@namzu/sdk 0.1.1

package/src/rag/chunking.ts

@@ -0,0 +1,145 @@
+import type { Chunker, ChunkContent, ChunkingConfig, ChunkingStrategy } from '../types/rag/index.js'
+
+const DEFAULT_SEPARATORS: Record<ChunkingStrategy, string[]> = {
+  fixed: [],
+  sentence: ['. ', '! ', '? ', '.\n', '!\n', '?\n'],
+  paragraph: ['\n\n', '\n'],
+  recursive: ['\n\n', '\n', '. ', ' ', ''],
+}
+
+export class TextChunker implements Chunker {
+  chunk(content: string, config: ChunkingConfig): ChunkContent[] {
+    switch (config.strategy) {
+      case 'fixed':
+        return this.fixedChunk(content, config)
+      case 'sentence':
+        return this.sentenceChunk(content, config)
+      case 'paragraph':
+        return this.paragraphChunk(content, config)
+      case 'recursive':
+        return this.recursiveChunk(content, config)
+      default: {
+        const _exhaustive: never = config.strategy
+        throw new Error(`Unhandled chunking strategy: ${_exhaustive}`)
+      }
+    }
+  }
+
+  private fixedChunk(content: string, config: ChunkingConfig): ChunkContent[] {
+    const { chunkSize, chunkOverlap } = config
+    const chunks: ChunkContent[] = []
+    const step = Math.max(1, chunkSize - chunkOverlap)
+    let idx = 0
+
+    for (let start = 0; start < content.length; start += step) {
+      const slice = content.slice(start, start + chunkSize).trim()
+      if (slice.length > 0) {
+        chunks.push({ content: slice, index: idx++ })
+      }
+    }
+    return chunks
+  }
+
+  private sentenceChunk(content: string, config: ChunkingConfig): ChunkContent[] {
+    const separators = config.separators ?? DEFAULT_SEPARATORS.sentence
+    const sentences = this.splitBySeparators(content, separators)
+    return this.mergeSmallParts(sentences, config)
+  }
+
+  private paragraphChunk(content: string, config: ChunkingConfig): ChunkContent[] {
+    const separators = config.separators ?? DEFAULT_SEPARATORS.paragraph
+    const paragraphs = this.splitBySeparators(content, separators)
+    return this.mergeSmallParts(paragraphs, config)
+  }
+
+  private recursiveChunk(content: string, config: ChunkingConfig): ChunkContent[] {
+    const separators = config.separators ?? DEFAULT_SEPARATORS.recursive
+    return this.recursiveSplit(content, separators, config, 0)
+  }
+
+  private recursiveSplit(
+    text: string,
+    separators: string[],
+    config: ChunkingConfig,
+    startIndex: number,
+  ): ChunkContent[] {
+    if (text.length <= config.chunkSize) {
+      const trimmed = text.trim()
+      return trimmed.length > 0 ? [{ content: trimmed, index: startIndex }] : []
+    }
+
+    for (const sep of separators) {
+      const parts = sep === '' ? [...text] : text.split(sep).filter(p => p.trim().length > 0)
+      if (parts.length <= 1) continue
+
+      const remaining = separators.slice(separators.indexOf(sep) + 1)
+      const merged = this.mergeSmallParts(parts, config)
+
+      if (merged.every(c => c.content.length <= config.chunkSize)) {
+        return merged
+      }
+
+      const results: ChunkContent[] = []
+      let idx = 0
+      for (const chunk of merged) {
+        if (chunk.content.length > config.chunkSize && remaining.length > 0) {
+          const subChunks = this.recursiveSplit(chunk.content, remaining, config, idx)
+          results.push(...subChunks)
+          idx += subChunks.length
+        } else {
+          results.push({ content: chunk.content, index: idx++ })
+        }
+      }
+      return results
+    }
+
+    return this.fixedChunk(text, config)
+  }
+
+  private splitBySeparators(text: string, separators: string[]): string[] {
+    let parts = [text]
+    for (const sep of separators) {
+      const nextParts: string[] = []
+      for (const part of parts) {
+        const splits = part.split(sep)
+        for (let i = 0; i < splits.length; i++) {
+          const piece = i < splits.length - 1 ? splits[i]! + sep : splits[i]!
+          if (piece.trim().length > 0) {
+            nextParts.push(piece)
+          }
+        }
+      }
+      parts = nextParts
+    }
+    return parts
+  }
+
+  private mergeSmallParts(parts: string[], config: ChunkingConfig): ChunkContent[] {
+    const { chunkSize, chunkOverlap } = config
+    const chunks: ChunkContent[] = []
+    let current = ''
+    let idx = 0
+
+    for (const part of parts) {
+      if (current.length + part.length > chunkSize && current.length > 0) {
+        chunks.push({ content: current.trim(), index: idx++ })
+        const overlapStart = Math.max(0, current.length - chunkOverlap)
+        current = current.slice(overlapStart) + part
+      } else {
+        current += part
+      }
+    }
+
+    if (current.trim().length > 0) {
+      chunks.push({ content: current.trim(), index: idx })
+    }
+
+    return chunks
+  }
+}
+
+export const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {
+  strategy: 'recursive',
+  chunkSize: 512,
+  chunkOverlap: 64,
+}

package/src/rag/context-assembler.ts

@@ -0,0 +1,65 @@
+import type { RAGContext, RAGContextConfig, VectorSearchResult } from '../types/rag/index.js'
+
+export const DEFAULT_RAG_CONTEXT_CONFIG: RAGContextConfig = {
+  maxTokens: 4096,
+  separator: '\n\n---\n\n',
+  includeMetadata: true,
+  headerTemplate: '## Retrieved Knowledge\n\nThe following information was retrieved from the knowledge base:\n',
+}
+
+export function assembleRAGContext(
+  results: VectorSearchResult[],
+  config: Partial<RAGContextConfig> = {},
+): RAGContext {
+  const effectiveConfig = { ...DEFAULT_RAG_CONTEXT_CONFIG, ...config }
+
+  if (results.length === 0) {
+    return { content: '', sources: [], tokenCount: 0 }
+  }
+
+  const sources: RAGContext['sources'] = []
+  const parts: string[] = []
+
+  if (effectiveConfig.headerTemplate) {
+    parts.push(effectiveConfig.headerTemplate)
+  }
+
+  let currentTokens = estimateTokens(parts.join(''))
+
+  for (const result of results) {
+    const chunkTokens = estimateTokens(result.chunk.content)
+    if (currentTokens + chunkTokens > effectiveConfig.maxTokens) break
+
+    let entry = result.chunk.content
+
+    if (effectiveConfig.includeMetadata) {
+      const meta = formatMetadata(result)
+      entry = `${meta}\n${entry}`
+    }
+
+    parts.push(entry)
+    currentTokens += chunkTokens
+
+    sources.push({
+      documentId: result.chunk.documentId,
+      chunk: result.chunk.content.slice(0, 200),
+      score: result.score,
+    })
+  }
+
+  const content = parts.join(effectiveConfig.separator)
+  return { content, sources, tokenCount: estimateTokens(content) }
+}
+
+function formatMetadata(result: VectorSearchResult): string {
+  const meta = result.chunk.metadata
+  const parts: string[] = []
+  if (meta.source) parts.push(`Source: ${meta.source}`)
+  if (meta.title) parts.push(`Title: ${meta.title}`)
+  parts.push(`Relevance: ${(result.score * 100).toFixed(1)}%`)
+  return `[${parts.join(' | ')}]`
+}
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4)
+}

package/src/rag/embedding.ts

@@ -0,0 +1,72 @@
+import type { EmbeddingProvider, OpenRouterEmbeddingConfig } from '../types/rag/index.js'
+
+export class OpenRouterEmbeddingProvider implements EmbeddingProvider {
+  readonly id = 'openrouter-embedding'
+  readonly model: string
+  readonly dimensions: number
+
+  private readonly apiKey: string
+  private readonly baseUrl: string
+  private readonly batchSize: number
+
+  constructor(config: OpenRouterEmbeddingConfig) {
+    this.model = config.model
+    this.dimensions = config.dimensions ?? 1536
+    this.apiKey = config.apiKey
+    this.baseUrl = config.baseUrl ?? 'https://openrouter.ai/api/v1'
+    this.batchSize = config.batchSize ?? 64
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    const results: number[][] = []
+
+    for (let i = 0; i < texts.length; i += this.batchSize) {
+      const batch = texts.slice(i, i + this.batchSize)
+      const batchResults = await this.callEmbeddingApi(batch)
+      results.push(...batchResults)
+    }
+
+    return results
+  }
+
+  async embedQuery(query: string): Promise<number[]> {
+    const [result] = await this.embed([query])
+    return result!
+  }
+
+  private async callEmbeddingApi(texts: string[]): Promise<number[][]> {
+    const response = await fetch(`${this.baseUrl}/embeddings`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: this.model,
+        input: texts,
+        dimensions: this.dimensions,
+      }),
+    })
+
+    if (!response.ok) {
+      const errorText = await response.text()
+      throw new Error(`Embedding API error (${response.status}): ${errorText}`)
+    }
+
+    const data = (await response.json()) as EmbeddingApiResponse
+    return data.data
+      .sort((a, b) => a.index - b.index)
+      .map(d => d.embedding)
+  }
+}
+
+interface EmbeddingApiResponse {
+  data: Array<{
+    index: number
+    embedding: number[]
+  }>
+  usage: {
+    prompt_tokens: number
+    total_tokens: number
+  }
+}

package/src/rag/index.ts

@@ -0,0 +1,43 @@
+export type {
+  TenantScope,
+  DocumentMetadata,
+  Document,
+  Chunk,
+  ChunkContent,
+  ChunkingStrategy,
+  ChunkingConfig,
+  Chunker,
+  EmbeddingProvider,
+  EmbeddingConfig,
+  VectorSearchResult,
+  VectorStoreQuery,
+  VectorStore,
+  RetrievalMode,
+  RetrievalConfig,
+  RetrievalQuery,
+  RetrievalResult,
+  Retriever,
+  IngestionResult,
+  IngestionPipeline,
+  KnowledgeBaseConfig,
+  KnowledgeBase,
+  RAGToolInput,
+  RAGContextConfig,
+  RAGContext,
+} from '../types/rag/index.js'
+
+export { TextChunker, DEFAULT_CHUNKING_CONFIG } from './chunking.js'
+
+export { OpenRouterEmbeddingProvider } from './embedding.js'
+
+export { InMemoryVectorStore, cosineSimilarity } from './vector-store.js'
+
+export { DefaultRetriever, DEFAULT_RETRIEVAL_CONFIG } from './retriever.js'
+
+export { DefaultIngestionPipeline } from './ingestion.js'
+
+export { assembleRAGContext, DEFAULT_RAG_CONTEXT_CONFIG } from './context-assembler.js'
+
+export { DefaultKnowledgeBase } from './knowledge-base.js'
+
+export { createRAGTool } from './rag-tool.js'

package/src/rag/ingestion.ts

@@ -0,0 +1,77 @@
+import type { DocumentId, KnowledgeBaseId } from '../types/ids/index.js'
+import type {
+  Chunk,
+  ChunkingConfig,
+  DocumentMetadata,
+  EmbeddingProvider,
+  IngestionPipeline,
+  IngestionResult,
+  TenantScope,
+  VectorStore,
+} from '../types/rag/index.js'
+import { generateDocumentId, generateChunkId } from '../utils/id.js'
+import { TextChunker, DEFAULT_CHUNKING_CONFIG } from './chunking.js'
+
+export class DefaultIngestionPipeline implements IngestionPipeline {
+  private readonly chunker: TextChunker
+  private readonly chunkingConfig: ChunkingConfig
+
+  constructor(
+    private readonly vectorStore: VectorStore,
+    private readonly embeddingProvider: EmbeddingProvider,
+    chunkingConfig?: Partial<ChunkingConfig>,
+  ) {
+    this.chunker = new TextChunker()
+    this.chunkingConfig = { ...DEFAULT_CHUNKING_CONFIG, ...chunkingConfig }
+  }
+
+  async ingest(
+    content: string,
+    metadata: DocumentMetadata,
+    scope: TenantScope,
+    knowledgeBaseId: KnowledgeBaseId,
+  ): Promise<IngestionResult> {
+    const startTime = Date.now()
+    const documentId = generateDocumentId()
+
+    const chunkContents = this.chunker.chunk(content, this.chunkingConfig)
+    if (chunkContents.length === 0) {
+      return { documentId, chunkCount: 0, totalTokens: 0, durationMs: Date.now() - startTime }
+    }
+
+    const texts = chunkContents.map(c => c.content)
+    const embeddings = await this.embeddingProvider.embed(texts)
+
+    const now = Date.now()
+    const chunks: Chunk[] = chunkContents.map((cc, i) => ({
+      id: generateChunkId(),
+      documentId,
+      knowledgeBaseId,
+      tenantId: scope.tenantId,
+      content: cc.content,
+      index: cc.index,
+      tokenCount: estimateTokens(cc.content),
+      embedding: embeddings[i],
+      metadata: { ...metadata, chunkIndex: cc.index },
+      createdAt: now,
+    }))
+
+    await this.vectorStore.upsert(chunks)
+
+    const totalTokens = chunks.reduce((sum, c) => sum + c.tokenCount, 0)
+    return {
+      documentId,
+      chunkCount: chunks.length,
+      totalTokens,
+      durationMs: Date.now() - startTime,
+    }
+  }
+
+  async remove(documentId: DocumentId): Promise<void> {
+    await this.vectorStore.deleteByDocument(documentId)
+  }
+}
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4)
+}

package/src/rag/knowledge-base.ts

@@ -0,0 +1,72 @@
+import type { DocumentId, KnowledgeBaseId } from '../types/ids/index.js'
+import type {
+  ChunkingConfig,
+  DocumentMetadata,
+  EmbeddingProvider,
+  IngestionResult,
+  KnowledgeBase,
+  KnowledgeBaseConfig,
+  RetrievalConfig,
+  RetrievalQuery,
+  RetrievalResult,
+  TenantScope,
+  VectorStore,
+} from '../types/rag/index.js'
+import { generateKnowledgeBaseId } from '../utils/id.js'
+import { DefaultIngestionPipeline } from './ingestion.js'
+import { DefaultRetriever, DEFAULT_RETRIEVAL_CONFIG } from './retriever.js'
+import { DEFAULT_CHUNKING_CONFIG } from './chunking.js'
+
+export class DefaultKnowledgeBase implements KnowledgeBase {
+  readonly id: KnowledgeBaseId
+  readonly config: KnowledgeBaseConfig
+
+  private readonly ingestion: DefaultIngestionPipeline
+  private readonly retriever: DefaultRetriever
+  private readonly scope: TenantScope
+  private readonly vectorStore: VectorStore
+
+  constructor(
+    config: KnowledgeBaseConfig,
+    vectorStore: VectorStore,
+    embeddingProvider: EmbeddingProvider,
+  ) {
+    this.vectorStore = vectorStore
+    this.id = config.id ?? generateKnowledgeBaseId()
+    this.config = { ...config, id: this.id }
+
+    this.scope = {
+      tenantId: config.tenantId,
+      namespace: config.namespace,
+    }
+
+    const chunkingConfig: ChunkingConfig = {
+      ...DEFAULT_CHUNKING_CONFIG,
+      ...config.chunking,
+    }
+
+    const retrievalConfig: RetrievalConfig = {
+      ...DEFAULT_RETRIEVAL_CONFIG,
+      ...config.retrieval,
+    }
+
+    this.ingestion = new DefaultIngestionPipeline(vectorStore, embeddingProvider, chunkingConfig)
+    this.retriever = new DefaultRetriever(vectorStore, embeddingProvider, retrievalConfig)
+  }
+
+  async ingest(content: string, metadata: DocumentMetadata = {}): Promise<IngestionResult> {
+    return this.ingestion.ingest(content, metadata, this.scope, this.id)
+  }
+
+  async remove(documentId: DocumentId): Promise<void> {
+    return this.ingestion.remove(documentId)
+  }
+
+  async query(query: RetrievalQuery): Promise<RetrievalResult> {
+    return this.retriever.retrieve(query, this.scope, this.id)
+  }
+
+  async clear(): Promise<void> {
+    await this.vectorStore.deleteByKnowledgeBase(this.id, this.scope.tenantId)
+  }
+}

package/src/rag/rag-tool.ts

@@ -0,0 +1,62 @@
+import { z } from 'zod'
+import { defineTool } from '../tools/defineTool.js'
+import type { KnowledgeBaseId } from '../types/ids/index.js'
+import type { RAGToolConfig } from '../types/rag/index.js'
+import { assembleRAGContext } from './context-assembler.js'
+
+const ragInputSchema = z.object({
+  query: z.string().min(1).describe('The search query to find relevant knowledge'),
+  knowledge_base_id: z.string().optional().describe('ID of a specific knowledge base to search'),
+  top_k: z.number().int().min(1).max(20).optional().describe('Number of results to retrieve'),
+})
+
+export function createRAGTool(config: RAGToolConfig) {
+  return defineTool({
+    name: 'knowledge_search',
+    description:
+      'Search the knowledge base for relevant information. Use this tool when you need to find specific facts, documentation, or context that may be stored in the knowledge base.',
+    inputSchema: ragInputSchema,
+    category: 'analysis',
+    permissions: ['network_access'],
+    readOnly: true,
+    destructive: false,
+    concurrencySafe: true,
+
+    async execute(input) {
+      const kbId = (input.knowledge_base_id as KnowledgeBaseId | undefined) ?? config.defaultKnowledgeBaseId
+      const kb = kbId ? config.knowledgeBases.get(kbId) : config.knowledgeBases.values().next().value
+
+      if (!kb) {
+        return {
+          success: false,
+          output: '',
+          error: `Knowledge base not found: ${kbId ?? 'none configured'}`,
+        }
+      }
+
+      const result = await kb.query({
+        text: input.query,
+        config: { topK: input.top_k ?? config.topK ?? 5 },
+      })
+
+      if (result.chunks.length === 0) {
+        return {
+          success: true,
+          output: 'No relevant information found in the knowledge base for this query.',
+        }
+      }
+
+      const context = assembleRAGContext(result.chunks, config.contextConfig)
+
+      return {
+        success: true,
+        output: context.content,
+        data: {
+          sources: context.sources,
+          mode: result.mode,
+          durationMs: result.durationMs,
+        },
+      }
+    },
+  })
+}