@operor/knowledge 0.1.0

package/src/SQLiteKnowledgeStore.ts

@@ -0,0 +1,379 @@
+import { statSync } from 'node:fs';
+import Database from 'better-sqlite3';
+import * as sqliteVec from 'sqlite-vec';
+import type { KBDocument, KBChunk, KBSearchResult, KBSearchOptions, KnowledgeStore, KBStats } from './types.js';
+
+export class SQLiteKnowledgeStore implements KnowledgeStore {
+  private db: Database.Database;
+  private dbPath: string;
+  private dimensions: number;
+  private dimensionWarned = false;
+
+  constructor(dbPath: string = './knowledge.db', dimensions: number = 1536) {
+    this.db = new Database(dbPath);
+    this.dbPath = dbPath;
+    this.dimensions = dimensions;
+    this.db.pragma('journal_mode = WAL');
+    this.db.pragma('foreign_keys = ON');
+    sqliteVec.load(this.db);
+  }
+
+  getDimensions(): number {
+    return this.dimensions;
+  }
+
+  async initialize(): Promise<void> {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS kb_documents (
+        id TEXT PRIMARY KEY,
+        source_type TEXT NOT NULL,
+        source_url TEXT,
+        file_name TEXT,
+        title TEXT,
+        content TEXT NOT NULL,
+        metadata TEXT,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        priority INTEGER DEFAULT 2,
+        content_hash TEXT
+      );
+
+      CREATE TABLE IF NOT EXISTS kb_chunks (
+        id TEXT PRIMARY KEY,
+        document_id TEXT NOT NULL,
+        content TEXT NOT NULL,
+        chunk_index INTEGER NOT NULL,
+        metadata TEXT,
+        FOREIGN KEY (document_id) REFERENCES kb_documents(id) ON DELETE CASCADE
+      );
+      CREATE INDEX IF NOT EXISTS idx_chunks_document ON kb_chunks(document_id);
+      CREATE INDEX IF NOT EXISTS idx_documents_source_url ON kb_documents(source_url);
+    `);
+
+    // Migration: add columns for existing DBs
+    try { this.db.exec('ALTER TABLE kb_documents ADD COLUMN priority INTEGER DEFAULT 2'); } catch {}
+    try { this.db.exec('ALTER TABLE kb_documents ADD COLUMN content_hash TEXT'); } catch {}
+    try { this.db.exec('CREATE INDEX IF NOT EXISTS idx_documents_source_url ON kb_documents(source_url)'); } catch {}
+
+    this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
+        chunk_id TEXT PRIMARY KEY,
+        embedding float[${this.dimensions}]
+      );
+    `);
+
+    this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS fts_chunks USING fts5(
+        chunk_id UNINDEXED,
+        content,
+        tokenize='porter unicode61'
+      );
+    `);
+  }
+
+  async close(): Promise<void> {
+    this.db.close();
+  }
+
+  async addDocument(doc: KBDocument): Promise<void> {
+    this.db.prepare(`
+      INSERT OR REPLACE INTO kb_documents (id, source_type, source_url, file_name, title, content, metadata, created_at, updated_at, priority, content_hash)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      doc.id,
+      doc.sourceType,
+      doc.sourceUrl || null,
+      doc.fileName || null,
+      doc.title || null,
+      doc.content,
+      doc.metadata ? JSON.stringify(doc.metadata) : null,
+      doc.createdAt,
+      doc.updatedAt,
+      doc.priority ?? 2,
+      doc.contentHash || null,
+    );
+  }
+
+  async getDocument(id: string): Promise<KBDocument | null> {
+    const row = this.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(id) as any;
+    return row ? this.rowToDocument(row) : null;
+  }
+
+  async listDocuments(): Promise<KBDocument[]> {
+    const rows = this.db.prepare('SELECT * FROM kb_documents ORDER BY created_at DESC').all() as any[];
+    return rows.map((r) => this.rowToDocument(r));
+  }
+
+  async deleteDocument(id: string): Promise<void> {
+    const chunks = this.db.prepare('SELECT id FROM kb_chunks WHERE document_id = ?').all(id) as any[];
+    for (const chunk of chunks) {
+      this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?').run(chunk.id);
+      this.db.prepare('DELETE FROM fts_chunks WHERE chunk_id = ?').run(chunk.id);
+    }
+    this.db.prepare('DELETE FROM kb_chunks WHERE document_id = ?').run(id);
+    this.db.prepare('DELETE FROM kb_documents WHERE id = ?').run(id);
+  }
+
+  async addChunks(chunks: KBChunk[]): Promise<void> {
+    const insertChunk = this.db.prepare(`
+      INSERT OR REPLACE INTO kb_chunks (id, document_id, content, chunk_index, metadata)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+    const insertVec = this.db.prepare(`
+      INSERT OR REPLACE INTO vec_chunks (chunk_id, embedding)
+      VALUES (?, ?)
+    `);
+    const deleteFts = this.db.prepare(`
+      DELETE FROM fts_chunks WHERE chunk_id = ?
+    `);
+    const insertFts = this.db.prepare(`
+      INSERT INTO fts_chunks (chunk_id, content)
+      VALUES (?, ?)
+    `);
+
+    const transaction = this.db.transaction((items: KBChunk[]) => {
+      for (const chunk of items) {
+        insertChunk.run(
+          chunk.id,
+          chunk.documentId,
+          chunk.content,
+          chunk.chunkIndex,
+          chunk.metadata ? JSON.stringify(chunk.metadata) : null,
+        );
+        if (chunk.embedding) {
+          if (chunk.embedding.length !== this.dimensions && !this.dimensionWarned) {
+            this.dimensionWarned = true;
+            console.warn(
+              `[KB] Dimension mismatch: store expects ${this.dimensions}d vectors but received ${chunk.embedding.length}d. ` +
+              `This will cause search errors. Re-ingest your documents after switching embedding providers, ` +
+              `or set the correct dimensions when creating the store.`
+            );
+          }
+          insertVec.run(chunk.id, new Float32Array(chunk.embedding));
+        }
+        // FTS5: delete-then-insert for idempotent upsert (FTS5 has no OR REPLACE)
+        deleteFts.run(chunk.id);
+        insertFts.run(chunk.id, chunk.content);
+      }
+    });
+
+    transaction(chunks);
+  }
+
+  getChunkCount(documentId: string): number {
+    const row = this.db.prepare(
+      'SELECT COUNT(*) as count FROM kb_chunks WHERE document_id = ?'
+    ).get(documentId) as any;
+    return row?.count || 0;
+  }
+
+  async search(query: string, embedding: number[], options?: KBSearchOptions): Promise<KBSearchResult[]> {
+    return this.searchByEmbedding(embedding, options);
+  }
+
+  async searchByEmbedding(embedding: number[], options?: KBSearchOptions): Promise<KBSearchResult[]> {
+    const limit = options?.limit || 5;
+    // When filtering by sourceTypes post-query, over-fetch to avoid missing
+    // results that would be filtered out (e.g., FAQ fast-path requesting limit=1
+    // but the closest vector is a non-FAQ document chunk).
+    const fetchLimit = options?.sourceTypes ? Math.min(limit * 10, 100) : limit;
+
+    const vecRows = this.db.prepare(`
+      SELECT chunk_id, distance
+      FROM vec_chunks
+      WHERE embedding MATCH ?
+      ORDER BY distance
+      LIMIT ?
+    `).all(new Float32Array(embedding), fetchLimit) as any[];
+
+    if (vecRows.length === 0) return [];
+
+    const results: KBSearchResult[] = [];
+    for (const vecRow of vecRows) {
+      const distance = vecRow.distance as number;
+      const score = 1 / (1 + distance);
+
+      if (options?.scoreThreshold && score < options.scoreThreshold) continue;
+
+      const chunk = this.db.prepare('SELECT * FROM kb_chunks WHERE id = ?').get(vecRow.chunk_id) as any;
+      if (!chunk) continue;
+
+      const doc = this.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(chunk.document_id) as any;
+      if (!doc) continue;
+
+      if (options?.sourceTypes && !options.sourceTypes.includes(doc.source_type)) continue;
+
+      results.push({
+        chunk: {
+          id: chunk.id,
+          documentId: chunk.document_id,
+          content: chunk.content,
+          chunkIndex: chunk.chunk_index,
+          metadata: chunk.metadata ? JSON.parse(chunk.metadata) : undefined,
+        },
+        document: this.rowToDocument(doc),
+        score,
+        distance,
+      });
+    }
+
+    return results.slice(0, limit);
+  }
+
+  async searchByKeyword(query: string, options?: KBSearchOptions): Promise<KBSearchResult[]> {
+    const limit = options?.limit || 5;
+
+    // FTS5 MATCH query — escape double quotes in user input
+    const safeQuery = query.replace(/"/g, '""');
+    let ftsRows: any[];
+    try {
+      ftsRows = this.db.prepare(`
+        SELECT chunk_id, rank
+        FROM fts_chunks
+        WHERE fts_chunks MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `).all(safeQuery, limit * 2) as any[]; // fetch extra to allow post-filtering
+    } catch {
+      // FTS5 can throw on malformed queries (e.g. special chars)
+      return [];
+    }
+
+    if (ftsRows.length === 0) return [];
+
+    const results: KBSearchResult[] = [];
+    for (const ftsRow of ftsRows) {
+      if (results.length >= limit) break;
+
+      const chunk = this.db.prepare('SELECT * FROM kb_chunks WHERE id = ?').get(ftsRow.chunk_id) as any;
+      if (!chunk) continue;
+
+      const doc = this.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(chunk.document_id) as any;
+      if (!doc) continue;
+
+      if (options?.sourceTypes && !options.sourceTypes.includes(doc.source_type)) continue;
+
+      // BM25 rank is negative (lower = better), convert to a positive score
+      const bm25Score = -ftsRow.rank;
+
+      if (options?.scoreThreshold && bm25Score < options.scoreThreshold) continue;
+
+      results.push({
+        chunk: {
+          id: chunk.id,
+          documentId: chunk.document_id,
+          content: chunk.content,
+          chunkIndex: chunk.chunk_index,
+          metadata: chunk.metadata ? JSON.parse(chunk.metadata) : undefined,
+        },
+        document: this.rowToDocument(doc),
+        score: bm25Score,
+        distance: 0,
+      });
+    }
+
+    return results;
+  }
+
+  /**
+   * Get all chunks from kb_chunks (text content only, no embeddings).
+   * Used by rebuild to re-embed all content.
+   */
+  getAllChunks(): { id: string; documentId: string; content: string; chunkIndex: number; metadata?: string }[] {
+    return this.db.prepare(
+      'SELECT id, document_id AS documentId, content, chunk_index AS chunkIndex, metadata FROM kb_chunks ORDER BY document_id, chunk_index'
+    ).all() as any[];
+  }
+
+  /**
+   * Drop and recreate the vec_chunks virtual table with new dimensions.
+   * Preserves kb_chunks, kb_documents, and fts_chunks — only vector data is affected.
+   */
+  rebuildVecTable(newDimensions: number): void {
+    this.db.exec('DROP TABLE IF EXISTS vec_chunks');
+    this.db.exec(`
+      CREATE VIRTUAL TABLE vec_chunks USING vec0(
+        chunk_id TEXT PRIMARY KEY,
+        embedding float[${newDimensions}]
+      );
+    `);
+    this.dimensions = newDimensions;
+    this.dimensionWarned = false;
+  }
+
+  /**
+   * Batch-insert embeddings into vec_chunks.
+   * Expects an array of { chunkId, embedding } pairs.
+   */
+  batchInsertEmbeddings(items: { chunkId: string; embedding: number[] }[]): void {
+    const insert = this.db.prepare(
+      'INSERT OR REPLACE INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)'
+    );
+    const tx = this.db.transaction((batch: { chunkId: string; embedding: number[] }[]) => {
+      for (const item of batch) {
+        insert.run(item.chunkId, new Float32Array(item.embedding));
+      }
+    });
+    tx(items);
+  }
+
+  async getStats(): Promise<KBStats> {
+    const docCount = this.db.prepare('SELECT COUNT(*) as count FROM kb_documents').get() as any;
+    const chunkCount = this.db.prepare('SELECT COUNT(*) as count FROM kb_chunks').get() as any;
+    let dbSizeBytes = 0;
+    try {
+      dbSizeBytes = statSync(this.dbPath).size;
+    } catch {}
+    return {
+      documentCount: docCount.count,
+      chunkCount: chunkCount.count,
+      embeddingDimensions: this.dimensions,
+      dbSizeBytes,
+    };
+  }
+
+  async findBySourceUrl(url: string): Promise<KBDocument | null> {
+    const row = this.db.prepare('SELECT * FROM kb_documents WHERE source_url = ?').get(url) as any;
+    return row ? this.rowToDocument(row) : null;
+  }
+
+  async findByContentHash(hash: string): Promise<KBDocument | null> {
+    const row = this.db.prepare('SELECT * FROM kb_documents WHERE content_hash = ?').get(hash) as any;
+    return row ? this.rowToDocument(row) : null;
+  }
+
+  async updateDocument(id: string, updates: { content?: string; title?: string; contentHash?: string; priority?: number; metadata?: Record<string, any> }): Promise<void> {
+    const sets: string[] = [];
+    const values: any[] = [];
+    if (updates.content !== undefined) { sets.push('content = ?'); values.push(updates.content); }
+    if (updates.title !== undefined) { sets.push('title = ?'); values.push(updates.title); }
+    if (updates.contentHash !== undefined) { sets.push('content_hash = ?'); values.push(updates.contentHash); }
+    if (updates.priority !== undefined) { sets.push('priority = ?'); values.push(updates.priority); }
+    if (updates.metadata !== undefined) { sets.push('metadata = ?'); values.push(JSON.stringify(updates.metadata)); }
+    sets.push('updated_at = ?'); values.push(Date.now());
+    values.push(id);
+    this.db.prepare(`UPDATE kb_documents SET ${sets.join(', ')} WHERE id = ?`).run(...values);
+  }
+
+  async findSimilarFaq(embedding: number[], threshold: number): Promise<KBSearchResult | null> {
+    const results = await this.searchByEmbedding(embedding, { sourceTypes: ['faq'], limit: 1 });
+    if (results.length > 0 && results[0].score >= threshold) return results[0];
+    return null;
+  }
+
+  private rowToDocument(row: any): KBDocument {
+    return {
+      id: row.id,
+      sourceType: row.source_type,
+      sourceUrl: row.source_url || undefined,
+      fileName: row.file_name || undefined,
+      title: row.title || undefined,
+      content: row.content,
+      metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
+      createdAt: row.created_at,
+      updatedAt: row.updated_at,
+      priority: row.priority ?? 2,
+      contentHash: row.content_hash || undefined,
+    };
+  }
+}

package/src/TextChunker.ts

@@ -0,0 +1,34 @@
+import { RecursiveCharacterTextSplitter, MarkdownTextSplitter } from '@langchain/textsplitters';
+
+export interface ChunkOptions {
+  chunkSize?: number;
+  chunkOverlap?: number;
+}
+
+export class TextChunker {
+  private defaultChunkSize: number;
+  private defaultChunkOverlap: number;
+
+  constructor(options?: ChunkOptions) {
+    this.defaultChunkSize = options?.chunkSize || 3200;
+    this.defaultChunkOverlap = options?.chunkOverlap || 200;
+  }
+
+  async chunk(text: string, options?: ChunkOptions): Promise<string[]> {
+    const splitter = new RecursiveCharacterTextSplitter({
+      chunkSize: options?.chunkSize || this.defaultChunkSize,
+      chunkOverlap: options?.chunkOverlap || this.defaultChunkOverlap,
+    });
+    const docs = await splitter.createDocuments([text]);
+    return docs.map((d) => d.pageContent);
+  }
+
+  async chunkMarkdown(markdown: string, options?: ChunkOptions): Promise<string[]> {
+    const splitter = new MarkdownTextSplitter({
+      chunkSize: options?.chunkSize || this.defaultChunkSize,
+      chunkOverlap: options?.chunkOverlap || this.defaultChunkOverlap,
+    });
+    const docs = await splitter.createDocuments([markdown]);
+    return docs.map((d) => d.pageContent);
+  }
+}

package/src/__tests__/cli-integration.test.ts

@@ -0,0 +1,134 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { unlinkSync } from 'node:fs';
+import { EmbeddingService } from '../EmbeddingService.js';
+import { SQLiteKnowledgeStore } from '../SQLiteKnowledgeStore.js';
+import { TextChunker } from '../TextChunker.js';
+import { IngestionPipeline } from '../IngestionPipeline.js';
+import { UrlIngestor } from '../ingestors/UrlIngestor.js';
+import { FileIngestor } from '../ingestors/FileIngestor.js';
+
+// Mock AI SDK
+vi.mock('ai', () => ({
+  embed: vi.fn(async ({ value }: { value: string }) => ({
+    embedding: mockEmbed(value),
+  })),
+  embedMany: vi.fn(async ({ values }: { values: string[] }) => ({
+    embeddings: values.map(mockEmbed),
+  })),
+}));
+
+vi.mock('@ai-sdk/openai', () => ({
+  createOpenAI: vi.fn(() => ({
+    embedding: vi.fn(() => ({})),
+  })),
+}));
+
+vi.mock('@ai-sdk/google', () => ({
+  createGoogleGenerativeAI: vi.fn(() => ({
+    textEmbeddingModel: vi.fn(() => ({})),
+  })),
+}));
+
+vi.mock('@ai-sdk/mistral', () => ({
+  mistral: {
+    embedding: vi.fn(() => ({})),
+  },
+}));
+
+vi.mock('@ai-sdk/cohere', () => ({
+  cohere: {
+    embedding: vi.fn(() => ({})),
+  },
+}));
+
+function mockEmbed(text: string): number[] {
+  const hash = text.split('').reduce((acc, c) => acc + c.charCodeAt(0), 0);
+  return Array.from({ length: 1536 }, (_, i) => Math.sin(hash + i) * 0.1);
+}
+
+describe('CLI Integration — Bug Fixes', () => {
+  let store: SQLiteKnowledgeStore;
+  let embedder: EmbeddingService;
+  let chunker: TextChunker;
+  let pipeline: IngestionPipeline;
+  const dbPath = './test-cli-integration.db';
+
+  beforeEach(async () => {
+    store = new SQLiteKnowledgeStore(dbPath);
+    // BUG 1 FIX: Must call initialize() to create tables
+    await store.initialize();
+    embedder = new EmbeddingService({ provider: 'openai', apiKey: 'test' });
+    chunker = new TextChunker({ chunkSize: 100, chunkOverlap: 20 });
+    pipeline = new IngestionPipeline(store, embedder, chunker);
+  });
+
+  afterEach(async () => {
+    await store.close();
+    try {
+      unlinkSync(dbPath);
+      unlinkSync(`${dbPath}-shm`);
+      unlinkSync(`${dbPath}-wal`);
+    } catch {}
+  });
+
+  it('should list documents after initialization (Bug 1)', async () => {
+    // This would fail with "no such table: kb_documents" without initialize()
+    const docs = await store.listDocuments();
+    expect(docs).toEqual([]);
+  });
+
+  it('should ingest URL via UrlIngestor (Bug 2)', async () => {
+    // Mock fetch for URL ingestion
+    global.fetch = vi.fn(async () => ({
+      ok: true,
+      text: async () => '<html><head><title>Test</title></head><body>Test content</body></html>',
+    })) as any;
+
+    const urlIngestor = new UrlIngestor(pipeline);
+    const doc = await urlIngestor.ingestUrl('https://example.com');
+
+    expect(doc.id).toBeDefined();
+    expect(doc.sourceType).toBe('url');
+    expect(doc.sourceUrl).toBe('https://example.com');
+  });
+
+  it('should ingest FAQ via IngestionPipeline (Bug 2)', async () => {
+    const doc = await pipeline.ingestFaq('What is Operor?', 'Operor is a framework.');
+
+    expect(doc.id).toBeDefined();
+    expect(doc.sourceType).toBe('faq');
+    expect(doc.title).toBe('What is Operor?');
+  });
+
+  it('should search using embedding array (Bug 2)', async () => {
+    await pipeline.ingestFaq('What is the return policy?', 'You can return within 30 days.');
+
+    const queryEmbedding = await embedder.embed('return policy');
+    const results = await store.search('return policy', queryEmbedding, { limit: 5 });
+
+    expect(results.length).toBeGreaterThan(0);
+  });
+
+  it('should return stats via getStats()', async () => {
+    // Empty DB
+    const emptyStats = await store.getStats();
+    expect(emptyStats.documentCount).toBe(0);
+    expect(emptyStats.chunkCount).toBe(0);
+    expect(emptyStats.embeddingDimensions).toBe(1536);
+    expect(emptyStats.dbSizeBytes).toBeGreaterThan(0);
+
+    // After ingestion
+    await pipeline.ingestFaq('Q1?', 'A1');
+    await pipeline.ingestFaq('Q2?', 'A2');
+    const stats = await store.getStats();
+    expect(stats.documentCount).toBe(2);
+    expect(stats.chunkCount).toBe(2);
+  });
+
+  it('should delete document by string ID', async () => {
+    const doc = await pipeline.ingestFaq('Delete me?', 'OK');
+    await store.deleteDocument(doc.id);
+    const retrieved = await store.getDocument(doc.id);
+    expect(retrieved).toBeNull();
+  });
+});