docorbit 0.1.0 → 0.1.2

package/packages/storage/src/repositories/chunk-repository.ts

@@ -1,316 +0,0 @@
-import type {
-  DocumentChunk,
-  ChunkRelationship,
-  ChunkCode,
-  SymbolReference,
-  ChunkType,
-  Provenance,
-} from '../../../shared/src/index.ts';
-import type { DocOrbitDb } from '../db.ts';
-import type { IChunkRepository } from '../interfaces.ts';
-
-export class ChunkRepository implements IChunkRepository {
-  private db: DocOrbitDb;
-
-  constructor(db: DocOrbitDb) {
-    this.db = db;
-  }
-
-  /**
-   * Saves chunks, code snippets, symbols, and relationships in a single atomic transaction.
-   * Transactionally synchronizes the FTS5 chunks_fts table with chunks.
-   */
-  saveChunks(
-    chunks: DocumentChunk[],
-    relationships: ChunkRelationship[] = [],
-    codeSnippets: ChunkCode[] = [],
-    symbols: SymbolReference[] = []
-  ): void {
-    if (chunks.length === 0) return;
-
-    const raw = this.db.getRawDb();
-    const createdAt = new Date().toISOString();
-
-    // Group symbols by chunkId for fast indexing into chunks_fts
-    const symbolsByChunk = new Map<string, string[]>();
-    for (const s of symbols) {
-      const list = symbolsByChunk.get(s.chunkId) || [];
-      list.push(s.name);
-      symbolsByChunk.set(s.chunkId, list);
-    }
-
-    raw.exec('BEGIN IMMEDIATE TRANSACTION;');
-    try {
-      const chunkStmt = raw.prepare(`
-        INSERT INTO chunks (
-          id, page_id, snapshot_id, title, section_path_json, content,
-          chunk_type, language, token_estimate, ordinal, content_hash,
-          doc_version, provenance_json, created_at
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        ON CONFLICT(id) DO UPDATE SET
-          title = excluded.title,
-          section_path_json = excluded.section_path_json,
-          content = excluded.content,
-          chunk_type = excluded.chunk_type,
-          language = excluded.language,
-          token_estimate = excluded.token_estimate,
-          ordinal = excluded.ordinal,
-          content_hash = excluded.content_hash,
-          doc_version = excluded.doc_version,
-          provenance_json = excluded.provenance_json
-      `);
-
-      const ftsDelStmt = this.db.isFtsAvailable()
-        ? raw.prepare('DELETE FROM chunks_fts WHERE chunk_id = ?')
-        : null;
-
-      const ftsInsStmt = this.db.isFtsAvailable()
-        ? raw.prepare(`
-            INSERT INTO chunks_fts (chunk_id, title, section_path, content, symbols)
-            VALUES (?, ?, ?, ?, ?)
-          `)
-        : null;
-
-      for (const chunk of chunks) {
-        const sectionPath = Array.isArray(chunk.sectionPath)
-          ? chunk.sectionPath
-          : (Array.isArray((chunk as any).headingPath) ? (chunk as any).headingPath : []);
-        const chunkType = chunk.chunkType || (chunk as any).type || 'prose';
-        const tokenEstimate = chunk.tokenEstimate ?? (chunk as any).tokenCount ?? 0;
-        const ordinal = chunk.ordinal ?? (chunk as any).orderIndex ?? 0;
-        const provenance = chunk.provenance ? JSON.stringify(chunk.provenance) : null;
-
-        chunkStmt.run(
-          chunk.id,
-          chunk.pageId || null,
-          chunk.snapshotId || 'default',
-          chunk.title || null,
-          JSON.stringify(sectionPath),
-          chunk.content,
-          chunkType,
-          chunk.language || null,
-          tokenEstimate,
-          ordinal,
-          chunk.contentHash,
-          chunk.docVersion || null,
-          provenance,
-          createdAt
-        );
-
-        if (ftsDelStmt && ftsInsStmt) {
-          ftsDelStmt.run(chunk.id);
-          const chunkSymbols = symbolsByChunk.get(chunk.id)?.join(' ') || '';
-          const sectionPathStr = sectionPath.join(' > ');
-          ftsInsStmt.run(chunk.id, chunk.title || '', sectionPathStr, chunk.content, chunkSymbols);
-        }
-      }
-
-      // Save code snippets
-      if (codeSnippets.length > 0) {
-        const codeStmt = raw.prepare(`
-          INSERT OR REPLACE INTO chunk_code (id, chunk_id, language, code)
-          VALUES (?, ?, ?, ?)
-        `);
-        for (const cs of codeSnippets) {
-          codeStmt.run(cs.id, cs.chunkId, cs.language || null, cs.code);
-        }
-      }
-
-      // Save symbols
-      if (symbols.length > 0) {
-        const symStmt = raw.prepare(`
-          INSERT OR REPLACE INTO symbol_references (id, chunk_id, name, kind)
-          VALUES (?, ?, ?, ?)
-        `);
-        for (const s of symbols) {
-          symStmt.run(s.id, s.chunkId, s.name, s.kind);
-        }
-      }
-
-      // Save relationships
-      if (relationships.length > 0) {
-        const relStmt = raw.prepare(`
-          INSERT OR REPLACE INTO chunk_relationships (source_chunk_id, target_chunk_id, type)
-          VALUES (?, ?, ?)
-        `);
-        for (const r of relationships) {
-          relStmt.run(r.sourceChunkId, r.targetChunkId, r.type);
-        }
-      }
-
-      raw.exec('COMMIT;');
-    } catch (err) {
-      raw.exec('ROLLBACK;');
-      throw err;
-    }
-  }
-
-  getChunk(id: string): DocumentChunk | null {
-    const raw = this.db.getRawDb();
-    const row = raw.prepare('SELECT * FROM chunks WHERE id = ?').get(id) as Record<string, unknown> | undefined;
-    if (!row) return null;
-    return this.hydrateChunk(row);
-  }
-
-  getChunksByPage(pageId: string): DocumentChunk[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT * FROM chunks WHERE page_id = ? ORDER BY ordinal ASC').all(pageId) as Array<Record<string, unknown>>;
-    return rows.map(r => this.hydrateChunk(r));
-  }
-
-  getChunksBySnapshot(snapshotId: string): DocumentChunk[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT * FROM chunks WHERE snapshot_id = ? ORDER BY page_id ASC, ordinal ASC').all(snapshotId) as Array<Record<string, unknown>>;
-    return rows.map(r => this.hydrateChunk(r));
-  }
-
-  getChunksByVersion(docVersion: string): DocumentChunk[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT * FROM chunks WHERE doc_version = ? ORDER BY page_id ASC, ordinal ASC').all(docVersion) as Array<Record<string, unknown>>;
-    return rows.map(r => this.hydrateChunk(r));
-  }
-
-  getChunkCode(chunkId: string): ChunkCode[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT id, chunk_id, language, code FROM chunk_code WHERE chunk_id = ?').all(chunkId) as Array<Record<string, unknown>>;
-    return rows.map(r => ({
-      id: String(r.id),
-      chunkId: String(r.chunk_id),
-      language: r.language ? String(r.language) : undefined,
-      code: String(r.code),
-    }));
-  }
-
-  getChunkSymbols(chunkId: string): SymbolReference[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT id, chunk_id, name, kind FROM symbol_references WHERE chunk_id = ?').all(chunkId) as Array<Record<string, unknown>>;
-    return rows.map(r => ({
-      id: String(r.id),
-      chunkId: String(r.chunk_id),
-      name: String(r.name),
-      kind: r.kind as any,
-    }));
-  }
-
-  getChunkRelationships(chunkId: string): ChunkRelationship[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare(`
-      SELECT source_chunk_id, target_chunk_id, type
-      FROM chunk_relationships
-      WHERE source_chunk_id = ? OR target_chunk_id = ?
-    `).all(chunkId, chunkId) as Array<Record<string, unknown>>;
-    return rows.map(r => ({
-      sourceChunkId: String(r.source_chunk_id),
-      targetChunkId: String(r.target_chunk_id),
-      type: r.type as any,
-    }));
-  }
-
-  countChunks(snapshotId?: string): number {
-    const raw = this.db.getRawDb();
-    if (snapshotId) {
-      const row = raw.prepare('SELECT COUNT(*) as count FROM chunks WHERE snapshot_id = ?').get(snapshotId) as { count: number };
-      return row?.count || 0;
-    }
-    const row = raw.prepare('SELECT COUNT(*) as count FROM chunks').get() as { count: number };
-    return row?.count || 0;
-  }
-
-  searchChunksFts(
-    query: string,
-    options: { limit?: number; snapshotId?: string; chunkType?: ChunkType; docVersion?: string } = {}
-  ): Array<{ chunk: DocumentChunk; ftsRank: number; symbols: SymbolReference[]; codeSnippets: ChunkCode[] }> {
-    if (!this.db.isFtsAvailable()) {
-      return [];
-    }
-
-    const raw = this.db.getRawDb();
-    const limit = options.limit || 20;
-
-    // Sanitize query for FTS5: strip dangerous punctuation, quote bare words or phrases
-    const cleanTokens = query
-      .replace(/[^\w\s-]/g, ' ')
-      .trim()
-      .split(/\s+/)
-      .filter(t => t.length > 0);
-
-    if (cleanTokens.length === 0) {
-      return [];
-    }
-
-    // Use OR-joined quoted tokens or phrase search
-    const sanitizedFtsQuery = cleanTokens.map(t => `"${t.replace(/"/g, '""')}"`).join(' OR ');
-
-    try {
-      let sql = `
-        SELECT c.*, fts.rank as fts_rank
-        FROM chunks c
-        JOIN chunks_fts fts ON c.id = fts.chunk_id
-        WHERE chunks_fts MATCH ?
-      `;
-      const params: unknown[] = [sanitizedFtsQuery];
-
-      if (options.snapshotId) {
-        sql += ' AND c.snapshot_id = ?';
-        params.push(options.snapshotId);
-      }
-
-      if (options.chunkType) {
-        sql += ' AND c.chunk_type = ?';
-        params.push(options.chunkType);
-      }
-
-      if (options.docVersion) {
-        sql += ' AND c.doc_version = ?';
-        params.push(options.docVersion);
-      }
-
-      sql += ' ORDER BY fts.rank LIMIT ?';
-      params.push(limit);
-
-      const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
-
-      return rows.map(r => {
-        const chunk = this.hydrateChunk(r);
-        return {
-          chunk,
-          ftsRank: Number(r.fts_rank),
-          symbols: this.getChunkSymbols(chunk.id),
-          codeSnippets: this.getChunkCode(chunk.id),
-        };
-      });
-    } catch {
-      return [];
-    }
-  }
-
-  private hydrateChunk(row: Record<string, unknown>): DocumentChunk {
-    const sectionPath: string[] = row.section_path_json ? JSON.parse(String(row.section_path_json)) : [];
-    const provenance: Provenance = row.provenance_json
-      ? JSON.parse(String(row.provenance_json))
-      : {
-          sourceUrl: '',
-          targetUrl: '',
-          fetchedAt: String(row.created_at || ''),
-          discoveredBy: 'direct',
-          contentHash: String(row.content_hash || ''),
-          snapshotId: String(row.snapshot_id || ''),
-        };
-
-    return {
-      id: String(row.id),
-      pageId: String(row.page_id),
-      snapshotId: String(row.snapshot_id),
-      title: row.title ? String(row.title) : undefined,
-      sectionPath,
-      content: String(row.content),
-      chunkType: row.chunk_type as ChunkType,
-      language: row.language ? String(row.language) : undefined,
-      tokenEstimate: Number(row.token_estimate),
-      ordinal: Number(row.ordinal),
-      contentHash: String(row.content_hash),
-      docVersion: row.doc_version ? String(row.doc_version) : undefined,
-      provenance,
-    };
-  }
-}

package/packages/storage/src/repositories/example-repository.ts

@@ -1,206 +0,0 @@
-import type { IndexedExample } from '../../../shared/src/index.ts';
-import type { DocOrbitDb } from '../db.ts';
-import type { IExampleRepository } from '../interfaces.ts';
-import { extractSearchTokens } from '../search-tokens.ts';
-
-export class ExampleRepository implements IExampleRepository {
-  private db: DocOrbitDb;
-
-  constructor(db: DocOrbitDb) {
-    this.db = db;
-  }
-
-  saveIndexedExamples(examples: IndexedExample[]): void {
-    if (examples.length === 0) return;
-    const raw = this.db.getRawDb();
-
-    const insertEx = raw.prepare(`
-      INSERT INTO indexed_examples (
-        id, chunk_id, page_id, snapshot_id, language, framework,
-        task, code, source_url, source_authority, related_api,
-        related_symbol, doc_version, provenance_json, created_at
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      ON CONFLICT(id) DO UPDATE SET
-        chunk_id = excluded.chunk_id,
-        language = excluded.language,
-        framework = excluded.framework,
-        task = excluded.task,
-        code = excluded.code,
-        source_url = excluded.source_url,
-        source_authority = excluded.source_authority,
-        related_api = excluded.related_api,
-        related_symbol = excluded.related_symbol,
-        doc_version = excluded.doc_version,
-        provenance_json = excluded.provenance_json
-    `);
-
-    const insertFts = this.db.isFtsAvailable()
-      ? raw.prepare(`
-          INSERT INTO indexed_examples_fts (example_id, task, language, framework, code, related_api, related_symbol)
-          VALUES (?, ?, ?, ?, ?, ?, ?)
-        `)
-      : null;
-
-    const deleteFts = this.db.isFtsAvailable()
-      ? raw.prepare('DELETE FROM indexed_examples_fts WHERE example_id = ?')
-      : null;
-
-    raw.exec('BEGIN TRANSACTION;');
-    try {
-      for (const ex of examples) {
-        insertEx.run(
-          ex.id,
-          ex.chunkId || null,
-          ex.pageId,
-          ex.snapshotId || 'snap_default',
-          ex.language,
-          ex.framework || null,
-          ex.task,
-          ex.code,
-          ex.sourceUrl,
-          ex.sourceAuthority,
-          ex.relatedApi || null,
-          ex.relatedSymbol || null,
-          ex.docVersion || null,
-          ex.provenance ? JSON.stringify(ex.provenance) : null,
-          ex.createdAt || new Date().toISOString()
-        );
-
-        if (insertFts && deleteFts) {
-          deleteFts.run(ex.id);
-          insertFts.run(
-            ex.id,
-            ex.task,
-            ex.language,
-            ex.framework || '',
-            ex.code,
-            ex.relatedApi || '',
-            ex.relatedSymbol || ''
-          );
-        }
-      }
-      raw.exec('COMMIT;');
-    } catch (err) {
-      raw.exec('ROLLBACK;');
-      throw err;
-    }
-  }
-
-  getIndexedExamplesBySnapshot(snapshotId: string): IndexedExample[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT * FROM indexed_examples WHERE snapshot_id = ?').all(snapshotId) as Array<Record<string, unknown>>;
-    return rows.map(r => this.hydrateIndexedExample(r));
-  }
-
-  searchIndexedExamples(
-    query: string,
-    options: { language?: string; framework?: string; docVersion?: string; limit?: number } = {}
-  ): IndexedExample[] {
-    const raw = this.db.getRawDb();
-    const limit = options.limit || 10;
-    const term = query.trim();
-
-    let results: IndexedExample[] = [];
-
-    // FTS attempt
-    if (this.db.isFtsAvailable() && term.length > 0) {
-      const cleanTokens = extractSearchTokens(term);
-
-      if (cleanTokens.length > 0) {
-        const sanitizedFts = cleanTokens
-          .map(t => t.length >= 4 ? `"${t.replace(/"/g, '""')}"*` : `"${t.replace(/"/g, '""')}"`)
-          .join(' OR ');
-        try {
-          let sql = `
-            SELECT ex.*, fts.rank as fts_rank
-            FROM indexed_examples ex
-            JOIN indexed_examples_fts fts ON ex.id = fts.example_id
-            WHERE indexed_examples_fts MATCH ?
-          `;
-          const params: unknown[] = [sanitizedFts];
-
-          if (options.language) {
-            sql += ' AND lower(ex.language) = ?';
-            params.push(options.language.toLowerCase());
-          }
-          if (options.framework) {
-            sql += ' AND lower(ex.framework) = ?';
-            params.push(options.framework.toLowerCase());
-          }
-          if (options.docVersion) {
-            sql += ' AND (ex.doc_version = ? OR ex.doc_version IS NULL)';
-            params.push(options.docVersion);
-          }
-
-          sql += " ORDER BY CASE WHEN ex.source_authority = 'official' THEN 0 ELSE 1 END, fts.rank LIMIT ?";
-          params.push(limit);
-
-          const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
-          results = rows.map(r => this.hydrateIndexedExample(r));
-        } catch {
-          // Fall through to LIKE
-        }
-      }
-    }
-
-    if (results.length === 0) {
-      // LIKE fallback
-      let sql = 'SELECT * FROM indexed_examples WHERE 1=1';
-      const params: unknown[] = [];
-
-      if (options.language) {
-        sql += ' AND lower(language) = ?';
-        params.push(options.language.toLowerCase());
-      }
-      if (options.framework) {
-        sql += ' AND lower(framework) = ?';
-        params.push(options.framework.toLowerCase());
-      }
-      if (options.docVersion) {
-        sql += ' AND (doc_version = ? OR doc_version IS NULL)';
-        params.push(options.docVersion);
-      }
-      if (term.length > 0) {
-        const tokens = extractSearchTokens(term);
-        if (tokens.length > 0) {
-          const tokenClauses = tokens.map(() => '(task LIKE ? OR code LIKE ? OR related_api LIKE ? OR related_symbol LIKE ?)');
-          sql += ` AND (${tokenClauses.join(' OR ')})`;
-          for (const t of tokens) {
-            const lp = `%${t}%`;
-            params.push(lp, lp, lp, lp);
-          }
-        } else {
-          return [];
-        }
-      }
-
-      sql += " ORDER BY CASE WHEN source_authority = 'official' THEN 0 ELSE 1 END LIMIT ?";
-      params.push(limit);
-
-      const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
-      results = rows.map(r => this.hydrateIndexedExample(r));
-    }
-
-    return results;
-  }
-
-  private hydrateIndexedExample(row: Record<string, unknown>): IndexedExample {
-    return {
-      id: String(row.id),
-      chunkId: row.chunk_id ? String(row.chunk_id) : undefined,
-      pageId: String(row.page_id),
-      snapshotId: String(row.snapshot_id),
-      language: String(row.language),
-      framework: row.framework ? String(row.framework) : undefined,
-      task: String(row.task),
-      code: String(row.code),
-      sourceUrl: String(row.source_url),
-      sourceAuthority: String(row.source_authority) as IndexedExample['sourceAuthority'],
-      relatedApi: row.related_api ? String(row.related_api) : undefined,
-      relatedSymbol: row.related_symbol ? String(row.related_symbol) : undefined,
-      docVersion: row.doc_version ? String(row.doc_version) : undefined,
-      provenance: row.provenance_json ? JSON.parse(String(row.provenance_json)) : undefined,
-      createdAt: String(row.created_at),
-    };
-  }
-}