docorbit 0.1.0

package/packages/storage/src/repositories/page-repository.ts

@@ -0,0 +1,205 @@
+import type {
+  NormalizedPage,
+  Link,
+  CodeExample,
+  Heading,
+  SecurityAnnotation,
+  Provenance,
+} from '../../../shared/src/index.ts';
+import type { DocOrbitDb } from '../db.ts';
+
+export class PageRepository {
+  private db: DocOrbitDb;
+
+  constructor(db: DocOrbitDb) {
+    this.db = db;
+  }
+
+  savePage(page: NormalizedPage): void {
+    const raw = this.db.getRawDb();
+
+    const pageStmt = raw.prepare(`
+      INSERT INTO pages (
+        id, source_id, title, url, content, content_hash,
+        fetched_at, raw_bytes, estimated_tokens,
+        headings_json, security_annotations_json, provenance_json
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(url) DO UPDATE SET
+        title = excluded.title,
+        content = excluded.content,
+        content_hash = excluded.content_hash,
+        fetched_at = excluded.fetched_at,
+        raw_bytes = excluded.raw_bytes,
+        estimated_tokens = excluded.estimated_tokens,
+        headings_json = excluded.headings_json,
+        security_annotations_json = excluded.security_annotations_json,
+        provenance_json = excluded.provenance_json
+    `);
+
+    pageStmt.run(
+      page.id,
+      page.sourceId || null,
+      page.title,
+      page.url,
+      page.content,
+      page.contentHash,
+      page.fetchedAt,
+      page.rawBytes,
+      page.estimatedTokens,
+      JSON.stringify(page.headings),
+      JSON.stringify(page.securityAnnotations),
+      page.provenance ? JSON.stringify(page.provenance) : null
+    );
+
+    raw.prepare('DELETE FROM links WHERE page_id = ?').run(page.id);
+    const linkStmt = raw.prepare(`
+      INSERT INTO links (id, page_id, text, url, is_external)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+
+    for (let i = 0; i < page.links.length; i++) {
+      const link = page.links[i];
+      const linkId = `${page.id}_link_${i}`;
+      linkStmt.run(linkId, page.id, link.text, link.url, link.isExternal ? 1 : 0);
+    }
+
+    raw.prepare('DELETE FROM code_examples WHERE page_id = ?').run(page.id);
+    const codeStmt = raw.prepare(`
+      INSERT INTO code_examples (id, page_id, language, code, caption)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+
+    for (let i = 0; i < page.codeExamples.length; i++) {
+      const example = page.codeExamples[i];
+      const exampleId = `${page.id}_code_${i}`;
+      codeStmt.run(exampleId, page.id, example.language, example.code, example.caption || null);
+    }
+
+    if (this.db.isFtsAvailable()) {
+      try {
+        raw.prepare('DELETE FROM pages_fts WHERE page_id = ?').run(page.id);
+        raw.prepare(`
+          INSERT INTO pages_fts (page_id, title, content)
+          VALUES (?, ?, ?)
+        `).run(page.id, page.title, page.content);
+      } catch {
+        // Continue
+      }
+    }
+  }
+
+  getPage(id: string): NormalizedPage | null {
+    const raw = this.db.getRawDb();
+    const row = raw.prepare('SELECT * FROM pages WHERE id = ?').get(id) as Record<string, unknown> | undefined;
+    if (!row) return null;
+    return this.hydratePage(row);
+  }
+
+  getPageByUrl(url: string): NormalizedPage | null {
+    const raw = this.db.getRawDb();
+    const row = raw.prepare('SELECT * FROM pages WHERE url = ?').get(url) as Record<string, unknown> | undefined;
+    if (!row) return null;
+    return this.hydratePage(row);
+  }
+
+  getPagesBySource(sourceId: string): NormalizedPage[] {
+    const raw = this.db.getRawDb();
+    const rows = raw.prepare('SELECT * FROM pages WHERE source_id = ? ORDER BY fetched_at ASC').all(sourceId) as Array<Record<string, unknown>>;
+    return rows.map(r => this.hydratePage(r));
+  }
+
+  listPages(limit: number = 50, offset: number = 0): NormalizedPage[] {
+    const raw = this.db.getRawDb();
+    const rows = raw.prepare('SELECT * FROM pages ORDER BY fetched_at DESC LIMIT ? OFFSET ?').all(limit, offset) as Array<Record<string, unknown>>;
+    return rows.map(r => this.hydratePage(r));
+  }
+
+  countPages(sourceId?: string): number {
+    const raw = this.db.getRawDb();
+    if (sourceId) {
+      const row = raw.prepare('SELECT COUNT(*) as count FROM pages WHERE source_id = ?').get(sourceId) as { count: number };
+      return row?.count || 0;
+    }
+    const row = raw.prepare('SELECT COUNT(*) as count FROM pages').get() as { count: number };
+    return row?.count || 0;
+  }
+
+  searchPagesFts(query: string, limit: number = 10): NormalizedPage[] {
+    if (!this.db.isFtsAvailable()) {
+      return [];
+    }
+
+    const raw = this.db.getRawDb();
+    const sanitizedQuery = `"${query.replace(/"/g, '""')}"`;
+
+    try {
+      const rows = raw.prepare(`
+        SELECT p.* FROM pages p
+        JOIN pages_fts fts ON p.id = fts.page_id
+        WHERE pages_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `).all(sanitizedQuery, limit) as Array<Record<string, unknown>>;
+
+      return rows.map(r => this.hydratePage(r));
+    } catch {
+      return [];
+    }
+  }
+
+  getPageLinks(pageId: string): string[] {
+    const raw = this.db.getRawDb();
+    const rows = raw.prepare('SELECT url FROM links WHERE page_id = ?').all(pageId) as Array<Record<string, unknown>>;
+    return rows.map(r => String(r.url));
+  }
+
+  savePageLinks(pageId: string, targetUrls: string[]): void {
+    const raw = this.db.getRawDb();
+    raw.prepare('DELETE FROM links WHERE page_id = ?').run(pageId);
+    const stmt = raw.prepare('INSERT OR IGNORE INTO links (id, page_id, text, url, is_external) VALUES (?, ?, ?, ?, ?)');
+    for (let i = 0; i < targetUrls.length; i++) {
+      stmt.run(`${pageId}_link_${i}`, pageId, '', targetUrls[i], 0);
+    }
+  }
+
+  hydratePage(row: Record<string, unknown>): NormalizedPage {
+    const raw = this.db.getRawDb();
+    const pageId = String(row.id);
+
+    const linkRows = raw.prepare('SELECT text, url, is_external FROM links WHERE page_id = ?').all(pageId) as Array<Record<string, unknown>>;
+    const links: Link[] = linkRows.map(lr => ({
+      text: String(lr.text),
+      url: String(lr.url),
+      isExternal: Number(lr.is_external) === 1,
+    }));
+
+    const codeRows = raw.prepare('SELECT id, language, code, caption FROM code_examples WHERE page_id = ?').all(pageId) as Array<Record<string, unknown>>;
+    const codeExamples: CodeExample[] = codeRows.map(cr => ({
+      id: String(cr.id),
+      language: String(cr.language),
+      code: String(cr.code),
+      caption: cr.caption ? String(cr.caption) : undefined,
+    }));
+
+    const headings: Heading[] = row.headings_json ? JSON.parse(String(row.headings_json)) : [];
+    const securityAnnotations: SecurityAnnotation[] = row.security_annotations_json ? JSON.parse(String(row.security_annotations_json)) : [];
+    const provenance: Provenance | undefined = row.provenance_json ? JSON.parse(String(row.provenance_json)) : undefined;
+
+    return {
+      id: pageId,
+      sourceId: String(row.source_id),
+      title: String(row.title),
+      url: String(row.url),
+      content: String(row.content),
+      headings,
+      links,
+      codeExamples,
+      contentHash: String(row.content_hash),
+      fetchedAt: String(row.fetched_at),
+      rawBytes: Number(row.raw_bytes),
+      estimatedTokens: Number(row.estimated_tokens),
+      securityAnnotations,
+      provenance,
+    };
+  }
+}

package/packages/storage/src/repositories/pitfall-repository.ts

@@ -0,0 +1,188 @@
+import type { Pitfall, PitfallKind } from '../../../shared/src/index.ts';
+import type { DocOrbitDb } from '../db.ts';
+import type { IPitfallRepository } from '../interfaces.ts';
+import { extractSearchTokens } from '../search-tokens.ts';
+
+export class PitfallRepository implements IPitfallRepository {
+  private db: DocOrbitDb;
+
+  constructor(db: DocOrbitDb) {
+    this.db = db;
+  }
+
+  savePitfalls(pitfalls: Pitfall[]): void {
+    if (pitfalls.length === 0) return;
+    const raw = this.db.getRawDb();
+
+    const insertPf = raw.prepare(`
+      INSERT INTO pitfalls (
+        id, chunk_id, page_id, snapshot_id, kind, title,
+        content, related_api, related_symbol, doc_version,
+        provenance_json, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        chunk_id = excluded.chunk_id,
+        kind = excluded.kind,
+        title = excluded.title,
+        content = excluded.content,
+        related_api = excluded.related_api,
+        related_symbol = excluded.related_symbol,
+        doc_version = excluded.doc_version,
+        provenance_json = excluded.provenance_json
+    `);
+
+    const insertFts = this.db.isFtsAvailable()
+      ? raw.prepare(`
+          INSERT INTO pitfalls_fts (pitfall_id, kind, title, content, related_api, related_symbol)
+          VALUES (?, ?, ?, ?, ?, ?)
+        `)
+      : null;
+
+    const deleteFts = this.db.isFtsAvailable()
+      ? raw.prepare('DELETE FROM pitfalls_fts WHERE pitfall_id = ?')
+      : null;
+
+    raw.exec('BEGIN TRANSACTION;');
+    try {
+      for (const pf of pitfalls) {
+        insertPf.run(
+          pf.id,
+          pf.chunkId || null,
+          pf.pageId,
+          pf.snapshotId || 'snap_default',
+          pf.kind,
+          pf.title,
+          pf.content || pf.message || '',
+          pf.relatedApi || null,
+          pf.relatedSymbol || null,
+          pf.docVersion || null,
+          pf.provenance ? JSON.stringify(pf.provenance) : null,
+          pf.createdAt || new Date().toISOString()
+        );
+
+        if (insertFts && deleteFts) {
+          deleteFts.run(pf.id);
+          insertFts.run(
+            pf.id,
+            pf.kind,
+            pf.title,
+            pf.content || pf.message || '',
+            pf.relatedApi || '',
+            pf.relatedSymbol || ''
+          );
+        }
+      }
+      raw.exec('COMMIT;');
+    } catch (err) {
+      raw.exec('ROLLBACK;');
+      throw err;
+    }
+  }
+
+  getPitfallsBySnapshot(snapshotId: string): Pitfall[] {
+    const raw = this.db.getRawDb();
+    const rows = raw.prepare('SELECT * FROM pitfalls WHERE snapshot_id = ?').all(snapshotId) as Array<Record<string, unknown>>;
+    return rows.map(r => this.hydratePitfall(r));
+  }
+
+  searchPitfalls(
+    query: string,
+    options: { kind?: PitfallKind; docVersion?: string; limit?: number } = {}
+  ): Pitfall[] {
+    const raw = this.db.getRawDb();
+    const limit = options.limit || 10;
+    const term = query.trim();
+
+    let results: Pitfall[] = [];
+
+    // FTS attempt
+    if (this.db.isFtsAvailable() && term.length > 0) {
+      const cleanTokens = extractSearchTokens(term);
+
+      if (cleanTokens.length > 0) {
+        const sanitizedFts = cleanTokens
+          .map(t => t.length >= 4 ? `"${t.replace(/"/g, '""')}"*` : `"${t.replace(/"/g, '""')}"`)
+          .join(' OR ');
+        try {
+          let sql = `
+            SELECT pf.*, fts.rank as fts_rank
+            FROM pitfalls pf
+            JOIN pitfalls_fts fts ON pf.id = fts.pitfall_id
+            WHERE pitfalls_fts MATCH ?
+          `;
+          const params: unknown[] = [sanitizedFts];
+
+          if (options.kind) {
+            sql += ' AND pf.kind = ?';
+            params.push(options.kind);
+          }
+          if (options.docVersion) {
+            sql += ' AND (pf.doc_version = ? OR pf.doc_version IS NULL)';
+            params.push(options.docVersion);
+          }
+
+          sql += ' ORDER BY fts.rank LIMIT ?';
+          params.push(limit);
+
+          const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
+          results = rows.map(r => this.hydratePitfall(r));
+        } catch {
+          // Fall through to LIKE
+        }
+      }
+    }
+
+    if (results.length === 0) {
+      // LIKE fallback
+      let sql = 'SELECT * FROM pitfalls WHERE 1=1';
+      const params: unknown[] = [];
+
+      if (options.kind) {
+        sql += ' AND kind = ?';
+        params.push(options.kind);
+      }
+      if (options.docVersion) {
+        sql += ' AND (doc_version = ? OR doc_version IS NULL)';
+        params.push(options.docVersion);
+      }
+      if (term.length > 0) {
+        const tokens = extractSearchTokens(term);
+        if (tokens.length > 0) {
+          const tokenClauses = tokens.map(() => '(title LIKE ? OR content LIKE ? OR related_api LIKE ? OR related_symbol LIKE ?)');
+          sql += ` AND (${tokenClauses.join(' OR ')})`;
+          for (const t of tokens) {
+            const lp = `%${t}%`;
+            params.push(lp, lp, lp, lp);
+          }
+        } else {
+          return [];
+        }
+      }
+
+      sql += ' ORDER BY kind LIMIT ?';
+      params.push(limit);
+
+      const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
+      results = rows.map(r => this.hydratePitfall(r));
+    }
+
+    return results;
+  }
+
+  private hydratePitfall(row: Record<string, unknown>): Pitfall {
+    return {
+      id: String(row.id),
+      chunkId: row.chunk_id ? String(row.chunk_id) : undefined,
+      pageId: String(row.page_id),
+      snapshotId: String(row.snapshot_id),
+      kind: String(row.kind) as PitfallKind,
+      title: String(row.title),
+      content: String(row.content),
+      relatedApi: row.related_api ? String(row.related_api) : undefined,
+      relatedSymbol: row.related_symbol ? String(row.related_symbol) : undefined,
+      docVersion: row.doc_version ? String(row.doc_version) : undefined,
+      provenance: row.provenance_json ? JSON.parse(String(row.provenance_json)) : undefined,
+      createdAt: String(row.created_at),
+    };
+  }
+}

package/packages/storage/src/repositories/source-repository.ts

@@ -0,0 +1,205 @@
+import { computeSha256 } from '../../../shared/src/index.ts';
+import type { DiscoveredSource, NormalizedPage } from '../../../shared/src/index.ts';
+import type { DocOrbitDb } from '../db.ts';
+
+export interface SnapshotRecord {
+  id: string;
+  sourceId: string;
+  snapshotHash: string;
+  pageCount: number;
+  capturedAt: string;
+  docVersion?: string;
+  metadata?: Record<string, unknown>;
+}
+
+export class SourceRepository {
+  private db: DocOrbitDb;
+  private hydratePageFn?: (row: Record<string, unknown>) => NormalizedPage;
+
+  constructor(
+    db: DocOrbitDb,
+    hydratePageFn?: (row: Record<string, unknown>) => NormalizedPage
+  ) {
+    this.db = db;
+    this.hydratePageFn = hydratePageFn;
+  }
+
+  saveSource(source: DiscoveredSource): string {
+    const raw = this.db.getRawDb();
+    const id = `src_${computeSha256(source.url).slice(0, 16)}`;
+    const createdAt = new Date().toISOString();
+
+    const stmt = raw.prepare(`
+      INSERT INTO sources (
+        id, url, type, discovered_by, status, content_type,
+        confidence, authority, machine_readable, metadata, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(url) DO UPDATE SET
+        type = excluded.type,
+        discovered_by = excluded.discovered_by,
+        status = excluded.status,
+        content_type = excluded.content_type,
+        confidence = excluded.confidence,
+        authority = excluded.authority,
+        machine_readable = excluded.machine_readable,
+        metadata = excluded.metadata
+    `);
+
+    stmt.run(
+      id,
+      source.url,
+      source.type,
+      source.discoveredBy || 'manual',
+      source.status,
+      source.contentType || null,
+      source.confidence,
+      source.authority,
+      source.machineReadable ? 1 : 0,
+      source.metadata ? JSON.stringify(source.metadata) : null,
+      createdAt
+    );
+
+    const row = raw.prepare('SELECT id FROM sources WHERE url = ?').get(source.url) as { id: string } | undefined;
+    return row?.id || id;
+  }
+
+  getSource(id: string): (DiscoveredSource & { id: string }) | null {
+    const raw = this.db.getRawDb();
+    const row = raw.prepare('SELECT * FROM sources WHERE id = ?').get(id) as Record<string, unknown> | undefined;
+    if (!row) return null;
+    return this.mapSourceRow(row);
+  }
+
+  getSourceByUrl(url: string): (DiscoveredSource & { id: string }) | null {
+    const raw = this.db.getRawDb();
+    const row = raw.prepare('SELECT * FROM sources WHERE url = ?').get(url) as Record<string, unknown> | undefined;
+    if (!row) return null;
+    return this.mapSourceRow(row);
+  }
+
+  listSources(): Array<DiscoveredSource & { id: string; createdAt: string }> {
+    const raw = this.db.getRawDb();
+    const rows = raw.prepare('SELECT * FROM sources ORDER BY created_at DESC').all() as Array<Record<string, unknown>>;
+    return rows.map(r => ({
+      ...this.mapSourceRow(r),
+      createdAt: String(r.created_at),
+    }));
+  }
+
+  createSnapshot(sourceId: string, metadata?: Record<string, unknown>, docVersion?: string): string {
+    const raw = this.db.getRawDb();
+    const pages = raw.prepare('SELECT id, content_hash FROM pages WHERE source_id = ? ORDER BY id ASC').all(sourceId) as Array<{ id: string; content_hash: string }>;
+
+    const combinedHash = computeSha256(pages.map(p => p.content_hash).join(':'));
+    const snapshotId = `snap_${combinedHash.slice(0, 16)}`;
+    const now = new Date().toISOString();
+    const version = docVersion || (metadata?.docVersion as string) || null;
+
+    const stmt = raw.prepare(`
+      INSERT INTO snapshots (id, source_id, snapshot_hash, page_count, captured_at, doc_version, metadata)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        captured_at = excluded.captured_at,
+        page_count = excluded.page_count,
+        doc_version = COALESCE(excluded.doc_version, snapshots.doc_version),
+        metadata = excluded.metadata
+    `);
+
+    stmt.run(
+      snapshotId,
+      sourceId,
+      combinedHash,
+      pages.length,
+      now,
+      version,
+      metadata ? JSON.stringify(metadata) : null
+    );
+
+    // Link snapshot to page membership in snapshot_pages
+    const linkStmt = raw.prepare(`
+      INSERT OR REPLACE INTO snapshot_pages (snapshot_id, page_id, content_hash)
+      VALUES (?, ?, ?)
+    `);
+    for (const p of pages) {
+      linkStmt.run(snapshotId, p.id, p.content_hash);
+    }
+
+    return snapshotId;
+  }
+
+  linkSnapshotPages(snapshotId: string, pageIds: string[]): void {
+    const raw = this.db.getRawDb();
+    const linkStmt = raw.prepare(`
+      INSERT OR REPLACE INTO snapshot_pages (snapshot_id, page_id, content_hash)
+      VALUES (?, ?, COALESCE((SELECT content_hash FROM pages WHERE id = ?), 'hash_default'))
+    `);
+    for (const pageId of pageIds) {
+      linkStmt.run(snapshotId, pageId, pageId);
+    }
+  }
+
+  getSnapshot(snapshotId: string): SnapshotRecord | null {
+    const raw = this.db.getRawDb();
+    const row = raw.prepare('SELECT * FROM snapshots WHERE id = ?').get(snapshotId) as Record<string, unknown> | undefined;
+    if (!row) return null;
+    return {
+      id: String(row.id),
+      sourceId: String(row.source_id),
+      snapshotHash: String(row.snapshot_hash),
+      pageCount: Number(row.page_count),
+      capturedAt: String(row.captured_at),
+      docVersion: row.doc_version ? String(row.doc_version) : undefined,
+      metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined,
+    };
+  }
+
+  listSnapshots(sourceId?: string): SnapshotRecord[] {
+    const raw = this.db.getRawDb();
+    let sql = 'SELECT * FROM snapshots';
+    const params: unknown[] = [];
+    if (sourceId) {
+      sql += ' WHERE source_id = ?';
+      params.push(sourceId);
+    }
+    sql += ' ORDER BY captured_at DESC';
+    const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
+    return rows.map(row => ({
+      id: String(row.id),
+      sourceId: String(row.source_id),
+      snapshotHash: String(row.snapshot_hash),
+      pageCount: Number(row.page_count),
+      capturedAt: String(row.captured_at),
+      docVersion: row.doc_version ? String(row.doc_version) : undefined,
+      metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined,
+    }));
+  }
+
+  getSnapshotPages(snapshotId: string): NormalizedPage[] {
+    const raw = this.db.getRawDb();
+    const rows = raw.prepare(`
+      SELECT p.* FROM pages p
+      JOIN snapshot_pages sp ON p.id = sp.page_id
+      WHERE sp.snapshot_id = ?
+      ORDER BY p.id ASC
+    `).all(snapshotId) as Array<Record<string, unknown>>;
+    if (this.hydratePageFn) {
+      return rows.map(r => this.hydratePageFn!(r));
+    }
+    return [];
+  }
+
+  private mapSourceRow(row: Record<string, unknown>): DiscoveredSource & { id: string } {
+    return {
+      id: String(row.id),
+      url: String(row.url),
+      type: row.type as any,
+      discoveredBy: String(row.discovered_by),
+      status: row.status as any,
+      contentType: row.content_type ? String(row.content_type) : undefined,
+      confidence: Number(row.confidence),
+      authority: row.authority as any,
+      machineReadable: Number(row.machine_readable) === 1,
+      metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined,
+    };
+  }
+}