docorbit 0.1.0 → 0.1.1

package/packages/storage/src/repositories/page-repository.ts

@@ -1,205 +0,0 @@
-import type {
-  NormalizedPage,
-  Link,
-  CodeExample,
-  Heading,
-  SecurityAnnotation,
-  Provenance,
-} from '../../../shared/src/index.ts';
-import type { DocOrbitDb } from '../db.ts';
-
-export class PageRepository {
-  private db: DocOrbitDb;
-
-  constructor(db: DocOrbitDb) {
-    this.db = db;
-  }
-
-  savePage(page: NormalizedPage): void {
-    const raw = this.db.getRawDb();
-
-    const pageStmt = raw.prepare(`
-      INSERT INTO pages (
-        id, source_id, title, url, content, content_hash,
-        fetched_at, raw_bytes, estimated_tokens,
-        headings_json, security_annotations_json, provenance_json
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      ON CONFLICT(url) DO UPDATE SET
-        title = excluded.title,
-        content = excluded.content,
-        content_hash = excluded.content_hash,
-        fetched_at = excluded.fetched_at,
-        raw_bytes = excluded.raw_bytes,
-        estimated_tokens = excluded.estimated_tokens,
-        headings_json = excluded.headings_json,
-        security_annotations_json = excluded.security_annotations_json,
-        provenance_json = excluded.provenance_json
-    `);
-
-    pageStmt.run(
-      page.id,
-      page.sourceId || null,
-      page.title,
-      page.url,
-      page.content,
-      page.contentHash,
-      page.fetchedAt,
-      page.rawBytes,
-      page.estimatedTokens,
-      JSON.stringify(page.headings),
-      JSON.stringify(page.securityAnnotations),
-      page.provenance ? JSON.stringify(page.provenance) : null
-    );
-
-    raw.prepare('DELETE FROM links WHERE page_id = ?').run(page.id);
-    const linkStmt = raw.prepare(`
-      INSERT INTO links (id, page_id, text, url, is_external)
-      VALUES (?, ?, ?, ?, ?)
-    `);
-
-    for (let i = 0; i < page.links.length; i++) {
-      const link = page.links[i];
-      const linkId = `${page.id}_link_${i}`;
-      linkStmt.run(linkId, page.id, link.text, link.url, link.isExternal ? 1 : 0);
-    }
-
-    raw.prepare('DELETE FROM code_examples WHERE page_id = ?').run(page.id);
-    const codeStmt = raw.prepare(`
-      INSERT INTO code_examples (id, page_id, language, code, caption)
-      VALUES (?, ?, ?, ?, ?)
-    `);
-
-    for (let i = 0; i < page.codeExamples.length; i++) {
-      const example = page.codeExamples[i];
-      const exampleId = `${page.id}_code_${i}`;
-      codeStmt.run(exampleId, page.id, example.language, example.code, example.caption || null);
-    }
-
-    if (this.db.isFtsAvailable()) {
-      try {
-        raw.prepare('DELETE FROM pages_fts WHERE page_id = ?').run(page.id);
-        raw.prepare(`
-          INSERT INTO pages_fts (page_id, title, content)
-          VALUES (?, ?, ?)
-        `).run(page.id, page.title, page.content);
-      } catch {
-        // Continue
-      }
-    }
-  }
-
-  getPage(id: string): NormalizedPage | null {
-    const raw = this.db.getRawDb();
-    const row = raw.prepare('SELECT * FROM pages WHERE id = ?').get(id) as Record<string, unknown> | undefined;
-    if (!row) return null;
-    return this.hydratePage(row);
-  }
-
-  getPageByUrl(url: string): NormalizedPage | null {
-    const raw = this.db.getRawDb();
-    const row = raw.prepare('SELECT * FROM pages WHERE url = ?').get(url) as Record<string, unknown> | undefined;
-    if (!row) return null;
-    return this.hydratePage(row);
-  }
-
-  getPagesBySource(sourceId: string): NormalizedPage[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT * FROM pages WHERE source_id = ? ORDER BY fetched_at ASC').all(sourceId) as Array<Record<string, unknown>>;
-    return rows.map(r => this.hydratePage(r));
-  }
-
-  listPages(limit: number = 50, offset: number = 0): NormalizedPage[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT * FROM pages ORDER BY fetched_at DESC LIMIT ? OFFSET ?').all(limit, offset) as Array<Record<string, unknown>>;
-    return rows.map(r => this.hydratePage(r));
-  }
-
-  countPages(sourceId?: string): number {
-    const raw = this.db.getRawDb();
-    if (sourceId) {
-      const row = raw.prepare('SELECT COUNT(*) as count FROM pages WHERE source_id = ?').get(sourceId) as { count: number };
-      return row?.count || 0;
-    }
-    const row = raw.prepare('SELECT COUNT(*) as count FROM pages').get() as { count: number };
-    return row?.count || 0;
-  }
-
-  searchPagesFts(query: string, limit: number = 10): NormalizedPage[] {
-    if (!this.db.isFtsAvailable()) {
-      return [];
-    }
-
-    const raw = this.db.getRawDb();
-    const sanitizedQuery = `"${query.replace(/"/g, '""')}"`;
-
-    try {
-      const rows = raw.prepare(`
-        SELECT p.* FROM pages p
-        JOIN pages_fts fts ON p.id = fts.page_id
-        WHERE pages_fts MATCH ?
-        ORDER BY rank
-        LIMIT ?
-      `).all(sanitizedQuery, limit) as Array<Record<string, unknown>>;
-
-      return rows.map(r => this.hydratePage(r));
-    } catch {
-      return [];
-    }
-  }
-
-  getPageLinks(pageId: string): string[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT url FROM links WHERE page_id = ?').all(pageId) as Array<Record<string, unknown>>;
-    return rows.map(r => String(r.url));
-  }
-
-  savePageLinks(pageId: string, targetUrls: string[]): void {
-    const raw = this.db.getRawDb();
-    raw.prepare('DELETE FROM links WHERE page_id = ?').run(pageId);
-    const stmt = raw.prepare('INSERT OR IGNORE INTO links (id, page_id, text, url, is_external) VALUES (?, ?, ?, ?, ?)');
-    for (let i = 0; i < targetUrls.length; i++) {
-      stmt.run(`${pageId}_link_${i}`, pageId, '', targetUrls[i], 0);
-    }
-  }
-
-  hydratePage(row: Record<string, unknown>): NormalizedPage {
-    const raw = this.db.getRawDb();
-    const pageId = String(row.id);
-
-    const linkRows = raw.prepare('SELECT text, url, is_external FROM links WHERE page_id = ?').all(pageId) as Array<Record<string, unknown>>;
-    const links: Link[] = linkRows.map(lr => ({
-      text: String(lr.text),
-      url: String(lr.url),
-      isExternal: Number(lr.is_external) === 1,
-    }));
-
-    const codeRows = raw.prepare('SELECT id, language, code, caption FROM code_examples WHERE page_id = ?').all(pageId) as Array<Record<string, unknown>>;
-    const codeExamples: CodeExample[] = codeRows.map(cr => ({
-      id: String(cr.id),
-      language: String(cr.language),
-      code: String(cr.code),
-      caption: cr.caption ? String(cr.caption) : undefined,
-    }));
-
-    const headings: Heading[] = row.headings_json ? JSON.parse(String(row.headings_json)) : [];
-    const securityAnnotations: SecurityAnnotation[] = row.security_annotations_json ? JSON.parse(String(row.security_annotations_json)) : [];
-    const provenance: Provenance | undefined = row.provenance_json ? JSON.parse(String(row.provenance_json)) : undefined;
-
-    return {
-      id: pageId,
-      sourceId: String(row.source_id),
-      title: String(row.title),
-      url: String(row.url),
-      content: String(row.content),
-      headings,
-      links,
-      codeExamples,
-      contentHash: String(row.content_hash),
-      fetchedAt: String(row.fetched_at),
-      rawBytes: Number(row.raw_bytes),
-      estimatedTokens: Number(row.estimated_tokens),
-      securityAnnotations,
-      provenance,
-    };
-  }
-}

package/packages/storage/src/repositories/pitfall-repository.ts

@@ -1,188 +0,0 @@
-import type { Pitfall, PitfallKind } from '../../../shared/src/index.ts';
-import type { DocOrbitDb } from '../db.ts';
-import type { IPitfallRepository } from '../interfaces.ts';
-import { extractSearchTokens } from '../search-tokens.ts';
-
-export class PitfallRepository implements IPitfallRepository {
-  private db: DocOrbitDb;
-
-  constructor(db: DocOrbitDb) {
-    this.db = db;
-  }
-
-  savePitfalls(pitfalls: Pitfall[]): void {
-    if (pitfalls.length === 0) return;
-    const raw = this.db.getRawDb();
-
-    const insertPf = raw.prepare(`
-      INSERT INTO pitfalls (
-        id, chunk_id, page_id, snapshot_id, kind, title,
-        content, related_api, related_symbol, doc_version,
-        provenance_json, created_at
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      ON CONFLICT(id) DO UPDATE SET
-        chunk_id = excluded.chunk_id,
-        kind = excluded.kind,
-        title = excluded.title,
-        content = excluded.content,
-        related_api = excluded.related_api,
-        related_symbol = excluded.related_symbol,
-        doc_version = excluded.doc_version,
-        provenance_json = excluded.provenance_json
-    `);
-
-    const insertFts = this.db.isFtsAvailable()
-      ? raw.prepare(`
-          INSERT INTO pitfalls_fts (pitfall_id, kind, title, content, related_api, related_symbol)
-          VALUES (?, ?, ?, ?, ?, ?)
-        `)
-      : null;
-
-    const deleteFts = this.db.isFtsAvailable()
-      ? raw.prepare('DELETE FROM pitfalls_fts WHERE pitfall_id = ?')
-      : null;
-
-    raw.exec('BEGIN TRANSACTION;');
-    try {
-      for (const pf of pitfalls) {
-        insertPf.run(
-          pf.id,
-          pf.chunkId || null,
-          pf.pageId,
-          pf.snapshotId || 'snap_default',
-          pf.kind,
-          pf.title,
-          pf.content || pf.message || '',
-          pf.relatedApi || null,
-          pf.relatedSymbol || null,
-          pf.docVersion || null,
-          pf.provenance ? JSON.stringify(pf.provenance) : null,
-          pf.createdAt || new Date().toISOString()
-        );
-
-        if (insertFts && deleteFts) {
-          deleteFts.run(pf.id);
-          insertFts.run(
-            pf.id,
-            pf.kind,
-            pf.title,
-            pf.content || pf.message || '',
-            pf.relatedApi || '',
-            pf.relatedSymbol || ''
-          );
-        }
-      }
-      raw.exec('COMMIT;');
-    } catch (err) {
-      raw.exec('ROLLBACK;');
-      throw err;
-    }
-  }
-
-  getPitfallsBySnapshot(snapshotId: string): Pitfall[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT * FROM pitfalls WHERE snapshot_id = ?').all(snapshotId) as Array<Record<string, unknown>>;
-    return rows.map(r => this.hydratePitfall(r));
-  }
-
-  searchPitfalls(
-    query: string,
-    options: { kind?: PitfallKind; docVersion?: string; limit?: number } = {}
-  ): Pitfall[] {
-    const raw = this.db.getRawDb();
-    const limit = options.limit || 10;
-    const term = query.trim();
-
-    let results: Pitfall[] = [];
-
-    // FTS attempt
-    if (this.db.isFtsAvailable() && term.length > 0) {
-      const cleanTokens = extractSearchTokens(term);
-
-      if (cleanTokens.length > 0) {
-        const sanitizedFts = cleanTokens
-          .map(t => t.length >= 4 ? `"${t.replace(/"/g, '""')}"*` : `"${t.replace(/"/g, '""')}"`)
-          .join(' OR ');
-        try {
-          let sql = `
-            SELECT pf.*, fts.rank as fts_rank
-            FROM pitfalls pf
-            JOIN pitfalls_fts fts ON pf.id = fts.pitfall_id
-            WHERE pitfalls_fts MATCH ?
-          `;
-          const params: unknown[] = [sanitizedFts];
-
-          if (options.kind) {
-            sql += ' AND pf.kind = ?';
-            params.push(options.kind);
-          }
-          if (options.docVersion) {
-            sql += ' AND (pf.doc_version = ? OR pf.doc_version IS NULL)';
-            params.push(options.docVersion);
-          }
-
-          sql += ' ORDER BY fts.rank LIMIT ?';
-          params.push(limit);
-
-          const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
-          results = rows.map(r => this.hydratePitfall(r));
-        } catch {
-          // Fall through to LIKE
-        }
-      }
-    }
-
-    if (results.length === 0) {
-      // LIKE fallback
-      let sql = 'SELECT * FROM pitfalls WHERE 1=1';
-      const params: unknown[] = [];
-
-      if (options.kind) {
-        sql += ' AND kind = ?';
-        params.push(options.kind);
-      }
-      if (options.docVersion) {
-        sql += ' AND (doc_version = ? OR doc_version IS NULL)';
-        params.push(options.docVersion);
-      }
-      if (term.length > 0) {
-        const tokens = extractSearchTokens(term);
-        if (tokens.length > 0) {
-          const tokenClauses = tokens.map(() => '(title LIKE ? OR content LIKE ? OR related_api LIKE ? OR related_symbol LIKE ?)');
-          sql += ` AND (${tokenClauses.join(' OR ')})`;
-          for (const t of tokens) {
-            const lp = `%${t}%`;
-            params.push(lp, lp, lp, lp);
-          }
-        } else {
-          return [];
-        }
-      }
-
-      sql += ' ORDER BY kind LIMIT ?';
-      params.push(limit);
-
-      const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
-      results = rows.map(r => this.hydratePitfall(r));
-    }
-
-    return results;
-  }
-
-  private hydratePitfall(row: Record<string, unknown>): Pitfall {
-    return {
-      id: String(row.id),
-      chunkId: row.chunk_id ? String(row.chunk_id) : undefined,
-      pageId: String(row.page_id),
-      snapshotId: String(row.snapshot_id),
-      kind: String(row.kind) as PitfallKind,
-      title: String(row.title),
-      content: String(row.content),
-      relatedApi: row.related_api ? String(row.related_api) : undefined,
-      relatedSymbol: row.related_symbol ? String(row.related_symbol) : undefined,
-      docVersion: row.doc_version ? String(row.doc_version) : undefined,
-      provenance: row.provenance_json ? JSON.parse(String(row.provenance_json)) : undefined,
-      createdAt: String(row.created_at),
-    };
-  }
-}

package/packages/storage/src/repositories/source-repository.ts

@@ -1,205 +0,0 @@
-import { computeSha256 } from '../../../shared/src/index.ts';
-import type { DiscoveredSource, NormalizedPage } from '../../../shared/src/index.ts';
-import type { DocOrbitDb } from '../db.ts';
-
-export interface SnapshotRecord {
-  id: string;
-  sourceId: string;
-  snapshotHash: string;
-  pageCount: number;
-  capturedAt: string;
-  docVersion?: string;
-  metadata?: Record<string, unknown>;
-}
-
-export class SourceRepository {
-  private db: DocOrbitDb;
-  private hydratePageFn?: (row: Record<string, unknown>) => NormalizedPage;
-
-  constructor(
-    db: DocOrbitDb,
-    hydratePageFn?: (row: Record<string, unknown>) => NormalizedPage
-  ) {
-    this.db = db;
-    this.hydratePageFn = hydratePageFn;
-  }
-
-  saveSource(source: DiscoveredSource): string {
-    const raw = this.db.getRawDb();
-    const id = `src_${computeSha256(source.url).slice(0, 16)}`;
-    const createdAt = new Date().toISOString();
-
-    const stmt = raw.prepare(`
-      INSERT INTO sources (
-        id, url, type, discovered_by, status, content_type,
-        confidence, authority, machine_readable, metadata, created_at
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      ON CONFLICT(url) DO UPDATE SET
-        type = excluded.type,
-        discovered_by = excluded.discovered_by,
-        status = excluded.status,
-        content_type = excluded.content_type,
-        confidence = excluded.confidence,
-        authority = excluded.authority,
-        machine_readable = excluded.machine_readable,
-        metadata = excluded.metadata
-    `);
-
-    stmt.run(
-      id,
-      source.url,
-      source.type,
-      source.discoveredBy || 'manual',
-      source.status,
-      source.contentType || null,
-      source.confidence,
-      source.authority,
-      source.machineReadable ? 1 : 0,
-      source.metadata ? JSON.stringify(source.metadata) : null,
-      createdAt
-    );
-
-    const row = raw.prepare('SELECT id FROM sources WHERE url = ?').get(source.url) as { id: string } | undefined;
-    return row?.id || id;
-  }
-
-  getSource(id: string): (DiscoveredSource & { id: string }) | null {
-    const raw = this.db.getRawDb();
-    const row = raw.prepare('SELECT * FROM sources WHERE id = ?').get(id) as Record<string, unknown> | undefined;
-    if (!row) return null;
-    return this.mapSourceRow(row);
-  }
-
-  getSourceByUrl(url: string): (DiscoveredSource & { id: string }) | null {
-    const raw = this.db.getRawDb();
-    const row = raw.prepare('SELECT * FROM sources WHERE url = ?').get(url) as Record<string, unknown> | undefined;
-    if (!row) return null;
-    return this.mapSourceRow(row);
-  }
-
-  listSources(): Array<DiscoveredSource & { id: string; createdAt: string }> {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT * FROM sources ORDER BY created_at DESC').all() as Array<Record<string, unknown>>;
-    return rows.map(r => ({
-      ...this.mapSourceRow(r),
-      createdAt: String(r.created_at),
-    }));
-  }
-
-  createSnapshot(sourceId: string, metadata?: Record<string, unknown>, docVersion?: string): string {
-    const raw = this.db.getRawDb();
-    const pages = raw.prepare('SELECT id, content_hash FROM pages WHERE source_id = ? ORDER BY id ASC').all(sourceId) as Array<{ id: string; content_hash: string }>;
-
-    const combinedHash = computeSha256(pages.map(p => p.content_hash).join(':'));
-    const snapshotId = `snap_${combinedHash.slice(0, 16)}`;
-    const now = new Date().toISOString();
-    const version = docVersion || (metadata?.docVersion as string) || null;
-
-    const stmt = raw.prepare(`
-      INSERT INTO snapshots (id, source_id, snapshot_hash, page_count, captured_at, doc_version, metadata)
-      VALUES (?, ?, ?, ?, ?, ?, ?)
-      ON CONFLICT(id) DO UPDATE SET
-        captured_at = excluded.captured_at,
-        page_count = excluded.page_count,
-        doc_version = COALESCE(excluded.doc_version, snapshots.doc_version),
-        metadata = excluded.metadata
-    `);
-
-    stmt.run(
-      snapshotId,
-      sourceId,
-      combinedHash,
-      pages.length,
-      now,
-      version,
-      metadata ? JSON.stringify(metadata) : null
-    );
-
-    // Link snapshot to page membership in snapshot_pages
-    const linkStmt = raw.prepare(`
-      INSERT OR REPLACE INTO snapshot_pages (snapshot_id, page_id, content_hash)
-      VALUES (?, ?, ?)
-    `);
-    for (const p of pages) {
-      linkStmt.run(snapshotId, p.id, p.content_hash);
-    }
-
-    return snapshotId;
-  }
-
-  linkSnapshotPages(snapshotId: string, pageIds: string[]): void {
-    const raw = this.db.getRawDb();
-    const linkStmt = raw.prepare(`
-      INSERT OR REPLACE INTO snapshot_pages (snapshot_id, page_id, content_hash)
-      VALUES (?, ?, COALESCE((SELECT content_hash FROM pages WHERE id = ?), 'hash_default'))
-    `);
-    for (const pageId of pageIds) {
-      linkStmt.run(snapshotId, pageId, pageId);
-    }
-  }
-
-  getSnapshot(snapshotId: string): SnapshotRecord | null {
-    const raw = this.db.getRawDb();
-    const row = raw.prepare('SELECT * FROM snapshots WHERE id = ?').get(snapshotId) as Record<string, unknown> | undefined;
-    if (!row) return null;
-    return {
-      id: String(row.id),
-      sourceId: String(row.source_id),
-      snapshotHash: String(row.snapshot_hash),
-      pageCount: Number(row.page_count),
-      capturedAt: String(row.captured_at),
-      docVersion: row.doc_version ? String(row.doc_version) : undefined,
-      metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined,
-    };
-  }
-
-  listSnapshots(sourceId?: string): SnapshotRecord[] {
-    const raw = this.db.getRawDb();
-    let sql = 'SELECT * FROM snapshots';
-    const params: unknown[] = [];
-    if (sourceId) {
-      sql += ' WHERE source_id = ?';
-      params.push(sourceId);
-    }
-    sql += ' ORDER BY captured_at DESC';
-    const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
-    return rows.map(row => ({
-      id: String(row.id),
-      sourceId: String(row.source_id),
-      snapshotHash: String(row.snapshot_hash),
-      pageCount: Number(row.page_count),
-      capturedAt: String(row.captured_at),
-      docVersion: row.doc_version ? String(row.doc_version) : undefined,
-      metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined,
-    }));
-  }
-
-  getSnapshotPages(snapshotId: string): NormalizedPage[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare(`
-      SELECT p.* FROM pages p
-      JOIN snapshot_pages sp ON p.id = sp.page_id
-      WHERE sp.snapshot_id = ?
-      ORDER BY p.id ASC
-    `).all(snapshotId) as Array<Record<string, unknown>>;
-    if (this.hydratePageFn) {
-      return rows.map(r => this.hydratePageFn!(r));
-    }
-    return [];
-  }
-
-  private mapSourceRow(row: Record<string, unknown>): DiscoveredSource & { id: string } {
-    return {
-      id: String(row.id),
-      url: String(row.url),
-      type: row.type as any,
-      discoveredBy: String(row.discovered_by),
-      status: row.status as any,
-      contentType: row.content_type ? String(row.content_type) : undefined,
-      confidence: Number(row.confidence),
-      authority: row.authority as any,
-      machineReadable: Number(row.machine_readable) === 1,
-      metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined,
-    };
-  }
-}