docorbit 0.1.0 → 0.1.2

package/dist/packages/storage/src/repositories/pitfall-repository.js

@@ -0,0 +1,144 @@
+import { extractSearchTokens } from "../search-tokens.js";
+export class PitfallRepository {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    savePitfalls(pitfalls) {
+        if (pitfalls.length === 0)
+            return;
+        const raw = this.db.getRawDb();
+        const insertPf = raw.prepare(`
+      INSERT INTO pitfalls (
+        id, chunk_id, page_id, snapshot_id, kind, title,
+        content, related_api, related_symbol, doc_version,
+        provenance_json, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        chunk_id = excluded.chunk_id,
+        kind = excluded.kind,
+        title = excluded.title,
+        content = excluded.content,
+        related_api = excluded.related_api,
+        related_symbol = excluded.related_symbol,
+        doc_version = excluded.doc_version,
+        provenance_json = excluded.provenance_json
+    `);
+        const insertFts = this.db.isFtsAvailable()
+            ? raw.prepare(`
+          INSERT INTO pitfalls_fts (pitfall_id, kind, title, content, related_api, related_symbol)
+          VALUES (?, ?, ?, ?, ?, ?)
+        `)
+            : null;
+        const deleteFts = this.db.isFtsAvailable()
+            ? raw.prepare('DELETE FROM pitfalls_fts WHERE pitfall_id = ?')
+            : null;
+        raw.exec('BEGIN TRANSACTION;');
+        try {
+            for (const pf of pitfalls) {
+                insertPf.run(pf.id, pf.chunkId || null, pf.pageId, pf.snapshotId || 'snap_default', pf.kind, pf.title, pf.content || pf.message || '', pf.relatedApi || null, pf.relatedSymbol || null, pf.docVersion || null, pf.provenance ? JSON.stringify(pf.provenance) : null, pf.createdAt || new Date().toISOString());
+                if (insertFts && deleteFts) {
+                    deleteFts.run(pf.id);
+                    insertFts.run(pf.id, pf.kind, pf.title, pf.content || pf.message || '', pf.relatedApi || '', pf.relatedSymbol || '');
+                }
+            }
+            raw.exec('COMMIT;');
+        }
+        catch (err) {
+            raw.exec('ROLLBACK;');
+            throw err;
+        }
+    }
+    getPitfallsBySnapshot(snapshotId) {
+        const raw = this.db.getRawDb();
+        const rows = raw.prepare('SELECT * FROM pitfalls WHERE snapshot_id = ?').all(snapshotId);
+        return rows.map(r => this.hydratePitfall(r));
+    }
+    searchPitfalls(query, options = {}) {
+        const raw = this.db.getRawDb();
+        const limit = options.limit || 10;
+        const term = query.trim();
+        let results = [];
+        // FTS attempt
+        if (this.db.isFtsAvailable() && term.length > 0) {
+            const cleanTokens = extractSearchTokens(term);
+            if (cleanTokens.length > 0) {
+                const sanitizedFts = cleanTokens
+                    .map(t => t.length >= 4 ? `"${t.replace(/"/g, '""')}"*` : `"${t.replace(/"/g, '""')}"`)
+                    .join(' OR ');
+                try {
+                    let sql = `
+            SELECT pf.*, fts.rank as fts_rank
+            FROM pitfalls pf
+            JOIN pitfalls_fts fts ON pf.id = fts.pitfall_id
+            WHERE pitfalls_fts MATCH ?
+          `;
+                    const params = [sanitizedFts];
+                    if (options.kind) {
+                        sql += ' AND pf.kind = ?';
+                        params.push(options.kind);
+                    }
+                    if (options.docVersion) {
+                        sql += ' AND (pf.doc_version = ? OR pf.doc_version IS NULL)';
+                        params.push(options.docVersion);
+                    }
+                    sql += ' ORDER BY fts.rank LIMIT ?';
+                    params.push(limit);
+                    const rows = raw.prepare(sql).all(...params);
+                    results = rows.map(r => this.hydratePitfall(r));
+                }
+                catch {
+                    // Fall through to LIKE
+                }
+            }
+        }
+        if (results.length === 0) {
+            // LIKE fallback
+            let sql = 'SELECT * FROM pitfalls WHERE 1=1';
+            const params = [];
+            if (options.kind) {
+                sql += ' AND kind = ?';
+                params.push(options.kind);
+            }
+            if (options.docVersion) {
+                sql += ' AND (doc_version = ? OR doc_version IS NULL)';
+                params.push(options.docVersion);
+            }
+            if (term.length > 0) {
+                const tokens = extractSearchTokens(term);
+                if (tokens.length > 0) {
+                    const tokenClauses = tokens.map(() => '(title LIKE ? OR content LIKE ? OR related_api LIKE ? OR related_symbol LIKE ?)');
+                    sql += ` AND (${tokenClauses.join(' OR ')})`;
+                    for (const t of tokens) {
+                        const lp = `%${t}%`;
+                        params.push(lp, lp, lp, lp);
+                    }
+                }
+                else {
+                    return [];
+                }
+            }
+            sql += ' ORDER BY kind LIMIT ?';
+            params.push(limit);
+            const rows = raw.prepare(sql).all(...params);
+            results = rows.map(r => this.hydratePitfall(r));
+        }
+        return results;
+    }
+    hydratePitfall(row) {
+        return {
+            id: String(row.id),
+            chunkId: row.chunk_id ? String(row.chunk_id) : undefined,
+            pageId: String(row.page_id),
+            snapshotId: String(row.snapshot_id),
+            kind: String(row.kind),
+            title: String(row.title),
+            content: String(row.content),
+            relatedApi: row.related_api ? String(row.related_api) : undefined,
+            relatedSymbol: row.related_symbol ? String(row.related_symbol) : undefined,
+            docVersion: row.doc_version ? String(row.doc_version) : undefined,
+            provenance: row.provenance_json ? JSON.parse(String(row.provenance_json)) : undefined,
+            createdAt: String(row.created_at),
+        };
+    }
+}

package/dist/packages/storage/src/repositories/source-repository.d.ts

@@ -0,0 +1,33 @@
+import type { DiscoveredSource, NormalizedPage } from '../../../shared/src/index.ts';
+import type { DocOrbitDb } from '../db.ts';
+export interface SnapshotRecord {
+    id: string;
+    sourceId: string;
+    snapshotHash: string;
+    pageCount: number;
+    capturedAt: string;
+    docVersion?: string;
+    metadata?: Record<string, unknown>;
+}
+export declare class SourceRepository {
+    private db;
+    private hydratePageFn?;
+    constructor(db: DocOrbitDb, hydratePageFn?: (row: Record<string, unknown>) => NormalizedPage);
+    saveSource(source: DiscoveredSource): string;
+    getSource(id: string): (DiscoveredSource & {
+        id: string;
+    }) | null;
+    getSourceByUrl(url: string): (DiscoveredSource & {
+        id: string;
+    }) | null;
+    listSources(): Array<DiscoveredSource & {
+        id: string;
+        createdAt: string;
+    }>;
+    createSnapshot(sourceId: string, metadata?: Record<string, unknown>, docVersion?: string): string;
+    linkSnapshotPages(snapshotId: string, pageIds: string[]): void;
+    getSnapshot(snapshotId: string): SnapshotRecord | null;
+    listSnapshots(sourceId?: string): SnapshotRecord[];
+    getSnapshotPages(snapshotId: string): NormalizedPage[];
+    private mapSourceRow;
+}

package/dist/packages/storage/src/repositories/source-repository.js

@@ -0,0 +1,153 @@
+import { computeSha256 } from "../../../shared/src/index.js";
+export class SourceRepository {
+    db;
+    hydratePageFn;
+    constructor(db, hydratePageFn) {
+        this.db = db;
+        this.hydratePageFn = hydratePageFn;
+    }
+    saveSource(source) {
+        const raw = this.db.getRawDb();
+        const id = `src_${computeSha256(source.url).slice(0, 16)}`;
+        const createdAt = new Date().toISOString();
+        const stmt = raw.prepare(`
+      INSERT INTO sources (
+        id, url, type, discovered_by, status, content_type,
+        confidence, authority, machine_readable, metadata, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(url) DO UPDATE SET
+        type = excluded.type,
+        discovered_by = excluded.discovered_by,
+        status = excluded.status,
+        content_type = excluded.content_type,
+        confidence = excluded.confidence,
+        authority = excluded.authority,
+        machine_readable = excluded.machine_readable,
+        metadata = excluded.metadata
+    `);
+        stmt.run(id, source.url, source.type, source.discoveredBy || 'manual', source.status, source.contentType || null, source.confidence, source.authority, source.machineReadable ? 1 : 0, source.metadata ? JSON.stringify(source.metadata) : null, createdAt);
+        const row = raw.prepare('SELECT id FROM sources WHERE url = ?').get(source.url);
+        return row?.id || id;
+    }
+    getSource(id) {
+        const raw = this.db.getRawDb();
+        const row = raw.prepare('SELECT * FROM sources WHERE id = ?').get(id);
+        if (!row)
+            return null;
+        return this.mapSourceRow(row);
+    }
+    getSourceByUrl(url) {
+        const raw = this.db.getRawDb();
+        const row = raw.prepare('SELECT * FROM sources WHERE url = ?').get(url);
+        if (!row)
+            return null;
+        return this.mapSourceRow(row);
+    }
+    listSources() {
+        const raw = this.db.getRawDb();
+        const rows = raw.prepare('SELECT * FROM sources ORDER BY created_at DESC').all();
+        return rows.map(r => ({
+            ...this.mapSourceRow(r),
+            createdAt: String(r.created_at),
+        }));
+    }
+    createSnapshot(sourceId, metadata, docVersion) {
+        const raw = this.db.getRawDb();
+        const pages = raw.prepare('SELECT id, content_hash FROM pages WHERE source_id = ? ORDER BY id ASC').all(sourceId);
+        const combinedHash = computeSha256(pages.map(p => p.content_hash).join(':'));
+        const snapshotId = `snap_${combinedHash.slice(0, 16)}`;
+        const now = new Date().toISOString();
+        const version = docVersion || metadata?.docVersion || null;
+        const stmt = raw.prepare(`
+      INSERT INTO snapshots (id, source_id, snapshot_hash, page_count, captured_at, doc_version, metadata)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        captured_at = excluded.captured_at,
+        page_count = excluded.page_count,
+        doc_version = COALESCE(excluded.doc_version, snapshots.doc_version),
+        metadata = excluded.metadata
+    `);
+        stmt.run(snapshotId, sourceId, combinedHash, pages.length, now, version, metadata ? JSON.stringify(metadata) : null);
+        // Link snapshot to page membership in snapshot_pages
+        const linkStmt = raw.prepare(`
+      INSERT OR REPLACE INTO snapshot_pages (snapshot_id, page_id, content_hash)
+      VALUES (?, ?, ?)
+    `);
+        for (const p of pages) {
+            linkStmt.run(snapshotId, p.id, p.content_hash);
+        }
+        return snapshotId;
+    }
+    linkSnapshotPages(snapshotId, pageIds) {
+        const raw = this.db.getRawDb();
+        const linkStmt = raw.prepare(`
+      INSERT OR REPLACE INTO snapshot_pages (snapshot_id, page_id, content_hash)
+      VALUES (?, ?, COALESCE((SELECT content_hash FROM pages WHERE id = ?), 'hash_default'))
+    `);
+        for (const pageId of pageIds) {
+            linkStmt.run(snapshotId, pageId, pageId);
+        }
+    }
+    getSnapshot(snapshotId) {
+        const raw = this.db.getRawDb();
+        const row = raw.prepare('SELECT * FROM snapshots WHERE id = ?').get(snapshotId);
+        if (!row)
+            return null;
+        return {
+            id: String(row.id),
+            sourceId: String(row.source_id),
+            snapshotHash: String(row.snapshot_hash),
+            pageCount: Number(row.page_count),
+            capturedAt: String(row.captured_at),
+            docVersion: row.doc_version ? String(row.doc_version) : undefined,
+            metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined,
+        };
+    }
+    listSnapshots(sourceId) {
+        const raw = this.db.getRawDb();
+        let sql = 'SELECT * FROM snapshots';
+        const params = [];
+        if (sourceId) {
+            sql += ' WHERE source_id = ?';
+            params.push(sourceId);
+        }
+        sql += ' ORDER BY captured_at DESC';
+        const rows = raw.prepare(sql).all(...params);
+        return rows.map(row => ({
+            id: String(row.id),
+            sourceId: String(row.source_id),
+            snapshotHash: String(row.snapshot_hash),
+            pageCount: Number(row.page_count),
+            capturedAt: String(row.captured_at),
+            docVersion: row.doc_version ? String(row.doc_version) : undefined,
+            metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined,
+        }));
+    }
+    getSnapshotPages(snapshotId) {
+        const raw = this.db.getRawDb();
+        const rows = raw.prepare(`
+      SELECT p.* FROM pages p
+      JOIN snapshot_pages sp ON p.id = sp.page_id
+      WHERE sp.snapshot_id = ?
+      ORDER BY p.id ASC
+    `).all(snapshotId);
+        if (this.hydratePageFn) {
+            return rows.map(r => this.hydratePageFn(r));
+        }
+        return [];
+    }
+    mapSourceRow(row) {
+        return {
+            id: String(row.id),
+            url: String(row.url),
+            type: row.type,
+            discoveredBy: String(row.discovered_by),
+            status: row.status,
+            contentType: row.content_type ? String(row.content_type) : undefined,
+            confidence: Number(row.confidence),
+            authority: row.authority,
+            machineReadable: Number(row.machine_readable) === 1,
+            metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined,
+        };
+    }
+}

package/dist/packages/storage/src/repository.d.ts

@@ -0,0 +1,92 @@
+import type { DiscoveredSource, NormalizedPage, DocumentChunk, ChunkCode, SymbolReference, ChunkRelationship, ChunkType, ApiEndpoint, IndexedExample, Pitfall, PitfallKind } from '../../shared/src/index.ts';
+import type { DocOrbitDb } from './db.ts';
+import type { Snapshot, ISourceRepository, IPageRepository, IChunkRepository, IApiRepository, IExampleRepository, IPitfallRepository } from './interfaces.ts';
+import { SourceRepository } from './repositories/source-repository.ts';
+import { PageRepository } from './repositories/page-repository.ts';
+import { ChunkRepository } from './repositories/chunk-repository.ts';
+import { ApiRepository } from './repositories/api-repository.ts';
+import { ExampleRepository } from './repositories/example-repository.ts';
+import { PitfallRepository } from './repositories/pitfall-repository.ts';
+export type { Snapshot } from './interfaces.ts';
+/**
+ * Unified Facade for the DocOrbit database storage layer.
+ * Delegates domain responsibilities to specialized sub-repositories:
+ * - sources: ISourceRepository (sources, snapshots, snapshot-page links)
+ * - pages: IPageRepository (normalized pages, page links, page FTS)
+ * - chunks: IChunkRepository (document chunks, code, symbols, relationships, chunk FTS)
+ * - apis: IApiRepository (API endpoints, OpenAPI schemas, endpoint FTS)
+ * - examples: IExampleRepository (code examples, example FTS)
+ * - pitfalls: IPitfallRepository (pitfalls, gotchas, pitfall FTS)
+ */
+export declare class DocOrbitRepository implements ISourceRepository, IPageRepository, IChunkRepository, IApiRepository, IExampleRepository, IPitfallRepository {
+    readonly sources: SourceRepository;
+    readonly pages: PageRepository;
+    readonly chunks: ChunkRepository;
+    readonly apis: ApiRepository;
+    readonly examples: ExampleRepository;
+    readonly pitfalls: PitfallRepository;
+    readonly db: DocOrbitDb;
+    constructor(db: DocOrbitDb);
+    saveSource(source: DiscoveredSource): string;
+    getSource(id: string): DiscoveredSource | null;
+    getSourceByUrl(url: string): DiscoveredSource | null;
+    listSources(): DiscoveredSource[];
+    createSnapshot(sourceId: string, metadata?: Record<string, unknown>, docVersion?: string): string;
+    saveSnapshot(snapshot: Snapshot): void;
+    getSnapshot(id: string): Snapshot | null;
+    getLatestSnapshot(sourceId: string): Snapshot | null;
+    listSnapshots(sourceId: string): Snapshot[];
+    linkSnapshotPages(snapshotId: string, pageIds: string[]): void;
+    getSnapshotPages(snapshotId: string): NormalizedPage[];
+    savePage(page: NormalizedPage): void;
+    getPage(id: string): NormalizedPage | null;
+    getPageByUrl(normalizedUrl: string): NormalizedPage | null;
+    listPages(limit?: number, offset?: number): NormalizedPage[];
+    countPages(sourceId?: string): number;
+    savePageLinks(pageId: string, targetUrls: string[]): void;
+    getPageLinks(pageId: string): string[];
+    searchPagesFts(query: string, limit?: number): NormalizedPage[];
+    saveChunks(chunks: DocumentChunk[], relationships?: ChunkRelationship[], codeSnippets?: ChunkCode[], symbols?: SymbolReference[]): void;
+    getChunk(id: string): DocumentChunk | null;
+    getChunksByPage(pageId: string): DocumentChunk[];
+    getChunksBySnapshot(snapshotId: string): DocumentChunk[];
+    getChunksByVersion(docVersion: string): DocumentChunk[];
+    getChunkCode(chunkId: string): ChunkCode[];
+    getChunkSymbols(chunkId: string): SymbolReference[];
+    getChunkRelationships(chunkId: string): ChunkRelationship[];
+    countChunks(snapshotId?: string): number;
+    searchChunksFts(query: string, options?: {
+        limit?: number;
+        snapshotId?: string;
+        chunkType?: ChunkType;
+        docVersion?: string;
+    }): Array<{
+        chunk: DocumentChunk;
+        ftsRank: number;
+        symbols: SymbolReference[];
+        codeSnippets: ChunkCode[];
+    }>;
+    saveApiEndpoints(endpoints: ApiEndpoint[]): void;
+    getApiEndpoint(id: string): ApiEndpoint | null;
+    getApiEndpointsBySnapshot(snapshotId: string): ApiEndpoint[];
+    searchApiEndpoints(query: string, options?: {
+        method?: string;
+        docVersion?: string;
+        limit?: number;
+    }): ApiEndpoint[];
+    saveIndexedExamples(examples: IndexedExample[]): void;
+    getIndexedExamplesBySnapshot(snapshotId: string): IndexedExample[];
+    searchIndexedExamples(query: string, options?: {
+        language?: string;
+        framework?: string;
+        docVersion?: string;
+        limit?: number;
+    }): IndexedExample[];
+    savePitfalls(pitfalls: Pitfall[]): void;
+    getPitfallsBySnapshot(snapshotId: string): Pitfall[];
+    searchPitfalls(query: string, options?: {
+        kind?: PitfallKind;
+        docVersion?: string;
+        limit?: number;
+    }): Pitfall[];
+}

package/dist/packages/storage/src/repository.js

@@ -0,0 +1,157 @@
+import { SourceRepository } from "./repositories/source-repository.js";
+import { PageRepository } from "./repositories/page-repository.js";
+import { ChunkRepository } from "./repositories/chunk-repository.js";
+import { ApiRepository } from "./repositories/api-repository.js";
+import { ExampleRepository } from "./repositories/example-repository.js";
+import { PitfallRepository } from "./repositories/pitfall-repository.js";
+/**
+ * Unified Facade for the DocOrbit database storage layer.
+ * Delegates domain responsibilities to specialized sub-repositories:
+ * - sources: ISourceRepository (sources, snapshots, snapshot-page links)
+ * - pages: IPageRepository (normalized pages, page links, page FTS)
+ * - chunks: IChunkRepository (document chunks, code, symbols, relationships, chunk FTS)
+ * - apis: IApiRepository (API endpoints, OpenAPI schemas, endpoint FTS)
+ * - examples: IExampleRepository (code examples, example FTS)
+ * - pitfalls: IPitfallRepository (pitfalls, gotchas, pitfall FTS)
+ */
+export class DocOrbitRepository {
+    sources;
+    pages;
+    chunks;
+    apis;
+    examples;
+    pitfalls;
+    db;
+    constructor(db) {
+        this.db = db;
+        this.pages = new PageRepository(db);
+        this.sources = new SourceRepository(db, row => this.pages.hydratePage(row));
+        this.chunks = new ChunkRepository(db);
+        this.apis = new ApiRepository(db);
+        this.examples = new ExampleRepository(db);
+        this.pitfalls = new PitfallRepository(db);
+    }
+    // --- ISourceRepository Delegations ---
+    saveSource(source) {
+        return this.sources.saveSource(source);
+    }
+    getSource(id) {
+        return this.sources.getSource(id);
+    }
+    getSourceByUrl(url) {
+        return this.sources.getSourceByUrl(url);
+    }
+    listSources() {
+        return this.sources.listSources();
+    }
+    createSnapshot(sourceId, metadata, docVersion) {
+        return this.sources.createSnapshot(sourceId, metadata, docVersion);
+    }
+    saveSnapshot(snapshot) {
+        this.sources.saveSnapshot(snapshot);
+    }
+    getSnapshot(id) {
+        return this.sources.getSnapshot(id);
+    }
+    getLatestSnapshot(sourceId) {
+        return this.sources.getLatestSnapshot(sourceId);
+    }
+    listSnapshots(sourceId) {
+        return this.sources.listSnapshots(sourceId);
+    }
+    linkSnapshotPages(snapshotId, pageIds) {
+        this.sources.linkSnapshotPages(snapshotId, pageIds);
+    }
+    getSnapshotPages(snapshotId) {
+        return this.sources.getSnapshotPages(snapshotId);
+    }
+    // --- IPageRepository Delegations ---
+    savePage(page) {
+        this.pages.savePage(page);
+    }
+    getPage(id) {
+        return this.pages.getPage(id);
+    }
+    getPageByUrl(normalizedUrl) {
+        return this.pages.getPageByUrl(normalizedUrl);
+    }
+    listPages(limit, offset) {
+        return this.pages.listPages(limit, offset);
+    }
+    countPages(sourceId) {
+        return this.pages.countPages(sourceId);
+    }
+    savePageLinks(pageId, targetUrls) {
+        this.pages.savePageLinks(pageId, targetUrls);
+    }
+    getPageLinks(pageId) {
+        return this.pages.getPageLinks(pageId);
+    }
+    searchPagesFts(query, limit = 10) {
+        return this.pages.searchPagesFts(query, limit);
+    }
+    // --- IChunkRepository Delegations ---
+    saveChunks(chunks, relationships = [], codeSnippets = [], symbols = []) {
+        this.chunks.saveChunks(chunks, relationships, codeSnippets, symbols);
+    }
+    getChunk(id) {
+        return this.chunks.getChunk(id);
+    }
+    getChunksByPage(pageId) {
+        return this.chunks.getChunksByPage(pageId);
+    }
+    getChunksBySnapshot(snapshotId) {
+        return this.chunks.getChunksBySnapshot(snapshotId);
+    }
+    getChunksByVersion(docVersion) {
+        return this.chunks.getChunksByVersion(docVersion);
+    }
+    getChunkCode(chunkId) {
+        return this.chunks.getChunkCode(chunkId);
+    }
+    getChunkSymbols(chunkId) {
+        return this.chunks.getChunkSymbols(chunkId);
+    }
+    getChunkRelationships(chunkId) {
+        return this.chunks.getChunkRelationships(chunkId);
+    }
+    countChunks(snapshotId) {
+        return this.chunks.countChunks(snapshotId);
+    }
+    searchChunksFts(query, options = {}) {
+        return this.chunks.searchChunksFts(query, options);
+    }
+    // --- IApiRepository Delegations ---
+    saveApiEndpoints(endpoints) {
+        this.apis.saveApiEndpoints(endpoints);
+    }
+    getApiEndpoint(id) {
+        return this.apis.getApiEndpoint(id);
+    }
+    getApiEndpointsBySnapshot(snapshotId) {
+        return this.apis.getApiEndpointsBySnapshot(snapshotId);
+    }
+    searchApiEndpoints(query, options = {}) {
+        return this.apis.searchApiEndpoints(query, options);
+    }
+    // --- IExampleRepository Delegations ---
+    saveIndexedExamples(examples) {
+        this.examples.saveIndexedExamples(examples);
+    }
+    getIndexedExamplesBySnapshot(snapshotId) {
+        return this.examples.getIndexedExamplesBySnapshot(snapshotId);
+    }
+    searchIndexedExamples(query, options = {}) {
+        return this.examples.searchIndexedExamples(query, options);
+    }
+    // --- IPitfallRepository Delegations ---
+    savePitfalls(pitfalls) {
+        this.pitfalls.savePitfalls(pitfalls);
+    }
+    getPitfallsBySnapshot(snapshotId) {
+        return this.pitfalls.getPitfallsBySnapshot(snapshotId);
+    }
+    searchPitfalls(query, options = {}) {
+        return this.pitfalls.searchPitfalls(query, options);
+    }
+}

package/dist/packages/storage/src/schema.d.ts

@@ -0,0 +1,2 @@
+export declare const SCHEMA_SQL = "\nCREATE TABLE IF NOT EXISTS sources (\n  id TEXT PRIMARY KEY,\n  url TEXT UNIQUE NOT NULL,\n  type TEXT NOT NULL,\n  discovered_by TEXT NOT NULL,\n  status TEXT NOT NULL,\n  content_type TEXT,\n  confidence REAL NOT NULL,\n  authority TEXT NOT NULL,\n  machine_readable INTEGER NOT NULL,\n  metadata TEXT,\n  created_at TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS pages (\n  id TEXT PRIMARY KEY,\n  source_id TEXT NOT NULL,\n  title TEXT NOT NULL,\n  url TEXT UNIQUE NOT NULL,\n  content TEXT NOT NULL,\n  content_hash TEXT NOT NULL,\n  fetched_at TEXT NOT NULL,\n  raw_bytes INTEGER NOT NULL,\n  estimated_tokens INTEGER NOT NULL,\n  headings_json TEXT,\n  security_annotations_json TEXT,\n  provenance_json TEXT,\n  FOREIGN KEY (source_id) REFERENCES sources(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS links (\n  id TEXT PRIMARY KEY,\n  page_id TEXT NOT NULL,\n  text TEXT NOT NULL,\n  url TEXT NOT NULL,\n  is_external INTEGER NOT NULL,\n  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS code_examples (\n  id TEXT PRIMARY KEY,\n  page_id TEXT NOT NULL,\n  language TEXT NOT NULL,\n  code TEXT NOT NULL,\n  caption TEXT,\n  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS snapshots (\n  id TEXT PRIMARY KEY,\n  source_id TEXT NOT NULL,\n  snapshot_hash TEXT NOT NULL,\n  page_count INTEGER NOT NULL,\n  captured_at TEXT NOT NULL,\n  doc_version TEXT,\n  metadata TEXT,\n  FOREIGN KEY (source_id) REFERENCES sources(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS snapshot_pages (\n  snapshot_id TEXT NOT NULL,\n  page_id TEXT NOT NULL,\n  content_hash TEXT NOT NULL,\n  PRIMARY KEY (snapshot_id, page_id),\n  FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,\n  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS chunks (\n  id TEXT PRIMARY KEY,\n  page_id TEXT NOT NULL,\n  snapshot_id TEXT NOT NULL,\n  title TEXT,\n  section_path_json TEXT NOT NULL,\n  content TEXT NOT NULL,\n  chunk_type TEXT NOT NULL,\n  language TEXT,\n  token_estimate INTEGER NOT NULL,\n  ordinal INTEGER NOT NULL,\n  content_hash TEXT NOT NULL,\n  doc_version TEXT,\n  provenance_json TEXT,\n  created_at TEXT NOT NULL,\n  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS chunk_code (\n  id TEXT PRIMARY KEY,\n  chunk_id TEXT NOT NULL,\n  language TEXT,\n  code TEXT NOT NULL,\n  FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS symbol_references (\n  id TEXT PRIMARY KEY,\n  chunk_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  kind TEXT NOT NULL,\n  FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS chunk_relationships (\n  source_chunk_id TEXT NOT NULL,\n  target_chunk_id TEXT NOT NULL,\n  type TEXT NOT NULL,\n  PRIMARY KEY (source_chunk_id, target_chunk_id, type)\n);\n\nCREATE INDEX IF NOT EXISTS idx_pages_source_id ON pages(source_id);\nCREATE INDEX IF NOT EXISTS idx_pages_content_hash ON pages(content_hash);\nCREATE INDEX IF NOT EXISTS idx_links_page_id ON links(page_id);\nCREATE INDEX IF NOT EXISTS idx_code_examples_page_id ON code_examples(page_id);\nCREATE INDEX IF NOT EXISTS idx_snapshots_source_id ON snapshots(source_id);\nCREATE INDEX IF NOT EXISTS idx_snapshots_doc_version ON snapshots(doc_version);\nCREATE INDEX IF NOT EXISTS idx_snapshot_pages_snapshot_id ON snapshot_pages(snapshot_id);\nCREATE INDEX IF NOT EXISTS idx_snapshot_pages_page_id ON snapshot_pages(page_id);\nCREATE INDEX IF NOT EXISTS idx_chunks_page_id ON chunks(page_id);\nCREATE INDEX IF NOT EXISTS idx_chunks_snapshot_id ON chunks(snapshot_id);\nCREATE INDEX IF NOT EXISTS idx_chunks_chunk_type ON chunks(chunk_type);\nCREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);\nCREATE INDEX IF NOT EXISTS idx_chunks_doc_version ON chunks(doc_version);\nCREATE INDEX IF NOT EXISTS idx_chunk_code_chunk_id ON chunk_code(chunk_id);\nCREATE INDEX IF NOT EXISTS idx_symbol_references_chunk_id ON symbol_references(chunk_id);\nCREATE INDEX IF NOT EXISTS idx_symbol_references_name ON symbol_references(name);\nCREATE INDEX IF NOT EXISTS idx_chunk_relationships_source ON chunk_relationships(source_chunk_id);\nCREATE INDEX IF NOT EXISTS idx_chunk_relationships_target ON chunk_relationships(target_chunk_id);\n\nCREATE TABLE IF NOT EXISTS api_endpoints (\n  id TEXT PRIMARY KEY,\n  page_id TEXT NOT NULL,\n  snapshot_id TEXT NOT NULL,\n  method TEXT NOT NULL,\n  path TEXT NOT NULL,\n  summary TEXT,\n  description TEXT,\n  operation_id TEXT,\n  parameters_json TEXT NOT NULL,\n  request_schema_json TEXT,\n  response_schema_json TEXT,\n  auth_json TEXT NOT NULL,\n  errors_json TEXT NOT NULL,\n  pagination_json TEXT,\n  deprecated INTEGER NOT NULL,\n  doc_version TEXT,\n  provenance_json TEXT,\n  created_at TEXT NOT NULL,\n  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_api_endpoints_page_id ON api_endpoints(page_id);\nCREATE INDEX IF NOT EXISTS idx_api_endpoints_snapshot_id ON api_endpoints(snapshot_id);\nCREATE INDEX IF NOT EXISTS idx_api_endpoints_method ON api_endpoints(method);\nCREATE INDEX IF NOT EXISTS idx_api_endpoints_path ON api_endpoints(path);\nCREATE INDEX IF NOT EXISTS idx_api_endpoints_doc_version ON api_endpoints(doc_version);\n\nCREATE TABLE IF NOT EXISTS indexed_examples (\n  id TEXT PRIMARY KEY,\n  chunk_id TEXT,\n  page_id TEXT NOT NULL,\n  snapshot_id TEXT NOT NULL,\n  language TEXT NOT NULL,\n  framework TEXT,\n  task TEXT NOT NULL,\n  code TEXT NOT NULL,\n  source_url TEXT NOT NULL,\n  source_authority TEXT NOT NULL,\n  related_api TEXT,\n  related_symbol TEXT,\n  doc_version TEXT,\n  provenance_json TEXT,\n  created_at TEXT NOT NULL,\n  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_indexed_examples_page_id ON indexed_examples(page_id);\nCREATE INDEX IF NOT EXISTS idx_indexed_examples_snapshot_id ON indexed_examples(snapshot_id);\nCREATE INDEX IF NOT EXISTS idx_indexed_examples_chunk_id ON indexed_examples(chunk_id);\nCREATE INDEX IF NOT EXISTS idx_indexed_examples_language ON indexed_examples(language);\nCREATE INDEX IF NOT EXISTS idx_indexed_examples_framework ON indexed_examples(framework);\nCREATE INDEX IF NOT EXISTS idx_indexed_examples_doc_version ON indexed_examples(doc_version);\nCREATE INDEX IF NOT EXISTS idx_indexed_examples_related_api ON indexed_examples(related_api);\n\nCREATE TABLE IF NOT EXISTS pitfalls (\n  id TEXT PRIMARY KEY,\n  chunk_id TEXT,\n  page_id TEXT NOT NULL,\n  snapshot_id TEXT NOT NULL,\n  kind TEXT NOT NULL,\n  title TEXT NOT NULL,\n  content TEXT NOT NULL,\n  related_api TEXT,\n  related_symbol TEXT,\n  doc_version TEXT,\n  provenance_json TEXT,\n  created_at TEXT NOT NULL,\n  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_pitfalls_page_id ON pitfalls(page_id);\nCREATE INDEX IF NOT EXISTS idx_pitfalls_snapshot_id ON pitfalls(snapshot_id);\nCREATE INDEX IF NOT EXISTS idx_pitfalls_chunk_id ON pitfalls(chunk_id);\nCREATE INDEX IF NOT EXISTS idx_pitfalls_kind ON pitfalls(kind);\nCREATE INDEX IF NOT EXISTS idx_pitfalls_doc_version ON pitfalls(doc_version);\nCREATE INDEX IF NOT EXISTS idx_pitfalls_related_api ON pitfalls(related_api);\n";
+export declare const FTS_SCHEMA_SQL = "\nCREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(\n  page_id UNINDEXED,\n  title,\n  content\n);\n\nCREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(\n  chunk_id UNINDEXED,\n  title,\n  section_path,\n  content,\n  symbols\n);\n\nCREATE TRIGGER IF NOT EXISTS trg_chunks_delete AFTER DELETE ON chunks BEGIN\n  DELETE FROM chunks_fts WHERE chunk_id = old.id;\nEND;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS api_endpoints_fts USING fts5(\n  endpoint_id UNINDEXED,\n  method,\n  path,\n  summary,\n  description,\n  parameters\n);\n\nCREATE TRIGGER IF NOT EXISTS trg_api_endpoints_delete AFTER DELETE ON api_endpoints BEGIN\n  DELETE FROM api_endpoints_fts WHERE endpoint_id = old.id;\nEND;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS indexed_examples_fts USING fts5(\n  example_id UNINDEXED,\n  task,\n  language,\n  framework,\n  code,\n  related_api,\n  related_symbol\n);\n\nCREATE TRIGGER IF NOT EXISTS trg_indexed_examples_delete AFTER DELETE ON indexed_examples BEGIN\n  DELETE FROM indexed_examples_fts WHERE example_id = old.id;\nEND;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS pitfalls_fts USING fts5(\n  pitfall_id UNINDEXED,\n  kind,\n  title,\n  content,\n  related_api,\n  related_symbol\n);\n\nCREATE TRIGGER IF NOT EXISTS trg_pitfalls_delete AFTER DELETE ON pitfalls BEGIN\n  DELETE FROM pitfalls_fts WHERE pitfall_id = old.id;\nEND;\n";

package/{packages/storage/src/schema.ts → dist/packages/storage/src/schema.js}

@@ -205,7 +205,6 @@ CREATE INDEX IF NOT EXISTS idx_pitfalls_kind ON pitfalls(kind);
 CREATE INDEX IF NOT EXISTS idx_pitfalls_doc_version ON pitfalls(doc_version);
 CREATE INDEX IF NOT EXISTS idx_pitfalls_related_api ON pitfalls(related_api);
 `;
-
 export const FTS_SCHEMA_SQL = `
 CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
   page_id UNINDEXED,
@@ -265,5 +264,3 @@ CREATE TRIGGER IF NOT EXISTS trg_pitfalls_delete AFTER DELETE ON pitfalls BEGIN
   DELETE FROM pitfalls_fts WHERE pitfall_id = old.id;
 END;
 `;
-
-

package/dist/packages/storage/src/search-tokens.d.ts

@@ -0,0 +1,6 @@
+export declare const SEARCH_STOPWORDS: Set<string>;
+/**
+ * Extracts substantive tokens from query text, filtering out common stopwords
+ * and short words (< 3 chars), while falling back to short words if query has no substantive words.
+ */
+export declare function extractSearchTokens(text: string): string[];

package/dist/packages/storage/src/search-tokens.js

@@ -0,0 +1,26 @@
+export const SEARCH_STOPWORDS = new Set([
+    'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are',
+    'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by',
+    'can', 'did', 'do', 'does', 'doing', 'down', 'during', 'each', 'few', 'for', 'from',
+    'further', 'had', 'has', 'have', 'having', 'he', 'her', 'here', 'hers', 'herself', 'him',
+    'himself', 'his', 'how', 'i', 'if', 'in', 'into', 'is', 'it', 'its', 'itself', 'just',
+    'me', 'more', 'most', 'my', 'myself', 'no', 'nor', 'not', 'now', 'of', 'off', 'on',
+    'once', 'only', 'or', 'other', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'same',
+    'she', 'should', 'so', 'some', 'such', 'than', 'that', 'the', 'their', 'theirs', 'them',
+    'themselves', 'then', 'there', 'these', 'they', 'this', 'those', 'through', 'to', 'too',
+    'under', 'until', 'up', 'very', 'was', 'we', 'were', 'what', 'when', 'where', 'which',
+    'while', 'who', 'whom', 'why', 'with', 'you', 'your', 'yours', 'yourself', 'yourselves'
+]);
+/**
+ * Extracts substantive tokens from query text, filtering out common stopwords
+ * and short words (< 3 chars), while falling back to short words if query has no substantive words.
+ */
+export function extractSearchTokens(text) {
+    const words = text
+        .replace(/[^\w\s-]/g, ' ')
+        .trim()
+        .split(/\s+/)
+        .filter(w => w.length > 0);
+    const substantive = words.filter(w => w.length >= 3 && !SEARCH_STOPWORDS.has(w.toLowerCase()));
+    return substantive.length > 0 ? substantive : words.filter(w => w.length >= 2);
+}

package/dist/packages/verification/src/diff-engine.d.ts

@@ -0,0 +1,18 @@
+import type { DocDiffResult } from '../../shared/src/index.ts';
+import type { DocOrbitRepository } from '../../storage/src/index.ts';
+export interface DiffOptions {
+    fromVersion?: string;
+    toVersion?: string;
+    fromSnapshotId?: string;
+    toSnapshotId?: string;
+    sourceId?: string;
+}
+export declare class DocDiffEngine {
+    private repo;
+    constructor(repo: DocOrbitRepository);
+    diff(options: DiffOptions): DocDiffResult;
+    private getEndpoints;
+    private getPitfalls;
+    private detectEndpointModifications;
+    private computeContentDiffs;
+}