docorbit 0.1.0

package/packages/storage/src/repository.ts

@@ -0,0 +1,256 @@
+import type {
+  DiscoveredSource,
+  NormalizedPage,
+  DocumentChunk,
+  ChunkCode,
+  SymbolReference,
+  ChunkRelationship,
+  ChunkType,
+  ApiEndpoint,
+  IndexedExample,
+  Pitfall,
+  PitfallKind,
+} from '../../shared/src/index.ts';
+import type { DocOrbitDb } from './db.ts';
+import type {
+  Snapshot,
+  ISourceRepository,
+  IPageRepository,
+  IChunkRepository,
+  IApiRepository,
+  IExampleRepository,
+  IPitfallRepository,
+} from './interfaces.ts';
+import { SourceRepository } from './repositories/source-repository.ts';
+import { PageRepository } from './repositories/page-repository.ts';
+import { ChunkRepository } from './repositories/chunk-repository.ts';
+import { ApiRepository } from './repositories/api-repository.ts';
+import { ExampleRepository } from './repositories/example-repository.ts';
+import { PitfallRepository } from './repositories/pitfall-repository.ts';
+
+export type { Snapshot } from './interfaces.ts';
+
+/**
+ * Unified Facade for the DocOrbit database storage layer.
+ * Delegates domain responsibilities to specialized sub-repositories:
+ * - sources: ISourceRepository (sources, snapshots, snapshot-page links)
+ * - pages: IPageRepository (normalized pages, page links, page FTS)
+ * - chunks: IChunkRepository (document chunks, code, symbols, relationships, chunk FTS)
+ * - apis: IApiRepository (API endpoints, OpenAPI schemas, endpoint FTS)
+ * - examples: IExampleRepository (code examples, example FTS)
+ * - pitfalls: IPitfallRepository (pitfalls, gotchas, pitfall FTS)
+ */
+export class DocOrbitRepository
+  implements
+    ISourceRepository,
+    IPageRepository,
+    IChunkRepository,
+    IApiRepository,
+    IExampleRepository,
+    IPitfallRepository
+{
+  public readonly sources: SourceRepository;
+  public readonly pages: PageRepository;
+  public readonly chunks: ChunkRepository;
+  public readonly apis: ApiRepository;
+  public readonly examples: ExampleRepository;
+  public readonly pitfalls: PitfallRepository;
+
+  public readonly db: DocOrbitDb;
+
+  constructor(db: DocOrbitDb) {
+    this.db = db;
+    this.pages = new PageRepository(db);
+    this.sources = new SourceRepository(db, row => this.pages.hydratePage(row));
+    this.chunks = new ChunkRepository(db);
+    this.apis = new ApiRepository(db);
+    this.examples = new ExampleRepository(db);
+    this.pitfalls = new PitfallRepository(db);
+  }
+
+  // --- ISourceRepository Delegations ---
+
+  saveSource(source: DiscoveredSource): string {
+    return this.sources.saveSource(source);
+  }
+
+  getSource(id: string): DiscoveredSource | null {
+    return this.sources.getSource(id);
+  }
+
+  getSourceByUrl(url: string): DiscoveredSource | null {
+    return this.sources.getSourceByUrl(url);
+  }
+
+  listSources(): DiscoveredSource[] {
+    return this.sources.listSources();
+  }
+
+  createSnapshot(sourceId: string, metadata?: Record<string, unknown>, docVersion?: string): string {
+    return this.sources.createSnapshot(sourceId, metadata, docVersion);
+  }
+
+  saveSnapshot(snapshot: Snapshot): void {
+    this.sources.saveSnapshot(snapshot);
+  }
+
+  getSnapshot(id: string): Snapshot | null {
+    return this.sources.getSnapshot(id);
+  }
+
+  getLatestSnapshot(sourceId: string): Snapshot | null {
+    return this.sources.getLatestSnapshot(sourceId);
+  }
+
+  listSnapshots(sourceId: string): Snapshot[] {
+    return this.sources.listSnapshots(sourceId);
+  }
+
+  linkSnapshotPages(snapshotId: string, pageIds: string[]): void {
+    this.sources.linkSnapshotPages(snapshotId, pageIds);
+  }
+
+  getSnapshotPages(snapshotId: string): NormalizedPage[] {
+    return this.sources.getSnapshotPages(snapshotId);
+  }
+
+  // --- IPageRepository Delegations ---
+
+  savePage(page: NormalizedPage): void {
+    this.pages.savePage(page);
+  }
+
+  getPage(id: string): NormalizedPage | null {
+    return this.pages.getPage(id);
+  }
+
+  getPageByUrl(normalizedUrl: string): NormalizedPage | null {
+    return this.pages.getPageByUrl(normalizedUrl);
+  }
+
+  listPages(limit?: number, offset?: number): NormalizedPage[] {
+    return this.pages.listPages(limit, offset);
+  }
+
+  countPages(sourceId?: string): number {
+    return this.pages.countPages(sourceId);
+  }
+
+  savePageLinks(pageId: string, targetUrls: string[]): void {
+    this.pages.savePageLinks(pageId, targetUrls);
+  }
+
+  getPageLinks(pageId: string): string[] {
+    return this.pages.getPageLinks(pageId);
+  }
+
+  searchPagesFts(query: string, limit: number = 10): NormalizedPage[] {
+    return this.pages.searchPagesFts(query, limit);
+  }
+
+  // --- IChunkRepository Delegations ---
+
+  saveChunks(
+    chunks: DocumentChunk[],
+    relationships: ChunkRelationship[] = [],
+    codeSnippets: ChunkCode[] = [],
+    symbols: SymbolReference[] = []
+  ): void {
+    this.chunks.saveChunks(chunks, relationships, codeSnippets, symbols);
+  }
+
+  getChunk(id: string): DocumentChunk | null {
+    return this.chunks.getChunk(id);
+  }
+
+  getChunksByPage(pageId: string): DocumentChunk[] {
+    return this.chunks.getChunksByPage(pageId);
+  }
+
+  getChunksBySnapshot(snapshotId: string): DocumentChunk[] {
+    return this.chunks.getChunksBySnapshot(snapshotId);
+  }
+
+  getChunksByVersion(docVersion: string): DocumentChunk[] {
+    return this.chunks.getChunksByVersion(docVersion);
+  }
+
+  getChunkCode(chunkId: string): ChunkCode[] {
+    return this.chunks.getChunkCode(chunkId);
+  }
+
+  getChunkSymbols(chunkId: string): SymbolReference[] {
+    return this.chunks.getChunkSymbols(chunkId);
+  }
+
+  getChunkRelationships(chunkId: string): ChunkRelationship[] {
+    return this.chunks.getChunkRelationships(chunkId);
+  }
+
+  countChunks(snapshotId?: string): number {
+    return this.chunks.countChunks(snapshotId);
+  }
+
+  searchChunksFts(
+    query: string,
+    options: { limit?: number; snapshotId?: string; chunkType?: ChunkType; docVersion?: string } = {}
+  ): Array<{ chunk: DocumentChunk; ftsRank: number; symbols: SymbolReference[]; codeSnippets: ChunkCode[] }> {
+    return this.chunks.searchChunksFts(query, options);
+  }
+
+  // --- IApiRepository Delegations ---
+
+  saveApiEndpoints(endpoints: ApiEndpoint[]): void {
+    this.apis.saveApiEndpoints(endpoints);
+  }
+
+  getApiEndpoint(id: string): ApiEndpoint | null {
+    return this.apis.getApiEndpoint(id);
+  }
+
+  getApiEndpointsBySnapshot(snapshotId: string): ApiEndpoint[] {
+    return this.apis.getApiEndpointsBySnapshot(snapshotId);
+  }
+
+  searchApiEndpoints(
+    query: string,
+    options: { method?: string; docVersion?: string; limit?: number } = {}
+  ): ApiEndpoint[] {
+    return this.apis.searchApiEndpoints(query, options);
+  }
+
+  // --- IExampleRepository Delegations ---
+
+  saveIndexedExamples(examples: IndexedExample[]): void {
+    this.examples.saveIndexedExamples(examples);
+  }
+
+  getIndexedExamplesBySnapshot(snapshotId: string): IndexedExample[] {
+    return this.examples.getIndexedExamplesBySnapshot(snapshotId);
+  }
+
+  searchIndexedExamples(
+    query: string,
+    options: { language?: string; framework?: string; docVersion?: string; limit?: number } = {}
+  ): IndexedExample[] {
+    return this.examples.searchIndexedExamples(query, options);
+  }
+
+  // --- IPitfallRepository Delegations ---
+
+  savePitfalls(pitfalls: Pitfall[]): void {
+    this.pitfalls.savePitfalls(pitfalls);
+  }
+
+  getPitfallsBySnapshot(snapshotId: string): Pitfall[] {
+    return this.pitfalls.getPitfallsBySnapshot(snapshotId);
+  }
+
+  searchPitfalls(
+    query: string,
+    options: { kind?: PitfallKind; docVersion?: string; limit?: number } = {}
+  ): Pitfall[] {
+    return this.pitfalls.searchPitfalls(query, options);
+  }
+}
+

package/packages/storage/src/schema.ts

@@ -0,0 +1,269 @@
+export const SCHEMA_SQL = `
+CREATE TABLE IF NOT EXISTS sources (
+  id TEXT PRIMARY KEY,
+  url TEXT UNIQUE NOT NULL,
+  type TEXT NOT NULL,
+  discovered_by TEXT NOT NULL,
+  status TEXT NOT NULL,
+  content_type TEXT,
+  confidence REAL NOT NULL,
+  authority TEXT NOT NULL,
+  machine_readable INTEGER NOT NULL,
+  metadata TEXT,
+  created_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS pages (
+  id TEXT PRIMARY KEY,
+  source_id TEXT NOT NULL,
+  title TEXT NOT NULL,
+  url TEXT UNIQUE NOT NULL,
+  content TEXT NOT NULL,
+  content_hash TEXT NOT NULL,
+  fetched_at TEXT NOT NULL,
+  raw_bytes INTEGER NOT NULL,
+  estimated_tokens INTEGER NOT NULL,
+  headings_json TEXT,
+  security_annotations_json TEXT,
+  provenance_json TEXT,
+  FOREIGN KEY (source_id) REFERENCES sources(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS links (
+  id TEXT PRIMARY KEY,
+  page_id TEXT NOT NULL,
+  text TEXT NOT NULL,
+  url TEXT NOT NULL,
+  is_external INTEGER NOT NULL,
+  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS code_examples (
+  id TEXT PRIMARY KEY,
+  page_id TEXT NOT NULL,
+  language TEXT NOT NULL,
+  code TEXT NOT NULL,
+  caption TEXT,
+  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS snapshots (
+  id TEXT PRIMARY KEY,
+  source_id TEXT NOT NULL,
+  snapshot_hash TEXT NOT NULL,
+  page_count INTEGER NOT NULL,
+  captured_at TEXT NOT NULL,
+  doc_version TEXT,
+  metadata TEXT,
+  FOREIGN KEY (source_id) REFERENCES sources(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS snapshot_pages (
+  snapshot_id TEXT NOT NULL,
+  page_id TEXT NOT NULL,
+  content_hash TEXT NOT NULL,
+  PRIMARY KEY (snapshot_id, page_id),
+  FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
+  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS chunks (
+  id TEXT PRIMARY KEY,
+  page_id TEXT NOT NULL,
+  snapshot_id TEXT NOT NULL,
+  title TEXT,
+  section_path_json TEXT NOT NULL,
+  content TEXT NOT NULL,
+  chunk_type TEXT NOT NULL,
+  language TEXT,
+  token_estimate INTEGER NOT NULL,
+  ordinal INTEGER NOT NULL,
+  content_hash TEXT NOT NULL,
+  doc_version TEXT,
+  provenance_json TEXT,
+  created_at TEXT NOT NULL,
+  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS chunk_code (
+  id TEXT PRIMARY KEY,
+  chunk_id TEXT NOT NULL,
+  language TEXT,
+  code TEXT NOT NULL,
+  FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS symbol_references (
+  id TEXT PRIMARY KEY,
+  chunk_id TEXT NOT NULL,
+  name TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS chunk_relationships (
+  source_chunk_id TEXT NOT NULL,
+  target_chunk_id TEXT NOT NULL,
+  type TEXT NOT NULL,
+  PRIMARY KEY (source_chunk_id, target_chunk_id, type)
+);
+
+CREATE INDEX IF NOT EXISTS idx_pages_source_id ON pages(source_id);
+CREATE INDEX IF NOT EXISTS idx_pages_content_hash ON pages(content_hash);
+CREATE INDEX IF NOT EXISTS idx_links_page_id ON links(page_id);
+CREATE INDEX IF NOT EXISTS idx_code_examples_page_id ON code_examples(page_id);
+CREATE INDEX IF NOT EXISTS idx_snapshots_source_id ON snapshots(source_id);
+CREATE INDEX IF NOT EXISTS idx_snapshots_doc_version ON snapshots(doc_version);
+CREATE INDEX IF NOT EXISTS idx_snapshot_pages_snapshot_id ON snapshot_pages(snapshot_id);
+CREATE INDEX IF NOT EXISTS idx_snapshot_pages_page_id ON snapshot_pages(page_id);
+CREATE INDEX IF NOT EXISTS idx_chunks_page_id ON chunks(page_id);
+CREATE INDEX IF NOT EXISTS idx_chunks_snapshot_id ON chunks(snapshot_id);
+CREATE INDEX IF NOT EXISTS idx_chunks_chunk_type ON chunks(chunk_type);
+CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
+CREATE INDEX IF NOT EXISTS idx_chunks_doc_version ON chunks(doc_version);
+CREATE INDEX IF NOT EXISTS idx_chunk_code_chunk_id ON chunk_code(chunk_id);
+CREATE INDEX IF NOT EXISTS idx_symbol_references_chunk_id ON symbol_references(chunk_id);
+CREATE INDEX IF NOT EXISTS idx_symbol_references_name ON symbol_references(name);
+CREATE INDEX IF NOT EXISTS idx_chunk_relationships_source ON chunk_relationships(source_chunk_id);
+CREATE INDEX IF NOT EXISTS idx_chunk_relationships_target ON chunk_relationships(target_chunk_id);
+
+CREATE TABLE IF NOT EXISTS api_endpoints (
+  id TEXT PRIMARY KEY,
+  page_id TEXT NOT NULL,
+  snapshot_id TEXT NOT NULL,
+  method TEXT NOT NULL,
+  path TEXT NOT NULL,
+  summary TEXT,
+  description TEXT,
+  operation_id TEXT,
+  parameters_json TEXT NOT NULL,
+  request_schema_json TEXT,
+  response_schema_json TEXT,
+  auth_json TEXT NOT NULL,
+  errors_json TEXT NOT NULL,
+  pagination_json TEXT,
+  deprecated INTEGER NOT NULL,
+  doc_version TEXT,
+  provenance_json TEXT,
+  created_at TEXT NOT NULL,
+  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_api_endpoints_page_id ON api_endpoints(page_id);
+CREATE INDEX IF NOT EXISTS idx_api_endpoints_snapshot_id ON api_endpoints(snapshot_id);
+CREATE INDEX IF NOT EXISTS idx_api_endpoints_method ON api_endpoints(method);
+CREATE INDEX IF NOT EXISTS idx_api_endpoints_path ON api_endpoints(path);
+CREATE INDEX IF NOT EXISTS idx_api_endpoints_doc_version ON api_endpoints(doc_version);
+
+CREATE TABLE IF NOT EXISTS indexed_examples (
+  id TEXT PRIMARY KEY,
+  chunk_id TEXT,
+  page_id TEXT NOT NULL,
+  snapshot_id TEXT NOT NULL,
+  language TEXT NOT NULL,
+  framework TEXT,
+  task TEXT NOT NULL,
+  code TEXT NOT NULL,
+  source_url TEXT NOT NULL,
+  source_authority TEXT NOT NULL,
+  related_api TEXT,
+  related_symbol TEXT,
+  doc_version TEXT,
+  provenance_json TEXT,
+  created_at TEXT NOT NULL,
+  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_indexed_examples_page_id ON indexed_examples(page_id);
+CREATE INDEX IF NOT EXISTS idx_indexed_examples_snapshot_id ON indexed_examples(snapshot_id);
+CREATE INDEX IF NOT EXISTS idx_indexed_examples_chunk_id ON indexed_examples(chunk_id);
+CREATE INDEX IF NOT EXISTS idx_indexed_examples_language ON indexed_examples(language);
+CREATE INDEX IF NOT EXISTS idx_indexed_examples_framework ON indexed_examples(framework);
+CREATE INDEX IF NOT EXISTS idx_indexed_examples_doc_version ON indexed_examples(doc_version);
+CREATE INDEX IF NOT EXISTS idx_indexed_examples_related_api ON indexed_examples(related_api);
+
+CREATE TABLE IF NOT EXISTS pitfalls (
+  id TEXT PRIMARY KEY,
+  chunk_id TEXT,
+  page_id TEXT NOT NULL,
+  snapshot_id TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  title TEXT NOT NULL,
+  content TEXT NOT NULL,
+  related_api TEXT,
+  related_symbol TEXT,
+  doc_version TEXT,
+  provenance_json TEXT,
+  created_at TEXT NOT NULL,
+  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_pitfalls_page_id ON pitfalls(page_id);
+CREATE INDEX IF NOT EXISTS idx_pitfalls_snapshot_id ON pitfalls(snapshot_id);
+CREATE INDEX IF NOT EXISTS idx_pitfalls_chunk_id ON pitfalls(chunk_id);
+CREATE INDEX IF NOT EXISTS idx_pitfalls_kind ON pitfalls(kind);
+CREATE INDEX IF NOT EXISTS idx_pitfalls_doc_version ON pitfalls(doc_version);
+CREATE INDEX IF NOT EXISTS idx_pitfalls_related_api ON pitfalls(related_api);
+`;
+
+export const FTS_SCHEMA_SQL = `
+CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
+  page_id UNINDEXED,
+  title,
+  content
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+  chunk_id UNINDEXED,
+  title,
+  section_path,
+  content,
+  symbols
+);
+
+CREATE TRIGGER IF NOT EXISTS trg_chunks_delete AFTER DELETE ON chunks BEGIN
+  DELETE FROM chunks_fts WHERE chunk_id = old.id;
+END;
+
+CREATE VIRTUAL TABLE IF NOT EXISTS api_endpoints_fts USING fts5(
+  endpoint_id UNINDEXED,
+  method,
+  path,
+  summary,
+  description,
+  parameters
+);
+
+CREATE TRIGGER IF NOT EXISTS trg_api_endpoints_delete AFTER DELETE ON api_endpoints BEGIN
+  DELETE FROM api_endpoints_fts WHERE endpoint_id = old.id;
+END;
+
+CREATE VIRTUAL TABLE IF NOT EXISTS indexed_examples_fts USING fts5(
+  example_id UNINDEXED,
+  task,
+  language,
+  framework,
+  code,
+  related_api,
+  related_symbol
+);
+
+CREATE TRIGGER IF NOT EXISTS trg_indexed_examples_delete AFTER DELETE ON indexed_examples BEGIN
+  DELETE FROM indexed_examples_fts WHERE example_id = old.id;
+END;
+
+CREATE VIRTUAL TABLE IF NOT EXISTS pitfalls_fts USING fts5(
+  pitfall_id UNINDEXED,
+  kind,
+  title,
+  content,
+  related_api,
+  related_symbol
+);
+
+CREATE TRIGGER IF NOT EXISTS trg_pitfalls_delete AFTER DELETE ON pitfalls BEGIN
+  DELETE FROM pitfalls_fts WHERE pitfall_id = old.id;
+END;
+`;
+
+

package/packages/storage/src/search-tokens.ts

@@ -0,0 +1,28 @@
+export const SEARCH_STOPWORDS = new Set([
+  'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are',
+  'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by',
+  'can', 'did', 'do', 'does', 'doing', 'down', 'during', 'each', 'few', 'for', 'from',
+  'further', 'had', 'has', 'have', 'having', 'he', 'her', 'here', 'hers', 'herself', 'him',
+  'himself', 'his', 'how', 'i', 'if', 'in', 'into', 'is', 'it', 'its', 'itself', 'just',
+  'me', 'more', 'most', 'my', 'myself', 'no', 'nor', 'not', 'now', 'of', 'off', 'on',
+  'once', 'only', 'or', 'other', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'same',
+  'she', 'should', 'so', 'some', 'such', 'than', 'that', 'the', 'their', 'theirs', 'them',
+  'themselves', 'then', 'there', 'these', 'they', 'this', 'those', 'through', 'to', 'too',
+  'under', 'until', 'up', 'very', 'was', 'we', 'were', 'what', 'when', 'where', 'which',
+  'while', 'who', 'whom', 'why', 'with', 'you', 'your', 'yours', 'yourself', 'yourselves'
+]);
+
+/**
+ * Extracts substantive tokens from query text, filtering out common stopwords
+ * and short words (< 3 chars), while falling back to short words if query has no substantive words.
+ */
+export function extractSearchTokens(text: string): string[] {
+  const words = text
+    .replace(/[^\w\s-]/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(w => w.length > 0);
+
+  const substantive = words.filter(w => w.length >= 3 && !SEARCH_STOPWORDS.has(w.toLowerCase()));
+  return substantive.length > 0 ? substantive : words.filter(w => w.length >= 2);
+}