docorbit 0.1.0

package/packages/storage/src/db.ts

@@ -0,0 +1,72 @@
+import { DatabaseSync } from 'node:sqlite';
+import { dirname } from 'node:path';
+import { mkdirSync } from 'node:fs';
+import { SCHEMA_SQL, FTS_SCHEMA_SQL } from './schema.ts';
+
+export class DocOrbitDb {
+  private db: DatabaseSync;
+  private ftsAvailable = false;
+
+  constructor(dbPath: string = ':memory:') {
+    if (dbPath !== ':memory:') {
+      mkdirSync(dirname(dbPath), { recursive: true });
+    }
+
+    this.db = new DatabaseSync(dbPath);
+
+    // Enable foreign keys
+    this.db.exec('PRAGMA foreign_keys = ON;');
+
+    // Enable WAL mode for file-based database for concurrent reads
+    if (dbPath !== ':memory:') {
+      this.db.exec('PRAGMA journal_mode = WAL;');
+    }
+
+    this.migrate();
+  }
+
+  private migrate(): void {
+    // Base relational tables
+    this.db.exec(SCHEMA_SQL);
+
+    // Schema evolution migrations
+    try {
+      this.db.exec('ALTER TABLE pages ADD COLUMN provenance_json TEXT;');
+    } catch {
+      // Column already exists
+    }
+
+    try {
+      this.db.exec('ALTER TABLE snapshots ADD COLUMN doc_version TEXT;');
+    } catch {
+      // Column already exists
+    }
+
+    try {
+      this.db.exec('ALTER TABLE chunks ADD COLUMN doc_version TEXT;');
+    } catch {
+      // Column already exists
+    }
+
+    // Test and enable FTS5 virtual table
+    try {
+      this.db.exec(FTS_SCHEMA_SQL);
+      this.ftsAvailable = true;
+    } catch {
+      this.ftsAvailable = false;
+    }
+  }
+
+  isFtsAvailable(): boolean {
+    return this.ftsAvailable;
+  }
+
+  getRawDb(): DatabaseSync {
+    return this.db;
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}
+

package/packages/storage/src/index.ts

@@ -0,0 +1,11 @@
+export * from './db.ts';
+export * from './schema.ts';
+export * from './repository.ts';
+export type * from './interfaces.ts';
+export * from './search-tokens.ts';
+export * from './repositories/source-repository.ts';
+export * from './repositories/page-repository.ts';
+export * from './repositories/chunk-repository.ts';
+export * from './repositories/api-repository.ts';
+export * from './repositories/example-repository.ts';
+export * from './repositories/pitfall-repository.ts';

package/packages/storage/src/interfaces.ts

@@ -0,0 +1,115 @@
+import type {
+  DiscoveredSource,
+  NormalizedPage,
+  DocumentChunk,
+  ChunkCode,
+  SymbolReference,
+  ChunkRelationship,
+  ChunkType,
+  ApiEndpoint,
+  IndexedExample,
+  Pitfall,
+  PitfallKind,
+} from '../../shared/src/index.ts';
+
+export interface Snapshot {
+  id: string;
+  sourceId: string;
+  hash: string;
+  pageCount: number;
+  docVersion?: string;
+  createdAt: string;
+}
+
+export interface ISourceRepository {
+  saveSource(source: DiscoveredSource): string;
+  getSource(id: string): DiscoveredSource | null;
+  getSourceByUrl(url: string): DiscoveredSource | null;
+  listSources(): DiscoveredSource[];
+  createSnapshot(sourceId: string, metadata?: Record<string, unknown>, docVersion?: string): string;
+  saveSnapshot(snapshot: Snapshot): void;
+  getSnapshot(id: string): Snapshot | null;
+  getLatestSnapshot(sourceId: string): Snapshot | null;
+  listSnapshots(sourceId: string): Snapshot[];
+  linkSnapshotPages(snapshotId: string, pageIds: string[]): void;
+  getSnapshotPages(snapshotId: string): NormalizedPage[];
+}
+
+export interface IPageRepository {
+  savePage(page: NormalizedPage): void;
+  getPage(id: string): NormalizedPage | null;
+  getPageByUrl(normalizedUrl: string): NormalizedPage | null;
+  listPages(limit?: number, offset?: number): NormalizedPage[];
+  countPages(sourceId?: string): number;
+  savePageLinks(pageId: string, targetUrls: string[]): void;
+  getPageLinks(pageId: string): string[];
+  searchPagesFts(query: string, limit?: number): NormalizedPage[];
+}
+
+export interface IChunkRepository {
+  saveChunks(
+    chunks: DocumentChunk[],
+    relationships?: ChunkRelationship[],
+    codeSnippets?: ChunkCode[],
+    symbols?: SymbolReference[]
+  ): void;
+  getChunk(id: string): DocumentChunk | null;
+  getChunksByPage(pageId: string): DocumentChunk[];
+  getChunksBySnapshot(snapshotId: string): DocumentChunk[];
+  getChunksByVersion(docVersion: string): DocumentChunk[];
+  getChunkCode(chunkId: string): ChunkCode[];
+  getChunkSymbols(chunkId: string): SymbolReference[];
+  getChunkRelationships(chunkId: string): ChunkRelationship[];
+  countChunks(snapshotId?: string): number;
+  searchChunksFts(
+    query: string,
+    options?: {
+      limit?: number;
+      snapshotId?: string;
+      chunkType?: ChunkType;
+      docVersion?: string;
+    }
+  ): Array<{
+    chunk: DocumentChunk;
+    ftsRank: number;
+    symbols: SymbolReference[];
+    codeSnippets: ChunkCode[];
+  }>;
+}
+
+export interface IApiRepository {
+  saveApiEndpoints(endpoints: ApiEndpoint[]): void;
+  getApiEndpoint(id: string): ApiEndpoint | null;
+  getApiEndpointsBySnapshot(snapshotId: string): ApiEndpoint[];
+  searchApiEndpoints(
+    query: string,
+    options?: { method?: string; docVersion?: string; limit?: number }
+  ): ApiEndpoint[];
+}
+
+export interface IExampleRepository {
+  saveIndexedExamples(examples: IndexedExample[]): void;
+  getIndexedExamplesBySnapshot(snapshotId: string): IndexedExample[];
+  searchIndexedExamples(
+    query: string,
+    options?: {
+      language?: string;
+      framework?: string;
+      docVersion?: string;
+      limit?: number;
+    }
+  ): IndexedExample[];
+}
+
+export interface IPitfallRepository {
+  savePitfalls(pitfalls: Pitfall[]): void;
+  getPitfallsBySnapshot(snapshotId: string): Pitfall[];
+  searchPitfalls(
+    query: string,
+    options?: {
+      kind?: PitfallKind;
+      docVersion?: string;
+      limit?: number;
+    }
+  ): Pitfall[];
+}

package/packages/storage/src/repositories/api-repository.ts

@@ -0,0 +1,219 @@
+import type { ApiEndpoint } from '../../../shared/src/index.ts';
+import type { DocOrbitDb } from '../db.ts';
+import type { IApiRepository } from '../interfaces.ts';
+import { extractSearchTokens } from '../search-tokens.ts';
+
+export class ApiRepository implements IApiRepository {
+  private db: DocOrbitDb;
+
+  constructor(db: DocOrbitDb) {
+    this.db = db;
+  }
+
+  saveApiEndpoints(endpoints: ApiEndpoint[]): void {
+    if (endpoints.length === 0) return;
+    const raw = this.db.getRawDb();
+
+    const insertEp = raw.prepare(`
+      INSERT INTO api_endpoints (
+        id, page_id, snapshot_id, method, path, summary, description,
+        operation_id, parameters_json, request_schema_json, response_schema_json,
+        auth_json, errors_json, pagination_json, deprecated, doc_version,
+        provenance_json, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        method = excluded.method,
+        path = excluded.path,
+        summary = excluded.summary,
+        description = excluded.description,
+        operation_id = excluded.operation_id,
+        parameters_json = excluded.parameters_json,
+        request_schema_json = excluded.request_schema_json,
+        response_schema_json = excluded.response_schema_json,
+        auth_json = excluded.auth_json,
+        errors_json = excluded.errors_json,
+        pagination_json = excluded.pagination_json,
+        deprecated = excluded.deprecated,
+        doc_version = excluded.doc_version,
+        provenance_json = excluded.provenance_json
+    `);
+
+    const insertFts = this.db.isFtsAvailable()
+      ? raw.prepare(`
+          INSERT INTO api_endpoints_fts (endpoint_id, method, path, summary, description, parameters)
+          VALUES (?, ?, ?, ?, ?, ?)
+        `)
+      : null;
+
+    const deleteFts = this.db.isFtsAvailable()
+      ? raw.prepare('DELETE FROM api_endpoints_fts WHERE endpoint_id = ?')
+      : null;
+
+    raw.exec('BEGIN TRANSACTION;');
+    try {
+      for (const ep of endpoints) {
+        insertEp.run(
+          ep.id,
+          ep.pageId,
+          ep.snapshotId || 'snap_default',
+          ep.method,
+          ep.path,
+          ep.summary || null,
+          ep.description || null,
+          ep.operationId || null,
+          JSON.stringify(ep.parameters || []),
+          ep.requestSchema ? JSON.stringify(ep.requestSchema) : null,
+          ep.responseSchema ? JSON.stringify(ep.responseSchema) : null,
+          ep.auth ? JSON.stringify(ep.auth) : '[]',
+          ep.errors ? JSON.stringify(ep.errors) : '[]',
+          ep.pagination ? JSON.stringify(ep.pagination) : null,
+          ep.deprecated ? 1 : 0,
+          ep.docVersion || null,
+          ep.provenance ? JSON.stringify(ep.provenance) : null,
+          ep.createdAt || new Date().toISOString()
+        );
+
+        if (insertFts && deleteFts) {
+          deleteFts.run(ep.id);
+          const paramNames = (ep.parameters || []).map(p => p.name).join(' ');
+          insertFts.run(
+            ep.id,
+            ep.method,
+            ep.path,
+            ep.summary || '',
+            ep.description || '',
+            paramNames
+          );
+        }
+      }
+      raw.exec('COMMIT;');
+    } catch (err) {
+      raw.exec('ROLLBACK;');
+      throw err;
+    }
+  }
+
+  getApiEndpoint(id: string): ApiEndpoint | null {
+    const raw = this.db.getRawDb();
+    const row = raw.prepare('SELECT * FROM api_endpoints WHERE id = ?').get(id) as Record<string, unknown> | undefined;
+    return row ? this.hydrateApiEndpoint(row) : null;
+  }
+
+  getApiEndpointsBySnapshot(snapshotId: string): ApiEndpoint[] {
+    const raw = this.db.getRawDb();
+    const rows = raw.prepare('SELECT * FROM api_endpoints WHERE snapshot_id = ? ORDER BY path, method').all(snapshotId) as Array<Record<string, unknown>>;
+    return rows.map(r => this.hydrateApiEndpoint(r));
+  }
+
+  searchApiEndpoints(
+    query: string,
+    options: { method?: string; docVersion?: string; limit?: number } = {}
+  ): ApiEndpoint[] {
+    const raw = this.db.getRawDb();
+    const limit = options.limit || 20;
+    const cleanQuery = query.trim();
+
+    // Check if query specifies method at start (e.g. "POST /v1/...")
+    let effectiveMethod = options.method?.toLowerCase();
+    let term = cleanQuery;
+    const methodMatch = cleanQuery.match(/^(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\s+(.*)$/i);
+    if (methodMatch) {
+      effectiveMethod = methodMatch[1].toLowerCase();
+      term = methodMatch[2].trim();
+    }
+
+    // Try FTS if available and term exists
+    if (this.db.isFtsAvailable() && term.length > 0) {
+      const cleanTokens = extractSearchTokens(term);
+
+      if (cleanTokens.length > 0) {
+        const sanitizedFts = cleanTokens
+          .map(t => t.length >= 4 ? `"${t.replace(/"/g, '""')}"*` : `"${t.replace(/"/g, '""')}"`)
+          .join(' OR ');
+        try {
+          let sql = `
+            SELECT ep.*, fts.rank as fts_rank
+            FROM api_endpoints ep
+            JOIN api_endpoints_fts fts ON ep.id = fts.endpoint_id
+            WHERE api_endpoints_fts MATCH ?
+          `;
+          const params: unknown[] = [sanitizedFts];
+
+          if (effectiveMethod) {
+            sql += ' AND lower(ep.method) = ?';
+            params.push(effectiveMethod);
+          }
+          if (options.docVersion) {
+            sql += ' AND (ep.doc_version = ? OR ep.doc_version IS NULL)';
+            params.push(options.docVersion);
+          }
+
+          sql += ' ORDER BY fts.rank LIMIT ?';
+          params.push(limit);
+
+          const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
+          if (rows.length > 0) {
+            return rows.map(r => this.hydrateApiEndpoint(r));
+          }
+        } catch {
+          // Fall through to LIKE search
+        }
+      }
+    }
+
+    // LIKE fallback search
+    let sql = 'SELECT * FROM api_endpoints WHERE 1=1';
+    const params: unknown[] = [];
+
+    if (effectiveMethod) {
+      sql += ' AND lower(method) = ?';
+      params.push(effectiveMethod);
+    }
+    if (options.docVersion) {
+      sql += ' AND (doc_version = ? OR doc_version IS NULL)';
+      params.push(options.docVersion);
+    }
+    if (term.length > 0) {
+      const tokens = extractSearchTokens(term);
+      if (tokens.length > 0) {
+        const tokenClauses = tokens.map(() => '(path LIKE ? OR summary LIKE ? OR description LIKE ? OR parameters_json LIKE ?)');
+        sql += ` AND (${tokenClauses.join(' OR ')})`;
+        for (const t of tokens) {
+          const lp = `%${t}%`;
+          params.push(lp, lp, lp, lp);
+        }
+      } else {
+        return [];
+      }
+    }
+
+    sql += ' ORDER BY path, method LIMIT ?';
+    params.push(limit);
+
+    const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
+    return rows.map(r => this.hydrateApiEndpoint(r));
+  }
+
+  private hydrateApiEndpoint(row: Record<string, unknown>): ApiEndpoint {
+    return {
+      id: String(row.id),
+      pageId: String(row.page_id),
+      snapshotId: String(row.snapshot_id),
+      method: String(row.method) as ApiEndpoint['method'],
+      path: String(row.path),
+      summary: row.summary ? String(row.summary) : undefined,
+      description: row.description ? String(row.description) : undefined,
+      operationId: row.operation_id ? String(row.operation_id) : undefined,
+      parameters: row.parameters_json ? JSON.parse(String(row.parameters_json)) : [],
+      requestSchema: row.request_schema_json ? JSON.parse(String(row.request_schema_json)) : undefined,
+      responseSchema: row.response_schema_json ? JSON.parse(String(row.response_schema_json)) : undefined,
+      auth: row.auth_json ? JSON.parse(String(row.auth_json)) : [],
+      errors: row.errors_json ? JSON.parse(String(row.errors_json)) : [],
+      pagination: row.pagination_json ? JSON.parse(String(row.pagination_json)) : undefined,
+      deprecated: Number(row.deprecated) === 1,
+      docVersion: row.doc_version ? String(row.doc_version) : undefined,
+      provenance: row.provenance_json ? JSON.parse(String(row.provenance_json)) : undefined,
+      createdAt: String(row.created_at),
+    };
+  }
+}