docorbit 0.1.0 → 0.1.2

package/packages/storage/src/db.ts

@@ -1,72 +0,0 @@
-import { DatabaseSync } from 'node:sqlite';
-import { dirname } from 'node:path';
-import { mkdirSync } from 'node:fs';
-import { SCHEMA_SQL, FTS_SCHEMA_SQL } from './schema.ts';
-
-export class DocOrbitDb {
-  private db: DatabaseSync;
-  private ftsAvailable = false;
-
-  constructor(dbPath: string = ':memory:') {
-    if (dbPath !== ':memory:') {
-      mkdirSync(dirname(dbPath), { recursive: true });
-    }
-
-    this.db = new DatabaseSync(dbPath);
-
-    // Enable foreign keys
-    this.db.exec('PRAGMA foreign_keys = ON;');
-
-    // Enable WAL mode for file-based database for concurrent reads
-    if (dbPath !== ':memory:') {
-      this.db.exec('PRAGMA journal_mode = WAL;');
-    }
-
-    this.migrate();
-  }
-
-  private migrate(): void {
-    // Base relational tables
-    this.db.exec(SCHEMA_SQL);
-
-    // Schema evolution migrations
-    try {
-      this.db.exec('ALTER TABLE pages ADD COLUMN provenance_json TEXT;');
-    } catch {
-      // Column already exists
-    }
-
-    try {
-      this.db.exec('ALTER TABLE snapshots ADD COLUMN doc_version TEXT;');
-    } catch {
-      // Column already exists
-    }
-
-    try {
-      this.db.exec('ALTER TABLE chunks ADD COLUMN doc_version TEXT;');
-    } catch {
-      // Column already exists
-    }
-
-    // Test and enable FTS5 virtual table
-    try {
-      this.db.exec(FTS_SCHEMA_SQL);
-      this.ftsAvailable = true;
-    } catch {
-      this.ftsAvailable = false;
-    }
-  }
-
-  isFtsAvailable(): boolean {
-    return this.ftsAvailable;
-  }
-
-  getRawDb(): DatabaseSync {
-    return this.db;
-  }
-
-  close(): void {
-    this.db.close();
-  }
-}
-

package/packages/storage/src/interfaces.ts

@@ -1,115 +0,0 @@
-import type {
-  DiscoveredSource,
-  NormalizedPage,
-  DocumentChunk,
-  ChunkCode,
-  SymbolReference,
-  ChunkRelationship,
-  ChunkType,
-  ApiEndpoint,
-  IndexedExample,
-  Pitfall,
-  PitfallKind,
-} from '../../shared/src/index.ts';
-
-export interface Snapshot {
-  id: string;
-  sourceId: string;
-  hash: string;
-  pageCount: number;
-  docVersion?: string;
-  createdAt: string;
-}
-
-export interface ISourceRepository {
-  saveSource(source: DiscoveredSource): string;
-  getSource(id: string): DiscoveredSource | null;
-  getSourceByUrl(url: string): DiscoveredSource | null;
-  listSources(): DiscoveredSource[];
-  createSnapshot(sourceId: string, metadata?: Record<string, unknown>, docVersion?: string): string;
-  saveSnapshot(snapshot: Snapshot): void;
-  getSnapshot(id: string): Snapshot | null;
-  getLatestSnapshot(sourceId: string): Snapshot | null;
-  listSnapshots(sourceId: string): Snapshot[];
-  linkSnapshotPages(snapshotId: string, pageIds: string[]): void;
-  getSnapshotPages(snapshotId: string): NormalizedPage[];
-}
-
-export interface IPageRepository {
-  savePage(page: NormalizedPage): void;
-  getPage(id: string): NormalizedPage | null;
-  getPageByUrl(normalizedUrl: string): NormalizedPage | null;
-  listPages(limit?: number, offset?: number): NormalizedPage[];
-  countPages(sourceId?: string): number;
-  savePageLinks(pageId: string, targetUrls: string[]): void;
-  getPageLinks(pageId: string): string[];
-  searchPagesFts(query: string, limit?: number): NormalizedPage[];
-}
-
-export interface IChunkRepository {
-  saveChunks(
-    chunks: DocumentChunk[],
-    relationships?: ChunkRelationship[],
-    codeSnippets?: ChunkCode[],
-    symbols?: SymbolReference[]
-  ): void;
-  getChunk(id: string): DocumentChunk | null;
-  getChunksByPage(pageId: string): DocumentChunk[];
-  getChunksBySnapshot(snapshotId: string): DocumentChunk[];
-  getChunksByVersion(docVersion: string): DocumentChunk[];
-  getChunkCode(chunkId: string): ChunkCode[];
-  getChunkSymbols(chunkId: string): SymbolReference[];
-  getChunkRelationships(chunkId: string): ChunkRelationship[];
-  countChunks(snapshotId?: string): number;
-  searchChunksFts(
-    query: string,
-    options?: {
-      limit?: number;
-      snapshotId?: string;
-      chunkType?: ChunkType;
-      docVersion?: string;
-    }
-  ): Array<{
-    chunk: DocumentChunk;
-    ftsRank: number;
-    symbols: SymbolReference[];
-    codeSnippets: ChunkCode[];
-  }>;
-}
-
-export interface IApiRepository {
-  saveApiEndpoints(endpoints: ApiEndpoint[]): void;
-  getApiEndpoint(id: string): ApiEndpoint | null;
-  getApiEndpointsBySnapshot(snapshotId: string): ApiEndpoint[];
-  searchApiEndpoints(
-    query: string,
-    options?: { method?: string; docVersion?: string; limit?: number }
-  ): ApiEndpoint[];
-}
-
-export interface IExampleRepository {
-  saveIndexedExamples(examples: IndexedExample[]): void;
-  getIndexedExamplesBySnapshot(snapshotId: string): IndexedExample[];
-  searchIndexedExamples(
-    query: string,
-    options?: {
-      language?: string;
-      framework?: string;
-      docVersion?: string;
-      limit?: number;
-    }
-  ): IndexedExample[];
-}
-
-export interface IPitfallRepository {
-  savePitfalls(pitfalls: Pitfall[]): void;
-  getPitfallsBySnapshot(snapshotId: string): Pitfall[];
-  searchPitfalls(
-    query: string,
-    options?: {
-      kind?: PitfallKind;
-      docVersion?: string;
-      limit?: number;
-    }
-  ): Pitfall[];
-}

package/packages/storage/src/repositories/api-repository.ts

@@ -1,219 +0,0 @@
-import type { ApiEndpoint } from '../../../shared/src/index.ts';
-import type { DocOrbitDb } from '../db.ts';
-import type { IApiRepository } from '../interfaces.ts';
-import { extractSearchTokens } from '../search-tokens.ts';
-
-export class ApiRepository implements IApiRepository {
-  private db: DocOrbitDb;
-
-  constructor(db: DocOrbitDb) {
-    this.db = db;
-  }
-
-  saveApiEndpoints(endpoints: ApiEndpoint[]): void {
-    if (endpoints.length === 0) return;
-    const raw = this.db.getRawDb();
-
-    const insertEp = raw.prepare(`
-      INSERT INTO api_endpoints (
-        id, page_id, snapshot_id, method, path, summary, description,
-        operation_id, parameters_json, request_schema_json, response_schema_json,
-        auth_json, errors_json, pagination_json, deprecated, doc_version,
-        provenance_json, created_at
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      ON CONFLICT(id) DO UPDATE SET
-        method = excluded.method,
-        path = excluded.path,
-        summary = excluded.summary,
-        description = excluded.description,
-        operation_id = excluded.operation_id,
-        parameters_json = excluded.parameters_json,
-        request_schema_json = excluded.request_schema_json,
-        response_schema_json = excluded.response_schema_json,
-        auth_json = excluded.auth_json,
-        errors_json = excluded.errors_json,
-        pagination_json = excluded.pagination_json,
-        deprecated = excluded.deprecated,
-        doc_version = excluded.doc_version,
-        provenance_json = excluded.provenance_json
-    `);
-
-    const insertFts = this.db.isFtsAvailable()
-      ? raw.prepare(`
-          INSERT INTO api_endpoints_fts (endpoint_id, method, path, summary, description, parameters)
-          VALUES (?, ?, ?, ?, ?, ?)
-        `)
-      : null;
-
-    const deleteFts = this.db.isFtsAvailable()
-      ? raw.prepare('DELETE FROM api_endpoints_fts WHERE endpoint_id = ?')
-      : null;
-
-    raw.exec('BEGIN TRANSACTION;');
-    try {
-      for (const ep of endpoints) {
-        insertEp.run(
-          ep.id,
-          ep.pageId,
-          ep.snapshotId || 'snap_default',
-          ep.method,
-          ep.path,
-          ep.summary || null,
-          ep.description || null,
-          ep.operationId || null,
-          JSON.stringify(ep.parameters || []),
-          ep.requestSchema ? JSON.stringify(ep.requestSchema) : null,
-          ep.responseSchema ? JSON.stringify(ep.responseSchema) : null,
-          ep.auth ? JSON.stringify(ep.auth) : '[]',
-          ep.errors ? JSON.stringify(ep.errors) : '[]',
-          ep.pagination ? JSON.stringify(ep.pagination) : null,
-          ep.deprecated ? 1 : 0,
-          ep.docVersion || null,
-          ep.provenance ? JSON.stringify(ep.provenance) : null,
-          ep.createdAt || new Date().toISOString()
-        );
-
-        if (insertFts && deleteFts) {
-          deleteFts.run(ep.id);
-          const paramNames = (ep.parameters || []).map(p => p.name).join(' ');
-          insertFts.run(
-            ep.id,
-            ep.method,
-            ep.path,
-            ep.summary || '',
-            ep.description || '',
-            paramNames
-          );
-        }
-      }
-      raw.exec('COMMIT;');
-    } catch (err) {
-      raw.exec('ROLLBACK;');
-      throw err;
-    }
-  }
-
-  getApiEndpoint(id: string): ApiEndpoint | null {
-    const raw = this.db.getRawDb();
-    const row = raw.prepare('SELECT * FROM api_endpoints WHERE id = ?').get(id) as Record<string, unknown> | undefined;
-    return row ? this.hydrateApiEndpoint(row) : null;
-  }
-
-  getApiEndpointsBySnapshot(snapshotId: string): ApiEndpoint[] {
-    const raw = this.db.getRawDb();
-    const rows = raw.prepare('SELECT * FROM api_endpoints WHERE snapshot_id = ? ORDER BY path, method').all(snapshotId) as Array<Record<string, unknown>>;
-    return rows.map(r => this.hydrateApiEndpoint(r));
-  }
-
-  searchApiEndpoints(
-    query: string,
-    options: { method?: string; docVersion?: string; limit?: number } = {}
-  ): ApiEndpoint[] {
-    const raw = this.db.getRawDb();
-    const limit = options.limit || 20;
-    const cleanQuery = query.trim();
-
-    // Check if query specifies method at start (e.g. "POST /v1/...")
-    let effectiveMethod = options.method?.toLowerCase();
-    let term = cleanQuery;
-    const methodMatch = cleanQuery.match(/^(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\s+(.*)$/i);
-    if (methodMatch) {
-      effectiveMethod = methodMatch[1].toLowerCase();
-      term = methodMatch[2].trim();
-    }
-
-    // Try FTS if available and term exists
-    if (this.db.isFtsAvailable() && term.length > 0) {
-      const cleanTokens = extractSearchTokens(term);
-
-      if (cleanTokens.length > 0) {
-        const sanitizedFts = cleanTokens
-          .map(t => t.length >= 4 ? `"${t.replace(/"/g, '""')}"*` : `"${t.replace(/"/g, '""')}"`)
-          .join(' OR ');
-        try {
-          let sql = `
-            SELECT ep.*, fts.rank as fts_rank
-            FROM api_endpoints ep
-            JOIN api_endpoints_fts fts ON ep.id = fts.endpoint_id
-            WHERE api_endpoints_fts MATCH ?
-          `;
-          const params: unknown[] = [sanitizedFts];
-
-          if (effectiveMethod) {
-            sql += ' AND lower(ep.method) = ?';
-            params.push(effectiveMethod);
-          }
-          if (options.docVersion) {
-            sql += ' AND (ep.doc_version = ? OR ep.doc_version IS NULL)';
-            params.push(options.docVersion);
-          }
-
-          sql += ' ORDER BY fts.rank LIMIT ?';
-          params.push(limit);
-
-          const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
-          if (rows.length > 0) {
-            return rows.map(r => this.hydrateApiEndpoint(r));
-          }
-        } catch {
-          // Fall through to LIKE search
-        }
-      }
-    }
-
-    // LIKE fallback search
-    let sql = 'SELECT * FROM api_endpoints WHERE 1=1';
-    const params: unknown[] = [];
-
-    if (effectiveMethod) {
-      sql += ' AND lower(method) = ?';
-      params.push(effectiveMethod);
-    }
-    if (options.docVersion) {
-      sql += ' AND (doc_version = ? OR doc_version IS NULL)';
-      params.push(options.docVersion);
-    }
-    if (term.length > 0) {
-      const tokens = extractSearchTokens(term);
-      if (tokens.length > 0) {
-        const tokenClauses = tokens.map(() => '(path LIKE ? OR summary LIKE ? OR description LIKE ? OR parameters_json LIKE ?)');
-        sql += ` AND (${tokenClauses.join(' OR ')})`;
-        for (const t of tokens) {
-          const lp = `%${t}%`;
-          params.push(lp, lp, lp, lp);
-        }
-      } else {
-        return [];
-      }
-    }
-
-    sql += ' ORDER BY path, method LIMIT ?';
-    params.push(limit);
-
-    const rows = raw.prepare(sql).all(...params) as Array<Record<string, unknown>>;
-    return rows.map(r => this.hydrateApiEndpoint(r));
-  }
-
-  private hydrateApiEndpoint(row: Record<string, unknown>): ApiEndpoint {
-    return {
-      id: String(row.id),
-      pageId: String(row.page_id),
-      snapshotId: String(row.snapshot_id),
-      method: String(row.method) as ApiEndpoint['method'],
-      path: String(row.path),
-      summary: row.summary ? String(row.summary) : undefined,
-      description: row.description ? String(row.description) : undefined,
-      operationId: row.operation_id ? String(row.operation_id) : undefined,
-      parameters: row.parameters_json ? JSON.parse(String(row.parameters_json)) : [],
-      requestSchema: row.request_schema_json ? JSON.parse(String(row.request_schema_json)) : undefined,
-      responseSchema: row.response_schema_json ? JSON.parse(String(row.response_schema_json)) : undefined,
-      auth: row.auth_json ? JSON.parse(String(row.auth_json)) : [],
-      errors: row.errors_json ? JSON.parse(String(row.errors_json)) : [],
-      pagination: row.pagination_json ? JSON.parse(String(row.pagination_json)) : undefined,
-      deprecated: Number(row.deprecated) === 1,
-      docVersion: row.doc_version ? String(row.doc_version) : undefined,
-      provenance: row.provenance_json ? JSON.parse(String(row.provenance_json)) : undefined,
-      createdAt: String(row.created_at),
-    };
-  }
-}