@geraldmaron/construct 1.0.23 → 1.1.0

package/lib/storage/hybrid-query.mjs

@@ -1,202 +1,21 @@
-#!/usr/bin/env node
 /**
- * lib/storage/hybrid-query.mjs — hybrid file + SQL + semantic retrieval.
+ * lib/storage/hybrid-query.mjs — unified search across knowledge and facts.
+ *
+ * This is a stub for the new LanceDB-backed search. 
  */
-import { loadStateSnapshot, summarizeStateSnapshot } from './state-source.mjs';
-import { describeSqlStore } from './sql-store.mjs';
-import { describeVectorStore, searchLocalVectorIndex, vectorSearchLocal } from './vector-store.mjs';
-import { createSqlClient, closeSqlClient, readVectorConfig } from './backend.mjs';
-import { embedText, getEmbeddingModelInfo } from './embeddings-engine.mjs';
-import { floatArrayToPgVector } from './vector-client.mjs';
-import { reciprocalRankFusion } from './rrf.mjs';
+import { VectorClient } from './vector-client.mjs';
 
-// iterative_scan (pgvector >= 0.8.0) keeps a filtered ANN query from
-// under-returning; harmless for the current unfiltered query, and the correct
-// default once tag/metadata filters are added to the vector search.
+export async function buildHybridSearchResultsAsync(rootDir, query, { limit = 10, env = process.env } = {}) {
+  // 1. Embed query
+  const { embedText } = await import('./embeddings-engine.mjs');
+  const { embedding } = await embedText(query, { env });
 
-async function supportsIterativeScan(client) {
-  try {
-    const [row] = await client`SELECT extversion AS v FROM pg_extension WHERE extname = 'vector'`;
-    if (!row?.v) return false;
-    const [major, minor] = String(row.v).split('.').map((n) => parseInt(n, 10));
-    return major > 0 || (major === 0 && minor >= 8);
-  } catch {
-    return false;
-  }
-}
-
-function collectFileCandidates(snapshot) {
-  const docs = [];
-
-  if (snapshot.context) {
-    docs.push({
-      id: '.cx/context.json',
-      kind: 'context',
-      title: 'Context state',
-      summary: snapshot.context.contextSummary || snapshot.context.summary || null,
-      body: JSON.stringify(snapshot.context, null, 2),
-      tags: ['context', 'state', 'cx'],
-    });
-  }
-
-  if (snapshot.architecture) {
-    docs.push({
-      id: 'docs/concepts/architecture.md',
-      kind: 'architecture',
-      title: 'Architecture docs',
-      summary: snapshot.architecture.slice(0, 240),
-      body: snapshot.architecture,
-      tags: ['architecture', 'docs'],
-    });
-  }
-
-  if (snapshot.docsReadme) {
-    docs.push({
-      id: 'docs/README.md',
-      kind: 'docs',
-      title: 'Docs index',
-      summary: snapshot.docsReadme.slice(0, 240),
-      body: snapshot.docsReadme,
-      tags: ['docs', 'index'],
-    });
-  }
-
-  for (const doc of snapshot.productIntelDocs ?? []) {
-    const kind = doc.path.startsWith('docs/prd/')
-      ? 'prd'
-      : doc.path.startsWith('docs/meta-prd/')
-        ? 'meta-prd'
-        : 'knowledge';
-    docs.push({
-      id: doc.path,
-      kind,
-      title: doc.title,
-      summary: doc.body.slice(0, 240),
-      body: doc.body,
-      tags: ['knowledge', kind],
-    });
-  }
-
-  return docs;
-}
-
-export function buildHybridSearchResults(rootDir, query, { limit = 10, env = process.env } = {}) {
-  const snapshot = loadStateSnapshot(rootDir);
-  const fileCandidates = collectFileCandidates(snapshot);
-  const sqlStore = describeSqlStore(env);
-  const vectorStore = describeVectorStore(env);
-  const fileHits = vectorSearchLocal(fileCandidates, query, { limit });
-  const localVectorHits = vectorStore.mode === 'local' && vectorStore.indexPath
-    ? searchLocalVectorIndex(vectorStore.indexPath, query, { limit })
-    : [];
-  const merged = [...fileHits];
-  for (const hit of localVectorHits) {
-    if (merged.some((entry) => entry.id === hit.id)) continue;
-    merged.push({
-      id: hit.id,
-      kind: hit.kind,
-      title: hit.title,
-      summary: hit.summary,
-      score: hit.score,
-      source_path: hit.source_path,
-    });
-  }
-
-  return {
-    query,
-    summary: summarizeStateSnapshot(snapshot),
-    stores: {
-      file: { configured: true, mode: 'canonical' },
-      sql: sqlStore,
-      vector: vectorStore,
-    },
-    results: merged.slice(0, limit).map((hit) => ({
-      id: hit.id,
-      kind: hit.kind,
-      title: hit.title,
-      summary: hit.summary,
-      score: hit.score,
-    })),
-  };
-}
-
-export async function buildHybridSearchResultsAsync(rootDir, query, { limit = 10, env = process.env, embed = embedText, embeddingModel: modelOverride = null } = {}) {
-  const base = buildHybridSearchResults(rootDir, query, { limit, env });
-  const client = createSqlClient(env);
-  if (!client) return base;
-
-  // Resolve the active embedding model so the SQL filter and the query
-  // embedding agree on dimensionality and identity.
-  const embeddingModel = modelOverride || (await getEmbeddingModelInfo({ env })).model;
-
-  try {
-    const queryVec = floatArrayToPgVector((await embed(query, { env })).embedding);
-    const iterative = await supportsIterativeScan(client);
-
-    // Native pgvector ANN over the HNSW index (not a JS full scan). When the
-    // extension supports it, relaxed_order iterative scan runs inside the txn.
-    const runVector = (sql) => sql`
-      select d.id, d.kind, d.title, d.summary, d.source_path
-      from construct_documents d
-      join construct_embeddings e on e.document_id = d.id
-      where d.project = 'construct' and e.model = ${embeddingModel}
-      order by e.embedding <=> ${queryVec}
-      limit ${limit}
-    `;
-    const vectorHits = iterative
-      ? await client.begin(async (sql) => {
-          await sql`SET LOCAL hnsw.iterative_scan = relaxed_order`;
-          return runVector(sql);
-        })
-      : await runVector(client);
-
-    const keywordHits = await client`
-      select id, kind, title, summary, source_path
-      from construct_documents
-      where project = 'construct'
-        and (title ilike ${`%${query}%`} or coalesce(summary, '') ilike ${`%${query}%`} or body ilike ${`%${query}%`})
-      order by updated_at desc
-      limit ${limit}
-    `;
-
-    // One consolidated ranking: file BM25 (base) + neural ANN + keyword, fused
-    // by Reciprocal Rank Fusion so three incompatible score scales merge by
-    // rank rather than magnitude.
-    const lists = [base.results, vectorHits, keywordHits];
-    const byId = new Map();
-    for (const list of lists) {
-      for (const it of list) if (it && !byId.has(it.id)) byId.set(it.id, it);
-    }
-    const results = reciprocalRankFusion(lists, { idOf: (x) => x.id, limit }).map(({ id, score }) => {
-      const it = byId.get(id);
-      return { id, kind: it.kind, title: it.title, summary: it.summary, score, source_path: it.source_path ?? null };
-    });
+  // 2. Search LanceDB
+  const client = new VectorClient({ env });
+  const docs = await client.searchDocuments({ project: 'construct', queryEmbedding: embedding, limit });
+  const obs = await client.searchObservations({ project: 'construct', queryEmbedding: embedding, limit });
 
-    return {
-      ...base,
-      results,
-      stores: {
-        ...base.stores,
-        vector: {
-          ...base.stores.vector,
-          ...readVectorConfig(env),
-          model: embeddingModel,
-          iterativeScan: iterative,
-        },
-        sql: {
-          ...base.stores.sql,
-          mode: 'postgres',
-          configured: true,
-          sharedReady: true,
-        },
-      },
-    };
-  } catch (error) {
-    return {
-      ...base,
-      error: error?.message || 'hybrid search failed',
-    };
-  } finally {
-    await closeSqlClient(client);
-  }
+  // 3. Merge and rank (simple merge for now)
+  const all = [...docs.map(d => ({ ...d, type: 'document' })), ...obs.map(o => ({ ...o, type: 'observation' }))];
+  return all.sort((a, b) => b.similarity - a.similarity).slice(0, limit);
 }

package/lib/storage/sync.mjs

@@ -1,15 +1,14 @@
 #!/usr/bin/env node
 /**
- * lib/storage/sync.mjs — sync file-state artifacts into shared Postgres indices.
+ * lib/storage/sync.mjs — sync file-state artifacts into local LanceDB indices.
+ *
+ * This implementation replaces the Postgres-backed sync with a LanceDB-backed
+ * one, matching the new local-first, Git-backed architecture.
  */
 import crypto from 'node:crypto';
 import { loadStateSnapshot } from './state-source.mjs';
-import { createSqlClient, closeSqlClient, readVectorConfig } from './backend.mjs';
 import { embedBatch, getEmbeddingModelInfo } from './embeddings-engine.mjs';
-import { writeLocalVectorIndex } from './vector-store.mjs';
-import { runMigrations } from './migrations.mjs';
-import { isManagedConstructPostgresUrl } from '../service-manager.mjs';
-import { floatArrayToPgVector } from './vector-client.mjs';
+import { VectorClient } from './vector-client.mjs';
 
 function hashContent(content) {
   return crypto.createHash('sha256').update(String(content || '')).digest('hex');
@@ -25,7 +24,7 @@ function toDocumentRows(rootDir, snapshot, project = 'construct') {
       project,
       kind: 'context',
       title: 'Context state',
-      summary: snapshot.context.contextSummary || snapshot.context.summary || null,
+      summary: snapshot.context.contextSummary || snapshot.context.summary || '',
       body,
       source_path: '.cx/context.json',
       tags: ['context', 'state', 'cx'],
@@ -87,187 +86,47 @@ function toDocumentRows(rootDir, snapshot, project = 'construct') {
   return rows;
 }
 
-/**
- * Ensure the hybrid storage schema is current by running every pending
- * migration in order. Uses `lib/storage/migrations.mjs` which tracks
- * applied state in `construct_schema_migrations` and surfaces drift on
- * changed files.
- */
-export async function ensureHybridSchema(client) {
-  await runMigrations(client);
-}
-
 export async function syncFileStateToSql(rootDir, { env = process.env, project = 'construct' } = {}) {
   const snapshot = loadStateSnapshot(rootDir);
   const rows = toDocumentRows(rootDir, snapshot, project);
-  const vectorConfig = readVectorConfig(env);
-
-  // Resolve the active embedding model once per sync so every record carries
-  // a consistent model id and dimensionality.
+  
   const modelInfo = await getEmbeddingModelInfo({ env });
   const embeddingModel = modelInfo.model;
 
-  // Pre-flight check: verify database is reachable before attempting sync
-  const probeClient = createSqlClient(env);
-  if (probeClient) {
-    try {
-      await probeClient`SELECT 1`;
-    } catch (error) {
-      // Database unreachable - check if it's a managed Postgres that needs startup
-      const databaseUrl = env.DATABASE_URL || '';
-      await closeSqlClient(probeClient);
-      
-      if (isManagedConstructPostgresUrl(databaseUrl)) {
-        return {
-          status: 'degraded',
-          error: 'Managed Postgres is not running. Run `construct up` to start services, then retry.',
-          documentsSynced: 0,
-          embeddingsSynced: 0,
-          embeddingModel,
-          vector: vectorConfig,
-          localVector: { status: 'unavailable', note: 'Sync failed - database offline' },
-          sql: { status: 'unavailable', error: 'ECONNREFUSED - Postgres not running' },
-          hint: 'Run `construct up` to start managed Postgres and other runtime services',
-        };
-      }
-      
-      return {
-        status: 'degraded',
-        error: `Database connection failed: ${error.message}`,
-        documentsSynced: 0,
-        embeddingsSynced: 0,
-        embeddingModel,
-        vector: vectorConfig,
-        localVector: { status: 'unavailable', note: 'Sync failed - database offline' },
-        sql: { status: 'unavailable', error: error.message },
-      };
-    }
-    await closeSqlClient(probeClient);
-  }
-
-  // Batch-embed all documents through the engine. The engine selects the
-  // configured adapter (local ONNX, openai, ollama, hashing) and returns
-  // Float32Array embeddings of `modelInfo.dimensions` length.
-  const embedTexts = rows.map((row) =>
-    [row.title, row.summary, row.body, row.source_path, row.kind].filter(Boolean).join('\n')
-  );
-  const embeddings = embedTexts.length > 0 ? await embedBatch(embedTexts, { env }) : [];
-
-  const localVectorRecords = rows.map((row, i) => ({
-    id: row.id,
-    project: row.project,
-    kind: row.kind,
-    title: row.title,
-    summary: row.summary,
-    body: row.body,
-    source_path: row.source_path,
-    tags: row.tags,
-    content_hash: row.content_hash,
-    embedding: Array.from(embeddings[i].embedding).map((v) => Number(v)),
-    updatedAt: new Date().toISOString(),
-  }));
-
-  let localVector = { status: 'unavailable', note: 'No local vector index configured', recordsSynced: 0 };
-  if (vectorConfig.indexPath) {
-    const payload = writeLocalVectorIndex(vectorConfig.indexPath, localVectorRecords, { model: embeddingModel });
-    localVector = {
-      status: 'ok',
-      indexPath: vectorConfig.indexPath,
-      model: payload.model,
-      recordsSynced: payload.records.length,
-      updatedAt: payload.updatedAt,
-    };
-  }
-
-  if (!probeClient) {
+  const vectorClient = new VectorClient({ env });
+  
+  if (rows.length === 0) {
     return {
-      status: localVector.status === 'ok' ? 'ok' : 'unavailable',
-      note: 'No DATABASE_URL configured',
-      documentsSynced: rows.length,
-      embeddingsSynced: localVector.recordsSynced,
+      status: 'ok',
+      documentsSynced: 0,
+      embeddingsSynced: 0,
       embeddingModel,
-      vector: vectorConfig,
-      localVector,
-      sql: { status: 'unavailable', note: 'No DATABASE_URL configured' },
+      backend: 'lancedb'
     };
   }
 
-  // Create a fresh client for the actual sync operation
-  const syncClient = createSqlClient(env);
-  if (!syncClient) {
-    return {
-      status: localVector.status === 'ok' ? 'ok' : 'unavailable',
-      note: 'No DATABASE_URL configured',
-      documentsSynced: rows.length,
-      embeddingsSynced: localVector.recordsSynced,
-      embeddingModel,
-      vector: vectorConfig,
-      localVector,
-      sql: { status: 'unavailable', note: 'No DATABASE_URL configured' },
-    };
+  // Batch-embed all documents
+  const embedTexts = rows.map((row) =>
+    [row.title, row.summary, row.body, row.source_path, row.kind].filter(Boolean).join('\n')
+  );
+  const embeddings = await embedBatch(embedTexts, { env });
+
+  let documentsSynced = 0;
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i];
+    await vectorClient.storeDocument({
+      ...row,
+      embedding: embeddings[i].embedding,
+      model: embeddingModel
+    });
+    documentsSynced += 1;
   }
 
-  try {
-    await ensureHybridSchema(syncClient);
-
-    let documentsSynced = 0;
-    let embeddingsSynced = 0;
-    for (const row of rows) {
-      await syncClient`
-        insert into construct_documents (id, project, kind, title, summary, body, source_path, tags, content_hash, updated_at)
-        values (${row.id}, ${row.project}, ${row.kind}, ${row.title}, ${row.summary}, ${row.body}, ${row.source_path}, ${JSON.stringify(row.tags)}, ${row.content_hash}, now())
-        on conflict (id) do update set
-          project = excluded.project,
-          kind = excluded.kind,
-          title = excluded.title,
-          summary = excluded.summary,
-          body = excluded.body,
-          source_path = excluded.source_path,
-          tags = excluded.tags,
-          content_hash = excluded.content_hash,
-          updated_at = now()
-      `;
-      documentsSynced += 1;
-
-      const embeddingVec = floatArrayToPgVector(localVectorRecords[documentsSynced - 1].embedding);
-      await syncClient`
-        insert into construct_embeddings (document_id, model, embedding, content_hash, updated_at)
-        values (${row.id}, ${embeddingModel}, ${embeddingVec}, ${row.content_hash}, now())
-        on conflict (document_id) do update set
-          model = excluded.model,
-          embedding = excluded.embedding,
-          content_hash = excluded.content_hash,
-          updated_at = now()
-      `;
-      embeddingsSynced += 1;
-    }
-
-    await syncClient`
-      insert into construct_sync_runs (project, source, documents_synced, embeddings_synced, status, note)
-      values (${project}, ${'file-state'}, ${documentsSynced}, ${0}, ${'ok'}, ${'synced file-state documents'})
-    `;
-
-    return {
-      status: 'ok',
-      documentsSynced,
-      embeddingsSynced,
-      embeddingModel,
-      vector: vectorConfig,
-      localVector,
-      sql: { status: 'ok' },
-    };
-  } catch (error) {
-    return {
-      status: localVector.status === 'ok' ? 'degraded' : 'degraded',
-      error: error?.message || 'sync failed',
-      documentsSynced: rows.length,
-      embeddingsSynced: localVector.recordsSynced,
-      embeddingModel,
-      vector: vectorConfig,
-      localVector,
-      sql: { status: 'degraded', error: error?.message || 'sync failed' },
-    };
-  } finally {
-    await closeSqlClient(syncClient);
-  }
+  return {
+    status: 'ok',
+    documentsSynced,
+    embeddingsSynced: documentsSynced,
+    embeddingModel,
+    backend: 'lancedb'
+  };
 }