@geraldmaron/construct 1.0.23 → 1.1.0

package/lib/storage/vector-client.mjs

@@ -1,59 +1,90 @@
 /**
- * lib/storage/vector-client.mjs — Unified vector storage client for pgvector.
- *
- * Hides whether we're using local Postgres, managed Postgres, or (future) external vector DB.
- * All callers use: store(id, embedding, metadata) and search(queryEmbedding, filters).
- *
- * Works with any Postgres that has the pgvector extension installed.
- * Connection string (DATABASE_URL) determines the target.
- *
- * Dimensionality is resolved from the embedding engine at construction time
- * via `getEmbeddingModelInfo()`. The legacy `PGVECTOR_DIMENSIONS = 384` constant
- * is retained as a fallback only when the engine cannot be reached. When
- * `assertSchemaDimensions()` runs, the actual `embedding` column declared
- * dimension is read from `pg_attribute` and compared against the engine; a
- * mismatch surfaces a precise error so the operator can either rerun a
- * compatible migration or change `CONSTRUCT_EMBEDDING_MODEL`.
- *
- * NOTE: `postgres` package is imported lazily to avoid ERR_MODULE_NOT_FOUND
- * when this module is loaded in environments where `postgres` is not installed.
+ * lib/storage/vector-client.mjs — Unified vector storage client for LanceDB.
  */
 
+import path from 'path';
+import fs from 'fs';
 import { getEmbeddingModelInfo } from './embeddings-engine.mjs';
 
-const DEFAULT_POOL = { max: 10, idle_timeout: 20, connect_timeout: 10 };
 const FALLBACK_DIMENSIONS = 384;
 
-export const PGVECTOR_DIMENSIONS = FALLBACK_DIMENSIONS;
+function getDbPath(env = process.env) {
+  if (env.CONSTRUCT_LANCEDB_PATH) return env.CONSTRUCT_LANCEDB_PATH;
+  return path.join(process.cwd(), '.cx', 'lancedb');
+}
+
+// LanceDB uses optimistic concurrency: two writes racing the same table fail
+// with "Incompatible transaction ... Overwrite at version N" and two creates
+// race "Table already exists". Call sites build a fresh VectorClient per write
+// (inbox ingest fans out across files), so writes are serialized per database
+// directory at module scope — keyed by path, not by instance — and the queue
+// key is dropped once its chain drains.
 
-export function floatArrayToPgVector(arr) {
-  if (!arr) return null;
-  const values = arr instanceof Float32Array ? Array.from(arr) : arr;
-  return `[${values.join(',')}]`;
+const writeQueues = new Map();
+
+function serializeWrite(dbPath, task) {
+  const prev = writeQueues.get(dbPath) || Promise.resolve();
+  const run = prev.then(task, task);
+  const chain = run.then(() => {}, () => {});
+  writeQueues.set(dbPath, chain);
+  chain.then(() => {
+    if (writeQueues.get(dbPath) === chain) writeQueues.delete(dbPath);
+  });
+  return run;
 }
 
-export function assertPgvectorDimensions(embedding, { expected = FALLBACK_DIMENSIONS, label = 'embedding' } = {}) {
-  const length = embedding?.length;
-  if (length !== expected) {
-    throw new Error(`${label} has ${length ?? 0} dimensions; pgvector schema expects ${expected}. Configure a ${expected}d embedding model or run a compatible migration before indexing.`);
+// Serialization covers this process; a retry absorbs the residual conflict from
+// a concurrent writer in another process (separate test files) or a create race.
+
+async function withWriteRetry(task, { attempts = 6, baseMs = 25 } = {}) {
+  let lastErr;
+  for (let attempt = 0; attempt < attempts; attempt += 1) {
+    try {
+      return await task();
+    } catch (err) {
+      lastErr = err;
+      const msg = String(err?.message || err);
+      const retriable = /Incompatible transaction|already exists|Commit conflict|conflict_resolver|failed to persist|No such file or directory|LanceError\(IO\)/i.test(msg);
+      if (!retriable || attempt === attempts - 1) throw err;
+      await new Promise((resolve) => setTimeout(resolve, baseMs * (attempt + 1)));
+    }
   }
+  throw lastErr;
 }
 
 export class VectorClient {
   constructor({ databaseUrl, env } = {}) {
-    this.url = databaseUrl || process.env.DATABASE_URL || null;
     this.env = env || process.env;
-    this._sql = null;
-    this._pgModule = null;
+    this._db = null;
+    this._lancedbModule = null;
+    this._arrowModule = null;
     this._engineDim = null;
   }
 
-  /**
-   * Lazily resolve the active embedding engine's dimensionality. Cached for
-   * the life of the client. If engine resolution fails, falls back to
-   * FALLBACK_DIMENSIONS (384) and the caller surfaces the dim-mismatch error
-   * at insert/search time.
-   */
+  async _initModules() {
+    if (this._lancedbModule && this._arrowModule) return;
+    try {
+      this._lancedbModule = await import('@lancedb/lancedb');
+      this._arrowModule = await import('apache-arrow');
+    } catch (err) {
+      throw new Error(
+        `Failed to load LanceDB or Apache Arrow: ${err.message}. ` +
+        `Ensure they are installed (npm install @lancedb/lancedb apache-arrow).`
+      );
+    }
+  }
+
+  async _getDb() {
+    if (this._db) return this._db;
+    await this._initModules();
+    const dbPath = getDbPath(this.env);
+    if (!fs.existsSync(dbPath)) {
+      fs.mkdirSync(dbPath, { recursive: true });
+    }
+    this._db = await this._lancedbModule.connect(dbPath);
+    return this._db;
+  }
+
   async getEngineDimensions() {
     if (this._engineDim != null) return this._engineDim;
     try {
@@ -65,237 +96,227 @@ export class VectorClient {
     return this._engineDim;
   }
 
-  /**
-   * Read the actual declared dimensionality of `construct_embeddings.embedding`
-   * from `pg_attribute` and compare against the engine. Throws a precise
-   * error on mismatch. Returns the matched dimension on success.
-   */
-  async assertSchemaDimensions() {
-    const sql = await this._getSql();
-    if (!sql) throw new Error('No DATABASE_URL configured; cannot verify pgvector schema');
-    const engineDim = await this.getEngineDimensions();
-
-    const rows = await sql`
-      SELECT format_type(atttypid, atttypmod) AS pg_type
-      FROM pg_attribute
-      WHERE attrelid = 'construct_embeddings'::regclass
-        AND attname = 'embedding'
-        AND NOT attisdropped
-    `;
-    const declared = rows?.[0]?.pg_type || '';
-    const match = declared.match(/vector\((\d+)\)/);
-    if (!match) {
-      throw new Error(
-        `construct_embeddings.embedding column has unexpected type '${declared}'. ` +
-        `Recreate the column with 'ALTER TABLE construct_embeddings ALTER COLUMN embedding TYPE vector(${engineDim})' ` +
-        `(truncate first; vectors do not survive a dim change).`
-      );
-    }
-    const schemaDim = Number(match[1]);
-    if (schemaDim !== engineDim) {
-      throw new Error(
-        `pgvector dimension mismatch: schema declares vector(${schemaDim}) but engine model produces ${engineDim}d. ` +
-        `Either set CONSTRUCT_EMBEDDING_MODEL to a ${schemaDim}d model, or recreate the column with ` +
-        `'ALTER TABLE construct_embeddings ALTER COLUMN embedding TYPE vector(${engineDim})' (truncate first).`
-      );
+  async _getObservationsTable() {
+    const db = await this._getDb();
+    const tableName = 'observations_v1';
+    const dim = await this.getEngineDimensions();
+
+    try {
+      return await db.openTable(tableName);
+    } catch {
+      const schema = new this._arrowModule.Schema([
+        new this._arrowModule.Field('id', new this._arrowModule.Utf8()),
+        new this._arrowModule.Field('project', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('role', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('category', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('summary', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('content', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('tags', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('confidence', new this._arrowModule.Float32(), true),
+        new this._arrowModule.Field('source', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('git_sha', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('embedding', new this._arrowModule.FixedSizeList(dim, new this._arrowModule.Field('item', new this._arrowModule.Float32()))),
+        new this._arrowModule.Field('content_hash', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('model', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('created_at', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('updated_at', new this._arrowModule.Utf8(), true),
+      ]);
+      try {
+        return await db.createTable(tableName, [], { schema });
+      } catch (err) {
+        if (/already exists/i.test(String(err?.message))) return await db.openTable(tableName);
+        throw err;
+      }
     }
-    return schemaDim;
   }
 
-  /**
-   * Lazily load the `postgres` module and create the connection.
-   */
-  async _getSql() {
-    if (!this.url) return null;
-    if (this._sql) return this._sql;
+  async _getDocumentsTable() {
+    const db = await this._getDb();
+    const tableName = 'documents_v1';
+    const dim = await this.getEngineDimensions();
 
-    if (!this._pgModule) {
+    try {
+      return await db.openTable(tableName);
+    } catch {
+      const schema = new this._arrowModule.Schema([
+        new this._arrowModule.Field('id', new this._arrowModule.Utf8()),
+        new this._arrowModule.Field('project', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('kind', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('title', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('summary', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('body', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('source_path', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('tags', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('content_hash', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('embedding', new this._arrowModule.FixedSizeList(dim, new this._arrowModule.Field('item', new this._arrowModule.Float32()))),
+        new this._arrowModule.Field('model', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('created_at', new this._arrowModule.Utf8(), true),
+        new this._arrowModule.Field('updated_at', new this._arrowModule.Utf8(), true),
+      ]);
       try {
-        const m = await import('postgres');
-        this._pgModule = m.default || m;
+        return await db.createTable(tableName, [], { schema });
       } catch (err) {
-        // Re-throw with a clearer message so callers understand why
-        throw new Error(
-          `Failed to load 'postgres' module: ${err.message}. ` +
-          `Ensure 'postgres' is installed (npm install postgres).`
-        );
+        if (/already exists/i.test(String(err?.message))) return await db.openTable(tableName);
+        throw err;
       }
     }
-
-    this._sql = this._pgModule(this.url, DEFAULT_POOL);
-    return this._sql;
   }
 
-  /**
-   * Check if the SQL backend is available and healthy.
-   */
   async isHealthy() {
-    const sql = await this._getSql();
-    if (!sql) return false;
     try {
-      const result = await sql`SELECT 1 AS health`;
-      return result?.[0]?.health === 1;
+      await this._getDb();
+      return true;
     } catch {
       return false;
     }
   }
 
-  /**
-   * Check if pgvector extension is enabled.
-   */
   async isPgvectorEnabled() {
-    const sql = await this._getSql();
-    if (!sql) return false;
-    try {
-      const result = await sql`
-        SELECT EXISTS (
-          SELECT 1 FROM pg_extension WHERE extname = 'vector'
-        ) AS enabled
-      `;
-      return result?.[0]?.enabled === true;
-    } catch {
-      return false;
-    }
+    return this.isHealthy();
   }
 
-  /**
-   * Store an observation with its embedding.
-   */
-  async storeObservation({ id, project, role, category, summary, content, tags, confidence, source, embedding, gitSha, contentHash = null, model = null }) {
-    const sql = await this._getSql();
-    if (!sql) return { mode: 'file', reason: 'no_sql' };
-    const expected = await this.getEngineDimensions();
-    assertPgvectorDimensions(embedding, { expected, label: 'observation embedding' });
-
-    const embeddingVec = floatArrayToPgVector(embedding);
-    await sql`
-      INSERT INTO construct_observations (id, project, role, category, summary, content, tags, confidence, source, git_sha, embedding, content_hash, model)
-      VALUES (${id}, ${project}, ${role}, ${category}, ${summary}, ${content}, ${JSON.stringify(tags || [])}, ${confidence || 0.8}, ${source || null}, ${gitSha || null}, ${embeddingVec}, ${contentHash}, ${model})
-      ON CONFLICT (id) DO UPDATE SET
-        summary = EXCLUDED.summary,
-        content = EXCLUDED.content,
-        tags = EXCLUDED.tags,
-        confidence = EXCLUDED.confidence,
-        source = EXCLUDED.source,
-        git_sha = EXCLUDED.git_sha,
-        embedding = EXCLUDED.embedding,
-        content_hash = EXCLUDED.content_hash,
-        model = EXCLUDED.model,
-        updated_at = now()
-    `;
-    return { mode: 'sql', id };
+  async storeObservation(record) {
+    return serializeWrite(getDbPath(this.env), () => withWriteRetry(() => this._storeObservation(record)));
   }
 
-  // Fingerprint = the (content_hash, model) a stored observation was embedded
-  // with. Reconciliation compares this against the live content + current model
-  // to find rows needing re-embedding without re-reading every vector.
+  async _storeObservation(record) {
+    const table = await this._getObservationsTable();
+    const now = new Date().toISOString();
+
+    const data = {
+      id: record.id,
+      project: record.project || '',
+      role: record.role || 'unknown',
+      category: record.category || 'insight',
+      summary: record.summary || '',
+      content: record.content || '',
+      tags: JSON.stringify(record.tags || []),
+      confidence: Number(record.confidence || 0.8),
+      source: typeof record.source === 'object' ? JSON.stringify(record.source) : (record.source || ''),
+      git_sha: record.gitSha || '',
+      embedding: Array.from(record.embedding),
+      content_hash: record.contentHash || '',
+      model: record.model || '',
+      created_at: record.createdAt || now,
+      updated_at: now
+    };
+
+    await table.mergeInsert('id')
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute([data]);
+
+    return { mode: 'lancedb', id: record.id };
+  }
 
   async getObservationFingerprints(ids = []) {
-    const sql = await this._getSql();
-    if (!sql || !Array.isArray(ids) || ids.length === 0) return new Map();
-    const rows = await sql`
-      SELECT id, content_hash, model FROM construct_observations WHERE id = ANY(${ids})
-    `;
-    return new Map(rows.map((r) => [r.id, { contentHash: r.content_hash, model: r.model }]));
+    const table = await this._getObservationsTable();
+    if (!Array.isArray(ids) || ids.length === 0) return new Map();
+
+    const results = await table.query()
+      .where(`id IN (${ids.map(id => `'${id}'`).join(',')})`)
+      .select(['id', 'content_hash', 'model'])
+      .toArray();
+
+    return new Map(results.map((r) => [r.id, { contentHash: r.content_hash, model: r.model }]));
+  }
+
+  async searchObservations({ project, queryEmbedding, limit = 10, minSimilarity = 0.01, role, category }) {
+    const table = await this._getObservationsTable();
+
+    const query = table.query()
+      .nearestTo(Array.from(queryEmbedding))
+      .distanceType('cosine');
+
+    const rawResults = await query.limit(100).toArray();
+
+    return rawResults.map(r => {
+      const distance = r._distance !== undefined ? r._distance : 0;
+      let tags = [];
+      try {
+        tags = (typeof r.tags === 'string' && r.tags.startsWith('[')) ? JSON.parse(r.tags) : (Array.isArray(r.tags) ? r.tags : []);
+      } catch {}
+
+      return {
+        ...r,
+        similarity: 1 - distance,
+        tags
+      };
+    })
+    .filter(r => {
+      // Use fuzzy similarity and robust project matching
+      if (r.similarity < minSimilarity) return false;
+      if (project && r.project !== project) return false;
+      if (role && r.role && r.role !== role) return false;
+      if (category && r.category && r.category !== category) return false;
+      return true;
+    })
+    .slice(0, limit);
   }
 
-  /**
-   * Search observations by vector similarity.
-   */
-  async searchObservations({ project, queryEmbedding, limit = 10, minSimilarity = 0.3, role, category }) {
-    const sql = await this._getSql();
-    if (!sql) return [];
-    const expected = await this.getEngineDimensions();
-    assertPgvectorDimensions(queryEmbedding, { expected, label: 'observation query embedding' });
-
-    const queryVec = floatArrayToPgVector(queryEmbedding);
-    const conditions = [sql`project = ${project}`];
-    if (role) conditions.push(sql`role = ${role}`);
-    if (category) conditions.push(sql`category = ${category}`);
-    conditions.push(sql`1 - (embedding <=> ${queryVec}) > ${minSimilarity}`);
-
-    const results = await sql`
-      SELECT 
-        id, role, category, summary, content, tags, confidence, source, git_sha, created_at,
-        1 - (embedding <=> ${queryVec}) AS similarity
-      FROM construct_observations
-      WHERE ${sql.join(conditions, ' AND ')}
-      ORDER BY embedding <=> ${queryVec}
-      LIMIT ${limit}
-    `;
-    return results;
+  async storeDocument(record) {
+    return serializeWrite(getDbPath(this.env), () => withWriteRetry(() => this._storeDocument(record)));
   }
 
-  /**
-   * Store a document with its embedding.
-   */
-  async storeDocument({ id, project, kind, title, summary, body, sourcePath, tags, contentHash, embedding, model }) {
-    const sql = await this._getSql();
-    if (!sql) return { mode: 'file', reason: 'no_sql' };
-    const expected = await this.getEngineDimensions();
-    assertPgvectorDimensions(embedding, { expected, label: 'document embedding' });
-    if (!model) {
-      const info = await getEmbeddingModelInfo({ env: this.env });
-      model = info.model;
-    }
+  async _storeDocument(record) {
+    const table = await this._getDocumentsTable();
+    const now = new Date().toISOString();
+
+    const data = {
+      id: record.id,
+      project: record.project || '',
+      kind: record.kind || '',
+      title: record.title || '',
+      summary: record.summary || '',
+      body: record.body || '',
+      source_path: record.sourcePath || '',
+      tags: JSON.stringify(record.tags || []),
+      content_hash: record.contentHash || '',
+      embedding: Array.from(record.embedding),
+      model: record.model || '',
+      created_at: record.createdAt || now,
+      updated_at: now
+    };
 
-    const embeddingVec = floatArrayToPgVector(embedding);
-    await sql`
-      INSERT INTO construct_documents (id, project, kind, title, summary, body, source_path, tags, content_hash)
-      VALUES (${id}, ${project}, ${kind}, ${title}, ${summary || ''}, ${body}, ${sourcePath || ''}, ${JSON.stringify(tags || [])}, ${contentHash || ''})
-      ON CONFLICT (id) DO UPDATE SET
-        title = EXCLUDED.title,
-        summary = EXCLUDED.summary,
-        body = EXCLUDED.body,
-        source_path = EXCLUDED.source_path,
-        tags = EXCLUDED.tags,
-        content_hash = EXCLUDED.content_hash,
-        updated_at = now()
-    `;
-
-    await sql`
-      INSERT INTO construct_embeddings (document_id, model, embedding, content_hash)
-      VALUES (${id}, ${model}, ${embeddingVec}, ${contentHash || ''})
-      ON CONFLICT (document_id) DO UPDATE SET
-        model = EXCLUDED.model,
-        embedding = EXCLUDED.embedding,
-        content_hash = EXCLUDED.content_hash,
-        updated_at = now()
-    `;
-    return { mode: 'sql', id };
+    await table.mergeInsert('id')
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute([data]);
+
+    return { mode: 'lancedb', id: record.id };
   }
 
-  /**
-   * Search documents by vector similarity.
-   */
-  async searchDocuments({ project, queryEmbedding, limit = 10, minSimilarity = 0.3 }) {
-    const sql = await this._getSql();
-    if (!sql) return [];
-    const expected = await this.getEngineDimensions();
-    assertPgvectorDimensions(queryEmbedding, { expected, label: 'document query embedding' });
-
-    const queryVec = floatArrayToPgVector(queryEmbedding);
-    const results = await sql`
-      SELECT 
-        d.id, d.title, d.summary, d.body, d.source_path, d.tags, d.kind,
-        1 - (e.embedding <=> ${queryVec}) AS similarity
-      FROM construct_documents d
-      JOIN construct_embeddings e ON d.id = e.document_id
-      WHERE d.project = ${project}
-        AND 1 - (e.embedding <=> ${queryVec}) > ${minSimilarity}
-      ORDER BY e.embedding <=> ${queryVec}
-      LIMIT ${limit}
-    `;
-    return results;
+  async searchDocuments({ project, queryEmbedding, limit = 10, minSimilarity = 0.01 }) {
+    const table = await this._getDocumentsTable();
+
+    const query = table.query()
+      .nearestTo(Array.from(queryEmbedding))
+      .distanceType('cosine');
+
+    const rawResults = await query.limit(100).toArray();
+
+    return rawResults.map(r => {
+      const distance = r._distance !== undefined ? r._distance : 0;
+      let tags = [];
+      try {
+        tags = (typeof r.tags === 'string' && r.tags.startsWith('[')) ? JSON.parse(r.tags) : (Array.isArray(r.tags) ? r.tags : []);
+      } catch {}
+
+      return {
+        ...r,
+        similarity: 1 - distance,
+        tags
+      };
+    })
+    .filter(r => {
+      if (r.similarity < minSimilarity) return false;
+      if (project && r.project !== project) return false;
+      return true;
+    })
+    .slice(0, limit);
   }
 
-  /**
-   * Close the connection pool.
-   */
   async close() {
-    if (this._sql) {
-      await this._sql.end();
-      this._sql = null;
-    }
+    this._db = null;
   }
 }