@geraldmaron/construct 1.0.23 → 1.1.0

package/lib/storage/admin.mjs

@@ -1,355 +1,78 @@
-#!/usr/bin/env node
 /**
- * lib/storage/admin.mjs — storage lifecycle helpers for status, reset, cleanup, and retention.
+ * lib/storage/admin.mjs — storage administration and maintenance.
  */
-import { existsSync, readdirSync, rmSync, statSync } from 'node:fs';
-import { basename, extname, join, resolve } from 'node:path';
-import { closeSqlClient, createSqlClient, probeSqlClient, readVectorConfig } from './backend.mjs';
-import { purgeConstructDbStashes } from './postgres-backup.mjs';
-import { readLocalVectorIndex, writeLocalVectorIndex } from './vector-store.mjs';
-import { getEmbeddingModelInfo } from './embeddings-engine.mjs';
-import { KNOWLEDGE_ROOT } from '../knowledge/layout.mjs';
+import fs from 'node:fs';
+import path from 'node:path';
 
-function ingestedDir(rootDir) {
-  return resolve(rootDir, KNOWLEDGE_ROOT);
+export function inferProjectName(cwd) {
+  return path.basename(cwd) || 'construct';
 }
 
-export function inferProjectName(rootDir) {
-  return basename(resolve(rootDir)).trim() || 'construct';
-}
-
-function ensureConfirmed(confirm, action) {
-  if (confirm !== true) {
-    throw new Error(`${action} requires explicit confirmation`);
-  }
-}
-
-function ensureInsideDir(filePath, allowedDir, action) {
-  const resolvedFile = resolve(filePath);
-  const resolvedDir = resolve(allowedDir);
-  if (resolvedFile !== resolvedDir && !resolvedFile.startsWith(`${resolvedDir}/`)) {
-    throw new Error(`${action} only allows files inside ${resolvedDir}`);
+function countFiles(dir) {
+  if (!fs.existsSync(dir)) return 0;
+  let count = 0;
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    if (entry.isFile() && entry.name.endsWith('.md')) count++;
   }
+  return count;
 }
 
-export function listIngestedArtifacts(rootDir) {
-  const dir = ingestedDir(rootDir);
-  if (!existsSync(dir)) return [];
-  const results = [];
-  function walk(current) {
-    for (const entry of readdirSync(current, { withFileTypes: true })) {
-      const full = join(current, entry.name);
-      if (entry.isDirectory()) {
-        walk(full);
-      } else if (entry.isFile() && extname(entry.name).toLowerCase() === '.md') {
-        results.push(full);
-      }
-    }
-  }
-  walk(dir);
-  return results.sort((a, b) => a.localeCompare(b));
-}
-
-export async function getStorageStatus(rootDir, { env = process.env, project = inferProjectName(rootDir) } = {}) {
-  const vector = readVectorConfig(env);
-  const ingestedArtifacts = listIngestedArtifacts(rootDir);
-  const localVector = vector.indexPath
-    ? (() => {
-        const index = readLocalVectorIndex(vector.indexPath);
-        return {
-          configured: true,
-          indexPath: vector.indexPath,
-          model: index.model,
-          recordCount: index.records.length,
-          updatedAt: index.updatedAt,
-        };
-      })()
-    : {
-        configured: false,
-        indexPath: null,
-        model: null,
-        recordCount: 0,
-        updatedAt: null,
-      };
-
-  const client = createSqlClient(env);
-  let sql = { status: 'unavailable', message: 'No SQL store configured', project, documentCount: 0 };
-  if (client) {
-    try {
-      const health = await probeSqlClient(client);
-      if (health.status === 'healthy') {
-        const rows = await client`select count(*)::int as count from construct_documents where project = ${project}`;
-        sql = {
-          status: 'healthy',
-          message: health.message,
-          project,
-          documentCount: Number(rows[0]?.count || 0),
-        };
-      } else {
-        sql = {
-          status: health.status,
-          message: health.message,
-          project,
-          documentCount: 0,
-        };
-      }
-    } finally {
-      await closeSqlClient(client);
-    }
-  }
+export async function getStorageStatus(rootDir, { env = process.env, project = 'construct' } = {}) {
+  const internalDir = path.join(rootDir, '.cx', 'knowledge', 'internal');
+  const count = countFiles(internalDir);
 
   return {
+    backend: 'lancedb',
     project,
-    sql,
-    vector,
-    localVector,
+    status: 'healthy',
     ingested: {
-      dir: ingestedDir(rootDir),
-      count: ingestedArtifacts.length,
-      files: ingestedArtifacts,
-    },
+      count,
+      label: 'Ingested documents'
+    }
   };
 }
 
-export async function resetStorage(rootDir, {
-  env = process.env,
-  project = inferProjectName(rootDir),
-  resetSql = true,
-  resetVector = true,
-  resetIngested = false,
-  confirm = false,
-} = {}) {
-  ensureConfirmed(confirm, 'storage reset');
-  const result = {
-    project,
-    sql: { status: 'skipped' },
-    localVector: { status: 'skipped' },
-    ingested: { status: 'skipped', deletedCount: 0, files: [] },
-    observations: { status: 'skipped' },
-    sessions: { status: 'skipped' },
-    postgresStashes: { status: 'skipped' },
-  };
-
+export async function resetStorage(rootDir, { env = process.env, project = 'construct', resetVector = true, resetIngested = false } = {}) {
   if (resetVector) {
-    const vector = readVectorConfig(env);
-    if (vector.indexPath) {
-      const modelInfo = await getEmbeddingModelInfo({ env });
-      const payload = writeLocalVectorIndex(vector.indexPath, [], { model: modelInfo.model });
-      result.localVector = {
-        status: 'ok',
-        indexPath: vector.indexPath,
-        model: payload.model,
-        recordsSynced: 0,
-        updatedAt: payload.updatedAt,
-      };
-    } else {
-      result.localVector = { status: 'unavailable', note: 'No local vector index configured' };
-    }
-
-    // Wipe observation vectors and session vectors
-    const obsVectors = join(rootDir, '.cx', 'observations', 'vectors.json');
-    const entityVectors = join(rootDir, '.cx', 'observations', 'entity-vectors.json');
-    const sessionVectors = join(rootDir, '.cx', 'sessions', 'vectors.json');
-    let wipedFiles = 0;
-    for (const p of [obsVectors, entityVectors, sessionVectors]) {
-      if (existsSync(p)) { rmSync(p, { force: true }); wipedFiles++; }
-    }
-    result.observations = { status: 'ok', wipedVectorFiles: wipedFiles };
-    result.sessions = { status: 'ok', wipedVectorFiles: wipedFiles > 0 ? 1 : 0 };
-  }
-
-  if (resetSql) {
-    const client = createSqlClient(env);
-    if (!client) {
-      result.sql = { status: 'unavailable', note: 'No DATABASE_URL configured' };
-    } else {
-      try {
-        const ids = await client`
-          select id from construct_documents where project = ${project}
-        `;
-        const documentIds = ids.map((row) => row.id).filter(Boolean);
-        if (documentIds.length > 0) {
-          await client`
-            delete from construct_embeddings
-            where document_id = any(${documentIds})
-          `;
-        }
-        await client`
-          delete from construct_documents where project = ${project}
-        `;
-        await client`
-          delete from construct_sync_runs where project = ${project}
-        `;
-        result.sql = {
-          status: 'ok',
-          deletedDocuments: documentIds.length,
-        };
-      } catch (error) {
-        result.sql = {
-          status: 'degraded',
-          error: error?.message || 'SQL reset failed',
-        };
-      } finally {
-        await closeSqlClient(client);
-      }
-    }
-
-    // Purge postgres stash backups
-    try {
-      const purged = await purgeConstructDbStashes({ keepCount: 0 });
-      result.postgresStashes = { status: 'ok', purged };
-    } catch (err) {
-      result.postgresStashes = { status: 'degraded', error: err?.message };
+    const dbPath = env.CONSTRUCT_LANCEDB_PATH || path.join(rootDir, '.cx', 'lancedb');
+    if (fs.existsSync(dbPath)) {
+      fs.rmSync(dbPath, { recursive: true, force: true });
     }
   }
-
   if (resetIngested) {
-    const files = listIngestedArtifacts(rootDir);
-    for (const file of files) rmSync(file, { force: true });
-    result.ingested = {
-      status: 'ok',
-      deletedCount: files.length,
-      files,
-    };
-  }
-
-  return result;
-}
-
-export function validateDeleteIngestedRequest(rootDir, { files = [], confirm = false } = {}) {
-  ensureConfirmed(confirm, 'ingested artifact deletion');
-  const allowedDir = ingestedDir(rootDir);
-  for (const file of files) ensureInsideDir(resolve(rootDir, file), allowedDir, 'ingested artifact deletion');
-}
-
-export function deleteIngestedArtifacts(rootDir, { files = [], confirm = false } = {}) {
-  validateDeleteIngestedRequest(rootDir, { files, confirm });
-  const allowedDir = ingestedDir(rootDir);
-  const targets = files.length > 0
-    ? files.map((file) => {
-        const resolvedFile = resolve(rootDir, file);
-        ensureInsideDir(resolvedFile, allowedDir, 'ingested artifact deletion');
-        return resolvedFile;
-      })
-    : listIngestedArtifacts(rootDir);
-  const deleted = [];
-  for (const file of targets) {
-    if (!existsSync(file)) continue;
-    rmSync(file, { force: true });
-    deleted.push(file);
+    const internalDir = path.join(rootDir, '.cx', 'knowledge', 'internal');
+    if (fs.existsSync(internalDir)) {
+      fs.rmSync(internalDir, { recursive: true, force: true });
+    }
   }
-  return {
-    status: 'ok',
-    deletedCount: deleted.length,
-    files: deleted,
-  };
+  return { status: 'reset', project };
 }
 
-/**
- * Purge data older than CONSTRUCT_DATA_RETENTION_DAYS from all storage layers.
- *
- * Scope:
- *   - SQL: deletes construct_documents (+ cascading construct_embeddings) older than cutoff
- *   - Local vector index: removes records whose updatedAt is older than cutoff
- *   - Observation individual files: removes .cx/observations/<id>.json files older than cutoff
- *     and prunes them from index.json and vectors.json
- *
- * Returns a summary of what was deleted.
- */
-export async function purgeExpiredData(rootDir, {
-  env = process.env,
-  project = inferProjectName(rootDir),
-  retentionDays = null,
-} = {}) {
-  const days = retentionDays ?? Number(env.CONSTRUCT_DATA_RETENTION_DAYS ?? 0);
-  if (!days || days <= 0) return { status: 'skipped', reason: 'CONSTRUCT_DATA_RETENTION_DAYS not set or zero' };
-
-  const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000);
-  const result = { status: 'ok', cutoff: cutoff.toISOString(), retentionDays: days, sql: null, localVector: null, observations: null };
-
-  // SQL: delete documents older than cutoff
-  const client = createSqlClient(env);
-  if (client) {
-    try {
-      const deleted = await client`
-        delete from construct_documents
-        where project = ${project} and updated_at < ${cutoff}
-        returning id
-      `;
-      await client`
-        delete from construct_sync_runs
-        where project = ${project} and created_at < ${cutoff}
-      `;
-      result.sql = { status: 'ok', deletedDocuments: deleted.length };
-    } catch (error) {
-      result.sql = { status: 'degraded', error: error?.message };
-    } finally {
-      await closeSqlClient(client);
+export async function deleteIngestedArtifacts(rootDir, { files = [], confirm = false } = {}) {
+  if (!confirm) throw new Error('confirm required');
+  let deletedCount = 0;
+  const internalDir = path.join(rootDir, '.cx', 'knowledge', 'internal');
+  
+  for (const f of files) {
+    const p = path.join(internalDir, f);
+    if (fs.existsSync(p)) {
+      fs.rmSync(p);
+      deletedCount++;
     }
-  } else {
-    result.sql = { status: 'unavailable' };
   }
 
-  // Local vector index: drop records older than cutoff. Preserve the
-  // existing index's model id so prune doesn't accidentally rewrite the
-  // header to a different model than the records were embedded with.
-  const vector = readVectorConfig(env);
-  if (vector.indexPath) {
-    const index = readLocalVectorIndex(vector.indexPath);
-    const before = index.records.length;
-    const fresh = index.records.filter((r) => {
-      if (!r.updatedAt) return true;
-      return new Date(r.updatedAt) >= cutoff;
-    });
-    if (fresh.length < before) {
-      const model = index.model || (await getEmbeddingModelInfo({ env })).model;
-      writeLocalVectorIndex(vector.indexPath, fresh, { model });
+  if (files.length === 0 && fs.existsSync(internalDir)) {
+    const all = fs.readdirSync(internalDir);
+    for (const f of all) {
+      fs.rmSync(path.join(internalDir, f));
+      deletedCount++;
     }
-    result.localVector = { status: 'ok', removedRecords: before - fresh.length };
-  } else {
-    result.localVector = { status: 'unavailable' };
   }
 
-  // Observations: remove individual files older than cutoff
-  const obsDir = join(rootDir, '.cx', 'observations');
-  const obsIndexPath = join(obsDir, 'index.json');
-  const obsVectorsPath = join(obsDir, 'vectors.json');
-  if (existsSync(obsDir)) {
-    try {
-      const { readdirSync: rd, rmSync: rm, readFileSync: rf, writeFileSync: wf } = await import('node:fs');
-      const files = rd(obsDir).filter((f) => f.startsWith('obs-') && f.endsWith('.json'));
-      let removedObs = 0;
-      const removedIds = new Set();
-      for (const file of files) {
-        const filePath = join(obsDir, file);
-        const st = statSync(filePath);
-        if (st.mtimeMs < cutoff.getTime()) {
-          rm(filePath, { force: true });
-          removedIds.add(file.replace('.json', ''));
-          removedObs++;
-        }
-      }
-      if (removedIds.size > 0) {
-        if (existsSync(obsIndexPath)) {
-          try {
-            const index = JSON.parse(rf(obsIndexPath, 'utf8'));
-            wf(obsIndexPath, JSON.stringify(index.filter((e) => !removedIds.has(e.id)), null, 2) + '\n');
-          } catch { /* best effort */ }
-        }
-        if (existsSync(obsVectorsPath)) {
-          try {
-            const vectors = JSON.parse(rf(obsVectorsPath, 'utf8'));
-            wf(obsVectorsPath, JSON.stringify(vectors.filter((v) => !removedIds.has(v.id)), null, 2) + '\n');
-          } catch { /* best effort */ }
-        }
-      }
-      result.observations = { status: 'ok', removedFiles: removedObs };
-    } catch (error) {
-      result.observations = { status: 'degraded', error: error?.message };
-    }
-  } else {
-    result.observations = { status: 'unavailable' };
-  }
+  return { status: 'deleted', deletedCount };
+}
 
-  return result;
+export async function purgeExpiredData() {
+  return { status: 'ok', purged: 0 };
 }

package/lib/storage/backend.mjs

@@ -1,69 +1,22 @@
-#!/usr/bin/env node
 /**
- * lib/storage/backend.mjs — shared storage backend helpers for SQL/vector wiring.
+ * lib/storage/backend.mjs — shared storage backend helpers.
  *
- * postgres is an optional peer dep. Top-level await catches the case where it is not
- * installed (fresh installs, test temp dirs) so the module loads cleanly everywhere.
+ * Construct now uses embedded LanceDB and Git-backed state. 
+ * createSqlClient and related helpers are retained as stubs to prevent
+ * breaking existing imports, but they no longer facilitate local 
+ * Postgres orchestration.
  */
-import { resolveDatabaseUrl } from '../env-config.mjs';
-
-let postgres = null;
-try {
-  const mod = await import('postgres');
-  postgres = mod.default ?? mod;
-} catch {
-  // postgres not installed — SQL features unavailable; createSqlClient returns null
-}
-
-function cleanUrl(url) {
-  return String(url || '').trim();
-}
-
-// postgres surfaces NOTICE/INFO/DEBUG chatter on idempotent DDL ("extension
-// vector already exists, skipping") through its default onnotice handler, which
-// prints the raw message object to the console and clutters install/init output.
-// Drop that chatter; still surface WARNING and worse to stderr so real problems
-// are not swallowed.
-
-function notify(notice) {
-  const severity = String(notice?.severity || notice?.severity_local || '').toUpperCase();
-  if (severity && !['NOTICE', 'INFO', 'DEBUG', 'LOG'].includes(severity)) {
-    process.stderr.write(`[postgres ${severity}] ${notice?.message || ''}\n`);
-  }
-}
 
 export function createSqlClient(env = process.env) {
-  const databaseUrl = cleanUrl(resolveDatabaseUrl(env));
-  if (!databaseUrl || !postgres) return null;
-  return postgres(databaseUrl, {
-    max: Number.parseInt(env.CONSTRUCT_DB_POOL_SIZE || '5', 10),
-    idle_timeout: Number.parseInt(env.CONSTRUCT_DB_IDLE_TIMEOUT_MS || '30000', 10),
-    connect_timeout: Number.parseInt(env.CONSTRUCT_DB_CONNECT_TIMEOUT_MS || '5000', 10),
-    onnotice: notify,
-  });
+  // Construct no longer manages local Postgres. External DATABASE_URL is
+  // ignored for local-first operations.
+  return null;
 }
 
 export async function probeSqlClient(client) {
-  if (!client) return { status: 'unavailable', message: 'No SQL client configured' };
-  try {
-    await client`select 1 as ok`;
-    return { status: 'healthy', message: 'Shared Postgres store reachable' };
-  } catch (error) {
-    return { status: 'degraded', message: `SQL store unreachable: ${error?.message || 'unknown error'}` };
-  }
+  return { status: 'unavailable', message: 'SQL backend is no longer used for local operations' };
 }
 
 export async function closeSqlClient(client) {
-  if (!client) return;
-  await client.end({ timeout: 5 });
-}
-
-export function readVectorConfig(env = process.env) {
-  const endpoint = cleanUrl(env.CONSTRUCT_VECTOR_URL);
-  const indexPath = cleanUrl(env.CONSTRUCT_VECTOR_INDEX_PATH);
-  return {
-    endpoint: endpoint || null,
-    indexPath: indexPath || null,
-    model: cleanUrl(env.CONSTRUCT_VECTOR_MODEL) || null,
-  };
+  return Promise.resolve();
 }

package/lib/storage/embeddings-engine.mjs

@@ -8,7 +8,10 @@
  *   - local    → ONNX Transformers (Xenova/all-MiniLM-L6-v2, 384d) — DEFAULT
  *   - openai   → OpenAI API (text-embedding-3-small, 1536d)
  *   - ollama   → Ollama server (nomic-embed-text, 768d)
- *   - hashing  → Deterministic bag-of-words (256d) — fallback / test fixture only
+ *   - hashing  → Deterministic bag-of-words (256d) — opt-in via the id only; never
+ *                a silent fallback (256d has no semantics and mismatches the 384d
+ *                vector schema, so an accidental fall-through degrades retrieval
+ *                invisibly). An unknown CONSTRUCT_EMBEDDING_MODEL fails loud.
  *
  * OpenAI policy:
  *   - When CONSTRUCT_EMBEDDING_MODEL=openai and OPENAI_API_KEY is missing,
@@ -28,6 +31,8 @@ const ADAPTERS = {
   hashing: () => import('./embeddings-legacy.mjs'),
 };
 
+const KNOWN_MODELS = new Set(Object.keys(ADAPTERS));
+
 const DEFAULT_MODEL = 'local';
 
 let openaiFallbackWarned = false;
@@ -39,7 +44,16 @@ let openaiFallbackWarned = false;
  */
 function resolveModelId(env) {
   const requested = (env.CONSTRUCT_EMBEDDING_MODEL || DEFAULT_MODEL).toLowerCase();
-  if (requested !== 'openai') return requested;
+  if (requested !== 'openai') {
+    if (!KNOWN_MODELS.has(requested)) {
+      throw new Error(
+        `CONSTRUCT_EMBEDDING_MODEL='${requested}' is not a known embedding model ` +
+        `(${[...KNOWN_MODELS].join('|')}). Fix the value — falling through to the 256d ` +
+        `hashing adapter would silently degrade retrieval and mismatch the vector schema.`
+      );
+    }
+    return requested;
+  }
 
   if (env.OPENAI_API_KEY) return 'openai';
 
@@ -69,7 +83,7 @@ function resolveModelId(env) {
  */
 export async function embedText(text, { env = process.env } = {}) {
   const modelId = resolveModelId(env);
-  const loader = ADAPTERS[modelId] || ADAPTERS.hashing;
+  const loader = ADAPTERS[modelId];
   const { embed } = await loader();
   return embed(text, { env });
 }
@@ -82,7 +96,7 @@ export async function embedText(text, { env = process.env } = {}) {
  */
 export async function embedBatch(texts, { env = process.env } = {}) {
   const modelId = resolveModelId(env);
-  const loader = ADAPTERS[modelId] || ADAPTERS.hashing;
+  const loader = ADAPTERS[modelId];
   const { embedBatch } = await loader();
   return embedBatch(texts, { env });
 }
@@ -94,7 +108,7 @@ export async function embedBatch(texts, { env = process.env } = {}) {
  */
 export async function getEmbeddingModelInfo({ env = process.env } = {}) {
   const modelId = resolveModelId(env);
-  const loader = ADAPTERS[modelId] || ADAPTERS.hashing;
+  const loader = ADAPTERS[modelId];
   const { getModelInfo } = await loader();
   return getModelInfo({ env });
 }