@geraldmaron/construct 1.4.2 → 1.5.1

package/lib/storage/admin.mjs

@@ -1,9 +1,24 @@
 /**
  * lib/storage/admin.mjs — storage administration and maintenance.
+ *
+ * purgeExpiredData() is the TTL/size eviction path for the machine-scoped
+ * vector-indexed observations store (ADR-0066: <stateRoot>/lancedb,
+ * construct-rf26.17). Checking or pruning never provisions the LanceDB
+ * index as a side effect — a project that never ran `construct ingest`
+ * has no lancedb/ dir yet, so the call is a no-op rather than a trigger to
+ * materialize one. Caps resolve env var > construct.config.json
+ * (resources.disk.observationsMaxDays / observationsMaxRows) > default,
+ * matching the precedence documented in lib/config/project-config.mjs.
  */
 import fs from 'node:fs';
 import path from 'node:path';
 import { resolveWithinRoot } from '../path-policy.mjs';
+import { resolveStateDir } from '../state-root.mjs';
+import { loadProjectConfig, resolveSetting } from '../config/project-config.mjs';
+import { VectorClient } from './vector-client.mjs';
+
+const OBSERVATIONS_MAX_DAYS_DEFAULT = 90;
+const OBSERVATIONS_MAX_ROWS_DEFAULT = 5000;
 
 export function inferProjectName(cwd) {
   return path.basename(cwd) || 'construct';
@@ -19,14 +34,24 @@ function countFiles(dir) {
   return count;
 }
 
-export async function getStorageStatus(rootDir, { env = process.env, project = 'construct' } = {}) {
+export async function getStorageStatus(rootDir, { env = process.env, project = 'construct', fsExistsSync = fs.existsSync } = {}) {
   const internalDir = path.join(rootDir, '.cx', 'knowledge', 'internal');
   const count = countFiles(internalDir);
 
+  // Health mirrors probeStorageHealth (lib/status.mjs), derived from filesystem state
+  // rather than hardcoded: no .cx/ is unavailable, no lancedb store yet is degraded
+  // (no vector index), both present is healthy. The lancedb store itself resolves
+  // through the same machine-scoped state root (ADR-0066) resetStorage below already
+  // uses, not a project-relative `.cx/lancedb` — an override wins when set.
+
+  const cxExists = fsExistsSync(path.join(rootDir, '.cx'));
+  const lancedbExists = fsExistsSync(env.CONSTRUCT_LANCEDB_PATH || resolveStateDir(rootDir, 'lancedb', { ensureDir: false }));
+  const status = !cxExists ? 'unavailable' : lancedbExists ? 'healthy' : 'degraded';
+
   return {
     backend: 'lancedb',
     project,
-    status: 'healthy',
+    status,
     ingested: {
       count,
       label: 'Ingested documents'
@@ -36,7 +61,7 @@ export async function getStorageStatus(rootDir, { env = process.env, project = '
 
 export async function resetStorage(rootDir, { env = process.env, project = 'construct', resetVector = true, resetIngested = false } = {}) {
   if (resetVector) {
-    const dbPath = env.CONSTRUCT_LANCEDB_PATH || path.join(rootDir, '.cx', 'lancedb');
+    const dbPath = env.CONSTRUCT_LANCEDB_PATH || resolveStateDir(rootDir, 'lancedb', { ensureDir: false });
     if (fs.existsSync(dbPath)) {
       fs.rmSync(dbPath, { recursive: true, force: true });
     }
@@ -78,6 +103,53 @@ export async function deleteIngestedArtifacts(rootDir, { files = [], confirm = f
   return { status: 'deleted', deletedCount };
 }
 
-export async function purgeExpiredData() {
-  return { status: 'ok', purged: 0 };
+// Recursive byte count for the "current size" figure doctor/prune report —
+// LanceDB stores each table as a directory of Lance fragment files, so a
+// single stat() on the db path itself would always read as 0.
+
+function dirSizeBytes(dir) {
+  if (!fs.existsSync(dir)) return 0;
+  let bytes = 0;
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    bytes += entry.isDirectory() ? dirSizeBytes(full) : fs.statSync(full).size;
+  }
+  return bytes;
+}
+
+/**
+ * Evict expired/overflow rows from the machine-scoped observations_v1 table.
+ * A no-op when the vector index was never provisioned — nothing to evict,
+ * and provisioning one just to check would defeat the lazy-index contract.
+ * Returns { status: 'skipped', reason } in that case, or
+ * { status: 'ok', evictedCount, remainingCount, oldestRetainedAt, sizeBytes,
+ *   maxAgeDays, maxRows } once a purge ran.
+ */
+export async function purgeExpiredData(rootDir, { env = process.env, project = 'construct', maxAgeDays, maxRows } = {}) {
+  const dbPath = env.CONSTRUCT_LANCEDB_PATH || resolveStateDir(rootDir, 'lancedb', { ensureDir: false });
+  if (!fs.existsSync(dbPath)) {
+    return { status: 'skipped', reason: 'no vector index provisioned yet' };
+  }
+
+  const { config } = loadProjectConfig(rootDir, env);
+  const resolvedMaxAgeDays = maxAgeDays ?? Number(resolveSetting({
+    config, env, jsonPath: 'resources.disk.observationsMaxDays',
+    envKey: 'CONSTRUCT_OBSERVATIONS_MAX_DAYS', defaultValue: OBSERVATIONS_MAX_DAYS_DEFAULT,
+  }).value);
+  const resolvedMaxRows = maxRows ?? Number(resolveSetting({
+    config, env, jsonPath: 'resources.disk.observationsMaxRows',
+    envKey: 'CONSTRUCT_OBSERVATIONS_MAX_ROWS', defaultValue: OBSERVATIONS_MAX_ROWS_DEFAULT,
+  }).value);
+
+  const vectorClient = new VectorClient({ env: { ...env, CONSTRUCT_LANCEDB_PATH: dbPath } });
+  const result = await vectorClient.pruneObservations({ maxAgeDays: resolvedMaxAgeDays, maxRows: resolvedMaxRows });
+
+  return {
+    status: 'ok',
+    project,
+    ...result,
+    sizeBytes: dirSizeBytes(dbPath),
+    maxAgeDays: resolvedMaxAgeDays,
+    maxRows: resolvedMaxRows,
+  };
 }

package/lib/storage/backend-registry.mjs

@@ -0,0 +1,73 @@
+/**
+ * lib/storage/backend-registry.mjs — kind:'storage' provider resolution.
+ *
+ * Run-store selection uses the same extension-manifest substrate as queue
+ * selection: builtin backends are always available, while project/user storage
+ * manifests can name an implementation without central dispatch edits.
+ */
+
+import { loadManifestsFromDir, mergeManifests, resolveManifestDirs } from '../extensions/loader.mjs';
+
+export const BUILTIN_STORAGE_BACKENDS = ['filesystem', 'sqlite', 'postgres'];
+export const SUPPORTED_RUN_STORE_IMPLEMENTATIONS = ['filesystem', 'sqlite', 'postgres'];
+
+export class UnregisteredStorageProviderError extends Error {
+  constructor(backend) {
+    super(
+      `Storage backend '${backend}' has no registered kind:'storage' provider. ` +
+      `Builtin providers: ${BUILTIN_STORAGE_BACKENDS.join(', ')}.`,
+    );
+    this.name = 'UnregisteredStorageProviderError';
+    this.backend = backend;
+  }
+}
+
+export class UnsupportedStorageProviderError extends Error {
+  constructor(backend, implementation) {
+    super(
+      `Storage backend '${backend}' resolves to unsupported run-store implementation ` +
+      `'${implementation || 'unknown'}'. Supported implementations: ` +
+      `${SUPPORTED_RUN_STORE_IMPLEMENTATIONS.join(', ')}.`,
+    );
+    this.name = 'UnsupportedStorageProviderError';
+    this.backend = backend;
+    this.implementation = implementation || null;
+  }
+}
+
+export function listStorageProviders({ rootDir = process.cwd(), env = process.env } = {}) {
+  const homeDir = env?.HOME || env?.USERPROFILE || undefined;
+  const dirs = resolveManifestDirs({ rootDir, homeDir });
+  const builtin = loadManifestsFromDir(dirs.builtin).manifests;
+  const user = loadManifestsFromDir(dirs.user).manifests;
+  const project = loadManifestsFromDir(dirs.project).manifests;
+  return mergeManifests(builtin, user, project).filter((m) => m.kind === 'storage');
+}
+
+export function findStorageProvider(backend, { rootDir = process.cwd(), env = process.env } = {}) {
+  if (!backend) return null;
+  return listStorageProviders({ rootDir, env }).find((m) => m.id === backend) || null;
+}
+
+export function runStoreImplementationFor(provider) {
+  const impl = provider?.operations?.runStore || provider?.runStore || provider?.implementation;
+  return typeof impl === 'string' && impl.trim() ? impl.trim().toLowerCase() : null;
+}
+
+export function resolveStorageBackend(backend, { rootDir = process.cwd(), env = process.env } = {}) {
+  const requested = String(backend || '').trim().toLowerCase();
+  if (!requested) return null;
+  if (BUILTIN_STORAGE_BACKENDS.includes(requested)) {
+    return { backend: requested, implementation: requested, provider: null };
+  }
+
+  const provider = findStorageProvider(requested, { rootDir, env });
+  if (!provider) throw new UnregisteredStorageProviderError(requested);
+
+  const implementation = runStoreImplementationFor(provider);
+  if (!SUPPORTED_RUN_STORE_IMPLEMENTATIONS.includes(implementation)) {
+    throw new UnsupportedStorageProviderError(requested, implementation);
+  }
+
+  return { backend: requested, implementation, provider };
+}

package/lib/storage/backend.mjs

@@ -1,22 +1,76 @@
 /**
  * lib/storage/backend.mjs — shared storage backend helpers.
  *
- * Construct uses embedded LanceDB and Git-backed state. createSqlClient and
- * related helpers are stubs that keep existing imports working without
- * orchestrating local Postgres.
+ * Construct uses embedded LanceDB and Git-backed state by default. When a
+ * Postgres URL is configured, createSqlClient returns the optional Postgres.js
+ * tagged-template client for shared run-store and migration paths.
  */
 
-export function createSqlClient(env = process.env) {
-  // Stub: local-first operations skip Postgres, so external DATABASE_URL has
-  // no effect here.
+import { createRequire } from 'node:module';
+import { resolveDatabaseUrl } from '../env-config.mjs';
 
-  return null;
+const require = createRequire(import.meta.url);
+
+function clean(value) {
+  return String(value ?? '').trim();
+}
+
+function positiveInt(value, fallback) {
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function resolvePostgresUrl(env) {
+  return clean(env?.DATABASE_URL || env?.CONSTRUCT_DATABASE_URL || resolveDatabaseUrl(env));
+}
+
+function resolveSsl(value) {
+  const raw = clean(value).toLowerCase();
+  if (!raw || raw === 'false' || raw === '0' || raw === 'disable') return false;
+  if (raw === 'true' || raw === '1') return true;
+  if (['require', 'prefer', 'verify-full'].includes(raw)) return raw;
+  return false;
+}
+
+function loadPostgresFactory(postgresFactory) {
+  if (postgresFactory) return postgresFactory;
+  try {
+    const mod = require('postgres');
+    return mod.default || mod;
+  } catch (err) {
+    if (err?.code === 'MODULE_NOT_FOUND' || err?.code === 'ERR_MODULE_NOT_FOUND') return null;
+    throw err;
+  }
+}
+
+export function createSqlClient(env = process.env, { postgresFactory } = {}) {
+  const url = resolvePostgresUrl(env);
+  if (!url) return null;
+
+  const postgres = loadPostgresFactory(postgresFactory);
+  if (!postgres) return null;
+
+  return postgres(url, {
+    max: positiveInt(env?.CONSTRUCT_DB_MAX_CONNECTIONS, 5),
+    idle_timeout: positiveInt(env?.CONSTRUCT_DB_IDLE_TIMEOUT_SECONDS, 20),
+    connect_timeout: positiveInt(env?.CONSTRUCT_DB_CONNECT_TIMEOUT_SECONDS, 10),
+    ssl: resolveSsl(env?.CONSTRUCT_DB_SSL || env?.PGSSLMODE),
+    prepare: true,
+    onnotice: false,
+    connection: { application_name: clean(env?.CONSTRUCT_DB_APPLICATION_NAME) || 'construct' },
+  });
 }
 
 export async function probeSqlClient(client) {
-  return { status: 'unavailable', message: 'SQL backend is no longer used for local operations' };
+  if (!client) return { status: 'unavailable', message: 'SQL client is not configured. Set DATABASE_URL or CONSTRUCT_DATABASE_URL.' };
+  try {
+    await client`SELECT 1 AS ok`;
+    return { status: 'available', message: 'Postgres connection succeeded' };
+  } catch (err) {
+    return { status: 'unavailable', message: err?.message || String(err) };
+  }
 }
 
 export async function closeSqlClient(client) {
-  return Promise.resolve();
+  if (client?.end) await client.end({ timeout: 5 });
 }

package/lib/storage/embeddings-openai.mjs

@@ -5,10 +5,20 @@
  * Supports native batching for up to 2048 inputs per request.
  */
 
+import { resolveSecret } from '../providers/secret-resolver.mjs';
+
 const MODEL_ID = 'text-embedding-3-small';
 const DIMENSIONS = 1536;
 const MAX_BATCH_SIZE = 2048;
 
+function resolveOpenAiKey(env) {
+  try {
+    return resolveSecret('OPENAI_API_KEY', { env, allowAmbient: env === process.env });
+  } catch {
+    return null;
+  }
+}
+
 async function openaiEmbed(key, input) {
   const res = await fetch('https://api.openai.com/v1/embeddings', {
     method: 'POST',
@@ -34,7 +44,7 @@ async function openaiEmbed(key, input) {
  * @returns {Promise<{ embedding: Float32Array, model: string, dimensions: number }>}
  */
 export async function embed(text, { env = process.env } = {}) {
-  const key = env.OPENAI_API_KEY;
+  const key = resolveOpenAiKey(env);
   if (!key) throw new Error('OPENAI_API_KEY required for OpenAI embeddings');
 
   const data = await openaiEmbed(key, text);
@@ -52,7 +62,7 @@ export async function embed(text, { env = process.env } = {}) {
  * @returns {Promise<{ embedding: Float32Array, model: string, dimensions: number }[]>}
  */
 export async function embedBatch(texts, { env = process.env } = {}) {
-  const key = env.OPENAI_API_KEY;
+  const key = resolveOpenAiKey(env);
   if (!key) throw new Error('OPENAI_API_KEY required for OpenAI embeddings');
 
   const results = [];

package/lib/storage/hybrid-query.mjs

@@ -1,9 +1,14 @@
 /**
  * lib/storage/hybrid-query.mjs — unified search across knowledge and facts.
  *
- * Stub for the LanceDB-backed search.
+ * Stub for the LanceDB-backed search. Results pass through the N4
+ * retrieval-hardening pipeline (lib/storage/retrieval-hardening.mjs) before
+ * being returned, so vector-store poisoning defenses (trust-weighted
+ * ranking, per-source recall cap, similarity sanity threshold, duplicate
+ * collapse) apply to every hybrid-query caller uniformly.
  */
 import { VectorClient } from './vector-client.mjs';
+import { hardenRetrieval } from './retrieval-hardening.mjs';
 
 export async function buildHybridSearchResultsAsync(rootDir, query, { limit = 10, env = process.env } = {}) {
   // Embed the query
@@ -17,8 +22,11 @@ export async function buildHybridSearchResultsAsync(rootDir, query, { limit = 10
   const docs = await client.searchDocuments({ project: 'construct', queryEmbedding: embedding, limit });
   const obs = await client.searchObservations({ project: 'construct', queryEmbedding: embedding, limit });
 
-  // Merge both result sets and rank by similarity
+  // Merge both result sets, then rank by similarity, provenance, and recall
+  // caps rather than similarity alone — an adversarial embedding tuned for
+  // top cosine similarity must not dominate assembled context.
 
   const all = [...docs.map(d => ({ ...d, type: 'document' })), ...obs.map(o => ({ ...o, type: 'observation' }))];
-  return all.sort((a, b) => b.similarity - a.similarity).slice(0, limit);
+  const ranked = all.sort((a, b) => b.similarity - a.similarity);
+  return hardenRetrieval(ranked, { limit });
 }