@geraldmaron/construct 1.0.21 → 1.0.24

package/lib/storage/migrations.mjs

@@ -1,187 +0,0 @@
-/**
- * lib/storage/migrations.mjs — Postgres migration runner with hash tracking.
- *
- * Walks `db/schema/*.sql` in lexical order and applies any not yet
- * recorded in `construct_schema_migrations`. Each applied file's SHA256 is
- * stored alongside the filename. Re-running the runner against an unchanged
- * tree is a no-op.
- *
- * Drift detection: if an applied file's SHA changes between runs, the runner
- * fails fast with a precise error. SQL files are append-only by convention —
- * to evolve schema after a migration ships, write a new file with a higher
- * sequence number.
- *
- * Repair mode: long-lived developer databases pre-date the append-only policy
- * and carry stale SHAs for files that have since evolved. Call runMigrations
- * with `repair: true` (or `construct storage repair-migrations`) to re-apply
- * each drifted file. The repair refuses on any file containing a destructive
- * statement (`DROP`, `TRUNCATE`, `ALTER … DROP`) so the safety bar stays
- * "only fully-idempotent migrations may be re-recorded."
- *
- * The bookkeeping table (`construct_schema_migrations`) is created inline
- * before applied state is checked, so the runner self-bootstraps and a fresh
- * database can run from any starting migration without a prerequisite file.
- */
-
-import { readdirSync, readFileSync } from 'node:fs';
-import { join, dirname } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { createHash } from 'node:crypto';
-
-const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
-const DEFAULT_MIGRATIONS_DIR = join(MODULE_DIR, '..', '..', 'db', 'schema');
-
-function sha256(text) {
-  return createHash('sha256').update(text).digest('hex');
-}
-
-function listMigrationFiles(dir) {
-  return readdirSync(dir)
-    .filter((name) => name.endsWith('.sql'))
-    .sort();
-}
-
-const BOOKKEEPING_DDL = `
-CREATE TABLE IF NOT EXISTS construct_schema_migrations (
-    filename TEXT PRIMARY KEY,
-    sha TEXT NOT NULL,
-    applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
-);
-`;
-
-// Heuristic: a migration is safe to re-apply (idempotent) when it contains no
-// destructive statements. Comments are stripped first so a `-- DROP …` note in
-// a docstring does not falsely flag the file.
-const DESTRUCTIVE_RE = /\b(drop\s+(table|index|view|column|schema|extension|function|sequence)\b|truncate\b|alter\s+table\s+[^;]*?\bdrop\b|delete\s+from\b)/i;
-
-function stripSqlComments(sql) {
-  return sql
-    .replace(/--[^\n]*/g, '')
-    .replace(/\/\*[\s\S]*?\*\//g, '');
-}
-
-export function isIdempotentMigration(sql) {
-  return !DESTRUCTIVE_RE.test(stripSqlComments(sql));
-}
-
-/**
- * Apply pending migrations. Returns a per-file outcome summary.
- *
- * @param {object} client — postgres client (from createSqlClient)
- * @param {object} [opts]
- * @param {string} [opts.dir] — schema/migrations directory, defaults to db/schema
- * @param {boolean} [opts.repair] — when true, re-apply drifted files that are
- *   verifiably idempotent and update their recorded SHA. Destructive drift
- *   still throws. Default false.
- * @returns {Promise<{ applied: string[], skipped: string[], repaired: string[], drift: Array<{ filename: string, expected: string, actual: string, idempotent: boolean }> }>}
- */
-export async function runMigrations(client, { dir = DEFAULT_MIGRATIONS_DIR, repair = false } = {}) {
-  if (!client) throw new Error('runMigrations requires a postgres client');
-
-  await client.unsafe(BOOKKEEPING_DDL);
-
-  const recorded = await client`
-    SELECT filename, sha FROM construct_schema_migrations
-  `;
-  const recordedMap = new Map(recorded.map((r) => [r.filename, r.sha]));
-
-  const files = listMigrationFiles(dir);
-  const applied = [];
-  const skipped = [];
-  const repaired = [];
-  const drift = [];
-
-  for (const filename of files) {
-    const path = join(dir, filename);
-    const contents = readFileSync(path, 'utf8');
-    const sha = sha256(contents);
-
-    const prev = recordedMap.get(filename);
-    if (prev) {
-      if (prev !== sha) {
-        const idempotent = isIdempotentMigration(contents);
-        if (repair && idempotent) {
-          // Re-apply the migration (its IF NOT EXISTS clauses make this a
-          // no-op against the live schema) and update the recorded SHA.
-          await client.begin(async (tx) => {
-            await tx.unsafe(contents);
-            await tx`
-              UPDATE construct_schema_migrations
-                 SET sha = ${sha}, applied_at = now()
-               WHERE filename = ${filename}
-            `;
-          });
-          repaired.push(filename);
-        } else {
-          drift.push({ filename, expected: prev, actual: sha, idempotent });
-        }
-      } else {
-        skipped.push(filename);
-      }
-      continue;
-    }
-
-    await client.begin(async (tx) => {
-      await tx.unsafe(contents);
-      await tx`
-        INSERT INTO construct_schema_migrations (filename, sha)
-        VALUES (${filename}, ${sha})
-        ON CONFLICT (filename) DO UPDATE SET sha = EXCLUDED.sha, applied_at = now()
-      `;
-    });
-    applied.push(filename);
-  }
-
-  if (drift.length > 0) {
-    const detail = drift
-      .map((d) => `  ${d.filename}: applied SHA ${d.expected.slice(0, 12)}…, on-disk SHA ${d.actual.slice(0, 12)}…${d.idempotent ? ' (idempotent — repairable via `construct storage repair-migrations`)' : ' (NOT idempotent — write a new migration file)'}`)
-      .join('\n');
-    throw new Error(
-      `Migration drift detected (a previously-applied migration file has changed). ` +
-      `Migrations are append-only — write a new file with a higher sequence number ` +
-      `to evolve the schema. Drift:\n${detail}`
-    );
-  }
-
-  return { applied, skipped, repaired, drift };
-}
-
-/**
- * Diagnostic for `construct doctor`: returns last applied filename, count,
- * and any drift, without throwing.
- */
-export async function describeMigrations(client, { dir = DEFAULT_MIGRATIONS_DIR } = {}) {
-  if (!client) return { ok: false, reason: 'no_sql_client' };
-  try {
-    await client.unsafe(BOOKKEEPING_DDL);
-    const rows = await client`
-      SELECT filename, sha, applied_at
-        FROM construct_schema_migrations
-       ORDER BY applied_at DESC, filename DESC
-       LIMIT 1
-    `;
-    const onDisk = listMigrationFiles(dir);
-    const applied = await client`SELECT count(*)::int AS n FROM construct_schema_migrations`;
-    const recorded = await client`SELECT filename, sha FROM construct_schema_migrations`;
-    const recordedMap = new Map(recorded.map((r) => [r.filename, r.sha]));
-    const drift = [];
-    for (const filename of onDisk) {
-      const contents = readFileSync(join(dir, filename), 'utf8');
-      const sha = sha256(contents);
-      const prev = recordedMap.get(filename);
-      if (prev && prev !== sha) {
-        drift.push({ filename, idempotent: isIdempotentMigration(contents) });
-      }
-    }
-    return {
-      ok: true,
-      lastApplied: rows[0]?.filename || null,
-      lastAppliedAt: rows[0]?.applied_at || null,
-      appliedCount: applied[0]?.n ?? 0,
-      onDiskCount: onDisk.length,
-      drift,
-    };
-  } catch (err) {
-    return { ok: false, reason: err?.message || 'describe-migrations failed' };
-  }
-}

package/lib/storage/postgres-backup.mjs

@@ -1,124 +0,0 @@
-#!/usr/bin/env node
-/**
- * lib/storage/postgres-backup.mjs — durable stash/restore for the managed construct-postgres container.
- *
- * Mirrors the telemetry backup pattern in service-manager.mjs. Entry points:
- *   - construct down  → stashConstructDb  (dump before container stop)
- *   - construct init → restoreConstructDb (reload after fresh container start)
- *
- * Dumps are stored in ~/.construct/backups/postgres/ as pg_dump custom-format
- * files. The N most recent are kept; older ones are pruned automatically.
- */
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-import { spawnSync } from 'node:child_process';
-
-const POSTGRES_CONTAINER = 'construct-postgres';
-const POSTGRES_USER = 'construct';
-const POSTGRES_DB = 'construct';
-const DEFAULT_KEEP = 5;
-
-function pruneBackupDir(dir, keep) {
-  try {
-    const dumps = fs.readdirSync(dir)
-      .filter((f) => f.startsWith('construct-') && f.endsWith('.dump'))
-      .sort()
-      .reverse();
-    for (const dump of dumps.slice(keep)) {
-      fs.rmSync(path.join(dir, dump), { force: true });
-      fs.rmSync(path.join(dir, dump.replace('.dump', '.json')), { force: true });
-    }
-  } catch { /* non-critical */ }
-}
-
-/**
- * Dump the construct Postgres database to a timestamped file under
- * ~/.construct/backups/postgres/. Safe to call when the container is not
- * running — returns { status: 'no-data' } silently.
- */
-export function stashConstructDb({
-  homeDir = os.homedir(),
-  spawnSyncFn = spawnSync,
-  keep = DEFAULT_KEEP,
-} = {}) {
-  const stashDir = path.join(homeDir, '.construct', 'backups', 'postgres');
-  fs.mkdirSync(stashDir, { recursive: true });
-
-  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-  const dumpFile = path.join(stashDir, `construct-${timestamp}.dump`);
-  const manifestFile = path.join(stashDir, `construct-${timestamp}.json`);
-
-  const dump = spawnSyncFn('docker', [
-    'exec', POSTGRES_CONTAINER,
-    'pg_dump', '-U', POSTGRES_USER, '-d', POSTGRES_DB,
-    '-Fc', '--data-only',
-  ], { stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 200 * 1024 * 1024 });
-
-  const hasData = dump.status === 0 && dump.stdout?.length > 100;
-  if (!hasData) return { status: 'no-data', stashPath: null };
-
-  fs.writeFileSync(dumpFile, dump.stdout);
-  fs.writeFileSync(manifestFile, JSON.stringify({
-    version: 1,
-    createdAt: new Date().toISOString(),
-    reason: 'pre-shutdown-stash',
-    dumpFile: path.basename(dumpFile),
-    dumpBytes: dump.stdout.length,
-  }, null, 2) + '\n');
-
-  pruneBackupDir(stashDir, keep);
-  return { status: 'ok', stashPath: dumpFile, bytes: dump.stdout.length };
-}
-
-/**
- * Restore the most recent stash into a running construct-postgres container.
- * Safe to call when no stash exists — returns { status: 'no-stash' } silently.
- * The schema must already exist (run migration before calling this).
- */
-export function restoreConstructDb({
-  homeDir = os.homedir(),
-  spawnSyncFn = spawnSync,
-} = {}) {
-  const stashDir = path.join(homeDir, '.construct', 'backups', 'postgres');
-  if (!fs.existsSync(stashDir)) return { status: 'no-stash' };
-
-  const dumps = fs.readdirSync(stashDir)
-    .filter((f) => f.startsWith('construct-') && f.endsWith('.dump'))
-    .sort()
-    .reverse();
-  if (dumps.length === 0) return { status: 'no-stash' };
-
-  const latestDump = path.join(stashDir, dumps[0]);
-
-  const cp = spawnSyncFn('docker', [
-    'cp', latestDump, `${POSTGRES_CONTAINER}:/tmp/construct.dump`,
-  ], { stdio: 'ignore' });
-  if (cp.status !== 0) return { status: 'copy-failed', stashPath: latestDump };
-
-  const restore = spawnSyncFn('docker', [
-    'exec', POSTGRES_CONTAINER,
-    'pg_restore', '-U', POSTGRES_USER, '-d', POSTGRES_DB,
-    '--data-only', '--disable-triggers', '--no-owner',
-    '--if-exists',
-    '/tmp/construct.dump',
-  ], { stdio: 'ignore' });
-
-  return {
-    status: restore.status === 0 ? 'restored' : 'restore-failed',
-    stashPath: latestDump,
-    exitCode: restore.status,
-  };
-}
-
-/**
- * Delete all stashed backups under ~/.construct/backups/postgres/.
- * Called by `construct storage reset` and uninstall flows.
- */
-export function purgeConstructDbStashes({ homeDir = os.homedir() } = {}) {
-  const stashDir = path.join(homeDir, '.construct', 'backups', 'postgres');
-  if (!fs.existsSync(stashDir)) return { status: 'ok', deletedCount: 0 };
-  const files = fs.readdirSync(stashDir).map((f) => path.join(stashDir, f));
-  for (const f of files) fs.rmSync(f, { force: true });
-  return { status: 'ok', deletedCount: files.length };
-}

package/lib/storage/sql-store.mjs

@@ -1,45 +0,0 @@
-#!/usr/bin/env node
-/**
- * lib/storage/sql-store.mjs — lightweight shared SQL storage facade.
- */
-import { createSqlClient, probeSqlClient } from './backend.mjs';
-import { resolveDatabaseUrl } from '../env-config.mjs';
-
-export function hasSqlStore(env = process.env) {
-  return Boolean(resolveDatabaseUrl(env));
-}
-
-export function detectStoreMode(env = process.env) {
-  if (resolveDatabaseUrl(env)) return 'postgres';
-  return 'file';
-}
-
-export function describeSqlStore(env = process.env) {
-  const mode = detectStoreMode(env);
-  return {
-    mode,
-    configured: mode !== 'file',
-    sharedReady: mode === 'postgres',
-    fallbackAvailable: mode === 'file',
-    hasDatabaseUrl: Boolean(resolveDatabaseUrl(env)),
-  };
-}
-
-export async function describeSqlStoreHealth(env = process.env) {
-  const store = describeSqlStore(env);
-  if (store.mode !== 'postgres') return sqlStoreHealth(env);
-  const client = createSqlClient(env);
-  try {
-    return await probeSqlClient(client);
-  } finally {
-    if (client) await client.end({ timeout: 5 }).catch(() => {});
-  }
-}
-
-export function sqlStoreHealth(env = process.env) {
-  const store = describeSqlStore(env);
-  if (store.mode === 'postgres') {
-    return { status: 'configured', message: 'Shared Postgres store is configured' };
-  }
-  return { status: 'unavailable', message: 'No SQL store configured; using file-state only' };
-}

package/lib/storage/store-version.mjs

@@ -1,115 +0,0 @@
-/**
- * lib/storage/store-version.mjs — versioned schema for the JSON stores.
- *
- * The observation, entity, and session stores write JSON arrays (or objects)
- * to disk. Until now, the on-disk format had no version stamp, which meant
- * any future schema change had to either fork the file or risk silently
- * breaking older data. This module provides a thin, opt-in versioning
- * layer:
- *
- *   readVersioned(filePath, storeId)
- *       Returns the on-disk records, lazily upgrading them through the
- *       store's migration chain if the embedded version is older than the
- *       current one. Existing unversioned files (plain arrays) are treated
- *       as version 1 and stamped on the next write.
- *
- *   writeVersioned(filePath, storeId, records)
- *       Wraps the records in `{ schemaVersion, storeId, records }` and
- *       writes atomically. Callers always read/write through this helper
- *       so the on-disk format stays canonical.
- *
- *   registerMigration(storeId, fromVersion, toVersion, migrate)
- *       Record a one-step migration. The runtime walks migrations in
- *       order until the records reach the current version.
- *
- * Migrations are pure functions over records; they cannot reach the network,
- * fork processes, or call other stores. That keeps lazy upgrade safe to run
- * inside a hot Read path.
- */
-
-import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
-import { dirname } from 'node:path';
-
-const CURRENT_VERSIONS = {
-  observations: 1,
-  observations_vectors: 1,
-  observations_index: 1,
-  entities: 1,
-  entity_vectors: 1,
-  sessions: 1,
-  session_vectors: 1,
-};
-
-const MIGRATIONS = new Map();
-
-export function registerMigration(storeId, fromVersion, migrate) {
-  const key = `${storeId}@${fromVersion}`;
-  if (MIGRATIONS.has(key)) {
-    throw new Error(`Migration already registered: ${key}`);
-  }
-  MIGRATIONS.set(key, migrate);
-}
-
-export function currentVersion(storeId) {
-  return CURRENT_VERSIONS[storeId] ?? 1;
-}
-
-function unwrap(parsed) {
-  if (parsed && typeof parsed === 'object' && Array.isArray(parsed.records)) {
-    return {
-      schemaVersion: Number.isInteger(parsed.schemaVersion) ? parsed.schemaVersion : 1,
-      records: parsed.records,
-    };
-  }
-  if (Array.isArray(parsed)) {
-    return { schemaVersion: 1, records: parsed };
-  }
-  return { schemaVersion: 1, records: [] };
-}
-
-function migrateOnce(storeId, schemaVersion, records) {
-  const migrate = MIGRATIONS.get(`${storeId}@${schemaVersion}`);
-  if (!migrate) return null;
-  const next = migrate(records);
-  if (!Array.isArray(next)) {
-    throw new Error(`Migration ${storeId}@${schemaVersion}→ must return an array of records`);
-  }
-  return next;
-}
-
-/**
- * Read a versioned JSON store. Lazily applies migrations to bring records up
- * to the current version. Returns plain records (caller doesn't see the
- * version envelope).
- */
-export function readVersioned(filePath, storeId, fallback = []) {
-  if (!existsSync(filePath)) return fallback;
-  let parsed;
-  try { parsed = JSON.parse(readFileSync(filePath, 'utf8')); }
-  catch { return fallback; }
-
-  let { schemaVersion, records } = unwrap(parsed);
-  const target = currentVersion(storeId);
-
-  while (schemaVersion < target) {
-    const next = migrateOnce(storeId, schemaVersion, records);
-    if (next === null) break;
-    records = next;
-    schemaVersion += 1;
-  }
-
-  return records;
-}
-
-/**
- * Write a versioned JSON store. Always stamps the current version.
- */
-export function writeVersioned(filePath, storeId, records) {
-  mkdirSync(dirname(filePath), { recursive: true });
-  const payload = {
-    schemaVersion: currentVersion(storeId),
-    storeId,
-    records: Array.isArray(records) ? records : [],
-  };
-  writeFileSync(filePath, JSON.stringify(payload, null, 2) + '\n');
-}

package/lib/storage/vector-store.mjs

@@ -1,100 +0,0 @@
-#!/usr/bin/env node
-/**
- * lib/storage/vector-store.mjs — Local JSON vector index facade.
- *
- * Stores vector records on disk for fallback retrieval when no SQL backend is
- * available. The model field is metadata supplied by the caller — this module
- * does not embed; it only persists what the engine produces.
- *
- * Search APIs accept a pre-computed query embedding so the embedding model is
- * the caller's choice and there is no implicit dependency on a specific model
- * inside this module.
- */
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
-import { dirname, resolve } from 'node:path';
-import { scoreByEmbedding, rankByBm25 } from './embeddings.mjs';
-import { embedSync } from './embeddings-legacy.mjs';
-
-export function vectorStoreMode(env = process.env) {
-  if (env.CONSTRUCT_VECTOR_URL) return 'remote';
-  if (env.CONSTRUCT_VECTOR_INDEX_PATH) return 'local';
-  return 'file';
-}
-
-export function describeVectorStore(env = process.env) {
-  const mode = vectorStoreMode(env);
-  return {
-    mode,
-    configured: mode !== 'file',
-    sharedReady: mode === 'remote',
-    fallbackAvailable: true,
-    endpoint: env.CONSTRUCT_VECTOR_URL || null,
-    indexPath: env.CONSTRUCT_VECTOR_INDEX_PATH || null,
-    model: env.CONSTRUCT_VECTOR_MODEL || null,
-  };
-}
-
-export function readLocalVectorIndex(indexPath) {
-  const resolvedPath = resolve(String(indexPath || ''));
-  if (!resolvedPath || !existsSync(resolvedPath)) {
-    return { version: 1, model: null, updatedAt: null, records: [] };
-  }
-
-  try {
-    const parsed = JSON.parse(readFileSync(resolvedPath, 'utf8'));
-    return {
-      version: parsed.version ?? 1,
-      model: parsed.model || null,
-      updatedAt: parsed.updatedAt || null,
-      records: Array.isArray(parsed.records) ? parsed.records : [],
-    };
-  } catch {
-    return { version: 1, model: null, updatedAt: null, records: [] };
-  }
-}
-
-export function writeLocalVectorIndex(indexPath, records = [], { model } = {}) {
-  const resolvedPath = resolve(String(indexPath || ''));
-  if (!resolvedPath) throw new Error('Missing local vector index path');
-  if (!model) throw new Error('writeLocalVectorIndex requires an explicit model id (caller must supply)');
-  mkdirSync(dirname(resolvedPath), { recursive: true });
-  const payload = {
-    version: 1,
-    model,
-    updatedAt: new Date().toISOString(),
-    records,
-  };
-  writeFileSync(resolvedPath, `${JSON.stringify(payload, null, 2)}\n`);
-  return payload;
-}
-
-/**
- * Search the local vector index. Accepts either a query string (which is
- * embedded via the legacy hashing adapter inline — fast, deterministic, no
- * engine init) or a pre-computed query embedding for engine-aligned callers.
- *
- * The hashing path matches what `addObservation` and `syncFileStateToSql`
- * write to disk when no SQL backend is configured, so dimensions agree.
- * Callers that store engine-dim vectors should pass an explicit Float32Array.
- */
-export function searchLocalVectorIndex(indexPath, query, { limit = 10 } = {}) {
-  const index = readLocalVectorIndex(indexPath);
-  const queryEmbedding = typeof query === 'string'
-    ? embedSync(query)
-    : (query instanceof Float32Array ? Array.from(query) : (query || []));
-  return scoreByEmbedding(index.records, queryEmbedding, { limit });
-}
-
-export function vectorSearchLocal(records = [], query = '', { limit = 10 } = {}) {
-  const bm25Docs = records.map((record) => ({
-    ...record,
-    text: [
-      record.title,
-      record.summary,
-      record.body,
-      record.text,
-      Array.isArray(record.tags) ? record.tags.join(' ') : '',
-    ].filter(Boolean).join('\n'),
-  }));
-  return rankByBm25(bm25Docs, query, { limit });
-}