@geraldmaron/construct 1.0.0 → 1.0.1

package/lib/storage/migrations.mjs

@@ -0,0 +1,187 @@
+/**
+ * lib/storage/migrations.mjs — Postgres migration runner with hash tracking.
+ *
+ * Walks `db/schema/*.sql` in lexical order and applies any not yet
+ * recorded in `construct_schema_migrations`. Each applied file's SHA256 is
+ * stored alongside the filename. Re-running the runner against an unchanged
+ * tree is a no-op.
+ *
+ * Drift detection: if an applied file's SHA changes between runs, the runner
+ * fails fast with a precise error. SQL files are append-only by convention —
+ * to evolve schema after a migration ships, write a new file with a higher
+ * sequence number.
+ *
+ * Repair mode: long-lived developer databases pre-date the append-only policy
+ * and carry stale SHAs for files that have since evolved. Call runMigrations
+ * with `repair: true` (or `construct storage repair-migrations`) to re-apply
+ * each drifted file. The repair refuses on any file containing a destructive
+ * statement (`DROP`, `TRUNCATE`, `ALTER … DROP`) so the safety bar stays
+ * "only fully-idempotent migrations may be re-recorded."
+ *
+ * The bookkeeping table (`construct_schema_migrations`) is created inline
+ * before applied state is checked, so the runner self-bootstraps and a fresh
+ * database can run from any starting migration without a prerequisite file.
+ */
+
+import { readdirSync, readFileSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createHash } from 'node:crypto';
+
+const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
+const DEFAULT_MIGRATIONS_DIR = join(MODULE_DIR, '..', '..', 'db', 'schema');
+
+function sha256(text) {
+  return createHash('sha256').update(text).digest('hex');
+}
+
+function listMigrationFiles(dir) {
+  return readdirSync(dir)
+    .filter((name) => name.endsWith('.sql'))
+    .sort();
+}
+
+const BOOKKEEPING_DDL = `
+CREATE TABLE IF NOT EXISTS construct_schema_migrations (
+    filename TEXT PRIMARY KEY,
+    sha TEXT NOT NULL,
+    applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+`;
+
+// Heuristic: a migration is safe to re-apply (idempotent) when it contains no
+// destructive statements. Comments are stripped first so a `-- DROP …` note in
+// a docstring does not falsely flag the file.
+const DESTRUCTIVE_RE = /\b(drop\s+(table|index|view|column|schema|extension|function|sequence)\b|truncate\b|alter\s+table\s+[^;]*?\bdrop\b|delete\s+from\b)/i;
+
+function stripSqlComments(sql) {
+  return sql
+    .replace(/--[^\n]*/g, '')
+    .replace(/\/\*[\s\S]*?\*\//g, '');
+}
+
+export function isIdempotentMigration(sql) {
+  return !DESTRUCTIVE_RE.test(stripSqlComments(sql));
+}
+
+/**
+ * Apply pending migrations. Returns a per-file outcome summary.
+ *
+ * @param {object} client — postgres client (from createSqlClient)
+ * @param {object} [opts]
+ * @param {string} [opts.dir] — schema/migrations directory, defaults to db/schema
+ * @param {boolean} [opts.repair] — when true, re-apply drifted files that are
+ *   verifiably idempotent and update their recorded SHA. Destructive drift
+ *   still throws. Default false.
+ * @returns {Promise<{ applied: string[], skipped: string[], repaired: string[], drift: Array<{ filename: string, expected: string, actual: string, idempotent: boolean }> }>}
+ */
+export async function runMigrations(client, { dir = DEFAULT_MIGRATIONS_DIR, repair = false } = {}) {
+  if (!client) throw new Error('runMigrations requires a postgres client');
+
+  await client.unsafe(BOOKKEEPING_DDL);
+
+  const recorded = await client`
+    SELECT filename, sha FROM construct_schema_migrations
+  `;
+  const recordedMap = new Map(recorded.map((r) => [r.filename, r.sha]));
+
+  const files = listMigrationFiles(dir);
+  const applied = [];
+  const skipped = [];
+  const repaired = [];
+  const drift = [];
+
+  for (const filename of files) {
+    const path = join(dir, filename);
+    const contents = readFileSync(path, 'utf8');
+    const sha = sha256(contents);
+
+    const prev = recordedMap.get(filename);
+    if (prev) {
+      if (prev !== sha) {
+        const idempotent = isIdempotentMigration(contents);
+        if (repair && idempotent) {
+          // Re-apply the migration (its IF NOT EXISTS clauses make this a
+          // no-op against the live schema) and update the recorded SHA.
+          await client.begin(async (tx) => {
+            await tx.unsafe(contents);
+            await tx`
+              UPDATE construct_schema_migrations
+                 SET sha = ${sha}, applied_at = now()
+               WHERE filename = ${filename}
+            `;
+          });
+          repaired.push(filename);
+        } else {
+          drift.push({ filename, expected: prev, actual: sha, idempotent });
+        }
+      } else {
+        skipped.push(filename);
+      }
+      continue;
+    }
+
+    await client.begin(async (tx) => {
+      await tx.unsafe(contents);
+      await tx`
+        INSERT INTO construct_schema_migrations (filename, sha)
+        VALUES (${filename}, ${sha})
+        ON CONFLICT (filename) DO UPDATE SET sha = EXCLUDED.sha, applied_at = now()
+      `;
+    });
+    applied.push(filename);
+  }
+
+  if (drift.length > 0) {
+    const detail = drift
+      .map((d) => `  ${d.filename}: applied SHA ${d.expected.slice(0, 12)}…, on-disk SHA ${d.actual.slice(0, 12)}…${d.idempotent ? ' (idempotent — repairable via `construct storage repair-migrations`)' : ' (NOT idempotent — write a new migration file)'}`)
+      .join('\n');
+    throw new Error(
+      `Migration drift detected (a previously-applied migration file has changed). ` +
+      `Migrations are append-only — write a new file with a higher sequence number ` +
+      `to evolve the schema. Drift:\n${detail}`
+    );
+  }
+
+  return { applied, skipped, repaired, drift };
+}
+
+/**
+ * Diagnostic for `construct doctor`: returns last applied filename, count,
+ * and any drift, without throwing.
+ */
+export async function describeMigrations(client, { dir = DEFAULT_MIGRATIONS_DIR } = {}) {
+  if (!client) return { ok: false, reason: 'no_sql_client' };
+  try {
+    await client.unsafe(BOOKKEEPING_DDL);
+    const rows = await client`
+      SELECT filename, sha, applied_at
+        FROM construct_schema_migrations
+       ORDER BY applied_at DESC, filename DESC
+       LIMIT 1
+    `;
+    const onDisk = listMigrationFiles(dir);
+    const applied = await client`SELECT count(*)::int AS n FROM construct_schema_migrations`;
+    const recorded = await client`SELECT filename, sha FROM construct_schema_migrations`;
+    const recordedMap = new Map(recorded.map((r) => [r.filename, r.sha]));
+    const drift = [];
+    for (const filename of onDisk) {
+      const contents = readFileSync(join(dir, filename), 'utf8');
+      const sha = sha256(contents);
+      const prev = recordedMap.get(filename);
+      if (prev && prev !== sha) {
+        drift.push({ filename, idempotent: isIdempotentMigration(contents) });
+      }
+    }
+    return {
+      ok: true,
+      lastApplied: rows[0]?.filename || null,
+      lastAppliedAt: rows[0]?.applied_at || null,
+      appliedCount: applied[0]?.n ?? 0,
+      onDiskCount: onDisk.length,
+      drift,
+    };
+  } catch (err) {
+    return { ok: false, reason: err?.message || 'describe-migrations failed' };
+  }
+}

package/lib/storage/postgres-backup.mjs

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+/**
+ * lib/storage/postgres-backup.mjs — durable stash/restore for the managed construct-postgres container.
+ *
+ * Mirrors the telemetry backup pattern in service-manager.mjs. Entry points:
+ *   - construct down  → stashConstructDb  (dump before container stop)
+ *   - construct init → restoreConstructDb (reload after fresh container start)
+ *
+ * Dumps are stored in ~/.construct/backups/postgres/ as pg_dump custom-format
+ * files. The N most recent are kept; older ones are pruned automatically.
+ */
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+const POSTGRES_CONTAINER = 'construct-postgres';
+const POSTGRES_USER = 'construct';
+const POSTGRES_DB = 'construct';
+const DEFAULT_KEEP = 5;
+
+function pruneBackupDir(dir, keep) {
+  try {
+    const dumps = fs.readdirSync(dir)
+      .filter((f) => f.startsWith('construct-') && f.endsWith('.dump'))
+      .sort()
+      .reverse();
+    for (const dump of dumps.slice(keep)) {
+      fs.rmSync(path.join(dir, dump), { force: true });
+      fs.rmSync(path.join(dir, dump.replace('.dump', '.json')), { force: true });
+    }
+  } catch { /* non-critical */ }
+}
+
+/**
+ * Dump the construct Postgres database to a timestamped file under
+ * ~/.construct/backups/postgres/. Safe to call when the container is not
+ * running — returns { status: 'no-data' } silently.
+ */
+export function stashConstructDb({
+  homeDir = os.homedir(),
+  spawnSyncFn = spawnSync,
+  keep = DEFAULT_KEEP,
+} = {}) {
+  const stashDir = path.join(homeDir, '.construct', 'backups', 'postgres');
+  fs.mkdirSync(stashDir, { recursive: true });
+
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const dumpFile = path.join(stashDir, `construct-${timestamp}.dump`);
+  const manifestFile = path.join(stashDir, `construct-${timestamp}.json`);
+
+  const dump = spawnSyncFn('docker', [
+    'exec', POSTGRES_CONTAINER,
+    'pg_dump', '-U', POSTGRES_USER, '-d', POSTGRES_DB,
+    '-Fc', '--data-only',
+  ], { stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 200 * 1024 * 1024 });
+
+  const hasData = dump.status === 0 && dump.stdout?.length > 100;
+  if (!hasData) return { status: 'no-data', stashPath: null };
+
+  fs.writeFileSync(dumpFile, dump.stdout);
+  fs.writeFileSync(manifestFile, JSON.stringify({
+    version: 1,
+    createdAt: new Date().toISOString(),
+    reason: 'pre-shutdown-stash',
+    dumpFile: path.basename(dumpFile),
+    dumpBytes: dump.stdout.length,
+  }, null, 2) + '\n');
+
+  pruneBackupDir(stashDir, keep);
+  return { status: 'ok', stashPath: dumpFile, bytes: dump.stdout.length };
+}
+
+/**
+ * Restore the most recent stash into a running construct-postgres container.
+ * Safe to call when no stash exists — returns { status: 'no-stash' } silently.
+ * The schema must already exist (run migration before calling this).
+ */
+export function restoreConstructDb({
+  homeDir = os.homedir(),
+  spawnSyncFn = spawnSync,
+} = {}) {
+  const stashDir = path.join(homeDir, '.construct', 'backups', 'postgres');
+  if (!fs.existsSync(stashDir)) return { status: 'no-stash' };
+
+  const dumps = fs.readdirSync(stashDir)
+    .filter((f) => f.startsWith('construct-') && f.endsWith('.dump'))
+    .sort()
+    .reverse();
+  if (dumps.length === 0) return { status: 'no-stash' };
+
+  const latestDump = path.join(stashDir, dumps[0]);
+
+  const cp = spawnSyncFn('docker', [
+    'cp', latestDump, `${POSTGRES_CONTAINER}:/tmp/construct.dump`,
+  ], { stdio: 'ignore' });
+  if (cp.status !== 0) return { status: 'copy-failed', stashPath: latestDump };
+
+  const restore = spawnSyncFn('docker', [
+    'exec', POSTGRES_CONTAINER,
+    'pg_restore', '-U', POSTGRES_USER, '-d', POSTGRES_DB,
+    '--data-only', '--disable-triggers', '--no-owner',
+    '--if-exists',
+    '/tmp/construct.dump',
+  ], { stdio: 'ignore' });
+
+  return {
+    status: restore.status === 0 ? 'restored' : 'restore-failed',
+    stashPath: latestDump,
+    exitCode: restore.status,
+  };
+}
+
+/**
+ * Delete all stashed backups under ~/.construct/backups/postgres/.
+ * Called by `construct storage reset` and uninstall flows.
+ */
+export function purgeConstructDbStashes({ homeDir = os.homedir() } = {}) {
+  const stashDir = path.join(homeDir, '.construct', 'backups', 'postgres');
+  if (!fs.existsSync(stashDir)) return { status: 'ok', deletedCount: 0 };
+  const files = fs.readdirSync(stashDir).map((f) => path.join(stashDir, f));
+  for (const f of files) fs.rmSync(f, { force: true });
+  return { status: 'ok', deletedCount: files.length };
+}

package/lib/storage/sql-store.mjs

@@ -2,16 +2,15 @@
 /**
  * lib/storage/sql-store.mjs — lightweight shared SQL storage facade.
  */
-import { existsSync } from 'node:fs';
 import { createSqlClient, probeSqlClient } from './backend.mjs';
+import { resolveDatabaseUrl } from '../env-config.mjs';
 
 export function hasSqlStore(env = process.env) {
-  return Boolean(env.DATABASE_URL || env.CONSTRUCT_SQLITE_PATH || env.CONSTRUCT_DB_PATH);
+  return Boolean(resolveDatabaseUrl(env));
 }
 
 export function detectStoreMode(env = process.env) {
-  if (env.DATABASE_URL) return 'postgres';
-  if (env.CONSTRUCT_SQLITE_PATH || env.CONSTRUCT_DB_PATH) return 'sqlite';
+  if (resolveDatabaseUrl(env)) return 'postgres';
   return 'file';
 }
 
@@ -21,9 +20,8 @@ export function describeSqlStore(env = process.env) {
     mode,
     configured: mode !== 'file',
     sharedReady: mode === 'postgres',
-    fallbackAvailable: mode === 'file' || mode === 'sqlite',
-    path: env.CONSTRUCT_SQLITE_PATH || env.CONSTRUCT_DB_PATH || null,
-    hasDatabaseUrl: Boolean(env.DATABASE_URL),
+    fallbackAvailable: mode === 'file',
+    hasDatabaseUrl: Boolean(resolveDatabaseUrl(env)),
   };
 }
 
@@ -43,13 +41,5 @@ export function sqlStoreHealth(env = process.env) {
   if (store.mode === 'postgres') {
     return { status: 'configured', message: 'Shared Postgres store is configured' };
   }
-  if (store.mode === 'sqlite') {
-    return { status: 'configured', message: 'SQLite-backed store is configured' };
-  }
   return { status: 'unavailable', message: 'No SQL store configured; using file-state only' };
 }
-
-export function canOpenLocalSqlite(env = process.env) {
-  const candidate = env.CONSTRUCT_SQLITE_PATH || env.CONSTRUCT_DB_PATH || null;
-  return candidate ? existsSync(candidate) : false;
-}

package/lib/storage/state-source.mjs

@@ -2,8 +2,9 @@
 /**
  * lib/storage/state-source.mjs — normalize file-state artifacts for hybrid retrieval.
  */
-import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { existsSync, readdirSync, statSync, readFileSync } from 'node:fs';
 import { basename, extname, join, relative } from 'node:path';
+import { EXTRACTABLE_DOCUMENT_EXTS, extractDocumentText } from '../document-extract.mjs';
 
 function readJsonIfExists(filePath) {
   if (!existsSync(filePath)) return null;
@@ -27,7 +28,7 @@ function titleFromPath(filePath) {
   return basename(filePath, extname(filePath)).replace(/[-_]+/g, ' ');
 }
 
-function collectTextFiles(rootDir, dirs, { extensions = new Set(['.md', '.json']) } = {}) {
+function collectTextFiles(rootDir, dirs, { extensions = EXTRACTABLE_DOCUMENT_EXTS } = {}) {
   const files = [];
   for (const dir of dirs) {
     const abs = join(rootDir, dir);
@@ -41,11 +42,12 @@ function collectTextFiles(rootDir, dirs, { extensions = new Set(['.md', '.json']
           stack.push(full);
           continue;
         }
-        if (!entry.isFile() || !extensions.has(extname(entry.name))) continue;
+        const extension = extname(entry.name).toLowerCase();
+        if (!entry.isFile() || !extensions.has(extension)) continue;
         try {
           const stat = statSync(full);
-          if (stat.size > 512_000) continue;
-          const body = readFileSync(full, 'utf8');
+          if (stat.size > 5_000_000) continue;
+          const body = extractDocumentText(full, { maxChars: 200_000 }).text;
           files.push({
             path: relative(rootDir, full),
             title: titleFromPath(full),
@@ -62,24 +64,24 @@ function collectTextFiles(rootDir, dirs, { extensions = new Set(['.md', '.json']
 
 export function loadStateSnapshot(rootDir) {
   const contextJsonPath = join(rootDir, '.cx', 'context.json');
-  const workflowJsonPath = join(rootDir, '.cx', 'workflow.json');
+  const planPath = join(rootDir, 'plan.md');
   const architecturePath = join(rootDir, 'docs', 'architecture.md');
   const docsReadmePath = join(rootDir, 'docs', 'README.md');
   const productIntelDocs = collectTextFiles(rootDir, [
-    '.cx/product-intel',
+    '.cx/knowledge',
     'docs/prd',
     'docs/meta-prd',
   ]);
 
   return {
     context: readJsonIfExists(contextJsonPath),
-    workflow: readJsonIfExists(workflowJsonPath),
+    plan: readTextIfExists(planPath),
     architecture: readTextIfExists(architecturePath),
     docsReadme: readTextIfExists(docsReadmePath),
     productIntelDocs,
     paths: {
       contextJsonPath,
-      workflowJsonPath,
+      planPath,
       architecturePath,
       docsReadmePath,
     },
@@ -89,11 +91,8 @@ export function loadStateSnapshot(rootDir) {
 export function summarizeStateSnapshot(snapshot) {
   return {
     contextSummary: snapshot.context?.contextSummary ?? snapshot.context?.summary ?? null,
-    workflowTitle: snapshot.workflow?.title ?? null,
-    workflowPhase: snapshot.workflow?.phase ?? null,
-    workflowStatus: snapshot.workflow?.status ?? null,
+    hasPlan: Boolean(snapshot.plan),
     lastContextSavedAt: snapshot.context?.savedAt ?? null,
-    lastWorkflowUpdatedAt: snapshot.workflow?.updatedAt ?? null,
     hasArchitectureDoc: Boolean(snapshot.architecture),
     hasDocsReadme: Boolean(snapshot.docsReadme),
     productIntelDocCount: snapshot.productIntelDocs?.length ?? 0,

package/lib/storage/store-version.mjs

@@ -0,0 +1,115 @@
+/**
+ * lib/storage/store-version.mjs — versioned schema for the JSON stores.
+ *
+ * The observation, entity, and session stores write JSON arrays (or objects)
+ * to disk. Until now, the on-disk format had no version stamp, which meant
+ * any future schema change had to either fork the file or risk silently
+ * breaking older data. This module provides a thin, opt-in versioning
+ * layer:
+ *
+ *   readVersioned(filePath, storeId)
+ *       Returns the on-disk records, lazily upgrading them through the
+ *       store's migration chain if the embedded version is older than the
+ *       current one. Existing unversioned files (plain arrays) are treated
+ *       as version 1 and stamped on the next write.
+ *
+ *   writeVersioned(filePath, storeId, records)
+ *       Wraps the records in `{ schemaVersion, storeId, records }` and
+ *       writes atomically. Callers always read/write through this helper
+ *       so the on-disk format stays canonical.
+ *
+ *   registerMigration(storeId, fromVersion, toVersion, migrate)
+ *       Record a one-step migration. The runtime walks migrations in
+ *       order until the records reach the current version.
+ *
+ * Migrations are pure functions over records; they cannot reach the network,
+ * fork processes, or call other stores. That keeps lazy upgrade safe to run
+ * inside a hot Read path.
+ */
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+
+const CURRENT_VERSIONS = {
+  observations: 1,
+  observations_vectors: 1,
+  observations_index: 1,
+  entities: 1,
+  entity_vectors: 1,
+  sessions: 1,
+  session_vectors: 1,
+};
+
+const MIGRATIONS = new Map();
+
+export function registerMigration(storeId, fromVersion, migrate) {
+  const key = `${storeId}@${fromVersion}`;
+  if (MIGRATIONS.has(key)) {
+    throw new Error(`Migration already registered: ${key}`);
+  }
+  MIGRATIONS.set(key, migrate);
+}
+
+export function currentVersion(storeId) {
+  return CURRENT_VERSIONS[storeId] ?? 1;
+}
+
+function unwrap(parsed) {
+  if (parsed && typeof parsed === 'object' && Array.isArray(parsed.records)) {
+    return {
+      schemaVersion: Number.isInteger(parsed.schemaVersion) ? parsed.schemaVersion : 1,
+      records: parsed.records,
+    };
+  }
+  if (Array.isArray(parsed)) {
+    return { schemaVersion: 1, records: parsed };
+  }
+  return { schemaVersion: 1, records: [] };
+}
+
+function migrateOnce(storeId, schemaVersion, records) {
+  const migrate = MIGRATIONS.get(`${storeId}@${schemaVersion}`);
+  if (!migrate) return null;
+  const next = migrate(records);
+  if (!Array.isArray(next)) {
+    throw new Error(`Migration ${storeId}@${schemaVersion}→ must return an array of records`);
+  }
+  return next;
+}
+
+/**
+ * Read a versioned JSON store. Lazily applies migrations to bring records up
+ * to the current version. Returns plain records (caller doesn't see the
+ * version envelope).
+ */
+export function readVersioned(filePath, storeId, fallback = []) {
+  if (!existsSync(filePath)) return fallback;
+  let parsed;
+  try { parsed = JSON.parse(readFileSync(filePath, 'utf8')); }
+  catch { return fallback; }
+
+  let { schemaVersion, records } = unwrap(parsed);
+  const target = currentVersion(storeId);
+
+  while (schemaVersion < target) {
+    const next = migrateOnce(storeId, schemaVersion, records);
+    if (next === null) break;
+    records = next;
+    schemaVersion += 1;
+  }
+
+  return records;
+}
+
+/**
+ * Write a versioned JSON store. Always stamps the current version.
+ */
+export function writeVersioned(filePath, storeId, records) {
+  mkdirSync(dirname(filePath), { recursive: true });
+  const payload = {
+    schemaVersion: currentVersion(storeId),
+    storeId,
+    records: Array.isArray(records) ? records : [],
+  };
+  writeFileSync(filePath, JSON.stringify(payload, null, 2) + '\n');
+}