@geraldmaron/construct 1.0.0

package/lib/storage/backend.mjs

@@ -0,0 +1,44 @@
+#!/usr/bin/env node
+/**
+ * lib/storage/backend.mjs — shared storage backend helpers for SQL/vector wiring.
+ */
+import postgres from 'postgres';
+
+function cleanUrl(url) {
+  return String(url || '').trim();
+}
+
+export function createSqlClient(env = process.env) {
+  const databaseUrl = cleanUrl(env.DATABASE_URL);
+  if (!databaseUrl) return null;
+  return postgres(databaseUrl, {
+    max: Number.parseInt(env.CONSTRUCT_DB_POOL_SIZE || '5', 10),
+    idle_timeout: Number.parseInt(env.CONSTRUCT_DB_IDLE_TIMEOUT_MS || '30000', 10),
+    connect_timeout: Number.parseInt(env.CONSTRUCT_DB_CONNECT_TIMEOUT_MS || '5000', 10),
+  });
+}
+
+export async function probeSqlClient(client) {
+  if (!client) return { status: 'unavailable', message: 'No SQL client configured' };
+  try {
+    await client`select 1 as ok`;
+    return { status: 'healthy', message: 'Shared Postgres store reachable' };
+  } catch (error) {
+    return { status: 'degraded', message: `SQL store unreachable: ${error?.message || 'unknown error'}` };
+  }
+}
+
+export async function closeSqlClient(client) {
+  if (!client) return;
+  await client.end({ timeout: 5 });
+}
+
+export function readVectorConfig(env = process.env) {
+  const endpoint = cleanUrl(env.CONSTRUCT_VECTOR_URL);
+  const indexPath = cleanUrl(env.CONSTRUCT_VECTOR_INDEX_PATH);
+  return {
+    endpoint: endpoint || null,
+    indexPath: indexPath || null,
+    model: cleanUrl(env.CONSTRUCT_VECTOR_MODEL) || null,
+  };
+}

package/lib/storage/embeddings.mjs

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+/**
+ * lib/storage/embeddings.mjs — deterministic local embeddings for hybrid retrieval.
+ */
+import crypto from 'node:crypto';
+
+export const EMBEDDING_MODEL = 'hashing-bow-v1';
+export const EMBEDDING_DIMENSIONS = 256;
+
+function tokenize(text) {
+  return String(text || '')
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((term) => term.length >= 3);
+}
+
+function bucketForToken(token, dimensions) {
+  const hash = crypto.createHash('sha256').update(token).digest();
+  return hash.readUInt32BE(0) % dimensions;
+}
+
+function signForToken(token) {
+  const hash = crypto.createHash('sha256').update(token).digest();
+  return (hash[4] & 1) === 0 ? 1 : -1;
+}
+
+export function embedText(text, dimensions = EMBEDDING_DIMENSIONS) {
+  const vector = new Array(dimensions).fill(0);
+  const tokens = tokenize(text);
+  if (tokens.length === 0) return vector;
+
+  for (const token of tokens) {
+    const bucket = bucketForToken(token, dimensions);
+    vector[bucket] += signForToken(token);
+  }
+
+  const norm = Math.sqrt(vector.reduce((sum, value) => sum + (value * value), 0));
+  if (norm === 0) return vector;
+  return vector.map((value) => Number((value / norm).toFixed(8)));
+}
+
+export function cosineSimilarity(left = [], right = []) {
+  const length = Math.min(left.length, right.length);
+  if (length === 0) return 0;
+  let dot = 0;
+  let leftNorm = 0;
+  let rightNorm = 0;
+  for (let i = 0; i < length; i += 1) {
+    const lv = Number(left[i] || 0);
+    const rv = Number(right[i] || 0);
+    dot += lv * rv;
+    leftNorm += lv * lv;
+    rightNorm += rv * rv;
+  }
+  const denom = Math.sqrt(leftNorm) * Math.sqrt(rightNorm);
+  return denom > 0 ? dot / denom : 0;
+}
+
+export function scoreEmbeddedDocuments(documents = [], query, { limit = 10 } = {}) {
+  const queryEmbedding = embedText(query);
+  return documents
+    .map((document) => ({
+      document,
+      score: cosineSimilarity(queryEmbedding, document.embedding || []),
+    }))
+    .filter(({ score }) => score > 0)
+    .sort((a, b) => b.score - a.score || String(a.document.id || '').localeCompare(String(b.document.id || '')))
+    .slice(0, limit)
+    .map(({ document, score }) => ({ ...document, score }));
+}

package/lib/storage/hybrid-query.mjs

@@ -0,0 +1,184 @@
+#!/usr/bin/env node
+/**
+ * lib/storage/hybrid-query.mjs — hybrid file + SQL + semantic retrieval.
+ */
+import { loadStateSnapshot, summarizeStateSnapshot } from './state-source.mjs';
+import { describeSqlStore } from './sql-store.mjs';
+import { describeVectorStore, vectorSearchLocal } from './vector-store.mjs';
+import { createSqlClient, closeSqlClient, readVectorConfig } from './backend.mjs';
+import { EMBEDDING_MODEL, embedText, scoreEmbeddedDocuments } from './embeddings.mjs';
+
+function collectFileCandidates(snapshot) {
+  const docs = [];
+
+  if (snapshot.context) {
+    docs.push({
+      id: '.cx/context.json',
+      kind: 'context',
+      title: 'Context state',
+      summary: snapshot.context.contextSummary || snapshot.context.summary || null,
+      body: JSON.stringify(snapshot.context, null, 2),
+      tags: ['context', 'state', 'cx'],
+    });
+  }
+
+  if (snapshot.workflow) {
+    docs.push({
+      id: '.cx/workflow.json',
+      kind: 'workflow',
+      title: snapshot.workflow.title || 'Workflow state',
+      summary: snapshot.workflow.summary || null,
+      body: JSON.stringify(snapshot.workflow, null, 2),
+      tags: ['workflow', 'state', 'cx'],
+    });
+  }
+
+  if (snapshot.architecture) {
+    docs.push({
+      id: 'docs/architecture.md',
+      kind: 'architecture',
+      title: 'Architecture docs',
+      summary: snapshot.architecture.slice(0, 240),
+      body: snapshot.architecture,
+      tags: ['architecture', 'docs'],
+    });
+  }
+
+  if (snapshot.docsReadme) {
+    docs.push({
+      id: 'docs/README.md',
+      kind: 'docs',
+      title: 'Docs index',
+      summary: snapshot.docsReadme.slice(0, 240),
+      body: snapshot.docsReadme,
+      tags: ['docs', 'index'],
+    });
+  }
+
+  for (const doc of snapshot.productIntelDocs ?? []) {
+    const kind = doc.path.startsWith('docs/prd/')
+      ? 'prd'
+      : doc.path.startsWith('docs/meta-prd/')
+        ? 'meta-prd'
+        : 'product-intel';
+    docs.push({
+      id: doc.path,
+      kind,
+      title: doc.title,
+      summary: doc.body.slice(0, 240),
+      body: doc.body,
+      tags: ['product-intel', kind],
+    });
+  }
+
+  return docs;
+}
+
+export function buildHybridSearchResults(rootDir, query, { limit = 10, env = process.env } = {}) {
+  const snapshot = loadStateSnapshot(rootDir);
+  const fileCandidates = collectFileCandidates(snapshot);
+  const fileHits = vectorSearchLocal(fileCandidates, query, { limit });
+
+  const sqlStore = describeSqlStore(env);
+  const vectorStore = describeVectorStore(env);
+
+  return {
+    query,
+    summary: summarizeStateSnapshot(snapshot),
+    stores: {
+      file: { configured: true, mode: 'canonical' },
+      sql: sqlStore,
+      vector: vectorStore,
+    },
+    results: fileHits.map((hit) => ({
+      id: hit.id,
+      kind: hit.kind,
+      title: hit.title,
+      summary: hit.summary,
+      score: hit.score,
+    })),
+  };
+}
+
+export async function buildHybridSearchResultsAsync(rootDir, query, { limit = 10, env = process.env } = {}) {
+  const base = buildHybridSearchResults(rootDir, query, { limit, env });
+  const client = createSqlClient(env);
+  if (!client) return base;
+
+  try {
+    const embeddingRows = await client`
+      select d.id, d.kind, d.title, d.summary, d.body, d.source_path, d.tags, e.embedding
+      from construct_documents d
+      join construct_embeddings e on e.document_id = d.id
+      where d.project = 'construct' and e.model = ${EMBEDDING_MODEL}
+    `;
+
+    const embeddedHits = scoreEmbeddedDocuments(
+      embeddingRows.map((row) => ({
+        ...row,
+        embedding: row.embedding,
+      })),
+      query,
+      { limit },
+    );
+
+    const sqlHits = await client`
+      select id, kind, title, summary, body, source_path
+      from construct_documents
+      where project = 'construct'
+        and (title ilike ${`%${query}%`} or coalesce(summary, '') ilike ${`%${query}%`} or body ilike ${`%${query}%`})
+      order by updated_at desc
+      limit ${limit}
+    `;
+
+    const merged = [...base.results];
+    for (const hit of embeddedHits) {
+      if (merged.some((entry) => entry.id === hit.id)) continue;
+      merged.push({
+        id: hit.id,
+        kind: hit.kind,
+        title: hit.title,
+        summary: hit.summary,
+        score: hit.score,
+        source_path: hit.source_path,
+      });
+    }
+    for (const hit of sqlHits) {
+      if (merged.some((entry) => entry.id === hit.id)) continue;
+      merged.push({
+        id: hit.id,
+        kind: hit.kind,
+        title: hit.title,
+        summary: hit.summary,
+        score: 1,
+        source_path: hit.source_path,
+      });
+    }
+
+    return {
+      ...base,
+      results: merged.slice(0, limit),
+      stores: {
+        ...base.stores,
+        vector: {
+          ...base.stores.vector,
+          ...readVectorConfig(env),
+          model: EMBEDDING_MODEL,
+        },
+        sql: {
+          ...base.stores.sql,
+          mode: 'postgres',
+          configured: true,
+          sharedReady: true,
+        },
+      },
+    };
+  } catch (error) {
+    return {
+      ...base,
+      error: error?.message || 'hybrid search failed',
+    };
+  } finally {
+    await closeSqlClient(client);
+  }
+}

package/lib/storage/sql-store.mjs

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+/**
+ * lib/storage/sql-store.mjs — lightweight shared SQL storage facade.
+ */
+import { existsSync } from 'node:fs';
+import { createSqlClient, probeSqlClient } from './backend.mjs';
+
+export function hasSqlStore(env = process.env) {
+  return Boolean(env.DATABASE_URL || env.CONSTRUCT_SQLITE_PATH || env.CONSTRUCT_DB_PATH);
+}
+
+export function detectStoreMode(env = process.env) {
+  if (env.DATABASE_URL) return 'postgres';
+  if (env.CONSTRUCT_SQLITE_PATH || env.CONSTRUCT_DB_PATH) return 'sqlite';
+  return 'file';
+}
+
+export function describeSqlStore(env = process.env) {
+  const mode = detectStoreMode(env);
+  return {
+    mode,
+    configured: mode !== 'file',
+    sharedReady: mode === 'postgres',
+    fallbackAvailable: mode === 'file' || mode === 'sqlite',
+    path: env.CONSTRUCT_SQLITE_PATH || env.CONSTRUCT_DB_PATH || null,
+    hasDatabaseUrl: Boolean(env.DATABASE_URL),
+  };
+}
+
+export async function describeSqlStoreHealth(env = process.env) {
+  const store = describeSqlStore(env);
+  if (store.mode !== 'postgres') return sqlStoreHealth(env);
+  const client = createSqlClient(env);
+  try {
+    return await probeSqlClient(client);
+  } finally {
+    if (client) await client.end({ timeout: 5 }).catch(() => {});
+  }
+}
+
+export function sqlStoreHealth(env = process.env) {
+  const store = describeSqlStore(env);
+  if (store.mode === 'postgres') {
+    return { status: 'configured', message: 'Shared Postgres store is configured' };
+  }
+  if (store.mode === 'sqlite') {
+    return { status: 'configured', message: 'SQLite-backed store is configured' };
+  }
+  return { status: 'unavailable', message: 'No SQL store configured; using file-state only' };
+}
+
+export function canOpenLocalSqlite(env = process.env) {
+  const candidate = env.CONSTRUCT_SQLITE_PATH || env.CONSTRUCT_DB_PATH || null;
+  return candidate ? existsSync(candidate) : false;
+}

package/lib/storage/state-source.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+/**
+ * lib/storage/state-source.mjs — normalize file-state artifacts for hybrid retrieval.
+ */
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { basename, extname, join, relative } from 'node:path';
+
+function readJsonIfExists(filePath) {
+  if (!existsSync(filePath)) return null;
+  try {
+    return JSON.parse(readFileSync(filePath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+function readTextIfExists(filePath) {
+  if (!existsSync(filePath)) return null;
+  try {
+    return readFileSync(filePath, 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+function titleFromPath(filePath) {
+  return basename(filePath, extname(filePath)).replace(/[-_]+/g, ' ');
+}
+
+function collectTextFiles(rootDir, dirs, { extensions = new Set(['.md', '.json']) } = {}) {
+  const files = [];
+  for (const dir of dirs) {
+    const abs = join(rootDir, dir);
+    if (!existsSync(abs)) continue;
+    const stack = [abs];
+    while (stack.length > 0) {
+      const current = stack.pop();
+      for (const entry of readdirSync(current, { withFileTypes: true })) {
+        const full = join(current, entry.name);
+        if (entry.isDirectory()) {
+          stack.push(full);
+          continue;
+        }
+        if (!entry.isFile() || !extensions.has(extname(entry.name))) continue;
+        try {
+          const stat = statSync(full);
+          if (stat.size > 512_000) continue;
+          const body = readFileSync(full, 'utf8');
+          files.push({
+            path: relative(rootDir, full),
+            title: titleFromPath(full),
+            body,
+          });
+        } catch {
+          // Ignore unreadable files in optional product intelligence folders.
+        }
+      }
+    }
+  }
+  return files.sort((a, b) => a.path.localeCompare(b.path));
+}
+
+export function loadStateSnapshot(rootDir) {
+  const contextJsonPath = join(rootDir, '.cx', 'context.json');
+  const workflowJsonPath = join(rootDir, '.cx', 'workflow.json');
+  const architecturePath = join(rootDir, 'docs', 'architecture.md');
+  const docsReadmePath = join(rootDir, 'docs', 'README.md');
+  const productIntelDocs = collectTextFiles(rootDir, [
+    '.cx/product-intel',
+    'docs/prd',
+    'docs/meta-prd',
+  ]);
+
+  return {
+    context: readJsonIfExists(contextJsonPath),
+    workflow: readJsonIfExists(workflowJsonPath),
+    architecture: readTextIfExists(architecturePath),
+    docsReadme: readTextIfExists(docsReadmePath),
+    productIntelDocs,
+    paths: {
+      contextJsonPath,
+      workflowJsonPath,
+      architecturePath,
+      docsReadmePath,
+    },
+  };
+}
+
+export function summarizeStateSnapshot(snapshot) {
+  return {
+    contextSummary: snapshot.context?.contextSummary ?? snapshot.context?.summary ?? null,
+    workflowTitle: snapshot.workflow?.title ?? null,
+    workflowPhase: snapshot.workflow?.phase ?? null,
+    workflowStatus: snapshot.workflow?.status ?? null,
+    lastContextSavedAt: snapshot.context?.savedAt ?? null,
+    lastWorkflowUpdatedAt: snapshot.workflow?.updatedAt ?? null,
+    hasArchitectureDoc: Boolean(snapshot.architecture),
+    hasDocsReadme: Boolean(snapshot.docsReadme),
+    productIntelDocCount: snapshot.productIntelDocs?.length ?? 0,
+  };
+}

package/lib/storage/sync.mjs

@@ -0,0 +1,164 @@
+#!/usr/bin/env node
+/**
+ * lib/storage/sync.mjs — sync file-state artifacts into shared Postgres indices.
+ */
+import crypto from 'node:crypto';
+import { loadStateSnapshot } from './state-source.mjs';
+import { createSqlClient, closeSqlClient, readVectorConfig } from './backend.mjs';
+import { EMBEDDING_MODEL, embedText } from './embeddings.mjs';
+
+function hashContent(content) {
+  return crypto.createHash('sha256').update(String(content || '')).digest('hex');
+}
+
+function toDocumentRows(rootDir, snapshot, project = 'construct') {
+  const rows = [];
+
+  if (snapshot.context) {
+    const body = JSON.stringify(snapshot.context, null, 2);
+    rows.push({
+      id: `${project}:context`,
+      project,
+      kind: 'context',
+      title: 'Context state',
+      summary: snapshot.context.contextSummary || snapshot.context.summary || null,
+      body,
+      source_path: '.cx/context.json',
+      tags: ['context', 'state', 'cx'],
+      content_hash: hashContent(body),
+    });
+  }
+
+  if (snapshot.workflow) {
+    const body = JSON.stringify(snapshot.workflow, null, 2);
+    rows.push({
+      id: `${project}:workflow`,
+      project,
+      kind: 'workflow',
+      title: snapshot.workflow.title || 'Workflow state',
+      summary: snapshot.workflow.summary || null,
+      body,
+      source_path: '.cx/workflow.json',
+      tags: ['workflow', 'state', 'cx'],
+      content_hash: hashContent(body),
+    });
+  }
+
+  if (snapshot.architecture) {
+    const body = snapshot.architecture;
+    rows.push({
+      id: `${project}:architecture`,
+      project,
+      kind: 'architecture',
+      title: 'Architecture docs',
+      summary: body.slice(0, 240),
+      body,
+      source_path: 'docs/architecture.md',
+      tags: ['architecture', 'docs'],
+      content_hash: hashContent(body),
+    });
+  }
+
+  if (snapshot.docsReadme) {
+    const body = snapshot.docsReadme;
+    rows.push({
+      id: `${project}:docs-readme`,
+      project,
+      kind: 'docs',
+      title: 'Docs index',
+      summary: body.slice(0, 240),
+      body,
+      source_path: 'docs/README.md',
+      tags: ['docs', 'index'],
+      content_hash: hashContent(body),
+    });
+  }
+
+  for (const doc of snapshot.productIntelDocs ?? []) {
+    const body = doc.body;
+    const rel = doc.path;
+    const kind = rel.startsWith('docs/prd/')
+      ? 'prd'
+      : rel.startsWith('docs/meta-prd/')
+        ? 'meta-prd'
+        : 'product-intel';
+    rows.push({
+      id: `${project}:${rel}`,
+      project,
+      kind,
+      title: doc.title,
+      summary: body.slice(0, 240),
+      body,
+      source_path: rel,
+      tags: ['product-intel', kind],
+      content_hash: hashContent(body),
+    });
+  }
+
+  return rows;
+}
+
+export async function ensureHybridSchema(client) {
+  const schema = await import('node:fs/promises').then((fs) => fs.readFile(new URL('../../db/migrations/001_init.sql', import.meta.url), 'utf8'));
+  await client.unsafe(schema);
+}
+
+export async function syncFileStateToSql(rootDir, { env = process.env, project = 'construct' } = {}) {
+  const client = createSqlClient(env);
+  if (!client) return { status: 'unavailable', note: 'No DATABASE_URL configured' };
+
+  try {
+    await ensureHybridSchema(client);
+    const snapshot = loadStateSnapshot(rootDir);
+    const rows = toDocumentRows(rootDir, snapshot, project);
+
+    let documentsSynced = 0;
+    let embeddingsSynced = 0;
+    for (const row of rows) {
+      await client`
+        insert into construct_documents (id, project, kind, title, summary, body, source_path, tags, content_hash, updated_at)
+        values (${row.id}, ${row.project}, ${row.kind}, ${row.title}, ${row.summary}, ${row.body}, ${row.source_path}, ${JSON.stringify(row.tags)}, ${row.content_hash}, now())
+        on conflict (id) do update set
+          project = excluded.project,
+          kind = excluded.kind,
+          title = excluded.title,
+          summary = excluded.summary,
+          body = excluded.body,
+          source_path = excluded.source_path,
+          tags = excluded.tags,
+          content_hash = excluded.content_hash,
+          updated_at = now()
+      `;
+      documentsSynced += 1;
+
+      const embedding = embedText([row.title, row.summary, row.body, row.source_path, row.kind].filter(Boolean).join('\n'));
+      await client`
+        insert into construct_embeddings (document_id, model, embedding, content_hash, updated_at)
+        values (${row.id}, ${EMBEDDING_MODEL}, ${embedding}, ${row.content_hash}, now())
+        on conflict (document_id) do update set
+          model = excluded.model,
+          embedding = excluded.embedding,
+          content_hash = excluded.content_hash,
+          updated_at = now()
+      `;
+      embeddingsSynced += 1;
+    }
+
+    await client`
+      insert into construct_sync_runs (project, source, documents_synced, embeddings_synced, status, note)
+      values (${project}, ${'file-state'}, ${documentsSynced}, ${0}, ${'ok'}, ${'synced file-state documents'})
+    `;
+
+    return {
+      status: 'ok',
+      documentsSynced,
+      embeddingsSynced,
+      embeddingModel: EMBEDDING_MODEL,
+      vector: readVectorConfig(env),
+    };
+  } catch (error) {
+    return { status: 'degraded', error: error?.message || 'sync failed' };
+  } finally {
+    await closeSqlClient(client);
+  }
+}

package/lib/storage/vector-store.mjs

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+/**
+ * lib/storage/vector-store.mjs — derived semantic retrieval facade.
+ */
+
+export function vectorStoreMode(env = process.env) {
+  if (env.CONSTRUCT_VECTOR_URL) return 'remote';
+  if (env.CONSTRUCT_VECTOR_INDEX_PATH) return 'local';
+  return 'file';
+}
+
+export function describeVectorStore(env = process.env) {
+  const mode = vectorStoreMode(env);
+  return {
+    mode,
+    configured: mode !== 'file',
+    sharedReady: mode === 'remote',
+    fallbackAvailable: true,
+    endpoint: env.CONSTRUCT_VECTOR_URL || null,
+    indexPath: env.CONSTRUCT_VECTOR_INDEX_PATH || null,
+    model: env.CONSTRUCT_VECTOR_MODEL || null,
+  };
+}
+
+function normalizeTerms(input) {
+  return String(input || '')
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((term) => term.length >= 3);
+}
+
+export function scoreSemanticText(text, query) {
+  const terms = normalizeTerms(query);
+  if (terms.length === 0) return 0;
+  const haystack = String(text || '').toLowerCase();
+  let score = 0;
+  for (const term of terms) {
+    if (haystack.includes(term)) score += 1;
+  }
+  return score;
+}
+
+export function vectorSearchLocal(records = [], query = '', { limit = 10 } = {}) {
+  return records
+    .map((record) => ({
+      record,
+      score: scoreSemanticText([
+        record.title,
+        record.summary,
+        record.body,
+        record.text,
+        Array.isArray(record.tags) ? record.tags.join(' ') : '',
+      ].filter(Boolean).join('\n'), query),
+    }))
+    .filter(({ score }) => score > 0)
+    .sort((a, b) => b.score - a.score || String(a.record.id || '').localeCompare(String(b.record.id || '')))
+    .slice(0, limit)
+    .map(({ record, score }) => ({ ...record, score }));
+}