@geraldmaron/construct 1.0.23 → 1.1.0

package/lib/observation-store.mjs

@@ -4,31 +4,36 @@
  * Stores distilled insights that specialists learn during work:
  *   - patterns, anti-patterns, dependency relationships, decisions, insights
  *   - each scoped to a role (cx-engineer, cx-architect, etc.)
- *   - vector-indexed for semantic search via hashing-bow-v1
+ *   - vector-indexed for semantic search via embedded LanceDB
  *
  * Storage layout:
  *   .cx/observations/index.json      — lightweight listing for fast filtering
  *   .cx/observations/<id>.json       — full observation record
- *   .cx/observations/vectors.json    — local vector index for semantic search
  */
 import crypto from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
-import { cosineSimilarity, rankByBm25 } from './storage/embeddings.mjs';
-import { embedSync as embedText } from './storage/embeddings-legacy.mjs';
 import { embedText as embedTextEngine } from './storage/embeddings-engine.mjs';
 import { VectorClient } from './storage/vector-client.mjs';
-import { withFileLockSync } from './storage/file-lock.mjs';
 import { ensureCxDir } from './project-init-shared.mjs';
 
 const OBS_DIR = '.cx/observations';
 const INDEX_FILE = 'index.json';
-const VECTORS_FILE = 'vectors.json';
 const MAX_INDEX = 1000;
 
-// The exact text fed to the embedder, and its content hash. addObservation and
-// the reconciliation pass MUST derive both the same way, so the (content_hash,
-// model) fingerprint they compare is computed identically.
+// VectorClient resolves its store from CONSTRUCT_LANCEDB_PATH and otherwise
+// falls back to process.cwd(). Observations are scoped to rootDir, so when no
+// explicit path is configured (tests, unmanaged checkouts) the vector store
+// must follow rootDir — not the caller's cwd — or a reader in a different cwd
+// than the writer sees an empty table. A configured path (managed installs)
+// is left untouched so the home-global store keeps working.
+
+function vectorClientFor(rootDir) {
+  const env = process.env.CONSTRUCT_LANCEDB_PATH
+    ? process.env
+    : { ...process.env, CONSTRUCT_LANCEDB_PATH: path.join(rootDir, '.cx', 'lancedb') };
+  return new VectorClient({ env });
+}
 
 export function observationSearchText(obs) {
   return [obs.summary, obs.content, ...(obs.tags || [])].filter(Boolean).join(' ');
@@ -40,9 +45,6 @@ export function observationContentHash(text) {
 const MAX_SUMMARY = 500;
 const MAX_CONTENT = 2000;
 const MAX_TAGS = 10;
-// Size guardrail: surfaces in `construct doctor` when the observation tree
-// exceeds this byte cap. Doesn't drop data — the operator decides whether to
-// run `construct memory consolidate` (which prunes archive) or raise the cap.
 const MAX_BYTES = 50 * 1024 * 1024;
 
 const VALID_CATEGORIES = new Set([
@@ -50,8 +52,6 @@ const VALID_CATEGORIES = new Set([
 ]);
 
 function ensureDir(dir, rootDir = null) {
-  // Preserves the construct context invariant: any code that creates a .cx/
-  // subdirectory also initializes context.md from the project template.
   if (rootDir) ensureCxDir(rootDir);
   if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
 }
@@ -64,10 +64,6 @@ function indexPath(rootDir) {
   return path.join(obsDir(rootDir), INDEX_FILE);
 }
 
-function vectorsPath(rootDir) {
-  return path.join(obsDir(rootDir), VECTORS_FILE);
-}
-
 function generateId() {
   const ts = Date.now().toString(36);
   const rand = crypto.randomBytes(4).toString('hex');
@@ -132,30 +128,6 @@ function writeIndex(rootDir, entries) {
   return { dropped };
 }
 
-function readVectors(rootDir) {
-  const p = vectorsPath(rootDir);
-  if (!fs.existsSync(p)) return [];
-  try {
-    return JSON.parse(fs.readFileSync(p, 'utf8'));
-  } catch {
-    return [];
-  }
-}
-
-function writeVectors(rootDir, records) {
-  ensureDir(obsDir(rootDir), rootDir);
-  const dropped = Math.max(0, records.length - MAX_INDEX);
-  if (dropped > 0) {
-    logCapDrop(rootDir, 'observation-vectors', dropped, records.length);
-  }
-  fs.writeFileSync(vectorsPath(rootDir), JSON.stringify(records, null, 2) + '\n');
-  return { dropped };
-}
-
-/**
- * Add a new observation and vectorize it for semantic search.
- * Uses SQL + neural embeddings when available, falls back to local JSON + hashing.
- */
 export async function addObservation(rootDir, {
   role = 'unknown',
   category = 'insight',
@@ -175,10 +147,9 @@ export async function addObservation(rootDir, {
   const clampedSummary = clamp(String(summary), MAX_SUMMARY);
   const clampedContent = clamp(String(content), MAX_CONTENT);
   const clampedTags = (Array.isArray(tags) ? tags : []).slice(0, MAX_TAGS).map(String);
-  // Structured metadata: opt-in. Capped at 2 KB stringified so a misuse can't
-  // bloat the per-id JSON. Plain object only; rejects arrays and functions.
   const clampedExtras = sanitizeExtras(extras);
 
+  const effectiveProject = project ? String(project) : rootDir.split('/').pop();
   const record = {
     id,
     role: String(role),
@@ -186,7 +157,7 @@ export async function addObservation(rootDir, {
     summary: clampedSummary,
     content: clampedContent,
     tags: clampedTags,
-    project: project ? String(project) : null,
+    project: effectiveProject,
     confidence: Math.max(0, Math.min(1, Number(confidence) || 0.8)),
     source: source || null,
     gitSha: gitSha ? String(gitSha).slice(0, 40) : null,
@@ -195,37 +166,30 @@ export async function addObservation(rootDir, {
     updatedAt: now,
   };
 
-  // Lock the per-store index so concurrent writers (CLI + hooks) cannot
-  // corrupt the index/vectors JSON pair. The observation JSON file is
-  // unique-per-id and doesn't need the lock; the shared structures do.
   ensureDir(obsDir(rootDir), rootDir);
   fs.writeFileSync(
     path.join(obsDir(rootDir), `${id}.json`),
     JSON.stringify(record, null, 2) + '\n',
   );
 
-  let indexDropped = 0;
-  withFileLockSync(indexPath(rootDir), () => {
-    const index = readIndex(rootDir);
-    index.unshift({
-      id,
-      role: record.role,
-      category: record.category,
-      summary: record.summary,
-      project: record.project,
-      createdAt: now,
-    });
-    indexDropped = writeIndex(rootDir, index).dropped;
+  const index = readIndex(rootDir);
+  index.unshift({
+    id,
+    role: record.role,
+    category: record.category,
+    summary: record.summary,
+    project: record.project,
+    createdAt: now,
   });
+  const indexDropped = writeIndex(rootDir, index).dropped;
   if (indexDropped > 0) {
     record.capDropped = indexDropped;
     process.stderr.write(`[observation-store] observation cap reached: ${indexDropped} oldest entries evicted (cap=${MAX_INDEX})\n`);
   }
 
-  // Try SQL storage with neural embeddings
   try {
-    const client = new VectorClient();
-    if (await client.isHealthy() && await client.isPgvectorEnabled()) {
+    const client = vectorClientFor(rootDir);
+    if (await client.isHealthy()) {
       const searchText = observationSearchText(record);
       const { embedding, model } = await embedTextEngine(searchText);
       await client.storeObservation({
@@ -234,34 +198,15 @@ export async function addObservation(rootDir, {
         contentHash: observationContentHash(searchText),
         model,
       });
-      await client.close();
-      return record;
     }
     await client.close();
   } catch (err) {
-    // A reachable backend that still fails to store (dimension drift, auth, transient
-    // network) must not look identical to "no backend configured" — surface it, then
-    // preserve the observation in the local index so the write never silently vanishes.
-
-    process.stderr.write(`[observation-store] SQL store failed; kept observation in local index: ${err?.message || err}\n`);
+    process.stderr.write(`[observation-store] vector store failed; kept observation in local index: ${err?.message || err}\n`);
   }
 
-  // Fallback: local JSON vector index with hashing-bow-v1.
-  const searchText = [record.summary, record.content, ...record.tags].filter(Boolean).join(' ');
-  const embedding = embedText(searchText);
-  withFileLockSync(vectorsPath(rootDir), () => {
-    const vectors = readVectors(rootDir);
-    vectors.push({ id, embedding, role: record.role, category: record.category, project: record.project });
-    writeVectors(rootDir, vectors.slice(-MAX_INDEX));
-  });
-
   return record;
 }
 
-/**
- * Search observations using SQL + neural embeddings when available,
- * falling back to hybrid cosine + BM25 on local JSON.
- */
 export async function searchObservations(rootDir, query, {
   role = null,
   category = null,
@@ -270,16 +215,15 @@ export async function searchObservations(rootDir, query, {
 } = {}) {
   if (!query) return [];
 
-  // Try SQL + neural embeddings first
   try {
-    const client = new VectorClient();
-    if (await client.isHealthy() && await client.isPgvectorEnabled()) {
+    const client = vectorClientFor(rootDir);
+    if (await client.isHealthy()) {
       const { embedding } = await embedTextEngine(String(query));
       const results = await client.searchObservations({
         project: project || rootDir.split('/').pop(),
         queryEmbedding: embedding,
         limit,
-        minSimilarity: 0.1,
+        minSimilarity: 0.01,
         role,
         category,
       });
@@ -288,6 +232,7 @@ export async function searchObservations(rootDir, query, {
         id: r.id,
         role: r.role,
         category: r.category,
+        project: r.project,
         summary: r.summary,
         content: r.content,
         tags: r.tags || [],
@@ -299,65 +244,12 @@ export async function searchObservations(rootDir, query, {
     }
     await client.close();
   } catch {
-    // Fall through to local JSON search
+    // If Vector search fails, return empty since fallback is removed.
   }
 
-  // Fallback: local JSON + hashing-bow-v1
-  const queryEmbedding = embedText(String(query));
-  let vectors = readVectors(rootDir);
-
-  if (role) vectors = vectors.filter((v) => v.role === role);
-  if (category) vectors = vectors.filter((v) => v.category === category);
-  if (project) vectors = vectors.filter((v) => v.project === project);
-
-  // Cosine pass
-  const cosineScored = vectors
-    .map((v) => ({
-      id: v.id,
-      score: cosineSimilarity(queryEmbedding, v.embedding || []),
-    }))
-    .filter(({ score }) => score > 0.05);
-
-  // Load full records for BM25 pass
-  const candidateIds = new Set(cosineScored.map((v) => v.id));
-  const recentIndex = readIndex(rootDir);
-  const filtered = role ? recentIndex.filter((e) => e.role === role) : recentIndex;
-  const filteredCat = category ? filtered.filter((e) => e.category === category) : filtered;
-  const filteredProj = project ? filteredCat.filter((e) => e.project === project) : filteredCat;
-  for (const entry of filteredProj.slice(0, Math.max(limit * 3, 30))) candidateIds.add(entry.id);
-
-  const candidateRecords = [...candidateIds]
-    .map((id) => getObservation(rootDir, id))
-    .filter(Boolean)
-    .map((r) => ({ ...r, text: [r.summary, r.content, ...(r.tags || [])].filter(Boolean).join(' ') }));
-
-  const bm25Scored = rankByBm25(candidateRecords, query, { limit: limit * 2 });
-
-  // Merge: highest score wins per id
-  const scoreMap = new Map();
-  for (const { id, score } of cosineScored) {
-    scoreMap.set(id, score);
-  }
-  for (const item of bm25Scored) {
-    const prev = scoreMap.get(item.id) || 0;
-    const bm25Max = bm25Scored[0]?.score || 1;
-    const normalized = Math.min(item.score / bm25Max, 1);
-    scoreMap.set(item.id, Math.max(prev, normalized));
-  }
-
-  return [...scoreMap.entries()]
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, limit)
-    .map(([id, score]) => {
-      const record = getObservation(rootDir, id);
-      return record ? { ...record, score } : null;
-    })
-    .filter(Boolean);
+  return [];
 }
 
-/**
- * List observations from the index with optional filters.
- */
 export function listObservations(rootDir, {
   role = null,
   category = null,
@@ -371,9 +263,6 @@ export function listObservations(rootDir, {
   return entries.slice(0, limit);
 }
 
-/**
- * Load a full observation record by ID.
- */
 export function getObservation(rootDir, id) {
   const filePath = path.join(obsDir(rootDir), `${id}.json`);
   if (!fs.existsSync(filePath)) return null;
@@ -384,31 +273,17 @@ export function getObservation(rootDir, id) {
   }
 }
 
-/**
- * Delete an observation by id.
- */
 export function deleteObservation(rootDir, id) {
   const filePath = path.join(obsDir(rootDir), `${id}.json`);
   if (fs.existsSync(filePath)) fs.rmSync(filePath, { force: true });
 
-  withFileLockSync(indexPath(rootDir), () => {
-    const index = readIndex(rootDir);
-    const filtered = index.filter((e) => e.id !== id);
-    if (filtered.length !== index.length) writeIndex(rootDir, filtered);
-  });
-
-  withFileLockSync(vectorsPath(rootDir), () => {
-    const vectors = readVectors(rootDir);
-    const filteredVec = vectors.filter((v) => v.id !== id);
-    if (filteredVec.length !== vectors.length) writeVectors(rootDir, filteredVec);
-  });
+  const index = readIndex(rootDir);
+  const filtered = index.filter((e) => e.id !== id);
+  if (filtered.length !== index.length) writeIndex(rootDir, filtered);
 
   return true;
 }
 
-/**
- * Count observations, optionally filtered.
- */
 export function countObservations(rootDir, { role = null, project = null } = {}) {
   let entries = readIndex(rootDir);
   if (role) entries = entries.filter((e) => e.role === role);
@@ -416,9 +291,6 @@ export function countObservations(rootDir, { role = null, project = null } = {})
   return entries.length;
 }
 
-// On-disk byte size of the observation tree (live + archive) at rootDir.
-// Backs the `Observation size` doctor surface so size pressure shows up
-// independently of count-based caps.
 export function getObservationsSize(rootDir) {
   const root = obsDir(rootDir);
   if (!fs.existsSync(root)) return 0;
@@ -439,4 +311,4 @@ export function getObservationsSize(rootDir) {
 export function checkObservationsSize(rootDir, { cap = MAX_BYTES } = {}) {
   const size = getObservationsSize(rootDir);
   return { size, cap, ok: size <= cap };
-}
+}

package/lib/ollama/capability-store.mjs

@@ -0,0 +1,78 @@
+/**
+ * lib/ollama/capability-store.mjs — durable record of local-model agentic coherence.
+ *
+ * `construct doctor --probe-local` measures whether each Ollama model can drive
+ * the agentic loop or collapses into word salad, and writes the verdict here.
+ * sync and doctor READ this store (they never probe — probing loads each model and
+ * costs minutes) to skip Modelfile provisioning for collapsed models, inform the
+ * MCP-trim decision, and warn when a configured default model cannot do agentic
+ * work. Verdicts are keyed by the model's `ollama list` digest so a re-pulled tag
+ * goes stale and is re-probed rather than trusted blindly.
+ *
+ * Lives at ~/.cx/local-models.json because local-model capability is a property of
+ * the machine's Ollama install, not of any one project.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { cxDir } from "../paths.mjs";
+
+const STORE_VERSION = 1;
+
+export function localModelsPath() {
+  return path.join(cxDir(), "local-models.json");
+}
+
+export function readCapabilityStore() {
+  try {
+    const raw = JSON.parse(fs.readFileSync(localModelsPath(), "utf8"));
+    if (raw && raw.version === STORE_VERSION && raw.models && typeof raw.models === "object") return raw;
+  } catch { /* missing or malformed — treat as empty */ }
+  return { version: STORE_VERSION, models: {} };
+}
+
+// Persist one probe result. probeResult is the object probeAgenticCoherence
+// returns ({ ok, coherent, repeatRatio, uniqueRatio, ... }); a probe that errored
+// (ok=false) is not recorded, so a transient failure never overwrites a real verdict.
+
+export function recordProbeResult(model, probeResult, digest = null) {
+  if (!probeResult || probeResult.ok !== true) return readCapabilityStore();
+  const store = readCapabilityStore();
+  store.models[model] = {
+    verdict: probeResult.coherent ? "COHERENT" : "COLLAPSED",
+    coherent: probeResult.coherent === true,
+    calledTool: probeResult.calledTool === true,
+    repeatRatio: probeResult.repeatRatio ?? null,
+    uniqueRatio: probeResult.uniqueRatio ?? null,
+    digest: digest ?? null,
+    probedAt: new Date().toISOString(),
+  };
+  const dir = path.dirname(localModelsPath());
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(localModelsPath(), `${JSON.stringify(store, null, 2)}\n`);
+  return store;
+}
+
+export function getModelVerdict(model) {
+  return readCapabilityStore().models[model] ?? null;
+}
+
+// A verdict is stale when no digest is recorded for it or the current digest
+// differs from the one it was measured against — callers re-probe rather than
+// trust a verdict about different model bytes.
+
+export function isVerdictStale(model, currentDigest) {
+  const entry = getModelVerdict(model);
+  if (!entry) return true;
+  if (!currentDigest || !entry.digest) return true;
+  return entry.digest !== currentDigest;
+}
+
+// COLLAPSED only counts when the digest still matches: a stale or unknown verdict
+// must not strand a model. Callers warn-and-allow-override rather than hide.
+
+export function isKnownCollapsed(model, currentDigest = null) {
+  const entry = getModelVerdict(model);
+  if (!entry) return false;
+  if (currentDigest && isVerdictStale(model, currentDigest)) return false;
+  return entry.verdict === "COLLAPSED";
+}