mindforge-cc 11.9.1 → 11.9.3

package/bin/learning/instinct-cli.js

@@ -23,7 +23,9 @@ const fs = require('fs');
 const path = require('path');
 
 const guard = require('./lib/ssrf-guard');
+const { redactSecrets } = require('../utils/redact-secrets');
 const { detectProject } = require('../hooks/lib/detect-project');
+const { withFileLock } = require('../utils/file-lock');
 
 const CONFIG_PATH = path.join(process.cwd(), '.mindforge', 'config.json');
 
@@ -71,33 +73,15 @@ function writeStoreAtomic(p, entries) {
 }
 
 /**
- * Advisory lock (Node has no fcntl): exclusive lockfile create, spin-with-timeout,
- * stale-break after 10s by mtime. Runs fn() while held, releases in finally.
+ * Advisory lock — now a thin delegate to the shared fail-closed lock (LOCK-01).
+ * The implementation moved to bin/utils/file-lock.js so the audit chain and the
+ * knowledge graph use the SAME lock semantics as the instinct store. Behaviour is
+ * unchanged: O_EXCL create, 50 tries, ~20ms waits, 10s stale reclaim, unlink in
+ * finally, THROW (never write anyway) when unacquirable — same error text.
  * Read the store INSIDE this so a prune/import rewrite can't race a hook append.
  */
 function withStoreLock(p, fn) {
-  const lock = `${p}.lock`;
-  const maxTries = 50, waitMs = 20, staleMs = 10000;
-  let held = false;
-  for (let i = 0; i < maxTries && !held; i++) {
-    try {
-      const fd = fs.openSync(lock, 'wx');
-      fs.closeSync(fd);
-      held = true;
-    } catch (err) {
-      if (err.code !== 'EEXIST') throw err;
-      // stale-break: if the lockfile is older than staleMs, remove it.
-      try {
-        const age = Date.now() - fs.statSync(lock).mtimeMs;
-        if (age > staleMs) { fs.unlinkSync(lock); continue; }
-      } catch { /* lock vanished — retry */ }
-      const until = Date.now() + waitMs;
-      while (Date.now() < until) { /* busy-wait (short) */ }
-    }
-  }
-  if (!held) throw new Error(`could not acquire instinct-store lock: ${lock}`);
-  try { return fn(); }
-  finally { try { fs.unlinkSync(lock); } catch { /* already gone */ } }
+  return withFileLock(p, fn, { label: 'instinct-store' });
 }
 
 function currentProjectId() {
@@ -214,7 +198,13 @@ async function cmdImport(args, cfg) {
   for (const e of incoming) {
     if (!e || !guard.validateInstinctId(e.id || '')) { process.stderr.write(`skipping entry with invalid id: ${e && e.id}\n`); continue; }
     if ((Number(e.confidence) || 0) < minC) continue;
+    // Import is the store's second ingress and it accepts an https:// source, so incoming text is
+    // untrusted twice over: it may carry a credential from whoever exported it. Redacting here
+    // rather than at every egress means export, promote, list and the LLM promotion path into a
+    // published SKILL.md all inherit already-clean data — one choke point instead of five.
     valid.push(Object.assign({}, e, {
+      observation: redactSecrets(e.observation),
+      behavior: redactSecrets(e.behavior),
       project: projectName, project_id: scopeId, source: 'imported',
       imported_from: source, created_at: now, updated_at: now,
     }));

package/bin/memory/knowledge-capture.js

@@ -133,7 +133,14 @@ function createCausalEdges(bugId, allEntries, vectors) {
           weight: sim,
           reason: `Bug pattern potentially caused by code pattern (sim: ${sim.toFixed(3)})`,
         });
-      } catch { /* skip duplicates or self-refs */ }
+      } catch (err) {
+        // LOCK-01: this catch predates the graph-edges lock. Without the guard below,
+        // an edge LOST to lock contention is reported as a skipped duplicate — a
+        // silent write loss wearing the label of a no-op.
+        if (/could not acquire/.test(err.message)) {
+          console.warn(`[KnowledgeCapture] edge NOT written — graph-edges lock unavailable: ${err.message}`);
+        }
+      }
     }
   }
 }
@@ -165,7 +172,14 @@ function createInformsEdges(decisionId, allEntries, vectors) {
           weight: sim,
           reason: `Decision informs domain knowledge (sim: ${sim.toFixed(3)})`,
         });
-      } catch { /* skip duplicates or self-refs */ }
+      } catch (err) {
+        // LOCK-01: this catch predates the graph-edges lock. Without the guard below,
+        // an edge LOST to lock contention is reported as a skipped duplicate — a
+        // silent write loss wearing the label of a no-op.
+        if (/could not acquire/.test(err.message)) {
+          console.warn(`[KnowledgeCapture] edge NOT written — graph-edges lock unavailable: ${err.message}`);
+        }
+      }
     }
   }
 }
@@ -180,7 +194,13 @@ function reinforceRelatedEdges(nodeId) {
     for (const edge of edges.slice(0, 3)) { // Top 3 edges only
       Graph.reinforceEdge(edge.id);
     }
-  } catch { /* non-critical */ }
+  } catch (err) {
+    // LOCK-01: reinforceEdge now takes the graph-edges lock, so "non-critical" would
+    // also absorb a contention failure. Surface that one; keep the rest quiet.
+    if (/could not acquire/.test(err.message)) {
+      console.warn(`[KnowledgeCapture] reinforcement NOT recorded — graph-edges lock unavailable: ${err.message}`);
+    }
+  }
 }
 
 // ── Event-specific capture functions ─────────────────────────────────────────

package/bin/memory/knowledge-graph.js

@@ -17,6 +17,7 @@ const crypto   = require('crypto');
 const Store    = require('./knowledge-store');
 const Embedder = require('./embedding-engine');
 const EISClient = require('./eis-client');
+const { withFileLock } = require('../utils/file-lock');
 
 // ── Edge Types ────────────────────────────────────────────────────────────────
 const EDGE_TYPES = Object.freeze({
@@ -59,6 +60,19 @@ function ensureDir(dir) {
   if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
 }
 
+/**
+ * LOCK-01: runs fn() holding the exclusive lock on graph-edges.jsonl.
+ * Every edge write below is a READ-modify-APPEND (readAllEdges -> mutate -> append),
+ * so the read MUST be inside the same critical section as the append or concurrent
+ * writers lose updates. Measured over 4 runs of 8 processes x 20 reinforceEdge calls:
+ * unlocked lost 93-129 of 160 increments (final traversal_count 31-67); locked lost 0
+ * of 160 in every run.
+ * Re-entrant: applyDecay() holds this lock and calls deprecateEdge(), which re-takes it.
+ */
+function withEdgesLock(fn, opts = {}) {
+  return withFileLock(getPaths().EDGES_PATH, fn, { maxTries: 150, label: 'graph-edges', ...opts });
+}
+
 // ── Edge CRUD ─────────────────────────────────────────────────────────────────
 
 /**
@@ -108,8 +122,12 @@ function addEdge(edge) {
   const payload = JSON.stringify({ ...record, checksum: '' });
   record.checksum = crypto.createHash('sha256').update(payload).digest('hex');
 
-  fs.appendFileSync(paths.EDGES_PATH, JSON.stringify(record) + '\n');
-  invalidateAdjacencyCache();
+  // LOCK-01: serialise against the read-modify-append paths below, whose readAllEdges()
+  // snapshot would otherwise go stale if this append landed inside their window.
+  withEdgesLock(() => {
+    fs.appendFileSync(paths.EDGES_PATH, JSON.stringify(record) + '\n');
+    invalidateAdjacencyCache();
+  });
   return id;
 }
 
@@ -144,19 +162,23 @@ function readAllEdges() {
  */
 function deprecateEdge(edgeId, reason) {
   const paths = getPaths();
-  const edges = readAllEdges();
-  const edge = edges.find(e => e.id === edgeId);
-  if (!edge) return;
-
-  const deprecated = {
-    ...edge,
-    deprecated: true,
-    deprecated_reason: reason,
-    deprecated_at: new Date().toISOString(),
-  };
-
-  fs.appendFileSync(paths.EDGES_PATH, JSON.stringify(deprecated) + '\n');
-  invalidateAdjacencyCache();
+  // LOCK-01: read-modify-append is ONE critical section — the readAllEdges() snapshot
+  // must not be able to go stale before our append lands.
+  return withEdgesLock(() => {
+    const edges = readAllEdges();
+    const edge = edges.find(e => e.id === edgeId);
+    if (!edge) return;
+
+    const deprecated = {
+      ...edge,
+      deprecated: true,
+      deprecated_reason: reason,
+      deprecated_at: new Date().toISOString(),
+    };
+
+    fs.appendFileSync(paths.EDGES_PATH, JSON.stringify(deprecated) + '\n');
+    invalidateAdjacencyCache();
+  });
 }
 
 /**
@@ -165,22 +187,26 @@ function deprecateEdge(edgeId, reason) {
  */
 function reinforceEdge(edgeId) {
   const paths = getPaths();
-  const edges = readAllEdges();
-  const edge = edges.find(e => e.id === edgeId);
-  if (!edge) return;
-
-  const reinforced = {
-    ...edge,
-    weight: Math.min(2.0, edge.weight + 0.1),
-    last_traversed: new Date().toISOString(),
-    traversal_count: (edge.traversal_count || 0) + 1,
-  };
-
-  // Recompute checksum
-  const payload = JSON.stringify({ ...reinforced, checksum: '' });
-  reinforced.checksum = crypto.createHash('sha256').update(payload).digest('hex');
-
-  fs.appendFileSync(paths.EDGES_PATH, JSON.stringify(reinforced) + '\n');
+  // LOCK-01: weight += 0.1 and traversal_count += 1 are read-modify-writes. Unlocked,
+  // 8 concurrent processes x 20 reinforcements lost 93-129 of 160 increments (4 runs).
+  return withEdgesLock(() => {
+    const edges = readAllEdges();
+    const edge = edges.find(e => e.id === edgeId);
+    if (!edge) return;
+
+    const reinforced = {
+      ...edge,
+      weight: Math.min(2.0, edge.weight + 0.1),
+      last_traversed: new Date().toISOString(),
+      traversal_count: (edge.traversal_count || 0) + 1,
+    };
+
+    // Recompute checksum
+    const payload = JSON.stringify({ ...reinforced, checksum: '' });
+    reinforced.checksum = crypto.createHash('sha256').update(payload).digest('hex');
+
+    fs.appendFileSync(paths.EDGES_PATH, JSON.stringify(reinforced) + '\n');
+  });
 }
 
 // ── Adjacency Index (with persistent cache) ─────────────────────────────────
@@ -489,6 +515,19 @@ function autoCreateEdges(entryId, vectors) {
  * @returns {{ decayed: number, pruned: number }}
  */
 function applyDecay() {
+  // LOCK-01: the ENTIRE pass is one read-modify-write — one readAllEdges() snapshot
+  // drives appends at the bottom of this loop AND nested deprecateEdge() calls. The
+  // nested calls re-take this same lock; withFileLock is re-entrant for that reason.
+  // staleMs is raised far above the 10s module default. This pass is O(n^2) — the
+  // nested deprecateEdge() re-reads the whole edges file per prune — and was measured
+  // holding this lock 9.1s at 2,000 edges and 14.4s at 2,500. Past STALE_MS another
+  // process classifies this LIVE lock as orphaned, unlinks it, and enters the critical
+  // section alongside us, so mutual exclusion would silently vanish on any real-sized
+  // graph — the same "looks guarded, isn't" failure this item exists to remove.
+  return withEdgesLock(() => applyDecayLocked(), { staleMs: 900000 });
+}
+
+function applyDecayLocked() {
   const edges = readAllEdges();
   const now = Date.now();
   let decayed = 0;