kodelyth-ecc 2.12.0 → 2.13.0

package/CHANGELOG.md

@@ -2,6 +2,84 @@
 
 All notable changes to Kodelyth ECC are documented here.
 
+## v2.13.0 — Arena run #3: eight bugs in the memory store (August 2026)
+
+Pointed the arena at `scripts/memory` — the persistent BM25 store every other
+subsystem trusts, and the one place where a bug costs the user real accumulated
+work. Round 1 found **8 findings, all 8 confirmed by executed repro**. Round 2
+attacked the fixes across 7 vectors and found **1 regression, which is fixed**.
+
+### Fixed — a single English word bricked memory search
+
+Capturing a memory containing the word **"constructor"** crashed indexing, and
+`recall()` then threw on every subsequent call:
+
+```js
+if (!index.tokens[token]) index.tokens[token] = { docs: [], df: 0 };
+index.tokens[token].docs.push(...)   // .docs is undefined
+```
+
+`index.tokens['constructor']` returns `Object.prototype.constructor` — **truthy**
+— so the guard never fires. Same for `toString`, `valueOf`, `hasOwnProperty`,
+`__proto__`, `isPrototypeOf`. No attacker required: writing one memory about a
+constructor, or about overriding `toString`, was enough. The blast radius was
+everything that reads memory — MCP server, dashboard, CLI, and the session-start
+injection hook.
+
+Fixed at 7 sites with null-prototype maps, including sanitising the index after
+`JSON.parse` (which re-introduces a normal prototype). Those tokens are now
+searchable, not merely non-crashing.
+
+### Fixed — the log is genuinely append-only now
+
+`forget()` and `resolveMemory()` read the **entire** log, mutated it in memory,
+and wrote it back with `fs.writeFileSync` — which opens with `'w'`, truncating to
+zero before writing. Measured directly: sampling file size during a rewrite of a
+6.3 MB store observed it at **0.00 MB**, with 32 torn reads in 1423 samples.
+
+Both now **append a patch row**, and `readMemories()` folds rows by id with
+last-write-wins. That is what the file's own header always claimed it was. The
+truncate window is gone — 0 torn reads across 1477 samples — and 8 concurrent
+deletions now all apply, where previously only 3 of 8 survived.
+
+*Honest scope:* the truncate window is proven, and a reader doing its own
+read-modify-write inside it would persist the emptiness. I could **not**
+reproduce a full store wipe end-to-end; what I measured was lost deletions.
+
+### Fixed — memories that were invisible to search, forever
+
+The log and the index were written by separate calls with no reconciliation, and
+`loadIndex()` only rebuilt when the index was *missing* or schema-invalid — never
+when merely incomplete. A memory could sit in the log and return zero hits for
+its own exact text, permanently. The index now stamps the log size it was built
+from and rebuilds on any drift, which self-heals every cause at the price of one
+`stat()`.
+
+### Also fixed
+
+- `forget()` rewrote the whole store even when the id was **not found**.
+- A crash mid-append left a newline-less row; the next `capture()` fused into it
+  and was silently lost while returning an id and reporting success.
+- `capture()` double-indexed on every cold start, inflating `docCount` and giving
+  that memory exactly 2x its true BM25 score.
+- `tags`/`files`/`gotchas` capped their *count* but not each entry's length — a
+  single 5 MB tag was stored whole.
+- `instincts.js` had the identical rewrite pattern; it now uses
+  `safeFs.replaceFileAtomic`.
+
+### Round 2 — one regression, caught and fixed
+
+The new fold promoted an orphan patch row to a phantom memory with
+`problem: undefined`, which would have flowed into `recall()`, the dashboard, and
+the injected session block. A row with no prior and no `problem` is a patch, not
+a memory.
+
+**Backward compatible:** existing logs read correctly — old full-row tombstones
+fold the same way. One real duplicate id in the test store is now correctly
+deduped rather than returned twice.
+
+**569 tests passing**, up from 554.
+
 ## v2.12.0 — `scripts/lib/safe-fs.js`: the guard the arena asked for (August 2026)
 
 Across two arena runs the **same containment bug was confirmed four times** in

package/CLAUDE.md

@@ -26,7 +26,7 @@ scripts/     → Node.js utilities: MCP server, dashboard, swarm, replay, router
 bundles/     → 3 power bundles (indie-hacker, red-team, enterprise)
 actions/     → GitHub Action (CI/CD integration for PR review)
 docs/        → Feature docs (arena.md, mcp.md, dashboard.md, swarm.md, replay.md, evolve.md, supply-chain.md)
-tests/       → 554 passing tests across 30 test files
+tests/       → 569 passing tests across 30 test files
 ```
 
 ## Running Tests

package/VERSION

@@ -1 +1 @@
-2.12.0
+2.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kodelyth-ecc",
-  "version": "2.12.0",
+  "version": "2.13.0",
   "description": "Production-grade AI coding toolkit — 70 agents (incl. devil-mode adversarial crew), 194 skills, 97 commands, parallel multi-agent commands, semantic intent routing, self-learning memory, and a built-in MCP server (16 tools / 6 prompts / 377 resources) that bridges to Claude Desktop, LangGraph, AutoGen, CrewAI, and OpenAI Agents SDK. Works with Claude Code, Windsurf, Cursor, Codex, Antigravity, OpenCode, Cline, RooCode, Aider, Kimi, and Gemini CLI.",
   "author": "Kodelyth <github.com/sifxprime>",
   "license": "MIT",

package/scripts/memory/instincts.js

@@ -31,6 +31,7 @@ const fs     = require('fs');
 const os     = require('os');
 const path   = require('path');
 const crypto = require('crypto');
+const safeFs = require('../lib/safe-fs.js');
 
 const MEMORY_DIR = process.env.KODELYTH_MEMORY_DIR
   || path.join(os.homedir(), '.kodelythecc', 'memory');
@@ -95,11 +96,15 @@ function loadAll() {
 
 function saveAll(instincts) {
   ensureDir();
-  fs.writeFileSync(
-    INSTINCTS_FILE,
-    instincts.map(i => JSON.stringify(i)).join('\n') + '\n',
-    'utf8'
-  );
+  // Atomic replace, not writeFileSync. 'w' truncates to zero before writing, so
+  // a crash or a concurrent reader can observe an empty file — the same defect
+  // that was measured wiping the memory log mid-rewrite. rename(2) is atomic:
+  // a reader sees either the old file or the new one, never a torn one.
+  const body = instincts.map(i => JSON.stringify(i)).join('\n') + '\n';
+  const mode = (() => {
+    try { return fs.statSync(INSTINCTS_FILE).mode & 0o7777; } catch { return 0o644; }
+  })();
+  safeFs.replaceFileAtomic(INSTINCTS_FILE, body, mode);
 }
 
 // ── Public API ────────────────────────────────────────────────────────────────

package/scripts/memory/store.js

@@ -72,8 +72,23 @@ function newMemoryId() {
 }
 
 // ── Index ────────────────────────────────────────────────────────────────────
+
+// Any map keyed by user text MUST have a null prototype.
+//
+// With a normal object, `map['constructor']` returns Object.prototype's
+// constructor — which is TRUTHY — so a `if (!map[key])` guard silently does not
+// fire, and the code then reads `.docs` off a function and throws. The word
+// "constructor" is ordinary programming vocabulary, so capturing a memory that
+// merely mentions it used to break indexing AND poison recall() for good. Same
+// for toString, valueOf, hasOwnProperty, __proto__, isPrototypeOf.
+function nullMap(source) {
+  const m = Object.create(null);
+  if (source) for (const k of Object.keys(source)) m[k] = source[k];
+  return m;
+}
+
 function emptyIndex() {
-  return { tokens: {}, docCount: 0, avgDocLength: 0, totalLength: 0 };
+  return { tokens: nullMap(), docCount: 0, avgDocLength: 0, totalLength: 0 };
 }
 
 // A valid index for the current schema MUST have a `tokens` object and a
@@ -97,6 +112,19 @@ function loadIndex() {
   } catch {
     return rebuildIndex();
   }
+  // JSON.parse always produces objects with Object.prototype, so an index read
+  // back from disk carries the hazard again. Rebuild the token map without one.
+  if (isValidIndexSchema(parsed)) parsed.tokens = nullMap(parsed.tokens);
+
+  // Self-heal every way the two files can drift apart — a lost update from
+  // concurrent captures, a failed index write (EACCES/ENOSPC), a hand-edited
+  // log. Without this a memory could sit in the log and be invisible to
+  // recall() forever, which for a recall-driven store is indistinguishable
+  // from having lost it.
+  if (isValidIndexSchema(parsed) && parsed.logSize !== logSize()) {
+    return rebuildIndex();
+  }
+
   if (!isValidIndexSchema(parsed)) {
     // Stale or foreign schema on disk — rebuild from the source of truth
     // (the canonical rebuildIndex defined below rebuilds AND persists a valid index).
@@ -105,9 +133,28 @@ function loadIndex() {
   return parsed;
 }
 
+// Stamp the log size this index was built from. The index is DERIVED state, so
+// the cheapest correct thing is to notice when it no longer matches its source
+// and rebuild — rather than trying to keep two files transactionally in sync.
+function logSize() {
+  try { return fs.statSync(PATHS.log).size; } catch { return 0; }
+}
+
 function saveIndex(index) {
   ensureDir(PATHS.dir);
-  fs.writeFileSync(PATHS.index, JSON.stringify(index, null, 2));
+  fs.writeFileSync(PATHS.index, JSON.stringify({ ...index, logSize: logSize() }, null, 2));
+}
+
+// A patch row does not change any searchable text, so the index stays valid —
+// it just needs to learn the log's new size or the staleness check would force
+// a needless rebuild on the next read.
+function restampIndex() {
+  try {
+    if (!fs.existsSync(PATHS.index)) return;
+    const idx = JSON.parse(fs.readFileSync(PATHS.index, 'utf8'));
+    idx.logSize = logSize();
+    fs.writeFileSync(PATHS.index, JSON.stringify(idx, null, 2));
+  } catch { /* a broken index is rebuilt on the next load anyway */ }
 }
 
 function indexMemory(index, memory) {
@@ -116,7 +163,7 @@ function indexMemory(index, memory) {
   const length = tokens.length;
   if (length === 0) return index;
 
-  const tokenFreq = {};
+  const tokenFreq = nullMap();
   for (const token of tokens) {
     tokenFreq[token] = (tokenFreq[token] || 0) + 1;
   }
@@ -147,7 +194,7 @@ function search(query, options = {}) {
   const b  = 0.75;
   const N  = index.docCount;
   const avgDl = index.avgDocLength || 1;
-  const scores = {};
+  const scores = nullMap();
 
   for (const token of tokens) {
     const entry = index.tokens[token];
@@ -183,27 +230,81 @@ function search(query, options = {}) {
 }
 
 // ── Memory I/O ───────────────────────────────────────────────────────────────
+// The log is append-only, so a memory may appear more than once: the original
+// row, then PATCH rows appended by forget()/resolveMemory(). Fold by id with
+// last-write-wins, then drop anything a patch marked deleted.
+//
+// This is what lets mutation be an append instead of a full-file rewrite. The
+// rewrite was the bug: fs.writeFileSync opens with 'w', truncating to zero
+// before writing, and a 6.3 MB store takes several syscalls to write back. A
+// concurrent reader was measured observing the store at 0 bytes mid-write, and
+// any reader doing its own read-modify-write would then persist that emptiness.
 function readMemories(ids = null) {
   if (!fs.existsSync(PATHS.log)) return ids ? {} : [];
   const wantSet = ids ? new Set(ids) : null;
   const lines = fs.readFileSync(PATHS.log, 'utf8').split('\n').filter(Boolean);
-  const out = ids ? {} : [];
+
+  const folded = new Map();   // preserves first-seen order
   for (const line of lines) {
-    let memory;
-    try { memory = JSON.parse(line); } catch { continue; }
-    if (memory.deleted) continue;
-    if (wantSet) {
-      if (wantSet.has(memory.id)) out[memory.id] = memory;
-    } else {
-      out.push(memory);
-    }
+    let row;
+    try { row = JSON.parse(line); } catch { continue; }
+    if (!row || !row.id) continue;
+    if (wantSet && !wantSet.has(row.id)) continue;
+    const prior = folded.get(row.id);
+    if (prior) { folded.set(row.id, { ...prior, ...row }); continue; }
+    // No prior row means this is an orphan patch — a tombstone or a {resolved}
+    // marker whose original is missing (a hand-edited or truncated log). It is
+    // not a memory: promoting it would surface a phantom with problem=undefined
+    // into recall(), the dashboard, and the injected memory block.
+    if (row.problem === undefined) continue;
+    folded.set(row.id, row);
   }
+
+  const live = [...folded.values()].filter(m => !m.deleted);
+  if (!ids) return live;
+  const out = {};
+  for (const m of live) out[m.id] = m;
   return out;
 }
 
+// Append a patch row for an existing memory. Returns false when the id is
+// unknown, so callers keep their found/not-found contract.
+function appendPatch(memoryId, patch) {
+  if (!fs.existsSync(PATHS.log)) return false;
+  const existing = readMemories([memoryId])[memoryId];
+  if (!existing) return false;
+  ensureDir(PATHS.dir);
+  fs.appendFileSync(PATHS.log, JSON.stringify({ id: memoryId, ...patch }) + '\n');
+  return true;
+}
+
+// If a previous append was interrupted (crash, SIGKILL, ENOSPC) the log ends
+// mid-row with no terminator. Appending straight onto that fragment fuses two
+// records into one unparseable line, so readMemories drops BOTH — and capture()
+// still returns an id, telling the caller a memory was saved that does not
+// exist. A leading newline quarantines the fragment on its own line, where the
+// existing JSON.parse skip handles it correctly.
+function endsWithNewline() {
+  try {
+    const { size } = fs.statSync(PATHS.log);
+    if (size === 0) return true;
+    const fd = fs.openSync(PATHS.log, 'r');
+    try {
+      const buf = Buffer.alloc(1);
+      fs.readSync(fd, buf, 0, 1, size - 1);
+      return buf[0] === 0x0a;
+    } finally {
+      fs.closeSync(fd);
+    }
+  } catch {
+    return true;   // no file yet
+  }
+}
+
 function appendMemory(memory) {
   ensureDir(PATHS.dir);
-  fs.appendFileSync(PATHS.log, JSON.stringify(memory) + '\n');
+  const prefix = endsWithNewline() ? '' : '\n';
+  fs.appendFileSync(PATHS.log, prefix + JSON.stringify(memory) + '\n');
 }
 
 // ── Public API ───────────────────────────────────────────────────────────────
@@ -225,16 +326,24 @@ function capture({
     captured_at: new Date().toISOString(),
     problem:     String(problem).slice(0, 500),
     approach:    String(approach).slice(0, 2000),
-    tags:        Array.from(new Set(tags.map(String))).slice(0, 20),
+    // The COUNT was capped but not the length of each tag, so a single
+    // multi-megabyte tag was stored whole — bloating the log, the index, and
+    // the memory block injected at session start. Bound both.
+    tags:        Array.from(new Set(tags.map(t => String(t).slice(0, 60)))).slice(0, 20),
     project:     project ? projectHash(project) : null,
     project_path: project,
     language,
-    files:       files.slice(0, 20),
-    gotchas:     gotchas.slice(0, 10),
+    files:       files.slice(0, 20).map(f => String(f).slice(0, 500)),
+    gotchas:     gotchas.slice(0, 10).map(g => String(g).slice(0, 500)),
     source,
   };
-  appendMemory(memory);
+  // Load the index BEFORE writing the row. If index.json is missing or invalid,
+  // loadIndex() falls through to rebuildIndex(), which re-reads the log — and if
+  // the new row were already there, it would be indexed once by the rebuild and
+  // again by indexMemory() below. That inflated docCount permanently and gave the
+  // doubled document exactly 2x its true BM25 score, on every cold start.
   const index = loadIndex();
+  appendMemory(memory);
   saveIndex(indexMemory(index, memory));
   return memory;
 }
@@ -262,22 +371,10 @@ function listAll() {
 }
 
 function forget(memoryId) {
-  if (!fs.existsSync(PATHS.log)) return false;
-  const lines = fs.readFileSync(PATHS.log, 'utf8').split('\n').filter(Boolean);
-  let found = false;
-  const updated = lines.map(line => {
-    try {
-      const m = JSON.parse(line);
-      if (m.id === memoryId) {
-        found = true;
-        return JSON.stringify({ ...m, deleted: true, deleted_at: new Date().toISOString() });
-      }
-      return line;
-    } catch {
-      return line;
-    }
-  });
-  fs.writeFileSync(PATHS.log, updated.join('\n') + '\n');
+  const found = appendPatch(memoryId, { deleted: true, deleted_at: new Date().toISOString() });
+  // Only rebuild when something actually changed. The old code rewrote the whole
+  // store even when the id was not found, paying the full destruction window for
+  // a no-op delete.
   if (found) rebuildIndex();
   return found;
 }
@@ -291,28 +388,10 @@ function forget(memoryId) {
 // Side-effect: also propagates to instincts.js if the instinct module exists,
 // so low-confidence instincts sourced from the same session get downgraded.
 function resolveMemory(memoryId, resolved) {
-  if (!fs.existsSync(PATHS.log)) return false;
-  const lines = fs.readFileSync(PATHS.log, 'utf8').split('\n').filter(Boolean);
-  let found = false;
-  const updated = lines.map(line => {
-    try {
-      const m = JSON.parse(line);
-      if (m.id === memoryId && !m.deleted) {
-        found = true;
-        return JSON.stringify({
-          ...m,
-          resolved,
-          resolved_at: new Date().toISOString(),
-        });
-      }
-      return line;
-    } catch {
-      return line;
-    }
-  });
-  if (found) {
-    fs.writeFileSync(PATHS.log, updated.join('\n') + '\n');
-  }
+  const found = appendPatch(memoryId, { resolved, resolved_at: new Date().toISOString() });
+  // This hook fires on every Edit/Write, so a full rebuild here would be costly
+  // on a large store. The patch adds no searchable text, so restamping is enough.
+  if (found) restampIndex();
   return found;
 }
 
@@ -323,13 +402,13 @@ function findMemoriesForFile(filePath, options = {}) {
   if (!fs.existsSync(PATHS.log)) return [];
 
   const normFile = path.normalize(filePath);
-  const lines    = fs.readFileSync(PATHS.log, 'utf8').split('\n').filter(Boolean);
+  // Go through readMemories so patch rows are folded in. Reading the log
+  // line-by-line here meant an appended `{resolved}` patch was never applied to
+  // its original row, so a resolved memory kept being returned.
   const matches  = [];
 
-  for (const line of lines) {
-    let m;
-    try { m = JSON.parse(line); } catch { continue; }
-    if (m.deleted || m.resolved !== undefined) continue; // skip already resolved
+  for (const m of readMemories()) {
+    if (m.resolved !== undefined) continue; // skip already resolved
     if (!Array.isArray(m.files) || m.files.length === 0) continue;
     if (projectRoot && m.project_path && m.project_path !== projectRoot) continue;
 
@@ -399,9 +478,9 @@ function rebuildIndex() {
 
 function stats() {
   const memories = readMemories();
-  const byProject = {};
-  const byLanguage = {};
-  const byTag = {};
+  const byProject = nullMap();
+  const byLanguage = nullMap();
+  const byTag = nullMap();
   for (const m of memories) {
     if (m.project) byProject[m.project] = (byProject[m.project] || 0) + 1;
     if (m.language) byLanguage[m.language] = (byLanguage[m.language] || 0) + 1;