kodelyth-ecc 2.25.0 → 2.29.0

package/scripts/memory/store.js

@@ -19,6 +19,8 @@ const os   = require('os');
 const path = require('path');
 const crypto = require('crypto');
 const safeFs = require('../lib/safe-fs.js');
+const pid = require('./project-id.js');
+const { redact } = require('../lib/redact.js');
 
 // Auto-migrate legacy ~/.kodelyth/ → ~/.kodelythecc/ before we touch any path.
 try { require('../migrate-legacy').main(); } catch { /* best-effort */ }
@@ -27,11 +29,14 @@ const MEMORY_DIR = process.env.KODELYTH_MEMORY_DIR
   || path.join(os.homedir(), '.kodelythecc', 'memory');
 
 const PATHS = {
-  dir:       MEMORY_DIR,
-  log:       path.join(MEMORY_DIR, 'memories.jsonl'),
-  index:     path.join(MEMORY_DIR, 'index.json'),
-  patterns:  path.join(MEMORY_DIR, 'patterns.json'),
-  projects:  path.join(MEMORY_DIR, 'projects'),
+  dir:        MEMORY_DIR,
+  log:        path.join(MEMORY_DIR, 'memories.jsonl'),
+  index:      path.join(MEMORY_DIR, 'index.json'),
+  patterns:   path.join(MEMORY_DIR, 'patterns.json'),
+  projects:   path.join(MEMORY_DIR, 'projects'),
+  // Opt-in semantic layer only. Written when embeddings are enabled; absent and
+  // untouched on a default (BM25-only) install.
+  embeddings: path.join(MEMORY_DIR, 'embeddings.jsonl'),
 };
 
 // ── Stop words (English + common code noise) ─────────────────────────────────
@@ -208,6 +213,28 @@ function indexMemory(index, memory) {
  */
 const RELATIVE_SCORE_FLOOR = 0.25;
 
+// Rank fusion. BM25 decides RELEVANCE (and the floor below gates it); this only
+// reorders the survivors so that, among similarly-relevant memories, a recent and
+// proven one outranks an old or failed one. Pure function of one memory — the
+// BM25 `score`, the `captured_at` timestamp, and the `resolved` outcome — so it
+// is unit-tested directly.
+function fuseScore(m) {
+  const base = m.score || 0;
+  // Recency: up to +25% for a just-captured memory, decaying over ~90 days.
+  let recency = 1;
+  const t = Date.parse(m.captured_at || '');
+  if (!Number.isNaN(t)) {
+    const ageDays = Math.max(0, (Date.now() - t) / 86400000);
+    recency = 1 + 0.25 * Math.exp(-ageDays / 90);
+  }
+  // Outcome: an approach confirmed to work (resolved:true) is worth more; one
+  // that didn't stick (resolved:false) is worth less. Unknown stays neutral.
+  let outcome = 1;
+  if (m.resolved === true) outcome = 1.3;
+  else if (m.resolved === false) outcome = 0.6;
+  return base * recency * outcome;
+}
+
 function search(query, options = {}) {
   // minScore stays honoured when passed, so existing callers keep their exact
   // behaviour; only the default changes from an absolute floor to a relative one.
@@ -253,9 +280,15 @@ function search(query, options = {}) {
     .filter(Boolean);
 
   if (projectFilter) {
-    results = results.filter(m => m.project === projectFilter);
+    // projectFilter may be a single id or a list of ids (read-both: the current
+    // git-remote id plus the legacy path id). Match on membership either way.
+    const allowed = Array.isArray(projectFilter) ? new Set(projectFilter) : new Set([projectFilter]);
+    results = results.filter(m => allowed.has(m.project));
   }
 
+  // Re-rank the relevant survivors by the fused score (relevance × recency ×
+  // outcome), then trim. `score` (raw BM25) stays on each result for callers.
+  results.sort((a, b) => fuseScore(b) - fuseScore(a));
   return results.slice(0, limit);
 }
 
@@ -354,17 +387,23 @@ function capture({
   const memory = {
     id:          newMemoryId(),
     captured_at: new Date().toISOString(),
-    problem:     String(problem).slice(0, 500),
-    approach:    String(approach).slice(0, 2000),
+    // Redact credentials BEFORE storing. A memory is written to memories.jsonl,
+    // surfaced by recall, and injected into future sessions — so a leaked key in
+    // a captured problem/approach would otherwise persist and re-surface.
+    problem:     redact(String(problem)).slice(0, 500),
+    approach:    redact(String(approach)).slice(0, 2000),
     // The COUNT was capped but not the length of each tag, so a single
     // multi-megabyte tag was stored whole — bloating the log, the index, and
     // the memory block injected at session start. Bound both.
-    tags:        Array.from(new Set(tags.map(t => String(t).slice(0, 60)))).slice(0, 20),
-    project:     project ? projectHash(project) : null,
+    tags:        Array.from(new Set(tags.map(t => redact(String(t)).slice(0, 60)))).slice(0, 20),
+    // Git-remote-first id: the same repo keeps its memories across clone / move /
+    // rename, where the old sha256(path) scheme orphaned them. Legacy memories
+    // (path id) are still found via read-both in recallForProject().
+    project:     project ? pid.resolveProjectId(project) : null,
     project_path: project,
     language,
-    files:       files.slice(0, 20).map(f => String(f).slice(0, 500)),
-    gotchas:     gotchas.slice(0, 10).map(g => String(g).slice(0, 500)),
+    files:       files.slice(0, 20).map(f => redact(String(f)).slice(0, 500)),
+    gotchas:     gotchas.slice(0, 10).map(g => redact(String(g)).slice(0, 500)),
     source,
   };
   // Load the index BEFORE writing the row. If index.json is missing or invalid,
@@ -383,7 +422,9 @@ function recall(query, options = {}) {
 }
 
 function recallForProject(projectRoot, query, options = {}) {
-  const opts = { ...options, projectFilter: projectHash(projectRoot) };
+  // Read-both: match the current git-remote id AND the legacy path id, so
+  // memories captured under either scheme are found.
+  const opts = { ...options, projectFilter: pid.matchingProjectIds(projectRoot) };
   const results = search(query, opts);
   if (results.length >= (options.limit || 5)) return results;
   // Fall back to global memories if project-specific are sparse
@@ -525,6 +566,69 @@ function stats() {
   };
 }
 
+// ── Optional semantic layer (opt-in; a default install never touches this) ───
+// Embeddings live in a sidecar keyed by memory id, so memories.jsonl and its
+// BM25 index are byte-for-byte identical whether or not embeddings are enabled.
+
+function saveEmbedding(id, vec) {
+  if (!id || !Array.isArray(vec) || vec.length === 0) return false;
+  ensureDir(PATHS.dir);
+  fs.appendFileSync(PATHS.embeddings, JSON.stringify({ id, vec }) + '\n');
+  return true;
+}
+
+function loadEmbeddings(ids = null) {
+  if (!fs.existsSync(PATHS.embeddings)) return {};
+  const want = ids ? new Set(ids) : null;
+  const out = {};
+  for (const line of fs.readFileSync(PATHS.embeddings, 'utf8').split('\n')) {
+    if (!line) continue;
+    let row; try { row = JSON.parse(line); } catch { continue; }
+    if (!row || !row.id || !Array.isArray(row.vec)) continue;
+    if (want && !want.has(row.id)) continue;
+    out[row.id] = row.vec; // append-only; last write wins
+  }
+  return out;
+}
+
+// Capture a memory and, when embeddings are enabled AND a provider is available,
+// also store its embedding. Otherwise behaves exactly like the sync capture().
+async function captureWithEmbedding(args) {
+  const memory = capture(args);
+  try {
+    const provider = await require('./embeddings').getProvider();
+    if (provider) {
+      const vec = await provider.embed(`${memory.problem} ${memory.approach}`);
+      if (vec) saveEmbedding(memory.id, vec);
+    }
+  } catch { /* embeddings are best-effort — never fail a capture over them */ }
+  return memory;
+}
+
+// Hybrid recall: BM25 selects the relevant set (sync, as always), then — only
+// when embeddings are enabled and available — re-ranks that set by blending in
+// semantic similarity. With embeddings off/unavailable this returns exactly what
+// recall() returns.
+async function recallHybrid(query, options = {}) {
+  const limit = options.limit || 5;
+  // Pull a wider candidate pool so semantic similarity can promote a strong
+  // match that BM25 alone ranked just below the cut.
+  const base = search(query, { ...options, limit: limit * 4 });
+  if (base.length === 0) return base;
+  try {
+    const emb = require('./embeddings');
+    const provider = await emb.getProvider();
+    if (!provider) return base.slice(0, limit);
+    const queryVec = await provider.embed(query);
+    if (!queryVec) return base.slice(0, limit);
+    const vecById = loadEmbeddings(base.map(m => m.id));
+    const fused = emb.fuseHybrid(base, { queryVec, vecById, weight: options.semanticWeight });
+    return fused.map(({ _bm25norm, _semantic, _hybrid, ...m }) => m).slice(0, limit);
+  } catch {
+    return base.slice(0, limit); // any failure in the opt-in path degrades to BM25
+  }
+}
+
 module.exports = {
   PATHS,
   capture,
@@ -536,6 +640,12 @@ module.exports = {
   stats,
   tokenise,
   projectHash,
+  fuseScore,
+  // Opt-in semantic layer
+  saveEmbedding,
+  loadEmbeddings,
+  captureWithEmbedding,
+  recallHybrid,
   // Improvement C
   resolveMemory,
   findMemoriesForFile,

package/scripts/super-agent/cli.js

@@ -0,0 +1,135 @@
+// scripts/super-agent/cli.js
+//
+// The driver surface for the super-agent orchestrator. Like the Arena CLI, this
+// does NOT run agents itself — it advances a tokenless state machine and tells
+// the driving LLM what to do next. The loop is:
+//
+//   super-agent start --goal "..."        → create a run, print the next action
+//   super-agent next <id>                 → print the next action (JSON)
+//   super-agent plan <id> --file p.json   → install the decomposed step DAG
+//   super-agent step <id> <step> --status done|failed [--branch b] [--tokens n]
+//   super-agent integrate <id> [--conflicts a,b] [--tokens n]
+//   super-agent verify <id> --passed|--failed [--detail "..."]
+//   super-agent status <id> | list
+//
+// The LLM performs each action with the existing primitives (blueprint to
+// decompose, the swarm to dispatch, the GAN loop to evaluate, git to integrate)
+// and reports back through the submit commands.
+'use strict';
+
+const fs = require('fs');
+const state = require('./state.js');
+const orch = require('./orchestrator.js');
+
+function parse(argv) {
+  const positional = [];
+  const flags = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith('--')) {
+      const key = a.slice(2);
+      const next = argv[i + 1];
+      if (next && !next.startsWith('--')) { flags[key] = next; i++; }
+      else flags[key] = true;
+    } else positional.push(a);
+  }
+  return { positional, flags };
+}
+
+const out = (o) => process.stdout.write(JSON.stringify(o, null, 2) + '\n');
+const err = (m) => { process.stderr.write(`super-agent: ${m}\n`); process.exit(2); };
+
+function loadOr(id) {
+  if (!id) err('a run id is required');
+  if (!state.exists(id)) err(`no run ${id}`);
+  return state.load(id);
+}
+
+function main(argv) {
+  const { positional, flags } = parse(argv);
+  const cmd = positional[0];
+
+  switch (cmd) {
+    case 'start': {
+      const goal = flags.goal || positional.slice(1).join(' ');
+      if (!goal) err('usage: super-agent start --goal "<objective>" [--token-cap N] [--step-cap N] [--scope .]');
+      const run = state.createRun({
+        goal,
+        scope: flags.scope || '.',
+        limits: { tokenCap: Number(flags['token-cap']) || 0, stepCap: Number(flags['step-cap']) || 0 },
+      });
+      state.save(run);
+      out({ runId: run.id, next: orch.nextAction(run) });
+      return;
+    }
+    case 'next': {
+      out(orch.nextAction(loadOr(positional[1])));
+      return;
+    }
+    case 'plan': {
+      const run = loadOr(positional[1]);
+      if (!flags.file) err('usage: super-agent plan <id> --file <plan.json>  (JSON array of { id, description, deps, exitCriteria })');
+      let steps;
+      try { steps = JSON.parse(fs.readFileSync(flags.file, 'utf8')); }
+      catch (e) { err(`cannot read plan: ${e.message}`); }
+      orch.submitPlan(run, steps);
+      out({ ok: true, next: orch.nextAction(run) });
+      return;
+    }
+    case 'dispatch': {
+      // Mark the given steps in-progress (optional bookkeeping before running them).
+      const run = loadOr(positional[1]);
+      const ids = (flags.steps ? String(flags.steps).split(',') : positional.slice(2)).filter(Boolean);
+      orch.markDispatched(run, ids);
+      out({ ok: true, inProgress: ids });
+      return;
+    }
+    case 'step': {
+      const run = loadOr(positional[1]);
+      const stepId = positional[2];
+      if (!stepId) err('usage: super-agent step <id> <stepId> --status done|failed [--branch b] [--tokens n]');
+      orch.submitStepResult(run, stepId, {
+        status: flags.status === 'failed' ? 'failed' : 'done',
+        branch: flags.branch || null,
+        result: flags.result || null,
+        tokensSpent: Number(flags.tokens) || 0,
+      });
+      out({ ok: true, next: orch.nextAction(run) });
+      return;
+    }
+    case 'integrate': {
+      const run = loadOr(positional[1]);
+      const conflicts = flags.conflicts ? String(flags.conflicts).split(',').filter(Boolean) : [];
+      orch.submitIntegration(run, {
+        merged: conflicts.length === 0 && flags.merged !== false,
+        conflicts,
+        tokensSpent: Number(flags.tokens) || 0,
+      });
+      out({ ok: true, next: orch.nextAction(run) });
+      return;
+    }
+    case 'verify': {
+      const run = loadOr(positional[1]);
+      if (!flags.passed && !flags.failed) err('usage: super-agent verify <id> --passed | --failed [--detail "..."]');
+      orch.submitVerification(run, {
+        passed: !!flags.passed && !flags.failed,
+        detail: typeof flags.detail === 'string' ? flags.detail : '',
+        tokensSpent: Number(flags.tokens) || 0,
+      });
+      out({ ok: true, next: orch.nextAction(run) });
+      return;
+    }
+    case 'status': {
+      out(state.summarize(loadOr(positional[1])));
+      return;
+    }
+    case 'list': {
+      out(state.list().map(state.summarize));
+      return;
+    }
+    default:
+      err('commands: start | next | plan | dispatch | step | integrate | verify | status | list');
+  }
+}
+
+module.exports = { main, parse };

package/scripts/super-agent/dag.js

@@ -0,0 +1,123 @@
+// scripts/super-agent/dag.js
+//
+// Pure dependency-graph model for a super-agent run's plan. No I/O. A step is
+//   { id, description, deps: [ids], exitCriteria?, status, ... }
+// and this module answers the only questions the orchestrator needs: which steps
+// are runnable right now, is the plan finished, and is it stuck. Keeping this
+// pure is what makes the whole control plane testable without spawning an agent.
+'use strict';
+
+const STATUS = Object.freeze({
+  PENDING: 'pending',
+  IN_PROGRESS: 'in_progress',
+  DONE: 'done',
+  FAILED: 'failed',
+  SKIPPED: 'skipped',
+});
+
+// Normalize a raw plan into well-formed steps (stable string ids, deps array,
+// a starting status). Ids are coerced to strings so numeric ids from a blueprint
+// and string deps compare correctly.
+function normalizeSteps(steps) {
+  return (Array.isArray(steps) ? steps : []).map((s) => ({
+    status: STATUS.PENDING,
+    ...s,
+    id: String(s && s.id != null ? s.id : ''),
+    deps: Array.isArray(s && s.deps) ? s.deps.map(String) : [],
+  }));
+}
+
+// Validate the plan is a sane DAG: unique non-empty ids, deps that exist, no
+// self-dependency, and no cycles. Returns { ok, errors: [] }.
+function validate(steps) {
+  const errors = [];
+  const ids = new Set();
+  for (const s of steps) {
+    if (!s.id) errors.push('a step has no id');
+    else if (ids.has(s.id)) errors.push(`duplicate step id: ${s.id}`);
+    else ids.add(s.id);
+  }
+  for (const s of steps) {
+    for (const d of s.deps) {
+      if (d === s.id) errors.push(`step ${s.id} depends on itself`);
+      else if (!ids.has(d)) errors.push(`step ${s.id} depends on unknown step ${d}`);
+    }
+  }
+  if (errors.length === 0 && hasCycle(steps)) errors.push('the plan has a dependency cycle');
+  return { ok: errors.length === 0, errors };
+}
+
+// Kahn's algorithm: if we cannot topologically order every step, there is a cycle.
+function hasCycle(steps) {
+  const indeg = new Map(steps.map((s) => [s.id, 0]));
+  const byId = new Map(steps.map((s) => [s.id, s]));
+  for (const s of steps) {
+    for (const d of s.deps) {
+      if (byId.has(d)) indeg.set(s.id, indeg.get(s.id) + 1);
+    }
+  }
+  const queue = [...indeg.entries()].filter(([, n]) => n === 0).map(([id]) => id);
+  const dependents = new Map(steps.map((s) => [s.id, []]));
+  for (const s of steps) {
+    for (const d of s.deps) {
+      if (dependents.has(d)) dependents.get(d).push(s.id);
+    }
+  }
+  let visited = 0;
+  while (queue.length) {
+    const id = queue.shift();
+    visited++;
+    for (const dep of dependents.get(id) || []) {
+      indeg.set(dep, indeg.get(dep) - 1);
+      if (indeg.get(dep) === 0) queue.push(dep);
+    }
+  }
+  return visited !== steps.length;
+}
+
+// Steps that can run now: still pending, with every dependency DONE (or SKIPPED,
+// which counts as satisfied — a skipped step did not fail).
+function readySteps(steps) {
+  const satisfied = new Set(
+    steps.filter((s) => s.status === STATUS.DONE || s.status === STATUS.SKIPPED).map((s) => s.id),
+  );
+  return steps.filter(
+    (s) => s.status === STATUS.PENDING && s.deps.every((d) => satisfied.has(d)),
+  );
+}
+
+function isComplete(steps) {
+  return steps.length > 0 && steps.every((s) => s.status === STATUS.DONE || s.status === STATUS.SKIPPED);
+}
+
+// Stuck: there is still work pending, but nothing is runnable and nothing is in
+// progress — so a dependency failed (or, if validate() passed, it cannot be a
+// cycle). The orchestrator turns this into a halt rather than spinning forever.
+function isStalled(steps) {
+  const pending = steps.filter((s) => s.status === STATUS.PENDING);
+  const running = steps.some((s) => s.status === STATUS.IN_PROGRESS);
+  return pending.length > 0 && !running && readySteps(steps).length === 0;
+}
+
+function progress(steps) {
+  const count = (st) => steps.filter((s) => s.status === st).length;
+  return {
+    total: steps.length,
+    done: count(STATUS.DONE),
+    failed: count(STATUS.FAILED),
+    inProgress: count(STATUS.IN_PROGRESS),
+    pending: count(STATUS.PENDING),
+    skipped: count(STATUS.SKIPPED),
+  };
+}
+
+module.exports = {
+  STATUS,
+  normalizeSteps,
+  validate,
+  hasCycle,
+  readySteps,
+  isComplete,
+  isStalled,
+  progress,
+};

package/scripts/super-agent/integrate.js

@@ -0,0 +1,91 @@
+// scripts/super-agent/integrate.js
+//
+// Branch integration for the super-agent — the piece Agent B flagged as missing:
+// "the swarm fans out but never merges worker branches." Two parts:
+//
+//   mergeOrder(steps)  — PURE. The order to merge worker branches in, respecting
+//                        step dependencies (a dependency's branch merges first).
+//   detectConflicts()  — uses `git merge-tree --write-tree` to find, WITHOUT
+//                        touching the working tree, which branches would conflict
+//                        if merged together.
+//
+// The orchestrator's integrate phase uses this to decide whether to proceed
+// (clean) or stop for a human (conflicts) — see submitIntegration.
+'use strict';
+
+const { spawnSync } = require('child_process');
+
+// Topologically order the done-with-branch steps so a dependency is merged
+// before anything built on top of it. Pure; only reorders what is given.
+function mergeOrder(steps) {
+  const withBranch = (steps || []).filter((s) => s.status === 'done' && s.branch);
+  const ids = new Set(withBranch.map((s) => s.id));
+  const byId = new Map(withBranch.map((s) => [s.id, s]));
+  const indeg = new Map(withBranch.map((s) => [s.id, 0]));
+  for (const s of withBranch) {
+    for (const d of s.deps || []) if (ids.has(String(d))) indeg.set(s.id, indeg.get(s.id) + 1);
+  }
+  // Stable: break ties by original order.
+  const order = [];
+  const queue = withBranch.filter((s) => indeg.get(s.id) === 0).map((s) => s.id);
+  while (queue.length) {
+    const id = queue.shift();
+    order.push(id);
+    for (const s of withBranch) {
+      if ((s.deps || []).map(String).includes(id) && indeg.has(s.id)) {
+        indeg.set(s.id, indeg.get(s.id) - 1);
+        if (indeg.get(s.id) === 0) queue.push(s.id);
+      }
+    }
+  }
+  // Any leftover (shouldn't happen for a validated plan) appended in input order.
+  for (const s of withBranch) if (!order.includes(s.id)) order.push(s.id);
+  return order.map((id) => ({ step: id, branch: byId.get(id).branch }));
+}
+
+function branchNames(branches) {
+  return (branches || [])
+    .map((b) => (typeof b === 'string' ? b : b && b.branch))
+    .filter(Boolean);
+}
+
+// Does merging branchB into branchA conflict? No working-tree side effects.
+function pairConflicts(repoRoot, branchA, branchB) {
+  const r = spawnSync('git', ['merge-tree', '--write-tree', branchA, branchB], {
+    cwd: repoRoot, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024,
+  });
+  if (r.status === 0) return { conflict: false };
+  if (r.status === 1) return { conflict: true, files: parseConflictFiles(r.stdout) };
+  return { conflict: false, error: (r.stderr || '').trim() || `git merge-tree exited ${r.status}` };
+}
+
+// Best-effort: pull file paths out of merge-tree's conflict section. The first
+// line is the tree oid; conflict detail follows. Version-tolerant — we just keep
+// anything that looks like a path, deduped.
+function parseConflictFiles(stdout) {
+  const files = new Set();
+  for (const line of String(stdout || '').split('\n').slice(1)) {
+    const m = line.match(/(?:^|\s)([\w.\-/]+\.[\w]+)(?:\s|$)/);
+    if (m) files.add(m[1]);
+  }
+  return [...files];
+}
+
+// Pairwise conflict scan across all worker branches. Catches the common case —
+// two workers editing the same file — before any real merge happens. Returns
+// { clean, conflicts: [{ branches:[a,b], files:[] }], errors: [] }.
+function detectConflicts(repoRoot, branches) {
+  const names = branchNames(branches);
+  const conflicts = [];
+  const errors = [];
+  for (let i = 0; i < names.length; i++) {
+    for (let j = i + 1; j < names.length; j++) {
+      const c = pairConflicts(repoRoot, names[i], names[j]);
+      if (c.error) errors.push({ branches: [names[i], names[j]], error: c.error });
+      else if (c.conflict) conflicts.push({ branches: [names[i], names[j]], files: c.files || [] });
+    }
+  }
+  return { clean: conflicts.length === 0, conflicts, errors };
+}
+
+module.exports = { mergeOrder, detectConflicts, pairConflicts, parseConflictFiles, branchNames };