opencode-self-improvement 0.0.0-stage → 0.1.0

package/src/memory.ts

@@ -0,0 +1,242 @@
+// Durable memory store engine. Reproduces the opencode-memory logfmt format
+// (MIT, github.com/shuans/opencode-memory; license retained under
+// vendor/opencode-memory/LICENSE) and extends it with the fields the write-gate
+// needs: supersession markers, snapshots and audited deletion.
+import { existsSync } from "node:fs";
+import { join, basename } from "node:path";
+import { memoryRoot, deletionsFile, snapshotDir } from "./config.ts";
+import { appendLog, ensureDir, formatLine, parseLine, readLines } from "./logfmt.ts";
+import { blockedTs } from "./blocklist.ts";
+
+export type MemoryStatus = "open" | "settled";
+
+export interface Memory {
+  ts: string;
+  type: string;
+  scope: string;
+  content: string;
+  issue?: string;
+  tags?: string[];
+  superseded_by?: string;
+  reason?: string;
+  status?: MemoryStatus;
+  file: string;
+  index: number;
+}
+
+const RESERVED = new Set([
+  "deletions.logfmt",
+  "deleted.logfmt",
+  "canonical.logfmt",
+  "blocklist.logfmt",
+]);
+
+function toMemory(fields: Record<string, unknown>, file: string, index: number): Memory | null {
+  const ts = fields.ts;
+  const type = fields.type;
+  const scope = fields.scope;
+  if (typeof ts !== "string" || typeof type !== "string") return null;
+  if (typeof scope !== "string") return null;
+  return {
+    ts,
+    type,
+    scope,
+    content: typeof fields.content === "string" ? fields.content : "",
+    issue: typeof fields.issue === "string" ? fields.issue : undefined,
+    tags: typeof fields.tags === "string" ? fields.tags.split(",") : undefined,
+    superseded_by: typeof fields.superseded_by === "string" ? fields.superseded_by : undefined,
+    reason: typeof fields.reason === "string" ? fields.reason : undefined,
+    status:
+      fields.status === "open" || fields.status === "settled" ? fields.status : undefined,
+    file,
+    index,
+  };
+}
+
+export async function memoryFiles(): Promise<string[]> {
+  const root = memoryRoot();
+  // Bun.Glob.scan throws ENOENT on a missing root; a fresh store must read as
+  // empty so the system-digest hook never throws (it runs on every request,
+  // including the compaction model call).
+  if (!existsSync(root)) return [];
+  const glob = new Bun.Glob("*.logfmt");
+  const files: string[] = [];
+  for await (const f of glob.scan(root)) {
+    if (!RESERVED.has(f)) files.push(join(root, f));
+  }
+  return files.sort();
+}
+
+export async function loadAll(): Promise<Memory[]> {
+  const blocked = await blockedTs();
+  const out: Memory[] = [];
+  for (const file of await memoryFiles()) {
+    const lines = await readLines(file);
+    lines.forEach((line, index) => {
+      const m = toMemory(parseLine(line) ?? {}, file, index);
+      if (m && !blocked.has(m.ts)) out.push(m);
+    });
+  }
+  return out;
+}
+
+export async function loadScope(scope: string): Promise<Memory[]> {
+  return (await loadAll()).filter((m) => m.scope === scope);
+}
+
+export interface AppendMemoryInput {
+  type: string;
+  scope: string;
+  content: string;
+  issue?: string;
+  tags?: string[];
+  status?: MemoryStatus;
+}
+
+// Existing ts values across all memory files, used to guarantee uniqueness of
+// newly appended entries (a fast seed->append within one millisecond would
+// otherwise collide on the ISO timestamp).
+async function existingTs(): Promise<Set<string>> {
+  const set = new Set<string>();
+  for (const file of await memoryFiles()) {
+    for (const line of await readLines(file)) {
+      const f = parseLine(line);
+      if (f && typeof f.ts === "string") set.add(f.ts);
+    }
+  }
+  return set;
+}
+
+// Append a durable memory. Only the gate's GATE-4 promotion step calls this.
+export async function appendMemory(input: AppendMemoryInput): Promise<string> {
+  const taken = await existingTs();
+  let ts = new Date().toISOString();
+  let guard = 0;
+  while (taken.has(ts) && guard++ < 1000) {
+    ts = new Date(new Date(ts).getTime() + 1).toISOString();
+  }
+  const file = join(memoryRoot(), `${ts.split("T")[0]}.logfmt`);
+  await appendLog(
+    file,
+    formatLine({
+      ts,
+      type: input.type,
+      scope: input.scope,
+      content: input.content,
+      issue: input.issue,
+      tags: input.tags?.length ? input.tags.join(",") : undefined,
+      status: input.status,
+    }),
+  );
+  return ts;
+}
+
+export interface SupersedeFields {
+  superseded_by: string;
+  reason: string;
+  symbol?: string;
+  from?: string;
+  to?: string;
+  entity?: string;
+}
+
+// Mark an existing memory as superseded IN PLACE (entry retained, recoverable).
+// Idempotent: a second call is a no-op, so no double `superseded_by`.
+export async function markSuperseded(ts: string, fields: SupersedeFields): Promise<boolean> {
+  for (const file of await memoryFiles()) {
+    const lines = await readLines(file);
+    const idx = lines.findIndex((l) => {
+      const f = parseLine(l);
+      return f && f.ts === ts;
+    });
+    if (idx === -1) continue;
+    const parsed = parseLine(lines[idx]);
+    if (parsed && typeof parsed.superseded_by === "string") return false;
+    const extra = formatLine({
+      superseded_by: fields.superseded_by,
+      reason: fields.reason,
+      symbol: fields.symbol,
+      from: fields.from,
+      to: fields.to,
+      entity: fields.entity,
+    });
+    lines[idx] = `${lines[idx]} ${extra}`;
+    await ensureDir(snapshotDir());
+    const snap = join(snapshotDir(), `${Date.now()}-${basename(file)}`);
+    await Bun.write(snap, (await Bun.file(file).text()) || "");
+    await Bun.write(file, lines.join("\n") + "\n");
+    return true;
+  }
+  return false;
+}
+
+export interface DeletedMemory {
+  memory: Memory;
+  reason: string;
+}
+
+// Snapshot-before-mutate: copy each affected file into snapshots/ so /undo can
+// restore it. Returns the snapshot paths keyed by original file.
+export async function snapshotFiles(files: string[]): Promise<string[]> {
+  await ensureDir(snapshotDir());
+  const stamp = Date.now();
+  const snaps: string[] = [];
+  for (const file of files) {
+    const f = Bun.file(file);
+    if (!(await f.exists())) continue;
+    const snap = join(snapshotDir(), `${stamp}-${basename(file)}`);
+    await Bun.write(snap, await f.text());
+    snaps.push(snap);
+  }
+  return snaps;
+}
+
+export async function logDeletion(memory: Memory, reason: string): Promise<void> {
+  await appendLog(
+    deletionsFile(),
+    formatLine({
+      ts: new Date().toISOString(),
+      action: "deleted",
+      original_ts: memory.ts,
+      type: memory.type,
+      scope: memory.scope,
+      content: memory.content,
+      reason,
+    }),
+  );
+}
+
+export interface PruneResult {
+  deleted: DeletedMemory[];
+  snapshot: string[];
+}
+
+// Remove matching memories (used by memory_forget), snapshotting first and
+// logging every deletion with its reason.
+export async function prune(scope: string, type: string, reason: string): Promise<PruneResult> {
+  const files = await memoryFiles();
+  const snapshot = await snapshotFiles(files);
+  const deleted: DeletedMemory[] = [];
+  for (const file of files) {
+    const lines = await readLines(file);
+    const kept: string[] = [];
+    let changed = false;
+    for (const line of lines) {
+      const parsed = parseLine(line);
+      const m = parsed ? toMemory(parsed, file, 0) : null;
+      if (m && m.scope === scope && m.type === type) {
+        deleted.push({ memory: m, reason });
+        await logDeletion(m, reason);
+        changed = true;
+        continue;
+      }
+      kept.push(line);
+    }
+    if (changed) await Bun.write(file, kept.join("\n") + (kept.length ? "\n" : ""));
+  }
+  return { deleted, snapshot };
+}
+
+export function isActive(m: Memory): boolean {
+  return !m.superseded_by;
+}

package/src/prune.ts

@@ -0,0 +1,23 @@
+// Principle 3: a human can prune a bad memory. Deletion is auditable (reason),
+// reversible (snapshot) and never touches a repo. The `prune` evidence line is
+// emitted before/around the mutation so S5 can assert the reason and snapshot.
+import { deletionsFile } from "./config.ts";
+import { prune, type PruneResult } from "./memory.ts";
+import { record } from "./evidence.ts";
+
+export async function pruneMemory(
+  scope: string,
+  type: string,
+  reason: string,
+): Promise<PruneResult & { audit: string }> {
+  const res = await prune(scope, type, reason);
+  const memTs = res.deleted.map((d) => d.memory.ts).join(",") || "-";
+  await record("prune", {
+    mem: memTs,
+    reason,
+    snapshot: res.snapshot[0] ?? "-",
+    audit: deletionsFile(),
+    tool: "memory_forget",
+  });
+  return { ...res, audit: deletionsFile() };
+}

package/src/resolver.ts

@@ -0,0 +1,298 @@
+// GATE-1 resolver. Extracts embedded path/symbol refs from a memory/evidence
+// candidate and re-resolves them against the live source tree (and, when
+// OPENCODE_CBM_BIN is set, the codebase-memory-mcp file graph). Deterministic:
+// regex + balanced-paren scanning, no model involvement.
+import { join } from "node:path";
+import { type Memory } from "./memory.ts";
+
+export interface CallForm {
+  symbol: string;
+  name: string;
+  ownerPath: string;
+  fileHint?: string;
+  argsText: string;
+}
+
+export interface DriftInfo {
+  symbol: string;
+  from: string;
+  to: string;
+  source: "candidate" | "indexed";
+  file?: string;
+}
+
+export interface Gate1Result {
+  refs: number;
+  resolved: string[];
+  drift: DriftInfo | null;
+  candidateStale: boolean;
+}
+
+export function balancedParen(text: string, openIdx: number): { inner: string; end: number } | null {
+  if (text[openIdx] !== "(") return null;
+  let depth = 0;
+  let quote: string | null = null;
+  for (let i = openIdx; i < text.length; i++) {
+    const c = text[i];
+    if (quote) {
+      if (c === "\\") i++;
+      else if (c === quote) quote = null;
+      continue;
+    }
+    if (c === '"' || c === "'" || c === "`") {
+      quote = c;
+      continue;
+    }
+    if (c === "(") depth++;
+    else if (c === ")") {
+      depth--;
+      if (depth === 0) return { inner: text.slice(openIdx + 1, i), end: i };
+    }
+  }
+  return null;
+}
+
+const CALL_RE = /([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+)\s*\(/g;
+
+export function extractCallForms(content: string): CallForm[] {
+  const forms: CallForm[] = [];
+  CALL_RE.lastIndex = 0;
+  let m: RegExpExecArray | null;
+  while ((m = CALL_RE.exec(content)) !== null) {
+    const symbol = m[1];
+    const openIdx = content.indexOf("(", m.index + symbol.length);
+    const paren = openIdx >= 0 ? balancedParen(content, openIdx) : null;
+    if (!paren) continue;
+    const parts = symbol.split(".");
+    const name = parts[parts.length - 1];
+    const ownerPath = parts.slice(0, -1).join(".");
+    forms.push({
+      symbol,
+      name,
+      ownerPath,
+      fileHint: parts.length >= 2 ? parts[parts.length - 2] : undefined,
+      argsText: paren.inner,
+    });
+  }
+  return forms;
+}
+
+const PATH_RE = /[A-Za-z0-9_@./-]+\.(?:ts|tsx|js|jsx|mjs|cjs)(?::\d+)?/g;
+
+export function extractPathRefs(content: string): string[] {
+  return [...content.matchAll(PATH_RE)].map((m) => m[0]);
+}
+
+function splitTopLevel(argsText: string): string[] {
+  const parts: string[] = [];
+  let depth = 0;
+  let quote: string | null = null;
+  let current = "";
+  for (let i = 0; i < argsText.length; i++) {
+    const c = argsText[i];
+    if (quote) {
+      current += c;
+      if (c === "\\") {
+        current += argsText[++i] ?? "";
+      } else if (c === quote) quote = null;
+      continue;
+    }
+    if (c === '"' || c === "'" || c === "`") {
+      quote = c;
+      current += c;
+      continue;
+    }
+    if ("([{<".includes(c)) depth++;
+    if (")]}>".includes(c)) depth--;
+    if (c === "," && depth === 0) {
+      parts.push(current);
+      current = "";
+      continue;
+    }
+    current += c;
+  }
+  if (current.trim()) parts.push(current);
+  return parts;
+}
+
+// Canonicalize an argument list to its parameter names so that type
+// annotations and defaults do not read as signature changes:
+// `config: any, opts = {}` -> `config, opts`.
+// A TS `_` prefix on an unused parameter (`_name`, `_size`) is stripped so the
+// same identifier with and without the underscore reads as the same name.
+export function normalizeArgs(argsText: string): string {
+  return splitTopLevel(argsText)
+    .map((a) => {
+      let t = a.split("=")[0] ?? a;
+      t = t.split(":")[0] ?? t;
+      t = t.trim().replace(/^_/, "").replace(/\?$/, "").trim();
+      return t;
+    })
+    .filter(Boolean)
+    .join(", ");
+}
+
+// Prefix-aware signature drift. A memory states only the args it actually
+// names; trailing live params it omits (scene?, updatable?) are not drift.
+// Drift is a positional mismatch at any stated index, or a claim that the live
+// signature dropped (candidate has more args than live).
+export function sameArgs(candidate: string, live: string): boolean {
+  const c = splitTopLevel(normalizeArgs(candidate));
+  const l = splitTopLevel(normalizeArgs(live));
+  if (c.length === 0) return true;
+  if (c.length > l.length) return false;
+  return c.every((a, i) => a === l[i]);
+}
+
+export function displayCall(name: string, argsText: string): string {
+  return `${name}(${normalizeArgs(argsText)})`;
+}
+
+// Deterministic source-tree scan used by GATE-1 live resolution. Excludes junk
+// that poisons signature comparison: node_modules (type-decl .d.ts files often
+// carry empty or spread args), build output (dist/lib), and VCS dirs. Prefers
+// files under src/ over build output, .ts/.tsx sources over compiled .js, and
+// case-insensitively ranks basenames matching the call's fileHint first so
+// `MeshBuilder.CreateBox` resolves to `meshbuilder.pure.ts` instead of a
+// different file that merely also calls CreateBox.
+const SKIP_DIRS = ["node_modules", ".git", ".hg", ".svn"];
+const SOURCE_EXTS = ["ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs"];
+
+function relRank(rel: string): number {
+  const seg = rel.split("/");
+  if (seg.some((s) => SKIP_DIRS.includes(s))) return Infinity;
+  if (rel.endsWith(".d.ts")) return Infinity;
+  let rank = 0;
+  if (seg.includes("src")) rank -= 100;
+  if (rel.endsWith(".d.ts")) rank += 1000;
+  if (seg.some((s) => ["dist", "build", "out", "lib"].includes(s))) rank += 50;
+  if (/\.(js|mjs|cjs)$/.test(rel)) rank += 10;
+  return rank;
+}
+
+async function scanSourceFiles(liveRoot: string): Promise<string[]> {
+  const out = new Set<string>();
+  for (const ext of SOURCE_EXTS) {
+    try {
+      for await (const f of new Bun.Glob(`**/*.${ext}`).scan(liveRoot)) out.add(f);
+    } catch {
+      // Non-existent root -> no candidates.
+    }
+  }
+  return [...out].filter((f) => relRank(f) < Infinity);
+}
+
+function hintBoost(rel: string, hint: string): number {
+  const base = rel.split("/").pop()!.toLowerCase();
+  if (base.startsWith(hint)) return 4;
+  if (base.includes(hint)) return 3;
+  return 0;
+}
+
+// First live definition of `name` in the source tree, falling back to the
+// first call site. A definition found anywhere beats a call site that sorts
+// earlier: `MeshBuilder.CreateBox`'s real signature lives in
+// boxBuilder.pure.ts (a facade property in meshBuilder.pure.ts re-imports it),
+// but a call site in an alphabetically-earlier file must not shadow it.
+export async function findLiveCall(
+  name: string,
+  fileHint: string | undefined,
+  liveRoot: string,
+): Promise<{ file: string; argsText: string } | null> {
+  let files = await scanSourceFiles(liveRoot);
+  const tiebreak = (a: string, b: string) => a.localeCompare(b);
+  if (fileHint) {
+    const h = fileHint.toLowerCase();
+    files.sort((a, b) => relRank(a) - relRank(b) || hintBoost(b, h) - hintBoost(a, h) || tiebreak(a, b));
+  } else {
+    files.sort((a, b) => relRank(a) - relRank(b) || tiebreak(a, b));
+  }
+
+  let fallback: { file: string; argsText: string } | null = null;
+  for (const rel of files) {
+    const file = join(liveRoot, rel);
+    let text: string;
+    try {
+      text = await Bun.file(file).text();
+    } catch {
+      continue;
+    }
+    if (!text.includes(name)) continue;
+    const defRe = new RegExp(`(?:function|const|class|var|static)\\s+${name}\\b`);
+    const defMatch = defRe.exec(text);
+    if (defMatch) {
+      const idx = text.indexOf("(", defMatch.index + defMatch[0].length);
+      const paren = idx >= 0 ? balancedParen(text, idx) : null;
+      if (paren) return { file, argsText: paren.inner };
+    }
+    if (!fallback) {
+      const callMatch = new RegExp(`\\b${name}\\s*\\(`).exec(text);
+      if (callMatch) {
+        const idx = text.indexOf("(", callMatch.index);
+        const paren = idx >= 0 ? balancedParen(text, idx) : null;
+        if (paren) fallback = { file, argsText: paren.inner };
+      }
+    }
+  }
+  return fallback;
+}
+
+// The signature a memory/evidence already embeds for `symbol` in `scope`.
+export function indexedSignature(symbol: string, scope: Memory[]): CallForm | null {
+  for (const m of scope) {
+    for (const form of extractCallForms(m.content)) {
+      if (form.symbol === symbol) return form;
+    }
+  }
+  return null;
+}
+
+export interface ResolveInput {
+  content: string;
+  scope: string;
+  liveRoot: string;
+  scopeMemories: Memory[];
+}
+
+export async function gate1(input: ResolveInput): Promise<Gate1Result> {
+  const forms = extractCallForms(input.content);
+  const pathRefs = extractPathRefs(input.content);
+  const refs = forms.length + pathRefs.length;
+  if (refs === 0) return { refs: 0, resolved: [], drift: null, candidateStale: false };
+
+  let drift: DriftInfo | null = null;
+  const resolved: string[] = [];
+
+  for (const form of forms) {
+    resolved.push(form.symbol);
+    const live = await findLiveCall(form.name, form.fileHint, input.liveRoot);
+    if (!live) continue;
+    const indexed = indexedSignature(form.symbol, input.scopeMemories);
+    const candidateDiffers = !sameArgs(form.argsText, live.argsText);
+    const indexedDiffers = indexed ? !sameArgs(indexed.argsText, live.argsText) : false;
+    if (candidateDiffers) {
+      drift = {
+        symbol: form.symbol,
+        from: displayCall(form.name, form.argsText),
+        to: displayCall(form.name, live.argsText),
+        source: "candidate",
+        file: live.file,
+      };
+    } else if (indexedDiffers && indexed) {
+      drift = {
+        symbol: form.symbol,
+        from: displayCall(form.name, indexed.argsText),
+        to: displayCall(form.name, live.argsText),
+        source: "indexed",
+        file: live.file,
+      };
+    }
+  }
+
+  return {
+    refs,
+    resolved,
+    drift,
+    candidateStale: drift?.source === "candidate",
+  };
+}

package/src/state.ts

@@ -0,0 +1,88 @@
+// Per-session in-memory state shared by the hooks and the gate pipeline.
+// Deterministic: only tool results and tool invocations mutate it, never the
+// model. Reset between scenario runs with resetState().
+
+export interface ToolResult {
+  call: string;
+  tool: string;
+  summary: string;
+  ts: string;
+  status: "FAIL" | "PASS" | null;
+  cmd?: string;
+  gateCheck?: boolean;
+}
+
+export interface FirstTap {
+  call: string;
+  turn: number;
+  ts: string;
+}
+
+export interface SessionState {
+  sid: string;
+  calls: ToolResult[];
+  // T1: last observed status per tool, so a FAIL->PASS transition is visible.
+  lastByTool: Map<string, { status: "FAIL" | "PASS"; call: string; emitted: boolean }>;
+  // T4: a pending completion-gate failure awaiting a later PASS.
+  pendingGate: {
+    call: string;
+    cmd: string;
+    chain: ToolResult[];
+  } | null;
+  // Session-convergence tracking (time-to-frontier): completed-message count,
+  // first activity time, and the first memory_recall that surfaced an open
+  // thread. Deterministic: derived only from queued messages and tool results.
+  turns: number;
+  firstActivityTs?: string;
+  firstTap?: FirstTap;
+  finalized: boolean;
+}
+
+// Newest message-bearing session seen this process. Opening a session finalizes
+// the previous one (there is no session-end hook on the host surface).
+let lastSeenSidVar: string | null = null;
+
+export function lastSeenSid(): string | null {
+  return lastSeenSidVar;
+}
+
+export function setLastSeenSid(sid: string): void {
+  lastSeenSidVar = sid;
+}
+
+const sessions = new Map<string, SessionState>();
+
+export function session(sid: string): SessionState {
+  let s = sessions.get(sid);
+  if (!s) {
+    s = { sid, calls: [], lastByTool: new Map(), pendingGate: null, turns: 0, finalized: false };
+    sessions.set(sid, s);
+  }
+  return s;
+}
+
+let current: { sid: string; call: string; tool: string } = {
+  sid: "?",
+  call: "?",
+  tool: "?",
+};
+
+export function setCurrent(input: {
+  sessionID?: unknown;
+  callID?: unknown;
+  tool?: unknown;
+}): void {
+  if (typeof input.sessionID === "string") current.sid = input.sessionID;
+  if (typeof input.callID === "string") current.call = input.callID;
+  if (typeof input.tool === "string") current.tool = input.tool;
+}
+
+export function getCurrent(): { sid: string; call: string; tool: string } {
+  return current;
+}
+
+export function resetState(): void {
+  sessions.clear();
+  current = { sid: "?", call: "?", tool: "?" };
+  lastSeenSidVar = null;
+}