opencode-self-improvement 0.0.0-stage → 0.1.0

package/src/init.ts

@@ -0,0 +1,300 @@
+// Repository init: ensure the codebase-memory graph exists for this repo and
+// re-derive build/test command memories through the write gate. Deterministic
+// (no model calls): the graph binary is spawned one-shot via `cli`, manifests
+// are parsed locally, and every write goes through the 4-gate pipeline with the
+// `trusted` flag (plugin-authored bookkeeping, not model claims). Degrades
+// gracefully when the binary or git is absent, so install-from-scratch still
+// writes script memories.
+import { basename, join } from "node:path";
+import { existsSync } from "node:fs";
+import { cbmBin } from "./config.ts";
+import { loadAll, markSuperseded } from "./memory.ts";
+import { runWriteGate, type GateContext } from "./gates.ts";
+import { record } from "./evidence.ts";
+import { evidenceCalls, hasGateCheck } from "./triggers.ts";
+import { getCurrent } from "./state.ts";
+
+export interface ScriptRecord {
+  key: string;
+  cmd: string;
+  source: string;
+}
+
+export interface GraphStatus {
+  action: "indexed" | "unavailable" | "index-failed";
+  name?: string;
+  mode: string;
+}
+
+export interface InitArgs {
+  liveRoot: string;
+  force?: boolean;
+  mode?: string;
+}
+
+export interface InitSummary {
+  scope: string;
+  root: string;
+  head: string;
+  graph: GraphStatus;
+  scripts: ScriptRecord[];
+  stampTs?: string;
+  skipped: boolean;
+  written: number;
+  collapsed: number;
+  blocked: number;
+}
+
+const NOTABLE_KEYS = ["test", "e2e", "lint", "typecheck", "check", "build", "format"];
+
+function isNotable(key: string): boolean {
+  return NOTABLE_KEYS.includes(key) || NOTABLE_KEYS.some((k) => key.startsWith(k));
+}
+
+async function run(cmd: string[], cwd: string): Promise<{ ok: boolean; out: string }> {
+  try {
+    const p = Bun.spawn(cmd, { cwd, stdout: "pipe", stderr: "pipe" });
+    const [so, se] = await Promise.all([
+      new Response(p.stdout).text(),
+      new Response(p.stderr).text(),
+    ]);
+    const code = await p.exited;
+    return { ok: code === 0, out: (so || se || "").trim() };
+  } catch {
+    return { ok: false, out: "" };
+  }
+}
+
+function runSync(cmd: string[], cwd: string): { ok: boolean; out: string } {
+  try {
+    const p = Bun.spawnSync(cmd, { cwd, stdout: "pipe", stderr: "pipe" });
+    const out = p.stdout?.toString() ?? "";
+    const err = p.stderr?.toString() ?? "";
+    return { ok: p.exitCode === 0, out: (out || err).trim() };
+  } catch {
+    return { ok: false, out: "" };
+  }
+}
+
+export async function gitRootOf(dir: string): Promise<string> {
+  const r = runSync(["git", "rev-parse", "--show-toplevel"], dir);
+  return r.ok && r.out ? r.out : dir;
+}
+
+export async function gitHead(root: string): Promise<string> {
+  const r = runSync(["git", "rev-parse", "--short", "HEAD"], root);
+  return r.ok && r.out ? r.out : "-";
+}
+
+async function readText(root: string, file: string): Promise<string | null> {
+  const p = join(root, file);
+  if (!existsSync(p)) return null;
+  try {
+    return await Bun.file(p).text();
+  } catch {
+    return null;
+  }
+}
+
+async function detectPackageScripts(root: string, records: ScriptRecord[]): Promise<void> {
+  const text = await readText(root, "package.json");
+  if (!text) return;
+  let pkg: Record<string, unknown>;
+  try {
+    pkg = JSON.parse(text);
+  } catch {
+    return;
+  }
+  if (!pkg.scripts || typeof pkg.scripts !== "object") return;
+  const seen = new Set<string>();
+  for (const name of Object.keys(pkg.scripts as Record<string, unknown>)) {
+    if (seen.has(name) || !isNotable(name.toLowerCase())) continue;
+    seen.add(name);
+    records.push({ key: name, cmd: `npm run ${name}`, source: "package.json" });
+  }
+}
+
+async function detectMakeTargets(root: string, records: ScriptRecord[]): Promise<void> {
+  const targetRe = /^([A-Za-z0-9_-]+)\s*:/;
+  const seen = new Set<string>();
+  for (const file of ["Makefile", "Justfile"]) {
+    const text = await readText(root, file);
+    if (!text) continue;
+    for (const line of text.split("\n")) {
+      if (/^[\t ]|^\s*#|^\./.test(line)) continue;
+      const m = targetRe.exec(line);
+      if (!m) continue;
+      const name = m[1].toLowerCase();
+      if (!isNotable(name) || seen.has(name)) continue;
+      seen.add(name);
+      const runner = file === "Justfile" ? "just" : "make";
+      records.push({ key: name, cmd: `${runner} ${m[1]}`, source: file });
+    }
+  }
+}
+
+async function detectLangManifests(root: string, records: ScriptRecord[]): Promise<void> {
+  if (existsSync(join(root, "Cargo.toml"))) {
+    records.push({ key: "test", cmd: "cargo test", source: "Cargo.toml" });
+  }
+  const py = await readText(root, "pyproject.toml");
+  if (py && /pytest|tool\.pytest|\[tool\]\.poetry/.test(py)) {
+    records.push({ key: "test", cmd: "pytest", source: "pyproject.toml" });
+  }
+}
+
+export async function detectScripts(root: string): Promise<ScriptRecord[]> {
+  const records: ScriptRecord[] = [];
+  await detectPackageScripts(root, records);
+  await detectMakeTargets(root, records);
+  await detectLangManifests(root, records);
+  return records.slice(0, 8);
+}
+
+async function cbmProject(bin: string, root: string): Promise<{ name: string } | null> {
+  const r = await run(
+    [bin, "cli", "--quiet", "list_projects", "--format", "json", "--metadata-only"],
+    root,
+  );
+  if (!r.ok || !r.out) return null;
+  try {
+    const env = JSON.parse(r.out) as { projects?: { name: string; root_path: string }[] };
+    const hit = env.projects?.find((p) => p.root_path === root);
+    return hit ? { name: hit.name } : null;
+  } catch {
+    return null;
+  }
+}
+
+async function ensureGraph(root: string, mode: string, force: boolean): Promise<GraphStatus> {
+  const bin = cbmBin() ?? "codebase-memory-mcp";
+  const existing = await cbmProject(bin, root);
+  if (existing && !force) return { action: "indexed", name: existing.name, mode };
+  const r = await run(
+    [bin, "cli", "--quiet", "index_repository", "--repo-path", root, "--mode", mode],
+    root,
+  );
+  if (!r.ok) return { action: "unavailable", mode };
+  const after = await cbmProject(bin, root);
+  return { action: after ? "indexed" : "index-failed", name: after?.name, mode };
+}
+
+export interface InitStamp {
+  ts: string;
+  head: string;
+}
+
+export async function initStamp(scope: string): Promise<InitStamp | null> {
+  const mems = (await loadAll()).filter((m) => !m.superseded_by);
+  for (const m of mems) {
+    if (m.type !== "context" || m.scope !== scope) continue;
+    if (!m.tags?.some((t) => t === "init-stamp")) continue;
+    const head = /head=([^ ,]+)/.exec(m.content)?.[1] ?? "-";
+    return { ts: m.ts, head };
+  }
+  return null;
+}
+
+export async function runInit(args: InitArgs): Promise<InitSummary> {
+  const root = await gitRootOf(args.liveRoot);
+  const scope = basename(root) || basename(args.liveRoot);
+  const head = await gitHead(root);
+  const force = Boolean(args.force);
+  const mode = args.mode ?? "moderate";
+
+  const gateCtx: GateContext = {
+    liveRoot: root,
+    evidenceCalls: evidenceCalls(getCurrent().sid),
+    hasGateCheck: hasGateCheck(getCurrent().sid),
+    trusted: true,
+  };
+
+  const prior = await initStamp(scope);
+  if (prior && !force && prior.head === head) {
+    await record("init_skip", { scope, head, since: prior.ts }, false);
+    return {
+      scope,
+      root,
+      head,
+      graph: { action: "indexed", mode },
+      scripts: [],
+      stampTs: prior.ts,
+      skipped: true,
+      written: 0,
+      collapsed: 0,
+      blocked: 0,
+    };
+  }
+
+  await record("init_start", { scope, head, force, mode }, false);
+
+  const graph = await ensureGraph(root, mode, force);
+  const scripts = await detectScripts(root);
+
+  let written = 0;
+  let collapsed = 0;
+  let blocked = 0;
+  for (const s of scripts) {
+    const content = `${scope} ${s.key} command: run "${s.cmd}" from the repo root (${s.source}).`;
+    const outcome = await runWriteGate(
+      { type: "pattern", scope, content, tags: ["init", s.source] },
+      gateCtx,
+    );
+    if (outcome.action === "promoted") written++;
+    else if (outcome.action === "collapsed") collapsed++;
+    else blocked++;
+  }
+
+  const stampContent = `Init stamp: repo=${scope}, head=${head}, graph=${graph.action}, mode=${graph.mode}, scripts=${scripts.length}`;
+  const stampOutcome = await runWriteGate(
+    { type: "context", scope, content: stampContent, tags: ["init-stamp"] },
+    gateCtx,
+  );
+  const stampTs = stampOutcome.ts ?? prior?.ts;
+  if (stampOutcome.action === "promoted" && prior && stampOutcome.ts) {
+    await markSuperseded(prior.ts, { superseded_by: stampOutcome.ts, reason: "init" });
+  }
+
+  await record(
+    "init_complete",
+    {
+      scope,
+      head,
+      force,
+      mode,
+      graph: graph.action,
+      scripts: scripts.length,
+      written,
+      collapsed,
+      blocked,
+      stamp: stampTs ?? "-",
+    },
+    false,
+  );
+
+  return {
+    scope,
+    root,
+    head,
+    graph,
+    scripts,
+    stampTs,
+    skipped: false,
+    written,
+    collapsed,
+    blocked,
+  };
+}
+
+export function formatInit(s: InitSummary): string {
+  if (s.skipped) {
+    return `Already initialized for ${s.scope} at head ${s.head} (stamp ${s.stampTs}). Nothing to do.`;
+  }
+  const scr = s.scripts.map((r) => `- ${r.source}: "${r.cmd}"`).join("\n");
+  return [
+    `Initialized ${s.scope} (root ${s.root}, head ${s.head}).`,
+    `Graph: ${s.graph.action}${s.graph.name ? ` (${s.graph.name})` : ""} [mode=${s.graph.mode}]`,
+    scr ? `Scripts (${s.scripts.length}):\n${scr}` : "Scripts: none detected",
+    `Memories: written=${s.written}, collapsed=${s.collapsed}, blocked=${s.blocked}; stamp=${s.stampTs ?? "-"}`,
+  ].join("\n");
+}

package/src/inject.ts

@@ -0,0 +1,150 @@
+// Context surfaces. The system digest is TINY (≤2 lines): durable knowledge is
+// pulled on demand, not pushed wholesale. Deterministic memory-first injection
+// rides chat.message (synthetic parts), so relevant knowledge is in the model's
+// context before it chooses any tool. Contextual pull rides grep/glob/read
+// results. Evidence is never injected (invariant I3). Open threads (status=
+// "open") get a single capped frontier block in the digest so planning sessions
+// start at the previous session's frontier instead of re-deriving it.
+import { isActive, loadAll, type Memory } from "./memory.ts";
+import { record } from "./evidence.ts";
+import { dateStr } from "./config.ts";
+import { bracketList } from "./logfmt.ts";
+import { recentConvergenceSummary } from "./triggers.ts";
+
+export interface TransformOutput {
+  system: string[];
+}
+
+export interface PullInput {
+  tool?: unknown;
+  args?: unknown;
+}
+
+export interface PullOutput {
+  output?: unknown;
+}
+
+function dateOf(ts: string): string {
+  return ts.split("T")[0];
+}
+
+export function formatMemory(m: Memory): string {
+  const tag = m.status === "open" ? " [open]" : "";
+  return `- [${dateOf(m.ts)}]${tag} ${m.type}/${m.scope}: ${m.content}`;
+}
+
+function argText(args: unknown): string {
+  if (!args || typeof args !== "object") return "";
+  const a = args as Record<string, unknown>;
+  for (const k of ["pattern", "query", "path", "filePath", "command"]) {
+    const v = a[k];
+    if (typeof v === "string" && v) return v;
+  }
+  return "";
+}
+
+function words(text: string): string[] {
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9_]+/)
+    .filter((t) => t.length > 2);
+}
+
+function scoreMemories(term: string, mems: Memory[], limit: number): Memory[] {
+  const query = new Set(words(term));
+  if (!query.size) return [];
+  return mems
+    .map((m) => {
+      const hay = new Set(words(`${m.scope} ${m.type} ${m.content}`));
+      let score = 0;
+      for (const w of query) if (hay.has(w)) score++;
+      return { m, score };
+    })
+    .filter((s) => s.score > 0)
+    .sort((a, b) => b.score - a.score || openRank(b.m) - openRank(a.m))
+    .slice(0, limit)
+    .map((s) => s.m);
+}
+
+function openRank(m: Memory): number {
+  return m.status === "open" ? 1 : 0;
+}
+
+// Only the newest open threads are surfaced, and each session sees the block
+// once (per process), so the frontier costs a bounded number of tokens.
+const OPEN_THREAD_LIMIT = 3;
+const frontierInjectedSessions = new Set<string>();
+
+export async function systemDigest(
+  _input: unknown,
+  output: TransformOutput,
+): Promise<void> {
+  const mems = (await loadAll()).filter(isActive);
+  if (!mems.length) return;
+  const latest = mems.map((m) => m.ts).sort().pop() ?? "?";
+  output.system.push(
+    "\n## Persistent memory (opencode-self-improvement)\n" +
+      `${mems.length} durable memories; latest: ${latest}. ` +
+      "Check memory_recall BEFORE grepping or reading, for build/test/lint commands, decisions, and conventions. Relevant memories are also auto-injected when your message matches.",
+  );
+
+  const open = mems
+    .filter((m) => m.status === "open")
+    .sort((a, b) => (a.ts < b.ts ? 1 : -1))
+    .slice(0, OPEN_THREAD_LIMIT);
+  if (!open.length) return;
+
+  output.system.push(
+    "\n## Open threads (frontier)\n" + open.map(formatMemory).join("\n"),
+  );
+
+  // Aggregate of past sessions' convergence (one line, gated on data existing;
+  // I3: it is a derived fact, never the raw log). Gives the next session the
+  // signal to recall-first and stop re-deriving.
+  const conv = await recentConvergenceSummary();
+  if (conv) output.system.push(`\nConvergence tracking: ${conv}`);
+
+  const sid =
+    typeof _input === "object" && _input !== null &&
+    "sessionID" in _input &&
+    typeof (_input as Record<string, unknown>).sessionID === "string"
+      ? String((_input as Record<string, unknown>).sessionID)
+      : "?";
+  if (frontierInjectedSessions.has(sid)) return;
+  frontierInjectedSessions.add(sid);
+  const openMems = mems.filter((m) => m.status === "open");
+  await record(
+    "frontier_injected",
+    {
+      session: sid,
+      scopes: bracketList([...new Set(open.map((m) => m.scope))]),
+      shown: open.length,
+      open_total: openMems.length,
+      ts_since: dateStr(),
+    },
+    false,
+  );
+}
+
+export async function relevantKnowledge(message: string, limit = 3): Promise<string> {
+  const text = message.trim();
+  if (!text) return "";
+  const mems = (await loadAll()).filter(isActive);
+  const top = scoreMemories(text, mems, limit);
+  if (!top.length) return "";
+  return `## Relevant durable memory\n${top.map(formatMemory).join("\n")}`;
+}
+
+export async function contextualPull(input: PullInput, output: PullOutput): Promise<void> {
+  const tool = typeof input.tool === "string" ? input.tool : "";
+  if (!["grep", "glob", "read"].includes(tool)) return;
+  const term = argText(input.args);
+  if (!term) return;
+  const mems = (await loadAll()).filter(isActive);
+  const top = scoreMemories(term, mems, 3);
+  if (!top.length) return;
+  const block = top.map(formatMemory).join("\n");
+  if (typeof output.output === "string") {
+    output.output += `\n## Related durable memory (pull)\n${block}`;
+  }
+}

package/src/logfmt.ts

@@ -0,0 +1,67 @@
+// Minimal logfmt reader/writer shared by the durable store and the evidence
+// log. Field order is preserved on write so chains are diffable.
+import { dirname } from "node:path";
+import { mkdir } from "node:fs/promises";
+
+export type Fields = Record<string, unknown>;
+
+export function quote(value: unknown): string {
+  const s = String(value);
+  if (s === "") return '""';
+  return /[\s"=]/.test(s) ? `"${s.replace(/"/g, '\\"')}"` : s;
+}
+
+export function formatLine(fields: Fields): string {
+  const parts: string[] = [];
+  for (const [k, v] of Object.entries(fields)) {
+    if (v === undefined || v === null) continue;
+    parts.push(`${k}=${quote(v)}`);
+  }
+  return parts.join(" ");
+}
+
+// Unescape a quoted logfmt value and read it back.
+function unquote(s: string): string {
+  return s.replace(/\\"/g, '"');
+}
+
+export function parseLine(line: string): Fields | null {
+  const trimmed = line.trim();
+  if (!trimmed || !trimmed.includes("=")) return null;
+  const fields: Fields = {};
+  const re = /([A-Za-z0-9_.-]+)=("(?:[^"\\]|\\.)*"|[^\s]*)/g;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(trimmed)) !== null) {
+    const key = m[1];
+    let raw = m[2];
+    if (raw.startsWith('"') && raw.endsWith('"')) {
+      raw = unquote(raw.slice(1, -1));
+    }
+    fields[key] = raw;
+  }
+  // parse booleans/numbers as strings is fine; callers coerce.
+  return fields;
+}
+
+export async function ensureDir(path: string): Promise<void> {
+  await mkdir(path, { recursive: true });
+}
+
+export async function appendLog(file: string, line: string): Promise<void> {
+  await ensureDir(dirname(file));
+  const next = line.endsWith("\n") ? line : line + "\n";
+  const f = Bun.file(file);
+  const existing = (await f.exists()) ? await f.text() : "";
+  await Bun.write(file, existing + next);
+}
+
+export async function readLines(file: string): Promise<string[]> {
+  const f = Bun.file(file);
+  if (!(await f.exists())) return [];
+  const text = await f.text();
+  return text.split("\n").filter((l) => l.trim().length > 0);
+}
+
+export function bracketList(items: string[]): string {
+  return `[${items.join(",")}]`;
+}