opencode-self-improvement 0.8.1 → 0.8.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-self-improvement",
-  "version": "0.8.1",
+  "version": "0.8.3",
   "description": "Durable memory for opencode: remember facts, decisions, and commands across sessions, plus a one-command per-repo setup (/repo-init).",
   "main": "index.ts",
   "exports": {

package/src/capture.ts

@@ -7,6 +7,7 @@
 // All parsing functions are TOTAL: any malformed input yields [] / "" / null,
 // never a throw.
 import { STATUSES, TYPES } from "./types.ts";
+import { flattenMemoryText } from "./sanitize.ts";
 
 export interface CaptureOptions {
   model: string; // "provider/id"
@@ -97,10 +98,16 @@ export function parseCandidates(raw: string, max: number): CaptureCandidate[] {
       if (out.length >= max) break;
       if (!item || typeof item !== "object" || Array.isArray(item)) continue;
       const o = item as Record<string, unknown>;
-      const type = typeof o.type === "string" ? o.type.trim() : "";
-      const scope = typeof o.scope === "string" ? o.scope.trim() : "";
+      const rawType = typeof o.type === "string" ? o.type : "";
+      const rawScope = typeof o.scope === "string" ? o.scope : "";
+      const type = rawType.trim();
+      const scope = rawScope.trim();
       const content = typeof o.content === "string" ? o.content.trim() : "";
       if (!type || !scope || !content) continue;
+      if (
+        flattenMemoryText(rawType) !== rawType ||
+        flattenMemoryText(rawScope) !== rawScope
+      ) continue;
       if (!VALID_TYPES.has(type)) continue;
       if (content.length < 20 || content.length > 600) continue;
       const status = o.status === "open" || o.status === "settled" ? o.status : undefined;

package/src/gates.ts

@@ -14,6 +14,7 @@ import {
   type Memory,
 } from "./memory.ts";
 import { chainId, contentHash, record, recordWriteBlocked } from "./evidence.ts";
+import { flattenMemoryText } from "./sanitize.ts";
 import {
   displayCall,
   extractCallForms,
@@ -222,6 +223,34 @@ export async function runWriteGate(
     };
   }
 
+  const badMetadata: string[] = ([
+    ["type", candidate.type],
+    ["scope", candidate.scope],
+    ["issue", candidate.issue],
+  ] as const)
+    .filter(
+      ([, value]) =>
+        value !== undefined &&
+        (typeof value !== "string" || flattenMemoryText(value) !== value),
+    )
+    .map(([field]) => field);
+  if (
+    candidate.tags !== undefined &&
+    (!Array.isArray(candidate.tags) ||
+      candidate.tags.some((tag) => typeof tag !== "string" || flattenMemoryText(tag) !== tag))
+  ) {
+    badMetadata.push("tags");
+  }
+  if (badMetadata.length) {
+    await recordWriteBlocked(4, "invalid_metadata", [`fields=${badMetadata.join(",")}`], chain);
+    return {
+      action: "blocked",
+      reason: "invalid_metadata",
+      chain,
+      hint: `${badMetadata.join(", ")} must not contain control characters.`,
+    };
+  }
+
   const ch = contentHash(candidate.content);
   await record("gate_start", {
     chain,

package/src/index.ts

@@ -65,6 +65,7 @@ import { getCurrent, setCurrent } from "./state.ts";
 import { resolveRepo, setRepoScope, rememberSessionDir, sessionDir } from "./repokey.ts";
 import { readLines } from "./logfmt.ts";
 import { buildCapturePrompt, flattenTranscript, parseCandidates, parseModelRef } from "./capture.ts";
+import { flattenMemoryText } from "./sanitize.ts";
 
 // Tool/input boundaries are zod schemas (src/schema.ts): the JSON Schema the
 // host sees is GENERATED from them, and raw args are re-parsed at execution.
@@ -387,7 +388,10 @@ const impl = {
             types.set(m.type, (types.get(m.type) ?? 0) + 1);
           }
           const fmt = (map: Map<string, number>) =>
-            [...map.entries()].sort((a, b) => b[1] - a[1]).map(([k, c]) => `  ${k}: ${c}`).join("\n");
+            [...map.entries()]
+              .sort((a, b) => b[1] - a[1])
+              .map(([k, c]) => `  ${flattenMemoryText(k)}: ${c}`)
+              .join("\n");
           return { content: `Total: ${mems.length}\n\nScopes:\n${fmt(scopes)}\n\nTypes:\n${fmt(types)}` };
         },
       }));
@@ -523,7 +527,10 @@ const impl = {
             return { content: `No active open threads older than ${days} days.` };
           }
           const listing = stale
-            .map((m) => `  ${m.ts} ${m.type}/${m.scope}: ${m.content.slice(0, 100)}`)
+            .map(
+              (m) =>
+                `  ${flattenMemoryText(m.ts)} ${flattenMemoryText(m.type)}/${flattenMemoryText(m.scope)}: ${flattenMemoryText(m.content).slice(0, 100)}`,
+            )
             .join("\n");
           if (dryRun) {
             await record(
@@ -595,10 +602,13 @@ const impl = {
 
     // Per-session frozen injection epochs: the relevance block plus the store
     // fingerprint it was computed against. Recomputed ONLY on the first prompt
-    // of a session, when the store changes, or after a compaction (see the
-    // prompt/compaction hooks) — so the system prefix, and with it the
-    // provider prompt cache, survives ordinary turns.
+    // of a session, when injectable rows change, or after a compaction (see the
+    // prompt/context/compaction hooks) — so the system prefix, and with it the
+    // provider prompt cache, survives ordinary requests.
     const epoch = new Map<string, { version: string; block: string; at: string; reason: "first" | "store_changed" }>();
+    // Latest user prompt per session. A changed store is re-scored against it
+    // even when the change happened during a tool loop with no new prompt.
+    const latestPrompt = new Map<string, string>();
     // Recomputes since server start: every one invalidated a prompt cache, so
     // it is the honest cost number the surfaces show (status files / tools).
     let epochRecomputes = 0;
@@ -647,6 +657,47 @@ const impl = {
     const flushStatus = (sid: string): Promise<void> =>
       buildPayload(sid).then((p) => (p ? writeStatus(p) : undefined));
 
+    // Called when a prompt arrives (to keep status surfaces current) and again
+    // at the injection decision on every model request. The latter keeps
+    // removals fail-closed: an out-of-band redaction or a block during a tool
+    // loop cannot ride one more request waiting for another user prompt.
+    const injectableRows = async (): Promise<Memory[]> =>
+      (await loadAll()).filter((m) => isActive(m) && isVisibleHere(m));
+    const refreshEpoch = async (sid: string, rows?: Memory[]): Promise<void> => {
+      if (!latestPrompt.has(sid)) return;
+      const mems = rows ?? await injectableRows();
+      const version = await storeVersion(mems);
+      const cached = epoch.get(sid);
+      if (cached?.version === version) return;
+      const raw = await relevantKnowledge(latestPrompt.get(sid) ?? "", 5, mems); // widen candidate pool
+      const block = raw ? capToBudget(raw, budget) : "";
+      const at = new Date().toISOString();
+      const reason: "first" | "store_changed" = cached ? "store_changed" : "first";
+      epoch.set(sid, { version, block, at, reason });
+      await record(
+        "epoch_recompute",
+        {
+          session: sid,
+          reason,
+          chars: block.length,
+        },
+        false,
+      );
+      epochRecomputes++;
+      // Ledger: a store_changed recompute of a live epoch is an
+      // invalidation (the first epoch of a session busts nothing). The
+      // epoch request count restarts with the new block.
+      const st = statsFor(sid);
+      st.epochRequests = 0;
+      if (reason === "store_changed") {
+        st.invalidations++;
+        st.lastInvalidation = { at, reason };
+      }
+      // Status hand-off for the terminal panel (src/tui.tsx) and any
+      // out-of-process surface. Best-effort; never breaks injection.
+      await flushStatus(sid);
+    };
+
     // ---- hooks (all fail-closed) -------------------------------------------
     // Bookkeeping: track the active session/tool/call before a tool runs.
     await ctx.tool?.hook("execute.before", (event: any) =>
@@ -684,12 +735,14 @@ const impl = {
       guarded("session.context (systemDigest)", async () => {
         // Route store reads in this hook through the right session's scope.
         setCurrent({ sessionID: event?.sessionID });
+        const sid = event?.sessionID;
+        const mems = await injectableRows();
+        if (typeof sid === "string") await refreshEpoch(sid, mems);
         const parts: string[] = [];
-        await systemDigest({ sessionID: event?.sessionID }, { system: parts });
+        await systemDigest({ sessionID: event?.sessionID }, { system: parts }, mems);
         for (const text of parts) event?.system?.push?.({ type: "text", text });
         // Deliver this epoch's relevance-ranked memory via the system channel.
         // The UI never renders system parts, unlike appended prompt text.
-        const sid = event?.sessionID;
         const knowledge = typeof sid === "string" ? epoch.get(sid)?.block : undefined;
         if (knowledge) event?.system?.push?.({ type: "text", text: knowledge });
         // Count AT THE INJECTION DECISION so badge/panel numbers match exactly
@@ -705,10 +758,11 @@ const impl = {
       }),
     );
 
-    // Memory-first injection, frozen per epoch (cache-safe). The block is
-    // relevance-scored against the prompt that STARTED the epoch — accepted
-    // staleness, traded for a stable system prefix across ordinary turns. An
-    // epoch is invalidated only on first prompt, store change, or compaction.
+    // Memory-first injection, frozen per epoch (cache-safe). Remember the
+    // latest user prompt here; the request-time context hook builds or
+    // invalidates the epoch immediately before delivery. Relevance staleness
+    // is accepted while the injectable rows stay unchanged, preserving a
+    // stable system prefix across ordinary turns.
     // Never mutate event.prompt.text — it is stored verbatim and rendered in
     // the UI.
     await ctx.session?.hook("prompt", (event: any) =>
@@ -719,38 +773,8 @@ const impl = {
           markTurn(sid);
           await checkNewSession(sid);
           await bindSessionFromMessages(sid);
-          const version = await storeVersion();
-          const cached = epoch.get(sid);
-          if (!cached || cached.version !== version) {
-            const text = event?.prompt?.text ?? "";
-            const raw = await relevantKnowledge(text, 5); // widen candidate pool
-            const block = raw ? capToBudget(raw, budget) : "";
-            const at = new Date().toISOString();
-            const reason: "first" | "store_changed" = cached ? "store_changed" : "first";
-            epoch.set(sid, { version, block, at, reason });
-            await record(
-              "epoch_recompute",
-              {
-                session: sid,
-                reason,
-                chars: block.length,
-              },
-              false,
-            );
-            epochRecomputes++;
-            // Ledger: a store_changed recompute of a live epoch is an
-            // invalidation (the first epoch of a session busts nothing). The
-            // epoch request count restarts with the new block.
-            const st = statsFor(sid);
-            st.epochRequests = 0;
-            if (reason === "store_changed") {
-              st.invalidations++;
-              st.lastInvalidation = { at, reason };
-            }
-            // Status hand-off for the terminal panel (src/tui.tsx) and any
-            // out-of-process surface. Best-effort; never breaks injection.
-            await flushStatus(sid);
-          }
+          latestPrompt.set(sid, event?.prompt?.text ?? "");
+          await refreshEpoch(sid);
         }
       }),
     );
@@ -765,7 +789,10 @@ const impl = {
       guarded("session.compaction (onCompacting)", async () => {
         setCurrent({ sessionID: event?.sessionID });
         await onCompacting({ sessionID: event?.sessionID });
-        if (typeof event?.sessionID === "string") epoch.delete(event.sessionID);
+        if (typeof event?.sessionID === "string") {
+          epoch.delete(event.sessionID);
+          latestPrompt.delete(event.sessionID);
+        }
         const anchor = await compactionAnchor();
         if (anchor) event?.system?.push?.({ type: "text", text: anchor });
       }),

package/src/inject.ts

@@ -5,11 +5,13 @@
 // results. Evidence is never injected. Open threads (status=
 // "open") get a single capped frontier block in the digest so planning sessions
 // start at the previous session's frontier instead of re-deriving it.
+import { createHash } from "node:crypto";
 import { isActive, loadAll, visibleHere, type Memory } from "./memory.ts";
 import { record } from "./evidence.ts";
 import { dateStr } from "./config.ts";
 import { bracketList } from "./logfmt.ts";
 import { recentConvergenceSummary } from "./triggers.ts";
+import { flattenMemoryText } from "./sanitize.ts";
 
 export interface TransformOutput {
   system: string[];
@@ -53,9 +55,12 @@ export function formatMemory(m: Memory): string {
   // One memory is ONE line in every injected block: the store round-trips real
   // newlines, so a planted "\n## Fake header" would forge its own section and
   // escape the framing (and throw capToBudget's line accounting off). Line
-  // breaks collapse to spaces; ordinary spacing survives.
-  const content = m.content.replace(/[\r\n\t]+/g, " ");
-  return `- [${dateOf(m.ts)}]${tag}${src} ${m.type}/${m.scope}: ${content}`;
+  // breaks and other controls collapse to spaces; ordinary spacing survives.
+  const date = flattenMemoryText(dateOf(m.ts));
+  const type = flattenMemoryText(m.type);
+  const scope = flattenMemoryText(m.scope);
+  const content = flattenMemoryText(m.content);
+  return `- [${date}]${tag}${src} ${type}/${scope}: ${content}`;
 }
 
 function argText(args: unknown): string {
@@ -137,11 +142,12 @@ const frontierInjectedSessions = new Set<string>();
 export async function systemDigest(
   _input: unknown,
   output: TransformOutput,
+  rows?: Memory[],
 ): Promise<void> {
   // Repo-scoped: model-facing reads see this repo + global rows.
-  const mems = visibleHere((await loadAll()).filter(isActive));
+  const mems = rows ?? visibleHere((await loadAll()).filter(isActive));
   if (!mems.length) return;
-  const latest = mems.map((m) => m.ts).sort().pop() ?? "?";
+  const latest = flattenMemoryText(mems.map((m) => m.ts).sort().pop() ?? "?");
   output.system.push(
     "\n## Persistent memory (opencode-self-improvement)\n" +
       `${mems.length} durable memories; latest: ${latest}. ` +
@@ -177,7 +183,7 @@ export async function systemDigest(
     "frontier_injected",
     {
       session: sid,
-      scopes: bracketList([...new Set(open.map((m) => m.scope))]),
+      scopes: bracketList([...new Set(open.map((m) => flattenMemoryText(m.scope)))]),
       shown: open.length,
       open_total: openMems.length,
       ts_since: dateStr(),
@@ -186,26 +192,33 @@ export async function systemDigest(
   );
 }
 
-export async function relevantKnowledge(message: string, limit = 3): Promise<string> {
+export async function relevantKnowledge(message: string, limit = 3, rows?: Memory[]): Promise<string> {
   const text = message.trim();
   if (!text) return "";
   // Repo-scoped: model-facing reads see this repo + global rows.
-  const mems = visibleHere((await loadAll()).filter(isActive));
+  const mems = rows ?? visibleHere((await loadAll()).filter(isActive));
   const top = scoreMemories(text, mems, limit);
   if (!top.length) return "";
   return `## Relevant durable memory${FRAME}\n${top.map(formatMemory).join("\n")}`;
 }
 
-// Deterministic fingerprint of the active store: `${activeCount}:${maxActiveTs}`.
-// Cheap change detector for the per-session epoch cache in index.ts: the frozen
-// injection block is recomputed only when this string changes (or after a
-// compaction). Evidence and superseded rows never affect it (I3, memory.ts).
-export async function storeVersion(): Promise<string> {
+// Deterministic fingerprint of the rows that can affect injection. Hash the
+// exact rendered row plus its full id, in load order: content edits, blocklist
+// changes and supersession all invalidate an epoch, while evidence, hidden
+// rows and non-rendered metadata do not. This is only a linear hash over rows
+// loadAll() already had to read and parse for the request.
+export async function storeVersion(rows?: Memory[]): Promise<string> {
   // Repo-scoped: model-facing reads see this repo + global rows.
-  const mems = visibleHere((await loadAll()).filter(isActive));
+  const mems = rows ?? visibleHere((await loadAll()).filter(isActive));
+  if (!mems.length) return "0:";
   let max = "";
-  for (const m of mems) if (m.ts > max) max = m.ts;
-  return `${mems.length}:${max}`;
+  const hash = createHash("sha256");
+  for (const m of mems) {
+    if (m.ts > max) max = m.ts;
+    hash.update(JSON.stringify([m.ts, formatMemory(m)]));
+    hash.update("\n");
+  }
+  return `${mems.length}:${max}:${hash.digest("hex")}`;
 }
 
 // Trim an injection block to `budget` characters at WHOLE-line boundaries:

package/src/memory.ts

@@ -11,6 +11,7 @@ import { getCurrent } from "./state.ts";
 import { appendLog, ensureDir, formatLine, parseLine, readLines, writeAtomic, type Fields } from "./logfmt.ts";
 import { withStoreLock, LOCK_WRITE } from "./lock.ts";
 import { blockedTs } from "./blocklist.ts";
+import { flattenMemoryText } from "./sanitize.ts";
 
 export type MemoryStatus = "open" | "settled";
 
@@ -167,6 +168,18 @@ export async function appendMemory(input: AppendMemoryInput): Promise<string> {
   if (input.content.length > MAX_CONTENT_CHARS) {
     throw new Error(`memory content exceeds ${MAX_CONTENT_CHARS} chars (${input.content.length})`);
   }
+  for (const [field, value] of [
+    ["type", input.type],
+    ["scope", input.scope],
+    ["issue", input.issue],
+  ] as const) {
+    if (value !== undefined && flattenMemoryText(value) !== value) {
+      throw new Error(`memory ${field} must not contain control characters`);
+    }
+  }
+  if (input.tags?.some((tag) => flattenMemoryText(tag) !== tag)) {
+    throw new Error("memory tags must not contain control characters");
+  }
   // Repo stamp at the single write choke point: gated writes, viewer restores
   // and edits all pass through here. Explicit "global" survives; an unscoped
   // process (viewer) stamps nothing = global. Session identity wins over the

package/src/portable.ts

@@ -8,6 +8,7 @@ import { memoryRoot } from "./config.ts";
 import { isActive, isVisibleHere, loadAll, type Memory } from "./memory.ts";
 import { TYPES } from "./types.ts";
 import type { WriteCandidate } from "./gates.ts";
+import { flattenMemoryText } from "./sanitize.ts";
 
 // --- export -------------------------------------------------------------------
 
@@ -101,7 +102,8 @@ function exportHeader(line: string): { version: number } | null {
 
 // A line is malformed (skipped, never gated) unless it is a JSON object with a
 // known type, a non-empty scope, and non-empty content. Optional fields are
-// sanitized rather than fatal: an unusable status/tags/issue is dropped.
+// sanitized rather than fatal where noted. Control characters in structured
+// metadata make the row malformed; content alone may remain multiline.
 function parseRow(line: string): WriteCandidate | null {
   let row: unknown;
   try {
@@ -113,7 +115,13 @@ function parseRow(line: string): WriteCandidate | null {
   const r = row as Record<string, unknown>;
   if (typeof r.type !== "string" || !(TYPES as readonly string[]).includes(r.type)) return null;
   if (typeof r.scope !== "string" || !r.scope.trim()) return null;
+  if (flattenMemoryText(r.scope) !== r.scope) return null;
   if (typeof r.content !== "string" || !r.content.trim()) return null;
+  if (
+    Array.isArray(r.tags) &&
+    r.tags.some((t) => typeof t === "string" && flattenMemoryText(t) !== t)
+  ) return null;
+  if (typeof r.issue === "string" && flattenMemoryText(r.issue) !== r.issue) return null;
   const c: WriteCandidate = { type: r.type, scope: r.scope, content: r.content };
   if (r.status === "open" || r.status === "settled") c.status = r.status;
   if (Array.isArray(r.tags) && r.tags.every((t) => typeof t === "string")) c.tags = r.tags;

package/src/sanitize.ts

@@ -0,0 +1,9 @@
+// Memory content may legitimately be multiline in the store, but every field
+// rendered into a model-facing line must stay on that line. C0/C1 controls and
+// Unicode line/paragraph separators all collapse through this one helper so
+// renderers and write-boundary validators cannot drift.
+const CONTROL_CHARS = /[\p{Cc}\p{Zl}\p{Zp}]+/gu;
+
+export function flattenMemoryText(value: string): string {
+  return value.replace(CONTROL_CHARS, " ");
+}

package/src/schema.ts

@@ -16,20 +16,30 @@
 //     check in memory.ts (row identity), where the row schema lives.
 import { z, type ZodTypeAny } from "zod";
 import { STATUSES, TYPES } from "./types.ts";
+import { flattenMemoryText } from "./sanitize.ts";
 
 type JSONSchema = Record<string, unknown>;
 
 /** Row identity anywhere it arrives off a boundary: an ISO instant. */
 export const isoTs = z.iso.datetime();
 
+// Structured row fields become labels in model-facing text. Content remains
+// multiline by design and is flattened only when rendered; metadata controls
+// are rejected at every write boundary instead. Lookup-only scopes (recall,
+// forget, export) stay plain strings so a row stored before this check can
+// still be found and forgotten.
+const memoryField = z.string().refine((value) => flattenMemoryText(value) === value, {
+  message: "must not contain control characters",
+});
+
 // ---- tool argument schemas (JSON Schema generated from these) ------------
 
 export const RememberArgs = z.object({
   type: z.enum(TYPES).describe("Type of memory"),
-  scope: z.string().describe("Scope/area (e.g., auth, api, ctl)"),
+  scope: memoryField.describe("Scope/area (e.g., auth, api, ctl)"),
   content: z.string().describe("The memory content"),
-  issue: z.string().describe("Related issue").optional(),
-  tags: z.array(z.string()).optional(),
+  issue: memoryField.describe("Related issue").optional(),
+  tags: z.array(memoryField).optional(),
   status: z
     .enum(STATUSES)
     .describe(
@@ -60,7 +70,7 @@ export const RecallArgs = z.object({
 });
 
 export const UpdateArgs = z.object({
-  scope: z.string(),
+  scope: memoryField,
   type: z.enum(TYPES),
   content: z.string(),
   query: z
@@ -140,10 +150,10 @@ export const DecayArgs = z.object({
 export const EditBody = z.object({
   ts: isoTs,
   type: z.enum(TYPES),
-  scope: z.string().min(1),
+  scope: memoryField.min(1),
   content: z.string().min(1),
-  issue: z.string().optional(),
-  tags: z.array(z.string()).optional(),
+  issue: memoryField.optional(),
+  tags: z.array(memoryField).optional(),
   // min NOT enforced here: the mutate layer owns the audit-reason contract
   // (and its tests pin its message). Shape is the boundary's job only.
   reason: z.string(),
@@ -156,7 +166,7 @@ export const UnblockBody = z.object({ ts: isoTs });
 // POSTs append rows with empty type/scope/content.
 export const RestoreEntryBody = z.object({
   type: z.enum(TYPES),
-  scope: z.string().min(1),
+  scope: memoryField.min(1),
   content: z.string().min(1),
 });