agent-procedures 0.3.0 → 0.3.2

package/lib/engine.js

@@ -1,11 +1,14 @@
 import {
-  appendProcedure,
+  appendPending,
   appendTrace,
+  countIngest,
   deleteTrace,
-  readProcedures,
+  readAllProcedures,
   readTrace,
   startTrace,
 } from "./store.js";
+import { sanitizeCommand, sanitizeText } from "./sanitize.js";
+import { recall } from "./recall.js";
 
 /**
  * Harness-free hot path.
@@ -15,8 +18,9 @@ import {
  *   { type: "tool",   sessionId, promptId, tool, target, ok }
  *   { type: "stop",   sessionId, promptId, interrupted, busy }
  *
- * Returns { context: string | null }. `context` is text the harness should
- * inject into the agent's next turn, if it knows how.
+ * Returns { context: string | null, notice?: string | null }.
+ * `context` is text the harness should inject into the agent's next turn.
+ * `notice` is a short user-visible line; the harness maps it if the host supports it.
  */
 export function handleEvent(evt, reflexDir, deps = {}) {
   if (!evt || !evt.sessionId || !evt.promptId) return { context: null };
@@ -33,47 +37,35 @@ export const GROOM_THRESHOLD = 5;
 
 function onPrompt(evt, reflexDir) {
   const { sessionId, promptId, prompt } = evt;
-  startTrace(reflexDir, sessionId, promptId, { t: "Goal", prompt });
+  const sanitized = sanitizeCommand(prompt);
+  startTrace(reflexDir, sessionId, promptId, {
+    t: "Goal",
+    prompt: sanitized.text,
+    ...(sanitized.requiresEnv.length > 0 ? { requires_env: sanitized.requiresEnv } : {}),
+    ...(sanitized.unsafe ? { contains_unresolvable_secret: true } : {}),
+  });
 
-  const hit = recall(prompt, readProcedures(reflexDir));
-  if (!hit) return { context: null };
+  const hit = recall(sanitized.text, readAllProcedures(reflexDir));
+  if (!hit) return { context: null, notice: null };
 
   appendTrace(reflexDir, sessionId, promptId, { t: "Recall", hit: hit.node.id });
-  return { context: hit.text };
-}
-
-function recall(prompt, procs) {
-  if (!prompt) return null;
-  const p = prompt.toLowerCase();
-  const fmt = (steps) => steps.map((s) => `- ${s.t}: ${s.target}`).join("\n");
-
-  for (const node of procs) {
-    if (node.enabled === false || node.parent_id) continue;
-
-    let match = node.trigger && p.includes(node.trigger.toLowerCase());
-    if (!match && Array.isArray(node.aliases)) {
-      match = node.aliases.some((a) => p.includes(a.toLowerCase()));
-    }
-    if (!match) continue;
-
-    let text = fmt(node.steps);
-    const alt = procs.find((e) => e.parent_id === node.id && e.type === "alt" && e.enabled !== false);
-    const onFail = procs.find((e) => e.parent_id === node.id && e.type === "on_fail" && e.enabled !== false);
-    if (alt) text += "\n\nAlternative path that worked:\n" + fmt(alt.steps);
-    if (onFail) text += "\n\nIf it fails, this recovery worked:\n" + fmt(onFail.steps);
-
-    return {
-      node,
-      text: `Reflex memory found a procedure for this task:\n\n${text}\n\nFollow these steps instead of figuring it out from scratch.`,
-    };
-  }
-  return null;
+  return { context: hit.text, notice: hit.notice };
 }
 
 // --- tool: trace ----------------------------------------------------------
 
 function onTool(evt, reflexDir) {
-  appendTrace(reflexDir, evt.sessionId, evt.promptId, { t: evt.tool, target: evt.target || "", ok: evt.ok });
+  const sanitized =
+    evt.tool === "Bash" || evt.tool === "bash"
+      ? sanitizeCommand(evt.target)
+      : sanitizeText(evt.target);
+  appendTrace(reflexDir, evt.sessionId, evt.promptId, {
+    t: evt.tool,
+    target: sanitized.text,
+    ok: evt.ok,
+    ...(sanitized.requiresEnv.length > 0 ? { requires_env: sanitized.requiresEnv } : {}),
+    ...(sanitized.unsafe ? { contains_unresolvable_secret: true } : {}),
+  });
   return { context: null };
 }
 
@@ -83,48 +75,63 @@ function onStop(evt, reflexDir, deps) {
   const { sessionId, promptId } = evt;
   const lines = readTrace(reflexDir, sessionId, promptId);
   if (lines.length === 0) return { context: null };
+  if (lines.some((line) => line.contains_unresolvable_secret)) {
+    deleteTrace(reflexDir, sessionId, promptId);
+    return { context: null };
+  }
 
   let goal = "";
   let recallHit = null;
+  let hasGoal = false;
   for (const l of lines) {
-    if (l.t === "Goal") goal = l.prompt;
+    if (l.t === "Goal") {
+      goal = l.prompt;
+      hasGoal = true;
+    }
     if (l.t === "Recall" && l.hit) recallHit = l.hit;
   }
+  // Continuation after another Stop hook can recreate a trace via appendTrace
+  // with no Goal line. Don't learn from that stump.
+  if (!hasGoal) {
+    deleteTrace(reflexDir, sessionId, promptId);
+    return { context: null };
+  }
 
   if (judge(lines, evt)) {
-    const steps = synthesize(lines);
+    const { steps, requiresEnv } = synthesize(lines);
     if (steps.length > 0) {
+      const id = Math.random().toString(36).slice(2, 11);
       if (!recallHit) {
-        appendProcedure(reflexDir, {
-          id: Math.random().toString(36).slice(2, 11),
+        appendPending(reflexDir, {
+          id,
           title: goal,
           trigger: goal,
           steps,
+          ...(requiresEnv.length > 0 ? { requires_env: requiresEnv } : {}),
           enabled: true,
-          hits: 0,
-          last_used_at: new Date().toISOString(),
           created_at: new Date().toISOString(),
           status: "pending_review",
         });
       } else {
-        const parent = readProcedures(reflexDir).find((p) => p.id === recallHit);
+        const parent = readAllProcedures(reflexDir).find((p) => p.id === recallHit);
         const parentSteps = parent ? parent.steps : [];
         const same =
           parentSteps.length === steps.length &&
           parentSteps.every((s, i) => s.t === steps[i].t && s.target === steps[i].target);
         if (!same) {
-          appendProcedure(reflexDir, {
+          appendPending(reflexDir, {
+            id,
             parent_id: recallHit,
             type: "alt",
             steps,
+            ...(requiresEnv.length > 0 ? { requires_env: requiresEnv } : {}),
             created_at: new Date().toISOString(),
             status: "pending_review",
           });
         }
       }
 
-      const pending = readProcedures(reflexDir).filter((p) => p.status === "pending_review").length;
-      if (pending >= GROOM_THRESHOLD && deps.spawnGroomer) deps.spawnGroomer();
+      if (countIngest(reflexDir) >= GROOM_THRESHOLD && deps.spawnGroomer) deps.spawnGroomer();
     }
   }
 
@@ -165,15 +172,18 @@ function judge(lines, evt) {
 
 function synthesize(lines) {
   const steps = [];
+  const requiresEnv = new Set();
   for (const l of lines) {
+    for (const name of l.requires_env || []) requiresEnv.add(name);
     if (l.t === "Goal" || l.t === "Recall" || l.ok === false) continue;
     if (l.t === "Read" || l.t === "View") continue;
     if (l.t === "Bash" || l.t === "bash") {
       const target = l.target || "";
+      if (!target.trim()) continue;
       if (PURE_READ_CMDS.includes(target.trim().split(" ")[0])) continue;
       if (/^git (status|log|diff)/.test(target)) continue;
     }
     steps.push({ t: l.t, target: l.target });
   }
-  return steps;
+  return { steps, requiresEnv: [...requiresEnv] };
 }

package/lib/groomer.js

@@ -1,60 +1,55 @@
 import fs from "node:fs";
 import path from "node:path";
 import { PROVIDERS, resolveCredentials } from "./providers.js";
+import {
+  appendPending,
+  claimPendingBatch,
+  countIngest,
+  migrateLegacyPending,
+  paths,
+  readProcedures,
+  readProcessing,
+  releaseGroomerLock,
+  tryAcquireGroomerLock,
+  unlinkWithRetry,
+  writeJsonlAtomic,
+} from "./store.js";
 
-// Silent Failure Logging
-function logError(proceduresFile, msg) {
-  const reflexDir = path.dirname(proceduresFile);
+export const GROOM_MAX_ATTEMPTS = 3;
+
+function logError(reflexDir, msg) {
   const logFile = path.join(reflexDir, "worker.log");
   const timestamp = new Date().toISOString();
   try {
     fs.appendFileSync(logFile, `[${timestamp}] ${msg}\n`, "utf8");
-  } catch (e) {
-    // nowhere to log, silently fail
+  } catch {
+    /* nowhere to log */
   }
 }
 
-export async function runGroomer(proceduresFile) {
-  let creds;
-  try {
-    creds = resolveCredentials();
-  } catch (e) {
-    logError(proceduresFile, e.message);
-    return;
+function mergeStable(stable, batchIds, keptFromBatch) {
+  const withoutBatch = stable.filter((n) => !n.id || !batchIds.has(n.id));
+  const byId = new Map();
+  const anonymous = [];
+  for (const n of [...withoutBatch, ...keptFromBatch]) {
+    if (n.id) byId.set(n.id, n);
+    else anonymous.push(n);
   }
-  if (!creds) {
-    logError(
-      proceduresFile,
-      "No API key found. Run `npx agent-procedures auth login` or set ANTHROPIC_API_KEY / OPENAI_API_KEY / GEMINI_API_KEY.",
-    );
-    return;
+  return [...byId.values(), ...anonymous];
+}
+
+async function processBatch(reflexDir, processingFile, creds) {
+  const batch = readProcessing(reflexDir);
+  if (batch.length === 0) {
+    unlinkWithRetry(processingFile);
+    return { ok: true, deferred: 0 };
   }
-  const { provider, apiKey, model } = creds;
-
-  if (!fs.existsSync(proceduresFile)) return;
-
-  const lines = fs
-    .readFileSync(proceduresFile, "utf8")
-    .split("\n")
-    .filter(Boolean);
-  const nodes = lines
-    .map((l) => {
-      try {
-        return JSON.parse(l);
-      } catch (e) {
-        return null;
-      }
-    })
-    .filter(Boolean);
-
-  const pending = nodes.filter((n) => n.status === "pending_review");
-  if (pending.length === 0) return;
 
   const prompt = `
-Here are ${pending.length} new execution nodes/edges added by a coding agent.
+Here are ${batch.length} new execution nodes/edges added by a coding agent.
 
 <edges>
-${JSON.stringify(pending, null, 2)}
+${JSON.stringify(batch, null, 2)}
 </edges>
 
 For each item:
@@ -77,10 +72,10 @@ Return ONLY a JSON array of objects with this schema:
 
   let responseText;
   try {
-    responseText = await PROVIDERS[provider].complete(apiKey, model, prompt);
+    responseText = await PROVIDERS[creds.provider].complete(creds.apiKey, creds.model, prompt);
   } catch (err) {
-    logError(proceduresFile, err.message);
-    return;
+    logError(reflexDir, err.message);
+    return { ok: false, deferred: 0 };
   }
 
   let actions = [];
@@ -90,40 +85,81 @@ Return ONLY a JSON array of objects with this schema:
       .replace(/```/g, "")
       .trim();
     actions = JSON.parse(jsonStr);
-  } catch (e) {
-    logError(
-      proceduresFile,
-      "Failed to parse LLM JSON response: " + responseText,
-    );
-    return;
+  } catch {
+    logError(reflexDir, "Failed to parse LLM JSON response.");
+    return { ok: false, deferred: 0 };
   }
 
-  const newNodes = [];
-  for (const node of nodes) {
-    if (node.status === "pending_review") {
-      const actionDef = actions.find((a) => a.id === node.id);
-      if (actionDef) {
-        if (actionDef.action === "DROP") {
-          continue;
-        } else {
-          delete node.status;
-          if (!node.parent_id && actionDef.aliases) {
-            node.aliases = actionDef.aliases;
-          }
-        }
-      }
+  const kept = [];
+  const deferred = [];
+  for (const node of batch) {
+    const actionDef = actions.find((a) => a.id === node.id);
+    if (!actionDef) {
+      // No verdict from the model. Retry a few times, then give up on it.
+      const attempts = (node.groom_attempts || 0) + 1;
+      if (attempts < GROOM_MAX_ATTEMPTS) deferred.push({ ...node, groom_attempts: attempts });
+      continue;
     }
-    newNodes.push(node);
+    if (actionDef.action === "DROP") continue;
+    const next = { ...node };
+    delete next.status;
+    delete next.groom_attempts;
+    if (!next.parent_id && Array.isArray(actionDef.aliases)) {
+      next.aliases = actionDef.aliases;
+    }
+    kept.push(next);
   }
 
-  // Atomic Writes for the Groomer
-  const newFileContent =
-    newNodes.map((n) => JSON.stringify(n)).join("\n") + "\n";
-  const tmpFile = proceduresFile + ".tmp";
+  // Only remove ids we actually resolved; deferred ids stay out of stable.
+  const resolvedIds = new Set(
+    batch.filter((n) => actions.some((a) => a.id === n.id)).map((n) => n.id).filter(Boolean),
+  );
+  const merged = mergeStable(readProcedures(reflexDir), resolvedIds, kept);
+  writeJsonlAtomic(paths(reflexDir).proceduresFile, merged);
+  unlinkWithRetry(processingFile);
+
+  for (const node of deferred) appendPending(reflexDir, node);
+  return { ok: true, deferred: deferred.length };
+}
+
+/**
+ * Compact pending → procedures. Single-flight. Safe vs concurrent hot-path appends.
+ * @param {string} reflexDir path to .reflex/
+ */
+export async function runGroomer(reflexDir) {
+  if (!tryAcquireGroomerLock(reflexDir)) return { ran: false, reason: "locked" };
+
   try {
-    fs.writeFileSync(tmpFile, newFileContent, "utf8");
-    fs.renameSync(tmpFile, proceduresFile);
-  } catch (err) {
-    logError(proceduresFile, `Failed atomic write: ${err.message}`);
+    migrateLegacyPending(reflexDir);
+
+    let creds;
+    try {
+      creds = resolveCredentials();
+    } catch (e) {
+      logError(reflexDir, e.message);
+      return { ran: false, reason: "creds" };
+    }
+    if (!creds) {
+      logError(
+        reflexDir,
+        "No API key found. Run `npx agent-procedures auth login` or set ANTHROPIC_API_KEY / OPENAI_API_KEY / GEMINI_API_KEY.",
+      );
+      return { ran: false, reason: "creds" };
+    }
+
+    let batches = 0;
+    while (batches < 8) {
+      const processingFile = claimPendingBatch(reflexDir);
+      if (!processingFile) break;
+      const { ok, deferred } = await processBatch(reflexDir, processingFile, creds);
+      if (!ok) break;
+      batches++;
+      // Deferred rows were written back to pending; don't spin on the same ids.
+      if (deferred > 0) break;
+      if (countIngest(reflexDir) === 0) break;
+    }
+    return { ran: true, batches };
+  } finally {
+    releaseGroomerLock(reflexDir);
   }
 }

package/lib/harnesses/claude.js

@@ -54,10 +54,12 @@ export default {
       return { type: "tool", ...base, tool, target, ok: name === "PostToolUse" };
     }
     if (name === "Stop") {
+      // stop_hook_active is a Stop-hook loop guard, not a user cancel.
+      // Claude's Stop does not fire on interrupt, so there is no interrupt signal here.
       return {
         type: "stop",
         ...base,
-        interrupted: payload.stop_hook_active === true,
+        interrupted: false,
         busy: Array.isArray(payload.background_tasks) && payload.background_tasks.length > 0,
       };
     }
@@ -68,8 +70,11 @@ export default {
   render(evt, result) {
     if (evt.type !== "prompt") return "";
     if (!result.context) return "{}";
-    return JSON.stringify({
+    const out = {
       hookSpecificOutput: { hookEventName: "UserPromptSubmit", additionalContext: result.context },
-    });
+    };
+    // Top-level systemMessage is shown to the user; not model context.
+    if (result.notice) out.systemMessage = result.notice;
+    return JSON.stringify(out);
   },
 };

package/lib/init.js

@@ -2,13 +2,13 @@ import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { DEFAULT_HARNESS, getHarness } from "./harnesses/index.js";
+import { migrateLegacyPending } from "./store.js";
 
 const LIB_DIR = path.dirname(fileURLToPath(import.meta.url));
 
 const DEFAULT_CONFIG = {
   version: 1,
   harness: "claude",
-  minConfidence: 0.7,
   store: {
     procedures: "procedures.jsonl",
     runs: "runs.jsonl",
@@ -38,27 +38,34 @@ function writeAlways(filePath, contents) {
 
 function ensureGitignore(cwd) {
   const gitignorePath = path.join(cwd, ".gitignore");
-  const entry = ".reflex/traces/";
-
-  if (!fs.existsSync(gitignorePath)) {
-    fs.writeFileSync(gitignorePath, `${entry}\n`, "utf8");
-    return "created";
-  }
-
-  const current = fs.readFileSync(gitignorePath, "utf8");
-  const lines = current.split(/\r?\n/);
-  const alreadyIgnored = lines.some(
-    (line) =>
-      line.trim() === ".reflex/traces/" || line.trim() === ".reflex/traces",
-  );
-
-  if (alreadyIgnored) {
-    return "exists";
-  }
-
-  const suffix = current.endsWith("\n") || current.length === 0 ? "" : "\n";
-  fs.writeFileSync(gitignorePath, `${current}${suffix}${entry}\n`, "utf8");
-  return "updated";
+  const block = [
+    "# Reflex: ignore runtime state; commit only the shared allowlist",
+    ".reflex/*",
+    "!.reflex/config.json",
+    "!.reflex/procedures.jsonl",
+    "!.reflex/runs.jsonl",
+    "!.reflex/runtime.js",
+  ];
+  // A whole-dir ignore (".reflex/") would defeat the allowlist: git cannot
+  // re-include a file whose parent directory is excluded.
+  const managed = new Set([
+    ...block,
+    ".reflex",
+    ".reflex/",
+    ".reflex/traces",
+    ".reflex/traces/",
+  ]);
+  const existed = fs.existsSync(gitignorePath);
+  const current = existed ? fs.readFileSync(gitignorePath, "utf8") : "";
+  const kept = current
+    .split(/\r?\n/)
+    .filter((line) => !managed.has(line.trim()));
+  while (kept.length > 0 && kept.at(-1) === "") kept.pop();
+  const next = [...kept, ...(kept.length > 0 ? [""] : []), ...block].join("\n") + "\n";
+
+  if (next === current) return "exists";
+  fs.writeFileSync(gitignorePath, next, "utf8");
+  return existed ? "updated" : "created";
 }
 
 // One bundled file, built by `npm run build` into dist/runtime.js.
@@ -117,6 +124,7 @@ export function init(cwd = process.cwd(), { harness: harnessId = DEFAULT_HARNESS
   writeShim(cwd, harness);
   harness.register(cwd);
   results.hooks = "updated";
+  results.migrate = migrateLegacyPending(reflexDir).moved > 0 ? "moved" : "noop";
 
   return results;
 }

package/lib/main.js

@@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url";
 import { handleEvent } from "./engine.js";
 import { runGroomer } from "./groomer.js";
 import { getHarness } from "./harnesses/index.js";
+import { isGroomerLocked } from "./store.js";
 
 // Bundled to <repo>/.reflex/runtime.js. That file sits inside .reflex/.
 const REFLEX_DIR = path.dirname(fileURLToPath(import.meta.url));
@@ -19,6 +20,7 @@ function readStdin() {
 }
 
 function spawnGroomer() {
+  if (isGroomerLocked(REFLEX_DIR)) return;
   const child = spawn(process.argv[0], [process.argv[1], "Groom"], { detached: true, stdio: "ignore" });
   child.unref();
 }
@@ -28,7 +30,7 @@ export async function run(harnessId, argv = process.argv.slice(2)) {
   if (!eventName) return;
 
   if (eventName === "Groom") {
-    await runGroomer(path.join(REFLEX_DIR, "procedures.jsonl"));
+    await runGroomer(REFLEX_DIR);
     return;
   }
 

package/lib/recall.js

@@ -0,0 +1,110 @@
+const STOP_WORDS = new Set([
+  "the",
+  "a",
+  "an",
+  "to",
+  "in",
+  "on",
+  "please",
+  "can",
+  "you",
+  "my",
+  "this",
+  "for",
+  "with",
+]);
+
+export function tokenize(text) {
+  if (!text) return [];
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s]+/g, " ")
+    .split(/\s+/)
+    .filter((w) => w.length > 1 && !STOP_WORDS.has(w));
+}
+
+function scoreTarget(promptTokens, targetString) {
+  const targetTokens = tokenize(targetString);
+  if (targetTokens.length === 0) return { coverage: 0, matchCount: 0, targetLen: 0 };
+
+  let matchCount = 0;
+  let hasHighValueWord = false;
+
+  for (const tWord of targetTokens) {
+    const isMatch = promptTokens.some(
+      (pWord) => pWord === tWord || (tWord.length >= 3 && pWord.startsWith(tWord)),
+    );
+    if (isMatch) {
+      matchCount++;
+      if (tWord.length >= 6) hasHighValueWord = true;
+    }
+  }
+
+  const coverage = matchCount / targetTokens.length;
+  const isReliable = coverage >= 0.75 && (matchCount >= 2 || hasHighValueWord);
+  if (!isReliable) return { coverage: 0, matchCount: 0, targetLen: 0 };
+
+  return { coverage, matchCount, targetLen: targetTokens.length };
+}
+
+/** Coverage of target tokens present in the prompt. 0 if below the reliability floor. */
+export function calculateCoverage(promptTokens, targetString) {
+  return scoreTarget(promptTokens, targetString).coverage;
+}
+
+function isBetter(a, b) {
+  if (a.coverage !== b.coverage) return a.coverage > b.coverage;
+  if (a.matchCount !== b.matchCount) return a.matchCount > b.matchCount;
+  return a.targetLen > b.targetLen;
+}
+
+function scoreProcedure(promptTokens, node) {
+  let best = scoreTarget(promptTokens, node.trigger || "");
+  if (Array.isArray(node.aliases)) {
+    for (const alias of node.aliases) {
+      const next = scoreTarget(promptTokens, alias);
+      if (isBetter(next, best)) best = next;
+    }
+  }
+  return best;
+}
+
+function formatSteps(steps, requiresEnv = []) {
+  const required =
+    requiresEnv.length > 0 ? `Requires environment: ${requiresEnv.join(", ")}\n\n` : "";
+  return required + steps.map((s) => `- ${s.t}: ${s.target}`).join("\n");
+}
+
+/**
+ * Best-scoring root procedure for this prompt, or null.
+ * Returns { node, text, notice } — text for the agent, notice for the harness UI.
+ */
+export function recall(prompt, procs) {
+  if (!prompt) return null;
+
+  const promptTokens = tokenize(prompt);
+  let best = null;
+  let bestScore = { coverage: 0, matchCount: 0, targetLen: 0 };
+
+  for (const node of procs) {
+    if (node.enabled === false || node.parent_id) continue;
+    const score = scoreProcedure(promptTokens, node);
+    if (isBetter(score, bestScore)) {
+      bestScore = score;
+      best = node;
+    }
+  }
+
+  if (!best) return null;
+
+  let text = formatSteps(best.steps, best.requires_env);
+  const alt = procs.find((e) => e.parent_id === best.id && e.type === "alt" && e.enabled !== false);
+  if (alt) text += "\n\nAlternative path that worked:\n" + formatSteps(alt.steps, alt.requires_env);
+
+  const label = best.trigger || "procedure";
+  return {
+    node: best,
+    text: `Reflex memory found a procedure for this task:\n\n${text}\n\nFollow these steps instead of figuring it out from scratch.`,
+    notice: `Reflex recalled: "${label}"`,
+  };
+}