forge-workflow 0.1.0-beta.3 → 0.1.0-beta.5

package/lib/commands/hooks.js

@@ -27,16 +27,151 @@ const {
   renderGlobalHookBlock,
   installGlobalHooks,
 } = require('../hook-global-installer');
+const fs = require('node:fs');
+const path = require('node:path');
 const { sessionStartCapability, userPromptSubmitCapability, sessionEndCapability } = require('../hook-renderer');
-const { collectDigestData, buildMemoryDigest, defaultFetchIssues, defaultFetchNotes } = require('../memory-digest');
+const { buildMemoryDigest, defaultFetchIssues, defaultFetchNotes } = require('../memory-digest');
 const { collectInbox, buildInboxNudge } = require('../inbox');
 const { collectDigest } = require('../pr-monitor/digest');
+const { loadDispatchText } = require('../using-forge');
+const projectMemory = require('../project-memory');
+const { parseHookInput, selectInjection, meaningfulTokens, DEFAULT_TOKEN_BUDGET, DEFAULT_SCORE_FLOOR } = require('../memory-recall');
+const { launchMemoryRecallEvent } = require('../memory-recall-events');
+const { getResolvedRuntimeGraph } = require('../core/runtime-graph');
+const { fireAndForget } = require('../pr-monitor/reconcile-executor');
+
+// Default-ON rail; `forge gate disable rail.memory_recall` turns tier-2 off. Mirrors
+// autoShepherdRailEnabled (lib/commands/ship.js): absent id = enabled, fail-open to true.
+const MEMORY_RECALL_RAIL = 'rail.memory_recall';
+// bm25 candidate pool to rank/floor down from. Wider than what we inject so the floor +
+// dedupe have room (mirrors the research "widen before ranking" guidance).
+const MEMORY_RECALL_CANDIDATES = 25;
+// Cross-turn dedupe memory: how many recently-injected keys to remember per session.
+const SEEN_KEYS_CAP = 40;
+const SESSION_START_DEADLINE_MS = 9_000;
+const PROMPT_RECALL_DEADLINE_MS = 4_500;
+const PROMPT_RECALL_SQLITE_BUSY_MS = 1_000;
+
+function withinDeadline(work, fallback, deadlineMs, onTimeout) {
+  return new Promise((resolve) => {
+    const timer = setTimeout(() => {
+      if (onTimeout) onTimeout();
+      resolve(fallback);
+    }, deadlineMs);
+    Promise.resolve().then(work).then(
+      value => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      () => {
+        clearTimeout(timer);
+        resolve(fallback);
+      },
+    );
+  });
+}
+
+async function collectSessionStartData(projectRoot, opts, deadlineMs) {
+  const fetchNotes = opts.fetchNotes || defaultFetchNotes;
+  const fetchIssues = opts.fetchIssues || defaultFetchIssues;
+  const fetchInbox = opts.fetchInbox || collectInbox;
+  const bounded = work => withinDeadline(work, [], deadlineMs);
+  const [notes, ready, claimed, inbox] = await Promise.all([
+    bounded(() => fetchNotes(projectRoot, opts)),
+    bounded(() => fetchIssues(projectRoot, 'ready', opts)),
+    bounded(() => fetchIssues(projectRoot, 'in_progress', opts)),
+    bounded(() => fetchInbox(projectRoot, opts)),
+  ]);
+  return { notes, ready, claimed, inbox };
+}
+
+function memoryRecallRailEnabled(projectRoot, resolveGraph = getResolvedRuntimeGraph) {
+  try {
+    const graph = resolveGraph({ projectRoot });
+    const rail = [...(graph.rails || []), ...(graph.gates || [])].find(entry => entry.id === MEMORY_RECALL_RAIL);
+    return !(rail?.enabled === false);
+  } catch {
+    return true;
+  }
+}
+
+// Read the hook's stdin payload (Claude delivers UserPromptSubmit JSON on fd 0). Never
+// throws — no stdin / a closed fd yields '' so the hook fails open.
+function readHookStdin() {
+  try {
+    return fs.readFileSync(0, 'utf8');
+  } catch {
+    return '';
+  }
+}
+
+function seenPath(projectRoot, sessionId) {
+  const safe = String(sessionId || 'nosession').replace(/[^a-zA-Z0-9_-]/g, '_');
+  return path.join(projectRoot, '.forge', 'memory-recall', `${safe}.json`);
+}
+
+// Keys injected on recent turns of THIS session (cross-turn dedupe). Best-effort: any read
+// error yields [] so a first turn or a corrupt file simply injects without exclusion.
+function loadSeenKeys(projectRoot, sessionId) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(seenPath(projectRoot, sessionId), 'utf8'));
+    return Array.isArray(parsed) ? parsed.filter(k => typeof k === 'string') : [];
+  } catch {
+    return [];
+  }
+}
+
+// Append the just-injected keys to the session's seen-list (newest last, capped). Best-effort.
+function saveSeenKeys(projectRoot, sessionId, keys) {
+  try {
+    const merged = [...loadSeenKeys(projectRoot, sessionId), ...keys].slice(-SEEN_KEYS_CAP);
+    const file = seenPath(projectRoot, sessionId);
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.writeFileSync(file, JSON.stringify(merged), 'utf8');
+  } catch {
+    // Dedupe is a nicety, not a correctness gate — a failed write never breaks the prompt.
+  }
+}
+
+// Shadow-log the tier-2 recall decision (kernel issue f71784d3 step-0 instrument): one JSON
+// line per run records what was retrieved and what cleared the floor, so the corpus-dependent
+// scoreFloor can be tuned from real data instead of guessed. Capped so it never grows unbounded.
+const SHADOW_LOG_MAX_BYTES = 512 * 1024;
+// A shadow record is a tuning sample, not an archive. It contains only aggregate counts/scores.
+
+function shadowLogPath(projectRoot) {
+  return path.join(projectRoot, '.forge', 'memory-recall', 'shadow.jsonl');
+}
+
+// Append one JSON line, then evict whole records oldest-first until the file fits the byte cap.
+// Trimming by bytes (not line count) is what keeps this hook's synchronous read/write bounded: a
+// record that exceeds the cap alone leaves the file empty rather than permanently oversized.
+// Best-effort — any failure is swallowed by the caller so logging never affects the hook result.
+function appendShadowLog(projectRoot, record) {
+  const file = shadowLogPath(projectRoot);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.appendFileSync(file, `${JSON.stringify(record)}\n`, 'utf8');
+  let size;
+  try { size = fs.statSync(file).size; } catch { size = 0; }
+  if (size <= SHADOW_LOG_MAX_BYTES) return;
+  const lines = fs.readFileSync(file, 'utf8').split('\n').filter(Boolean);
+  const keep = [];
+  let bytes = 0;
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const cost = Buffer.byteLength(`${lines[i]}\n`, 'utf8');
+    if (bytes + cost > SHADOW_LOG_MAX_BYTES) break;
+    bytes += cost;
+    keep.unshift(lines[i]);
+  }
+  fs.writeFileSync(file, keep.length ? `${keep.join('\n')}\n` : '', 'utf8');
+}
 
 function usage() {
   return 'Usage: forge hooks install --global [--harness codex|hermes|all] [--dry-run]\n'
     + '       forge hooks session-start --harness <claude> (machine-facing; emits SessionStart context)\n'
     + '       forge hooks inbox-pickup --harness <claude> (machine-facing; emits UserPromptSubmit context)\n'
     + '       forge hooks shepherd-events --harness <claude> (machine-facing; emits UserPromptSubmit PR-monitor deltas)\n'
+    + '       forge hooks memory-recall --harness <claude> (machine-facing; emits UserPromptSubmit query-relevant memory)\n'
     + '       forge hooks capture --harness <claude> --trigger <precompact|stop> (machine-facing; captures a session summary on exit)';
 }
 
@@ -111,21 +246,46 @@ function formatSessionStart(harness, text) {
  *
  * @param {string[]} rest - args after the `session-start` action.
  * @param {string} projectRoot
- * @param {object} [opts] - injectable digest fetchers ({ fetchNotes, fetchIssues }).
+ * @param {object} [opts] - injectable digest fetchers ({ fetchNotes, fetchIssues }) and an
+ *   optional host `harness` capability ({ hasBgShell: true, runBgShell(argv, { cwd }) }).
  * @returns {Promise<{ success: boolean, output: string }>}
  */
 async function handleSessionStart(rest, projectRoot, opts = {}) {
+  const harness = parseHarness(rest);
+  const capability = sessionStartCapability(harness);
+  if (!capability.rendered) return { success: true, output: '', reason: capability.reason };
   try {
-    const harness = parseHarness(rest);
-    if (!sessionStartCapability(harness).rendered) return { success: true, output: '' };
-    const data = await collectDigestData(projectRoot, opts);
+    const triggerContext = { projectRoot };
+    if (opts.harness !== undefined) triggerContext.harness = opts.harness;
+    (opts.fireAndForget || fireAndForget)(triggerContext);
+  } catch { /* the automatic trigger must never break session start */ }
+
+  // The using-forge dispatch bootstrap is injected FIRST so Forge skills auto-trigger from turn
+  // one (the Superpowers mechanism): a reasoning-driven system, not just harness description
+  // matching. It is a deterministic file read that survives a kernel outage. The memory digest
+  // (remembered notes + top open issues) is appended when present. Either alone is enough to
+  // inject; a total blank yields '' (the harness injects nothing). FAIL-OPEN throughout.
+  let dispatch = '';
+  try {
+    // No projectRoot: the dispatch skill is read from the Forge PACKAGE's canonical skills/
+    // (a set-up consumer project has no root skills/, only generated mirrors).
+    dispatch = (opts.loadDispatchText || loadDispatchText)() || '';
+  } catch { /* keep '' — a missing dispatch skill must not break session start */ }
+
+  let digestText = '';
+  try {
+    const data = await collectSessionStartData(
+      projectRoot,
+      opts,
+      opts.sessionStartDeadlineMs || SESSION_START_DEADLINE_MS,
+    );
     const digest = buildMemoryDigest(data, opts);
-    if (digest.empty) return { success: true, output: '' };
-    return { success: true, output: formatSessionStart(harness, digest.text) };
-  } catch {
-    // Fail-open: a context hook must never break a session.
-    return { success: true, output: '' };
-  }
+    if (!digest.empty) digestText = digest.text;
+  } catch { /* fail-open: a kernel outage must not suppress the dispatch bootstrap */ }
+
+  const combined = [dispatch, digestText].filter(Boolean).join('\n\n');
+  if (!combined) return { success: true, output: '' };
+  return { success: true, output: formatSessionStart(harness, combined) };
 }
 
 /** Wrap a digest into a harness-native UserPromptSubmit payload, or '' when unsupported. */
@@ -160,7 +320,8 @@ function formatUserPromptSubmit(harness, text) {
 async function handleInboxPickup(rest, projectRoot, opts = {}) {
   try {
     const harness = parseHarness(rest);
-    if (!userPromptSubmitCapability(harness).rendered) return { success: true, output: '' };
+    const capability = userPromptSubmitCapability(harness);
+    if (!capability.rendered) return { success: true, output: '', reason: capability.reason };
     const pending = await collectInbox(projectRoot, opts);
     const nudge = buildInboxNudge(pending);
     if (nudge.empty) return { success: true, output: '' };
@@ -205,6 +366,187 @@ function handleShepherdEvents(rest, projectRoot, opts = {}) {
   }
 }
 
+/**
+ * `forge hooks memory-recall --harness <h>` — the QUERY-RELEVANT memory tier (tier-2). On each
+ * prompt it reads the submitted prompt from the hook's stdin, ranks stored memories by BM25
+ * relevance to it (NOT recency — that is the SessionStart digest's job), applies a relevance
+ * floor + anaphora guard + cross-turn dedupe + a hard token budget, and emits the survivors as
+ * harness-native UserPromptSubmit context. Complements the always-on recency digest: this one
+ * answers "what memory is relevant to THIS turn".
+ *
+ * Compliance + safety (verified against the Claude Code hooks contract, kernel issue 781f6f65):
+ *   - It READS its own hook stdin (the supported input channel) — it does NOT inject into a
+ *     running session's stdin and never drives the agent (Anthropic Usage Policy).
+ *   - additionalContext APPENDS to history every prompt, so the injection is tiny and gated:
+ *     below the relevance floor, or on a low-signal (anaphora) prompt, it injects NOTHING.
+ *   - Injected memory bodies are UNTRUSTED (a planted note could carry directives), so each is
+ *     provenance-fenced.
+ *   - UserPromptSubmit has a 30s timeout and blocks the prompt; the read is local BM25 over
+ *     SQLite (sub-ms) and FAIL-OPEN — any error, a disabled rail, an unsupported harness, or
+ *     nothing relevant yields '' and the prompt proceeds untouched. NEVER throws.
+ *   - Kill-switch: `forge gate disable rail.memory_recall`.
+ *
+ * @param {string[]} rest - args after the `memory-recall` action.
+ * @param {string} projectRoot
+ * @param {object} [opts] - injectable seams for tests: { railEnabled, readInput, search,
+ *   loadSeen, saveSeen, scoreFloor, tokenBudget }.
+ * @returns {{ success: boolean, output: string }}
+ */
+async function runMemoryRecall(rest, projectRoot, opts = {}) {
+  const observation = opts.observation || {};
+  try {
+    const harness = parseHarness(rest);
+    observation.harness = harness;
+    const capability = userPromptSubmitCapability(harness);
+    if (!capability.rendered) {
+      observation.outcome = 'unsupported';
+      return { success: true, output: '', reason: capability.reason };
+    }
+
+    const railEnabled = opts.railEnabled || memoryRecallRailEnabled;
+    if (!railEnabled(projectRoot)) {
+      observation.outcome = 'filtered';
+      return { success: true, output: '' };
+    }
+
+    const readInput = opts.readInput || readHookStdin;
+    const { prompt, sessionId } = parseHookInput(readInput());
+    if (!prompt) {
+      observation.outcome = 'filtered';
+      return { success: true, output: '' };
+    }
+
+    const search = opts.search
+      || ((root, query, limit, options) => projectMemory.searchRankedScored(root, query, limit, options));
+    const loadSeen = opts.loadSeen || loadSeenKeys;
+    const saveSeen = opts.saveSeen || saveSeenKeys;
+    const appendShadow = opts.appendShadow || appendShadowLog;
+
+    // Stopword-aware, unicode-safe tokenization lives HERE (single source); the driver's
+    // buildMemoryFtsMatchOr only quotes+ORs whatever tokens it receives. Passing the raw prompt
+    // to a token-AND match was the 0-recall bug (kernel issue f71784d3).
+    const tokens = meaningfulTokens(prompt);
+    const excludeKeys = await loadSeen(projectRoot, sessionId) || [];
+    const hits = await search(
+      projectRoot,
+      tokens.join(' '),
+      MEMORY_RECALL_CANDIDATES,
+      {
+        excludeKeys: excludeKeys.slice(0, 256),
+        busyTimeoutMs: Math.max(0, Math.min(
+          opts.promptRecallSqliteBusyMs ?? PROMPT_RECALL_SQLITE_BUSY_MS,
+          (opts.promptRecallDeadlineMs || PROMPT_RECALL_DEADLINE_MS) - 100,
+        )),
+      },
+    ) || [];
+    observation.candidateCount = hits.length;
+    observation.eligibleCount = hits.length;
+    if (opts.deadline?.expired) {
+      observation.outcome = 'timeout';
+      return { success: true, output: '', reason: 'timeout' };
+    }
+    const scoreFloor = typeof opts.scoreFloor === 'number' ? opts.scoreFloor : DEFAULT_SCORE_FLOOR;
+    const { lines, entries, injectedKeys } = selectInjection({
+      query: prompt,
+      hits,
+      scoreFloor,
+      tokenBudget: opts.tokenBudget || DEFAULT_TOKEN_BUDGET,
+      excludeKeys,
+    });
+
+    // Step-0 instrument: record every real query (even ones that inject nothing) so the floor is
+    // tuned from data. Own try/catch — a logging failure must NEVER fall through to the fail-open
+    // outer catch (which would suppress a legitimate injection).
+    if (tokens.length) {
+      try {
+        appendShadow(projectRoot, {
+          candidateCount: hits.length,
+          injectedCount: injectedKeys.length,
+          candidateScoreMin: hits.length ? Math.min(...hits.map(hit => Number(hit?.score) || 0)) : null,
+          candidateScoreMax: hits.length ? Math.max(...hits.map(hit => Number(hit?.score) || 0)) : null,
+          scoreFloor,
+        });
+      } catch { /* best-effort: shadow logging never affects the hook result */ }
+    }
+
+    if (!lines.length) {
+      observation.outcome = hits.length ? 'filtered' : 'empty';
+      return { success: true, output: '' };
+    }
+
+    if (opts.deadline?.expired) return { success: true, output: '', reason: 'timeout' };
+    try {
+      const pending = saveSeen(projectRoot, sessionId, injectedKeys);
+      if (pending && typeof pending.catch === 'function') pending.catch(() => {});
+    } catch { /* dedupe persistence never suppresses an injection */ }
+    const sections = [
+      ['confirmed', 'Confirmed memory (project-local; provenance shown)'],
+      ['suggested', 'Suggested memory — verify before relying'],
+    ].map(([trust, heading]) => {
+      const selectedLines = entries.filter(entry => entry.trust === trust).map(entry => entry.line);
+      return selectedLines.length ? `${heading}\n${selectedLines.join('\n')}` : '';
+    }).filter(Boolean);
+    const fenced = sections.join('\n');
+    const selected = hits.filter(hit => injectedKeys.includes(hit.memory_id || hit.key));
+    observation.outcome = 'selected';
+    observation.selectedIds = injectedKeys;
+    observation.sourceMix = selected.reduce((mix, hit) => {
+      const sourceAgent = hit.provenance?.source_agent || hit.sourceAgent || 'unknown';
+      mix[sourceAgent] = (mix[sourceAgent] || 0) + 1;
+      return mix;
+    }, {});
+    observation.trustMix = selected.reduce((mix, hit) => {
+      const trust = hit.trust_status || 'unknown';
+      mix[trust] = (mix[trust] || 0) + 1;
+      return mix;
+    }, {});
+    observation.tokenEstimate = Math.ceil(fenced.length / 4);
+    return { success: true, output: formatUserPromptSubmit(harness, fenced) };
+  } catch {
+    // Fail-open: a context hook must never break a prompt.
+    observation.outcome = 'error';
+    return { success: true, output: '' };
+  }
+}
+
+async function handleMemoryRecall(rest, projectRoot, opts = {}) {
+  const harness = parseHarness(rest);
+  const startedAt = Date.now();
+  const observation = { harness };
+  const record = result => {
+    const outcome = result.reason === 'timeout'
+      ? 'timeout'
+      : (observation.outcome || (result.output ? 'selected' : 'empty'));
+    const eventObservation = {
+      ...observation,
+      outcome,
+      harness,
+      elapsedMs: Date.now() - startedAt,
+    };
+    try {
+      const pending = (opts.recordRecallEvent || opts.launchRecallEvent || launchMemoryRecallEvent)(
+        projectRoot,
+        eventObservation,
+      );
+      if (pending && typeof pending.catch === 'function') pending.catch(() => {});
+    } catch { /* Operational evidence is best-effort and must never delay or suppress a prompt. */ }
+    return result;
+  };
+  const capability = userPromptSubmitCapability(harness);
+  if (!capability.rendered) {
+    observation.outcome = 'unsupported';
+    return record({ success: true, output: '', reason: capability.reason });
+  }
+  const deadline = { expired: false };
+  const result = await withinDeadline(
+    () => runMemoryRecall(rest, projectRoot, { ...opts, deadline, observation }),
+    { success: true, output: '', reason: 'timeout' },
+    opts.promptRecallDeadlineMs || PROMPT_RECALL_DEADLINE_MS,
+    () => { deadline.expired = true; },
+  );
+  return record(result);
+}
+
 /**
  * `forge hooks capture --harness <h> --trigger <precompact|stop>` — the CAPTURE-on-exit hook.
  * PreCompact (before context compaction) and Stop (turn end) fire it; it snapshots a bounded
@@ -228,7 +570,8 @@ function handleShepherdEvents(rest, projectRoot, opts = {}) {
 async function handleCapture(rest, projectRoot, opts = {}) {
   try {
     const harness = parseHarness(rest);
-    if (!sessionEndCapability(harness).rendered) return { success: true, output: '' };
+    const capability = sessionEndCapability(harness);
+    if (!capability.rendered) return { success: true, output: '', reason: capability.reason };
     const trigger = parseTrigger(rest);
 
     const fetchIssues = opts.fetchIssues || defaultFetchIssues;
@@ -356,6 +699,7 @@ async function handler(args, flags = {}, projectRoot, opts = {}) {
   if (action === 'session-start') return handleSessionStart(args.slice(1), projectRoot, opts);
   if (action === 'inbox-pickup') return handleInboxPickup(args.slice(1), projectRoot, opts);
   if (action === 'shepherd-events') return handleShepherdEvents(args.slice(1), projectRoot, opts);
+  if (action === 'memory-recall') return handleMemoryRecall(args.slice(1), projectRoot, opts);
   if (action === 'capture') return handleCapture(args.slice(1), projectRoot, opts);
   if (action === 'install') return handleInstall(args, flags, opts);
   return {
@@ -374,4 +718,5 @@ module.exports = {
     '--dry-run': 'Preview the merge without writing anything',
   },
   handler,
+  _internal: { appendShadowLog, SHADOW_LOG_MAX_BYTES },
 };

package/lib/commands/insights.js

@@ -36,7 +36,7 @@ function positionals(args) {
   return values;
 }
 
-async function handler(args, flags, projectRoot) {
+async function handler(args, flags, projectRoot, opts = {}) {
   const commandFlags = flags ?? {};
   const [subcommand, candidateId] = positionals(args);
   if (subcommand === 'accept' || subcommand === 'reject') {
@@ -54,14 +54,19 @@ async function handler(args, flags, projectRoot) {
     };
   }
 
-  const result = analyzeInsights(projectRoot, {
+  const result = await analyzeInsights(projectRoot, {
     limit: readOption(args, '--limit', undefined),
     minCount: readOption(args, '--min-count', undefined),
     since: readOption(args, '--since', undefined),
+    // Injectable kernel-read seams (Slice C2); undefined falls back to the real
+    // cli-broker-factory-backed reads inside analyzeInsights.
+    runIssueOperation: opts.runIssueOperation,
+    listRecentEvents: opts.listRecentEvents,
+    env: opts.env,
   });
   if (args.includes('--review-feedback')) {
     result.limitations = [
-      'Compatibility note: --review-feedback now reads Beads interactions and issue evidence; external review-provider comments are not inferred.',
+      'Compatibility note: --review-feedback now reads kernel events and issue evidence; external review-provider comments are not inferred.',
       ...result.limitations,
     ];
   }