forge-workflow 0.1.0-beta.4 → 0.1.0-beta.5

package/lib/commands/hooks.js

@@ -30,14 +30,15 @@ const {
 const fs = require('node:fs');
 const path = require('node:path');
 const { sessionStartCapability, userPromptSubmitCapability, sessionEndCapability } = require('../hook-renderer');
-const { collectDigestData, buildMemoryDigest, defaultFetchIssues, defaultFetchNotes } = require('../memory-digest');
+const { buildMemoryDigest, defaultFetchIssues, defaultFetchNotes } = require('../memory-digest');
 const { collectInbox, buildInboxNudge } = require('../inbox');
 const { collectDigest } = require('../pr-monitor/digest');
 const { loadDispatchText } = require('../using-forge');
 const projectMemory = require('../project-memory');
-const { parseHookInput, selectInjection, DEFAULT_TOKEN_BUDGET, DEFAULT_SCORE_FLOOR } = require('../memory-recall');
-const { fenceUntrusted } = require('../untrusted-content');
+const { parseHookInput, selectInjection, meaningfulTokens, DEFAULT_TOKEN_BUDGET, DEFAULT_SCORE_FLOOR } = require('../memory-recall');
+const { launchMemoryRecallEvent } = require('../memory-recall-events');
 const { getResolvedRuntimeGraph } = require('../core/runtime-graph');
+const { fireAndForget } = require('../pr-monitor/reconcile-executor');
 
 // Default-ON rail; `forge gate disable rail.memory_recall` turns tier-2 off. Mirrors
 // autoShepherdRailEnabled (lib/commands/ship.js): absent id = enabled, fail-open to true.
@@ -47,6 +48,42 @@ const MEMORY_RECALL_RAIL = 'rail.memory_recall';
 const MEMORY_RECALL_CANDIDATES = 25;
 // Cross-turn dedupe memory: how many recently-injected keys to remember per session.
 const SEEN_KEYS_CAP = 40;
+const SESSION_START_DEADLINE_MS = 9_000;
+const PROMPT_RECALL_DEADLINE_MS = 4_500;
+const PROMPT_RECALL_SQLITE_BUSY_MS = 1_000;
+
+function withinDeadline(work, fallback, deadlineMs, onTimeout) {
+  return new Promise((resolve) => {
+    const timer = setTimeout(() => {
+      if (onTimeout) onTimeout();
+      resolve(fallback);
+    }, deadlineMs);
+    Promise.resolve().then(work).then(
+      value => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      () => {
+        clearTimeout(timer);
+        resolve(fallback);
+      },
+    );
+  });
+}
+
+async function collectSessionStartData(projectRoot, opts, deadlineMs) {
+  const fetchNotes = opts.fetchNotes || defaultFetchNotes;
+  const fetchIssues = opts.fetchIssues || defaultFetchIssues;
+  const fetchInbox = opts.fetchInbox || collectInbox;
+  const bounded = work => withinDeadline(work, [], deadlineMs);
+  const [notes, ready, claimed, inbox] = await Promise.all([
+    bounded(() => fetchNotes(projectRoot, opts)),
+    bounded(() => fetchIssues(projectRoot, 'ready', opts)),
+    bounded(() => fetchIssues(projectRoot, 'in_progress', opts)),
+    bounded(() => fetchInbox(projectRoot, opts)),
+  ]);
+  return { notes, ready, claimed, inbox };
+}
 
 function memoryRecallRailEnabled(projectRoot, resolveGraph = getResolvedRuntimeGraph) {
   try {
@@ -96,6 +133,39 @@ function saveSeenKeys(projectRoot, sessionId, keys) {
   }
 }
 
+// Shadow-log the tier-2 recall decision (kernel issue f71784d3 step-0 instrument): one JSON
+// line per run records what was retrieved and what cleared the floor, so the corpus-dependent
+// scoreFloor can be tuned from real data instead of guessed. Capped so it never grows unbounded.
+const SHADOW_LOG_MAX_BYTES = 512 * 1024;
+// A shadow record is a tuning sample, not an archive. It contains only aggregate counts/scores.
+
+function shadowLogPath(projectRoot) {
+  return path.join(projectRoot, '.forge', 'memory-recall', 'shadow.jsonl');
+}
+
+// Append one JSON line, then evict whole records oldest-first until the file fits the byte cap.
+// Trimming by bytes (not line count) is what keeps this hook's synchronous read/write bounded: a
+// record that exceeds the cap alone leaves the file empty rather than permanently oversized.
+// Best-effort — any failure is swallowed by the caller so logging never affects the hook result.
+function appendShadowLog(projectRoot, record) {
+  const file = shadowLogPath(projectRoot);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.appendFileSync(file, `${JSON.stringify(record)}\n`, 'utf8');
+  let size;
+  try { size = fs.statSync(file).size; } catch { size = 0; }
+  if (size <= SHADOW_LOG_MAX_BYTES) return;
+  const lines = fs.readFileSync(file, 'utf8').split('\n').filter(Boolean);
+  const keep = [];
+  let bytes = 0;
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const cost = Buffer.byteLength(`${lines[i]}\n`, 'utf8');
+    if (bytes + cost > SHADOW_LOG_MAX_BYTES) break;
+    bytes += cost;
+    keep.unshift(lines[i]);
+  }
+  fs.writeFileSync(file, keep.length ? `${keep.join('\n')}\n` : '', 'utf8');
+}
+
 function usage() {
   return 'Usage: forge hooks install --global [--harness codex|hermes|all] [--dry-run]\n'
     + '       forge hooks session-start --harness <claude> (machine-facing; emits SessionStart context)\n'
@@ -176,16 +246,19 @@ function formatSessionStart(harness, text) {
  *
  * @param {string[]} rest - args after the `session-start` action.
  * @param {string} projectRoot
- * @param {object} [opts] - injectable digest fetchers ({ fetchNotes, fetchIssues }).
+ * @param {object} [opts] - injectable digest fetchers ({ fetchNotes, fetchIssues }) and an
+ *   optional host `harness` capability ({ hasBgShell: true, runBgShell(argv, { cwd }) }).
  * @returns {Promise<{ success: boolean, output: string }>}
  */
 async function handleSessionStart(rest, projectRoot, opts = {}) {
   const harness = parseHarness(rest);
-  if (!sessionStartCapability(harness).rendered) return { success: true, output: '' };
-
-  // NOTE: the autonomous-shepherd session-start trigger is deferred to W-S4c along with the
-  // per-command dispatch trigger — both need a firing-policy + containment design (a naive
-  // trigger spawns a session-outliving daemon that breaks test isolation).
+  const capability = sessionStartCapability(harness);
+  if (!capability.rendered) return { success: true, output: '', reason: capability.reason };
+  try {
+    const triggerContext = { projectRoot };
+    if (opts.harness !== undefined) triggerContext.harness = opts.harness;
+    (opts.fireAndForget || fireAndForget)(triggerContext);
+  } catch { /* the automatic trigger must never break session start */ }
 
   // The using-forge dispatch bootstrap is injected FIRST so Forge skills auto-trigger from turn
   // one (the Superpowers mechanism): a reasoning-driven system, not just harness description
@@ -201,7 +274,11 @@ async function handleSessionStart(rest, projectRoot, opts = {}) {
 
   let digestText = '';
   try {
-    const data = await collectDigestData(projectRoot, opts);
+    const data = await collectSessionStartData(
+      projectRoot,
+      opts,
+      opts.sessionStartDeadlineMs || SESSION_START_DEADLINE_MS,
+    );
     const digest = buildMemoryDigest(data, opts);
     if (!digest.empty) digestText = digest.text;
   } catch { /* fail-open: a kernel outage must not suppress the dispatch bootstrap */ }
@@ -243,7 +320,8 @@ function formatUserPromptSubmit(harness, text) {
 async function handleInboxPickup(rest, projectRoot, opts = {}) {
   try {
     const harness = parseHarness(rest);
-    if (!userPromptSubmitCapability(harness).rendered) return { success: true, output: '' };
+    const capability = userPromptSubmitCapability(harness);
+    if (!capability.rendered) return { success: true, output: '', reason: capability.reason };
     const pending = await collectInbox(projectRoot, opts);
     const nudge = buildInboxNudge(pending);
     if (nudge.empty) return { success: true, output: '' };
@@ -314,43 +392,161 @@ function handleShepherdEvents(rest, projectRoot, opts = {}) {
  *   loadSeen, saveSeen, scoreFloor, tokenBudget }.
  * @returns {{ success: boolean, output: string }}
  */
-function handleMemoryRecall(rest, projectRoot, opts = {}) {
+async function runMemoryRecall(rest, projectRoot, opts = {}) {
+  const observation = opts.observation || {};
   try {
     const harness = parseHarness(rest);
-    if (!userPromptSubmitCapability(harness).rendered) return { success: true, output: '' };
+    observation.harness = harness;
+    const capability = userPromptSubmitCapability(harness);
+    if (!capability.rendered) {
+      observation.outcome = 'unsupported';
+      return { success: true, output: '', reason: capability.reason };
+    }
 
     const railEnabled = opts.railEnabled || memoryRecallRailEnabled;
-    if (!railEnabled(projectRoot)) return { success: true, output: '' };
+    if (!railEnabled(projectRoot)) {
+      observation.outcome = 'filtered';
+      return { success: true, output: '' };
+    }
 
     const readInput = opts.readInput || readHookStdin;
     const { prompt, sessionId } = parseHookInput(readInput());
-    if (!prompt) return { success: true, output: '' };
+    if (!prompt) {
+      observation.outcome = 'filtered';
+      return { success: true, output: '' };
+    }
 
     const search = opts.search
-      || ((root, query, limit) => projectMemory.searchRankedScored(root, query, limit));
+      || ((root, query, limit, options) => projectMemory.searchRankedScored(root, query, limit, options));
     const loadSeen = opts.loadSeen || loadSeenKeys;
     const saveSeen = opts.saveSeen || saveSeenKeys;
-
-    const hits = search(projectRoot, prompt, MEMORY_RECALL_CANDIDATES) || [];
-    const excludeKeys = loadSeen(projectRoot, sessionId) || [];
-    const { lines, injectedKeys } = selectInjection({
+    const appendShadow = opts.appendShadow || appendShadowLog;
+
+    // Stopword-aware, unicode-safe tokenization lives HERE (single source); the driver's
+    // buildMemoryFtsMatchOr only quotes+ORs whatever tokens it receives. Passing the raw prompt
+    // to a token-AND match was the 0-recall bug (kernel issue f71784d3).
+    const tokens = meaningfulTokens(prompt);
+    const excludeKeys = await loadSeen(projectRoot, sessionId) || [];
+    const hits = await search(
+      projectRoot,
+      tokens.join(' '),
+      MEMORY_RECALL_CANDIDATES,
+      {
+        excludeKeys: excludeKeys.slice(0, 256),
+        busyTimeoutMs: Math.max(0, Math.min(
+          opts.promptRecallSqliteBusyMs ?? PROMPT_RECALL_SQLITE_BUSY_MS,
+          (opts.promptRecallDeadlineMs || PROMPT_RECALL_DEADLINE_MS) - 100,
+        )),
+      },
+    ) || [];
+    observation.candidateCount = hits.length;
+    observation.eligibleCount = hits.length;
+    if (opts.deadline?.expired) {
+      observation.outcome = 'timeout';
+      return { success: true, output: '', reason: 'timeout' };
+    }
+    const scoreFloor = typeof opts.scoreFloor === 'number' ? opts.scoreFloor : DEFAULT_SCORE_FLOOR;
+    const { lines, entries, injectedKeys } = selectInjection({
       query: prompt,
       hits,
-      scoreFloor: typeof opts.scoreFloor === 'number' ? opts.scoreFloor : DEFAULT_SCORE_FLOOR,
+      scoreFloor,
       tokenBudget: opts.tokenBudget || DEFAULT_TOKEN_BUDGET,
       excludeKeys,
     });
-    if (!lines.length) return { success: true, output: '' };
 
-    saveSeen(projectRoot, sessionId, injectedKeys);
-    const fenced = lines.map(line => fenceUntrusted(line, { source: 'memory' })).join('\n');
+    // Step-0 instrument: record every real query (even ones that inject nothing) so the floor is
+    // tuned from data. Own try/catch — a logging failure must NEVER fall through to the fail-open
+    // outer catch (which would suppress a legitimate injection).
+    if (tokens.length) {
+      try {
+        appendShadow(projectRoot, {
+          candidateCount: hits.length,
+          injectedCount: injectedKeys.length,
+          candidateScoreMin: hits.length ? Math.min(...hits.map(hit => Number(hit?.score) || 0)) : null,
+          candidateScoreMax: hits.length ? Math.max(...hits.map(hit => Number(hit?.score) || 0)) : null,
+          scoreFloor,
+        });
+      } catch { /* best-effort: shadow logging never affects the hook result */ }
+    }
+
+    if (!lines.length) {
+      observation.outcome = hits.length ? 'filtered' : 'empty';
+      return { success: true, output: '' };
+    }
+
+    if (opts.deadline?.expired) return { success: true, output: '', reason: 'timeout' };
+    try {
+      const pending = saveSeen(projectRoot, sessionId, injectedKeys);
+      if (pending && typeof pending.catch === 'function') pending.catch(() => {});
+    } catch { /* dedupe persistence never suppresses an injection */ }
+    const sections = [
+      ['confirmed', 'Confirmed memory (project-local; provenance shown)'],
+      ['suggested', 'Suggested memory — verify before relying'],
+    ].map(([trust, heading]) => {
+      const selectedLines = entries.filter(entry => entry.trust === trust).map(entry => entry.line);
+      return selectedLines.length ? `${heading}\n${selectedLines.join('\n')}` : '';
+    }).filter(Boolean);
+    const fenced = sections.join('\n');
+    const selected = hits.filter(hit => injectedKeys.includes(hit.memory_id || hit.key));
+    observation.outcome = 'selected';
+    observation.selectedIds = injectedKeys;
+    observation.sourceMix = selected.reduce((mix, hit) => {
+      const sourceAgent = hit.provenance?.source_agent || hit.sourceAgent || 'unknown';
+      mix[sourceAgent] = (mix[sourceAgent] || 0) + 1;
+      return mix;
+    }, {});
+    observation.trustMix = selected.reduce((mix, hit) => {
+      const trust = hit.trust_status || 'unknown';
+      mix[trust] = (mix[trust] || 0) + 1;
+      return mix;
+    }, {});
+    observation.tokenEstimate = Math.ceil(fenced.length / 4);
     return { success: true, output: formatUserPromptSubmit(harness, fenced) };
   } catch {
     // Fail-open: a context hook must never break a prompt.
+    observation.outcome = 'error';
     return { success: true, output: '' };
   }
 }
 
+async function handleMemoryRecall(rest, projectRoot, opts = {}) {
+  const harness = parseHarness(rest);
+  const startedAt = Date.now();
+  const observation = { harness };
+  const record = result => {
+    const outcome = result.reason === 'timeout'
+      ? 'timeout'
+      : (observation.outcome || (result.output ? 'selected' : 'empty'));
+    const eventObservation = {
+      ...observation,
+      outcome,
+      harness,
+      elapsedMs: Date.now() - startedAt,
+    };
+    try {
+      const pending = (opts.recordRecallEvent || opts.launchRecallEvent || launchMemoryRecallEvent)(
+        projectRoot,
+        eventObservation,
+      );
+      if (pending && typeof pending.catch === 'function') pending.catch(() => {});
+    } catch { /* Operational evidence is best-effort and must never delay or suppress a prompt. */ }
+    return result;
+  };
+  const capability = userPromptSubmitCapability(harness);
+  if (!capability.rendered) {
+    observation.outcome = 'unsupported';
+    return record({ success: true, output: '', reason: capability.reason });
+  }
+  const deadline = { expired: false };
+  const result = await withinDeadline(
+    () => runMemoryRecall(rest, projectRoot, { ...opts, deadline, observation }),
+    { success: true, output: '', reason: 'timeout' },
+    opts.promptRecallDeadlineMs || PROMPT_RECALL_DEADLINE_MS,
+    () => { deadline.expired = true; },
+  );
+  return record(result);
+}
+
 /**
  * `forge hooks capture --harness <h> --trigger <precompact|stop>` — the CAPTURE-on-exit hook.
  * PreCompact (before context compaction) and Stop (turn end) fire it; it snapshots a bounded
@@ -374,7 +570,8 @@ function handleMemoryRecall(rest, projectRoot, opts = {}) {
 async function handleCapture(rest, projectRoot, opts = {}) {
   try {
     const harness = parseHarness(rest);
-    if (!sessionEndCapability(harness).rendered) return { success: true, output: '' };
+    const capability = sessionEndCapability(harness);
+    if (!capability.rendered) return { success: true, output: '', reason: capability.reason };
     const trigger = parseTrigger(rest);
 
     const fetchIssues = opts.fetchIssues || defaultFetchIssues;
@@ -521,4 +718,5 @@ module.exports = {
     '--dry-run': 'Preview the merge without writing anything',
   },
   handler,
+  _internal: { appendShadowLog, SHADOW_LOG_MAX_BYTES },
 };

package/lib/commands/insights.js

@@ -36,7 +36,7 @@ function positionals(args) {
   return values;
 }
 
-async function handler(args, flags, projectRoot) {
+async function handler(args, flags, projectRoot, opts = {}) {
   const commandFlags = flags ?? {};
   const [subcommand, candidateId] = positionals(args);
   if (subcommand === 'accept' || subcommand === 'reject') {
@@ -54,14 +54,19 @@ async function handler(args, flags, projectRoot) {
     };
   }
 
-  const result = analyzeInsights(projectRoot, {
+  const result = await analyzeInsights(projectRoot, {
     limit: readOption(args, '--limit', undefined),
     minCount: readOption(args, '--min-count', undefined),
     since: readOption(args, '--since', undefined),
+    // Injectable kernel-read seams (Slice C2); undefined falls back to the real
+    // cli-broker-factory-backed reads inside analyzeInsights.
+    runIssueOperation: opts.runIssueOperation,
+    listRecentEvents: opts.listRecentEvents,
+    env: opts.env,
   });
   if (args.includes('--review-feedback')) {
     result.limitations = [
-      'Compatibility note: --review-feedback now reads Beads interactions and issue evidence; external review-provider comments are not inferred.',
+      'Compatibility note: --review-feedback now reads kernel events and issue evidence; external review-provider comments are not inferred.',
       ...result.limitations,
     ];
   }