@gcunharodrigues/wrxn 0.13.2 → 0.14.0

package/payload/.claude/hooks/recall-surface.cjs

@@ -23,6 +23,8 @@
 const fs = require('fs');
 const http = require('http');
 const path = require('path');
+const { coalesceSidecar } = require('./sidecar.cjs'); // shared coalesced read/rewrite/fail-open/secret-scan
+const { rewardFactor, selectRewardMode, RECORDED_REWARD_VERDICT } = require('./reward.cjs'); // S3 re-rank + S5 verdict-derived mode
 
 const MIN_PROMPT_LEN = 8;          // skip trivial prompts ("ok", "yes")
 const MAX_QUERY_CHARS = 512;       // trim the prompt before querying the door
@@ -37,7 +39,19 @@ const PROSE_TYPES = new Set(['Page', 'Section']); // prose scope — drop code s
 const ENDPOINT_REL = path.join('.recon-wrxn', 'serve-endpoint.json');
 const FIND_PATH = '/api/tools/recon_find';
 const REINFORCE_REL = path.join('.wrxn', 'reinforce.json'); // coalesced access-recency sidecar (STATE)
+const SURFACED_REL = path.join('.wrxn', 'surfaced.json'); // per-session surfaced-log sidecar (STATE)
 const WIKI_PREFIX = '.wrxn/wiki/'; // the wiki root — stripped to form the D1 join key
+const REWARD_REL = path.join('.wrxn', 'reward.json'); // per-page Beta-Bernoulli store (STATE) — read-only here
+const HISTORY_REL = path.join('.wrxn', 'history'); // per-session edit trail dir (STATE) — .touched lives here (S4)
+
+// The single shipped mode gating the reward re-rank (mirrors recon's SHIPPED_DECAY_MODE). It is DERIVED
+// from the recorded lift-gate verdict via selectRewardMode — never hard-coded — so the live state is never
+// a silent default. Today RECORDED_REWARD_VERDICT is NOT passing (no real session corpus accrued yet), so
+// this resolves to 'shadow': reward counts accrue at session-end but the factor NEVER moves a recall rank
+// — recall output is byte-identical to pre-reward behaviour. It flips to 'live' ONLY when the recorded
+// verdict passes (the offline lift gate proves lift on real data AND the operator ratifies the git-only
+// signal — docs/eval/0001-reward-lift-gate.md). Tests force 'live' via the recallFromDoor option.
+const SHIPPED_REWARD_MODE = selectRewardMode(RECORDED_REWARD_VERDICT);
 
 function emit(envelope) {
   process.stdout.write(JSON.stringify(envelope));
@@ -82,6 +96,118 @@ function qualifies(hit) {
   return floorOk || hasConsensus(hit);
 }
 
+// ── S1 exclusion: drop curation-retired pages (stale / superseded) before the gate (#20) ─────────────
+//
+// Harvest stamps retirement into a page's FRONTMATTER — `stale: <missing-source>` (orphaned) or
+// `superseded_by: <path>` (replaced). A recon FindHit carries no frontmatter, so the exclusion reads it
+// off a { hit.file → frontmatter } VIEW supplied by the caller (the IO shell reads the page files; tests
+// pass a literal map) — buildExclusion stays a PURE, deterministic black box. A page is RETIRED when its
+// frontmatter sets a truthy `stale:` OR it is not its own live head (it carries a `superseded_by:` that
+// resolves forward). resolveHead walks the supersession chain to the live head and fails SAFE on a cycle
+// or a dangling successor: the affected page is excluded and recall NEVER throws.
+
+const MAX_SUPERSESSION_HOPS = 64; // a belt-and-braces bound; the visited-set already stops a cycle
+
+// Walk `superseded_by` from `start` to the live head (a page with no successor). Returns the head path,
+// or null when the chain is malformed — a cycle (a path seen twice) or a dangling successor (a pointer
+// to a page absent from the view). PURE over the frontmatter view; a null head ⇒ fail-safe exclude.
+function resolveHead(start, view) {
+  let cur = start;
+  const seen = new Set();
+  for (let i = 0; i < MAX_SUPERSESSION_HOPS; i++) {
+    if (seen.has(cur)) return null; // cycle → fail safe
+    seen.add(cur);
+    const fm = view && view[cur];
+    if (!fm) return null; // dangling: a successor points at a page not in the view → fail safe
+    const next = fm.superseded_by;
+    if (!next || typeof next !== 'string') return cur; // no (usable) successor → cur is the live head
+    cur = next;
+  }
+  return null; // over the hop bound (pathological chain) → fail safe
+}
+
+// Build the exclude predicate (true ⇒ drop the hit) from the frontmatter view. A page is excluded when it
+// is truthy-`stale:`, OR it is superseded (its resolved live head is not itself — including the fail-safe
+// null head from a cycle / dangling chain). A page absent from the view, or with empty frontmatter, is
+// kept. TOTAL: a non-object view yields a never-exclude predicate (fail-open → recall unchanged).
+function buildExclusion(view) {
+  const v = view && typeof view === 'object' ? view : {};
+  return (hit) => {
+    const file = hit && hit.file;
+    if (typeof file !== 'string' || !file) return false; // no path to judge → keep
+    const fm = v[file];
+    if (!fm || typeof fm !== 'object') return false; // unflagged / absent → keep
+    if (fm.stale) return true; // truthy stale (a missing-source orphan) → retired
+    if (fm.superseded_by) return resolveHead(file, v) !== file; // not its own head → retired (incl. fail-safe)
+    return false;
+  };
+}
+
+// ── S4 structural arm: edited paths → a recon_find seed, and reciprocal-rank fusion (#23) ────────────
+//
+// Adapted (no-invention): recon-wrxn exposes no DOCUMENTED_BY code→wiki graph edge, so the structural
+// arm is kernel-only — it REUSES the .touched per-session edited-paths list and issues a SECOND
+// recon_find seeded by those edits, then RRF-fuses that ranked list with the prompt-semantic one BEFORE
+// the exclusion / gate / reward / top-N. Both helpers here are PURE black boxes (the fetch + the .touched
+// read are the injectable IO shell below).
+
+// Seed a recon_find query from the session's edited paths: take each path's basename (sans extension),
+// tokenize on non-alphanumerics, dedup case-insensitively, join with spaces, cap at the query budget. A
+// dir-segment carries little semantic signal (and a code-seeded arm fetches mostly code that the prose
+// filter drops anyway), so only basenames are seeded — they are what a wiki page about the file is titled
+// after. Empty / non-array / all-junk input → '' (the no-op seed: the structural arm then never fires).
+function buildStructuralQuery(paths) {
+  const list = Array.isArray(paths) ? paths : [];
+  const seen = new Set();
+  const tokens = [];
+  for (const p of list) {
+    if (typeof p !== 'string' || !p) continue;
+    const base = path.basename(p).replace(/\.[^.]+$/, '');
+    for (const raw of base.split(/[^A-Za-z0-9]+/)) {
+      const tok = raw.trim();
+      if (!tok) continue;
+      const key = tok.toLowerCase();
+      if (seen.has(key)) continue;
+      seen.add(key);
+      tokens.push(tok);
+    }
+  }
+  return tokens.join(' ').slice(0, MAX_QUERY_CHARS);
+}
+
+// Reciprocal-rank fusion of two ranked hit lists into one, deduped by page key. Each hit contributes
+// 1/(k + rank) (rank 1-based) to its key's score; a page in BOTH arms accrues both contributions and so
+// outranks a single-arm page (rank-based consensus, never a score magnitude — same posture as the gate).
+// IDENTITY when one list is empty: a single list's per-rank scores strictly decrease, so the sort returns
+// it in its original order with its original hit objects — this is what makes the empty-.touched path a
+// byte-identical no-op. On a key collision listA's hit object is the kept representative (the semantic arm
+// is primary; the structural arm only boosts an already-present page's rank). Ties break by first-seen
+// order (stable). A hit with no usable key (no file/id/name) is skipped.
+const RRF_K = 60; // the standard RRF damping constant (Cormack et al.) — large k softens top-rank dominance
+
+function hitKey(h) {
+  return (h && (h.file || h.id || h.name)) || null;
+}
+
+function rrfFuse(listA, listB, k) {
+  const K = Number.isFinite(k) && k > 0 ? k : RRF_K;
+  const acc = new Map(); // key → { hit, score, order }
+  let order = 0;
+  const fold = (list) => {
+    (Array.isArray(list) ? list : []).forEach((h, i) => {
+      const key = hitKey(h);
+      if (key == null) return;
+      const contrib = 1 / (K + i + 1); // rank is 1-based
+      const cur = acc.get(key);
+      if (cur) cur.score += contrib; // page in both arms → summed reciprocal ranks (consensus boost)
+      else acc.set(key, { hit: h, score: contrib, order: order++ }); // first arm to see it owns the object
+    });
+  };
+  fold(listA); // listA (semantic) folded first → it owns the representative on a collision
+  fold(listB);
+  return [...acc.values()].sort((x, y) => y.score - x.score || x.order - y.order).map((x) => x.hit);
+}
+
 function slugify(s) {
   return String(s || '')
     .toLowerCase()
@@ -124,16 +250,64 @@ function renderBlock(hits) {
   return block;
 }
 
-// PURE: the prose hits that clear the gate, capped at TOP_N — exactly the hits decideRecall renders
-// (and the pages reinforce stamps). Factored out so the IO shell can stamp the surfaced pages by path.
-function qualifyingHits(hits) {
+// ── S3 reward re-rank (PURE) ─────────────────────────────────────────────────────────
+//
+// The learning-moat's value axis applied to recall: a per-page reward factor (∈(0,2), neutral 1) from
+// the Beta-Bernoulli store re-ranks the qualifying prose candidates by `base relevance × factor` BEFORE
+// the top-N cut, so a page that has preceded good sessions rises and one that preceded nothing fades.
+// The factor is INJECTED as a lookup { <wiki-rel-path>: factor } (the impure shell reads .wrxn/reward.json
+// and pre-computes factors via reward.cjs's rewardFactor) — decideRecall stays pure and reward-math-free.
+//
+// SHADOW BY DEFAULT: the re-rank only fires when a NON-EMPTY lookup is passed. An absent / empty lookup
+// is the IDENTITY — the door order is preserved byte-for-byte, exactly today's behaviour. The shell
+// passes a lookup only in 'live' mode (SHIPPED_REWARD_MODE), so the shipped default is a provable no-op.
+
+// The relevance magnitude the reward factor modulates: the door's fused score, with the dense cosine as
+// a fallback when a hit carries no fused score (totality), else 0.
+function baseScore(hit) {
+  const sc = Number(hit && hit.score);
+  if (Number.isFinite(sc)) return sc;
+  const sem = Number(hit && hit.semanticScore);
+  return Number.isFinite(sem) ? sem : 0;
+}
+
+// A hit's reward factor from the lookup, keyed by its wiki-rel path. A page absent from the lookup, a
+// non-wiki hit, or a non-positive / garbage factor → neutral 1 (fail-open: reward never zeroes a rank).
+function factorFor(lookup, hit) {
+  const key = wikiRelPath(hit && hit.file);
+  const f = key ? Number(lookup[key]) : NaN;
+  return Number.isFinite(f) && f > 0 ? f : 1;
+}
+
+// Re-rank prose candidates by base relevance × reward factor, descending. STABLE: equal effective
+// scores keep door order (so an all-neutral lookup is the exact identity). An absent / empty / non-object
+// lookup short-circuits to the identity — the shadow no-op, by construction byte-identical to today.
+function rerankByReward(prose, rewardLookup) {
+  if (!rewardLookup || typeof rewardLookup !== 'object' || !Object.keys(rewardLookup).length) return prose;
+  return prose
+    .map((h, i) => ({ h, i, eff: baseScore(h) * factorFor(rewardLookup, h) }))
+    .sort((a, b) => b.eff - a.eff || a.i - b.i)
+    .map((x) => x.h);
+}
+
+// PURE: the prose hits that clear the gate, reward-re-ranked, capped at TOP_N — exactly the hits
+// decideRecall renders (and the pages reinforce stamps). The optional reward lookup is injected; absent
+// → door order (shadow). The optional `exclude` predicate (true ⇒ drop) is applied FIRST — before the
+// prose-filter, gate, reward re-rank, and top-N cut — so a retired page (stale / superseded) never
+// occupies a slot a live page should hold; absent ⇒ no exclusion (byte-identical to today). Factored
+// out so the IO shell can stamp the surfaced pages by path.
+function qualifyingHits(hits, rewardLookup, exclude) {
   const list = Array.isArray(hits) ? hits : [];
-  return list.filter((h) => isProse(h) && qualifies(h)).slice(0, TOP_N);
+  const kept = typeof exclude === 'function' ? list.filter((h) => !exclude(h)) : list;
+  const prose = kept.filter((h) => isProse(h) && qualifies(h));
+  return rerankByReward(prose, rewardLookup).slice(0, TOP_N);
 }
 
-// PURE: prose-filter → gate → top-N → format. Returns the block string, or null (Abstain).
-function decideRecall(hits) {
-  const qualified = qualifyingHits(hits);
+// PURE: exclusion → prose-filter → gate → reward re-rank → top-N → format. Returns the block string,
+// or null (Abstain). The reward lookup and the exclusion predicate are optional and injected; with
+// neither (the shipped default) the output is byte-identical to the pre-exclusion behaviour.
+function decideRecall(hits, rewardLookup, exclude) {
+  const qualified = qualifyingHits(hits, rewardLookup, exclude);
   if (!qualified.length) return null;
   return renderBlock(qualified);
 }
@@ -169,27 +343,13 @@ function dayStamp(now) {
 // Stamp each surfaced prose hit's wiki-rel path → today into <root>/.wrxn/reinforce.json. Writes only
 // when the map actually changes (coalesced). Wholly best-effort: never throws, never blocks recall.
 function reinforce(root, hits, now) {
-  try {
-    const list = Array.isArray(hits) ? hits : [];
-    if (!root || !list.length) return;
-    const file = path.join(root, REINFORCE_REL);
-    let map = {};
-    let raw = null;
-    try {
-      raw = fs.readFileSync(file, 'utf8');
-    } catch {
-      raw = null; // absent → fresh map (normal, not a fault)
-    }
-    if (raw !== null) {
-      let parsed;
-      try {
-        parsed = JSON.parse(raw);
-      } catch {
-        return; // malformed existing sidecar → skip silently, leave it untouched (never clobber)
-      }
-      if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return; // not a map → skip
-      map = parsed;
-    }
+  const list = Array.isArray(hits) ? hits : [];
+  if (!root || !list.length) return;
+  // The whole read/parse/coalesce/rewrite/fail-open/secret-scan mechanism lives in coalesceSidecar; here
+  // we supply only the domain mutation — stamp each surfaced prose hit's wiki-rel join key to today,
+  // reporting whether the map actually changed (so an all-already-today recall is a coalesced no-op). The
+  // day-stamp is computed INSIDE the mutate so an invalid clock is caught by the helper's fail-open envelope.
+  coalesceSidecar(path.join(root, REINFORCE_REL), (map) => {
     const day = dayStamp(now);
     let changed = false;
     for (const h of list) {
@@ -200,12 +360,86 @@ function reinforce(root, hits, now) {
         changed = true;
       }
     }
-    if (!changed) return; // coalesced no-op → file stays byte-identical (<= 1 write/page/day)
-    fs.mkdirSync(path.dirname(file), { recursive: true });
-    fs.writeFileSync(file, JSON.stringify(map, null, 2) + '\n');
+    return changed;
+  });
+}
+
+// ── surfaced-log: the per-session record of what recall surfaced (S1 / kernel #12) ────
+//
+// When Recall surfaces prose pages, record THIS session's surfaced (qualifying) page-paths into
+// <root>/.wrxn/surfaced.json — a compact map { "<session_id>": ["<wiki-rel-path>", …] } via the shared
+// coalesced-sidecar helper. The value uses the SAME wiki-root-relative join key reinforce stamps (a
+// slug/path mismatch silently breaks downstream consumers). Coalesced: re-surfacing the identical set
+// for a session rewrites nothing. Best-effort + non-blocking — the helper swallows every fault, so a
+// surfaced-log write can never alter or break the recall surfacing.
+
+// Map the surfaced hits to their wiki-rel join keys, de-duplicated, order preserved. (qualifyingHits
+// has already prose-filtered + gated + capped; here we only project to the path key and drop non-wiki
+// hits / dupes.)
+function surfacedPaths(hits) {
+  const list = Array.isArray(hits) ? hits : [];
+  const seen = new Set();
+  const out = [];
+  for (const h of list) {
+    const key = wikiRelPath(h && h.file);
+    if (!key || seen.has(key)) continue;
+    seen.add(key);
+    out.push(key);
+  }
+  return out;
+}
+
+// Record `paths(hits)` under `sessionId` in the surfaced-log. No session id or no surfaced paths → no
+// write. Coalesced: an unchanged set for the session leaves the file byte-identical.
+function surfacedLog(root, sessionId, hits) {
+  if (!root || !sessionId) return;
+  const paths = surfacedPaths(hits);
+  if (!paths.length) return;
+  coalesceSidecar(path.join(root, SURFACED_REL), (map) => {
+    const prev = map[sessionId];
+    if (Array.isArray(prev) && prev.length === paths.length && prev.every((v, i) => v === paths[i])) {
+      return false; // identical surfaced set for this session → coalesced no-op
+    }
+    map[sessionId] = paths;
+    return true;
+  });
+}
+
+// ── S4 structural arm: read the .touched edit trail (IO shell) (#23) ─────────────────────────────────
+//
+// The session-id → filename transform MUST match code-intel-push's safeId byte-for-byte, or the arm reads
+// the wrong file. Replicated here (self-contained: no shared import) — the same discipline that duplicates
+// secretScan across the install-only modules.
+function safeSessionId(sid) {
+  return String(sid || 'session')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 48) || 'session';
+}
+
+// Read this session's edited paths back from <root>/.wrxn/history/<safeId>.touched — the list
+// code-intel-push already maintains (REUSE; no new persistence path). Deduped on read, order preserved.
+// FAIL-OPEN: an absent file (the common no-edits case) or any read fault → [] (the structural arm then
+// never fires). The values are treated ONLY as a query seed — never as filesystem paths.
+function readTouched(root, sessionId) {
+  if (!root) return [];
+  const marker = path.join(root, HISTORY_REL, `${safeSessionId(sessionId)}.touched`);
+  let raw;
+  try {
+    raw = fs.readFileSync(marker, 'utf8');
   } catch {
-    /* best-effort: a reinforce fault must NEVER alter or break the recall surfacing */
+    return []; // absent / unreadable → no edits this session
   }
+  const seen = new Set();
+  const out = [];
+  for (const line of raw.split('\n')) {
+    const p = line.trim();
+    if (!p || seen.has(p)) continue;
+    seen.add(p);
+    out.push(p);
+  }
+  return out;
 }
 
 // ── the door (IO shell, injectable transport) ───────────────────────────────────────
@@ -308,36 +542,203 @@ function httpTransport({ port, path: reqPath, body, timeoutMs }) {
   });
 }
 
+// Read the reward sidecar (.wrxn/reward.json = { "<wiki-rel-path>": { s, f } }) and pre-compute each
+// page's reward factor → { "<wiki-rel-path>": factor } for the live re-rank. FAIL-OPEN: a missing /
+// corrupt / non-object store → null (a neutral lookup → recall unaffected). The keys are treated ONLY
+// as join keys (map lookups against wikiRelPath), NEVER as filesystem paths (sec posture). rewardFactor
+// is total, so a malformed per-page slot reads as zero evidence → neutral 1.
+function readRewardLookup(root) {
+  if (!root) return null;
+  let raw;
+  try {
+    raw = fs.readFileSync(path.join(root, REWARD_REL), 'utf8');
+  } catch {
+    return null; // absent → neutral (the common case before the gate flips, and on any read fault)
+  }
+  let counts;
+  try {
+    counts = JSON.parse(raw);
+  } catch {
+    return null; // corrupt JSON → neutral, recall proceeds unchanged
+  }
+  if (!counts || typeof counts !== 'object' || Array.isArray(counts)) return null;
+  const lookup = {};
+  for (const key of Object.keys(counts)) lookup[key] = rewardFactor(counts[key]);
+  return lookup;
+}
+
+// ── S1 exclusion: read each candidate page's retirement frontmatter from disk (#20) ──────────────────
+//
+// buildExclusion is PURE over a { hit.file → frontmatter } view; this shell builds that view by reading
+// each candidate page's frontmatter off disk under <root>. A recon FindHit carries no frontmatter, so
+// this is the only source of the `stale:` / `superseded_by:` flags harvest stamps. BEST-EFFORT: a page
+// read fault (missing/unreadable file, no fence) contributes empty frontmatter → the page is treated as
+// unflagged (kept) and nothing throws — so a since-deleted page or a parse hiccup never breaks recall.
+
+// Extract the frontmatter block (between the leading `---` fence and the next `---`). Mirrors drift-detect:
+// a page without a closed fence yields '' (no flags → kept).
+function frontmatterBlock(content) {
+  const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(content);
+  return m ? m[1] : '';
+}
+
+// A YAML-falsy scalar — a `stale:`/`superseded_by:` set to any of these means NOT retired (the page is
+// kept). It mirrors YAML's boolean/null primitives so that, e.g., `stale: false` un-flags a page rather
+// than being read as the truthy STRING 'false' (#25). Compared case-insensitively after quote-stripping.
+const YAML_FALSY = new Set(['false', 'no', 'null', '~', '0', '']);
+
+// Parse the two retirement keys from a page's frontmatter into { stale?, superseded_by? }. Values are
+// simple scalars (harvest writes `stale: <path>` / `superseded_by: <path>`); surrounding quotes are
+// stripped. The scalar is interpreted as a YAML boolean: a falsy scalar (false / no / null / ~ / 0 /
+// empty) means NOT retired and is DROPPED — only a truthy flag (a non-empty non-falsy value, e.g.
+// `true`, `yes`, or a harvest source path) is kept, so the downstream truthiness check is correct (#25).
+// Only these two keys are read — everything else in the frontmatter is ignored.
+function parseRetirement(content) {
+  const fm = frontmatterBlock(content);
+  if (!fm) return {};
+  const out = {};
+  for (const line of fm.split(/\r?\n/)) {
+    const m = /^(stale|superseded_by):\s*(.*)$/.exec(line);
+    if (!m) continue;
+    const val = m[2].trim().replace(/^['"]|['"]$/g, '');
+    if (val && !YAML_FALSY.has(val.toLowerCase())) out[m[1]] = val;
+  }
+  return out;
+}
+
+// Resolve a candidate hit.file to the absolute path to OPEN, confined to the REAL wiki root — or null
+// when it escapes (#26). wikiRelPath is a lenient substring join-key (an absolute or `..`-bearing path
+// passes it), so it is NOT a read guard: a hit.file like `.wrxn/wiki/../../../etc/passwd` would slip it
+// and `path.join(root, file)` would resolve OUTSIDE the wiki root (readFileSync also follows symlinks).
+// Here we canonicalize and enforce a resolved-prefix check — the same discipline as lib/connect.cjs'
+// `state:` confinement: the resolved path (and, when it exists, its symlink-realpath) MUST sit at or
+// under <root>/.wrxn/wiki/. A non-string path, an escaping/absolute path, or a symlink whose target
+// escapes → null (the caller treats it as unreadable/unflagged → kept, fail-open). A path that does not
+// yet exist on disk stays confined lexically and is returned (the read then fails-open as a missing page).
+function confinedWikiPath(root, file) {
+  if (typeof file !== 'string' || !file) return null;
+  const wikiRoot = path.resolve(root, WIKI_PREFIX); // <root>/.wrxn/wiki
+  const underWiki = (p) => p === wikiRoot || p.startsWith(wikiRoot + path.sep);
+  const abs = path.resolve(root, file);
+  if (!underWiki(abs)) return null; // lexical `..`/absolute escape → refused, never opened
+  let real;
+  try {
+    real = fs.realpathSync(abs); // follows symlinks — a link pointing outside is caught here
+  } catch {
+    return abs; // does not (yet) exist / unresolvable → lexically confined; read fails-open as missing
+  }
+  return underWiki(real) ? abs : null; // a symlink whose real target escapes the wiki root → refused
+}
+
+// Read the frontmatter of each prose candidate page into a view keyed by the EXACT path the hit carries
+// (hit.file), so buildExclusion's predicate — which reads off hit.file — joins to it directly. The path
+// is CONFINED to the real wiki root via confinedWikiPath: a `..`-escaping, absolute, or symlink-escaping
+// hit.file is never opened (#26 — defense-in-depth, mirrors lib/connect.cjs' resolved-prefix discipline).
+// A read fault, a parse fault, or a refused path all contribute {} (unflagged → kept). Returns {path: fm}.
+function readExclusionView(root, hits) {
+  const view = {};
+  if (!root || !Array.isArray(hits)) return view;
+  for (const h of hits) {
+    const file = h && h.file;
+    if (typeof file !== 'string' || !file || view[file]) continue;
+    if (wikiRelPath(file) == null) continue; // not under the wiki root (no join key) → never opened
+    const abs = confinedWikiPath(root, file); // resolved-prefix confinement (the real read guard)
+    if (abs == null) { view[file] = {}; continue; } // escapes the wiki root → refused, treated as unflagged
+    let content;
+    try {
+      content = fs.readFileSync(abs, 'utf8');
+    } catch {
+      view[file] = {}; // unreadable (e.g. a since-deleted page) → unflagged, kept
+      continue;
+    }
+    try {
+      view[file] = parseRetirement(content);
+    } catch {
+      view[file] = {}; // any parse fault → unflagged, kept (fail-open)
+    }
+  }
+  return view;
+}
+
+// Build the exclude predicate for this candidate set from the page frontmatter on disk. Wholly fail-open:
+// any fault yields a never-exclude predicate, so a broken read can only ever LEAVE RECALL UNCHANGED — it
+// can never wrongly drop a live page or throw.
+function diskExclusion(root, hits) {
+  try {
+    return buildExclusion(readExclusionView(root, hits));
+  } catch {
+    return () => false;
+  }
+}
+
+// One recon_find POST against the warm door → the hits array. Both arms (prompt-semantic and S4
+// structural) share this single fetch path, so they hit the SAME door identically. THROWS on any fault
+// (transport reject / non-200 / malformed body); each caller decides what a fault means — the semantic
+// arm aborts recall (the existing contract), the structural arm swallows it (fail-open).
+async function findHits(transport, port, query, timeoutMs) {
+  const resp = await (transport || httpTransport)({
+    port,
+    path: FIND_PATH,
+    body: { query, limit: FETCH_LIMIT },
+    timeoutMs: timeoutMs || TIMEOUT_MS,
+  });
+  if (!resp || resp.statusCode !== 200) throw new Error('recall door non-200');
+  const parsed = JSON.parse(resp.body); // throws on a malformed body
+  return Array.isArray(parsed.hits) ? parsed.hits : [];
+}
+
+// S4 structural arm: seed a query from this session's edited paths (.touched) and issue a SECOND
+// recon_find for it. Empty .touched → empty seed → [] (the arm never fires — the no-op). The hits are
+// returned RAW; the caller fuses them with the semantic arm and the shared pipeline then prose-filters,
+// excludes, gates, and caps — so a code-seeded arm's code hits are dropped exactly as in the semantic arm.
+async function structuralArm(root, sessionId, port, { transport, timeoutMs } = {}) {
+  try {
+    const query = buildStructuralQuery(readTouched(root, sessionId));
+    if (!query) return []; // no edits this session → no structural arm
+    return await findHits(transport, port, query, timeoutMs);
+  } catch {
+    return []; // any structural fault (read / fetch / non-200 / malformed) → empty arm; recall proceeds on the semantic arm
+  }
+}
+
 // IO shell: discover the door, POST the prose query, gate the hits. Returns the block string or null.
 // `transport` is injected in tests; production uses httpTransport. Sends NO `type` (recon_find takes a
 // single NodeType, not an array) — prose scope is enforced by decideRecall's post-filter.
-async function recallFromDoor(root, prompt, { transport, timeoutMs, now } = {}) {
+async function recallFromDoor(root, prompt, { transport, timeoutMs, now, sessionId, rewardMode } = {}) {
   const door = discoverEndpoint(root);
   if (!door) return null; // not warm → Abstain (silent)
   const query = String(prompt || '').trim().slice(0, MAX_QUERY_CHARS);
   if (!query) return null;
-  let resp;
+  let semanticHits;
   try {
-    resp = await (transport || httpTransport)({
-      port: door.port,
-      path: FIND_PATH,
-      body: { query, limit: FETCH_LIMIT },
-      timeoutMs: timeoutMs || TIMEOUT_MS,
-    });
+    semanticHits = await findHits(transport, door.port, query, timeoutMs);
   } catch {
-    return null; // timeout / connection refused / abort → silent
+    return null; // timeout / connection refused / abort / non-200 / malformed body → silent (existing contract)
   }
-  if (!resp || resp.statusCode !== 200) return null;
-  let parsed;
-  try {
-    parsed = JSON.parse(resp.body);
-  } catch {
-    return null; // malformed body → silent
+  // S4 structural arm: a SECOND recon_find seeded by this session's edited paths (.touched), RRF-fused
+  // with the prompt-semantic arm BEFORE the exclusion / gate / reward / top-N. Fuse only when the arm has
+  // hits: empty .touched (or any structural fault) → no structural hits → the fused set IS the semantic
+  // set object, so recall output stays byte-identical to pre-S4 (the strict no-op guarantee).
+  const structuralHits = await structuralArm(root, sessionId, door.port, { transport, timeoutMs });
+  const hits = structuralHits.length ? rrfFuse(semanticHits, structuralHits) : semanticHits;
+  // SHADOW (shipped default): pass NO lookup → decideRecall is the identity (door order), byte-identical
+  // to pre-reward recall regardless of what sits in reward.json. LIVE (gate-flipped / test-forced): read
+  // the reward sidecar and re-rank the qualifying candidates by reward factor before the top-N cut.
+  const mode = rewardMode || SHIPPED_REWARD_MODE;
+  const rewardLookup = mode === 'live' ? readRewardLookup(root) : null;
+  // S1 exclusion: drop curation-retired (stale / superseded) candidates BEFORE the gate/rerank/top-N, by
+  // reading each candidate page's frontmatter from disk. Fail-open: a fault yields a never-exclude
+  // predicate (recall unchanged), and an unflagged corpus makes this the identity (no-op, AC-6).
+  const exclude = diskExclusion(root, hits);
+  const block = decideRecall(hits, rewardLookup, exclude);
+  // Side effects on a surfacing (both best-effort, never block): stamp access-recency for the pages we
+  // surfaced, and record this session's surfaced set in the per-session surfaced-log. Use the SAME lookup
+  // and exclusion so the recorded set matches the re-ranked, excluded top-N that was rendered.
+  if (block) {
+    const surfaced = qualifyingHits(hits, rewardLookup, exclude);
+    reinforce(root, surfaced, now);
+    surfacedLog(root, sessionId, surfaced);
   }
-  const hits = Array.isArray(parsed.hits) ? parsed.hits : [];
-  const block = decideRecall(hits);
-  // Side effect: stamp access-recency for the pages we actually surfaced (best-effort, never blocks).
-  if (block) reinforce(root, qualifyingHits(hits), now);
   return block;
 }
 
@@ -360,7 +761,7 @@ async function main() {
 
   let block = null;
   try {
-    block = await recallFromDoor(root, prompt.trim());
+    block = await recallFromDoor(root, prompt.trim(), { sessionId: event.session_id });
   } catch {
     return emit({});
   }
@@ -376,8 +777,14 @@ if (require.main === module) {
 module.exports = {
   decideRecall,
   qualifyingHits,
+  rerankByReward,
   recallFromDoor,
+  readRewardLookup,
+  readExclusionView,
+  confinedWikiPath,
+  parseRetirement,
   reinforce,
+  surfacedLog,
   wikiRelPath,
   dayStamp,
   discoverEndpoint,
@@ -386,8 +793,14 @@ module.exports = {
   isProse,
   hasConsensus,
   qualifies,
+  buildExclusion,
+  resolveHead,
+  buildStructuralQuery,
+  rrfFuse,
+  readTouched,
   renderBlock,
   findInstallRoot,
   SEMANTIC_FLOOR,
   PROSE_TYPES,
+  SHIPPED_REWARD_MODE,
 };