@ctxr/skill-llm-wiki 1.0.1 → 1.1.0

package/scripts/lib/operators.mjs

@@ -40,8 +40,10 @@ import {
   writeFileSync,
 } from "node:fs";
 import { basename, dirname, join, relative } from "node:path";
+import pRetry, { AbortError } from "p-retry";
+import pTimeout, { TimeoutError } from "p-timeout";
 import { parseFrontmatter, renderFrontmatter } from "./frontmatter.mjs";
-import { collectFrontmatterOnly } from "./chunk.mjs";
+import { collectFrontmatterOnly, readFrontmatterStreaming } from "./chunk.mjs";
 import { listChildren, rebuildAllIndices } from "./indices.mjs";
 import { buildComparisonModel } from "./similarity.mjs";
 import {
@@ -53,13 +55,21 @@ import {
 } from "./tiered.mjs";
 import {
   buildProposeStructureRequest,
+  buildSiblingIdfContext,
   detectClusters,
+  deterministicPurpose,
+  generateDeterministicSlug,
   MAX_CLUSTER_SIZE,
   MIN_CLUSTER_SIZE,
   MIN_MATH_CLUSTER_SIZE,
   MIN_TIER2_CLUSTER_SIZE,
 } from "./cluster-detect.mjs";
-import { applyNest, resolveNestSlug, validateSlug } from "./nest-applier.mjs";
+import {
+  applyNest,
+  buildWikiForbiddenIndex,
+  resolveNestSlug,
+  validateSlug,
+} from "./nest-applier.mjs";
 import { computeRoutingCost } from "./quality-metric.mjs";
 import { loadFixture, resolveFromFixture } from "./tier2-protocol.mjs";
 import { appendMetricTrajectory, appendNestDecision } from "./decision-log.mjs";
@@ -68,7 +78,31 @@ import { appendMetricTrajectory, appendNestDecision } from "./decision-log.mjs";
 // termination. The methodology's convergence argument proves it
 // halts, but we still cap defensively in case two operators
 // interact pathologically.
-const MAX_CONVERGENCE_ITERATIONS = 20;
+//
+// Why 200, not 20: the architecture applies at most ONE PAIRWISE
+// operator (DESCEND/LIFT/MERGE) per outer iteration, and pairwise
+// ops always win priority over the cluster-NEST fallback
+// (`tryClusterNestIteration` runs only when no pairwise op fires
+// in a given iteration — see the while loop in `runConvergence`).
+// Note that once cluster-NEST IS reached, it can apply multiple
+// NEST commits in that single pass (multi-NEST selection inside
+// `tryClusterNestIteration`); the scarcity is at the outer
+// iteration level, not per-NEST. On a 596-leaf hand-authored
+// corpus observed in the field (skill-code-review/reviewers.src),
+// Tier 1 similarity produces ~20 viable MERGE pairs; the old cap
+// of 20 iterations burned entirely on those MERGEs and cluster
+// NEST — where the Phase X.10 coarse-partition emits ~75
+// top-level NESTs — never got a chance to run even once. Result:
+// the downstream balance phase saw a 576-leaf root, tried linear
+// carving, hit its own 20-iter cap, and the whole build rolled
+// back.
+//
+// 200 gives a typical large corpus (20 pairwise ops + ~75 cluster
+// NESTs + some follow-up MERGEs after NESTs reveal new overlaps)
+// plenty of headroom. Small wikis exit early via the "no ops
+// fired this iteration" break and never approach the cap — so the
+// raise is effectively free for them.
+const MAX_CONVERGENCE_ITERATIONS = 200;
 
 // Each operator returns an array of `Proposal` objects describing
 // a change to apply. The loop priority-orders proposals, applies
@@ -94,11 +128,24 @@ const PRIORITY = {
 // Detection: a non-root directory that contains exactly one leaf
 // file and no indexed subdirs. Apply: move the leaf up one level,
 // delete the now-empty folder.
+//
+// X.11 constraint: LIFT refuses to land a leaf at the wiki root
+// (`dirname(dir) === wikiRoot`). The root-containment invariant
+// forbids non-index leaves at depth 0, and without this guard LIFT +
+// X.11 root-containment (Phase 4.4.5) would oscillate forever on
+// single-member subcategories that X.11 itself creates for outliers.
+// Single-member subcategories
+// produced by X.11 are a valid transient end state — they stay until
+// future builds accrete topically-adjacent leaves that make the
+// cluster worth merging up. Flatten-to-root is the only direction
+// LIFT is forbidden; every deeper single-child passthrough is still
+// fair game.
 export function detectLift(wikiRoot) {
   const proposals = [];
   const dirs = walkDirs(wikiRoot);
   for (const dir of dirs) {
     if (dir === wikiRoot) continue;
+    if (dirname(dir) === wikiRoot) continue;
     const { leaves, subdirs } = listChildren(dir);
     if (leaves.length === 1 && subdirs.length === 0) {
       const leaf = leaves[0];
@@ -163,6 +210,48 @@ async function applyLift(wikiRoot, dir, leaf) {
 // says "same". Apply: produce a merged entry carrying the union
 // of covers[] (deduped), the more general focus, both source ids as
 // aliases, and delete the second source leaf.
+// Batch size for parallel pairwise `decide()` calls inside
+// `detectMerge`. Each pair is independent (no cross-pair state;
+// tiered.decide() reads/writes its own cache file + appends one
+// decision-log entry), so an O(N²) sequential loop can be turned
+// into ceil(N²/BATCH) sequential batches of concurrent awaits.
+// 32 is empirically chosen to saturate I/O without flooding the
+// kernel's open-file table on flat similarity-cache layouts.
+// Larger batches don't help because most of decide()'s latency is
+// filesystem-bound, and the bottleneck is dir-lookup contention
+// on the cache dir, not the number of in-flight readers.
+export const DETECT_MERGE_PAIR_BATCH_SIZE = 32;
+
+// Per-pair timeout for `decide()` calls wrapped with `pTimeout`.
+// A healthy pair resolves in <200 ms (cache hit) or <2 s (Tier 1
+// inference cold). 30 s is a 15× safety margin over the worst
+// realistic case — catches genuine hangs (stuck model load, NFS
+// mount stall, runaway fs write) without false-positiving on a
+// slow-but-progressing embed call. When the timeout fires the
+// inner `decide()` call is still running on the event loop; it
+// will eventually settle, but its result is discarded. For a
+// 178k-pair sweep that's negligible overhead.
+export const DETECT_MERGE_PAIR_TIMEOUT_MS = 30_000;
+
+// Retry budget per pair. Most "retryable" failures are transient
+// fs glitches (EAGAIN on a contested shard dir, a partial write
+// that parse rejected) — three attempts total (1 initial + 2
+// retries) with 500 ms → 5 s exponential backoff handles those
+// cleanly. Only timeouts (wrapped as AbortError) are explicitly
+// non-retryable; all other errors (including deterministic
+// validation errors from `decide()` — unknown qualityMode, missing
+// ctx fields, etc.) currently consume the full retry budget. That
+// wastes a few seconds on what would be a repeat failure, but
+// classifying decide()'s error surface would require pattern-
+// matching on error messages or a dedicated error-code layer, and
+// the waste in practice is small vs the correctness risk of
+// misclassifying a legitimate transient failure. Acceptable
+// trade-off for now; a targeted shouldRetry() refinement is
+// tracked as a follow-up.
+export const DETECT_MERGE_PAIR_RETRIES = 2;
+export const DETECT_MERGE_PAIR_RETRY_MIN_MS = 500;
+export const DETECT_MERGE_PAIR_RETRY_MAX_MS = 5_000;
+
 export async function detectMerge(wikiRoot, ctx) {
   const proposals = [];
   const dirs = walkDirs(wikiRoot);
@@ -176,19 +265,136 @@ export async function detectMerge(wikiRoot, ctx) {
     // between 10⁹ and 10⁶ operations.
     const corpus = leaves.map((l) => l.data);
     const model = buildComparisonModel(corpus);
-    for (let i = 0; i < leaves.length; i++) {
-      for (let j = i + 1; j < leaves.length; j++) {
-        const a = leaves[i];
-        const b = leaves[j];
-        const r = await decide(a.data, b.data, corpus, {
-          wikiRoot,
-          opId: ctx.opId,
-          operator: "MERGE",
-          qualityMode: ctx.qualityMode,
-          interactive: ctx.interactive,
-          tier2Handler: ctx.tier2Handler,
-          precomputedModel: model,
-        });
+    // Stream pairs in batches without ever materialising the full
+    // O(N²) pair list. The outer/inner (i, j) indices advance
+    // monotonically; once a batch of DETECT_MERGE_PAIR_BATCH_SIZE
+    // pairs is ready we run them concurrently, drain results in
+    // deterministic order, and continue. Memory stays O(batch
+    // size) instead of O(N²). For a flat 600-leaf directory the
+    // pre-allocation would have been ~180k pair objects at ~200
+    // bytes each ≈ 36 MB — not fatal but wasteful when all we need
+    // is a rolling 32-pair window.
+    //
+    // Reliability: each `decide()` call is wrapped with
+    // `pTimeout` (per-call deadline) + `pRetry` (transient retry
+    // with exponential backoff). The outer batch uses
+    // `Promise.allSettled` so a pair that exhausts retries logs
+    // and skips instead of aborting the whole batch — the previous
+    // `Promise.all` semantic would have discarded every
+    // successfully-completed pair alongside the one failure.
+    // Timeout is non-retryable: a `TimeoutError` is re-thrown as
+    // `AbortError` so pRetry gives up immediately on the first
+    // deadline hit. Rationale: the underlying `decide()` call is
+    // still running on the event loop when pTimeout fires and may
+    // still write its cache entry + decision log. A retry would
+    // run a second `decide()` for the same pair and duplicate
+    // those side effects. Fail-fast on timeout keeps the write
+    // footprint to one cache/log entry per pair, which matters for
+    // post-run auditability. Transient errors (EAGAIN, partial
+    // writes, parse rejects) still retry up to
+    // `DETECT_MERGE_PAIR_RETRIES` times with backoff.
+    //
+    // Determinism: each batch's pairs are processed in the order
+    // they were generated (pure i-ascending, j-ascending), and the
+    // within-batch allSettled[] preserves that order via positional
+    // result semantics. Cross-batch ordering is preserved by
+    // draining each batch before advancing.
+    const BATCH = DETECT_MERGE_PAIR_BATCH_SIZE;
+    const decideCtx = {
+      wikiRoot,
+      opId: ctx.opId,
+      operator: "MERGE",
+      qualityMode: ctx.qualityMode,
+      interactive: ctx.interactive,
+      tier2Handler: ctx.tier2Handler,
+      precomputedModel: model,
+    };
+    const pending = [];
+    let failureCount = 0;
+    const decideWithGuards = (a, b) =>
+      pRetry(
+        async () => {
+          try {
+            return await pTimeout(decide(a.data, b.data, corpus, decideCtx), {
+              milliseconds: DETECT_MERGE_PAIR_TIMEOUT_MS,
+              message:
+                `detectMerge: decide() timed out after ` +
+                `${DETECT_MERGE_PAIR_TIMEOUT_MS}ms for pair ` +
+                `(${a.data.id ?? "?"} ↔ ${b.data.id ?? "?"})`,
+            });
+          } catch (err) {
+            // Convert timeouts into non-retryable AbortErrors. The
+            // underlying `decide()` call is still running on the
+            // event loop when pTimeout rejects — it will eventually
+            // settle and may still write the similarity-cache entry
+            // AND append a decision-log entry for this pair. Running
+            // a retry attempt after that would trigger a SECOND
+            // `decide()` for the same pair, doubling the cache writes
+            // and decision-log entries (on the happy-if-slow path)
+            // or doubling the timeout cost (on the stuck path).
+            // Treating timeouts as fail-fast skips the pair cleanly
+            // so the only visible residue is whatever the dangling
+            // decide() eventually writes — one cache entry, one log
+            // entry, never amplified. True cancellation would
+            // require an AbortSignal plumbed through decide →
+            // similarity-cache → embeddings, which is a bigger
+            // refactor; the non-retry shortcut is the honest
+            // trade-off for this release.
+            if (err instanceof TimeoutError) throw new AbortError(err);
+            throw err;
+          }
+        },
+        {
+          retries: DETECT_MERGE_PAIR_RETRIES,
+          minTimeout: DETECT_MERGE_PAIR_RETRY_MIN_MS,
+          maxTimeout: DETECT_MERGE_PAIR_RETRY_MAX_MS,
+          onFailedAttempt: (err) => {
+            // Single-line breadcrumb per failure. Quiet-but-visible
+            // — an operator inspecting a slow run should SEE the
+            // retries happening rather than wonder why a batch
+            // ended up with a few skipped pairs.
+            process.stderr.write(
+              `[detectMerge] retry ${err.attemptNumber}/${err.retriesLeft + err.attemptNumber}` +
+                ` pair (${a.data.id ?? "?"} ↔ ${b.data.id ?? "?"}): ` +
+                `${err.message}\n`,
+            );
+          },
+        },
+      );
+    const flush = async () => {
+      if (pending.length === 0) return;
+      const settled = await Promise.allSettled(
+        pending.map(({ a, b }) => decideWithGuards(a, b)),
+      );
+      for (let k = 0; k < settled.length; k++) {
+        const outcome = settled[k];
+        const { a, b } = pending[k];
+        if (outcome.status === "rejected") {
+          failureCount++;
+          const reason = outcome.reason;
+          // Timeout path: decideWithGuards converts `TimeoutError`
+          // into `AbortError` so pRetry skips the pair
+          // immediately (see the note in decideWithGuards on why
+          // re-running decide() after a timeout would duplicate
+          // side effects). At this point `reason` is therefore an
+          // AbortError whose originalError is the TimeoutError —
+          // unwrap to distinguish "timed out and bailed" from
+          // "exhausted retries on a transient error". The phrasing
+          // in the breadcrumb matters for operators grepping logs
+          // after a slow run.
+          const original = reason?.originalError;
+          const isTimeout =
+            reason instanceof TimeoutError || original instanceof TimeoutError;
+          const kind = isTimeout ? "timeout" : "error";
+          const verb = isTimeout ? "aborted" : "exhausted retries";
+          const msg = (isTimeout ? original?.message : reason?.message) ?? reason;
+          process.stderr.write(
+            `[detectMerge] pair (${a.data.id ?? "?"} ↔ ${b.data.id ?? "?"}) ` +
+              `${verb} (${kind}): ${msg}\n`,
+          );
+          continue;
+        }
+        const r = outcome.value;
         if (r.decision === "same") {
           proposals.push({
             operator: "MERGE",
@@ -199,6 +405,19 @@ export async function detectMerge(wikiRoot, ctx) {
           });
         }
       }
+      pending.length = 0;
+    };
+    for (let i = 0; i < leaves.length; i++) {
+      for (let j = i + 1; j < leaves.length; j++) {
+        pending.push({ a: leaves[i], b: leaves[j] });
+        if (pending.length >= BATCH) await flush();
+      }
+    }
+    await flush(); // drain the tail
+    if (failureCount > 0) {
+      process.stderr.write(
+        `[detectMerge] finished ${dir}: ${failureCount} pair(s) skipped after exhausting retries (or on timeout)\n`,
+      );
     }
   }
   return proposals;
@@ -263,6 +482,51 @@ async function applyMerge(wikiRoot, a, b, decision) {
   aliases.delete(data.id);
   data.aliases = Array.from(aliases);
 
+  // Pre-apply alias-collision guard. If any alias the MERGE is about to
+  // introduce would collide with a live id elsewhere in the wiki, refuse
+  // the merge BEFORE writing the keeper or deleting the absorbed. The
+  // v1.0 validator catches this downstream as ALIAS-COLLIDES-ID, but by
+  // then the phase has committed and the whole convergence iteration
+  // has to roll back. Checking here pre-empts the rollback.
+  //
+  // Scope: only aliases the MERGE is *newly introducing* can create a
+  // new collision — that's absorbed's id and absorbed's pre-existing
+  // aliases, rolled into the keeper. The keeper's pre-existing aliases
+  // were already validated on their way into the tree, so re-checking
+  // them here would only produce misattributed errors. We filter the
+  // candidate set accordingly and report the absorbed leaf as the
+  // source of the conflict.
+  //
+  // We exclude absorbed.path from the live-id scan because the merge
+  // deletes the absorbed file, freeing its id namespace, so absorbed's
+  // own id shouldn't register as an "elsewhere" collision against
+  // itself. keeper.path does NOT need to be excluded: MERGE rewrites
+  // keeper in place but keeps its id unchanged, and the filter on
+  // `newlyIntroducedAliases` already drops keeper's id before the
+  // alias check (see `.delete(data.id)` below), so keeper's id
+  // appearing in `liveIds` is harmless and keeping the scan focused on
+  // "every other entry in the wiki" is clearer than two asymmetric
+  // exclusions. Anything else carrying a colliding id is a real
+  // conflict that would surface downstream as ALIAS-COLLIDES-ID if we
+  // proceeded.
+  const newlyIntroducedAliases = new Set([
+    absorbed.data.id,
+    ...(absorbed.data.aliases ?? []),
+  ]);
+  newlyIntroducedAliases.delete(data.id);
+  const liveIds = collectLiveIds(wikiRoot, new Set([absorbed.path]));
+  for (const alias of newlyIntroducedAliases) {
+    if (liveIds.has(alias)) {
+      throw new Error(
+        `MERGE: alias "${alias}" (from absorbed ${absorbed.data.id}) ` +
+          `collides with an existing live id; refusing to merge ` +
+          `${absorbed.data.id} into ${keeper.data.id}. ` +
+          `Resolve the conflict (rename the other leaf, or drop ` +
+          `the alias) and retry.`,
+      );
+    }
+  }
+
   writeFileSync(keeper.path, renderFrontmatter(data, body), "utf8");
   rmSync(absorbed.path, { force: true });
   return {
@@ -541,6 +805,7 @@ export async function runConvergence(wikiRoot, ctx = {}) {
       metricTrajectory,
       commitBetweenIterations,
       nestedParents,
+      qualityMode,
     });
     if (nestOutcome === "applied") continue;
     if (nestOutcome === "pending-tier2") {
@@ -620,7 +885,9 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
     metricTrajectory,
     commitBetweenIterations,
     nestedParents = new Set(),
+    qualityMode = "tiered-fast",
   } = ctx;
+  const deterministic = qualityMode === "deterministic";
 
   // Collect candidate proposals across every parent directory.
   // For each directory:
@@ -670,23 +937,28 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
 
     const relDir = relative(wikiRoot, dir) || ".";
 
-    // Step 1: propose_structure Tier 2 request. Park on pending
-    // without short-circuiting the math phase below.
+    // Step 1: propose_structure Tier 2 request. Skipped in
+    // deterministic mode — the whole point is no LLM in the loop.
+    // Math detection alone drives structural proposals; no Tier 2
+    // subcategory suggestions are ever ingested, so byte-reproducible
+    // outputs are guaranteed from the same inputs.
     let tier2Clusters = [];
-    const proposeReq = buildProposeStructureRequest(relDir, leaves);
-    const proposeResp = resolveTier2Response(wikiRoot, fixture, proposeReq);
-    if (proposeResp === "pending") {
-      enqueuePending(wikiRoot, proposeReq);
-      suggestions.push({
-        operator: "NEST",
-        sources: leaves.map((l) => l.path),
-        reason: `propose_structure parked for ${relDir} (awaiting Tier 2)`,
-      });
-      // Fall through — math still runs so any cluster this dir
-      // carries is evaluated (and its gate/naming requests are
-      // batched alongside every other directory's) before we exit 7.
-    } else {
-      tier2Clusters = extractTier2Clusters(proposeResp, leaves, dir);
+    if (!deterministic) {
+      const proposeReq = buildProposeStructureRequest(relDir, leaves);
+      const proposeResp = resolveTier2Response(wikiRoot, fixture, proposeReq);
+      if (proposeResp === "pending") {
+        enqueuePending(wikiRoot, proposeReq);
+        suggestions.push({
+          operator: "NEST",
+          sources: leaves.map((l) => l.path),
+          reason: `propose_structure parked for ${relDir} (awaiting Tier 2)`,
+        });
+        // Fall through — math still runs so any cluster this dir
+        // carries is evaluated (and its gate/naming requests are
+        // batched alongside every other directory's) before we exit 7.
+      } else {
+        tier2Clusters = extractTier2Clusters(proposeResp, leaves, dir);
+      }
     }
 
     // Step 2: math cluster detection (aggressive scan).
@@ -705,6 +977,40 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
     // Step 3: merge proposals, dedup by member set.
     const merged = mergeClusterProposals(tier2Clusters, mathClusters);
     for (const c of merged) c.parent_dir = dir;
+    // Step 3b (deterministic only): pre-name every math candidate
+    // using `generateDeterministicSlug` so the naming loop below
+    // treats them as already-named and skips the cluster_name
+    // Tier 2 request. The sibling leaves at `dir` act as the IDF
+    // context for the slug generator so terms that are common
+    // across the entire directory get down-weighted relative to
+    // terms that distinguish the cluster from its siblings — the
+    // slug becomes the cluster's most-distinguishing feature.
+    //
+    // Precompute the IDF map ONCE per directory and reuse it across
+    // every candidate. Without this, each `generateDeterministicSlug`
+    // call would re-tokenize + recompute IDF over the full sibling
+    // corpus — an O(|siblings| × |candidates|) repeat on dirs with
+    // multiple candidates. A shared map collapses that to
+    // O(|siblings| + |candidates| × |cluster|). Same byte-output;
+    // faster for dense dirs.
+    if (deterministic) {
+      // Only build the sibling-IDF context when there's at least one
+      // math candidate that will actually consume it. Skipping the
+      // tokenisation + IDF computation on directories that produced
+      // only tier2-sourced (or no) candidates avoids a hot-path cost
+      // on large sibling sets for no gain.
+      const hasMathCandidate = merged.some((c) => c.source === "math");
+      if (hasMathCandidate) {
+        const deterministicIdf = buildSiblingIdfContext(leaves);
+        for (const cand of merged) {
+          if (cand.source !== "math") continue;
+          cand.slug = generateDeterministicSlug(cand.leaves, leaves, {
+            precomputedIdf: deterministicIdf,
+          });
+          cand.purpose = deterministicPurpose(cand.leaves);
+        }
+      }
+    }
     allCandidates.push(...merged);
   }
 
@@ -715,6 +1021,12 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
   // Step 4: math-only candidates go through a mandatory
   // nest_decision gate. Candidates that came from propose_structure
   // are already structurally approved by Tier 2 — skip the gate.
+  //
+  // In deterministic mode the gate is also skipped: math candidates
+  // produced by the aggressive-threshold scan are auto-approved
+  // because the partition-shape score + metric regression gate
+  // downstream are the algorithmic equivalents of the gate and are
+  // fully deterministic.
   const gatedCandidates = [];
   for (const cand of allCandidates) {
     if (cand.source === "tier2" || cand.source === "both") {
@@ -733,6 +1045,11 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
       dropStaleMathCandidate(wikiRoot, cand, opId, suggestions);
       continue;
     }
+    if (deterministic) {
+      cand.gate_reason = "deterministic mode: algorithmic auto-approve";
+      gatedCandidates.push(cand);
+      continue;
+    }
     // math-only: run the gate.
     const gateReq = cand.gate_request;
     const gateResp = resolveTier2Response(wikiRoot, fixture, gateReq);
@@ -923,6 +1240,23 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
     /* best effort: indices may not be set up yet on a fresh wiki */
   }
 
+  // Precompute the wiki-wide forbidden-id index once per convergence
+  // iteration, but ONLY when at least one picked proposal will consult
+  // it. resolveNestSlug below reuses it via opts.wikiIndex so each
+  // picked proposal's slug resolution is O(parent-dir) instead of
+  // O(full-tree). After each successful apply, we mutate the index
+  // (`wikiIndex.add(resolvedSlug)`) so the next proposal sees the new
+  // directory/id as occupied. Total cost across a multi-NEST iteration
+  // drops from O(#applies × #files) to O(#files + #applies).
+  //
+  // Guarding on `picked.length > 0` avoids an otherwise-pointless
+  // full-tree walk on iterations where detection ran but the non-
+  // conflict selection culled every candidate — a concrete saving on
+  // the convergence loop's last iteration, where the picked set is
+  // typically empty and we're one step away from breaking out.
+  const wikiIndex =
+    picked.length > 0 ? buildWikiForbiddenIndex(wikiRoot) : null;
+
   let appliedCount = 0;
   for (const proposal of picked) {
     // Re-check freshness RIGHT BEFORE apply. An earlier pick in the
@@ -935,12 +1269,28 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
       dropStaleMathCandidate(wikiRoot, proposal, opId, suggestions);
       continue;
     }
+    // Confidence band + tier_used reflect how the candidate got here.
+    // Under `--quality-mode deterministic` every math candidate was
+    // auto-approved algorithmically (no sub-agent ever consulted),
+    // so the audit trail must say so:
+    //   - `confidence_band: "deterministic-math"` distinguishes these
+    //     entries from math candidates that passed a Tier 2
+    //     `nest_decision` gate.
+    //   - `tier_used: 0` correctly records that no Tier was consulted
+    //     at decision time. Tooling/tests that interpret tier_used as
+    //     actual Tier usage no longer see a misleading `2`.
+    // Under every other quality mode the paths that reach here
+    // touched Tier 2 at least once (propose_structure, nest_decision,
+    // or both), so tier_used stays at the legacy default of 2.
     const confBand =
-      proposal.source === "both"
-        ? "tier2-and-math"
-        : proposal.source === "tier2"
-          ? "tier2-proposed"
-          : "math-gated";
+      deterministic && proposal.source === "math"
+        ? "deterministic-math"
+        : proposal.source === "both"
+          ? "tier2-and-math"
+          : proposal.source === "tier2"
+            ? "tier2-proposed"
+            : "math-gated";
+    const tierUsed = confBand === "deterministic-math" ? 0 : 2;
     const preMetric = computeRoutingCost(wikiRoot).cost;
     // Snapshot the files we're about to mutate so we can roll back
     // on a metric regression. We ONLY need the old leaf contents
@@ -957,7 +1307,12 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
     // is written AFTER applyNest succeeds so decisions.yaml never
     // records a rename for an op that ultimately failed.
     const originalSlug = proposal.slug;
-    const resolvedSlug = resolveNestSlug(originalSlug, proposal);
+    const resolvedSlug = resolveNestSlug(
+      originalSlug,
+      proposal,
+      wikiRoot,
+      wikiIndex ? { wikiIndex } : {},
+    );
     let result;
     try {
       result = applyNest(wikiRoot, proposal, resolvedSlug);
@@ -972,6 +1327,7 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
         sources: proposal.leaves.map((l) => l.data.id),
         similarity: proposal.average_affinity ?? 0,
         confidence_band: confBand,
+        tier_used: tierUsed,
         decision: "rejected-by-gate",
         reason: `applyNest threw: ${err.message}`,
       });
@@ -1021,6 +1377,7 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
         sources: proposal.leaves.map((l) => l.data.id),
         similarity: proposal.average_affinity ?? 0,
         confidence_band: confBand,
+        tier_used: tierUsed,
         decision: "rejected-by-metric",
         reason: `metric ${preMetric.toFixed(4)} → ${postMetric.toFixed(4)} exceeds ${policyLabel}`,
       });
@@ -1038,6 +1395,7 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
         sources: proposal.leaves.map((l) => l.data.id),
         similarity: proposal.average_affinity ?? 0,
         confidence_band: confBand,
+        tier_used: tierUsed,
         decision: "slug-renamed",
         reason: `slug "${originalSlug}" collided with existing id; renamed to "${resolvedSlug}"`,
       });
@@ -1071,6 +1429,7 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
       sources: proposal.leaves.map((l) => l.data.id),
       similarity: proposal.average_affinity ?? 0,
       confidence_band: confBand,
+      tier_used: tierUsed,
       decision: "applied",
       reason:
         `slug=${proposal.slug}, ` +
@@ -1080,6 +1439,12 @@ async function tryClusterNestIteration(wikiRoot, ctx) {
     // recursively sub-cluster it in later iterations of the
     // same run.
     nestedParents.add(result.target_dir);
+    // Update the wiki-wide forbidden index with the new slug so later
+    // picks in this same iteration see the directory as occupied and
+    // auto-suffix against it. Only the slug needs adding — member leaf
+    // ids were already in the index (NEST moves files but preserves
+    // ids) and the applier doesn't delete anything.
+    if (wikiIndex) wikiIndex.add(resolvedSlug);
     appliedCount++;
   }
 
@@ -1343,6 +1708,75 @@ export function mathCandidateIsFresh(cand) {
 
 // ── Directory walk helper ────────────────────────────────────────────
 
+// Collect live entry frontmatter ids from the wiki on a best-effort
+// basis, excluding any paths the caller names. "Entry" means every
+// `.md` file we parse — leaves AND the `index.md` at every directory
+// depth, since the validator treats both as id-bearing entries and a
+// MERGE alias colliding with either shape is equally a conflict.
+//
+// Best-effort: unreadable files and malformed frontmatter are
+// silently skipped (see the `catch` block in the inner loop). The
+// direction of that error matters: skipping means the returned Set
+// is a SUBSET of "every id currently live in the wiki", not a
+// superset. A MERGE that passes this guard is therefore protected
+// against collisions with every cleanly-parseable entry in the tree,
+// but NOT against a collision hidden inside a malformed file —
+// that's a potential false negative. Malformed entries would also
+// fail the validator's post-op pass (which is what actually enforces
+// the invariant at commit time), so the guard's job is to pre-empt
+// the common case of a healthy wiki hitting a runtime MERGE
+// collision, not to substitute for validation. Callers that need a
+// strict complete live-id set should validate the wiki first.
+//
+// Used by applyMerge for its pre-apply alias-collision guard (before
+// MERGE commits new aliases, ensure none of them already exist as
+// live ids elsewhere in the corpus).
+//
+// Skips every dot-directory as a blanket rule — matches the discipline
+// in `scripts/lib/chunk.mjs::collectEntryPaths` and other wiki-tree
+// walks in the pipeline. This covers skill internals (`.llmwiki/`,
+// `.work/`, `.shape/`) as well as arbitrary user dot-directories the
+// corpus might carry (`.git/`, `.github/`, `.cache/`, etc). Without
+// this, ids found in unrelated dotfolders would produce false-positive
+// collision refusals the validator never reports.
+//
+// Per-file frontmatter is extracted via `readFrontmatterStreaming` so
+// this walk reads bounded (≤ `MAX_FRONTMATTER_BYTES`) from each leaf
+// rather than slurping the full body — matching the pattern used by
+// other full-tree id walks in the pipeline.
+export function collectLiveIds(wikiRoot, excludePaths = new Set()) {
+  const liveIds = new Set();
+  const stack = [wikiRoot];
+  while (stack.length > 0) {
+    const dir = stack.pop();
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      if (entry.name.startsWith(".")) continue;
+      const entryPath = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(entryPath);
+        continue;
+      }
+      if (!entry.name.endsWith(".md")) continue;
+      if (excludePaths.has(entryPath)) continue;
+      try {
+        const captured = readFrontmatterStreaming(entryPath);
+        if (captured === null) continue;
+        const { data } = parseFrontmatter(captured.frontmatterText, entryPath);
+        if (data?.id) liveIds.add(data.id);
+      } catch {
+        /* skip unreadable / malformed frontmatter */
+      }
+    }
+  }
+  return liveIds;
+}
+
 function walkDirs(wikiRoot) {
   const out = [wikiRoot];
   const stack = [wikiRoot];