@ctxr/skill-llm-wiki 1.0.2 → 1.2.0

package/scripts/lib/cluster-detect.mjs

@@ -3,10 +3,18 @@
 // Given a set of leaves at a single depth (one directory's worth
 // of children), compute an affinity matrix using several signals,
 // find candidate clusters as connected components under a
-// threshold, and propose NEST applications. Every proposal is
-// named by asking Tier 2 (cluster_name kind) — we never invent
-// names from keyword shortcuts, because the whole point of Tier 2
-// is to let the sub-agent exercise judgment at naming time.
+// threshold, and propose NEST applications.
+//
+// Cluster naming depends on the active quality mode:
+//   - tiered-fast / claude-first: proposals are named by asking
+//     Tier 2 (the `cluster_name` request kind), because the point
+//     of Tier 2 is to let the sub-agent exercise judgment at
+//     naming time.
+//   - deterministic: naming is derived locally from member
+//     frontmatters via `generateDeterministicSlug` +
+//     `deterministicPurpose`, bypassing the `cluster_name` request
+//     entirely so the mode's "no LLM in the loop" contract holds
+//     end-to-end. See these helpers' doc comments for the algorithm.
 //
 // Signals used for the affinity matrix:
 //
@@ -68,6 +76,7 @@ import {
 } from "./embeddings.mjs";
 import {
   buildComparisonModel,
+  computeIdf,
   cosine,
   entryText,
   tfidfVector,
@@ -137,6 +146,45 @@ export const MAX_CLUSTER_SIZE = 8;
 // usually a noise floor hit and is structurally useless.
 export const GIANT_BLOB_FRACTION = 0.75;
 
+// ── Coarse-partition pre-pass for flat large-diverse directories ──
+//
+// The HAC path above (`findComponents` + `partitionShapeScore`) is
+// tuned for FINE-GRAINED sub-clustering inside already-bounded
+// directories: it maximises the count of 3-8-size components at
+// some candidate threshold. On a flat 600-leaf root that's the
+// wrong optimisation — the best partition at any threshold is
+// dominated by one giant component plus many singletons, and the
+// handful of 3-8-size clusters that do emerge score poorly.
+// Practical symptom: a 596-leaf hand-authored corpus observed in
+// the field produced zero NEST proposals during convergence under
+// `--quality-mode deterministic`, which left the balance phase to
+// carve categories linearly and hit its 20-iter cap far short of
+// convergence.
+//
+// The coarse-partition pre-pass uses deterministic K-means (farthest-
+// first init + mean-member-similarity assignment) to force K top-
+// level clusters when the directory's leaf count exceeds
+// `COARSE_PARTITION_THRESHOLD`. K is chosen as
+// `ceil(N / COARSE_TARGET_CLUSTER_SIZE)` so the average cluster
+// lands around the `COARSE_TARGET_CLUSTER_SIZE` mark. Clusters
+// smaller than `MIN_CLUSTER_SIZE` or larger than
+// `MAX_COARSE_CLUSTER_SIZE` are rejected post-hoc — small ones
+// aren't worth nesting (the `MIN_CLUSTER_SIZE` floor) and giant
+// ones are usually noise-floor hits that would themselves need
+// sub-clustering (the `MAX_COARSE_CLUSTER_SIZE` ceiling, 30, is
+// ~4× the target so only egregiously-concentrated clusters get
+// pruned — the rest pass through and balance enforcement can
+// refine them in a second pass if `--fanout-target` is tight).
+//
+// Determinism: all ordering uses lex-first tie-breaking (first
+// seed is always index 0, subsequent seeds via farthest-first,
+// members iterate in leaf-array order). Two runs on the same
+// corpus produce byte-identical cluster membership.
+export const COARSE_PARTITION_THRESHOLD = 50;
+export const COARSE_TARGET_CLUSTER_SIZE = 8;
+export const MAX_COARSE_CLUSTER_SIZE = 30;
+export const COARSE_KMEANS_MAX_ITERS = 20;
+
 // Read the first ~1 KB of a leaf's body for the Tier 1 signal.
 // We skip the frontmatter (between the first two `---` lines)
 // and take a prefix of the remaining bytes. Short-body leaves
@@ -453,6 +501,161 @@ export function buildProposeStructureRequest(relativeDir, leaves) {
   });
 }
 
+// Coarse-partition K-means for flat large-diverse directories.
+// Called from `detectClusters` when `leaves.length` exceeds
+// `COARSE_PARTITION_THRESHOLD`. The HAC path used for ≤-threshold
+// directories can't produce usable 3-8-sized clusters on a flat
+// 600-leaf root (see the constant block at the top of the file);
+// this function forces K top-level clusters via deterministic
+// K-means with farthest-first seed init.
+//
+// Algorithm:
+//
+//   1. Compute the same NxN affinity matrix `detectClusters` uses
+//      (Tier 0 + Tier 1 blend via `computeAffinityMatrix`). Reused
+//      downstream — we do NOT recompute it in the HAC path when
+//      we dispatch here.
+//
+//   2. Pick K = ceil(N / COARSE_TARGET_CLUSTER_SIZE) seeds via
+//      farthest-first selection. First seed is leaves[0] (lex-first
+//      by the caller's ordering). Each subsequent seed maximises
+//      its minimum similarity-distance (1 - max(sim-to-existing))
+//      so seeds spread across the similarity space.
+//
+//   3. Iterate assignment: each leaf → cluster whose current
+//      members have the highest MEAN similarity to it. Using mean
+//      member similarity rather than vector-centroid distance lets
+//      us work with the existing `matrix` directly — no need to
+//      expose or recompute per-leaf vectors. Stops when assignments
+//      stop changing or the iteration cap fires.
+//
+//   4. Build proposals via `buildNestProposal`. Clusters smaller
+//      than `MIN_CLUSTER_SIZE` or larger than
+//      `MAX_COARSE_CLUSTER_SIZE` are rejected (small: not worth
+//      nesting; giant: noise-floor concentration, leave to a
+//      second pass or to balance enforcement).
+//
+// Returns an array of NEST proposals in
+// `(average_affinity desc, member-path asc)` order. Returns `[]`
+// if no cluster passed filters — the caller decides whether to
+// fall back to HAC or escalate.
+export async function detectCoarseClusters(wikiRoot, leaves, opts = {}) {
+  if (leaves.length < MIN_CLUSTER_SIZE) return [];
+  const matrix =
+    opts.precomputedMatrix ??
+    (await computeAffinityMatrix(wikiRoot, leaves, opts));
+  const N = leaves.length;
+  const K = Math.min(
+    Math.ceil(N / COARSE_TARGET_CLUSTER_SIZE),
+    // Guard: K cannot exceed N (degenerate) or produce clusters
+    // smaller than MIN on average. ceil(N / TARGET) hits both
+    // floors naturally, but pin the upper bound so a user tuning
+    // TARGET down to 1 doesn't blow up.
+    Math.floor(N / MIN_CLUSTER_SIZE),
+  );
+  if (K < 2) return []; // nothing meaningful to partition into
+
+  // Step 1: deterministic farthest-first seeds. First seed is the
+  // lex-first leaf (index 0). Each subsequent seed maximises its
+  // minimum similarity-distance (1 - max(sim-to-any-existing-seed))
+  // so seeds don't pile up in a dense region of the affinity graph.
+  // Ties broken by index-ascending, preserving determinism.
+  const seeds = [0];
+  while (seeds.length < K) {
+    let bestIdx = -1;
+    let bestMinDist = -1;
+    for (let i = 0; i < N; i++) {
+      if (seeds.includes(i)) continue;
+      let maxSimToSeed = -Infinity;
+      for (const s of seeds) {
+        if (matrix[i][s] > maxSimToSeed) maxSimToSeed = matrix[i][s];
+      }
+      const minDistToSeed = 1 - maxSimToSeed;
+      if (minDistToSeed > bestMinDist) {
+        bestMinDist = minDistToSeed;
+        bestIdx = i;
+      }
+    }
+    if (bestIdx === -1) break;
+    seeds.push(bestIdx);
+  }
+
+  // Step 2: initial assignment = nearest-seed (max similarity).
+  const assignments = new Array(N);
+  for (let i = 0; i < N; i++) {
+    let bestK = 0;
+    let bestSim = -Infinity;
+    for (let k = 0; k < seeds.length; k++) {
+      const sim = matrix[i][seeds[k]];
+      if (sim > bestSim) {
+        bestSim = sim;
+        bestK = k;
+      }
+    }
+    assignments[i] = bestK;
+  }
+
+  // Step 3: iterate. Each leaf re-assigns to the cluster whose
+  // current members have the highest mean similarity to it.
+  // Converges in a handful of iterations on most corpora; the
+  // COARSE_KMEANS_MAX_ITERS cap is defensive against pathological
+  // oscillation.
+  for (let iter = 0; iter < COARSE_KMEANS_MAX_ITERS; iter++) {
+    const members = Array.from({ length: seeds.length }, () => []);
+    for (let i = 0; i < N; i++) members[assignments[i]].push(i);
+    let changed = false;
+    for (let i = 0; i < N; i++) {
+      let bestK = assignments[i];
+      let bestMean = -Infinity;
+      for (let k = 0; k < seeds.length; k++) {
+        const mem = members[k];
+        if (mem.length === 0) continue;
+        let sum = 0;
+        for (const m of mem) sum += matrix[i][m];
+        const mean = sum / mem.length;
+        if (mean > bestMean) {
+          bestMean = mean;
+          bestK = k;
+        }
+      }
+      if (bestK !== assignments[i]) {
+        assignments[i] = bestK;
+        changed = true;
+      }
+    }
+    if (!changed) break;
+  }
+
+  // Step 4: build proposals from each non-trivial cluster.
+  const proposals = [];
+  for (let k = 0; k < seeds.length; k++) {
+    const componentIndices = [];
+    for (let i = 0; i < N; i++) {
+      if (assignments[i] === k) componentIndices.push(i);
+    }
+    if (componentIndices.length < MIN_CLUSTER_SIZE) continue;
+    if (componentIndices.length > MAX_COARSE_CLUSTER_SIZE) continue;
+    if (componentIndices.length === N) continue; // single-cluster-everything
+    const componentLeaves = componentIndices.map((i) => leaves[i]);
+    const proposal = buildNestProposal(componentLeaves, matrix, componentIndices);
+    proposal.threshold = null; // n/a for K-means; left null to signal coarse-mode
+    proposal.source = "math-coarse";
+    proposals.push(proposal);
+  }
+  // Deterministic sort: highest-affinity clusters first, ties
+  // broken by the lex-first member path so the on-disk apply
+  // order is stable across runs.
+  proposals.sort((a, b) => {
+    if (b.average_affinity !== a.average_affinity) {
+      return b.average_affinity - a.average_affinity;
+    }
+    const aKey = a.leaves?.[0]?.path ?? "";
+    const bKey = b.leaves?.[0]?.path ?? "";
+    return aKey.localeCompare(bKey);
+  });
+  return proposals;
+}
+
 // Detect all NEST proposals for a single parent directory's
 // leaves. Tries each candidate threshold (aggressive range), picks
 // the best by shape score, and emits a proposal for each
@@ -465,9 +668,32 @@ export function buildProposeStructureRequest(relativeDir, leaves) {
 // marker and returns `[]` instead — used by tests and the
 // cluster_name unit tests that don't want the marker in their
 // output.
+//
+// Dispatch: for directories above `COARSE_PARTITION_THRESHOLD`
+// leaves, skip the HAC path entirely and run the coarse K-means
+// partitioner. The HAC path's shape-score optimiser is tuned for
+// fine-grained sub-clustering (3-8-size components), which can't
+// structure a flat large-diverse root — see the constant block.
+// Coarse clusters returned in the same shape the HAC path would
+// emit, so downstream (operators.mjs::tryClusterNestIteration,
+// balance.mjs::runBalance) is untouched.
 export async function detectClusters(wikiRoot, leaves, opts = {}) {
   const { returnEmptyMarker = true } = opts;
   if (leaves.length < MIN_CLUSTER_SIZE) return [];
+
+  // Coarse-partition dispatch for flat large-diverse roots. This
+  // path doesn't honour `returnEmptyMarker` (no empty-partition
+  // marker is emitted) because Tier 2's propose_structure is the
+  // wrong tool for these inputs anyway — the LLM would be asked
+  // to partition 500+ leaves in one shot, which is both a huge
+  // token cost and typically produces worse structure than the
+  // deterministic K-means. If coarse produces zero valid clusters,
+  // return empty; the caller (balance / operators) handles zero-
+  // proposal days gracefully.
+  if (leaves.length > COARSE_PARTITION_THRESHOLD) {
+    return detectCoarseClusters(wikiRoot, leaves, opts);
+  }
+
   const matrix = await computeAffinityMatrix(wikiRoot, leaves, opts);
   let bestPartition = null;
   let bestScore = -1;
@@ -514,3 +740,255 @@ export async function detectClusters(wikiRoot, leaves, opts = {}) {
   proposals.sort((a, b) => b.average_affinity - a.average_affinity);
   return proposals;
 }
+
+// Deterministic slug generator for the `deterministic` quality mode.
+// Given a cluster's member leaves and optional corpus context (for
+// IDF), returns a reproducible kebab-case slug derived from the
+// members' frontmatter terms alone — no LLM, no network, no
+// randomness. Repeated invocations on the same inputs always return
+// the same slug; shuffling the member order never changes the output.
+//
+// Algorithm:
+//
+//   1. Build a TF-IDF vector over each member's `entryText` (focus +
+//      covers + tags + domains) using the supplied corpus context
+//      for IDF weighting. Without context, members form their own
+//      micro-corpus — less semantically interesting but still
+//      deterministic.
+//   2. Sum the per-member vectors (weights stay dominated by terms
+//      that are rare in the corpus but common inside the cluster —
+//      exactly the "distinguishing" terms we want in the slug).
+//   3. Rank terms by (weight desc, term asc). The lex tie-break is
+//      the ONLY source of determinism when two terms share a weight.
+//   4. Walk the ranked list, taking the first 1–2 terms that are
+//      valid slug components (lowercase, ≥ 2 chars, start with a
+//      letter, pass the `SLUG_RE` check when joined).
+//   5. If still no valid slug (terse frontmatters, every top term
+//      numeric/short), fall back to a 7-hex-char content hash of
+//      the sorted member ids — deterministic in its inputs, but NOT
+//      globally unique. Seven hex characters is ~28 bits of entropy
+//      from a truncated FNV-1a-32 output, so hash collisions are
+//      mathematically possible (~0.1% collision rate at 1000 distinct
+//      clusters per the birthday bound). That's fine at this layer:
+//      the caller passes every slug — hash-derived or term-derived —
+//      through `resolveNestSlug` next, which auto-suffixes any
+//      collision with an existing id / alias / directory basename
+//      into the `-group`/`-group-N` deterministic sequence. The hash
+//      fallback just needs to be reproducible from the same inputs,
+//      not collision-free across the whole corpus.
+//
+// The caller (operators.mjs::tryClusterNestIteration) passes the
+// result through `resolveNestSlug` so collisions with existing ids
+// auto-suffix deterministically.
+//
+// `opts.precomputedIdf` lets the caller share an IDF map across
+// sibling clusters in the same directory — cuts the per-candidate
+// cost from `O(|corpus|)` tokenization + IDF to `O(|cluster|)`
+// tokenization alone. Semantically identical to a fresh derivation
+// from the passed `corpusContext`; pass whichever you already have.
+export function generateDeterministicSlug(
+  componentLeaves,
+  corpusContext,
+  opts = {},
+) {
+  // Sort members by a stable key BEFORE building text/token lists.
+  // Floating-point summation is order-sensitive, so an unsorted input
+  // could theoretically flip near-tie ordering under shuffled input.
+  // Sorting on leaf id (path fallback for tests that omit id) removes
+  // that entire class of ambiguity at trivial cost.
+  // Normalise each member: accept either a leaf wrapper `{ path, data }`
+  // or a plain frontmatter object (the shape corpusContext also
+  // tolerates below). Without this, a caller passing plain frontmatter
+  // would hit `entryText(undefined)` for every member, producing empty
+  // token lists and collapsing every such cluster onto the identical
+  // `cluster-<hash>` fallback — so multiple unrelated clusters could
+  // end up with the same slug. Symmetrising with the corpusContext
+  // path closes that footgun.
+  const normalisedMembers = componentLeaves.map((leaf) => ({
+    data: leaf?.data ?? leaf,
+    path: leaf?.path,
+  }));
+  const stableMembers = [...normalisedMembers].sort((a, b) => {
+    const ka = a?.data?.id ?? a?.path ?? "";
+    const kb = b?.data?.id ?? b?.path ?? "";
+    return ka < kb ? -1 : ka > kb ? 1 : 0;
+  });
+  const tokenLists = stableMembers.map((leaf) => tokenize(entryText(leaf.data)));
+  // IDF context: precomputed > corpusContext > cluster itself.
+  const idfMap =
+    opts.precomputedIdf ??
+    (corpusContext && corpusContext.length > 0
+      ? computeIdf(
+          corpusContext.map((e) => tokenize(entryText(e.data ?? e))),
+        )
+      : computeIdf(tokenLists));
+  // Per-member tf-idf, then sum into a single cluster-wide vector.
+  // Stable member order + lex tie-break on the final ranking below
+  // means the output is byte-identical regardless of caller-side
+  // ordering.
+  const sum = new Map();
+  for (const tokens of tokenLists) {
+    const vec = tfidfVector(tokens, idfMap);
+    for (const [term, weight] of vec) {
+      sum.set(term, (sum.get(term) ?? 0) + weight);
+    }
+  }
+  // Rank: weight desc, term asc (lex tie-break → determinism).
+  const ranked = Array.from(sum.entries()).sort((a, b) => {
+    if (b[1] !== a[1]) return b[1] - a[1];
+    return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
+  });
+
+  const SLUG_RE = /^[a-z][a-z0-9-]{0,63}$/;
+  const VALID_TOKEN = /^[a-z][a-z0-9]*$/;
+  // Collect up to MAX_TOKENS_TO_CONSIDER ranked tokens, bounded to
+  // keep the O(n²) pair search below fast on corpora with many
+  // distinct terms. The pair search checks `C(n, 2) = n·(n−1)/2`
+  // combinations, so with n=16 that's at most 120 candidate slugs
+  // to test — trivial, and far more than practical frontmatters
+  // actually supply in their combined focus + covers + tags
+  // token bag.
+  const MAX_TOKENS_TO_CONSIDER = 16;
+  const takeable = [];
+  for (const [term] of ranked) {
+    if (!VALID_TOKEN.test(term)) continue;
+    takeable.push(term);
+    if (takeable.length >= MAX_TOKENS_TO_CONSIDER) break;
+  }
+  // Priority 1: highest-ranked TWO tokens that, when joined with "-",
+  // produce a valid SLUG_RE slug. The outer loop walks rank-first;
+  // the inner loop fills the second slot. Because both axes march
+  // top-to-bottom in ranked order, the first valid combo we find is
+  // the one carrying the highest total rank weight — semantically
+  // the "best" two-term slug.
+  //
+  // Bugfix vs. the v1 impl, which stopped after the top 2 ranked
+  // tokens and fell back to the hash whenever that specific combo
+  // overflowed SLUG_RE's 64-char cap. Walking further ranked terms
+  // surfaces a valid slug in every case where member frontmatters
+  // supply at least one kebab-compatible short pair, instead of
+  // producing an opaque `cluster-<hash>` when a valid slug was
+  // reachable just one rank away.
+  for (let i = 0; i < takeable.length; i++) {
+    for (let j = i + 1; j < takeable.length; j++) {
+      const candidate = `${takeable[i]}-${takeable[j]}`;
+      if (SLUG_RE.test(candidate)) return candidate;
+    }
+  }
+  // Priority 2: highest-ranked SINGLE token that passes SLUG_RE.
+  // Walks in ranked order for the same reason.
+  for (const term of takeable) {
+    if (SLUG_RE.test(term)) return term;
+  }
+  // Deterministic hash fallback — member ids sorted lex, hashed.
+  // Use the normalisedMembers we built earlier so plain-frontmatter
+  // callers get a stable hash too (their id lives at `.data.id` after
+  // normalisation, not `.id` directly).
+  const sortedIds = normalisedMembers
+    .map((leaf) => leaf?.data?.id ?? leaf?.path ?? "")
+    .filter(Boolean)
+    .sort();
+  const hash = hashString(sortedIds.join("|")).slice(0, 7);
+  return `cluster-${hash}`;
+}
+
+// Build an IDF map over a sibling leaf set once, for reuse across
+// multiple `generateDeterministicSlug` calls on clusters within the
+// same parent directory. Every candidate cluster under a given parent
+// shares the same corpus context, so computing IDF once per directory
+// — rather than once per candidate — is strictly better for any
+// directory with ≥ 2 candidate clusters. Drop the return value into
+// `generateDeterministicSlug(.., .., { precomputedIdf: idfMap })`.
+export function buildSiblingIdfContext(siblings) {
+  const tokenLists = siblings.map((leaf) =>
+    tokenize(entryText(leaf?.data ?? leaf)),
+  );
+  return computeIdf(tokenLists);
+}
+
+// Maximum number of cover phrases to splice into a multi-member
+// cluster's synthesised focus. Four is a soft cap that keeps the
+// resulting string short enough to read at a glance while still
+// telling the orchestrator the cluster is multi-topic.
+const PURPOSE_MAX_COVERS = 4;
+
+// Deterministic purpose for the NEST stub's `focus:` field.
+//
+// Tiered policy:
+//   - Single-member cluster: return the member's own `focus` directly
+//     — concise + accurate, e.g. for an X.11 root-containment outlier
+//     whose folder will hold exactly one leaf.
+//   - Multi-member cluster: aggregate the top-N cover phrases across
+//     members ranked by frequency desc, then lex asc, joined with
+//     "; ". This is the corrected behaviour: previously the
+//     algorithm returned just the lex-first highest-count cover,
+//     which on coarse k-means clusters of diverse content (where no
+//     cover appears in multiple members) collapsed to "the
+//     alphabetically-first cover of any member" — producing
+//     misleading single-leaf focus strings on multi-topic clusters
+//     (see X.11 wiki output where an 8-leaf ops/observability
+//     cluster's focus read "Action items without owner or deadline",
+//     which was just one detail of one member).
+//   - Multi-member cluster with no covers anywhere: fall back to the
+//     focus of the member whose id sorts first. Still deterministic,
+//     still driven by member content alone.
+//
+// Accepts either `{ path, data }` leaf wrappers or plain frontmatter
+// objects. Input is normalised via `leaf?.data ?? leaf` at the top so
+// this helper matches `generateDeterministicSlug` + `buildSiblingIdfContext`'s
+// API shape — callers can pass whichever form they already have
+// without getting silent empty results for the plain-object path.
+export function deterministicPurpose(componentLeaves) {
+  const normalised = componentLeaves.map((leaf) => leaf?.data ?? leaf);
+  if (normalised.length === 1) {
+    return typeof normalised[0]?.focus === "string" ? normalised[0].focus : "";
+  }
+  const counts = new Map();
+  for (const data of normalised) {
+    const covers = Array.isArray(data?.covers) ? data.covers : [];
+    const seenInLeaf = new Set();
+    for (const cover of covers) {
+      const key = typeof cover === "string" ? cover.trim() : "";
+      if (!key || seenInLeaf.has(key)) continue;
+      seenInLeaf.add(key);
+      counts.set(key, (counts.get(key) ?? 0) + 1);
+    }
+  }
+  if (counts.size > 0) {
+    const ranked = Array.from(counts.entries()).sort((a, b) => {
+      if (b[1] !== a[1]) return b[1] - a[1];
+      return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
+    });
+    const top = ranked.slice(0, PURPOSE_MAX_COVERS).map(([cover]) => cover);
+    return top.length === 1 ? top[0] : top.join("; ");
+  }
+  const sorted = normalised
+    .map((data) => ({
+      id: data?.id ?? "",
+      // Mirror the single-member branch: only accept string focus
+      // values, normalise everything else to "". Without this, a
+      // hand-authored leaf with a non-string `focus:` (e.g. a YAML
+      // number `0` or `false`) would propagate through and make
+      // `deterministicPurpose()` return a non-string, breaking the
+      // documented contract that this helper always returns a
+      // string.
+      focus: typeof data?.focus === "string" ? data.focus : "",
+    }))
+    .filter((x) => x.id)
+    .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+  return sorted[0]?.focus ?? "";
+}
+
+// Local helper — a simple, stable non-crypto hash. `createHash` would
+// be fine but adds a node:crypto import and is overkill for a 7-char
+// slug suffix. FNV-1a 32-bit is widely used, stable across Node
+// versions, and deterministic on the same string input.
+function hashString(str) {
+  let h = 0x811c9dc5 >>> 0;
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i);
+    h = Math.imul(h, 0x01000193) >>> 0;
+  }
+  return h.toString(16).padStart(8, "0");
+}
+

package/scripts/lib/contract.mjs

@@ -65,8 +65,29 @@ const FRONTMATTER_SCHEMA = {
       shared_covers: { kind: "string[]" },
       // Only present (and required) when type === "overlay".
       overlay_targets: { kind: "string[]" },
-      links: { kind: "string[]" },
+      links: {
+        kind: "object[]",
+        description: "Cross-leaf references; each entry carries an `id` (and optional metadata) — see scripts/lib/join.mjs for how runtime code reads link.id.",
+      },
+      // Consumer-defined fields (e.g. skill-code-review's
+      // `dimensions`, `audit_surface`, `languages`, `tools`) are
+      // carried through rebuilds via the deny-list forwarding in
+      // draft.mjs; their VALUES are preserved (not dropped). Exact
+      // bytes can change because the renderer applies canonical
+      // top-level key ordering and YAML formatting. The contract
+      // here describes only the fields the wiki framework itself
+      // reads / writes; consumers ship their own schemas alongside.
     },
+    // Reserved fields that the rebuild ALWAYS re-derives from the
+    // target-tree position, regardless of what the author wrote.
+    // `parents` is NOT in this set — it's hand-authored when the
+    // soft-DAG layout requires it (the drafter picks the authored
+    // value over the heuristic).
+    reserved: ["id", "type", "depth_role", "source"],
+    // Deny-list semantics for everything else: any authored field not
+    // in `reserved` flows through verbatim. This keeps the wiki
+    // framework agnostic to consumer-specific schemas.
+    pass_through_authored: true,
   },
   index: {
     required: ["id", "type", "depth_role", "focus"],
@@ -135,6 +156,9 @@ const SUBCOMMANDS = {
       "--layout-mode",
       "--target",
       "--quality-mode",
+      "--fanout-target",
+      "--max-depth",
+      "--soft-dag-parents",
       "--no-prompt",
       "--accept-dirty",
       "--accept-foreign-target",
@@ -143,17 +167,42 @@ const SUBCOMMANDS = {
   },
   extend: {
     positionals: ["wiki"],
-    flags: ["--quality-mode", "--no-prompt", "--json"],
+    flags: [
+      "--quality-mode",
+      "--no-prompt",
+      "--json",
+    ],
   },
   validate: { positionals: ["wiki"], flags: ["--json"] },
   rebuild: {
     positionals: ["wiki"],
-    flags: ["--quality-mode", "--review", "--no-prompt", "--json"],
+    flags: [
+      "--quality-mode",
+      "--fanout-target",
+      "--max-depth",
+      "--soft-dag-parents",
+      "--review",
+      "--no-prompt",
+      "--json",
+    ],
   },
   fix: { positionals: ["wiki"], flags: ["--json"] },
   join: {
+    // Variadic positionals — the CLI accepts
+    // `join <wiki-a> <wiki-b> [<wiki-c>...]`. `positionals` lists
+    // the minimum shape; `min_positionals` / `variadic` describe
+    // the full contract so consumers generating invocations or
+    // validating argument counts don't assume exactly two sources.
     positionals: ["wiki-a", "wiki-b"],
-    flags: ["--target", "--canonical", "--json"],
+    min_positionals: 2,
+    variadic: true,
+    flags: [
+      "--target",
+      "--canonical",
+      "--id-collision",
+      "--quality-mode",
+      "--json",
+    ],
   },
   rollback: { positionals: ["wiki"], flags: ["--to", "--json"] },
   init: {