@ctxr/skill-llm-wiki 1.0.2 → 1.2.0

package/scripts/lib/orchestrator.mjs

@@ -25,6 +25,7 @@ import {
   mkdirSync,
   readFileSync,
   readdirSync,
+  rmSync,
   writeFileSync,
 } from "node:fs";
 import { basename, dirname, join, relative } from "node:path";
@@ -49,8 +50,17 @@ import {
   recordSource,
   startCorpus,
 } from "./provenance.mjs";
-import { rmSync } from "node:fs";
+import { MAX_BALANCE_ITERATIONS, runBalance } from "./balance.mjs";
+import {
+  FANOUT_TARGET_MAX,
+  FANOUT_TARGET_MIN,
+  MAX_DEPTH_MAX,
+  MAX_DEPTH_MIN,
+} from "./intent.mjs";
+import { applySoftParentEntries, runSoftDagParents } from "./soft-dag.mjs";
 import { runConvergence } from "./operators.mjs";
+import { runJoin } from "./join.mjs";
+import { runRootContainment } from "./root-containment.mjs";
 import { runReviewCycle } from "../commands/review.mjs";
 import {
   deriveBatchId,
@@ -60,6 +70,7 @@ import {
 } from "./tier2-protocol.mjs";
 import {
   clearTier2Responses,
+  resolveQualityMode,
   seedTier2Responses,
   takePendingRequests,
 } from "./tiered.mjs";
@@ -68,7 +79,22 @@ import {
 // { operation, layout_mode, source, target, is_new_wiki, flags }.
 // Returns { op_id, final_sha, phases: [...] } on success; throws on
 // validation failure (after rolling the working tree back to pre-op).
-export async function runOperation(plan, { opId, source, startedIso } = {}) {
+export async function runOperation(plan, {
+  opId,
+  source,
+  startedIso,
+  // Optional per-phase progress hook. Invoked inside `record()`
+  // so the caller (typically CLI) can surface phase-by-phase
+  // progress to the user in real time — a 596-leaf deterministic
+  // build otherwise prints nothing for 2+ minutes. Shape:
+  //   (phase: { name: string, summary: string, index: number }) => void
+  // The hook runs synchronously; any errors it throws are caught
+  // and swallowed inside `record()` so a misbehaving progress
+  // reporter can never halt the operation. When absent,
+  // `runOperation` is silent on phase progression (preserving the
+  // pre-X.9 behaviour tests may have relied on).
+  onProgress = null,
+} = {}) {
   if (!plan || !plan.target) {
     throw new Error("runOperation requires a resolved plan with a target");
   }
@@ -80,7 +106,141 @@ export async function runOperation(plan, { opId, source, startedIso } = {}) {
   mkdirSync(workDir, { recursive: true });
 
   const phases = [];
-  const record = (name, summary) => phases.push({ name, summary });
+  const record = (name, summary) => {
+    const phase = { name, summary };
+    phases.push(phase);
+    if (onProgress) {
+      // Two failure modes to cover: a synchronous throw, and a
+      // returned Promise that rejects (an `async` hook never
+      // throws synchronously — the rejection escapes as an
+      // unhandledRejection and would terminate the process under
+      // Node's default policy, violating the "progress-hook
+      // failures must never halt the op" contract). The try/catch
+      // handles the sync throw; `Promise.resolve(...).catch()`
+      // handles the async rejection. Both swallow silently — a
+      // user with a broken progress reporter should still get a
+      // successful op completion.
+      try {
+        const ret = onProgress({ name, summary, index: phases.length });
+        if (ret && typeof ret.then === "function") {
+          Promise.resolve(ret).catch(() => {
+            /* async progress-hook rejection silently swallowed */
+          });
+        }
+      } catch {
+        /* sync progress-hook throw silently swallowed */
+      }
+    }
+  };
+
+  // `join` is the one top-level operation whose pipeline shape does
+  // not match the build/rebuild/fix family. Instead of threading
+  // an "ingest from N wikis" path through the build-shaped phase
+  // cascade, we delegate to `scripts/lib/join.mjs::runJoin` which
+  // implements all 11 phases from `guide/operations/ingest/join.md`
+  // end-to-end on an already-prepared empty target. The caller
+  // (intent → CLI) is responsible for ensuring `plan.target` is a
+  // fresh empty directory before runOperation fires; runJoin writes
+  // the unified tree there and then runs the same
+  // convergence / indices / validation tail the build path would.
+  // Per-phase commits are routed through runJoin's onPhaseCommit
+  // callback so the private git log shows join progression at the
+  // same granularity a build does.
+  if (plan.operation === "join") {
+    if (!Array.isArray(plan.sources) || plan.sources.length < 2) {
+      throw new Error("join requires at least 2 resolved source wiki paths");
+    }
+    const snap = preOpSnapshot(wikiRoot, opId);
+    record(
+      "snapshot",
+      `tag ${snap.tag} sha=${(snap.sha ?? "n/a").slice(0, 12)}`,
+    );
+    try {
+      const result = await runJoin(plan.sources, wikiRoot, {
+        opId,
+        qualityMode: resolveQualityMode(plan.flags || {}),
+        idCollisionPolicy: plan.flags?.id_collision,
+        onPhaseCommit: async ({ phase, summary }) => {
+          gitRunChecked(wikiRoot, ["add", "-A"]);
+          if (!gitWorkingTreeClean(wikiRoot)) {
+            gitCommit(wikiRoot, `phase ${phase}: ${summary}`);
+          }
+        },
+        // Stream join's phases through the orchestrator's own
+        // `record()` the moment each one fires — that routes the
+        // phase into the outer `phases[]` AND invokes the CLI's
+        // `onProgress` breadcrumb callback in real time. Without
+        // this hook we'd only see join's phases relayed AFTER
+        // `runJoin` returns (the pre-X.9-followup post-call
+        // `for (const p of result.phases)` loop), which made the
+        // whole join look like one silent block in the breadcrumb
+        // stream. runJoin's `phaseLog` still accumulates for the
+        // returned-result shape, but the outer `phases[]` now
+        // gets populated live.
+        onPhase: ({ phase, summary }) => {
+          record(phase, summary);
+        },
+      });
+      // Tier 2 suspend: convergence parked decisions that can't be
+      // resolved without a sub-agent. Drain the pending queue,
+      // write the batch, and throw NeedsTier2Error — the caller
+      // catches it via the same path the build/rebuild uses, exits
+      // 7, and the wiki-runner spawns sub-agents, writes responses,
+      // and re-invokes the CLI. Must happen BEFORE the op tag so a
+      // resume picks up at the same pre-op/... anchor.
+      if (result.needs_tier2) {
+        const requests = takePendingRequests(wikiRoot);
+        if (requests.length > 0) {
+          const batchId = deriveBatchId(
+            opId,
+            "join-convergence",
+            result.convergence.iterations,
+          );
+          const path = writePending(wikiRoot, batchId, requests);
+          throw new NeedsTier2Error(
+            `join: operator-convergence parked ${requests.length} Tier 2 request(s) ` +
+              `(batch ${batchId}); wiki-runner must resolve and re-invoke`,
+            opId,
+            path,
+          );
+        }
+      }
+      // Finalise — tag the op and emit the op-log entry.
+      const finalTag = `op/${opId}`;
+      gitTag(wikiRoot, finalTag);
+      const finalSha = gitHeadSha(wikiRoot);
+      appendOpLog(wikiRoot, {
+        op_id: opId,
+        operation: "join",
+        layout_mode: plan.layout_mode,
+        started: startedIso || new Date().toISOString(),
+        finished: new Date().toISOString(),
+        base_commit: snap.sha || "",
+        final_commit: finalSha || "",
+        summary:
+          `join target=${plan.target} sources=${plan.sources.length} ` +
+          `mode=${plan.layout_mode} phases=${phases.length}`,
+      });
+      record("commit-finalize", `tagged ${finalTag}`);
+      return { op_id: opId, final_sha: finalSha, phases };
+    } catch (err) {
+      // NeedsTier2Error is NOT a failure — it's a suspend signal.
+      // Don't roll back the working tree; the wiki-runner will
+      // resume on re-invoke. Matches the build-path semantics.
+      if (err instanceof NeedsTier2Error) {
+        throw err;
+      }
+      // Roll working tree back to the pre-op snapshot, matching the
+      // build path's failure semantics.
+      try {
+        gitResetHard(wikiRoot, snap.tag);
+        gitClean(wikiRoot);
+      } catch {
+        /* best-effort rollback */
+      }
+      throw err;
+    }
+  }
 
   // Map of authored index hints keyed by POSIX-relative directory
   // path from the wiki root. Populated in the ingest phase for Build
@@ -354,7 +514,14 @@ export async function runOperation(plan, { opId, source, startedIso } = {}) {
     }
     const convergence = await runConvergence(wikiRoot, {
       opId,
-      qualityMode: plan.flags?.quality_mode || "tiered-fast",
+      // Resolve through tiered.mjs::resolveQualityMode so the
+      // `LLM_WIKI_QUALITY_MODE` env var is actually consulted (the
+      // prior `plan.flags?.quality_mode || "tiered-fast"` shortcut
+      // silently dropped the env-var path documented in
+      // methodology.md). Intent-layer INT-13 already catches an
+      // invalid flag value before we reach here; an invalid env
+      // var surfaces from `resolveQualityMode` as a plain throw.
+      qualityMode: resolveQualityMode(plan.flags || {}),
       interactive: false, // orchestrator runs non-interactive
       commitBetweenIterations: async ({ iteration, operator, summary }) => {
         gitRunChecked(wikiRoot, ["add", "-A"]);
@@ -390,14 +557,235 @@ export async function runOperation(plan, { opId, source, startedIso } = {}) {
         `${convergence.suggestions.length} suggestion(s) recorded`,
     );
 
+    // Phase 4.3 — balance enforcement. Runs only when at least one
+    // of `--fanout-target` / `--max-depth` is set on the plan (both
+    // validated at intent time). No-op otherwise. Iterates until
+    // fixed point applying two transform classes:
+    //   - Sub-cluster an overfull directory (movable leaf count >
+    //     target × 1.5 — subdirs are structurally cemented and can't
+    //     be carved by the math cluster detector, so a dir overfull
+    //     *only* due to subdirs is un-actionable and is skipped) via
+    //     the math cluster detector + deterministic naming, reusing
+    //     the same helpers Phase X.3 built for the deterministic
+    //     quality mode so two runs on the same tree produce identical
+    //     sub-clusters.
+    //   - Flatten an overdeep single-child passthrough by promoting
+    //     its only subdir up one level. Descendants' `parents[]`
+    //     paths are left unchanged — they are relative to the direct
+    //     parent's `index.md`, so promoting an entire subtree up one
+    //     level preserves every relative path by construction. Only
+    //     pure passthroughs qualify; multi-child subcategories would
+    //     lose structure.
+    // The phase has its own commit cadence (one commit per apply);
+    // the same git-add + git-commit callback convergence uses wires
+    // into the private-git machinery. A no-op run (no overfull /
+    // overdeep candidates) leaves the working tree byte-identical.
+    // Balance-enforcement is build/rebuild-only per contract.mjs and
+    // `--help`. Intent-resolution already rejects the flags on other
+    // subcommands (INT-14a / INT-15a) and rejects non-integer values
+    // (INT-14 / INT-15), but gate here as well in defense-in-depth —
+    // any future code path that constructs a plan with balance flags
+    // outside of build/rebuild (or with garbage values) would otherwise
+    // trigger structural mutations outside the documented surface.
+    // `Number.isFinite` rejects NaN from a non-numeric string: a NaN
+    // maxDepth would make `detectDepthOverage` treat every depth as
+    // over (NaN comparisons are false), potentially flattening the
+    // wiki root and moving directories outside the wiki.
+    // Single source of truth for the set of operations that can
+    // reach the post-convergence hooks (balance enforcement + soft-
+    // DAG synthesis). Named for the actual invariant (operation
+    // membership), not a specific feature, so adding a third
+    // post-convergence phase in the future doesn't demand another
+    // one-purpose constant.
+    const BUILD_REBUILD_OPS = new Set(["build", "rebuild"]);
+    // Strict parse: must be a canonical base-10 integer string (no
+    // leading `+`/`-`, no trailing garbage, no decimal parts, no
+    // scientific notation), and must fall inside the intent-validated
+    // range for its flag. `Number.parseInt` would accept `"3.5"` as
+    // `3` and `"5xyz"` as `5`, which an adversarial or faulty caller
+    // could use to bypass the range bounds. `/^\d+$/` + explicit
+    // bounds keep orchestrator behaviour aligned with what intent
+    // resolution already enforces (INT-14 / INT-15).
+    const parseBalanceInt = (raw, min, max) => {
+      if (raw == null) return null;
+      const s = String(raw);
+      if (!/^\d+$/.test(s)) return null;
+      const n = Number.parseInt(s, 10);
+      if (!Number.isFinite(n)) return null;
+      if (n < min || n > max) return null;
+      return n;
+    };
+    const opSupportsBalance = BUILD_REBUILD_OPS.has(plan.operation);
+    const fanoutTarget = opSupportsBalance
+      ? parseBalanceInt(plan.flags?.fanout_target, FANOUT_TARGET_MIN, FANOUT_TARGET_MAX)
+      : null;
+    const maxDepth = opSupportsBalance
+      ? parseBalanceInt(plan.flags?.max_depth, MAX_DEPTH_MIN, MAX_DEPTH_MAX)
+      : null;
+    let balance = null;
+    if (fanoutTarget != null || maxDepth != null) {
+      balance = await runBalance(wikiRoot, {
+        opId,
+        // Same resolveQualityMode indirection as the convergence
+        // phase above — keeps the env-var path working on builds
+        // that use the balance-enforcement flags.
+        qualityMode: resolveQualityMode(plan.flags || {}),
+        fanoutTarget,
+        maxDepth,
+        commitBetweenIterations: async ({ iteration, operator, summary }) => {
+          gitRunChecked(wikiRoot, ["add", "-A"]);
+          if (!gitWorkingTreeClean(wikiRoot)) {
+            gitCommit(
+              wikiRoot,
+              `phase balance-enforcement: iteration ${iteration} ${operator} — ${summary}`,
+            );
+          }
+        },
+      });
+      record(
+        "balance-enforcement",
+        `${balance.applied.length} operation(s) applied across ` +
+          `${balance.iterations} iteration(s); converged=${balance.converged}`,
+      );
+      if (!balance.converged) {
+        // Enforcement contract: a user who asked for a balanced tree
+        // expects the post-convergence shape to honour `--fanout-target`
+        // / `--max-depth`. Hitting the 20-iteration cap means the
+        // rebalance didn't reach a fixed point — any downstream
+        // assumption "the tree is now balanced" would silently be
+        // wrong. Fail loud here so the orchestrator's pre-op snapshot
+        // restores and the user sees the problem, instead of shipping
+        // a half-balanced wiki with no error.
+        throw new Error(
+          `balance enforcement did not converge after ${balance.iterations} ` +
+            `iteration(s) (cap=${MAX_BALANCE_ITERATIONS}); applied ` +
+            `${balance.applied.length} op(s). Inspect the private git ` +
+            `history via the skill's hidden-git passthrough (e.g. ` +
+            `\`node scripts/cli.mjs log ${wikiRoot}\`) — the private repo ` +
+            `lives under \`<wiki>/.llmwiki/git\` and requires the skill's ` +
+            `isolation env, which the passthrough wraps. Reduce ` +
+            `--fanout-target / --max-depth strictness, or file a ` +
+            `ping-pong repro.`,
+        );
+      }
+    }
+
+    // Phase 4.4 — optional soft-DAG parent synthesis. Runs only when
+    // the user passed `--soft-dag-parents` on build / rebuild (intent
+    // rejects the flag everywhere else via INT-16a). Writes soft
+    // parents into each routable leaf's `parents[]` frontmatter based
+    // on TF-IDF cosine similarity against candidate category
+    // directories. The downstream `applySoftParentEntries` pass, which
+    // runs in Phase 5 AFTER `rebuildAllIndices`, propagates each
+    // leaf's soft claims into the corresponding index `entries[]`
+    // arrays so a Claude navigator arriving at any claimed parent
+    // sees the leaf in that parent's entries.
+    //
+    // Fan-in control is via `SOFT_PARENT_MAX_PER_LEAF` (default 3) and
+    // `SOFT_PARENT_AFFINITY_THRESHOLD` (default 0.35). Both exported
+    // from `soft-dag.mjs` — tests that want boundary behaviour can
+    // pass overrides, but the CLI flag is a pure boolean.
+    const softDagRequested =
+      plan.flags?.soft_dag_parents === true && BUILD_REBUILD_OPS.has(plan.operation);
+    let softDag = null;
+    // Tracks whether the soft-DAG phase actually created a commit
+    // (not just whether it found soft parents). A run with zero
+    // softParentsAdded can still dirty the tree — e.g., a rerun
+    // that removes previously-synthesised soft parents now below
+    // threshold, or a canonical-order frontmatter rewrite that
+    // leaves no net soft-parent change but still alters bytes. The
+    // `--review` gate below consumes this flag to decide whether
+    // Phase 4.4 contributed to the diff.
+    let softDagDidCommit = false;
+    if (softDagRequested) {
+      softDag = await runSoftDagParents(wikiRoot);
+      record(
+        "soft-dag-parents",
+        `${softDag.leavesProcessed} leaf/leaves scanned; ` +
+          `${softDag.softParentsAdded} soft-parent pointer(s) selected`,
+      );
+      // Commit the frontmatter rewrites as a single iteration so the
+      // private-git history preserves the DAG synthesis as a
+      // distinguishable artefact. Review cycle picks up this commit
+      // via the anyMutation gate below.
+      gitRunChecked(wikiRoot, ["add", "-A"]);
+      if (!gitWorkingTreeClean(wikiRoot)) {
+        // Commit subject stays generic over the leaf count alone.
+        // A per-run delta (added / removed / reordered) against the
+        // pre-run parents[] would be the ideal audit trail, but the
+        // orchestrator doesn't snapshot pre-run frontmatter state
+        // and wiring that plumbing adds cost for a diagnostic
+        // message; the per-phase record() call above already surfaces
+        // `softParentsAdded` (the count of selected pointers this
+        // run, which equals the "pointers present after synthesis"
+        // since runSoftDagParents rewrites parents[] top-to-bottom
+        // from scratch each pass). The private-git diff on the
+        // commit itself is the byte-exact source of truth.
+        gitCommit(
+          wikiRoot,
+          `phase soft-dag-parents: parents[] synthesis across ` +
+            `${softDag.leavesProcessed} leaf/leaves`,
+        );
+        softDagDidCommit = true;
+      }
+    }
+
+    // Phase 4.4.5 — root-leaf containment (X.11). Invariant: the wiki
+    // root holds only `index.md` plus subdirectories — never a direct
+    // `.md` leaf. Any outlier leaf that survived convergence + balance
+    // (because its affinity to every other leaf fell below clustering
+    // thresholds) is moved into its own semantically-named
+    // subcategory derived from its own TF-IDF distinguishing tokens.
+    // Runs BEFORE Phase 4.5 review so the containment commit is part
+    // of the reviewable diff — users can drop/abort it like any
+    // other tree-mutating phase. Runs BEFORE Phase 5 so
+    // `rebuildAllIndices` sees the final tree shape (and the new
+    // `<slug>/index.md` stubs get their `entries[]` populated as part
+    // of the regular pass, not a separate write).
+    let containmentDidCommit = false;
+    const containment = await runRootContainment(wikiRoot);
+    if (containment.moved > 0) {
+      record(
+        "root-containment",
+        `moved ${containment.moved} outlier leaf/leaves into per-slug subcategories`,
+      );
+      gitRunChecked(wikiRoot, ["add", "-A"]);
+      if (!gitWorkingTreeClean(wikiRoot)) {
+        gitCommit(
+          wikiRoot,
+          `phase root-containment: moved ${containment.moved} outlier(s) into subcategories`,
+        );
+        containmentDidCommit = true;
+      }
+    }
+
     // Phase 4.5 — optional interactive review. Fires only when the
-    // user passed --review AND convergence actually produced at
-    // least one commit. The review flow prints a diff + commit
-    // list and lets the user approve, abort, or drop specific
-    // iterations before validation runs. Abort throws so the
-    // orchestrator's catch block handles the rollback uniformly
-    // with any other failure path.
-    if (plan.flags?.review && convergence.applied.length > 0) {
+    // user passed --review AND at least one tree-mutating phase
+    // actually produced commits. That's ANY of convergence, the
+    // balance-enforcement phase, the soft-DAG parent phase above, or
+    // the root-containment phase — each commits separately via its
+    // own git callback. Pre-round-6 the gate only checked
+    // convergence, which meant a no-op-convergence + active-balance
+    // op (e.g. a hand-authored corpus that's already operator-clean
+    // but violates the fanout/depth targets) would silently skip
+    // review. The review flow prints a diff + commit list and lets
+    // the user approve, abort, or drop specific iterations before
+    // validation runs. Abort throws so the orchestrator's catch
+    // block handles the rollback uniformly with any other failure
+    // path.
+    const balanceApplied = balance?.applied?.length ?? 0;
+    // Use the commit-did-fire flag for soft-DAG and root-containment,
+    // not their planned-work counters: a rerun that removes
+    // previously-synthesised soft parents below threshold, or a
+    // canonical-order frontmatter rewrite that preserves logical
+    // content while altering bytes, produces the planned-work
+    // counter = 0 but still dirties the tree and commits.
+    const anyMutation =
+      convergence.applied.length > 0 ||
+      balanceApplied > 0 ||
+      softDagDidCommit ||
+      containmentDidCommit;
+    if (plan.flags?.review && anyMutation) {
       const reviewResult = await runReviewCycle(wikiRoot, opId, {
         forceInteractive: plan.flags?.force_interactive === true,
       });
@@ -429,11 +817,30 @@ export async function runOperation(plan, { opId, source, startedIso } = {}) {
     // the rebuild pass can fill them in with frontmatter.
     bootstrapIndexStubs(wikiRoot);
     const rebuilt = rebuildAllIndices(wikiRoot, { indexInputs });
+    // Phase 5.1 — soft-DAG propagation. When Phase 4.4 synthesised
+    // soft-parent pointers, `rebuildAllIndices` has just placed every
+    // leaf in its DIRECT parent's `entries[]` (the primary parent) —
+    // but the DAG view requires leaves to also show up under every
+    // parent they claim. `applySoftParentEntries` walks each leaf's
+    // `parents[]` and appends its record into each claimed non-primary
+    // parent's index. Idempotent: records are deduped by id, so
+    // re-running the pass produces byte-identical output.
+    let softDagAppends = null;
+    if (softDagRequested) {
+      softDagAppends = applySoftParentEntries(wikiRoot);
+      record(
+        "soft-dag-propagate",
+        `${softDagAppends.indicesTouched} index(es) updated; ` +
+          `${softDagAppends.softEntriesAdded} soft-entry record(s) added`,
+      );
+    }
     gitRunChecked(wikiRoot, ["add", "-A"]);
     if (!gitWorkingTreeClean(wikiRoot)) {
       gitCommit(
         wikiRoot,
-        `phase index-generation: rebuilt ${rebuilt.length} index.md files`,
+        softDagRequested
+          ? `phase index-generation: rebuilt ${rebuilt.length} index.md files + soft-DAG propagation`
+          : `phase index-generation: rebuilt ${rebuilt.length} index.md files`,
       );
     }
     record("index-generation", `rebuilt ${rebuilt.length} indices`);