mandrel 1.93.0 → 1.94.0

package/.agents/scripts/lib/orchestration/bookkeeping-outbox.js

@@ -0,0 +1,270 @@
+/**
+ * bookkeeping-outbox.js — Epic #4476 (M5): buffered GitHub bookkeeping.
+ *
+ * Delivery bookkeeping — structured-comment upserts (progress, friction,
+ * wave-stall, …) and `agent::*` label flips — is human-visible surface that,
+ * in an **unattended** (`--yes` / headless) run, no operator is watching in
+ * real time. Emitting each one as a live GitHub round-trip mid-run costs an
+ * LLM turn per transition for a surface nobody reads until the run finishes.
+ *
+ * This module lets those non-urgent mutations be **buffered to a local NDJSON
+ * outbox** during the run and **reconciled to GitHub once at finalize**.
+ * GitHub stays the source of truth *at rest* (post-reconcile); only the
+ * per-transition chatter moves off the token stream. The outbox file survives
+ * a crash, so crash recovery drains whatever was buffered before the finalize
+ * reconcile ran.
+ *
+ * ## What is NEVER buffered
+ * The `agent::blocked` HITL gate (`.agents/instructions.md` §1.J) is the single
+ * authoritative runtime pause point. A genuine blocker MUST surface on GitHub
+ * **immediately**, not batched to finalize — an operator can only resume a run
+ * they can see is blocked. The {@link transitionStateOrBuffer} facade forces
+ * `agent::blocked` (and any caller-marked `urgent`) transition through the
+ * live path regardless of headless mode.
+ *
+ * ## Attended runs are unchanged
+ * Buffering is gated on the explicit `headless` signal (Story #4427's
+ * `--headless`/`--yes` plumbing). An attended run passes `headless: false` and
+ * every comment/label posts live exactly as before — byte-for-byte behaviour.
+ */
+
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+} from 'node:fs';
+import path from 'node:path';
+
+import { epicTempDir } from '../config/temp-paths.js';
+import { STATE_LABELS } from './ticketing/reads.js';
+import { upsertStructuredComment } from './ticketing/state.js';
+import { transitionTicketState } from './ticketing/transition.js';
+
+/** Canonical basename for the per-Epic bookkeeping outbox. */
+const OUTBOX_BASENAME = 'bookkeeping-outbox.ndjson';
+
+/**
+ * Resolve the canonical outbox path for an Epic:
+ * `temp/epic-<id>/bookkeeping-outbox.ndjson`.
+ *
+ * @param {number} epicId
+ * @param {object} [config] Resolved config (tempRoot).
+ * @returns {string}
+ */
+export function outboxPathFor(epicId, config) {
+  return path.join(epicTempDir(epicId, config), OUTBOX_BASENAME);
+}
+
+/**
+ * Append one operation record to the outbox, creating the parent dir on
+ * demand. Best-effort at the storage layer is the caller's concern; here we
+ * let a genuine fs failure propagate so a mis-configured outbox path surfaces
+ * loudly in tests.
+ *
+ * @param {string} outboxPath
+ * @param {object} op
+ */
+function appendOp(outboxPath, op) {
+  mkdirSync(path.dirname(outboxPath), { recursive: true });
+  appendFileSync(outboxPath, `${JSON.stringify(op)}\n`, 'utf8');
+}
+
+/**
+ * Buffer a structured-comment upsert. The op captures everything
+ * `upsertStructuredComment` needs at drain time so the reconcile is a pure
+ * replay with no re-derivation.
+ *
+ * @param {{ outboxPath: string, ticketId: number, marker: string,
+ *           body: string, attrs?: Record<string, string|number>|null,
+ *           ts?: string }} args
+ */
+export function enqueueComment({
+  outboxPath,
+  ticketId,
+  marker,
+  body,
+  attrs = null,
+  ts = new Date().toISOString(),
+}) {
+  appendOp(outboxPath, {
+    kind: 'comment',
+    ts,
+    ticketId,
+    marker,
+    body,
+    ...(attrs ? { attrs } : {}),
+  });
+}
+
+/**
+ * Buffer an `agent::*` label transition.
+ *
+ * @param {{ outboxPath: string, ticketId: number, state: string,
+ *           ts?: string }} args
+ */
+export function enqueueLabel({
+  outboxPath,
+  ticketId,
+  state,
+  ts = new Date().toISOString(),
+}) {
+  appendOp(outboxPath, { kind: 'label', ts, ticketId, state });
+}
+
+/**
+ * Parse the outbox into an ordered array of operation records. Malformed
+ * lines are skipped (never throw). A missing / empty outbox yields `[]`.
+ *
+ * @param {string} outboxPath
+ * @returns {Array<object>}
+ */
+export function readOutbox(outboxPath) {
+  if (!outboxPath || !existsSync(outboxPath)) return [];
+  let raw;
+  try {
+    raw = readFileSync(outboxPath, 'utf8');
+  } catch {
+    return [];
+  }
+  if (!raw) return [];
+  const ops = [];
+  for (const line of raw.split(/\r?\n/)) {
+    if (!line) continue;
+    try {
+      const op = JSON.parse(line);
+      if (op && typeof op === 'object') ops.push(op);
+    } catch {
+      // skip a torn / malformed line — a partial write from a crash is not
+      // fatal to the rest of the batch.
+    }
+  }
+  return ops;
+}
+
+/**
+ * Drain the outbox to GitHub in FIFO order and clear it on success. Both sink
+ * operations are idempotent (comment upserts are marker-scoped;
+ * `transitionTicketState` applies one state via the canonical
+ * remove-all-then-add path), so a re-run after a partial drain converges. A
+ * per-op failure is recorded and the drain continues — a single bad ticket
+ * must not strand the rest of the batch. The outbox file is only truncated
+ * when EVERY op succeeded, so a crash mid-drain leaves the un-applied
+ * remainder for the next reconcile.
+ *
+ * @param {{ outboxPath: string,
+ *           provider: import('../ITicketingProvider.js').ITicketingProvider,
+ *           logger?: { warn?: (m: string) => void } }} args
+ * @returns {Promise<{ drained: number, comments: number, labels: number,
+ *   errors: Array<{ op: object, error: string }>, cleared: boolean }>}
+ */
+export async function reconcileOutbox({ outboxPath, provider, logger }) {
+  const ops = readOutbox(outboxPath);
+  const result = {
+    drained: 0,
+    comments: 0,
+    labels: 0,
+    errors: [],
+    cleared: false,
+  };
+  if (ops.length === 0) {
+    result.cleared = true;
+    return result;
+  }
+
+  for (const op of ops) {
+    try {
+      if (op.kind === 'comment') {
+        await upsertStructuredComment(
+          provider,
+          op.ticketId,
+          op.marker,
+          op.body,
+          op.attrs ?? null,
+        );
+        result.comments += 1;
+        result.drained += 1;
+      } else if (op.kind === 'label') {
+        await transitionTicketState(provider, op.ticketId, op.state);
+        result.labels += 1;
+        result.drained += 1;
+      } else {
+        result.errors.push({ op, error: `unknown op kind "${op.kind}"` });
+      }
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      result.errors.push({ op, error: message });
+      logger?.warn?.(
+        `[bookkeeping-outbox] reconcile op failed (${op.kind} #${op.ticketId}): ${message}`,
+      );
+    }
+  }
+
+  // Only truncate when the whole batch landed — otherwise the un-applied
+  // ops must survive for the next reconcile (crash-recovery contract).
+  if (result.errors.length === 0) {
+    try {
+      writeFileSync(outboxPath, '', 'utf8');
+      result.cleared = true;
+    } catch {
+      // A clear failure is non-fatal: the ops all landed, and the idempotent
+      // sinks make a redundant re-drain a no-op.
+    }
+  }
+  return result;
+}
+
+/**
+ * Facade: post a structured comment live, OR buffer it to the outbox when the
+ * run is headless. Comments are never urgent (the operator reads them at
+ * finalize), so headless always buffers when an `outboxPath` is available.
+ *
+ * @param {{ provider: object, ticketId: number, marker: string, body: string,
+ *           attrs?: Record<string, string|number>|null, headless?: boolean,
+ *           outboxPath?: string|null }} args
+ * @returns {Promise<{ buffered: boolean }>}
+ */
+export async function postCommentOrBuffer({
+  provider,
+  ticketId,
+  marker,
+  body,
+  attrs = null,
+  headless = false,
+  outboxPath = null,
+}) {
+  if (headless && outboxPath) {
+    enqueueComment({ outboxPath, ticketId, marker, body, attrs });
+    return { buffered: true };
+  }
+  await upsertStructuredComment(provider, ticketId, marker, body, attrs);
+  return { buffered: false };
+}
+
+/**
+ * Facade: flip an `agent::*` state live, OR buffer it when headless — EXCEPT
+ * `agent::blocked` (and any caller-marked `urgent` flip), which ALWAYS goes
+ * live so the HITL gate surfaces immediately (§1.J).
+ *
+ * @param {{ provider: object, ticketId: number, state: string,
+ *           headless?: boolean, outboxPath?: string|null,
+ *           urgent?: boolean }} args
+ * @returns {Promise<{ buffered: boolean }>}
+ */
+export async function transitionStateOrBuffer({
+  provider,
+  ticketId,
+  state,
+  headless = false,
+  outboxPath = null,
+  urgent = false,
+}) {
+  const mustSurfaceNow = urgent || state === STATE_LABELS.BLOCKED;
+  if (headless && outboxPath && !mustSurfaceNow) {
+    enqueueLabel({ outboxPath, ticketId, state });
+    return { buffered: true };
+  }
+  await transitionTicketState(provider, ticketId, state);
+  return { buffered: false };
+}

package/.agents/scripts/lib/orchestration/ceremony-routing.js

@@ -0,0 +1,141 @@
+/**
+ * lib/orchestration/ceremony-routing.js — risk-routed acceptance ceremony
+ * resolver (Epic #4478, M7-B, Part 2).
+ *
+ * The sibling of `review-depth.js` (risk → review depth) and
+ * `code-review.js#resolveAuditLenses` (risk → audit lens): it folds the
+ * planner-judged risk envelope into a per-cluster ceremony decision for the
+ * single-delivery acceptance critic — **fresh-context spawn** vs the
+ * contract-identical **inline** critic. It does NOT invent a new risk score
+ * and it does NOT own clustering.
+ *
+ * ## The load-bearing invariant (M4-B acceptance floor — DO NOT VIOLATE)
+ *
+ * Risk-routing chooses fresh-vs-inline **PER CLUSTER**. It NEVER changes the
+ * cluster COUNT. The cluster count is `ceil(totalACs / clusterCeiling)` with
+ * the non-disableable `[1, 8]` clamp, owned entirely by
+ * `acceptance-clusters.js` and untouched here. A low-risk Epic still gets one
+ * verdict per cluster — just possibly authored inline instead of by a fresh
+ * sub-agent. This module takes the cluster index as an INPUT and returns a
+ * decision for that one cluster; it has no way to add or remove clusters.
+ *
+ * ## Tier rules (per cluster)
+ *
+ *   - `high` risk        → `fresh`   (a fresh-context maker-blind spawn).
+ *   - `medium` risk      → `fresh`   (fail toward more ceremony, never less —
+ *                                     matches `review-depth`'s fail-to-middle).
+ *   - `low` risk         → `inline`  (the contract-identical inline critic),
+ *                                     UNLESS the maker-checker sampling floor
+ *                                     selects this cluster → `fresh`.
+ *   - missing / unknown  → `fresh`   (fail-safe: an Epic that skipped `/plan`
+ *                                     has no risk verdict; treat it as needing
+ *                                     the full fresh-context ceremony, exactly
+ *                                     as `review-depth` degrades to `standard`
+ *                                     and `deriveRiskEnvelope` degrades to
+ *                                     review-required).
+ *
+ * ## Maker-checker sampling floor
+ *
+ * Even at `low` risk, a fraction of clusters (`freshCriticSampleRate`, default
+ * 0.2) is forced `fresh` so low risk never means zero independent checking. The
+ * selection is **deterministic** in the cluster index (a fixed stride), so it
+ * is stable across re-runs and — critically — never changes the cluster count:
+ * it only re-labels which of the fixed set of clusters run fresh.
+ *
+ * Pure and total: inputs in, decision out. No I/O, no throws. `null` /
+ * `undefined` / malformed inputs degrade to `fresh` + `full` ceremony.
+ *
+ * @typedef {'fresh'|'inline'} CeremonyMode
+ * @typedef {'low'|'medium'|'high'} RiskLevel
+ */
+
+/**
+ * Decide whether the sampling floor forces this low-risk cluster fresh.
+ *
+ * Deterministic in the cluster index: with rate `r` (0 < r ≤ 1) the stride is
+ * `round(1 / r)` and every `stride`-th cluster (0-based indices 0, stride,
+ * 2·stride, …) is forced fresh, yielding ≈`r` of clusters fresh. `r <= 0`
+ * disables the floor (no cluster forced); `r >= 1` forces every cluster.
+ *
+ * @param {number} clusterIndex  Zero-based cluster position (from the fixed
+ *   `ceil(totalACs / clusterCeiling)` fan-out — an INPUT, never mutated here).
+ * @param {number} rate          Sampling rate, already clamped into [0, 1] by
+ *   `getDeliveryRouting`.
+ * @returns {boolean} `true` when the floor forces this cluster fresh.
+ */
+export function sampledFresh(clusterIndex, rate) {
+  if (typeof rate !== 'number' || !Number.isFinite(rate) || rate <= 0) {
+    return false;
+  }
+  if (rate >= 1) return true;
+  const idx =
+    typeof clusterIndex === 'number' &&
+    Number.isInteger(clusterIndex) &&
+    clusterIndex >= 0
+      ? clusterIndex
+      : 0;
+  const stride = Math.max(1, Math.round(1 / rate));
+  return idx % stride === 0;
+}
+
+/**
+ * Resolve the acceptance ceremony for one cluster from the judged risk level
+ * and the maker-checker sampling floor. See the module header for the tier
+ * rules and the untouchable cluster-count invariant.
+ *
+ * @param {{
+ *   overallLevel?: (RiskLevel|string|null|undefined),
+ *   clusterIndex?: (number|null|undefined),
+ *   freshCriticSampleRate?: (number|null|undefined),
+ * }} [input]
+ * @returns {{ mode: CeremonyMode, reason: string, sampled: boolean }}
+ */
+export function resolveCeremonyForRisk(input = {}) {
+  const overallLevel =
+    input && typeof input === 'object' ? input.overallLevel : undefined;
+  const clusterIndex =
+    input && typeof input === 'object' ? input.clusterIndex : undefined;
+  const rate =
+    input && typeof input === 'object'
+      ? input.freshCriticSampleRate
+      : undefined;
+
+  if (overallLevel === 'high') {
+    return {
+      mode: 'fresh',
+      reason: 'high-risk: fresh-context critic',
+      sampled: false,
+    };
+  }
+  if (overallLevel === 'medium') {
+    return {
+      mode: 'fresh',
+      reason: 'medium-risk: fresh-context critic (fail toward more ceremony)',
+      sampled: false,
+    };
+  }
+  if (overallLevel === 'low') {
+    if (sampledFresh(clusterIndex, rate)) {
+      return {
+        mode: 'fresh',
+        reason:
+          'low-risk cluster forced fresh by the maker-checker sampling floor',
+        sampled: true,
+      };
+    }
+    return {
+      mode: 'inline',
+      reason: 'low-risk: contract-identical inline critic',
+      sampled: false,
+    };
+  }
+  // Missing / unknown / malformed risk → fail-safe fresh + full ceremony,
+  // matching how review-depth.js and deriveRiskEnvelope degrade on an
+  // unjudged Epic.
+  return {
+    mode: 'fresh',
+    reason:
+      'risk absent/unknown: fail-safe fresh-context critic + full ceremony',
+    sampled: false,
+  };
+}

package/.agents/scripts/lib/orchestration/context-hydration-engine.js

@@ -35,12 +35,10 @@ import {
   elideEnvelope,
   envelopeToPrompt,
 } from './context-envelope.js';
-import { loadSkillCapsule } from './skill-capsule-loader.js';
 
 // ---------------------------------------------------------------------------
-// File-content cache — the agent-protocol template and persona files are
-// read-only during a dispatch run. Skill bodies are loaded via
-// `skills.index.json` + `loadSkillCapsule` (not cached here).
+// File-content cache — the agent-protocol template is read-only during a
+// dispatch run.
 // ---------------------------------------------------------------------------
 
 const _fileCache = new Map();
@@ -52,81 +50,6 @@ function readFileCached(absPath) {
   return content;
 }
 
-let _skillsIndexCache = null;
-
-function loadSkillsIndex() {
-  if (!_skillsIndexCache) {
-    const indexPath = path.join(
-      PROJECT_ROOT,
-      '.agents',
-      'skills',
-      'skills.index.json',
-    );
-    _skillsIndexCache = JSON.parse(fs.readFileSync(indexPath, 'utf8'));
-  }
-  return _skillsIndexCache;
-}
-
-/**
- * Test-only seam: clear the persona/skill/template cache between runs.
- * The `__` prefix matches the project convention for test-only exports
- * (see `git-utils.__setGitRunners`, `git-utils.__setSleep`).
- */
-export function __resetContextCache() {
-  _fileCache.clear();
-  _skillsIndexCache = null;
-}
-
-/**
- * Resolve activated skills to Policy Capsule payloads via `skills.index.json`.
- *
- * Capsule-only is the contract (Story #3863, hard cutover): only the Policy
- * Capsule is hydrated. The full `SKILL.md` body is never inlined into a task
- * prompt — the sub-agent reads it on demand via the rendered pointer path.
- *
- * @param {object} task - Normalized task (skills[]).
- * @param {object} skillsIndex - Parsed `skills.index.json` body.
- * @returns {Array<{ skill: string, capsule: string, source: string, path: string }>}
- */
-export function buildSkillCapsuleSections(task, skillsIndex) {
-  const entries = [];
-
-  for (const skill of task.skills ?? []) {
-    try {
-      const {
-        capsule,
-        source,
-        path: skillPath,
-      } = loadSkillCapsule(skill, skillsIndex);
-      entries.push({ skill, capsule, source, path: skillPath });
-    } catch (err) {
-      Logger.warn(`[Hydrator] Failed to load skill ${skill}: ${err.message}`);
-    }
-  }
-
-  return entries;
-}
-
-/**
- * Render skill capsule entries for the prose prompt and envelope
- * `skillCapsules` section. Each entry carries its capsule plus a pointer
- * instruction so the sub-agent knows to `Read` the full `SKILL.md` when the
- * task needs the playbook beyond the capsule's non-negotiables. The source is
- * recorded per skill for auditors.
- *
- * @param {Array<{ skill: string, capsule: string, source: string, path: string }>} entries
- * @returns {string}
- */
-export function formatSkillCapsulesSection(entries) {
-  if (!entries.length) return '';
-  let out = '## Activated Skills\n\n';
-  for (const { skill, capsule, source, path: skillPath } of entries) {
-    out += `### Skill: ${skill} (source: ${source})\n${capsule}\n\n`;
-    out += `Read the full playbook on demand: \`Read ${skillPath}\`.\n\n`;
-  }
-  return out.trimEnd();
-}
-
 // ---------------------------------------------------------------------------
 // Internal helpers
 // ---------------------------------------------------------------------------
@@ -377,8 +300,6 @@ function envelopeTaskFrom(task) {
   return {
     id: task.id,
     title: task.title,
-    persona: task.persona,
-    skills: task.skills,
     protocolVersion: task.protocolVersion,
   };
 }
@@ -446,7 +367,7 @@ async function buildHierarchySections(task, provider, epicId, agentSettings) {
 
 /**
  * @param {object} task
- * @param {{ templatesRoot: string, personasRoot: string, skillsRoot: string }} paths
+ * @param {{ templatesRoot: string }} paths
  * @param {object} agentSettings - Legacy-shim settings bag or resolved config.
  * @param {string} currentVersion
  * @param {string} taskBranch
@@ -491,48 +412,6 @@ function buildStaticSections(
     });
   }
 
-  if (task.persona) {
-    try {
-      const pPath = path.join(
-        PROJECT_ROOT,
-        paths.personasRoot,
-        `${task.persona}.md`,
-      );
-      if (fs.existsSync(pPath)) {
-        sections.push({
-          name: 'persona',
-          priority: DEFAULT_SECTION_PRIORITIES.persona,
-          elideWhenOverBudget: DEFAULT_ELIDE_POLICIES.persona,
-          content: `## Persona: ${task.persona}\n\n${readFileCached(pPath)}`,
-          source: { kind: 'file', ref: `personas/${task.persona}.md` },
-        });
-      }
-    } catch (err) {
-      Logger.warn(
-        `[Hydrator] Failed to load persona ${task.persona}: ${err.message}`,
-      );
-    }
-  }
-
-  if (task.skills?.length > 0) {
-    try {
-      const skillsIndex = loadSkillsIndex();
-      const entries = buildSkillCapsuleSections(task, skillsIndex);
-      const skillsContext = formatSkillCapsulesSection(entries);
-      if (skillsContext) {
-        sections.push({
-          name: 'skillCapsules',
-          priority: DEFAULT_SECTION_PRIORITIES.skillCapsules,
-          elideWhenOverBudget: DEFAULT_ELIDE_POLICIES.skillCapsules,
-          content: skillsContext,
-          source: { kind: 'derived', ref: 'activated-skills' },
-        });
-      }
-    } catch (err) {
-      Logger.warn(`[Hydrator] Failed to load skills index: ${err.message}`);
-    }
-  }
-
   // Track which dedicated section(s) were emitted so taskInstructions drops
   // only the inline sections that were actually reproduced elsewhere in the
   // envelope — keeping each binding acceptance/verify item present exactly