@mmerterden/multi-agent-pipeline 17.6.0 → 18.0.0

package/pipeline/lib/run-paths.sh

@@ -0,0 +1,335 @@
+#!/bin/bash
+# run-paths.sh  -  the single resolver for pipeline run-state paths (shell side).
+#
+# Shell twin of pipeline/scripts/_run-paths.mjs. The two must agree; the
+# contract is asserted by pipeline/scripts/smoke-run-path-canonical.sh, which
+# runs both over the same fixture tree and diffs their answers.
+#
+# A run's files live under the log root in ONE of two layouts:
+#
+#   nested   <root>/<project>/<task_id>/     documented canonical
+#   flat     <root>/<task_id>/               what phase-tracker.sh writes
+#
+# Both are populated on real machines, so every reader accepts both. This file
+# does NOT change where anything is written: measured on a real install, 90 of
+# 103 runs are flat and every tracker-state.json is. A task id present in both
+# layouts is ONE run; the most recently written directory wins. Relocation is
+# opt-in and explicit: `migrate-state.mjs --relocate`.
+#
+# No `set -e` on purpose: this is a sourced library and changing the caller's
+# shell options is not this file's business.
+#
+# Usage:
+#   . "$(dirname "$0")/../lib/run-paths.sh"
+#   root=$(ma_logs_root)
+#   dir=$(ma_resolve_run_dir "{JIRA_KEY}-123") || echo "unknown run"
+#   f=$(ma_resolve_run_file "{JIRA_KEY}-123" tracker-state.json)
+#   ma_list_runs   # task_id<TAB>project<TAB>dir<TAB>layout, one per run
+#
+# Bash 3.2 compatible: no associative arrays, no `mapfile`, no `grep -P`.
+
+# Marker files and reserved names are spelled out at each test rather than held
+# in a space-separated variable iterated with `for x in $VAR`. bash word-splits
+# an unquoted variable; zsh does not, so there the loop would run ONCE with the
+# whole string as a single word, every test would fail, and the library would
+# answer "no runs" instead of erroring. This file is sourced, so the caller's
+# shell decides - and answering zero is the failure mode that looks like data.
+
+ma_logs_root() {
+  printf '%s\n' "${LOGS_ROOT:-$HOME/.claude/logs/multi-agent}"
+}
+
+# ma_is_run_dir <dir> -> 0 when the directory holds at least one run marker
+# Phase 6 removes the worktree and salvages the run's files into `artifacts/`
+# inside the same run directory, so a shipped run keeps its state one level
+# deeper. A reader that only looks at the top level reports it as stateless.
+MA_ARTIFACTS_SUBDIR=artifacts
+
+ma_is_run_dir() {
+  local d="$1"
+  [ -f "$d/agent-state.json" ] && return 0
+  [ -f "$d/tracker-state.json" ] && return 0
+  [ -f "$d/agent-log.md" ] && return 0
+  [ -f "$d/$MA_ARTIFACTS_SUBDIR/agent-state.json" ] && return 0
+  [ -f "$d/$MA_ARTIFACTS_SUBDIR/tracker-state.json" ] && return 0
+  [ -f "$d/$MA_ARTIFACTS_SUBDIR/agent-log.md" ] && return 0
+  return 1
+}
+
+ma_is_reserved() {
+  case "$1" in
+    review-watch | jira-backups | shadow-git | _analysis-jira | prompts) return 0 ;;
+  esac
+  return 1
+}
+
+# Newest mtime across a run directory's markers, as epoch seconds. GNU-first
+# `stat -c` ordering is required: `stat -f` is a valid GNU flag
+# (--file-system) that SUCCEEDS, so BSD-first would silently win on Linux.
+ma_run_mtime() {
+  local d="$1" m t newest=0 b
+  for b in "$d" "$d/$MA_ARTIFACTS_SUBDIR"; do
+    for m in agent-state.json tracker-state.json agent-log.md; do
+      [ -f "$b/$m" ] || continue
+      # -L follows symlinks. Some nested run directories are symlink bridges to
+      # the flat copy of the same run (3 such pairs on the install this was
+      # measured on); without -L the shell reads the LINK's mtime while the
+      # .mjs twin's fs.statSync reads the TARGET's, and the two picked
+      # different winners for one record. Both must read the same clock.
+      t=$(stat -L -c %Y "$b/$m" 2>/dev/null || stat -L -f %m "$b/$m" 2>/dev/null || echo 0)
+      [ -n "$t" ] || t=0
+      [ "$t" -gt "$newest" ] && newest="$t"
+    done
+  done
+  printf '%s\n' "$newest"
+}
+
+# ma_canonical_run_dir <task_id> [project] -> where a NEW run should be written
+ma_canonical_run_dir() {
+  local task_id="$1" project="${2:-}" root
+  root=$(ma_logs_root)
+  if [ -n "$project" ]; then
+    printf '%s\n' "$root/$project/$task_id"
+  else
+    printf '%s\n' "$root/$task_id"
+  fi
+}
+
+# ma_run_dir_candidates <task_id> [project] -> candidate dirs, most specific
+# first. Existence is not checked here.
+ma_run_dir_candidates() {
+  local task_id="$1" project="${2:-}" root d n
+  root=$(ma_logs_root)
+  [ -n "$project" ] && printf '%s\n' "$root/$project/$task_id"
+  printf '%s\n' "$root/$task_id"
+  [ -n "$project" ] && return 0
+  # No project given: the run may still be nested under one.
+  #
+  # `find` rather than a `*/` glob on purpose: an unmatched glob is a literal
+  # string in bash and a hard error in zsh, and this file is sourced, so the
+  # caller's shell decides. find behaves identically in both.
+  while IFS= read -r d; do
+    [ -n "$d" ] || continue
+    n=$(basename "$d")
+    ma_is_reserved "$n" && continue
+    [ "$n" = "$task_id" ] && continue
+    ma_is_run_dir "$root/$n/$task_id" && printf '%s\n' "$root/$n/$task_id"
+  done <<EOF
+$(find "$root" -mindepth 1 -maxdepth 1 -type d 2>/dev/null)
+EOF
+  return 0
+}
+
+# ma_realpath <path> -> the path with every symlink resolved, or the input when
+# it cannot be resolved. BSD realpath first, python3 as the fallback (both are
+# already hard requirements of this tree).
+ma_realpath() {
+  realpath "$1" 2>/dev/null ||
+    python3 -c 'import os,sys; print(os.path.realpath(sys.argv[1]))' "$1" 2>/dev/null ||
+    printf '%s\n' "$1"
+}
+
+# ma_is_bridge_dir <dir> -> 0 when any marker in it is a symlink elsewhere.
+# Some nested run directories are symlink bridges into the flat copy of the
+# same run. They are one record seen twice, not two copies.
+ma_is_bridge_dir() {
+  local d="$1" m
+  for m in agent-state.json tracker-state.json agent-log.md; do
+    [ -L "$d/$m" ] && return 0
+  done
+  return 1
+}
+
+# ma_same_record <dir_a> <dir_b> -> 0 when both are views of ONE record.
+ma_same_record() {
+  local a="$1" b="$2" m ra rb
+  for m in agent-state.json tracker-state.json agent-log.md; do
+    [ -e "$a/$m" ] && [ -e "$b/$m" ] || continue
+    ra=$(ma_realpath "$a/$m")
+    rb=$(ma_realpath "$b/$m")
+    [ "$ra" = "$rb" ] && return 0
+  done
+  return 1
+}
+
+# ma_rank_fields <dir> -> "<mtime>\t<has_state>\t<depth>", the three ranking
+# columns used to pick between candidate directories for one task id.
+#
+# ma_sort_key packs the same three into one space-separated key for callers
+# that sort a single column; both must match compareCandidates in
+# _run-paths.mjs exactly.
+#
+# mtime alone is not an order: two directories written in the same second tie,
+# and each implementation then fell back to its own traversal order - which is
+# how the two came to disagree about a run present in both layouts. The tail of
+# the key defines the answer: richer record first (agent-state.json is what
+# every reader wants), then the documented nested layout, then the path.
+ma_rank_fields() {
+  local d="$1" t has_state depth real
+  t=$(ma_run_mtime "$d")
+  has_state=0
+  [ -f "$d/agent-state.json" ] && has_state=1
+  depth=$(printf '%s' "$d" | awk -F/ '{print NF}')
+  real=1
+  ma_is_bridge_dir "$d" && real=0
+  printf '%s\t%s\t%s\t%s\n' "$real" "$t" "$has_state" "$depth"
+}
+
+ma_sort_key() {
+  local d="$1" t has_state depth real
+  t=$(ma_run_mtime "$d")
+  has_state=0
+  [ -f "$d/agent-state.json" ] && has_state=1
+  depth=$(printf '%s' "$d" | awk -F/ '{print NF}')
+  real=1
+  ma_is_bridge_dir "$d" && real=0
+  printf '%d %012d %d %04d %s\n' "$real" "$t" "$has_state" "$depth" "$d"
+}
+
+# ma_resolve_run_dir <task_id> [project] -> the directory, or rc 1 when unknown.
+ma_resolve_run_dir() {
+  local task_id="$1" project="${2:-}" c best
+  best=$(
+    while IFS= read -r c; do
+      [ -n "$c" ] || continue
+      ma_is_run_dir "$c" || continue
+      ma_sort_key "$c"
+    done <<EOF
+$(ma_run_dir_candidates "$task_id" "$project")
+EOF
+  )
+  [ -n "$best" ] || return 1
+  # Descending on mtime, then has-state, then depth; ascending on path.
+  printf '%s\n' "$best" | sort -k1,1nr -k2,2nr -k3,3nr -k4,4nr -k5,5 | head -1 | cut -d' ' -f5-
+}
+
+# ma_task_id_variants <id> -> the spellings one task id has been written under,
+# in preference order. `#316` arrives from a GitHub issue reference, `316` from
+# the bare-number input class, `task-316` from an older directory convention.
+# Callers used to inline this list; dropping a spelling silently stops old runs
+# from resolving.
+ma_task_id_variants() {
+  local raw="$1" bare
+  bare="${raw#\#}"
+  printf '%s\n' "$raw"
+  [ "$bare" != "$raw" ] && printf '%s\n' "$bare"
+  printf '%s\n' "task-$bare"
+}
+
+# ma_resolve_run_dir_any <task_id> [project] -> dir for any spelling, or rc 1
+ma_resolve_run_dir_any() {
+  local task_id="$1" project="${2:-}" v dir
+  while IFS= read -r v; do
+    [ -n "$v" ] || continue
+    dir=$(ma_resolve_run_dir "$v" "$project") && {
+      printf '%s\n' "$dir"
+      return 0
+    }
+  done <<EOF
+$(ma_task_id_variants "$task_id")
+EOF
+  return 1
+}
+
+# ma_resolve_run_file <task_id> <filename> [project] -> path, or rc 1.
+# Every spelling of the id is tried.
+ma_resolve_run_file() {
+  local task_id="$1" filename="$2" project="${3:-}" v dir
+  while IFS= read -r v; do
+    [ -n "$v" ] || continue
+    dir=$(ma_resolve_run_dir "$v" "$project") || continue
+    if [ -f "$dir/$filename" ]; then
+      printf '%s\n' "$dir/$filename"
+      return 0
+    fi
+    # The salvaged copy Phase 6 leaves behind.
+    if [ -f "$dir/$MA_ARTIFACTS_SUBDIR/$filename" ]; then
+      printf '%s\n' "$dir/$MA_ARTIFACTS_SUBDIR/$filename"
+      return 0
+    fi
+  done <<EOF
+$(ma_task_id_variants "$task_id")
+EOF
+  return 1
+}
+
+# ma_list_runs -> one TAB-separated row per run, deduplicated by task id:
+#   task_id<TAB>project<TAB>dir<TAB>layout
+# `project` is "-" when the layout does not name one.
+ma_list_runs() {
+  local root d n kd kn rows row id
+  root=$(ma_logs_root)
+  [ -d "$root" ] || return 0
+  rows=""
+  while IFS= read -r d; do
+    [ -n "$d" ] || continue
+    n=$(basename "$d")
+    ma_is_reserved "$n" && continue
+    if ma_is_run_dir "$root/$n"; then
+      rows="$rows$n	-	$root/$n	flat	$(ma_rank_fields "$root/$n")
+"
+      continue
+    fi
+    while IFS= read -r kd; do
+      [ -n "$kd" ] || continue
+      kn=$(basename "$kd")
+      ma_is_run_dir "$root/$n/$kn" || continue
+      rows="$rows$kn	$n	$root/$n/$kn	nested	$(ma_rank_fields "$root/$n/$kn")
+"
+    done <<EOF
+$(find "$root/$n" -mindepth 1 -maxdepth 1 -type d 2>/dev/null)
+EOF
+  done <<EOF
+$(find "$root" -mindepth 1 -maxdepth 1 -type d 2>/dev/null)
+EOF
+  # Dedup by task id using the same total order as resolveRunDir and as
+  # compareCandidates in the .mjs twin: mtime desc, has-state desc, depth desc,
+  # path ASC. The sub-keys are separate columns on purpose - a single reverse
+  # sort over a composite key also reverses the path, which is the one
+  # component that has to ascend.
+  printf '%s' "$rows" | grep -v '^$' |
+    sort -t'	' -k1,1 -k5,5nr -k6,6nr -k7,7nr -k8,8nr -k3,3 | awk -F'	' '
+    !seen[$1]++ { print $1 "\t" $2 "\t" $3 "\t" $4 }
+  ' | sort -t'	' -k1,1
+}
+
+# ma_duplicate_run_ids -> task ids that exist as two SEPARATE records in the two
+# layouts, one per line. A symlink bridge is excluded: it is one record seen
+# twice, and reporting it overstates the drift.
+ma_duplicate_run_ids() {
+  local root d n kd kn pairs id dirs a b
+  root=$(ma_logs_root)
+  [ -d "$root" ] || return 0
+  pairs=$(
+    while IFS= read -r d; do
+      [ -n "$d" ] || continue
+      n=$(basename "$d")
+      ma_is_reserved "$n" && continue
+      if ma_is_run_dir "$root/$n"; then
+        printf '%s\t%s\n' "$n" "$root/$n"
+        continue
+      fi
+      while IFS= read -r kd; do
+        [ -n "$kd" ] || continue
+        kn=$(basename "$kd")
+        ma_is_run_dir "$root/$n/$kn" && printf '%s\t%s\n' "$kn" "$root/$n/$kn"
+      done <<EOF
+$(find "$root/$n" -mindepth 1 -maxdepth 1 -type d 2>/dev/null)
+EOF
+    done <<EOF
+$(find "$root" -mindepth 1 -maxdepth 1 -type d 2>/dev/null)
+EOF
+  )
+  # Ids seen more than once, minus the ones whose two directories resolve to the
+  # same underlying files (a symlink bridge is one record, not drift).
+  while IFS= read -r id; do
+    [ -n "$id" ] || continue
+    dirs=$(printf '%s\n' "$pairs" | awk -F'\t' -v k="$id" '$1==k {print $2}')
+    a=$(printf '%s\n' "$dirs" | sed -n 1p)
+    b=$(printf '%s\n' "$dirs" | sed -n 2p)
+    [ -n "$b" ] || continue
+    ma_same_record "$a" "$b" || printf '%s\n' "$id"
+  done <<EOF
+$(printf '%s\n' "$pairs" | awk -F'\t' '{print $1}' | sort | uniq -d)
+EOF
+}

package/pipeline/multi-agent-refs/features/autopilot-circuit-breaker.md

@@ -1,5 +1,14 @@
 # Feature: Autopilot Circuit-Breaker
 
+<!-- toc -->
+- [Trip conditions](#trip-conditions)
+- [Wiring status](#wiring-status)
+- [Action on trip](#action-on-trip)
+- [Why this is the right autopilot exception](#why-this-is-the-right-autopilot-exception)
+- [The runner-level breaker (a different failure, a different layer)](#the-runner-level-breaker-a-different-failure-a-different-layer)
+- [Two more things a long-running runner needs](#two-more-things-a-long-running-runner-needs)
+<!-- /toc -->
+
 **Pattern**: autopilot runs with zero interaction, which is exactly when a silent failure loop is most expensive  -  an agent can burn a budget re-attempting the same broken fix, or thrash between two phases, with nobody watching. A circuit-breaker converts "keep going no matter what" into "keep going until a defined unsafe condition, then halt and hand back to the user." This is the sanctioned autopilot pause (same class as the Phase 7 channels pause): the run stops, records why, and waits for an explicit `resume`.
 
 **Gated by `prefs.global.autopilotCircuitBreaker`** (`enabled` default true, `identicalFindingCycles` default 2, `maxReworkCycles` default 3; `schemas/prefs.schema.json`). Halting is always safe, so the breaker itself defaults on. Disable per-run only with an explicit override. Complements, does not replace, the existing autopilot safety rules (build-fail max 3 retries, Phase 4 blocking-finding rework, destructive-op confirmations).
@@ -43,3 +52,64 @@ State shape: `state.circuitBreaker = {tripped, trigger, detail, checkpoint: {pha
 Autopilot's contract is "no interaction on the happy path." The circuit-breaker fires only off the happy path, where continuing unattended is the *less* safe choice: repeating a proven-broken action, or spending past a ceiling the user set, is not autonomy, it is a runaway. Halting with a precise reason is cheaper than the tokens (and trust) a silent loop burns.
 
 Inspired by autonomous-loop operators that gate on explicit stop conditions (no-progress, identical-stack-trace repetition, cost drift, conflict blocking), adapted to this pipeline's phase/rework/state model.
+
+## The runner-level breaker (a different failure, a different layer)
+
+Everything above is the IN-RUN breaker: one run, one state file, triggers that
+mean "this task is not converging". It cannot see the failure a server actually
+hits, which is not about any one task.
+
+An expired token, a `claude` binary that no longer launches, a full disk: every
+item fails identically, a few minutes apart, until the queue is empty. Nothing
+above notices, because each individual run failed for its own apparently local
+reason. What is left afterwards is a ledger of failures with no indication of
+which came first and no queue to resume.
+
+`autopilot-runner.mjs` therefore refuses to take NEW work after
+`MA_AP_BREAKER_LIMIT` consecutive attempts produced nothing (default 3, zero
+disables). The count is CONSECUTIVE and read from `attempted.jsonl` rather than
+kept as a counter, because a counter is a second piece of state that can
+disagree with the ledger - and the ledger is what a person reads when they ask
+why the runner stopped.
+
+Two outcomes are deliberately not failures:
+
+- `needs-input` is a run that did its work and is waiting for a person. That is
+  the system behaving correctly, and counting it would stop a machine whose only
+  problem is that somebody has not answered yet.
+- `blocked-*` items never ran at all - arming refused them - so they say nothing
+  about whether a run would have worked.
+
+When it opens, the reason goes into `queue.json.blockedReason` (so
+`autopilot-status` shows it rather than reporting an idle queue), a
+`breaker-open` line goes into `ticks.jsonl`, and the log names the outcome that
+opened it plus the override.
+
+It is HALF-OPEN rather than latched, and that distinction is load-bearing. The
+breaker returns before the only writer of `attempted.jsonl`, so a version that
+simply stopped could never record another attempt, the consecutive count could
+never fall, and the runner would be off for good - while its own log said
+"until an attempt succeeds". After `MA_AP_BREAKER_COOLDOWN_SEC` (30 min by
+default) one probe is let through: a recovered machine succeeds and the count
+resets on its own, and a still-broken one fails, becomes the newest attempt, and
+closes the breaker for another cooldown. One wasted run per cooldown is the
+price of not needing a person to notice.
+
+## Two more things a long-running runner needs
+
+**The log.** launchd appends the runner's stdout to `runner.log` forever. On a
+laptop that file is never read; on a machine ticking every few minutes for
+months it is what fills the disk, and a full disk stops the runs the log existed
+to record. Each tick truncates it past `MA_AP_LOG_MAX_BYTES` (default 5MB),
+keeping the last 256KB in `runner.log.1`.
+
+Truncating the same inode is the mechanism, not renaming: launchd holds the file
+open with `O_APPEND`, so a rename leaves that descriptor writing into the renamed
+file while the new one stays empty forever. That is the rotation that looks
+correct and silently stops logging.
+
+**Telemetry.** `ticks.jsonl` gets one JSON line per tick - what the tick did,
+what it took, what came out, how many consecutive failures preceded it. The
+human log answers "what happened just now"; this answers "how has this been
+behaving for a week", which is the question a server raises and prose cannot be
+asked.

package/pipeline/multi-agent-refs/features/cost-analysis.md

@@ -0,0 +1,93 @@
+# cost analysis - projection, anomaly, burn, diff
+
+`cost-budget-check.mjs` answers one question, live, for one run: is THIS task
+about to cross its ceiling. Two ways a budget actually empties are invisible to
+it. A slow drift never trips any single run's ceiling. And one pathological
+session can burn a week's worth in an hour while every individual run stays
+comfortably under its cap.
+
+`cost-analyze.mjs` answers the other four questions off one series.
+
+| Sub-command | Question |
+|---|---|
+| `projection` | at this rate, what does a week / a month / a quarter cost, and when does a stated budget run out |
+| `anomaly` | which day or session is out of family |
+| `burn` | is the last day accelerating against the preceding week |
+| `diff` | two saved snapshots, side by side |
+
+```bash
+node "$HOME/.claude/scripts/cost-analyze.mjs" projection --days 14 [--monthly-usd 200]
+node "$HOME/.claude/scripts/cost-analyze.mjs" anomaly --days 30 [--by day|session] [--threshold 3.5]
+node "$HOME/.claude/scripts/cost-analyze.mjs" burn [--factor 3]
+node "$HOME/.claude/scripts/cost-analyze.mjs" --save [--days 30]
+node "$HOME/.claude/scripts/cost-analyze.mjs" diff [a.json b.json]
+```
+
+Exit 0 means nothing to report, 10 means a finding, 2 is a usage error. Every
+sub-command takes `--json`, and the human output and the JSON are rendered from
+the same numbers.
+
+## Where the numbers come from, and what that makes them worth
+
+The per-run token accumulators in `tracker-state.json` are the pipeline's own
+ledger, and they are the obvious source. They are also empty: the schema has
+`tokens_in` / `tokens_out` / `tokens_cached` per phase, they are written only
+when a phase reports them, and on a real machine no tracker carries them at all.
+
+The series that does exist is the host's own transcripts, one usage record per
+assistant turn, under `~/.claude/projects/<slug>/<session>.jsonl`. So that is
+what this reads, and three consequences follow that are stated here rather than
+discovered later:
+
+- It covers everything Claude Code did on this machine, not only pipeline runs.
+- The figures are ESTIMATES priced from `cost-table.json`, at LIST price. On a
+  subscription they are the right number for comparing one day to another and
+  the wrong number to call a bill. The human output says so on every run.
+- A host with no such transcripts (Copilot, Codex) reports `UNMEASURED`, never
+  zero. Zero would read as "you spent nothing" when it means "I could not see".
+
+Records whose model is not in `cost-table.json` are COUNTED as unpriced and left
+out of the total. Folding them in at zero would make an unknown model look free,
+which is the direction that hurts.
+
+Cache writes are priced at 1.25x input, derived rather than read from the table,
+because `cost-table.json` carries a cache READ rate only. Dropping the component
+would make every long-context session look cheap.
+
+## Why the median and the MAD, not the mean
+
+The single expensive session `anomaly` exists to find is also the observation
+that inflates a mean and a standard deviation - it hides inside the statistic
+measured against it. With six ordinary days and one at fifty times the rest, a
+mean-based z-score puts the outlier at about 2.3 sigma, under every usual
+threshold. The median does not move for one observation, and neither does the
+median absolute deviation.
+
+The score is the modified z-score of Iglewicz and Hoaglin, `0.6745 * (x - median)
+/ MAD`, flagged at 3.5.
+
+When more than half the values are identical the MAD is zero. That is not an
+error and not a licence to divide by zero: the published fallback is the mean
+absolute deviation scaled by 1.253314. When THAT is zero too there is no
+dispersion at all, and the honest answer is that no anomaly can be called -
+which is what it says.
+
+Fewer than five points is refused outright, with the number it needs.
+
+## Why projection divides by the calendar window
+
+Total over the window, divided by the WINDOW, not by the days that happen to
+have data. Dividing by active days answers a different question - what a working
+day costs - and then projecting a month as thirty working days overstates it by
+roughly a third.
+
+`--monthly-usd` is optional and there is no default. Without it the projection
+is reported and the exhaustion date is not, with the reason given. Inventing a
+budget to compare against would produce a number that looks measured.
+
+## Snapshots
+
+`--save` writes `~/.claude/state/cost/<iso>.json`: the window, the total, the
+per-day rate and the per-day breakdown. `diff` compares two, by path or the last
+two by default, and says when the windows differ - the per-day rate is
+comparable across different windows, the total is not.

package/pipeline/multi-agent-refs/features/doctor.md

@@ -8,6 +8,7 @@
 - [The line shape](#the-line-shape)
 - [It recommends, it never fixes](#it-recommends-it-never-fixes)
 - [Checks](#checks)
+- [The server profile (`--profile=server`)](#the-server-profile---profileserver)
 <!-- /toc -->
 
 Every check `/multi-agent:doctor` can report has a `### <id>` heading here, and
@@ -252,3 +253,47 @@ a full second checkout, so the total is measured in gigabytes rather than
 megabytes. SKIP outside a git repository - a project name in prefs is a name,
 not a path, and guessing checkout locations to produce a number is how a
 diagnostic starts lying.
+
+## The server profile (`--profile=server`)
+
+Four checks that only run under `--profile=server`. They are ADDITIVE: a
+default run is byte-for-byte what it was, and `--list-checks` reports whichever
+set the invocation would actually perform, because a list that disagrees with
+the run is worse than no list.
+
+None of them may BLOCK. The blocking set is closed at six and these are
+readiness, not correctness: a machine that fails all four still runs the
+pipeline perfectly well with someone watching it. What they catch is the
+failure mode of an unwatched run - stopping without saying so.
+
+### unattended-contract
+
+Is `multi-agent-refs/unattended-contract.md` installed. It is the only place
+that says which entry points honour `MULTI_AGENT_UNATTENDED=1` and what each
+one resolves to. Without it an operator setting up a server has to read the
+scripts to find out, which is how the wrong assumption gets made.
+
+### unattended-permissions
+
+Does `settings.json` carry `permissions.allow` entries covering Bash, Edit and
+Write. autopilot spawns its child with `--permission-prompts none`, but the
+tools that child then calls still have to be allowed, and on a fresh machine
+they are not - the run stops at the first prompt with nobody there to answer
+it, printing nothing. This is the single most likely reason a server sits idle
+and looks healthy.
+
+### scheduler
+
+Is a multi-agent launchd agent loaded. Something has to start the work; a
+server with no agent and an empty queue looks exactly like a server that has
+finished everything.
+
+### keychain-unlock
+
+Is a default keychain reachable from this session. Every credential the
+pipeline reads lives there, and a LaunchDaemon started before login sees a
+LOCKED keychain: each fetch fails, and the failures surface much later as 401s
+that blame the token rather than the lock. The check is a proxy - reading a
+real credential would be a network call and a side effect - so it asks whether
+a login keychain is reachable at all, which is the condition a boot-time
+daemon fails.

package/pipeline/multi-agent-refs/features/verify.md

@@ -0,0 +1,83 @@
+# verify - is this install the thing that was published
+
+The install is a COPY. `install.js` writes the pipeline tree into `~/.claude`,
+`~/.copilot` and `~/.codex`, and from that moment the two halves drift
+independently. Both directions produce bugs that are hard to name:
+
+- an edit made in the installed copy is a behaviour with no source, and the next
+  update silently reverts it;
+- a file the installer failed to write is a script the docs describe and nobody
+  has, which reads as a documentation error.
+
+`multi-agent-pipeline verify` answers both mechanically.
+
+```bash
+npx @mmerterden/multi-agent-pipeline verify              # package + install
+npx @mmerterden/multi-agent-pipeline verify --package    # package integrity only
+npx @mmerterden/multi-agent-pipeline verify --install    # install drift only
+npx @mmerterden/multi-agent-pipeline verify --json
+```
+
+| Code | Meaning |
+|---|---|
+| 0 | everything matches |
+| 1 | a difference was found, named file by file |
+| 2 | nothing to verify - a source checkout, or a version published before manifests existed |
+
+Exit 2 is not a pass and not a failure. A dev checkout has no manifest by
+design, and reporting that as either would be a lie in one direction or the
+other.
+
+## The manifest
+
+`manifest.json` is written at pack time by `prepack`, never committed. A
+manifest in git is stale one commit after it is written, and a stale manifest
+reports honest edits as tampering - which is worse than having none, because
+people learn to ignore it.
+
+The file list is not guessed. It comes from `npm pack --dry-run --json`, so by
+construction it is the same set npm publishes, `files` globs and all. The gate
+asserts the two counts agree, which is what catches a `files` entry and a
+manifest that have stopped describing the same package.
+
+Two things it cannot cover, said here rather than discovered later: it cannot
+hash itself, and a signature over it does not authenticate the tarball.
+
+## What a green result proves, and what it does not
+
+It proves the bytes match what the publisher recorded. It is not proof of WHO
+published them. The manifest, the signature and the verifier all travel inside
+the same tarball, so anyone able to rewrite one can rewrite the others.
+Provenance belongs to npm's own integrity field.
+
+What this does catch is the set of failures that actually happen: a damaged or
+partial install, a file edited after install, and an update that did not land.
+
+Signing is optional. `make-manifest.mjs --sign` reads an ed25519 private key
+from the credential store (or `MULTI_AGENT_SIGNING_KEY` on a build host with no
+store) and writes `manifest.sig`; `verify` checks it against
+`MULTI_AGENT_SIGNING_PUBKEY` when one is pinned. Without a key it says "signed,
+no public key to check it against" rather than claiming valid - a signature
+nobody can check is not a signature that passed.
+
+## How each tree is compared
+
+| Tree | Mode | Why |
+|---|---|---|
+| `scripts` | bytes | verbatim copy, minus the dev-only set |
+| `lib` | bytes | verbatim copy |
+| `multi-agent-refs` | bytes | verbatim copy |
+| `agents` | bytes | verbatim copy |
+| `commands/multi-agent` | presence | `install.js` rewrites each SKILL.md `description` into the user's `outputLanguage` |
+
+Byte-comparing `commands/` reports every command as drift on a perfectly
+healthy machine. Measured here: all 57 command files differ, and 56 of them
+differ by nothing except the translated description. A report that is wrong by
+default is a report nobody reads.
+
+The dev-only filter matters just as much: smokes, linters and fixtures ship in
+the package and are deliberately NOT installed. Without excluding them, `verify`
+would report 252 files as "the installer skipped this".
+
+`~/.copilot` and `~/.codex` are reported as present, not compared: the installer
+rewrites paths for both on purpose, so a byte difference there is the design.

package/pipeline/multi-agent-refs/phases/operations.md

@@ -78,8 +78,9 @@ Update `agent-state.json` at EVERY phase transition.
 
 ### Writing `agent-state.json` (required mechanism)
 
-Every state update goes through `write-state.mjs`. Never write the file with a
-plain read-modify-write (`jq ... > tmp && mv`, an editor tool, `cat >`):
+Every state write goes through `write-state.mjs`, **including the first one in
+Phase 0**. Never write the file with a plain read-modify-write (`jq ... > tmp &&
+mv`, an editor tool, `cat >`):
 
 ```bash
 # Merge a patch into the current state (the normal case).
@@ -97,6 +98,16 @@ the first landed. `write-state.mjs` does tmpfile + rename (atomic on POSIX) unde
 an advisory `.lock`, reclaims a lock whose holder PID is dead, and releases the
 lock on every error path.
 
+Why the CREATE matters as much as the updates: the writer stamps `rev` on every
+write and `schemaVersion` on the first one. A document written by hand in Phase 0
+starts with neither, so every later writer compares against an absent revision
+and `migrate-state.mjs` can never place the file on a migration path. Measured on
+a real install before this was fixed: 39 of 43 `agent-state.json` files carried no
+`rev` and 43 of 43 carried no `schemaVersion`, which is the whole of
+`$HOME/.claude/schemas/migrations/` sitting unreachable. `migrate-state.mjs --all`
+reports the legacy ones; it does not stamp them, because a stamp would assert a
+conformance nothing checked.
+
 Exit codes the caller must handle: `0` written, `1` invalid JSON on stdin, `2`
 lock timeout (another writer held it past the acquire window  -  retry once, then
 halt per the halt-visibility rule), `3` I/O error.