@caddis/cli 0.4.52 → 0.4.53

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/bundles/antigravity-plugin/CADDIS-GLOSSARY.md +1 -0
  2. package/bundles/antigravity-plugin/plugin.json +1 -1
  3. package/bundles/antigravity-plugin/repo_freshness.py +132 -5
  4. package/bundles/antigravity-plugin/scripts/caddis_checkout_guard.py +223 -0
  5. package/bundles/antigravity-plugin/scripts/caddis_docs_pr.py +284 -0
  6. package/bundles/antigravity-plugin/scripts/caddis_exit.py +2 -0
  7. package/bundles/antigravity-plugin/scripts/caddis_frontmatter.py +1 -0
  8. package/bundles/antigravity-plugin/scripts/caddis_gate.py +226 -195
  9. package/bundles/antigravity-plugin/scripts/caddis_glossary.py +3 -3
  10. package/bundles/antigravity-plugin/scripts/caddis_governor.py +48 -4
  11. package/bundles/antigravity-plugin/scripts/caddis_inventory.py +38 -0
  12. package/bundles/antigravity-plugin/scripts/caddis_lanes.py +294 -127
  13. package/bundles/antigravity-plugin/scripts/caddis_lineage_publish.py +458 -0
  14. package/bundles/antigravity-plugin/scripts/caddis_merge_chain.py +237 -0
  15. package/bundles/antigravity-plugin/scripts/caddis_notify.py +121 -0
  16. package/bundles/antigravity-plugin/scripts/caddis_prod_check.py +380 -0
  17. package/bundles/antigravity-plugin/scripts/caddis_redact.py +99 -0
  18. package/bundles/antigravity-plugin/scripts/caddis_relay_git.py +2 -2
  19. package/bundles/antigravity-plugin/scripts/caddis_review.py +1309 -0
  20. package/bundles/antigravity-plugin/scripts/caddis_roadmap.py +2056 -0
  21. package/bundles/antigravity-plugin/scripts/caddis_roadmap_run.py +2572 -0
  22. package/bundles/antigravity-plugin/scripts/caddis_routing.py +3 -2
  23. package/bundles/antigravity-plugin/scripts/caddis_rows.py +476 -0
  24. package/bundles/antigravity-plugin/scripts/caddis_spawn.py +8 -9
  25. package/bundles/antigravity-plugin/scripts/caddis_spend.py +415 -0
  26. package/bundles/antigravity-plugin/scripts/caddis_tidy.py +357 -20
  27. package/bundles/antigravity-plugin/scripts/caddis_todo.py +55 -0
  28. package/bundles/antigravity-plugin/scripts/caddis_workstreams.py +178 -19
  29. package/bundles/antigravity-plugin/scripts/caddis_worktree_close.py +349 -0
  30. package/bundles/antigravity-plugin/scripts/check_doc_coverage.py +0 -56
  31. package/bundles/antigravity-plugin/scripts/claudster_doctor.py +1 -1
  32. package/bundles/antigravity-plugin/scripts/enforcement_inventory.py +0 -1
  33. package/bundles/antigravity-plugin/scripts/oss_review.py +2 -2
  34. package/bundles/antigravity-plugin/scripts/roadmap-recheck.ps1 +89 -0
  35. package/bundles/antigravity-plugin/scripts/roadmap-run.ps1 +88 -0
  36. package/bundles/antigravity-plugin/scripts/setup_project_ai.py +43 -1
  37. package/bundles/antigravity-plugin/session_end_agy.py +3 -3
  38. package/bundles/antigravity-plugin/skills/_registry.md +2 -6
  39. package/bundles/antigravity-plugin/skills/best-practices/agents/best-practices-referencer.md +1 -1
  40. package/bundles/antigravity-plugin/skills/best-practices/agents/codebase-context-builder.md +1 -1
  41. package/bundles/antigravity-plugin/skills/best-practices/agents/task-intent-analyzer.md +1 -1
  42. package/bundles/antigravity-plugin/skills/catchup/SKILL.md +7 -0
  43. package/bundles/antigravity-plugin/skills/cloud-phases/SKILL.md +1 -1
  44. package/bundles/antigravity-plugin/skills/cross-review/SKILL.md +5 -2
  45. package/bundles/antigravity-plugin/skills/feature-plan/SKILL.md +29 -83
  46. package/bundles/antigravity-plugin/skills/handoff/SKILL.md +77 -48
  47. package/bundles/antigravity-plugin/skills/implement/SKILL.md +87 -147
  48. package/bundles/antigravity-plugin/skills/migrate-dir/SKILL.md +1 -1
  49. package/bundles/antigravity-plugin/skills/mmr/SKILL.md +5 -2
  50. package/bundles/antigravity-plugin/skills/office-documents-official/SKILL.md +24 -0
  51. package/bundles/antigravity-plugin/skills/park/SKILL.md +0 -1
  52. package/bundles/antigravity-plugin/skills/pr-merge/SKILL.md +8 -14
  53. package/bundles/antigravity-plugin/skills/prd/SKILL.md +2 -10
  54. package/bundles/antigravity-plugin/skills/roadmap/SKILL.md +259 -0
  55. package/bundles/antigravity-plugin/skills/roadmap/assets/plan-resume-prompt.md +77 -0
  56. package/bundles/antigravity-plugin/skills/roadmap/assets/render-check.template.mjs +68 -0
  57. package/bundles/antigravity-plugin/skills/roadmap/assets/review-protocol.md +78 -0
  58. package/bundles/antigravity-plugin/skills/roadmap/assets/roadmap.template.md +32 -0
  59. package/bundles/antigravity-plugin/skills/roadmap/assets/value-check.template.py +161 -0
  60. package/bundles/antigravity-plugin/skills/ship/SKILL.md +14 -14
  61. package/bundles/antigravity-plugin/skills/ship-merge/SKILL.md +6 -0
  62. package/bundles/antigravity-plugin/skills/ship-pr/SKILL.md +33 -14
  63. package/bundles/antigravity-plugin/skills/spawn-session/SKILL.md +4 -8
  64. package/bundles/antigravity-plugin/warm_start_agy.py +1 -3
  65. package/bundles/antigravity-plugin-extras/plugin.json +1 -1
  66. package/bundles/antigravity-plugin-extras/skills/_registry.md +1 -2
  67. package/bundles/antigravity-plugin-extras/skills/agent-orchestration/SKILL.md +5 -5
  68. package/bundles/antigravity-plugin-extras/skills/cross-review/SKILL.md +5 -2
  69. package/bundles/antigravity-plugin-extras/skills/runner-move/SKILL.md +214 -0
  70. package/bundles/codex/.codex/skills/docs/office-documents-official/SKILL.md +24 -0
  71. package/bundles/codex/.codex/skills/workflow/best-practices/agents/best-practices-referencer.md +1 -1
  72. package/bundles/codex/.codex/skills/workflow/best-practices/agents/codebase-context-builder.md +1 -1
  73. package/bundles/codex/.codex/skills/workflow/best-practices/agents/task-intent-analyzer.md +1 -1
  74. package/bundles/codex/.codex/skills/workflow/roadmap/SKILL.md +259 -0
  75. package/bundles/codex/.codex/skills/workflow/roadmap/assets/plan-resume-prompt.md +77 -0
  76. package/bundles/codex/.codex/skills/workflow/roadmap/assets/render-check.template.mjs +68 -0
  77. package/bundles/codex/.codex/skills/workflow/roadmap/assets/review-protocol.md +78 -0
  78. package/bundles/codex/.codex/skills/workflow/roadmap/assets/roadmap.template.md +32 -0
  79. package/bundles/codex/.codex/skills/workflow/roadmap/assets/value-check.template.py +161 -0
  80. package/bundles/codex/AGENTS.md +5 -4
  81. package/bundles/manifest.json +3 -3
  82. package/package.json +1 -1
  83. package/bundles/antigravity-plugin/skills/advisory-hub/SKILL.md +0 -233
  84. package/bundles/antigravity-plugin/skills/advisory-hub/reference/context-template.md +0 -214
  85. package/bundles/antigravity-plugin/skills/advisory-hub/reference/hub-handbook.md +0 -233
  86. package/bundles/antigravity-plugin/skills/advisory-hub/reference/report-contract.md +0 -94
  87. package/bundles/antigravity-plugin/skills/advisory-hub/reference/succession.md +0 -118
  88. package/bundles/antigravity-plugin/skills/resume/SKILL.md +0 -6
  89. package/bundles/antigravity-plugin/skills/review/SKILL.md +0 -6
  90. package/bundles/antigravity-plugin/skills/spawn-hub/SKILL.md +0 -341
  91. package/bundles/antigravity-plugin/skills/statusline/SKILL.md +0 -6
  92. package/bundles/antigravity-plugin/skills/validate-phase/SKILL.md +0 -143
  93. package/bundles/antigravity-plugin-extras/skills/data-lineage/SKILL.md +0 -158
  94. package/bundles/antigravity-plugin-extras/skills/data-lineage/scripts/convert_doc_to_confluence.py +0 -1430
  95. package/bundles/antigravity-plugin-extras/skills/golden-plan/SKILL.md +0 -586
  96. package/bundles/codex/.codex/skills/workflow/advisory-hub/SKILL.md +0 -233
  97. package/bundles/codex/.codex/skills/workflow/advisory-hub/reference/context-template.md +0 -214
  98. package/bundles/codex/.codex/skills/workflow/advisory-hub/reference/hub-handbook.md +0 -233
  99. package/bundles/codex/.codex/skills/workflow/advisory-hub/reference/report-contract.md +0 -94
  100. package/bundles/codex/.codex/skills/workflow/advisory-hub/reference/succession.md +0 -118
@@ -0,0 +1,2572 @@
1
+ """Runner: start one fresh headless session per batch, until a stop. Plan mode runs a plain plan; roadmap mode runs
2
+ the next batches of a roadmap.
3
+
4
+ Design: programme-method-design.md sections 8.1 to 8.5 and 9.2; caddis-next Phases 7 and 9. `roadmap-run.ps1` is a
5
+ thin shim over this file. The programme layer (programme folders, ledger, budget block) is gone (caddis-next Phase 7).
6
+
7
+ One session: stop checks (STOP file, session, hour and dollar caps); for agy, a reservation under the global spend cap
8
+ (`caddis_spend.py`, settled after the session); read the relay hash; launch ONE session through the governor with the
9
+ resume prompt and `CADDIS_HEADLESS=1` in its environment; parse its JSON result (cost, error); read the relay status
10
+ line; decide. CONTINUE with an unchanged relay is `FAILED no progress`; three sessions in a row with no newly closed
11
+ row is `BLOCKED stuck`.
12
+
13
+ Plan mode (`--plan <slug>`, decision D-I, design section 8.5) runs `.caddis/plans/<slug>.md`. A closed row in its
14
+ tracker or in `.caddis/open-work.md` is progress; the relay is `.caddis/relay/<slug>.md`; the log and STOP file are in
15
+ `.caddis/plans/.local/<slug>/`; the lock is `<git common dir>/caddis-runner/plan-<slug>/runner.lock`. The loop goes on only
16
+ at CONTINUE. Its lock holds the whole plan (`batch` `<slug>#*`). A plan that another live runner lock names is refused
17
+ (exit 3 at start, 1 at any later iteration: the check runs before every session).
18
+
19
+ Roadmap mode (`--roadmap <slug>`, caddis-next Phase 9) runs `.caddis/roadmaps/<slug>/roadmap.md` as it is on
20
+ `origin/<default>`; the log and STOP file are in `.caddis/roadmaps/<slug>/.local/`; the lock is
21
+ `<git common dir>/caddis-runner/roadmap-<slug>/runner.lock`. It refuses to start (exit 3) on a host that is not the roadmap's
22
+ `Runner host`. Each iteration: `caddis_roadmap.next_batch`; none is `BLOCKED nothing ready` (exit 1, the notification
23
+ carries the `status` line). `next` and the batch claim in the lock happen under one repo-level claim lock, so two
24
+ runners never choose the same batch. Found: print `next: <milestone> <plan>#<phase>`. A phase whose `**Lane:**` line
25
+ is not this engine (a cloud session, the owner, another tool), or that has no Lane line, is not run: the runner adds an
26
+ `owner-gate` blocker `phase <n> lane: <lane>` (`none` when there is no line) and goes on. Else a `todo` milestone is
27
+ set `in-progress`, the runner makes a fresh branch `caddis/roadmap/<slug>/<plan>-p<n>` from origin/<default>, and ONE
28
+ session runs `plan-resume-prompt.md` for that plan and phase on it. Sessions never push. A relay `STATUS: CONTINUE`
29
+ with new commits on that branch (still checked out, nothing under `.caddis/roadmaps/` changed): the RUNNER writes a
30
+ persistent claim for the phase, pushes the branch, opens a PR (`gh api`) and stops `BLOCKED waiting for merge
31
+ <plan>#<phase> <url>` (exit 1); `--last-stop` lets the recheck task restart it once the PR merged; the restart checks
32
+ the merged origin roadmap against the hash taken at publish time, removes the claim, rolls that milestone up and goes
33
+ on. Every roadmap runs its sessions in ONE dedicated worktree, `%TEMP%/caddis-roadmap-<slug>-<repo tag>` (made with
34
+ `git worktree add --detach` from origin/<default> on the first session, reused on restart, recorded in the runner
35
+ folder, never the `--repo` checkout); when the roadmap is finished the runner removes it if it is clean, and keeps it
36
+ (saying so in the stop line) if not. The caps (hours, sessions, USD per session and in total) are per run, in memory. CONTINUE with the phase already merged on
37
+ origin: `rollup` when every phase of the milestone is merged, then the next batch. A relay `STATUS: OWNER-GATE <r>`
38
+ becomes an `owner-gate` blocker, `STATUS: BLOCKED <r>` an `external` one, and the loop goes on.
39
+
40
+ Review gate (caddis-next Phase 11): a phase is pending review when its Tracker row is done in the working copy of
41
+ the plan (its path, or `.caddis/plans/done/` after tidy moved it), not done on origin/<default>, marked
42
+ `**Risk:** high` in any copy (working, origin, the copy from before the session; `caddis_review.phase_risk`), and not
43
+ recorded clean in this runner's own `<git common dir>/caddis-runner/<kind>-<slug>/reviewed/<plan>.json`
44
+ (phase -> head, utc, result; another runner's record does not count). Plan mode
45
+ reviews every pending phase before each session (so a restart still reviews what an earlier run left); roadmap mode
46
+ reviews the plan's pending phases BEFORE it pushes and opens the PR, so a `risk: high` phase is never published
47
+ unreviewed. Pending is recomputed after every review and every fix session. Each review is `caddis_review.py --plan
48
+ <plan> --phase <n> --runs 2 --out-dir <local>/reviews/<plan>`, a child with `CADDIS_HEADLESS=1` (the `review_cmd`
49
+ seam). Exit 0 records the phase clean. Exit 1: ONE fix session for that phase (a session against the session cap and
50
+ the USD caps) with the prompt `Read <out-dir>/review-<n>-merged.md and fix every blocking finding
51
+ ...`, then one more review; exit 1 again stops `BLOCKED review`. Exit 3, 4, any other code, a timeout, exit 1 with
52
+ no merged file, a plan that cannot be read, or a done row with no `### Phase <n>` heading stops `BLOCKED review not
53
+ run`. Neither stop is restartable by the recheck task; an owner restart reviews the phase again. The review's
54
+ `review-<n>-cost.json` `total_cost_usd` is added to the spend; a missing or unreadable file adds USD 10 (USD 5 per
55
+ run, 2 runs). A review that could take the spend over the total cap does not start (a cap stop). The
56
+ review child (the reviewer has full Bash) and the fix session are each guarded by the runner-state check like any
57
+ session; the runner writes `reviewed/<plan>.json` only between children, before the next child's snapshot.
58
+
59
+ Snapshot guard: one baseline across the whole run. The Milestones and Blockers tables (every cell of every row, rows
60
+ added, removed or moved) and the `Runner host` line are frozen. Every origin read (each `next`, and the reads after
61
+ the runner's own writes and after each session) is compared with the previous read: the only differences allowed are
62
+ the exact edits the runner made and recorded for that read (a Status it set, a blocker row it added, and what a
63
+ rollup's audit entry says it actually wrote). Anything else stops the run `FAILED roadmap changed` (exit 1). Every
64
+ roadmap change goes through `caddis_roadmap.py` as a child process with `CADDIS_HEADLESS=1`; the runner never passes
65
+ `--by-owner`.
66
+
67
+ Runner state: one folder per runner, `<git common dir>/caddis-runner/<roadmap|plan>-<slug>/` (lock, last stop,
68
+ options, branches, worktree, saved roadmap text); shared at the top level: `claims/` and `claim.lock`. Sessions
69
+ are cooperative (our own sessions): the runner guards against mistakes and runaway spend, not against a hostile
70
+ session. Around every session it hashes its own folder and its own claims; a change there is `FAILED runner state
71
+ changed`. Another runner's writes never stop it. A restart checks that the claim its last stop records
72
+ is still there, unchanged (`FAILED own claim missing or changed`).
73
+
74
+ Lock: `<git common dir>/caddis-runner/<roadmap|plan>-<slug>/runner.lock` (every worktree sees it; outside a git repo,
75
+ `<local>/runner.lock`) holds `{"host", "pid", "started_utc", "batch", "max_hours"}`. On this host a lock is stale when
76
+ its pid is not running, or the process now holding that pid started after `started_utc` (pid reuse after a reboot;
77
+ when the start time cannot be read, live). Another host's lock is stale when `started_utc` is older than the lock's
78
+ own `max_hours` plus 1 h. A stale lock is removed with a toast and the run goes on; a live lock is exit 3. The runner
79
+ that holds the lock records `running` as its last stop at once, with its launch options, and its real last stop at the
80
+ end; an unexpected exception records `FAILED crash: <type>`, notifies once and re-raises. `roadmap-recheck.ps1`
81
+ restarts the runner only after `BLOCKED nothing ready`, a merged `BLOCKED waiting for merge`, or no recorded stop
82
+ (`--last-stop`), with the recorded options clamped to the built-in caps (`--restart-args`). Every lock write is a temp
83
+ file in the same folder, then a hard link (create) or os.replace.
84
+
85
+ On every stop: one Windows toast, and the push-notification fallback when the last session did not report `PUSH-SENT`
86
+ (not for a STOP file, a lock or a usage limit). One JSON line per session and one per stop go to `<local>/runner.jsonl`.
87
+ Session output, relay text and every log line, printed stop line, toast and push pass through `caddis_redact` first.
88
+
89
+ The runner never passes an owner gate, never raises a cap and never deletes a STOP file.
90
+
91
+ Exit codes: 0 stopped at a status the session wrote (plan mode: OWNER-GATE, BLOCKED), or at a STOP file after a
92
+ session; 1 a FAILED stop, a cap, the agy spend cap, a usage limit, `BLOCKED stuck`, `BLOCKED nothing ready`,
93
+ `BLOCKED waiting for merge`, `BLOCKED review`, `BLOCKED review not run`, or the snapshot guard; 3 never started (lock, STOP file, no governor, agy spend cap not configured,
94
+ another host, a plan a roadmap runner holds); 4 bad input (no plan or roadmap, both, a bad slug, no prompt template,
95
+ bad engine).
96
+
97
+ Engines: `claude` is built and live-tested. `agy` and `codex` are command-line adapters with unit tests only (not
98
+ live-tested; design 8.2).
99
+ """
100
+
101
+ from __future__ import annotations
102
+
103
+ import argparse
104
+ import contextlib
105
+ import difflib
106
+ import hashlib
107
+ import json
108
+ import os
109
+ import re
110
+ import socket
111
+ import subprocess
112
+ import sys
113
+ import tempfile
114
+ import time
115
+ import uuid
116
+ from collections import Counter
117
+ from dataclasses import dataclass, field
118
+ from datetime import datetime, timezone
119
+ from pathlib import Path
120
+ from typing import Callable, Sequence
121
+
122
+ _HERE = Path(__file__).resolve().parent
123
+ try:
124
+ import caddis_exit
125
+ except ModuleNotFoundError as exc:
126
+ if exc.name != "caddis_exit":
127
+ raise
128
+ sys.path.insert(0, str(_HERE))
129
+ sys.path.insert(0, str(_HERE.parent / "claude-harness" / "scripts"))
130
+ import caddis_exit
131
+
132
+ import caddis_notify # noqa: E402
133
+ import caddis_redact # noqa: E402
134
+ import caddis_review # noqa: E402
135
+ import caddis_roadmap # noqa: E402
136
+ import caddis_spend # noqa: E402
137
+ import caddis_workstreams # noqa: E402
138
+
139
+ ENGINES: dict[str, dict] = {
140
+ "claude": {"live_tested": True, "default_model": "opus", "limit_key": "claude"},
141
+ "agy": {"live_tested": False, "default_model": "gemini-4-argon-medium", "limit_key": "agy"},
142
+ "codex": {"live_tested": False, "default_model": "gpt-5", "limit_key": "codex-sol"},
143
+ }
144
+ PROMPT_ON_STDIN = {"claude"}
145
+ STUCK_SESSIONS = 3
146
+ PLAN_PROMPT_NAME = "plan-resume-prompt.md"
147
+ HEADLESS_ENV = {"CADDIS_HEADLESS": "1"}
148
+ ROADMAP_WRITE_TIMEOUT_S = caddis_roadmap.MERGE_CHAIN_TIMEOUT + 600 # a backstop: every call inside is bounded
149
+ RUNNING = "running" # the last-stop record while a runner holds the lock; a killed run leaves it (needs the owner)
150
+ GH_STOP_PREFIX = "BLOCKED gh cannot see" # a gh error at a restart: waiting.json stays, a later tick may resume
151
+ RESTARTABLE_STOP = "BLOCKED nothing ready" # restartable once a blocker cleared
152
+ WAITING_PREFIX = "BLOCKED waiting for merge" # restartable once its PR merged
153
+ ROLLUP_BLOCK_RE = r"exit check failing \((exit -?\d+|timed out after \d+ s)\)" # the blocker rollup writes
154
+ _SLUG_RE = re.compile(r"[a-z0-9][a-z0-9-]*")
155
+ _CLOSED = {"done", "not-needed"}
156
+ _LANE_RE = re.compile(r"^\s*\*\*Lane:\*\*\s*(.*?)\s*$")
157
+ _PHASE_HEAD_RE = re.compile(r"^ {0,3}#{2,4}\s+Phase\s+(\d+)\b", re.IGNORECASE)
158
+ _HEAD_RE = re.compile(r"^ {0,3}#{1,6}\s")
159
+ _LANE_TOOL_RE = re.compile(r"^\W*([A-Za-z][A-Za-z0-9_-]*)")
160
+ _NOT_RUNNER_LANE_RE = re.compile(r"cloud|owner", re.IGNORECASE)
161
+
162
+
163
+ @dataclass
164
+ class Caps:
165
+ max_hours: float = 10.0
166
+ max_sessions: int = 12
167
+ session_timeout_min: float = 120.0
168
+ max_usd_per_session: float = 25.0
169
+ max_usd_total: float = 150.0
170
+
171
+
172
+ def _usd(value: float) -> str:
173
+ return f"{round(value, 2):g}"
174
+
175
+
176
+ def headless_env(base: dict | None = None) -> dict:
177
+ """A copy of the environment with CADDIS_HEADLESS=1: headless detection cannot see a process started in a fresh
178
+ console, so every session and every caddis_roadmap.py call the runner starts carries the marker."""
179
+ env = dict(os.environ if base is None else base)
180
+ env.update(HEADLESS_ENV)
181
+ return env
182
+
183
+
184
+ def build_command(engine: str, model: str, prompt: str, *, usd: float, timeout_min: float,
185
+ governor: Sequence[str], engine_cmd: Sequence[str] | None = None) -> list[str]:
186
+ """The full command line for one session: the governor prefix, then the engine's print-mode call.
187
+
188
+ `claude` gets its prompt on stdin, not as an argument (PROMPT_ON_STDIN): on Windows `claude` is an npm `.cmd`
189
+ shim, and cmd.exe cuts a multi-line argument at its first newline, which drops the rest of the prompt and every
190
+ argument after it."""
191
+ if engine not in ENGINES:
192
+ raise ValueError(f"unknown engine {engine!r}; known: {', '.join(ENGINES)}")
193
+ exe = list(engine_cmd) if engine_cmd else [engine]
194
+ if engine == "claude":
195
+ tail = [*exe, "-p", "--model", model, "--output-format", "json", "--max-budget-usd", _usd(usd)]
196
+ elif engine == "agy": # probe 5: --model must come before --print
197
+ tail = [*exe, "--model", model, "--print", prompt, "--output-format", "json",
198
+ "--print-timeout", f"{max(1, int(timeout_min))}m"]
199
+ else:
200
+ tail = [*exe, "exec", "--model", model, "--json", prompt]
201
+ return [*governor, *tail]
202
+
203
+
204
+ def default_governor() -> list[str] | None:
205
+ """`caddis_governor.py exec --scope session --`, preferring the admin-installed copy (as caddis-run.ps1 does)."""
206
+ installed = Path(os.environ.get("ProgramData", r"C:\ProgramData")) / "caddis" / "caddis_governor.py"
207
+ for gov in (installed, _HERE / "caddis_governor.py", _HERE.parent / "claude-harness" / "scripts" / "caddis_governor.py"):
208
+ if gov.is_file():
209
+ return [sys.executable, str(gov), "exec", "--scope", "session", "--"]
210
+ return None
211
+
212
+
213
+ def default_roadmap_cmd() -> list[str]:
214
+ return [sys.executable, str(_HERE / "caddis_roadmap.py")]
215
+
216
+
217
+ def find_prompt_template(name: str = PLAN_PROMPT_NAME) -> Path | None:
218
+ for assets in (_HERE.parent / ".github" / "skills" / "workflow" / "roadmap" / "assets",
219
+ _HERE.parent / "skills" / "roadmap" / "assets"):
220
+ if (assets / name).is_file():
221
+ return assets / name
222
+ return None
223
+
224
+
225
+ def _local_dir(repo: Path, kind: str, slug: str) -> Path:
226
+ """The runner's local folder: plan `.caddis/plans/.local/<slug>`, roadmap `.caddis/roadmaps/<slug>/.local`."""
227
+ if kind == "plan":
228
+ return Path(repo) / ".caddis" / "plans" / ".local" / slug
229
+ if kind == "roadmap":
230
+ return Path(repo) / ".caddis" / "roadmaps" / slug / ".local"
231
+ raise ValueError(f"unknown target kind {kind!r}")
232
+
233
+
234
+ def _ensure_local(path: Path) -> Path:
235
+ """Create the local folder. A plan's `.caddis/plans/.local/` and a roadmap's `.local/` get a `.gitignore` of `*`
236
+ when they have none (the repo's own `.gitignore` may not cover them)."""
237
+ path = Path(path)
238
+ path.mkdir(parents=True, exist_ok=True)
239
+ if path.parent.name == ".local" and path.parent.parent.name == "plans":
240
+ gitignore = path.parent / ".gitignore"
241
+ elif path.name == ".local" and path.parent.parent.name == "roadmaps":
242
+ gitignore = path / ".gitignore"
243
+ else:
244
+ return path
245
+ if not gitignore.exists():
246
+ gitignore.write_text("*\n", encoding="utf-8")
247
+ return path
248
+
249
+
250
+ @dataclass
251
+ class Target:
252
+ """What the runner runs: a plain plan file (plan mode) or a roadmap (roadmap mode)."""
253
+ kind: str # "plan" | "roadmap"
254
+ slug: str
255
+ local: Path # lock, runner.jsonl, STOP
256
+ progress: tuple[Path, ...] # plan mode: files whose closed rows count as progress
257
+ prompt_name: str
258
+
259
+
260
+ def resolve_target(repo: Path, plan: str | None) -> Target:
261
+ """The plan-mode target; ValueError when the slug is missing or bad, or its plan file is missing."""
262
+ slug = plan
263
+ if not isinstance(slug, str) or not _SLUG_RE.fullmatch(slug):
264
+ raise ValueError(f"bad plan {slug!r}: invalid slug: {slug!r}")
265
+ plan_file = repo / ".caddis" / "plans" / f"{slug}.md"
266
+ if not plan_file.is_file():
267
+ raise ValueError(f"no plan {slug!r}: {plan_file} does not exist")
268
+ return Target("plan", slug, _local_dir(repo, "plan", slug), (plan_file, repo / ".caddis" / "open-work.md"),
269
+ PLAN_PROMPT_NAME)
270
+
271
+
272
+ def render_prompt(template: Path, slug: str, phase: int | None = None) -> str:
273
+ """The template body with `{{slug}}`, `{{phase}}` (empty when None) and `{{scripts}}` (this scripts folder: an
274
+ app repo has no caddis `scripts/`) filled in."""
275
+ text = template.read_text(encoding="utf-8")
276
+ _meta, body = caddis_workstreams.split_document(text)
277
+ for key, value in (("{{slug}}", slug), ("{{phase}}", "" if phase is None else str(phase)),
278
+ ("{{scripts}}", _HERE.as_posix())):
279
+ body = body.replace(key, value)
280
+ return body.strip() + "\n"
281
+
282
+
283
+ # -- open work: closed rows -------------------------------------------------------------------------------------------
284
+
285
+
286
+ def closed_rows(open_work: Path) -> int:
287
+ """Rows whose `Status` column (by name) is done or not-needed, over every table of the file."""
288
+ try:
289
+ lines = open_work.read_text(encoding="utf-8").splitlines()
290
+ except OSError:
291
+ return 0
292
+ count, status_col = 0, None
293
+ for line in lines:
294
+ s = line.strip()
295
+ if not s.startswith("|"):
296
+ status_col = None
297
+ continue
298
+ cells = [c.strip() for c in s.strip("|").split("|")]
299
+ if status_col is None:
300
+ status_col = cells.index("Status") if "Status" in cells else -1
301
+ continue
302
+ if status_col >= 0 and status_col < len(cells) and cells[status_col].lower() in _CLOSED:
303
+ count += 1
304
+ return count
305
+
306
+
307
+ # -- session output ---------------------------------------------------------------------------------------------------
308
+
309
+
310
+ def parse_result(stdout: str) -> dict | None:
311
+ text = (stdout or "").strip()
312
+ candidates = [text] + [line for line in reversed(text.splitlines()) if line.lstrip().startswith("{")]
313
+ for candidate in candidates:
314
+ try:
315
+ value = json.loads(candidate)
316
+ except ValueError:
317
+ continue
318
+ if isinstance(value, dict):
319
+ return value
320
+ return None
321
+
322
+
323
+ def _agy_engine_model(repo: Path) -> str:
324
+ """`<agy model>-medium`: the `[lanes.models] agy` pin in `<repo>/.caddis/config.toml` when set, else the default
325
+ agy model as `agy models` offers it here (see caddis_lanes.resolve_models)."""
326
+ try:
327
+ try:
328
+ import caddis_lanes
329
+ except ModuleNotFoundError:
330
+ sys.path.insert(0, str(_HERE))
331
+ import caddis_lanes
332
+ return caddis_lanes.resolve_models(caddis_lanes.load_lanes_config(repo))["agy"] + "-medium"
333
+ except Exception: # the lane engine is missing or the config is bad: the display default
334
+ return ENGINES["agy"]["default_model"]
335
+
336
+
337
+ def usage_limited(engine: str, text: str) -> bool:
338
+ try:
339
+ import caddis_lanes
340
+ patterns = caddis_lanes.BUDGET_PATTERNS.get(ENGINES[engine]["limit_key"], ())
341
+ except Exception: # the lane engine is missing: use the Claude wording only
342
+ patterns = (r"You[’']ve hit your (?:session|weekly|Opus|Sonnet|Fable|usage credit) limit\b",)
343
+ return any(re.search(p, text, re.IGNORECASE) for p in patterns)
344
+
345
+
346
+ def _read_relay(repo: Path, slug: str) -> tuple[str, str | None]:
347
+ """The plan's relay text and hash; an unsafe relay path (a symlink, an escape) counts as no relay, never a crash."""
348
+ try:
349
+ return caddis_workstreams.read_relay(repo, slug)
350
+ except (ValueError, OSError) as exc: # OSError: the relay path is a folder or locked - still no crash
351
+ print(f"caddis_roadmap_run: relay not read: {exc}", file=sys.stderr)
352
+ return "", None
353
+
354
+
355
+ def _cost(result: dict | None) -> float:
356
+ value = (result or {}).get("total_cost_usd", 0)
357
+ return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) and value >= 0 else 0.0
358
+
359
+
360
+ # -- roadmap mode helpers ---------------------------------------------------------------------------------------------
361
+
362
+
363
+ def phase_lane(plan_text: str, phase: int) -> str | None:
364
+ """The `**Lane:**` text of the `Phase <n>` heading's section in a plan (until the next heading of the same or a
365
+ higher level), or None when the phase has no Lane line. Fenced code is skipped."""
366
+ level = 0 # the open phase heading's level; 0 = not inside the phase
367
+ for line in caddis_roadmap._mask_fences(plan_text.splitlines()):
368
+ head = _HEAD_RE.match(line)
369
+ if head:
370
+ this_level = len(line.lstrip(" ")) - len(line.lstrip(" ").lstrip("#"))
371
+ phase_head = _PHASE_HEAD_RE.match(line)
372
+ if phase_head:
373
+ level = this_level if int(phase_head.group(1)) == phase else 0
374
+ elif level and this_level <= level:
375
+ level = 0
376
+ continue
377
+ if level:
378
+ match = _LANE_RE.match(line)
379
+ if match:
380
+ return match.group(1)
381
+ return None
382
+
383
+
384
+ def lane_runs_here(lane: str, engine: str) -> bool:
385
+ """True when the runner may run a phase with this Lane: its first word names this engine, and it names neither
386
+ a cloud session nor the owner. Anything else (another tool, no tool named) is not the runner's (fail closed)."""
387
+ if _NOT_RUNNER_LANE_RE.search(lane):
388
+ return False
389
+ match = _LANE_TOOL_RE.match(lane)
390
+ return bool(match) and match.group(1).lower() == engine
391
+
392
+
393
+ def _milestone_cells(ms: "caddis_roadmap.Milestone") -> list:
394
+ return [ms.id, ms.name, list(ms.phases), list(ms.depends), ms.exit_check, ms.status, ms.owner, ms.target]
395
+
396
+
397
+ def roadmap_guard(before: "caddis_roadmap.Roadmap", after: "caddis_roadmap.Roadmap",
398
+ edits: Sequence[tuple] = ()) -> list[str]:
399
+ """Every difference between two origin reads that the runner's own recorded `edits` do not explain.
400
+
401
+ The Milestones and Blockers tables and the `Runner host` line are frozen: every cell of every milestone row, the
402
+ row order, and the multiset of blocker rows. `edits` are what the runner itself wrote since `before`:
403
+ `("status", mid, value)`, `("blocker", mid, type, reason)` (a row with no Check; nothing when the same
404
+ milestone, type and reason exist, as `caddis_roadmap.add_blocker` does), and `("blocker-like", mid, type,
405
+ regex)` (one added row whose reason matches, for the blocker a rollup writes)."""
406
+ cols = caddis_roadmap.MILESTONE_COLS
407
+ rows = {ms.id: _milestone_cells(ms) for ms in before.milestones}
408
+ blockers = Counter(b.key for b in before.blockers)
409
+ patterns: list[tuple[str, str, str]] = []
410
+ for edit in edits:
411
+ if edit[0] == "status" and edit[1] in rows:
412
+ rows[edit[1]][cols.index("Status")] = edit[2]
413
+ elif edit[0] == "blocker":
414
+ _kind, mid, btype, reason = edit
415
+ if not any(k[0] == mid and k[1] == btype and k[2] == reason for k in blockers):
416
+ blockers[(mid, btype, reason, "")] += 1
417
+ elif edit[0] == "blocker-like":
418
+ patterns.append(tuple(edit[1:]))
419
+ changes: list[str] = []
420
+ if after.runner_host != before.runner_host:
421
+ changes.append("Runner host changed")
422
+ now = {ms.id: _milestone_cells(ms) for ms in after.milestones}
423
+ changes += [f"{mid} milestone removed" for mid in rows if mid not in now]
424
+ changes += [f"{mid} milestone added" for mid in now if mid not in rows]
425
+ for mid in (m for m in rows if m in now):
426
+ changed = [col for col, old, new in zip(cols, rows[mid], now[mid]) if old != new]
427
+ if changed:
428
+ changes.append(f"{mid} {', '.join(changed)} changed")
429
+ kept_before = [ms.id for ms in before.milestones if ms.id in now]
430
+ kept_after = [ms.id for ms in after.milestones if ms.id in rows]
431
+ if kept_before != kept_after:
432
+ changes.append("milestone rows reordered")
433
+ found = Counter(b.key for b in after.blockers)
434
+ for key, n in sorted((blockers - found).items()):
435
+ changes.append(f"{key[0]} {key[1]} blocker removed ({key[2]})" + (f" x{n}" if n > 1 else ""))
436
+ for key, n in sorted((found - blockers).items()):
437
+ for _ in range(n):
438
+ match = next((p for p in patterns if p[0] == key[0] and p[1] == key[1] and not key[3]
439
+ and re.fullmatch(p[2], key[2])), None)
440
+ if match is not None:
441
+ patterns.remove(match)
442
+ else:
443
+ changes.append(f"{key[0]} {key[1]} blocker added ({key[2]})")
444
+ return changes
445
+
446
+
447
+ def _done_phases(snap: "caddis_roadmap.Snapshot", plan: str) -> int:
448
+ rows = snap.phases.tracker(plan) or {}
449
+ return sum(1 for cell in rows.values() if caddis_roadmap._is_done_status(cell))
450
+
451
+
452
+ def _plan_text(snap: "caddis_roadmap.Snapshot", plan: str) -> str:
453
+ for path in (f".caddis/plans/{plan}.md", f".caddis/plans/done/{plan}.md"):
454
+ text = caddis_roadmap.show(snap.root, snap.branch, path)
455
+ if text is not None:
456
+ return text
457
+ return ""
458
+
459
+
460
+ def _status_line(repo: Path, slug: str) -> str:
461
+ try:
462
+ return caddis_roadmap.status(repo, slug)["line"]
463
+ except Exception as exc: # the stop must still notify
464
+ return f"status not read: {type(exc).__name__}"
465
+
466
+
467
+ # -- the lock ---------------------------------------------------------------------------------------------------------
468
+
469
+
470
+ def _tmp_beside(path: Path, text: str) -> Path:
471
+ tmp = path.with_name(f"{path.name}.{os.getpid()}-{uuid.uuid4().hex}.tmp")
472
+ tmp.write_text(text, encoding="utf-8")
473
+ return tmp
474
+
475
+
476
+ def _create_with(path: Path, text: str) -> None:
477
+ """Create `path` holding `text` in one step (a reader never sees it empty); FileExistsError when it exists.
478
+ A temp file in the same folder is hard-linked to the name (the link fails when the name exists)."""
479
+ tmp = _tmp_beside(path, text)
480
+ try:
481
+ os.link(tmp, path)
482
+ except FileExistsError:
483
+ raise
484
+ except OSError: # no hard links on this file system: O_EXCL, then the text
485
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
486
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
487
+ fh.write(text)
488
+ finally:
489
+ tmp.unlink(missing_ok=True)
490
+
491
+
492
+ def _replace_with(path: Path, text: str) -> None:
493
+ """Rewrite `path` with a temp file in the same folder and os.replace (a reader sees the old or the new text)."""
494
+ tmp = _tmp_beside(path, text)
495
+ try:
496
+ os.replace(tmp, path)
497
+ finally:
498
+ tmp.unlink(missing_ok=True)
499
+
500
+
501
+ class _RunnerLock:
502
+ """The runner lock: JSON `{"host", "pid", "started_utc", "batch", "max_hours"}`, created with O_EXCL, in
503
+ `<git common dir>/caddis-runner/<kind>-<slug>/` (every worktree sees it; outside a git repo, the local folder).
504
+
505
+ A stale lock (caddis_roadmap.runner_lock_stale: judged with the `max_hours` the lock itself records) is taken over
506
+ race-safely: rename it to a unique name (only one contender's rename succeeds), check the renamed file still
507
+ holds what was judged stale, then create the lock."""
508
+
509
+ def __init__(self, path: Path, max_hours: float, batch: str = "", claim_path: Path | None = None):
510
+ self.path, self.max_hours = Path(path), max_hours
511
+ self.claim_path = claim_path # takeovers of a stale lock happen under this repo-level lock
512
+ self.text: str | None = None
513
+ self.holder = {"host": socket.gethostname(), "pid": os.getpid(),
514
+ "started_utc": _iso(datetime.now(timezone.utc)), "batch": batch, "max_hours": max_hours}
515
+
516
+ def acquire(self, on_stale: Callable[[str], None], claim_held: bool = False) -> None:
517
+ """`claim_held`: the caller already holds claim.lock (plan mode): the stale takeover must not take it again."""
518
+ self.path.parent.mkdir(parents=True, exist_ok=True)
519
+ for attempt in range(3):
520
+ text = json.dumps(self.holder)
521
+ try:
522
+ _create_with(self.path, text)
523
+ except FileExistsError:
524
+ if attempt < 2 and self._take_stale(on_stale, claim_held):
525
+ continue
526
+ raise _Stop(f"BLOCKED another runner holds {self.path} (live; delete it only when no runner runs)",
527
+ caddis_exit.NOT_RUN, push=False) from None
528
+ self.text = text
529
+ return
530
+ raise _Stop(f"BLOCKED another runner holds {self.path}", caddis_exit.NOT_RUN, push=False)
531
+
532
+ def _take_stale(self, on_stale: Callable[[str], None], claim_held: bool = False) -> bool:
533
+ if self.claim_path is not None: # serialised: of several contenders on one stale lock, one takes it over
534
+ try:
535
+ with (contextlib.nullcontext() if claim_held
536
+ else caddis_roadmap._Lock(self.claim_path, timeout=caddis_roadmap.CLAIM_TIMEOUT)):
537
+ try:
538
+ judged = self.path.read_bytes()
539
+ except FileNotFoundError:
540
+ return True
541
+ stale = caddis_roadmap.runner_lock_stale(self.path)
542
+ if stale is None:
543
+ return True # gone between the read and the judgement: free, try the create again
544
+ if not stale:
545
+ return False # live: a contender before us won
546
+ self.path.unlink(missing_ok=True)
547
+ except caddis_roadmap.RoadmapError:
548
+ return False
549
+ on_stale(f"removed a stale runner lock {self.path}: {judged.decode('utf-8', 'replace')[:200]}")
550
+ return True
551
+ try:
552
+ judged = self.path.read_bytes()
553
+ except FileNotFoundError:
554
+ return True # gone: try to create it again
555
+ except OSError:
556
+ return False
557
+ stale = caddis_roadmap.runner_lock_stale(self.path)
558
+ if stale is None:
559
+ return True
560
+ if not stale:
561
+ return False
562
+ grave = self.path.with_name(f"{self.path.name}.stale-{os.getpid()}-{uuid.uuid4().hex}")
563
+ try:
564
+ os.rename(self.path, grave)
565
+ except OSError: # another contender took it first, or the holder still has it open
566
+ return False
567
+ try:
568
+ if grave.read_bytes() != judged: # a fresh lock replaced the stale one: put it back
569
+ with contextlib.suppress(OSError):
570
+ os.link(grave, self.path)
571
+ return False
572
+ finally:
573
+ grave.unlink(missing_ok=True)
574
+ on_stale(f"removed a stale runner lock {self.path}: {judged.decode('utf-8', 'replace')[:200]}")
575
+ return True
576
+
577
+ def set_batch(self, batch: str) -> None:
578
+ self.holder["batch"] = batch
579
+ text = json.dumps(self.holder)
580
+ try:
581
+ _replace_with(self.path, text)
582
+ self.text = text
583
+ except OSError as exc: # other runners could not see the batch: never run it unclaimed
584
+ raise _Stop(f"BLOCKED claim not written: {batch} ({type(exc).__name__})", caddis_exit.BLOCKED) from None
585
+
586
+ def release(self) -> None:
587
+ try:
588
+ if self.text is not None and self.path.read_text(encoding="utf-8") == self.text:
589
+ self.path.unlink()
590
+ except OSError:
591
+ pass
592
+
593
+
594
+ def runner_lock_file(repo: Path, kind: str, slug: str) -> Path:
595
+ """The runner lock path: in the git common dir, or the local folder outside a git repo."""
596
+ return caddis_roadmap.runner_lock_path(repo, kind, slug) or _local_dir(repo, kind, slug) / "runner.lock"
597
+
598
+
599
+ def last_stop_file(repo: Path, kind: str, slug: str) -> Path:
600
+ return caddis_roadmap.last_stop_path(repo, kind, slug) or _local_dir(repo, kind, slug) / "last-stop.json"
601
+
602
+
603
+ @contextlib.contextmanager
604
+ def _claim(repo: Path):
605
+ """One repo-level lock around `next` plus the batch claim (and plan mode's holder check plus its lock), so two
606
+ runners cannot choose the same batch. Outside a git repo there is nothing to share it with."""
607
+ path = caddis_roadmap.claim_lock_path(repo)
608
+ if path is None:
609
+ yield
610
+ return
611
+ try:
612
+ lock = caddis_roadmap._Lock(path, timeout=caddis_roadmap.CLAIM_TIMEOUT)
613
+ lock.__enter__()
614
+ except caddis_roadmap.RoadmapError as exc:
615
+ raise _Stop(f"BLOCKED claim lock busy: {exc}", caddis_exit.BLOCKED) from None
616
+ try:
617
+ yield
618
+ finally:
619
+ lock.__exit__(None, None, None)
620
+
621
+
622
+ def _roadmap_holder_of(repo: Path, plan: str, own_lock: Path | None = None) -> tuple[dict, str] | None:
623
+ """(holder, batch) when another live runner lock holds a phase of this plan (or the whole plan)."""
624
+ for batch, holder in caddis_roadmap.live_runner_holders(repo, exclude=own_lock).items():
625
+ if caddis_roadmap.match_ref(batch) == plan:
626
+ return holder, batch
627
+ return None
628
+
629
+
630
+ def _read_record(path: Path) -> dict:
631
+ try:
632
+ record = json.loads(Path(path).read_text(encoding="utf-8"))
633
+ except (OSError, ValueError):
634
+ return {}
635
+ return record if isinstance(record, dict) else {}
636
+
637
+
638
+ class GhError(caddis_roadmap.RoadmapError):
639
+ """gh could not answer (not installed, no login, 401/404, bad output): an error, never "not merged"."""
640
+
641
+
642
+ GH_ACCOUNT = {"name": None} # the account the runner's gh calls use (GH_TOKEN set from it), None: the active one
643
+
644
+
645
+ def use_gh_account(account: str | None) -> str | None:
646
+ """Make every gh call of this process (and of the caddis_roadmap.py children, which inherit the environment) use
647
+ `account`: GH_TOKEN from `gh auth token -u <account>`. None keeps the environment / active account. Returns the
648
+ value set, so it can be masked; GhError when gh cannot give it."""
649
+ GH_ACCOUNT["name"] = account or None
650
+ if not account:
651
+ return None
652
+ try:
653
+ proc = subprocess.run(["gh", "auth", "token", "-u", account], capture_output=True, text=True,
654
+ encoding="utf-8", errors="replace", stdin=subprocess.DEVNULL,
655
+ timeout=caddis_roadmap.GH_TIMEOUT)
656
+ except (OSError, subprocess.TimeoutExpired) as exc:
657
+ raise GhError(f"gh cannot give a login for account {account}: {type(exc).__name__}",
658
+ caddis_exit.BLOCKED) from None
659
+ gh_auth = (proc.stdout or "").strip()
660
+ if proc.returncode != 0 or not gh_auth:
661
+ raise GhError(f"gh has no login for account {account}", caddis_exit.BLOCKED)
662
+ os.environ["GH_TOKEN"] = gh_auth
663
+ return gh_auth
664
+
665
+
666
+ def _gh_who() -> str:
667
+ return GH_ACCOUNT["name"] or "(the active gh account)"
668
+
669
+
670
+ def _pr_state(url: str) -> dict | None:
671
+ """The PR document from `gh api repos/<o>/<r>/pulls/<n>`; None for a URL that is not a GitHub PR. GhError when
672
+ gh cannot answer (missing, 401, 404, a timeout, bad output)."""
673
+ match = re.search(r"github\.com/([^/]+/[^/]+)/pull/(\d+)$", url or "")
674
+ if not match:
675
+ return None
676
+ gh_repo = match.group(1)
677
+ try:
678
+ proc = caddis_roadmap.run_bounded(["gh", "api", f"repos/{gh_repo}/pulls/{match.group(2)}"],
679
+ caddis_roadmap.GH_TIMEOUT, "gh api (PR state)")
680
+ except (OSError, caddis_roadmap.RoadmapError) as exc:
681
+ raise GhError(f"gh cannot see {gh_repo} with account {_gh_who()}: {type(exc).__name__}",
682
+ caddis_exit.BLOCKED) from None
683
+ if proc.returncode != 0:
684
+ tail = " ".join((proc.stderr or proc.stdout or "").split())[:160]
685
+ raise GhError(f"gh cannot see {gh_repo} with account {_gh_who()}: {tail}", caddis_exit.BLOCKED)
686
+ try:
687
+ doc = json.loads(proc.stdout)
688
+ except ValueError:
689
+ raise GhError(f"gh gave no PR document for {url}", caddis_exit.BLOCKED) from None
690
+ if not isinstance(doc, dict):
691
+ raise GhError(f"gh gave no PR document for {url}", caddis_exit.BLOCKED)
692
+ return doc
693
+
694
+
695
+ def default_pr_merged(repo: Path, url: str) -> bool:
696
+ """GitHub says the PR merged. GhError when gh cannot answer."""
697
+ doc = _pr_state(url)
698
+ return bool(doc) and doc.get("merged") is True
699
+
700
+
701
+ def default_pr_closed(repo: Path, url: str) -> bool:
702
+ """GitHub says the PR is closed without a merge. GhError when gh cannot answer."""
703
+ doc = _pr_state(url)
704
+ return bool(doc) and doc.get("state") == "closed" and doc.get("merged") is not True
705
+
706
+
707
+ def last_stop_restartable(repo: Path, roadmap: str,
708
+ pr_merged: Callable[[Path, str], bool] | None = None,
709
+ pr_closed: Callable[[Path, str], bool] | None = None) -> tuple[bool, str]:
710
+ """(may the recheck task restart the runner, why). Yes after no recorded stop or `BLOCKED nothing ready` (then
711
+ only when recheck cleared a blocker), and after `BLOCKED waiting for merge` once GitHub says that PR merged (the
712
+ why starts with `merged:`; no cleared blocker is needed). Read only: no fetch, no ref or file changes (the runner
713
+ checks the Tracker on origin after it holds the lock). Everything else, `running` included, needs the owner."""
714
+ path = last_stop_file(repo, "roadmap", roadmap)
715
+ try:
716
+ record = json.loads(path.read_text(encoding="utf-8"))
717
+ except FileNotFoundError:
718
+ return True, "no recorded stop"
719
+ except (OSError, ValueError) as exc:
720
+ return False, f"last stop not readable ({type(exc).__name__}); the owner decides"
721
+ if not isinstance(record, dict):
722
+ return False, "last stop not readable (not an object); the owner decides"
723
+ reason = str(record.get("reason"))
724
+ if reason == RESTARTABLE_STOP:
725
+ held = [h for h in record.get("held") or [] if isinstance(h, dict) and h.get("ref")]
726
+ if held: # skipped only because another runner held them: free now?
727
+ live = caddis_roadmap.live_runner_batches(Path(repo), exclude=runner_lock_file(Path(repo), "roadmap",
728
+ roadmap))
729
+ held_plans = {caddis_roadmap.match_ref(b) for b in live}
730
+ still = [h["ref"] for h in held if caddis_roadmap.match_ref(h["ref"]) in held_plans]
731
+ if not still:
732
+ return True, "freed: " + ", ".join(h["ref"] for h in held)
733
+ ready = _rollup_ready_without_fetch(Path(repo), roadmap)
734
+ if ready: # a milestone waits only for its rollup (its blocker went): restart, no cleared blocker needed
735
+ return True, "rollup-ready: " + ", ".join(ready)
736
+ return True, f"last stop: {reason}"
737
+ waiting = _read_record(_state_file(Path(repo), roadmap, "waiting"))
738
+ if waiting and (reason.startswith(WAITING_PREFIX) or reason.startswith(GH_STOP_PREFIX)):
739
+ ref, url = str(waiting.get("ref", "")), str(waiting.get("pr", ""))
740
+ try:
741
+ if (pr_merged or default_pr_merged)(Path(repo), url): # gh only: no fetch, no ref changes
742
+ return True, f"merged: {ref} {url}"
743
+ if (pr_closed or default_pr_closed)(Path(repo), url): # read only: the runner's start cleans it up
744
+ return False, f"last stop needs the owner: the PR closed without a merge: {ref} {url}"
745
+ except Exception as exc: # gh missing, 401, 404: an error, never "not merged"
746
+ detail = str(exc).splitlines()[0] if str(exc) else type(exc).__name__
747
+ return False, f"gh error: {detail}"
748
+ return False, f"waiting for merge: {ref} {url}"
749
+ if reason == RUNNING:
750
+ return False, ("last stop needs the owner: running (a runner is running, or it was killed without a stop: "
751
+ "a crash, a reboot)")
752
+ return False, f"last stop needs the owner: {reason} (exit {record.get('exit')})"
753
+
754
+
755
+ # -- the runner -------------------------------------------------------------------------------------------------------
756
+
757
+
758
+ class _Stop(Exception):
759
+ def __init__(self, reason: str, code: int, *, push: bool = True, from_relay: bool = False,
760
+ status: tuple[str, str] | None = None, detail: str | None = None, record: dict | None = None):
761
+ super().__init__(reason)
762
+ self.reason, self.code, self.push, self.from_relay = reason, code, push, from_relay
763
+ self.status, self.detail, self.record = status, detail, record or {}
764
+
765
+
766
+ def _iso(moment: datetime) -> str:
767
+ return moment.astimezone(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
768
+
769
+
770
+ @dataclass
771
+ class _Ctx:
772
+ """What one run shares between its sessions."""
773
+ repo: Path
774
+ target: Target
775
+ caps: Caps
776
+ engine: str
777
+ model: str
778
+ engine_cmd: Sequence[str] | None
779
+ governor: Sequence[str]
780
+ home: Path
781
+ now: Callable[[], datetime]
782
+ clock: Callable[[], float]
783
+ toast: Callable[[str, str], int]
784
+ push: Callable[[str], int]
785
+ masks: list[str]
786
+ last_stop: Path | None = None # set once this run holds the lock: only the holder records its last stop
787
+ written_claims: list = field(default_factory=list) # claims this run wrote (removed again if it crashes)
788
+ gh_account: str | None = None # recorded in the launch options; recheck restarts with it
789
+ review_cmd: Sequence[str] = () # the `caddis_review.py` command (a seam: tests pass a fake)
790
+
791
+ @property
792
+ def log_path(self) -> Path:
793
+ return self.target.local / "runner.jsonl"
794
+
795
+ def log(self, record: dict) -> None:
796
+ _ensure_local(self.target.local)
797
+ with self.log_path.open("a", encoding="utf-8") as fh:
798
+ fh.write(json.dumps(caddis_redact.redact_obj(record, self.masks), ensure_ascii=False) + "\n")
799
+
800
+ def stop_file(self) -> bool:
801
+ return (self.home / ".caddis" / "STOP").exists() or (self.target.local / "STOP").exists()
802
+
803
+ def checks(self, sessions: int, spent: float, elapsed: float) -> None:
804
+ """Raise _Stop for the first reason not to launch another session."""
805
+ started = sessions > 0
806
+ if self.stop_file():
807
+ raise _Stop("STOP file", caddis_exit.CLEAN if started else caddis_exit.NOT_RUN, push=False)
808
+ if sessions >= self.caps.max_sessions:
809
+ raise _Stop(f"cap: sessions {self.caps.max_sessions}", caddis_exit.BLOCKED)
810
+ if elapsed >= self.caps.max_hours * 3600:
811
+ raise _Stop(f"cap: hours {elapsed / 3600:.2f} of {self.caps.max_hours:g}", caddis_exit.BLOCKED)
812
+ if spent >= self.caps.max_usd_total:
813
+ raise _Stop(f"cap: usd {spent:.2f} of {self.caps.max_usd_total:g}", caddis_exit.BLOCKED)
814
+
815
+
816
+ @dataclass
817
+ class _Session:
818
+ outcome: "_Outcome"
819
+ result: dict | None
820
+ cost: float
821
+ pushed: bool
822
+ relay_text: str
823
+ hash_before: str | None
824
+ hash_after: str | None
825
+ record: dict
826
+
827
+
828
+ def _one_session(ctx: _Ctx, prompt: str, relay_slug: str, n: int, spent: float, elapsed: float,
829
+ spend_slug: str, cwd: Path | None = None) -> _Session:
830
+ """Launch one session (agy: under a spend reservation) in `cwd` (default the repo) and read what it left."""
831
+ caps = ctx.caps
832
+ cwd = cwd or ctx.repo
833
+ remaining_s = max(1.0, min(caps.session_timeout_min * 60, caps.max_hours * 3600 - elapsed))
834
+ usd = min(caps.max_usd_per_session, caps.max_usd_total - spent)
835
+ cmd = build_command(ctx.engine, ctx.model, prompt, usd=usd, timeout_min=remaining_s / 60, governor=ctx.governor,
836
+ engine_cmd=ctx.engine_cmd)
837
+ rid = _spend_reserve(ctx.home, spend_slug, ctx.now(), n > 1) if ctx.engine == "agy" else None
838
+ launched, outcome = ctx.clock(), None
839
+ try:
840
+ _, hash_before = _read_relay(cwd, relay_slug)
841
+ record = {"kind": "session", "n": n, "start": _iso(ctx.now()), "relay_hash_before": hash_before}
842
+ outcome = _launch(cmd, cwd, remaining_s, prompt if ctx.engine in PROMPT_ON_STDIN else None,
843
+ env=headless_env())
844
+ finally: # any exception, KeyboardInterrupt too, still settles the reservation
845
+ if rid is not None:
846
+ _spend_settle(ctx.home, rid, cmd, outcome, ctx.now(), ctx.clock() - launched)
847
+ outcome = _Outcome(outcome.code, caddis_redact.redact(outcome.stdout, ctx.masks),
848
+ caddis_redact.redact(outcome.stderr, ctx.masks), outcome.timed_out, outcome.started)
849
+ record["end"] = _iso(ctx.now())
850
+ # The hour cap is checked again by the next iteration's checks(), before any further session.
851
+ result = parse_result(outcome.stdout)
852
+ cost = _cost(result)
853
+ text = result.get("result", "") if isinstance(result, dict) else ""
854
+ pushed = caddis_notify.PUSH_MARKER in (text if isinstance(text, str) else "") or \
855
+ caddis_notify.PUSH_MARKER in outcome.stdout
856
+ relay_text, hash_after = _read_relay(cwd, relay_slug)
857
+ record.update({"exit": outcome.code, "cost_usd": cost, "relay_hash_after": hash_after, "push_sent": pushed})
858
+ return _Session(outcome, result, cost, pushed, relay_text, hash_before, hash_after, record)
859
+
860
+
861
+ # -- the review gate (caddis-next Phase 11) ---------------------------------------------------------------------------
862
+
863
+ REVIEW_RUNS = 2
864
+ REVIEW_MAX_USD = REVIEW_RUNS * caddis_review.MAX_BUDGET_USD # the most one review may cost; counted when unknown
865
+ REVIEW_TIMEOUT_S = REVIEW_RUNS * caddis_review.CLAUDE_TIMEOUT_SECS + 900 # a backstop: each run has its own timeout
866
+ REVIEW_BLOCKED = "BLOCKED review" # findings left after the one fix round; needs the owner
867
+ REVIEW_NOT_RUN = "BLOCKED review not run" # exit 3, 4, any other code, no heading, no plan; needs the owner
868
+ REVIEW_ROUND_LIMIT = "BLOCKED review round limit: the owner decides" # exit 5: never review again
869
+
870
+
871
+ def default_review_cmd() -> list[str]:
872
+ return [sys.executable, str(_HERE / "caddis_review.py")]
873
+
874
+
875
+ @dataclass
876
+ class _Tally:
877
+ """The run's counters while the review gate runs: its fix session and its cost count against the caps."""
878
+ sessions: int
879
+ spent: float
880
+ pushed: bool
881
+
882
+
883
+ def plan_paths(repo: Path, plan: str) -> tuple[Path, Path]:
884
+ """Where a plan lives: `.caddis/plans/<plan>.md`, or `.caddis/plans/done/<plan>.md` after tidy moved it."""
885
+ plans = Path(repo) / ".caddis" / "plans"
886
+ return plans / f"{plan}.md", plans / "done" / f"{plan}.md"
887
+
888
+
889
+ def _working_plan(repo: Path, plan: str) -> tuple[Path, str] | None:
890
+ """The working copy of the plan (its current path, else done/) and its text; None when neither is readable."""
891
+ for path in plan_paths(repo, plan):
892
+ if path.is_file():
893
+ try:
894
+ return path, path.read_text(encoding="utf-8")
895
+ except OSError:
896
+ continue
897
+ return None
898
+
899
+
900
+ def _origin_plan_text(repo: Path, plan: str) -> str | None:
901
+ """The plan on `origin/<default>` as last fetched (no fetch and no remote call here), from its path or done/;
902
+ None when absent or when origin/HEAD is not set locally."""
903
+ ref = _git_out(repo, "symbolic-ref", "refs/remotes/origin/HEAD") or ""
904
+ if not ref.startswith("refs/remotes/origin/"):
905
+ return None
906
+ branch = ref[len("refs/remotes/origin/"):]
907
+ for path in (f".caddis/plans/{plan}.md", f".caddis/plans/done/{plan}.md"):
908
+ text = caddis_roadmap.show(repo, branch, path)
909
+ if text is not None:
910
+ return text
911
+ return None
912
+
913
+
914
+ def phase_info(texts: Sequence[str], phase: int, scratch: Path) -> tuple[bool, bool]:
915
+ """(some copy has a `### Phase <n> ` heading, some copy marks the phase high). `caddis_review.phase_risk` fails
916
+ closed, so anything but normal, low or medium is high. Every copy counts: a session cannot lower its own risk."""
917
+ found = high = False
918
+ _ensure_local(scratch) # the local folder gets its `*` .gitignore before anything is written there
919
+ probe = scratch / f"risk-probe-{os.getpid()}-{uuid.uuid4().hex}.md"
920
+ try:
921
+ for text in texts:
922
+ if not text:
923
+ continue
924
+ probe.write_text(text, encoding="utf-8")
925
+ try:
926
+ risk = caddis_review.phase_risk(probe, phase)
927
+ except ValueError: # no such phase heading in this copy
928
+ continue
929
+ found = True
930
+ high = high or risk == "high"
931
+ finally:
932
+ probe.unlink(missing_ok=True)
933
+ return found, high
934
+
935
+
936
+ def _done_rows(text: str | None) -> list[int]:
937
+ rows = caddis_roadmap.parse_tracker(text or "")
938
+ return [n for n in sorted(rows) if caddis_roadmap._is_done_status(rows[n])]
939
+
940
+
941
+ def reviewed_file(repo: Path, kind: str, slug: str, plan: str) -> Path:
942
+ """The record of reviewed phases of `plan`, in THIS runner's own folder:
943
+ `<git common dir>/caddis-runner/<kind>-<slug>/reviewed/<plan>.json` (every worktree and every restart of this
944
+ runner sees it; outside a git repo, `<local>/reviewed/<plan>.json`). A runner trusts only records in its own
945
+ folder, so the per-runner state check covers them and a session cannot forge them unseen during this runner's
946
+ sessions. Another runner's record for the same plan does not count: a phase reviewed by a plan-mode runner is
947
+ reviewed again when a roadmap runner picks it (safe, accepted). Limit: the state check hashes only this runner's
948
+ own folder, so a session can edit ANOTHER runner's folder unseen by this runner (sessions are cooperative, same
949
+ OS user; the state check is not a security boundary)."""
950
+ return runner_folder(repo, kind, slug) / "reviewed" / f"{plan}.json"
951
+
952
+
953
+ def read_reviewed(path: Path) -> dict:
954
+ """phase (str) -> {"head", "utc", "result"}. A missing or unreadable record is empty: every phase is pending
955
+ again (fail closed: review once more, never skip)."""
956
+ record = _read_record(path)
957
+ return {k: v for k, v in record.items() if isinstance(v, dict)}
958
+
959
+
960
+ def _record_review(path: Path, phase: int, head: str | None, result: str) -> None:
961
+ """The ONLY writer of `reviewed/<plan>.json`. The runner calls it only between children (after a review child
962
+ exited and its state check passed, never while a session or a review runs); every session and review child takes
963
+ its own state snapshot right before it starts, so the runner's write is already in that baseline."""
964
+ record = read_reviewed(path)
965
+ record[str(phase)] = {"head": head, "utc": _iso(datetime.now(timezone.utc)), "result": result}
966
+ _write_record(path, record)
967
+
968
+
969
+ def pending_reviews(working: str, origin: str | None, reviewed: dict, also: Sequence[int] = ()) -> list[int]:
970
+ """Phases to review: done in the working copy's Tracker, not done on origin/<default> (merged phases were reviewed
971
+ before their PR, or merged by the owner), and not recorded as reviewed clean. The caller then keeps only the
972
+ `risk: high` ones."""
973
+ merged = set(_done_rows(origin)) if origin is not None else set()
974
+ return [n for n in sorted(set(_done_rows(working)) | set(also)) if n not in merged
975
+ and (reviewed.get(str(n)) or {}).get("result") != "clean"]
976
+
977
+
978
+ def _rel(repo: Path, path: Path) -> str:
979
+ try:
980
+ return Path(path).resolve().relative_to(repo).as_posix()
981
+ except ValueError:
982
+ return Path(path).as_posix()
983
+
984
+
985
+ def _review_cost(cost_file: Path) -> tuple[float, str]:
986
+ """The review's `total_cost_usd`, or REVIEW_MAX_USD when the file is missing or unreadable (fail toward
987
+ spending less: the caps then stop the run sooner, never later)."""
988
+ try:
989
+ value = json.loads(cost_file.read_text(encoding="utf-8")).get("total_cost_usd")
990
+ except FileNotFoundError:
991
+ return float(REVIEW_MAX_USD), "missing"
992
+ except (OSError, ValueError, AttributeError):
993
+ return float(REVIEW_MAX_USD), "unreadable"
994
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not (0 <= value < float("inf")):
995
+ return float(REVIEW_MAX_USD), "unreadable"
996
+ return float(value), "file"
997
+
998
+
999
+ def _run_review(ctx: _Ctx, tally: _Tally, plan_file: Path, phase: int, out_dir: Path, elapsed: Callable[[], float],
1000
+ ref: str, round_no: int, cwd: Path | None = None) -> int | None:
1001
+ """One `caddis_review.py --plan --phase --runs 2` child with CADDIS_HEADLESS=1. Its cost is added to `spent`.
1002
+ Returns its exit code; None when it timed out, did not start, or exit 1 left no merged findings file. It reviews
1003
+ the checkout `cwd` (default the repo; roadmap mode: the roadmap's worktree, where the phase session ran)."""
1004
+ cwd = Path(cwd) if cwd is not None else ctx.repo
1005
+ caps = ctx.caps
1006
+ if ctx.stop_file():
1007
+ raise _Stop("STOP file", caddis_exit.CLEAN if tally.sessions else caddis_exit.NOT_RUN, push=False)
1008
+ left_s = caps.max_hours * 3600 - elapsed()
1009
+ if left_s <= 0:
1010
+ raise _Stop(f"cap: hours {caps.max_hours:g} reached before the review of {ref}", caddis_exit.BLOCKED)
1011
+ if tally.spent + REVIEW_MAX_USD > caps.max_usd_total:
1012
+ raise _Stop(f"cap: usd {tally.spent:.2f} of {caps.max_usd_total:g} (the review of {ref} may cost up to "
1013
+ f"{REVIEW_MAX_USD:g})", caddis_exit.BLOCKED)
1014
+ out_dir.mkdir(parents=True, exist_ok=True)
1015
+ cost_file = out_dir / f"review-{phase}-cost.json"
1016
+ merged = out_dir / f"review-{phase}-merged.md"
1017
+ stale = False
1018
+ for old in (cost_file, merged): # a file from an earlier review never counts for this one
1019
+ try:
1020
+ old.unlink(missing_ok=True)
1021
+ except OSError:
1022
+ stale = True
1023
+ argv = [*ctx.review_cmd, "--worktree", str(cwd), "--plan", str(plan_file), "--phase", str(phase),
1024
+ "--runs", str(REVIEW_RUNS), "--out-dir", str(out_dir)]
1025
+ print(f"review: {ref} round {round_no} ({REVIEW_RUNS} runs)")
1026
+ code: int | None = None
1027
+ kind, slug = ctx.target.kind, ctx.target.slug
1028
+ state_before = state_snapshot(ctx.repo, kind, slug) # the reviewer has full Bash: it is guarded like a session
1029
+ try:
1030
+ proc = subprocess.run(argv, cwd=cwd, env=headless_env(), stdin=subprocess.DEVNULL, capture_output=True,
1031
+ text=True, encoding="utf-8", errors="replace",
1032
+ timeout=max(1.0, min(REVIEW_TIMEOUT_S, left_s)))
1033
+ code, output = proc.returncode, (proc.stdout or "") + (proc.stderr or "")
1034
+ except subprocess.TimeoutExpired:
1035
+ output = "the review timed out"
1036
+ except OSError as exc:
1037
+ output = f"the review did not start: {exc}"
1038
+ for line in output.splitlines():
1039
+ if line.strip():
1040
+ print(caddis_redact.redact(f"review {ref}: {line}", ctx.masks))
1041
+ cost, source = (float(REVIEW_MAX_USD), "stale") if stale else _review_cost(cost_file)
1042
+ tally.spent += cost
1043
+ if source != "file":
1044
+ print(f"caddis_roadmap_run: review cost file {cost_file} is {source}; counting USD {REVIEW_MAX_USD:g} "
1045
+ f"(the most {REVIEW_RUNS} runs may cost)", file=sys.stderr)
1046
+ note = None
1047
+ if code == caddis_exit.BLOCKED and (stale or not merged.is_file()):
1048
+ note, code = "exit 1 with no merged findings file", None # nothing for a fix session to read: not run
1049
+ ctx.log({"kind": "review", "batch": ref, "round": round_no, "exit": code, "cost_usd": cost,
1050
+ "cost_source": source, **({"note": note} if note else {}), "time": _iso(ctx.now())})
1051
+ _check_state(ctx.repo, kind, slug, state_before) # after its cost is counted, before the record is written
1052
+ return code
1053
+
1054
+
1055
+ def _review_gate(ctx: _Ctx, tally: _Tally, *, plan: str, extra_texts: Sequence[str] = (),
1056
+ origin_text: str | None = None, elapsed: Callable[[], float], spend_slug: str,
1057
+ also: Sequence[int] = (), cwd: Path | None = None) -> bool:
1058
+ """Review every pending `risk: high` phase of `plan`, in order; True when it reviewed one (all now clean).
1059
+
1060
+ Pending (recomputed after every review and every fix session): a Tracker row done in the working copy, not done on
1061
+ origin/<default>, not recorded clean in `reviewed_file`, and high in any copy (working, origin, `extra_texts`).
1062
+ The plan is read from its path or from done/; neither readable stops `BLOCKED review not run`, and so does a done
1063
+ row with no `### Phase <n>` heading in any copy. Review exit 0 records the phase clean. Exit 1 starts ONE fix
1064
+ session per phase (a session against the caps), then that phase is reviewed again; exit 1 again stops
1065
+ `BLOCKED review`; exit 5 (round limit) stops at once `BLOCKED review round limit: the owner decides`, with no
1066
+ fix session and no later review; exit 3, 4 or any other code stops `BLOCKED review not run`. Neither stop is restartable by the
1067
+ recheck task; an owner restart reviews the phase again (it is still pending). `also`: phases that count as done
1068
+ even when their row is not (roadmap mode: the phase it is about to publish). `cwd`: where the phase session ran
1069
+ (roadmap mode: the roadmap's worktree; default the repo): the plan is read there, the review reviews that
1070
+ checkout, and the fix session runs there."""
1071
+ cwd = Path(cwd) if cwd is not None else ctx.repo
1072
+ record_path = reviewed_file(ctx.repo, ctx.target.kind, ctx.target.slug, plan)
1073
+ out_dir = ctx.target.local / "reviews" / plan
1074
+ fixed: set[int] = set()
1075
+ reviewed_any = False
1076
+ while True:
1077
+ working = _working_plan(cwd, plan)
1078
+ if working is None:
1079
+ raise _Stop(f"{REVIEW_NOT_RUN}: plan {plan} not found", caddis_exit.BLOCKED, record={"plan": plan})
1080
+ plan_file, text = working
1081
+ texts = [text, origin_text or "", *extra_texts]
1082
+ phase = None
1083
+ for n in pending_reviews(text, origin_text, read_reviewed(record_path), also):
1084
+ found, high = phase_info(texts, n, ctx.target.local)
1085
+ if not found:
1086
+ raise _Stop(f"{REVIEW_NOT_RUN}: no heading for phase {n}", caddis_exit.BLOCKED,
1087
+ record={"ref": f"{plan}#{n}"})
1088
+ if high:
1089
+ phase = n
1090
+ break
1091
+ if phase is None:
1092
+ return reviewed_any
1093
+ reviewed_any = True
1094
+ ref = f"{plan}#{phase}"
1095
+ merged = out_dir / f"review-{phase}-merged.md"
1096
+ record = {"ref": ref, "review": str(merged)}
1097
+ code = _run_review(ctx, tally, plan_file, phase, out_dir, elapsed, ref, 2 if phase in fixed else 1, cwd)
1098
+ head = _git_out(cwd, "rev-parse", "HEAD")
1099
+ result = {caddis_exit.CLEAN: "clean", caddis_exit.BLOCKED: "blocked"}.get(code, "not-run")
1100
+ _record_review(record_path, phase, head, result)
1101
+ if code == caddis_exit.CLEAN:
1102
+ continue
1103
+ if code == caddis_exit.ROUND_LIMIT: # the phase used its review rounds: stop at once, never review again
1104
+ raise _Stop(REVIEW_ROUND_LIMIT, caddis_exit.BLOCKED, record=record,
1105
+ detail=f"{ref}: review exit {code} (round limit); see {merged}")
1106
+ if code != caddis_exit.BLOCKED:
1107
+ raise _Stop(REVIEW_NOT_RUN, caddis_exit.BLOCKED, record=record,
1108
+ detail=f"{ref}: review exit {'none' if code is None else code}; see {merged}")
1109
+ if phase in fixed:
1110
+ raise _Stop(REVIEW_BLOCKED, caddis_exit.BLOCKED, record=record,
1111
+ detail=f"{ref}: blocking findings left after one fix round; see {merged}")
1112
+ fixed.add(phase)
1113
+ ctx.checks(tally.sessions, tally.spent, elapsed()) # the fix session is a session like any other
1114
+ prompt = (f"Read {_rel(cwd, merged)} and fix every blocking finding in Phase {phase} of "
1115
+ f"{_rel(cwd, plan_file)}; commit. Work on the branch that is checked out; do not push.\n")
1116
+ state_before = state_snapshot(ctx.repo, ctx.target.kind, ctx.target.slug) # AFTER the record write
1117
+ s = _one_session(ctx, prompt, plan, tally.sessions + 1, tally.spent, elapsed(), spend_slug, cwd=cwd)
1118
+ tally.sessions += 1
1119
+ tally.spent += s.cost
1120
+ tally.pushed = s.pushed
1121
+ failed = _session_failed(ctx.engine, s.outcome, s.result)
1122
+ s.record.update({"batch": ref, "fix_for": "review", "status": "FIX" if failed is None else failed.reason})
1123
+ ctx.log(s.record)
1124
+ _check_state(ctx.repo, ctx.target.kind, ctx.target.slug, state_before) # before any runner write
1125
+ if failed is not None:
1126
+ raise failed
1127
+
1128
+
1129
+ def run(*, repo: Path, plan: str | None = None, roadmap: str | None = None, caps: Caps, engine: str = "claude",
1130
+ model: str | None = None, engine_cmd: Sequence[str] | None = None, governor: Sequence[str] | None = None,
1131
+ prompt_template: Path | None = None, dry_run: bool = False,
1132
+ home: Path | None = None, toast: Callable[[str, str], int] | None = None,
1133
+ push: Callable[[str], int] | None = None, clock: Callable[[], float] = time.monotonic,
1134
+ now: Callable[[], datetime] | None = None, after_session: Callable[[int], None] | None = None,
1135
+ roadmap_cmd: Sequence[str] | None = None, hostname: str | None = None,
1136
+ publish: Callable[..., str] | None = None, pr_merged: Callable[[Path, str], bool] | None = None,
1137
+ worktree_root: Path | None = None, delete_remote_branch: Callable[[Path, str], None] | None = None,
1138
+ pr_closed: Callable[[Path, str], bool] | None = None, gh_account: str | None = None,
1139
+ gh_login: Callable[[str | None], str | None] | None = None,
1140
+ review_cmd: Sequence[str] | None = None) -> int:
1141
+ repo = Path(repo).resolve()
1142
+ if (plan is None) == (roadmap is None):
1143
+ print("caddis_roadmap_run: give exactly one of --plan and --roadmap", file=sys.stderr)
1144
+ return caddis_exit.ERROR
1145
+ if roadmap is not None:
1146
+ if not isinstance(roadmap, str) or not caddis_roadmap.SLUG_RE.match(roadmap):
1147
+ print(f"caddis_roadmap_run: bad roadmap {roadmap!r}: invalid slug", file=sys.stderr)
1148
+ return caddis_exit.ERROR
1149
+ target = Target("roadmap", roadmap, _local_dir(repo, "roadmap", roadmap), (), PLAN_PROMPT_NAME)
1150
+ else:
1151
+ try:
1152
+ target = resolve_target(repo, plan)
1153
+ except ValueError as exc:
1154
+ print(f"caddis_roadmap_run: {exc}", file=sys.stderr)
1155
+ return caddis_exit.ERROR
1156
+ if engine not in ENGINES:
1157
+ print(f"caddis_roadmap_run: unknown engine {engine!r}", file=sys.stderr)
1158
+ return caddis_exit.ERROR
1159
+ if not model and engine == "agy": # the model this machine offers (Argon, else the fallback); the table is display only
1160
+ model = _agy_engine_model(repo)
1161
+ model = model or ENGINES[engine]["default_model"]
1162
+ template = prompt_template or find_prompt_template(target.prompt_name)
1163
+ if template is None or not Path(template).is_file():
1164
+ print(f"caddis_roadmap_run: resume prompt template {target.prompt_name} not found", file=sys.stderr)
1165
+ return caddis_exit.ERROR
1166
+ template = Path(template)
1167
+ if governor is None:
1168
+ governor = default_governor()
1169
+ if governor is None:
1170
+ print("caddis_roadmap_run: caddis_governor.py not found; refusing to start ungoverned", file=sys.stderr)
1171
+ return caddis_exit.NOT_RUN
1172
+ home = home or Path.home()
1173
+ now = now or (lambda: datetime.now(timezone.utc))
1174
+ toast = toast or (lambda title, message: caddis_notify.toast(title, message))
1175
+ push = push or (lambda message: caddis_notify.push(message))
1176
+ ctx = _Ctx(repo, target, caps, engine, model, engine_cmd, governor, home, now, clock, toast, push,
1177
+ caddis_redact.load_secrets(home), gh_account=gh_account or None,
1178
+ review_cmd=list(review_cmd or default_review_cmd()))
1179
+ if gh_account and not dry_run: # every gh call of this run (and its caddis_roadmap.py children) uses it
1180
+ try:
1181
+ value = (gh_login or use_gh_account)(gh_account)
1182
+ except caddis_roadmap.RoadmapError as exc:
1183
+ print(f"caddis_roadmap_run: {exc}", file=sys.stderr)
1184
+ ctx.toast(f"caddis {target.kind} {target.slug}", caddis_redact.redact(f"RUNNER STOP: BLOCKED {exc}",
1185
+ ctx.masks))
1186
+ return caddis_exit.BLOCKED
1187
+ if value:
1188
+ ctx.masks.append(value)
1189
+ if roadmap is not None:
1190
+ return _run_roadmap(ctx, template, dry_run, after_session, list(roadmap_cmd or default_roadmap_cmd()),
1191
+ hostname or socket.gethostname(), publish or default_publish,
1192
+ pr_merged or default_pr_merged, worktree_root,
1193
+ delete_remote_branch or default_delete_remote_branch, pr_closed or default_pr_closed)
1194
+ return _run_plan(ctx, template, dry_run, after_session)
1195
+
1196
+
1197
+ def _dry_run_tail(ctx: _Ctx, template: Path, prompt: str, cwd: Path | None = None) -> None:
1198
+ caps = ctx.caps
1199
+ cmd = build_command(ctx.engine, ctx.model, "<resume prompt>", usd=min(caps.max_usd_per_session, caps.max_usd_total),
1200
+ timeout_min=caps.session_timeout_min, governor=ctx.governor, engine_cmd=ctx.engine_cmd)
1201
+ print(f"would launch (cwd {cwd or ctx.repo}, CADDIS_HEADLESS=1): {subprocess.list2cmdline(cmd)}"
1202
+ + (" (resume prompt on stdin)" if ctx.engine in PROMPT_ON_STDIN else ""))
1203
+ print(f"resume prompt from {template}:\n{prompt}")
1204
+ if ctx.engine != "claude":
1205
+ print(f"note: engine {ctx.engine} is an adapter, not live-tested")
1206
+
1207
+
1208
+ def _dry_lock_check(lock: Path) -> None:
1209
+ stale = caddis_roadmap.runner_lock_stale(lock)
1210
+ if stale is False:
1211
+ raise _Stop("BLOCKED another runner holds the lock", caddis_exit.NOT_RUN, push=False)
1212
+ if stale:
1213
+ print(f"would remove a stale lock {lock}")
1214
+
1215
+
1216
+ def _run_plan(ctx: _Ctx, template: Path, dry_run: bool, after_session: Callable[[int], None] | None) -> int:
1217
+ repo, target, slug, masks = ctx.repo, ctx.target, ctx.target.slug, ctx.masks
1218
+ prompt = render_prompt(template, slug)
1219
+ lock_file = runner_lock_file(repo, "plan", slug)
1220
+ lock = _RunnerLock(lock_file, ctx.caps.max_hours, batch=f"{slug}#*", # the whole plan is held
1221
+ claim_path=caddis_roadmap.claim_lock_path(repo))
1222
+
1223
+ def held_stop(code: int) -> _Stop | None:
1224
+ holder = _roadmap_holder_of(repo, slug, own_lock=lock_file)
1225
+ if holder is None:
1226
+ return None
1227
+ what = "its claim: its PR waits for a merge" if holder[0].get("kind") == "claim" else "its runner lock"
1228
+ return _Stop(f"BLOCKED {holder[0].get('holder')} holds {holder[1]} ({what})", code, push=False)
1229
+
1230
+ if dry_run:
1231
+ try:
1232
+ held = held_stop(caddis_exit.NOT_RUN)
1233
+ if held is not None:
1234
+ raise held
1235
+ _dry_lock_check(lock.path)
1236
+ ctx.checks(0, 0.0, 0.0)
1237
+ if ctx.engine == "agy":
1238
+ _spend_preview(ctx.home, ctx.now())
1239
+ except _Stop as stop:
1240
+ print(f"would stop: {caddis_redact.redact(stop.reason, masks)}")
1241
+ return caddis_exit.NOT_RUN
1242
+ progress = ", ".join(str(p) for p in target.progress)
1243
+ print(f"{target.kind} {slug}: progress from {progress}; local files in {target.local}; lock {lock.path}")
1244
+ _dry_run_tail(ctx, template, prompt)
1245
+ return caddis_exit.CLEAN
1246
+
1247
+ try:
1248
+ with _claim(repo): # the holder check and the lock, atomic against a roadmap runner's choice
1249
+ held = held_stop(caddis_exit.NOT_RUN)
1250
+ if held is not None:
1251
+ raise held
1252
+ lock.acquire(lambda message: _stale_notice(ctx, message),
1253
+ claim_held=caddis_roadmap.claim_lock_path(repo) is not None) # _claim holds it already
1254
+ except _Stop as stop:
1255
+ return _finish(stop, ctx, False, 0, 0.0, slug)
1256
+ except OSError as exc:
1257
+ return _finish(_Stop(f"BLOCKED claim not written: the runner lock ({type(exc).__name__})",
1258
+ caddis_exit.BLOCKED), ctx, False, 0, 0.0, slug)
1259
+ _mark_running(ctx, last_stop_file(repo, "plan", slug))
1260
+ if not ENGINES[ctx.engine]["live_tested"]:
1261
+ print(f"caddis_roadmap_run: engine {ctx.engine} is an adapter, not live-tested", file=sys.stderr)
1262
+
1263
+ sessions, spent, stuck, pushed = 0, 0.0, 0, False
1264
+ plan_before = ""
1265
+ start = ctx.clock()
1266
+ try:
1267
+ while True:
1268
+ elapsed = ctx.clock() - start
1269
+ ctx.checks(sessions, spent, elapsed)
1270
+ held = held_stop(caddis_exit.BLOCKED) # checked on every iteration, not only at start
1271
+ if held is not None:
1272
+ raise held
1273
+ # Review gate: every pending `risk: high` phase is reviewed before another session builds on it (also
1274
+ # after a restart: the record of reviewed phases is in the runner state, not in this process).
1275
+ tally = _Tally(sessions, spent, pushed)
1276
+ try:
1277
+ reviewed = _review_gate(ctx, tally, plan=slug, extra_texts=(plan_before,),
1278
+ origin_text=_origin_plan_text(repo, slug),
1279
+ elapsed=lambda: ctx.clock() - start, spend_slug=slug)
1280
+ finally:
1281
+ sessions, spent, pushed = tally.sessions, tally.spent, tally.pushed
1282
+ if reviewed:
1283
+ ctx.checks(sessions, spent, ctx.clock() - start) # the review and its fix session count
1284
+ closed_before = _closed(target)
1285
+ working = _working_plan(repo, slug)
1286
+ plan_before = working[1] if working else ""
1287
+ state_before = state_snapshot(repo, "plan", slug)
1288
+ s = _one_session(ctx, prompt, slug, sessions + 1, spent, elapsed, slug)
1289
+ _check_state(repo, "plan", slug, state_before)
1290
+ sessions += 1
1291
+ spent += s.cost
1292
+ pushed = s.pushed
1293
+ try:
1294
+ stop = _decide(ctx.engine, s.outcome, s.result, s.relay_text, s.hash_before, s.hash_after)
1295
+ if stop is None:
1296
+ closed = _closed(target)
1297
+ stuck = 0 if closed > closed_before else stuck + 1
1298
+ s.record["status"] = "CONTINUE"
1299
+ if stuck >= STUCK_SESSIONS:
1300
+ raise _Stop("BLOCKED stuck", caddis_exit.BLOCKED)
1301
+ else:
1302
+ s.record["status"] = stop.reason
1303
+ raise stop
1304
+ finally:
1305
+ s.record.setdefault("status", "FAILED")
1306
+ ctx.log(s.record)
1307
+ if after_session is not None:
1308
+ after_session(sessions)
1309
+ except _Stop as stop:
1310
+ return _finish(stop, ctx, pushed, sessions, spent, slug)
1311
+ except Exception as exc:
1312
+ _crashed(ctx, exc, pushed, sessions, spent, slug)
1313
+ raise
1314
+ finally:
1315
+ lock.release()
1316
+
1317
+
1318
+ def _stale_notice(ctx: _Ctx, message: str) -> None:
1319
+ message = caddis_redact.redact(message, ctx.masks)
1320
+ print(f"caddis_roadmap_run: {message}", file=sys.stderr)
1321
+ ctx.log({"kind": "stale-lock", "message": message, "time": _iso(datetime.now(timezone.utc))})
1322
+ ctx.toast(f"caddis {ctx.target.kind} {ctx.target.slug}", message)
1323
+
1324
+
1325
+ def _roadmap_write(ctx: _Ctx, roadmap_cmd: Sequence[str], args: list[str]) -> int:
1326
+ """Run one `caddis_roadmap.py` write as a child process with CADDIS_HEADLESS=1; return its exit code."""
1327
+ if "--by-owner" in args:
1328
+ raise ValueError("the runner never makes an owner-only roadmap change")
1329
+ argv = [*roadmap_cmd, *args, "--roadmap", ctx.target.slug, "--root", str(ctx.repo)]
1330
+ try:
1331
+ proc = subprocess.run(argv, cwd=ctx.repo, env=headless_env(), stdin=subprocess.DEVNULL, capture_output=True,
1332
+ text=True, encoding="utf-8", errors="replace", timeout=ROADMAP_WRITE_TIMEOUT_S)
1333
+ except subprocess.TimeoutExpired:
1334
+ print(f"caddis_roadmap_run: roadmap {args[0]} timed out", file=sys.stderr)
1335
+ return caddis_exit.NOT_RUN
1336
+ except OSError as exc:
1337
+ print(f"caddis_roadmap_run: roadmap {args[0]} not started: {exc}", file=sys.stderr)
1338
+ return caddis_exit.NOT_RUN
1339
+ for line in ((proc.stdout or "") + (proc.stderr or "")).splitlines():
1340
+ if line.strip():
1341
+ print(caddis_redact.redact(f"roadmap {args[0]}: {line}", ctx.masks))
1342
+ ctx.log({"kind": "roadmap", "args": args, "exit": proc.returncode, "time": _iso(datetime.now(timezone.utc))})
1343
+ return proc.returncode
1344
+
1345
+
1346
+ class _Baseline:
1347
+ """The snapshot guard's baseline, kept across iterations. Every origin read is compared with the previous read
1348
+ first: the Milestones and Blockers tables and the Runner host line are frozen, except for the exact edits the
1349
+ runner itself made and recorded (`expect`) since that read. Then the new read becomes the baseline."""
1350
+
1351
+ def __init__(self, roadmap: "caddis_roadmap.Roadmap"):
1352
+ self.roadmap = roadmap
1353
+ self.edits: list[tuple] = []
1354
+
1355
+ def expect(self, *edit) -> None:
1356
+ self.edits.append(edit)
1357
+
1358
+ def observe(self, snap: "caddis_roadmap.Snapshot") -> "caddis_roadmap.Snapshot":
1359
+ changes = roadmap_guard(self.roadmap, snap.roadmap, self.edits)
1360
+ self.edits = []
1361
+ if changes:
1362
+ raise _Stop("FAILED roadmap changed: " + "; ".join(changes), caddis_exit.BLOCKED)
1363
+ self.roadmap = snap.roadmap
1364
+ return snap
1365
+
1366
+
1367
+ def _load_observed(repo: Path, slug: str, baseline: _Baseline) -> "caddis_roadmap.Snapshot":
1368
+ try:
1369
+ snap = caddis_roadmap.load(repo, slug)
1370
+ except caddis_roadmap.RoadmapError as exc:
1371
+ raise _Stop(f"FAILED roadmap changed: the origin copy no longer loads: {str(exc).splitlines()[0]}",
1372
+ caddis_exit.BLOCKED) from None
1373
+ return baseline.observe(snap)
1374
+
1375
+
1376
+ def _audit_lines(repo: Path, slug: str) -> list[str]:
1377
+ try:
1378
+ return (caddis_roadmap.local_dir(repo, slug) / "changes.jsonl").read_text(encoding="utf-8").splitlines()
1379
+ except OSError:
1380
+ return []
1381
+
1382
+
1383
+ def _git_out(repo: Path, *args: str) -> str | None:
1384
+ proc = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8",
1385
+ errors="replace", stdin=subprocess.DEVNULL)
1386
+ return proc.stdout.strip() if proc.returncode == 0 else None
1387
+
1388
+
1389
+ def _phase_title(plan_text: str, phase: int) -> str:
1390
+ for line in plan_text.splitlines():
1391
+ head = _PHASE_HEAD_RE.match(line)
1392
+ if head and int(head.group(1)) == phase:
1393
+ title = re.sub(r"^ {0,3}#{2,4}\s+Phase\s+\d+\s*[-:–—]*\s*", "", line).strip()
1394
+ return " ".join(title.split())[:100]
1395
+ return ""
1396
+
1397
+
1398
+ def default_publish(repo: Path, branch: str, base: str, title: str, body: str) -> str:
1399
+ """Push `branch` to origin and open a PR with `gh api` (REST; the account comes from GH_TOKEN). Returns its URL.
1400
+ When a PR for the branch is already open, returns that one. RoadmapError when it cannot."""
1401
+ caddis_roadmap._git(repo, "push", "--quiet", "-u", "origin", f"{branch}:{branch}")
1402
+ gh_repo = caddis_roadmap.gh_repo_of(repo)
1403
+ if gh_repo is None:
1404
+ raise caddis_roadmap.RoadmapError("origin is not a GitHub remote; cannot open the PR", caddis_exit.NOT_RUN)
1405
+ try:
1406
+ number = caddis_roadmap.open_pr(repo, gh_repo, branch, base, title, body)
1407
+ except caddis_roadmap.RoadmapError:
1408
+ owner = gh_repo.split("/", 1)[0]
1409
+ proc = caddis_roadmap.run_bounded(["gh", "api", f"repos/{gh_repo}/pulls?head={owner}:{branch}&state=open"],
1410
+ caddis_roadmap.GH_TIMEOUT, "gh api (find the PR)")
1411
+ found = json.loads(proc.stdout or "[]") if proc.returncode == 0 else []
1412
+ if not found:
1413
+ raise
1414
+ number = int(found[0]["number"])
1415
+ return f"https://github.com/{gh_repo}/pull/{number}"
1416
+
1417
+
1418
+ def session_branch(slug: str, plan: str, phase: int) -> str:
1419
+ """The branch the runner makes for one roadmap-mode session: `caddis/roadmap/<slug>/<plan>-p<n>`."""
1420
+ return f"caddis/roadmap/{slug}/{plan}-p{phase}"
1421
+
1422
+
1423
+ def runner_folder(repo: Path, kind: str, slug: str) -> Path:
1424
+ """This runner's own state folder: `<git common dir>/caddis-runner/<kind>-<slug>/` (outside a git repo, the
1425
+ local folder). Shared at the top level: `claims/` and `claim.lock`."""
1426
+ lock = caddis_roadmap.runner_lock_path(repo, kind, slug)
1427
+ return lock.parent if lock is not None else _local_dir(repo, kind, slug)
1428
+
1429
+
1430
+ def _state_file(repo: Path, slug: str, name: str) -> Path:
1431
+ """Roadmap runner state in its own folder: `<git common dir>/caddis-runner/roadmap-<slug>/<name>.json`."""
1432
+ return runner_folder(repo, "roadmap", slug) / f"{name}.json"
1433
+
1434
+
1435
+ def claim_file(repo: Path, ref: str) -> Path | None:
1436
+ base = caddis_roadmap.runner_state_dir(repo)
1437
+ return None if base is None else base / "claims" / f"{ref}.json"
1438
+
1439
+
1440
+ def _roadmap_sha(text: str) -> str:
1441
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
1442
+
1443
+
1444
+ def _branch_on_origin(repo: Path, branch: str) -> bool:
1445
+ proc = caddis_roadmap._git(repo, "ls-remote", "--heads", "origin", f"refs/heads/{branch}", check=False)
1446
+ return proc.returncode == 0 and bool(proc.stdout.strip())
1447
+
1448
+
1449
+ WORKTREE_PREFIX = "caddis-roadmap-"
1450
+
1451
+
1452
+ def _registered_worktrees(repo: Path) -> set[Path]:
1453
+ out = _git_out(repo, "worktree", "list", "--porcelain") or ""
1454
+ return {Path(line[len("worktree "):]).resolve() for line in out.splitlines() if line.startswith("worktree ")}
1455
+
1456
+
1457
+ def worktree_path(repo: Path, slug: str, root: Path | None = None) -> Path:
1458
+ """The roadmap's session worktree: the recorded path, else `<root or %TEMP%>/caddis-roadmap-<slug>-<repo tag>`."""
1459
+ recorded = _read_record(_state_file(repo, slug, "worktree")).get("path")
1460
+ if isinstance(recorded, str) and recorded:
1461
+ return Path(recorded)
1462
+ return Path(root or tempfile.gettempdir()) / f"{WORKTREE_PREFIX}{slug}-{_repo_tag(repo)}"
1463
+
1464
+
1465
+ def _repo_tag(repo: Path) -> str:
1466
+ """8 hex of sha256 of the resolved git common dir: two repos with the same roadmap slug get two worktrees."""
1467
+ base = caddis_roadmap.runner_state_dir(repo)
1468
+ common = base.parent if base is not None else Path(repo)
1469
+ return hashlib.sha256(os.path.normcase(str(common.resolve())).encode("utf-8")).hexdigest()[:8]
1470
+
1471
+
1472
+ def ensure_worktree(repo: Path, slug: str, default: str, root: Path | None = None) -> Path:
1473
+ """Reuse the roadmap's worktree, or make it (`git worktree add --detach` at origin/<default>) and record it."""
1474
+ path = worktree_path(repo, slug, root)
1475
+ if path.resolve() in _registered_worktrees(repo):
1476
+ if path.is_dir():
1477
+ return path
1478
+ _git_out(repo, "worktree", "prune") # listed, but its folder is gone (temp cleanup): prune, make it again
1479
+ if path.exists():
1480
+ raise _Stop(f"FAILED worktree path {path} exists and is not a worktree of this repo", caddis_exit.BLOCKED)
1481
+ path.parent.mkdir(parents=True, exist_ok=True)
1482
+ if _git_out(repo, "worktree", "add", "--quiet", "--detach", str(path), f"origin/{default}") is None:
1483
+ raise _Stop(f"FAILED cannot make the roadmap worktree {path}", caddis_exit.BLOCKED)
1484
+ _write_record(_state_file(repo, slug, "worktree"), {"path": str(path), "utc": _iso(datetime.now(timezone.utc))})
1485
+ return path
1486
+
1487
+
1488
+ def _worktree_work(path: Path) -> list[str] | None:
1489
+ """Uncommitted work in a worktree (`git status --porcelain --untracked-files=all`, untracked `.caddis/relay/`
1490
+ files left out); None when the status cannot be read."""
1491
+ status = _git_out(path, "status", "--porcelain", "--untracked-files=all")
1492
+ if status is None:
1493
+ return None
1494
+ return [line for line in status.splitlines()
1495
+ if line.strip() and not (line.startswith("??") and line[3:].replace("\\", "/").startswith(".caddis/relay/"))]
1496
+
1497
+
1498
+ def remove_worktree_if_clean(repo: Path, slug: str, root: Path | None = None) -> str:
1499
+ """At the end of a roadmap: remove its worktree when it holds no work (only session relays may be left);
1500
+ keep it otherwise. Returns a line for the stop."""
1501
+ path = worktree_path(repo, slug, root)
1502
+ if path.resolve() not in _registered_worktrees(repo):
1503
+ _state_file(repo, slug, "worktree").unlink(missing_ok=True)
1504
+ return "no worktree"
1505
+ work = _worktree_work(path)
1506
+ if work is None:
1507
+ return f"worktree kept (status not readable): {path}"
1508
+ if work:
1509
+ return f"worktree kept (not clean, {len(work)} change(s)): {path}"
1510
+ if _git_out(repo, "worktree", "remove", "--force", str(path)) is None:
1511
+ return f"worktree kept (remove failed): {path}"
1512
+ _git_out(repo, "worktree", "prune")
1513
+ _state_file(repo, slug, "worktree").unlink(missing_ok=True)
1514
+ return f"worktree removed: {path}"
1515
+
1516
+
1517
+ def default_delete_remote_branch(repo: Path, branch: str) -> None:
1518
+ """Delete the branch on GitHub (`gh api -X DELETE .../git/refs/heads/<branch>`; the account comes from GH_TOKEN).
1519
+ RoadmapError when it cannot."""
1520
+ gh_repo = caddis_roadmap.gh_repo_of(repo)
1521
+ if gh_repo is None:
1522
+ raise caddis_roadmap.RoadmapError("origin is not a GitHub remote; cannot delete the branch", caddis_exit.NOT_RUN)
1523
+ try:
1524
+ proc = caddis_roadmap.run_bounded(["gh", "api", "-X", "DELETE", f"repos/{gh_repo}/git/refs/heads/{branch}"],
1525
+ caddis_roadmap.GH_TIMEOUT, "gh api (delete the branch)")
1526
+ except OSError as exc:
1527
+ raise GhError(f"gh cannot delete {branch}: {type(exc).__name__}", caddis_exit.BLOCKED) from None
1528
+ if proc.returncode != 0:
1529
+ raise caddis_roadmap.RoadmapError(f"could not delete {branch} on origin: {(proc.stderr or '').strip()[:200]}",
1530
+ caddis_exit.BLOCKED)
1531
+
1532
+
1533
+ def _checked_out_in(repo: Path, branch: str) -> list[Path]:
1534
+ """The worktrees (`git worktree list --porcelain`) that have `branch` checked out."""
1535
+ out, path = [], None
1536
+ for line in (_git_out(repo, "worktree", "list", "--porcelain") or "").splitlines():
1537
+ if line.startswith("worktree "):
1538
+ path = Path(line[len("worktree "):])
1539
+ elif line == f"branch refs/heads/{branch}" and path is not None:
1540
+ out.append(path)
1541
+ return out
1542
+
1543
+
1544
+ def _delete_local_branch(repo: Path, branch: str, wt: Path | None) -> str | None:
1545
+ """Delete a local branch. Only the roadmap worktree `wt` may be detached for it; HEAD never moves in any other
1546
+ checkout: a branch checked out elsewhere is kept, and that checkout is returned as the reason."""
1547
+ for path in _checked_out_in(repo, branch):
1548
+ if wt is not None and path.resolve() == Path(wt).resolve():
1549
+ _git_out(path, "checkout", "-q", "--detach")
1550
+ else:
1551
+ return f"kept locally: checked out in {path}"
1552
+ if _git_out(repo, "branch", "-q", "-D", branch) is None and \
1553
+ _git_out(repo, "rev-parse", "--verify", "--quiet", f"refs/heads/{branch}") is not None:
1554
+ return "kept locally: git branch -D failed"
1555
+ return None
1556
+
1557
+
1558
+ def _cleanup_merged_branch(ctx: _Ctx, repo: Path, branch: str, wt: Path | None,
1559
+ delete_remote_branch: Callable[[Path, str], None], made_branches: set,
1560
+ branches_file: Path) -> None:
1561
+ """Whoever creates a branch removes it when its PR merges: the local branch, the branch on origin, the record."""
1562
+ if not branch:
1563
+ return
1564
+ kept = _delete_local_branch(repo, branch, wt)
1565
+ if kept:
1566
+ print(f"caddis_roadmap_run: {branch} {kept}", file=sys.stderr)
1567
+ try:
1568
+ delete_remote_branch(repo, branch)
1569
+ except caddis_roadmap.RoadmapError as exc:
1570
+ print(f"caddis_roadmap_run: {str(exc).splitlines()[0]}", file=sys.stderr)
1571
+ made_branches.discard(branch)
1572
+ prs = dict(_read_record(branches_file).get("prs") or {})
1573
+ prs.pop(branch, None)
1574
+ _write_record(branches_file, {"branches": sorted(made_branches), "prs": prs})
1575
+ ctx.log({"kind": "branch-removed", "branch": branch, "local": kept or "deleted",
1576
+ "time": _iso(datetime.now(timezone.utc))})
1577
+
1578
+
1579
+ def _cleanup_finished_branches(repo: Path, slug: str, default: str,
1580
+ pr_merged: Callable[[Path, str], bool] = lambda repo, url: False,
1581
+ delete_remote_branch: Callable[[Path, str], None] = lambda repo, branch: None) -> str:
1582
+ """At the end of a roadmap: delete the local `caddis/roadmap/<slug>/*` branches that are merged: in
1583
+ origin/<default>'s history, or with a PR that GitHub says merged (a squash merge). A branch with a merged PR is
1584
+ deleted on origin too. Keep the others and name them."""
1585
+ names = (_git_out(repo, "for-each-ref", "--format=%(refname:short)", f"refs/heads/caddis/roadmap/{slug}/") or "")
1586
+ branches_file = _state_file(repo, slug, "branches")
1587
+ record = _read_record(branches_file)
1588
+ prs = dict(record.get("prs") or {})
1589
+ kept: list[str] = []
1590
+ for name in (n.strip() for n in names.splitlines() if n.strip()):
1591
+ url = prs.get(name)
1592
+ try:
1593
+ by_pr = bool(url) and pr_merged(repo, url)
1594
+ except Exception: # gh cannot answer: keep the branch and name it
1595
+ by_pr = False
1596
+ merged = by_pr or subprocess.run(["git", "-C", str(repo), "merge-base", "--is-ancestor", name,
1597
+ f"origin/{default}"], capture_output=True,
1598
+ stdin=subprocess.DEVNULL).returncode == 0
1599
+ if not (merged and _git_out(repo, "branch", "-q", "-D", name) is not None):
1600
+ kept.append(name)
1601
+ continue
1602
+ if by_pr:
1603
+ try:
1604
+ delete_remote_branch(repo, name)
1605
+ except caddis_roadmap.RoadmapError as exc:
1606
+ print(f"caddis_roadmap_run: {str(exc).splitlines()[0]}", file=sys.stderr)
1607
+ prs.pop(name, None)
1608
+ branches = sorted(set(record.get("branches") or []) & set(kept))
1609
+ if record:
1610
+ _write_record(branches_file, {"branches": branches, "prs": prs})
1611
+ return f"unmerged branches kept: {', '.join(kept)}" if kept else "no roadmap branches left"
1612
+
1613
+
1614
+ def rollup_ready(snap: "caddis_roadmap.Snapshot", failed: dict | None = None) -> list[str]:
1615
+ """Milestones that wait only for their rollup: Status not done/not-needed, every phase merged (Tracker done on
1616
+ origin), and the same runner rule as `next` (caddis_roadmap.runner_ineligible: dependencies, blockers, Owner).
1617
+ `failed` (milestone -> roadmap hash) leaves out a milestone whose rollup already failed on this same roadmap."""
1618
+ rm = snap.roadmap
1619
+ failed = failed or {}
1620
+ sha = _roadmap_sha(snap.text)
1621
+ out = []
1622
+ for ms in rm.milestones:
1623
+ if ms.status in caddis_roadmap.FINISHED or not ms.phases or caddis_roadmap.runner_ineligible(rm, ms):
1624
+ continue
1625
+ if failed.get(ms.id) == sha: # tried on this very roadmap and nothing was written: the owner decides
1626
+ continue
1627
+ if all(snap.phases.state(r) == "done" for r in ms.phases):
1628
+ out.append(ms.id)
1629
+ return out
1630
+
1631
+
1632
+ def _rollup_ready_without_fetch(repo: Path, slug: str) -> list[str]:
1633
+ """rollup_ready on the origin copy as the local refs have it (no fetch: --last-stop only reads)."""
1634
+ try:
1635
+ branch = caddis_roadmap.default_branch(repo)
1636
+ text = caddis_roadmap.show(repo, branch, caddis_roadmap.roadmap_path(slug))
1637
+ if text is None:
1638
+ return []
1639
+ rm = caddis_roadmap.parse_roadmap(text)
1640
+ if rm.errors:
1641
+ return []
1642
+ snap = caddis_roadmap.Snapshot(repo, slug, branch, text, rm, caddis_roadmap.PhaseIndex(repo, branch), [])
1643
+ return rollup_ready(snap, _read_record(_state_file(repo, slug, "rollup-failed")))
1644
+ except Exception: # a read that fails: nothing to report
1645
+ return []
1646
+
1647
+
1648
+ def _check_orphaned_claims(repo: Path, slug: str, waiting: dict, pending: dict) -> None:
1649
+ """An own claim (`roadmap` = this slug) that neither the waiting record nor pending-rollup.json names is from a run
1650
+ killed between writing the claim and recording the wait. It is never deleted here: the owner checks the branch and
1651
+ its PR first."""
1652
+ known: set[str] = set()
1653
+ if waiting.get("ref"):
1654
+ named = claim_file(repo, str(waiting["ref"]))
1655
+ if named is not None:
1656
+ known.add(os.path.normcase(str(named)))
1657
+ if isinstance(pending.get("claim"), str) and pending["claim"]:
1658
+ known.add(os.path.normcase(pending["claim"]))
1659
+ for _name, path in sorted(_claims_of(repo, "roadmap", slug).items()):
1660
+ if os.path.normcase(str(path)) in known:
1661
+ continue
1662
+ batch = str(_read_record(path).get("batch") or path.stem)
1663
+ match = caddis_roadmap.REF_RE.match(batch)
1664
+ branch = session_branch(slug, match.group(1), int(match.group(2))) if match else "its branch"
1665
+ raise _Stop(f"BLOCKED orphaned claim {path} from a killed run: check {branch} / its PR, then delete the claim",
1666
+ caddis_exit.BLOCKED)
1667
+
1668
+
1669
+ def _held_batches(repo: Path, slug: str, snap: "caddis_roadmap.Snapshot") -> list[dict]:
1670
+ """The batches `next` skipped only because another runner's lock or claim held them (ref, kind, holder), from
1671
+ caddis_roadmap's structure, never from the reason text."""
1672
+ held: list[dict] = []
1673
+ caddis_roadmap._select(snap, caddis_roadmap.other_roadmap_holders(repo, slug), held_out=held)
1674
+ return held
1675
+
1676
+
1677
+ def _clean_merged(ctx: _Ctx, repo: Path, slug: str, merged: dict, worktree_root: Path | None,
1678
+ delete_remote_branch: Callable[[Path, str], None], made_branches: set, branches_file: Path) -> None:
1679
+ """A merged phase: its claim and its branch go (local, origin, the record)."""
1680
+ claim = merged.get("claim")
1681
+ if isinstance(claim, str) and claim:
1682
+ Path(claim).unlink(missing_ok=True)
1683
+ wt_now = worktree_path(repo, slug, worktree_root)
1684
+ _cleanup_merged_branch(ctx, repo, str(merged.get("branch") or ""), wt_now if wt_now.is_dir() else None,
1685
+ delete_remote_branch, made_branches, branches_file)
1686
+
1687
+
1688
+ def _run_roadmap(ctx: _Ctx, template: Path, dry_run: bool, after_session: Callable[[int], None] | None,
1689
+ roadmap_cmd: list[str], hostname: str, publish: Callable[..., str],
1690
+ pr_merged: Callable[[Path, str], bool] = lambda repo, url: False,
1691
+ worktree_root: Path | None = None,
1692
+ delete_remote_branch: Callable[[Path, str], None] = lambda repo, branch: None,
1693
+ pr_closed: Callable[[Path, str], bool] = lambda repo, url: False) -> int:
1694
+ repo, slug, masks, caps = ctx.repo, ctx.target.slug, ctx.masks, ctx.caps
1695
+ lock = _RunnerLock(runner_lock_file(repo, "roadmap", slug), caps.max_hours,
1696
+ claim_path=caddis_roadmap.claim_lock_path(repo))
1697
+ baseline: _Baseline | None = None
1698
+
1699
+ def write(args: list[str], edit: tuple) -> None:
1700
+ rc = _roadmap_write(ctx, roadmap_cmd, args)
1701
+ if rc != caddis_exit.CLEAN:
1702
+ raise _Stop(f"FAILED roadmap {' '.join(args[:3])} exit {rc}", caddis_exit.BLOCKED)
1703
+ baseline.expect(*edit)
1704
+
1705
+ def rollup(mid: str) -> int:
1706
+ """Roll the milestone up; the guard excuses only what the audit log says this rollup WROTE (merged). Returns
1707
+ how many changes it wrote."""
1708
+ written = 0
1709
+ seen = len(_audit_lines(repo, slug))
1710
+ rc = _roadmap_write(ctx, roadmap_cmd, ["rollup", "--id", mid])
1711
+ if rc not in (caddis_exit.CLEAN, caddis_exit.BLOCKED):
1712
+ raise _Stop(f"FAILED roadmap rollup {mid} exit {rc}", caddis_exit.BLOCKED)
1713
+ for line in _audit_lines(repo, slug)[seen:]:
1714
+ try:
1715
+ entry = json.loads(line)
1716
+ except ValueError:
1717
+ continue
1718
+ if not isinstance(entry, dict) or entry.get("id") != mid or entry.get("merge_exit") != caddis_exit.CLEAN:
1719
+ continue
1720
+ change = str(entry.get("change", ""))
1721
+ if change.startswith("Status "):
1722
+ baseline.expect("status", mid, change[len("Status "):])
1723
+ written += 1
1724
+ elif change == "block external":
1725
+ baseline.expect("blocker-like", mid, "external", ROLLUP_BLOCK_RE)
1726
+ written += 1
1727
+ return written
1728
+
1729
+ if dry_run: # no host check, no lock, no write: read origin and print what one iteration would do
1730
+ try:
1731
+ _dry_lock_check(lock.path)
1732
+ ctx.checks(0, 0.0, 0.0)
1733
+ if ctx.engine == "agy":
1734
+ _spend_preview(ctx.home, ctx.now())
1735
+ try:
1736
+ batch, snap = caddis_roadmap.next_with_snapshot(repo, slug)
1737
+ except caddis_roadmap.RoadmapError as exc:
1738
+ raise _Stop(f"FAILED roadmap not read: {exc}", exc.code, push=False) from None
1739
+ waiting = _read_record(_state_file(repo, slug, "waiting"))
1740
+ if waiting: # a real start begins with the recorded wait
1741
+ ref, url = str(waiting.get("ref", "")), str(waiting.get("pr", ""))
1742
+ try:
1743
+ merged = pr_merged(repo, url)
1744
+ except Exception as exc:
1745
+ raise _Stop(f"{GH_STOP_PREFIX} {url} with account {_gh_who()}: {type(exc).__name__}",
1746
+ caddis_exit.BLOCKED, push=False) from None
1747
+ if not merged:
1748
+ print(f"still waiting for merge of {ref}: {url} (a real start stops here)")
1749
+ return caddis_exit.CLEAN
1750
+ if snap.phases.state(ref) != "done":
1751
+ raise _Stop(f"BLOCKED {ref} merged but its Tracker row is not done", caddis_exit.BLOCKED)
1752
+ print(f"merged path: {ref} merged ({url}); would check the roadmap against the published one, roll up "
1753
+ f"{waiting.get('milestone')} when every phase is merged, then remove the claim and "
1754
+ f"{waiting.get('branch')}, then take the next batch")
1755
+ return caddis_exit.CLEAN
1756
+ if batch is None:
1757
+ raise _Stop(f"BLOCKED nothing ready: {_status_line(repo, slug)}", caddis_exit.BLOCKED)
1758
+ except _Stop as stop:
1759
+ print(f"would stop: {caddis_redact.redact(stop.reason, masks)}")
1760
+ return stop.code if stop.code != caddis_exit.CLEAN else caddis_exit.NOT_RUN
1761
+ mid, plan, phase = batch["milestone"], batch["plan"], batch["phase"]
1762
+ ready = rollup_ready(snap, _read_record(_state_file(repo, slug, "rollup-failed")))
1763
+ if ready:
1764
+ print(f"would roll up first: {', '.join(ready)}")
1765
+ pending = _read_record(_state_file(repo, slug, "pending-rollup"))
1766
+ if pending:
1767
+ print(f"pending rollup: {pending.get('ref')} (would roll up {pending.get('milestone')} and clean up "
1768
+ f"{pending.get('branch')} first)")
1769
+ print(f"next: {mid} {plan}#{phase}")
1770
+ print(f"roadmap {slug}: runner host {snap.roadmap.runner_host or '(none)'} (not checked in a dry run); "
1771
+ f"lock {lock.path}; local files in {ctx.target.local}")
1772
+ lane = phase_lane(_plan_text(snap, plan), phase) or "none"
1773
+ if not lane_runs_here(lane, ctx.engine):
1774
+ print(f"would block {mid}: owner-gate phase {phase} lane: {lane}")
1775
+ return caddis_exit.CLEAN
1776
+ if snap.roadmap.get(mid).status == "todo":
1777
+ print(f"would set {mid} Status in-progress (docs PR)")
1778
+ wt = worktree_path(repo, slug, worktree_root)
1779
+ if wt.resolve() in _registered_worktrees(repo):
1780
+ print(f"would reuse the roadmap worktree {wt}")
1781
+ else:
1782
+ print(f"would create the roadmap worktree {wt} from origin/{snap.branch} (git worktree add --detach)")
1783
+ print(f"would make the session branch {session_branch(slug, plan, phase)} from origin/{snap.branch} in it")
1784
+ _dry_run_tail(ctx, template, render_prompt(template, plan, phase), cwd=wt)
1785
+ print("then: on CONTINUE with new commits on that branch, push it, open a PR and stop "
1786
+ f"`{WAITING_PREFIX}`; the recheck task restarts the runner after the merge")
1787
+ return caddis_exit.CLEAN
1788
+
1789
+ try: # the Runner host first, read only: a wrong-host start must change nothing (no lock, no record)
1790
+ host = caddis_roadmap.load(repo, slug).roadmap.runner_host
1791
+ except caddis_roadmap.RoadmapError:
1792
+ host = None # the read below, under the lock, reports and records it
1793
+ if host is not None and host.casefold() != hostname.casefold():
1794
+ return _finish(_Stop(f"BLOCKED this host {hostname} is not the roadmap's Runner host {host}",
1795
+ caddis_exit.NOT_RUN, push=False), ctx, False, 0, 0.0, None)
1796
+ try:
1797
+ lock.acquire(lambda message: _stale_notice(ctx, message))
1798
+ except _Stop as stop:
1799
+ return _finish(stop, ctx, False, 0, 0.0, None)
1800
+ except OSError as exc:
1801
+ return _finish(_Stop(f"BLOCKED claim not written: the runner lock ({type(exc).__name__})",
1802
+ caddis_exit.BLOCKED), ctx, False, 0, 0.0, None)
1803
+ try:
1804
+ first = caddis_roadmap.load(repo, slug)
1805
+ except caddis_roadmap.RoadmapError as exc: # like any stop: notify once and record it (not restartable)
1806
+ ctx.last_stop = last_stop_file(repo, "roadmap", slug)
1807
+ try:
1808
+ return _finish(_Stop(f"FAILED roadmap not read: {str(exc).splitlines()[0]}", exc.code), ctx, False, 0,
1809
+ 0.0, None)
1810
+ finally:
1811
+ lock.release()
1812
+ if first.roadmap.runner_host.casefold() != hostname.casefold():
1813
+ lock.release()
1814
+ return _finish(_Stop(f"BLOCKED this host {hostname} is not the roadmap's Runner host "
1815
+ f"{first.roadmap.runner_host}", caddis_exit.NOT_RUN, push=False), ctx, False, 0, 0.0, None)
1816
+ waiting_file = _state_file(repo, slug, "waiting")
1817
+ waiting = _read_record(waiting_file) # the waiting-for-merge data lives here, never in the last stop
1818
+ merged_by_gh = tracker_done = False
1819
+ if waiting:
1820
+ ref, url = str(waiting.get("ref", "")), str(waiting.get("pr", ""))
1821
+ tracker_done = first.phases.state(ref) == "done"
1822
+ try: # GitHub must confirm the merge, also when the Tracker row is done: no branch goes on the row alone
1823
+ merged_by_gh = pr_merged(repo, url)
1824
+ closed = not merged_by_gh and pr_closed(repo, url)
1825
+ except Exception as exc: # gh missing, 401, 404: a recorded, notified stop; the lock is released
1826
+ gh_repo = re.search(r"github\.com/([^/]+/[^/]+)/pull/", url or "")
1827
+ ctx.last_stop = last_stop_file(repo, "roadmap", slug)
1828
+ try:
1829
+ return _finish(_Stop(f"{GH_STOP_PREFIX} {gh_repo.group(1) if gh_repo else url} with account "
1830
+ f"{_gh_who()}: {type(exc).__name__}", caddis_exit.BLOCKED),
1831
+ ctx, False, 0, 0.0, None) # waiting.json stays: the next start resumes
1832
+ finally:
1833
+ lock.release()
1834
+ if tracker_done and not merged_by_gh: # the row says done, GitHub does not: keep everything, owner decides
1835
+ ctx.last_stop = last_stop_file(repo, "roadmap", slug)
1836
+ try:
1837
+ return _finish(_Stop(f"BLOCKED {ref} Tracker row is done but PR {url} is not merged",
1838
+ caddis_exit.BLOCKED), ctx, False, 0, 0.0, None)
1839
+ finally:
1840
+ lock.release()
1841
+ if not tracker_done:
1842
+ if closed: # closed without a merge: clean up under the lock
1843
+ claim = claim_file(repo, ref) if ref else None
1844
+ if claim is not None:
1845
+ claim.unlink(missing_ok=True)
1846
+ waiting_file.unlink(missing_ok=True)
1847
+ ctx.last_stop = last_stop_file(repo, "roadmap", slug)
1848
+ try:
1849
+ return _finish(_Stop(f"BLOCKED {ref} PR closed without a merge", caddis_exit.BLOCKED),
1850
+ ctx, False, 0, 0.0, None)
1851
+ finally:
1852
+ lock.release()
1853
+ if not merged_by_gh: # still waiting: stop now, leave the waiting record, the claim and the last stop
1854
+ lock.release()
1855
+ return _finish(_Stop(f"BLOCKED still waiting for merge of {ref}: {url}", caddis_exit.BLOCKED,
1856
+ push=False), ctx, False, 0, 0.0, None)
1857
+ _mark_running(ctx, last_stop_file(repo, "roadmap", slug))
1858
+ if not ENGINES[ctx.engine]["live_tested"]:
1859
+ print(f"caddis_roadmap_run: engine {ctx.engine} is an adapter, not live-tested", file=sys.stderr)
1860
+ branches_file = _state_file(repo, slug, "branches")
1861
+ made_branches = set(_read_record(branches_file).get("branches") or [])
1862
+
1863
+ sessions, spent, stuck, pushed = 0, 0.0, 0, False # per run, in memory
1864
+ pending_file = _state_file(repo, slug, "pending-rollup")
1865
+ swept = False
1866
+ relay_slug: str | None = None
1867
+ last_write: tuple[str, str] | None = None
1868
+ wt: Path | None = None
1869
+ start = ctx.clock()
1870
+
1871
+ try:
1872
+ baseline = _Baseline(first.roadmap)
1873
+ if waiting:
1874
+ # A restart after `BLOCKED waiting for merge`: the merged origin roadmap must still be the one the runner
1875
+ # published against. Never take a fresh baseline without this check.
1876
+ saved = _read_text(_state_file(repo, slug, "published-roadmap").with_suffix(".md"))
1877
+ hint = (f" (if the owner made this change: roadmap-run.ps1 -Roadmap {slug} -AcceptRoadmap, then start "
1878
+ "again)")
1879
+ if waiting.get("roadmap_sha") != _roadmap_sha(first.text):
1880
+ if saved is None or _roadmap_sha(saved) != waiting.get("roadmap_sha"):
1881
+ raise _Stop("FAILED roadmap changed: the origin roadmap differs from the one published against"
1882
+ + hint, caddis_exit.BLOCKED)
1883
+ try:
1884
+ _Baseline(caddis_roadmap.parse_roadmap(saved)).observe(first)
1885
+ except _Stop as stop:
1886
+ raise _Stop(stop.reason + hint, stop.code) from None
1887
+ ref = str(waiting.get("ref", ""))
1888
+ claim = claim_file(repo, ref) if ref else None
1889
+ if claim is not None and (not waiting.get("claim_sha") or not claim.is_file()
1890
+ or _sha_file(claim) != waiting.get("claim_sha")): # fail closed
1891
+ raise _Stop(f"FAILED own claim missing or changed: {ref}", caddis_exit.BLOCKED)
1892
+ merged = {"ref": ref, "pr": waiting.get("pr"), "milestone": waiting.get("milestone"),
1893
+ "claim": str(claim) if claim is not None else None, "branch": waiting.get("branch")}
1894
+ if merged_by_gh and not tracker_done: # GitHub says merged, the row is not done: the owner decides
1895
+ _clean_merged(ctx, repo, slug, merged, worktree_root, delete_remote_branch, made_branches,
1896
+ branches_file)
1897
+ waiting_file.unlink(missing_ok=True)
1898
+ raise _Stop(f"BLOCKED {ref} merged but its Tracker row is not done", caddis_exit.BLOCKED)
1899
+ if tracker_done and merged_by_gh:
1900
+ # The merged work moves to its own file (rollup first, cleanup after, then the file goes), and the
1901
+ # wait is over. Any stop before that records itself; the next start finds pending-rollup.json.
1902
+ _write_record(pending_file, {**merged, "utc": _iso(datetime.now(timezone.utc))})
1903
+ waiting_file.unlink(missing_ok=True)
1904
+ _check_orphaned_claims(repo, slug, _read_record(waiting_file), _read_record(pending_file))
1905
+ while True:
1906
+ ctx.checks(sessions, spent, ctx.clock() - start)
1907
+ pending = _read_record(pending_file)
1908
+ if pending:
1909
+ mid = pending.get("milestone")
1910
+ if isinstance(mid, str):
1911
+ snap = _load_observed(repo, slug, baseline)
1912
+ ms = snap.roadmap.get(mid)
1913
+ if ms is not None and ms.status in caddis_roadmap.OPEN and ms.phases and \
1914
+ all(snap.phases.state(r) == "done" for r in ms.phases):
1915
+ rollup(mid) # raises _Stop on a failed rollup: recorded, the file stays
1916
+ _clean_merged(ctx, repo, slug, pending, worktree_root, delete_remote_branch, made_branches,
1917
+ branches_file)
1918
+ pending_file.unlink(missing_ok=True)
1919
+ if not swept: # once per run: roll up every milestone that waits only for its rollup
1920
+ swept = True
1921
+ snap = _load_observed(repo, slug, baseline)
1922
+ failed_file = _state_file(repo, slug, "rollup-failed")
1923
+ for mid in rollup_ready(snap, _read_record(failed_file)):
1924
+ if rollup(mid) == 0: # nothing written: a recorded stop, and recheck must not retry it
1925
+ _write_record(failed_file, {**_read_record(failed_file), mid: _roadmap_sha(snap.text)})
1926
+ raise _Stop(f"BLOCKED rollup of {mid} wrote nothing; the owner decides", caddis_exit.BLOCKED)
1927
+ with _claim(repo): # choose and claim atomically: no other runner can choose the same batch
1928
+ try:
1929
+ batch, snap = caddis_roadmap.next_with_snapshot(repo, slug)
1930
+ except caddis_roadmap.RoadmapError as exc:
1931
+ raise _Stop(f"FAILED roadmap changed: the origin copy no longer loads: "
1932
+ f"{str(exc).splitlines()[0]}", caddis_exit.BLOCKED) from None
1933
+ baseline.observe(snap)
1934
+ if batch is not None:
1935
+ lock.set_batch(f"{batch['plan']}#{batch['phase']}")
1936
+ if batch is None:
1937
+ detail = _status_line(repo, slug)
1938
+ if snap.roadmap.milestones and all(ms.status in caddis_roadmap.FINISHED
1939
+ for ms in snap.roadmap.milestones):
1940
+ gone = remove_worktree_if_clean(repo, slug, worktree_root)
1941
+ detail += "; roadmap finished; " + gone
1942
+ if gone.startswith(("worktree removed", "no worktree")):
1943
+ detail += "; " + _cleanup_finished_branches(repo, slug, snap.branch, pr_merged,
1944
+ delete_remote_branch)
1945
+ raise _Stop(RESTARTABLE_STOP, caddis_exit.BLOCKED, detail=detail,
1946
+ record={"held": _held_batches(repo, slug, snap)})
1947
+ mid, plan, phase = batch["milestone"], batch["plan"], batch["phase"]
1948
+ ref = f"{plan}#{phase}"
1949
+ print(f"next: {mid} {ref}")
1950
+ if last_write == (mid, ref): # the runner's own block did not take the batch off `next`
1951
+ raise _Stop(f"FAILED roadmap unchanged after the runner's write for {mid} {ref}",
1952
+ caddis_exit.BLOCKED)
1953
+ relay_slug = plan
1954
+ plan_text = _plan_text(snap, plan)
1955
+ lane = phase_lane(plan_text, phase) or "none" # no Lane line: not the runner's (fail closed)
1956
+ if not lane_runs_here(lane, ctx.engine):
1957
+ reason = f"phase {phase} lane: {' '.join(lane.split())[:160]}".strip()
1958
+ write(["block", "--id", mid, "--type", "owner-gate", "--reason", reason],
1959
+ ("blocker", mid, "owner-gate", reason))
1960
+ last_write = (mid, ref)
1961
+ continue
1962
+ last_write = None
1963
+ if snap.roadmap.get(mid).status == "todo":
1964
+ write(["set", "--id", mid, "--col", "Status", "--value", "in-progress"],
1965
+ ("status", mid, "in-progress"))
1966
+ snap = _load_observed(repo, slug, baseline) # the write window is guarded too
1967
+ # Every session runs in the roadmap's own worktree, on a fresh branch made from origin/<default>.
1968
+ wt = wt or ensure_worktree(repo, slug, snap.branch, worktree_root)
1969
+ branch = session_branch(slug, plan, phase)
1970
+ if branch not in made_branches and _branch_on_origin(repo, branch):
1971
+ raise _Stop(f"BLOCKED branch {branch} is on origin and this runner did not make it",
1972
+ caddis_exit.BLOCKED)
1973
+ for ref_name in (f"refs/heads/{branch}", f"refs/remotes/origin/{branch}"):
1974
+ if _git_out(wt, "rev-parse", "--verify", "--quiet", ref_name) is None:
1975
+ continue
1976
+ ahead = _git_out(wt, "rev-list", "--count", f"origin/{snap.branch}..{ref_name}")
1977
+ if ahead is None or ahead != "0": # an earlier session's work: never reset it
1978
+ raise _Stop(f"BLOCKED {branch} has unpublished commits from an earlier session",
1979
+ caddis_exit.BLOCKED)
1980
+ left = _worktree_work(wt) # a blocked session's uncommitted work must never ride into another branch
1981
+ if left is None or left:
1982
+ was = _git_out(wt, "rev-parse", "--abbrev-ref", "HEAD") or "?"
1983
+ raise _Stop(f"BLOCKED roadmap worktree has uncommitted work from {was} ({wt})", caddis_exit.BLOCKED)
1984
+ made = _git_out(wt, "checkout", "-q", "-B", branch, f"origin/{snap.branch}")
1985
+ if made is None:
1986
+ raise _Stop(f"FAILED cannot make the session branch {branch} in {wt} (is it clean?)",
1987
+ caddis_exit.BLOCKED)
1988
+ head_before = _git_out(wt, "rev-parse", "HEAD")
1989
+ done_before = _done_phases(snap, plan)
1990
+ state_before = state_snapshot(repo, "roadmap", slug)
1991
+ s = _one_session(ctx, render_prompt(template, plan, phase), plan, sessions + 1, spent,
1992
+ ctx.clock() - start, slug, cwd=wt)
1993
+ _check_state(repo, "roadmap", slug, state_before) # before the runner writes anything after it
1994
+ sessions += 1
1995
+ spent += s.cost
1996
+ pushed = s.pushed
1997
+ s.record.update({"milestone": mid, "batch": ref, "branch": branch})
1998
+ try:
1999
+ stop = _decide(ctx.engine, s.outcome, s.result, s.relay_text, s.hash_before, s.hash_after)
2000
+ s.record["status"] = "CONTINUE" if stop is None else stop.reason
2001
+ after = _load_observed(repo, slug, baseline) # one read after the session (no merge wait)
2002
+ stuck = 0 if _done_phases(after, plan) > done_before else stuck + 1
2003
+ if stop is not None:
2004
+ if stop.status is None or stop.status[0] not in ("OWNER-GATE", "BLOCKED"):
2005
+ raise stop
2006
+ btype = "owner-gate" if stop.status[0] == "OWNER-GATE" else "external"
2007
+ reason = f"{ref}: {stop.status[1]}".strip()
2008
+ write(["block", "--id", mid, "--type", btype, "--reason", reason], ("blocker", mid, btype, reason))
2009
+ last_write = (mid, ref)
2010
+ elif after.phases.state(ref) != "done":
2011
+ def review_before_publish() -> None:
2012
+ # The review gate runs BEFORE the claim and the push: a `risk: high` phase is never
2013
+ # published unreviewed. Its review and fix session count against the caps.
2014
+ nonlocal sessions, spent, pushed
2015
+ tally = _Tally(sessions, spent, pushed)
2016
+ try:
2017
+ reviewed = _review_gate(ctx, tally, plan=plan, extra_texts=(plan_text,),
2018
+ origin_text=_plan_text(after, plan) or None,
2019
+ elapsed=lambda: ctx.clock() - start, spend_slug=slug,
2020
+ also=(phase,), cwd=wt)
2021
+ finally:
2022
+ sessions, spent, pushed = tally.sessions, tally.spent, tally.pushed
2023
+ if reviewed:
2024
+ _load_observed(repo, slug, baseline) # the review and fix window is guarded too
2025
+ now_on = _git_out(wt, "rev-parse", "--abbrev-ref", "HEAD")
2026
+ if now_on != branch:
2027
+ raise _Stop(f"FAILED {ref}: the review fix round left the runner's branch {branch} "
2028
+ f"(now on {now_on})", caddis_exit.BLOCKED)
2029
+
2030
+ _publish_and_wait(ctx, wt, slug, mid, plan, phase, ref, branch, head_before, after, plan_text,
2031
+ publish, made_branches, branches_file, s.record,
2032
+ before_publish=review_before_publish)
2033
+ else:
2034
+ ms = after.roadmap.get(mid)
2035
+ if ms is not None and all(after.phases.state(r) == "done" for r in ms.phases):
2036
+ rollup(mid)
2037
+ if stuck >= STUCK_SESSIONS:
2038
+ raise _Stop("BLOCKED stuck", caddis_exit.BLOCKED)
2039
+ except _Stop as stop_now:
2040
+ s.record.setdefault("stop", stop_now.reason)
2041
+ raise
2042
+ finally:
2043
+ s.record.setdefault("status", "FAILED")
2044
+ ctx.log(s.record)
2045
+ if after_session is not None:
2046
+ after_session(sessions)
2047
+ except _Stop as stop:
2048
+ return _finish(stop, ctx, pushed, sessions, spent, relay_slug, wt)
2049
+ except Exception as exc:
2050
+ _crashed(ctx, exc, pushed, sessions, spent, relay_slug, wt)
2051
+ raise
2052
+ finally:
2053
+ lock.release()
2054
+
2055
+
2056
+ def _sha_file(path: Path) -> str:
2057
+ try:
2058
+ return hashlib.sha256(path.read_bytes()).hexdigest()
2059
+ except OSError:
2060
+ return "unreadable"
2061
+
2062
+
2063
+ def _claims_of(repo: Path, kind: str, slug: str) -> dict[str, Path]:
2064
+ """This runner's claims (`claims/*.json` whose `roadmap` field, or `plan` for plan mode, is `slug`)."""
2065
+ base = caddis_roadmap.runner_state_dir(repo)
2066
+ out: dict[str, Path] = {}
2067
+ if base is None:
2068
+ return out
2069
+ try:
2070
+ files = sorted((base / "claims").glob("*.json"))
2071
+ except OSError:
2072
+ return out
2073
+ for path in files:
2074
+ if _read_record(path).get(kind) == slug:
2075
+ out[f"claims/{path.name}"] = path
2076
+ return out
2077
+
2078
+
2079
+ def state_snapshot(repo: Path, kind: str, slug: str) -> dict[str, str]:
2080
+ """sha256 of every file in THIS runner's folder (`<git common dir>/caddis-runner/<kind>-<slug>/`, recursively,
2081
+ so later records such as `reviewed/` too) and of this runner's own claims. Another runner's state is not
2082
+ watched: it never stops this one. Outside a git repo: empty."""
2083
+ base = caddis_roadmap.runner_state_dir(repo)
2084
+ out: dict[str, str] = {}
2085
+ if base is None:
2086
+ return out
2087
+ own = base / f"{kind}-{slug}"
2088
+ if own.is_dir():
2089
+ for path in sorted(own.rglob("*")):
2090
+ if path.is_file():
2091
+ out[path.relative_to(base).as_posix()] = _sha_file(path)
2092
+ for name, path in _claims_of(repo, kind, slug).items():
2093
+ out[name] = _sha_file(path)
2094
+ return out
2095
+
2096
+
2097
+ def _state_changes(before: dict[str, str], after: dict[str, str]) -> list[str]:
2098
+ changes = [f"{name} removed" for name in before if name not in after]
2099
+ changes += [f"{name} added" for name in after if name not in before]
2100
+ changes += [f"{name} changed" for name in before if name in after and before[name] != after[name]]
2101
+ return changes
2102
+
2103
+
2104
+ def _check_state(repo: Path, kind: str, slug: str, before: dict[str, str]) -> None:
2105
+ """The runner writes none of its own state while a session runs, so a change there is a session's mistake:
2106
+ stop, not restartable. Sessions are cooperative; this catches accidents, it is not a security boundary."""
2107
+ changes = _state_changes(before, state_snapshot(repo, kind, slug))
2108
+ if changes:
2109
+ raise _Stop("FAILED runner state changed: " + "; ".join(changes[:5]), caddis_exit.BLOCKED)
2110
+
2111
+
2112
+ def _read_text(path: Path) -> str | None:
2113
+ try:
2114
+ return Path(path).read_bytes().decode("utf-8")
2115
+ except OSError:
2116
+ return None
2117
+
2118
+
2119
+ def _publish_and_wait(ctx: _Ctx, repo: Path, slug: str, mid: str, plan: str, phase: int, ref: str, branch: str,
2120
+ head_before: str | None, after: "caddis_roadmap.Snapshot", plan_text: str,
2121
+ publish: Callable[..., str], made_branches: set, branches_file: Path, record: dict,
2122
+ before_publish: Callable[[], None] | None = None) -> None:
2123
+ """CONTINUE with the phase not merged: publish the runner's session branch and raise the waiting stop.
2124
+ Refuses (FAILED, not restartable) when the session left that branch, made no commit, or touched the roadmap."""
2125
+ def verify() -> None:
2126
+ current = _git_out(repo, "rev-parse", "--abbrev-ref", "HEAD")
2127
+ if current != branch:
2128
+ raise _Stop(f"FAILED {ref}: the session left the runner's branch {branch} (now on {current})",
2129
+ caddis_exit.BLOCKED)
2130
+ head_after = _git_out(repo, "rev-parse", "HEAD")
2131
+ ahead = _git_out(repo, "rev-list", "--count", f"origin/{after.branch}..{branch}")
2132
+ if head_after == head_before or not ahead or ahead == "0":
2133
+ raise _Stop(f"FAILED {ref}: CONTINUE with no new commits on {branch} and not merged",
2134
+ caddis_exit.BLOCKED)
2135
+ listing = _git_out(repo, "diff", "--name-only", f"origin/{after.branch}...{branch}")
2136
+ if listing is None: # fail closed: without the list of changed files, never publish
2137
+ raise _Stop(f"FAILED {ref}: cannot list the session's changes (git diff --name-only failed); "
2138
+ "not published", caddis_exit.BLOCKED)
2139
+ touched = [p for p in listing.splitlines() if p.replace("\\", "/").startswith(".caddis/roadmaps/")]
2140
+ if touched:
2141
+ raise _Stop(f"FAILED session changed the roadmap ({', '.join(touched[:3])}); not published",
2142
+ caddis_exit.BLOCKED)
2143
+
2144
+ verify()
2145
+ if before_publish is not None: # the review gate (caddis-next Phase 11); a fix session may have committed
2146
+ before_publish()
2147
+ verify()
2148
+ claim = claim_file(repo, ref)
2149
+ utc = _iso(datetime.now(timezone.utc))
2150
+ if claim is not None: # the phase stays held while its PR waits (written before the push)
2151
+ try:
2152
+ claim.parent.mkdir(parents=True, exist_ok=True)
2153
+ _replace_with(claim, json.dumps({"batch": ref, "roadmap": slug, "pr": None, "utc": utc}))
2154
+ except OSError as exc:
2155
+ raise _Stop(f"BLOCKED claim not written: {ref} ({type(exc).__name__})", caddis_exit.BLOCKED) from None
2156
+ ctx.written_claims.append(claim)
2157
+ try: # from here to the waiting record, ANY failure removes the claim and stops FAILED (not restartable)
2158
+ title = f"{plan} phase {phase}" + (f": {_phase_title(plan_text, phase)}"
2159
+ if _phase_title(plan_text, phase) else "")
2160
+ body = f"Opened by caddis_roadmap_run.py for roadmap {slug}, milestone {mid}, {ref}."
2161
+ made_branches.add(branch) # recorded BEFORE the push: a push that lands and a PR that fails do not lock it out
2162
+ _write_record(branches_file, {**_read_record(branches_file), "branches": sorted(made_branches)})
2163
+ url = publish(repo, branch, after.branch, title, body)
2164
+ prs = dict(_read_record(branches_file).get("prs") or {})
2165
+ prs[branch] = url # the finished roadmap uses it to tell a squash-merged branch from an unmerged one
2166
+ _write_record(branches_file, {"branches": sorted(made_branches), "prs": prs})
2167
+ claim_sha = None
2168
+ if claim is not None:
2169
+ _write_record(claim, {"batch": ref, "roadmap": slug, "pr": url, "utc": utc})
2170
+ claim_sha = _sha_file(claim)
2171
+ saved = _state_file(repo, slug, "published-roadmap").with_suffix(".md")
2172
+ try:
2173
+ saved.parent.mkdir(parents=True, exist_ok=True)
2174
+ tmp = saved.with_name(f"{saved.name}.{os.getpid()}.tmp")
2175
+ tmp.write_bytes(after.text.encode("utf-8")) # bytes: the exact text the hash was taken of
2176
+ os.replace(tmp, saved)
2177
+ except OSError as exc:
2178
+ print(f"caddis_roadmap_run: published roadmap not saved: {exc}", file=sys.stderr)
2179
+ record["pr"] = url
2180
+ except Exception as exc:
2181
+ if claim is not None:
2182
+ claim.unlink(missing_ok=True)
2183
+ if claim in ctx.written_claims:
2184
+ ctx.written_claims.remove(claim)
2185
+ detail = str(exc).splitlines()[0] if str(exc) else type(exc).__name__
2186
+ raise _Stop(f"FAILED {ref}: could not push or open the PR: {detail}", caddis_exit.BLOCKED) from None
2187
+ if claim is not None and claim in ctx.written_claims:
2188
+ ctx.written_claims.remove(claim) # the waiting record below holds it now
2189
+ _write_record(_state_file(repo, slug, "waiting"), # the waiting data, in its own file (never in the last stop)
2190
+ {"ref": ref, "pr": url, "milestone": mid, "branch": branch,
2191
+ "claim": str(claim) if claim is not None else None, "claim_sha": claim_sha,
2192
+ "roadmap_sha": _roadmap_sha(after.text), "utc": _iso(datetime.now(timezone.utc))})
2193
+ raise _Stop(f"{WAITING_PREFIX} {ref} {url}", caddis_exit.BLOCKED)
2194
+
2195
+
2196
+ def _spend_reserve(home: Path, slug: str, moment: datetime, started: bool) -> str:
2197
+ """Book one agy session under the global spend cap (caddis_spend.py); _Stop when the cap refuses it or cannot be
2198
+ read (a file error never crashes the runner: the stop is logged and notified, redacted, like any other).
2199
+ The caller name in the spend log is `runner-<slug>`."""
2200
+ try:
2201
+ rid = caddis_spend.reserve(home, f"runner-{slug}", moment)
2202
+ except caddis_spend.NotConfigured:
2203
+ code = caddis_exit.BLOCKED if started else caddis_exit.NOT_RUN
2204
+ raise _Stop("FAILED spend cap not configured", code) from None
2205
+ except TimeoutError:
2206
+ raise _Stop("BLOCKED spend cap lock busy", caddis_exit.BLOCKED) from None
2207
+ except Exception as exc:
2208
+ raise _Stop(f"FAILED spend cap error: {exc!r}", caddis_exit.BLOCKED) from None
2209
+ if rid is None:
2210
+ raise _Stop("BLOCKED spend cap", caddis_exit.BLOCKED)
2211
+ return rid
2212
+
2213
+
2214
+ def _spend_settle(home: Path, rid: str, cmd: Sequence[str], outcome: "_Outcome | None", moment: datetime,
2215
+ duration_s: float) -> None:
2216
+ """Record the session's real cost: (0, 0) when it never started, its reserve (estimated) when there is no outcome
2217
+ (an exception) or no readable token counts."""
2218
+ try:
2219
+ if outcome is None:
2220
+ usage_counts = None
2221
+ elif not outcome.started:
2222
+ usage_counts = (0, 0) # nothing ran, nothing was spent
2223
+ elif caddis_spend.json_mode(list(cmd)):
2224
+ usage_counts = caddis_spend.tokens_from_output(outcome.stdout)
2225
+ else:
2226
+ usage_counts = None
2227
+ caddis_spend.settle(home, rid, usage_counts, moment, duration_s=duration_s)
2228
+ except Exception as exc: # the pending reserve keeps counting for 24 h; never lose the session's result
2229
+ print(f"caddis_roadmap_run: could not settle spend {rid}: {exc}", file=sys.stderr)
2230
+
2231
+
2232
+ def _spend_preview(home: Path, moment: datetime) -> None:
2233
+ """The dry run's read-only spend check: the same stops as _spend_reserve, with no pending line written."""
2234
+ try:
2235
+ cap = caddis_spend.load_cap(home)
2236
+ over = caddis_spend.committed_last_24h(home, moment) + cap.reserve > cap.eur_per_24h
2237
+ except caddis_spend.NotConfigured:
2238
+ raise _Stop("FAILED spend cap not configured", caddis_exit.NOT_RUN) from None
2239
+ except Exception as exc:
2240
+ raise _Stop(f"FAILED spend cap error: {exc!r}", caddis_exit.NOT_RUN) from None
2241
+ if over:
2242
+ raise _Stop("BLOCKED spend cap", caddis_exit.BLOCKED)
2243
+
2244
+
2245
+ @dataclass
2246
+ class _Outcome:
2247
+ code: int
2248
+ stdout: str
2249
+ stderr: str
2250
+ timed_out: bool = False
2251
+ started: bool = True
2252
+
2253
+
2254
+ def _launch(cmd: list[str], cwd: Path, timeout_s: float, stdin_text: str | None = None,
2255
+ env: dict | None = None) -> _Outcome:
2256
+ try:
2257
+ proc = subprocess.run(cmd, cwd=cwd, input=stdin_text if stdin_text is not None else "", capture_output=True,
2258
+ text=True, encoding="utf-8", errors="replace", timeout=timeout_s,
2259
+ env=env if env is not None else headless_env())
2260
+ except subprocess.TimeoutExpired as exc:
2261
+ out = exc.stdout.decode("utf-8", "replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "")
2262
+ err = exc.stderr.decode("utf-8", "replace") if isinstance(exc.stderr, bytes) else (exc.stderr or "")
2263
+ return _Outcome(-1, out, err, timed_out=True)
2264
+ except OSError as exc:
2265
+ return _Outcome(-1, "", f"cannot start session: {exc}", started=False)
2266
+ return _Outcome(proc.returncode, proc.stdout or "", proc.stderr or "")
2267
+
2268
+
2269
+ def _decide(engine: str, outcome: _Outcome, result: dict | None, relay_text: str, hash_before: str | None,
2270
+ hash_after: str | None) -> _Stop | None:
2271
+ """None to go on; else the stop. A missing or bad status line is FAILED, never CONTINUE (design 8.1)."""
2272
+ failed = _session_failed(engine, outcome, result)
2273
+ if failed is not None:
2274
+ return failed
2275
+ if hash_after is None:
2276
+ return _Stop("FAILED no relay", caddis_exit.BLOCKED)
2277
+ try:
2278
+ kind, arg = caddis_workstreams.parse_status(relay_text)
2279
+ except ValueError as exc:
2280
+ return _Stop(f"FAILED bad status line: {exc}", caddis_exit.BLOCKED)
2281
+ if kind == "CONTINUE":
2282
+ if hash_after == hash_before:
2283
+ return _Stop("FAILED no progress", caddis_exit.BLOCKED)
2284
+ return None
2285
+ if kind == "FAILED":
2286
+ return _Stop(f"FAILED {arg}", caddis_exit.BLOCKED, from_relay=True, status=(kind, arg))
2287
+ return _Stop(f"{kind} {arg}", caddis_exit.CLEAN, from_relay=True, status=(kind, arg))
2288
+
2289
+
2290
+ def _session_failed(engine: str, outcome: _Outcome, result: dict | None) -> _Stop | None:
2291
+ """The stop for a session that did not finish cleanly (usage limit, timeout, exit code, output, error); None when
2292
+ it did. The relay is not read here: a review fix session writes no status line."""
2293
+ if usage_limited(engine, outcome.stdout + "\n" + outcome.stderr):
2294
+ return _Stop("BLOCKED usage limit", caddis_exit.BLOCKED, push=False)
2295
+ if outcome.timed_out:
2296
+ return _Stop("FAILED session timeout", caddis_exit.BLOCKED)
2297
+ if outcome.code != 0:
2298
+ tail = " ".join(outcome.stderr.split())[-160:]
2299
+ return _Stop(f"FAILED session exit {outcome.code}" + (f": {tail}" if tail else ""), caddis_exit.BLOCKED)
2300
+ if result is None:
2301
+ return _Stop("FAILED session output is not JSON", caddis_exit.BLOCKED)
2302
+ if result.get("is_error") is True:
2303
+ return _Stop(f"FAILED session error: {result.get('subtype', 'unknown')}", caddis_exit.BLOCKED)
2304
+ return None
2305
+
2306
+
2307
+ def _closed(target: Target) -> int:
2308
+ return sum(closed_rows(path) for path in target.progress)
2309
+
2310
+
2311
+ def _finish(stop: _Stop, ctx: _Ctx, pushed: bool, sessions: int, spent: float, relay_slug: str | None,
2312
+ relay_root: Path | None = None) -> int:
2313
+ masks, target = ctx.masks, ctx.target
2314
+ line = f"STATUS: {stop.reason}" if stop.from_relay else f"RUNNER STOP: {stop.reason}"
2315
+ if stop.detail:
2316
+ message = caddis_redact.redact(f"{line}. {stop.detail}", masks)
2317
+ else:
2318
+ relay_text = _read_relay(relay_root or ctx.repo, relay_slug)[0] if relay_slug else ""
2319
+ # Redact BEFORE _next_step cuts the text to 120 characters: a key cut in half is no longer an exact match.
2320
+ relay_text = caddis_redact.redact(relay_text, masks)
2321
+ next_step = caddis_workstreams._next_step(relay_text) if relay_text else "next step unrecorded"
2322
+ message = caddis_redact.redact(f"{line}. Next: {next_step}", masks) # relay text is session-written
2323
+ if ctx.last_stop is not None:
2324
+ _write_options(ctx) # re-written from memory at every stop: a changed file can never raise the caps
2325
+ _write_last_stop(ctx.last_stop, stop) # always the REAL stop
2326
+ toast_exit = ctx.toast(f"caddis {target.kind} {target.slug}", message)
2327
+ push_exit = None
2328
+ if stop.push and not (pushed and stop.from_relay): # PUSH-SENT covers only the stop the session reported
2329
+ push_exit = ctx.push(f"{target.kind} {target.slug}: {message}")
2330
+ ctx.log({"kind": "stop", "stop": stop.reason, "exit": stop.code, "sessions": sessions, "usd": round(spent, 4),
2331
+ "toast_exit": toast_exit, "push_exit": push_exit, "time": _iso(datetime.now(timezone.utc))})
2332
+ print(caddis_redact.redact(f"RUNNER STOP: {stop.reason} (sessions {sessions}, USD {spent:.2f}, exit {stop.code})",
2333
+ masks))
2334
+ return stop.code
2335
+
2336
+
2337
+ def _write_record(path: Path, record: dict) -> None:
2338
+ try:
2339
+ path.parent.mkdir(parents=True, exist_ok=True)
2340
+ _replace_with(path, json.dumps(record))
2341
+ except OSError as exc: # the recheck task then finds an old record; it restarts only after "nothing ready"
2342
+ print(f"caddis_roadmap_run: runner state not written to {path}: {exc}", file=sys.stderr)
2343
+
2344
+
2345
+ def _write_last_stop(path: Path, stop: _Stop) -> None:
2346
+ _write_record(path, {**stop.record, "reason": stop.reason, "exit": stop.code,
2347
+ "utc": _iso(datetime.now(timezone.utc))})
2348
+
2349
+
2350
+ def options_file(last_stop: Path) -> Path:
2351
+ """The launch options of the run that holds (or last held) the lock, next to its last stop."""
2352
+ return last_stop.with_name("options.json")
2353
+
2354
+
2355
+ def _write_options(ctx: _Ctx) -> None:
2356
+ """The launch options, from the run's memory (never from the file, which a session can write)."""
2357
+ caps = ctx.caps
2358
+ _write_record(options_file(ctx.last_stop), {
2359
+ "engine": ctx.engine, "model": ctx.model, "max_hours": caps.max_hours, "max_sessions": caps.max_sessions,
2360
+ "session_timeout_min": caps.session_timeout_min, "max_usd_per_session": caps.max_usd_per_session,
2361
+ "max_usd_total": caps.max_usd_total, "gh_account": ctx.gh_account})
2362
+
2363
+
2364
+ def _mark_running(ctx: _Ctx, last_stop: Path) -> None:
2365
+ """At lock time: the last stop becomes `running` (not restartable) and the launch options are recorded, so a run
2366
+ that is killed without a stop needs the owner, and the recheck task restarts with the same options."""
2367
+ ctx.last_stop = last_stop
2368
+ _write_options(ctx)
2369
+ _write_record(last_stop, {"reason": RUNNING, "exit": None, "utc": _iso(datetime.now(timezone.utc)),
2370
+ "pid": os.getpid()})
2371
+
2372
+
2373
+ def _crashed(ctx: _Ctx, exc: Exception, pushed: bool, sessions: int, spent: float, relay_slug: str | None,
2374
+ relay_root: Path | None = None) -> None:
2375
+ """An unexpected exception in the loop: record `FAILED crash: <type>` and notify once; the caller re-raises.
2376
+ `relay_root`: where the session relay is (roadmap mode: the roadmap worktree)."""
2377
+ stop = _Stop(f"FAILED crash: {type(exc).__name__}", caddis_exit.BLOCKED)
2378
+ for claim in list(ctx.written_claims): # a claim with no waiting record must not hold the phase for ever
2379
+ Path(claim).unlink(missing_ok=True)
2380
+ ctx.written_claims.clear()
2381
+ try:
2382
+ _finish(stop, ctx, pushed, sessions, spent, relay_slug, relay_root)
2383
+ except Exception as again: # never hide the first exception
2384
+ print(f"caddis_roadmap_run: crash not notified: {again!r}", file=sys.stderr)
2385
+ if ctx.last_stop is not None:
2386
+ _write_last_stop(ctx.last_stop, stop)
2387
+
2388
+
2389
+ _ARG_RE = re.compile(r"^[A-Za-z0-9._\[\]-]+$")
2390
+ _RESTART_ARGS = (("engine", "-Engine"), ("model", "-Model"), ("max_hours", "-MaxHours"),
2391
+ ("max_sessions", "-MaxSessions"), ("session_timeout_min", "-SessionTimeoutMin"),
2392
+ ("max_usd_per_session", "-MaxUsdPerSession"), ("max_usd_total", "-MaxUsdTotal"),
2393
+ ("gh_account", "-GhAccount"))
2394
+
2395
+
2396
+ def restart_args(repo: Path, roadmap: str) -> list[str]:
2397
+ """`roadmap-run.ps1` arguments that repeat the last run's launch options; [] when none are recorded. ValueError
2398
+ for a record that is not readable or holds an unsafe value."""
2399
+ path = options_file(last_stop_file(Path(repo), "roadmap", roadmap))
2400
+ try:
2401
+ record = json.loads(path.read_text(encoding="utf-8"))
2402
+ except FileNotFoundError:
2403
+ return []
2404
+ except (OSError, ValueError) as exc:
2405
+ raise ValueError(f"launch options not readable: {type(exc).__name__}") from None
2406
+ if not isinstance(record, dict):
2407
+ raise ValueError("launch options are not an object")
2408
+ out: list[str] = []
2409
+ defaults = Caps()
2410
+ for key, flag in _RESTART_ARGS:
2411
+ value = record.get(key)
2412
+ if value is None:
2413
+ continue
2414
+ if hasattr(defaults, key): # a cap: never above the built-in default (the options file is writable)
2415
+ cap = getattr(defaults, key)
2416
+ number = value if isinstance(value, (int, float)) and not isinstance(value, bool) else None
2417
+ if number is None or not (0 < number <= cap):
2418
+ print(f"caddis_roadmap_run: launch option {key} {value!r} is not in (0, {cap:g}]; using {cap:g}",
2419
+ file=sys.stderr)
2420
+ number = cap
2421
+ value = int(number) if key == "max_sessions" else float(number)
2422
+ if isinstance(value, bool) or not isinstance(value, (str, int, float)):
2423
+ raise ValueError(f"launch option {key} is not a value")
2424
+ text = value if isinstance(value, str) else (f"{value:g}" if isinstance(value, float) else str(value))
2425
+ if key == "engine" and text not in ENGINES:
2426
+ raise ValueError(f"launch option engine {text!r} is unknown")
2427
+ if not _ARG_RE.match(text):
2428
+ raise ValueError(f"launch option {key} holds an unsafe value")
2429
+ out += [flag, text]
2430
+ return out
2431
+
2432
+
2433
+ def accept_roadmap(repo: Path, roadmap: str, yes: bool = False, dry_run: bool = False,
2434
+ confirm: Callable[[str], str] = input) -> int:
2435
+ """Owner only (refused headless): accept the origin roadmap as it is now as the baseline of the current merge
2436
+ wait (the roadmap hash and the saved text in waiting.json), after an owner edit during the wait. It takes the runner
2437
+ lock (refuses a live runner), prints the unified diff from the published baseline, and asks for `accept` at the
2438
+ console unless `yes`. `dry_run` prints the diff and writes nothing (no lock either)."""
2439
+ if not dry_run and caddis_roadmap.is_headless(): # the dry run only reads, so it may run anywhere
2440
+ print("caddis_roadmap_run: --accept-roadmap is owner-only and this session is headless "
2441
+ f"({', '.join(caddis_roadmap.HEADLESS_VARS)} in the environment, or stdin is not a console)",
2442
+ file=sys.stderr)
2443
+ return caddis_exit.ERROR
2444
+ repo = Path(repo)
2445
+ lock = None
2446
+ if not dry_run: # the lock FIRST; everything below is read under it
2447
+ lock = _RunnerLock(runner_lock_file(repo, "roadmap", roadmap), Caps().max_hours,
2448
+ claim_path=caddis_roadmap.claim_lock_path(repo))
2449
+ try:
2450
+ lock.acquire(lambda message: print(f"caddis_roadmap_run: {message}", file=sys.stderr))
2451
+ except _Stop as stop:
2452
+ print(f"caddis_roadmap_run: {stop.reason}", file=sys.stderr)
2453
+ return stop.code
2454
+ try:
2455
+ waiting_file = _state_file(repo, roadmap, "waiting")
2456
+ waiting = _read_record(waiting_file)
2457
+ if not waiting:
2458
+ print("nothing to accept: no merge wait recorded")
2459
+ return caddis_exit.CLEAN
2460
+ try:
2461
+ snap = caddis_roadmap.load(repo, roadmap)
2462
+ except caddis_roadmap.RoadmapError as exc:
2463
+ print(f"caddis_roadmap_run: {exc}", file=sys.stderr)
2464
+ return exc.code
2465
+ saved = _state_file(repo, roadmap, "published-roadmap").with_suffix(".md")
2466
+ before = _read_text(saved) or ""
2467
+ diff = "".join(difflib.unified_diff(before.splitlines(keepends=True), snap.text.splitlines(keepends=True),
2468
+ "published (baseline of the wait)", f"origin/{snap.branch} (now)"))
2469
+ print(diff or "no difference")
2470
+ if dry_run:
2471
+ print("dry run: nothing written")
2472
+ return caddis_exit.CLEAN
2473
+ if not yes and confirm("type accept to accept this roadmap as the baseline of the wait: ").strip() != "accept":
2474
+ print("not accepted; nothing written")
2475
+ return caddis_exit.BLOCKED
2476
+ saved.parent.mkdir(parents=True, exist_ok=True)
2477
+ tmp = saved.with_name(f"{saved.name}.{os.getpid()}.tmp")
2478
+ tmp.write_bytes(snap.text.encode("utf-8"))
2479
+ os.replace(tmp, saved)
2480
+ _write_record(waiting_file, {**waiting, "roadmap_sha": _roadmap_sha(snap.text),
2481
+ "accepted_by_owner_utc": _iso(datetime.now(timezone.utc))})
2482
+ print(f"roadmap {roadmap}: the origin roadmap is accepted as the baseline of the wait for {waiting.get('ref')}")
2483
+ return caddis_exit.CLEAN
2484
+ finally:
2485
+ if lock is not None:
2486
+ lock.release()
2487
+
2488
+
2489
+ def lock_state(repo: Path, roadmap: str) -> str:
2490
+ """`absent`, `stale` or `live`: the roadmap runner lock, judged with the lock's own `max_hours`."""
2491
+ stale = caddis_roadmap.runner_lock_stale(runner_lock_file(Path(repo), "roadmap", roadmap))
2492
+ return "absent" if stale is None else ("stale" if stale else "live")
2493
+
2494
+
2495
+ def main(argv: list[str] | None = None) -> int:
2496
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
2497
+ mode = parser.add_mutually_exclusive_group(required=True)
2498
+ mode.add_argument("--plan", help="run a plain plan: .caddis/plans/<slug>.md (plan mode)")
2499
+ mode.add_argument("--roadmap", help="run a roadmap: .caddis/roadmaps/<slug>/roadmap.md on origin (roadmap mode)")
2500
+ parser.add_argument("--repo", type=Path, default=Path("."))
2501
+ parser.add_argument("--max-hours", type=float, default=Caps.max_hours)
2502
+ parser.add_argument("--max-sessions", type=int, default=Caps.max_sessions)
2503
+ parser.add_argument("--session-timeout-min", type=float, default=Caps.session_timeout_min)
2504
+ parser.add_argument("--max-usd-per-session", type=float, default=Caps.max_usd_per_session)
2505
+ parser.add_argument("--max-usd-total", type=float, default=Caps.max_usd_total)
2506
+ parser.add_argument("--engine", choices=sorted(ENGINES), default="claude")
2507
+ parser.add_argument("--model", default=None)
2508
+ parser.add_argument("--dry-run", action="store_true")
2509
+ parser.add_argument("--gh-account", default=None,
2510
+ help="the gh account every gh call uses (GH_TOKEN from `gh auth token -u <account>`); give "
2511
+ "the repo's account on a machine with several gh logins")
2512
+ parser.add_argument("--lock-state", action="store_true",
2513
+ help="roadmap mode: print absent, stale or live for the runner lock; exit 0 unless live (3)")
2514
+ parser.add_argument("--restart-args", action="store_true",
2515
+ help="roadmap mode: print the roadmap-run.ps1 arguments of the last launch options, one per "
2516
+ "line (none when not recorded); exit 1 when they cannot be read")
2517
+ parser.add_argument("--accept-roadmap", action="store_true",
2518
+ help="roadmap mode, owner only (refused headless): accept the origin roadmap as it is now as "
2519
+ "the baseline of the current merge wait, after an owner edit during the wait")
2520
+ parser.add_argument("--yes", action="store_true", help="--accept-roadmap without the console confirmation")
2521
+ parser.add_argument("--last-stop", action="store_true",
2522
+ help="roadmap mode: exit 0 when the recheck task may restart the runner: no recorded "
2523
+ "stop, BLOCKED nothing ready, or BLOCKED waiting for merge once that PR merged (the "
2524
+ "line then starts with merged:); else 1 (running, any FAILED stop, a cap); prints why")
2525
+ args = parser.parse_args(argv)
2526
+ for name in ("max_hours", "max_sessions", "session_timeout_min", "max_usd_per_session", "max_usd_total"):
2527
+ value = getattr(args, name)
2528
+ if not (value > 0) or value != value or value == float("inf"):
2529
+ parser.error(f"--{name.replace('_', '-')} must be a finite number above 0")
2530
+ if args.lock_state:
2531
+ if not args.roadmap or not caddis_roadmap.SLUG_RE.match(args.roadmap):
2532
+ parser.error("--lock-state needs --roadmap <slug>")
2533
+ state = lock_state(args.repo, args.roadmap)
2534
+ print(state)
2535
+ return caddis_exit.NOT_RUN if state == "live" else caddis_exit.CLEAN
2536
+ if args.accept_roadmap:
2537
+ if not args.roadmap or not caddis_roadmap.SLUG_RE.match(args.roadmap):
2538
+ parser.error("--accept-roadmap needs --roadmap <slug>")
2539
+ return accept_roadmap(args.repo, args.roadmap, yes=args.yes, dry_run=args.dry_run)
2540
+ if args.restart_args:
2541
+ if not args.roadmap or not caddis_roadmap.SLUG_RE.match(args.roadmap):
2542
+ parser.error("--restart-args needs --roadmap <slug>")
2543
+ try:
2544
+ print("\n".join(restart_args(args.repo, args.roadmap)))
2545
+ except ValueError as exc:
2546
+ print(f"caddis_roadmap_run: {exc}", file=sys.stderr)
2547
+ return caddis_exit.BLOCKED
2548
+ return caddis_exit.CLEAN
2549
+ account = args.gh_account
2550
+ if args.last_stop and not account and args.roadmap and caddis_roadmap.SLUG_RE.match(args.roadmap):
2551
+ account = _read_record(options_file(last_stop_file(Path(args.repo), "roadmap", args.roadmap))).get(
2552
+ "gh_account") or None # the account the run recorded
2553
+ if account and args.last_stop:
2554
+ try:
2555
+ use_gh_account(account)
2556
+ except caddis_roadmap.RoadmapError as exc:
2557
+ print(f"gh error: {exc}")
2558
+ return caddis_exit.BLOCKED
2559
+ if args.last_stop:
2560
+ if not args.roadmap or not caddis_roadmap.SLUG_RE.match(args.roadmap):
2561
+ parser.error("--last-stop needs --roadmap <slug>")
2562
+ ok, why = last_stop_restartable(args.repo, args.roadmap)
2563
+ print(why)
2564
+ return caddis_exit.CLEAN if ok else caddis_exit.BLOCKED
2565
+ caps = Caps(args.max_hours, args.max_sessions, args.session_timeout_min, args.max_usd_per_session,
2566
+ args.max_usd_total)
2567
+ return run(repo=args.repo, plan=args.plan, roadmap=args.roadmap, caps=caps, engine=args.engine, model=args.model,
2568
+ dry_run=args.dry_run, gh_account=args.gh_account)
2569
+
2570
+
2571
+ if __name__ == "__main__":
2572
+ raise SystemExit(main())