@caddis/cli 0.4.52 → 0.4.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundles/antigravity-plugin/CADDIS-GLOSSARY.md +1 -0
- package/bundles/antigravity-plugin/plugin.json +1 -1
- package/bundles/antigravity-plugin/repo_freshness.py +132 -5
- package/bundles/antigravity-plugin/scripts/caddis_checkout_guard.py +223 -0
- package/bundles/antigravity-plugin/scripts/caddis_docs_pr.py +284 -0
- package/bundles/antigravity-plugin/scripts/caddis_exit.py +2 -0
- package/bundles/antigravity-plugin/scripts/caddis_frontmatter.py +1 -0
- package/bundles/antigravity-plugin/scripts/caddis_gate.py +226 -195
- package/bundles/antigravity-plugin/scripts/caddis_glossary.py +3 -3
- package/bundles/antigravity-plugin/scripts/caddis_governor.py +48 -4
- package/bundles/antigravity-plugin/scripts/caddis_inventory.py +38 -0
- package/bundles/antigravity-plugin/scripts/caddis_lanes.py +294 -127
- package/bundles/antigravity-plugin/scripts/caddis_lineage_publish.py +458 -0
- package/bundles/antigravity-plugin/scripts/caddis_merge_chain.py +237 -0
- package/bundles/antigravity-plugin/scripts/caddis_notify.py +121 -0
- package/bundles/antigravity-plugin/scripts/caddis_prod_check.py +380 -0
- package/bundles/antigravity-plugin/scripts/caddis_redact.py +99 -0
- package/bundles/antigravity-plugin/scripts/caddis_relay_git.py +2 -2
- package/bundles/antigravity-plugin/scripts/caddis_review.py +1309 -0
- package/bundles/antigravity-plugin/scripts/caddis_roadmap.py +2056 -0
- package/bundles/antigravity-plugin/scripts/caddis_roadmap_run.py +2572 -0
- package/bundles/antigravity-plugin/scripts/caddis_routing.py +3 -2
- package/bundles/antigravity-plugin/scripts/caddis_rows.py +476 -0
- package/bundles/antigravity-plugin/scripts/caddis_spawn.py +8 -9
- package/bundles/antigravity-plugin/scripts/caddis_spend.py +415 -0
- package/bundles/antigravity-plugin/scripts/caddis_tidy.py +357 -20
- package/bundles/antigravity-plugin/scripts/caddis_todo.py +55 -0
- package/bundles/antigravity-plugin/scripts/caddis_workstreams.py +178 -19
- package/bundles/antigravity-plugin/scripts/caddis_worktree_close.py +349 -0
- package/bundles/antigravity-plugin/scripts/check_doc_coverage.py +0 -56
- package/bundles/antigravity-plugin/scripts/claudster_doctor.py +1 -1
- package/bundles/antigravity-plugin/scripts/enforcement_inventory.py +0 -1
- package/bundles/antigravity-plugin/scripts/oss_review.py +2 -2
- package/bundles/antigravity-plugin/scripts/roadmap-recheck.ps1 +89 -0
- package/bundles/antigravity-plugin/scripts/roadmap-run.ps1 +88 -0
- package/bundles/antigravity-plugin/scripts/setup_project_ai.py +43 -1
- package/bundles/antigravity-plugin/session_end_agy.py +3 -3
- package/bundles/antigravity-plugin/skills/_registry.md +2 -6
- package/bundles/antigravity-plugin/skills/best-practices/agents/best-practices-referencer.md +1 -1
- package/bundles/antigravity-plugin/skills/best-practices/agents/codebase-context-builder.md +1 -1
- package/bundles/antigravity-plugin/skills/best-practices/agents/task-intent-analyzer.md +1 -1
- package/bundles/antigravity-plugin/skills/catchup/SKILL.md +7 -0
- package/bundles/antigravity-plugin/skills/cloud-phases/SKILL.md +1 -1
- package/bundles/antigravity-plugin/skills/cross-review/SKILL.md +5 -2
- package/bundles/antigravity-plugin/skills/feature-plan/SKILL.md +29 -83
- package/bundles/antigravity-plugin/skills/handoff/SKILL.md +77 -48
- package/bundles/antigravity-plugin/skills/implement/SKILL.md +87 -147
- package/bundles/antigravity-plugin/skills/migrate-dir/SKILL.md +1 -1
- package/bundles/antigravity-plugin/skills/mmr/SKILL.md +5 -2
- package/bundles/antigravity-plugin/skills/office-documents-official/SKILL.md +24 -0
- package/bundles/antigravity-plugin/skills/park/SKILL.md +0 -1
- package/bundles/antigravity-plugin/skills/pr-merge/SKILL.md +8 -14
- package/bundles/antigravity-plugin/skills/prd/SKILL.md +2 -10
- package/bundles/antigravity-plugin/skills/roadmap/SKILL.md +259 -0
- package/bundles/antigravity-plugin/skills/roadmap/assets/plan-resume-prompt.md +77 -0
- package/bundles/antigravity-plugin/skills/roadmap/assets/render-check.template.mjs +68 -0
- package/bundles/antigravity-plugin/skills/roadmap/assets/review-protocol.md +78 -0
- package/bundles/antigravity-plugin/skills/roadmap/assets/roadmap.template.md +32 -0
- package/bundles/antigravity-plugin/skills/roadmap/assets/value-check.template.py +161 -0
- package/bundles/antigravity-plugin/skills/ship/SKILL.md +14 -14
- package/bundles/antigravity-plugin/skills/ship-merge/SKILL.md +6 -0
- package/bundles/antigravity-plugin/skills/ship-pr/SKILL.md +33 -14
- package/bundles/antigravity-plugin/skills/spawn-session/SKILL.md +4 -8
- package/bundles/antigravity-plugin/warm_start_agy.py +1 -3
- package/bundles/antigravity-plugin-extras/plugin.json +1 -1
- package/bundles/antigravity-plugin-extras/skills/_registry.md +1 -2
- package/bundles/antigravity-plugin-extras/skills/agent-orchestration/SKILL.md +5 -5
- package/bundles/antigravity-plugin-extras/skills/cross-review/SKILL.md +5 -2
- package/bundles/antigravity-plugin-extras/skills/runner-move/SKILL.md +214 -0
- package/bundles/codex/.codex/skills/docs/office-documents-official/SKILL.md +24 -0
- package/bundles/codex/.codex/skills/workflow/best-practices/agents/best-practices-referencer.md +1 -1
- package/bundles/codex/.codex/skills/workflow/best-practices/agents/codebase-context-builder.md +1 -1
- package/bundles/codex/.codex/skills/workflow/best-practices/agents/task-intent-analyzer.md +1 -1
- package/bundles/codex/.codex/skills/workflow/roadmap/SKILL.md +259 -0
- package/bundles/codex/.codex/skills/workflow/roadmap/assets/plan-resume-prompt.md +77 -0
- package/bundles/codex/.codex/skills/workflow/roadmap/assets/render-check.template.mjs +68 -0
- package/bundles/codex/.codex/skills/workflow/roadmap/assets/review-protocol.md +78 -0
- package/bundles/codex/.codex/skills/workflow/roadmap/assets/roadmap.template.md +32 -0
- package/bundles/codex/.codex/skills/workflow/roadmap/assets/value-check.template.py +161 -0
- package/bundles/codex/AGENTS.md +5 -4
- package/bundles/manifest.json +3 -3
- package/package.json +1 -1
- package/bundles/antigravity-plugin/skills/advisory-hub/SKILL.md +0 -233
- package/bundles/antigravity-plugin/skills/advisory-hub/reference/context-template.md +0 -214
- package/bundles/antigravity-plugin/skills/advisory-hub/reference/hub-handbook.md +0 -233
- package/bundles/antigravity-plugin/skills/advisory-hub/reference/report-contract.md +0 -94
- package/bundles/antigravity-plugin/skills/advisory-hub/reference/succession.md +0 -118
- package/bundles/antigravity-plugin/skills/resume/SKILL.md +0 -6
- package/bundles/antigravity-plugin/skills/review/SKILL.md +0 -6
- package/bundles/antigravity-plugin/skills/spawn-hub/SKILL.md +0 -341
- package/bundles/antigravity-plugin/skills/statusline/SKILL.md +0 -6
- package/bundles/antigravity-plugin/skills/validate-phase/SKILL.md +0 -143
- package/bundles/antigravity-plugin-extras/skills/data-lineage/SKILL.md +0 -158
- package/bundles/antigravity-plugin-extras/skills/data-lineage/scripts/convert_doc_to_confluence.py +0 -1430
- package/bundles/antigravity-plugin-extras/skills/golden-plan/SKILL.md +0 -586
- package/bundles/codex/.codex/skills/workflow/advisory-hub/SKILL.md +0 -233
- package/bundles/codex/.codex/skills/workflow/advisory-hub/reference/context-template.md +0 -214
- package/bundles/codex/.codex/skills/workflow/advisory-hub/reference/hub-handbook.md +0 -233
- package/bundles/codex/.codex/skills/workflow/advisory-hub/reference/report-contract.md +0 -94
- package/bundles/codex/.codex/skills/workflow/advisory-hub/reference/succession.md +0 -118
|
@@ -0,0 +1,2572 @@
|
|
|
1
|
+
"""Runner: start one fresh headless session per batch, until a stop. Plan mode runs a plain plan; roadmap mode runs
|
|
2
|
+
the next batches of a roadmap.
|
|
3
|
+
|
|
4
|
+
Design: programme-method-design.md sections 8.1 to 8.5 and 9.2; caddis-next Phases 7 and 9. `roadmap-run.ps1` is a
|
|
5
|
+
thin shim over this file. The programme layer (programme folders, ledger, budget block) is gone (caddis-next Phase 7).
|
|
6
|
+
|
|
7
|
+
One session: stop checks (STOP file, session, hour and dollar caps); for agy, a reservation under the global spend cap
|
|
8
|
+
(`caddis_spend.py`, settled after the session); read the relay hash; launch ONE session through the governor with the
|
|
9
|
+
resume prompt and `CADDIS_HEADLESS=1` in its environment; parse its JSON result (cost, error); read the relay status
|
|
10
|
+
line; decide. CONTINUE with an unchanged relay is `FAILED no progress`; three sessions in a row with no newly closed
|
|
11
|
+
row is `BLOCKED stuck`.
|
|
12
|
+
|
|
13
|
+
Plan mode (`--plan <slug>`, decision D-I, design section 8.5) runs `.caddis/plans/<slug>.md`. A closed row in its
|
|
14
|
+
tracker or in `.caddis/open-work.md` is progress; the relay is `.caddis/relay/<slug>.md`; the log and STOP file are in
|
|
15
|
+
`.caddis/plans/.local/<slug>/`; the lock is `<git common dir>/caddis-runner/plan-<slug>/runner.lock`. The loop goes on only
|
|
16
|
+
at CONTINUE. Its lock holds the whole plan (`batch` `<slug>#*`). A plan that another live runner lock names is refused
|
|
17
|
+
(exit 3 at start, 1 at any later iteration: the check runs before every session).
|
|
18
|
+
|
|
19
|
+
Roadmap mode (`--roadmap <slug>`, caddis-next Phase 9) runs `.caddis/roadmaps/<slug>/roadmap.md` as it is on
|
|
20
|
+
`origin/<default>`; the log and STOP file are in `.caddis/roadmaps/<slug>/.local/`; the lock is
|
|
21
|
+
`<git common dir>/caddis-runner/roadmap-<slug>/runner.lock`. It refuses to start (exit 3) on a host that is not the roadmap's
|
|
22
|
+
`Runner host`. Each iteration: `caddis_roadmap.next_batch`; none is `BLOCKED nothing ready` (exit 1, the notification
|
|
23
|
+
carries the `status` line). `next` and the batch claim in the lock happen under one repo-level claim lock, so two
|
|
24
|
+
runners never choose the same batch. Found: print `next: <milestone> <plan>#<phase>`. A phase whose `**Lane:**` line
|
|
25
|
+
is not this engine (a cloud session, the owner, another tool), or that has no Lane line, is not run: the runner adds an
|
|
26
|
+
`owner-gate` blocker `phase <n> lane: <lane>` (`none` when there is no line) and goes on. Else a `todo` milestone is
|
|
27
|
+
set `in-progress`, the runner makes a fresh branch `caddis/roadmap/<slug>/<plan>-p<n>` from origin/<default>, and ONE
|
|
28
|
+
session runs `plan-resume-prompt.md` for that plan and phase on it. Sessions never push. A relay `STATUS: CONTINUE`
|
|
29
|
+
with new commits on that branch (still checked out, nothing under `.caddis/roadmaps/` changed): the RUNNER writes a
|
|
30
|
+
persistent claim for the phase, pushes the branch, opens a PR (`gh api`) and stops `BLOCKED waiting for merge
|
|
31
|
+
<plan>#<phase> <url>` (exit 1); `--last-stop` lets the recheck task restart it once the PR merged; the restart checks
|
|
32
|
+
the merged origin roadmap against the hash taken at publish time, removes the claim, rolls that milestone up and goes
|
|
33
|
+
on. Every roadmap runs its sessions in ONE dedicated worktree, `%TEMP%/caddis-roadmap-<slug>-<repo tag>` (made with
|
|
34
|
+
`git worktree add --detach` from origin/<default> on the first session, reused on restart, recorded in the runner
|
|
35
|
+
folder, never the `--repo` checkout); when the roadmap is finished the runner removes it if it is clean, and keeps it
|
|
36
|
+
(saying so in the stop line) if not. The caps (hours, sessions, USD per session and in total) are per run, in memory. CONTINUE with the phase already merged on
|
|
37
|
+
origin: `rollup` when every phase of the milestone is merged, then the next batch. A relay `STATUS: OWNER-GATE <r>`
|
|
38
|
+
becomes an `owner-gate` blocker, `STATUS: BLOCKED <r>` an `external` one, and the loop goes on.
|
|
39
|
+
|
|
40
|
+
Review gate (caddis-next Phase 11): a phase is pending review when its Tracker row is done in the working copy of
|
|
41
|
+
the plan (its path, or `.caddis/plans/done/` after tidy moved it), not done on origin/<default>, marked
|
|
42
|
+
`**Risk:** high` in any copy (working, origin, the copy from before the session; `caddis_review.phase_risk`), and not
|
|
43
|
+
recorded clean in this runner's own `<git common dir>/caddis-runner/<kind>-<slug>/reviewed/<plan>.json`
|
|
44
|
+
(phase -> head, utc, result; another runner's record does not count). Plan mode
|
|
45
|
+
reviews every pending phase before each session (so a restart still reviews what an earlier run left); roadmap mode
|
|
46
|
+
reviews the plan's pending phases BEFORE it pushes and opens the PR, so a `risk: high` phase is never published
|
|
47
|
+
unreviewed. Pending is recomputed after every review and every fix session. Each review is `caddis_review.py --plan
|
|
48
|
+
<plan> --phase <n> --runs 2 --out-dir <local>/reviews/<plan>`, a child with `CADDIS_HEADLESS=1` (the `review_cmd`
|
|
49
|
+
seam). Exit 0 records the phase clean. Exit 1: ONE fix session for that phase (a session against the session cap and
|
|
50
|
+
the USD caps) with the prompt `Read <out-dir>/review-<n>-merged.md and fix every blocking finding
|
|
51
|
+
...`, then one more review; exit 1 again stops `BLOCKED review`. Exit 3, 4, any other code, a timeout, exit 1 with
|
|
52
|
+
no merged file, a plan that cannot be read, or a done row with no `### Phase <n>` heading stops `BLOCKED review not
|
|
53
|
+
run`. Neither stop is restartable by the recheck task; an owner restart reviews the phase again. The review's
|
|
54
|
+
`review-<n>-cost.json` `total_cost_usd` is added to the spend; a missing or unreadable file adds USD 10 (USD 5 per
|
|
55
|
+
run, 2 runs). A review that could take the spend over the total cap does not start (a cap stop). The
|
|
56
|
+
review child (the reviewer has full Bash) and the fix session are each guarded by the runner-state check like any
|
|
57
|
+
session; the runner writes `reviewed/<plan>.json` only between children, before the next child's snapshot.
|
|
58
|
+
|
|
59
|
+
Snapshot guard: one baseline across the whole run. The Milestones and Blockers tables (every cell of every row, rows
|
|
60
|
+
added, removed or moved) and the `Runner host` line are frozen. Every origin read (each `next`, and the reads after
|
|
61
|
+
the runner's own writes and after each session) is compared with the previous read: the only differences allowed are
|
|
62
|
+
the exact edits the runner made and recorded for that read (a Status it set, a blocker row it added, and what a
|
|
63
|
+
rollup's audit entry says it actually wrote). Anything else stops the run `FAILED roadmap changed` (exit 1). Every
|
|
64
|
+
roadmap change goes through `caddis_roadmap.py` as a child process with `CADDIS_HEADLESS=1`; the runner never passes
|
|
65
|
+
`--by-owner`.
|
|
66
|
+
|
|
67
|
+
Runner state: one folder per runner, `<git common dir>/caddis-runner/<roadmap|plan>-<slug>/` (lock, last stop,
|
|
68
|
+
options, branches, worktree, saved roadmap text); shared at the top level: `claims/` and `claim.lock`. Sessions
|
|
69
|
+
are cooperative (our own sessions): the runner guards against mistakes and runaway spend, not against a hostile
|
|
70
|
+
session. Around every session it hashes its own folder and its own claims; a change there is `FAILED runner state
|
|
71
|
+
changed`. Another runner's writes never stop it. A restart checks that the claim its last stop records
|
|
72
|
+
is still there, unchanged (`FAILED own claim missing or changed`).
|
|
73
|
+
|
|
74
|
+
Lock: `<git common dir>/caddis-runner/<roadmap|plan>-<slug>/runner.lock` (every worktree sees it; outside a git repo,
|
|
75
|
+
`<local>/runner.lock`) holds `{"host", "pid", "started_utc", "batch", "max_hours"}`. On this host a lock is stale when
|
|
76
|
+
its pid is not running, or the process now holding that pid started after `started_utc` (pid reuse after a reboot;
|
|
77
|
+
when the start time cannot be read, live). Another host's lock is stale when `started_utc` is older than the lock's
|
|
78
|
+
own `max_hours` plus 1 h. A stale lock is removed with a toast and the run goes on; a live lock is exit 3. The runner
|
|
79
|
+
that holds the lock records `running` as its last stop at once, with its launch options, and its real last stop at the
|
|
80
|
+
end; an unexpected exception records `FAILED crash: <type>`, notifies once and re-raises. `roadmap-recheck.ps1`
|
|
81
|
+
restarts the runner only after `BLOCKED nothing ready`, a merged `BLOCKED waiting for merge`, or no recorded stop
|
|
82
|
+
(`--last-stop`), with the recorded options clamped to the built-in caps (`--restart-args`). Every lock write is a temp
|
|
83
|
+
file in the same folder, then a hard link (create) or os.replace.
|
|
84
|
+
|
|
85
|
+
On every stop: one Windows toast, and the push-notification fallback when the last session did not report `PUSH-SENT`
|
|
86
|
+
(not for a STOP file, a lock or a usage limit). One JSON line per session and one per stop go to `<local>/runner.jsonl`.
|
|
87
|
+
Session output, relay text and every log line, printed stop line, toast and push pass through `caddis_redact` first.
|
|
88
|
+
|
|
89
|
+
The runner never passes an owner gate, never raises a cap and never deletes a STOP file.
|
|
90
|
+
|
|
91
|
+
Exit codes: 0 stopped at a status the session wrote (plan mode: OWNER-GATE, BLOCKED), or at a STOP file after a
|
|
92
|
+
session; 1 a FAILED stop, a cap, the agy spend cap, a usage limit, `BLOCKED stuck`, `BLOCKED nothing ready`,
|
|
93
|
+
`BLOCKED waiting for merge`, `BLOCKED review`, `BLOCKED review not run`, or the snapshot guard; 3 never started (lock, STOP file, no governor, agy spend cap not configured,
|
|
94
|
+
another host, a plan a roadmap runner holds); 4 bad input (no plan or roadmap, both, a bad slug, no prompt template,
|
|
95
|
+
bad engine).
|
|
96
|
+
|
|
97
|
+
Engines: `claude` is built and live-tested. `agy` and `codex` are command-line adapters with unit tests only (not
|
|
98
|
+
live-tested; design 8.2).
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
from __future__ import annotations
|
|
102
|
+
|
|
103
|
+
import argparse
|
|
104
|
+
import contextlib
|
|
105
|
+
import difflib
|
|
106
|
+
import hashlib
|
|
107
|
+
import json
|
|
108
|
+
import os
|
|
109
|
+
import re
|
|
110
|
+
import socket
|
|
111
|
+
import subprocess
|
|
112
|
+
import sys
|
|
113
|
+
import tempfile
|
|
114
|
+
import time
|
|
115
|
+
import uuid
|
|
116
|
+
from collections import Counter
|
|
117
|
+
from dataclasses import dataclass, field
|
|
118
|
+
from datetime import datetime, timezone
|
|
119
|
+
from pathlib import Path
|
|
120
|
+
from typing import Callable, Sequence
|
|
121
|
+
|
|
122
|
+
_HERE = Path(__file__).resolve().parent
|
|
123
|
+
try:
|
|
124
|
+
import caddis_exit
|
|
125
|
+
except ModuleNotFoundError as exc:
|
|
126
|
+
if exc.name != "caddis_exit":
|
|
127
|
+
raise
|
|
128
|
+
sys.path.insert(0, str(_HERE))
|
|
129
|
+
sys.path.insert(0, str(_HERE.parent / "claude-harness" / "scripts"))
|
|
130
|
+
import caddis_exit
|
|
131
|
+
|
|
132
|
+
import caddis_notify # noqa: E402
|
|
133
|
+
import caddis_redact # noqa: E402
|
|
134
|
+
import caddis_review # noqa: E402
|
|
135
|
+
import caddis_roadmap # noqa: E402
|
|
136
|
+
import caddis_spend # noqa: E402
|
|
137
|
+
import caddis_workstreams # noqa: E402
|
|
138
|
+
|
|
139
|
+
ENGINES: dict[str, dict] = {
|
|
140
|
+
"claude": {"live_tested": True, "default_model": "opus", "limit_key": "claude"},
|
|
141
|
+
"agy": {"live_tested": False, "default_model": "gemini-4-argon-medium", "limit_key": "agy"},
|
|
142
|
+
"codex": {"live_tested": False, "default_model": "gpt-5", "limit_key": "codex-sol"},
|
|
143
|
+
}
|
|
144
|
+
PROMPT_ON_STDIN = {"claude"}
|
|
145
|
+
STUCK_SESSIONS = 3
|
|
146
|
+
PLAN_PROMPT_NAME = "plan-resume-prompt.md"
|
|
147
|
+
HEADLESS_ENV = {"CADDIS_HEADLESS": "1"}
|
|
148
|
+
ROADMAP_WRITE_TIMEOUT_S = caddis_roadmap.MERGE_CHAIN_TIMEOUT + 600 # a backstop: every call inside is bounded
|
|
149
|
+
RUNNING = "running" # the last-stop record while a runner holds the lock; a killed run leaves it (needs the owner)
|
|
150
|
+
GH_STOP_PREFIX = "BLOCKED gh cannot see" # a gh error at a restart: waiting.json stays, a later tick may resume
|
|
151
|
+
RESTARTABLE_STOP = "BLOCKED nothing ready" # restartable once a blocker cleared
|
|
152
|
+
WAITING_PREFIX = "BLOCKED waiting for merge" # restartable once its PR merged
|
|
153
|
+
ROLLUP_BLOCK_RE = r"exit check failing \((exit -?\d+|timed out after \d+ s)\)" # the blocker rollup writes
|
|
154
|
+
_SLUG_RE = re.compile(r"[a-z0-9][a-z0-9-]*")
|
|
155
|
+
_CLOSED = {"done", "not-needed"}
|
|
156
|
+
_LANE_RE = re.compile(r"^\s*\*\*Lane:\*\*\s*(.*?)\s*$")
|
|
157
|
+
_PHASE_HEAD_RE = re.compile(r"^ {0,3}#{2,4}\s+Phase\s+(\d+)\b", re.IGNORECASE)
|
|
158
|
+
_HEAD_RE = re.compile(r"^ {0,3}#{1,6}\s")
|
|
159
|
+
_LANE_TOOL_RE = re.compile(r"^\W*([A-Za-z][A-Za-z0-9_-]*)")
|
|
160
|
+
_NOT_RUNNER_LANE_RE = re.compile(r"cloud|owner", re.IGNORECASE)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@dataclass
|
|
164
|
+
class Caps:
|
|
165
|
+
max_hours: float = 10.0
|
|
166
|
+
max_sessions: int = 12
|
|
167
|
+
session_timeout_min: float = 120.0
|
|
168
|
+
max_usd_per_session: float = 25.0
|
|
169
|
+
max_usd_total: float = 150.0
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _usd(value: float) -> str:
|
|
173
|
+
return f"{round(value, 2):g}"
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def headless_env(base: dict | None = None) -> dict:
|
|
177
|
+
"""A copy of the environment with CADDIS_HEADLESS=1: headless detection cannot see a process started in a fresh
|
|
178
|
+
console, so every session and every caddis_roadmap.py call the runner starts carries the marker."""
|
|
179
|
+
env = dict(os.environ if base is None else base)
|
|
180
|
+
env.update(HEADLESS_ENV)
|
|
181
|
+
return env
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def build_command(engine: str, model: str, prompt: str, *, usd: float, timeout_min: float,
|
|
185
|
+
governor: Sequence[str], engine_cmd: Sequence[str] | None = None) -> list[str]:
|
|
186
|
+
"""The full command line for one session: the governor prefix, then the engine's print-mode call.
|
|
187
|
+
|
|
188
|
+
`claude` gets its prompt on stdin, not as an argument (PROMPT_ON_STDIN): on Windows `claude` is an npm `.cmd`
|
|
189
|
+
shim, and cmd.exe cuts a multi-line argument at its first newline, which drops the rest of the prompt and every
|
|
190
|
+
argument after it."""
|
|
191
|
+
if engine not in ENGINES:
|
|
192
|
+
raise ValueError(f"unknown engine {engine!r}; known: {', '.join(ENGINES)}")
|
|
193
|
+
exe = list(engine_cmd) if engine_cmd else [engine]
|
|
194
|
+
if engine == "claude":
|
|
195
|
+
tail = [*exe, "-p", "--model", model, "--output-format", "json", "--max-budget-usd", _usd(usd)]
|
|
196
|
+
elif engine == "agy": # probe 5: --model must come before --print
|
|
197
|
+
tail = [*exe, "--model", model, "--print", prompt, "--output-format", "json",
|
|
198
|
+
"--print-timeout", f"{max(1, int(timeout_min))}m"]
|
|
199
|
+
else:
|
|
200
|
+
tail = [*exe, "exec", "--model", model, "--json", prompt]
|
|
201
|
+
return [*governor, *tail]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def default_governor() -> list[str] | None:
|
|
205
|
+
"""`caddis_governor.py exec --scope session --`, preferring the admin-installed copy (as caddis-run.ps1 does)."""
|
|
206
|
+
installed = Path(os.environ.get("ProgramData", r"C:\ProgramData")) / "caddis" / "caddis_governor.py"
|
|
207
|
+
for gov in (installed, _HERE / "caddis_governor.py", _HERE.parent / "claude-harness" / "scripts" / "caddis_governor.py"):
|
|
208
|
+
if gov.is_file():
|
|
209
|
+
return [sys.executable, str(gov), "exec", "--scope", "session", "--"]
|
|
210
|
+
return None
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def default_roadmap_cmd() -> list[str]:
|
|
214
|
+
return [sys.executable, str(_HERE / "caddis_roadmap.py")]
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def find_prompt_template(name: str = PLAN_PROMPT_NAME) -> Path | None:
|
|
218
|
+
for assets in (_HERE.parent / ".github" / "skills" / "workflow" / "roadmap" / "assets",
|
|
219
|
+
_HERE.parent / "skills" / "roadmap" / "assets"):
|
|
220
|
+
if (assets / name).is_file():
|
|
221
|
+
return assets / name
|
|
222
|
+
return None
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _local_dir(repo: Path, kind: str, slug: str) -> Path:
|
|
226
|
+
"""The runner's local folder: plan `.caddis/plans/.local/<slug>`, roadmap `.caddis/roadmaps/<slug>/.local`."""
|
|
227
|
+
if kind == "plan":
|
|
228
|
+
return Path(repo) / ".caddis" / "plans" / ".local" / slug
|
|
229
|
+
if kind == "roadmap":
|
|
230
|
+
return Path(repo) / ".caddis" / "roadmaps" / slug / ".local"
|
|
231
|
+
raise ValueError(f"unknown target kind {kind!r}")
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _ensure_local(path: Path) -> Path:
|
|
235
|
+
"""Create the local folder. A plan's `.caddis/plans/.local/` and a roadmap's `.local/` get a `.gitignore` of `*`
|
|
236
|
+
when they have none (the repo's own `.gitignore` may not cover them)."""
|
|
237
|
+
path = Path(path)
|
|
238
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
239
|
+
if path.parent.name == ".local" and path.parent.parent.name == "plans":
|
|
240
|
+
gitignore = path.parent / ".gitignore"
|
|
241
|
+
elif path.name == ".local" and path.parent.parent.name == "roadmaps":
|
|
242
|
+
gitignore = path / ".gitignore"
|
|
243
|
+
else:
|
|
244
|
+
return path
|
|
245
|
+
if not gitignore.exists():
|
|
246
|
+
gitignore.write_text("*\n", encoding="utf-8")
|
|
247
|
+
return path
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@dataclass
|
|
251
|
+
class Target:
|
|
252
|
+
"""What the runner runs: a plain plan file (plan mode) or a roadmap (roadmap mode)."""
|
|
253
|
+
kind: str # "plan" | "roadmap"
|
|
254
|
+
slug: str
|
|
255
|
+
local: Path # lock, runner.jsonl, STOP
|
|
256
|
+
progress: tuple[Path, ...] # plan mode: files whose closed rows count as progress
|
|
257
|
+
prompt_name: str
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def resolve_target(repo: Path, plan: str | None) -> Target:
|
|
261
|
+
"""The plan-mode target; ValueError when the slug is missing or bad, or its plan file is missing."""
|
|
262
|
+
slug = plan
|
|
263
|
+
if not isinstance(slug, str) or not _SLUG_RE.fullmatch(slug):
|
|
264
|
+
raise ValueError(f"bad plan {slug!r}: invalid slug: {slug!r}")
|
|
265
|
+
plan_file = repo / ".caddis" / "plans" / f"{slug}.md"
|
|
266
|
+
if not plan_file.is_file():
|
|
267
|
+
raise ValueError(f"no plan {slug!r}: {plan_file} does not exist")
|
|
268
|
+
return Target("plan", slug, _local_dir(repo, "plan", slug), (plan_file, repo / ".caddis" / "open-work.md"),
|
|
269
|
+
PLAN_PROMPT_NAME)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def render_prompt(template: Path, slug: str, phase: int | None = None) -> str:
|
|
273
|
+
"""The template body with `{{slug}}`, `{{phase}}` (empty when None) and `{{scripts}}` (this scripts folder: an
|
|
274
|
+
app repo has no caddis `scripts/`) filled in."""
|
|
275
|
+
text = template.read_text(encoding="utf-8")
|
|
276
|
+
_meta, body = caddis_workstreams.split_document(text)
|
|
277
|
+
for key, value in (("{{slug}}", slug), ("{{phase}}", "" if phase is None else str(phase)),
|
|
278
|
+
("{{scripts}}", _HERE.as_posix())):
|
|
279
|
+
body = body.replace(key, value)
|
|
280
|
+
return body.strip() + "\n"
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# -- open work: closed rows -------------------------------------------------------------------------------------------
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def closed_rows(open_work: Path) -> int:
|
|
287
|
+
"""Rows whose `Status` column (by name) is done or not-needed, over every table of the file."""
|
|
288
|
+
try:
|
|
289
|
+
lines = open_work.read_text(encoding="utf-8").splitlines()
|
|
290
|
+
except OSError:
|
|
291
|
+
return 0
|
|
292
|
+
count, status_col = 0, None
|
|
293
|
+
for line in lines:
|
|
294
|
+
s = line.strip()
|
|
295
|
+
if not s.startswith("|"):
|
|
296
|
+
status_col = None
|
|
297
|
+
continue
|
|
298
|
+
cells = [c.strip() for c in s.strip("|").split("|")]
|
|
299
|
+
if status_col is None:
|
|
300
|
+
status_col = cells.index("Status") if "Status" in cells else -1
|
|
301
|
+
continue
|
|
302
|
+
if status_col >= 0 and status_col < len(cells) and cells[status_col].lower() in _CLOSED:
|
|
303
|
+
count += 1
|
|
304
|
+
return count
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
# -- session output ---------------------------------------------------------------------------------------------------
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def parse_result(stdout: str) -> dict | None:
|
|
311
|
+
text = (stdout or "").strip()
|
|
312
|
+
candidates = [text] + [line for line in reversed(text.splitlines()) if line.lstrip().startswith("{")]
|
|
313
|
+
for candidate in candidates:
|
|
314
|
+
try:
|
|
315
|
+
value = json.loads(candidate)
|
|
316
|
+
except ValueError:
|
|
317
|
+
continue
|
|
318
|
+
if isinstance(value, dict):
|
|
319
|
+
return value
|
|
320
|
+
return None
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _agy_engine_model(repo: Path) -> str:
|
|
324
|
+
"""`<agy model>-medium`: the `[lanes.models] agy` pin in `<repo>/.caddis/config.toml` when set, else the default
|
|
325
|
+
agy model as `agy models` offers it here (see caddis_lanes.resolve_models)."""
|
|
326
|
+
try:
|
|
327
|
+
try:
|
|
328
|
+
import caddis_lanes
|
|
329
|
+
except ModuleNotFoundError:
|
|
330
|
+
sys.path.insert(0, str(_HERE))
|
|
331
|
+
import caddis_lanes
|
|
332
|
+
return caddis_lanes.resolve_models(caddis_lanes.load_lanes_config(repo))["agy"] + "-medium"
|
|
333
|
+
except Exception: # the lane engine is missing or the config is bad: the display default
|
|
334
|
+
return ENGINES["agy"]["default_model"]
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def usage_limited(engine: str, text: str) -> bool:
|
|
338
|
+
try:
|
|
339
|
+
import caddis_lanes
|
|
340
|
+
patterns = caddis_lanes.BUDGET_PATTERNS.get(ENGINES[engine]["limit_key"], ())
|
|
341
|
+
except Exception: # the lane engine is missing: use the Claude wording only
|
|
342
|
+
patterns = (r"You[’']ve hit your (?:session|weekly|Opus|Sonnet|Fable|usage credit) limit\b",)
|
|
343
|
+
return any(re.search(p, text, re.IGNORECASE) for p in patterns)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _read_relay(repo: Path, slug: str) -> tuple[str, str | None]:
|
|
347
|
+
"""The plan's relay text and hash; an unsafe relay path (a symlink, an escape) counts as no relay, never a crash."""
|
|
348
|
+
try:
|
|
349
|
+
return caddis_workstreams.read_relay(repo, slug)
|
|
350
|
+
except (ValueError, OSError) as exc: # OSError: the relay path is a folder or locked - still no crash
|
|
351
|
+
print(f"caddis_roadmap_run: relay not read: {exc}", file=sys.stderr)
|
|
352
|
+
return "", None
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _cost(result: dict | None) -> float:
|
|
356
|
+
value = (result or {}).get("total_cost_usd", 0)
|
|
357
|
+
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) and value >= 0 else 0.0
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
# -- roadmap mode helpers ---------------------------------------------------------------------------------------------
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def phase_lane(plan_text: str, phase: int) -> str | None:
|
|
364
|
+
"""The `**Lane:**` text of the `Phase <n>` heading's section in a plan (until the next heading of the same or a
|
|
365
|
+
higher level), or None when the phase has no Lane line. Fenced code is skipped."""
|
|
366
|
+
level = 0 # the open phase heading's level; 0 = not inside the phase
|
|
367
|
+
for line in caddis_roadmap._mask_fences(plan_text.splitlines()):
|
|
368
|
+
head = _HEAD_RE.match(line)
|
|
369
|
+
if head:
|
|
370
|
+
this_level = len(line.lstrip(" ")) - len(line.lstrip(" ").lstrip("#"))
|
|
371
|
+
phase_head = _PHASE_HEAD_RE.match(line)
|
|
372
|
+
if phase_head:
|
|
373
|
+
level = this_level if int(phase_head.group(1)) == phase else 0
|
|
374
|
+
elif level and this_level <= level:
|
|
375
|
+
level = 0
|
|
376
|
+
continue
|
|
377
|
+
if level:
|
|
378
|
+
match = _LANE_RE.match(line)
|
|
379
|
+
if match:
|
|
380
|
+
return match.group(1)
|
|
381
|
+
return None
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def lane_runs_here(lane: str, engine: str) -> bool:
|
|
385
|
+
"""True when the runner may run a phase with this Lane: its first word names this engine, and it names neither
|
|
386
|
+
a cloud session nor the owner. Anything else (another tool, no tool named) is not the runner's (fail closed)."""
|
|
387
|
+
if _NOT_RUNNER_LANE_RE.search(lane):
|
|
388
|
+
return False
|
|
389
|
+
match = _LANE_TOOL_RE.match(lane)
|
|
390
|
+
return bool(match) and match.group(1).lower() == engine
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _milestone_cells(ms: "caddis_roadmap.Milestone") -> list:
|
|
394
|
+
return [ms.id, ms.name, list(ms.phases), list(ms.depends), ms.exit_check, ms.status, ms.owner, ms.target]
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def roadmap_guard(before: "caddis_roadmap.Roadmap", after: "caddis_roadmap.Roadmap",
|
|
398
|
+
edits: Sequence[tuple] = ()) -> list[str]:
|
|
399
|
+
"""Every difference between two origin reads that the runner's own recorded `edits` do not explain.
|
|
400
|
+
|
|
401
|
+
The Milestones and Blockers tables and the `Runner host` line are frozen: every cell of every milestone row, the
|
|
402
|
+
row order, and the multiset of blocker rows. `edits` are what the runner itself wrote since `before`:
|
|
403
|
+
`("status", mid, value)`, `("blocker", mid, type, reason)` (a row with no Check; nothing when the same
|
|
404
|
+
milestone, type and reason exist, as `caddis_roadmap.add_blocker` does), and `("blocker-like", mid, type,
|
|
405
|
+
regex)` (one added row whose reason matches, for the blocker a rollup writes)."""
|
|
406
|
+
cols = caddis_roadmap.MILESTONE_COLS
|
|
407
|
+
rows = {ms.id: _milestone_cells(ms) for ms in before.milestones}
|
|
408
|
+
blockers = Counter(b.key for b in before.blockers)
|
|
409
|
+
patterns: list[tuple[str, str, str]] = []
|
|
410
|
+
for edit in edits:
|
|
411
|
+
if edit[0] == "status" and edit[1] in rows:
|
|
412
|
+
rows[edit[1]][cols.index("Status")] = edit[2]
|
|
413
|
+
elif edit[0] == "blocker":
|
|
414
|
+
_kind, mid, btype, reason = edit
|
|
415
|
+
if not any(k[0] == mid and k[1] == btype and k[2] == reason for k in blockers):
|
|
416
|
+
blockers[(mid, btype, reason, "")] += 1
|
|
417
|
+
elif edit[0] == "blocker-like":
|
|
418
|
+
patterns.append(tuple(edit[1:]))
|
|
419
|
+
changes: list[str] = []
|
|
420
|
+
if after.runner_host != before.runner_host:
|
|
421
|
+
changes.append("Runner host changed")
|
|
422
|
+
now = {ms.id: _milestone_cells(ms) for ms in after.milestones}
|
|
423
|
+
changes += [f"{mid} milestone removed" for mid in rows if mid not in now]
|
|
424
|
+
changes += [f"{mid} milestone added" for mid in now if mid not in rows]
|
|
425
|
+
for mid in (m for m in rows if m in now):
|
|
426
|
+
changed = [col for col, old, new in zip(cols, rows[mid], now[mid]) if old != new]
|
|
427
|
+
if changed:
|
|
428
|
+
changes.append(f"{mid} {', '.join(changed)} changed")
|
|
429
|
+
kept_before = [ms.id for ms in before.milestones if ms.id in now]
|
|
430
|
+
kept_after = [ms.id for ms in after.milestones if ms.id in rows]
|
|
431
|
+
if kept_before != kept_after:
|
|
432
|
+
changes.append("milestone rows reordered")
|
|
433
|
+
found = Counter(b.key for b in after.blockers)
|
|
434
|
+
for key, n in sorted((blockers - found).items()):
|
|
435
|
+
changes.append(f"{key[0]} {key[1]} blocker removed ({key[2]})" + (f" x{n}" if n > 1 else ""))
|
|
436
|
+
for key, n in sorted((found - blockers).items()):
|
|
437
|
+
for _ in range(n):
|
|
438
|
+
match = next((p for p in patterns if p[0] == key[0] and p[1] == key[1] and not key[3]
|
|
439
|
+
and re.fullmatch(p[2], key[2])), None)
|
|
440
|
+
if match is not None:
|
|
441
|
+
patterns.remove(match)
|
|
442
|
+
else:
|
|
443
|
+
changes.append(f"{key[0]} {key[1]} blocker added ({key[2]})")
|
|
444
|
+
return changes
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _done_phases(snap: "caddis_roadmap.Snapshot", plan: str) -> int:
|
|
448
|
+
rows = snap.phases.tracker(plan) or {}
|
|
449
|
+
return sum(1 for cell in rows.values() if caddis_roadmap._is_done_status(cell))
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _plan_text(snap: "caddis_roadmap.Snapshot", plan: str) -> str:
|
|
453
|
+
for path in (f".caddis/plans/{plan}.md", f".caddis/plans/done/{plan}.md"):
|
|
454
|
+
text = caddis_roadmap.show(snap.root, snap.branch, path)
|
|
455
|
+
if text is not None:
|
|
456
|
+
return text
|
|
457
|
+
return ""
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _status_line(repo: Path, slug: str) -> str:
|
|
461
|
+
try:
|
|
462
|
+
return caddis_roadmap.status(repo, slug)["line"]
|
|
463
|
+
except Exception as exc: # the stop must still notify
|
|
464
|
+
return f"status not read: {type(exc).__name__}"
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
# -- the lock ---------------------------------------------------------------------------------------------------------
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _tmp_beside(path: Path, text: str) -> Path:
|
|
471
|
+
tmp = path.with_name(f"{path.name}.{os.getpid()}-{uuid.uuid4().hex}.tmp")
|
|
472
|
+
tmp.write_text(text, encoding="utf-8")
|
|
473
|
+
return tmp
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _create_with(path: Path, text: str) -> None:
|
|
477
|
+
"""Create `path` holding `text` in one step (a reader never sees it empty); FileExistsError when it exists.
|
|
478
|
+
A temp file in the same folder is hard-linked to the name (the link fails when the name exists)."""
|
|
479
|
+
tmp = _tmp_beside(path, text)
|
|
480
|
+
try:
|
|
481
|
+
os.link(tmp, path)
|
|
482
|
+
except FileExistsError:
|
|
483
|
+
raise
|
|
484
|
+
except OSError: # no hard links on this file system: O_EXCL, then the text
|
|
485
|
+
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
486
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
487
|
+
fh.write(text)
|
|
488
|
+
finally:
|
|
489
|
+
tmp.unlink(missing_ok=True)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _replace_with(path: Path, text: str) -> None:
|
|
493
|
+
"""Rewrite `path` with a temp file in the same folder and os.replace (a reader sees the old or the new text)."""
|
|
494
|
+
tmp = _tmp_beside(path, text)
|
|
495
|
+
try:
|
|
496
|
+
os.replace(tmp, path)
|
|
497
|
+
finally:
|
|
498
|
+
tmp.unlink(missing_ok=True)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
class _RunnerLock:
|
|
502
|
+
"""The runner lock: JSON `{"host", "pid", "started_utc", "batch", "max_hours"}`, created with O_EXCL, in
|
|
503
|
+
`<git common dir>/caddis-runner/<kind>-<slug>/` (every worktree sees it; outside a git repo, the local folder).
|
|
504
|
+
|
|
505
|
+
A stale lock (caddis_roadmap.runner_lock_stale: judged with the `max_hours` the lock itself records) is taken over
|
|
506
|
+
race-safely: rename it to a unique name (only one contender's rename succeeds), check the renamed file still
|
|
507
|
+
holds what was judged stale, then create the lock."""
|
|
508
|
+
|
|
509
|
+
def __init__(self, path: Path, max_hours: float, batch: str = "", claim_path: Path | None = None):
|
|
510
|
+
self.path, self.max_hours = Path(path), max_hours
|
|
511
|
+
self.claim_path = claim_path # takeovers of a stale lock happen under this repo-level lock
|
|
512
|
+
self.text: str | None = None
|
|
513
|
+
self.holder = {"host": socket.gethostname(), "pid": os.getpid(),
|
|
514
|
+
"started_utc": _iso(datetime.now(timezone.utc)), "batch": batch, "max_hours": max_hours}
|
|
515
|
+
|
|
516
|
+
def acquire(self, on_stale: Callable[[str], None], claim_held: bool = False) -> None:
|
|
517
|
+
"""`claim_held`: the caller already holds claim.lock (plan mode): the stale takeover must not take it again."""
|
|
518
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
519
|
+
for attempt in range(3):
|
|
520
|
+
text = json.dumps(self.holder)
|
|
521
|
+
try:
|
|
522
|
+
_create_with(self.path, text)
|
|
523
|
+
except FileExistsError:
|
|
524
|
+
if attempt < 2 and self._take_stale(on_stale, claim_held):
|
|
525
|
+
continue
|
|
526
|
+
raise _Stop(f"BLOCKED another runner holds {self.path} (live; delete it only when no runner runs)",
|
|
527
|
+
caddis_exit.NOT_RUN, push=False) from None
|
|
528
|
+
self.text = text
|
|
529
|
+
return
|
|
530
|
+
raise _Stop(f"BLOCKED another runner holds {self.path}", caddis_exit.NOT_RUN, push=False)
|
|
531
|
+
|
|
532
|
+
def _take_stale(self, on_stale: Callable[[str], None], claim_held: bool = False) -> bool:
|
|
533
|
+
if self.claim_path is not None: # serialised: of several contenders on one stale lock, one takes it over
|
|
534
|
+
try:
|
|
535
|
+
with (contextlib.nullcontext() if claim_held
|
|
536
|
+
else caddis_roadmap._Lock(self.claim_path, timeout=caddis_roadmap.CLAIM_TIMEOUT)):
|
|
537
|
+
try:
|
|
538
|
+
judged = self.path.read_bytes()
|
|
539
|
+
except FileNotFoundError:
|
|
540
|
+
return True
|
|
541
|
+
stale = caddis_roadmap.runner_lock_stale(self.path)
|
|
542
|
+
if stale is None:
|
|
543
|
+
return True # gone between the read and the judgement: free, try the create again
|
|
544
|
+
if not stale:
|
|
545
|
+
return False # live: a contender before us won
|
|
546
|
+
self.path.unlink(missing_ok=True)
|
|
547
|
+
except caddis_roadmap.RoadmapError:
|
|
548
|
+
return False
|
|
549
|
+
on_stale(f"removed a stale runner lock {self.path}: {judged.decode('utf-8', 'replace')[:200]}")
|
|
550
|
+
return True
|
|
551
|
+
try:
|
|
552
|
+
judged = self.path.read_bytes()
|
|
553
|
+
except FileNotFoundError:
|
|
554
|
+
return True # gone: try to create it again
|
|
555
|
+
except OSError:
|
|
556
|
+
return False
|
|
557
|
+
stale = caddis_roadmap.runner_lock_stale(self.path)
|
|
558
|
+
if stale is None:
|
|
559
|
+
return True
|
|
560
|
+
if not stale:
|
|
561
|
+
return False
|
|
562
|
+
grave = self.path.with_name(f"{self.path.name}.stale-{os.getpid()}-{uuid.uuid4().hex}")
|
|
563
|
+
try:
|
|
564
|
+
os.rename(self.path, grave)
|
|
565
|
+
except OSError: # another contender took it first, or the holder still has it open
|
|
566
|
+
return False
|
|
567
|
+
try:
|
|
568
|
+
if grave.read_bytes() != judged: # a fresh lock replaced the stale one: put it back
|
|
569
|
+
with contextlib.suppress(OSError):
|
|
570
|
+
os.link(grave, self.path)
|
|
571
|
+
return False
|
|
572
|
+
finally:
|
|
573
|
+
grave.unlink(missing_ok=True)
|
|
574
|
+
on_stale(f"removed a stale runner lock {self.path}: {judged.decode('utf-8', 'replace')[:200]}")
|
|
575
|
+
return True
|
|
576
|
+
|
|
577
|
+
def set_batch(self, batch: str) -> None:
|
|
578
|
+
self.holder["batch"] = batch
|
|
579
|
+
text = json.dumps(self.holder)
|
|
580
|
+
try:
|
|
581
|
+
_replace_with(self.path, text)
|
|
582
|
+
self.text = text
|
|
583
|
+
except OSError as exc: # other runners could not see the batch: never run it unclaimed
|
|
584
|
+
raise _Stop(f"BLOCKED claim not written: {batch} ({type(exc).__name__})", caddis_exit.BLOCKED) from None
|
|
585
|
+
|
|
586
|
+
def release(self) -> None:
|
|
587
|
+
try:
|
|
588
|
+
if self.text is not None and self.path.read_text(encoding="utf-8") == self.text:
|
|
589
|
+
self.path.unlink()
|
|
590
|
+
except OSError:
|
|
591
|
+
pass
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def runner_lock_file(repo: Path, kind: str, slug: str) -> Path:
|
|
595
|
+
"""The runner lock path: in the git common dir, or the local folder outside a git repo."""
|
|
596
|
+
return caddis_roadmap.runner_lock_path(repo, kind, slug) or _local_dir(repo, kind, slug) / "runner.lock"
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def last_stop_file(repo: Path, kind: str, slug: str) -> Path:
|
|
600
|
+
return caddis_roadmap.last_stop_path(repo, kind, slug) or _local_dir(repo, kind, slug) / "last-stop.json"
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
@contextlib.contextmanager
|
|
604
|
+
def _claim(repo: Path):
|
|
605
|
+
"""One repo-level lock around `next` plus the batch claim (and plan mode's holder check plus its lock), so two
|
|
606
|
+
runners cannot choose the same batch. Outside a git repo there is nothing to share it with."""
|
|
607
|
+
path = caddis_roadmap.claim_lock_path(repo)
|
|
608
|
+
if path is None:
|
|
609
|
+
yield
|
|
610
|
+
return
|
|
611
|
+
try:
|
|
612
|
+
lock = caddis_roadmap._Lock(path, timeout=caddis_roadmap.CLAIM_TIMEOUT)
|
|
613
|
+
lock.__enter__()
|
|
614
|
+
except caddis_roadmap.RoadmapError as exc:
|
|
615
|
+
raise _Stop(f"BLOCKED claim lock busy: {exc}", caddis_exit.BLOCKED) from None
|
|
616
|
+
try:
|
|
617
|
+
yield
|
|
618
|
+
finally:
|
|
619
|
+
lock.__exit__(None, None, None)
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def _roadmap_holder_of(repo: Path, plan: str, own_lock: Path | None = None) -> tuple[dict, str] | None:
|
|
623
|
+
"""(holder, batch) when another live runner lock holds a phase of this plan (or the whole plan)."""
|
|
624
|
+
for batch, holder in caddis_roadmap.live_runner_holders(repo, exclude=own_lock).items():
|
|
625
|
+
if caddis_roadmap.match_ref(batch) == plan:
|
|
626
|
+
return holder, batch
|
|
627
|
+
return None
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _read_record(path: Path) -> dict:
|
|
631
|
+
try:
|
|
632
|
+
record = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
633
|
+
except (OSError, ValueError):
|
|
634
|
+
return {}
|
|
635
|
+
return record if isinstance(record, dict) else {}
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
class GhError(caddis_roadmap.RoadmapError):
|
|
639
|
+
"""gh could not answer (not installed, no login, 401/404, bad output): an error, never "not merged"."""
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
GH_ACCOUNT = {"name": None} # the account the runner's gh calls use (GH_TOKEN set from it), None: the active one
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def use_gh_account(account: str | None) -> str | None:
|
|
646
|
+
"""Make every gh call of this process (and of the caddis_roadmap.py children, which inherit the environment) use
|
|
647
|
+
`account`: GH_TOKEN from `gh auth token -u <account>`. None keeps the environment / active account. Returns the
|
|
648
|
+
value set, so it can be masked; GhError when gh cannot give it."""
|
|
649
|
+
GH_ACCOUNT["name"] = account or None
|
|
650
|
+
if not account:
|
|
651
|
+
return None
|
|
652
|
+
try:
|
|
653
|
+
proc = subprocess.run(["gh", "auth", "token", "-u", account], capture_output=True, text=True,
|
|
654
|
+
encoding="utf-8", errors="replace", stdin=subprocess.DEVNULL,
|
|
655
|
+
timeout=caddis_roadmap.GH_TIMEOUT)
|
|
656
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
657
|
+
raise GhError(f"gh cannot give a login for account {account}: {type(exc).__name__}",
|
|
658
|
+
caddis_exit.BLOCKED) from None
|
|
659
|
+
gh_auth = (proc.stdout or "").strip()
|
|
660
|
+
if proc.returncode != 0 or not gh_auth:
|
|
661
|
+
raise GhError(f"gh has no login for account {account}", caddis_exit.BLOCKED)
|
|
662
|
+
os.environ["GH_TOKEN"] = gh_auth
|
|
663
|
+
return gh_auth
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _gh_who() -> str:
|
|
667
|
+
return GH_ACCOUNT["name"] or "(the active gh account)"
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _pr_state(url: str) -> dict | None:
|
|
671
|
+
"""The PR document from `gh api repos/<o>/<r>/pulls/<n>`; None for a URL that is not a GitHub PR. GhError when
|
|
672
|
+
gh cannot answer (missing, 401, 404, a timeout, bad output)."""
|
|
673
|
+
match = re.search(r"github\.com/([^/]+/[^/]+)/pull/(\d+)$", url or "")
|
|
674
|
+
if not match:
|
|
675
|
+
return None
|
|
676
|
+
gh_repo = match.group(1)
|
|
677
|
+
try:
|
|
678
|
+
proc = caddis_roadmap.run_bounded(["gh", "api", f"repos/{gh_repo}/pulls/{match.group(2)}"],
|
|
679
|
+
caddis_roadmap.GH_TIMEOUT, "gh api (PR state)")
|
|
680
|
+
except (OSError, caddis_roadmap.RoadmapError) as exc:
|
|
681
|
+
raise GhError(f"gh cannot see {gh_repo} with account {_gh_who()}: {type(exc).__name__}",
|
|
682
|
+
caddis_exit.BLOCKED) from None
|
|
683
|
+
if proc.returncode != 0:
|
|
684
|
+
tail = " ".join((proc.stderr or proc.stdout or "").split())[:160]
|
|
685
|
+
raise GhError(f"gh cannot see {gh_repo} with account {_gh_who()}: {tail}", caddis_exit.BLOCKED)
|
|
686
|
+
try:
|
|
687
|
+
doc = json.loads(proc.stdout)
|
|
688
|
+
except ValueError:
|
|
689
|
+
raise GhError(f"gh gave no PR document for {url}", caddis_exit.BLOCKED) from None
|
|
690
|
+
if not isinstance(doc, dict):
|
|
691
|
+
raise GhError(f"gh gave no PR document for {url}", caddis_exit.BLOCKED)
|
|
692
|
+
return doc
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def default_pr_merged(repo: Path, url: str) -> bool:
|
|
696
|
+
"""GitHub says the PR merged. GhError when gh cannot answer."""
|
|
697
|
+
doc = _pr_state(url)
|
|
698
|
+
return bool(doc) and doc.get("merged") is True
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def default_pr_closed(repo: Path, url: str) -> bool:
|
|
702
|
+
"""GitHub says the PR is closed without a merge. GhError when gh cannot answer."""
|
|
703
|
+
doc = _pr_state(url)
|
|
704
|
+
return bool(doc) and doc.get("state") == "closed" and doc.get("merged") is not True
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def last_stop_restartable(repo: Path, roadmap: str,
|
|
708
|
+
pr_merged: Callable[[Path, str], bool] | None = None,
|
|
709
|
+
pr_closed: Callable[[Path, str], bool] | None = None) -> tuple[bool, str]:
|
|
710
|
+
"""(may the recheck task restart the runner, why). Yes after no recorded stop or `BLOCKED nothing ready` (then
|
|
711
|
+
only when recheck cleared a blocker), and after `BLOCKED waiting for merge` once GitHub says that PR merged (the
|
|
712
|
+
why starts with `merged:`; no cleared blocker is needed). Read only: no fetch, no ref or file changes (the runner
|
|
713
|
+
checks the Tracker on origin after it holds the lock). Everything else, `running` included, needs the owner."""
|
|
714
|
+
path = last_stop_file(repo, "roadmap", roadmap)
|
|
715
|
+
try:
|
|
716
|
+
record = json.loads(path.read_text(encoding="utf-8"))
|
|
717
|
+
except FileNotFoundError:
|
|
718
|
+
return True, "no recorded stop"
|
|
719
|
+
except (OSError, ValueError) as exc:
|
|
720
|
+
return False, f"last stop not readable ({type(exc).__name__}); the owner decides"
|
|
721
|
+
if not isinstance(record, dict):
|
|
722
|
+
return False, "last stop not readable (not an object); the owner decides"
|
|
723
|
+
reason = str(record.get("reason"))
|
|
724
|
+
if reason == RESTARTABLE_STOP:
|
|
725
|
+
held = [h for h in record.get("held") or [] if isinstance(h, dict) and h.get("ref")]
|
|
726
|
+
if held: # skipped only because another runner held them: free now?
|
|
727
|
+
live = caddis_roadmap.live_runner_batches(Path(repo), exclude=runner_lock_file(Path(repo), "roadmap",
|
|
728
|
+
roadmap))
|
|
729
|
+
held_plans = {caddis_roadmap.match_ref(b) for b in live}
|
|
730
|
+
still = [h["ref"] for h in held if caddis_roadmap.match_ref(h["ref"]) in held_plans]
|
|
731
|
+
if not still:
|
|
732
|
+
return True, "freed: " + ", ".join(h["ref"] for h in held)
|
|
733
|
+
ready = _rollup_ready_without_fetch(Path(repo), roadmap)
|
|
734
|
+
if ready: # a milestone waits only for its rollup (its blocker went): restart, no cleared blocker needed
|
|
735
|
+
return True, "rollup-ready: " + ", ".join(ready)
|
|
736
|
+
return True, f"last stop: {reason}"
|
|
737
|
+
waiting = _read_record(_state_file(Path(repo), roadmap, "waiting"))
|
|
738
|
+
if waiting and (reason.startswith(WAITING_PREFIX) or reason.startswith(GH_STOP_PREFIX)):
|
|
739
|
+
ref, url = str(waiting.get("ref", "")), str(waiting.get("pr", ""))
|
|
740
|
+
try:
|
|
741
|
+
if (pr_merged or default_pr_merged)(Path(repo), url): # gh only: no fetch, no ref changes
|
|
742
|
+
return True, f"merged: {ref} {url}"
|
|
743
|
+
if (pr_closed or default_pr_closed)(Path(repo), url): # read only: the runner's start cleans it up
|
|
744
|
+
return False, f"last stop needs the owner: the PR closed without a merge: {ref} {url}"
|
|
745
|
+
except Exception as exc: # gh missing, 401, 404: an error, never "not merged"
|
|
746
|
+
detail = str(exc).splitlines()[0] if str(exc) else type(exc).__name__
|
|
747
|
+
return False, f"gh error: {detail}"
|
|
748
|
+
return False, f"waiting for merge: {ref} {url}"
|
|
749
|
+
if reason == RUNNING:
|
|
750
|
+
return False, ("last stop needs the owner: running (a runner is running, or it was killed without a stop: "
|
|
751
|
+
"a crash, a reboot)")
|
|
752
|
+
return False, f"last stop needs the owner: {reason} (exit {record.get('exit')})"
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
# -- the runner -------------------------------------------------------------------------------------------------------
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
class _Stop(Exception):
|
|
759
|
+
def __init__(self, reason: str, code: int, *, push: bool = True, from_relay: bool = False,
|
|
760
|
+
status: tuple[str, str] | None = None, detail: str | None = None, record: dict | None = None):
|
|
761
|
+
super().__init__(reason)
|
|
762
|
+
self.reason, self.code, self.push, self.from_relay = reason, code, push, from_relay
|
|
763
|
+
self.status, self.detail, self.record = status, detail, record or {}
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def _iso(moment: datetime) -> str:
|
|
767
|
+
return moment.astimezone(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
@dataclass
|
|
771
|
+
class _Ctx:
|
|
772
|
+
"""What one run shares between its sessions."""
|
|
773
|
+
repo: Path
|
|
774
|
+
target: Target
|
|
775
|
+
caps: Caps
|
|
776
|
+
engine: str
|
|
777
|
+
model: str
|
|
778
|
+
engine_cmd: Sequence[str] | None
|
|
779
|
+
governor: Sequence[str]
|
|
780
|
+
home: Path
|
|
781
|
+
now: Callable[[], datetime]
|
|
782
|
+
clock: Callable[[], float]
|
|
783
|
+
toast: Callable[[str, str], int]
|
|
784
|
+
push: Callable[[str], int]
|
|
785
|
+
masks: list[str]
|
|
786
|
+
last_stop: Path | None = None # set once this run holds the lock: only the holder records its last stop
|
|
787
|
+
written_claims: list = field(default_factory=list) # claims this run wrote (removed again if it crashes)
|
|
788
|
+
gh_account: str | None = None # recorded in the launch options; recheck restarts with it
|
|
789
|
+
review_cmd: Sequence[str] = () # the `caddis_review.py` command (a seam: tests pass a fake)
|
|
790
|
+
|
|
791
|
+
@property
|
|
792
|
+
def log_path(self) -> Path:
|
|
793
|
+
return self.target.local / "runner.jsonl"
|
|
794
|
+
|
|
795
|
+
def log(self, record: dict) -> None:
|
|
796
|
+
_ensure_local(self.target.local)
|
|
797
|
+
with self.log_path.open("a", encoding="utf-8") as fh:
|
|
798
|
+
fh.write(json.dumps(caddis_redact.redact_obj(record, self.masks), ensure_ascii=False) + "\n")
|
|
799
|
+
|
|
800
|
+
def stop_file(self) -> bool:
|
|
801
|
+
return (self.home / ".caddis" / "STOP").exists() or (self.target.local / "STOP").exists()
|
|
802
|
+
|
|
803
|
+
def checks(self, sessions: int, spent: float, elapsed: float) -> None:
|
|
804
|
+
"""Raise _Stop for the first reason not to launch another session."""
|
|
805
|
+
started = sessions > 0
|
|
806
|
+
if self.stop_file():
|
|
807
|
+
raise _Stop("STOP file", caddis_exit.CLEAN if started else caddis_exit.NOT_RUN, push=False)
|
|
808
|
+
if sessions >= self.caps.max_sessions:
|
|
809
|
+
raise _Stop(f"cap: sessions {self.caps.max_sessions}", caddis_exit.BLOCKED)
|
|
810
|
+
if elapsed >= self.caps.max_hours * 3600:
|
|
811
|
+
raise _Stop(f"cap: hours {elapsed / 3600:.2f} of {self.caps.max_hours:g}", caddis_exit.BLOCKED)
|
|
812
|
+
if spent >= self.caps.max_usd_total:
|
|
813
|
+
raise _Stop(f"cap: usd {spent:.2f} of {self.caps.max_usd_total:g}", caddis_exit.BLOCKED)
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
@dataclass
|
|
817
|
+
class _Session:
|
|
818
|
+
outcome: "_Outcome"
|
|
819
|
+
result: dict | None
|
|
820
|
+
cost: float
|
|
821
|
+
pushed: bool
|
|
822
|
+
relay_text: str
|
|
823
|
+
hash_before: str | None
|
|
824
|
+
hash_after: str | None
|
|
825
|
+
record: dict
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
def _one_session(ctx: _Ctx, prompt: str, relay_slug: str, n: int, spent: float, elapsed: float,
|
|
829
|
+
spend_slug: str, cwd: Path | None = None) -> _Session:
|
|
830
|
+
"""Launch one session (agy: under a spend reservation) in `cwd` (default the repo) and read what it left."""
|
|
831
|
+
caps = ctx.caps
|
|
832
|
+
cwd = cwd or ctx.repo
|
|
833
|
+
remaining_s = max(1.0, min(caps.session_timeout_min * 60, caps.max_hours * 3600 - elapsed))
|
|
834
|
+
usd = min(caps.max_usd_per_session, caps.max_usd_total - spent)
|
|
835
|
+
cmd = build_command(ctx.engine, ctx.model, prompt, usd=usd, timeout_min=remaining_s / 60, governor=ctx.governor,
|
|
836
|
+
engine_cmd=ctx.engine_cmd)
|
|
837
|
+
rid = _spend_reserve(ctx.home, spend_slug, ctx.now(), n > 1) if ctx.engine == "agy" else None
|
|
838
|
+
launched, outcome = ctx.clock(), None
|
|
839
|
+
try:
|
|
840
|
+
_, hash_before = _read_relay(cwd, relay_slug)
|
|
841
|
+
record = {"kind": "session", "n": n, "start": _iso(ctx.now()), "relay_hash_before": hash_before}
|
|
842
|
+
outcome = _launch(cmd, cwd, remaining_s, prompt if ctx.engine in PROMPT_ON_STDIN else None,
|
|
843
|
+
env=headless_env())
|
|
844
|
+
finally: # any exception, KeyboardInterrupt too, still settles the reservation
|
|
845
|
+
if rid is not None:
|
|
846
|
+
_spend_settle(ctx.home, rid, cmd, outcome, ctx.now(), ctx.clock() - launched)
|
|
847
|
+
outcome = _Outcome(outcome.code, caddis_redact.redact(outcome.stdout, ctx.masks),
|
|
848
|
+
caddis_redact.redact(outcome.stderr, ctx.masks), outcome.timed_out, outcome.started)
|
|
849
|
+
record["end"] = _iso(ctx.now())
|
|
850
|
+
# The hour cap is checked again by the next iteration's checks(), before any further session.
|
|
851
|
+
result = parse_result(outcome.stdout)
|
|
852
|
+
cost = _cost(result)
|
|
853
|
+
text = result.get("result", "") if isinstance(result, dict) else ""
|
|
854
|
+
pushed = caddis_notify.PUSH_MARKER in (text if isinstance(text, str) else "") or \
|
|
855
|
+
caddis_notify.PUSH_MARKER in outcome.stdout
|
|
856
|
+
relay_text, hash_after = _read_relay(cwd, relay_slug)
|
|
857
|
+
record.update({"exit": outcome.code, "cost_usd": cost, "relay_hash_after": hash_after, "push_sent": pushed})
|
|
858
|
+
return _Session(outcome, result, cost, pushed, relay_text, hash_before, hash_after, record)
|
|
859
|
+
|
|
860
|
+
|
|
861
|
+
# -- the review gate (caddis-next Phase 11) ---------------------------------------------------------------------------
|
|
862
|
+
|
|
863
|
+
REVIEW_RUNS = 2
|
|
864
|
+
REVIEW_MAX_USD = REVIEW_RUNS * caddis_review.MAX_BUDGET_USD # the most one review may cost; counted when unknown
|
|
865
|
+
REVIEW_TIMEOUT_S = REVIEW_RUNS * caddis_review.CLAUDE_TIMEOUT_SECS + 900 # a backstop: each run has its own timeout
|
|
866
|
+
REVIEW_BLOCKED = "BLOCKED review" # findings left after the one fix round; needs the owner
|
|
867
|
+
REVIEW_NOT_RUN = "BLOCKED review not run" # exit 3, 4, any other code, no heading, no plan; needs the owner
|
|
868
|
+
REVIEW_ROUND_LIMIT = "BLOCKED review round limit: the owner decides" # exit 5: never review again
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
def default_review_cmd() -> list[str]:
|
|
872
|
+
return [sys.executable, str(_HERE / "caddis_review.py")]
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
@dataclass
|
|
876
|
+
class _Tally:
|
|
877
|
+
"""The run's counters while the review gate runs: its fix session and its cost count against the caps."""
|
|
878
|
+
sessions: int
|
|
879
|
+
spent: float
|
|
880
|
+
pushed: bool
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
def plan_paths(repo: Path, plan: str) -> tuple[Path, Path]:
|
|
884
|
+
"""Where a plan lives: `.caddis/plans/<plan>.md`, or `.caddis/plans/done/<plan>.md` after tidy moved it."""
|
|
885
|
+
plans = Path(repo) / ".caddis" / "plans"
|
|
886
|
+
return plans / f"{plan}.md", plans / "done" / f"{plan}.md"
|
|
887
|
+
|
|
888
|
+
|
|
889
|
+
def _working_plan(repo: Path, plan: str) -> tuple[Path, str] | None:
|
|
890
|
+
"""The working copy of the plan (its current path, else done/) and its text; None when neither is readable."""
|
|
891
|
+
for path in plan_paths(repo, plan):
|
|
892
|
+
if path.is_file():
|
|
893
|
+
try:
|
|
894
|
+
return path, path.read_text(encoding="utf-8")
|
|
895
|
+
except OSError:
|
|
896
|
+
continue
|
|
897
|
+
return None
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def _origin_plan_text(repo: Path, plan: str) -> str | None:
|
|
901
|
+
"""The plan on `origin/<default>` as last fetched (no fetch and no remote call here), from its path or done/;
|
|
902
|
+
None when absent or when origin/HEAD is not set locally."""
|
|
903
|
+
ref = _git_out(repo, "symbolic-ref", "refs/remotes/origin/HEAD") or ""
|
|
904
|
+
if not ref.startswith("refs/remotes/origin/"):
|
|
905
|
+
return None
|
|
906
|
+
branch = ref[len("refs/remotes/origin/"):]
|
|
907
|
+
for path in (f".caddis/plans/{plan}.md", f".caddis/plans/done/{plan}.md"):
|
|
908
|
+
text = caddis_roadmap.show(repo, branch, path)
|
|
909
|
+
if text is not None:
|
|
910
|
+
return text
|
|
911
|
+
return None
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
def phase_info(texts: Sequence[str], phase: int, scratch: Path) -> tuple[bool, bool]:
|
|
915
|
+
"""(some copy has a `### Phase <n> ` heading, some copy marks the phase high). `caddis_review.phase_risk` fails
|
|
916
|
+
closed, so anything but normal, low or medium is high. Every copy counts: a session cannot lower its own risk."""
|
|
917
|
+
found = high = False
|
|
918
|
+
_ensure_local(scratch) # the local folder gets its `*` .gitignore before anything is written there
|
|
919
|
+
probe = scratch / f"risk-probe-{os.getpid()}-{uuid.uuid4().hex}.md"
|
|
920
|
+
try:
|
|
921
|
+
for text in texts:
|
|
922
|
+
if not text:
|
|
923
|
+
continue
|
|
924
|
+
probe.write_text(text, encoding="utf-8")
|
|
925
|
+
try:
|
|
926
|
+
risk = caddis_review.phase_risk(probe, phase)
|
|
927
|
+
except ValueError: # no such phase heading in this copy
|
|
928
|
+
continue
|
|
929
|
+
found = True
|
|
930
|
+
high = high or risk == "high"
|
|
931
|
+
finally:
|
|
932
|
+
probe.unlink(missing_ok=True)
|
|
933
|
+
return found, high
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def _done_rows(text: str | None) -> list[int]:
|
|
937
|
+
rows = caddis_roadmap.parse_tracker(text or "")
|
|
938
|
+
return [n for n in sorted(rows) if caddis_roadmap._is_done_status(rows[n])]
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
def reviewed_file(repo: Path, kind: str, slug: str, plan: str) -> Path:
|
|
942
|
+
"""The record of reviewed phases of `plan`, in THIS runner's own folder:
|
|
943
|
+
`<git common dir>/caddis-runner/<kind>-<slug>/reviewed/<plan>.json` (every worktree and every restart of this
|
|
944
|
+
runner sees it; outside a git repo, `<local>/reviewed/<plan>.json`). A runner trusts only records in its own
|
|
945
|
+
folder, so the per-runner state check covers them and a session cannot forge them unseen during this runner's
|
|
946
|
+
sessions. Another runner's record for the same plan does not count: a phase reviewed by a plan-mode runner is
|
|
947
|
+
reviewed again when a roadmap runner picks it (safe, accepted). Limit: the state check hashes only this runner's
|
|
948
|
+
own folder, so a session can edit ANOTHER runner's folder unseen by this runner (sessions are cooperative, same
|
|
949
|
+
OS user; the state check is not a security boundary)."""
|
|
950
|
+
return runner_folder(repo, kind, slug) / "reviewed" / f"{plan}.json"
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
def read_reviewed(path: Path) -> dict:
|
|
954
|
+
"""phase (str) -> {"head", "utc", "result"}. A missing or unreadable record is empty: every phase is pending
|
|
955
|
+
again (fail closed: review once more, never skip)."""
|
|
956
|
+
record = _read_record(path)
|
|
957
|
+
return {k: v for k, v in record.items() if isinstance(v, dict)}
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def _record_review(path: Path, phase: int, head: str | None, result: str) -> None:
|
|
961
|
+
"""The ONLY writer of `reviewed/<plan>.json`. The runner calls it only between children (after a review child
|
|
962
|
+
exited and its state check passed, never while a session or a review runs); every session and review child takes
|
|
963
|
+
its own state snapshot right before it starts, so the runner's write is already in that baseline."""
|
|
964
|
+
record = read_reviewed(path)
|
|
965
|
+
record[str(phase)] = {"head": head, "utc": _iso(datetime.now(timezone.utc)), "result": result}
|
|
966
|
+
_write_record(path, record)
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
def pending_reviews(working: str, origin: str | None, reviewed: dict, also: Sequence[int] = ()) -> list[int]:
|
|
970
|
+
"""Phases to review: done in the working copy's Tracker, not done on origin/<default> (merged phases were reviewed
|
|
971
|
+
before their PR, or merged by the owner), and not recorded as reviewed clean. The caller then keeps only the
|
|
972
|
+
`risk: high` ones."""
|
|
973
|
+
merged = set(_done_rows(origin)) if origin is not None else set()
|
|
974
|
+
return [n for n in sorted(set(_done_rows(working)) | set(also)) if n not in merged
|
|
975
|
+
and (reviewed.get(str(n)) or {}).get("result") != "clean"]
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
def _rel(repo: Path, path: Path) -> str:
|
|
979
|
+
try:
|
|
980
|
+
return Path(path).resolve().relative_to(repo).as_posix()
|
|
981
|
+
except ValueError:
|
|
982
|
+
return Path(path).as_posix()
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
def _review_cost(cost_file: Path) -> tuple[float, str]:
|
|
986
|
+
"""The review's `total_cost_usd`, or REVIEW_MAX_USD when the file is missing or unreadable (fail toward
|
|
987
|
+
spending less: the caps then stop the run sooner, never later)."""
|
|
988
|
+
try:
|
|
989
|
+
value = json.loads(cost_file.read_text(encoding="utf-8")).get("total_cost_usd")
|
|
990
|
+
except FileNotFoundError:
|
|
991
|
+
return float(REVIEW_MAX_USD), "missing"
|
|
992
|
+
except (OSError, ValueError, AttributeError):
|
|
993
|
+
return float(REVIEW_MAX_USD), "unreadable"
|
|
994
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not (0 <= value < float("inf")):
|
|
995
|
+
return float(REVIEW_MAX_USD), "unreadable"
|
|
996
|
+
return float(value), "file"
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _run_review(ctx: _Ctx, tally: _Tally, plan_file: Path, phase: int, out_dir: Path, elapsed: Callable[[], float],
|
|
1000
|
+
ref: str, round_no: int, cwd: Path | None = None) -> int | None:
|
|
1001
|
+
"""One `caddis_review.py --plan --phase --runs 2` child with CADDIS_HEADLESS=1. Its cost is added to `spent`.
|
|
1002
|
+
Returns its exit code; None when it timed out, did not start, or exit 1 left no merged findings file. It reviews
|
|
1003
|
+
the checkout `cwd` (default the repo; roadmap mode: the roadmap's worktree, where the phase session ran)."""
|
|
1004
|
+
cwd = Path(cwd) if cwd is not None else ctx.repo
|
|
1005
|
+
caps = ctx.caps
|
|
1006
|
+
if ctx.stop_file():
|
|
1007
|
+
raise _Stop("STOP file", caddis_exit.CLEAN if tally.sessions else caddis_exit.NOT_RUN, push=False)
|
|
1008
|
+
left_s = caps.max_hours * 3600 - elapsed()
|
|
1009
|
+
if left_s <= 0:
|
|
1010
|
+
raise _Stop(f"cap: hours {caps.max_hours:g} reached before the review of {ref}", caddis_exit.BLOCKED)
|
|
1011
|
+
if tally.spent + REVIEW_MAX_USD > caps.max_usd_total:
|
|
1012
|
+
raise _Stop(f"cap: usd {tally.spent:.2f} of {caps.max_usd_total:g} (the review of {ref} may cost up to "
|
|
1013
|
+
f"{REVIEW_MAX_USD:g})", caddis_exit.BLOCKED)
|
|
1014
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
1015
|
+
cost_file = out_dir / f"review-{phase}-cost.json"
|
|
1016
|
+
merged = out_dir / f"review-{phase}-merged.md"
|
|
1017
|
+
stale = False
|
|
1018
|
+
for old in (cost_file, merged): # a file from an earlier review never counts for this one
|
|
1019
|
+
try:
|
|
1020
|
+
old.unlink(missing_ok=True)
|
|
1021
|
+
except OSError:
|
|
1022
|
+
stale = True
|
|
1023
|
+
argv = [*ctx.review_cmd, "--worktree", str(cwd), "--plan", str(plan_file), "--phase", str(phase),
|
|
1024
|
+
"--runs", str(REVIEW_RUNS), "--out-dir", str(out_dir)]
|
|
1025
|
+
print(f"review: {ref} round {round_no} ({REVIEW_RUNS} runs)")
|
|
1026
|
+
code: int | None = None
|
|
1027
|
+
kind, slug = ctx.target.kind, ctx.target.slug
|
|
1028
|
+
state_before = state_snapshot(ctx.repo, kind, slug) # the reviewer has full Bash: it is guarded like a session
|
|
1029
|
+
try:
|
|
1030
|
+
proc = subprocess.run(argv, cwd=cwd, env=headless_env(), stdin=subprocess.DEVNULL, capture_output=True,
|
|
1031
|
+
text=True, encoding="utf-8", errors="replace",
|
|
1032
|
+
timeout=max(1.0, min(REVIEW_TIMEOUT_S, left_s)))
|
|
1033
|
+
code, output = proc.returncode, (proc.stdout or "") + (proc.stderr or "")
|
|
1034
|
+
except subprocess.TimeoutExpired:
|
|
1035
|
+
output = "the review timed out"
|
|
1036
|
+
except OSError as exc:
|
|
1037
|
+
output = f"the review did not start: {exc}"
|
|
1038
|
+
for line in output.splitlines():
|
|
1039
|
+
if line.strip():
|
|
1040
|
+
print(caddis_redact.redact(f"review {ref}: {line}", ctx.masks))
|
|
1041
|
+
cost, source = (float(REVIEW_MAX_USD), "stale") if stale else _review_cost(cost_file)
|
|
1042
|
+
tally.spent += cost
|
|
1043
|
+
if source != "file":
|
|
1044
|
+
print(f"caddis_roadmap_run: review cost file {cost_file} is {source}; counting USD {REVIEW_MAX_USD:g} "
|
|
1045
|
+
f"(the most {REVIEW_RUNS} runs may cost)", file=sys.stderr)
|
|
1046
|
+
note = None
|
|
1047
|
+
if code == caddis_exit.BLOCKED and (stale or not merged.is_file()):
|
|
1048
|
+
note, code = "exit 1 with no merged findings file", None # nothing for a fix session to read: not run
|
|
1049
|
+
ctx.log({"kind": "review", "batch": ref, "round": round_no, "exit": code, "cost_usd": cost,
|
|
1050
|
+
"cost_source": source, **({"note": note} if note else {}), "time": _iso(ctx.now())})
|
|
1051
|
+
_check_state(ctx.repo, kind, slug, state_before) # after its cost is counted, before the record is written
|
|
1052
|
+
return code
|
|
1053
|
+
|
|
1054
|
+
|
|
1055
|
+
def _review_gate(ctx: _Ctx, tally: _Tally, *, plan: str, extra_texts: Sequence[str] = (),
|
|
1056
|
+
origin_text: str | None = None, elapsed: Callable[[], float], spend_slug: str,
|
|
1057
|
+
also: Sequence[int] = (), cwd: Path | None = None) -> bool:
|
|
1058
|
+
"""Review every pending `risk: high` phase of `plan`, in order; True when it reviewed one (all now clean).
|
|
1059
|
+
|
|
1060
|
+
Pending (recomputed after every review and every fix session): a Tracker row done in the working copy, not done on
|
|
1061
|
+
origin/<default>, not recorded clean in `reviewed_file`, and high in any copy (working, origin, `extra_texts`).
|
|
1062
|
+
The plan is read from its path or from done/; neither readable stops `BLOCKED review not run`, and so does a done
|
|
1063
|
+
row with no `### Phase <n>` heading in any copy. Review exit 0 records the phase clean. Exit 1 starts ONE fix
|
|
1064
|
+
session per phase (a session against the caps), then that phase is reviewed again; exit 1 again stops
|
|
1065
|
+
`BLOCKED review`; exit 5 (round limit) stops at once `BLOCKED review round limit: the owner decides`, with no
|
|
1066
|
+
fix session and no later review; exit 3, 4 or any other code stops `BLOCKED review not run`. Neither stop is restartable by the
|
|
1067
|
+
recheck task; an owner restart reviews the phase again (it is still pending). `also`: phases that count as done
|
|
1068
|
+
even when their row is not (roadmap mode: the phase it is about to publish). `cwd`: where the phase session ran
|
|
1069
|
+
(roadmap mode: the roadmap's worktree; default the repo): the plan is read there, the review reviews that
|
|
1070
|
+
checkout, and the fix session runs there."""
|
|
1071
|
+
cwd = Path(cwd) if cwd is not None else ctx.repo
|
|
1072
|
+
record_path = reviewed_file(ctx.repo, ctx.target.kind, ctx.target.slug, plan)
|
|
1073
|
+
out_dir = ctx.target.local / "reviews" / plan
|
|
1074
|
+
fixed: set[int] = set()
|
|
1075
|
+
reviewed_any = False
|
|
1076
|
+
while True:
|
|
1077
|
+
working = _working_plan(cwd, plan)
|
|
1078
|
+
if working is None:
|
|
1079
|
+
raise _Stop(f"{REVIEW_NOT_RUN}: plan {plan} not found", caddis_exit.BLOCKED, record={"plan": plan})
|
|
1080
|
+
plan_file, text = working
|
|
1081
|
+
texts = [text, origin_text or "", *extra_texts]
|
|
1082
|
+
phase = None
|
|
1083
|
+
for n in pending_reviews(text, origin_text, read_reviewed(record_path), also):
|
|
1084
|
+
found, high = phase_info(texts, n, ctx.target.local)
|
|
1085
|
+
if not found:
|
|
1086
|
+
raise _Stop(f"{REVIEW_NOT_RUN}: no heading for phase {n}", caddis_exit.BLOCKED,
|
|
1087
|
+
record={"ref": f"{plan}#{n}"})
|
|
1088
|
+
if high:
|
|
1089
|
+
phase = n
|
|
1090
|
+
break
|
|
1091
|
+
if phase is None:
|
|
1092
|
+
return reviewed_any
|
|
1093
|
+
reviewed_any = True
|
|
1094
|
+
ref = f"{plan}#{phase}"
|
|
1095
|
+
merged = out_dir / f"review-{phase}-merged.md"
|
|
1096
|
+
record = {"ref": ref, "review": str(merged)}
|
|
1097
|
+
code = _run_review(ctx, tally, plan_file, phase, out_dir, elapsed, ref, 2 if phase in fixed else 1, cwd)
|
|
1098
|
+
head = _git_out(cwd, "rev-parse", "HEAD")
|
|
1099
|
+
result = {caddis_exit.CLEAN: "clean", caddis_exit.BLOCKED: "blocked"}.get(code, "not-run")
|
|
1100
|
+
_record_review(record_path, phase, head, result)
|
|
1101
|
+
if code == caddis_exit.CLEAN:
|
|
1102
|
+
continue
|
|
1103
|
+
if code == caddis_exit.ROUND_LIMIT: # the phase used its review rounds: stop at once, never review again
|
|
1104
|
+
raise _Stop(REVIEW_ROUND_LIMIT, caddis_exit.BLOCKED, record=record,
|
|
1105
|
+
detail=f"{ref}: review exit {code} (round limit); see {merged}")
|
|
1106
|
+
if code != caddis_exit.BLOCKED:
|
|
1107
|
+
raise _Stop(REVIEW_NOT_RUN, caddis_exit.BLOCKED, record=record,
|
|
1108
|
+
detail=f"{ref}: review exit {'none' if code is None else code}; see {merged}")
|
|
1109
|
+
if phase in fixed:
|
|
1110
|
+
raise _Stop(REVIEW_BLOCKED, caddis_exit.BLOCKED, record=record,
|
|
1111
|
+
detail=f"{ref}: blocking findings left after one fix round; see {merged}")
|
|
1112
|
+
fixed.add(phase)
|
|
1113
|
+
ctx.checks(tally.sessions, tally.spent, elapsed()) # the fix session is a session like any other
|
|
1114
|
+
prompt = (f"Read {_rel(cwd, merged)} and fix every blocking finding in Phase {phase} of "
|
|
1115
|
+
f"{_rel(cwd, plan_file)}; commit. Work on the branch that is checked out; do not push.\n")
|
|
1116
|
+
state_before = state_snapshot(ctx.repo, ctx.target.kind, ctx.target.slug) # AFTER the record write
|
|
1117
|
+
s = _one_session(ctx, prompt, plan, tally.sessions + 1, tally.spent, elapsed(), spend_slug, cwd=cwd)
|
|
1118
|
+
tally.sessions += 1
|
|
1119
|
+
tally.spent += s.cost
|
|
1120
|
+
tally.pushed = s.pushed
|
|
1121
|
+
failed = _session_failed(ctx.engine, s.outcome, s.result)
|
|
1122
|
+
s.record.update({"batch": ref, "fix_for": "review", "status": "FIX" if failed is None else failed.reason})
|
|
1123
|
+
ctx.log(s.record)
|
|
1124
|
+
_check_state(ctx.repo, ctx.target.kind, ctx.target.slug, state_before) # before any runner write
|
|
1125
|
+
if failed is not None:
|
|
1126
|
+
raise failed
|
|
1127
|
+
|
|
1128
|
+
|
|
1129
|
+
def run(*, repo: Path, plan: str | None = None, roadmap: str | None = None, caps: Caps, engine: str = "claude",
|
|
1130
|
+
model: str | None = None, engine_cmd: Sequence[str] | None = None, governor: Sequence[str] | None = None,
|
|
1131
|
+
prompt_template: Path | None = None, dry_run: bool = False,
|
|
1132
|
+
home: Path | None = None, toast: Callable[[str, str], int] | None = None,
|
|
1133
|
+
push: Callable[[str], int] | None = None, clock: Callable[[], float] = time.monotonic,
|
|
1134
|
+
now: Callable[[], datetime] | None = None, after_session: Callable[[int], None] | None = None,
|
|
1135
|
+
roadmap_cmd: Sequence[str] | None = None, hostname: str | None = None,
|
|
1136
|
+
publish: Callable[..., str] | None = None, pr_merged: Callable[[Path, str], bool] | None = None,
|
|
1137
|
+
worktree_root: Path | None = None, delete_remote_branch: Callable[[Path, str], None] | None = None,
|
|
1138
|
+
pr_closed: Callable[[Path, str], bool] | None = None, gh_account: str | None = None,
|
|
1139
|
+
gh_login: Callable[[str | None], str | None] | None = None,
|
|
1140
|
+
review_cmd: Sequence[str] | None = None) -> int:
|
|
1141
|
+
repo = Path(repo).resolve()
|
|
1142
|
+
if (plan is None) == (roadmap is None):
|
|
1143
|
+
print("caddis_roadmap_run: give exactly one of --plan and --roadmap", file=sys.stderr)
|
|
1144
|
+
return caddis_exit.ERROR
|
|
1145
|
+
if roadmap is not None:
|
|
1146
|
+
if not isinstance(roadmap, str) or not caddis_roadmap.SLUG_RE.match(roadmap):
|
|
1147
|
+
print(f"caddis_roadmap_run: bad roadmap {roadmap!r}: invalid slug", file=sys.stderr)
|
|
1148
|
+
return caddis_exit.ERROR
|
|
1149
|
+
target = Target("roadmap", roadmap, _local_dir(repo, "roadmap", roadmap), (), PLAN_PROMPT_NAME)
|
|
1150
|
+
else:
|
|
1151
|
+
try:
|
|
1152
|
+
target = resolve_target(repo, plan)
|
|
1153
|
+
except ValueError as exc:
|
|
1154
|
+
print(f"caddis_roadmap_run: {exc}", file=sys.stderr)
|
|
1155
|
+
return caddis_exit.ERROR
|
|
1156
|
+
if engine not in ENGINES:
|
|
1157
|
+
print(f"caddis_roadmap_run: unknown engine {engine!r}", file=sys.stderr)
|
|
1158
|
+
return caddis_exit.ERROR
|
|
1159
|
+
if not model and engine == "agy": # the model this machine offers (Argon, else the fallback); the table is display only
|
|
1160
|
+
model = _agy_engine_model(repo)
|
|
1161
|
+
model = model or ENGINES[engine]["default_model"]
|
|
1162
|
+
template = prompt_template or find_prompt_template(target.prompt_name)
|
|
1163
|
+
if template is None or not Path(template).is_file():
|
|
1164
|
+
print(f"caddis_roadmap_run: resume prompt template {target.prompt_name} not found", file=sys.stderr)
|
|
1165
|
+
return caddis_exit.ERROR
|
|
1166
|
+
template = Path(template)
|
|
1167
|
+
if governor is None:
|
|
1168
|
+
governor = default_governor()
|
|
1169
|
+
if governor is None:
|
|
1170
|
+
print("caddis_roadmap_run: caddis_governor.py not found; refusing to start ungoverned", file=sys.stderr)
|
|
1171
|
+
return caddis_exit.NOT_RUN
|
|
1172
|
+
home = home or Path.home()
|
|
1173
|
+
now = now or (lambda: datetime.now(timezone.utc))
|
|
1174
|
+
toast = toast or (lambda title, message: caddis_notify.toast(title, message))
|
|
1175
|
+
push = push or (lambda message: caddis_notify.push(message))
|
|
1176
|
+
ctx = _Ctx(repo, target, caps, engine, model, engine_cmd, governor, home, now, clock, toast, push,
|
|
1177
|
+
caddis_redact.load_secrets(home), gh_account=gh_account or None,
|
|
1178
|
+
review_cmd=list(review_cmd or default_review_cmd()))
|
|
1179
|
+
if gh_account and not dry_run: # every gh call of this run (and its caddis_roadmap.py children) uses it
|
|
1180
|
+
try:
|
|
1181
|
+
value = (gh_login or use_gh_account)(gh_account)
|
|
1182
|
+
except caddis_roadmap.RoadmapError as exc:
|
|
1183
|
+
print(f"caddis_roadmap_run: {exc}", file=sys.stderr)
|
|
1184
|
+
ctx.toast(f"caddis {target.kind} {target.slug}", caddis_redact.redact(f"RUNNER STOP: BLOCKED {exc}",
|
|
1185
|
+
ctx.masks))
|
|
1186
|
+
return caddis_exit.BLOCKED
|
|
1187
|
+
if value:
|
|
1188
|
+
ctx.masks.append(value)
|
|
1189
|
+
if roadmap is not None:
|
|
1190
|
+
return _run_roadmap(ctx, template, dry_run, after_session, list(roadmap_cmd or default_roadmap_cmd()),
|
|
1191
|
+
hostname or socket.gethostname(), publish or default_publish,
|
|
1192
|
+
pr_merged or default_pr_merged, worktree_root,
|
|
1193
|
+
delete_remote_branch or default_delete_remote_branch, pr_closed or default_pr_closed)
|
|
1194
|
+
return _run_plan(ctx, template, dry_run, after_session)
|
|
1195
|
+
|
|
1196
|
+
|
|
1197
|
+
def _dry_run_tail(ctx: _Ctx, template: Path, prompt: str, cwd: Path | None = None) -> None:
|
|
1198
|
+
caps = ctx.caps
|
|
1199
|
+
cmd = build_command(ctx.engine, ctx.model, "<resume prompt>", usd=min(caps.max_usd_per_session, caps.max_usd_total),
|
|
1200
|
+
timeout_min=caps.session_timeout_min, governor=ctx.governor, engine_cmd=ctx.engine_cmd)
|
|
1201
|
+
print(f"would launch (cwd {cwd or ctx.repo}, CADDIS_HEADLESS=1): {subprocess.list2cmdline(cmd)}"
|
|
1202
|
+
+ (" (resume prompt on stdin)" if ctx.engine in PROMPT_ON_STDIN else ""))
|
|
1203
|
+
print(f"resume prompt from {template}:\n{prompt}")
|
|
1204
|
+
if ctx.engine != "claude":
|
|
1205
|
+
print(f"note: engine {ctx.engine} is an adapter, not live-tested")
|
|
1206
|
+
|
|
1207
|
+
|
|
1208
|
+
def _dry_lock_check(lock: Path) -> None:
|
|
1209
|
+
stale = caddis_roadmap.runner_lock_stale(lock)
|
|
1210
|
+
if stale is False:
|
|
1211
|
+
raise _Stop("BLOCKED another runner holds the lock", caddis_exit.NOT_RUN, push=False)
|
|
1212
|
+
if stale:
|
|
1213
|
+
print(f"would remove a stale lock {lock}")
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def _run_plan(ctx: _Ctx, template: Path, dry_run: bool, after_session: Callable[[int], None] | None) -> int:
|
|
1217
|
+
repo, target, slug, masks = ctx.repo, ctx.target, ctx.target.slug, ctx.masks
|
|
1218
|
+
prompt = render_prompt(template, slug)
|
|
1219
|
+
lock_file = runner_lock_file(repo, "plan", slug)
|
|
1220
|
+
lock = _RunnerLock(lock_file, ctx.caps.max_hours, batch=f"{slug}#*", # the whole plan is held
|
|
1221
|
+
claim_path=caddis_roadmap.claim_lock_path(repo))
|
|
1222
|
+
|
|
1223
|
+
def held_stop(code: int) -> _Stop | None:
|
|
1224
|
+
holder = _roadmap_holder_of(repo, slug, own_lock=lock_file)
|
|
1225
|
+
if holder is None:
|
|
1226
|
+
return None
|
|
1227
|
+
what = "its claim: its PR waits for a merge" if holder[0].get("kind") == "claim" else "its runner lock"
|
|
1228
|
+
return _Stop(f"BLOCKED {holder[0].get('holder')} holds {holder[1]} ({what})", code, push=False)
|
|
1229
|
+
|
|
1230
|
+
if dry_run:
|
|
1231
|
+
try:
|
|
1232
|
+
held = held_stop(caddis_exit.NOT_RUN)
|
|
1233
|
+
if held is not None:
|
|
1234
|
+
raise held
|
|
1235
|
+
_dry_lock_check(lock.path)
|
|
1236
|
+
ctx.checks(0, 0.0, 0.0)
|
|
1237
|
+
if ctx.engine == "agy":
|
|
1238
|
+
_spend_preview(ctx.home, ctx.now())
|
|
1239
|
+
except _Stop as stop:
|
|
1240
|
+
print(f"would stop: {caddis_redact.redact(stop.reason, masks)}")
|
|
1241
|
+
return caddis_exit.NOT_RUN
|
|
1242
|
+
progress = ", ".join(str(p) for p in target.progress)
|
|
1243
|
+
print(f"{target.kind} {slug}: progress from {progress}; local files in {target.local}; lock {lock.path}")
|
|
1244
|
+
_dry_run_tail(ctx, template, prompt)
|
|
1245
|
+
return caddis_exit.CLEAN
|
|
1246
|
+
|
|
1247
|
+
try:
|
|
1248
|
+
with _claim(repo): # the holder check and the lock, atomic against a roadmap runner's choice
|
|
1249
|
+
held = held_stop(caddis_exit.NOT_RUN)
|
|
1250
|
+
if held is not None:
|
|
1251
|
+
raise held
|
|
1252
|
+
lock.acquire(lambda message: _stale_notice(ctx, message),
|
|
1253
|
+
claim_held=caddis_roadmap.claim_lock_path(repo) is not None) # _claim holds it already
|
|
1254
|
+
except _Stop as stop:
|
|
1255
|
+
return _finish(stop, ctx, False, 0, 0.0, slug)
|
|
1256
|
+
except OSError as exc:
|
|
1257
|
+
return _finish(_Stop(f"BLOCKED claim not written: the runner lock ({type(exc).__name__})",
|
|
1258
|
+
caddis_exit.BLOCKED), ctx, False, 0, 0.0, slug)
|
|
1259
|
+
_mark_running(ctx, last_stop_file(repo, "plan", slug))
|
|
1260
|
+
if not ENGINES[ctx.engine]["live_tested"]:
|
|
1261
|
+
print(f"caddis_roadmap_run: engine {ctx.engine} is an adapter, not live-tested", file=sys.stderr)
|
|
1262
|
+
|
|
1263
|
+
sessions, spent, stuck, pushed = 0, 0.0, 0, False
|
|
1264
|
+
plan_before = ""
|
|
1265
|
+
start = ctx.clock()
|
|
1266
|
+
try:
|
|
1267
|
+
while True:
|
|
1268
|
+
elapsed = ctx.clock() - start
|
|
1269
|
+
ctx.checks(sessions, spent, elapsed)
|
|
1270
|
+
held = held_stop(caddis_exit.BLOCKED) # checked on every iteration, not only at start
|
|
1271
|
+
if held is not None:
|
|
1272
|
+
raise held
|
|
1273
|
+
# Review gate: every pending `risk: high` phase is reviewed before another session builds on it (also
|
|
1274
|
+
# after a restart: the record of reviewed phases is in the runner state, not in this process).
|
|
1275
|
+
tally = _Tally(sessions, spent, pushed)
|
|
1276
|
+
try:
|
|
1277
|
+
reviewed = _review_gate(ctx, tally, plan=slug, extra_texts=(plan_before,),
|
|
1278
|
+
origin_text=_origin_plan_text(repo, slug),
|
|
1279
|
+
elapsed=lambda: ctx.clock() - start, spend_slug=slug)
|
|
1280
|
+
finally:
|
|
1281
|
+
sessions, spent, pushed = tally.sessions, tally.spent, tally.pushed
|
|
1282
|
+
if reviewed:
|
|
1283
|
+
ctx.checks(sessions, spent, ctx.clock() - start) # the review and its fix session count
|
|
1284
|
+
closed_before = _closed(target)
|
|
1285
|
+
working = _working_plan(repo, slug)
|
|
1286
|
+
plan_before = working[1] if working else ""
|
|
1287
|
+
state_before = state_snapshot(repo, "plan", slug)
|
|
1288
|
+
s = _one_session(ctx, prompt, slug, sessions + 1, spent, elapsed, slug)
|
|
1289
|
+
_check_state(repo, "plan", slug, state_before)
|
|
1290
|
+
sessions += 1
|
|
1291
|
+
spent += s.cost
|
|
1292
|
+
pushed = s.pushed
|
|
1293
|
+
try:
|
|
1294
|
+
stop = _decide(ctx.engine, s.outcome, s.result, s.relay_text, s.hash_before, s.hash_after)
|
|
1295
|
+
if stop is None:
|
|
1296
|
+
closed = _closed(target)
|
|
1297
|
+
stuck = 0 if closed > closed_before else stuck + 1
|
|
1298
|
+
s.record["status"] = "CONTINUE"
|
|
1299
|
+
if stuck >= STUCK_SESSIONS:
|
|
1300
|
+
raise _Stop("BLOCKED stuck", caddis_exit.BLOCKED)
|
|
1301
|
+
else:
|
|
1302
|
+
s.record["status"] = stop.reason
|
|
1303
|
+
raise stop
|
|
1304
|
+
finally:
|
|
1305
|
+
s.record.setdefault("status", "FAILED")
|
|
1306
|
+
ctx.log(s.record)
|
|
1307
|
+
if after_session is not None:
|
|
1308
|
+
after_session(sessions)
|
|
1309
|
+
except _Stop as stop:
|
|
1310
|
+
return _finish(stop, ctx, pushed, sessions, spent, slug)
|
|
1311
|
+
except Exception as exc:
|
|
1312
|
+
_crashed(ctx, exc, pushed, sessions, spent, slug)
|
|
1313
|
+
raise
|
|
1314
|
+
finally:
|
|
1315
|
+
lock.release()
|
|
1316
|
+
|
|
1317
|
+
|
|
1318
|
+
def _stale_notice(ctx: _Ctx, message: str) -> None:
|
|
1319
|
+
message = caddis_redact.redact(message, ctx.masks)
|
|
1320
|
+
print(f"caddis_roadmap_run: {message}", file=sys.stderr)
|
|
1321
|
+
ctx.log({"kind": "stale-lock", "message": message, "time": _iso(datetime.now(timezone.utc))})
|
|
1322
|
+
ctx.toast(f"caddis {ctx.target.kind} {ctx.target.slug}", message)
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _roadmap_write(ctx: _Ctx, roadmap_cmd: Sequence[str], args: list[str]) -> int:
|
|
1326
|
+
"""Run one `caddis_roadmap.py` write as a child process with CADDIS_HEADLESS=1; return its exit code."""
|
|
1327
|
+
if "--by-owner" in args:
|
|
1328
|
+
raise ValueError("the runner never makes an owner-only roadmap change")
|
|
1329
|
+
argv = [*roadmap_cmd, *args, "--roadmap", ctx.target.slug, "--root", str(ctx.repo)]
|
|
1330
|
+
try:
|
|
1331
|
+
proc = subprocess.run(argv, cwd=ctx.repo, env=headless_env(), stdin=subprocess.DEVNULL, capture_output=True,
|
|
1332
|
+
text=True, encoding="utf-8", errors="replace", timeout=ROADMAP_WRITE_TIMEOUT_S)
|
|
1333
|
+
except subprocess.TimeoutExpired:
|
|
1334
|
+
print(f"caddis_roadmap_run: roadmap {args[0]} timed out", file=sys.stderr)
|
|
1335
|
+
return caddis_exit.NOT_RUN
|
|
1336
|
+
except OSError as exc:
|
|
1337
|
+
print(f"caddis_roadmap_run: roadmap {args[0]} not started: {exc}", file=sys.stderr)
|
|
1338
|
+
return caddis_exit.NOT_RUN
|
|
1339
|
+
for line in ((proc.stdout or "") + (proc.stderr or "")).splitlines():
|
|
1340
|
+
if line.strip():
|
|
1341
|
+
print(caddis_redact.redact(f"roadmap {args[0]}: {line}", ctx.masks))
|
|
1342
|
+
ctx.log({"kind": "roadmap", "args": args, "exit": proc.returncode, "time": _iso(datetime.now(timezone.utc))})
|
|
1343
|
+
return proc.returncode
|
|
1344
|
+
|
|
1345
|
+
|
|
1346
|
+
class _Baseline:
|
|
1347
|
+
"""The snapshot guard's baseline, kept across iterations. Every origin read is compared with the previous read
|
|
1348
|
+
first: the Milestones and Blockers tables and the Runner host line are frozen, except for the exact edits the
|
|
1349
|
+
runner itself made and recorded (`expect`) since that read. Then the new read becomes the baseline."""
|
|
1350
|
+
|
|
1351
|
+
def __init__(self, roadmap: "caddis_roadmap.Roadmap"):
|
|
1352
|
+
self.roadmap = roadmap
|
|
1353
|
+
self.edits: list[tuple] = []
|
|
1354
|
+
|
|
1355
|
+
def expect(self, *edit) -> None:
|
|
1356
|
+
self.edits.append(edit)
|
|
1357
|
+
|
|
1358
|
+
def observe(self, snap: "caddis_roadmap.Snapshot") -> "caddis_roadmap.Snapshot":
|
|
1359
|
+
changes = roadmap_guard(self.roadmap, snap.roadmap, self.edits)
|
|
1360
|
+
self.edits = []
|
|
1361
|
+
if changes:
|
|
1362
|
+
raise _Stop("FAILED roadmap changed: " + "; ".join(changes), caddis_exit.BLOCKED)
|
|
1363
|
+
self.roadmap = snap.roadmap
|
|
1364
|
+
return snap
|
|
1365
|
+
|
|
1366
|
+
|
|
1367
|
+
def _load_observed(repo: Path, slug: str, baseline: _Baseline) -> "caddis_roadmap.Snapshot":
|
|
1368
|
+
try:
|
|
1369
|
+
snap = caddis_roadmap.load(repo, slug)
|
|
1370
|
+
except caddis_roadmap.RoadmapError as exc:
|
|
1371
|
+
raise _Stop(f"FAILED roadmap changed: the origin copy no longer loads: {str(exc).splitlines()[0]}",
|
|
1372
|
+
caddis_exit.BLOCKED) from None
|
|
1373
|
+
return baseline.observe(snap)
|
|
1374
|
+
|
|
1375
|
+
|
|
1376
|
+
def _audit_lines(repo: Path, slug: str) -> list[str]:
|
|
1377
|
+
try:
|
|
1378
|
+
return (caddis_roadmap.local_dir(repo, slug) / "changes.jsonl").read_text(encoding="utf-8").splitlines()
|
|
1379
|
+
except OSError:
|
|
1380
|
+
return []
|
|
1381
|
+
|
|
1382
|
+
|
|
1383
|
+
def _git_out(repo: Path, *args: str) -> str | None:
|
|
1384
|
+
proc = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8",
|
|
1385
|
+
errors="replace", stdin=subprocess.DEVNULL)
|
|
1386
|
+
return proc.stdout.strip() if proc.returncode == 0 else None
|
|
1387
|
+
|
|
1388
|
+
|
|
1389
|
+
def _phase_title(plan_text: str, phase: int) -> str:
|
|
1390
|
+
for line in plan_text.splitlines():
|
|
1391
|
+
head = _PHASE_HEAD_RE.match(line)
|
|
1392
|
+
if head and int(head.group(1)) == phase:
|
|
1393
|
+
title = re.sub(r"^ {0,3}#{2,4}\s+Phase\s+\d+\s*[-:–—]*\s*", "", line).strip()
|
|
1394
|
+
return " ".join(title.split())[:100]
|
|
1395
|
+
return ""
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def default_publish(repo: Path, branch: str, base: str, title: str, body: str) -> str:
|
|
1399
|
+
"""Push `branch` to origin and open a PR with `gh api` (REST; the account comes from GH_TOKEN). Returns its URL.
|
|
1400
|
+
When a PR for the branch is already open, returns that one. RoadmapError when it cannot."""
|
|
1401
|
+
caddis_roadmap._git(repo, "push", "--quiet", "-u", "origin", f"{branch}:{branch}")
|
|
1402
|
+
gh_repo = caddis_roadmap.gh_repo_of(repo)
|
|
1403
|
+
if gh_repo is None:
|
|
1404
|
+
raise caddis_roadmap.RoadmapError("origin is not a GitHub remote; cannot open the PR", caddis_exit.NOT_RUN)
|
|
1405
|
+
try:
|
|
1406
|
+
number = caddis_roadmap.open_pr(repo, gh_repo, branch, base, title, body)
|
|
1407
|
+
except caddis_roadmap.RoadmapError:
|
|
1408
|
+
owner = gh_repo.split("/", 1)[0]
|
|
1409
|
+
proc = caddis_roadmap.run_bounded(["gh", "api", f"repos/{gh_repo}/pulls?head={owner}:{branch}&state=open"],
|
|
1410
|
+
caddis_roadmap.GH_TIMEOUT, "gh api (find the PR)")
|
|
1411
|
+
found = json.loads(proc.stdout or "[]") if proc.returncode == 0 else []
|
|
1412
|
+
if not found:
|
|
1413
|
+
raise
|
|
1414
|
+
number = int(found[0]["number"])
|
|
1415
|
+
return f"https://github.com/{gh_repo}/pull/{number}"
|
|
1416
|
+
|
|
1417
|
+
|
|
1418
|
+
def session_branch(slug: str, plan: str, phase: int) -> str:
|
|
1419
|
+
"""The branch the runner makes for one roadmap-mode session: `caddis/roadmap/<slug>/<plan>-p<n>`."""
|
|
1420
|
+
return f"caddis/roadmap/{slug}/{plan}-p{phase}"
|
|
1421
|
+
|
|
1422
|
+
|
|
1423
|
+
def runner_folder(repo: Path, kind: str, slug: str) -> Path:
|
|
1424
|
+
"""This runner's own state folder: `<git common dir>/caddis-runner/<kind>-<slug>/` (outside a git repo, the
|
|
1425
|
+
local folder). Shared at the top level: `claims/` and `claim.lock`."""
|
|
1426
|
+
lock = caddis_roadmap.runner_lock_path(repo, kind, slug)
|
|
1427
|
+
return lock.parent if lock is not None else _local_dir(repo, kind, slug)
|
|
1428
|
+
|
|
1429
|
+
|
|
1430
|
+
def _state_file(repo: Path, slug: str, name: str) -> Path:
|
|
1431
|
+
"""Roadmap runner state in its own folder: `<git common dir>/caddis-runner/roadmap-<slug>/<name>.json`."""
|
|
1432
|
+
return runner_folder(repo, "roadmap", slug) / f"{name}.json"
|
|
1433
|
+
|
|
1434
|
+
|
|
1435
|
+
def claim_file(repo: Path, ref: str) -> Path | None:
|
|
1436
|
+
base = caddis_roadmap.runner_state_dir(repo)
|
|
1437
|
+
return None if base is None else base / "claims" / f"{ref}.json"
|
|
1438
|
+
|
|
1439
|
+
|
|
1440
|
+
def _roadmap_sha(text: str) -> str:
|
|
1441
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
def _branch_on_origin(repo: Path, branch: str) -> bool:
|
|
1445
|
+
proc = caddis_roadmap._git(repo, "ls-remote", "--heads", "origin", f"refs/heads/{branch}", check=False)
|
|
1446
|
+
return proc.returncode == 0 and bool(proc.stdout.strip())
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
WORKTREE_PREFIX = "caddis-roadmap-"
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _registered_worktrees(repo: Path) -> set[Path]:
|
|
1453
|
+
out = _git_out(repo, "worktree", "list", "--porcelain") or ""
|
|
1454
|
+
return {Path(line[len("worktree "):]).resolve() for line in out.splitlines() if line.startswith("worktree ")}
|
|
1455
|
+
|
|
1456
|
+
|
|
1457
|
+
def worktree_path(repo: Path, slug: str, root: Path | None = None) -> Path:
|
|
1458
|
+
"""The roadmap's session worktree: the recorded path, else `<root or %TEMP%>/caddis-roadmap-<slug>-<repo tag>`."""
|
|
1459
|
+
recorded = _read_record(_state_file(repo, slug, "worktree")).get("path")
|
|
1460
|
+
if isinstance(recorded, str) and recorded:
|
|
1461
|
+
return Path(recorded)
|
|
1462
|
+
return Path(root or tempfile.gettempdir()) / f"{WORKTREE_PREFIX}{slug}-{_repo_tag(repo)}"
|
|
1463
|
+
|
|
1464
|
+
|
|
1465
|
+
def _repo_tag(repo: Path) -> str:
|
|
1466
|
+
"""8 hex of sha256 of the resolved git common dir: two repos with the same roadmap slug get two worktrees."""
|
|
1467
|
+
base = caddis_roadmap.runner_state_dir(repo)
|
|
1468
|
+
common = base.parent if base is not None else Path(repo)
|
|
1469
|
+
return hashlib.sha256(os.path.normcase(str(common.resolve())).encode("utf-8")).hexdigest()[:8]
|
|
1470
|
+
|
|
1471
|
+
|
|
1472
|
+
def ensure_worktree(repo: Path, slug: str, default: str, root: Path | None = None) -> Path:
|
|
1473
|
+
"""Reuse the roadmap's worktree, or make it (`git worktree add --detach` at origin/<default>) and record it."""
|
|
1474
|
+
path = worktree_path(repo, slug, root)
|
|
1475
|
+
if path.resolve() in _registered_worktrees(repo):
|
|
1476
|
+
if path.is_dir():
|
|
1477
|
+
return path
|
|
1478
|
+
_git_out(repo, "worktree", "prune") # listed, but its folder is gone (temp cleanup): prune, make it again
|
|
1479
|
+
if path.exists():
|
|
1480
|
+
raise _Stop(f"FAILED worktree path {path} exists and is not a worktree of this repo", caddis_exit.BLOCKED)
|
|
1481
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1482
|
+
if _git_out(repo, "worktree", "add", "--quiet", "--detach", str(path), f"origin/{default}") is None:
|
|
1483
|
+
raise _Stop(f"FAILED cannot make the roadmap worktree {path}", caddis_exit.BLOCKED)
|
|
1484
|
+
_write_record(_state_file(repo, slug, "worktree"), {"path": str(path), "utc": _iso(datetime.now(timezone.utc))})
|
|
1485
|
+
return path
|
|
1486
|
+
|
|
1487
|
+
|
|
1488
|
+
def _worktree_work(path: Path) -> list[str] | None:
|
|
1489
|
+
"""Uncommitted work in a worktree (`git status --porcelain --untracked-files=all`, untracked `.caddis/relay/`
|
|
1490
|
+
files left out); None when the status cannot be read."""
|
|
1491
|
+
status = _git_out(path, "status", "--porcelain", "--untracked-files=all")
|
|
1492
|
+
if status is None:
|
|
1493
|
+
return None
|
|
1494
|
+
return [line for line in status.splitlines()
|
|
1495
|
+
if line.strip() and not (line.startswith("??") and line[3:].replace("\\", "/").startswith(".caddis/relay/"))]
|
|
1496
|
+
|
|
1497
|
+
|
|
1498
|
+
def remove_worktree_if_clean(repo: Path, slug: str, root: Path | None = None) -> str:
|
|
1499
|
+
"""At the end of a roadmap: remove its worktree when it holds no work (only session relays may be left);
|
|
1500
|
+
keep it otherwise. Returns a line for the stop."""
|
|
1501
|
+
path = worktree_path(repo, slug, root)
|
|
1502
|
+
if path.resolve() not in _registered_worktrees(repo):
|
|
1503
|
+
_state_file(repo, slug, "worktree").unlink(missing_ok=True)
|
|
1504
|
+
return "no worktree"
|
|
1505
|
+
work = _worktree_work(path)
|
|
1506
|
+
if work is None:
|
|
1507
|
+
return f"worktree kept (status not readable): {path}"
|
|
1508
|
+
if work:
|
|
1509
|
+
return f"worktree kept (not clean, {len(work)} change(s)): {path}"
|
|
1510
|
+
if _git_out(repo, "worktree", "remove", "--force", str(path)) is None:
|
|
1511
|
+
return f"worktree kept (remove failed): {path}"
|
|
1512
|
+
_git_out(repo, "worktree", "prune")
|
|
1513
|
+
_state_file(repo, slug, "worktree").unlink(missing_ok=True)
|
|
1514
|
+
return f"worktree removed: {path}"
|
|
1515
|
+
|
|
1516
|
+
|
|
1517
|
+
def default_delete_remote_branch(repo: Path, branch: str) -> None:
|
|
1518
|
+
"""Delete the branch on GitHub (`gh api -X DELETE .../git/refs/heads/<branch>`; the account comes from GH_TOKEN).
|
|
1519
|
+
RoadmapError when it cannot."""
|
|
1520
|
+
gh_repo = caddis_roadmap.gh_repo_of(repo)
|
|
1521
|
+
if gh_repo is None:
|
|
1522
|
+
raise caddis_roadmap.RoadmapError("origin is not a GitHub remote; cannot delete the branch", caddis_exit.NOT_RUN)
|
|
1523
|
+
try:
|
|
1524
|
+
proc = caddis_roadmap.run_bounded(["gh", "api", "-X", "DELETE", f"repos/{gh_repo}/git/refs/heads/{branch}"],
|
|
1525
|
+
caddis_roadmap.GH_TIMEOUT, "gh api (delete the branch)")
|
|
1526
|
+
except OSError as exc:
|
|
1527
|
+
raise GhError(f"gh cannot delete {branch}: {type(exc).__name__}", caddis_exit.BLOCKED) from None
|
|
1528
|
+
if proc.returncode != 0:
|
|
1529
|
+
raise caddis_roadmap.RoadmapError(f"could not delete {branch} on origin: {(proc.stderr or '').strip()[:200]}",
|
|
1530
|
+
caddis_exit.BLOCKED)
|
|
1531
|
+
|
|
1532
|
+
|
|
1533
|
+
def _checked_out_in(repo: Path, branch: str) -> list[Path]:
|
|
1534
|
+
"""The worktrees (`git worktree list --porcelain`) that have `branch` checked out."""
|
|
1535
|
+
out, path = [], None
|
|
1536
|
+
for line in (_git_out(repo, "worktree", "list", "--porcelain") or "").splitlines():
|
|
1537
|
+
if line.startswith("worktree "):
|
|
1538
|
+
path = Path(line[len("worktree "):])
|
|
1539
|
+
elif line == f"branch refs/heads/{branch}" and path is not None:
|
|
1540
|
+
out.append(path)
|
|
1541
|
+
return out
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
def _delete_local_branch(repo: Path, branch: str, wt: Path | None) -> str | None:
|
|
1545
|
+
"""Delete a local branch. Only the roadmap worktree `wt` may be detached for it; HEAD never moves in any other
|
|
1546
|
+
checkout: a branch checked out elsewhere is kept, and that checkout is returned as the reason."""
|
|
1547
|
+
for path in _checked_out_in(repo, branch):
|
|
1548
|
+
if wt is not None and path.resolve() == Path(wt).resolve():
|
|
1549
|
+
_git_out(path, "checkout", "-q", "--detach")
|
|
1550
|
+
else:
|
|
1551
|
+
return f"kept locally: checked out in {path}"
|
|
1552
|
+
if _git_out(repo, "branch", "-q", "-D", branch) is None and \
|
|
1553
|
+
_git_out(repo, "rev-parse", "--verify", "--quiet", f"refs/heads/{branch}") is not None:
|
|
1554
|
+
return "kept locally: git branch -D failed"
|
|
1555
|
+
return None
|
|
1556
|
+
|
|
1557
|
+
|
|
1558
|
+
def _cleanup_merged_branch(ctx: _Ctx, repo: Path, branch: str, wt: Path | None,
|
|
1559
|
+
delete_remote_branch: Callable[[Path, str], None], made_branches: set,
|
|
1560
|
+
branches_file: Path) -> None:
|
|
1561
|
+
"""Whoever creates a branch removes it when its PR merges: the local branch, the branch on origin, the record."""
|
|
1562
|
+
if not branch:
|
|
1563
|
+
return
|
|
1564
|
+
kept = _delete_local_branch(repo, branch, wt)
|
|
1565
|
+
if kept:
|
|
1566
|
+
print(f"caddis_roadmap_run: {branch} {kept}", file=sys.stderr)
|
|
1567
|
+
try:
|
|
1568
|
+
delete_remote_branch(repo, branch)
|
|
1569
|
+
except caddis_roadmap.RoadmapError as exc:
|
|
1570
|
+
print(f"caddis_roadmap_run: {str(exc).splitlines()[0]}", file=sys.stderr)
|
|
1571
|
+
made_branches.discard(branch)
|
|
1572
|
+
prs = dict(_read_record(branches_file).get("prs") or {})
|
|
1573
|
+
prs.pop(branch, None)
|
|
1574
|
+
_write_record(branches_file, {"branches": sorted(made_branches), "prs": prs})
|
|
1575
|
+
ctx.log({"kind": "branch-removed", "branch": branch, "local": kept or "deleted",
|
|
1576
|
+
"time": _iso(datetime.now(timezone.utc))})
|
|
1577
|
+
|
|
1578
|
+
|
|
1579
|
+
def _cleanup_finished_branches(repo: Path, slug: str, default: str,
|
|
1580
|
+
pr_merged: Callable[[Path, str], bool] = lambda repo, url: False,
|
|
1581
|
+
delete_remote_branch: Callable[[Path, str], None] = lambda repo, branch: None) -> str:
|
|
1582
|
+
"""At the end of a roadmap: delete the local `caddis/roadmap/<slug>/*` branches that are merged: in
|
|
1583
|
+
origin/<default>'s history, or with a PR that GitHub says merged (a squash merge). A branch with a merged PR is
|
|
1584
|
+
deleted on origin too. Keep the others and name them."""
|
|
1585
|
+
names = (_git_out(repo, "for-each-ref", "--format=%(refname:short)", f"refs/heads/caddis/roadmap/{slug}/") or "")
|
|
1586
|
+
branches_file = _state_file(repo, slug, "branches")
|
|
1587
|
+
record = _read_record(branches_file)
|
|
1588
|
+
prs = dict(record.get("prs") or {})
|
|
1589
|
+
kept: list[str] = []
|
|
1590
|
+
for name in (n.strip() for n in names.splitlines() if n.strip()):
|
|
1591
|
+
url = prs.get(name)
|
|
1592
|
+
try:
|
|
1593
|
+
by_pr = bool(url) and pr_merged(repo, url)
|
|
1594
|
+
except Exception: # gh cannot answer: keep the branch and name it
|
|
1595
|
+
by_pr = False
|
|
1596
|
+
merged = by_pr or subprocess.run(["git", "-C", str(repo), "merge-base", "--is-ancestor", name,
|
|
1597
|
+
f"origin/{default}"], capture_output=True,
|
|
1598
|
+
stdin=subprocess.DEVNULL).returncode == 0
|
|
1599
|
+
if not (merged and _git_out(repo, "branch", "-q", "-D", name) is not None):
|
|
1600
|
+
kept.append(name)
|
|
1601
|
+
continue
|
|
1602
|
+
if by_pr:
|
|
1603
|
+
try:
|
|
1604
|
+
delete_remote_branch(repo, name)
|
|
1605
|
+
except caddis_roadmap.RoadmapError as exc:
|
|
1606
|
+
print(f"caddis_roadmap_run: {str(exc).splitlines()[0]}", file=sys.stderr)
|
|
1607
|
+
prs.pop(name, None)
|
|
1608
|
+
branches = sorted(set(record.get("branches") or []) & set(kept))
|
|
1609
|
+
if record:
|
|
1610
|
+
_write_record(branches_file, {"branches": branches, "prs": prs})
|
|
1611
|
+
return f"unmerged branches kept: {', '.join(kept)}" if kept else "no roadmap branches left"
|
|
1612
|
+
|
|
1613
|
+
|
|
1614
|
+
def rollup_ready(snap: "caddis_roadmap.Snapshot", failed: dict | None = None) -> list[str]:
|
|
1615
|
+
"""Milestones that wait only for their rollup: Status not done/not-needed, every phase merged (Tracker done on
|
|
1616
|
+
origin), and the same runner rule as `next` (caddis_roadmap.runner_ineligible: dependencies, blockers, Owner).
|
|
1617
|
+
`failed` (milestone -> roadmap hash) leaves out a milestone whose rollup already failed on this same roadmap."""
|
|
1618
|
+
rm = snap.roadmap
|
|
1619
|
+
failed = failed or {}
|
|
1620
|
+
sha = _roadmap_sha(snap.text)
|
|
1621
|
+
out = []
|
|
1622
|
+
for ms in rm.milestones:
|
|
1623
|
+
if ms.status in caddis_roadmap.FINISHED or not ms.phases or caddis_roadmap.runner_ineligible(rm, ms):
|
|
1624
|
+
continue
|
|
1625
|
+
if failed.get(ms.id) == sha: # tried on this very roadmap and nothing was written: the owner decides
|
|
1626
|
+
continue
|
|
1627
|
+
if all(snap.phases.state(r) == "done" for r in ms.phases):
|
|
1628
|
+
out.append(ms.id)
|
|
1629
|
+
return out
|
|
1630
|
+
|
|
1631
|
+
|
|
1632
|
+
def _rollup_ready_without_fetch(repo: Path, slug: str) -> list[str]:
|
|
1633
|
+
"""rollup_ready on the origin copy as the local refs have it (no fetch: --last-stop only reads)."""
|
|
1634
|
+
try:
|
|
1635
|
+
branch = caddis_roadmap.default_branch(repo)
|
|
1636
|
+
text = caddis_roadmap.show(repo, branch, caddis_roadmap.roadmap_path(slug))
|
|
1637
|
+
if text is None:
|
|
1638
|
+
return []
|
|
1639
|
+
rm = caddis_roadmap.parse_roadmap(text)
|
|
1640
|
+
if rm.errors:
|
|
1641
|
+
return []
|
|
1642
|
+
snap = caddis_roadmap.Snapshot(repo, slug, branch, text, rm, caddis_roadmap.PhaseIndex(repo, branch), [])
|
|
1643
|
+
return rollup_ready(snap, _read_record(_state_file(repo, slug, "rollup-failed")))
|
|
1644
|
+
except Exception: # a read that fails: nothing to report
|
|
1645
|
+
return []
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
def _check_orphaned_claims(repo: Path, slug: str, waiting: dict, pending: dict) -> None:
|
|
1649
|
+
"""An own claim (`roadmap` = this slug) that neither the waiting record nor pending-rollup.json names is from a run
|
|
1650
|
+
killed between writing the claim and recording the wait. It is never deleted here: the owner checks the branch and
|
|
1651
|
+
its PR first."""
|
|
1652
|
+
known: set[str] = set()
|
|
1653
|
+
if waiting.get("ref"):
|
|
1654
|
+
named = claim_file(repo, str(waiting["ref"]))
|
|
1655
|
+
if named is not None:
|
|
1656
|
+
known.add(os.path.normcase(str(named)))
|
|
1657
|
+
if isinstance(pending.get("claim"), str) and pending["claim"]:
|
|
1658
|
+
known.add(os.path.normcase(pending["claim"]))
|
|
1659
|
+
for _name, path in sorted(_claims_of(repo, "roadmap", slug).items()):
|
|
1660
|
+
if os.path.normcase(str(path)) in known:
|
|
1661
|
+
continue
|
|
1662
|
+
batch = str(_read_record(path).get("batch") or path.stem)
|
|
1663
|
+
match = caddis_roadmap.REF_RE.match(batch)
|
|
1664
|
+
branch = session_branch(slug, match.group(1), int(match.group(2))) if match else "its branch"
|
|
1665
|
+
raise _Stop(f"BLOCKED orphaned claim {path} from a killed run: check {branch} / its PR, then delete the claim",
|
|
1666
|
+
caddis_exit.BLOCKED)
|
|
1667
|
+
|
|
1668
|
+
|
|
1669
|
+
def _held_batches(repo: Path, slug: str, snap: "caddis_roadmap.Snapshot") -> list[dict]:
|
|
1670
|
+
"""The batches `next` skipped only because another runner's lock or claim held them (ref, kind, holder), from
|
|
1671
|
+
caddis_roadmap's structure, never from the reason text."""
|
|
1672
|
+
held: list[dict] = []
|
|
1673
|
+
caddis_roadmap._select(snap, caddis_roadmap.other_roadmap_holders(repo, slug), held_out=held)
|
|
1674
|
+
return held
|
|
1675
|
+
|
|
1676
|
+
|
|
1677
|
+
def _clean_merged(ctx: _Ctx, repo: Path, slug: str, merged: dict, worktree_root: Path | None,
|
|
1678
|
+
delete_remote_branch: Callable[[Path, str], None], made_branches: set, branches_file: Path) -> None:
|
|
1679
|
+
"""A merged phase: its claim and its branch go (local, origin, the record)."""
|
|
1680
|
+
claim = merged.get("claim")
|
|
1681
|
+
if isinstance(claim, str) and claim:
|
|
1682
|
+
Path(claim).unlink(missing_ok=True)
|
|
1683
|
+
wt_now = worktree_path(repo, slug, worktree_root)
|
|
1684
|
+
_cleanup_merged_branch(ctx, repo, str(merged.get("branch") or ""), wt_now if wt_now.is_dir() else None,
|
|
1685
|
+
delete_remote_branch, made_branches, branches_file)
|
|
1686
|
+
|
|
1687
|
+
|
|
1688
|
+
def _run_roadmap(ctx: _Ctx, template: Path, dry_run: bool, after_session: Callable[[int], None] | None,
|
|
1689
|
+
roadmap_cmd: list[str], hostname: str, publish: Callable[..., str],
|
|
1690
|
+
pr_merged: Callable[[Path, str], bool] = lambda repo, url: False,
|
|
1691
|
+
worktree_root: Path | None = None,
|
|
1692
|
+
delete_remote_branch: Callable[[Path, str], None] = lambda repo, branch: None,
|
|
1693
|
+
pr_closed: Callable[[Path, str], bool] = lambda repo, url: False) -> int:
|
|
1694
|
+
repo, slug, masks, caps = ctx.repo, ctx.target.slug, ctx.masks, ctx.caps
|
|
1695
|
+
lock = _RunnerLock(runner_lock_file(repo, "roadmap", slug), caps.max_hours,
|
|
1696
|
+
claim_path=caddis_roadmap.claim_lock_path(repo))
|
|
1697
|
+
baseline: _Baseline | None = None
|
|
1698
|
+
|
|
1699
|
+
def write(args: list[str], edit: tuple) -> None:
|
|
1700
|
+
rc = _roadmap_write(ctx, roadmap_cmd, args)
|
|
1701
|
+
if rc != caddis_exit.CLEAN:
|
|
1702
|
+
raise _Stop(f"FAILED roadmap {' '.join(args[:3])} exit {rc}", caddis_exit.BLOCKED)
|
|
1703
|
+
baseline.expect(*edit)
|
|
1704
|
+
|
|
1705
|
+
def rollup(mid: str) -> int:
|
|
1706
|
+
"""Roll the milestone up; the guard excuses only what the audit log says this rollup WROTE (merged). Returns
|
|
1707
|
+
how many changes it wrote."""
|
|
1708
|
+
written = 0
|
|
1709
|
+
seen = len(_audit_lines(repo, slug))
|
|
1710
|
+
rc = _roadmap_write(ctx, roadmap_cmd, ["rollup", "--id", mid])
|
|
1711
|
+
if rc not in (caddis_exit.CLEAN, caddis_exit.BLOCKED):
|
|
1712
|
+
raise _Stop(f"FAILED roadmap rollup {mid} exit {rc}", caddis_exit.BLOCKED)
|
|
1713
|
+
for line in _audit_lines(repo, slug)[seen:]:
|
|
1714
|
+
try:
|
|
1715
|
+
entry = json.loads(line)
|
|
1716
|
+
except ValueError:
|
|
1717
|
+
continue
|
|
1718
|
+
if not isinstance(entry, dict) or entry.get("id") != mid or entry.get("merge_exit") != caddis_exit.CLEAN:
|
|
1719
|
+
continue
|
|
1720
|
+
change = str(entry.get("change", ""))
|
|
1721
|
+
if change.startswith("Status "):
|
|
1722
|
+
baseline.expect("status", mid, change[len("Status "):])
|
|
1723
|
+
written += 1
|
|
1724
|
+
elif change == "block external":
|
|
1725
|
+
baseline.expect("blocker-like", mid, "external", ROLLUP_BLOCK_RE)
|
|
1726
|
+
written += 1
|
|
1727
|
+
return written
|
|
1728
|
+
|
|
1729
|
+
if dry_run: # no host check, no lock, no write: read origin and print what one iteration would do
|
|
1730
|
+
try:
|
|
1731
|
+
_dry_lock_check(lock.path)
|
|
1732
|
+
ctx.checks(0, 0.0, 0.0)
|
|
1733
|
+
if ctx.engine == "agy":
|
|
1734
|
+
_spend_preview(ctx.home, ctx.now())
|
|
1735
|
+
try:
|
|
1736
|
+
batch, snap = caddis_roadmap.next_with_snapshot(repo, slug)
|
|
1737
|
+
except caddis_roadmap.RoadmapError as exc:
|
|
1738
|
+
raise _Stop(f"FAILED roadmap not read: {exc}", exc.code, push=False) from None
|
|
1739
|
+
waiting = _read_record(_state_file(repo, slug, "waiting"))
|
|
1740
|
+
if waiting: # a real start begins with the recorded wait
|
|
1741
|
+
ref, url = str(waiting.get("ref", "")), str(waiting.get("pr", ""))
|
|
1742
|
+
try:
|
|
1743
|
+
merged = pr_merged(repo, url)
|
|
1744
|
+
except Exception as exc:
|
|
1745
|
+
raise _Stop(f"{GH_STOP_PREFIX} {url} with account {_gh_who()}: {type(exc).__name__}",
|
|
1746
|
+
caddis_exit.BLOCKED, push=False) from None
|
|
1747
|
+
if not merged:
|
|
1748
|
+
print(f"still waiting for merge of {ref}: {url} (a real start stops here)")
|
|
1749
|
+
return caddis_exit.CLEAN
|
|
1750
|
+
if snap.phases.state(ref) != "done":
|
|
1751
|
+
raise _Stop(f"BLOCKED {ref} merged but its Tracker row is not done", caddis_exit.BLOCKED)
|
|
1752
|
+
print(f"merged path: {ref} merged ({url}); would check the roadmap against the published one, roll up "
|
|
1753
|
+
f"{waiting.get('milestone')} when every phase is merged, then remove the claim and "
|
|
1754
|
+
f"{waiting.get('branch')}, then take the next batch")
|
|
1755
|
+
return caddis_exit.CLEAN
|
|
1756
|
+
if batch is None:
|
|
1757
|
+
raise _Stop(f"BLOCKED nothing ready: {_status_line(repo, slug)}", caddis_exit.BLOCKED)
|
|
1758
|
+
except _Stop as stop:
|
|
1759
|
+
print(f"would stop: {caddis_redact.redact(stop.reason, masks)}")
|
|
1760
|
+
return stop.code if stop.code != caddis_exit.CLEAN else caddis_exit.NOT_RUN
|
|
1761
|
+
mid, plan, phase = batch["milestone"], batch["plan"], batch["phase"]
|
|
1762
|
+
ready = rollup_ready(snap, _read_record(_state_file(repo, slug, "rollup-failed")))
|
|
1763
|
+
if ready:
|
|
1764
|
+
print(f"would roll up first: {', '.join(ready)}")
|
|
1765
|
+
pending = _read_record(_state_file(repo, slug, "pending-rollup"))
|
|
1766
|
+
if pending:
|
|
1767
|
+
print(f"pending rollup: {pending.get('ref')} (would roll up {pending.get('milestone')} and clean up "
|
|
1768
|
+
f"{pending.get('branch')} first)")
|
|
1769
|
+
print(f"next: {mid} {plan}#{phase}")
|
|
1770
|
+
print(f"roadmap {slug}: runner host {snap.roadmap.runner_host or '(none)'} (not checked in a dry run); "
|
|
1771
|
+
f"lock {lock.path}; local files in {ctx.target.local}")
|
|
1772
|
+
lane = phase_lane(_plan_text(snap, plan), phase) or "none"
|
|
1773
|
+
if not lane_runs_here(lane, ctx.engine):
|
|
1774
|
+
print(f"would block {mid}: owner-gate phase {phase} lane: {lane}")
|
|
1775
|
+
return caddis_exit.CLEAN
|
|
1776
|
+
if snap.roadmap.get(mid).status == "todo":
|
|
1777
|
+
print(f"would set {mid} Status in-progress (docs PR)")
|
|
1778
|
+
wt = worktree_path(repo, slug, worktree_root)
|
|
1779
|
+
if wt.resolve() in _registered_worktrees(repo):
|
|
1780
|
+
print(f"would reuse the roadmap worktree {wt}")
|
|
1781
|
+
else:
|
|
1782
|
+
print(f"would create the roadmap worktree {wt} from origin/{snap.branch} (git worktree add --detach)")
|
|
1783
|
+
print(f"would make the session branch {session_branch(slug, plan, phase)} from origin/{snap.branch} in it")
|
|
1784
|
+
_dry_run_tail(ctx, template, render_prompt(template, plan, phase), cwd=wt)
|
|
1785
|
+
print("then: on CONTINUE with new commits on that branch, push it, open a PR and stop "
|
|
1786
|
+
f"`{WAITING_PREFIX}`; the recheck task restarts the runner after the merge")
|
|
1787
|
+
return caddis_exit.CLEAN
|
|
1788
|
+
|
|
1789
|
+
try: # the Runner host first, read only: a wrong-host start must change nothing (no lock, no record)
|
|
1790
|
+
host = caddis_roadmap.load(repo, slug).roadmap.runner_host
|
|
1791
|
+
except caddis_roadmap.RoadmapError:
|
|
1792
|
+
host = None # the read below, under the lock, reports and records it
|
|
1793
|
+
if host is not None and host.casefold() != hostname.casefold():
|
|
1794
|
+
return _finish(_Stop(f"BLOCKED this host {hostname} is not the roadmap's Runner host {host}",
|
|
1795
|
+
caddis_exit.NOT_RUN, push=False), ctx, False, 0, 0.0, None)
|
|
1796
|
+
try:
|
|
1797
|
+
lock.acquire(lambda message: _stale_notice(ctx, message))
|
|
1798
|
+
except _Stop as stop:
|
|
1799
|
+
return _finish(stop, ctx, False, 0, 0.0, None)
|
|
1800
|
+
except OSError as exc:
|
|
1801
|
+
return _finish(_Stop(f"BLOCKED claim not written: the runner lock ({type(exc).__name__})",
|
|
1802
|
+
caddis_exit.BLOCKED), ctx, False, 0, 0.0, None)
|
|
1803
|
+
try:
|
|
1804
|
+
first = caddis_roadmap.load(repo, slug)
|
|
1805
|
+
except caddis_roadmap.RoadmapError as exc: # like any stop: notify once and record it (not restartable)
|
|
1806
|
+
ctx.last_stop = last_stop_file(repo, "roadmap", slug)
|
|
1807
|
+
try:
|
|
1808
|
+
return _finish(_Stop(f"FAILED roadmap not read: {str(exc).splitlines()[0]}", exc.code), ctx, False, 0,
|
|
1809
|
+
0.0, None)
|
|
1810
|
+
finally:
|
|
1811
|
+
lock.release()
|
|
1812
|
+
if first.roadmap.runner_host.casefold() != hostname.casefold():
|
|
1813
|
+
lock.release()
|
|
1814
|
+
return _finish(_Stop(f"BLOCKED this host {hostname} is not the roadmap's Runner host "
|
|
1815
|
+
f"{first.roadmap.runner_host}", caddis_exit.NOT_RUN, push=False), ctx, False, 0, 0.0, None)
|
|
1816
|
+
waiting_file = _state_file(repo, slug, "waiting")
|
|
1817
|
+
waiting = _read_record(waiting_file) # the waiting-for-merge data lives here, never in the last stop
|
|
1818
|
+
merged_by_gh = tracker_done = False
|
|
1819
|
+
if waiting:
|
|
1820
|
+
ref, url = str(waiting.get("ref", "")), str(waiting.get("pr", ""))
|
|
1821
|
+
tracker_done = first.phases.state(ref) == "done"
|
|
1822
|
+
try: # GitHub must confirm the merge, also when the Tracker row is done: no branch goes on the row alone
|
|
1823
|
+
merged_by_gh = pr_merged(repo, url)
|
|
1824
|
+
closed = not merged_by_gh and pr_closed(repo, url)
|
|
1825
|
+
except Exception as exc: # gh missing, 401, 404: a recorded, notified stop; the lock is released
|
|
1826
|
+
gh_repo = re.search(r"github\.com/([^/]+/[^/]+)/pull/", url or "")
|
|
1827
|
+
ctx.last_stop = last_stop_file(repo, "roadmap", slug)
|
|
1828
|
+
try:
|
|
1829
|
+
return _finish(_Stop(f"{GH_STOP_PREFIX} {gh_repo.group(1) if gh_repo else url} with account "
|
|
1830
|
+
f"{_gh_who()}: {type(exc).__name__}", caddis_exit.BLOCKED),
|
|
1831
|
+
ctx, False, 0, 0.0, None) # waiting.json stays: the next start resumes
|
|
1832
|
+
finally:
|
|
1833
|
+
lock.release()
|
|
1834
|
+
if tracker_done and not merged_by_gh: # the row says done, GitHub does not: keep everything, owner decides
|
|
1835
|
+
ctx.last_stop = last_stop_file(repo, "roadmap", slug)
|
|
1836
|
+
try:
|
|
1837
|
+
return _finish(_Stop(f"BLOCKED {ref} Tracker row is done but PR {url} is not merged",
|
|
1838
|
+
caddis_exit.BLOCKED), ctx, False, 0, 0.0, None)
|
|
1839
|
+
finally:
|
|
1840
|
+
lock.release()
|
|
1841
|
+
if not tracker_done:
|
|
1842
|
+
if closed: # closed without a merge: clean up under the lock
|
|
1843
|
+
claim = claim_file(repo, ref) if ref else None
|
|
1844
|
+
if claim is not None:
|
|
1845
|
+
claim.unlink(missing_ok=True)
|
|
1846
|
+
waiting_file.unlink(missing_ok=True)
|
|
1847
|
+
ctx.last_stop = last_stop_file(repo, "roadmap", slug)
|
|
1848
|
+
try:
|
|
1849
|
+
return _finish(_Stop(f"BLOCKED {ref} PR closed without a merge", caddis_exit.BLOCKED),
|
|
1850
|
+
ctx, False, 0, 0.0, None)
|
|
1851
|
+
finally:
|
|
1852
|
+
lock.release()
|
|
1853
|
+
if not merged_by_gh: # still waiting: stop now, leave the waiting record, the claim and the last stop
|
|
1854
|
+
lock.release()
|
|
1855
|
+
return _finish(_Stop(f"BLOCKED still waiting for merge of {ref}: {url}", caddis_exit.BLOCKED,
|
|
1856
|
+
push=False), ctx, False, 0, 0.0, None)
|
|
1857
|
+
_mark_running(ctx, last_stop_file(repo, "roadmap", slug))
|
|
1858
|
+
if not ENGINES[ctx.engine]["live_tested"]:
|
|
1859
|
+
print(f"caddis_roadmap_run: engine {ctx.engine} is an adapter, not live-tested", file=sys.stderr)
|
|
1860
|
+
branches_file = _state_file(repo, slug, "branches")
|
|
1861
|
+
made_branches = set(_read_record(branches_file).get("branches") or [])
|
|
1862
|
+
|
|
1863
|
+
sessions, spent, stuck, pushed = 0, 0.0, 0, False # per run, in memory
|
|
1864
|
+
pending_file = _state_file(repo, slug, "pending-rollup")
|
|
1865
|
+
swept = False
|
|
1866
|
+
relay_slug: str | None = None
|
|
1867
|
+
last_write: tuple[str, str] | None = None
|
|
1868
|
+
wt: Path | None = None
|
|
1869
|
+
start = ctx.clock()
|
|
1870
|
+
|
|
1871
|
+
try:
|
|
1872
|
+
baseline = _Baseline(first.roadmap)
|
|
1873
|
+
if waiting:
|
|
1874
|
+
# A restart after `BLOCKED waiting for merge`: the merged origin roadmap must still be the one the runner
|
|
1875
|
+
# published against. Never take a fresh baseline without this check.
|
|
1876
|
+
saved = _read_text(_state_file(repo, slug, "published-roadmap").with_suffix(".md"))
|
|
1877
|
+
hint = (f" (if the owner made this change: roadmap-run.ps1 -Roadmap {slug} -AcceptRoadmap, then start "
|
|
1878
|
+
"again)")
|
|
1879
|
+
if waiting.get("roadmap_sha") != _roadmap_sha(first.text):
|
|
1880
|
+
if saved is None or _roadmap_sha(saved) != waiting.get("roadmap_sha"):
|
|
1881
|
+
raise _Stop("FAILED roadmap changed: the origin roadmap differs from the one published against"
|
|
1882
|
+
+ hint, caddis_exit.BLOCKED)
|
|
1883
|
+
try:
|
|
1884
|
+
_Baseline(caddis_roadmap.parse_roadmap(saved)).observe(first)
|
|
1885
|
+
except _Stop as stop:
|
|
1886
|
+
raise _Stop(stop.reason + hint, stop.code) from None
|
|
1887
|
+
ref = str(waiting.get("ref", ""))
|
|
1888
|
+
claim = claim_file(repo, ref) if ref else None
|
|
1889
|
+
if claim is not None and (not waiting.get("claim_sha") or not claim.is_file()
|
|
1890
|
+
or _sha_file(claim) != waiting.get("claim_sha")): # fail closed
|
|
1891
|
+
raise _Stop(f"FAILED own claim missing or changed: {ref}", caddis_exit.BLOCKED)
|
|
1892
|
+
merged = {"ref": ref, "pr": waiting.get("pr"), "milestone": waiting.get("milestone"),
|
|
1893
|
+
"claim": str(claim) if claim is not None else None, "branch": waiting.get("branch")}
|
|
1894
|
+
if merged_by_gh and not tracker_done: # GitHub says merged, the row is not done: the owner decides
|
|
1895
|
+
_clean_merged(ctx, repo, slug, merged, worktree_root, delete_remote_branch, made_branches,
|
|
1896
|
+
branches_file)
|
|
1897
|
+
waiting_file.unlink(missing_ok=True)
|
|
1898
|
+
raise _Stop(f"BLOCKED {ref} merged but its Tracker row is not done", caddis_exit.BLOCKED)
|
|
1899
|
+
if tracker_done and merged_by_gh:
|
|
1900
|
+
# The merged work moves to its own file (rollup first, cleanup after, then the file goes), and the
|
|
1901
|
+
# wait is over. Any stop before that records itself; the next start finds pending-rollup.json.
|
|
1902
|
+
_write_record(pending_file, {**merged, "utc": _iso(datetime.now(timezone.utc))})
|
|
1903
|
+
waiting_file.unlink(missing_ok=True)
|
|
1904
|
+
_check_orphaned_claims(repo, slug, _read_record(waiting_file), _read_record(pending_file))
|
|
1905
|
+
while True:
|
|
1906
|
+
ctx.checks(sessions, spent, ctx.clock() - start)
|
|
1907
|
+
pending = _read_record(pending_file)
|
|
1908
|
+
if pending:
|
|
1909
|
+
mid = pending.get("milestone")
|
|
1910
|
+
if isinstance(mid, str):
|
|
1911
|
+
snap = _load_observed(repo, slug, baseline)
|
|
1912
|
+
ms = snap.roadmap.get(mid)
|
|
1913
|
+
if ms is not None and ms.status in caddis_roadmap.OPEN and ms.phases and \
|
|
1914
|
+
all(snap.phases.state(r) == "done" for r in ms.phases):
|
|
1915
|
+
rollup(mid) # raises _Stop on a failed rollup: recorded, the file stays
|
|
1916
|
+
_clean_merged(ctx, repo, slug, pending, worktree_root, delete_remote_branch, made_branches,
|
|
1917
|
+
branches_file)
|
|
1918
|
+
pending_file.unlink(missing_ok=True)
|
|
1919
|
+
if not swept: # once per run: roll up every milestone that waits only for its rollup
|
|
1920
|
+
swept = True
|
|
1921
|
+
snap = _load_observed(repo, slug, baseline)
|
|
1922
|
+
failed_file = _state_file(repo, slug, "rollup-failed")
|
|
1923
|
+
for mid in rollup_ready(snap, _read_record(failed_file)):
|
|
1924
|
+
if rollup(mid) == 0: # nothing written: a recorded stop, and recheck must not retry it
|
|
1925
|
+
_write_record(failed_file, {**_read_record(failed_file), mid: _roadmap_sha(snap.text)})
|
|
1926
|
+
raise _Stop(f"BLOCKED rollup of {mid} wrote nothing; the owner decides", caddis_exit.BLOCKED)
|
|
1927
|
+
with _claim(repo): # choose and claim atomically: no other runner can choose the same batch
|
|
1928
|
+
try:
|
|
1929
|
+
batch, snap = caddis_roadmap.next_with_snapshot(repo, slug)
|
|
1930
|
+
except caddis_roadmap.RoadmapError as exc:
|
|
1931
|
+
raise _Stop(f"FAILED roadmap changed: the origin copy no longer loads: "
|
|
1932
|
+
f"{str(exc).splitlines()[0]}", caddis_exit.BLOCKED) from None
|
|
1933
|
+
baseline.observe(snap)
|
|
1934
|
+
if batch is not None:
|
|
1935
|
+
lock.set_batch(f"{batch['plan']}#{batch['phase']}")
|
|
1936
|
+
if batch is None:
|
|
1937
|
+
detail = _status_line(repo, slug)
|
|
1938
|
+
if snap.roadmap.milestones and all(ms.status in caddis_roadmap.FINISHED
|
|
1939
|
+
for ms in snap.roadmap.milestones):
|
|
1940
|
+
gone = remove_worktree_if_clean(repo, slug, worktree_root)
|
|
1941
|
+
detail += "; roadmap finished; " + gone
|
|
1942
|
+
if gone.startswith(("worktree removed", "no worktree")):
|
|
1943
|
+
detail += "; " + _cleanup_finished_branches(repo, slug, snap.branch, pr_merged,
|
|
1944
|
+
delete_remote_branch)
|
|
1945
|
+
raise _Stop(RESTARTABLE_STOP, caddis_exit.BLOCKED, detail=detail,
|
|
1946
|
+
record={"held": _held_batches(repo, slug, snap)})
|
|
1947
|
+
mid, plan, phase = batch["milestone"], batch["plan"], batch["phase"]
|
|
1948
|
+
ref = f"{plan}#{phase}"
|
|
1949
|
+
print(f"next: {mid} {ref}")
|
|
1950
|
+
if last_write == (mid, ref): # the runner's own block did not take the batch off `next`
|
|
1951
|
+
raise _Stop(f"FAILED roadmap unchanged after the runner's write for {mid} {ref}",
|
|
1952
|
+
caddis_exit.BLOCKED)
|
|
1953
|
+
relay_slug = plan
|
|
1954
|
+
plan_text = _plan_text(snap, plan)
|
|
1955
|
+
lane = phase_lane(plan_text, phase) or "none" # no Lane line: not the runner's (fail closed)
|
|
1956
|
+
if not lane_runs_here(lane, ctx.engine):
|
|
1957
|
+
reason = f"phase {phase} lane: {' '.join(lane.split())[:160]}".strip()
|
|
1958
|
+
write(["block", "--id", mid, "--type", "owner-gate", "--reason", reason],
|
|
1959
|
+
("blocker", mid, "owner-gate", reason))
|
|
1960
|
+
last_write = (mid, ref)
|
|
1961
|
+
continue
|
|
1962
|
+
last_write = None
|
|
1963
|
+
if snap.roadmap.get(mid).status == "todo":
|
|
1964
|
+
write(["set", "--id", mid, "--col", "Status", "--value", "in-progress"],
|
|
1965
|
+
("status", mid, "in-progress"))
|
|
1966
|
+
snap = _load_observed(repo, slug, baseline) # the write window is guarded too
|
|
1967
|
+
# Every session runs in the roadmap's own worktree, on a fresh branch made from origin/<default>.
|
|
1968
|
+
wt = wt or ensure_worktree(repo, slug, snap.branch, worktree_root)
|
|
1969
|
+
branch = session_branch(slug, plan, phase)
|
|
1970
|
+
if branch not in made_branches and _branch_on_origin(repo, branch):
|
|
1971
|
+
raise _Stop(f"BLOCKED branch {branch} is on origin and this runner did not make it",
|
|
1972
|
+
caddis_exit.BLOCKED)
|
|
1973
|
+
for ref_name in (f"refs/heads/{branch}", f"refs/remotes/origin/{branch}"):
|
|
1974
|
+
if _git_out(wt, "rev-parse", "--verify", "--quiet", ref_name) is None:
|
|
1975
|
+
continue
|
|
1976
|
+
ahead = _git_out(wt, "rev-list", "--count", f"origin/{snap.branch}..{ref_name}")
|
|
1977
|
+
if ahead is None or ahead != "0": # an earlier session's work: never reset it
|
|
1978
|
+
raise _Stop(f"BLOCKED {branch} has unpublished commits from an earlier session",
|
|
1979
|
+
caddis_exit.BLOCKED)
|
|
1980
|
+
left = _worktree_work(wt) # a blocked session's uncommitted work must never ride into another branch
|
|
1981
|
+
if left is None or left:
|
|
1982
|
+
was = _git_out(wt, "rev-parse", "--abbrev-ref", "HEAD") or "?"
|
|
1983
|
+
raise _Stop(f"BLOCKED roadmap worktree has uncommitted work from {was} ({wt})", caddis_exit.BLOCKED)
|
|
1984
|
+
made = _git_out(wt, "checkout", "-q", "-B", branch, f"origin/{snap.branch}")
|
|
1985
|
+
if made is None:
|
|
1986
|
+
raise _Stop(f"FAILED cannot make the session branch {branch} in {wt} (is it clean?)",
|
|
1987
|
+
caddis_exit.BLOCKED)
|
|
1988
|
+
head_before = _git_out(wt, "rev-parse", "HEAD")
|
|
1989
|
+
done_before = _done_phases(snap, plan)
|
|
1990
|
+
state_before = state_snapshot(repo, "roadmap", slug)
|
|
1991
|
+
s = _one_session(ctx, render_prompt(template, plan, phase), plan, sessions + 1, spent,
|
|
1992
|
+
ctx.clock() - start, slug, cwd=wt)
|
|
1993
|
+
_check_state(repo, "roadmap", slug, state_before) # before the runner writes anything after it
|
|
1994
|
+
sessions += 1
|
|
1995
|
+
spent += s.cost
|
|
1996
|
+
pushed = s.pushed
|
|
1997
|
+
s.record.update({"milestone": mid, "batch": ref, "branch": branch})
|
|
1998
|
+
try:
|
|
1999
|
+
stop = _decide(ctx.engine, s.outcome, s.result, s.relay_text, s.hash_before, s.hash_after)
|
|
2000
|
+
s.record["status"] = "CONTINUE" if stop is None else stop.reason
|
|
2001
|
+
after = _load_observed(repo, slug, baseline) # one read after the session (no merge wait)
|
|
2002
|
+
stuck = 0 if _done_phases(after, plan) > done_before else stuck + 1
|
|
2003
|
+
if stop is not None:
|
|
2004
|
+
if stop.status is None or stop.status[0] not in ("OWNER-GATE", "BLOCKED"):
|
|
2005
|
+
raise stop
|
|
2006
|
+
btype = "owner-gate" if stop.status[0] == "OWNER-GATE" else "external"
|
|
2007
|
+
reason = f"{ref}: {stop.status[1]}".strip()
|
|
2008
|
+
write(["block", "--id", mid, "--type", btype, "--reason", reason], ("blocker", mid, btype, reason))
|
|
2009
|
+
last_write = (mid, ref)
|
|
2010
|
+
elif after.phases.state(ref) != "done":
|
|
2011
|
+
def review_before_publish() -> None:
|
|
2012
|
+
# The review gate runs BEFORE the claim and the push: a `risk: high` phase is never
|
|
2013
|
+
# published unreviewed. Its review and fix session count against the caps.
|
|
2014
|
+
nonlocal sessions, spent, pushed
|
|
2015
|
+
tally = _Tally(sessions, spent, pushed)
|
|
2016
|
+
try:
|
|
2017
|
+
reviewed = _review_gate(ctx, tally, plan=plan, extra_texts=(plan_text,),
|
|
2018
|
+
origin_text=_plan_text(after, plan) or None,
|
|
2019
|
+
elapsed=lambda: ctx.clock() - start, spend_slug=slug,
|
|
2020
|
+
also=(phase,), cwd=wt)
|
|
2021
|
+
finally:
|
|
2022
|
+
sessions, spent, pushed = tally.sessions, tally.spent, tally.pushed
|
|
2023
|
+
if reviewed:
|
|
2024
|
+
_load_observed(repo, slug, baseline) # the review and fix window is guarded too
|
|
2025
|
+
now_on = _git_out(wt, "rev-parse", "--abbrev-ref", "HEAD")
|
|
2026
|
+
if now_on != branch:
|
|
2027
|
+
raise _Stop(f"FAILED {ref}: the review fix round left the runner's branch {branch} "
|
|
2028
|
+
f"(now on {now_on})", caddis_exit.BLOCKED)
|
|
2029
|
+
|
|
2030
|
+
_publish_and_wait(ctx, wt, slug, mid, plan, phase, ref, branch, head_before, after, plan_text,
|
|
2031
|
+
publish, made_branches, branches_file, s.record,
|
|
2032
|
+
before_publish=review_before_publish)
|
|
2033
|
+
else:
|
|
2034
|
+
ms = after.roadmap.get(mid)
|
|
2035
|
+
if ms is not None and all(after.phases.state(r) == "done" for r in ms.phases):
|
|
2036
|
+
rollup(mid)
|
|
2037
|
+
if stuck >= STUCK_SESSIONS:
|
|
2038
|
+
raise _Stop("BLOCKED stuck", caddis_exit.BLOCKED)
|
|
2039
|
+
except _Stop as stop_now:
|
|
2040
|
+
s.record.setdefault("stop", stop_now.reason)
|
|
2041
|
+
raise
|
|
2042
|
+
finally:
|
|
2043
|
+
s.record.setdefault("status", "FAILED")
|
|
2044
|
+
ctx.log(s.record)
|
|
2045
|
+
if after_session is not None:
|
|
2046
|
+
after_session(sessions)
|
|
2047
|
+
except _Stop as stop:
|
|
2048
|
+
return _finish(stop, ctx, pushed, sessions, spent, relay_slug, wt)
|
|
2049
|
+
except Exception as exc:
|
|
2050
|
+
_crashed(ctx, exc, pushed, sessions, spent, relay_slug, wt)
|
|
2051
|
+
raise
|
|
2052
|
+
finally:
|
|
2053
|
+
lock.release()
|
|
2054
|
+
|
|
2055
|
+
|
|
2056
|
+
def _sha_file(path: Path) -> str:
|
|
2057
|
+
try:
|
|
2058
|
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
2059
|
+
except OSError:
|
|
2060
|
+
return "unreadable"
|
|
2061
|
+
|
|
2062
|
+
|
|
2063
|
+
def _claims_of(repo: Path, kind: str, slug: str) -> dict[str, Path]:
|
|
2064
|
+
"""This runner's claims (`claims/*.json` whose `roadmap` field, or `plan` for plan mode, is `slug`)."""
|
|
2065
|
+
base = caddis_roadmap.runner_state_dir(repo)
|
|
2066
|
+
out: dict[str, Path] = {}
|
|
2067
|
+
if base is None:
|
|
2068
|
+
return out
|
|
2069
|
+
try:
|
|
2070
|
+
files = sorted((base / "claims").glob("*.json"))
|
|
2071
|
+
except OSError:
|
|
2072
|
+
return out
|
|
2073
|
+
for path in files:
|
|
2074
|
+
if _read_record(path).get(kind) == slug:
|
|
2075
|
+
out[f"claims/{path.name}"] = path
|
|
2076
|
+
return out
|
|
2077
|
+
|
|
2078
|
+
|
|
2079
|
+
def state_snapshot(repo: Path, kind: str, slug: str) -> dict[str, str]:
|
|
2080
|
+
"""sha256 of every file in THIS runner's folder (`<git common dir>/caddis-runner/<kind>-<slug>/`, recursively,
|
|
2081
|
+
so later records such as `reviewed/` too) and of this runner's own claims. Another runner's state is not
|
|
2082
|
+
watched: it never stops this one. Outside a git repo: empty."""
|
|
2083
|
+
base = caddis_roadmap.runner_state_dir(repo)
|
|
2084
|
+
out: dict[str, str] = {}
|
|
2085
|
+
if base is None:
|
|
2086
|
+
return out
|
|
2087
|
+
own = base / f"{kind}-{slug}"
|
|
2088
|
+
if own.is_dir():
|
|
2089
|
+
for path in sorted(own.rglob("*")):
|
|
2090
|
+
if path.is_file():
|
|
2091
|
+
out[path.relative_to(base).as_posix()] = _sha_file(path)
|
|
2092
|
+
for name, path in _claims_of(repo, kind, slug).items():
|
|
2093
|
+
out[name] = _sha_file(path)
|
|
2094
|
+
return out
|
|
2095
|
+
|
|
2096
|
+
|
|
2097
|
+
def _state_changes(before: dict[str, str], after: dict[str, str]) -> list[str]:
|
|
2098
|
+
changes = [f"{name} removed" for name in before if name not in after]
|
|
2099
|
+
changes += [f"{name} added" for name in after if name not in before]
|
|
2100
|
+
changes += [f"{name} changed" for name in before if name in after and before[name] != after[name]]
|
|
2101
|
+
return changes
|
|
2102
|
+
|
|
2103
|
+
|
|
2104
|
+
def _check_state(repo: Path, kind: str, slug: str, before: dict[str, str]) -> None:
|
|
2105
|
+
"""The runner writes none of its own state while a session runs, so a change there is a session's mistake:
|
|
2106
|
+
stop, not restartable. Sessions are cooperative; this catches accidents, it is not a security boundary."""
|
|
2107
|
+
changes = _state_changes(before, state_snapshot(repo, kind, slug))
|
|
2108
|
+
if changes:
|
|
2109
|
+
raise _Stop("FAILED runner state changed: " + "; ".join(changes[:5]), caddis_exit.BLOCKED)
|
|
2110
|
+
|
|
2111
|
+
|
|
2112
|
+
def _read_text(path: Path) -> str | None:
|
|
2113
|
+
try:
|
|
2114
|
+
return Path(path).read_bytes().decode("utf-8")
|
|
2115
|
+
except OSError:
|
|
2116
|
+
return None
|
|
2117
|
+
|
|
2118
|
+
|
|
2119
|
+
def _publish_and_wait(ctx: _Ctx, repo: Path, slug: str, mid: str, plan: str, phase: int, ref: str, branch: str,
|
|
2120
|
+
head_before: str | None, after: "caddis_roadmap.Snapshot", plan_text: str,
|
|
2121
|
+
publish: Callable[..., str], made_branches: set, branches_file: Path, record: dict,
|
|
2122
|
+
before_publish: Callable[[], None] | None = None) -> None:
|
|
2123
|
+
"""CONTINUE with the phase not merged: publish the runner's session branch and raise the waiting stop.
|
|
2124
|
+
Refuses (FAILED, not restartable) when the session left that branch, made no commit, or touched the roadmap."""
|
|
2125
|
+
def verify() -> None:
|
|
2126
|
+
current = _git_out(repo, "rev-parse", "--abbrev-ref", "HEAD")
|
|
2127
|
+
if current != branch:
|
|
2128
|
+
raise _Stop(f"FAILED {ref}: the session left the runner's branch {branch} (now on {current})",
|
|
2129
|
+
caddis_exit.BLOCKED)
|
|
2130
|
+
head_after = _git_out(repo, "rev-parse", "HEAD")
|
|
2131
|
+
ahead = _git_out(repo, "rev-list", "--count", f"origin/{after.branch}..{branch}")
|
|
2132
|
+
if head_after == head_before or not ahead or ahead == "0":
|
|
2133
|
+
raise _Stop(f"FAILED {ref}: CONTINUE with no new commits on {branch} and not merged",
|
|
2134
|
+
caddis_exit.BLOCKED)
|
|
2135
|
+
listing = _git_out(repo, "diff", "--name-only", f"origin/{after.branch}...{branch}")
|
|
2136
|
+
if listing is None: # fail closed: without the list of changed files, never publish
|
|
2137
|
+
raise _Stop(f"FAILED {ref}: cannot list the session's changes (git diff --name-only failed); "
|
|
2138
|
+
"not published", caddis_exit.BLOCKED)
|
|
2139
|
+
touched = [p for p in listing.splitlines() if p.replace("\\", "/").startswith(".caddis/roadmaps/")]
|
|
2140
|
+
if touched:
|
|
2141
|
+
raise _Stop(f"FAILED session changed the roadmap ({', '.join(touched[:3])}); not published",
|
|
2142
|
+
caddis_exit.BLOCKED)
|
|
2143
|
+
|
|
2144
|
+
verify()
|
|
2145
|
+
if before_publish is not None: # the review gate (caddis-next Phase 11); a fix session may have committed
|
|
2146
|
+
before_publish()
|
|
2147
|
+
verify()
|
|
2148
|
+
claim = claim_file(repo, ref)
|
|
2149
|
+
utc = _iso(datetime.now(timezone.utc))
|
|
2150
|
+
if claim is not None: # the phase stays held while its PR waits (written before the push)
|
|
2151
|
+
try:
|
|
2152
|
+
claim.parent.mkdir(parents=True, exist_ok=True)
|
|
2153
|
+
_replace_with(claim, json.dumps({"batch": ref, "roadmap": slug, "pr": None, "utc": utc}))
|
|
2154
|
+
except OSError as exc:
|
|
2155
|
+
raise _Stop(f"BLOCKED claim not written: {ref} ({type(exc).__name__})", caddis_exit.BLOCKED) from None
|
|
2156
|
+
ctx.written_claims.append(claim)
|
|
2157
|
+
try: # from here to the waiting record, ANY failure removes the claim and stops FAILED (not restartable)
|
|
2158
|
+
title = f"{plan} phase {phase}" + (f": {_phase_title(plan_text, phase)}"
|
|
2159
|
+
if _phase_title(plan_text, phase) else "")
|
|
2160
|
+
body = f"Opened by caddis_roadmap_run.py for roadmap {slug}, milestone {mid}, {ref}."
|
|
2161
|
+
made_branches.add(branch) # recorded BEFORE the push: a push that lands and a PR that fails do not lock it out
|
|
2162
|
+
_write_record(branches_file, {**_read_record(branches_file), "branches": sorted(made_branches)})
|
|
2163
|
+
url = publish(repo, branch, after.branch, title, body)
|
|
2164
|
+
prs = dict(_read_record(branches_file).get("prs") or {})
|
|
2165
|
+
prs[branch] = url # the finished roadmap uses it to tell a squash-merged branch from an unmerged one
|
|
2166
|
+
_write_record(branches_file, {"branches": sorted(made_branches), "prs": prs})
|
|
2167
|
+
claim_sha = None
|
|
2168
|
+
if claim is not None:
|
|
2169
|
+
_write_record(claim, {"batch": ref, "roadmap": slug, "pr": url, "utc": utc})
|
|
2170
|
+
claim_sha = _sha_file(claim)
|
|
2171
|
+
saved = _state_file(repo, slug, "published-roadmap").with_suffix(".md")
|
|
2172
|
+
try:
|
|
2173
|
+
saved.parent.mkdir(parents=True, exist_ok=True)
|
|
2174
|
+
tmp = saved.with_name(f"{saved.name}.{os.getpid()}.tmp")
|
|
2175
|
+
tmp.write_bytes(after.text.encode("utf-8")) # bytes: the exact text the hash was taken of
|
|
2176
|
+
os.replace(tmp, saved)
|
|
2177
|
+
except OSError as exc:
|
|
2178
|
+
print(f"caddis_roadmap_run: published roadmap not saved: {exc}", file=sys.stderr)
|
|
2179
|
+
record["pr"] = url
|
|
2180
|
+
except Exception as exc:
|
|
2181
|
+
if claim is not None:
|
|
2182
|
+
claim.unlink(missing_ok=True)
|
|
2183
|
+
if claim in ctx.written_claims:
|
|
2184
|
+
ctx.written_claims.remove(claim)
|
|
2185
|
+
detail = str(exc).splitlines()[0] if str(exc) else type(exc).__name__
|
|
2186
|
+
raise _Stop(f"FAILED {ref}: could not push or open the PR: {detail}", caddis_exit.BLOCKED) from None
|
|
2187
|
+
if claim is not None and claim in ctx.written_claims:
|
|
2188
|
+
ctx.written_claims.remove(claim) # the waiting record below holds it now
|
|
2189
|
+
_write_record(_state_file(repo, slug, "waiting"), # the waiting data, in its own file (never in the last stop)
|
|
2190
|
+
{"ref": ref, "pr": url, "milestone": mid, "branch": branch,
|
|
2191
|
+
"claim": str(claim) if claim is not None else None, "claim_sha": claim_sha,
|
|
2192
|
+
"roadmap_sha": _roadmap_sha(after.text), "utc": _iso(datetime.now(timezone.utc))})
|
|
2193
|
+
raise _Stop(f"{WAITING_PREFIX} {ref} {url}", caddis_exit.BLOCKED)
|
|
2194
|
+
|
|
2195
|
+
|
|
2196
|
+
def _spend_reserve(home: Path, slug: str, moment: datetime, started: bool) -> str:
|
|
2197
|
+
"""Book one agy session under the global spend cap (caddis_spend.py); _Stop when the cap refuses it or cannot be
|
|
2198
|
+
read (a file error never crashes the runner: the stop is logged and notified, redacted, like any other).
|
|
2199
|
+
The caller name in the spend log is `runner-<slug>`."""
|
|
2200
|
+
try:
|
|
2201
|
+
rid = caddis_spend.reserve(home, f"runner-{slug}", moment)
|
|
2202
|
+
except caddis_spend.NotConfigured:
|
|
2203
|
+
code = caddis_exit.BLOCKED if started else caddis_exit.NOT_RUN
|
|
2204
|
+
raise _Stop("FAILED spend cap not configured", code) from None
|
|
2205
|
+
except TimeoutError:
|
|
2206
|
+
raise _Stop("BLOCKED spend cap lock busy", caddis_exit.BLOCKED) from None
|
|
2207
|
+
except Exception as exc:
|
|
2208
|
+
raise _Stop(f"FAILED spend cap error: {exc!r}", caddis_exit.BLOCKED) from None
|
|
2209
|
+
if rid is None:
|
|
2210
|
+
raise _Stop("BLOCKED spend cap", caddis_exit.BLOCKED)
|
|
2211
|
+
return rid
|
|
2212
|
+
|
|
2213
|
+
|
|
2214
|
+
def _spend_settle(home: Path, rid: str, cmd: Sequence[str], outcome: "_Outcome | None", moment: datetime,
|
|
2215
|
+
duration_s: float) -> None:
|
|
2216
|
+
"""Record the session's real cost: (0, 0) when it never started, its reserve (estimated) when there is no outcome
|
|
2217
|
+
(an exception) or no readable token counts."""
|
|
2218
|
+
try:
|
|
2219
|
+
if outcome is None:
|
|
2220
|
+
usage_counts = None
|
|
2221
|
+
elif not outcome.started:
|
|
2222
|
+
usage_counts = (0, 0) # nothing ran, nothing was spent
|
|
2223
|
+
elif caddis_spend.json_mode(list(cmd)):
|
|
2224
|
+
usage_counts = caddis_spend.tokens_from_output(outcome.stdout)
|
|
2225
|
+
else:
|
|
2226
|
+
usage_counts = None
|
|
2227
|
+
caddis_spend.settle(home, rid, usage_counts, moment, duration_s=duration_s)
|
|
2228
|
+
except Exception as exc: # the pending reserve keeps counting for 24 h; never lose the session's result
|
|
2229
|
+
print(f"caddis_roadmap_run: could not settle spend {rid}: {exc}", file=sys.stderr)
|
|
2230
|
+
|
|
2231
|
+
|
|
2232
|
+
def _spend_preview(home: Path, moment: datetime) -> None:
|
|
2233
|
+
"""The dry run's read-only spend check: the same stops as _spend_reserve, with no pending line written."""
|
|
2234
|
+
try:
|
|
2235
|
+
cap = caddis_spend.load_cap(home)
|
|
2236
|
+
over = caddis_spend.committed_last_24h(home, moment) + cap.reserve > cap.eur_per_24h
|
|
2237
|
+
except caddis_spend.NotConfigured:
|
|
2238
|
+
raise _Stop("FAILED spend cap not configured", caddis_exit.NOT_RUN) from None
|
|
2239
|
+
except Exception as exc:
|
|
2240
|
+
raise _Stop(f"FAILED spend cap error: {exc!r}", caddis_exit.NOT_RUN) from None
|
|
2241
|
+
if over:
|
|
2242
|
+
raise _Stop("BLOCKED spend cap", caddis_exit.BLOCKED)
|
|
2243
|
+
|
|
2244
|
+
|
|
2245
|
+
@dataclass
|
|
2246
|
+
class _Outcome:
|
|
2247
|
+
code: int
|
|
2248
|
+
stdout: str
|
|
2249
|
+
stderr: str
|
|
2250
|
+
timed_out: bool = False
|
|
2251
|
+
started: bool = True
|
|
2252
|
+
|
|
2253
|
+
|
|
2254
|
+
def _launch(cmd: list[str], cwd: Path, timeout_s: float, stdin_text: str | None = None,
|
|
2255
|
+
env: dict | None = None) -> _Outcome:
|
|
2256
|
+
try:
|
|
2257
|
+
proc = subprocess.run(cmd, cwd=cwd, input=stdin_text if stdin_text is not None else "", capture_output=True,
|
|
2258
|
+
text=True, encoding="utf-8", errors="replace", timeout=timeout_s,
|
|
2259
|
+
env=env if env is not None else headless_env())
|
|
2260
|
+
except subprocess.TimeoutExpired as exc:
|
|
2261
|
+
out = exc.stdout.decode("utf-8", "replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "")
|
|
2262
|
+
err = exc.stderr.decode("utf-8", "replace") if isinstance(exc.stderr, bytes) else (exc.stderr or "")
|
|
2263
|
+
return _Outcome(-1, out, err, timed_out=True)
|
|
2264
|
+
except OSError as exc:
|
|
2265
|
+
return _Outcome(-1, "", f"cannot start session: {exc}", started=False)
|
|
2266
|
+
return _Outcome(proc.returncode, proc.stdout or "", proc.stderr or "")
|
|
2267
|
+
|
|
2268
|
+
|
|
2269
|
+
def _decide(engine: str, outcome: _Outcome, result: dict | None, relay_text: str, hash_before: str | None,
|
|
2270
|
+
hash_after: str | None) -> _Stop | None:
|
|
2271
|
+
"""None to go on; else the stop. A missing or bad status line is FAILED, never CONTINUE (design 8.1)."""
|
|
2272
|
+
failed = _session_failed(engine, outcome, result)
|
|
2273
|
+
if failed is not None:
|
|
2274
|
+
return failed
|
|
2275
|
+
if hash_after is None:
|
|
2276
|
+
return _Stop("FAILED no relay", caddis_exit.BLOCKED)
|
|
2277
|
+
try:
|
|
2278
|
+
kind, arg = caddis_workstreams.parse_status(relay_text)
|
|
2279
|
+
except ValueError as exc:
|
|
2280
|
+
return _Stop(f"FAILED bad status line: {exc}", caddis_exit.BLOCKED)
|
|
2281
|
+
if kind == "CONTINUE":
|
|
2282
|
+
if hash_after == hash_before:
|
|
2283
|
+
return _Stop("FAILED no progress", caddis_exit.BLOCKED)
|
|
2284
|
+
return None
|
|
2285
|
+
if kind == "FAILED":
|
|
2286
|
+
return _Stop(f"FAILED {arg}", caddis_exit.BLOCKED, from_relay=True, status=(kind, arg))
|
|
2287
|
+
return _Stop(f"{kind} {arg}", caddis_exit.CLEAN, from_relay=True, status=(kind, arg))
|
|
2288
|
+
|
|
2289
|
+
|
|
2290
|
+
def _session_failed(engine: str, outcome: _Outcome, result: dict | None) -> _Stop | None:
|
|
2291
|
+
"""The stop for a session that did not finish cleanly (usage limit, timeout, exit code, output, error); None when
|
|
2292
|
+
it did. The relay is not read here: a review fix session writes no status line."""
|
|
2293
|
+
if usage_limited(engine, outcome.stdout + "\n" + outcome.stderr):
|
|
2294
|
+
return _Stop("BLOCKED usage limit", caddis_exit.BLOCKED, push=False)
|
|
2295
|
+
if outcome.timed_out:
|
|
2296
|
+
return _Stop("FAILED session timeout", caddis_exit.BLOCKED)
|
|
2297
|
+
if outcome.code != 0:
|
|
2298
|
+
tail = " ".join(outcome.stderr.split())[-160:]
|
|
2299
|
+
return _Stop(f"FAILED session exit {outcome.code}" + (f": {tail}" if tail else ""), caddis_exit.BLOCKED)
|
|
2300
|
+
if result is None:
|
|
2301
|
+
return _Stop("FAILED session output is not JSON", caddis_exit.BLOCKED)
|
|
2302
|
+
if result.get("is_error") is True:
|
|
2303
|
+
return _Stop(f"FAILED session error: {result.get('subtype', 'unknown')}", caddis_exit.BLOCKED)
|
|
2304
|
+
return None
|
|
2305
|
+
|
|
2306
|
+
|
|
2307
|
+
def _closed(target: Target) -> int:
|
|
2308
|
+
return sum(closed_rows(path) for path in target.progress)
|
|
2309
|
+
|
|
2310
|
+
|
|
2311
|
+
def _finish(stop: _Stop, ctx: _Ctx, pushed: bool, sessions: int, spent: float, relay_slug: str | None,
|
|
2312
|
+
relay_root: Path | None = None) -> int:
|
|
2313
|
+
masks, target = ctx.masks, ctx.target
|
|
2314
|
+
line = f"STATUS: {stop.reason}" if stop.from_relay else f"RUNNER STOP: {stop.reason}"
|
|
2315
|
+
if stop.detail:
|
|
2316
|
+
message = caddis_redact.redact(f"{line}. {stop.detail}", masks)
|
|
2317
|
+
else:
|
|
2318
|
+
relay_text = _read_relay(relay_root or ctx.repo, relay_slug)[0] if relay_slug else ""
|
|
2319
|
+
# Redact BEFORE _next_step cuts the text to 120 characters: a key cut in half is no longer an exact match.
|
|
2320
|
+
relay_text = caddis_redact.redact(relay_text, masks)
|
|
2321
|
+
next_step = caddis_workstreams._next_step(relay_text) if relay_text else "next step unrecorded"
|
|
2322
|
+
message = caddis_redact.redact(f"{line}. Next: {next_step}", masks) # relay text is session-written
|
|
2323
|
+
if ctx.last_stop is not None:
|
|
2324
|
+
_write_options(ctx) # re-written from memory at every stop: a changed file can never raise the caps
|
|
2325
|
+
_write_last_stop(ctx.last_stop, stop) # always the REAL stop
|
|
2326
|
+
toast_exit = ctx.toast(f"caddis {target.kind} {target.slug}", message)
|
|
2327
|
+
push_exit = None
|
|
2328
|
+
if stop.push and not (pushed and stop.from_relay): # PUSH-SENT covers only the stop the session reported
|
|
2329
|
+
push_exit = ctx.push(f"{target.kind} {target.slug}: {message}")
|
|
2330
|
+
ctx.log({"kind": "stop", "stop": stop.reason, "exit": stop.code, "sessions": sessions, "usd": round(spent, 4),
|
|
2331
|
+
"toast_exit": toast_exit, "push_exit": push_exit, "time": _iso(datetime.now(timezone.utc))})
|
|
2332
|
+
print(caddis_redact.redact(f"RUNNER STOP: {stop.reason} (sessions {sessions}, USD {spent:.2f}, exit {stop.code})",
|
|
2333
|
+
masks))
|
|
2334
|
+
return stop.code
|
|
2335
|
+
|
|
2336
|
+
|
|
2337
|
+
def _write_record(path: Path, record: dict) -> None:
|
|
2338
|
+
try:
|
|
2339
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
2340
|
+
_replace_with(path, json.dumps(record))
|
|
2341
|
+
except OSError as exc: # the recheck task then finds an old record; it restarts only after "nothing ready"
|
|
2342
|
+
print(f"caddis_roadmap_run: runner state not written to {path}: {exc}", file=sys.stderr)
|
|
2343
|
+
|
|
2344
|
+
|
|
2345
|
+
def _write_last_stop(path: Path, stop: _Stop) -> None:
|
|
2346
|
+
_write_record(path, {**stop.record, "reason": stop.reason, "exit": stop.code,
|
|
2347
|
+
"utc": _iso(datetime.now(timezone.utc))})
|
|
2348
|
+
|
|
2349
|
+
|
|
2350
|
+
def options_file(last_stop: Path) -> Path:
|
|
2351
|
+
"""The launch options of the run that holds (or last held) the lock, next to its last stop."""
|
|
2352
|
+
return last_stop.with_name("options.json")
|
|
2353
|
+
|
|
2354
|
+
|
|
2355
|
+
def _write_options(ctx: _Ctx) -> None:
|
|
2356
|
+
"""The launch options, from the run's memory (never from the file, which a session can write)."""
|
|
2357
|
+
caps = ctx.caps
|
|
2358
|
+
_write_record(options_file(ctx.last_stop), {
|
|
2359
|
+
"engine": ctx.engine, "model": ctx.model, "max_hours": caps.max_hours, "max_sessions": caps.max_sessions,
|
|
2360
|
+
"session_timeout_min": caps.session_timeout_min, "max_usd_per_session": caps.max_usd_per_session,
|
|
2361
|
+
"max_usd_total": caps.max_usd_total, "gh_account": ctx.gh_account})
|
|
2362
|
+
|
|
2363
|
+
|
|
2364
|
+
def _mark_running(ctx: _Ctx, last_stop: Path) -> None:
|
|
2365
|
+
"""At lock time: the last stop becomes `running` (not restartable) and the launch options are recorded, so a run
|
|
2366
|
+
that is killed without a stop needs the owner, and the recheck task restarts with the same options."""
|
|
2367
|
+
ctx.last_stop = last_stop
|
|
2368
|
+
_write_options(ctx)
|
|
2369
|
+
_write_record(last_stop, {"reason": RUNNING, "exit": None, "utc": _iso(datetime.now(timezone.utc)),
|
|
2370
|
+
"pid": os.getpid()})
|
|
2371
|
+
|
|
2372
|
+
|
|
2373
|
+
def _crashed(ctx: _Ctx, exc: Exception, pushed: bool, sessions: int, spent: float, relay_slug: str | None,
|
|
2374
|
+
relay_root: Path | None = None) -> None:
|
|
2375
|
+
"""An unexpected exception in the loop: record `FAILED crash: <type>` and notify once; the caller re-raises.
|
|
2376
|
+
`relay_root`: where the session relay is (roadmap mode: the roadmap worktree)."""
|
|
2377
|
+
stop = _Stop(f"FAILED crash: {type(exc).__name__}", caddis_exit.BLOCKED)
|
|
2378
|
+
for claim in list(ctx.written_claims): # a claim with no waiting record must not hold the phase for ever
|
|
2379
|
+
Path(claim).unlink(missing_ok=True)
|
|
2380
|
+
ctx.written_claims.clear()
|
|
2381
|
+
try:
|
|
2382
|
+
_finish(stop, ctx, pushed, sessions, spent, relay_slug, relay_root)
|
|
2383
|
+
except Exception as again: # never hide the first exception
|
|
2384
|
+
print(f"caddis_roadmap_run: crash not notified: {again!r}", file=sys.stderr)
|
|
2385
|
+
if ctx.last_stop is not None:
|
|
2386
|
+
_write_last_stop(ctx.last_stop, stop)
|
|
2387
|
+
|
|
2388
|
+
|
|
2389
|
+
_ARG_RE = re.compile(r"^[A-Za-z0-9._\[\]-]+$")
|
|
2390
|
+
_RESTART_ARGS = (("engine", "-Engine"), ("model", "-Model"), ("max_hours", "-MaxHours"),
|
|
2391
|
+
("max_sessions", "-MaxSessions"), ("session_timeout_min", "-SessionTimeoutMin"),
|
|
2392
|
+
("max_usd_per_session", "-MaxUsdPerSession"), ("max_usd_total", "-MaxUsdTotal"),
|
|
2393
|
+
("gh_account", "-GhAccount"))
|
|
2394
|
+
|
|
2395
|
+
|
|
2396
|
+
def restart_args(repo: Path, roadmap: str) -> list[str]:
|
|
2397
|
+
"""`roadmap-run.ps1` arguments that repeat the last run's launch options; [] when none are recorded. ValueError
|
|
2398
|
+
for a record that is not readable or holds an unsafe value."""
|
|
2399
|
+
path = options_file(last_stop_file(Path(repo), "roadmap", roadmap))
|
|
2400
|
+
try:
|
|
2401
|
+
record = json.loads(path.read_text(encoding="utf-8"))
|
|
2402
|
+
except FileNotFoundError:
|
|
2403
|
+
return []
|
|
2404
|
+
except (OSError, ValueError) as exc:
|
|
2405
|
+
raise ValueError(f"launch options not readable: {type(exc).__name__}") from None
|
|
2406
|
+
if not isinstance(record, dict):
|
|
2407
|
+
raise ValueError("launch options are not an object")
|
|
2408
|
+
out: list[str] = []
|
|
2409
|
+
defaults = Caps()
|
|
2410
|
+
for key, flag in _RESTART_ARGS:
|
|
2411
|
+
value = record.get(key)
|
|
2412
|
+
if value is None:
|
|
2413
|
+
continue
|
|
2414
|
+
if hasattr(defaults, key): # a cap: never above the built-in default (the options file is writable)
|
|
2415
|
+
cap = getattr(defaults, key)
|
|
2416
|
+
number = value if isinstance(value, (int, float)) and not isinstance(value, bool) else None
|
|
2417
|
+
if number is None or not (0 < number <= cap):
|
|
2418
|
+
print(f"caddis_roadmap_run: launch option {key} {value!r} is not in (0, {cap:g}]; using {cap:g}",
|
|
2419
|
+
file=sys.stderr)
|
|
2420
|
+
number = cap
|
|
2421
|
+
value = int(number) if key == "max_sessions" else float(number)
|
|
2422
|
+
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
|
|
2423
|
+
raise ValueError(f"launch option {key} is not a value")
|
|
2424
|
+
text = value if isinstance(value, str) else (f"{value:g}" if isinstance(value, float) else str(value))
|
|
2425
|
+
if key == "engine" and text not in ENGINES:
|
|
2426
|
+
raise ValueError(f"launch option engine {text!r} is unknown")
|
|
2427
|
+
if not _ARG_RE.match(text):
|
|
2428
|
+
raise ValueError(f"launch option {key} holds an unsafe value")
|
|
2429
|
+
out += [flag, text]
|
|
2430
|
+
return out
|
|
2431
|
+
|
|
2432
|
+
|
|
2433
|
+
def accept_roadmap(repo: Path, roadmap: str, yes: bool = False, dry_run: bool = False,
|
|
2434
|
+
confirm: Callable[[str], str] = input) -> int:
|
|
2435
|
+
"""Owner only (refused headless): accept the origin roadmap as it is now as the baseline of the current merge
|
|
2436
|
+
wait (the roadmap hash and the saved text in waiting.json), after an owner edit during the wait. It takes the runner
|
|
2437
|
+
lock (refuses a live runner), prints the unified diff from the published baseline, and asks for `accept` at the
|
|
2438
|
+
console unless `yes`. `dry_run` prints the diff and writes nothing (no lock either)."""
|
|
2439
|
+
if not dry_run and caddis_roadmap.is_headless(): # the dry run only reads, so it may run anywhere
|
|
2440
|
+
print("caddis_roadmap_run: --accept-roadmap is owner-only and this session is headless "
|
|
2441
|
+
f"({', '.join(caddis_roadmap.HEADLESS_VARS)} in the environment, or stdin is not a console)",
|
|
2442
|
+
file=sys.stderr)
|
|
2443
|
+
return caddis_exit.ERROR
|
|
2444
|
+
repo = Path(repo)
|
|
2445
|
+
lock = None
|
|
2446
|
+
if not dry_run: # the lock FIRST; everything below is read under it
|
|
2447
|
+
lock = _RunnerLock(runner_lock_file(repo, "roadmap", roadmap), Caps().max_hours,
|
|
2448
|
+
claim_path=caddis_roadmap.claim_lock_path(repo))
|
|
2449
|
+
try:
|
|
2450
|
+
lock.acquire(lambda message: print(f"caddis_roadmap_run: {message}", file=sys.stderr))
|
|
2451
|
+
except _Stop as stop:
|
|
2452
|
+
print(f"caddis_roadmap_run: {stop.reason}", file=sys.stderr)
|
|
2453
|
+
return stop.code
|
|
2454
|
+
try:
|
|
2455
|
+
waiting_file = _state_file(repo, roadmap, "waiting")
|
|
2456
|
+
waiting = _read_record(waiting_file)
|
|
2457
|
+
if not waiting:
|
|
2458
|
+
print("nothing to accept: no merge wait recorded")
|
|
2459
|
+
return caddis_exit.CLEAN
|
|
2460
|
+
try:
|
|
2461
|
+
snap = caddis_roadmap.load(repo, roadmap)
|
|
2462
|
+
except caddis_roadmap.RoadmapError as exc:
|
|
2463
|
+
print(f"caddis_roadmap_run: {exc}", file=sys.stderr)
|
|
2464
|
+
return exc.code
|
|
2465
|
+
saved = _state_file(repo, roadmap, "published-roadmap").with_suffix(".md")
|
|
2466
|
+
before = _read_text(saved) or ""
|
|
2467
|
+
diff = "".join(difflib.unified_diff(before.splitlines(keepends=True), snap.text.splitlines(keepends=True),
|
|
2468
|
+
"published (baseline of the wait)", f"origin/{snap.branch} (now)"))
|
|
2469
|
+
print(diff or "no difference")
|
|
2470
|
+
if dry_run:
|
|
2471
|
+
print("dry run: nothing written")
|
|
2472
|
+
return caddis_exit.CLEAN
|
|
2473
|
+
if not yes and confirm("type accept to accept this roadmap as the baseline of the wait: ").strip() != "accept":
|
|
2474
|
+
print("not accepted; nothing written")
|
|
2475
|
+
return caddis_exit.BLOCKED
|
|
2476
|
+
saved.parent.mkdir(parents=True, exist_ok=True)
|
|
2477
|
+
tmp = saved.with_name(f"{saved.name}.{os.getpid()}.tmp")
|
|
2478
|
+
tmp.write_bytes(snap.text.encode("utf-8"))
|
|
2479
|
+
os.replace(tmp, saved)
|
|
2480
|
+
_write_record(waiting_file, {**waiting, "roadmap_sha": _roadmap_sha(snap.text),
|
|
2481
|
+
"accepted_by_owner_utc": _iso(datetime.now(timezone.utc))})
|
|
2482
|
+
print(f"roadmap {roadmap}: the origin roadmap is accepted as the baseline of the wait for {waiting.get('ref')}")
|
|
2483
|
+
return caddis_exit.CLEAN
|
|
2484
|
+
finally:
|
|
2485
|
+
if lock is not None:
|
|
2486
|
+
lock.release()
|
|
2487
|
+
|
|
2488
|
+
|
|
2489
|
+
def lock_state(repo: Path, roadmap: str) -> str:
|
|
2490
|
+
"""`absent`, `stale` or `live`: the roadmap runner lock, judged with the lock's own `max_hours`."""
|
|
2491
|
+
stale = caddis_roadmap.runner_lock_stale(runner_lock_file(Path(repo), "roadmap", roadmap))
|
|
2492
|
+
return "absent" if stale is None else ("stale" if stale else "live")
|
|
2493
|
+
|
|
2494
|
+
|
|
2495
|
+
def main(argv: list[str] | None = None) -> int:
|
|
2496
|
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
2497
|
+
mode = parser.add_mutually_exclusive_group(required=True)
|
|
2498
|
+
mode.add_argument("--plan", help="run a plain plan: .caddis/plans/<slug>.md (plan mode)")
|
|
2499
|
+
mode.add_argument("--roadmap", help="run a roadmap: .caddis/roadmaps/<slug>/roadmap.md on origin (roadmap mode)")
|
|
2500
|
+
parser.add_argument("--repo", type=Path, default=Path("."))
|
|
2501
|
+
parser.add_argument("--max-hours", type=float, default=Caps.max_hours)
|
|
2502
|
+
parser.add_argument("--max-sessions", type=int, default=Caps.max_sessions)
|
|
2503
|
+
parser.add_argument("--session-timeout-min", type=float, default=Caps.session_timeout_min)
|
|
2504
|
+
parser.add_argument("--max-usd-per-session", type=float, default=Caps.max_usd_per_session)
|
|
2505
|
+
parser.add_argument("--max-usd-total", type=float, default=Caps.max_usd_total)
|
|
2506
|
+
parser.add_argument("--engine", choices=sorted(ENGINES), default="claude")
|
|
2507
|
+
parser.add_argument("--model", default=None)
|
|
2508
|
+
parser.add_argument("--dry-run", action="store_true")
|
|
2509
|
+
parser.add_argument("--gh-account", default=None,
|
|
2510
|
+
help="the gh account every gh call uses (GH_TOKEN from `gh auth token -u <account>`); give "
|
|
2511
|
+
"the repo's account on a machine with several gh logins")
|
|
2512
|
+
parser.add_argument("--lock-state", action="store_true",
|
|
2513
|
+
help="roadmap mode: print absent, stale or live for the runner lock; exit 0 unless live (3)")
|
|
2514
|
+
parser.add_argument("--restart-args", action="store_true",
|
|
2515
|
+
help="roadmap mode: print the roadmap-run.ps1 arguments of the last launch options, one per "
|
|
2516
|
+
"line (none when not recorded); exit 1 when they cannot be read")
|
|
2517
|
+
parser.add_argument("--accept-roadmap", action="store_true",
|
|
2518
|
+
help="roadmap mode, owner only (refused headless): accept the origin roadmap as it is now as "
|
|
2519
|
+
"the baseline of the current merge wait, after an owner edit during the wait")
|
|
2520
|
+
parser.add_argument("--yes", action="store_true", help="--accept-roadmap without the console confirmation")
|
|
2521
|
+
parser.add_argument("--last-stop", action="store_true",
|
|
2522
|
+
help="roadmap mode: exit 0 when the recheck task may restart the runner: no recorded "
|
|
2523
|
+
"stop, BLOCKED nothing ready, or BLOCKED waiting for merge once that PR merged (the "
|
|
2524
|
+
"line then starts with merged:); else 1 (running, any FAILED stop, a cap); prints why")
|
|
2525
|
+
args = parser.parse_args(argv)
|
|
2526
|
+
for name in ("max_hours", "max_sessions", "session_timeout_min", "max_usd_per_session", "max_usd_total"):
|
|
2527
|
+
value = getattr(args, name)
|
|
2528
|
+
if not (value > 0) or value != value or value == float("inf"):
|
|
2529
|
+
parser.error(f"--{name.replace('_', '-')} must be a finite number above 0")
|
|
2530
|
+
if args.lock_state:
|
|
2531
|
+
if not args.roadmap or not caddis_roadmap.SLUG_RE.match(args.roadmap):
|
|
2532
|
+
parser.error("--lock-state needs --roadmap <slug>")
|
|
2533
|
+
state = lock_state(args.repo, args.roadmap)
|
|
2534
|
+
print(state)
|
|
2535
|
+
return caddis_exit.NOT_RUN if state == "live" else caddis_exit.CLEAN
|
|
2536
|
+
if args.accept_roadmap:
|
|
2537
|
+
if not args.roadmap or not caddis_roadmap.SLUG_RE.match(args.roadmap):
|
|
2538
|
+
parser.error("--accept-roadmap needs --roadmap <slug>")
|
|
2539
|
+
return accept_roadmap(args.repo, args.roadmap, yes=args.yes, dry_run=args.dry_run)
|
|
2540
|
+
if args.restart_args:
|
|
2541
|
+
if not args.roadmap or not caddis_roadmap.SLUG_RE.match(args.roadmap):
|
|
2542
|
+
parser.error("--restart-args needs --roadmap <slug>")
|
|
2543
|
+
try:
|
|
2544
|
+
print("\n".join(restart_args(args.repo, args.roadmap)))
|
|
2545
|
+
except ValueError as exc:
|
|
2546
|
+
print(f"caddis_roadmap_run: {exc}", file=sys.stderr)
|
|
2547
|
+
return caddis_exit.BLOCKED
|
|
2548
|
+
return caddis_exit.CLEAN
|
|
2549
|
+
account = args.gh_account
|
|
2550
|
+
if args.last_stop and not account and args.roadmap and caddis_roadmap.SLUG_RE.match(args.roadmap):
|
|
2551
|
+
account = _read_record(options_file(last_stop_file(Path(args.repo), "roadmap", args.roadmap))).get(
|
|
2552
|
+
"gh_account") or None # the account the run recorded
|
|
2553
|
+
if account and args.last_stop:
|
|
2554
|
+
try:
|
|
2555
|
+
use_gh_account(account)
|
|
2556
|
+
except caddis_roadmap.RoadmapError as exc:
|
|
2557
|
+
print(f"gh error: {exc}")
|
|
2558
|
+
return caddis_exit.BLOCKED
|
|
2559
|
+
if args.last_stop:
|
|
2560
|
+
if not args.roadmap or not caddis_roadmap.SLUG_RE.match(args.roadmap):
|
|
2561
|
+
parser.error("--last-stop needs --roadmap <slug>")
|
|
2562
|
+
ok, why = last_stop_restartable(args.repo, args.roadmap)
|
|
2563
|
+
print(why)
|
|
2564
|
+
return caddis_exit.CLEAN if ok else caddis_exit.BLOCKED
|
|
2565
|
+
caps = Caps(args.max_hours, args.max_sessions, args.session_timeout_min, args.max_usd_per_session,
|
|
2566
|
+
args.max_usd_total)
|
|
2567
|
+
return run(repo=args.repo, plan=args.plan, roadmap=args.roadmap, caps=caps, engine=args.engine, model=args.model,
|
|
2568
|
+
dry_run=args.dry_run, gh_account=args.gh_account)
|
|
2569
|
+
|
|
2570
|
+
|
|
2571
|
+
if __name__ == "__main__":
|
|
2572
|
+
raise SystemExit(main())
|