@opsee/cli 0.11.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +1962 -0
  2. package/bin/opsee.js +28 -0
  3. package/package.json +40 -0
  4. package/skills/README.md +3 -0
  5. package/skills/to-issues/SKILL.md +92 -0
  6. package/skills/to-issues/agents/openai.yaml +5 -0
  7. package/skills/to-spec/SKILL.md +79 -0
  8. package/skills/to-spec/agents/openai.yaml +5 -0
  9. package/skills/wayfinder/SKILL.md +138 -0
  10. package/skills/wayfinder/agents/openai.yaml +5 -0
  11. package/src/args.ts +676 -0
  12. package/src/cli.ts +341 -0
  13. package/src/commands/account.ts +121 -0
  14. package/src/commands/deps.ts +11 -0
  15. package/src/commands/foreman-control.ts +242 -0
  16. package/src/commands/foreman-debug.ts +131 -0
  17. package/src/commands/foreman-plan.ts +213 -0
  18. package/src/commands/foreman-service.ts +186 -0
  19. package/src/commands/foreman-up.ts +165 -0
  20. package/src/commands/foreman-views.ts +398 -0
  21. package/src/commands/foreman.ts +465 -0
  22. package/src/commands/init.ts +176 -0
  23. package/src/commands/initiative.ts +192 -0
  24. package/src/commands/login.ts +24 -0
  25. package/src/commands/whoami.ts +15 -0
  26. package/src/foreman/account-store.ts +96 -0
  27. package/src/foreman/account.ts +474 -0
  28. package/src/foreman/claude-worker-adapter.ts +412 -0
  29. package/src/foreman/codex-worker-adapter.ts +472 -0
  30. package/src/foreman/completion-report.ts +153 -0
  31. package/src/foreman/core/context.ts +169 -0
  32. package/src/foreman/core/defects.ts +280 -0
  33. package/src/foreman/core/exec.ts +20 -0
  34. package/src/foreman/core/gates.ts +493 -0
  35. package/src/foreman/core/handoff.ts +163 -0
  36. package/src/foreman/core/install.ts +109 -0
  37. package/src/foreman/core/learnings.ts +368 -0
  38. package/src/foreman/core/outbox-tracker.ts +192 -0
  39. package/src/foreman/core/pin.ts +226 -0
  40. package/src/foreman/core/plan-context.ts +238 -0
  41. package/src/foreman/core/process-table.ts +535 -0
  42. package/src/foreman/core/reconcile.ts +227 -0
  43. package/src/foreman/core/report.ts +60 -0
  44. package/src/foreman/core/run.ts +2836 -0
  45. package/src/foreman/core/scheduler.ts +244 -0
  46. package/src/foreman/core/summary.ts +166 -0
  47. package/src/foreman/core/text.ts +97 -0
  48. package/src/foreman/core/transcripts.ts +38 -0
  49. package/src/foreman/core/triage.ts +138 -0
  50. package/src/foreman/core/verifier.ts +800 -0
  51. package/src/foreman/core/views.ts +940 -0
  52. package/src/foreman/core/work-contract.ts +152 -0
  53. package/src/foreman/core/workspace.ts +335 -0
  54. package/src/foreman/fake-handoff.ts +33 -0
  55. package/src/foreman/fake-learnings.ts +26 -0
  56. package/src/foreman/fake-remote-api.ts +70 -0
  57. package/src/foreman/fake-tracker-adapter.ts +355 -0
  58. package/src/foreman/fake-worker-adapter.ts +221 -0
  59. package/src/foreman/host.ts +75 -0
  60. package/src/foreman/local-dir.ts +28 -0
  61. package/src/foreman/opsee-tracker-adapter.ts +612 -0
  62. package/src/foreman/process-group.ts +160 -0
  63. package/src/foreman/remote-api.ts +283 -0
  64. package/src/foreman/run-recipe.ts +274 -0
  65. package/src/foreman/service-unit.ts +257 -0
  66. package/src/foreman/tracker-adapter.ts +298 -0
  67. package/src/foreman/triage-draft.ts +40 -0
  68. package/src/foreman/vendor.ts +23 -0
  69. package/src/foreman/verdict.ts +120 -0
  70. package/src/foreman/worker-adapter.ts +177 -0
  71. package/src/foreman/worker-process.ts +488 -0
  72. package/src/identity.ts +49 -0
  73. package/src/index.ts +3 -0
  74. package/src/init/managed.ts +84 -0
  75. package/src/init/mcp-config.ts +77 -0
  76. package/src/init/paths.ts +16 -0
  77. package/src/init/pointer-block.ts +45 -0
  78. package/src/init/project.ts +22 -0
  79. package/src/init/prompt.ts +45 -0
  80. package/src/init/run-recipe-config.ts +133 -0
  81. package/src/init/skills.ts +38 -0
  82. package/src/init/text.ts +22 -0
  83. package/src/init/tracker-doc.ts +106 -0
  84. package/src/opsee-config.ts +116 -0
  85. package/templates/issue-tracker.md +162 -0
@@ -0,0 +1,2836 @@
1
+ /**
2
+ * The Run loop (see ../../../CONTEXT.md: Run, Ready Issue, Workspace, Context Assembly, Completion
3
+ * Report, Status Label, Run Record, Hand-off). Each tick takes one Ready Task of the Initiative,
4
+ * creates its Workspace, installs dependencies once, assembles the Worker's context, runs one
5
+ * unattended turn pinned to the Workspace, hands a `done` turn off as a draft pull request
6
+ * (core/handoff.ts) linked to the Task, then writes the Completion Report as Initiative memory
7
+ * with a linking comment, moves the Task through the board with a Status Label, and appends
8
+ * dispatch and attempt events to the Run Record. It stops when no Ready Task remains, and then
9
+ * gathers the Run's Proposed Learnings into one draft pull request against the repository's
10
+ * learnings file (core/learnings.ts, story 45).
11
+ *
12
+ * Gates (gates.ts, OPS-271) sit between the Hand-off and the settlement: once the draft pull request
13
+ * is open, the base's Run Recipe test, lint and typecheck commands run in the Workspace and are
14
+ * judged by exit code alone. A pass settles the Task In review as before; a failure resumes the same
15
+ * Worker session with the failing command and its output tail, and once its next `done` turn has
16
+ * updated the Hand-off the Gates run again, up to the retry cap, past which the Task is blocked
17
+ * with a comment naming the Gate. Every Gate result reaches the Run Record as a `gate` event in the
18
+ * orchestrator Ledger's Verification shape before anything else moves, and the rounds are
19
+ * summarised as a comment on the pull request, whatever the verdict; Reconcile reads those events
20
+ * back to finish a settlement, so a Hand-off the Gates held is never settled as handed off.
21
+ *
22
+ * The Verifier (verifier.ts, OPS-272) follows the Gates: once they pass (or the recipe names none),
23
+ * a Task whose Verification section describes a user journey gets the app started from the base's
24
+ * Run Recipe on a leased port and a separate Verifier Worker with Playwright MCP, whose prompt is
25
+ * that section and the URL and nothing of the implementer's turn. Its Verdict goes to the Run
26
+ * Record as a `verdict` event (the Ledger's shape, screenshots uploaded to the pull request first
27
+ * so the Defects carry their evidence URL), then to the pull request as a comment; a passed
28
+ * Verdict settles the Task In review, Defects leave it In Progress under `foreman:blocked` for the
29
+ * Defect filer (OPS-273). A Task with commands only in its Verification section skips the browser
30
+ * and is verified at its Gates.
31
+ *
32
+ * Scheduling (core/scheduler.ts, OPS-276): each tick fills the Account's free Slots rather than
33
+ * dispatching one Task and waiting. The Account's `cap` is how many Slots it has, one of them
34
+ * reserved for Verifiers, so `cap` 3 is two implementers and a Verifier round beside them; a
35
+ * dispatch swaps its implementer Slot for the reserved one when it reaches verification, which is
36
+ * what keeps a fleet of implementers from starving it. Ready Tasks are taken in the Tracker's
37
+ * readiness set but the Foreman's order: priority descending, then oldest first. `--once` and
38
+ * `--task` still mean exactly one dispatch. A dispatch that throws is caught, left blocked as
39
+ * before, and reported in the Run's outcome; the other Slots go on.
40
+ *
41
+ * `runForeman` is the seam the daemon wraps: it takes the adapters and an Account and
42
+ * knows nothing about daemons. Reconcile is here (story 51): with a Process Table in the deps, each
43
+ * tick first Reconciles (reconcile.ts) and drains the outbox, every live Worker is recorded in the
44
+ * table (its session id the moment `started` arrives, its last output as it streams), and a turn
45
+ * that the stored session id can continue is resumed through the same settle path as a fresh one.
46
+ * Transcripts go to the local store; the Run Record gets an output tail (story 49).
47
+ *
48
+ * Readiness is the Tracker's (`listReadyTasks`, ADR-0008); the loop never walks blockers itself.
49
+ * Run Record writes come before the board moves they describe (ADR-0009: that write is the commit).
50
+ *
51
+ * Before the dispatch event the Task's work contract is checked (work-contract.ts, story 23): a Task
52
+ * missing Goal or Acceptance Criteria is blocked and loses its dispatch label; one missing only
53
+ * Verification gets a Triage turn (triage.ts) whose draft is posted as a comment and joins the
54
+ * Worker's context, or reuses a draft an earlier turn left; a complete one dispatches as is.
55
+ *
56
+ * The human steers through the Process Table (stories 53-56; commands/foreman-control.ts): a
57
+ * paused Initiative stops the loop from picking a Task, and the Run either returns with `paused`
58
+ * set (the daemon then puts the request back and serves others) or, asked to wait, idles at the
59
+ * daemon's tick until resumed or stopped; a
60
+ * turn whose row was marked `attached` when its stream ended is left open (no attempt event, the
61
+ * row stays) for the human's attended turn, and Reconcile resumes it unattended once `released`;
62
+ * one marked `cancelled` is closed as a `stopped` attempt whose summary starts with the cancelled
63
+ * marker, the Task blocked with a comment. The attempt outcome vocabulary is the Run Record's
64
+ * (RunAttemptEvent.outcome), which has no cancelled value; the `control` event beside it says who
65
+ * stopped the Worker and why.
66
+ */
67
+ import { DEFECT_LABEL } from "@opsee/mcp-server/src/utils/defect.js";
68
+ import { timestampFromMs } from "@bufbuild/protobuf/wkt";
69
+ import { existsSync, readFileSync } from "node:fs";
70
+ import { basename, join } from "node:path";
71
+ import { create, type MessageInitShape } from "@bufbuild/protobuf";
72
+ import { RunEventInputSchema, type RunEventInput } from "@opsee/mcp-server/gen/api/v1/initiative_pb.js";
73
+ import {
74
+ clearCredentialFailures,
75
+ clearResumed,
76
+ CREDENTIAL_FAILURES_BEFORE_QUARANTINE,
77
+ DEFAULT_PAUSE_MS,
78
+ isPaused,
79
+ isQuarantined,
80
+ MIN_PAUSE_MS,
81
+ noteCredentialFailure,
82
+ pauseAccount,
83
+ pausedUntil,
84
+ pauseUntilFrom,
85
+ quarantineAccount,
86
+ type Account,
87
+ } from "../account.js";
88
+ import type { AccountStore } from "../account-store.js";
89
+ import type { CompletionReport } from "../completion-report.js";
90
+ import { blockerIdsOf, DISPATCH_LABEL, type TaskWithContext, type TrackerAdapter, type TrackerTask } from "../tracker-adapter.js";
91
+ import type { Vendor } from "../vendor.js";
92
+ import type { Verdict } from "../verdict.js";
93
+ import type { AdapterEvent, TurnFailureReason, TurnHandle, WorkerAdapter } from "../worker-adapter.js";
94
+ import { triageDraftFrom } from "../triage-draft.js";
95
+ import type { RemoteApi } from "../remote-api.js";
96
+ import { assembleContext, SIBLING_LEARNING_LIMIT, type AcceptedLearnings, type BlockerReports, type SiblingLearning } from "./context.js";
97
+ import { isRoundFailure, type DefectEvidenceLink, type DefectsFn, type FiledDefect } from "./defects.js";
98
+ import { DEFAULT_GATE_RETRIES, gateComment, gateEvent, gateFixPrompt, gateLine, gatePassed, gateSummary, gateVerdictOf, type GateOutcome, type GateReason, type GateRound, type GatesFn, truncateCodePoints } from "./gates.js";
99
+ import type { HandOffFn, HandOffResult } from "./handoff.js";
100
+ import { inferInstallCommand, runInstall, tail, type InstallResult } from "./install.js";
101
+ import { collectProposedLearnings, DEFAULT_LEARNINGS_FILE, type LearningsFn, type LearningsResult } from "./learnings.js";
102
+ import { describePin, pinAllows, pinOf, pinToken, unplaceable, type Pin, type Unplaceable } from "./pin.js";
103
+ import { otherForemanWorkersOn, type DrainResult, type ProcessTableApi, type WorkerRow } from "./process-table.js";
104
+ import { reconcile } from "./reconcile.js";
105
+ import { failureComment, reportComment, reportToMemory } from "./report.js";
106
+ import { orderReadyTasks, priorityScale, Slots, type SlotHold } from "./scheduler.js";
107
+ import { runSummaryComment } from "./summary.js";
108
+ import { count, printableOneLine } from "./text.js";
109
+ import { driveTriageTurn, isTriageSummary, TRIAGE_MARKER } from "./triage.js";
110
+ import { failureVerdict, verdictComment, verdictEvent, verdictOf, verdictStatusOf, verificationPassed, verifySummary, type DefectEvidence, type VerifierFn, type VerifyOutcome } from "./verifier.js";
111
+ import {
112
+ blockedComment,
113
+ draftedVerificationFrom,
114
+ parseWorkContract,
115
+ triageComment,
116
+ triageCappedComment,
117
+ triageFailedComment,
118
+ validateWorkContract,
119
+ type WorkContract,
120
+ type WorkContractSection,
121
+ } from "./work-contract.js";
122
+ import type { TranscriptStore, TranscriptWriter } from "./transcripts.js";
123
+ import type { Workspace, WorkspaceManager } from "./workspace.js";
124
+
125
+ /** A refusal the Foreman makes on purpose, reported as a plain message. */
126
+ export class ForemanError extends Error {
127
+ constructor(message: string) {
128
+ super(message);
129
+ this.name = "ForemanError";
130
+ }
131
+ }
132
+
133
+ /** The Status Labels (story 63) the Foreman keeps on the Tasks it touches; created when missing. */
134
+ export const STATUS_LABELS = {
135
+ running: "foreman:running",
136
+ /** Worn *alongside* `running` for the Verifier's browser round, not instead of it. The round is
137
+ * the longest and most failure-prone phase and the board could not tell it from implementation.
138
+ * Additive because Reconcile finds a dead Foreman's orphans by `running` alone (core/reconcile.ts,
139
+ * RUNNING_LABEL): swapping the label out would hide a Task that died mid-verify from the one
140
+ * thing built to recover it. Cleared when the attempt settles. */
141
+ verifying: "foreman:verifying",
142
+ blocked: "foreman:blocked",
143
+ done: "foreman:done",
144
+ } as const;
145
+ type StatusLabel = (typeof STATUS_LABELS)[keyof typeof STATUS_LABELS];
146
+
147
+ /** What the Run Record calls each vendor (RunDispatchEvent.vendor). */
148
+ export const RUN_RECORD_VENDOR: Readonly<Record<Vendor, string>> = { claude: "claude-code", codex: "codex" };
149
+
150
+ /** RunAttemptEvent.outcome values. */
151
+ export type AttemptOutcome = "completed" | "failed" | "stalled" | "rate_limited" | "stopped";
152
+
153
+ /** RunControlEvent.action values: what a human did from the CLI. */
154
+ export type ControlAction = "attach" | "release" | "pause" | "resume" | "cancel";
155
+
156
+ /** What a cancelled attempt's summary starts with: the outcome is `stopped` (the Run Record's
157
+ * vocabulary), the marker says the human stopped it on purpose. */
158
+ export const CANCELLED_MARKER = "cancelled:";
159
+
160
+ /** How often a foreground Run that waits through a pause looks for its resume: the daemon's idle
161
+ * tick (foreman-up.ts takes its DEFAULT_DAEMON_TICK_MS from here), since each look reads the Run
162
+ * Record for every row the Initiative has. */
163
+ export const PAUSE_POLL_MS = 5_000;
164
+
165
+ /** The shortest that poll may be set to. `pausePollMs` reaches a `sleep` inside the loop, so 0 is
166
+ * not "check often", it is a spin: the Run would re-read the Run Record for every row of the
167
+ * Initiative thousands of times a second while an Account waits out an hour. */
168
+ export const MIN_PAUSE_POLL_MS = 100;
169
+
170
+ /** How long a dependency install may run before it is killed and recorded as such; a wedged
171
+ * install must not hold the Run where no Worker stall timer can reach it. */
172
+ export const INSTALL_TIMEOUT_MS = 15 * 60_000;
173
+
174
+ /** How many further attempts a Task gets after a turn ended without reporting for a reason that is
175
+ * the machine's rather than the work's (`RETRYABLE_FAILURES`). Under the cap the Task is made Ready
176
+ * again and the next tick dispatches it afresh — on another Account of the same vendor, when its
177
+ * own is Paused by then (story 31); at the cap it is settled blocked with a comment naming the
178
+ * cause that used the attempts up (story 34, `Settlement.capped`), and the Run goes on with the
179
+ * other Tasks.
180
+ *
181
+ * The attempts are spaced by `retryBackoff` rather than taken as fast as the machine can fail. */
182
+ export const DEFAULT_ATTEMPT_RETRIES = 2;
183
+
184
+ /**
185
+ * How long the first retry of a Task waits, with each further one waiting twice as long
186
+ * (`retryBackoff`).
187
+ *
188
+ * Without a delay the retry cap bounds the number of attempts and nothing bounds their speed. An
189
+ * adapter that fails in milliseconds — a vendor that answers at once, a launch that cannot start —
190
+ * takes a Task through every attempt it has and gives the Slot back before a human sees a line of
191
+ * log, and every free Slot does it at the same time. The Paused Account was the only clock of that
192
+ * kind, and it only covers the vendor saying "not yet" out loud.
193
+ *
194
+ * Thirty seconds is chosen against what the delay is for: long enough that a transient condition
195
+ * (a vendor's bad minute, a machine still releasing a port) has time to clear, short enough to be
196
+ * invisible on the overnight Run this exists to protect. At the default cap a Task waits 30s and
197
+ * then 60s across its whole life.
198
+ */
199
+ export const DEFAULT_RETRY_BACKOFF_MS = 30_000;
200
+
201
+ /** The longest a retry waits, however many attempts a Task has been given. Doubling is only useful
202
+ * while the wait is short: past a few minutes a Task that could have run is simply idle, and the
203
+ * Run has other Tasks it should be spending the night on. Reached at the fifth retry, which only a
204
+ * raised `attemptRetries` gets to. */
205
+ export const MAX_RETRY_BACKOFF_MS = 5 * 60_000;
206
+
207
+ /**
208
+ * How long a Task waits before its next attempt, given how many retryable failures it has had
209
+ * (`retryableFailuresOf`, this one included). Doubles per failure from `base`, capped.
210
+ *
211
+ * `spent` is the count the retry cap itself is measured in, so the backoff and the cap always agree
212
+ * about which attempt this is; a count of zero or less is not a retry and waits not at all.
213
+ */
214
+ /** " in 30s" for a wait worth mentioning, and nothing at all for one that is not. A Run configured
215
+ * with no backoff must not tell a human its Task comes back "in 0s", which reads as a bug in the
216
+ * sentence rather than as the absence of a delay. */
217
+ function inSeconds(ms: number): string {
218
+ return ms > 0 ? ` in ${Math.round(ms / 1000)}s` : "";
219
+ }
220
+
221
+ export function retryBackoff(spent: number, base: number = DEFAULT_RETRY_BACKOFF_MS): number {
222
+ if (spent < 1) return 0;
223
+ // Shifting rather than `Math.pow` would overflow at 31 doublings; the cap is applied to a number
224
+ // that may be Infinity, which `Math.min` handles, so a huge `spent` is the cap and not a NaN.
225
+ return Math.min(base * 2 ** (spent - 1), MAX_RETRY_BACKOFF_MS);
226
+ }
227
+
228
+ /** How many Defect Tasks one Run may dispatch before the Breaker stops it starting anything new.
229
+ *
230
+ * A Defect is filed as a sibling Task carrying the dispatch label, so the same Run picks it up; its
231
+ * own Verdict can file more, and nothing counted. One bad plan could spend a whole night on
232
+ * defect, fix, defect, and the morning summary would show a Run that never stopped and never
233
+ * finished. This is the count that ends it.
234
+ *
235
+ * The count is this Run's own and in memory, so a Foreman that crashes and is restarted begins a
236
+ * fresh Run with a fresh count. That is the honest scope of the guard: it bounds a night, not a
237
+ * week, and a crash-looping Foreman is a different failure with a different fix. */
238
+ export const DEFAULT_DEFECT_DISPATCH_CAP = 10;
239
+
240
+ /**
241
+ * The turn failures another attempt is worth trying, because they are the machine's or the
242
+ * vendor's rather than the work's: the Worker went quiet and the adapter stopped it (story 33), or
243
+ * the vendor refused the turn on a rate limit (story 30) — neither of which says anything about
244
+ * whether the Task can be done.
245
+ *
246
+ * A rate limit is here for the same reason Failover exists: with the Account Paused, the next tick
247
+ * puts the Task on a sibling Account of the same vendor, or on the same one once the reset has
248
+ * passed, and nothing is lost. The rest are deliberately not retried — `invalid_report` and
249
+ * `max_turns` are turns that really ran and would only run the same way again, `launch_failed` and
250
+ * `vendor_error` are a machine or a configuration a human must look at, and `stopped` is a human's
251
+ * own doing.
252
+ */
253
+ export const RETRYABLE_FAILURES: ReadonlySet<AttemptOutcome> = new Set<AttemptOutcome>(["stalled", "rate_limited"]);
254
+
255
+ /**
256
+ * The same question asked of the turn's reason rather than of the outcome it was recorded under
257
+ * (OPS-288).
258
+ *
259
+ * `credential_failed` has to be retried for the reason a rate limit does — with its Account
260
+ * quarantined, the next tick puts the Task on a sibling and nothing is lost, where blocking it
261
+ * outright would strand work that a healthy Account could do. But it cannot join
262
+ * `RETRYABLE_FAILURES`: that set is keyed by `AttemptOutcome`, a credential failure is recorded as
263
+ * the generic `failed` (the proto's outcome list is closed), and adding `failed` there would make
264
+ * every `vendor_error` and `invalid_report` retryable too.
265
+ *
266
+ * So the live decision reads the reason, and the Run Record read-back reads the outcome plus
267
+ * `CREDENTIAL_MARKER` in the summary, which is how a past credential failure is still counted
268
+ * against the retry cap. The alternative — a new outcome in the proto — is four generated trees for
269
+ * a fact the summary already carries, and the marker precedent is `TRIAGE_MARKER`'s.
270
+ */
271
+ export const RETRYABLE_FAILURE_REASONS: ReadonlySet<TurnFailureReason> = new Set<TurnFailureReason>(["stalled", "rate_limited", "credential_failed"]);
272
+
273
+ /** What an attempt event's summary starts with when the turn failed for a credential reason, so
274
+ * `retryableFailuresOf` can count it back off the Run Record despite its generic `failed` outcome. */
275
+ export const CREDENTIAL_MARKER = "[credential]";
276
+
277
+ export interface RunDeps {
278
+ tracker: TrackerAdapter;
279
+ worker: WorkerAdapter;
280
+ account: Account;
281
+ /** Sibling Accounts of the same vendor a not-yet-started Ready Task may Fail over to while this
282
+ * Run's own Account is Paused (story 31, CONTEXT.md: Failover). Each carries its own Worker
283
+ * Adapter and the per-Account steps built from it, and gets its own Slots inside the Run, so the
284
+ * concurrency one vendor identity sees is still its own cap and no more (ADR-0013). A live
285
+ * session never changes Account: a Task with a Process Table row runs on that row's Account
286
+ * alone. Its Workspace is not pinned with it — the row goes when the attempt is settled, so a
287
+ * retry after a machine-side failure takes whichever Lane is open and reuses the same git
288
+ * worktree; nothing of the old Account travels with it but the commits. */
289
+ failover?: RunAccount[];
290
+ /** Accounts of *another* vendor that only a pinned Task may use (story 24, core/pin.ts). A Run on
291
+ * a Claude Account with a Codex Account registered beside it carries that Codex Account here: a
292
+ * Task labelled `foreman:vendor:codex` (or `foreman:account:<its name>`) runs there, and nothing
293
+ * else ever does. They are Lanes like any other — their own Slots from their own cap, checked
294
+ * against the Process Table each tick — so one vendor identity still runs its own cap and no more
295
+ * (ADR-0013); what they are not is Failover targets, which stays within a vendor because a
296
+ * session, a prompt contract and an adapter do not cross one. `pinnableAccountsFor`
297
+ * (commands/foreman.ts) builds them. */
298
+ pinnable?: RunAccount[];
299
+ /** Where an Account's Paused state is written when a vendor reports a rate limit, and read back
300
+ * each time Slots are filled (story 30). It is the store rather than a callback because the fact
301
+ * outlives the Run: a pause this Foreman hit is one the next Foreman, and `foreman account list`,
302
+ * must see. Without one a rate limit is logged and no Account is Paused. */
303
+ accounts?: AccountStore;
304
+ /** How long an Account is Paused when the vendor named no reset; `DEFAULT_PAUSE_MS` when unset,
305
+ * and never below `MIN_PAUSE_MS`, since a window of zero is no pause at all. */
306
+ pauseMs?: number;
307
+ workspaces: Pick<WorkspaceManager, "create">;
308
+ /** Installs dependencies in a new Workspace; resolves undefined when there is nothing to install.
309
+ * Defaults to the Run Recipe's `commands.install`, else the lockfile inference in install.ts. */
310
+ install?: (workspacePath: string) => Promise<InstallResult | undefined>;
311
+ /** The Hand-off after a `done` turn: pushes the Workspace branch and opens or updates the draft
312
+ * pull request (core/handoff.ts). Its result decides whether the Task goes to In review. Absent,
313
+ * every `done` turn is an incomplete attempt that says so; `foreman run` always wires one. */
314
+ handOff?: HandOffFn;
315
+ /** The Gates after a Hand-off (gates.ts): runs the base's Run Recipe commands in the Workspace
316
+ * and reports each by exit code. `foreman run` always wires the real ones (`gatesWith`, which
317
+ * needs the guarded git and the checkout); absent, the Gates are skipped and the Hand-off stands. */
318
+ gates?: GatesFn;
319
+ /** How many times a failing Gate goes back to the same Worker before the Task is blocked;
320
+ * `DEFAULT_GATE_RETRIES` when unset. */
321
+ gateRetries?: number;
322
+ /** The code host of the Hand-off, for the Gates' summary and the Verdict on the pull request
323
+ * and the Verifier's screenshots. Absent, the summary is logged and the Run Record still has
324
+ * every result. */
325
+ remote?: Pick<RemoteApi, "addPullRequestComment" | "uploadAttachment">;
326
+ /** The Verifier after the Gates (verifier.ts): starts the app from the base's Run Recipe and
327
+ * runs the Verifier Worker on the Task's Verification section. `foreman run` always wires the
328
+ * real one (`verifierWith`); absent, verification is skipped and the Gates are the whole of it. */
329
+ verifier?: VerifierFn;
330
+ /** The Defect filer after a failed Verdict (defects.ts): one sibling Bug Task per observed
331
+ * Defect, related to the Task the Verifier found it on, so the next tick dispatches the fix.
332
+ * `foreman run` always wires the real one (`defectFilerWith`); absent, the Verdict is recorded
333
+ * and nothing is filed, which is what the loop's own tests want. */
334
+ defects?: DefectsFn;
335
+ /** The learnings step at the end of the Run: every Proposed Learning of its Completion Reports
336
+ * into one draft pull request against the learnings file (core/learnings.ts). Absent, the
337
+ * learnings stay in Initiative memory only; `foreman run` always wires one. */
338
+ learnings?: LearningsFn;
339
+ /** The repository's learnings file, relative to its root, read from each Workspace for Context
340
+ * Assembly as the Accepted Learnings; `docs/learnings.md` unless the Run Recipe says otherwise. */
341
+ learningsFile?: string;
342
+ /** The daemon log: every decision the loop makes goes here. */
343
+ log: (line: string) => void;
344
+ maxTurns?: number;
345
+ stallTimeoutMs?: number;
346
+ /** How many further attempts a stalled Task gets before it is settled blocked;
347
+ * `DEFAULT_ATTEMPT_RETRIES` when unset. */
348
+ attemptRetries?: number;
349
+ /** How long the first retry of a Task waits, doubling per further retry (`retryBackoff`);
350
+ * `DEFAULT_RETRY_BACKOFF_MS` when unset. */
351
+ retryBackoffMs?: number;
352
+ /** How many Defect Tasks this Run may dispatch before the Breaker halts new dispatch;
353
+ * `DEFAULT_DEFECT_DISPATCH_CAP` when unset. */
354
+ defectDispatchCap?: number;
355
+ /** Pause between ticks, so a Tracker that just released a successor is not hammered; 0 in tests. */
356
+ tickDelayMs?: number;
357
+ now?: () => number;
358
+ sleep?: (ms: number) => Promise<void>;
359
+ /** The Process Table (ADR-0009). Without one the loop keeps no local record and never Reconciles. */
360
+ processTable?: ProcessTableApi;
361
+ /** Where each turn's raw event stream is written; unset keeps no transcript. */
362
+ transcripts?: TranscriptStore;
363
+ /** Sends the outbox's queued Run Record batches (OutboxTracker.drain); called at the top of each tick. */
364
+ drainOutbox?: (initiativeId: number) => Promise<DrainResult>;
365
+ /** Whether a Process Table row's pid is a live process; the real check by default. */
366
+ isAlive?: (pid: number | undefined) => boolean;
367
+ /** How often a Run that waits through a pause (`RunOptions.waitWhilePaused`) checks for its
368
+ * resume; PAUSE_POLL_MS by default. */
369
+ pausePollMs?: number;
370
+ }
371
+
372
+ /** How often at most an `output` event bumps the row's last_output_at. */
373
+ export const TOUCH_INTERVAL_MS = 1000;
374
+
375
+ export interface RunOptions {
376
+ initiativeId: number;
377
+ /** Stop after one dispatch. */
378
+ once?: boolean;
379
+ /** Dispatch this Task and stop; it must be a Ready Task of the Initiative. */
380
+ taskId?: number;
381
+ /** Asked before each pick: true ends the Run after the dispatch in flight (the daemon was told
382
+ * to stop). The turn itself is stopped by the process hooks; this keeps the loop from starting
383
+ * another. */
384
+ shouldStop?: () => boolean;
385
+ /** Resolves when the Run should stop, for the waits a poll cannot reach: a Reconcile turn that
386
+ * needs an implementer Slot waits on this beside the Slot, and abandons the wait rather than
387
+ * holding the loop — and with it the stop and the pause — until a Worker turn ends. The daemon
388
+ * passes the same promise its `shouldStop` reads. */
389
+ untilStop?: Promise<void>;
390
+ /** Idle through a pause of the Initiative (story 55), polling for the resume, instead of returning
391
+ * with `paused` set; a foreground `foreman run` does, the daemon does not. */
392
+ waitWhilePaused?: boolean;
393
+ }
394
+
395
+ /** A Triage turn the Foreman ran on a Task before dispatching it. On the Run Record as an `attempt`
396
+ * event whose summary starts `triage:`, so it counts as a turn on the Account like any other. */
397
+ export interface TriageTurn {
398
+ attempt: number;
399
+ outcome: AttemptOutcome;
400
+ /** The drafted Verification section, when the turn produced one. */
401
+ verification?: string;
402
+ /** Why it produced none: the turn's failure, or what the Worker reported instead. */
403
+ reason?: string;
404
+ sessionId?: string;
405
+ durationMs: number;
406
+ }
407
+
408
+ export interface Dispatch {
409
+ task: TrackerTask;
410
+ workspace: Workspace;
411
+ attempt: number;
412
+ outcome: AttemptOutcome;
413
+ report?: CompletionReport;
414
+ sessionId?: string;
415
+ durationMs: number;
416
+ install?: InstallResult;
417
+ /** Present for a `done` report: the draft pull request, or why there is none. */
418
+ handOff?: HandOffResult;
419
+ /** The Gates that ran on the Hand-off, round by round; absent when the recipe names none. */
420
+ gates?: GateOutcome;
421
+ /** The Verifier's round after the Gates passed: skipped, a Verdict, or a failed round; absent
422
+ * when the Gates held the Hand-off or there was none. */
423
+ verification?: VerifyOutcome;
424
+ /** The Triage turn that ran before this dispatch, when the Task had no Verification section. */
425
+ triage?: TriageTurn;
426
+ /** The Verification section the Worker was given because the Task had none: this dispatch's
427
+ * Triage turn's, or one an earlier turn left in the comments. */
428
+ draftedVerification?: string;
429
+ /** True when Reconcile continued this attempt from its stored session id (story 51). */
430
+ resumed?: boolean;
431
+ /** True when the turn ended because a human attached (story 53): nothing was recorded, the
432
+ * attempt is still open and its row stays until the human releases it. */
433
+ attached?: boolean;
434
+ /** True when the attempt was closed by `foreman cancel` (story 56). */
435
+ cancelled?: boolean;
436
+ /** True when the attempt ended stalled or rate limited under the retry cap: the Task was made
437
+ * Ready again instead of blocked, and a later tick dispatches it as the next attempt (story 33). */
438
+ retrying?: boolean;
439
+ /** True when that retry cap is what blocked the Task (story 34): the attempt failed for a reason
440
+ * another attempt would have been worth, and there were none left. */
441
+ blockedByCap?: boolean;
442
+ /** When the Task may be dispatched again, for a `retrying` dispatch (`retryBackoff`). Absolute
443
+ * rather than a duration, so the Run compares it against the same clock it schedules on.
444
+ *
445
+ * Decided here and held by the Run: this is where the facts are (which attempt, how many
446
+ * retryable failures the Run Record counts), while the waiting is the tick's business. Carrying
447
+ * it on the Dispatch keeps the backoff out of `RunDeps` — the settlement path has no mutable Run
448
+ * state to write to, and giving it some would be a seam every future turn could reach through. */
449
+ retryNotBefore?: number;
450
+ }
451
+
452
+ /** A dispatch whose work reached review: the turn reported `done`, the Hand-off is open, the
453
+ * Gates passed (or there were none to run), and the Verifier passed it or had no browser journey
454
+ * to run. */
455
+ export function isHandedOff(d: Dispatch): boolean {
456
+ return d.outcome === "completed" && d.report?.outcome === "done" && d.handOff?.kind === "handed_off" && (d.gates?.passed ?? true) && verificationPassed(d.verification);
457
+ }
458
+
459
+ /** A Ready Task the Foreman refused to dispatch: its work contract is incomplete, and the Triage
460
+ * turn, when one applied, drafted nothing. It lost its dispatch label and carries `foreman:blocked`. */
461
+ export interface BlockedTask {
462
+ task: TrackerTask;
463
+ missing: WorkContractSection[];
464
+ triage?: TriageTurn;
465
+ /** The Workspace created for the Triage turn, when one ran; kept, and reused by a later dispatch. */
466
+ workspace?: Workspace;
467
+ /** The Task was left Ready because the Triage turn failed for a reason that is not the contract's
468
+ * — a stall, a rate limit — and the retry cap still allows another go. A later tick of *this* Run
469
+ * picks it up again (on a sibling Account when its own is Paused by then); it is not deferred to
470
+ * the next Run, which would cost a Task the whole night for one bad minute at 3am. `attemptsLeft`
471
+ * is what the retry cap has left, counted the same way a failed dispatch counts it
472
+ * (`retryableFailuresOf`), and `notBefore` is when that later tick may take it (`retryBackoff`):
473
+ * a Triage turn that fails in milliseconds is the same hot loop a dispatch's is. */
474
+ retry?: { attemptsLeft: number; notBefore: number };
475
+ /** The Task keeps its dispatch label but this Run will not look at it again: the Triage turn
476
+ * failed for something no tick of this Run can change (an adapter that will not start, a vendor
477
+ * error), so trying again in a second would only spin. The next Run picks it up. */
478
+ defer?: boolean;
479
+ }
480
+
481
+ /** A Slot whose dispatch threw something that is not a turn outcome: a Tracker write refused, a
482
+ * board without the column, an adapter that would not start. `dispatch` has already commented on
483
+ * the Task and marked it blocked; the Run reports it here and fills the Slot with the next Task
484
+ * rather than dying, so one bad Task does not end the Run (OPS-276). */
485
+ export interface DispatchFailure {
486
+ task: TrackerTask;
487
+ reason: string;
488
+ }
489
+
490
+ /**
491
+ * A Ready Task this Run left alone because its label pins it somewhere this Run cannot reach: an
492
+ * Account it has no Lane for, a vendor it has no Account of, or a `foreman:` label it cannot read
493
+ * as a pin at all (story 24, core/pin.ts).
494
+ *
495
+ * It is deliberately *not* a `BlockedTask`. Nothing is wrong with the Task: it keeps its dispatch
496
+ * label, stays Ready, is never marked `foreman:blocked`, and spends none of its retry cap — the
497
+ * Foreman simply has nowhere to put it. Registering the Account (or fixing the label) is the whole
498
+ * of the fix, and the next Run dispatches it with nothing to undo. It is skipped for the rest of
499
+ * *this* Run, because no tick of it can change the answer.
500
+ */
501
+ export interface QueuedTask {
502
+ task: TrackerTask;
503
+ pin: Pin;
504
+ /** What the comment on the Task said, in one sentence. */
505
+ reason: string;
506
+ }
507
+
508
+ export interface RunOutcome {
509
+ /** In the order they settled, which with Slots filled in parallel is not the order they were
510
+ * dispatched in: the Run Record's `dispatch` events are what carry that. */
511
+ dispatches: Dispatch[];
512
+ blocked: BlockedTask[];
513
+ /** Ready Tasks left queued because their pin named an Account this Run does not have (story 24).
514
+ * Absent when there were none. */
515
+ queued?: QueuedTask[];
516
+ /** Dispatches that threw; each Task is blocked on the Tracker and left for a human. */
517
+ failures: DispatchFailure[];
518
+ /** Set when a Breaker halted new dispatch: what tripped, and what a human should read. The Run
519
+ * still drained what was in flight before returning. */
520
+ breaker?: { condition: string; message: string };
521
+ /** What became of the Run's Proposed Learnings; absent when the Run has no learnings step. */
522
+ learnings?: LearningsResult;
523
+ /** The Run stopped because its Initiative is paused (story 55) and it was not asked to wait: the
524
+ * caller decides what to do with the rest of the work (the daemon puts the request back). */
525
+ paused?: boolean;
526
+ }
527
+
528
+ const defaultInstall = async (workspacePath: string): Promise<InstallResult | undefined> => {
529
+ const command = inferInstallCommand(workspacePath);
530
+ return command ? runInstall(command, workspacePath, { timeoutMs: INSTALL_TIMEOUT_MS }) : undefined;
531
+ };
532
+
533
+ const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
534
+
535
+ const noHandOff: HandOffFn = async () => ({ kind: "incomplete", reason: "this Run has no Hand-off configured (RunDeps.handOff), so nothing was pushed" });
536
+
537
+ const noGates: GatesFn = async () => ({ skipped: true, reason: "this Run has no Gates configured (RunDeps.gates)" });
538
+
539
+
540
+ /**
541
+ * One Account a Run may dispatch on: the Account, the Worker Adapter for its vendor, and the
542
+ * per-Account steps built from the two (the Verifier runs a second turn on the same Account; the
543
+ * Gates run their shell without its key). `RunDeps` carries the Run's own; `RunDeps.failover`
544
+ * carries the siblings (`runDepsFor`, commands/foreman.ts, builds them from the registered
545
+ * Accounts of the same vendor).
546
+ */
547
+ export type RunAccount = Pick<RunDeps, "account" | "worker" | "verifier" | "gates" | "maxTurns" | "stallTimeoutMs">;
548
+
549
+ /** An Account inside a Run: its `RunAccount` and the Slots the Run gives it. Slots are per Account
550
+ * because the cap is (ADR-0013): a Run with two Accounts runs each one's cap, not the sum on one. */
551
+ interface Lane extends RunAccount {
552
+ slots: Slots;
553
+ /** True for a Lane of another vendor (`RunDeps.pinnable`): only a Task whose label pins it here
554
+ * may use it, so an unpinned Ready Task never sees it and Failover never reaches it. */
555
+ pinOnly?: boolean;
556
+ }
557
+
558
+ /** The Run's deps with its Slots and the Tasks it has in flight: what every dispatch of one Run
559
+ * shares. `runForeman` builds one per Lane; nothing outside this module constructs one. */
560
+ interface Scheduling extends RunDeps {
561
+ slots: Slots;
562
+ /** Whether this Run has the Task's turn in flight, for Reconcile (a row it is driving is not a
563
+ * row a tick may restart). In-process only: it is this Run's own map, so a *second* Foreman on
564
+ * the same Process Table still sees a row whose adapter reports no pid as one to restart, and
565
+ * would dispatch a live Task twice. What keeps that from happening is the guard on starting a
566
+ * second Run on the Account at all (commands/foreman.ts), not this. */
567
+ owns: (taskId: number) => boolean;
568
+ /** Resolves when the Run is stopping; the Slot waits of Reconcile race against it. */
569
+ untilStop?: Promise<void>;
570
+ }
571
+
572
+ /**
573
+ * Runs the Initiative's Ready Tasks, filling the free Slots of every Account it may use each tick
574
+ * until none remains (or once, for `--once` and `--task`).
575
+ *
576
+ * A tick Reconciles, checks the pause and the stop, then takes Ready Tasks in priority-then-age
577
+ * order and starts each in a free Slot without waiting for the ones already going. It then waits
578
+ * for a dispatch to settle — or, when it ran out of Slots before it ran out of Tasks, for a Slot to
579
+ * come back; or, when every Account it could have used is Paused, for the soonest reset — and ticks
580
+ * again. The Run ends when nothing is in flight, no Ready Task remains and nothing is waiting on a
581
+ * reset, and its Proposed Learnings go out then (story 45).
582
+ *
583
+ * **Accounts (stories 30-31).** The Run's own Account is its first Lane; `RunDeps.failover` adds a
584
+ * Lane per sibling Account of the same vendor, each with its own Slots. A Ready Task takes the
585
+ * first Lane that is not Paused, is not another Foreman's (`contendedNow`) and has a Slot, which is
586
+ * the Failover: while Account A is Paused, the next Ready Task starts on Account B and A's own
587
+ * Workers finish where they are. A live session never changes Account — a Task with a Process Table
588
+ * row is pinned to that row's Lane and waits for it — so a session and the credential behind it
589
+ * always stay together; the Workspace is the Task's rather than the Account's, so a retry after a
590
+ * machine-side failure reuses it wherever it lands, and is told the tree is not clean
591
+ * (core/context.ts). Paused itself is
592
+ * read from the AccountStore each tick and is nothing but a timestamp: the tick after the reset
593
+ * passes fills that Account's Slots again, with no operator action.
594
+ *
595
+ * **The pin (story 24, core/pin.ts).** A Task labelled `foreman:vendor:codex` or
596
+ * `foreman:account:spare` narrows that Lane set to the Lanes the label allows, before any of the
597
+ * above is asked. `RunDeps.pinnable` adds a Lane per Account of another vendor, which only a pinned
598
+ * Task ever sees, so a Codex-pinned Task runs on the Codex Account while the Claude Slots beside it
599
+ * sit free. Everything the Lane set means afterwards is unchanged: a pinned Task whose only Lane is
600
+ * Paused **waits for that reset** rather than failing over — routing the work elsewhere is the one
601
+ * thing the label ruled out — and a pinned Task whose Lane is another Foreman's waits the same way.
602
+ * A pin no Lane can satisfy leaves the Task Ready with a comment (`leaveQueued`); it is never run
603
+ * somewhere else and never blocked.
604
+ */
605
+ export async function runForeman(deps: RunDeps, options: RunOptions): Promise<RunOutcome> {
606
+ const { tracker, account, log } = deps;
607
+ const { initiativeId } = options;
608
+ const now = deps.now ?? Date.now;
609
+ const dispatches: Dispatch[] = [];
610
+ /** Defect Tasks this Run has dispatched, against `defectDispatchCap`. */
611
+ let defectDispatches = 0;
612
+ /**
613
+ * The earliest this Run may dispatch a Task again, by Task id, for Tasks a retryable failure made
614
+ * Ready again (`retryBackoff`). Written where an attempt settles, read where the tick picks.
615
+ *
616
+ * This Run's own and in memory, exactly as `defectDispatches` is, so a Foreman that crashes and
617
+ * is restarted begins a fresh Run with nothing held back. That is the honest scope: the delay
618
+ * exists to stop one Run burning a Task's attempts as fast as an adapter can fail them, and a
619
+ * restarted Foreman is not that loop. Nothing durable is needed for it either — the Run Record
620
+ * already counts what the retry cap spends, and this only decides *when*, not *whether*.
621
+ */
622
+ const retryBackoffs = new Map<number, number>();
623
+ const blocked: BlockedTask[] = [];
624
+ const failures: DispatchFailure[] = [];
625
+ /** The Tasks this Run has a turn going for, by Task id; the promise never rejects. */
626
+ const inFlight = new Map<number, Promise<void>>();
627
+ const owns = (taskId: number) => inFlight.has(taskId);
628
+
629
+ // The Lanes, in the order a Ready Task tries them: this Run's own Account first, so Failover is
630
+ // the exception and the default Account is where work lands while nothing is Paused.
631
+ const siblings = (deps.failover ?? []).filter((s) => {
632
+ if (s.account.name === account.name) return false;
633
+ // Failover is within a vendor (CONTEXT.md): the Workspace is vendor-neutral but the session,
634
+ // the prompt contract and the adapter are not, so another vendor is a different Worker, not
635
+ // the same work moved.
636
+ if (s.account.vendor !== account.vendor) {
637
+ log(`run: Account "${s.account.name}" is ${s.account.vendor}, not ${account.vendor}; Failover is within a vendor, so it is not used for this Run`);
638
+ return false;
639
+ }
640
+ return true;
641
+ });
642
+ // Lanes only a pinned Task may use (story 24): the Accounts of another vendor. Deduplicated
643
+ // against the Lanes above by name, because two Lanes on one Account would be two Slots objects on
644
+ // one vendor identity — the cap counted twice, which is the one thing Lanes must never do.
645
+ const pinnable = (deps.pinnable ?? []).filter((p) => {
646
+ if (p.account.name === account.name || siblings.some((s) => s.account.name === p.account.name)) return false;
647
+ // `RunDeps.pinnable` says another vendor, and `pinnableAccountsFor` only ever builds it that
648
+ // way; enforced here as well because this is where it is relied on. A same-vendor Account is a
649
+ // Failover Lane — one an *unpinned* Task may use too — and quietly making it pin-only would
650
+ // narrow the fleet by dropping it out of `openLanes`.
651
+ if (p.account.vendor === account.vendor) {
652
+ log(`run: Account "${p.account.name}" is ${p.account.vendor}, the same vendor as "${account.name}"; a pin-only Lane is another vendor's Account, so it is not used for this Run (a sibling of the same vendor belongs in Failover)`);
653
+ return false;
654
+ }
655
+ return true;
656
+ });
657
+ const lanes: Lane[] = [
658
+ { account, worker: deps.worker, verifier: deps.verifier, gates: deps.gates, slots: new Slots(account.cap) },
659
+ ...siblings.map((s) => ({ ...s, slots: new Slots(s.account.cap) })),
660
+ ...pinnable.map((p) => ({ ...p, slots: new Slots(p.account.cap), pinOnly: true })),
661
+ ];
662
+ const slots = lanes[0].slots;
663
+ /** The Lanes an unpinned Ready Task may use: every Lane but the pin-only ones. */
664
+ const openLanes = lanes.filter((l) => !l.pinOnly);
665
+
666
+ // Lifts a human made through `foreman account resume`, which had no Initiative to record them
667
+ // against. This Run has one and a Lane on the Account, so it reports them now, dated when they
668
+ // happened rather than when they were noticed.
669
+ await reportResumedAccounts(deps, initiativeId, lanes);
670
+ /** One Lane's view of the Run: its Account and adapter in place of the Run's own, its per-turn
671
+ * limits (story 33: `foreman account set --max-turns` on a Failover Account must reach its own
672
+ * Workers, not the primary Account's numbers), its Slots. */
673
+ const schedulingFor = (lane: Lane): Scheduling => ({
674
+ ...deps,
675
+ account: lane.account,
676
+ worker: lane.worker,
677
+ verifier: lane.verifier,
678
+ gates: lane.gates,
679
+ maxTurns: lane.maxTurns ?? deps.maxTurns,
680
+ stallTimeoutMs: lane.stallTimeoutMs ?? deps.stallTimeoutMs,
681
+ slots: lane.slots,
682
+ owns,
683
+ untilStop: options.untilStop,
684
+ });
685
+ const scheduling = schedulingFor(lanes[0]);
686
+ const sleep = deps.sleep ?? defaultSleep;
687
+ // A poll of 0 is not "poll as fast as possible", it is a 1ms hot loop that burns a core while an
688
+ // Account waits out an hour; the floor makes a mistyped flag slow rather than harmful.
689
+ const pausePollMs = Math.max(MIN_PAUSE_POLL_MS, deps.pausePollMs ?? PAUSE_POLL_MS);
690
+ // `--once` and `--task` are one dispatch, whatever the cap: the human asked for one.
691
+ const single = options.once === true || options.taskId !== undefined;
692
+ log(
693
+ `run: Initiative ${initiativeId} on Account "${account.name}" (${account.vendor}), cap ${slots.cap}: ${single ? "one dispatch (asked for)" : `${count(slots.implementerCap, "implementer Slot")}${slots.reservesVerifierSlot ? " and one reserved for Verifiers" : " and no Slot to reserve for Verifiers"}`}${siblings.length ? `; Failover to ${siblings.map((s) => `"${s.account.name}" (cap ${s.account.cap})`).join(", ")} while it is Paused` : ""}${pinnable.length ? `; ${pinnable.map((p) => `"${p.account.name}" (${p.account.vendor}, cap ${p.account.cap})`).join(", ")} for a Task pinned there and nothing else` : ""}`,
694
+ );
695
+
696
+ /**
697
+ * Which Accounts are Paused right now, and until when (story 30). Read from the AccountStore
698
+ * rather than from the Lane, so a pause written by the turn that hit the limit — or by another
699
+ * Foreman on the same Account — is seen the moment the next Slot is filled, and a pause whose
700
+ * reset has passed is already gone (`pausedUntil`). Without a store the Lane's own snapshot is
701
+ * all there is, which is the shape a test that hands in no store wants.
702
+ */
703
+ const pausedNow = (at: number, stored = deps.accounts?.load()): Map<string, Date> => {
704
+ const paused = new Map<string, Date>();
705
+ for (const lane of lanes) {
706
+ const current = stored?.find((a) => a.name === lane.account.name) ?? lane.account;
707
+ const until = pausedUntil(current, at);
708
+ if (until) paused.set(lane.account.name, until);
709
+ }
710
+ return paused;
711
+ };
712
+
713
+ /**
714
+ * Which Lanes are quarantined right now (OPS-288). Read from the AccountStore every tick for the
715
+ * reasons `pausedNow` is: the turn that proved the credential dead wrote it a moment ago, and a
716
+ * quarantine another Foreman wrote is just as binding — the Account is one vendor identity and
717
+ * its credential is dead for everyone.
718
+ *
719
+ * Unlike a pause this takes no `at`: nothing about a quarantine changes with the clock, which is
720
+ * exactly what makes it worth reporting differently when a Run runs out of Lanes.
721
+ */
722
+ const quarantinedNow = (stored = deps.accounts?.load()): Map<string, string> => {
723
+ const quarantined = new Map<string, string>();
724
+ for (const lane of lanes) {
725
+ const current = stored?.find((a) => a.name === lane.account.name) ?? lane.account;
726
+ if (current.quarantined) quarantined.set(lane.account.name, current.quarantined.reason);
727
+ }
728
+ return quarantined;
729
+ };
730
+
731
+ /** Lanes already reported as quarantined, so the log says it once rather than once per tick. */
732
+ const quarantineLogged = new Set<string>();
733
+
734
+ /** Lanes already reported as another Foreman's, so the log says it once. */
735
+ const contendedLogged = new Set<string>();
736
+
737
+ /** Which of the Lanes after the first — Failover Lanes and pin-only Lanes alike — another Foreman
738
+ * has Workers on, and how many, at `at`.
739
+ *
740
+ * `failoverAccountsFor` and `pinnableAccountsFor` (commands/foreman.ts) ask the same question,
741
+ * but they can only ask once, before the Run starts: an Account with no live rows then is a Lane,
742
+ * and nothing stops a second `foreman run --account b` from starting on it a minute later, since
743
+ * that Run's own guard (`refuseSecondRunOnAccount`) sees no Workers on b either. Two Runs would
744
+ * then have their own Slots on one vendor identity, which is the thing per-Account caps exist to
745
+ * prevent (ADR-0013). So the Process Table is asked again every tick, and a Lane that is not
746
+ * clean is not used this tick. This Run's own turns are not contention (`owns`), and the Run's
747
+ * first Lane is left out: it is guarded at start, and dropping the Account the human named would
748
+ * be a Run that does nothing rather than one that keeps within the cap.
749
+ *
750
+ * **Without a Process Table there is no second ask**, and this returns nothing: the Run has only
751
+ * the snapshot the two functions above took, which is `RunDeps` without `processTable` — the
752
+ * shape a unit test hands in, never `foreman run`, which always opens one (commands/foreman.ts).
753
+ */
754
+ const contendedNow = (at: number): Map<string, number> => {
755
+ const contended = new Map<string, number>();
756
+ if (!deps.processTable || lanes.length < 2) return contended;
757
+ for (const lane of lanes.slice(1)) {
758
+ const live = otherForemanWorkersOn(deps.processTable, lane.account.name, at, { isAlive: deps.isAlive, owns });
759
+ if (live.length === 0) {
760
+ contendedLogged.delete(lane.account.name);
761
+ continue;
762
+ }
763
+ contended.set(lane.account.name, live.length);
764
+ // Once per spell, not once per tick: a Foreman that holds the Lane for an hour must not write
765
+ // a line every poll.
766
+ if (contendedLogged.has(lane.account.name)) continue;
767
+ contendedLogged.add(lane.account.name);
768
+ log(
769
+ `run: Account "${lane.account.name}" now has ${count(live.length, "Worker")} running under another Foreman (${live.map((r) => `${r.identifier}${r.pid === undefined ? " (starting)" : ` (pid ${r.pid})`}`).join(", ")}); ` +
770
+ `it is not used for ${lane.pinOnly ? "a pinned Task" : "Failover"} while that lasts, since its cap is that Foreman's to fill`,
771
+ );
772
+ }
773
+ return contended;
774
+ };
775
+
776
+ /** Tasks whose row and whose label disagree, so the log says it once rather than every tick. */
777
+ const disagreementLogged = new Set<number>();
778
+
779
+ /** Where a Ready Task may run: the Lanes it is allowed, or why there are none. */
780
+ type LanePick =
781
+ | { kind: "lanes"; lanes: Lane[] }
782
+ /** Its Process Table row is under an Account this Run has no Lane for. */
783
+ | { kind: "foreign"; account: string }
784
+ /** Its label pins it somewhere this Run cannot reach, or cannot be read as a pin at all. */
785
+ | { kind: "unplaceable"; pin: Pin; why: Unplaceable };
786
+
787
+ /**
788
+ * The Lanes a Task may run in, narrowed twice.
789
+ *
790
+ * First by its Process Table row, when it has one: a row is what "started" means (ADR-0009: it
791
+ * exists from the moment the dispatch is committed), and started work does not change Account.
792
+ * **The row wins over the label**, always — a vendor session cannot be resumed under another
793
+ * identity, so a label added after a Worker started cannot move it, and re-pinning takes effect
794
+ * on the Task's next attempt. A disagreement between the two is worth a line, because to the
795
+ * developer who wrote the label it looks like the pin was ignored.
796
+ *
797
+ * Then by the pin its labels carry (story 24, core/pin.ts): a vendor, one named Account, or
798
+ * nothing. An unpinned Task never sees a pin-only Lane. A pin that no Lane satisfies does *not*
799
+ * fall back to running somewhere else — that is the whole point of a pin — so the Task comes back
800
+ * unplaceable and is left Ready with a comment. `registered` is every Account on the machine,
801
+ * which is what lets that comment tell "you never registered this Account" apart from "another
802
+ * Foreman is on it, wait": two conditions with opposite remedies (`unplaceable`, core/pin.ts).
803
+ */
804
+ const lanesFor = (task: TrackerTask, registered?: readonly Account[]): LanePick => {
805
+ const row = deps.processTable?.worker(task.id);
806
+ const pin = pinOf(task.labels);
807
+ if (row) {
808
+ const started = lanes.find((l) => l.account.name === row.account);
809
+ // A row under an Account this Run does not have is not this Run's Task at all; Reconcile skips
810
+ // it for the same reason, and picking it here would take it off its own Foreman.
811
+ if (!started) return { kind: "foreign", account: row.account };
812
+ if (pin && !pinAllows(pin, started.account) && !disagreementLogged.has(task.id)) {
813
+ disagreementLogged.add(task.id);
814
+ log(
815
+ `run: ${task.identifier} is ${describePin(pin)} but has a live Worker on Account "${started.account.name}"; ` +
816
+ `a started session never changes Account, so the row wins and the pin applies to its next attempt`,
817
+ );
818
+ }
819
+ return { kind: "lanes", lanes: [started] };
820
+ }
821
+ if (!pin) return { kind: "lanes", lanes: openLanes };
822
+ const why = unplaceable(
823
+ pin,
824
+ lanes.map((l) => l.account),
825
+ registered,
826
+ );
827
+ if (why) return { kind: "unplaceable", pin, why };
828
+ return { kind: "lanes", lanes: lanes.filter((l) => pinAllows(pin, l.account)) };
829
+ };
830
+
831
+ // Tasks whose dispatch threw and Tasks pinned where this Run cannot reach: retried by the next
832
+ // Run, never by this one, so a Run cannot spin on them.
833
+ //
834
+ // A Triage turn that stalled or hit a rate limit is deliberately *not* here. It is a machine-side
835
+ // failure like any other, and the thing that bounds it is the retry cap the Run Record already
836
+ // counts (`retryableFailuresOf`), not the length of the night: deferring it would cost a Task the
837
+ // whole Run for one bad minute at 3am, when a rate limit resolves by itself as soon as the
838
+ // Account's window resets.
839
+ const deferred = new Set<number>();
840
+ const queued: QueuedTask[] = [];
841
+
842
+ /**
843
+ * Leaves a pinned Task Ready with one comment saying where it wanted to go, what this Run has,
844
+ * and which of the three things fixes it (story 24).
845
+ *
846
+ * Deferred, so it is passed over for the rest of the Run: no tick of this Run can register an
847
+ * Account or fix a label, so looking again would only be a way to comment twice. Nothing else is
848
+ * touched — the dispatch label stays on, no Status Label goes on, no Run Record event is written
849
+ * and this Run spends no attempt on it — so the Task is exactly as Ready as it was, for the next
850
+ * Run or for a human. A Tracker that refuses the comment is logged and the Task is still left
851
+ * alone: failing to say why must not turn into running the Task somewhere the developer ruled out.
852
+ *
853
+ * **Once per condition, not once per Run.** The deferral above only covers this process:
854
+ * `deferred` does not survive a restart, a second `foreman run`, or the daemon building a fresh
855
+ * Run for the next request, and a Task pinned at an Account nobody intends to register is a
856
+ * condition that lasts indefinitely — a nightly daemon would leave an identical comment every
857
+ * night. So the Tracker is the record, the way the Defect filer makes filing idempotent
858
+ * (core/defects.ts): the comment carries `pinToken(pin)`, and a Task that already has one for
859
+ * this pin is left in silence. The token is the pin's, not the Run's, so correcting a label into
860
+ * a *different* unplaceable pin is a new condition and is said again.
861
+ */
862
+ const leaveQueued = async (task: TrackerTask, pin: Pin, why: Unplaceable): Promise<void> => {
863
+ deferred.add(task.id);
864
+ queued.push({ task, pin, reason: why.reason });
865
+ log(`run: ${task.identifier} ${why.reason} It stays Ready with its "${DISPATCH_LABEL}" label; nothing was dispatched and this Run spent no attempt on it.`);
866
+ const token = pinToken(pin);
867
+ try {
868
+ const { comments } = await tracker.getTask(task.id);
869
+ if (comments.some((c) => c.body.includes(token))) {
870
+ log(`run: ${task.identifier} was already told this about ${token}; the comment is not repeated`);
871
+ return;
872
+ }
873
+ } catch (error) {
874
+ // The Task's own comments could not be read. Saying it twice is a worse outcome than saying
875
+ // it once, but a silent Task is worse than both, so the comment goes out.
876
+ log(`run: ${task.identifier} comments could not be read back, so this may repeat an earlier one: ${error instanceof Error ? error.message : String(error)}`);
877
+ }
878
+ try {
879
+ await tracker.addComment(task.id, `Foreman: this Task ${why.reason} It is left Ready and untouched — this Run spent no attempt on it and it is not blocked — so ${why.remedy}. ${token}`);
880
+ } catch (error) {
881
+ log(`run: ${task.identifier} could not be commented on: ${error instanceof Error ? error.message : String(error)}`);
882
+ }
883
+ };
884
+ const record = (settled: Dispatch | BlockedTask) => {
885
+ if (!("missing" in settled)) {
886
+ dispatches.push(settled);
887
+ // A retried dispatch says when it may be taken again; anything else has finished with the
888
+ // Task, so an entry an earlier attempt left is dropped rather than outliving what set it.
889
+ if (settled.retryNotBefore !== undefined) retryBackoffs.set(settled.task.id, settled.retryNotBefore);
890
+ else retryBackoffs.delete(settled.task.id);
891
+ return;
892
+ }
893
+ blocked.push(settled);
894
+ // `retry` means the Task kept its dispatch label and is Ready again, and a later tick of this
895
+ // Run takes it: not deferred, but not immediately either. `defer` is the Task that kept its
896
+ // label for a reason no tick of this Run can change. Everything else is blocked and has lost
897
+ // the label anyway.
898
+ if (settled.retry) retryBackoffs.set(settled.task.id, settled.retry.notBefore);
899
+ else retryBackoffs.delete(settled.task.id);
900
+ if (settled.defer) deferred.add(settled.task.id);
901
+ };
902
+
903
+ /** Pin-only Lanes this Run has actually dispatched onto, which is what makes one of them a Lane
904
+ * this Run may Reconcile (see the Reconcile loop). */
905
+ const workedPinOnly = new Set<string>();
906
+
907
+ /** Starts a dispatch in the Lane's Slot it has been given and keeps its promise until it settles. */
908
+ const fill = (task: TrackerTask, lane: Lane, hold: SlotHold): void => {
909
+ if (lane.pinOnly) workedPinOnly.add(lane.account.name);
910
+ const running = (async () => {
911
+ try {
912
+ record(await dispatch(schedulingFor(lane), initiativeId, task, hold));
913
+ } catch (error) {
914
+ // `dispatch` has already commented on the Task and marked it blocked; a Slot's failure is
915
+ // the Run's to report, never the other Slots' to die of.
916
+ const reason = error instanceof Error ? error.message : String(error);
917
+ log(`run: ${task.identifier} was left blocked by a failed dispatch; the other Slots go on: ${reason}`);
918
+ failures.push({ task, reason });
919
+ deferred.add(task.id);
920
+ } finally {
921
+ hold.release();
922
+ inFlight.delete(task.id);
923
+ }
924
+ })();
925
+ if (task.labels?.includes(DEFECT_LABEL)) defectDispatches += 1;
926
+ inFlight.set(task.id, running);
927
+ };
928
+
929
+ /** Settles every turn still going, so the Run never returns with Workers it started unaccounted. */
930
+ const drain = async () => {
931
+ while (inFlight.size) await Promise.all([...inFlight.values()]);
932
+ };
933
+
934
+ let pausedLogged = false;
935
+ let started = 0;
936
+ /** The project's priority scale, logged the first tick that shows one (core/scheduler.ts). */
937
+ let scaleLogged = false;
938
+ // Whatever ends the loop — no Ready Task, a stop, or a throw from the Tracker — the turns this
939
+ // Run started are settled before it reports, so it never leaves Workers unaccounted.
940
+ try {
941
+ for (;;) {
942
+ // The stop is read before Reconcile, not after it: a Reconcile that must resume or restart a row
943
+ // takes an implementer Slot for it, and at the default cap there is one, so it can be waiting
944
+ // on a Worker turn to end. Asking here means a stop that arrived while the last tick waited is
945
+ // acted on now rather than a whole turn later, and the wait inside Reconcile is abandoned by
946
+ // `untilStop` besides. The pause is deliberately *not* moved up with it: a paused Run still
947
+ // finishes what is in flight (story 55), and Reconcile is what finishes it.
948
+ if (options.shouldStop?.()) {
949
+ log(`run: stopping as asked, after ${count(dispatches.length, "dispatch", "dispatches")}${inFlight.size ? `, waiting for ${count(inFlight.size, "turn")} in flight` : ""}`);
950
+ break;
951
+ }
952
+
953
+ // Reconcile before anything new is picked (story 51): what a previous process left in the
954
+ // Process Table is settled first, so a resumed Task is never dispatched a second time. Rows
955
+ // this Run is driving are left alone (`owns`), whatever their pid says. One pass per Lane,
956
+ // since a row is reconciled by a Run under its own Account and skipped by every other
957
+ // (reconcile.ts): that is how a Worker left on Account B is resumed on B, whichever Account
958
+ // this Run calls its own. The outbox is drained by the first pass alone — it is the Run's,
959
+ // not the Lane's.
960
+ for (const [index, lane] of lanes.entries()) {
961
+ // A pin-only Lane is another vendor's Account this Run adopted for pinned Tasks alone, and
962
+ // until it has dispatched onto one it is not this Run's to settle. `owns` is per-Run and
963
+ // in-process, so a row another Foreman on that Account wrote — one whose settle path is
964
+ // between the Worker ending and `removeWorker` — reads here as a dead pid with a session,
965
+ // which `planReconcile` calls a resume: this Run would drive a second turn on that
966
+ // Foreman's Task, in its Workspace, under its identity. Merely having a Codex Account
967
+ // registered must not make a Claude Run adopt its rows. Nothing is orphaned by waiting:
968
+ // the daemon's idle Reconcile groups live rows by their own Account and settles them under
969
+ // it (commands/foreman-up.ts), and a Run started on that Account settles them itself.
970
+ if (lane.pinOnly && !workedPinOnly.has(lane.account.name)) continue;
971
+ const laneDeps = index === 0 ? scheduling : { ...schedulingFor(lane), drainOutbox: undefined };
972
+ for (const settled of await reconcileTick(laneDeps, initiativeId)) record(settled);
973
+ }
974
+ // Paused (story 55): nothing new starts, and what is in flight is settled before the Run
975
+ // reports. A Run asked to wait idles here until the resume, at the daemon's tick; any other
976
+ // returns the pause as its outcome, so a daemon can put the request back and serve others.
977
+ if (deps.processTable?.isPaused(initiativeId)) {
978
+ if (!options.waitWhilePaused) {
979
+ await drain();
980
+ log(`run: Initiative ${initiativeId} is paused; stopping after ${count(dispatches.length, "dispatch", "dispatches")}, the rest waits for foreman resume ${initiativeId}`);
981
+ return { dispatches, blocked, failures, ...(queued.length ? { queued } : {}), paused: true };
982
+ }
983
+ if (!pausedLogged) log(`run: Initiative ${initiativeId} is paused; no new dispatch until foreman resume ${initiativeId}`);
984
+ pausedLogged = true;
985
+ await sleep(pausePollMs);
986
+ continue;
987
+ }
988
+ if (pausedLogged) log(`run: Initiative ${initiativeId} resumed; dispatch continues`);
989
+ pausedLogged = false;
990
+
991
+ // The Breaker (CONTEXT.md: Breaker). Same shape as the pause above and for the same reason:
992
+ // nothing new starts, what is in flight is settled before the Run reports, and the fact is on
993
+ // the Run Record for the morning. Checked here rather than at the dispatch site so a Run that
994
+ // trips mid-tick still finishes the turns it already started.
995
+ const defectCap = deps.defectDispatchCap ?? DEFAULT_DEFECT_DISPATCH_CAP;
996
+ if (defectDispatches >= defectCap) {
997
+ const message =
998
+ `${count(defectDispatches, "Defect Task")} dispatched in this Run, which is the cap (${defectCap}).` +
999
+ ` New dispatch is stopped so a defect-fix-defect loop cannot spend the rest of the night;` +
1000
+ ` the Tasks already running finish. Raise it with defectDispatchCap, or look at the Initiative's plan.`;
1001
+ log(`run: breaker: ${message}`);
1002
+ await tracker.appendRunEvents(initiativeId, [runEvent(undefined, { payload: { case: "breaker", value: { condition: "defect_loop", message } } })]);
1003
+ await drain();
1004
+ return await summarised(deps, initiativeId, { dispatches, blocked, failures, ...(queued.length ? { queued } : {}), breaker: { condition: "defect_loop", message } });
1005
+ }
1006
+
1007
+ /** True when Ready Tasks were left over for want of a Slot: the tick then waits for one back. */
1008
+ let slotLimited = false;
1009
+ /** True when the only Lanes left for a Ready Task are ones another Foreman has Workers on:
1010
+ * the tick polls rather than stopping, since that Foreman's Workers do end. */
1011
+ let laneContended = false;
1012
+ /** Ready Tasks whose Process Table row is under an Account this Run has no Lane for; counted
1013
+ * so the Run's tally says they were left rather than implying there was nothing to do. */
1014
+ let skippedForeign = 0;
1015
+ /** The soonest reset a Ready Task is waiting on, when every Lane it may use is Paused: the
1016
+ * tick sleeps until then rather than spinning, and dispatch resumes with no operator action. */
1017
+ let waitingOnReset: Date | undefined;
1018
+ /** True when a Ready Task's every Lane is quarantined (OPS-288). Unlike a pause there is
1019
+ * nothing to wait for and unlike contention nobody else's Workers will end, so this is the one
1020
+ * of the three that ends the Run rather than making it poll. */
1021
+ let laneQuarantined = false;
1022
+ /** The soonest a Ready Task held back by `retryBackoff` may be taken. Shaped like
1023
+ * `waitingOnReset` and waited on the same way: both are a moment this tick knows about and
1024
+ * neither needs anyone to do anything when it arrives. */
1025
+ let waitingOnBackoff: number | undefined;
1026
+ /** Which Accounts are Paused this tick; read once, and reused by the wait below. */
1027
+ let paused = new Map<string, Date>();
1028
+ if (single && started > 0) {
1029
+ // The one dispatch has started; nothing more is picked, only awaited.
1030
+ if (inFlight.size === 0) break;
1031
+ } else {
1032
+ const candidates = await pickTasks(tracker, options, deferred, inFlight);
1033
+ if (!scaleLogged) {
1034
+ // Dispatch takes the highest `TaskPriority.level` first, which is the seeded scale's
1035
+ // convention and not something Opsee enforces (core/scheduler.ts): a project that numbers
1036
+ // the other way would silently run in reverse, so the Run says what it is seeing.
1037
+ const scale = priorityScale(candidates);
1038
+ if (scale) {
1039
+ log(`run: priority levels in this project, highest first: ${scale}; dispatch takes the highest level first`);
1040
+ scaleLogged = true;
1041
+ }
1042
+ }
1043
+ // Every Account on the machine, read once and used twice: for Paused below, and for the
1044
+ // comment on a Task no Lane can take, where an Account that is registered but not a Lane is
1045
+ // one to wait for rather than one to register (core/pin.ts).
1046
+ const registered = deps.accounts?.load();
1047
+ paused = pausedNow(now(), registered);
1048
+ const quarantined = quarantinedNow(registered);
1049
+ // Which Lanes after the first — Failover and pin-only alike — another Foreman is on right
1050
+ // now, asked once per tick and again every tick: `failoverAccountsFor` and
1051
+ // `pinnableAccountsFor` (commands/foreman.ts) can only ask before the Run starts, and a
1052
+ // Lane that was latent then may be a second Foreman's by the time it is first used.
1053
+ const contended = contendedNow(now());
1054
+ let taken = 0;
1055
+ for (const task of candidates) {
1056
+ // Held back after a retryable failure, so its next attempt is not taken as fast as the
1057
+ // last one failed (`retryBackoff`). Asked before the Lanes, because the answer does not
1058
+ // depend on them: a Task waiting out its own backoff is waiting wherever it would run,
1059
+ // and a free Slot found for it here would only be handed straight back.
1060
+ // An entry whose moment has passed is simply not a reason to skip; it is left for the
1061
+ // Task's next settlement to replace or drop (`record`), rather than swept here, since a
1062
+ // Map keyed by Task id cannot grow past the Initiative and a second place that edits it
1063
+ // is a second place to get it wrong.
1064
+ const notBefore = retryBackoffs.get(task.id);
1065
+ if (notBefore !== undefined && notBefore > now()) {
1066
+ if (waitingOnBackoff === undefined || notBefore < waitingOnBackoff) waitingOnBackoff = notBefore;
1067
+ continue;
1068
+ }
1069
+ // The Lanes this Task may use, then the ones that are neither Paused nor another
1070
+ // Foreman's, then the first with a free Slot. A Task pinned to a Lane gets exactly one
1071
+ // candidate, so a Paused Account holds its own started work rather than handing it to a
1072
+ // sibling.
1073
+ const pick = lanesFor(task, registered);
1074
+ if (pick.kind === "foreign") {
1075
+ // The row is under an Account this Run does not have: another Foreman's Task, which
1076
+ // Reconcile skips for the same reason. Said out loud, because a silent skip here made
1077
+ // the Run claim below that no Ready Task remained.
1078
+ log(`run: ${task.identifier} has a Process Table row under Account "${pick.account}", which is not one of this Run's Lanes (${lanes.map((l) => `"${l.account.name}"`).join(", ")}); it belongs to that Account's Foreman and is left alone`);
1079
+ skippedForeign++;
1080
+ continue;
1081
+ }
1082
+ if (pick.kind === "unplaceable") {
1083
+ // Pinned somewhere this Run has no Lane for, or a `foreman:` label that is not a pin at
1084
+ // all. Either way running it here is the one thing the label rules out.
1085
+ await leaveQueued(task, pick.pin, pick.why);
1086
+ continue;
1087
+ }
1088
+ // Quarantine is asked before Paused, because it is the stronger fact: an Account whose
1089
+ // credential is dead cannot run this Task when its pause lifts either, and a Task waiting
1090
+ // on the reset of an Account that will never work again is a Task that never runs
1091
+ // (OPS-288). A quarantined Lane contributes no Slots to anyone.
1092
+ const usable = pick.lanes.filter((l) => !quarantined.has(l.account.name));
1093
+ if (usable.length === 0) {
1094
+ for (const lane of pick.lanes) {
1095
+ if (quarantineLogged.has(lane.account.name)) continue;
1096
+ quarantineLogged.add(lane.account.name);
1097
+ log(
1098
+ `run: Account "${lane.account.name}" (${lane.account.vendor}) is quarantined (${printableOneLine(quarantined.get(lane.account.name) ?? "")}); ` +
1099
+ `it contributes no Slots and is not a Failover target. Log the vendor back in, then: opsee foreman account resume ${lane.account.name}`,
1100
+ );
1101
+ }
1102
+ // A pinned Task is refused rather than merely skipped, and is told so on the Task
1103
+ // itself: the label ruled out running it anywhere else, so nothing this Run does next
1104
+ // will place it, and the human who wrote the pin is the only one who can. An unpinned
1105
+ // Task gets no comment — it can run on any Lane, so what it is waiting for is the Run's
1106
+ // fact and not the Task's, and a comment on every Ready Task would be noise.
1107
+ // A pin that could not be read never reaches here: `lanesFor` answers `unplaceable`
1108
+ // for those and the branch above has already refused the Task.
1109
+ const pin = pinOf(task.labels);
1110
+ if (pin) {
1111
+ await leaveQueued(task, pin, {
1112
+ reason: `is ${describePin(pin)}, and every Account that satisfies it is quarantined: its credential is dead, not rate limited.`,
1113
+ remedy: `log the vendor back in and run "opsee foreman account resume <name>" (or re-register the Account), or change the pin`,
1114
+ });
1115
+ continue;
1116
+ }
1117
+ laneQuarantined = true;
1118
+ continue;
1119
+ }
1120
+ const notPaused = usable.filter((l) => !paused.has(l.account.name));
1121
+ if (notPaused.length === 0) {
1122
+ // Every Lane this Task may use is Paused: it waits for the soonest of their resets.
1123
+ for (const lane of usable) {
1124
+ const until = paused.get(lane.account.name)!;
1125
+ if (!waitingOnReset || until < waitingOnReset) waitingOnReset = until;
1126
+ }
1127
+ continue;
1128
+ }
1129
+ const open = notPaused.filter((l) => !contended.has(l.account.name));
1130
+ if (open.length === 0) {
1131
+ // Only Lanes another Foreman has Workers on are left: opening Slots of our own there
1132
+ // would double one vendor identity's concurrency (ADR-0013), so the Task waits.
1133
+ laneContended = true;
1134
+ continue;
1135
+ }
1136
+ const lane = open.find((l) => l.slots.free > 0);
1137
+ if (!lane) {
1138
+ slotLimited = true;
1139
+ continue;
1140
+ }
1141
+ if (lane !== lanes[0]) {
1142
+ const pin = pinOf(task.labels);
1143
+ // Landing off the Run's own Account is either the pin doing its job or Failover doing
1144
+ // its job, and the two want different sentences: one is what the developer asked for,
1145
+ // the other is what the Foreman did because it had to.
1146
+ // Why the Run's own Account was passed over. There are two answers now, and the wrong
1147
+ // one is worse than none: a quarantine has no reset, so the pause wording would print
1148
+ // "Paused until ?" for a state that is not a pause and never lifts (OPS-288).
1149
+ const why = quarantined.has(lanes[0].account.name)
1150
+ ? `"${lanes[0].account.name}" is quarantined (${printableOneLine(quarantined.get(lanes[0].account.name) ?? "")})`
1151
+ : `"${lanes[0].account.name}" is Paused until ${paused.get(lanes[0].account.name)?.toISOString() ?? "?"}`;
1152
+ log(
1153
+ pin && pinAllows(pin, lane.account)
1154
+ ? `run: ${task.identifier} runs on Account "${lane.account.name}" (${lane.account.vendor}): it is ${describePin(pin)}`
1155
+ : `run: ${task.identifier} fails over to Account "${lane.account.name}": ${why}`,
1156
+ );
1157
+ }
1158
+ fill(task, lane, lane.slots.acquire());
1159
+ taken++;
1160
+ started++;
1161
+ if (single) break;
1162
+ }
1163
+ // Before the tally below, which would otherwise report "no Ready Task remains" for a Run
1164
+ // that has plenty of Ready Tasks and nowhere to run them. A quarantine is not waited out:
1165
+ // nothing lifts it but a human, so the Run says what to do and ends (OPS-288).
1166
+ //
1167
+ // `waitingOnBackoff` holds this back too: a Task serving out the delay after its last
1168
+ // failed attempt may still have a Lane to run on, so ending the Run on another Task's
1169
+ // quarantine would abandon work that was only waiting.
1170
+ if (laneQuarantined && taken === 0 && !slotLimited && !waitingOnReset && !laneContended && waitingOnBackoff === undefined && inFlight.size === 0) {
1171
+ log(
1172
+ `run: every Account a Ready Task could use is quarantined; ${count(dispatches.length, "dispatch", "dispatches")} made and Ready Tasks remain. ` +
1173
+ `Nothing lifts a quarantine by itself: log the vendor back in and run "opsee foreman account resume <name>", or register another Account, then start the Run again.`,
1174
+ );
1175
+ break;
1176
+ }
1177
+ if (taken === 0 && !slotLimited && !waitingOnReset && !laneContended && !laneQuarantined && waitingOnBackoff === undefined && inFlight.size === 0) {
1178
+ const tally = `${count(dispatches.length, "dispatch", "dispatches")}${blocked.length ? `, ${count(blocked.length, "Task")} blocked before dispatch` : ""}${failures.length ? `, ${count(failures.length, "failed dispatch", "failed dispatches")}` : ""}${skippedForeign ? `, ${count(skippedForeign, "Task")} left to another Account's Foreman` : ""}${queued.length ? `, ${count(queued.length, "Task")} left Ready with a pin this Run has no Account for` : ""}`;
1179
+ log(`run: no Ready Task remains in Initiative ${initiativeId}; stopping after ${tally}`);
1180
+ break;
1181
+ }
1182
+ if (taken) {
1183
+ const inUse = lanes.map((l) => `"${l.account.name}" ${l.slots.inFlight}/${l.slots.cap}`).join(", ");
1184
+ log(`run: ${count(taken, "Slot")} filled, ${inUse} in use${slotLimited ? "; Ready Tasks are waiting for a Slot" : ""}`);
1185
+ }
1186
+ if (waitingOnReset && taken === 0 && inFlight.size === 0) {
1187
+ log(`run: every Account a Ready Task could use is Paused; waiting for the reset at ${waitingOnReset.toISOString()}`);
1188
+ }
1189
+ if (waitingOnBackoff !== undefined && taken === 0 && inFlight.size === 0) {
1190
+ log(`run: every Ready Task is waiting out the delay after its last failed attempt; the next may be dispatched in ${Math.max(0, Math.round((waitingOnBackoff - now()) / 1000))}s`);
1191
+ }
1192
+ // One dispatch was asked for and no Lane can take it: say why and stop, rather than idling
1193
+ // out a pause that is up to `MAX_PAUSE_MS` away with nobody watching. The daemon's own Runs
1194
+ // are `--once` too, and a request it cannot serve now belongs back in its queue.
1195
+ if (single && taken === 0 && inFlight.size === 0 && (waitingOnReset || laneContended)) {
1196
+ log(
1197
+ waitingOnReset
1198
+ ? `run: one dispatch was asked for and every Account a Ready Task could use is Paused until ${waitingOnReset.toISOString()}; nothing is dispatched. Try again after the reset, or run it on another Account (--account <name>).`
1199
+ : `run: one dispatch was asked for and every Account a Ready Task could use has Workers under another Foreman; nothing is dispatched. Wait for that Foreman, or run it on another Account (--account <name>).`,
1200
+ );
1201
+ break;
1202
+ }
1203
+ }
1204
+
1205
+ // Wait for something to change: a dispatch settling (its Task may release successors); or,
1206
+ // when Ready Tasks were waiting, a Slot coming back on any Lane that could take one — which a
1207
+ // dispatch entering verification gives without settling anything; or, when the only thing in
1208
+ // the way is a Paused Account, the reset itself, capped at the pause poll so a pause lifted
1209
+ // by hand (or by another Foreman) is noticed too.
1210
+ const waits: Promise<unknown>[] = [...inFlight.values()];
1211
+ if (slotLimited) for (const lane of lanes) if (!paused.has(lane.account.name)) waits.push(lane.slots.nextRelease());
1212
+ if (waitingOnReset) waits.push(sleep(Math.max(1, Math.min(waitingOnReset.getTime() - now(), pausePollMs))));
1213
+ // Unlike a pause, nothing but this Run's own clock can end a backoff early, so it is slept
1214
+ // out whole rather than polled. `Math.max(1, ...)` keeps a wait that has just elapsed from
1215
+ // becoming a zero-length sleep the loop would spin on.
1216
+ if (waitingOnBackoff !== undefined) waits.push(sleep(Math.max(1, waitingOnBackoff - now())));
1217
+ // Another Foreman's Workers on the only Lane left: nothing here is notified when they end, so
1218
+ // the Process Table is polled at the same interval a pause is.
1219
+ if (laneContended && !waitingOnReset) waits.push(sleep(pausePollMs));
1220
+ if (waits.length === 0) break;
1221
+ await Promise.race(waits);
1222
+ if (deps.tickDelayMs) await sleep(deps.tickDelayMs);
1223
+ }
1224
+ } finally {
1225
+ await drain();
1226
+ }
1227
+ return await summarised(deps, initiativeId, { dispatches, blocked, failures, ...(queued.length ? { queued } : {}), learnings: await submitLearnings(deps, initiativeId, dispatches) });
1228
+ }
1229
+
1230
+ /**
1231
+ * Leaves the Run's summary on the Initiative and hands the outcome back unchanged (story 62).
1232
+ *
1233
+ * Here rather than in the command, so the daemon's Runs get it too: `foreman run` and `foreman up`
1234
+ * both come through `runForeman` and both used to end with a line on a terminal nobody was watching
1235
+ * at 3am. Every exit goes through this, an early stop included — a Breaker halting dispatch is the
1236
+ * night that most needs explaining, and it is exactly the night that would otherwise be silent.
1237
+ *
1238
+ * The Run's own facts are already committed by the time this runs: the Run Record has every event
1239
+ * and each Task carries its comment and its Status Label. So a Tracker that refuses this write costs
1240
+ * the morning its summary and nothing else, and it must not turn a Run that worked into a Run that
1241
+ * threw on its last line.
1242
+ */
1243
+ async function summarised(deps: RunDeps, initiativeId: number, outcome: RunOutcome): Promise<RunOutcome> {
1244
+ try {
1245
+ // No line on success. `submitLearnings`, the other last step of a Run, reports through the
1246
+ // outcome and logs only what went wrong; a routine write announcing itself every Run is noise
1247
+ // in a log read to find out what did not happen.
1248
+ await deps.tracker.addInitiativeComment(initiativeId, runSummaryComment(outcome));
1249
+ } catch (error) {
1250
+ deps.log(`run: the Run summary could not be posted on Initiative ${initiativeId}, so the morning has only this terminal: ${error instanceof Error ? error.message : String(error)}`);
1251
+ }
1252
+ return outcome;
1253
+ }
1254
+
1255
+ /** The Run's last step: its Proposed Learnings, gathered from every dispatch that reported, into
1256
+ * the learnings pull request. Its failure is the Run's to report, never to die of: the work is
1257
+ * handed off already, and the learnings are in Initiative memory whatever happens here. */
1258
+ async function submitLearnings(deps: RunDeps, initiativeId: number, dispatches: Dispatch[]): Promise<LearningsResult | undefined> {
1259
+ if (!deps.learnings) return undefined;
1260
+ const learnings = collectProposedLearnings(dispatches);
1261
+ try {
1262
+ return await deps.learnings({ initiativeId, learnings });
1263
+ } catch (error) {
1264
+ const reason = error instanceof Error ? error.message : String(error);
1265
+ deps.log(`learnings: failed: ${reason}`);
1266
+ return { kind: "failed", count: learnings.length, reason };
1267
+ }
1268
+ }
1269
+
1270
+ /**
1271
+ * One Reconcile pass for an Initiative: drain the outbox, then resume, restart or release every
1272
+ * Process Table row. The daemon runs this on its idle ticks too, per Initiative it has rows for.
1273
+ * Without a Process Table there is nothing to reconcile.
1274
+ *
1275
+ * Inside a Run the deps carry its Slots: a resumed or restarted turn is a turn on the Account like
1276
+ * any other, so it takes an implementer Slot and waits for one when the Run has none free. Rows the
1277
+ * Run is driving in its own Slots are left alone. The daemon's idle tick hands in plain `RunDeps`,
1278
+ * where there is no Run to share Slots with and its one Reconcile at a time is the whole of it.
1279
+ */
1280
+ export async function reconcileTick(deps: RunDeps, initiativeId: number): Promise<Array<Dispatch | BlockedTask>> {
1281
+ if (deps.drainOutbox) await deps.drainOutbox(initiativeId);
1282
+ if (!deps.processTable) return [];
1283
+ const scheduling = "slots" in deps ? (deps as Scheduling) : undefined;
1284
+ // A Paused Account starts no new Workers, and a restart is a new Worker: a row with no session to
1285
+ // resume waits, exactly as it waits through a paused Initiative. Resuming is not held back — the
1286
+ // glossary's promise is that a Paused Account's running Workers stay resumable, and a resumed
1287
+ // session is the turn that was already going, not a new one.
1288
+ const current = deps.accounts?.load().find((a) => a.name === deps.account.name) ?? deps.account;
1289
+ const accountPaused = isPaused(current, (deps.now ?? Date.now)());
1290
+ if (accountPaused) deps.log(`reconcile: Account "${deps.account.name}" is Paused until ${current.paused?.until}; its Workers are resumed, nothing is dispatched afresh on it`);
1291
+ /** Runs one Reconcile turn in a Slot of the Run, when there is a Run to take one from. A Run that
1292
+ * is stopping does not wait for a Slot that is in use: the row stays as it is, for the next tick
1293
+ * or the next Foreman, and the loop gets its stop back without a Worker turn's delay. */
1294
+ const inSlot = async (row: WorkerRow, turn: (hold?: SlotHold) => Promise<Dispatch | BlockedTask>): Promise<Dispatch | BlockedTask | undefined> => {
1295
+ if (!scheduling) return turn();
1296
+ const hold = await scheduling.slots.acquireWhenFree(scheduling.untilStop);
1297
+ if (!hold) {
1298
+ deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: the Run is stopping and no Slot came free; leaving the row as it is`);
1299
+ return undefined;
1300
+ }
1301
+ try {
1302
+ return await turn(hold);
1303
+ } finally {
1304
+ hold.release();
1305
+ }
1306
+ };
1307
+ const { results } = await reconcile<Dispatch | BlockedTask>(
1308
+ {
1309
+ table: deps.processTable,
1310
+ tracker: deps.tracker,
1311
+ account: deps.account,
1312
+ log: deps.log,
1313
+ isAlive: deps.isAlive,
1314
+ paused: (id) => accountPaused || (deps.processTable?.isPaused(id) ?? false),
1315
+ owns: scheduling?.owns,
1316
+ resume: (row, sessionId, released) => inSlot(row, (hold) => resumeTurn(deps, initiativeId, row, sessionId, released, hold)),
1317
+ restart: (row) => inSlot(row, (hold) => restartTurn(deps, initiativeId, row, hold)),
1318
+ cancel: (row) => cancelTurn(deps, initiativeId, row),
1319
+ finishSettlement: (row, outcome) => finishSettlement(deps, initiativeId, row, outcome),
1320
+ },
1321
+ initiativeId,
1322
+ );
1323
+ return results;
1324
+ }
1325
+
1326
+ /** Finishes what a crash between the attempt event and the board left undone. The attempt event is
1327
+ * the commit (ADR-0009); memory is agent-written and may already be there, so only the comment and
1328
+ * the board are replayed: a Task whose attempt completed goes to In review under `foreman:done`
1329
+ * when the event names a Hand-off and the attempt's `gate` events say its last round passed (or
1330
+ * there are none, the recipe having named no Gate), else it is blocked, and the comment points at
1331
+ * the events. The full report is not reconstructed: the transcript and the event's summary carry
1332
+ * what a reader needs. */
1333
+ async function finishSettlement(deps: RunDeps, initiativeId: number, row: WorkerRow, outcome: string): Promise<void> {
1334
+ const { tracker, log } = deps;
1335
+ const { task } = await tracker.getTask(row.taskId);
1336
+ const { events } = await tracker.readRunRecord(initiativeId, { taskId: row.taskId, kinds: ["attempt"] });
1337
+ // Never the Triage turn's, which carries this same attempt number and is written before the
1338
+ // dispatch event (core/triage.ts `TRIAGE_MARKER`): settling from it would report the drafting
1339
+ // turn's outcome as the implementer attempt's.
1340
+ const event = events.find(
1341
+ (e) => e.payload?.payload.case === "attempt" && e.payload.payload.value.attempt === row.attempt && !isTriageSummary(e.payload.payload.value.summary),
1342
+ );
1343
+ // The attempt event is the commit (ADR-0009): it carries the Hand-off URL when the draft pull
1344
+ // request was opened, whether or not the Gates then held it. The Gates' own events, written before
1345
+ // the attempt, carry the verdict, so both are read: a URL with a failed last round is a blocked
1346
+ // Task whose pull request is still open, not a handed-off one.
1347
+ const handOffUrl = event?.payload?.payload.case === "attempt" ? event.payload.payload.value.handOffUrl : undefined;
1348
+ const verdict = gateVerdictOf((await tracker.readRunRecord(initiativeId, { taskId: row.taskId, kinds: ["gate"] })).events, row.attempt);
1349
+ // The Verifier's word, written after the Gates': a failed Verdict holds the Hand-off the way a
1350
+ // failed Gate does. The dispatch events are read with it: a Verdict written without an app (the
1351
+ // round failed before the app started) is the attempt's by its place after that attempt's
1352
+ // dispatch event (verifier.ts, the invariant on `verdictEvent`).
1353
+ const verified = verdictStatusOf((await tracker.readRunRecord(initiativeId, { taskId: row.taskId, kinds: ["dispatch", "verdict"] })).events, row.attempt, row.identifier);
1354
+ const handedOff = outcome === "completed" && !!handOffUrl && verdict !== "failed" && verified !== "failed";
1355
+ const state = handedOff
1356
+ ? ` Handed off as ${handOffUrl}${verdict === "passed" ? ", its Gates passed" : ""}${verified === "passed" ? ", its Verdict passed" : ""}.`
1357
+ : handOffUrl
1358
+ ? verdict === "failed"
1359
+ ? ` Hand-off ${handOffUrl} is held by its Gates: the last round failed. The Task stays In Progress under foreman:blocked; see the gate events and the transcript.`
1360
+ : ` Hand-off ${handOffUrl} is held by its Verdict: the Verifier found Defects. The Task stays In Progress under foreman:blocked; see the verdict event and the transcript.`
1361
+ : " The Task stays In Progress under foreman:blocked; see the attempt event and the transcript.";
1362
+ await tracker.addComment(task.id, `Foreman: attempt ${row.attempt} was recorded (${outcome}) but the Foreman stopped before finishing; settled now from the Run Record.${state}`);
1363
+ // A Task that died mid-round still wears it; detach is a no-op when it does not.
1364
+ await tracker.detachLabel(row.taskId, STATUS_LABELS.verifying);
1365
+ if (handedOff) {
1366
+ await tracker.detachLabel(task.id, STATUS_LABELS.running);
1367
+ await tracker.moveTask(task.id, "in_review");
1368
+ await markStatus(tracker, task, undefined, STATUS_LABELS.done);
1369
+ } else {
1370
+ await markStatus(tracker, task, STATUS_LABELS.running, STATUS_LABELS.blocked);
1371
+ }
1372
+ log(`reconcile: ${row.identifier} attempt ${row.attempt}: settlement finished (${handedOff ? "handed off" : handOffUrl ? (verdict === "failed" ? "blocked, Hand-off held by Gates" : "blocked, Hand-off held by the Verdict") : "blocked"})`);
1373
+ }
1374
+
1375
+ /** Continues an interrupted attempt from its stored session id, in the same Workspace, and settles
1376
+ * it the way a fresh turn is settled; the attempt event carries a `resumed` marker. A `released`
1377
+ * row (story 54) is resumed the same way, told that a human had the session in between; its mark
1378
+ * comes off as the turn starts, so the row is an ordinary live Worker's again. */
1379
+ async function resumeTurn(deps: RunDeps, initiativeId: number, row: WorkerRow, sessionId: string, released = false, slot?: SlotHold): Promise<Dispatch> {
1380
+ const { tracker, worker, account, log } = deps;
1381
+ const now = deps.now ?? Date.now;
1382
+ const ctx = await tracker.getTask(row.taskId);
1383
+ const workspace: Workspace = { path: row.workspace, branch: row.branch, created: false };
1384
+ if (released) deps.processTable?.setControl(row.taskId, undefined, now());
1385
+ const started = now();
1386
+ const handle = worker.resume(sessionId, { account, cwd: workspace.path, prompt: resumePrompt(ctx.task, released), maxTurns: deps.maxTurns, stallTimeoutMs: deps.stallTimeoutMs });
1387
+ if (handle.pid !== undefined) deps.processTable?.setPid(row.taskId, handle.pid);
1388
+ log(`worker: ${account.vendor} resumed turn on ${row.identifier} in ${workspace.path} (session ${sessionId}${released ? ", after an attended turn" : ""})`);
1389
+ return observeTurn(deps, initiativeId, ctx, workspace, row.attempt, handle, { started, installNote: "", resumed: true, draftedVerification: draftedVerificationFrom(ctx.comments)?.verification, slot });
1390
+ }
1391
+
1392
+ /** What a resumed Worker is told: the turn it was in ended with the Foreman, not with it; or, after
1393
+ * a release, that a human took the session over for a while and has handed it back. */
1394
+ export function resumePrompt(task: TrackerTask, released = false): string {
1395
+ const first = released
1396
+ ? `A human attached to your session on Task ${task.identifier} ("${task.title}") for an attended turn and has now released it; the Foreman has restarted you unattended in the same Workspace with everything that was said and done.`
1397
+ : `The Foreman that ran your previous turn on Task ${task.identifier} ("${task.title}") stopped before you reported, and has restarted you in the same Workspace with your earlier context.`;
1398
+ return [first, "Check the working tree for what you had already done, continue from there, finish the Task, and end with the Completion Report."].join("\n");
1399
+ }
1400
+
1401
+ /** Closes the attempt of a Worker `foreman cancel` stopped when the process that owned its turn is
1402
+ * not there to do it (it died, or the row was left by a daemon that is gone): the same settlement
1403
+ * the owner would have made, from the row alone. */
1404
+ async function cancelTurn(deps: RunDeps, initiativeId: number, row: WorkerRow): Promise<Dispatch> {
1405
+ const ctx = await deps.tracker.getTask(row.taskId);
1406
+ const workspace: Workspace = { path: row.workspace, branch: row.branch, created: false };
1407
+ return settleCancelled(deps, initiativeId, ctx, workspace, row.attempt, `\n\nThe process that owned the turn did not close the attempt; closed by Reconcile.`, { started: deps.now?.() ?? Date.now(), installNote: "", resumed: true });
1408
+ }
1409
+
1410
+ /** A cancelled attempt (story 56): `stopped` on the Run Record with the cancelled marker first in
1411
+ * the summary, a comment saying the human cancelled it, the Task blocked, the row released. */
1412
+ async function settleCancelled(deps: RunDeps, initiativeId: number, ctx: TaskWithContext, workspace: Workspace, attempt: number, detail: string, obs: TurnObservation): Promise<Dispatch> {
1413
+ const { tracker, log } = deps;
1414
+ const task = ctx.task;
1415
+ const durationMs = (deps.now ?? Date.now)() - obs.started;
1416
+ const marker = obs.resumed ? `${RESUMED_MARKER} ` : "";
1417
+ await settle(
1418
+ tracker,
1419
+ initiativeId,
1420
+ task,
1421
+ {
1422
+ attempt,
1423
+ outcome: "stopped",
1424
+ summary: `${marker}${CANCELLED_MARKER} stopped by opsee foreman cancel${detail}${obs.installNote}\n\nTurn: ${durationMs}ms.`,
1425
+ memory: [],
1426
+ comment: () => `Foreman: attempt ${attempt} was cancelled (opsee foreman cancel ${task.id}); the Worker was stopped and the Task stays In Progress under ${STATUS_LABELS.blocked}. Its Workspace ${workspace.path} (branch ${workspace.branch}) is kept for the next attempt.`,
1427
+ gateRetries: deps.gateRetries ?? DEFAULT_GATE_RETRIES,
1428
+ },
1429
+ log,
1430
+ );
1431
+ deps.processTable?.removeWorker(task.id);
1432
+ log(`attempt: ${task.identifier} cancelled by the human after ${durationMs}ms; the Task is blocked`);
1433
+ return { task, workspace, attempt, outcome: "stopped", durationMs, install: obs.install, resumed: obs.resumed, cancelled: true };
1434
+ }
1435
+
1436
+ /** A Worker that died before it had a session cannot be resumed: the Task is dispatched again,
1437
+ * which the Run Record counts as the next attempt. */
1438
+ async function restartTurn(deps: RunDeps, initiativeId: number, row: WorkerRow, slot?: SlotHold): Promise<Dispatch | BlockedTask> {
1439
+ deps.processTable?.removeWorker(row.taskId);
1440
+ const { task } = await deps.tracker.getTask(row.taskId);
1441
+ return dispatch(deps, initiativeId, task, slot);
1442
+ }
1443
+
1444
+ /** The Task outside any Initiative is refused (story 15); so is one in a different Initiative. */
1445
+ export function assertInInitiative(task: TrackerTask, initiativeId: number): void {
1446
+ if (task.initiativeId === undefined || task.initiativeId === 0) {
1447
+ throw new ForemanError(
1448
+ `Task ${task.identifier} (id ${task.id}) is not in any Initiative. The Foreman only runs Initiative Tasks, so every Completion Report has a memory log to land in: link it to Initiative ${initiativeId} (or another) and label it "${DISPATCH_LABEL}".`,
1449
+ );
1450
+ }
1451
+ if (task.initiativeId !== initiativeId) {
1452
+ throw new ForemanError(`Task ${task.identifier} (id ${task.id}) is in Initiative ${task.initiativeId}, not Initiative ${initiativeId}.`);
1453
+ }
1454
+ }
1455
+
1456
+ /**
1457
+ * The Ready Tasks this tick may fill Slots with, in dispatch order: priority descending, then
1458
+ * oldest first (core/scheduler.ts, story 28). Readiness is the Tracker's (ADR-0008); what is
1459
+ * dropped here is only what this Run must not pick again — a Task it already has in flight, and one
1460
+ * it deferred. `--task` names one Task instead and is refused when that Task is not Ready.
1461
+ */
1462
+ async function pickTasks(
1463
+ tracker: TrackerAdapter,
1464
+ options: RunOptions,
1465
+ deferred: ReadonlySet<number>,
1466
+ inFlight: ReadonlyMap<number, unknown>,
1467
+ ): Promise<TrackerTask[]> {
1468
+ const ready = await tracker.listReadyTasks(options.initiativeId);
1469
+ if (!ready.dispatchLabelExists) {
1470
+ throw new ForemanError(`Initiative ${options.initiativeId}: the project has no "${ready.dispatchLabel}" label, so no Task can be ready.`);
1471
+ }
1472
+ if (options.taskId === undefined) {
1473
+ return orderReadyTasks(ready.tasks.filter((t) => !deferred.has(t.id) && !inFlight.has(t.id)));
1474
+ }
1475
+
1476
+ const { task } = await tracker.getTask(options.taskId);
1477
+ assertInInitiative(task, options.initiativeId);
1478
+ const hit = ready.tasks.find((t) => t.id === options.taskId);
1479
+ if (!hit) {
1480
+ throw new ForemanError(
1481
+ `Task ${task.identifier} (id ${task.id}) is not Ready in Initiative ${options.initiativeId}: it needs the "${ready.dispatchLabel}" label, an open column, and every blocker Done.`,
1482
+ );
1483
+ }
1484
+ return [hit];
1485
+ }
1486
+
1487
+ function runEvent(taskId: number | undefined, payload: MessageInitShape<typeof RunEventInputSchema>["payload"]): RunEventInput {
1488
+ return create(RunEventInputSchema, { taskId, payload });
1489
+ }
1490
+
1491
+ /** `runEvent` for a fact that happened at a moment other than now: the server stamps `occurredAt`
1492
+ * itself when it is absent, which is right for everything the Run does as it does it, and wrong for
1493
+ * a lift a human made before this Run started (`reportResumedAccounts`). */
1494
+ function runEventAt(taskId: number | undefined, occurredAt: Date | undefined, payload: MessageInitShape<typeof RunEventInputSchema>["payload"]): RunEventInput {
1495
+ return create(RunEventInputSchema, { taskId, payload, ...(occurredAt ? { occurredAt: timestampFromMs(occurredAt.getTime()) } : {}) });
1496
+ }
1497
+
1498
+ /** A `spend` event: what one turn cost, on which Account. Written for any turn that reported a
1499
+ * cost whatever its outcome, because a turn that failed still spent; only a `completed` turn
1500
+ * carries one, since that is the only terminal event the vendors put a total on.
1501
+ *
1502
+ * Credits are the Run Record's unit -- a thousandth of a dollar (`InitiativeCostRollup`: 1 credit
1503
+ * = $0.001) -- rounded to the nearest whole one. A cost that rounds to nothing is left unwritten
1504
+ * rather than recorded as free: an event saying a turn cost zero is worse than no event, because
1505
+ * the rollup would sum it as fact.
1506
+ *
1507
+ * `model`, `inputTokens` and `outputTokens` stay unset. The streams this Foreman reads report a
1508
+ * turn's total and not its token split, and a zero in those fields would read as "no tokens" where
1509
+ * the truth is "not reported". */
1510
+ export function spendEvent(taskId: number, vendor: Vendor, accountName: string, costUsd: number | undefined): RunEventInput | undefined {
1511
+ if (typeof costUsd !== "number" || !Number.isFinite(costUsd) || costUsd <= 0) return undefined;
1512
+ const credits = Math.round(costUsd * 1000);
1513
+ if (credits <= 0) return undefined;
1514
+ return runEvent(taskId, { payload: { case: "spend", value: { credits: BigInt(credits), vendor: RUN_RECORD_VENDOR[vendor], account: accountName } } });
1515
+ }
1516
+
1517
+ /** A `control` event (stories 53-56): what the human did, from the CLI. Task-scoped for attach,
1518
+ * release and cancel; Run-wide for pause and resume. */
1519
+ /** The state an Account moved to, as the Run Record names it (`RunAccountEvent.state`). */
1520
+ export type AccountState = "paused" | "quarantined" | "active";
1521
+
1522
+ /**
1523
+ * An Account's state on the Run Record: the vendor refused a turn on it, its credential was judged
1524
+ * dead, or a human brought it back.
1525
+ *
1526
+ * `state` is where the Account arrived rather than what happened to it, so one vocabulary covers a
1527
+ * lift from either a pause or a quarantine. What identifies the Account is the label its operator
1528
+ * gave it and nothing else — never the config directory or key variable behind it (ADR-0013), which
1529
+ * is the same line `RunDispatchEvent.account` holds.
1530
+ *
1531
+ * `taskId` is the turn that discovered it, for a pause or a quarantine; an Account brought back was
1532
+ * caused by no Task and carries none.
1533
+ */
1534
+ export function accountEvent(state: AccountState, vendor: Vendor, accountName: string, reason: string, options: { taskId?: number; until?: Date; occurredAt?: Date } = {}): RunEventInput {
1535
+ return runEventAt(options.taskId, options.occurredAt, {
1536
+ payload: {
1537
+ case: "account",
1538
+ value: {
1539
+ state,
1540
+ vendor: RUN_RECORD_VENDOR[vendor],
1541
+ account: accountName,
1542
+ // The vendor's own prose in places, on its way to a record other people read: one printable
1543
+ // line, as it is for the accounts file and the terminal views.
1544
+ reason: printableOneLine(reason, ACCOUNT_REASON_MAX),
1545
+ ...(options.until ? { until: timestampFromMs(options.until.getTime()) } : {}),
1546
+ },
1547
+ },
1548
+ });
1549
+ }
1550
+
1551
+ /** What `RunAccountEvent.reason` takes (proto: `max_len: 4000`). Well under it: the reason is one
1552
+ * line of a vendor's refusal, and the Run Record is not where a stack trace belongs. */
1553
+ const ACCOUNT_REASON_MAX = 500;
1554
+
1555
+ export function controlEvent(action: ControlAction, message: string, taskId?: number): RunEventInput {
1556
+ return runEvent(taskId, { payload: { case: "control", value: { action, message } } });
1557
+ }
1558
+
1559
+ function attemptOutcomeOf(reason: TurnFailureReason): AttemptOutcome {
1560
+ switch (reason) {
1561
+ case "stalled":
1562
+ return "stalled";
1563
+ case "rate_limited":
1564
+ return "rate_limited";
1565
+ case "stopped":
1566
+ return "stopped";
1567
+ // A credential failure has no outcome of its own: `RunAttemptEvent.outcome` is a closed set in
1568
+ // the proto and widening it is a change to four generated trees for a fact the summary already
1569
+ // carries in words. Written as its own case rather than left to the default, so that reading
1570
+ // this function answers the question instead of raising it.
1571
+ case "credential_failed":
1572
+ return "failed";
1573
+ default:
1574
+ return "failed";
1575
+ }
1576
+ }
1577
+
1578
+ /**
1579
+ * How many attempts on this Task have already ended in a `RETRYABLE_FAILURES` outcome, read back
1580
+ * off the Run Record: what the retry cap is spent on (story 34).
1581
+ *
1582
+ * Deliberately not `attempt - 1`. The attempt number counts every turn the Task ever had, whatever
1583
+ * ended it — a failed Gate round, a Verdict with Defects, a human relabelling a blocked Task — and
1584
+ * none of those is a machine-side failure the cap exists to bound. A Triage turn that ended
1585
+ * `rate_limited` does count: it is a turn on the Account that the vendor refused, exactly the thing
1586
+ * another attempt is worth trying.
1587
+ */
1588
+ async function retryableFailuresOf(tracker: TrackerAdapter, initiativeId: number, taskId: number): Promise<number> {
1589
+ const { events } = await tracker.readRunRecord(initiativeId, { taskId, kinds: ["attempt"] });
1590
+ return events.filter((e) => {
1591
+ const payload = e.payload?.payload;
1592
+ if (payload?.case !== "attempt") return false;
1593
+ // The outcome answers for `stalled` and `rate_limited`; a credential failure was recorded as
1594
+ // the generic `failed` and is recognised by its marker instead (`RETRYABLE_FAILURE_REASONS`).
1595
+ return RETRYABLE_FAILURES.has(payload.value.outcome as AttemptOutcome) || isCredentialSummary(payload.value.summary);
1596
+ }).length;
1597
+ }
1598
+
1599
+ /** Whether an attempt event's summary is a turn that failed for a credential reason. The marker is
1600
+ * written after the resumed and triage markers, so this is a search rather than a prefix test. */
1601
+ export function isCredentialSummary(summary: string): boolean {
1602
+ return summary.includes(CREDENTIAL_MARKER);
1603
+ }
1604
+
1605
+ /** Swaps the Task's Status Label: `from` comes off, `to` goes on (created in the project first). */
1606
+ async function markStatus(tracker: TrackerAdapter, task: TrackerTask, from: StatusLabel | undefined, to: StatusLabel): Promise<void> {
1607
+ if (from) await tracker.detachLabel(task.id, from);
1608
+ await tracker.ensureLabel(task.projectId, to);
1609
+ await tracker.attachLabel(task.id, to);
1610
+ }
1611
+
1612
+ type HandedOff = Extract<HandOffResult, { kind: "handed_off" }>;
1613
+
1614
+ interface Settlement {
1615
+ attempt: number;
1616
+ outcome: AttemptOutcome;
1617
+ /** The attempt event's summary: what happened, the install exit code, the output tail. */
1618
+ summary: string;
1619
+ /** The linking comment, given the ids of the memory entries just written (outcome first) and
1620
+ * the Run loop's note about the Hand-off. */
1621
+ comment: (memoryIds: number[], note: string) => string;
1622
+ /** Memory entries to write before the comment; empty for a turn without a report. */
1623
+ memory: Parameters<TrackerAdapter["addMemory"]>[1][];
1624
+ /** The Hand-off of a `done` turn. Open: the Task goes to In review under `foreman:done` with the
1625
+ * pull request linked, provided the Gates passed. Incomplete or absent: it stays In Progress
1626
+ * under `foreman:blocked`. */
1627
+ handOff?: HandOffResult;
1628
+ /** The Gates that ran on the Hand-off. Failed: the pull request is still linked and named in the
1629
+ * attempt event, but the Task stays In Progress under `foreman:blocked`. */
1630
+ gates?: GateOutcome;
1631
+ /** The Verifier's round after passing Gates. Defects or a failed round: the pull request stays
1632
+ * linked, the Task stays In Progress under `foreman:blocked`. */
1633
+ verification?: VerifyOutcome;
1634
+ gateRetries: number;
1635
+ /** Set when the attempt failed for a reason worth another go under the retry cap
1636
+ * (`RETRYABLE_FAILURES`): the Task gets its dispatch label back instead of `foreman:blocked`, so
1637
+ * the Tracker counts it Ready again and a later tick dispatches the next attempt — on a sibling
1638
+ * Account when its own is Paused by then (story 31), and no sooner than `waitMs` from now
1639
+ * (`retryBackoff`). */
1640
+ retry?: { attemptsLeft: number; waitMs: number };
1641
+ /** Set when that same kind of failure has used the retry cap up (story 34): the Task is blocked
1642
+ * like any other failed attempt, and the comment says which cause spent the attempts, so a human
1643
+ * reading the Task alone knows whether to retry it or to fix something first. `attempts` is how
1644
+ * many attempts ended in a retryable failure (`retryableFailuresOf`, this one included), which is
1645
+ * what the comment claims — never the Task's total dispatch count. */
1646
+ capped?: { cause: TurnFailureReason; attempts: number; retries: number };
1647
+ }
1648
+
1649
+ /** Records how a turn ended: the attempt event first (ADR-0009, carrying the Hand-off URL when
1650
+ * there is one), then the pull request link, memory, the comment, and the board with its Status
1651
+ * Label. Returns the ids of the memory entries written. */
1652
+ async function settle(tracker: TrackerAdapter, initiativeId: number, task: TrackerTask, s: Settlement, log: (line: string) => void): Promise<number[]> {
1653
+ const opened = s.handOff?.kind === "handed_off" ? s.handOff : undefined;
1654
+ const gatesPassed = s.gates?.passed ?? true;
1655
+ const handedOff = gatesPassed && verificationPassed(s.verification) ? opened : undefined;
1656
+ await tracker.appendRunEvents(initiativeId, [
1657
+ runEvent(task.id, { payload: { case: "attempt", value: { attempt: s.attempt, outcome: s.outcome, summary: attemptSummary(s.summary), handOffUrl: opened?.pullRequest.url } } }),
1658
+ ]);
1659
+ let note = "";
1660
+ if (opened) note = await linkHandOff(tracker, task, opened, log);
1661
+ else if (s.handOff?.kind === "incomplete") note = ` Hand-off incomplete: ${s.handOff.reason}. The Task stays In Progress.`;
1662
+ if (s.gates) note += gateNote(s.gates, s.gateRetries);
1663
+ if (s.verification) note += verifyNote(s.verification);
1664
+ const retry = handedOff ? undefined : s.retry;
1665
+ if (retry) note += ` The Task is Ready again and the Foreman dispatches it as attempt ${s.attempt + 1}${inSeconds(retry.waitMs)}: ${count(retry.attemptsLeft, "attempt")} left under the retry cap.`;
1666
+ const capped = handedOff ? undefined : s.capped;
1667
+ if (capped) {
1668
+ note +=
1669
+ ` Every attempt this Task gets is spent: ${count(capped.attempts, "attempt")} ended ${capped.cause}, which is the retry cap (${count(capped.retries, "further attempt")}) reached.` +
1670
+ ` It stays In Progress under ${STATUS_LABELS.blocked} for a human, and the Foreman goes on with the Initiative's other Tasks.`;
1671
+ }
1672
+ const ids: number[] = [];
1673
+ for (const entry of s.memory) ids.push((await tracker.addMemory(initiativeId, entry)).id);
1674
+ await tracker.addComment(task.id, s.comment(ids, note));
1675
+ await tracker.detachLabel(task.id, STATUS_LABELS.verifying);
1676
+ if (handedOff) {
1677
+ await tracker.detachLabel(task.id, STATUS_LABELS.running);
1678
+ await tracker.moveTask(task.id, "in_review");
1679
+ await markStatus(tracker, task, undefined, STATUS_LABELS.done);
1680
+ } else if (retry) {
1681
+ // The dispatch label back on is what makes the Task Ready again; the column is already an open
1682
+ // one (In Progress), so the Tracker's own readiness rule lets the next tick pick it up.
1683
+ await tracker.detachLabel(task.id, STATUS_LABELS.running);
1684
+ await tracker.attachLabel(task.id, DISPATCH_LABEL);
1685
+ } else {
1686
+ await markStatus(tracker, task, STATUS_LABELS.running, STATUS_LABELS.blocked);
1687
+ }
1688
+ return ids;
1689
+ }
1690
+
1691
+ /** Links the pull request to the Task through the project's connected repository that matches
1692
+ * the origin, so it rolls up into the Initiative. With no match the link is not guessed: the
1693
+ * comment says which repository to connect. Returns the note for the comment. */
1694
+ async function linkHandOff(tracker: TrackerAdapter, task: TrackerTask, handOff: HandedOff, log: (line: string) => void): Promise<string> {
1695
+ const repositories = await tracker.projectRepositories(task.projectId);
1696
+ const wanted = handOff.repositoryFullName?.toLowerCase();
1697
+ const repository =
1698
+ repositories.find((r) => wanted !== undefined && r.fullName.toLowerCase() === wanted) ??
1699
+ (wanted === undefined && repositories.length === 1 ? repositories[0] : undefined);
1700
+ if (!repository) {
1701
+ const connected = repositories.map((r) => r.fullName).join(", ") || "none";
1702
+ log(`hand-off: ${task.identifier} not linked: no connected repository matches origin ${wanted ?? "(unknown)"} (connected: ${connected})`);
1703
+ return ` Not linked to a repository: the project has no connected repository "${wanted ?? "?"}" (connected: ${connected}); connect it so the pull request rolls up into the Initiative.`;
1704
+ }
1705
+ const { pullRequest } = handOff;
1706
+ await tracker.linkPullRequest(task.id, {
1707
+ projectRepositoryId: repository.id,
1708
+ number: pullRequest.number,
1709
+ url: pullRequest.url,
1710
+ title: pullRequest.title,
1711
+ headBranch: pullRequest.headBranch,
1712
+ headSha: handOff.headSha,
1713
+ });
1714
+ log(`hand-off: ${task.identifier} linked ${pullRequest.url} to repository ${repository.fullName}`);
1715
+ return "";
1716
+ }
1717
+
1718
+ /** What the Task comment says about the Gates: short when they passed, and naming the Gate,
1719
+ * its command and exit code when they did not (story 34). */
1720
+ export function gateNote(gates: GateOutcome, retries: number): string {
1721
+ // Said as plainly as a skipped Verifier says it (`verifyNote`): a reader of the Task alone must
1722
+ // never be able to conclude the work was checked when nothing checked it.
1723
+ if (gates.reason === "skipped") {
1724
+ return ` Gates ${gateSummary(gates, retries)}. Nothing on this machine ran the repository's tests, lint or typecheck against this Hand-off; read it as unchecked, not as passing.`;
1725
+ }
1726
+ if (gates.passed) {
1727
+ return ` Gates passed in ${count(gates.rounds.length, "round")}${gates.resumes ? ` after ${count(gates.resumes, "resumed turn")}` : ""}.`;
1728
+ }
1729
+ return ` Gates ${gateSummary(gates, retries)}. The Task stays In Progress under ${STATUS_LABELS.blocked}.`;
1730
+ }
1731
+
1732
+ /** What the Task comment says about the Verifier: passed, skipped and why, or the Defects. */
1733
+ export function verifyNote(outcome: VerifyOutcome): string {
1734
+ if (verificationPassed(outcome)) return ` Verification ${verifySummary(outcome)}.`;
1735
+ return ` Verification ${verifySummary(outcome)}. The Task stays In Progress under ${STATUS_LABELS.blocked}.`;
1736
+ }
1737
+
1738
+ /** Refuses a Ready Task before its dispatch event: the dispatch label comes off first so the Task
1739
+ * is not picked again next tick, then the comment says why, then the blocked Status Label. No Run
1740
+ * Record event: nothing was dispatched, and a Triage turn that ran has recorded itself already. */
1741
+ async function refuse(
1742
+ deps: RunDeps,
1743
+ task: TrackerTask,
1744
+ missing: WorkContractSection[],
1745
+ comment: string,
1746
+ why: string,
1747
+ after?: { triage: TriageTurn; workspace: Workspace },
1748
+ ): Promise<BlockedTask> {
1749
+ const { tracker, log } = deps;
1750
+ log(`contract: ${task.identifier} blocked, ${why}; "${DISPATCH_LABEL}" removed${after ? `, Workspace ${after.workspace.path} kept` : ""}`);
1751
+ await tracker.detachLabel(task.id, DISPATCH_LABEL);
1752
+ await tracker.addComment(task.id, comment);
1753
+ await markStatus(tracker, task, undefined, STATUS_LABELS.blocked);
1754
+ return { task, missing, ...after };
1755
+ }
1756
+
1757
+ /** Runs the Triage turn and records it: the attempt event first (ADR-0009), then the draft as a
1758
+ * comment, which is the record later slices read (`draftedVerificationFrom`). */
1759
+ async function triageTurn(deps: RunDeps, initiativeId: number, task: TrackerTask, contract: WorkContract, workspace: Workspace, attempt: number, now: () => number): Promise<TriageTurn> {
1760
+ const { tracker, worker, account, log } = deps;
1761
+ const { terminal, durationMs, outputTail, rateLimit } = await driveTriageTurn({
1762
+ worker,
1763
+ account,
1764
+ task,
1765
+ contract,
1766
+ workspace,
1767
+ maxTurns: deps.maxTurns,
1768
+ stallTimeoutMs: deps.stallTimeoutMs,
1769
+ log,
1770
+ now,
1771
+ });
1772
+ const trailer = `\n\nTurn: ${durationMs}ms.${outputTail ? `\n\nOutput tail:\n${outputTail}` : ""}`;
1773
+ // A Triage turn is a turn on the Account, so a limit it hit Pauses the Account exactly as a
1774
+ // Worker turn's would (story 30); the Task itself keeps its dispatch label and is retried.
1775
+ if (terminal.type === "failed" && terminal.reason === "rate_limited") await pauseOnRateLimit(deps, initiativeId, task.id, rateLimit, terminal.message, now());
1776
+ // And a credential it refused counts against the Account exactly as a Worker turn's would
1777
+ // (OPS-288): a Triage turn is a turn on the Account, and a dead login refuses it identically.
1778
+ if (terminal.type === "failed" && terminal.reason === "credential_failed") await noteCredentialFailureOnAccount(deps, initiativeId, task.id, terminal.message, now());
1779
+ // A Triage turn the vendor accepted is proof the credential is live, so it closes an open streak
1780
+ // the same way an implementer turn does.
1781
+ if (terminal.type === "completed") clearCredentialStreak(deps);
1782
+
1783
+ let result: TriageTurn;
1784
+ let summary: string;
1785
+ if (terminal.type === "completed") {
1786
+ const parsed = triageDraftFrom(terminal.report);
1787
+ if (parsed.ok) {
1788
+ result = { attempt, outcome: "completed", verification: parsed.draft.verification, sessionId: terminal.sessionId, durationMs };
1789
+ summary = `triage: drafted Verification section (${parsed.draft.verification.length} chars)${trailer}`;
1790
+ } else {
1791
+ result = { attempt, outcome: "completed", reason: parsed.reason, sessionId: terminal.sessionId, durationMs };
1792
+ summary = `triage: no draft: ${parsed.reason}${trailer}`;
1793
+ }
1794
+ } else {
1795
+ const reason = `${terminal.reason}: ${terminal.message}`;
1796
+ const details = terminal.details?.length ? `\n${terminal.details.join("\n")}` : "";
1797
+ result = { attempt, outcome: attemptOutcomeOf(terminal.reason), reason, sessionId: terminal.sessionId, durationMs };
1798
+ summary = `triage: ${reason}${details}${trailer}`;
1799
+ }
1800
+
1801
+ // The attempt event has no account field, so the summary names it: a Triage turn is a turn on
1802
+ // the Account, and a reader of the Run Record must be able to charge it to one. The marker comes
1803
+ // first and is machine-read: this event shares its attempt number with the implementer turn that
1804
+ // follows it, and the two settlement paths tell them apart by it (core/triage.ts `TRIAGE_MARKER`).
1805
+ summary = `${TRIAGE_MARKER} ${summary.replace(/^triage:/, `triage on ${RUN_RECORD_VENDOR[account.vendor]} Account "${account.name}":`)}`;
1806
+ const turnSpend = spendEvent(task.id, account.vendor, account.name, terminal.type === "completed" ? terminal.costUsd : undefined);
1807
+ await tracker.appendRunEvents(initiativeId, [
1808
+ ...(turnSpend ? [turnSpend] : []),
1809
+ runEvent(task.id, { payload: { case: "attempt", value: { attempt, outcome: result.outcome, summary: attemptSummary(summary) } } }),
1810
+ ]);
1811
+ if (result.verification !== undefined) {
1812
+ await tracker.addComment(task.id, triageComment(result.verification));
1813
+ log(`triage: ${task.identifier} drafted Verification section (${result.verification.length} chars) after ${durationMs}ms`);
1814
+ } else {
1815
+ log(`triage: ${task.identifier} ${result.outcome} after ${durationMs}ms: ${result.reason}`);
1816
+ }
1817
+ return result;
1818
+ }
1819
+
1820
+ async function dispatch(deps: RunDeps, initiativeId: number, picked: TrackerTask, slot?: SlotHold): Promise<Dispatch | BlockedTask> {
1821
+ const { tracker, worker, account, log } = deps;
1822
+ const now = deps.now ?? Date.now;
1823
+ const ctx = await tracker.getTask(picked.id);
1824
+ const task = ctx.task;
1825
+ assertInInitiative(task, initiativeId);
1826
+
1827
+ // The work contract, before anything is written: a Task that cannot be worked is refused here.
1828
+ const contract = parseWorkContract(task.description);
1829
+ const verdict = validateWorkContract(contract);
1830
+ if (verdict.status === "blocked") {
1831
+ return refuse(deps, task, verdict.missing, blockedComment(verdict.missing), `the work contract is missing: ${verdict.missing.join(", ")}`);
1832
+ }
1833
+
1834
+ const priorDispatches = (await tracker.readRunRecord(initiativeId, { taskId: task.id, kinds: ["dispatch"] })).events.length;
1835
+ const attempt = priorDispatches + 1;
1836
+ const workspace = await deps.workspaces.create(task.identifier);
1837
+
1838
+ let triage: TriageTurn | undefined;
1839
+ let draftedVerification: string | undefined;
1840
+ if (verdict.status === "ok") {
1841
+ log(`contract: ${task.identifier} complete`);
1842
+ } else {
1843
+ const earlier = draftedVerificationFrom(ctx.comments);
1844
+ if (earlier) {
1845
+ log(`contract: ${task.identifier} has no Verification section; using the one drafted in comment #${earlier.commentId}`);
1846
+ draftedVerification = earlier.verification;
1847
+ } else {
1848
+ log(`contract: ${task.identifier} has no Verification section; Triage turn`);
1849
+ triage = await triageTurn(deps, initiativeId, task, contract, workspace, attempt, now);
1850
+ if (triage.verification === undefined) {
1851
+ // Only the Worker saying the contract is too thin to draft is a contract defect. A limit,
1852
+ // a stall, or an adapter failure is the Account's or the machine's, so the Task stays
1853
+ // Ready and the next tick retries, as a Worker turn in the same state would.
1854
+ if (triage.outcome === "completed") {
1855
+ return refuse(deps, task, verdict.missing, triageFailedComment(triage.reason ?? "no draft"), `the Triage turn drafted nothing (${triage.reason})`, { triage, workspace });
1856
+ }
1857
+ // A stall or a rate limit is worth another go within this Run; anything else that ended the
1858
+ // turn without a report (an adapter that would not start, a vendor error, a stop) is not:
1859
+ // no tick of this Run can change it, and looking again in a second would only spin. Only
1860
+ // the retryable outcomes write an attempt event `retryableFailuresOf` counts, so only they
1861
+ // have a cap to be bounded by.
1862
+ if (!RETRYABLE_FAILURES.has(triage.outcome)) {
1863
+ log(`contract: ${task.identifier} Triage turn ${triage.outcome} (${triage.reason}); "${DISPATCH_LABEL}" kept, left to the next Run`);
1864
+ await tracker.addComment(task.id, `Foreman: the Triage turn ended ${triage.outcome} (${triage.reason}); the Task stays Ready and the next Run retries it.`);
1865
+ return { task, missing: verdict.missing, triage, workspace, defer: true };
1866
+ }
1867
+ // Bounded by the same retry cap a failed dispatch is bounded by, and by nothing else: the
1868
+ // Triage turn's own attempt event is one of the retryable failures `retryableFailuresOf`
1869
+ // counts (deliberately, see its comment), so this cannot spin — but it is retried on a
1870
+ // later tick of this Run rather than deferred to the next one.
1871
+ const attemptRetries = deps.attemptRetries ?? DEFAULT_ATTEMPT_RETRIES;
1872
+ const spent = await retryableFailuresOf(tracker, initiativeId, task.id);
1873
+ const attemptsLeft = attemptRetries - (spent - 1);
1874
+ if (attemptsLeft <= 0) {
1875
+ return refuse(
1876
+ deps,
1877
+ task,
1878
+ verdict.missing,
1879
+ triageCappedComment(triage.outcome, triage.reason ?? "no reason given", spent, attemptRetries),
1880
+ `every Triage turn it is allowed ended ${triage.outcome} (${count(spent, "attempt")}, the retry cap)`,
1881
+ { triage, workspace },
1882
+ );
1883
+ }
1884
+ const waitMs = retryBackoff(spent, deps.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS);
1885
+ log(`contract: ${task.identifier} Triage turn ${triage.outcome} (${triage.reason}); "${DISPATCH_LABEL}" kept, retried on a later tick${inSeconds(waitMs)} (${count(attemptsLeft, "attempt")} left under the retry cap)`);
1886
+ await tracker.addComment(
1887
+ task.id,
1888
+ `Foreman: the Triage turn ended ${triage.outcome} (${triage.reason}); the Task stays Ready and is retried${inSeconds(waitMs)}: ${count(attemptsLeft, "attempt")} left under the retry cap.`,
1889
+ );
1890
+ return { task, missing: verdict.missing, triage, workspace, retry: { attemptsLeft, notBefore: (deps.now ?? Date.now)() + waitMs } };
1891
+ }
1892
+ draftedVerification = triage.verification;
1893
+ }
1894
+ }
1895
+
1896
+ const origin = workspace.created ? `new Workspace${workspace.base ? ` from ${workspace.base}` : ""}` : "existing Workspace";
1897
+ log(`dispatch: ${task.identifier} "${task.title}" attempt ${attempt} -> ${workspace.path} (branch ${workspace.branch}, ${origin})`);
1898
+
1899
+ // Run Record first (ADR-0009), then the board. The dispatch label comes off at dispatch so the
1900
+ // Task is never picked twice; the Status Label says who has it.
1901
+ await tracker.appendRunEvents(initiativeId, [
1902
+ runEvent(task.id, {
1903
+ payload: {
1904
+ case: "dispatch",
1905
+ value: { vendor: RUN_RECORD_VENDOR[account.vendor], account: account.name, workspace: workspace.path, branch: workspace.branch, attempt },
1906
+ },
1907
+ }),
1908
+ ]);
1909
+ // The row exists from the moment the dispatch is committed (story 50): a crash anywhere between
1910
+ // here and the vendor's `started` leaves a row with no session, which Reconcile restarts.
1911
+ deps.processTable?.upsertWorker({
1912
+ taskId: task.id,
1913
+ initiativeId,
1914
+ identifier: task.identifier,
1915
+ account: account.name,
1916
+ vendor: account.vendor,
1917
+ workspace: workspace.path,
1918
+ branch: workspace.branch,
1919
+ startedAt: now(),
1920
+ lastOutputAt: now(),
1921
+ attempt,
1922
+ });
1923
+ // From here the Task is being taken. Whatever goes wrong below that is not a typed turn outcome
1924
+ // (a Tracker write failing, a board without the column, an adapter rejecting) must not leave the
1925
+ // Task half-claimed with no trace: it is marked blocked with the reason, and the error still
1926
+ // propagates. The abort writes are best-effort so they cannot mask the original error.
1927
+ let running = false;
1928
+ try {
1929
+ await tracker.detachLabel(task.id, DISPATCH_LABEL);
1930
+ await tracker.moveTask(task.id, "in_progress");
1931
+ await markStatus(tracker, task, undefined, STATUS_LABELS.running);
1932
+ running = true;
1933
+ const done = await work(deps, initiativeId, ctx, workspace, attempt, now, draftedVerification, slot);
1934
+ return { ...done, triage, draftedVerification };
1935
+ } catch (error) {
1936
+ const message = error instanceof Error ? error.message : String(error);
1937
+ log(`dispatch: ${task.identifier} aborted: ${message}`);
1938
+ // No Worker is running for the Task any more, so the row would only ever be released.
1939
+ deps.processTable?.removeWorker(task.id);
1940
+ try {
1941
+ await tracker.addComment(task.id, `Foreman: dispatch aborted: ${message}`);
1942
+ await markStatus(tracker, task, running ? STATUS_LABELS.running : undefined, STATUS_LABELS.blocked);
1943
+ } catch (abortError) {
1944
+ log(`dispatch: ${task.identifier} could not record the abort: ${abortError instanceof Error ? abortError.message : String(abortError)}`);
1945
+ }
1946
+ throw error;
1947
+ }
1948
+ }
1949
+
1950
+ async function work(
1951
+ deps: RunDeps,
1952
+ initiativeId: number,
1953
+ ctx: TaskWithContext,
1954
+ workspace: Workspace,
1955
+ attempt: number,
1956
+ now: () => number,
1957
+ draftedVerification?: string,
1958
+ slot?: SlotHold,
1959
+ ): Promise<Dispatch> {
1960
+ const { tracker, worker, account, log } = deps;
1961
+ const task = ctx.task;
1962
+
1963
+ let install: InstallResult | undefined;
1964
+ if (!workspace.created) {
1965
+ log("install: skipped, the Workspace already existed");
1966
+ } else {
1967
+ try {
1968
+ install = await (deps.install ?? defaultInstall)(workspace.path);
1969
+ } catch (error) {
1970
+ install = { command: "(install)", exitCode: null, durationMs: 0, outputTail: error instanceof Error ? error.message : String(error) };
1971
+ }
1972
+ if (install) log(`install: ${install.command} exited ${install.exitCode ?? "without a code"} after ${install.durationMs}ms`);
1973
+ else log("install: no install command in the Run Recipe and no lockfile at the Workspace root; nothing to install");
1974
+ }
1975
+ const installNote = install ? `\n\nInstall: ${install.command} exited ${install.exitCode ?? "without a code"}.` : "";
1976
+
1977
+ const prompt = await assemble(deps, initiativeId, ctx, workspace, log, draftedVerification);
1978
+
1979
+ const started = now();
1980
+ const handle = worker.launch({ account, cwd: workspace.path, prompt, maxTurns: deps.maxTurns, stallTimeoutMs: deps.stallTimeoutMs });
1981
+ // The pid goes to the row as soon as there is one (story 50): a crash before the vendor's first
1982
+ // line then leaves a row Reconcile can see is still running, rather than one it would restart
1983
+ // beside the live Worker.
1984
+ if (handle.pid !== undefined) deps.processTable?.setPid(task.id, handle.pid);
1985
+ log(`worker: ${account.vendor} turn on ${task.identifier} in ${workspace.path}`);
1986
+ return observeTurn(deps, initiativeId, ctx, workspace, attempt, handle, { started, installNote, install, draftedVerification, slot });
1987
+ }
1988
+
1989
+ interface TurnObservation {
1990
+ started: number;
1991
+ installNote: string;
1992
+ install?: InstallResult;
1993
+ resumed?: boolean;
1994
+ /** The Verification section the Worker was given because the Task has none; what the Verifier
1995
+ * follows in that case. */
1996
+ draftedVerification?: string;
1997
+ /** The Slot this turn is running in, when a Run gave it one: swapped for the Account's reserved
1998
+ * Verifier Slot before the Verifier round (core/scheduler.ts). Unset for a turn outside a Run's
1999
+ * Slots (the daemon's idle Reconcile), which is one turn at a time anyway. */
2000
+ slot?: SlotHold;
2001
+ }
2002
+
2003
+ /** The Verification section the Verifier follows: the Task's own, else the drafted one. */
2004
+ function verificationSectionOf(task: TrackerTask, drafted: string | undefined): string | undefined {
2005
+ return parseWorkContract(task.description).sections.Verification ?? drafted;
2006
+ }
2007
+
2008
+ type TerminalEvent = Extract<AdapterEvent, { type: "completed" | "failed" }>;
2009
+ type CompletedEvent = Extract<AdapterEvent, { type: "completed" }>;
2010
+
2011
+ /** Where one attempt's turns write: the local transcript and the output lines the tail is cut
2012
+ * from. A Gate's resumed turns append to the same. */
2013
+ interface TurnSink {
2014
+ transcript?: TranscriptWriter;
2015
+ output: string[];
2016
+ /** The vendor's last rate-limit report of this turn (story 30). The terminal `failed` event says
2017
+ * only that the turn ended `rate_limited`; the reset the vendor named came on the event before
2018
+ * it, and that is what decides how long the Account is Paused. */
2019
+ rateLimit?: { resetAt?: string; message: string };
2020
+ }
2021
+
2022
+ /** Follows one turn's event stream to its terminal event: the Process Table learns the session id
2023
+ * the moment `started` arrives and the last output time as text streams (throttled); every event
2024
+ * goes to the local transcript; the output lines are kept for the tail. Shared by a fresh launch,
2025
+ * a Reconcile resume and a Gate's resumed turn. */
2026
+ async function followTurn(deps: RunDeps, task: TrackerTask, handle: TurnHandle, sink: TurnSink, started: number): Promise<TerminalEvent> {
2027
+ const { log } = deps;
2028
+ const now = deps.now ?? Date.now;
2029
+ let lastTouch = started;
2030
+ let terminal: TerminalEvent | undefined;
2031
+ for await (const event of handle.events) {
2032
+ sink.transcript?.append(event);
2033
+ switch (event.type) {
2034
+ case "started":
2035
+ deps.processTable?.setSession(task.id, event.sessionId, handle.pid);
2036
+ log(`worker: session ${event.sessionId}${handle.pid !== undefined ? ` (pid ${handle.pid})` : ""}`);
2037
+ break;
2038
+ case "output": {
2039
+ if (deps.processTable) {
2040
+ const at = now();
2041
+ if (at - lastTouch >= TOUCH_INTERVAL_MS) {
2042
+ deps.processTable.touch(task.id, at);
2043
+ lastTouch = at;
2044
+ }
2045
+ }
2046
+ for (const line of event.text.split("\n")) {
2047
+ sink.output.push(line);
2048
+ log(` | ${line}`);
2049
+ }
2050
+ break;
2051
+ }
2052
+ case "tool":
2053
+ log(` tool ${event.name}`);
2054
+ break;
2055
+ case "rate_limited":
2056
+ sink.rateLimit = { resetAt: event.resetAt, message: event.message };
2057
+ log(`worker: rate limited: ${event.message}${event.resetAt ? ` (resets ${event.resetAt})` : ""}`);
2058
+ break;
2059
+ case "stalled":
2060
+ log(`worker: stalled, no output for ${event.silentMs}ms`);
2061
+ break;
2062
+ case "completed":
2063
+ case "failed":
2064
+ terminal = event;
2065
+ break;
2066
+ }
2067
+ }
2068
+ if (!terminal) throw new Error("Worker Adapter contract violated: the event stream ended without a terminal event");
2069
+ return terminal;
2070
+ }
2071
+
2072
+ /** Where a `done` turn stands after its Hand-off (and, when it opened, its Gates). */
2073
+ interface DoneState {
2074
+ terminal: CompletedEvent;
2075
+ /** The report as recorded: with the Hand-off the Foreman verified on the remote, over anything
2076
+ * the Worker claimed. */
2077
+ report: CompletionReport;
2078
+ handOff: HandOffResult;
2079
+ handOffNote: string;
2080
+ gates?: GateOutcome;
2081
+ verification?: VerifyOutcome;
2082
+ /** The last `done` report whose Hand-off stands, when `report` is a later turn's that did not
2083
+ * reach one (a blocked report, an incomplete Hand-off); written to memory beside it. */
2084
+ priorDone?: CompletionReport;
2085
+ }
2086
+
2087
+ /** A `done` turn whose Hand-off opened: the state the Gates judge and the pull request they hold. */
2088
+ type HandedOffState = DoneState & { handOff: HandedOff };
2089
+
2090
+ /** The Hand-off of a `done` report: pushes and opens or updates the draft pull request, and puts
2091
+ * the verified pull request into the report so memory and the comment name it. */
2092
+ async function handOffTurn(deps: RunDeps, initiativeId: number, task: TrackerTask, workspace: Workspace, terminal: CompletedEvent): Promise<DoneState> {
2093
+ let report = terminal.report;
2094
+ const handOff = await (deps.handOff ?? noHandOff)({ task, workspace, report, initiativeId });
2095
+ let handOffNote: string;
2096
+ if (handOff.kind === "handed_off") {
2097
+ report = { ...report, handOff: { branch: workspace.branch, prUrl: handOff.pullRequest.url } };
2098
+ handOffNote = `\n\nHand-off: ${handOff.pullRequest.url}${handOff.reused ? " (updated)" : ""}`;
2099
+ } else {
2100
+ deps.log(`hand-off: ${task.identifier} incomplete: ${handOff.reason}`);
2101
+ handOffNote = `\n\nHand-off incomplete: ${handOff.reason}`;
2102
+ }
2103
+ return { terminal, report, handOff, handOffNote };
2104
+ }
2105
+
2106
+ /** What the Gates left the attempt in: the state of its last `done` turn (with the Hand-off that
2107
+ * stands, whichever turn opened it), or the failure of a resumed turn beside the last handed-off
2108
+ * state, with the Gates' rounds either way. */
2109
+ type GateDrive = { done: DoneState } | { failed: Extract<AdapterEvent, { type: "failed" }>; last: HandedOffState; gates: GateOutcome };
2110
+
2111
+ /** Runs the Gates on an open Hand-off and, while they fail and the retry cap allows, resumes the
2112
+ * same Worker session with the failing command and its output, hands its next `done` turn off
2113
+ * again (the same pull request, updated) and runs the Gates once more. Every round's results go to
2114
+ * the Run Record before anything else (ADR-0009); the rounds are summarised on the pull request at
2115
+ * the end, whatever the end was. A resumed turn that ends without a `done` report, or whose
2116
+ * Hand-off is incomplete, ends the Gates as failed with that as the reason: the Task is settled
2117
+ * blocked with what happened, and the last Hand-off that did open stays the attempt's, on the Run
2118
+ * Record, linked to the Task and carrying the comment. */
2119
+ async function driveGates(deps: RunDeps, initiativeId: number, ctx: TaskWithContext, workspace: Workspace, attempt: number, state: DoneState, sink: TurnSink): Promise<GateDrive> {
2120
+ const { tracker, worker, account, log } = deps;
2121
+ const now = deps.now ?? Date.now;
2122
+ const task = ctx.task;
2123
+ const gates = deps.gates ?? noGates;
2124
+ const retries = deps.gateRetries ?? DEFAULT_GATE_RETRIES;
2125
+ const rounds: GateRound[] = [];
2126
+ let resumes = 0;
2127
+ // A skip does not hold the Hand-off (a repository may name no Gate), but it is not a pass either:
2128
+ // `gateSummary` and `gateNote` say "skipped" wherever a pass would have said "passed".
2129
+ const outcome = (reason: GateReason, detail?: string): GateOutcome => ({ rounds, passed: reason === "passed" || reason === "skipped", resumes, reason, detail });
2130
+
2131
+ // No Gates without a Hand-off: a turn with nothing to push is settled as it is.
2132
+ if (state.handOff.kind !== "handed_off") return { done: state };
2133
+ let handedOff: HandedOffState = { ...state, handOff: state.handOff };
2134
+ const summarise = async (gates: GateOutcome) => {
2135
+ const { pullRequest } = handedOff.handOff;
2136
+ await commentOnPullRequest(deps, task, pullRequest.number, pullRequest.url, gateComment(gates, retries));
2137
+ };
2138
+
2139
+ for (;;) {
2140
+ const round = rounds.length + 1;
2141
+ const run = await gates(workspace);
2142
+ if (run.skipped) {
2143
+ log(`gate: ${task.identifier} skipped: ${run.reason}; the Hand-off stands as it is, unchecked`);
2144
+ // Past round 1 the base stopped naming Gates between rounds: earlier rounds did judge this
2145
+ // Hand-off, so that is a pass with the reason. At round 1 nothing judged it at all, and that
2146
+ // is carried as a `skipped` outcome rather than as no outcome: with none, `settle` writes no
2147
+ // gate note on the Task and no gate trailer on the attempt event, the Verifier skips too, and
2148
+ // the Task reaches In review under foreman:done with the only trace a line in the daemon log.
2149
+ return { done: { ...handedOff, gates: outcome(rounds.length ? "passed" : "skipped", run.reason) } };
2150
+ }
2151
+ const passed = run.results.every(gatePassed);
2152
+ // Run Record first (ADR-0009): the results are the commit, whatever the resume or the board do next.
2153
+ await tracker.appendRunEvents(
2154
+ initiativeId,
2155
+ run.results.map((r) => gateEvent(task, attempt, round, r)),
2156
+ );
2157
+ rounds.push({ round, results: run.results, passed });
2158
+ for (const r of run.results) log(`gate: ${task.identifier} round ${round}: ${gateLine(r)}`);
2159
+ if (passed) {
2160
+ log(`gate: ${task.identifier} passed in round ${round}${resumes ? ` after ${count(resumes, "resumed turn")}` : ""}`);
2161
+ const done = { ...handedOff, gates: outcome("passed") };
2162
+ await summarise(done.gates);
2163
+ return { done };
2164
+ }
2165
+ const failing = run.results.find((r) => !gatePassed(r))!;
2166
+ if (resumes >= retries) {
2167
+ log(`gate: ${task.identifier} ${failing.name} failed in round ${round}, past the retry cap (${retries}); the Task is blocked`);
2168
+ const done = { ...handedOff, gates: outcome("cap") };
2169
+ await summarise(done.gates);
2170
+ return { done };
2171
+ }
2172
+
2173
+ // The same session, so the Worker keeps its context; the same Process Table row, since the
2174
+ // attempt goes on. The next Hand-off updates the pull request already open for the branch.
2175
+ resumes++;
2176
+ const sessionId = handedOff.terminal.sessionId;
2177
+ log(`gate: ${task.identifier} ${failing.name} failed in round ${round} (${gateLine(failing)}); resuming session ${sessionId}, retry ${resumes} of ${retries}`);
2178
+ const handle = worker.resume(sessionId, {
2179
+ account,
2180
+ cwd: workspace.path,
2181
+ prompt: gateFixPrompt(task, failing, round, retries - resumes),
2182
+ maxTurns: deps.maxTurns,
2183
+ stallTimeoutMs: deps.stallTimeoutMs,
2184
+ });
2185
+ if (handle.pid !== undefined) deps.processTable?.setPid(task.id, handle.pid);
2186
+ const terminal = await followTurn(deps, task, handle, sink, now());
2187
+ if (terminal.type === "failed") {
2188
+ log(`gate: ${task.identifier} resumed turn ${terminal.reason}: ${terminal.message}; the Task is blocked`);
2189
+ const gates = outcome("turn_failed", `${terminal.reason}: ${terminal.message}`);
2190
+ await summarise(gates);
2191
+ return { failed: terminal, last: handedOff, gates };
2192
+ }
2193
+ if (terminal.report.outcome !== "done") {
2194
+ log(`gate: ${task.identifier} resumed turn reported ${terminal.report.outcome}; the Task is blocked`);
2195
+ // The pull request from before is still the attempt's Hand-off; the new report says why it
2196
+ // stopped, and the report it supersedes goes to memory beside it.
2197
+ const gates = outcome("turn_not_done", terminal.report.outcome);
2198
+ await summarise(gates);
2199
+ return { done: { ...handedOff, terminal, report: { ...terminal.report, handOff: handedOff.report.handOff }, gates, priorDone: handedOff.report } };
2200
+ }
2201
+ const next = await handOffTurn(deps, initiativeId, task, workspace, terminal);
2202
+ if (next.handOff.kind !== "handed_off") {
2203
+ log(`gate: ${task.identifier} resumed turn's Hand-off incomplete: ${next.handOff.reason}; the Task is blocked`);
2204
+ // The new report is the work's latest word, but the pull request that stands is the earlier one.
2205
+ const gates = outcome("handoff_incomplete", next.handOff.reason);
2206
+ await summarise(gates);
2207
+ return {
2208
+ done: {
2209
+ ...handedOff,
2210
+ terminal: next.terminal,
2211
+ report: { ...next.report, handOff: handedOff.report.handOff },
2212
+ handOffNote: `${handedOff.handOffNote}\n\nHand-off of the resumed turn incomplete: ${next.handOff.reason}`,
2213
+ gates,
2214
+ },
2215
+ };
2216
+ }
2217
+ handedOff = { ...next, handOff: next.handOff };
2218
+ }
2219
+ }
2220
+
2221
+ /** Posts the Gates' summary on the pull request; best effort, since the Run Record already has
2222
+ * every result, and a Run without a code host client only logs it. */
2223
+ async function commentOnPullRequest(deps: RunDeps, task: TrackerTask, number: number, url: string, body: string, stage: "gate" | "verify" = "gate"): Promise<void> {
2224
+ const what = stage === "gate" ? "summary" : "Verdict";
2225
+ if (!deps.remote) {
2226
+ deps.log(`${stage}: ${task.identifier}: no code host client in this Run, so the ${what} is not posted on ${url}`);
2227
+ return;
2228
+ }
2229
+ try {
2230
+ await deps.remote.addPullRequestComment(number, body);
2231
+ deps.log(`${stage}: ${task.identifier} ${what} posted on ${url}`);
2232
+ } catch (error) {
2233
+ deps.log(`${stage}: ${task.identifier} could not post the ${what} on ${url}: ${error instanceof Error ? error.message : String(error)}`);
2234
+ }
2235
+ }
2236
+
2237
+ /** The Verifier's round on a Hand-off whose Gates passed: the seam runs the app and the Verifier
2238
+ * Worker; here the screenshots go to the pull request (or stay on this machine, named), the
2239
+ * Verdict goes to the Run Record first (ADR-0009), then to the pull request as a comment. A round
2240
+ * that failed without a Verdict is recorded as a failed Verdict with one Defect saying why. */
2241
+ async function driveVerifier(deps: RunDeps, initiativeId: number, task: TrackerTask, workspace: Workspace, attempt: number, handedOff: HandedOff, verification: string | undefined): Promise<VerifyOutcome | undefined> {
2242
+ const { tracker, log } = deps;
2243
+ const now = deps.now ?? Date.now;
2244
+ // No Verifier in the deps: as with the Gates, the loop skips by itself and records nothing;
2245
+ // `foreman run` is what wires the real one.
2246
+ if (!deps.verifier) {
2247
+ log(`verify: ${task.identifier} skipped: this Run has no Verifier configured (RunDeps.verifier); the Hand-off stands as it is`);
2248
+ return undefined;
2249
+ }
2250
+ if (verification === undefined) {
2251
+ log(`verify: ${task.identifier} skipped: the Task has no Verification section and none was drafted`);
2252
+ return { kind: "skipped", reason: "the Task has no Verification section and none was drafted" };
2253
+ }
2254
+ // The Verifier's processes go on the Task's Process Table row: the turn's pid as the row's pid
2255
+ // (the implementer's turn is over), the app's beside it, so `foreman cancel` stops both and
2256
+ // Reconcile stops an app a dead Foreman left running.
2257
+ const onPid = (which: "app" | "verifier", pid: number | undefined) => {
2258
+ if (!deps.processTable) return;
2259
+ if (which === "app") deps.processTable.setAppPid(task.id, pid);
2260
+ else if (pid !== undefined) deps.processTable.setPid(task.id, pid);
2261
+ };
2262
+ let outcome: VerifyOutcome;
2263
+ await markStatus(tracker, task, undefined, STATUS_LABELS.verifying);
2264
+ try {
2265
+ outcome = await deps.verifier({ task, workspace, attempt, initiativeId, verification, onPid });
2266
+ } catch (error) {
2267
+ // The seam threw instead of reporting: a failed round, never an aborted dispatch, so the attempt
2268
+ // event is still written and the Task still lands somewhere a human can see.
2269
+ const reason = `the Verifier failed unexpectedly: ${error instanceof Error ? error.message : String(error)}`;
2270
+ log(`verify: ${task.identifier} ${reason}; the round failed`);
2271
+ outcome = { kind: "failed", reason, durationMs: 0 };
2272
+ }
2273
+ // Off as soon as the round is over, whichever way it went. The catch above does not rethrow, so
2274
+ // every path reaches here; the settlement clears it again as a backstop.
2275
+ await tracker.detachLabel(task.id, STATUS_LABELS.verifying);
2276
+ if (outcome.kind === "skipped") return outcome;
2277
+ let recorded: RecordedVerdict | undefined;
2278
+ try {
2279
+ recorded = await recordVerdict(deps, initiativeId, task, attempt, handedOff, outcome);
2280
+ } catch (error) {
2281
+ // The Verdict could not be recorded (the backend refused the event, say): the round is still
2282
+ // a failed one, recorded as such with a short reason if that at least will go.
2283
+ const reason = `the Verdict could not be recorded: ${error instanceof Error ? error.message : String(error)}`;
2284
+ log(`verify: ${task.identifier} ${reason}; the round failed`);
2285
+ const failed: VerifyOutcome = { kind: "failed", reason, app: outcome.app, durationMs: outcome.durationMs, sessionId: outcome.sessionId, outputTail: outcome.outputTail };
2286
+ try {
2287
+ await tracker.appendRunEvents(initiativeId, [verdictEvent(task, attempt, failureVerdict(reason), [], outcome.app, now())]);
2288
+ } catch (again) {
2289
+ log(`verify: ${task.identifier} the failed round could not be recorded either: ${again instanceof Error ? again.message : String(again)}`);
2290
+ }
2291
+ return failed;
2292
+ }
2293
+ // Every Defect becomes a sibling Task (OPS-273), after the Verdict is committed and outside its
2294
+ // try: a filer that fails leaves a recorded Verdict a human can still act on, never a round
2295
+ // reported as failed for a reason that has nothing to do with what the Verifier saw.
2296
+ await fileDefects(deps, initiativeId, task, handedOff, recorded);
2297
+ return outcome;
2298
+ }
2299
+
2300
+ /** Files each observed Defect of a failed Verdict as a sibling Task through the seam (defects.ts).
2301
+ * Never throws and never changes what the round was: the Verdict is already the commit, and the
2302
+ * original Task's Hand-off stays open and blocked whether or not its fixes could be filed. */
2303
+ async function fileDefects(deps: RunDeps, initiativeId: number, task: TrackerTask, handedOff: HandedOff, recorded: RecordedVerdict): Promise<FiledDefect[]> {
2304
+ const { log } = deps;
2305
+ const { verdict, evidence } = recorded;
2306
+ if (verdict.passed) return [];
2307
+ if (!deps.defects) {
2308
+ log(`defect: ${task.identifier} ${count(verdict.defects.length, "Defect")} not filed: this Run has no Defect filer configured (RunDeps.defects)`);
2309
+ return [];
2310
+ }
2311
+ // A round that failed before the Verifier observed anything carries one stand-in Defect that
2312
+ // says so; there is nothing there for a Worker to fix, and the filer skips it. The Verdict's own
2313
+ // flag says which kind it is, so no Verifier can suppress filing by how it titles a Defect.
2314
+ if (isRoundFailure(verdict)) {
2315
+ log(`defect: ${task.identifier} nothing to file: the round failed before the Verifier observed a Defect`);
2316
+ return [];
2317
+ }
2318
+ try {
2319
+ // No attempt: a Defect is keyed by its title alone, so the same failure seen on a later
2320
+ // attempt is the same Defect and is not filed twice (see core/defects.ts).
2321
+ const filed = await deps.defects({ initiativeId, task, verdict, evidence, pullRequestUrl: handedOff.pullRequest.url });
2322
+ const created = filed.filter((f) => f.task);
2323
+ log(
2324
+ `defect: ${task.identifier} ${count(created.length, "Defect")} filed as ${created.length === 1 ? "a sibling Task" : "sibling Tasks"}${created.length ? `: ${created.map((f) => f.task!.identifier).join(", ")}` : ""}${filed.length > created.length ? ` (${filed.length - created.length} skipped)` : ""}`,
2325
+ );
2326
+ return filed;
2327
+ } catch (error) {
2328
+ log(`defect: ${task.identifier} the Defects could not be filed: ${error instanceof Error ? error.message : String(error)}; the Verdict stands on ${handedOff.pullRequest.url}`);
2329
+ return [];
2330
+ }
2331
+ }
2332
+
2333
+ /** What `recordVerdict` committed: the Verdict as it was written to the Run Record and where each
2334
+ * Defect's screenshot ended up, so the Defect filer links the evidence rather than pasting it. */
2335
+ interface RecordedVerdict {
2336
+ verdict: Verdict;
2337
+ attempt: number;
2338
+ evidence: DefectEvidenceLink[];
2339
+ }
2340
+
2341
+ /** The screenshots to the pull request (or kept here, named), the `verdict` event to the Run
2342
+ * Record first (ADR-0009), then the Verdict as a comment. Throws only when the Run Record refused
2343
+ * the event; the upload and the comment are best effort and logged. */
2344
+ async function recordVerdict(deps: RunDeps, initiativeId: number, task: TrackerTask, attempt: number, handedOff: HandedOff, outcome: Exclude<VerifyOutcome, { kind: "skipped" }>): Promise<RecordedVerdict> {
2345
+ const { tracker, log } = deps;
2346
+ const now = deps.now ?? Date.now;
2347
+ const verdict = verdictOf(outcome);
2348
+ const { pullRequest } = handedOff;
2349
+ const evidence: DefectEvidence[] = [];
2350
+ const links: DefectEvidenceLink[] = [];
2351
+ const urls: string[] = [];
2352
+ for (let i = 0; i < verdict.defects.length; i++) {
2353
+ const shot = outcome.kind === "verified" ? outcome.screenshots.find((s) => s.defect === i) : undefined;
2354
+ if (!shot?.path) {
2355
+ evidence.push({});
2356
+ links.push({});
2357
+ urls.push("");
2358
+ continue;
2359
+ }
2360
+ let uploaded: Awaited<ReturnType<NonNullable<RunDeps["remote"]>["uploadAttachment"]>> | undefined;
2361
+ try {
2362
+ uploaded = await deps.remote?.uploadAttachment(pullRequest.number, shot.path);
2363
+ } catch (error) {
2364
+ log(`verify: ${task.identifier} could not upload ${shot.path} to ${pullRequest.url}: ${error instanceof Error ? error.message : String(error)}`);
2365
+ }
2366
+ if (uploaded) {
2367
+ log(`verify: ${task.identifier} uploaded ${shot.path} as ${uploaded.url}`);
2368
+ evidence.push({ markdown: uploaded.markdown });
2369
+ // The Defect Task links the upload; the bytes stay on the pull request.
2370
+ links.push({ url: uploaded.url });
2371
+ urls.push(uploaded.url);
2372
+ } else {
2373
+ evidence.push({ localPath: shot.path });
2374
+ // No upload on this code host: the Defect Task names the file, never the Foreman's home.
2375
+ links.push({ localFile: basename(shot.path) });
2376
+ urls.push(shot.path);
2377
+ }
2378
+ }
2379
+ // Run Record first (ADR-0009): the Verdict is the commit, whatever the comment or the board do next.
2380
+ await tracker.appendRunEvents(initiativeId, [verdictEvent(task, attempt, verdict, urls, outcome.app, now())]);
2381
+ await commentOnPullRequest(deps, task, pullRequest.number, pullRequest.url, verdictComment(verdict, evidence, outcome.app), "verify");
2382
+ return { verdict, attempt, evidence: links };
2383
+ }
2384
+
2385
+ /** Follows one attempt to its settlement: the turn, then for a `done` turn the Hand-off and the
2386
+ * Gates (with their resumed turns), then the Run Record, memory, comment and board. Shared by a
2387
+ * fresh launch and a Reconcile resume. */
2388
+ async function observeTurn(deps: RunDeps, initiativeId: number, ctx: TaskWithContext, workspace: Workspace, attempt: number, handle: TurnHandle, obs: TurnObservation): Promise<Dispatch> {
2389
+ const { tracker, log } = deps;
2390
+ const now = deps.now ?? Date.now;
2391
+ const task = ctx.task;
2392
+ const { started, installNote, install } = obs;
2393
+ const gateRetries = deps.gateRetries ?? DEFAULT_GATE_RETRIES;
2394
+ const sink: TurnSink = { transcript: deps.transcripts?.open(initiativeId, task.identifier, attempt), output: [] };
2395
+ const first = await followTurn(deps, task, handle, sink, started);
2396
+
2397
+ // Only a `done` turn is handed off: a blocked or failed one keeps its Workspace, and the branch
2398
+ // with it, for the next attempt. What gets recorded is the report with the Hand-off the Foreman
2399
+ // verified on the remote (over anything the Worker claimed), so the memory entries get it as
2400
+ // their source URL and the comment names it; the Worker's own report is left as it was.
2401
+ let terminal: TerminalEvent = first;
2402
+ let done: DoneState | undefined;
2403
+ /** The last handed-off state when a resumed turn then failed: its pull request stands. */
2404
+ let lastDone: HandedOffState | undefined;
2405
+ let gates: GateOutcome | undefined;
2406
+ let handOff: HandOffResult | undefined;
2407
+ if (first.type === "completed" && first.report.outcome === "done") {
2408
+ const drive = await driveGates(deps, initiativeId, ctx, workspace, attempt, await handOffTurn(deps, initiativeId, task, workspace, first), sink);
2409
+ if ("done" in drive) {
2410
+ done = drive.done;
2411
+ terminal = done.terminal;
2412
+ // The Verifier follows passing Gates (story 37): a Hand-off the Gates held, or none, gets no round.
2413
+ if (done.handOff.kind === "handed_off" && (done.gates?.passed ?? true)) {
2414
+ const section = verificationSectionOf(task, obs.draftedVerification);
2415
+ // Into the Account's reserved Verifier Slot (story 29), which gives the implementer Slot
2416
+ // back so the next Ready Task can start. Only for a round that will really run: a Run
2417
+ // without a Verifier, or a Task with nothing to verify, must not hold the reserved Slot
2418
+ // while it decides to skip. (A Task the seam itself then skips — commands only, no user
2419
+ // journey — does hold it, for as long as that decision takes, which is no time at all.)
2420
+ if (deps.verifier && section !== undefined) await obs.slot?.enterVerification();
2421
+ let verification: VerifyOutcome | undefined;
2422
+ try {
2423
+ verification = await driveVerifier(deps, initiativeId, task, workspace, attempt, done.handOff, section);
2424
+ } finally {
2425
+ // The round is over, so the reserved Slot goes to the next dispatch waiting to verify.
2426
+ // What follows — the Verdict comment, the settlement, the board, the row — is round-trips
2427
+ // with nothing running on the Account, and holding the one reserved Slot through them
2428
+ // would stall every other Verifier for the length of a settlement.
2429
+ obs.slot?.leaveVerification();
2430
+ }
2431
+ if (verification) done = { ...done, verification };
2432
+ }
2433
+ } else {
2434
+ terminal = drive.failed;
2435
+ lastDone = drive.last;
2436
+ handOff = lastDone.handOff;
2437
+ gates = drive.gates;
2438
+ }
2439
+ }
2440
+ const durationMs = now() - started;
2441
+ // Before the attempt event, because the spend happened during the turn the attempt settles.
2442
+ const turnSpend = spendEvent(task.id, deps.account.vendor, deps.account.name, terminal.type === "completed" ? terminal.costUsd : undefined);
2443
+ if (turnSpend) await deps.tracker.appendRunEvents(initiativeId, [turnSpend]);
2444
+ const outputTail = tail(sink.output.join("\n"));
2445
+ const trailer = `${installNote}\n\nTurn: ${durationMs}ms.${outputTail ? `\n\nOutput tail:\n${outputTail}` : ""}`;
2446
+ const marker = obs.resumed ? `${RESUMED_MARKER} ` : "";
2447
+ const resumed = obs.resumed ? " (resumed)" : "";
2448
+ const gateTrailer = (g: GateOutcome | undefined) => (g ? `\n\nGates: ${gateSummary(g, gateRetries)}` : "");
2449
+ const verifyTrailer = (v: VerifyOutcome | undefined) => (v ? `\n\nVerification: ${verifySummary(v)}` : "");
2450
+
2451
+ if (terminal.type === "completed") {
2452
+ // A turn the vendor accepted is proof the credential is live: it closes any open streak, so the
2453
+ // failures that quarantine are consecutive ones and a Run's unlucky 401 cannot accumulate with
2454
+ // another an hour later (OPS-288).
2455
+ clearCredentialStreak(deps);
2456
+ const report = done?.report ?? terminal.report;
2457
+ handOff = done?.handOff;
2458
+ gates = done?.gates;
2459
+ const verification = done?.verification;
2460
+ // The recorded report's entries first, so the comment's outcome entry is its own; then the
2461
+ // `done` report it superseded without reaching a Hand-off, so that report is not lost.
2462
+ const own = reportToMemory(task, report);
2463
+ const prior = done?.priorDone ? reportToMemory(task, done.priorDone) : [];
2464
+ const memory = [...own, ...prior];
2465
+ await settle(
2466
+ tracker,
2467
+ initiativeId,
2468
+ task,
2469
+ {
2470
+ attempt,
2471
+ outcome: "completed",
2472
+ summary: `${marker}${report.outcome}: ${report.summary}${done?.handOffNote ?? ""}${gateTrailer(gates)}${verifyTrailer(verification)}${trailer}`,
2473
+ memory,
2474
+ comment: (ids, note) => reportComment(report, ids, `${prior.length ? priorReportNote(ids[own.length]) : ""}${note}`),
2475
+ handOff,
2476
+ gates,
2477
+ verification,
2478
+ gateRetries,
2479
+ },
2480
+ log,
2481
+ );
2482
+ deps.processTable?.removeWorker(task.id);
2483
+ const handedOff =
2484
+ handOff?.kind === "handed_off"
2485
+ ? gates?.passed === false
2486
+ ? `; Hand-off ${handOff.pullRequest.url} held by Gates`
2487
+ : !verificationPassed(verification)
2488
+ ? `; Hand-off ${handOff.pullRequest.url} held by the Verdict`
2489
+ : `; handed off as ${handOff.pullRequest.url}`
2490
+ : handOff
2491
+ ? "; Hand-off incomplete"
2492
+ : "";
2493
+ log(`attempt: ${task.identifier} completed${resumed} (${report.outcome}) after ${durationMs}ms; ${count(memory.length, "memory entry", "memory entries")} written${handedOff}`);
2494
+ return { task, workspace, attempt, outcome: "completed", report, sessionId: terminal.sessionId, durationMs, install, handOff, gates, verification, resumed: obs.resumed };
2495
+ }
2496
+
2497
+ // The stream ended without a report. If a human stopped it from the CLI, the row says so and
2498
+ // the settlement is theirs, not the failure's (stories 53, 56): an attached Worker's attempt
2499
+ // stays open for the attended turn; a cancelled one is closed as cancelled.
2500
+ const control = deps.processTable?.worker(task.id)?.control;
2501
+ if (control === "attached" || control === "released") {
2502
+ // Released already: the attach could not open its session (or the human was quick) and handed
2503
+ // the Worker back before this stream ended; the attempt stays open all the same, for Reconcile.
2504
+ log(
2505
+ control === "attached"
2506
+ ? `attempt: ${task.identifier} attempt ${attempt} stopped for a human to attach after ${durationMs}ms; the attempt stays open until foreman release`
2507
+ : `attempt: ${task.identifier} attempt ${attempt} stopped for a human to attach after ${durationMs}ms and already released; the attempt stays open for the next tick to resume`,
2508
+ );
2509
+ return { task, workspace, attempt, outcome: "stopped", sessionId: terminal.sessionId, durationMs, install, resumed: obs.resumed, attached: true };
2510
+ }
2511
+ if (control === "cancelled") {
2512
+ const details = terminal.details?.length ? `\n${terminal.details.join("\n")}` : "";
2513
+ return settleCancelled(deps, initiativeId, ctx, workspace, attempt, `\n\nThe turn ended ${terminal.reason}: ${terminal.message}${details}${outputTail ? `\n\nOutput tail:\n${outputTail}` : ""}`, obs);
2514
+ }
2515
+
2516
+
2517
+ const failure = terminal;
2518
+ const outcome = attemptOutcomeOf(failure.reason);
2519
+ // After the resumed marker, so a resumed turn still reads as one; `isCredentialSummary` searches
2520
+ // rather than testing a prefix for exactly that reason.
2521
+ const credentialMarker = failure.reason === "credential_failed" ? `${CREDENTIAL_MARKER} ` : "";
2522
+ const details = failure.details?.length ? `\n${failure.details.join("\n")}` : "";
2523
+ const handOffNote = handOff?.kind === "handed_off" ? `\n\nHand-off: ${handOff.pullRequest.url}` : "";
2524
+ // A resumed turn that failed after a `done` turn had been handed off: that report is still the
2525
+ // work's record and goes to memory, though the attempt itself ended without one.
2526
+ const memory = lastDone ? reportToMemory(task, lastDone.report) : [];
2527
+ // The vendor said it is rate limited: the Account is Paused until the reset it named (story 30),
2528
+ // before the Task is settled, so the very next Slot this Run fills already sees it and goes to a
2529
+ // sibling Account instead. The pause is written through the AccountStore rather than held in the
2530
+ // Run, because it outlives the Run: the next Foreman and `foreman account list` read the same
2531
+ // fact. Its own Workers are untouched — they finish where they are, and Reconcile may resume
2532
+ // them; what is Paused is starting new ones.
2533
+ if (failure.reason === "rate_limited") await pauseOnRateLimit(deps, initiativeId, task.id, sink.rateLimit, failure.message, now());
2534
+ // The credential behind the Account is dead rather than busy (OPS-288). Recorded before the Task
2535
+ // is settled for the same reason a pause is: the Slot this frees is filled on the same tick, and
2536
+ // it must not be filled on the Account that just proved it cannot run anything.
2537
+ if (failure.reason === "credential_failed") await noteCredentialFailureOnAccount(deps, initiativeId, task.id, failure.message, now());
2538
+
2539
+ // A stall or a rate limit is the machine's or the vendor's failure, not the work's
2540
+ // (`RETRYABLE_FAILURES`): the Worker went quiet and the adapter stopped it, or the vendor refused
2541
+ // the turn, neither of which says anything about whether the Task can be done. Under the retry
2542
+ // cap the Task is made Ready again and a later tick dispatches it as the next attempt — on
2543
+ // whichever Account is not Paused by then — rather than being left blocked for a human who would
2544
+ // only tell it to try again. At the cap it is blocked like any other failed attempt, with the
2545
+ // comment naming the cause that spent the attempts (story 34), and the Run goes on with its other
2546
+ // Tasks. The next attempt comes on the next tick with no backoff (`DEFAULT_ATTEMPT_RETRIES`): a
2547
+ // fast-failing adapter spends them in milliseconds, on every Slot at once.
2548
+ //
2549
+ // What the cap counts is retryable failures, not dispatches: `attempt` numbers every turn the
2550
+ // Task ever had (the Run Record's numbering, which other components match on), and a Task whose
2551
+ // Gates failed, or whose Verdict found Defects, spent no machine-side attempt at all. Counting
2552
+ // dispatches would let a Task that a human relabelled twice hit its very first rate limit with
2553
+ // nothing left, blocked on a vendor's bad minute with no Failover ever tried; and the comment
2554
+ // would then say "3 attempts ended rate_limited" when one did.
2555
+ const attemptRetries = deps.attemptRetries ?? DEFAULT_ATTEMPT_RETRIES;
2556
+ // The reason, not the outcome: a credential failure is recorded as the generic `failed` and would
2557
+ // otherwise be read as work that cannot be retried (`RETRYABLE_FAILURE_REASONS`).
2558
+ const retryable = RETRYABLE_FAILURE_REASONS.has(failure.reason) && !handOff;
2559
+ // Read only when it can matter: one more Run Record read per retryable failure, none otherwise.
2560
+ const spent = retryable ? (await retryableFailuresOf(tracker, initiativeId, task.id)) + 1 : 0;
2561
+ const attemptsLeft = attemptRetries - (spent - 1);
2562
+ // How long the next attempt waits. Decided from `spent`, the same count the cap is measured in,
2563
+ // so the two never disagree about which attempt this is, and turned into an absolute moment here
2564
+ // against the Run's own clock rather than a duration the tick would have to date itself.
2565
+ const backoffMs = retryable && attemptsLeft > 0 ? retryBackoff(spent, deps.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS) : 0;
2566
+ const retry = retryable && attemptsLeft > 0 ? { attemptsLeft, waitMs: backoffMs } : undefined;
2567
+ const retryNotBefore = retry ? now() + backoffMs : undefined;
2568
+ const capped = retryable && attemptsLeft <= 0 ? { cause: failure.reason, attempts: spent, retries: attemptRetries } : undefined;
2569
+ await settle(
2570
+ tracker,
2571
+ initiativeId,
2572
+ task,
2573
+ {
2574
+ attempt,
2575
+ outcome,
2576
+ summary: `${marker}${credentialMarker}${failure.reason}: ${failure.message}${details}${handOffNote}${gateTrailer(gates)}${trailer}`,
2577
+ memory,
2578
+ comment: (ids, note) => `${failureComment(failure.reason, failure.message)}${memory.length ? priorReportNote(ids[0]) : ""}${note}`,
2579
+ handOff,
2580
+ gates,
2581
+ gateRetries,
2582
+ retry,
2583
+ capped,
2584
+ },
2585
+ log,
2586
+ );
2587
+ deps.processTable?.removeWorker(task.id);
2588
+ log(
2589
+ `attempt: ${task.identifier} ${outcome}${resumed} (${failure.reason}) after ${durationMs}ms: ${failure.message}${memory.length ? `; ${count(memory.length, "memory entry", "memory entries")} written for the earlier done report` : ""}${retry ? `; Ready again${inSeconds(backoffMs)}, ${count(retry.attemptsLeft, "attempt")} left under the retry cap` : ""}${capped ? `; the retry cap (${capped.retries}) is spent on ${capped.cause}, so the Task is blocked and the Run goes on` : ""}`,
2590
+ );
2591
+ return { task, workspace, attempt, outcome, sessionId: failure.sessionId, durationMs, install, handOff, gates, resumed: obs.resumed, retrying: retry !== undefined, blockedByCap: capped !== undefined, retryNotBefore };
2592
+ }
2593
+
2594
+ /** Pauses the turn's Account after a rate limit (story 30): until the reset the vendor named, or
2595
+ * `DEFAULT_PAUSE_MS` when it named none (`pauseUntilFrom` also refuses a reset far enough ahead to
2596
+ * be a parsing mistake). A Run with no AccountStore has nowhere to put the fact and says so, since
2597
+ * "the Account is Paused" that only one process can see is worse than none. */
2598
+ async function pauseOnRateLimit(deps: RunDeps, initiativeId: number, taskId: number | undefined, reported: TurnSink["rateLimit"], fallbackMessage: string, at: number): Promise<void> {
2599
+ const { account, log } = deps;
2600
+ // The reason is the vendor's own prose, parsed out of the Worker's stdout: it is stored in
2601
+ // `~/.opsee/foreman-accounts.json` and printed by `foreman account list`, so it is made one
2602
+ // printable line of a sane length first. An ANSI escape reaching that table would move the
2603
+ // operator's cursor, and its own padding would measure a width the terminal never shows.
2604
+ const reason = printableOneLine(reported?.message ?? fallbackMessage);
2605
+ if (!deps.accounts) {
2606
+ log(`account: "${account.name}" (${account.vendor}) reported a rate limit (${reason}) but this Run has no Account store to record the pause in; nothing is Paused`);
2607
+ return;
2608
+ }
2609
+ // `pauseMs` 0 is not "do not pause": it would set the reset to now, which `pausedUntil` reads as
2610
+ // no pause at all, so the Run would go straight back at the limit that just refused it. A pause
2611
+ // is at least `MIN_PAUSE_MS`.
2612
+ const until = pauseUntilFrom(at, reported?.resetAt, Math.max(MIN_PAUSE_MS, deps.pauseMs ?? DEFAULT_PAUSE_MS));
2613
+ const paused = pauseAccount(deps.accounts, account.name, until, reason);
2614
+ if (!paused) {
2615
+ log(`account: "${account.name}" reported a rate limit but is no longer registered; nothing is Paused`);
2616
+ return;
2617
+ }
2618
+ log(
2619
+ `account: "${account.name}" (${account.vendor}) is Paused until ${until.toISOString()}${reported?.resetAt ? " (the reset the vendor named)" : ` (the vendor named no reset, so ${deps.pauseMs ?? DEFAULT_PAUSE_MS}ms)`}: ${reason}. ` +
2620
+ `No new Worker starts on it until then; the Workers it has keep going, and a Ready Task that has not started goes to another Account of this vendor.`,
2621
+ );
2622
+ await recordAccountState(deps, initiativeId, accountEvent("paused", account.vendor, account.name, reason, { taskId, until }));
2623
+ }
2624
+
2625
+ /**
2626
+ * Reports on the Run Record every hand-made lift this Run's Accounts are still carrying, then
2627
+ * forgets them (`ResumedState`).
2628
+ *
2629
+ * `foreman account resume` cannot write the event itself: it is a local command over the accounts
2630
+ * file with no Initiative, and a Run Record belongs to one. So it stamps the moment and the next
2631
+ * Run with a Lane on that Account reports it here, carrying the stamp as `occurredAt` — the field
2632
+ * that exists because facts reach this Record late.
2633
+ *
2634
+ * Two consequences, both deliberate. The lift is missing from the Record until a Run next touches
2635
+ * that Account, which is honest rather than invented: nothing had read it yet. And where several
2636
+ * Initiatives could report it, whichever Run gets there first does — an Account belongs to no one
2637
+ * Initiative, so the attribution is arbitrary, which is why the event carries no `taskId`.
2638
+ *
2639
+ * The stamp is cleared only after the append is accepted. Through the outbox that includes a write
2640
+ * that was queued rather than delivered, which is the same commit the rest of the Run Record takes;
2641
+ * a refusal leaves the stamp for the next Run rather than losing the lift.
2642
+ */
2643
+ async function reportResumedAccounts(deps: RunDeps, initiativeId: number, lanes: Lane[]): Promise<void> {
2644
+ if (!deps.accounts) return;
2645
+ const stored = deps.accounts.load();
2646
+ for (const lane of lanes) {
2647
+ const current = stored.find((a) => a.name === lane.account.name);
2648
+ const resumed = current?.resumedAt;
2649
+ if (!resumed) continue;
2650
+ const at = new Date(resumed.at);
2651
+ if (!Number.isFinite(at.getTime())) {
2652
+ // A hand-edited accounts file. Dropped rather than dated to now, which would put a moment on
2653
+ // the Record that nobody chose.
2654
+ deps.log(`account: "${lane.account.name}" carries an unreadable resume timestamp (${resumed.at}); it is dropped rather than recorded at the wrong time`);
2655
+ clearResumed(deps.accounts, lane.account.name);
2656
+ continue;
2657
+ }
2658
+ const reason = resumed.from === "both" ? "a human lifted its quarantine and its pause" : `a human lifted its ${resumed.from === "quarantined" ? "quarantine" : "pause"}`;
2659
+ try {
2660
+ await deps.tracker.appendRunEvents(initiativeId, [accountEvent("active", lane.account.vendor, lane.account.name, reason, { occurredAt: at })]);
2661
+ clearResumed(deps.accounts, lane.account.name);
2662
+ deps.log(`account: "${lane.account.name}" was brought back by hand at ${resumed.at}; recorded on the Run Record of Initiative ${initiativeId}`);
2663
+ } catch (error) {
2664
+ deps.log(`account: "${lane.account.name}" was brought back by hand at ${resumed.at}, but the Run Record would not take it, so it stays for the next Run: ${error instanceof Error ? error.message : String(error)}`);
2665
+ }
2666
+ }
2667
+ }
2668
+
2669
+ /**
2670
+ * Puts an Account's state on the Run Record, and never lets that write cost the Run anything.
2671
+ *
2672
+ * The Run Record is the commit for an *attempt* (ADR-0009): losing one loses the fact that a turn
2673
+ * happened. This is not that. The state itself is already in the accounts file, which is the copy
2674
+ * every Run and every `account list` reads, so a backend that refuses this leaves the schedule
2675
+ * correct and only the record of *when* it changed missing. Failing the turn over it would trade
2676
+ * something that matters for something that does not.
2677
+ */
2678
+ async function recordAccountState(deps: RunDeps, initiativeId: number, event: RunEventInput): Promise<void> {
2679
+ try {
2680
+ await deps.tracker.appendRunEvents(initiativeId, [event]);
2681
+ } catch (error) {
2682
+ deps.log(`account: the Account state change could not be written to the Run Record, so the morning summary will not explain it: ${error instanceof Error ? error.message : String(error)}`);
2683
+ }
2684
+ }
2685
+
2686
+ /**
2687
+ * Records one credential failure against the turn's Account, and quarantines it once the streak
2688
+ * reaches `CREDENTIAL_FAILURES_BEFORE_QUARANTINE` (OPS-288, story 32).
2689
+ *
2690
+ * The counting is why this is not simply "quarantine on a credential failure". Every vendor pattern
2691
+ * the adapters match is a guess about prose, and the cost of guessing wrong is asymmetric: a false
2692
+ * positive pulls a working Account until a human re-adds it, which overnight is the whole Lane,
2693
+ * where a false negative only spends the Task's retry cap. A `CredentialError` is exempt because it
2694
+ * is not a guess — the Foreman established it about its own Account record before spawning anything
2695
+ * — but it reaches here as a `credential_failed` turn like any other, so the exemption is the
2696
+ * adapter's `launchFailureReason` doing its job upstream and not a second rule here.
2697
+ *
2698
+ * The transition goes to the Initiative's memory log as a `blocker`, which is what `foreman review`
2699
+ * reads in the morning: a Run that went quiet because its only Account died must be able to say so
2700
+ * somewhere a human looks. Deliberately not a Run Record event — there is no Account-state kind, and
2701
+ * `control` means a human steered the Run — which is also why a Pause has never been recorded there.
2702
+ * Written once, on the transition: the check is that the Account was not already quarantined, so a
2703
+ * later Run that meets the same dead login says nothing rather than repeating itself.
2704
+ */
2705
+ async function noteCredentialFailureOnAccount(deps: RunDeps, initiativeId: number, taskId: number | undefined, message: string, at: number): Promise<void> {
2706
+ const { account, log } = deps;
2707
+ // The vendor's own prose, on its way to `~/.opsee/foreman-accounts.json` and to two terminal
2708
+ // views: made one printable line first, exactly as a pause reason is.
2709
+ const reason = printableOneLine(message);
2710
+ if (!deps.accounts) {
2711
+ log(`account: "${account.name}" (${account.vendor}) failed for a credential reason (${reason}) but this Run has no Account store to record it in; nothing is quarantined`);
2712
+ return;
2713
+ }
2714
+ const noted = noteCredentialFailure(deps.accounts, account.name, new Date(at), reason);
2715
+ if (!noted) {
2716
+ log(`account: "${account.name}" failed for a credential reason but is no longer registered; nothing is quarantined`);
2717
+ return;
2718
+ }
2719
+ const count = noted.credentialFailures?.count ?? 1;
2720
+ if (count < CREDENTIAL_FAILURES_BEFORE_QUARANTINE) {
2721
+ log(
2722
+ `account: "${account.name}" (${account.vendor}) failed for a credential reason (${reason}); that is ${count} in a row and ${CREDENTIAL_FAILURES_BEFORE_QUARANTINE} quarantines it. ` +
2723
+ `The Account stays in the schedule: one such failure is not enough to take a working Account out of rotation, and a turn that completes on it clears the count.`,
2724
+ );
2725
+ return;
2726
+ }
2727
+ const alreadyQuarantined = isQuarantined(noted);
2728
+ const quarantined = quarantineAccount(deps.accounts, account.name, new Date(at), reason);
2729
+ if (!quarantined) {
2730
+ log(`account: "${account.name}" failed for a credential reason but is no longer registered; nothing is quarantined`);
2731
+ return;
2732
+ }
2733
+ log(
2734
+ `account: "${account.name}" (${account.vendor}) is quarantined after ${count} credential failures in a row: ${reason}. ` +
2735
+ `It contributes no Slots and is not a Failover target; nothing lifts it by itself. Log the vendor back in, then: opsee foreman account resume ${account.name}`,
2736
+ );
2737
+ if (alreadyQuarantined) return;
2738
+ // The Run Record, not the Initiative's memory log. The memory entry only ever existed because the
2739
+ // Record had no Account-state event to put this in; now that it does, one fact lives in one place
2740
+ // and `foreman review` reads it from there.
2741
+ await recordAccountState(deps, initiativeId, accountEvent("quarantined", account.vendor, account.name, reason, { taskId }));
2742
+ }
2743
+
2744
+ /** Closes an open credential-failure streak on the turn's Account after the vendor accepted a turn.
2745
+ * A no-op without an Account store, and — inside `clearCredentialFailures` — no store write at all
2746
+ * when there is no streak, which is every turn of a healthy Account. */
2747
+ function clearCredentialStreak(deps: RunDeps): void {
2748
+ if (deps.accounts) clearCredentialFailures(deps.accounts, deps.account.name);
2749
+ }
2750
+
2751
+ /** The comment's pointer at the `done` report a later turn superseded without reaching a Hand-off. */
2752
+ function priorReportNote(outcomeId: number): string {
2753
+ return ` The earlier done report is in Initiative memory (outcome entry #${outcomeId}).`;
2754
+ }
2755
+
2756
+ /** What a resumed attempt's summary starts with, so the Run Record says the turn was continued. */
2757
+ export const RESUMED_MARKER = "[resumed]";
2758
+
2759
+ /** What `RunAttemptEvent.summary` takes (proto/api/v1/initiative.proto: `max_len: 20000`, counted
2760
+ * in code points). */
2761
+ export const ATTEMPT_SUMMARY_MAX = 20_000;
2762
+
2763
+ /** What the cut says, so the Run Record never ends mid-word with no explanation — every other
2764
+ * truncation here (a Defect's text, a Gate's command and tails) names itself the same way. */
2765
+ const SUMMARY_CUT = "\n\n… (summary cut to the Run Record's 20000-character limit; the rest is in the transcript)";
2766
+
2767
+ /**
2768
+ * An attempt event's summary within the proto's limit.
2769
+ *
2770
+ * The summary is assembled from a Completion Report's own summary (which the report schema does not
2771
+ * bound), the Hand-off note, the Gate and Verification trailers, the install note and an output
2772
+ * tail. Past 20000 code points the backend answers `InvalidArgument`, which is not queueable
2773
+ * (core/outbox-tracker.ts) — `appendNow` rethrows, `settle` throws, and the dispatch is marked
2774
+ * blocked with no attempt event on the Run Record at all. Under ADR-0009 that is the loss of the
2775
+ * commit itself, so the summary is cut here rather than refused there.
2776
+ */
2777
+ export function attemptSummary(text: string): string {
2778
+ if (Array.from(text).length <= ATTEMPT_SUMMARY_MAX) return text;
2779
+ return `${truncateCodePoints(text, ATTEMPT_SUMMARY_MAX - Array.from(SUMMARY_CUT).length)}${SUMMARY_CUT}`;
2780
+ }
2781
+
2782
+ /** A Task a memory entry or edge points at, or a stand-in when the Tracker no longer has it (a
2783
+ * reconcile deleted it): a missing neighbour must not abort the dispatch of a Task that is fine. */
2784
+ async function taskOrPlaceholder(tracker: TrackerAdapter, id: number): Promise<TrackerTask> {
2785
+ try {
2786
+ return (await tracker.getTask(id)).task;
2787
+ } catch {
2788
+ return { id, identifier: `deleted Task ${id}`, title: "(no longer on the Tracker)", description: "", projectId: 0, labels: [] };
2789
+ }
2790
+ }
2791
+
2792
+ /** The learnings file as the Workspace has it: what the default branch carries, since a Workspace
2793
+ * starts from its remote head. Absent when the repository has none yet. */
2794
+ export function acceptedLearningsIn(workspace: Pick<Workspace, "path">, file: string = DEFAULT_LEARNINGS_FILE): AcceptedLearnings | undefined {
2795
+ const path = join(workspace.path, file);
2796
+ return existsSync(path) ? { file, body: readFileSync(path, "utf8") } : undefined;
2797
+ }
2798
+
2799
+ /** Context Assembly for one Task: reads the blockers' Completion Reports and the siblings'
2800
+ * Proposed Learnings from the Initiative's memory log and the Accepted Learnings from the
2801
+ * Workspace, renders the prompt, and logs what went in. */
2802
+ async function assemble(
2803
+ deps: RunDeps,
2804
+ initiativeId: number,
2805
+ ctx: TaskWithContext,
2806
+ workspace: Workspace,
2807
+ log: (line: string) => void,
2808
+ draftedVerification?: string,
2809
+ ): Promise<string> {
2810
+ const { tracker } = deps;
2811
+ const identifiers = new Map<number, string>([[ctx.task.id, ctx.task.identifier]]);
2812
+ const blockers: BlockerReports[] = [];
2813
+ for (const id of blockerIdsOf(ctx.task.id, ctx.dependencies)) {
2814
+ const blocker = await taskOrPlaceholder(tracker, id);
2815
+ identifiers.set(id, blocker.identifier);
2816
+ // The Tracker writes its own `outcome` entries when a Task moves to Done; those are bookkeeping,
2817
+ // not a Completion Report, and stay out.
2818
+ const reports = (await tracker.listMemory(initiativeId, { kinds: ["outcome"], sourceTaskId: id })).filter((r) => !r.isSystem);
2819
+ blockers.push({ task: blocker, reports });
2820
+ }
2821
+
2822
+ const learnings = (await tracker.listMemory(initiativeId, { kinds: ["learning"] }))
2823
+ .filter((l) => !l.isSystem && l.sourceTaskId !== ctx.task.id)
2824
+ .slice(0, SIBLING_LEARNING_LIMIT);
2825
+ const siblingLearnings: SiblingLearning[] = [];
2826
+ for (const l of learnings) {
2827
+ if (l.sourceTaskId !== undefined && !identifiers.has(l.sourceTaskId)) {
2828
+ identifiers.set(l.sourceTaskId, (await taskOrPlaceholder(tracker, l.sourceTaskId)).identifier);
2829
+ }
2830
+ siblingLearnings.push({ body: l.body, identifier: l.sourceTaskId === undefined ? undefined : identifiers.get(l.sourceTaskId), isAgent: l.isAgent });
2831
+ }
2832
+
2833
+ const assembled = assembleContext({ task: ctx.task, parent: ctx.parent, blockers, acceptedLearnings: acceptedLearningsIn(workspace, deps.learningsFile), siblingLearnings, workspace, draftedVerification });
2834
+ for (const line of assembled.summary) log(`context: ${line}`);
2835
+ return assembled.prompt;
2836
+ }