@opsee/cli 0.11.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1962 -0
- package/bin/opsee.js +28 -0
- package/package.json +40 -0
- package/skills/README.md +3 -0
- package/skills/to-issues/SKILL.md +92 -0
- package/skills/to-issues/agents/openai.yaml +5 -0
- package/skills/to-spec/SKILL.md +79 -0
- package/skills/to-spec/agents/openai.yaml +5 -0
- package/skills/wayfinder/SKILL.md +138 -0
- package/skills/wayfinder/agents/openai.yaml +5 -0
- package/src/args.ts +676 -0
- package/src/cli.ts +341 -0
- package/src/commands/account.ts +121 -0
- package/src/commands/deps.ts +11 -0
- package/src/commands/foreman-control.ts +242 -0
- package/src/commands/foreman-debug.ts +131 -0
- package/src/commands/foreman-plan.ts +213 -0
- package/src/commands/foreman-service.ts +186 -0
- package/src/commands/foreman-up.ts +165 -0
- package/src/commands/foreman-views.ts +398 -0
- package/src/commands/foreman.ts +465 -0
- package/src/commands/init.ts +176 -0
- package/src/commands/initiative.ts +192 -0
- package/src/commands/login.ts +24 -0
- package/src/commands/whoami.ts +15 -0
- package/src/foreman/account-store.ts +96 -0
- package/src/foreman/account.ts +474 -0
- package/src/foreman/claude-worker-adapter.ts +412 -0
- package/src/foreman/codex-worker-adapter.ts +472 -0
- package/src/foreman/completion-report.ts +153 -0
- package/src/foreman/core/context.ts +169 -0
- package/src/foreman/core/defects.ts +280 -0
- package/src/foreman/core/exec.ts +20 -0
- package/src/foreman/core/gates.ts +493 -0
- package/src/foreman/core/handoff.ts +163 -0
- package/src/foreman/core/install.ts +109 -0
- package/src/foreman/core/learnings.ts +368 -0
- package/src/foreman/core/outbox-tracker.ts +192 -0
- package/src/foreman/core/pin.ts +226 -0
- package/src/foreman/core/plan-context.ts +238 -0
- package/src/foreman/core/process-table.ts +535 -0
- package/src/foreman/core/reconcile.ts +227 -0
- package/src/foreman/core/report.ts +60 -0
- package/src/foreman/core/run.ts +2836 -0
- package/src/foreman/core/scheduler.ts +244 -0
- package/src/foreman/core/summary.ts +166 -0
- package/src/foreman/core/text.ts +97 -0
- package/src/foreman/core/transcripts.ts +38 -0
- package/src/foreman/core/triage.ts +138 -0
- package/src/foreman/core/verifier.ts +800 -0
- package/src/foreman/core/views.ts +940 -0
- package/src/foreman/core/work-contract.ts +152 -0
- package/src/foreman/core/workspace.ts +335 -0
- package/src/foreman/fake-handoff.ts +33 -0
- package/src/foreman/fake-learnings.ts +26 -0
- package/src/foreman/fake-remote-api.ts +70 -0
- package/src/foreman/fake-tracker-adapter.ts +355 -0
- package/src/foreman/fake-worker-adapter.ts +221 -0
- package/src/foreman/host.ts +75 -0
- package/src/foreman/local-dir.ts +28 -0
- package/src/foreman/opsee-tracker-adapter.ts +612 -0
- package/src/foreman/process-group.ts +160 -0
- package/src/foreman/remote-api.ts +283 -0
- package/src/foreman/run-recipe.ts +274 -0
- package/src/foreman/service-unit.ts +257 -0
- package/src/foreman/tracker-adapter.ts +298 -0
- package/src/foreman/triage-draft.ts +40 -0
- package/src/foreman/vendor.ts +23 -0
- package/src/foreman/verdict.ts +120 -0
- package/src/foreman/worker-adapter.ts +177 -0
- package/src/foreman/worker-process.ts +488 -0
- package/src/identity.ts +49 -0
- package/src/index.ts +3 -0
- package/src/init/managed.ts +84 -0
- package/src/init/mcp-config.ts +77 -0
- package/src/init/paths.ts +16 -0
- package/src/init/pointer-block.ts +45 -0
- package/src/init/project.ts +22 -0
- package/src/init/prompt.ts +45 -0
- package/src/init/run-recipe-config.ts +133 -0
- package/src/init/skills.ts +38 -0
- package/src/init/text.ts +22 -0
- package/src/init/tracker-doc.ts +106 -0
- package/src/opsee-config.ts +116 -0
- package/templates/issue-tracker.md +162 -0
|
@@ -0,0 +1,2836 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Run loop (see ../../../CONTEXT.md: Run, Ready Issue, Workspace, Context Assembly, Completion
|
|
3
|
+
* Report, Status Label, Run Record, Hand-off). Each tick takes one Ready Task of the Initiative,
|
|
4
|
+
* creates its Workspace, installs dependencies once, assembles the Worker's context, runs one
|
|
5
|
+
* unattended turn pinned to the Workspace, hands a `done` turn off as a draft pull request
|
|
6
|
+
* (core/handoff.ts) linked to the Task, then writes the Completion Report as Initiative memory
|
|
7
|
+
* with a linking comment, moves the Task through the board with a Status Label, and appends
|
|
8
|
+
* dispatch and attempt events to the Run Record. It stops when no Ready Task remains, and then
|
|
9
|
+
* gathers the Run's Proposed Learnings into one draft pull request against the repository's
|
|
10
|
+
* learnings file (core/learnings.ts, story 45).
|
|
11
|
+
*
|
|
12
|
+
* Gates (gates.ts, OPS-271) sit between the Hand-off and the settlement: once the draft pull request
|
|
13
|
+
* is open, the base's Run Recipe test, lint and typecheck commands run in the Workspace and are
|
|
14
|
+
* judged by exit code alone. A pass settles the Task In review as before; a failure resumes the same
|
|
15
|
+
* Worker session with the failing command and its output tail, and once its next `done` turn has
|
|
16
|
+
* updated the Hand-off the Gates run again, up to the retry cap, past which the Task is blocked
|
|
17
|
+
* with a comment naming the Gate. Every Gate result reaches the Run Record as a `gate` event in the
|
|
18
|
+
* orchestrator Ledger's Verification shape before anything else moves, and the rounds are
|
|
19
|
+
* summarised as a comment on the pull request, whatever the verdict; Reconcile reads those events
|
|
20
|
+
* back to finish a settlement, so a Hand-off the Gates held is never settled as handed off.
|
|
21
|
+
*
|
|
22
|
+
* The Verifier (verifier.ts, OPS-272) follows the Gates: once they pass (or the recipe names none),
|
|
23
|
+
* a Task whose Verification section describes a user journey gets the app started from the base's
|
|
24
|
+
* Run Recipe on a leased port and a separate Verifier Worker with Playwright MCP, whose prompt is
|
|
25
|
+
* that section and the URL and nothing of the implementer's turn. Its Verdict goes to the Run
|
|
26
|
+
* Record as a `verdict` event (the Ledger's shape, screenshots uploaded to the pull request first
|
|
27
|
+
* so the Defects carry their evidence URL), then to the pull request as a comment; a passed
|
|
28
|
+
* Verdict settles the Task In review, Defects leave it In Progress under `foreman:blocked` for the
|
|
29
|
+
* Defect filer (OPS-273). A Task with commands only in its Verification section skips the browser
|
|
30
|
+
* and is verified at its Gates.
|
|
31
|
+
*
|
|
32
|
+
* Scheduling (core/scheduler.ts, OPS-276): each tick fills the Account's free Slots rather than
|
|
33
|
+
* dispatching one Task and waiting. The Account's `cap` is how many Slots it has, one of them
|
|
34
|
+
* reserved for Verifiers, so `cap` 3 is two implementers and a Verifier round beside them; a
|
|
35
|
+
* dispatch swaps its implementer Slot for the reserved one when it reaches verification, which is
|
|
36
|
+
* what keeps a fleet of implementers from starving it. Ready Tasks are taken in the Tracker's
|
|
37
|
+
* readiness set but the Foreman's order: priority descending, then oldest first. `--once` and
|
|
38
|
+
* `--task` still mean exactly one dispatch. A dispatch that throws is caught, left blocked as
|
|
39
|
+
* before, and reported in the Run's outcome; the other Slots go on.
|
|
40
|
+
*
|
|
41
|
+
* `runForeman` is the seam the daemon wraps: it takes the adapters and an Account and
|
|
42
|
+
* knows nothing about daemons. Reconcile is here (story 51): with a Process Table in the deps, each
|
|
43
|
+
* tick first Reconciles (reconcile.ts) and drains the outbox, every live Worker is recorded in the
|
|
44
|
+
* table (its session id the moment `started` arrives, its last output as it streams), and a turn
|
|
45
|
+
* that the stored session id can continue is resumed through the same settle path as a fresh one.
|
|
46
|
+
* Transcripts go to the local store; the Run Record gets an output tail (story 49).
|
|
47
|
+
*
|
|
48
|
+
* Readiness is the Tracker's (`listReadyTasks`, ADR-0008); the loop never walks blockers itself.
|
|
49
|
+
* Run Record writes come before the board moves they describe (ADR-0009: that write is the commit).
|
|
50
|
+
*
|
|
51
|
+
* Before the dispatch event the Task's work contract is checked (work-contract.ts, story 23): a Task
|
|
52
|
+
* missing Goal or Acceptance Criteria is blocked and loses its dispatch label; one missing only
|
|
53
|
+
* Verification gets a Triage turn (triage.ts) whose draft is posted as a comment and joins the
|
|
54
|
+
* Worker's context, or reuses a draft an earlier turn left; a complete one dispatches as is.
|
|
55
|
+
*
|
|
56
|
+
* The human steers through the Process Table (stories 53-56; commands/foreman-control.ts): a
|
|
57
|
+
* paused Initiative stops the loop from picking a Task, and the Run either returns with `paused`
|
|
58
|
+
* set (the daemon then puts the request back and serves others) or, asked to wait, idles at the
|
|
59
|
+
* daemon's tick until resumed or stopped; a
|
|
60
|
+
* turn whose row was marked `attached` when its stream ended is left open (no attempt event, the
|
|
61
|
+
* row stays) for the human's attended turn, and Reconcile resumes it unattended once `released`;
|
|
62
|
+
* one marked `cancelled` is closed as a `stopped` attempt whose summary starts with the cancelled
|
|
63
|
+
* marker, the Task blocked with a comment. The attempt outcome vocabulary is the Run Record's
|
|
64
|
+
* (RunAttemptEvent.outcome), which has no cancelled value; the `control` event beside it says who
|
|
65
|
+
* stopped the Worker and why.
|
|
66
|
+
*/
|
|
67
|
+
import { DEFECT_LABEL } from "@opsee/mcp-server/src/utils/defect.js";
|
|
68
|
+
import { timestampFromMs } from "@bufbuild/protobuf/wkt";
|
|
69
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
70
|
+
import { basename, join } from "node:path";
|
|
71
|
+
import { create, type MessageInitShape } from "@bufbuild/protobuf";
|
|
72
|
+
import { RunEventInputSchema, type RunEventInput } from "@opsee/mcp-server/gen/api/v1/initiative_pb.js";
|
|
73
|
+
import {
|
|
74
|
+
clearCredentialFailures,
|
|
75
|
+
clearResumed,
|
|
76
|
+
CREDENTIAL_FAILURES_BEFORE_QUARANTINE,
|
|
77
|
+
DEFAULT_PAUSE_MS,
|
|
78
|
+
isPaused,
|
|
79
|
+
isQuarantined,
|
|
80
|
+
MIN_PAUSE_MS,
|
|
81
|
+
noteCredentialFailure,
|
|
82
|
+
pauseAccount,
|
|
83
|
+
pausedUntil,
|
|
84
|
+
pauseUntilFrom,
|
|
85
|
+
quarantineAccount,
|
|
86
|
+
type Account,
|
|
87
|
+
} from "../account.js";
|
|
88
|
+
import type { AccountStore } from "../account-store.js";
|
|
89
|
+
import type { CompletionReport } from "../completion-report.js";
|
|
90
|
+
import { blockerIdsOf, DISPATCH_LABEL, type TaskWithContext, type TrackerAdapter, type TrackerTask } from "../tracker-adapter.js";
|
|
91
|
+
import type { Vendor } from "../vendor.js";
|
|
92
|
+
import type { Verdict } from "../verdict.js";
|
|
93
|
+
import type { AdapterEvent, TurnFailureReason, TurnHandle, WorkerAdapter } from "../worker-adapter.js";
|
|
94
|
+
import { triageDraftFrom } from "../triage-draft.js";
|
|
95
|
+
import type { RemoteApi } from "../remote-api.js";
|
|
96
|
+
import { assembleContext, SIBLING_LEARNING_LIMIT, type AcceptedLearnings, type BlockerReports, type SiblingLearning } from "./context.js";
|
|
97
|
+
import { isRoundFailure, type DefectEvidenceLink, type DefectsFn, type FiledDefect } from "./defects.js";
|
|
98
|
+
import { DEFAULT_GATE_RETRIES, gateComment, gateEvent, gateFixPrompt, gateLine, gatePassed, gateSummary, gateVerdictOf, type GateOutcome, type GateReason, type GateRound, type GatesFn, truncateCodePoints } from "./gates.js";
|
|
99
|
+
import type { HandOffFn, HandOffResult } from "./handoff.js";
|
|
100
|
+
import { inferInstallCommand, runInstall, tail, type InstallResult } from "./install.js";
|
|
101
|
+
import { collectProposedLearnings, DEFAULT_LEARNINGS_FILE, type LearningsFn, type LearningsResult } from "./learnings.js";
|
|
102
|
+
import { describePin, pinAllows, pinOf, pinToken, unplaceable, type Pin, type Unplaceable } from "./pin.js";
|
|
103
|
+
import { otherForemanWorkersOn, type DrainResult, type ProcessTableApi, type WorkerRow } from "./process-table.js";
|
|
104
|
+
import { reconcile } from "./reconcile.js";
|
|
105
|
+
import { failureComment, reportComment, reportToMemory } from "./report.js";
|
|
106
|
+
import { orderReadyTasks, priorityScale, Slots, type SlotHold } from "./scheduler.js";
|
|
107
|
+
import { runSummaryComment } from "./summary.js";
|
|
108
|
+
import { count, printableOneLine } from "./text.js";
|
|
109
|
+
import { driveTriageTurn, isTriageSummary, TRIAGE_MARKER } from "./triage.js";
|
|
110
|
+
import { failureVerdict, verdictComment, verdictEvent, verdictOf, verdictStatusOf, verificationPassed, verifySummary, type DefectEvidence, type VerifierFn, type VerifyOutcome } from "./verifier.js";
|
|
111
|
+
import {
|
|
112
|
+
blockedComment,
|
|
113
|
+
draftedVerificationFrom,
|
|
114
|
+
parseWorkContract,
|
|
115
|
+
triageComment,
|
|
116
|
+
triageCappedComment,
|
|
117
|
+
triageFailedComment,
|
|
118
|
+
validateWorkContract,
|
|
119
|
+
type WorkContract,
|
|
120
|
+
type WorkContractSection,
|
|
121
|
+
} from "./work-contract.js";
|
|
122
|
+
import type { TranscriptStore, TranscriptWriter } from "./transcripts.js";
|
|
123
|
+
import type { Workspace, WorkspaceManager } from "./workspace.js";
|
|
124
|
+
|
|
125
|
+
/** A refusal the Foreman makes on purpose, reported as a plain message. */
|
|
126
|
+
export class ForemanError extends Error {
|
|
127
|
+
constructor(message: string) {
|
|
128
|
+
super(message);
|
|
129
|
+
this.name = "ForemanError";
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** The Status Labels (story 63) the Foreman keeps on the Tasks it touches; created when missing. */
|
|
134
|
+
export const STATUS_LABELS = {
|
|
135
|
+
running: "foreman:running",
|
|
136
|
+
/** Worn *alongside* `running` for the Verifier's browser round, not instead of it. The round is
|
|
137
|
+
* the longest and most failure-prone phase and the board could not tell it from implementation.
|
|
138
|
+
* Additive because Reconcile finds a dead Foreman's orphans by `running` alone (core/reconcile.ts,
|
|
139
|
+
* RUNNING_LABEL): swapping the label out would hide a Task that died mid-verify from the one
|
|
140
|
+
* thing built to recover it. Cleared when the attempt settles. */
|
|
141
|
+
verifying: "foreman:verifying",
|
|
142
|
+
blocked: "foreman:blocked",
|
|
143
|
+
done: "foreman:done",
|
|
144
|
+
} as const;
|
|
145
|
+
type StatusLabel = (typeof STATUS_LABELS)[keyof typeof STATUS_LABELS];
|
|
146
|
+
|
|
147
|
+
/** What the Run Record calls each vendor (RunDispatchEvent.vendor). */
|
|
148
|
+
export const RUN_RECORD_VENDOR: Readonly<Record<Vendor, string>> = { claude: "claude-code", codex: "codex" };
|
|
149
|
+
|
|
150
|
+
/** RunAttemptEvent.outcome values. */
|
|
151
|
+
export type AttemptOutcome = "completed" | "failed" | "stalled" | "rate_limited" | "stopped";
|
|
152
|
+
|
|
153
|
+
/** RunControlEvent.action values: what a human did from the CLI. */
|
|
154
|
+
export type ControlAction = "attach" | "release" | "pause" | "resume" | "cancel";
|
|
155
|
+
|
|
156
|
+
/** What a cancelled attempt's summary starts with: the outcome is `stopped` (the Run Record's
|
|
157
|
+
* vocabulary), the marker says the human stopped it on purpose. */
|
|
158
|
+
export const CANCELLED_MARKER = "cancelled:";
|
|
159
|
+
|
|
160
|
+
/** How often a foreground Run that waits through a pause looks for its resume: the daemon's idle
|
|
161
|
+
* tick (foreman-up.ts takes its DEFAULT_DAEMON_TICK_MS from here), since each look reads the Run
|
|
162
|
+
* Record for every row the Initiative has. */
|
|
163
|
+
export const PAUSE_POLL_MS = 5_000;
|
|
164
|
+
|
|
165
|
+
/** The shortest that poll may be set to. `pausePollMs` reaches a `sleep` inside the loop, so 0 is
|
|
166
|
+
* not "check often", it is a spin: the Run would re-read the Run Record for every row of the
|
|
167
|
+
* Initiative thousands of times a second while an Account waits out an hour. */
|
|
168
|
+
export const MIN_PAUSE_POLL_MS = 100;
|
|
169
|
+
|
|
170
|
+
/** How long a dependency install may run before it is killed and recorded as such; a wedged
|
|
171
|
+
* install must not hold the Run where no Worker stall timer can reach it. */
|
|
172
|
+
export const INSTALL_TIMEOUT_MS = 15 * 60_000;
|
|
173
|
+
|
|
174
|
+
/** How many further attempts a Task gets after a turn ended without reporting for a reason that is
|
|
175
|
+
* the machine's rather than the work's (`RETRYABLE_FAILURES`). Under the cap the Task is made Ready
|
|
176
|
+
* again and the next tick dispatches it afresh — on another Account of the same vendor, when its
|
|
177
|
+
* own is Paused by then (story 31); at the cap it is settled blocked with a comment naming the
|
|
178
|
+
* cause that used the attempts up (story 34, `Settlement.capped`), and the Run goes on with the
|
|
179
|
+
* other Tasks.
|
|
180
|
+
*
|
|
181
|
+
* The attempts are spaced by `retryBackoff` rather than taken as fast as the machine can fail. */
|
|
182
|
+
export const DEFAULT_ATTEMPT_RETRIES = 2;
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* How long the first retry of a Task waits, with each further one waiting twice as long
|
|
186
|
+
* (`retryBackoff`).
|
|
187
|
+
*
|
|
188
|
+
* Without a delay the retry cap bounds the number of attempts and nothing bounds their speed. An
|
|
189
|
+
* adapter that fails in milliseconds — a vendor that answers at once, a launch that cannot start —
|
|
190
|
+
* takes a Task through every attempt it has and gives the Slot back before a human sees a line of
|
|
191
|
+
* log, and every free Slot does it at the same time. The Paused Account was the only clock of that
|
|
192
|
+
* kind, and it only covers the vendor saying "not yet" out loud.
|
|
193
|
+
*
|
|
194
|
+
* Thirty seconds is chosen against what the delay is for: long enough that a transient condition
|
|
195
|
+
* (a vendor's bad minute, a machine still releasing a port) has time to clear, short enough to be
|
|
196
|
+
* invisible on the overnight Run this exists to protect. At the default cap a Task waits 30s and
|
|
197
|
+
* then 60s across its whole life.
|
|
198
|
+
*/
|
|
199
|
+
export const DEFAULT_RETRY_BACKOFF_MS = 30_000;
|
|
200
|
+
|
|
201
|
+
/** The longest a retry waits, however many attempts a Task has been given. Doubling is only useful
|
|
202
|
+
* while the wait is short: past a few minutes a Task that could have run is simply idle, and the
|
|
203
|
+
* Run has other Tasks it should be spending the night on. Reached at the fifth retry, which only a
|
|
204
|
+
* raised `attemptRetries` gets to. */
|
|
205
|
+
export const MAX_RETRY_BACKOFF_MS = 5 * 60_000;
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* How long a Task waits before its next attempt, given how many retryable failures it has had
|
|
209
|
+
* (`retryableFailuresOf`, this one included). Doubles per failure from `base`, capped.
|
|
210
|
+
*
|
|
211
|
+
* `spent` is the count the retry cap itself is measured in, so the backoff and the cap always agree
|
|
212
|
+
* about which attempt this is; a count of zero or less is not a retry and waits not at all.
|
|
213
|
+
*/
|
|
214
|
+
/** " in 30s" for a wait worth mentioning, and nothing at all for one that is not. A Run configured
|
|
215
|
+
* with no backoff must not tell a human its Task comes back "in 0s", which reads as a bug in the
|
|
216
|
+
* sentence rather than as the absence of a delay. */
|
|
217
|
+
function inSeconds(ms: number): string {
|
|
218
|
+
return ms > 0 ? ` in ${Math.round(ms / 1000)}s` : "";
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export function retryBackoff(spent: number, base: number = DEFAULT_RETRY_BACKOFF_MS): number {
|
|
222
|
+
if (spent < 1) return 0;
|
|
223
|
+
// Shifting rather than `Math.pow` would overflow at 31 doublings; the cap is applied to a number
|
|
224
|
+
// that may be Infinity, which `Math.min` handles, so a huge `spent` is the cap and not a NaN.
|
|
225
|
+
return Math.min(base * 2 ** (spent - 1), MAX_RETRY_BACKOFF_MS);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** How many Defect Tasks one Run may dispatch before the Breaker stops it starting anything new.
|
|
229
|
+
*
|
|
230
|
+
* A Defect is filed as a sibling Task carrying the dispatch label, so the same Run picks it up; its
|
|
231
|
+
* own Verdict can file more, and nothing counted. One bad plan could spend a whole night on
|
|
232
|
+
* defect, fix, defect, and the morning summary would show a Run that never stopped and never
|
|
233
|
+
* finished. This is the count that ends it.
|
|
234
|
+
*
|
|
235
|
+
* The count is this Run's own and in memory, so a Foreman that crashes and is restarted begins a
|
|
236
|
+
* fresh Run with a fresh count. That is the honest scope of the guard: it bounds a night, not a
|
|
237
|
+
* week, and a crash-looping Foreman is a different failure with a different fix. */
|
|
238
|
+
export const DEFAULT_DEFECT_DISPATCH_CAP = 10;
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* The turn failures another attempt is worth trying, because they are the machine's or the
|
|
242
|
+
* vendor's rather than the work's: the Worker went quiet and the adapter stopped it (story 33), or
|
|
243
|
+
* the vendor refused the turn on a rate limit (story 30) — neither of which says anything about
|
|
244
|
+
* whether the Task can be done.
|
|
245
|
+
*
|
|
246
|
+
* A rate limit is here for the same reason Failover exists: with the Account Paused, the next tick
|
|
247
|
+
* puts the Task on a sibling Account of the same vendor, or on the same one once the reset has
|
|
248
|
+
* passed, and nothing is lost. The rest are deliberately not retried — `invalid_report` and
|
|
249
|
+
* `max_turns` are turns that really ran and would only run the same way again, `launch_failed` and
|
|
250
|
+
* `vendor_error` are a machine or a configuration a human must look at, and `stopped` is a human's
|
|
251
|
+
* own doing.
|
|
252
|
+
*/
|
|
253
|
+
export const RETRYABLE_FAILURES: ReadonlySet<AttemptOutcome> = new Set<AttemptOutcome>(["stalled", "rate_limited"]);
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* The same question asked of the turn's reason rather than of the outcome it was recorded under
|
|
257
|
+
* (OPS-288).
|
|
258
|
+
*
|
|
259
|
+
* `credential_failed` has to be retried for the reason a rate limit does — with its Account
|
|
260
|
+
* quarantined, the next tick puts the Task on a sibling and nothing is lost, where blocking it
|
|
261
|
+
* outright would strand work that a healthy Account could do. But it cannot join
|
|
262
|
+
* `RETRYABLE_FAILURES`: that set is keyed by `AttemptOutcome`, a credential failure is recorded as
|
|
263
|
+
* the generic `failed` (the proto's outcome list is closed), and adding `failed` there would make
|
|
264
|
+
* every `vendor_error` and `invalid_report` retryable too.
|
|
265
|
+
*
|
|
266
|
+
* So the live decision reads the reason, and the Run Record read-back reads the outcome plus
|
|
267
|
+
* `CREDENTIAL_MARKER` in the summary, which is how a past credential failure is still counted
|
|
268
|
+
* against the retry cap. The alternative — a new outcome in the proto — is four generated trees for
|
|
269
|
+
* a fact the summary already carries, and the marker precedent is `TRIAGE_MARKER`'s.
|
|
270
|
+
*/
|
|
271
|
+
export const RETRYABLE_FAILURE_REASONS: ReadonlySet<TurnFailureReason> = new Set<TurnFailureReason>(["stalled", "rate_limited", "credential_failed"]);
|
|
272
|
+
|
|
273
|
+
/** What an attempt event's summary starts with when the turn failed for a credential reason, so
|
|
274
|
+
* `retryableFailuresOf` can count it back off the Run Record despite its generic `failed` outcome. */
|
|
275
|
+
export const CREDENTIAL_MARKER = "[credential]";
|
|
276
|
+
|
|
277
|
+
export interface RunDeps {
|
|
278
|
+
tracker: TrackerAdapter;
|
|
279
|
+
worker: WorkerAdapter;
|
|
280
|
+
account: Account;
|
|
281
|
+
/** Sibling Accounts of the same vendor a not-yet-started Ready Task may Fail over to while this
|
|
282
|
+
* Run's own Account is Paused (story 31, CONTEXT.md: Failover). Each carries its own Worker
|
|
283
|
+
* Adapter and the per-Account steps built from it, and gets its own Slots inside the Run, so the
|
|
284
|
+
* concurrency one vendor identity sees is still its own cap and no more (ADR-0013). A live
|
|
285
|
+
* session never changes Account: a Task with a Process Table row runs on that row's Account
|
|
286
|
+
* alone. Its Workspace is not pinned with it — the row goes when the attempt is settled, so a
|
|
287
|
+
* retry after a machine-side failure takes whichever Lane is open and reuses the same git
|
|
288
|
+
* worktree; nothing of the old Account travels with it but the commits. */
|
|
289
|
+
failover?: RunAccount[];
|
|
290
|
+
/** Accounts of *another* vendor that only a pinned Task may use (story 24, core/pin.ts). A Run on
|
|
291
|
+
* a Claude Account with a Codex Account registered beside it carries that Codex Account here: a
|
|
292
|
+
* Task labelled `foreman:vendor:codex` (or `foreman:account:<its name>`) runs there, and nothing
|
|
293
|
+
* else ever does. They are Lanes like any other — their own Slots from their own cap, checked
|
|
294
|
+
* against the Process Table each tick — so one vendor identity still runs its own cap and no more
|
|
295
|
+
* (ADR-0013); what they are not is Failover targets, which stays within a vendor because a
|
|
296
|
+
* session, a prompt contract and an adapter do not cross one. `pinnableAccountsFor`
|
|
297
|
+
* (commands/foreman.ts) builds them. */
|
|
298
|
+
pinnable?: RunAccount[];
|
|
299
|
+
/** Where an Account's Paused state is written when a vendor reports a rate limit, and read back
|
|
300
|
+
* each time Slots are filled (story 30). It is the store rather than a callback because the fact
|
|
301
|
+
* outlives the Run: a pause this Foreman hit is one the next Foreman, and `foreman account list`,
|
|
302
|
+
* must see. Without one a rate limit is logged and no Account is Paused. */
|
|
303
|
+
accounts?: AccountStore;
|
|
304
|
+
/** How long an Account is Paused when the vendor named no reset; `DEFAULT_PAUSE_MS` when unset,
|
|
305
|
+
* and never below `MIN_PAUSE_MS`, since a window of zero is no pause at all. */
|
|
306
|
+
pauseMs?: number;
|
|
307
|
+
workspaces: Pick<WorkspaceManager, "create">;
|
|
308
|
+
/** Installs dependencies in a new Workspace; resolves undefined when there is nothing to install.
|
|
309
|
+
* Defaults to the Run Recipe's `commands.install`, else the lockfile inference in install.ts. */
|
|
310
|
+
install?: (workspacePath: string) => Promise<InstallResult | undefined>;
|
|
311
|
+
/** The Hand-off after a `done` turn: pushes the Workspace branch and opens or updates the draft
|
|
312
|
+
* pull request (core/handoff.ts). Its result decides whether the Task goes to In review. Absent,
|
|
313
|
+
* every `done` turn is an incomplete attempt that says so; `foreman run` always wires one. */
|
|
314
|
+
handOff?: HandOffFn;
|
|
315
|
+
/** The Gates after a Hand-off (gates.ts): runs the base's Run Recipe commands in the Workspace
|
|
316
|
+
* and reports each by exit code. `foreman run` always wires the real ones (`gatesWith`, which
|
|
317
|
+
* needs the guarded git and the checkout); absent, the Gates are skipped and the Hand-off stands. */
|
|
318
|
+
gates?: GatesFn;
|
|
319
|
+
/** How many times a failing Gate goes back to the same Worker before the Task is blocked;
|
|
320
|
+
* `DEFAULT_GATE_RETRIES` when unset. */
|
|
321
|
+
gateRetries?: number;
|
|
322
|
+
/** The code host of the Hand-off, for the Gates' summary and the Verdict on the pull request
|
|
323
|
+
* and the Verifier's screenshots. Absent, the summary is logged and the Run Record still has
|
|
324
|
+
* every result. */
|
|
325
|
+
remote?: Pick<RemoteApi, "addPullRequestComment" | "uploadAttachment">;
|
|
326
|
+
/** The Verifier after the Gates (verifier.ts): starts the app from the base's Run Recipe and
|
|
327
|
+
* runs the Verifier Worker on the Task's Verification section. `foreman run` always wires the
|
|
328
|
+
* real one (`verifierWith`); absent, verification is skipped and the Gates are the whole of it. */
|
|
329
|
+
verifier?: VerifierFn;
|
|
330
|
+
/** The Defect filer after a failed Verdict (defects.ts): one sibling Bug Task per observed
|
|
331
|
+
* Defect, related to the Task the Verifier found it on, so the next tick dispatches the fix.
|
|
332
|
+
* `foreman run` always wires the real one (`defectFilerWith`); absent, the Verdict is recorded
|
|
333
|
+
* and nothing is filed, which is what the loop's own tests want. */
|
|
334
|
+
defects?: DefectsFn;
|
|
335
|
+
/** The learnings step at the end of the Run: every Proposed Learning of its Completion Reports
|
|
336
|
+
* into one draft pull request against the learnings file (core/learnings.ts). Absent, the
|
|
337
|
+
* learnings stay in Initiative memory only; `foreman run` always wires one. */
|
|
338
|
+
learnings?: LearningsFn;
|
|
339
|
+
/** The repository's learnings file, relative to its root, read from each Workspace for Context
|
|
340
|
+
* Assembly as the Accepted Learnings; `docs/learnings.md` unless the Run Recipe says otherwise. */
|
|
341
|
+
learningsFile?: string;
|
|
342
|
+
/** The daemon log: every decision the loop makes goes here. */
|
|
343
|
+
log: (line: string) => void;
|
|
344
|
+
maxTurns?: number;
|
|
345
|
+
stallTimeoutMs?: number;
|
|
346
|
+
/** How many further attempts a stalled Task gets before it is settled blocked;
|
|
347
|
+
* `DEFAULT_ATTEMPT_RETRIES` when unset. */
|
|
348
|
+
attemptRetries?: number;
|
|
349
|
+
/** How long the first retry of a Task waits, doubling per further retry (`retryBackoff`);
|
|
350
|
+
* `DEFAULT_RETRY_BACKOFF_MS` when unset. */
|
|
351
|
+
retryBackoffMs?: number;
|
|
352
|
+
/** How many Defect Tasks this Run may dispatch before the Breaker halts new dispatch;
|
|
353
|
+
* `DEFAULT_DEFECT_DISPATCH_CAP` when unset. */
|
|
354
|
+
defectDispatchCap?: number;
|
|
355
|
+
/** Pause between ticks, so a Tracker that just released a successor is not hammered; 0 in tests. */
|
|
356
|
+
tickDelayMs?: number;
|
|
357
|
+
now?: () => number;
|
|
358
|
+
sleep?: (ms: number) => Promise<void>;
|
|
359
|
+
/** The Process Table (ADR-0009). Without one the loop keeps no local record and never Reconciles. */
|
|
360
|
+
processTable?: ProcessTableApi;
|
|
361
|
+
/** Where each turn's raw event stream is written; unset keeps no transcript. */
|
|
362
|
+
transcripts?: TranscriptStore;
|
|
363
|
+
/** Sends the outbox's queued Run Record batches (OutboxTracker.drain); called at the top of each tick. */
|
|
364
|
+
drainOutbox?: (initiativeId: number) => Promise<DrainResult>;
|
|
365
|
+
/** Whether a Process Table row's pid is a live process; the real check by default. */
|
|
366
|
+
isAlive?: (pid: number | undefined) => boolean;
|
|
367
|
+
/** How often a Run that waits through a pause (`RunOptions.waitWhilePaused`) checks for its
|
|
368
|
+
* resume; PAUSE_POLL_MS by default. */
|
|
369
|
+
pausePollMs?: number;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** How often at most an `output` event bumps the row's last_output_at. */
|
|
373
|
+
export const TOUCH_INTERVAL_MS = 1000;
|
|
374
|
+
|
|
375
|
+
export interface RunOptions {
|
|
376
|
+
initiativeId: number;
|
|
377
|
+
/** Stop after one dispatch. */
|
|
378
|
+
once?: boolean;
|
|
379
|
+
/** Dispatch this Task and stop; it must be a Ready Task of the Initiative. */
|
|
380
|
+
taskId?: number;
|
|
381
|
+
/** Asked before each pick: true ends the Run after the dispatch in flight (the daemon was told
|
|
382
|
+
* to stop). The turn itself is stopped by the process hooks; this keeps the loop from starting
|
|
383
|
+
* another. */
|
|
384
|
+
shouldStop?: () => boolean;
|
|
385
|
+
/** Resolves when the Run should stop, for the waits a poll cannot reach: a Reconcile turn that
|
|
386
|
+
* needs an implementer Slot waits on this beside the Slot, and abandons the wait rather than
|
|
387
|
+
* holding the loop — and with it the stop and the pause — until a Worker turn ends. The daemon
|
|
388
|
+
* passes the same promise its `shouldStop` reads. */
|
|
389
|
+
untilStop?: Promise<void>;
|
|
390
|
+
/** Idle through a pause of the Initiative (story 55), polling for the resume, instead of returning
|
|
391
|
+
* with `paused` set; a foreground `foreman run` does, the daemon does not. */
|
|
392
|
+
waitWhilePaused?: boolean;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/** A Triage turn the Foreman ran on a Task before dispatching it. On the Run Record as an `attempt`
|
|
396
|
+
* event whose summary starts `triage:`, so it counts as a turn on the Account like any other. */
|
|
397
|
+
export interface TriageTurn {
|
|
398
|
+
attempt: number;
|
|
399
|
+
outcome: AttemptOutcome;
|
|
400
|
+
/** The drafted Verification section, when the turn produced one. */
|
|
401
|
+
verification?: string;
|
|
402
|
+
/** Why it produced none: the turn's failure, or what the Worker reported instead. */
|
|
403
|
+
reason?: string;
|
|
404
|
+
sessionId?: string;
|
|
405
|
+
durationMs: number;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
export interface Dispatch {
|
|
409
|
+
task: TrackerTask;
|
|
410
|
+
workspace: Workspace;
|
|
411
|
+
attempt: number;
|
|
412
|
+
outcome: AttemptOutcome;
|
|
413
|
+
report?: CompletionReport;
|
|
414
|
+
sessionId?: string;
|
|
415
|
+
durationMs: number;
|
|
416
|
+
install?: InstallResult;
|
|
417
|
+
/** Present for a `done` report: the draft pull request, or why there is none. */
|
|
418
|
+
handOff?: HandOffResult;
|
|
419
|
+
/** The Gates that ran on the Hand-off, round by round; absent when the recipe names none. */
|
|
420
|
+
gates?: GateOutcome;
|
|
421
|
+
/** The Verifier's round after the Gates passed: skipped, a Verdict, or a failed round; absent
|
|
422
|
+
* when the Gates held the Hand-off or there was none. */
|
|
423
|
+
verification?: VerifyOutcome;
|
|
424
|
+
/** The Triage turn that ran before this dispatch, when the Task had no Verification section. */
|
|
425
|
+
triage?: TriageTurn;
|
|
426
|
+
/** The Verification section the Worker was given because the Task had none: this dispatch's
|
|
427
|
+
* Triage turn's, or one an earlier turn left in the comments. */
|
|
428
|
+
draftedVerification?: string;
|
|
429
|
+
/** True when Reconcile continued this attempt from its stored session id (story 51). */
|
|
430
|
+
resumed?: boolean;
|
|
431
|
+
/** True when the turn ended because a human attached (story 53): nothing was recorded, the
|
|
432
|
+
* attempt is still open and its row stays until the human releases it. */
|
|
433
|
+
attached?: boolean;
|
|
434
|
+
/** True when the attempt was closed by `foreman cancel` (story 56). */
|
|
435
|
+
cancelled?: boolean;
|
|
436
|
+
/** True when the attempt ended stalled or rate limited under the retry cap: the Task was made
|
|
437
|
+
* Ready again instead of blocked, and a later tick dispatches it as the next attempt (story 33). */
|
|
438
|
+
retrying?: boolean;
|
|
439
|
+
/** True when that retry cap is what blocked the Task (story 34): the attempt failed for a reason
|
|
440
|
+
* another attempt would have been worth, and there were none left. */
|
|
441
|
+
blockedByCap?: boolean;
|
|
442
|
+
/** When the Task may be dispatched again, for a `retrying` dispatch (`retryBackoff`). Absolute
|
|
443
|
+
* rather than a duration, so the Run compares it against the same clock it schedules on.
|
|
444
|
+
*
|
|
445
|
+
* Decided here and held by the Run: this is where the facts are (which attempt, how many
|
|
446
|
+
* retryable failures the Run Record counts), while the waiting is the tick's business. Carrying
|
|
447
|
+
* it on the Dispatch keeps the backoff out of `RunDeps` — the settlement path has no mutable Run
|
|
448
|
+
* state to write to, and giving it some would be a seam every future turn could reach through. */
|
|
449
|
+
retryNotBefore?: number;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** A dispatch whose work reached review: the turn reported `done`, the Hand-off is open, the
|
|
453
|
+
* Gates passed (or there were none to run), and the Verifier passed it or had no browser journey
|
|
454
|
+
* to run. */
|
|
455
|
+
export function isHandedOff(d: Dispatch): boolean {
|
|
456
|
+
return d.outcome === "completed" && d.report?.outcome === "done" && d.handOff?.kind === "handed_off" && (d.gates?.passed ?? true) && verificationPassed(d.verification);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/** A Ready Task the Foreman refused to dispatch: its work contract is incomplete, and the Triage
|
|
460
|
+
* turn, when one applied, drafted nothing. It lost its dispatch label and carries `foreman:blocked`. */
|
|
461
|
+
export interface BlockedTask {
|
|
462
|
+
task: TrackerTask;
|
|
463
|
+
missing: WorkContractSection[];
|
|
464
|
+
triage?: TriageTurn;
|
|
465
|
+
/** The Workspace created for the Triage turn, when one ran; kept, and reused by a later dispatch. */
|
|
466
|
+
workspace?: Workspace;
|
|
467
|
+
/** The Task was left Ready because the Triage turn failed for a reason that is not the contract's
|
|
468
|
+
* — a stall, a rate limit — and the retry cap still allows another go. A later tick of *this* Run
|
|
469
|
+
* picks it up again (on a sibling Account when its own is Paused by then); it is not deferred to
|
|
470
|
+
* the next Run, which would cost a Task the whole night for one bad minute at 3am. `attemptsLeft`
|
|
471
|
+
* is what the retry cap has left, counted the same way a failed dispatch counts it
|
|
472
|
+
* (`retryableFailuresOf`), and `notBefore` is when that later tick may take it (`retryBackoff`):
|
|
473
|
+
* a Triage turn that fails in milliseconds is the same hot loop a dispatch's is. */
|
|
474
|
+
retry?: { attemptsLeft: number; notBefore: number };
|
|
475
|
+
/** The Task keeps its dispatch label but this Run will not look at it again: the Triage turn
|
|
476
|
+
* failed for something no tick of this Run can change (an adapter that will not start, a vendor
|
|
477
|
+
* error), so trying again in a second would only spin. The next Run picks it up. */
|
|
478
|
+
defer?: boolean;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/** A Slot whose dispatch threw something that is not a turn outcome: a Tracker write refused, a
|
|
482
|
+
* board without the column, an adapter that would not start. `dispatch` has already commented on
|
|
483
|
+
* the Task and marked it blocked; the Run reports it here and fills the Slot with the next Task
|
|
484
|
+
* rather than dying, so one bad Task does not end the Run (OPS-276). */
|
|
485
|
+
export interface DispatchFailure {
|
|
486
|
+
task: TrackerTask;
|
|
487
|
+
reason: string;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* A Ready Task this Run left alone because its label pins it somewhere this Run cannot reach: an
|
|
492
|
+
* Account it has no Lane for, a vendor it has no Account of, or a `foreman:` label it cannot read
|
|
493
|
+
* as a pin at all (story 24, core/pin.ts).
|
|
494
|
+
*
|
|
495
|
+
* It is deliberately *not* a `BlockedTask`. Nothing is wrong with the Task: it keeps its dispatch
|
|
496
|
+
* label, stays Ready, is never marked `foreman:blocked`, and spends none of its retry cap — the
|
|
497
|
+
* Foreman simply has nowhere to put it. Registering the Account (or fixing the label) is the whole
|
|
498
|
+
* of the fix, and the next Run dispatches it with nothing to undo. It is skipped for the rest of
|
|
499
|
+
* *this* Run, because no tick of it can change the answer.
|
|
500
|
+
*/
|
|
501
|
+
export interface QueuedTask {
|
|
502
|
+
task: TrackerTask;
|
|
503
|
+
pin: Pin;
|
|
504
|
+
/** What the comment on the Task said, in one sentence. */
|
|
505
|
+
reason: string;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
export interface RunOutcome {
|
|
509
|
+
/** In the order they settled, which with Slots filled in parallel is not the order they were
|
|
510
|
+
* dispatched in: the Run Record's `dispatch` events are what carry that. */
|
|
511
|
+
dispatches: Dispatch[];
|
|
512
|
+
blocked: BlockedTask[];
|
|
513
|
+
/** Ready Tasks left queued because their pin named an Account this Run does not have (story 24).
|
|
514
|
+
* Absent when there were none. */
|
|
515
|
+
queued?: QueuedTask[];
|
|
516
|
+
/** Dispatches that threw; each Task is blocked on the Tracker and left for a human. */
|
|
517
|
+
failures: DispatchFailure[];
|
|
518
|
+
/** Set when a Breaker halted new dispatch: what tripped, and what a human should read. The Run
|
|
519
|
+
* still drained what was in flight before returning. */
|
|
520
|
+
breaker?: { condition: string; message: string };
|
|
521
|
+
/** What became of the Run's Proposed Learnings; absent when the Run has no learnings step. */
|
|
522
|
+
learnings?: LearningsResult;
|
|
523
|
+
/** The Run stopped because its Initiative is paused (story 55) and it was not asked to wait: the
|
|
524
|
+
* caller decides what to do with the rest of the work (the daemon puts the request back). */
|
|
525
|
+
paused?: boolean;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
const defaultInstall = async (workspacePath: string): Promise<InstallResult | undefined> => {
|
|
529
|
+
const command = inferInstallCommand(workspacePath);
|
|
530
|
+
return command ? runInstall(command, workspacePath, { timeoutMs: INSTALL_TIMEOUT_MS }) : undefined;
|
|
531
|
+
};
|
|
532
|
+
|
|
533
|
+
const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
534
|
+
|
|
535
|
+
const noHandOff: HandOffFn = async () => ({ kind: "incomplete", reason: "this Run has no Hand-off configured (RunDeps.handOff), so nothing was pushed" });
|
|
536
|
+
|
|
537
|
+
const noGates: GatesFn = async () => ({ skipped: true, reason: "this Run has no Gates configured (RunDeps.gates)" });
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* One Account a Run may dispatch on: the Account, the Worker Adapter for its vendor, and the
|
|
542
|
+
* per-Account steps built from the two (the Verifier runs a second turn on the same Account; the
|
|
543
|
+
* Gates run their shell without its key). `RunDeps` carries the Run's own; `RunDeps.failover`
|
|
544
|
+
* carries the siblings (`runDepsFor`, commands/foreman.ts, builds them from the registered
|
|
545
|
+
* Accounts of the same vendor).
|
|
546
|
+
*/
|
|
547
|
+
export type RunAccount = Pick<RunDeps, "account" | "worker" | "verifier" | "gates" | "maxTurns" | "stallTimeoutMs">;
|
|
548
|
+
|
|
549
|
+
/** An Account inside a Run: its `RunAccount` and the Slots the Run gives it. Slots are per Account
|
|
550
|
+
* because the cap is (ADR-0013): a Run with two Accounts runs each one's cap, not the sum on one. */
|
|
551
|
+
interface Lane extends RunAccount {
|
|
552
|
+
slots: Slots;
|
|
553
|
+
/** True for a Lane of another vendor (`RunDeps.pinnable`): only a Task whose label pins it here
|
|
554
|
+
* may use it, so an unpinned Ready Task never sees it and Failover never reaches it. */
|
|
555
|
+
pinOnly?: boolean;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/** The Run's deps with its Slots and the Tasks it has in flight: what every dispatch of one Run
|
|
559
|
+
* shares. `runForeman` builds one per Lane; nothing outside this module constructs one. */
|
|
560
|
+
interface Scheduling extends RunDeps {
|
|
561
|
+
slots: Slots;
|
|
562
|
+
/** Whether this Run has the Task's turn in flight, for Reconcile (a row it is driving is not a
|
|
563
|
+
* row a tick may restart). In-process only: it is this Run's own map, so a *second* Foreman on
|
|
564
|
+
* the same Process Table still sees a row whose adapter reports no pid as one to restart, and
|
|
565
|
+
* would dispatch a live Task twice. What keeps that from happening is the guard on starting a
|
|
566
|
+
* second Run on the Account at all (commands/foreman.ts), not this. */
|
|
567
|
+
owns: (taskId: number) => boolean;
|
|
568
|
+
/** Resolves when the Run is stopping; the Slot waits of Reconcile race against it. */
|
|
569
|
+
untilStop?: Promise<void>;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* Runs the Initiative's Ready Tasks, filling the free Slots of every Account it may use each tick
|
|
574
|
+
* until none remains (or once, for `--once` and `--task`).
|
|
575
|
+
*
|
|
576
|
+
* A tick Reconciles, checks the pause and the stop, then takes Ready Tasks in priority-then-age
|
|
577
|
+
* order and starts each in a free Slot without waiting for the ones already going. It then waits
|
|
578
|
+
* for a dispatch to settle — or, when it ran out of Slots before it ran out of Tasks, for a Slot to
|
|
579
|
+
* come back; or, when every Account it could have used is Paused, for the soonest reset — and ticks
|
|
580
|
+
* again. The Run ends when nothing is in flight, no Ready Task remains and nothing is waiting on a
|
|
581
|
+
* reset, and its Proposed Learnings go out then (story 45).
|
|
582
|
+
*
|
|
583
|
+
* **Accounts (stories 30-31).** The Run's own Account is its first Lane; `RunDeps.failover` adds a
|
|
584
|
+
* Lane per sibling Account of the same vendor, each with its own Slots. A Ready Task takes the
|
|
585
|
+
* first Lane that is not Paused, is not another Foreman's (`contendedNow`) and has a Slot, which is
|
|
586
|
+
* the Failover: while Account A is Paused, the next Ready Task starts on Account B and A's own
|
|
587
|
+
* Workers finish where they are. A live session never changes Account — a Task with a Process Table
|
|
588
|
+
* row is pinned to that row's Lane and waits for it — so a session and the credential behind it
|
|
589
|
+
* always stay together; the Workspace is the Task's rather than the Account's, so a retry after a
|
|
590
|
+
* machine-side failure reuses it wherever it lands, and is told the tree is not clean
|
|
591
|
+
* (core/context.ts). Paused itself is
|
|
592
|
+
* read from the AccountStore each tick and is nothing but a timestamp: the tick after the reset
|
|
593
|
+
* passes fills that Account's Slots again, with no operator action.
|
|
594
|
+
*
|
|
595
|
+
* **The pin (story 24, core/pin.ts).** A Task labelled `foreman:vendor:codex` or
|
|
596
|
+
* `foreman:account:spare` narrows that Lane set to the Lanes the label allows, before any of the
|
|
597
|
+
* above is asked. `RunDeps.pinnable` adds a Lane per Account of another vendor, which only a pinned
|
|
598
|
+
* Task ever sees, so a Codex-pinned Task runs on the Codex Account while the Claude Slots beside it
|
|
599
|
+
* sit free. Everything the Lane set means afterwards is unchanged: a pinned Task whose only Lane is
|
|
600
|
+
* Paused **waits for that reset** rather than failing over — routing the work elsewhere is the one
|
|
601
|
+
* thing the label ruled out — and a pinned Task whose Lane is another Foreman's waits the same way.
|
|
602
|
+
* A pin no Lane can satisfy leaves the Task Ready with a comment (`leaveQueued`); it is never run
|
|
603
|
+
* somewhere else and never blocked.
|
|
604
|
+
*/
|
|
605
|
+
export async function runForeman(deps: RunDeps, options: RunOptions): Promise<RunOutcome> {
|
|
606
|
+
const { tracker, account, log } = deps;
|
|
607
|
+
const { initiativeId } = options;
|
|
608
|
+
const now = deps.now ?? Date.now;
|
|
609
|
+
const dispatches: Dispatch[] = [];
|
|
610
|
+
/** Defect Tasks this Run has dispatched, against `defectDispatchCap`. */
|
|
611
|
+
let defectDispatches = 0;
|
|
612
|
+
/**
|
|
613
|
+
* The earliest this Run may dispatch a Task again, by Task id, for Tasks a retryable failure made
|
|
614
|
+
* Ready again (`retryBackoff`). Written where an attempt settles, read where the tick picks.
|
|
615
|
+
*
|
|
616
|
+
* This Run's own and in memory, exactly as `defectDispatches` is, so a Foreman that crashes and
|
|
617
|
+
* is restarted begins a fresh Run with nothing held back. That is the honest scope: the delay
|
|
618
|
+
* exists to stop one Run burning a Task's attempts as fast as an adapter can fail them, and a
|
|
619
|
+
* restarted Foreman is not that loop. Nothing durable is needed for it either — the Run Record
|
|
620
|
+
* already counts what the retry cap spends, and this only decides *when*, not *whether*.
|
|
621
|
+
*/
|
|
622
|
+
const retryBackoffs = new Map<number, number>();
|
|
623
|
+
const blocked: BlockedTask[] = [];
|
|
624
|
+
const failures: DispatchFailure[] = [];
|
|
625
|
+
/** The Tasks this Run has a turn going for, by Task id; the promise never rejects. */
|
|
626
|
+
const inFlight = new Map<number, Promise<void>>();
|
|
627
|
+
const owns = (taskId: number) => inFlight.has(taskId);
|
|
628
|
+
|
|
629
|
+
// The Lanes, in the order a Ready Task tries them: this Run's own Account first, so Failover is
|
|
630
|
+
// the exception and the default Account is where work lands while nothing is Paused.
|
|
631
|
+
const siblings = (deps.failover ?? []).filter((s) => {
|
|
632
|
+
if (s.account.name === account.name) return false;
|
|
633
|
+
// Failover is within a vendor (CONTEXT.md): the Workspace is vendor-neutral but the session,
|
|
634
|
+
// the prompt contract and the adapter are not, so another vendor is a different Worker, not
|
|
635
|
+
// the same work moved.
|
|
636
|
+
if (s.account.vendor !== account.vendor) {
|
|
637
|
+
log(`run: Account "${s.account.name}" is ${s.account.vendor}, not ${account.vendor}; Failover is within a vendor, so it is not used for this Run`);
|
|
638
|
+
return false;
|
|
639
|
+
}
|
|
640
|
+
return true;
|
|
641
|
+
});
|
|
642
|
+
// Lanes only a pinned Task may use (story 24): the Accounts of another vendor. Deduplicated
|
|
643
|
+
// against the Lanes above by name, because two Lanes on one Account would be two Slots objects on
|
|
644
|
+
// one vendor identity — the cap counted twice, which is the one thing Lanes must never do.
|
|
645
|
+
const pinnable = (deps.pinnable ?? []).filter((p) => {
|
|
646
|
+
if (p.account.name === account.name || siblings.some((s) => s.account.name === p.account.name)) return false;
|
|
647
|
+
// `RunDeps.pinnable` says another vendor, and `pinnableAccountsFor` only ever builds it that
|
|
648
|
+
// way; enforced here as well because this is where it is relied on. A same-vendor Account is a
|
|
649
|
+
// Failover Lane — one an *unpinned* Task may use too — and quietly making it pin-only would
|
|
650
|
+
// narrow the fleet by dropping it out of `openLanes`.
|
|
651
|
+
if (p.account.vendor === account.vendor) {
|
|
652
|
+
log(`run: Account "${p.account.name}" is ${p.account.vendor}, the same vendor as "${account.name}"; a pin-only Lane is another vendor's Account, so it is not used for this Run (a sibling of the same vendor belongs in Failover)`);
|
|
653
|
+
return false;
|
|
654
|
+
}
|
|
655
|
+
return true;
|
|
656
|
+
});
|
|
657
|
+
const lanes: Lane[] = [
|
|
658
|
+
{ account, worker: deps.worker, verifier: deps.verifier, gates: deps.gates, slots: new Slots(account.cap) },
|
|
659
|
+
...siblings.map((s) => ({ ...s, slots: new Slots(s.account.cap) })),
|
|
660
|
+
...pinnable.map((p) => ({ ...p, slots: new Slots(p.account.cap), pinOnly: true })),
|
|
661
|
+
];
|
|
662
|
+
const slots = lanes[0].slots;
|
|
663
|
+
/** The Lanes an unpinned Ready Task may use: every Lane but the pin-only ones. */
|
|
664
|
+
const openLanes = lanes.filter((l) => !l.pinOnly);
|
|
665
|
+
|
|
666
|
+
// Lifts a human made through `foreman account resume`, which had no Initiative to record them
|
|
667
|
+
// against. This Run has one and a Lane on the Account, so it reports them now, dated when they
|
|
668
|
+
// happened rather than when they were noticed.
|
|
669
|
+
await reportResumedAccounts(deps, initiativeId, lanes);
|
|
670
|
+
/** One Lane's view of the Run: its Account and adapter in place of the Run's own, its per-turn
|
|
671
|
+
* limits (story 33: `foreman account set --max-turns` on a Failover Account must reach its own
|
|
672
|
+
* Workers, not the primary Account's numbers), its Slots. */
|
|
673
|
+
const schedulingFor = (lane: Lane): Scheduling => ({
|
|
674
|
+
...deps,
|
|
675
|
+
account: lane.account,
|
|
676
|
+
worker: lane.worker,
|
|
677
|
+
verifier: lane.verifier,
|
|
678
|
+
gates: lane.gates,
|
|
679
|
+
maxTurns: lane.maxTurns ?? deps.maxTurns,
|
|
680
|
+
stallTimeoutMs: lane.stallTimeoutMs ?? deps.stallTimeoutMs,
|
|
681
|
+
slots: lane.slots,
|
|
682
|
+
owns,
|
|
683
|
+
untilStop: options.untilStop,
|
|
684
|
+
});
|
|
685
|
+
const scheduling = schedulingFor(lanes[0]);
|
|
686
|
+
const sleep = deps.sleep ?? defaultSleep;
|
|
687
|
+
// A poll of 0 is not "poll as fast as possible", it is a 1ms hot loop that burns a core while an
|
|
688
|
+
// Account waits out an hour; the floor makes a mistyped flag slow rather than harmful.
|
|
689
|
+
const pausePollMs = Math.max(MIN_PAUSE_POLL_MS, deps.pausePollMs ?? PAUSE_POLL_MS);
|
|
690
|
+
// `--once` and `--task` are one dispatch, whatever the cap: the human asked for one.
|
|
691
|
+
const single = options.once === true || options.taskId !== undefined;
|
|
692
|
+
log(
|
|
693
|
+
`run: Initiative ${initiativeId} on Account "${account.name}" (${account.vendor}), cap ${slots.cap}: ${single ? "one dispatch (asked for)" : `${count(slots.implementerCap, "implementer Slot")}${slots.reservesVerifierSlot ? " and one reserved for Verifiers" : " and no Slot to reserve for Verifiers"}`}${siblings.length ? `; Failover to ${siblings.map((s) => `"${s.account.name}" (cap ${s.account.cap})`).join(", ")} while it is Paused` : ""}${pinnable.length ? `; ${pinnable.map((p) => `"${p.account.name}" (${p.account.vendor}, cap ${p.account.cap})`).join(", ")} for a Task pinned there and nothing else` : ""}`,
|
|
694
|
+
);
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* Which Accounts are Paused right now, and until when (story 30). Read from the AccountStore
|
|
698
|
+
* rather than from the Lane, so a pause written by the turn that hit the limit — or by another
|
|
699
|
+
* Foreman on the same Account — is seen the moment the next Slot is filled, and a pause whose
|
|
700
|
+
* reset has passed is already gone (`pausedUntil`). Without a store the Lane's own snapshot is
|
|
701
|
+
* all there is, which is the shape a test that hands in no store wants.
|
|
702
|
+
*/
|
|
703
|
+
const pausedNow = (at: number, stored = deps.accounts?.load()): Map<string, Date> => {
|
|
704
|
+
const paused = new Map<string, Date>();
|
|
705
|
+
for (const lane of lanes) {
|
|
706
|
+
const current = stored?.find((a) => a.name === lane.account.name) ?? lane.account;
|
|
707
|
+
const until = pausedUntil(current, at);
|
|
708
|
+
if (until) paused.set(lane.account.name, until);
|
|
709
|
+
}
|
|
710
|
+
return paused;
|
|
711
|
+
};
|
|
712
|
+
|
|
713
|
+
/**
|
|
714
|
+
* Which Lanes are quarantined right now (OPS-288). Read from the AccountStore every tick for the
|
|
715
|
+
* reasons `pausedNow` is: the turn that proved the credential dead wrote it a moment ago, and a
|
|
716
|
+
* quarantine another Foreman wrote is just as binding — the Account is one vendor identity and
|
|
717
|
+
* its credential is dead for everyone.
|
|
718
|
+
*
|
|
719
|
+
* Unlike a pause this takes no `at`: nothing about a quarantine changes with the clock, which is
|
|
720
|
+
* exactly what makes it worth reporting differently when a Run runs out of Lanes.
|
|
721
|
+
*/
|
|
722
|
+
const quarantinedNow = (stored = deps.accounts?.load()): Map<string, string> => {
|
|
723
|
+
const quarantined = new Map<string, string>();
|
|
724
|
+
for (const lane of lanes) {
|
|
725
|
+
const current = stored?.find((a) => a.name === lane.account.name) ?? lane.account;
|
|
726
|
+
if (current.quarantined) quarantined.set(lane.account.name, current.quarantined.reason);
|
|
727
|
+
}
|
|
728
|
+
return quarantined;
|
|
729
|
+
};
|
|
730
|
+
|
|
731
|
+
/** Lanes already reported as quarantined, so the log says it once rather than once per tick. */
|
|
732
|
+
const quarantineLogged = new Set<string>();
|
|
733
|
+
|
|
734
|
+
/** Lanes already reported as another Foreman's, so the log says it once. */
|
|
735
|
+
const contendedLogged = new Set<string>();
|
|
736
|
+
|
|
737
|
+
/** Which of the Lanes after the first — Failover Lanes and pin-only Lanes alike — another Foreman
|
|
738
|
+
* has Workers on, and how many, at `at`.
|
|
739
|
+
*
|
|
740
|
+
* `failoverAccountsFor` and `pinnableAccountsFor` (commands/foreman.ts) ask the same question,
|
|
741
|
+
* but they can only ask once, before the Run starts: an Account with no live rows then is a Lane,
|
|
742
|
+
* and nothing stops a second `foreman run --account b` from starting on it a minute later, since
|
|
743
|
+
* that Run's own guard (`refuseSecondRunOnAccount`) sees no Workers on b either. Two Runs would
|
|
744
|
+
* then have their own Slots on one vendor identity, which is the thing per-Account caps exist to
|
|
745
|
+
* prevent (ADR-0013). So the Process Table is asked again every tick, and a Lane that is not
|
|
746
|
+
* clean is not used this tick. This Run's own turns are not contention (`owns`), and the Run's
|
|
747
|
+
* first Lane is left out: it is guarded at start, and dropping the Account the human named would
|
|
748
|
+
* be a Run that does nothing rather than one that keeps within the cap.
|
|
749
|
+
*
|
|
750
|
+
* **Without a Process Table there is no second ask**, and this returns nothing: the Run has only
|
|
751
|
+
* the snapshot the two functions above took, which is `RunDeps` without `processTable` — the
|
|
752
|
+
* shape a unit test hands in, never `foreman run`, which always opens one (commands/foreman.ts).
|
|
753
|
+
*/
|
|
754
|
+
const contendedNow = (at: number): Map<string, number> => {
|
|
755
|
+
const contended = new Map<string, number>();
|
|
756
|
+
if (!deps.processTable || lanes.length < 2) return contended;
|
|
757
|
+
for (const lane of lanes.slice(1)) {
|
|
758
|
+
const live = otherForemanWorkersOn(deps.processTable, lane.account.name, at, { isAlive: deps.isAlive, owns });
|
|
759
|
+
if (live.length === 0) {
|
|
760
|
+
contendedLogged.delete(lane.account.name);
|
|
761
|
+
continue;
|
|
762
|
+
}
|
|
763
|
+
contended.set(lane.account.name, live.length);
|
|
764
|
+
// Once per spell, not once per tick: a Foreman that holds the Lane for an hour must not write
|
|
765
|
+
// a line every poll.
|
|
766
|
+
if (contendedLogged.has(lane.account.name)) continue;
|
|
767
|
+
contendedLogged.add(lane.account.name);
|
|
768
|
+
log(
|
|
769
|
+
`run: Account "${lane.account.name}" now has ${count(live.length, "Worker")} running under another Foreman (${live.map((r) => `${r.identifier}${r.pid === undefined ? " (starting)" : ` (pid ${r.pid})`}`).join(", ")}); ` +
|
|
770
|
+
`it is not used for ${lane.pinOnly ? "a pinned Task" : "Failover"} while that lasts, since its cap is that Foreman's to fill`,
|
|
771
|
+
);
|
|
772
|
+
}
|
|
773
|
+
return contended;
|
|
774
|
+
};
|
|
775
|
+
|
|
776
|
+
/** Tasks whose row and whose label disagree, so the log says it once rather than every tick. */
|
|
777
|
+
const disagreementLogged = new Set<number>();
|
|
778
|
+
|
|
779
|
+
/** Where a Ready Task may run: the Lanes it is allowed, or why there are none. */
|
|
780
|
+
type LanePick =
|
|
781
|
+
| { kind: "lanes"; lanes: Lane[] }
|
|
782
|
+
/** Its Process Table row is under an Account this Run has no Lane for. */
|
|
783
|
+
| { kind: "foreign"; account: string }
|
|
784
|
+
/** Its label pins it somewhere this Run cannot reach, or cannot be read as a pin at all. */
|
|
785
|
+
| { kind: "unplaceable"; pin: Pin; why: Unplaceable };
|
|
786
|
+
|
|
787
|
+
/**
|
|
788
|
+
* The Lanes a Task may run in, narrowed twice.
|
|
789
|
+
*
|
|
790
|
+
* First by its Process Table row, when it has one: a row is what "started" means (ADR-0009: it
|
|
791
|
+
* exists from the moment the dispatch is committed), and started work does not change Account.
|
|
792
|
+
* **The row wins over the label**, always — a vendor session cannot be resumed under another
|
|
793
|
+
* identity, so a label added after a Worker started cannot move it, and re-pinning takes effect
|
|
794
|
+
* on the Task's next attempt. A disagreement between the two is worth a line, because to the
|
|
795
|
+
* developer who wrote the label it looks like the pin was ignored.
|
|
796
|
+
*
|
|
797
|
+
* Then by the pin its labels carry (story 24, core/pin.ts): a vendor, one named Account, or
|
|
798
|
+
* nothing. An unpinned Task never sees a pin-only Lane. A pin that no Lane satisfies does *not*
|
|
799
|
+
* fall back to running somewhere else — that is the whole point of a pin — so the Task comes back
|
|
800
|
+
* unplaceable and is left Ready with a comment. `registered` is every Account on the machine,
|
|
801
|
+
* which is what lets that comment tell "you never registered this Account" apart from "another
|
|
802
|
+
* Foreman is on it, wait": two conditions with opposite remedies (`unplaceable`, core/pin.ts).
|
|
803
|
+
*/
|
|
804
|
+
const lanesFor = (task: TrackerTask, registered?: readonly Account[]): LanePick => {
|
|
805
|
+
const row = deps.processTable?.worker(task.id);
|
|
806
|
+
const pin = pinOf(task.labels);
|
|
807
|
+
if (row) {
|
|
808
|
+
const started = lanes.find((l) => l.account.name === row.account);
|
|
809
|
+
// A row under an Account this Run does not have is not this Run's Task at all; Reconcile skips
|
|
810
|
+
// it for the same reason, and picking it here would take it off its own Foreman.
|
|
811
|
+
if (!started) return { kind: "foreign", account: row.account };
|
|
812
|
+
if (pin && !pinAllows(pin, started.account) && !disagreementLogged.has(task.id)) {
|
|
813
|
+
disagreementLogged.add(task.id);
|
|
814
|
+
log(
|
|
815
|
+
`run: ${task.identifier} is ${describePin(pin)} but has a live Worker on Account "${started.account.name}"; ` +
|
|
816
|
+
`a started session never changes Account, so the row wins and the pin applies to its next attempt`,
|
|
817
|
+
);
|
|
818
|
+
}
|
|
819
|
+
return { kind: "lanes", lanes: [started] };
|
|
820
|
+
}
|
|
821
|
+
if (!pin) return { kind: "lanes", lanes: openLanes };
|
|
822
|
+
const why = unplaceable(
|
|
823
|
+
pin,
|
|
824
|
+
lanes.map((l) => l.account),
|
|
825
|
+
registered,
|
|
826
|
+
);
|
|
827
|
+
if (why) return { kind: "unplaceable", pin, why };
|
|
828
|
+
return { kind: "lanes", lanes: lanes.filter((l) => pinAllows(pin, l.account)) };
|
|
829
|
+
};
|
|
830
|
+
|
|
831
|
+
// Tasks whose dispatch threw and Tasks pinned where this Run cannot reach: retried by the next
|
|
832
|
+
// Run, never by this one, so a Run cannot spin on them.
|
|
833
|
+
//
|
|
834
|
+
// A Triage turn that stalled or hit a rate limit is deliberately *not* here. It is a machine-side
|
|
835
|
+
// failure like any other, and the thing that bounds it is the retry cap the Run Record already
|
|
836
|
+
// counts (`retryableFailuresOf`), not the length of the night: deferring it would cost a Task the
|
|
837
|
+
// whole Run for one bad minute at 3am, when a rate limit resolves by itself as soon as the
|
|
838
|
+
// Account's window resets.
|
|
839
|
+
const deferred = new Set<number>();
|
|
840
|
+
const queued: QueuedTask[] = [];
|
|
841
|
+
|
|
842
|
+
/**
|
|
843
|
+
* Leaves a pinned Task Ready with one comment saying where it wanted to go, what this Run has,
|
|
844
|
+
* and which of the three things fixes it (story 24).
|
|
845
|
+
*
|
|
846
|
+
* Deferred, so it is passed over for the rest of the Run: no tick of this Run can register an
|
|
847
|
+
* Account or fix a label, so looking again would only be a way to comment twice. Nothing else is
|
|
848
|
+
* touched — the dispatch label stays on, no Status Label goes on, no Run Record event is written
|
|
849
|
+
* and this Run spends no attempt on it — so the Task is exactly as Ready as it was, for the next
|
|
850
|
+
* Run or for a human. A Tracker that refuses the comment is logged and the Task is still left
|
|
851
|
+
* alone: failing to say why must not turn into running the Task somewhere the developer ruled out.
|
|
852
|
+
*
|
|
853
|
+
* **Once per condition, not once per Run.** The deferral above only covers this process:
|
|
854
|
+
* `deferred` does not survive a restart, a second `foreman run`, or the daemon building a fresh
|
|
855
|
+
* Run for the next request, and a Task pinned at an Account nobody intends to register is a
|
|
856
|
+
* condition that lasts indefinitely — a nightly daemon would leave an identical comment every
|
|
857
|
+
* night. So the Tracker is the record, the way the Defect filer makes filing idempotent
|
|
858
|
+
* (core/defects.ts): the comment carries `pinToken(pin)`, and a Task that already has one for
|
|
859
|
+
* this pin is left in silence. The token is the pin's, not the Run's, so correcting a label into
|
|
860
|
+
* a *different* unplaceable pin is a new condition and is said again.
|
|
861
|
+
*/
|
|
862
|
+
const leaveQueued = async (task: TrackerTask, pin: Pin, why: Unplaceable): Promise<void> => {
|
|
863
|
+
deferred.add(task.id);
|
|
864
|
+
queued.push({ task, pin, reason: why.reason });
|
|
865
|
+
log(`run: ${task.identifier} ${why.reason} It stays Ready with its "${DISPATCH_LABEL}" label; nothing was dispatched and this Run spent no attempt on it.`);
|
|
866
|
+
const token = pinToken(pin);
|
|
867
|
+
try {
|
|
868
|
+
const { comments } = await tracker.getTask(task.id);
|
|
869
|
+
if (comments.some((c) => c.body.includes(token))) {
|
|
870
|
+
log(`run: ${task.identifier} was already told this about ${token}; the comment is not repeated`);
|
|
871
|
+
return;
|
|
872
|
+
}
|
|
873
|
+
} catch (error) {
|
|
874
|
+
// The Task's own comments could not be read. Saying it twice is a worse outcome than saying
|
|
875
|
+
// it once, but a silent Task is worse than both, so the comment goes out.
|
|
876
|
+
log(`run: ${task.identifier} comments could not be read back, so this may repeat an earlier one: ${error instanceof Error ? error.message : String(error)}`);
|
|
877
|
+
}
|
|
878
|
+
try {
|
|
879
|
+
await tracker.addComment(task.id, `Foreman: this Task ${why.reason} It is left Ready and untouched — this Run spent no attempt on it and it is not blocked — so ${why.remedy}. ${token}`);
|
|
880
|
+
} catch (error) {
|
|
881
|
+
log(`run: ${task.identifier} could not be commented on: ${error instanceof Error ? error.message : String(error)}`);
|
|
882
|
+
}
|
|
883
|
+
};
|
|
884
|
+
const record = (settled: Dispatch | BlockedTask) => {
|
|
885
|
+
if (!("missing" in settled)) {
|
|
886
|
+
dispatches.push(settled);
|
|
887
|
+
// A retried dispatch says when it may be taken again; anything else has finished with the
|
|
888
|
+
// Task, so an entry an earlier attempt left is dropped rather than outliving what set it.
|
|
889
|
+
if (settled.retryNotBefore !== undefined) retryBackoffs.set(settled.task.id, settled.retryNotBefore);
|
|
890
|
+
else retryBackoffs.delete(settled.task.id);
|
|
891
|
+
return;
|
|
892
|
+
}
|
|
893
|
+
blocked.push(settled);
|
|
894
|
+
// `retry` means the Task kept its dispatch label and is Ready again, and a later tick of this
|
|
895
|
+
// Run takes it: not deferred, but not immediately either. `defer` is the Task that kept its
|
|
896
|
+
// label for a reason no tick of this Run can change. Everything else is blocked and has lost
|
|
897
|
+
// the label anyway.
|
|
898
|
+
if (settled.retry) retryBackoffs.set(settled.task.id, settled.retry.notBefore);
|
|
899
|
+
else retryBackoffs.delete(settled.task.id);
|
|
900
|
+
if (settled.defer) deferred.add(settled.task.id);
|
|
901
|
+
};
|
|
902
|
+
|
|
903
|
+
/** Pin-only Lanes this Run has actually dispatched onto, which is what makes one of them a Lane
|
|
904
|
+
* this Run may Reconcile (see the Reconcile loop). */
|
|
905
|
+
const workedPinOnly = new Set<string>();
|
|
906
|
+
|
|
907
|
+
/** Starts a dispatch in the Lane's Slot it has been given and keeps its promise until it settles. */
|
|
908
|
+
const fill = (task: TrackerTask, lane: Lane, hold: SlotHold): void => {
|
|
909
|
+
if (lane.pinOnly) workedPinOnly.add(lane.account.name);
|
|
910
|
+
const running = (async () => {
|
|
911
|
+
try {
|
|
912
|
+
record(await dispatch(schedulingFor(lane), initiativeId, task, hold));
|
|
913
|
+
} catch (error) {
|
|
914
|
+
// `dispatch` has already commented on the Task and marked it blocked; a Slot's failure is
|
|
915
|
+
// the Run's to report, never the other Slots' to die of.
|
|
916
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
917
|
+
log(`run: ${task.identifier} was left blocked by a failed dispatch; the other Slots go on: ${reason}`);
|
|
918
|
+
failures.push({ task, reason });
|
|
919
|
+
deferred.add(task.id);
|
|
920
|
+
} finally {
|
|
921
|
+
hold.release();
|
|
922
|
+
inFlight.delete(task.id);
|
|
923
|
+
}
|
|
924
|
+
})();
|
|
925
|
+
if (task.labels?.includes(DEFECT_LABEL)) defectDispatches += 1;
|
|
926
|
+
inFlight.set(task.id, running);
|
|
927
|
+
};
|
|
928
|
+
|
|
929
|
+
/** Settles every turn still going, so the Run never returns with Workers it started unaccounted. */
|
|
930
|
+
const drain = async () => {
|
|
931
|
+
while (inFlight.size) await Promise.all([...inFlight.values()]);
|
|
932
|
+
};
|
|
933
|
+
|
|
934
|
+
let pausedLogged = false;
|
|
935
|
+
let started = 0;
|
|
936
|
+
/** The project's priority scale, logged the first tick that shows one (core/scheduler.ts). */
|
|
937
|
+
let scaleLogged = false;
|
|
938
|
+
// Whatever ends the loop — no Ready Task, a stop, or a throw from the Tracker — the turns this
|
|
939
|
+
// Run started are settled before it reports, so it never leaves Workers unaccounted.
|
|
940
|
+
try {
|
|
941
|
+
for (;;) {
|
|
942
|
+
// The stop is read before Reconcile, not after it: a Reconcile that must resume or restart a row
|
|
943
|
+
// takes an implementer Slot for it, and at the default cap there is one, so it can be waiting
|
|
944
|
+
// on a Worker turn to end. Asking here means a stop that arrived while the last tick waited is
|
|
945
|
+
// acted on now rather than a whole turn later, and the wait inside Reconcile is abandoned by
|
|
946
|
+
// `untilStop` besides. The pause is deliberately *not* moved up with it: a paused Run still
|
|
947
|
+
// finishes what is in flight (story 55), and Reconcile is what finishes it.
|
|
948
|
+
if (options.shouldStop?.()) {
|
|
949
|
+
log(`run: stopping as asked, after ${count(dispatches.length, "dispatch", "dispatches")}${inFlight.size ? `, waiting for ${count(inFlight.size, "turn")} in flight` : ""}`);
|
|
950
|
+
break;
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
// Reconcile before anything new is picked (story 51): what a previous process left in the
|
|
954
|
+
// Process Table is settled first, so a resumed Task is never dispatched a second time. Rows
|
|
955
|
+
// this Run is driving are left alone (`owns`), whatever their pid says. One pass per Lane,
|
|
956
|
+
// since a row is reconciled by a Run under its own Account and skipped by every other
|
|
957
|
+
// (reconcile.ts): that is how a Worker left on Account B is resumed on B, whichever Account
|
|
958
|
+
// this Run calls its own. The outbox is drained by the first pass alone — it is the Run's,
|
|
959
|
+
// not the Lane's.
|
|
960
|
+
for (const [index, lane] of lanes.entries()) {
|
|
961
|
+
// A pin-only Lane is another vendor's Account this Run adopted for pinned Tasks alone, and
|
|
962
|
+
// until it has dispatched onto one it is not this Run's to settle. `owns` is per-Run and
|
|
963
|
+
// in-process, so a row another Foreman on that Account wrote — one whose settle path is
|
|
964
|
+
// between the Worker ending and `removeWorker` — reads here as a dead pid with a session,
|
|
965
|
+
// which `planReconcile` calls a resume: this Run would drive a second turn on that
|
|
966
|
+
// Foreman's Task, in its Workspace, under its identity. Merely having a Codex Account
|
|
967
|
+
// registered must not make a Claude Run adopt its rows. Nothing is orphaned by waiting:
|
|
968
|
+
// the daemon's idle Reconcile groups live rows by their own Account and settles them under
|
|
969
|
+
// it (commands/foreman-up.ts), and a Run started on that Account settles them itself.
|
|
970
|
+
if (lane.pinOnly && !workedPinOnly.has(lane.account.name)) continue;
|
|
971
|
+
const laneDeps = index === 0 ? scheduling : { ...schedulingFor(lane), drainOutbox: undefined };
|
|
972
|
+
for (const settled of await reconcileTick(laneDeps, initiativeId)) record(settled);
|
|
973
|
+
}
|
|
974
|
+
// Paused (story 55): nothing new starts, and what is in flight is settled before the Run
|
|
975
|
+
// reports. A Run asked to wait idles here until the resume, at the daemon's tick; any other
|
|
976
|
+
// returns the pause as its outcome, so a daemon can put the request back and serve others.
|
|
977
|
+
if (deps.processTable?.isPaused(initiativeId)) {
|
|
978
|
+
if (!options.waitWhilePaused) {
|
|
979
|
+
await drain();
|
|
980
|
+
log(`run: Initiative ${initiativeId} is paused; stopping after ${count(dispatches.length, "dispatch", "dispatches")}, the rest waits for foreman resume ${initiativeId}`);
|
|
981
|
+
return { dispatches, blocked, failures, ...(queued.length ? { queued } : {}), paused: true };
|
|
982
|
+
}
|
|
983
|
+
if (!pausedLogged) log(`run: Initiative ${initiativeId} is paused; no new dispatch until foreman resume ${initiativeId}`);
|
|
984
|
+
pausedLogged = true;
|
|
985
|
+
await sleep(pausePollMs);
|
|
986
|
+
continue;
|
|
987
|
+
}
|
|
988
|
+
if (pausedLogged) log(`run: Initiative ${initiativeId} resumed; dispatch continues`);
|
|
989
|
+
pausedLogged = false;
|
|
990
|
+
|
|
991
|
+
// The Breaker (CONTEXT.md: Breaker). Same shape as the pause above and for the same reason:
|
|
992
|
+
// nothing new starts, what is in flight is settled before the Run reports, and the fact is on
|
|
993
|
+
// the Run Record for the morning. Checked here rather than at the dispatch site so a Run that
|
|
994
|
+
// trips mid-tick still finishes the turns it already started.
|
|
995
|
+
const defectCap = deps.defectDispatchCap ?? DEFAULT_DEFECT_DISPATCH_CAP;
|
|
996
|
+
if (defectDispatches >= defectCap) {
|
|
997
|
+
const message =
|
|
998
|
+
`${count(defectDispatches, "Defect Task")} dispatched in this Run, which is the cap (${defectCap}).` +
|
|
999
|
+
` New dispatch is stopped so a defect-fix-defect loop cannot spend the rest of the night;` +
|
|
1000
|
+
` the Tasks already running finish. Raise it with defectDispatchCap, or look at the Initiative's plan.`;
|
|
1001
|
+
log(`run: breaker: ${message}`);
|
|
1002
|
+
await tracker.appendRunEvents(initiativeId, [runEvent(undefined, { payload: { case: "breaker", value: { condition: "defect_loop", message } } })]);
|
|
1003
|
+
await drain();
|
|
1004
|
+
return await summarised(deps, initiativeId, { dispatches, blocked, failures, ...(queued.length ? { queued } : {}), breaker: { condition: "defect_loop", message } });
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
/** True when Ready Tasks were left over for want of a Slot: the tick then waits for one back. */
|
|
1008
|
+
let slotLimited = false;
|
|
1009
|
+
/** True when the only Lanes left for a Ready Task are ones another Foreman has Workers on:
|
|
1010
|
+
* the tick polls rather than stopping, since that Foreman's Workers do end. */
|
|
1011
|
+
let laneContended = false;
|
|
1012
|
+
/** Ready Tasks whose Process Table row is under an Account this Run has no Lane for; counted
|
|
1013
|
+
* so the Run's tally says they were left rather than implying there was nothing to do. */
|
|
1014
|
+
let skippedForeign = 0;
|
|
1015
|
+
/** The soonest reset a Ready Task is waiting on, when every Lane it may use is Paused: the
|
|
1016
|
+
* tick sleeps until then rather than spinning, and dispatch resumes with no operator action. */
|
|
1017
|
+
let waitingOnReset: Date | undefined;
|
|
1018
|
+
/** True when a Ready Task's every Lane is quarantined (OPS-288). Unlike a pause there is
|
|
1019
|
+
* nothing to wait for and unlike contention nobody else's Workers will end, so this is the one
|
|
1020
|
+
* of the three that ends the Run rather than making it poll. */
|
|
1021
|
+
let laneQuarantined = false;
|
|
1022
|
+
/** The soonest a Ready Task held back by `retryBackoff` may be taken. Shaped like
|
|
1023
|
+
* `waitingOnReset` and waited on the same way: both are a moment this tick knows about and
|
|
1024
|
+
* neither needs anyone to do anything when it arrives. */
|
|
1025
|
+
let waitingOnBackoff: number | undefined;
|
|
1026
|
+
/** Which Accounts are Paused this tick; read once, and reused by the wait below. */
|
|
1027
|
+
let paused = new Map<string, Date>();
|
|
1028
|
+
if (single && started > 0) {
|
|
1029
|
+
// The one dispatch has started; nothing more is picked, only awaited.
|
|
1030
|
+
if (inFlight.size === 0) break;
|
|
1031
|
+
} else {
|
|
1032
|
+
const candidates = await pickTasks(tracker, options, deferred, inFlight);
|
|
1033
|
+
if (!scaleLogged) {
|
|
1034
|
+
// Dispatch takes the highest `TaskPriority.level` first, which is the seeded scale's
|
|
1035
|
+
// convention and not something Opsee enforces (core/scheduler.ts): a project that numbers
|
|
1036
|
+
// the other way would silently run in reverse, so the Run says what it is seeing.
|
|
1037
|
+
const scale = priorityScale(candidates);
|
|
1038
|
+
if (scale) {
|
|
1039
|
+
log(`run: priority levels in this project, highest first: ${scale}; dispatch takes the highest level first`);
|
|
1040
|
+
scaleLogged = true;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
// Every Account on the machine, read once and used twice: for Paused below, and for the
|
|
1044
|
+
// comment on a Task no Lane can take, where an Account that is registered but not a Lane is
|
|
1045
|
+
// one to wait for rather than one to register (core/pin.ts).
|
|
1046
|
+
const registered = deps.accounts?.load();
|
|
1047
|
+
paused = pausedNow(now(), registered);
|
|
1048
|
+
const quarantined = quarantinedNow(registered);
|
|
1049
|
+
// Which Lanes after the first — Failover and pin-only alike — another Foreman is on right
|
|
1050
|
+
// now, asked once per tick and again every tick: `failoverAccountsFor` and
|
|
1051
|
+
// `pinnableAccountsFor` (commands/foreman.ts) can only ask before the Run starts, and a
|
|
1052
|
+
// Lane that was latent then may be a second Foreman's by the time it is first used.
|
|
1053
|
+
const contended = contendedNow(now());
|
|
1054
|
+
let taken = 0;
|
|
1055
|
+
for (const task of candidates) {
|
|
1056
|
+
// Held back after a retryable failure, so its next attempt is not taken as fast as the
|
|
1057
|
+
// last one failed (`retryBackoff`). Asked before the Lanes, because the answer does not
|
|
1058
|
+
// depend on them: a Task waiting out its own backoff is waiting wherever it would run,
|
|
1059
|
+
// and a free Slot found for it here would only be handed straight back.
|
|
1060
|
+
// An entry whose moment has passed is simply not a reason to skip; it is left for the
|
|
1061
|
+
// Task's next settlement to replace or drop (`record`), rather than swept here, since a
|
|
1062
|
+
// Map keyed by Task id cannot grow past the Initiative and a second place that edits it
|
|
1063
|
+
// is a second place to get it wrong.
|
|
1064
|
+
const notBefore = retryBackoffs.get(task.id);
|
|
1065
|
+
if (notBefore !== undefined && notBefore > now()) {
|
|
1066
|
+
if (waitingOnBackoff === undefined || notBefore < waitingOnBackoff) waitingOnBackoff = notBefore;
|
|
1067
|
+
continue;
|
|
1068
|
+
}
|
|
1069
|
+
// The Lanes this Task may use, then the ones that are neither Paused nor another
|
|
1070
|
+
// Foreman's, then the first with a free Slot. A Task pinned to a Lane gets exactly one
|
|
1071
|
+
// candidate, so a Paused Account holds its own started work rather than handing it to a
|
|
1072
|
+
// sibling.
|
|
1073
|
+
const pick = lanesFor(task, registered);
|
|
1074
|
+
if (pick.kind === "foreign") {
|
|
1075
|
+
// The row is under an Account this Run does not have: another Foreman's Task, which
|
|
1076
|
+
// Reconcile skips for the same reason. Said out loud, because a silent skip here made
|
|
1077
|
+
// the Run claim below that no Ready Task remained.
|
|
1078
|
+
log(`run: ${task.identifier} has a Process Table row under Account "${pick.account}", which is not one of this Run's Lanes (${lanes.map((l) => `"${l.account.name}"`).join(", ")}); it belongs to that Account's Foreman and is left alone`);
|
|
1079
|
+
skippedForeign++;
|
|
1080
|
+
continue;
|
|
1081
|
+
}
|
|
1082
|
+
if (pick.kind === "unplaceable") {
|
|
1083
|
+
// Pinned somewhere this Run has no Lane for, or a `foreman:` label that is not a pin at
|
|
1084
|
+
// all. Either way running it here is the one thing the label rules out.
|
|
1085
|
+
await leaveQueued(task, pick.pin, pick.why);
|
|
1086
|
+
continue;
|
|
1087
|
+
}
|
|
1088
|
+
// Quarantine is asked before Paused, because it is the stronger fact: an Account whose
|
|
1089
|
+
// credential is dead cannot run this Task when its pause lifts either, and a Task waiting
|
|
1090
|
+
// on the reset of an Account that will never work again is a Task that never runs
|
|
1091
|
+
// (OPS-288). A quarantined Lane contributes no Slots to anyone.
|
|
1092
|
+
const usable = pick.lanes.filter((l) => !quarantined.has(l.account.name));
|
|
1093
|
+
if (usable.length === 0) {
|
|
1094
|
+
for (const lane of pick.lanes) {
|
|
1095
|
+
if (quarantineLogged.has(lane.account.name)) continue;
|
|
1096
|
+
quarantineLogged.add(lane.account.name);
|
|
1097
|
+
log(
|
|
1098
|
+
`run: Account "${lane.account.name}" (${lane.account.vendor}) is quarantined (${printableOneLine(quarantined.get(lane.account.name) ?? "")}); ` +
|
|
1099
|
+
`it contributes no Slots and is not a Failover target. Log the vendor back in, then: opsee foreman account resume ${lane.account.name}`,
|
|
1100
|
+
);
|
|
1101
|
+
}
|
|
1102
|
+
// A pinned Task is refused rather than merely skipped, and is told so on the Task
|
|
1103
|
+
// itself: the label ruled out running it anywhere else, so nothing this Run does next
|
|
1104
|
+
// will place it, and the human who wrote the pin is the only one who can. An unpinned
|
|
1105
|
+
// Task gets no comment — it can run on any Lane, so what it is waiting for is the Run's
|
|
1106
|
+
// fact and not the Task's, and a comment on every Ready Task would be noise.
|
|
1107
|
+
// A pin that could not be read never reaches here: `lanesFor` answers `unplaceable`
|
|
1108
|
+
// for those and the branch above has already refused the Task.
|
|
1109
|
+
const pin = pinOf(task.labels);
|
|
1110
|
+
if (pin) {
|
|
1111
|
+
await leaveQueued(task, pin, {
|
|
1112
|
+
reason: `is ${describePin(pin)}, and every Account that satisfies it is quarantined: its credential is dead, not rate limited.`,
|
|
1113
|
+
remedy: `log the vendor back in and run "opsee foreman account resume <name>" (or re-register the Account), or change the pin`,
|
|
1114
|
+
});
|
|
1115
|
+
continue;
|
|
1116
|
+
}
|
|
1117
|
+
laneQuarantined = true;
|
|
1118
|
+
continue;
|
|
1119
|
+
}
|
|
1120
|
+
const notPaused = usable.filter((l) => !paused.has(l.account.name));
|
|
1121
|
+
if (notPaused.length === 0) {
|
|
1122
|
+
// Every Lane this Task may use is Paused: it waits for the soonest of their resets.
|
|
1123
|
+
for (const lane of usable) {
|
|
1124
|
+
const until = paused.get(lane.account.name)!;
|
|
1125
|
+
if (!waitingOnReset || until < waitingOnReset) waitingOnReset = until;
|
|
1126
|
+
}
|
|
1127
|
+
continue;
|
|
1128
|
+
}
|
|
1129
|
+
const open = notPaused.filter((l) => !contended.has(l.account.name));
|
|
1130
|
+
if (open.length === 0) {
|
|
1131
|
+
// Only Lanes another Foreman has Workers on are left: opening Slots of our own there
|
|
1132
|
+
// would double one vendor identity's concurrency (ADR-0013), so the Task waits.
|
|
1133
|
+
laneContended = true;
|
|
1134
|
+
continue;
|
|
1135
|
+
}
|
|
1136
|
+
const lane = open.find((l) => l.slots.free > 0);
|
|
1137
|
+
if (!lane) {
|
|
1138
|
+
slotLimited = true;
|
|
1139
|
+
continue;
|
|
1140
|
+
}
|
|
1141
|
+
if (lane !== lanes[0]) {
|
|
1142
|
+
const pin = pinOf(task.labels);
|
|
1143
|
+
// Landing off the Run's own Account is either the pin doing its job or Failover doing
|
|
1144
|
+
// its job, and the two want different sentences: one is what the developer asked for,
|
|
1145
|
+
// the other is what the Foreman did because it had to.
|
|
1146
|
+
// Why the Run's own Account was passed over. There are two answers now, and the wrong
|
|
1147
|
+
// one is worse than none: a quarantine has no reset, so the pause wording would print
|
|
1148
|
+
// "Paused until ?" for a state that is not a pause and never lifts (OPS-288).
|
|
1149
|
+
const why = quarantined.has(lanes[0].account.name)
|
|
1150
|
+
? `"${lanes[0].account.name}" is quarantined (${printableOneLine(quarantined.get(lanes[0].account.name) ?? "")})`
|
|
1151
|
+
: `"${lanes[0].account.name}" is Paused until ${paused.get(lanes[0].account.name)?.toISOString() ?? "?"}`;
|
|
1152
|
+
log(
|
|
1153
|
+
pin && pinAllows(pin, lane.account)
|
|
1154
|
+
? `run: ${task.identifier} runs on Account "${lane.account.name}" (${lane.account.vendor}): it is ${describePin(pin)}`
|
|
1155
|
+
: `run: ${task.identifier} fails over to Account "${lane.account.name}": ${why}`,
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
1158
|
+
fill(task, lane, lane.slots.acquire());
|
|
1159
|
+
taken++;
|
|
1160
|
+
started++;
|
|
1161
|
+
if (single) break;
|
|
1162
|
+
}
|
|
1163
|
+
// Before the tally below, which would otherwise report "no Ready Task remains" for a Run
|
|
1164
|
+
// that has plenty of Ready Tasks and nowhere to run them. A quarantine is not waited out:
|
|
1165
|
+
// nothing lifts it but a human, so the Run says what to do and ends (OPS-288).
|
|
1166
|
+
//
|
|
1167
|
+
// `waitingOnBackoff` holds this back too: a Task serving out the delay after its last
|
|
1168
|
+
// failed attempt may still have a Lane to run on, so ending the Run on another Task's
|
|
1169
|
+
// quarantine would abandon work that was only waiting.
|
|
1170
|
+
if (laneQuarantined && taken === 0 && !slotLimited && !waitingOnReset && !laneContended && waitingOnBackoff === undefined && inFlight.size === 0) {
|
|
1171
|
+
log(
|
|
1172
|
+
`run: every Account a Ready Task could use is quarantined; ${count(dispatches.length, "dispatch", "dispatches")} made and Ready Tasks remain. ` +
|
|
1173
|
+
`Nothing lifts a quarantine by itself: log the vendor back in and run "opsee foreman account resume <name>", or register another Account, then start the Run again.`,
|
|
1174
|
+
);
|
|
1175
|
+
break;
|
|
1176
|
+
}
|
|
1177
|
+
if (taken === 0 && !slotLimited && !waitingOnReset && !laneContended && !laneQuarantined && waitingOnBackoff === undefined && inFlight.size === 0) {
|
|
1178
|
+
const tally = `${count(dispatches.length, "dispatch", "dispatches")}${blocked.length ? `, ${count(blocked.length, "Task")} blocked before dispatch` : ""}${failures.length ? `, ${count(failures.length, "failed dispatch", "failed dispatches")}` : ""}${skippedForeign ? `, ${count(skippedForeign, "Task")} left to another Account's Foreman` : ""}${queued.length ? `, ${count(queued.length, "Task")} left Ready with a pin this Run has no Account for` : ""}`;
|
|
1179
|
+
log(`run: no Ready Task remains in Initiative ${initiativeId}; stopping after ${tally}`);
|
|
1180
|
+
break;
|
|
1181
|
+
}
|
|
1182
|
+
if (taken) {
|
|
1183
|
+
const inUse = lanes.map((l) => `"${l.account.name}" ${l.slots.inFlight}/${l.slots.cap}`).join(", ");
|
|
1184
|
+
log(`run: ${count(taken, "Slot")} filled, ${inUse} in use${slotLimited ? "; Ready Tasks are waiting for a Slot" : ""}`);
|
|
1185
|
+
}
|
|
1186
|
+
if (waitingOnReset && taken === 0 && inFlight.size === 0) {
|
|
1187
|
+
log(`run: every Account a Ready Task could use is Paused; waiting for the reset at ${waitingOnReset.toISOString()}`);
|
|
1188
|
+
}
|
|
1189
|
+
if (waitingOnBackoff !== undefined && taken === 0 && inFlight.size === 0) {
|
|
1190
|
+
log(`run: every Ready Task is waiting out the delay after its last failed attempt; the next may be dispatched in ${Math.max(0, Math.round((waitingOnBackoff - now()) / 1000))}s`);
|
|
1191
|
+
}
|
|
1192
|
+
// One dispatch was asked for and no Lane can take it: say why and stop, rather than idling
|
|
1193
|
+
// out a pause that is up to `MAX_PAUSE_MS` away with nobody watching. The daemon's own Runs
|
|
1194
|
+
// are `--once` too, and a request it cannot serve now belongs back in its queue.
|
|
1195
|
+
if (single && taken === 0 && inFlight.size === 0 && (waitingOnReset || laneContended)) {
|
|
1196
|
+
log(
|
|
1197
|
+
waitingOnReset
|
|
1198
|
+
? `run: one dispatch was asked for and every Account a Ready Task could use is Paused until ${waitingOnReset.toISOString()}; nothing is dispatched. Try again after the reset, or run it on another Account (--account <name>).`
|
|
1199
|
+
: `run: one dispatch was asked for and every Account a Ready Task could use has Workers under another Foreman; nothing is dispatched. Wait for that Foreman, or run it on another Account (--account <name>).`,
|
|
1200
|
+
);
|
|
1201
|
+
break;
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
// Wait for something to change: a dispatch settling (its Task may release successors); or,
|
|
1206
|
+
// when Ready Tasks were waiting, a Slot coming back on any Lane that could take one — which a
|
|
1207
|
+
// dispatch entering verification gives without settling anything; or, when the only thing in
|
|
1208
|
+
// the way is a Paused Account, the reset itself, capped at the pause poll so a pause lifted
|
|
1209
|
+
// by hand (or by another Foreman) is noticed too.
|
|
1210
|
+
const waits: Promise<unknown>[] = [...inFlight.values()];
|
|
1211
|
+
if (slotLimited) for (const lane of lanes) if (!paused.has(lane.account.name)) waits.push(lane.slots.nextRelease());
|
|
1212
|
+
if (waitingOnReset) waits.push(sleep(Math.max(1, Math.min(waitingOnReset.getTime() - now(), pausePollMs))));
|
|
1213
|
+
// Unlike a pause, nothing but this Run's own clock can end a backoff early, so it is slept
|
|
1214
|
+
// out whole rather than polled. `Math.max(1, ...)` keeps a wait that has just elapsed from
|
|
1215
|
+
// becoming a zero-length sleep the loop would spin on.
|
|
1216
|
+
if (waitingOnBackoff !== undefined) waits.push(sleep(Math.max(1, waitingOnBackoff - now())));
|
|
1217
|
+
// Another Foreman's Workers on the only Lane left: nothing here is notified when they end, so
|
|
1218
|
+
// the Process Table is polled at the same interval a pause is.
|
|
1219
|
+
if (laneContended && !waitingOnReset) waits.push(sleep(pausePollMs));
|
|
1220
|
+
if (waits.length === 0) break;
|
|
1221
|
+
await Promise.race(waits);
|
|
1222
|
+
if (deps.tickDelayMs) await sleep(deps.tickDelayMs);
|
|
1223
|
+
}
|
|
1224
|
+
} finally {
|
|
1225
|
+
await drain();
|
|
1226
|
+
}
|
|
1227
|
+
return await summarised(deps, initiativeId, { dispatches, blocked, failures, ...(queued.length ? { queued } : {}), learnings: await submitLearnings(deps, initiativeId, dispatches) });
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
/**
|
|
1231
|
+
* Leaves the Run's summary on the Initiative and hands the outcome back unchanged (story 62).
|
|
1232
|
+
*
|
|
1233
|
+
* Here rather than in the command, so the daemon's Runs get it too: `foreman run` and `foreman up`
|
|
1234
|
+
* both come through `runForeman` and both used to end with a line on a terminal nobody was watching
|
|
1235
|
+
* at 3am. Every exit goes through this, an early stop included — a Breaker halting dispatch is the
|
|
1236
|
+
* night that most needs explaining, and it is exactly the night that would otherwise be silent.
|
|
1237
|
+
*
|
|
1238
|
+
* The Run's own facts are already committed by the time this runs: the Run Record has every event
|
|
1239
|
+
* and each Task carries its comment and its Status Label. So a Tracker that refuses this write costs
|
|
1240
|
+
* the morning its summary and nothing else, and it must not turn a Run that worked into a Run that
|
|
1241
|
+
* threw on its last line.
|
|
1242
|
+
*/
|
|
1243
|
+
async function summarised(deps: RunDeps, initiativeId: number, outcome: RunOutcome): Promise<RunOutcome> {
|
|
1244
|
+
try {
|
|
1245
|
+
// No line on success. `submitLearnings`, the other last step of a Run, reports through the
|
|
1246
|
+
// outcome and logs only what went wrong; a routine write announcing itself every Run is noise
|
|
1247
|
+
// in a log read to find out what did not happen.
|
|
1248
|
+
await deps.tracker.addInitiativeComment(initiativeId, runSummaryComment(outcome));
|
|
1249
|
+
} catch (error) {
|
|
1250
|
+
deps.log(`run: the Run summary could not be posted on Initiative ${initiativeId}, so the morning has only this terminal: ${error instanceof Error ? error.message : String(error)}`);
|
|
1251
|
+
}
|
|
1252
|
+
return outcome;
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
/** The Run's last step: its Proposed Learnings, gathered from every dispatch that reported, into
|
|
1256
|
+
* the learnings pull request. Its failure is the Run's to report, never to die of: the work is
|
|
1257
|
+
* handed off already, and the learnings are in Initiative memory whatever happens here. */
|
|
1258
|
+
async function submitLearnings(deps: RunDeps, initiativeId: number, dispatches: Dispatch[]): Promise<LearningsResult | undefined> {
|
|
1259
|
+
if (!deps.learnings) return undefined;
|
|
1260
|
+
const learnings = collectProposedLearnings(dispatches);
|
|
1261
|
+
try {
|
|
1262
|
+
return await deps.learnings({ initiativeId, learnings });
|
|
1263
|
+
} catch (error) {
|
|
1264
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
1265
|
+
deps.log(`learnings: failed: ${reason}`);
|
|
1266
|
+
return { kind: "failed", count: learnings.length, reason };
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
/**
|
|
1271
|
+
* One Reconcile pass for an Initiative: drain the outbox, then resume, restart or release every
|
|
1272
|
+
* Process Table row. The daemon runs this on its idle ticks too, per Initiative it has rows for.
|
|
1273
|
+
* Without a Process Table there is nothing to reconcile.
|
|
1274
|
+
*
|
|
1275
|
+
* Inside a Run the deps carry its Slots: a resumed or restarted turn is a turn on the Account like
|
|
1276
|
+
* any other, so it takes an implementer Slot and waits for one when the Run has none free. Rows the
|
|
1277
|
+
* Run is driving in its own Slots are left alone. The daemon's idle tick hands in plain `RunDeps`,
|
|
1278
|
+
* where there is no Run to share Slots with and its one Reconcile at a time is the whole of it.
|
|
1279
|
+
*/
|
|
1280
|
+
export async function reconcileTick(deps: RunDeps, initiativeId: number): Promise<Array<Dispatch | BlockedTask>> {
|
|
1281
|
+
if (deps.drainOutbox) await deps.drainOutbox(initiativeId);
|
|
1282
|
+
if (!deps.processTable) return [];
|
|
1283
|
+
const scheduling = "slots" in deps ? (deps as Scheduling) : undefined;
|
|
1284
|
+
// A Paused Account starts no new Workers, and a restart is a new Worker: a row with no session to
|
|
1285
|
+
// resume waits, exactly as it waits through a paused Initiative. Resuming is not held back — the
|
|
1286
|
+
// glossary's promise is that a Paused Account's running Workers stay resumable, and a resumed
|
|
1287
|
+
// session is the turn that was already going, not a new one.
|
|
1288
|
+
const current = deps.accounts?.load().find((a) => a.name === deps.account.name) ?? deps.account;
|
|
1289
|
+
const accountPaused = isPaused(current, (deps.now ?? Date.now)());
|
|
1290
|
+
if (accountPaused) deps.log(`reconcile: Account "${deps.account.name}" is Paused until ${current.paused?.until}; its Workers are resumed, nothing is dispatched afresh on it`);
|
|
1291
|
+
/** Runs one Reconcile turn in a Slot of the Run, when there is a Run to take one from. A Run that
|
|
1292
|
+
* is stopping does not wait for a Slot that is in use: the row stays as it is, for the next tick
|
|
1293
|
+
* or the next Foreman, and the loop gets its stop back without a Worker turn's delay. */
|
|
1294
|
+
const inSlot = async (row: WorkerRow, turn: (hold?: SlotHold) => Promise<Dispatch | BlockedTask>): Promise<Dispatch | BlockedTask | undefined> => {
|
|
1295
|
+
if (!scheduling) return turn();
|
|
1296
|
+
const hold = await scheduling.slots.acquireWhenFree(scheduling.untilStop);
|
|
1297
|
+
if (!hold) {
|
|
1298
|
+
deps.log(`reconcile: ${row.identifier} attempt ${row.attempt}: the Run is stopping and no Slot came free; leaving the row as it is`);
|
|
1299
|
+
return undefined;
|
|
1300
|
+
}
|
|
1301
|
+
try {
|
|
1302
|
+
return await turn(hold);
|
|
1303
|
+
} finally {
|
|
1304
|
+
hold.release();
|
|
1305
|
+
}
|
|
1306
|
+
};
|
|
1307
|
+
const { results } = await reconcile<Dispatch | BlockedTask>(
|
|
1308
|
+
{
|
|
1309
|
+
table: deps.processTable,
|
|
1310
|
+
tracker: deps.tracker,
|
|
1311
|
+
account: deps.account,
|
|
1312
|
+
log: deps.log,
|
|
1313
|
+
isAlive: deps.isAlive,
|
|
1314
|
+
paused: (id) => accountPaused || (deps.processTable?.isPaused(id) ?? false),
|
|
1315
|
+
owns: scheduling?.owns,
|
|
1316
|
+
resume: (row, sessionId, released) => inSlot(row, (hold) => resumeTurn(deps, initiativeId, row, sessionId, released, hold)),
|
|
1317
|
+
restart: (row) => inSlot(row, (hold) => restartTurn(deps, initiativeId, row, hold)),
|
|
1318
|
+
cancel: (row) => cancelTurn(deps, initiativeId, row),
|
|
1319
|
+
finishSettlement: (row, outcome) => finishSettlement(deps, initiativeId, row, outcome),
|
|
1320
|
+
},
|
|
1321
|
+
initiativeId,
|
|
1322
|
+
);
|
|
1323
|
+
return results;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
/** Finishes what a crash between the attempt event and the board left undone. The attempt event is
|
|
1327
|
+
* the commit (ADR-0009); memory is agent-written and may already be there, so only the comment and
|
|
1328
|
+
* the board are replayed: a Task whose attempt completed goes to In review under `foreman:done`
|
|
1329
|
+
* when the event names a Hand-off and the attempt's `gate` events say its last round passed (or
|
|
1330
|
+
* there are none, the recipe having named no Gate), else it is blocked, and the comment points at
|
|
1331
|
+
* the events. The full report is not reconstructed: the transcript and the event's summary carry
|
|
1332
|
+
* what a reader needs. */
|
|
1333
|
+
async function finishSettlement(deps: RunDeps, initiativeId: number, row: WorkerRow, outcome: string): Promise<void> {
|
|
1334
|
+
const { tracker, log } = deps;
|
|
1335
|
+
const { task } = await tracker.getTask(row.taskId);
|
|
1336
|
+
const { events } = await tracker.readRunRecord(initiativeId, { taskId: row.taskId, kinds: ["attempt"] });
|
|
1337
|
+
// Never the Triage turn's, which carries this same attempt number and is written before the
|
|
1338
|
+
// dispatch event (core/triage.ts `TRIAGE_MARKER`): settling from it would report the drafting
|
|
1339
|
+
// turn's outcome as the implementer attempt's.
|
|
1340
|
+
const event = events.find(
|
|
1341
|
+
(e) => e.payload?.payload.case === "attempt" && e.payload.payload.value.attempt === row.attempt && !isTriageSummary(e.payload.payload.value.summary),
|
|
1342
|
+
);
|
|
1343
|
+
// The attempt event is the commit (ADR-0009): it carries the Hand-off URL when the draft pull
|
|
1344
|
+
// request was opened, whether or not the Gates then held it. The Gates' own events, written before
|
|
1345
|
+
// the attempt, carry the verdict, so both are read: a URL with a failed last round is a blocked
|
|
1346
|
+
// Task whose pull request is still open, not a handed-off one.
|
|
1347
|
+
const handOffUrl = event?.payload?.payload.case === "attempt" ? event.payload.payload.value.handOffUrl : undefined;
|
|
1348
|
+
const verdict = gateVerdictOf((await tracker.readRunRecord(initiativeId, { taskId: row.taskId, kinds: ["gate"] })).events, row.attempt);
|
|
1349
|
+
// The Verifier's word, written after the Gates': a failed Verdict holds the Hand-off the way a
|
|
1350
|
+
// failed Gate does. The dispatch events are read with it: a Verdict written without an app (the
|
|
1351
|
+
// round failed before the app started) is the attempt's by its place after that attempt's
|
|
1352
|
+
// dispatch event (verifier.ts, the invariant on `verdictEvent`).
|
|
1353
|
+
const verified = verdictStatusOf((await tracker.readRunRecord(initiativeId, { taskId: row.taskId, kinds: ["dispatch", "verdict"] })).events, row.attempt, row.identifier);
|
|
1354
|
+
const handedOff = outcome === "completed" && !!handOffUrl && verdict !== "failed" && verified !== "failed";
|
|
1355
|
+
const state = handedOff
|
|
1356
|
+
? ` Handed off as ${handOffUrl}${verdict === "passed" ? ", its Gates passed" : ""}${verified === "passed" ? ", its Verdict passed" : ""}.`
|
|
1357
|
+
: handOffUrl
|
|
1358
|
+
? verdict === "failed"
|
|
1359
|
+
? ` Hand-off ${handOffUrl} is held by its Gates: the last round failed. The Task stays In Progress under foreman:blocked; see the gate events and the transcript.`
|
|
1360
|
+
: ` Hand-off ${handOffUrl} is held by its Verdict: the Verifier found Defects. The Task stays In Progress under foreman:blocked; see the verdict event and the transcript.`
|
|
1361
|
+
: " The Task stays In Progress under foreman:blocked; see the attempt event and the transcript.";
|
|
1362
|
+
await tracker.addComment(task.id, `Foreman: attempt ${row.attempt} was recorded (${outcome}) but the Foreman stopped before finishing; settled now from the Run Record.${state}`);
|
|
1363
|
+
// A Task that died mid-round still wears it; detach is a no-op when it does not.
|
|
1364
|
+
await tracker.detachLabel(row.taskId, STATUS_LABELS.verifying);
|
|
1365
|
+
if (handedOff) {
|
|
1366
|
+
await tracker.detachLabel(task.id, STATUS_LABELS.running);
|
|
1367
|
+
await tracker.moveTask(task.id, "in_review");
|
|
1368
|
+
await markStatus(tracker, task, undefined, STATUS_LABELS.done);
|
|
1369
|
+
} else {
|
|
1370
|
+
await markStatus(tracker, task, STATUS_LABELS.running, STATUS_LABELS.blocked);
|
|
1371
|
+
}
|
|
1372
|
+
log(`reconcile: ${row.identifier} attempt ${row.attempt}: settlement finished (${handedOff ? "handed off" : handOffUrl ? (verdict === "failed" ? "blocked, Hand-off held by Gates" : "blocked, Hand-off held by the Verdict") : "blocked"})`);
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
/** Continues an interrupted attempt from its stored session id, in the same Workspace, and settles
|
|
1376
|
+
* it the way a fresh turn is settled; the attempt event carries a `resumed` marker. A `released`
|
|
1377
|
+
* row (story 54) is resumed the same way, told that a human had the session in between; its mark
|
|
1378
|
+
* comes off as the turn starts, so the row is an ordinary live Worker's again. */
|
|
1379
|
+
async function resumeTurn(deps: RunDeps, initiativeId: number, row: WorkerRow, sessionId: string, released = false, slot?: SlotHold): Promise<Dispatch> {
|
|
1380
|
+
const { tracker, worker, account, log } = deps;
|
|
1381
|
+
const now = deps.now ?? Date.now;
|
|
1382
|
+
const ctx = await tracker.getTask(row.taskId);
|
|
1383
|
+
const workspace: Workspace = { path: row.workspace, branch: row.branch, created: false };
|
|
1384
|
+
if (released) deps.processTable?.setControl(row.taskId, undefined, now());
|
|
1385
|
+
const started = now();
|
|
1386
|
+
const handle = worker.resume(sessionId, { account, cwd: workspace.path, prompt: resumePrompt(ctx.task, released), maxTurns: deps.maxTurns, stallTimeoutMs: deps.stallTimeoutMs });
|
|
1387
|
+
if (handle.pid !== undefined) deps.processTable?.setPid(row.taskId, handle.pid);
|
|
1388
|
+
log(`worker: ${account.vendor} resumed turn on ${row.identifier} in ${workspace.path} (session ${sessionId}${released ? ", after an attended turn" : ""})`);
|
|
1389
|
+
return observeTurn(deps, initiativeId, ctx, workspace, row.attempt, handle, { started, installNote: "", resumed: true, draftedVerification: draftedVerificationFrom(ctx.comments)?.verification, slot });
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
/** What a resumed Worker is told: the turn it was in ended with the Foreman, not with it; or, after
|
|
1393
|
+
* a release, that a human took the session over for a while and has handed it back. */
|
|
1394
|
+
export function resumePrompt(task: TrackerTask, released = false): string {
|
|
1395
|
+
const first = released
|
|
1396
|
+
? `A human attached to your session on Task ${task.identifier} ("${task.title}") for an attended turn and has now released it; the Foreman has restarted you unattended in the same Workspace with everything that was said and done.`
|
|
1397
|
+
: `The Foreman that ran your previous turn on Task ${task.identifier} ("${task.title}") stopped before you reported, and has restarted you in the same Workspace with your earlier context.`;
|
|
1398
|
+
return [first, "Check the working tree for what you had already done, continue from there, finish the Task, and end with the Completion Report."].join("\n");
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
/** Closes the attempt of a Worker `foreman cancel` stopped when the process that owned its turn is
|
|
1402
|
+
* not there to do it (it died, or the row was left by a daemon that is gone): the same settlement
|
|
1403
|
+
* the owner would have made, from the row alone. */
|
|
1404
|
+
async function cancelTurn(deps: RunDeps, initiativeId: number, row: WorkerRow): Promise<Dispatch> {
|
|
1405
|
+
const ctx = await deps.tracker.getTask(row.taskId);
|
|
1406
|
+
const workspace: Workspace = { path: row.workspace, branch: row.branch, created: false };
|
|
1407
|
+
return settleCancelled(deps, initiativeId, ctx, workspace, row.attempt, `\n\nThe process that owned the turn did not close the attempt; closed by Reconcile.`, { started: deps.now?.() ?? Date.now(), installNote: "", resumed: true });
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
/** A cancelled attempt (story 56): `stopped` on the Run Record with the cancelled marker first in
|
|
1411
|
+
* the summary, a comment saying the human cancelled it, the Task blocked, the row released. */
|
|
1412
|
+
async function settleCancelled(deps: RunDeps, initiativeId: number, ctx: TaskWithContext, workspace: Workspace, attempt: number, detail: string, obs: TurnObservation): Promise<Dispatch> {
|
|
1413
|
+
const { tracker, log } = deps;
|
|
1414
|
+
const task = ctx.task;
|
|
1415
|
+
const durationMs = (deps.now ?? Date.now)() - obs.started;
|
|
1416
|
+
const marker = obs.resumed ? `${RESUMED_MARKER} ` : "";
|
|
1417
|
+
await settle(
|
|
1418
|
+
tracker,
|
|
1419
|
+
initiativeId,
|
|
1420
|
+
task,
|
|
1421
|
+
{
|
|
1422
|
+
attempt,
|
|
1423
|
+
outcome: "stopped",
|
|
1424
|
+
summary: `${marker}${CANCELLED_MARKER} stopped by opsee foreman cancel${detail}${obs.installNote}\n\nTurn: ${durationMs}ms.`,
|
|
1425
|
+
memory: [],
|
|
1426
|
+
comment: () => `Foreman: attempt ${attempt} was cancelled (opsee foreman cancel ${task.id}); the Worker was stopped and the Task stays In Progress under ${STATUS_LABELS.blocked}. Its Workspace ${workspace.path} (branch ${workspace.branch}) is kept for the next attempt.`,
|
|
1427
|
+
gateRetries: deps.gateRetries ?? DEFAULT_GATE_RETRIES,
|
|
1428
|
+
},
|
|
1429
|
+
log,
|
|
1430
|
+
);
|
|
1431
|
+
deps.processTable?.removeWorker(task.id);
|
|
1432
|
+
log(`attempt: ${task.identifier} cancelled by the human after ${durationMs}ms; the Task is blocked`);
|
|
1433
|
+
return { task, workspace, attempt, outcome: "stopped", durationMs, install: obs.install, resumed: obs.resumed, cancelled: true };
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
/** A Worker that died before it had a session cannot be resumed: the Task is dispatched again,
|
|
1437
|
+
* which the Run Record counts as the next attempt. */
|
|
1438
|
+
async function restartTurn(deps: RunDeps, initiativeId: number, row: WorkerRow, slot?: SlotHold): Promise<Dispatch | BlockedTask> {
|
|
1439
|
+
deps.processTable?.removeWorker(row.taskId);
|
|
1440
|
+
const { task } = await deps.tracker.getTask(row.taskId);
|
|
1441
|
+
return dispatch(deps, initiativeId, task, slot);
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
/** The Task outside any Initiative is refused (story 15); so is one in a different Initiative. */
|
|
1445
|
+
export function assertInInitiative(task: TrackerTask, initiativeId: number): void {
|
|
1446
|
+
if (task.initiativeId === undefined || task.initiativeId === 0) {
|
|
1447
|
+
throw new ForemanError(
|
|
1448
|
+
`Task ${task.identifier} (id ${task.id}) is not in any Initiative. The Foreman only runs Initiative Tasks, so every Completion Report has a memory log to land in: link it to Initiative ${initiativeId} (or another) and label it "${DISPATCH_LABEL}".`,
|
|
1449
|
+
);
|
|
1450
|
+
}
|
|
1451
|
+
if (task.initiativeId !== initiativeId) {
|
|
1452
|
+
throw new ForemanError(`Task ${task.identifier} (id ${task.id}) is in Initiative ${task.initiativeId}, not Initiative ${initiativeId}.`);
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
/**
|
|
1457
|
+
* The Ready Tasks this tick may fill Slots with, in dispatch order: priority descending, then
|
|
1458
|
+
* oldest first (core/scheduler.ts, story 28). Readiness is the Tracker's (ADR-0008); what is
|
|
1459
|
+
* dropped here is only what this Run must not pick again — a Task it already has in flight, and one
|
|
1460
|
+
* it deferred. `--task` names one Task instead and is refused when that Task is not Ready.
|
|
1461
|
+
*/
|
|
1462
|
+
async function pickTasks(
|
|
1463
|
+
tracker: TrackerAdapter,
|
|
1464
|
+
options: RunOptions,
|
|
1465
|
+
deferred: ReadonlySet<number>,
|
|
1466
|
+
inFlight: ReadonlyMap<number, unknown>,
|
|
1467
|
+
): Promise<TrackerTask[]> {
|
|
1468
|
+
const ready = await tracker.listReadyTasks(options.initiativeId);
|
|
1469
|
+
if (!ready.dispatchLabelExists) {
|
|
1470
|
+
throw new ForemanError(`Initiative ${options.initiativeId}: the project has no "${ready.dispatchLabel}" label, so no Task can be ready.`);
|
|
1471
|
+
}
|
|
1472
|
+
if (options.taskId === undefined) {
|
|
1473
|
+
return orderReadyTasks(ready.tasks.filter((t) => !deferred.has(t.id) && !inFlight.has(t.id)));
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
const { task } = await tracker.getTask(options.taskId);
|
|
1477
|
+
assertInInitiative(task, options.initiativeId);
|
|
1478
|
+
const hit = ready.tasks.find((t) => t.id === options.taskId);
|
|
1479
|
+
if (!hit) {
|
|
1480
|
+
throw new ForemanError(
|
|
1481
|
+
`Task ${task.identifier} (id ${task.id}) is not Ready in Initiative ${options.initiativeId}: it needs the "${ready.dispatchLabel}" label, an open column, and every blocker Done.`,
|
|
1482
|
+
);
|
|
1483
|
+
}
|
|
1484
|
+
return [hit];
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
function runEvent(taskId: number | undefined, payload: MessageInitShape<typeof RunEventInputSchema>["payload"]): RunEventInput {
|
|
1488
|
+
return create(RunEventInputSchema, { taskId, payload });
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
/** `runEvent` for a fact that happened at a moment other than now: the server stamps `occurredAt`
|
|
1492
|
+
* itself when it is absent, which is right for everything the Run does as it does it, and wrong for
|
|
1493
|
+
* a lift a human made before this Run started (`reportResumedAccounts`). */
|
|
1494
|
+
function runEventAt(taskId: number | undefined, occurredAt: Date | undefined, payload: MessageInitShape<typeof RunEventInputSchema>["payload"]): RunEventInput {
|
|
1495
|
+
return create(RunEventInputSchema, { taskId, payload, ...(occurredAt ? { occurredAt: timestampFromMs(occurredAt.getTime()) } : {}) });
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
/** A `spend` event: what one turn cost, on which Account. Written for any turn that reported a
|
|
1499
|
+
* cost whatever its outcome, because a turn that failed still spent; only a `completed` turn
|
|
1500
|
+
* carries one, since that is the only terminal event the vendors put a total on.
|
|
1501
|
+
*
|
|
1502
|
+
* Credits are the Run Record's unit -- a thousandth of a dollar (`InitiativeCostRollup`: 1 credit
|
|
1503
|
+
* = $0.001) -- rounded to the nearest whole one. A cost that rounds to nothing is left unwritten
|
|
1504
|
+
* rather than recorded as free: an event saying a turn cost zero is worse than no event, because
|
|
1505
|
+
* the rollup would sum it as fact.
|
|
1506
|
+
*
|
|
1507
|
+
* `model`, `inputTokens` and `outputTokens` stay unset. The streams this Foreman reads report a
|
|
1508
|
+
* turn's total and not its token split, and a zero in those fields would read as "no tokens" where
|
|
1509
|
+
* the truth is "not reported". */
|
|
1510
|
+
export function spendEvent(taskId: number, vendor: Vendor, accountName: string, costUsd: number | undefined): RunEventInput | undefined {
|
|
1511
|
+
if (typeof costUsd !== "number" || !Number.isFinite(costUsd) || costUsd <= 0) return undefined;
|
|
1512
|
+
const credits = Math.round(costUsd * 1000);
|
|
1513
|
+
if (credits <= 0) return undefined;
|
|
1514
|
+
return runEvent(taskId, { payload: { case: "spend", value: { credits: BigInt(credits), vendor: RUN_RECORD_VENDOR[vendor], account: accountName } } });
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
/** A `control` event (stories 53-56): what the human did, from the CLI. Task-scoped for attach,
|
|
1518
|
+
* release and cancel; Run-wide for pause and resume. */
|
|
1519
|
+
/** The state an Account moved to, as the Run Record names it (`RunAccountEvent.state`). */
|
|
1520
|
+
export type AccountState = "paused" | "quarantined" | "active";
|
|
1521
|
+
|
|
1522
|
+
/**
|
|
1523
|
+
* An Account's state on the Run Record: the vendor refused a turn on it, its credential was judged
|
|
1524
|
+
* dead, or a human brought it back.
|
|
1525
|
+
*
|
|
1526
|
+
* `state` is where the Account arrived rather than what happened to it, so one vocabulary covers a
|
|
1527
|
+
* lift from either a pause or a quarantine. What identifies the Account is the label its operator
|
|
1528
|
+
* gave it and nothing else — never the config directory or key variable behind it (ADR-0013), which
|
|
1529
|
+
* is the same line `RunDispatchEvent.account` holds.
|
|
1530
|
+
*
|
|
1531
|
+
* `taskId` is the turn that discovered it, for a pause or a quarantine; an Account brought back was
|
|
1532
|
+
* caused by no Task and carries none.
|
|
1533
|
+
*/
|
|
1534
|
+
export function accountEvent(state: AccountState, vendor: Vendor, accountName: string, reason: string, options: { taskId?: number; until?: Date; occurredAt?: Date } = {}): RunEventInput {
|
|
1535
|
+
return runEventAt(options.taskId, options.occurredAt, {
|
|
1536
|
+
payload: {
|
|
1537
|
+
case: "account",
|
|
1538
|
+
value: {
|
|
1539
|
+
state,
|
|
1540
|
+
vendor: RUN_RECORD_VENDOR[vendor],
|
|
1541
|
+
account: accountName,
|
|
1542
|
+
// The vendor's own prose in places, on its way to a record other people read: one printable
|
|
1543
|
+
// line, as it is for the accounts file and the terminal views.
|
|
1544
|
+
reason: printableOneLine(reason, ACCOUNT_REASON_MAX),
|
|
1545
|
+
...(options.until ? { until: timestampFromMs(options.until.getTime()) } : {}),
|
|
1546
|
+
},
|
|
1547
|
+
},
|
|
1548
|
+
});
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
/** What `RunAccountEvent.reason` takes (proto: `max_len: 4000`). Well under it: the reason is one
|
|
1552
|
+
* line of a vendor's refusal, and the Run Record is not where a stack trace belongs. */
|
|
1553
|
+
const ACCOUNT_REASON_MAX = 500;
|
|
1554
|
+
|
|
1555
|
+
export function controlEvent(action: ControlAction, message: string, taskId?: number): RunEventInput {
|
|
1556
|
+
return runEvent(taskId, { payload: { case: "control", value: { action, message } } });
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
function attemptOutcomeOf(reason: TurnFailureReason): AttemptOutcome {
|
|
1560
|
+
switch (reason) {
|
|
1561
|
+
case "stalled":
|
|
1562
|
+
return "stalled";
|
|
1563
|
+
case "rate_limited":
|
|
1564
|
+
return "rate_limited";
|
|
1565
|
+
case "stopped":
|
|
1566
|
+
return "stopped";
|
|
1567
|
+
// A credential failure has no outcome of its own: `RunAttemptEvent.outcome` is a closed set in
|
|
1568
|
+
// the proto and widening it is a change to four generated trees for a fact the summary already
|
|
1569
|
+
// carries in words. Written as its own case rather than left to the default, so that reading
|
|
1570
|
+
// this function answers the question instead of raising it.
|
|
1571
|
+
case "credential_failed":
|
|
1572
|
+
return "failed";
|
|
1573
|
+
default:
|
|
1574
|
+
return "failed";
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
/**
|
|
1579
|
+
* How many attempts on this Task have already ended in a `RETRYABLE_FAILURES` outcome, read back
|
|
1580
|
+
* off the Run Record: what the retry cap is spent on (story 34).
|
|
1581
|
+
*
|
|
1582
|
+
* Deliberately not `attempt - 1`. The attempt number counts every turn the Task ever had, whatever
|
|
1583
|
+
* ended it — a failed Gate round, a Verdict with Defects, a human relabelling a blocked Task — and
|
|
1584
|
+
* none of those is a machine-side failure the cap exists to bound. A Triage turn that ended
|
|
1585
|
+
* `rate_limited` does count: it is a turn on the Account that the vendor refused, exactly the thing
|
|
1586
|
+
* another attempt is worth trying.
|
|
1587
|
+
*/
|
|
1588
|
+
async function retryableFailuresOf(tracker: TrackerAdapter, initiativeId: number, taskId: number): Promise<number> {
|
|
1589
|
+
const { events } = await tracker.readRunRecord(initiativeId, { taskId, kinds: ["attempt"] });
|
|
1590
|
+
return events.filter((e) => {
|
|
1591
|
+
const payload = e.payload?.payload;
|
|
1592
|
+
if (payload?.case !== "attempt") return false;
|
|
1593
|
+
// The outcome answers for `stalled` and `rate_limited`; a credential failure was recorded as
|
|
1594
|
+
// the generic `failed` and is recognised by its marker instead (`RETRYABLE_FAILURE_REASONS`).
|
|
1595
|
+
return RETRYABLE_FAILURES.has(payload.value.outcome as AttemptOutcome) || isCredentialSummary(payload.value.summary);
|
|
1596
|
+
}).length;
|
|
1597
|
+
}
|
|
1598
|
+
|
|
1599
|
+
/** Whether an attempt event's summary is a turn that failed for a credential reason. The marker is
|
|
1600
|
+
* written after the resumed and triage markers, so this is a search rather than a prefix test. */
|
|
1601
|
+
export function isCredentialSummary(summary: string): boolean {
|
|
1602
|
+
return summary.includes(CREDENTIAL_MARKER);
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
/** Swaps the Task's Status Label: `from` comes off, `to` goes on (created in the project first). */
|
|
1606
|
+
async function markStatus(tracker: TrackerAdapter, task: TrackerTask, from: StatusLabel | undefined, to: StatusLabel): Promise<void> {
|
|
1607
|
+
if (from) await tracker.detachLabel(task.id, from);
|
|
1608
|
+
await tracker.ensureLabel(task.projectId, to);
|
|
1609
|
+
await tracker.attachLabel(task.id, to);
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
type HandedOff = Extract<HandOffResult, { kind: "handed_off" }>;
|
|
1613
|
+
|
|
1614
|
+
interface Settlement {
|
|
1615
|
+
attempt: number;
|
|
1616
|
+
outcome: AttemptOutcome;
|
|
1617
|
+
/** The attempt event's summary: what happened, the install exit code, the output tail. */
|
|
1618
|
+
summary: string;
|
|
1619
|
+
/** The linking comment, given the ids of the memory entries just written (outcome first) and
|
|
1620
|
+
* the Run loop's note about the Hand-off. */
|
|
1621
|
+
comment: (memoryIds: number[], note: string) => string;
|
|
1622
|
+
/** Memory entries to write before the comment; empty for a turn without a report. */
|
|
1623
|
+
memory: Parameters<TrackerAdapter["addMemory"]>[1][];
|
|
1624
|
+
/** The Hand-off of a `done` turn. Open: the Task goes to In review under `foreman:done` with the
|
|
1625
|
+
* pull request linked, provided the Gates passed. Incomplete or absent: it stays In Progress
|
|
1626
|
+
* under `foreman:blocked`. */
|
|
1627
|
+
handOff?: HandOffResult;
|
|
1628
|
+
/** The Gates that ran on the Hand-off. Failed: the pull request is still linked and named in the
|
|
1629
|
+
* attempt event, but the Task stays In Progress under `foreman:blocked`. */
|
|
1630
|
+
gates?: GateOutcome;
|
|
1631
|
+
/** The Verifier's round after passing Gates. Defects or a failed round: the pull request stays
|
|
1632
|
+
* linked, the Task stays In Progress under `foreman:blocked`. */
|
|
1633
|
+
verification?: VerifyOutcome;
|
|
1634
|
+
gateRetries: number;
|
|
1635
|
+
/** Set when the attempt failed for a reason worth another go under the retry cap
|
|
1636
|
+
* (`RETRYABLE_FAILURES`): the Task gets its dispatch label back instead of `foreman:blocked`, so
|
|
1637
|
+
* the Tracker counts it Ready again and a later tick dispatches the next attempt — on a sibling
|
|
1638
|
+
* Account when its own is Paused by then (story 31), and no sooner than `waitMs` from now
|
|
1639
|
+
* (`retryBackoff`). */
|
|
1640
|
+
retry?: { attemptsLeft: number; waitMs: number };
|
|
1641
|
+
/** Set when that same kind of failure has used the retry cap up (story 34): the Task is blocked
|
|
1642
|
+
* like any other failed attempt, and the comment says which cause spent the attempts, so a human
|
|
1643
|
+
* reading the Task alone knows whether to retry it or to fix something first. `attempts` is how
|
|
1644
|
+
* many attempts ended in a retryable failure (`retryableFailuresOf`, this one included), which is
|
|
1645
|
+
* what the comment claims — never the Task's total dispatch count. */
|
|
1646
|
+
capped?: { cause: TurnFailureReason; attempts: number; retries: number };
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
/** Records how a turn ended: the attempt event first (ADR-0009, carrying the Hand-off URL when
|
|
1650
|
+
* there is one), then the pull request link, memory, the comment, and the board with its Status
|
|
1651
|
+
* Label. Returns the ids of the memory entries written. */
|
|
1652
|
+
async function settle(tracker: TrackerAdapter, initiativeId: number, task: TrackerTask, s: Settlement, log: (line: string) => void): Promise<number[]> {
|
|
1653
|
+
const opened = s.handOff?.kind === "handed_off" ? s.handOff : undefined;
|
|
1654
|
+
const gatesPassed = s.gates?.passed ?? true;
|
|
1655
|
+
const handedOff = gatesPassed && verificationPassed(s.verification) ? opened : undefined;
|
|
1656
|
+
await tracker.appendRunEvents(initiativeId, [
|
|
1657
|
+
runEvent(task.id, { payload: { case: "attempt", value: { attempt: s.attempt, outcome: s.outcome, summary: attemptSummary(s.summary), handOffUrl: opened?.pullRequest.url } } }),
|
|
1658
|
+
]);
|
|
1659
|
+
let note = "";
|
|
1660
|
+
if (opened) note = await linkHandOff(tracker, task, opened, log);
|
|
1661
|
+
else if (s.handOff?.kind === "incomplete") note = ` Hand-off incomplete: ${s.handOff.reason}. The Task stays In Progress.`;
|
|
1662
|
+
if (s.gates) note += gateNote(s.gates, s.gateRetries);
|
|
1663
|
+
if (s.verification) note += verifyNote(s.verification);
|
|
1664
|
+
const retry = handedOff ? undefined : s.retry;
|
|
1665
|
+
if (retry) note += ` The Task is Ready again and the Foreman dispatches it as attempt ${s.attempt + 1}${inSeconds(retry.waitMs)}: ${count(retry.attemptsLeft, "attempt")} left under the retry cap.`;
|
|
1666
|
+
const capped = handedOff ? undefined : s.capped;
|
|
1667
|
+
if (capped) {
|
|
1668
|
+
note +=
|
|
1669
|
+
` Every attempt this Task gets is spent: ${count(capped.attempts, "attempt")} ended ${capped.cause}, which is the retry cap (${count(capped.retries, "further attempt")}) reached.` +
|
|
1670
|
+
` It stays In Progress under ${STATUS_LABELS.blocked} for a human, and the Foreman goes on with the Initiative's other Tasks.`;
|
|
1671
|
+
}
|
|
1672
|
+
const ids: number[] = [];
|
|
1673
|
+
for (const entry of s.memory) ids.push((await tracker.addMemory(initiativeId, entry)).id);
|
|
1674
|
+
await tracker.addComment(task.id, s.comment(ids, note));
|
|
1675
|
+
await tracker.detachLabel(task.id, STATUS_LABELS.verifying);
|
|
1676
|
+
if (handedOff) {
|
|
1677
|
+
await tracker.detachLabel(task.id, STATUS_LABELS.running);
|
|
1678
|
+
await tracker.moveTask(task.id, "in_review");
|
|
1679
|
+
await markStatus(tracker, task, undefined, STATUS_LABELS.done);
|
|
1680
|
+
} else if (retry) {
|
|
1681
|
+
// The dispatch label back on is what makes the Task Ready again; the column is already an open
|
|
1682
|
+
// one (In Progress), so the Tracker's own readiness rule lets the next tick pick it up.
|
|
1683
|
+
await tracker.detachLabel(task.id, STATUS_LABELS.running);
|
|
1684
|
+
await tracker.attachLabel(task.id, DISPATCH_LABEL);
|
|
1685
|
+
} else {
|
|
1686
|
+
await markStatus(tracker, task, STATUS_LABELS.running, STATUS_LABELS.blocked);
|
|
1687
|
+
}
|
|
1688
|
+
return ids;
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
/** Links the pull request to the Task through the project's connected repository that matches
|
|
1692
|
+
* the origin, so it rolls up into the Initiative. With no match the link is not guessed: the
|
|
1693
|
+
* comment says which repository to connect. Returns the note for the comment. */
|
|
1694
|
+
async function linkHandOff(tracker: TrackerAdapter, task: TrackerTask, handOff: HandedOff, log: (line: string) => void): Promise<string> {
|
|
1695
|
+
const repositories = await tracker.projectRepositories(task.projectId);
|
|
1696
|
+
const wanted = handOff.repositoryFullName?.toLowerCase();
|
|
1697
|
+
const repository =
|
|
1698
|
+
repositories.find((r) => wanted !== undefined && r.fullName.toLowerCase() === wanted) ??
|
|
1699
|
+
(wanted === undefined && repositories.length === 1 ? repositories[0] : undefined);
|
|
1700
|
+
if (!repository) {
|
|
1701
|
+
const connected = repositories.map((r) => r.fullName).join(", ") || "none";
|
|
1702
|
+
log(`hand-off: ${task.identifier} not linked: no connected repository matches origin ${wanted ?? "(unknown)"} (connected: ${connected})`);
|
|
1703
|
+
return ` Not linked to a repository: the project has no connected repository "${wanted ?? "?"}" (connected: ${connected}); connect it so the pull request rolls up into the Initiative.`;
|
|
1704
|
+
}
|
|
1705
|
+
const { pullRequest } = handOff;
|
|
1706
|
+
await tracker.linkPullRequest(task.id, {
|
|
1707
|
+
projectRepositoryId: repository.id,
|
|
1708
|
+
number: pullRequest.number,
|
|
1709
|
+
url: pullRequest.url,
|
|
1710
|
+
title: pullRequest.title,
|
|
1711
|
+
headBranch: pullRequest.headBranch,
|
|
1712
|
+
headSha: handOff.headSha,
|
|
1713
|
+
});
|
|
1714
|
+
log(`hand-off: ${task.identifier} linked ${pullRequest.url} to repository ${repository.fullName}`);
|
|
1715
|
+
return "";
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
/** What the Task comment says about the Gates: short when they passed, and naming the Gate,
|
|
1719
|
+
* its command and exit code when they did not (story 34). */
|
|
1720
|
+
export function gateNote(gates: GateOutcome, retries: number): string {
|
|
1721
|
+
// Said as plainly as a skipped Verifier says it (`verifyNote`): a reader of the Task alone must
|
|
1722
|
+
// never be able to conclude the work was checked when nothing checked it.
|
|
1723
|
+
if (gates.reason === "skipped") {
|
|
1724
|
+
return ` Gates ${gateSummary(gates, retries)}. Nothing on this machine ran the repository's tests, lint or typecheck against this Hand-off; read it as unchecked, not as passing.`;
|
|
1725
|
+
}
|
|
1726
|
+
if (gates.passed) {
|
|
1727
|
+
return ` Gates passed in ${count(gates.rounds.length, "round")}${gates.resumes ? ` after ${count(gates.resumes, "resumed turn")}` : ""}.`;
|
|
1728
|
+
}
|
|
1729
|
+
return ` Gates ${gateSummary(gates, retries)}. The Task stays In Progress under ${STATUS_LABELS.blocked}.`;
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
/** What the Task comment says about the Verifier: passed, skipped and why, or the Defects. */
|
|
1733
|
+
export function verifyNote(outcome: VerifyOutcome): string {
|
|
1734
|
+
if (verificationPassed(outcome)) return ` Verification ${verifySummary(outcome)}.`;
|
|
1735
|
+
return ` Verification ${verifySummary(outcome)}. The Task stays In Progress under ${STATUS_LABELS.blocked}.`;
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
/** Refuses a Ready Task before its dispatch event: the dispatch label comes off first so the Task
|
|
1739
|
+
* is not picked again next tick, then the comment says why, then the blocked Status Label. No Run
|
|
1740
|
+
* Record event: nothing was dispatched, and a Triage turn that ran has recorded itself already. */
|
|
1741
|
+
async function refuse(
|
|
1742
|
+
deps: RunDeps,
|
|
1743
|
+
task: TrackerTask,
|
|
1744
|
+
missing: WorkContractSection[],
|
|
1745
|
+
comment: string,
|
|
1746
|
+
why: string,
|
|
1747
|
+
after?: { triage: TriageTurn; workspace: Workspace },
|
|
1748
|
+
): Promise<BlockedTask> {
|
|
1749
|
+
const { tracker, log } = deps;
|
|
1750
|
+
log(`contract: ${task.identifier} blocked, ${why}; "${DISPATCH_LABEL}" removed${after ? `, Workspace ${after.workspace.path} kept` : ""}`);
|
|
1751
|
+
await tracker.detachLabel(task.id, DISPATCH_LABEL);
|
|
1752
|
+
await tracker.addComment(task.id, comment);
|
|
1753
|
+
await markStatus(tracker, task, undefined, STATUS_LABELS.blocked);
|
|
1754
|
+
return { task, missing, ...after };
|
|
1755
|
+
}
|
|
1756
|
+
|
|
1757
|
+
/** Runs the Triage turn and records it: the attempt event first (ADR-0009), then the draft as a
|
|
1758
|
+
* comment, which is the record later slices read (`draftedVerificationFrom`). */
|
|
1759
|
+
async function triageTurn(deps: RunDeps, initiativeId: number, task: TrackerTask, contract: WorkContract, workspace: Workspace, attempt: number, now: () => number): Promise<TriageTurn> {
|
|
1760
|
+
const { tracker, worker, account, log } = deps;
|
|
1761
|
+
const { terminal, durationMs, outputTail, rateLimit } = await driveTriageTurn({
|
|
1762
|
+
worker,
|
|
1763
|
+
account,
|
|
1764
|
+
task,
|
|
1765
|
+
contract,
|
|
1766
|
+
workspace,
|
|
1767
|
+
maxTurns: deps.maxTurns,
|
|
1768
|
+
stallTimeoutMs: deps.stallTimeoutMs,
|
|
1769
|
+
log,
|
|
1770
|
+
now,
|
|
1771
|
+
});
|
|
1772
|
+
const trailer = `\n\nTurn: ${durationMs}ms.${outputTail ? `\n\nOutput tail:\n${outputTail}` : ""}`;
|
|
1773
|
+
// A Triage turn is a turn on the Account, so a limit it hit Pauses the Account exactly as a
|
|
1774
|
+
// Worker turn's would (story 30); the Task itself keeps its dispatch label and is retried.
|
|
1775
|
+
if (terminal.type === "failed" && terminal.reason === "rate_limited") await pauseOnRateLimit(deps, initiativeId, task.id, rateLimit, terminal.message, now());
|
|
1776
|
+
// And a credential it refused counts against the Account exactly as a Worker turn's would
|
|
1777
|
+
// (OPS-288): a Triage turn is a turn on the Account, and a dead login refuses it identically.
|
|
1778
|
+
if (terminal.type === "failed" && terminal.reason === "credential_failed") await noteCredentialFailureOnAccount(deps, initiativeId, task.id, terminal.message, now());
|
|
1779
|
+
// A Triage turn the vendor accepted is proof the credential is live, so it closes an open streak
|
|
1780
|
+
// the same way an implementer turn does.
|
|
1781
|
+
if (terminal.type === "completed") clearCredentialStreak(deps);
|
|
1782
|
+
|
|
1783
|
+
let result: TriageTurn;
|
|
1784
|
+
let summary: string;
|
|
1785
|
+
if (terminal.type === "completed") {
|
|
1786
|
+
const parsed = triageDraftFrom(terminal.report);
|
|
1787
|
+
if (parsed.ok) {
|
|
1788
|
+
result = { attempt, outcome: "completed", verification: parsed.draft.verification, sessionId: terminal.sessionId, durationMs };
|
|
1789
|
+
summary = `triage: drafted Verification section (${parsed.draft.verification.length} chars)${trailer}`;
|
|
1790
|
+
} else {
|
|
1791
|
+
result = { attempt, outcome: "completed", reason: parsed.reason, sessionId: terminal.sessionId, durationMs };
|
|
1792
|
+
summary = `triage: no draft: ${parsed.reason}${trailer}`;
|
|
1793
|
+
}
|
|
1794
|
+
} else {
|
|
1795
|
+
const reason = `${terminal.reason}: ${terminal.message}`;
|
|
1796
|
+
const details = terminal.details?.length ? `\n${terminal.details.join("\n")}` : "";
|
|
1797
|
+
result = { attempt, outcome: attemptOutcomeOf(terminal.reason), reason, sessionId: terminal.sessionId, durationMs };
|
|
1798
|
+
summary = `triage: ${reason}${details}${trailer}`;
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
// The attempt event has no account field, so the summary names it: a Triage turn is a turn on
|
|
1802
|
+
// the Account, and a reader of the Run Record must be able to charge it to one. The marker comes
|
|
1803
|
+
// first and is machine-read: this event shares its attempt number with the implementer turn that
|
|
1804
|
+
// follows it, and the two settlement paths tell them apart by it (core/triage.ts `TRIAGE_MARKER`).
|
|
1805
|
+
summary = `${TRIAGE_MARKER} ${summary.replace(/^triage:/, `triage on ${RUN_RECORD_VENDOR[account.vendor]} Account "${account.name}":`)}`;
|
|
1806
|
+
const turnSpend = spendEvent(task.id, account.vendor, account.name, terminal.type === "completed" ? terminal.costUsd : undefined);
|
|
1807
|
+
await tracker.appendRunEvents(initiativeId, [
|
|
1808
|
+
...(turnSpend ? [turnSpend] : []),
|
|
1809
|
+
runEvent(task.id, { payload: { case: "attempt", value: { attempt, outcome: result.outcome, summary: attemptSummary(summary) } } }),
|
|
1810
|
+
]);
|
|
1811
|
+
if (result.verification !== undefined) {
|
|
1812
|
+
await tracker.addComment(task.id, triageComment(result.verification));
|
|
1813
|
+
log(`triage: ${task.identifier} drafted Verification section (${result.verification.length} chars) after ${durationMs}ms`);
|
|
1814
|
+
} else {
|
|
1815
|
+
log(`triage: ${task.identifier} ${result.outcome} after ${durationMs}ms: ${result.reason}`);
|
|
1816
|
+
}
|
|
1817
|
+
return result;
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
async function dispatch(deps: RunDeps, initiativeId: number, picked: TrackerTask, slot?: SlotHold): Promise<Dispatch | BlockedTask> {
|
|
1821
|
+
const { tracker, worker, account, log } = deps;
|
|
1822
|
+
const now = deps.now ?? Date.now;
|
|
1823
|
+
const ctx = await tracker.getTask(picked.id);
|
|
1824
|
+
const task = ctx.task;
|
|
1825
|
+
assertInInitiative(task, initiativeId);
|
|
1826
|
+
|
|
1827
|
+
// The work contract, before anything is written: a Task that cannot be worked is refused here.
|
|
1828
|
+
const contract = parseWorkContract(task.description);
|
|
1829
|
+
const verdict = validateWorkContract(contract);
|
|
1830
|
+
if (verdict.status === "blocked") {
|
|
1831
|
+
return refuse(deps, task, verdict.missing, blockedComment(verdict.missing), `the work contract is missing: ${verdict.missing.join(", ")}`);
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
const priorDispatches = (await tracker.readRunRecord(initiativeId, { taskId: task.id, kinds: ["dispatch"] })).events.length;
|
|
1835
|
+
const attempt = priorDispatches + 1;
|
|
1836
|
+
const workspace = await deps.workspaces.create(task.identifier);
|
|
1837
|
+
|
|
1838
|
+
let triage: TriageTurn | undefined;
|
|
1839
|
+
let draftedVerification: string | undefined;
|
|
1840
|
+
if (verdict.status === "ok") {
|
|
1841
|
+
log(`contract: ${task.identifier} complete`);
|
|
1842
|
+
} else {
|
|
1843
|
+
const earlier = draftedVerificationFrom(ctx.comments);
|
|
1844
|
+
if (earlier) {
|
|
1845
|
+
log(`contract: ${task.identifier} has no Verification section; using the one drafted in comment #${earlier.commentId}`);
|
|
1846
|
+
draftedVerification = earlier.verification;
|
|
1847
|
+
} else {
|
|
1848
|
+
log(`contract: ${task.identifier} has no Verification section; Triage turn`);
|
|
1849
|
+
triage = await triageTurn(deps, initiativeId, task, contract, workspace, attempt, now);
|
|
1850
|
+
if (triage.verification === undefined) {
|
|
1851
|
+
// Only the Worker saying the contract is too thin to draft is a contract defect. A limit,
|
|
1852
|
+
// a stall, or an adapter failure is the Account's or the machine's, so the Task stays
|
|
1853
|
+
// Ready and the next tick retries, as a Worker turn in the same state would.
|
|
1854
|
+
if (triage.outcome === "completed") {
|
|
1855
|
+
return refuse(deps, task, verdict.missing, triageFailedComment(triage.reason ?? "no draft"), `the Triage turn drafted nothing (${triage.reason})`, { triage, workspace });
|
|
1856
|
+
}
|
|
1857
|
+
// A stall or a rate limit is worth another go within this Run; anything else that ended the
|
|
1858
|
+
// turn without a report (an adapter that would not start, a vendor error, a stop) is not:
|
|
1859
|
+
// no tick of this Run can change it, and looking again in a second would only spin. Only
|
|
1860
|
+
// the retryable outcomes write an attempt event `retryableFailuresOf` counts, so only they
|
|
1861
|
+
// have a cap to be bounded by.
|
|
1862
|
+
if (!RETRYABLE_FAILURES.has(triage.outcome)) {
|
|
1863
|
+
log(`contract: ${task.identifier} Triage turn ${triage.outcome} (${triage.reason}); "${DISPATCH_LABEL}" kept, left to the next Run`);
|
|
1864
|
+
await tracker.addComment(task.id, `Foreman: the Triage turn ended ${triage.outcome} (${triage.reason}); the Task stays Ready and the next Run retries it.`);
|
|
1865
|
+
return { task, missing: verdict.missing, triage, workspace, defer: true };
|
|
1866
|
+
}
|
|
1867
|
+
// Bounded by the same retry cap a failed dispatch is bounded by, and by nothing else: the
|
|
1868
|
+
// Triage turn's own attempt event is one of the retryable failures `retryableFailuresOf`
|
|
1869
|
+
// counts (deliberately, see its comment), so this cannot spin — but it is retried on a
|
|
1870
|
+
// later tick of this Run rather than deferred to the next one.
|
|
1871
|
+
const attemptRetries = deps.attemptRetries ?? DEFAULT_ATTEMPT_RETRIES;
|
|
1872
|
+
const spent = await retryableFailuresOf(tracker, initiativeId, task.id);
|
|
1873
|
+
const attemptsLeft = attemptRetries - (spent - 1);
|
|
1874
|
+
if (attemptsLeft <= 0) {
|
|
1875
|
+
return refuse(
|
|
1876
|
+
deps,
|
|
1877
|
+
task,
|
|
1878
|
+
verdict.missing,
|
|
1879
|
+
triageCappedComment(triage.outcome, triage.reason ?? "no reason given", spent, attemptRetries),
|
|
1880
|
+
`every Triage turn it is allowed ended ${triage.outcome} (${count(spent, "attempt")}, the retry cap)`,
|
|
1881
|
+
{ triage, workspace },
|
|
1882
|
+
);
|
|
1883
|
+
}
|
|
1884
|
+
const waitMs = retryBackoff(spent, deps.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS);
|
|
1885
|
+
log(`contract: ${task.identifier} Triage turn ${triage.outcome} (${triage.reason}); "${DISPATCH_LABEL}" kept, retried on a later tick${inSeconds(waitMs)} (${count(attemptsLeft, "attempt")} left under the retry cap)`);
|
|
1886
|
+
await tracker.addComment(
|
|
1887
|
+
task.id,
|
|
1888
|
+
`Foreman: the Triage turn ended ${triage.outcome} (${triage.reason}); the Task stays Ready and is retried${inSeconds(waitMs)}: ${count(attemptsLeft, "attempt")} left under the retry cap.`,
|
|
1889
|
+
);
|
|
1890
|
+
return { task, missing: verdict.missing, triage, workspace, retry: { attemptsLeft, notBefore: (deps.now ?? Date.now)() + waitMs } };
|
|
1891
|
+
}
|
|
1892
|
+
draftedVerification = triage.verification;
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
const origin = workspace.created ? `new Workspace${workspace.base ? ` from ${workspace.base}` : ""}` : "existing Workspace";
|
|
1897
|
+
log(`dispatch: ${task.identifier} "${task.title}" attempt ${attempt} -> ${workspace.path} (branch ${workspace.branch}, ${origin})`);
|
|
1898
|
+
|
|
1899
|
+
// Run Record first (ADR-0009), then the board. The dispatch label comes off at dispatch so the
|
|
1900
|
+
// Task is never picked twice; the Status Label says who has it.
|
|
1901
|
+
await tracker.appendRunEvents(initiativeId, [
|
|
1902
|
+
runEvent(task.id, {
|
|
1903
|
+
payload: {
|
|
1904
|
+
case: "dispatch",
|
|
1905
|
+
value: { vendor: RUN_RECORD_VENDOR[account.vendor], account: account.name, workspace: workspace.path, branch: workspace.branch, attempt },
|
|
1906
|
+
},
|
|
1907
|
+
}),
|
|
1908
|
+
]);
|
|
1909
|
+
// The row exists from the moment the dispatch is committed (story 50): a crash anywhere between
|
|
1910
|
+
// here and the vendor's `started` leaves a row with no session, which Reconcile restarts.
|
|
1911
|
+
deps.processTable?.upsertWorker({
|
|
1912
|
+
taskId: task.id,
|
|
1913
|
+
initiativeId,
|
|
1914
|
+
identifier: task.identifier,
|
|
1915
|
+
account: account.name,
|
|
1916
|
+
vendor: account.vendor,
|
|
1917
|
+
workspace: workspace.path,
|
|
1918
|
+
branch: workspace.branch,
|
|
1919
|
+
startedAt: now(),
|
|
1920
|
+
lastOutputAt: now(),
|
|
1921
|
+
attempt,
|
|
1922
|
+
});
|
|
1923
|
+
// From here the Task is being taken. Whatever goes wrong below that is not a typed turn outcome
|
|
1924
|
+
// (a Tracker write failing, a board without the column, an adapter rejecting) must not leave the
|
|
1925
|
+
// Task half-claimed with no trace: it is marked blocked with the reason, and the error still
|
|
1926
|
+
// propagates. The abort writes are best-effort so they cannot mask the original error.
|
|
1927
|
+
let running = false;
|
|
1928
|
+
try {
|
|
1929
|
+
await tracker.detachLabel(task.id, DISPATCH_LABEL);
|
|
1930
|
+
await tracker.moveTask(task.id, "in_progress");
|
|
1931
|
+
await markStatus(tracker, task, undefined, STATUS_LABELS.running);
|
|
1932
|
+
running = true;
|
|
1933
|
+
const done = await work(deps, initiativeId, ctx, workspace, attempt, now, draftedVerification, slot);
|
|
1934
|
+
return { ...done, triage, draftedVerification };
|
|
1935
|
+
} catch (error) {
|
|
1936
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1937
|
+
log(`dispatch: ${task.identifier} aborted: ${message}`);
|
|
1938
|
+
// No Worker is running for the Task any more, so the row would only ever be released.
|
|
1939
|
+
deps.processTable?.removeWorker(task.id);
|
|
1940
|
+
try {
|
|
1941
|
+
await tracker.addComment(task.id, `Foreman: dispatch aborted: ${message}`);
|
|
1942
|
+
await markStatus(tracker, task, running ? STATUS_LABELS.running : undefined, STATUS_LABELS.blocked);
|
|
1943
|
+
} catch (abortError) {
|
|
1944
|
+
log(`dispatch: ${task.identifier} could not record the abort: ${abortError instanceof Error ? abortError.message : String(abortError)}`);
|
|
1945
|
+
}
|
|
1946
|
+
throw error;
|
|
1947
|
+
}
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
async function work(
|
|
1951
|
+
deps: RunDeps,
|
|
1952
|
+
initiativeId: number,
|
|
1953
|
+
ctx: TaskWithContext,
|
|
1954
|
+
workspace: Workspace,
|
|
1955
|
+
attempt: number,
|
|
1956
|
+
now: () => number,
|
|
1957
|
+
draftedVerification?: string,
|
|
1958
|
+
slot?: SlotHold,
|
|
1959
|
+
): Promise<Dispatch> {
|
|
1960
|
+
const { tracker, worker, account, log } = deps;
|
|
1961
|
+
const task = ctx.task;
|
|
1962
|
+
|
|
1963
|
+
let install: InstallResult | undefined;
|
|
1964
|
+
if (!workspace.created) {
|
|
1965
|
+
log("install: skipped, the Workspace already existed");
|
|
1966
|
+
} else {
|
|
1967
|
+
try {
|
|
1968
|
+
install = await (deps.install ?? defaultInstall)(workspace.path);
|
|
1969
|
+
} catch (error) {
|
|
1970
|
+
install = { command: "(install)", exitCode: null, durationMs: 0, outputTail: error instanceof Error ? error.message : String(error) };
|
|
1971
|
+
}
|
|
1972
|
+
if (install) log(`install: ${install.command} exited ${install.exitCode ?? "without a code"} after ${install.durationMs}ms`);
|
|
1973
|
+
else log("install: no install command in the Run Recipe and no lockfile at the Workspace root; nothing to install");
|
|
1974
|
+
}
|
|
1975
|
+
const installNote = install ? `\n\nInstall: ${install.command} exited ${install.exitCode ?? "without a code"}.` : "";
|
|
1976
|
+
|
|
1977
|
+
const prompt = await assemble(deps, initiativeId, ctx, workspace, log, draftedVerification);
|
|
1978
|
+
|
|
1979
|
+
const started = now();
|
|
1980
|
+
const handle = worker.launch({ account, cwd: workspace.path, prompt, maxTurns: deps.maxTurns, stallTimeoutMs: deps.stallTimeoutMs });
|
|
1981
|
+
// The pid goes to the row as soon as there is one (story 50): a crash before the vendor's first
|
|
1982
|
+
// line then leaves a row Reconcile can see is still running, rather than one it would restart
|
|
1983
|
+
// beside the live Worker.
|
|
1984
|
+
if (handle.pid !== undefined) deps.processTable?.setPid(task.id, handle.pid);
|
|
1985
|
+
log(`worker: ${account.vendor} turn on ${task.identifier} in ${workspace.path}`);
|
|
1986
|
+
return observeTurn(deps, initiativeId, ctx, workspace, attempt, handle, { started, installNote, install, draftedVerification, slot });
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1989
|
+
interface TurnObservation {
|
|
1990
|
+
started: number;
|
|
1991
|
+
installNote: string;
|
|
1992
|
+
install?: InstallResult;
|
|
1993
|
+
resumed?: boolean;
|
|
1994
|
+
/** The Verification section the Worker was given because the Task has none; what the Verifier
|
|
1995
|
+
* follows in that case. */
|
|
1996
|
+
draftedVerification?: string;
|
|
1997
|
+
/** The Slot this turn is running in, when a Run gave it one: swapped for the Account's reserved
|
|
1998
|
+
* Verifier Slot before the Verifier round (core/scheduler.ts). Unset for a turn outside a Run's
|
|
1999
|
+
* Slots (the daemon's idle Reconcile), which is one turn at a time anyway. */
|
|
2000
|
+
slot?: SlotHold;
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
/** The Verification section the Verifier follows: the Task's own, else the drafted one. */
|
|
2004
|
+
function verificationSectionOf(task: TrackerTask, drafted: string | undefined): string | undefined {
|
|
2005
|
+
return parseWorkContract(task.description).sections.Verification ?? drafted;
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
type TerminalEvent = Extract<AdapterEvent, { type: "completed" | "failed" }>;
|
|
2009
|
+
type CompletedEvent = Extract<AdapterEvent, { type: "completed" }>;
|
|
2010
|
+
|
|
2011
|
+
/** Where one attempt's turns write: the local transcript and the output lines the tail is cut
|
|
2012
|
+
* from. A Gate's resumed turns append to the same. */
|
|
2013
|
+
interface TurnSink {
|
|
2014
|
+
transcript?: TranscriptWriter;
|
|
2015
|
+
output: string[];
|
|
2016
|
+
/** The vendor's last rate-limit report of this turn (story 30). The terminal `failed` event says
|
|
2017
|
+
* only that the turn ended `rate_limited`; the reset the vendor named came on the event before
|
|
2018
|
+
* it, and that is what decides how long the Account is Paused. */
|
|
2019
|
+
rateLimit?: { resetAt?: string; message: string };
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
/** Follows one turn's event stream to its terminal event: the Process Table learns the session id
|
|
2023
|
+
* the moment `started` arrives and the last output time as text streams (throttled); every event
|
|
2024
|
+
* goes to the local transcript; the output lines are kept for the tail. Shared by a fresh launch,
|
|
2025
|
+
* a Reconcile resume and a Gate's resumed turn. */
|
|
2026
|
+
async function followTurn(deps: RunDeps, task: TrackerTask, handle: TurnHandle, sink: TurnSink, started: number): Promise<TerminalEvent> {
|
|
2027
|
+
const { log } = deps;
|
|
2028
|
+
const now = deps.now ?? Date.now;
|
|
2029
|
+
let lastTouch = started;
|
|
2030
|
+
let terminal: TerminalEvent | undefined;
|
|
2031
|
+
for await (const event of handle.events) {
|
|
2032
|
+
sink.transcript?.append(event);
|
|
2033
|
+
switch (event.type) {
|
|
2034
|
+
case "started":
|
|
2035
|
+
deps.processTable?.setSession(task.id, event.sessionId, handle.pid);
|
|
2036
|
+
log(`worker: session ${event.sessionId}${handle.pid !== undefined ? ` (pid ${handle.pid})` : ""}`);
|
|
2037
|
+
break;
|
|
2038
|
+
case "output": {
|
|
2039
|
+
if (deps.processTable) {
|
|
2040
|
+
const at = now();
|
|
2041
|
+
if (at - lastTouch >= TOUCH_INTERVAL_MS) {
|
|
2042
|
+
deps.processTable.touch(task.id, at);
|
|
2043
|
+
lastTouch = at;
|
|
2044
|
+
}
|
|
2045
|
+
}
|
|
2046
|
+
for (const line of event.text.split("\n")) {
|
|
2047
|
+
sink.output.push(line);
|
|
2048
|
+
log(` | ${line}`);
|
|
2049
|
+
}
|
|
2050
|
+
break;
|
|
2051
|
+
}
|
|
2052
|
+
case "tool":
|
|
2053
|
+
log(` tool ${event.name}`);
|
|
2054
|
+
break;
|
|
2055
|
+
case "rate_limited":
|
|
2056
|
+
sink.rateLimit = { resetAt: event.resetAt, message: event.message };
|
|
2057
|
+
log(`worker: rate limited: ${event.message}${event.resetAt ? ` (resets ${event.resetAt})` : ""}`);
|
|
2058
|
+
break;
|
|
2059
|
+
case "stalled":
|
|
2060
|
+
log(`worker: stalled, no output for ${event.silentMs}ms`);
|
|
2061
|
+
break;
|
|
2062
|
+
case "completed":
|
|
2063
|
+
case "failed":
|
|
2064
|
+
terminal = event;
|
|
2065
|
+
break;
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
if (!terminal) throw new Error("Worker Adapter contract violated: the event stream ended without a terminal event");
|
|
2069
|
+
return terminal;
|
|
2070
|
+
}
|
|
2071
|
+
|
|
2072
|
+
/** Where a `done` turn stands after its Hand-off (and, when it opened, its Gates). */
|
|
2073
|
+
interface DoneState {
|
|
2074
|
+
terminal: CompletedEvent;
|
|
2075
|
+
/** The report as recorded: with the Hand-off the Foreman verified on the remote, over anything
|
|
2076
|
+
* the Worker claimed. */
|
|
2077
|
+
report: CompletionReport;
|
|
2078
|
+
handOff: HandOffResult;
|
|
2079
|
+
handOffNote: string;
|
|
2080
|
+
gates?: GateOutcome;
|
|
2081
|
+
verification?: VerifyOutcome;
|
|
2082
|
+
/** The last `done` report whose Hand-off stands, when `report` is a later turn's that did not
|
|
2083
|
+
* reach one (a blocked report, an incomplete Hand-off); written to memory beside it. */
|
|
2084
|
+
priorDone?: CompletionReport;
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
/** A `done` turn whose Hand-off opened: the state the Gates judge and the pull request they hold. */
|
|
2088
|
+
type HandedOffState = DoneState & { handOff: HandedOff };
|
|
2089
|
+
|
|
2090
|
+
/** The Hand-off of a `done` report: pushes and opens or updates the draft pull request, and puts
|
|
2091
|
+
* the verified pull request into the report so memory and the comment name it. */
|
|
2092
|
+
async function handOffTurn(deps: RunDeps, initiativeId: number, task: TrackerTask, workspace: Workspace, terminal: CompletedEvent): Promise<DoneState> {
|
|
2093
|
+
let report = terminal.report;
|
|
2094
|
+
const handOff = await (deps.handOff ?? noHandOff)({ task, workspace, report, initiativeId });
|
|
2095
|
+
let handOffNote: string;
|
|
2096
|
+
if (handOff.kind === "handed_off") {
|
|
2097
|
+
report = { ...report, handOff: { branch: workspace.branch, prUrl: handOff.pullRequest.url } };
|
|
2098
|
+
handOffNote = `\n\nHand-off: ${handOff.pullRequest.url}${handOff.reused ? " (updated)" : ""}`;
|
|
2099
|
+
} else {
|
|
2100
|
+
deps.log(`hand-off: ${task.identifier} incomplete: ${handOff.reason}`);
|
|
2101
|
+
handOffNote = `\n\nHand-off incomplete: ${handOff.reason}`;
|
|
2102
|
+
}
|
|
2103
|
+
return { terminal, report, handOff, handOffNote };
|
|
2104
|
+
}
|
|
2105
|
+
|
|
2106
|
+
/** What the Gates left the attempt in: the state of its last `done` turn (with the Hand-off that
|
|
2107
|
+
* stands, whichever turn opened it), or the failure of a resumed turn beside the last handed-off
|
|
2108
|
+
* state, with the Gates' rounds either way. */
|
|
2109
|
+
type GateDrive = { done: DoneState } | { failed: Extract<AdapterEvent, { type: "failed" }>; last: HandedOffState; gates: GateOutcome };
|
|
2110
|
+
|
|
2111
|
+
/** Runs the Gates on an open Hand-off and, while they fail and the retry cap allows, resumes the
|
|
2112
|
+
* same Worker session with the failing command and its output, hands its next `done` turn off
|
|
2113
|
+
* again (the same pull request, updated) and runs the Gates once more. Every round's results go to
|
|
2114
|
+
* the Run Record before anything else (ADR-0009); the rounds are summarised on the pull request at
|
|
2115
|
+
* the end, whatever the end was. A resumed turn that ends without a `done` report, or whose
|
|
2116
|
+
* Hand-off is incomplete, ends the Gates as failed with that as the reason: the Task is settled
|
|
2117
|
+
* blocked with what happened, and the last Hand-off that did open stays the attempt's, on the Run
|
|
2118
|
+
* Record, linked to the Task and carrying the comment. */
|
|
2119
|
+
async function driveGates(deps: RunDeps, initiativeId: number, ctx: TaskWithContext, workspace: Workspace, attempt: number, state: DoneState, sink: TurnSink): Promise<GateDrive> {
|
|
2120
|
+
const { tracker, worker, account, log } = deps;
|
|
2121
|
+
const now = deps.now ?? Date.now;
|
|
2122
|
+
const task = ctx.task;
|
|
2123
|
+
const gates = deps.gates ?? noGates;
|
|
2124
|
+
const retries = deps.gateRetries ?? DEFAULT_GATE_RETRIES;
|
|
2125
|
+
const rounds: GateRound[] = [];
|
|
2126
|
+
let resumes = 0;
|
|
2127
|
+
// A skip does not hold the Hand-off (a repository may name no Gate), but it is not a pass either:
|
|
2128
|
+
// `gateSummary` and `gateNote` say "skipped" wherever a pass would have said "passed".
|
|
2129
|
+
const outcome = (reason: GateReason, detail?: string): GateOutcome => ({ rounds, passed: reason === "passed" || reason === "skipped", resumes, reason, detail });
|
|
2130
|
+
|
|
2131
|
+
// No Gates without a Hand-off: a turn with nothing to push is settled as it is.
|
|
2132
|
+
if (state.handOff.kind !== "handed_off") return { done: state };
|
|
2133
|
+
let handedOff: HandedOffState = { ...state, handOff: state.handOff };
|
|
2134
|
+
const summarise = async (gates: GateOutcome) => {
|
|
2135
|
+
const { pullRequest } = handedOff.handOff;
|
|
2136
|
+
await commentOnPullRequest(deps, task, pullRequest.number, pullRequest.url, gateComment(gates, retries));
|
|
2137
|
+
};
|
|
2138
|
+
|
|
2139
|
+
for (;;) {
|
|
2140
|
+
const round = rounds.length + 1;
|
|
2141
|
+
const run = await gates(workspace);
|
|
2142
|
+
if (run.skipped) {
|
|
2143
|
+
log(`gate: ${task.identifier} skipped: ${run.reason}; the Hand-off stands as it is, unchecked`);
|
|
2144
|
+
// Past round 1 the base stopped naming Gates between rounds: earlier rounds did judge this
|
|
2145
|
+
// Hand-off, so that is a pass with the reason. At round 1 nothing judged it at all, and that
|
|
2146
|
+
// is carried as a `skipped` outcome rather than as no outcome: with none, `settle` writes no
|
|
2147
|
+
// gate note on the Task and no gate trailer on the attempt event, the Verifier skips too, and
|
|
2148
|
+
// the Task reaches In review under foreman:done with the only trace a line in the daemon log.
|
|
2149
|
+
return { done: { ...handedOff, gates: outcome(rounds.length ? "passed" : "skipped", run.reason) } };
|
|
2150
|
+
}
|
|
2151
|
+
const passed = run.results.every(gatePassed);
|
|
2152
|
+
// Run Record first (ADR-0009): the results are the commit, whatever the resume or the board do next.
|
|
2153
|
+
await tracker.appendRunEvents(
|
|
2154
|
+
initiativeId,
|
|
2155
|
+
run.results.map((r) => gateEvent(task, attempt, round, r)),
|
|
2156
|
+
);
|
|
2157
|
+
rounds.push({ round, results: run.results, passed });
|
|
2158
|
+
for (const r of run.results) log(`gate: ${task.identifier} round ${round}: ${gateLine(r)}`);
|
|
2159
|
+
if (passed) {
|
|
2160
|
+
log(`gate: ${task.identifier} passed in round ${round}${resumes ? ` after ${count(resumes, "resumed turn")}` : ""}`);
|
|
2161
|
+
const done = { ...handedOff, gates: outcome("passed") };
|
|
2162
|
+
await summarise(done.gates);
|
|
2163
|
+
return { done };
|
|
2164
|
+
}
|
|
2165
|
+
const failing = run.results.find((r) => !gatePassed(r))!;
|
|
2166
|
+
if (resumes >= retries) {
|
|
2167
|
+
log(`gate: ${task.identifier} ${failing.name} failed in round ${round}, past the retry cap (${retries}); the Task is blocked`);
|
|
2168
|
+
const done = { ...handedOff, gates: outcome("cap") };
|
|
2169
|
+
await summarise(done.gates);
|
|
2170
|
+
return { done };
|
|
2171
|
+
}
|
|
2172
|
+
|
|
2173
|
+
// The same session, so the Worker keeps its context; the same Process Table row, since the
|
|
2174
|
+
// attempt goes on. The next Hand-off updates the pull request already open for the branch.
|
|
2175
|
+
resumes++;
|
|
2176
|
+
const sessionId = handedOff.terminal.sessionId;
|
|
2177
|
+
log(`gate: ${task.identifier} ${failing.name} failed in round ${round} (${gateLine(failing)}); resuming session ${sessionId}, retry ${resumes} of ${retries}`);
|
|
2178
|
+
const handle = worker.resume(sessionId, {
|
|
2179
|
+
account,
|
|
2180
|
+
cwd: workspace.path,
|
|
2181
|
+
prompt: gateFixPrompt(task, failing, round, retries - resumes),
|
|
2182
|
+
maxTurns: deps.maxTurns,
|
|
2183
|
+
stallTimeoutMs: deps.stallTimeoutMs,
|
|
2184
|
+
});
|
|
2185
|
+
if (handle.pid !== undefined) deps.processTable?.setPid(task.id, handle.pid);
|
|
2186
|
+
const terminal = await followTurn(deps, task, handle, sink, now());
|
|
2187
|
+
if (terminal.type === "failed") {
|
|
2188
|
+
log(`gate: ${task.identifier} resumed turn ${terminal.reason}: ${terminal.message}; the Task is blocked`);
|
|
2189
|
+
const gates = outcome("turn_failed", `${terminal.reason}: ${terminal.message}`);
|
|
2190
|
+
await summarise(gates);
|
|
2191
|
+
return { failed: terminal, last: handedOff, gates };
|
|
2192
|
+
}
|
|
2193
|
+
if (terminal.report.outcome !== "done") {
|
|
2194
|
+
log(`gate: ${task.identifier} resumed turn reported ${terminal.report.outcome}; the Task is blocked`);
|
|
2195
|
+
// The pull request from before is still the attempt's Hand-off; the new report says why it
|
|
2196
|
+
// stopped, and the report it supersedes goes to memory beside it.
|
|
2197
|
+
const gates = outcome("turn_not_done", terminal.report.outcome);
|
|
2198
|
+
await summarise(gates);
|
|
2199
|
+
return { done: { ...handedOff, terminal, report: { ...terminal.report, handOff: handedOff.report.handOff }, gates, priorDone: handedOff.report } };
|
|
2200
|
+
}
|
|
2201
|
+
const next = await handOffTurn(deps, initiativeId, task, workspace, terminal);
|
|
2202
|
+
if (next.handOff.kind !== "handed_off") {
|
|
2203
|
+
log(`gate: ${task.identifier} resumed turn's Hand-off incomplete: ${next.handOff.reason}; the Task is blocked`);
|
|
2204
|
+
// The new report is the work's latest word, but the pull request that stands is the earlier one.
|
|
2205
|
+
const gates = outcome("handoff_incomplete", next.handOff.reason);
|
|
2206
|
+
await summarise(gates);
|
|
2207
|
+
return {
|
|
2208
|
+
done: {
|
|
2209
|
+
...handedOff,
|
|
2210
|
+
terminal: next.terminal,
|
|
2211
|
+
report: { ...next.report, handOff: handedOff.report.handOff },
|
|
2212
|
+
handOffNote: `${handedOff.handOffNote}\n\nHand-off of the resumed turn incomplete: ${next.handOff.reason}`,
|
|
2213
|
+
gates,
|
|
2214
|
+
},
|
|
2215
|
+
};
|
|
2216
|
+
}
|
|
2217
|
+
handedOff = { ...next, handOff: next.handOff };
|
|
2218
|
+
}
|
|
2219
|
+
}
|
|
2220
|
+
|
|
2221
|
+
/** Posts the Gates' summary on the pull request; best effort, since the Run Record already has
|
|
2222
|
+
* every result, and a Run without a code host client only logs it. */
|
|
2223
|
+
async function commentOnPullRequest(deps: RunDeps, task: TrackerTask, number: number, url: string, body: string, stage: "gate" | "verify" = "gate"): Promise<void> {
|
|
2224
|
+
const what = stage === "gate" ? "summary" : "Verdict";
|
|
2225
|
+
if (!deps.remote) {
|
|
2226
|
+
deps.log(`${stage}: ${task.identifier}: no code host client in this Run, so the ${what} is not posted on ${url}`);
|
|
2227
|
+
return;
|
|
2228
|
+
}
|
|
2229
|
+
try {
|
|
2230
|
+
await deps.remote.addPullRequestComment(number, body);
|
|
2231
|
+
deps.log(`${stage}: ${task.identifier} ${what} posted on ${url}`);
|
|
2232
|
+
} catch (error) {
|
|
2233
|
+
deps.log(`${stage}: ${task.identifier} could not post the ${what} on ${url}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
|
|
2237
|
+
/** The Verifier's round on a Hand-off whose Gates passed: the seam runs the app and the Verifier
|
|
2238
|
+
* Worker; here the screenshots go to the pull request (or stay on this machine, named), the
|
|
2239
|
+
* Verdict goes to the Run Record first (ADR-0009), then to the pull request as a comment. A round
|
|
2240
|
+
* that failed without a Verdict is recorded as a failed Verdict with one Defect saying why. */
|
|
2241
|
+
async function driveVerifier(deps: RunDeps, initiativeId: number, task: TrackerTask, workspace: Workspace, attempt: number, handedOff: HandedOff, verification: string | undefined): Promise<VerifyOutcome | undefined> {
|
|
2242
|
+
const { tracker, log } = deps;
|
|
2243
|
+
const now = deps.now ?? Date.now;
|
|
2244
|
+
// No Verifier in the deps: as with the Gates, the loop skips by itself and records nothing;
|
|
2245
|
+
// `foreman run` is what wires the real one.
|
|
2246
|
+
if (!deps.verifier) {
|
|
2247
|
+
log(`verify: ${task.identifier} skipped: this Run has no Verifier configured (RunDeps.verifier); the Hand-off stands as it is`);
|
|
2248
|
+
return undefined;
|
|
2249
|
+
}
|
|
2250
|
+
if (verification === undefined) {
|
|
2251
|
+
log(`verify: ${task.identifier} skipped: the Task has no Verification section and none was drafted`);
|
|
2252
|
+
return { kind: "skipped", reason: "the Task has no Verification section and none was drafted" };
|
|
2253
|
+
}
|
|
2254
|
+
// The Verifier's processes go on the Task's Process Table row: the turn's pid as the row's pid
|
|
2255
|
+
// (the implementer's turn is over), the app's beside it, so `foreman cancel` stops both and
|
|
2256
|
+
// Reconcile stops an app a dead Foreman left running.
|
|
2257
|
+
const onPid = (which: "app" | "verifier", pid: number | undefined) => {
|
|
2258
|
+
if (!deps.processTable) return;
|
|
2259
|
+
if (which === "app") deps.processTable.setAppPid(task.id, pid);
|
|
2260
|
+
else if (pid !== undefined) deps.processTable.setPid(task.id, pid);
|
|
2261
|
+
};
|
|
2262
|
+
let outcome: VerifyOutcome;
|
|
2263
|
+
await markStatus(tracker, task, undefined, STATUS_LABELS.verifying);
|
|
2264
|
+
try {
|
|
2265
|
+
outcome = await deps.verifier({ task, workspace, attempt, initiativeId, verification, onPid });
|
|
2266
|
+
} catch (error) {
|
|
2267
|
+
// The seam threw instead of reporting: a failed round, never an aborted dispatch, so the attempt
|
|
2268
|
+
// event is still written and the Task still lands somewhere a human can see.
|
|
2269
|
+
const reason = `the Verifier failed unexpectedly: ${error instanceof Error ? error.message : String(error)}`;
|
|
2270
|
+
log(`verify: ${task.identifier} ${reason}; the round failed`);
|
|
2271
|
+
outcome = { kind: "failed", reason, durationMs: 0 };
|
|
2272
|
+
}
|
|
2273
|
+
// Off as soon as the round is over, whichever way it went. The catch above does not rethrow, so
|
|
2274
|
+
// every path reaches here; the settlement clears it again as a backstop.
|
|
2275
|
+
await tracker.detachLabel(task.id, STATUS_LABELS.verifying);
|
|
2276
|
+
if (outcome.kind === "skipped") return outcome;
|
|
2277
|
+
let recorded: RecordedVerdict | undefined;
|
|
2278
|
+
try {
|
|
2279
|
+
recorded = await recordVerdict(deps, initiativeId, task, attempt, handedOff, outcome);
|
|
2280
|
+
} catch (error) {
|
|
2281
|
+
// The Verdict could not be recorded (the backend refused the event, say): the round is still
|
|
2282
|
+
// a failed one, recorded as such with a short reason if that at least will go.
|
|
2283
|
+
const reason = `the Verdict could not be recorded: ${error instanceof Error ? error.message : String(error)}`;
|
|
2284
|
+
log(`verify: ${task.identifier} ${reason}; the round failed`);
|
|
2285
|
+
const failed: VerifyOutcome = { kind: "failed", reason, app: outcome.app, durationMs: outcome.durationMs, sessionId: outcome.sessionId, outputTail: outcome.outputTail };
|
|
2286
|
+
try {
|
|
2287
|
+
await tracker.appendRunEvents(initiativeId, [verdictEvent(task, attempt, failureVerdict(reason), [], outcome.app, now())]);
|
|
2288
|
+
} catch (again) {
|
|
2289
|
+
log(`verify: ${task.identifier} the failed round could not be recorded either: ${again instanceof Error ? again.message : String(again)}`);
|
|
2290
|
+
}
|
|
2291
|
+
return failed;
|
|
2292
|
+
}
|
|
2293
|
+
// Every Defect becomes a sibling Task (OPS-273), after the Verdict is committed and outside its
|
|
2294
|
+
// try: a filer that fails leaves a recorded Verdict a human can still act on, never a round
|
|
2295
|
+
// reported as failed for a reason that has nothing to do with what the Verifier saw.
|
|
2296
|
+
await fileDefects(deps, initiativeId, task, handedOff, recorded);
|
|
2297
|
+
return outcome;
|
|
2298
|
+
}
|
|
2299
|
+
|
|
2300
|
+
/** Files each observed Defect of a failed Verdict as a sibling Task through the seam (defects.ts).
|
|
2301
|
+
* Never throws and never changes what the round was: the Verdict is already the commit, and the
|
|
2302
|
+
* original Task's Hand-off stays open and blocked whether or not its fixes could be filed. */
|
|
2303
|
+
async function fileDefects(deps: RunDeps, initiativeId: number, task: TrackerTask, handedOff: HandedOff, recorded: RecordedVerdict): Promise<FiledDefect[]> {
|
|
2304
|
+
const { log } = deps;
|
|
2305
|
+
const { verdict, evidence } = recorded;
|
|
2306
|
+
if (verdict.passed) return [];
|
|
2307
|
+
if (!deps.defects) {
|
|
2308
|
+
log(`defect: ${task.identifier} ${count(verdict.defects.length, "Defect")} not filed: this Run has no Defect filer configured (RunDeps.defects)`);
|
|
2309
|
+
return [];
|
|
2310
|
+
}
|
|
2311
|
+
// A round that failed before the Verifier observed anything carries one stand-in Defect that
|
|
2312
|
+
// says so; there is nothing there for a Worker to fix, and the filer skips it. The Verdict's own
|
|
2313
|
+
// flag says which kind it is, so no Verifier can suppress filing by how it titles a Defect.
|
|
2314
|
+
if (isRoundFailure(verdict)) {
|
|
2315
|
+
log(`defect: ${task.identifier} nothing to file: the round failed before the Verifier observed a Defect`);
|
|
2316
|
+
return [];
|
|
2317
|
+
}
|
|
2318
|
+
try {
|
|
2319
|
+
// No attempt: a Defect is keyed by its title alone, so the same failure seen on a later
|
|
2320
|
+
// attempt is the same Defect and is not filed twice (see core/defects.ts).
|
|
2321
|
+
const filed = await deps.defects({ initiativeId, task, verdict, evidence, pullRequestUrl: handedOff.pullRequest.url });
|
|
2322
|
+
const created = filed.filter((f) => f.task);
|
|
2323
|
+
log(
|
|
2324
|
+
`defect: ${task.identifier} ${count(created.length, "Defect")} filed as ${created.length === 1 ? "a sibling Task" : "sibling Tasks"}${created.length ? `: ${created.map((f) => f.task!.identifier).join(", ")}` : ""}${filed.length > created.length ? ` (${filed.length - created.length} skipped)` : ""}`,
|
|
2325
|
+
);
|
|
2326
|
+
return filed;
|
|
2327
|
+
} catch (error) {
|
|
2328
|
+
log(`defect: ${task.identifier} the Defects could not be filed: ${error instanceof Error ? error.message : String(error)}; the Verdict stands on ${handedOff.pullRequest.url}`);
|
|
2329
|
+
return [];
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
/** What `recordVerdict` committed: the Verdict as it was written to the Run Record and where each
|
|
2334
|
+
* Defect's screenshot ended up, so the Defect filer links the evidence rather than pasting it. */
|
|
2335
|
+
interface RecordedVerdict {
|
|
2336
|
+
verdict: Verdict;
|
|
2337
|
+
attempt: number;
|
|
2338
|
+
evidence: DefectEvidenceLink[];
|
|
2339
|
+
}
|
|
2340
|
+
|
|
2341
|
+
/** The screenshots to the pull request (or kept here, named), the `verdict` event to the Run
|
|
2342
|
+
* Record first (ADR-0009), then the Verdict as a comment. Throws only when the Run Record refused
|
|
2343
|
+
* the event; the upload and the comment are best effort and logged. */
|
|
2344
|
+
async function recordVerdict(deps: RunDeps, initiativeId: number, task: TrackerTask, attempt: number, handedOff: HandedOff, outcome: Exclude<VerifyOutcome, { kind: "skipped" }>): Promise<RecordedVerdict> {
|
|
2345
|
+
const { tracker, log } = deps;
|
|
2346
|
+
const now = deps.now ?? Date.now;
|
|
2347
|
+
const verdict = verdictOf(outcome);
|
|
2348
|
+
const { pullRequest } = handedOff;
|
|
2349
|
+
const evidence: DefectEvidence[] = [];
|
|
2350
|
+
const links: DefectEvidenceLink[] = [];
|
|
2351
|
+
const urls: string[] = [];
|
|
2352
|
+
for (let i = 0; i < verdict.defects.length; i++) {
|
|
2353
|
+
const shot = outcome.kind === "verified" ? outcome.screenshots.find((s) => s.defect === i) : undefined;
|
|
2354
|
+
if (!shot?.path) {
|
|
2355
|
+
evidence.push({});
|
|
2356
|
+
links.push({});
|
|
2357
|
+
urls.push("");
|
|
2358
|
+
continue;
|
|
2359
|
+
}
|
|
2360
|
+
let uploaded: Awaited<ReturnType<NonNullable<RunDeps["remote"]>["uploadAttachment"]>> | undefined;
|
|
2361
|
+
try {
|
|
2362
|
+
uploaded = await deps.remote?.uploadAttachment(pullRequest.number, shot.path);
|
|
2363
|
+
} catch (error) {
|
|
2364
|
+
log(`verify: ${task.identifier} could not upload ${shot.path} to ${pullRequest.url}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2365
|
+
}
|
|
2366
|
+
if (uploaded) {
|
|
2367
|
+
log(`verify: ${task.identifier} uploaded ${shot.path} as ${uploaded.url}`);
|
|
2368
|
+
evidence.push({ markdown: uploaded.markdown });
|
|
2369
|
+
// The Defect Task links the upload; the bytes stay on the pull request.
|
|
2370
|
+
links.push({ url: uploaded.url });
|
|
2371
|
+
urls.push(uploaded.url);
|
|
2372
|
+
} else {
|
|
2373
|
+
evidence.push({ localPath: shot.path });
|
|
2374
|
+
// No upload on this code host: the Defect Task names the file, never the Foreman's home.
|
|
2375
|
+
links.push({ localFile: basename(shot.path) });
|
|
2376
|
+
urls.push(shot.path);
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
// Run Record first (ADR-0009): the Verdict is the commit, whatever the comment or the board do next.
|
|
2380
|
+
await tracker.appendRunEvents(initiativeId, [verdictEvent(task, attempt, verdict, urls, outcome.app, now())]);
|
|
2381
|
+
await commentOnPullRequest(deps, task, pullRequest.number, pullRequest.url, verdictComment(verdict, evidence, outcome.app), "verify");
|
|
2382
|
+
return { verdict, attempt, evidence: links };
|
|
2383
|
+
}
|
|
2384
|
+
|
|
2385
|
+
/** Follows one attempt to its settlement: the turn, then for a `done` turn the Hand-off and the
|
|
2386
|
+
* Gates (with their resumed turns), then the Run Record, memory, comment and board. Shared by a
|
|
2387
|
+
* fresh launch and a Reconcile resume. */
|
|
2388
|
+
async function observeTurn(deps: RunDeps, initiativeId: number, ctx: TaskWithContext, workspace: Workspace, attempt: number, handle: TurnHandle, obs: TurnObservation): Promise<Dispatch> {
|
|
2389
|
+
const { tracker, log } = deps;
|
|
2390
|
+
const now = deps.now ?? Date.now;
|
|
2391
|
+
const task = ctx.task;
|
|
2392
|
+
const { started, installNote, install } = obs;
|
|
2393
|
+
const gateRetries = deps.gateRetries ?? DEFAULT_GATE_RETRIES;
|
|
2394
|
+
const sink: TurnSink = { transcript: deps.transcripts?.open(initiativeId, task.identifier, attempt), output: [] };
|
|
2395
|
+
const first = await followTurn(deps, task, handle, sink, started);
|
|
2396
|
+
|
|
2397
|
+
// Only a `done` turn is handed off: a blocked or failed one keeps its Workspace, and the branch
|
|
2398
|
+
// with it, for the next attempt. What gets recorded is the report with the Hand-off the Foreman
|
|
2399
|
+
// verified on the remote (over anything the Worker claimed), so the memory entries get it as
|
|
2400
|
+
// their source URL and the comment names it; the Worker's own report is left as it was.
|
|
2401
|
+
let terminal: TerminalEvent = first;
|
|
2402
|
+
let done: DoneState | undefined;
|
|
2403
|
+
/** The last handed-off state when a resumed turn then failed: its pull request stands. */
|
|
2404
|
+
let lastDone: HandedOffState | undefined;
|
|
2405
|
+
let gates: GateOutcome | undefined;
|
|
2406
|
+
let handOff: HandOffResult | undefined;
|
|
2407
|
+
if (first.type === "completed" && first.report.outcome === "done") {
|
|
2408
|
+
const drive = await driveGates(deps, initiativeId, ctx, workspace, attempt, await handOffTurn(deps, initiativeId, task, workspace, first), sink);
|
|
2409
|
+
if ("done" in drive) {
|
|
2410
|
+
done = drive.done;
|
|
2411
|
+
terminal = done.terminal;
|
|
2412
|
+
// The Verifier follows passing Gates (story 37): a Hand-off the Gates held, or none, gets no round.
|
|
2413
|
+
if (done.handOff.kind === "handed_off" && (done.gates?.passed ?? true)) {
|
|
2414
|
+
const section = verificationSectionOf(task, obs.draftedVerification);
|
|
2415
|
+
// Into the Account's reserved Verifier Slot (story 29), which gives the implementer Slot
|
|
2416
|
+
// back so the next Ready Task can start. Only for a round that will really run: a Run
|
|
2417
|
+
// without a Verifier, or a Task with nothing to verify, must not hold the reserved Slot
|
|
2418
|
+
// while it decides to skip. (A Task the seam itself then skips — commands only, no user
|
|
2419
|
+
// journey — does hold it, for as long as that decision takes, which is no time at all.)
|
|
2420
|
+
if (deps.verifier && section !== undefined) await obs.slot?.enterVerification();
|
|
2421
|
+
let verification: VerifyOutcome | undefined;
|
|
2422
|
+
try {
|
|
2423
|
+
verification = await driveVerifier(deps, initiativeId, task, workspace, attempt, done.handOff, section);
|
|
2424
|
+
} finally {
|
|
2425
|
+
// The round is over, so the reserved Slot goes to the next dispatch waiting to verify.
|
|
2426
|
+
// What follows — the Verdict comment, the settlement, the board, the row — is round-trips
|
|
2427
|
+
// with nothing running on the Account, and holding the one reserved Slot through them
|
|
2428
|
+
// would stall every other Verifier for the length of a settlement.
|
|
2429
|
+
obs.slot?.leaveVerification();
|
|
2430
|
+
}
|
|
2431
|
+
if (verification) done = { ...done, verification };
|
|
2432
|
+
}
|
|
2433
|
+
} else {
|
|
2434
|
+
terminal = drive.failed;
|
|
2435
|
+
lastDone = drive.last;
|
|
2436
|
+
handOff = lastDone.handOff;
|
|
2437
|
+
gates = drive.gates;
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
const durationMs = now() - started;
|
|
2441
|
+
// Before the attempt event, because the spend happened during the turn the attempt settles.
|
|
2442
|
+
const turnSpend = spendEvent(task.id, deps.account.vendor, deps.account.name, terminal.type === "completed" ? terminal.costUsd : undefined);
|
|
2443
|
+
if (turnSpend) await deps.tracker.appendRunEvents(initiativeId, [turnSpend]);
|
|
2444
|
+
const outputTail = tail(sink.output.join("\n"));
|
|
2445
|
+
const trailer = `${installNote}\n\nTurn: ${durationMs}ms.${outputTail ? `\n\nOutput tail:\n${outputTail}` : ""}`;
|
|
2446
|
+
const marker = obs.resumed ? `${RESUMED_MARKER} ` : "";
|
|
2447
|
+
const resumed = obs.resumed ? " (resumed)" : "";
|
|
2448
|
+
const gateTrailer = (g: GateOutcome | undefined) => (g ? `\n\nGates: ${gateSummary(g, gateRetries)}` : "");
|
|
2449
|
+
const verifyTrailer = (v: VerifyOutcome | undefined) => (v ? `\n\nVerification: ${verifySummary(v)}` : "");
|
|
2450
|
+
|
|
2451
|
+
if (terminal.type === "completed") {
|
|
2452
|
+
// A turn the vendor accepted is proof the credential is live: it closes any open streak, so the
|
|
2453
|
+
// failures that quarantine are consecutive ones and a Run's unlucky 401 cannot accumulate with
|
|
2454
|
+
// another an hour later (OPS-288).
|
|
2455
|
+
clearCredentialStreak(deps);
|
|
2456
|
+
const report = done?.report ?? terminal.report;
|
|
2457
|
+
handOff = done?.handOff;
|
|
2458
|
+
gates = done?.gates;
|
|
2459
|
+
const verification = done?.verification;
|
|
2460
|
+
// The recorded report's entries first, so the comment's outcome entry is its own; then the
|
|
2461
|
+
// `done` report it superseded without reaching a Hand-off, so that report is not lost.
|
|
2462
|
+
const own = reportToMemory(task, report);
|
|
2463
|
+
const prior = done?.priorDone ? reportToMemory(task, done.priorDone) : [];
|
|
2464
|
+
const memory = [...own, ...prior];
|
|
2465
|
+
await settle(
|
|
2466
|
+
tracker,
|
|
2467
|
+
initiativeId,
|
|
2468
|
+
task,
|
|
2469
|
+
{
|
|
2470
|
+
attempt,
|
|
2471
|
+
outcome: "completed",
|
|
2472
|
+
summary: `${marker}${report.outcome}: ${report.summary}${done?.handOffNote ?? ""}${gateTrailer(gates)}${verifyTrailer(verification)}${trailer}`,
|
|
2473
|
+
memory,
|
|
2474
|
+
comment: (ids, note) => reportComment(report, ids, `${prior.length ? priorReportNote(ids[own.length]) : ""}${note}`),
|
|
2475
|
+
handOff,
|
|
2476
|
+
gates,
|
|
2477
|
+
verification,
|
|
2478
|
+
gateRetries,
|
|
2479
|
+
},
|
|
2480
|
+
log,
|
|
2481
|
+
);
|
|
2482
|
+
deps.processTable?.removeWorker(task.id);
|
|
2483
|
+
const handedOff =
|
|
2484
|
+
handOff?.kind === "handed_off"
|
|
2485
|
+
? gates?.passed === false
|
|
2486
|
+
? `; Hand-off ${handOff.pullRequest.url} held by Gates`
|
|
2487
|
+
: !verificationPassed(verification)
|
|
2488
|
+
? `; Hand-off ${handOff.pullRequest.url} held by the Verdict`
|
|
2489
|
+
: `; handed off as ${handOff.pullRequest.url}`
|
|
2490
|
+
: handOff
|
|
2491
|
+
? "; Hand-off incomplete"
|
|
2492
|
+
: "";
|
|
2493
|
+
log(`attempt: ${task.identifier} completed${resumed} (${report.outcome}) after ${durationMs}ms; ${count(memory.length, "memory entry", "memory entries")} written${handedOff}`);
|
|
2494
|
+
return { task, workspace, attempt, outcome: "completed", report, sessionId: terminal.sessionId, durationMs, install, handOff, gates, verification, resumed: obs.resumed };
|
|
2495
|
+
}
|
|
2496
|
+
|
|
2497
|
+
// The stream ended without a report. If a human stopped it from the CLI, the row says so and
|
|
2498
|
+
// the settlement is theirs, not the failure's (stories 53, 56): an attached Worker's attempt
|
|
2499
|
+
// stays open for the attended turn; a cancelled one is closed as cancelled.
|
|
2500
|
+
const control = deps.processTable?.worker(task.id)?.control;
|
|
2501
|
+
if (control === "attached" || control === "released") {
|
|
2502
|
+
// Released already: the attach could not open its session (or the human was quick) and handed
|
|
2503
|
+
// the Worker back before this stream ended; the attempt stays open all the same, for Reconcile.
|
|
2504
|
+
log(
|
|
2505
|
+
control === "attached"
|
|
2506
|
+
? `attempt: ${task.identifier} attempt ${attempt} stopped for a human to attach after ${durationMs}ms; the attempt stays open until foreman release`
|
|
2507
|
+
: `attempt: ${task.identifier} attempt ${attempt} stopped for a human to attach after ${durationMs}ms and already released; the attempt stays open for the next tick to resume`,
|
|
2508
|
+
);
|
|
2509
|
+
return { task, workspace, attempt, outcome: "stopped", sessionId: terminal.sessionId, durationMs, install, resumed: obs.resumed, attached: true };
|
|
2510
|
+
}
|
|
2511
|
+
if (control === "cancelled") {
|
|
2512
|
+
const details = terminal.details?.length ? `\n${terminal.details.join("\n")}` : "";
|
|
2513
|
+
return settleCancelled(deps, initiativeId, ctx, workspace, attempt, `\n\nThe turn ended ${terminal.reason}: ${terminal.message}${details}${outputTail ? `\n\nOutput tail:\n${outputTail}` : ""}`, obs);
|
|
2514
|
+
}
|
|
2515
|
+
|
|
2516
|
+
|
|
2517
|
+
const failure = terminal;
|
|
2518
|
+
const outcome = attemptOutcomeOf(failure.reason);
|
|
2519
|
+
// After the resumed marker, so a resumed turn still reads as one; `isCredentialSummary` searches
|
|
2520
|
+
// rather than testing a prefix for exactly that reason.
|
|
2521
|
+
const credentialMarker = failure.reason === "credential_failed" ? `${CREDENTIAL_MARKER} ` : "";
|
|
2522
|
+
const details = failure.details?.length ? `\n${failure.details.join("\n")}` : "";
|
|
2523
|
+
const handOffNote = handOff?.kind === "handed_off" ? `\n\nHand-off: ${handOff.pullRequest.url}` : "";
|
|
2524
|
+
// A resumed turn that failed after a `done` turn had been handed off: that report is still the
|
|
2525
|
+
// work's record and goes to memory, though the attempt itself ended without one.
|
|
2526
|
+
const memory = lastDone ? reportToMemory(task, lastDone.report) : [];
|
|
2527
|
+
// The vendor said it is rate limited: the Account is Paused until the reset it named (story 30),
|
|
2528
|
+
// before the Task is settled, so the very next Slot this Run fills already sees it and goes to a
|
|
2529
|
+
// sibling Account instead. The pause is written through the AccountStore rather than held in the
|
|
2530
|
+
// Run, because it outlives the Run: the next Foreman and `foreman account list` read the same
|
|
2531
|
+
// fact. Its own Workers are untouched — they finish where they are, and Reconcile may resume
|
|
2532
|
+
// them; what is Paused is starting new ones.
|
|
2533
|
+
if (failure.reason === "rate_limited") await pauseOnRateLimit(deps, initiativeId, task.id, sink.rateLimit, failure.message, now());
|
|
2534
|
+
// The credential behind the Account is dead rather than busy (OPS-288). Recorded before the Task
|
|
2535
|
+
// is settled for the same reason a pause is: the Slot this frees is filled on the same tick, and
|
|
2536
|
+
// it must not be filled on the Account that just proved it cannot run anything.
|
|
2537
|
+
if (failure.reason === "credential_failed") await noteCredentialFailureOnAccount(deps, initiativeId, task.id, failure.message, now());
|
|
2538
|
+
|
|
2539
|
+
// A stall or a rate limit is the machine's or the vendor's failure, not the work's
|
|
2540
|
+
// (`RETRYABLE_FAILURES`): the Worker went quiet and the adapter stopped it, or the vendor refused
|
|
2541
|
+
// the turn, neither of which says anything about whether the Task can be done. Under the retry
|
|
2542
|
+
// cap the Task is made Ready again and a later tick dispatches it as the next attempt — on
|
|
2543
|
+
// whichever Account is not Paused by then — rather than being left blocked for a human who would
|
|
2544
|
+
// only tell it to try again. At the cap it is blocked like any other failed attempt, with the
|
|
2545
|
+
// comment naming the cause that spent the attempts (story 34), and the Run goes on with its other
|
|
2546
|
+
// Tasks. The next attempt comes on the next tick with no backoff (`DEFAULT_ATTEMPT_RETRIES`): a
|
|
2547
|
+
// fast-failing adapter spends them in milliseconds, on every Slot at once.
|
|
2548
|
+
//
|
|
2549
|
+
// What the cap counts is retryable failures, not dispatches: `attempt` numbers every turn the
|
|
2550
|
+
// Task ever had (the Run Record's numbering, which other components match on), and a Task whose
|
|
2551
|
+
// Gates failed, or whose Verdict found Defects, spent no machine-side attempt at all. Counting
|
|
2552
|
+
// dispatches would let a Task that a human relabelled twice hit its very first rate limit with
|
|
2553
|
+
// nothing left, blocked on a vendor's bad minute with no Failover ever tried; and the comment
|
|
2554
|
+
// would then say "3 attempts ended rate_limited" when one did.
|
|
2555
|
+
const attemptRetries = deps.attemptRetries ?? DEFAULT_ATTEMPT_RETRIES;
|
|
2556
|
+
// The reason, not the outcome: a credential failure is recorded as the generic `failed` and would
|
|
2557
|
+
// otherwise be read as work that cannot be retried (`RETRYABLE_FAILURE_REASONS`).
|
|
2558
|
+
const retryable = RETRYABLE_FAILURE_REASONS.has(failure.reason) && !handOff;
|
|
2559
|
+
// Read only when it can matter: one more Run Record read per retryable failure, none otherwise.
|
|
2560
|
+
const spent = retryable ? (await retryableFailuresOf(tracker, initiativeId, task.id)) + 1 : 0;
|
|
2561
|
+
const attemptsLeft = attemptRetries - (spent - 1);
|
|
2562
|
+
// How long the next attempt waits. Decided from `spent`, the same count the cap is measured in,
|
|
2563
|
+
// so the two never disagree about which attempt this is, and turned into an absolute moment here
|
|
2564
|
+
// against the Run's own clock rather than a duration the tick would have to date itself.
|
|
2565
|
+
const backoffMs = retryable && attemptsLeft > 0 ? retryBackoff(spent, deps.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS) : 0;
|
|
2566
|
+
const retry = retryable && attemptsLeft > 0 ? { attemptsLeft, waitMs: backoffMs } : undefined;
|
|
2567
|
+
const retryNotBefore = retry ? now() + backoffMs : undefined;
|
|
2568
|
+
const capped = retryable && attemptsLeft <= 0 ? { cause: failure.reason, attempts: spent, retries: attemptRetries } : undefined;
|
|
2569
|
+
await settle(
|
|
2570
|
+
tracker,
|
|
2571
|
+
initiativeId,
|
|
2572
|
+
task,
|
|
2573
|
+
{
|
|
2574
|
+
attempt,
|
|
2575
|
+
outcome,
|
|
2576
|
+
summary: `${marker}${credentialMarker}${failure.reason}: ${failure.message}${details}${handOffNote}${gateTrailer(gates)}${trailer}`,
|
|
2577
|
+
memory,
|
|
2578
|
+
comment: (ids, note) => `${failureComment(failure.reason, failure.message)}${memory.length ? priorReportNote(ids[0]) : ""}${note}`,
|
|
2579
|
+
handOff,
|
|
2580
|
+
gates,
|
|
2581
|
+
gateRetries,
|
|
2582
|
+
retry,
|
|
2583
|
+
capped,
|
|
2584
|
+
},
|
|
2585
|
+
log,
|
|
2586
|
+
);
|
|
2587
|
+
deps.processTable?.removeWorker(task.id);
|
|
2588
|
+
log(
|
|
2589
|
+
`attempt: ${task.identifier} ${outcome}${resumed} (${failure.reason}) after ${durationMs}ms: ${failure.message}${memory.length ? `; ${count(memory.length, "memory entry", "memory entries")} written for the earlier done report` : ""}${retry ? `; Ready again${inSeconds(backoffMs)}, ${count(retry.attemptsLeft, "attempt")} left under the retry cap` : ""}${capped ? `; the retry cap (${capped.retries}) is spent on ${capped.cause}, so the Task is blocked and the Run goes on` : ""}`,
|
|
2590
|
+
);
|
|
2591
|
+
return { task, workspace, attempt, outcome, sessionId: failure.sessionId, durationMs, install, handOff, gates, resumed: obs.resumed, retrying: retry !== undefined, blockedByCap: capped !== undefined, retryNotBefore };
|
|
2592
|
+
}
|
|
2593
|
+
|
|
2594
|
+
/** Pauses the turn's Account after a rate limit (story 30): until the reset the vendor named, or
|
|
2595
|
+
* `DEFAULT_PAUSE_MS` when it named none (`pauseUntilFrom` also refuses a reset far enough ahead to
|
|
2596
|
+
* be a parsing mistake). A Run with no AccountStore has nowhere to put the fact and says so, since
|
|
2597
|
+
* "the Account is Paused" that only one process can see is worse than none. */
|
|
2598
|
+
async function pauseOnRateLimit(deps: RunDeps, initiativeId: number, taskId: number | undefined, reported: TurnSink["rateLimit"], fallbackMessage: string, at: number): Promise<void> {
|
|
2599
|
+
const { account, log } = deps;
|
|
2600
|
+
// The reason is the vendor's own prose, parsed out of the Worker's stdout: it is stored in
|
|
2601
|
+
// `~/.opsee/foreman-accounts.json` and printed by `foreman account list`, so it is made one
|
|
2602
|
+
// printable line of a sane length first. An ANSI escape reaching that table would move the
|
|
2603
|
+
// operator's cursor, and its own padding would measure a width the terminal never shows.
|
|
2604
|
+
const reason = printableOneLine(reported?.message ?? fallbackMessage);
|
|
2605
|
+
if (!deps.accounts) {
|
|
2606
|
+
log(`account: "${account.name}" (${account.vendor}) reported a rate limit (${reason}) but this Run has no Account store to record the pause in; nothing is Paused`);
|
|
2607
|
+
return;
|
|
2608
|
+
}
|
|
2609
|
+
// `pauseMs` 0 is not "do not pause": it would set the reset to now, which `pausedUntil` reads as
|
|
2610
|
+
// no pause at all, so the Run would go straight back at the limit that just refused it. A pause
|
|
2611
|
+
// is at least `MIN_PAUSE_MS`.
|
|
2612
|
+
const until = pauseUntilFrom(at, reported?.resetAt, Math.max(MIN_PAUSE_MS, deps.pauseMs ?? DEFAULT_PAUSE_MS));
|
|
2613
|
+
const paused = pauseAccount(deps.accounts, account.name, until, reason);
|
|
2614
|
+
if (!paused) {
|
|
2615
|
+
log(`account: "${account.name}" reported a rate limit but is no longer registered; nothing is Paused`);
|
|
2616
|
+
return;
|
|
2617
|
+
}
|
|
2618
|
+
log(
|
|
2619
|
+
`account: "${account.name}" (${account.vendor}) is Paused until ${until.toISOString()}${reported?.resetAt ? " (the reset the vendor named)" : ` (the vendor named no reset, so ${deps.pauseMs ?? DEFAULT_PAUSE_MS}ms)`}: ${reason}. ` +
|
|
2620
|
+
`No new Worker starts on it until then; the Workers it has keep going, and a Ready Task that has not started goes to another Account of this vendor.`,
|
|
2621
|
+
);
|
|
2622
|
+
await recordAccountState(deps, initiativeId, accountEvent("paused", account.vendor, account.name, reason, { taskId, until }));
|
|
2623
|
+
}
|
|
2624
|
+
|
|
2625
|
+
/**
|
|
2626
|
+
* Reports on the Run Record every hand-made lift this Run's Accounts are still carrying, then
|
|
2627
|
+
* forgets them (`ResumedState`).
|
|
2628
|
+
*
|
|
2629
|
+
* `foreman account resume` cannot write the event itself: it is a local command over the accounts
|
|
2630
|
+
* file with no Initiative, and a Run Record belongs to one. So it stamps the moment and the next
|
|
2631
|
+
* Run with a Lane on that Account reports it here, carrying the stamp as `occurredAt` — the field
|
|
2632
|
+
* that exists because facts reach this Record late.
|
|
2633
|
+
*
|
|
2634
|
+
* Two consequences, both deliberate. The lift is missing from the Record until a Run next touches
|
|
2635
|
+
* that Account, which is honest rather than invented: nothing had read it yet. And where several
|
|
2636
|
+
* Initiatives could report it, whichever Run gets there first does — an Account belongs to no one
|
|
2637
|
+
* Initiative, so the attribution is arbitrary, which is why the event carries no `taskId`.
|
|
2638
|
+
*
|
|
2639
|
+
* The stamp is cleared only after the append is accepted. Through the outbox that includes a write
|
|
2640
|
+
* that was queued rather than delivered, which is the same commit the rest of the Run Record takes;
|
|
2641
|
+
* a refusal leaves the stamp for the next Run rather than losing the lift.
|
|
2642
|
+
*/
|
|
2643
|
+
async function reportResumedAccounts(deps: RunDeps, initiativeId: number, lanes: Lane[]): Promise<void> {
|
|
2644
|
+
if (!deps.accounts) return;
|
|
2645
|
+
const stored = deps.accounts.load();
|
|
2646
|
+
for (const lane of lanes) {
|
|
2647
|
+
const current = stored.find((a) => a.name === lane.account.name);
|
|
2648
|
+
const resumed = current?.resumedAt;
|
|
2649
|
+
if (!resumed) continue;
|
|
2650
|
+
const at = new Date(resumed.at);
|
|
2651
|
+
if (!Number.isFinite(at.getTime())) {
|
|
2652
|
+
// A hand-edited accounts file. Dropped rather than dated to now, which would put a moment on
|
|
2653
|
+
// the Record that nobody chose.
|
|
2654
|
+
deps.log(`account: "${lane.account.name}" carries an unreadable resume timestamp (${resumed.at}); it is dropped rather than recorded at the wrong time`);
|
|
2655
|
+
clearResumed(deps.accounts, lane.account.name);
|
|
2656
|
+
continue;
|
|
2657
|
+
}
|
|
2658
|
+
const reason = resumed.from === "both" ? "a human lifted its quarantine and its pause" : `a human lifted its ${resumed.from === "quarantined" ? "quarantine" : "pause"}`;
|
|
2659
|
+
try {
|
|
2660
|
+
await deps.tracker.appendRunEvents(initiativeId, [accountEvent("active", lane.account.vendor, lane.account.name, reason, { occurredAt: at })]);
|
|
2661
|
+
clearResumed(deps.accounts, lane.account.name);
|
|
2662
|
+
deps.log(`account: "${lane.account.name}" was brought back by hand at ${resumed.at}; recorded on the Run Record of Initiative ${initiativeId}`);
|
|
2663
|
+
} catch (error) {
|
|
2664
|
+
deps.log(`account: "${lane.account.name}" was brought back by hand at ${resumed.at}, but the Run Record would not take it, so it stays for the next Run: ${error instanceof Error ? error.message : String(error)}`);
|
|
2665
|
+
}
|
|
2666
|
+
}
|
|
2667
|
+
}
|
|
2668
|
+
|
|
2669
|
+
/**
|
|
2670
|
+
* Puts an Account's state on the Run Record, and never lets that write cost the Run anything.
|
|
2671
|
+
*
|
|
2672
|
+
* The Run Record is the commit for an *attempt* (ADR-0009): losing one loses the fact that a turn
|
|
2673
|
+
* happened. This is not that. The state itself is already in the accounts file, which is the copy
|
|
2674
|
+
* every Run and every `account list` reads, so a backend that refuses this leaves the schedule
|
|
2675
|
+
* correct and only the record of *when* it changed missing. Failing the turn over it would trade
|
|
2676
|
+
* something that matters for something that does not.
|
|
2677
|
+
*/
|
|
2678
|
+
async function recordAccountState(deps: RunDeps, initiativeId: number, event: RunEventInput): Promise<void> {
|
|
2679
|
+
try {
|
|
2680
|
+
await deps.tracker.appendRunEvents(initiativeId, [event]);
|
|
2681
|
+
} catch (error) {
|
|
2682
|
+
deps.log(`account: the Account state change could not be written to the Run Record, so the morning summary will not explain it: ${error instanceof Error ? error.message : String(error)}`);
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
/**
|
|
2687
|
+
* Records one credential failure against the turn's Account, and quarantines it once the streak
|
|
2688
|
+
* reaches `CREDENTIAL_FAILURES_BEFORE_QUARANTINE` (OPS-288, story 32).
|
|
2689
|
+
*
|
|
2690
|
+
* The counting is why this is not simply "quarantine on a credential failure". Every vendor pattern
|
|
2691
|
+
* the adapters match is a guess about prose, and the cost of guessing wrong is asymmetric: a false
|
|
2692
|
+
* positive pulls a working Account until a human re-adds it, which overnight is the whole Lane,
|
|
2693
|
+
* where a false negative only spends the Task's retry cap. A `CredentialError` is exempt because it
|
|
2694
|
+
* is not a guess — the Foreman established it about its own Account record before spawning anything
|
|
2695
|
+
* — but it reaches here as a `credential_failed` turn like any other, so the exemption is the
|
|
2696
|
+
* adapter's `launchFailureReason` doing its job upstream and not a second rule here.
|
|
2697
|
+
*
|
|
2698
|
+
* The transition goes to the Initiative's memory log as a `blocker`, which is what `foreman review`
|
|
2699
|
+
* reads in the morning: a Run that went quiet because its only Account died must be able to say so
|
|
2700
|
+
* somewhere a human looks. Deliberately not a Run Record event — there is no Account-state kind, and
|
|
2701
|
+
* `control` means a human steered the Run — which is also why a Pause has never been recorded there.
|
|
2702
|
+
* Written once, on the transition: the check is that the Account was not already quarantined, so a
|
|
2703
|
+
* later Run that meets the same dead login says nothing rather than repeating itself.
|
|
2704
|
+
*/
|
|
2705
|
+
async function noteCredentialFailureOnAccount(deps: RunDeps, initiativeId: number, taskId: number | undefined, message: string, at: number): Promise<void> {
|
|
2706
|
+
const { account, log } = deps;
|
|
2707
|
+
// The vendor's own prose, on its way to `~/.opsee/foreman-accounts.json` and to two terminal
|
|
2708
|
+
// views: made one printable line first, exactly as a pause reason is.
|
|
2709
|
+
const reason = printableOneLine(message);
|
|
2710
|
+
if (!deps.accounts) {
|
|
2711
|
+
log(`account: "${account.name}" (${account.vendor}) failed for a credential reason (${reason}) but this Run has no Account store to record it in; nothing is quarantined`);
|
|
2712
|
+
return;
|
|
2713
|
+
}
|
|
2714
|
+
const noted = noteCredentialFailure(deps.accounts, account.name, new Date(at), reason);
|
|
2715
|
+
if (!noted) {
|
|
2716
|
+
log(`account: "${account.name}" failed for a credential reason but is no longer registered; nothing is quarantined`);
|
|
2717
|
+
return;
|
|
2718
|
+
}
|
|
2719
|
+
const count = noted.credentialFailures?.count ?? 1;
|
|
2720
|
+
if (count < CREDENTIAL_FAILURES_BEFORE_QUARANTINE) {
|
|
2721
|
+
log(
|
|
2722
|
+
`account: "${account.name}" (${account.vendor}) failed for a credential reason (${reason}); that is ${count} in a row and ${CREDENTIAL_FAILURES_BEFORE_QUARANTINE} quarantines it. ` +
|
|
2723
|
+
`The Account stays in the schedule: one such failure is not enough to take a working Account out of rotation, and a turn that completes on it clears the count.`,
|
|
2724
|
+
);
|
|
2725
|
+
return;
|
|
2726
|
+
}
|
|
2727
|
+
const alreadyQuarantined = isQuarantined(noted);
|
|
2728
|
+
const quarantined = quarantineAccount(deps.accounts, account.name, new Date(at), reason);
|
|
2729
|
+
if (!quarantined) {
|
|
2730
|
+
log(`account: "${account.name}" failed for a credential reason but is no longer registered; nothing is quarantined`);
|
|
2731
|
+
return;
|
|
2732
|
+
}
|
|
2733
|
+
log(
|
|
2734
|
+
`account: "${account.name}" (${account.vendor}) is quarantined after ${count} credential failures in a row: ${reason}. ` +
|
|
2735
|
+
`It contributes no Slots and is not a Failover target; nothing lifts it by itself. Log the vendor back in, then: opsee foreman account resume ${account.name}`,
|
|
2736
|
+
);
|
|
2737
|
+
if (alreadyQuarantined) return;
|
|
2738
|
+
// The Run Record, not the Initiative's memory log. The memory entry only ever existed because the
|
|
2739
|
+
// Record had no Account-state event to put this in; now that it does, one fact lives in one place
|
|
2740
|
+
// and `foreman review` reads it from there.
|
|
2741
|
+
await recordAccountState(deps, initiativeId, accountEvent("quarantined", account.vendor, account.name, reason, { taskId }));
|
|
2742
|
+
}
|
|
2743
|
+
|
|
2744
|
+
/** Closes an open credential-failure streak on the turn's Account after the vendor accepted a turn.
|
|
2745
|
+
* A no-op without an Account store, and — inside `clearCredentialFailures` — no store write at all
|
|
2746
|
+
* when there is no streak, which is every turn of a healthy Account. */
|
|
2747
|
+
function clearCredentialStreak(deps: RunDeps): void {
|
|
2748
|
+
if (deps.accounts) clearCredentialFailures(deps.accounts, deps.account.name);
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
/** The comment's pointer at the `done` report a later turn superseded without reaching a Hand-off. */
|
|
2752
|
+
function priorReportNote(outcomeId: number): string {
|
|
2753
|
+
return ` The earlier done report is in Initiative memory (outcome entry #${outcomeId}).`;
|
|
2754
|
+
}
|
|
2755
|
+
|
|
2756
|
+
/** What a resumed attempt's summary starts with, so the Run Record says the turn was continued. */
|
|
2757
|
+
export const RESUMED_MARKER = "[resumed]";
|
|
2758
|
+
|
|
2759
|
+
/** What `RunAttemptEvent.summary` takes (proto/api/v1/initiative.proto: `max_len: 20000`, counted
|
|
2760
|
+
* in code points). */
|
|
2761
|
+
export const ATTEMPT_SUMMARY_MAX = 20_000;
|
|
2762
|
+
|
|
2763
|
+
/** What the cut says, so the Run Record never ends mid-word with no explanation — every other
|
|
2764
|
+
* truncation here (a Defect's text, a Gate's command and tails) names itself the same way. */
|
|
2765
|
+
const SUMMARY_CUT = "\n\n… (summary cut to the Run Record's 20000-character limit; the rest is in the transcript)";
|
|
2766
|
+
|
|
2767
|
+
/**
|
|
2768
|
+
* An attempt event's summary within the proto's limit.
|
|
2769
|
+
*
|
|
2770
|
+
* The summary is assembled from a Completion Report's own summary (which the report schema does not
|
|
2771
|
+
* bound), the Hand-off note, the Gate and Verification trailers, the install note and an output
|
|
2772
|
+
* tail. Past 20000 code points the backend answers `InvalidArgument`, which is not queueable
|
|
2773
|
+
* (core/outbox-tracker.ts) — `appendNow` rethrows, `settle` throws, and the dispatch is marked
|
|
2774
|
+
* blocked with no attempt event on the Run Record at all. Under ADR-0009 that is the loss of the
|
|
2775
|
+
* commit itself, so the summary is cut here rather than refused there.
|
|
2776
|
+
*/
|
|
2777
|
+
export function attemptSummary(text: string): string {
|
|
2778
|
+
if (Array.from(text).length <= ATTEMPT_SUMMARY_MAX) return text;
|
|
2779
|
+
return `${truncateCodePoints(text, ATTEMPT_SUMMARY_MAX - Array.from(SUMMARY_CUT).length)}${SUMMARY_CUT}`;
|
|
2780
|
+
}
|
|
2781
|
+
|
|
2782
|
+
/** A Task a memory entry or edge points at, or a stand-in when the Tracker no longer has it (a
|
|
2783
|
+
* reconcile deleted it): a missing neighbour must not abort the dispatch of a Task that is fine. */
|
|
2784
|
+
async function taskOrPlaceholder(tracker: TrackerAdapter, id: number): Promise<TrackerTask> {
|
|
2785
|
+
try {
|
|
2786
|
+
return (await tracker.getTask(id)).task;
|
|
2787
|
+
} catch {
|
|
2788
|
+
return { id, identifier: `deleted Task ${id}`, title: "(no longer on the Tracker)", description: "", projectId: 0, labels: [] };
|
|
2789
|
+
}
|
|
2790
|
+
}
|
|
2791
|
+
|
|
2792
|
+
/** The learnings file as the Workspace has it: what the default branch carries, since a Workspace
|
|
2793
|
+
* starts from its remote head. Absent when the repository has none yet. */
|
|
2794
|
+
export function acceptedLearningsIn(workspace: Pick<Workspace, "path">, file: string = DEFAULT_LEARNINGS_FILE): AcceptedLearnings | undefined {
|
|
2795
|
+
const path = join(workspace.path, file);
|
|
2796
|
+
return existsSync(path) ? { file, body: readFileSync(path, "utf8") } : undefined;
|
|
2797
|
+
}
|
|
2798
|
+
|
|
2799
|
+
/** Context Assembly for one Task: reads the blockers' Completion Reports and the siblings'
|
|
2800
|
+
* Proposed Learnings from the Initiative's memory log and the Accepted Learnings from the
|
|
2801
|
+
* Workspace, renders the prompt, and logs what went in. */
|
|
2802
|
+
async function assemble(
|
|
2803
|
+
deps: RunDeps,
|
|
2804
|
+
initiativeId: number,
|
|
2805
|
+
ctx: TaskWithContext,
|
|
2806
|
+
workspace: Workspace,
|
|
2807
|
+
log: (line: string) => void,
|
|
2808
|
+
draftedVerification?: string,
|
|
2809
|
+
): Promise<string> {
|
|
2810
|
+
const { tracker } = deps;
|
|
2811
|
+
const identifiers = new Map<number, string>([[ctx.task.id, ctx.task.identifier]]);
|
|
2812
|
+
const blockers: BlockerReports[] = [];
|
|
2813
|
+
for (const id of blockerIdsOf(ctx.task.id, ctx.dependencies)) {
|
|
2814
|
+
const blocker = await taskOrPlaceholder(tracker, id);
|
|
2815
|
+
identifiers.set(id, blocker.identifier);
|
|
2816
|
+
// The Tracker writes its own `outcome` entries when a Task moves to Done; those are bookkeeping,
|
|
2817
|
+
// not a Completion Report, and stay out.
|
|
2818
|
+
const reports = (await tracker.listMemory(initiativeId, { kinds: ["outcome"], sourceTaskId: id })).filter((r) => !r.isSystem);
|
|
2819
|
+
blockers.push({ task: blocker, reports });
|
|
2820
|
+
}
|
|
2821
|
+
|
|
2822
|
+
const learnings = (await tracker.listMemory(initiativeId, { kinds: ["learning"] }))
|
|
2823
|
+
.filter((l) => !l.isSystem && l.sourceTaskId !== ctx.task.id)
|
|
2824
|
+
.slice(0, SIBLING_LEARNING_LIMIT);
|
|
2825
|
+
const siblingLearnings: SiblingLearning[] = [];
|
|
2826
|
+
for (const l of learnings) {
|
|
2827
|
+
if (l.sourceTaskId !== undefined && !identifiers.has(l.sourceTaskId)) {
|
|
2828
|
+
identifiers.set(l.sourceTaskId, (await taskOrPlaceholder(tracker, l.sourceTaskId)).identifier);
|
|
2829
|
+
}
|
|
2830
|
+
siblingLearnings.push({ body: l.body, identifier: l.sourceTaskId === undefined ? undefined : identifiers.get(l.sourceTaskId), isAgent: l.isAgent });
|
|
2831
|
+
}
|
|
2832
|
+
|
|
2833
|
+
const assembled = assembleContext({ task: ctx.task, parent: ctx.parent, blockers, acceptedLearnings: acceptedLearningsIn(workspace, deps.learningsFile), siblingLearnings, workspace, draftedVerification });
|
|
2834
|
+
for (const line of assembled.summary) log(`context: ${line}`);
|
|
2835
|
+
return assembled.prompt;
|
|
2836
|
+
}
|