@ai-dossier/sched 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -15
- package/dist/batch-dispatch.d.ts +121 -0
- package/dist/batch-dispatch.d.ts.map +1 -0
- package/dist/batch-dispatch.js +1114 -0
- package/dist/batch-dispatch.js.map +1 -0
- package/dist/dispatch.d.ts +99 -14
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +204 -21
- package/dist/dispatch.js.map +1 -1
- package/dist/engine.d.ts +43 -2
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +181 -30
- package/dist/engine.js.map +1 -1
- package/dist/enqueue.d.ts.map +1 -1
- package/dist/enqueue.js +17 -2
- package/dist/enqueue.js.map +1 -1
- package/dist/fence.d.ts +78 -0
- package/dist/fence.d.ts.map +1 -0
- package/dist/fence.js +107 -0
- package/dist/fence.js.map +1 -0
- package/dist/groundtruth.d.ts +23 -0
- package/dist/groundtruth.d.ts.map +1 -1
- package/dist/groundtruth.js +41 -0
- package/dist/groundtruth.js.map +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +23 -3
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts +2 -0
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +8 -0
- package/dist/journal.js.map +1 -1
- package/dist/persist.js +25 -1
- package/dist/persist.js.map +1 -1
- package/dist/state.d.ts +7 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +56 -2
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +84 -6
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +15 -4
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,1114 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Batch dispatch (#523, RFC-0001 §C.4/D.2/D.3): the missing driver that
|
|
4
|
+
* executes `batch:<id>` units. #498 landed the batch failure-recovery library
|
|
5
|
+
* (attribution/bisect/eviction/dissolve, `recovery.ts`) and the batch state
|
|
6
|
+
* machine (`state.ts`); readiness/placement already treat a `ready` batch as
|
|
7
|
+
* a runnable unit (`readiness.ts`, `scheduler.ts`). Nothing dispatched one
|
|
8
|
+
* until now.
|
|
9
|
+
*
|
|
10
|
+
* Shape, mirroring `engine.ts`'s per-issue dispatch: claim a slot → spawn an
|
|
11
|
+
* agent → poll ground truth → verify → transition. Generalized to `BatchEntry`
|
|
12
|
+
* at batch-phase granularity instead of per-issue-phase granularity:
|
|
13
|
+
*
|
|
14
|
+
* ```
|
|
15
|
+
* ready → executing(member i/N) ⟲ → validating → reviewing → shipping
|
|
16
|
+
* → awaiting-merge → merged → deployed → reported → done
|
|
17
|
+
* failure rails (RFC F.2/F.8/F.9):
|
|
18
|
+
* executing → dissolving (a member self-reports blocked, RFC F.1)
|
|
19
|
+
* validating → attributing → (fixing | evicting) → validating
|
|
20
|
+
* → dissolving
|
|
21
|
+
* ```
|
|
22
|
+
*
|
|
23
|
+
* NO batch claim — not the first (`ready → executing`) nor any continuation
|
|
24
|
+
* (a later member, the tail agent, the report agent, the fix agent) — ever
|
|
25
|
+
* goes through `computeAssignments`/`runnableUnits`. Every one is a bespoke
|
|
26
|
+
* free-capacity-gated assignment, the same shape `engine.ts`'s
|
|
27
|
+
* `dispatchReportAgents` already uses (`runnableUnits` only ever offers a
|
|
28
|
+
* `status === 'ready'` batch, i.e. the moment BEFORE any claim). Between
|
|
29
|
+
* steps — a suite run, a PR merge wait — the slot is released to `idle` and
|
|
30
|
+
* holds no capacity (AC5): only a live member/tail/report/fix agent holds a
|
|
31
|
+
* slot.
|
|
32
|
+
*
|
|
33
|
+
* The aggregate suite itself is deterministic engine work, not an LLM step —
|
|
34
|
+
* it runs with no slot claimed at all, matching AC5's "member or batch-LLM-step"
|
|
35
|
+
* wording precisely.
|
|
36
|
+
*
|
|
37
|
+
* Two distinct failure rails, deliberately different:
|
|
38
|
+
* - A member's OWN agent reports itself blocked (its own gate never went
|
|
39
|
+
* green) — evicted directly, no attribution needed: the offender is already
|
|
40
|
+
* known, and either it has no commits yet (blocked before implementing) or
|
|
41
|
+
* its commits are exactly what gets reverted.
|
|
42
|
+
* - The AGGREGATE suite (run by the engine after every member individually
|
|
43
|
+
* went green) comes back red — an integration-level conflict no member's own
|
|
44
|
+
* gate caught. THIS is what `recovery.ts`'s attribution/fix/evict pipeline
|
|
45
|
+
* exists for (RFC F.2).
|
|
46
|
+
*
|
|
47
|
+
* Scope decisions recorded here, not silently cut: no `git bisect` stage for
|
|
48
|
+
* an ambiguous aggregate failure (bisect needs a per-project "run only these
|
|
49
|
+
* tests" command this module has no generic way to construct) — an
|
|
50
|
+
* unattributable red aggregate suite dissolves the batch rather than
|
|
51
|
+
* bisecting, which `attributing → dissolving` already models. No worktree-pool
|
|
52
|
+
* integration for batch-setup — cold `git worktree add` only, mirroring
|
|
53
|
+
* `teardown.ts`'s cold path. No per-phase stall/escalation ladder for batch
|
|
54
|
+
* sub-agents — a dead-without-verification agent is treated as blocked and
|
|
55
|
+
* evicted/reported rather than redispatched stronger. Both are documented
|
|
56
|
+
* follow-ups, not gaps discovered later.
|
|
57
|
+
*/
|
|
58
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
59
|
+
if (k2 === undefined) k2 = k;
|
|
60
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
61
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
62
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
63
|
+
}
|
|
64
|
+
Object.defineProperty(o, k2, desc);
|
|
65
|
+
}) : (function(o, m, k, k2) {
|
|
66
|
+
if (k2 === undefined) k2 = k;
|
|
67
|
+
o[k2] = m[k];
|
|
68
|
+
}));
|
|
69
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
70
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
71
|
+
}) : function(o, v) {
|
|
72
|
+
o["default"] = v;
|
|
73
|
+
});
|
|
74
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
75
|
+
var ownKeys = function(o) {
|
|
76
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
77
|
+
var ar = [];
|
|
78
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
79
|
+
return ar;
|
|
80
|
+
};
|
|
81
|
+
return ownKeys(o);
|
|
82
|
+
};
|
|
83
|
+
return function (mod) {
|
|
84
|
+
if (mod && mod.__esModule) return mod;
|
|
85
|
+
var result = {};
|
|
86
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
87
|
+
__setModuleDefault(result, mod);
|
|
88
|
+
return result;
|
|
89
|
+
};
|
|
90
|
+
})();
|
|
91
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
92
|
+
exports.runBatchTick = runBatchTick;
|
|
93
|
+
const path = __importStar(require("node:path"));
|
|
94
|
+
const attribution_1 = require("./attribution");
|
|
95
|
+
const dispatch_1 = require("./dispatch");
|
|
96
|
+
const groundtruth_1 = require("./groundtruth");
|
|
97
|
+
const journal_1 = require("./journal");
|
|
98
|
+
const recovery_1 = require("./recovery");
|
|
99
|
+
const scheduler_1 = require("./scheduler");
|
|
100
|
+
const state_1 = require("./state");
|
|
101
|
+
const teardown_1 = require("./teardown");
|
|
102
|
+
function emptyResult() {
|
|
103
|
+
return { spawned: [], completed: [], parked: [], mergeAccepted: [], failed: [], blocked: [] };
|
|
104
|
+
}
|
|
105
|
+
function unit(batchId) {
|
|
106
|
+
return `batch:${batchId}`;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Sanitize one untrusted string before it lands in persisted state, the
|
|
110
|
+
* journal, or a `sched status` terminal render (CWE-117/150): a milestone
|
|
111
|
+
* `reason=` value originates from a GitHub issue comment (anyone who can
|
|
112
|
+
* comment on a member issue can set it) and `parseMilestoneJson` copies it
|
|
113
|
+
* verbatim with no charset or length bound. Strips control characters
|
|
114
|
+
* (including the ANSI escape prefix) and bounds the length.
|
|
115
|
+
*/
|
|
116
|
+
function sanitizeUntrustedText(value) {
|
|
117
|
+
return (value
|
|
118
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: flattening control characters is the point
|
|
119
|
+
.replace(/[\u0000-\u001F\u007F]/g, ' ')
|
|
120
|
+
.slice(0, 200));
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Journal one event. Loosely-typed `extra`, matching `engine.ts`'s own local
|
|
124
|
+
* `journal()` wrapper — `unitEvent`'s stricter `Omit<JournalEvent, ...>` typing
|
|
125
|
+
* excess-property-checks an inline object literal (e.g. rejecting `pr`, a key
|
|
126
|
+
* `JournalEvent` doesn't declare), where a pre-typed `Record<string, unknown>`
|
|
127
|
+
* value passed through a variable does not.
|
|
128
|
+
*/
|
|
129
|
+
function journalEvent(deps, event, unitId, extra = {}) {
|
|
130
|
+
deps.journal.append((0, journal_1.unitEvent)(event, unitId, extra), deps.now());
|
|
131
|
+
}
|
|
132
|
+
function slotFor(state, batchId) {
|
|
133
|
+
return state.slots.find((s) => s.unit === unit(batchId));
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Re-apply a state computed OUTSIDE the lock (by `recovery.ts`'s functions,
|
|
137
|
+
* which necessarily shell out — `git revert`, `ai-dossier runstate post` —
|
|
138
|
+
* and so cannot themselves run inside `store.withLock`) onto a FRESHLY
|
|
139
|
+
* loaded state, touching only `batchId`'s own batch record (plus any new
|
|
140
|
+
* half-batches a dissolve split created) and the named issues' queue
|
|
141
|
+
* entries. Anything a concurrent process wrote to `fresh` in the meantime —
|
|
142
|
+
* `sched enqueue`, `sched abandon`, `sched pause` all take the same
|
|
143
|
+
* cross-process lock — survives, where blindly returning `computed` wholesale
|
|
144
|
+
* would have silently clobbered it.
|
|
145
|
+
*/
|
|
146
|
+
function applyBatchAndIssues(fresh, computed, batchId, issues) {
|
|
147
|
+
const updatedBatch = computed.batches.find((b) => b.id === batchId);
|
|
148
|
+
const batches = fresh.batches.map((b) => (b.id === batchId && updatedBatch ? updatedBatch : b));
|
|
149
|
+
// A `halved` dissolve creates new batch ids (`<id>-a`/`<id>-b`) that exist
|
|
150
|
+
// in `computed` but not yet in `fresh`.
|
|
151
|
+
for (const cb of computed.batches) {
|
|
152
|
+
if (!batches.some((b) => b.id === cb.id))
|
|
153
|
+
batches.push(cb);
|
|
154
|
+
}
|
|
155
|
+
const issueSet = new Set(issues);
|
|
156
|
+
const entries = fresh.entries.map((e) => {
|
|
157
|
+
if (!issueSet.has(e.issue))
|
|
158
|
+
return e;
|
|
159
|
+
return computed.entries.find((ce) => ce.issue === e.issue) ?? e;
|
|
160
|
+
});
|
|
161
|
+
return { ...fresh, batches, entries };
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Run the aggregate suite, treating a THROWING runner as a red suite with no
|
|
165
|
+
* failing tests — `recovery.ts`'s own internal `runSuite` wrapper already
|
|
166
|
+
* does this for calls that go through `beginAttribution`/`evictMembers`/etc,
|
|
167
|
+
* but `runValidate`/`reconcileFixSlot` call `deps.runSuite` directly (they
|
|
168
|
+
* need the result before deciding whether to call into `recovery.ts` at
|
|
169
|
+
* all), so an unguarded throw there would propagate out of `runBatchTick`
|
|
170
|
+
* into `tick()`'s own catch — a bare `tick-failed` with no unit id, repeating
|
|
171
|
+
* every reconcile interval forever since nothing about the batch changed.
|
|
172
|
+
*/
|
|
173
|
+
function safeSuite(deps, batchId, worktree) {
|
|
174
|
+
try {
|
|
175
|
+
return deps.runSuite(worktree);
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
const detail = `suite runner threw: ${err.message}`;
|
|
179
|
+
journalEvent(deps, 'suite-failed', unit(batchId), { detail });
|
|
180
|
+
return { ok: false, failing: [], detail };
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
function recoveryDeps(deps, batch, now) {
|
|
184
|
+
return {
|
|
185
|
+
exec: deps.exec,
|
|
186
|
+
repoDir: batch.worktree ?? deps.repoDir,
|
|
187
|
+
journal: deps.journal,
|
|
188
|
+
postMilestone: (0, recovery_1.createExecMilestonePoster)(deps.exec, { repoDir: deps.repoDir }),
|
|
189
|
+
runSuite: batch.worktree !== null ? () => deps.runSuite(batch.worktree) : undefined,
|
|
190
|
+
now: () => now,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* The declared edge one step closer to `idle` from each `SlotStatus`
|
|
195
|
+
* (`state.ts`'s `SLOT_BASE_TRANSITIONS`) — `recovering` has no direct edge to
|
|
196
|
+
* `idle`, only `running`/`failed`, so it routes through `failed` first;
|
|
197
|
+
* getting this wrong throws `IllegalTransitionError` inside a lock.
|
|
198
|
+
*/
|
|
199
|
+
const NEXT_TOWARD_IDLE = {
|
|
200
|
+
idle: null,
|
|
201
|
+
assigned: 'idle',
|
|
202
|
+
running: 'exited',
|
|
203
|
+
exited: 'verifying',
|
|
204
|
+
verifying: 'complete',
|
|
205
|
+
complete: 'idle',
|
|
206
|
+
recovering: 'failed',
|
|
207
|
+
failed: 'idle',
|
|
208
|
+
};
|
|
209
|
+
/** Release a batch's slot to idle, whatever status it currently holds (mirrors `walkSlotToIdle`). */
|
|
210
|
+
function releaseSlot(state, batchId, now) {
|
|
211
|
+
let next = state;
|
|
212
|
+
let slot = slotFor(next, batchId);
|
|
213
|
+
// Bounded: the longest real walk (recovering → failed → idle, or
|
|
214
|
+
// running → exited → verifying → complete → idle) is 4 hops.
|
|
215
|
+
for (let i = 0; i < 8 && slot && slot.status !== 'idle'; i++) {
|
|
216
|
+
const to = NEXT_TOWARD_IDLE[slot.status];
|
|
217
|
+
if (to === null)
|
|
218
|
+
break;
|
|
219
|
+
next = (0, state_1.transitionSlot)(next, slot.id, to, {}, now);
|
|
220
|
+
slot = slotFor(next, batchId);
|
|
221
|
+
}
|
|
222
|
+
return next;
|
|
223
|
+
}
|
|
224
|
+
// --- Batch setup (ready → executing, member 1) ---
|
|
225
|
+
/**
|
|
226
|
+
* `batch/<id>-<YYYYMMDD>` off `base_branch`, a worktree at
|
|
227
|
+
* `<repoDir>/worktrees/batch-<id>-<YYYYMMDD>` (cold git only — no pool
|
|
228
|
+
* integration in this version, see the module doc), and a fresh runstate run
|
|
229
|
+
* id minted against the anchor. All-or-nothing: any failed step reports the
|
|
230
|
+
* step name and nothing is partially recorded on the batch.
|
|
231
|
+
*/
|
|
232
|
+
function runBatchSetup(deps, batch, now) {
|
|
233
|
+
if (batch.anchor === null)
|
|
234
|
+
return { ok: false, reason: 'no-anchor' };
|
|
235
|
+
const date = now.toISOString().slice(0, 10).replaceAll('-', '');
|
|
236
|
+
const branch = `batch/${batch.id}-${date}`;
|
|
237
|
+
const worktree = path.join(deps.repoDir, 'worktrees', `batch-${batch.id}-${date}`);
|
|
238
|
+
if (!attribution_1.SAFE_REF_RE.test(branch) || !attribution_1.SAFE_REF_RE.test(batch.base_branch)) {
|
|
239
|
+
return { ok: false, reason: 'invalid-branch-name' };
|
|
240
|
+
}
|
|
241
|
+
// Defense in depth alongside enqueue.ts's `BATCH_ID_RE` (CWE-22): the batch
|
|
242
|
+
// id is enqueue-time-validated against path-hostile characters already, but
|
|
243
|
+
// this is the actual point where it becomes a filesystem path — the same
|
|
244
|
+
// containment check teardown applies on the way OUT must hold on the way IN.
|
|
245
|
+
const root = deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir;
|
|
246
|
+
if (!(0, teardown_1.isSafeWorktree)(path.resolve(root), worktree)) {
|
|
247
|
+
return { ok: false, reason: 'invalid-worktree-path' };
|
|
248
|
+
}
|
|
249
|
+
const runId = deps.exec('ai-dossier', ['runstate', 'mint', '--issue', String(batch.anchor)], deps.repoDir);
|
|
250
|
+
if (runId === null || runId.trim() === '')
|
|
251
|
+
return { ok: false, reason: 'runstate-mint-failed' };
|
|
252
|
+
const mintedRunId = runId.trim();
|
|
253
|
+
if (deps.exec('git', ['fetch', 'origin', '--', batch.base_branch], deps.repoDir) === null) {
|
|
254
|
+
return { ok: false, reason: 'fetch-failed', runId: mintedRunId };
|
|
255
|
+
}
|
|
256
|
+
if (deps.exec('git', ['branch', branch, `origin/${batch.base_branch}`], deps.repoDir) === null) {
|
|
257
|
+
return { ok: false, reason: 'branch-create-failed', runId: mintedRunId };
|
|
258
|
+
}
|
|
259
|
+
if (deps.exec('git', ['push', '-u', 'origin', '--', branch], deps.repoDir) === null) {
|
|
260
|
+
return { ok: false, reason: 'branch-push-failed', runId: mintedRunId };
|
|
261
|
+
}
|
|
262
|
+
if (deps.exec('git', ['worktree', 'add', '--', worktree, branch], deps.repoDir) === null) {
|
|
263
|
+
return { ok: false, reason: 'worktree-add-failed', runId: mintedRunId };
|
|
264
|
+
}
|
|
265
|
+
return { ok: true, branch, worktree, runId: mintedRunId };
|
|
266
|
+
}
|
|
267
|
+
/** Spawn one batch member's `slot-cycle` agent into the slot batch-setup (or a prior member) just released. */
|
|
268
|
+
/**
|
|
269
|
+
* Drive a member's `QueueEntry` through the D.1 slot-line states it must pass
|
|
270
|
+
* through before `shipped-in-batch` becomes a legal edge (`validated` is the
|
|
271
|
+
* only state that transitions there) — `classified → batched → waiting →
|
|
272
|
+
* in-work`, each a no-op waypoint from the batch's perspective (the real
|
|
273
|
+
* waiting/working happens at BATCH granularity), applied idempotently so a
|
|
274
|
+
* member already past a given waypoint is left alone.
|
|
275
|
+
*/
|
|
276
|
+
function advanceMemberToInWork(state, memberIssue, now) {
|
|
277
|
+
let next = state;
|
|
278
|
+
const chain = [
|
|
279
|
+
['queued', 'classified'],
|
|
280
|
+
['classified', 'batched'],
|
|
281
|
+
['batched', 'waiting'],
|
|
282
|
+
['waiting', 'in-work'],
|
|
283
|
+
];
|
|
284
|
+
for (const [from, to] of chain) {
|
|
285
|
+
if ((0, state_1.findEntry)(next, memberIssue)?.status === from) {
|
|
286
|
+
next = (0, state_1.transitionIssue)(next, memberIssue, to, {}, now);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return next;
|
|
290
|
+
}
|
|
291
|
+
/** The completion half of the same chain: `in-work → committed → validated` (see `advanceMemberToInWork`). */
|
|
292
|
+
function advanceMemberToValidated(state, memberIssue, now) {
|
|
293
|
+
let next = state;
|
|
294
|
+
const chain = [
|
|
295
|
+
['in-work', 'committed'],
|
|
296
|
+
['committed', 'validated'],
|
|
297
|
+
];
|
|
298
|
+
for (const [from, to] of chain) {
|
|
299
|
+
if ((0, state_1.findEntry)(next, memberIssue)?.status === from) {
|
|
300
|
+
next = (0, state_1.transitionIssue)(next, memberIssue, to, {}, now);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return next;
|
|
304
|
+
}
|
|
305
|
+
function spawnMember(deps, dispatch, state, slot, batchId, now, result) {
|
|
306
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
307
|
+
if (!batch || batch.worktree === null) {
|
|
308
|
+
// A leaked `assigned` slot with `pid: null` is invisible to `dead`
|
|
309
|
+
// detection (nothing ever kills/reclaims it) — release it here rather
|
|
310
|
+
// than leaving the batch permanently down one slot of capacity.
|
|
311
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
312
|
+
reason: 'no-worktree',
|
|
313
|
+
detail: 'spawnMember: batch has no worktree — batch-setup has not landed',
|
|
314
|
+
});
|
|
315
|
+
return releaseSlot(state, batchId, now);
|
|
316
|
+
}
|
|
317
|
+
const memberIssue = batch.members[batch.executing_member - 1];
|
|
318
|
+
if (memberIssue === undefined) {
|
|
319
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
320
|
+
reason: 'no-member',
|
|
321
|
+
detail: `spawnMember: executing_member ${batch.executing_member} has no member issue`,
|
|
322
|
+
});
|
|
323
|
+
return releaseSlot(state, batchId, now);
|
|
324
|
+
}
|
|
325
|
+
const withStatus = advanceMemberToInWork(state, memberIssue, now);
|
|
326
|
+
const tier = (0, state_1.findEntry)(withStatus, memberIssue)?.tier ?? 'mid';
|
|
327
|
+
const cmd = (0, dispatch_1.buildAgentCommand)(dispatch.command, tier, memberIssue, dispatch.tierModels);
|
|
328
|
+
const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, batch.worktree);
|
|
329
|
+
const logFile = path.join(deps.store.runsDir, `${(0, dispatch_1.unitLogName)(unit(batchId))}-m${batch.executing_member}-${memberIssue}.log`);
|
|
330
|
+
let pid;
|
|
331
|
+
try {
|
|
332
|
+
pid = deps.spawnDeps.spawn(cmd, prompt, logFile);
|
|
333
|
+
}
|
|
334
|
+
catch (err) {
|
|
335
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
336
|
+
issue: memberIssue,
|
|
337
|
+
reason: 'spawn-error',
|
|
338
|
+
detail: `member #${memberIssue} spawn failed: ${err.message}`,
|
|
339
|
+
});
|
|
340
|
+
result.failed.push(unit(batchId));
|
|
341
|
+
return releaseSlot(withStatus, batchId, now);
|
|
342
|
+
}
|
|
343
|
+
const patch = {
|
|
344
|
+
pid,
|
|
345
|
+
pid_start: deps.spawnDeps.processStart(pid),
|
|
346
|
+
phase: 'member',
|
|
347
|
+
last_progress_at: now.toISOString(),
|
|
348
|
+
};
|
|
349
|
+
const next = slot.status === 'assigned' || slot.status === 'recovering'
|
|
350
|
+
? (0, state_1.transitionSlot)(withStatus, slot.id, 'running', patch, now)
|
|
351
|
+
: withStatus;
|
|
352
|
+
deps.journal.append((0, journal_1.unitEvent)('spawned', unit(batchId), {
|
|
353
|
+
pid,
|
|
354
|
+
tier,
|
|
355
|
+
slot: slot.id,
|
|
356
|
+
issue: memberIssue,
|
|
357
|
+
detail: `member ${batch.executing_member}/${batch.members.length}`,
|
|
358
|
+
}), now);
|
|
359
|
+
result.spawned.push(unit(batchId));
|
|
360
|
+
return next;
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* First claim of a `ready` batch: assign it a slot, run batch-setup (real
|
|
364
|
+
* network round trips — `ai-dossier runstate mint`, `git fetch`/`push`), land
|
|
365
|
+
* the results, then spawn member 1 in the SAME slot rather than releasing and
|
|
366
|
+
* re-claiming: setup already holds the slot for its own duration, and
|
|
367
|
+
* splitting it into two capacity-gated claims would only add a second gate
|
|
368
|
+
* for no benefit — the slot is going to member 1 immediately either way.
|
|
369
|
+
*/
|
|
370
|
+
function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
371
|
+
const claimedSlot = deps.store.withLock((state) => {
|
|
372
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
373
|
+
if (!batch || batch.status !== 'ready' || batch.anchor === null || slotFor(state, batchId)) {
|
|
374
|
+
return { state, result: null };
|
|
375
|
+
}
|
|
376
|
+
if ((0, scheduler_1.freeCapacity)(state, config) === 0)
|
|
377
|
+
return { state, result: null };
|
|
378
|
+
const assigned = (0, scheduler_1.assignToIdleSlot)(state, unit(batchId), 'batch-setup', now);
|
|
379
|
+
return { state: assigned.state, result: assigned.slotId };
|
|
380
|
+
});
|
|
381
|
+
if (claimedSlot === null)
|
|
382
|
+
return;
|
|
383
|
+
const state = deps.store.load();
|
|
384
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
385
|
+
if (!batch)
|
|
386
|
+
return;
|
|
387
|
+
const setup = runBatchSetup(deps, batch, now);
|
|
388
|
+
const poster = (0, recovery_1.createExecMilestonePoster)(deps.exec, { repoDir: deps.repoDir });
|
|
389
|
+
if (!setup.ok) {
|
|
390
|
+
// `ai-dossier runstate post` REQUIRES a run id (types.ts's `BatchEntry.run_id`
|
|
391
|
+
// doc) — posting with an empty string silently fails the CLI call. Only post
|
|
392
|
+
// when the mint step actually landed one (`setup.runId`, when a LATER step
|
|
393
|
+
// failed) or the batch already carries one from an earlier attempt.
|
|
394
|
+
const runId = setup.runId ?? batch.run_id;
|
|
395
|
+
if (batch.anchor !== null && runId !== null) {
|
|
396
|
+
poster(batch.anchor, runId, {
|
|
397
|
+
phase: 'batch-setup',
|
|
398
|
+
status: 'blocked',
|
|
399
|
+
kv: { reason: setup.reason },
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
else {
|
|
403
|
+
journalEvent(deps, 'milestone-post-failed', unit(batchId), {
|
|
404
|
+
detail: `batch-setup blocked (${setup.reason}) — no run id to post to yet`,
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
deps.journal.append((0, journal_1.unitEvent)('batch-setup-failed', unit(batchId), { detail: setup.reason }), now);
|
|
408
|
+
result.failed.push(unit(batchId));
|
|
409
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
poster(batch.anchor, setup.runId, {
|
|
413
|
+
phase: 'batch-setup',
|
|
414
|
+
status: 'done',
|
|
415
|
+
kv: { branch: setup.branch, worktree: setup.worktree, base_branch: batch.base_branch },
|
|
416
|
+
});
|
|
417
|
+
deps.journal.append((0, journal_1.unitEvent)('batch-setup-done', unit(batchId), { detail: setup.worktree }), now);
|
|
418
|
+
deps.store.withLock((s) => {
|
|
419
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
420
|
+
if (!b || b.status !== 'ready') {
|
|
421
|
+
// The batch moved (dissolved/abandoned) between the claim and here — a
|
|
422
|
+
// real worktree now exists that nothing else knows about, and the slot
|
|
423
|
+
// this claim took is still `assigned` with no agent in it. Release the
|
|
424
|
+
// slot rather than leaking capacity; the worktree is orphaned (named in
|
|
425
|
+
// the journal for manual cleanup — it is not this rare-race path's job
|
|
426
|
+
// to guess whether reusing or removing it is safe).
|
|
427
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
428
|
+
reason: 'batch-left-ready-during-setup',
|
|
429
|
+
detail: `worktree ${setup.worktree} created but batch is now '${b?.status ?? 'gone'}' — orphaned, manual cleanup required`,
|
|
430
|
+
});
|
|
431
|
+
return { state: releaseSlot(s, batchId, now), result: undefined };
|
|
432
|
+
}
|
|
433
|
+
let next = (0, state_1.patchBatch)(s, batchId, { branch: setup.branch, worktree: setup.worktree, run_id: setup.runId }, now);
|
|
434
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 1 }, now);
|
|
435
|
+
const slot = slotFor(next, batchId);
|
|
436
|
+
if (slot)
|
|
437
|
+
next = spawnMember(deps, dispatch, next, slot, batchId, now, result);
|
|
438
|
+
return { state: next, result: undefined };
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
// --- Continuation: claim a fresh slot for the next live step ---
|
|
442
|
+
/**
|
|
443
|
+
* Claim a fresh idle slot for the batch's next live step (a later member, the
|
|
444
|
+
* tail agent, the report agent), gated on free capacity exactly like
|
|
445
|
+
* `dispatchReportAgents` — never through `computeAssignments`/`runnableUnits`
|
|
446
|
+
* again (those only ever offer a `status === 'ready'` batch).
|
|
447
|
+
*/
|
|
448
|
+
function claimAndSpawn(deps, config, batchId, phase, now, spawn) {
|
|
449
|
+
return deps.store.withLock((state) => {
|
|
450
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
451
|
+
if (!batch || slotFor(state, batchId))
|
|
452
|
+
return { state, result: false };
|
|
453
|
+
if ((0, scheduler_1.freeCapacity)(state, config) === 0)
|
|
454
|
+
return { state, result: false };
|
|
455
|
+
const assigned = (0, scheduler_1.assignToIdleSlot)(state, unit(batchId), phase, now, 'cycle');
|
|
456
|
+
const slot = assigned.state.slots.find((s) => s.id === assigned.slotId);
|
|
457
|
+
if (!slot)
|
|
458
|
+
return { state, result: false };
|
|
459
|
+
return { state: spawn(assigned.state, slot), result: true };
|
|
460
|
+
});
|
|
461
|
+
}
|
|
462
|
+
function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
463
|
+
claimAndSpawn(deps, config, batchId, 'member', now, (state, slot) => spawnMember(deps, dispatch, state, slot, batchId, now, result));
|
|
464
|
+
}
|
|
465
|
+
function spawnTailAgent(deps, config, dispatch, batchId, now, result) {
|
|
466
|
+
claimAndSpawn(deps, config, batchId, 'reviewing', now, (state, slot) => {
|
|
467
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
468
|
+
if (!batch || batch.worktree === null || batch.anchor === null) {
|
|
469
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
470
|
+
reason: 'no-worktree-or-anchor',
|
|
471
|
+
detail: 'spawnTailAgent: batch has no worktree/anchor',
|
|
472
|
+
});
|
|
473
|
+
return releaseSlot(state, batchId, now);
|
|
474
|
+
}
|
|
475
|
+
const cmd = (0, dispatch_1.buildAgentCommand)(dispatch.command, 'strong', batch.anchor, dispatch.tierModels);
|
|
476
|
+
const prompt = (0, dispatch_1.buildBatchTailPrompt)(dispatch.batchTailPrompt, batchId, batch.anchor, batch.members, batch.worktree);
|
|
477
|
+
const logFile = path.join(deps.store.runsDir, `${(0, dispatch_1.unitLogName)(unit(batchId))}-tail.log`);
|
|
478
|
+
let pid;
|
|
479
|
+
try {
|
|
480
|
+
pid = deps.spawnDeps.spawn(cmd, prompt, logFile);
|
|
481
|
+
}
|
|
482
|
+
catch (err) {
|
|
483
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
484
|
+
reason: 'spawn-error',
|
|
485
|
+
detail: `tail agent spawn failed: ${err.message}`,
|
|
486
|
+
});
|
|
487
|
+
result.failed.push(unit(batchId));
|
|
488
|
+
return releaseSlot(state, batchId, now);
|
|
489
|
+
}
|
|
490
|
+
const patch = {
|
|
491
|
+
pid,
|
|
492
|
+
pid_start: deps.spawnDeps.processStart(pid),
|
|
493
|
+
phase: 'reviewing',
|
|
494
|
+
last_progress_at: now.toISOString(),
|
|
495
|
+
};
|
|
496
|
+
const next = slot.status === 'assigned' ? (0, state_1.transitionSlot)(state, slot.id, 'running', patch, now) : state;
|
|
497
|
+
deps.journal.append((0, journal_1.unitEvent)('spawned', unit(batchId), { pid, slot: slot.id }), now);
|
|
498
|
+
result.spawned.push(unit(batchId));
|
|
499
|
+
return next;
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
function spawnReportAgent(deps, config, dispatch, batchId, now, result) {
|
|
503
|
+
claimAndSpawn(deps, config, batchId, 'report', now, (state, slot) => {
|
|
504
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
505
|
+
if (!batch || batch.anchor === null) {
|
|
506
|
+
journalEvent(deps, 'report-failed', unit(batchId), {
|
|
507
|
+
detail: 'spawnReportAgent: batch has no anchor',
|
|
508
|
+
});
|
|
509
|
+
return releaseSlot(state, batchId, now);
|
|
510
|
+
}
|
|
511
|
+
const prNumber = batch.pr;
|
|
512
|
+
if (prNumber === null) {
|
|
513
|
+
journalEvent(deps, 'report-failed', unit(batchId), {
|
|
514
|
+
detail: 'spawnReportAgent: batch has no parked pr recorded',
|
|
515
|
+
});
|
|
516
|
+
return releaseSlot(state, batchId, now);
|
|
517
|
+
}
|
|
518
|
+
const cmd = (0, dispatch_1.buildAgentCommand)(dispatch.command, 'mechanical', batch.anchor, dispatch.tierModels);
|
|
519
|
+
const prompt = (0, dispatch_1.buildBatchReportPrompt)(dispatch.batchReportPrompt, batchId, batch.anchor, prNumber);
|
|
520
|
+
const logFile = path.join(deps.store.runsDir, `${(0, dispatch_1.unitLogName)(unit(batchId))}-report.log`);
|
|
521
|
+
let pid;
|
|
522
|
+
try {
|
|
523
|
+
pid = deps.spawnDeps.spawn(cmd, prompt, logFile);
|
|
524
|
+
}
|
|
525
|
+
catch (err) {
|
|
526
|
+
deps.journal.append((0, journal_1.unitEvent)('report-failed', unit(batchId), { detail: err.message }), now);
|
|
527
|
+
return releaseSlot(state, batchId, now);
|
|
528
|
+
}
|
|
529
|
+
const patchState = {
|
|
530
|
+
pid,
|
|
531
|
+
pid_start: deps.spawnDeps.processStart(pid),
|
|
532
|
+
phase: 'report',
|
|
533
|
+
last_progress_at: now.toISOString(),
|
|
534
|
+
};
|
|
535
|
+
const next = slot.status === 'assigned'
|
|
536
|
+
? (0, state_1.transitionSlot)(state, slot.id, 'running', patchState, now)
|
|
537
|
+
: state;
|
|
538
|
+
journalEvent(deps, 'report-dispatched', unit(batchId), { pid, slot: slot.id, pr: prNumber });
|
|
539
|
+
result.spawned.push(unit(batchId));
|
|
540
|
+
return next;
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
// --- Aggregate validate + attribution/fix/evict (RFC F.2) ---
|
|
544
|
+
function memberFootprints(deps, batch) {
|
|
545
|
+
if (batch.worktree === null)
|
|
546
|
+
return [];
|
|
547
|
+
return batch.ranges.map((range) => {
|
|
548
|
+
// `range.commits` is persisted state — validate as shas before they
|
|
549
|
+
// become git argv (CWE-88), the same discipline `recovery.ts`'s revert
|
|
550
|
+
// path applies to the identical values (attribution.ts's `SHA_RE` doc).
|
|
551
|
+
const commits = range.commits.filter((c) => attribution_1.SHA_RE.test(c));
|
|
552
|
+
if (commits.length === 0)
|
|
553
|
+
return { issue: range.issue, changedPaths: [], focusedTests: [] };
|
|
554
|
+
const out = deps.exec('git', ['show', '--name-only', '--format=', ...commits], batch.worktree);
|
|
555
|
+
const changedPaths = (out ?? '')
|
|
556
|
+
.split('\n')
|
|
557
|
+
.map((l) => l.trim())
|
|
558
|
+
.filter((l) => l.length > 0);
|
|
559
|
+
return { issue: range.issue, changedPaths, focusedTests: [] };
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
function boundaryCommits(deps, batch) {
|
|
563
|
+
if (batch.worktree === null || batch.branch === null)
|
|
564
|
+
return [];
|
|
565
|
+
const out = deps.exec('git', ['log', '--reverse', '--format=%H%x09%s', `origin/${batch.base_branch}..${batch.branch}`], batch.worktree);
|
|
566
|
+
return (0, attribution_1.parseBoundaryCommits)(out);
|
|
567
|
+
}
|
|
568
|
+
/**
|
|
569
|
+
* `validating`, no live slot: run the aggregate suite (deterministic — no
|
|
570
|
+
* agent, no slot claimed, matching AC5's "member or batch-LLM-step" wording).
|
|
571
|
+
* Green proceeds to the tail; red attributes and either fixes one offender or
|
|
572
|
+
* dissolves when nothing could be attributed (RFC F.2/F.8).
|
|
573
|
+
*/
|
|
574
|
+
function runValidate(deps, config, dispatch, batchId, now, result) {
|
|
575
|
+
const state = deps.store.load();
|
|
576
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
577
|
+
if (!batch || batch.worktree === null)
|
|
578
|
+
return;
|
|
579
|
+
const suite = safeSuite(deps, batchId, batch.worktree);
|
|
580
|
+
const rDeps = recoveryDeps(deps, batch, now);
|
|
581
|
+
const poster = (0, recovery_1.createExecMilestonePoster)(deps.exec, { repoDir: deps.repoDir });
|
|
582
|
+
if (suite.ok) {
|
|
583
|
+
if (batch.anchor !== null && batch.run_id !== null) {
|
|
584
|
+
poster(batch.anchor, batch.run_id, { phase: 'batch-validate', status: 'done', kv: {} });
|
|
585
|
+
}
|
|
586
|
+
deps.journal.append((0, journal_1.unitEvent)('verify-complete', unit(batchId), { detail: 'suite green' }), now);
|
|
587
|
+
deps.store.withLock((s) => {
|
|
588
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
589
|
+
if (!b || b.status !== 'validating')
|
|
590
|
+
return { state: s, result: undefined };
|
|
591
|
+
return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), result: undefined };
|
|
592
|
+
});
|
|
593
|
+
spawnTailAgent(deps, config, dispatch, batchId, now, result);
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
const { state: attributed, outcome } = (0, recovery_1.beginAttribution)(state, batchId, { failing: suite.failing, footprints: memberFootprints(deps, batch) }, rDeps);
|
|
597
|
+
if (outcome.offenders.length === 0) {
|
|
598
|
+
const dissolve = (0, recovery_1.dissolveBatch)(attributed, batchId, { strategy: 'full', reason: 'unattributable-suite-failure' }, rDeps);
|
|
599
|
+
deps.store.withLock((s) => ({
|
|
600
|
+
state: applyBatchAndIssues(s, dissolve.state, batchId, dissolve.requeued),
|
|
601
|
+
result: undefined,
|
|
602
|
+
}));
|
|
603
|
+
teardownBatch(deps, batchId);
|
|
604
|
+
result.blocked.push(...dissolve.requeued);
|
|
605
|
+
result.failed.push(unit(batchId));
|
|
606
|
+
return;
|
|
607
|
+
}
|
|
608
|
+
const offender = outcome.offenders[0];
|
|
609
|
+
const { state: fixing, dispatch: fixDispatch } = (0, recovery_1.beginFixAttempt)(attributed, batchId, offender, rDeps, { config, tests: outcome.attributed.get(offender) ?? [] });
|
|
610
|
+
deps.store.withLock((s) => ({
|
|
611
|
+
state: applyBatchAndIssues(s, fixing, batchId, []),
|
|
612
|
+
result: undefined,
|
|
613
|
+
}));
|
|
614
|
+
if (fixDispatch === null) {
|
|
615
|
+
// Already had its one attempt — evict directly (mirrors the module's own
|
|
616
|
+
// documented next step when `beginFixAttempt` refuses).
|
|
617
|
+
evictOffender(deps, config, batchId, offender, outcome.method, now, result);
|
|
618
|
+
return;
|
|
619
|
+
}
|
|
620
|
+
claimAndSpawn(deps, config, batchId, 'fixing', now, (s, slot) => {
|
|
621
|
+
const logFile = path.join(deps.store.runsDir, `${(0, dispatch_1.unitLogName)(unit(batchId))}-fix-${offender}.log`);
|
|
622
|
+
let pid;
|
|
623
|
+
try {
|
|
624
|
+
pid = deps.spawnDeps.spawn(fixDispatch.command, fixDispatch.prompt, logFile);
|
|
625
|
+
}
|
|
626
|
+
catch (err) {
|
|
627
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
628
|
+
issue: offender,
|
|
629
|
+
reason: 'fix-spawn-error',
|
|
630
|
+
detail: `fix agent spawn failed: ${err.message}`,
|
|
631
|
+
});
|
|
632
|
+
// The fix attempt was already recorded `dispatched` by `beginFixAttempt`
|
|
633
|
+
// — a spawn failure never dispatched anything, so resolve it `red`
|
|
634
|
+
// (pure, no I/O — safe inside this lock) rather than leaving the state
|
|
635
|
+
// claiming an attempt is in flight forever.
|
|
636
|
+
const resolved = (0, recovery_1.resolveFixAttempt)(s, batchId, offender, 'red', rDeps).state;
|
|
637
|
+
return releaseSlot(resolved, batchId, now);
|
|
638
|
+
}
|
|
639
|
+
const patch = {
|
|
640
|
+
pid,
|
|
641
|
+
pid_start: deps.spawnDeps.processStart(pid),
|
|
642
|
+
phase: 'fixing',
|
|
643
|
+
last_progress_at: now.toISOString(),
|
|
644
|
+
};
|
|
645
|
+
const next = slot.status === 'assigned' ? (0, state_1.transitionSlot)(s, slot.id, 'running', patch, now) : s;
|
|
646
|
+
result.spawned.push(unit(batchId));
|
|
647
|
+
return next;
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
function evictOffender(deps, _config, batchId, offender, attribution, now, result) {
|
|
651
|
+
const state = deps.store.load();
|
|
652
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
653
|
+
if (!batch)
|
|
654
|
+
return;
|
|
655
|
+
const rDeps = recoveryDeps(deps, batch, now);
|
|
656
|
+
const outcome = (0, recovery_1.evictMembers)(state, batchId, { issues: [offender], reason: 'suite-red-after-fix', attribution, ranges: batch.ranges }, rDeps);
|
|
657
|
+
deps.store.withLock((s) => ({
|
|
658
|
+
state: applyBatchAndIssues(s, outcome.state, batchId, outcome.requeued),
|
|
659
|
+
result: undefined,
|
|
660
|
+
}));
|
|
661
|
+
if (outcome.dissolved) {
|
|
662
|
+
result.failed.push(unit(batchId));
|
|
663
|
+
return;
|
|
664
|
+
}
|
|
665
|
+
if (outcome.suite?.ok) {
|
|
666
|
+
deps.store.withLock((s) => {
|
|
667
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
668
|
+
if (!b || b.status !== 'validating')
|
|
669
|
+
return { state: s, result: undefined };
|
|
670
|
+
return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), result: undefined };
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
// --- Reconcile a batch currently holding a live/exited slot ---
|
|
675
|
+
/**
|
|
676
|
+
* After a member leaves `executing` (green, gate-failed, or self-blocked):
|
|
677
|
+
* advance to the next member, or — on the last member — transition to
|
|
678
|
+
* `validating` and run the aggregate suite. Shared by every exit from
|
|
679
|
+
* `reconcileMemberSlot` so the pointer-advance rail exists once.
|
|
680
|
+
*/
|
|
681
|
+
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result) {
|
|
682
|
+
const isLast = currentMember >= memberCount;
|
|
683
|
+
if (isLast) {
|
|
684
|
+
deps.store.withLock((s) => {
|
|
685
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
686
|
+
if (!b || b.status !== 'executing')
|
|
687
|
+
return { state: s, result: undefined };
|
|
688
|
+
return { state: (0, state_1.transitionBatch)(s, batchId, 'validating', {}, now), result: undefined };
|
|
689
|
+
});
|
|
690
|
+
runValidate(deps, config, dispatch, batchId, now, result);
|
|
691
|
+
return;
|
|
692
|
+
}
|
|
693
|
+
deps.store.withLock((s) => {
|
|
694
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
695
|
+
if (!b || b.status !== 'executing')
|
|
696
|
+
return { state: s, result: undefined };
|
|
697
|
+
return {
|
|
698
|
+
state: (0, state_1.transitionBatch)(s, batchId, 'executing', { executing_member: b.executing_member + 1 }, now),
|
|
699
|
+
result: undefined,
|
|
700
|
+
};
|
|
701
|
+
});
|
|
702
|
+
journalEvent(deps, 'member-advanced', unit(batchId), { issue: memberIssue });
|
|
703
|
+
spawnMemberContinuation(deps, config, dispatch, batchId, now, result);
|
|
704
|
+
}
|
|
705
|
+
/**
|
|
706
|
+
* Evict the current member and either dissolve, or continue the batch via
|
|
707
|
+
* `advanceMemberOrValidate` — the shared tail of both member-failure rails
|
|
708
|
+
* (self-reported blocked, and the incremental gate below).
|
|
709
|
+
*/
|
|
710
|
+
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, reason, now, result) {
|
|
711
|
+
const dissolved = evictMemberDirectly(deps, batchId, memberIssue, reason, now);
|
|
712
|
+
if (dissolved) {
|
|
713
|
+
result.failed.push(unit(batchId));
|
|
714
|
+
return;
|
|
715
|
+
}
|
|
716
|
+
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
717
|
+
}
|
|
718
|
+
function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result) {
|
|
719
|
+
const state0 = deps.store.load();
|
|
720
|
+
const batch = (0, state_1.findBatch)(state0, batchId);
|
|
721
|
+
if (!batch)
|
|
722
|
+
return;
|
|
723
|
+
const memberIssue = batch.members[batch.executing_member - 1];
|
|
724
|
+
if (memberIssue === undefined)
|
|
725
|
+
return;
|
|
726
|
+
const dead = slot.pid !== null && !deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
|
|
727
|
+
const milestone = deps.groundTruth.latestMilestone(memberIssue);
|
|
728
|
+
if (milestone === undefined)
|
|
729
|
+
return; // unreachable — pause this batch's decisions
|
|
730
|
+
if ((0, groundtruth_1.isMemberComplete)(milestone)) {
|
|
731
|
+
deps.journal.append((0, journal_1.unitEvent)('external-advance', unit(batchId), {
|
|
732
|
+
issue: memberIssue,
|
|
733
|
+
detail: 'member review done',
|
|
734
|
+
}), now);
|
|
735
|
+
// Incremental gate (#523 AC2): typecheck + focused tests via `cap run`,
|
|
736
|
+
// when the repo has a manifest for them — a second, independent check
|
|
737
|
+
// that the member's own self-reported "done" is real, matching this
|
|
738
|
+
// codebase's "never trust a claimed completion" ethos (AC2/#464's
|
|
739
|
+
// `isVerifiedComplete`). `task-failed` evicts the member directly, same
|
|
740
|
+
// rail as a self-reported block (RFC F.1) — no aggregate suite has run
|
|
741
|
+
// yet, so there is nothing to attribute.
|
|
742
|
+
if (batch.worktree !== null && deps.runCapability) {
|
|
743
|
+
const gateFailure = ['typecheck.run', 'test.focused']
|
|
744
|
+
.map((id) => ({ id, outcome: deps.runCapability?.(batch.worktree, id) }))
|
|
745
|
+
.find((r) => r.outcome === 'task-failed');
|
|
746
|
+
if (gateFailure) {
|
|
747
|
+
const reason = `incremental-gate-failed:${gateFailure.id}`;
|
|
748
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
749
|
+
issue: memberIssue,
|
|
750
|
+
reason,
|
|
751
|
+
detail: `cap run ${gateFailure.id} reported task-failed after member review done`,
|
|
752
|
+
});
|
|
753
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
754
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, reason, now, result);
|
|
755
|
+
return;
|
|
756
|
+
}
|
|
757
|
+
// `ok` / `automation-broken` / `capability-unavailable` all proceed —
|
|
758
|
+
// only a definite task failure blocks a member here.
|
|
759
|
+
}
|
|
760
|
+
// The commit-range recompute (`git log`) is a blocking subprocess call —
|
|
761
|
+
// it must run OUTSIDE the lock, like every other exec in this module;
|
|
762
|
+
// the result then lands as a pure data patch under the lock (Convention
|
|
763
|
+
// review: `recordRanges` used to run `git log` INSIDE the withLock
|
|
764
|
+
// mutator, which is exactly what `engine.ts`'s own "a slow git call never
|
|
765
|
+
// holds the lock" invariant exists to prevent).
|
|
766
|
+
const ranges = (0, attribution_1.memberRanges)(boundaryCommits(deps, batch));
|
|
767
|
+
deps.store.withLock((s) => {
|
|
768
|
+
let n = releaseSlot(s, batchId, now);
|
|
769
|
+
n = (0, state_1.patchBatch)(n, batchId, { ranges }, now);
|
|
770
|
+
n = advanceMemberToValidated(n, memberIssue, now);
|
|
771
|
+
return { state: n, result: undefined };
|
|
772
|
+
});
|
|
773
|
+
result.completed.push(unit(batchId));
|
|
774
|
+
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
if ((0, groundtruth_1.isMemberBlocked)(milestone) || dead) {
|
|
778
|
+
const rawReason = milestone?.keys.reason;
|
|
779
|
+
const reason = typeof rawReason === 'string' && rawReason.length > 0
|
|
780
|
+
? sanitizeUntrustedText(rawReason)
|
|
781
|
+
: dead
|
|
782
|
+
? 'agent-exited-unverified'
|
|
783
|
+
: 'member-blocked';
|
|
784
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
785
|
+
issue: memberIssue,
|
|
786
|
+
reason,
|
|
787
|
+
detail: 'member blocked',
|
|
788
|
+
});
|
|
789
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
790
|
+
// A member that never went green (RFC F.1) evicts DIRECTLY — no aggregate
|
|
791
|
+
// suite has run yet, so there is nothing for `attributing`/`evicting` (the
|
|
792
|
+
// AGGREGATE-suite-red pipeline, RFC F.2) to attribute or revert: the
|
|
793
|
+
// offender is already known, and `batch.ranges` has no entry for a member
|
|
794
|
+
// that never reached `isMemberComplete`. `executing → validating →
|
|
795
|
+
// attributing → evicting` is not even a legal edge from mid-`executing`
|
|
796
|
+
// (BATCH_TRANSITIONS has no `executing → evicting`) — this stays entirely
|
|
797
|
+
// within `executing`/`dissolving`, both of which ARE legal from here.
|
|
798
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, reason, now, result);
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* Requeue a member full-cycle, record the eviction, and dissolve if this tips
|
|
803
|
+
* the batch past the ⅓ threshold (RFC F.1/F.8) — WITHOUT going through
|
|
804
|
+
* `recovery.ts`'s `evictMembers` (which needs `attributing`/`evicting` status
|
|
805
|
+
* and a commit range to revert; a member evicted here has neither). Returns
|
|
806
|
+
* whether the batch dissolved.
|
|
807
|
+
*/
|
|
808
|
+
function evictMemberDirectly(deps, batchId, memberIssue, reason, now) {
|
|
809
|
+
// Pass 1 (pure — requeue + record the eviction): safe to run entirely
|
|
810
|
+
// inside the lock, unlike `dissolveBatch` below, which shells out
|
|
811
|
+
// (`deps.exec`/`postMilestone`) and so must NOT hold the lock while it runs.
|
|
812
|
+
const triggered = deps.store.withLock((s) => {
|
|
813
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
814
|
+
if (!b)
|
|
815
|
+
return { state: s, result: false };
|
|
816
|
+
const evidence = {
|
|
817
|
+
batch: batchId,
|
|
818
|
+
reason,
|
|
819
|
+
failing_tests: [],
|
|
820
|
+
attribution: 'none',
|
|
821
|
+
reverted_commits: [],
|
|
822
|
+
at: now.toISOString(),
|
|
823
|
+
};
|
|
824
|
+
const requeueResult = (0, state_1.requeueMember)(s, memberIssue, { mode: 'full', batch: null }, reason, now, { failure_evidence: evidence });
|
|
825
|
+
let next = requeueResult.state;
|
|
826
|
+
next = (0, state_1.patchBatch)(next, batchId, {
|
|
827
|
+
evictions: [
|
|
828
|
+
...b.evictions,
|
|
829
|
+
{
|
|
830
|
+
issue: memberIssue,
|
|
831
|
+
reason,
|
|
832
|
+
attribution: 'none',
|
|
833
|
+
reverted_commits: [],
|
|
834
|
+
group: [],
|
|
835
|
+
at: now.toISOString(),
|
|
836
|
+
},
|
|
837
|
+
],
|
|
838
|
+
}, now);
|
|
839
|
+
const updated = (0, state_1.findBatch)(next, batchId);
|
|
840
|
+
return { state: next, result: updated !== undefined && (0, recovery_1.checkDissolveTrigger)(updated) };
|
|
841
|
+
});
|
|
842
|
+
if (!triggered)
|
|
843
|
+
return false;
|
|
844
|
+
// Pass 2 (outside the lock — dissolveBatch shells out): re-load fresh
|
|
845
|
+
// (pass 1's write already landed), dissolve, then re-apply just this
|
|
846
|
+
// batch's + the requeued members' state under a fresh lock.
|
|
847
|
+
const state = deps.store.load();
|
|
848
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
849
|
+
if (!batch)
|
|
850
|
+
return false;
|
|
851
|
+
const rDeps = recoveryDeps(deps, batch, now);
|
|
852
|
+
const outcome = (0, recovery_1.dissolveBatch)(state, batchId, { strategy: 'full', reason: 'eviction-threshold' }, rDeps);
|
|
853
|
+
deps.store.withLock((s) => ({
|
|
854
|
+
state: applyBatchAndIssues(s, outcome.state, batchId, outcome.requeued),
|
|
855
|
+
result: undefined,
|
|
856
|
+
}));
|
|
857
|
+
teardownBatch(deps, batchId);
|
|
858
|
+
return true;
|
|
859
|
+
}
|
|
860
|
+
function reconcileFixSlot(deps, config, batchId, slot, now, result) {
|
|
861
|
+
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined))
|
|
862
|
+
return; // still running
|
|
863
|
+
const state = deps.store.load();
|
|
864
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
865
|
+
if (!batch || batch.worktree === null)
|
|
866
|
+
return;
|
|
867
|
+
const offenderRecord = [...batch.fix_attempts].reverse().find((a) => a.outcome === 'dispatched');
|
|
868
|
+
if (!offenderRecord)
|
|
869
|
+
return;
|
|
870
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
871
|
+
const suite = safeSuite(deps, batchId, batch.worktree);
|
|
872
|
+
const rDeps = recoveryDeps(deps, batch, now);
|
|
873
|
+
const { state: resolved } = (0, recovery_1.resolveFixAttempt)(deps.store.load(), batchId, offenderRecord.issue, suite.ok ? 'green' : 'red', rDeps);
|
|
874
|
+
deps.store.withLock((s) => ({
|
|
875
|
+
state: applyBatchAndIssues(s, resolved, batchId, []),
|
|
876
|
+
result: undefined,
|
|
877
|
+
}));
|
|
878
|
+
if (suite.ok) {
|
|
879
|
+
deps.store.withLock((s) => {
|
|
880
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
881
|
+
if (!b || b.status !== 'validating')
|
|
882
|
+
return { state: s, result: undefined };
|
|
883
|
+
return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), result: undefined };
|
|
884
|
+
});
|
|
885
|
+
return;
|
|
886
|
+
}
|
|
887
|
+
evictOffender(deps, config, batchId, offenderRecord.issue, 'overlap', now, result);
|
|
888
|
+
}
|
|
889
|
+
function reconcileTailSlot(deps, batchId, slot, now, result) {
|
|
890
|
+
const state = deps.store.load();
|
|
891
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
892
|
+
if (!batch || batch.anchor === null)
|
|
893
|
+
return;
|
|
894
|
+
const dead = slot.pid !== null && !deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
|
|
895
|
+
const milestone = deps.groundTruth.latestMilestone(batch.anchor);
|
|
896
|
+
if (milestone === undefined)
|
|
897
|
+
return;
|
|
898
|
+
if (batch.status === 'reviewing' && (0, groundtruth_1.isBatchPhaseDone)(milestone, 'batch-review')) {
|
|
899
|
+
deps.store.withLock((s) => {
|
|
900
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
901
|
+
if (!b || b.status !== 'reviewing')
|
|
902
|
+
return { state: s, result: undefined };
|
|
903
|
+
return { state: (0, state_1.transitionBatch)(s, batchId, 'shipping', {}, now), result: undefined };
|
|
904
|
+
});
|
|
905
|
+
return;
|
|
906
|
+
}
|
|
907
|
+
if ((0, groundtruth_1.isBatchTailParked)(milestone)) {
|
|
908
|
+
const pr = (0, groundtruth_1.prOfMilestone)(milestone);
|
|
909
|
+
journalEvent(deps, 'pr-parked', unit(batchId), { pr: pr ?? undefined });
|
|
910
|
+
deps.store.withLock((s) => {
|
|
911
|
+
let n = releaseSlot(s, batchId, now);
|
|
912
|
+
const b = (0, state_1.findBatch)(n, batchId);
|
|
913
|
+
if (!b)
|
|
914
|
+
return { state: n, result: undefined };
|
|
915
|
+
n = b.status === 'reviewing' ? (0, state_1.transitionBatch)(n, batchId, 'shipping', {}, now) : n;
|
|
916
|
+
n = (0, state_1.transitionBatch)(n, batchId, 'awaiting-merge', { pr }, now);
|
|
917
|
+
return { state: n, result: undefined };
|
|
918
|
+
});
|
|
919
|
+
result.parked.push(unit(batchId));
|
|
920
|
+
return;
|
|
921
|
+
}
|
|
922
|
+
if (dead) {
|
|
923
|
+
deps.journal.append((0, journal_1.unitEvent)('unit-failed', unit(batchId), { reason: 'tail-agent-exited-unverified' }), now);
|
|
924
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
925
|
+
result.failed.push(unit(batchId));
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
function reconcileReportSlot(deps, batchId, slot, now, result) {
|
|
929
|
+
const state = deps.store.load();
|
|
930
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
931
|
+
if (!batch || batch.anchor === null)
|
|
932
|
+
return;
|
|
933
|
+
const dead = slot.pid !== null && !deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
|
|
934
|
+
const milestone = deps.groundTruth.latestMilestone(batch.anchor);
|
|
935
|
+
if (milestone === undefined)
|
|
936
|
+
return;
|
|
937
|
+
if ((0, groundtruth_1.isBatchPhaseDone)(milestone, 'batch-report')) {
|
|
938
|
+
deps.journal.append((0, journal_1.unitEvent)('external-advance', unit(batchId), { detail: 'batch report done' }), now);
|
|
939
|
+
deps.store.withLock((s) => {
|
|
940
|
+
let n = releaseSlot(s, batchId, now);
|
|
941
|
+
const b = (0, state_1.findBatch)(n, batchId);
|
|
942
|
+
if (!b || b.status !== 'deployed')
|
|
943
|
+
return { state: n, result: undefined };
|
|
944
|
+
n = (0, state_1.transitionBatch)(n, batchId, 'reported', {}, now);
|
|
945
|
+
n = (0, state_1.transitionBatch)(n, batchId, 'done', {}, now);
|
|
946
|
+
return { state: n, result: undefined };
|
|
947
|
+
});
|
|
948
|
+
result.completed.push(unit(batchId));
|
|
949
|
+
teardownBatch(deps, batchId);
|
|
950
|
+
return;
|
|
951
|
+
}
|
|
952
|
+
if (dead) {
|
|
953
|
+
deps.journal.append((0, journal_1.unitEvent)('report-failed', unit(batchId), { detail: 'unverified exit' }), now);
|
|
954
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
// --- PR watch for `awaiting-merge` batches (mirrors `pollParkedPrs`/`reconcileParked`) ---
|
|
958
|
+
function reconcilePrWatch(deps, now, result) {
|
|
959
|
+
const state = deps.store.load();
|
|
960
|
+
for (const batch of state.batches) {
|
|
961
|
+
if (batch.status !== 'awaiting-merge')
|
|
962
|
+
continue;
|
|
963
|
+
const pr = batch.pr;
|
|
964
|
+
if (pr === null)
|
|
965
|
+
continue;
|
|
966
|
+
const truth = deps.groundTruth.prState(pr);
|
|
967
|
+
if (truth === undefined)
|
|
968
|
+
continue; // unreachable — keep watching
|
|
969
|
+
if (truth.state === 'MERGED' && truth.mergedAt !== null) {
|
|
970
|
+
journalEvent(deps, 'merge-accepted', unit(batch.id), { pr });
|
|
971
|
+
deps.store.withLock((s) => {
|
|
972
|
+
const b = (0, state_1.findBatch)(s, batch.id);
|
|
973
|
+
if (!b || b.status !== 'awaiting-merge')
|
|
974
|
+
return { state: s, result: undefined };
|
|
975
|
+
let n = (0, state_1.transitionBatch)(s, batch.id, 'merged', {}, now);
|
|
976
|
+
n = (0, state_1.transitionBatch)(n, batch.id, 'deployed', {}, now);
|
|
977
|
+
for (const issue of b.members) {
|
|
978
|
+
const entry = (0, state_1.findEntry)(n, issue);
|
|
979
|
+
if (entry && entry.status !== 'shipped-in-batch') {
|
|
980
|
+
try {
|
|
981
|
+
n = (0, state_1.transitionIssue)(n, issue, 'shipped-in-batch', {}, now);
|
|
982
|
+
n = (0, state_1.transitionIssue)(n, issue, 'done', {}, now);
|
|
983
|
+
}
|
|
984
|
+
catch {
|
|
985
|
+
// Already terminal via another rail — leave it.
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
return { state: n, result: undefined };
|
|
990
|
+
});
|
|
991
|
+
result.mergeAccepted.push(unit(batch.id));
|
|
992
|
+
continue;
|
|
993
|
+
}
|
|
994
|
+
if (truth.blocked || truth.mergeable === 'CONFLICTING') {
|
|
995
|
+
journalEvent(deps, 'pr-watch-failed', unit(batch.id), {
|
|
996
|
+
reason: truth.blocked ? 'auto-merge-blocked' : 'pr-conflicting',
|
|
997
|
+
pr,
|
|
998
|
+
});
|
|
999
|
+
// #472's own rebase-and-reship path (RFC F.9) is a documented follow-up
|
|
1000
|
+
// for the batch PR-conflict rail; for now the batch stays parked and
|
|
1001
|
+
// the block is visible via the journal + `sched status`.
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
/**
|
|
1006
|
+
* Remove the batch's shared worktree — called both on the happy path
|
|
1007
|
+
* (`reconcileReportSlot`, after `batch-report done`) and on every dissolve
|
|
1008
|
+
* path (`runValidate`'s unattributable-suite dissolve, `evictMemberDirectly`'s
|
|
1009
|
+
* eviction-threshold dissolve): a dissolved batch's worktree is otherwise
|
|
1010
|
+
* left on disk forever, since nothing else ever calls this for it.
|
|
1011
|
+
*/
|
|
1012
|
+
function teardownBatch(deps, batchId) {
|
|
1013
|
+
const state = deps.store.load();
|
|
1014
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
1015
|
+
if (!batch || batch.worktree === null)
|
|
1016
|
+
return;
|
|
1017
|
+
const root = deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir;
|
|
1018
|
+
if (!(0, teardown_1.isSafeWorktree)(path.resolve(root), batch.worktree)) {
|
|
1019
|
+
journalEvent(deps, 'teardown-failed', unit(batchId), {
|
|
1020
|
+
reason: 'unsafe-worktree-path',
|
|
1021
|
+
detail: batch.worktree,
|
|
1022
|
+
});
|
|
1023
|
+
return;
|
|
1024
|
+
}
|
|
1025
|
+
const result = (0, teardown_1.runTeardown)(deps.exec, deps.repoDir, { worktree: batch.worktree, poolClaimed: false, branch: batch.branch }, deps.fsExists);
|
|
1026
|
+
journalEvent(deps, result.cleanup === 'done' ? 'teardown-done' : 'teardown-failed', unit(batchId), {
|
|
1027
|
+
cleanup: result.cleanup,
|
|
1028
|
+
detail: result.detail,
|
|
1029
|
+
worktree: batch.worktree,
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
// --- Entry point ---
|
|
1033
|
+
/**
|
|
1034
|
+
* One batch reconcile+refill pass. Called from `engine.ts`'s `tick()` after
|
|
1035
|
+
* the issue-level pass — batches never compete with issues for a slot within
|
|
1036
|
+
* the same tick because this pass runs strictly after `dispatchAssignments`
|
|
1037
|
+
* already filled every slot it could (see the module doc: batch claims never
|
|
1038
|
+
* go through `computeAssignments`/`runnableUnits` at all). Loads and saves
|
|
1039
|
+
* state itself via `deps.store.withLock` — the caller holds no lock across
|
|
1040
|
+
* this call. `deps.exec` and `deps.runSuite` are mandatory; `deps.
|
|
1041
|
+
* runCapability` is independently optional (AC2's incremental gate is itself
|
|
1042
|
+
* a "when available" fast path).
|
|
1043
|
+
*/
|
|
1044
|
+
function runBatchTick(deps, config, dispatch) {
|
|
1045
|
+
const result = emptyResult();
|
|
1046
|
+
const now = deps.now();
|
|
1047
|
+
for (const batch of deps.store.load().batches) {
|
|
1048
|
+
if (batch.status === 'ready' && slotFor(deps.store.load(), batch.id) === undefined) {
|
|
1049
|
+
claimAndSetup(deps, config, dispatch, batch.id, now, result);
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
for (const batch of deps.store.load().batches) {
|
|
1053
|
+
const slot = slotFor(deps.store.load(), batch.id);
|
|
1054
|
+
if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
|
|
1055
|
+
if (batch.status === 'executing') {
|
|
1056
|
+
reconcileMemberSlot(deps, config, dispatch, batch.id, slot, now, result);
|
|
1057
|
+
}
|
|
1058
|
+
else if (batch.status === 'fixing') {
|
|
1059
|
+
reconcileFixSlot(deps, config, batch.id, slot, now, result);
|
|
1060
|
+
}
|
|
1061
|
+
else if (batch.status === 'reviewing' || batch.status === 'shipping') {
|
|
1062
|
+
reconcileTailSlot(deps, batch.id, slot, now, result);
|
|
1063
|
+
}
|
|
1064
|
+
else if (batch.status === 'deployed') {
|
|
1065
|
+
reconcileReportSlot(deps, batch.id, slot, now, result);
|
|
1066
|
+
}
|
|
1067
|
+
continue;
|
|
1068
|
+
}
|
|
1069
|
+
if (slot)
|
|
1070
|
+
continue; // live but neither running/assigned (e.g. mid-verify) — next tick
|
|
1071
|
+
if (batch.status === 'validating') {
|
|
1072
|
+
runValidate(deps, config, dispatch, batch.id, now, result);
|
|
1073
|
+
}
|
|
1074
|
+
else if (batch.status === 'deployed') {
|
|
1075
|
+
spawnReportAgent(deps, config, dispatch, batch.id, now, result);
|
|
1076
|
+
}
|
|
1077
|
+
else if (batch.status === 'executing') {
|
|
1078
|
+
// A prior spawn threw, or `claimAndSpawn` found zero free capacity —
|
|
1079
|
+
// either way the batch is stuck mid-member with no slot and nothing
|
|
1080
|
+
// else will ever retry it (Conformance review AC5 caveat; Supportability
|
|
1081
|
+
// review #12). Retrying every tick is safe: `claimAndSpawn` itself is
|
|
1082
|
+
// the capacity gate, so this is a no-op until a slot actually frees up.
|
|
1083
|
+
spawnMemberContinuation(deps, config, dispatch, batch.id, now, result);
|
|
1084
|
+
}
|
|
1085
|
+
else if (batch.status === 'reviewing' || batch.status === 'shipping') {
|
|
1086
|
+
// Same wedge, for a dead-or-never-claimed tail agent.
|
|
1087
|
+
spawnTailAgent(deps, config, dispatch, batch.id, now, result);
|
|
1088
|
+
}
|
|
1089
|
+
else if (batch.status === 'fixing') {
|
|
1090
|
+
// `beginFixAttempt` already recorded this member's ONE attempt as
|
|
1091
|
+
// `dispatched` before `claimAndSpawn` could find capacity — retrying the
|
|
1092
|
+
// exact same dispatch isn't reconstructible from persisted state (only
|
|
1093
|
+
// the outcome is persisted, not the command/prompt), so resolve it
|
|
1094
|
+
// `red` (conservatively: the member loses its one attempt and evicts on
|
|
1095
|
+
// the next validate pass, which is safe — never a permanent wedge).
|
|
1096
|
+
const state = deps.store.load();
|
|
1097
|
+
const b = (0, state_1.findBatch)(state, batch.id);
|
|
1098
|
+
const offenderRecord = b
|
|
1099
|
+
? [...b.fix_attempts].reverse().find((a) => a.outcome === 'dispatched')
|
|
1100
|
+
: undefined;
|
|
1101
|
+
if (b && offenderRecord) {
|
|
1102
|
+
const rDeps = recoveryDeps(deps, b, now);
|
|
1103
|
+
const { state: resolved } = (0, recovery_1.resolveFixAttempt)(state, batch.id, offenderRecord.issue, 'red', rDeps);
|
|
1104
|
+
deps.store.withLock((s) => ({
|
|
1105
|
+
state: applyBatchAndIssues(s, resolved, batch.id, []),
|
|
1106
|
+
result: undefined,
|
|
1107
|
+
}));
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
reconcilePrWatch(deps, now, result);
|
|
1112
|
+
return result;
|
|
1113
|
+
}
|
|
1114
|
+
//# sourceMappingURL=batch-dispatch.js.map
|