@ai-dossier/sched 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +179 -17
- package/dist/attribution.d.ts +132 -0
- package/dist/attribution.d.ts.map +1 -0
- package/dist/attribution.js +268 -0
- package/dist/attribution.js.map +1 -0
- package/dist/bisect.d.ts +67 -0
- package/dist/bisect.d.ts.map +1 -0
- package/dist/bisect.js +122 -0
- package/dist/bisect.js.map +1 -0
- package/dist/dispatch.d.ts +48 -1
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +92 -5
- package/dist/dispatch.js.map +1 -1
- package/dist/engine.d.ts +53 -13
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +421 -75
- package/dist/engine.js.map +1 -1
- package/dist/enqueue.d.ts +9 -0
- package/dist/enqueue.d.ts.map +1 -1
- package/dist/enqueue.js +96 -12
- package/dist/enqueue.js.map +1 -1
- package/dist/groundtruth.d.ts +66 -5
- package/dist/groundtruth.d.ts.map +1 -1
- package/dist/groundtruth.js +150 -5
- package/dist/groundtruth.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +51 -2
- package/dist/index.js.map +1 -1
- package/dist/persist.d.ts.map +1 -1
- package/dist/persist.js +14 -0
- package/dist/persist.js.map +1 -1
- package/dist/recovery.d.ts +263 -0
- package/dist/recovery.d.ts.map +1 -0
- package/dist/recovery.js +730 -0
- package/dist/recovery.js.map +1 -0
- package/dist/scheduler.d.ts +12 -0
- package/dist/scheduler.d.ts.map +1 -1
- package/dist/scheduler.js +53 -58
- package/dist/scheduler.js.map +1 -1
- package/dist/state.d.ts +57 -4
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +238 -8
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts +16 -4
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +10 -4
- package/dist/status.js.map +1 -1
- package/dist/teardown.d.ts +57 -0
- package/dist/teardown.d.ts.map +1 -0
- package/dist/teardown.js +242 -0
- package/dist/teardown.js.map +1 -0
- package/dist/types.d.ts +155 -4
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +39 -5
- package/dist/types.js.map +1 -1
- package/package.json +2 -2
package/dist/engine.js
CHANGED
|
@@ -2,12 +2,16 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* The scheduler engine (#464): dispatch, completion verification, and the
|
|
4
4
|
* stall/escalation ladder (RFC-0001 §C.1) — the deterministic organs that
|
|
5
|
-
* replace fleet-cycle's LLM supervision.
|
|
5
|
+
* replace fleet-cycle's LLM supervision. Since #468 it also owns the
|
|
6
|
+
* detached-ship tail: the PR watcher, script-based teardown, and the
|
|
7
|
+
* cheap-tier report dispatch.
|
|
6
8
|
*
|
|
7
9
|
* One `tick()` is a full reconcile+refill cycle:
|
|
8
10
|
*
|
|
9
|
-
* 1. **Poll** ground truth for every live unit
|
|
10
|
-
* subprocesses are slow; the lock must never wait
|
|
11
|
+
* 1. **Poll** ground truth for every live unit and every parked PR OUTSIDE
|
|
12
|
+
* the state lock (gh/git subprocesses are slow; the lock must never wait
|
|
13
|
+
* on a network call). Parked PRs poll on their own cadence
|
|
14
|
+
* (`pr_poll_interval_ms`, persisted `last_pr_poll_at`) — every 2–3 min.
|
|
11
15
|
* 2. **Apply** under `SchedStore.withLock`:
|
|
12
16
|
* - `assigned` slots (crash between assign and spawn) are spawned/re-attached
|
|
13
17
|
* - `running` slots: dead pid → exit rail; ground truth says complete →
|
|
@@ -15,13 +19,25 @@
|
|
|
15
19
|
* pushed commit → progress; no progress for `stall_timeout_ms` →
|
|
16
20
|
* redispatch one tier stronger (cap 2, then failed)
|
|
17
21
|
* - `exited`/`verifying` slots: the agent exited — completion is verified
|
|
18
|
-
* against ground truth, never assumed (AC2);
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
+
* against ground truth, never assumed (AC2); an exit whose milestone is
|
|
23
|
+
* the ship phase's `awaiting-merge` (with `pr=`) is a VERIFIED park:
|
|
24
|
+
* entry → parked, slot released (a parked unit holds no slot — AC5)
|
|
25
|
+
* - `parked` entries: the watcher applies the PR truth — merge accepted
|
|
26
|
+
* only when state MERGED AND mergedAt non-null AND the issue is closed
|
|
27
|
+
* (AC1) → shipped (gating on MERGE, never the park — AC4);
|
|
28
|
+
* CONFLICTING / closed-unmerged / auto-merge-blocked → failed + transitive
|
|
29
|
+
* dependents blocked (AC3)
|
|
22
30
|
* - failures block their TRANSITIVE dependents (AC4)
|
|
23
|
-
*
|
|
24
|
-
*
|
|
31
|
+
* - report agents are dispatched for merged units whose teardown is
|
|
32
|
+
* already recorded (before queue refill — cheap reports don't queue
|
|
33
|
+
* behind long runs)
|
|
34
|
+
* - **Refill** in the SAME lock pass: `computeAssignments` fills every
|
|
35
|
+
* freed slot — a runnable unit never waits while a slot is idle (AC5)
|
|
36
|
+
* 3. **Teardown** (outside the lock — pool/git subprocesses are slow): for
|
|
37
|
+
* every freshly-merged unit, recover the setup milestone's worktree info
|
|
38
|
+
* and run pool return / worktree remove, VERIFIED before claimed
|
|
39
|
+
* (`cleanup=failed-<step>` on mismatch, AC2). Results land in a second
|
|
40
|
+
* short lock pass together with the report dispatch.
|
|
25
41
|
*
|
|
26
42
|
* Everything that touches the world (processes, GitHub, git) is injected;
|
|
27
43
|
* the state machine is pure. Only `issue:<n>` units are dispatched — batch
|
|
@@ -69,12 +85,19 @@ const groundtruth_1 = require("./groundtruth");
|
|
|
69
85
|
const journal_1 = require("./journal");
|
|
70
86
|
const scheduler_1 = require("./scheduler");
|
|
71
87
|
const state_1 = require("./state");
|
|
88
|
+
const teardown_1 = require("./teardown");
|
|
72
89
|
const types_1 = require("./types");
|
|
73
90
|
function emptyResult() {
|
|
74
91
|
return {
|
|
75
92
|
spawned: [],
|
|
76
93
|
externalAdvances: [],
|
|
77
94
|
completed: [],
|
|
95
|
+
parked: [],
|
|
96
|
+
mergeAccepted: [],
|
|
97
|
+
reportDispatched: [],
|
|
98
|
+
reportWaiting: 0,
|
|
99
|
+
teardownDone: [],
|
|
100
|
+
teardownFailed: [],
|
|
78
101
|
redispatched: [],
|
|
79
102
|
failed: [],
|
|
80
103
|
blocked: [],
|
|
@@ -138,39 +161,53 @@ function pollUnits(deps, state) {
|
|
|
138
161
|
}
|
|
139
162
|
return out;
|
|
140
163
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
164
|
+
/**
|
|
165
|
+
* Poll parked PRs on their own cadence (#468 AC1 — every 2–3 min, persisted
|
|
166
|
+
* `last_pr_poll_at` so a restart honors it). Runs only when parked entries
|
|
167
|
+
* exist AND the interval elapsed; every subprocess stays outside the lock.
|
|
168
|
+
* The issue-closed signal rides along — it is the second half of the
|
|
169
|
+
* merge-acceptance gate and must not be re-queried under the lock.
|
|
170
|
+
*/
|
|
171
|
+
function pollParkedPrs(deps, state, dispatch) {
|
|
172
|
+
const parked = state.entries.filter((e) => e.status === 'parked' && e.pr !== null);
|
|
173
|
+
if (parked.length === 0)
|
|
174
|
+
return { ran: false, truths: new Map(), closed: new Map() };
|
|
175
|
+
const now = deps.now().getTime();
|
|
176
|
+
const last = state.last_pr_poll_at !== null ? Date.parse(state.last_pr_poll_at) : 0;
|
|
177
|
+
if (Number.isFinite(last) && now - last < dispatch.prPollIntervalMs) {
|
|
178
|
+
return { ran: false, truths: new Map(), closed: new Map() };
|
|
155
179
|
}
|
|
156
|
-
const
|
|
157
|
-
const
|
|
180
|
+
const truths = new Map();
|
|
181
|
+
const closed = new Map();
|
|
182
|
+
for (const entry of parked) {
|
|
183
|
+
truths.set(entry.issue, deps.groundTruth.prState(entry.pr));
|
|
184
|
+
closed.set(entry.issue, deps.groundTruth.issueClosed(entry.issue));
|
|
185
|
+
}
|
|
186
|
+
return { ran: true, truths, closed };
|
|
187
|
+
}
|
|
188
|
+
// --- Spawn / fail / complete / park ---
|
|
189
|
+
/**
|
|
190
|
+
* The shared spawn-and-record tail of every agent dispatch (#464 full-cycle,
|
|
191
|
+
* #468 report): log file, try-spawn (a throw fails the unit through the
|
|
192
|
+
* declared failure rail — visible in `sched status`, never a tick abort),
|
|
193
|
+
* pid/phase/progress patch, the `assigned|recovering → running` transition,
|
|
194
|
+
* and the `spawned` journal event. `spawnUnit`/`spawnReportAgent` differ only
|
|
195
|
+
* in tier, prompt, phase, and failure opts.
|
|
196
|
+
*/
|
|
197
|
+
function spawnAndRecord(ctx, state, unit, slot, opts) {
|
|
158
198
|
const logFile = path.join(ctx.deps.store.runsDir, `${(0, dispatch_1.unitLogName)(unit)}.log`);
|
|
159
199
|
let pid;
|
|
160
200
|
try {
|
|
161
|
-
pid = ctx.deps.spawnDeps.spawn(cmd, prompt, logFile);
|
|
201
|
+
pid = ctx.deps.spawnDeps.spawn(opts.cmd, opts.prompt, logFile);
|
|
162
202
|
}
|
|
163
203
|
catch (err) {
|
|
164
|
-
|
|
165
|
-
// visible in `sched status` (Failed: … spawn-error) instead of aborting
|
|
166
|
-
// the whole tick and silently discarding every other unit's reconcile.
|
|
167
|
-
return failUnit(ctx, state, unit, `spawn-error: ${err.message}`);
|
|
204
|
+
return failUnit(ctx, state, unit, `spawn-error: ${err.message}`, opts.failOpts);
|
|
168
205
|
}
|
|
169
206
|
const now = ctx.deps.now();
|
|
170
207
|
const patch = {
|
|
171
208
|
pid,
|
|
172
209
|
pid_start: ctx.deps.spawnDeps.processStart(pid),
|
|
173
|
-
phase:
|
|
210
|
+
phase: opts.phase,
|
|
174
211
|
last_progress_at: now.toISOString(),
|
|
175
212
|
};
|
|
176
213
|
const next = slot.status === 'assigned' || slot.status === 'recovering'
|
|
@@ -178,16 +215,75 @@ function spawnUnit(ctx, state, unit) {
|
|
|
178
215
|
: patchSlot(state, slot.id, patch, now);
|
|
179
216
|
journal(ctx, 'spawned', unit, {
|
|
180
217
|
pid,
|
|
181
|
-
tier:
|
|
218
|
+
tier: opts.tier,
|
|
182
219
|
slot: slot.id,
|
|
183
|
-
cmd: cmd.join(' '),
|
|
220
|
+
cmd: opts.cmd.join(' '),
|
|
184
221
|
log: logFile,
|
|
222
|
+
...(opts.journalExtra ?? {}),
|
|
185
223
|
});
|
|
186
224
|
ctx.result.spawned.push(unit);
|
|
187
225
|
return next;
|
|
188
226
|
}
|
|
189
|
-
/**
|
|
190
|
-
function
|
|
227
|
+
/** Spawn (or respawn) the agent for `unit` and move its slot to `running`. */
|
|
228
|
+
function spawnUnit(ctx, state, unit) {
|
|
229
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
230
|
+
if (issue === null)
|
|
231
|
+
return state;
|
|
232
|
+
const entry = (0, state_1.findEntry)(state, issue);
|
|
233
|
+
const slot = slotOf(state, unit);
|
|
234
|
+
if (!entry ||
|
|
235
|
+
!slot ||
|
|
236
|
+
slot.status === 'idle' ||
|
|
237
|
+
slot.status === 'complete' ||
|
|
238
|
+
slot.status === 'failed') {
|
|
239
|
+
return state;
|
|
240
|
+
}
|
|
241
|
+
// Report slots (crash recovery, ladder redispatch) respawn as report agents.
|
|
242
|
+
if (slot.phase === 'report') {
|
|
243
|
+
return spawnReportAgent(ctx, state, unit);
|
|
244
|
+
}
|
|
245
|
+
return spawnAndRecord(ctx, state, unit, slot, {
|
|
246
|
+
tier: entry.tier,
|
|
247
|
+
cmd: (0, dispatch_1.buildAgentCommand)(ctx.dispatch.command, entry.tier, issue, ctx.dispatch.tierModels),
|
|
248
|
+
prompt: (0, dispatch_1.buildPrompt)(ctx.dispatch.prompt, issue),
|
|
249
|
+
phase: 'gate',
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Spawn the report agent for a merged unit (#468 AC2 — "a cheap-tier report
|
|
254
|
+
* agent is dispatched", never a full-cycle tail run). The tier climbs the
|
|
255
|
+
* ladder from mechanical across redispatches (`reportTierFor`).
|
|
256
|
+
*/
|
|
257
|
+
function spawnReportAgent(ctx, state, unit) {
|
|
258
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
259
|
+
if (issue === null)
|
|
260
|
+
return state;
|
|
261
|
+
const entry = (0, state_1.findEntry)(state, issue);
|
|
262
|
+
const slot = slotOf(state, unit);
|
|
263
|
+
if (!entry || !slot || entry.pr === null || entry.cleanup === null)
|
|
264
|
+
return state;
|
|
265
|
+
const tier = (0, dispatch_1.reportTierFor)(slot.recoveries);
|
|
266
|
+
if (tier === null)
|
|
267
|
+
return state;
|
|
268
|
+
return spawnAndRecord(ctx, state, unit, slot, {
|
|
269
|
+
tier,
|
|
270
|
+
cmd: (0, dispatch_1.buildAgentCommand)(ctx.dispatch.command, tier, issue, ctx.dispatch.tierModels),
|
|
271
|
+
prompt: (0, dispatch_1.buildReportPrompt)(ctx.dispatch.reportPrompt, issue, entry.pr, entry.cleanup),
|
|
272
|
+
phase: 'report',
|
|
273
|
+
// Merged-aware: the PR is merged — a report spawn failure never blocks
|
|
274
|
+
// dependents (gating already released at `shipped`).
|
|
275
|
+
failOpts: { merged: true },
|
|
276
|
+
journalExtra: { detail: 'report agent' },
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Fail a unit: entry → failed, slot released, transitive dependents blocked
|
|
281
|
+
* (AC4). `merged: true` (report-agent failures on a merged unit, #468) is the
|
|
282
|
+
* merged-aware rail: the PR is merged, so the unit COMPLETES (done, reason
|
|
283
|
+
* recorded) instead of failing — a failed report never fails shipped work and
|
|
284
|
+
* never blocks dependents whose dependency actually merged.
|
|
285
|
+
*/
|
|
286
|
+
function failUnit(ctx, state, unit, reason, opts = {}) {
|
|
191
287
|
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
192
288
|
if (issue === null)
|
|
193
289
|
return state;
|
|
@@ -196,12 +292,18 @@ function failUnit(ctx, state, unit, reason) {
|
|
|
196
292
|
let next = releaseSlotViaFailure(ctx, state, unit);
|
|
197
293
|
const entry = (0, state_1.findEntry)(next, issue);
|
|
198
294
|
if (entry && !types_1.TERMINAL_ISSUE_STATUSES.has(entry.status)) {
|
|
199
|
-
next = (0, state_1.transitionIssue)(next, issue, 'failed', { reason }, now);
|
|
200
|
-
journal(ctx, 'unit-failed', unit, { reason });
|
|
201
295
|
ctx.result.failed.push(unit);
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
296
|
+
if (opts.merged === true && entry.status === 'shipped') {
|
|
297
|
+
next = (0, state_1.transitionIssue)(next, issue, 'done', { reason }, now);
|
|
298
|
+
journal(ctx, 'report-failed', unit, { reason });
|
|
299
|
+
}
|
|
300
|
+
else {
|
|
301
|
+
next = (0, state_1.transitionIssue)(next, issue, 'failed', { reason }, now);
|
|
302
|
+
journal(ctx, 'unit-failed', unit, { reason });
|
|
303
|
+
const blocked = blockTransitiveDependents(ctx, next, issue);
|
|
304
|
+
next = blocked.state;
|
|
305
|
+
ctx.result.blocked.push(...blocked.issues);
|
|
306
|
+
}
|
|
205
307
|
}
|
|
206
308
|
return next;
|
|
207
309
|
}
|
|
@@ -256,7 +358,9 @@ function blockTransitiveDependents(ctx, state, failedIssue) {
|
|
|
256
358
|
* The recovery decision for a unit that must be redispatched one tier
|
|
257
359
|
* stronger (stall or unverified exit, AC4). At the escalation cap or the
|
|
258
360
|
* strongest tier, the unit fails instead — the designed signal that a human,
|
|
259
|
-
* not a stronger model, is next.
|
|
361
|
+
* not a stronger model, is next. Report agents (#468) climb their own
|
|
362
|
+
* mechanical-starting ladder and fail MERGED-AWARE at the cap: the PR is
|
|
363
|
+
* already merged, so dependents stay released.
|
|
260
364
|
*/
|
|
261
365
|
function enterRecovery(ctx, state, unit, causeEvent, cause, evidence = {}) {
|
|
262
366
|
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
@@ -268,18 +372,25 @@ function enterRecovery(ctx, state, unit, causeEvent, cause, evidence = {}) {
|
|
|
268
372
|
if (!entry || !slot)
|
|
269
373
|
return state;
|
|
270
374
|
killUnitAgent(ctx, state, unit);
|
|
271
|
-
const
|
|
375
|
+
const report = slot.phase === 'report';
|
|
376
|
+
const nextTier = report ? (0, dispatch_1.reportTierFor)(slot.recoveries + 1) : (0, dispatch_1.escalateTier)(entry.tier);
|
|
272
377
|
if (slot.recoveries >= types_1.ESCALATION_CAP || nextTier === null) {
|
|
273
378
|
// Cap reached (2 escalations) or already at the strongest tier — the
|
|
274
379
|
// designed signal that a human, not a stronger model, is next.
|
|
275
|
-
const reason =
|
|
276
|
-
|
|
380
|
+
const reason = report
|
|
381
|
+
? 'report-escalation-cap'
|
|
382
|
+
: slot.recoveries >= types_1.ESCALATION_CAP
|
|
383
|
+
? 'escalation-cap'
|
|
384
|
+
: `${cause}-at-strongest-tier`;
|
|
385
|
+
return failUnit(ctx, state, unit, reason, { merged: report });
|
|
277
386
|
}
|
|
278
387
|
let next = (0, state_1.transitionSlot)(state, slot.id, 'recovering', { pid: null, recoveries: slot.recoveries + 1 }, now);
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
388
|
+
if (!report) {
|
|
389
|
+
next = {
|
|
390
|
+
...next,
|
|
391
|
+
entries: next.entries.map((e) => e.issue === issue ? { ...e, tier: nextTier, updated_at: now.toISOString() } : e),
|
|
392
|
+
};
|
|
393
|
+
}
|
|
283
394
|
journal(ctx, causeEvent, unit, {
|
|
284
395
|
detail: cause,
|
|
285
396
|
slot: slot.id,
|
|
@@ -288,41 +399,68 @@ function enterRecovery(ctx, state, unit, causeEvent, cause, evidence = {}) {
|
|
|
288
399
|
});
|
|
289
400
|
journal(ctx, 'redispatched', unit, { tier: nextTier, slot: slot.id });
|
|
290
401
|
ctx.result.redispatched.push(unit);
|
|
291
|
-
// Respawn immediately on the recovering rail — recovering → running.
|
|
402
|
+
// Respawn immediately on the recovering rail — recovering → running. A
|
|
403
|
+
// report-phase slot routes to the report agent with its escalated tier.
|
|
292
404
|
return spawnUnit(ctx, next, unit);
|
|
293
405
|
}
|
|
406
|
+
/**
|
|
407
|
+
* Verified-exit walk to idle: `complete` is reachable only via
|
|
408
|
+
* exited → verifying → complete → idle; the fallback keeps the walk from
|
|
409
|
+
* ever wedging (assigned/recovering have nothing verified yet).
|
|
410
|
+
*/
|
|
411
|
+
function stepVerifiedExitToIdle(status) {
|
|
412
|
+
if (status === 'complete' || status === 'failed')
|
|
413
|
+
return 'idle';
|
|
414
|
+
if (status === 'running')
|
|
415
|
+
return 'exited';
|
|
416
|
+
if (status === 'exited')
|
|
417
|
+
return 'verifying';
|
|
418
|
+
if (status === 'verifying')
|
|
419
|
+
return 'complete';
|
|
420
|
+
return 'failed';
|
|
421
|
+
}
|
|
294
422
|
/** Complete a unit whose ground truth is verified (AC2). */
|
|
295
423
|
function completeUnit(ctx, state, unit, via) {
|
|
296
424
|
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
297
425
|
if (issue === null)
|
|
298
426
|
return state;
|
|
299
427
|
const now = ctx.deps.now();
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
let next = walkSlotToIdle(state, unit, now, (status) => {
|
|
303
|
-
if (status === 'complete' || status === 'failed')
|
|
304
|
-
return 'idle';
|
|
305
|
-
if (status === 'running')
|
|
306
|
-
return 'exited';
|
|
307
|
-
if (status === 'exited')
|
|
308
|
-
return 'verifying';
|
|
309
|
-
if (status === 'verifying')
|
|
310
|
-
return 'complete';
|
|
311
|
-
return 'failed'; // assigned/recovering: nothing verified yet — never wedge
|
|
312
|
-
});
|
|
428
|
+
const next = walkSlotToIdle(state, unit, now, stepVerifiedExitToIdle);
|
|
429
|
+
let withEntry = next;
|
|
313
430
|
const entry = (0, state_1.findEntry)(next, issue);
|
|
314
431
|
if (entry && entry.status === 'dispatched') {
|
|
315
|
-
|
|
316
|
-
|
|
432
|
+
withEntry = (0, state_1.transitionIssue)(next, issue, 'shipped', {}, now);
|
|
433
|
+
withEntry = (0, state_1.transitionIssue)(withEntry, issue, 'done', {}, now);
|
|
317
434
|
}
|
|
318
435
|
else if (entry && entry.status === 'shipped') {
|
|
319
|
-
|
|
436
|
+
// A report agent completing its run (#468): shipped → done.
|
|
437
|
+
withEntry = (0, state_1.transitionIssue)(next, issue, 'done', {}, now);
|
|
320
438
|
}
|
|
321
439
|
journal(ctx, via, unit);
|
|
322
440
|
if (via === 'external-advance')
|
|
323
441
|
ctx.result.externalAdvances.push(unit);
|
|
324
442
|
else
|
|
325
443
|
ctx.result.completed.push(unit);
|
|
444
|
+
return withEntry;
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* Park a unit whose agent exited after parking its PR on auto-merge (#468):
|
|
448
|
+
* the exit is VERIFIED (the ship phase's `awaiting-merge` milestone with
|
|
449
|
+
* `pr=`), entry → parked (pr recorded), slot released — a waiting unit
|
|
450
|
+
* consumes zero slots (AC5) and the watcher owns it from here.
|
|
451
|
+
*/
|
|
452
|
+
function parkUnit(ctx, state, unit, milestone) {
|
|
453
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
454
|
+
if (issue === null)
|
|
455
|
+
return state;
|
|
456
|
+
const now = ctx.deps.now();
|
|
457
|
+
const pr = (0, groundtruth_1.prOfMilestone)(milestone);
|
|
458
|
+
if (pr === null)
|
|
459
|
+
return state; // isParkedMilestone guarantees this
|
|
460
|
+
let next = walkSlotToIdle(state, unit, now, stepVerifiedExitToIdle);
|
|
461
|
+
next = (0, state_1.transitionIssue)(next, issue, 'parked', { pr }, now);
|
|
462
|
+
journal(ctx, 'pr-parked', unit, { pr });
|
|
463
|
+
ctx.result.parked.push(unit);
|
|
326
464
|
return next;
|
|
327
465
|
}
|
|
328
466
|
// --- Per-slot reconciliation ---
|
|
@@ -361,6 +499,14 @@ function applyProgressSignals(ctx, state, slot, truth, unit) {
|
|
|
361
499
|
}
|
|
362
500
|
return { state: next, progressed };
|
|
363
501
|
}
|
|
502
|
+
/**
|
|
503
|
+
* The issue-closed completion signal for a live unit (#468): a report agent's
|
|
504
|
+
* issue is already closed (closed AT MERGE), so for report-phase slots the
|
|
505
|
+
* closed signal is suppressed — only the report milestone can complete them.
|
|
506
|
+
*/
|
|
507
|
+
function effectiveClosedSignal(slot, truth) {
|
|
508
|
+
return slot.phase === 'report' ? false : truth.closed;
|
|
509
|
+
}
|
|
364
510
|
/** Reconcile one running slot against its polled ground truth. */
|
|
365
511
|
function reconcileRunning(ctx, state, slot, truth, unit) {
|
|
366
512
|
const now = ctx.deps.now();
|
|
@@ -384,7 +530,11 @@ function reconcileRunning(ctx, state, slot, truth, unit) {
|
|
|
384
530
|
}
|
|
385
531
|
// Ground truth says the unit is DONE while the agent still holds the slot —
|
|
386
532
|
// externally-advanced state (AC3): reclaim the slot, kill the leftover agent.
|
|
387
|
-
|
|
533
|
+
// A parked milestone is deliberately NOT an advance: a detached run parks
|
|
534
|
+
// and stops — the watcher owns the tail (#468); the exit/stall rails take
|
|
535
|
+
// the agent from here.
|
|
536
|
+
if ((0, groundtruth_1.isVerifiedComplete)(truth.milestone, effectiveClosedSignal(slot, truth)) &&
|
|
537
|
+
!(0, groundtruth_1.isParkedMilestone)(truth.milestone)) {
|
|
388
538
|
journal(ctx, 'external-advance', unit, {
|
|
389
539
|
pid: slot.pid,
|
|
390
540
|
slot: slot.id,
|
|
@@ -406,8 +556,10 @@ function reconcileRunning(ctx, state, slot, truth, unit) {
|
|
|
406
556
|
}
|
|
407
557
|
/**
|
|
408
558
|
* An exited/verifying slot: verify the claimed state against ground truth
|
|
409
|
-
* (AC2). Verified → complete;
|
|
410
|
-
*
|
|
559
|
+
* (AC2). Verified → complete; a ship-phase `awaiting-merge` milestone with
|
|
560
|
+
* `pr=` → parked (the detached-run exit, #468); unverified → the same
|
|
561
|
+
* recovery ladder as a stall. `via` distinguishes the journal's completion
|
|
562
|
+
* event.
|
|
411
563
|
*/
|
|
412
564
|
function completeUnitOrRecover(ctx, state, unit, truth, via) {
|
|
413
565
|
const now = ctx.deps.now();
|
|
@@ -429,7 +581,15 @@ function completeUnitOrRecover(ctx, state, unit, truth, via) {
|
|
|
429
581
|
});
|
|
430
582
|
return next;
|
|
431
583
|
}
|
|
432
|
-
|
|
584
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
585
|
+
const entry = issue !== null ? (0, state_1.findEntry)(next, issue) : undefined;
|
|
586
|
+
// A verified park: the agent exited having parked its PR — the watcher
|
|
587
|
+
// takes the unit (AC2's "never inferred from agent exit" cut both ways:
|
|
588
|
+
// the park IS the milestone, the merge is not).
|
|
589
|
+
if (entry !== undefined && entry.status === 'dispatched' && (0, groundtruth_1.isParkedMilestone)(truth.milestone)) {
|
|
590
|
+
return parkUnit(ctx, next, unit, truth.milestone);
|
|
591
|
+
}
|
|
592
|
+
if ((0, groundtruth_1.isVerifiedComplete)(truth.milestone, effectiveClosedSignal(slot, truth))) {
|
|
433
593
|
return completeUnit(ctx, next, unit, via);
|
|
434
594
|
}
|
|
435
595
|
return enterRecovery(ctx, next, unit, 'verify-incomplete', 'unverified-exit', {
|
|
@@ -506,6 +666,135 @@ function requeueOrphanedDispatches(ctx, state) {
|
|
|
506
666
|
}
|
|
507
667
|
return next;
|
|
508
668
|
}
|
|
669
|
+
/**
|
|
670
|
+
* The PR watcher's decision pass (#468): apply polled PR truths to parked
|
|
671
|
+
* entries. Merge acceptance (AC1) requires state MERGED AND mergedAt non-null
|
|
672
|
+
* AND the issue closed — never an agent exit. Failure states (AC3) fail the
|
|
673
|
+
* unit and block transitive dependents; the engine never merges anything
|
|
674
|
+
* itself. `shipped` (not `parked`) is what unblocks dependents (AC4).
|
|
675
|
+
*
|
|
676
|
+
* Entries are re-read fresh each iteration: a mid-loop failure blocks OTHER
|
|
677
|
+
* parked entries (transitive dependents), and acting on a stale snapshot
|
|
678
|
+
* would drive an already-blocked entry through `parked → shipped`.
|
|
679
|
+
*/
|
|
680
|
+
function reconcileParked(ctx, state, prPoll) {
|
|
681
|
+
if (!prPoll.ran)
|
|
682
|
+
return state;
|
|
683
|
+
let next = { ...state, last_pr_poll_at: ctx.deps.now().toISOString() };
|
|
684
|
+
const parkedIssues = state.entries
|
|
685
|
+
.filter((e) => e.status === 'parked' && e.pr !== null)
|
|
686
|
+
.map((e) => e.issue);
|
|
687
|
+
for (const issue of parkedIssues) {
|
|
688
|
+
const entry = (0, state_1.findEntry)(next, issue);
|
|
689
|
+
if (!entry || entry.status !== 'parked' || entry.pr === null)
|
|
690
|
+
continue; // blocked mid-loop
|
|
691
|
+
const unit = `issue:${issue}`;
|
|
692
|
+
const truth = prPoll.truths.get(issue);
|
|
693
|
+
if (truth === undefined) {
|
|
694
|
+
if (!prPoll.truths.has(issue)) {
|
|
695
|
+
continue; // parked AFTER the poll ran (this tick) — next cadence picks it up
|
|
696
|
+
}
|
|
697
|
+
journal(ctx, 'ground-truth-unreachable', unit, {
|
|
698
|
+
detail: 'pr watch paused until truth returns',
|
|
699
|
+
});
|
|
700
|
+
continue;
|
|
701
|
+
}
|
|
702
|
+
const failWatch = (reason) => {
|
|
703
|
+
journal(ctx, 'pr-watch-failed', unit, { reason, pr: entry.pr });
|
|
704
|
+
return failUnit(ctx, next, unit, reason);
|
|
705
|
+
};
|
|
706
|
+
if (truth.blocked) {
|
|
707
|
+
next = failWatch('auto-merge-blocked');
|
|
708
|
+
continue;
|
|
709
|
+
}
|
|
710
|
+
if (truth.mergeable === 'CONFLICTING') {
|
|
711
|
+
next = failWatch('pr-conflicting');
|
|
712
|
+
continue;
|
|
713
|
+
}
|
|
714
|
+
if (truth.state === 'CLOSED' && truth.mergedAt === null) {
|
|
715
|
+
next = failWatch('pr-closed-unmerged');
|
|
716
|
+
continue;
|
|
717
|
+
}
|
|
718
|
+
if (truth.state === 'MERGED' && truth.mergedAt !== null) {
|
|
719
|
+
// AC1: the issue must ALSO be closed (a merged PR auto-closes it) —
|
|
720
|
+
// until GitHub propagates, the unit stays parked and keeps watching.
|
|
721
|
+
if (prPoll.closed.get(issue) !== true) {
|
|
722
|
+
journal(ctx, 'pr-watch-waiting', unit, {
|
|
723
|
+
pr: entry.pr,
|
|
724
|
+
mergedAt: truth.mergedAt,
|
|
725
|
+
detail: 'merge seen but issue not closed — keep watching',
|
|
726
|
+
});
|
|
727
|
+
continue;
|
|
728
|
+
}
|
|
729
|
+
next = (0, state_1.transitionIssue)(next, issue, 'shipped', { reason: null }, ctx.deps.now());
|
|
730
|
+
journal(ctx, 'merge-accepted', unit, { pr: entry.pr, mergedAt: truth.mergedAt });
|
|
731
|
+
ctx.result.mergeAccepted.push(unit);
|
|
732
|
+
}
|
|
733
|
+
// OPEN (or MERGED without a date / mergeable UNKNOWN) — keep watching.
|
|
734
|
+
}
|
|
735
|
+
return next;
|
|
736
|
+
}
|
|
737
|
+
/**
|
|
738
|
+
* Dispatch report agents for merged units whose teardown is recorded (#468
|
|
739
|
+
* AC2): shipped + pr + cleanup + no live slot + free capacity → a slot is
|
|
740
|
+
* assigned (phase `report`) and a mechanical-tier agent spawned with the
|
|
741
|
+
* report prompt. Reports run BEFORE queue refill — a cheap report never
|
|
742
|
+
* queues behind long full-cycle runs. A unit waiting for capacity consumes
|
|
743
|
+
* zero slots (AC5) and is surfaced via `result.reportWaiting`.
|
|
744
|
+
*/
|
|
745
|
+
function dispatchReportAgents(ctx, state, config) {
|
|
746
|
+
let next = state;
|
|
747
|
+
for (const entry of state.entries) {
|
|
748
|
+
if (entry.status !== 'shipped' || entry.pr === null || entry.cleanup === null)
|
|
749
|
+
continue;
|
|
750
|
+
const unit = `issue:${entry.issue}`;
|
|
751
|
+
if (slotOf(next, unit) !== undefined)
|
|
752
|
+
continue; // a slot already holds the unit
|
|
753
|
+
if ((0, scheduler_1.freeCapacity)(next, config) === 0) {
|
|
754
|
+
// Full — the report waits (zero slots consumed). Count what is waiting
|
|
755
|
+
// so the wait is visible, then stop scanning.
|
|
756
|
+
ctx.result.reportWaiting = state.entries.filter((e) => e.status === 'shipped' &&
|
|
757
|
+
e.pr !== null &&
|
|
758
|
+
e.cleanup !== null &&
|
|
759
|
+
slotOf(next, `issue:${e.issue}`) === undefined).length;
|
|
760
|
+
break;
|
|
761
|
+
}
|
|
762
|
+
const now = ctx.deps.now();
|
|
763
|
+
const assigned = (0, scheduler_1.assignToIdleSlot)(next, unit, 'report', now);
|
|
764
|
+
next = assigned.state;
|
|
765
|
+
journal(ctx, 'assigned', unit, { slot: assigned.slotId, detail: 'report agent' });
|
|
766
|
+
journal(ctx, 'report-dispatched', unit, {
|
|
767
|
+
slot: assigned.slotId,
|
|
768
|
+
pr: entry.pr,
|
|
769
|
+
cleanup: entry.cleanup,
|
|
770
|
+
});
|
|
771
|
+
ctx.result.reportDispatched.push(unit);
|
|
772
|
+
next = spawnReportAgent(ctx, next, unit);
|
|
773
|
+
}
|
|
774
|
+
return next;
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* Run the teardown script for one merged unit OUTSIDE the state lock (#468
|
|
778
|
+
* AC2). Returns null when teardown must be retried next tick (setup info
|
|
779
|
+
* unreachable — a transient outage, never a failure).
|
|
780
|
+
*/
|
|
781
|
+
function runTeardownFor(deps, issue) {
|
|
782
|
+
const info = deps.groundTruth.setupInfo(issue);
|
|
783
|
+
const unit = `issue:${issue}`;
|
|
784
|
+
if (info === undefined) {
|
|
785
|
+
deps.journal.append((0, journal_1.unitEvent)('ground-truth-unreachable', unit, {
|
|
786
|
+
detail: 'teardown paused until truth returns',
|
|
787
|
+
}), deps.now());
|
|
788
|
+
return null;
|
|
789
|
+
}
|
|
790
|
+
if (info === null) {
|
|
791
|
+
return {
|
|
792
|
+
cleanup: 'failed-missing-setup-info',
|
|
793
|
+
detail: 'no setup milestone with worktree= found on the issue',
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
return (0, teardown_1.runTeardown)(deps.teardownExec, deps.repoDir, info);
|
|
797
|
+
}
|
|
509
798
|
/** Phase 3: refill — every freed slot is filled in THIS tick (AC5). */
|
|
510
799
|
function dispatchAssignments(ctx, state, config) {
|
|
511
800
|
const now = ctx.deps.now();
|
|
@@ -527,20 +816,77 @@ function dispatchAssignments(ctx, state, config) {
|
|
|
527
816
|
}
|
|
528
817
|
return next;
|
|
529
818
|
}
|
|
819
|
+
/** Entries freshly shipped whose teardown has not run yet (cleanup still null). */
|
|
820
|
+
function teardownPendingIssues(state) {
|
|
821
|
+
return state.entries
|
|
822
|
+
.filter((e) => e.status === 'shipped' && e.pr !== null && e.cleanup === null)
|
|
823
|
+
.map((e) => e.issue);
|
|
824
|
+
}
|
|
825
|
+
/** Record teardown results on their entries + journal (the lock pass after the subprocesses). */
|
|
826
|
+
function recordTeardowns(ctx, state, results) {
|
|
827
|
+
const now = ctx.deps.now();
|
|
828
|
+
let next = state;
|
|
829
|
+
for (const [issue, result] of results) {
|
|
830
|
+
const unit = `issue:${issue}`;
|
|
831
|
+
next = {
|
|
832
|
+
...next,
|
|
833
|
+
entries: next.entries.map((e) => e.issue === issue ? { ...e, cleanup: result.cleanup, updated_at: now.toISOString() } : e),
|
|
834
|
+
};
|
|
835
|
+
journal(ctx, result.cleanup === 'done' ? 'teardown-done' : 'teardown-failed', unit, {
|
|
836
|
+
cleanup: result.cleanup,
|
|
837
|
+
detail: result.detail,
|
|
838
|
+
});
|
|
839
|
+
if (result.cleanup === 'done')
|
|
840
|
+
ctx.result.teardownDone.push(unit);
|
|
841
|
+
else
|
|
842
|
+
ctx.result.teardownFailed.push(unit);
|
|
843
|
+
}
|
|
844
|
+
return next;
|
|
845
|
+
}
|
|
530
846
|
/**
|
|
531
|
-
* One full reconcile+refill cycle. Ground truth is polled WITHOUT the lock
|
|
532
|
-
* every state mutation happens under
|
|
533
|
-
* the same pass, so a freed slot
|
|
847
|
+
* One full reconcile+refill cycle. Ground truth is polled WITHOUT the lock
|
|
848
|
+
* (live units AND parked PRs); every state mutation happens under
|
|
849
|
+
* `store.withLock`; refills are computed in the same pass, so a freed slot
|
|
850
|
+
* is reused within this tick (AC5). Teardown subprocesses run between two
|
|
851
|
+
* short lock passes — a slow `npx`/`git` call never holds the lock.
|
|
534
852
|
*/
|
|
535
853
|
function tick(deps, config) {
|
|
536
854
|
const dispatch = (0, dispatch_1.resolveDispatch)(config);
|
|
537
|
-
const
|
|
538
|
-
|
|
855
|
+
const state0 = deps.store.load();
|
|
856
|
+
const polled = pollUnits(deps, state0);
|
|
857
|
+
const prPoll = pollParkedPrs(deps, state0, dispatch);
|
|
858
|
+
const pass1 = deps.store.withLock((state) => {
|
|
539
859
|
const ctx = { deps, dispatch, result: emptyResult() };
|
|
540
860
|
let next = reconcileSlots(ctx, state, polled);
|
|
861
|
+
next = reconcileParked(ctx, next, prPoll);
|
|
541
862
|
next = requeueOrphanedDispatches(ctx, next);
|
|
863
|
+
next = dispatchReportAgents(ctx, next, config);
|
|
542
864
|
next = dispatchAssignments(ctx, next, config);
|
|
543
|
-
return {
|
|
865
|
+
return {
|
|
866
|
+
state: next,
|
|
867
|
+
result: { tick: ctx.result, teardownPending: teardownPendingIssues(next) },
|
|
868
|
+
};
|
|
869
|
+
});
|
|
870
|
+
if (pass1.teardownPending.length === 0) {
|
|
871
|
+
return pass1.tick;
|
|
872
|
+
}
|
|
873
|
+
// Teardown subprocesses (pool return / worktree remove) run OUTSIDE the
|
|
874
|
+
// lock; results land in a second short lock pass together with the report
|
|
875
|
+
// dispatch (AC2: teardown, THEN the cheap-tier report agent).
|
|
876
|
+
const results = new Map();
|
|
877
|
+
for (const issue of pass1.teardownPending) {
|
|
878
|
+
const result = runTeardownFor(deps, issue);
|
|
879
|
+
if (result !== null)
|
|
880
|
+
results.set(issue, result);
|
|
881
|
+
}
|
|
882
|
+
if (results.size === 0) {
|
|
883
|
+
return pass1.tick; // all deferred (setup info unreachable) — retried next tick
|
|
884
|
+
}
|
|
885
|
+
return deps.store.withLock((state) => {
|
|
886
|
+
const ctx = { deps, dispatch, result: pass1.tick };
|
|
887
|
+
const next = recordTeardowns(ctx, state, results);
|
|
888
|
+
const withReport = dispatchReportAgents(ctx, next, config);
|
|
889
|
+
return { state: withReport, result: ctx.result };
|
|
544
890
|
});
|
|
545
891
|
}
|
|
546
892
|
/**
|