@ai-dossier/sched 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -17
- package/dist/dispatch.d.ts +128 -0
- package/dist/dispatch.d.ts.map +1 -0
- package/dist/dispatch.js +300 -0
- package/dist/dispatch.js.map +1 -0
- package/dist/engine.d.ts +111 -0
- package/dist/engine.d.ts.map +1 -0
- package/dist/engine.js +936 -0
- package/dist/engine.js.map +1 -0
- package/dist/enqueue.d.ts +38 -0
- package/dist/enqueue.d.ts.map +1 -0
- package/dist/enqueue.js +222 -0
- package/dist/enqueue.js.map +1 -0
- package/dist/groundtruth.d.ts +135 -0
- package/dist/groundtruth.d.ts.map +1 -0
- package/dist/groundtruth.js +259 -0
- package/dist/groundtruth.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +101 -0
- package/dist/index.js.map +1 -0
- package/dist/journal.d.ts +32 -0
- package/dist/journal.d.ts.map +1 -0
- package/dist/journal.js +113 -0
- package/dist/journal.js.map +1 -0
- package/dist/persist.d.ts +54 -0
- package/dist/persist.d.ts.map +1 -0
- package/dist/persist.js +348 -0
- package/dist/persist.js.map +1 -0
- package/dist/project.d.ts +41 -0
- package/dist/project.d.ts.map +1 -0
- package/dist/project.js +124 -0
- package/dist/project.js.map +1 -0
- package/dist/readiness.d.ts +43 -0
- package/dist/readiness.d.ts.map +1 -0
- package/dist/readiness.js +102 -0
- package/dist/readiness.js.map +1 -0
- package/dist/scheduler.d.ts +78 -0
- package/dist/scheduler.d.ts.map +1 -0
- package/dist/scheduler.js +191 -0
- package/dist/scheduler.js.map +1 -0
- package/dist/state.d.ts +40 -0
- package/dist/state.d.ts.map +1 -0
- package/dist/state.js +391 -0
- package/dist/state.js.map +1 -0
- package/dist/status.d.ts +44 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +82 -0
- package/dist/status.js.map +1 -0
- package/dist/teardown.d.ts +57 -0
- package/dist/teardown.d.ts.map +1 -0
- package/dist/teardown.js +242 -0
- package/dist/teardown.js.map +1 -0
- package/dist/types.d.ts +253 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +91 -0
- package/dist/types.js.map +1 -0
- package/package.json +1 -1
package/dist/engine.js
ADDED
|
@@ -0,0 +1,936 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The scheduler engine (#464): dispatch, completion verification, and the
|
|
4
|
+
* stall/escalation ladder (RFC-0001 §C.1) — the deterministic organs that
|
|
5
|
+
* replace fleet-cycle's LLM supervision. Since #468 it also owns the
|
|
6
|
+
* detached-ship tail: the PR watcher, script-based teardown, and the
|
|
7
|
+
* cheap-tier report dispatch.
|
|
8
|
+
*
|
|
9
|
+
* One `tick()` is a full reconcile+refill cycle:
|
|
10
|
+
*
|
|
11
|
+
* 1. **Poll** ground truth for every live unit and every parked PR OUTSIDE
|
|
12
|
+
* the state lock (gh/git subprocesses are slow; the lock must never wait
|
|
13
|
+
* on a network call). Parked PRs poll on their own cadence
|
|
14
|
+
* (`pr_poll_interval_ms`, persisted `last_pr_poll_at`) — every 2–3 min.
|
|
15
|
+
* 2. **Apply** under `SchedStore.withLock`:
|
|
16
|
+
* - `assigned` slots (crash between assign and spawn) are spawned/re-attached
|
|
17
|
+
* - `running` slots: dead pid → exit rail; ground truth says complete →
|
|
18
|
+
* external advance (kill the leftover agent, complete); new milestone or
|
|
19
|
+
* pushed commit → progress; no progress for `stall_timeout_ms` →
|
|
20
|
+
* redispatch one tier stronger (cap 2, then failed)
|
|
21
|
+
* - `exited`/`verifying` slots: the agent exited — completion is verified
|
|
22
|
+
* against ground truth, never assumed (AC2); an exit whose milestone is
|
|
23
|
+
* the ship phase's `awaiting-merge` (with `pr=`) is a VERIFIED park:
|
|
24
|
+
* entry → parked, slot released (a parked unit holds no slot — AC5)
|
|
25
|
+
* - `parked` entries: the watcher applies the PR truth — merge accepted
|
|
26
|
+
* only when state MERGED AND mergedAt non-null AND the issue is closed
|
|
27
|
+
* (AC1) → shipped (gating on MERGE, never the park — AC4);
|
|
28
|
+
* CONFLICTING / closed-unmerged / auto-merge-blocked → failed + transitive
|
|
29
|
+
* dependents blocked (AC3)
|
|
30
|
+
* - failures block their TRANSITIVE dependents (AC4)
|
|
31
|
+
* - report agents are dispatched for merged units whose teardown is
|
|
32
|
+
* already recorded (before queue refill — cheap reports don't queue
|
|
33
|
+
* behind long runs)
|
|
34
|
+
* - **Refill** in the SAME lock pass: `computeAssignments` fills every
|
|
35
|
+
* freed slot — a runnable unit never waits while a slot is idle (AC5)
|
|
36
|
+
* 3. **Teardown** (outside the lock — pool/git subprocesses are slow): for
|
|
37
|
+
* every freshly-merged unit, recover the setup milestone's worktree info
|
|
38
|
+
* and run pool return / worktree remove, VERIFIED before claimed
|
|
39
|
+
* (`cleanup=failed-<step>` on mismatch, AC2). Results land in a second
|
|
40
|
+
* short lock pass together with the report dispatch.
|
|
41
|
+
*
|
|
42
|
+
* Everything that touches the world (processes, GitHub, git) is injected;
|
|
43
|
+
* the state machine is pure. Only `issue:<n>` units are dispatched — batch
|
|
44
|
+
* member sequencing is a follow-up (#464 non-goal).
|
|
45
|
+
*/
|
|
46
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
47
|
+
if (k2 === undefined) k2 = k;
|
|
48
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
49
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
50
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
51
|
+
}
|
|
52
|
+
Object.defineProperty(o, k2, desc);
|
|
53
|
+
}) : (function(o, m, k, k2) {
|
|
54
|
+
if (k2 === undefined) k2 = k;
|
|
55
|
+
o[k2] = m[k];
|
|
56
|
+
}));
|
|
57
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
58
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
59
|
+
}) : function(o, v) {
|
|
60
|
+
o["default"] = v;
|
|
61
|
+
});
|
|
62
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
63
|
+
var ownKeys = function(o) {
|
|
64
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
65
|
+
var ar = [];
|
|
66
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
67
|
+
return ar;
|
|
68
|
+
};
|
|
69
|
+
return ownKeys(o);
|
|
70
|
+
};
|
|
71
|
+
return function (mod) {
|
|
72
|
+
if (mod && mod.__esModule) return mod;
|
|
73
|
+
var result = {};
|
|
74
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
75
|
+
__setModuleDefault(result, mod);
|
|
76
|
+
return result;
|
|
77
|
+
};
|
|
78
|
+
})();
|
|
79
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
80
|
+
exports.tick = tick;
|
|
81
|
+
exports.runLoop = runLoop;
|
|
82
|
+
const path = __importStar(require("node:path"));
|
|
83
|
+
const dispatch_1 = require("./dispatch");
|
|
84
|
+
const groundtruth_1 = require("./groundtruth");
|
|
85
|
+
const journal_1 = require("./journal");
|
|
86
|
+
const scheduler_1 = require("./scheduler");
|
|
87
|
+
const state_1 = require("./state");
|
|
88
|
+
const teardown_1 = require("./teardown");
|
|
89
|
+
const types_1 = require("./types");
|
|
90
|
+
function emptyResult() {
|
|
91
|
+
return {
|
|
92
|
+
spawned: [],
|
|
93
|
+
externalAdvances: [],
|
|
94
|
+
completed: [],
|
|
95
|
+
parked: [],
|
|
96
|
+
mergeAccepted: [],
|
|
97
|
+
reportDispatched: [],
|
|
98
|
+
reportWaiting: 0,
|
|
99
|
+
teardownDone: [],
|
|
100
|
+
teardownFailed: [],
|
|
101
|
+
redispatched: [],
|
|
102
|
+
failed: [],
|
|
103
|
+
blocked: [],
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
function journal(ctx, event, unit, extra = {}) {
|
|
107
|
+
ctx.deps.journal.append((0, journal_1.unitEvent)(event, unit, extra), ctx.deps.now());
|
|
108
|
+
}
|
|
109
|
+
function slotOf(state, unit) {
|
|
110
|
+
return state.slots.find((s) => s.unit === unit);
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Metadata patch on a slot WITHOUT a status transition (pid/phase/branch/
|
|
114
|
+
* last_head/last_progress are data, not machine states — RFC-0001 §D.3 keeps
|
|
115
|
+
* them alongside the status, and the transition tables stay pure).
|
|
116
|
+
*/
|
|
117
|
+
function patchSlot(state, slotId, patch, now) {
|
|
118
|
+
return {
|
|
119
|
+
...state,
|
|
120
|
+
slots: state.slots.map((s) => s.id === slotId ? { ...s, ...patch, updated_at: now.toISOString() } : s),
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
/** Walk a slot to `idle` one declared edge per iteration; `step` picks the next status. */
|
|
124
|
+
function walkSlotToIdle(state, unit, now, step) {
|
|
125
|
+
let next = state;
|
|
126
|
+
let slot = slotOf(next, unit);
|
|
127
|
+
while (slot && slot.status !== 'idle') {
|
|
128
|
+
next = (0, state_1.transitionSlot)(next, slot.id, step(slot.status), {}, now);
|
|
129
|
+
slot = slotOf(next, unit);
|
|
130
|
+
}
|
|
131
|
+
return next;
|
|
132
|
+
}
|
|
133
|
+
/** Kill the agent holding `unit`'s slot, if it is alive. */
|
|
134
|
+
function killUnitAgent(ctx, state, unit) {
|
|
135
|
+
const slot = slotOf(state, unit);
|
|
136
|
+
if (slot &&
|
|
137
|
+
slot.pid !== null &&
|
|
138
|
+
ctx.deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
|
|
139
|
+
ctx.deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
// --- Poll (outside the lock) ---
|
|
143
|
+
function pollUnits(deps, state) {
|
|
144
|
+
const out = new Map();
|
|
145
|
+
for (const slot of state.slots) {
|
|
146
|
+
if (slot.unit === null)
|
|
147
|
+
continue;
|
|
148
|
+
if (slot.status !== 'running' && slot.status !== 'verifying' && slot.status !== 'exited') {
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
const issue = (0, journal_1.issueOfUnit)(slot.unit);
|
|
152
|
+
if (issue === null || out.has(slot.unit))
|
|
153
|
+
continue;
|
|
154
|
+
const milestone = deps.groundTruth.latestMilestone(issue);
|
|
155
|
+
out.set(slot.unit, {
|
|
156
|
+
reachable: milestone !== undefined,
|
|
157
|
+
milestone: milestone ?? null,
|
|
158
|
+
closed: deps.groundTruth.issueClosed(issue),
|
|
159
|
+
head: slot.branch !== null ? deps.groundTruth.branchHead(slot.branch) : null,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
return out;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Poll parked PRs on their own cadence (#468 AC1 — every 2–3 min, persisted
|
|
166
|
+
* `last_pr_poll_at` so a restart honors it). Runs only when parked entries
|
|
167
|
+
* exist AND the interval elapsed; every subprocess stays outside the lock.
|
|
168
|
+
* The issue-closed signal rides along — it is the second half of the
|
|
169
|
+
* merge-acceptance gate and must not be re-queried under the lock.
|
|
170
|
+
*/
|
|
171
|
+
function pollParkedPrs(deps, state, dispatch) {
|
|
172
|
+
const parked = state.entries.filter((e) => e.status === 'parked' && e.pr !== null);
|
|
173
|
+
if (parked.length === 0)
|
|
174
|
+
return { ran: false, truths: new Map(), closed: new Map() };
|
|
175
|
+
const now = deps.now().getTime();
|
|
176
|
+
const last = state.last_pr_poll_at !== null ? Date.parse(state.last_pr_poll_at) : 0;
|
|
177
|
+
if (Number.isFinite(last) && now - last < dispatch.prPollIntervalMs) {
|
|
178
|
+
return { ran: false, truths: new Map(), closed: new Map() };
|
|
179
|
+
}
|
|
180
|
+
const truths = new Map();
|
|
181
|
+
const closed = new Map();
|
|
182
|
+
for (const entry of parked) {
|
|
183
|
+
truths.set(entry.issue, deps.groundTruth.prState(entry.pr));
|
|
184
|
+
closed.set(entry.issue, deps.groundTruth.issueClosed(entry.issue));
|
|
185
|
+
}
|
|
186
|
+
return { ran: true, truths, closed };
|
|
187
|
+
}
|
|
188
|
+
// --- Spawn / fail / complete / park ---
|
|
189
|
+
/**
|
|
190
|
+
* The shared spawn-and-record tail of every agent dispatch (#464 full-cycle,
|
|
191
|
+
* #468 report): log file, try-spawn (a throw fails the unit through the
|
|
192
|
+
* declared failure rail — visible in `sched status`, never a tick abort),
|
|
193
|
+
* pid/phase/progress patch, the `assigned|recovering → running` transition,
|
|
194
|
+
* and the `spawned` journal event. `spawnUnit`/`spawnReportAgent` differ only
|
|
195
|
+
* in tier, prompt, phase, and failure opts.
|
|
196
|
+
*/
|
|
197
|
+
function spawnAndRecord(ctx, state, unit, slot, opts) {
|
|
198
|
+
const logFile = path.join(ctx.deps.store.runsDir, `${(0, dispatch_1.unitLogName)(unit)}.log`);
|
|
199
|
+
let pid;
|
|
200
|
+
try {
|
|
201
|
+
pid = ctx.deps.spawnDeps.spawn(opts.cmd, opts.prompt, logFile);
|
|
202
|
+
}
|
|
203
|
+
catch (err) {
|
|
204
|
+
return failUnit(ctx, state, unit, `spawn-error: ${err.message}`, opts.failOpts);
|
|
205
|
+
}
|
|
206
|
+
const now = ctx.deps.now();
|
|
207
|
+
const patch = {
|
|
208
|
+
pid,
|
|
209
|
+
pid_start: ctx.deps.spawnDeps.processStart(pid),
|
|
210
|
+
phase: opts.phase,
|
|
211
|
+
last_progress_at: now.toISOString(),
|
|
212
|
+
};
|
|
213
|
+
const next = slot.status === 'assigned' || slot.status === 'recovering'
|
|
214
|
+
? (0, state_1.transitionSlot)(state, slot.id, 'running', patch, now)
|
|
215
|
+
: patchSlot(state, slot.id, patch, now);
|
|
216
|
+
journal(ctx, 'spawned', unit, {
|
|
217
|
+
pid,
|
|
218
|
+
tier: opts.tier,
|
|
219
|
+
slot: slot.id,
|
|
220
|
+
cmd: opts.cmd.join(' '),
|
|
221
|
+
log: logFile,
|
|
222
|
+
...(opts.journalExtra ?? {}),
|
|
223
|
+
});
|
|
224
|
+
ctx.result.spawned.push(unit);
|
|
225
|
+
return next;
|
|
226
|
+
}
|
|
227
|
+
/** Spawn (or respawn) the agent for `unit` and move its slot to `running`. */
|
|
228
|
+
function spawnUnit(ctx, state, unit) {
|
|
229
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
230
|
+
if (issue === null)
|
|
231
|
+
return state;
|
|
232
|
+
const entry = (0, state_1.findEntry)(state, issue);
|
|
233
|
+
const slot = slotOf(state, unit);
|
|
234
|
+
if (!entry ||
|
|
235
|
+
!slot ||
|
|
236
|
+
slot.status === 'idle' ||
|
|
237
|
+
slot.status === 'complete' ||
|
|
238
|
+
slot.status === 'failed') {
|
|
239
|
+
return state;
|
|
240
|
+
}
|
|
241
|
+
// Report slots (crash recovery, ladder redispatch) respawn as report agents.
|
|
242
|
+
if (slot.phase === 'report') {
|
|
243
|
+
return spawnReportAgent(ctx, state, unit);
|
|
244
|
+
}
|
|
245
|
+
return spawnAndRecord(ctx, state, unit, slot, {
|
|
246
|
+
tier: entry.tier,
|
|
247
|
+
cmd: (0, dispatch_1.buildAgentCommand)(ctx.dispatch.command, entry.tier, issue, ctx.dispatch.tierModels),
|
|
248
|
+
prompt: (0, dispatch_1.buildPrompt)(ctx.dispatch.prompt, issue),
|
|
249
|
+
phase: 'gate',
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Spawn the report agent for a merged unit (#468 AC2 — "a cheap-tier report
|
|
254
|
+
* agent is dispatched", never a full-cycle tail run). The tier climbs the
|
|
255
|
+
* ladder from mechanical across redispatches (`reportTierFor`).
|
|
256
|
+
*/
|
|
257
|
+
function spawnReportAgent(ctx, state, unit) {
|
|
258
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
259
|
+
if (issue === null)
|
|
260
|
+
return state;
|
|
261
|
+
const entry = (0, state_1.findEntry)(state, issue);
|
|
262
|
+
const slot = slotOf(state, unit);
|
|
263
|
+
if (!entry || !slot || entry.pr === null || entry.cleanup === null)
|
|
264
|
+
return state;
|
|
265
|
+
const tier = (0, dispatch_1.reportTierFor)(slot.recoveries);
|
|
266
|
+
if (tier === null)
|
|
267
|
+
return state;
|
|
268
|
+
return spawnAndRecord(ctx, state, unit, slot, {
|
|
269
|
+
tier,
|
|
270
|
+
cmd: (0, dispatch_1.buildAgentCommand)(ctx.dispatch.command, tier, issue, ctx.dispatch.tierModels),
|
|
271
|
+
prompt: (0, dispatch_1.buildReportPrompt)(ctx.dispatch.reportPrompt, issue, entry.pr, entry.cleanup),
|
|
272
|
+
phase: 'report',
|
|
273
|
+
// Merged-aware: the PR is merged — a report spawn failure never blocks
|
|
274
|
+
// dependents (gating already released at `shipped`).
|
|
275
|
+
failOpts: { merged: true },
|
|
276
|
+
journalExtra: { detail: 'report agent' },
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Fail a unit: entry → failed, slot released, transitive dependents blocked
|
|
281
|
+
* (AC4). `merged: true` (report-agent failures on a merged unit, #468) is the
|
|
282
|
+
* merged-aware rail: the PR is merged, so the unit COMPLETES (done, reason
|
|
283
|
+
* recorded) instead of failing — a failed report never fails shipped work and
|
|
284
|
+
* never blocks dependents whose dependency actually merged.
|
|
285
|
+
*/
|
|
286
|
+
function failUnit(ctx, state, unit, reason, opts = {}) {
|
|
287
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
288
|
+
if (issue === null)
|
|
289
|
+
return state;
|
|
290
|
+
const now = ctx.deps.now();
|
|
291
|
+
killUnitAgent(ctx, state, unit);
|
|
292
|
+
let next = releaseSlotViaFailure(ctx, state, unit);
|
|
293
|
+
const entry = (0, state_1.findEntry)(next, issue);
|
|
294
|
+
if (entry && !types_1.TERMINAL_ISSUE_STATUSES.has(entry.status)) {
|
|
295
|
+
ctx.result.failed.push(unit);
|
|
296
|
+
if (opts.merged === true && entry.status === 'shipped') {
|
|
297
|
+
next = (0, state_1.transitionIssue)(next, issue, 'done', { reason }, now);
|
|
298
|
+
journal(ctx, 'report-failed', unit, { reason });
|
|
299
|
+
}
|
|
300
|
+
else {
|
|
301
|
+
next = (0, state_1.transitionIssue)(next, issue, 'failed', { reason }, now);
|
|
302
|
+
journal(ctx, 'unit-failed', unit, { reason });
|
|
303
|
+
const blocked = blockTransitiveDependents(ctx, next, issue);
|
|
304
|
+
next = blocked.state;
|
|
305
|
+
ctx.result.blocked.push(...blocked.issues);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return next;
|
|
309
|
+
}
|
|
310
|
+
/** Release a unit's slot to idle through the failure rail (failed → idle). */
|
|
311
|
+
function releaseSlotViaFailure(ctx, state, unit) {
|
|
312
|
+
return walkSlotToIdle(state, unit, ctx.deps.now(), (status) => status === 'complete' || status === 'failed' ? 'idle' : 'failed');
|
|
313
|
+
}
|
|
314
|
+
/** Block every entry that transitively depends on `failedIssue` (AC4). */
|
|
315
|
+
function blockTransitiveDependents(ctx, state, failedIssue) {
|
|
316
|
+
const now = ctx.deps.now();
|
|
317
|
+
const reason = `dep-failed:${failedIssue}`;
|
|
318
|
+
// BFS over reversed dependency edges: who depends on the failed issue,
|
|
319
|
+
// directly or through another to-be-blocked entry?
|
|
320
|
+
const queue = [failedIssue];
|
|
321
|
+
const seen = new Set([failedIssue]);
|
|
322
|
+
const dependents = [];
|
|
323
|
+
while (queue.length > 0) {
|
|
324
|
+
const current = queue.shift();
|
|
325
|
+
for (const entry of state.entries) {
|
|
326
|
+
if (seen.has(entry.issue) || !entry.deps.includes(current))
|
|
327
|
+
continue;
|
|
328
|
+
seen.add(entry.issue);
|
|
329
|
+
queue.push(entry.issue);
|
|
330
|
+
dependents.push(entry.issue);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const blockedIssues = [];
|
|
334
|
+
let next = state;
|
|
335
|
+
for (const issue of dependents) {
|
|
336
|
+
const entry = (0, state_1.findEntry)(next, issue);
|
|
337
|
+
if (!entry)
|
|
338
|
+
continue;
|
|
339
|
+
// Terminal/satisfied entries keep their outcome; already-blocked entries
|
|
340
|
+
// keep their (possibly more specific) reason.
|
|
341
|
+
if (types_1.TERMINAL_ISSUE_STATUSES.has(entry.status) ||
|
|
342
|
+
entry.status === 'blocked' ||
|
|
343
|
+
entry.status === 'shipped' ||
|
|
344
|
+
entry.status === 'shipped-in-batch') {
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
// A dependent mid-run is working toward a doomed merge — release its slot.
|
|
348
|
+
const unit = `issue:${issue}`;
|
|
349
|
+
killUnitAgent(ctx, next, unit);
|
|
350
|
+
next = releaseSlotViaFailure(ctx, next, unit);
|
|
351
|
+
next = (0, state_1.transitionIssue)(next, issue, 'blocked', { reason }, now);
|
|
352
|
+
journal(ctx, 'dependents-blocked', unit, { reason });
|
|
353
|
+
blockedIssues.push(issue);
|
|
354
|
+
}
|
|
355
|
+
return { state: next, issues: blockedIssues };
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* The recovery decision for a unit that must be redispatched one tier
|
|
359
|
+
* stronger (stall or unverified exit, AC4). At the escalation cap or the
|
|
360
|
+
* strongest tier, the unit fails instead — the designed signal that a human,
|
|
361
|
+
* not a stronger model, is next. Report agents (#468) climb their own
|
|
362
|
+
* mechanical-starting ladder and fail MERGED-AWARE at the cap: the PR is
|
|
363
|
+
* already merged, so dependents stay released.
|
|
364
|
+
*/
|
|
365
|
+
function enterRecovery(ctx, state, unit, causeEvent, cause, evidence = {}) {
|
|
366
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
367
|
+
if (issue === null)
|
|
368
|
+
return state;
|
|
369
|
+
const now = ctx.deps.now();
|
|
370
|
+
const entry = (0, state_1.findEntry)(state, issue);
|
|
371
|
+
const slot = slotOf(state, unit);
|
|
372
|
+
if (!entry || !slot)
|
|
373
|
+
return state;
|
|
374
|
+
killUnitAgent(ctx, state, unit);
|
|
375
|
+
const report = slot.phase === 'report';
|
|
376
|
+
const nextTier = report ? (0, dispatch_1.reportTierFor)(slot.recoveries + 1) : (0, dispatch_1.escalateTier)(entry.tier);
|
|
377
|
+
if (slot.recoveries >= types_1.ESCALATION_CAP || nextTier === null) {
|
|
378
|
+
// Cap reached (2 escalations) or already at the strongest tier — the
|
|
379
|
+
// designed signal that a human, not a stronger model, is next.
|
|
380
|
+
const reason = report
|
|
381
|
+
? 'report-escalation-cap'
|
|
382
|
+
: slot.recoveries >= types_1.ESCALATION_CAP
|
|
383
|
+
? 'escalation-cap'
|
|
384
|
+
: `${cause}-at-strongest-tier`;
|
|
385
|
+
return failUnit(ctx, state, unit, reason, { merged: report });
|
|
386
|
+
}
|
|
387
|
+
let next = (0, state_1.transitionSlot)(state, slot.id, 'recovering', { pid: null, recoveries: slot.recoveries + 1 }, now);
|
|
388
|
+
if (!report) {
|
|
389
|
+
next = {
|
|
390
|
+
...next,
|
|
391
|
+
entries: next.entries.map((e) => e.issue === issue ? { ...e, tier: nextTier, updated_at: now.toISOString() } : e),
|
|
392
|
+
};
|
|
393
|
+
}
|
|
394
|
+
journal(ctx, causeEvent, unit, {
|
|
395
|
+
detail: cause,
|
|
396
|
+
slot: slot.id,
|
|
397
|
+
...(slot.last_progress_at !== null ? { last_progress_at: slot.last_progress_at } : {}),
|
|
398
|
+
...evidence,
|
|
399
|
+
});
|
|
400
|
+
journal(ctx, 'redispatched', unit, { tier: nextTier, slot: slot.id });
|
|
401
|
+
ctx.result.redispatched.push(unit);
|
|
402
|
+
// Respawn immediately on the recovering rail — recovering → running. A
|
|
403
|
+
// report-phase slot routes to the report agent with its escalated tier.
|
|
404
|
+
return spawnUnit(ctx, next, unit);
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Verified-exit walk to idle: `complete` is reachable only via
|
|
408
|
+
* exited → verifying → complete → idle; the fallback keeps the walk from
|
|
409
|
+
* ever wedging (assigned/recovering have nothing verified yet).
|
|
410
|
+
*/
|
|
411
|
+
function stepVerifiedExitToIdle(status) {
|
|
412
|
+
if (status === 'complete' || status === 'failed')
|
|
413
|
+
return 'idle';
|
|
414
|
+
if (status === 'running')
|
|
415
|
+
return 'exited';
|
|
416
|
+
if (status === 'exited')
|
|
417
|
+
return 'verifying';
|
|
418
|
+
if (status === 'verifying')
|
|
419
|
+
return 'complete';
|
|
420
|
+
return 'failed';
|
|
421
|
+
}
|
|
422
|
+
/** Complete a unit whose ground truth is verified (AC2). */
|
|
423
|
+
function completeUnit(ctx, state, unit, via) {
|
|
424
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
425
|
+
if (issue === null)
|
|
426
|
+
return state;
|
|
427
|
+
const now = ctx.deps.now();
|
|
428
|
+
const next = walkSlotToIdle(state, unit, now, stepVerifiedExitToIdle);
|
|
429
|
+
let withEntry = next;
|
|
430
|
+
const entry = (0, state_1.findEntry)(next, issue);
|
|
431
|
+
if (entry && entry.status === 'dispatched') {
|
|
432
|
+
withEntry = (0, state_1.transitionIssue)(next, issue, 'shipped', {}, now);
|
|
433
|
+
withEntry = (0, state_1.transitionIssue)(withEntry, issue, 'done', {}, now);
|
|
434
|
+
}
|
|
435
|
+
else if (entry && entry.status === 'shipped') {
|
|
436
|
+
// A report agent completing its run (#468): shipped → done.
|
|
437
|
+
withEntry = (0, state_1.transitionIssue)(next, issue, 'done', {}, now);
|
|
438
|
+
}
|
|
439
|
+
journal(ctx, via, unit);
|
|
440
|
+
if (via === 'external-advance')
|
|
441
|
+
ctx.result.externalAdvances.push(unit);
|
|
442
|
+
else
|
|
443
|
+
ctx.result.completed.push(unit);
|
|
444
|
+
return withEntry;
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* Park a unit whose agent exited after parking its PR on auto-merge (#468):
|
|
448
|
+
* the exit is VERIFIED (the ship phase's `awaiting-merge` milestone with
|
|
449
|
+
* `pr=`), entry → parked (pr recorded), slot released — a waiting unit
|
|
450
|
+
* consumes zero slots (AC5) and the watcher owns it from here.
|
|
451
|
+
*/
|
|
452
|
+
function parkUnit(ctx, state, unit, milestone) {
|
|
453
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
454
|
+
if (issue === null)
|
|
455
|
+
return state;
|
|
456
|
+
const now = ctx.deps.now();
|
|
457
|
+
const pr = (0, groundtruth_1.prOfMilestone)(milestone);
|
|
458
|
+
if (pr === null)
|
|
459
|
+
return state; // isParkedMilestone guarantees this
|
|
460
|
+
let next = walkSlotToIdle(state, unit, now, stepVerifiedExitToIdle);
|
|
461
|
+
next = (0, state_1.transitionIssue)(next, issue, 'parked', { pr }, now);
|
|
462
|
+
journal(ctx, 'pr-parked', unit, { pr });
|
|
463
|
+
ctx.result.parked.push(unit);
|
|
464
|
+
return next;
|
|
465
|
+
}
|
|
466
|
+
// --- Per-slot reconciliation ---
|
|
467
|
+
/** Apply the polled milestone/branch/head signals; returns whether progress happened. */
|
|
468
|
+
function applyProgressSignals(ctx, state, slot, truth, unit) {
|
|
469
|
+
const now = ctx.deps.now();
|
|
470
|
+
let next = state;
|
|
471
|
+
let progressed = false;
|
|
472
|
+
if (truth.milestone !== null) {
|
|
473
|
+
// Live phase per unit (AC6).
|
|
474
|
+
if (truth.milestone.phase !== slot.phase) {
|
|
475
|
+
next = patchSlot(next, slot.id, { phase: truth.milestone.phase }, now);
|
|
476
|
+
journal(ctx, 'phase-updated', unit, { phase: truth.milestone.phase, slot: slot.id });
|
|
477
|
+
}
|
|
478
|
+
// The setup milestone carries the branch name — capture it once so the
|
|
479
|
+
// pushed-commit stall signal can watch the remote head.
|
|
480
|
+
if (slot.branch === null && typeof truth.milestone.keys.branch === 'string') {
|
|
481
|
+
next = patchSlot(next, slot.id, { branch: truth.milestone.keys.branch }, now);
|
|
482
|
+
}
|
|
483
|
+
if (Date.parse(truth.milestone.at) > Date.parse(slot.last_progress_at ?? '')) {
|
|
484
|
+
progressed = true;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
if (truth.head !== null && truth.head !== slot.last_head) {
|
|
488
|
+
progressed = true;
|
|
489
|
+
next = patchSlot(next, slot.id, { last_head: truth.head }, now);
|
|
490
|
+
}
|
|
491
|
+
if (progressed) {
|
|
492
|
+
next = patchSlot(next, slot.id, { last_progress_at: now.toISOString() }, now);
|
|
493
|
+
journal(ctx, 'progress', unit, {
|
|
494
|
+
slot: slot.id,
|
|
495
|
+
detail: truth.milestone
|
|
496
|
+
? `milestone ${truth.milestone.phase}/${truth.milestone.status}`
|
|
497
|
+
: 'new pushed commit',
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
return { state: next, progressed };
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* The issue-closed completion signal for a live unit (#468): a report agent's
|
|
504
|
+
* issue is already closed (closed AT MERGE), so for report-phase slots the
|
|
505
|
+
* closed signal is suppressed — only the report milestone can complete them.
|
|
506
|
+
*/
|
|
507
|
+
function effectiveClosedSignal(slot, truth) {
|
|
508
|
+
return slot.phase === 'report' ? false : truth.closed;
|
|
509
|
+
}
|
|
510
|
+
/** Reconcile one running slot against its polled ground truth. */
|
|
511
|
+
function reconcileRunning(ctx, state, slot, truth, unit) {
|
|
512
|
+
const now = ctx.deps.now();
|
|
513
|
+
// Orphaned pid after a sched restart, or a normally-exited agent: the exit
|
|
514
|
+
// is DETECTED, never trusted as completion (AC2/AC3).
|
|
515
|
+
if (slot.pid !== null && !ctx.deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
|
|
516
|
+
journal(ctx, 'exit-detected', unit, { pid: slot.pid, slot: slot.id });
|
|
517
|
+
const exited = (0, state_1.transitionSlot)(state, slot.id, 'exited', {}, now);
|
|
518
|
+
return completeUnitOrRecover(ctx, exited, unit, truth, 'verify-complete');
|
|
519
|
+
}
|
|
520
|
+
// The milestone poll FAILED (gh outage, missing binary) — unreachable is NOT
|
|
521
|
+
// known-absent (decision 2, option A): stall and advance decisions pause for
|
|
522
|
+
// this unit until truth returns. The dead-pid rail above still ran — local
|
|
523
|
+
// truth needs no network.
|
|
524
|
+
if (!truth.reachable) {
|
|
525
|
+
journal(ctx, 'ground-truth-unreachable', unit, {
|
|
526
|
+
slot: slot.id,
|
|
527
|
+
detail: 'stall/advance decisions paused until truth returns',
|
|
528
|
+
});
|
|
529
|
+
return state;
|
|
530
|
+
}
|
|
531
|
+
// Ground truth says the unit is DONE while the agent still holds the slot —
|
|
532
|
+
// externally-advanced state (AC3): reclaim the slot, kill the leftover agent.
|
|
533
|
+
// A parked milestone is deliberately NOT an advance: a detached run parks
|
|
534
|
+
// and stops — the watcher owns the tail (#468); the exit/stall rails take
|
|
535
|
+
// the agent from here.
|
|
536
|
+
if ((0, groundtruth_1.isVerifiedComplete)(truth.milestone, effectiveClosedSignal(slot, truth)) &&
|
|
537
|
+
!(0, groundtruth_1.isParkedMilestone)(truth.milestone)) {
|
|
538
|
+
journal(ctx, 'external-advance', unit, {
|
|
539
|
+
pid: slot.pid,
|
|
540
|
+
slot: slot.id,
|
|
541
|
+
detail: truth.closed ? 'issue closed' : 'report done',
|
|
542
|
+
});
|
|
543
|
+
killUnitAgent(ctx, state, unit);
|
|
544
|
+
const exited = (0, state_1.transitionSlot)(state, slot.id, 'exited', {}, now);
|
|
545
|
+
return completeUnitOrRecover(ctx, exited, unit, truth, 'external-advance');
|
|
546
|
+
}
|
|
547
|
+
const progress = applyProgressSignals(ctx, state, slot, truth, unit);
|
|
548
|
+
if (progress.progressed)
|
|
549
|
+
return progress.state;
|
|
550
|
+
// No progress: the stall timer (AC4).
|
|
551
|
+
const lastProgress = slot.last_progress_at ? Date.parse(slot.last_progress_at) : 0;
|
|
552
|
+
if (now.getTime() - lastProgress >= ctx.dispatch.stallTimeoutMs) {
|
|
553
|
+
return enterRecovery(ctx, progress.state, unit, 'stalled', 'stall');
|
|
554
|
+
}
|
|
555
|
+
return progress.state;
|
|
556
|
+
}
|
|
557
|
+
/**
|
|
558
|
+
* An exited/verifying slot: verify the claimed state against ground truth
|
|
559
|
+
* (AC2). Verified → complete; a ship-phase `awaiting-merge` milestone with
|
|
560
|
+
* `pr=` → parked (the detached-run exit, #468); unverified → the same
|
|
561
|
+
* recovery ladder as a stall. `via` distinguishes the journal's completion
|
|
562
|
+
* event.
|
|
563
|
+
*/
|
|
564
|
+
function completeUnitOrRecover(ctx, state, unit, truth, via) {
|
|
565
|
+
const now = ctx.deps.now();
|
|
566
|
+
let next = state;
|
|
567
|
+
let slot = slotOf(next, unit);
|
|
568
|
+
if (slot && slot.status === 'exited') {
|
|
569
|
+
next = (0, state_1.transitionSlot)(next, slot.id, 'verifying', {}, now);
|
|
570
|
+
slot = slotOf(next, unit);
|
|
571
|
+
}
|
|
572
|
+
if (!slot || slot.status !== 'verifying')
|
|
573
|
+
return next;
|
|
574
|
+
// The poll failed — unreachable is not "unverified" (decision 2, option A):
|
|
575
|
+
// hold the exit in `verifying` until truth returns, then decide. The agent
|
|
576
|
+
// is already gone; no slot work is lost by waiting.
|
|
577
|
+
if (!truth.reachable) {
|
|
578
|
+
journal(ctx, 'ground-truth-unreachable', unit, {
|
|
579
|
+
slot: slot.id,
|
|
580
|
+
detail: 'exit verification paused until truth returns',
|
|
581
|
+
});
|
|
582
|
+
return next;
|
|
583
|
+
}
|
|
584
|
+
const issue = (0, journal_1.issueOfUnit)(unit);
|
|
585
|
+
const entry = issue !== null ? (0, state_1.findEntry)(next, issue) : undefined;
|
|
586
|
+
// A verified park: the agent exited having parked its PR — the watcher
|
|
587
|
+
// takes the unit (AC2's "never inferred from agent exit" cut both ways:
|
|
588
|
+
// the park IS the milestone, the merge is not).
|
|
589
|
+
if (entry !== undefined && entry.status === 'dispatched' && (0, groundtruth_1.isParkedMilestone)(truth.milestone)) {
|
|
590
|
+
return parkUnit(ctx, next, unit, truth.milestone);
|
|
591
|
+
}
|
|
592
|
+
if ((0, groundtruth_1.isVerifiedComplete)(truth.milestone, effectiveClosedSignal(slot, truth))) {
|
|
593
|
+
return completeUnit(ctx, next, unit, via);
|
|
594
|
+
}
|
|
595
|
+
return enterRecovery(ctx, next, unit, 'verify-incomplete', 'unverified-exit', {
|
|
596
|
+
observed: truth.milestone
|
|
597
|
+
? `milestone ${truth.milestone.phase}/${truth.milestone.status}; closed=${truth.closed}`
|
|
598
|
+
: `no milestone; closed=${truth.closed}`,
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
/** Re-attach or spawn a slot left `assigned` by a crash between assign and spawn. */
|
|
602
|
+
function reconcileAssigned(ctx, state, slot, unit) {
|
|
603
|
+
if (slot.pid !== null && ctx.deps.spawnDeps.isAlive(slot.pid)) {
|
|
604
|
+
// Crash after spawn, before the running transition: re-attach by pid.
|
|
605
|
+
journal(ctx, 'orphan-pid', unit, {
|
|
606
|
+
pid: slot.pid,
|
|
607
|
+
slot: slot.id,
|
|
608
|
+
detail: 're-attached after restart',
|
|
609
|
+
});
|
|
610
|
+
return (0, state_1.transitionSlot)(state, slot.id, 'running', {}, ctx.deps.now());
|
|
611
|
+
}
|
|
612
|
+
journal(ctx, 'assigned', unit, { slot: slot.id, detail: 'crash-recovery spawn' });
|
|
613
|
+
return spawnUnit(ctx, state, unit);
|
|
614
|
+
}
|
|
615
|
+
/** Reconcile a `recovering` slot: respawn with the escalated tier. */
|
|
616
|
+
function reconcileRecovering(ctx, state, unit) {
|
|
617
|
+
return spawnUnit(ctx, state, unit);
|
|
618
|
+
}
|
|
619
|
+
// --- Tick phases ---
|
|
620
|
+
/** Phase 1: reconcile every existing slot against its polled ground truth. */
|
|
621
|
+
function reconcileSlots(ctx, state, polled) {
|
|
622
|
+
let next = state;
|
|
623
|
+
for (const slot of state.slots) {
|
|
624
|
+
const unit = slot.unit;
|
|
625
|
+
if (unit === null)
|
|
626
|
+
continue;
|
|
627
|
+
const truth = polled.get(unit) ?? {
|
|
628
|
+
reachable: true,
|
|
629
|
+
milestone: null,
|
|
630
|
+
closed: false,
|
|
631
|
+
head: null,
|
|
632
|
+
};
|
|
633
|
+
switch (slot.status) {
|
|
634
|
+
case 'assigned':
|
|
635
|
+
next = reconcileAssigned(ctx, next, slot, unit);
|
|
636
|
+
break;
|
|
637
|
+
case 'running':
|
|
638
|
+
next = reconcileRunning(ctx, next, slot, truth, unit);
|
|
639
|
+
break;
|
|
640
|
+
case 'exited':
|
|
641
|
+
case 'verifying':
|
|
642
|
+
next = completeUnitOrRecover(ctx, next, unit, truth, 'verify-complete');
|
|
643
|
+
break;
|
|
644
|
+
case 'recovering':
|
|
645
|
+
next = reconcileRecovering(ctx, next, unit);
|
|
646
|
+
break;
|
|
647
|
+
default:
|
|
648
|
+
break;
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
return next;
|
|
652
|
+
}
|
|
653
|
+
/** Phase 2: self-heal orphaned dispatches (crash between the entry and slot transitions). */
|
|
654
|
+
function requeueOrphanedDispatches(ctx, state) {
|
|
655
|
+
const held = new Set(state.slots.map((s) => s.unit).filter((u) => u !== null));
|
|
656
|
+
let next = state;
|
|
657
|
+
for (const entry of state.entries) {
|
|
658
|
+
if (entry.status !== 'dispatched' || held.has(`issue:${entry.issue}`))
|
|
659
|
+
continue;
|
|
660
|
+
const now = ctx.deps.now();
|
|
661
|
+
next = (0, state_1.transitionIssue)(next, entry.issue, 'blocked', { reason: 'orphaned-dispatch' }, now);
|
|
662
|
+
next = (0, state_1.transitionIssue)(next, entry.issue, 'queued', { reason: null }, now);
|
|
663
|
+
journal(ctx, 'requeued', `issue:${entry.issue}`, {
|
|
664
|
+
detail: 'orphaned dispatch requeued after restart',
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
return next;
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* The PR watcher's decision pass (#468): apply polled PR truths to parked
|
|
671
|
+
* entries. Merge acceptance (AC1) requires state MERGED AND mergedAt non-null
|
|
672
|
+
* AND the issue closed — never an agent exit. Failure states (AC3) fail the
|
|
673
|
+
* unit and block transitive dependents; the engine never merges anything
|
|
674
|
+
* itself. `shipped` (not `parked`) is what unblocks dependents (AC4).
|
|
675
|
+
*
|
|
676
|
+
* Entries are re-read fresh each iteration: a mid-loop failure blocks OTHER
|
|
677
|
+
* parked entries (transitive dependents), and acting on a stale snapshot
|
|
678
|
+
* would drive an already-blocked entry through `parked → shipped`.
|
|
679
|
+
*/
|
|
680
|
+
function reconcileParked(ctx, state, prPoll) {
|
|
681
|
+
if (!prPoll.ran)
|
|
682
|
+
return state;
|
|
683
|
+
let next = { ...state, last_pr_poll_at: ctx.deps.now().toISOString() };
|
|
684
|
+
const parkedIssues = state.entries
|
|
685
|
+
.filter((e) => e.status === 'parked' && e.pr !== null)
|
|
686
|
+
.map((e) => e.issue);
|
|
687
|
+
for (const issue of parkedIssues) {
|
|
688
|
+
const entry = (0, state_1.findEntry)(next, issue);
|
|
689
|
+
if (!entry || entry.status !== 'parked' || entry.pr === null)
|
|
690
|
+
continue; // blocked mid-loop
|
|
691
|
+
const unit = `issue:${issue}`;
|
|
692
|
+
const truth = prPoll.truths.get(issue);
|
|
693
|
+
if (truth === undefined) {
|
|
694
|
+
if (!prPoll.truths.has(issue)) {
|
|
695
|
+
continue; // parked AFTER the poll ran (this tick) — next cadence picks it up
|
|
696
|
+
}
|
|
697
|
+
journal(ctx, 'ground-truth-unreachable', unit, {
|
|
698
|
+
detail: 'pr watch paused until truth returns',
|
|
699
|
+
});
|
|
700
|
+
continue;
|
|
701
|
+
}
|
|
702
|
+
const failWatch = (reason) => {
|
|
703
|
+
journal(ctx, 'pr-watch-failed', unit, { reason, pr: entry.pr });
|
|
704
|
+
return failUnit(ctx, next, unit, reason);
|
|
705
|
+
};
|
|
706
|
+
if (truth.blocked) {
|
|
707
|
+
next = failWatch('auto-merge-blocked');
|
|
708
|
+
continue;
|
|
709
|
+
}
|
|
710
|
+
if (truth.mergeable === 'CONFLICTING') {
|
|
711
|
+
next = failWatch('pr-conflicting');
|
|
712
|
+
continue;
|
|
713
|
+
}
|
|
714
|
+
if (truth.state === 'CLOSED' && truth.mergedAt === null) {
|
|
715
|
+
next = failWatch('pr-closed-unmerged');
|
|
716
|
+
continue;
|
|
717
|
+
}
|
|
718
|
+
if (truth.state === 'MERGED' && truth.mergedAt !== null) {
|
|
719
|
+
// AC1: the issue must ALSO be closed (a merged PR auto-closes it) —
|
|
720
|
+
// until GitHub propagates, the unit stays parked and keeps watching.
|
|
721
|
+
if (prPoll.closed.get(issue) !== true) {
|
|
722
|
+
journal(ctx, 'pr-watch-waiting', unit, {
|
|
723
|
+
pr: entry.pr,
|
|
724
|
+
mergedAt: truth.mergedAt,
|
|
725
|
+
detail: 'merge seen but issue not closed — keep watching',
|
|
726
|
+
});
|
|
727
|
+
continue;
|
|
728
|
+
}
|
|
729
|
+
next = (0, state_1.transitionIssue)(next, issue, 'shipped', { reason: null }, ctx.deps.now());
|
|
730
|
+
journal(ctx, 'merge-accepted', unit, { pr: entry.pr, mergedAt: truth.mergedAt });
|
|
731
|
+
ctx.result.mergeAccepted.push(unit);
|
|
732
|
+
}
|
|
733
|
+
// OPEN (or MERGED without a date / mergeable UNKNOWN) — keep watching.
|
|
734
|
+
}
|
|
735
|
+
return next;
|
|
736
|
+
}
|
|
737
|
+
/**
|
|
738
|
+
* Dispatch report agents for merged units whose teardown is recorded (#468
|
|
739
|
+
* AC2): shipped + pr + cleanup + no live slot + free capacity → a slot is
|
|
740
|
+
* assigned (phase `report`) and a mechanical-tier agent spawned with the
|
|
741
|
+
* report prompt. Reports run BEFORE queue refill — a cheap report never
|
|
742
|
+
* queues behind long full-cycle runs. A unit waiting for capacity consumes
|
|
743
|
+
* zero slots (AC5) and is surfaced via `result.reportWaiting`.
|
|
744
|
+
*/
|
|
745
|
+
function dispatchReportAgents(ctx, state, config) {
|
|
746
|
+
let next = state;
|
|
747
|
+
for (const entry of state.entries) {
|
|
748
|
+
if (entry.status !== 'shipped' || entry.pr === null || entry.cleanup === null)
|
|
749
|
+
continue;
|
|
750
|
+
const unit = `issue:${entry.issue}`;
|
|
751
|
+
if (slotOf(next, unit) !== undefined)
|
|
752
|
+
continue; // a slot already holds the unit
|
|
753
|
+
if ((0, scheduler_1.freeCapacity)(next, config) === 0) {
|
|
754
|
+
// Full — the report waits (zero slots consumed). Count what is waiting
|
|
755
|
+
// so the wait is visible, then stop scanning.
|
|
756
|
+
ctx.result.reportWaiting = state.entries.filter((e) => e.status === 'shipped' &&
|
|
757
|
+
e.pr !== null &&
|
|
758
|
+
e.cleanup !== null &&
|
|
759
|
+
slotOf(next, `issue:${e.issue}`) === undefined).length;
|
|
760
|
+
break;
|
|
761
|
+
}
|
|
762
|
+
const now = ctx.deps.now();
|
|
763
|
+
const assigned = (0, scheduler_1.assignToIdleSlot)(next, unit, 'report', now);
|
|
764
|
+
next = assigned.state;
|
|
765
|
+
journal(ctx, 'assigned', unit, { slot: assigned.slotId, detail: 'report agent' });
|
|
766
|
+
journal(ctx, 'report-dispatched', unit, {
|
|
767
|
+
slot: assigned.slotId,
|
|
768
|
+
pr: entry.pr,
|
|
769
|
+
cleanup: entry.cleanup,
|
|
770
|
+
});
|
|
771
|
+
ctx.result.reportDispatched.push(unit);
|
|
772
|
+
next = spawnReportAgent(ctx, next, unit);
|
|
773
|
+
}
|
|
774
|
+
return next;
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* Run the teardown script for one merged unit OUTSIDE the state lock (#468
|
|
778
|
+
* AC2). Returns null when teardown must be retried next tick (setup info
|
|
779
|
+
* unreachable — a transient outage, never a failure).
|
|
780
|
+
*/
|
|
781
|
+
function runTeardownFor(deps, issue) {
|
|
782
|
+
const info = deps.groundTruth.setupInfo(issue);
|
|
783
|
+
const unit = `issue:${issue}`;
|
|
784
|
+
if (info === undefined) {
|
|
785
|
+
deps.journal.append((0, journal_1.unitEvent)('ground-truth-unreachable', unit, {
|
|
786
|
+
detail: 'teardown paused until truth returns',
|
|
787
|
+
}), deps.now());
|
|
788
|
+
return null;
|
|
789
|
+
}
|
|
790
|
+
if (info === null) {
|
|
791
|
+
return {
|
|
792
|
+
cleanup: 'failed-missing-setup-info',
|
|
793
|
+
detail: 'no setup milestone with worktree= found on the issue',
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
return (0, teardown_1.runTeardown)(deps.teardownExec, deps.repoDir, info);
|
|
797
|
+
}
|
|
798
|
+
/** Phase 3: refill — every freed slot is filled in THIS tick (AC5). */
|
|
799
|
+
function dispatchAssignments(ctx, state, config) {
|
|
800
|
+
const now = ctx.deps.now();
|
|
801
|
+
const { state: assigned, assignments } = (0, scheduler_1.computeAssignments)(state, config, now, ['issue']);
|
|
802
|
+
let next = assigned;
|
|
803
|
+
for (const assignment of assignments) {
|
|
804
|
+
if (assignment.kind !== 'issue')
|
|
805
|
+
continue;
|
|
806
|
+
const unit = `issue:${assignment.issue}`;
|
|
807
|
+
journal(ctx, 'assigned', unit, { slot: assignment.slot });
|
|
808
|
+
const entry = (0, state_1.findEntry)(next, assignment.issue);
|
|
809
|
+
if (!entry)
|
|
810
|
+
continue;
|
|
811
|
+
if (entry.status === 'queued') {
|
|
812
|
+
next = (0, state_1.transitionIssue)(next, assignment.issue, 'classified', {}, now);
|
|
813
|
+
}
|
|
814
|
+
next = (0, state_1.transitionIssue)(next, assignment.issue, 'dispatched', {}, now);
|
|
815
|
+
next = spawnUnit(ctx, next, unit);
|
|
816
|
+
}
|
|
817
|
+
return next;
|
|
818
|
+
}
|
|
819
|
+
/** Entries freshly shipped whose teardown has not run yet (cleanup still null). */
|
|
820
|
+
function teardownPendingIssues(state) {
|
|
821
|
+
return state.entries
|
|
822
|
+
.filter((e) => e.status === 'shipped' && e.pr !== null && e.cleanup === null)
|
|
823
|
+
.map((e) => e.issue);
|
|
824
|
+
}
|
|
825
|
+
/** Record teardown results on their entries + journal (the lock pass after the subprocesses). */
|
|
826
|
+
function recordTeardowns(ctx, state, results) {
|
|
827
|
+
const now = ctx.deps.now();
|
|
828
|
+
let next = state;
|
|
829
|
+
for (const [issue, result] of results) {
|
|
830
|
+
const unit = `issue:${issue}`;
|
|
831
|
+
next = {
|
|
832
|
+
...next,
|
|
833
|
+
entries: next.entries.map((e) => e.issue === issue ? { ...e, cleanup: result.cleanup, updated_at: now.toISOString() } : e),
|
|
834
|
+
};
|
|
835
|
+
journal(ctx, result.cleanup === 'done' ? 'teardown-done' : 'teardown-failed', unit, {
|
|
836
|
+
cleanup: result.cleanup,
|
|
837
|
+
detail: result.detail,
|
|
838
|
+
});
|
|
839
|
+
if (result.cleanup === 'done')
|
|
840
|
+
ctx.result.teardownDone.push(unit);
|
|
841
|
+
else
|
|
842
|
+
ctx.result.teardownFailed.push(unit);
|
|
843
|
+
}
|
|
844
|
+
return next;
|
|
845
|
+
}
|
|
846
|
+
/**
|
|
847
|
+
* One full reconcile+refill cycle. Ground truth is polled WITHOUT the lock
|
|
848
|
+
* (live units AND parked PRs); every state mutation happens under
|
|
849
|
+
* `store.withLock`; refills are computed in the same pass, so a freed slot
|
|
850
|
+
* is reused within this tick (AC5). Teardown subprocesses run between two
|
|
851
|
+
* short lock passes — a slow `npx`/`git` call never holds the lock.
|
|
852
|
+
*/
|
|
853
|
+
function tick(deps, config) {
|
|
854
|
+
const dispatch = (0, dispatch_1.resolveDispatch)(config);
|
|
855
|
+
const state0 = deps.store.load();
|
|
856
|
+
const polled = pollUnits(deps, state0);
|
|
857
|
+
const prPoll = pollParkedPrs(deps, state0, dispatch);
|
|
858
|
+
const pass1 = deps.store.withLock((state) => {
|
|
859
|
+
const ctx = { deps, dispatch, result: emptyResult() };
|
|
860
|
+
let next = reconcileSlots(ctx, state, polled);
|
|
861
|
+
next = reconcileParked(ctx, next, prPoll);
|
|
862
|
+
next = requeueOrphanedDispatches(ctx, next);
|
|
863
|
+
next = dispatchReportAgents(ctx, next, config);
|
|
864
|
+
next = dispatchAssignments(ctx, next, config);
|
|
865
|
+
return {
|
|
866
|
+
state: next,
|
|
867
|
+
result: { tick: ctx.result, teardownPending: teardownPendingIssues(next) },
|
|
868
|
+
};
|
|
869
|
+
});
|
|
870
|
+
if (pass1.teardownPending.length === 0) {
|
|
871
|
+
return pass1.tick;
|
|
872
|
+
}
|
|
873
|
+
// Teardown subprocesses (pool return / worktree remove) run OUTSIDE the
|
|
874
|
+
// lock; results land in a second short lock pass together with the report
|
|
875
|
+
// dispatch (AC2: teardown, THEN the cheap-tier report agent).
|
|
876
|
+
const results = new Map();
|
|
877
|
+
for (const issue of pass1.teardownPending) {
|
|
878
|
+
const result = runTeardownFor(deps, issue);
|
|
879
|
+
if (result !== null)
|
|
880
|
+
results.set(issue, result);
|
|
881
|
+
}
|
|
882
|
+
if (results.size === 0) {
|
|
883
|
+
return pass1.tick; // all deferred (setup info unreachable) — retried next tick
|
|
884
|
+
}
|
|
885
|
+
return deps.store.withLock((state) => {
|
|
886
|
+
const ctx = { deps, dispatch, result: pass1.tick };
|
|
887
|
+
const next = recordTeardowns(ctx, state, results);
|
|
888
|
+
const withReport = dispatchReportAgents(ctx, next, config);
|
|
889
|
+
return { state: withReport, result: ctx.result };
|
|
890
|
+
});
|
|
891
|
+
}
|
|
892
|
+
/**
|
|
893
|
+
* The long-running loop behind `sched start`: tick, sleep, repeat. Returns
|
|
894
|
+
* when `shouldStop()` says so (the CLI wires SIGINT). A failed tick is
|
|
895
|
+
* journaled (`tick-failed`, with the error name — `LockTimeoutError` and
|
|
896
|
+
* `CorruptStateError` demand different operator actions) and reported on
|
|
897
|
+
* stderr, and the loop continues — one bad tick (e.g. a transient gh
|
|
898
|
+
* failure) never stops the scheduler.
|
|
899
|
+
*/
|
|
900
|
+
async function runLoop(deps, config, shouldStop, onTick) {
|
|
901
|
+
const interval = (0, dispatch_1.resolveDispatch)(config).reconcileIntervalMs;
|
|
902
|
+
while (!shouldStop()) {
|
|
903
|
+
try {
|
|
904
|
+
const result = tick(deps, config);
|
|
905
|
+
onTick?.(result);
|
|
906
|
+
}
|
|
907
|
+
catch (err) {
|
|
908
|
+
const detail = `${err.name}: ${err.message}`;
|
|
909
|
+
process.stderr.write(`⚠ sched tick failed: ${detail}\n`);
|
|
910
|
+
deps.journal.append({ event: 'tick-failed', detail }, deps.now());
|
|
911
|
+
}
|
|
912
|
+
if (shouldStop())
|
|
913
|
+
break;
|
|
914
|
+
await sleep(interval, shouldStop);
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
function sleep(ms, shouldStop) {
|
|
918
|
+
return new Promise((resolve) => {
|
|
919
|
+
const timer = setTimeout(() => {
|
|
920
|
+
clearInterval(stopCheck);
|
|
921
|
+
resolve();
|
|
922
|
+
}, ms);
|
|
923
|
+
const stopCheck = setInterval(() => {
|
|
924
|
+
if (shouldStop()) {
|
|
925
|
+
clearTimeout(timer);
|
|
926
|
+
clearInterval(stopCheck);
|
|
927
|
+
resolve();
|
|
928
|
+
}
|
|
929
|
+
}, Math.min(dispatch_1.STOP_POLL_MAX_MS, Math.max(dispatch_1.STOP_POLL_MIN_MS, Math.floor(ms / 10))));
|
|
930
|
+
if (typeof timer.unref === 'function')
|
|
931
|
+
timer.unref();
|
|
932
|
+
if (typeof stopCheck.unref === 'function')
|
|
933
|
+
stopCheck.unref();
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
//# sourceMappingURL=engine.js.map
|