@ai-dossier/sched 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -6
- package/dist/batch-dispatch.d.ts +45 -20
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +269 -41
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +12 -0
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +15 -0
- package/dist/dispatch.js.map +1 -1
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +40 -7
- package/dist/engine.js.map +1 -1
- package/dist/groundtruth.d.ts +19 -2
- package/dist/groundtruth.d.ts.map +1 -1
- package/dist/groundtruth.js +54 -7
- package/dist/groundtruth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -4
- package/dist/index.js.map +1 -1
- package/dist/recovery.d.ts.map +1 -1
- package/dist/recovery.js +4 -1
- package/dist/recovery.js.map +1 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +18 -2
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +17 -0
- package/dist/status.js.map +1 -1
- package/dist/types.d.ts +51 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +3 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -88,7 +88,13 @@ where every mechanical supervision decision is code, not remembered prose:
|
|
|
88
88
|
runstate milestone is `report done`, or GitHub says the issue is closed — except a
|
|
89
89
|
report-agent slot (`role: 'report'`), whose issue is already closed at merge: the
|
|
90
90
|
closed signal is suppressed and only a `report done` milestone completes it (#500).
|
|
91
|
-
An unverified exit rides the recovery ladder like a stall.
|
|
91
|
+
An unverified exit rides the recovery ladder like a stall. A `report done` milestone
|
|
92
|
+
must also postdate the slot's own `spawned_at` (±60s clock-skew tolerance, #575) — a
|
|
93
|
+
re-enqueued issue's PREVIOUS run's report milestone is ignored (journaled
|
|
94
|
+
`stale-milestone-ignored`) rather than instantly completing a freshly-spawned agent on
|
|
95
|
+
its first reconcile tick; a legacy slot with no `spawned_at` degrades to the old,
|
|
96
|
+
unfenced check. Batch members get the same fence on their own completion signal
|
|
97
|
+
(`isMemberComplete`, `phase=review status=done mode=slot`).
|
|
92
98
|
3. **Reconciliation tick (AC3)** — every tick detects externally-advanced state (someone
|
|
93
99
|
finished the work outside sched → complete, kill the leftover agent, reclaim the
|
|
94
100
|
slot), orphaned pids after a restart (dead pid on a running slot → exit rail →
|
|
@@ -122,7 +128,7 @@ where every mechanical supervision decision is code, not remembered prose:
|
|
|
122
128
|
progress, stalled, redispatched, fence-written, fence-failed, unit-failed,
|
|
123
129
|
dependents-blocked, slot-released, suspect-dispatch, dispatch-unhealthy,
|
|
124
130
|
run-log-recorded, run-log-no-usage, run-log-skipped, run-log-failed, engine-stale,
|
|
125
|
-
engine-auto-upgrade-attempted, engine-auto-upgrade-failed, …) is
|
|
131
|
+
engine-auto-upgrade-attempted, engine-auto-upgrade-failed, stale-milestone-ignored, …) is
|
|
126
132
|
appended to `events.jsonl`; `sched status` shows the live phase per unit, plus each
|
|
127
133
|
slot's `gen` and `fenced` state (#504).
|
|
128
134
|
`engine-stale`/`engine-auto-upgrade-attempted`/`engine-auto-upgrade-failed`
|
|
@@ -207,6 +213,13 @@ Two engine-safety policies were explicit product decisions on #464:
|
|
|
207
213
|
never kill a healthy agent or fail a unit as "unverified". An agent that exits during
|
|
208
214
|
an outage holds in `verifying` until truth returns. Each pause is journaled as
|
|
209
215
|
`ground-truth-unreachable`.
|
|
216
|
+
- **A completion milestone is fenced to its own dispatch (#575).** `isVerifiedComplete`
|
|
217
|
+
(issues) and `isMemberComplete` (batch members) both accept the current dispatch's
|
|
218
|
+
`SlotEntry.spawned_at` and reject a `report done` / `review done mode=slot` milestone
|
|
219
|
+
that predates it (±60s clock-skew tolerance) — a re-enqueued issue or a member re-added
|
|
220
|
+
to a fresh batch run must not read as instantly complete against a PREVIOUS run's
|
|
221
|
+
milestone. The rejection is journaled as `stale-milestone-ignored`; `spawned_at=null`
|
|
222
|
+
(a legacy slot) degrades to the old, unfenced check.
|
|
210
223
|
|
|
211
224
|
This applies to `issue:<n>` unit dispatch (`dispatchAssignments`). `batch:<id>` units run
|
|
212
225
|
through a separate pass with its own claim/reconcile logic — see
|
|
@@ -442,6 +455,8 @@ ready → executing(member i/N) ⟲ → validating → reviewing → shipping
|
|
|
442
455
|
failure rails: executing → dissolving (a member self-reports blocked)
|
|
443
456
|
validating → attributing → (fixing | evicting) → validating → dissolving
|
|
444
457
|
validating → blocked (suite report unreadable, #562) → validating
|
|
458
|
+
executing → blocked (gate-inconclusive:<cap>, #583) → executing
|
|
459
|
+
(`sched resume --batch <id>` re-runs the gate; nothing requeued/reverted)
|
|
445
460
|
```
|
|
446
461
|
|
|
447
462
|
- **One shared worktree/branch per batch**, claimed once by a deterministic (no LLM)
|
|
@@ -460,7 +475,12 @@ failure rails: executing → dissolving (a member self-reports blocked)
|
|
|
460
475
|
kept on `BatchEntry.ranges` for eviction. An incremental gate (`ai-dossier cap run
|
|
461
476
|
typecheck.run` / `test.focused`, when the repo has a manifest) runs after each member
|
|
462
477
|
before advancing — a second, independent check that the member's self-reported "done"
|
|
463
|
-
is real.
|
|
478
|
+
is real. Three-way outcome policy (#583): `task-failed` evicts the member (same rail
|
|
479
|
+
as a self-reported block); `automation-broken`/`capability-unavailable` — the gate
|
|
480
|
+
itself couldn't reach a verdict — block the batch instead of silently proceeding
|
|
481
|
+
(`gate-inconclusive:<cap>`, `member_gates`/`blocked_reason` on `BatchEntry`, surfaced
|
|
482
|
+
in `sched status`); `sched resume --batch <id>` re-runs the gate later to resolve the
|
|
483
|
+
block once the capability is fixed.
|
|
464
484
|
- **The batch's single slot is claimed FRESH for each live step** (a member, the tail
|
|
465
485
|
agent, the report agent, a bounded fix agent) — never held across a wait. The aggregate
|
|
466
486
|
suite itself runs with NO slot claimed at all (deterministic engine work, not an LLM
|
|
@@ -491,10 +511,21 @@ batch-setup had pool integration). Config schema moves to 1.3.0: `dispatch` gain
|
|
|
491
511
|
`member_prompt`, `batch_tail_prompt` and `batch_report_prompt` (the three new agent
|
|
492
512
|
prompt templates).
|
|
493
513
|
|
|
514
|
+
Schema 1.11.0 (#583): `BatchEntry` gains `member_gates` (most recent incremental-gate
|
|
515
|
+
result per member, keyed by issue number as a string — `{capability, outcome,
|
|
516
|
+
output_tail, at}`) and `blocked_reason` (why the batch is `blocked` — persisted so
|
|
517
|
+
`sched status` can show it; previously `blockBatch` only journaled/posted the reason,
|
|
518
|
+
never stored it on the entry, so this also retroactively covers the #562 case). 1.10.0
|
|
519
|
+
states migrate on load: no gate has ever produced a non-`ok` verdict, and no batch has
|
|
520
|
+
ever been blocked, under them, so `{}`/`null` is the exact backfill, not a guess.
|
|
521
|
+
|
|
494
522
|
New journal events: `batch-setup-done`, `batch-setup-failed`, `member-advanced`,
|
|
495
523
|
`batch-warmup-done`, `batch-warmup-failed` (#561 — the cold-path warm step only; a pool
|
|
496
|
-
claim emits neither).
|
|
497
|
-
|
|
524
|
+
claim emits neither). `gate-inconclusive` (#583 — the incremental gate came back
|
|
525
|
+
`automation-broken`/`capability-unavailable` rather than a definite `ok`/`task-failed`;
|
|
526
|
+
sits alongside `batch-blocked` as the per-member analogue of the aggregate suite's
|
|
527
|
+
"block, don't dissolve" precedent). Member/tail/report/fix-agent spawn, progress,
|
|
528
|
+
completion and park events reuse the existing unit-generic names (`assigned`/`spawned`/`unit-failed`/
|
|
498
529
|
`external-advance`/`pr-parked`/`merge-accepted`/`report-dispatched`/`teardown-done`/
|
|
499
530
|
`teardown-failed`) with `unit = batch:<id>`.
|
|
500
531
|
|
|
@@ -611,10 +642,12 @@ import {
|
|
|
611
642
|
// the issue pass — loads/saves state itself, holds no lock
|
|
612
643
|
// across the call
|
|
613
644
|
type BatchDispatchDeps, // inject store/journal/groundTruth/spawnDeps/exec/runSuite/
|
|
614
|
-
// runCapability(optional)/fsExists(optional)
|
|
645
|
+
// runCapability(optional, returns CapabilityGateResult)/fsExists(optional)
|
|
615
646
|
type BatchTickResult, // spawned/completed/parked/mergeAccepted/failed (batch:<id> ids)
|
|
616
647
|
// + blocked (issue numbers, dissolve-requeued)
|
|
617
648
|
type CapOutcome, // ok | task-failed | automation-broken | capability-unavailable
|
|
649
|
+
type CapabilityGateResult, // {outcome: CapOutcome, outputTail?, reason?} — runCapability's return shape (#583)
|
|
650
|
+
resumeBlockedGate, // #583: sched resume --batch <id> — re-run the gate that blocked a batch
|
|
618
651
|
buildMemberPrompt, buildBatchTailPrompt, buildBatchReportPrompt, // #523 prompt builders
|
|
619
652
|
DEFAULT_MEMBER_PROMPT_TEMPLATE, DEFAULT_BATCH_TAIL_PROMPT_TEMPLATE,
|
|
620
653
|
DEFAULT_BATCH_REPORT_PROMPT_TEMPLATE,
|
package/dist/batch-dispatch.d.ts
CHANGED
|
@@ -67,15 +67,8 @@ import type { SchedStore } from './persist';
|
|
|
67
67
|
import type { ExecFn } from './project';
|
|
68
68
|
import { type SuiteResult } from './recovery';
|
|
69
69
|
import { type FsExists } from './teardown';
|
|
70
|
-
import type { SchedConfig } from './types';
|
|
71
|
-
|
|
72
|
-
* The four `ai-dossier cap run` outcomes (docs/reference/capabilities.md):
|
|
73
|
-
* `ok` = the capability ran and passed; `task-failed` = it ran and the TASK
|
|
74
|
-
* itself failed (trust the result); `automation-broken` = do not trust the
|
|
75
|
-
* machinery (missing tool, timeout, bad manifest); `capability-unavailable` =
|
|
76
|
-
* no manifest / no such id / `lifecycle: shadow` — no fast path here.
|
|
77
|
-
*/
|
|
78
|
-
export type CapOutcome = 'ok' | 'task-failed' | 'automation-broken' | 'capability-unavailable';
|
|
70
|
+
import type { CapabilityGateResult, SchedConfig } from './types';
|
|
71
|
+
export type { CapOutcome } from './types';
|
|
79
72
|
/** Everything batch dispatch needs from the outside world. */
|
|
80
73
|
export interface BatchDispatchDeps {
|
|
81
74
|
store: SchedStore;
|
|
@@ -100,19 +93,24 @@ export interface BatchDispatchDeps {
|
|
|
100
93
|
/**
|
|
101
94
|
* Runs one `ai-dossier cap run <capabilityId>` in a batch worktree. Two call
|
|
102
95
|
* sites, different degrade contracts:
|
|
103
|
-
* - the per-member incremental gate (#523 AC2
|
|
104
|
-
*
|
|
105
|
-
* this
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
96
|
+
* - the per-member incremental gate (#523 AC2, revised #583): without this
|
|
97
|
+
* hook, the gate is skipped entirely (the member's own `slot-cycle` run
|
|
98
|
+
* already attempted this fast path before ever posting `review done`, so
|
|
99
|
+
* a repo with no manifest loses nothing but the engine's independent
|
|
100
|
+
* re-check). With the hook: `ok` advances, `task-failed` evicts,
|
|
101
|
+
* `automation-broken`/`capability-unavailable` BLOCK the batch
|
|
102
|
+
* (`gate-inconclusive`) rather than silently proceeding — #583 found the
|
|
103
|
+
* old "proceed on anything but task-failed" policy let a script's own
|
|
104
|
+
* "I could not run this" signal (a non-zero exit reporting it never
|
|
105
|
+
* really tested anything) masquerade as either a pass or a real failure.
|
|
109
106
|
* - batch-setup's `worktree.prepare` warm step (#561, `warmColdBatchWorktree`):
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
107
|
+
* unaffected by #583's gate policy change — `undefined`/
|
|
108
|
+
* `capability-unavailable`/`automation-broken` still fall through to
|
|
109
|
+
* package-manager detection, and a declared-and-`task-failed` capability
|
|
110
|
+
* still hard-fails the whole batch setup (a repo that owns its warm-up
|
|
111
|
+
* should never be silently second-guessed by a fallback underneath it).
|
|
114
112
|
*/
|
|
115
|
-
runCapability?: (worktree: string, capabilityId: string) =>
|
|
113
|
+
runCapability?: (worktree: string, capabilityId: string) => CapabilityGateResult;
|
|
116
114
|
fsExists?: FsExists;
|
|
117
115
|
/**
|
|
118
116
|
* Home directory for `~/.dossier/runs.jsonl` (#564) — mirrors `EngineDeps.homeDir`.
|
|
@@ -130,6 +128,33 @@ export interface BatchTickResult {
|
|
|
130
128
|
/** Issue numbers requeued full-cycle by a dissolve — matches `TickResult.blocked`'s shape. */
|
|
131
129
|
blocked: number[];
|
|
132
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* `sched resume --batch <id>` (#583 AC4): an operator-triggered, synchronous
|
|
133
|
+
* one-shot recheck of a batch blocked on `gate-inconclusive:<capabilityId>`
|
|
134
|
+
* — re-runs exactly that capability against the current member and resolves
|
|
135
|
+
* the block:
|
|
136
|
+
*
|
|
137
|
+
* - still `automation-broken`/`capability-unavailable` → stays `blocked`,
|
|
138
|
+
* no state change (the capability still isn't fixed).
|
|
139
|
+
* - `task-failed` → the member really is broken; evict via the same rail
|
|
140
|
+
* the live gate uses.
|
|
141
|
+
* - `ok` → the member really was fine; complete it via the same rail the
|
|
142
|
+
* live gate uses.
|
|
143
|
+
*
|
|
144
|
+
* Deliberately NOT a generic engine tick: `runBatchTick`'s own "executing
|
|
145
|
+
* with no live slot" wedge-recovery path (`spawnMemberContinuation`,
|
|
146
|
+
* unconditional) exists for a dispatch that never happened — reusing it here
|
|
147
|
+
* would redispatch a fresh agent for a member whose work is already
|
|
148
|
+
* committed and reviewed. This function transitions `blocked → executing`
|
|
149
|
+
* and immediately, synchronously, calls the same completion/eviction
|
|
150
|
+
* functions the live gate calls — no intervening tick ever sees the batch
|
|
151
|
+
* `executing` with nothing in flight.
|
|
152
|
+
*/
|
|
153
|
+
export declare function resumeBlockedGate(deps: BatchDispatchDeps, config: SchedConfig, dispatch: ResolvedDispatch, batchId: string, now: Date): {
|
|
154
|
+
outcome: 'still-blocked' | 'evicted' | 'completed';
|
|
155
|
+
capability: string;
|
|
156
|
+
detail?: string;
|
|
157
|
+
};
|
|
133
158
|
/**
|
|
134
159
|
* One batch reconcile+refill pass. Called from `engine.ts`'s `tick()` after
|
|
135
160
|
* the issue-level pass — this pass never claims a slot the issue pass already
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AAiBH,OAAO,
|
|
1
|
+
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AAiBH,OAAO,EAUL,KAAK,gBAAgB,EAErB,KAAK,SAAS,EACf,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,KAAK,WAAW,EAOjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAUL,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAYpB,OAAO,EAAE,KAAK,QAAQ,EAA2D,MAAM,YAAY,CAAC;AACpG,OAAO,KAAK,EAGV,oBAAoB,EAIpB,WAAW,EAIZ,MAAM,SAAS,CAAC;AAMjB,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sGAAsG;IACtG,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,WAAW,CAAC;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAywCD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IAAE,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CA+E7F;AAieD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CAsFjB"}
|
package/dist/batch-dispatch.js
CHANGED
|
@@ -95,6 +95,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
95
95
|
};
|
|
96
96
|
})();
|
|
97
97
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
98
|
+
exports.resumeBlockedGate = resumeBlockedGate;
|
|
98
99
|
exports.runBatchTick = runBatchTick;
|
|
99
100
|
const fs = __importStar(require("node:fs"));
|
|
100
101
|
const path = __importStar(require("node:path"));
|
|
@@ -140,6 +141,65 @@ function sanitizeUntrustedText(value) {
|
|
|
140
141
|
function journalEvent(deps, event, unitId, extra = {}) {
|
|
141
142
|
deps.journal.append((0, journal_1.unitEvent)(event, unitId, extra), deps.now());
|
|
142
143
|
}
|
|
144
|
+
/**
|
|
145
|
+
* Last ~500 bytes of a gate's output tail, falling back to the envelope's
|
|
146
|
+
* `reason` when no subprocess ran (#583 AC1/AC3 review: `capability-unavailable`
|
|
147
|
+
* and a failed assumption probe carry no output_tail — `reason` is the only
|
|
148
|
+
* explanation available for those) — journal details stay compact; the full
|
|
149
|
+
* tail lives in the per-gate log file. UTF-8-safe (never splits a multi-byte
|
|
150
|
+
* character) — chars would risk exactly that, hence `Buffer`, matching
|
|
151
|
+
* `cli/src/capability.ts`'s `truncateTailBytes`.
|
|
152
|
+
*/
|
|
153
|
+
function gateDetailExcerpt(outputTail, reason) {
|
|
154
|
+
const text = outputTail || reason;
|
|
155
|
+
if (!text)
|
|
156
|
+
return undefined;
|
|
157
|
+
const buf = Buffer.from(text, 'utf-8');
|
|
158
|
+
return buf.length > 500 ? buf.subarray(buf.length - 500).toString('utf-8') : text;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Best-effort per-gate diagnostic log (#583 AC1) — mirrors `appendCapLog`'s
|
|
162
|
+
* never-crash contract: a log-write failure must not interrupt the gate
|
|
163
|
+
* decision itself.
|
|
164
|
+
*/
|
|
165
|
+
function writeGateLog(deps, batchId, capabilityId, issue, outputTail) {
|
|
166
|
+
if (!outputTail)
|
|
167
|
+
return;
|
|
168
|
+
try {
|
|
169
|
+
const logPath = (0, dispatch_1.batchGateLogPath)(deps.store.runsDir, batchId, capabilityId, issue);
|
|
170
|
+
fs.mkdirSync(path.dirname(logPath), { recursive: true, mode: 0o700 });
|
|
171
|
+
fs.writeFileSync(logPath, outputTail, { mode: 0o600 });
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
// diagnostic only — never fail the gate decision over a log write
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Persist the incremental gate's most recent verdict for a member on
|
|
179
|
+
* `BatchEntry.member_gates` (#583 AC4) — shared by the live gate
|
|
180
|
+
* (`reconcileMemberSlot`) and `sched resume --batch`'s recheck
|
|
181
|
+
* (`resumeBlockedGate`), so a resumed batch's status never shows a stale
|
|
182
|
+
* outcome from before the recheck ran.
|
|
183
|
+
*/
|
|
184
|
+
function recordMemberGate(deps, batchId, memberIssue, gate, now) {
|
|
185
|
+
deps.store.withLock((s) => {
|
|
186
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
187
|
+
return {
|
|
188
|
+
state: (0, state_1.patchBatch)(s, batchId, {
|
|
189
|
+
member_gates: {
|
|
190
|
+
...(b?.member_gates ?? {}),
|
|
191
|
+
[String(memberIssue)]: {
|
|
192
|
+
capability: gate.id,
|
|
193
|
+
outcome: gate.outcome,
|
|
194
|
+
output_tail: gate.outputTail ?? null,
|
|
195
|
+
at: now.toISOString(),
|
|
196
|
+
},
|
|
197
|
+
},
|
|
198
|
+
}, now),
|
|
199
|
+
result: undefined,
|
|
200
|
+
};
|
|
201
|
+
});
|
|
202
|
+
}
|
|
143
203
|
function slotFor(state, batchId) {
|
|
144
204
|
return state.slots.find((s) => s.unit === unit(batchId));
|
|
145
205
|
}
|
|
@@ -288,7 +348,7 @@ poolNote = '') {
|
|
|
288
348
|
}), now);
|
|
289
349
|
return { ok: true };
|
|
290
350
|
};
|
|
291
|
-
const capOutcome = deps.runCapability?.(worktree, 'worktree.prepare');
|
|
351
|
+
const capOutcome = deps.runCapability?.(worktree, 'worktree.prepare')?.outcome;
|
|
292
352
|
if (capOutcome === 'ok')
|
|
293
353
|
return done('cap:worktree.prepare');
|
|
294
354
|
if (capOutcome === 'task-failed')
|
|
@@ -932,6 +992,187 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
932
992
|
}
|
|
933
993
|
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
934
994
|
}
|
|
995
|
+
/**
|
|
996
|
+
* The incremental gate (#523 AC2, revised #583): typecheck + focused tests
|
|
997
|
+
* via `cap run`, when the repo has a manifest for them — a second,
|
|
998
|
+
* independent check that the member's own self-reported "done" is real,
|
|
999
|
+
* matching this codebase's "never trust a claimed completion" ethos
|
|
1000
|
+
* (AC2/#464's `isVerifiedComplete`). Three-way policy: `task-failed` evicts
|
|
1001
|
+
* the member directly, same rail as a self-reported block (RFC F.1) — no
|
|
1002
|
+
* aggregate suite has run yet, so there is nothing to attribute.
|
|
1003
|
+
* `automation-broken`/`capability-unavailable` — the gate itself could not
|
|
1004
|
+
* reach a verdict — BLOCK the batch instead of silently proceeding (#583: a
|
|
1005
|
+
* script that legitimately could not run its suite must not be read as
|
|
1006
|
+
* either a pass or a real failure). Only both `ok` falls through.
|
|
1007
|
+
*
|
|
1008
|
+
* Returns `true` when the gate already decided the member's fate (evicted or
|
|
1009
|
+
* blocked, both of which `return` from the caller); `false` when no hook is
|
|
1010
|
+
* configured or both checks came back `ok`, meaning the caller should treat
|
|
1011
|
+
* the member as genuinely complete.
|
|
1012
|
+
*/
|
|
1013
|
+
function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
|
|
1014
|
+
if (batch.worktree === null || !deps.runCapability)
|
|
1015
|
+
return false;
|
|
1016
|
+
const worktree = batch.worktree;
|
|
1017
|
+
const runCapability = deps.runCapability;
|
|
1018
|
+
const gateResults = ['typecheck.run', 'test.focused'].map((id) => ({
|
|
1019
|
+
id,
|
|
1020
|
+
...runCapability(worktree, id),
|
|
1021
|
+
}));
|
|
1022
|
+
const gateFailure = gateResults.find((r) => r.outcome === 'task-failed');
|
|
1023
|
+
const gateInconclusive = gateResults.find((r) => r.outcome === 'automation-broken' || r.outcome === 'capability-unavailable');
|
|
1024
|
+
const worstGate = gateFailure ?? gateInconclusive;
|
|
1025
|
+
if (worstGate) {
|
|
1026
|
+
recordMemberGate(deps, batchId, memberIssue, worstGate, now);
|
|
1027
|
+
}
|
|
1028
|
+
if (gateFailure) {
|
|
1029
|
+
const reason = `incremental-gate-failed:${gateFailure.id}`;
|
|
1030
|
+
writeGateLog(deps, batchId, gateFailure.id, memberIssue, gateFailure.outputTail);
|
|
1031
|
+
const excerpt = gateDetailExcerpt(gateFailure.outputTail, gateFailure.reason);
|
|
1032
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1033
|
+
issue: memberIssue,
|
|
1034
|
+
reason,
|
|
1035
|
+
detail: excerpt
|
|
1036
|
+
? `cap run ${gateFailure.id} reported task-failed after member review done: ${excerpt}`
|
|
1037
|
+
: `cap run ${gateFailure.id} reported task-failed after member review done`,
|
|
1038
|
+
});
|
|
1039
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1040
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, reason, now, result);
|
|
1041
|
+
return true;
|
|
1042
|
+
}
|
|
1043
|
+
if (gateInconclusive) {
|
|
1044
|
+
const reason = `gate-inconclusive:${gateInconclusive.id}`;
|
|
1045
|
+
writeGateLog(deps, batchId, gateInconclusive.id, memberIssue, gateInconclusive.outputTail);
|
|
1046
|
+
const excerpt = gateDetailExcerpt(gateInconclusive.outputTail, gateInconclusive.reason);
|
|
1047
|
+
journalEvent(deps, 'gate-inconclusive', unit(batchId), {
|
|
1048
|
+
issue: memberIssue,
|
|
1049
|
+
reason,
|
|
1050
|
+
detail: excerpt
|
|
1051
|
+
? `cap run ${gateInconclusive.id} reported ${gateInconclusive.outcome} after member review done: ${excerpt}`
|
|
1052
|
+
: `cap run ${gateInconclusive.id} reported ${gateInconclusive.outcome} after member review done`,
|
|
1053
|
+
});
|
|
1054
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1055
|
+
const stateNow = deps.store.load();
|
|
1056
|
+
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
1057
|
+
const blocked = (0, recovery_1.blockBatch)(stateNow, batchId, { reason, milestonePhase: 'batch-review' }, rDeps);
|
|
1058
|
+
deps.store.withLock((s) => ({
|
|
1059
|
+
state: applyBatchAndIssues(s, blocked.state, batchId, []),
|
|
1060
|
+
result: undefined,
|
|
1061
|
+
}));
|
|
1062
|
+
result.failed.push(unit(batchId));
|
|
1063
|
+
return true;
|
|
1064
|
+
}
|
|
1065
|
+
return false;
|
|
1066
|
+
}
|
|
1067
|
+
/**
|
|
1068
|
+
* The member is verifiably done — self-reported complete AND (when checked)
|
|
1069
|
+
* the incremental gate agrees. Recompute the member's commit range, mark it
|
|
1070
|
+
* validated, and advance the batch. Shared by `reconcileMemberSlot`'s
|
|
1071
|
+
* immediate "both gate checks ok" fallthrough and `resumeBlockedGate` (#583)
|
|
1072
|
+
* — a member confirmed complete via a delayed `sched resume --batch` recheck
|
|
1073
|
+
* gets exactly the same treatment as one confirmed complete on the first try.
|
|
1074
|
+
* `releaseSlot` is idempotent (a no-op once the slot is already idle), so
|
|
1075
|
+
* this is safe to call whether or not the caller already released it.
|
|
1076
|
+
*/
|
|
1077
|
+
function completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
|
|
1078
|
+
// The commit-range recompute (`git log`) is a blocking subprocess call — it
|
|
1079
|
+
// must run OUTSIDE the lock, like every other exec in this module; the
|
|
1080
|
+
// result then lands as a pure data patch under the lock (Convention review:
|
|
1081
|
+
// `recordRanges` used to run `git log` INSIDE the withLock mutator, which
|
|
1082
|
+
// is exactly what `engine.ts`'s own "a slow git call never holds the lock"
|
|
1083
|
+
// invariant exists to prevent).
|
|
1084
|
+
const ranges = (0, attribution_1.memberRanges)(boundaryCommits(deps, batch));
|
|
1085
|
+
deps.store.withLock((s) => {
|
|
1086
|
+
let n = releaseSlot(s, batchId, now);
|
|
1087
|
+
n = (0, state_1.patchBatch)(n, batchId, { ranges }, now);
|
|
1088
|
+
n = advanceMemberToValidated(n, memberIssue, now);
|
|
1089
|
+
return { state: n, result: undefined };
|
|
1090
|
+
});
|
|
1091
|
+
result.completed.push(unit(batchId));
|
|
1092
|
+
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
1093
|
+
}
|
|
1094
|
+
/**
|
|
1095
|
+
* `sched resume --batch <id>` (#583 AC4): an operator-triggered, synchronous
|
|
1096
|
+
* one-shot recheck of a batch blocked on `gate-inconclusive:<capabilityId>`
|
|
1097
|
+
* — re-runs exactly that capability against the current member and resolves
|
|
1098
|
+
* the block:
|
|
1099
|
+
*
|
|
1100
|
+
* - still `automation-broken`/`capability-unavailable` → stays `blocked`,
|
|
1101
|
+
* no state change (the capability still isn't fixed).
|
|
1102
|
+
* - `task-failed` → the member really is broken; evict via the same rail
|
|
1103
|
+
* the live gate uses.
|
|
1104
|
+
* - `ok` → the member really was fine; complete it via the same rail the
|
|
1105
|
+
* live gate uses.
|
|
1106
|
+
*
|
|
1107
|
+
* Deliberately NOT a generic engine tick: `runBatchTick`'s own "executing
|
|
1108
|
+
* with no live slot" wedge-recovery path (`spawnMemberContinuation`,
|
|
1109
|
+
* unconditional) exists for a dispatch that never happened — reusing it here
|
|
1110
|
+
* would redispatch a fresh agent for a member whose work is already
|
|
1111
|
+
* committed and reviewed. This function transitions `blocked → executing`
|
|
1112
|
+
* and immediately, synchronously, calls the same completion/eviction
|
|
1113
|
+
* functions the live gate calls — no intervening tick ever sees the batch
|
|
1114
|
+
* `executing` with nothing in flight.
|
|
1115
|
+
*/
|
|
1116
|
+
function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
1117
|
+
const state = deps.store.load();
|
|
1118
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
1119
|
+
if (!batch)
|
|
1120
|
+
throw new types_1.SchedNotFoundError(`Batch not found: ${batchId}`);
|
|
1121
|
+
if (batch.status !== 'blocked' || !batch.blocked_reason?.startsWith('gate-inconclusive:')) {
|
|
1122
|
+
throw new types_1.IllegalTransitionError('batch', batch.status, 'executing');
|
|
1123
|
+
}
|
|
1124
|
+
const capabilityId = batch.blocked_reason.slice('gate-inconclusive:'.length);
|
|
1125
|
+
const memberIssue = batch.members[batch.executing_member - 1];
|
|
1126
|
+
if (batch.worktree === null || !deps.runCapability || memberIssue === undefined) {
|
|
1127
|
+
throw new types_1.SchedNotFoundError(`Batch ${batchId} has no worktree/current member/gate hook to recheck`);
|
|
1128
|
+
}
|
|
1129
|
+
const recheck = deps.runCapability(batch.worktree, capabilityId);
|
|
1130
|
+
const result = emptyResult();
|
|
1131
|
+
const excerpt = gateDetailExcerpt(recheck.outputTail, recheck.reason);
|
|
1132
|
+
recordMemberGate(deps, batchId, memberIssue, { id: capabilityId, ...recheck }, now);
|
|
1133
|
+
if (recheck.outcome === 'automation-broken' || recheck.outcome === 'capability-unavailable') {
|
|
1134
|
+
// Still inconclusive: leave an audit trail (log + journal) exactly like
|
|
1135
|
+
// the live gate does, even though the batch stays `blocked` — otherwise
|
|
1136
|
+
// a repeated `sched resume --batch` leaves no record of when it was last
|
|
1137
|
+
// checked or what it said (#583 review).
|
|
1138
|
+
writeGateLog(deps, batchId, capabilityId, memberIssue, recheck.outputTail);
|
|
1139
|
+
journalEvent(deps, 'gate-inconclusive', unit(batchId), {
|
|
1140
|
+
issue: memberIssue,
|
|
1141
|
+
reason: `gate-inconclusive:${capabilityId}`,
|
|
1142
|
+
detail: excerpt
|
|
1143
|
+
? `sched resume --batch: cap run ${capabilityId} still reports ${recheck.outcome} on recheck: ${excerpt}`
|
|
1144
|
+
: `sched resume --batch: cap run ${capabilityId} still reports ${recheck.outcome} on recheck`,
|
|
1145
|
+
});
|
|
1146
|
+
return {
|
|
1147
|
+
outcome: 'still-blocked',
|
|
1148
|
+
capability: capabilityId,
|
|
1149
|
+
detail: excerpt,
|
|
1150
|
+
};
|
|
1151
|
+
}
|
|
1152
|
+
deps.store.withLock((s) => ({
|
|
1153
|
+
state: (0, state_1.patchBatch)((0, state_1.transitionBatch)(s, batchId, 'executing', {}, now), batchId, { blocked_reason: null }, now),
|
|
1154
|
+
result: undefined,
|
|
1155
|
+
}));
|
|
1156
|
+
if (recheck.outcome === 'task-failed') {
|
|
1157
|
+
const reason = `incremental-gate-failed:${capabilityId}`;
|
|
1158
|
+
writeGateLog(deps, batchId, capabilityId, memberIssue, recheck.outputTail);
|
|
1159
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1160
|
+
issue: memberIssue,
|
|
1161
|
+
reason,
|
|
1162
|
+
detail: excerpt
|
|
1163
|
+
? `sched resume --batch: cap run ${capabilityId} reported task-failed on recheck: ${excerpt}`
|
|
1164
|
+
: `sched resume --batch: cap run ${capabilityId} reported task-failed on recheck`,
|
|
1165
|
+
});
|
|
1166
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, reason, now, result);
|
|
1167
|
+
return { outcome: 'evicted', capability: capabilityId, detail: excerpt };
|
|
1168
|
+
}
|
|
1169
|
+
journalEvent(deps, 'external-advance', unit(batchId), {
|
|
1170
|
+
issue: memberIssue,
|
|
1171
|
+
detail: `sched resume --batch: cap run ${capabilityId} reported ok on recheck`,
|
|
1172
|
+
});
|
|
1173
|
+
completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
1174
|
+
return { outcome: 'completed', capability: capabilityId };
|
|
1175
|
+
}
|
|
935
1176
|
/**
|
|
936
1177
|
* Record one member dispatch's tokens/cost to `runs.jsonl` (#564) — the
|
|
937
1178
|
* `batch-dispatch.ts` analogue of `engine.ts`'s `recordDispatchRunLog`.
|
|
@@ -989,7 +1230,25 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
989
1230
|
const milestone = deps.groundTruth.latestMilestone(memberIssue);
|
|
990
1231
|
if (milestone === undefined)
|
|
991
1232
|
return; // unreachable — pause this batch's decisions
|
|
992
|
-
|
|
1233
|
+
// #575: fence to THIS member dispatch's `spawned_at` — a member re-added to
|
|
1234
|
+
// a fresh batch run after a PREVIOUS batch already posted its
|
|
1235
|
+
// `review done mode=slot` milestone (pilot re-run, requeue-with-context)
|
|
1236
|
+
// must not read as instantly complete against that stale milestone. Mirrors
|
|
1237
|
+
// the per-issue fence in `engine.ts`'s `reconcileRunning`/
|
|
1238
|
+
// `completeUnitOrRecover` — same bug class, same fix, different completion
|
|
1239
|
+
// predicate (`isMemberComplete` vs `isVerifiedComplete`).
|
|
1240
|
+
if (milestone !== null &&
|
|
1241
|
+
!(0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at) &&
|
|
1242
|
+
(0, groundtruth_1.isMemberComplete)(milestone)) {
|
|
1243
|
+
journalEvent(deps, 'stale-milestone-ignored', unit(batchId), {
|
|
1244
|
+
issue: memberIssue,
|
|
1245
|
+
slot: slot.id,
|
|
1246
|
+
run: milestone.run,
|
|
1247
|
+
at: milestone.at,
|
|
1248
|
+
detail: `predates dispatch spawned_at=${slot.spawned_at}`,
|
|
1249
|
+
});
|
|
1250
|
+
}
|
|
1251
|
+
if ((0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at)) {
|
|
993
1252
|
deps.journal.append((0, journal_1.unitEvent)('external-advance', unit(batchId), {
|
|
994
1253
|
issue: memberIssue,
|
|
995
1254
|
detail: 'member review done',
|
|
@@ -998,46 +1257,15 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
998
1257
|
// incremental gate below decides — record its telemetry once here (#564)
|
|
999
1258
|
// rather than at each of this branch's two later exit points.
|
|
1000
1259
|
recordMemberRunLog(deps, dispatch, state0, batchId, batch, memberIssue, slot, now);
|
|
1001
|
-
// Incremental gate (#523 AC2
|
|
1002
|
-
//
|
|
1003
|
-
//
|
|
1004
|
-
//
|
|
1005
|
-
//
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
if (batch.worktree !== null && deps.runCapability) {
|
|
1009
|
-
const gateFailure = ['typecheck.run', 'test.focused']
|
|
1010
|
-
.map((id) => ({ id, outcome: deps.runCapability?.(batch.worktree, id) }))
|
|
1011
|
-
.find((r) => r.outcome === 'task-failed');
|
|
1012
|
-
if (gateFailure) {
|
|
1013
|
-
const reason = `incremental-gate-failed:${gateFailure.id}`;
|
|
1014
|
-
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1015
|
-
issue: memberIssue,
|
|
1016
|
-
reason,
|
|
1017
|
-
detail: `cap run ${gateFailure.id} reported task-failed after member review done`,
|
|
1018
|
-
});
|
|
1019
|
-
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1020
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, reason, now, result);
|
|
1021
|
-
return;
|
|
1022
|
-
}
|
|
1023
|
-
// `ok` / `automation-broken` / `capability-unavailable` all proceed —
|
|
1024
|
-
// only a definite task failure blocks a member here.
|
|
1260
|
+
// Incremental gate (#523 AC2, revised #583) — see `runIncrementalGate`'s
|
|
1261
|
+
// own doc comment for the three-way policy. A `true` return means the
|
|
1262
|
+
// gate already decided the member's fate (evicted or blocked) and
|
|
1263
|
+
// returned early; `false` means both checks were `ok` and the member is
|
|
1264
|
+
// genuinely done.
|
|
1265
|
+
if (runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result)) {
|
|
1266
|
+
return;
|
|
1025
1267
|
}
|
|
1026
|
-
|
|
1027
|
-
// it must run OUTSIDE the lock, like every other exec in this module;
|
|
1028
|
-
// the result then lands as a pure data patch under the lock (Convention
|
|
1029
|
-
// review: `recordRanges` used to run `git log` INSIDE the withLock
|
|
1030
|
-
// mutator, which is exactly what `engine.ts`'s own "a slow git call never
|
|
1031
|
-
// holds the lock" invariant exists to prevent).
|
|
1032
|
-
const ranges = (0, attribution_1.memberRanges)(boundaryCommits(deps, batch));
|
|
1033
|
-
deps.store.withLock((s) => {
|
|
1034
|
-
let n = releaseSlot(s, batchId, now);
|
|
1035
|
-
n = (0, state_1.patchBatch)(n, batchId, { ranges }, now);
|
|
1036
|
-
n = advanceMemberToValidated(n, memberIssue, now);
|
|
1037
|
-
return { state: n, result: undefined };
|
|
1038
|
-
});
|
|
1039
|
-
result.completed.push(unit(batchId));
|
|
1040
|
-
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
1268
|
+
completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
1041
1269
|
return;
|
|
1042
1270
|
}
|
|
1043
1271
|
if ((0, groundtruth_1.isMemberBlocked)(milestone) || dead) {
|