@ai-dossier/sched 0.38.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -125,6 +125,8 @@ export interface BatchTickResult {
125
125
  parked: string[];
126
126
  mergeAccepted: string[];
127
127
  failed: string[];
128
+ /** Per-batch exceptions contained so unrelated units can continue this tick (#635). */
129
+ reconciliation_errors: string[];
128
130
  /** Issue numbers requeued full-cycle by a dissolve — matches `TickResult.blocked`'s shape. */
129
131
  blocked: number[];
130
132
  }
@@ -1 +1 @@
1
- {"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AAmBH,OAAO,EAYL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,KAAK,WAAW,EAQjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAUL,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAoBpB,OAAO,EAAE,KAAK,QAAQ,EAA2D,MAAM,YAAY,CAAC;AACpG,OAAO,KAAK,EAEV,UAAU,EAEV,oBAAoB,EAMpB,WAAW,EAGZ,MAAM,SAAS,CAAC;AAYjB,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sGAAsG;IACtG,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,WAAW,CAAC;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAugBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3F;AA0uCD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,GACtB,IAAI,CA+BN;AAiSD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IACD,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,cAAc,GAAG,qBAAqB,CAAC;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CA2KA;AA+gCD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CAgOjB"}
1
+ {"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AAmBH,OAAO,EAYL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,KAAK,WAAW,EAQjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAUL,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAoBpB,OAAO,EAAE,KAAK,QAAQ,EAA2D,MAAM,YAAY,CAAC;AACpG,OAAO,KAAK,EAEV,UAAU,EAEV,oBAAoB,EAMpB,WAAW,EAGZ,MAAM,SAAS,CAAC;AAYjB,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sGAAsG;IACtG,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,WAAW,CAAC;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,uFAAuF;IACvF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AA0hBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3F;AAswCD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,GACtB,IAAI,CA+BN;AAiSD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IACD,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,cAAc,GAAG,qBAAqB,CAAC;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CA2KA;AA+gCD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CA+PjB"}
@@ -116,7 +116,15 @@ const state_1 = require("./state");
116
116
  const teardown_1 = require("./teardown");
117
117
  const types_1 = require("./types");
118
118
  function emptyResult() {
119
- return { spawned: [], completed: [], parked: [], mergeAccepted: [], failed: [], blocked: [] };
119
+ return {
120
+ spawned: [],
121
+ completed: [],
122
+ parked: [],
123
+ mergeAccepted: [],
124
+ failed: [],
125
+ reconciliation_errors: [],
126
+ blocked: [],
127
+ };
120
128
  }
121
129
  function unit(batchId) {
122
130
  return `batch:${batchId}`;
@@ -309,9 +317,21 @@ function slotFor(state, batchId) {
309
317
  * cross-process lock — survives, where blindly returning `computed` wholesale
310
318
  * would have silently clobbered it.
311
319
  */
312
- function applyBatchAndIssues(fresh, computed, batchId, issues) {
320
+ function applyBatchAndIssues(fresh, computed, batchId, issues, membersBeforeRecovery) {
313
321
  const updatedBatch = computed.batches.find((b) => b.id === batchId);
314
- const batches = fresh.batches.map((b) => (b.id === batchId && updatedBatch ? updatedBatch : b));
322
+ const batches = fresh.batches.map((b) => {
323
+ if (b.id !== batchId || !updatedBatch)
324
+ return b;
325
+ if (!membersBeforeRecovery)
326
+ return updatedBatch;
327
+ // Recovery runs outside the lock. Retain members admitted while its suite
328
+ // or remediation work was running rather than overwriting them with the
329
+ // recovery snapshot captured before admission.
330
+ const admitted = b.members.filter((member) => !membersBeforeRecovery.includes(member));
331
+ return admitted.length === 0
332
+ ? updatedBatch
333
+ : { ...updatedBatch, members: [...updatedBatch.members, ...admitted] };
334
+ });
315
335
  // A `halved` dissolve creates new batch ids (`<id>-a`/`<id>-b`) that exist
316
336
  // in `computed` but not yet in `fresh`.
317
337
  for (const cb of computed.batches) {
@@ -1330,12 +1350,22 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
1330
1350
  poster(batch.anchor, batch.run_id, { phase: 'batch-validate', status: 'done', kv: {} });
1331
1351
  }
1332
1352
  deps.journal.append((0, journal_1.unitEvent)('verify-complete', unit(batchId), { detail: suite.detail ?? 'suite green' }), now);
1333
- deps.store.withLock((s) => {
1353
+ const admitted = deps.store.withLock((s) => {
1334
1354
  const b = (0, state_1.findBatch)(s, batchId);
1335
1355
  if (!b || b.status !== 'validating')
1336
- return { state: s, result: undefined };
1356
+ return { state: s, result: false };
1357
+ if (b.executing_member < b.members.length) {
1358
+ return {
1359
+ state: (0, state_1.transitionBatch)(s, batchId, 'executing', { executing_member: b.executing_member + 1 }, now),
1360
+ result: true,
1361
+ };
1362
+ }
1337
1363
  return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), result: undefined };
1338
1364
  });
1365
+ if (admitted) {
1366
+ spawnMemberContinuation(deps, config, dispatch, batchId, now, result);
1367
+ return;
1368
+ }
1339
1369
  spawnTailAgent(deps, config, dispatch, batchId, now, result);
1340
1370
  return;
1341
1371
  }
@@ -1352,7 +1382,7 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
1352
1382
  }), now);
1353
1383
  const blocked = (0, recovery_1.blockBatch)(state, batchId, { reason: 'suite-unreadable' }, rDeps);
1354
1384
  deps.store.withLock((s) => ({
1355
- state: applyBatchAndIssues(s, blocked.state, batchId, []),
1385
+ state: applyBatchAndIssues(s, blocked.state, batchId, [], batch.members),
1356
1386
  result: undefined,
1357
1387
  }));
1358
1388
  // `result.blocked` is "issue numbers requeued full-cycle by a dissolve"
@@ -1365,7 +1395,7 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
1365
1395
  if (outcome.offenders.length === 0) {
1366
1396
  const dissolve = (0, recovery_1.dissolveBatch)(attributed, batchId, { strategy: 'full', reason: 'unattributable-suite-failure' }, rDeps);
1367
1397
  deps.store.withLock((s) => ({
1368
- state: applyBatchAndIssues(s, dissolve.state, batchId, dissolve.requeued),
1398
+ state: applyBatchAndIssues(s, dissolve.state, batchId, dissolve.requeued, batch.members),
1369
1399
  result: undefined,
1370
1400
  }));
1371
1401
  teardownBatch(deps, batchId);
@@ -1379,7 +1409,7 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
1379
1409
  // fix agent rides the same family the members ran on.
1380
1410
  { config, tests: outcome.attributed.get(offender) ?? [], dispatch });
1381
1411
  deps.store.withLock((s) => ({
1382
- state: applyBatchAndIssues(s, fixing, batchId, []),
1412
+ state: applyBatchAndIssues(s, fixing, batchId, [], batch.members),
1383
1413
  result: undefined,
1384
1414
  }));
1385
1415
  if (fixDispatch === null) {
@@ -1430,7 +1460,7 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
1430
1460
  const rDeps = recoveryDeps(deps, config, batch, now);
1431
1461
  const outcome = (0, recovery_1.evictMembers)(state, batchId, { issues: [offender], reason: 'suite-red-after-fix', attribution, ranges: batch.ranges }, rDeps);
1432
1462
  deps.store.withLock((s) => ({
1433
- state: applyBatchAndIssues(s, outcome.state, batchId, outcome.requeued),
1463
+ state: applyBatchAndIssues(s, outcome.state, batchId, outcome.requeued, batch.members),
1434
1464
  result: undefined,
1435
1465
  }));
1436
1466
  if (outcome.dissolved) {
@@ -1442,6 +1472,12 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
1442
1472
  const b = (0, state_1.findBatch)(s, batchId);
1443
1473
  if (!b || b.status !== 'validating')
1444
1474
  return { state: s, result: undefined };
1475
+ if (b.executing_member < b.members.length) {
1476
+ return {
1477
+ state: (0, state_1.transitionBatch)(s, batchId, 'executing', { executing_member: b.executing_member + 1 }, now),
1478
+ result: undefined,
1479
+ };
1480
+ }
1445
1481
  return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), result: undefined };
1446
1482
  });
1447
1483
  }
@@ -2874,6 +2910,22 @@ function runBatchTick(deps, config, dispatch) {
2874
2910
  }
2875
2911
  }
2876
2912
  };
2913
+ const containFailure = (batchId, err) => {
2914
+ const detail = `${err.name}: ${err.message}`;
2915
+ // A corrupt unit must not turn into a project-wide scheduler outage.
2916
+ // Keep the failed batch inspectable, release its slot, and continue.
2917
+ deps.store.withLock((state) => {
2918
+ const fresh = (0, state_1.findBatch)(state, batchId);
2919
+ if (!fresh || types_1.TERMINAL_BATCH_STATUSES.has(fresh.status) || fresh.status === 'blocked') {
2920
+ return { state, result: undefined };
2921
+ }
2922
+ const blocked = (0, state_1.transitionBatch)(state, batchId, 'blocked', { blocked_reason: 'reconcile-error' }, now);
2923
+ return { state: (0, state_1.releaseBatchSlot)(blocked, batchId, now), result: undefined };
2924
+ });
2925
+ journalEvent(deps, 'unit-failed', unit(batchId), { reason: 'reconcile-error', detail });
2926
+ result.failed.push(unit(batchId));
2927
+ result.reconciliation_errors.push(`${unit(batchId)}: ${detail}`);
2928
+ };
2877
2929
  // #565: when more ready batches exist than free capacity, claim them in
2878
2930
  // the same priority order `runnableUnits` would (desc priority → asc
2879
2931
  // readiness age → anchor) — this loop never goes through
@@ -2905,11 +2957,16 @@ function runBatchTick(deps, config, dispatch) {
2905
2957
  const state = deps.store.load();
2906
2958
  const batch = (0, state_1.findBatch)(state, batchId);
2907
2959
  if (batch && batch.status === 'ready' && slotFor(state, batchId) === undefined) {
2908
- // #707: batch-setup dispatches through the batch's own profile too — it
2909
- // is the batch's first agent, not an engine-internal step.
2910
- const batchDispatch = dispatchFor(batch);
2911
- if (batchDispatch !== null) {
2912
- claimAndSetup(deps, config, batchDispatch, batchId, now, result);
2960
+ try {
2961
+ // #707: batch-setup dispatches through the batch's own profile too — it
2962
+ // is the batch's first agent, not an engine-internal step.
2963
+ const batchDispatch = dispatchFor(batch);
2964
+ if (batchDispatch !== null) {
2965
+ claimAndSetup(deps, config, batchDispatch, batchId, now, result);
2966
+ }
2967
+ }
2968
+ catch (err) {
2969
+ containFailure(batchId, err);
2913
2970
  }
2914
2971
  }
2915
2972
  }
@@ -2938,78 +2995,83 @@ function runBatchTick(deps, config, dispatch) {
2938
2995
  }
2939
2996
  continue;
2940
2997
  }
2941
- // #707: resolve this batch's profile once per pass — or dissolve/fall
2942
- // back loudly when it no longer resolves (see `dispatchFor` above).
2943
- // Terminal batches are skipped before this call so a removed profile does
2944
- // not generate a missing-profile event on every later tick.
2945
- const batchDispatch = dispatchFor(batch);
2946
- if (batchDispatch === null)
2947
- continue;
2948
- if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
2949
- if (batch.status === 'executing') {
2950
- reconcileMemberSlot(deps, config, batchDispatch, batch.id, slot, now, result);
2998
+ try {
2999
+ // #707: resolve this batch's profile once per pass — or dissolve/fall
3000
+ // back loudly when it no longer resolves (see `dispatchFor` above).
3001
+ // Terminal batches are skipped before this call so a removed profile does
3002
+ // not generate a missing-profile event on every later tick.
3003
+ const batchDispatch = dispatchFor(batch);
3004
+ if (batchDispatch === null)
3005
+ continue;
3006
+ if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
3007
+ if (batch.status === 'executing') {
3008
+ reconcileMemberSlot(deps, config, batchDispatch, batch.id, slot, now, result);
3009
+ }
3010
+ else if (batch.status === 'fixing') {
3011
+ reconcileFixSlot(deps, config, batch.id, slot, now, result);
3012
+ }
3013
+ else if (batch.status === 'reviewing' || batch.status === 'shipping') {
3014
+ reconcileTailSlot(deps, batch.id, slot, now, result);
3015
+ }
3016
+ else if (batch.status === 'deployed') {
3017
+ reconcileReportSlot(deps, batch.id, slot, now, result);
3018
+ }
3019
+ continue;
2951
3020
  }
2952
- else if (batch.status === 'fixing') {
2953
- reconcileFixSlot(deps, config, batch.id, slot, now, result);
3021
+ if (slot)
3022
+ continue; // live but neither running/assigned (e.g. mid-verify) — next tick
3023
+ if (batch.status === 'validating') {
3024
+ // A local suite run, not a provider dispatch — unaffected by `paused`.
3025
+ runValidate(deps, config, batchDispatch, batch.id, now, result);
3026
+ continue;
3027
+ }
3028
+ // #629: every branch below spawns a provider agent — hold the wedge
3029
+ // until `sched resume` (see the top of this function for why `paused`
3030
+ // is read once, up front, rather than per iteration).
3031
+ if (paused)
3032
+ continue;
3033
+ if (batch.status === 'deployed') {
3034
+ spawnReportAgent(deps, config, batchDispatch, batch.id, now, result);
3035
+ }
3036
+ else if (batch.status === 'executing') {
3037
+ // A prior spawn threw, or `claimAndSpawn` found zero free capacity —
3038
+ // either way the batch is stuck mid-member with no slot and nothing
3039
+ // else will ever retry it (Conformance review AC5 caveat; Supportability
3040
+ // review #12). Retrying every tick is cheap-by-contract (#677 review):
3041
+ // `spawnMemberContinuation` runs the free-capacity and member-identity
3042
+ // pre-checks BEFORE any pool/git work, so a full scheduler (or stale
3043
+ // member context) exits here without exec'ing, and `claimAndSpawn`
3044
+ // remains the authoritative capacity gate.
3045
+ spawnMemberContinuation(deps, config, batchDispatch, batch.id, now, result);
2954
3046
  }
2955
3047
  else if (batch.status === 'reviewing' || batch.status === 'shipping') {
2956
- reconcileTailSlot(deps, batch.id, slot, now, result);
3048
+ // Same wedge, for a dead-or-never-claimed tail agent.
3049
+ spawnTailAgent(deps, config, batchDispatch, batch.id, now, result);
2957
3050
  }
2958
- else if (batch.status === 'deployed') {
2959
- reconcileReportSlot(deps, batch.id, slot, now, result);
3051
+ else if (batch.status === 'fixing') {
3052
+ // `beginFixAttempt` already recorded this member's ONE attempt as
3053
+ // `dispatched` before `claimAndSpawn` could find capacity — retrying the
3054
+ // exact same dispatch isn't reconstructible from persisted state (only
3055
+ // the outcome is persisted, not the command/prompt), so resolve it
3056
+ // `red` (conservatively: the member loses its one attempt and evicts on
3057
+ // the next validate pass, which is safe — never a permanent wedge).
3058
+ const state = deps.store.load();
3059
+ const b = (0, state_1.findBatch)(state, batch.id);
3060
+ const offenderRecord = b
3061
+ ? [...b.fix_attempts].reverse().find((a) => a.outcome === 'dispatched')
3062
+ : undefined;
3063
+ if (b && offenderRecord) {
3064
+ const rDeps = recoveryDeps(deps, config, b, now);
3065
+ const { state: resolved } = (0, recovery_1.resolveFixAttempt)(state, batch.id, offenderRecord.issue, 'red', rDeps);
3066
+ deps.store.withLock((s) => ({
3067
+ state: applyBatchAndIssues(s, resolved, batch.id, []),
3068
+ result: undefined,
3069
+ }));
3070
+ }
2960
3071
  }
2961
- continue;
2962
3072
  }
2963
- if (slot)
2964
- continue; // live but neither running/assigned (e.g. mid-verify) — next tick
2965
- if (batch.status === 'validating') {
2966
- // A local suite run, not a provider dispatch — unaffected by `paused`.
2967
- runValidate(deps, config, batchDispatch, batch.id, now, result);
2968
- continue;
2969
- }
2970
- // #629: every branch below spawns a provider agent — hold the wedge
2971
- // until `sched resume` (see the top of this function for why `paused`
2972
- // is read once, up front, rather than per iteration).
2973
- if (paused)
2974
- continue;
2975
- if (batch.status === 'deployed') {
2976
- spawnReportAgent(deps, config, batchDispatch, batch.id, now, result);
2977
- }
2978
- else if (batch.status === 'executing') {
2979
- // A prior spawn threw, or `claimAndSpawn` found zero free capacity —
2980
- // either way the batch is stuck mid-member with no slot and nothing
2981
- // else will ever retry it (Conformance review AC5 caveat; Supportability
2982
- // review #12). Retrying every tick is cheap-by-contract (#677 review):
2983
- // `spawnMemberContinuation` runs the free-capacity and member-identity
2984
- // pre-checks BEFORE any pool/git work, so a full scheduler (or stale
2985
- // member context) exits here without exec'ing, and `claimAndSpawn`
2986
- // remains the authoritative capacity gate.
2987
- spawnMemberContinuation(deps, config, batchDispatch, batch.id, now, result);
2988
- }
2989
- else if (batch.status === 'reviewing' || batch.status === 'shipping') {
2990
- // Same wedge, for a dead-or-never-claimed tail agent.
2991
- spawnTailAgent(deps, config, batchDispatch, batch.id, now, result);
2992
- }
2993
- else if (batch.status === 'fixing') {
2994
- // `beginFixAttempt` already recorded this member's ONE attempt as
2995
- // `dispatched` before `claimAndSpawn` could find capacity — retrying the
2996
- // exact same dispatch isn't reconstructible from persisted state (only
2997
- // the outcome is persisted, not the command/prompt), so resolve it
2998
- // `red` (conservatively: the member loses its one attempt and evicts on
2999
- // the next validate pass, which is safe — never a permanent wedge).
3000
- const state = deps.store.load();
3001
- const b = (0, state_1.findBatch)(state, batch.id);
3002
- const offenderRecord = b
3003
- ? [...b.fix_attempts].reverse().find((a) => a.outcome === 'dispatched')
3004
- : undefined;
3005
- if (b && offenderRecord) {
3006
- const rDeps = recoveryDeps(deps, config, b, now);
3007
- const { state: resolved } = (0, recovery_1.resolveFixAttempt)(state, batch.id, offenderRecord.issue, 'red', rDeps);
3008
- deps.store.withLock((s) => ({
3009
- state: applyBatchAndIssues(s, resolved, batch.id, []),
3010
- result: undefined,
3011
- }));
3012
- }
3073
+ catch (err) {
3074
+ containFailure(batch.id, err);
3013
3075
  }
3014
3076
  }
3015
3077
  reconcilePrWatch(deps, now, result);