@ctrl-spc/cs 0.7.14 → 0.7.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,14 +1,17 @@
1
+ import { executionIdentity } from './agents.js';
2
+ import { CodexHomeFailure } from './codex-home.js';
1
3
  import { execFile, spawn as spawnChild } from 'node:child_process';
2
4
  import { randomUUID } from 'node:crypto';
5
+ import { beginOwnedClaim, ownedWorkAllowed, hasOwnedWorkContext, pendingTerminalOutcomes, acknowledgeTerminalOutcome, updateTerminalOutcome, ownedWorkerClaimedAt, ownedWorkerExecutionHeld, reserveOwnedWork, legacyHeldTodoIds, interruptionReceipts, acknowledgeInterruption, legacySnapshotNeeded, saveLegacySnapshot, legacyTodoHolds, releaseLegacyTodoHold, recordExitedInterruption, beginRpcClaim, deferRpcClaim, pendingRpcClaims, recordRpcClaims, finishRpcClaims, recoveredRpcClaims, pendingRpcReference, retireRpcReference } from './daemon-processes.js';
3
6
  import { promisify } from 'node:util';
4
7
  import { agentPath, detectAgents } from './agents.js';
5
8
  /* 18a Slice 7: the two Windows spawn rules, in one module so a caller cannot
6
9
  take the shell and forget the quoting. `quoteForCmd` is re-exported because
7
10
  this module's own tests reach for it here. */
8
- import { killTree, needsShell, processIsAlive, quoteForCmd } from './win-shell.js';
11
+ import { killTree, needsShell, processIsAlive, quoteForCmd, spawnOwnedProcess, releaseOwnedProcess } from './win-shell.js';
9
12
  export { quoteForCmd } from './win-shell.js';
10
13
  import { ORCHESTRATOR_MAX_CONCURRENT, ORCHESTRATOR_SPAWN_TIMEOUT_MS } from './env.js';
11
- import { agentDisplayName, plainFailureReason } from './failure-reason.js';
14
+ import { agentDisplayName, nativeFailureKind, failureMessage } from './failure-reason.js';
12
15
  /* 18c SLICE 9 (GAP 15): the two request-scoped releases. They live in mcp.ts
13
16
  beside the session-end routes and the reservations table's other writers, so
14
17
  there is ONE place that knows how a lease is let go, rather than two that can
@@ -199,7 +202,7 @@ export async function claimNextTodo(client, machineId, agent,
199
202
  because the claim must stay ONE statement (see below), and because a
200
203
  blocked unit is the OLDEST claimable row: released and re-claimed it would
201
204
  be picked first every tick, starving everything behind it. */
202
- skipIds = []) {
205
+ skipIds = [], recoveryOnly = false) {
203
206
  /* THROUGH AN RPC, NOT PostgREST — and this is not a style preference, it is
204
207
  the fix for a bug that made the whole slice inert.
205
208
 
@@ -224,11 +227,11 @@ skipIds = []) {
224
227
  row rather than blocking or double-claiming. `userId` is no longer passed:
225
228
  the function scopes itself with `auth.uid()`, which is the same identity
226
229
  RLS uses and cannot be spoofed by a caller. */
227
- const { data, error } = await client.rpc('cliv2_claim_next_loose_todo', {
230
+ const { data, error, operationId } = await cliClaim(client, 'cliv2_claim_next_loose_todo', {
228
231
  p_machine_id: machineId,
229
232
  p_agent: agent,
230
233
  p_skip_ids: skipIds,
231
- });
234
+ }, recoveryOnly);
232
235
  if (error)
233
236
  throw error;
234
237
  /* 18b Slice 2: the claim also carries the request's OWN project (id and name),
@@ -240,40 +243,12 @@ skipIds = []) {
240
243
  const row = (data ?? [])[0];
241
244
  if (!row)
242
245
  return null;
243
- /* ═══ 18d SLICE 4 — THE FENCE TOKEN FOR THIS DISPATCH. ═══
244
-
245
- `claimed_at` is what this run OWNS. It is read here, immediately after the
246
- claim, rather than added to the RPC's return table, because that keeps the
247
- shipped function's contract alone and needs no migration.
248
-
249
- WHY IT IS NEEDED. hand_off_stage clears the claim so the next stage is
250
- dispatched to a fresh worker, and the handing-off worker then keeps talking
251
- for as long as it takes to write its closing answer. Its exit path wrote
252
- `state='done'` unconditionally, over a row a SECOND worker was already
253
- working: the card read Done carrying stage 1's answer, and because the claim
254
- RPC skips `done` rows the request could never be claimed again, so the
255
- remaining stages were never run. Found by the gate's audit, and it matches
256
- what the walk saw.
257
-
258
- Every terminal write is now fenced on this value, so a run that no longer
259
- holds the claim writes NOTHING. Null is possible only if the row vanished
260
- between claiming and reading it, and an unfenceable dispatch is treated as
261
- already superseded rather than allowed to clobber. */
262
- let claimedAt = null;
263
- try {
264
- const { data: claimRow } = await client
265
- .from('cliv2_loose_todos')
266
- .select('claimed_at')
267
- .eq('id', row.id)
268
- .maybeSingle();
269
- claimedAt = claimRow?.claimed_at ?? null;
270
- }
271
- catch {
272
- /* Unreadable is treated as unfenceable, which only loses the protection —
273
- it never blocks the dispatch that was already claimed successfully. */
274
- }
246
+ const claimedAt = row.claimed_at ?? null;
247
+ if (hasOwnedWorkContext() && !claimedAt)
248
+ throw new Error('The claim did not return an exact attempt identity. Update the database before starting work.');
275
249
  return {
276
250
  id: row.id,
251
+ ...(operationId ? { claimOperationId: operationId } : {}),
277
252
  instruction: row.instruction,
278
253
  /* `?? null` rather than trusting the shape: a daemon running against a
279
254
  database that has not taken this slice's migration yet gets neither
@@ -514,10 +489,13 @@ claimedAt) {
514
489
  const write = client
515
490
  .from('cliv2_loose_todos')
516
491
  .update({ answer, state: 'done' })
517
- .eq('id', todoId);
518
- const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
492
+ .eq('id', todoId)
493
+ .in('state', ['working', 'needs-input']);
494
+ const { data: written, error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write).select('id');
519
495
  if (error)
520
496
  throw error;
497
+ if (!written?.length)
498
+ return false;
521
499
  /* 18c SLICE 1 — CLOSE OUT THE LAST ACTIVITY LINE.
522
500
  *
523
501
  * report_activity only ever flips the PREVIOUS 'doing' row to 'done' when a
@@ -541,6 +519,7 @@ claimedAt) {
541
519
  catch (err) {
542
520
  console.warn(`answerTodo: closing out the last activity line failed: ${err.message}`);
543
521
  }
522
+ return true;
544
523
  }
545
524
  /**
546
525
  * !Cleanup PHASE 6 (I15) — DID THIS WORKER ASK SOMETHING AND EXIT?
@@ -566,7 +545,7 @@ claimedAt) {
566
545
  * a card asking the user for something that does not exist, unclearable except
567
546
  * by Stop. A wrong `done` is recoverable by Retry; a phantom question is not.
568
547
  */
569
- export async function todoHasOpenAsk(client, todoId, warn) {
548
+ export async function todoHasOpenAsk(client, todoId, warn, strict = false) {
570
549
  try {
571
550
  const { data, error } = await client
572
551
  .from('cliv2_todo_asks')
@@ -578,11 +557,13 @@ export async function todoHasOpenAsk(client, todoId, warn) {
578
557
  throw new Error(error.message);
579
558
  if ((data ?? []).length > 0)
580
559
  return true;
581
- if (await todoHasOpenDecision(client, todoId, warn))
560
+ if (await todoHasOpenDecision(client, todoId, warn, strict))
582
561
  return true;
583
- return await todoHasOpenBlockedNotice(client, todoId, warn);
562
+ return await todoHasOpenBlockedNotice(client, todoId, warn, strict);
584
563
  }
585
564
  catch (err) {
565
+ if (strict)
566
+ throw err;
586
567
  warn(`[orchestrator] could not check for open questions: ${err.message}`);
587
568
  return false;
588
569
  }
@@ -606,7 +587,7 @@ export async function todoHasOpenAsk(client, todoId, warn) {
606
587
  * Best-effort and false on error, like its two siblings, and for the same
607
588
  * reason: a wrong `done` is recoverable by Retry, a phantom park is not.
608
589
  */
609
- export async function todoHasOpenBlockedNotice(client, todoId, warn) {
590
+ export async function todoHasOpenBlockedNotice(client, todoId, warn, strict = false) {
610
591
  try {
611
592
  const { data, error } = await client
612
593
  .from('cliv2_blocked_notices')
@@ -619,6 +600,8 @@ export async function todoHasOpenBlockedNotice(client, todoId, warn) {
619
600
  return (data ?? []).length > 0;
620
601
  }
621
602
  catch (err) {
603
+ if (strict)
604
+ throw err;
622
605
  warn(`[orchestrator] could not check for an open stop: ${err.message}`);
623
606
  return false;
624
607
  }
@@ -659,7 +642,7 @@ const PERMISSION_CATEGORY = 'cliv2_permission';
659
642
  * BEST-EFFORT, like every read in the tick. A failure returns false, which costs
660
643
  * a wrongly-finished card rather than a lost run.
661
644
  */
662
- export async function todoHasOpenDecision(client, todoId, warn) {
645
+ export async function todoHasOpenDecision(client, todoId, warn, strict = false) {
663
646
  try {
664
647
  const { data: links, error: linkError } = await client
665
648
  .from('cliv2_loose_todo_links')
@@ -691,6 +674,8 @@ export async function todoHasOpenDecision(client, todoId, warn) {
691
674
  return (data ?? []).length > 0;
692
675
  }
693
676
  catch (err) {
677
+ if (strict)
678
+ throw err;
694
679
  warn(`[orchestrator] could not check the work item's questions: ${err.message}`);
695
680
  return false;
696
681
  }
@@ -803,7 +788,7 @@ export async function todoRunLeftATrace(client, todoId, warn,
803
788
  * same answer for the scope gate and resolving it twice would let the two
804
789
  * disagree about which item this run is for. Null for a panel request with
805
790
  * no work item, which is the greenfield case. */
806
- workItemId = null) {
791
+ workItemId = null, strict = false) {
807
792
  try {
808
793
  /* ANY ONE IS ENOUGH, so the cheapest and most likely is asked first and the
809
794
  rest are skipped once one answers yes. A session is what the guidance
@@ -852,6 +837,8 @@ workItemId = null) {
852
837
  return (dropped.data?.dropped_activity ?? 0) > 0;
853
838
  }
854
839
  catch (err) {
840
+ if (strict)
841
+ throw err;
855
842
  /* See the block comment: towards `done`, so a transient read failure never
856
843
  discards a run that really did the work. */
857
844
  warn(`[orchestrator] could not check whether the run recorded anything: ${err.message}`);
@@ -888,10 +875,12 @@ claimedAt) {
888
875
  // `answer` is constrained nonblank-when-present, so '' must go in as null
889
876
  // rather than as an empty string the check would refuse.
890
877
  .update({ answer: trimmed || null, state: 'needs-input' })
891
- .eq('id', todoId);
892
- const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
878
+ .eq('id', todoId)
879
+ .in('state', ['working', 'needs-input']);
880
+ const { data, error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write).select('id');
893
881
  if (error)
894
882
  throw error;
883
+ return !!data?.length;
895
884
  }
896
885
  /**
897
886
  * !Cleanup PHASE 6 (I17) — HAS THE USER STOPPED THIS?
@@ -934,12 +923,13 @@ export async function todoWasStopped(client, todoId) {
934
923
  * already hit Retry, the unit is `working` again and this matches nothing. That
935
924
  * is the Gherkin line "the daemon does not later overwrite that".
936
925
  */
937
- export async function releaseStoppedTodo(client, todoId) {
938
- const { error } = await client
926
+ export async function releaseStoppedTodo(client, todoId, claimedAt) {
927
+ const write = client
939
928
  .from('cliv2_loose_todos')
940
929
  .update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
941
930
  .eq('id', todoId)
942
931
  .eq('state', 'stopped');
932
+ const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
943
933
  if (error)
944
934
  throw error;
945
935
  }
@@ -972,6 +962,41 @@ export const MAX_ATTEMPTS = 3;
972
962
  * so there is one count: a second read could land after another process had
973
963
  * touched the row, and a wait computed from a stale count is a wait the card's
974
964
  * own countdown would disagree with. */
965
+ /** The existing terminal write, also used for a nonretryable native refusal. */
966
+ async function failClaimedTodo(client, todoId, reason, expectedClaim, count) {
967
+ let attempts = count ?? 0;
968
+ if (count === undefined) {
969
+ const { data, error } = await client.from('cliv2_loose_todos').select('attempts,claimed_at').eq('id', todoId).maybeSingle();
970
+ if (error)
971
+ throw error;
972
+ if (!data || data.claimed_at !== expectedClaim)
973
+ return { fate: 'stale', attempts: data?.attempts ?? 0 };
974
+ attempts = Number(data.attempts ?? 0) + 1;
975
+ }
976
+ let failure = client
977
+ .from('cliv2_loose_todos')
978
+ .update({
979
+ state: 'failed',
980
+ attempts,
981
+ stopped_reason: reason.trim() || 'the work failed repeatedly',
982
+ claimed_machine_id: null,
983
+ claimed_agent: null,
984
+ claimed_at: null,
985
+ /* 18a SLICE 6: cleared at the bound. A wait on a terminal row would be
986
+ read by nothing and would outlive the row's own meaning; the only
987
+ route back is Retry, which means now. */
988
+ retry_after: null,
989
+ })
990
+ .eq('id', todoId)
991
+ .in('state', ['working', 'needs-input']);
992
+ failure = expectedClaim ? failure.eq('claimed_at', expectedClaim) : failure.is('claimed_at', null);
993
+ const { data: failed, error: failError } = await failure.select('id');
994
+ if (failError)
995
+ throw failError;
996
+ if (!failed?.length)
997
+ return { fate: 'stale', attempts: attempts - 1 };
998
+ return { fate: 'failed', attempts };
999
+ }
975
1000
  export async function releaseOrFail(client, todoId, reason,
976
1001
  /** 18a Slice 6: injectable so a test can place the wait exactly. */
977
1002
  nowMs = Date.now(),
@@ -992,34 +1017,17 @@ claimedAt) {
992
1017
  already claimed by the run that replaced this one. */
993
1018
  const current = data;
994
1019
  if (claimedAt && current && current.claimed_at !== claimedAt) {
995
- return { fate: 'released', attempts: current.attempts ?? 0 };
1020
+ return { fate: 'stale', attempts: current.attempts ?? 0 };
996
1021
  }
997
1022
  const attempts = (current?.attempts ?? 0) + 1;
998
- if (attempts >= MAX_ATTEMPTS) {
999
- const { error: failError } = await client
1000
- .from('cliv2_loose_todos')
1001
- .update({
1002
- state: 'failed',
1003
- attempts,
1004
- stopped_reason: reason.trim() || 'the work failed repeatedly',
1005
- claimed_machine_id: null,
1006
- claimed_agent: null,
1007
- claimed_at: null,
1008
- /* 18a SLICE 6: cleared at the bound. A wait on a terminal row would be
1009
- read by nothing and would outlive the row's own meaning; the only
1010
- route back is Retry, which means now. */
1011
- retry_after: null,
1012
- })
1013
- .eq('id', todoId);
1014
- if (failError)
1015
- throw failError;
1016
- return { fate: 'failed', attempts };
1017
- }
1023
+ const expectedClaim = claimedAt ?? current?.claimed_at ?? null;
1024
+ if (attempts >= MAX_ATTEMPTS)
1025
+ return failClaimedTodo(client, todoId, reason, expectedClaim, attempts);
1018
1026
  /* 18a SLICE 6: THE WAIT IS WRITTEN ON THE SAME STATEMENT AS THE COUNT, for
1019
1027
  the reason the count is on the same statement as the release: a wait that
1020
1028
  can fail on its own is a wait the daemon might not honour, and the failure
1021
1029
  mode is the fork bomb this bound exists to prevent. */
1022
- const { error: releaseError } = await client
1030
+ let release = client
1023
1031
  .from('cliv2_loose_todos')
1024
1032
  .update({
1025
1033
  attempts,
@@ -1028,9 +1036,14 @@ claimedAt) {
1028
1036
  claimed_at: null,
1029
1037
  retry_after: new Date(nowMs + retryDelayMs(attempts)).toISOString(),
1030
1038
  })
1031
- .eq('id', todoId);
1039
+ .eq('id', todoId)
1040
+ .in('state', ['working', 'needs-input']);
1041
+ release = expectedClaim ? release.eq('claimed_at', expectedClaim) : release.is('claimed_at', null);
1042
+ const { data: released, error: releaseError } = await release.select('id');
1032
1043
  if (releaseError)
1033
1044
  throw releaseError;
1045
+ if (!released?.length)
1046
+ return { fate: 'stale', attempts: current?.attempts ?? 0 };
1034
1047
  return { fate: 'released', attempts };
1035
1048
  }
1036
1049
  /**
@@ -1172,11 +1185,13 @@ export async function scopeIsApproved(client, workItemId) {
1172
1185
  * See the call site for why the claim stays: an unclaimed todo is immediately
1173
1186
  * re-claimable, so releasing would spin the tick loop.
1174
1187
  */
1175
- export async function parkTodoForScope(client, todoId) {
1176
- const { error } = await client
1188
+ export async function parkTodoForScope(client, todoId, claimedAt) {
1189
+ const write = client
1177
1190
  .from('cliv2_loose_todos')
1178
1191
  .update({ state: 'needs-input' })
1179
- .eq('id', todoId);
1192
+ .eq('id', todoId)
1193
+ .in('state', ['working', 'needs-input']);
1194
+ const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
1180
1195
  if (error)
1181
1196
  throw error;
1182
1197
  }
@@ -1279,7 +1294,7 @@ injectedSpawn,
1279
1294
  * worker's, and injected by the same convention (`deps.resolveContext`) so a
1280
1295
  * test can point it somewhere without `resolveClaimContext` growing
1281
1296
  * parameters for the suite's sake. */
1282
- resolveContext = (claim) => resolveClaimContext(client, claim, warn)) {
1297
+ resolveContext = (claim) => resolveClaimContext(client, claim, warn), observeHarness) {
1283
1298
  /* ═══ 18k SLICE 10b — NO TOOLS MEANS NO ANSWER, the tick's own rule. ═══
1284
1299
  An answerer now holds the worker's toolset, so one started while the tools
1285
1300
  server is down would have NONE of it: it could not read the work item, could
@@ -1319,17 +1334,19 @@ resolveContext = (claim) => resolveClaimContext(client, claim, warn)) {
1319
1334
  returns; what bounds a long answer is `state.answering`, keyed per run, and
1320
1335
  `state.maxAnswering`. This guard is back to meaning what it says: one take
1321
1336
  at a time, which is a few queries. */
1322
- if (state.takeInFlight)
1337
+ if (state.takeInFlight || !ownedWorkAllowed())
1323
1338
  return;
1339
+ const endClaim = beginOwnedClaim();
1324
1340
  state.takeInFlight = true;
1325
1341
  try {
1326
- await takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext);
1342
+ await takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext, observeHarness);
1327
1343
  }
1328
1344
  finally {
1329
1345
  state.takeInFlight = false;
1346
+ endClaim();
1330
1347
  }
1331
1348
  }
1332
- async function takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext) {
1349
+ async function takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext, observeHarness) {
1333
1350
  /* ═══ 18k SLICE 10 — A MACHINE ANSWERS ITS OWN RUNS, DESIGNATED OR NOT. ═══
1334
1351
  Slice 7 gated this whole function on being the designated orchestrator, and
1335
1352
  that is wrong for work already claimed: the user may switch the toolbar to
@@ -1342,6 +1359,7 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
1342
1359
  old check survives on its own. */
1343
1360
  if (agents.length === 0)
1344
1361
  return;
1362
+ const claimOperations = [];
1345
1363
  /* THE RUNS THIS MACHINE CLAIMS, which is what it may answer for. `claimed_at`
1346
1364
  is the real predicate rather than the state alone: `releaseStoppedTodo` and
1347
1365
  `releaseOrFail` null the claim on `stopped` and `failed`, while `answerTodo`
@@ -1400,6 +1418,13 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
1400
1418
  scoped take with an empty id list, which correctly returns nothing — so this
1401
1419
  is a saved round trip every ten seconds on every machine in the fleet that is
1402
1420
  not participating, not a safety property. The safety is the scope itself. */
1421
+ if (hasOwnedWorkContext()) {
1422
+ for (const claim of recoveredRpcClaims('cli').filter(row => row.action !== 'cliv2_claim_next_loose_todo')) {
1423
+ for (const ref of cliClaimReferences(claim.action, claim.result))
1424
+ if (ref.surface === 'reply' && pendingRpcReference(ref) && !state.answering.has(ref.todoId) && !ownTodoIds.includes(ref.todoId))
1425
+ ownTodoIds.push(ref.todoId);
1426
+ }
1427
+ }
1403
1428
  if (ownTodoIds.length === 0 && !designated)
1404
1429
  return;
1405
1430
  try {
@@ -1412,13 +1437,17 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
1412
1437
  which is the whole of the starvation this scoping removes. */
1413
1438
  const taken = [];
1414
1439
  if (ownTodoIds.length > 0) {
1415
- const own = await client.rpc('cliv2_take_run_messages', { p_todo_ids: ownTodoIds, p_machine_id: machineId });
1440
+ const own = await cliClaim(client, 'cliv2_take_run_messages', { p_todo_ids: ownTodoIds, p_machine_id: machineId, p_skip_todo_ids: [...legacyHeldTodoIds(), ...state.answering] });
1441
+ if (own.operationId)
1442
+ claimOperations.push(own.operationId);
1416
1443
  if (own.error)
1417
1444
  throw new Error(own.error.message);
1418
1445
  taken.push(...(own.data ?? []));
1419
1446
  }
1420
1447
  if (designated) {
1421
- const rest = await client.rpc('cliv2_take_run_messages', { p_todo_ids: null, p_machine_id: machineId });
1448
+ const rest = await cliClaim(client, 'cliv2_take_run_messages', { p_todo_ids: null, p_machine_id: machineId, p_skip_todo_ids: [...legacyHeldTodoIds(), ...state.answering] });
1449
+ if (rest.operationId)
1450
+ claimOperations.push(rest.operationId);
1422
1451
  if (rest.error)
1423
1452
  throw new Error(rest.error.message);
1424
1453
  /* DEDUPED BY ID rather than trusted to be disjoint. The database will not
@@ -1450,7 +1479,13 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
1450
1479
  cover: the answer is not to a message, it is to an approval. Everything
1451
1480
  downstream already handles an empty id list — the covering RPC is skipped
1452
1481
  and the lease is not involved, because no message was leased. */
1453
- const granted = await readGrantedRuns(client, userId, warn);
1482
+ const grants = await cliClaim(client, 'cliv2_take_granted_resumes', { p_todo_ids: designated ? null : ownTodoIds,
1483
+ p_machine_id: machineId, p_skip_todo_ids: [...legacyHeldTodoIds(), ...state.answering] });
1484
+ if (grants.operationId)
1485
+ claimOperations.push(grants.operationId);
1486
+ if (grants.error)
1487
+ throw grants.error;
1488
+ const granted = (grants.data ?? []);
1454
1489
  /* AN EMPTY PASS SAYS NOTHING. This runs every 10 seconds and the common case
1455
1490
  is nothing to take and nothing approved; a line each time would bury every
1456
1491
  useful one, the same rule `reportRole` and `reportNoTools` follow.
@@ -1495,7 +1530,8 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
1495
1530
  }
1496
1531
  /* AND THE APPROVED RUNS JOIN THEM, with no message ids: there is nothing to
1497
1532
  cover, because the answer is to an approval rather than to a message. */
1498
- for (const todoId of granted) {
1533
+ for (const grant of granted) {
1534
+ const todoId = grant.todo_id;
1499
1535
  if (!byRun.has(todoId))
1500
1536
  byRun.set(todoId, { messageIds: [], wasExpired: false });
1501
1537
  }
@@ -1553,7 +1589,10 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
1553
1589
  `answerRunMessages` warns and returns false rather than throwing, and
1554
1590
  the pass's job is to have STARTED every answer. The `catch` is the
1555
1591
  daemon's never-throw contract, not a second failure path. */
1556
- void answerRunMessages(client, todoId, group.messageIds, { agent: plan.agent, cwd: context.checkout, workingCopyElsewhere: plan.workingCopyElsewhere }, state, spawn, warn).catch(err => {
1592
+ void answerRunMessages(client, todoId, group.messageIds, { agent: plan.agent, cwd: context.checkout, workingCopyElsewhere: plan.workingCopyElsewhere }, state, spawn, warn, { surface: 'reply', todoId,
1593
+ messageLeases: taken.filter((row) => row.todo_id === todoId).map((row) => ({ id: row.id, taken_at: row.taken_at })),
1594
+ grantLeases: granted.filter((row) => row.todo_id === todoId).map((row) => ({ id: row.id, resume_taken_at: row.resume_taken_at })),
1595
+ observedPendingMessageIds: null }, machineId, undefined, observeHarness).catch(err => {
1557
1596
  warn(`[orchestrator] could not answer ${shortId(todoId)}: ${err.message}`);
1558
1597
  });
1559
1598
  }
@@ -1561,6 +1600,10 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
1561
1600
  catch (err) {
1562
1601
  reportTakeFailed(state, err.message, warn);
1563
1602
  }
1603
+ finally {
1604
+ if (hasOwnedWorkContext())
1605
+ finishRpcClaims(claimOperations, true);
1606
+ }
1564
1607
  }
1565
1608
  /**
1566
1609
  * 18k Slice 10 — THE CODEBASE THIS RUN IS FOR, recovered the way the tick
@@ -2195,7 +2238,7 @@ plan, state,
2195
2238
  /** 18k Slice 10b — THE WORKER'S OWN SPAWN SHAPE, not a responder's. There is no
2196
2239
  * second adapter any more: this is what `defaultSpawn(mcpServer)` returns, and
2197
2240
  * the answerer differs from a dispatch only in passing no `alsoRunningIn`. */
2198
- spawn, warn) {
2241
+ spawn, warn, attempt, machineId, owned, observeHarness) {
2199
2242
  /* ═══ 18k SLICE 10b — ONE ANSWER PER RUN AT A TIME. ═══
2200
2243
  DEPTH, NOT THE PRIMARY DEFENCE. `takeRunMessagesInner` subtracts
2201
2244
  `state.answering` from the scoped take's id list, so a run this machine is
@@ -2220,26 +2263,64 @@ spawn, warn) {
2220
2263
  into an inner function so the clear is in ONE place rather than repeated at
2221
2264
  every `return` below, which is the same shape `orchestratorTick` uses to
2222
2265
  release its own slot. */
2266
+ if (hasOwnedWorkContext() && (!attempt || !machineId))
2267
+ throw new Error('The reply has no exact source lease identity.');
2268
+ owned = hasOwnedWorkContext() ? reserveOwnedWork(attempt, plan.agent) : undefined;
2223
2269
  state.answering.add(todoId);
2224
2270
  try {
2225
- return await answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn);
2271
+ const answered = await answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned, observeHarness);
2272
+ if (answered)
2273
+ owned?.complete();
2274
+ return answered;
2226
2275
  }
2227
2276
  finally {
2228
2277
  state.answering.delete(todoId);
2278
+ owned?.finishPreparation();
2279
+ }
2280
+ }
2281
+ async function settleReplyOutcome(client, machineId, attempt, outcome, onAccepted) {
2282
+ const todoId = attempt.todoId;
2283
+ const finished = await client.rpc('cliv2_finish_reply_attempt', {
2284
+ p_todo_id: todoId, p_machine_id: machineId, p_message_leases: attempt.messageLeases,
2285
+ p_grant_leases: attempt.grantLeases, p_body: outcome.text,
2286
+ ...(!outcome.ok && !outcome.stopped ? { p_failure_kind: outcome.failureKind ?? 'unknown' } : {}),
2287
+ });
2288
+ if (finished.error)
2289
+ throw finished.error;
2290
+ const receipt = finished.data?.[0]?.outcome;
2291
+ if (receipt === 'stale')
2292
+ return false;
2293
+ if (receipt === 'failed' || receipt === 'already_failed') {
2294
+ onAccepted?.();
2295
+ return false;
2229
2296
  }
2297
+ if (receipt !== 'accepted' && receipt !== 'already_settled')
2298
+ throw new Error('The reply acknowledgement was incomplete.');
2299
+ onAccepted?.();
2300
+ if (outcome.stopped && outcome.replyingToClaim)
2301
+ await releaseStoppedTodo(client, todoId, outcome.replyingToClaim);
2302
+ else if (outcome.workerIsLive && outcome.replyingToClaim) {
2303
+ const scheduled = await client.from('cliv2_loose_todos')
2304
+ .update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
2305
+ .eq('id', todoId).eq('claimed_at', outcome.replyingToClaim).in('state', ['working', 'needs-input']);
2306
+ if (scheduled.error)
2307
+ throw scheduled.error;
2308
+ }
2309
+ return !outcome.stopped;
2230
2310
  }
2231
- async function answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn) {
2311
+ async function answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned, observeHarness) {
2232
2312
  /* ALREADY GIVEN UP ON. Said ONCE, at the bound, and then silently skipped —
2233
2313
  the `reportTakeFailed` rule: a line every two minutes forever would bury
2234
2314
  every useful one. */
2235
- const attempts = Math.max(...messageIds.map(id => state.replyAttempts.get(id) ?? 0));
2315
+ const sourceIds = [...messageIds, ...(attempt?.grantLeases ?? []).map(grant => `grant:${grant.id}`)];
2316
+ const attempts = Math.max(0, ...sourceIds.map(id => state.replyAttempts.get(id) ?? 0));
2236
2317
  if (attempts >= MAX_REPLY_ATTEMPTS)
2237
2318
  return false;
2238
2319
  const fail = (reason) => {
2239
- for (const id of messageIds) {
2320
+ for (const id of sourceIds) {
2240
2321
  state.replyAttempts.set(id, (state.replyAttempts.get(id) ?? 0) + 1);
2241
2322
  }
2242
- const spent = Math.max(...messageIds.map(id => state.replyAttempts.get(id) ?? 0));
2323
+ const spent = Math.max(0, ...sourceIds.map(id => state.replyAttempts.get(id) ?? 0));
2243
2324
  if (spent >= MAX_REPLY_ATTEMPTS) {
2244
2325
  warn(`[orchestrator] gave up answering ${shortId(todoId)} after ${MAX_REPLY_ATTEMPTS} attempts: ${reason}`);
2245
2326
  }
@@ -2287,6 +2368,12 @@ async function answerRunMessagesInner(client, todoId, messageIds, plan, state, s
2287
2368
  would either promise a boundary that never comes, or re-dispatch a worker
2288
2369
  for a change the agent already made itself. */
2289
2370
  const workerIsLive = await liveWorkerForTodo(client, todoId, warn);
2371
+ // A stopped request may retain a claim even though its worker has ended.
2372
+ // Capture it before answering so a late stopped reply cannot clear a newer one.
2373
+ const current = await client.from('cliv2_loose_todos').select('claimed_at').eq('id', todoId).maybeSingle();
2374
+ if (current.error)
2375
+ throw current.error;
2376
+ const replyingToClaim = current.data?.claimed_at ?? null;
2290
2377
  /* ═══ 18k SLICE 10b — SPAWNED AS A WORKER, WITH NO CLOCK ON IT. ═══
2291
2378
  The worker's own adapter, so the answerer holds the worker's toolset (ruling
2292
2379
  1) and no timeout at all (ruling 5) — an agent asked for a real change may
@@ -2304,166 +2391,53 @@ async function answerRunMessagesInner(client, todoId, messageIds, plan, state, s
2304
2391
  NO `alsoRunningIn`: the answerer works one run, in that run's own checkout.
2305
2392
  `runTodoId` IS the run, so a tool call it makes is attributed without
2306
2393
  depending on the agent repeating an id back to us. */
2394
+ if (attempt) {
2395
+ const pending = await client.from('cliv2_run_messages').select('id').eq('todo_id', todoId).is('answered_by', null);
2396
+ if (pending.error)
2397
+ throw pending.error;
2398
+ attempt.observedPendingMessageIds = (pending.data ?? []).map((row) => row.id);
2399
+ owned?.setReference(attempt);
2400
+ }
2401
+ const harnessIdentity = executionIdentity(plan.agent);
2307
2402
  let spawnedPid = null;
2308
2403
  const result = await spawn(plan.agent, buildReplyPrompt(record, bodies, { hasCheckout: plan.cwd !== null, workingCopyElsewhere: plan.workingCopyElsewhere }, workerIsLive), undefined, plan.cwd, () => todoWasStopped(client, todoId), (pid) => {
2309
2404
  state.livePids.add(pid);
2310
2405
  spawnedPid = pid;
2311
- }, undefined, todoId);
2406
+ }, undefined, todoId, owned);
2312
2407
  /* HOWEVER THE ANSWER ENDED. The pid must leave `livePids` on every path, or a
2313
2408
  dead child looks alive to the liveness watch forever. */
2314
2409
  if (spawnedPid !== null)
2315
2410
  state.livePids.delete(spawnedPid);
2316
- /* ═══ 18k SLICE 10b — THE USER STOPPED IT, AND THAT IS NOT A FAILURE. ═══
2317
- CHECKED BEFORE `!result.ok`, because a killed child also reports not-ok, and
2318
- falling through would count the user's own decision as an attempt and
2319
- re-spawn the very thing they cancelled — `dispatchClaimInner` makes the same
2320
- check for the same reason.
2321
-
2322
- BUT STOPPING ALONE WOULD BE AN INFINITE SPAWN LOOP. A stopped run stays
2323
- stopped, so the message stays uncovered, the lease lapses, the next take
2324
- spawns another answerer, and that one is stopped too — forever, burning no
2325
- budget because no attempt is counted. So the messages are COVERED, and
2326
- `cliv2_mark_messages_answered` needs a reply id, so one short reply is what
2327
- covers them. That is also the honest thread: it says the message was not
2328
- answered because the work was stopped, rather than showing "Queued" forever.
2329
- ux.md's north star forbids "this will never be answered", and an uncovered
2330
- stopped message is exactly that state.
2331
-
2332
- AND THE CLAIM. This function set no claim, but the run may carry one from an
2333
- earlier dispatch, and `releaseStoppedTodo` is the only thing that clears it —
2334
- without this call Retry cannot pick the run up. Its own `.eq('state','stopped')`
2335
- guard makes it a no-op when Retry has already re-armed the run. */
2336
- if (result.stopped) {
2337
- try {
2338
- const replyId = await writeRunReply(client, todoId, 'Stopped.');
2339
- const { error } = await client.rpc('cliv2_mark_messages_answered', {
2340
- p_reply_id: replyId,
2341
- p_message_ids: messageIds,
2342
- });
2343
- if (error)
2344
- throw new Error(error.message);
2345
- for (const id of messageIds)
2346
- state.replyAttempts.delete(id);
2347
- }
2348
- catch (err) {
2349
- warn(`[orchestrator] ${shortId(todoId)} was stopped and the conversation could not be closed off: ${err.message}`);
2350
- }
2351
- try {
2352
- await releaseStoppedTodo(client, todoId);
2353
- }
2354
- catch (err) {
2355
- warn(`[orchestrator] ${shortId(todoId)} was stopped and its claim could not be released: ${err.message}`);
2356
- }
2411
+ if (result.interrupted)
2357
2412
  return false;
2358
- }
2359
- if (!result.ok) {
2360
- fail(result.error);
2361
- return false;
2362
- }
2363
- const body = result.output.trim();
2364
- if (body === '') {
2365
- fail('it said nothing');
2413
+ const failureKind = result.failureKind ?? nativeFailureKind(result.saidTail ?? '');
2414
+ if (!result.ok && !result.stopped && failureKind === 'transient' && attempts + 1 < MAX_REPLY_ATTEMPTS) {
2415
+ fail(failureMessage(plan.agent, failureKind));
2416
+ owned?.complete();
2366
2417
  return false;
2367
2418
  }
2368
- let replyId;
2369
- try {
2370
- replyId = await writeRunReply(client, todoId, body);
2371
- }
2372
- catch (err) {
2373
- fail(`the reply could not be saved: ${err.message}`);
2374
- return false;
2375
- }
2376
- /* ANSWERED. The count is cleared so a LATER message on this run that fails
2377
- starts from zero rather than inheriting a spent budget.
2378
-
2379
- ═══ 18k SLICE 9: AND IT IS CLEARED HERE, BEFORE THE COVERING WRITE. ═══
2380
- The reply EXISTS from this line on, so the attempt is spent whatever happens
2381
- next, and the covering below must NOT go through `fail()`. `fail()`'s
2382
- contract is "nothing was written, so retry the whole thing", and that
2383
- contract does not hold once a reply is in the thread. Had the covering's
2384
- failure counted an attempt, three consecutive attribution failures — a reply
2385
- written EACH time — would exhaust `MAX_REPLY_ATTEMPTS` while `answered_by`
2386
- stayed null, so the take would keep re-offering the row forever and nothing
2387
- would ever spawn again: three replies, then permanent silence with "Queued"
2388
- showing. Strictly worse than the predicate this slice replaces, which at
2389
- least excluded such a message after the first reply. */
2390
- for (const id of messageIds)
2419
+ if (!result.ok && !result.stopped && failureKind === 'transient')
2420
+ fail(failureMessage(plan.agent, failureKind));
2421
+ const succeeded = result.ok && !!result.output.trim();
2422
+ const body = result.stopped ? 'Stopped.' : succeeded ? result.output.trim() : null;
2423
+ if (!attempt || !machineId)
2424
+ throw new Error('The reply has no exact source lease identity.');
2425
+ const terminal = { surface: 'reply', completedAt: Date.now(), harnessIdentity, ok: succeeded, text: body, reason: succeeded ? '' : failureMessage(plan.agent, failureKind), failureKind: succeeded ? undefined : failureKind, stopped: !!result.stopped, replyingToClaim, workerIsLive: result.ok && workerIsLive };
2426
+ owned?.deferOutcome(terminal);
2427
+ // A durable terminal result is awaiting acknowledgement, never another native attempt.
2428
+ for (const id of sourceIds)
2391
2429
  state.replyAttempts.delete(id);
2392
- /* ═══ 18k SLICE 9: RECORD WHAT THE REPLY ANSWERED. ═══
2393
- "Was this message in front of the responder that wrote that reply" is known
2394
- only here, to the code that handed it over — no timestamp anywhere records
2395
- it, which is why 20260814190000's inference was wrong by the width of the
2396
- responder's runtime.
2397
-
2398
- REPLY FIRST, COVERING SECOND, DELIBERATELY. If the covering fails the reply
2399
- exists and the messages stay uncovered, so the next take re-takes them and
2400
- they get a second reply: visible duplication the user can see and react to.
2401
- The reverse order risks a covering for a reply that was never written — a
2402
- message marked answered by nothing, invisible forever. A visible duplicate
2403
- beats a silent drop, which is the north star's own ranking (ux.md:739).
2404
-
2405
- AND THE BOUND IS NOT LOST, IT MOVES: the RPC's own `answered_by is null`
2406
- term is what bounds a repeated failure here. Each retry round writes at most
2407
- one extra reply and rounds are one pass apart, so the user-visible bound is
2408
- duplicate replies at one per pass, never an unbounded spawn loop and never
2409
- permanent silence. A database that will not accept the mark is an outage,
2410
- and the honest behaviour under an outage is a visible duplicate rather than
2411
- a message that disappears. */
2412
- /* 18k Slice 10b — NOTHING TO ATTRIBUTE WHEN AN APPROVAL WOKE THIS. The reply
2413
- answers a grant, not a message, so there is no row to mark and the RPC is
2414
- skipped rather than called with an empty list. The grant's own `consumed_at`
2415
- is what stops it being offered again, stamped by the tool when the approved
2416
- call finally goes through. */
2417
- if (messageIds.length > 0) {
2418
- try {
2419
- const { error } = await client.rpc('cliv2_mark_messages_answered', {
2420
- p_reply_id: replyId,
2421
- p_message_ids: messageIds,
2422
- });
2423
- if (error)
2424
- throw new Error(error.message);
2425
- }
2426
- catch (err) {
2427
- warn(`[orchestrator] the reply to ${shortId(todoId)} could not be attributed, it will be answered again: ${err.message}`);
2428
- return false;
2429
- }
2430
- }
2431
- /* ═══ 18k SLICE 10b — CASE B: THE BOUNDARY. ═══
2432
- An agent was already working this checkout, so the reply said what would
2433
- happen rather than doing it. Clearing the claim is what makes that true: the
2434
- next tick dispatches a fresh worker, and `readRunDirection` hands it the
2435
- message. Nothing interrupts the live worker — it finishes what it is on, and
2436
- the change rides the NEXT dispatch, exactly as `hand_off_stage` already does
2437
- mid-run.
2438
-
2439
- AFTER THE REPLY AND ITS COVERING, NEVER BEFORE. Clearing first and then
2440
- failing to write the reply would re-dispatch a worker for a change nobody
2441
- was ever told about.
2442
-
2443
- GUARDED ON `working`/`needs-input`, for `releaseTodoForRedispatch`'s own
2444
- reason: a run that has since finished, failed or been stopped must not be
2445
- resurrected into `working` by a claim clear. And all three columns move
2446
- together — `cliv2_loose_todos_claim_complete_check` makes a half-release a
2447
- database error.
2448
-
2449
- BEST-EFFORT: the reply is in the thread whatever happens here, so a failed
2450
- clear costs the user a change that does not get dispatched — which the next
2451
- message re-raises — where throwing would lose the reply's own success. */
2452
- if (workerIsLive) {
2453
- try {
2454
- const { error } = await client
2455
- .from('cliv2_loose_todos')
2456
- .update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
2457
- .eq('id', todoId)
2458
- .in('state', ['working', 'needs-input']);
2459
- if (error)
2460
- throw new Error(error.message);
2461
- }
2462
- catch (err) {
2463
- warn(`[orchestrator] ${shortId(todoId)} was answered but the change could not be scheduled: ${err.message}`);
2464
- }
2465
- }
2466
- return true;
2430
+ const answered = await settleReplyOutcome(client, machineId, attempt, terminal, () => {
2431
+ if (succeeded && !result.stopped)
2432
+ observeHarness?.(plan.agent, 'authenticated', 'dispatch-success', terminal.completedAt, terminal.harnessIdentity);
2433
+ else if (failureKind === 'authentication')
2434
+ observeHarness?.(plan.agent, 'sign-in-required', 'provider-rejected', terminal.completedAt, terminal.harnessIdentity);
2435
+ });
2436
+ owned?.acknowledgeOutcome();
2437
+ owned?.complete();
2438
+ for (const id of sourceIds)
2439
+ state.replyAttempts.delete(id);
2440
+ return answered;
2467
2441
  }
2468
2442
  /** 18k Slice 7 — say ONCE that the take is failing, not every 10 seconds.
2469
2443
  *
@@ -2504,7 +2478,7 @@ export function reportTakeFailed(state, reason, warn) {
2504
2478
  * is the brake, exactly as it is for the scope park, and answering is what lets
2505
2479
  * it off.
2506
2480
  */
2507
- export async function askWhichCodebase(client, todoId, projectName, remotes, warn) {
2481
+ export async function askWhichCodebase(client, todoId, projectName, remotes, warn, claimedAt) {
2508
2482
  try {
2509
2483
  const { error } = await client.rpc('cliv2_ask_about_request', {
2510
2484
  p_todo: todoId,
@@ -2517,7 +2491,7 @@ export async function askWhichCodebase(client, todoId, projectName, remotes, war
2517
2491
  });
2518
2492
  if (error)
2519
2493
  throw new Error(error.message);
2520
- await parkTodoForScope(client, todoId);
2494
+ await parkTodoForScope(client, todoId, claimedAt);
2521
2495
  return true;
2522
2496
  }
2523
2497
  catch (err) {
@@ -2797,11 +2771,13 @@ export async function collisionForTodo(client, todoId, gitRemoteUrl, warn) {
2797
2771
  return null;
2798
2772
  }
2799
2773
  }
2800
- export async function releaseTodo(client, todoId) {
2801
- const { error } = await client
2774
+ export async function releaseTodo(client, todoId, claimedAt) {
2775
+ const write = client
2802
2776
  .from('cliv2_loose_todos')
2803
2777
  .update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
2804
- .eq('id', todoId);
2778
+ .eq('id', todoId)
2779
+ .in('state', ['working', 'needs-input']);
2780
+ const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
2805
2781
  if (error)
2806
2782
  throw error;
2807
2783
  }
@@ -3707,12 +3683,13 @@ run = spawnAgent) {
3707
3683
  alsoRunningIn,
3708
3684
  /** 18c Slice 1 — the request this spawn is working, so
3709
3685
  * `report_activity` lines can be attributed without the prompt. */
3710
- runTodoId) =>
3711
- /* !Cleanup Phase 6b (I42): NO TIMEOUT. `NO_SPAWN_TIMEOUT` disables the kill
3712
- entirely — a working agent is never killed for elapsed time (user ruling
3713
- 2026-08-06). Liveness is watched from outside the run instead, by
3714
- `reapDeadWorkers` on the heartbeat. */
3715
- run(agent, prompt, NO_SPAWN_TIMEOUT, { onStep, mcpServer, cwd, stopRequested, onPid, alsoRunningIn, runTodoId });
3686
+ runTodoId, owned) => {
3687
+ /* !Cleanup Phase 6b (I42): NO TIMEOUT. `NO_SPAWN_TIMEOUT` disables the kill
3688
+ entirely — a working agent is never killed for elapsed time (user ruling
3689
+ 2026-08-06). Liveness is watched from outside the run instead, by
3690
+ `reapDeadWorkers` on the heartbeat. */
3691
+ return run(agent, prompt, NO_SPAWN_TIMEOUT, { onStep, mcpServer, cwd, stopRequested, onPid, alsoRunningIn, runTodoId, owned });
3692
+ };
3716
3693
  }
3717
3694
  /** 18k Slice 8 — how many times one message may be attempted before the daemon
3718
3695
  * stops spawning for it.
@@ -3727,7 +3704,9 @@ export const MAX_REPLY_ATTEMPTS = 3;
3727
3704
  /** Bound on retained stdout. Generous — a real run's JSONL is a few hundred KB —
3728
3705
  * but finite, so a runaway agent cannot exhaust memory. */
3729
3706
  const MAX_OUTPUT_BYTES = 32 * 1024 * 1024;
3730
- export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_TIMEOUT_MS, { resolveBin = agentPath, exec, onStep, spawnProcess = spawnChild, maxOutputBytes = MAX_OUTPUT_BYTES, mcpServer = null, cwd = null, stopRequested, stopPollMs = STOP_POLL_MS, onPid, alsoRunningIn = [], runTodoId = null } = {}) {
3707
+ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_TIMEOUT_MS, { resolveBin = agentPath, exec, onStep, spawnProcess = spawnChild, maxOutputBytes = MAX_OUTPUT_BYTES, mcpServer = null, cwd = null, stopRequested, stopPollMs = STOP_POLL_MS, onPid, alsoRunningIn = [], runTodoId = null, owned } = {}) {
3708
+ if (owned?.interrupted())
3709
+ return { ok: false, output: '', error: 'Work was interrupted by the service command.', interrupted: true };
3731
3710
  const bin = resolveBin(agent);
3732
3711
  if (!bin)
3733
3712
  return { ok: false, output: '', error: `${agent} is not installed on this machine` };
@@ -3741,7 +3720,7 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
3741
3720
  Windows path holds spaces (`C:\\Program Files\\…`) and the MCP config is
3742
3721
  JSON, so both would still be split by cmd.exe. */
3743
3722
  const viaShell = needsShell(bin);
3744
- const rawArgs = headlessAgentArgs(agent, prompt, mcpServer, alsoRunningIn, viaShell, runTodoId);
3723
+ const rawArgs = headlessAgentArgs(agent, prompt, mcpServer, alsoRunningIn, true, runTodoId);
3745
3724
  const args = viaShell ? rawArgs.map(quoteForCmd) : rawArgs;
3746
3725
  /* !Cleanup Phase 2 (I5), CORRECTED BY 18b SLICE 2.
3747
3726
  It used to be `cwd ?? undefined`, and `undefined` means "inherit the
@@ -3787,10 +3766,8 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
3787
3766
  const codexHome = agent === 'claude'
3788
3767
  ? null
3789
3768
  : ensureCodexRunHome(mcpServer, runTodoId, runTodoId ? `${runTodoId}-${randomUUID().slice(0, 8)}` : null);
3790
- if (agent !== 'claude' && !codexHome) {
3791
- console.warn('[orchestrator] codex is running WITHOUT an isolated configuration: no signed-in codex ' +
3792
- 'credential to seed a per-run home. This worker inherits the MCP servers, instruction ' +
3793
- 'files and permission posture on this machine.');
3769
+ if (codexHome instanceof CodexHomeFailure) {
3770
+ return { ok: false, output: '', error: codexHome.message, failureKind: codexHome.kind };
3794
3771
  }
3795
3772
  /* Spread the parent's env rather than replacing it: codex needs PATH, HOME,
3796
3773
  the proxy variables and the user's shell environment to run at all. Only the
@@ -3829,13 +3806,18 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
3829
3806
  return new Promise((resolve) => {
3830
3807
  let child;
3831
3808
  try {
3832
- child = spawnProcess(bin, args, {
3809
+ owned?.beforeSpawn();
3810
+ const spawnExecution = owned && spawnProcess === spawnChild
3811
+ ? (file, argv, options) => spawnOwnedProcess(file, argv, options, owned.id)
3812
+ : spawnProcess;
3813
+ child = spawnExecution(bin, args, {
3833
3814
  // The repo-wide invariant for ANY child process (MEMORY: "Windows
3834
3815
  // silence decision"). A user must never see a console flash.
3835
3816
  windowsHide: true,
3836
3817
  shell: viaShell,
3837
3818
  // 18a Slice 7: stdin is a pipe only where the prompt travels down it.
3838
- stdio: [viaShell ? 'pipe' : 'ignore', 'pipe', 'pipe'],
3819
+ stdio: ['pipe', 'pipe', 'pipe'],
3820
+ detached: process.platform !== 'win32',
3839
3821
  // !Cleanup Phase 2 (I5) — the worker starts inside the work item's own
3840
3822
  // checkout instead of at the daemon's cwd (`/` under launchd).
3841
3823
  cwd: childCwd,
@@ -3849,17 +3831,15 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
3849
3831
  /* Nothing was spawned, so nothing can still be holding the directory. */
3850
3832
  if (codexHome)
3851
3833
  removeCodexRunHome(codexHome);
3852
- return resolve(spawnFailure(agent, err));
3834
+ return resolve({ ...spawnFailure(agent, err), ...(owned?.interrupted() ? { interrupted: true } : {}) });
3853
3835
  }
3854
3836
  /* 18a SLICE 7 — AND THE PROMPT GOES DOWN IT, then the pipe is CLOSED.
3855
3837
  Both agents wait for end-of-input before they start, so leaving it open
3856
3838
  hangs the run forever. The write is best-effort in the same sense every
3857
3839
  other pipe operation here is: a child that died before reading raises
3858
3840
  EPIPE, and that is the child's exit to report, not this write's. */
3859
- if (viaShell && child.stdin) {
3860
- child.stdin.on('error', () => { });
3861
- child.stdin.end(prompt);
3862
- }
3841
+ let registration = Promise.resolve();
3842
+ child.stdin?.on('error', () => { });
3863
3843
  /* !Cleanup Phase 6b (I43) — hand the pid up the moment it exists, so the
3864
3844
  liveness watch can tell a dead agent from a quiet one. Guarded on
3865
3845
  `typeof`: a child that failed to spawn has no pid, and reporting
@@ -4020,9 +4000,24 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
4020
4000
  it stripped to '', the column would take null and the card would lose
4021
4001
  the context its question needs. `redactSecrets` is identity when the run
4022
4002
  read no credential, which is almost every run. */
4023
- resolve(result.output
4024
- ? { ...result, output: redactSecrets(runTodoId, result.output) }
4025
- : result);
4003
+ void (async () => {
4004
+ await registration.catch(() => { });
4005
+ if (owned && typeof child.pid === 'number') {
4006
+ for (;;) {
4007
+ try {
4008
+ await owned.exited();
4009
+ break;
4010
+ }
4011
+ catch {
4012
+ await new Promise((done) => setTimeout(done, 250));
4013
+ }
4014
+ }
4015
+ }
4016
+ if (codexHome && owned)
4017
+ removeCodexRunHome(codexHome);
4018
+ resolve({ ...(result.output ? { ...result, output: redactSecrets(runTodoId, result.output) } : result),
4019
+ ...(owned?.interrupted() ? { interrupted: true } : {}) });
4020
+ })();
4026
4021
  };
4027
4022
  /* ═══ 18c SLICE 8 — THE HOME IS SWEPT WHEN THE CHILD IS GONE, NOT WHEN THE
4028
4023
  PROMISE SETTLES. ═══
@@ -4058,7 +4053,8 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
4058
4053
  sweptHome = true;
4059
4054
  removeCodexRunHome(codexHome);
4060
4055
  };
4061
- child.on('close', sweepHomeWhenChildIsGone);
4056
+ if (!owned)
4057
+ child.on('close', sweepHomeWhenChildIsGone);
4062
4058
  /* !Cleanup PHASE 6b (I42) — THE ELAPSED-TIME KILL IS OFF BY DEFAULT NOW.
4063
4059
  User ruling 2026-08-06: "No agent that is actively working should ever be
4064
4060
  killed automatically, regardless of how much time has elapsed."
@@ -4186,7 +4182,7 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
4186
4182
  401 that means signed out. Each tailed separately so a chatty stdout
4187
4183
  cannot push stderr out of the window. */
4188
4184
  const saidTail = `${stdout.slice(-STDERR_TAIL_BYTES)}\n${stderrTail}`;
4189
- return settle({ ok: false, output: '', error: `${agent} ${how}`, saidTail });
4185
+ return settle({ ok: false, output: '', error: `${agent} ${how}`, saidTail, failureKind: nativeFailureKind(stdout, stderrTail) });
4190
4186
  }
4191
4187
  /* 18c SLICE 8 — CODEX CAN EXIT 0 ON A TURN THAT FAILED, so the exit code
4192
4188
  above is not the last word. Checked AFTER the code, because a non-zero
@@ -4196,7 +4192,7 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
4196
4192
  release path, like every other failure, so the unit is retried. */
4197
4193
  if (codexTurnFailed !== null) {
4198
4194
  const saidTail = `${stdout.slice(-STDERR_TAIL_BYTES)}\n${stderrTail}`;
4199
- return settle({ ok: false, output: '', error: `${agent} ${codexTurnFailed}`, saidTail });
4195
+ return settle({ ok: false, output: '', error: `${agent} ${codexTurnFailed}`, saidTail, failureKind: nativeFailureKind(stdout, stderrTail) });
4200
4196
  }
4201
4197
  /* PREFER WHAT WAS CAPTURED LIVE, FOR BOTH HARNESSES (18c Slice 8 — it used
4202
4198
  to be claude only). `stdout` is now the fallback only for a run that
@@ -4214,6 +4210,18 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
4214
4210
  way: anything parseable was already captured live above. */
4215
4211
  settle(finishSpawn(agent, stdout));
4216
4212
  });
4213
+ registration = owned ? owned.register(child) : Promise.resolve();
4214
+ void registration.then(() => {
4215
+ if (!owned?.interrupted()) {
4216
+ if (owned)
4217
+ releaseOwnedProcess(child);
4218
+ child.stdin?.end(prompt);
4219
+ }
4220
+ }).catch((error) => {
4221
+ child.stdin?.destroy();
4222
+ killTree(child);
4223
+ settle(spawnFailure(agent, error));
4224
+ });
4217
4225
  });
4218
4226
  }
4219
4227
  /**
@@ -4401,7 +4409,7 @@ export async function startWorker(client, todoId, machineId, agent, warn = conso
4401
4409
  /** Move a worker through its lifecycle. `died` REQUIRES a reason — the table's
4402
4410
  * check constraint refuses the row otherwise, deliberately: a failure the panel
4403
4411
  * cannot explain gives the user nothing to act on. */
4404
- export async function updateWorker(client, workerId, state, error = null, warn = console.warn) {
4412
+ export async function updateWorker(client, workerId, state, error = null, warn = console.warn, strict = false) {
4405
4413
  if (!workerId)
4406
4414
  return;
4407
4415
  try {
@@ -4412,11 +4420,14 @@ export async function updateWorker(client, workerId, state, error = null, warn =
4412
4420
  const { error: writeError } = await client
4413
4421
  .from('cliv2_workers')
4414
4422
  .update(patch)
4415
- .eq('id', workerId);
4423
+ .eq('id', workerId)
4424
+ .in('state', ['working', 'dispatching']);
4416
4425
  if (writeError)
4417
4426
  throw writeError;
4418
4427
  }
4419
4428
  catch (err) {
4429
+ if (strict)
4430
+ throw err;
4420
4431
  warn(`[orchestrator] could not update the worker: ${err.message}`);
4421
4432
  }
4422
4433
  }
@@ -4596,6 +4607,11 @@ export async function reapDeadWorkers(client, machineId, livePids, now = () => n
4596
4607
  const cutoff = now().getTime() - STALE_WORKER_MS;
4597
4608
  let reaped = 0;
4598
4609
  for (const row of rows) {
4610
+ if (hasOwnedWorkContext() && ownedWorkerExecutionHeld(row.id))
4611
+ continue;
4612
+ const claimedAt = hasOwnedWorkContext() ? ownedWorkerClaimedAt(row.id) : undefined;
4613
+ if (claimedAt === null)
4614
+ continue;
4599
4615
  /* CONDITION 1 — quiet. A worker that has written a step recently is
4600
4616
  working, and nothing else is even considered. */
4601
4617
  const seen = Date.parse(row.updated_at);
@@ -4641,7 +4657,9 @@ export async function reapDeadWorkers(client, machineId, livePids, now = () => n
4641
4657
  card everything Slices 5 and 6 already draw (the attempt, the
4642
4658
  reason, the countdown) for free, and at the bound it gives up and
4643
4659
  says why instead of being fed back into the same machine. */
4644
- const { fate, attempts } = await releaseOrFail(client, row.todo_id, plain, now().getTime());
4660
+ const { fate, attempts } = await releaseOrFail(client, row.todo_id, plain, now().getTime(), claimedAt);
4661
+ if (fate === 'stale')
4662
+ continue;
4645
4663
  /* 18c SLICE 9 (GAP 15): AND IT HOLDS NOTHING ANY MORE.
4646
4664
  THIS PATH NEEDS ITS OWN CALL and is not covered by `dispatchClaim`'s
4647
4665
  `finally`: the reaper exists precisely for a worker that died while
@@ -4705,6 +4723,7 @@ export async function reapDeadWorkers(client, machineId, livePids, now = () => n
4705
4723
  */
4706
4724
  export async function recoverStrandedWorkers(client, machineId, log = console.log, warn = console.warn) {
4707
4725
  try {
4726
+ const lifecycleOwned = hasOwnedWorkContext();
4708
4727
  const { data, error } = await client
4709
4728
  .from('cliv2_workers')
4710
4729
  .update({
@@ -4732,7 +4751,11 @@ export async function recoverStrandedWorkers(client, machineId, log = console.lo
4732
4751
  /* The todo goes back in the queue. Order matters: the worker is closed
4733
4752
  FIRST, so there is no instant at which a claimable todo still has a live
4734
4753
  worker that would refuse the dispatch. */
4735
- for (const row of stranded) {
4754
+ // Lifecycle startup has already reconciled exact attempt receipts. A stale
4755
+ // worker row can outlive a newer claim, including one on another machine;
4756
+ // closing that obsolete worker must never release the newer assignment.
4757
+ // Keep the pre-lifecycle compatibility path only for its existing callers.
4758
+ for (const row of lifecycleOwned ? [] : stranded) {
4736
4759
  try {
4737
4760
  await releaseTodo(client, row.todo_id);
4738
4761
  }
@@ -5287,7 +5310,7 @@ export async function orchestratorTick(deps) {
5287
5310
  query, so a tick that cannot start anything costs nothing at all — no read,
5288
5311
  no claim, no log. Was `if (state.busy)`, which is why a second request typed
5289
5312
  three seconds after the first sat untouched until the first finished.
5290
-
5313
+ *
5291
5314
  THE SLOT IS TAKEN HERE, NOT AFTER THE CLAIM, and that is not tidiness — it
5292
5315
  is the difference between a cap and a suggestion. `pollOrchestrator` is an
5293
5316
  interval that does NOT await, so ticks genuinely overlap, and there are
@@ -5297,7 +5320,7 @@ export async function orchestratorTick(deps) {
5297
5320
  the cap would bound nothing. Taking the slot before the first await makes
5298
5321
  the check-and-take atomic with respect to the event loop, because there is
5299
5322
  no suspension point between them.
5300
-
5323
+ *
5301
5324
  EVERY early return below MUST give it back, which is why the whole body from
5302
5325
  here down is wrapped so the release is in one place rather than repeated at
5303
5326
  each `return`. */
@@ -5305,8 +5328,14 @@ export async function orchestratorTick(deps) {
5305
5328
  return 'busy';
5306
5329
  state.inFlight += 1;
5307
5330
  try {
5331
+ await reconcileInterruptedWork(deps.client, deps.machineId, deps.observeHarness);
5308
5332
  return await runTick(deps, state, now, log, warn);
5309
5333
  }
5334
+ catch (error) {
5335
+ // This supervising tick must survive an outage and retry its saved recovery.
5336
+ warn(`orchestrator poll failed, will retry: ${error instanceof Error ? error.message : JSON.stringify(error)}`);
5337
+ return 'failed';
5338
+ }
5310
5339
  finally {
5311
5340
  state.inFlight -= 1;
5312
5341
  }
@@ -5314,208 +5343,230 @@ export async function orchestratorTick(deps) {
5314
5343
  /** The tick body, with its concurrency slot already taken. Split out so the slot
5315
5344
  * is released in exactly one place — see the caller. */
5316
5345
  async function runTick(deps, state, now, log, warn) {
5317
- /* Only what CHOOSING a unit needs. The spawn's own inputs (`spawn`,
5318
- `mcpServer`) are destructured in `dispatchClaim`, where they are used.
5319
- `resolveContext` is needed in BOTH: here it answers "which repo would this
5320
- touch" for the collision read, and there it answers "where does the worker
5321
- start" — deliberately the same resolver, so the two cannot disagree. */
5322
- const { client, userId, machineId, machineName, agents, resolveContext = (claim) => resolveClaimContext(client, claim, warn), } = deps;
5323
- /* 18a SLICE 6: THE BACKOFF USED TO BE CHECKED HERE, and returning from this
5324
- line is precisely what was wrong with it: a failing agent stopped the whole
5325
- daemon rather than stopping itself. The wait is now per request, on the row,
5326
- and the claim itself skips a request whose moment has not come. */
5327
- let role;
5346
+ if (!ownedWorkAllowed())
5347
+ return 'nothing-to-do';
5348
+ const endClaim = beginOwnedClaim();
5349
+ const claimOperations = [];
5328
5350
  try {
5329
- const { data, error } = await client
5330
- .from('cliv2_orchestrator_preference')
5331
- .select('machine_id, agent')
5332
- .eq('user_id', userId)
5333
- .maybeSingle();
5334
- if (error)
5335
- throw error;
5336
- role = resolveOrchestratorRole(data ?? null, machineId, agents);
5337
- }
5338
- catch (err) {
5339
- // Do NOT fall through to a claim on a failed read: an unreadable designation
5340
- // is not a designation, and guessing would be exactly the fleet-wide race
5341
- // the machine check exists to prevent.
5342
- warn(`orchestrator designation read failed, will retry: ${err.message}`);
5343
- return 'failed';
5344
- }
5345
- reportRole(state, role, machineName, log);
5346
- if (role.kind !== 'this-machine')
5347
- return 'not-designated';
5348
- /* ═══ 18d — NO TOOLS MEANS NO DISPATCH. ═══
5349
-
5350
- FOUND BY WALKING IT, 2026-08-12. A daemon restart raced its own port: the
5351
- replacement process came up while the old one still held 4579, so
5352
- `toolsServerStatus().running` was false, `presence.ts` passed
5353
- `mcpServer: null`, and the very next tick claimed a request and spawned a
5354
- worker WITH NO CTRL+SPC TOOLS AT ALL. That worker could not read the work
5355
- item, could not read its own stage document, could not call `ask_question`
5356
- and could not `hand_off_stage`. It did the only thing left: it wrote an
5357
- essay into its final answer explaining that the tools were missing, and the
5358
- request was burned — `attempts` spent, the user's card finished, and
5359
- nothing to show for it.
5360
-
5361
- Null is the RIGHT value there: the alternative that comment rejects is
5362
- letting the worker inherit the user's own agent config, where a second
5363
- CTRL+SPC server pointed at a DIFFERENT ACCOUNT was once found registered and
5364
- listening. That is worse. But there was a third option nobody took, which is
5365
- to not dispatch at all.
5366
-
5367
- WAITING IS FREE AND CORRECT. The request keeps its place, unclaimed and
5368
- unspent, and the tools server comes up moments later — the port race
5369
- resolves as soon as the old process lets go. A dispatch made in this window
5370
- cannot be retried usefully, because it is not the request that is broken.
5371
-
5372
- BEFORE THE CLAIM, deliberately: claiming and then refusing to spawn would
5373
- spend `attempts` on a condition that has nothing to do with the request.
5374
-
5375
- THE CONDITION IS "A REAL SPAWN WITH NO SERVER", not "no server". An injected
5376
- `spawn` is a caller who has taken over the launching entirely — every test in
5377
- the suite, and any future embedder — and for them `mcpServer` describes
5378
- nothing, because `defaultSpawn` is what turns it into an argv. Gating on the
5379
- null alone would refuse to dispatch in exactly the cases where the tools are
5380
- not in question, which is not a stricter check but a wrong one. */
5381
- if (!deps.mcpServer && !deps.spawn) {
5382
- reportNoTools(state, warn);
5383
- return 'no-tools';
5384
- }
5385
- state.warnedNoTools = false;
5386
- /* ═══ !Cleanup PHASE 7 SLICE 2: CLAIM PAST THE BLOCKED ONES. ═══
5387
-
5388
- A dependent unit must WAIT, and everything behind it must NOT. Those two
5389
- requirements together are why this is a loop rather than a single claim.
5390
-
5391
- The claim is this design's only queue position, and it takes the OLDEST
5392
- claimable row. So a blocked unit that is merely released is re-claimed FIRST
5393
- on the very next tick, forever — it stops being a waiting unit and becomes a
5394
- queue stopper. Releasing it AND telling the next claim to skip it is what
5395
- makes the rest of the queue reachable.
5396
-
5397
- BOUNDED BY THE NUMBER OF UNITS SKIPPED, not by a fixed count: each iteration
5398
- either dispatches (returns), finds nothing (returns), or adds exactly one id
5399
- to `skip`, and a skipped id can never be claimed again this tick. So the
5400
- loop runs at most once per blocked unit and then ends. No timeout needed,
5401
- because it cannot revisit a row.
5402
-
5403
- THE UNIT STAYS `working` AND UNCLAIMED, deliberately — it is NOT parked to
5404
- `needs-input` the way the scope gate parks. `needs-input` means "this needs
5405
- YOU", and a dependency needs nothing from the user: it resolves when the
5406
- blocker finishes. Parking it would put a card under "Needs you" offering no
5407
- action, which is precisely the defect Phase 1 existed to remove. The panel
5408
- reads the same `blocked_tasks` view and renders "Waiting on <name>" from it,
5409
- so the card explains itself without a state that lies. */
5410
- const skip = [];
5411
- let blockedCount = 0;
5412
- for (;;) {
5413
- let claim;
5351
+ /* Only what CHOOSING a unit needs. The spawn's own inputs (`spawn`,
5352
+ `mcpServer`) are destructured in `dispatchClaim`, where they are used.
5353
+ `resolveContext` is needed in BOTH: here it answers "which repo would this
5354
+ touch" for the collision read, and there it answers "where does the worker
5355
+ start" — deliberately the same resolver, so the two cannot disagree. */
5356
+ const { client, userId, machineId, machineName, agents, resolveContext = (claim) => resolveClaimContext(client, claim, warn), } = deps;
5357
+ /* 18a SLICE 6: THE BACKOFF USED TO BE CHECKED HERE, and returning from this
5358
+ line is precisely what was wrong with it: a failing agent stopped the whole
5359
+ daemon rather than stopping itself. The wait is now per request, on the row,
5360
+ and the claim itself skips a request whose moment has not come. */
5361
+ const recovered = hasOwnedWorkContext() ? recoveredRpcClaims('cli').find(row => row.action === 'cliv2_claim_next_loose_todo' && agents.includes(row.args?.p_agent)) : undefined;
5362
+ let role;
5414
5363
  try {
5415
- claim = await claimNextTodo(client, machineId, role.agent, skip);
5364
+ const { data, error } = await client
5365
+ .from('cliv2_orchestrator_preference')
5366
+ .select('machine_id, agent')
5367
+ .eq('user_id', userId)
5368
+ .maybeSingle();
5369
+ if (error)
5370
+ throw error;
5371
+ role = resolveOrchestratorRole(data ?? null, machineId, agents);
5416
5372
  }
5417
5373
  catch (err) {
5418
- warn(`orchestrator poll failed, will retry: ${err.message}`);
5419
- return 'failed';
5374
+ // Do NOT fall through to a claim on a failed read: an unreadable designation
5375
+ // is not a designation, and guessing would be exactly the fleet-wide race
5376
+ // the machine check exists to prevent.
5377
+ warn(`orchestrator designation read failed, will retry: ${err.message}`);
5378
+ if (!recovered)
5379
+ return 'failed';
5380
+ role = { kind: 'none' };
5420
5381
  }
5421
- if (!claim) {
5422
- /* Nothing claimable. If we skipped something, the queue is not empty —
5423
- it is WAITING, which is a different fact and must not be logged as
5424
- idleness. */
5425
- return blockedCount > 0 ? 'blocked' : 'nothing-to-do';
5382
+ if (recovered)
5383
+ role = { kind: 'this-machine', agent: recovered.args.p_agent };
5384
+ reportRole(state, role, machineName, log);
5385
+ if (role.kind !== 'this-machine')
5386
+ return 'not-designated';
5387
+ /* ═══ 18d — NO TOOLS MEANS NO DISPATCH. ═══
5388
+ *
5389
+ FOUND BY WALKING IT, 2026-08-12. A daemon restart raced its own port: the
5390
+ replacement process came up while the old one still held 4579, so
5391
+ `toolsServerStatus().running` was false, `presence.ts` passed
5392
+ `mcpServer: null`, and the very next tick claimed a request and spawned a
5393
+ worker WITH NO CTRL+SPC TOOLS AT ALL. That worker could not read the work
5394
+ item, could not read its own stage document, could not call `ask_question`
5395
+ and could not `hand_off_stage`. It did the only thing left: it wrote an
5396
+ essay into its final answer explaining that the tools were missing, and the
5397
+ request was burned — `attempts` spent, the user's card finished, and
5398
+ nothing to show for it.
5399
+ *
5400
+ Null is the RIGHT value there: the alternative that comment rejects is
5401
+ letting the worker inherit the user's own agent config, where a second
5402
+ CTRL+SPC server pointed at a DIFFERENT ACCOUNT was once found registered and
5403
+ listening. That is worse. But there was a third option nobody took, which is
5404
+ to not dispatch at all.
5405
+ *
5406
+ WAITING IS FREE AND CORRECT. The request keeps its place, unclaimed and
5407
+ unspent, and the tools server comes up moments later — the port race
5408
+ resolves as soon as the old process lets go. A dispatch made in this window
5409
+ cannot be retried usefully, because it is not the request that is broken.
5410
+ *
5411
+ BEFORE THE CLAIM, deliberately: claiming and then refusing to spawn would
5412
+ spend `attempts` on a condition that has nothing to do with the request.
5413
+ *
5414
+ THE CONDITION IS "A REAL SPAWN WITH NO SERVER", not "no server". An injected
5415
+ `spawn` is a caller who has taken over the launching entirely — every test in
5416
+ the suite, and any future embedder — and for them `mcpServer` describes
5417
+ nothing, because `defaultSpawn` is what turns it into an argv. Gating on the
5418
+ null alone would refuse to dispatch in exactly the cases where the tools are
5419
+ not in question, which is not a stricter check but a wrong one. */
5420
+ if (!deps.mcpServer && !deps.spawn) {
5421
+ reportNoTools(state, warn);
5422
+ return 'no-tools';
5426
5423
  }
5427
- /* ═══ THE TWO REASONS WORK WAITS, ASKED IN ORDER. ═══
5428
-
5429
- A DEPENDENCY IS ASKED FIRST, and the order is meaningful rather than
5430
- incidental: a dependency is a FACT about what must happen before what,
5431
- while a collision is a CHOICE to serialise safely. When both are true the
5432
- user is better told the one that will not change on its own — a collision
5433
- clears the moment the other agent releases, a dependency does not clear
5434
- until the blocker is finished.
5435
-
5436
- Slice 3 adds the second. Both hold the SAME WAY (release, skip, report
5437
- once), so the shared tail below is written once rather than twice. */
5438
- const blocker = await blockerForTodo(client, claim.id, warn);
5439
- /* THE CODEBASE COMES FROM `resolveContext`, the SAME resolver the dispatch
5440
- uses to choose the worker's working directory not a second lookup that
5441
- could disagree with it. A reservation is scoped by `git_remote_url`, so
5442
- "which repo would this run touch" and "which repo will it start in" have
5443
- to be one answer. Only reached when there is no dependency, so an ordinary
5444
- dispatchable request costs one extra read, not two. */
5445
- /* ═══ 18b SLICE 4 THE CODEBASE THE USER CHOSE, AND THE FAN-OUT. ═══
5446
-
5447
- Read BEFORE the context is resolved, because it is an input to it: given a
5448
- choice, the resolver treats the project as if that one codebase were its
5449
- only one, and the ambiguity never arises.
5450
-
5451
- MORE THAN ONE CHOSEN MEANS MORE THAN ONE RUN (user ruling, 2026-08-07).
5452
- This tick dispatches the FIRST; `spawnSiblingRuns` below creates a sibling
5453
- request for each of the rest, which the daemon then claims on its own
5454
- ticks exactly like any other request. Each run carries ONE codebase and
5455
- starts in that codebase's own checkout, wherever it is on disk — nothing
5456
- here reads folder layout, which is the whole of the location ruling.
5457
-
5458
- Nothing changes for a request that was never asked: `chosen` is empty, the
5459
- claim carries null, and every path below behaves as it did in Slice 2. */
5460
- /* A SIBLING ALREADY KNOWS ITS OWN, and asks nothing. `chosen_codebase` comes
5461
- down on the claim itself (this slice's migration), so a fanned-out run
5462
- needs no ask to read and cannot fan out again — which is what stops the
5463
- fan-out being infinite. Only a request that was ASKED reads the answer. */
5464
- const chosen = blocker
5465
- ? []
5466
- : claim.chosenCodebase
5467
- ? [claim.chosenCodebase]
5468
- : await readChosenCodebases(client, claim.id, warn);
5469
- const runClaim = chosen.length > 0
5470
- ? { ...claim, chosenCodebase: chosen[0] }
5471
- : claim;
5472
- const context = blocker ? null : await resolveContext(runClaim);
5473
- const collision = context
5474
- ? await collisionForTodo(client, claim.id, context.gitRemoteUrl, warn)
5475
- : null;
5476
- if (!blocker && !collision) {
5477
- /* NO LONGER WAITING, so the next time it does wait it is reported again.
5478
- Without this, a unit that blocked, ran, and blocked again on something
5479
- else would stay silent forever. */
5480
- state.reportedWaiting.delete(claim.id);
5481
- /* THE RESOLVED CONTEXT IS HANDED ON rather than resolved again. It reads
5482
- several tables and touches the filesystem, and the dispatch needs the
5483
- very same answer — resolving twice would both cost a second pass and
5484
- allow the collision check and the working directory to disagree. */
5485
- /* THE SIBLINGS ARE CREATED BEFORE THIS RUN DISPATCHES, so a user who chose
5486
- three codebases sees three cards immediately rather than one that
5487
- mysteriously multiplies later. Best-effort: a failed sibling costs that
5488
- codebase its run, and is logged; it must not cost the user the run that
5489
- is about to start. */
5490
- if (chosen.length > 1) {
5491
- await spawnSiblingRuns(client, claim, chosen.slice(1), log, warn);
5424
+ state.warnedNoTools = false;
5425
+ /* ═══ !Cleanup PHASE 7 SLICE 2: CLAIM PAST THE BLOCKED ONES. ═══
5426
+ *
5427
+ A dependent unit must WAIT, and everything behind it must NOT. Those two
5428
+ requirements together are why this is a loop rather than a single claim.
5429
+ *
5430
+ The claim is this design's only queue position, and it takes the OLDEST
5431
+ claimable row. So a blocked unit that is merely released is re-claimed FIRST
5432
+ on the very next tick, forever — it stops being a waiting unit and becomes a
5433
+ queue stopper. Releasing it AND telling the next claim to skip it is what
5434
+ makes the rest of the queue reachable.
5435
+ *
5436
+ BOUNDED BY THE NUMBER OF UNITS SKIPPED, not by a fixed count: each iteration
5437
+ either dispatches (returns), finds nothing (returns), or adds exactly one id
5438
+ to `skip`, and a skipped id can never be claimed again this tick. So the
5439
+ loop runs at most once per blocked unit and then ends. No timeout needed,
5440
+ because it cannot revisit a row.
5441
+ *
5442
+ THE UNIT STAYS `working` AND UNCLAIMED, deliberately it is NOT parked to
5443
+ `needs-input` the way the scope gate parks. `needs-input` means "this needs
5444
+ YOU", and a dependency needs nothing from the user: it resolves when the
5445
+ blocker finishes. Parking it would put a card under "Needs you" offering no
5446
+ action, which is precisely the defect Phase 1 existed to remove. The panel
5447
+ reads the same `blocked_tasks` view and renders "Waiting on <name>" from it,
5448
+ so the card explains itself without a state that lies. */
5449
+ const skip = [];
5450
+ let blockedCount = 0;
5451
+ for (;;) {
5452
+ let claim;
5453
+ try {
5454
+ claim = await claimNextTodo(client, machineId, role.agent, [...skip, ...legacyHeldTodoIds()], !!recovered);
5455
+ if (claim?.claimOperationId)
5456
+ claimOperations.push(claim.claimOperationId);
5492
5457
  }
5493
- return await dispatchClaim(deps, state, now, log, warn, role, runClaim, context, chosen);
5494
- }
5495
- /* SAID ONCE PER (unit, REASON) — see `reportedWaiting`. The tick runs every
5496
- 3s and this branch is reached on every one of them for as long as the wait
5497
- holds. The key carries the reason, not just the unit, so a request that
5498
- stops waiting on a dependency and starts waiting on a FILE says so. */
5499
- const waitKey = blocker
5500
- ? `dep:${blocker.dependsOnId}`
5501
- : `path:${collision.heldBySessionId}:${collision.path}`;
5502
- const waitLine = blocker
5503
- ? `is waiting on "${blocker.dependsOnName}"`
5504
- : `is waiting on the same files as "${collision.heldByInstruction ?? 'another run'}"`
5505
- + ` (${collision.path}${collision.pathCount > 1 ? ` and ${collision.pathCount - 1} more` : ''})`;
5506
- if (state.reportedWaiting.get(claim.id) !== waitKey) {
5507
- state.reportedWaiting.set(claim.id, waitKey);
5508
- log(`[orchestrator] "${claim.instruction}" (${shortId(claim.id)}) ${waitLine} not dispatching`);
5458
+ catch (err) {
5459
+ warn(`orchestrator poll failed, will retry: ${err.message}`);
5460
+ return 'failed';
5461
+ }
5462
+ if (!claim) {
5463
+ /* Nothing claimable. If we skipped something, the queue is not empty
5464
+ it is WAITING, which is a different fact and must not be logged as
5465
+ idleness. */
5466
+ return blockedCount > 0 ? 'blocked' : 'nothing-to-do';
5467
+ }
5468
+ /* ═══ THE TWO REASONS WORK WAITS, ASKED IN ORDER. ═══
5469
+ *
5470
+ A DEPENDENCY IS ASKED FIRST, and the order is meaningful rather than
5471
+ incidental: a dependency is a FACT about what must happen before what,
5472
+ while a collision is a CHOICE to serialise safely. When both are true the
5473
+ user is better told the one that will not change on its own — a collision
5474
+ clears the moment the other agent releases, a dependency does not clear
5475
+ until the blocker is finished.
5476
+ *
5477
+ Slice 3 adds the second. Both hold the SAME WAY (release, skip, report
5478
+ once), so the shared tail below is written once rather than twice. */
5479
+ const blocker = await blockerForTodo(client, claim.id, warn);
5480
+ /* THE CODEBASE COMES FROM `resolveContext`, the SAME resolver the dispatch
5481
+ uses to choose the worker's working directory — not a second lookup that
5482
+ could disagree with it. A reservation is scoped by `git_remote_url`, so
5483
+ "which repo would this run touch" and "which repo will it start in" have
5484
+ to be one answer. Only reached when there is no dependency, so an ordinary
5485
+ dispatchable request costs one extra read, not two. */
5486
+ /* ═══ 18b SLICE 4 — THE CODEBASE THE USER CHOSE, AND THE FAN-OUT. ═══
5487
+ *
5488
+ Read BEFORE the context is resolved, because it is an input to it: given a
5489
+ choice, the resolver treats the project as if that one codebase were its
5490
+ only one, and the ambiguity never arises.
5491
+ *
5492
+ MORE THAN ONE CHOSEN MEANS MORE THAN ONE RUN (user ruling, 2026-08-07).
5493
+ This tick dispatches the FIRST; `spawnSiblingRuns` below creates a sibling
5494
+ request for each of the rest, which the daemon then claims on its own
5495
+ ticks exactly like any other request. Each run carries ONE codebase and
5496
+ starts in that codebase's own checkout, wherever it is on disk — nothing
5497
+ here reads folder layout, which is the whole of the location ruling.
5498
+ *
5499
+ Nothing changes for a request that was never asked: `chosen` is empty, the
5500
+ claim carries null, and every path below behaves as it did in Slice 2. */
5501
+ /* A SIBLING ALREADY KNOWS ITS OWN, and asks nothing. `chosen_codebase` comes
5502
+ down on the claim itself (this slice's migration), so a fanned-out run
5503
+ needs no ask to read and cannot fan out again — which is what stops the
5504
+ fan-out being infinite. Only a request that was ASKED reads the answer. */
5505
+ const chosen = blocker
5506
+ ? []
5507
+ : claim.chosenCodebase
5508
+ ? [claim.chosenCodebase]
5509
+ : await readChosenCodebases(client, claim.id, warn);
5510
+ const runClaim = chosen.length > 0
5511
+ ? { ...claim, chosenCodebase: chosen[0] }
5512
+ : claim;
5513
+ const context = blocker ? null : await resolveContext(runClaim);
5514
+ const collision = context
5515
+ ? await collisionForTodo(client, claim.id, context.gitRemoteUrl, warn)
5516
+ : null;
5517
+ if (!blocker && !collision) {
5518
+ /* NO LONGER WAITING, so the next time it does wait it is reported again.
5519
+ Without this, a unit that blocked, ran, and blocked again on something
5520
+ else would stay silent forever. */
5521
+ state.reportedWaiting.delete(claim.id);
5522
+ /* THE RESOLVED CONTEXT IS HANDED ON rather than resolved again. It reads
5523
+ several tables and touches the filesystem, and the dispatch needs the
5524
+ very same answer — resolving twice would both cost a second pass and
5525
+ allow the collision check and the working directory to disagree. */
5526
+ /* THE SIBLINGS ARE CREATED BEFORE THIS RUN DISPATCHES, so a user who chose
5527
+ three codebases sees three cards immediately rather than one that
5528
+ mysteriously multiplies later. Best-effort: a failed sibling costs that
5529
+ codebase its run, and is logged; it must not cost the user the run that
5530
+ is about to start. */
5531
+ if (chosen.length > 1) {
5532
+ await spawnSiblingRuns(client, claim, chosen.slice(1), log, warn);
5533
+ }
5534
+ const dispatched = dispatchClaim(deps, state, now, log, warn, role, runClaim, context, chosen);
5535
+ if (hasOwnedWorkContext())
5536
+ finishRpcClaims(claimOperations, true);
5537
+ endClaim();
5538
+ return await dispatched;
5539
+ }
5540
+ /* SAID ONCE PER (unit, REASON) — see `reportedWaiting`. The tick runs every
5541
+ 3s and this branch is reached on every one of them for as long as the wait
5542
+ holds. The key carries the reason, not just the unit, so a request that
5543
+ stops waiting on a dependency and starts waiting on a FILE says so. */
5544
+ const waitKey = blocker
5545
+ ? `dep:${blocker.dependsOnId}`
5546
+ : `path:${collision.heldBySessionId}:${collision.path}`;
5547
+ const waitLine = blocker
5548
+ ? `is waiting on "${blocker.dependsOnName}"`
5549
+ : `is waiting on the same files as "${collision.heldByInstruction ?? 'another run'}"`
5550
+ + ` (${collision.path}${collision.pathCount > 1 ? ` and ${collision.pathCount - 1} more` : ''})`;
5551
+ if (state.reportedWaiting.get(claim.id) !== waitKey) {
5552
+ state.reportedWaiting.set(claim.id, waitKey);
5553
+ log(`[orchestrator] "${claim.instruction}" (${shortId(claim.id)}) ${waitLine} — not dispatching`);
5554
+ }
5555
+ /* RELEASED, so that the moment the blocker is done the ordinary claim picks
5556
+ it up with no user action and no special path. That is the Gherkin line
5557
+ "123.1 starts without my intervention", and it is why this must not keep
5558
+ the claim the way `parkTodoForScope` does. */
5559
+ await releaseTodo(client, claim.id, claim.claimedAt).catch((err) => {
5560
+ warn(`could not release a blocked todo: ${err.message}`);
5561
+ });
5562
+ skip.push(claim.id);
5563
+ blockedCount += 1;
5509
5564
  }
5510
- /* RELEASED, so that the moment the blocker is done the ordinary claim picks
5511
- it up with no user action and no special path. That is the Gherkin line
5512
- "123.1 starts without my intervention", and it is why this must not keep
5513
- the claim the way `parkTodoForScope` does. */
5514
- await releaseTodo(client, claim.id).catch((err) => {
5515
- warn(`could not release a blocked todo: ${err.message}`);
5516
- });
5517
- skip.push(claim.id);
5518
- blockedCount += 1;
5565
+ }
5566
+ finally {
5567
+ if (hasOwnedWorkContext())
5568
+ finishRpcClaims(claimOperations, true);
5569
+ endClaim();
5519
5570
  }
5520
5571
  }
5521
5572
  /**
@@ -5549,10 +5600,14 @@ async function runTick(deps, state, now, log, warn) {
5549
5600
  * returns is not a funnel.
5550
5601
  */
5551
5602
  async function dispatchClaim(deps, state, now, log, warn, role, claim, context, chosenCodebases = []) {
5603
+ const owned = hasOwnedWorkContext() ? reserveOwnedWork({ surface: 'worker', todoId: claim.id, workerId: null, claimedAt: claim.claimedAt, observedPendingMessageIds: null }, role.agent) : undefined;
5552
5604
  try {
5553
- return await dispatchClaimInner(deps, state, now, log, warn, role, claim, context, chosenCodebases);
5605
+ const outcome = await dispatchClaimInner(deps, state, now, log, warn, role, claim, context, chosenCodebases, owned);
5606
+ owned?.complete();
5607
+ return outcome;
5554
5608
  }
5555
5609
  finally {
5610
+ owned?.finishPreparation();
5556
5611
  /* THE RUN IS OVER, SO IT HOLDS NOTHING, however it ended and wherever it
5557
5612
  returned from. Best-effort by contract: `releaseTodoReservations` never
5558
5613
  throws, so this cannot change what the body returned, and a `finally`
@@ -5567,6 +5622,57 @@ async function dispatchClaim(deps, state, now, log, warn, role, claim, context,
5567
5622
  forgetSecrets(claim.id);
5568
5623
  }
5569
5624
  }
5625
+ async function settleWorkerOutcome(client, workerId, claim, role, result, targeted, now, startedAt, log, warn, saved, persist, onAccepted) {
5626
+ let disposition = saved?.disposition;
5627
+ if (!disposition) {
5628
+ if (!result.ok) {
5629
+ const kind = result.failureKind ?? nativeFailureKind(result.saidTail ?? '');
5630
+ const reason = failureMessage(role.agent, kind);
5631
+ disposition = { worker: 'died', reason, todo: kind === 'transient' ? 'retry' : 'failed' };
5632
+ }
5633
+ else if (await todoHasOpenAsk(client, claim.id, warn, true)) {
5634
+ disposition = { worker: 'done', reason: '', todo: 'needs-input' };
5635
+ }
5636
+ else if (!await todoRunLeftATrace(client, claim.id, warn, targeted, true)) {
5637
+ disposition = { worker: 'died', reason: NOTHING_RECORDED_REASON, todo: 'retry' };
5638
+ }
5639
+ else
5640
+ disposition = { worker: 'done', reason: '', todo: 'done' };
5641
+ if (saved) {
5642
+ saved.disposition = disposition;
5643
+ persist?.(saved);
5644
+ }
5645
+ }
5646
+ if (disposition.todo === 'needs-input') {
5647
+ const accepted = await suspendTodoForAsk(client, claim.id, result.output, claim.claimedAt);
5648
+ if (accepted)
5649
+ onAccepted?.();
5650
+ await updateWorker(client, workerId, 'done', null, warn, true);
5651
+ log(`[orchestrator] ${shortId(claim.id)} is waiting on the user`);
5652
+ return 'answered';
5653
+ }
5654
+ if (disposition.todo === 'done') {
5655
+ const accepted = await answerTodo(client, claim.id, result.output, claim.claimedAt);
5656
+ if (accepted)
5657
+ onAccepted?.();
5658
+ await updateWorker(client, workerId, 'done', null, warn, true);
5659
+ log(`[orchestrator] answered ${shortId(claim.id)} in ${((now().getTime() - startedAt) / 1000).toFixed(1)}s`);
5660
+ return 'answered';
5661
+ }
5662
+ await updateWorker(client, workerId, 'died', disposition.reason, warn, true);
5663
+ const { fate, attempts } = disposition.todo === 'retry'
5664
+ ? await releaseOrFail(client, claim.id, disposition.reason, now().getTime(), claim.claimedAt)
5665
+ : await failClaimedTodo(client, claim.id, disposition.reason, claim.claimedAt ?? null);
5666
+ if (fate === 'stale')
5667
+ return 'nothing-to-do';
5668
+ onAccepted?.();
5669
+ if (fate === 'failed') {
5670
+ warn(`[orchestrator] ${shortId(claim.id)} failed: ${disposition.reason}`);
5671
+ return 'failed';
5672
+ }
5673
+ warn(`[orchestrator] retrying ${shortId(claim.id)} in ${Math.round(retryDelayMs(attempts) / 1000)}s`);
5674
+ return 'released';
5675
+ }
5570
5676
  async function dispatchClaimInner(deps, state, now, log, warn, role, claim,
5571
5677
  /* !Cleanup Phase 7 Slice 3 — RESOLVED BY THE CALLER, and passed in rather than
5572
5678
  resolved again here. `runTick` needs the same answer first, to know which
@@ -5577,7 +5683,7 @@ context,
5577
5683
  /** 18b Slice 4 — every codebase the user chose, including this run's own. The
5578
5684
  * siblings are derived from it below; empty on every request that was never
5579
5685
  * asked, which is the ordinary case. */
5580
- chosenCodebases = []) {
5686
+ chosenCodebases = [], owned) {
5581
5687
  const { client, machineId, mcpServer = null, spawn = defaultSpawn(mcpServer), } = deps;
5582
5688
  /* ═══ 16g SLICE 1 — THE SECOND ENFORCEMENT POINT. ═══
5583
5689
  `No approved scope-version pointer → no execution write and no coding lane.`
@@ -5638,7 +5744,7 @@ chosenCodebases = []) {
5638
5744
  uses the mirror of this: answering an ask CLEARS the claim to make the
5639
5745
  work dispatchable again — so approving scope re-dispatches through the
5640
5746
  path that already exists, with no new mechanism. */
5641
- await parkTodoForScope(client, claim.id).catch((err) => {
5747
+ await parkTodoForScope(client, claim.id, claim.claimedAt).catch((err) => {
5642
5748
  warn(`could not park an unapproved todo: ${err.message}`);
5643
5749
  });
5644
5750
  return 'nothing-to-do';
@@ -5665,7 +5771,7 @@ chosenCodebases = []) {
5665
5771
  hold a dead end too. Building stays held exactly as shipped. */
5666
5772
  if (targeted && building && await todoIsHeldForScope(client, claim.id)) {
5667
5773
  log(`[orchestrator] "${claim.instruction}" has an unresolved scope change — not dispatching`);
5668
- await parkTodoForScope(client, claim.id).catch((err) => {
5774
+ await parkTodoForScope(client, claim.id, claim.claimedAt).catch((err) => {
5669
5775
  warn(`could not park a held todo: ${err.message}`);
5670
5776
  });
5671
5777
  return 'nothing-to-do';
@@ -5682,6 +5788,7 @@ chosenCodebases = []) {
5682
5788
  let workerId = null;
5683
5789
  /* !Cleanup Phase 6b (I43) — outside the try for the same reason: the `finally`
5684
5790
  has to remove it from `livePids` however this run ended. */
5791
+ let terminalDeferred = false;
5685
5792
  let spawnedPid = null;
5686
5793
  try {
5687
5794
  log(`[orchestrator] picked up "${claim.instruction}" (${shortId(claim.id)})`);
@@ -5801,7 +5908,7 @@ chosenCodebases = []) {
5801
5908
  with the words it had in Slice 2: never parked with no question on it,
5802
5909
  which the user could only clear with Stop. */
5803
5910
  if (context.miss?.kind === 'ambiguous-codebase' && instructionNeedsCheckout(claim.instruction)) {
5804
- const asked = await askWhichCodebase(client, claim.id, context.miss.projectName, context.miss.gitRemoteUrls, warn);
5911
+ const asked = await askWhichCodebase(client, claim.id, context.miss.projectName, context.miss.gitRemoteUrls, warn, claim.claimedAt);
5805
5912
  if (asked) {
5806
5913
  log(`[orchestrator] asking which codebase "${claim.instruction}" should run in`);
5807
5914
  await updateWorker(client, workerId, 'died', 'waiting for the user to choose a codebase', warn);
@@ -5811,7 +5918,7 @@ chosenCodebases = []) {
5811
5918
  if (context.miss && instructionNeedsCheckout(claim.instruction)) {
5812
5919
  log(`[orchestrator] "${claim.instruction}" needs the code, which is not on this machine`);
5813
5920
  await updateWorker(client, workerId, 'died', 'the codebase is not available on this machine', warn);
5814
- await answerTodo(client, claim.id, checkoutMissAnswer(context.miss));
5921
+ await answerTodo(client, claim.id, checkoutMissAnswer(context.miss), claim.claimedAt);
5815
5922
  return 'answered';
5816
5923
  }
5817
5924
  /* ═══ 18d SLICE 4 — ONE STAGE PER DISPATCH (user ruling, 2026-08-12). ═══
@@ -5823,6 +5930,15 @@ chosenCodebases = []) {
5823
5930
  if (stage) {
5824
5931
  log(`[orchestrator] ${shortId(claim.id)} is working stage ${stage.position}/${stage.total}: ${stage.title}`);
5825
5932
  }
5933
+ if (owned) {
5934
+ if (!claim.claimedAt || !workerId)
5935
+ throw new Error('The worker has no exact attempt identity.');
5936
+ const pending = await client.from('cliv2_run_messages').select('id').eq('todo_id', claim.id).is('answered_by', null);
5937
+ if (pending.error)
5938
+ throw pending.error;
5939
+ owned.setReference({ surface: 'worker', todoId: claim.id, workerId, claimedAt: claim.claimedAt,
5940
+ observedPendingMessageIds: (pending.data ?? []).map((row) => row.id) });
5941
+ }
5826
5942
  const result = await spawn(role.agent, buildPrompt(claim.instruction, claim.id, answered, targeted && scopeVersion ? { workItemId: targeted, version: scopeVersion } : null, replies, context, planningOnly, interrupted, selectedTools, stage, directed), (step) => steps.offer(step), context.checkout,
5827
5943
  // !Cleanup Phase 6 (I17) — polled by the child; true kills it.
5828
5944
  () => todoWasStopped(client, claim.id),
@@ -5850,118 +5966,44 @@ chosenCodebases = []) {
5850
5966
  not the prompt, so report_activity attributes without depending on
5851
5967
  agent cooperation (ux.md § Corrections: codex dropped a prompt-carried
5852
5968
  id twice). */
5853
- claim.id);
5969
+ claim.id, owned);
5854
5970
  /* !Cleanup PHASE 6 (I17) — THE USER STOPPED IT, and that is not a failure.
5855
5971
  Checked BEFORE `!result.ok`, because a killed child also reports not-ok:
5856
5972
  falling through would count the user's own decision as an attempt and
5857
5973
  retry the very thing they cancelled. Nothing is written but the claim
5858
5974
  release — the browser already set `stopped`. */
5975
+ if (result.interrupted)
5976
+ return 'nothing-to-do';
5859
5977
  if (result.stopped) {
5860
- await updateWorker(client, workerId, 'died', 'stopped by the user', warn);
5861
- await releaseStoppedTodo(client, claim.id);
5978
+ terminalDeferred = !!owned;
5979
+ owned?.deferOutcome({ surface: 'worker', ok: false, text: null, reason: 'stopped by the user', stopped: true });
5980
+ await updateWorker(client, workerId, 'died', 'stopped by the user', warn, true);
5981
+ await releaseStoppedTodo(client, claim.id, claim.claimedAt);
5982
+ owned?.acknowledgeOutcome();
5983
+ terminalDeferred = false;
5862
5984
  log(`[orchestrator] ${shortId(claim.id)} was stopped by the user`);
5863
5985
  return 'released';
5864
5986
  }
5865
- if (!result.ok) {
5866
- /* THE WORKER DIED. Its row says so, WITH the reason, before the unit is
5867
- released so a panel watching this moment sees "worker stopped" rather
5868
- than a unit that silently went back to unclaimed. Order matters: the
5869
- release is what makes the todo claimable again, so anything that must
5870
- be true about the dead worker has to be written first. */
5871
- /* 18a SLICE 5: THE WORKER ROW CARRIES THE PLAIN REASON, because that row
5872
- is what the card reads BETWEEN attempts. `result.error` stays the log's
5873
- string; the stored one is the sentence. */
5874
- const plain = plainFailureReason(role.agent, result.error, result.saidTail ?? '');
5875
- await updateWorker(client, workerId, 'died', plain, warn);
5876
- /* ux.md state D — released, not left claimed and silent. !Cleanup Phase 6
5877
- (I18): released ONLY while attempts remain. At the bound this writes
5878
- `failed` with the reason instead, which is what stops the unit being
5879
- re-claimed forever while reading as Working. */
5880
- const { fate, attempts } = await releaseOrFail(client, claim.id, plain, now().getTime(), claim.claimedAt);
5881
- if (fate === 'failed') {
5882
- warn(`[orchestrator] ${result.error} — ${shortId(claim.id)} failed after ${MAX_ATTEMPTS} attempts`);
5883
- return 'failed';
5884
- }
5885
- /* 18a SLICE 6: THIS REQUEST WAITS, AND NOTHING ELSE DOES. The wait went
5886
- onto the row with the attempt count; what is left here is saying so, in
5887
- the line that used to say only "releasing". */
5888
- const seconds = Math.round(retryDelayMs(attempts) / 1000);
5889
- warn(`[orchestrator] ${result.error}: retrying ${shortId(claim.id)} in ${seconds}s`);
5890
- return 'released';
5891
- }
5892
- /* !Cleanup PHASE 6 (I15) — A QUESTION IS NOT A FINISHED JOB.
5893
- The child exited cleanly, which until now meant `done` unconditionally.
5894
- But a one-shot headless run ASKS AND EXITS — that is the only way it can
5895
- handle a question — so the clean exit says nothing about whether the work
5896
- is over. If an ask is open on this unit, the truthful state is that it
5897
- needs the user, and the Inbox already has the question. */
5898
- if (await todoHasOpenAsk(client, claim.id, warn)) {
5899
- await suspendTodoForAsk(client, claim.id, result.output, claim.claimedAt);
5900
- await updateWorker(client, workerId, 'done', null, warn);
5901
- log(`[orchestrator] ${shortId(claim.id)} is waiting on the user`);
5902
- return 'answered';
5903
- }
5904
- /* A CONFIDENT ANSWER WITH NOTHING BEHIND IT IS NOT A FINISHED JOB.
5905
- See `todoRunLeftATrace` for the measurement: a codex worker that was
5906
- correctly configured, connected, and fetched the tool list still reported
5907
- that it had no tools, twice in three identical runs, and one reported
5908
- ATTACHING a screenshot it never attached. Every one of them exited 0 with
5909
- prose on stdout, which is all the two checks above look at, so every one
5910
- of them landed here and was written `done`.
5911
-
5912
- PLACED AFTER THE ASK CHECK, and that order is load-bearing. A run that
5913
- asked a question and exited has legitimately recorded something and is
5914
- already handled as `needs-input`; reaching this check first would relabel
5915
- the product's most important state as a failure.
5916
-
5917
- RELEASED FOR RETRY RATHER THAN FAILED OUTRIGHT, through the SAME
5918
- `releaseOrFail` the dead-worker path uses. This is intermittent by nature
5919
- (the third run of the identical prompt worked perfectly), so a retry is
5920
- very likely to succeed, and the attempt bound is what stops it looping
5921
- forever. At the bound the card says why, in words about what is KNOWN
5922
- (nothing was recorded) rather than repeating the agent's own account of a
5923
- run that has just been shown to be an unreliable narrator.
5924
-
5925
- `targeted` IS PASSED, and without it this check is blind to the primary
5926
- flow. A session opened on a work item has `todo_id` NULL by the table's
5927
- own one-anchor constraint, and the prompt tells the agent to open it that
5928
- way whenever there is an item, so a `todo_id`-only lookup could never
5929
- match a targeted run. See `todoRunLeftATrace`. It is the same value the
5930
- scope gate above already resolved, so the two cannot disagree about which
5931
- item this request is for. */
5932
- if (!await todoRunLeftATrace(client, claim.id, warn, targeted)) {
5933
- /* THE WORKER DIED WITH NO STEP, and that blank is load-bearing rather
5934
- than incidental. `interruptedRunStep` looks up the most recent `died`
5935
- worker on this request and `describeInterruptedRun` then tells the
5936
- REPLACEMENT agent that a previous attempt "stopped partway" and that
5937
- anything it did "IS STILL THERE". For a genuinely toolless run that is
5938
- true and useful. Clearing the step keeps the claim honest about the one
5939
- thing that is known: this run recorded nothing, so there is no last
5940
- action to hand on and nothing for the next agent to resume from. */
5941
- await updateWorker(client, workerId, 'died', NOTHING_RECORDED_REASON, warn);
5942
- const { fate, attempts } = await releaseOrFail(client, claim.id, NOTHING_RECORDED_REASON, now().getTime(), claim.claimedAt);
5943
- if (fate === 'failed') {
5944
- warn(`[orchestrator] ${shortId(claim.id)} recorded nothing and failed after ${MAX_ATTEMPTS} attempts`);
5945
- return 'failed';
5946
- }
5947
- const seconds = Math.round(retryDelayMs(attempts) / 1000);
5948
- warn(`[orchestrator] ${shortId(claim.id)} recorded nothing: retrying in ${seconds}s`);
5949
- return 'released';
5950
- }
5951
- /* FENCED ON THE CLAIM THIS DISPATCH TOOK (18d Slice 4). If this run handed
5952
- its stage on, the claim is already cleared and a fresh worker is on the
5953
- row, so this write must do nothing rather than mark the request finished
5954
- over work that is still running. See answerTodo's own comment. */
5955
- await answerTodo(client, claim.id, result.output, claim.claimedAt);
5956
- // The worker finished and its answer is committed. Marked done AFTER the
5957
- // answer write, so a worker is never reported finished for a unit that has
5958
- // no answer on it.
5959
- await updateWorker(client, workerId, 'done', null, warn);
5960
- const seconds = ((now().getTime() - startedAt) / 1000).toFixed(1);
5961
- log(`[orchestrator] answered ${shortId(claim.id)} in ${seconds}s`);
5962
- return 'answered';
5987
+ terminalDeferred = !!owned;
5988
+ const terminal = { surface: 'worker', ok: result.ok, text: result.ok ? result.output : null,
5989
+ reason: result.ok ? '' : failureMessage(role.agent, result.failureKind ?? nativeFailureKind(result.saidTail ?? '')),
5990
+ failureKind: result.failureKind ?? nativeFailureKind(result.saidTail ?? ''), workItemId: targeted };
5991
+ owned?.deferOutcome(terminal);
5992
+ const settled = await settleWorkerOutcome(client, workerId, claim, role, result, targeted, now, startedAt, log, warn, terminal, owned?.deferOutcome, () => {
5993
+ if (result.ok)
5994
+ deps.observeHarness?.(role.agent, 'authenticated', 'dispatch-success');
5995
+ else if (result.failureKind === 'authentication')
5996
+ deps.observeHarness?.(role.agent, 'sign-in-required', 'provider-rejected');
5997
+ });
5998
+ owned?.acknowledgeOutcome();
5999
+ terminalDeferred = false;
6000
+ return settled;
5963
6001
  }
5964
6002
  catch (err) {
6003
+ if (terminalDeferred) {
6004
+ warn('The completed execution is waiting for its original account to acknowledge the result.');
6005
+ return 'failed';
6006
+ }
5965
6007
  // The answer write itself failed (network, RLS, a reverted migration). The
5966
6008
  // todo is CLAIMED and has no answer — the worst state — so release it here
5967
6009
  // too. A release that also fails is warned and the row is left claimed;
@@ -5979,6 +6021,8 @@ chosenCodebases = []) {
5979
6021
  repeatedly by a durable cause — a reverted migration, an RLS change —
5980
6022
  and before the bound it was the same forever-loop as a failed spawn. */
5981
6023
  const { fate, attempts } = await releaseOrFail(client, claim.id, err.message, now().getTime(), claim.claimedAt);
6024
+ if (fate === 'stale')
6025
+ return 'nothing-to-do';
5982
6026
  warn(fate === 'failed'
5983
6027
  ? `[orchestrator] ${shortId(claim.id)} failed after ${MAX_ATTEMPTS} attempts`
5984
6028
  : `[orchestrator] retrying ${shortId(claim.id)} in ${Math.round(retryDelayMs(attempts) / 1000)}s`);
@@ -6009,3 +6053,276 @@ chosenCodebases = []) {
6009
6053
  export function liveAgents() {
6010
6054
  return detectAgents();
6011
6055
  }
6056
+ function cliClaimReferences(action, result) {
6057
+ if (!Array.isArray(result))
6058
+ throw new Error('The claim journal returned no complete result.');
6059
+ return result.map((row) => {
6060
+ if (action === 'cliv2_claim_next_loose_todo') {
6061
+ const attempt = row?._attempt;
6062
+ if (!attempt || typeof attempt.todo_id !== 'string' || typeof attempt.claimed_at !== 'string')
6063
+ throw new Error('The claim journal returned an incomplete request identity.');
6064
+ return { surface: 'worker', todoId: attempt.todo_id, workerId: null, claimedAt: attempt.claimed_at, observedPendingMessageIds: null };
6065
+ }
6066
+ if (typeof row?.id !== 'string' || typeof row?.todo_id !== 'string')
6067
+ throw new Error('The claim journal returned an incomplete reply source.');
6068
+ if (action === 'cliv2_take_run_messages' && typeof row.taken_at === 'string')
6069
+ return {
6070
+ surface: 'reply', todoId: row.todo_id, messageLeases: [{ id: row.id, taken_at: row.taken_at }], grantLeases: [], observedPendingMessageIds: null,
6071
+ };
6072
+ if (action === 'cliv2_take_granted_resumes' && typeof row.resume_taken_at === 'string')
6073
+ return {
6074
+ surface: 'reply', todoId: row.todo_id, messageLeases: [], grantLeases: [{ id: row.id, resume_taken_at: row.resume_taken_at }], observedPendingMessageIds: null,
6075
+ };
6076
+ throw new Error('The claim journal returned an unrecognized lease identity.');
6077
+ });
6078
+ }
6079
+ async function recoveredCliClaim(client, action, args) {
6080
+ const skip = (args.p_skip_ids ?? args.p_skip_todo_ids ?? []);
6081
+ for (const claim of recoveredRpcClaims('cli').filter(row => row.action === action)) {
6082
+ if (action === 'cliv2_claim_next_loose_todo' && claim.args?.p_agent !== args.p_agent)
6083
+ continue;
6084
+ const rows = [];
6085
+ for (const row of claim.result ?? []) {
6086
+ const ref = cliClaimReferences(action, [row])[0];
6087
+ if (ref.surface === 'panel' || !pendingRpcReference(ref))
6088
+ continue;
6089
+ if (skip.includes(ref.todoId) || Array.isArray(args.p_todo_ids) && !args.p_todo_ids.includes(ref.todoId))
6090
+ continue;
6091
+ let current = false;
6092
+ if (ref.surface === 'worker') {
6093
+ const read = await client.from('cliv2_loose_todos').select('state,claimed_at,claimed_machine_id').eq('id', ref.todoId).maybeSingle();
6094
+ if (read.error)
6095
+ throw read.error;
6096
+ current = !!read.data && read.data.claimed_at === ref.claimedAt && read.data.claimed_machine_id === args.p_machine_id && ['working', 'needs-input'].includes(read.data.state);
6097
+ }
6098
+ else if (action === 'cliv2_take_run_messages') {
6099
+ const source = ref.messageLeases[0];
6100
+ const read = await client.from('cliv2_run_messages').select('taken_at,taken_machine_id,answered_by,interrupted_at').eq('id', source.id).maybeSingle();
6101
+ if (read.error)
6102
+ throw read.error;
6103
+ current = !!read.data && read.data.taken_at === source.taken_at && read.data.taken_machine_id === args.p_machine_id && !read.data.answered_by && !read.data.interrupted_at;
6104
+ }
6105
+ else {
6106
+ const source = ref.grantLeases[0];
6107
+ const read = await client.from('cliv2_permission_requests').select('resume_taken_at,resume_machine_id,resume_answered_by,resume_interrupted_at,state').eq('id', source.id).maybeSingle();
6108
+ if (read.error)
6109
+ throw read.error;
6110
+ current = !!read.data && read.data.resume_taken_at === source.resume_taken_at && read.data.resume_machine_id === args.p_machine_id && !read.data.resume_answered_by && !read.data.resume_interrupted_at && read.data.state === 'granted';
6111
+ }
6112
+ if (current)
6113
+ rows.push(row);
6114
+ else
6115
+ retireRpcReference(ref);
6116
+ }
6117
+ if (rows.length)
6118
+ return { data: rows, error: null, operationId: claim.id };
6119
+ finishRpcClaims([claim.id], true);
6120
+ }
6121
+ return null;
6122
+ }
6123
+ async function cliClaim(client, action, args, recoveryOnly = false) {
6124
+ if (!hasOwnedWorkContext())
6125
+ return { ...await client.rpc(action, args), operationId: undefined };
6126
+ const recovered = await recoveredCliClaim(client, action, args);
6127
+ if (recovered)
6128
+ return recovered;
6129
+ if (recoveryOnly)
6130
+ return { data: [], error: null, operationId: undefined };
6131
+ const operationId = beginRpcClaim('cli', action, args);
6132
+ try {
6133
+ const response = await client.rpc(action, { ...args, p_operation_id: operationId });
6134
+ if (!response.error) {
6135
+ const journal = await client.rpc('cliv2_reconcile_claim', { p_machine_id: args.p_machine_id, p_operation_id: operationId });
6136
+ if (journal.error)
6137
+ throw journal.error;
6138
+ const row = journal.data?.[0];
6139
+ if (row?.outcome !== 'completed' || row.action !== action)
6140
+ throw new Error('The claim was cancelled before it could start.');
6141
+ recordRpcClaims(operationId, cliClaimReferences(action, row.result), false, row.result);
6142
+ if (row.result.length === 0)
6143
+ finishRpcClaims([operationId]);
6144
+ return { ...response, data: row.result, operationId: row.result.length ? operationId : undefined };
6145
+ }
6146
+ if (response.error)
6147
+ deferRpcClaim(operationId);
6148
+ return { ...response, operationId };
6149
+ }
6150
+ catch (error) {
6151
+ deferRpcClaim(operationId);
6152
+ throw error;
6153
+ }
6154
+ }
6155
+ /** Read every legacy source before committing a baseline; a failed page keeps admission closed. */
6156
+ async function legacyRows(client, table, columns, todoId) {
6157
+ const rows = [];
6158
+ let after = null;
6159
+ for (;;) {
6160
+ let query = client.from(table).select(columns).order('id').limit(500);
6161
+ if (todoId)
6162
+ query = query.eq('todo_id', todoId);
6163
+ if (after)
6164
+ query = query.gt('id', after);
6165
+ const { data, error } = await query;
6166
+ if (error)
6167
+ throw error;
6168
+ if (!Array.isArray(data))
6169
+ throw new Error('The old work snapshot did not return a complete page.');
6170
+ rows.push(...data);
6171
+ if (data.length < 500)
6172
+ return rows;
6173
+ after = data[data.length - 1].id;
6174
+ if (typeof after !== 'string')
6175
+ throw new Error('The old work snapshot has no continuation identity.');
6176
+ }
6177
+ }
6178
+ export async function reconcileInterruptedWork(client, machineId, observeHarness) {
6179
+ if (!hasOwnedWorkContext())
6180
+ return;
6181
+ const capability = await client.rpc('cliv2_interrupt_machine_work', { p_machine_id: machineId,
6182
+ p_operation_id: randomUUID(), p_interrupted_at: new Date().toISOString(), p_workers: [], p_replies: [] });
6183
+ if (capability.error)
6184
+ throw capability.error;
6185
+ for (const pending of pendingTerminalOutcomes()) {
6186
+ if (pending.reference.surface === 'reply') {
6187
+ await settleReplyOutcome(client, machineId, pending.reference, pending.outcome, () => {
6188
+ const outcome = pending.outcome;
6189
+ if (!outcome.completedAt || !outcome.harnessIdentity || !['claude', 'codex'].includes(pending.harness ?? '') || outcome.stopped)
6190
+ return;
6191
+ if (outcome.ok || outcome.failureKind === 'authentication')
6192
+ observeHarness?.(pending.harness, outcome.ok ? 'authenticated' : 'sign-in-required', outcome.ok ? 'dispatch-success' : 'provider-rejected', outcome.completedAt, outcome.harnessIdentity);
6193
+ });
6194
+ acknowledgeTerminalOutcome(pending.id);
6195
+ continue;
6196
+ }
6197
+ if (pending.reference.surface !== 'worker')
6198
+ continue;
6199
+ const ref = pending.reference;
6200
+ if (pending.outcome.stopped) {
6201
+ await updateWorker(client, ref.workerId, 'died', 'stopped by the user', message => console.warn(message), true);
6202
+ await releaseStoppedTodo(client, ref.todoId, ref.claimedAt);
6203
+ acknowledgeTerminalOutcome(pending.id);
6204
+ continue;
6205
+ }
6206
+ const { data, error } = await client.from('cliv2_loose_todos').select('claimed_at,state').eq('id', ref.todoId).maybeSingle();
6207
+ if (error)
6208
+ throw error;
6209
+ if (data?.claimed_at === ref.claimedAt && ['working', 'needs-input'].includes(data.state)) {
6210
+ await settleWorkerOutcome(client, ref.workerId, { id: ref.todoId, claimedAt: ref.claimedAt }, { agent: pending.harness ?? 'Agent' }, { ok: pending.outcome.ok, output: pending.outcome.text ?? '', error: pending.outcome.reason, failureKind: pending.outcome.failureKind }, pending.outcome.workItemId ?? null, () => new Date(), Date.now(), () => { }, message => console.warn(message), pending.outcome, value => updateTerminalOutcome(pending.id, value));
6211
+ }
6212
+ else {
6213
+ const disposition = pending.outcome.disposition;
6214
+ await updateWorker(client, ref.workerId, disposition?.worker ?? (pending.outcome.ok ? 'done' : 'died'), disposition?.reason || pending.outcome.reason || null, message => console.warn(message), true);
6215
+ }
6216
+ acknowledgeTerminalOutcome(pending.id);
6217
+ }
6218
+ for (const claim of pendingRpcClaims('cli')) {
6219
+ const journal = await client.rpc('cliv2_reconcile_claim', { p_machine_id: machineId, p_operation_id: claim.id });
6220
+ if (journal.error)
6221
+ throw journal.error;
6222
+ const row = journal.data?.[0];
6223
+ if (row?.outcome === 'cancelled')
6224
+ recordRpcClaims(claim.id, [], true);
6225
+ else if (row?.outcome === 'completed' && row.action === claim.action)
6226
+ recordRpcClaims(claim.id, cliClaimReferences(claim.action, row.result), false, row.result);
6227
+ else
6228
+ throw new Error('The pending claim could not be reconciled.');
6229
+ }
6230
+ if (legacySnapshotNeeded()) {
6231
+ const todos = await legacyRows(client, 'cliv2_loose_todos', 'id,claimed_at,claimed_machine_id');
6232
+ const workers = await legacyRows(client, 'cliv2_workers', 'id,todo_id,machine_id,state');
6233
+ const messages = await legacyRows(client, 'cliv2_run_messages', 'id,todo_id,taken_at,taken_machine_id,answered_by');
6234
+ const permissions = await legacyRows(client, 'cliv2_permission_requests', 'id,todo_id,state,consumed_at,resume_taken_at,resume_machine_id,resume_answered_by');
6235
+ const holds = new Map();
6236
+ const hold = (todoId) => {
6237
+ let value = holds.get(todoId);
6238
+ if (!value) {
6239
+ value = { todoId, messageIds: [], messageSources: [], permissionSources: [] };
6240
+ holds.set(todoId, value);
6241
+ }
6242
+ return value;
6243
+ };
6244
+ for (const message of messages) {
6245
+ if (message.answered_by != null || message.taken_machine_id && message.taken_machine_id !== machineId)
6246
+ continue;
6247
+ const pending = hold(message.todo_id);
6248
+ pending.messageIds.push(message.id);
6249
+ if (message.taken_at)
6250
+ pending.messageSources.push({ id: message.id, taken_at: message.taken_at });
6251
+ if (message.taken_machine_id === machineId && message.taken_at)
6252
+ recordExitedInterruption({
6253
+ surface: 'reply', todoId: message.todo_id, messageLeases: [{ id: message.id, taken_at: message.taken_at }],
6254
+ grantLeases: [], observedPendingMessageIds: null,
6255
+ });
6256
+ }
6257
+ for (const permission of permissions) {
6258
+ if (!permission.todo_id || permission.state !== 'granted' || permission.resume_answered_by != null
6259
+ || permission.resume_machine_id && permission.resume_machine_id !== machineId)
6260
+ continue;
6261
+ hold(permission.todo_id).permissionSources.push({ id: permission.id, state: permission.state, consumed_at: permission.consumed_at });
6262
+ if (permission.resume_machine_id === machineId && permission.resume_taken_at)
6263
+ recordExitedInterruption({
6264
+ surface: 'reply', todoId: permission.todo_id, messageLeases: [],
6265
+ grantLeases: [{ id: permission.id, resume_taken_at: permission.resume_taken_at }], observedPendingMessageIds: null,
6266
+ });
6267
+ }
6268
+ for (const worker of workers) {
6269
+ if (worker.machine_id !== machineId || !['working', 'dispatching'].includes(worker.state))
6270
+ continue;
6271
+ const todo = todos.find((row) => row.id === worker.todo_id);
6272
+ if (!todo || todo.claimed_machine_id !== machineId || !todo.claimed_at)
6273
+ continue;
6274
+ recordExitedInterruption({ surface: 'worker', todoId: todo.id, workerId: worker.id,
6275
+ claimedAt: todo.claimed_at, observedPendingMessageIds: null });
6276
+ }
6277
+ for (const baseline of holds.values())
6278
+ baseline.messageIds = messages.filter((row) => row.todo_id === baseline.todoId).map((row) => row.id);
6279
+ saveLegacySnapshot([...holds.values()]);
6280
+ }
6281
+ for (const surface of ['worker', 'reply']) {
6282
+ for (const receipt of interruptionReceipts(surface)) {
6283
+ const ref = receipt.reference;
6284
+ const workers = ref.surface === 'worker' ? [{ worker_id: ref.workerId, todo_id: ref.todoId,
6285
+ claimed_at: ref.claimedAt, observed_pending_message_ids: ref.observedPendingMessageIds }] : [];
6286
+ const replies = ref.surface === 'reply' ? [{ todo_id: ref.todoId, message_leases: ref.messageLeases,
6287
+ grant_leases: ref.grantLeases, observed_pending_message_ids: ref.observedPendingMessageIds }] : [];
6288
+ const { data, error } = await client.rpc('cliv2_interrupt_machine_work', {
6289
+ p_machine_id: machineId, p_operation_id: receipt.operationId, p_interrupted_at: receipt.interruptedAt,
6290
+ p_workers: workers, p_replies: replies,
6291
+ });
6292
+ if (error)
6293
+ throw error;
6294
+ const expected = ref.surface === 'worker' ? [{ kind: ref.workerId ? 'worker' : 'claim', id: ref.workerId ?? ref.todoId }] : ref.surface === 'reply'
6295
+ ? [...ref.messageLeases.map((source) => ({ kind: 'message', id: source.id })), ...ref.grantLeases.map((source) => ({ kind: 'grant', id: source.id }))] : [];
6296
+ const results = data;
6297
+ if (!Array.isArray(results) || expected.some((expectedRow) => !results.some((row) => row.id === expectedRow.id && row.kind === expectedRow.kind
6298
+ && ['interrupted', 'already_interrupted', 'already_settled', 'continued', 'stale'].includes(row.outcome)))) {
6299
+ throw new Error('Interrupted work was not fully acknowledged. Work remains protected.');
6300
+ }
6301
+ acknowledgeInterruption(receipt.id);
6302
+ }
6303
+ }
6304
+ await refreshLegacyTodoHolds(client, machineId);
6305
+ }
6306
+ export async function refreshLegacyTodoHolds(client, machineId) {
6307
+ for (const hold of legacyTodoHolds()) {
6308
+ const todo = await client.from('cliv2_loose_todos').select('id,claimed_machine_id').eq('id', hold.todoId).maybeSingle();
6309
+ if (todo.error)
6310
+ throw todo.error;
6311
+ if (!todo.data) {
6312
+ releaseLegacyTodoHold(hold.todoId);
6313
+ continue;
6314
+ }
6315
+ const messages = { data: await legacyRows(client, 'cliv2_run_messages', 'id,answered_by,taken_at,taken_machine_id', hold.todoId) };
6316
+ if (messages.data.some((row) => !hold.messageIds.includes(row.id))) {
6317
+ releaseLegacyTodoHold(hold.todoId);
6318
+ continue;
6319
+ }
6320
+ const permissions = { data: await legacyRows(client, 'cliv2_permission_requests', 'id,state,consumed_at,resume_answered_by,resume_machine_id', hold.todoId) };
6321
+ const messagesRemain = messages.data.some((row) => hold.messageIds.includes(row.id) && row.answered_by === null
6322
+ && (!row.taken_machine_id || row.taken_machine_id === machineId));
6323
+ const grantsRemain = permissions.data.some((row) => hold.permissionSources.some((source) => source.id === row.id)
6324
+ && row.state === 'granted' && row.resume_answered_by === null && (!row.resume_machine_id || row.resume_machine_id === machineId));
6325
+ if (!messagesRemain && !grantsRemain)
6326
+ releaseLegacyTodoHold(hold.todoId);
6327
+ }
6328
+ }