@ctrl-spc/cs 0.7.14 → 0.7.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -3
- package/dist/agents.js +175 -1
- package/dist/autostart.js +103 -122
- package/dist/codex-home.js +14 -11
- package/dist/companion-ui.js +54 -6
- package/dist/companion.js +86 -169
- package/dist/config.js +199 -17
- package/dist/daemon-lifecycle.js +575 -0
- package/dist/daemon-lock.js +149 -42
- package/dist/daemon-processes.js +860 -0
- package/dist/daemon.js +14 -46
- package/dist/darwin-coalition.js +340 -0
- package/dist/failure-reason.js +76 -16
- package/dist/index.js +75 -76
- package/dist/login.js +9 -9
- package/dist/mcp.js +55 -56
- package/dist/native/darwin-coalition +0 -0
- package/dist/native/darwin-coalition.build.json +1 -0
- package/dist/native/darwin-coalition.c +145 -0
- package/dist/orchestrator.js +892 -575
- package/dist/panel3/coordinator.js +3 -1
- package/dist/panel3/presence.js +1 -1
- package/dist/panel3/run.js +996 -558
- package/dist/panel3/spawn.js +85 -23
- package/dist/panel3/tools.js +5 -0
- package/dist/presence-heartbeat.js +3 -0
- package/dist/presence.js +271 -135
- package/dist/supabase.js +173 -37
- package/dist/win-shell.js +464 -1
- package/dist/windows-job.js +312 -0
- package/package.json +4 -3
package/dist/orchestrator.js
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
|
+
import { executionIdentity } from './agents.js';
|
|
2
|
+
import { CodexHomeFailure } from './codex-home.js';
|
|
1
3
|
import { execFile, spawn as spawnChild } from 'node:child_process';
|
|
2
4
|
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { beginOwnedClaim, ownedWorkAllowed, hasOwnedWorkContext, pendingTerminalOutcomes, acknowledgeTerminalOutcome, updateTerminalOutcome, ownedWorkerClaimedAt, ownedWorkerExecutionHeld, reserveOwnedWork, legacyHeldTodoIds, interruptionReceipts, acknowledgeInterruption, legacySnapshotNeeded, saveLegacySnapshot, legacyTodoHolds, releaseLegacyTodoHold, recordExitedInterruption, beginRpcClaim, deferRpcClaim, pendingRpcClaims, recordRpcClaims, finishRpcClaims, recoveredRpcClaims, pendingRpcReference, retireRpcReference } from './daemon-processes.js';
|
|
3
6
|
import { promisify } from 'node:util';
|
|
4
7
|
import { agentPath, detectAgents } from './agents.js';
|
|
5
8
|
/* 18a Slice 7: the two Windows spawn rules, in one module so a caller cannot
|
|
6
9
|
take the shell and forget the quoting. `quoteForCmd` is re-exported because
|
|
7
10
|
this module's own tests reach for it here. */
|
|
8
|
-
import { killTree, needsShell, processIsAlive, quoteForCmd } from './win-shell.js';
|
|
11
|
+
import { killTree, needsShell, processIsAlive, quoteForCmd, spawnOwnedProcess, releaseOwnedProcess } from './win-shell.js';
|
|
9
12
|
export { quoteForCmd } from './win-shell.js';
|
|
10
13
|
import { ORCHESTRATOR_MAX_CONCURRENT, ORCHESTRATOR_SPAWN_TIMEOUT_MS } from './env.js';
|
|
11
|
-
import { agentDisplayName,
|
|
14
|
+
import { agentDisplayName, nativeFailureKind, failureMessage } from './failure-reason.js';
|
|
12
15
|
/* 18c SLICE 9 (GAP 15): the two request-scoped releases. They live in mcp.ts
|
|
13
16
|
beside the session-end routes and the reservations table's other writers, so
|
|
14
17
|
there is ONE place that knows how a lease is let go, rather than two that can
|
|
@@ -199,7 +202,7 @@ export async function claimNextTodo(client, machineId, agent,
|
|
|
199
202
|
because the claim must stay ONE statement (see below), and because a
|
|
200
203
|
blocked unit is the OLDEST claimable row: released and re-claimed it would
|
|
201
204
|
be picked first every tick, starving everything behind it. */
|
|
202
|
-
skipIds = []) {
|
|
205
|
+
skipIds = [], recoveryOnly = false) {
|
|
203
206
|
/* THROUGH AN RPC, NOT PostgREST — and this is not a style preference, it is
|
|
204
207
|
the fix for a bug that made the whole slice inert.
|
|
205
208
|
|
|
@@ -224,11 +227,11 @@ skipIds = []) {
|
|
|
224
227
|
row rather than blocking or double-claiming. `userId` is no longer passed:
|
|
225
228
|
the function scopes itself with `auth.uid()`, which is the same identity
|
|
226
229
|
RLS uses and cannot be spoofed by a caller. */
|
|
227
|
-
const { data, error } = await client
|
|
230
|
+
const { data, error, operationId } = await cliClaim(client, 'cliv2_claim_next_loose_todo', {
|
|
228
231
|
p_machine_id: machineId,
|
|
229
232
|
p_agent: agent,
|
|
230
233
|
p_skip_ids: skipIds,
|
|
231
|
-
});
|
|
234
|
+
}, recoveryOnly);
|
|
232
235
|
if (error)
|
|
233
236
|
throw error;
|
|
234
237
|
/* 18b Slice 2: the claim also carries the request's OWN project (id and name),
|
|
@@ -240,40 +243,12 @@ skipIds = []) {
|
|
|
240
243
|
const row = (data ?? [])[0];
|
|
241
244
|
if (!row)
|
|
242
245
|
return null;
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
claim, rather than added to the RPC's return table, because that keeps the
|
|
247
|
-
shipped function's contract alone and needs no migration.
|
|
248
|
-
|
|
249
|
-
WHY IT IS NEEDED. hand_off_stage clears the claim so the next stage is
|
|
250
|
-
dispatched to a fresh worker, and the handing-off worker then keeps talking
|
|
251
|
-
for as long as it takes to write its closing answer. Its exit path wrote
|
|
252
|
-
`state='done'` unconditionally, over a row a SECOND worker was already
|
|
253
|
-
working: the card read Done carrying stage 1's answer, and because the claim
|
|
254
|
-
RPC skips `done` rows the request could never be claimed again, so the
|
|
255
|
-
remaining stages were never run. Found by the gate's audit, and it matches
|
|
256
|
-
what the walk saw.
|
|
257
|
-
|
|
258
|
-
Every terminal write is now fenced on this value, so a run that no longer
|
|
259
|
-
holds the claim writes NOTHING. Null is possible only if the row vanished
|
|
260
|
-
between claiming and reading it, and an unfenceable dispatch is treated as
|
|
261
|
-
already superseded rather than allowed to clobber. */
|
|
262
|
-
let claimedAt = null;
|
|
263
|
-
try {
|
|
264
|
-
const { data: claimRow } = await client
|
|
265
|
-
.from('cliv2_loose_todos')
|
|
266
|
-
.select('claimed_at')
|
|
267
|
-
.eq('id', row.id)
|
|
268
|
-
.maybeSingle();
|
|
269
|
-
claimedAt = claimRow?.claimed_at ?? null;
|
|
270
|
-
}
|
|
271
|
-
catch {
|
|
272
|
-
/* Unreadable is treated as unfenceable, which only loses the protection —
|
|
273
|
-
it never blocks the dispatch that was already claimed successfully. */
|
|
274
|
-
}
|
|
246
|
+
const claimedAt = row.claimed_at ?? null;
|
|
247
|
+
if (hasOwnedWorkContext() && !claimedAt)
|
|
248
|
+
throw new Error('The claim did not return an exact attempt identity. Update the database before starting work.');
|
|
275
249
|
return {
|
|
276
250
|
id: row.id,
|
|
251
|
+
...(operationId ? { claimOperationId: operationId } : {}),
|
|
277
252
|
instruction: row.instruction,
|
|
278
253
|
/* `?? null` rather than trusting the shape: a daemon running against a
|
|
279
254
|
database that has not taken this slice's migration yet gets neither
|
|
@@ -514,10 +489,13 @@ claimedAt) {
|
|
|
514
489
|
const write = client
|
|
515
490
|
.from('cliv2_loose_todos')
|
|
516
491
|
.update({ answer, state: 'done' })
|
|
517
|
-
.eq('id', todoId)
|
|
518
|
-
|
|
492
|
+
.eq('id', todoId)
|
|
493
|
+
.in('state', ['working', 'needs-input']);
|
|
494
|
+
const { data: written, error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write).select('id');
|
|
519
495
|
if (error)
|
|
520
496
|
throw error;
|
|
497
|
+
if (!written?.length)
|
|
498
|
+
return false;
|
|
521
499
|
/* 18c SLICE 1 — CLOSE OUT THE LAST ACTIVITY LINE.
|
|
522
500
|
*
|
|
523
501
|
* report_activity only ever flips the PREVIOUS 'doing' row to 'done' when a
|
|
@@ -541,6 +519,7 @@ claimedAt) {
|
|
|
541
519
|
catch (err) {
|
|
542
520
|
console.warn(`answerTodo: closing out the last activity line failed: ${err.message}`);
|
|
543
521
|
}
|
|
522
|
+
return true;
|
|
544
523
|
}
|
|
545
524
|
/**
|
|
546
525
|
* !Cleanup PHASE 6 (I15) — DID THIS WORKER ASK SOMETHING AND EXIT?
|
|
@@ -566,7 +545,7 @@ claimedAt) {
|
|
|
566
545
|
* a card asking the user for something that does not exist, unclearable except
|
|
567
546
|
* by Stop. A wrong `done` is recoverable by Retry; a phantom question is not.
|
|
568
547
|
*/
|
|
569
|
-
export async function todoHasOpenAsk(client, todoId, warn) {
|
|
548
|
+
export async function todoHasOpenAsk(client, todoId, warn, strict = false) {
|
|
570
549
|
try {
|
|
571
550
|
const { data, error } = await client
|
|
572
551
|
.from('cliv2_todo_asks')
|
|
@@ -578,11 +557,13 @@ export async function todoHasOpenAsk(client, todoId, warn) {
|
|
|
578
557
|
throw new Error(error.message);
|
|
579
558
|
if ((data ?? []).length > 0)
|
|
580
559
|
return true;
|
|
581
|
-
if (await todoHasOpenDecision(client, todoId, warn))
|
|
560
|
+
if (await todoHasOpenDecision(client, todoId, warn, strict))
|
|
582
561
|
return true;
|
|
583
|
-
return await todoHasOpenBlockedNotice(client, todoId, warn);
|
|
562
|
+
return await todoHasOpenBlockedNotice(client, todoId, warn, strict);
|
|
584
563
|
}
|
|
585
564
|
catch (err) {
|
|
565
|
+
if (strict)
|
|
566
|
+
throw err;
|
|
586
567
|
warn(`[orchestrator] could not check for open questions: ${err.message}`);
|
|
587
568
|
return false;
|
|
588
569
|
}
|
|
@@ -606,7 +587,7 @@ export async function todoHasOpenAsk(client, todoId, warn) {
|
|
|
606
587
|
* Best-effort and false on error, like its two siblings, and for the same
|
|
607
588
|
* reason: a wrong `done` is recoverable by Retry, a phantom park is not.
|
|
608
589
|
*/
|
|
609
|
-
export async function todoHasOpenBlockedNotice(client, todoId, warn) {
|
|
590
|
+
export async function todoHasOpenBlockedNotice(client, todoId, warn, strict = false) {
|
|
610
591
|
try {
|
|
611
592
|
const { data, error } = await client
|
|
612
593
|
.from('cliv2_blocked_notices')
|
|
@@ -619,6 +600,8 @@ export async function todoHasOpenBlockedNotice(client, todoId, warn) {
|
|
|
619
600
|
return (data ?? []).length > 0;
|
|
620
601
|
}
|
|
621
602
|
catch (err) {
|
|
603
|
+
if (strict)
|
|
604
|
+
throw err;
|
|
622
605
|
warn(`[orchestrator] could not check for an open stop: ${err.message}`);
|
|
623
606
|
return false;
|
|
624
607
|
}
|
|
@@ -659,7 +642,7 @@ const PERMISSION_CATEGORY = 'cliv2_permission';
|
|
|
659
642
|
* BEST-EFFORT, like every read in the tick. A failure returns false, which costs
|
|
660
643
|
* a wrongly-finished card rather than a lost run.
|
|
661
644
|
*/
|
|
662
|
-
export async function todoHasOpenDecision(client, todoId, warn) {
|
|
645
|
+
export async function todoHasOpenDecision(client, todoId, warn, strict = false) {
|
|
663
646
|
try {
|
|
664
647
|
const { data: links, error: linkError } = await client
|
|
665
648
|
.from('cliv2_loose_todo_links')
|
|
@@ -691,6 +674,8 @@ export async function todoHasOpenDecision(client, todoId, warn) {
|
|
|
691
674
|
return (data ?? []).length > 0;
|
|
692
675
|
}
|
|
693
676
|
catch (err) {
|
|
677
|
+
if (strict)
|
|
678
|
+
throw err;
|
|
694
679
|
warn(`[orchestrator] could not check the work item's questions: ${err.message}`);
|
|
695
680
|
return false;
|
|
696
681
|
}
|
|
@@ -803,7 +788,7 @@ export async function todoRunLeftATrace(client, todoId, warn,
|
|
|
803
788
|
* same answer for the scope gate and resolving it twice would let the two
|
|
804
789
|
* disagree about which item this run is for. Null for a panel request with
|
|
805
790
|
* no work item, which is the greenfield case. */
|
|
806
|
-
workItemId = null) {
|
|
791
|
+
workItemId = null, strict = false) {
|
|
807
792
|
try {
|
|
808
793
|
/* ANY ONE IS ENOUGH, so the cheapest and most likely is asked first and the
|
|
809
794
|
rest are skipped once one answers yes. A session is what the guidance
|
|
@@ -852,6 +837,8 @@ workItemId = null) {
|
|
|
852
837
|
return (dropped.data?.dropped_activity ?? 0) > 0;
|
|
853
838
|
}
|
|
854
839
|
catch (err) {
|
|
840
|
+
if (strict)
|
|
841
|
+
throw err;
|
|
855
842
|
/* See the block comment: towards `done`, so a transient read failure never
|
|
856
843
|
discards a run that really did the work. */
|
|
857
844
|
warn(`[orchestrator] could not check whether the run recorded anything: ${err.message}`);
|
|
@@ -888,10 +875,12 @@ claimedAt) {
|
|
|
888
875
|
// `answer` is constrained nonblank-when-present, so '' must go in as null
|
|
889
876
|
// rather than as an empty string the check would refuse.
|
|
890
877
|
.update({ answer: trimmed || null, state: 'needs-input' })
|
|
891
|
-
.eq('id', todoId)
|
|
892
|
-
|
|
878
|
+
.eq('id', todoId)
|
|
879
|
+
.in('state', ['working', 'needs-input']);
|
|
880
|
+
const { data, error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write).select('id');
|
|
893
881
|
if (error)
|
|
894
882
|
throw error;
|
|
883
|
+
return !!data?.length;
|
|
895
884
|
}
|
|
896
885
|
/**
|
|
897
886
|
* !Cleanup PHASE 6 (I17) — HAS THE USER STOPPED THIS?
|
|
@@ -934,12 +923,13 @@ export async function todoWasStopped(client, todoId) {
|
|
|
934
923
|
* already hit Retry, the unit is `working` again and this matches nothing. That
|
|
935
924
|
* is the Gherkin line "the daemon does not later overwrite that".
|
|
936
925
|
*/
|
|
937
|
-
export async function releaseStoppedTodo(client, todoId) {
|
|
938
|
-
const
|
|
926
|
+
export async function releaseStoppedTodo(client, todoId, claimedAt) {
|
|
927
|
+
const write = client
|
|
939
928
|
.from('cliv2_loose_todos')
|
|
940
929
|
.update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
|
|
941
930
|
.eq('id', todoId)
|
|
942
931
|
.eq('state', 'stopped');
|
|
932
|
+
const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
|
|
943
933
|
if (error)
|
|
944
934
|
throw error;
|
|
945
935
|
}
|
|
@@ -972,6 +962,41 @@ export const MAX_ATTEMPTS = 3;
|
|
|
972
962
|
* so there is one count: a second read could land after another process had
|
|
973
963
|
* touched the row, and a wait computed from a stale count is a wait the card's
|
|
974
964
|
* own countdown would disagree with. */
|
|
965
|
+
/** The existing terminal write, also used for a nonretryable native refusal. */
|
|
966
|
+
async function failClaimedTodo(client, todoId, reason, expectedClaim, count) {
|
|
967
|
+
let attempts = count ?? 0;
|
|
968
|
+
if (count === undefined) {
|
|
969
|
+
const { data, error } = await client.from('cliv2_loose_todos').select('attempts,claimed_at').eq('id', todoId).maybeSingle();
|
|
970
|
+
if (error)
|
|
971
|
+
throw error;
|
|
972
|
+
if (!data || data.claimed_at !== expectedClaim)
|
|
973
|
+
return { fate: 'stale', attempts: data?.attempts ?? 0 };
|
|
974
|
+
attempts = Number(data.attempts ?? 0) + 1;
|
|
975
|
+
}
|
|
976
|
+
let failure = client
|
|
977
|
+
.from('cliv2_loose_todos')
|
|
978
|
+
.update({
|
|
979
|
+
state: 'failed',
|
|
980
|
+
attempts,
|
|
981
|
+
stopped_reason: reason.trim() || 'the work failed repeatedly',
|
|
982
|
+
claimed_machine_id: null,
|
|
983
|
+
claimed_agent: null,
|
|
984
|
+
claimed_at: null,
|
|
985
|
+
/* 18a SLICE 6: cleared at the bound. A wait on a terminal row would be
|
|
986
|
+
read by nothing and would outlive the row's own meaning; the only
|
|
987
|
+
route back is Retry, which means now. */
|
|
988
|
+
retry_after: null,
|
|
989
|
+
})
|
|
990
|
+
.eq('id', todoId)
|
|
991
|
+
.in('state', ['working', 'needs-input']);
|
|
992
|
+
failure = expectedClaim ? failure.eq('claimed_at', expectedClaim) : failure.is('claimed_at', null);
|
|
993
|
+
const { data: failed, error: failError } = await failure.select('id');
|
|
994
|
+
if (failError)
|
|
995
|
+
throw failError;
|
|
996
|
+
if (!failed?.length)
|
|
997
|
+
return { fate: 'stale', attempts: attempts - 1 };
|
|
998
|
+
return { fate: 'failed', attempts };
|
|
999
|
+
}
|
|
975
1000
|
export async function releaseOrFail(client, todoId, reason,
|
|
976
1001
|
/** 18a Slice 6: injectable so a test can place the wait exactly. */
|
|
977
1002
|
nowMs = Date.now(),
|
|
@@ -992,34 +1017,17 @@ claimedAt) {
|
|
|
992
1017
|
already claimed by the run that replaced this one. */
|
|
993
1018
|
const current = data;
|
|
994
1019
|
if (claimedAt && current && current.claimed_at !== claimedAt) {
|
|
995
|
-
return { fate: '
|
|
1020
|
+
return { fate: 'stale', attempts: current.attempts ?? 0 };
|
|
996
1021
|
}
|
|
997
1022
|
const attempts = (current?.attempts ?? 0) + 1;
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
.update({
|
|
1002
|
-
state: 'failed',
|
|
1003
|
-
attempts,
|
|
1004
|
-
stopped_reason: reason.trim() || 'the work failed repeatedly',
|
|
1005
|
-
claimed_machine_id: null,
|
|
1006
|
-
claimed_agent: null,
|
|
1007
|
-
claimed_at: null,
|
|
1008
|
-
/* 18a SLICE 6: cleared at the bound. A wait on a terminal row would be
|
|
1009
|
-
read by nothing and would outlive the row's own meaning; the only
|
|
1010
|
-
route back is Retry, which means now. */
|
|
1011
|
-
retry_after: null,
|
|
1012
|
-
})
|
|
1013
|
-
.eq('id', todoId);
|
|
1014
|
-
if (failError)
|
|
1015
|
-
throw failError;
|
|
1016
|
-
return { fate: 'failed', attempts };
|
|
1017
|
-
}
|
|
1023
|
+
const expectedClaim = claimedAt ?? current?.claimed_at ?? null;
|
|
1024
|
+
if (attempts >= MAX_ATTEMPTS)
|
|
1025
|
+
return failClaimedTodo(client, todoId, reason, expectedClaim, attempts);
|
|
1018
1026
|
/* 18a SLICE 6: THE WAIT IS WRITTEN ON THE SAME STATEMENT AS THE COUNT, for
|
|
1019
1027
|
the reason the count is on the same statement as the release: a wait that
|
|
1020
1028
|
can fail on its own is a wait the daemon might not honour, and the failure
|
|
1021
1029
|
mode is the fork bomb this bound exists to prevent. */
|
|
1022
|
-
|
|
1030
|
+
let release = client
|
|
1023
1031
|
.from('cliv2_loose_todos')
|
|
1024
1032
|
.update({
|
|
1025
1033
|
attempts,
|
|
@@ -1028,9 +1036,14 @@ claimedAt) {
|
|
|
1028
1036
|
claimed_at: null,
|
|
1029
1037
|
retry_after: new Date(nowMs + retryDelayMs(attempts)).toISOString(),
|
|
1030
1038
|
})
|
|
1031
|
-
.eq('id', todoId)
|
|
1039
|
+
.eq('id', todoId)
|
|
1040
|
+
.in('state', ['working', 'needs-input']);
|
|
1041
|
+
release = expectedClaim ? release.eq('claimed_at', expectedClaim) : release.is('claimed_at', null);
|
|
1042
|
+
const { data: released, error: releaseError } = await release.select('id');
|
|
1032
1043
|
if (releaseError)
|
|
1033
1044
|
throw releaseError;
|
|
1045
|
+
if (!released?.length)
|
|
1046
|
+
return { fate: 'stale', attempts: current?.attempts ?? 0 };
|
|
1034
1047
|
return { fate: 'released', attempts };
|
|
1035
1048
|
}
|
|
1036
1049
|
/**
|
|
@@ -1172,11 +1185,13 @@ export async function scopeIsApproved(client, workItemId) {
|
|
|
1172
1185
|
* See the call site for why the claim stays: an unclaimed todo is immediately
|
|
1173
1186
|
* re-claimable, so releasing would spin the tick loop.
|
|
1174
1187
|
*/
|
|
1175
|
-
export async function parkTodoForScope(client, todoId) {
|
|
1176
|
-
const
|
|
1188
|
+
export async function parkTodoForScope(client, todoId, claimedAt) {
|
|
1189
|
+
const write = client
|
|
1177
1190
|
.from('cliv2_loose_todos')
|
|
1178
1191
|
.update({ state: 'needs-input' })
|
|
1179
|
-
.eq('id', todoId)
|
|
1192
|
+
.eq('id', todoId)
|
|
1193
|
+
.in('state', ['working', 'needs-input']);
|
|
1194
|
+
const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
|
|
1180
1195
|
if (error)
|
|
1181
1196
|
throw error;
|
|
1182
1197
|
}
|
|
@@ -1279,7 +1294,7 @@ injectedSpawn,
|
|
|
1279
1294
|
* worker's, and injected by the same convention (`deps.resolveContext`) so a
|
|
1280
1295
|
* test can point it somewhere without `resolveClaimContext` growing
|
|
1281
1296
|
* parameters for the suite's sake. */
|
|
1282
|
-
resolveContext = (claim) => resolveClaimContext(client, claim, warn)) {
|
|
1297
|
+
resolveContext = (claim) => resolveClaimContext(client, claim, warn), observeHarness) {
|
|
1283
1298
|
/* ═══ 18k SLICE 10b — NO TOOLS MEANS NO ANSWER, the tick's own rule. ═══
|
|
1284
1299
|
An answerer now holds the worker's toolset, so one started while the tools
|
|
1285
1300
|
server is down would have NONE of it: it could not read the work item, could
|
|
@@ -1319,17 +1334,19 @@ resolveContext = (claim) => resolveClaimContext(client, claim, warn)) {
|
|
|
1319
1334
|
returns; what bounds a long answer is `state.answering`, keyed per run, and
|
|
1320
1335
|
`state.maxAnswering`. This guard is back to meaning what it says: one take
|
|
1321
1336
|
at a time, which is a few queries. */
|
|
1322
|
-
if (state.takeInFlight)
|
|
1337
|
+
if (state.takeInFlight || !ownedWorkAllowed())
|
|
1323
1338
|
return;
|
|
1339
|
+
const endClaim = beginOwnedClaim();
|
|
1324
1340
|
state.takeInFlight = true;
|
|
1325
1341
|
try {
|
|
1326
|
-
await takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext);
|
|
1342
|
+
await takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext, observeHarness);
|
|
1327
1343
|
}
|
|
1328
1344
|
finally {
|
|
1329
1345
|
state.takeInFlight = false;
|
|
1346
|
+
endClaim();
|
|
1330
1347
|
}
|
|
1331
1348
|
}
|
|
1332
|
-
async function takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext) {
|
|
1349
|
+
async function takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext, observeHarness) {
|
|
1333
1350
|
/* ═══ 18k SLICE 10 — A MACHINE ANSWERS ITS OWN RUNS, DESIGNATED OR NOT. ═══
|
|
1334
1351
|
Slice 7 gated this whole function on being the designated orchestrator, and
|
|
1335
1352
|
that is wrong for work already claimed: the user may switch the toolbar to
|
|
@@ -1342,6 +1359,7 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1342
1359
|
old check survives on its own. */
|
|
1343
1360
|
if (agents.length === 0)
|
|
1344
1361
|
return;
|
|
1362
|
+
const claimOperations = [];
|
|
1345
1363
|
/* THE RUNS THIS MACHINE CLAIMS, which is what it may answer for. `claimed_at`
|
|
1346
1364
|
is the real predicate rather than the state alone: `releaseStoppedTodo` and
|
|
1347
1365
|
`releaseOrFail` null the claim on `stopped` and `failed`, while `answerTodo`
|
|
@@ -1400,6 +1418,13 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1400
1418
|
scoped take with an empty id list, which correctly returns nothing — so this
|
|
1401
1419
|
is a saved round trip every ten seconds on every machine in the fleet that is
|
|
1402
1420
|
not participating, not a safety property. The safety is the scope itself. */
|
|
1421
|
+
if (hasOwnedWorkContext()) {
|
|
1422
|
+
for (const claim of recoveredRpcClaims('cli').filter(row => row.action !== 'cliv2_claim_next_loose_todo')) {
|
|
1423
|
+
for (const ref of cliClaimReferences(claim.action, claim.result))
|
|
1424
|
+
if (ref.surface === 'reply' && pendingRpcReference(ref) && !state.answering.has(ref.todoId) && !ownTodoIds.includes(ref.todoId))
|
|
1425
|
+
ownTodoIds.push(ref.todoId);
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1403
1428
|
if (ownTodoIds.length === 0 && !designated)
|
|
1404
1429
|
return;
|
|
1405
1430
|
try {
|
|
@@ -1412,13 +1437,17 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1412
1437
|
which is the whole of the starvation this scoping removes. */
|
|
1413
1438
|
const taken = [];
|
|
1414
1439
|
if (ownTodoIds.length > 0) {
|
|
1415
|
-
const own = await client
|
|
1440
|
+
const own = await cliClaim(client, 'cliv2_take_run_messages', { p_todo_ids: ownTodoIds, p_machine_id: machineId, p_skip_todo_ids: [...legacyHeldTodoIds(), ...state.answering] });
|
|
1441
|
+
if (own.operationId)
|
|
1442
|
+
claimOperations.push(own.operationId);
|
|
1416
1443
|
if (own.error)
|
|
1417
1444
|
throw new Error(own.error.message);
|
|
1418
1445
|
taken.push(...(own.data ?? []));
|
|
1419
1446
|
}
|
|
1420
1447
|
if (designated) {
|
|
1421
|
-
const rest = await client
|
|
1448
|
+
const rest = await cliClaim(client, 'cliv2_take_run_messages', { p_todo_ids: null, p_machine_id: machineId, p_skip_todo_ids: [...legacyHeldTodoIds(), ...state.answering] });
|
|
1449
|
+
if (rest.operationId)
|
|
1450
|
+
claimOperations.push(rest.operationId);
|
|
1422
1451
|
if (rest.error)
|
|
1423
1452
|
throw new Error(rest.error.message);
|
|
1424
1453
|
/* DEDUPED BY ID rather than trusted to be disjoint. The database will not
|
|
@@ -1450,7 +1479,13 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1450
1479
|
cover: the answer is not to a message, it is to an approval. Everything
|
|
1451
1480
|
downstream already handles an empty id list — the covering RPC is skipped
|
|
1452
1481
|
and the lease is not involved, because no message was leased. */
|
|
1453
|
-
const
|
|
1482
|
+
const grants = await cliClaim(client, 'cliv2_take_granted_resumes', { p_todo_ids: designated ? null : ownTodoIds,
|
|
1483
|
+
p_machine_id: machineId, p_skip_todo_ids: [...legacyHeldTodoIds(), ...state.answering] });
|
|
1484
|
+
if (grants.operationId)
|
|
1485
|
+
claimOperations.push(grants.operationId);
|
|
1486
|
+
if (grants.error)
|
|
1487
|
+
throw grants.error;
|
|
1488
|
+
const granted = (grants.data ?? []);
|
|
1454
1489
|
/* AN EMPTY PASS SAYS NOTHING. This runs every 10 seconds and the common case
|
|
1455
1490
|
is nothing to take and nothing approved; a line each time would bury every
|
|
1456
1491
|
useful one, the same rule `reportRole` and `reportNoTools` follow.
|
|
@@ -1495,7 +1530,8 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1495
1530
|
}
|
|
1496
1531
|
/* AND THE APPROVED RUNS JOIN THEM, with no message ids: there is nothing to
|
|
1497
1532
|
cover, because the answer is to an approval rather than to a message. */
|
|
1498
|
-
for (const
|
|
1533
|
+
for (const grant of granted) {
|
|
1534
|
+
const todoId = grant.todo_id;
|
|
1499
1535
|
if (!byRun.has(todoId))
|
|
1500
1536
|
byRun.set(todoId, { messageIds: [], wasExpired: false });
|
|
1501
1537
|
}
|
|
@@ -1553,7 +1589,10 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1553
1589
|
`answerRunMessages` warns and returns false rather than throwing, and
|
|
1554
1590
|
the pass's job is to have STARTED every answer. The `catch` is the
|
|
1555
1591
|
daemon's never-throw contract, not a second failure path. */
|
|
1556
|
-
void answerRunMessages(client, todoId, group.messageIds, { agent: plan.agent, cwd: context.checkout, workingCopyElsewhere: plan.workingCopyElsewhere }, state, spawn, warn
|
|
1592
|
+
void answerRunMessages(client, todoId, group.messageIds, { agent: plan.agent, cwd: context.checkout, workingCopyElsewhere: plan.workingCopyElsewhere }, state, spawn, warn, { surface: 'reply', todoId,
|
|
1593
|
+
messageLeases: taken.filter((row) => row.todo_id === todoId).map((row) => ({ id: row.id, taken_at: row.taken_at })),
|
|
1594
|
+
grantLeases: granted.filter((row) => row.todo_id === todoId).map((row) => ({ id: row.id, resume_taken_at: row.resume_taken_at })),
|
|
1595
|
+
observedPendingMessageIds: null }, machineId, undefined, observeHarness).catch(err => {
|
|
1557
1596
|
warn(`[orchestrator] could not answer ${shortId(todoId)}: ${err.message}`);
|
|
1558
1597
|
});
|
|
1559
1598
|
}
|
|
@@ -1561,6 +1600,10 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1561
1600
|
catch (err) {
|
|
1562
1601
|
reportTakeFailed(state, err.message, warn);
|
|
1563
1602
|
}
|
|
1603
|
+
finally {
|
|
1604
|
+
if (hasOwnedWorkContext())
|
|
1605
|
+
finishRpcClaims(claimOperations, true);
|
|
1606
|
+
}
|
|
1564
1607
|
}
|
|
1565
1608
|
/**
|
|
1566
1609
|
* 18k Slice 10 — THE CODEBASE THIS RUN IS FOR, recovered the way the tick
|
|
@@ -2195,7 +2238,7 @@ plan, state,
|
|
|
2195
2238
|
/** 18k Slice 10b — THE WORKER'S OWN SPAWN SHAPE, not a responder's. There is no
|
|
2196
2239
|
* second adapter any more: this is what `defaultSpawn(mcpServer)` returns, and
|
|
2197
2240
|
* the answerer differs from a dispatch only in passing no `alsoRunningIn`. */
|
|
2198
|
-
spawn, warn) {
|
|
2241
|
+
spawn, warn, attempt, machineId, owned, observeHarness) {
|
|
2199
2242
|
/* ═══ 18k SLICE 10b — ONE ANSWER PER RUN AT A TIME. ═══
|
|
2200
2243
|
DEPTH, NOT THE PRIMARY DEFENCE. `takeRunMessagesInner` subtracts
|
|
2201
2244
|
`state.answering` from the scoped take's id list, so a run this machine is
|
|
@@ -2220,26 +2263,64 @@ spawn, warn) {
|
|
|
2220
2263
|
into an inner function so the clear is in ONE place rather than repeated at
|
|
2221
2264
|
every `return` below, which is the same shape `orchestratorTick` uses to
|
|
2222
2265
|
release its own slot. */
|
|
2266
|
+
if (hasOwnedWorkContext() && (!attempt || !machineId))
|
|
2267
|
+
throw new Error('The reply has no exact source lease identity.');
|
|
2268
|
+
owned = hasOwnedWorkContext() ? reserveOwnedWork(attempt, plan.agent) : undefined;
|
|
2223
2269
|
state.answering.add(todoId);
|
|
2224
2270
|
try {
|
|
2225
|
-
|
|
2271
|
+
const answered = await answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned, observeHarness);
|
|
2272
|
+
if (answered)
|
|
2273
|
+
owned?.complete();
|
|
2274
|
+
return answered;
|
|
2226
2275
|
}
|
|
2227
2276
|
finally {
|
|
2228
2277
|
state.answering.delete(todoId);
|
|
2278
|
+
owned?.finishPreparation();
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
async function settleReplyOutcome(client, machineId, attempt, outcome, onAccepted) {
|
|
2282
|
+
const todoId = attempt.todoId;
|
|
2283
|
+
const finished = await client.rpc('cliv2_finish_reply_attempt', {
|
|
2284
|
+
p_todo_id: todoId, p_machine_id: machineId, p_message_leases: attempt.messageLeases,
|
|
2285
|
+
p_grant_leases: attempt.grantLeases, p_body: outcome.text,
|
|
2286
|
+
...(!outcome.ok && !outcome.stopped ? { p_failure_kind: outcome.failureKind ?? 'unknown' } : {}),
|
|
2287
|
+
});
|
|
2288
|
+
if (finished.error)
|
|
2289
|
+
throw finished.error;
|
|
2290
|
+
const receipt = finished.data?.[0]?.outcome;
|
|
2291
|
+
if (receipt === 'stale')
|
|
2292
|
+
return false;
|
|
2293
|
+
if (receipt === 'failed' || receipt === 'already_failed') {
|
|
2294
|
+
onAccepted?.();
|
|
2295
|
+
return false;
|
|
2229
2296
|
}
|
|
2297
|
+
if (receipt !== 'accepted' && receipt !== 'already_settled')
|
|
2298
|
+
throw new Error('The reply acknowledgement was incomplete.');
|
|
2299
|
+
onAccepted?.();
|
|
2300
|
+
if (outcome.stopped && outcome.replyingToClaim)
|
|
2301
|
+
await releaseStoppedTodo(client, todoId, outcome.replyingToClaim);
|
|
2302
|
+
else if (outcome.workerIsLive && outcome.replyingToClaim) {
|
|
2303
|
+
const scheduled = await client.from('cliv2_loose_todos')
|
|
2304
|
+
.update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
|
|
2305
|
+
.eq('id', todoId).eq('claimed_at', outcome.replyingToClaim).in('state', ['working', 'needs-input']);
|
|
2306
|
+
if (scheduled.error)
|
|
2307
|
+
throw scheduled.error;
|
|
2308
|
+
}
|
|
2309
|
+
return !outcome.stopped;
|
|
2230
2310
|
}
|
|
2231
|
-
async function answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn) {
|
|
2311
|
+
async function answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned, observeHarness) {
|
|
2232
2312
|
/* ALREADY GIVEN UP ON. Said ONCE, at the bound, and then silently skipped —
|
|
2233
2313
|
the `reportTakeFailed` rule: a line every two minutes forever would bury
|
|
2234
2314
|
every useful one. */
|
|
2235
|
-
const
|
|
2315
|
+
const sourceIds = [...messageIds, ...(attempt?.grantLeases ?? []).map(grant => `grant:${grant.id}`)];
|
|
2316
|
+
const attempts = Math.max(0, ...sourceIds.map(id => state.replyAttempts.get(id) ?? 0));
|
|
2236
2317
|
if (attempts >= MAX_REPLY_ATTEMPTS)
|
|
2237
2318
|
return false;
|
|
2238
2319
|
const fail = (reason) => {
|
|
2239
|
-
for (const id of
|
|
2320
|
+
for (const id of sourceIds) {
|
|
2240
2321
|
state.replyAttempts.set(id, (state.replyAttempts.get(id) ?? 0) + 1);
|
|
2241
2322
|
}
|
|
2242
|
-
const spent = Math.max(...
|
|
2323
|
+
const spent = Math.max(0, ...sourceIds.map(id => state.replyAttempts.get(id) ?? 0));
|
|
2243
2324
|
if (spent >= MAX_REPLY_ATTEMPTS) {
|
|
2244
2325
|
warn(`[orchestrator] gave up answering ${shortId(todoId)} after ${MAX_REPLY_ATTEMPTS} attempts: ${reason}`);
|
|
2245
2326
|
}
|
|
@@ -2287,6 +2368,12 @@ async function answerRunMessagesInner(client, todoId, messageIds, plan, state, s
|
|
|
2287
2368
|
would either promise a boundary that never comes, or re-dispatch a worker
|
|
2288
2369
|
for a change the agent already made itself. */
|
|
2289
2370
|
const workerIsLive = await liveWorkerForTodo(client, todoId, warn);
|
|
2371
|
+
// A stopped request may retain a claim even though its worker has ended.
|
|
2372
|
+
// Capture it before answering so a late stopped reply cannot clear a newer one.
|
|
2373
|
+
const current = await client.from('cliv2_loose_todos').select('claimed_at').eq('id', todoId).maybeSingle();
|
|
2374
|
+
if (current.error)
|
|
2375
|
+
throw current.error;
|
|
2376
|
+
const replyingToClaim = current.data?.claimed_at ?? null;
|
|
2290
2377
|
/* ═══ 18k SLICE 10b — SPAWNED AS A WORKER, WITH NO CLOCK ON IT. ═══
|
|
2291
2378
|
The worker's own adapter, so the answerer holds the worker's toolset (ruling
|
|
2292
2379
|
1) and no timeout at all (ruling 5) — an agent asked for a real change may
|
|
@@ -2304,166 +2391,53 @@ async function answerRunMessagesInner(client, todoId, messageIds, plan, state, s
|
|
|
2304
2391
|
NO `alsoRunningIn`: the answerer works one run, in that run's own checkout.
|
|
2305
2392
|
`runTodoId` IS the run, so a tool call it makes is attributed without
|
|
2306
2393
|
depending on the agent repeating an id back to us. */
|
|
2394
|
+
if (attempt) {
|
|
2395
|
+
const pending = await client.from('cliv2_run_messages').select('id').eq('todo_id', todoId).is('answered_by', null);
|
|
2396
|
+
if (pending.error)
|
|
2397
|
+
throw pending.error;
|
|
2398
|
+
attempt.observedPendingMessageIds = (pending.data ?? []).map((row) => row.id);
|
|
2399
|
+
owned?.setReference(attempt);
|
|
2400
|
+
}
|
|
2401
|
+
const harnessIdentity = executionIdentity(plan.agent);
|
|
2307
2402
|
let spawnedPid = null;
|
|
2308
2403
|
const result = await spawn(plan.agent, buildReplyPrompt(record, bodies, { hasCheckout: plan.cwd !== null, workingCopyElsewhere: plan.workingCopyElsewhere }, workerIsLive), undefined, plan.cwd, () => todoWasStopped(client, todoId), (pid) => {
|
|
2309
2404
|
state.livePids.add(pid);
|
|
2310
2405
|
spawnedPid = pid;
|
|
2311
|
-
}, undefined, todoId);
|
|
2406
|
+
}, undefined, todoId, owned);
|
|
2312
2407
|
/* HOWEVER THE ANSWER ENDED. The pid must leave `livePids` on every path, or a
|
|
2313
2408
|
dead child looks alive to the liveness watch forever. */
|
|
2314
2409
|
if (spawnedPid !== null)
|
|
2315
2410
|
state.livePids.delete(spawnedPid);
|
|
2316
|
-
|
|
2317
|
-
CHECKED BEFORE `!result.ok`, because a killed child also reports not-ok, and
|
|
2318
|
-
falling through would count the user's own decision as an attempt and
|
|
2319
|
-
re-spawn the very thing they cancelled — `dispatchClaimInner` makes the same
|
|
2320
|
-
check for the same reason.
|
|
2321
|
-
|
|
2322
|
-
BUT STOPPING ALONE WOULD BE AN INFINITE SPAWN LOOP. A stopped run stays
|
|
2323
|
-
stopped, so the message stays uncovered, the lease lapses, the next take
|
|
2324
|
-
spawns another answerer, and that one is stopped too — forever, burning no
|
|
2325
|
-
budget because no attempt is counted. So the messages are COVERED, and
|
|
2326
|
-
`cliv2_mark_messages_answered` needs a reply id, so one short reply is what
|
|
2327
|
-
covers them. That is also the honest thread: it says the message was not
|
|
2328
|
-
answered because the work was stopped, rather than showing "Queued" forever.
|
|
2329
|
-
ux.md's north star forbids "this will never be answered", and an uncovered
|
|
2330
|
-
stopped message is exactly that state.
|
|
2331
|
-
|
|
2332
|
-
AND THE CLAIM. This function set no claim, but the run may carry one from an
|
|
2333
|
-
earlier dispatch, and `releaseStoppedTodo` is the only thing that clears it —
|
|
2334
|
-
without this call Retry cannot pick the run up. Its own `.eq('state','stopped')`
|
|
2335
|
-
guard makes it a no-op when Retry has already re-armed the run. */
|
|
2336
|
-
if (result.stopped) {
|
|
2337
|
-
try {
|
|
2338
|
-
const replyId = await writeRunReply(client, todoId, 'Stopped.');
|
|
2339
|
-
const { error } = await client.rpc('cliv2_mark_messages_answered', {
|
|
2340
|
-
p_reply_id: replyId,
|
|
2341
|
-
p_message_ids: messageIds,
|
|
2342
|
-
});
|
|
2343
|
-
if (error)
|
|
2344
|
-
throw new Error(error.message);
|
|
2345
|
-
for (const id of messageIds)
|
|
2346
|
-
state.replyAttempts.delete(id);
|
|
2347
|
-
}
|
|
2348
|
-
catch (err) {
|
|
2349
|
-
warn(`[orchestrator] ${shortId(todoId)} was stopped and the conversation could not be closed off: ${err.message}`);
|
|
2350
|
-
}
|
|
2351
|
-
try {
|
|
2352
|
-
await releaseStoppedTodo(client, todoId);
|
|
2353
|
-
}
|
|
2354
|
-
catch (err) {
|
|
2355
|
-
warn(`[orchestrator] ${shortId(todoId)} was stopped and its claim could not be released: ${err.message}`);
|
|
2356
|
-
}
|
|
2411
|
+
if (result.interrupted)
|
|
2357
2412
|
return false;
|
|
2358
|
-
|
|
2359
|
-
if (!result.ok) {
|
|
2360
|
-
fail(
|
|
2361
|
-
|
|
2362
|
-
}
|
|
2363
|
-
const body = result.output.trim();
|
|
2364
|
-
if (body === '') {
|
|
2365
|
-
fail('it said nothing');
|
|
2413
|
+
const failureKind = result.failureKind ?? nativeFailureKind(result.saidTail ?? '');
|
|
2414
|
+
if (!result.ok && !result.stopped && failureKind === 'transient' && attempts + 1 < MAX_REPLY_ATTEMPTS) {
|
|
2415
|
+
fail(failureMessage(plan.agent, failureKind));
|
|
2416
|
+
owned?.complete();
|
|
2366
2417
|
return false;
|
|
2367
2418
|
}
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
═══ 18k SLICE 9: AND IT IS CLEARED HERE, BEFORE THE COVERING WRITE. ═══
|
|
2380
|
-
The reply EXISTS from this line on, so the attempt is spent whatever happens
|
|
2381
|
-
next, and the covering below must NOT go through `fail()`. `fail()`'s
|
|
2382
|
-
contract is "nothing was written, so retry the whole thing", and that
|
|
2383
|
-
contract does not hold once a reply is in the thread. Had the covering's
|
|
2384
|
-
failure counted an attempt, three consecutive attribution failures — a reply
|
|
2385
|
-
written EACH time — would exhaust `MAX_REPLY_ATTEMPTS` while `answered_by`
|
|
2386
|
-
stayed null, so the take would keep re-offering the row forever and nothing
|
|
2387
|
-
would ever spawn again: three replies, then permanent silence with "Queued"
|
|
2388
|
-
showing. Strictly worse than the predicate this slice replaces, which at
|
|
2389
|
-
least excluded such a message after the first reply. */
|
|
2390
|
-
for (const id of messageIds)
|
|
2419
|
+
if (!result.ok && !result.stopped && failureKind === 'transient')
|
|
2420
|
+
fail(failureMessage(plan.agent, failureKind));
|
|
2421
|
+
const succeeded = result.ok && !!result.output.trim();
|
|
2422
|
+
const body = result.stopped ? 'Stopped.' : succeeded ? result.output.trim() : null;
|
|
2423
|
+
if (!attempt || !machineId)
|
|
2424
|
+
throw new Error('The reply has no exact source lease identity.');
|
|
2425
|
+
const terminal = { surface: 'reply', completedAt: Date.now(), harnessIdentity, ok: succeeded, text: body, reason: succeeded ? '' : failureMessage(plan.agent, failureKind), failureKind: succeeded ? undefined : failureKind, stopped: !!result.stopped, replyingToClaim, workerIsLive: result.ok && workerIsLive };
|
|
2426
|
+
owned?.deferOutcome(terminal);
|
|
2427
|
+
// A durable terminal result is awaiting acknowledgement, never another native attempt.
|
|
2428
|
+
for (const id of sourceIds)
|
|
2391
2429
|
state.replyAttempts.delete(id);
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
beats a silent drop, which is the north star's own ranking (ux.md:739).
|
|
2404
|
-
|
|
2405
|
-
AND THE BOUND IS NOT LOST, IT MOVES: the RPC's own `answered_by is null`
|
|
2406
|
-
term is what bounds a repeated failure here. Each retry round writes at most
|
|
2407
|
-
one extra reply and rounds are one pass apart, so the user-visible bound is
|
|
2408
|
-
duplicate replies at one per pass, never an unbounded spawn loop and never
|
|
2409
|
-
permanent silence. A database that will not accept the mark is an outage,
|
|
2410
|
-
and the honest behaviour under an outage is a visible duplicate rather than
|
|
2411
|
-
a message that disappears. */
|
|
2412
|
-
/* 18k Slice 10b — NOTHING TO ATTRIBUTE WHEN AN APPROVAL WOKE THIS. The reply
|
|
2413
|
-
answers a grant, not a message, so there is no row to mark and the RPC is
|
|
2414
|
-
skipped rather than called with an empty list. The grant's own `consumed_at`
|
|
2415
|
-
is what stops it being offered again, stamped by the tool when the approved
|
|
2416
|
-
call finally goes through. */
|
|
2417
|
-
if (messageIds.length > 0) {
|
|
2418
|
-
try {
|
|
2419
|
-
const { error } = await client.rpc('cliv2_mark_messages_answered', {
|
|
2420
|
-
p_reply_id: replyId,
|
|
2421
|
-
p_message_ids: messageIds,
|
|
2422
|
-
});
|
|
2423
|
-
if (error)
|
|
2424
|
-
throw new Error(error.message);
|
|
2425
|
-
}
|
|
2426
|
-
catch (err) {
|
|
2427
|
-
warn(`[orchestrator] the reply to ${shortId(todoId)} could not be attributed, it will be answered again: ${err.message}`);
|
|
2428
|
-
return false;
|
|
2429
|
-
}
|
|
2430
|
-
}
|
|
2431
|
-
/* ═══ 18k SLICE 10b — CASE B: THE BOUNDARY. ═══
|
|
2432
|
-
An agent was already working this checkout, so the reply said what would
|
|
2433
|
-
happen rather than doing it. Clearing the claim is what makes that true: the
|
|
2434
|
-
next tick dispatches a fresh worker, and `readRunDirection` hands it the
|
|
2435
|
-
message. Nothing interrupts the live worker — it finishes what it is on, and
|
|
2436
|
-
the change rides the NEXT dispatch, exactly as `hand_off_stage` already does
|
|
2437
|
-
mid-run.
|
|
2438
|
-
|
|
2439
|
-
AFTER THE REPLY AND ITS COVERING, NEVER BEFORE. Clearing first and then
|
|
2440
|
-
failing to write the reply would re-dispatch a worker for a change nobody
|
|
2441
|
-
was ever told about.
|
|
2442
|
-
|
|
2443
|
-
GUARDED ON `working`/`needs-input`, for `releaseTodoForRedispatch`'s own
|
|
2444
|
-
reason: a run that has since finished, failed or been stopped must not be
|
|
2445
|
-
resurrected into `working` by a claim clear. And all three columns move
|
|
2446
|
-
together — `cliv2_loose_todos_claim_complete_check` makes a half-release a
|
|
2447
|
-
database error.
|
|
2448
|
-
|
|
2449
|
-
BEST-EFFORT: the reply is in the thread whatever happens here, so a failed
|
|
2450
|
-
clear costs the user a change that does not get dispatched — which the next
|
|
2451
|
-
message re-raises — where throwing would lose the reply's own success. */
|
|
2452
|
-
if (workerIsLive) {
|
|
2453
|
-
try {
|
|
2454
|
-
const { error } = await client
|
|
2455
|
-
.from('cliv2_loose_todos')
|
|
2456
|
-
.update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
|
|
2457
|
-
.eq('id', todoId)
|
|
2458
|
-
.in('state', ['working', 'needs-input']);
|
|
2459
|
-
if (error)
|
|
2460
|
-
throw new Error(error.message);
|
|
2461
|
-
}
|
|
2462
|
-
catch (err) {
|
|
2463
|
-
warn(`[orchestrator] ${shortId(todoId)} was answered but the change could not be scheduled: ${err.message}`);
|
|
2464
|
-
}
|
|
2465
|
-
}
|
|
2466
|
-
return true;
|
|
2430
|
+
const answered = await settleReplyOutcome(client, machineId, attempt, terminal, () => {
|
|
2431
|
+
if (succeeded && !result.stopped)
|
|
2432
|
+
observeHarness?.(plan.agent, 'authenticated', 'dispatch-success', terminal.completedAt, terminal.harnessIdentity);
|
|
2433
|
+
else if (failureKind === 'authentication')
|
|
2434
|
+
observeHarness?.(plan.agent, 'sign-in-required', 'provider-rejected', terminal.completedAt, terminal.harnessIdentity);
|
|
2435
|
+
});
|
|
2436
|
+
owned?.acknowledgeOutcome();
|
|
2437
|
+
owned?.complete();
|
|
2438
|
+
for (const id of sourceIds)
|
|
2439
|
+
state.replyAttempts.delete(id);
|
|
2440
|
+
return answered;
|
|
2467
2441
|
}
|
|
2468
2442
|
/** 18k Slice 7 — say ONCE that the take is failing, not every 10 seconds.
|
|
2469
2443
|
*
|
|
@@ -2504,7 +2478,7 @@ export function reportTakeFailed(state, reason, warn) {
|
|
|
2504
2478
|
* is the brake, exactly as it is for the scope park, and answering is what lets
|
|
2505
2479
|
* it off.
|
|
2506
2480
|
*/
|
|
2507
|
-
export async function askWhichCodebase(client, todoId, projectName, remotes, warn) {
|
|
2481
|
+
export async function askWhichCodebase(client, todoId, projectName, remotes, warn, claimedAt) {
|
|
2508
2482
|
try {
|
|
2509
2483
|
const { error } = await client.rpc('cliv2_ask_about_request', {
|
|
2510
2484
|
p_todo: todoId,
|
|
@@ -2517,7 +2491,7 @@ export async function askWhichCodebase(client, todoId, projectName, remotes, war
|
|
|
2517
2491
|
});
|
|
2518
2492
|
if (error)
|
|
2519
2493
|
throw new Error(error.message);
|
|
2520
|
-
await parkTodoForScope(client, todoId);
|
|
2494
|
+
await parkTodoForScope(client, todoId, claimedAt);
|
|
2521
2495
|
return true;
|
|
2522
2496
|
}
|
|
2523
2497
|
catch (err) {
|
|
@@ -2797,11 +2771,13 @@ export async function collisionForTodo(client, todoId, gitRemoteUrl, warn) {
|
|
|
2797
2771
|
return null;
|
|
2798
2772
|
}
|
|
2799
2773
|
}
|
|
2800
|
-
export async function releaseTodo(client, todoId) {
|
|
2801
|
-
const
|
|
2774
|
+
export async function releaseTodo(client, todoId, claimedAt) {
|
|
2775
|
+
const write = client
|
|
2802
2776
|
.from('cliv2_loose_todos')
|
|
2803
2777
|
.update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
|
|
2804
|
-
.eq('id', todoId)
|
|
2778
|
+
.eq('id', todoId)
|
|
2779
|
+
.in('state', ['working', 'needs-input']);
|
|
2780
|
+
const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
|
|
2805
2781
|
if (error)
|
|
2806
2782
|
throw error;
|
|
2807
2783
|
}
|
|
@@ -3707,12 +3683,13 @@ run = spawnAgent) {
|
|
|
3707
3683
|
alsoRunningIn,
|
|
3708
3684
|
/** 18c Slice 1 — the request this spawn is working, so
|
|
3709
3685
|
* `report_activity` lines can be attributed without the prompt. */
|
|
3710
|
-
runTodoId) =>
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
3686
|
+
runTodoId, owned) => {
|
|
3687
|
+
/* !Cleanup Phase 6b (I42): NO TIMEOUT. `NO_SPAWN_TIMEOUT` disables the kill
|
|
3688
|
+
entirely — a working agent is never killed for elapsed time (user ruling
|
|
3689
|
+
2026-08-06). Liveness is watched from outside the run instead, by
|
|
3690
|
+
`reapDeadWorkers` on the heartbeat. */
|
|
3691
|
+
return run(agent, prompt, NO_SPAWN_TIMEOUT, { onStep, mcpServer, cwd, stopRequested, onPid, alsoRunningIn, runTodoId, owned });
|
|
3692
|
+
};
|
|
3716
3693
|
}
|
|
3717
3694
|
/** 18k Slice 8 — how many times one message may be attempted before the daemon
|
|
3718
3695
|
* stops spawning for it.
|
|
@@ -3727,7 +3704,9 @@ export const MAX_REPLY_ATTEMPTS = 3;
|
|
|
3727
3704
|
/** Bound on retained stdout. Generous — a real run's JSONL is a few hundred KB —
|
|
3728
3705
|
* but finite, so a runaway agent cannot exhaust memory. */
|
|
3729
3706
|
const MAX_OUTPUT_BYTES = 32 * 1024 * 1024;
|
|
3730
|
-
export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_TIMEOUT_MS, { resolveBin = agentPath, exec, onStep, spawnProcess = spawnChild, maxOutputBytes = MAX_OUTPUT_BYTES, mcpServer = null, cwd = null, stopRequested, stopPollMs = STOP_POLL_MS, onPid, alsoRunningIn = [], runTodoId = null } = {}) {
|
|
3707
|
+
export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_TIMEOUT_MS, { resolveBin = agentPath, exec, onStep, spawnProcess = spawnChild, maxOutputBytes = MAX_OUTPUT_BYTES, mcpServer = null, cwd = null, stopRequested, stopPollMs = STOP_POLL_MS, onPid, alsoRunningIn = [], runTodoId = null, owned } = {}) {
|
|
3708
|
+
if (owned?.interrupted())
|
|
3709
|
+
return { ok: false, output: '', error: 'Work was interrupted by the service command.', interrupted: true };
|
|
3731
3710
|
const bin = resolveBin(agent);
|
|
3732
3711
|
if (!bin)
|
|
3733
3712
|
return { ok: false, output: '', error: `${agent} is not installed on this machine` };
|
|
@@ -3741,7 +3720,7 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
3741
3720
|
Windows path holds spaces (`C:\\Program Files\\…`) and the MCP config is
|
|
3742
3721
|
JSON, so both would still be split by cmd.exe. */
|
|
3743
3722
|
const viaShell = needsShell(bin);
|
|
3744
|
-
const rawArgs = headlessAgentArgs(agent, prompt, mcpServer, alsoRunningIn,
|
|
3723
|
+
const rawArgs = headlessAgentArgs(agent, prompt, mcpServer, alsoRunningIn, true, runTodoId);
|
|
3745
3724
|
const args = viaShell ? rawArgs.map(quoteForCmd) : rawArgs;
|
|
3746
3725
|
/* !Cleanup Phase 2 (I5), CORRECTED BY 18b SLICE 2.
|
|
3747
3726
|
It used to be `cwd ?? undefined`, and `undefined` means "inherit the
|
|
@@ -3787,10 +3766,8 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
3787
3766
|
const codexHome = agent === 'claude'
|
|
3788
3767
|
? null
|
|
3789
3768
|
: ensureCodexRunHome(mcpServer, runTodoId, runTodoId ? `${runTodoId}-${randomUUID().slice(0, 8)}` : null);
|
|
3790
|
-
if (
|
|
3791
|
-
|
|
3792
|
-
'credential to seed a per-run home. This worker inherits the MCP servers, instruction ' +
|
|
3793
|
-
'files and permission posture on this machine.');
|
|
3769
|
+
if (codexHome instanceof CodexHomeFailure) {
|
|
3770
|
+
return { ok: false, output: '', error: codexHome.message, failureKind: codexHome.kind };
|
|
3794
3771
|
}
|
|
3795
3772
|
/* Spread the parent's env rather than replacing it: codex needs PATH, HOME,
|
|
3796
3773
|
the proxy variables and the user's shell environment to run at all. Only the
|
|
@@ -3829,13 +3806,18 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
3829
3806
|
return new Promise((resolve) => {
|
|
3830
3807
|
let child;
|
|
3831
3808
|
try {
|
|
3832
|
-
|
|
3809
|
+
owned?.beforeSpawn();
|
|
3810
|
+
const spawnExecution = owned && spawnProcess === spawnChild
|
|
3811
|
+
? (file, argv, options) => spawnOwnedProcess(file, argv, options, owned.id)
|
|
3812
|
+
: spawnProcess;
|
|
3813
|
+
child = spawnExecution(bin, args, {
|
|
3833
3814
|
// The repo-wide invariant for ANY child process (MEMORY: "Windows
|
|
3834
3815
|
// silence decision"). A user must never see a console flash.
|
|
3835
3816
|
windowsHide: true,
|
|
3836
3817
|
shell: viaShell,
|
|
3837
3818
|
// 18a Slice 7: stdin is a pipe only where the prompt travels down it.
|
|
3838
|
-
stdio: [
|
|
3819
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
3820
|
+
detached: process.platform !== 'win32',
|
|
3839
3821
|
// !Cleanup Phase 2 (I5) — the worker starts inside the work item's own
|
|
3840
3822
|
// checkout instead of at the daemon's cwd (`/` under launchd).
|
|
3841
3823
|
cwd: childCwd,
|
|
@@ -3849,17 +3831,15 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
3849
3831
|
/* Nothing was spawned, so nothing can still be holding the directory. */
|
|
3850
3832
|
if (codexHome)
|
|
3851
3833
|
removeCodexRunHome(codexHome);
|
|
3852
|
-
return resolve(spawnFailure(agent, err));
|
|
3834
|
+
return resolve({ ...spawnFailure(agent, err), ...(owned?.interrupted() ? { interrupted: true } : {}) });
|
|
3853
3835
|
}
|
|
3854
3836
|
/* 18a SLICE 7 — AND THE PROMPT GOES DOWN IT, then the pipe is CLOSED.
|
|
3855
3837
|
Both agents wait for end-of-input before they start, so leaving it open
|
|
3856
3838
|
hangs the run forever. The write is best-effort in the same sense every
|
|
3857
3839
|
other pipe operation here is: a child that died before reading raises
|
|
3858
3840
|
EPIPE, and that is the child's exit to report, not this write's. */
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
child.stdin.end(prompt);
|
|
3862
|
-
}
|
|
3841
|
+
let registration = Promise.resolve();
|
|
3842
|
+
child.stdin?.on('error', () => { });
|
|
3863
3843
|
/* !Cleanup Phase 6b (I43) — hand the pid up the moment it exists, so the
|
|
3864
3844
|
liveness watch can tell a dead agent from a quiet one. Guarded on
|
|
3865
3845
|
`typeof`: a child that failed to spawn has no pid, and reporting
|
|
@@ -4020,9 +4000,24 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
4020
4000
|
it stripped to '', the column would take null and the card would lose
|
|
4021
4001
|
the context its question needs. `redactSecrets` is identity when the run
|
|
4022
4002
|
read no credential, which is almost every run. */
|
|
4023
|
-
|
|
4024
|
-
|
|
4025
|
-
|
|
4003
|
+
void (async () => {
|
|
4004
|
+
await registration.catch(() => { });
|
|
4005
|
+
if (owned && typeof child.pid === 'number') {
|
|
4006
|
+
for (;;) {
|
|
4007
|
+
try {
|
|
4008
|
+
await owned.exited();
|
|
4009
|
+
break;
|
|
4010
|
+
}
|
|
4011
|
+
catch {
|
|
4012
|
+
await new Promise((done) => setTimeout(done, 250));
|
|
4013
|
+
}
|
|
4014
|
+
}
|
|
4015
|
+
}
|
|
4016
|
+
if (codexHome && owned)
|
|
4017
|
+
removeCodexRunHome(codexHome);
|
|
4018
|
+
resolve({ ...(result.output ? { ...result, output: redactSecrets(runTodoId, result.output) } : result),
|
|
4019
|
+
...(owned?.interrupted() ? { interrupted: true } : {}) });
|
|
4020
|
+
})();
|
|
4026
4021
|
};
|
|
4027
4022
|
/* ═══ 18c SLICE 8 — THE HOME IS SWEPT WHEN THE CHILD IS GONE, NOT WHEN THE
|
|
4028
4023
|
PROMISE SETTLES. ═══
|
|
@@ -4058,7 +4053,8 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
4058
4053
|
sweptHome = true;
|
|
4059
4054
|
removeCodexRunHome(codexHome);
|
|
4060
4055
|
};
|
|
4061
|
-
|
|
4056
|
+
if (!owned)
|
|
4057
|
+
child.on('close', sweepHomeWhenChildIsGone);
|
|
4062
4058
|
/* !Cleanup PHASE 6b (I42) — THE ELAPSED-TIME KILL IS OFF BY DEFAULT NOW.
|
|
4063
4059
|
User ruling 2026-08-06: "No agent that is actively working should ever be
|
|
4064
4060
|
killed automatically, regardless of how much time has elapsed."
|
|
@@ -4186,7 +4182,7 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
4186
4182
|
401 that means signed out. Each tailed separately so a chatty stdout
|
|
4187
4183
|
cannot push stderr out of the window. */
|
|
4188
4184
|
const saidTail = `${stdout.slice(-STDERR_TAIL_BYTES)}\n${stderrTail}`;
|
|
4189
|
-
return settle({ ok: false, output: '', error: `${agent} ${how}`, saidTail });
|
|
4185
|
+
return settle({ ok: false, output: '', error: `${agent} ${how}`, saidTail, failureKind: nativeFailureKind(stdout, stderrTail) });
|
|
4190
4186
|
}
|
|
4191
4187
|
/* 18c SLICE 8 — CODEX CAN EXIT 0 ON A TURN THAT FAILED, so the exit code
|
|
4192
4188
|
above is not the last word. Checked AFTER the code, because a non-zero
|
|
@@ -4196,7 +4192,7 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
4196
4192
|
release path, like every other failure, so the unit is retried. */
|
|
4197
4193
|
if (codexTurnFailed !== null) {
|
|
4198
4194
|
const saidTail = `${stdout.slice(-STDERR_TAIL_BYTES)}\n${stderrTail}`;
|
|
4199
|
-
return settle({ ok: false, output: '', error: `${agent} ${codexTurnFailed}`, saidTail });
|
|
4195
|
+
return settle({ ok: false, output: '', error: `${agent} ${codexTurnFailed}`, saidTail, failureKind: nativeFailureKind(stdout, stderrTail) });
|
|
4200
4196
|
}
|
|
4201
4197
|
/* PREFER WHAT WAS CAPTURED LIVE, FOR BOTH HARNESSES (18c Slice 8 — it used
|
|
4202
4198
|
to be claude only). `stdout` is now the fallback only for a run that
|
|
@@ -4214,6 +4210,18 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
4214
4210
|
way: anything parseable was already captured live above. */
|
|
4215
4211
|
settle(finishSpawn(agent, stdout));
|
|
4216
4212
|
});
|
|
4213
|
+
registration = owned ? owned.register(child) : Promise.resolve();
|
|
4214
|
+
void registration.then(() => {
|
|
4215
|
+
if (!owned?.interrupted()) {
|
|
4216
|
+
if (owned)
|
|
4217
|
+
releaseOwnedProcess(child);
|
|
4218
|
+
child.stdin?.end(prompt);
|
|
4219
|
+
}
|
|
4220
|
+
}).catch((error) => {
|
|
4221
|
+
child.stdin?.destroy();
|
|
4222
|
+
killTree(child);
|
|
4223
|
+
settle(spawnFailure(agent, error));
|
|
4224
|
+
});
|
|
4217
4225
|
});
|
|
4218
4226
|
}
|
|
4219
4227
|
/**
|
|
@@ -4401,7 +4409,7 @@ export async function startWorker(client, todoId, machineId, agent, warn = conso
|
|
|
4401
4409
|
/** Move a worker through its lifecycle. `died` REQUIRES a reason — the table's
|
|
4402
4410
|
* check constraint refuses the row otherwise, deliberately: a failure the panel
|
|
4403
4411
|
* cannot explain gives the user nothing to act on. */
|
|
4404
|
-
export async function updateWorker(client, workerId, state, error = null, warn = console.warn) {
|
|
4412
|
+
export async function updateWorker(client, workerId, state, error = null, warn = console.warn, strict = false) {
|
|
4405
4413
|
if (!workerId)
|
|
4406
4414
|
return;
|
|
4407
4415
|
try {
|
|
@@ -4412,11 +4420,14 @@ export async function updateWorker(client, workerId, state, error = null, warn =
|
|
|
4412
4420
|
const { error: writeError } = await client
|
|
4413
4421
|
.from('cliv2_workers')
|
|
4414
4422
|
.update(patch)
|
|
4415
|
-
.eq('id', workerId)
|
|
4423
|
+
.eq('id', workerId)
|
|
4424
|
+
.in('state', ['working', 'dispatching']);
|
|
4416
4425
|
if (writeError)
|
|
4417
4426
|
throw writeError;
|
|
4418
4427
|
}
|
|
4419
4428
|
catch (err) {
|
|
4429
|
+
if (strict)
|
|
4430
|
+
throw err;
|
|
4420
4431
|
warn(`[orchestrator] could not update the worker: ${err.message}`);
|
|
4421
4432
|
}
|
|
4422
4433
|
}
|
|
@@ -4596,6 +4607,11 @@ export async function reapDeadWorkers(client, machineId, livePids, now = () => n
|
|
|
4596
4607
|
const cutoff = now().getTime() - STALE_WORKER_MS;
|
|
4597
4608
|
let reaped = 0;
|
|
4598
4609
|
for (const row of rows) {
|
|
4610
|
+
if (hasOwnedWorkContext() && ownedWorkerExecutionHeld(row.id))
|
|
4611
|
+
continue;
|
|
4612
|
+
const claimedAt = hasOwnedWorkContext() ? ownedWorkerClaimedAt(row.id) : undefined;
|
|
4613
|
+
if (claimedAt === null)
|
|
4614
|
+
continue;
|
|
4599
4615
|
/* CONDITION 1 — quiet. A worker that has written a step recently is
|
|
4600
4616
|
working, and nothing else is even considered. */
|
|
4601
4617
|
const seen = Date.parse(row.updated_at);
|
|
@@ -4641,7 +4657,9 @@ export async function reapDeadWorkers(client, machineId, livePids, now = () => n
|
|
|
4641
4657
|
card everything Slices 5 and 6 already draw (the attempt, the
|
|
4642
4658
|
reason, the countdown) for free, and at the bound it gives up and
|
|
4643
4659
|
says why instead of being fed back into the same machine. */
|
|
4644
|
-
const { fate, attempts } = await releaseOrFail(client, row.todo_id, plain, now().getTime());
|
|
4660
|
+
const { fate, attempts } = await releaseOrFail(client, row.todo_id, plain, now().getTime(), claimedAt);
|
|
4661
|
+
if (fate === 'stale')
|
|
4662
|
+
continue;
|
|
4645
4663
|
/* 18c SLICE 9 (GAP 15): AND IT HOLDS NOTHING ANY MORE.
|
|
4646
4664
|
THIS PATH NEEDS ITS OWN CALL and is not covered by `dispatchClaim`'s
|
|
4647
4665
|
`finally`: the reaper exists precisely for a worker that died while
|
|
@@ -4705,6 +4723,7 @@ export async function reapDeadWorkers(client, machineId, livePids, now = () => n
|
|
|
4705
4723
|
*/
|
|
4706
4724
|
export async function recoverStrandedWorkers(client, machineId, log = console.log, warn = console.warn) {
|
|
4707
4725
|
try {
|
|
4726
|
+
const lifecycleOwned = hasOwnedWorkContext();
|
|
4708
4727
|
const { data, error } = await client
|
|
4709
4728
|
.from('cliv2_workers')
|
|
4710
4729
|
.update({
|
|
@@ -4732,7 +4751,11 @@ export async function recoverStrandedWorkers(client, machineId, log = console.lo
|
|
|
4732
4751
|
/* The todo goes back in the queue. Order matters: the worker is closed
|
|
4733
4752
|
FIRST, so there is no instant at which a claimable todo still has a live
|
|
4734
4753
|
worker that would refuse the dispatch. */
|
|
4735
|
-
|
|
4754
|
+
// Lifecycle startup has already reconciled exact attempt receipts. A stale
|
|
4755
|
+
// worker row can outlive a newer claim, including one on another machine;
|
|
4756
|
+
// closing that obsolete worker must never release the newer assignment.
|
|
4757
|
+
// Keep the pre-lifecycle compatibility path only for its existing callers.
|
|
4758
|
+
for (const row of lifecycleOwned ? [] : stranded) {
|
|
4736
4759
|
try {
|
|
4737
4760
|
await releaseTodo(client, row.todo_id);
|
|
4738
4761
|
}
|
|
@@ -5287,7 +5310,7 @@ export async function orchestratorTick(deps) {
|
|
|
5287
5310
|
query, so a tick that cannot start anything costs nothing at all — no read,
|
|
5288
5311
|
no claim, no log. Was `if (state.busy)`, which is why a second request typed
|
|
5289
5312
|
three seconds after the first sat untouched until the first finished.
|
|
5290
|
-
|
|
5313
|
+
*
|
|
5291
5314
|
THE SLOT IS TAKEN HERE, NOT AFTER THE CLAIM, and that is not tidiness — it
|
|
5292
5315
|
is the difference between a cap and a suggestion. `pollOrchestrator` is an
|
|
5293
5316
|
interval that does NOT await, so ticks genuinely overlap, and there are
|
|
@@ -5297,7 +5320,7 @@ export async function orchestratorTick(deps) {
|
|
|
5297
5320
|
the cap would bound nothing. Taking the slot before the first await makes
|
|
5298
5321
|
the check-and-take atomic with respect to the event loop, because there is
|
|
5299
5322
|
no suspension point between them.
|
|
5300
|
-
|
|
5323
|
+
*
|
|
5301
5324
|
EVERY early return below MUST give it back, which is why the whole body from
|
|
5302
5325
|
here down is wrapped so the release is in one place rather than repeated at
|
|
5303
5326
|
each `return`. */
|
|
@@ -5305,8 +5328,14 @@ export async function orchestratorTick(deps) {
|
|
|
5305
5328
|
return 'busy';
|
|
5306
5329
|
state.inFlight += 1;
|
|
5307
5330
|
try {
|
|
5331
|
+
await reconcileInterruptedWork(deps.client, deps.machineId, deps.observeHarness);
|
|
5308
5332
|
return await runTick(deps, state, now, log, warn);
|
|
5309
5333
|
}
|
|
5334
|
+
catch (error) {
|
|
5335
|
+
// This supervising tick must survive an outage and retry its saved recovery.
|
|
5336
|
+
warn(`orchestrator poll failed, will retry: ${error instanceof Error ? error.message : JSON.stringify(error)}`);
|
|
5337
|
+
return 'failed';
|
|
5338
|
+
}
|
|
5310
5339
|
finally {
|
|
5311
5340
|
state.inFlight -= 1;
|
|
5312
5341
|
}
|
|
@@ -5314,208 +5343,230 @@ export async function orchestratorTick(deps) {
|
|
|
5314
5343
|
/** The tick body, with its concurrency slot already taken. Split out so the slot
|
|
5315
5344
|
* is released in exactly one place — see the caller. */
|
|
5316
5345
|
async function runTick(deps, state, now, log, warn) {
|
|
5317
|
-
|
|
5318
|
-
|
|
5319
|
-
|
|
5320
|
-
|
|
5321
|
-
start" — deliberately the same resolver, so the two cannot disagree. */
|
|
5322
|
-
const { client, userId, machineId, machineName, agents, resolveContext = (claim) => resolveClaimContext(client, claim, warn), } = deps;
|
|
5323
|
-
/* 18a SLICE 6: THE BACKOFF USED TO BE CHECKED HERE, and returning from this
|
|
5324
|
-
line is precisely what was wrong with it: a failing agent stopped the whole
|
|
5325
|
-
daemon rather than stopping itself. The wait is now per request, on the row,
|
|
5326
|
-
and the claim itself skips a request whose moment has not come. */
|
|
5327
|
-
let role;
|
|
5346
|
+
if (!ownedWorkAllowed())
|
|
5347
|
+
return 'nothing-to-do';
|
|
5348
|
+
const endClaim = beginOwnedClaim();
|
|
5349
|
+
const claimOperations = [];
|
|
5328
5350
|
try {
|
|
5329
|
-
|
|
5330
|
-
|
|
5331
|
-
|
|
5332
|
-
|
|
5333
|
-
|
|
5334
|
-
|
|
5335
|
-
|
|
5336
|
-
|
|
5337
|
-
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
|
|
5341
|
-
// the machine check exists to prevent.
|
|
5342
|
-
warn(`orchestrator designation read failed, will retry: ${err.message}`);
|
|
5343
|
-
return 'failed';
|
|
5344
|
-
}
|
|
5345
|
-
reportRole(state, role, machineName, log);
|
|
5346
|
-
if (role.kind !== 'this-machine')
|
|
5347
|
-
return 'not-designated';
|
|
5348
|
-
/* ═══ 18d — NO TOOLS MEANS NO DISPATCH. ═══
|
|
5349
|
-
|
|
5350
|
-
FOUND BY WALKING IT, 2026-08-12. A daemon restart raced its own port: the
|
|
5351
|
-
replacement process came up while the old one still held 4579, so
|
|
5352
|
-
`toolsServerStatus().running` was false, `presence.ts` passed
|
|
5353
|
-
`mcpServer: null`, and the very next tick claimed a request and spawned a
|
|
5354
|
-
worker WITH NO CTRL+SPC TOOLS AT ALL. That worker could not read the work
|
|
5355
|
-
item, could not read its own stage document, could not call `ask_question`
|
|
5356
|
-
and could not `hand_off_stage`. It did the only thing left: it wrote an
|
|
5357
|
-
essay into its final answer explaining that the tools were missing, and the
|
|
5358
|
-
request was burned — `attempts` spent, the user's card finished, and
|
|
5359
|
-
nothing to show for it.
|
|
5360
|
-
|
|
5361
|
-
Null is the RIGHT value there: the alternative that comment rejects is
|
|
5362
|
-
letting the worker inherit the user's own agent config, where a second
|
|
5363
|
-
CTRL+SPC server pointed at a DIFFERENT ACCOUNT was once found registered and
|
|
5364
|
-
listening. That is worse. But there was a third option nobody took, which is
|
|
5365
|
-
to not dispatch at all.
|
|
5366
|
-
|
|
5367
|
-
WAITING IS FREE AND CORRECT. The request keeps its place, unclaimed and
|
|
5368
|
-
unspent, and the tools server comes up moments later — the port race
|
|
5369
|
-
resolves as soon as the old process lets go. A dispatch made in this window
|
|
5370
|
-
cannot be retried usefully, because it is not the request that is broken.
|
|
5371
|
-
|
|
5372
|
-
BEFORE THE CLAIM, deliberately: claiming and then refusing to spawn would
|
|
5373
|
-
spend `attempts` on a condition that has nothing to do with the request.
|
|
5374
|
-
|
|
5375
|
-
THE CONDITION IS "A REAL SPAWN WITH NO SERVER", not "no server". An injected
|
|
5376
|
-
`spawn` is a caller who has taken over the launching entirely — every test in
|
|
5377
|
-
the suite, and any future embedder — and for them `mcpServer` describes
|
|
5378
|
-
nothing, because `defaultSpawn` is what turns it into an argv. Gating on the
|
|
5379
|
-
null alone would refuse to dispatch in exactly the cases where the tools are
|
|
5380
|
-
not in question, which is not a stricter check but a wrong one. */
|
|
5381
|
-
if (!deps.mcpServer && !deps.spawn) {
|
|
5382
|
-
reportNoTools(state, warn);
|
|
5383
|
-
return 'no-tools';
|
|
5384
|
-
}
|
|
5385
|
-
state.warnedNoTools = false;
|
|
5386
|
-
/* ═══ !Cleanup PHASE 7 SLICE 2: CLAIM PAST THE BLOCKED ONES. ═══
|
|
5387
|
-
|
|
5388
|
-
A dependent unit must WAIT, and everything behind it must NOT. Those two
|
|
5389
|
-
requirements together are why this is a loop rather than a single claim.
|
|
5390
|
-
|
|
5391
|
-
The claim is this design's only queue position, and it takes the OLDEST
|
|
5392
|
-
claimable row. So a blocked unit that is merely released is re-claimed FIRST
|
|
5393
|
-
on the very next tick, forever — it stops being a waiting unit and becomes a
|
|
5394
|
-
queue stopper. Releasing it AND telling the next claim to skip it is what
|
|
5395
|
-
makes the rest of the queue reachable.
|
|
5396
|
-
|
|
5397
|
-
BOUNDED BY THE NUMBER OF UNITS SKIPPED, not by a fixed count: each iteration
|
|
5398
|
-
either dispatches (returns), finds nothing (returns), or adds exactly one id
|
|
5399
|
-
to `skip`, and a skipped id can never be claimed again this tick. So the
|
|
5400
|
-
loop runs at most once per blocked unit and then ends. No timeout needed,
|
|
5401
|
-
because it cannot revisit a row.
|
|
5402
|
-
|
|
5403
|
-
THE UNIT STAYS `working` AND UNCLAIMED, deliberately — it is NOT parked to
|
|
5404
|
-
`needs-input` the way the scope gate parks. `needs-input` means "this needs
|
|
5405
|
-
YOU", and a dependency needs nothing from the user: it resolves when the
|
|
5406
|
-
blocker finishes. Parking it would put a card under "Needs you" offering no
|
|
5407
|
-
action, which is precisely the defect Phase 1 existed to remove. The panel
|
|
5408
|
-
reads the same `blocked_tasks` view and renders "Waiting on <name>" from it,
|
|
5409
|
-
so the card explains itself without a state that lies. */
|
|
5410
|
-
const skip = [];
|
|
5411
|
-
let blockedCount = 0;
|
|
5412
|
-
for (;;) {
|
|
5413
|
-
let claim;
|
|
5351
|
+
/* Only what CHOOSING a unit needs. The spawn's own inputs (`spawn`,
|
|
5352
|
+
`mcpServer`) are destructured in `dispatchClaim`, where they are used.
|
|
5353
|
+
`resolveContext` is needed in BOTH: here it answers "which repo would this
|
|
5354
|
+
touch" for the collision read, and there it answers "where does the worker
|
|
5355
|
+
start" — deliberately the same resolver, so the two cannot disagree. */
|
|
5356
|
+
const { client, userId, machineId, machineName, agents, resolveContext = (claim) => resolveClaimContext(client, claim, warn), } = deps;
|
|
5357
|
+
/* 18a SLICE 6: THE BACKOFF USED TO BE CHECKED HERE, and returning from this
|
|
5358
|
+
line is precisely what was wrong with it: a failing agent stopped the whole
|
|
5359
|
+
daemon rather than stopping itself. The wait is now per request, on the row,
|
|
5360
|
+
and the claim itself skips a request whose moment has not come. */
|
|
5361
|
+
const recovered = hasOwnedWorkContext() ? recoveredRpcClaims('cli').find(row => row.action === 'cliv2_claim_next_loose_todo' && agents.includes(row.args?.p_agent)) : undefined;
|
|
5362
|
+
let role;
|
|
5414
5363
|
try {
|
|
5415
|
-
|
|
5364
|
+
const { data, error } = await client
|
|
5365
|
+
.from('cliv2_orchestrator_preference')
|
|
5366
|
+
.select('machine_id, agent')
|
|
5367
|
+
.eq('user_id', userId)
|
|
5368
|
+
.maybeSingle();
|
|
5369
|
+
if (error)
|
|
5370
|
+
throw error;
|
|
5371
|
+
role = resolveOrchestratorRole(data ?? null, machineId, agents);
|
|
5416
5372
|
}
|
|
5417
5373
|
catch (err) {
|
|
5418
|
-
|
|
5419
|
-
|
|
5374
|
+
// Do NOT fall through to a claim on a failed read: an unreadable designation
|
|
5375
|
+
// is not a designation, and guessing would be exactly the fleet-wide race
|
|
5376
|
+
// the machine check exists to prevent.
|
|
5377
|
+
warn(`orchestrator designation read failed, will retry: ${err.message}`);
|
|
5378
|
+
if (!recovered)
|
|
5379
|
+
return 'failed';
|
|
5380
|
+
role = { kind: 'none' };
|
|
5420
5381
|
}
|
|
5421
|
-
if (
|
|
5422
|
-
|
|
5423
|
-
|
|
5424
|
-
|
|
5425
|
-
return
|
|
5382
|
+
if (recovered)
|
|
5383
|
+
role = { kind: 'this-machine', agent: recovered.args.p_agent };
|
|
5384
|
+
reportRole(state, role, machineName, log);
|
|
5385
|
+
if (role.kind !== 'this-machine')
|
|
5386
|
+
return 'not-designated';
|
|
5387
|
+
/* ═══ 18d — NO TOOLS MEANS NO DISPATCH. ═══
|
|
5388
|
+
*
|
|
5389
|
+
FOUND BY WALKING IT, 2026-08-12. A daemon restart raced its own port: the
|
|
5390
|
+
replacement process came up while the old one still held 4579, so
|
|
5391
|
+
`toolsServerStatus().running` was false, `presence.ts` passed
|
|
5392
|
+
`mcpServer: null`, and the very next tick claimed a request and spawned a
|
|
5393
|
+
worker WITH NO CTRL+SPC TOOLS AT ALL. That worker could not read the work
|
|
5394
|
+
item, could not read its own stage document, could not call `ask_question`
|
|
5395
|
+
and could not `hand_off_stage`. It did the only thing left: it wrote an
|
|
5396
|
+
essay into its final answer explaining that the tools were missing, and the
|
|
5397
|
+
request was burned — `attempts` spent, the user's card finished, and
|
|
5398
|
+
nothing to show for it.
|
|
5399
|
+
*
|
|
5400
|
+
Null is the RIGHT value there: the alternative that comment rejects is
|
|
5401
|
+
letting the worker inherit the user's own agent config, where a second
|
|
5402
|
+
CTRL+SPC server pointed at a DIFFERENT ACCOUNT was once found registered and
|
|
5403
|
+
listening. That is worse. But there was a third option nobody took, which is
|
|
5404
|
+
to not dispatch at all.
|
|
5405
|
+
*
|
|
5406
|
+
WAITING IS FREE AND CORRECT. The request keeps its place, unclaimed and
|
|
5407
|
+
unspent, and the tools server comes up moments later — the port race
|
|
5408
|
+
resolves as soon as the old process lets go. A dispatch made in this window
|
|
5409
|
+
cannot be retried usefully, because it is not the request that is broken.
|
|
5410
|
+
*
|
|
5411
|
+
BEFORE THE CLAIM, deliberately: claiming and then refusing to spawn would
|
|
5412
|
+
spend `attempts` on a condition that has nothing to do with the request.
|
|
5413
|
+
*
|
|
5414
|
+
THE CONDITION IS "A REAL SPAWN WITH NO SERVER", not "no server". An injected
|
|
5415
|
+
`spawn` is a caller who has taken over the launching entirely — every test in
|
|
5416
|
+
the suite, and any future embedder — and for them `mcpServer` describes
|
|
5417
|
+
nothing, because `defaultSpawn` is what turns it into an argv. Gating on the
|
|
5418
|
+
null alone would refuse to dispatch in exactly the cases where the tools are
|
|
5419
|
+
not in question, which is not a stricter check but a wrong one. */
|
|
5420
|
+
if (!deps.mcpServer && !deps.spawn) {
|
|
5421
|
+
reportNoTools(state, warn);
|
|
5422
|
+
return 'no-tools';
|
|
5426
5423
|
}
|
|
5427
|
-
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
|
|
5431
|
-
|
|
5432
|
-
|
|
5433
|
-
|
|
5434
|
-
|
|
5435
|
-
|
|
5436
|
-
|
|
5437
|
-
|
|
5438
|
-
|
|
5439
|
-
|
|
5440
|
-
|
|
5441
|
-
|
|
5442
|
-
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5448
|
-
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
5454
|
-
|
|
5455
|
-
|
|
5456
|
-
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
|
|
5460
|
-
/* A SIBLING ALREADY KNOWS ITS OWN, and asks nothing. `chosen_codebase` comes
|
|
5461
|
-
down on the claim itself (this slice's migration), so a fanned-out run
|
|
5462
|
-
needs no ask to read and cannot fan out again — which is what stops the
|
|
5463
|
-
fan-out being infinite. Only a request that was ASKED reads the answer. */
|
|
5464
|
-
const chosen = blocker
|
|
5465
|
-
? []
|
|
5466
|
-
: claim.chosenCodebase
|
|
5467
|
-
? [claim.chosenCodebase]
|
|
5468
|
-
: await readChosenCodebases(client, claim.id, warn);
|
|
5469
|
-
const runClaim = chosen.length > 0
|
|
5470
|
-
? { ...claim, chosenCodebase: chosen[0] }
|
|
5471
|
-
: claim;
|
|
5472
|
-
const context = blocker ? null : await resolveContext(runClaim);
|
|
5473
|
-
const collision = context
|
|
5474
|
-
? await collisionForTodo(client, claim.id, context.gitRemoteUrl, warn)
|
|
5475
|
-
: null;
|
|
5476
|
-
if (!blocker && !collision) {
|
|
5477
|
-
/* NO LONGER WAITING, so the next time it does wait it is reported again.
|
|
5478
|
-
Without this, a unit that blocked, ran, and blocked again on something
|
|
5479
|
-
else would stay silent forever. */
|
|
5480
|
-
state.reportedWaiting.delete(claim.id);
|
|
5481
|
-
/* THE RESOLVED CONTEXT IS HANDED ON rather than resolved again. It reads
|
|
5482
|
-
several tables and touches the filesystem, and the dispatch needs the
|
|
5483
|
-
very same answer — resolving twice would both cost a second pass and
|
|
5484
|
-
allow the collision check and the working directory to disagree. */
|
|
5485
|
-
/* THE SIBLINGS ARE CREATED BEFORE THIS RUN DISPATCHES, so a user who chose
|
|
5486
|
-
three codebases sees three cards immediately rather than one that
|
|
5487
|
-
mysteriously multiplies later. Best-effort: a failed sibling costs that
|
|
5488
|
-
codebase its run, and is logged; it must not cost the user the run that
|
|
5489
|
-
is about to start. */
|
|
5490
|
-
if (chosen.length > 1) {
|
|
5491
|
-
await spawnSiblingRuns(client, claim, chosen.slice(1), log, warn);
|
|
5424
|
+
state.warnedNoTools = false;
|
|
5425
|
+
/* ═══ !Cleanup PHASE 7 SLICE 2: CLAIM PAST THE BLOCKED ONES. ═══
|
|
5426
|
+
*
|
|
5427
|
+
A dependent unit must WAIT, and everything behind it must NOT. Those two
|
|
5428
|
+
requirements together are why this is a loop rather than a single claim.
|
|
5429
|
+
*
|
|
5430
|
+
The claim is this design's only queue position, and it takes the OLDEST
|
|
5431
|
+
claimable row. So a blocked unit that is merely released is re-claimed FIRST
|
|
5432
|
+
on the very next tick, forever — it stops being a waiting unit and becomes a
|
|
5433
|
+
queue stopper. Releasing it AND telling the next claim to skip it is what
|
|
5434
|
+
makes the rest of the queue reachable.
|
|
5435
|
+
*
|
|
5436
|
+
BOUNDED BY THE NUMBER OF UNITS SKIPPED, not by a fixed count: each iteration
|
|
5437
|
+
either dispatches (returns), finds nothing (returns), or adds exactly one id
|
|
5438
|
+
to `skip`, and a skipped id can never be claimed again this tick. So the
|
|
5439
|
+
loop runs at most once per blocked unit and then ends. No timeout needed,
|
|
5440
|
+
because it cannot revisit a row.
|
|
5441
|
+
*
|
|
5442
|
+
THE UNIT STAYS `working` AND UNCLAIMED, deliberately — it is NOT parked to
|
|
5443
|
+
`needs-input` the way the scope gate parks. `needs-input` means "this needs
|
|
5444
|
+
YOU", and a dependency needs nothing from the user: it resolves when the
|
|
5445
|
+
blocker finishes. Parking it would put a card under "Needs you" offering no
|
|
5446
|
+
action, which is precisely the defect Phase 1 existed to remove. The panel
|
|
5447
|
+
reads the same `blocked_tasks` view and renders "Waiting on <name>" from it,
|
|
5448
|
+
so the card explains itself without a state that lies. */
|
|
5449
|
+
const skip = [];
|
|
5450
|
+
let blockedCount = 0;
|
|
5451
|
+
for (;;) {
|
|
5452
|
+
let claim;
|
|
5453
|
+
try {
|
|
5454
|
+
claim = await claimNextTodo(client, machineId, role.agent, [...skip, ...legacyHeldTodoIds()], !!recovered);
|
|
5455
|
+
if (claim?.claimOperationId)
|
|
5456
|
+
claimOperations.push(claim.claimOperationId);
|
|
5492
5457
|
}
|
|
5493
|
-
|
|
5494
|
-
|
|
5495
|
-
|
|
5496
|
-
|
|
5497
|
-
|
|
5498
|
-
|
|
5499
|
-
|
|
5500
|
-
|
|
5501
|
-
|
|
5502
|
-
|
|
5503
|
-
|
|
5504
|
-
|
|
5505
|
-
|
|
5506
|
-
|
|
5507
|
-
|
|
5508
|
-
|
|
5458
|
+
catch (err) {
|
|
5459
|
+
warn(`orchestrator poll failed, will retry: ${err.message}`);
|
|
5460
|
+
return 'failed';
|
|
5461
|
+
}
|
|
5462
|
+
if (!claim) {
|
|
5463
|
+
/* Nothing claimable. If we skipped something, the queue is not empty —
|
|
5464
|
+
it is WAITING, which is a different fact and must not be logged as
|
|
5465
|
+
idleness. */
|
|
5466
|
+
return blockedCount > 0 ? 'blocked' : 'nothing-to-do';
|
|
5467
|
+
}
|
|
5468
|
+
/* ═══ THE TWO REASONS WORK WAITS, ASKED IN ORDER. ═══
|
|
5469
|
+
*
|
|
5470
|
+
A DEPENDENCY IS ASKED FIRST, and the order is meaningful rather than
|
|
5471
|
+
incidental: a dependency is a FACT about what must happen before what,
|
|
5472
|
+
while a collision is a CHOICE to serialise safely. When both are true the
|
|
5473
|
+
user is better told the one that will not change on its own — a collision
|
|
5474
|
+
clears the moment the other agent releases, a dependency does not clear
|
|
5475
|
+
until the blocker is finished.
|
|
5476
|
+
*
|
|
5477
|
+
Slice 3 adds the second. Both hold the SAME WAY (release, skip, report
|
|
5478
|
+
once), so the shared tail below is written once rather than twice. */
|
|
5479
|
+
const blocker = await blockerForTodo(client, claim.id, warn);
|
|
5480
|
+
/* THE CODEBASE COMES FROM `resolveContext`, the SAME resolver the dispatch
|
|
5481
|
+
uses to choose the worker's working directory — not a second lookup that
|
|
5482
|
+
could disagree with it. A reservation is scoped by `git_remote_url`, so
|
|
5483
|
+
"which repo would this run touch" and "which repo will it start in" have
|
|
5484
|
+
to be one answer. Only reached when there is no dependency, so an ordinary
|
|
5485
|
+
dispatchable request costs one extra read, not two. */
|
|
5486
|
+
/* ═══ 18b SLICE 4 — THE CODEBASE THE USER CHOSE, AND THE FAN-OUT. ═══
|
|
5487
|
+
*
|
|
5488
|
+
Read BEFORE the context is resolved, because it is an input to it: given a
|
|
5489
|
+
choice, the resolver treats the project as if that one codebase were its
|
|
5490
|
+
only one, and the ambiguity never arises.
|
|
5491
|
+
*
|
|
5492
|
+
MORE THAN ONE CHOSEN MEANS MORE THAN ONE RUN (user ruling, 2026-08-07).
|
|
5493
|
+
This tick dispatches the FIRST; `spawnSiblingRuns` below creates a sibling
|
|
5494
|
+
request for each of the rest, which the daemon then claims on its own
|
|
5495
|
+
ticks exactly like any other request. Each run carries ONE codebase and
|
|
5496
|
+
starts in that codebase's own checkout, wherever it is on disk — nothing
|
|
5497
|
+
here reads folder layout, which is the whole of the location ruling.
|
|
5498
|
+
*
|
|
5499
|
+
Nothing changes for a request that was never asked: `chosen` is empty, the
|
|
5500
|
+
claim carries null, and every path below behaves as it did in Slice 2. */
|
|
5501
|
+
/* A SIBLING ALREADY KNOWS ITS OWN, and asks nothing. `chosen_codebase` comes
|
|
5502
|
+
down on the claim itself (this slice's migration), so a fanned-out run
|
|
5503
|
+
needs no ask to read and cannot fan out again — which is what stops the
|
|
5504
|
+
fan-out being infinite. Only a request that was ASKED reads the answer. */
|
|
5505
|
+
const chosen = blocker
|
|
5506
|
+
? []
|
|
5507
|
+
: claim.chosenCodebase
|
|
5508
|
+
? [claim.chosenCodebase]
|
|
5509
|
+
: await readChosenCodebases(client, claim.id, warn);
|
|
5510
|
+
const runClaim = chosen.length > 0
|
|
5511
|
+
? { ...claim, chosenCodebase: chosen[0] }
|
|
5512
|
+
: claim;
|
|
5513
|
+
const context = blocker ? null : await resolveContext(runClaim);
|
|
5514
|
+
const collision = context
|
|
5515
|
+
? await collisionForTodo(client, claim.id, context.gitRemoteUrl, warn)
|
|
5516
|
+
: null;
|
|
5517
|
+
if (!blocker && !collision) {
|
|
5518
|
+
/* NO LONGER WAITING, so the next time it does wait it is reported again.
|
|
5519
|
+
Without this, a unit that blocked, ran, and blocked again on something
|
|
5520
|
+
else would stay silent forever. */
|
|
5521
|
+
state.reportedWaiting.delete(claim.id);
|
|
5522
|
+
/* THE RESOLVED CONTEXT IS HANDED ON rather than resolved again. It reads
|
|
5523
|
+
several tables and touches the filesystem, and the dispatch needs the
|
|
5524
|
+
very same answer — resolving twice would both cost a second pass and
|
|
5525
|
+
allow the collision check and the working directory to disagree. */
|
|
5526
|
+
/* THE SIBLINGS ARE CREATED BEFORE THIS RUN DISPATCHES, so a user who chose
|
|
5527
|
+
three codebases sees three cards immediately rather than one that
|
|
5528
|
+
mysteriously multiplies later. Best-effort: a failed sibling costs that
|
|
5529
|
+
codebase its run, and is logged; it must not cost the user the run that
|
|
5530
|
+
is about to start. */
|
|
5531
|
+
if (chosen.length > 1) {
|
|
5532
|
+
await spawnSiblingRuns(client, claim, chosen.slice(1), log, warn);
|
|
5533
|
+
}
|
|
5534
|
+
const dispatched = dispatchClaim(deps, state, now, log, warn, role, runClaim, context, chosen);
|
|
5535
|
+
if (hasOwnedWorkContext())
|
|
5536
|
+
finishRpcClaims(claimOperations, true);
|
|
5537
|
+
endClaim();
|
|
5538
|
+
return await dispatched;
|
|
5539
|
+
}
|
|
5540
|
+
/* SAID ONCE PER (unit, REASON) — see `reportedWaiting`. The tick runs every
|
|
5541
|
+
3s and this branch is reached on every one of them for as long as the wait
|
|
5542
|
+
holds. The key carries the reason, not just the unit, so a request that
|
|
5543
|
+
stops waiting on a dependency and starts waiting on a FILE says so. */
|
|
5544
|
+
const waitKey = blocker
|
|
5545
|
+
? `dep:${blocker.dependsOnId}`
|
|
5546
|
+
: `path:${collision.heldBySessionId}:${collision.path}`;
|
|
5547
|
+
const waitLine = blocker
|
|
5548
|
+
? `is waiting on "${blocker.dependsOnName}"`
|
|
5549
|
+
: `is waiting on the same files as "${collision.heldByInstruction ?? 'another run'}"`
|
|
5550
|
+
+ ` (${collision.path}${collision.pathCount > 1 ? ` and ${collision.pathCount - 1} more` : ''})`;
|
|
5551
|
+
if (state.reportedWaiting.get(claim.id) !== waitKey) {
|
|
5552
|
+
state.reportedWaiting.set(claim.id, waitKey);
|
|
5553
|
+
log(`[orchestrator] "${claim.instruction}" (${shortId(claim.id)}) ${waitLine} — not dispatching`);
|
|
5554
|
+
}
|
|
5555
|
+
/* RELEASED, so that the moment the blocker is done the ordinary claim picks
|
|
5556
|
+
it up with no user action and no special path. That is the Gherkin line
|
|
5557
|
+
"123.1 starts without my intervention", and it is why this must not keep
|
|
5558
|
+
the claim the way `parkTodoForScope` does. */
|
|
5559
|
+
await releaseTodo(client, claim.id, claim.claimedAt).catch((err) => {
|
|
5560
|
+
warn(`could not release a blocked todo: ${err.message}`);
|
|
5561
|
+
});
|
|
5562
|
+
skip.push(claim.id);
|
|
5563
|
+
blockedCount += 1;
|
|
5509
5564
|
}
|
|
5510
|
-
|
|
5511
|
-
|
|
5512
|
-
|
|
5513
|
-
|
|
5514
|
-
|
|
5515
|
-
warn(`could not release a blocked todo: ${err.message}`);
|
|
5516
|
-
});
|
|
5517
|
-
skip.push(claim.id);
|
|
5518
|
-
blockedCount += 1;
|
|
5565
|
+
}
|
|
5566
|
+
finally {
|
|
5567
|
+
if (hasOwnedWorkContext())
|
|
5568
|
+
finishRpcClaims(claimOperations, true);
|
|
5569
|
+
endClaim();
|
|
5519
5570
|
}
|
|
5520
5571
|
}
|
|
5521
5572
|
/**
|
|
@@ -5549,10 +5600,14 @@ async function runTick(deps, state, now, log, warn) {
|
|
|
5549
5600
|
* returns is not a funnel.
|
|
5550
5601
|
*/
|
|
5551
5602
|
async function dispatchClaim(deps, state, now, log, warn, role, claim, context, chosenCodebases = []) {
|
|
5603
|
+
const owned = hasOwnedWorkContext() ? reserveOwnedWork({ surface: 'worker', todoId: claim.id, workerId: null, claimedAt: claim.claimedAt, observedPendingMessageIds: null }, role.agent) : undefined;
|
|
5552
5604
|
try {
|
|
5553
|
-
|
|
5605
|
+
const outcome = await dispatchClaimInner(deps, state, now, log, warn, role, claim, context, chosenCodebases, owned);
|
|
5606
|
+
owned?.complete();
|
|
5607
|
+
return outcome;
|
|
5554
5608
|
}
|
|
5555
5609
|
finally {
|
|
5610
|
+
owned?.finishPreparation();
|
|
5556
5611
|
/* THE RUN IS OVER, SO IT HOLDS NOTHING, however it ended and wherever it
|
|
5557
5612
|
returned from. Best-effort by contract: `releaseTodoReservations` never
|
|
5558
5613
|
throws, so this cannot change what the body returned, and a `finally`
|
|
@@ -5567,6 +5622,57 @@ async function dispatchClaim(deps, state, now, log, warn, role, claim, context,
|
|
|
5567
5622
|
forgetSecrets(claim.id);
|
|
5568
5623
|
}
|
|
5569
5624
|
}
|
|
5625
|
+
async function settleWorkerOutcome(client, workerId, claim, role, result, targeted, now, startedAt, log, warn, saved, persist, onAccepted) {
|
|
5626
|
+
let disposition = saved?.disposition;
|
|
5627
|
+
if (!disposition) {
|
|
5628
|
+
if (!result.ok) {
|
|
5629
|
+
const kind = result.failureKind ?? nativeFailureKind(result.saidTail ?? '');
|
|
5630
|
+
const reason = failureMessage(role.agent, kind);
|
|
5631
|
+
disposition = { worker: 'died', reason, todo: kind === 'transient' ? 'retry' : 'failed' };
|
|
5632
|
+
}
|
|
5633
|
+
else if (await todoHasOpenAsk(client, claim.id, warn, true)) {
|
|
5634
|
+
disposition = { worker: 'done', reason: '', todo: 'needs-input' };
|
|
5635
|
+
}
|
|
5636
|
+
else if (!await todoRunLeftATrace(client, claim.id, warn, targeted, true)) {
|
|
5637
|
+
disposition = { worker: 'died', reason: NOTHING_RECORDED_REASON, todo: 'retry' };
|
|
5638
|
+
}
|
|
5639
|
+
else
|
|
5640
|
+
disposition = { worker: 'done', reason: '', todo: 'done' };
|
|
5641
|
+
if (saved) {
|
|
5642
|
+
saved.disposition = disposition;
|
|
5643
|
+
persist?.(saved);
|
|
5644
|
+
}
|
|
5645
|
+
}
|
|
5646
|
+
if (disposition.todo === 'needs-input') {
|
|
5647
|
+
const accepted = await suspendTodoForAsk(client, claim.id, result.output, claim.claimedAt);
|
|
5648
|
+
if (accepted)
|
|
5649
|
+
onAccepted?.();
|
|
5650
|
+
await updateWorker(client, workerId, 'done', null, warn, true);
|
|
5651
|
+
log(`[orchestrator] ${shortId(claim.id)} is waiting on the user`);
|
|
5652
|
+
return 'answered';
|
|
5653
|
+
}
|
|
5654
|
+
if (disposition.todo === 'done') {
|
|
5655
|
+
const accepted = await answerTodo(client, claim.id, result.output, claim.claimedAt);
|
|
5656
|
+
if (accepted)
|
|
5657
|
+
onAccepted?.();
|
|
5658
|
+
await updateWorker(client, workerId, 'done', null, warn, true);
|
|
5659
|
+
log(`[orchestrator] answered ${shortId(claim.id)} in ${((now().getTime() - startedAt) / 1000).toFixed(1)}s`);
|
|
5660
|
+
return 'answered';
|
|
5661
|
+
}
|
|
5662
|
+
await updateWorker(client, workerId, 'died', disposition.reason, warn, true);
|
|
5663
|
+
const { fate, attempts } = disposition.todo === 'retry'
|
|
5664
|
+
? await releaseOrFail(client, claim.id, disposition.reason, now().getTime(), claim.claimedAt)
|
|
5665
|
+
: await failClaimedTodo(client, claim.id, disposition.reason, claim.claimedAt ?? null);
|
|
5666
|
+
if (fate === 'stale')
|
|
5667
|
+
return 'nothing-to-do';
|
|
5668
|
+
onAccepted?.();
|
|
5669
|
+
if (fate === 'failed') {
|
|
5670
|
+
warn(`[orchestrator] ${shortId(claim.id)} failed: ${disposition.reason}`);
|
|
5671
|
+
return 'failed';
|
|
5672
|
+
}
|
|
5673
|
+
warn(`[orchestrator] retrying ${shortId(claim.id)} in ${Math.round(retryDelayMs(attempts) / 1000)}s`);
|
|
5674
|
+
return 'released';
|
|
5675
|
+
}
|
|
5570
5676
|
async function dispatchClaimInner(deps, state, now, log, warn, role, claim,
|
|
5571
5677
|
/* !Cleanup Phase 7 Slice 3 — RESOLVED BY THE CALLER, and passed in rather than
|
|
5572
5678
|
resolved again here. `runTick` needs the same answer first, to know which
|
|
@@ -5577,7 +5683,7 @@ context,
|
|
|
5577
5683
|
/** 18b Slice 4 — every codebase the user chose, including this run's own. The
|
|
5578
5684
|
* siblings are derived from it below; empty on every request that was never
|
|
5579
5685
|
* asked, which is the ordinary case. */
|
|
5580
|
-
chosenCodebases = []) {
|
|
5686
|
+
chosenCodebases = [], owned) {
|
|
5581
5687
|
const { client, machineId, mcpServer = null, spawn = defaultSpawn(mcpServer), } = deps;
|
|
5582
5688
|
/* ═══ 16g SLICE 1 — THE SECOND ENFORCEMENT POINT. ═══
|
|
5583
5689
|
`No approved scope-version pointer → no execution write and no coding lane.`
|
|
@@ -5638,7 +5744,7 @@ chosenCodebases = []) {
|
|
|
5638
5744
|
uses the mirror of this: answering an ask CLEARS the claim to make the
|
|
5639
5745
|
work dispatchable again — so approving scope re-dispatches through the
|
|
5640
5746
|
path that already exists, with no new mechanism. */
|
|
5641
|
-
await parkTodoForScope(client, claim.id).catch((err) => {
|
|
5747
|
+
await parkTodoForScope(client, claim.id, claim.claimedAt).catch((err) => {
|
|
5642
5748
|
warn(`could not park an unapproved todo: ${err.message}`);
|
|
5643
5749
|
});
|
|
5644
5750
|
return 'nothing-to-do';
|
|
@@ -5665,7 +5771,7 @@ chosenCodebases = []) {
|
|
|
5665
5771
|
hold a dead end too. Building stays held exactly as shipped. */
|
|
5666
5772
|
if (targeted && building && await todoIsHeldForScope(client, claim.id)) {
|
|
5667
5773
|
log(`[orchestrator] "${claim.instruction}" has an unresolved scope change — not dispatching`);
|
|
5668
|
-
await parkTodoForScope(client, claim.id).catch((err) => {
|
|
5774
|
+
await parkTodoForScope(client, claim.id, claim.claimedAt).catch((err) => {
|
|
5669
5775
|
warn(`could not park a held todo: ${err.message}`);
|
|
5670
5776
|
});
|
|
5671
5777
|
return 'nothing-to-do';
|
|
@@ -5682,6 +5788,7 @@ chosenCodebases = []) {
|
|
|
5682
5788
|
let workerId = null;
|
|
5683
5789
|
/* !Cleanup Phase 6b (I43) — outside the try for the same reason: the `finally`
|
|
5684
5790
|
has to remove it from `livePids` however this run ended. */
|
|
5791
|
+
let terminalDeferred = false;
|
|
5685
5792
|
let spawnedPid = null;
|
|
5686
5793
|
try {
|
|
5687
5794
|
log(`[orchestrator] picked up "${claim.instruction}" (${shortId(claim.id)})`);
|
|
@@ -5801,7 +5908,7 @@ chosenCodebases = []) {
|
|
|
5801
5908
|
with the words it had in Slice 2: never parked with no question on it,
|
|
5802
5909
|
which the user could only clear with Stop. */
|
|
5803
5910
|
if (context.miss?.kind === 'ambiguous-codebase' && instructionNeedsCheckout(claim.instruction)) {
|
|
5804
|
-
const asked = await askWhichCodebase(client, claim.id, context.miss.projectName, context.miss.gitRemoteUrls, warn);
|
|
5911
|
+
const asked = await askWhichCodebase(client, claim.id, context.miss.projectName, context.miss.gitRemoteUrls, warn, claim.claimedAt);
|
|
5805
5912
|
if (asked) {
|
|
5806
5913
|
log(`[orchestrator] asking which codebase "${claim.instruction}" should run in`);
|
|
5807
5914
|
await updateWorker(client, workerId, 'died', 'waiting for the user to choose a codebase', warn);
|
|
@@ -5811,7 +5918,7 @@ chosenCodebases = []) {
|
|
|
5811
5918
|
if (context.miss && instructionNeedsCheckout(claim.instruction)) {
|
|
5812
5919
|
log(`[orchestrator] "${claim.instruction}" needs the code, which is not on this machine`);
|
|
5813
5920
|
await updateWorker(client, workerId, 'died', 'the codebase is not available on this machine', warn);
|
|
5814
|
-
await answerTodo(client, claim.id, checkoutMissAnswer(context.miss));
|
|
5921
|
+
await answerTodo(client, claim.id, checkoutMissAnswer(context.miss), claim.claimedAt);
|
|
5815
5922
|
return 'answered';
|
|
5816
5923
|
}
|
|
5817
5924
|
/* ═══ 18d SLICE 4 — ONE STAGE PER DISPATCH (user ruling, 2026-08-12). ═══
|
|
@@ -5823,6 +5930,15 @@ chosenCodebases = []) {
|
|
|
5823
5930
|
if (stage) {
|
|
5824
5931
|
log(`[orchestrator] ${shortId(claim.id)} is working stage ${stage.position}/${stage.total}: ${stage.title}`);
|
|
5825
5932
|
}
|
|
5933
|
+
if (owned) {
|
|
5934
|
+
if (!claim.claimedAt || !workerId)
|
|
5935
|
+
throw new Error('The worker has no exact attempt identity.');
|
|
5936
|
+
const pending = await client.from('cliv2_run_messages').select('id').eq('todo_id', claim.id).is('answered_by', null);
|
|
5937
|
+
if (pending.error)
|
|
5938
|
+
throw pending.error;
|
|
5939
|
+
owned.setReference({ surface: 'worker', todoId: claim.id, workerId, claimedAt: claim.claimedAt,
|
|
5940
|
+
observedPendingMessageIds: (pending.data ?? []).map((row) => row.id) });
|
|
5941
|
+
}
|
|
5826
5942
|
const result = await spawn(role.agent, buildPrompt(claim.instruction, claim.id, answered, targeted && scopeVersion ? { workItemId: targeted, version: scopeVersion } : null, replies, context, planningOnly, interrupted, selectedTools, stage, directed), (step) => steps.offer(step), context.checkout,
|
|
5827
5943
|
// !Cleanup Phase 6 (I17) — polled by the child; true kills it.
|
|
5828
5944
|
() => todoWasStopped(client, claim.id),
|
|
@@ -5850,118 +5966,44 @@ chosenCodebases = []) {
|
|
|
5850
5966
|
not the prompt, so report_activity attributes without depending on
|
|
5851
5967
|
agent cooperation (ux.md § Corrections: codex dropped a prompt-carried
|
|
5852
5968
|
id twice). */
|
|
5853
|
-
claim.id);
|
|
5969
|
+
claim.id, owned);
|
|
5854
5970
|
/* !Cleanup PHASE 6 (I17) — THE USER STOPPED IT, and that is not a failure.
|
|
5855
5971
|
Checked BEFORE `!result.ok`, because a killed child also reports not-ok:
|
|
5856
5972
|
falling through would count the user's own decision as an attempt and
|
|
5857
5973
|
retry the very thing they cancelled. Nothing is written but the claim
|
|
5858
5974
|
release — the browser already set `stopped`. */
|
|
5975
|
+
if (result.interrupted)
|
|
5976
|
+
return 'nothing-to-do';
|
|
5859
5977
|
if (result.stopped) {
|
|
5860
|
-
|
|
5861
|
-
|
|
5978
|
+
terminalDeferred = !!owned;
|
|
5979
|
+
owned?.deferOutcome({ surface: 'worker', ok: false, text: null, reason: 'stopped by the user', stopped: true });
|
|
5980
|
+
await updateWorker(client, workerId, 'died', 'stopped by the user', warn, true);
|
|
5981
|
+
await releaseStoppedTodo(client, claim.id, claim.claimedAt);
|
|
5982
|
+
owned?.acknowledgeOutcome();
|
|
5983
|
+
terminalDeferred = false;
|
|
5862
5984
|
log(`[orchestrator] ${shortId(claim.id)} was stopped by the user`);
|
|
5863
5985
|
return 'released';
|
|
5864
5986
|
}
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
5869
|
-
|
|
5870
|
-
|
|
5871
|
-
|
|
5872
|
-
|
|
5873
|
-
|
|
5874
|
-
|
|
5875
|
-
|
|
5876
|
-
|
|
5877
|
-
|
|
5878
|
-
|
|
5879
|
-
re-claimed forever while reading as Working. */
|
|
5880
|
-
const { fate, attempts } = await releaseOrFail(client, claim.id, plain, now().getTime(), claim.claimedAt);
|
|
5881
|
-
if (fate === 'failed') {
|
|
5882
|
-
warn(`[orchestrator] ${result.error} — ${shortId(claim.id)} failed after ${MAX_ATTEMPTS} attempts`);
|
|
5883
|
-
return 'failed';
|
|
5884
|
-
}
|
|
5885
|
-
/* 18a SLICE 6: THIS REQUEST WAITS, AND NOTHING ELSE DOES. The wait went
|
|
5886
|
-
onto the row with the attempt count; what is left here is saying so, in
|
|
5887
|
-
the line that used to say only "releasing". */
|
|
5888
|
-
const seconds = Math.round(retryDelayMs(attempts) / 1000);
|
|
5889
|
-
warn(`[orchestrator] ${result.error}: retrying ${shortId(claim.id)} in ${seconds}s`);
|
|
5890
|
-
return 'released';
|
|
5891
|
-
}
|
|
5892
|
-
/* !Cleanup PHASE 6 (I15) — A QUESTION IS NOT A FINISHED JOB.
|
|
5893
|
-
The child exited cleanly, which until now meant `done` unconditionally.
|
|
5894
|
-
But a one-shot headless run ASKS AND EXITS — that is the only way it can
|
|
5895
|
-
handle a question — so the clean exit says nothing about whether the work
|
|
5896
|
-
is over. If an ask is open on this unit, the truthful state is that it
|
|
5897
|
-
needs the user, and the Inbox already has the question. */
|
|
5898
|
-
if (await todoHasOpenAsk(client, claim.id, warn)) {
|
|
5899
|
-
await suspendTodoForAsk(client, claim.id, result.output, claim.claimedAt);
|
|
5900
|
-
await updateWorker(client, workerId, 'done', null, warn);
|
|
5901
|
-
log(`[orchestrator] ${shortId(claim.id)} is waiting on the user`);
|
|
5902
|
-
return 'answered';
|
|
5903
|
-
}
|
|
5904
|
-
/* A CONFIDENT ANSWER WITH NOTHING BEHIND IT IS NOT A FINISHED JOB.
|
|
5905
|
-
See `todoRunLeftATrace` for the measurement: a codex worker that was
|
|
5906
|
-
correctly configured, connected, and fetched the tool list still reported
|
|
5907
|
-
that it had no tools, twice in three identical runs, and one reported
|
|
5908
|
-
ATTACHING a screenshot it never attached. Every one of them exited 0 with
|
|
5909
|
-
prose on stdout, which is all the two checks above look at, so every one
|
|
5910
|
-
of them landed here and was written `done`.
|
|
5911
|
-
|
|
5912
|
-
PLACED AFTER THE ASK CHECK, and that order is load-bearing. A run that
|
|
5913
|
-
asked a question and exited has legitimately recorded something and is
|
|
5914
|
-
already handled as `needs-input`; reaching this check first would relabel
|
|
5915
|
-
the product's most important state as a failure.
|
|
5916
|
-
|
|
5917
|
-
RELEASED FOR RETRY RATHER THAN FAILED OUTRIGHT, through the SAME
|
|
5918
|
-
`releaseOrFail` the dead-worker path uses. This is intermittent by nature
|
|
5919
|
-
(the third run of the identical prompt worked perfectly), so a retry is
|
|
5920
|
-
very likely to succeed, and the attempt bound is what stops it looping
|
|
5921
|
-
forever. At the bound the card says why, in words about what is KNOWN
|
|
5922
|
-
(nothing was recorded) rather than repeating the agent's own account of a
|
|
5923
|
-
run that has just been shown to be an unreliable narrator.
|
|
5924
|
-
|
|
5925
|
-
`targeted` IS PASSED, and without it this check is blind to the primary
|
|
5926
|
-
flow. A session opened on a work item has `todo_id` NULL by the table's
|
|
5927
|
-
own one-anchor constraint, and the prompt tells the agent to open it that
|
|
5928
|
-
way whenever there is an item, so a `todo_id`-only lookup could never
|
|
5929
|
-
match a targeted run. See `todoRunLeftATrace`. It is the same value the
|
|
5930
|
-
scope gate above already resolved, so the two cannot disagree about which
|
|
5931
|
-
item this request is for. */
|
|
5932
|
-
if (!await todoRunLeftATrace(client, claim.id, warn, targeted)) {
|
|
5933
|
-
/* THE WORKER DIED WITH NO STEP, and that blank is load-bearing rather
|
|
5934
|
-
than incidental. `interruptedRunStep` looks up the most recent `died`
|
|
5935
|
-
worker on this request and `describeInterruptedRun` then tells the
|
|
5936
|
-
REPLACEMENT agent that a previous attempt "stopped partway" and that
|
|
5937
|
-
anything it did "IS STILL THERE". For a genuinely toolless run that is
|
|
5938
|
-
true and useful. Clearing the step keeps the claim honest about the one
|
|
5939
|
-
thing that is known: this run recorded nothing, so there is no last
|
|
5940
|
-
action to hand on and nothing for the next agent to resume from. */
|
|
5941
|
-
await updateWorker(client, workerId, 'died', NOTHING_RECORDED_REASON, warn);
|
|
5942
|
-
const { fate, attempts } = await releaseOrFail(client, claim.id, NOTHING_RECORDED_REASON, now().getTime(), claim.claimedAt);
|
|
5943
|
-
if (fate === 'failed') {
|
|
5944
|
-
warn(`[orchestrator] ${shortId(claim.id)} recorded nothing and failed after ${MAX_ATTEMPTS} attempts`);
|
|
5945
|
-
return 'failed';
|
|
5946
|
-
}
|
|
5947
|
-
const seconds = Math.round(retryDelayMs(attempts) / 1000);
|
|
5948
|
-
warn(`[orchestrator] ${shortId(claim.id)} recorded nothing: retrying in ${seconds}s`);
|
|
5949
|
-
return 'released';
|
|
5950
|
-
}
|
|
5951
|
-
/* FENCED ON THE CLAIM THIS DISPATCH TOOK (18d Slice 4). If this run handed
|
|
5952
|
-
its stage on, the claim is already cleared and a fresh worker is on the
|
|
5953
|
-
row, so this write must do nothing rather than mark the request finished
|
|
5954
|
-
over work that is still running. See answerTodo's own comment. */
|
|
5955
|
-
await answerTodo(client, claim.id, result.output, claim.claimedAt);
|
|
5956
|
-
// The worker finished and its answer is committed. Marked done AFTER the
|
|
5957
|
-
// answer write, so a worker is never reported finished for a unit that has
|
|
5958
|
-
// no answer on it.
|
|
5959
|
-
await updateWorker(client, workerId, 'done', null, warn);
|
|
5960
|
-
const seconds = ((now().getTime() - startedAt) / 1000).toFixed(1);
|
|
5961
|
-
log(`[orchestrator] answered ${shortId(claim.id)} in ${seconds}s`);
|
|
5962
|
-
return 'answered';
|
|
5987
|
+
terminalDeferred = !!owned;
|
|
5988
|
+
const terminal = { surface: 'worker', ok: result.ok, text: result.ok ? result.output : null,
|
|
5989
|
+
reason: result.ok ? '' : failureMessage(role.agent, result.failureKind ?? nativeFailureKind(result.saidTail ?? '')),
|
|
5990
|
+
failureKind: result.failureKind ?? nativeFailureKind(result.saidTail ?? ''), workItemId: targeted };
|
|
5991
|
+
owned?.deferOutcome(terminal);
|
|
5992
|
+
const settled = await settleWorkerOutcome(client, workerId, claim, role, result, targeted, now, startedAt, log, warn, terminal, owned?.deferOutcome, () => {
|
|
5993
|
+
if (result.ok)
|
|
5994
|
+
deps.observeHarness?.(role.agent, 'authenticated', 'dispatch-success');
|
|
5995
|
+
else if (result.failureKind === 'authentication')
|
|
5996
|
+
deps.observeHarness?.(role.agent, 'sign-in-required', 'provider-rejected');
|
|
5997
|
+
});
|
|
5998
|
+
owned?.acknowledgeOutcome();
|
|
5999
|
+
terminalDeferred = false;
|
|
6000
|
+
return settled;
|
|
5963
6001
|
}
|
|
5964
6002
|
catch (err) {
|
|
6003
|
+
if (terminalDeferred) {
|
|
6004
|
+
warn('The completed execution is waiting for its original account to acknowledge the result.');
|
|
6005
|
+
return 'failed';
|
|
6006
|
+
}
|
|
5965
6007
|
// The answer write itself failed (network, RLS, a reverted migration). The
|
|
5966
6008
|
// todo is CLAIMED and has no answer — the worst state — so release it here
|
|
5967
6009
|
// too. A release that also fails is warned and the row is left claimed;
|
|
@@ -5979,6 +6021,8 @@ chosenCodebases = []) {
|
|
|
5979
6021
|
repeatedly by a durable cause — a reverted migration, an RLS change —
|
|
5980
6022
|
and before the bound it was the same forever-loop as a failed spawn. */
|
|
5981
6023
|
const { fate, attempts } = await releaseOrFail(client, claim.id, err.message, now().getTime(), claim.claimedAt);
|
|
6024
|
+
if (fate === 'stale')
|
|
6025
|
+
return 'nothing-to-do';
|
|
5982
6026
|
warn(fate === 'failed'
|
|
5983
6027
|
? `[orchestrator] ${shortId(claim.id)} failed after ${MAX_ATTEMPTS} attempts`
|
|
5984
6028
|
: `[orchestrator] retrying ${shortId(claim.id)} in ${Math.round(retryDelayMs(attempts) / 1000)}s`);
|
|
@@ -6009,3 +6053,276 @@ chosenCodebases = []) {
|
|
|
6009
6053
|
export function liveAgents() {
|
|
6010
6054
|
return detectAgents();
|
|
6011
6055
|
}
|
|
6056
|
+
function cliClaimReferences(action, result) {
|
|
6057
|
+
if (!Array.isArray(result))
|
|
6058
|
+
throw new Error('The claim journal returned no complete result.');
|
|
6059
|
+
return result.map((row) => {
|
|
6060
|
+
if (action === 'cliv2_claim_next_loose_todo') {
|
|
6061
|
+
const attempt = row?._attempt;
|
|
6062
|
+
if (!attempt || typeof attempt.todo_id !== 'string' || typeof attempt.claimed_at !== 'string')
|
|
6063
|
+
throw new Error('The claim journal returned an incomplete request identity.');
|
|
6064
|
+
return { surface: 'worker', todoId: attempt.todo_id, workerId: null, claimedAt: attempt.claimed_at, observedPendingMessageIds: null };
|
|
6065
|
+
}
|
|
6066
|
+
if (typeof row?.id !== 'string' || typeof row?.todo_id !== 'string')
|
|
6067
|
+
throw new Error('The claim journal returned an incomplete reply source.');
|
|
6068
|
+
if (action === 'cliv2_take_run_messages' && typeof row.taken_at === 'string')
|
|
6069
|
+
return {
|
|
6070
|
+
surface: 'reply', todoId: row.todo_id, messageLeases: [{ id: row.id, taken_at: row.taken_at }], grantLeases: [], observedPendingMessageIds: null,
|
|
6071
|
+
};
|
|
6072
|
+
if (action === 'cliv2_take_granted_resumes' && typeof row.resume_taken_at === 'string')
|
|
6073
|
+
return {
|
|
6074
|
+
surface: 'reply', todoId: row.todo_id, messageLeases: [], grantLeases: [{ id: row.id, resume_taken_at: row.resume_taken_at }], observedPendingMessageIds: null,
|
|
6075
|
+
};
|
|
6076
|
+
throw new Error('The claim journal returned an unrecognized lease identity.');
|
|
6077
|
+
});
|
|
6078
|
+
}
|
|
6079
|
+
async function recoveredCliClaim(client, action, args) {
|
|
6080
|
+
const skip = (args.p_skip_ids ?? args.p_skip_todo_ids ?? []);
|
|
6081
|
+
for (const claim of recoveredRpcClaims('cli').filter(row => row.action === action)) {
|
|
6082
|
+
if (action === 'cliv2_claim_next_loose_todo' && claim.args?.p_agent !== args.p_agent)
|
|
6083
|
+
continue;
|
|
6084
|
+
const rows = [];
|
|
6085
|
+
for (const row of claim.result ?? []) {
|
|
6086
|
+
const ref = cliClaimReferences(action, [row])[0];
|
|
6087
|
+
if (ref.surface === 'panel' || !pendingRpcReference(ref))
|
|
6088
|
+
continue;
|
|
6089
|
+
if (skip.includes(ref.todoId) || Array.isArray(args.p_todo_ids) && !args.p_todo_ids.includes(ref.todoId))
|
|
6090
|
+
continue;
|
|
6091
|
+
let current = false;
|
|
6092
|
+
if (ref.surface === 'worker') {
|
|
6093
|
+
const read = await client.from('cliv2_loose_todos').select('state,claimed_at,claimed_machine_id').eq('id', ref.todoId).maybeSingle();
|
|
6094
|
+
if (read.error)
|
|
6095
|
+
throw read.error;
|
|
6096
|
+
current = !!read.data && read.data.claimed_at === ref.claimedAt && read.data.claimed_machine_id === args.p_machine_id && ['working', 'needs-input'].includes(read.data.state);
|
|
6097
|
+
}
|
|
6098
|
+
else if (action === 'cliv2_take_run_messages') {
|
|
6099
|
+
const source = ref.messageLeases[0];
|
|
6100
|
+
const read = await client.from('cliv2_run_messages').select('taken_at,taken_machine_id,answered_by,interrupted_at').eq('id', source.id).maybeSingle();
|
|
6101
|
+
if (read.error)
|
|
6102
|
+
throw read.error;
|
|
6103
|
+
current = !!read.data && read.data.taken_at === source.taken_at && read.data.taken_machine_id === args.p_machine_id && !read.data.answered_by && !read.data.interrupted_at;
|
|
6104
|
+
}
|
|
6105
|
+
else {
|
|
6106
|
+
const source = ref.grantLeases[0];
|
|
6107
|
+
const read = await client.from('cliv2_permission_requests').select('resume_taken_at,resume_machine_id,resume_answered_by,resume_interrupted_at,state').eq('id', source.id).maybeSingle();
|
|
6108
|
+
if (read.error)
|
|
6109
|
+
throw read.error;
|
|
6110
|
+
current = !!read.data && read.data.resume_taken_at === source.resume_taken_at && read.data.resume_machine_id === args.p_machine_id && !read.data.resume_answered_by && !read.data.resume_interrupted_at && read.data.state === 'granted';
|
|
6111
|
+
}
|
|
6112
|
+
if (current)
|
|
6113
|
+
rows.push(row);
|
|
6114
|
+
else
|
|
6115
|
+
retireRpcReference(ref);
|
|
6116
|
+
}
|
|
6117
|
+
if (rows.length)
|
|
6118
|
+
return { data: rows, error: null, operationId: claim.id };
|
|
6119
|
+
finishRpcClaims([claim.id], true);
|
|
6120
|
+
}
|
|
6121
|
+
return null;
|
|
6122
|
+
}
|
|
6123
|
+
async function cliClaim(client, action, args, recoveryOnly = false) {
|
|
6124
|
+
if (!hasOwnedWorkContext())
|
|
6125
|
+
return { ...await client.rpc(action, args), operationId: undefined };
|
|
6126
|
+
const recovered = await recoveredCliClaim(client, action, args);
|
|
6127
|
+
if (recovered)
|
|
6128
|
+
return recovered;
|
|
6129
|
+
if (recoveryOnly)
|
|
6130
|
+
return { data: [], error: null, operationId: undefined };
|
|
6131
|
+
const operationId = beginRpcClaim('cli', action, args);
|
|
6132
|
+
try {
|
|
6133
|
+
const response = await client.rpc(action, { ...args, p_operation_id: operationId });
|
|
6134
|
+
if (!response.error) {
|
|
6135
|
+
const journal = await client.rpc('cliv2_reconcile_claim', { p_machine_id: args.p_machine_id, p_operation_id: operationId });
|
|
6136
|
+
if (journal.error)
|
|
6137
|
+
throw journal.error;
|
|
6138
|
+
const row = journal.data?.[0];
|
|
6139
|
+
if (row?.outcome !== 'completed' || row.action !== action)
|
|
6140
|
+
throw new Error('The claim was cancelled before it could start.');
|
|
6141
|
+
recordRpcClaims(operationId, cliClaimReferences(action, row.result), false, row.result);
|
|
6142
|
+
if (row.result.length === 0)
|
|
6143
|
+
finishRpcClaims([operationId]);
|
|
6144
|
+
return { ...response, data: row.result, operationId: row.result.length ? operationId : undefined };
|
|
6145
|
+
}
|
|
6146
|
+
if (response.error)
|
|
6147
|
+
deferRpcClaim(operationId);
|
|
6148
|
+
return { ...response, operationId };
|
|
6149
|
+
}
|
|
6150
|
+
catch (error) {
|
|
6151
|
+
deferRpcClaim(operationId);
|
|
6152
|
+
throw error;
|
|
6153
|
+
}
|
|
6154
|
+
}
|
|
6155
|
+
/** Read every legacy source before committing a baseline; a failed page keeps admission closed. */
|
|
6156
|
+
async function legacyRows(client, table, columns, todoId) {
|
|
6157
|
+
const rows = [];
|
|
6158
|
+
let after = null;
|
|
6159
|
+
for (;;) {
|
|
6160
|
+
let query = client.from(table).select(columns).order('id').limit(500);
|
|
6161
|
+
if (todoId)
|
|
6162
|
+
query = query.eq('todo_id', todoId);
|
|
6163
|
+
if (after)
|
|
6164
|
+
query = query.gt('id', after);
|
|
6165
|
+
const { data, error } = await query;
|
|
6166
|
+
if (error)
|
|
6167
|
+
throw error;
|
|
6168
|
+
if (!Array.isArray(data))
|
|
6169
|
+
throw new Error('The old work snapshot did not return a complete page.');
|
|
6170
|
+
rows.push(...data);
|
|
6171
|
+
if (data.length < 500)
|
|
6172
|
+
return rows;
|
|
6173
|
+
after = data[data.length - 1].id;
|
|
6174
|
+
if (typeof after !== 'string')
|
|
6175
|
+
throw new Error('The old work snapshot has no continuation identity.');
|
|
6176
|
+
}
|
|
6177
|
+
}
|
|
6178
|
+
export async function reconcileInterruptedWork(client, machineId, observeHarness) {
|
|
6179
|
+
if (!hasOwnedWorkContext())
|
|
6180
|
+
return;
|
|
6181
|
+
const capability = await client.rpc('cliv2_interrupt_machine_work', { p_machine_id: machineId,
|
|
6182
|
+
p_operation_id: randomUUID(), p_interrupted_at: new Date().toISOString(), p_workers: [], p_replies: [] });
|
|
6183
|
+
if (capability.error)
|
|
6184
|
+
throw capability.error;
|
|
6185
|
+
for (const pending of pendingTerminalOutcomes()) {
|
|
6186
|
+
if (pending.reference.surface === 'reply') {
|
|
6187
|
+
await settleReplyOutcome(client, machineId, pending.reference, pending.outcome, () => {
|
|
6188
|
+
const outcome = pending.outcome;
|
|
6189
|
+
if (!outcome.completedAt || !outcome.harnessIdentity || !['claude', 'codex'].includes(pending.harness ?? '') || outcome.stopped)
|
|
6190
|
+
return;
|
|
6191
|
+
if (outcome.ok || outcome.failureKind === 'authentication')
|
|
6192
|
+
observeHarness?.(pending.harness, outcome.ok ? 'authenticated' : 'sign-in-required', outcome.ok ? 'dispatch-success' : 'provider-rejected', outcome.completedAt, outcome.harnessIdentity);
|
|
6193
|
+
});
|
|
6194
|
+
acknowledgeTerminalOutcome(pending.id);
|
|
6195
|
+
continue;
|
|
6196
|
+
}
|
|
6197
|
+
if (pending.reference.surface !== 'worker')
|
|
6198
|
+
continue;
|
|
6199
|
+
const ref = pending.reference;
|
|
6200
|
+
if (pending.outcome.stopped) {
|
|
6201
|
+
await updateWorker(client, ref.workerId, 'died', 'stopped by the user', message => console.warn(message), true);
|
|
6202
|
+
await releaseStoppedTodo(client, ref.todoId, ref.claimedAt);
|
|
6203
|
+
acknowledgeTerminalOutcome(pending.id);
|
|
6204
|
+
continue;
|
|
6205
|
+
}
|
|
6206
|
+
const { data, error } = await client.from('cliv2_loose_todos').select('claimed_at,state').eq('id', ref.todoId).maybeSingle();
|
|
6207
|
+
if (error)
|
|
6208
|
+
throw error;
|
|
6209
|
+
if (data?.claimed_at === ref.claimedAt && ['working', 'needs-input'].includes(data.state)) {
|
|
6210
|
+
await settleWorkerOutcome(client, ref.workerId, { id: ref.todoId, claimedAt: ref.claimedAt }, { agent: pending.harness ?? 'Agent' }, { ok: pending.outcome.ok, output: pending.outcome.text ?? '', error: pending.outcome.reason, failureKind: pending.outcome.failureKind }, pending.outcome.workItemId ?? null, () => new Date(), Date.now(), () => { }, message => console.warn(message), pending.outcome, value => updateTerminalOutcome(pending.id, value));
|
|
6211
|
+
}
|
|
6212
|
+
else {
|
|
6213
|
+
const disposition = pending.outcome.disposition;
|
|
6214
|
+
await updateWorker(client, ref.workerId, disposition?.worker ?? (pending.outcome.ok ? 'done' : 'died'), disposition?.reason || pending.outcome.reason || null, message => console.warn(message), true);
|
|
6215
|
+
}
|
|
6216
|
+
acknowledgeTerminalOutcome(pending.id);
|
|
6217
|
+
}
|
|
6218
|
+
for (const claim of pendingRpcClaims('cli')) {
|
|
6219
|
+
const journal = await client.rpc('cliv2_reconcile_claim', { p_machine_id: machineId, p_operation_id: claim.id });
|
|
6220
|
+
if (journal.error)
|
|
6221
|
+
throw journal.error;
|
|
6222
|
+
const row = journal.data?.[0];
|
|
6223
|
+
if (row?.outcome === 'cancelled')
|
|
6224
|
+
recordRpcClaims(claim.id, [], true);
|
|
6225
|
+
else if (row?.outcome === 'completed' && row.action === claim.action)
|
|
6226
|
+
recordRpcClaims(claim.id, cliClaimReferences(claim.action, row.result), false, row.result);
|
|
6227
|
+
else
|
|
6228
|
+
throw new Error('The pending claim could not be reconciled.');
|
|
6229
|
+
}
|
|
6230
|
+
if (legacySnapshotNeeded()) {
|
|
6231
|
+
const todos = await legacyRows(client, 'cliv2_loose_todos', 'id,claimed_at,claimed_machine_id');
|
|
6232
|
+
const workers = await legacyRows(client, 'cliv2_workers', 'id,todo_id,machine_id,state');
|
|
6233
|
+
const messages = await legacyRows(client, 'cliv2_run_messages', 'id,todo_id,taken_at,taken_machine_id,answered_by');
|
|
6234
|
+
const permissions = await legacyRows(client, 'cliv2_permission_requests', 'id,todo_id,state,consumed_at,resume_taken_at,resume_machine_id,resume_answered_by');
|
|
6235
|
+
const holds = new Map();
|
|
6236
|
+
const hold = (todoId) => {
|
|
6237
|
+
let value = holds.get(todoId);
|
|
6238
|
+
if (!value) {
|
|
6239
|
+
value = { todoId, messageIds: [], messageSources: [], permissionSources: [] };
|
|
6240
|
+
holds.set(todoId, value);
|
|
6241
|
+
}
|
|
6242
|
+
return value;
|
|
6243
|
+
};
|
|
6244
|
+
for (const message of messages) {
|
|
6245
|
+
if (message.answered_by != null || message.taken_machine_id && message.taken_machine_id !== machineId)
|
|
6246
|
+
continue;
|
|
6247
|
+
const pending = hold(message.todo_id);
|
|
6248
|
+
pending.messageIds.push(message.id);
|
|
6249
|
+
if (message.taken_at)
|
|
6250
|
+
pending.messageSources.push({ id: message.id, taken_at: message.taken_at });
|
|
6251
|
+
if (message.taken_machine_id === machineId && message.taken_at)
|
|
6252
|
+
recordExitedInterruption({
|
|
6253
|
+
surface: 'reply', todoId: message.todo_id, messageLeases: [{ id: message.id, taken_at: message.taken_at }],
|
|
6254
|
+
grantLeases: [], observedPendingMessageIds: null,
|
|
6255
|
+
});
|
|
6256
|
+
}
|
|
6257
|
+
for (const permission of permissions) {
|
|
6258
|
+
if (!permission.todo_id || permission.state !== 'granted' || permission.resume_answered_by != null
|
|
6259
|
+
|| permission.resume_machine_id && permission.resume_machine_id !== machineId)
|
|
6260
|
+
continue;
|
|
6261
|
+
hold(permission.todo_id).permissionSources.push({ id: permission.id, state: permission.state, consumed_at: permission.consumed_at });
|
|
6262
|
+
if (permission.resume_machine_id === machineId && permission.resume_taken_at)
|
|
6263
|
+
recordExitedInterruption({
|
|
6264
|
+
surface: 'reply', todoId: permission.todo_id, messageLeases: [],
|
|
6265
|
+
grantLeases: [{ id: permission.id, resume_taken_at: permission.resume_taken_at }], observedPendingMessageIds: null,
|
|
6266
|
+
});
|
|
6267
|
+
}
|
|
6268
|
+
for (const worker of workers) {
|
|
6269
|
+
if (worker.machine_id !== machineId || !['working', 'dispatching'].includes(worker.state))
|
|
6270
|
+
continue;
|
|
6271
|
+
const todo = todos.find((row) => row.id === worker.todo_id);
|
|
6272
|
+
if (!todo || todo.claimed_machine_id !== machineId || !todo.claimed_at)
|
|
6273
|
+
continue;
|
|
6274
|
+
recordExitedInterruption({ surface: 'worker', todoId: todo.id, workerId: worker.id,
|
|
6275
|
+
claimedAt: todo.claimed_at, observedPendingMessageIds: null });
|
|
6276
|
+
}
|
|
6277
|
+
for (const baseline of holds.values())
|
|
6278
|
+
baseline.messageIds = messages.filter((row) => row.todo_id === baseline.todoId).map((row) => row.id);
|
|
6279
|
+
saveLegacySnapshot([...holds.values()]);
|
|
6280
|
+
}
|
|
6281
|
+
for (const surface of ['worker', 'reply']) {
|
|
6282
|
+
for (const receipt of interruptionReceipts(surface)) {
|
|
6283
|
+
const ref = receipt.reference;
|
|
6284
|
+
const workers = ref.surface === 'worker' ? [{ worker_id: ref.workerId, todo_id: ref.todoId,
|
|
6285
|
+
claimed_at: ref.claimedAt, observed_pending_message_ids: ref.observedPendingMessageIds }] : [];
|
|
6286
|
+
const replies = ref.surface === 'reply' ? [{ todo_id: ref.todoId, message_leases: ref.messageLeases,
|
|
6287
|
+
grant_leases: ref.grantLeases, observed_pending_message_ids: ref.observedPendingMessageIds }] : [];
|
|
6288
|
+
const { data, error } = await client.rpc('cliv2_interrupt_machine_work', {
|
|
6289
|
+
p_machine_id: machineId, p_operation_id: receipt.operationId, p_interrupted_at: receipt.interruptedAt,
|
|
6290
|
+
p_workers: workers, p_replies: replies,
|
|
6291
|
+
});
|
|
6292
|
+
if (error)
|
|
6293
|
+
throw error;
|
|
6294
|
+
const expected = ref.surface === 'worker' ? [{ kind: ref.workerId ? 'worker' : 'claim', id: ref.workerId ?? ref.todoId }] : ref.surface === 'reply'
|
|
6295
|
+
? [...ref.messageLeases.map((source) => ({ kind: 'message', id: source.id })), ...ref.grantLeases.map((source) => ({ kind: 'grant', id: source.id }))] : [];
|
|
6296
|
+
const results = data;
|
|
6297
|
+
if (!Array.isArray(results) || expected.some((expectedRow) => !results.some((row) => row.id === expectedRow.id && row.kind === expectedRow.kind
|
|
6298
|
+
&& ['interrupted', 'already_interrupted', 'already_settled', 'continued', 'stale'].includes(row.outcome)))) {
|
|
6299
|
+
throw new Error('Interrupted work was not fully acknowledged. Work remains protected.');
|
|
6300
|
+
}
|
|
6301
|
+
acknowledgeInterruption(receipt.id);
|
|
6302
|
+
}
|
|
6303
|
+
}
|
|
6304
|
+
await refreshLegacyTodoHolds(client, machineId);
|
|
6305
|
+
}
|
|
6306
|
+
export async function refreshLegacyTodoHolds(client, machineId) {
|
|
6307
|
+
for (const hold of legacyTodoHolds()) {
|
|
6308
|
+
const todo = await client.from('cliv2_loose_todos').select('id,claimed_machine_id').eq('id', hold.todoId).maybeSingle();
|
|
6309
|
+
if (todo.error)
|
|
6310
|
+
throw todo.error;
|
|
6311
|
+
if (!todo.data) {
|
|
6312
|
+
releaseLegacyTodoHold(hold.todoId);
|
|
6313
|
+
continue;
|
|
6314
|
+
}
|
|
6315
|
+
const messages = { data: await legacyRows(client, 'cliv2_run_messages', 'id,answered_by,taken_at,taken_machine_id', hold.todoId) };
|
|
6316
|
+
if (messages.data.some((row) => !hold.messageIds.includes(row.id))) {
|
|
6317
|
+
releaseLegacyTodoHold(hold.todoId);
|
|
6318
|
+
continue;
|
|
6319
|
+
}
|
|
6320
|
+
const permissions = { data: await legacyRows(client, 'cliv2_permission_requests', 'id,state,consumed_at,resume_answered_by,resume_machine_id', hold.todoId) };
|
|
6321
|
+
const messagesRemain = messages.data.some((row) => hold.messageIds.includes(row.id) && row.answered_by === null
|
|
6322
|
+
&& (!row.taken_machine_id || row.taken_machine_id === machineId));
|
|
6323
|
+
const grantsRemain = permissions.data.some((row) => hold.permissionSources.some((source) => source.id === row.id)
|
|
6324
|
+
&& row.state === 'granted' && row.resume_answered_by === null && (!row.resume_machine_id || row.resume_machine_id === machineId));
|
|
6325
|
+
if (!messagesRemain && !grantsRemain)
|
|
6326
|
+
releaseLegacyTodoHold(hold.todoId);
|
|
6327
|
+
}
|
|
6328
|
+
}
|