@ctrl-spc/cs 0.7.13 → 0.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -3
- package/dist/autostart.js +103 -122
- package/dist/companion-ui.js +34 -6
- package/dist/companion.js +59 -150
- package/dist/config.js +52 -1
- package/dist/daemon-lifecycle.js +548 -0
- package/dist/daemon-lock.js +149 -42
- package/dist/daemon-processes.js +756 -0
- package/dist/daemon.js +14 -46
- package/dist/darwin-coalition.js +340 -0
- package/dist/index.js +70 -74
- package/dist/login.js +5 -3
- package/dist/native/darwin-coalition +0 -0
- package/dist/native/darwin-coalition.build.json +1 -0
- package/dist/native/darwin-coalition.c +145 -0
- package/dist/orchestrator.js +620 -428
- package/dist/panel3/checkout.js +100 -8
- package/dist/panel3/prompt.js +16 -8
- package/dist/panel3/run.js +891 -541
- package/dist/panel3/spawn.js +59 -11
- package/dist/panel3/tools.js +28 -5
- package/dist/presence.js +183 -24
- package/dist/supabase.js +43 -9
- package/dist/win-shell.js +464 -1
- package/dist/windows-job.js +312 -0
- package/package.json +4 -3
package/dist/orchestrator.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { execFile, spawn as spawnChild } from 'node:child_process';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { beginOwnedClaim, ownedWorkAllowed, hasOwnedWorkContext, ownedWorkerClaimedAt, ownedWorkerExecutionHeld, reserveOwnedWork, legacyHeldTodoIds, interruptionReceipts, acknowledgeInterruption, legacySnapshotNeeded, saveLegacySnapshot, legacyTodoHolds, releaseLegacyTodoHold, recordExitedInterruption, beginRpcClaim, deferRpcClaim, pendingRpcClaims, recordRpcClaims, finishRpcClaims } from './daemon-processes.js';
|
|
3
4
|
import { promisify } from 'node:util';
|
|
4
5
|
import { agentPath, detectAgents } from './agents.js';
|
|
5
6
|
/* 18a Slice 7: the two Windows spawn rules, in one module so a caller cannot
|
|
6
7
|
take the shell and forget the quoting. `quoteForCmd` is re-exported because
|
|
7
8
|
this module's own tests reach for it here. */
|
|
8
|
-
import { killTree, needsShell, processIsAlive, quoteForCmd } from './win-shell.js';
|
|
9
|
+
import { killTree, needsShell, processIsAlive, quoteForCmd, spawnOwnedProcess, releaseOwnedProcess } from './win-shell.js';
|
|
9
10
|
export { quoteForCmd } from './win-shell.js';
|
|
10
11
|
import { ORCHESTRATOR_MAX_CONCURRENT, ORCHESTRATOR_SPAWN_TIMEOUT_MS } from './env.js';
|
|
11
12
|
import { agentDisplayName, plainFailureReason } from './failure-reason.js';
|
|
@@ -224,7 +225,7 @@ skipIds = []) {
|
|
|
224
225
|
row rather than blocking or double-claiming. `userId` is no longer passed:
|
|
225
226
|
the function scopes itself with `auth.uid()`, which is the same identity
|
|
226
227
|
RLS uses and cannot be spoofed by a caller. */
|
|
227
|
-
const { data, error } = await client
|
|
228
|
+
const { data, error, operationId } = await cliClaim(client, 'cliv2_claim_next_loose_todo', {
|
|
228
229
|
p_machine_id: machineId,
|
|
229
230
|
p_agent: agent,
|
|
230
231
|
p_skip_ids: skipIds,
|
|
@@ -240,40 +241,12 @@ skipIds = []) {
|
|
|
240
241
|
const row = (data ?? [])[0];
|
|
241
242
|
if (!row)
|
|
242
243
|
return null;
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
claim, rather than added to the RPC's return table, because that keeps the
|
|
247
|
-
shipped function's contract alone and needs no migration.
|
|
248
|
-
|
|
249
|
-
WHY IT IS NEEDED. hand_off_stage clears the claim so the next stage is
|
|
250
|
-
dispatched to a fresh worker, and the handing-off worker then keeps talking
|
|
251
|
-
for as long as it takes to write its closing answer. Its exit path wrote
|
|
252
|
-
`state='done'` unconditionally, over a row a SECOND worker was already
|
|
253
|
-
working: the card read Done carrying stage 1's answer, and because the claim
|
|
254
|
-
RPC skips `done` rows the request could never be claimed again, so the
|
|
255
|
-
remaining stages were never run. Found by the gate's audit, and it matches
|
|
256
|
-
what the walk saw.
|
|
257
|
-
|
|
258
|
-
Every terminal write is now fenced on this value, so a run that no longer
|
|
259
|
-
holds the claim writes NOTHING. Null is possible only if the row vanished
|
|
260
|
-
between claiming and reading it, and an unfenceable dispatch is treated as
|
|
261
|
-
already superseded rather than allowed to clobber. */
|
|
262
|
-
let claimedAt = null;
|
|
263
|
-
try {
|
|
264
|
-
const { data: claimRow } = await client
|
|
265
|
-
.from('cliv2_loose_todos')
|
|
266
|
-
.select('claimed_at')
|
|
267
|
-
.eq('id', row.id)
|
|
268
|
-
.maybeSingle();
|
|
269
|
-
claimedAt = claimRow?.claimed_at ?? null;
|
|
270
|
-
}
|
|
271
|
-
catch {
|
|
272
|
-
/* Unreadable is treated as unfenceable, which only loses the protection —
|
|
273
|
-
it never blocks the dispatch that was already claimed successfully. */
|
|
274
|
-
}
|
|
244
|
+
const claimedAt = row.claimed_at ?? null;
|
|
245
|
+
if (hasOwnedWorkContext() && !claimedAt)
|
|
246
|
+
throw new Error('The claim did not return an exact attempt identity. Update the database before starting work.');
|
|
275
247
|
return {
|
|
276
248
|
id: row.id,
|
|
249
|
+
...(operationId ? { claimOperationId: operationId } : {}),
|
|
277
250
|
instruction: row.instruction,
|
|
278
251
|
/* `?? null` rather than trusting the shape: a daemon running against a
|
|
279
252
|
database that has not taken this slice's migration yet gets neither
|
|
@@ -514,10 +487,13 @@ claimedAt) {
|
|
|
514
487
|
const write = client
|
|
515
488
|
.from('cliv2_loose_todos')
|
|
516
489
|
.update({ answer, state: 'done' })
|
|
517
|
-
.eq('id', todoId)
|
|
518
|
-
|
|
490
|
+
.eq('id', todoId)
|
|
491
|
+
.in('state', ['working', 'needs-input']);
|
|
492
|
+
const { data: written, error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write).select('id');
|
|
519
493
|
if (error)
|
|
520
494
|
throw error;
|
|
495
|
+
if (!written?.length)
|
|
496
|
+
return;
|
|
521
497
|
/* 18c SLICE 1 — CLOSE OUT THE LAST ACTIVITY LINE.
|
|
522
498
|
*
|
|
523
499
|
* report_activity only ever flips the PREVIOUS 'doing' row to 'done' when a
|
|
@@ -888,7 +864,8 @@ claimedAt) {
|
|
|
888
864
|
// `answer` is constrained nonblank-when-present, so '' must go in as null
|
|
889
865
|
// rather than as an empty string the check would refuse.
|
|
890
866
|
.update({ answer: trimmed || null, state: 'needs-input' })
|
|
891
|
-
.eq('id', todoId)
|
|
867
|
+
.eq('id', todoId)
|
|
868
|
+
.in('state', ['working', 'needs-input']);
|
|
892
869
|
const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
|
|
893
870
|
if (error)
|
|
894
871
|
throw error;
|
|
@@ -934,12 +911,13 @@ export async function todoWasStopped(client, todoId) {
|
|
|
934
911
|
* already hit Retry, the unit is `working` again and this matches nothing. That
|
|
935
912
|
* is the Gherkin line "the daemon does not later overwrite that".
|
|
936
913
|
*/
|
|
937
|
-
export async function releaseStoppedTodo(client, todoId) {
|
|
938
|
-
const
|
|
914
|
+
export async function releaseStoppedTodo(client, todoId, claimedAt) {
|
|
915
|
+
const write = client
|
|
939
916
|
.from('cliv2_loose_todos')
|
|
940
917
|
.update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
|
|
941
918
|
.eq('id', todoId)
|
|
942
919
|
.eq('state', 'stopped');
|
|
920
|
+
const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
|
|
943
921
|
if (error)
|
|
944
922
|
throw error;
|
|
945
923
|
}
|
|
@@ -992,11 +970,12 @@ claimedAt) {
|
|
|
992
970
|
already claimed by the run that replaced this one. */
|
|
993
971
|
const current = data;
|
|
994
972
|
if (claimedAt && current && current.claimed_at !== claimedAt) {
|
|
995
|
-
return { fate: '
|
|
973
|
+
return { fate: 'stale', attempts: current.attempts ?? 0 };
|
|
996
974
|
}
|
|
997
975
|
const attempts = (current?.attempts ?? 0) + 1;
|
|
976
|
+
const expectedClaim = claimedAt ?? current?.claimed_at ?? null;
|
|
998
977
|
if (attempts >= MAX_ATTEMPTS) {
|
|
999
|
-
|
|
978
|
+
let failure = client
|
|
1000
979
|
.from('cliv2_loose_todos')
|
|
1001
980
|
.update({
|
|
1002
981
|
state: 'failed',
|
|
@@ -1010,16 +989,21 @@ claimedAt) {
|
|
|
1010
989
|
route back is Retry, which means now. */
|
|
1011
990
|
retry_after: null,
|
|
1012
991
|
})
|
|
1013
|
-
.eq('id', todoId)
|
|
992
|
+
.eq('id', todoId)
|
|
993
|
+
.in('state', ['working', 'needs-input']);
|
|
994
|
+
failure = expectedClaim ? failure.eq('claimed_at', expectedClaim) : failure.is('claimed_at', null);
|
|
995
|
+
const { data: failed, error: failError } = await failure.select('id');
|
|
1014
996
|
if (failError)
|
|
1015
997
|
throw failError;
|
|
998
|
+
if (!failed?.length)
|
|
999
|
+
return { fate: 'stale', attempts: current?.attempts ?? 0 };
|
|
1016
1000
|
return { fate: 'failed', attempts };
|
|
1017
1001
|
}
|
|
1018
1002
|
/* 18a SLICE 6: THE WAIT IS WRITTEN ON THE SAME STATEMENT AS THE COUNT, for
|
|
1019
1003
|
the reason the count is on the same statement as the release: a wait that
|
|
1020
1004
|
can fail on its own is a wait the daemon might not honour, and the failure
|
|
1021
1005
|
mode is the fork bomb this bound exists to prevent. */
|
|
1022
|
-
|
|
1006
|
+
let release = client
|
|
1023
1007
|
.from('cliv2_loose_todos')
|
|
1024
1008
|
.update({
|
|
1025
1009
|
attempts,
|
|
@@ -1028,9 +1012,14 @@ claimedAt) {
|
|
|
1028
1012
|
claimed_at: null,
|
|
1029
1013
|
retry_after: new Date(nowMs + retryDelayMs(attempts)).toISOString(),
|
|
1030
1014
|
})
|
|
1031
|
-
.eq('id', todoId)
|
|
1015
|
+
.eq('id', todoId)
|
|
1016
|
+
.in('state', ['working', 'needs-input']);
|
|
1017
|
+
release = expectedClaim ? release.eq('claimed_at', expectedClaim) : release.is('claimed_at', null);
|
|
1018
|
+
const { data: released, error: releaseError } = await release.select('id');
|
|
1032
1019
|
if (releaseError)
|
|
1033
1020
|
throw releaseError;
|
|
1021
|
+
if (!released?.length)
|
|
1022
|
+
return { fate: 'stale', attempts: current?.attempts ?? 0 };
|
|
1034
1023
|
return { fate: 'released', attempts };
|
|
1035
1024
|
}
|
|
1036
1025
|
/**
|
|
@@ -1172,11 +1161,13 @@ export async function scopeIsApproved(client, workItemId) {
|
|
|
1172
1161
|
* See the call site for why the claim stays: an unclaimed todo is immediately
|
|
1173
1162
|
* re-claimable, so releasing would spin the tick loop.
|
|
1174
1163
|
*/
|
|
1175
|
-
export async function parkTodoForScope(client, todoId) {
|
|
1176
|
-
const
|
|
1164
|
+
export async function parkTodoForScope(client, todoId, claimedAt) {
|
|
1165
|
+
const write = client
|
|
1177
1166
|
.from('cliv2_loose_todos')
|
|
1178
1167
|
.update({ state: 'needs-input' })
|
|
1179
|
-
.eq('id', todoId)
|
|
1168
|
+
.eq('id', todoId)
|
|
1169
|
+
.in('state', ['working', 'needs-input']);
|
|
1170
|
+
const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
|
|
1180
1171
|
if (error)
|
|
1181
1172
|
throw error;
|
|
1182
1173
|
}
|
|
@@ -1319,14 +1310,16 @@ resolveContext = (claim) => resolveClaimContext(client, claim, warn)) {
|
|
|
1319
1310
|
returns; what bounds a long answer is `state.answering`, keyed per run, and
|
|
1320
1311
|
`state.maxAnswering`. This guard is back to meaning what it says: one take
|
|
1321
1312
|
at a time, which is a few queries. */
|
|
1322
|
-
if (state.takeInFlight)
|
|
1313
|
+
if (state.takeInFlight || !ownedWorkAllowed())
|
|
1323
1314
|
return;
|
|
1315
|
+
const endClaim = beginOwnedClaim();
|
|
1324
1316
|
state.takeInFlight = true;
|
|
1325
1317
|
try {
|
|
1326
1318
|
await takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext);
|
|
1327
1319
|
}
|
|
1328
1320
|
finally {
|
|
1329
1321
|
state.takeInFlight = false;
|
|
1322
|
+
endClaim();
|
|
1330
1323
|
}
|
|
1331
1324
|
}
|
|
1332
1325
|
async function takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext) {
|
|
@@ -1342,6 +1335,7 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1342
1335
|
old check survives on its own. */
|
|
1343
1336
|
if (agents.length === 0)
|
|
1344
1337
|
return;
|
|
1338
|
+
const claimOperations = [];
|
|
1345
1339
|
/* THE RUNS THIS MACHINE CLAIMS, which is what it may answer for. `claimed_at`
|
|
1346
1340
|
is the real predicate rather than the state alone: `releaseStoppedTodo` and
|
|
1347
1341
|
`releaseOrFail` null the claim on `stopped` and `failed`, while `answerTodo`
|
|
@@ -1412,13 +1406,17 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1412
1406
|
which is the whole of the starvation this scoping removes. */
|
|
1413
1407
|
const taken = [];
|
|
1414
1408
|
if (ownTodoIds.length > 0) {
|
|
1415
|
-
const own = await client
|
|
1409
|
+
const own = await cliClaim(client, 'cliv2_take_run_messages', { p_todo_ids: ownTodoIds, p_machine_id: machineId, p_skip_todo_ids: [...legacyHeldTodoIds(), ...state.answering] });
|
|
1410
|
+
if (own.operationId)
|
|
1411
|
+
claimOperations.push(own.operationId);
|
|
1416
1412
|
if (own.error)
|
|
1417
1413
|
throw new Error(own.error.message);
|
|
1418
1414
|
taken.push(...(own.data ?? []));
|
|
1419
1415
|
}
|
|
1420
1416
|
if (designated) {
|
|
1421
|
-
const rest = await client
|
|
1417
|
+
const rest = await cliClaim(client, 'cliv2_take_run_messages', { p_todo_ids: null, p_machine_id: machineId, p_skip_todo_ids: [...legacyHeldTodoIds(), ...state.answering] });
|
|
1418
|
+
if (rest.operationId)
|
|
1419
|
+
claimOperations.push(rest.operationId);
|
|
1422
1420
|
if (rest.error)
|
|
1423
1421
|
throw new Error(rest.error.message);
|
|
1424
1422
|
/* DEDUPED BY ID rather than trusted to be disjoint. The database will not
|
|
@@ -1450,7 +1448,13 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1450
1448
|
cover: the answer is not to a message, it is to an approval. Everything
|
|
1451
1449
|
downstream already handles an empty id list — the covering RPC is skipped
|
|
1452
1450
|
and the lease is not involved, because no message was leased. */
|
|
1453
|
-
const
|
|
1451
|
+
const grants = await cliClaim(client, 'cliv2_take_granted_resumes', { p_todo_ids: designated ? null : ownTodoIds,
|
|
1452
|
+
p_machine_id: machineId, p_skip_todo_ids: [...legacyHeldTodoIds(), ...state.answering] });
|
|
1453
|
+
if (grants.operationId)
|
|
1454
|
+
claimOperations.push(grants.operationId);
|
|
1455
|
+
if (grants.error)
|
|
1456
|
+
throw grants.error;
|
|
1457
|
+
const granted = (grants.data ?? []);
|
|
1454
1458
|
/* AN EMPTY PASS SAYS NOTHING. This runs every 10 seconds and the common case
|
|
1455
1459
|
is nothing to take and nothing approved; a line each time would bury every
|
|
1456
1460
|
useful one, the same rule `reportRole` and `reportNoTools` follow.
|
|
@@ -1495,7 +1499,8 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1495
1499
|
}
|
|
1496
1500
|
/* AND THE APPROVED RUNS JOIN THEM, with no message ids: there is nothing to
|
|
1497
1501
|
cover, because the answer is to an approval rather than to a message. */
|
|
1498
|
-
for (const
|
|
1502
|
+
for (const grant of granted) {
|
|
1503
|
+
const todoId = grant.todo_id;
|
|
1499
1504
|
if (!byRun.has(todoId))
|
|
1500
1505
|
byRun.set(todoId, { messageIds: [], wasExpired: false });
|
|
1501
1506
|
}
|
|
@@ -1553,7 +1558,10 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1553
1558
|
`answerRunMessages` warns and returns false rather than throwing, and
|
|
1554
1559
|
the pass's job is to have STARTED every answer. The `catch` is the
|
|
1555
1560
|
daemon's never-throw contract, not a second failure path. */
|
|
1556
|
-
void answerRunMessages(client, todoId, group.messageIds, { agent: plan.agent, cwd: context.checkout, workingCopyElsewhere: plan.workingCopyElsewhere }, state, spawn, warn
|
|
1561
|
+
void answerRunMessages(client, todoId, group.messageIds, { agent: plan.agent, cwd: context.checkout, workingCopyElsewhere: plan.workingCopyElsewhere }, state, spawn, warn, { surface: 'reply', todoId,
|
|
1562
|
+
messageLeases: taken.filter((row) => row.todo_id === todoId).map((row) => ({ id: row.id, taken_at: row.taken_at })),
|
|
1563
|
+
grantLeases: granted.filter((row) => row.todo_id === todoId).map((row) => ({ id: row.id, resume_taken_at: row.resume_taken_at })),
|
|
1564
|
+
observedPendingMessageIds: null }, machineId).catch(err => {
|
|
1557
1565
|
warn(`[orchestrator] could not answer ${shortId(todoId)}: ${err.message}`);
|
|
1558
1566
|
});
|
|
1559
1567
|
}
|
|
@@ -1561,6 +1569,10 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
|
|
|
1561
1569
|
catch (err) {
|
|
1562
1570
|
reportTakeFailed(state, err.message, warn);
|
|
1563
1571
|
}
|
|
1572
|
+
finally {
|
|
1573
|
+
if (hasOwnedWorkContext())
|
|
1574
|
+
finishRpcClaims(claimOperations);
|
|
1575
|
+
}
|
|
1564
1576
|
}
|
|
1565
1577
|
/**
|
|
1566
1578
|
* 18k Slice 10 — THE CODEBASE THIS RUN IS FOR, recovered the way the tick
|
|
@@ -2195,7 +2207,7 @@ plan, state,
|
|
|
2195
2207
|
/** 18k Slice 10b — THE WORKER'S OWN SPAWN SHAPE, not a responder's. There is no
|
|
2196
2208
|
* second adapter any more: this is what `defaultSpawn(mcpServer)` returns, and
|
|
2197
2209
|
* the answerer differs from a dispatch only in passing no `alsoRunningIn`. */
|
|
2198
|
-
spawn, warn) {
|
|
2210
|
+
spawn, warn, attempt, machineId, owned) {
|
|
2199
2211
|
/* ═══ 18k SLICE 10b — ONE ANSWER PER RUN AT A TIME. ═══
|
|
2200
2212
|
DEPTH, NOT THE PRIMARY DEFENCE. `takeRunMessagesInner` subtracts
|
|
2201
2213
|
`state.answering` from the scoped take's id list, so a run this machine is
|
|
@@ -2220,15 +2232,21 @@ spawn, warn) {
|
|
|
2220
2232
|
into an inner function so the clear is in ONE place rather than repeated at
|
|
2221
2233
|
every `return` below, which is the same shape `orchestratorTick` uses to
|
|
2222
2234
|
release its own slot. */
|
|
2235
|
+
if (hasOwnedWorkContext() && (!attempt || !machineId))
|
|
2236
|
+
throw new Error('The reply has no exact source lease identity.');
|
|
2237
|
+
owned = hasOwnedWorkContext() ? reserveOwnedWork(attempt, plan.agent) : undefined;
|
|
2223
2238
|
state.answering.add(todoId);
|
|
2224
2239
|
try {
|
|
2225
|
-
|
|
2240
|
+
const answered = await answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned);
|
|
2241
|
+
owned?.complete();
|
|
2242
|
+
return answered;
|
|
2226
2243
|
}
|
|
2227
2244
|
finally {
|
|
2228
2245
|
state.answering.delete(todoId);
|
|
2246
|
+
owned?.finishPreparation();
|
|
2229
2247
|
}
|
|
2230
2248
|
}
|
|
2231
|
-
async function answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn) {
|
|
2249
|
+
async function answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned) {
|
|
2232
2250
|
/* ALREADY GIVEN UP ON. Said ONCE, at the bound, and then silently skipped —
|
|
2233
2251
|
the `reportTakeFailed` rule: a line every two minutes forever would bury
|
|
2234
2252
|
every useful one. */
|
|
@@ -2287,6 +2305,12 @@ async function answerRunMessagesInner(client, todoId, messageIds, plan, state, s
|
|
|
2287
2305
|
would either promise a boundary that never comes, or re-dispatch a worker
|
|
2288
2306
|
for a change the agent already made itself. */
|
|
2289
2307
|
const workerIsLive = await liveWorkerForTodo(client, todoId, warn);
|
|
2308
|
+
// A stopped request may retain a claim even though its worker has ended.
|
|
2309
|
+
// Capture it before answering so a late stopped reply cannot clear a newer one.
|
|
2310
|
+
const current = await client.from('cliv2_loose_todos').select('claimed_at').eq('id', todoId).maybeSingle();
|
|
2311
|
+
if (current.error)
|
|
2312
|
+
throw current.error;
|
|
2313
|
+
const replyingToClaim = current.data?.claimed_at ?? null;
|
|
2290
2314
|
/* ═══ 18k SLICE 10b — SPAWNED AS A WORKER, WITH NO CLOCK ON IT. ═══
|
|
2291
2315
|
The worker's own adapter, so the answerer holds the worker's toolset (ruling
|
|
2292
2316
|
1) and no timeout at all (ruling 5) — an agent asked for a real change may
|
|
@@ -2304,166 +2328,58 @@ async function answerRunMessagesInner(client, todoId, messageIds, plan, state, s
|
|
|
2304
2328
|
NO `alsoRunningIn`: the answerer works one run, in that run's own checkout.
|
|
2305
2329
|
`runTodoId` IS the run, so a tool call it makes is attributed without
|
|
2306
2330
|
depending on the agent repeating an id back to us. */
|
|
2331
|
+
if (attempt) {
|
|
2332
|
+
const pending = await client.from('cliv2_run_messages').select('id').eq('todo_id', todoId).is('answered_by', null);
|
|
2333
|
+
if (pending.error)
|
|
2334
|
+
throw pending.error;
|
|
2335
|
+
attempt.observedPendingMessageIds = (pending.data ?? []).map((row) => row.id);
|
|
2336
|
+
owned?.setReference(attempt);
|
|
2337
|
+
}
|
|
2307
2338
|
let spawnedPid = null;
|
|
2308
2339
|
const result = await spawn(plan.agent, buildReplyPrompt(record, bodies, { hasCheckout: plan.cwd !== null, workingCopyElsewhere: plan.workingCopyElsewhere }, workerIsLive), undefined, plan.cwd, () => todoWasStopped(client, todoId), (pid) => {
|
|
2309
2340
|
state.livePids.add(pid);
|
|
2310
2341
|
spawnedPid = pid;
|
|
2311
|
-
}, undefined, todoId);
|
|
2342
|
+
}, undefined, todoId, owned);
|
|
2312
2343
|
/* HOWEVER THE ANSWER ENDED. The pid must leave `livePids` on every path, or a
|
|
2313
2344
|
dead child looks alive to the liveness watch forever. */
|
|
2314
2345
|
if (spawnedPid !== null)
|
|
2315
2346
|
state.livePids.delete(spawnedPid);
|
|
2316
|
-
|
|
2317
|
-
CHECKED BEFORE `!result.ok`, because a killed child also reports not-ok, and
|
|
2318
|
-
falling through would count the user's own decision as an attempt and
|
|
2319
|
-
re-spawn the very thing they cancelled — `dispatchClaimInner` makes the same
|
|
2320
|
-
check for the same reason.
|
|
2321
|
-
|
|
2322
|
-
BUT STOPPING ALONE WOULD BE AN INFINITE SPAWN LOOP. A stopped run stays
|
|
2323
|
-
stopped, so the message stays uncovered, the lease lapses, the next take
|
|
2324
|
-
spawns another answerer, and that one is stopped too — forever, burning no
|
|
2325
|
-
budget because no attempt is counted. So the messages are COVERED, and
|
|
2326
|
-
`cliv2_mark_messages_answered` needs a reply id, so one short reply is what
|
|
2327
|
-
covers them. That is also the honest thread: it says the message was not
|
|
2328
|
-
answered because the work was stopped, rather than showing "Queued" forever.
|
|
2329
|
-
ux.md's north star forbids "this will never be answered", and an uncovered
|
|
2330
|
-
stopped message is exactly that state.
|
|
2331
|
-
|
|
2332
|
-
AND THE CLAIM. This function set no claim, but the run may carry one from an
|
|
2333
|
-
earlier dispatch, and `releaseStoppedTodo` is the only thing that clears it —
|
|
2334
|
-
without this call Retry cannot pick the run up. Its own `.eq('state','stopped')`
|
|
2335
|
-
guard makes it a no-op when Retry has already re-armed the run. */
|
|
2336
|
-
if (result.stopped) {
|
|
2337
|
-
try {
|
|
2338
|
-
const replyId = await writeRunReply(client, todoId, 'Stopped.');
|
|
2339
|
-
const { error } = await client.rpc('cliv2_mark_messages_answered', {
|
|
2340
|
-
p_reply_id: replyId,
|
|
2341
|
-
p_message_ids: messageIds,
|
|
2342
|
-
});
|
|
2343
|
-
if (error)
|
|
2344
|
-
throw new Error(error.message);
|
|
2345
|
-
for (const id of messageIds)
|
|
2346
|
-
state.replyAttempts.delete(id);
|
|
2347
|
-
}
|
|
2348
|
-
catch (err) {
|
|
2349
|
-
warn(`[orchestrator] ${shortId(todoId)} was stopped and the conversation could not be closed off: ${err.message}`);
|
|
2350
|
-
}
|
|
2351
|
-
try {
|
|
2352
|
-
await releaseStoppedTodo(client, todoId);
|
|
2353
|
-
}
|
|
2354
|
-
catch (err) {
|
|
2355
|
-
warn(`[orchestrator] ${shortId(todoId)} was stopped and its claim could not be released: ${err.message}`);
|
|
2356
|
-
}
|
|
2347
|
+
if (result.interrupted)
|
|
2357
2348
|
return false;
|
|
2358
|
-
|
|
2359
|
-
if (!result.ok) {
|
|
2349
|
+
if (!result.ok && !result.stopped) {
|
|
2360
2350
|
fail(result.error);
|
|
2361
2351
|
return false;
|
|
2362
2352
|
}
|
|
2363
|
-
const body = result.output.trim();
|
|
2364
|
-
if (body
|
|
2353
|
+
const body = result.stopped ? 'Stopped.' : result.output.trim();
|
|
2354
|
+
if (!body) {
|
|
2365
2355
|
fail('it said nothing');
|
|
2366
2356
|
return false;
|
|
2367
2357
|
}
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2358
|
+
if (!attempt || !machineId)
|
|
2359
|
+
throw new Error('The reply has no exact source lease identity.');
|
|
2360
|
+
const finished = await client.rpc('cliv2_finish_reply_attempt', {
|
|
2361
|
+
p_todo_id: todoId, p_machine_id: machineId, p_message_leases: attempt.messageLeases,
|
|
2362
|
+
p_grant_leases: attempt.grantLeases, p_body: body,
|
|
2363
|
+
});
|
|
2364
|
+
if (finished.error)
|
|
2365
|
+
throw finished.error;
|
|
2366
|
+
const outcome = finished.data?.[0]?.outcome;
|
|
2367
|
+
if (outcome === 'stale')
|
|
2374
2368
|
return false;
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
starts from zero rather than inheriting a spent budget.
|
|
2378
|
-
|
|
2379
|
-
═══ 18k SLICE 9: AND IT IS CLEARED HERE, BEFORE THE COVERING WRITE. ═══
|
|
2380
|
-
The reply EXISTS from this line on, so the attempt is spent whatever happens
|
|
2381
|
-
next, and the covering below must NOT go through `fail()`. `fail()`'s
|
|
2382
|
-
contract is "nothing was written, so retry the whole thing", and that
|
|
2383
|
-
contract does not hold once a reply is in the thread. Had the covering's
|
|
2384
|
-
failure counted an attempt, three consecutive attribution failures — a reply
|
|
2385
|
-
written EACH time — would exhaust `MAX_REPLY_ATTEMPTS` while `answered_by`
|
|
2386
|
-
stayed null, so the take would keep re-offering the row forever and nothing
|
|
2387
|
-
would ever spawn again: three replies, then permanent silence with "Queued"
|
|
2388
|
-
showing. Strictly worse than the predicate this slice replaces, which at
|
|
2389
|
-
least excluded such a message after the first reply. */
|
|
2369
|
+
if (outcome !== 'accepted' && outcome !== 'already_settled')
|
|
2370
|
+
throw new Error('The reply acknowledgement was incomplete.');
|
|
2390
2371
|
for (const id of messageIds)
|
|
2391
2372
|
state.replyAttempts.delete(id);
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
message marked answered by nothing, invisible forever. A visible duplicate
|
|
2403
|
-
beats a silent drop, which is the north star's own ranking (ux.md:739).
|
|
2404
|
-
|
|
2405
|
-
AND THE BOUND IS NOT LOST, IT MOVES: the RPC's own `answered_by is null`
|
|
2406
|
-
term is what bounds a repeated failure here. Each retry round writes at most
|
|
2407
|
-
one extra reply and rounds are one pass apart, so the user-visible bound is
|
|
2408
|
-
duplicate replies at one per pass, never an unbounded spawn loop and never
|
|
2409
|
-
permanent silence. A database that will not accept the mark is an outage,
|
|
2410
|
-
and the honest behaviour under an outage is a visible duplicate rather than
|
|
2411
|
-
a message that disappears. */
|
|
2412
|
-
/* 18k Slice 10b — NOTHING TO ATTRIBUTE WHEN AN APPROVAL WOKE THIS. The reply
|
|
2413
|
-
answers a grant, not a message, so there is no row to mark and the RPC is
|
|
2414
|
-
skipped rather than called with an empty list. The grant's own `consumed_at`
|
|
2415
|
-
is what stops it being offered again, stamped by the tool when the approved
|
|
2416
|
-
call finally goes through. */
|
|
2417
|
-
if (messageIds.length > 0) {
|
|
2418
|
-
try {
|
|
2419
|
-
const { error } = await client.rpc('cliv2_mark_messages_answered', {
|
|
2420
|
-
p_reply_id: replyId,
|
|
2421
|
-
p_message_ids: messageIds,
|
|
2422
|
-
});
|
|
2423
|
-
if (error)
|
|
2424
|
-
throw new Error(error.message);
|
|
2425
|
-
}
|
|
2426
|
-
catch (err) {
|
|
2427
|
-
warn(`[orchestrator] the reply to ${shortId(todoId)} could not be attributed, it will be answered again: ${err.message}`);
|
|
2428
|
-
return false;
|
|
2429
|
-
}
|
|
2430
|
-
}
|
|
2431
|
-
/* ═══ 18k SLICE 10b — CASE B: THE BOUNDARY. ═══
|
|
2432
|
-
An agent was already working this checkout, so the reply said what would
|
|
2433
|
-
happen rather than doing it. Clearing the claim is what makes that true: the
|
|
2434
|
-
next tick dispatches a fresh worker, and `readRunDirection` hands it the
|
|
2435
|
-
message. Nothing interrupts the live worker — it finishes what it is on, and
|
|
2436
|
-
the change rides the NEXT dispatch, exactly as `hand_off_stage` already does
|
|
2437
|
-
mid-run.
|
|
2438
|
-
|
|
2439
|
-
AFTER THE REPLY AND ITS COVERING, NEVER BEFORE. Clearing first and then
|
|
2440
|
-
failing to write the reply would re-dispatch a worker for a change nobody
|
|
2441
|
-
was ever told about.
|
|
2442
|
-
|
|
2443
|
-
GUARDED ON `working`/`needs-input`, for `releaseTodoForRedispatch`'s own
|
|
2444
|
-
reason: a run that has since finished, failed or been stopped must not be
|
|
2445
|
-
resurrected into `working` by a claim clear. And all three columns move
|
|
2446
|
-
together — `cliv2_loose_todos_claim_complete_check` makes a half-release a
|
|
2447
|
-
database error.
|
|
2448
|
-
|
|
2449
|
-
BEST-EFFORT: the reply is in the thread whatever happens here, so a failed
|
|
2450
|
-
clear costs the user a change that does not get dispatched — which the next
|
|
2451
|
-
message re-raises — where throwing would lose the reply's own success. */
|
|
2452
|
-
if (workerIsLive) {
|
|
2453
|
-
try {
|
|
2454
|
-
const { error } = await client
|
|
2455
|
-
.from('cliv2_loose_todos')
|
|
2456
|
-
.update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
|
|
2457
|
-
.eq('id', todoId)
|
|
2458
|
-
.in('state', ['working', 'needs-input']);
|
|
2459
|
-
if (error)
|
|
2460
|
-
throw new Error(error.message);
|
|
2461
|
-
}
|
|
2462
|
-
catch (err) {
|
|
2463
|
-
warn(`[orchestrator] ${shortId(todoId)} was answered but the change could not be scheduled: ${err.message}`);
|
|
2464
|
-
}
|
|
2465
|
-
}
|
|
2466
|
-
return true;
|
|
2373
|
+
if (result.stopped && replyingToClaim)
|
|
2374
|
+
await releaseStoppedTodo(client, todoId, replyingToClaim);
|
|
2375
|
+
else if (workerIsLive && replyingToClaim) {
|
|
2376
|
+
const scheduled = await client.from('cliv2_loose_todos')
|
|
2377
|
+
.update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
|
|
2378
|
+
.eq('id', todoId).eq('claimed_at', replyingToClaim).in('state', ['working', 'needs-input']);
|
|
2379
|
+
if (scheduled.error)
|
|
2380
|
+
throw scheduled.error;
|
|
2381
|
+
}
|
|
2382
|
+
return !result.stopped;
|
|
2467
2383
|
}
|
|
2468
2384
|
/** 18k Slice 7 — say ONCE that the take is failing, not every 10 seconds.
|
|
2469
2385
|
*
|
|
@@ -2504,7 +2420,7 @@ export function reportTakeFailed(state, reason, warn) {
|
|
|
2504
2420
|
* is the brake, exactly as it is for the scope park, and answering is what lets
|
|
2505
2421
|
* it off.
|
|
2506
2422
|
*/
|
|
2507
|
-
export async function askWhichCodebase(client, todoId, projectName, remotes, warn) {
|
|
2423
|
+
export async function askWhichCodebase(client, todoId, projectName, remotes, warn, claimedAt) {
|
|
2508
2424
|
try {
|
|
2509
2425
|
const { error } = await client.rpc('cliv2_ask_about_request', {
|
|
2510
2426
|
p_todo: todoId,
|
|
@@ -2517,7 +2433,7 @@ export async function askWhichCodebase(client, todoId, projectName, remotes, war
|
|
|
2517
2433
|
});
|
|
2518
2434
|
if (error)
|
|
2519
2435
|
throw new Error(error.message);
|
|
2520
|
-
await parkTodoForScope(client, todoId);
|
|
2436
|
+
await parkTodoForScope(client, todoId, claimedAt);
|
|
2521
2437
|
return true;
|
|
2522
2438
|
}
|
|
2523
2439
|
catch (err) {
|
|
@@ -2797,11 +2713,13 @@ export async function collisionForTodo(client, todoId, gitRemoteUrl, warn) {
|
|
|
2797
2713
|
return null;
|
|
2798
2714
|
}
|
|
2799
2715
|
}
|
|
2800
|
-
export async function releaseTodo(client, todoId) {
|
|
2801
|
-
const
|
|
2716
|
+
export async function releaseTodo(client, todoId, claimedAt) {
|
|
2717
|
+
const write = client
|
|
2802
2718
|
.from('cliv2_loose_todos')
|
|
2803
2719
|
.update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
|
|
2804
|
-
.eq('id', todoId)
|
|
2720
|
+
.eq('id', todoId)
|
|
2721
|
+
.in('state', ['working', 'needs-input']);
|
|
2722
|
+
const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
|
|
2805
2723
|
if (error)
|
|
2806
2724
|
throw error;
|
|
2807
2725
|
}
|
|
@@ -3707,12 +3625,13 @@ run = spawnAgent) {
|
|
|
3707
3625
|
alsoRunningIn,
|
|
3708
3626
|
/** 18c Slice 1 — the request this spawn is working, so
|
|
3709
3627
|
* `report_activity` lines can be attributed without the prompt. */
|
|
3710
|
-
runTodoId) =>
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
3628
|
+
runTodoId, owned) => {
|
|
3629
|
+
/* !Cleanup Phase 6b (I42): NO TIMEOUT. `NO_SPAWN_TIMEOUT` disables the kill
|
|
3630
|
+
entirely — a working agent is never killed for elapsed time (user ruling
|
|
3631
|
+
2026-08-06). Liveness is watched from outside the run instead, by
|
|
3632
|
+
`reapDeadWorkers` on the heartbeat. */
|
|
3633
|
+
return run(agent, prompt, NO_SPAWN_TIMEOUT, { onStep, mcpServer, cwd, stopRequested, onPid, alsoRunningIn, runTodoId, owned });
|
|
3634
|
+
};
|
|
3716
3635
|
}
|
|
3717
3636
|
/** 18k Slice 8 — how many times one message may be attempted before the daemon
|
|
3718
3637
|
* stops spawning for it.
|
|
@@ -3727,7 +3646,9 @@ export const MAX_REPLY_ATTEMPTS = 3;
|
|
|
3727
3646
|
/** Bound on retained stdout. Generous — a real run's JSONL is a few hundred KB —
|
|
3728
3647
|
* but finite, so a runaway agent cannot exhaust memory. */
|
|
3729
3648
|
const MAX_OUTPUT_BYTES = 32 * 1024 * 1024;
|
|
3730
|
-
export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_TIMEOUT_MS, { resolveBin = agentPath, exec, onStep, spawnProcess = spawnChild, maxOutputBytes = MAX_OUTPUT_BYTES, mcpServer = null, cwd = null, stopRequested, stopPollMs = STOP_POLL_MS, onPid, alsoRunningIn = [], runTodoId = null } = {}) {
|
|
3649
|
+
export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_TIMEOUT_MS, { resolveBin = agentPath, exec, onStep, spawnProcess = spawnChild, maxOutputBytes = MAX_OUTPUT_BYTES, mcpServer = null, cwd = null, stopRequested, stopPollMs = STOP_POLL_MS, onPid, alsoRunningIn = [], runTodoId = null, owned } = {}) {
|
|
3650
|
+
if (owned?.interrupted())
|
|
3651
|
+
return { ok: false, output: '', error: 'Work was interrupted by the service command.', interrupted: true };
|
|
3731
3652
|
const bin = resolveBin(agent);
|
|
3732
3653
|
if (!bin)
|
|
3733
3654
|
return { ok: false, output: '', error: `${agent} is not installed on this machine` };
|
|
@@ -3741,7 +3662,7 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
3741
3662
|
Windows path holds spaces (`C:\\Program Files\\…`) and the MCP config is
|
|
3742
3663
|
JSON, so both would still be split by cmd.exe. */
|
|
3743
3664
|
const viaShell = needsShell(bin);
|
|
3744
|
-
const rawArgs = headlessAgentArgs(agent, prompt, mcpServer, alsoRunningIn,
|
|
3665
|
+
const rawArgs = headlessAgentArgs(agent, prompt, mcpServer, alsoRunningIn, true, runTodoId);
|
|
3745
3666
|
const args = viaShell ? rawArgs.map(quoteForCmd) : rawArgs;
|
|
3746
3667
|
/* !Cleanup Phase 2 (I5), CORRECTED BY 18b SLICE 2.
|
|
3747
3668
|
It used to be `cwd ?? undefined`, and `undefined` means "inherit the
|
|
@@ -3829,13 +3750,18 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
3829
3750
|
return new Promise((resolve) => {
|
|
3830
3751
|
let child;
|
|
3831
3752
|
try {
|
|
3832
|
-
|
|
3753
|
+
owned?.beforeSpawn();
|
|
3754
|
+
const spawnExecution = owned && spawnProcess === spawnChild
|
|
3755
|
+
? (file, argv, options) => spawnOwnedProcess(file, argv, options, owned.id)
|
|
3756
|
+
: spawnProcess;
|
|
3757
|
+
child = spawnExecution(bin, args, {
|
|
3833
3758
|
// The repo-wide invariant for ANY child process (MEMORY: "Windows
|
|
3834
3759
|
// silence decision"). A user must never see a console flash.
|
|
3835
3760
|
windowsHide: true,
|
|
3836
3761
|
shell: viaShell,
|
|
3837
3762
|
// 18a Slice 7: stdin is a pipe only where the prompt travels down it.
|
|
3838
|
-
stdio: [
|
|
3763
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
3764
|
+
detached: process.platform !== 'win32',
|
|
3839
3765
|
// !Cleanup Phase 2 (I5) — the worker starts inside the work item's own
|
|
3840
3766
|
// checkout instead of at the daemon's cwd (`/` under launchd).
|
|
3841
3767
|
cwd: childCwd,
|
|
@@ -3849,17 +3775,15 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
3849
3775
|
/* Nothing was spawned, so nothing can still be holding the directory. */
|
|
3850
3776
|
if (codexHome)
|
|
3851
3777
|
removeCodexRunHome(codexHome);
|
|
3852
|
-
return resolve(spawnFailure(agent, err));
|
|
3778
|
+
return resolve({ ...spawnFailure(agent, err), ...(owned?.interrupted() ? { interrupted: true } : {}) });
|
|
3853
3779
|
}
|
|
3854
3780
|
/* 18a SLICE 7 — AND THE PROMPT GOES DOWN IT, then the pipe is CLOSED.
|
|
3855
3781
|
Both agents wait for end-of-input before they start, so leaving it open
|
|
3856
3782
|
hangs the run forever. The write is best-effort in the same sense every
|
|
3857
3783
|
other pipe operation here is: a child that died before reading raises
|
|
3858
3784
|
EPIPE, and that is the child's exit to report, not this write's. */
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
child.stdin.end(prompt);
|
|
3862
|
-
}
|
|
3785
|
+
let registration = Promise.resolve();
|
|
3786
|
+
child.stdin?.on('error', () => { });
|
|
3863
3787
|
/* !Cleanup Phase 6b (I43) — hand the pid up the moment it exists, so the
|
|
3864
3788
|
liveness watch can tell a dead agent from a quiet one. Guarded on
|
|
3865
3789
|
`typeof`: a child that failed to spawn has no pid, and reporting
|
|
@@ -4020,9 +3944,24 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
4020
3944
|
it stripped to '', the column would take null and the card would lose
|
|
4021
3945
|
the context its question needs. `redactSecrets` is identity when the run
|
|
4022
3946
|
read no credential, which is almost every run. */
|
|
4023
|
-
|
|
4024
|
-
|
|
4025
|
-
|
|
3947
|
+
void (async () => {
|
|
3948
|
+
await registration.catch(() => { });
|
|
3949
|
+
if (owned && typeof child.pid === 'number') {
|
|
3950
|
+
for (;;) {
|
|
3951
|
+
try {
|
|
3952
|
+
await owned.exited();
|
|
3953
|
+
break;
|
|
3954
|
+
}
|
|
3955
|
+
catch {
|
|
3956
|
+
await new Promise((done) => setTimeout(done, 250));
|
|
3957
|
+
}
|
|
3958
|
+
}
|
|
3959
|
+
}
|
|
3960
|
+
if (codexHome && owned)
|
|
3961
|
+
removeCodexRunHome(codexHome);
|
|
3962
|
+
resolve({ ...(result.output ? { ...result, output: redactSecrets(runTodoId, result.output) } : result),
|
|
3963
|
+
...(owned?.interrupted() ? { interrupted: true } : {}) });
|
|
3964
|
+
})();
|
|
4026
3965
|
};
|
|
4027
3966
|
/* ═══ 18c SLICE 8 — THE HOME IS SWEPT WHEN THE CHILD IS GONE, NOT WHEN THE
|
|
4028
3967
|
PROMISE SETTLES. ═══
|
|
@@ -4058,7 +3997,8 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
4058
3997
|
sweptHome = true;
|
|
4059
3998
|
removeCodexRunHome(codexHome);
|
|
4060
3999
|
};
|
|
4061
|
-
|
|
4000
|
+
if (!owned)
|
|
4001
|
+
child.on('close', sweepHomeWhenChildIsGone);
|
|
4062
4002
|
/* !Cleanup PHASE 6b (I42) — THE ELAPSED-TIME KILL IS OFF BY DEFAULT NOW.
|
|
4063
4003
|
User ruling 2026-08-06: "No agent that is actively working should ever be
|
|
4064
4004
|
killed automatically, regardless of how much time has elapsed."
|
|
@@ -4214,6 +4154,18 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
|
|
|
4214
4154
|
way: anything parseable was already captured live above. */
|
|
4215
4155
|
settle(finishSpawn(agent, stdout));
|
|
4216
4156
|
});
|
|
4157
|
+
registration = owned ? owned.register(child) : Promise.resolve();
|
|
4158
|
+
void registration.then(() => {
|
|
4159
|
+
if (!owned?.interrupted()) {
|
|
4160
|
+
if (owned)
|
|
4161
|
+
releaseOwnedProcess(child);
|
|
4162
|
+
child.stdin?.end(prompt);
|
|
4163
|
+
}
|
|
4164
|
+
}).catch((error) => {
|
|
4165
|
+
child.stdin?.destroy();
|
|
4166
|
+
killTree(child);
|
|
4167
|
+
settle(spawnFailure(agent, error));
|
|
4168
|
+
});
|
|
4217
4169
|
});
|
|
4218
4170
|
}
|
|
4219
4171
|
/**
|
|
@@ -4412,7 +4364,8 @@ export async function updateWorker(client, workerId, state, error = null, warn =
|
|
|
4412
4364
|
const { error: writeError } = await client
|
|
4413
4365
|
.from('cliv2_workers')
|
|
4414
4366
|
.update(patch)
|
|
4415
|
-
.eq('id', workerId)
|
|
4367
|
+
.eq('id', workerId)
|
|
4368
|
+
.in('state', ['working', 'dispatching']);
|
|
4416
4369
|
if (writeError)
|
|
4417
4370
|
throw writeError;
|
|
4418
4371
|
}
|
|
@@ -4596,6 +4549,11 @@ export async function reapDeadWorkers(client, machineId, livePids, now = () => n
|
|
|
4596
4549
|
const cutoff = now().getTime() - STALE_WORKER_MS;
|
|
4597
4550
|
let reaped = 0;
|
|
4598
4551
|
for (const row of rows) {
|
|
4552
|
+
if (hasOwnedWorkContext() && ownedWorkerExecutionHeld(row.id))
|
|
4553
|
+
continue;
|
|
4554
|
+
const claimedAt = hasOwnedWorkContext() ? ownedWorkerClaimedAt(row.id) : undefined;
|
|
4555
|
+
if (claimedAt === null)
|
|
4556
|
+
continue;
|
|
4599
4557
|
/* CONDITION 1 — quiet. A worker that has written a step recently is
|
|
4600
4558
|
working, and nothing else is even considered. */
|
|
4601
4559
|
const seen = Date.parse(row.updated_at);
|
|
@@ -4641,7 +4599,9 @@ export async function reapDeadWorkers(client, machineId, livePids, now = () => n
|
|
|
4641
4599
|
card everything Slices 5 and 6 already draw (the attempt, the
|
|
4642
4600
|
reason, the countdown) for free, and at the bound it gives up and
|
|
4643
4601
|
says why instead of being fed back into the same machine. */
|
|
4644
|
-
const { fate, attempts } = await releaseOrFail(client, row.todo_id, plain, now().getTime());
|
|
4602
|
+
const { fate, attempts } = await releaseOrFail(client, row.todo_id, plain, now().getTime(), claimedAt);
|
|
4603
|
+
if (fate === 'stale')
|
|
4604
|
+
continue;
|
|
4645
4605
|
/* 18c SLICE 9 (GAP 15): AND IT HOLDS NOTHING ANY MORE.
|
|
4646
4606
|
THIS PATH NEEDS ITS OWN CALL and is not covered by `dispatchClaim`'s
|
|
4647
4607
|
`finally`: the reaper exists precisely for a worker that died while
|
|
@@ -4705,6 +4665,7 @@ export async function reapDeadWorkers(client, machineId, livePids, now = () => n
|
|
|
4705
4665
|
*/
|
|
4706
4666
|
export async function recoverStrandedWorkers(client, machineId, log = console.log, warn = console.warn) {
|
|
4707
4667
|
try {
|
|
4668
|
+
const lifecycleOwned = hasOwnedWorkContext();
|
|
4708
4669
|
const { data, error } = await client
|
|
4709
4670
|
.from('cliv2_workers')
|
|
4710
4671
|
.update({
|
|
@@ -4732,7 +4693,11 @@ export async function recoverStrandedWorkers(client, machineId, log = console.lo
|
|
|
4732
4693
|
/* The todo goes back in the queue. Order matters: the worker is closed
|
|
4733
4694
|
FIRST, so there is no instant at which a claimable todo still has a live
|
|
4734
4695
|
worker that would refuse the dispatch. */
|
|
4735
|
-
|
|
4696
|
+
// Lifecycle startup has already reconciled exact attempt receipts. A stale
|
|
4697
|
+
// worker row can outlive a newer claim, including one on another machine;
|
|
4698
|
+
// closing that obsolete worker must never release the newer assignment.
|
|
4699
|
+
// Keep the pre-lifecycle compatibility path only for its existing callers.
|
|
4700
|
+
for (const row of lifecycleOwned ? [] : stranded) {
|
|
4736
4701
|
try {
|
|
4737
4702
|
await releaseTodo(client, row.todo_id);
|
|
4738
4703
|
}
|
|
@@ -5287,7 +5252,7 @@ export async function orchestratorTick(deps) {
|
|
|
5287
5252
|
query, so a tick that cannot start anything costs nothing at all — no read,
|
|
5288
5253
|
no claim, no log. Was `if (state.busy)`, which is why a second request typed
|
|
5289
5254
|
three seconds after the first sat untouched until the first finished.
|
|
5290
|
-
|
|
5255
|
+
*
|
|
5291
5256
|
THE SLOT IS TAKEN HERE, NOT AFTER THE CLAIM, and that is not tidiness — it
|
|
5292
5257
|
is the difference between a cap and a suggestion. `pollOrchestrator` is an
|
|
5293
5258
|
interval that does NOT await, so ticks genuinely overlap, and there are
|
|
@@ -5297,7 +5262,7 @@ export async function orchestratorTick(deps) {
|
|
|
5297
5262
|
the cap would bound nothing. Taking the slot before the first await makes
|
|
5298
5263
|
the check-and-take atomic with respect to the event loop, because there is
|
|
5299
5264
|
no suspension point between them.
|
|
5300
|
-
|
|
5265
|
+
*
|
|
5301
5266
|
EVERY early return below MUST give it back, which is why the whole body from
|
|
5302
5267
|
here down is wrapped so the release is in one place rather than repeated at
|
|
5303
5268
|
each `return`. */
|
|
@@ -5305,6 +5270,7 @@ export async function orchestratorTick(deps) {
|
|
|
5305
5270
|
return 'busy';
|
|
5306
5271
|
state.inFlight += 1;
|
|
5307
5272
|
try {
|
|
5273
|
+
await reconcileInterruptedWork(deps.client, deps.machineId);
|
|
5308
5274
|
return await runTick(deps, state, now, log, warn);
|
|
5309
5275
|
}
|
|
5310
5276
|
finally {
|
|
@@ -5314,208 +5280,225 @@ export async function orchestratorTick(deps) {
|
|
|
5314
5280
|
/** The tick body, with its concurrency slot already taken. Split out so the slot
|
|
5315
5281
|
* is released in exactly one place — see the caller. */
|
|
5316
5282
|
async function runTick(deps, state, now, log, warn) {
|
|
5317
|
-
|
|
5318
|
-
|
|
5319
|
-
|
|
5320
|
-
|
|
5321
|
-
start" — deliberately the same resolver, so the two cannot disagree. */
|
|
5322
|
-
const { client, userId, machineId, machineName, agents, resolveContext = (claim) => resolveClaimContext(client, claim, warn), } = deps;
|
|
5323
|
-
/* 18a SLICE 6: THE BACKOFF USED TO BE CHECKED HERE, and returning from this
|
|
5324
|
-
line is precisely what was wrong with it: a failing agent stopped the whole
|
|
5325
|
-
daemon rather than stopping itself. The wait is now per request, on the row,
|
|
5326
|
-
and the claim itself skips a request whose moment has not come. */
|
|
5327
|
-
let role;
|
|
5283
|
+
if (!ownedWorkAllowed())
|
|
5284
|
+
return 'nothing-to-do';
|
|
5285
|
+
const endClaim = beginOwnedClaim();
|
|
5286
|
+
const claimOperations = [];
|
|
5328
5287
|
try {
|
|
5329
|
-
|
|
5330
|
-
|
|
5331
|
-
|
|
5332
|
-
|
|
5333
|
-
|
|
5334
|
-
|
|
5335
|
-
|
|
5336
|
-
|
|
5337
|
-
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
// is not a designation, and guessing would be exactly the fleet-wide race
|
|
5341
|
-
// the machine check exists to prevent.
|
|
5342
|
-
warn(`orchestrator designation read failed, will retry: ${err.message}`);
|
|
5343
|
-
return 'failed';
|
|
5344
|
-
}
|
|
5345
|
-
reportRole(state, role, machineName, log);
|
|
5346
|
-
if (role.kind !== 'this-machine')
|
|
5347
|
-
return 'not-designated';
|
|
5348
|
-
/* ═══ 18d — NO TOOLS MEANS NO DISPATCH. ═══
|
|
5349
|
-
|
|
5350
|
-
FOUND BY WALKING IT, 2026-08-12. A daemon restart raced its own port: the
|
|
5351
|
-
replacement process came up while the old one still held 4579, so
|
|
5352
|
-
`toolsServerStatus().running` was false, `presence.ts` passed
|
|
5353
|
-
`mcpServer: null`, and the very next tick claimed a request and spawned a
|
|
5354
|
-
worker WITH NO CTRL+SPC TOOLS AT ALL. That worker could not read the work
|
|
5355
|
-
item, could not read its own stage document, could not call `ask_question`
|
|
5356
|
-
and could not `hand_off_stage`. It did the only thing left: it wrote an
|
|
5357
|
-
essay into its final answer explaining that the tools were missing, and the
|
|
5358
|
-
request was burned — `attempts` spent, the user's card finished, and
|
|
5359
|
-
nothing to show for it.
|
|
5360
|
-
|
|
5361
|
-
Null is the RIGHT value there: the alternative that comment rejects is
|
|
5362
|
-
letting the worker inherit the user's own agent config, where a second
|
|
5363
|
-
CTRL+SPC server pointed at a DIFFERENT ACCOUNT was once found registered and
|
|
5364
|
-
listening. That is worse. But there was a third option nobody took, which is
|
|
5365
|
-
to not dispatch at all.
|
|
5366
|
-
|
|
5367
|
-
WAITING IS FREE AND CORRECT. The request keeps its place, unclaimed and
|
|
5368
|
-
unspent, and the tools server comes up moments later — the port race
|
|
5369
|
-
resolves as soon as the old process lets go. A dispatch made in this window
|
|
5370
|
-
cannot be retried usefully, because it is not the request that is broken.
|
|
5371
|
-
|
|
5372
|
-
BEFORE THE CLAIM, deliberately: claiming and then refusing to spawn would
|
|
5373
|
-
spend `attempts` on a condition that has nothing to do with the request.
|
|
5374
|
-
|
|
5375
|
-
THE CONDITION IS "A REAL SPAWN WITH NO SERVER", not "no server". An injected
|
|
5376
|
-
`spawn` is a caller who has taken over the launching entirely — every test in
|
|
5377
|
-
the suite, and any future embedder — and for them `mcpServer` describes
|
|
5378
|
-
nothing, because `defaultSpawn` is what turns it into an argv. Gating on the
|
|
5379
|
-
null alone would refuse to dispatch in exactly the cases where the tools are
|
|
5380
|
-
not in question, which is not a stricter check but a wrong one. */
|
|
5381
|
-
if (!deps.mcpServer && !deps.spawn) {
|
|
5382
|
-
reportNoTools(state, warn);
|
|
5383
|
-
return 'no-tools';
|
|
5384
|
-
}
|
|
5385
|
-
state.warnedNoTools = false;
|
|
5386
|
-
/* ═══ !Cleanup PHASE 7 SLICE 2: CLAIM PAST THE BLOCKED ONES. ═══
|
|
5387
|
-
|
|
5388
|
-
A dependent unit must WAIT, and everything behind it must NOT. Those two
|
|
5389
|
-
requirements together are why this is a loop rather than a single claim.
|
|
5390
|
-
|
|
5391
|
-
The claim is this design's only queue position, and it takes the OLDEST
|
|
5392
|
-
claimable row. So a blocked unit that is merely released is re-claimed FIRST
|
|
5393
|
-
on the very next tick, forever — it stops being a waiting unit and becomes a
|
|
5394
|
-
queue stopper. Releasing it AND telling the next claim to skip it is what
|
|
5395
|
-
makes the rest of the queue reachable.
|
|
5396
|
-
|
|
5397
|
-
BOUNDED BY THE NUMBER OF UNITS SKIPPED, not by a fixed count: each iteration
|
|
5398
|
-
either dispatches (returns), finds nothing (returns), or adds exactly one id
|
|
5399
|
-
to `skip`, and a skipped id can never be claimed again this tick. So the
|
|
5400
|
-
loop runs at most once per blocked unit and then ends. No timeout needed,
|
|
5401
|
-
because it cannot revisit a row.
|
|
5402
|
-
|
|
5403
|
-
THE UNIT STAYS `working` AND UNCLAIMED, deliberately — it is NOT parked to
|
|
5404
|
-
`needs-input` the way the scope gate parks. `needs-input` means "this needs
|
|
5405
|
-
YOU", and a dependency needs nothing from the user: it resolves when the
|
|
5406
|
-
blocker finishes. Parking it would put a card under "Needs you" offering no
|
|
5407
|
-
action, which is precisely the defect Phase 1 existed to remove. The panel
|
|
5408
|
-
reads the same `blocked_tasks` view and renders "Waiting on <name>" from it,
|
|
5409
|
-
so the card explains itself without a state that lies. */
|
|
5410
|
-
const skip = [];
|
|
5411
|
-
let blockedCount = 0;
|
|
5412
|
-
for (;;) {
|
|
5413
|
-
let claim;
|
|
5288
|
+
/* Only what CHOOSING a unit needs. The spawn's own inputs (`spawn`,
|
|
5289
|
+
`mcpServer`) are destructured in `dispatchClaim`, where they are used.
|
|
5290
|
+
`resolveContext` is needed in BOTH: here it answers "which repo would this
|
|
5291
|
+
touch" for the collision read, and there it answers "where does the worker
|
|
5292
|
+
start" — deliberately the same resolver, so the two cannot disagree. */
|
|
5293
|
+
const { client, userId, machineId, machineName, agents, resolveContext = (claim) => resolveClaimContext(client, claim, warn), } = deps;
|
|
5294
|
+
/* 18a SLICE 6: THE BACKOFF USED TO BE CHECKED HERE, and returning from this
|
|
5295
|
+
line is precisely what was wrong with it: a failing agent stopped the whole
|
|
5296
|
+
daemon rather than stopping itself. The wait is now per request, on the row,
|
|
5297
|
+
and the claim itself skips a request whose moment has not come. */
|
|
5298
|
+
let role;
|
|
5414
5299
|
try {
|
|
5415
|
-
|
|
5300
|
+
const { data, error } = await client
|
|
5301
|
+
.from('cliv2_orchestrator_preference')
|
|
5302
|
+
.select('machine_id, agent')
|
|
5303
|
+
.eq('user_id', userId)
|
|
5304
|
+
.maybeSingle();
|
|
5305
|
+
if (error)
|
|
5306
|
+
throw error;
|
|
5307
|
+
role = resolveOrchestratorRole(data ?? null, machineId, agents);
|
|
5416
5308
|
}
|
|
5417
5309
|
catch (err) {
|
|
5418
|
-
|
|
5310
|
+
// Do NOT fall through to a claim on a failed read: an unreadable designation
|
|
5311
|
+
// is not a designation, and guessing would be exactly the fleet-wide race
|
|
5312
|
+
// the machine check exists to prevent.
|
|
5313
|
+
warn(`orchestrator designation read failed, will retry: ${err.message}`);
|
|
5419
5314
|
return 'failed';
|
|
5420
5315
|
}
|
|
5421
|
-
|
|
5422
|
-
|
|
5423
|
-
|
|
5424
|
-
|
|
5425
|
-
|
|
5316
|
+
reportRole(state, role, machineName, log);
|
|
5317
|
+
if (role.kind !== 'this-machine')
|
|
5318
|
+
return 'not-designated';
|
|
5319
|
+
/* ═══ 18d — NO TOOLS MEANS NO DISPATCH. ═══
|
|
5320
|
+
*
|
|
5321
|
+
FOUND BY WALKING IT, 2026-08-12. A daemon restart raced its own port: the
|
|
5322
|
+
replacement process came up while the old one still held 4579, so
|
|
5323
|
+
`toolsServerStatus().running` was false, `presence.ts` passed
|
|
5324
|
+
`mcpServer: null`, and the very next tick claimed a request and spawned a
|
|
5325
|
+
worker WITH NO CTRL+SPC TOOLS AT ALL. That worker could not read the work
|
|
5326
|
+
item, could not read its own stage document, could not call `ask_question`
|
|
5327
|
+
and could not `hand_off_stage`. It did the only thing left: it wrote an
|
|
5328
|
+
essay into its final answer explaining that the tools were missing, and the
|
|
5329
|
+
request was burned — `attempts` spent, the user's card finished, and
|
|
5330
|
+
nothing to show for it.
|
|
5331
|
+
*
|
|
5332
|
+
Null is the RIGHT value there: the alternative that comment rejects is
|
|
5333
|
+
letting the worker inherit the user's own agent config, where a second
|
|
5334
|
+
CTRL+SPC server pointed at a DIFFERENT ACCOUNT was once found registered and
|
|
5335
|
+
listening. That is worse. But there was a third option nobody took, which is
|
|
5336
|
+
to not dispatch at all.
|
|
5337
|
+
*
|
|
5338
|
+
WAITING IS FREE AND CORRECT. The request keeps its place, unclaimed and
|
|
5339
|
+
unspent, and the tools server comes up moments later — the port race
|
|
5340
|
+
resolves as soon as the old process lets go. A dispatch made in this window
|
|
5341
|
+
cannot be retried usefully, because it is not the request that is broken.
|
|
5342
|
+
*
|
|
5343
|
+
BEFORE THE CLAIM, deliberately: claiming and then refusing to spawn would
|
|
5344
|
+
spend `attempts` on a condition that has nothing to do with the request.
|
|
5345
|
+
*
|
|
5346
|
+
THE CONDITION IS "A REAL SPAWN WITH NO SERVER", not "no server". An injected
|
|
5347
|
+
`spawn` is a caller who has taken over the launching entirely — every test in
|
|
5348
|
+
the suite, and any future embedder — and for them `mcpServer` describes
|
|
5349
|
+
nothing, because `defaultSpawn` is what turns it into an argv. Gating on the
|
|
5350
|
+
null alone would refuse to dispatch in exactly the cases where the tools are
|
|
5351
|
+
not in question, which is not a stricter check but a wrong one. */
|
|
5352
|
+
if (!deps.mcpServer && !deps.spawn) {
|
|
5353
|
+
reportNoTools(state, warn);
|
|
5354
|
+
return 'no-tools';
|
|
5426
5355
|
}
|
|
5427
|
-
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
|
|
5431
|
-
|
|
5432
|
-
|
|
5433
|
-
|
|
5434
|
-
|
|
5435
|
-
|
|
5436
|
-
|
|
5437
|
-
|
|
5438
|
-
|
|
5439
|
-
|
|
5440
|
-
|
|
5441
|
-
|
|
5442
|
-
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5448
|
-
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
5454
|
-
|
|
5455
|
-
|
|
5456
|
-
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
|
|
5460
|
-
/* A SIBLING ALREADY KNOWS ITS OWN, and asks nothing. `chosen_codebase` comes
|
|
5461
|
-
down on the claim itself (this slice's migration), so a fanned-out run
|
|
5462
|
-
needs no ask to read and cannot fan out again — which is what stops the
|
|
5463
|
-
fan-out being infinite. Only a request that was ASKED reads the answer. */
|
|
5464
|
-
const chosen = blocker
|
|
5465
|
-
? []
|
|
5466
|
-
: claim.chosenCodebase
|
|
5467
|
-
? [claim.chosenCodebase]
|
|
5468
|
-
: await readChosenCodebases(client, claim.id, warn);
|
|
5469
|
-
const runClaim = chosen.length > 0
|
|
5470
|
-
? { ...claim, chosenCodebase: chosen[0] }
|
|
5471
|
-
: claim;
|
|
5472
|
-
const context = blocker ? null : await resolveContext(runClaim);
|
|
5473
|
-
const collision = context
|
|
5474
|
-
? await collisionForTodo(client, claim.id, context.gitRemoteUrl, warn)
|
|
5475
|
-
: null;
|
|
5476
|
-
if (!blocker && !collision) {
|
|
5477
|
-
/* NO LONGER WAITING, so the next time it does wait it is reported again.
|
|
5478
|
-
Without this, a unit that blocked, ran, and blocked again on something
|
|
5479
|
-
else would stay silent forever. */
|
|
5480
|
-
state.reportedWaiting.delete(claim.id);
|
|
5481
|
-
/* THE RESOLVED CONTEXT IS HANDED ON rather than resolved again. It reads
|
|
5482
|
-
several tables and touches the filesystem, and the dispatch needs the
|
|
5483
|
-
very same answer — resolving twice would both cost a second pass and
|
|
5484
|
-
allow the collision check and the working directory to disagree. */
|
|
5485
|
-
/* THE SIBLINGS ARE CREATED BEFORE THIS RUN DISPATCHES, so a user who chose
|
|
5486
|
-
three codebases sees three cards immediately rather than one that
|
|
5487
|
-
mysteriously multiplies later. Best-effort: a failed sibling costs that
|
|
5488
|
-
codebase its run, and is logged; it must not cost the user the run that
|
|
5489
|
-
is about to start. */
|
|
5490
|
-
if (chosen.length > 1) {
|
|
5491
|
-
await spawnSiblingRuns(client, claim, chosen.slice(1), log, warn);
|
|
5356
|
+
state.warnedNoTools = false;
|
|
5357
|
+
/* ═══ !Cleanup PHASE 7 SLICE 2: CLAIM PAST THE BLOCKED ONES. ═══
|
|
5358
|
+
*
|
|
5359
|
+
A dependent unit must WAIT, and everything behind it must NOT. Those two
|
|
5360
|
+
requirements together are why this is a loop rather than a single claim.
|
|
5361
|
+
*
|
|
5362
|
+
The claim is this design's only queue position, and it takes the OLDEST
|
|
5363
|
+
claimable row. So a blocked unit that is merely released is re-claimed FIRST
|
|
5364
|
+
on the very next tick, forever — it stops being a waiting unit and becomes a
|
|
5365
|
+
queue stopper. Releasing it AND telling the next claim to skip it is what
|
|
5366
|
+
makes the rest of the queue reachable.
|
|
5367
|
+
*
|
|
5368
|
+
BOUNDED BY THE NUMBER OF UNITS SKIPPED, not by a fixed count: each iteration
|
|
5369
|
+
either dispatches (returns), finds nothing (returns), or adds exactly one id
|
|
5370
|
+
to `skip`, and a skipped id can never be claimed again this tick. So the
|
|
5371
|
+
loop runs at most once per blocked unit and then ends. No timeout needed,
|
|
5372
|
+
because it cannot revisit a row.
|
|
5373
|
+
*
|
|
5374
|
+
THE UNIT STAYS `working` AND UNCLAIMED, deliberately — it is NOT parked to
|
|
5375
|
+
`needs-input` the way the scope gate parks. `needs-input` means "this needs
|
|
5376
|
+
YOU", and a dependency needs nothing from the user: it resolves when the
|
|
5377
|
+
blocker finishes. Parking it would put a card under "Needs you" offering no
|
|
5378
|
+
action, which is precisely the defect Phase 1 existed to remove. The panel
|
|
5379
|
+
reads the same `blocked_tasks` view and renders "Waiting on <name>" from it,
|
|
5380
|
+
so the card explains itself without a state that lies. */
|
|
5381
|
+
const skip = [];
|
|
5382
|
+
let blockedCount = 0;
|
|
5383
|
+
for (;;) {
|
|
5384
|
+
let claim;
|
|
5385
|
+
try {
|
|
5386
|
+
claim = await claimNextTodo(client, machineId, role.agent, [...skip, ...legacyHeldTodoIds()]);
|
|
5387
|
+
if (claim?.claimOperationId)
|
|
5388
|
+
claimOperations.push(claim.claimOperationId);
|
|
5492
5389
|
}
|
|
5493
|
-
|
|
5494
|
-
|
|
5495
|
-
|
|
5496
|
-
|
|
5497
|
-
|
|
5498
|
-
|
|
5499
|
-
|
|
5500
|
-
|
|
5501
|
-
|
|
5502
|
-
|
|
5503
|
-
|
|
5504
|
-
|
|
5505
|
-
|
|
5506
|
-
|
|
5507
|
-
|
|
5508
|
-
|
|
5390
|
+
catch (err) {
|
|
5391
|
+
warn(`orchestrator poll failed, will retry: ${err.message}`);
|
|
5392
|
+
return 'failed';
|
|
5393
|
+
}
|
|
5394
|
+
if (!claim) {
|
|
5395
|
+
/* Nothing claimable. If we skipped something, the queue is not empty —
|
|
5396
|
+
it is WAITING, which is a different fact and must not be logged as
|
|
5397
|
+
idleness. */
|
|
5398
|
+
return blockedCount > 0 ? 'blocked' : 'nothing-to-do';
|
|
5399
|
+
}
|
|
5400
|
+
/* ═══ THE TWO REASONS WORK WAITS, ASKED IN ORDER. ═══
|
|
5401
|
+
*
|
|
5402
|
+
A DEPENDENCY IS ASKED FIRST, and the order is meaningful rather than
|
|
5403
|
+
incidental: a dependency is a FACT about what must happen before what,
|
|
5404
|
+
while a collision is a CHOICE to serialise safely. When both are true the
|
|
5405
|
+
user is better told the one that will not change on its own — a collision
|
|
5406
|
+
clears the moment the other agent releases, a dependency does not clear
|
|
5407
|
+
until the blocker is finished.
|
|
5408
|
+
*
|
|
5409
|
+
Slice 3 adds the second. Both hold the SAME WAY (release, skip, report
|
|
5410
|
+
once), so the shared tail below is written once rather than twice. */
|
|
5411
|
+
const blocker = await blockerForTodo(client, claim.id, warn);
|
|
5412
|
+
/* THE CODEBASE COMES FROM `resolveContext`, the SAME resolver the dispatch
|
|
5413
|
+
uses to choose the worker's working directory — not a second lookup that
|
|
5414
|
+
could disagree with it. A reservation is scoped by `git_remote_url`, so
|
|
5415
|
+
"which repo would this run touch" and "which repo will it start in" have
|
|
5416
|
+
to be one answer. Only reached when there is no dependency, so an ordinary
|
|
5417
|
+
dispatchable request costs one extra read, not two. */
|
|
5418
|
+
/* ═══ 18b SLICE 4 — THE CODEBASE THE USER CHOSE, AND THE FAN-OUT. ═══
|
|
5419
|
+
*
|
|
5420
|
+
Read BEFORE the context is resolved, because it is an input to it: given a
|
|
5421
|
+
choice, the resolver treats the project as if that one codebase were its
|
|
5422
|
+
only one, and the ambiguity never arises.
|
|
5423
|
+
*
|
|
5424
|
+
MORE THAN ONE CHOSEN MEANS MORE THAN ONE RUN (user ruling, 2026-08-07).
|
|
5425
|
+
This tick dispatches the FIRST; `spawnSiblingRuns` below creates a sibling
|
|
5426
|
+
request for each of the rest, which the daemon then claims on its own
|
|
5427
|
+
ticks exactly like any other request. Each run carries ONE codebase and
|
|
5428
|
+
starts in that codebase's own checkout, wherever it is on disk — nothing
|
|
5429
|
+
here reads folder layout, which is the whole of the location ruling.
|
|
5430
|
+
*
|
|
5431
|
+
Nothing changes for a request that was never asked: `chosen` is empty, the
|
|
5432
|
+
claim carries null, and every path below behaves as it did in Slice 2. */
|
|
5433
|
+
/* A SIBLING ALREADY KNOWS ITS OWN, and asks nothing. `chosen_codebase` comes
|
|
5434
|
+
down on the claim itself (this slice's migration), so a fanned-out run
|
|
5435
|
+
needs no ask to read and cannot fan out again — which is what stops the
|
|
5436
|
+
fan-out being infinite. Only a request that was ASKED reads the answer. */
|
|
5437
|
+
const chosen = blocker
|
|
5438
|
+
? []
|
|
5439
|
+
: claim.chosenCodebase
|
|
5440
|
+
? [claim.chosenCodebase]
|
|
5441
|
+
: await readChosenCodebases(client, claim.id, warn);
|
|
5442
|
+
const runClaim = chosen.length > 0
|
|
5443
|
+
? { ...claim, chosenCodebase: chosen[0] }
|
|
5444
|
+
: claim;
|
|
5445
|
+
const context = blocker ? null : await resolveContext(runClaim);
|
|
5446
|
+
const collision = context
|
|
5447
|
+
? await collisionForTodo(client, claim.id, context.gitRemoteUrl, warn)
|
|
5448
|
+
: null;
|
|
5449
|
+
if (!blocker && !collision) {
|
|
5450
|
+
/* NO LONGER WAITING, so the next time it does wait it is reported again.
|
|
5451
|
+
Without this, a unit that blocked, ran, and blocked again on something
|
|
5452
|
+
else would stay silent forever. */
|
|
5453
|
+
state.reportedWaiting.delete(claim.id);
|
|
5454
|
+
/* THE RESOLVED CONTEXT IS HANDED ON rather than resolved again. It reads
|
|
5455
|
+
several tables and touches the filesystem, and the dispatch needs the
|
|
5456
|
+
very same answer — resolving twice would both cost a second pass and
|
|
5457
|
+
allow the collision check and the working directory to disagree. */
|
|
5458
|
+
/* THE SIBLINGS ARE CREATED BEFORE THIS RUN DISPATCHES, so a user who chose
|
|
5459
|
+
three codebases sees three cards immediately rather than one that
|
|
5460
|
+
mysteriously multiplies later. Best-effort: a failed sibling costs that
|
|
5461
|
+
codebase its run, and is logged; it must not cost the user the run that
|
|
5462
|
+
is about to start. */
|
|
5463
|
+
if (chosen.length > 1) {
|
|
5464
|
+
await spawnSiblingRuns(client, claim, chosen.slice(1), log, warn);
|
|
5465
|
+
}
|
|
5466
|
+
const dispatched = dispatchClaim(deps, state, now, log, warn, role, runClaim, context, chosen);
|
|
5467
|
+
if (hasOwnedWorkContext())
|
|
5468
|
+
finishRpcClaims(claimOperations);
|
|
5469
|
+
endClaim();
|
|
5470
|
+
return await dispatched;
|
|
5471
|
+
}
|
|
5472
|
+
/* SAID ONCE PER (unit, REASON) — see `reportedWaiting`. The tick runs every
|
|
5473
|
+
3s and this branch is reached on every one of them for as long as the wait
|
|
5474
|
+
holds. The key carries the reason, not just the unit, so a request that
|
|
5475
|
+
stops waiting on a dependency and starts waiting on a FILE says so. */
|
|
5476
|
+
const waitKey = blocker
|
|
5477
|
+
? `dep:${blocker.dependsOnId}`
|
|
5478
|
+
: `path:${collision.heldBySessionId}:${collision.path}`;
|
|
5479
|
+
const waitLine = blocker
|
|
5480
|
+
? `is waiting on "${blocker.dependsOnName}"`
|
|
5481
|
+
: `is waiting on the same files as "${collision.heldByInstruction ?? 'another run'}"`
|
|
5482
|
+
+ ` (${collision.path}${collision.pathCount > 1 ? ` and ${collision.pathCount - 1} more` : ''})`;
|
|
5483
|
+
if (state.reportedWaiting.get(claim.id) !== waitKey) {
|
|
5484
|
+
state.reportedWaiting.set(claim.id, waitKey);
|
|
5485
|
+
log(`[orchestrator] "${claim.instruction}" (${shortId(claim.id)}) ${waitLine} — not dispatching`);
|
|
5486
|
+
}
|
|
5487
|
+
/* RELEASED, so that the moment the blocker is done the ordinary claim picks
|
|
5488
|
+
it up with no user action and no special path. That is the Gherkin line
|
|
5489
|
+
"123.1 starts without my intervention", and it is why this must not keep
|
|
5490
|
+
the claim the way `parkTodoForScope` does. */
|
|
5491
|
+
await releaseTodo(client, claim.id, claim.claimedAt).catch((err) => {
|
|
5492
|
+
warn(`could not release a blocked todo: ${err.message}`);
|
|
5493
|
+
});
|
|
5494
|
+
skip.push(claim.id);
|
|
5495
|
+
blockedCount += 1;
|
|
5509
5496
|
}
|
|
5510
|
-
|
|
5511
|
-
|
|
5512
|
-
|
|
5513
|
-
|
|
5514
|
-
|
|
5515
|
-
warn(`could not release a blocked todo: ${err.message}`);
|
|
5516
|
-
});
|
|
5517
|
-
skip.push(claim.id);
|
|
5518
|
-
blockedCount += 1;
|
|
5497
|
+
}
|
|
5498
|
+
finally {
|
|
5499
|
+
if (hasOwnedWorkContext())
|
|
5500
|
+
finishRpcClaims(claimOperations);
|
|
5501
|
+
endClaim();
|
|
5519
5502
|
}
|
|
5520
5503
|
}
|
|
5521
5504
|
/**
|
|
@@ -5549,10 +5532,14 @@ async function runTick(deps, state, now, log, warn) {
|
|
|
5549
5532
|
* returns is not a funnel.
|
|
5550
5533
|
*/
|
|
5551
5534
|
async function dispatchClaim(deps, state, now, log, warn, role, claim, context, chosenCodebases = []) {
|
|
5535
|
+
const owned = hasOwnedWorkContext() ? reserveOwnedWork({ surface: 'worker', todoId: claim.id, workerId: null, claimedAt: claim.claimedAt, observedPendingMessageIds: null }, role.agent) : undefined;
|
|
5552
5536
|
try {
|
|
5553
|
-
|
|
5537
|
+
const outcome = await dispatchClaimInner(deps, state, now, log, warn, role, claim, context, chosenCodebases, owned);
|
|
5538
|
+
owned?.complete();
|
|
5539
|
+
return outcome;
|
|
5554
5540
|
}
|
|
5555
5541
|
finally {
|
|
5542
|
+
owned?.finishPreparation();
|
|
5556
5543
|
/* THE RUN IS OVER, SO IT HOLDS NOTHING, however it ended and wherever it
|
|
5557
5544
|
returned from. Best-effort by contract: `releaseTodoReservations` never
|
|
5558
5545
|
throws, so this cannot change what the body returned, and a `finally`
|
|
@@ -5577,7 +5564,7 @@ context,
|
|
|
5577
5564
|
/** 18b Slice 4 — every codebase the user chose, including this run's own. The
|
|
5578
5565
|
* siblings are derived from it below; empty on every request that was never
|
|
5579
5566
|
* asked, which is the ordinary case. */
|
|
5580
|
-
chosenCodebases = []) {
|
|
5567
|
+
chosenCodebases = [], owned) {
|
|
5581
5568
|
const { client, machineId, mcpServer = null, spawn = defaultSpawn(mcpServer), } = deps;
|
|
5582
5569
|
/* ═══ 16g SLICE 1 — THE SECOND ENFORCEMENT POINT. ═══
|
|
5583
5570
|
`No approved scope-version pointer → no execution write and no coding lane.`
|
|
@@ -5638,7 +5625,7 @@ chosenCodebases = []) {
|
|
|
5638
5625
|
uses the mirror of this: answering an ask CLEARS the claim to make the
|
|
5639
5626
|
work dispatchable again — so approving scope re-dispatches through the
|
|
5640
5627
|
path that already exists, with no new mechanism. */
|
|
5641
|
-
await parkTodoForScope(client, claim.id).catch((err) => {
|
|
5628
|
+
await parkTodoForScope(client, claim.id, claim.claimedAt).catch((err) => {
|
|
5642
5629
|
warn(`could not park an unapproved todo: ${err.message}`);
|
|
5643
5630
|
});
|
|
5644
5631
|
return 'nothing-to-do';
|
|
@@ -5665,7 +5652,7 @@ chosenCodebases = []) {
|
|
|
5665
5652
|
hold a dead end too. Building stays held exactly as shipped. */
|
|
5666
5653
|
if (targeted && building && await todoIsHeldForScope(client, claim.id)) {
|
|
5667
5654
|
log(`[orchestrator] "${claim.instruction}" has an unresolved scope change — not dispatching`);
|
|
5668
|
-
await parkTodoForScope(client, claim.id).catch((err) => {
|
|
5655
|
+
await parkTodoForScope(client, claim.id, claim.claimedAt).catch((err) => {
|
|
5669
5656
|
warn(`could not park a held todo: ${err.message}`);
|
|
5670
5657
|
});
|
|
5671
5658
|
return 'nothing-to-do';
|
|
@@ -5801,7 +5788,7 @@ chosenCodebases = []) {
|
|
|
5801
5788
|
with the words it had in Slice 2: never parked with no question on it,
|
|
5802
5789
|
which the user could only clear with Stop. */
|
|
5803
5790
|
if (context.miss?.kind === 'ambiguous-codebase' && instructionNeedsCheckout(claim.instruction)) {
|
|
5804
|
-
const asked = await askWhichCodebase(client, claim.id, context.miss.projectName, context.miss.gitRemoteUrls, warn);
|
|
5791
|
+
const asked = await askWhichCodebase(client, claim.id, context.miss.projectName, context.miss.gitRemoteUrls, warn, claim.claimedAt);
|
|
5805
5792
|
if (asked) {
|
|
5806
5793
|
log(`[orchestrator] asking which codebase "${claim.instruction}" should run in`);
|
|
5807
5794
|
await updateWorker(client, workerId, 'died', 'waiting for the user to choose a codebase', warn);
|
|
@@ -5811,7 +5798,7 @@ chosenCodebases = []) {
|
|
|
5811
5798
|
if (context.miss && instructionNeedsCheckout(claim.instruction)) {
|
|
5812
5799
|
log(`[orchestrator] "${claim.instruction}" needs the code, which is not on this machine`);
|
|
5813
5800
|
await updateWorker(client, workerId, 'died', 'the codebase is not available on this machine', warn);
|
|
5814
|
-
await answerTodo(client, claim.id, checkoutMissAnswer(context.miss));
|
|
5801
|
+
await answerTodo(client, claim.id, checkoutMissAnswer(context.miss), claim.claimedAt);
|
|
5815
5802
|
return 'answered';
|
|
5816
5803
|
}
|
|
5817
5804
|
/* ═══ 18d SLICE 4 — ONE STAGE PER DISPATCH (user ruling, 2026-08-12). ═══
|
|
@@ -5823,6 +5810,15 @@ chosenCodebases = []) {
|
|
|
5823
5810
|
if (stage) {
|
|
5824
5811
|
log(`[orchestrator] ${shortId(claim.id)} is working stage ${stage.position}/${stage.total}: ${stage.title}`);
|
|
5825
5812
|
}
|
|
5813
|
+
if (owned) {
|
|
5814
|
+
if (!claim.claimedAt || !workerId)
|
|
5815
|
+
throw new Error('The worker has no exact attempt identity.');
|
|
5816
|
+
const pending = await client.from('cliv2_run_messages').select('id').eq('todo_id', claim.id).is('answered_by', null);
|
|
5817
|
+
if (pending.error)
|
|
5818
|
+
throw pending.error;
|
|
5819
|
+
owned.setReference({ surface: 'worker', todoId: claim.id, workerId, claimedAt: claim.claimedAt,
|
|
5820
|
+
observedPendingMessageIds: (pending.data ?? []).map((row) => row.id) });
|
|
5821
|
+
}
|
|
5826
5822
|
const result = await spawn(role.agent, buildPrompt(claim.instruction, claim.id, answered, targeted && scopeVersion ? { workItemId: targeted, version: scopeVersion } : null, replies, context, planningOnly, interrupted, selectedTools, stage, directed), (step) => steps.offer(step), context.checkout,
|
|
5827
5823
|
// !Cleanup Phase 6 (I17) — polled by the child; true kills it.
|
|
5828
5824
|
() => todoWasStopped(client, claim.id),
|
|
@@ -5850,15 +5846,17 @@ chosenCodebases = []) {
|
|
|
5850
5846
|
not the prompt, so report_activity attributes without depending on
|
|
5851
5847
|
agent cooperation (ux.md § Corrections: codex dropped a prompt-carried
|
|
5852
5848
|
id twice). */
|
|
5853
|
-
claim.id);
|
|
5849
|
+
claim.id, owned);
|
|
5854
5850
|
/* !Cleanup PHASE 6 (I17) — THE USER STOPPED IT, and that is not a failure.
|
|
5855
5851
|
Checked BEFORE `!result.ok`, because a killed child also reports not-ok:
|
|
5856
5852
|
falling through would count the user's own decision as an attempt and
|
|
5857
5853
|
retry the very thing they cancelled. Nothing is written but the claim
|
|
5858
5854
|
release — the browser already set `stopped`. */
|
|
5855
|
+
if (result.interrupted)
|
|
5856
|
+
return 'nothing-to-do';
|
|
5859
5857
|
if (result.stopped) {
|
|
5860
5858
|
await updateWorker(client, workerId, 'died', 'stopped by the user', warn);
|
|
5861
|
-
await releaseStoppedTodo(client, claim.id);
|
|
5859
|
+
await releaseStoppedTodo(client, claim.id, claim.claimedAt);
|
|
5862
5860
|
log(`[orchestrator] ${shortId(claim.id)} was stopped by the user`);
|
|
5863
5861
|
return 'released';
|
|
5864
5862
|
}
|
|
@@ -5878,6 +5876,8 @@ chosenCodebases = []) {
|
|
|
5878
5876
|
`failed` with the reason instead, which is what stops the unit being
|
|
5879
5877
|
re-claimed forever while reading as Working. */
|
|
5880
5878
|
const { fate, attempts } = await releaseOrFail(client, claim.id, plain, now().getTime(), claim.claimedAt);
|
|
5879
|
+
if (fate === 'stale')
|
|
5880
|
+
return 'nothing-to-do';
|
|
5881
5881
|
if (fate === 'failed') {
|
|
5882
5882
|
warn(`[orchestrator] ${result.error} — ${shortId(claim.id)} failed after ${MAX_ATTEMPTS} attempts`);
|
|
5883
5883
|
return 'failed';
|
|
@@ -5940,6 +5940,8 @@ chosenCodebases = []) {
|
|
|
5940
5940
|
action to hand on and nothing for the next agent to resume from. */
|
|
5941
5941
|
await updateWorker(client, workerId, 'died', NOTHING_RECORDED_REASON, warn);
|
|
5942
5942
|
const { fate, attempts } = await releaseOrFail(client, claim.id, NOTHING_RECORDED_REASON, now().getTime(), claim.claimedAt);
|
|
5943
|
+
if (fate === 'stale')
|
|
5944
|
+
return 'nothing-to-do';
|
|
5943
5945
|
if (fate === 'failed') {
|
|
5944
5946
|
warn(`[orchestrator] ${shortId(claim.id)} recorded nothing and failed after ${MAX_ATTEMPTS} attempts`);
|
|
5945
5947
|
return 'failed';
|
|
@@ -5979,6 +5981,8 @@ chosenCodebases = []) {
|
|
|
5979
5981
|
repeatedly by a durable cause — a reverted migration, an RLS change —
|
|
5980
5982
|
and before the bound it was the same forever-loop as a failed spawn. */
|
|
5981
5983
|
const { fate, attempts } = await releaseOrFail(client, claim.id, err.message, now().getTime(), claim.claimedAt);
|
|
5984
|
+
if (fate === 'stale')
|
|
5985
|
+
return 'nothing-to-do';
|
|
5982
5986
|
warn(fate === 'failed'
|
|
5983
5987
|
? `[orchestrator] ${shortId(claim.id)} failed after ${MAX_ATTEMPTS} attempts`
|
|
5984
5988
|
: `[orchestrator] retrying ${shortId(claim.id)} in ${Math.round(retryDelayMs(attempts) / 1000)}s`);
|
|
@@ -6009,3 +6013,191 @@ chosenCodebases = []) {
|
|
|
6009
6013
|
export function liveAgents() {
|
|
6010
6014
|
return detectAgents();
|
|
6011
6015
|
}
|
|
6016
|
+
function cliClaimReferences(action, result) {
|
|
6017
|
+
if (!Array.isArray(result))
|
|
6018
|
+
throw new Error('The claim journal returned no complete result.');
|
|
6019
|
+
return result.map((row) => {
|
|
6020
|
+
if (action === 'cliv2_claim_next_loose_todo') {
|
|
6021
|
+
const attempt = row?._attempt;
|
|
6022
|
+
if (!attempt || typeof attempt.todo_id !== 'string' || typeof attempt.claimed_at !== 'string')
|
|
6023
|
+
throw new Error('The claim journal returned an incomplete request identity.');
|
|
6024
|
+
return { surface: 'worker', todoId: attempt.todo_id, workerId: null, claimedAt: attempt.claimed_at, observedPendingMessageIds: null };
|
|
6025
|
+
}
|
|
6026
|
+
if (typeof row?.id !== 'string' || typeof row?.todo_id !== 'string')
|
|
6027
|
+
throw new Error('The claim journal returned an incomplete reply source.');
|
|
6028
|
+
if (action === 'cliv2_take_run_messages' && typeof row.taken_at === 'string')
|
|
6029
|
+
return {
|
|
6030
|
+
surface: 'reply', todoId: row.todo_id, messageLeases: [{ id: row.id, taken_at: row.taken_at }], grantLeases: [], observedPendingMessageIds: null,
|
|
6031
|
+
};
|
|
6032
|
+
if (action === 'cliv2_take_granted_resumes' && typeof row.resume_taken_at === 'string')
|
|
6033
|
+
return {
|
|
6034
|
+
surface: 'reply', todoId: row.todo_id, messageLeases: [], grantLeases: [{ id: row.id, resume_taken_at: row.resume_taken_at }], observedPendingMessageIds: null,
|
|
6035
|
+
};
|
|
6036
|
+
throw new Error('The claim journal returned an unrecognized lease identity.');
|
|
6037
|
+
});
|
|
6038
|
+
}
|
|
6039
|
+
async function cliClaim(client, action, args) {
|
|
6040
|
+
if (!hasOwnedWorkContext())
|
|
6041
|
+
return { ...await client.rpc(action, args), operationId: undefined };
|
|
6042
|
+
const operationId = beginRpcClaim('cli', action);
|
|
6043
|
+
try {
|
|
6044
|
+
const response = await client.rpc(action, { ...args, p_operation_id: operationId });
|
|
6045
|
+
if (!response.error) {
|
|
6046
|
+
const journal = await client.rpc('cliv2_reconcile_claim', { p_machine_id: args.p_machine_id, p_operation_id: operationId });
|
|
6047
|
+
if (journal.error)
|
|
6048
|
+
throw journal.error;
|
|
6049
|
+
const row = journal.data?.[0];
|
|
6050
|
+
if (row?.outcome !== 'completed' || row.action !== action)
|
|
6051
|
+
throw new Error('The claim was cancelled before it could start.');
|
|
6052
|
+
recordRpcClaims(operationId, cliClaimReferences(action, row.result));
|
|
6053
|
+
}
|
|
6054
|
+
if (response.error)
|
|
6055
|
+
deferRpcClaim(operationId);
|
|
6056
|
+
return { ...response, operationId };
|
|
6057
|
+
}
|
|
6058
|
+
catch (error) {
|
|
6059
|
+
deferRpcClaim(operationId);
|
|
6060
|
+
throw error;
|
|
6061
|
+
}
|
|
6062
|
+
}
|
|
6063
|
+
/** Read every legacy source before committing a baseline; a failed page keeps admission closed. */
|
|
6064
|
+
async function legacyRows(client, table, columns, todoId) {
|
|
6065
|
+
const rows = [];
|
|
6066
|
+
let after = null;
|
|
6067
|
+
for (;;) {
|
|
6068
|
+
let query = client.from(table).select(columns).order('id').limit(500);
|
|
6069
|
+
if (todoId)
|
|
6070
|
+
query = query.eq('todo_id', todoId);
|
|
6071
|
+
if (after)
|
|
6072
|
+
query = query.gt('id', after);
|
|
6073
|
+
const { data, error } = await query;
|
|
6074
|
+
if (error)
|
|
6075
|
+
throw error;
|
|
6076
|
+
if (!Array.isArray(data))
|
|
6077
|
+
throw new Error('The old work snapshot did not return a complete page.');
|
|
6078
|
+
rows.push(...data);
|
|
6079
|
+
if (data.length < 500)
|
|
6080
|
+
return rows;
|
|
6081
|
+
after = data[data.length - 1].id;
|
|
6082
|
+
if (typeof after !== 'string')
|
|
6083
|
+
throw new Error('The old work snapshot has no continuation identity.');
|
|
6084
|
+
}
|
|
6085
|
+
}
|
|
6086
|
+
export async function reconcileInterruptedWork(client, machineId) {
|
|
6087
|
+
if (!hasOwnedWorkContext())
|
|
6088
|
+
return;
|
|
6089
|
+
const capability = await client.rpc('cliv2_interrupt_machine_work', { p_machine_id: machineId,
|
|
6090
|
+
p_operation_id: randomUUID(), p_interrupted_at: new Date().toISOString(), p_workers: [], p_replies: [] });
|
|
6091
|
+
if (capability.error)
|
|
6092
|
+
throw capability.error;
|
|
6093
|
+
for (const claim of pendingRpcClaims('cli')) {
|
|
6094
|
+
const journal = await client.rpc('cliv2_reconcile_claim', { p_machine_id: machineId, p_operation_id: claim.id });
|
|
6095
|
+
if (journal.error)
|
|
6096
|
+
throw journal.error;
|
|
6097
|
+
const row = journal.data?.[0];
|
|
6098
|
+
if (row?.outcome === 'cancelled')
|
|
6099
|
+
recordRpcClaims(claim.id, [], true);
|
|
6100
|
+
else if (row?.outcome === 'completed' && row.action === claim.action)
|
|
6101
|
+
recordRpcClaims(claim.id, cliClaimReferences(claim.action, row.result), true);
|
|
6102
|
+
else
|
|
6103
|
+
throw new Error('The pending claim could not be reconciled.');
|
|
6104
|
+
}
|
|
6105
|
+
if (legacySnapshotNeeded()) {
|
|
6106
|
+
const todos = await legacyRows(client, 'cliv2_loose_todos', 'id,claimed_at,claimed_machine_id');
|
|
6107
|
+
const workers = await legacyRows(client, 'cliv2_workers', 'id,todo_id,machine_id,state');
|
|
6108
|
+
const messages = await legacyRows(client, 'cliv2_run_messages', 'id,todo_id,taken_at,taken_machine_id,answered_by');
|
|
6109
|
+
const permissions = await legacyRows(client, 'cliv2_permission_requests', 'id,todo_id,state,consumed_at,resume_taken_at,resume_machine_id,resume_answered_by');
|
|
6110
|
+
const holds = new Map();
|
|
6111
|
+
const hold = (todoId) => {
|
|
6112
|
+
let value = holds.get(todoId);
|
|
6113
|
+
if (!value) {
|
|
6114
|
+
value = { todoId, messageIds: [], messageSources: [], permissionSources: [] };
|
|
6115
|
+
holds.set(todoId, value);
|
|
6116
|
+
}
|
|
6117
|
+
return value;
|
|
6118
|
+
};
|
|
6119
|
+
for (const message of messages) {
|
|
6120
|
+
if (message.answered_by != null || message.taken_machine_id && message.taken_machine_id !== machineId)
|
|
6121
|
+
continue;
|
|
6122
|
+
const pending = hold(message.todo_id);
|
|
6123
|
+
pending.messageIds.push(message.id);
|
|
6124
|
+
if (message.taken_at)
|
|
6125
|
+
pending.messageSources.push({ id: message.id, taken_at: message.taken_at });
|
|
6126
|
+
if (message.taken_machine_id === machineId && message.taken_at)
|
|
6127
|
+
recordExitedInterruption({
|
|
6128
|
+
surface: 'reply', todoId: message.todo_id, messageLeases: [{ id: message.id, taken_at: message.taken_at }],
|
|
6129
|
+
grantLeases: [], observedPendingMessageIds: null,
|
|
6130
|
+
});
|
|
6131
|
+
}
|
|
6132
|
+
for (const permission of permissions) {
|
|
6133
|
+
if (!permission.todo_id || permission.state !== 'granted' || permission.resume_answered_by != null
|
|
6134
|
+
|| permission.resume_machine_id && permission.resume_machine_id !== machineId)
|
|
6135
|
+
continue;
|
|
6136
|
+
hold(permission.todo_id).permissionSources.push({ id: permission.id, state: permission.state, consumed_at: permission.consumed_at });
|
|
6137
|
+
if (permission.resume_machine_id === machineId && permission.resume_taken_at)
|
|
6138
|
+
recordExitedInterruption({
|
|
6139
|
+
surface: 'reply', todoId: permission.todo_id, messageLeases: [],
|
|
6140
|
+
grantLeases: [{ id: permission.id, resume_taken_at: permission.resume_taken_at }], observedPendingMessageIds: null,
|
|
6141
|
+
});
|
|
6142
|
+
}
|
|
6143
|
+
for (const worker of workers) {
|
|
6144
|
+
if (worker.machine_id !== machineId || !['working', 'dispatching'].includes(worker.state))
|
|
6145
|
+
continue;
|
|
6146
|
+
const todo = todos.find((row) => row.id === worker.todo_id);
|
|
6147
|
+
if (!todo || todo.claimed_machine_id !== machineId || !todo.claimed_at)
|
|
6148
|
+
continue;
|
|
6149
|
+
recordExitedInterruption({ surface: 'worker', todoId: todo.id, workerId: worker.id,
|
|
6150
|
+
claimedAt: todo.claimed_at, observedPendingMessageIds: null });
|
|
6151
|
+
}
|
|
6152
|
+
for (const baseline of holds.values())
|
|
6153
|
+
baseline.messageIds = messages.filter((row) => row.todo_id === baseline.todoId).map((row) => row.id);
|
|
6154
|
+
saveLegacySnapshot([...holds.values()]);
|
|
6155
|
+
}
|
|
6156
|
+
for (const surface of ['worker', 'reply']) {
|
|
6157
|
+
for (const receipt of interruptionReceipts(surface)) {
|
|
6158
|
+
const ref = receipt.reference;
|
|
6159
|
+
const workers = ref.surface === 'worker' ? [{ worker_id: ref.workerId, todo_id: ref.todoId,
|
|
6160
|
+
claimed_at: ref.claimedAt, observed_pending_message_ids: ref.observedPendingMessageIds }] : [];
|
|
6161
|
+
const replies = ref.surface === 'reply' ? [{ todo_id: ref.todoId, message_leases: ref.messageLeases,
|
|
6162
|
+
grant_leases: ref.grantLeases, observed_pending_message_ids: ref.observedPendingMessageIds }] : [];
|
|
6163
|
+
const { data, error } = await client.rpc('cliv2_interrupt_machine_work', {
|
|
6164
|
+
p_machine_id: machineId, p_operation_id: receipt.operationId, p_interrupted_at: receipt.interruptedAt,
|
|
6165
|
+
p_workers: workers, p_replies: replies,
|
|
6166
|
+
});
|
|
6167
|
+
if (error)
|
|
6168
|
+
throw error;
|
|
6169
|
+
const expected = ref.surface === 'worker' ? [{ kind: ref.workerId ? 'worker' : 'claim', id: ref.workerId ?? ref.todoId }] : ref.surface === 'reply'
|
|
6170
|
+
? [...ref.messageLeases.map((source) => ({ kind: 'message', id: source.id })), ...ref.grantLeases.map((source) => ({ kind: 'grant', id: source.id }))] : [];
|
|
6171
|
+
const results = data;
|
|
6172
|
+
if (!Array.isArray(results) || expected.some((expectedRow) => !results.some((row) => row.id === expectedRow.id && row.kind === expectedRow.kind
|
|
6173
|
+
&& ['interrupted', 'already_interrupted', 'already_settled', 'continued', 'stale'].includes(row.outcome)))) {
|
|
6174
|
+
throw new Error('Interrupted work was not fully acknowledged. Work remains protected.');
|
|
6175
|
+
}
|
|
6176
|
+
acknowledgeInterruption(receipt.id);
|
|
6177
|
+
}
|
|
6178
|
+
}
|
|
6179
|
+
await refreshLegacyTodoHolds(client, machineId);
|
|
6180
|
+
}
|
|
6181
|
+
export async function refreshLegacyTodoHolds(client, machineId) {
|
|
6182
|
+
for (const hold of legacyTodoHolds()) {
|
|
6183
|
+
const todo = await client.from('cliv2_loose_todos').select('id,claimed_machine_id').eq('id', hold.todoId).maybeSingle();
|
|
6184
|
+
if (todo.error)
|
|
6185
|
+
throw todo.error;
|
|
6186
|
+
if (!todo.data) {
|
|
6187
|
+
releaseLegacyTodoHold(hold.todoId);
|
|
6188
|
+
continue;
|
|
6189
|
+
}
|
|
6190
|
+
const messages = { data: await legacyRows(client, 'cliv2_run_messages', 'id,answered_by,taken_at,taken_machine_id', hold.todoId) };
|
|
6191
|
+
if (messages.data.some((row) => !hold.messageIds.includes(row.id))) {
|
|
6192
|
+
releaseLegacyTodoHold(hold.todoId);
|
|
6193
|
+
continue;
|
|
6194
|
+
}
|
|
6195
|
+
const permissions = { data: await legacyRows(client, 'cliv2_permission_requests', 'id,state,consumed_at,resume_answered_by,resume_machine_id', hold.todoId) };
|
|
6196
|
+
const messagesRemain = messages.data.some((row) => hold.messageIds.includes(row.id) && row.answered_by === null
|
|
6197
|
+
&& (!row.taken_machine_id || row.taken_machine_id === machineId));
|
|
6198
|
+
const grantsRemain = permissions.data.some((row) => hold.permissionSources.some((source) => source.id === row.id)
|
|
6199
|
+
&& row.state === 'granted' && row.resume_answered_by === null && (!row.resume_machine_id || row.resume_machine_id === machineId));
|
|
6200
|
+
if (!messagesRemain && !grantsRemain)
|
|
6201
|
+
releaseLegacyTodoHold(hold.todoId);
|
|
6202
|
+
}
|
|
6203
|
+
}
|