@ctrl-spc/cs 0.7.14 → 0.7.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -3
- package/dist/agents.js +175 -1
- package/dist/autostart.js +103 -122
- package/dist/codex-home.js +14 -11
- package/dist/companion-ui.js +54 -6
- package/dist/companion.js +86 -169
- package/dist/config.js +199 -17
- package/dist/daemon-lifecycle.js +575 -0
- package/dist/daemon-lock.js +149 -42
- package/dist/daemon-processes.js +860 -0
- package/dist/daemon.js +14 -46
- package/dist/darwin-coalition.js +340 -0
- package/dist/failure-reason.js +76 -16
- package/dist/index.js +75 -76
- package/dist/login.js +9 -9
- package/dist/mcp.js +55 -56
- package/dist/native/darwin-coalition +0 -0
- package/dist/native/darwin-coalition.build.json +1 -0
- package/dist/native/darwin-coalition.c +145 -0
- package/dist/orchestrator.js +892 -575
- package/dist/panel3/coordinator.js +3 -1
- package/dist/panel3/presence.js +1 -1
- package/dist/panel3/run.js +996 -558
- package/dist/panel3/spawn.js +85 -23
- package/dist/panel3/tools.js +5 -0
- package/dist/presence-heartbeat.js +3 -0
- package/dist/presence.js +271 -135
- package/dist/supabase.js +173 -37
- package/dist/win-shell.js +464 -1
- package/dist/windows-job.js +312 -0
- package/package.json +4 -3
package/dist/panel3/run.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { failureMessage } from '../failure-reason.js';
|
|
1
2
|
/**
|
|
2
3
|
* ═══ AGENT PANEL v3: the poll loop that answers a card. ═══
|
|
3
4
|
*
|
|
@@ -137,7 +138,7 @@ import { answerPrompt, escalationPrompt, levelOnePrompt, ownerActivationPrompt,
|
|
|
137
138
|
import { ASK_CONTENT_COLUMNS, attachmentLine, gitRulesFor, loadAttachments, loadOutputNames, outputOf, recordBaseProtection, standingRulesFor, withAskContent, } from './show.js';
|
|
138
139
|
import { forgetSecrets, redactSecrets } from './secrets.js';
|
|
139
140
|
import { sayListening, sayPollingProblem, stopListening } from './presence.js';
|
|
140
|
-
import { selectedHarness } from './coordinator.js';
|
|
141
|
+
import { selectedHarness, recoveryHarnesses } from './coordinator.js';
|
|
141
142
|
import { baseBranchState, checkoutForCodebase, commitCardWork, detectBaseProtection, folderIsBranch, hasCheckoutForCodebase, mergeIntoBase, releaseBaseBranch, settleCardWorktree, worktreeForCard, ownsReconciliation, prepareCardReconciliation, finishCardReconciliation, worktreesOnThisMachine, } from './checkout.js';
|
|
142
143
|
import { harness, startAgent } from './spawn.js';
|
|
143
144
|
import { establishOwnerSession, listOwnerSessionIds, OWNER_SESSION_GRACE_MS, readOwnerSession, removeOwnerSession, validSessionUuid, writeOwnerSession, } from './session.js';
|
|
@@ -146,11 +147,187 @@ import { listPanel3CodexOwnerHomeIds, removePanel3CodexOwnerHome, } from '../cod
|
|
|
146
147
|
import { getMachineIdentity, scratchDir } from '../config.js';
|
|
147
148
|
import { listCodebases } from '../codebases.js';
|
|
148
149
|
import { killTree, processIsAlive } from '../win-shell.js';
|
|
150
|
+
import { randomUUID } from 'node:crypto';
|
|
149
151
|
import { hostname, uptime } from 'node:os';
|
|
150
152
|
import { execFileSync } from 'node:child_process';
|
|
151
153
|
import { existsSync } from 'node:fs';
|
|
152
154
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
153
155
|
import { join } from 'node:path';
|
|
156
|
+
function recoveryAttempt(run) {
|
|
157
|
+
return { runId: run.id, processToken: run.process_token, startedAt: run.started_at, resumedAt: run.resumed_at };
|
|
158
|
+
}
|
|
159
|
+
async function startTrackedAgent(client, runId, preparation, work, captured, ...args) {
|
|
160
|
+
if (!preparation)
|
|
161
|
+
return { ...startAgent(...args), attempt: captured };
|
|
162
|
+
const claimed = captured ?? work?.claimAttempt(runId);
|
|
163
|
+
if (!claimed)
|
|
164
|
+
throw new Error('This run no longer belongs to the claimed attempt.');
|
|
165
|
+
preparation.setAttempt(claimed);
|
|
166
|
+
let preparationFailed = false;
|
|
167
|
+
let failureReason = '';
|
|
168
|
+
const execution = {
|
|
169
|
+
...preparation.execution,
|
|
170
|
+
register: async (child, agent) => {
|
|
171
|
+
await preparation.execution.register(child, agent);
|
|
172
|
+
// An ordinary stop can reopen admission after a refusal. Every such
|
|
173
|
+
// wait needs a fresh cloud check before the prompt is authorized.
|
|
174
|
+
try {
|
|
175
|
+
let waited;
|
|
176
|
+
do {
|
|
177
|
+
const { data, error } = await client.from('panel3_runs')
|
|
178
|
+
.select('process_token,started_at,resumed_at,state,ended_at').eq('id', runId).single();
|
|
179
|
+
if (error)
|
|
180
|
+
throw error;
|
|
181
|
+
if (!data || data.state !== 'running' || data.ended_at !== null
|
|
182
|
+
|| claimed.processToken !== (data.process_token ?? null) || claimed.startedAt !== data.started_at
|
|
183
|
+
|| claimed.resumedAt !== (data.resumed_at ?? null)) {
|
|
184
|
+
throw new Error('This run no longer belongs to the claimed attempt.');
|
|
185
|
+
}
|
|
186
|
+
waited = await preparation.execution.waitForPromptAdmission();
|
|
187
|
+
} while (waited);
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
if (!preparation.execution.interrupted()) {
|
|
191
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
192
|
+
preparation.deferFailure(reason, args[1]);
|
|
193
|
+
preparationFailed = true;
|
|
194
|
+
failureReason = reason;
|
|
195
|
+
}
|
|
196
|
+
throw error;
|
|
197
|
+
}
|
|
198
|
+
},
|
|
199
|
+
};
|
|
200
|
+
const started = startAgent(args[0], args[1], args[2], args[3], args[4], args[5], execution);
|
|
201
|
+
// Do not hold the claim barrier while registration waits for an ordinary
|
|
202
|
+
// stop to be refused. The original answer promises proven process closure.
|
|
203
|
+
const answered = started.answered.then(async (answer) => {
|
|
204
|
+
if (preparationFailed && !preparation.execution.interrupted()) {
|
|
205
|
+
await endRun(client, args[1], runId, claimed.cardId, failureReason, claimed);
|
|
206
|
+
preparation.execution.complete();
|
|
207
|
+
}
|
|
208
|
+
return answer;
|
|
209
|
+
});
|
|
210
|
+
void answered.catch(() => { }); // The caller records the PID before awaiting settlement.
|
|
211
|
+
return { ...started, answered, attempt: claimed, deferOutcome: preparation.deferOutcome, acknowledgeOutcome: preparation.acknowledgeOutcome, preparationFailed: () => preparationFailed };
|
|
212
|
+
}
|
|
213
|
+
function panelClaimAttempts(result) {
|
|
214
|
+
if (!Array.isArray(result))
|
|
215
|
+
throw new Error('The claim journal returned no complete result.');
|
|
216
|
+
const unique = new Map();
|
|
217
|
+
for (const row of result) {
|
|
218
|
+
const attempt = row?._attempt;
|
|
219
|
+
if (!attempt || typeof attempt.run_id !== 'string' || typeof attempt.card_id !== 'string' || typeof attempt.started_at !== 'string') {
|
|
220
|
+
throw new Error('The claim journal returned an incomplete run identity.');
|
|
221
|
+
}
|
|
222
|
+
unique.set(attempt.run_id, { runId: attempt.run_id, cardId: attempt.card_id, processToken: attempt.process_token ?? null,
|
|
223
|
+
pid: attempt.pid ?? null, startedAt: attempt.started_at, resumedAt: attempt.resumed_at ?? null,
|
|
224
|
+
observedPendingTurnIds: attempt.observed_pending_turn_ids ?? null, authRecoveryRunId: attempt.auth_recovery_run_id ?? null, harness: attempt.harness ?? null });
|
|
225
|
+
}
|
|
226
|
+
return [...unique.values()];
|
|
227
|
+
}
|
|
228
|
+
function capturedPanelAttempt(rows, runId, work) {
|
|
229
|
+
return work?.claimAttempt(runId) ?? panelClaimAttempts(rows).find(attempt => attempt.runId === runId);
|
|
230
|
+
}
|
|
231
|
+
async function panelClaim(client, work, action, args, nativeContext) {
|
|
232
|
+
const operationId = work?.beginRpc(action, { ...args, ...(nativeContext ? { nativeContext } : {}) }) ?? randomUUID();
|
|
233
|
+
try {
|
|
234
|
+
const response = await client.rpc(action, { ...args, p_operation_id: operationId });
|
|
235
|
+
if (!response.error) {
|
|
236
|
+
const journal = await client.rpc('panel3_reconcile_claim', { p_machine_id: args.p_machine_id, p_operation_id: operationId });
|
|
237
|
+
if (journal.error)
|
|
238
|
+
throw journal.error;
|
|
239
|
+
const row = journal.data?.[0];
|
|
240
|
+
if (row?.outcome !== 'completed' || row.action !== action)
|
|
241
|
+
throw new Error('The claim was cancelled before it could start.');
|
|
242
|
+
const attempts = panelClaimAttempts(row.result);
|
|
243
|
+
work?.recordRpc(operationId, attempts, false, row.result);
|
|
244
|
+
return { ...response, data: row.result, operationId };
|
|
245
|
+
}
|
|
246
|
+
if (response.error)
|
|
247
|
+
work?.deferRpc(operationId);
|
|
248
|
+
return { ...response, operationId };
|
|
249
|
+
}
|
|
250
|
+
catch (error) {
|
|
251
|
+
work?.deferRpc(operationId);
|
|
252
|
+
throw error;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
async function reconcilePanelInterruptions(client, machineId, work) {
|
|
256
|
+
const capability = await client.rpc('panel3_interrupt_machine_runs', { p_machine_id: machineId,
|
|
257
|
+
p_operation_id: randomUUID(), p_interrupted_at: new Date().toISOString(), p_attempts: [] });
|
|
258
|
+
if (capability.error)
|
|
259
|
+
throw new Error(`could not verify interruption recovery: ${capability.error.message}`);
|
|
260
|
+
for (const pending of work.pendingOutcomes()) {
|
|
261
|
+
const { attempt, outcome } = pending;
|
|
262
|
+
const { data, error } = await client.from('panel3_runs').select('started_at,resumed_at,process_token,state,ended_at,level,card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)').eq('id', attempt.runId).maybeSingle();
|
|
263
|
+
if (error)
|
|
264
|
+
throw new Error(`could not read the completed attempt: ${error.message}`);
|
|
265
|
+
if (data && data.started_at === attempt.startedAt && (data.resumed_at ?? null) === attempt.resumedAt
|
|
266
|
+
&& (data.process_token ?? null) === attempt.processToken && data.ended_at === null) {
|
|
267
|
+
const card = (Array.isArray(data.card) ? data.card[0] : data.card);
|
|
268
|
+
const successfulToolEnding = !outcome.ok && outcome.failureKind !== 'authentication' && (data.state === 'asked' || (data.level === 1 && !!card?.conversation_run_id));
|
|
269
|
+
if (outcome.ok || successfulToolEnding)
|
|
270
|
+
await writeAnswer(client, attempt.runId, attempt.cardId, successfulToolEnding ? null : outcome.text, attempt.processToken ?? undefined, attempt);
|
|
271
|
+
else
|
|
272
|
+
await endRun(client, outcome.level ?? 1, attempt.runId, attempt.cardId, outcome.reason, attempt, outcome.failureKind);
|
|
273
|
+
}
|
|
274
|
+
work.acknowledgeOutcome(pending.id);
|
|
275
|
+
}
|
|
276
|
+
for (const failure of work.failedPreparations()) {
|
|
277
|
+
await endRun(client, failure.level, failure.attempt.runId, failure.attempt.cardId, failure.reason, failure.attempt);
|
|
278
|
+
work.acknowledgePreparationFailure(failure.id);
|
|
279
|
+
}
|
|
280
|
+
for (const claim of work.pendingRpcs()) {
|
|
281
|
+
const journal = await client.rpc('panel3_reconcile_claim', { p_machine_id: machineId, p_operation_id: claim.id });
|
|
282
|
+
if (journal.error)
|
|
283
|
+
throw new Error(`could not read the claim receipt: ${journal.error.message}`);
|
|
284
|
+
const row = journal.data?.[0];
|
|
285
|
+
if (row?.outcome === 'cancelled')
|
|
286
|
+
work.recordRpc(claim.id, [], true);
|
|
287
|
+
else if (row?.outcome === 'completed' && row.action === claim.action)
|
|
288
|
+
work.recordRpc(claim.id, panelClaimAttempts(row.result), false, row.result);
|
|
289
|
+
else
|
|
290
|
+
throw new Error('The pending claim could not be reconciled.');
|
|
291
|
+
}
|
|
292
|
+
if (work.legacyRequired()) {
|
|
293
|
+
const attempts = [];
|
|
294
|
+
for (let offset = 0;; offset += 500) {
|
|
295
|
+
const { data, error } = await client.from('panel3_runs')
|
|
296
|
+
.select('id,card_id,process_token,pid,started_at,resumed_at')
|
|
297
|
+
.eq('machine_id', machineId).is('ended_at', null).order('id').range(offset, offset + 499);
|
|
298
|
+
if (error)
|
|
299
|
+
throw error;
|
|
300
|
+
if (!Array.isArray(data))
|
|
301
|
+
throw new Error('The old run snapshot did not return a complete page.');
|
|
302
|
+
for (const row of data) {
|
|
303
|
+
if (typeof row.started_at !== 'string')
|
|
304
|
+
throw new Error('An old run has no exact attempt identity.');
|
|
305
|
+
attempts.push({ runId: row.id, cardId: row.card_id, processToken: row.process_token ?? null,
|
|
306
|
+
pid: row.pid ?? null, startedAt: row.started_at, resumedAt: row.resumed_at ?? null, observedPendingTurnIds: null });
|
|
307
|
+
}
|
|
308
|
+
if (data.length < 500)
|
|
309
|
+
break;
|
|
310
|
+
}
|
|
311
|
+
for (const attempt of attempts)
|
|
312
|
+
work.recordLegacy(attempt);
|
|
313
|
+
work.legacyComplete();
|
|
314
|
+
}
|
|
315
|
+
for (const receipt of work.receipts()) {
|
|
316
|
+
const attempt = receipt.attempt;
|
|
317
|
+
const { data, error } = await client.rpc('panel3_interrupt_machine_runs', {
|
|
318
|
+
p_machine_id: machineId, p_operation_id: receipt.operationId, p_interrupted_at: receipt.interruptedAt,
|
|
319
|
+
p_attempts: [{ run_id: attempt.runId, process_token: attempt.processToken, pid: receipt.pid ?? attempt.pid ?? null,
|
|
320
|
+
started_at: attempt.startedAt, resumed_at: attempt.resumedAt, observed_pending_turn_ids: attempt.observedPendingTurnIds }],
|
|
321
|
+
});
|
|
322
|
+
if (error)
|
|
323
|
+
throw error;
|
|
324
|
+
const result = data?.[0];
|
|
325
|
+
if (!result || result.run_id !== attempt.runId || !['interrupted', 'already_interrupted', 'continued', 'stale'].includes(result.outcome)) {
|
|
326
|
+
throw new Error('The interrupted run was not acknowledged. Work remains protected.');
|
|
327
|
+
}
|
|
328
|
+
work.acknowledge(receipt.id);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
154
331
|
const USAGE = 'usage: run [--once]';
|
|
155
332
|
/** How long between takes. Short, because it is the whole delay between a user
|
|
156
333
|
* sending and a card showing an agent on it, and the take is one small indexed
|
|
@@ -550,18 +727,24 @@ pictures = []) => {
|
|
|
550
727
|
* a dispatch knows the brief before the row exists and writes it there. Passing
|
|
551
728
|
* it again would be rewriting a brief that ux.md fixes at dispatch.
|
|
552
729
|
*
|
|
553
|
-
* A failure
|
|
554
|
-
*
|
|
555
|
-
*
|
|
556
|
-
* which errs towards answering again rather than stranding.
|
|
730
|
+
* A bookkeeping failure does not discard the locally tracked execution. Exact
|
|
731
|
+
* activation predicates prevent a delayed write from reviving an ended run or
|
|
732
|
+
* attaching this process to a later continuation.
|
|
557
733
|
*/
|
|
558
|
-
async function recordProcess(client, runId, pid, brief, processToken) {
|
|
734
|
+
async function recordProcess(client, runId, pid, brief, processToken, attempt) {
|
|
559
735
|
let query = client
|
|
560
736
|
.from('panel3_runs')
|
|
561
737
|
.update({ pid, ...(brief === undefined ? {} : { brief }) })
|
|
562
|
-
.eq('id', runId)
|
|
738
|
+
.eq('id', runId)
|
|
739
|
+
.eq('state', 'running')
|
|
740
|
+
.is('ended_at', null);
|
|
563
741
|
if (processToken !== undefined)
|
|
564
742
|
query = query.eq('process_token', processToken);
|
|
743
|
+
if (attempt) {
|
|
744
|
+
query = query.eq('started_at', attempt.startedAt);
|
|
745
|
+
query = attempt.resumedAt === null ? query.is('resumed_at', null) : query.eq('resumed_at', attempt.resumedAt);
|
|
746
|
+
query = attempt.processToken === null ? query.is('process_token', null) : query.eq('process_token', attempt.processToken);
|
|
747
|
+
}
|
|
565
748
|
const written = await returned(query.select('id'), 'record what is running', `run ${runId}`);
|
|
566
749
|
if (written.length === 0)
|
|
567
750
|
throw new Error(`could not record what is running for run ${runId}: its activation has ended`);
|
|
@@ -579,15 +762,24 @@ async function recordProcess(client, runId, pid, brief, processToken) {
|
|
|
579
762
|
* there first. It is a fact about the record rather than a failure, so it does
|
|
580
763
|
* not go through `returned()`, exactly as the answer's own null does not.
|
|
581
764
|
*/
|
|
582
|
-
async function giveUp(client, runId, reason, processToken) {
|
|
765
|
+
async function giveUp(client, runId, reason, processToken, attempt, failureKind) {
|
|
583
766
|
const { data, error } = await client
|
|
584
767
|
.rpc('panel3_give_up', {
|
|
585
768
|
p_run_id: runId,
|
|
586
769
|
p_reason: reason,
|
|
770
|
+
p_failure_kind: failureKind ?? null,
|
|
771
|
+
p_process_token: processToken ?? null,
|
|
772
|
+
p_expected_attempt: null,
|
|
587
773
|
...(processToken === undefined ? {} : { p_process_token: processToken }),
|
|
774
|
+
...(attempt ? { p_process_token: attempt.processToken, p_expected_attempt: {
|
|
775
|
+
started_at: attempt.startedAt, resumed_at: attempt.resumedAt,
|
|
776
|
+
} } : {}),
|
|
588
777
|
});
|
|
589
778
|
if (error)
|
|
590
779
|
throw new Error(`could not give up run ${runId}: ${error.message}`);
|
|
780
|
+
if (attempt && data !== null && (!Number.isInteger(data) || data < 0)) {
|
|
781
|
+
throw new Error(`could not give up run ${runId}: no ending was acknowledged`);
|
|
782
|
+
}
|
|
591
783
|
/* THE RUN IS OVER, so whatever it read is dropped. Every ending does this —
|
|
592
784
|
here, `failRun` and `writeAnswer` — because a daemon stays up for days and
|
|
593
785
|
has no business holding Tuesday's secret. */
|
|
@@ -630,7 +822,7 @@ async function giveUp(client, runId, reason, processToken) {
|
|
|
630
822
|
* a run can be the last thing live on it — but no turn is written, exactly as
|
|
631
823
|
* none is written for a run that stopped to ask.
|
|
632
824
|
*/
|
|
633
|
-
async function writeAnswer(client, runId, cardId, text, processToken) {
|
|
825
|
+
async function writeAnswer(client, runId, cardId, text, processToken, attempt) {
|
|
634
826
|
/* ═══ THE SECOND OF THE TWO CHOKEPOINTS THE CREDENTIAL RULE RESTS ON. ═══
|
|
635
827
|
This is the ONE path an agent's own words take to a hosted row — a turn on
|
|
636
828
|
the card at levels 1 and 2, and a level 3's `panel3_runs.report`, which
|
|
@@ -640,6 +832,9 @@ async function writeAnswer(client, runId, cardId, text, processToken) {
|
|
|
640
832
|
const { data: turnId, error } = await client
|
|
641
833
|
.rpc('panel3_answer', {
|
|
642
834
|
p_run_id: runId,
|
|
835
|
+
p_process_token: processToken ?? attempt?.processToken ?? null,
|
|
836
|
+
p_expected_attempt: attempt ? { started_at: attempt.startedAt, resumed_at: attempt.resumedAt } : null,
|
|
837
|
+
p_auth_recovery_run_id: attempt?.authRecoveryRunId ?? null,
|
|
643
838
|
p_body: text === null ? null : redactSecrets(processToken === undefined ? runId : `${runId}:${processToken}`, text),
|
|
644
839
|
...(processToken === undefined ? {} : { p_process_token: processToken }),
|
|
645
840
|
});
|
|
@@ -707,14 +902,15 @@ async function writeAnswer(client, runId, cardId, text, processToken) {
|
|
|
707
902
|
away. */
|
|
708
903
|
said(`run ${runId} had already ended, so its answer was not written to card ${cardId}`);
|
|
709
904
|
}
|
|
710
|
-
return ownerSettlementAccepted(run, processToken);
|
|
905
|
+
return ownerSettlementAccepted(run, processToken, attempt);
|
|
711
906
|
}
|
|
712
907
|
out(`answered card ${cardId} run ${runId} ${text?.length ?? 0} characters`);
|
|
713
908
|
return true;
|
|
714
909
|
}
|
|
715
|
-
export function ownerSettlementAccepted(run, processToken) {
|
|
910
|
+
export function ownerSettlementAccepted(run, processToken, attempt) {
|
|
716
911
|
return (run.state === 'asked' || run.state === 'finished')
|
|
717
|
-
&& (processToken === undefined || run.processToken === processToken)
|
|
912
|
+
&& (processToken === undefined || run.processToken === processToken)
|
|
913
|
+
&& (!attempt || run.processToken === attempt.processToken && run.startedAt === attempt.startedAt && (run.resumedAt ?? null) === attempt.resumedAt);
|
|
718
914
|
}
|
|
719
915
|
/** What the record says a run is now, and at what level. Read only to say the
|
|
720
916
|
* right sentence about something that has already happened; nothing branches on
|
|
@@ -722,7 +918,7 @@ export function ownerSettlementAccepted(run, processToken) {
|
|
|
722
918
|
async function runNow(client, runId) {
|
|
723
919
|
const runs = await returned(client
|
|
724
920
|
.from('panel3_runs')
|
|
725
|
-
.select('state, level, process_token, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)')
|
|
921
|
+
.select('state, level, process_token, started_at, resumed_at, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)')
|
|
726
922
|
.eq('id', runId), 'read', `the state of run ${runId}`);
|
|
727
923
|
const run = runs[0];
|
|
728
924
|
return run
|
|
@@ -730,9 +926,9 @@ async function runNow(client, runId) {
|
|
|
730
926
|
state: run.state,
|
|
731
927
|
level: run.level,
|
|
732
928
|
conversationOwnerId: run.card?.conversation_run_id ?? null,
|
|
733
|
-
processToken: run.process_token,
|
|
929
|
+
processToken: run.process_token, startedAt: run.started_at, resumedAt: run.resumed_at,
|
|
734
930
|
}
|
|
735
|
-
: { state: 'no longer on the record', level: null, conversationOwnerId: null, processToken: null };
|
|
931
|
+
: { state: 'no longer on the record', level: null, conversationOwnerId: null, processToken: null, startedAt: null, resumedAt: null };
|
|
736
932
|
}
|
|
737
933
|
/**
|
|
738
934
|
* A run whose process failed, ended with the reason ON ITS OWN COLUMN, and its
|
|
@@ -828,76 +1024,83 @@ export async function settingsForRun(client, runId) {
|
|
|
828
1024
|
* the level is what tells it how a run of this shape ends.
|
|
829
1025
|
*/
|
|
830
1026
|
async function answerCard(client, tools, machineId, cardId, turns) {
|
|
831
|
-
const
|
|
832
|
-
/* BEFORE THE SPAWN, AND ITS FAILURE IS THE SPAWN'S FAILURE. The receipts are
|
|
833
|
-
part of what the agent is sent, so a read that fails must not be papered
|
|
834
|
-
over with an empty list: that reads as a card that has made nothing, which
|
|
835
|
-
is how an agent creates a second epic beside the one it cannot see. The
|
|
836
|
-
attachments read carries the same rule: a failed read here must not read
|
|
837
|
-
as "nothing is attached", which is a different card than the one that was
|
|
838
|
-
actually sent. */
|
|
839
|
-
/* ═══ AND A FAILURE HERE ENDS THE RUN, RATHER THAN LEAVING IT RUNNING WITH NO
|
|
840
|
-
PROCESS. ═══ The take already wrote the run row in the statement that leased
|
|
841
|
-
the turns, so a throw between here and `startAgent` leaves a run reading
|
|
842
|
-
`running` with a null pid and nothing on stderr the person can see.
|
|
843
|
-
`recoverStranded` then reads that as a machine that went away, hands the
|
|
844
|
-
message back, and `panel3_take_turns` leases it to A BRAND NEW RUN whose
|
|
845
|
-
attempts start again at one — so a permanent failure, such as a read this
|
|
846
|
-
build cannot make against the current schema, repeats forever while the card
|
|
847
|
-
says Working and never says why. This is the sixth of `endRun`'s endings and
|
|
848
|
-
the last one that was missing: `resumeRun` and `startRearmed` already end
|
|
849
|
-
their two post-claim failures this way for exactly this reason.
|
|
850
|
-
THE REASON IS SHAREABLE. All three reads are `returned()` calls against the
|
|
851
|
-
database, whose messages name tables and columns and never a local path, so
|
|
852
|
-
constraint 6 is satisfied without a level fork here. */
|
|
853
|
-
let brief;
|
|
854
|
-
/* WHERE IT RUNS AND WHAT IT IS TOLD ABOUT THE CODEBASES, THROUGH THE SAME
|
|
855
|
-
SEAM AS EVERY OTHER SPAWN. A level 1 run works in an empty directory of the
|
|
856
|
-
user's own, and it is the fifth start site rather than a special case: the
|
|
857
|
-
generated git block reaches every level, and one place resolving it is what
|
|
858
|
-
makes that true without four copies of the read. */
|
|
859
|
-
let where;
|
|
860
|
-
/* THE MANDATE IS READ IN THE SAME WINDOW AND UNDER THE SAME RULE. A launcher
|
|
861
|
-
is an agent like any other and gets the project's standing rules before it
|
|
862
|
-
decides anything, and a read that fails must not read as "this project has
|
|
863
|
-
no rules" — that is a different project than the one the person is on. So it
|
|
864
|
-
joins the three reads above inside this ending rather than beside it. */
|
|
865
|
-
let rules;
|
|
866
|
-
let settings;
|
|
867
|
-
try {
|
|
868
|
-
brief = briefFor(turns[0].card_title, turns, await receiptsFor(client, cardId), await attachmentsFor(client, cardId), await codebasesForRun(client, runId));
|
|
869
|
-
rules = await standingRulesFor(client, runId);
|
|
870
|
-
settings = await settingsForRun(client, runId);
|
|
871
|
-
where = await workingDirectory(client, runId, LEVEL, false);
|
|
872
|
-
}
|
|
873
|
-
catch (error) {
|
|
874
|
-
const why = error instanceof Error ? error.message : String(error);
|
|
875
|
-
await endRun(client, LEVEL, runId, cardId, why);
|
|
876
|
-
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
877
|
-
}
|
|
878
|
-
/* ═══ THE RUN ID IS ON THE URL, AND THAT IS THE WHOLE OF WHAT THE AGENT IS
|
|
879
|
-
TOLD ABOUT ITS OWN STANDING. ═══ The tools server reads the level off the
|
|
880
|
-
run row this id names, so the daemon does not tell the child what it may do
|
|
881
|
-
and the child has nothing to claim. `LEVEL` below decides argv only — which
|
|
882
|
-
of the harness's own tools the process gets — and the two can never disagree
|
|
883
|
-
about the record, because only one of them consults it. */
|
|
884
|
-
/* AN EMPTY DIRECTORY OF THE USER'S OWN, because level 1 has no code tool to
|
|
885
|
-
use a real one with, and the daemon's inherited cwd under a launchd login
|
|
886
|
-
item is the filesystem root. */
|
|
887
|
-
const started = startAgent(withStandingRules(rules, brief, where.block), LEVEL, tools.urlFor(runId), where.cwd, undefined, settings);
|
|
888
|
-
out(`run ${runId} level ${LEVEL} ${started.pid ? `pid ${started.pid}` : 'no process'}`);
|
|
1027
|
+
const preparation = tools.work?.prepare(turns[0].run_id);
|
|
889
1028
|
try {
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
1029
|
+
const runId = turns[0].run_id;
|
|
1030
|
+
const attempt = capturedPanelAttempt(turns, runId, tools.work);
|
|
1031
|
+
/* BEFORE THE SPAWN, AND ITS FAILURE IS THE SPAWN'S FAILURE. The receipts are
|
|
1032
|
+
part of what the agent is sent, so a read that fails must not be papered
|
|
1033
|
+
over with an empty list: that reads as a card that has made nothing, which
|
|
1034
|
+
is how an agent creates a second epic beside the one it cannot see. The
|
|
1035
|
+
attachments read carries the same rule: a failed read here must not read
|
|
1036
|
+
as "nothing is attached", which is a different card than the one that was
|
|
1037
|
+
actually sent. */
|
|
1038
|
+
/* ═══ AND A FAILURE HERE ENDS THE RUN, RATHER THAN LEAVING IT RUNNING WITH NO
|
|
1039
|
+
PROCESS. ═══ The take already wrote the run row in the statement that leased
|
|
1040
|
+
the turns, so a throw between here and `startAgent` leaves a run reading
|
|
1041
|
+
`running` with a null pid and nothing on stderr the person can see.
|
|
1042
|
+
`recoverStranded` then reads that as a machine that went away, hands the
|
|
1043
|
+
message back, and `panel3_take_turns` leases it to A BRAND NEW RUN whose
|
|
1044
|
+
attempts start again at one — so a permanent failure, such as a read this
|
|
1045
|
+
build cannot make against the current schema, repeats forever while the card
|
|
1046
|
+
says Working and never says why. This is the sixth of `endRun`'s endings and
|
|
1047
|
+
the last one that was missing: `resumeRun` and `startRearmed` already end
|
|
1048
|
+
their two post-claim failures this way for exactly this reason.
|
|
1049
|
+
THE REASON IS SHAREABLE. All three reads are `returned()` calls against the
|
|
1050
|
+
database, whose messages name tables and columns and never a local path, so
|
|
1051
|
+
constraint 6 is satisfied without a level fork here. */
|
|
1052
|
+
let brief;
|
|
1053
|
+
/* WHERE IT RUNS AND WHAT IT IS TOLD ABOUT THE CODEBASES, THROUGH THE SAME
|
|
1054
|
+
SEAM AS EVERY OTHER SPAWN. A level 1 run works in an empty directory of the
|
|
1055
|
+
user's own, and it is the fifth start site rather than a special case: the
|
|
1056
|
+
generated git block reaches every level, and one place resolving it is what
|
|
1057
|
+
makes that true without four copies of the read. */
|
|
1058
|
+
let where;
|
|
1059
|
+
/* THE MANDATE IS READ IN THE SAME WINDOW AND UNDER THE SAME RULE. A launcher
|
|
1060
|
+
is an agent like any other and gets the project's standing rules before it
|
|
1061
|
+
decides anything, and a read that fails must not read as "this project has
|
|
1062
|
+
no rules" — that is a different project than the one the person is on. So it
|
|
1063
|
+
joins the three reads above inside this ending rather than beside it. */
|
|
1064
|
+
let rules;
|
|
1065
|
+
let settings;
|
|
1066
|
+
try {
|
|
1067
|
+
brief = briefFor(turns[0].card_title, turns, await receiptsFor(client, cardId), await attachmentsFor(client, cardId), await codebasesForRun(client, runId));
|
|
1068
|
+
rules = await standingRulesFor(client, runId);
|
|
1069
|
+
settings = await settingsForRun(client, runId);
|
|
1070
|
+
where = await workingDirectory(client, runId, LEVEL, false);
|
|
1071
|
+
}
|
|
1072
|
+
catch (error) {
|
|
1073
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
1074
|
+
await endRun(client, LEVEL, runId, cardId, why, attempt);
|
|
1075
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
1076
|
+
}
|
|
1077
|
+
/* ═══ THE RUN ID IS ON THE URL, AND THAT IS THE WHOLE OF WHAT THE AGENT IS
|
|
1078
|
+
TOLD ABOUT ITS OWN STANDING. ═══ The tools server reads the level off the
|
|
1079
|
+
run row this id names, so the daemon does not tell the child what it may do
|
|
1080
|
+
and the child has nothing to claim. `LEVEL` below decides argv only — which
|
|
1081
|
+
of the harness's own tools the process gets — and the two can never disagree
|
|
1082
|
+
about the record, because only one of them consults it. */
|
|
1083
|
+
/* AN EMPTY DIRECTORY OF THE USER'S OWN, because level 1 has no code tool to
|
|
1084
|
+
use a real one with, and the daemon's inherited cwd under a launchd login
|
|
1085
|
+
item is the filesystem root. */
|
|
1086
|
+
const started = await startTrackedAgent(client, runId, preparation, tools.work, attempt, withStandingRules(rules, brief, where.block), LEVEL, tools.urlFor(runId), where.cwd, undefined, settings);
|
|
1087
|
+
out(`run ${runId} level ${LEVEL} ${started.pid ? `pid ${started.pid}` : 'no process'}`);
|
|
1088
|
+
try {
|
|
1089
|
+
/* THE BRIEF, NOT WHAT THE PROCESS WAS HANDED. The rules are current at the
|
|
1090
|
+
activation and the brief is immutable, so storing the composed string
|
|
1091
|
+
would freeze one inside the other and every respawn would replay it. */
|
|
1092
|
+
await recordProcess(client, runId, started.pid, brief, undefined, started.attempt);
|
|
1093
|
+
}
|
|
1094
|
+
catch (error) {
|
|
1095
|
+
// Said, not fatal. See `recordProcess` for what this costs and why the
|
|
1096
|
+
// agent is not killed over it.
|
|
1097
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1098
|
+
}
|
|
1099
|
+
return settle(client, tools, machineId, LEVEL, runId, cardId, started, false);
|
|
894
1100
|
}
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
// agent is not killed over it.
|
|
898
|
-
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1101
|
+
finally {
|
|
1102
|
+
preparation?.finish();
|
|
899
1103
|
}
|
|
900
|
-
return settle(client, tools, machineId, LEVEL, runId, cardId, started, false);
|
|
901
1104
|
}
|
|
902
1105
|
/**
|
|
903
1106
|
* WHICH PROJECT A RUN'S CARD IS FILED UNDER, or null when the card has none.
|
|
@@ -1224,92 +1427,106 @@ async function workingDirectory(client, runId, level, isOwner, knownCodebase) {
|
|
|
1224
1427
|
* Then the row, then the process, then the pid — constraint 8, in the only order
|
|
1225
1428
|
* that satisfies it.
|
|
1226
1429
|
*/
|
|
1227
|
-
async function startChild(client, tools, machineId, parentRunId, brief, codebase, parentProcessToken, choice = {}) {
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
p_brief: brief,
|
|
1231
|
-
p_machine_id: machineId,
|
|
1232
|
-
p_codebase_id: codebase?.id ?? null,
|
|
1233
|
-
p_codebase_label: codebase?.name ?? null,
|
|
1234
|
-
p_process_token: parentProcessToken ?? null,
|
|
1235
|
-
p_model: choice.model ?? null,
|
|
1236
|
-
p_effort: choice.effort ?? null,
|
|
1237
|
-
});
|
|
1238
|
-
if (error)
|
|
1239
|
-
throw new Error(`could not start an agent under run ${parentRunId}: ${readableWriteError(error.message)}`);
|
|
1240
|
-
const row = data?.[0];
|
|
1241
|
-
if (!row) {
|
|
1242
|
-
/* NOTHING WAS WRITTEN AND NOTHING IS RUNNING, and the two reasons are said
|
|
1243
|
-
together because the caller cannot tell them apart from here and both mean
|
|
1244
|
-
the same thing to it. */
|
|
1245
|
-
throw new Error(`NO AGENT WAS STARTED and nothing was written: run ${parentRunId} has ended, is already as `
|
|
1246
|
-
+ 'deep as anything may be sent from, or this conversation already has its owner. Exit now.');
|
|
1247
|
-
}
|
|
1248
|
-
if (row.run_level !== 2 && row.run_level !== 3) {
|
|
1249
|
-
/* UNREACHABLE, AND STILL SETTLED. `panel3_dispatch` writes `parent.level + 1`
|
|
1250
|
-
from a parent it has just checked is below 3, so there is no level here
|
|
1251
|
-
this cannot spawn. If that ever stops being true, the row exists and
|
|
1252
|
-
nothing will ever start for it, and leaving it `running` would make a
|
|
1253
|
-
recovery sweep wait out the pid grace window to conclude what is already
|
|
1254
|
-
known. */
|
|
1255
|
-
const why = `run ${row.run_id} was written at level ${row.run_level}, which cannot be spawned`;
|
|
1256
|
-
await giveUp(client, row.run_id, why, row.process_token ?? undefined);
|
|
1257
|
-
throw new Error(why);
|
|
1258
|
-
}
|
|
1259
|
-
const level = row.run_level;
|
|
1260
|
-
let where;
|
|
1261
|
-
let prompt;
|
|
1262
|
-
let pictures;
|
|
1263
|
-
/* READ AGAINST THE CHILD'S OWN RUN ROW, NOT THE PARENT'S. The row already
|
|
1264
|
-
exists (`panel3_dispatch` wrote it above) and it carries this child's
|
|
1265
|
-
codebase, which is what decides which codebase-scoped rules it is under. */
|
|
1266
|
-
let rules;
|
|
1267
|
-
let settings;
|
|
1268
|
-
try {
|
|
1269
|
-
where = await workingDirectory(client, row.run_id, level, level === 2, codebase);
|
|
1270
|
-
prompt = level === 2
|
|
1271
|
-
? await initialOwnerPrompt(client, row.run_card_id, parentRunId, brief)
|
|
1272
|
-
: brief;
|
|
1273
|
-
rules = await standingRulesFor(client, row.run_id);
|
|
1274
|
-
settings = await settingsForRun(client, row.run_id);
|
|
1275
|
-
/* Inside this site's own try, for `standingRulesFor`'s reason: a failure to
|
|
1276
|
-
assemble what the agent needs ends the run the way this path already ends
|
|
1277
|
-
runs, rather than starting a process that is missing it. */
|
|
1278
|
-
pictures = await picturesOnDisk(client, row.run_card_id, where, level);
|
|
1279
|
-
}
|
|
1280
|
-
catch (error) {
|
|
1281
|
-
const why = error instanceof Error ? error.message : String(error);
|
|
1282
|
-
await giveUp(client, row.run_id, why, row.process_token ?? undefined);
|
|
1283
|
-
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
1284
|
-
}
|
|
1285
|
-
const processToken = row.process_token ?? undefined;
|
|
1286
|
-
const isOwner = level === 2;
|
|
1287
|
-
const started = startAgent(withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id, processToken), where.cwd, isOwner ? { ownerId: row.run_id } : undefined, settings);
|
|
1288
|
-
if (started.pid === null) {
|
|
1289
|
-
/* THERE IS A ROW AND THERE IS NO PROCESS, which is the one shape the record
|
|
1290
|
-
must never be left in quietly. The answer is already settled — nothing ran
|
|
1291
|
-
— so the reason is read off it, the run is ended with that reason on it,
|
|
1292
|
-
and the tool call fails saying no agent was started. */
|
|
1293
|
-
const answer = await started.answered;
|
|
1294
|
-
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
1295
|
-
await giveUp(client, row.run_id, reason, processToken);
|
|
1296
|
-
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
1297
|
-
}
|
|
1298
|
-
out(`dispatch run ${row.run_id} level ${level} under ${parentRunId} pid ${started.pid}`);
|
|
1430
|
+
async function startChild(client, tools, machineId, parentRunId, brief, codebase, parentProcessToken, choice = {}, recovered) {
|
|
1431
|
+
let claimOperationId;
|
|
1432
|
+
const preparation = tools.work?.prepare();
|
|
1299
1433
|
try {
|
|
1300
|
-
await
|
|
1434
|
+
const response = recovered ?? await panelClaim(client, tools.work, 'panel3_dispatch', {
|
|
1435
|
+
p_parent_run_id: parentRunId,
|
|
1436
|
+
p_brief: brief,
|
|
1437
|
+
p_machine_id: machineId,
|
|
1438
|
+
p_codebase_id: codebase?.id ?? null,
|
|
1439
|
+
p_codebase_label: codebase?.name ?? null,
|
|
1440
|
+
p_process_token: parentProcessToken ?? null,
|
|
1441
|
+
p_model: choice.model ?? null,
|
|
1442
|
+
p_effort: choice.effort ?? null,
|
|
1443
|
+
});
|
|
1444
|
+
const { data, error, operationId } = response;
|
|
1445
|
+
claimOperationId = operationId;
|
|
1446
|
+
if (error)
|
|
1447
|
+
throw new Error(`could not start an agent under run ${parentRunId}: ${readableWriteError(error.message)}`);
|
|
1448
|
+
const row = data?.[0];
|
|
1449
|
+
if (!row) {
|
|
1450
|
+
/* NOTHING WAS WRITTEN AND NOTHING IS RUNNING, and the two reasons are said
|
|
1451
|
+
together because the caller cannot tell them apart from here and both mean
|
|
1452
|
+
the same thing to it. */
|
|
1453
|
+
throw new Error(`NO AGENT WAS STARTED and nothing was written: run ${parentRunId} has ended, is already as `
|
|
1454
|
+
+ 'deep as anything may be sent from, or this conversation already has its owner. Exit now.');
|
|
1455
|
+
}
|
|
1456
|
+
const attempt = capturedPanelAttempt(data ?? [], row.run_id, tools.work);
|
|
1457
|
+
if (row.run_level !== 2 && row.run_level !== 3) {
|
|
1458
|
+
/* UNREACHABLE, AND STILL SETTLED. `panel3_dispatch` writes `parent.level + 1`
|
|
1459
|
+
from a parent it has just checked is below 3, so there is no level here
|
|
1460
|
+
this cannot spawn. If that ever stops being true, the row exists and
|
|
1461
|
+
nothing will ever start for it, and leaving it `running` would make a
|
|
1462
|
+
recovery sweep wait out the pid grace window to conclude what is already
|
|
1463
|
+
known. */
|
|
1464
|
+
const why = `run ${row.run_id} was written at level ${row.run_level}, which cannot be spawned`;
|
|
1465
|
+
await giveUp(client, row.run_id, why, row.process_token ?? undefined, attempt);
|
|
1466
|
+
throw new Error(why);
|
|
1467
|
+
}
|
|
1468
|
+
const level = row.run_level;
|
|
1469
|
+
let where;
|
|
1470
|
+
let prompt;
|
|
1471
|
+
let pictures;
|
|
1472
|
+
/* READ AGAINST THE CHILD'S OWN RUN ROW, NOT THE PARENT'S. The row already
|
|
1473
|
+
exists (`panel3_dispatch` wrote it above) and it carries this child's
|
|
1474
|
+
codebase, which is what decides which codebase-scoped rules it is under. */
|
|
1475
|
+
let rules;
|
|
1476
|
+
let settings;
|
|
1477
|
+
try {
|
|
1478
|
+
where = await workingDirectory(client, row.run_id, level, level === 2, codebase);
|
|
1479
|
+
prompt = level === 2
|
|
1480
|
+
? await initialOwnerPrompt(client, row.run_card_id, parentRunId, brief)
|
|
1481
|
+
: brief;
|
|
1482
|
+
rules = await standingRulesFor(client, row.run_id);
|
|
1483
|
+
settings = await settingsForRun(client, row.run_id);
|
|
1484
|
+
/* Inside this site's own try, for `standingRulesFor`'s reason: a failure to
|
|
1485
|
+
assemble what the agent needs ends the run the way this path already ends
|
|
1486
|
+
runs, rather than starting a process that is missing it. */
|
|
1487
|
+
pictures = await picturesOnDisk(client, row.run_card_id, where, level);
|
|
1488
|
+
}
|
|
1489
|
+
catch (error) {
|
|
1490
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
1491
|
+
await giveUp(client, row.run_id, why, row.process_token ?? undefined, attempt);
|
|
1492
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
1493
|
+
}
|
|
1494
|
+
const processToken = row.process_token ?? undefined;
|
|
1495
|
+
const isOwner = level === 2;
|
|
1496
|
+
const started = await startTrackedAgent(client, row.run_id, preparation, tools.work, attempt, withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id, processToken), where.cwd, isOwner ? { ownerId: row.run_id } : undefined, settings);
|
|
1497
|
+
if (started.pid === null) {
|
|
1498
|
+
if (started.interrupted?.())
|
|
1499
|
+
throw new Error('Work was interrupted by the service command.');
|
|
1500
|
+
/* THERE IS A ROW AND THERE IS NO PROCESS, which is the one shape the record
|
|
1501
|
+
must never be left in quietly. The answer is already settled — nothing ran
|
|
1502
|
+
— so the reason is read off it, the run is ended with that reason on it,
|
|
1503
|
+
and the tool call fails saying no agent was started. */
|
|
1504
|
+
const answer = await started.answered;
|
|
1505
|
+
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
1506
|
+
await giveUp(client, row.run_id, reason, processToken, attempt);
|
|
1507
|
+
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
1508
|
+
}
|
|
1509
|
+
out(`dispatch run ${row.run_id} level ${level} under ${parentRunId} pid ${started.pid}`);
|
|
1510
|
+
try {
|
|
1511
|
+
await recordProcess(client, row.run_id, started.pid, undefined, processToken, started.attempt);
|
|
1512
|
+
}
|
|
1513
|
+
catch (error) {
|
|
1514
|
+
// Said, not fatal, exactly as at level 1: the agent is running and killing
|
|
1515
|
+
// it over a bookkeeping write would cost the user the work.
|
|
1516
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1517
|
+
}
|
|
1518
|
+
return {
|
|
1519
|
+
runId: row.run_id,
|
|
1520
|
+
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, true, processToken, isOwner && processToken
|
|
1521
|
+
? ownerSessionLifecycle(started, row.run_id, settings.harness ?? harness(), processToken)
|
|
1522
|
+
: undefined),
|
|
1523
|
+
};
|
|
1301
1524
|
}
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1525
|
+
finally {
|
|
1526
|
+
preparation?.finish();
|
|
1527
|
+
if (claimOperationId)
|
|
1528
|
+
tools.work?.finishRpcs([claimOperationId]);
|
|
1306
1529
|
}
|
|
1307
|
-
return {
|
|
1308
|
-
runId: row.run_id,
|
|
1309
|
-
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, true, processToken, isOwner && processToken
|
|
1310
|
-
? ownerSessionLifecycle(started, row.run_id, settings.harness ?? harness(), processToken)
|
|
1311
|
-
: undefined),
|
|
1312
|
-
};
|
|
1313
1530
|
}
|
|
1314
1531
|
/**
|
|
1315
1532
|
* HOW MANY PROCESSES HAVE BEEN STARTED FOR THIS RUN, counting the first.
|
|
@@ -1377,9 +1594,22 @@ async function attemptsSoFar(client, runId) {
|
|
|
1377
1594
|
* Returns whether THIS call is what ended it. False is a fact about the record,
|
|
1378
1595
|
* not a failure: the run had already ended, and the caller says so.
|
|
1379
1596
|
*/
|
|
1380
|
-
async function endRun(client, level, runId, cardId, why) {
|
|
1597
|
+
async function endRun(client, level, runId, cardId, why, attempt, failureKind) {
|
|
1381
1598
|
if (level !== 1)
|
|
1382
|
-
return (await giveUp(client, runId, why)) !== null;
|
|
1599
|
+
return (await giveUp(client, runId, why, undefined, attempt, failureKind)) !== null;
|
|
1600
|
+
if (attempt) {
|
|
1601
|
+
const { data, error } = await client.rpc('panel3_end_run', {
|
|
1602
|
+
p_run_id: runId, p_reason: why, p_process_token: attempt.processToken,
|
|
1603
|
+
p_failure_kind: failureKind ?? null,
|
|
1604
|
+
p_expected_attempt: { started_at: attempt.startedAt, resumed_at: attempt.resumedAt },
|
|
1605
|
+
});
|
|
1606
|
+
if (error)
|
|
1607
|
+
throw new Error(`could not end run ${runId}: ${error.message}`);
|
|
1608
|
+
if (typeof data !== 'boolean')
|
|
1609
|
+
throw new Error(`could not end run ${runId}: no ending was acknowledged`);
|
|
1610
|
+
forgetSecrets(runId);
|
|
1611
|
+
return data;
|
|
1612
|
+
}
|
|
1383
1613
|
/* ═══ THE RUN FIRST, AND THE CARD ONLY IF THIS RUN WAS STILL THE CARD'S TO
|
|
1384
1614
|
FAIL. ═══
|
|
1385
1615
|
The card used to be written first, and the argument for that was a daemon
|
|
@@ -1464,6 +1694,13 @@ function ownerSessionLifecycle(started, ownerId, ownerHarness, processToken, exp
|
|
|
1464
1694
|
}
|
|
1465
1695
|
function settle(client, tools, machineId, level, runId, cardId, started, speaksToTheCard = true, processToken, ownerSession) {
|
|
1466
1696
|
return started.answered.then(async (answer) => {
|
|
1697
|
+
if (started.interrupted?.() || started.preparationFailed?.())
|
|
1698
|
+
return;
|
|
1699
|
+
if (!answer.ok)
|
|
1700
|
+
answer = { ...answer, reason: failureMessage(started.attempt?.harness ?? harness(), answer.failureKind ?? 'unknown') };
|
|
1701
|
+
started.deferOutcome?.({ surface: 'panel', ok: answer.ok,
|
|
1702
|
+
text: answer.ok && speaksToTheCard ? redactSecrets(processToken === undefined ? runId : `${runId}:${processToken}`, answer.text) : null,
|
|
1703
|
+
reason: answer.ok ? '' : answer.reason, failureKind: answer.ok ? undefined : answer.failureKind, level });
|
|
1467
1704
|
/* ═══ A RUN THAT STOPPED TO ASK DID NOT DIE, WHATEVER THE HARNESS PRINTED
|
|
1468
1705
|
ON ITS WAY OUT. ═══
|
|
1469
1706
|
|
|
@@ -1488,12 +1725,20 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
1488
1725
|
into an ordinary ending, and then handed to `writeAnswer` — which already
|
|
1489
1726
|
owns this case and already prints its sentence — rather than to a second
|
|
1490
1727
|
ending written here beside it. */
|
|
1491
|
-
|
|
1728
|
+
const observe = (authenticated) => {
|
|
1729
|
+
const selected = started.attempt?.harness;
|
|
1730
|
+
if (selected)
|
|
1731
|
+
tools.work?.observeHarness(selected, authenticated ? 'authenticated' : 'sign-in-required', authenticated ? 'dispatch-success' : 'provider-rejected');
|
|
1732
|
+
};
|
|
1733
|
+
if (!answer.ok && answer.failureKind !== 'authentication') {
|
|
1492
1734
|
const run = await runNow(client, runId);
|
|
1493
1735
|
if (run.state === 'asked') {
|
|
1494
|
-
|
|
1495
|
-
|
|
1736
|
+
started.deferOutcome?.({ surface: 'panel', ok: true, text: null, reason: '', level });
|
|
1737
|
+
const accepted = await writeAnswer(client, runId, cardId, null, processToken, started.attempt);
|
|
1738
|
+
if (accepted) {
|
|
1739
|
+
observe(true);
|
|
1496
1740
|
await ownerSession?.established();
|
|
1741
|
+
}
|
|
1497
1742
|
else
|
|
1498
1743
|
await ownerSession?.failed();
|
|
1499
1744
|
return;
|
|
@@ -1503,7 +1748,9 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
1503
1748
|
exit cleanly without an agent message after the tool succeeds. The
|
|
1504
1749
|
record, not prose the launcher was told not to write, decides success. */
|
|
1505
1750
|
if (level === 1 && run.conversationOwnerId !== null) {
|
|
1506
|
-
|
|
1751
|
+
started.deferOutcome?.({ surface: 'panel', ok: true, text: null, reason: '', level });
|
|
1752
|
+
if (await writeAnswer(client, runId, cardId, null, processToken, started.attempt))
|
|
1753
|
+
observe(true);
|
|
1507
1754
|
return;
|
|
1508
1755
|
}
|
|
1509
1756
|
}
|
|
@@ -1514,7 +1761,7 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
1514
1761
|
Repeating an invalid model/effort request cannot repair it. Unknown
|
|
1515
1762
|
process deaths retain the existing recovery policy. */
|
|
1516
1763
|
const attempts = await attemptsSoFar(client, runId);
|
|
1517
|
-
if (answer.retryable
|
|
1764
|
+
if (answer.retryable === true && started.pid !== null && attempts < MAX_ATTEMPTS) {
|
|
1518
1765
|
out(`retry run ${runId} attempt ${attempts} of ${MAX_ATTEMPTS} died: ${answer.reason}`);
|
|
1519
1766
|
const again = processToken === undefined
|
|
1520
1767
|
? await resumeRun(client, tools, machineId, runId, started.pid)
|
|
@@ -1552,8 +1799,10 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
1552
1799
|
than two. */
|
|
1553
1800
|
const why = attempts > 1 ? `${answer.reason} (after ${attempts} attempts)` : answer.reason;
|
|
1554
1801
|
const ended = processToken === undefined
|
|
1555
|
-
? await endRun(client, level, runId, cardId, why)
|
|
1556
|
-
: (await giveUp(client, runId, why, processToken)) !== null;
|
|
1802
|
+
? await endRun(client, level, runId, cardId, why, started.attempt, answer.failureKind)
|
|
1803
|
+
: (await giveUp(client, runId, why, processToken, started.attempt, answer.failureKind)) !== null;
|
|
1804
|
+
if (ended && answer.failureKind === 'authentication')
|
|
1805
|
+
observe(false);
|
|
1557
1806
|
if (!ended) {
|
|
1558
1807
|
/* IT WAS ALREADY SETTLED, by recovery, which decided this process was
|
|
1559
1808
|
gone before it said so itself, or by the person's Stop. Nothing was
|
|
@@ -1570,12 +1819,14 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
1570
1819
|
/* ═══ ITS OWN WORDS, ONTO THE CARD, THROUGH THE SAME STATEMENT EVERY LEVEL
|
|
1571
1820
|
USES. ═══ Nothing reads them, nothing shortens them and nothing waits to
|
|
1572
1821
|
approve them: ux.md's "whoever did the work writes the answer". */
|
|
1573
|
-
const accepted = await writeAnswer(client, runId, cardId, speaksToTheCard ? answer.text : null, processToken);
|
|
1574
|
-
if (accepted)
|
|
1822
|
+
const accepted = await writeAnswer(client, runId, cardId, speaksToTheCard ? answer.text : null, processToken, started.attempt);
|
|
1823
|
+
if (accepted) {
|
|
1824
|
+
observe(true);
|
|
1575
1825
|
await ownerSession?.established();
|
|
1826
|
+
}
|
|
1576
1827
|
else
|
|
1577
1828
|
await ownerSession?.failed();
|
|
1578
|
-
});
|
|
1829
|
+
}).then(() => { started.acknowledgeOutcome?.(); started.completed?.(); });
|
|
1579
1830
|
}
|
|
1580
1831
|
/**
|
|
1581
1832
|
* ═══ ONE RUN, STARTED AGAIN AS ITSELF, WITH WHAT IT WAS SENT AND WHAT IT HAD
|
|
@@ -1633,13 +1884,13 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
1633
1884
|
* the record refuses a fourth, and the caller is the one that knows what to say
|
|
1634
1885
|
* about the failure it was holding.
|
|
1635
1886
|
*/
|
|
1636
|
-
async function resumeRun(client, tools, machineId, runId, afterPid) {
|
|
1887
|
+
async function resumeRun(client, tools, machineId, runId, afterPid, recovered) {
|
|
1637
1888
|
/* ═══ THE SWEEP CHECKS BEFORE THE CLAIM AND THE RETRY CANNOT, AND `afterPid`
|
|
1638
1889
|
IS THE WHOLE OF WHAT DECIDES IT. ═══ Said once, here. See the header for
|
|
1639
1890
|
both halves of the argument: a machine with no checkout must not take a run
|
|
1640
1891
|
it cannot start, and the level a retry needs the answer for is not known
|
|
1641
1892
|
until the claim returns.
|
|
1642
|
-
|
|
1893
|
+
*
|
|
1643
1894
|
═══ IT IS THE EXISTENCE CHECK AND NOTHING MORE, WHICH IS WHAT CHANGED IN
|
|
1644
1895
|
worktrees-8. ═══ Resolving the working copy now CREATES a branch, a folder
|
|
1645
1896
|
and a row write, and every poll tick that loses the claim race would leave
|
|
@@ -1648,126 +1899,139 @@ async function resumeRun(client, tools, machineId, runId, afterPid) {
|
|
|
1648
1899
|
this position was ever asking, and the copy is made after the claim. */
|
|
1649
1900
|
if (afterPid === null)
|
|
1650
1901
|
checkoutForCodebase(await codebaseOfRun(client, runId), hostname());
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
p_machine_id: machineId,
|
|
1654
|
-
p_after_pid: afterPid,
|
|
1655
|
-
});
|
|
1656
|
-
if (error)
|
|
1657
|
-
throw new Error(`could not start run ${runId} again: ${error.message}`);
|
|
1658
|
-
const claimed = data?.[0];
|
|
1659
|
-
if (!claimed)
|
|
1660
|
-
return null;
|
|
1661
|
-
const level = claimed.run_level === 1 ? 1 : claimed.run_level === 2 ? 2 : 3;
|
|
1662
|
-
if (claimed.run_level !== level) {
|
|
1663
|
-
/* UNREACHABLE, AND STILL SETTLED, exactly as in `startChild`: the level
|
|
1664
|
-
column is checked at three, so there is no level here this cannot spawn.
|
|
1665
|
-
The claim has already happened, so leaving it would strand the run for a
|
|
1666
|
-
whole grace window before anything looked at it again. */
|
|
1667
|
-
const why = `run ${runId} is at level ${claimed.run_level}, which cannot be spawned`;
|
|
1668
|
-
await giveUp(client, runId, why);
|
|
1669
|
-
throw new Error(why);
|
|
1670
|
-
}
|
|
1671
|
-
/* ONE RESOLUTION FOR BOTH PATHS, AFTER THE CLAIM. It used to fork on whether
|
|
1672
|
-
the sweep had already resolved a folder before the claim; since the copy is
|
|
1673
|
-
the card's own and making it writes, both paths make it here, in the branch
|
|
1674
|
-
that can end the run when it cannot be made. */
|
|
1675
|
-
let where;
|
|
1676
|
-
try {
|
|
1677
|
-
where = await workingDirectory(client, runId, level, false);
|
|
1678
|
-
}
|
|
1679
|
-
catch (error) {
|
|
1680
|
-
/* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
|
|
1681
|
-
═══ Constraint 6, and `startRearmed`'s own handling of the same two
|
|
1682
|
-
calls: `worktreeForCard()` is already careful about this and says so;
|
|
1683
|
-
`scratchDir()` is not, because it is `mkdirSync`, whose EACCES and
|
|
1684
|
-
ENOTDIR messages name the directory they failed on. So the machine's own
|
|
1685
|
-
error is kept for stderr and the record is told only what is true and
|
|
1686
|
-
shareable. */
|
|
1687
|
-
const stderrOnly = error instanceof Error ? error.message : String(error);
|
|
1688
|
-
const why = level === 1
|
|
1689
|
-
? 'this machine could not make the empty directory this runs in'
|
|
1690
|
-
: stderrOnly;
|
|
1691
|
-
/* ═══ AND IT ENDS THE WAY A RUN OF THIS LEVEL ENDS. ═══ It was
|
|
1692
|
-
`panel3_give_up` outright, which was right while only a dispatched run
|
|
1693
|
-
could reach this function and is a leak now that the retry brings level
|
|
1694
|
-
1 here: handing the person's message back mints a new run with its
|
|
1695
|
-
attempts at one, which is the bound the retry is under, undone by the
|
|
1696
|
-
one path that could not start. See `endRun`. */
|
|
1697
|
-
await endRun(client, level, runId, claimed.run_card_id, why);
|
|
1698
|
-
throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? stderrOnly : why}`);
|
|
1699
|
-
}
|
|
1700
|
-
/* WHAT IT SENT OTHERS TO DO, FROM THE RECORD. Level 3 has no `dispatch`, so it
|
|
1701
|
-
has no children to have: null says that, where an empty list would say it
|
|
1702
|
-
chose to send nobody.
|
|
1703
|
-
|
|
1704
|
-
═══ AND LEVEL 1 HAS THEM TOO, WHICH ONLY THE RETRY CAN REACH. ═══ This read
|
|
1705
|
-
used to be `level === 2`, which was correct only because nothing at level 1
|
|
1706
|
-
ever got here. Left alone it would hand a coordinator a null child list, and
|
|
1707
|
-
a coordinator that came back to no children dispatches its workers a second
|
|
1708
|
-
time: ux.md's single most expensive failure. `startRearmed` reads it the
|
|
1709
|
-
same way, for the same reason. */
|
|
1710
|
-
const children = level === 3 ? null : await childrenOf(client, runId);
|
|
1711
|
-
/* ═══ READ AGAIN ON EVERY START, WHICH IS THE WHOLE OF CONTRACT POINT 2. ═══
|
|
1712
|
-
A resumed or retried run is handed the rules AS THEY STAND NOW, not as they
|
|
1713
|
-
stood when it first began: a rule edited while the conversation was running
|
|
1714
|
-
governs the rest of it, and a rule deleted while it was running stops
|
|
1715
|
-
applying to it. That is only true because this read happens here rather than
|
|
1716
|
-
once, at the top of the run's life.
|
|
1717
|
-
|
|
1718
|
-
═══ AND IT ENDS THE WAY THIS PATH ALREADY ENDS. ═══ The claim has already
|
|
1719
|
-
happened, so a throw here would leave a run reading `running` with no
|
|
1720
|
-
process. `endRun` with the level fork is this path's own ending (see the cwd
|
|
1721
|
-
branch above for why level 1 may not simply be given up on), and the reason
|
|
1722
|
-
is a `returned()` message naming tables and columns, which carries no local
|
|
1723
|
-
path and is therefore shareable. */
|
|
1724
|
-
let rules;
|
|
1725
|
-
let settings;
|
|
1726
|
-
let pictures;
|
|
1902
|
+
let claimOperationId;
|
|
1903
|
+
const preparation = tools.work?.prepare();
|
|
1727
1904
|
try {
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
const
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
it
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1905
|
+
const response = recovered ?? await panelClaim(client, tools.work, 'panel3_resume', {
|
|
1906
|
+
p_run_id: runId,
|
|
1907
|
+
p_machine_id: machineId,
|
|
1908
|
+
p_after_pid: afterPid,
|
|
1909
|
+
});
|
|
1910
|
+
const { data, error, operationId } = response;
|
|
1911
|
+
claimOperationId = operationId;
|
|
1912
|
+
if (error)
|
|
1913
|
+
throw new Error(`could not start run ${runId} again: ${error.message}`);
|
|
1914
|
+
const claimed = data?.[0];
|
|
1915
|
+
if (!claimed)
|
|
1916
|
+
return null;
|
|
1917
|
+
const attempt = capturedPanelAttempt(data ?? [], runId, tools.work);
|
|
1918
|
+
const level = claimed.run_level === 1 ? 1 : claimed.run_level === 2 ? 2 : 3;
|
|
1919
|
+
if (claimed.run_level !== level) {
|
|
1920
|
+
/* UNREACHABLE, AND STILL SETTLED, exactly as in `startChild`: the level
|
|
1921
|
+
column is checked at three, so there is no level here this cannot spawn.
|
|
1922
|
+
The claim has already happened, so leaving it would strand the run for a
|
|
1923
|
+
whole grace window before anything looked at it again. */
|
|
1924
|
+
const why = `run ${runId} is at level ${claimed.run_level}, which cannot be spawned`;
|
|
1925
|
+
await giveUp(client, runId, why, undefined, attempt);
|
|
1926
|
+
throw new Error(why);
|
|
1927
|
+
}
|
|
1928
|
+
/* ONE RESOLUTION FOR BOTH PATHS, AFTER THE CLAIM. It used to fork on whether
|
|
1929
|
+
the sweep had already resolved a folder before the claim; since the copy is
|
|
1930
|
+
the card's own and making it writes, both paths make it here, in the branch
|
|
1931
|
+
that can end the run when it cannot be made. */
|
|
1932
|
+
let where;
|
|
1933
|
+
try {
|
|
1934
|
+
where = await workingDirectory(client, runId, level, false);
|
|
1935
|
+
}
|
|
1936
|
+
catch (error) {
|
|
1937
|
+
/* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
|
|
1938
|
+
═══ Constraint 6, and `startRearmed`'s own handling of the same two
|
|
1939
|
+
calls: `worktreeForCard()` is already careful about this and says so;
|
|
1940
|
+
`scratchDir()` is not, because it is `mkdirSync`, whose EACCES and
|
|
1941
|
+
ENOTDIR messages name the directory they failed on. So the machine's own
|
|
1942
|
+
error is kept for stderr and the record is told only what is true and
|
|
1943
|
+
shareable. */
|
|
1944
|
+
const stderrOnly = error instanceof Error ? error.message : String(error);
|
|
1945
|
+
const why = level === 1
|
|
1946
|
+
? 'this machine could not make the empty directory this runs in'
|
|
1947
|
+
: stderrOnly;
|
|
1948
|
+
/* ═══ AND IT ENDS THE WAY A RUN OF THIS LEVEL ENDS. ═══ It was
|
|
1949
|
+
`panel3_give_up` outright, which was right while only a dispatched run
|
|
1950
|
+
could reach this function and is a leak now that the retry brings level
|
|
1951
|
+
1 here: handing the person's message back mints a new run with its
|
|
1952
|
+
attempts at one, which is the bound the retry is under, undone by the
|
|
1953
|
+
one path that could not start. See `endRun`. */
|
|
1954
|
+
await endRun(client, level, runId, claimed.run_card_id, why, attempt);
|
|
1955
|
+
throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? stderrOnly : why}`);
|
|
1956
|
+
}
|
|
1957
|
+
/* WHAT IT SENT OTHERS TO DO, FROM THE RECORD. Level 3 has no `dispatch`, so it
|
|
1958
|
+
has no children to have: null says that, where an empty list would say it
|
|
1959
|
+
chose to send nobody.
|
|
1960
|
+
*
|
|
1961
|
+
═══ AND LEVEL 1 HAS THEM TOO, WHICH ONLY THE RETRY CAN REACH. ═══ This read
|
|
1962
|
+
used to be `level === 2`, which was correct only because nothing at level 1
|
|
1963
|
+
ever got here. Left alone it would hand a coordinator a null child list, and
|
|
1964
|
+
a coordinator that came back to no children dispatches its workers a second
|
|
1965
|
+
time: ux.md's single most expensive failure. `startRearmed` reads it the
|
|
1966
|
+
same way, for the same reason. */
|
|
1967
|
+
const children = level === 3 ? null : await childrenOf(client, runId);
|
|
1968
|
+
/* ═══ READ AGAIN ON EVERY START, WHICH IS THE WHOLE OF CONTRACT POINT 2. ═══
|
|
1969
|
+
A resumed or retried run is handed the rules AS THEY STAND NOW, not as they
|
|
1970
|
+
stood when it first began: a rule edited while the conversation was running
|
|
1971
|
+
governs the rest of it, and a rule deleted while it was running stops
|
|
1972
|
+
applying to it. That is only true because this read happens here rather than
|
|
1973
|
+
once, at the top of the run's life.
|
|
1974
|
+
*
|
|
1975
|
+
═══ AND IT ENDS THE WAY THIS PATH ALREADY ENDS. ═══ The claim has already
|
|
1976
|
+
happened, so a throw here would leave a run reading `running` with no
|
|
1977
|
+
process. `endRun` with the level fork is this path's own ending (see the cwd
|
|
1978
|
+
branch above for why level 1 may not simply be given up on), and the reason
|
|
1979
|
+
is a `returned()` message naming tables and columns, which carries no local
|
|
1980
|
+
path and is therefore shareable. */
|
|
1981
|
+
let rules;
|
|
1982
|
+
let settings;
|
|
1983
|
+
let pictures;
|
|
1984
|
+
try {
|
|
1985
|
+
rules = await standingRulesFor(client, runId);
|
|
1986
|
+
settings = await settingsForRun(client, runId);
|
|
1987
|
+
/* ═══ WRITTEN AGAIN ON EVERY START, LIKE THE RULES. ═══ A resumed process is
|
|
1988
|
+
a NEW process with a new copy of the working directory, so the files a
|
|
1989
|
+
previous one was handed are not there any more, and the stored brief this
|
|
1990
|
+
path replays cannot carry a path that was not known when it was written. */
|
|
1991
|
+
pictures = await picturesOnDisk(client, claimed.run_card_id, where, level);
|
|
1992
|
+
}
|
|
1993
|
+
catch (error) {
|
|
1994
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
1995
|
+
await endRun(client, level, runId, claimed.run_card_id, why, attempt);
|
|
1996
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
1997
|
+
}
|
|
1998
|
+
/* ═══ WHY IT DIED IS WHAT DIFFERS, AND IT IS TOLD THE TRUTH ABOUT IT. ═══
|
|
1999
|
+
`resumePrompt` opens by saying the machine went down, which is true of the
|
|
2000
|
+
sweep and false of a retry: the daemon that watched this harness exit is
|
|
2001
|
+
still running. See `retryPrompt`. */
|
|
2002
|
+
const started = await startTrackedAgent(client, runId, preparation, tools.work, attempt, withStandingRules(rules, afterPid === null
|
|
2003
|
+
? resumePrompt(claimed.run_brief, claimed.run_report, children)
|
|
2004
|
+
: retryPrompt(claimed.run_brief, claimed.run_report, children), where.block, [], pictures), level, tools.urlFor(runId), where.cwd, undefined, settings);
|
|
2005
|
+
if (started.pid === null) {
|
|
2006
|
+
if (started.interrupted?.())
|
|
2007
|
+
throw new Error('Work was interrupted by the service command.');
|
|
2008
|
+
/* THE CLAIM HAPPENED AND NO PROCESS DID, which is the one shape the record
|
|
2009
|
+
must never be left in quietly. Same handling as a dispatch that could not
|
|
2010
|
+
start: the reason is read off the settled answer and the run is ended with
|
|
2011
|
+
it. */
|
|
2012
|
+
const answer = await started.answered;
|
|
2013
|
+
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
2014
|
+
// The same level fork, for the same reason. See `endRun`.
|
|
2015
|
+
await endRun(client, level, runId, claimed.run_card_id, reason, attempt);
|
|
2016
|
+
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
2017
|
+
}
|
|
2018
|
+
out(`resume run ${runId} level ${level} pid ${started.pid} `
|
|
2019
|
+
+ `${claimed.run_report === null ? 'no report to carry' : 'carrying its report'}`);
|
|
2020
|
+
try {
|
|
2021
|
+
await recordProcess(client, runId, started.pid, undefined, undefined, started.attempt);
|
|
2022
|
+
}
|
|
2023
|
+
catch (error) {
|
|
2024
|
+
// Said, not fatal, as everywhere else: the agent is running and killing it
|
|
2025
|
+
// over a bookkeeping write would cost the user the work a second time.
|
|
2026
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
2027
|
+
}
|
|
2028
|
+
return { settled: settle(client, tools, machineId, level, runId, claimed.run_card_id, started) };
|
|
1764
2029
|
}
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
2030
|
+
finally {
|
|
2031
|
+
preparation?.finish();
|
|
2032
|
+
if (claimOperationId)
|
|
2033
|
+
tools.work?.finishRpcs([claimOperationId]);
|
|
1769
2034
|
}
|
|
1770
|
-
return { settled: settle(client, tools, machineId, level, runId, claimed.run_card_id, started) };
|
|
1771
2035
|
}
|
|
1772
2036
|
/**
|
|
1773
2037
|
* ═══ ONE RUN, STARTED AGAIN BECAUSE SOMETHING IT WAS WAITING ON EXISTS NOW. ═══
|
|
@@ -1796,105 +2060,114 @@ async function resumeRun(client, tools, machineId, runId, afterPid) {
|
|
|
1796
2060
|
* and in `cs show`, rather than quietly hoping the next poll finds it.
|
|
1797
2061
|
*/
|
|
1798
2062
|
async function startRearmed(client, tools, machineId, row) {
|
|
1799
|
-
const
|
|
1800
|
-
if (row.run_level !== level) {
|
|
1801
|
-
const why = `run ${row.run_id} is at level ${row.run_level}, which cannot be spawned`;
|
|
1802
|
-
await giveUp(client, row.run_id, why);
|
|
1803
|
-
throw new Error(why);
|
|
1804
|
-
}
|
|
1805
|
-
let where;
|
|
2063
|
+
const preparation = tools.work?.prepare(row.run_id);
|
|
1806
2064
|
try {
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
? await
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
try {
|
|
1858
|
-
rules = await standingRulesFor(client, row.run_id);
|
|
1859
|
-
settings = await settingsForRun(client, row.run_id);
|
|
1860
|
-
pictures = await picturesOnDisk(client, row.run_card_id, where, level);
|
|
1861
|
-
}
|
|
1862
|
-
catch (error) {
|
|
1863
|
-
const why = error instanceof Error ? error.message : String(error);
|
|
1864
|
-
await endRun(client, level, row.run_id, row.run_card_id, why);
|
|
1865
|
-
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
1866
|
-
}
|
|
1867
|
-
const started = startAgent(withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id), where.cwd, undefined, settings);
|
|
1868
|
-
if (started.pid === null) {
|
|
1869
|
-
const answer = await started.answered;
|
|
1870
|
-
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
1871
|
-
// The same level fork, for the same reason. See `endRun`.
|
|
1872
|
-
await endRun(client, level, row.run_id, row.run_card_id, reason);
|
|
1873
|
-
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
1874
|
-
}
|
|
1875
|
-
out(`rearm run ${row.run_id} level ${level} pid ${started.pid} `
|
|
1876
|
-
+ `${row.ask_id === null
|
|
1877
|
-
? 'to read back everybody it sent'
|
|
2065
|
+
const attempt = capturedPanelAttempt([row], row.run_id, tools.work);
|
|
2066
|
+
const level = row.run_level === 1 ? 1 : row.run_level === 2 ? 2 : 3;
|
|
2067
|
+
if (row.run_level !== level) {
|
|
2068
|
+
const why = `run ${row.run_id} is at level ${row.run_level}, which cannot be spawned`;
|
|
2069
|
+
await giveUp(client, row.run_id, why, undefined, attempt);
|
|
2070
|
+
throw new Error(why);
|
|
2071
|
+
}
|
|
2072
|
+
let where;
|
|
2073
|
+
try {
|
|
2074
|
+
where = await workingDirectory(client, row.run_id, level, false);
|
|
2075
|
+
}
|
|
2076
|
+
catch (error) {
|
|
2077
|
+
/* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
|
|
2078
|
+
═══ Constraint 6. `workingCopy()` is already careful about this and says
|
|
2079
|
+
so; `scratchDir()` is not — it is `mkdirSync`, whose EACCES and ENOTDIR
|
|
2080
|
+
messages name the directory they failed on — and `failed_because` is a
|
|
2081
|
+
column `cs show` prints. So the machine's own error is kept for stderr
|
|
2082
|
+
and the record is told only what is true and shareable. */
|
|
2083
|
+
const said = error instanceof Error ? error.message : String(error);
|
|
2084
|
+
const why = level === 1
|
|
2085
|
+
? 'this machine could not make the empty directory this runs in'
|
|
2086
|
+
: said;
|
|
2087
|
+
// The level fork, which this path needs for the same reason `resumeRun`'s
|
|
2088
|
+
// two do: a re-arm serves level 1, and a level 1 run holds the person's
|
|
2089
|
+
// message. See `endRun`.
|
|
2090
|
+
await endRun(client, level, row.run_id, row.run_card_id, why, attempt);
|
|
2091
|
+
throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? said : why}`);
|
|
2092
|
+
}
|
|
2093
|
+
/* WHAT IT SENT OTHERS TO DO AND WHAT THEY WROTE, FROM THE RECORD, AS LATE AS
|
|
2094
|
+
POSSIBLE. Level 3 has no `dispatch`, so it has no children to have: null
|
|
2095
|
+
says that, where an empty list would say it chose to send nobody. */
|
|
2096
|
+
const children = level === 3 ? null : await childrenOf(client, row.run_id);
|
|
2097
|
+
const deliveredArtifact = row.ask_id !== null && row.mine
|
|
2098
|
+
? await rearmedArtifactAnswer(client, row.ask_id)
|
|
2099
|
+
: null;
|
|
2100
|
+
/* ═══ THE MERGE HAPPENS HERE, OFF THE MARK ON THE ASK, BEFORE THE AGENT THAT
|
|
2101
|
+
WILL SPEAK ABOUT IT IS STARTED. ═══ The re-arm is the last thing that runs
|
|
2102
|
+
before the spawn, which is why it is the only place the outcome can reach
|
|
2103
|
+
the prompt. See `landCardWork`. */
|
|
2104
|
+
const offered = row.ask_id !== null && row.mine
|
|
2105
|
+
? await landingOffer(client, row.ask_id)
|
|
2106
|
+
: null;
|
|
2107
|
+
const landing = offered === null ? null : landCardWork(where, offered);
|
|
2108
|
+
/* ═══ THREE REASONS, AND THE ROW SAYS WHICH. ═══ No question is ux.md's third
|
|
2109
|
+
re-arm: everybody it sent has finished, and it is started to read them back.
|
|
2110
|
+
`children` cannot be null on that path — only a run with children is ever
|
|
2111
|
+
claimed for it — and the prompt takes the list rather than the maybe-list so
|
|
2112
|
+
that is a fact of the signature rather than of a comment. */
|
|
2113
|
+
const prompt = row.ask_id === null
|
|
2114
|
+
? readBackPrompt(row.run_brief, row.run_report, children ?? [])
|
|
1878
2115
|
: row.mine
|
|
1879
|
-
? '
|
|
1880
|
-
:
|
|
1881
|
-
|
|
1882
|
-
|
|
2116
|
+
? answerPrompt(row.run_brief, row.run_report, children, row.question ?? '', row.answer ?? '', deliveredArtifact, landing)
|
|
2117
|
+
: escalationPrompt(row.run_brief, row.run_report, children, row.ask_id, row.question ?? '');
|
|
2118
|
+
/* CURRENT AT THIS ACTIVATION, exactly as on the resume path, and ended the
|
|
2119
|
+
same way: the re-arm's claim has already happened, so a failure here ends
|
|
2120
|
+
the run with its reason rather than leaving it claimed with no process. */
|
|
2121
|
+
let rules;
|
|
2122
|
+
let settings;
|
|
2123
|
+
let pictures;
|
|
2124
|
+
try {
|
|
2125
|
+
rules = await standingRulesFor(client, row.run_id);
|
|
2126
|
+
settings = await settingsForRun(client, row.run_id);
|
|
2127
|
+
pictures = await picturesOnDisk(client, row.run_card_id, where, level);
|
|
2128
|
+
}
|
|
2129
|
+
catch (error) {
|
|
2130
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
2131
|
+
await endRun(client, level, row.run_id, row.run_card_id, why, attempt);
|
|
2132
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
2133
|
+
}
|
|
2134
|
+
const started = await startTrackedAgent(client, row.run_id, preparation, tools.work, attempt, withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id), where.cwd, undefined, settings);
|
|
2135
|
+
if (started.pid === null) {
|
|
2136
|
+
if (started.interrupted?.())
|
|
2137
|
+
throw new Error('Work was interrupted by the service command.');
|
|
2138
|
+
const answer = await started.answered;
|
|
2139
|
+
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
2140
|
+
// The same level fork, for the same reason. See `endRun`.
|
|
2141
|
+
await endRun(client, level, row.run_id, row.run_card_id, reason, attempt);
|
|
2142
|
+
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
2143
|
+
}
|
|
2144
|
+
out(`rearm run ${row.run_id} level ${level} pid ${started.pid} `
|
|
2145
|
+
+ `${row.ask_id === null
|
|
2146
|
+
? 'to read back everybody it sent'
|
|
2147
|
+
: row.mine
|
|
2148
|
+
? 'with the answer to its own question'
|
|
2149
|
+
: 'with a question it has to settle'}`);
|
|
2150
|
+
try {
|
|
2151
|
+
await recordProcess(client, row.run_id, started.pid, undefined, undefined, started.attempt);
|
|
2152
|
+
}
|
|
2153
|
+
catch (error) {
|
|
2154
|
+
// Said, not fatal, as everywhere else: the agent is running and killing it
|
|
2155
|
+
// over a bookkeeping write would cost the user the work.
|
|
2156
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
2157
|
+
}
|
|
2158
|
+
/* ═══ AND IT SPEAKS TO THE CARD ONLY IF IT IS DOING THE PERSON'S WORK. ═══
|
|
2159
|
+
Exactly the distinction ux.md draws: a run carrying on with its own work has
|
|
2160
|
+
something to say when it finishes, and a run started only to settle somebody
|
|
2161
|
+
else's question does not. A run started to read back everybody it sent is
|
|
2162
|
+
the first kind and the clearest case of it — that reply IS the answer to the
|
|
2163
|
+
request. See `settle`. */
|
|
2164
|
+
return {
|
|
2165
|
+
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, row.ask_id === null || !!row.mine),
|
|
2166
|
+
};
|
|
2167
|
+
}
|
|
2168
|
+
finally {
|
|
2169
|
+
preparation?.finish();
|
|
1883
2170
|
}
|
|
1884
|
-
catch (error) {
|
|
1885
|
-
// Said, not fatal, as everywhere else: the agent is running and killing it
|
|
1886
|
-
// over a bookkeeping write would cost the user the work.
|
|
1887
|
-
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1888
|
-
}
|
|
1889
|
-
/* ═══ AND IT SPEAKS TO THE CARD ONLY IF IT IS DOING THE PERSON'S WORK. ═══
|
|
1890
|
-
Exactly the distinction ux.md draws: a run carrying on with its own work has
|
|
1891
|
-
something to say when it finishes, and a run started only to settle somebody
|
|
1892
|
-
else's question does not. A run started to read back everybody it sent is
|
|
1893
|
-
the first kind and the clearest case of it — that reply IS the answer to the
|
|
1894
|
-
request. See `settle`. */
|
|
1895
|
-
return {
|
|
1896
|
-
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, row.ask_id === null || !!row.mine),
|
|
1897
|
-
};
|
|
1898
2171
|
}
|
|
1899
2172
|
/**
|
|
1900
2173
|
* The runs one run dispatched, AND WHAT EACH OF THEM WROTE, in the words the
|
|
@@ -2047,140 +2320,159 @@ export function resumableOwnerSessionId(candidate, machineId, machineHarness) {
|
|
|
2047
2320
|
? local.nativeSessionId
|
|
2048
2321
|
: undefined;
|
|
2049
2322
|
}
|
|
2050
|
-
async function activateOwner(client, tools, machineId, runId, afterProcessToken = null, afterPid = null) {
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
return null;
|
|
2054
|
-
// Existing conversations retain their agent; an explicit hand-back follows
|
|
2055
|
-
// the new selection. Never reuse a native session across different agents.
|
|
2056
|
-
if (candidate.handed_back_at === null && candidate.machine_id !== machineId)
|
|
2057
|
-
return null;
|
|
2058
|
-
const machineHarness = candidate.handed_back_at === null && candidate.harness !== null
|
|
2059
|
-
? harness({ CTRL_SPC_V3_AGENT: candidate.harness })
|
|
2060
|
-
: await selectedHarness(client, machineId);
|
|
2061
|
-
const resumeSessionId = resumableOwnerSessionId(candidate, machineId, machineHarness);
|
|
2062
|
-
/* ═══ THE EXISTENCE CHECK BEFORE THE CLAIM, AND THE COPY AFTER IT. ═══ This
|
|
2063
|
-
was the whole resolution, which was right while resolving meant reading a
|
|
2064
|
-
folder out of a file. Since worktrees-8 it also CREATES one, and a daemon
|
|
2065
|
-
that lost the activation race would leave a branch and a folder behind for
|
|
2066
|
-
an owner it never activated. The refusal this position exists for — a
|
|
2067
|
-
machine that does not have this codebase must not take the activation —
|
|
2068
|
-
is unchanged, because it is the located checkout that answers it. */
|
|
2069
|
-
if (candidate.codebase_id !== null) {
|
|
2070
|
-
checkoutForCodebase(await codebaseOfRun(client, candidate.id), hostname());
|
|
2071
|
-
}
|
|
2072
|
-
const { data, error } = await client.rpc('panel3_take_owner_activation', {
|
|
2073
|
-
p_run_id: runId,
|
|
2074
|
-
p_machine_id: machineId,
|
|
2075
|
-
p_agent: machineHarness,
|
|
2076
|
-
p_after_process_token: afterProcessToken,
|
|
2077
|
-
p_after_pid: afterPid,
|
|
2078
|
-
});
|
|
2079
|
-
if (error)
|
|
2080
|
-
throw new Error(`could not activate conversation owner ${runId}: ${error.message}`);
|
|
2081
|
-
const claimed = data?.[0];
|
|
2082
|
-
if (!claimed)
|
|
2083
|
-
return null;
|
|
2084
|
-
const [events, children] = await Promise.all([
|
|
2085
|
-
ownerConversation(client, claimed.run_card_id, new Set(claimed.turn_ids ?? [])),
|
|
2086
|
-
ownerChildren(client, runId),
|
|
2087
|
-
]);
|
|
2088
|
-
const currentArtifactAnswer = deliveredArtifactAnswer(events, claimed);
|
|
2089
|
-
const delivered = claimed.ask_id === null ? null : {
|
|
2090
|
-
id: claimed.ask_id,
|
|
2091
|
-
question: claimed.question ?? '(question unavailable)',
|
|
2092
|
-
answer: claimed.answer,
|
|
2093
|
-
mine: claimed.mine === true,
|
|
2094
|
-
artifactAnswer: currentArtifactAnswer,
|
|
2095
|
-
};
|
|
2096
|
-
/* ═══ AFTER THE CLAIM, SO THE ENDING IS THE ONE THIS PATH HAS. ═══ Every
|
|
2097
|
-
failure below the claim ends the activation with `giveUp` and its process
|
|
2098
|
-
token; a throw above it would merely be an activation that did not happen.
|
|
2099
|
-
|
|
2100
|
-
═══ AND IT MATTERS MOST HERE. ═══ This is the owner, which lives for the
|
|
2101
|
-
whole card and whose native session is RESUMED, so it is the one agent that
|
|
2102
|
-
can be running while a person edits or deletes a rule. Reading at every
|
|
2103
|
-
activation is what makes an edit govern the rest of the conversation, and
|
|
2104
|
-
the block's own supersession sentence is what makes a DELETION take effect
|
|
2105
|
-
in a session that still holds the older copy.
|
|
2106
|
-
|
|
2107
|
-
═══ AND IT IS BEFORE THE PROMPT SINCE worktrees-8 C1, because the prompt now
|
|
2108
|
-
says what the product DID with the person's answer, and the landing needs
|
|
2109
|
-
the card's copy. Nothing in the prompt depended on it before. */
|
|
2110
|
-
let rules;
|
|
2111
|
-
let settings;
|
|
2112
|
-
/* THE CARD'S COPY IS MADE IN THE SAME WINDOW AND UNDER THE SAME ENDING, for
|
|
2113
|
-
the reason the check above gives: it writes, so it happens after the claim,
|
|
2114
|
-
and a failure to make it is an activation that ends rather than one that
|
|
2115
|
-
sits `running` with no process. */
|
|
2116
|
-
let where;
|
|
2117
|
-
/* ═══ AND WHAT IS ATTACHED, READ IN THE SAME WINDOW AND UNDER THE SAME
|
|
2118
|
-
ENDING. ═══ attaching-after-the-fact-10: a person may attach to a card that
|
|
2119
|
-
is already running, and the owner's brief is immutable, so the only account
|
|
2120
|
-
of attachments it would otherwise get is the one frozen at dispatch. Read
|
|
2121
|
-
at every activation, exactly as the rules are, and for the same reason a
|
|
2122
|
-
failed read ends the activation rather than continuing: "nothing is
|
|
2123
|
-
attached" is a different card from the one the person sent.
|
|
2124
|
-
|
|
2125
|
-
THE CODEBASE LINES ARE DROPPED. `whatWasAttached` partitions those into a
|
|
2126
|
-
section whose own text says "its one codebase is named separately below",
|
|
2127
|
-
a forward reference to something `workBrief` supplies and an owner
|
|
2128
|
-
activation does not. This block carries the PERSON's attachments; where the
|
|
2129
|
-
owner is working is `where.block`'s answer. */
|
|
2130
|
-
let attached;
|
|
2131
|
-
let pictures;
|
|
2323
|
+
async function activateOwner(client, tools, machineId, runId, afterProcessToken = null, afterPid = null, recovered) {
|
|
2324
|
+
let claimOperationId;
|
|
2325
|
+
const preparation = tools.work?.prepare();
|
|
2132
2326
|
try {
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
const
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2327
|
+
const candidate = await ownerCandidate(client, runId);
|
|
2328
|
+
if (!candidate)
|
|
2329
|
+
return null;
|
|
2330
|
+
// Existing conversations retain their agent; an explicit hand-back follows
|
|
2331
|
+
// the new selection. Never reuse a native session across different agents.
|
|
2332
|
+
if (candidate.handed_back_at === null && candidate.machine_id !== machineId)
|
|
2333
|
+
return null;
|
|
2334
|
+
const machineHarness = candidate.handed_back_at === null && candidate.harness !== null
|
|
2335
|
+
? harness({ CTRL_SPC_V3_AGENT: candidate.harness })
|
|
2336
|
+
: await selectedHarness(client, machineId);
|
|
2337
|
+
if (recovered?.nativeContext && (recovered.nativeContext.harness !== machineHarness || recovered.nativeContext.processToken !== null && typeof recovered.nativeContext.processToken !== 'string' || recovered.nativeContext.nativeSessionId !== undefined && typeof recovered.nativeContext.nativeSessionId !== 'string'))
|
|
2338
|
+
throw new Error('The saved native conversation context is invalid.');
|
|
2339
|
+
const resumeSessionId = recovered?.nativeContext
|
|
2340
|
+
? resumableOwnerSessionId({ ...candidate, process_token: recovered.nativeContext.processToken }, machineId, recovered.nativeContext.harness)
|
|
2341
|
+
: resumableOwnerSessionId(candidate, machineId, machineHarness);
|
|
2342
|
+
if (recovered?.nativeContext?.nativeSessionId && resumeSessionId !== recovered.nativeContext.nativeSessionId)
|
|
2343
|
+
throw new Error('The saved native conversation is no longer available for this attempt.');
|
|
2344
|
+
/* ═══ THE EXISTENCE CHECK BEFORE THE CLAIM, AND THE COPY AFTER IT. ═══ This
|
|
2345
|
+
was the whole resolution, which was right while resolving meant reading a
|
|
2346
|
+
folder out of a file. Since worktrees-8 it also CREATES one, and a daemon
|
|
2347
|
+
that lost the activation race would leave a branch and a folder behind for
|
|
2348
|
+
an owner it never activated. The refusal this position exists for — a
|
|
2349
|
+
machine that does not have this codebase must not take the activation —
|
|
2350
|
+
is unchanged, because it is the located checkout that answers it. */
|
|
2351
|
+
if (candidate.codebase_id !== null) {
|
|
2352
|
+
checkoutForCodebase(await codebaseOfRun(client, candidate.id), hostname());
|
|
2353
|
+
}
|
|
2354
|
+
const response = recovered ?? await panelClaim(client, tools.work, 'panel3_take_owner_activation', {
|
|
2355
|
+
p_run_id: runId,
|
|
2356
|
+
p_machine_id: machineId,
|
|
2357
|
+
p_agent: machineHarness,
|
|
2358
|
+
p_after_process_token: afterProcessToken,
|
|
2359
|
+
p_after_pid: afterPid,
|
|
2360
|
+
}, { nativeSessionId: resumeSessionId, processToken: candidate.process_token, harness: machineHarness });
|
|
2361
|
+
const { data, error, operationId } = response;
|
|
2362
|
+
claimOperationId = operationId;
|
|
2363
|
+
if (error)
|
|
2364
|
+
throw new Error(`could not activate conversation owner ${runId}: ${error.message}`);
|
|
2365
|
+
const claimed = data?.[0];
|
|
2366
|
+
if (!claimed)
|
|
2367
|
+
return null;
|
|
2368
|
+
const attempt = capturedPanelAttempt(data ?? [], runId, tools.work);
|
|
2369
|
+
const [events, children] = await Promise.all([
|
|
2370
|
+
ownerConversation(client, claimed.run_card_id, new Set(claimed.turn_ids ?? [])),
|
|
2371
|
+
ownerChildren(client, runId),
|
|
2372
|
+
]);
|
|
2373
|
+
const currentArtifactAnswer = deliveredArtifactAnswer(events, claimed);
|
|
2374
|
+
const delivered = claimed.ask_id === null ? null : {
|
|
2375
|
+
id: claimed.ask_id,
|
|
2376
|
+
question: claimed.question ?? '(question unavailable)',
|
|
2377
|
+
answer: claimed.answer,
|
|
2378
|
+
mine: claimed.mine === true,
|
|
2379
|
+
artifactAnswer: currentArtifactAnswer,
|
|
2380
|
+
};
|
|
2381
|
+
/* ═══ AFTER THE CLAIM, SO THE ENDING IS THE ONE THIS PATH HAS. ═══ Every
|
|
2382
|
+
failure below the claim ends the activation with `giveUp` and its process
|
|
2383
|
+
token; a throw above it would merely be an activation that did not happen.
|
|
2384
|
+
*
|
|
2385
|
+
═══ AND IT MATTERS MOST HERE. ═══ This is the owner, which lives for the
|
|
2386
|
+
whole card and whose native session is RESUMED, so it is the one agent that
|
|
2387
|
+
can be running while a person edits or deletes a rule. Reading at every
|
|
2388
|
+
activation is what makes an edit govern the rest of the conversation, and
|
|
2389
|
+
the block's own supersession sentence is what makes a DELETION take effect
|
|
2390
|
+
in a session that still holds the older copy.
|
|
2391
|
+
*
|
|
2392
|
+
═══ AND IT IS BEFORE THE PROMPT SINCE worktrees-8 C1, because the prompt now
|
|
2393
|
+
says what the product DID with the person's answer, and the landing needs
|
|
2394
|
+
the card's copy. Nothing in the prompt depended on it before. */
|
|
2395
|
+
let rules;
|
|
2396
|
+
let settings;
|
|
2397
|
+
/* THE CARD'S COPY IS MADE IN THE SAME WINDOW AND UNDER THE SAME ENDING, for
|
|
2398
|
+
the reason the check above gives: it writes, so it happens after the claim,
|
|
2399
|
+
and a failure to make it is an activation that ends rather than one that
|
|
2400
|
+
sits `running` with no process. */
|
|
2401
|
+
let where;
|
|
2402
|
+
/* ═══ AND WHAT IS ATTACHED, READ IN THE SAME WINDOW AND UNDER THE SAME
|
|
2403
|
+
ENDING. ═══ attaching-after-the-fact-10: a person may attach to a card that
|
|
2404
|
+
is already running, and the owner's brief is immutable, so the only account
|
|
2405
|
+
of attachments it would otherwise get is the one frozen at dispatch. Read
|
|
2406
|
+
at every activation, exactly as the rules are, and for the same reason a
|
|
2407
|
+
failed read ends the activation rather than continuing: "nothing is
|
|
2408
|
+
attached" is a different card from the one the person sent.
|
|
2409
|
+
*
|
|
2410
|
+
THE CODEBASE LINES ARE DROPPED. `whatWasAttached` partitions those into a
|
|
2411
|
+
section whose own text says "its one codebase is named separately below",
|
|
2412
|
+
a forward reference to something `workBrief` supplies and an owner
|
|
2413
|
+
activation does not. This block carries the PERSON's attachments; where the
|
|
2414
|
+
owner is working is `where.block`'s answer. */
|
|
2415
|
+
let attached;
|
|
2416
|
+
let pictures;
|
|
2417
|
+
try {
|
|
2418
|
+
where = await ownerDirectory(client, candidate);
|
|
2419
|
+
rules = await standingRulesFor(client, runId);
|
|
2420
|
+
settings = await settingsForRun(client, runId);
|
|
2421
|
+
attached = whatWasAttached((await attachmentsFor(client, claimed.run_card_id))
|
|
2422
|
+
.filter((line) => !line.startsWith('codebase ')));
|
|
2423
|
+
/* ═══ THE OWNER'S OWN ACTIVATION, WHICH IS WHERE MOST PICTURES ARRIVE. ═══
|
|
2424
|
+
The person sends one and this is the run that reads it. Written on every
|
|
2425
|
+
activation rather than once, so an owner resumed into an existing native
|
|
2426
|
+
conversation is told again about a directory a previous process wrote. */
|
|
2427
|
+
pictures = await picturesOnDisk(client, claimed.run_card_id, where, 2);
|
|
2428
|
+
}
|
|
2429
|
+
catch (error) {
|
|
2430
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
2431
|
+
await giveUp(client, runId, why, claimed.process_token, attempt);
|
|
2432
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
2433
|
+
}
|
|
2434
|
+
/* THE MERGE, OFF THE MARK ON THE ASK, BEFORE THE AGENT THAT WILL SPEAK ABOUT
|
|
2435
|
+
IT IS STARTED. The owner reaches an answer by this route as often as by the
|
|
2436
|
+
re-arm, which is why the artifact answer is read on both and this is too. */
|
|
2437
|
+
const offered = claimed.ask_id !== null && claimed.mine === true
|
|
2438
|
+
? await landingOffer(client, claimed.ask_id)
|
|
2439
|
+
: null;
|
|
2440
|
+
const landing = offered === null ? null : landCardWork(where, offered);
|
|
2161
2441
|
/* ═══ A PROCESS OF ITS OWN ENDED BEFORE IT FINISHED. ═══ `afterPid` is the
|
|
2162
2442
|
fact, and it is non-null on all three paths that follow one: a harness
|
|
2163
2443
|
that crashed, a machine that went down, and now a person's correction.
|
|
2164
2444
|
The sentences it adds say what to do and never why, because those three
|
|
2165
2445
|
are not the same event and `prompt.ts` exists to stop an agent being
|
|
2166
2446
|
told an untrue reason for its own restart. */
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
await
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2447
|
+
const prompt = resumeSessionId
|
|
2448
|
+
? ownerContinuationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered, landing)
|
|
2449
|
+
: ownerActivationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered !== null && !delivered.mine
|
|
2450
|
+
? { id: delivered.id, question: delivered.question }
|
|
2451
|
+
: null, currentArtifactAnswer, afterPid !== null, landing);
|
|
2452
|
+
const started = await startTrackedAgent(client, runId, preparation, tools.work, attempt, withStandingRules(rules, prompt, where.block, attached, pictures), 2, tools.urlFor(runId, claimed.process_token), where.cwd, { ownerId: runId, ...(resumeSessionId ? { resumeSessionId } : {}) }, settings);
|
|
2453
|
+
if (started.pid === null) {
|
|
2454
|
+
if (started.interrupted?.())
|
|
2455
|
+
throw new Error('Work was interrupted by the service command.');
|
|
2456
|
+
const answer = await started.answered;
|
|
2457
|
+
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
2458
|
+
await giveUp(client, runId, reason, claimed.process_token, attempt);
|
|
2459
|
+
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
2460
|
+
}
|
|
2461
|
+
try {
|
|
2462
|
+
await recordProcess(client, runId, started.pid, undefined, claimed.process_token, started.attempt);
|
|
2463
|
+
}
|
|
2464
|
+
catch (error) {
|
|
2465
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
2466
|
+
}
|
|
2467
|
+
return {
|
|
2468
|
+
settled: settle(client, tools, machineId, 2, runId, claimed.run_card_id, started, true, claimed.process_token, ownerSessionLifecycle(started, runId, machineHarness, claimed.process_token, resumeSessionId)),
|
|
2469
|
+
};
|
|
2177
2470
|
}
|
|
2178
|
-
|
|
2179
|
-
|
|
2471
|
+
finally {
|
|
2472
|
+
preparation?.finish();
|
|
2473
|
+
if (claimOperationId)
|
|
2474
|
+
tools.work?.finishRpcs([claimOperationId]);
|
|
2180
2475
|
}
|
|
2181
|
-
return {
|
|
2182
|
-
settled: settle(client, tools, machineId, 2, runId, claimed.run_card_id, started, true, claimed.process_token, ownerSessionLifecycle(started, runId, machineHarness, claimed.process_token, resumeSessionId)),
|
|
2183
|
-
};
|
|
2184
2476
|
}
|
|
2185
2477
|
/** The only owner rows the normal poll may try, including an explicit hand-back move.
|
|
2186
2478
|
*
|
|
@@ -2256,7 +2548,8 @@ export function redirectedProcess(candidate, machineId,
|
|
|
2256
2548
|
/** The newest unaddressed person turn per card. MAX, never first-seen: a turn
|
|
2257
2549
|
* left unaddressed while a question was open would otherwise pin its card
|
|
2258
2550
|
* below `resumed_at` for good and nothing on it could ever redirect. */
|
|
2259
|
-
waiting, cardId, booted) {
|
|
2551
|
+
waiting, cardId, booted, executionHeld = false) {
|
|
2552
|
+
// A dead transport can still own live tools in the durable execution boundary.
|
|
2260
2553
|
// The claim's own five, mirrored.
|
|
2261
2554
|
if (candidate.state !== 'running' || candidate.ended_at !== null)
|
|
2262
2555
|
return null;
|
|
@@ -2288,7 +2581,7 @@ waiting, cardId, booted) {
|
|
|
2288
2581
|
if (new Date(said).getTime() <= new Date(candidate.resumed_at ?? candidate.started_at).getTime()) {
|
|
2289
2582
|
return null;
|
|
2290
2583
|
}
|
|
2291
|
-
return runProcessIsAlive(candidate, booted) ? candidate.pid : null;
|
|
2584
|
+
return executionHeld || runProcessIsAlive(candidate, booted) ? candidate.pid : null;
|
|
2292
2585
|
}
|
|
2293
2586
|
export function pendingOwnerSessionWithinGrace(candidate, now = Date.now()) {
|
|
2294
2587
|
const local = readOwnerSession(candidate.id);
|
|
@@ -2331,9 +2624,17 @@ async function takeOwnerActivations(client, tools, machineId, mine, hold) {
|
|
|
2331
2624
|
else and there is a dead process, a row still reading `running` and a card
|
|
2332
2625
|
still saying `working`, with nothing local to retry it. */
|
|
2333
2626
|
if (mine.has(candidate.id)) {
|
|
2334
|
-
const
|
|
2627
|
+
const attempt = recoveryAttempt(candidate);
|
|
2628
|
+
const executionHeld = tools.work?.heldAttempt(attempt) ?? false;
|
|
2629
|
+
// A run ID alone cannot authorize stopping a newer activation.
|
|
2630
|
+
if (tools.work && !executionHeld)
|
|
2631
|
+
continue;
|
|
2632
|
+
const pid = redirectedProcess({ ...candidate, card_state: candidate.card?.state ?? null }, machineId, waiting, candidate.card_id, bootedAt(), executionHeld);
|
|
2335
2633
|
if (pid !== null) {
|
|
2336
|
-
|
|
2634
|
+
if (executionHeld)
|
|
2635
|
+
await tools.work.stopHeldAttempt(attempt);
|
|
2636
|
+
else
|
|
2637
|
+
killTree({ pid, kill: (signal) => process.kill(pid, signal) });
|
|
2337
2638
|
/* ═══ SIGNALLED, NOT KILLED, AND THE WORD IS THE POINT. ═══ `killTree`
|
|
2338
2639
|
swallows a refused signal on both platforms, so saying "killed" would
|
|
2339
2640
|
claim a death this daemon never observed. A process that survives is
|
|
@@ -2344,6 +2645,8 @@ async function takeOwnerActivations(client, tools, machineId, mine, hold) {
|
|
|
2344
2645
|
}
|
|
2345
2646
|
continue;
|
|
2346
2647
|
}
|
|
2648
|
+
if (tools.work?.heldAttempt(recoveryAttempt(candidate)))
|
|
2649
|
+
continue;
|
|
2347
2650
|
if (pendingOwnerSessionWithinGrace(candidate))
|
|
2348
2651
|
continue;
|
|
2349
2652
|
if (candidate.handed_back_at !== null
|
|
@@ -2407,7 +2710,7 @@ function runProcessIsAlive(run, booted) {
|
|
|
2407
2710
|
* touched here. A current owner row, live PID, in-flight process, or fresh
|
|
2408
2711
|
* pid-null claim always defers cleanup. Stable state is removed only after the
|
|
2409
2712
|
* owner row disappears, becomes terminal, or no longer owns its card. */
|
|
2410
|
-
export async function reconcileOwnerSessions(client, machineId, _machineHarness, inFlightOwnerIds = new Set()) {
|
|
2713
|
+
export async function reconcileOwnerSessions(client, machineId, _machineHarness, inFlightOwnerIds = new Set(), work) {
|
|
2411
2714
|
const mappingIds = listOwnerSessionIds();
|
|
2412
2715
|
const homeIds = listPanel3CodexOwnerHomeIds();
|
|
2413
2716
|
const all = [...new Set([...mappingIds, ...homeIds])];
|
|
@@ -2424,7 +2727,11 @@ export async function reconcileOwnerSessions(client, machineId, _machineHarness,
|
|
|
2424
2727
|
for (const id of ids) {
|
|
2425
2728
|
if (inFlightOwnerIds.has(id))
|
|
2426
2729
|
continue;
|
|
2730
|
+
if (work?.heldLocalOwner(id))
|
|
2731
|
+
continue;
|
|
2427
2732
|
const row = byId.get(id);
|
|
2733
|
+
if (row && work?.heldAttempt(recoveryAttempt(row)))
|
|
2734
|
+
continue;
|
|
2428
2735
|
const localProcessInUse = !!row
|
|
2429
2736
|
&& row.machine_id === machineId
|
|
2430
2737
|
&& ((row.pid !== null && processIsAlive(row.pid))
|
|
@@ -2477,10 +2784,10 @@ export async function reconcileOwnerSessions(client, machineId, _machineHarness,
|
|
|
2477
2784
|
* the next poll tries again: clearing it after an EPERM would say this machine
|
|
2478
2785
|
* has no process for a run whose agent is still working.
|
|
2479
2786
|
*/
|
|
2480
|
-
async function killStopped(client, machineId) {
|
|
2787
|
+
async function killStopped(client, machineId, work) {
|
|
2481
2788
|
const stopped = await returned(client
|
|
2482
2789
|
.from('panel3_runs')
|
|
2483
|
-
.select('id, card_id, pid, state, started_at, resumed_at')
|
|
2790
|
+
.select('id, card_id, pid, state, started_at, resumed_at, process_token')
|
|
2484
2791
|
.eq('machine_id', machineId)
|
|
2485
2792
|
.in('state', [...ENDED_BY_THE_PERSON, 'finished', 'failed'])
|
|
2486
2793
|
.not('pid', 'is', null), 'read', 'the runs on this machine that are not coming back');
|
|
@@ -2488,7 +2795,16 @@ async function killStopped(client, machineId) {
|
|
|
2488
2795
|
return;
|
|
2489
2796
|
const booted = bootedAt();
|
|
2490
2797
|
for (const run of stopped) {
|
|
2491
|
-
|
|
2798
|
+
let stoppedByOwner = false;
|
|
2799
|
+
if (work?.heldAttempt(recoveryAttempt(run))) {
|
|
2800
|
+
if (!ENDED_BY_THE_PERSON.includes(run.state))
|
|
2801
|
+
continue;
|
|
2802
|
+
// The recorded bridge can be gone while its owned tools still run.
|
|
2803
|
+
// Only the durable owner can confirm that the complete execution ended.
|
|
2804
|
+
await work.stopHeldAttempt(recoveryAttempt(run));
|
|
2805
|
+
stoppedByOwner = true;
|
|
2806
|
+
}
|
|
2807
|
+
if (!stoppedByOwner && runProcessIsAlive(run, booted)) {
|
|
2492
2808
|
if (!ENDED_BY_THE_PERSON.includes(run.state))
|
|
2493
2809
|
continue;
|
|
2494
2810
|
try {
|
|
@@ -2509,15 +2825,18 @@ async function killStopped(client, machineId) {
|
|
|
2509
2825
|
continue;
|
|
2510
2826
|
}
|
|
2511
2827
|
try {
|
|
2512
|
-
|
|
2828
|
+
let cleared = client
|
|
2513
2829
|
.from('panel3_runs')
|
|
2514
2830
|
.update({ pid: null })
|
|
2515
|
-
// A finished owner may activate again between this read and write.
|
|
2516
|
-
// Only acknowledge the exact process whose exit was observed.
|
|
2517
2831
|
.eq('id', run.id)
|
|
2518
2832
|
.eq('pid', run.pid)
|
|
2519
|
-
.
|
|
2520
|
-
.
|
|
2833
|
+
.eq('started_at', run.started_at)
|
|
2834
|
+
.in('state', [...ENDED_BY_THE_PERSON, 'finished', 'failed']);
|
|
2835
|
+
cleared = run.process_token === null ? cleared.is('process_token', null) : cleared.eq('process_token', run.process_token);
|
|
2836
|
+
cleared = run.resumed_at === null ? cleared.is('resumed_at', null) : cleared.eq('resumed_at', run.resumed_at);
|
|
2837
|
+
await returned(
|
|
2838
|
+
// A finished owner may activate again while its old execution closes.
|
|
2839
|
+
cleared.select('id'), 'clear the process id of', `run ${run.id}`);
|
|
2521
2840
|
}
|
|
2522
2841
|
catch (error) {
|
|
2523
2842
|
// Said, not fatal. The kill has already happened; this is bookkeeping, and
|
|
@@ -2602,6 +2921,8 @@ async function recoverStranded(client, tools, machineId, mine, hold) {
|
|
|
2602
2921
|
for (const run of live) {
|
|
2603
2922
|
if (mine.has(run.id))
|
|
2604
2923
|
continue;
|
|
2924
|
+
if (tools.work?.heldAttempt(recoveryAttempt(run)))
|
|
2925
|
+
continue;
|
|
2605
2926
|
// WHEN THE ATTEMPT NOW RUNNING BEGAN, which is the first one until a resume
|
|
2606
2927
|
// says otherwise. See the header.
|
|
2607
2928
|
const startedAt = new Date(run.resumed_at ?? run.started_at).getTime();
|
|
@@ -2838,6 +3159,8 @@ export async function takeHandedBack(client, tools, machineId, mine, hold) {
|
|
|
2838
3159
|
// ONCE, OUTSIDE THE LOOP. It is a property of this machine, not of a row.
|
|
2839
3160
|
const booted = bootedAt();
|
|
2840
3161
|
for (const run of offered) {
|
|
3162
|
+
if (tools.work?.heldAttempt(recoveryAttempt(run)))
|
|
3163
|
+
continue;
|
|
2841
3164
|
if (mine.has(run.id)) {
|
|
2842
3165
|
/* THIS DAEMON'S OWN LIVE WORK, OFFERED WHILE IT WAS BUSY BEING QUIET. See
|
|
2843
3166
|
the header. Said rather than passed over in silence, because a machine
|
|
@@ -3028,7 +3351,7 @@ export async function sweepFinishedWorktrees(client) {
|
|
|
3028
3351
|
export function clientReader(injected) {
|
|
3029
3352
|
return typeof injected === 'function' ? injected : () => injected;
|
|
3030
3353
|
}
|
|
3031
|
-
export async function run(args, injected, signal, lifecycle) {
|
|
3354
|
+
export async function run(args, injected, signal, lifecycle, work) {
|
|
3032
3355
|
let once = false;
|
|
3033
3356
|
for (const arg of args) {
|
|
3034
3357
|
if (arg === '--once')
|
|
@@ -3059,6 +3382,7 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3059
3382
|
const machineId = getMachineIdentity().id;
|
|
3060
3383
|
const machineName = hostname();
|
|
3061
3384
|
let listeningHarness = null;
|
|
3385
|
+
const listeningRoutes = new Set();
|
|
3062
3386
|
/* THE RUNS THIS PROCESS IS HOLDING RIGHT NOW, so recovery cannot declare its
|
|
3063
3387
|
own live work dead in the moment before a pid is recorded. It covers THIS
|
|
3064
3388
|
daemon only, which is why `PID_GRACE_MS` exists for the other ones. Keyed by
|
|
@@ -3084,11 +3408,22 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3084
3408
|
`tools` is referenced inside the callback it is being given, which is safe
|
|
3085
3409
|
for the plain reason that the callback can only run once a request has
|
|
3086
3410
|
arrived at a server that by then exists. */
|
|
3411
|
+
if (work && !work.cloudAllowed())
|
|
3412
|
+
throw new Error('Cloud operations are suspended for this sign-in.');
|
|
3413
|
+
if (work)
|
|
3414
|
+
await reconcilePanelInterruptions(current(), machineId, work);
|
|
3087
3415
|
const tools = await startToolsServer(current(), async (parentRunId, brief, codebase, processToken, choice) => {
|
|
3088
|
-
const
|
|
3089
|
-
|
|
3090
|
-
|
|
3416
|
+
const endClaim = work?.beginClaim();
|
|
3417
|
+
try {
|
|
3418
|
+
const child = await startChild(current(), tools, machineId, parentRunId, brief, codebase, processToken, choice);
|
|
3419
|
+
hold(child.runId, child.settled);
|
|
3420
|
+
return { runId: child.runId };
|
|
3421
|
+
}
|
|
3422
|
+
finally {
|
|
3423
|
+
endClaim?.();
|
|
3424
|
+
}
|
|
3091
3425
|
}, (runId, processToken, action) => recoverLanding(current(), machineId, runId, processToken, action));
|
|
3426
|
+
tools.work = work;
|
|
3092
3427
|
out(`daemon machine ${machineId}`);
|
|
3093
3428
|
out(`tools ${tools.urlFor('<run-id>')}`);
|
|
3094
3429
|
out(once ? 'mode one poll' : `mode polling every ${POLL_INTERVAL_MS / 1000}s, Ctrl-C to stop`);
|
|
@@ -3129,6 +3464,8 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3129
3464
|
// Restart only at a poll boundary, before any claims, with no live work.
|
|
3130
3465
|
if (lifecycle && !lifecycle.beforePoll(inFlight.size))
|
|
3131
3466
|
break;
|
|
3467
|
+
let endClaim;
|
|
3468
|
+
const claimOperations = [];
|
|
3132
3469
|
/* ═══ ONE POLL FAILING IS NOT THE DAEMON FAILING. ═══ Every read and write
|
|
3133
3470
|
here throws on a network or database error, by design (constraint 7), and
|
|
3134
3471
|
until Slice 4 that threw straight out of `panel3/cli.js run` and exited the process.
|
|
@@ -3141,37 +3478,119 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3141
3478
|
`--once` still fails loudly, because the acceptance harness reads the exit
|
|
3142
3479
|
code and a swallowed failure there would make a broken suite look green. */
|
|
3143
3480
|
try {
|
|
3481
|
+
if (work && !work.cloudAllowed()) {
|
|
3482
|
+
await sleep(POLL_INTERVAL_MS);
|
|
3483
|
+
continue;
|
|
3484
|
+
}
|
|
3485
|
+
if (work)
|
|
3486
|
+
await reconcilePanelInterruptions(current(), machineId, work);
|
|
3487
|
+
if (work && !work.allowed()) {
|
|
3488
|
+
// A successful read proves polling before recovery opens admission.
|
|
3489
|
+
const { error } = await current().from('panel3_cards').select('id').limit(1);
|
|
3490
|
+
if (error)
|
|
3491
|
+
throw error;
|
|
3492
|
+
lifecycle?.ready();
|
|
3493
|
+
await sleep(POLL_INTERVAL_MS);
|
|
3494
|
+
continue;
|
|
3495
|
+
}
|
|
3496
|
+
endClaim = work?.beginClaim();
|
|
3144
3497
|
/* ═══ THE USER'S STOP IS HONOURED BEFORE ANYTHING ELSE ON THE POLL. ═══ It
|
|
3145
3498
|
is the only thing here that a person is waiting on, and the two takes
|
|
3146
3499
|
below can spend the rest of the poll starting agents. Nothing else needs
|
|
3147
3500
|
to run first: `panel3_stop_card` has already ended the runs, so recovery
|
|
3148
3501
|
cannot see them and neither take can start them. */
|
|
3149
|
-
await killStopped(current(), machineId);
|
|
3502
|
+
await killStopped(current(), machineId, work);
|
|
3503
|
+
for (const claim of work?.recoveredClaims() ?? []) {
|
|
3504
|
+
const attempts = panelClaimAttempts(claim.result);
|
|
3505
|
+
const eligible = new Set();
|
|
3506
|
+
for (const attempt of attempts) {
|
|
3507
|
+
if (inFlight.has(attempt.runId))
|
|
3508
|
+
continue;
|
|
3509
|
+
const pending = work?.claimAttempt(attempt.runId);
|
|
3510
|
+
if (!pending)
|
|
3511
|
+
continue; // Preparation or an actual stop already owns it.
|
|
3512
|
+
const { data, error } = await current().from('panel3_runs')
|
|
3513
|
+
.select('started_at,resumed_at,process_token,state,ended_at').eq('id', attempt.runId).maybeSingle();
|
|
3514
|
+
if (error)
|
|
3515
|
+
throw error;
|
|
3516
|
+
if (data?.state === 'running' && data.ended_at === null && data.started_at === attempt.startedAt
|
|
3517
|
+
&& (data.resumed_at ?? null) === attempt.resumedAt && (data.process_token ?? null) === attempt.processToken)
|
|
3518
|
+
eligible.add(attempt.runId);
|
|
3519
|
+
}
|
|
3520
|
+
const rows = (claim.result ?? []).filter((row) => eligible.has(row._attempt?.run_id ?? row.run_id));
|
|
3521
|
+
const response = { data: rows, error: null, operationId: claim.id, nativeContext: claim.args?.nativeContext };
|
|
3522
|
+
const args = claim.args ?? {};
|
|
3523
|
+
if (rows.length) {
|
|
3524
|
+
if (claim.action === 'panel3_take_turns') {
|
|
3525
|
+
for (const [cardId, turns] of byCard(rows))
|
|
3526
|
+
hold(turns[0].run_id, answerCard(current(), tools, machineId, cardId, turns));
|
|
3527
|
+
}
|
|
3528
|
+
else if (claim.action === 'panel3_take_rearms') {
|
|
3529
|
+
for (const row of rows) {
|
|
3530
|
+
const started = await startRearmed(current(), tools, machineId, row);
|
|
3531
|
+
hold(row.run_id, started.settled);
|
|
3532
|
+
}
|
|
3533
|
+
}
|
|
3534
|
+
else if (claim.action === 'panel3_dispatch') {
|
|
3535
|
+
const row = rows[0];
|
|
3536
|
+
const codebase = args.p_codebase_id ? await codebaseOfRun(current(), row.run_id) : null;
|
|
3537
|
+
const started = await startChild(current(), tools, machineId, String(args.p_parent_run_id), String(args.p_brief), codebase, typeof args.p_process_token === 'string' ? args.p_process_token : undefined, {}, response);
|
|
3538
|
+
hold(started.runId, started.settled);
|
|
3539
|
+
}
|
|
3540
|
+
else if (claim.action === 'panel3_resume') {
|
|
3541
|
+
const runId = String(args.p_run_id);
|
|
3542
|
+
const started = await resumeRun(current(), tools, machineId, runId, typeof args.p_after_pid === 'number' ? args.p_after_pid : null, response);
|
|
3543
|
+
if (started)
|
|
3544
|
+
hold(runId, started.settled);
|
|
3545
|
+
}
|
|
3546
|
+
else if (claim.action === 'panel3_take_owner_activation') {
|
|
3547
|
+
const runId = String(args.p_run_id);
|
|
3548
|
+
const started = await activateOwner(current(), tools, machineId, runId, typeof args.p_after_process_token === 'string' ? args.p_after_process_token : null, typeof args.p_after_pid === 'number' ? args.p_after_pid : null, response);
|
|
3549
|
+
if (started)
|
|
3550
|
+
hold(runId, started.settled);
|
|
3551
|
+
}
|
|
3552
|
+
else
|
|
3553
|
+
throw new Error('The saved panel claim has an unsupported action.');
|
|
3554
|
+
}
|
|
3555
|
+
work?.finishRpcs([claim.id]);
|
|
3556
|
+
}
|
|
3150
3557
|
/* Publish readiness before claiming work. A card with an untaken turn reads the same whether a daemon
|
|
3151
3558
|
is two seconds away or nobody has one running; this row is the only place
|
|
3152
3559
|
the difference exists. It is written before the takes rather than after
|
|
3153
3560
|
so that a machine which is up but busy still reads as up. */
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3561
|
+
const installed = recoveryHarnesses();
|
|
3562
|
+
for (const route of [...listeningRoutes]) {
|
|
3563
|
+
if (!installed.includes(route)) {
|
|
3564
|
+
await stopListening(current(), machineId, route);
|
|
3565
|
+
listeningRoutes.delete(route);
|
|
3566
|
+
}
|
|
3159
3567
|
}
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3568
|
+
// A primary selection failure cannot skip an installed recovery route.
|
|
3569
|
+
for (const route of installed) {
|
|
3570
|
+
try {
|
|
3571
|
+
const response = await panelClaim(current(), work, 'panel3_take_turns', { p_machine_id: machineId, p_agent: route, p_recovery_only: true });
|
|
3572
|
+
if (response.operationId)
|
|
3573
|
+
claimOperations.push(response.operationId);
|
|
3574
|
+
const taken = await returned(Promise.resolve(response), 'take', 'explicit authentication recovery');
|
|
3575
|
+
await sayListening(current(), machineId, machineName, route);
|
|
3576
|
+
await sayPollingProblem(current(), machineId, route, null);
|
|
3577
|
+
listeningRoutes.add(route);
|
|
3578
|
+
for (const [cardId, turns] of byCard(taken))
|
|
3579
|
+
hold(turns[0].run_id, answerCard(current(), tools, machineId, cardId, turns));
|
|
3580
|
+
}
|
|
3581
|
+
catch (error) {
|
|
3582
|
+
try {
|
|
3583
|
+
await sayPollingProblem(current(), machineId, route, 'poll_failed');
|
|
3584
|
+
}
|
|
3585
|
+
catch { /* The last successful observation expires. */ }
|
|
3586
|
+
said(`Authentication recovery polling failed for ${route}: ${error instanceof Error ? error.message : String(error)}`);
|
|
3164
3587
|
}
|
|
3165
|
-
throw error;
|
|
3166
3588
|
}
|
|
3589
|
+
const machineHarness = await selectedHarness(current(), machineId);
|
|
3590
|
+
listeningHarness = machineHarness;
|
|
3167
3591
|
if (signal?.aborted)
|
|
3168
3592
|
break;
|
|
3169
|
-
|
|
3170
|
-
await stopListening(current(), machineId, listeningHarness);
|
|
3171
|
-
}
|
|
3172
|
-
listeningHarness = machineHarness;
|
|
3173
|
-
await sayListening(current(), machineId, machineName, machineHarness);
|
|
3174
|
-
await reconcileOwnerSessions(current(), machineId, machineHarness, new Set(inFlight.keys()));
|
|
3593
|
+
await reconcileOwnerSessions(current(), machineId, machineHarness, new Set(inFlight.keys()), work);
|
|
3175
3594
|
await recoverStranded(current(), tools, machineId, new Set(inFlight.keys()), hold);
|
|
3176
3595
|
/* ═══ AND THE COPIES OF CARDS THAT ARE OVER. ═══ After recovery,
|
|
3177
3596
|
deliberately: a run this machine is about to resume is one whose card is
|
|
@@ -3197,7 +3616,10 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3197
3616
|
/* THE MACHINE ID GOES IN because the take writes the run row, and a run has
|
|
3198
3617
|
to say where it is running: the exclusion is cross-machine and recovery is
|
|
3199
3618
|
per-machine, so a row with nobody's machine on it could be neither. */
|
|
3200
|
-
const
|
|
3619
|
+
const takenResponse = await panelClaim(current(), work, 'panel3_take_turns', { p_machine_id: machineId, p_agent: machineHarness, p_recovery_only: false });
|
|
3620
|
+
if (takenResponse.operationId)
|
|
3621
|
+
claimOperations.push(takenResponse.operationId);
|
|
3622
|
+
const taken = await returned(Promise.resolve(takenResponse), 'take', 'turns');
|
|
3201
3623
|
/* ═══ THE OTHER KIND OF TAKEABLE WORK. ═══ ux.md's re-arm: an answered
|
|
3202
3624
|
question makes the branch that asked it takeable again, and a question
|
|
3203
3625
|
still walking up makes the run it reached takeable so that level gets its
|
|
@@ -3211,7 +3633,10 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3211
3633
|
to it on a later poll rather than putting two of them on one card. The
|
|
3212
3634
|
other order would decide the same question from a snapshot taken before
|
|
3213
3635
|
the run existed. */
|
|
3214
|
-
const
|
|
3636
|
+
const rearmedResponse = await panelClaim(current(), work, 'panel3_take_rearms', { p_machine_id: machineId, p_agent: machineHarness });
|
|
3637
|
+
if (rearmedResponse.operationId)
|
|
3638
|
+
claimOperations.push(rearmedResponse.operationId);
|
|
3639
|
+
const rearmed = await returned(Promise.resolve(rearmedResponse), 'take', 'runs that can carry on');
|
|
3215
3640
|
for (const row of rearmed) {
|
|
3216
3641
|
/* NOT AWAITED PAST THE SPAWN, exactly as a taken card is not: the work is a
|
|
3217
3642
|
real agent and holding the poll open for it would put every other card
|
|
@@ -3239,6 +3664,8 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3239
3664
|
// rather than becoming an unhandled rejection.
|
|
3240
3665
|
hold(runId, answerCard(current(), tools, machineId, cardId, turns));
|
|
3241
3666
|
}
|
|
3667
|
+
await sayListening(current(), machineId, machineName, machineHarness);
|
|
3668
|
+
listeningRoutes.add(machineHarness);
|
|
3242
3669
|
await sayPollingProblem(current(), machineId, machineHarness, null);
|
|
3243
3670
|
lifecycle?.ready();
|
|
3244
3671
|
if (once) {
|
|
@@ -3265,14 +3692,18 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3265
3692
|
}
|
|
3266
3693
|
}
|
|
3267
3694
|
}
|
|
3695
|
+
finally {
|
|
3696
|
+
work?.finishRpcs(claimOperations);
|
|
3697
|
+
endClaim?.();
|
|
3698
|
+
}
|
|
3268
3699
|
if (!signal?.aborted)
|
|
3269
3700
|
await sleep(POLL_INTERVAL_MS);
|
|
3270
3701
|
}
|
|
3271
3702
|
}
|
|
3272
3703
|
finally {
|
|
3273
3704
|
try {
|
|
3274
|
-
|
|
3275
|
-
await stopListening(current(), machineId,
|
|
3705
|
+
for (const route of listeningRoutes)
|
|
3706
|
+
await stopListening(current(), machineId, route);
|
|
3276
3707
|
}
|
|
3277
3708
|
finally {
|
|
3278
3709
|
await tools.close();
|
|
@@ -3284,8 +3715,12 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3284
3715
|
* client. A stopped worker is restarted automatically. Explicit restarts happen
|
|
3285
3716
|
* at an idle poll boundary and confirm only after a new worker completes a poll.
|
|
3286
3717
|
* Sign-out stops the supervisor, so it cannot restart behind the user's back. */
|
|
3287
|
-
export function startPanel(injected) {
|
|
3718
|
+
export function startPanel(injected, work) {
|
|
3288
3719
|
const controller = new AbortController();
|
|
3720
|
+
let resolveReady;
|
|
3721
|
+
let rejectReady;
|
|
3722
|
+
const ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; });
|
|
3723
|
+
void ready.catch(() => { });
|
|
3289
3724
|
let requested = false;
|
|
3290
3725
|
let restarting = null;
|
|
3291
3726
|
let resolveRestart = null;
|
|
@@ -3304,11 +3739,12 @@ export function startPanel(injected) {
|
|
|
3304
3739
|
}
|
|
3305
3740
|
return false;
|
|
3306
3741
|
},
|
|
3307
|
-
ready: () => { if (!requested)
|
|
3742
|
+
ready: () => { resolveReady(); if (!requested)
|
|
3308
3743
|
resolveRestart?.(); },
|
|
3309
|
-
});
|
|
3744
|
+
}, work);
|
|
3310
3745
|
}
|
|
3311
3746
|
catch (error) {
|
|
3747
|
+
rejectReady(error instanceof Error ? error : new Error(String(error)));
|
|
3312
3748
|
said(`the agent panel stopped polling: ${error instanceof Error ? error.message : String(error)}`);
|
|
3313
3749
|
// Startup failures must not leave an online machine with a dead worker.
|
|
3314
3750
|
if (!controller.signal.aborted)
|
|
@@ -3317,6 +3753,7 @@ export function startPanel(injected) {
|
|
|
3317
3753
|
}
|
|
3318
3754
|
})();
|
|
3319
3755
|
return {
|
|
3756
|
+
ready,
|
|
3320
3757
|
restart: () => {
|
|
3321
3758
|
if (controller.signal.aborted)
|
|
3322
3759
|
return Promise.reject(new Error('This machine is signing out. Open Companion and sign in again.'));
|
|
@@ -3339,6 +3776,7 @@ export function startPanel(injected) {
|
|
|
3339
3776
|
},
|
|
3340
3777
|
stop: async () => {
|
|
3341
3778
|
controller.abort();
|
|
3779
|
+
rejectReady(new Error('Cloud startup was cancelled by the service command.'));
|
|
3342
3780
|
rejectRestart?.(new Error('The machine disconnected before the worker restarted.'));
|
|
3343
3781
|
await running;
|
|
3344
3782
|
},
|