@ctrl-spc/cs 0.7.14 → 0.7.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,4 @@
1
+ import { failureMessage } from '../failure-reason.js';
1
2
  /**
2
3
  * ═══ AGENT PANEL v3: the poll loop that answers a card. ═══
3
4
  *
@@ -137,7 +138,7 @@ import { answerPrompt, escalationPrompt, levelOnePrompt, ownerActivationPrompt,
137
138
  import { ASK_CONTENT_COLUMNS, attachmentLine, gitRulesFor, loadAttachments, loadOutputNames, outputOf, recordBaseProtection, standingRulesFor, withAskContent, } from './show.js';
138
139
  import { forgetSecrets, redactSecrets } from './secrets.js';
139
140
  import { sayListening, sayPollingProblem, stopListening } from './presence.js';
140
- import { selectedHarness } from './coordinator.js';
141
+ import { selectedHarness, recoveryHarnesses } from './coordinator.js';
141
142
  import { baseBranchState, checkoutForCodebase, commitCardWork, detectBaseProtection, folderIsBranch, hasCheckoutForCodebase, mergeIntoBase, releaseBaseBranch, settleCardWorktree, worktreeForCard, ownsReconciliation, prepareCardReconciliation, finishCardReconciliation, worktreesOnThisMachine, } from './checkout.js';
142
143
  import { harness, startAgent } from './spawn.js';
143
144
  import { establishOwnerSession, listOwnerSessionIds, OWNER_SESSION_GRACE_MS, readOwnerSession, removeOwnerSession, validSessionUuid, writeOwnerSession, } from './session.js';
@@ -146,11 +147,187 @@ import { listPanel3CodexOwnerHomeIds, removePanel3CodexOwnerHome, } from '../cod
146
147
  import { getMachineIdentity, scratchDir } from '../config.js';
147
148
  import { listCodebases } from '../codebases.js';
148
149
  import { killTree, processIsAlive } from '../win-shell.js';
150
+ import { randomUUID } from 'node:crypto';
149
151
  import { hostname, uptime } from 'node:os';
150
152
  import { execFileSync } from 'node:child_process';
151
153
  import { existsSync } from 'node:fs';
152
154
  import { mkdir, writeFile } from 'node:fs/promises';
153
155
  import { join } from 'node:path';
156
+ function recoveryAttempt(run) {
157
+ return { runId: run.id, processToken: run.process_token, startedAt: run.started_at, resumedAt: run.resumed_at };
158
+ }
159
+ async function startTrackedAgent(client, runId, preparation, work, captured, ...args) {
160
+ if (!preparation)
161
+ return { ...startAgent(...args), attempt: captured };
162
+ const claimed = captured ?? work?.claimAttempt(runId);
163
+ if (!claimed)
164
+ throw new Error('This run no longer belongs to the claimed attempt.');
165
+ preparation.setAttempt(claimed);
166
+ let preparationFailed = false;
167
+ let failureReason = '';
168
+ const execution = {
169
+ ...preparation.execution,
170
+ register: async (child, agent) => {
171
+ await preparation.execution.register(child, agent);
172
+ // An ordinary stop can reopen admission after a refusal. Every such
173
+ // wait needs a fresh cloud check before the prompt is authorized.
174
+ try {
175
+ let waited;
176
+ do {
177
+ const { data, error } = await client.from('panel3_runs')
178
+ .select('process_token,started_at,resumed_at,state,ended_at').eq('id', runId).single();
179
+ if (error)
180
+ throw error;
181
+ if (!data || data.state !== 'running' || data.ended_at !== null
182
+ || claimed.processToken !== (data.process_token ?? null) || claimed.startedAt !== data.started_at
183
+ || claimed.resumedAt !== (data.resumed_at ?? null)) {
184
+ throw new Error('This run no longer belongs to the claimed attempt.');
185
+ }
186
+ waited = await preparation.execution.waitForPromptAdmission();
187
+ } while (waited);
188
+ }
189
+ catch (error) {
190
+ if (!preparation.execution.interrupted()) {
191
+ const reason = error instanceof Error ? error.message : String(error);
192
+ preparation.deferFailure(reason, args[1]);
193
+ preparationFailed = true;
194
+ failureReason = reason;
195
+ }
196
+ throw error;
197
+ }
198
+ },
199
+ };
200
+ const started = startAgent(args[0], args[1], args[2], args[3], args[4], args[5], execution);
201
+ // Do not hold the claim barrier while registration waits for an ordinary
202
+ // stop to be refused. The original answer promises proven process closure.
203
+ const answered = started.answered.then(async (answer) => {
204
+ if (preparationFailed && !preparation.execution.interrupted()) {
205
+ await endRun(client, args[1], runId, claimed.cardId, failureReason, claimed);
206
+ preparation.execution.complete();
207
+ }
208
+ return answer;
209
+ });
210
+ void answered.catch(() => { }); // The caller records the PID before awaiting settlement.
211
+ return { ...started, answered, attempt: claimed, deferOutcome: preparation.deferOutcome, acknowledgeOutcome: preparation.acknowledgeOutcome, preparationFailed: () => preparationFailed };
212
+ }
213
+ function panelClaimAttempts(result) {
214
+ if (!Array.isArray(result))
215
+ throw new Error('The claim journal returned no complete result.');
216
+ const unique = new Map();
217
+ for (const row of result) {
218
+ const attempt = row?._attempt;
219
+ if (!attempt || typeof attempt.run_id !== 'string' || typeof attempt.card_id !== 'string' || typeof attempt.started_at !== 'string') {
220
+ throw new Error('The claim journal returned an incomplete run identity.');
221
+ }
222
+ unique.set(attempt.run_id, { runId: attempt.run_id, cardId: attempt.card_id, processToken: attempt.process_token ?? null,
223
+ pid: attempt.pid ?? null, startedAt: attempt.started_at, resumedAt: attempt.resumed_at ?? null,
224
+ observedPendingTurnIds: attempt.observed_pending_turn_ids ?? null, authRecoveryRunId: attempt.auth_recovery_run_id ?? null, harness: attempt.harness ?? null });
225
+ }
226
+ return [...unique.values()];
227
+ }
228
+ function capturedPanelAttempt(rows, runId, work) {
229
+ return work?.claimAttempt(runId) ?? panelClaimAttempts(rows).find(attempt => attempt.runId === runId);
230
+ }
231
+ async function panelClaim(client, work, action, args, nativeContext) {
232
+ const operationId = work?.beginRpc(action, { ...args, ...(nativeContext ? { nativeContext } : {}) }) ?? randomUUID();
233
+ try {
234
+ const response = await client.rpc(action, { ...args, p_operation_id: operationId });
235
+ if (!response.error) {
236
+ const journal = await client.rpc('panel3_reconcile_claim', { p_machine_id: args.p_machine_id, p_operation_id: operationId });
237
+ if (journal.error)
238
+ throw journal.error;
239
+ const row = journal.data?.[0];
240
+ if (row?.outcome !== 'completed' || row.action !== action)
241
+ throw new Error('The claim was cancelled before it could start.');
242
+ const attempts = panelClaimAttempts(row.result);
243
+ work?.recordRpc(operationId, attempts, false, row.result);
244
+ return { ...response, data: row.result, operationId };
245
+ }
246
+ if (response.error)
247
+ work?.deferRpc(operationId);
248
+ return { ...response, operationId };
249
+ }
250
+ catch (error) {
251
+ work?.deferRpc(operationId);
252
+ throw error;
253
+ }
254
+ }
255
+ async function reconcilePanelInterruptions(client, machineId, work) {
256
+ const capability = await client.rpc('panel3_interrupt_machine_runs', { p_machine_id: machineId,
257
+ p_operation_id: randomUUID(), p_interrupted_at: new Date().toISOString(), p_attempts: [] });
258
+ if (capability.error)
259
+ throw new Error(`could not verify interruption recovery: ${capability.error.message}`);
260
+ for (const pending of work.pendingOutcomes()) {
261
+ const { attempt, outcome } = pending;
262
+ const { data, error } = await client.from('panel3_runs').select('started_at,resumed_at,process_token,state,ended_at,level,card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)').eq('id', attempt.runId).maybeSingle();
263
+ if (error)
264
+ throw new Error(`could not read the completed attempt: ${error.message}`);
265
+ if (data && data.started_at === attempt.startedAt && (data.resumed_at ?? null) === attempt.resumedAt
266
+ && (data.process_token ?? null) === attempt.processToken && data.ended_at === null) {
267
+ const card = (Array.isArray(data.card) ? data.card[0] : data.card);
268
+ const successfulToolEnding = !outcome.ok && outcome.failureKind !== 'authentication' && (data.state === 'asked' || (data.level === 1 && !!card?.conversation_run_id));
269
+ if (outcome.ok || successfulToolEnding)
270
+ await writeAnswer(client, attempt.runId, attempt.cardId, successfulToolEnding ? null : outcome.text, attempt.processToken ?? undefined, attempt);
271
+ else
272
+ await endRun(client, outcome.level ?? 1, attempt.runId, attempt.cardId, outcome.reason, attempt, outcome.failureKind);
273
+ }
274
+ work.acknowledgeOutcome(pending.id);
275
+ }
276
+ for (const failure of work.failedPreparations()) {
277
+ await endRun(client, failure.level, failure.attempt.runId, failure.attempt.cardId, failure.reason, failure.attempt);
278
+ work.acknowledgePreparationFailure(failure.id);
279
+ }
280
+ for (const claim of work.pendingRpcs()) {
281
+ const journal = await client.rpc('panel3_reconcile_claim', { p_machine_id: machineId, p_operation_id: claim.id });
282
+ if (journal.error)
283
+ throw new Error(`could not read the claim receipt: ${journal.error.message}`);
284
+ const row = journal.data?.[0];
285
+ if (row?.outcome === 'cancelled')
286
+ work.recordRpc(claim.id, [], true);
287
+ else if (row?.outcome === 'completed' && row.action === claim.action)
288
+ work.recordRpc(claim.id, panelClaimAttempts(row.result), false, row.result);
289
+ else
290
+ throw new Error('The pending claim could not be reconciled.');
291
+ }
292
+ if (work.legacyRequired()) {
293
+ const attempts = [];
294
+ for (let offset = 0;; offset += 500) {
295
+ const { data, error } = await client.from('panel3_runs')
296
+ .select('id,card_id,process_token,pid,started_at,resumed_at')
297
+ .eq('machine_id', machineId).is('ended_at', null).order('id').range(offset, offset + 499);
298
+ if (error)
299
+ throw error;
300
+ if (!Array.isArray(data))
301
+ throw new Error('The old run snapshot did not return a complete page.');
302
+ for (const row of data) {
303
+ if (typeof row.started_at !== 'string')
304
+ throw new Error('An old run has no exact attempt identity.');
305
+ attempts.push({ runId: row.id, cardId: row.card_id, processToken: row.process_token ?? null,
306
+ pid: row.pid ?? null, startedAt: row.started_at, resumedAt: row.resumed_at ?? null, observedPendingTurnIds: null });
307
+ }
308
+ if (data.length < 500)
309
+ break;
310
+ }
311
+ for (const attempt of attempts)
312
+ work.recordLegacy(attempt);
313
+ work.legacyComplete();
314
+ }
315
+ for (const receipt of work.receipts()) {
316
+ const attempt = receipt.attempt;
317
+ const { data, error } = await client.rpc('panel3_interrupt_machine_runs', {
318
+ p_machine_id: machineId, p_operation_id: receipt.operationId, p_interrupted_at: receipt.interruptedAt,
319
+ p_attempts: [{ run_id: attempt.runId, process_token: attempt.processToken, pid: receipt.pid ?? attempt.pid ?? null,
320
+ started_at: attempt.startedAt, resumed_at: attempt.resumedAt, observed_pending_turn_ids: attempt.observedPendingTurnIds }],
321
+ });
322
+ if (error)
323
+ throw error;
324
+ const result = data?.[0];
325
+ if (!result || result.run_id !== attempt.runId || !['interrupted', 'already_interrupted', 'continued', 'stale'].includes(result.outcome)) {
326
+ throw new Error('The interrupted run was not acknowledged. Work remains protected.');
327
+ }
328
+ work.acknowledge(receipt.id);
329
+ }
330
+ }
154
331
  const USAGE = 'usage: run [--once]';
155
332
  /** How long between takes. Short, because it is the whole delay between a user
156
333
  * sending and a card showing an agent on it, and the take is one small indexed
@@ -550,18 +727,24 @@ pictures = []) => {
550
727
  * a dispatch knows the brief before the row exists and writes it there. Passing
551
728
  * it again would be rewriting a brief that ux.md fixes at dispatch.
552
729
  *
553
- * A failure here is NOT fatal. The agent is already running and killing it over
554
- * a bookkeeping write would cost the user the work. What is lost is precision:
555
- * the run has no pid, so once this daemon is gone recovery treats it as dead —
556
- * which errs towards answering again rather than stranding.
730
+ * A bookkeeping failure does not discard the locally tracked execution. Exact
731
+ * activation predicates prevent a delayed write from reviving an ended run or
732
+ * attaching this process to a later continuation.
557
733
  */
558
- async function recordProcess(client, runId, pid, brief, processToken) {
734
+ async function recordProcess(client, runId, pid, brief, processToken, attempt) {
559
735
  let query = client
560
736
  .from('panel3_runs')
561
737
  .update({ pid, ...(brief === undefined ? {} : { brief }) })
562
- .eq('id', runId);
738
+ .eq('id', runId)
739
+ .eq('state', 'running')
740
+ .is('ended_at', null);
563
741
  if (processToken !== undefined)
564
742
  query = query.eq('process_token', processToken);
743
+ if (attempt) {
744
+ query = query.eq('started_at', attempt.startedAt);
745
+ query = attempt.resumedAt === null ? query.is('resumed_at', null) : query.eq('resumed_at', attempt.resumedAt);
746
+ query = attempt.processToken === null ? query.is('process_token', null) : query.eq('process_token', attempt.processToken);
747
+ }
565
748
  const written = await returned(query.select('id'), 'record what is running', `run ${runId}`);
566
749
  if (written.length === 0)
567
750
  throw new Error(`could not record what is running for run ${runId}: its activation has ended`);
@@ -579,15 +762,24 @@ async function recordProcess(client, runId, pid, brief, processToken) {
579
762
  * there first. It is a fact about the record rather than a failure, so it does
580
763
  * not go through `returned()`, exactly as the answer's own null does not.
581
764
  */
582
- async function giveUp(client, runId, reason, processToken) {
765
+ async function giveUp(client, runId, reason, processToken, attempt, failureKind) {
583
766
  const { data, error } = await client
584
767
  .rpc('panel3_give_up', {
585
768
  p_run_id: runId,
586
769
  p_reason: reason,
770
+ p_failure_kind: failureKind ?? null,
771
+ p_process_token: processToken ?? null,
772
+ p_expected_attempt: null,
587
773
  ...(processToken === undefined ? {} : { p_process_token: processToken }),
774
+ ...(attempt ? { p_process_token: attempt.processToken, p_expected_attempt: {
775
+ started_at: attempt.startedAt, resumed_at: attempt.resumedAt,
776
+ } } : {}),
588
777
  });
589
778
  if (error)
590
779
  throw new Error(`could not give up run ${runId}: ${error.message}`);
780
+ if (attempt && data !== null && (!Number.isInteger(data) || data < 0)) {
781
+ throw new Error(`could not give up run ${runId}: no ending was acknowledged`);
782
+ }
591
783
  /* THE RUN IS OVER, so whatever it read is dropped. Every ending does this —
592
784
  here, `failRun` and `writeAnswer` — because a daemon stays up for days and
593
785
  has no business holding Tuesday's secret. */
@@ -630,7 +822,7 @@ async function giveUp(client, runId, reason, processToken) {
630
822
  * a run can be the last thing live on it — but no turn is written, exactly as
631
823
  * none is written for a run that stopped to ask.
632
824
  */
633
- async function writeAnswer(client, runId, cardId, text, processToken) {
825
+ async function writeAnswer(client, runId, cardId, text, processToken, attempt) {
634
826
  /* ═══ THE SECOND OF THE TWO CHOKEPOINTS THE CREDENTIAL RULE RESTS ON. ═══
635
827
  This is the ONE path an agent's own words take to a hosted row — a turn on
636
828
  the card at levels 1 and 2, and a level 3's `panel3_runs.report`, which
@@ -640,6 +832,9 @@ async function writeAnswer(client, runId, cardId, text, processToken) {
640
832
  const { data: turnId, error } = await client
641
833
  .rpc('panel3_answer', {
642
834
  p_run_id: runId,
835
+ p_process_token: processToken ?? attempt?.processToken ?? null,
836
+ p_expected_attempt: attempt ? { started_at: attempt.startedAt, resumed_at: attempt.resumedAt } : null,
837
+ p_auth_recovery_run_id: attempt?.authRecoveryRunId ?? null,
643
838
  p_body: text === null ? null : redactSecrets(processToken === undefined ? runId : `${runId}:${processToken}`, text),
644
839
  ...(processToken === undefined ? {} : { p_process_token: processToken }),
645
840
  });
@@ -707,14 +902,15 @@ async function writeAnswer(client, runId, cardId, text, processToken) {
707
902
  away. */
708
903
  said(`run ${runId} had already ended, so its answer was not written to card ${cardId}`);
709
904
  }
710
- return ownerSettlementAccepted(run, processToken);
905
+ return ownerSettlementAccepted(run, processToken, attempt);
711
906
  }
712
907
  out(`answered card ${cardId} run ${runId} ${text?.length ?? 0} characters`);
713
908
  return true;
714
909
  }
715
- export function ownerSettlementAccepted(run, processToken) {
910
+ export function ownerSettlementAccepted(run, processToken, attempt) {
716
911
  return (run.state === 'asked' || run.state === 'finished')
717
- && (processToken === undefined || run.processToken === processToken);
912
+ && (processToken === undefined || run.processToken === processToken)
913
+ && (!attempt || run.processToken === attempt.processToken && run.startedAt === attempt.startedAt && (run.resumedAt ?? null) === attempt.resumedAt);
718
914
  }
719
915
  /** What the record says a run is now, and at what level. Read only to say the
720
916
  * right sentence about something that has already happened; nothing branches on
@@ -722,7 +918,7 @@ export function ownerSettlementAccepted(run, processToken) {
722
918
  async function runNow(client, runId) {
723
919
  const runs = await returned(client
724
920
  .from('panel3_runs')
725
- .select('state, level, process_token, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)')
921
+ .select('state, level, process_token, started_at, resumed_at, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)')
726
922
  .eq('id', runId), 'read', `the state of run ${runId}`);
727
923
  const run = runs[0];
728
924
  return run
@@ -730,9 +926,9 @@ async function runNow(client, runId) {
730
926
  state: run.state,
731
927
  level: run.level,
732
928
  conversationOwnerId: run.card?.conversation_run_id ?? null,
733
- processToken: run.process_token,
929
+ processToken: run.process_token, startedAt: run.started_at, resumedAt: run.resumed_at,
734
930
  }
735
- : { state: 'no longer on the record', level: null, conversationOwnerId: null, processToken: null };
931
+ : { state: 'no longer on the record', level: null, conversationOwnerId: null, processToken: null, startedAt: null, resumedAt: null };
736
932
  }
737
933
  /**
738
934
  * A run whose process failed, ended with the reason ON ITS OWN COLUMN, and its
@@ -828,76 +1024,83 @@ export async function settingsForRun(client, runId) {
828
1024
  * the level is what tells it how a run of this shape ends.
829
1025
  */
830
1026
  async function answerCard(client, tools, machineId, cardId, turns) {
831
- const runId = turns[0].run_id;
832
- /* BEFORE THE SPAWN, AND ITS FAILURE IS THE SPAWN'S FAILURE. The receipts are
833
- part of what the agent is sent, so a read that fails must not be papered
834
- over with an empty list: that reads as a card that has made nothing, which
835
- is how an agent creates a second epic beside the one it cannot see. The
836
- attachments read carries the same rule: a failed read here must not read
837
- as "nothing is attached", which is a different card than the one that was
838
- actually sent. */
839
- /* ═══ AND A FAILURE HERE ENDS THE RUN, RATHER THAN LEAVING IT RUNNING WITH NO
840
- PROCESS. ═══ The take already wrote the run row in the statement that leased
841
- the turns, so a throw between here and `startAgent` leaves a run reading
842
- `running` with a null pid and nothing on stderr the person can see.
843
- `recoverStranded` then reads that as a machine that went away, hands the
844
- message back, and `panel3_take_turns` leases it to A BRAND NEW RUN whose
845
- attempts start again at one — so a permanent failure, such as a read this
846
- build cannot make against the current schema, repeats forever while the card
847
- says Working and never says why. This is the sixth of `endRun`'s endings and
848
- the last one that was missing: `resumeRun` and `startRearmed` already end
849
- their two post-claim failures this way for exactly this reason.
850
- THE REASON IS SHAREABLE. All three reads are `returned()` calls against the
851
- database, whose messages name tables and columns and never a local path, so
852
- constraint 6 is satisfied without a level fork here. */
853
- let brief;
854
- /* WHERE IT RUNS AND WHAT IT IS TOLD ABOUT THE CODEBASES, THROUGH THE SAME
855
- SEAM AS EVERY OTHER SPAWN. A level 1 run works in an empty directory of the
856
- user's own, and it is the fifth start site rather than a special case: the
857
- generated git block reaches every level, and one place resolving it is what
858
- makes that true without four copies of the read. */
859
- let where;
860
- /* THE MANDATE IS READ IN THE SAME WINDOW AND UNDER THE SAME RULE. A launcher
861
- is an agent like any other and gets the project's standing rules before it
862
- decides anything, and a read that fails must not read as "this project has
863
- no rules" — that is a different project than the one the person is on. So it
864
- joins the three reads above inside this ending rather than beside it. */
865
- let rules;
866
- let settings;
867
- try {
868
- brief = briefFor(turns[0].card_title, turns, await receiptsFor(client, cardId), await attachmentsFor(client, cardId), await codebasesForRun(client, runId));
869
- rules = await standingRulesFor(client, runId);
870
- settings = await settingsForRun(client, runId);
871
- where = await workingDirectory(client, runId, LEVEL, false);
872
- }
873
- catch (error) {
874
- const why = error instanceof Error ? error.message : String(error);
875
- await endRun(client, LEVEL, runId, cardId, why);
876
- throw new Error(`NO AGENT IS RUNNING: ${why}`);
877
- }
878
- /* ═══ THE RUN ID IS ON THE URL, AND THAT IS THE WHOLE OF WHAT THE AGENT IS
879
- TOLD ABOUT ITS OWN STANDING. ═══ The tools server reads the level off the
880
- run row this id names, so the daemon does not tell the child what it may do
881
- and the child has nothing to claim. `LEVEL` below decides argv only — which
882
- of the harness's own tools the process gets — and the two can never disagree
883
- about the record, because only one of them consults it. */
884
- /* AN EMPTY DIRECTORY OF THE USER'S OWN, because level 1 has no code tool to
885
- use a real one with, and the daemon's inherited cwd under a launchd login
886
- item is the filesystem root. */
887
- const started = startAgent(withStandingRules(rules, brief, where.block), LEVEL, tools.urlFor(runId), where.cwd, undefined, settings);
888
- out(`run ${runId} level ${LEVEL} ${started.pid ? `pid ${started.pid}` : 'no process'}`);
1027
+ const preparation = tools.work?.prepare(turns[0].run_id);
889
1028
  try {
890
- /* THE BRIEF, NOT WHAT THE PROCESS WAS HANDED. The rules are current at the
891
- activation and the brief is immutable, so storing the composed string
892
- would freeze one inside the other and every respawn would replay it. */
893
- await recordProcess(client, runId, started.pid, brief);
1029
+ const runId = turns[0].run_id;
1030
+ const attempt = capturedPanelAttempt(turns, runId, tools.work);
1031
+ /* BEFORE THE SPAWN, AND ITS FAILURE IS THE SPAWN'S FAILURE. The receipts are
1032
+ part of what the agent is sent, so a read that fails must not be papered
1033
+ over with an empty list: that reads as a card that has made nothing, which
1034
+ is how an agent creates a second epic beside the one it cannot see. The
1035
+ attachments read carries the same rule: a failed read here must not read
1036
+ as "nothing is attached", which is a different card than the one that was
1037
+ actually sent. */
1038
+ /* ═══ AND A FAILURE HERE ENDS THE RUN, RATHER THAN LEAVING IT RUNNING WITH NO
1039
+ PROCESS. ═══ The take already wrote the run row in the statement that leased
1040
+ the turns, so a throw between here and `startAgent` leaves a run reading
1041
+ `running` with a null pid and nothing on stderr the person can see.
1042
+ `recoverStranded` then reads that as a machine that went away, hands the
1043
+ message back, and `panel3_take_turns` leases it to A BRAND NEW RUN whose
1044
+ attempts start again at one — so a permanent failure, such as a read this
1045
+ build cannot make against the current schema, repeats forever while the card
1046
+ says Working and never says why. This is the sixth of `endRun`'s endings and
1047
+ the last one that was missing: `resumeRun` and `startRearmed` already end
1048
+ their two post-claim failures this way for exactly this reason.
1049
+ THE REASON IS SHAREABLE. All three reads are `returned()` calls against the
1050
+ database, whose messages name tables and columns and never a local path, so
1051
+ constraint 6 is satisfied without a level fork here. */
1052
+ let brief;
1053
+ /* WHERE IT RUNS AND WHAT IT IS TOLD ABOUT THE CODEBASES, THROUGH THE SAME
1054
+ SEAM AS EVERY OTHER SPAWN. A level 1 run works in an empty directory of the
1055
+ user's own, and it is the fifth start site rather than a special case: the
1056
+ generated git block reaches every level, and one place resolving it is what
1057
+ makes that true without four copies of the read. */
1058
+ let where;
1059
+ /* THE MANDATE IS READ IN THE SAME WINDOW AND UNDER THE SAME RULE. A launcher
1060
+ is an agent like any other and gets the project's standing rules before it
1061
+ decides anything, and a read that fails must not read as "this project has
1062
+ no rules" — that is a different project than the one the person is on. So it
1063
+ joins the three reads above inside this ending rather than beside it. */
1064
+ let rules;
1065
+ let settings;
1066
+ try {
1067
+ brief = briefFor(turns[0].card_title, turns, await receiptsFor(client, cardId), await attachmentsFor(client, cardId), await codebasesForRun(client, runId));
1068
+ rules = await standingRulesFor(client, runId);
1069
+ settings = await settingsForRun(client, runId);
1070
+ where = await workingDirectory(client, runId, LEVEL, false);
1071
+ }
1072
+ catch (error) {
1073
+ const why = error instanceof Error ? error.message : String(error);
1074
+ await endRun(client, LEVEL, runId, cardId, why, attempt);
1075
+ throw new Error(`NO AGENT IS RUNNING: ${why}`);
1076
+ }
1077
+ /* ═══ THE RUN ID IS ON THE URL, AND THAT IS THE WHOLE OF WHAT THE AGENT IS
1078
+ TOLD ABOUT ITS OWN STANDING. ═══ The tools server reads the level off the
1079
+ run row this id names, so the daemon does not tell the child what it may do
1080
+ and the child has nothing to claim. `LEVEL` below decides argv only — which
1081
+ of the harness's own tools the process gets — and the two can never disagree
1082
+ about the record, because only one of them consults it. */
1083
+ /* AN EMPTY DIRECTORY OF THE USER'S OWN, because level 1 has no code tool to
1084
+ use a real one with, and the daemon's inherited cwd under a launchd login
1085
+ item is the filesystem root. */
1086
+ const started = await startTrackedAgent(client, runId, preparation, tools.work, attempt, withStandingRules(rules, brief, where.block), LEVEL, tools.urlFor(runId), where.cwd, undefined, settings);
1087
+ out(`run ${runId} level ${LEVEL} ${started.pid ? `pid ${started.pid}` : 'no process'}`);
1088
+ try {
1089
+ /* THE BRIEF, NOT WHAT THE PROCESS WAS HANDED. The rules are current at the
1090
+ activation and the brief is immutable, so storing the composed string
1091
+ would freeze one inside the other and every respawn would replay it. */
1092
+ await recordProcess(client, runId, started.pid, brief, undefined, started.attempt);
1093
+ }
1094
+ catch (error) {
1095
+ // Said, not fatal. See `recordProcess` for what this costs and why the
1096
+ // agent is not killed over it.
1097
+ said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
1098
+ }
1099
+ return settle(client, tools, machineId, LEVEL, runId, cardId, started, false);
894
1100
  }
895
- catch (error) {
896
- // Said, not fatal. See `recordProcess` for what this costs and why the
897
- // agent is not killed over it.
898
- said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
1101
+ finally {
1102
+ preparation?.finish();
899
1103
  }
900
- return settle(client, tools, machineId, LEVEL, runId, cardId, started, false);
901
1104
  }
902
1105
  /**
903
1106
  * WHICH PROJECT A RUN'S CARD IS FILED UNDER, or null when the card has none.
@@ -1224,92 +1427,106 @@ async function workingDirectory(client, runId, level, isOwner, knownCodebase) {
1224
1427
  * Then the row, then the process, then the pid — constraint 8, in the only order
1225
1428
  * that satisfies it.
1226
1429
  */
1227
- async function startChild(client, tools, machineId, parentRunId, brief, codebase, parentProcessToken, choice = {}) {
1228
- const { data, error } = await client.rpc('panel3_dispatch', {
1229
- p_parent_run_id: parentRunId,
1230
- p_brief: brief,
1231
- p_machine_id: machineId,
1232
- p_codebase_id: codebase?.id ?? null,
1233
- p_codebase_label: codebase?.name ?? null,
1234
- p_process_token: parentProcessToken ?? null,
1235
- p_model: choice.model ?? null,
1236
- p_effort: choice.effort ?? null,
1237
- });
1238
- if (error)
1239
- throw new Error(`could not start an agent under run ${parentRunId}: ${readableWriteError(error.message)}`);
1240
- const row = data?.[0];
1241
- if (!row) {
1242
- /* NOTHING WAS WRITTEN AND NOTHING IS RUNNING, and the two reasons are said
1243
- together because the caller cannot tell them apart from here and both mean
1244
- the same thing to it. */
1245
- throw new Error(`NO AGENT WAS STARTED and nothing was written: run ${parentRunId} has ended, is already as `
1246
- + 'deep as anything may be sent from, or this conversation already has its owner. Exit now.');
1247
- }
1248
- if (row.run_level !== 2 && row.run_level !== 3) {
1249
- /* UNREACHABLE, AND STILL SETTLED. `panel3_dispatch` writes `parent.level + 1`
1250
- from a parent it has just checked is below 3, so there is no level here
1251
- this cannot spawn. If that ever stops being true, the row exists and
1252
- nothing will ever start for it, and leaving it `running` would make a
1253
- recovery sweep wait out the pid grace window to conclude what is already
1254
- known. */
1255
- const why = `run ${row.run_id} was written at level ${row.run_level}, which cannot be spawned`;
1256
- await giveUp(client, row.run_id, why, row.process_token ?? undefined);
1257
- throw new Error(why);
1258
- }
1259
- const level = row.run_level;
1260
- let where;
1261
- let prompt;
1262
- let pictures;
1263
- /* READ AGAINST THE CHILD'S OWN RUN ROW, NOT THE PARENT'S. The row already
1264
- exists (`panel3_dispatch` wrote it above) and it carries this child's
1265
- codebase, which is what decides which codebase-scoped rules it is under. */
1266
- let rules;
1267
- let settings;
1268
- try {
1269
- where = await workingDirectory(client, row.run_id, level, level === 2, codebase);
1270
- prompt = level === 2
1271
- ? await initialOwnerPrompt(client, row.run_card_id, parentRunId, brief)
1272
- : brief;
1273
- rules = await standingRulesFor(client, row.run_id);
1274
- settings = await settingsForRun(client, row.run_id);
1275
- /* Inside this site's own try, for `standingRulesFor`'s reason: a failure to
1276
- assemble what the agent needs ends the run the way this path already ends
1277
- runs, rather than starting a process that is missing it. */
1278
- pictures = await picturesOnDisk(client, row.run_card_id, where, level);
1279
- }
1280
- catch (error) {
1281
- const why = error instanceof Error ? error.message : String(error);
1282
- await giveUp(client, row.run_id, why, row.process_token ?? undefined);
1283
- throw new Error(`NO AGENT IS RUNNING: ${why}`);
1284
- }
1285
- const processToken = row.process_token ?? undefined;
1286
- const isOwner = level === 2;
1287
- const started = startAgent(withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id, processToken), where.cwd, isOwner ? { ownerId: row.run_id } : undefined, settings);
1288
- if (started.pid === null) {
1289
- /* THERE IS A ROW AND THERE IS NO PROCESS, which is the one shape the record
1290
- must never be left in quietly. The answer is already settled — nothing ran
1291
- — so the reason is read off it, the run is ended with that reason on it,
1292
- and the tool call fails saying no agent was started. */
1293
- const answer = await started.answered;
1294
- const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
1295
- await giveUp(client, row.run_id, reason, processToken);
1296
- throw new Error(`NO AGENT IS RUNNING: ${reason}`);
1297
- }
1298
- out(`dispatch run ${row.run_id} level ${level} under ${parentRunId} pid ${started.pid}`);
1430
+ async function startChild(client, tools, machineId, parentRunId, brief, codebase, parentProcessToken, choice = {}, recovered) {
1431
+ let claimOperationId;
1432
+ const preparation = tools.work?.prepare();
1299
1433
  try {
1300
- await recordProcess(client, row.run_id, started.pid, undefined, processToken);
1434
+ const response = recovered ?? await panelClaim(client, tools.work, 'panel3_dispatch', {
1435
+ p_parent_run_id: parentRunId,
1436
+ p_brief: brief,
1437
+ p_machine_id: machineId,
1438
+ p_codebase_id: codebase?.id ?? null,
1439
+ p_codebase_label: codebase?.name ?? null,
1440
+ p_process_token: parentProcessToken ?? null,
1441
+ p_model: choice.model ?? null,
1442
+ p_effort: choice.effort ?? null,
1443
+ });
1444
+ const { data, error, operationId } = response;
1445
+ claimOperationId = operationId;
1446
+ if (error)
1447
+ throw new Error(`could not start an agent under run ${parentRunId}: ${readableWriteError(error.message)}`);
1448
+ const row = data?.[0];
1449
+ if (!row) {
1450
+ /* NOTHING WAS WRITTEN AND NOTHING IS RUNNING, and the two reasons are said
1451
+ together because the caller cannot tell them apart from here and both mean
1452
+ the same thing to it. */
1453
+ throw new Error(`NO AGENT WAS STARTED and nothing was written: run ${parentRunId} has ended, is already as `
1454
+ + 'deep as anything may be sent from, or this conversation already has its owner. Exit now.');
1455
+ }
1456
+ const attempt = capturedPanelAttempt(data ?? [], row.run_id, tools.work);
1457
+ if (row.run_level !== 2 && row.run_level !== 3) {
1458
+ /* UNREACHABLE, AND STILL SETTLED. `panel3_dispatch` writes `parent.level + 1`
1459
+ from a parent it has just checked is below 3, so there is no level here
1460
+ this cannot spawn. If that ever stops being true, the row exists and
1461
+ nothing will ever start for it, and leaving it `running` would make a
1462
+ recovery sweep wait out the pid grace window to conclude what is already
1463
+ known. */
1464
+ const why = `run ${row.run_id} was written at level ${row.run_level}, which cannot be spawned`;
1465
+ await giveUp(client, row.run_id, why, row.process_token ?? undefined, attempt);
1466
+ throw new Error(why);
1467
+ }
1468
+ const level = row.run_level;
1469
+ let where;
1470
+ let prompt;
1471
+ let pictures;
1472
+ /* READ AGAINST THE CHILD'S OWN RUN ROW, NOT THE PARENT'S. The row already
1473
+ exists (`panel3_dispatch` wrote it above) and it carries this child's
1474
+ codebase, which is what decides which codebase-scoped rules it is under. */
1475
+ let rules;
1476
+ let settings;
1477
+ try {
1478
+ where = await workingDirectory(client, row.run_id, level, level === 2, codebase);
1479
+ prompt = level === 2
1480
+ ? await initialOwnerPrompt(client, row.run_card_id, parentRunId, brief)
1481
+ : brief;
1482
+ rules = await standingRulesFor(client, row.run_id);
1483
+ settings = await settingsForRun(client, row.run_id);
1484
+ /* Inside this site's own try, for `standingRulesFor`'s reason: a failure to
1485
+ assemble what the agent needs ends the run the way this path already ends
1486
+ runs, rather than starting a process that is missing it. */
1487
+ pictures = await picturesOnDisk(client, row.run_card_id, where, level);
1488
+ }
1489
+ catch (error) {
1490
+ const why = error instanceof Error ? error.message : String(error);
1491
+ await giveUp(client, row.run_id, why, row.process_token ?? undefined, attempt);
1492
+ throw new Error(`NO AGENT IS RUNNING: ${why}`);
1493
+ }
1494
+ const processToken = row.process_token ?? undefined;
1495
+ const isOwner = level === 2;
1496
+ const started = await startTrackedAgent(client, row.run_id, preparation, tools.work, attempt, withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id, processToken), where.cwd, isOwner ? { ownerId: row.run_id } : undefined, settings);
1497
+ if (started.pid === null) {
1498
+ if (started.interrupted?.())
1499
+ throw new Error('Work was interrupted by the service command.');
1500
+ /* THERE IS A ROW AND THERE IS NO PROCESS, which is the one shape the record
1501
+ must never be left in quietly. The answer is already settled — nothing ran
1502
+ — so the reason is read off it, the run is ended with that reason on it,
1503
+ and the tool call fails saying no agent was started. */
1504
+ const answer = await started.answered;
1505
+ const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
1506
+ await giveUp(client, row.run_id, reason, processToken, attempt);
1507
+ throw new Error(`NO AGENT IS RUNNING: ${reason}`);
1508
+ }
1509
+ out(`dispatch run ${row.run_id} level ${level} under ${parentRunId} pid ${started.pid}`);
1510
+ try {
1511
+ await recordProcess(client, row.run_id, started.pid, undefined, processToken, started.attempt);
1512
+ }
1513
+ catch (error) {
1514
+ // Said, not fatal, exactly as at level 1: the agent is running and killing
1515
+ // it over a bookkeeping write would cost the user the work.
1516
+ said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
1517
+ }
1518
+ return {
1519
+ runId: row.run_id,
1520
+ settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, true, processToken, isOwner && processToken
1521
+ ? ownerSessionLifecycle(started, row.run_id, settings.harness ?? harness(), processToken)
1522
+ : undefined),
1523
+ };
1301
1524
  }
1302
- catch (error) {
1303
- // Said, not fatal, exactly as at level 1: the agent is running and killing
1304
- // it over a bookkeeping write would cost the user the work.
1305
- said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
1525
+ finally {
1526
+ preparation?.finish();
1527
+ if (claimOperationId)
1528
+ tools.work?.finishRpcs([claimOperationId]);
1306
1529
  }
1307
- return {
1308
- runId: row.run_id,
1309
- settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, true, processToken, isOwner && processToken
1310
- ? ownerSessionLifecycle(started, row.run_id, settings.harness ?? harness(), processToken)
1311
- : undefined),
1312
- };
1313
1530
  }
1314
1531
  /**
1315
1532
  * HOW MANY PROCESSES HAVE BEEN STARTED FOR THIS RUN, counting the first.
@@ -1377,9 +1594,22 @@ async function attemptsSoFar(client, runId) {
1377
1594
  * Returns whether THIS call is what ended it. False is a fact about the record,
1378
1595
  * not a failure: the run had already ended, and the caller says so.
1379
1596
  */
1380
- async function endRun(client, level, runId, cardId, why) {
1597
+ async function endRun(client, level, runId, cardId, why, attempt, failureKind) {
1381
1598
  if (level !== 1)
1382
- return (await giveUp(client, runId, why)) !== null;
1599
+ return (await giveUp(client, runId, why, undefined, attempt, failureKind)) !== null;
1600
+ if (attempt) {
1601
+ const { data, error } = await client.rpc('panel3_end_run', {
1602
+ p_run_id: runId, p_reason: why, p_process_token: attempt.processToken,
1603
+ p_failure_kind: failureKind ?? null,
1604
+ p_expected_attempt: { started_at: attempt.startedAt, resumed_at: attempt.resumedAt },
1605
+ });
1606
+ if (error)
1607
+ throw new Error(`could not end run ${runId}: ${error.message}`);
1608
+ if (typeof data !== 'boolean')
1609
+ throw new Error(`could not end run ${runId}: no ending was acknowledged`);
1610
+ forgetSecrets(runId);
1611
+ return data;
1612
+ }
1383
1613
  /* ═══ THE RUN FIRST, AND THE CARD ONLY IF THIS RUN WAS STILL THE CARD'S TO
1384
1614
  FAIL. ═══
1385
1615
  The card used to be written first, and the argument for that was a daemon
@@ -1464,6 +1694,13 @@ function ownerSessionLifecycle(started, ownerId, ownerHarness, processToken, exp
1464
1694
  }
1465
1695
  function settle(client, tools, machineId, level, runId, cardId, started, speaksToTheCard = true, processToken, ownerSession) {
1466
1696
  return started.answered.then(async (answer) => {
1697
+ if (started.interrupted?.() || started.preparationFailed?.())
1698
+ return;
1699
+ if (!answer.ok)
1700
+ answer = { ...answer, reason: failureMessage(started.attempt?.harness ?? harness(), answer.failureKind ?? 'unknown') };
1701
+ started.deferOutcome?.({ surface: 'panel', ok: answer.ok,
1702
+ text: answer.ok && speaksToTheCard ? redactSecrets(processToken === undefined ? runId : `${runId}:${processToken}`, answer.text) : null,
1703
+ reason: answer.ok ? '' : answer.reason, failureKind: answer.ok ? undefined : answer.failureKind, level });
1467
1704
  /* ═══ A RUN THAT STOPPED TO ASK DID NOT DIE, WHATEVER THE HARNESS PRINTED
1468
1705
  ON ITS WAY OUT. ═══
1469
1706
 
@@ -1488,12 +1725,20 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
1488
1725
  into an ordinary ending, and then handed to `writeAnswer` — which already
1489
1726
  owns this case and already prints its sentence — rather than to a second
1490
1727
  ending written here beside it. */
1491
- if (!answer.ok) {
1728
+ const observe = (authenticated) => {
1729
+ const selected = started.attempt?.harness;
1730
+ if (selected)
1731
+ tools.work?.observeHarness(selected, authenticated ? 'authenticated' : 'sign-in-required', authenticated ? 'dispatch-success' : 'provider-rejected');
1732
+ };
1733
+ if (!answer.ok && answer.failureKind !== 'authentication') {
1492
1734
  const run = await runNow(client, runId);
1493
1735
  if (run.state === 'asked') {
1494
- const accepted = await writeAnswer(client, runId, cardId, null, processToken);
1495
- if (accepted)
1736
+ started.deferOutcome?.({ surface: 'panel', ok: true, text: null, reason: '', level });
1737
+ const accepted = await writeAnswer(client, runId, cardId, null, processToken, started.attempt);
1738
+ if (accepted) {
1739
+ observe(true);
1496
1740
  await ownerSession?.established();
1741
+ }
1497
1742
  else
1498
1743
  await ownerSession?.failed();
1499
1744
  return;
@@ -1503,7 +1748,9 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
1503
1748
  exit cleanly without an agent message after the tool succeeds. The
1504
1749
  record, not prose the launcher was told not to write, decides success. */
1505
1750
  if (level === 1 && run.conversationOwnerId !== null) {
1506
- await writeAnswer(client, runId, cardId, null, processToken);
1751
+ started.deferOutcome?.({ surface: 'panel', ok: true, text: null, reason: '', level });
1752
+ if (await writeAnswer(client, runId, cardId, null, processToken, started.attempt))
1753
+ observe(true);
1507
1754
  return;
1508
1755
  }
1509
1756
  }
@@ -1514,7 +1761,7 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
1514
1761
  Repeating an invalid model/effort request cannot repair it. Unknown
1515
1762
  process deaths retain the existing recovery policy. */
1516
1763
  const attempts = await attemptsSoFar(client, runId);
1517
- if (answer.retryable !== false && started.pid !== null && attempts < MAX_ATTEMPTS) {
1764
+ if (answer.retryable === true && started.pid !== null && attempts < MAX_ATTEMPTS) {
1518
1765
  out(`retry run ${runId} attempt ${attempts} of ${MAX_ATTEMPTS} died: ${answer.reason}`);
1519
1766
  const again = processToken === undefined
1520
1767
  ? await resumeRun(client, tools, machineId, runId, started.pid)
@@ -1552,8 +1799,10 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
1552
1799
  than two. */
1553
1800
  const why = attempts > 1 ? `${answer.reason} (after ${attempts} attempts)` : answer.reason;
1554
1801
  const ended = processToken === undefined
1555
- ? await endRun(client, level, runId, cardId, why)
1556
- : (await giveUp(client, runId, why, processToken)) !== null;
1802
+ ? await endRun(client, level, runId, cardId, why, started.attempt, answer.failureKind)
1803
+ : (await giveUp(client, runId, why, processToken, started.attempt, answer.failureKind)) !== null;
1804
+ if (ended && answer.failureKind === 'authentication')
1805
+ observe(false);
1557
1806
  if (!ended) {
1558
1807
  /* IT WAS ALREADY SETTLED, by recovery, which decided this process was
1559
1808
  gone before it said so itself, or by the person's Stop. Nothing was
@@ -1570,12 +1819,14 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
1570
1819
  /* ═══ ITS OWN WORDS, ONTO THE CARD, THROUGH THE SAME STATEMENT EVERY LEVEL
1571
1820
  USES. ═══ Nothing reads them, nothing shortens them and nothing waits to
1572
1821
  approve them: ux.md's "whoever did the work writes the answer". */
1573
- const accepted = await writeAnswer(client, runId, cardId, speaksToTheCard ? answer.text : null, processToken);
1574
- if (accepted)
1822
+ const accepted = await writeAnswer(client, runId, cardId, speaksToTheCard ? answer.text : null, processToken, started.attempt);
1823
+ if (accepted) {
1824
+ observe(true);
1575
1825
  await ownerSession?.established();
1826
+ }
1576
1827
  else
1577
1828
  await ownerSession?.failed();
1578
- });
1829
+ }).then(() => { started.acknowledgeOutcome?.(); started.completed?.(); });
1579
1830
  }
1580
1831
  /**
1581
1832
  * ═══ ONE RUN, STARTED AGAIN AS ITSELF, WITH WHAT IT WAS SENT AND WHAT IT HAD
@@ -1633,13 +1884,13 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
1633
1884
  * the record refuses a fourth, and the caller is the one that knows what to say
1634
1885
  * about the failure it was holding.
1635
1886
  */
1636
- async function resumeRun(client, tools, machineId, runId, afterPid) {
1887
+ async function resumeRun(client, tools, machineId, runId, afterPid, recovered) {
1637
1888
  /* ═══ THE SWEEP CHECKS BEFORE THE CLAIM AND THE RETRY CANNOT, AND `afterPid`
1638
1889
  IS THE WHOLE OF WHAT DECIDES IT. ═══ Said once, here. See the header for
1639
1890
  both halves of the argument: a machine with no checkout must not take a run
1640
1891
  it cannot start, and the level a retry needs the answer for is not known
1641
1892
  until the claim returns.
1642
-
1893
+ *
1643
1894
  ═══ IT IS THE EXISTENCE CHECK AND NOTHING MORE, WHICH IS WHAT CHANGED IN
1644
1895
  worktrees-8. ═══ Resolving the working copy now CREATES a branch, a folder
1645
1896
  and a row write, and every poll tick that loses the claim race would leave
@@ -1648,126 +1899,139 @@ async function resumeRun(client, tools, machineId, runId, afterPid) {
1648
1899
  this position was ever asking, and the copy is made after the claim. */
1649
1900
  if (afterPid === null)
1650
1901
  checkoutForCodebase(await codebaseOfRun(client, runId), hostname());
1651
- const { data, error } = await client.rpc('panel3_resume', {
1652
- p_run_id: runId,
1653
- p_machine_id: machineId,
1654
- p_after_pid: afterPid,
1655
- });
1656
- if (error)
1657
- throw new Error(`could not start run ${runId} again: ${error.message}`);
1658
- const claimed = data?.[0];
1659
- if (!claimed)
1660
- return null;
1661
- const level = claimed.run_level === 1 ? 1 : claimed.run_level === 2 ? 2 : 3;
1662
- if (claimed.run_level !== level) {
1663
- /* UNREACHABLE, AND STILL SETTLED, exactly as in `startChild`: the level
1664
- column is checked at three, so there is no level here this cannot spawn.
1665
- The claim has already happened, so leaving it would strand the run for a
1666
- whole grace window before anything looked at it again. */
1667
- const why = `run ${runId} is at level ${claimed.run_level}, which cannot be spawned`;
1668
- await giveUp(client, runId, why);
1669
- throw new Error(why);
1670
- }
1671
- /* ONE RESOLUTION FOR BOTH PATHS, AFTER THE CLAIM. It used to fork on whether
1672
- the sweep had already resolved a folder before the claim; since the copy is
1673
- the card's own and making it writes, both paths make it here, in the branch
1674
- that can end the run when it cannot be made. */
1675
- let where;
1676
- try {
1677
- where = await workingDirectory(client, runId, level, false);
1678
- }
1679
- catch (error) {
1680
- /* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
1681
- ═══ Constraint 6, and `startRearmed`'s own handling of the same two
1682
- calls: `worktreeForCard()` is already careful about this and says so;
1683
- `scratchDir()` is not, because it is `mkdirSync`, whose EACCES and
1684
- ENOTDIR messages name the directory they failed on. So the machine's own
1685
- error is kept for stderr and the record is told only what is true and
1686
- shareable. */
1687
- const stderrOnly = error instanceof Error ? error.message : String(error);
1688
- const why = level === 1
1689
- ? 'this machine could not make the empty directory this runs in'
1690
- : stderrOnly;
1691
- /* ═══ AND IT ENDS THE WAY A RUN OF THIS LEVEL ENDS. ═══ It was
1692
- `panel3_give_up` outright, which was right while only a dispatched run
1693
- could reach this function and is a leak now that the retry brings level
1694
- 1 here: handing the person's message back mints a new run with its
1695
- attempts at one, which is the bound the retry is under, undone by the
1696
- one path that could not start. See `endRun`. */
1697
- await endRun(client, level, runId, claimed.run_card_id, why);
1698
- throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? stderrOnly : why}`);
1699
- }
1700
- /* WHAT IT SENT OTHERS TO DO, FROM THE RECORD. Level 3 has no `dispatch`, so it
1701
- has no children to have: null says that, where an empty list would say it
1702
- chose to send nobody.
1703
-
1704
- ═══ AND LEVEL 1 HAS THEM TOO, WHICH ONLY THE RETRY CAN REACH. ═══ This read
1705
- used to be `level === 2`, which was correct only because nothing at level 1
1706
- ever got here. Left alone it would hand a coordinator a null child list, and
1707
- a coordinator that came back to no children dispatches its workers a second
1708
- time: ux.md's single most expensive failure. `startRearmed` reads it the
1709
- same way, for the same reason. */
1710
- const children = level === 3 ? null : await childrenOf(client, runId);
1711
- /* ═══ READ AGAIN ON EVERY START, WHICH IS THE WHOLE OF CONTRACT POINT 2. ═══
1712
- A resumed or retried run is handed the rules AS THEY STAND NOW, not as they
1713
- stood when it first began: a rule edited while the conversation was running
1714
- governs the rest of it, and a rule deleted while it was running stops
1715
- applying to it. That is only true because this read happens here rather than
1716
- once, at the top of the run's life.
1717
-
1718
- ═══ AND IT ENDS THE WAY THIS PATH ALREADY ENDS. ═══ The claim has already
1719
- happened, so a throw here would leave a run reading `running` with no
1720
- process. `endRun` with the level fork is this path's own ending (see the cwd
1721
- branch above for why level 1 may not simply be given up on), and the reason
1722
- is a `returned()` message naming tables and columns, which carries no local
1723
- path and is therefore shareable. */
1724
- let rules;
1725
- let settings;
1726
- let pictures;
1902
+ let claimOperationId;
1903
+ const preparation = tools.work?.prepare();
1727
1904
  try {
1728
- rules = await standingRulesFor(client, runId);
1729
- settings = await settingsForRun(client, runId);
1730
- /* ═══ WRITTEN AGAIN ON EVERY START, LIKE THE RULES. ═══ A resumed process is
1731
- a NEW process with a new copy of the working directory, so the files a
1732
- previous one was handed are not there any more, and the stored brief this
1733
- path replays cannot carry a path that was not known when it was written. */
1734
- pictures = await picturesOnDisk(client, claimed.run_card_id, where, level);
1735
- }
1736
- catch (error) {
1737
- const why = error instanceof Error ? error.message : String(error);
1738
- await endRun(client, level, runId, claimed.run_card_id, why);
1739
- throw new Error(`NO AGENT IS RUNNING: ${why}`);
1740
- }
1741
- const started = startAgent(
1742
- /* ═══ WHY IT DIED IS WHAT DIFFERS, AND IT IS TOLD THE TRUTH ABOUT IT. ═══
1743
- `resumePrompt` opens by saying the machine went down, which is true of the
1744
- sweep and false of a retry: the daemon that watched this harness exit is
1745
- still running. See `retryPrompt`. */
1746
- withStandingRules(rules, afterPid === null
1747
- ? resumePrompt(claimed.run_brief, claimed.run_report, children)
1748
- : retryPrompt(claimed.run_brief, claimed.run_report, children), where.block, [], pictures), level, tools.urlFor(runId), where.cwd, undefined, settings);
1749
- if (started.pid === null) {
1750
- /* THE CLAIM HAPPENED AND NO PROCESS DID, which is the one shape the record
1751
- must never be left in quietly. Same handling as a dispatch that could not
1752
- start: the reason is read off the settled answer and the run is ended with
1753
- it. */
1754
- const answer = await started.answered;
1755
- const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
1756
- // The same level fork, for the same reason. See `endRun`.
1757
- await endRun(client, level, runId, claimed.run_card_id, reason);
1758
- throw new Error(`NO AGENT IS RUNNING: ${reason}`);
1759
- }
1760
- out(`resume run ${runId} level ${level} pid ${started.pid} `
1761
- + `${claimed.run_report === null ? 'no report to carry' : 'carrying its report'}`);
1762
- try {
1763
- await recordProcess(client, runId, started.pid);
1905
+ const response = recovered ?? await panelClaim(client, tools.work, 'panel3_resume', {
1906
+ p_run_id: runId,
1907
+ p_machine_id: machineId,
1908
+ p_after_pid: afterPid,
1909
+ });
1910
+ const { data, error, operationId } = response;
1911
+ claimOperationId = operationId;
1912
+ if (error)
1913
+ throw new Error(`could not start run ${runId} again: ${error.message}`);
1914
+ const claimed = data?.[0];
1915
+ if (!claimed)
1916
+ return null;
1917
+ const attempt = capturedPanelAttempt(data ?? [], runId, tools.work);
1918
+ const level = claimed.run_level === 1 ? 1 : claimed.run_level === 2 ? 2 : 3;
1919
+ if (claimed.run_level !== level) {
1920
+ /* UNREACHABLE, AND STILL SETTLED, exactly as in `startChild`: the level
1921
+ column is checked at three, so there is no level here this cannot spawn.
1922
+ The claim has already happened, so leaving it would strand the run for a
1923
+ whole grace window before anything looked at it again. */
1924
+ const why = `run ${runId} is at level ${claimed.run_level}, which cannot be spawned`;
1925
+ await giveUp(client, runId, why, undefined, attempt);
1926
+ throw new Error(why);
1927
+ }
1928
+ /* ONE RESOLUTION FOR BOTH PATHS, AFTER THE CLAIM. It used to fork on whether
1929
+ the sweep had already resolved a folder before the claim; since the copy is
1930
+ the card's own and making it writes, both paths make it here, in the branch
1931
+ that can end the run when it cannot be made. */
1932
+ let where;
1933
+ try {
1934
+ where = await workingDirectory(client, runId, level, false);
1935
+ }
1936
+ catch (error) {
1937
+ /* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
1938
+ ═══ Constraint 6, and `startRearmed`'s own handling of the same two
1939
+ calls: `worktreeForCard()` is already careful about this and says so;
1940
+ `scratchDir()` is not, because it is `mkdirSync`, whose EACCES and
1941
+ ENOTDIR messages name the directory they failed on. So the machine's own
1942
+ error is kept for stderr and the record is told only what is true and
1943
+ shareable. */
1944
+ const stderrOnly = error instanceof Error ? error.message : String(error);
1945
+ const why = level === 1
1946
+ ? 'this machine could not make the empty directory this runs in'
1947
+ : stderrOnly;
1948
+ /* ═══ AND IT ENDS THE WAY A RUN OF THIS LEVEL ENDS. ═══ It was
1949
+ `panel3_give_up` outright, which was right while only a dispatched run
1950
+ could reach this function and is a leak now that the retry brings level
1951
+ 1 here: handing the person's message back mints a new run with its
1952
+ attempts at one, which is the bound the retry is under, undone by the
1953
+ one path that could not start. See `endRun`. */
1954
+ await endRun(client, level, runId, claimed.run_card_id, why, attempt);
1955
+ throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? stderrOnly : why}`);
1956
+ }
1957
+ /* WHAT IT SENT OTHERS TO DO, FROM THE RECORD. Level 3 has no `dispatch`, so it
1958
+ has no children to have: null says that, where an empty list would say it
1959
+ chose to send nobody.
1960
+ *
1961
+ ═══ AND LEVEL 1 HAS THEM TOO, WHICH ONLY THE RETRY CAN REACH. ═══ This read
1962
+ used to be `level === 2`, which was correct only because nothing at level 1
1963
+ ever got here. Left alone it would hand a coordinator a null child list, and
1964
+ a coordinator that came back to no children dispatches its workers a second
1965
+ time: ux.md's single most expensive failure. `startRearmed` reads it the
1966
+ same way, for the same reason. */
1967
+ const children = level === 3 ? null : await childrenOf(client, runId);
1968
+ /* ═══ READ AGAIN ON EVERY START, WHICH IS THE WHOLE OF CONTRACT POINT 2. ═══
1969
+ A resumed or retried run is handed the rules AS THEY STAND NOW, not as they
1970
+ stood when it first began: a rule edited while the conversation was running
1971
+ governs the rest of it, and a rule deleted while it was running stops
1972
+ applying to it. That is only true because this read happens here rather than
1973
+ once, at the top of the run's life.
1974
+ *
1975
+ ═══ AND IT ENDS THE WAY THIS PATH ALREADY ENDS. ═══ The claim has already
1976
+ happened, so a throw here would leave a run reading `running` with no
1977
+ process. `endRun` with the level fork is this path's own ending (see the cwd
1978
+ branch above for why level 1 may not simply be given up on), and the reason
1979
+ is a `returned()` message naming tables and columns, which carries no local
1980
+ path and is therefore shareable. */
1981
+ let rules;
1982
+ let settings;
1983
+ let pictures;
1984
+ try {
1985
+ rules = await standingRulesFor(client, runId);
1986
+ settings = await settingsForRun(client, runId);
1987
+ /* ═══ WRITTEN AGAIN ON EVERY START, LIKE THE RULES. ═══ A resumed process is
1988
+ a NEW process with a new copy of the working directory, so the files a
1989
+ previous one was handed are not there any more, and the stored brief this
1990
+ path replays cannot carry a path that was not known when it was written. */
1991
+ pictures = await picturesOnDisk(client, claimed.run_card_id, where, level);
1992
+ }
1993
+ catch (error) {
1994
+ const why = error instanceof Error ? error.message : String(error);
1995
+ await endRun(client, level, runId, claimed.run_card_id, why, attempt);
1996
+ throw new Error(`NO AGENT IS RUNNING: ${why}`);
1997
+ }
1998
+ /* ═══ WHY IT DIED IS WHAT DIFFERS, AND IT IS TOLD THE TRUTH ABOUT IT. ═══
1999
+ `resumePrompt` opens by saying the machine went down, which is true of the
2000
+ sweep and false of a retry: the daemon that watched this harness exit is
2001
+ still running. See `retryPrompt`. */
2002
+ const started = await startTrackedAgent(client, runId, preparation, tools.work, attempt, withStandingRules(rules, afterPid === null
2003
+ ? resumePrompt(claimed.run_brief, claimed.run_report, children)
2004
+ : retryPrompt(claimed.run_brief, claimed.run_report, children), where.block, [], pictures), level, tools.urlFor(runId), where.cwd, undefined, settings);
2005
+ if (started.pid === null) {
2006
+ if (started.interrupted?.())
2007
+ throw new Error('Work was interrupted by the service command.');
2008
+ /* THE CLAIM HAPPENED AND NO PROCESS DID, which is the one shape the record
2009
+ must never be left in quietly. Same handling as a dispatch that could not
2010
+ start: the reason is read off the settled answer and the run is ended with
2011
+ it. */
2012
+ const answer = await started.answered;
2013
+ const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
2014
+ // The same level fork, for the same reason. See `endRun`.
2015
+ await endRun(client, level, runId, claimed.run_card_id, reason, attempt);
2016
+ throw new Error(`NO AGENT IS RUNNING: ${reason}`);
2017
+ }
2018
+ out(`resume run ${runId} level ${level} pid ${started.pid} `
2019
+ + `${claimed.run_report === null ? 'no report to carry' : 'carrying its report'}`);
2020
+ try {
2021
+ await recordProcess(client, runId, started.pid, undefined, undefined, started.attempt);
2022
+ }
2023
+ catch (error) {
2024
+ // Said, not fatal, as everywhere else: the agent is running and killing it
2025
+ // over a bookkeeping write would cost the user the work a second time.
2026
+ said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
2027
+ }
2028
+ return { settled: settle(client, tools, machineId, level, runId, claimed.run_card_id, started) };
1764
2029
  }
1765
- catch (error) {
1766
- // Said, not fatal, as everywhere else: the agent is running and killing it
1767
- // over a bookkeeping write would cost the user the work a second time.
1768
- said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
2030
+ finally {
2031
+ preparation?.finish();
2032
+ if (claimOperationId)
2033
+ tools.work?.finishRpcs([claimOperationId]);
1769
2034
  }
1770
- return { settled: settle(client, tools, machineId, level, runId, claimed.run_card_id, started) };
1771
2035
  }
1772
2036
  /**
1773
2037
  * ═══ ONE RUN, STARTED AGAIN BECAUSE SOMETHING IT WAS WAITING ON EXISTS NOW. ═══
@@ -1796,105 +2060,114 @@ async function resumeRun(client, tools, machineId, runId, afterPid) {
1796
2060
  * and in `cs show`, rather than quietly hoping the next poll finds it.
1797
2061
  */
1798
2062
  async function startRearmed(client, tools, machineId, row) {
1799
- const level = row.run_level === 1 ? 1 : row.run_level === 2 ? 2 : 3;
1800
- if (row.run_level !== level) {
1801
- const why = `run ${row.run_id} is at level ${row.run_level}, which cannot be spawned`;
1802
- await giveUp(client, row.run_id, why);
1803
- throw new Error(why);
1804
- }
1805
- let where;
2063
+ const preparation = tools.work?.prepare(row.run_id);
1806
2064
  try {
1807
- where = await workingDirectory(client, row.run_id, level, false);
1808
- }
1809
- catch (error) {
1810
- /* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
1811
- ═══ Constraint 6. `workingCopy()` is already careful about this and says
1812
- so; `scratchDir()` is not — it is `mkdirSync`, whose EACCES and ENOTDIR
1813
- messages name the directory they failed on — and `failed_because` is a
1814
- column `cs show` prints. So the machine's own error is kept for stderr
1815
- and the record is told only what is true and shareable. */
1816
- const said = error instanceof Error ? error.message : String(error);
1817
- const why = level === 1
1818
- ? 'this machine could not make the empty directory this runs in'
1819
- : said;
1820
- // The level fork, which this path needs for the same reason `resumeRun`'s
1821
- // two do: a re-arm serves level 1, and a level 1 run holds the person's
1822
- // message. See `endRun`.
1823
- await endRun(client, level, row.run_id, row.run_card_id, why);
1824
- throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? said : why}`);
1825
- }
1826
- /* WHAT IT SENT OTHERS TO DO AND WHAT THEY WROTE, FROM THE RECORD, AS LATE AS
1827
- POSSIBLE. Level 3 has no `dispatch`, so it has no children to have: null
1828
- says that, where an empty list would say it chose to send nobody. */
1829
- const children = level === 3 ? null : await childrenOf(client, row.run_id);
1830
- const deliveredArtifact = row.ask_id !== null && row.mine
1831
- ? await rearmedArtifactAnswer(client, row.ask_id)
1832
- : null;
1833
- /* ═══ THE MERGE HAPPENS HERE, OFF THE MARK ON THE ASK, BEFORE THE AGENT THAT
1834
- WILL SPEAK ABOUT IT IS STARTED. ═══ The re-arm is the last thing that runs
1835
- before the spawn, which is why it is the only place the outcome can reach
1836
- the prompt. See `landCardWork`. */
1837
- const offered = row.ask_id !== null && row.mine
1838
- ? await landingOffer(client, row.ask_id)
1839
- : null;
1840
- const landing = offered === null ? null : landCardWork(where, offered);
1841
- /* ═══ THREE REASONS, AND THE ROW SAYS WHICH. ═══ No question is ux.md's third
1842
- re-arm: everybody it sent has finished, and it is started to read them back.
1843
- `children` cannot be null on that path only a run with children is ever
1844
- claimed for it and the prompt takes the list rather than the maybe-list so
1845
- that is a fact of the signature rather than of a comment. */
1846
- const prompt = row.ask_id === null
1847
- ? readBackPrompt(row.run_brief, row.run_report, children ?? [])
1848
- : row.mine
1849
- ? answerPrompt(row.run_brief, row.run_report, children, row.question ?? '', row.answer ?? '', deliveredArtifact, landing)
1850
- : escalationPrompt(row.run_brief, row.run_report, children, row.ask_id, row.question ?? '');
1851
- /* CURRENT AT THIS ACTIVATION, exactly as on the resume path, and ended the
1852
- same way: the re-arm's claim has already happened, so a failure here ends
1853
- the run with its reason rather than leaving it claimed with no process. */
1854
- let rules;
1855
- let settings;
1856
- let pictures;
1857
- try {
1858
- rules = await standingRulesFor(client, row.run_id);
1859
- settings = await settingsForRun(client, row.run_id);
1860
- pictures = await picturesOnDisk(client, row.run_card_id, where, level);
1861
- }
1862
- catch (error) {
1863
- const why = error instanceof Error ? error.message : String(error);
1864
- await endRun(client, level, row.run_id, row.run_card_id, why);
1865
- throw new Error(`NO AGENT IS RUNNING: ${why}`);
1866
- }
1867
- const started = startAgent(withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id), where.cwd, undefined, settings);
1868
- if (started.pid === null) {
1869
- const answer = await started.answered;
1870
- const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
1871
- // The same level fork, for the same reason. See `endRun`.
1872
- await endRun(client, level, row.run_id, row.run_card_id, reason);
1873
- throw new Error(`NO AGENT IS RUNNING: ${reason}`);
1874
- }
1875
- out(`rearm run ${row.run_id} level ${level} pid ${started.pid} `
1876
- + `${row.ask_id === null
1877
- ? 'to read back everybody it sent'
2065
+ const attempt = capturedPanelAttempt([row], row.run_id, tools.work);
2066
+ const level = row.run_level === 1 ? 1 : row.run_level === 2 ? 2 : 3;
2067
+ if (row.run_level !== level) {
2068
+ const why = `run ${row.run_id} is at level ${row.run_level}, which cannot be spawned`;
2069
+ await giveUp(client, row.run_id, why, undefined, attempt);
2070
+ throw new Error(why);
2071
+ }
2072
+ let where;
2073
+ try {
2074
+ where = await workingDirectory(client, row.run_id, level, false);
2075
+ }
2076
+ catch (error) {
2077
+ /* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
2078
+ ═══ Constraint 6. `workingCopy()` is already careful about this and says
2079
+ so; `scratchDir()` is not it is `mkdirSync`, whose EACCES and ENOTDIR
2080
+ messages name the directory they failed on — and `failed_because` is a
2081
+ column `cs show` prints. So the machine's own error is kept for stderr
2082
+ and the record is told only what is true and shareable. */
2083
+ const said = error instanceof Error ? error.message : String(error);
2084
+ const why = level === 1
2085
+ ? 'this machine could not make the empty directory this runs in'
2086
+ : said;
2087
+ // The level fork, which this path needs for the same reason `resumeRun`'s
2088
+ // two do: a re-arm serves level 1, and a level 1 run holds the person's
2089
+ // message. See `endRun`.
2090
+ await endRun(client, level, row.run_id, row.run_card_id, why, attempt);
2091
+ throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? said : why}`);
2092
+ }
2093
+ /* WHAT IT SENT OTHERS TO DO AND WHAT THEY WROTE, FROM THE RECORD, AS LATE AS
2094
+ POSSIBLE. Level 3 has no `dispatch`, so it has no children to have: null
2095
+ says that, where an empty list would say it chose to send nobody. */
2096
+ const children = level === 3 ? null : await childrenOf(client, row.run_id);
2097
+ const deliveredArtifact = row.ask_id !== null && row.mine
2098
+ ? await rearmedArtifactAnswer(client, row.ask_id)
2099
+ : null;
2100
+ /* ═══ THE MERGE HAPPENS HERE, OFF THE MARK ON THE ASK, BEFORE THE AGENT THAT
2101
+ WILL SPEAK ABOUT IT IS STARTED. ═══ The re-arm is the last thing that runs
2102
+ before the spawn, which is why it is the only place the outcome can reach
2103
+ the prompt. See `landCardWork`. */
2104
+ const offered = row.ask_id !== null && row.mine
2105
+ ? await landingOffer(client, row.ask_id)
2106
+ : null;
2107
+ const landing = offered === null ? null : landCardWork(where, offered);
2108
+ /* ═══ THREE REASONS, AND THE ROW SAYS WHICH. ═══ No question is ux.md's third
2109
+ re-arm: everybody it sent has finished, and it is started to read them back.
2110
+ `children` cannot be null on that path only a run with children is ever
2111
+ claimed for it and the prompt takes the list rather than the maybe-list so
2112
+ that is a fact of the signature rather than of a comment. */
2113
+ const prompt = row.ask_id === null
2114
+ ? readBackPrompt(row.run_brief, row.run_report, children ?? [])
1878
2115
  : row.mine
1879
- ? 'with the answer to its own question'
1880
- : 'with a question it has to settle'}`);
1881
- try {
1882
- await recordProcess(client, row.run_id, started.pid);
2116
+ ? answerPrompt(row.run_brief, row.run_report, children, row.question ?? '', row.answer ?? '', deliveredArtifact, landing)
2117
+ : escalationPrompt(row.run_brief, row.run_report, children, row.ask_id, row.question ?? '');
2118
+ /* CURRENT AT THIS ACTIVATION, exactly as on the resume path, and ended the
2119
+ same way: the re-arm's claim has already happened, so a failure here ends
2120
+ the run with its reason rather than leaving it claimed with no process. */
2121
+ let rules;
2122
+ let settings;
2123
+ let pictures;
2124
+ try {
2125
+ rules = await standingRulesFor(client, row.run_id);
2126
+ settings = await settingsForRun(client, row.run_id);
2127
+ pictures = await picturesOnDisk(client, row.run_card_id, where, level);
2128
+ }
2129
+ catch (error) {
2130
+ const why = error instanceof Error ? error.message : String(error);
2131
+ await endRun(client, level, row.run_id, row.run_card_id, why, attempt);
2132
+ throw new Error(`NO AGENT IS RUNNING: ${why}`);
2133
+ }
2134
+ const started = await startTrackedAgent(client, row.run_id, preparation, tools.work, attempt, withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id), where.cwd, undefined, settings);
2135
+ if (started.pid === null) {
2136
+ if (started.interrupted?.())
2137
+ throw new Error('Work was interrupted by the service command.');
2138
+ const answer = await started.answered;
2139
+ const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
2140
+ // The same level fork, for the same reason. See `endRun`.
2141
+ await endRun(client, level, row.run_id, row.run_card_id, reason, attempt);
2142
+ throw new Error(`NO AGENT IS RUNNING: ${reason}`);
2143
+ }
2144
+ out(`rearm run ${row.run_id} level ${level} pid ${started.pid} `
2145
+ + `${row.ask_id === null
2146
+ ? 'to read back everybody it sent'
2147
+ : row.mine
2148
+ ? 'with the answer to its own question'
2149
+ : 'with a question it has to settle'}`);
2150
+ try {
2151
+ await recordProcess(client, row.run_id, started.pid, undefined, undefined, started.attempt);
2152
+ }
2153
+ catch (error) {
2154
+ // Said, not fatal, as everywhere else: the agent is running and killing it
2155
+ // over a bookkeeping write would cost the user the work.
2156
+ said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
2157
+ }
2158
+ /* ═══ AND IT SPEAKS TO THE CARD ONLY IF IT IS DOING THE PERSON'S WORK. ═══
2159
+ Exactly the distinction ux.md draws: a run carrying on with its own work has
2160
+ something to say when it finishes, and a run started only to settle somebody
2161
+ else's question does not. A run started to read back everybody it sent is
2162
+ the first kind and the clearest case of it — that reply IS the answer to the
2163
+ request. See `settle`. */
2164
+ return {
2165
+ settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, row.ask_id === null || !!row.mine),
2166
+ };
2167
+ }
2168
+ finally {
2169
+ preparation?.finish();
1883
2170
  }
1884
- catch (error) {
1885
- // Said, not fatal, as everywhere else: the agent is running and killing it
1886
- // over a bookkeeping write would cost the user the work.
1887
- said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
1888
- }
1889
- /* ═══ AND IT SPEAKS TO THE CARD ONLY IF IT IS DOING THE PERSON'S WORK. ═══
1890
- Exactly the distinction ux.md draws: a run carrying on with its own work has
1891
- something to say when it finishes, and a run started only to settle somebody
1892
- else's question does not. A run started to read back everybody it sent is
1893
- the first kind and the clearest case of it — that reply IS the answer to the
1894
- request. See `settle`. */
1895
- return {
1896
- settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, row.ask_id === null || !!row.mine),
1897
- };
1898
2171
  }
1899
2172
  /**
1900
2173
  * The runs one run dispatched, AND WHAT EACH OF THEM WROTE, in the words the
@@ -2047,140 +2320,159 @@ export function resumableOwnerSessionId(candidate, machineId, machineHarness) {
2047
2320
  ? local.nativeSessionId
2048
2321
  : undefined;
2049
2322
  }
2050
- async function activateOwner(client, tools, machineId, runId, afterProcessToken = null, afterPid = null) {
2051
- const candidate = await ownerCandidate(client, runId);
2052
- if (!candidate)
2053
- return null;
2054
- // Existing conversations retain their agent; an explicit hand-back follows
2055
- // the new selection. Never reuse a native session across different agents.
2056
- if (candidate.handed_back_at === null && candidate.machine_id !== machineId)
2057
- return null;
2058
- const machineHarness = candidate.handed_back_at === null && candidate.harness !== null
2059
- ? harness({ CTRL_SPC_V3_AGENT: candidate.harness })
2060
- : await selectedHarness(client, machineId);
2061
- const resumeSessionId = resumableOwnerSessionId(candidate, machineId, machineHarness);
2062
- /* ═══ THE EXISTENCE CHECK BEFORE THE CLAIM, AND THE COPY AFTER IT. ═══ This
2063
- was the whole resolution, which was right while resolving meant reading a
2064
- folder out of a file. Since worktrees-8 it also CREATES one, and a daemon
2065
- that lost the activation race would leave a branch and a folder behind for
2066
- an owner it never activated. The refusal this position exists for — a
2067
- machine that does not have this codebase must not take the activation —
2068
- is unchanged, because it is the located checkout that answers it. */
2069
- if (candidate.codebase_id !== null) {
2070
- checkoutForCodebase(await codebaseOfRun(client, candidate.id), hostname());
2071
- }
2072
- const { data, error } = await client.rpc('panel3_take_owner_activation', {
2073
- p_run_id: runId,
2074
- p_machine_id: machineId,
2075
- p_agent: machineHarness,
2076
- p_after_process_token: afterProcessToken,
2077
- p_after_pid: afterPid,
2078
- });
2079
- if (error)
2080
- throw new Error(`could not activate conversation owner ${runId}: ${error.message}`);
2081
- const claimed = data?.[0];
2082
- if (!claimed)
2083
- return null;
2084
- const [events, children] = await Promise.all([
2085
- ownerConversation(client, claimed.run_card_id, new Set(claimed.turn_ids ?? [])),
2086
- ownerChildren(client, runId),
2087
- ]);
2088
- const currentArtifactAnswer = deliveredArtifactAnswer(events, claimed);
2089
- const delivered = claimed.ask_id === null ? null : {
2090
- id: claimed.ask_id,
2091
- question: claimed.question ?? '(question unavailable)',
2092
- answer: claimed.answer,
2093
- mine: claimed.mine === true,
2094
- artifactAnswer: currentArtifactAnswer,
2095
- };
2096
- /* ═══ AFTER THE CLAIM, SO THE ENDING IS THE ONE THIS PATH HAS. ═══ Every
2097
- failure below the claim ends the activation with `giveUp` and its process
2098
- token; a throw above it would merely be an activation that did not happen.
2099
-
2100
- ═══ AND IT MATTERS MOST HERE. ═══ This is the owner, which lives for the
2101
- whole card and whose native session is RESUMED, so it is the one agent that
2102
- can be running while a person edits or deletes a rule. Reading at every
2103
- activation is what makes an edit govern the rest of the conversation, and
2104
- the block's own supersession sentence is what makes a DELETION take effect
2105
- in a session that still holds the older copy.
2106
-
2107
- ═══ AND IT IS BEFORE THE PROMPT SINCE worktrees-8 C1, because the prompt now
2108
- says what the product DID with the person's answer, and the landing needs
2109
- the card's copy. Nothing in the prompt depended on it before. */
2110
- let rules;
2111
- let settings;
2112
- /* THE CARD'S COPY IS MADE IN THE SAME WINDOW AND UNDER THE SAME ENDING, for
2113
- the reason the check above gives: it writes, so it happens after the claim,
2114
- and a failure to make it is an activation that ends rather than one that
2115
- sits `running` with no process. */
2116
- let where;
2117
- /* ═══ AND WHAT IS ATTACHED, READ IN THE SAME WINDOW AND UNDER THE SAME
2118
- ENDING. ═══ attaching-after-the-fact-10: a person may attach to a card that
2119
- is already running, and the owner's brief is immutable, so the only account
2120
- of attachments it would otherwise get is the one frozen at dispatch. Read
2121
- at every activation, exactly as the rules are, and for the same reason a
2122
- failed read ends the activation rather than continuing: "nothing is
2123
- attached" is a different card from the one the person sent.
2124
-
2125
- THE CODEBASE LINES ARE DROPPED. `whatWasAttached` partitions those into a
2126
- section whose own text says "its one codebase is named separately below",
2127
- a forward reference to something `workBrief` supplies and an owner
2128
- activation does not. This block carries the PERSON's attachments; where the
2129
- owner is working is `where.block`'s answer. */
2130
- let attached;
2131
- let pictures;
2323
+ async function activateOwner(client, tools, machineId, runId, afterProcessToken = null, afterPid = null, recovered) {
2324
+ let claimOperationId;
2325
+ const preparation = tools.work?.prepare();
2132
2326
  try {
2133
- where = await ownerDirectory(client, candidate);
2134
- rules = await standingRulesFor(client, runId);
2135
- settings = await settingsForRun(client, runId);
2136
- attached = whatWasAttached((await attachmentsFor(client, claimed.run_card_id))
2137
- .filter((line) => !line.startsWith('codebase ')));
2138
- /* ═══ THE OWNER'S OWN ACTIVATION, WHICH IS WHERE MOST PICTURES ARRIVE. ═══
2139
- The person sends one and this is the run that reads it. Written on every
2140
- activation rather than once, so an owner resumed into an existing native
2141
- conversation is told again about a directory a previous process wrote. */
2142
- pictures = await picturesOnDisk(client, claimed.run_card_id, where, 2);
2143
- }
2144
- catch (error) {
2145
- const why = error instanceof Error ? error.message : String(error);
2146
- await giveUp(client, runId, why, claimed.process_token);
2147
- throw new Error(`NO AGENT IS RUNNING: ${why}`);
2148
- }
2149
- /* THE MERGE, OFF THE MARK ON THE ASK, BEFORE THE AGENT THAT WILL SPEAK ABOUT
2150
- IT IS STARTED. The owner reaches an answer by this route as often as by the
2151
- re-arm, which is why the artifact answer is read on both and this is too. */
2152
- const offered = claimed.ask_id !== null && claimed.mine === true
2153
- ? await landingOffer(client, claimed.ask_id)
2154
- : null;
2155
- const landing = offered === null ? null : landCardWork(where, offered);
2156
- const prompt = resumeSessionId
2157
- ? ownerContinuationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered, landing)
2158
- : ownerActivationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered !== null && !delivered.mine
2159
- ? { id: delivered.id, question: delivered.question }
2160
- : null, currentArtifactAnswer,
2327
+ const candidate = await ownerCandidate(client, runId);
2328
+ if (!candidate)
2329
+ return null;
2330
+ // Existing conversations retain their agent; an explicit hand-back follows
2331
+ // the new selection. Never reuse a native session across different agents.
2332
+ if (candidate.handed_back_at === null && candidate.machine_id !== machineId)
2333
+ return null;
2334
+ const machineHarness = candidate.handed_back_at === null && candidate.harness !== null
2335
+ ? harness({ CTRL_SPC_V3_AGENT: candidate.harness })
2336
+ : await selectedHarness(client, machineId);
2337
+ if (recovered?.nativeContext && (recovered.nativeContext.harness !== machineHarness || recovered.nativeContext.processToken !== null && typeof recovered.nativeContext.processToken !== 'string' || recovered.nativeContext.nativeSessionId !== undefined && typeof recovered.nativeContext.nativeSessionId !== 'string'))
2338
+ throw new Error('The saved native conversation context is invalid.');
2339
+ const resumeSessionId = recovered?.nativeContext
2340
+ ? resumableOwnerSessionId({ ...candidate, process_token: recovered.nativeContext.processToken }, machineId, recovered.nativeContext.harness)
2341
+ : resumableOwnerSessionId(candidate, machineId, machineHarness);
2342
+ if (recovered?.nativeContext?.nativeSessionId && resumeSessionId !== recovered.nativeContext.nativeSessionId)
2343
+ throw new Error('The saved native conversation is no longer available for this attempt.');
2344
+ /* ═══ THE EXISTENCE CHECK BEFORE THE CLAIM, AND THE COPY AFTER IT. ═══ This
2345
+ was the whole resolution, which was right while resolving meant reading a
2346
+ folder out of a file. Since worktrees-8 it also CREATES one, and a daemon
2347
+ that lost the activation race would leave a branch and a folder behind for
2348
+ an owner it never activated. The refusal this position exists for — a
2349
+ machine that does not have this codebase must not take the activation —
2350
+ is unchanged, because it is the located checkout that answers it. */
2351
+ if (candidate.codebase_id !== null) {
2352
+ checkoutForCodebase(await codebaseOfRun(client, candidate.id), hostname());
2353
+ }
2354
+ const response = recovered ?? await panelClaim(client, tools.work, 'panel3_take_owner_activation', {
2355
+ p_run_id: runId,
2356
+ p_machine_id: machineId,
2357
+ p_agent: machineHarness,
2358
+ p_after_process_token: afterProcessToken,
2359
+ p_after_pid: afterPid,
2360
+ }, { nativeSessionId: resumeSessionId, processToken: candidate.process_token, harness: machineHarness });
2361
+ const { data, error, operationId } = response;
2362
+ claimOperationId = operationId;
2363
+ if (error)
2364
+ throw new Error(`could not activate conversation owner ${runId}: ${error.message}`);
2365
+ const claimed = data?.[0];
2366
+ if (!claimed)
2367
+ return null;
2368
+ const attempt = capturedPanelAttempt(data ?? [], runId, tools.work);
2369
+ const [events, children] = await Promise.all([
2370
+ ownerConversation(client, claimed.run_card_id, new Set(claimed.turn_ids ?? [])),
2371
+ ownerChildren(client, runId),
2372
+ ]);
2373
+ const currentArtifactAnswer = deliveredArtifactAnswer(events, claimed);
2374
+ const delivered = claimed.ask_id === null ? null : {
2375
+ id: claimed.ask_id,
2376
+ question: claimed.question ?? '(question unavailable)',
2377
+ answer: claimed.answer,
2378
+ mine: claimed.mine === true,
2379
+ artifactAnswer: currentArtifactAnswer,
2380
+ };
2381
+ /* ═══ AFTER THE CLAIM, SO THE ENDING IS THE ONE THIS PATH HAS. ═══ Every
2382
+ failure below the claim ends the activation with `giveUp` and its process
2383
+ token; a throw above it would merely be an activation that did not happen.
2384
+ *
2385
+ ═══ AND IT MATTERS MOST HERE. ═══ This is the owner, which lives for the
2386
+ whole card and whose native session is RESUMED, so it is the one agent that
2387
+ can be running while a person edits or deletes a rule. Reading at every
2388
+ activation is what makes an edit govern the rest of the conversation, and
2389
+ the block's own supersession sentence is what makes a DELETION take effect
2390
+ in a session that still holds the older copy.
2391
+ *
2392
+ ═══ AND IT IS BEFORE THE PROMPT SINCE worktrees-8 C1, because the prompt now
2393
+ says what the product DID with the person's answer, and the landing needs
2394
+ the card's copy. Nothing in the prompt depended on it before. */
2395
+ let rules;
2396
+ let settings;
2397
+ /* THE CARD'S COPY IS MADE IN THE SAME WINDOW AND UNDER THE SAME ENDING, for
2398
+ the reason the check above gives: it writes, so it happens after the claim,
2399
+ and a failure to make it is an activation that ends rather than one that
2400
+ sits `running` with no process. */
2401
+ let where;
2402
+ /* ═══ AND WHAT IS ATTACHED, READ IN THE SAME WINDOW AND UNDER THE SAME
2403
+ ENDING. ═══ attaching-after-the-fact-10: a person may attach to a card that
2404
+ is already running, and the owner's brief is immutable, so the only account
2405
+ of attachments it would otherwise get is the one frozen at dispatch. Read
2406
+ at every activation, exactly as the rules are, and for the same reason a
2407
+ failed read ends the activation rather than continuing: "nothing is
2408
+ attached" is a different card from the one the person sent.
2409
+ *
2410
+ THE CODEBASE LINES ARE DROPPED. `whatWasAttached` partitions those into a
2411
+ section whose own text says "its one codebase is named separately below",
2412
+ a forward reference to something `workBrief` supplies and an owner
2413
+ activation does not. This block carries the PERSON's attachments; where the
2414
+ owner is working is `where.block`'s answer. */
2415
+ let attached;
2416
+ let pictures;
2417
+ try {
2418
+ where = await ownerDirectory(client, candidate);
2419
+ rules = await standingRulesFor(client, runId);
2420
+ settings = await settingsForRun(client, runId);
2421
+ attached = whatWasAttached((await attachmentsFor(client, claimed.run_card_id))
2422
+ .filter((line) => !line.startsWith('codebase ')));
2423
+ /* ═══ THE OWNER'S OWN ACTIVATION, WHICH IS WHERE MOST PICTURES ARRIVE. ═══
2424
+ The person sends one and this is the run that reads it. Written on every
2425
+ activation rather than once, so an owner resumed into an existing native
2426
+ conversation is told again about a directory a previous process wrote. */
2427
+ pictures = await picturesOnDisk(client, claimed.run_card_id, where, 2);
2428
+ }
2429
+ catch (error) {
2430
+ const why = error instanceof Error ? error.message : String(error);
2431
+ await giveUp(client, runId, why, claimed.process_token, attempt);
2432
+ throw new Error(`NO AGENT IS RUNNING: ${why}`);
2433
+ }
2434
+ /* THE MERGE, OFF THE MARK ON THE ASK, BEFORE THE AGENT THAT WILL SPEAK ABOUT
2435
+ IT IS STARTED. The owner reaches an answer by this route as often as by the
2436
+ re-arm, which is why the artifact answer is read on both and this is too. */
2437
+ const offered = claimed.ask_id !== null && claimed.mine === true
2438
+ ? await landingOffer(client, claimed.ask_id)
2439
+ : null;
2440
+ const landing = offered === null ? null : landCardWork(where, offered);
2161
2441
  /* ═══ A PROCESS OF ITS OWN ENDED BEFORE IT FINISHED. ═══ `afterPid` is the
2162
2442
  fact, and it is non-null on all three paths that follow one: a harness
2163
2443
  that crashed, a machine that went down, and now a person's correction.
2164
2444
  The sentences it adds say what to do and never why, because those three
2165
2445
  are not the same event and `prompt.ts` exists to stop an agent being
2166
2446
  told an untrue reason for its own restart. */
2167
- afterPid !== null, landing);
2168
- const started = startAgent(withStandingRules(rules, prompt, where.block, attached, pictures), 2, tools.urlFor(runId, claimed.process_token), where.cwd, { ownerId: runId, ...(resumeSessionId ? { resumeSessionId } : {}) }, settings);
2169
- if (started.pid === null) {
2170
- const answer = await started.answered;
2171
- const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
2172
- await giveUp(client, runId, reason, claimed.process_token);
2173
- throw new Error(`NO AGENT IS RUNNING: ${reason}`);
2174
- }
2175
- try {
2176
- await recordProcess(client, runId, started.pid, undefined, claimed.process_token);
2447
+ const prompt = resumeSessionId
2448
+ ? ownerContinuationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered, landing)
2449
+ : ownerActivationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered !== null && !delivered.mine
2450
+ ? { id: delivered.id, question: delivered.question }
2451
+ : null, currentArtifactAnswer, afterPid !== null, landing);
2452
+ const started = await startTrackedAgent(client, runId, preparation, tools.work, attempt, withStandingRules(rules, prompt, where.block, attached, pictures), 2, tools.urlFor(runId, claimed.process_token), where.cwd, { ownerId: runId, ...(resumeSessionId ? { resumeSessionId } : {}) }, settings);
2453
+ if (started.pid === null) {
2454
+ if (started.interrupted?.())
2455
+ throw new Error('Work was interrupted by the service command.');
2456
+ const answer = await started.answered;
2457
+ const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
2458
+ await giveUp(client, runId, reason, claimed.process_token, attempt);
2459
+ throw new Error(`NO AGENT IS RUNNING: ${reason}`);
2460
+ }
2461
+ try {
2462
+ await recordProcess(client, runId, started.pid, undefined, claimed.process_token, started.attempt);
2463
+ }
2464
+ catch (error) {
2465
+ said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
2466
+ }
2467
+ return {
2468
+ settled: settle(client, tools, machineId, 2, runId, claimed.run_card_id, started, true, claimed.process_token, ownerSessionLifecycle(started, runId, machineHarness, claimed.process_token, resumeSessionId)),
2469
+ };
2177
2470
  }
2178
- catch (error) {
2179
- said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
2471
+ finally {
2472
+ preparation?.finish();
2473
+ if (claimOperationId)
2474
+ tools.work?.finishRpcs([claimOperationId]);
2180
2475
  }
2181
- return {
2182
- settled: settle(client, tools, machineId, 2, runId, claimed.run_card_id, started, true, claimed.process_token, ownerSessionLifecycle(started, runId, machineHarness, claimed.process_token, resumeSessionId)),
2183
- };
2184
2476
  }
2185
2477
  /** The only owner rows the normal poll may try, including an explicit hand-back move.
2186
2478
  *
@@ -2256,7 +2548,8 @@ export function redirectedProcess(candidate, machineId,
2256
2548
  /** The newest unaddressed person turn per card. MAX, never first-seen: a turn
2257
2549
  * left unaddressed while a question was open would otherwise pin its card
2258
2550
  * below `resumed_at` for good and nothing on it could ever redirect. */
2259
- waiting, cardId, booted) {
2551
+ waiting, cardId, booted, executionHeld = false) {
2552
+ // A dead transport can still own live tools in the durable execution boundary.
2260
2553
  // The claim's own five, mirrored.
2261
2554
  if (candidate.state !== 'running' || candidate.ended_at !== null)
2262
2555
  return null;
@@ -2288,7 +2581,7 @@ waiting, cardId, booted) {
2288
2581
  if (new Date(said).getTime() <= new Date(candidate.resumed_at ?? candidate.started_at).getTime()) {
2289
2582
  return null;
2290
2583
  }
2291
- return runProcessIsAlive(candidate, booted) ? candidate.pid : null;
2584
+ return executionHeld || runProcessIsAlive(candidate, booted) ? candidate.pid : null;
2292
2585
  }
2293
2586
  export function pendingOwnerSessionWithinGrace(candidate, now = Date.now()) {
2294
2587
  const local = readOwnerSession(candidate.id);
@@ -2331,9 +2624,17 @@ async function takeOwnerActivations(client, tools, machineId, mine, hold) {
2331
2624
  else and there is a dead process, a row still reading `running` and a card
2332
2625
  still saying `working`, with nothing local to retry it. */
2333
2626
  if (mine.has(candidate.id)) {
2334
- const pid = redirectedProcess({ ...candidate, card_state: candidate.card?.state ?? null }, machineId, waiting, candidate.card_id, bootedAt());
2627
+ const attempt = recoveryAttempt(candidate);
2628
+ const executionHeld = tools.work?.heldAttempt(attempt) ?? false;
2629
+ // A run ID alone cannot authorize stopping a newer activation.
2630
+ if (tools.work && !executionHeld)
2631
+ continue;
2632
+ const pid = redirectedProcess({ ...candidate, card_state: candidate.card?.state ?? null }, machineId, waiting, candidate.card_id, bootedAt(), executionHeld);
2335
2633
  if (pid !== null) {
2336
- killTree({ pid, kill: (signal) => process.kill(pid, signal) });
2634
+ if (executionHeld)
2635
+ await tools.work.stopHeldAttempt(attempt);
2636
+ else
2637
+ killTree({ pid, kill: (signal) => process.kill(pid, signal) });
2337
2638
  /* ═══ SIGNALLED, NOT KILLED, AND THE WORD IS THE POINT. ═══ `killTree`
2338
2639
  swallows a refused signal on both platforms, so saying "killed" would
2339
2640
  claim a death this daemon never observed. A process that survives is
@@ -2344,6 +2645,8 @@ async function takeOwnerActivations(client, tools, machineId, mine, hold) {
2344
2645
  }
2345
2646
  continue;
2346
2647
  }
2648
+ if (tools.work?.heldAttempt(recoveryAttempt(candidate)))
2649
+ continue;
2347
2650
  if (pendingOwnerSessionWithinGrace(candidate))
2348
2651
  continue;
2349
2652
  if (candidate.handed_back_at !== null
@@ -2407,7 +2710,7 @@ function runProcessIsAlive(run, booted) {
2407
2710
  * touched here. A current owner row, live PID, in-flight process, or fresh
2408
2711
  * pid-null claim always defers cleanup. Stable state is removed only after the
2409
2712
  * owner row disappears, becomes terminal, or no longer owns its card. */
2410
- export async function reconcileOwnerSessions(client, machineId, _machineHarness, inFlightOwnerIds = new Set()) {
2713
+ export async function reconcileOwnerSessions(client, machineId, _machineHarness, inFlightOwnerIds = new Set(), work) {
2411
2714
  const mappingIds = listOwnerSessionIds();
2412
2715
  const homeIds = listPanel3CodexOwnerHomeIds();
2413
2716
  const all = [...new Set([...mappingIds, ...homeIds])];
@@ -2424,7 +2727,11 @@ export async function reconcileOwnerSessions(client, machineId, _machineHarness,
2424
2727
  for (const id of ids) {
2425
2728
  if (inFlightOwnerIds.has(id))
2426
2729
  continue;
2730
+ if (work?.heldLocalOwner(id))
2731
+ continue;
2427
2732
  const row = byId.get(id);
2733
+ if (row && work?.heldAttempt(recoveryAttempt(row)))
2734
+ continue;
2428
2735
  const localProcessInUse = !!row
2429
2736
  && row.machine_id === machineId
2430
2737
  && ((row.pid !== null && processIsAlive(row.pid))
@@ -2477,10 +2784,10 @@ export async function reconcileOwnerSessions(client, machineId, _machineHarness,
2477
2784
  * the next poll tries again: clearing it after an EPERM would say this machine
2478
2785
  * has no process for a run whose agent is still working.
2479
2786
  */
2480
- async function killStopped(client, machineId) {
2787
+ async function killStopped(client, machineId, work) {
2481
2788
  const stopped = await returned(client
2482
2789
  .from('panel3_runs')
2483
- .select('id, card_id, pid, state, started_at, resumed_at')
2790
+ .select('id, card_id, pid, state, started_at, resumed_at, process_token')
2484
2791
  .eq('machine_id', machineId)
2485
2792
  .in('state', [...ENDED_BY_THE_PERSON, 'finished', 'failed'])
2486
2793
  .not('pid', 'is', null), 'read', 'the runs on this machine that are not coming back');
@@ -2488,7 +2795,16 @@ async function killStopped(client, machineId) {
2488
2795
  return;
2489
2796
  const booted = bootedAt();
2490
2797
  for (const run of stopped) {
2491
- if (runProcessIsAlive(run, booted)) {
2798
+ let stoppedByOwner = false;
2799
+ if (work?.heldAttempt(recoveryAttempt(run))) {
2800
+ if (!ENDED_BY_THE_PERSON.includes(run.state))
2801
+ continue;
2802
+ // The recorded bridge can be gone while its owned tools still run.
2803
+ // Only the durable owner can confirm that the complete execution ended.
2804
+ await work.stopHeldAttempt(recoveryAttempt(run));
2805
+ stoppedByOwner = true;
2806
+ }
2807
+ if (!stoppedByOwner && runProcessIsAlive(run, booted)) {
2492
2808
  if (!ENDED_BY_THE_PERSON.includes(run.state))
2493
2809
  continue;
2494
2810
  try {
@@ -2509,15 +2825,18 @@ async function killStopped(client, machineId) {
2509
2825
  continue;
2510
2826
  }
2511
2827
  try {
2512
- await returned(client
2828
+ let cleared = client
2513
2829
  .from('panel3_runs')
2514
2830
  .update({ pid: null })
2515
- // A finished owner may activate again between this read and write.
2516
- // Only acknowledge the exact process whose exit was observed.
2517
2831
  .eq('id', run.id)
2518
2832
  .eq('pid', run.pid)
2519
- .in('state', [...ENDED_BY_THE_PERSON, 'finished', 'failed'])
2520
- .select('id'), 'clear the process id of', `run ${run.id}`);
2833
+ .eq('started_at', run.started_at)
2834
+ .in('state', [...ENDED_BY_THE_PERSON, 'finished', 'failed']);
2835
+ cleared = run.process_token === null ? cleared.is('process_token', null) : cleared.eq('process_token', run.process_token);
2836
+ cleared = run.resumed_at === null ? cleared.is('resumed_at', null) : cleared.eq('resumed_at', run.resumed_at);
2837
+ await returned(
2838
+ // A finished owner may activate again while its old execution closes.
2839
+ cleared.select('id'), 'clear the process id of', `run ${run.id}`);
2521
2840
  }
2522
2841
  catch (error) {
2523
2842
  // Said, not fatal. The kill has already happened; this is bookkeeping, and
@@ -2602,6 +2921,8 @@ async function recoverStranded(client, tools, machineId, mine, hold) {
2602
2921
  for (const run of live) {
2603
2922
  if (mine.has(run.id))
2604
2923
  continue;
2924
+ if (tools.work?.heldAttempt(recoveryAttempt(run)))
2925
+ continue;
2605
2926
  // WHEN THE ATTEMPT NOW RUNNING BEGAN, which is the first one until a resume
2606
2927
  // says otherwise. See the header.
2607
2928
  const startedAt = new Date(run.resumed_at ?? run.started_at).getTime();
@@ -2838,6 +3159,8 @@ export async function takeHandedBack(client, tools, machineId, mine, hold) {
2838
3159
  // ONCE, OUTSIDE THE LOOP. It is a property of this machine, not of a row.
2839
3160
  const booted = bootedAt();
2840
3161
  for (const run of offered) {
3162
+ if (tools.work?.heldAttempt(recoveryAttempt(run)))
3163
+ continue;
2841
3164
  if (mine.has(run.id)) {
2842
3165
  /* THIS DAEMON'S OWN LIVE WORK, OFFERED WHILE IT WAS BUSY BEING QUIET. See
2843
3166
  the header. Said rather than passed over in silence, because a machine
@@ -3028,7 +3351,7 @@ export async function sweepFinishedWorktrees(client) {
3028
3351
  export function clientReader(injected) {
3029
3352
  return typeof injected === 'function' ? injected : () => injected;
3030
3353
  }
3031
- export async function run(args, injected, signal, lifecycle) {
3354
+ export async function run(args, injected, signal, lifecycle, work) {
3032
3355
  let once = false;
3033
3356
  for (const arg of args) {
3034
3357
  if (arg === '--once')
@@ -3059,6 +3382,7 @@ export async function run(args, injected, signal, lifecycle) {
3059
3382
  const machineId = getMachineIdentity().id;
3060
3383
  const machineName = hostname();
3061
3384
  let listeningHarness = null;
3385
+ const listeningRoutes = new Set();
3062
3386
  /* THE RUNS THIS PROCESS IS HOLDING RIGHT NOW, so recovery cannot declare its
3063
3387
  own live work dead in the moment before a pid is recorded. It covers THIS
3064
3388
  daemon only, which is why `PID_GRACE_MS` exists for the other ones. Keyed by
@@ -3084,11 +3408,22 @@ export async function run(args, injected, signal, lifecycle) {
3084
3408
  `tools` is referenced inside the callback it is being given, which is safe
3085
3409
  for the plain reason that the callback can only run once a request has
3086
3410
  arrived at a server that by then exists. */
3411
+ if (work && !work.cloudAllowed())
3412
+ throw new Error('Cloud operations are suspended for this sign-in.');
3413
+ if (work)
3414
+ await reconcilePanelInterruptions(current(), machineId, work);
3087
3415
  const tools = await startToolsServer(current(), async (parentRunId, brief, codebase, processToken, choice) => {
3088
- const child = await startChild(current(), tools, machineId, parentRunId, brief, codebase, processToken, choice);
3089
- hold(child.runId, child.settled);
3090
- return { runId: child.runId };
3416
+ const endClaim = work?.beginClaim();
3417
+ try {
3418
+ const child = await startChild(current(), tools, machineId, parentRunId, brief, codebase, processToken, choice);
3419
+ hold(child.runId, child.settled);
3420
+ return { runId: child.runId };
3421
+ }
3422
+ finally {
3423
+ endClaim?.();
3424
+ }
3091
3425
  }, (runId, processToken, action) => recoverLanding(current(), machineId, runId, processToken, action));
3426
+ tools.work = work;
3092
3427
  out(`daemon machine ${machineId}`);
3093
3428
  out(`tools ${tools.urlFor('<run-id>')}`);
3094
3429
  out(once ? 'mode one poll' : `mode polling every ${POLL_INTERVAL_MS / 1000}s, Ctrl-C to stop`);
@@ -3129,6 +3464,8 @@ export async function run(args, injected, signal, lifecycle) {
3129
3464
  // Restart only at a poll boundary, before any claims, with no live work.
3130
3465
  if (lifecycle && !lifecycle.beforePoll(inFlight.size))
3131
3466
  break;
3467
+ let endClaim;
3468
+ const claimOperations = [];
3132
3469
  /* ═══ ONE POLL FAILING IS NOT THE DAEMON FAILING. ═══ Every read and write
3133
3470
  here throws on a network or database error, by design (constraint 7), and
3134
3471
  until Slice 4 that threw straight out of `panel3/cli.js run` and exited the process.
@@ -3141,37 +3478,119 @@ export async function run(args, injected, signal, lifecycle) {
3141
3478
  `--once` still fails loudly, because the acceptance harness reads the exit
3142
3479
  code and a swallowed failure there would make a broken suite look green. */
3143
3480
  try {
3481
+ if (work && !work.cloudAllowed()) {
3482
+ await sleep(POLL_INTERVAL_MS);
3483
+ continue;
3484
+ }
3485
+ if (work)
3486
+ await reconcilePanelInterruptions(current(), machineId, work);
3487
+ if (work && !work.allowed()) {
3488
+ // A successful read proves polling before recovery opens admission.
3489
+ const { error } = await current().from('panel3_cards').select('id').limit(1);
3490
+ if (error)
3491
+ throw error;
3492
+ lifecycle?.ready();
3493
+ await sleep(POLL_INTERVAL_MS);
3494
+ continue;
3495
+ }
3496
+ endClaim = work?.beginClaim();
3144
3497
  /* ═══ THE USER'S STOP IS HONOURED BEFORE ANYTHING ELSE ON THE POLL. ═══ It
3145
3498
  is the only thing here that a person is waiting on, and the two takes
3146
3499
  below can spend the rest of the poll starting agents. Nothing else needs
3147
3500
  to run first: `panel3_stop_card` has already ended the runs, so recovery
3148
3501
  cannot see them and neither take can start them. */
3149
- await killStopped(current(), machineId);
3502
+ await killStopped(current(), machineId, work);
3503
+ for (const claim of work?.recoveredClaims() ?? []) {
3504
+ const attempts = panelClaimAttempts(claim.result);
3505
+ const eligible = new Set();
3506
+ for (const attempt of attempts) {
3507
+ if (inFlight.has(attempt.runId))
3508
+ continue;
3509
+ const pending = work?.claimAttempt(attempt.runId);
3510
+ if (!pending)
3511
+ continue; // Preparation or an actual stop already owns it.
3512
+ const { data, error } = await current().from('panel3_runs')
3513
+ .select('started_at,resumed_at,process_token,state,ended_at').eq('id', attempt.runId).maybeSingle();
3514
+ if (error)
3515
+ throw error;
3516
+ if (data?.state === 'running' && data.ended_at === null && data.started_at === attempt.startedAt
3517
+ && (data.resumed_at ?? null) === attempt.resumedAt && (data.process_token ?? null) === attempt.processToken)
3518
+ eligible.add(attempt.runId);
3519
+ }
3520
+ const rows = (claim.result ?? []).filter((row) => eligible.has(row._attempt?.run_id ?? row.run_id));
3521
+ const response = { data: rows, error: null, operationId: claim.id, nativeContext: claim.args?.nativeContext };
3522
+ const args = claim.args ?? {};
3523
+ if (rows.length) {
3524
+ if (claim.action === 'panel3_take_turns') {
3525
+ for (const [cardId, turns] of byCard(rows))
3526
+ hold(turns[0].run_id, answerCard(current(), tools, machineId, cardId, turns));
3527
+ }
3528
+ else if (claim.action === 'panel3_take_rearms') {
3529
+ for (const row of rows) {
3530
+ const started = await startRearmed(current(), tools, machineId, row);
3531
+ hold(row.run_id, started.settled);
3532
+ }
3533
+ }
3534
+ else if (claim.action === 'panel3_dispatch') {
3535
+ const row = rows[0];
3536
+ const codebase = args.p_codebase_id ? await codebaseOfRun(current(), row.run_id) : null;
3537
+ const started = await startChild(current(), tools, machineId, String(args.p_parent_run_id), String(args.p_brief), codebase, typeof args.p_process_token === 'string' ? args.p_process_token : undefined, {}, response);
3538
+ hold(started.runId, started.settled);
3539
+ }
3540
+ else if (claim.action === 'panel3_resume') {
3541
+ const runId = String(args.p_run_id);
3542
+ const started = await resumeRun(current(), tools, machineId, runId, typeof args.p_after_pid === 'number' ? args.p_after_pid : null, response);
3543
+ if (started)
3544
+ hold(runId, started.settled);
3545
+ }
3546
+ else if (claim.action === 'panel3_take_owner_activation') {
3547
+ const runId = String(args.p_run_id);
3548
+ const started = await activateOwner(current(), tools, machineId, runId, typeof args.p_after_process_token === 'string' ? args.p_after_process_token : null, typeof args.p_after_pid === 'number' ? args.p_after_pid : null, response);
3549
+ if (started)
3550
+ hold(runId, started.settled);
3551
+ }
3552
+ else
3553
+ throw new Error('The saved panel claim has an unsupported action.');
3554
+ }
3555
+ work?.finishRpcs([claim.id]);
3556
+ }
3150
3557
  /* Publish readiness before claiming work. A card with an untaken turn reads the same whether a daemon
3151
3558
  is two seconds away or nobody has one running; this row is the only place
3152
3559
  the difference exists. It is written before the takes rather than after
3153
3560
  so that a machine which is up but busy still reads as up. */
3154
- // Resolve before claiming work. A failed read or missing binary must not
3155
- // leave the previous agent advertised as ready for new assignments.
3156
- let machineHarness;
3157
- try {
3158
- machineHarness = await selectedHarness(current(), machineId);
3561
+ const installed = recoveryHarnesses();
3562
+ for (const route of [...listeningRoutes]) {
3563
+ if (!installed.includes(route)) {
3564
+ await stopListening(current(), machineId, route);
3565
+ listeningRoutes.delete(route);
3566
+ }
3159
3567
  }
3160
- catch (error) {
3161
- if (listeningHarness !== null) {
3162
- await stopListening(current(), machineId, listeningHarness);
3163
- listeningHarness = null;
3568
+ // A primary selection failure cannot skip an installed recovery route.
3569
+ for (const route of installed) {
3570
+ try {
3571
+ const response = await panelClaim(current(), work, 'panel3_take_turns', { p_machine_id: machineId, p_agent: route, p_recovery_only: true });
3572
+ if (response.operationId)
3573
+ claimOperations.push(response.operationId);
3574
+ const taken = await returned(Promise.resolve(response), 'take', 'explicit authentication recovery');
3575
+ await sayListening(current(), machineId, machineName, route);
3576
+ await sayPollingProblem(current(), machineId, route, null);
3577
+ listeningRoutes.add(route);
3578
+ for (const [cardId, turns] of byCard(taken))
3579
+ hold(turns[0].run_id, answerCard(current(), tools, machineId, cardId, turns));
3580
+ }
3581
+ catch (error) {
3582
+ try {
3583
+ await sayPollingProblem(current(), machineId, route, 'poll_failed');
3584
+ }
3585
+ catch { /* The last successful observation expires. */ }
3586
+ said(`Authentication recovery polling failed for ${route}: ${error instanceof Error ? error.message : String(error)}`);
3164
3587
  }
3165
- throw error;
3166
3588
  }
3589
+ const machineHarness = await selectedHarness(current(), machineId);
3590
+ listeningHarness = machineHarness;
3167
3591
  if (signal?.aborted)
3168
3592
  break;
3169
- if (listeningHarness !== null && listeningHarness !== machineHarness) {
3170
- await stopListening(current(), machineId, listeningHarness);
3171
- }
3172
- listeningHarness = machineHarness;
3173
- await sayListening(current(), machineId, machineName, machineHarness);
3174
- await reconcileOwnerSessions(current(), machineId, machineHarness, new Set(inFlight.keys()));
3593
+ await reconcileOwnerSessions(current(), machineId, machineHarness, new Set(inFlight.keys()), work);
3175
3594
  await recoverStranded(current(), tools, machineId, new Set(inFlight.keys()), hold);
3176
3595
  /* ═══ AND THE COPIES OF CARDS THAT ARE OVER. ═══ After recovery,
3177
3596
  deliberately: a run this machine is about to resume is one whose card is
@@ -3197,7 +3616,10 @@ export async function run(args, injected, signal, lifecycle) {
3197
3616
  /* THE MACHINE ID GOES IN because the take writes the run row, and a run has
3198
3617
  to say where it is running: the exclusion is cross-machine and recovery is
3199
3618
  per-machine, so a row with nobody's machine on it could be neither. */
3200
- const taken = await returned(current().rpc('panel3_take_turns', { p_machine_id: machineId, p_agent: machineHarness }), 'take', 'turns');
3619
+ const takenResponse = await panelClaim(current(), work, 'panel3_take_turns', { p_machine_id: machineId, p_agent: machineHarness, p_recovery_only: false });
3620
+ if (takenResponse.operationId)
3621
+ claimOperations.push(takenResponse.operationId);
3622
+ const taken = await returned(Promise.resolve(takenResponse), 'take', 'turns');
3201
3623
  /* ═══ THE OTHER KIND OF TAKEABLE WORK. ═══ ux.md's re-arm: an answered
3202
3624
  question makes the branch that asked it takeable again, and a question
3203
3625
  still walking up makes the run it reached takeable so that level gets its
@@ -3211,7 +3633,10 @@ export async function run(args, injected, signal, lifecycle) {
3211
3633
  to it on a later poll rather than putting two of them on one card. The
3212
3634
  other order would decide the same question from a snapshot taken before
3213
3635
  the run existed. */
3214
- const rearmed = await returned(current().rpc('panel3_take_rearms', { p_machine_id: machineId, p_agent: machineHarness }), 'take', 'runs that can carry on');
3636
+ const rearmedResponse = await panelClaim(current(), work, 'panel3_take_rearms', { p_machine_id: machineId, p_agent: machineHarness });
3637
+ if (rearmedResponse.operationId)
3638
+ claimOperations.push(rearmedResponse.operationId);
3639
+ const rearmed = await returned(Promise.resolve(rearmedResponse), 'take', 'runs that can carry on');
3215
3640
  for (const row of rearmed) {
3216
3641
  /* NOT AWAITED PAST THE SPAWN, exactly as a taken card is not: the work is a
3217
3642
  real agent and holding the poll open for it would put every other card
@@ -3239,6 +3664,8 @@ export async function run(args, injected, signal, lifecycle) {
3239
3664
  // rather than becoming an unhandled rejection.
3240
3665
  hold(runId, answerCard(current(), tools, machineId, cardId, turns));
3241
3666
  }
3667
+ await sayListening(current(), machineId, machineName, machineHarness);
3668
+ listeningRoutes.add(machineHarness);
3242
3669
  await sayPollingProblem(current(), machineId, machineHarness, null);
3243
3670
  lifecycle?.ready();
3244
3671
  if (once) {
@@ -3265,14 +3692,18 @@ export async function run(args, injected, signal, lifecycle) {
3265
3692
  }
3266
3693
  }
3267
3694
  }
3695
+ finally {
3696
+ work?.finishRpcs(claimOperations);
3697
+ endClaim?.();
3698
+ }
3268
3699
  if (!signal?.aborted)
3269
3700
  await sleep(POLL_INTERVAL_MS);
3270
3701
  }
3271
3702
  }
3272
3703
  finally {
3273
3704
  try {
3274
- if (listeningHarness !== null)
3275
- await stopListening(current(), machineId, listeningHarness);
3705
+ for (const route of listeningRoutes)
3706
+ await stopListening(current(), machineId, route);
3276
3707
  }
3277
3708
  finally {
3278
3709
  await tools.close();
@@ -3284,8 +3715,12 @@ export async function run(args, injected, signal, lifecycle) {
3284
3715
  * client. A stopped worker is restarted automatically. Explicit restarts happen
3285
3716
  * at an idle poll boundary and confirm only after a new worker completes a poll.
3286
3717
  * Sign-out stops the supervisor, so it cannot restart behind the user's back. */
3287
- export function startPanel(injected) {
3718
+ export function startPanel(injected, work) {
3288
3719
  const controller = new AbortController();
3720
+ let resolveReady;
3721
+ let rejectReady;
3722
+ const ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; });
3723
+ void ready.catch(() => { });
3289
3724
  let requested = false;
3290
3725
  let restarting = null;
3291
3726
  let resolveRestart = null;
@@ -3304,11 +3739,12 @@ export function startPanel(injected) {
3304
3739
  }
3305
3740
  return false;
3306
3741
  },
3307
- ready: () => { if (!requested)
3742
+ ready: () => { resolveReady(); if (!requested)
3308
3743
  resolveRestart?.(); },
3309
- });
3744
+ }, work);
3310
3745
  }
3311
3746
  catch (error) {
3747
+ rejectReady(error instanceof Error ? error : new Error(String(error)));
3312
3748
  said(`the agent panel stopped polling: ${error instanceof Error ? error.message : String(error)}`);
3313
3749
  // Startup failures must not leave an online machine with a dead worker.
3314
3750
  if (!controller.signal.aborted)
@@ -3317,6 +3753,7 @@ export function startPanel(injected) {
3317
3753
  }
3318
3754
  })();
3319
3755
  return {
3756
+ ready,
3320
3757
  restart: () => {
3321
3758
  if (controller.signal.aborted)
3322
3759
  return Promise.reject(new Error('This machine is signing out. Open Companion and sign in again.'));
@@ -3339,6 +3776,7 @@ export function startPanel(injected) {
3339
3776
  },
3340
3777
  stop: async () => {
3341
3778
  controller.abort();
3779
+ rejectReady(new Error('Cloud startup was cancelled by the service command.'));
3342
3780
  rejectRestart?.(new Error('The machine disconnected before the worker restarted.'));
3343
3781
  await running;
3344
3782
  },