@ctrl-spc/cs 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/codex-home.js +62 -1
- package/dist/panel3/prompt.js +329 -61
- package/dist/panel3/run.js +584 -56
- package/dist/panel3/session.js +128 -0
- package/dist/panel3/show.js +2 -1
- package/dist/panel3/spawn.js +80 -18
- package/dist/panel3/tools.js +353 -48
- package/dist/workflows.js +68 -0
- package/package.json +1 -1
package/dist/panel3/run.js
CHANGED
|
@@ -132,16 +132,18 @@
|
|
|
132
132
|
// and takes the leading comment with it, so a file whose first statement is
|
|
133
133
|
// `import type` loses its v3 header in the published `dist/`.
|
|
134
134
|
import { out, returned, signedInClient } from './client.js';
|
|
135
|
-
import { answerPrompt, escalationPrompt, levelOnePrompt, readBackPrompt, resumePrompt, retryPrompt, } from './prompt.js';
|
|
136
|
-
import { attachmentLine, loadAttachments, loadOutputNames, outputOf } from './show.js';
|
|
135
|
+
import { answerPrompt, escalationPrompt, levelOnePrompt, ownerActivationPrompt, readBackPrompt, presentedArtifactAnswerContext, resumePrompt, retryPrompt, } from './prompt.js';
|
|
136
|
+
import { ASK_CONTENT_COLUMNS, attachmentLine, loadAttachments, loadOutputNames, outputOf, withAskContent, } from './show.js';
|
|
137
137
|
import { forgetSecrets, redactSecrets } from './secrets.js';
|
|
138
138
|
import { sayListening, stopListening } from './presence.js';
|
|
139
139
|
import { checkoutForCodebase, hasCheckoutForCodebase } from './checkout.js';
|
|
140
140
|
import { harness, startAgent } from './spawn.js';
|
|
141
|
+
import { establishOwnerSession, listOwnerSessionIds, OWNER_SESSION_GRACE_MS, readOwnerSession, removeOwnerSession, validSessionUuid, writeOwnerSession, } from './session.js';
|
|
141
142
|
import { startToolsServer } from './tools.js';
|
|
143
|
+
import { listPanel3CodexOwnerHomeIds, removePanel3CodexOwnerHome, } from '../codex-home.js';
|
|
142
144
|
import { getMachineIdentity, scratchDir } from '../config.js';
|
|
143
145
|
import { listCodebases } from '../codebases.js';
|
|
144
|
-
import { processIsAlive } from '../win-shell.js';
|
|
146
|
+
import { killTree, processIsAlive } from '../win-shell.js';
|
|
145
147
|
import { hostname, uptime } from 'node:os';
|
|
146
148
|
const USAGE = 'usage: cs3 run [--once]';
|
|
147
149
|
/** How long between takes. Short, because it is the whole delay between a user
|
|
@@ -200,6 +202,28 @@ export const MAX_ATTEMPTS = 3;
|
|
|
200
202
|
* because an agent asked for it, and its level comes off the row
|
|
201
203
|
* `panel3_dispatch` wrote rather than from anything here. */
|
|
202
204
|
const LEVEL = 1;
|
|
205
|
+
const artifactAnswer = (id, revision, selected) => {
|
|
206
|
+
if (!id || revision === null || revision === undefined || selected?.length !== 1)
|
|
207
|
+
return null;
|
|
208
|
+
const answer = selected[0]?.trim().toLowerCase();
|
|
209
|
+
return answer === 'approve' || answer === 'request changes'
|
|
210
|
+
? { id, revision, answer }
|
|
211
|
+
: null;
|
|
212
|
+
};
|
|
213
|
+
export function deliveredArtifactAnswer(events, delivered) {
|
|
214
|
+
if (!delivered.mine || delivered.ask_id === null)
|
|
215
|
+
return null;
|
|
216
|
+
const event = events.find((candidate) => candidate.kind === 'question' && candidate.id === delivered.ask_id);
|
|
217
|
+
const answer = artifactAnswer(event?.relatedArtifactId, event?.relatedArtifactRevision, event?.artifactAnswer ? [event.artifactAnswer] : null);
|
|
218
|
+
return answer === null ? null : { questionId: delivered.ask_id, ...answer };
|
|
219
|
+
}
|
|
220
|
+
async function rearmedArtifactAnswer(client, askId) {
|
|
221
|
+
const asks = await withAskContent(client, await returned(client.from('panel3_asks').select(`id, ${ASK_CONTENT_COLUMNS}`).eq('id', askId), 'read', `question ${askId}`));
|
|
222
|
+
const ask = asks[0];
|
|
223
|
+
return ask === undefined
|
|
224
|
+
? null
|
|
225
|
+
: artifactAnswer(ask.related_artifact_id, ask.related_artifact_revision, ask.selected_options);
|
|
226
|
+
}
|
|
203
227
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
204
228
|
/**
|
|
205
229
|
* ═══ NOTHING IN THIS FILE STAMPS A COLUMN FROM THIS MACHINE'S CLOCK ANY MORE,
|
|
@@ -344,12 +368,16 @@ export const briefFor = (title, turns, produced, attachments, codebases = null)
|
|
|
344
368
|
* the run has no pid, so once this daemon is gone recovery treats it as dead —
|
|
345
369
|
* which errs towards answering again rather than stranding.
|
|
346
370
|
*/
|
|
347
|
-
async function recordProcess(client, runId, pid, brief) {
|
|
348
|
-
|
|
371
|
+
async function recordProcess(client, runId, pid, brief, processToken) {
|
|
372
|
+
let query = client
|
|
349
373
|
.from('panel3_runs')
|
|
350
374
|
.update({ pid, ...(brief === undefined ? {} : { brief }) })
|
|
351
|
-
.eq('id', runId)
|
|
352
|
-
|
|
375
|
+
.eq('id', runId);
|
|
376
|
+
if (processToken !== undefined)
|
|
377
|
+
query = query.eq('process_token', processToken);
|
|
378
|
+
const written = await returned(query.select('id'), 'record what is running', `run ${runId}`);
|
|
379
|
+
if (written.length === 0)
|
|
380
|
+
throw new Error(`could not record what is running for run ${runId}: its activation has ended`);
|
|
353
381
|
}
|
|
354
382
|
/**
|
|
355
383
|
* ═══ A RUN GIVEN UP ON: ENDED, ITS TURNS HANDED BACK, ITS CARD IN HAND. ═══
|
|
@@ -364,15 +392,19 @@ async function recordProcess(client, runId, pid, brief) {
|
|
|
364
392
|
* there first. It is a fact about the record rather than a failure, so it does
|
|
365
393
|
* not go through `returned()`, exactly as the answer's own null does not.
|
|
366
394
|
*/
|
|
367
|
-
async function giveUp(client, runId, reason) {
|
|
395
|
+
async function giveUp(client, runId, reason, processToken) {
|
|
368
396
|
const { data, error } = await client
|
|
369
|
-
.rpc('panel3_give_up', {
|
|
397
|
+
.rpc('panel3_give_up', {
|
|
398
|
+
p_run_id: runId,
|
|
399
|
+
p_reason: reason,
|
|
400
|
+
...(processToken === undefined ? {} : { p_process_token: processToken }),
|
|
401
|
+
});
|
|
370
402
|
if (error)
|
|
371
403
|
throw new Error(`could not give up run ${runId}: ${error.message}`);
|
|
372
404
|
/* THE RUN IS OVER, so whatever it read is dropped. Every ending does this —
|
|
373
405
|
here, `failRun` and `writeAnswer` — because a daemon stays up for days and
|
|
374
406
|
has no business holding Tuesday's secret. */
|
|
375
|
-
forgetSecrets(runId);
|
|
407
|
+
forgetSecrets(processToken === undefined ? runId : `${runId}:${processToken}`);
|
|
376
408
|
return data;
|
|
377
409
|
}
|
|
378
410
|
/**
|
|
@@ -411,7 +443,7 @@ async function giveUp(client, runId, reason) {
|
|
|
411
443
|
* a run can be the last thing live on it — but no turn is written, exactly as
|
|
412
444
|
* none is written for a run that stopped to ask.
|
|
413
445
|
*/
|
|
414
|
-
async function writeAnswer(client, runId, cardId, text) {
|
|
446
|
+
async function writeAnswer(client, runId, cardId, text, processToken) {
|
|
415
447
|
/* ═══ THE SECOND OF THE TWO CHOKEPOINTS THE CREDENTIAL RULE RESTS ON. ═══
|
|
416
448
|
This is the ONE path an agent's own words take to a hosted row — a turn on
|
|
417
449
|
the card at levels 1 and 2, and a level 3's `panel3_runs.report`, which
|
|
@@ -419,7 +451,11 @@ async function writeAnswer(client, runId, cardId, text) {
|
|
|
419
451
|
`buildServer`. Identity for a run that read no credential, so every existing
|
|
420
452
|
exact-string assertion is byte-identical. */
|
|
421
453
|
const { data: turnId, error } = await client
|
|
422
|
-
.rpc('panel3_answer', {
|
|
454
|
+
.rpc('panel3_answer', {
|
|
455
|
+
p_run_id: runId,
|
|
456
|
+
p_body: text === null ? null : redactSecrets(processToken === undefined ? runId : `${runId}:${processToken}`, text),
|
|
457
|
+
...(processToken === undefined ? {} : { p_process_token: processToken }),
|
|
458
|
+
});
|
|
423
459
|
if (error) {
|
|
424
460
|
/* Thrown, not settled as failed. The run stays `running` with a process
|
|
425
461
|
that has now exited, so recovery reaps it: at level 1 the turns are
|
|
@@ -433,7 +469,7 @@ async function writeAnswer(client, runId, cardId, text) {
|
|
|
433
469
|
credential again through `get_credential`. What it is handed to start from —
|
|
434
470
|
its own report, and the card — went through this same substitution, so there
|
|
435
471
|
is nothing left for a stale entry to protect. */
|
|
436
|
-
forgetSecrets(runId);
|
|
472
|
+
forgetSecrets(processToken === undefined ? runId : `${runId}:${processToken}`);
|
|
437
473
|
if (turnId === null) {
|
|
438
474
|
/* NO TURN WAS WRITTEN, and there are FOUR reasons, which mean different
|
|
439
475
|
things and must not be printed as one line.
|
|
@@ -469,6 +505,9 @@ async function writeAnswer(client, runId, cardId, text) {
|
|
|
469
505
|
else if (run.state !== 'finished') {
|
|
470
506
|
said(`run ${runId} was ${run.state}, so its answer was not written to card ${cardId}`);
|
|
471
507
|
}
|
|
508
|
+
else if (run.level === 1 && run.conversationOwnerId !== null) {
|
|
509
|
+
out(`settled card ${cardId} run ${runId} assigned the conversation, so it wrote nothing here`);
|
|
510
|
+
}
|
|
472
511
|
else if (text === null) {
|
|
473
512
|
out(`settled card ${cardId} run ${runId} answered a question, so it wrote nothing here`);
|
|
474
513
|
}
|
|
@@ -481,16 +520,32 @@ async function writeAnswer(client, runId, cardId, text) {
|
|
|
481
520
|
away. */
|
|
482
521
|
said(`run ${runId} had already ended, so its answer was not written to card ${cardId}`);
|
|
483
522
|
}
|
|
484
|
-
return;
|
|
523
|
+
return ownerSettlementAccepted(run, processToken);
|
|
485
524
|
}
|
|
486
525
|
out(`answered card ${cardId} run ${runId} ${text?.length ?? 0} characters`);
|
|
526
|
+
return true;
|
|
527
|
+
}
|
|
528
|
+
export function ownerSettlementAccepted(run, processToken) {
|
|
529
|
+
return (run.state === 'asked' || run.state === 'finished')
|
|
530
|
+
&& (processToken === undefined || run.processToken === processToken);
|
|
487
531
|
}
|
|
488
532
|
/** What the record says a run is now, and at what level. Read only to say the
|
|
489
533
|
* right sentence about something that has already happened; nothing branches on
|
|
490
534
|
* it that could have gone the other way. */
|
|
491
535
|
async function runNow(client, runId) {
|
|
492
|
-
const runs = await returned(client
|
|
493
|
-
|
|
536
|
+
const runs = await returned(client
|
|
537
|
+
.from('panel3_runs')
|
|
538
|
+
.select('state, level, process_token, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)')
|
|
539
|
+
.eq('id', runId), 'read', `the state of run ${runId}`);
|
|
540
|
+
const run = runs[0];
|
|
541
|
+
return run
|
|
542
|
+
? {
|
|
543
|
+
state: run.state,
|
|
544
|
+
level: run.level,
|
|
545
|
+
conversationOwnerId: run.card?.conversation_run_id ?? null,
|
|
546
|
+
processToken: run.process_token,
|
|
547
|
+
}
|
|
548
|
+
: { state: 'no longer on the record', level: null, conversationOwnerId: null, processToken: null };
|
|
494
549
|
}
|
|
495
550
|
/**
|
|
496
551
|
* A run whose process failed, ended with the reason ON ITS OWN COLUMN, and its
|
|
@@ -583,7 +638,29 @@ async function answerCard(client, tools, machineId, cardId, turns) {
|
|
|
583
638
|
attachments read carries the same rule: a failed read here must not read
|
|
584
639
|
as "nothing is attached", which is a different card than the one that was
|
|
585
640
|
actually sent. */
|
|
586
|
-
|
|
641
|
+
/* ═══ AND A FAILURE HERE ENDS THE RUN, RATHER THAN LEAVING IT RUNNING WITH NO
|
|
642
|
+
PROCESS. ═══ The take already wrote the run row in the statement that leased
|
|
643
|
+
the turns, so a throw between here and `startAgent` leaves a run reading
|
|
644
|
+
`running` with a null pid and nothing on stderr the person can see.
|
|
645
|
+
`recoverStranded` then reads that as a machine that went away, hands the
|
|
646
|
+
message back, and `panel3_take_turns` leases it to A BRAND NEW RUN whose
|
|
647
|
+
attempts start again at one — so a permanent failure, such as a read this
|
|
648
|
+
build cannot make against the current schema, repeats forever while the card
|
|
649
|
+
says Working and never says why. This is the sixth of `endRun`'s endings and
|
|
650
|
+
the last one that was missing: `resumeRun` and `startRearmed` already end
|
|
651
|
+
their two post-claim failures this way for exactly this reason.
|
|
652
|
+
THE REASON IS SHAREABLE. All three reads are `returned()` calls against the
|
|
653
|
+
database, whose messages name tables and columns and never a local path, so
|
|
654
|
+
constraint 6 is satisfied without a level fork here. */
|
|
655
|
+
let brief;
|
|
656
|
+
try {
|
|
657
|
+
brief = briefFor(turns[0].card_title, turns, await receiptsFor(client, cardId), await attachmentsFor(client, cardId), await codebasesForRun(client, runId));
|
|
658
|
+
}
|
|
659
|
+
catch (error) {
|
|
660
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
661
|
+
await endRun(client, LEVEL, runId, cardId, why);
|
|
662
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
663
|
+
}
|
|
587
664
|
/* ═══ THE RUN ID IS ON THE URL, AND THAT IS THE WHOLE OF WHAT THE AGENT IS
|
|
588
665
|
TOLD ABOUT ITS OWN STANDING. ═══ The tools server reads the level off the
|
|
589
666
|
run row this id names, so the daemon does not tell the child what it may do
|
|
@@ -603,7 +680,7 @@ async function answerCard(client, tools, machineId, cardId, turns) {
|
|
|
603
680
|
// agent is not killed over it.
|
|
604
681
|
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
605
682
|
}
|
|
606
|
-
return settle(client, tools, machineId, LEVEL, runId, cardId, started);
|
|
683
|
+
return settle(client, tools, machineId, LEVEL, runId, cardId, started, false);
|
|
607
684
|
}
|
|
608
685
|
/**
|
|
609
686
|
* WHICH PROJECT A RUN'S CARD IS FILED UNDER, or null when the card has none.
|
|
@@ -620,7 +697,7 @@ async function answerCard(client, tools, machineId, cardId, turns) {
|
|
|
620
697
|
* project's code.
|
|
621
698
|
*/
|
|
622
699
|
async function projectOfRun(client, runId) {
|
|
623
|
-
const rows = await returned(client.from('panel3_runs').select('card:panel3_cards(project_id)').eq('id', runId), 'read', 'which project this run belongs to');
|
|
700
|
+
const rows = await returned(client.from('panel3_runs').select('card:panel3_cards!panel3_runs_card_id_fkey(project_id)').eq('id', runId), 'read', 'which project this run belongs to');
|
|
624
701
|
return rows[0]?.card?.project_id ?? null;
|
|
625
702
|
}
|
|
626
703
|
async function codebasesForRun(client, runId) {
|
|
@@ -630,7 +707,7 @@ async function codebasesForRun(client, runId) {
|
|
|
630
707
|
async function codebaseOfRun(client, runId) {
|
|
631
708
|
const rows = await returned(client
|
|
632
709
|
.from('panel3_runs')
|
|
633
|
-
.select('codebase_id, card:panel3_cards(project_id)')
|
|
710
|
+
.select('codebase_id, card:panel3_cards!panel3_runs_card_id_fkey(project_id)')
|
|
634
711
|
.eq('id', runId), 'read', `which codebase run ${runId} belongs to`);
|
|
635
712
|
const row = rows[0];
|
|
636
713
|
if (!row?.codebase_id || !row.card?.project_id) {
|
|
@@ -643,6 +720,17 @@ async function codebaseOfRun(client, runId) {
|
|
|
643
720
|
}
|
|
644
721
|
return codebase;
|
|
645
722
|
}
|
|
723
|
+
/** One directory rule for every spawn whose level and ownership are known. */
|
|
724
|
+
async function workingDirectory(client, runId, level, isOwner, knownCodebase) {
|
|
725
|
+
if (level === 1)
|
|
726
|
+
return scratchDir();
|
|
727
|
+
if (knownCodebase === null) {
|
|
728
|
+
if (level === 2 && isOwner)
|
|
729
|
+
return scratchDir();
|
|
730
|
+
throw new Error('This code work does not name a registered project codebase.');
|
|
731
|
+
}
|
|
732
|
+
return checkoutForCodebase(knownCodebase ?? await codebaseOfRun(client, runId), hostname());
|
|
733
|
+
}
|
|
646
734
|
/**
|
|
647
735
|
* ═══ ONE AGENT, STARTED UNDER ANOTHER THAT IS STILL RUNNING. ═══
|
|
648
736
|
*
|
|
@@ -658,13 +746,14 @@ async function codebaseOfRun(client, runId) {
|
|
|
658
746
|
* Then the row, then the process, then the pid — constraint 8, in the only order
|
|
659
747
|
* that satisfies it.
|
|
660
748
|
*/
|
|
661
|
-
async function startChild(client, tools, machineId, parentRunId, brief, codebase) {
|
|
749
|
+
async function startChild(client, tools, machineId, parentRunId, brief, codebase, parentProcessToken) {
|
|
662
750
|
const { data, error } = await client.rpc('panel3_dispatch', {
|
|
663
751
|
p_parent_run_id: parentRunId,
|
|
664
752
|
p_brief: brief,
|
|
665
753
|
p_machine_id: machineId,
|
|
666
|
-
p_codebase_id: codebase
|
|
667
|
-
p_codebase_label: codebase
|
|
754
|
+
p_codebase_id: codebase?.id ?? null,
|
|
755
|
+
p_codebase_label: codebase?.name ?? null,
|
|
756
|
+
...(parentProcessToken === undefined ? {} : { p_process_token: parentProcessToken }),
|
|
668
757
|
});
|
|
669
758
|
if (error)
|
|
670
759
|
throw new Error(`could not start an agent under run ${parentRunId}: ${error.message}`);
|
|
@@ -673,8 +762,8 @@ async function startChild(client, tools, machineId, parentRunId, brief, codebase
|
|
|
673
762
|
/* NOTHING WAS WRITTEN AND NOTHING IS RUNNING, and the two reasons are said
|
|
674
763
|
together because the caller cannot tell them apart from here and both mean
|
|
675
764
|
the same thing to it. */
|
|
676
|
-
throw new Error(`NO AGENT WAS STARTED and nothing was written: run ${parentRunId} has ended,
|
|
677
|
-
+ '
|
|
765
|
+
throw new Error(`NO AGENT WAS STARTED and nothing was written: run ${parentRunId} has ended, is already as `
|
|
766
|
+
+ 'deep as anything may be sent from, or this conversation already has its owner. Exit now.');
|
|
678
767
|
}
|
|
679
768
|
if (row.run_level !== 2 && row.run_level !== 3) {
|
|
680
769
|
/* UNREACHABLE, AND STILL SETTLED. `panel3_dispatch` writes `parent.level + 1`
|
|
@@ -684,20 +773,26 @@ async function startChild(client, tools, machineId, parentRunId, brief, codebase
|
|
|
684
773
|
recovery sweep wait out the pid grace window to conclude what is already
|
|
685
774
|
known. */
|
|
686
775
|
const why = `run ${row.run_id} was written at level ${row.run_level}, which cannot be spawned`;
|
|
687
|
-
await giveUp(client, row.run_id, why);
|
|
776
|
+
await giveUp(client, row.run_id, why, row.process_token ?? undefined);
|
|
688
777
|
throw new Error(why);
|
|
689
778
|
}
|
|
690
779
|
const level = row.run_level;
|
|
691
780
|
let cwd;
|
|
781
|
+
let prompt;
|
|
692
782
|
try {
|
|
693
|
-
cwd =
|
|
783
|
+
cwd = await workingDirectory(client, row.run_id, level, level === 2, codebase);
|
|
784
|
+
prompt = level === 2
|
|
785
|
+
? await initialOwnerPrompt(client, row.run_card_id, parentRunId, brief)
|
|
786
|
+
: brief;
|
|
694
787
|
}
|
|
695
788
|
catch (error) {
|
|
696
789
|
const why = error instanceof Error ? error.message : String(error);
|
|
697
|
-
await giveUp(client, row.run_id, why);
|
|
790
|
+
await giveUp(client, row.run_id, why, row.process_token ?? undefined);
|
|
698
791
|
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
699
792
|
}
|
|
700
|
-
const
|
|
793
|
+
const processToken = row.process_token ?? undefined;
|
|
794
|
+
const isOwner = level === 2;
|
|
795
|
+
const started = startAgent(prompt, level, tools.urlFor(row.run_id, processToken), cwd, isOwner ? { ownerId: row.run_id } : undefined);
|
|
701
796
|
if (started.pid === null) {
|
|
702
797
|
/* THERE IS A ROW AND THERE IS NO PROCESS, which is the one shape the record
|
|
703
798
|
must never be left in quietly. The answer is already settled — nothing ran
|
|
@@ -705,12 +800,12 @@ async function startChild(client, tools, machineId, parentRunId, brief, codebase
|
|
|
705
800
|
and the tool call fails saying no agent was started. */
|
|
706
801
|
const answer = await started.answered;
|
|
707
802
|
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
708
|
-
await giveUp(client, row.run_id, reason);
|
|
803
|
+
await giveUp(client, row.run_id, reason, processToken);
|
|
709
804
|
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
710
805
|
}
|
|
711
806
|
out(`dispatch run ${row.run_id} level ${level} under ${parentRunId} pid ${started.pid}`);
|
|
712
807
|
try {
|
|
713
|
-
await recordProcess(client, row.run_id, started.pid);
|
|
808
|
+
await recordProcess(client, row.run_id, started.pid, undefined, processToken);
|
|
714
809
|
}
|
|
715
810
|
catch (error) {
|
|
716
811
|
// Said, not fatal, exactly as at level 1: the agent is running and killing
|
|
@@ -719,7 +814,9 @@ async function startChild(client, tools, machineId, parentRunId, brief, codebase
|
|
|
719
814
|
}
|
|
720
815
|
return {
|
|
721
816
|
runId: row.run_id,
|
|
722
|
-
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started
|
|
817
|
+
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, true, processToken, isOwner && processToken
|
|
818
|
+
? ownerSessionLifecycle(started, row.run_id, harness(), processToken)
|
|
819
|
+
: undefined),
|
|
723
820
|
};
|
|
724
821
|
}
|
|
725
822
|
/**
|
|
@@ -763,11 +860,13 @@ async function attemptsSoFar(client, runId) {
|
|
|
763
860
|
* ends through `panel3_end_run`, which hands no turns back, and the card says
|
|
764
861
|
* failed.
|
|
765
862
|
*
|
|
766
|
-
* ═══ AND THE FORK LIVES HERE AND NOWHERE ELSE. ═══
|
|
767
|
-
* way: the settle,
|
|
768
|
-
* and `
|
|
769
|
-
* drift, and it drifted the first time it could:
|
|
770
|
-
* same pair of endings and only one pair was looked
|
|
863
|
+
* ═══ AND THE FORK LIVES HERE AND NOWHERE ELSE. ═══ SIX places end a run this
|
|
864
|
+
* way: the settle, the two failures after the claim in each of `resumeRun` and
|
|
865
|
+
* `startRearmed`, and the pre-spawn failure in `answerCard`. A copy of this rule
|
|
866
|
+
* in each is six places for it to drift, and it drifted the first time it could:
|
|
867
|
+
* those two functions hold the same pair of endings and only one pair was looked
|
|
868
|
+
* at. `answerCard`'s was missing outright, which is how a build that could not
|
|
869
|
+
* read a card's project turned every take into a silent take/recover loop.
|
|
771
870
|
*
|
|
772
871
|
* ALL FIVE CAN SEE LEVEL 1. `resumeRun` serving the retry is what opened its
|
|
773
872
|
* two. `startRearmed`'s were always reachable, because a re-arm has always
|
|
@@ -834,7 +933,44 @@ async function endRun(client, level, runId, cardId, why) {
|
|
|
834
933
|
* cannot end the same way. What `answerCard` always did is what level 1 still
|
|
835
934
|
* does, and the only thing that has moved is where it is written.
|
|
836
935
|
*/
|
|
837
|
-
function
|
|
936
|
+
function ownerSessionLifecycle(started, ownerId, ownerHarness, processToken, expectedSessionId) {
|
|
937
|
+
const pending = started.session.then((nativeSessionId) => {
|
|
938
|
+
if (nativeSessionId === null)
|
|
939
|
+
return false;
|
|
940
|
+
if (expectedSessionId && nativeSessionId !== expectedSessionId) {
|
|
941
|
+
said(`the resumed harness returned a different conversation for owner ${ownerId}; it will not be reused`);
|
|
942
|
+
return false;
|
|
943
|
+
}
|
|
944
|
+
try {
|
|
945
|
+
writeOwnerSession({
|
|
946
|
+
ownerId,
|
|
947
|
+
harness: ownerHarness,
|
|
948
|
+
nativeSessionId,
|
|
949
|
+
processToken,
|
|
950
|
+
state: 'pending',
|
|
951
|
+
});
|
|
952
|
+
return true;
|
|
953
|
+
}
|
|
954
|
+
catch (error) {
|
|
955
|
+
said(`could not save the local conversation for owner ${ownerId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
956
|
+
return false;
|
|
957
|
+
}
|
|
958
|
+
});
|
|
959
|
+
// Start observing immediately; Codex reports thread.started before its answer.
|
|
960
|
+
void pending;
|
|
961
|
+
return {
|
|
962
|
+
established: async () => {
|
|
963
|
+
if (await pending)
|
|
964
|
+
establishOwnerSession(ownerId, processToken);
|
|
965
|
+
},
|
|
966
|
+
failed: async () => {
|
|
967
|
+
// A failed or rejected generation stays pending and therefore cannot be
|
|
968
|
+
// resumed. Reconciliation is the sole owner of stable-state deletion.
|
|
969
|
+
await pending;
|
|
970
|
+
},
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
function settle(client, tools, machineId, level, runId, cardId, started, speaksToTheCard = true, processToken, ownerSession) {
|
|
838
974
|
return started.answered.then(async (answer) => {
|
|
839
975
|
/* ═══ A RUN THAT STOPPED TO ASK DID NOT DIE, WHATEVER THE HARNESS PRINTED
|
|
840
976
|
ON ITS WAY OUT. ═══
|
|
@@ -860,11 +996,27 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
860
996
|
into an ordinary ending, and then handed to `writeAnswer` — which already
|
|
861
997
|
owns this case and already prints its sentence — rather than to a second
|
|
862
998
|
ending written here beside it. */
|
|
863
|
-
if (!answer.ok
|
|
864
|
-
await
|
|
865
|
-
|
|
999
|
+
if (!answer.ok) {
|
|
1000
|
+
const run = await runNow(client, runId);
|
|
1001
|
+
if (run.state === 'asked') {
|
|
1002
|
+
const accepted = await writeAnswer(client, runId, cardId, null, processToken);
|
|
1003
|
+
if (accepted)
|
|
1004
|
+
await ownerSession?.established();
|
|
1005
|
+
else
|
|
1006
|
+
await ownerSession?.failed();
|
|
1007
|
+
return;
|
|
1008
|
+
}
|
|
1009
|
+
/* Level 1's only successful result is the owner pointer written by
|
|
1010
|
+
dispatch. Its stdout is deliberately hidden, and Codex may therefore
|
|
1011
|
+
exit cleanly without an agent message after the tool succeeds. The
|
|
1012
|
+
record, not prose the launcher was told not to write, decides success. */
|
|
1013
|
+
if (level === 1 && run.conversationOwnerId !== null) {
|
|
1014
|
+
await writeAnswer(client, runId, cardId, null, processToken);
|
|
1015
|
+
return;
|
|
1016
|
+
}
|
|
866
1017
|
}
|
|
867
1018
|
if (!answer.ok) {
|
|
1019
|
+
await ownerSession?.failed();
|
|
868
1020
|
/* ═══ A PROCESS THAT STARTED AND THEN DIED BADLY IS STARTED AGAIN. ═══
|
|
869
1021
|
recovery-1/ux.md, Slice 3: "the agent's harness fails on something that
|
|
870
1022
|
is nobody's fault: a rate limit, a dropped connection, a provider
|
|
@@ -904,7 +1056,9 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
904
1056
|
const attempts = await attemptsSoFar(client, runId);
|
|
905
1057
|
if (started.pid !== null && attempts < MAX_ATTEMPTS) {
|
|
906
1058
|
out(`retry run ${runId} attempt ${attempts} of ${MAX_ATTEMPTS} died: ${answer.reason}`);
|
|
907
|
-
const again =
|
|
1059
|
+
const again = processToken === undefined
|
|
1060
|
+
? await resumeRun(client, tools, machineId, runId, started.pid)
|
|
1061
|
+
: await activateOwner(client, tools, machineId, runId, processToken, started.pid);
|
|
908
1062
|
if (again)
|
|
909
1063
|
return again.settled;
|
|
910
1064
|
/* ═══ THE CLAIM MATCHED NOTHING, AND THAT IS NOT ALWAYS SOMEBODY ELSE
|
|
@@ -937,7 +1091,9 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
937
1091
|
both already read, so the count reaches a person from one place rather
|
|
938
1092
|
than two. */
|
|
939
1093
|
const why = attempts > 1 ? `${answer.reason} (after ${attempts} attempts)` : answer.reason;
|
|
940
|
-
const ended =
|
|
1094
|
+
const ended = processToken === undefined
|
|
1095
|
+
? await endRun(client, level, runId, cardId, why)
|
|
1096
|
+
: (await giveUp(client, runId, why, processToken)) !== null;
|
|
941
1097
|
if (!ended) {
|
|
942
1098
|
/* IT WAS ALREADY SETTLED, by recovery, which decided this process was
|
|
943
1099
|
gone before it said so itself, or by the person's Stop. Nothing was
|
|
@@ -954,7 +1110,11 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
954
1110
|
/* ═══ ITS OWN WORDS, ONTO THE CARD, THROUGH THE SAME STATEMENT EVERY LEVEL
|
|
955
1111
|
USES. ═══ Nothing reads them, nothing shortens them and nothing waits to
|
|
956
1112
|
approve them: ux.md's "whoever did the work writes the answer". */
|
|
957
|
-
await writeAnswer(client, runId, cardId, speaksToTheCard ? answer.text : null);
|
|
1113
|
+
const accepted = await writeAnswer(client, runId, cardId, speaksToTheCard ? answer.text : null, processToken);
|
|
1114
|
+
if (accepted)
|
|
1115
|
+
await ownerSession?.established();
|
|
1116
|
+
else
|
|
1117
|
+
await ownerSession?.failed();
|
|
958
1118
|
});
|
|
959
1119
|
}
|
|
960
1120
|
/**
|
|
@@ -1052,9 +1212,7 @@ async function resumeRun(client, tools, machineId, runId, afterPid) {
|
|
|
1052
1212
|
}
|
|
1053
1213
|
else {
|
|
1054
1214
|
try {
|
|
1055
|
-
cwd = level
|
|
1056
|
-
? scratchDir()
|
|
1057
|
-
: checkoutForCodebase(await codebaseOfRun(client, runId), hostname());
|
|
1215
|
+
cwd = await workingDirectory(client, runId, level, false);
|
|
1058
1216
|
}
|
|
1059
1217
|
catch (error) {
|
|
1060
1218
|
/* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
|
|
@@ -1155,9 +1313,7 @@ async function startRearmed(client, tools, machineId, row) {
|
|
|
1155
1313
|
}
|
|
1156
1314
|
let cwd;
|
|
1157
1315
|
try {
|
|
1158
|
-
cwd = level
|
|
1159
|
-
? scratchDir()
|
|
1160
|
-
: checkoutForCodebase(await codebaseOfRun(client, row.run_id), hostname());
|
|
1316
|
+
cwd = await workingDirectory(client, row.run_id, level, false);
|
|
1161
1317
|
}
|
|
1162
1318
|
catch (error) {
|
|
1163
1319
|
/* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
|
|
@@ -1180,6 +1336,9 @@ async function startRearmed(client, tools, machineId, row) {
|
|
|
1180
1336
|
POSSIBLE. Level 3 has no `dispatch`, so it has no children to have: null
|
|
1181
1337
|
says that, where an empty list would say it chose to send nobody. */
|
|
1182
1338
|
const children = level === 3 ? null : await childrenOf(client, row.run_id);
|
|
1339
|
+
const deliveredArtifact = row.ask_id !== null && row.mine
|
|
1340
|
+
? await rearmedArtifactAnswer(client, row.ask_id)
|
|
1341
|
+
: null;
|
|
1183
1342
|
/* ═══ THREE REASONS, AND THE ROW SAYS WHICH. ═══ No question is ux.md's third
|
|
1184
1343
|
re-arm: everybody it sent has finished, and it is started to read them back.
|
|
1185
1344
|
`children` cannot be null on that path — only a run with children is ever
|
|
@@ -1188,7 +1347,7 @@ async function startRearmed(client, tools, machineId, row) {
|
|
|
1188
1347
|
const prompt = row.ask_id === null
|
|
1189
1348
|
? readBackPrompt(row.run_brief, row.run_report, children ?? [])
|
|
1190
1349
|
: row.mine
|
|
1191
|
-
? answerPrompt(row.run_brief, row.run_report, children, row.question ?? '', row.answer ?? '')
|
|
1350
|
+
? answerPrompt(row.run_brief, row.run_report, children, row.question ?? '', row.answer ?? '', deliveredArtifact)
|
|
1192
1351
|
: escalationPrompt(row.run_brief, row.run_report, children, row.ask_id, row.question ?? '');
|
|
1193
1352
|
const started = startAgent(prompt, level, tools.urlFor(row.run_id), cwd);
|
|
1194
1353
|
if (started.pid === null) {
|
|
@@ -1263,6 +1422,312 @@ async function childrenOf(client, runId) {
|
|
|
1263
1422
|
: c.report.split('\n').map((l) => ` ${l}`)),
|
|
1264
1423
|
]);
|
|
1265
1424
|
}
|
|
1425
|
+
/** Private owner context: enough to continue or stop work, without hierarchy or activity text. */
|
|
1426
|
+
async function ownerChildren(client, runId) {
|
|
1427
|
+
const children = await returned(client.from('panel3_runs').select('id, state, report, failed_because')
|
|
1428
|
+
.eq('parent_run_id', runId).order('started_at'), 'read', `what owner ${runId} sent others to do`);
|
|
1429
|
+
return children.map((child) => [
|
|
1430
|
+
`${child.id} ${child.state}`,
|
|
1431
|
+
...(child.failed_because ? [`ended: ${child.failed_because}`] : []),
|
|
1432
|
+
child.report ?? 'no report',
|
|
1433
|
+
].join('\n'));
|
|
1434
|
+
}
|
|
1435
|
+
async function ownerConversation(client, cardId, leased, initialLeaseRunId) {
|
|
1436
|
+
const [turns, rawAsks] = await Promise.all([
|
|
1437
|
+
returned(client.from('panel3_turns').select('id, role, body, created_at, lease_id')
|
|
1438
|
+
.eq('card_id', cardId).order('created_at'), 'read', `the visible conversation on card ${cardId}`),
|
|
1439
|
+
returned(client.from('panel3_asks').select(`id, created_at, ${ASK_CONTENT_COLUMNS}`)
|
|
1440
|
+
.eq('card_id', cardId).not('decision_id', 'is', null).order('created_at'), 'read', `the visible questions on card ${cardId}`),
|
|
1441
|
+
]);
|
|
1442
|
+
const asks = await withAskContent(client, rawAsks);
|
|
1443
|
+
const events = [
|
|
1444
|
+
...turns.map((turn) => ({
|
|
1445
|
+
at: turn.created_at,
|
|
1446
|
+
kind: turn.role,
|
|
1447
|
+
id: turn.id,
|
|
1448
|
+
body: turn.body,
|
|
1449
|
+
needsReply: turn.role === 'user'
|
|
1450
|
+
&& (leased.has(turn.id) || turn.lease_id === initialLeaseRunId),
|
|
1451
|
+
})),
|
|
1452
|
+
...asks.map((ask) => ({
|
|
1453
|
+
at: ask.created_at,
|
|
1454
|
+
kind: 'question',
|
|
1455
|
+
id: ask.id,
|
|
1456
|
+
body: ask.question ?? '(question unavailable)',
|
|
1457
|
+
answer: ask.answer,
|
|
1458
|
+
relatedArtifactId: ask.related_artifact_id,
|
|
1459
|
+
relatedArtifactRevision: ask.related_artifact_revision,
|
|
1460
|
+
artifactAnswer: artifactAnswer(ask.related_artifact_id, ask.related_artifact_revision, ask.selected_options)?.answer ?? null,
|
|
1461
|
+
})),
|
|
1462
|
+
];
|
|
1463
|
+
return events.sort((a, b) => a.at.localeCompare(b.at) || a.id.localeCompare(b.id));
|
|
1464
|
+
}
|
|
1465
|
+
/** The first owner gets the same durable context as every later activation. */
|
|
1466
|
+
export async function initialOwnerPrompt(client, cardId, launcherRunId, brief) {
|
|
1467
|
+
const events = await ownerConversation(client, cardId, new Set(), launcherRunId);
|
|
1468
|
+
return ownerActivationPrompt(brief, null, events, [], null);
|
|
1469
|
+
}
|
|
1470
|
+
/** Only facts delivered by this activation enter an already-resumed native
|
|
1471
|
+
* conversation. Older visible turns are already in that conversation. */
|
|
1472
|
+
export function ownerContinuationPrompt(brief, report, events, children, delivered) {
|
|
1473
|
+
const messages = events.filter((event) => event.kind === 'user' && event.needsReply);
|
|
1474
|
+
return [
|
|
1475
|
+
'CONTINUE THE CTRL+SPC CONVERSATION YOU ALREADY OWN.',
|
|
1476
|
+
'Use only the new facts below as the new turn. Keep coordinating the whole card and speak as CTRL+SPC.',
|
|
1477
|
+
'',
|
|
1478
|
+
'NEW USER MESSAGES',
|
|
1479
|
+
...(messages.length === 0
|
|
1480
|
+
? ['None.']
|
|
1481
|
+
: messages.flatMap((event) => [`Turn ${event.id}`, event.body])),
|
|
1482
|
+
...(delivered === null ? [] : delivered.mine ? [
|
|
1483
|
+
'',
|
|
1484
|
+
'ANSWER TO YOUR QUESTION',
|
|
1485
|
+
`Question ${delivered.id}: ${delivered.question}`,
|
|
1486
|
+
`Answer: ${delivered.answer ?? '(no answer text)'}`,
|
|
1487
|
+
...presentedArtifactAnswerContext(delivered.artifactAnswer),
|
|
1488
|
+
] : [
|
|
1489
|
+
'',
|
|
1490
|
+
'A WORKER NEEDS YOUR DECISION',
|
|
1491
|
+
`Question ${delivered.id}: ${delivered.question}`,
|
|
1492
|
+
'Answer it with answer_escalation if you can. If the person must decide, ask them directly.',
|
|
1493
|
+
]),
|
|
1494
|
+
'',
|
|
1495
|
+
'CURRENT PRIVATE WORKER SNAPSHOT — THIS REPLACES THE PREVIOUS SNAPSHOT',
|
|
1496
|
+
...(children.length === 0 ? ['You have sent nobody.'] : children),
|
|
1497
|
+
'',
|
|
1498
|
+
'YOUR CURRENT DURABLE REPORT',
|
|
1499
|
+
report ?? 'Nothing yet.',
|
|
1500
|
+
'',
|
|
1501
|
+
'YOUR IMMUTABLE RESPONSIBILITY',
|
|
1502
|
+
brief,
|
|
1503
|
+
].join('\n');
|
|
1504
|
+
}
|
|
1505
|
+
async function ownerCandidate(client, runId) {
|
|
1506
|
+
const candidates = await returned(client.from('panel3_runs')
|
|
1507
|
+
.select('id, card_id, codebase_id, machine_id, harness, state, ended_at, attempts, pid, started_at, resumed_at, process_token, handed_back_at, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id, state)')
|
|
1508
|
+
.eq('id', runId), 'read', `conversation owner ${runId}`);
|
|
1509
|
+
const candidate = candidates[0];
|
|
1510
|
+
return candidate?.card?.conversation_run_id === candidate?.id ? candidate : null;
|
|
1511
|
+
}
|
|
1512
|
+
async function ownerDirectory(client, candidate) {
|
|
1513
|
+
return workingDirectory(client, candidate.id, 2, true, candidate.codebase_id === null ? null : undefined);
|
|
1514
|
+
}
|
|
1515
|
+
export function resumableOwnerSessionId(candidate, machineId, machineHarness) {
|
|
1516
|
+
const local = readOwnerSession(candidate.id);
|
|
1517
|
+
return candidate.machine_id === machineId
|
|
1518
|
+
&& candidate.harness === machineHarness
|
|
1519
|
+
&& local?.state === 'established'
|
|
1520
|
+
&& local.harness === machineHarness
|
|
1521
|
+
&& local.processToken === candidate.process_token
|
|
1522
|
+
? local.nativeSessionId
|
|
1523
|
+
: undefined;
|
|
1524
|
+
}
|
|
1525
|
+
async function activateOwner(client, tools, machineId, runId, afterProcessToken = null, afterPid = null) {
|
|
1526
|
+
const candidate = await ownerCandidate(client, runId);
|
|
1527
|
+
if (!candidate)
|
|
1528
|
+
return null;
|
|
1529
|
+
const machineHarness = harness();
|
|
1530
|
+
const resumeSessionId = resumableOwnerSessionId(candidate, machineId, machineHarness);
|
|
1531
|
+
const cwd = await ownerDirectory(client, candidate);
|
|
1532
|
+
const { data, error } = await client.rpc('panel3_take_owner_activation', {
|
|
1533
|
+
p_run_id: runId,
|
|
1534
|
+
p_machine_id: machineId,
|
|
1535
|
+
p_agent: machineHarness,
|
|
1536
|
+
p_after_process_token: afterProcessToken,
|
|
1537
|
+
p_after_pid: afterPid,
|
|
1538
|
+
});
|
|
1539
|
+
if (error)
|
|
1540
|
+
throw new Error(`could not activate conversation owner ${runId}: ${error.message}`);
|
|
1541
|
+
const claimed = data?.[0];
|
|
1542
|
+
if (!claimed)
|
|
1543
|
+
return null;
|
|
1544
|
+
const [events, children] = await Promise.all([
|
|
1545
|
+
ownerConversation(client, claimed.run_card_id, new Set(claimed.turn_ids ?? [])),
|
|
1546
|
+
ownerChildren(client, runId),
|
|
1547
|
+
]);
|
|
1548
|
+
const currentArtifactAnswer = deliveredArtifactAnswer(events, claimed);
|
|
1549
|
+
const delivered = claimed.ask_id === null ? null : {
|
|
1550
|
+
id: claimed.ask_id,
|
|
1551
|
+
question: claimed.question ?? '(question unavailable)',
|
|
1552
|
+
answer: claimed.answer,
|
|
1553
|
+
mine: claimed.mine === true,
|
|
1554
|
+
artifactAnswer: currentArtifactAnswer,
|
|
1555
|
+
};
|
|
1556
|
+
const prompt = resumeSessionId
|
|
1557
|
+
? ownerContinuationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered)
|
|
1558
|
+
: ownerActivationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered !== null && !delivered.mine
|
|
1559
|
+
? { id: delivered.id, question: delivered.question }
|
|
1560
|
+
: null, currentArtifactAnswer,
|
|
1561
|
+
/* ═══ A PROCESS OF ITS OWN ENDED BEFORE IT FINISHED. ═══ `afterPid` is the
|
|
1562
|
+
fact, and it is non-null on all three paths that follow one: a harness
|
|
1563
|
+
that crashed, a machine that went down, and now a person's correction.
|
|
1564
|
+
The sentences it adds say what to do and never why, because those three
|
|
1565
|
+
are not the same event and `prompt.ts` exists to stop an agent being
|
|
1566
|
+
told an untrue reason for its own restart. */
|
|
1567
|
+
afterPid !== null);
|
|
1568
|
+
const started = startAgent(prompt, 2, tools.urlFor(runId, claimed.process_token), cwd, { ownerId: runId, ...(resumeSessionId ? { resumeSessionId } : {}) });
|
|
1569
|
+
if (started.pid === null) {
|
|
1570
|
+
const answer = await started.answered;
|
|
1571
|
+
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
1572
|
+
await giveUp(client, runId, reason, claimed.process_token);
|
|
1573
|
+
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
1574
|
+
}
|
|
1575
|
+
try {
|
|
1576
|
+
await recordProcess(client, runId, started.pid, undefined, claimed.process_token);
|
|
1577
|
+
}
|
|
1578
|
+
catch (error) {
|
|
1579
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1580
|
+
}
|
|
1581
|
+
return {
|
|
1582
|
+
settled: settle(client, tools, machineId, 2, runId, claimed.run_card_id, started, true, claimed.process_token, ownerSessionLifecycle(started, runId, machineHarness, claimed.process_token, resumeSessionId)),
|
|
1583
|
+
};
|
|
1584
|
+
}
|
|
1585
|
+
/** The only owner rows the normal poll may try, including an explicit hand-back move. */
|
|
1586
|
+
export function ownerPollClaim(candidate) {
|
|
1587
|
+
if (candidate.state === 'running') {
|
|
1588
|
+
if (candidate.handed_back_at === null || candidate.process_token === null)
|
|
1589
|
+
return null;
|
|
1590
|
+
return { afterProcessToken: candidate.process_token, afterPid: null };
|
|
1591
|
+
}
|
|
1592
|
+
if (candidate.state === 'finished' || candidate.state === 'asked') {
|
|
1593
|
+
return { afterProcessToken: null, afterPid: null };
|
|
1594
|
+
}
|
|
1595
|
+
return null;
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* ═══ THE PROCESS A PERSON'S CORRECTION CANNOT HAVE REACHED. ═══
|
|
1599
|
+
*
|
|
1600
|
+
* redirect-5/ux.md: stop ends work, and changing its direction mid-flight did
|
|
1601
|
+
* not exist. ux.md's mechanism is "redirect is stop plus respawn", and the
|
|
1602
|
+
* respawn half is already built: a process that dies is started again as itself
|
|
1603
|
+
* by `settle`, and the claim it goes through leases every unaddressed person
|
|
1604
|
+
* turn on its way. **So the only thing missing was the ending**, and this is it.
|
|
1605
|
+
*
|
|
1606
|
+
* ═══ IT IS A MIRROR OF THE RETRY CLAIM, AND MUST BE CHANGED WITH IT. ═══
|
|
1607
|
+
* `20260914090000`'s retry branch requires the token, `running`, `ended_at` null,
|
|
1608
|
+
* the pid, and `attempts < 3`. The kill is the ONE irreversible act in this file:
|
|
1609
|
+
* every other predicate here pre-filters an RPC that can refuse harmlessly, and
|
|
1610
|
+
* a claim refused AFTER a kill falls through `settle` to `giveUp`, which turns a
|
|
1611
|
+
* person's correction into a FAILED CARD. So every condition below is either
|
|
1612
|
+
* that mirror, or a product rule with a named cost.
|
|
1613
|
+
*
|
|
1614
|
+
* Two things are NOT here, because neither is a property of the row: whether
|
|
1615
|
+
* this run owns its card, and whether this daemon holds its process. Both are
|
|
1616
|
+
* the caller's, and `mine` is what makes the respawn possible at all.
|
|
1617
|
+
*/
|
|
1618
|
+
export function redirectedProcess(candidate, machineId,
|
|
1619
|
+
/** The newest unaddressed person turn per card. MAX, never first-seen: a turn
|
|
1620
|
+
* left unaddressed while a question was open would otherwise pin its card
|
|
1621
|
+
* below `resumed_at` for good and nothing on it could ever redirect. */
|
|
1622
|
+
waiting, cardId, booted) {
|
|
1623
|
+
// The claim's own five, mirrored.
|
|
1624
|
+
if (candidate.state !== 'running' || candidate.ended_at !== null)
|
|
1625
|
+
return null;
|
|
1626
|
+
if (candidate.attempts >= MAX_ATTEMPTS)
|
|
1627
|
+
return null;
|
|
1628
|
+
// The person moved this card to another machine. Killing here makes `settle`
|
|
1629
|
+
// retry locally and the claim clears `handed_back_at`, silently reversing them.
|
|
1630
|
+
if (candidate.handed_back_at !== null)
|
|
1631
|
+
return null;
|
|
1632
|
+
// `mine` is not enough: a takeover moves the row while this daemon still holds
|
|
1633
|
+
// a promise for it, and the claim then refuses on the machine predicate.
|
|
1634
|
+
if (candidate.machine_id !== machineId)
|
|
1635
|
+
return null;
|
|
1636
|
+
/* ═══ A CARD THAT NEEDS THE PERSON IS NOT A CARD THEY ARE REDIRECTING. ═══
|
|
1637
|
+
The claim refuses to lease turns while a person's question is open, so a
|
|
1638
|
+
kill here would destroy the work and NOT deliver the correction. The card's
|
|
1639
|
+
state is written from a character-identical predicate by the
|
|
1640
|
+
`panel3_prepare_user_turn` trigger on every person message, so it is the
|
|
1641
|
+
same fact, one read earlier. */
|
|
1642
|
+
if (candidate.card_state !== 'working')
|
|
1643
|
+
return null;
|
|
1644
|
+
/* ═══ AND IT HAS TO BE A MESSAGE THIS PROCESS CANNOT HAVE SEEN. ═══ The rule
|
|
1645
|
+
`panel3_take_turns` already applies one level up. Measured against the
|
|
1646
|
+
attempt that is running NOW, which is also what stops a replacement being
|
|
1647
|
+
killed by the very message that caused it. */
|
|
1648
|
+
const said = waiting.get(cardId);
|
|
1649
|
+
if (said === undefined)
|
|
1650
|
+
return null;
|
|
1651
|
+
if (new Date(said).getTime() <= new Date(candidate.resumed_at ?? candidate.started_at).getTime()) {
|
|
1652
|
+
return null;
|
|
1653
|
+
}
|
|
1654
|
+
return runProcessIsAlive(candidate, booted) ? candidate.pid : null;
|
|
1655
|
+
}
|
|
1656
|
+
export function pendingOwnerSessionWithinGrace(candidate, now = Date.now()) {
|
|
1657
|
+
const local = readOwnerSession(candidate.id);
|
|
1658
|
+
return !!local
|
|
1659
|
+
&& local.state === 'pending'
|
|
1660
|
+
&& local.harness === candidate.harness
|
|
1661
|
+
&& local.processToken === candidate.process_token
|
|
1662
|
+
&& now - new Date(local.updatedAt).getTime() < OWNER_SESSION_GRACE_MS;
|
|
1663
|
+
}
|
|
1664
|
+
async function takeOwnerActivations(client, tools, machineId, mine, hold) {
|
|
1665
|
+
/* ═══ THE MESSAGES NOBODY HAS TAKEN, READ BEFORE THE RUNS, DELIBERATELY. ═══
|
|
1666
|
+
A claim landing between these two reads stamps a `resumed_at` NEWER than
|
|
1667
|
+
every turn it just leased, so the redirect below refuses and nothing is
|
|
1668
|
+
killed. In the other order the same claim leaves a stale `resumed_at` beside
|
|
1669
|
+
a turn that now reads unaddressed, and the daemon kills the replacement it
|
|
1670
|
+
has just started. One narrow indexed read (`panel3_turns_takeable_idx`),
|
|
1671
|
+
unconditional because that ordering is the whole point of it. */
|
|
1672
|
+
const outstanding = await returned(client.from('panel3_turns')
|
|
1673
|
+
.select('card_id, created_at')
|
|
1674
|
+
.eq('role', 'user')
|
|
1675
|
+
.is('addressed_at', null), 'read', 'the messages nobody has taken yet');
|
|
1676
|
+
const waiting = new Map();
|
|
1677
|
+
for (const turn of outstanding) {
|
|
1678
|
+
// THE NEWEST PER CARD. See `redirectedProcess`: first-wins would pin a card
|
|
1679
|
+
// below its own `resumed_at` for good.
|
|
1680
|
+
const held = waiting.get(turn.card_id);
|
|
1681
|
+
if (held === undefined || turn.created_at > held)
|
|
1682
|
+
waiting.set(turn.card_id, turn.created_at);
|
|
1683
|
+
}
|
|
1684
|
+
const candidates = await returned(client.from('panel3_runs')
|
|
1685
|
+
.select('id, card_id, codebase_id, machine_id, harness, state, ended_at, attempts, pid, started_at, resumed_at, process_token, handed_back_at, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id, state)')
|
|
1686
|
+
.in('state', ['finished', 'asked', 'running']), 'read', 'conversation owners that may be ready');
|
|
1687
|
+
for (const candidate of candidates) {
|
|
1688
|
+
if (candidate.card?.conversation_run_id !== candidate.id)
|
|
1689
|
+
continue;
|
|
1690
|
+
/* ═══ A RUN THIS DAEMON IS HOLDING IS THE ONLY ONE A CORRECTION CAN
|
|
1691
|
+
REDIRECT. ═══ Not because of the claim, which is not made here at all, but
|
|
1692
|
+
because of what happens AFTER the kill: the respawn is the dying process's
|
|
1693
|
+
own `settle`, and only the daemon that started it has one. Kill anything
|
|
1694
|
+
else and there is a dead process, a row still reading `running` and a card
|
|
1695
|
+
still saying `working`, with nothing local to retry it. */
|
|
1696
|
+
if (mine.has(candidate.id)) {
|
|
1697
|
+
const pid = redirectedProcess({ ...candidate, card_state: candidate.card?.state ?? null }, machineId, waiting, candidate.card_id, bootedAt());
|
|
1698
|
+
if (pid !== null) {
|
|
1699
|
+
killTree({ pid, kill: (signal) => process.kill(pid, signal) });
|
|
1700
|
+
/* ═══ SIGNALLED, NOT KILLED, AND THE WORD IS THE POINT. ═══ `killTree`
|
|
1701
|
+
swallows a refused signal on both platforms, so saying "killed" would
|
|
1702
|
+
claim a death this daemon never observed. A process that survives is
|
|
1703
|
+
still running, still held, and its turn is still unaddressed and still
|
|
1704
|
+
newer than `resumed_at`, so the next poll signals it again — the same
|
|
1705
|
+
rule `killStopped` states two hundred lines below. */
|
|
1706
|
+
out(`redirect card ${candidate.card_id} run ${candidate.id} signalled pid ${pid}`);
|
|
1707
|
+
}
|
|
1708
|
+
continue;
|
|
1709
|
+
}
|
|
1710
|
+
if (pendingOwnerSessionWithinGrace(candidate))
|
|
1711
|
+
continue;
|
|
1712
|
+
if (candidate.handed_back_at !== null
|
|
1713
|
+
&& candidate.machine_id === machineId
|
|
1714
|
+
&& (runProcessIsAlive(candidate, bootedAt())
|
|
1715
|
+
|| (candidate.pid === null
|
|
1716
|
+
&& Date.now() - new Date(candidate.resumed_at ?? candidate.started_at).getTime() < PID_GRACE_MS)))
|
|
1717
|
+
continue;
|
|
1718
|
+
const claim = ownerPollClaim(candidate);
|
|
1719
|
+
if (!claim)
|
|
1720
|
+
continue;
|
|
1721
|
+
try {
|
|
1722
|
+
const activated = await activateOwner(client, tools, machineId, candidate.id, claim.afterProcessToken, claim.afterPid);
|
|
1723
|
+
if (activated)
|
|
1724
|
+
hold(candidate.id, activated.settled);
|
|
1725
|
+
}
|
|
1726
|
+
catch (error) {
|
|
1727
|
+
said(`could not activate owner ${candidate.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1266
1731
|
// ---------------------------------------------------------------------------
|
|
1267
1732
|
/**
|
|
1268
1733
|
* ═══ THE TWO ENDS THE PERSON CAUSES, WHICH ARE THE TWO THIS DAEMON KILLS. ═══
|
|
@@ -1300,6 +1765,46 @@ function runProcessIsAlive(run, booted) {
|
|
|
1300
1765
|
return false;
|
|
1301
1766
|
return processIsAlive(run.pid);
|
|
1302
1767
|
}
|
|
1768
|
+
/** Reconcile only CTRL+SPC-owned local session state. Native Claude and Windows
|
|
1769
|
+
* Codex transcripts live in harness-owned locations and are deliberately not
|
|
1770
|
+
* touched here. A current owner row, live PID, in-flight process, or fresh
|
|
1771
|
+
* pid-null claim always defers cleanup. Stable state is removed only after the
|
|
1772
|
+
* owner row disappears, becomes terminal, or no longer owns its card. */
|
|
1773
|
+
export async function reconcileOwnerSessions(client, machineId, _machineHarness, inFlightOwnerIds = new Set()) {
|
|
1774
|
+
const mappingIds = listOwnerSessionIds();
|
|
1775
|
+
const homeIds = listPanel3CodexOwnerHomeIds();
|
|
1776
|
+
const all = [...new Set([...mappingIds, ...homeIds])];
|
|
1777
|
+
const invalid = all.filter((id) => !validSessionUuid(id));
|
|
1778
|
+
for (const id of invalid)
|
|
1779
|
+
removePanel3CodexOwnerHome(id);
|
|
1780
|
+
const ids = all.filter(validSessionUuid);
|
|
1781
|
+
if (ids.length === 0)
|
|
1782
|
+
return;
|
|
1783
|
+
const rows = await returned(client.from('panel3_runs')
|
|
1784
|
+
.select('id, machine_id, harness, state, pid, started_at, resumed_at, ended_at, process_token, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)')
|
|
1785
|
+
.in('id', ids), 'read', 'which local conversation sessions still have an owner');
|
|
1786
|
+
const byId = new Map(rows.map((row) => [row.id, row]));
|
|
1787
|
+
for (const id of ids) {
|
|
1788
|
+
if (inFlightOwnerIds.has(id))
|
|
1789
|
+
continue;
|
|
1790
|
+
const row = byId.get(id);
|
|
1791
|
+
const localProcessInUse = !!row
|
|
1792
|
+
&& row.machine_id === machineId
|
|
1793
|
+
&& ((row.pid !== null && processIsAlive(row.pid))
|
|
1794
|
+
|| (row.state === 'running'
|
|
1795
|
+
&& row.ended_at === null
|
|
1796
|
+
&& row.pid === null
|
|
1797
|
+
&& Date.now() - new Date(row.resumed_at ?? row.started_at).getTime() < PID_GRACE_MS));
|
|
1798
|
+
const rowStillOwnsConversation = !!row
|
|
1799
|
+
&& row.card?.conversation_run_id === id
|
|
1800
|
+
&& ['running', 'asked', 'finished'].includes(row.state);
|
|
1801
|
+
if (localProcessInUse || rowStillOwnsConversation)
|
|
1802
|
+
continue;
|
|
1803
|
+
removeOwnerSession(id);
|
|
1804
|
+
if (homeIds.includes(id))
|
|
1805
|
+
removePanel3CodexOwnerHome(id);
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1303
1808
|
/**
|
|
1304
1809
|
* ═══ THE USER STOPPED IT, OR THEIR NEXT MESSAGE REPLACED IT, SO THE PROCESS
|
|
1305
1810
|
* STOPS. ═══
|
|
@@ -1441,7 +1946,7 @@ async function killStopped(client, machineId) {
|
|
|
1441
1946
|
async function recoverStranded(client, tools, machineId, mine, hold) {
|
|
1442
1947
|
const live = await returned(client
|
|
1443
1948
|
.from('panel3_runs')
|
|
1444
|
-
.select('id, card_id, parent_run_id, state, pid, started_at, resumed_at')
|
|
1949
|
+
.select('id, card_id, parent_run_id, state, pid, started_at, resumed_at, process_token, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)')
|
|
1445
1950
|
.eq('machine_id', machineId)
|
|
1446
1951
|
/* ═══ AND A RUN THAT STOPPED TO ASK IS SWEPT TOO, WHICH IS WHAT MAKES THE
|
|
1447
1952
|
RE-ARM'S OWN PREDICATE SAFE. ═══ `panel3_take_rearms` will only start a run
|
|
@@ -1468,6 +1973,23 @@ async function recoverStranded(client, tools, machineId, mine, hold) {
|
|
|
1468
1973
|
continue;
|
|
1469
1974
|
if (runProcessIsAlive(run, booted))
|
|
1470
1975
|
continue;
|
|
1976
|
+
const isOwner = run.card?.conversation_run_id === run.id && run.process_token !== null;
|
|
1977
|
+
if (isOwner) {
|
|
1978
|
+
try {
|
|
1979
|
+
if (run.state === 'asked') {
|
|
1980
|
+
await writeAnswer(client, run.id, run.card_id, null, run.process_token ?? undefined);
|
|
1981
|
+
}
|
|
1982
|
+
else {
|
|
1983
|
+
const resumed = await activateOwner(client, tools, machineId, run.id, run.process_token, run.pid);
|
|
1984
|
+
if (resumed)
|
|
1985
|
+
hold(run.id, resumed.settled);
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
catch (error) {
|
|
1989
|
+
said(`could not recover owner ${run.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1990
|
+
}
|
|
1991
|
+
continue;
|
|
1992
|
+
}
|
|
1471
1993
|
if (run.state === 'asked') {
|
|
1472
1994
|
/* ═══ ITS END IS STAMPED AND ITS TURNS ARE LEFT ALONE. ═══ This run is not
|
|
1473
1995
|
handed back and is not started again from here: its continuation is the
|
|
@@ -1662,7 +2184,7 @@ export async function takeHandedBack(client, tools, machineId, mine, hold) {
|
|
|
1662
2184
|
whether its pid means anything here. See the header. */
|
|
1663
2185
|
const offered = await returned(client
|
|
1664
2186
|
.from('panel3_runs')
|
|
1665
|
-
.select('id, card_id, machine_id, pid, started_at, resumed_at')
|
|
2187
|
+
.select('id, card_id, machine_id, pid, started_at, resumed_at, process_token, card:panel3_cards!panel3_runs_card_id_fkey(conversation_run_id)')
|
|
1666
2188
|
.not('handed_back_at', 'is', null)
|
|
1667
2189
|
/* WHAT THE OFFER MEANS, RESTATED IN THE READ. `panel3_hand_back` only ever
|
|
1668
2190
|
stamps a dispatched run that is still going, and `panel3_resume` refuses
|
|
@@ -1708,7 +2230,9 @@ export async function takeHandedBack(client, tools, machineId, mine, hold) {
|
|
|
1708
2230
|
continue;
|
|
1709
2231
|
}
|
|
1710
2232
|
try {
|
|
1711
|
-
const resumed =
|
|
2233
|
+
const resumed = run.card?.conversation_run_id === run.id && run.process_token !== null
|
|
2234
|
+
? await activateOwner(client, tools, machineId, run.id, run.process_token, null)
|
|
2235
|
+
: await resumeRun(client, tools, machineId, run.id, null);
|
|
1712
2236
|
if (resumed)
|
|
1713
2237
|
hold(run.id, resumed.settled);
|
|
1714
2238
|
else
|
|
@@ -1794,8 +2318,8 @@ export async function run(args, injected) {
|
|
|
1794
2318
|
`tools` is referenced inside the callback it is being given, which is safe
|
|
1795
2319
|
for the plain reason that the callback can only run once a request has
|
|
1796
2320
|
arrived at a server that by then exists. */
|
|
1797
|
-
const tools = await startToolsServer(client, async (parentRunId, brief, codebase) => {
|
|
1798
|
-
const child = await startChild(client, tools, machineId, parentRunId, brief, codebase);
|
|
2321
|
+
const tools = await startToolsServer(client, async (parentRunId, brief, codebase, processToken) => {
|
|
2322
|
+
const child = await startChild(client, tools, machineId, parentRunId, brief, codebase, processToken);
|
|
1799
2323
|
hold(child.runId, child.settled);
|
|
1800
2324
|
return { runId: child.runId };
|
|
1801
2325
|
});
|
|
@@ -1856,6 +2380,7 @@ export async function run(args, injected) {
|
|
|
1856
2380
|
to run first: `panel3_stop_card` has already ended the runs, so recovery
|
|
1857
2381
|
cannot see them and neither take can start them. */
|
|
1858
2382
|
await killStopped(client, machineId);
|
|
2383
|
+
await reconcileOwnerSessions(client, machineId, machineHarness, new Set(inFlight.keys()));
|
|
1859
2384
|
await recoverStranded(client, tools, machineId, new Set(inFlight.keys()), hold);
|
|
1860
2385
|
/* ═══ AND THEN WHAT SOMEBODY ELSE'S MACHINE WAS HOLDING, IF A PERSON HANDED
|
|
1861
2386
|
IT BACK. ═══ AFTER the sweep, deliberately: this machine settles its own
|
|
@@ -1868,6 +2393,9 @@ export async function run(args, injected) {
|
|
|
1868
2393
|
with a brief and a report behind it, comes before work that has not
|
|
1869
2394
|
started. */
|
|
1870
2395
|
await takeHandedBack(client, tools, machineId, new Set(inFlight.keys()), hold);
|
|
2396
|
+
/* Every reason a conversational owner may continue is claimed together.
|
|
2397
|
+
The legacy turn and re-arm takes exclude named owners in the database. */
|
|
2398
|
+
await takeOwnerActivations(client, tools, machineId, new Set(inFlight.keys()), hold);
|
|
1871
2399
|
/* THE MACHINE ID GOES IN because the take writes the run row, and a run has
|
|
1872
2400
|
to say where it is running: the exclusion is cross-machine and recovery is
|
|
1873
2401
|
per-machine, so a row with nobody's machine on it could be neither. */
|