@ctrl-spc/cs 0.7.14 → 0.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -3
- package/dist/autostart.js +103 -122
- package/dist/companion-ui.js +34 -6
- package/dist/companion.js +59 -150
- package/dist/config.js +52 -1
- package/dist/daemon-lifecycle.js +548 -0
- package/dist/daemon-lock.js +149 -42
- package/dist/daemon-processes.js +756 -0
- package/dist/daemon.js +14 -46
- package/dist/darwin-coalition.js +340 -0
- package/dist/index.js +70 -74
- package/dist/login.js +5 -3
- package/dist/native/darwin-coalition +0 -0
- package/dist/native/darwin-coalition.build.json +1 -0
- package/dist/native/darwin-coalition.c +145 -0
- package/dist/orchestrator.js +620 -428
- package/dist/panel3/run.js +822 -520
- package/dist/panel3/spawn.js +59 -11
- package/dist/presence.js +183 -24
- package/dist/supabase.js +43 -9
- package/dist/win-shell.js +464 -1
- package/dist/windows-job.js +312 -0
- package/package.json +4 -3
package/dist/panel3/run.js
CHANGED
|
@@ -146,11 +146,168 @@ import { listPanel3CodexOwnerHomeIds, removePanel3CodexOwnerHome, } from '../cod
|
|
|
146
146
|
import { getMachineIdentity, scratchDir } from '../config.js';
|
|
147
147
|
import { listCodebases } from '../codebases.js';
|
|
148
148
|
import { killTree, processIsAlive } from '../win-shell.js';
|
|
149
|
+
import { randomUUID } from 'node:crypto';
|
|
149
150
|
import { hostname, uptime } from 'node:os';
|
|
150
151
|
import { execFileSync } from 'node:child_process';
|
|
151
152
|
import { existsSync } from 'node:fs';
|
|
152
153
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
153
154
|
import { join } from 'node:path';
|
|
155
|
+
function recoveryAttempt(run) {
|
|
156
|
+
return { runId: run.id, processToken: run.process_token, startedAt: run.started_at, resumedAt: run.resumed_at };
|
|
157
|
+
}
|
|
158
|
+
async function startTrackedAgent(client, runId, preparation, work, ...args) {
|
|
159
|
+
if (!preparation)
|
|
160
|
+
return startAgent(...args);
|
|
161
|
+
const claimed = work?.claimAttempt(runId);
|
|
162
|
+
if (!claimed)
|
|
163
|
+
throw new Error('This run no longer belongs to the claimed attempt.');
|
|
164
|
+
preparation.setAttempt(claimed);
|
|
165
|
+
let preparationFailed = false;
|
|
166
|
+
let failureReason = '';
|
|
167
|
+
const execution = {
|
|
168
|
+
...preparation.execution,
|
|
169
|
+
register: async (child, agent) => {
|
|
170
|
+
await preparation.execution.register(child, agent);
|
|
171
|
+
// An ordinary stop can reopen admission after a refusal. Every such
|
|
172
|
+
// wait needs a fresh cloud check before the prompt is authorized.
|
|
173
|
+
try {
|
|
174
|
+
let waited;
|
|
175
|
+
do {
|
|
176
|
+
const { data, error } = await client.from('panel3_runs')
|
|
177
|
+
.select('process_token,started_at,resumed_at,state,ended_at').eq('id', runId).single();
|
|
178
|
+
if (error)
|
|
179
|
+
throw error;
|
|
180
|
+
if (!data || data.state !== 'running' || data.ended_at !== null
|
|
181
|
+
|| claimed.processToken !== (data.process_token ?? null) || claimed.startedAt !== data.started_at
|
|
182
|
+
|| claimed.resumedAt !== (data.resumed_at ?? null)) {
|
|
183
|
+
throw new Error('This run no longer belongs to the claimed attempt.');
|
|
184
|
+
}
|
|
185
|
+
waited = await preparation.execution.waitForPromptAdmission();
|
|
186
|
+
} while (waited);
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
if (!preparation.execution.interrupted()) {
|
|
190
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
191
|
+
preparation.deferFailure(reason, args[1]);
|
|
192
|
+
preparationFailed = true;
|
|
193
|
+
failureReason = reason;
|
|
194
|
+
}
|
|
195
|
+
throw error;
|
|
196
|
+
}
|
|
197
|
+
},
|
|
198
|
+
};
|
|
199
|
+
const started = startAgent(args[0], args[1], args[2], args[3], args[4], args[5], execution);
|
|
200
|
+
// Do not hold the claim barrier while registration waits for an ordinary
|
|
201
|
+
// stop to be refused. The original answer promises proven process closure.
|
|
202
|
+
const answered = started.answered.then(async (answer) => {
|
|
203
|
+
if (preparationFailed && !preparation.execution.interrupted()) {
|
|
204
|
+
await endRun(client, args[1], runId, claimed.cardId, failureReason, claimed);
|
|
205
|
+
preparation.execution.complete();
|
|
206
|
+
}
|
|
207
|
+
return answer;
|
|
208
|
+
});
|
|
209
|
+
void answered.catch(() => { }); // The caller records the PID before awaiting settlement.
|
|
210
|
+
return { ...started, answered, attempt: claimed, preparationFailed: () => preparationFailed };
|
|
211
|
+
}
|
|
212
|
+
function panelClaimAttempts(result) {
|
|
213
|
+
if (!Array.isArray(result))
|
|
214
|
+
throw new Error('The claim journal returned no complete result.');
|
|
215
|
+
const unique = new Map();
|
|
216
|
+
for (const row of result) {
|
|
217
|
+
const attempt = row?._attempt;
|
|
218
|
+
if (!attempt || typeof attempt.run_id !== 'string' || typeof attempt.card_id !== 'string' || typeof attempt.started_at !== 'string') {
|
|
219
|
+
throw new Error('The claim journal returned an incomplete run identity.');
|
|
220
|
+
}
|
|
221
|
+
unique.set(attempt.run_id, { runId: attempt.run_id, cardId: attempt.card_id, processToken: attempt.process_token ?? null,
|
|
222
|
+
pid: attempt.pid ?? null, startedAt: attempt.started_at, resumedAt: attempt.resumed_at ?? null,
|
|
223
|
+
observedPendingTurnIds: attempt.observed_pending_turn_ids ?? null });
|
|
224
|
+
}
|
|
225
|
+
return [...unique.values()];
|
|
226
|
+
}
|
|
227
|
+
async function panelClaim(client, work, action, args) {
|
|
228
|
+
if (!work)
|
|
229
|
+
return { ...await client.rpc(action, args), operationId: undefined };
|
|
230
|
+
const operationId = work.beginRpc(action);
|
|
231
|
+
try {
|
|
232
|
+
const response = await client.rpc(action, { ...args, p_operation_id: operationId });
|
|
233
|
+
if (!response.error) {
|
|
234
|
+
const journal = await client.rpc('panel3_reconcile_claim', { p_machine_id: args.p_machine_id, p_operation_id: operationId });
|
|
235
|
+
if (journal.error)
|
|
236
|
+
throw journal.error;
|
|
237
|
+
const row = journal.data?.[0];
|
|
238
|
+
if (row?.outcome !== 'completed' || row.action !== action)
|
|
239
|
+
throw new Error('The claim was cancelled before it could start.');
|
|
240
|
+
work.recordRpc(operationId, panelClaimAttempts(row.result), false);
|
|
241
|
+
}
|
|
242
|
+
if (response.error)
|
|
243
|
+
work.deferRpc(operationId);
|
|
244
|
+
return { ...response, operationId };
|
|
245
|
+
}
|
|
246
|
+
catch (error) {
|
|
247
|
+
work.deferRpc(operationId);
|
|
248
|
+
throw error;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
async function reconcilePanelInterruptions(client, machineId, work) {
|
|
252
|
+
const capability = await client.rpc('panel3_interrupt_machine_runs', { p_machine_id: machineId,
|
|
253
|
+
p_operation_id: randomUUID(), p_interrupted_at: new Date().toISOString(), p_attempts: [] });
|
|
254
|
+
if (capability.error)
|
|
255
|
+
throw capability.error;
|
|
256
|
+
for (const failure of work.failedPreparations()) {
|
|
257
|
+
await endRun(client, failure.level, failure.attempt.runId, failure.attempt.cardId, failure.reason, failure.attempt);
|
|
258
|
+
work.acknowledgePreparationFailure(failure.id);
|
|
259
|
+
}
|
|
260
|
+
for (const claim of work.pendingRpcs()) {
|
|
261
|
+
const journal = await client.rpc('panel3_reconcile_claim', { p_machine_id: machineId, p_operation_id: claim.id });
|
|
262
|
+
if (journal.error)
|
|
263
|
+
throw journal.error;
|
|
264
|
+
const row = journal.data?.[0];
|
|
265
|
+
if (row?.outcome === 'cancelled')
|
|
266
|
+
work.recordRpc(claim.id, [], true);
|
|
267
|
+
else if (row?.outcome === 'completed' && row.action === claim.action)
|
|
268
|
+
work.recordRpc(claim.id, panelClaimAttempts(row.result), true);
|
|
269
|
+
else
|
|
270
|
+
throw new Error('The pending claim could not be reconciled.');
|
|
271
|
+
}
|
|
272
|
+
if (work.legacyRequired()) {
|
|
273
|
+
const attempts = [];
|
|
274
|
+
for (let offset = 0;; offset += 500) {
|
|
275
|
+
const { data, error } = await client.from('panel3_runs')
|
|
276
|
+
.select('id,card_id,process_token,pid,started_at,resumed_at')
|
|
277
|
+
.eq('machine_id', machineId).is('ended_at', null).order('id').range(offset, offset + 499);
|
|
278
|
+
if (error)
|
|
279
|
+
throw error;
|
|
280
|
+
if (!Array.isArray(data))
|
|
281
|
+
throw new Error('The old run snapshot did not return a complete page.');
|
|
282
|
+
for (const row of data) {
|
|
283
|
+
if (typeof row.started_at !== 'string')
|
|
284
|
+
throw new Error('An old run has no exact attempt identity.');
|
|
285
|
+
attempts.push({ runId: row.id, cardId: row.card_id, processToken: row.process_token ?? null,
|
|
286
|
+
pid: row.pid ?? null, startedAt: row.started_at, resumedAt: row.resumed_at ?? null, observedPendingTurnIds: null });
|
|
287
|
+
}
|
|
288
|
+
if (data.length < 500)
|
|
289
|
+
break;
|
|
290
|
+
}
|
|
291
|
+
for (const attempt of attempts)
|
|
292
|
+
work.recordLegacy(attempt);
|
|
293
|
+
work.legacyComplete();
|
|
294
|
+
}
|
|
295
|
+
for (const receipt of work.receipts()) {
|
|
296
|
+
const attempt = receipt.attempt;
|
|
297
|
+
const { data, error } = await client.rpc('panel3_interrupt_machine_runs', {
|
|
298
|
+
p_machine_id: machineId, p_operation_id: receipt.operationId, p_interrupted_at: receipt.interruptedAt,
|
|
299
|
+
p_attempts: [{ run_id: attempt.runId, process_token: attempt.processToken, pid: receipt.pid ?? attempt.pid ?? null,
|
|
300
|
+
started_at: attempt.startedAt, resumed_at: attempt.resumedAt, observed_pending_turn_ids: attempt.observedPendingTurnIds }],
|
|
301
|
+
});
|
|
302
|
+
if (error)
|
|
303
|
+
throw error;
|
|
304
|
+
const result = data?.[0];
|
|
305
|
+
if (!result || result.run_id !== attempt.runId || !['interrupted', 'already_interrupted', 'continued', 'stale'].includes(result.outcome)) {
|
|
306
|
+
throw new Error('The interrupted run was not acknowledged. Work remains protected.');
|
|
307
|
+
}
|
|
308
|
+
work.acknowledge(receipt.id);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
154
311
|
const USAGE = 'usage: run [--once]';
|
|
155
312
|
/** How long between takes. Short, because it is the whole delay between a user
|
|
156
313
|
* sending and a card showing an agent on it, and the take is one small indexed
|
|
@@ -550,18 +707,24 @@ pictures = []) => {
|
|
|
550
707
|
* a dispatch knows the brief before the row exists and writes it there. Passing
|
|
551
708
|
* it again would be rewriting a brief that ux.md fixes at dispatch.
|
|
552
709
|
*
|
|
553
|
-
* A failure
|
|
554
|
-
*
|
|
555
|
-
*
|
|
556
|
-
* which errs towards answering again rather than stranding.
|
|
710
|
+
* A bookkeeping failure does not discard the locally tracked execution. Exact
|
|
711
|
+
* activation predicates prevent a delayed write from reviving an ended run or
|
|
712
|
+
* attaching this process to a later continuation.
|
|
557
713
|
*/
|
|
558
|
-
async function recordProcess(client, runId, pid, brief, processToken) {
|
|
714
|
+
async function recordProcess(client, runId, pid, brief, processToken, attempt) {
|
|
559
715
|
let query = client
|
|
560
716
|
.from('panel3_runs')
|
|
561
717
|
.update({ pid, ...(brief === undefined ? {} : { brief }) })
|
|
562
|
-
.eq('id', runId)
|
|
718
|
+
.eq('id', runId)
|
|
719
|
+
.eq('state', 'running')
|
|
720
|
+
.is('ended_at', null);
|
|
563
721
|
if (processToken !== undefined)
|
|
564
722
|
query = query.eq('process_token', processToken);
|
|
723
|
+
if (attempt) {
|
|
724
|
+
query = query.eq('started_at', attempt.startedAt);
|
|
725
|
+
query = attempt.resumedAt === null ? query.is('resumed_at', null) : query.eq('resumed_at', attempt.resumedAt);
|
|
726
|
+
query = attempt.processToken === null ? query.is('process_token', null) : query.eq('process_token', attempt.processToken);
|
|
727
|
+
}
|
|
565
728
|
const written = await returned(query.select('id'), 'record what is running', `run ${runId}`);
|
|
566
729
|
if (written.length === 0)
|
|
567
730
|
throw new Error(`could not record what is running for run ${runId}: its activation has ended`);
|
|
@@ -579,15 +742,21 @@ async function recordProcess(client, runId, pid, brief, processToken) {
|
|
|
579
742
|
* there first. It is a fact about the record rather than a failure, so it does
|
|
580
743
|
* not go through `returned()`, exactly as the answer's own null does not.
|
|
581
744
|
*/
|
|
582
|
-
async function giveUp(client, runId, reason, processToken) {
|
|
745
|
+
async function giveUp(client, runId, reason, processToken, attempt) {
|
|
583
746
|
const { data, error } = await client
|
|
584
747
|
.rpc('panel3_give_up', {
|
|
585
748
|
p_run_id: runId,
|
|
586
749
|
p_reason: reason,
|
|
587
750
|
...(processToken === undefined ? {} : { p_process_token: processToken }),
|
|
751
|
+
...(attempt ? { p_process_token: attempt.processToken, p_expected_attempt: {
|
|
752
|
+
started_at: attempt.startedAt, resumed_at: attempt.resumedAt,
|
|
753
|
+
} } : {}),
|
|
588
754
|
});
|
|
589
755
|
if (error)
|
|
590
756
|
throw new Error(`could not give up run ${runId}: ${error.message}`);
|
|
757
|
+
if (attempt && data !== null && (!Number.isInteger(data) || data < 0)) {
|
|
758
|
+
throw new Error(`could not give up run ${runId}: no ending was acknowledged`);
|
|
759
|
+
}
|
|
591
760
|
/* THE RUN IS OVER, so whatever it read is dropped. Every ending does this —
|
|
592
761
|
here, `failRun` and `writeAnswer` — because a daemon stays up for days and
|
|
593
762
|
has no business holding Tuesday's secret. */
|
|
@@ -828,76 +997,82 @@ export async function settingsForRun(client, runId) {
|
|
|
828
997
|
* the level is what tells it how a run of this shape ends.
|
|
829
998
|
*/
|
|
830
999
|
async function answerCard(client, tools, machineId, cardId, turns) {
|
|
831
|
-
const
|
|
832
|
-
/* BEFORE THE SPAWN, AND ITS FAILURE IS THE SPAWN'S FAILURE. The receipts are
|
|
833
|
-
part of what the agent is sent, so a read that fails must not be papered
|
|
834
|
-
over with an empty list: that reads as a card that has made nothing, which
|
|
835
|
-
is how an agent creates a second epic beside the one it cannot see. The
|
|
836
|
-
attachments read carries the same rule: a failed read here must not read
|
|
837
|
-
as "nothing is attached", which is a different card than the one that was
|
|
838
|
-
actually sent. */
|
|
839
|
-
/* ═══ AND A FAILURE HERE ENDS THE RUN, RATHER THAN LEAVING IT RUNNING WITH NO
|
|
840
|
-
PROCESS. ═══ The take already wrote the run row in the statement that leased
|
|
841
|
-
the turns, so a throw between here and `startAgent` leaves a run reading
|
|
842
|
-
`running` with a null pid and nothing on stderr the person can see.
|
|
843
|
-
`recoverStranded` then reads that as a machine that went away, hands the
|
|
844
|
-
message back, and `panel3_take_turns` leases it to A BRAND NEW RUN whose
|
|
845
|
-
attempts start again at one — so a permanent failure, such as a read this
|
|
846
|
-
build cannot make against the current schema, repeats forever while the card
|
|
847
|
-
says Working and never says why. This is the sixth of `endRun`'s endings and
|
|
848
|
-
the last one that was missing: `resumeRun` and `startRearmed` already end
|
|
849
|
-
their two post-claim failures this way for exactly this reason.
|
|
850
|
-
THE REASON IS SHAREABLE. All three reads are `returned()` calls against the
|
|
851
|
-
database, whose messages name tables and columns and never a local path, so
|
|
852
|
-
constraint 6 is satisfied without a level fork here. */
|
|
853
|
-
let brief;
|
|
854
|
-
/* WHERE IT RUNS AND WHAT IT IS TOLD ABOUT THE CODEBASES, THROUGH THE SAME
|
|
855
|
-
SEAM AS EVERY OTHER SPAWN. A level 1 run works in an empty directory of the
|
|
856
|
-
user's own, and it is the fifth start site rather than a special case: the
|
|
857
|
-
generated git block reaches every level, and one place resolving it is what
|
|
858
|
-
makes that true without four copies of the read. */
|
|
859
|
-
let where;
|
|
860
|
-
/* THE MANDATE IS READ IN THE SAME WINDOW AND UNDER THE SAME RULE. A launcher
|
|
861
|
-
is an agent like any other and gets the project's standing rules before it
|
|
862
|
-
decides anything, and a read that fails must not read as "this project has
|
|
863
|
-
no rules" — that is a different project than the one the person is on. So it
|
|
864
|
-
joins the three reads above inside this ending rather than beside it. */
|
|
865
|
-
let rules;
|
|
866
|
-
let settings;
|
|
867
|
-
try {
|
|
868
|
-
brief = briefFor(turns[0].card_title, turns, await receiptsFor(client, cardId), await attachmentsFor(client, cardId), await codebasesForRun(client, runId));
|
|
869
|
-
rules = await standingRulesFor(client, runId);
|
|
870
|
-
settings = await settingsForRun(client, runId);
|
|
871
|
-
where = await workingDirectory(client, runId, LEVEL, false);
|
|
872
|
-
}
|
|
873
|
-
catch (error) {
|
|
874
|
-
const why = error instanceof Error ? error.message : String(error);
|
|
875
|
-
await endRun(client, LEVEL, runId, cardId, why);
|
|
876
|
-
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
877
|
-
}
|
|
878
|
-
/* ═══ THE RUN ID IS ON THE URL, AND THAT IS THE WHOLE OF WHAT THE AGENT IS
|
|
879
|
-
TOLD ABOUT ITS OWN STANDING. ═══ The tools server reads the level off the
|
|
880
|
-
run row this id names, so the daemon does not tell the child what it may do
|
|
881
|
-
and the child has nothing to claim. `LEVEL` below decides argv only — which
|
|
882
|
-
of the harness's own tools the process gets — and the two can never disagree
|
|
883
|
-
about the record, because only one of them consults it. */
|
|
884
|
-
/* AN EMPTY DIRECTORY OF THE USER'S OWN, because level 1 has no code tool to
|
|
885
|
-
use a real one with, and the daemon's inherited cwd under a launchd login
|
|
886
|
-
item is the filesystem root. */
|
|
887
|
-
const started = startAgent(withStandingRules(rules, brief, where.block), LEVEL, tools.urlFor(runId), where.cwd, undefined, settings);
|
|
888
|
-
out(`run ${runId} level ${LEVEL} ${started.pid ? `pid ${started.pid}` : 'no process'}`);
|
|
1000
|
+
const preparation = tools.work?.prepare(turns[0].run_id);
|
|
889
1001
|
try {
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
1002
|
+
const runId = turns[0].run_id;
|
|
1003
|
+
/* BEFORE THE SPAWN, AND ITS FAILURE IS THE SPAWN'S FAILURE. The receipts are
|
|
1004
|
+
part of what the agent is sent, so a read that fails must not be papered
|
|
1005
|
+
over with an empty list: that reads as a card that has made nothing, which
|
|
1006
|
+
is how an agent creates a second epic beside the one it cannot see. The
|
|
1007
|
+
attachments read carries the same rule: a failed read here must not read
|
|
1008
|
+
as "nothing is attached", which is a different card than the one that was
|
|
1009
|
+
actually sent. */
|
|
1010
|
+
/* ═══ AND A FAILURE HERE ENDS THE RUN, RATHER THAN LEAVING IT RUNNING WITH NO
|
|
1011
|
+
PROCESS. ═══ The take already wrote the run row in the statement that leased
|
|
1012
|
+
the turns, so a throw between here and `startAgent` leaves a run reading
|
|
1013
|
+
`running` with a null pid and nothing on stderr the person can see.
|
|
1014
|
+
`recoverStranded` then reads that as a machine that went away, hands the
|
|
1015
|
+
message back, and `panel3_take_turns` leases it to A BRAND NEW RUN whose
|
|
1016
|
+
attempts start again at one — so a permanent failure, such as a read this
|
|
1017
|
+
build cannot make against the current schema, repeats forever while the card
|
|
1018
|
+
says Working and never says why. This is the sixth of `endRun`'s endings and
|
|
1019
|
+
the last one that was missing: `resumeRun` and `startRearmed` already end
|
|
1020
|
+
their two post-claim failures this way for exactly this reason.
|
|
1021
|
+
THE REASON IS SHAREABLE. All three reads are `returned()` calls against the
|
|
1022
|
+
database, whose messages name tables and columns and never a local path, so
|
|
1023
|
+
constraint 6 is satisfied without a level fork here. */
|
|
1024
|
+
let brief;
|
|
1025
|
+
/* WHERE IT RUNS AND WHAT IT IS TOLD ABOUT THE CODEBASES, THROUGH THE SAME
|
|
1026
|
+
SEAM AS EVERY OTHER SPAWN. A level 1 run works in an empty directory of the
|
|
1027
|
+
user's own, and it is the fifth start site rather than a special case: the
|
|
1028
|
+
generated git block reaches every level, and one place resolving it is what
|
|
1029
|
+
makes that true without four copies of the read. */
|
|
1030
|
+
let where;
|
|
1031
|
+
/* THE MANDATE IS READ IN THE SAME WINDOW AND UNDER THE SAME RULE. A launcher
|
|
1032
|
+
is an agent like any other and gets the project's standing rules before it
|
|
1033
|
+
decides anything, and a read that fails must not read as "this project has
|
|
1034
|
+
no rules" — that is a different project than the one the person is on. So it
|
|
1035
|
+
joins the three reads above inside this ending rather than beside it. */
|
|
1036
|
+
let rules;
|
|
1037
|
+
let settings;
|
|
1038
|
+
try {
|
|
1039
|
+
brief = briefFor(turns[0].card_title, turns, await receiptsFor(client, cardId), await attachmentsFor(client, cardId), await codebasesForRun(client, runId));
|
|
1040
|
+
rules = await standingRulesFor(client, runId);
|
|
1041
|
+
settings = await settingsForRun(client, runId);
|
|
1042
|
+
where = await workingDirectory(client, runId, LEVEL, false);
|
|
1043
|
+
}
|
|
1044
|
+
catch (error) {
|
|
1045
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
1046
|
+
await endRun(client, LEVEL, runId, cardId, why);
|
|
1047
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
1048
|
+
}
|
|
1049
|
+
/* ═══ THE RUN ID IS ON THE URL, AND THAT IS THE WHOLE OF WHAT THE AGENT IS
|
|
1050
|
+
TOLD ABOUT ITS OWN STANDING. ═══ The tools server reads the level off the
|
|
1051
|
+
run row this id names, so the daemon does not tell the child what it may do
|
|
1052
|
+
and the child has nothing to claim. `LEVEL` below decides argv only — which
|
|
1053
|
+
of the harness's own tools the process gets — and the two can never disagree
|
|
1054
|
+
about the record, because only one of them consults it. */
|
|
1055
|
+
/* AN EMPTY DIRECTORY OF THE USER'S OWN, because level 1 has no code tool to
|
|
1056
|
+
use a real one with, and the daemon's inherited cwd under a launchd login
|
|
1057
|
+
item is the filesystem root. */
|
|
1058
|
+
const started = await startTrackedAgent(client, runId, preparation, tools.work, withStandingRules(rules, brief, where.block), LEVEL, tools.urlFor(runId), where.cwd, undefined, settings);
|
|
1059
|
+
out(`run ${runId} level ${LEVEL} ${started.pid ? `pid ${started.pid}` : 'no process'}`);
|
|
1060
|
+
try {
|
|
1061
|
+
/* THE BRIEF, NOT WHAT THE PROCESS WAS HANDED. The rules are current at the
|
|
1062
|
+
activation and the brief is immutable, so storing the composed string
|
|
1063
|
+
would freeze one inside the other and every respawn would replay it. */
|
|
1064
|
+
await recordProcess(client, runId, started.pid, brief, undefined, started.attempt);
|
|
1065
|
+
}
|
|
1066
|
+
catch (error) {
|
|
1067
|
+
// Said, not fatal. See `recordProcess` for what this costs and why the
|
|
1068
|
+
// agent is not killed over it.
|
|
1069
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1070
|
+
}
|
|
1071
|
+
return settle(client, tools, machineId, LEVEL, runId, cardId, started, false);
|
|
894
1072
|
}
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
// agent is not killed over it.
|
|
898
|
-
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1073
|
+
finally {
|
|
1074
|
+
preparation?.finish();
|
|
899
1075
|
}
|
|
900
|
-
return settle(client, tools, machineId, LEVEL, runId, cardId, started, false);
|
|
901
1076
|
}
|
|
902
1077
|
/**
|
|
903
1078
|
* WHICH PROJECT A RUN'S CARD IS FILED UNDER, or null when the card has none.
|
|
@@ -1225,91 +1400,104 @@ async function workingDirectory(client, runId, level, isOwner, knownCodebase) {
|
|
|
1225
1400
|
* that satisfies it.
|
|
1226
1401
|
*/
|
|
1227
1402
|
async function startChild(client, tools, machineId, parentRunId, brief, codebase, parentProcessToken, choice = {}) {
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
p_brief: brief,
|
|
1231
|
-
p_machine_id: machineId,
|
|
1232
|
-
p_codebase_id: codebase?.id ?? null,
|
|
1233
|
-
p_codebase_label: codebase?.name ?? null,
|
|
1234
|
-
p_process_token: parentProcessToken ?? null,
|
|
1235
|
-
p_model: choice.model ?? null,
|
|
1236
|
-
p_effort: choice.effort ?? null,
|
|
1237
|
-
});
|
|
1238
|
-
if (error)
|
|
1239
|
-
throw new Error(`could not start an agent under run ${parentRunId}: ${readableWriteError(error.message)}`);
|
|
1240
|
-
const row = data?.[0];
|
|
1241
|
-
if (!row) {
|
|
1242
|
-
/* NOTHING WAS WRITTEN AND NOTHING IS RUNNING, and the two reasons are said
|
|
1243
|
-
together because the caller cannot tell them apart from here and both mean
|
|
1244
|
-
the same thing to it. */
|
|
1245
|
-
throw new Error(`NO AGENT WAS STARTED and nothing was written: run ${parentRunId} has ended, is already as `
|
|
1246
|
-
+ 'deep as anything may be sent from, or this conversation already has its owner. Exit now.');
|
|
1247
|
-
}
|
|
1248
|
-
if (row.run_level !== 2 && row.run_level !== 3) {
|
|
1249
|
-
/* UNREACHABLE, AND STILL SETTLED. `panel3_dispatch` writes `parent.level + 1`
|
|
1250
|
-
from a parent it has just checked is below 3, so there is no level here
|
|
1251
|
-
this cannot spawn. If that ever stops being true, the row exists and
|
|
1252
|
-
nothing will ever start for it, and leaving it `running` would make a
|
|
1253
|
-
recovery sweep wait out the pid grace window to conclude what is already
|
|
1254
|
-
known. */
|
|
1255
|
-
const why = `run ${row.run_id} was written at level ${row.run_level}, which cannot be spawned`;
|
|
1256
|
-
await giveUp(client, row.run_id, why, row.process_token ?? undefined);
|
|
1257
|
-
throw new Error(why);
|
|
1258
|
-
}
|
|
1259
|
-
const level = row.run_level;
|
|
1260
|
-
let where;
|
|
1261
|
-
let prompt;
|
|
1262
|
-
let pictures;
|
|
1263
|
-
/* READ AGAINST THE CHILD'S OWN RUN ROW, NOT THE PARENT'S. The row already
|
|
1264
|
-
exists (`panel3_dispatch` wrote it above) and it carries this child's
|
|
1265
|
-
codebase, which is what decides which codebase-scoped rules it is under. */
|
|
1266
|
-
let rules;
|
|
1267
|
-
let settings;
|
|
1403
|
+
let claimOperationId;
|
|
1404
|
+
const preparation = tools.work?.prepare();
|
|
1268
1405
|
try {
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
:
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1406
|
+
const response = await panelClaim(client, tools.work, 'panel3_dispatch', {
|
|
1407
|
+
p_parent_run_id: parentRunId,
|
|
1408
|
+
p_brief: brief,
|
|
1409
|
+
p_machine_id: machineId,
|
|
1410
|
+
p_codebase_id: codebase?.id ?? null,
|
|
1411
|
+
p_codebase_label: codebase?.name ?? null,
|
|
1412
|
+
p_process_token: parentProcessToken ?? null,
|
|
1413
|
+
p_model: choice.model ?? null,
|
|
1414
|
+
p_effort: choice.effort ?? null,
|
|
1415
|
+
});
|
|
1416
|
+
const { data, error, operationId } = response;
|
|
1417
|
+
claimOperationId = operationId;
|
|
1418
|
+
if (error)
|
|
1419
|
+
throw new Error(`could not start an agent under run ${parentRunId}: ${readableWriteError(error.message)}`);
|
|
1420
|
+
const row = data?.[0];
|
|
1421
|
+
if (!row) {
|
|
1422
|
+
/* NOTHING WAS WRITTEN AND NOTHING IS RUNNING, and the two reasons are said
|
|
1423
|
+
together because the caller cannot tell them apart from here and both mean
|
|
1424
|
+
the same thing to it. */
|
|
1425
|
+
throw new Error(`NO AGENT WAS STARTED and nothing was written: run ${parentRunId} has ended, is already as `
|
|
1426
|
+
+ 'deep as anything may be sent from, or this conversation already has its owner. Exit now.');
|
|
1427
|
+
}
|
|
1428
|
+
if (row.run_level !== 2 && row.run_level !== 3) {
|
|
1429
|
+
/* UNREACHABLE, AND STILL SETTLED. `panel3_dispatch` writes `parent.level + 1`
|
|
1430
|
+
from a parent it has just checked is below 3, so there is no level here
|
|
1431
|
+
this cannot spawn. If that ever stops being true, the row exists and
|
|
1432
|
+
nothing will ever start for it, and leaving it `running` would make a
|
|
1433
|
+
recovery sweep wait out the pid grace window to conclude what is already
|
|
1434
|
+
known. */
|
|
1435
|
+
const why = `run ${row.run_id} was written at level ${row.run_level}, which cannot be spawned`;
|
|
1436
|
+
await giveUp(client, row.run_id, why, row.process_token ?? undefined);
|
|
1437
|
+
throw new Error(why);
|
|
1438
|
+
}
|
|
1439
|
+
const level = row.run_level;
|
|
1440
|
+
let where;
|
|
1441
|
+
let prompt;
|
|
1442
|
+
let pictures;
|
|
1443
|
+
/* READ AGAINST THE CHILD'S OWN RUN ROW, NOT THE PARENT'S. The row already
|
|
1444
|
+
exists (`panel3_dispatch` wrote it above) and it carries this child's
|
|
1445
|
+
codebase, which is what decides which codebase-scoped rules it is under. */
|
|
1446
|
+
let rules;
|
|
1447
|
+
let settings;
|
|
1448
|
+
try {
|
|
1449
|
+
where = await workingDirectory(client, row.run_id, level, level === 2, codebase);
|
|
1450
|
+
prompt = level === 2
|
|
1451
|
+
? await initialOwnerPrompt(client, row.run_card_id, parentRunId, brief)
|
|
1452
|
+
: brief;
|
|
1453
|
+
rules = await standingRulesFor(client, row.run_id);
|
|
1454
|
+
settings = await settingsForRun(client, row.run_id);
|
|
1455
|
+
/* Inside this site's own try, for `standingRulesFor`'s reason: a failure to
|
|
1456
|
+
assemble what the agent needs ends the run the way this path already ends
|
|
1457
|
+
runs, rather than starting a process that is missing it. */
|
|
1458
|
+
pictures = await picturesOnDisk(client, row.run_card_id, where, level);
|
|
1459
|
+
}
|
|
1460
|
+
catch (error) {
|
|
1461
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
1462
|
+
await giveUp(client, row.run_id, why, row.process_token ?? undefined);
|
|
1463
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
1464
|
+
}
|
|
1465
|
+
const processToken = row.process_token ?? undefined;
|
|
1466
|
+
const isOwner = level === 2;
|
|
1467
|
+
const started = await startTrackedAgent(client, row.run_id, preparation, tools.work, withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id, processToken), where.cwd, isOwner ? { ownerId: row.run_id } : undefined, settings);
|
|
1468
|
+
if (started.pid === null) {
|
|
1469
|
+
if (started.interrupted?.())
|
|
1470
|
+
throw new Error('Work was interrupted by the service command.');
|
|
1471
|
+
/* THERE IS A ROW AND THERE IS NO PROCESS, which is the one shape the record
|
|
1472
|
+
must never be left in quietly. The answer is already settled — nothing ran
|
|
1473
|
+
— so the reason is read off it, the run is ended with that reason on it,
|
|
1474
|
+
and the tool call fails saying no agent was started. */
|
|
1475
|
+
const answer = await started.answered;
|
|
1476
|
+
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
1477
|
+
await giveUp(client, row.run_id, reason, processToken);
|
|
1478
|
+
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
1479
|
+
}
|
|
1480
|
+
out(`dispatch run ${row.run_id} level ${level} under ${parentRunId} pid ${started.pid}`);
|
|
1481
|
+
try {
|
|
1482
|
+
await recordProcess(client, row.run_id, started.pid, undefined, processToken, started.attempt);
|
|
1483
|
+
}
|
|
1484
|
+
catch (error) {
|
|
1485
|
+
// Said, not fatal, exactly as at level 1: the agent is running and killing
|
|
1486
|
+
// it over a bookkeeping write would cost the user the work.
|
|
1487
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1488
|
+
}
|
|
1489
|
+
return {
|
|
1490
|
+
runId: row.run_id,
|
|
1491
|
+
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, true, processToken, isOwner && processToken
|
|
1492
|
+
? ownerSessionLifecycle(started, row.run_id, settings.harness ?? harness(), processToken)
|
|
1493
|
+
: undefined),
|
|
1494
|
+
};
|
|
1301
1495
|
}
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1496
|
+
finally {
|
|
1497
|
+
preparation?.finish();
|
|
1498
|
+
if (claimOperationId)
|
|
1499
|
+
tools.work?.finishRpcs([claimOperationId]);
|
|
1306
1500
|
}
|
|
1307
|
-
return {
|
|
1308
|
-
runId: row.run_id,
|
|
1309
|
-
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, true, processToken, isOwner && processToken
|
|
1310
|
-
? ownerSessionLifecycle(started, row.run_id, settings.harness ?? harness(), processToken)
|
|
1311
|
-
: undefined),
|
|
1312
|
-
};
|
|
1313
1501
|
}
|
|
1314
1502
|
/**
|
|
1315
1503
|
* HOW MANY PROCESSES HAVE BEEN STARTED FOR THIS RUN, counting the first.
|
|
@@ -1377,9 +1565,21 @@ async function attemptsSoFar(client, runId) {
|
|
|
1377
1565
|
* Returns whether THIS call is what ended it. False is a fact about the record,
|
|
1378
1566
|
* not a failure: the run had already ended, and the caller says so.
|
|
1379
1567
|
*/
|
|
1380
|
-
async function endRun(client, level, runId, cardId, why) {
|
|
1568
|
+
async function endRun(client, level, runId, cardId, why, attempt) {
|
|
1381
1569
|
if (level !== 1)
|
|
1382
|
-
return (await giveUp(client, runId, why)) !== null;
|
|
1570
|
+
return (await giveUp(client, runId, why, undefined, attempt)) !== null;
|
|
1571
|
+
if (attempt) {
|
|
1572
|
+
const { data, error } = await client.rpc('panel3_end_run', {
|
|
1573
|
+
p_run_id: runId, p_reason: why, p_process_token: attempt.processToken,
|
|
1574
|
+
p_expected_attempt: { started_at: attempt.startedAt, resumed_at: attempt.resumedAt },
|
|
1575
|
+
});
|
|
1576
|
+
if (error)
|
|
1577
|
+
throw new Error(`could not end run ${runId}: ${error.message}`);
|
|
1578
|
+
if (typeof data !== 'boolean')
|
|
1579
|
+
throw new Error(`could not end run ${runId}: no ending was acknowledged`);
|
|
1580
|
+
forgetSecrets(runId);
|
|
1581
|
+
return data;
|
|
1582
|
+
}
|
|
1383
1583
|
/* ═══ THE RUN FIRST, AND THE CARD ONLY IF THIS RUN WAS STILL THE CARD'S TO
|
|
1384
1584
|
FAIL. ═══
|
|
1385
1585
|
The card used to be written first, and the argument for that was a daemon
|
|
@@ -1464,6 +1664,8 @@ function ownerSessionLifecycle(started, ownerId, ownerHarness, processToken, exp
|
|
|
1464
1664
|
}
|
|
1465
1665
|
function settle(client, tools, machineId, level, runId, cardId, started, speaksToTheCard = true, processToken, ownerSession) {
|
|
1466
1666
|
return started.answered.then(async (answer) => {
|
|
1667
|
+
if (started.interrupted?.() || started.preparationFailed?.())
|
|
1668
|
+
return;
|
|
1467
1669
|
/* ═══ A RUN THAT STOPPED TO ASK DID NOT DIE, WHATEVER THE HARNESS PRINTED
|
|
1468
1670
|
ON ITS WAY OUT. ═══
|
|
1469
1671
|
|
|
@@ -1575,7 +1777,7 @@ function settle(client, tools, machineId, level, runId, cardId, started, speaksT
|
|
|
1575
1777
|
await ownerSession?.established();
|
|
1576
1778
|
else
|
|
1577
1779
|
await ownerSession?.failed();
|
|
1578
|
-
});
|
|
1780
|
+
}).then(() => { started.completed?.(); });
|
|
1579
1781
|
}
|
|
1580
1782
|
/**
|
|
1581
1783
|
* ═══ ONE RUN, STARTED AGAIN AS ITSELF, WITH WHAT IT WAS SENT AND WHAT IT HAD
|
|
@@ -1639,7 +1841,7 @@ async function resumeRun(client, tools, machineId, runId, afterPid) {
|
|
|
1639
1841
|
both halves of the argument: a machine with no checkout must not take a run
|
|
1640
1842
|
it cannot start, and the level a retry needs the answer for is not known
|
|
1641
1843
|
until the claim returns.
|
|
1642
|
-
|
|
1844
|
+
*
|
|
1643
1845
|
═══ IT IS THE EXISTENCE CHECK AND NOTHING MORE, WHICH IS WHAT CHANGED IN
|
|
1644
1846
|
worktrees-8. ═══ Resolving the working copy now CREATES a branch, a folder
|
|
1645
1847
|
and a row write, and every poll tick that loses the claim race would leave
|
|
@@ -1648,126 +1850,138 @@ async function resumeRun(client, tools, machineId, runId, afterPid) {
|
|
|
1648
1850
|
this position was ever asking, and the copy is made after the claim. */
|
|
1649
1851
|
if (afterPid === null)
|
|
1650
1852
|
checkoutForCodebase(await codebaseOfRun(client, runId), hostname());
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
p_machine_id: machineId,
|
|
1654
|
-
p_after_pid: afterPid,
|
|
1655
|
-
});
|
|
1656
|
-
if (error)
|
|
1657
|
-
throw new Error(`could not start run ${runId} again: ${error.message}`);
|
|
1658
|
-
const claimed = data?.[0];
|
|
1659
|
-
if (!claimed)
|
|
1660
|
-
return null;
|
|
1661
|
-
const level = claimed.run_level === 1 ? 1 : claimed.run_level === 2 ? 2 : 3;
|
|
1662
|
-
if (claimed.run_level !== level) {
|
|
1663
|
-
/* UNREACHABLE, AND STILL SETTLED, exactly as in `startChild`: the level
|
|
1664
|
-
column is checked at three, so there is no level here this cannot spawn.
|
|
1665
|
-
The claim has already happened, so leaving it would strand the run for a
|
|
1666
|
-
whole grace window before anything looked at it again. */
|
|
1667
|
-
const why = `run ${runId} is at level ${claimed.run_level}, which cannot be spawned`;
|
|
1668
|
-
await giveUp(client, runId, why);
|
|
1669
|
-
throw new Error(why);
|
|
1670
|
-
}
|
|
1671
|
-
/* ONE RESOLUTION FOR BOTH PATHS, AFTER THE CLAIM. It used to fork on whether
|
|
1672
|
-
the sweep had already resolved a folder before the claim; since the copy is
|
|
1673
|
-
the card's own and making it writes, both paths make it here, in the branch
|
|
1674
|
-
that can end the run when it cannot be made. */
|
|
1675
|
-
let where;
|
|
1676
|
-
try {
|
|
1677
|
-
where = await workingDirectory(client, runId, level, false);
|
|
1678
|
-
}
|
|
1679
|
-
catch (error) {
|
|
1680
|
-
/* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
|
|
1681
|
-
═══ Constraint 6, and `startRearmed`'s own handling of the same two
|
|
1682
|
-
calls: `worktreeForCard()` is already careful about this and says so;
|
|
1683
|
-
`scratchDir()` is not, because it is `mkdirSync`, whose EACCES and
|
|
1684
|
-
ENOTDIR messages name the directory they failed on. So the machine's own
|
|
1685
|
-
error is kept for stderr and the record is told only what is true and
|
|
1686
|
-
shareable. */
|
|
1687
|
-
const stderrOnly = error instanceof Error ? error.message : String(error);
|
|
1688
|
-
const why = level === 1
|
|
1689
|
-
? 'this machine could not make the empty directory this runs in'
|
|
1690
|
-
: stderrOnly;
|
|
1691
|
-
/* ═══ AND IT ENDS THE WAY A RUN OF THIS LEVEL ENDS. ═══ It was
|
|
1692
|
-
`panel3_give_up` outright, which was right while only a dispatched run
|
|
1693
|
-
could reach this function and is a leak now that the retry brings level
|
|
1694
|
-
1 here: handing the person's message back mints a new run with its
|
|
1695
|
-
attempts at one, which is the bound the retry is under, undone by the
|
|
1696
|
-
one path that could not start. See `endRun`. */
|
|
1697
|
-
await endRun(client, level, runId, claimed.run_card_id, why);
|
|
1698
|
-
throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? stderrOnly : why}`);
|
|
1699
|
-
}
|
|
1700
|
-
/* WHAT IT SENT OTHERS TO DO, FROM THE RECORD. Level 3 has no `dispatch`, so it
|
|
1701
|
-
has no children to have: null says that, where an empty list would say it
|
|
1702
|
-
chose to send nobody.
|
|
1703
|
-
|
|
1704
|
-
═══ AND LEVEL 1 HAS THEM TOO, WHICH ONLY THE RETRY CAN REACH. ═══ This read
|
|
1705
|
-
used to be `level === 2`, which was correct only because nothing at level 1
|
|
1706
|
-
ever got here. Left alone it would hand a coordinator a null child list, and
|
|
1707
|
-
a coordinator that came back to no children dispatches its workers a second
|
|
1708
|
-
time: ux.md's single most expensive failure. `startRearmed` reads it the
|
|
1709
|
-
same way, for the same reason. */
|
|
1710
|
-
const children = level === 3 ? null : await childrenOf(client, runId);
|
|
1711
|
-
/* ═══ READ AGAIN ON EVERY START, WHICH IS THE WHOLE OF CONTRACT POINT 2. ═══
|
|
1712
|
-
A resumed or retried run is handed the rules AS THEY STAND NOW, not as they
|
|
1713
|
-
stood when it first began: a rule edited while the conversation was running
|
|
1714
|
-
governs the rest of it, and a rule deleted while it was running stops
|
|
1715
|
-
applying to it. That is only true because this read happens here rather than
|
|
1716
|
-
once, at the top of the run's life.
|
|
1717
|
-
|
|
1718
|
-
═══ AND IT ENDS THE WAY THIS PATH ALREADY ENDS. ═══ The claim has already
|
|
1719
|
-
happened, so a throw here would leave a run reading `running` with no
|
|
1720
|
-
process. `endRun` with the level fork is this path's own ending (see the cwd
|
|
1721
|
-
branch above for why level 1 may not simply be given up on), and the reason
|
|
1722
|
-
is a `returned()` message naming tables and columns, which carries no local
|
|
1723
|
-
path and is therefore shareable. */
|
|
1724
|
-
let rules;
|
|
1725
|
-
let settings;
|
|
1726
|
-
let pictures;
|
|
1853
|
+
let claimOperationId;
|
|
1854
|
+
const preparation = tools.work?.prepare();
|
|
1727
1855
|
try {
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
const
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
/*
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
it. */
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1856
|
+
const response = await panelClaim(client, tools.work, 'panel3_resume', {
|
|
1857
|
+
p_run_id: runId,
|
|
1858
|
+
p_machine_id: machineId,
|
|
1859
|
+
p_after_pid: afterPid,
|
|
1860
|
+
});
|
|
1861
|
+
const { data, error, operationId } = response;
|
|
1862
|
+
claimOperationId = operationId;
|
|
1863
|
+
if (error)
|
|
1864
|
+
throw new Error(`could not start run ${runId} again: ${error.message}`);
|
|
1865
|
+
const claimed = data?.[0];
|
|
1866
|
+
if (!claimed)
|
|
1867
|
+
return null;
|
|
1868
|
+
const level = claimed.run_level === 1 ? 1 : claimed.run_level === 2 ? 2 : 3;
|
|
1869
|
+
if (claimed.run_level !== level) {
|
|
1870
|
+
/* UNREACHABLE, AND STILL SETTLED, exactly as in `startChild`: the level
|
|
1871
|
+
column is checked at three, so there is no level here this cannot spawn.
|
|
1872
|
+
The claim has already happened, so leaving it would strand the run for a
|
|
1873
|
+
whole grace window before anything looked at it again. */
|
|
1874
|
+
const why = `run ${runId} is at level ${claimed.run_level}, which cannot be spawned`;
|
|
1875
|
+
await giveUp(client, runId, why);
|
|
1876
|
+
throw new Error(why);
|
|
1877
|
+
}
|
|
1878
|
+
/* ONE RESOLUTION FOR BOTH PATHS, AFTER THE CLAIM. It used to fork on whether
|
|
1879
|
+
the sweep had already resolved a folder before the claim; since the copy is
|
|
1880
|
+
the card's own and making it writes, both paths make it here, in the branch
|
|
1881
|
+
that can end the run when it cannot be made. */
|
|
1882
|
+
let where;
|
|
1883
|
+
try {
|
|
1884
|
+
where = await workingDirectory(client, runId, level, false);
|
|
1885
|
+
}
|
|
1886
|
+
catch (error) {
|
|
1887
|
+
/* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
|
|
1888
|
+
═══ Constraint 6, and `startRearmed`'s own handling of the same two
|
|
1889
|
+
calls: `worktreeForCard()` is already careful about this and says so;
|
|
1890
|
+
`scratchDir()` is not, because it is `mkdirSync`, whose EACCES and
|
|
1891
|
+
ENOTDIR messages name the directory they failed on. So the machine's own
|
|
1892
|
+
error is kept for stderr and the record is told only what is true and
|
|
1893
|
+
shareable. */
|
|
1894
|
+
const stderrOnly = error instanceof Error ? error.message : String(error);
|
|
1895
|
+
const why = level === 1
|
|
1896
|
+
? 'this machine could not make the empty directory this runs in'
|
|
1897
|
+
: stderrOnly;
|
|
1898
|
+
/* ═══ AND IT ENDS THE WAY A RUN OF THIS LEVEL ENDS. ═══ It was
|
|
1899
|
+
`panel3_give_up` outright, which was right while only a dispatched run
|
|
1900
|
+
could reach this function and is a leak now that the retry brings level
|
|
1901
|
+
1 here: handing the person's message back mints a new run with its
|
|
1902
|
+
attempts at one, which is the bound the retry is under, undone by the
|
|
1903
|
+
one path that could not start. See `endRun`. */
|
|
1904
|
+
await endRun(client, level, runId, claimed.run_card_id, why);
|
|
1905
|
+
throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? stderrOnly : why}`);
|
|
1906
|
+
}
|
|
1907
|
+
/* WHAT IT SENT OTHERS TO DO, FROM THE RECORD. Level 3 has no `dispatch`, so it
|
|
1908
|
+
has no children to have: null says that, where an empty list would say it
|
|
1909
|
+
chose to send nobody.
|
|
1910
|
+
*
|
|
1911
|
+
═══ AND LEVEL 1 HAS THEM TOO, WHICH ONLY THE RETRY CAN REACH. ═══ This read
|
|
1912
|
+
used to be `level === 2`, which was correct only because nothing at level 1
|
|
1913
|
+
ever got here. Left alone it would hand a coordinator a null child list, and
|
|
1914
|
+
a coordinator that came back to no children dispatches its workers a second
|
|
1915
|
+
time: ux.md's single most expensive failure. `startRearmed` reads it the
|
|
1916
|
+
same way, for the same reason. */
|
|
1917
|
+
const children = level === 3 ? null : await childrenOf(client, runId);
|
|
1918
|
+
/* ═══ READ AGAIN ON EVERY START, WHICH IS THE WHOLE OF CONTRACT POINT 2. ═══
|
|
1919
|
+
A resumed or retried run is handed the rules AS THEY STAND NOW, not as they
|
|
1920
|
+
stood when it first began: a rule edited while the conversation was running
|
|
1921
|
+
governs the rest of it, and a rule deleted while it was running stops
|
|
1922
|
+
applying to it. That is only true because this read happens here rather than
|
|
1923
|
+
once, at the top of the run's life.
|
|
1924
|
+
*
|
|
1925
|
+
═══ AND IT ENDS THE WAY THIS PATH ALREADY ENDS. ═══ The claim has already
|
|
1926
|
+
happened, so a throw here would leave a run reading `running` with no
|
|
1927
|
+
process. `endRun` with the level fork is this path's own ending (see the cwd
|
|
1928
|
+
branch above for why level 1 may not simply be given up on), and the reason
|
|
1929
|
+
is a `returned()` message naming tables and columns, which carries no local
|
|
1930
|
+
path and is therefore shareable. */
|
|
1931
|
+
let rules;
|
|
1932
|
+
let settings;
|
|
1933
|
+
let pictures;
|
|
1934
|
+
try {
|
|
1935
|
+
rules = await standingRulesFor(client, runId);
|
|
1936
|
+
settings = await settingsForRun(client, runId);
|
|
1937
|
+
/* ═══ WRITTEN AGAIN ON EVERY START, LIKE THE RULES. ═══ A resumed process is
|
|
1938
|
+
a NEW process with a new copy of the working directory, so the files a
|
|
1939
|
+
previous one was handed are not there any more, and the stored brief this
|
|
1940
|
+
path replays cannot carry a path that was not known when it was written. */
|
|
1941
|
+
pictures = await picturesOnDisk(client, claimed.run_card_id, where, level);
|
|
1942
|
+
}
|
|
1943
|
+
catch (error) {
|
|
1944
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
1945
|
+
await endRun(client, level, runId, claimed.run_card_id, why);
|
|
1946
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
1947
|
+
}
|
|
1948
|
+
/* ═══ WHY IT DIED IS WHAT DIFFERS, AND IT IS TOLD THE TRUTH ABOUT IT. ═══
|
|
1949
|
+
`resumePrompt` opens by saying the machine went down, which is true of the
|
|
1950
|
+
sweep and false of a retry: the daemon that watched this harness exit is
|
|
1951
|
+
still running. See `retryPrompt`. */
|
|
1952
|
+
const started = await startTrackedAgent(client, runId, preparation, tools.work, withStandingRules(rules, afterPid === null
|
|
1953
|
+
? resumePrompt(claimed.run_brief, claimed.run_report, children)
|
|
1954
|
+
: retryPrompt(claimed.run_brief, claimed.run_report, children), where.block, [], pictures), level, tools.urlFor(runId), where.cwd, undefined, settings);
|
|
1955
|
+
if (started.pid === null) {
|
|
1956
|
+
if (started.interrupted?.())
|
|
1957
|
+
throw new Error('Work was interrupted by the service command.');
|
|
1958
|
+
/* THE CLAIM HAPPENED AND NO PROCESS DID, which is the one shape the record
|
|
1959
|
+
must never be left in quietly. Same handling as a dispatch that could not
|
|
1960
|
+
start: the reason is read off the settled answer and the run is ended with
|
|
1961
|
+
it. */
|
|
1962
|
+
const answer = await started.answered;
|
|
1963
|
+
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
1964
|
+
// The same level fork, for the same reason. See `endRun`.
|
|
1965
|
+
await endRun(client, level, runId, claimed.run_card_id, reason);
|
|
1966
|
+
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
1967
|
+
}
|
|
1968
|
+
out(`resume run ${runId} level ${level} pid ${started.pid} `
|
|
1969
|
+
+ `${claimed.run_report === null ? 'no report to carry' : 'carrying its report'}`);
|
|
1970
|
+
try {
|
|
1971
|
+
await recordProcess(client, runId, started.pid, undefined, undefined, started.attempt);
|
|
1972
|
+
}
|
|
1973
|
+
catch (error) {
|
|
1974
|
+
// Said, not fatal, as everywhere else: the agent is running and killing it
|
|
1975
|
+
// over a bookkeeping write would cost the user the work a second time.
|
|
1976
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1977
|
+
}
|
|
1978
|
+
return { settled: settle(client, tools, machineId, level, runId, claimed.run_card_id, started) };
|
|
1764
1979
|
}
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1980
|
+
finally {
|
|
1981
|
+
preparation?.finish();
|
|
1982
|
+
if (claimOperationId)
|
|
1983
|
+
tools.work?.finishRpcs([claimOperationId]);
|
|
1769
1984
|
}
|
|
1770
|
-
return { settled: settle(client, tools, machineId, level, runId, claimed.run_card_id, started) };
|
|
1771
1985
|
}
|
|
1772
1986
|
/**
|
|
1773
1987
|
* ═══ ONE RUN, STARTED AGAIN BECAUSE SOMETHING IT WAS WAITING ON EXISTS NOW. ═══
|
|
@@ -1796,105 +2010,113 @@ async function resumeRun(client, tools, machineId, runId, afterPid) {
|
|
|
1796
2010
|
* and in `cs show`, rather than quietly hoping the next poll finds it.
|
|
1797
2011
|
*/
|
|
1798
2012
|
async function startRearmed(client, tools, machineId, row) {
|
|
1799
|
-
const
|
|
1800
|
-
if (row.run_level !== level) {
|
|
1801
|
-
const why = `run ${row.run_id} is at level ${row.run_level}, which cannot be spawned`;
|
|
1802
|
-
await giveUp(client, row.run_id, why);
|
|
1803
|
-
throw new Error(why);
|
|
1804
|
-
}
|
|
1805
|
-
let where;
|
|
1806
|
-
try {
|
|
1807
|
-
where = await workingDirectory(client, row.run_id, level, false);
|
|
1808
|
-
}
|
|
1809
|
-
catch (error) {
|
|
1810
|
-
/* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
|
|
1811
|
-
═══ Constraint 6. `workingCopy()` is already careful about this and says
|
|
1812
|
-
so; `scratchDir()` is not — it is `mkdirSync`, whose EACCES and ENOTDIR
|
|
1813
|
-
messages name the directory they failed on — and `failed_because` is a
|
|
1814
|
-
column `cs show` prints. So the machine's own error is kept for stderr
|
|
1815
|
-
and the record is told only what is true and shareable. */
|
|
1816
|
-
const said = error instanceof Error ? error.message : String(error);
|
|
1817
|
-
const why = level === 1
|
|
1818
|
-
? 'this machine could not make the empty directory this runs in'
|
|
1819
|
-
: said;
|
|
1820
|
-
// The level fork, which this path needs for the same reason `resumeRun`'s
|
|
1821
|
-
// two do: a re-arm serves level 1, and a level 1 run holds the person's
|
|
1822
|
-
// message. See `endRun`.
|
|
1823
|
-
await endRun(client, level, row.run_id, row.run_card_id, why);
|
|
1824
|
-
throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? said : why}`);
|
|
1825
|
-
}
|
|
1826
|
-
/* WHAT IT SENT OTHERS TO DO AND WHAT THEY WROTE, FROM THE RECORD, AS LATE AS
|
|
1827
|
-
POSSIBLE. Level 3 has no `dispatch`, so it has no children to have: null
|
|
1828
|
-
says that, where an empty list would say it chose to send nobody. */
|
|
1829
|
-
const children = level === 3 ? null : await childrenOf(client, row.run_id);
|
|
1830
|
-
const deliveredArtifact = row.ask_id !== null && row.mine
|
|
1831
|
-
? await rearmedArtifactAnswer(client, row.ask_id)
|
|
1832
|
-
: null;
|
|
1833
|
-
/* ═══ THE MERGE HAPPENS HERE, OFF THE MARK ON THE ASK, BEFORE THE AGENT THAT
|
|
1834
|
-
WILL SPEAK ABOUT IT IS STARTED. ═══ The re-arm is the last thing that runs
|
|
1835
|
-
before the spawn, which is why it is the only place the outcome can reach
|
|
1836
|
-
the prompt. See `landCardWork`. */
|
|
1837
|
-
const offered = row.ask_id !== null && row.mine
|
|
1838
|
-
? await landingOffer(client, row.ask_id)
|
|
1839
|
-
: null;
|
|
1840
|
-
const landing = offered === null ? null : landCardWork(where, offered);
|
|
1841
|
-
/* ═══ THREE REASONS, AND THE ROW SAYS WHICH. ═══ No question is ux.md's third
|
|
1842
|
-
re-arm: everybody it sent has finished, and it is started to read them back.
|
|
1843
|
-
`children` cannot be null on that path — only a run with children is ever
|
|
1844
|
-
claimed for it — and the prompt takes the list rather than the maybe-list so
|
|
1845
|
-
that is a fact of the signature rather than of a comment. */
|
|
1846
|
-
const prompt = row.ask_id === null
|
|
1847
|
-
? readBackPrompt(row.run_brief, row.run_report, children ?? [])
|
|
1848
|
-
: row.mine
|
|
1849
|
-
? answerPrompt(row.run_brief, row.run_report, children, row.question ?? '', row.answer ?? '', deliveredArtifact, landing)
|
|
1850
|
-
: escalationPrompt(row.run_brief, row.run_report, children, row.ask_id, row.question ?? '');
|
|
1851
|
-
/* CURRENT AT THIS ACTIVATION, exactly as on the resume path, and ended the
|
|
1852
|
-
same way: the re-arm's claim has already happened, so a failure here ends
|
|
1853
|
-
the run with its reason rather than leaving it claimed with no process. */
|
|
1854
|
-
let rules;
|
|
1855
|
-
let settings;
|
|
1856
|
-
let pictures;
|
|
2013
|
+
const preparation = tools.work?.prepare(row.run_id);
|
|
1857
2014
|
try {
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
2015
|
+
const level = row.run_level === 1 ? 1 : row.run_level === 2 ? 2 : 3;
|
|
2016
|
+
if (row.run_level !== level) {
|
|
2017
|
+
const why = `run ${row.run_id} is at level ${row.run_level}, which cannot be spawned`;
|
|
2018
|
+
await giveUp(client, row.run_id, why);
|
|
2019
|
+
throw new Error(why);
|
|
2020
|
+
}
|
|
2021
|
+
let where;
|
|
2022
|
+
try {
|
|
2023
|
+
where = await workingDirectory(client, row.run_id, level, false);
|
|
2024
|
+
}
|
|
2025
|
+
catch (error) {
|
|
2026
|
+
/* ═══ THE REASON IS WRITTEN TO THE DATABASE, SO IT MAY NOT CARRY A PATH.
|
|
2027
|
+
═══ Constraint 6. `workingCopy()` is already careful about this and says
|
|
2028
|
+
so; `scratchDir()` is not — it is `mkdirSync`, whose EACCES and ENOTDIR
|
|
2029
|
+
messages name the directory they failed on — and `failed_because` is a
|
|
2030
|
+
column `cs show` prints. So the machine's own error is kept for stderr
|
|
2031
|
+
and the record is told only what is true and shareable. */
|
|
2032
|
+
const said = error instanceof Error ? error.message : String(error);
|
|
2033
|
+
const why = level === 1
|
|
2034
|
+
? 'this machine could not make the empty directory this runs in'
|
|
2035
|
+
: said;
|
|
2036
|
+
// The level fork, which this path needs for the same reason `resumeRun`'s
|
|
2037
|
+
// two do: a re-arm serves level 1, and a level 1 run holds the person's
|
|
2038
|
+
// message. See `endRun`.
|
|
2039
|
+
await endRun(client, level, row.run_id, row.run_card_id, why);
|
|
2040
|
+
throw new Error(`NO AGENT IS RUNNING: ${level === 1 ? said : why}`);
|
|
2041
|
+
}
|
|
2042
|
+
/* WHAT IT SENT OTHERS TO DO AND WHAT THEY WROTE, FROM THE RECORD, AS LATE AS
|
|
2043
|
+
POSSIBLE. Level 3 has no `dispatch`, so it has no children to have: null
|
|
2044
|
+
says that, where an empty list would say it chose to send nobody. */
|
|
2045
|
+
const children = level === 3 ? null : await childrenOf(client, row.run_id);
|
|
2046
|
+
const deliveredArtifact = row.ask_id !== null && row.mine
|
|
2047
|
+
? await rearmedArtifactAnswer(client, row.ask_id)
|
|
2048
|
+
: null;
|
|
2049
|
+
/* ═══ THE MERGE HAPPENS HERE, OFF THE MARK ON THE ASK, BEFORE THE AGENT THAT
|
|
2050
|
+
WILL SPEAK ABOUT IT IS STARTED. ═══ The re-arm is the last thing that runs
|
|
2051
|
+
before the spawn, which is why it is the only place the outcome can reach
|
|
2052
|
+
the prompt. See `landCardWork`. */
|
|
2053
|
+
const offered = row.ask_id !== null && row.mine
|
|
2054
|
+
? await landingOffer(client, row.ask_id)
|
|
2055
|
+
: null;
|
|
2056
|
+
const landing = offered === null ? null : landCardWork(where, offered);
|
|
2057
|
+
/* ═══ THREE REASONS, AND THE ROW SAYS WHICH. ═══ No question is ux.md's third
|
|
2058
|
+
re-arm: everybody it sent has finished, and it is started to read them back.
|
|
2059
|
+
`children` cannot be null on that path — only a run with children is ever
|
|
2060
|
+
claimed for it — and the prompt takes the list rather than the maybe-list so
|
|
2061
|
+
that is a fact of the signature rather than of a comment. */
|
|
2062
|
+
const prompt = row.ask_id === null
|
|
2063
|
+
? readBackPrompt(row.run_brief, row.run_report, children ?? [])
|
|
1878
2064
|
: row.mine
|
|
1879
|
-
? '
|
|
1880
|
-
:
|
|
1881
|
-
|
|
1882
|
-
|
|
2065
|
+
? answerPrompt(row.run_brief, row.run_report, children, row.question ?? '', row.answer ?? '', deliveredArtifact, landing)
|
|
2066
|
+
: escalationPrompt(row.run_brief, row.run_report, children, row.ask_id, row.question ?? '');
|
|
2067
|
+
/* CURRENT AT THIS ACTIVATION, exactly as on the resume path, and ended the
|
|
2068
|
+
same way: the re-arm's claim has already happened, so a failure here ends
|
|
2069
|
+
the run with its reason rather than leaving it claimed with no process. */
|
|
2070
|
+
let rules;
|
|
2071
|
+
let settings;
|
|
2072
|
+
let pictures;
|
|
2073
|
+
try {
|
|
2074
|
+
rules = await standingRulesFor(client, row.run_id);
|
|
2075
|
+
settings = await settingsForRun(client, row.run_id);
|
|
2076
|
+
pictures = await picturesOnDisk(client, row.run_card_id, where, level);
|
|
2077
|
+
}
|
|
2078
|
+
catch (error) {
|
|
2079
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
2080
|
+
await endRun(client, level, row.run_id, row.run_card_id, why);
|
|
2081
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
2082
|
+
}
|
|
2083
|
+
const started = await startTrackedAgent(client, row.run_id, preparation, tools.work, withStandingRules(rules, prompt, where.block, [], pictures), level, tools.urlFor(row.run_id), where.cwd, undefined, settings);
|
|
2084
|
+
if (started.pid === null) {
|
|
2085
|
+
if (started.interrupted?.())
|
|
2086
|
+
throw new Error('Work was interrupted by the service command.');
|
|
2087
|
+
const answer = await started.answered;
|
|
2088
|
+
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
2089
|
+
// The same level fork, for the same reason. See `endRun`.
|
|
2090
|
+
await endRun(client, level, row.run_id, row.run_card_id, reason);
|
|
2091
|
+
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
2092
|
+
}
|
|
2093
|
+
out(`rearm run ${row.run_id} level ${level} pid ${started.pid} `
|
|
2094
|
+
+ `${row.ask_id === null
|
|
2095
|
+
? 'to read back everybody it sent'
|
|
2096
|
+
: row.mine
|
|
2097
|
+
? 'with the answer to its own question'
|
|
2098
|
+
: 'with a question it has to settle'}`);
|
|
2099
|
+
try {
|
|
2100
|
+
await recordProcess(client, row.run_id, started.pid, undefined, undefined, started.attempt);
|
|
2101
|
+
}
|
|
2102
|
+
catch (error) {
|
|
2103
|
+
// Said, not fatal, as everywhere else: the agent is running and killing it
|
|
2104
|
+
// over a bookkeeping write would cost the user the work.
|
|
2105
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
2106
|
+
}
|
|
2107
|
+
/* ═══ AND IT SPEAKS TO THE CARD ONLY IF IT IS DOING THE PERSON'S WORK. ═══
|
|
2108
|
+
Exactly the distinction ux.md draws: a run carrying on with its own work has
|
|
2109
|
+
something to say when it finishes, and a run started only to settle somebody
|
|
2110
|
+
else's question does not. A run started to read back everybody it sent is
|
|
2111
|
+
the first kind and the clearest case of it — that reply IS the answer to the
|
|
2112
|
+
request. See `settle`. */
|
|
2113
|
+
return {
|
|
2114
|
+
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, row.ask_id === null || !!row.mine),
|
|
2115
|
+
};
|
|
2116
|
+
}
|
|
2117
|
+
finally {
|
|
2118
|
+
preparation?.finish();
|
|
1883
2119
|
}
|
|
1884
|
-
catch (error) {
|
|
1885
|
-
// Said, not fatal, as everywhere else: the agent is running and killing it
|
|
1886
|
-
// over a bookkeeping write would cost the user the work.
|
|
1887
|
-
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
1888
|
-
}
|
|
1889
|
-
/* ═══ AND IT SPEAKS TO THE CARD ONLY IF IT IS DOING THE PERSON'S WORK. ═══
|
|
1890
|
-
Exactly the distinction ux.md draws: a run carrying on with its own work has
|
|
1891
|
-
something to say when it finishes, and a run started only to settle somebody
|
|
1892
|
-
else's question does not. A run started to read back everybody it sent is
|
|
1893
|
-
the first kind and the clearest case of it — that reply IS the answer to the
|
|
1894
|
-
request. See `settle`. */
|
|
1895
|
-
return {
|
|
1896
|
-
settled: settle(client, tools, machineId, level, row.run_id, row.run_card_id, started, row.ask_id === null || !!row.mine),
|
|
1897
|
-
};
|
|
1898
2120
|
}
|
|
1899
2121
|
/**
|
|
1900
2122
|
* The runs one run dispatched, AND WHAT EACH OF THEM WROTE, in the words the
|
|
@@ -2048,139 +2270,151 @@ export function resumableOwnerSessionId(candidate, machineId, machineHarness) {
|
|
|
2048
2270
|
: undefined;
|
|
2049
2271
|
}
|
|
2050
2272
|
async function activateOwner(client, tools, machineId, runId, afterProcessToken = null, afterPid = null) {
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
return null;
|
|
2054
|
-
// Existing conversations retain their agent; an explicit hand-back follows
|
|
2055
|
-
// the new selection. Never reuse a native session across different agents.
|
|
2056
|
-
if (candidate.handed_back_at === null && candidate.machine_id !== machineId)
|
|
2057
|
-
return null;
|
|
2058
|
-
const machineHarness = candidate.handed_back_at === null && candidate.harness !== null
|
|
2059
|
-
? harness({ CTRL_SPC_V3_AGENT: candidate.harness })
|
|
2060
|
-
: await selectedHarness(client, machineId);
|
|
2061
|
-
const resumeSessionId = resumableOwnerSessionId(candidate, machineId, machineHarness);
|
|
2062
|
-
/* ═══ THE EXISTENCE CHECK BEFORE THE CLAIM, AND THE COPY AFTER IT. ═══ This
|
|
2063
|
-
was the whole resolution, which was right while resolving meant reading a
|
|
2064
|
-
folder out of a file. Since worktrees-8 it also CREATES one, and a daemon
|
|
2065
|
-
that lost the activation race would leave a branch and a folder behind for
|
|
2066
|
-
an owner it never activated. The refusal this position exists for — a
|
|
2067
|
-
machine that does not have this codebase must not take the activation —
|
|
2068
|
-
is unchanged, because it is the located checkout that answers it. */
|
|
2069
|
-
if (candidate.codebase_id !== null) {
|
|
2070
|
-
checkoutForCodebase(await codebaseOfRun(client, candidate.id), hostname());
|
|
2071
|
-
}
|
|
2072
|
-
const { data, error } = await client.rpc('panel3_take_owner_activation', {
|
|
2073
|
-
p_run_id: runId,
|
|
2074
|
-
p_machine_id: machineId,
|
|
2075
|
-
p_agent: machineHarness,
|
|
2076
|
-
p_after_process_token: afterProcessToken,
|
|
2077
|
-
p_after_pid: afterPid,
|
|
2078
|
-
});
|
|
2079
|
-
if (error)
|
|
2080
|
-
throw new Error(`could not activate conversation owner ${runId}: ${error.message}`);
|
|
2081
|
-
const claimed = data?.[0];
|
|
2082
|
-
if (!claimed)
|
|
2083
|
-
return null;
|
|
2084
|
-
const [events, children] = await Promise.all([
|
|
2085
|
-
ownerConversation(client, claimed.run_card_id, new Set(claimed.turn_ids ?? [])),
|
|
2086
|
-
ownerChildren(client, runId),
|
|
2087
|
-
]);
|
|
2088
|
-
const currentArtifactAnswer = deliveredArtifactAnswer(events, claimed);
|
|
2089
|
-
const delivered = claimed.ask_id === null ? null : {
|
|
2090
|
-
id: claimed.ask_id,
|
|
2091
|
-
question: claimed.question ?? '(question unavailable)',
|
|
2092
|
-
answer: claimed.answer,
|
|
2093
|
-
mine: claimed.mine === true,
|
|
2094
|
-
artifactAnswer: currentArtifactAnswer,
|
|
2095
|
-
};
|
|
2096
|
-
/* ═══ AFTER THE CLAIM, SO THE ENDING IS THE ONE THIS PATH HAS. ═══ Every
|
|
2097
|
-
failure below the claim ends the activation with `giveUp` and its process
|
|
2098
|
-
token; a throw above it would merely be an activation that did not happen.
|
|
2099
|
-
|
|
2100
|
-
═══ AND IT MATTERS MOST HERE. ═══ This is the owner, which lives for the
|
|
2101
|
-
whole card and whose native session is RESUMED, so it is the one agent that
|
|
2102
|
-
can be running while a person edits or deletes a rule. Reading at every
|
|
2103
|
-
activation is what makes an edit govern the rest of the conversation, and
|
|
2104
|
-
the block's own supersession sentence is what makes a DELETION take effect
|
|
2105
|
-
in a session that still holds the older copy.
|
|
2106
|
-
|
|
2107
|
-
═══ AND IT IS BEFORE THE PROMPT SINCE worktrees-8 C1, because the prompt now
|
|
2108
|
-
says what the product DID with the person's answer, and the landing needs
|
|
2109
|
-
the card's copy. Nothing in the prompt depended on it before. */
|
|
2110
|
-
let rules;
|
|
2111
|
-
let settings;
|
|
2112
|
-
/* THE CARD'S COPY IS MADE IN THE SAME WINDOW AND UNDER THE SAME ENDING, for
|
|
2113
|
-
the reason the check above gives: it writes, so it happens after the claim,
|
|
2114
|
-
and a failure to make it is an activation that ends rather than one that
|
|
2115
|
-
sits `running` with no process. */
|
|
2116
|
-
let where;
|
|
2117
|
-
/* ═══ AND WHAT IS ATTACHED, READ IN THE SAME WINDOW AND UNDER THE SAME
|
|
2118
|
-
ENDING. ═══ attaching-after-the-fact-10: a person may attach to a card that
|
|
2119
|
-
is already running, and the owner's brief is immutable, so the only account
|
|
2120
|
-
of attachments it would otherwise get is the one frozen at dispatch. Read
|
|
2121
|
-
at every activation, exactly as the rules are, and for the same reason a
|
|
2122
|
-
failed read ends the activation rather than continuing: "nothing is
|
|
2123
|
-
attached" is a different card from the one the person sent.
|
|
2124
|
-
|
|
2125
|
-
THE CODEBASE LINES ARE DROPPED. `whatWasAttached` partitions those into a
|
|
2126
|
-
section whose own text says "its one codebase is named separately below",
|
|
2127
|
-
a forward reference to something `workBrief` supplies and an owner
|
|
2128
|
-
activation does not. This block carries the PERSON's attachments; where the
|
|
2129
|
-
owner is working is `where.block`'s answer. */
|
|
2130
|
-
let attached;
|
|
2131
|
-
let pictures;
|
|
2273
|
+
let claimOperationId;
|
|
2274
|
+
const preparation = tools.work?.prepare();
|
|
2132
2275
|
try {
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2276
|
+
const candidate = await ownerCandidate(client, runId);
|
|
2277
|
+
if (!candidate)
|
|
2278
|
+
return null;
|
|
2279
|
+
// Existing conversations retain their agent; an explicit hand-back follows
|
|
2280
|
+
// the new selection. Never reuse a native session across different agents.
|
|
2281
|
+
if (candidate.handed_back_at === null && candidate.machine_id !== machineId)
|
|
2282
|
+
return null;
|
|
2283
|
+
const machineHarness = candidate.handed_back_at === null && candidate.harness !== null
|
|
2284
|
+
? harness({ CTRL_SPC_V3_AGENT: candidate.harness })
|
|
2285
|
+
: await selectedHarness(client, machineId);
|
|
2286
|
+
const resumeSessionId = resumableOwnerSessionId(candidate, machineId, machineHarness);
|
|
2287
|
+
/* ═══ THE EXISTENCE CHECK BEFORE THE CLAIM, AND THE COPY AFTER IT. ═══ This
|
|
2288
|
+
was the whole resolution, which was right while resolving meant reading a
|
|
2289
|
+
folder out of a file. Since worktrees-8 it also CREATES one, and a daemon
|
|
2290
|
+
that lost the activation race would leave a branch and a folder behind for
|
|
2291
|
+
an owner it never activated. The refusal this position exists for — a
|
|
2292
|
+
machine that does not have this codebase must not take the activation —
|
|
2293
|
+
is unchanged, because it is the located checkout that answers it. */
|
|
2294
|
+
if (candidate.codebase_id !== null) {
|
|
2295
|
+
checkoutForCodebase(await codebaseOfRun(client, candidate.id), hostname());
|
|
2296
|
+
}
|
|
2297
|
+
const response = await panelClaim(client, tools.work, 'panel3_take_owner_activation', {
|
|
2298
|
+
p_run_id: runId,
|
|
2299
|
+
p_machine_id: machineId,
|
|
2300
|
+
p_agent: machineHarness,
|
|
2301
|
+
p_after_process_token: afterProcessToken,
|
|
2302
|
+
p_after_pid: afterPid,
|
|
2303
|
+
});
|
|
2304
|
+
const { data, error, operationId } = response;
|
|
2305
|
+
claimOperationId = operationId;
|
|
2306
|
+
if (error)
|
|
2307
|
+
throw new Error(`could not activate conversation owner ${runId}: ${error.message}`);
|
|
2308
|
+
const claimed = data?.[0];
|
|
2309
|
+
if (!claimed)
|
|
2310
|
+
return null;
|
|
2311
|
+
const [events, children] = await Promise.all([
|
|
2312
|
+
ownerConversation(client, claimed.run_card_id, new Set(claimed.turn_ids ?? [])),
|
|
2313
|
+
ownerChildren(client, runId),
|
|
2314
|
+
]);
|
|
2315
|
+
const currentArtifactAnswer = deliveredArtifactAnswer(events, claimed);
|
|
2316
|
+
const delivered = claimed.ask_id === null ? null : {
|
|
2317
|
+
id: claimed.ask_id,
|
|
2318
|
+
question: claimed.question ?? '(question unavailable)',
|
|
2319
|
+
answer: claimed.answer,
|
|
2320
|
+
mine: claimed.mine === true,
|
|
2321
|
+
artifactAnswer: currentArtifactAnswer,
|
|
2322
|
+
};
|
|
2323
|
+
/* ═══ AFTER THE CLAIM, SO THE ENDING IS THE ONE THIS PATH HAS. ═══ Every
|
|
2324
|
+
failure below the claim ends the activation with `giveUp` and its process
|
|
2325
|
+
token; a throw above it would merely be an activation that did not happen.
|
|
2326
|
+
*
|
|
2327
|
+
═══ AND IT MATTERS MOST HERE. ═══ This is the owner, which lives for the
|
|
2328
|
+
whole card and whose native session is RESUMED, so it is the one agent that
|
|
2329
|
+
can be running while a person edits or deletes a rule. Reading at every
|
|
2330
|
+
activation is what makes an edit govern the rest of the conversation, and
|
|
2331
|
+
the block's own supersession sentence is what makes a DELETION take effect
|
|
2332
|
+
in a session that still holds the older copy.
|
|
2333
|
+
*
|
|
2334
|
+
═══ AND IT IS BEFORE THE PROMPT SINCE worktrees-8 C1, because the prompt now
|
|
2335
|
+
says what the product DID with the person's answer, and the landing needs
|
|
2336
|
+
the card's copy. Nothing in the prompt depended on it before. */
|
|
2337
|
+
let rules;
|
|
2338
|
+
let settings;
|
|
2339
|
+
/* THE CARD'S COPY IS MADE IN THE SAME WINDOW AND UNDER THE SAME ENDING, for
|
|
2340
|
+
the reason the check above gives: it writes, so it happens after the claim,
|
|
2341
|
+
and a failure to make it is an activation that ends rather than one that
|
|
2342
|
+
sits `running` with no process. */
|
|
2343
|
+
let where;
|
|
2344
|
+
/* ═══ AND WHAT IS ATTACHED, READ IN THE SAME WINDOW AND UNDER THE SAME
|
|
2345
|
+
ENDING. ═══ attaching-after-the-fact-10: a person may attach to a card that
|
|
2346
|
+
is already running, and the owner's brief is immutable, so the only account
|
|
2347
|
+
of attachments it would otherwise get is the one frozen at dispatch. Read
|
|
2348
|
+
at every activation, exactly as the rules are, and for the same reason a
|
|
2349
|
+
failed read ends the activation rather than continuing: "nothing is
|
|
2350
|
+
attached" is a different card from the one the person sent.
|
|
2351
|
+
*
|
|
2352
|
+
THE CODEBASE LINES ARE DROPPED. `whatWasAttached` partitions those into a
|
|
2353
|
+
section whose own text says "its one codebase is named separately below",
|
|
2354
|
+
a forward reference to something `workBrief` supplies and an owner
|
|
2355
|
+
activation does not. This block carries the PERSON's attachments; where the
|
|
2356
|
+
owner is working is `where.block`'s answer. */
|
|
2357
|
+
let attached;
|
|
2358
|
+
let pictures;
|
|
2359
|
+
try {
|
|
2360
|
+
where = await ownerDirectory(client, candidate);
|
|
2361
|
+
rules = await standingRulesFor(client, runId);
|
|
2362
|
+
settings = await settingsForRun(client, runId);
|
|
2363
|
+
attached = whatWasAttached((await attachmentsFor(client, claimed.run_card_id))
|
|
2364
|
+
.filter((line) => !line.startsWith('codebase ')));
|
|
2365
|
+
/* ═══ THE OWNER'S OWN ACTIVATION, WHICH IS WHERE MOST PICTURES ARRIVE. ═══
|
|
2366
|
+
The person sends one and this is the run that reads it. Written on every
|
|
2367
|
+
activation rather than once, so an owner resumed into an existing native
|
|
2368
|
+
conversation is told again about a directory a previous process wrote. */
|
|
2369
|
+
pictures = await picturesOnDisk(client, claimed.run_card_id, where, 2);
|
|
2370
|
+
}
|
|
2371
|
+
catch (error) {
|
|
2372
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
2373
|
+
await giveUp(client, runId, why, claimed.process_token);
|
|
2374
|
+
throw new Error(`NO AGENT IS RUNNING: ${why}`);
|
|
2375
|
+
}
|
|
2376
|
+
/* THE MERGE, OFF THE MARK ON THE ASK, BEFORE THE AGENT THAT WILL SPEAK ABOUT
|
|
2377
|
+
IT IS STARTED. The owner reaches an answer by this route as often as by the
|
|
2378
|
+
re-arm, which is why the artifact answer is read on both and this is too. */
|
|
2379
|
+
const offered = claimed.ask_id !== null && claimed.mine === true
|
|
2380
|
+
? await landingOffer(client, claimed.ask_id)
|
|
2381
|
+
: null;
|
|
2382
|
+
const landing = offered === null ? null : landCardWork(where, offered);
|
|
2161
2383
|
/* ═══ A PROCESS OF ITS OWN ENDED BEFORE IT FINISHED. ═══ `afterPid` is the
|
|
2162
2384
|
fact, and it is non-null on all three paths that follow one: a harness
|
|
2163
2385
|
that crashed, a machine that went down, and now a person's correction.
|
|
2164
2386
|
The sentences it adds say what to do and never why, because those three
|
|
2165
2387
|
are not the same event and `prompt.ts` exists to stop an agent being
|
|
2166
2388
|
told an untrue reason for its own restart. */
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
await
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2389
|
+
const prompt = resumeSessionId
|
|
2390
|
+
? ownerContinuationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered, landing)
|
|
2391
|
+
: ownerActivationPrompt(claimed.run_brief, claimed.run_report, events, children, delivered !== null && !delivered.mine
|
|
2392
|
+
? { id: delivered.id, question: delivered.question }
|
|
2393
|
+
: null, currentArtifactAnswer, afterPid !== null, landing);
|
|
2394
|
+
const started = await startTrackedAgent(client, runId, preparation, tools.work, withStandingRules(rules, prompt, where.block, attached, pictures), 2, tools.urlFor(runId, claimed.process_token), where.cwd, { ownerId: runId, ...(resumeSessionId ? { resumeSessionId } : {}) }, settings);
|
|
2395
|
+
if (started.pid === null) {
|
|
2396
|
+
if (started.interrupted?.())
|
|
2397
|
+
throw new Error('Work was interrupted by the service command.');
|
|
2398
|
+
const answer = await started.answered;
|
|
2399
|
+
const reason = answer.ok ? 'the process ended before it could be identified' : answer.reason;
|
|
2400
|
+
await giveUp(client, runId, reason, claimed.process_token);
|
|
2401
|
+
throw new Error(`NO AGENT IS RUNNING: ${reason}`);
|
|
2402
|
+
}
|
|
2403
|
+
try {
|
|
2404
|
+
await recordProcess(client, runId, started.pid, undefined, claimed.process_token, started.attempt);
|
|
2405
|
+
}
|
|
2406
|
+
catch (error) {
|
|
2407
|
+
said(`${error instanceof Error ? error.message : String(error)} (the run is still going)`);
|
|
2408
|
+
}
|
|
2409
|
+
return {
|
|
2410
|
+
settled: settle(client, tools, machineId, 2, runId, claimed.run_card_id, started, true, claimed.process_token, ownerSessionLifecycle(started, runId, machineHarness, claimed.process_token, resumeSessionId)),
|
|
2411
|
+
};
|
|
2177
2412
|
}
|
|
2178
|
-
|
|
2179
|
-
|
|
2413
|
+
finally {
|
|
2414
|
+
preparation?.finish();
|
|
2415
|
+
if (claimOperationId)
|
|
2416
|
+
tools.work?.finishRpcs([claimOperationId]);
|
|
2180
2417
|
}
|
|
2181
|
-
return {
|
|
2182
|
-
settled: settle(client, tools, machineId, 2, runId, claimed.run_card_id, started, true, claimed.process_token, ownerSessionLifecycle(started, runId, machineHarness, claimed.process_token, resumeSessionId)),
|
|
2183
|
-
};
|
|
2184
2418
|
}
|
|
2185
2419
|
/** The only owner rows the normal poll may try, including an explicit hand-back move.
|
|
2186
2420
|
*
|
|
@@ -2256,7 +2490,8 @@ export function redirectedProcess(candidate, machineId,
|
|
|
2256
2490
|
/** The newest unaddressed person turn per card. MAX, never first-seen: a turn
|
|
2257
2491
|
* left unaddressed while a question was open would otherwise pin its card
|
|
2258
2492
|
* below `resumed_at` for good and nothing on it could ever redirect. */
|
|
2259
|
-
waiting, cardId, booted) {
|
|
2493
|
+
waiting, cardId, booted, executionHeld = false) {
|
|
2494
|
+
// A dead transport can still own live tools in the durable execution boundary.
|
|
2260
2495
|
// The claim's own five, mirrored.
|
|
2261
2496
|
if (candidate.state !== 'running' || candidate.ended_at !== null)
|
|
2262
2497
|
return null;
|
|
@@ -2288,7 +2523,7 @@ waiting, cardId, booted) {
|
|
|
2288
2523
|
if (new Date(said).getTime() <= new Date(candidate.resumed_at ?? candidate.started_at).getTime()) {
|
|
2289
2524
|
return null;
|
|
2290
2525
|
}
|
|
2291
|
-
return runProcessIsAlive(candidate, booted) ? candidate.pid : null;
|
|
2526
|
+
return executionHeld || runProcessIsAlive(candidate, booted) ? candidate.pid : null;
|
|
2292
2527
|
}
|
|
2293
2528
|
export function pendingOwnerSessionWithinGrace(candidate, now = Date.now()) {
|
|
2294
2529
|
const local = readOwnerSession(candidate.id);
|
|
@@ -2331,9 +2566,17 @@ async function takeOwnerActivations(client, tools, machineId, mine, hold) {
|
|
|
2331
2566
|
else and there is a dead process, a row still reading `running` and a card
|
|
2332
2567
|
still saying `working`, with nothing local to retry it. */
|
|
2333
2568
|
if (mine.has(candidate.id)) {
|
|
2334
|
-
const
|
|
2569
|
+
const attempt = recoveryAttempt(candidate);
|
|
2570
|
+
const executionHeld = tools.work?.heldAttempt(attempt) ?? false;
|
|
2571
|
+
// A run ID alone cannot authorize stopping a newer activation.
|
|
2572
|
+
if (tools.work && !executionHeld)
|
|
2573
|
+
continue;
|
|
2574
|
+
const pid = redirectedProcess({ ...candidate, card_state: candidate.card?.state ?? null }, machineId, waiting, candidate.card_id, bootedAt(), executionHeld);
|
|
2335
2575
|
if (pid !== null) {
|
|
2336
|
-
|
|
2576
|
+
if (executionHeld)
|
|
2577
|
+
await tools.work.stopHeldAttempt(attempt);
|
|
2578
|
+
else
|
|
2579
|
+
killTree({ pid, kill: (signal) => process.kill(pid, signal) });
|
|
2337
2580
|
/* ═══ SIGNALLED, NOT KILLED, AND THE WORD IS THE POINT. ═══ `killTree`
|
|
2338
2581
|
swallows a refused signal on both platforms, so saying "killed" would
|
|
2339
2582
|
claim a death this daemon never observed. A process that survives is
|
|
@@ -2344,6 +2587,8 @@ async function takeOwnerActivations(client, tools, machineId, mine, hold) {
|
|
|
2344
2587
|
}
|
|
2345
2588
|
continue;
|
|
2346
2589
|
}
|
|
2590
|
+
if (tools.work?.heldAttempt(recoveryAttempt(candidate)))
|
|
2591
|
+
continue;
|
|
2347
2592
|
if (pendingOwnerSessionWithinGrace(candidate))
|
|
2348
2593
|
continue;
|
|
2349
2594
|
if (candidate.handed_back_at !== null
|
|
@@ -2407,7 +2652,7 @@ function runProcessIsAlive(run, booted) {
|
|
|
2407
2652
|
* touched here. A current owner row, live PID, in-flight process, or fresh
|
|
2408
2653
|
* pid-null claim always defers cleanup. Stable state is removed only after the
|
|
2409
2654
|
* owner row disappears, becomes terminal, or no longer owns its card. */
|
|
2410
|
-
export async function reconcileOwnerSessions(client, machineId, _machineHarness, inFlightOwnerIds = new Set()) {
|
|
2655
|
+
export async function reconcileOwnerSessions(client, machineId, _machineHarness, inFlightOwnerIds = new Set(), work) {
|
|
2411
2656
|
const mappingIds = listOwnerSessionIds();
|
|
2412
2657
|
const homeIds = listPanel3CodexOwnerHomeIds();
|
|
2413
2658
|
const all = [...new Set([...mappingIds, ...homeIds])];
|
|
@@ -2424,7 +2669,11 @@ export async function reconcileOwnerSessions(client, machineId, _machineHarness,
|
|
|
2424
2669
|
for (const id of ids) {
|
|
2425
2670
|
if (inFlightOwnerIds.has(id))
|
|
2426
2671
|
continue;
|
|
2672
|
+
if (work?.heldLocalOwner(id))
|
|
2673
|
+
continue;
|
|
2427
2674
|
const row = byId.get(id);
|
|
2675
|
+
if (row && work?.heldAttempt(recoveryAttempt(row)))
|
|
2676
|
+
continue;
|
|
2428
2677
|
const localProcessInUse = !!row
|
|
2429
2678
|
&& row.machine_id === machineId
|
|
2430
2679
|
&& ((row.pid !== null && processIsAlive(row.pid))
|
|
@@ -2477,10 +2726,10 @@ export async function reconcileOwnerSessions(client, machineId, _machineHarness,
|
|
|
2477
2726
|
* the next poll tries again: clearing it after an EPERM would say this machine
|
|
2478
2727
|
* has no process for a run whose agent is still working.
|
|
2479
2728
|
*/
|
|
2480
|
-
async function killStopped(client, machineId) {
|
|
2729
|
+
async function killStopped(client, machineId, work) {
|
|
2481
2730
|
const stopped = await returned(client
|
|
2482
2731
|
.from('panel3_runs')
|
|
2483
|
-
.select('id, card_id, pid, state, started_at, resumed_at')
|
|
2732
|
+
.select('id, card_id, pid, state, started_at, resumed_at, process_token')
|
|
2484
2733
|
.eq('machine_id', machineId)
|
|
2485
2734
|
.in('state', [...ENDED_BY_THE_PERSON, 'finished', 'failed'])
|
|
2486
2735
|
.not('pid', 'is', null), 'read', 'the runs on this machine that are not coming back');
|
|
@@ -2488,7 +2737,16 @@ async function killStopped(client, machineId) {
|
|
|
2488
2737
|
return;
|
|
2489
2738
|
const booted = bootedAt();
|
|
2490
2739
|
for (const run of stopped) {
|
|
2491
|
-
|
|
2740
|
+
let stoppedByOwner = false;
|
|
2741
|
+
if (work?.heldAttempt(recoveryAttempt(run))) {
|
|
2742
|
+
if (!ENDED_BY_THE_PERSON.includes(run.state))
|
|
2743
|
+
continue;
|
|
2744
|
+
// The recorded bridge can be gone while its owned tools still run.
|
|
2745
|
+
// Only the durable owner can confirm that the complete execution ended.
|
|
2746
|
+
await work.stopHeldAttempt(recoveryAttempt(run));
|
|
2747
|
+
stoppedByOwner = true;
|
|
2748
|
+
}
|
|
2749
|
+
if (!stoppedByOwner && runProcessIsAlive(run, booted)) {
|
|
2492
2750
|
if (!ENDED_BY_THE_PERSON.includes(run.state))
|
|
2493
2751
|
continue;
|
|
2494
2752
|
try {
|
|
@@ -2509,15 +2767,18 @@ async function killStopped(client, machineId) {
|
|
|
2509
2767
|
continue;
|
|
2510
2768
|
}
|
|
2511
2769
|
try {
|
|
2512
|
-
|
|
2770
|
+
let cleared = client
|
|
2513
2771
|
.from('panel3_runs')
|
|
2514
2772
|
.update({ pid: null })
|
|
2515
|
-
// A finished owner may activate again between this read and write.
|
|
2516
|
-
// Only acknowledge the exact process whose exit was observed.
|
|
2517
2773
|
.eq('id', run.id)
|
|
2518
2774
|
.eq('pid', run.pid)
|
|
2519
|
-
.
|
|
2520
|
-
.
|
|
2775
|
+
.eq('started_at', run.started_at)
|
|
2776
|
+
.in('state', [...ENDED_BY_THE_PERSON, 'finished', 'failed']);
|
|
2777
|
+
cleared = run.process_token === null ? cleared.is('process_token', null) : cleared.eq('process_token', run.process_token);
|
|
2778
|
+
cleared = run.resumed_at === null ? cleared.is('resumed_at', null) : cleared.eq('resumed_at', run.resumed_at);
|
|
2779
|
+
await returned(
|
|
2780
|
+
// A finished owner may activate again while its old execution closes.
|
|
2781
|
+
cleared.select('id'), 'clear the process id of', `run ${run.id}`);
|
|
2521
2782
|
}
|
|
2522
2783
|
catch (error) {
|
|
2523
2784
|
// Said, not fatal. The kill has already happened; this is bookkeeping, and
|
|
@@ -2602,6 +2863,8 @@ async function recoverStranded(client, tools, machineId, mine, hold) {
|
|
|
2602
2863
|
for (const run of live) {
|
|
2603
2864
|
if (mine.has(run.id))
|
|
2604
2865
|
continue;
|
|
2866
|
+
if (tools.work?.heldAttempt(recoveryAttempt(run)))
|
|
2867
|
+
continue;
|
|
2605
2868
|
// WHEN THE ATTEMPT NOW RUNNING BEGAN, which is the first one until a resume
|
|
2606
2869
|
// says otherwise. See the header.
|
|
2607
2870
|
const startedAt = new Date(run.resumed_at ?? run.started_at).getTime();
|
|
@@ -2838,6 +3101,8 @@ export async function takeHandedBack(client, tools, machineId, mine, hold) {
|
|
|
2838
3101
|
// ONCE, OUTSIDE THE LOOP. It is a property of this machine, not of a row.
|
|
2839
3102
|
const booted = bootedAt();
|
|
2840
3103
|
for (const run of offered) {
|
|
3104
|
+
if (tools.work?.heldAttempt(recoveryAttempt(run)))
|
|
3105
|
+
continue;
|
|
2841
3106
|
if (mine.has(run.id)) {
|
|
2842
3107
|
/* THIS DAEMON'S OWN LIVE WORK, OFFERED WHILE IT WAS BUSY BEING QUIET. See
|
|
2843
3108
|
the header. Said rather than passed over in silence, because a machine
|
|
@@ -3028,7 +3293,7 @@ export async function sweepFinishedWorktrees(client) {
|
|
|
3028
3293
|
export function clientReader(injected) {
|
|
3029
3294
|
return typeof injected === 'function' ? injected : () => injected;
|
|
3030
3295
|
}
|
|
3031
|
-
export async function run(args, injected, signal, lifecycle) {
|
|
3296
|
+
export async function run(args, injected, signal, lifecycle, work) {
|
|
3032
3297
|
let once = false;
|
|
3033
3298
|
for (const arg of args) {
|
|
3034
3299
|
if (arg === '--once')
|
|
@@ -3084,11 +3349,21 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3084
3349
|
`tools` is referenced inside the callback it is being given, which is safe
|
|
3085
3350
|
for the plain reason that the callback can only run once a request has
|
|
3086
3351
|
arrived at a server that by then exists. */
|
|
3352
|
+
if (work)
|
|
3353
|
+
await reconcilePanelInterruptions(current(), machineId, work);
|
|
3354
|
+
lifecycle?.reconciled?.();
|
|
3087
3355
|
const tools = await startToolsServer(current(), async (parentRunId, brief, codebase, processToken, choice) => {
|
|
3088
|
-
const
|
|
3089
|
-
|
|
3090
|
-
|
|
3356
|
+
const endClaim = work?.beginClaim();
|
|
3357
|
+
try {
|
|
3358
|
+
const child = await startChild(current(), tools, machineId, parentRunId, brief, codebase, processToken, choice);
|
|
3359
|
+
hold(child.runId, child.settled);
|
|
3360
|
+
return { runId: child.runId };
|
|
3361
|
+
}
|
|
3362
|
+
finally {
|
|
3363
|
+
endClaim?.();
|
|
3364
|
+
}
|
|
3091
3365
|
}, (runId, processToken, action) => recoverLanding(current(), machineId, runId, processToken, action));
|
|
3366
|
+
tools.work = work;
|
|
3092
3367
|
out(`daemon machine ${machineId}`);
|
|
3093
3368
|
out(`tools ${tools.urlFor('<run-id>')}`);
|
|
3094
3369
|
out(once ? 'mode one poll' : `mode polling every ${POLL_INTERVAL_MS / 1000}s, Ctrl-C to stop`);
|
|
@@ -3129,6 +3404,8 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3129
3404
|
// Restart only at a poll boundary, before any claims, with no live work.
|
|
3130
3405
|
if (lifecycle && !lifecycle.beforePoll(inFlight.size))
|
|
3131
3406
|
break;
|
|
3407
|
+
let endClaim;
|
|
3408
|
+
const claimOperations = [];
|
|
3132
3409
|
/* ═══ ONE POLL FAILING IS NOT THE DAEMON FAILING. ═══ Every read and write
|
|
3133
3410
|
here throws on a network or database error, by design (constraint 7), and
|
|
3134
3411
|
until Slice 4 that threw straight out of `panel3/cli.js run` and exited the process.
|
|
@@ -3141,12 +3418,19 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3141
3418
|
`--once` still fails loudly, because the acceptance harness reads the exit
|
|
3142
3419
|
code and a swallowed failure there would make a broken suite look green. */
|
|
3143
3420
|
try {
|
|
3421
|
+
if (work)
|
|
3422
|
+
await reconcilePanelInterruptions(current(), machineId, work);
|
|
3423
|
+
if (work && !work.allowed()) {
|
|
3424
|
+
await sleep(POLL_INTERVAL_MS);
|
|
3425
|
+
continue;
|
|
3426
|
+
}
|
|
3427
|
+
endClaim = work?.beginClaim();
|
|
3144
3428
|
/* ═══ THE USER'S STOP IS HONOURED BEFORE ANYTHING ELSE ON THE POLL. ═══ It
|
|
3145
3429
|
is the only thing here that a person is waiting on, and the two takes
|
|
3146
3430
|
below can spend the rest of the poll starting agents. Nothing else needs
|
|
3147
3431
|
to run first: `panel3_stop_card` has already ended the runs, so recovery
|
|
3148
3432
|
cannot see them and neither take can start them. */
|
|
3149
|
-
await killStopped(current(), machineId);
|
|
3433
|
+
await killStopped(current(), machineId, work);
|
|
3150
3434
|
/* Publish readiness before claiming work. A card with an untaken turn reads the same whether a daemon
|
|
3151
3435
|
is two seconds away or nobody has one running; this row is the only place
|
|
3152
3436
|
the difference exists. It is written before the takes rather than after
|
|
@@ -3171,7 +3455,7 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3171
3455
|
}
|
|
3172
3456
|
listeningHarness = machineHarness;
|
|
3173
3457
|
await sayListening(current(), machineId, machineName, machineHarness);
|
|
3174
|
-
await reconcileOwnerSessions(current(), machineId, machineHarness, new Set(inFlight.keys()));
|
|
3458
|
+
await reconcileOwnerSessions(current(), machineId, machineHarness, new Set(inFlight.keys()), work);
|
|
3175
3459
|
await recoverStranded(current(), tools, machineId, new Set(inFlight.keys()), hold);
|
|
3176
3460
|
/* ═══ AND THE COPIES OF CARDS THAT ARE OVER. ═══ After recovery,
|
|
3177
3461
|
deliberately: a run this machine is about to resume is one whose card is
|
|
@@ -3197,7 +3481,10 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3197
3481
|
/* THE MACHINE ID GOES IN because the take writes the run row, and a run has
|
|
3198
3482
|
to say where it is running: the exclusion is cross-machine and recovery is
|
|
3199
3483
|
per-machine, so a row with nobody's machine on it could be neither. */
|
|
3200
|
-
const
|
|
3484
|
+
const takenResponse = await panelClaim(current(), work, 'panel3_take_turns', { p_machine_id: machineId, p_agent: machineHarness });
|
|
3485
|
+
if (takenResponse.operationId)
|
|
3486
|
+
claimOperations.push(takenResponse.operationId);
|
|
3487
|
+
const taken = await returned(Promise.resolve(takenResponse), 'take', 'turns');
|
|
3201
3488
|
/* ═══ THE OTHER KIND OF TAKEABLE WORK. ═══ ux.md's re-arm: an answered
|
|
3202
3489
|
question makes the branch that asked it takeable again, and a question
|
|
3203
3490
|
still walking up makes the run it reached takeable so that level gets its
|
|
@@ -3211,7 +3498,10 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3211
3498
|
to it on a later poll rather than putting two of them on one card. The
|
|
3212
3499
|
other order would decide the same question from a snapshot taken before
|
|
3213
3500
|
the run existed. */
|
|
3214
|
-
const
|
|
3501
|
+
const rearmedResponse = await panelClaim(current(), work, 'panel3_take_rearms', { p_machine_id: machineId, p_agent: machineHarness });
|
|
3502
|
+
if (rearmedResponse.operationId)
|
|
3503
|
+
claimOperations.push(rearmedResponse.operationId);
|
|
3504
|
+
const rearmed = await returned(Promise.resolve(rearmedResponse), 'take', 'runs that can carry on');
|
|
3215
3505
|
for (const row of rearmed) {
|
|
3216
3506
|
/* NOT AWAITED PAST THE SPAWN, exactly as a taken card is not: the work is a
|
|
3217
3507
|
real agent and holding the poll open for it would put every other card
|
|
@@ -3265,6 +3555,10 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3265
3555
|
}
|
|
3266
3556
|
}
|
|
3267
3557
|
}
|
|
3558
|
+
finally {
|
|
3559
|
+
work?.finishRpcs(claimOperations);
|
|
3560
|
+
endClaim?.();
|
|
3561
|
+
}
|
|
3268
3562
|
if (!signal?.aborted)
|
|
3269
3563
|
await sleep(POLL_INTERVAL_MS);
|
|
3270
3564
|
}
|
|
@@ -3284,8 +3578,12 @@ export async function run(args, injected, signal, lifecycle) {
|
|
|
3284
3578
|
* client. A stopped worker is restarted automatically. Explicit restarts happen
|
|
3285
3579
|
* at an idle poll boundary and confirm only after a new worker completes a poll.
|
|
3286
3580
|
* Sign-out stops the supervisor, so it cannot restart behind the user's back. */
|
|
3287
|
-
export function startPanel(injected) {
|
|
3581
|
+
export function startPanel(injected, work) {
|
|
3288
3582
|
const controller = new AbortController();
|
|
3583
|
+
let resolveReady;
|
|
3584
|
+
let rejectReady;
|
|
3585
|
+
const ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; });
|
|
3586
|
+
void ready.catch(() => { });
|
|
3289
3587
|
let requested = false;
|
|
3290
3588
|
let restarting = null;
|
|
3291
3589
|
let resolveRestart = null;
|
|
@@ -3304,11 +3602,13 @@ export function startPanel(injected) {
|
|
|
3304
3602
|
}
|
|
3305
3603
|
return false;
|
|
3306
3604
|
},
|
|
3605
|
+
reconciled: () => resolveReady(),
|
|
3307
3606
|
ready: () => { if (!requested)
|
|
3308
3607
|
resolveRestart?.(); },
|
|
3309
|
-
});
|
|
3608
|
+
}, work);
|
|
3310
3609
|
}
|
|
3311
3610
|
catch (error) {
|
|
3611
|
+
rejectReady(error instanceof Error ? error : new Error(String(error)));
|
|
3312
3612
|
said(`the agent panel stopped polling: ${error instanceof Error ? error.message : String(error)}`);
|
|
3313
3613
|
// Startup failures must not leave an online machine with a dead worker.
|
|
3314
3614
|
if (!controller.signal.aborted)
|
|
@@ -3317,6 +3617,7 @@ export function startPanel(injected) {
|
|
|
3317
3617
|
}
|
|
3318
3618
|
})();
|
|
3319
3619
|
return {
|
|
3620
|
+
ready,
|
|
3320
3621
|
restart: () => {
|
|
3321
3622
|
if (controller.signal.aborted)
|
|
3322
3623
|
return Promise.reject(new Error('This machine is signing out. Open Companion and sign in again.'));
|
|
@@ -3339,6 +3640,7 @@ export function startPanel(injected) {
|
|
|
3339
3640
|
},
|
|
3340
3641
|
stop: async () => {
|
|
3341
3642
|
controller.abort();
|
|
3643
|
+
rejectReady(new Error('Cloud startup was cancelled by the service command.'));
|
|
3342
3644
|
rejectRestart?.(new Error('The machine disconnected before the worker restarted.'));
|
|
3343
3645
|
await running;
|
|
3344
3646
|
},
|