mtech-ai-local-runner 0.1.38 → 0.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/src/adapters/codex/adapter.js +47 -44
- package/dist/src/adapters/codex/exec-launch-host.js +12 -10
- package/dist/src/daemon/execution-coordinator.js +20 -1
- package/dist/src/process/node-process-group-host.js +14 -2
- package/dist/src/process/owned-process-guardian.js +58 -16
- package/dist/src/safety/execution-failure.js +27 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,3 +1,18 @@
|
|
|
1
|
+
## 0.1.40: say why the owned guardian could not confirm cleanup
|
|
2
|
+
|
|
3
|
+
When the owned guardian reports an unconfirmed cleanup, the `executor_cleanup` diagnostic now carries fixed
|
|
4
|
+
`cleanup_guardian_*` observations: what started the cleanup, how TERM and KILL went, whether the executor group was
|
|
5
|
+
still alive and its pipes still open, whether the root had exited, and whether the host's deadline passed. Cleanup
|
|
6
|
+
decisions, deadlines and recovery are unchanged.
|
|
7
|
+
|
|
8
|
+
## 0.1.39: keep the first failure when executor cleanup is unconfirmed
|
|
9
|
+
|
|
10
|
+
A job whose Codex executor could not confirm its cleanup was suspended with only `EXECUTION_UNACKNOWLEDGED` in the
|
|
11
|
+
daemon log: the unconfirmed stop was rethrown before the failure was logged. The daemon now writes the stage that
|
|
12
|
+
failed first (P1 runs use `p1_freeze`, `p1_verify` and `p1_receipts`) and a second `execution_failed` line with
|
|
13
|
+
stage `executor_cleanup`, whose fixed `cleanup_*` observations name the cleanup step that failed or was still
|
|
14
|
+
pending at the deadline. Suspension and manual reconciliation are unchanged.
|
|
15
|
+
|
|
1
16
|
## 0.1.38: find the Codex app shipped as ChatGPT.app
|
|
2
17
|
|
|
3
18
|
0.1.37 looked only for `Codex.app`, so a Mac whose Codex app is the ChatGPT app (`/Applications/ChatGPT.app`,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { executionError } from '../../safety/execution-failure.js';
|
|
1
|
+
import { executionError, unconfirmedCleanupError, cleanupObservations } from '../../safety/execution-failure.js';
|
|
2
2
|
import { UnsupportedSupervisorCommandError } from '../../safety/unsupported-supervisor-command.js';
|
|
3
3
|
import { CodexExecLaunchHost } from './exec-launch-host.js';
|
|
4
4
|
import { requireSecurityRequirements } from '../../safety/security-requirements.js';
|
|
@@ -250,9 +250,27 @@ export class CodexAdapter {
|
|
|
250
250
|
if (this.cancelPromise)
|
|
251
251
|
return this.cancelPromise;
|
|
252
252
|
this.cancelled = true;
|
|
253
|
-
let finish, reject
|
|
253
|
+
let finish, reject;
|
|
254
|
+
// Fixed step names only: which cleanup step failed, or was still pending at the deadline.
|
|
255
|
+
const failed = new Set(), pending = new Set();
|
|
256
|
+
const unconfirmed = () => unconfirmedCleanupError('Provider process group cleanup could not be confirmed', [...failed, ...pending]);
|
|
257
|
+
const step = async (name, work) => {
|
|
258
|
+
const waiting = `cleanup_${name}_pending`;
|
|
259
|
+
pending.add(waiting);
|
|
260
|
+
try {
|
|
261
|
+
await work();
|
|
262
|
+
}
|
|
263
|
+
catch (error) {
|
|
264
|
+
failed.add(`cleanup_${name}_failed`);
|
|
265
|
+
for (const inner of cleanupObservations(error))
|
|
266
|
+
failed.add(inner);
|
|
267
|
+
}
|
|
268
|
+
finally {
|
|
269
|
+
pending.delete(waiting);
|
|
270
|
+
}
|
|
271
|
+
};
|
|
254
272
|
this.cancelPromise = new Promise((resolve, rejectPromise) => { finish = resolve; reject = rejectPromise; });
|
|
255
|
-
const timer = setTimeout(() =>
|
|
273
|
+
const timer = setTimeout(() => { failed.add('cleanup_timeout'); reject(unconfirmed()); }, 6000);
|
|
256
274
|
// Let the official owner freeze its observations before external revocation.
|
|
257
275
|
// Known owner cleanup and endpoint closure start without waiting for a late
|
|
258
276
|
// retained handle; only confirmation waits for its acquisition/disposal.
|
|
@@ -262,68 +280,53 @@ export class CodexAdapter {
|
|
|
262
280
|
officialStop = this.ownedExec?.cancel();
|
|
263
281
|
}
|
|
264
282
|
catch {
|
|
265
|
-
failed
|
|
283
|
+
failed.add('cleanup_owner_stop_failed');
|
|
266
284
|
}
|
|
267
285
|
}
|
|
268
286
|
try {
|
|
269
287
|
this.options.revokeSecurity?.();
|
|
270
288
|
}
|
|
271
289
|
catch {
|
|
272
|
-
failed
|
|
290
|
+
failed.add('cleanup_security_revoke_failed');
|
|
273
291
|
}
|
|
274
292
|
try {
|
|
275
293
|
this.tools?.revoke();
|
|
276
294
|
}
|
|
277
295
|
catch {
|
|
278
|
-
failed
|
|
296
|
+
failed.add('cleanup_tools_revoke_failed');
|
|
279
297
|
}
|
|
280
298
|
this.startupAbort.abort();
|
|
281
|
-
const ownerStop = (async () => {
|
|
299
|
+
const ownerStop = step('owner_stop', async () => {
|
|
282
300
|
if (this.usingOfficialExec) {
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
}
|
|
286
|
-
catch {
|
|
287
|
-
failed = true;
|
|
288
|
-
}
|
|
301
|
+
await officialStop;
|
|
302
|
+
return;
|
|
289
303
|
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
}
|
|
297
|
-
try {
|
|
298
|
-
if (this.handle?.terminateOwned)
|
|
299
|
-
await this.handle.terminateOwned();
|
|
300
|
-
else
|
|
301
|
-
this.handle?.kill('SIGKILL');
|
|
302
|
-
}
|
|
303
|
-
catch {
|
|
304
|
-
failed = true;
|
|
305
|
-
}
|
|
304
|
+
let stopFailed = false;
|
|
305
|
+
try {
|
|
306
|
+
this.handle?.stdin.end();
|
|
307
|
+
}
|
|
308
|
+
catch {
|
|
309
|
+
stopFailed = true;
|
|
306
310
|
}
|
|
307
|
-
})();
|
|
308
|
-
const toolsClose = (async () => { try {
|
|
309
|
-
await this.tools?.close();
|
|
310
|
-
}
|
|
311
|
-
catch {
|
|
312
|
-
failed = true;
|
|
313
|
-
} })();
|
|
314
|
-
void (async () => {
|
|
315
|
-
const outcomes = await Promise.allSettled([ownerStop, toolsClose, this.pendingHandle]);
|
|
316
|
-
if (outcomes.some(outcome => outcome.status === 'rejected'))
|
|
317
|
-
failed = true;
|
|
318
311
|
try {
|
|
319
|
-
|
|
312
|
+
if (this.handle?.terminateOwned)
|
|
313
|
+
await this.handle.terminateOwned();
|
|
314
|
+
else
|
|
315
|
+
this.handle?.kill('SIGKILL');
|
|
320
316
|
}
|
|
321
317
|
catch {
|
|
322
|
-
|
|
318
|
+
stopFailed = true;
|
|
323
319
|
}
|
|
320
|
+
if (stopFailed)
|
|
321
|
+
throw Error('owner stop failed');
|
|
322
|
+
});
|
|
323
|
+
const toolsClose = step('tools_close', () => this.tools?.close());
|
|
324
|
+
void (async () => {
|
|
325
|
+
await Promise.all([ownerStop, toolsClose, step('pending_handle', () => this.pendingHandle)]);
|
|
326
|
+
await step('official_exec_close', () => this.options.closeOfficialExec?.());
|
|
324
327
|
clearTimeout(timer);
|
|
325
|
-
if (failed)
|
|
326
|
-
reject(
|
|
328
|
+
if (failed.size)
|
|
329
|
+
reject(unconfirmed());
|
|
327
330
|
else
|
|
328
331
|
finish();
|
|
329
332
|
})();
|
|
@@ -2,7 +2,7 @@ import { performance } from 'node:perf_hooks';
|
|
|
2
2
|
import { PreparationClock, CODEX_PREPARATION_BUDGET_MS } from '../../safety/preparation-timing.js';
|
|
3
3
|
import { localEndpointObservations, localEndpointReadObservations } from '../../safety/local-tool-server.js';
|
|
4
4
|
import { processStartupObservations, freezeProcessLaunchObservation } from '../../process/node-process-group-host.js';
|
|
5
|
-
import { executionError } from '../../safety/execution-failure.js';
|
|
5
|
+
import { executionError, unconfirmedCleanupError, cleanupObservations } from '../../safety/execution-failure.js';
|
|
6
6
|
import { UNMANAGED_EXEC_MODE } from './unmanaged-scope.js';
|
|
7
7
|
import { fingerprint as taskFingerprint } from '../../daemon/execution-coordinator.js';
|
|
8
8
|
import { securityRequirementsFingerprint as hash } from '../../safety/security-requirements.js';
|
|
@@ -280,19 +280,19 @@ export class CodexExecLaunchHost {
|
|
|
280
280
|
}
|
|
281
281
|
let finish, reject;
|
|
282
282
|
this.closePromise = new Promise((resolve, rejectPromise) => { finish = resolve; reject = rejectPromise; });
|
|
283
|
-
|
|
283
|
+
const failed = new Set();
|
|
284
284
|
// Publish closePromise first: synchronous exit/cancel cannot recurse or skip cleanup.
|
|
285
285
|
try {
|
|
286
286
|
this.deps.execTools?.revoke();
|
|
287
287
|
}
|
|
288
288
|
catch {
|
|
289
|
-
|
|
289
|
+
failed.add('cleanup_exec_tools_revoke_failed');
|
|
290
290
|
}
|
|
291
291
|
try {
|
|
292
292
|
this.deps.revokeTools();
|
|
293
293
|
}
|
|
294
294
|
catch {
|
|
295
|
-
|
|
295
|
+
failed.add('cleanup_job_tools_revoke_failed');
|
|
296
296
|
}
|
|
297
297
|
this.abort.abort();
|
|
298
298
|
Promise.resolve().then(async () => {
|
|
@@ -300,7 +300,7 @@ export class CodexExecLaunchHost {
|
|
|
300
300
|
this.handle?.stdin.end();
|
|
301
301
|
}
|
|
302
302
|
catch {
|
|
303
|
-
|
|
303
|
+
failed.add('cleanup_stdin_end_failed');
|
|
304
304
|
}
|
|
305
305
|
try {
|
|
306
306
|
if (this.handle?.terminateOwned)
|
|
@@ -308,17 +308,19 @@ export class CodexExecLaunchHost {
|
|
|
308
308
|
else
|
|
309
309
|
this.handle?.kill('SIGKILL');
|
|
310
310
|
}
|
|
311
|
-
catch {
|
|
312
|
-
|
|
311
|
+
catch (error) {
|
|
312
|
+
failed.add('cleanup_process_terminate_failed');
|
|
313
|
+
for (const observed of cleanupObservations(error))
|
|
314
|
+
failed.add(observed);
|
|
313
315
|
}
|
|
314
316
|
try {
|
|
315
317
|
await this.deps.closeTools();
|
|
316
318
|
}
|
|
317
319
|
catch {
|
|
318
|
-
|
|
320
|
+
failed.add('cleanup_tool_server_close_failed');
|
|
319
321
|
}
|
|
320
|
-
if (
|
|
321
|
-
reject(
|
|
322
|
+
if (failed.size)
|
|
323
|
+
reject(unconfirmedCleanupError('CODEX_EXEC_CLEANUP_UNCONFIRMED', failed));
|
|
322
324
|
else
|
|
323
325
|
finish();
|
|
324
326
|
});
|
|
@@ -476,9 +476,12 @@ export class ExecutionCoordinator {
|
|
|
476
476
|
await publicationStep(() => adapter.freezeForPublication());
|
|
477
477
|
lifecycle.frozen = true;
|
|
478
478
|
} await assertModelFrozen(); };
|
|
479
|
+
executionStage = 'p1_freeze';
|
|
479
480
|
await freezeOwnedModel();
|
|
480
481
|
await readControls();
|
|
482
|
+
executionStage = 'p1_verify';
|
|
481
483
|
p1Result = await publicationStep(() => this.deps.p1Host.verify({ job, checkoutPath: workspace.path, bindingId: binding.bindingId, signal: verificationAbort.signal, current, freezeOwnedModel, assertModelFrozen }));
|
|
484
|
+
executionStage = 'p1_receipts';
|
|
482
485
|
await readControls();
|
|
483
486
|
await assertModelFrozen();
|
|
484
487
|
const seen = new Set();
|
|
@@ -491,6 +494,7 @@ export class ExecutionCoordinator {
|
|
|
491
494
|
const id = await publicationStep(() => append({ eventType: 'test_result', summary: 'runner command observed', payload: { observer: 'runner', runner_receipt: receipt } }, true));
|
|
492
495
|
p1ReceiptIds.push(id);
|
|
493
496
|
}
|
|
497
|
+
executionStage = 'event_stream';
|
|
494
498
|
}
|
|
495
499
|
if (event.eventType === "completed" && fixSession) {
|
|
496
500
|
// Stop only this owned model before running verified host publication.
|
|
@@ -572,7 +576,22 @@ export class ExecutionCoordinator {
|
|
|
572
576
|
await append(terminal);
|
|
573
577
|
}
|
|
574
578
|
catch (error) {
|
|
575
|
-
|
|
579
|
+
try {
|
|
580
|
+
await stop(true);
|
|
581
|
+
}
|
|
582
|
+
catch (cleanupError) {
|
|
583
|
+
// An unconfirmed executor stop keeps the job suspended, but it must not erase
|
|
584
|
+
// the stage that failed first: log both fixed diagnostics before rethrowing.
|
|
585
|
+
try {
|
|
586
|
+
this.deps.reportExecutionFailure?.(executionFailure(error, executionStage));
|
|
587
|
+
}
|
|
588
|
+
catch { /* logging must not change terminal reconciliation */ }
|
|
589
|
+
try {
|
|
590
|
+
this.deps.reportExecutionFailure?.(executionFailure(cleanupError, 'executor_cleanup'));
|
|
591
|
+
}
|
|
592
|
+
catch { /* logging must not change terminal reconciliation */ }
|
|
593
|
+
throw cleanupError;
|
|
594
|
+
}
|
|
576
595
|
if (error instanceof PublicationOutcomeUnknownError || (publicationPending && (heartbeatFailure || requestedTerminal))) {
|
|
577
596
|
// The provider may already have committed/pushed/opened a PR. Do not
|
|
578
597
|
// release this run as failed or let restart begin another model session.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import { OwnedLaunchObserver } from './owned-launch-observer.js';
|
|
3
|
+
import { unconfirmedCleanupError } from '../safety/execution-failure.js';
|
|
3
4
|
const startup = new WeakMap();
|
|
4
5
|
const launchObservers = new WeakMap();
|
|
5
6
|
export function freezeProcessLaunchObservation(handle) { return launchObservers.get(handle)?.freeze(); }
|
|
@@ -142,7 +143,7 @@ export class NodeProcessGroupHost {
|
|
|
142
143
|
return;
|
|
143
144
|
realExit = true;
|
|
144
145
|
observations.add('host_process_exited');
|
|
145
|
-
void guardian.terminate().then(() => { owned = false; const result = guardian.outcome(); terminated(result?.code ?? null, result?.signal ?? null); }, () => { failure =
|
|
146
|
+
void guardian.terminate().then(() => { owned = false; const result = guardian.outcome(); terminated(result?.code ?? null, result?.signal ?? null); }, () => { failure = unconfirmedCleanupError('Provider process group cleanup could not be confirmed', guardian.cleanupObservations?.() ?? []); terminated(null, null); notify(); });
|
|
146
147
|
});
|
|
147
148
|
// Drain from spawn, not from the first adapter event read: official exec can
|
|
148
149
|
// emit startup events before it initializes MCP, while the adapter awaits
|
|
@@ -266,7 +267,18 @@ export class NodeProcessGroupHost {
|
|
|
266
267
|
clearTimeout(timer);
|
|
267
268
|
}
|
|
268
269
|
},
|
|
269
|
-
terminateOwned: () => cleanupPromise ??= (async () => {
|
|
270
|
+
terminateOwned: () => cleanupPromise ??= (async () => {
|
|
271
|
+
launch?.stop();
|
|
272
|
+
stopInput();
|
|
273
|
+
// Keep the fixed message; carry only the guardian's fixed cleanup observations.
|
|
274
|
+
try {
|
|
275
|
+
await guardian.terminate();
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
throw unconfirmedCleanupError('Provider process group cleanup could not be confirmed', guardian.cleanupObservations?.() ?? []);
|
|
279
|
+
}
|
|
280
|
+
owned = false;
|
|
281
|
+
})(),
|
|
270
282
|
once: (_event, callback) => { if (termination) {
|
|
271
283
|
const ended = termination;
|
|
272
284
|
queueMicrotask(() => callback(ended.code, ended.signal));
|
|
@@ -12,30 +12,35 @@ let child, ownedPid, ownedGone=false, cleaning, ready=Promise.resolve(), rootExi
|
|
|
12
12
|
const send=value=>{if(!control.destroyed)control.write(JSON.stringify(value)+'\n');};
|
|
13
13
|
const delay=ms=>new Promise(resolve=>setTimeout(resolve,ms));
|
|
14
14
|
const alive=()=>{if(!ownedPid||ownedGone)return false;try{process.kill(-ownedPid,0);return true;}catch(e){if(e.code==='ESRCH'){ownedGone=true;return false;}if(e.code==='EPERM')return true;throw Error('cleanup');}};
|
|
15
|
-
const signal=s=>{if(!ownedPid)throw Error('ownership');if(ownedGone)return;try{process.kill(-ownedPid,s);}catch(e){if(e.code==='ESRCH'){ownedGone=true;return;}if(e.code==='EPERM'){if(!rootExited)child.kill(s);return;}throw Error('cleanup');}};
|
|
16
|
-
|
|
15
|
+
const signal=s=>{if(!ownedPid)throw Error('ownership');if(ownedGone)return 'esrch';try{process.kill(-ownedPid,s);return 'sent';}catch(e){if(e.code==='ESRCH'){ownedGone=true;return 'esrch';}if(e.code==='EPERM'){if(!rootExited)child.kill(s);return 'eperm';}throw Error('cleanup');}};
|
|
16
|
+
// Fixed words only: what started cleanup, each signal outcome and what was still open.
|
|
17
|
+
function cleanup(immediate=false,trigger='terminate'){
|
|
17
18
|
if(cleaning)return cleaning;
|
|
19
|
+
const detail={trigger,term:'not_needed',kill:'not_needed'};
|
|
18
20
|
cleaning=(async()=>{
|
|
19
21
|
try{
|
|
20
22
|
if(!child)throw Error('ownership');
|
|
21
23
|
await ready;
|
|
22
24
|
child.stdin.end();
|
|
23
25
|
if(ownedPid){
|
|
24
|
-
if(alive())signal(immediate?'SIGKILL':'SIGTERM');
|
|
26
|
+
if(alive())detail.term=signal(immediate?'SIGKILL':'SIGTERM');
|
|
25
27
|
const deadline=Date.now()+(immediate?0:2000);
|
|
26
28
|
while(alive()&&Date.now()<deadline)await delay(20);
|
|
27
|
-
if(alive())signal('SIGKILL');
|
|
29
|
+
if(alive())detail.kill=signal('SIGKILL');
|
|
28
30
|
const killed=Date.now()+2000;
|
|
29
31
|
while((alive()||!closed)&&Date.now()<killed)await delay(20);
|
|
30
32
|
if(alive()||!closed)throw Error('cleanup');
|
|
31
33
|
}
|
|
32
34
|
send({type:'cleaned'});finish(0);
|
|
33
|
-
}catch{
|
|
35
|
+
}catch{
|
|
36
|
+
let groupAlive=true;try{groupAlive=alive();}catch{/* an unknown liveness error stays alive */}
|
|
37
|
+
send({type:'unconfirmed',detail:{...detail,alive:groupAlive,closed,root:rootExited}});finish(1);
|
|
38
|
+
}
|
|
34
39
|
})();
|
|
35
40
|
return cleaning;
|
|
36
41
|
}
|
|
37
42
|
function finish(code){process.exitCode=code;process.stdin.destroy();liveness.destroy();control.end(()=>control.destroy());process.stdout.end();process.stderr.end();}
|
|
38
|
-
function lost(){orphaned=true;if(configured)void cleanup();else finish(1);}
|
|
43
|
+
function lost(){orphaned=true;if(configured)void cleanup(false,'lost');else finish(1);}
|
|
39
44
|
liveness.on('data',()=>{});liveness.once('end',lost);liveness.once('error',lost);
|
|
40
45
|
control.on('error',lost);control.on('end',lost);
|
|
41
46
|
control.on('data',chunk=>{
|
|
@@ -47,19 +52,19 @@ control.on('data',chunk=>{
|
|
|
47
52
|
configured=true;
|
|
48
53
|
try{child=spawn(message.command,message.args,{cwd:message.cwd,env:message.env,shell:false,detached:true,stdio:['pipe','pipe','pipe']});}catch{send({type:'spawn_failed'});finish(0);return;}
|
|
49
54
|
ready=new Promise(resolve=>{
|
|
50
|
-
child.once('spawn',()=>{ownedPid=child.pid;send({type:'spawned',pid:ownedPid,parentPid:process.pid});resolve();if(orphaned)void cleanup();});
|
|
51
|
-
child.once('error',()=>{send({type:'spawn_failed'});closed=true;resolve();void cleanup(true);});
|
|
55
|
+
child.once('spawn',()=>{ownedPid=child.pid;send({type:'spawned',pid:ownedPid,parentPid:process.pid});resolve();if(orphaned)void cleanup(false,'lost');});
|
|
56
|
+
child.once('error',()=>{send({type:'spawn_failed'});closed=true;resolve();void cleanup(true,'spawn_error');});
|
|
52
57
|
});
|
|
53
58
|
child.stdin.on('error',()=>{});
|
|
54
59
|
process.stdin.pipe(child.stdin);child.stdout.pipe(process.stdout,{end:false});child.stderr.pipe(process.stderr,{end:false});
|
|
55
|
-
child.once('exit',(code,signal)=>{rootExited=true;send({type:'root_exit',code,signal});void cleanup();});
|
|
60
|
+
child.once('exit',(code,signal)=>{rootExited=true;send({type:'root_exit',code,signal});void cleanup(false,'root_exit');});
|
|
56
61
|
child.once('close',()=>{closed=true;});
|
|
57
|
-
}else if(message.type==='terminate')void cleanup(message.immediate===true);
|
|
62
|
+
}else if(message.type==='terminate')void cleanup(message.immediate===true,'terminate');
|
|
58
63
|
else lost();
|
|
59
64
|
}
|
|
60
65
|
});
|
|
61
|
-
process.stdout.on('error',()=>{child?.stdout.resume();void cleanup(true);});
|
|
62
|
-
process.stderr.on('error',()=>{child?.stderr.resume();void cleanup(true);});
|
|
66
|
+
process.stdout.on('error',()=>{child?.stdout.resume();void cleanup(true,'output_error');});
|
|
67
|
+
process.stderr.on('error',()=>{child?.stderr.resume();void cleanup(true,'output_error');});
|
|
63
68
|
`;
|
|
64
69
|
export function spawnOwnedGuardian(command, args, options) {
|
|
65
70
|
const child = spawn(process.execPath, ['--input-type=commonjs', '-e', GUARDIAN_SOURCE], {
|
|
@@ -73,6 +78,7 @@ export function spawnOwnedGuardian(command, args, options) {
|
|
|
73
78
|
// Hosts can attach observers after spawn returns without unhandled rejection.
|
|
74
79
|
void started.catch(() => { });
|
|
75
80
|
let confirmed = false, spawnFailed = false, protocolFailed = false, buffer = '';
|
|
81
|
+
let terminateWritten = false, hostTimeout = false, detail;
|
|
76
82
|
let outcome;
|
|
77
83
|
let closePromise;
|
|
78
84
|
const closed = new Promise(resolve => child.once('close', () => { liveness.destroy(); resolve(); }));
|
|
@@ -109,8 +115,11 @@ export function spawnOwnedGuardian(command, args, options) {
|
|
|
109
115
|
outcome = { code: message.code, signal: message.signal };
|
|
110
116
|
else if (message.type === 'cleaned')
|
|
111
117
|
confirmed = true;
|
|
112
|
-
else if (message.type === 'unconfirmed')
|
|
118
|
+
else if (message.type === 'unconfirmed') {
|
|
113
119
|
protocolFailed = true;
|
|
120
|
+
if (message.detail && typeof message.detail === 'object' && !Array.isArray(message.detail))
|
|
121
|
+
detail = message.detail;
|
|
122
|
+
}
|
|
114
123
|
else {
|
|
115
124
|
protocolFailed = true;
|
|
116
125
|
liveness.destroy();
|
|
@@ -121,14 +130,16 @@ export function spawnOwnedGuardian(command, args, options) {
|
|
|
121
130
|
child.once('close', () => { rejectStarted(new Error('Provider process could not start')); });
|
|
122
131
|
control.write(JSON.stringify({ type: 'launch', command, args, ...options }) + '\n');
|
|
123
132
|
return { child, started, closed, outcome: () => outcome, spawnFailed: () => spawnFailed,
|
|
124
|
-
signal(immediate) { if (child.exitCode === null && child.signalCode === null && !confirmed && !spawnFailed && !control.destroyed)
|
|
125
|
-
control.write(JSON.stringify({ type: 'terminate', immediate }) + '\n');
|
|
133
|
+
signal(immediate) { if (child.exitCode === null && child.signalCode === null && !confirmed && !spawnFailed && !control.destroyed) {
|
|
134
|
+
control.write(JSON.stringify({ type: 'terminate', immediate }) + '\n');
|
|
135
|
+
terminateWritten = true;
|
|
136
|
+
} },
|
|
126
137
|
terminate(immediate = false) {
|
|
127
138
|
if (closePromise)
|
|
128
139
|
return closePromise;
|
|
129
140
|
this.signal(immediate);
|
|
130
141
|
closePromise = new Promise((resolve, reject) => {
|
|
131
|
-
const timer = setTimeout(() => { protocolFailed = true; liveness.destroy(); reject(new Error('Provider process group cleanup could not be confirmed')); }, 5500);
|
|
142
|
+
const timer = setTimeout(() => { protocolFailed = true; hostTimeout = true; liveness.destroy(); reject(new Error('Provider process group cleanup could not be confirmed')); }, 5500);
|
|
132
143
|
void closed.then(() => { clearTimeout(timer); if (protocolFailed || !confirmed)
|
|
133
144
|
reject(new Error('Provider process group cleanup could not be confirmed'));
|
|
134
145
|
else
|
|
@@ -136,5 +147,36 @@ export function spawnOwnedGuardian(command, args, options) {
|
|
|
136
147
|
});
|
|
137
148
|
return closePromise;
|
|
138
149
|
},
|
|
150
|
+
/** Fixed cleanup observations after an unconfirmed cleanup; empty once cleanup is confirmed. */
|
|
151
|
+
cleanupObservations() {
|
|
152
|
+
if (confirmed && !protocolFailed)
|
|
153
|
+
return [];
|
|
154
|
+
if (!protocolFailed && !hostTimeout)
|
|
155
|
+
return [];
|
|
156
|
+
const observed = [terminateWritten ? 'cleanup_guardian_terminate_written' : 'cleanup_guardian_terminate_not_written'];
|
|
157
|
+
if (hostTimeout)
|
|
158
|
+
observed.push('cleanup_guardian_host_timeout');
|
|
159
|
+
if (!detail) {
|
|
160
|
+
if (!hostTimeout)
|
|
161
|
+
observed.push('cleanup_guardian_protocol_failed');
|
|
162
|
+
return observed;
|
|
163
|
+
}
|
|
164
|
+
const word = (value, allowed) => allowed.find(item => item === value);
|
|
165
|
+
const trigger = word(detail.trigger, ['terminate', 'root_exit', 'lost', 'output_error', 'spawn_error']);
|
|
166
|
+
const term = word(detail.term, ['sent', 'eperm', 'esrch', 'not_needed']), kill = word(detail.kill, ['sent', 'eperm', 'esrch', 'not_needed']);
|
|
167
|
+
if (trigger)
|
|
168
|
+
observed.push(`cleanup_guardian_trigger_${trigger}`);
|
|
169
|
+
if (term)
|
|
170
|
+
observed.push(`cleanup_guardian_term_${term}`);
|
|
171
|
+
if (kill)
|
|
172
|
+
observed.push(`cleanup_guardian_kill_${kill}`);
|
|
173
|
+
if (typeof detail.alive === 'boolean')
|
|
174
|
+
observed.push(detail.alive ? 'cleanup_guardian_group_alive' : 'cleanup_guardian_group_gone');
|
|
175
|
+
if (typeof detail.closed === 'boolean')
|
|
176
|
+
observed.push(detail.closed ? 'cleanup_guardian_pipes_closed' : 'cleanup_guardian_pipes_open');
|
|
177
|
+
if (typeof detail.root === 'boolean')
|
|
178
|
+
observed.push(detail.root ? 'cleanup_guardian_root_exited' : 'cleanup_guardian_root_running');
|
|
179
|
+
return observed;
|
|
180
|
+
},
|
|
139
181
|
};
|
|
140
182
|
}
|
|
@@ -13,6 +13,19 @@ export const STARTUP_OBSERVATIONS = Object.freeze([
|
|
|
13
13
|
'native_stdin_read', 'native_model_catalog_failed', 'native_cloud_policy_failed', 'native_cloud_policy_timeout',
|
|
14
14
|
'native_required_mcp_failed', 'native_config_failed', 'native_auth_refresh_unsupported',
|
|
15
15
|
'local_mcp_request_received', 'local_mcp_auth_rejected', 'local_mcp_origin_or_host_rejected', 'local_mcp_initialize_received',
|
|
16
|
+
// Unconfirmed executor cleanup: which fixed step failed, or was still pending at its deadline.
|
|
17
|
+
'cleanup_owner_stop_failed', 'cleanup_owner_stop_pending', 'cleanup_security_revoke_failed', 'cleanup_tools_revoke_failed',
|
|
18
|
+
'cleanup_tools_close_failed', 'cleanup_tools_close_pending', 'cleanup_pending_handle_failed', 'cleanup_pending_handle_pending',
|
|
19
|
+
'cleanup_official_exec_close_failed', 'cleanup_official_exec_close_pending',
|
|
20
|
+
'cleanup_exec_tools_revoke_failed', 'cleanup_job_tools_revoke_failed', 'cleanup_stdin_end_failed', 'cleanup_process_terminate_failed',
|
|
21
|
+
'cleanup_tool_server_close_failed', 'cleanup_timeout',
|
|
22
|
+
// The owned guardian's own account of an unconfirmed cleanup (fixed words, no PIDs or paths).
|
|
23
|
+
'cleanup_guardian_terminate_written', 'cleanup_guardian_terminate_not_written', 'cleanup_guardian_host_timeout', 'cleanup_guardian_protocol_failed',
|
|
24
|
+
'cleanup_guardian_trigger_terminate', 'cleanup_guardian_trigger_root_exit', 'cleanup_guardian_trigger_lost', 'cleanup_guardian_trigger_output_error', 'cleanup_guardian_trigger_spawn_error',
|
|
25
|
+
'cleanup_guardian_term_sent', 'cleanup_guardian_term_eperm', 'cleanup_guardian_term_esrch', 'cleanup_guardian_term_not_needed',
|
|
26
|
+
'cleanup_guardian_kill_sent', 'cleanup_guardian_kill_eperm', 'cleanup_guardian_kill_esrch', 'cleanup_guardian_kill_not_needed',
|
|
27
|
+
'cleanup_guardian_group_alive', 'cleanup_guardian_group_gone', 'cleanup_guardian_pipes_closed', 'cleanup_guardian_pipes_open',
|
|
28
|
+
'cleanup_guardian_root_exited', 'cleanup_guardian_root_running',
|
|
16
29
|
]);
|
|
17
30
|
const issued = new WeakMap();
|
|
18
31
|
export function executionFailure(error, stage, fallback = 'EXECUTION_FAILED') {
|
|
@@ -42,6 +55,20 @@ export function executionFailure(error, stage, fallback = 'EXECUTION_FAILED') {
|
|
|
42
55
|
}
|
|
43
56
|
return { stage, code: fallback };
|
|
44
57
|
}
|
|
58
|
+
/** An unconfirmed cleanup keeps its fixed message for callers and carries only fixed
|
|
59
|
+
* `cleanup_*` observations, never the underlying error, path or provider output. */
|
|
60
|
+
export function unconfirmedCleanupError(message, observations) {
|
|
61
|
+
const seen = new Set(observations);
|
|
62
|
+
const fixed = STARTUP_OBSERVATIONS.filter(value => value.startsWith('cleanup_') && seen.has(value));
|
|
63
|
+
const error = new Error(message);
|
|
64
|
+
issued.set(error, Object.freeze({ stage: 'executor_cleanup', code: 'EXECUTION_FAILED', ...(fixed.length ? { observations: fixed } : {}) }));
|
|
65
|
+
return error;
|
|
66
|
+
}
|
|
67
|
+
/** The fixed cleanup observations an inner owner issued, for an outer owner to carry. */
|
|
68
|
+
export function cleanupObservations(error) {
|
|
69
|
+
const trusted = error && typeof error === 'object' ? issued.get(error) : undefined;
|
|
70
|
+
return (trusted?.observations ?? []).filter(value => value.startsWith('cleanup_'));
|
|
71
|
+
}
|
|
45
72
|
export function executionError(error, stage, fallback, detail, observations, preparationTiming, launchObservation) {
|
|
46
73
|
const diagnostic = executionFailure(error, stage, fallback);
|
|
47
74
|
// Only trusted callers supply this fixed vocabulary. Serialized provider
|