@opengeni/runtime 4.6.0 → 4.7.0-canary.37090756480001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-instructions/modules/media.d.ts +3 -0
- package/dist/anthropic-messages.d.ts +13 -3
- package/dist/anthropic-request-error.d.ts +10 -0
- package/dist/assets/codemode-client.json +1 -1
- package/dist/{chunk-A22BKN4P.js → chunk-LJNCGUMB.js} +3 -3
- package/dist/chunk-LJNCGUMB.js.map +1 -0
- package/dist/{chunk-GTWKWJ67.js → chunk-P2OVGKDG.js} +505 -100
- package/dist/chunk-P2OVGKDG.js.map +1 -0
- package/dist/{chunk-D5DPVA2C.js → chunk-R25L27T3.js} +2047 -1275
- package/dist/chunk-R25L27T3.js.map +1 -0
- package/dist/claude-subscription-usage.d.ts +14 -4
- package/dist/context-compaction.d.ts +7 -7
- package/dist/index.d.ts +2 -0
- package/dist/index.js +13 -3
- package/dist/interaction-tools.d.ts +1 -1
- package/dist/mcp-network.js +1 -1
- package/dist/runtime-skills.d.ts +1 -0
- package/dist/sandbox/channel-a.d.ts +3 -0
- package/dist/sandbox/index.d.ts +3 -1
- package/dist/sandbox/index.js +7 -1
- package/dist/sandbox/provider-command-session.d.ts +21 -0
- package/dist/sandbox/providers/modal-command-argv.d.ts +4 -0
- package/dist/sandbox/providers/modal-command-control.d.ts +17 -5
- package/dist/sandbox/providers/modal-command-observation-errors.d.ts +3 -0
- package/dist/sandbox/providers/modal-command-raw-page.d.ts +100 -0
- package/dist/sandbox/providers/modal-command-router-wire.d.ts +17 -0
- package/dist/sandbox/routing/routing-session.d.ts +10 -0
- package/dist/sandbox/turn-tool-cancellation.d.ts +4 -0
- package/dist/workspace-tool-gateway.js +3 -3
- package/package.json +12 -12
- package/src/agent-instructions/PROMPT_CHANGELOG.md +23 -1
- package/src/agent-instructions/compose.ts +2 -0
- package/src/agent-instructions/modules/media.ts +20 -0
- package/src/agent-instructions/runtime-mechanics.ts +8 -0
- package/src/anthropic-messages.ts +223 -31
- package/src/anthropic-request-error.ts +45 -0
- package/src/bundled_default_skills/opengeni-client/SKILL.md +9 -7
- package/src/bundled_default_skills/opengeni-client/references/agent-recipes.md +1 -1
- package/src/bundled_default_skills/opengeni-client/references/api-workflows.md +61 -0
- package/src/bundled_default_skills/opengeni-client/references/compatibility-and-troubleshooting.md +86 -0
- package/src/bundled_default_skills/opengeni-client/references/data-tools-and-credentials.md +82 -0
- package/src/bundled_default_skills/opengeni-client/references/discovery-and-autonomy.md +32 -25
- package/src/bundled_default_skills/opengeni-client/references/external-users-and-connect.md +17 -0
- package/src/bundled_default_skills/opengeni-client/references/product-shapes-and-ui.md +156 -0
- package/src/bundled_schedule_skills/opengeni-schedules/SKILL.md +56 -0
- package/src/claude-subscription-usage.ts +55 -9
- package/src/context-compaction.ts +16 -9
- package/src/index.ts +85 -18
- package/src/interaction-tools.ts +113 -3
- package/src/lazy-tool-transport.ts +26 -10
- package/src/mcp-network.ts +4 -2
- package/src/model-provider-client.ts +11 -3
- package/src/model-provider-routing.ts +4 -0
- package/src/runtime-skills.ts +8 -0
- package/src/sandbox/channel-a.ts +3 -0
- package/src/sandbox/index.ts +11 -1
- package/src/sandbox/provider-command-session.ts +58 -0
- package/src/sandbox/provider-errors.ts +43 -9
- package/src/sandbox/providers/modal-command-argv.ts +20 -0
- package/src/sandbox/providers/modal-command-control.ts +422 -116
- package/src/sandbox/providers/modal-command-observation-errors.ts +65 -0
- package/src/sandbox/providers/modal-command-raw-page.ts +147 -0
- package/src/sandbox/providers/modal-command-router-wire.ts +104 -12
- package/src/sandbox/providers/modal-command-session.ts +26 -4
- package/src/sandbox/providers/modal-legacy-command-control.ts +2 -9
- package/src/sandbox/providers/modal-materialization-verification.ts +70 -6
- package/src/sandbox/routing/routing-session.ts +140 -41
- package/src/sandbox/run-credentials.ts +62 -19
- package/src/sandbox/turn-tool-cancellation.ts +159 -42
- package/dist/chunk-A22BKN4P.js.map +0 -1
- package/dist/chunk-D5DPVA2C.js.map +0 -1
- package/dist/chunk-GTWKWJ67.js.map +0 -1
|
@@ -2,11 +2,11 @@ import { randomUUID, randomBytes } from "node:crypto";
|
|
|
2
2
|
import { posix } from "node:path";
|
|
3
3
|
import { status } from "@grpc/grpc-js";
|
|
4
4
|
import { setTimeout as delay } from "node:timers/promises";
|
|
5
|
-
import {
|
|
5
|
+
import { modalCommandArgv } from "./modal-command-argv";
|
|
6
6
|
import {
|
|
7
7
|
SandboxProviderCommand,
|
|
8
8
|
CommandSupervisionReceipt,
|
|
9
|
-
|
|
9
|
+
ModalRouterProviderCommand,
|
|
10
10
|
} from "@opengeni/contracts";
|
|
11
11
|
import type { ChannelAExecArgs } from "../channel-a";
|
|
12
12
|
import type { ProviderCommandOutput } from "../provider-command-session";
|
|
@@ -15,11 +15,14 @@ import {
|
|
|
15
15
|
markPendingCommandSupervised,
|
|
16
16
|
reserveSupervisedLaunch,
|
|
17
17
|
ProviderCommandStartOutcomeUnknownError,
|
|
18
|
+
ProviderCommandObservationUnavailableError,
|
|
18
19
|
} from "../provider-command-session";
|
|
20
|
+
import { isModalCommandObservationTransportError } from "./modal-command-observation-errors";
|
|
21
|
+
import { ModalCommandStartOutcomeUnknownError } from "./modal-command-start-errors";
|
|
22
|
+
import { classifyProviderSandboxFailure } from "../provider-errors";
|
|
19
23
|
import {
|
|
20
24
|
ModalCommandControl as LegacyControl,
|
|
21
25
|
commandControlPlane,
|
|
22
|
-
decodePage,
|
|
23
26
|
} from "./modal-legacy-command-control";
|
|
24
27
|
import {
|
|
25
28
|
ModalCommandRouterWire,
|
|
@@ -27,6 +30,7 @@ import {
|
|
|
27
30
|
ModalCommandStartRejectedError,
|
|
28
31
|
ModalCommandStartNotDispatchedError,
|
|
29
32
|
} from "./modal-command-router-wire";
|
|
33
|
+
import { collectModalRawOutputPage, type ModalRawOutputPage } from "./modal-command-raw-page";
|
|
30
34
|
|
|
31
35
|
export { modalCommandAbortMiddleware } from "./modal-legacy-command-control";
|
|
32
36
|
export type ModalProviderCommand = SandboxProviderCommand;
|
|
@@ -39,11 +43,39 @@ type RouterEntry = {
|
|
|
39
43
|
refreshAt: number;
|
|
40
44
|
idle?: ReturnType<typeof setTimeout>;
|
|
41
45
|
};
|
|
46
|
+
type RouterLookup = { controller: AbortController; waiters: number; settled: boolean };
|
|
47
|
+
type ControlObservation = { command: ModalRouterProviderCommand; output: string };
|
|
48
|
+
type SupervisionControlResult = {
|
|
49
|
+
state: "idle" | "running" | "quiescent";
|
|
50
|
+
receipt?: CommandSupervisionReceipt;
|
|
51
|
+
};
|
|
52
|
+
type ControlHelper = ControlObservation & {
|
|
53
|
+
startPending: boolean;
|
|
54
|
+
startUnknown?: unknown;
|
|
55
|
+
inFlight?: Promise<SupervisionControlResult>;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/** Cancels this waiter, not the shared provider operation. The rejection
|
|
59
|
+
* handler remains attached even when an uncooperative provider settles late. */
|
|
60
|
+
function awaitRouterAccess<T>(pending: Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
61
|
+
if (!signal) return pending;
|
|
62
|
+
signal.throwIfAborted();
|
|
63
|
+
return new Promise<T>((resolve, reject) => {
|
|
64
|
+
const abort = () => {
|
|
65
|
+
signal.removeEventListener("abort", abort);
|
|
66
|
+
reject(signal.reason);
|
|
67
|
+
};
|
|
68
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
69
|
+
void pending.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
|
|
70
|
+
});
|
|
71
|
+
}
|
|
42
72
|
|
|
43
73
|
/** New commands use replayable task-router byte offsets. Legacy identifiers
|
|
44
74
|
* never cross that protocol boundary and are never used for new starts. */
|
|
45
75
|
export class ModalCommandControl {
|
|
46
76
|
private readonly routers = new Map<string, Promise<RouterEntry>>();
|
|
77
|
+
private readonly routerLookups = new Map<Promise<RouterEntry>, RouterLookup>();
|
|
78
|
+
private readonly controlHelpers = new Map<string, ControlHelper>();
|
|
47
79
|
private readonly client: ReturnType<typeof commandControlPlane>;
|
|
48
80
|
private readonly legacy: LegacyControl;
|
|
49
81
|
private closed = false;
|
|
@@ -85,10 +117,15 @@ export class ModalCommandControl {
|
|
|
85
117
|
let pending = this.routers.get(taskId);
|
|
86
118
|
const reused = Boolean(pending);
|
|
87
119
|
if (!pending) {
|
|
120
|
+
const lookup: RouterLookup = {
|
|
121
|
+
controller: new AbortController(),
|
|
122
|
+
waiters: 0,
|
|
123
|
+
settled: false,
|
|
124
|
+
};
|
|
88
125
|
pending = (async () => {
|
|
89
|
-
const access = await
|
|
90
|
-
{ taskId },
|
|
91
|
-
signal
|
|
126
|
+
const access = await awaitRouterAccess(
|
|
127
|
+
this.client.taskGetCommandRouterAccess({ taskId }, { signal: lookup.controller.signal }),
|
|
128
|
+
lookup.controller.signal,
|
|
92
129
|
);
|
|
93
130
|
let refreshAt = Date.now() + 60_000;
|
|
94
131
|
try {
|
|
@@ -104,16 +141,39 @@ export class ModalCommandControl {
|
|
|
104
141
|
return { router: new ModalCommandRouterWire(access), users: 0, refreshAt };
|
|
105
142
|
})();
|
|
106
143
|
this.routers.set(taskId, pending);
|
|
107
|
-
|
|
144
|
+
this.routerLookups.set(pending, lookup);
|
|
145
|
+
const settled = () => {
|
|
146
|
+
lookup.settled = true;
|
|
147
|
+
this.routerLookups.delete(pending!);
|
|
148
|
+
};
|
|
149
|
+
void pending.then(settled, () => {
|
|
150
|
+
settled();
|
|
108
151
|
if (this.routers.get(taskId) === pending) this.routers.delete(taskId);
|
|
109
152
|
});
|
|
110
153
|
}
|
|
111
|
-
const
|
|
154
|
+
const lookup = this.routerLookups.get(pending);
|
|
155
|
+
if (lookup) lookup.waiters++;
|
|
156
|
+
let entry: RouterEntry;
|
|
157
|
+
try {
|
|
158
|
+
entry = await awaitRouterAccess(pending, signal);
|
|
159
|
+
} finally {
|
|
160
|
+
if (lookup && --lookup.waiters === 0 && !lookup.settled) {
|
|
161
|
+
// No interested callers remain. Never cancel a sibling's lookup, and
|
|
162
|
+
// retire only this exact promise if a replacement has already won.
|
|
163
|
+
if (this.routers.get(taskId) === pending) this.routers.delete(taskId);
|
|
164
|
+
lookup.controller.abort(new Error("Modal router access has no remaining waiters"));
|
|
165
|
+
}
|
|
166
|
+
}
|
|
112
167
|
// Refresh credentials between operations, never close another active read.
|
|
113
168
|
if (reused && !entry.users && Date.now() > entry.refreshAt) {
|
|
114
|
-
|
|
115
|
-
entry
|
|
116
|
-
|
|
169
|
+
// Another continuation may already have installed fresh access. Retire
|
|
170
|
+
// only the captured cache entry, then join its replacement rather than
|
|
171
|
+
// deleting or closing the newer transport.
|
|
172
|
+
if (this.routers.get(taskId) === pending) {
|
|
173
|
+
this.routers.delete(taskId);
|
|
174
|
+
clearTimeout(entry.idle);
|
|
175
|
+
entry.router.close();
|
|
176
|
+
}
|
|
117
177
|
return await this.withRouter(taskId, signal, run);
|
|
118
178
|
}
|
|
119
179
|
clearTimeout(entry.idle);
|
|
@@ -137,6 +197,8 @@ export class ModalCommandControl {
|
|
|
137
197
|
|
|
138
198
|
async close(): Promise<void> {
|
|
139
199
|
this.closed = true;
|
|
200
|
+
for (const lookup of this.routerLookups.values())
|
|
201
|
+
lookup.controller.abort(new Error("Modal command control is closed"));
|
|
140
202
|
const entries = await Promise.allSettled(this.routers.values());
|
|
141
203
|
this.routers.clear();
|
|
142
204
|
for (const entry of entries)
|
|
@@ -195,20 +257,7 @@ export class ModalCommandControl {
|
|
|
195
257
|
controlPath: `/tmp/opengeni-supervision/${invocationId}.sock`,
|
|
196
258
|
}
|
|
197
259
|
: undefined;
|
|
198
|
-
let commandArgs =
|
|
199
|
-
args.shell ?? "/bin/sh",
|
|
200
|
-
args.shell && (args.login ?? true) ? "-lc" : "-c",
|
|
201
|
-
args.cmd,
|
|
202
|
-
];
|
|
203
|
-
if (args.runAs) {
|
|
204
|
-
const user = shellQuote(args.runAs),
|
|
205
|
-
invocation = commandArgs.map(shellQuote).join(" ");
|
|
206
|
-
commandArgs = [
|
|
207
|
-
"/bin/sh",
|
|
208
|
-
"-c",
|
|
209
|
-
`if [ "$(id -u)" = ${user} ] || [ "$(id -un 2>/dev/null)" = ${user} ]; then exec ${invocation}; elif [ "$(id -u)" = 0 ]; then exec su -s /bin/sh ${user} -c ${shellQuote(`exec ${invocation}`)}; else exec sudo -n -u ${user} -- ${invocation}; fi`,
|
|
210
|
-
];
|
|
211
|
-
}
|
|
260
|
+
let commandArgs = modalCommandArgv(args);
|
|
212
261
|
const env = typeof this.environment === "function" ? this.environment() : this.environment;
|
|
213
262
|
if (supervision)
|
|
214
263
|
commandArgs = [
|
|
@@ -292,28 +341,55 @@ export class ModalCommandControl {
|
|
|
292
341
|
if (!task.taskId || task.taskResult) throw new Error("Modal command task is unavailable");
|
|
293
342
|
await this.withStartRouter(task.taskId, signal, async (router) => {
|
|
294
343
|
const identity = { taskId: task.taskId!, execId: randomUUID() };
|
|
295
|
-
|
|
296
|
-
{
|
|
344
|
+
const observation: ControlObservation = {
|
|
345
|
+
command: {
|
|
346
|
+
kind: "modal-router-v1",
|
|
347
|
+
sandboxId,
|
|
297
348
|
...identity,
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
349
|
+
streams: {
|
|
350
|
+
stdout: { byteOffset: 0, utf8Remainder: "", eof: false, exitCode: null },
|
|
351
|
+
stderr: { byteOffset: 0, utf8Remainder: "", eof: false, exitCode: null },
|
|
352
|
+
},
|
|
301
353
|
},
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
let
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
354
|
+
output: "",
|
|
355
|
+
};
|
|
356
|
+
let startUnknown: ModalCommandStartOutcomeUnknownError | undefined;
|
|
357
|
+
try {
|
|
358
|
+
await router.start(
|
|
359
|
+
{
|
|
360
|
+
...identity,
|
|
361
|
+
commandArgs: ["/usr/local/bin/opengeni-command-supervisor", "capabilities"],
|
|
362
|
+
workdir: "/tmp",
|
|
363
|
+
env: {},
|
|
364
|
+
},
|
|
365
|
+
signal,
|
|
366
|
+
);
|
|
367
|
+
} catch (error) {
|
|
368
|
+
// A lost acknowledgement does not authorize another Start. This fixed
|
|
369
|
+
// read-only probe can still prove capability by observing its original
|
|
370
|
+
// invocation inside the same five-second budget.
|
|
371
|
+
if (!(error instanceof ModalCommandStartOutcomeUnknownError)) throw error;
|
|
372
|
+
if (error.taskId !== identity.taskId || error.execId !== identity.execId) throw error;
|
|
373
|
+
startUnknown = error;
|
|
316
374
|
}
|
|
375
|
+
let result: { output: string; exit: number };
|
|
376
|
+
try {
|
|
377
|
+
result = await this.readControlOutput(identity, 128, signal, observation);
|
|
378
|
+
} catch (error) {
|
|
379
|
+
if (!startUnknown) throw error;
|
|
380
|
+
// Missing/denied observation cannot erase a genuine unknown Start or
|
|
381
|
+
// turn it into sandbox-loss or replay authority.
|
|
382
|
+
throw new ProviderCommandObservationUnavailableError(
|
|
383
|
+
structuredClone(observation.command),
|
|
384
|
+
new AggregateError(
|
|
385
|
+
[startUnknown, error],
|
|
386
|
+
"Original capability Start and observation remain uncertain",
|
|
387
|
+
),
|
|
388
|
+
error instanceof ProviderCommandObservationUnavailableError && error.readRetryAllowed,
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
signal.throwIfAborted();
|
|
392
|
+
const { output, exit } = result;
|
|
317
393
|
if (exit !== 0 || output !== "native-subreaper-v1")
|
|
318
394
|
throw new Error(
|
|
319
395
|
"Exact Modal instance lacks compatible native supervision; command not admitted",
|
|
@@ -401,50 +477,111 @@ export class ModalCommandControl {
|
|
|
401
477
|
command: ModalRouterProviderCommand,
|
|
402
478
|
action: "release" | "cancel" | "status" | "ack",
|
|
403
479
|
receiptId?: string,
|
|
404
|
-
): Promise<
|
|
480
|
+
): Promise<SupervisionControlResult> {
|
|
405
481
|
SandboxProviderCommand.parse(command);
|
|
406
482
|
const descriptor = command.supervision;
|
|
407
483
|
if (!descriptor || command.sandboxId !== this.sandboxId)
|
|
408
484
|
throw new Error("Supervised command identity is unavailable");
|
|
409
|
-
const
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
485
|
+
const key = JSON.stringify([
|
|
486
|
+
command.sandboxId,
|
|
487
|
+
command.taskId,
|
|
488
|
+
command.execId,
|
|
489
|
+
descriptor.invocationId,
|
|
490
|
+
descriptor.nonce,
|
|
491
|
+
descriptor.controlPath,
|
|
492
|
+
action,
|
|
493
|
+
receiptId ?? null,
|
|
494
|
+
]);
|
|
495
|
+
let helper = this.controlHelpers.get(key);
|
|
496
|
+
if (!helper) {
|
|
497
|
+
helper = {
|
|
498
|
+
startPending: true,
|
|
499
|
+
output: "",
|
|
500
|
+
command: {
|
|
501
|
+
kind: "modal-router-v1",
|
|
502
|
+
sandboxId: command.sandboxId,
|
|
503
|
+
taskId: command.taskId,
|
|
504
|
+
execId: randomUUID(),
|
|
505
|
+
streams: {
|
|
506
|
+
stdout: { byteOffset: 0, utf8Remainder: "", eof: false, exitCode: null },
|
|
507
|
+
stderr: { byteOffset: 0, utf8Remainder: "", eof: false, exitCode: null },
|
|
508
|
+
},
|
|
430
509
|
},
|
|
431
|
-
|
|
432
|
-
);
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
510
|
+
};
|
|
511
|
+
this.controlHelpers.set(key, helper);
|
|
512
|
+
}
|
|
513
|
+
const observation = helper;
|
|
514
|
+
const run = async (): Promise<SupervisionControlResult> => {
|
|
515
|
+
const signal = AbortSignal.timeout(5_000);
|
|
516
|
+
if (observation.startPending) {
|
|
517
|
+
try {
|
|
518
|
+
await this.withStartRouter(command.taskId, signal, async (router) => {
|
|
519
|
+
// Once the wire may dispatch, neither a failed acknowledgement nor
|
|
520
|
+
// a later drain may create another helper invocation.
|
|
521
|
+
observation.startPending = false;
|
|
522
|
+
await router.start(
|
|
523
|
+
{
|
|
524
|
+
taskId: observation.command.taskId,
|
|
525
|
+
execId: observation.command.execId,
|
|
526
|
+
commandArgs: [
|
|
527
|
+
"/usr/local/bin/opengeni-command-supervisor",
|
|
528
|
+
"control",
|
|
529
|
+
"--invocation",
|
|
530
|
+
descriptor.invocationId,
|
|
531
|
+
"--nonce",
|
|
532
|
+
descriptor.nonce,
|
|
533
|
+
"--socket",
|
|
534
|
+
descriptor.controlPath,
|
|
535
|
+
"--action",
|
|
536
|
+
action,
|
|
537
|
+
...(receiptId ? ["--receipt", receiptId] : []),
|
|
538
|
+
],
|
|
539
|
+
workdir: "/tmp",
|
|
540
|
+
env: {},
|
|
541
|
+
},
|
|
542
|
+
signal,
|
|
543
|
+
);
|
|
544
|
+
});
|
|
545
|
+
} catch (error) {
|
|
546
|
+
if (
|
|
547
|
+
observation.startPending ||
|
|
548
|
+
error instanceof ModalCommandStartNotDispatchedError ||
|
|
549
|
+
error instanceof ModalCommandStartRejectedError ||
|
|
550
|
+
error instanceof ModalCommandStartPreDispatchUnavailableError
|
|
551
|
+
) {
|
|
552
|
+
this.controlHelpers.delete(key);
|
|
553
|
+
throw error;
|
|
554
|
+
}
|
|
555
|
+
// Preserve the genuine original Start boundary. A later drain reads
|
|
556
|
+
// this same helper; it never repeats a possibly dispatched Start.
|
|
557
|
+
observation.startUnknown = error;
|
|
558
|
+
throw error;
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
let response: { output: string; exit: number };
|
|
562
|
+
try {
|
|
563
|
+
response = await this.readControlOutput(observation.command, 4096, signal, observation);
|
|
564
|
+
} catch (error) {
|
|
565
|
+
if (
|
|
566
|
+
observation.startUnknown !== undefined &&
|
|
567
|
+
error instanceof ProviderCommandObservationUnavailableError
|
|
568
|
+
)
|
|
569
|
+
throw new ProviderCommandObservationUnavailableError(
|
|
570
|
+
error.command,
|
|
571
|
+
new AggregateError(
|
|
572
|
+
[observation.startUnknown, error],
|
|
573
|
+
"Original helper Start and observation remain uncertain",
|
|
574
|
+
),
|
|
575
|
+
error.readRetryAllowed,
|
|
576
|
+
);
|
|
577
|
+
throw error;
|
|
445
578
|
}
|
|
579
|
+
const { output, exit } = response;
|
|
580
|
+
// Only a complete authenticated helper response retires its private
|
|
581
|
+
// cursor/output cache. Uncertainty keeps the same action and UUID.
|
|
582
|
+
this.controlHelpers.delete(key);
|
|
446
583
|
if (exit !== 0) throw new Error("Supervisor control is unavailable");
|
|
447
|
-
const result = JSON.parse(
|
|
584
|
+
const result = JSON.parse(output);
|
|
448
585
|
if (!result || !["idle", "running", "quiescent"].includes(result.state))
|
|
449
586
|
throw new Error("Invalid supervisor control response");
|
|
450
587
|
if (result.receipt !== undefined) {
|
|
@@ -455,7 +592,66 @@ export class ModalCommandControl {
|
|
|
455
592
|
if ((result.state === "quiescent") !== Boolean(result.receipt))
|
|
456
593
|
throw new Error("Supervisor quiescence response lacks its receipt");
|
|
457
594
|
return result;
|
|
595
|
+
};
|
|
596
|
+
observation.inFlight ??= run().finally(() => {
|
|
597
|
+
delete observation.inFlight;
|
|
458
598
|
});
|
|
599
|
+
return await observation.inFlight;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/** These fixed helpers already started once. Observe only their original
|
|
603
|
+
* invocation within the caller's existing five-second budget. */
|
|
604
|
+
private async readControlOutput(
|
|
605
|
+
identity: { taskId: string; execId: string },
|
|
606
|
+
limit: number,
|
|
607
|
+
signal: AbortSignal,
|
|
608
|
+
observation?: ControlObservation,
|
|
609
|
+
): Promise<{ output: string; exit: number }> {
|
|
610
|
+
const state: ControlObservation = observation ?? {
|
|
611
|
+
command: {
|
|
612
|
+
kind: "modal-router-v1",
|
|
613
|
+
sandboxId: this.sandboxId,
|
|
614
|
+
...identity,
|
|
615
|
+
streams: {
|
|
616
|
+
stdout: { byteOffset: 0, utf8Remainder: "", eof: false, exitCode: null },
|
|
617
|
+
stderr: { byteOffset: 0, utf8Remainder: "", eof: false, exitCode: null },
|
|
618
|
+
},
|
|
619
|
+
},
|
|
620
|
+
output: "",
|
|
621
|
+
};
|
|
622
|
+
for (;;) {
|
|
623
|
+
try {
|
|
624
|
+
signal.throwIfAborted();
|
|
625
|
+
const page = await this.read(state.command, 250, signal);
|
|
626
|
+
state.command = page.command as ModalRouterProviderCommand;
|
|
627
|
+
if (
|
|
628
|
+
state.command.streams.stdout.byteOffset + state.command.streams.stderr.byteOffset >
|
|
629
|
+
limit
|
|
630
|
+
)
|
|
631
|
+
throw new Error("Supervisor control response exceeds its bound");
|
|
632
|
+
state.output += page.chunks
|
|
633
|
+
.filter((chunk) => chunk.stream === "stdout")
|
|
634
|
+
.map((chunk) => chunk.text)
|
|
635
|
+
.join("");
|
|
636
|
+
if (page.exitCode !== null) return { output: state.output, exit: page.exitCode };
|
|
637
|
+
} catch (error) {
|
|
638
|
+
if (signal.aborted)
|
|
639
|
+
throw new ProviderCommandObservationUnavailableError(
|
|
640
|
+
structuredClone(state.command),
|
|
641
|
+
error,
|
|
642
|
+
);
|
|
643
|
+
if (
|
|
644
|
+
!(error instanceof ProviderCommandObservationUnavailableError) ||
|
|
645
|
+
!error.readRetryAllowed
|
|
646
|
+
)
|
|
647
|
+
throw error;
|
|
648
|
+
}
|
|
649
|
+
try {
|
|
650
|
+
await delay(100, undefined, { signal });
|
|
651
|
+
} catch (error) {
|
|
652
|
+
throw new ProviderCommandObservationUnavailableError(structuredClone(state.command), error);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
459
655
|
}
|
|
460
656
|
|
|
461
657
|
async read(
|
|
@@ -467,13 +663,145 @@ export class ModalCommandControl {
|
|
|
467
663
|
if (command.sandboxId !== this.sandboxId)
|
|
468
664
|
throw new Error("Modal command does not belong to this sandbox");
|
|
469
665
|
if (command.kind === "modal-control-v1") return await this.legacy.read(command, waitMs, signal);
|
|
470
|
-
|
|
666
|
+
return (await this.readNative(command, waitMs, signal)).output;
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
/** Native capture plumbing only. Raw bytes are acquired by the same joined
|
|
670
|
+
* read/poll path as read(); no legacy locator or new command is admitted. */
|
|
671
|
+
async readRaw(
|
|
672
|
+
command: ModalRouterProviderCommand,
|
|
673
|
+
waitMs: number,
|
|
674
|
+
signal?: AbortSignal,
|
|
675
|
+
): Promise<ModalRawOutputPage> {
|
|
676
|
+
return (await this.readNative(command, waitMs, signal)).raw;
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
private async readNative(
|
|
680
|
+
input: ModalRouterProviderCommand,
|
|
681
|
+
waitMs: number,
|
|
682
|
+
signal?: AbortSignal,
|
|
683
|
+
): Promise<ReturnType<typeof collectModalRawOutputPage>> {
|
|
684
|
+
const command = structuredClone(ModalRouterProviderCommand.parse(input));
|
|
685
|
+
if (command.sandboxId !== this.sandboxId)
|
|
686
|
+
throw new Error("Modal command does not belong to this sandbox");
|
|
687
|
+
if (!Number.isFinite(waitMs) || waitMs < 0 || waitMs > 50_000)
|
|
688
|
+
throw new Error("Invalid Modal output read bounds");
|
|
689
|
+
signal?.throwIfAborted();
|
|
690
|
+
const stdout = command.streams.stdout;
|
|
691
|
+
const stderr = command.streams.stderr;
|
|
692
|
+
if (
|
|
693
|
+
stdout.eof &&
|
|
694
|
+
stderr.eof &&
|
|
695
|
+
stdout.utf8Remainder === "" &&
|
|
696
|
+
stderr.utf8Remainder === "" &&
|
|
697
|
+
stdout.exitCode !== null &&
|
|
698
|
+
stdout.exitCode === stderr.exitCode
|
|
699
|
+
) {
|
|
700
|
+
// Both complete streams and their matching terminal observation were
|
|
701
|
+
// already captured. An expired provider handle cannot revoke that exact
|
|
702
|
+
// evidence; the session still atomically verifies the retained cursor.
|
|
703
|
+
return collectModalRawOutputPage(
|
|
704
|
+
command,
|
|
705
|
+
{
|
|
706
|
+
stdout: { bytes: Buffer.alloc(0), eof: true },
|
|
707
|
+
stderr: { bytes: Buffer.alloc(0), eof: true },
|
|
708
|
+
},
|
|
709
|
+
{ source: "retained_terminal", code: stdout.exitCode },
|
|
710
|
+
);
|
|
711
|
+
}
|
|
712
|
+
const budget = new AbortController();
|
|
713
|
+
const abort = () => budget.abort(signal?.reason);
|
|
714
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
715
|
+
if (signal?.aborted) abort();
|
|
716
|
+
// The router deliberately ends a quiet stream at its read deadline with
|
|
717
|
+
// a partial page. Let that deadline settle before outer containment aborts
|
|
718
|
+
// the whole page, including bytes already read from the other stream.
|
|
719
|
+
// Caller cancellation still bounds this allowance independently.
|
|
720
|
+
const deadline = performance.now() + Math.max(1, waitMs);
|
|
721
|
+
const timeout = setTimeout(
|
|
722
|
+
() => budget.abort(new Error("Modal command read budget exhausted")),
|
|
723
|
+
Math.max(1, waitMs) + 5_000,
|
|
724
|
+
);
|
|
725
|
+
let lastError: unknown;
|
|
726
|
+
try {
|
|
727
|
+
for (let attempt = 0; attempt < 5; attempt++) {
|
|
728
|
+
signal?.throwIfAborted();
|
|
729
|
+
if (attempt > 0 && performance.now() >= deadline) break;
|
|
730
|
+
try {
|
|
731
|
+
return await this.readRouterPage(
|
|
732
|
+
command,
|
|
733
|
+
Math.min(Math.max(1, waitMs), Math.max(1, deadline - performance.now())),
|
|
734
|
+
budget.signal,
|
|
735
|
+
deadline,
|
|
736
|
+
);
|
|
737
|
+
} catch (error) {
|
|
738
|
+
signal?.throwIfAborted();
|
|
739
|
+
if (!budget.signal.aborted && !isModalCommandObservationTransportError(error)) {
|
|
740
|
+
// Containment is not retry permission. A structural provider fault
|
|
741
|
+
// dominates nested missing-handle prose, but mixed/unreadable
|
|
742
|
+
// graphs do not acquire the strict safe-read retry authority.
|
|
743
|
+
if (classifyProviderSandboxFailure("modal", error).kind === "transient_transport")
|
|
744
|
+
throw new ProviderCommandObservationUnavailableError(
|
|
745
|
+
structuredClone(command),
|
|
746
|
+
error,
|
|
747
|
+
false,
|
|
748
|
+
);
|
|
749
|
+
throw error;
|
|
750
|
+
}
|
|
751
|
+
lastError = error;
|
|
752
|
+
if (budget.signal.aborted || performance.now() >= deadline || attempt === 4) break;
|
|
753
|
+
try {
|
|
754
|
+
await delay(Math.min(100, Math.max(1, deadline - performance.now())), undefined, {
|
|
755
|
+
signal: budget.signal,
|
|
756
|
+
});
|
|
757
|
+
} catch {
|
|
758
|
+
signal?.throwIfAborted();
|
|
759
|
+
break;
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
throw new ProviderCommandObservationUnavailableError(structuredClone(command), lastError);
|
|
764
|
+
} finally {
|
|
765
|
+
clearTimeout(timeout);
|
|
766
|
+
signal?.removeEventListener("abort", abort);
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
private async readRouterPage(
|
|
771
|
+
command: ModalRouterProviderCommand,
|
|
772
|
+
waitMs: number,
|
|
773
|
+
signal: AbortSignal,
|
|
774
|
+
deadline: number,
|
|
775
|
+
): Promise<ReturnType<typeof collectModalRawOutputPage>> {
|
|
471
776
|
const cancellation = new AbortController();
|
|
472
|
-
const
|
|
777
|
+
const lookupCancellation = new AbortController();
|
|
778
|
+
const abort = () => {
|
|
779
|
+
cancellation.abort(signal.reason);
|
|
780
|
+
lookupCancellation.abort(signal.reason);
|
|
781
|
+
};
|
|
473
782
|
signal?.addEventListener("abort", abort, { once: true });
|
|
474
783
|
if (signal?.aborted) abort();
|
|
784
|
+
const lookupTimeout = setTimeout(
|
|
785
|
+
() =>
|
|
786
|
+
lookupCancellation.abort(
|
|
787
|
+
new ProviderCommandObservationUnavailableError(
|
|
788
|
+
structuredClone(command),
|
|
789
|
+
new Error("Modal command read access budget exhausted"),
|
|
790
|
+
),
|
|
791
|
+
),
|
|
792
|
+
Math.max(1, deadline - performance.now()),
|
|
793
|
+
);
|
|
475
794
|
try {
|
|
476
|
-
return await this.withRouter(command.taskId,
|
|
795
|
+
return await this.withRouter(command.taskId, lookupCancellation.signal, async (router) => {
|
|
796
|
+
clearTimeout(lookupTimeout);
|
|
797
|
+
lookupCancellation.signal.throwIfAborted();
|
|
798
|
+
signal.throwIfAborted();
|
|
799
|
+
if (performance.now() >= deadline)
|
|
800
|
+
throw new ProviderCommandObservationUnavailableError(
|
|
801
|
+
structuredClone(command),
|
|
802
|
+
new Error("Modal command read budget exhausted"),
|
|
803
|
+
);
|
|
804
|
+
const remainingWait = Math.max(1, Math.min(waitMs, deadline - performance.now()));
|
|
477
805
|
const operations = [
|
|
478
806
|
...(["stdout", "stderr"] as const).map(async (stream) =>
|
|
479
807
|
command.streams[stream].eof
|
|
@@ -482,7 +810,7 @@ export class ModalCommandControl {
|
|
|
482
810
|
command,
|
|
483
811
|
stream,
|
|
484
812
|
command.streams[stream].byteOffset,
|
|
485
|
-
|
|
813
|
+
remainingWait,
|
|
486
814
|
cancellation.signal,
|
|
487
815
|
),
|
|
488
816
|
),
|
|
@@ -497,44 +825,22 @@ export class ModalCommandControl {
|
|
|
497
825
|
),
|
|
498
826
|
);
|
|
499
827
|
const failed = results.find((result) => result.status === "rejected");
|
|
500
|
-
|
|
828
|
+
// Abort-induced sibling rejections may sort before the failing poll.
|
|
829
|
+
// Preserve the first actual provider fault rather than array order.
|
|
830
|
+
if (failed?.status === "rejected") throw cancellation.signal.reason ?? failed.reason;
|
|
501
831
|
const stdout = (results[0] as PromiseFulfilledResult<{ bytes: Buffer; eof: boolean }>)
|
|
502
832
|
.value;
|
|
503
833
|
const stderr = (results[1] as PromiseFulfilledResult<{ bytes: Buffer; eof: boolean }>)
|
|
504
834
|
.value;
|
|
505
835
|
const exit = (results[2] as PromiseFulfilledResult<number | null>).value;
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
const old = command.streams[stream];
|
|
512
|
-
const decoded = decodePage(old.utf8Remainder, [page.bytes], page.eof);
|
|
513
|
-
const byteOffset = old.byteOffset + page.bytes.length;
|
|
514
|
-
if (!Number.isSafeInteger(byteOffset)) throw new Error("Modal output offset exhausted");
|
|
515
|
-
next.streams[stream] = {
|
|
516
|
-
byteOffset,
|
|
517
|
-
utf8Remainder: decoded.remainder,
|
|
518
|
-
eof: page.eof,
|
|
519
|
-
exitCode: page.eof ? exit : null,
|
|
520
|
-
};
|
|
521
|
-
if (decoded.text)
|
|
522
|
-
chunks.push({
|
|
523
|
-
stream,
|
|
524
|
-
chunkId: `modal-router:${command.execId}:${stream}:${old.byteOffset}:${byteOffset}:${page.eof ? 1 : 0}`,
|
|
525
|
-
text: decoded.text,
|
|
526
|
-
});
|
|
527
|
-
}
|
|
528
|
-
return {
|
|
529
|
-
command: next,
|
|
530
|
-
expected: structuredClone(command),
|
|
531
|
-
chunks,
|
|
532
|
-
exitCode: next.streams.stdout.eof && next.streams.stderr.eof ? exit : null,
|
|
533
|
-
providerExited: exit !== null,
|
|
534
|
-
streamFidelity: command.pty ? "merged" : "separate",
|
|
535
|
-
};
|
|
836
|
+
return collectModalRawOutputPage(
|
|
837
|
+
command,
|
|
838
|
+
{ stdout, stderr },
|
|
839
|
+
{ source: "router_poll", code: exit },
|
|
840
|
+
);
|
|
536
841
|
});
|
|
537
842
|
} finally {
|
|
843
|
+
clearTimeout(lookupTimeout);
|
|
538
844
|
signal?.removeEventListener("abort", abort);
|
|
539
845
|
}
|
|
540
846
|
}
|