humanish 0.137.0 → 0.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/cli/commands/runs.js +2 -2
- package/dist/cli/doctor.js +24 -0
- package/dist/keys/key-resolution.js +53 -2
- package/dist/routes/computer-use/bundle-parts.d.ts +3 -0
- package/dist/routes/computer-use/bundle-parts.js +9 -1
- package/dist/routes/computer-use/bundle.js +3 -4
- package/dist/routes/computer-use/e2b-desktop/prepare.js +8 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -2
- package/dist/routes/computer-use/participant-deps.d.ts +1 -1
- package/dist/routes/computer-use/plan.js +66 -19
- package/dist/routes/computer-use/setup.js +14 -7
- package/dist/routes/computer-use/types.d.ts +3 -0
- package/dist/routes/terminal/live-sandbox.d.ts +2 -0
- package/dist/routes/terminal/live-sandbox.js +8 -3
- package/dist/routes/terminal/plan.js +49 -6
- package/dist/routes/terminal/session.js +8 -1
- package/dist/run/bundle-shape.js +1 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/cost-summary.js +9 -4
- package/dist/run/feedback-shape.js +1 -0
- package/dist/run/pricing.d.ts +11 -0
- package/dist/run/pricing.js +18 -5
- package/dist/run/reclaim-outcomes.d.ts +4 -1
- package/dist/run/reclaim-outcomes.js +8 -6
- package/dist/run/reclaim.d.ts +4 -1
- package/dist/run/reclaim.js +81 -34
- package/dist/run/sandbox-creates.d.ts +7 -4
- package/dist/run/sandbox-creates.js +17 -9
- package/dist/run/sandbox-receipts.d.ts +13 -5
- package/dist/run/sandbox-receipts.js +18 -6
- package/dist/study/composition-rules.js +14 -12
- package/dist/study/doctor.js +1 -1
- package/dist/study/parse/execution.js +6 -2
- package/dist/study/plan-base.d.ts +1 -0
- package/dist/study/plan-base.js +2 -0
- package/dist/study/plan-types.d.ts +6 -1
- package/dist/study/requirements.js +7 -2
- package/dist/study/routing.d.ts +1 -1
- package/dist/study/routing.js +1 -1
- package/dist/study/types.d.ts +1 -1
- package/dist/study/warnings.js +3 -1
- package/dist/substrates/e2b/desktop-resources.d.ts +4 -2
- package/dist/substrates/e2b/desktop-resources.js +2 -2
- package/dist/substrates/e2b/peer-install.js +1 -1
- package/dist/substrates/e2b/sandbox.d.ts +3 -0
- package/dist/substrates/e2b/sandbox.js +5 -4
- package/dist/substrates/e2b/sdk.d.ts +5 -0
- package/dist/substrates/e2b/sdk.js +12 -0
- package/dist/substrates/modal/commands.d.ts +16 -0
- package/dist/substrates/modal/commands.js +174 -0
- package/dist/substrates/modal/desktop.d.ts +16 -0
- package/dist/substrates/modal/desktop.js +195 -0
- package/dist/substrates/modal/image.d.ts +11 -0
- package/dist/substrates/modal/image.js +71 -0
- package/dist/substrates/modal/sandbox.d.ts +30 -0
- package/dist/substrates/modal/sandbox.js +325 -0
- package/dist/substrates/modal/sdk.d.ts +112 -0
- package/dist/substrates/modal/sdk.js +61 -0
- package/docs/architecture/terminal-product-route.md +1 -1
- package/docs/contracts/run-bundle.md +5 -2
- package/docs/contracts/schemas.md +8 -7
- package/package.json +8 -2
|
@@ -9,6 +9,7 @@ import { isNonEmpty, planBase } from "../../study/plan-base.js";
|
|
|
9
9
|
import { studyUrlCredentialReason } from "../../study/url-credentials.js";
|
|
10
10
|
import { desktopMediaValidationReason, networkCaptureValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
11
11
|
import { ceilingAdvice } from "../../substrates/e2b/lifetime.js";
|
|
12
|
+
import { MODAL_MAX_SANDBOX_MS, MODAL_TOKEN_ID_ENV, MODAL_TOKEN_SECRET_ENV, } from "../../substrates/modal/sdk.js";
|
|
12
13
|
import { NODE_BOOTSTRAP_TIMEOUT_MS } from "../../subject/node-bootstrap.js";
|
|
13
14
|
import { terminalSandboxTimeoutMs } from "./lifetime.js";
|
|
14
15
|
import { isExactRuntimeVersion, TERMINAL_RUNTIME_VERSION_TIMEOUT_MS } from "./runtime.js";
|
|
@@ -34,6 +35,10 @@ export function planTerminalStudy(config, input) {
|
|
|
34
35
|
const runtimeAuth = config.execution?.runtimeAuth;
|
|
35
36
|
if (runtimeAuth !== undefined && runtimeAuth !== "openai-egress" && runtimeAuth !== "openai-env")
|
|
36
37
|
return refuse("HUMANISH_TERMINAL_CREDENTIAL_DENIED", "`execution.runtimeAuth` must be openai-egress or openai-env. Leave it out for openai-egress, which keeps the OpenAI key outside the sandbox.");
|
|
38
|
+
const sandboxProvider = config.execution?.target === "modal-terminal" ? "modal" : "e2b";
|
|
39
|
+
const modal = sandboxProvider === "modal" ? modalRefusal(config) : undefined;
|
|
40
|
+
if (modal !== undefined)
|
|
41
|
+
return refuse(modal.code, modal.message);
|
|
37
42
|
const analysis = resolveAutomaticAnalysis(config.review?.analysis);
|
|
38
43
|
if (!analysis.ok)
|
|
39
44
|
return refuse("HUMANISH_STUDY_ANALYSIS_INVALID", analysis.message);
|
|
@@ -64,6 +69,7 @@ export function planTerminalStudy(config, input) {
|
|
|
64
69
|
const shared = {
|
|
65
70
|
...base,
|
|
66
71
|
route: "terminal",
|
|
72
|
+
sandboxProvider,
|
|
67
73
|
actor: descriptor.id,
|
|
68
74
|
product: { ...product, publicSurfaces: surfaces },
|
|
69
75
|
...(actor?.persona === undefined ? {} : { personaId: actor.persona }),
|
|
@@ -107,14 +113,16 @@ export function planTerminalStudy(config, input) {
|
|
|
107
113
|
// The sandbox's timeout covers the steps before the codex command, maxMinutes and the teardown
|
|
108
114
|
// buffer, and E2B refuses a sandbox past the plan's ceiling. Refuse here, with the arithmetic,
|
|
109
115
|
// rather than after the plan prints, from a provider 400 that names neither knob.
|
|
110
|
-
const ceiling =
|
|
116
|
+
const ceiling = sandboxProvider === "modal"
|
|
117
|
+
? { ok: true, ms: MODAL_MAX_SANDBOX_MS, source: "default" }
|
|
118
|
+
: input.sandboxCeiling;
|
|
111
119
|
if (!ceiling.ok)
|
|
112
120
|
return refuse("HUMANISH_TERMINAL_CAPS_INVALID", ceiling.message, descriptor.id);
|
|
113
121
|
const productInstall = shared.product.install !== undefined;
|
|
114
122
|
const sandboxTimeoutMs = terminalSandboxTimeoutMs({ maxMinutes, productInstall });
|
|
115
123
|
if (sandboxTimeoutMs > ceiling.ms) {
|
|
116
124
|
const headroomMinutes = (sandboxTimeoutMs - maxMinutes * 60_000) / 60_000;
|
|
117
|
-
return refuse("HUMANISH_TERMINAL_CAPS_INVALID", `caps.maxMinutes ${maxMinutes} derives a ${sandboxTimeoutMs / 60_000}m sandbox deadline, and a sandbox may not live longer than ${ceiling.ms / 60_000}m. The deadline is maxMinutes plus ${headroomMinutes}m: the Node bootstrap (${NODE_BOOTSTRAP_TIMEOUT_MS / 60_000}m), the runtime version check (${TERMINAL_RUNTIME_VERSION_TIMEOUT_MS / 60_000}m)${productInstall ? `, the product setup (${PRODUCT_SETUP_TIMEOUT_MS / 60_000}m)` : ""} and the teardown buffer (${TERMINAL_SANDBOX_TIMEOUT_BUFFER_MS / 60_000}m). Lower caps.maxMinutes to at most ${ceiling.ms / 60_000 - headroomMinutes}.${ceilingAdvice(sandboxTimeoutMs)} No sandbox was created and the runtime key was not used.`, descriptor.id);
|
|
125
|
+
return refuse("HUMANISH_TERMINAL_CAPS_INVALID", `caps.maxMinutes ${maxMinutes} derives a ${sandboxTimeoutMs / 60_000}m sandbox deadline, and a sandbox may not live longer than ${ceiling.ms / 60_000}m. The deadline is maxMinutes plus ${headroomMinutes}m: the Node bootstrap (${NODE_BOOTSTRAP_TIMEOUT_MS / 60_000}m), the runtime version check (${TERMINAL_RUNTIME_VERSION_TIMEOUT_MS / 60_000}m)${productInstall ? `, the product setup (${PRODUCT_SETUP_TIMEOUT_MS / 60_000}m)` : ""} and the teardown buffer (${TERMINAL_SANDBOX_TIMEOUT_BUFFER_MS / 60_000}m). Lower caps.maxMinutes to at most ${ceiling.ms / 60_000 - headroomMinutes}.${sandboxProvider === "modal" ? " Modal ends every sandbox after 24 hours." : ceilingAdvice(sandboxTimeoutMs)} No sandbox was created and the runtime key was not used.`, descriptor.id);
|
|
118
126
|
}
|
|
119
127
|
return {
|
|
120
128
|
ok: true,
|
|
@@ -122,10 +130,45 @@ export function planTerminalStudy(config, input) {
|
|
|
122
130
|
...shared,
|
|
123
131
|
dryRun: false,
|
|
124
132
|
caps: { ...caps, maxUsd, maxMinutes },
|
|
125
|
-
requirements:
|
|
126
|
-
{ kind: "key", name: "E2B_API_KEY" },
|
|
127
|
-
{ kind: "key-one-of", names: ["CODEX_API_KEY", "OPENAI_API_KEY"] },
|
|
128
|
-
],
|
|
133
|
+
requirements: liveRequirements(sandboxProvider),
|
|
129
134
|
},
|
|
130
135
|
};
|
|
131
136
|
}
|
|
137
|
+
/** The sandbox provider's keys, then the runtime key the agent's command gets. */
|
|
138
|
+
function liveRequirements(provider) {
|
|
139
|
+
return [
|
|
140
|
+
...(provider === "modal"
|
|
141
|
+
? [
|
|
142
|
+
{ kind: "key", name: MODAL_TOKEN_ID_ENV },
|
|
143
|
+
{ kind: "key", name: MODAL_TOKEN_SECRET_ENV },
|
|
144
|
+
]
|
|
145
|
+
: [{ kind: "key", name: "E2B_API_KEY" }]),
|
|
146
|
+
{ kind: "key-one-of", names: ["CODEX_API_KEY", "OPENAI_API_KEY"] },
|
|
147
|
+
];
|
|
148
|
+
}
|
|
149
|
+
/** What a terminal study on Modal cannot have that one on E2B can. */
|
|
150
|
+
function modalRefusal(config) {
|
|
151
|
+
// E2B adds the key to requests at its egress proxy for each sandbox. Modal does that only from a
|
|
152
|
+
// Secret stored in the operator's workspace, so humanish does not place the key there itself.
|
|
153
|
+
if (config.execution?.runtimeAuth !== "openai-env")
|
|
154
|
+
return {
|
|
155
|
+
code: "HUMANISH_TERMINAL_CREDENTIAL_DENIED",
|
|
156
|
+
message: "On Modal, humanish cannot keep the OpenAI key outside the sandbox: Modal adds a key to outbound requests only from a Secret stored in your Modal workspace. Set `execution.runtimeAuth: openai-env`, which passes the key to the codex command only, or use `execution.target: e2b-terminal`.",
|
|
157
|
+
};
|
|
158
|
+
const egress = modalEgressReason(config.execution?.egressAllow);
|
|
159
|
+
return egress === undefined
|
|
160
|
+
? undefined
|
|
161
|
+
: { code: "HUMANISH_TERMINAL_SUBJECT_INVALID", message: egress };
|
|
162
|
+
}
|
|
163
|
+
const CIDR = /^[0-9a-f:.]+\/\d{1,3}$/i;
|
|
164
|
+
/**
|
|
165
|
+
* Why a declared egress allowlist cannot run on Modal, or undefined when it can. Modal limits
|
|
166
|
+
* egress to listed domains or CIDR ranges; an entry that is neither, such as a bare IP address,
|
|
167
|
+
* has no Modal form.
|
|
168
|
+
*/
|
|
169
|
+
function modalEgressReason(egressAllow) {
|
|
170
|
+
const unsupported = (egressAllow ?? []).filter((entry) => !CIDR.test(entry) && !/^(\*\.)?([a-z0-9-]+\.)+[a-z0-9-]*[a-z][a-z0-9-]*$/i.test(entry));
|
|
171
|
+
return unsupported.length === 0
|
|
172
|
+
? undefined
|
|
173
|
+
: `Modal limits egress to domain names and CIDR ranges, and \`execution.egressAllow\` lists ${unsupported.map((entry) => `"${entry}"`).join(", ")}. Write a bare address as a /32 range, or use \`execution.target: e2b-terminal\`.`;
|
|
174
|
+
}
|
|
@@ -95,6 +95,7 @@ export async function runLiveTerminalSession(args) {
|
|
|
95
95
|
verdictNonce,
|
|
96
96
|
maxMinutes,
|
|
97
97
|
e2bApiKey,
|
|
98
|
+
env,
|
|
98
99
|
runPaths,
|
|
99
100
|
metadata: buildSandboxMetadata({ studyId: plan.studyId, recordId: "sim-001", runId }),
|
|
100
101
|
warnings,
|
|
@@ -160,7 +161,13 @@ async function prepareLivePrompt(args) {
|
|
|
160
161
|
// anything persists (a key has no detectable "shape" if it is an arbitrary token); redactText is
|
|
161
162
|
// the second pass for secret-shaped content. Applied pre-truncation so a cut can never split a
|
|
162
163
|
// value past the scrubber.
|
|
163
|
-
|
|
164
|
+
// The sandbox provider's credentials too: a provider error that quotes one reaches the run.
|
|
165
|
+
const secrets = new RunSecrets([
|
|
166
|
+
runtimeEnv.keyValue,
|
|
167
|
+
env.E2B_API_KEY?.trim() ?? "",
|
|
168
|
+
env.MODAL_TOKEN_ID?.trim() ?? "",
|
|
169
|
+
env.MODAL_TOKEN_SECRET?.trim() ?? "",
|
|
170
|
+
]);
|
|
164
171
|
const sanitize = (text) => redactText(secrets.scrub(text));
|
|
165
172
|
const persona = terminalPersonaRef(terminalPersona, promptDigest, secrets.scrub);
|
|
166
173
|
return {
|
package/dist/run/bundle-shape.js
CHANGED
|
@@ -93,7 +93,7 @@ export function isRunOutcome(value) {
|
|
|
93
93
|
function isRunProviderResource(value) {
|
|
94
94
|
return (isRecord(value) &&
|
|
95
95
|
value.schema === "humanish.provider-resource.v1" &&
|
|
96
|
-
value.provider === "e2b-desktop" &&
|
|
96
|
+
(value.provider === "e2b-desktop" || value.provider === "modal-desktop") &&
|
|
97
97
|
value.kind === "sandbox" &&
|
|
98
98
|
typeof value.id === "string" &&
|
|
99
99
|
value.id.trim().length > 0 &&
|
package/dist/run/bundle.d.ts
CHANGED
|
@@ -53,7 +53,7 @@ export interface RunFeedbackCandidate {
|
|
|
53
53
|
scenario_id: string;
|
|
54
54
|
persona_id: string;
|
|
55
55
|
actor: "codex-tui" | "codex-exec" | "codex-app-server" | "computer-use" | "synthetic-dry-run" | "unknown";
|
|
56
|
-
substrate: "e2b-desktop" | "local-desktop" | "e2b-terminal" | "local-filesystem" | "codex-app-server" | "unknown";
|
|
56
|
+
substrate: "e2b-desktop" | "modal-desktop" | "local-desktop" | "e2b-terminal" | "local-filesystem" | "codex-app-server" | "unknown";
|
|
57
57
|
failure_owner: "harness" | "target-app" | "actor" | "environment" | "unknown";
|
|
58
58
|
summary: string;
|
|
59
59
|
expected: string;
|
|
@@ -441,7 +441,8 @@ export interface RunCostLine {
|
|
|
441
441
|
cpuCount: number;
|
|
442
442
|
memoryMiB: number;
|
|
443
443
|
};
|
|
444
|
-
|
|
444
|
+
/** E2B's report of the allocation, or the size humanish asked Modal for. */
|
|
445
|
+
resourceSource?: "e2b.getInfo" | "modal.create";
|
|
445
446
|
resourceUnavailableReason?: "metadata_unavailable" | "metadata_invalid" | "metadata_timeout";
|
|
446
447
|
usdPerSecond?: number;
|
|
447
448
|
};
|
|
@@ -478,7 +479,8 @@ export interface RunCostSummary {
|
|
|
478
479
|
}
|
|
479
480
|
export interface RunProviderResource {
|
|
480
481
|
schema: "humanish.provider-resource.v1";
|
|
481
|
-
provider:
|
|
482
|
+
/** The hosted desktop's provider: E2B, or Modal for `execution.target: modal-desktop`. */
|
|
483
|
+
provider: "e2b-desktop" | "modal-desktop";
|
|
482
484
|
kind: "sandbox";
|
|
483
485
|
/**
|
|
484
486
|
* "[redacted-sandbox-id]" in runs from 0.110 on; earlier runs recorded the raw id here. The raw id
|
package/dist/run/cost-summary.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { estimateActorCostForExecution, estimateAllocatedDesktopCost, estimateDesktopCost, round6, } from "./pricing.js";
|
|
1
|
+
import { estimateActorCostForExecution, estimateAllocatedDesktopCost, estimateDesktopCost, MODAL_SANDBOX_RESOURCE_RATE, round6, } from "./pricing.js";
|
|
2
2
|
import {} from "./bundle.js";
|
|
3
3
|
export function buildRunCostSummary(args) {
|
|
4
4
|
const breakdown = [...modelCostLines(args.participants), ...desktopCostLines(args)];
|
|
@@ -76,8 +76,11 @@ function desktopCostLines(args) {
|
|
|
76
76
|
const breakdown = [];
|
|
77
77
|
for (const usage of args.desktops ?? []) {
|
|
78
78
|
const observation = usage.observation;
|
|
79
|
-
const
|
|
80
|
-
const
|
|
79
|
+
const observed = observation && "resources" in observation ? observation : undefined;
|
|
80
|
+
const resources = observed?.resources;
|
|
81
|
+
const estimate = observed?.source === "modal.create"
|
|
82
|
+
? estimateAllocatedDesktopCost(usage.minutes, resources, MODAL_SANDBOX_RESOURCE_RATE)
|
|
83
|
+
: estimateAllocatedDesktopCost(usage.minutes, resources);
|
|
81
84
|
breakdown.push({
|
|
82
85
|
kind: "desktop-minutes",
|
|
83
86
|
...(usage.participantId === undefined ? {} : { laneId: usage.participantId }),
|
|
@@ -88,7 +91,9 @@ function desktopCostLines(args) {
|
|
|
88
91
|
desktop: {
|
|
89
92
|
minutes: estimate.minutes,
|
|
90
93
|
durationBasis: "host-acquired-to-cleanup",
|
|
91
|
-
...(
|
|
94
|
+
...(observed === undefined
|
|
95
|
+
? {}
|
|
96
|
+
: { resources: observed.resources, resourceSource: observed.source }),
|
|
92
97
|
...(observation && "reason" in observation
|
|
93
98
|
? { resourceUnavailableReason: observation.reason }
|
|
94
99
|
: {}),
|
package/dist/run/pricing.d.ts
CHANGED
|
@@ -50,6 +50,12 @@ export interface DesktopResourceRate {
|
|
|
50
50
|
usdPerGiBSecond: number;
|
|
51
51
|
asOf: string;
|
|
52
52
|
source: string;
|
|
53
|
+
/** The sizes the public sheet prices; a size outside them stays unpriced. Absent: any size. */
|
|
54
|
+
sheet?: {
|
|
55
|
+
cpuCounts: readonly number[];
|
|
56
|
+
minMemoryMiB: number;
|
|
57
|
+
maxMemoryMiB: number;
|
|
58
|
+
};
|
|
53
59
|
}
|
|
54
60
|
/**
|
|
55
61
|
* The token-derived cost estimate for one actor trace. `estimatedCostUsd: null` = declared absent
|
|
@@ -104,6 +110,11 @@ export interface DesktopCostEstimate {
|
|
|
104
110
|
usdPerSecond?: number;
|
|
105
111
|
}
|
|
106
112
|
export declare const MODEL_RATES: Record<string, ModelRate>;
|
|
113
|
+
/**
|
|
114
|
+
* Modal's Sandbox rate, per vCPU: Modal prices a physical core, $0.00003942 a second, and counts
|
|
115
|
+
* it as two vCPU. Modal prices every size the same way.
|
|
116
|
+
*/
|
|
117
|
+
export declare const MODAL_SANDBOX_RESOURCE_RATE: DesktopResourceRate;
|
|
107
118
|
/**
|
|
108
119
|
* The refusal for a spend cap on a model this table cannot price. It names the priced models,
|
|
109
120
|
* since an npm install cannot edit this file.
|
package/dist/run/pricing.js
CHANGED
|
@@ -91,6 +91,19 @@ const DESKTOP_RESOURCE_RATE = {
|
|
|
91
91
|
asOf: "2026-09-05",
|
|
92
92
|
// A source label rather than an executable URL; runtime URL redaction protects E2B streams.
|
|
93
93
|
source: "e2b.dev/pricing",
|
|
94
|
+
// The public Hobby/Pro sheet stops at 8 CPU / 8 GiB; larger allocations may have negotiated
|
|
95
|
+
// prices, so their resource evidence is kept without extrapolating a standard rate to them.
|
|
96
|
+
sheet: { cpuCounts: [1, 2, 4, 6, 8], minMemoryMiB: 512, maxMemoryMiB: 8192 },
|
|
97
|
+
};
|
|
98
|
+
/**
|
|
99
|
+
* Modal's Sandbox rate, per vCPU: Modal prices a physical core, $0.00003942 a second, and counts
|
|
100
|
+
* it as two vCPU. Modal prices every size the same way.
|
|
101
|
+
*/
|
|
102
|
+
export const MODAL_SANDBOX_RESOURCE_RATE = {
|
|
103
|
+
usdPerCpuSecond: 0.00003942 / 2,
|
|
104
|
+
usdPerGiBSecond: 0.00000667,
|
|
105
|
+
asOf: "2026-10-19",
|
|
106
|
+
source: "modal.com/pricing",
|
|
94
107
|
};
|
|
95
108
|
// Planning assumption behind the public estimateDesktopCost helper: stock desktops observed on
|
|
96
109
|
// 2026-09-05 had 8 vCPU / 8 GiB. This is the largest CPU/RAM combination on that public sheet, not
|
|
@@ -131,11 +144,11 @@ export function estimateAllocatedDesktopCost(minutes, resources, rate = DESKTOP_
|
|
|
131
144
|
minutes: round6(minutes),
|
|
132
145
|
};
|
|
133
146
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
147
|
+
const { sheet } = rate;
|
|
148
|
+
if ((sheet !== undefined &&
|
|
149
|
+
(!sheet.cpuCounts.includes(resources.cpuCount) ||
|
|
150
|
+
resources.memoryMiB < sheet.minMemoryMiB ||
|
|
151
|
+
resources.memoryMiB > sheet.maxMemoryMiB)) ||
|
|
139
152
|
!Number.isFinite(rate.usdPerCpuSecond) ||
|
|
140
153
|
rate.usdPerCpuSecond < 0 ||
|
|
141
154
|
!Number.isFinite(rate.usdPerGiBSecond) ||
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { type E2BTagSearch } from "../substrates/e2b/sandbox.js";
|
|
2
2
|
import type { E2BAccount } from "../substrates/e2b/connection.js";
|
|
3
|
+
import { type SandboxProviderId } from "./sandbox-receipts.js";
|
|
4
|
+
/** The account reclaim reaches each provider through; absent when it could not be set up. */
|
|
5
|
+
export type ProviderAccounts = Partial<Record<SandboxProviderId, E2BAccount>>;
|
|
3
6
|
import { type ContainedRefusal, type PreparedOutputRoot } from "./contained-output.js";
|
|
4
7
|
export declare const RECLAIM_RESULT_SCHEMA = "humanish.reclaim-result.v1";
|
|
5
8
|
export declare const RECLAIM_RECEIPT_ARTIFACT = "reclaim-receipt.json";
|
|
@@ -50,7 +53,7 @@ export interface Target {
|
|
|
50
53
|
* kills leaves a record of each one not yet confirmed gone.
|
|
51
54
|
*/
|
|
52
55
|
export declare function targetSet(options: {
|
|
53
|
-
|
|
56
|
+
accounts: ProviderAccounts;
|
|
54
57
|
check: boolean;
|
|
55
58
|
requestTimeoutMs: number;
|
|
56
59
|
earlier: ReclaimOutcome[];
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// receipt writer. src/run/reclaim.ts decides what to find; this module records what happened.
|
|
5
5
|
import { REDACTED_SANDBOX_ID, redactText, sandboxIdDigest, toErrorMessage, } from "../evidence/redaction.js";
|
|
6
6
|
import { destroyE2BSandbox, findE2BSandboxesByTags, inspectE2BSandbox, } from "../substrates/e2b/sandbox.js";
|
|
7
|
+
import { isSandboxProviderId } from "./sandbox-receipts.js";
|
|
7
8
|
import { readContainedRegularFile, RUN_ARTIFACT_MAX_BYTES, writeContainedOutputFile, } from "./contained-output.js";
|
|
8
9
|
import { scrubSandboxIds } from "./sandbox-ids.js";
|
|
9
10
|
import { isRecord } from "./type-guards.js";
|
|
@@ -15,7 +16,7 @@ export const RECLAIM_RECEIPT_ARTIFACT = "reclaim-receipt.json";
|
|
|
15
16
|
* kills leaves a record of each one not yet confirmed gone.
|
|
16
17
|
*/
|
|
17
18
|
export function targetSet(options) {
|
|
18
|
-
const {
|
|
19
|
+
const { accounts, check, requestTimeoutMs, earlier, persist } = options;
|
|
19
20
|
const targets = new Map();
|
|
20
21
|
const work = [];
|
|
21
22
|
const queued = [];
|
|
@@ -25,7 +26,7 @@ export function targetSet(options) {
|
|
|
25
26
|
const seen = new Set(current.map((outcome) => outcome.sandboxIdDigest));
|
|
26
27
|
return [...current, ...carried.filter((outcome) => !seen.has(outcome.sandboxIdDigest))];
|
|
27
28
|
};
|
|
28
|
-
const act = (target) => actOn(
|
|
29
|
+
const act = (target) => actOn(accounts, target, { check, requestTimeoutMs }).then((settled) => {
|
|
29
30
|
target.outcome = settled;
|
|
30
31
|
void persist.write(snapshot(), "unknown");
|
|
31
32
|
});
|
|
@@ -95,17 +96,18 @@ export async function searchTagSets(e2b, tagSets, options) {
|
|
|
95
96
|
};
|
|
96
97
|
}
|
|
97
98
|
/** Kill (or look up) one sandbox, and say what happened in reclaim's vocabulary. */
|
|
98
|
-
async function actOn(
|
|
99
|
+
async function actOn(accounts, target, options) {
|
|
99
100
|
const { outcome, provider, rawId } = target;
|
|
100
|
-
|
|
101
|
+
const account = isSandboxProviderId(provider) ? accounts[provider] : undefined;
|
|
102
|
+
if (account === undefined)
|
|
101
103
|
return {
|
|
102
104
|
...outcome,
|
|
103
105
|
state: "unsupported-provider",
|
|
104
106
|
detail: `provider ${JSON.stringify(provider)} is not supported by this humanish version; nothing was attempted, and that provider's own timeout is the backstop`,
|
|
105
107
|
};
|
|
106
108
|
const attempt = options.check
|
|
107
|
-
? await inspectE2BSandbox(
|
|
108
|
-
: await destroyE2BSandbox(
|
|
109
|
+
? await inspectE2BSandbox(account, rawId, options)
|
|
110
|
+
: await destroyE2BSandbox(account, rawId, options);
|
|
109
111
|
const { detail: _previous, ...rest } = outcome;
|
|
110
112
|
// A provider error that quotes the id is scrubbed; the raw id stays in sandbox-receipts.ndjson.
|
|
111
113
|
return "detail" in attempt
|
package/dist/run/reclaim.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type E2BAccount } from "../substrates/e2b/connection.js";
|
|
1
2
|
import { type E2BDesktopModule } from "../substrates/e2b/sdk.js";
|
|
2
3
|
import { type StoppedSandboxCreates } from "./sandbox-creates.js";
|
|
3
4
|
import { RECLAIM_RECEIPT_ARTIFACT, RECLAIM_RESULT_SCHEMA, type ReclaimOutcome, type ReclaimState, type ReclaimTagSearch } from "./reclaim-outcomes.js";
|
|
@@ -33,13 +34,15 @@ export interface ReclaimResult {
|
|
|
33
34
|
export interface ReclaimHooks {
|
|
34
35
|
/** Tests inject a fake @e2b/desktop module; default loads the real one. */
|
|
35
36
|
loadModule?: () => Promise<E2BDesktopModule>;
|
|
37
|
+
/** Tests inject a fake Modal account; default reads the Modal credentials from `env`. */
|
|
38
|
+
loadModalAccount?: (env: Readonly<Record<string, string | undefined>>) => Promise<E2BAccount>;
|
|
36
39
|
/**
|
|
37
40
|
* The env whose E2B_API_KEY and E2B_DOMAIN every E2B call carries. Defaults to process.env, where
|
|
38
41
|
* the CLI put the run's key before the run started.
|
|
39
42
|
*/
|
|
40
43
|
env?: Readonly<Record<string, string | undefined>>;
|
|
41
44
|
requestTimeoutMs?: number;
|
|
42
|
-
/** Ask
|
|
45
|
+
/** Ask the provider whether each sandbox still exists, and kill nothing. */
|
|
43
46
|
check?: boolean;
|
|
44
47
|
/** The signal handler's view of this run's creates: the ids they reported, and those still
|
|
45
48
|
* in flight, whose sandboxes are killed as their ids arrive. */
|
package/dist/run/reclaim.js
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
// `humanish reclaim`: stop an interrupted run's sandboxes and report what happened to each. It
|
|
2
2
|
// finds them three ways: the exact ids the run journaled at create time (sandbox-receipts.ndjson),
|
|
3
3
|
// the ids a create in this process reported before its receipt landed (the signal handler only),
|
|
4
|
-
// and
|
|
5
|
-
// sandbox whose id never reached the run.
|
|
4
|
+
// and each provider's list of sandboxes tagged with the run's owner tags (sandboxOwnerTags), which
|
|
5
|
+
// covers a sandbox whose id never reached the run. Every sandbox is acted on through the provider
|
|
6
|
+
// its receipt or owner line names: E2B, or Modal (src/substrates/modal/sandbox.ts). The list is filtered server-side by the run's tags and
|
|
6
7
|
// each result is checked against every tag again, so it can only return this run's sandboxes; an
|
|
7
8
|
// account-wide operation once destroyed unrelated infrastructure. Kills run concurrently, and
|
|
8
9
|
// reclaim-receipt.json is written before the first kill and after each one, so an exit at any
|
|
9
|
-
// point leaves a record of every sandbox not yet confirmed gone. `--check` asks
|
|
10
|
-
// questions and kills nothing.
|
|
10
|
+
// point leaves a record of every sandbox not yet confirmed gone. `--check` asks the providers the
|
|
11
|
+
// same questions and kills nothing.
|
|
11
12
|
import { e2bAccount, e2bConnection } from "../substrates/e2b/connection.js";
|
|
13
|
+
import { modalAccount } from "../substrates/modal/sandbox.js";
|
|
12
14
|
import { E2B_DEBUG_ENV, e2bDebugMode, loadE2BDesktopModule, } from "../substrates/e2b/sdk.js";
|
|
13
15
|
import { readContainedRegularFile, refusalText, RUN_ARTIFACT_MAX_BYTES, } from "./contained-output.js";
|
|
14
16
|
import { discardPreflightJournal, listPreflightJournals, preflightReclaimDecision, } from "./preflight-receipts.js";
|
|
@@ -261,6 +263,45 @@ function unreadableMessage(file, refusal) {
|
|
|
261
263
|
const kept = file === SANDBOX_RECEIPTS_ARTIFACT ? "the receipts were kept" : "the receipt was kept";
|
|
262
264
|
return `${refusalText(file, refusal)}. Nothing was killed and ${kept}; check the file, then run reclaim again.`;
|
|
263
265
|
}
|
|
266
|
+
/**
|
|
267
|
+
* The account for each provider the run's journal names, or the message that stops reclaim when
|
|
268
|
+
* one it needs cannot be reached. A run with no record and creates in flight is an E2B run unless
|
|
269
|
+
* its owner line says otherwise, as every run was before Modal.
|
|
270
|
+
*/
|
|
271
|
+
async function reachProviders(receipts, owners, hooks) {
|
|
272
|
+
const env = hooks.env ?? process.env;
|
|
273
|
+
const recorded = new Set([
|
|
274
|
+
...receipts.map((receipt) => receipt.provider),
|
|
275
|
+
...owners.map((owner) => owner.provider),
|
|
276
|
+
]);
|
|
277
|
+
// A create this process began counts too: its receipt and owner line may not have landed.
|
|
278
|
+
for (const provider of hooks.creates?.providers() ?? [])
|
|
279
|
+
recorded.add(provider);
|
|
280
|
+
const needsE2B = recorded.has("e2b") || (hooks.creates !== undefined && !recorded.has("modal"));
|
|
281
|
+
// E2B's SDK lists by tag as well as killing by id, so it is needed even with no receipt. Without
|
|
282
|
+
// it, an E2B receipt cannot be acted on; with no E2B receipt, only the tag search is lost.
|
|
283
|
+
const accounts = {};
|
|
284
|
+
const unavailable = {};
|
|
285
|
+
try {
|
|
286
|
+
const module = await (hooks.loadModule ?? loadE2BDesktopModule)();
|
|
287
|
+
accounts.e2b = e2bAccount(module, e2bConnection(env));
|
|
288
|
+
}
|
|
289
|
+
catch (error) {
|
|
290
|
+
unavailable.e2b = `Cannot load @e2b/desktop to reach E2B: ${redactText(toErrorMessage(error))}`;
|
|
291
|
+
if (needsE2B)
|
|
292
|
+
return unavailable.e2b;
|
|
293
|
+
}
|
|
294
|
+
// Modal is reached only for a run that recorded a Modal sandbox or owner line.
|
|
295
|
+
if (recorded.has("modal")) {
|
|
296
|
+
try {
|
|
297
|
+
accounts.modal = await (hooks.loadModalAccount ?? modalAccount)(env);
|
|
298
|
+
}
|
|
299
|
+
catch (error) {
|
|
300
|
+
return `Cannot reach Modal: ${redactText(toErrorMessage(error))}`;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return { accounts, unavailable, recorded, needsE2B };
|
|
304
|
+
}
|
|
264
305
|
/**
|
|
265
306
|
* Find and stop (or, for `--check`, look up) every sandbox one directory owns: its receipts, the
|
|
266
307
|
* creates this process reported, and E2B's tagged list, each once by exact id.
|
|
@@ -277,19 +318,10 @@ async function reclaimRoot(root, label, hooks, warnings) {
|
|
|
277
318
|
const owners = parseSandboxOwners(journal, label);
|
|
278
319
|
const check = hooks.check === true;
|
|
279
320
|
const requestTimeoutMs = hooks.requestTimeoutMs ?? 60_000;
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
try {
|
|
285
|
-
const module = await (hooks.loadModule ?? loadE2BDesktopModule)();
|
|
286
|
-
e2b = e2bAccount(module, e2bConnection(hooks.env ?? process.env));
|
|
287
|
-
}
|
|
288
|
-
catch (error) {
|
|
289
|
-
moduleError = `Cannot load @e2b/desktop to reach E2B: ${redactText(toErrorMessage(error))}`;
|
|
290
|
-
if (receipts.some((receipt) => receipt.provider === "e2b") || hooks.creates !== undefined)
|
|
291
|
-
return { kind: "module-unavailable", receiptCount: receipts.length, message: moduleError };
|
|
292
|
-
}
|
|
321
|
+
const reached = await reachProviders(receipts, owners, hooks);
|
|
322
|
+
if (typeof reached === "string")
|
|
323
|
+
return { kind: "module-unavailable", receiptCount: receipts.length, message: reached };
|
|
324
|
+
const { accounts, unavailable, recorded, needsE2B } = reached;
|
|
293
325
|
// An earlier reclaim may have killed a sandbox no receipt names (the signal handler's creates
|
|
294
326
|
// and tag finds). Its outcome is carried forward, so a later reclaim never erases that record.
|
|
295
327
|
const earlier = check ? [] : await earlierOutcomes(root, label);
|
|
@@ -297,7 +329,7 @@ async function reclaimRoot(root, label, hooks, warnings) {
|
|
|
297
329
|
if (!Array.isArray(earlier))
|
|
298
330
|
return { kind: "unreadable", message: unreadableMessage(RECLAIM_RECEIPT_ARTIFACT, earlier) };
|
|
299
331
|
const set = targetSet({
|
|
300
|
-
|
|
332
|
+
accounts,
|
|
301
333
|
check,
|
|
302
334
|
requestTimeoutMs,
|
|
303
335
|
earlier,
|
|
@@ -312,38 +344,53 @@ async function reclaimRoot(root, label, hooks, warnings) {
|
|
|
312
344
|
if (replaying)
|
|
313
345
|
replayed.set(sandbox.sandboxId, sandbox);
|
|
314
346
|
else
|
|
315
|
-
set.add(
|
|
347
|
+
set.add(sandbox.provider, sandbox.sandboxId, sandbox.participantId, "create", sandbox.released);
|
|
316
348
|
});
|
|
317
349
|
replaying = false;
|
|
318
350
|
for (const receipt of receipts) {
|
|
319
|
-
const
|
|
351
|
+
const create = replayed.get(receipt.sandboxId);
|
|
352
|
+
const released = create?.provider === receipt.provider && create.released;
|
|
320
353
|
set.add(receipt.provider, receipt.sandboxId, receipt.laneId, "receipt", released === true);
|
|
321
354
|
}
|
|
322
355
|
for (const sandbox of replayed.values())
|
|
323
|
-
set.add(
|
|
356
|
+
set.add(sandbox.provider, sandbox.sandboxId, sandbox.participantId, "create", sandbox.released);
|
|
324
357
|
await set.start();
|
|
325
358
|
if (hooks.creates !== undefined)
|
|
326
359
|
await settledWithin(hooks.creates.settled, hooks.createsWaitMs);
|
|
327
|
-
//
|
|
328
|
-
// run directory computes other tags than the ones its
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
360
|
+
// Each provider is searched with the tags the run recorded for it before each create, and with
|
|
361
|
+
// this directory's own: a copied or moved run directory computes other tags than the ones its
|
|
362
|
+
// sandboxes carry. E2B is searched unless the journal names only Modal, as before Modal existed.
|
|
363
|
+
const providers = [
|
|
364
|
+
...(needsE2B || !recorded.has("modal") ? ["e2b"] : []),
|
|
365
|
+
...(accounts.modal === undefined ? [] : ["modal"]),
|
|
366
|
+
];
|
|
367
|
+
const searches = await Promise.all(providers.map(async (provider) => {
|
|
368
|
+
const tagSets = [
|
|
369
|
+
...owners.filter((owner) => owner.provider === provider).map((owner) => owner.tags),
|
|
370
|
+
sandboxOwnerTags(root),
|
|
371
|
+
];
|
|
372
|
+
const result = await searchTagSets(accounts[provider], tagSets, {
|
|
373
|
+
requestTimeoutMs,
|
|
374
|
+
unavailable: unavailable[provider] ?? `no ${provider} account`,
|
|
375
|
+
});
|
|
376
|
+
return { provider, result };
|
|
377
|
+
}));
|
|
333
378
|
let found = 0;
|
|
334
|
-
for (const
|
|
335
|
-
const
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
379
|
+
for (const { provider, result } of searches)
|
|
380
|
+
for (const listed of result.sandboxes) {
|
|
381
|
+
const metadata = listed.metadata ?? {};
|
|
382
|
+
const participant = metadata.participantId ?? metadata.kind ?? "unknown";
|
|
383
|
+
if (set.add(provider, listed.sandboxId, participant, "tag"))
|
|
384
|
+
found += 1;
|
|
385
|
+
}
|
|
386
|
+
const search = searches.find(({ result }) => result.status !== "done")?.result ?? searches[0].result;
|
|
340
387
|
const tagSearch = {
|
|
341
388
|
status: search.status,
|
|
342
389
|
found,
|
|
343
390
|
...(search.detail === undefined ? {} : { detail: search.detail }),
|
|
344
391
|
};
|
|
345
392
|
if (search.status !== "done")
|
|
346
|
-
warnings.push(`Could not search
|
|
393
|
+
warnings.push(`Could not search for sandboxes tagged with ${label}: ${search.detail ?? search.status}. A sandbox whose id never reached a receipt would not be found.`);
|
|
347
394
|
// A create that reports its id while the search or a kill is still running is killed too.
|
|
348
395
|
await set.drain();
|
|
349
396
|
unwatch?.();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { PreparedOutputRoot } from "./contained-output.js";
|
|
2
|
-
import type { SandboxOwnerTags } from "./sandbox-receipts.js";
|
|
2
|
+
import type { SandboxOwnerTags, SandboxProviderId } from "./sandbox-receipts.js";
|
|
3
3
|
/** Refuses a sandbox create for a run directory whose process is stopping. */
|
|
4
4
|
export declare class SandboxCreateRefusedError extends Error {
|
|
5
5
|
constructor();
|
|
@@ -7,6 +7,7 @@ export declare class SandboxCreateRefusedError extends Error {
|
|
|
7
7
|
/** One sandbox a create in this process reported, and what its route did with it. */
|
|
8
8
|
export interface CreatedSandbox {
|
|
9
9
|
sandboxId: string;
|
|
10
|
+
provider: SandboxProviderId;
|
|
10
11
|
/** The participant or subject label the receipt also records. */
|
|
11
12
|
participantId: string;
|
|
12
13
|
/** True once the route's own release confirmed the sandbox gone. */
|
|
@@ -32,14 +33,14 @@ export interface SandboxCreateTicket {
|
|
|
32
33
|
settled(): void;
|
|
33
34
|
}
|
|
34
35
|
/** Register a create for `root`. Throws SandboxCreateRefusedError once the directory is stopping. */
|
|
35
|
-
export declare function beginSandboxCreate(root: PreparedOutputRoot, participantId: string): SandboxCreateTicket;
|
|
36
|
+
export declare function beginSandboxCreate(root: PreparedOutputRoot, participantId: string, provider: SandboxProviderId): SandboxCreateTicket;
|
|
36
37
|
/**
|
|
37
|
-
* Run `write` once per directory and return its promise to every caller. Creates that start
|
|
38
|
+
* Run `write` once per directory and provider and return its promise to every caller. Creates that start
|
|
38
39
|
* together then resume in the order they called, so the order they reach the provider does not
|
|
39
40
|
* depend on which file append finishes first. A write that reports failure is forgotten, so the
|
|
40
41
|
* next create tries again.
|
|
41
42
|
*/
|
|
42
|
-
export declare function recordSandboxOwnerOnce(root: PreparedOutputRoot, write: () => Promise<boolean>): Promise<void>;
|
|
43
|
+
export declare function recordSandboxOwnerOnce(root: PreparedOutputRoot, provider: SandboxProviderId, write: () => Promise<boolean>): Promise<void>;
|
|
43
44
|
/** What the signal handler holds after stopping a directory's creates. */
|
|
44
45
|
export interface StoppedSandboxCreates {
|
|
45
46
|
/**
|
|
@@ -51,6 +52,8 @@ export interface StoppedSandboxCreates {
|
|
|
51
52
|
settled: Promise<void>;
|
|
52
53
|
/** How many of those creates have not settled yet. */
|
|
53
54
|
inFlight(): number;
|
|
55
|
+
/** The providers creates began on, including those whose id has not arrived. */
|
|
56
|
+
providers(): ReadonlySet<SandboxProviderId>;
|
|
54
57
|
}
|
|
55
58
|
/**
|
|
56
59
|
* Stop `root`'s creates: any create that starts from now on is refused. Called synchronously when
|
|
@@ -36,6 +36,8 @@ function directoryFor(root) {
|
|
|
36
36
|
return existing;
|
|
37
37
|
const created = {
|
|
38
38
|
stopping: false,
|
|
39
|
+
ownerLines: new Map(),
|
|
40
|
+
providers: new Set(),
|
|
39
41
|
created: new Map(),
|
|
40
42
|
inFlight: new Set(),
|
|
41
43
|
listeners: new Set(),
|
|
@@ -73,10 +75,11 @@ export function sandboxOwnerTags(root) {
|
|
|
73
75
|
return { tool: "humanish", runId: path.basename(where), runKey };
|
|
74
76
|
}
|
|
75
77
|
/** Register a create for `root`. Throws SandboxCreateRefusedError once the directory is stopping. */
|
|
76
|
-
export function beginSandboxCreate(root, participantId) {
|
|
78
|
+
export function beginSandboxCreate(root, participantId, provider) {
|
|
77
79
|
const directory = directoryFor(root);
|
|
78
80
|
if (directory.stopping)
|
|
79
81
|
throw new SandboxCreateRefusedError();
|
|
82
|
+
directory.providers.add(provider);
|
|
80
83
|
let settle;
|
|
81
84
|
const done = new Promise((resolve) => {
|
|
82
85
|
settle = resolve;
|
|
@@ -87,7 +90,7 @@ export function beginSandboxCreate(root, participantId) {
|
|
|
87
90
|
created(sandboxId) {
|
|
88
91
|
if (directory.created.has(sandboxId))
|
|
89
92
|
return;
|
|
90
|
-
const sandbox = { sandboxId, participantId, released: false };
|
|
93
|
+
const sandbox = { sandboxId, provider, participantId, released: false };
|
|
91
94
|
directory.created.set(sandboxId, sandbox);
|
|
92
95
|
for (const listener of directory.listeners)
|
|
93
96
|
listener(sandbox);
|
|
@@ -104,17 +107,21 @@ export function beginSandboxCreate(root, participantId) {
|
|
|
104
107
|
};
|
|
105
108
|
}
|
|
106
109
|
/**
|
|
107
|
-
* Run `write` once per directory and return its promise to every caller. Creates that start
|
|
110
|
+
* Run `write` once per directory and provider and return its promise to every caller. Creates that start
|
|
108
111
|
* together then resume in the order they called, so the order they reach the provider does not
|
|
109
112
|
* depend on which file append finishes first. A write that reports failure is forgotten, so the
|
|
110
113
|
* next create tries again.
|
|
111
114
|
*/
|
|
112
|
-
export function recordSandboxOwnerOnce(root, write) {
|
|
113
|
-
const
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
115
|
+
export function recordSandboxOwnerOnce(root, provider, write) {
|
|
116
|
+
const { ownerLines } = directoryFor(root);
|
|
117
|
+
const existing = ownerLines.get(provider);
|
|
118
|
+
if (existing !== undefined)
|
|
119
|
+
return existing;
|
|
120
|
+
const pending = write().then((written) => {
|
|
121
|
+
if (!written && ownerLines.get(provider) === pending)
|
|
122
|
+
ownerLines.delete(provider);
|
|
123
|
+
});
|
|
124
|
+
ownerLines.set(provider, pending);
|
|
118
125
|
return pending;
|
|
119
126
|
}
|
|
120
127
|
/**
|
|
@@ -136,5 +143,6 @@ export function stopSandboxCreates(root) {
|
|
|
136
143
|
},
|
|
137
144
|
settled: Promise.all(pending).then(() => undefined),
|
|
138
145
|
inFlight: () => pending.filter((create) => directory.inFlight.has(create)).length,
|
|
146
|
+
providers: () => directory.providers,
|
|
139
147
|
};
|
|
140
148
|
}
|