@cloudflare/sandbox 0.12.4 → 0.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bridge/index.d.ts +30 -0
- package/dist/bridge/index.d.ts.map +1 -1
- package/dist/bridge/index.js +196 -77
- package/dist/bridge/index.js.map +1 -1
- package/dist/{contexts-mjA8ZsDG.d.ts → contexts-C186NnAB.d.ts} +53 -4
- package/dist/contexts-C186NnAB.d.ts.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/openai/index.d.ts +1 -1
- package/dist/opencode/index.d.ts +2 -2
- package/dist/opencode/index.d.ts.map +1 -1
- package/dist/{sandbox-5MHau7vJ.d.ts → sandbox-BtaWcmmG.d.ts} +165 -6
- package/dist/{sandbox-5MHau7vJ.d.ts.map → sandbox-BtaWcmmG.d.ts.map} +1 -1
- package/dist/{sandbox-CyqG4jca.js → sandbox-sU3r5LSr.js} +879 -93
- package/dist/sandbox-sU3r5LSr.js.map +1 -0
- package/package.json +2 -2
- package/dist/contexts-mjA8ZsDG.d.ts.map +0 -1
- package/dist/sandbox-CyqG4jca.js.map +0 -1
|
@@ -3,8 +3,8 @@ import { n as getHttpStatus, r as ErrorCode, t as getSuggestion } from "./errors
|
|
|
3
3
|
import { Container, ContainerProxy, getContainer, switchPort } from "@cloudflare/containers";
|
|
4
4
|
import { AwsClient } from "aws4fetch";
|
|
5
5
|
import { RpcSession, RpcTarget } from "capnweb";
|
|
6
|
+
import { RpcTarget as RpcTarget$1, tracing } from "cloudflare:workers";
|
|
6
7
|
import path from "node:path/posix";
|
|
7
|
-
import { RpcTarget as RpcTarget$1 } from "cloudflare:workers";
|
|
8
8
|
|
|
9
9
|
//#region src/errors/classes.ts
|
|
10
10
|
/**
|
|
@@ -769,19 +769,55 @@ function isRetryableWebSocketUpgradeResponse(response) {
|
|
|
769
769
|
return RETRYABLE_WEBSOCKET_UPGRADE_STATUSES.has(response.status);
|
|
770
770
|
}
|
|
771
771
|
/**
|
|
772
|
+
* Compute the next backoff delay from the remaining retry budget. Returns null
|
|
773
|
+
* when another attempt should not begin.
|
|
774
|
+
*
|
|
775
|
+
* Delay is limited to the budget left after reserving `minTimeForRetryMs` for
|
|
776
|
+
* the subsequent attempt. An attempt already in progress owns its own
|
|
777
|
+
* per-request timeouts via the caller-provided `fetchResponse` function.
|
|
778
|
+
*/
|
|
779
|
+
function nextRetryDelayMs(attempt, remainingMs, minTimeForRetryMs) {
|
|
780
|
+
const availableForDelay = remainingMs - minTimeForRetryMs;
|
|
781
|
+
if (availableForDelay <= 0) return null;
|
|
782
|
+
const computedDelay = Math.min(DEFAULT_INITIAL_RETRY_DELAY_MS * 2 ** attempt, DEFAULT_MAX_RETRY_DELAY_MS);
|
|
783
|
+
return Math.min(computedDelay, availableForDelay);
|
|
784
|
+
}
|
|
785
|
+
/**
|
|
772
786
|
* Retry Response-returning operations while their response remains retryable.
|
|
773
|
-
*
|
|
774
|
-
*
|
|
787
|
+
*
|
|
788
|
+
* The retry budget decides whether another attempt may begin after a retryable
|
|
789
|
+
* response or thrown error. Each attempt owns any per-request timeout inside
|
|
790
|
+
* the caller-provided `fetchResponse` function. Backoff is limited to the
|
|
791
|
+
* remaining budget after reserving `minTimeForRetryMs` for that next attempt.
|
|
775
792
|
*/
|
|
776
793
|
async function fetchWithResponseRetry(fetchResponse, options) {
|
|
777
794
|
const startTime = Date.now();
|
|
778
795
|
let attempt = 0;
|
|
779
796
|
while (true) {
|
|
780
|
-
|
|
797
|
+
let response;
|
|
798
|
+
try {
|
|
799
|
+
response = await fetchResponse();
|
|
800
|
+
} catch (error) {
|
|
801
|
+
if (!options.shouldRetryError?.(error)) throw error;
|
|
802
|
+
const elapsed$1 = Date.now() - startTime;
|
|
803
|
+
const remaining$1 = options.retryTimeoutMs - elapsed$1;
|
|
804
|
+
const delay$1 = nextRetryDelayMs(attempt, remaining$1, options.minTimeForRetryMs);
|
|
805
|
+
if (delay$1 === null) throw error;
|
|
806
|
+
options.logger.info(options.retryLogMessage, {
|
|
807
|
+
attempt: attempt + 1,
|
|
808
|
+
delayMs: delay$1,
|
|
809
|
+
remainingSec: Math.floor(remaining$1 / 1e3),
|
|
810
|
+
error: error instanceof Error ? error.message : String(error)
|
|
811
|
+
});
|
|
812
|
+
await new Promise((resolve) => setTimeout(resolve, delay$1));
|
|
813
|
+
attempt++;
|
|
814
|
+
continue;
|
|
815
|
+
}
|
|
781
816
|
if (!options.shouldRetry(response)) return response;
|
|
782
817
|
const elapsed = Date.now() - startTime;
|
|
783
818
|
const remaining = options.retryTimeoutMs - elapsed;
|
|
784
|
-
|
|
819
|
+
const delay = nextRetryDelayMs(attempt, remaining, options.minTimeForRetryMs);
|
|
820
|
+
if (delay === null) {
|
|
785
821
|
options.onRetryExhausted?.({
|
|
786
822
|
attempts: attempt + 1,
|
|
787
823
|
elapsedMs: elapsed,
|
|
@@ -789,7 +825,6 @@ async function fetchWithResponseRetry(fetchResponse, options) {
|
|
|
789
825
|
});
|
|
790
826
|
return response;
|
|
791
827
|
}
|
|
792
|
-
const delay = Math.min(DEFAULT_INITIAL_RETRY_DELAY_MS * 2 ** attempt, DEFAULT_MAX_RETRY_DELAY_MS);
|
|
793
828
|
options.logger.info(options.retryLogMessage, {
|
|
794
829
|
status: response.status,
|
|
795
830
|
attempt: attempt + 1,
|
|
@@ -2495,6 +2530,52 @@ const DO_STORAGE_STARTUP_RESET_PATTERN = /internal error while starting up durab
|
|
|
2495
2530
|
function errorMessageOf(error) {
|
|
2496
2531
|
return error instanceof Error ? error.message : typeof error === "string" ? error : "";
|
|
2497
2532
|
}
|
|
2533
|
+
/**
|
|
2534
|
+
* Extract a matchable message from any thrown value, reading a `.message`
|
|
2535
|
+
* property even when the value is not an `Error` instance. The Containers
|
|
2536
|
+
* runtime raises admission failures from the container binding, which may
|
|
2537
|
+
* live in a different realm — so `instanceof Error` can be false for a
|
|
2538
|
+
* genuine error. Mirrors the base `@cloudflare/containers` `isErrorOfType`
|
|
2539
|
+
* helper: coerce to a string, then match case-insensitively.
|
|
2540
|
+
*/
|
|
2541
|
+
function realmSafeMessageOf(error) {
|
|
2542
|
+
const message = error?.message;
|
|
2543
|
+
return typeof message === "string" ? message : String(error);
|
|
2544
|
+
}
|
|
2545
|
+
/**
|
|
2546
|
+
* Platform messages emitted by the Containers runtime when it cannot admit a
|
|
2547
|
+
* container for a Durable Object during startup. Matched case-insensitively as
|
|
2548
|
+
* lowercase substrings. Single source of truth for both the RPC connection
|
|
2549
|
+
* path (connection.ts) and the HTTP `containerFetch` path (sandbox.ts).
|
|
2550
|
+
*/
|
|
2551
|
+
const CONTAINER_UNAVAILABLE_SIGNATURES = [
|
|
2552
|
+
{
|
|
2553
|
+
substring: "there is no container instance that can be provided to this durable object",
|
|
2554
|
+
reason: "no_container_instance_available"
|
|
2555
|
+
},
|
|
2556
|
+
{
|
|
2557
|
+
substring: "there is no container instance available at this time",
|
|
2558
|
+
reason: "no_container_instance_available"
|
|
2559
|
+
},
|
|
2560
|
+
{
|
|
2561
|
+
substring: "maximum number of running container instances exceeded",
|
|
2562
|
+
reason: "max_container_instances_exceeded"
|
|
2563
|
+
}
|
|
2564
|
+
];
|
|
2565
|
+
/**
|
|
2566
|
+
* Classify a container-startup error as a platform admission/capacity failure,
|
|
2567
|
+
* returning the categorical reason or null. Realm-safe (does not gate on
|
|
2568
|
+
* `instanceof Error`), case-insensitive, and walks the `.cause` chain so a
|
|
2569
|
+
* wrapped admission failure is still recognized.
|
|
2570
|
+
*/
|
|
2571
|
+
function matchContainerUnavailable(error) {
|
|
2572
|
+
for (const candidate of selfAndCauses(error)) {
|
|
2573
|
+
const text = realmSafeMessageOf(candidate).toLowerCase();
|
|
2574
|
+
const match = CONTAINER_UNAVAILABLE_SIGNATURES.find((sig) => text.includes(sig.substring));
|
|
2575
|
+
if (match) return match.reason;
|
|
2576
|
+
}
|
|
2577
|
+
return null;
|
|
2578
|
+
}
|
|
2498
2579
|
function* selfAndCauses(error) {
|
|
2499
2580
|
let current = error;
|
|
2500
2581
|
for (let depth = 0; depth < 8 && current != null; depth += 1) {
|
|
@@ -3003,6 +3084,123 @@ var RestoreLifecycleRunner = class {
|
|
|
3003
3084
|
}
|
|
3004
3085
|
};
|
|
3005
3086
|
|
|
3087
|
+
//#endregion
|
|
3088
|
+
//#region src/container-control/tracing.ts
|
|
3089
|
+
/**
|
|
3090
|
+
* Thin tracing helper for the RPC control path.
|
|
3091
|
+
*
|
|
3092
|
+
* Wraps `cloudflare:workers` `tracing.enterSpan` so the connection/client code
|
|
3093
|
+
* can emit spans for RPC calls, the connect/disconnect lifecycle, and
|
|
3094
|
+
* individual upgrade attempts — without hard-failing in environments where the
|
|
3095
|
+
* tracing API is unavailable.
|
|
3096
|
+
*
|
|
3097
|
+
* Error convention: the Cloudflare trace UI surfaces `error` and `error.stack`
|
|
3098
|
+
* span attributes specially (this differs from the OpenTelemetry standard,
|
|
3099
|
+
* which uses exception events). We therefore stamp both attributes on the span
|
|
3100
|
+
* when the wrapped work throws, so failures are visible in the GUI.
|
|
3101
|
+
*
|
|
3102
|
+
* Cause chain: the base `@cloudflare/containers` class wraps the true failure
|
|
3103
|
+
* inside `new Error(NO_CONTAINER_INSTANCE_ERROR, { cause })`, so the real
|
|
3104
|
+
* reason (e.g. "the container is not listening", "Network connection lost", a
|
|
3105
|
+
* non-zero exit code) is only visible on `.cause`. Every distinct failure mode
|
|
3106
|
+
* therefore collapses to the same generic top-level message in traces. We walk
|
|
3107
|
+
* the cause chain and stamp it so a trace names the actual root cause.
|
|
3108
|
+
*/
|
|
3109
|
+
/** Bound on cause-chain traversal — guards against cycles and runaway depth. */
|
|
3110
|
+
const MAX_CAUSE_DEPTH = 8;
|
|
3111
|
+
/** Read a string `code` property off an arbitrary value, if present. */
|
|
3112
|
+
function stringCode(value) {
|
|
3113
|
+
const code = value?.code;
|
|
3114
|
+
return typeof code === "string" ? code : void 0;
|
|
3115
|
+
}
|
|
3116
|
+
/** Render any thrown value as a short human-readable message. */
|
|
3117
|
+
function messageOf(value) {
|
|
3118
|
+
return value instanceof Error ? value.message : String(value);
|
|
3119
|
+
}
|
|
3120
|
+
/**
|
|
3121
|
+
* Compute the Cloudflare-convention error span attributes for any thrown value,
|
|
3122
|
+
* including the wrapped `.cause` chain. Pure and side-effect-free so it can be
|
|
3123
|
+
* unit-tested without a live tracer.
|
|
3124
|
+
*
|
|
3125
|
+
* Emits:
|
|
3126
|
+
* - `error` — top-level message (or stringified non-Error)
|
|
3127
|
+
* - `error.stack` — top-level stack, when available
|
|
3128
|
+
* - `error.code` — top-level string `code`, when available
|
|
3129
|
+
* - `error.cause` — immediate cause message, when a cause exists
|
|
3130
|
+
* - `error.cause.code` — immediate cause string `code`, when available
|
|
3131
|
+
* - `error.cause_chain`— all nested cause messages joined by " <- "
|
|
3132
|
+
*/
|
|
3133
|
+
function computeErrorAttributes(error) {
|
|
3134
|
+
const attrs = {};
|
|
3135
|
+
if (error instanceof Error) {
|
|
3136
|
+
attrs.error = error.message;
|
|
3137
|
+
if (typeof error.stack === "string") attrs["error.stack"] = error.stack;
|
|
3138
|
+
const code = stringCode(error);
|
|
3139
|
+
if (code !== void 0) attrs["error.code"] = code;
|
|
3140
|
+
} else {
|
|
3141
|
+
attrs.error = String(error);
|
|
3142
|
+
return attrs;
|
|
3143
|
+
}
|
|
3144
|
+
const chain = [];
|
|
3145
|
+
const seen = new Set([error]);
|
|
3146
|
+
let current = error.cause;
|
|
3147
|
+
let depth = 0;
|
|
3148
|
+
while (current !== void 0 && current !== null && depth < MAX_CAUSE_DEPTH) {
|
|
3149
|
+
if (seen.has(current)) break;
|
|
3150
|
+
seen.add(current);
|
|
3151
|
+
if (depth === 0) {
|
|
3152
|
+
attrs["error.cause"] = messageOf(current);
|
|
3153
|
+
const causeCode = stringCode(current);
|
|
3154
|
+
if (causeCode !== void 0) attrs["error.cause.code"] = causeCode;
|
|
3155
|
+
}
|
|
3156
|
+
chain.push(messageOf(current));
|
|
3157
|
+
current = current instanceof Error ? current.cause : void 0;
|
|
3158
|
+
depth++;
|
|
3159
|
+
}
|
|
3160
|
+
if (chain.length > 0) attrs["error.cause_chain"] = chain.join(" <- ");
|
|
3161
|
+
return attrs;
|
|
3162
|
+
}
|
|
3163
|
+
/**
|
|
3164
|
+
* Stamp the Cloudflare-convention error attributes onto a span. Safe to call
|
|
3165
|
+
* with any thrown value.
|
|
3166
|
+
*/
|
|
3167
|
+
function setErrorAttributes(span, error) {
|
|
3168
|
+
for (const [key, value] of Object.entries(computeErrorAttributes(error))) if (value !== void 0) span.setAttribute(key, value);
|
|
3169
|
+
}
|
|
3170
|
+
/**
|
|
3171
|
+
* Run `fn` inside a span named `name`, applying `attributes` up front and
|
|
3172
|
+
* stamping `error`/`error.stack` if it throws. Re-throws the original error.
|
|
3173
|
+
*
|
|
3174
|
+
* Falls back to running `fn` directly if the tracing API is unavailable, so a
|
|
3175
|
+
* missing tracer never changes behavior.
|
|
3176
|
+
*/
|
|
3177
|
+
async function withSpan(name, attributes, fn) {
|
|
3178
|
+
const enter = tracing?.enterSpan?.bind(tracing);
|
|
3179
|
+
if (typeof enter !== "function") return fn({ setAttribute: () => {} });
|
|
3180
|
+
return enter(name, async (span) => {
|
|
3181
|
+
for (const [key, value] of Object.entries(attributes)) if (value !== void 0) span.setAttribute(key, value);
|
|
3182
|
+
try {
|
|
3183
|
+
return await fn(span);
|
|
3184
|
+
} catch (error) {
|
|
3185
|
+
setErrorAttributes(span, error);
|
|
3186
|
+
throw error;
|
|
3187
|
+
}
|
|
3188
|
+
});
|
|
3189
|
+
}
|
|
3190
|
+
/**
|
|
3191
|
+
* Emit a zero-duration marker span for a lifecycle event (e.g. disconnect).
|
|
3192
|
+
* Best-effort: never throws, never changes behavior.
|
|
3193
|
+
*/
|
|
3194
|
+
function traceEvent(name, attributes) {
|
|
3195
|
+
const enter = tracing?.enterSpan?.bind(tracing);
|
|
3196
|
+
if (typeof enter !== "function") return;
|
|
3197
|
+
try {
|
|
3198
|
+
enter(name, (span) => {
|
|
3199
|
+
for (const [key, value] of Object.entries(attributes)) if (value !== void 0) span.setAttribute(key, value);
|
|
3200
|
+
});
|
|
3201
|
+
} catch {}
|
|
3202
|
+
}
|
|
3203
|
+
|
|
3006
3204
|
//#endregion
|
|
3007
3205
|
//#region src/container-control/connection.ts
|
|
3008
3206
|
/**
|
|
@@ -3015,14 +3213,14 @@ var RestoreLifecycleRunner = class {
|
|
|
3015
3213
|
* holds the original Response.
|
|
3016
3214
|
*/
|
|
3017
3215
|
async function tryParseContainerUnavailable(response) {
|
|
3018
|
-
if (
|
|
3019
|
-
try {
|
|
3216
|
+
if ((response.headers.get("content-type") ?? "").includes("application/json")) try {
|
|
3020
3217
|
const body = await response.clone().json();
|
|
3021
3218
|
if (body.code !== ErrorCode.CONTAINER_UNAVAILABLE) return null;
|
|
3022
3219
|
const context = {
|
|
3023
|
-
reason: body.context?.reason === "container_starting" || body.context?.reason === "container_unhealthy" || body.context?.reason === "container_replaced" || body.context?.reason === "rpc_upgrade_failed" ? body.context.reason : "container_replaced",
|
|
3220
|
+
reason: body.context?.reason === "container_starting" || body.context?.reason === "container_unhealthy" || body.context?.reason === "container_replaced" || body.context?.reason === "rpc_upgrade_failed" || body.context?.reason === "no_container_instance_available" || body.context?.reason === "max_container_instances_exceeded" ? body.context.reason : "container_replaced",
|
|
3024
3221
|
retryable: true,
|
|
3025
|
-
...typeof body.context?.retryAfterMs === "number" && { retryAfterMs: body.context.retryAfterMs }
|
|
3222
|
+
...typeof body.context?.retryAfterMs === "number" && { retryAfterMs: body.context.retryAfterMs },
|
|
3223
|
+
...typeof body.context?.originalMessage === "string" && { originalMessage: body.context.originalMessage }
|
|
3026
3224
|
};
|
|
3027
3225
|
return createErrorFromResponse({
|
|
3028
3226
|
code: ErrorCode.CONTAINER_UNAVAILABLE,
|
|
@@ -3035,6 +3233,53 @@ async function tryParseContainerUnavailable(response) {
|
|
|
3035
3233
|
} catch {
|
|
3036
3234
|
return null;
|
|
3037
3235
|
}
|
|
3236
|
+
try {
|
|
3237
|
+
const text = await response.clone().text();
|
|
3238
|
+
const reason = matchContainerUnavailable(text);
|
|
3239
|
+
if (!reason) return null;
|
|
3240
|
+
return buildContainerUnavailableError(reason, text);
|
|
3241
|
+
} catch {
|
|
3242
|
+
return null;
|
|
3243
|
+
}
|
|
3244
|
+
}
|
|
3245
|
+
/**
|
|
3246
|
+
* True when a thrown connection-startup error matches a known platform
|
|
3247
|
+
* container-admission failure. These are transient: the platform asks the
|
|
3248
|
+
* caller to try again later, so they are safe to retry within the budget.
|
|
3249
|
+
* Delegates to the shared matcher in platform-errors.ts.
|
|
3250
|
+
*/
|
|
3251
|
+
function isPlatformUnavailableError(error) {
|
|
3252
|
+
return matchContainerUnavailable(error) !== null;
|
|
3253
|
+
}
|
|
3254
|
+
/**
|
|
3255
|
+
* Build a typed ContainerUnavailableError for a matched platform
|
|
3256
|
+
* container-admission failure, preserving the original message verbatim.
|
|
3257
|
+
*/
|
|
3258
|
+
function buildContainerUnavailableError(reason, originalMessage, cause) {
|
|
3259
|
+
const context = {
|
|
3260
|
+
reason,
|
|
3261
|
+
retryable: true,
|
|
3262
|
+
originalMessage
|
|
3263
|
+
};
|
|
3264
|
+
return createErrorFromResponse({
|
|
3265
|
+
code: ErrorCode.CONTAINER_UNAVAILABLE,
|
|
3266
|
+
message: originalMessage,
|
|
3267
|
+
context,
|
|
3268
|
+
httpStatus: getHttpStatus(ErrorCode.CONTAINER_UNAVAILABLE),
|
|
3269
|
+
suggestion: getSuggestion(ErrorCode.CONTAINER_UNAVAILABLE, context),
|
|
3270
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
3271
|
+
}, cause !== void 0 ? { cause } : void 0);
|
|
3272
|
+
}
|
|
3273
|
+
/**
|
|
3274
|
+
* Convert a raw connection-startup error into a typed ContainerUnavailableError
|
|
3275
|
+
* when it matches a known platform container-admission failure. Returns null
|
|
3276
|
+
* for anything else so the caller preserves the original error.
|
|
3277
|
+
*/
|
|
3278
|
+
function tryConvertPlatformUnavailable(error) {
|
|
3279
|
+
if (error instanceof SandboxError) return error;
|
|
3280
|
+
const reason = matchContainerUnavailable(error);
|
|
3281
|
+
if (!reason) return null;
|
|
3282
|
+
return buildContainerUnavailableError(reason, error instanceof Error ? error.message : String(error), error);
|
|
3038
3283
|
}
|
|
3039
3284
|
const DEFAULT_CONNECT_TIMEOUT_MS = 3e4;
|
|
3040
3285
|
const DEFAULT_RETRY_TIMEOUT_MS = 12e4;
|
|
@@ -3058,12 +3303,20 @@ var ContainerControlConnection = class {
|
|
|
3058
3303
|
logger;
|
|
3059
3304
|
retryTimeoutMs;
|
|
3060
3305
|
onClose;
|
|
3306
|
+
onConnectionError;
|
|
3307
|
+
onConnected;
|
|
3308
|
+
startContainer;
|
|
3309
|
+
getSandboxInfo;
|
|
3061
3310
|
constructor(options) {
|
|
3062
3311
|
this.containerStub = options.stub;
|
|
3063
3312
|
this.port = options.port ?? 3e3;
|
|
3064
3313
|
this.logger = options.logger ?? createNoOpLogger();
|
|
3065
3314
|
this.retryTimeoutMs = options.retryTimeoutMs ?? DEFAULT_RETRY_TIMEOUT_MS;
|
|
3066
3315
|
this.onClose = options.onClose;
|
|
3316
|
+
this.onConnectionError = options.onConnectionError;
|
|
3317
|
+
this.onConnected = options.onConnected;
|
|
3318
|
+
this.startContainer = options.startContainer;
|
|
3319
|
+
this.getSandboxInfo = options.getSandboxInfo;
|
|
3067
3320
|
this.transport = new DeferredTransport();
|
|
3068
3321
|
this.session = new RpcSession(this.transport, options.localMain);
|
|
3069
3322
|
this.stub = this.session.getRemoteMain();
|
|
@@ -3090,6 +3343,24 @@ var ContainerControlConnection = class {
|
|
|
3090
3343
|
isConnected() {
|
|
3091
3344
|
return this.connected;
|
|
3092
3345
|
}
|
|
3346
|
+
/**
|
|
3347
|
+
* True while a connection attempt is in progress (upgrade not yet
|
|
3348
|
+
* established and not yet failed). Owners use this to avoid tearing the
|
|
3349
|
+
* connection down mid-attempt.
|
|
3350
|
+
*/
|
|
3351
|
+
isConnecting() {
|
|
3352
|
+
return this.connectPromise !== null;
|
|
3353
|
+
}
|
|
3354
|
+
/**
|
|
3355
|
+
* Resolve once any in-flight connection attempt settles (success or
|
|
3356
|
+
* failure). Resolves immediately when no attempt is in progress. Never
|
|
3357
|
+
* rejects — callers only care that the attempt is done.
|
|
3358
|
+
*/
|
|
3359
|
+
async whenSettled() {
|
|
3360
|
+
const pending = this.connectPromise;
|
|
3361
|
+
if (!pending) return;
|
|
3362
|
+
await pending.catch(() => {});
|
|
3363
|
+
}
|
|
3093
3364
|
async connect() {
|
|
3094
3365
|
if (this.connected) return;
|
|
3095
3366
|
if (this.connectPromise) return this.connectPromise;
|
|
@@ -3100,7 +3371,20 @@ var ContainerControlConnection = class {
|
|
|
3100
3371
|
this.connectPromise = null;
|
|
3101
3372
|
}
|
|
3102
3373
|
}
|
|
3103
|
-
|
|
3374
|
+
/**
|
|
3375
|
+
* Tear down the connection. `cause` (a lifecycle teardown reason from the
|
|
3376
|
+
* owning DO) is used only for the disconnect trace event here; the owner
|
|
3377
|
+
* is responsible for stamping it as a *weak* connection cause (one that
|
|
3378
|
+
* never overwrites an authoritative failure already captured from a
|
|
3379
|
+
* connection attempt), since a teardown is usually a downstream
|
|
3380
|
+
* consequence of that earlier failure rather than the root cause.
|
|
3381
|
+
*/
|
|
3382
|
+
disconnect(cause) {
|
|
3383
|
+
if (cause !== void 0) traceEvent("sandbox.rpc.disconnect", {
|
|
3384
|
+
...this.spanAttrs(),
|
|
3385
|
+
reason: cause instanceof Error ? cause.message : String(cause)
|
|
3386
|
+
});
|
|
3387
|
+
else traceEvent("sandbox.rpc.disconnect", this.spanAttrs());
|
|
3104
3388
|
try {
|
|
3105
3389
|
this.stub[Symbol.dispose]?.();
|
|
3106
3390
|
} catch {}
|
|
@@ -3124,6 +3408,19 @@ var ContainerControlConnection = class {
|
|
|
3124
3408
|
this.retryTimeoutMs = ms;
|
|
3125
3409
|
}
|
|
3126
3410
|
/**
|
|
3411
|
+
* Base span attributes for this connection: the container port plus the
|
|
3412
|
+
* sandbox identifiers (`sandbox.id` = DO id, `sandbox.name` = user name)
|
|
3413
|
+
* when available.
|
|
3414
|
+
*/
|
|
3415
|
+
spanAttrs() {
|
|
3416
|
+
const info = this.getSandboxInfo?.();
|
|
3417
|
+
return {
|
|
3418
|
+
"sandbox.rpc.port": this.port,
|
|
3419
|
+
"sandbox.id": info?.id,
|
|
3420
|
+
"sandbox.name": info?.name
|
|
3421
|
+
};
|
|
3422
|
+
}
|
|
3423
|
+
/**
|
|
3127
3424
|
* Run the owner-provided `onClose` callback exactly once per call,
|
|
3128
3425
|
* swallowing any errors so a buggy listener can't keep the connection
|
|
3129
3426
|
* object in a half-torn-down state.
|
|
@@ -3137,6 +3434,18 @@ var ContainerControlConnection = class {
|
|
|
3137
3434
|
}
|
|
3138
3435
|
}
|
|
3139
3436
|
/**
|
|
3437
|
+
* Run the owner-provided `onConnectionError` callback, swallowing any
|
|
3438
|
+
* listener errors.
|
|
3439
|
+
*/
|
|
3440
|
+
fireConnectionError(error) {
|
|
3441
|
+
if (!this.onConnectionError) return;
|
|
3442
|
+
try {
|
|
3443
|
+
this.onConnectionError(error);
|
|
3444
|
+
} catch (err) {
|
|
3445
|
+
this.logger.warn("ContainerControlConnection onConnectionError handler threw", { error: err instanceof Error ? err.message : String(err) });
|
|
3446
|
+
}
|
|
3447
|
+
}
|
|
3448
|
+
/**
|
|
3140
3449
|
* WebSocket `close` listener. Defined as a bound arrow field so the
|
|
3141
3450
|
* same reference can be passed to both `addEventListener` and
|
|
3142
3451
|
* `removeEventListener` — a fresh anonymous lambda would silently
|
|
@@ -3160,6 +3469,9 @@ var ContainerControlConnection = class {
|
|
|
3160
3469
|
if (wasConnected) this.fireOnClose();
|
|
3161
3470
|
};
|
|
3162
3471
|
async doConnect() {
|
|
3472
|
+
return withSpan("sandbox.rpc.connect", this.spanAttrs(), () => this.doConnectInner());
|
|
3473
|
+
}
|
|
3474
|
+
async doConnectInner() {
|
|
3163
3475
|
try {
|
|
3164
3476
|
const response = await this.fetchUpgradeWithRetry();
|
|
3165
3477
|
if (response.status !== 101) {
|
|
@@ -3186,13 +3498,20 @@ var ContainerControlConnection = class {
|
|
|
3186
3498
|
this.ws = ws;
|
|
3187
3499
|
this.transport.activate(ws);
|
|
3188
3500
|
this.connected = true;
|
|
3501
|
+
try {
|
|
3502
|
+
this.onConnected?.();
|
|
3503
|
+
} catch (err) {
|
|
3504
|
+
this.logger.warn("ContainerControlConnection onConnected handler threw", { error: err instanceof Error ? err.message : String(err) });
|
|
3505
|
+
}
|
|
3189
3506
|
this.logger.debug("ContainerControlConnection established", { port: this.port });
|
|
3190
3507
|
} catch (error) {
|
|
3191
3508
|
this.connected = false;
|
|
3192
|
-
|
|
3509
|
+
const connectionError = tryConvertPlatformUnavailable(error) ?? error;
|
|
3510
|
+
this.fireConnectionError(connectionError);
|
|
3511
|
+
this.transport.abort(connectionError);
|
|
3193
3512
|
this.fireOnClose();
|
|
3194
|
-
this.logger.error("ContainerControlConnection failed",
|
|
3195
|
-
throw
|
|
3513
|
+
this.logger.error("ContainerControlConnection failed", connectionError instanceof Error ? connectionError : new Error(String(connectionError)));
|
|
3514
|
+
throw connectionError;
|
|
3196
3515
|
}
|
|
3197
3516
|
}
|
|
3198
3517
|
/**
|
|
@@ -3206,30 +3525,56 @@ var ContainerControlConnection = class {
|
|
|
3206
3525
|
minTimeForRetryMs: MIN_TIME_FOR_RETRY_MS,
|
|
3207
3526
|
logger: this.logger,
|
|
3208
3527
|
retryLogMessage: "ContainerControlConnection upgrade returned retryable status, retrying",
|
|
3209
|
-
shouldRetry: isRetryableWebSocketUpgradeResponse
|
|
3528
|
+
shouldRetry: isRetryableWebSocketUpgradeResponse,
|
|
3529
|
+
shouldRetryError: isPlatformUnavailableError
|
|
3210
3530
|
});
|
|
3211
3531
|
}
|
|
3212
3532
|
/**
|
|
3213
|
-
* Single WebSocket-upgrade fetch attempt.
|
|
3214
|
-
*
|
|
3215
|
-
*
|
|
3533
|
+
* Single WebSocket-upgrade fetch attempt.
|
|
3534
|
+
*
|
|
3535
|
+
* Container start and the upgrade fetch have deliberately separate timeout
|
|
3536
|
+
* scopes:
|
|
3537
|
+
*
|
|
3538
|
+
* - `startContainer()` runs under the base `Container` class's own
|
|
3539
|
+
* instance-get / port-ready budget. It must NOT be cut short by the
|
|
3540
|
+
* per-attempt connect timeout: the base class only emits the
|
|
3541
|
+
* classifiable `NO_CONTAINER_INSTANCE_ERROR` after its internal budget
|
|
3542
|
+
* is exhausted, and it checks its abort signal *before* that throw
|
|
3543
|
+
* (see @cloudflare/containers `doStartContainer`). Aborting it early
|
|
3544
|
+
* yields a generic "Aborted waiting for container to start" error that
|
|
3545
|
+
* we can't classify — which is what masked the real cause as
|
|
3546
|
+
* OPERATION_INTERRUPTED. Letting it run its own budget means the
|
|
3547
|
+
* platform capacity failure throws here, in our retry loop, where
|
|
3548
|
+
* `shouldRetryError` retries it and `doConnect`'s catch converts it to
|
|
3549
|
+
* a typed ContainerUnavailableError.
|
|
3550
|
+
*
|
|
3551
|
+
* - The WebSocket upgrade fetch gets its own fresh AbortController so
|
|
3552
|
+
* each retry has an independent connect timeout for the *upgrade only*.
|
|
3216
3553
|
*/
|
|
3217
3554
|
async fetchUpgradeAttempt() {
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3555
|
+
return withSpan("sandbox.rpc.connect.attempt", this.spanAttrs(), async () => {
|
|
3556
|
+
if (this.startContainer) try {
|
|
3557
|
+
await this.startContainer();
|
|
3558
|
+
} catch (error) {
|
|
3559
|
+
this.fireConnectionError(tryConvertPlatformUnavailable(error) ?? error);
|
|
3560
|
+
throw error;
|
|
3561
|
+
}
|
|
3562
|
+
const controller = new AbortController();
|
|
3563
|
+
const timeout = setTimeout(() => controller.abort(), DEFAULT_CONNECT_TIMEOUT_MS);
|
|
3564
|
+
try {
|
|
3565
|
+
const url = `http://localhost:${this.port}/rpc`;
|
|
3566
|
+
const request = new Request(url, {
|
|
3567
|
+
headers: {
|
|
3568
|
+
Upgrade: "websocket",
|
|
3569
|
+
Connection: "Upgrade"
|
|
3570
|
+
},
|
|
3571
|
+
signal: controller.signal
|
|
3572
|
+
});
|
|
3573
|
+
return await this.containerStub.fetch(request);
|
|
3574
|
+
} finally {
|
|
3575
|
+
clearTimeout(timeout);
|
|
3576
|
+
}
|
|
3577
|
+
});
|
|
3233
3578
|
}
|
|
3234
3579
|
};
|
|
3235
3580
|
/**
|
|
@@ -3350,6 +3695,9 @@ function translateRPCError(error, context = {}) {
|
|
|
3350
3695
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
3351
3696
|
});
|
|
3352
3697
|
const transportResponse = buildTransportErrorResponse(error);
|
|
3698
|
+
const captured = maybePreferConnectionError(transportResponse, context);
|
|
3699
|
+
if (captured) throw captured;
|
|
3700
|
+
if (context.sessionEstablished === false) throw createErrorFromResponse(buildNeverConnectedUnavailableResponse(transportResponse, context) ?? transportResponse, { cause: error });
|
|
3353
3701
|
throw createErrorFromResponse(buildInterruptedOperationResponse(transportResponse, context) ?? transportResponse, { cause: error });
|
|
3354
3702
|
}
|
|
3355
3703
|
throw createErrorFromResponse(buildTransportErrorResponse(new Error(String(error))), { cause: error });
|
|
@@ -3398,6 +3746,146 @@ function buildTransportErrorResponse(error) {
|
|
|
3398
3746
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
3399
3747
|
};
|
|
3400
3748
|
}
|
|
3749
|
+
/**
|
|
3750
|
+
* Extract a recognized structured error shape from a captured connection cause.
|
|
3751
|
+
* Accepts both same-realm SandboxError instances and plain error-like objects
|
|
3752
|
+
* that carry a known `code` plus either `context` or `details`.
|
|
3753
|
+
*/
|
|
3754
|
+
function extractCapturedErrorShape(captured) {
|
|
3755
|
+
if (captured instanceof SandboxError) return {
|
|
3756
|
+
code: captured.code,
|
|
3757
|
+
message: captured.message,
|
|
3758
|
+
context: captured.context ?? {}
|
|
3759
|
+
};
|
|
3760
|
+
const shape = captured;
|
|
3761
|
+
if (!shape || typeof shape.code !== "string" || !Object.hasOwn(ErrorCode, shape.code)) return null;
|
|
3762
|
+
const code = shape.code;
|
|
3763
|
+
const structured = shape.context && typeof shape.context === "object" ? shape.context : shape.details && typeof shape.details === "object" ? shape.details : {};
|
|
3764
|
+
return {
|
|
3765
|
+
code,
|
|
3766
|
+
message: typeof shape.message === "string" ? shape.message : code,
|
|
3767
|
+
context: structured
|
|
3768
|
+
};
|
|
3769
|
+
}
|
|
3770
|
+
const OPERATION_INTERRUPTED_REASONS = new Set([
|
|
3771
|
+
"runtime_replaced",
|
|
3772
|
+
"transport_disposed",
|
|
3773
|
+
"sandbox_lifetime_changed",
|
|
3774
|
+
"recovery_exhausted"
|
|
3775
|
+
]);
|
|
3776
|
+
function asOperationInterruptedReason(value) {
|
|
3777
|
+
return typeof value === "string" && OPERATION_INTERRUPTED_REASONS.has(value) ? value : null;
|
|
3778
|
+
}
|
|
3779
|
+
function asInterruptedAdmitted(value) {
|
|
3780
|
+
if (typeof value === "boolean" || value === "unknown") return value;
|
|
3781
|
+
}
|
|
3782
|
+
/**
|
|
3783
|
+
* Lifecycle disconnect causes are stamped with a generic connection-level
|
|
3784
|
+
* operation identity. When they interrupt a concrete RPC method, rebind the
|
|
3785
|
+
* public interruption context onto that method while preserving the lifecycle
|
|
3786
|
+
* reason, retryability, and stop metadata.
|
|
3787
|
+
*/
|
|
3788
|
+
function rebindInterruptedOperationContext(shape, context, cause) {
|
|
3789
|
+
if (shape.code !== ErrorCode.OPERATION_INTERRUPTED || typeof context.operation !== "string" || context.operation.length === 0) return createErrorFromResponse({
|
|
3790
|
+
code: shape.code,
|
|
3791
|
+
message: shape.message,
|
|
3792
|
+
context: shape.context,
|
|
3793
|
+
httpStatus: getHttpStatus(shape.code),
|
|
3794
|
+
suggestion: getSuggestion(shape.code, shape.context),
|
|
3795
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
3796
|
+
}, { cause });
|
|
3797
|
+
const reason = asOperationInterruptedReason(shape.context.reason) ?? "transport_disposed";
|
|
3798
|
+
const admitted = context.sessionEstablished === true ? true : context.sessionEstablished === false ? false : asInterruptedAdmitted(shape.context.admitted) ?? "unknown";
|
|
3799
|
+
const retryable = typeof shape.context.retryable === "boolean" ? shape.context.retryable : false;
|
|
3800
|
+
const interruptedContext = {
|
|
3801
|
+
reason,
|
|
3802
|
+
operation: context.operation,
|
|
3803
|
+
phase: "rpc_call",
|
|
3804
|
+
admitted,
|
|
3805
|
+
retryable,
|
|
3806
|
+
...typeof shape.context.recoveryAttempts === "number" && { recoveryAttempts: shape.context.recoveryAttempts },
|
|
3807
|
+
...typeof shape.context.maxRecoveryAttempts === "number" && { maxRecoveryAttempts: shape.context.maxRecoveryAttempts },
|
|
3808
|
+
...typeof shape.context.containerExitCode === "number" && { containerExitCode: shape.context.containerExitCode },
|
|
3809
|
+
...typeof shape.context.stopReason === "string" && { stopReason: shape.context.stopReason }
|
|
3810
|
+
};
|
|
3811
|
+
return new OperationInterruptedError({
|
|
3812
|
+
code: ErrorCode.OPERATION_INTERRUPTED,
|
|
3813
|
+
message: shape.message,
|
|
3814
|
+
context: interruptedContext,
|
|
3815
|
+
httpStatus: getHttpStatus(ErrorCode.OPERATION_INTERRUPTED),
|
|
3816
|
+
suggestion: getSuggestion(ErrorCode.OPERATION_INTERRUPTED, interruptedContext),
|
|
3817
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
3818
|
+
}, { cause });
|
|
3819
|
+
}
|
|
3820
|
+
/**
|
|
3821
|
+
* When a queued RPC call rejects with a transport error that a connection
|
|
3822
|
+
* abort could have caused (session disposed, connection failed, upgrade
|
|
3823
|
+
* failed, or peer closed), and the connection captured a real startup or
|
|
3824
|
+
* lifecycle error, return that captured error in preference to the masking
|
|
3825
|
+
* transport error.
|
|
3826
|
+
*
|
|
3827
|
+
* The captured error is accepted whether it is:
|
|
3828
|
+
* - a same-realm `SandboxError`, or
|
|
3829
|
+
* - any structured, error-like value carrying a recognized `code` — e.g. a
|
|
3830
|
+
* raw or cross-realm error decorated with `code: 'CONTAINER_UNAVAILABLE'`
|
|
3831
|
+
* and optional `context`/`details`/`message`. Such values are rehydrated
|
|
3832
|
+
* via `createErrorFromResponse` into a typed SandboxError.
|
|
3833
|
+
*
|
|
3834
|
+
* Lifecycle interruption causes are rebound onto the pending RPC method so the
|
|
3835
|
+
* public error names that operation and keeps the lifecycle reason,
|
|
3836
|
+
* retryability, and stop metadata.
|
|
3837
|
+
*
|
|
3838
|
+
* Anything without a recognized code is ignored so the caller falls back to
|
|
3839
|
+
* normal transport classification.
|
|
3840
|
+
*/
|
|
3841
|
+
function maybePreferConnectionError(transportResponse, context) {
|
|
3842
|
+
const captured = context.connectionError;
|
|
3843
|
+
if (!captured) return null;
|
|
3844
|
+
const { kind } = transportResponse.context;
|
|
3845
|
+
if (kind !== "session_disposed" && kind !== "connection_failed" && kind !== "upgrade_failed" && kind !== "peer_closed") return null;
|
|
3846
|
+
const shape = extractCapturedErrorShape(captured);
|
|
3847
|
+
if (!shape) return null;
|
|
3848
|
+
return rebindInterruptedOperationContext(shape, context, captured);
|
|
3849
|
+
}
|
|
3850
|
+
/**
|
|
3851
|
+
* When the session never established a live connection and a queued RPC call
|
|
3852
|
+
* rejects with a teardown-family transport error (disposed / connection
|
|
3853
|
+
* failed / peer closed), surface a clean, retryable `ContainerUnavailableError`
|
|
3854
|
+
* instead of the raw capnweb string (e.g. "RPC session was shut down by
|
|
3855
|
+
* disposing the main stub").
|
|
3856
|
+
*
|
|
3857
|
+
* Reaching this point means: the container never became reachable (so no
|
|
3858
|
+
* OPERATION_INTERRUPTED — nothing was admitted) AND no structured connection
|
|
3859
|
+
* error was captured (so `maybePreferConnectionError` didn't fire). That's the
|
|
3860
|
+
* Durable Object being torn down/evicted mid-startup under capacity pressure
|
|
3861
|
+
* before `doConnect` recorded a cause — which is, from the caller's view, the
|
|
3862
|
+
* container being unavailable. Retryable, with the raw transport message
|
|
3863
|
+
* preserved as `originalMessage` for diagnostics.
|
|
3864
|
+
*
|
|
3865
|
+
* `upgrade_failed` is intentionally excluded here. Unlike disposal/close
|
|
3866
|
+
* during startup, an HTTP upgrade failure with no captured structured cause can
|
|
3867
|
+
* be a permanent configuration or routing error (for example, a 404 on `/rpc`),
|
|
3868
|
+
* so the generic transport classification is more accurate than retryable
|
|
3869
|
+
* container unavailability.
|
|
3870
|
+
*/
|
|
3871
|
+
function buildNeverConnectedUnavailableResponse(transportResponse, context) {
|
|
3872
|
+
if (context.sessionEstablished !== false) return null;
|
|
3873
|
+
const { kind } = transportResponse.context;
|
|
3874
|
+
if (kind !== "session_disposed" && kind !== "connection_failed" && kind !== "peer_closed") return null;
|
|
3875
|
+
const ctx = {
|
|
3876
|
+
reason: "container_unreachable",
|
|
3877
|
+
retryable: true,
|
|
3878
|
+
originalMessage: transportResponse.context.originalMessage
|
|
3879
|
+
};
|
|
3880
|
+
return {
|
|
3881
|
+
code: ErrorCode.CONTAINER_UNAVAILABLE,
|
|
3882
|
+
message: "The sandbox container was unavailable: the connection was torn down before it became reachable. Retry the operation.",
|
|
3883
|
+
context: ctx,
|
|
3884
|
+
httpStatus: getHttpStatus(ErrorCode.CONTAINER_UNAVAILABLE),
|
|
3885
|
+
suggestion: getSuggestion(ErrorCode.CONTAINER_UNAVAILABLE, ctx),
|
|
3886
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
3887
|
+
};
|
|
3888
|
+
}
|
|
3401
3889
|
function buildInterruptedOperationResponse(transportResponse, context) {
|
|
3402
3890
|
if (!context.operation) return null;
|
|
3403
3891
|
const { kind } = transportResponse.context;
|
|
@@ -3406,7 +3894,7 @@ function buildInterruptedOperationResponse(transportResponse, context) {
|
|
|
3406
3894
|
reason: kind === "session_disposed" ? "transport_disposed" : "runtime_replaced",
|
|
3407
3895
|
operation: context.operation,
|
|
3408
3896
|
phase: "rpc_call",
|
|
3409
|
-
admitted: "unknown",
|
|
3897
|
+
admitted: context.sessionEstablished === true ? true : context.sessionEstablished === false ? false : "unknown",
|
|
3410
3898
|
retryable: false
|
|
3411
3899
|
};
|
|
3412
3900
|
const action = kind === "session_disposed" ? "was closing" : "closed unexpectedly";
|
|
@@ -3424,17 +3912,18 @@ function buildInterruptedOperationResponse(transportResponse, context) {
|
|
|
3424
3912
|
* from the JSON wire format into typed SandboxError instances and signals
|
|
3425
3913
|
* activity at call start.
|
|
3426
3914
|
*
|
|
3427
|
-
* `onCallStarted` fires synchronously when an RPC method is invoked
|
|
3428
|
-
*
|
|
3429
|
-
*
|
|
3430
|
-
*
|
|
3915
|
+
* `onCallStarted` fires synchronously when an RPC method is invoked, and
|
|
3916
|
+
* `onCallSettled` fires when the returned promise settles. The
|
|
3917
|
+
* ContainerControlClient uses these hooks to keep the session marked busy
|
|
3918
|
+
* even if capnweb stats briefly report the bootstrap baseline while a call is
|
|
3919
|
+
* still pending.
|
|
3431
3920
|
*
|
|
3432
|
-
*
|
|
3433
|
-
*
|
|
3434
|
-
*
|
|
3435
|
-
*
|
|
3921
|
+
* A method whose returned promise resolves with a `ReadableStream` is *not*
|
|
3922
|
+
* finished when the promise settles — capnweb keeps the export alive until
|
|
3923
|
+
* the stream ends. The busy/idle poll on `getStats()` remains the source of
|
|
3924
|
+
* truth for stream lifetimes after the initial RPC promise settles.
|
|
3436
3925
|
*/
|
|
3437
|
-
function wrapStub(stub, domain, onCallStarted) {
|
|
3926
|
+
function wrapStub(stub, domain, onCallStarted, onCallSettled, getConnectionError, getSessionEstablished, getSpanAttrs) {
|
|
3438
3927
|
return new Proxy(stub, { get(target, prop, receiver) {
|
|
3439
3928
|
const value = Reflect.get(target, prop, receiver);
|
|
3440
3929
|
if (typeof value !== "function") return value;
|
|
@@ -3443,10 +3932,23 @@ function wrapStub(stub, domain, onCallStarted) {
|
|
|
3443
3932
|
const operation = typeof prop === "string" ? `${domain}.${prop}` : domain;
|
|
3444
3933
|
try {
|
|
3445
3934
|
const result = Reflect.apply(value, target, args);
|
|
3446
|
-
if (result != null && typeof result.then === "function") return
|
|
3935
|
+
if (result != null && typeof result.then === "function") return withSpan(`sandbox.rpc.call ${operation}`, {
|
|
3936
|
+
...getSpanAttrs(),
|
|
3937
|
+
operation
|
|
3938
|
+
}, () => result.catch((err) => translateRPCError(err, {
|
|
3939
|
+
operation,
|
|
3940
|
+
connectionError: getConnectionError(),
|
|
3941
|
+
sessionEstablished: getSessionEstablished()
|
|
3942
|
+
}))).finally(onCallSettled);
|
|
3943
|
+
onCallSettled();
|
|
3447
3944
|
return result;
|
|
3448
3945
|
} catch (err) {
|
|
3449
|
-
|
|
3946
|
+
onCallSettled();
|
|
3947
|
+
translateRPCError(err, {
|
|
3948
|
+
operation,
|
|
3949
|
+
connectionError: getConnectionError(),
|
|
3950
|
+
sessionEstablished: getSessionEstablished()
|
|
3951
|
+
});
|
|
3450
3952
|
}
|
|
3451
3953
|
};
|
|
3452
3954
|
} });
|
|
@@ -3473,8 +3975,34 @@ var ContainerControlClient = class {
|
|
|
3473
3975
|
onSessionBusy;
|
|
3474
3976
|
onSessionIdle;
|
|
3475
3977
|
conn = null;
|
|
3978
|
+
/**
|
|
3979
|
+
* Real cause captured by the connection during startup failure (e.g. a
|
|
3980
|
+
* platform container-allocation error). Preferred over the generic capnweb
|
|
3981
|
+
* disposal error when translating queued RPC rejections. Cleared each time a
|
|
3982
|
+
* fresh connection is created.
|
|
3983
|
+
*/
|
|
3984
|
+
lastConnectionError = null;
|
|
3985
|
+
/**
|
|
3986
|
+
* Whether `lastConnectionError` was captured from an actual connection
|
|
3987
|
+
* attempt failure (authoritative root cause) rather than a lifecycle
|
|
3988
|
+
* teardown reason (weak). A weak teardown cause — e.g. `onStop` firing
|
|
3989
|
+
* `runtime_replaced` — is usually a downstream *consequence* of the real
|
|
3990
|
+
* failure, so it must not overwrite an authoritative cause already
|
|
3991
|
+
* captured from the connect attempt.
|
|
3992
|
+
*/
|
|
3993
|
+
connectionErrorIsAuthoritative = false;
|
|
3994
|
+
/**
|
|
3995
|
+
* Whether the current connection ever established a live session to a
|
|
3996
|
+
* running container. Set true by the connection's `onConnected` callback,
|
|
3997
|
+
* reset when a fresh connection is created. Lets `translateRPCError`
|
|
3998
|
+
* distinguish a true interruption (established, then dropped) from a
|
|
3999
|
+
* never-connected failure (container never started).
|
|
4000
|
+
*/
|
|
4001
|
+
sessionEstablished = false;
|
|
3476
4002
|
idleTimer = null;
|
|
3477
4003
|
busyPollTimer = null;
|
|
4004
|
+
/** Number of RPC method promises that have started but not settled. */
|
|
4005
|
+
activeCalls = 0;
|
|
3478
4006
|
/** Tracks whether we currently believe the session is busy. */
|
|
3479
4007
|
busy = false;
|
|
3480
4008
|
constructor(options) {
|
|
@@ -3484,8 +4012,19 @@ var ContainerControlClient = class {
|
|
|
3484
4012
|
localMain: options.localMain,
|
|
3485
4013
|
logger: options.logger,
|
|
3486
4014
|
retryTimeoutMs: options.retryTimeoutMs,
|
|
4015
|
+
getSandboxInfo: options.getSandboxInfo,
|
|
4016
|
+
startContainer: options.startContainer,
|
|
4017
|
+
onConnected: () => {
|
|
4018
|
+
this.sessionEstablished = true;
|
|
4019
|
+
this.lastConnectionError = null;
|
|
4020
|
+
this.connectionErrorIsAuthoritative = false;
|
|
4021
|
+
},
|
|
3487
4022
|
onClose: () => {
|
|
3488
4023
|
if (this.conn) this.destroyConnection();
|
|
4024
|
+
},
|
|
4025
|
+
onConnectionError: (error) => {
|
|
4026
|
+
this.lastConnectionError = error;
|
|
4027
|
+
this.connectionErrorIsAuthoritative = true;
|
|
3489
4028
|
}
|
|
3490
4029
|
};
|
|
3491
4030
|
this.idleDisconnectMs = options.idleDisconnectMs ?? DEFAULT_IDLE_DISCONNECT_MS;
|
|
@@ -3501,18 +4040,103 @@ var ContainerControlClient = class {
|
|
|
3501
4040
|
*/
|
|
3502
4041
|
getConnection() {
|
|
3503
4042
|
if (!this.conn) {
|
|
4043
|
+
this.lastConnectionError = null;
|
|
4044
|
+
this.connectionErrorIsAuthoritative = false;
|
|
4045
|
+
this.sessionEstablished = false;
|
|
3504
4046
|
this.conn = new ContainerControlConnection(this.connOptions);
|
|
3505
4047
|
this.startBusyPoll();
|
|
3506
4048
|
}
|
|
3507
4049
|
return this.conn;
|
|
3508
4050
|
}
|
|
3509
4051
|
/**
|
|
4052
|
+
* Stamp a *weak* connection cause (a lifecycle teardown reason). Only takes
|
|
4053
|
+
* effect if no authoritative connect-attempt failure has been captured, so
|
|
4054
|
+
* a teardown consequence never masks the real root cause.
|
|
4055
|
+
*/
|
|
4056
|
+
stampWeakConnectionCause(cause) {
|
|
4057
|
+
if (cause === void 0) return;
|
|
4058
|
+
if (this.connectionErrorIsAuthoritative) return;
|
|
4059
|
+
this.lastConnectionError = cause;
|
|
4060
|
+
}
|
|
4061
|
+
fireUserCallback(name, callback) {
|
|
4062
|
+
if (!callback) return;
|
|
4063
|
+
try {
|
|
4064
|
+
callback();
|
|
4065
|
+
} catch (error) {
|
|
4066
|
+
this.logger.warn("ContainerControlClient callback threw", {
|
|
4067
|
+
callback: name,
|
|
4068
|
+
error: error instanceof Error ? error.message : String(error)
|
|
4069
|
+
});
|
|
4070
|
+
}
|
|
4071
|
+
}
|
|
4072
|
+
fireActivity() {
|
|
4073
|
+
this.fireUserCallback("onActivity", this.onActivity);
|
|
4074
|
+
}
|
|
4075
|
+
fireSessionBusy() {
|
|
4076
|
+
this.fireUserCallback("onSessionBusy", this.onSessionBusy);
|
|
4077
|
+
}
|
|
4078
|
+
fireSessionIdle() {
|
|
4079
|
+
this.fireUserCallback("onSessionIdle", this.onSessionIdle);
|
|
4080
|
+
}
|
|
4081
|
+
markBusy() {
|
|
4082
|
+
if (!this.busy) {
|
|
4083
|
+
this.busy = true;
|
|
4084
|
+
this.fireSessionBusy();
|
|
4085
|
+
}
|
|
4086
|
+
this.clearIdleTimer();
|
|
4087
|
+
}
|
|
4088
|
+
isSessionBusy(conn) {
|
|
4089
|
+
const { imports, exports } = conn.getStats();
|
|
4090
|
+
return this.activeCalls > 0 || imports > IDLE_IMPORT_THRESHOLD || exports > IDLE_EXPORT_THRESHOLD;
|
|
4091
|
+
}
|
|
4092
|
+
maybeTransitionIdle() {
|
|
4093
|
+
const conn = this.conn;
|
|
4094
|
+
if (!conn) return;
|
|
4095
|
+
if (!conn.isConnected()) return;
|
|
4096
|
+
this.transitionIdleIfReady(conn);
|
|
4097
|
+
}
|
|
4098
|
+
transitionIdleIfReady(conn) {
|
|
4099
|
+
if (this.isSessionBusy(conn)) {
|
|
4100
|
+
this.markBusy();
|
|
4101
|
+
return;
|
|
4102
|
+
}
|
|
4103
|
+
if (this.busy) {
|
|
4104
|
+
this.busy = false;
|
|
4105
|
+
this.fireSessionIdle();
|
|
4106
|
+
this.scheduleIdleDisconnect();
|
|
4107
|
+
} else if (!this.idleTimer) this.scheduleIdleDisconnect();
|
|
4108
|
+
}
|
|
4109
|
+
/**
|
|
3510
4110
|
* Called synchronously at the start of each RPC method invocation.
|
|
3511
4111
|
* Renews the DO activity timeout so the sleepAfter alarm is pushed
|
|
3512
|
-
* forward before the container processes the call
|
|
4112
|
+
* forward before the container processes the call, and pins the RPC
|
|
4113
|
+
* WebSocket as busy until the method's promise settles.
|
|
4114
|
+
*/
|
|
4115
|
+
recordCallStarted = () => {
|
|
4116
|
+
this.activeCalls++;
|
|
4117
|
+
this.markBusy();
|
|
4118
|
+
this.fireActivity();
|
|
4119
|
+
};
|
|
4120
|
+
recordCallSettled = () => {
|
|
4121
|
+
this.activeCalls = Math.max(0, this.activeCalls - 1);
|
|
4122
|
+
this.maybeTransitionIdle();
|
|
4123
|
+
};
|
|
4124
|
+
/** Return the last connection-startup error captured, if any. */
|
|
4125
|
+
getLastConnectionError = () => this.lastConnectionError;
|
|
4126
|
+
/** Whether the current connection ever established a live session. */
|
|
4127
|
+
getSessionEstablished = () => this.sessionEstablished;
|
|
4128
|
+
/**
|
|
4129
|
+
* Base span attributes for RPC-call spans: sandbox identifiers plus the
|
|
4130
|
+
* container port. Mirrors the connection's `spanAttrs()` so all
|
|
4131
|
+
* `sandbox.rpc.*` spans share a consistent shape.
|
|
3513
4132
|
*/
|
|
3514
|
-
|
|
3515
|
-
this.
|
|
4133
|
+
getSpanAttrs = () => {
|
|
4134
|
+
const info = this.connOptions.getSandboxInfo?.();
|
|
4135
|
+
return {
|
|
4136
|
+
"sandbox.id": info?.id,
|
|
4137
|
+
"sandbox.name": info?.name,
|
|
4138
|
+
"sandbox.rpc.port": this.connOptions.port !== void 0 ? String(this.connOptions.port) : void 0
|
|
4139
|
+
};
|
|
3516
4140
|
};
|
|
3517
4141
|
/**
|
|
3518
4142
|
* Sample `getStats()` and update busy/idle state. While busy, renews the
|
|
@@ -3529,19 +4153,12 @@ var ContainerControlClient = class {
|
|
|
3529
4153
|
const conn = this.conn;
|
|
3530
4154
|
if (!conn) return;
|
|
3531
4155
|
if (!conn.isConnected()) return;
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
this.onActivity?.();
|
|
3539
|
-
this.clearIdleTimer();
|
|
3540
|
-
} else if (this.busy) {
|
|
3541
|
-
this.busy = false;
|
|
3542
|
-
this.onSessionIdle?.();
|
|
3543
|
-
this.scheduleIdleDisconnect();
|
|
3544
|
-
} else if (!this.idleTimer) this.scheduleIdleDisconnect();
|
|
4156
|
+
if (this.isSessionBusy(conn)) {
|
|
4157
|
+
this.markBusy();
|
|
4158
|
+
this.fireActivity();
|
|
4159
|
+
return;
|
|
4160
|
+
}
|
|
4161
|
+
this.transitionIdleIfReady(conn);
|
|
3545
4162
|
};
|
|
3546
4163
|
startBusyPoll() {
|
|
3547
4164
|
if (this.busyPollTimer) return;
|
|
@@ -3559,8 +4176,7 @@ var ContainerControlClient = class {
|
|
|
3559
4176
|
this.idleTimer = null;
|
|
3560
4177
|
const conn = this.conn;
|
|
3561
4178
|
if (!conn || !conn.isConnected()) return;
|
|
3562
|
-
|
|
3563
|
-
if (imports <= IDLE_IMPORT_THRESHOLD && exports <= IDLE_EXPORT_THRESHOLD) {
|
|
4179
|
+
if (!this.isSessionBusy(conn)) {
|
|
3564
4180
|
this.logger.debug("Disconnecting idle RPC connection");
|
|
3565
4181
|
this.destroyConnection();
|
|
3566
4182
|
}
|
|
@@ -3572,47 +4188,49 @@ var ContainerControlClient = class {
|
|
|
3572
4188
|
this.idleTimer = null;
|
|
3573
4189
|
}
|
|
3574
4190
|
}
|
|
3575
|
-
destroyConnection() {
|
|
4191
|
+
destroyConnection(cause) {
|
|
3576
4192
|
this.stopBusyPoll();
|
|
3577
4193
|
this.clearIdleTimer();
|
|
4194
|
+
this.activeCalls = 0;
|
|
3578
4195
|
if (this.busy) {
|
|
3579
4196
|
this.busy = false;
|
|
3580
|
-
this.
|
|
4197
|
+
this.fireSessionIdle();
|
|
3581
4198
|
}
|
|
3582
4199
|
if (this.conn) {
|
|
3583
|
-
this.
|
|
4200
|
+
this.stampWeakConnectionCause(cause);
|
|
4201
|
+
this.conn.disconnect(cause);
|
|
3584
4202
|
this.conn = null;
|
|
3585
4203
|
}
|
|
3586
4204
|
}
|
|
3587
4205
|
get commands() {
|
|
3588
|
-
return wrapStub(this.getConnection().rpc().commands, "commands", this.
|
|
4206
|
+
return wrapStub(this.getConnection().rpc().commands, "commands", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3589
4207
|
}
|
|
3590
4208
|
get files() {
|
|
3591
|
-
return wrapStub(this.getConnection().rpc().files, "files", this.
|
|
4209
|
+
return wrapStub(this.getConnection().rpc().files, "files", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3592
4210
|
}
|
|
3593
4211
|
get processes() {
|
|
3594
|
-
return wrapStub(this.getConnection().rpc().processes, "processes", this.
|
|
4212
|
+
return wrapStub(this.getConnection().rpc().processes, "processes", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3595
4213
|
}
|
|
3596
4214
|
get ports() {
|
|
3597
|
-
return wrapStub(this.getConnection().rpc().ports, "ports", this.
|
|
4215
|
+
return wrapStub(this.getConnection().rpc().ports, "ports", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3598
4216
|
}
|
|
3599
4217
|
get git() {
|
|
3600
|
-
return wrapStub(this.getConnection().rpc().git, "git", this.
|
|
4218
|
+
return wrapStub(this.getConnection().rpc().git, "git", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3601
4219
|
}
|
|
3602
4220
|
get utils() {
|
|
3603
|
-
return wrapStub(this.getConnection().rpc().utils, "utils", this.
|
|
4221
|
+
return wrapStub(this.getConnection().rpc().utils, "utils", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3604
4222
|
}
|
|
3605
4223
|
get backup() {
|
|
3606
|
-
return wrapStub(this.getConnection().rpc().backup, "backup", this.
|
|
4224
|
+
return wrapStub(this.getConnection().rpc().backup, "backup", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3607
4225
|
}
|
|
3608
4226
|
get watch() {
|
|
3609
|
-
return wrapStub(this.getConnection().rpc().watch, "watch", this.
|
|
4227
|
+
return wrapStub(this.getConnection().rpc().watch, "watch", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3610
4228
|
}
|
|
3611
4229
|
get tunnels() {
|
|
3612
|
-
return wrapStub(this.getConnection().rpc().tunnels, "tunnels", this.
|
|
4230
|
+
return wrapStub(this.getConnection().rpc().tunnels, "tunnels", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3613
4231
|
}
|
|
3614
4232
|
get interpreter() {
|
|
3615
|
-
return wrapStub(this.getConnection().rpc().interpreter, "interpreter", this.
|
|
4233
|
+
return wrapStub(this.getConnection().rpc().interpreter, "interpreter", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
|
|
3616
4234
|
}
|
|
3617
4235
|
/**
|
|
3618
4236
|
* Update the upgrade retry budget. Applies to the current connection
|
|
@@ -3632,8 +4250,42 @@ var ContainerControlClient = class {
|
|
|
3632
4250
|
async connect() {
|
|
3633
4251
|
await this.getConnection().connect();
|
|
3634
4252
|
}
|
|
3635
|
-
|
|
3636
|
-
|
|
4253
|
+
/**
|
|
4254
|
+
* Tear down the active connection.
|
|
4255
|
+
*
|
|
4256
|
+
* When a connection attempt is still in progress, the teardown is deferred
|
|
4257
|
+
* until that attempt settles — so a lifecycle disconnect (e.g. the DO's
|
|
4258
|
+
* alarm firing `onStop`) cannot rip the transport out from under an
|
|
4259
|
+
* in-flight connect and reject queued calls with a generic disposal error.
|
|
4260
|
+
* The provided `cause` is stamped immediately so that if the attempt fails,
|
|
4261
|
+
* queued calls surface it; if the attempt succeeds, the (now-established)
|
|
4262
|
+
* connection is then torn down cleanly with the cause.
|
|
4263
|
+
*
|
|
4264
|
+
* During this deferral window `this.conn` still points at the connecting
|
|
4265
|
+
* transport so queued calls can keep their original connection context.
|
|
4266
|
+
* Container lifecycle code treats `disconnect()` as terminal and does not
|
|
4267
|
+
* issue more RPC calls after requesting it.
|
|
4268
|
+
*
|
|
4269
|
+
* `cause` should be a typed `SandboxError` describing why the connection is
|
|
4270
|
+
* being torn down (sandbox stopping, lifetime change, transport switch).
|
|
4271
|
+
*/
|
|
4272
|
+
disconnect(cause) {
|
|
4273
|
+
const conn = this.conn;
|
|
4274
|
+
if (conn?.isConnecting()) {
|
|
4275
|
+
this.stopBusyPoll();
|
|
4276
|
+
this.clearIdleTimer();
|
|
4277
|
+
this.activeCalls = 0;
|
|
4278
|
+
if (this.busy) {
|
|
4279
|
+
this.busy = false;
|
|
4280
|
+
this.fireSessionIdle();
|
|
4281
|
+
}
|
|
4282
|
+
this.stampWeakConnectionCause(cause);
|
|
4283
|
+
conn.whenSettled().then(() => {
|
|
4284
|
+
if (this.conn === conn) this.destroyConnection(cause);
|
|
4285
|
+
});
|
|
4286
|
+
return;
|
|
4287
|
+
}
|
|
4288
|
+
this.destroyConnection(cause);
|
|
3637
4289
|
}
|
|
3638
4290
|
};
|
|
3639
4291
|
|
|
@@ -6487,7 +7139,7 @@ function createTunnelsHandler(host) {
|
|
|
6487
7139
|
* This file is auto-updated by .github/changeset-version.ts during releases
|
|
6488
7140
|
* DO NOT EDIT MANUALLY - Changes will be overwritten on the next version bump
|
|
6489
7141
|
*/
|
|
6490
|
-
const SDK_VERSION = "0.12.
|
|
7142
|
+
const SDK_VERSION = "0.12.5";
|
|
6491
7143
|
|
|
6492
7144
|
//#endregion
|
|
6493
7145
|
//#region src/sandbox.ts
|
|
@@ -6540,6 +7192,22 @@ const R2_DEFAULT_S3FS_OPTIONS = {
|
|
|
6540
7192
|
const R2_DEFAULT_S3FS_OPTION_ENTRIES = Object.entries(R2_DEFAULT_S3FS_OPTIONS).map(([key, value]) => value === true ? key : `${key}=${value}`);
|
|
6541
7193
|
const S3FS_DISABLE_EXPECT_HEADER_CONFIG = " Expect:\n";
|
|
6542
7194
|
const BACKUP_DEFAULT_TTL_SECONDS = 259200;
|
|
7195
|
+
/**
|
|
7196
|
+
* Instance-get budget (ms) for the RPC transport's explicit container-start
|
|
7197
|
+
* hook, independent of the user-facing `containerTimeouts.instanceGetTimeoutMS`
|
|
7198
|
+
* (which still governs the HTTP path's `containerFetch`).
|
|
7199
|
+
*
|
|
7200
|
+
* Shorter than the SDK's user-facing 30s default, but long enough to match
|
|
7201
|
+
* the Containers platform's own 8s instance-get default. This keeps one RPC
|
|
7202
|
+
* start attempt from burning the full 30s under capacity pressure while still
|
|
7203
|
+
* letting ordinary 4-8s cold provisions succeed on the first attempt instead
|
|
7204
|
+
* of always paying the first backoff delay. The RPC control connection's own
|
|
7205
|
+
* retry loop (`fetchWithResponseRetry`, ~2 min budget with exponential
|
|
7206
|
+
* backoff) still owns cross-attempt retries. Once an instance exists,
|
|
7207
|
+
* `portReadyTimeoutMS` still governs app boot on the next attempt (the base
|
|
7208
|
+
* fast-path returns immediately when the container is already running).
|
|
7209
|
+
*/
|
|
7210
|
+
const RPC_START_INSTANCE_GET_TIMEOUT_MS = 8e3;
|
|
6543
7211
|
const BACKUP_MAX_NAME_LENGTH = 256;
|
|
6544
7212
|
const BACKUP_CONTAINER_DIR = "/var/backups";
|
|
6545
7213
|
const BACKUP_STORAGE_PREFIX = "backups";
|
|
@@ -6949,6 +7617,11 @@ var Sandbox = class Sandbox extends Container {
|
|
|
6949
7617
|
port: 3e3,
|
|
6950
7618
|
logger: this.logger,
|
|
6951
7619
|
retryTimeoutMs: this.computeRetryTimeoutMs(),
|
|
7620
|
+
getSandboxInfo: () => ({
|
|
7621
|
+
id: this.ctx.id.toString(),
|
|
7622
|
+
name: this.sandboxName ?? void 0
|
|
7623
|
+
}),
|
|
7624
|
+
startContainer: () => this.startContainerForRPC(),
|
|
6952
7625
|
localMain: this.controlCallback,
|
|
6953
7626
|
onActivity: () => {
|
|
6954
7627
|
this.renewActivityTimeout();
|
|
@@ -7069,7 +7742,7 @@ var Sandbox = class Sandbox extends Container {
|
|
|
7069
7742
|
this.tunnelsHandler = null;
|
|
7070
7743
|
this.tunnelExitHandler = null;
|
|
7071
7744
|
this.destroyAllTunnels = null;
|
|
7072
|
-
previousClient.disconnect();
|
|
7745
|
+
previousClient.disconnect(this.buildDisconnectCause("runtime_replaced", "The sandbox transport was switched while the operation was pending."));
|
|
7073
7746
|
}
|
|
7074
7747
|
if (storedTransport) this.hasStoredTransport = true;
|
|
7075
7748
|
const storedLabels = await this.ctx.storage.get("labels");
|
|
@@ -7154,7 +7827,7 @@ var Sandbox = class Sandbox extends Container {
|
|
|
7154
7827
|
this.tunnelsHandler = null;
|
|
7155
7828
|
this.tunnelExitHandler = null;
|
|
7156
7829
|
this.destroyAllTunnels = null;
|
|
7157
|
-
previousClient.disconnect();
|
|
7830
|
+
previousClient.disconnect(this.buildDisconnectCause("runtime_replaced", "The sandbox transport was switched while the operation was pending."));
|
|
7158
7831
|
this.renewActivityTimeout();
|
|
7159
7832
|
this.logger.debug("Transport updated", { transport });
|
|
7160
7833
|
}
|
|
@@ -7844,9 +8517,14 @@ var Sandbox = class Sandbox extends Container {
|
|
|
7844
8517
|
}
|
|
7845
8518
|
await this.ctx.storage.delete("tunnels");
|
|
7846
8519
|
await this.ctx.storage.delete("tunnels:meta");
|
|
7847
|
-
this.client.disconnect();
|
|
8520
|
+
this.client.disconnect(this.buildDisconnectCause("sandbox_lifetime_changed", "The sandbox was destroyed while the operation was pending."));
|
|
7848
8521
|
outcome = "success";
|
|
7849
|
-
|
|
8522
|
+
try {
|
|
8523
|
+
await super.destroy();
|
|
8524
|
+
} catch (error) {
|
|
8525
|
+
if (!this.isNoInstanceError(error)) throw error;
|
|
8526
|
+
this.logger.debug("super.destroy() reported no container instance; treating as no-op");
|
|
8527
|
+
}
|
|
7850
8528
|
} catch (error) {
|
|
7851
8529
|
caughtError = error instanceof Error ? error : new Error(String(error));
|
|
7852
8530
|
throw error;
|
|
@@ -7937,8 +8615,11 @@ var Sandbox = class Sandbox extends Container {
|
|
|
7937
8615
|
versionOutcome: outcome
|
|
7938
8616
|
}, { successLevel });
|
|
7939
8617
|
}
|
|
7940
|
-
async onStop() {
|
|
7941
|
-
this.logger.debug("Sandbox stopped"
|
|
8618
|
+
async onStop(params) {
|
|
8619
|
+
this.logger.debug("Sandbox stopped", {
|
|
8620
|
+
exitCode: params?.exitCode,
|
|
8621
|
+
reason: params?.reason
|
|
8622
|
+
});
|
|
7942
8623
|
this.containerGeneration++;
|
|
7943
8624
|
this.defaultSession = null;
|
|
7944
8625
|
this.defaultSessionInit = null;
|
|
@@ -7949,7 +8630,7 @@ var Sandbox = class Sandbox extends Container {
|
|
|
7949
8630
|
} catch (error) {
|
|
7950
8631
|
this.logger.error("Failed to reconcile tunnel storage after container stop", error instanceof Error ? error : new Error(String(error)));
|
|
7951
8632
|
}
|
|
7952
|
-
this.client.disconnect();
|
|
8633
|
+
this.client.disconnect(this.buildDisconnectCause("runtime_replaced", "The sandbox container stopped while the operation was pending.", params));
|
|
7953
8634
|
let hadR2EgressMount = false;
|
|
7954
8635
|
let hadCredentialProxyMount = false;
|
|
7955
8636
|
for (const [, m] of this.activeMounts) if (m.mountType === "local-sync") await m.syncManager.stop().catch(() => {});
|
|
@@ -7987,14 +8668,21 @@ var Sandbox = class Sandbox extends Container {
|
|
|
7987
8668
|
}
|
|
7988
8669
|
});
|
|
7989
8670
|
} catch (e) {
|
|
7990
|
-
|
|
8671
|
+
const admissionReason = matchContainerUnavailable(e);
|
|
8672
|
+
if (admissionReason) {
|
|
8673
|
+
const originalMessage = e instanceof Error ? e.message : String(e);
|
|
8674
|
+
const context = {
|
|
8675
|
+
reason: admissionReason,
|
|
8676
|
+
retryable: true,
|
|
8677
|
+
originalMessage
|
|
8678
|
+
};
|
|
7991
8679
|
const errorBody$1 = {
|
|
7992
|
-
code: ErrorCode.
|
|
7993
|
-
message:
|
|
7994
|
-
context
|
|
8680
|
+
code: ErrorCode.CONTAINER_UNAVAILABLE,
|
|
8681
|
+
message: originalMessage,
|
|
8682
|
+
context,
|
|
7995
8683
|
httpStatus: 503,
|
|
7996
8684
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7997
|
-
suggestion:
|
|
8685
|
+
suggestion: getSuggestion(ErrorCode.CONTAINER_UNAVAILABLE, context)
|
|
7998
8686
|
};
|
|
7999
8687
|
return new Response(JSON.stringify(errorBody$1), {
|
|
8000
8688
|
status: 503,
|
|
@@ -8085,7 +8773,105 @@ var Sandbox = class Sandbox extends Container {
|
|
|
8085
8773
|
* This indicates the container VM is still being provisioned.
|
|
8086
8774
|
*/
|
|
8087
8775
|
isNoInstanceError(error) {
|
|
8088
|
-
return error
|
|
8776
|
+
return matchContainerUnavailable(error) !== null;
|
|
8777
|
+
}
|
|
8778
|
+
/**
|
|
8779
|
+
* Explicit container-start hook for the RPC transport.
|
|
8780
|
+
*
|
|
8781
|
+
* Runs `startAndWaitForPorts` under a short instance-get budget
|
|
8782
|
+
* ({@link RPC_START_INSTANCE_GET_TIMEOUT_MS}) so a single attempt fails fast
|
|
8783
|
+
* under capacity pressure — the base class throws the classifiable
|
|
8784
|
+
* `NO_CONTAINER_INSTANCE_ERROR` once that budget is exhausted, and the RPC
|
|
8785
|
+
* control connection's own retry loop (with backoff) owns cross-attempt
|
|
8786
|
+
* retries. Port readiness still uses the full `portReadyTimeoutMS` so a
|
|
8787
|
+
* genuinely-booting app isn't cut short once an instance exists.
|
|
8788
|
+
*
|
|
8789
|
+
* Always throws a typed `SandboxError` on failure so the control connection
|
|
8790
|
+
* never surfaces a raw capnweb/transport string to the caller:
|
|
8791
|
+
* - container-admission/capacity failures → retryable
|
|
8792
|
+
* `ContainerUnavailableError` (preserving the platform message so the
|
|
8793
|
+
* connection's `shouldRetryError` still recognizes and retries it);
|
|
8794
|
+
* - anything else (platform-transient DO reset, network loss, unexpected
|
|
8795
|
+
* startup failure) → `INTERNAL_ERROR`-coded SandboxError carrying the
|
|
8796
|
+
* original message.
|
|
8797
|
+
*/
|
|
8798
|
+
async startContainerForRPC() {
|
|
8799
|
+
try {
|
|
8800
|
+
await this.startAndWaitForPorts({
|
|
8801
|
+
ports: 3e3,
|
|
8802
|
+
cancellationOptions: {
|
|
8803
|
+
instanceGetTimeoutMS: RPC_START_INSTANCE_GET_TIMEOUT_MS,
|
|
8804
|
+
portReadyTimeoutMS: this.containerTimeouts.portReadyTimeoutMS,
|
|
8805
|
+
waitInterval: this.containerTimeouts.waitIntervalMS
|
|
8806
|
+
}
|
|
8807
|
+
});
|
|
8808
|
+
} catch (error) {
|
|
8809
|
+
throw this.toContainerStartError(error);
|
|
8810
|
+
}
|
|
8811
|
+
}
|
|
8812
|
+
/**
|
|
8813
|
+
* Convert a container-start failure into a typed SandboxError. Preserves an
|
|
8814
|
+
* existing SandboxError as-is; maps platform admission/capacity failures to
|
|
8815
|
+
* a retryable ContainerUnavailableError; and wraps everything else as an
|
|
8816
|
+
* INTERNAL_ERROR carrying the original message (never a raw transport
|
|
8817
|
+
* string).
|
|
8818
|
+
*/
|
|
8819
|
+
toContainerStartError(error) {
|
|
8820
|
+
if (error instanceof SandboxError) return error;
|
|
8821
|
+
const originalMessage = error instanceof Error ? error.message : String(error);
|
|
8822
|
+
const admissionReason = matchContainerUnavailable(error);
|
|
8823
|
+
if (admissionReason) {
|
|
8824
|
+
const context$1 = {
|
|
8825
|
+
reason: admissionReason,
|
|
8826
|
+
retryable: true,
|
|
8827
|
+
originalMessage
|
|
8828
|
+
};
|
|
8829
|
+
return createErrorFromResponse({
|
|
8830
|
+
code: ErrorCode.CONTAINER_UNAVAILABLE,
|
|
8831
|
+
message: originalMessage,
|
|
8832
|
+
context: context$1,
|
|
8833
|
+
httpStatus: getHttpStatus(ErrorCode.CONTAINER_UNAVAILABLE),
|
|
8834
|
+
suggestion: getSuggestion(ErrorCode.CONTAINER_UNAVAILABLE, context$1),
|
|
8835
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
8836
|
+
}, { cause: error });
|
|
8837
|
+
}
|
|
8838
|
+
const context = {
|
|
8839
|
+
phase: "startup",
|
|
8840
|
+
error: originalMessage
|
|
8841
|
+
};
|
|
8842
|
+
return createErrorFromResponse({
|
|
8843
|
+
code: ErrorCode.INTERNAL_ERROR,
|
|
8844
|
+
message: `Container failed to start: ${originalMessage}`,
|
|
8845
|
+
context,
|
|
8846
|
+
httpStatus: getHttpStatus(ErrorCode.INTERNAL_ERROR),
|
|
8847
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
8848
|
+
}, { cause: error });
|
|
8849
|
+
}
|
|
8850
|
+
/**
|
|
8851
|
+
* Build a typed lifecycle cause to stamp on the RPC transport when the DO
|
|
8852
|
+
* tears the connection down for its own reasons (container stopped, sandbox
|
|
8853
|
+
* destroyed, transport switched). Handed to `client.disconnect(cause)` so
|
|
8854
|
+
* any RPC calls queued on the transport reject with this actionable reason
|
|
8855
|
+
* instead of the generic capnweb "disposing the main stub" message.
|
|
8856
|
+
*/
|
|
8857
|
+
buildDisconnectCause(reason, detail, stopParams) {
|
|
8858
|
+
const context = {
|
|
8859
|
+
reason,
|
|
8860
|
+
operation: "rpc.connect",
|
|
8861
|
+
phase: "connection",
|
|
8862
|
+
admitted: "unknown",
|
|
8863
|
+
retryable: reason === "runtime_replaced",
|
|
8864
|
+
...stopParams?.exitCode !== void 0 && { containerExitCode: stopParams.exitCode },
|
|
8865
|
+
...stopParams?.reason !== void 0 && { stopReason: stopParams.reason }
|
|
8866
|
+
};
|
|
8867
|
+
return new OperationInterruptedError({
|
|
8868
|
+
code: ErrorCode.OPERATION_INTERRUPTED,
|
|
8869
|
+
message: detail,
|
|
8870
|
+
context,
|
|
8871
|
+
httpStatus: getHttpStatus(ErrorCode.OPERATION_INTERRUPTED),
|
|
8872
|
+
suggestion: getSuggestion(ErrorCode.OPERATION_INTERRUPTED, context),
|
|
8873
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
8874
|
+
});
|
|
8089
8875
|
}
|
|
8090
8876
|
/**
|
|
8091
8877
|
* Helper: Check if error is a transient startup error that should trigger retry
|
|
@@ -10734,4 +11520,4 @@ var Sandbox = class Sandbox extends Container {
|
|
|
10734
11520
|
|
|
10735
11521
|
//#endregion
|
|
10736
11522
|
export { PortClient as A, InvalidBackupConfigError as B, collectFile as C, SandboxClient as D, isPlatformTransientError as E, BackupCreateError as F, SessionTerminatedError as G, ProcessExitedBeforeReadyError as H, BackupExpiredError as I, BackupNotFoundError as L, FileClient as M, CommandClient as N, UtilityClient as O, BackupClient as P, BackupRestoreError as R, validateTunnelName as S, isDurableObjectCodeUpdateReset as T, ProcessReadyTimeoutError as U, OperationInterruptedError as V, RPCTransportError as W, responseToAsyncIterable as _, PREVIEW_PROXY_HEADER as a, sanitizeSandboxId as b, PREVIEW_PROXY_SANDBOX_ID_HEADER as c, BucketUnmountError as d, InvalidMountConfigError as f, parseSSEStream as g, asyncIterableToSSEStream as h, proxyTerminal as i, GitClient as j, ProcessClient as k, PREVIEW_PROXY_TOKEN_HEADER as l, S3FSMountError as m, Sandbox as n, PREVIEW_PROXY_HEADERS as o, MissingCredentialsError as p, getSandbox as r, PREVIEW_PROXY_PORT_HEADER as s, ContainerProxy$1 as t, BucketMountError as u, CodeInterpreter as v, streamFile as w, validatePort as x, SandboxSecurityError as y, ContainerUnavailableError as z };
|
|
10737
|
-
//# sourceMappingURL=sandbox-
|
|
11523
|
+
//# sourceMappingURL=sandbox-sU3r5LSr.js.map
|