@cloudflare/sandbox 0.12.4 → 0.12.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,8 +3,8 @@ import { n as getHttpStatus, r as ErrorCode, t as getSuggestion } from "./errors
3
3
  import { Container, ContainerProxy, getContainer, switchPort } from "@cloudflare/containers";
4
4
  import { AwsClient } from "aws4fetch";
5
5
  import { RpcSession, RpcTarget } from "capnweb";
6
+ import { RpcTarget as RpcTarget$1, tracing } from "cloudflare:workers";
6
7
  import path from "node:path/posix";
7
- import { RpcTarget as RpcTarget$1 } from "cloudflare:workers";
8
8
 
9
9
  //#region src/errors/classes.ts
10
10
  /**
@@ -769,19 +769,55 @@ function isRetryableWebSocketUpgradeResponse(response) {
769
769
  return RETRYABLE_WEBSOCKET_UPGRADE_STATUSES.has(response.status);
770
770
  }
771
771
  /**
772
+ * Compute the next backoff delay from the remaining retry budget. Returns null
773
+ * when another attempt should not begin.
774
+ *
775
+ * Delay is limited to the budget left after reserving `minTimeForRetryMs` for
776
+ * the subsequent attempt. An attempt already in progress owns its own
777
+ * per-request timeouts via the caller-provided `fetchResponse` function.
778
+ */
779
+ function nextRetryDelayMs(attempt, remainingMs, minTimeForRetryMs) {
780
+ const availableForDelay = remainingMs - minTimeForRetryMs;
781
+ if (availableForDelay <= 0) return null;
782
+ const computedDelay = Math.min(DEFAULT_INITIAL_RETRY_DELAY_MS * 2 ** attempt, DEFAULT_MAX_RETRY_DELAY_MS);
783
+ return Math.min(computedDelay, availableForDelay);
784
+ }
785
+ /**
772
786
  * Retry Response-returning operations while their response remains retryable.
773
- * The retry budget covers the whole operation; each attempt owns any
774
- * per-request timeout inside the caller-provided `fetchResponse` function.
787
+ *
788
+ * The retry budget decides whether another attempt may begin after a retryable
789
+ * response or thrown error. Each attempt owns any per-request timeout inside
790
+ * the caller-provided `fetchResponse` function. Backoff is limited to the
791
+ * remaining budget after reserving `minTimeForRetryMs` for that next attempt.
775
792
  */
776
793
  async function fetchWithResponseRetry(fetchResponse, options) {
777
794
  const startTime = Date.now();
778
795
  let attempt = 0;
779
796
  while (true) {
780
- const response = await fetchResponse();
797
+ let response;
798
+ try {
799
+ response = await fetchResponse();
800
+ } catch (error) {
801
+ if (!options.shouldRetryError?.(error)) throw error;
802
+ const elapsed$1 = Date.now() - startTime;
803
+ const remaining$1 = options.retryTimeoutMs - elapsed$1;
804
+ const delay$1 = nextRetryDelayMs(attempt, remaining$1, options.minTimeForRetryMs);
805
+ if (delay$1 === null) throw error;
806
+ options.logger.info(options.retryLogMessage, {
807
+ attempt: attempt + 1,
808
+ delayMs: delay$1,
809
+ remainingSec: Math.floor(remaining$1 / 1e3),
810
+ error: error instanceof Error ? error.message : String(error)
811
+ });
812
+ await new Promise((resolve) => setTimeout(resolve, delay$1));
813
+ attempt++;
814
+ continue;
815
+ }
781
816
  if (!options.shouldRetry(response)) return response;
782
817
  const elapsed = Date.now() - startTime;
783
818
  const remaining = options.retryTimeoutMs - elapsed;
784
- if (remaining <= options.minTimeForRetryMs) {
819
+ const delay = nextRetryDelayMs(attempt, remaining, options.minTimeForRetryMs);
820
+ if (delay === null) {
785
821
  options.onRetryExhausted?.({
786
822
  attempts: attempt + 1,
787
823
  elapsedMs: elapsed,
@@ -789,7 +825,6 @@ async function fetchWithResponseRetry(fetchResponse, options) {
789
825
  });
790
826
  return response;
791
827
  }
792
- const delay = Math.min(DEFAULT_INITIAL_RETRY_DELAY_MS * 2 ** attempt, DEFAULT_MAX_RETRY_DELAY_MS);
793
828
  options.logger.info(options.retryLogMessage, {
794
829
  status: response.status,
795
830
  attempt: attempt + 1,
@@ -2495,6 +2530,52 @@ const DO_STORAGE_STARTUP_RESET_PATTERN = /internal error while starting up durab
2495
2530
  function errorMessageOf(error) {
2496
2531
  return error instanceof Error ? error.message : typeof error === "string" ? error : "";
2497
2532
  }
2533
+ /**
2534
+ * Extract a matchable message from any thrown value, reading a `.message`
2535
+ * property even when the value is not an `Error` instance. The Containers
2536
+ * runtime raises admission failures from the container binding, which may
2537
+ * live in a different realm — so `instanceof Error` can be false for a
2538
+ * genuine error. Mirrors the base `@cloudflare/containers` `isErrorOfType`
2539
+ * helper: coerce to a string, then match case-insensitively.
2540
+ */
2541
+ function realmSafeMessageOf(error) {
2542
+ const message = error?.message;
2543
+ return typeof message === "string" ? message : String(error);
2544
+ }
2545
+ /**
2546
+ * Platform messages emitted by the Containers runtime when it cannot admit a
2547
+ * container for a Durable Object during startup. Matched case-insensitively as
2548
+ * lowercase substrings. Single source of truth for both the RPC connection
2549
+ * path (connection.ts) and the HTTP `containerFetch` path (sandbox.ts).
2550
+ */
2551
+ const CONTAINER_UNAVAILABLE_SIGNATURES = [
2552
+ {
2553
+ substring: "there is no container instance that can be provided to this durable object",
2554
+ reason: "no_container_instance_available"
2555
+ },
2556
+ {
2557
+ substring: "there is no container instance available at this time",
2558
+ reason: "no_container_instance_available"
2559
+ },
2560
+ {
2561
+ substring: "maximum number of running container instances exceeded",
2562
+ reason: "max_container_instances_exceeded"
2563
+ }
2564
+ ];
2565
+ /**
2566
+ * Classify a container-startup error as a platform admission/capacity failure,
2567
+ * returning the categorical reason or null. Realm-safe (does not gate on
2568
+ * `instanceof Error`), case-insensitive, and walks the `.cause` chain so a
2569
+ * wrapped admission failure is still recognized.
2570
+ */
2571
+ function matchContainerUnavailable(error) {
2572
+ for (const candidate of selfAndCauses(error)) {
2573
+ const text = realmSafeMessageOf(candidate).toLowerCase();
2574
+ const match = CONTAINER_UNAVAILABLE_SIGNATURES.find((sig) => text.includes(sig.substring));
2575
+ if (match) return match.reason;
2576
+ }
2577
+ return null;
2578
+ }
2498
2579
  function* selfAndCauses(error) {
2499
2580
  let current = error;
2500
2581
  for (let depth = 0; depth < 8 && current != null; depth += 1) {
@@ -3003,6 +3084,123 @@ var RestoreLifecycleRunner = class {
3003
3084
  }
3004
3085
  };
3005
3086
 
3087
+ //#endregion
3088
+ //#region src/container-control/tracing.ts
3089
+ /**
3090
+ * Thin tracing helper for the RPC control path.
3091
+ *
3092
+ * Wraps `cloudflare:workers` `tracing.enterSpan` so the connection/client code
3093
+ * can emit spans for RPC calls, the connect/disconnect lifecycle, and
3094
+ * individual upgrade attempts — without hard-failing in environments where the
3095
+ * tracing API is unavailable.
3096
+ *
3097
+ * Error convention: the Cloudflare trace UI surfaces `error` and `error.stack`
3098
+ * span attributes specially (this differs from the OpenTelemetry standard,
3099
+ * which uses exception events). We therefore stamp both attributes on the span
3100
+ * when the wrapped work throws, so failures are visible in the GUI.
3101
+ *
3102
+ * Cause chain: the base `@cloudflare/containers` class wraps the true failure
3103
+ * inside `new Error(NO_CONTAINER_INSTANCE_ERROR, { cause })`, so the real
3104
+ * reason (e.g. "the container is not listening", "Network connection lost", a
3105
+ * non-zero exit code) is only visible on `.cause`. Every distinct failure mode
3106
+ * therefore collapses to the same generic top-level message in traces. We walk
3107
+ * the cause chain and stamp it so a trace names the actual root cause.
3108
+ */
3109
+ /** Bound on cause-chain traversal — guards against cycles and runaway depth. */
3110
+ const MAX_CAUSE_DEPTH = 8;
3111
+ /** Read a string `code` property off an arbitrary value, if present. */
3112
+ function stringCode(value) {
3113
+ const code = value?.code;
3114
+ return typeof code === "string" ? code : void 0;
3115
+ }
3116
+ /** Render any thrown value as a short human-readable message. */
3117
+ function messageOf(value) {
3118
+ return value instanceof Error ? value.message : String(value);
3119
+ }
3120
+ /**
3121
+ * Compute the Cloudflare-convention error span attributes for any thrown value,
3122
+ * including the wrapped `.cause` chain. Pure and side-effect-free so it can be
3123
+ * unit-tested without a live tracer.
3124
+ *
3125
+ * Emits:
3126
+ * - `error` — top-level message (or stringified non-Error)
3127
+ * - `error.stack` — top-level stack, when available
3128
+ * - `error.code` — top-level string `code`, when available
3129
+ * - `error.cause` — immediate cause message, when a cause exists
3130
+ * - `error.cause.code` — immediate cause string `code`, when available
3131
+ * - `error.cause_chain`— all nested cause messages joined by " <- "
3132
+ */
3133
+ function computeErrorAttributes(error) {
3134
+ const attrs = {};
3135
+ if (error instanceof Error) {
3136
+ attrs.error = error.message;
3137
+ if (typeof error.stack === "string") attrs["error.stack"] = error.stack;
3138
+ const code = stringCode(error);
3139
+ if (code !== void 0) attrs["error.code"] = code;
3140
+ } else {
3141
+ attrs.error = String(error);
3142
+ return attrs;
3143
+ }
3144
+ const chain = [];
3145
+ const seen = new Set([error]);
3146
+ let current = error.cause;
3147
+ let depth = 0;
3148
+ while (current !== void 0 && current !== null && depth < MAX_CAUSE_DEPTH) {
3149
+ if (seen.has(current)) break;
3150
+ seen.add(current);
3151
+ if (depth === 0) {
3152
+ attrs["error.cause"] = messageOf(current);
3153
+ const causeCode = stringCode(current);
3154
+ if (causeCode !== void 0) attrs["error.cause.code"] = causeCode;
3155
+ }
3156
+ chain.push(messageOf(current));
3157
+ current = current instanceof Error ? current.cause : void 0;
3158
+ depth++;
3159
+ }
3160
+ if (chain.length > 0) attrs["error.cause_chain"] = chain.join(" <- ");
3161
+ return attrs;
3162
+ }
3163
+ /**
3164
+ * Stamp the Cloudflare-convention error attributes onto a span. Safe to call
3165
+ * with any thrown value.
3166
+ */
3167
+ function setErrorAttributes(span, error) {
3168
+ for (const [key, value] of Object.entries(computeErrorAttributes(error))) if (value !== void 0) span.setAttribute(key, value);
3169
+ }
3170
+ /**
3171
+ * Run `fn` inside a span named `name`, applying `attributes` up front and
3172
+ * stamping `error`/`error.stack` if it throws. Re-throws the original error.
3173
+ *
3174
+ * Falls back to running `fn` directly if the tracing API is unavailable, so a
3175
+ * missing tracer never changes behavior.
3176
+ */
3177
+ async function withSpan(name, attributes, fn) {
3178
+ const enter = tracing?.enterSpan?.bind(tracing);
3179
+ if (typeof enter !== "function") return fn({ setAttribute: () => {} });
3180
+ return enter(name, async (span) => {
3181
+ for (const [key, value] of Object.entries(attributes)) if (value !== void 0) span.setAttribute(key, value);
3182
+ try {
3183
+ return await fn(span);
3184
+ } catch (error) {
3185
+ setErrorAttributes(span, error);
3186
+ throw error;
3187
+ }
3188
+ });
3189
+ }
3190
+ /**
3191
+ * Emit a zero-duration marker span for a lifecycle event (e.g. disconnect).
3192
+ * Best-effort: never throws, never changes behavior.
3193
+ */
3194
+ function traceEvent(name, attributes) {
3195
+ const enter = tracing?.enterSpan?.bind(tracing);
3196
+ if (typeof enter !== "function") return;
3197
+ try {
3198
+ enter(name, (span) => {
3199
+ for (const [key, value] of Object.entries(attributes)) if (value !== void 0) span.setAttribute(key, value);
3200
+ });
3201
+ } catch {}
3202
+ }
3203
+
3006
3204
  //#endregion
3007
3205
  //#region src/container-control/connection.ts
3008
3206
  /**
@@ -3015,14 +3213,14 @@ var RestoreLifecycleRunner = class {
3015
3213
  * holds the original Response.
3016
3214
  */
3017
3215
  async function tryParseContainerUnavailable(response) {
3018
- if (!(response.headers.get("content-type") ?? "").includes("application/json")) return null;
3019
- try {
3216
+ if ((response.headers.get("content-type") ?? "").includes("application/json")) try {
3020
3217
  const body = await response.clone().json();
3021
3218
  if (body.code !== ErrorCode.CONTAINER_UNAVAILABLE) return null;
3022
3219
  const context = {
3023
- reason: body.context?.reason === "container_starting" || body.context?.reason === "container_unhealthy" || body.context?.reason === "container_replaced" || body.context?.reason === "rpc_upgrade_failed" ? body.context.reason : "container_replaced",
3220
+ reason: body.context?.reason === "container_starting" || body.context?.reason === "container_unhealthy" || body.context?.reason === "container_replaced" || body.context?.reason === "rpc_upgrade_failed" || body.context?.reason === "no_container_instance_available" || body.context?.reason === "max_container_instances_exceeded" ? body.context.reason : "container_replaced",
3024
3221
  retryable: true,
3025
- ...typeof body.context?.retryAfterMs === "number" && { retryAfterMs: body.context.retryAfterMs }
3222
+ ...typeof body.context?.retryAfterMs === "number" && { retryAfterMs: body.context.retryAfterMs },
3223
+ ...typeof body.context?.originalMessage === "string" && { originalMessage: body.context.originalMessage }
3026
3224
  };
3027
3225
  return createErrorFromResponse({
3028
3226
  code: ErrorCode.CONTAINER_UNAVAILABLE,
@@ -3035,6 +3233,53 @@ async function tryParseContainerUnavailable(response) {
3035
3233
  } catch {
3036
3234
  return null;
3037
3235
  }
3236
+ try {
3237
+ const text = await response.clone().text();
3238
+ const reason = matchContainerUnavailable(text);
3239
+ if (!reason) return null;
3240
+ return buildContainerUnavailableError(reason, text);
3241
+ } catch {
3242
+ return null;
3243
+ }
3244
+ }
3245
+ /**
3246
+ * True when a thrown connection-startup error matches a known platform
3247
+ * container-admission failure. These are transient: the platform asks the
3248
+ * caller to try again later, so they are safe to retry within the budget.
3249
+ * Delegates to the shared matcher in platform-errors.ts.
3250
+ */
3251
+ function isPlatformUnavailableError(error) {
3252
+ return matchContainerUnavailable(error) !== null;
3253
+ }
3254
+ /**
3255
+ * Build a typed ContainerUnavailableError for a matched platform
3256
+ * container-admission failure, preserving the original message verbatim.
3257
+ */
3258
+ function buildContainerUnavailableError(reason, originalMessage, cause) {
3259
+ const context = {
3260
+ reason,
3261
+ retryable: true,
3262
+ originalMessage
3263
+ };
3264
+ return createErrorFromResponse({
3265
+ code: ErrorCode.CONTAINER_UNAVAILABLE,
3266
+ message: originalMessage,
3267
+ context,
3268
+ httpStatus: getHttpStatus(ErrorCode.CONTAINER_UNAVAILABLE),
3269
+ suggestion: getSuggestion(ErrorCode.CONTAINER_UNAVAILABLE, context),
3270
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
3271
+ }, cause !== void 0 ? { cause } : void 0);
3272
+ }
3273
+ /**
3274
+ * Convert a raw connection-startup error into a typed ContainerUnavailableError
3275
+ * when it matches a known platform container-admission failure. Returns null
3276
+ * for anything else so the caller preserves the original error.
3277
+ */
3278
+ function tryConvertPlatformUnavailable(error) {
3279
+ if (error instanceof SandboxError) return error;
3280
+ const reason = matchContainerUnavailable(error);
3281
+ if (!reason) return null;
3282
+ return buildContainerUnavailableError(reason, error instanceof Error ? error.message : String(error), error);
3038
3283
  }
3039
3284
  const DEFAULT_CONNECT_TIMEOUT_MS = 3e4;
3040
3285
  const DEFAULT_RETRY_TIMEOUT_MS = 12e4;
@@ -3058,12 +3303,20 @@ var ContainerControlConnection = class {
3058
3303
  logger;
3059
3304
  retryTimeoutMs;
3060
3305
  onClose;
3306
+ onConnectionError;
3307
+ onConnected;
3308
+ startContainer;
3309
+ getSandboxInfo;
3061
3310
  constructor(options) {
3062
3311
  this.containerStub = options.stub;
3063
3312
  this.port = options.port ?? 3e3;
3064
3313
  this.logger = options.logger ?? createNoOpLogger();
3065
3314
  this.retryTimeoutMs = options.retryTimeoutMs ?? DEFAULT_RETRY_TIMEOUT_MS;
3066
3315
  this.onClose = options.onClose;
3316
+ this.onConnectionError = options.onConnectionError;
3317
+ this.onConnected = options.onConnected;
3318
+ this.startContainer = options.startContainer;
3319
+ this.getSandboxInfo = options.getSandboxInfo;
3067
3320
  this.transport = new DeferredTransport();
3068
3321
  this.session = new RpcSession(this.transport, options.localMain);
3069
3322
  this.stub = this.session.getRemoteMain();
@@ -3090,6 +3343,24 @@ var ContainerControlConnection = class {
3090
3343
  isConnected() {
3091
3344
  return this.connected;
3092
3345
  }
3346
+ /**
3347
+ * True while a connection attempt is in progress (upgrade not yet
3348
+ * established and not yet failed). Owners use this to avoid tearing the
3349
+ * connection down mid-attempt.
3350
+ */
3351
+ isConnecting() {
3352
+ return this.connectPromise !== null;
3353
+ }
3354
+ /**
3355
+ * Resolve once any in-flight connection attempt settles (success or
3356
+ * failure). Resolves immediately when no attempt is in progress. Never
3357
+ * rejects — callers only care that the attempt is done.
3358
+ */
3359
+ async whenSettled() {
3360
+ const pending = this.connectPromise;
3361
+ if (!pending) return;
3362
+ await pending.catch(() => {});
3363
+ }
3093
3364
  async connect() {
3094
3365
  if (this.connected) return;
3095
3366
  if (this.connectPromise) return this.connectPromise;
@@ -3100,7 +3371,20 @@ var ContainerControlConnection = class {
3100
3371
  this.connectPromise = null;
3101
3372
  }
3102
3373
  }
3103
- disconnect() {
3374
+ /**
3375
+ * Tear down the connection. `cause` (a lifecycle teardown reason from the
3376
+ * owning DO) is used only for the disconnect trace event here; the owner
3377
+ * is responsible for stamping it as a *weak* connection cause (one that
3378
+ * never overwrites an authoritative failure already captured from a
3379
+ * connection attempt), since a teardown is usually a downstream
3380
+ * consequence of that earlier failure rather than the root cause.
3381
+ */
3382
+ disconnect(cause) {
3383
+ if (cause !== void 0) traceEvent("sandbox.rpc.disconnect", {
3384
+ ...this.spanAttrs(),
3385
+ reason: cause instanceof Error ? cause.message : String(cause)
3386
+ });
3387
+ else traceEvent("sandbox.rpc.disconnect", this.spanAttrs());
3104
3388
  try {
3105
3389
  this.stub[Symbol.dispose]?.();
3106
3390
  } catch {}
@@ -3124,6 +3408,19 @@ var ContainerControlConnection = class {
3124
3408
  this.retryTimeoutMs = ms;
3125
3409
  }
3126
3410
  /**
3411
+ * Base span attributes for this connection: the container port plus the
3412
+ * sandbox identifiers (`sandbox.id` = DO id, `sandbox.name` = user name)
3413
+ * when available.
3414
+ */
3415
+ spanAttrs() {
3416
+ const info = this.getSandboxInfo?.();
3417
+ return {
3418
+ "sandbox.rpc.port": this.port,
3419
+ "sandbox.id": info?.id,
3420
+ "sandbox.name": info?.name
3421
+ };
3422
+ }
3423
+ /**
3127
3424
  * Run the owner-provided `onClose` callback exactly once per call,
3128
3425
  * swallowing any errors so a buggy listener can't keep the connection
3129
3426
  * object in a half-torn-down state.
@@ -3137,6 +3434,18 @@ var ContainerControlConnection = class {
3137
3434
  }
3138
3435
  }
3139
3436
  /**
3437
+ * Run the owner-provided `onConnectionError` callback, swallowing any
3438
+ * listener errors.
3439
+ */
3440
+ fireConnectionError(error) {
3441
+ if (!this.onConnectionError) return;
3442
+ try {
3443
+ this.onConnectionError(error);
3444
+ } catch (err) {
3445
+ this.logger.warn("ContainerControlConnection onConnectionError handler threw", { error: err instanceof Error ? err.message : String(err) });
3446
+ }
3447
+ }
3448
+ /**
3140
3449
  * WebSocket `close` listener. Defined as a bound arrow field so the
3141
3450
  * same reference can be passed to both `addEventListener` and
3142
3451
  * `removeEventListener` — a fresh anonymous lambda would silently
@@ -3160,6 +3469,9 @@ var ContainerControlConnection = class {
3160
3469
  if (wasConnected) this.fireOnClose();
3161
3470
  };
3162
3471
  async doConnect() {
3472
+ return withSpan("sandbox.rpc.connect", this.spanAttrs(), () => this.doConnectInner());
3473
+ }
3474
+ async doConnectInner() {
3163
3475
  try {
3164
3476
  const response = await this.fetchUpgradeWithRetry();
3165
3477
  if (response.status !== 101) {
@@ -3186,13 +3498,20 @@ var ContainerControlConnection = class {
3186
3498
  this.ws = ws;
3187
3499
  this.transport.activate(ws);
3188
3500
  this.connected = true;
3501
+ try {
3502
+ this.onConnected?.();
3503
+ } catch (err) {
3504
+ this.logger.warn("ContainerControlConnection onConnected handler threw", { error: err instanceof Error ? err.message : String(err) });
3505
+ }
3189
3506
  this.logger.debug("ContainerControlConnection established", { port: this.port });
3190
3507
  } catch (error) {
3191
3508
  this.connected = false;
3192
- this.transport.abort(error);
3509
+ const connectionError = tryConvertPlatformUnavailable(error) ?? error;
3510
+ this.fireConnectionError(connectionError);
3511
+ this.transport.abort(connectionError);
3193
3512
  this.fireOnClose();
3194
- this.logger.error("ContainerControlConnection failed", error instanceof Error ? error : new Error(String(error)));
3195
- throw error;
3513
+ this.logger.error("ContainerControlConnection failed", connectionError instanceof Error ? connectionError : new Error(String(connectionError)));
3514
+ throw connectionError;
3196
3515
  }
3197
3516
  }
3198
3517
  /**
@@ -3206,30 +3525,56 @@ var ContainerControlConnection = class {
3206
3525
  minTimeForRetryMs: MIN_TIME_FOR_RETRY_MS,
3207
3526
  logger: this.logger,
3208
3527
  retryLogMessage: "ContainerControlConnection upgrade returned retryable status, retrying",
3209
- shouldRetry: isRetryableWebSocketUpgradeResponse
3528
+ shouldRetry: isRetryableWebSocketUpgradeResponse,
3529
+ shouldRetryError: isPlatformUnavailableError
3210
3530
  });
3211
3531
  }
3212
3532
  /**
3213
- * Single WebSocket-upgrade fetch attempt. Owns its own AbortController so
3214
- * each retry gets a fresh per-attempt connect timeout independent of the
3215
- * total retry budget.
3533
+ * Single WebSocket-upgrade fetch attempt.
3534
+ *
3535
+ * Container start and the upgrade fetch have deliberately separate timeout
3536
+ * scopes:
3537
+ *
3538
+ * - `startContainer()` runs under the base `Container` class's own
3539
+ * instance-get / port-ready budget. It must NOT be cut short by the
3540
+ * per-attempt connect timeout: the base class only emits the
3541
+ * classifiable `NO_CONTAINER_INSTANCE_ERROR` after its internal budget
3542
+ * is exhausted, and it checks its abort signal *before* that throw
3543
+ * (see @cloudflare/containers `doStartContainer`). Aborting it early
3544
+ * yields a generic "Aborted waiting for container to start" error that
3545
+ * we can't classify — which is what masked the real cause as
3546
+ * OPERATION_INTERRUPTED. Letting it run its own budget means the
3547
+ * platform capacity failure throws here, in our retry loop, where
3548
+ * `shouldRetryError` retries it and `doConnect`'s catch converts it to
3549
+ * a typed ContainerUnavailableError.
3550
+ *
3551
+ * - The WebSocket upgrade fetch gets its own fresh AbortController so
3552
+ * each retry has an independent connect timeout for the *upgrade only*.
3216
3553
  */
3217
3554
  async fetchUpgradeAttempt() {
3218
- const controller = new AbortController();
3219
- const timeout = setTimeout(() => controller.abort(), DEFAULT_CONNECT_TIMEOUT_MS);
3220
- try {
3221
- const url = `http://localhost:${this.port}/rpc`;
3222
- const request = new Request(url, {
3223
- headers: {
3224
- Upgrade: "websocket",
3225
- Connection: "Upgrade"
3226
- },
3227
- signal: controller.signal
3228
- });
3229
- return await this.containerStub.fetch(request);
3230
- } finally {
3231
- clearTimeout(timeout);
3232
- }
3555
+ return withSpan("sandbox.rpc.connect.attempt", this.spanAttrs(), async () => {
3556
+ if (this.startContainer) try {
3557
+ await this.startContainer();
3558
+ } catch (error) {
3559
+ this.fireConnectionError(tryConvertPlatformUnavailable(error) ?? error);
3560
+ throw error;
3561
+ }
3562
+ const controller = new AbortController();
3563
+ const timeout = setTimeout(() => controller.abort(), DEFAULT_CONNECT_TIMEOUT_MS);
3564
+ try {
3565
+ const url = `http://localhost:${this.port}/rpc`;
3566
+ const request = new Request(url, {
3567
+ headers: {
3568
+ Upgrade: "websocket",
3569
+ Connection: "Upgrade"
3570
+ },
3571
+ signal: controller.signal
3572
+ });
3573
+ return await this.containerStub.fetch(request);
3574
+ } finally {
3575
+ clearTimeout(timeout);
3576
+ }
3577
+ });
3233
3578
  }
3234
3579
  };
3235
3580
  /**
@@ -3350,6 +3695,9 @@ function translateRPCError(error, context = {}) {
3350
3695
  timestamp: (/* @__PURE__ */ new Date()).toISOString()
3351
3696
  });
3352
3697
  const transportResponse = buildTransportErrorResponse(error);
3698
+ const captured = maybePreferConnectionError(transportResponse, context);
3699
+ if (captured) throw captured;
3700
+ if (context.sessionEstablished === false) throw createErrorFromResponse(buildNeverConnectedUnavailableResponse(transportResponse, context) ?? transportResponse, { cause: error });
3353
3701
  throw createErrorFromResponse(buildInterruptedOperationResponse(transportResponse, context) ?? transportResponse, { cause: error });
3354
3702
  }
3355
3703
  throw createErrorFromResponse(buildTransportErrorResponse(new Error(String(error))), { cause: error });
@@ -3398,6 +3746,146 @@ function buildTransportErrorResponse(error) {
3398
3746
  timestamp: (/* @__PURE__ */ new Date()).toISOString()
3399
3747
  };
3400
3748
  }
3749
+ /**
3750
+ * Extract a recognized structured error shape from a captured connection cause.
3751
+ * Accepts both same-realm SandboxError instances and plain error-like objects
3752
+ * that carry a known `code` plus either `context` or `details`.
3753
+ */
3754
+ function extractCapturedErrorShape(captured) {
3755
+ if (captured instanceof SandboxError) return {
3756
+ code: captured.code,
3757
+ message: captured.message,
3758
+ context: captured.context ?? {}
3759
+ };
3760
+ const shape = captured;
3761
+ if (!shape || typeof shape.code !== "string" || !Object.hasOwn(ErrorCode, shape.code)) return null;
3762
+ const code = shape.code;
3763
+ const structured = shape.context && typeof shape.context === "object" ? shape.context : shape.details && typeof shape.details === "object" ? shape.details : {};
3764
+ return {
3765
+ code,
3766
+ message: typeof shape.message === "string" ? shape.message : code,
3767
+ context: structured
3768
+ };
3769
+ }
3770
+ const OPERATION_INTERRUPTED_REASONS = new Set([
3771
+ "runtime_replaced",
3772
+ "transport_disposed",
3773
+ "sandbox_lifetime_changed",
3774
+ "recovery_exhausted"
3775
+ ]);
3776
+ function asOperationInterruptedReason(value) {
3777
+ return typeof value === "string" && OPERATION_INTERRUPTED_REASONS.has(value) ? value : null;
3778
+ }
3779
+ function asInterruptedAdmitted(value) {
3780
+ if (typeof value === "boolean" || value === "unknown") return value;
3781
+ }
3782
+ /**
3783
+ * Lifecycle disconnect causes are stamped with a generic connection-level
3784
+ * operation identity. When they interrupt a concrete RPC method, rebind the
3785
+ * public interruption context onto that method while preserving the lifecycle
3786
+ * reason, retryability, and stop metadata.
3787
+ */
3788
+ function rebindInterruptedOperationContext(shape, context, cause) {
3789
+ if (shape.code !== ErrorCode.OPERATION_INTERRUPTED || typeof context.operation !== "string" || context.operation.length === 0) return createErrorFromResponse({
3790
+ code: shape.code,
3791
+ message: shape.message,
3792
+ context: shape.context,
3793
+ httpStatus: getHttpStatus(shape.code),
3794
+ suggestion: getSuggestion(shape.code, shape.context),
3795
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
3796
+ }, { cause });
3797
+ const reason = asOperationInterruptedReason(shape.context.reason) ?? "transport_disposed";
3798
+ const admitted = context.sessionEstablished === true ? true : context.sessionEstablished === false ? false : asInterruptedAdmitted(shape.context.admitted) ?? "unknown";
3799
+ const retryable = typeof shape.context.retryable === "boolean" ? shape.context.retryable : false;
3800
+ const interruptedContext = {
3801
+ reason,
3802
+ operation: context.operation,
3803
+ phase: "rpc_call",
3804
+ admitted,
3805
+ retryable,
3806
+ ...typeof shape.context.recoveryAttempts === "number" && { recoveryAttempts: shape.context.recoveryAttempts },
3807
+ ...typeof shape.context.maxRecoveryAttempts === "number" && { maxRecoveryAttempts: shape.context.maxRecoveryAttempts },
3808
+ ...typeof shape.context.containerExitCode === "number" && { containerExitCode: shape.context.containerExitCode },
3809
+ ...typeof shape.context.stopReason === "string" && { stopReason: shape.context.stopReason }
3810
+ };
3811
+ return new OperationInterruptedError({
3812
+ code: ErrorCode.OPERATION_INTERRUPTED,
3813
+ message: shape.message,
3814
+ context: interruptedContext,
3815
+ httpStatus: getHttpStatus(ErrorCode.OPERATION_INTERRUPTED),
3816
+ suggestion: getSuggestion(ErrorCode.OPERATION_INTERRUPTED, interruptedContext),
3817
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
3818
+ }, { cause });
3819
+ }
3820
+ /**
3821
+ * When a queued RPC call rejects with a transport error that a connection
3822
+ * abort could have caused (session disposed, connection failed, upgrade
3823
+ * failed, or peer closed), and the connection captured a real startup or
3824
+ * lifecycle error, return that captured error in preference to the masking
3825
+ * transport error.
3826
+ *
3827
+ * The captured error is accepted whether it is:
3828
+ * - a same-realm `SandboxError`, or
3829
+ * - any structured, error-like value carrying a recognized `code` — e.g. a
3830
+ * raw or cross-realm error decorated with `code: 'CONTAINER_UNAVAILABLE'`
3831
+ * and optional `context`/`details`/`message`. Such values are rehydrated
3832
+ * via `createErrorFromResponse` into a typed SandboxError.
3833
+ *
3834
+ * Lifecycle interruption causes are rebound onto the pending RPC method so the
3835
+ * public error names that operation and keeps the lifecycle reason,
3836
+ * retryability, and stop metadata.
3837
+ *
3838
+ * Anything without a recognized code is ignored so the caller falls back to
3839
+ * normal transport classification.
3840
+ */
3841
+ function maybePreferConnectionError(transportResponse, context) {
3842
+ const captured = context.connectionError;
3843
+ if (!captured) return null;
3844
+ const { kind } = transportResponse.context;
3845
+ if (kind !== "session_disposed" && kind !== "connection_failed" && kind !== "upgrade_failed" && kind !== "peer_closed") return null;
3846
+ const shape = extractCapturedErrorShape(captured);
3847
+ if (!shape) return null;
3848
+ return rebindInterruptedOperationContext(shape, context, captured);
3849
+ }
3850
+ /**
3851
+ * When the session never established a live connection and a queued RPC call
3852
+ * rejects with a teardown-family transport error (disposed / connection
3853
+ * failed / peer closed), surface a clean, retryable `ContainerUnavailableError`
3854
+ * instead of the raw capnweb string (e.g. "RPC session was shut down by
3855
+ * disposing the main stub").
3856
+ *
3857
+ * Reaching this point means: the container never became reachable (so no
3858
+ * OPERATION_INTERRUPTED — nothing was admitted) AND no structured connection
3859
+ * error was captured (so `maybePreferConnectionError` didn't fire). That's the
3860
+ * Durable Object being torn down/evicted mid-startup under capacity pressure
3861
+ * before `doConnect` recorded a cause — which is, from the caller's view, the
3862
+ * container being unavailable. Retryable, with the raw transport message
3863
+ * preserved as `originalMessage` for diagnostics.
3864
+ *
3865
+ * `upgrade_failed` is intentionally excluded here. Unlike disposal/close
3866
+ * during startup, an HTTP upgrade failure with no captured structured cause can
3867
+ * be a permanent configuration or routing error (for example, a 404 on `/rpc`),
3868
+ * so the generic transport classification is more accurate than retryable
3869
+ * container unavailability.
3870
+ */
3871
+ function buildNeverConnectedUnavailableResponse(transportResponse, context) {
3872
+ if (context.sessionEstablished !== false) return null;
3873
+ const { kind } = transportResponse.context;
3874
+ if (kind !== "session_disposed" && kind !== "connection_failed" && kind !== "peer_closed") return null;
3875
+ const ctx = {
3876
+ reason: "container_unreachable",
3877
+ retryable: true,
3878
+ originalMessage: transportResponse.context.originalMessage
3879
+ };
3880
+ return {
3881
+ code: ErrorCode.CONTAINER_UNAVAILABLE,
3882
+ message: "The sandbox container was unavailable: the connection was torn down before it became reachable. Retry the operation.",
3883
+ context: ctx,
3884
+ httpStatus: getHttpStatus(ErrorCode.CONTAINER_UNAVAILABLE),
3885
+ suggestion: getSuggestion(ErrorCode.CONTAINER_UNAVAILABLE, ctx),
3886
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
3887
+ };
3888
+ }
3401
3889
  function buildInterruptedOperationResponse(transportResponse, context) {
3402
3890
  if (!context.operation) return null;
3403
3891
  const { kind } = transportResponse.context;
@@ -3406,7 +3894,7 @@ function buildInterruptedOperationResponse(transportResponse, context) {
3406
3894
  reason: kind === "session_disposed" ? "transport_disposed" : "runtime_replaced",
3407
3895
  operation: context.operation,
3408
3896
  phase: "rpc_call",
3409
- admitted: "unknown",
3897
+ admitted: context.sessionEstablished === true ? true : context.sessionEstablished === false ? false : "unknown",
3410
3898
  retryable: false
3411
3899
  };
3412
3900
  const action = kind === "session_disposed" ? "was closing" : "closed unexpectedly";
@@ -3424,17 +3912,18 @@ function buildInterruptedOperationResponse(transportResponse, context) {
3424
3912
  * from the JSON wire format into typed SandboxError instances and signals
3425
3913
  * activity at call start.
3426
3914
  *
3427
- * `onCallStarted` fires synchronously when an RPC method is invoked. The
3428
- * ContainerControlClient uses this to renew the DO's activity timeout
3429
- * immediately, so even a call that completes entirely between two
3430
- * busy-poll ticks still pushes the sleepAfter deadline forward.
3915
+ * `onCallStarted` fires synchronously when an RPC method is invoked, and
3916
+ * `onCallSettled` fires when the returned promise settles. The
3917
+ * ContainerControlClient uses these hooks to keep the session marked busy
3918
+ * even if capnweb stats briefly report the bootstrap baseline while a call is
3919
+ * still pending.
3431
3920
  *
3432
- * Note: there is no `onCallSettled` hook. A method whose returned promise
3433
- * resolves with a `ReadableStream` is *not* finished when the promise
3434
- * settles — capnweb keeps the export alive until the stream ends. The
3435
- * busy/idle poll on `getStats()` is the source of truth for that.
3921
+ * A method whose returned promise resolves with a `ReadableStream` is *not*
3922
+ * finished when the promise settles — capnweb keeps the export alive until
3923
+ * the stream ends. The busy/idle poll on `getStats()` remains the source of
3924
+ * truth for stream lifetimes after the initial RPC promise settles.
3436
3925
  */
3437
- function wrapStub(stub, domain, onCallStarted) {
3926
+ function wrapStub(stub, domain, onCallStarted, onCallSettled, getConnectionError, getSessionEstablished, getSpanAttrs) {
3438
3927
  return new Proxy(stub, { get(target, prop, receiver) {
3439
3928
  const value = Reflect.get(target, prop, receiver);
3440
3929
  if (typeof value !== "function") return value;
@@ -3443,10 +3932,23 @@ function wrapStub(stub, domain, onCallStarted) {
3443
3932
  const operation = typeof prop === "string" ? `${domain}.${prop}` : domain;
3444
3933
  try {
3445
3934
  const result = Reflect.apply(value, target, args);
3446
- if (result != null && typeof result.then === "function") return result.catch((err) => translateRPCError(err, { operation }));
3935
+ if (result != null && typeof result.then === "function") return withSpan(`sandbox.rpc.call ${operation}`, {
3936
+ ...getSpanAttrs(),
3937
+ operation
3938
+ }, () => result.catch((err) => translateRPCError(err, {
3939
+ operation,
3940
+ connectionError: getConnectionError(),
3941
+ sessionEstablished: getSessionEstablished()
3942
+ }))).finally(onCallSettled);
3943
+ onCallSettled();
3447
3944
  return result;
3448
3945
  } catch (err) {
3449
- translateRPCError(err, { operation });
3946
+ onCallSettled();
3947
+ translateRPCError(err, {
3948
+ operation,
3949
+ connectionError: getConnectionError(),
3950
+ sessionEstablished: getSessionEstablished()
3951
+ });
3450
3952
  }
3451
3953
  };
3452
3954
  } });
@@ -3473,8 +3975,34 @@ var ContainerControlClient = class {
3473
3975
  onSessionBusy;
3474
3976
  onSessionIdle;
3475
3977
  conn = null;
3978
+ /**
3979
+ * Real cause captured by the connection during startup failure (e.g. a
3980
+ * platform container-allocation error). Preferred over the generic capnweb
3981
+ * disposal error when translating queued RPC rejections. Cleared each time a
3982
+ * fresh connection is created.
3983
+ */
3984
+ lastConnectionError = null;
3985
+ /**
3986
+ * Whether `lastConnectionError` was captured from an actual connection
3987
+ * attempt failure (authoritative root cause) rather than a lifecycle
3988
+ * teardown reason (weak). A weak teardown cause — e.g. `onStop` firing
3989
+ * `runtime_replaced` — is usually a downstream *consequence* of the real
3990
+ * failure, so it must not overwrite an authoritative cause already
3991
+ * captured from the connect attempt.
3992
+ */
3993
+ connectionErrorIsAuthoritative = false;
3994
+ /**
3995
+ * Whether the current connection ever established a live session to a
3996
+ * running container. Set true by the connection's `onConnected` callback,
3997
+ * reset when a fresh connection is created. Lets `translateRPCError`
3998
+ * distinguish a true interruption (established, then dropped) from a
3999
+ * never-connected failure (container never started).
4000
+ */
4001
+ sessionEstablished = false;
3476
4002
  idleTimer = null;
3477
4003
  busyPollTimer = null;
4004
+ /** Number of RPC method promises that have started but not settled. */
4005
+ activeCalls = 0;
3478
4006
  /** Tracks whether we currently believe the session is busy. */
3479
4007
  busy = false;
3480
4008
  constructor(options) {
@@ -3484,8 +4012,19 @@ var ContainerControlClient = class {
3484
4012
  localMain: options.localMain,
3485
4013
  logger: options.logger,
3486
4014
  retryTimeoutMs: options.retryTimeoutMs,
4015
+ getSandboxInfo: options.getSandboxInfo,
4016
+ startContainer: options.startContainer,
4017
+ onConnected: () => {
4018
+ this.sessionEstablished = true;
4019
+ this.lastConnectionError = null;
4020
+ this.connectionErrorIsAuthoritative = false;
4021
+ },
3487
4022
  onClose: () => {
3488
4023
  if (this.conn) this.destroyConnection();
4024
+ },
4025
+ onConnectionError: (error) => {
4026
+ this.lastConnectionError = error;
4027
+ this.connectionErrorIsAuthoritative = true;
3489
4028
  }
3490
4029
  };
3491
4030
  this.idleDisconnectMs = options.idleDisconnectMs ?? DEFAULT_IDLE_DISCONNECT_MS;
@@ -3501,18 +4040,103 @@ var ContainerControlClient = class {
3501
4040
  */
3502
4041
  getConnection() {
3503
4042
  if (!this.conn) {
4043
+ this.lastConnectionError = null;
4044
+ this.connectionErrorIsAuthoritative = false;
4045
+ this.sessionEstablished = false;
3504
4046
  this.conn = new ContainerControlConnection(this.connOptions);
3505
4047
  this.startBusyPoll();
3506
4048
  }
3507
4049
  return this.conn;
3508
4050
  }
3509
4051
  /**
4052
+ * Stamp a *weak* connection cause (a lifecycle teardown reason). Only takes
4053
+ * effect if no authoritative connect-attempt failure has been captured, so
4054
+ * a teardown consequence never masks the real root cause.
4055
+ */
4056
+ stampWeakConnectionCause(cause) {
4057
+ if (cause === void 0) return;
4058
+ if (this.connectionErrorIsAuthoritative) return;
4059
+ this.lastConnectionError = cause;
4060
+ }
4061
+ fireUserCallback(name, callback) {
4062
+ if (!callback) return;
4063
+ try {
4064
+ callback();
4065
+ } catch (error) {
4066
+ this.logger.warn("ContainerControlClient callback threw", {
4067
+ callback: name,
4068
+ error: error instanceof Error ? error.message : String(error)
4069
+ });
4070
+ }
4071
+ }
4072
+ fireActivity() {
4073
+ this.fireUserCallback("onActivity", this.onActivity);
4074
+ }
4075
+ fireSessionBusy() {
4076
+ this.fireUserCallback("onSessionBusy", this.onSessionBusy);
4077
+ }
4078
+ fireSessionIdle() {
4079
+ this.fireUserCallback("onSessionIdle", this.onSessionIdle);
4080
+ }
4081
+ markBusy() {
4082
+ if (!this.busy) {
4083
+ this.busy = true;
4084
+ this.fireSessionBusy();
4085
+ }
4086
+ this.clearIdleTimer();
4087
+ }
4088
+ isSessionBusy(conn) {
4089
+ const { imports, exports } = conn.getStats();
4090
+ return this.activeCalls > 0 || imports > IDLE_IMPORT_THRESHOLD || exports > IDLE_EXPORT_THRESHOLD;
4091
+ }
4092
+ maybeTransitionIdle() {
4093
+ const conn = this.conn;
4094
+ if (!conn) return;
4095
+ if (!conn.isConnected()) return;
4096
+ this.transitionIdleIfReady(conn);
4097
+ }
4098
+ transitionIdleIfReady(conn) {
4099
+ if (this.isSessionBusy(conn)) {
4100
+ this.markBusy();
4101
+ return;
4102
+ }
4103
+ if (this.busy) {
4104
+ this.busy = false;
4105
+ this.fireSessionIdle();
4106
+ this.scheduleIdleDisconnect();
4107
+ } else if (!this.idleTimer) this.scheduleIdleDisconnect();
4108
+ }
4109
+ /**
3510
4110
  * Called synchronously at the start of each RPC method invocation.
3511
4111
  * Renews the DO activity timeout so the sleepAfter alarm is pushed
3512
- * forward before the container processes the call.
4112
+ * forward before the container processes the call, and pins the RPC
4113
+ * WebSocket as busy until the method's promise settles.
4114
+ */
4115
+ recordCallStarted = () => {
4116
+ this.activeCalls++;
4117
+ this.markBusy();
4118
+ this.fireActivity();
4119
+ };
4120
+ recordCallSettled = () => {
4121
+ this.activeCalls = Math.max(0, this.activeCalls - 1);
4122
+ this.maybeTransitionIdle();
4123
+ };
4124
+ /** Return the last connection-startup error captured, if any. */
4125
+ getLastConnectionError = () => this.lastConnectionError;
4126
+ /** Whether the current connection ever established a live session. */
4127
+ getSessionEstablished = () => this.sessionEstablished;
4128
+ /**
4129
+ * Base span attributes for RPC-call spans: sandbox identifiers plus the
4130
+ * container port. Mirrors the connection's `spanAttrs()` so all
4131
+ * `sandbox.rpc.*` spans share a consistent shape.
3513
4132
  */
3514
- renewActivity = () => {
3515
- this.onActivity?.();
4133
+ getSpanAttrs = () => {
4134
+ const info = this.connOptions.getSandboxInfo?.();
4135
+ return {
4136
+ "sandbox.id": info?.id,
4137
+ "sandbox.name": info?.name,
4138
+ "sandbox.rpc.port": this.connOptions.port !== void 0 ? String(this.connOptions.port) : void 0
4139
+ };
3516
4140
  };
3517
4141
  /**
3518
4142
  * Sample `getStats()` and update busy/idle state. While busy, renews the
@@ -3529,19 +4153,12 @@ var ContainerControlClient = class {
3529
4153
  const conn = this.conn;
3530
4154
  if (!conn) return;
3531
4155
  if (!conn.isConnected()) return;
3532
- const { imports, exports } = conn.getStats();
3533
- if (imports > IDLE_IMPORT_THRESHOLD || exports > IDLE_EXPORT_THRESHOLD) {
3534
- if (!this.busy) {
3535
- this.busy = true;
3536
- this.onSessionBusy?.();
3537
- }
3538
- this.onActivity?.();
3539
- this.clearIdleTimer();
3540
- } else if (this.busy) {
3541
- this.busy = false;
3542
- this.onSessionIdle?.();
3543
- this.scheduleIdleDisconnect();
3544
- } else if (!this.idleTimer) this.scheduleIdleDisconnect();
4156
+ if (this.isSessionBusy(conn)) {
4157
+ this.markBusy();
4158
+ this.fireActivity();
4159
+ return;
4160
+ }
4161
+ this.transitionIdleIfReady(conn);
3545
4162
  };
3546
4163
  startBusyPoll() {
3547
4164
  if (this.busyPollTimer) return;
@@ -3559,8 +4176,7 @@ var ContainerControlClient = class {
3559
4176
  this.idleTimer = null;
3560
4177
  const conn = this.conn;
3561
4178
  if (!conn || !conn.isConnected()) return;
3562
- const { imports, exports } = conn.getStats();
3563
- if (imports <= IDLE_IMPORT_THRESHOLD && exports <= IDLE_EXPORT_THRESHOLD) {
4179
+ if (!this.isSessionBusy(conn)) {
3564
4180
  this.logger.debug("Disconnecting idle RPC connection");
3565
4181
  this.destroyConnection();
3566
4182
  }
@@ -3572,47 +4188,49 @@ var ContainerControlClient = class {
3572
4188
  this.idleTimer = null;
3573
4189
  }
3574
4190
  }
3575
- destroyConnection() {
4191
+ destroyConnection(cause) {
3576
4192
  this.stopBusyPoll();
3577
4193
  this.clearIdleTimer();
4194
+ this.activeCalls = 0;
3578
4195
  if (this.busy) {
3579
4196
  this.busy = false;
3580
- this.onSessionIdle?.();
4197
+ this.fireSessionIdle();
3581
4198
  }
3582
4199
  if (this.conn) {
3583
- this.conn.disconnect();
4200
+ this.stampWeakConnectionCause(cause);
4201
+ this.conn.disconnect(cause);
3584
4202
  this.conn = null;
3585
4203
  }
3586
4204
  }
3587
4205
  get commands() {
3588
- return wrapStub(this.getConnection().rpc().commands, "commands", this.renewActivity);
4206
+ return wrapStub(this.getConnection().rpc().commands, "commands", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3589
4207
  }
3590
4208
  get files() {
3591
- return wrapStub(this.getConnection().rpc().files, "files", this.renewActivity);
4209
+ return wrapStub(this.getConnection().rpc().files, "files", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3592
4210
  }
3593
4211
  get processes() {
3594
- return wrapStub(this.getConnection().rpc().processes, "processes", this.renewActivity);
4212
+ return wrapStub(this.getConnection().rpc().processes, "processes", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3595
4213
  }
3596
4214
  get ports() {
3597
- return wrapStub(this.getConnection().rpc().ports, "ports", this.renewActivity);
4215
+ return wrapStub(this.getConnection().rpc().ports, "ports", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3598
4216
  }
3599
4217
  get git() {
3600
- return wrapStub(this.getConnection().rpc().git, "git", this.renewActivity);
4218
+ return wrapStub(this.getConnection().rpc().git, "git", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3601
4219
  }
3602
4220
  get utils() {
3603
- return wrapStub(this.getConnection().rpc().utils, "utils", this.renewActivity);
4221
+ return wrapStub(this.getConnection().rpc().utils, "utils", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3604
4222
  }
3605
4223
  get backup() {
3606
- return wrapStub(this.getConnection().rpc().backup, "backup", this.renewActivity);
4224
+ return wrapStub(this.getConnection().rpc().backup, "backup", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3607
4225
  }
3608
4226
  get watch() {
3609
- return wrapStub(this.getConnection().rpc().watch, "watch", this.renewActivity);
4227
+ return wrapStub(this.getConnection().rpc().watch, "watch", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3610
4228
  }
3611
4229
  get tunnels() {
3612
- return wrapStub(this.getConnection().rpc().tunnels, "tunnels", this.renewActivity);
4230
+ return wrapStub(this.getConnection().rpc().tunnels, "tunnels", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3613
4231
  }
3614
4232
  get interpreter() {
3615
- return wrapStub(this.getConnection().rpc().interpreter, "interpreter", this.renewActivity);
4233
+ return wrapStub(this.getConnection().rpc().interpreter, "interpreter", this.recordCallStarted, this.recordCallSettled, this.getLastConnectionError, this.getSessionEstablished, this.getSpanAttrs);
3616
4234
  }
3617
4235
  /**
3618
4236
  * Update the upgrade retry budget. Applies to the current connection
@@ -3632,8 +4250,42 @@ var ContainerControlClient = class {
3632
4250
  async connect() {
3633
4251
  await this.getConnection().connect();
3634
4252
  }
3635
- disconnect() {
3636
- this.destroyConnection();
4253
+ /**
4254
+ * Tear down the active connection.
4255
+ *
4256
+ * When a connection attempt is still in progress, the teardown is deferred
4257
+ * until that attempt settles — so a lifecycle disconnect (e.g. the DO's
4258
+ * alarm firing `onStop`) cannot rip the transport out from under an
4259
+ * in-flight connect and reject queued calls with a generic disposal error.
4260
+ * The provided `cause` is stamped immediately so that if the attempt fails,
4261
+ * queued calls surface it; if the attempt succeeds, the (now-established)
4262
+ * connection is then torn down cleanly with the cause.
4263
+ *
4264
+ * During this deferral window `this.conn` still points at the connecting
4265
+ * transport so queued calls can keep their original connection context.
4266
+ * Container lifecycle code treats `disconnect()` as terminal and does not
4267
+ * issue more RPC calls after requesting it.
4268
+ *
4269
+ * `cause` should be a typed `SandboxError` describing why the connection is
4270
+ * being torn down (sandbox stopping, lifetime change, transport switch).
4271
+ */
4272
+ disconnect(cause) {
4273
+ const conn = this.conn;
4274
+ if (conn?.isConnecting()) {
4275
+ this.stopBusyPoll();
4276
+ this.clearIdleTimer();
4277
+ this.activeCalls = 0;
4278
+ if (this.busy) {
4279
+ this.busy = false;
4280
+ this.fireSessionIdle();
4281
+ }
4282
+ this.stampWeakConnectionCause(cause);
4283
+ conn.whenSettled().then(() => {
4284
+ if (this.conn === conn) this.destroyConnection(cause);
4285
+ });
4286
+ return;
4287
+ }
4288
+ this.destroyConnection(cause);
3637
4289
  }
3638
4290
  };
3639
4291
 
@@ -5212,6 +5864,33 @@ function isObjectKeyWithinPrefix(objectKey, prefix) {
5212
5864
  if (!normalizedPrefix) return true;
5213
5865
  return objectKey === normalizedPrefix || objectKey.startsWith(`${normalizedPrefix}/`);
5214
5866
  }
5867
+ function parseCopySourceHeader(value) {
5868
+ const encodedPath = value.split("?", 1)[0];
5869
+ if (!encodedPath) return null;
5870
+ let decodedPath;
5871
+ try {
5872
+ decodedPath = decodeURIComponent(encodedPath);
5873
+ } catch {
5874
+ return null;
5875
+ }
5876
+ const path$1 = decodedPath.startsWith("/") ? decodedPath.slice(1) : decodedPath;
5877
+ const separatorIndex = path$1.indexOf("/");
5878
+ if (separatorIndex <= 0 || separatorIndex === path$1.length - 1) return null;
5879
+ return {
5880
+ bucket: path$1.slice(0, separatorIndex),
5881
+ objectKey: path$1.slice(separatorIndex + 1)
5882
+ };
5883
+ }
5884
+ function validateCopySourceWithinMountScope(headers, bucket, prefix) {
5885
+ const copySources = [["x-amz-copy-source", headers.get("x-amz-copy-source")], ["x-goog-copy-source", headers.get("x-goog-copy-source")]].filter((entry) => entry[1] !== null);
5886
+ if (copySources.length > 1) return new Response("Bad Request: multiple copy source headers", { status: 400 });
5887
+ if (copySources.length === 0) return null;
5888
+ const [headerName, value] = copySources[0];
5889
+ const copySource = parseCopySourceHeader(value);
5890
+ if (!copySource) return new Response(`Bad Request: invalid ${headerName}`, { status: 400 });
5891
+ if (copySource.bucket !== bucket || !isObjectKeyWithinPrefix(copySource.objectKey, prefix)) return new Response("Forbidden: copy source is outside mounted scope", { status: 403 });
5892
+ return null;
5893
+ }
5215
5894
  function isRequestWithinMountScope(realPath, url, bucket, prefix) {
5216
5895
  const objectKey = getObjectKeyForPath(realPath, bucket);
5217
5896
  if (objectKey === null) return false;
@@ -5349,6 +6028,8 @@ const s3CredentialProxyHandler = async (request, env$1, ctx) => {
5349
6028
  const realUrl = new URL(realPath + (url.search || ""), mount.endpoint);
5350
6029
  if (isBucketRootProbe(request, realPath, url, mount.bucket)) return new Response(null, { status: 200 });
5351
6030
  if (!isRequestWithinMountScope(realPath, url, mount.bucket, mount.prefix)) return new Response("Forbidden: request is outside mounted bucket scope", { status: 403 });
6031
+ const copySourceError = validateCopySourceWithinMountScope(request.headers, mount.bucket, mount.prefix);
6032
+ if (copySourceError) return copySourceError;
5352
6033
  const cleanHeaders = buildCleanHeaders(request.headers);
5353
6034
  const cleanRequest = new Request(realUrl.toString(), {
5354
6035
  method: request.method,
@@ -6487,7 +7168,7 @@ function createTunnelsHandler(host) {
6487
7168
  * This file is auto-updated by .github/changeset-version.ts during releases
6488
7169
  * DO NOT EDIT MANUALLY - Changes will be overwritten on the next version bump
6489
7170
  */
6490
- const SDK_VERSION = "0.12.4";
7171
+ const SDK_VERSION = "0.12.6";
6491
7172
 
6492
7173
  //#endregion
6493
7174
  //#region src/sandbox.ts
@@ -6540,6 +7221,22 @@ const R2_DEFAULT_S3FS_OPTIONS = {
6540
7221
  const R2_DEFAULT_S3FS_OPTION_ENTRIES = Object.entries(R2_DEFAULT_S3FS_OPTIONS).map(([key, value]) => value === true ? key : `${key}=${value}`);
6541
7222
  const S3FS_DISABLE_EXPECT_HEADER_CONFIG = " Expect:\n";
6542
7223
  const BACKUP_DEFAULT_TTL_SECONDS = 259200;
7224
+ /**
7225
+ * Instance-get budget (ms) for the RPC transport's explicit container-start
7226
+ * hook, independent of the user-facing `containerTimeouts.instanceGetTimeoutMS`
7227
+ * (which still governs the HTTP path's `containerFetch`).
7228
+ *
7229
+ * Shorter than the SDK's user-facing 30s default, but long enough to match
7230
+ * the Containers platform's own 8s instance-get default. This keeps one RPC
7231
+ * start attempt from burning the full 30s under capacity pressure while still
7232
+ * letting ordinary 4-8s cold provisions succeed on the first attempt instead
7233
+ * of always paying the first backoff delay. The RPC control connection's own
7234
+ * retry loop (`fetchWithResponseRetry`, ~2 min budget with exponential
7235
+ * backoff) still owns cross-attempt retries. Once an instance exists,
7236
+ * `portReadyTimeoutMS` still governs app boot on the next attempt (the base
7237
+ * fast-path returns immediately when the container is already running).
7238
+ */
7239
+ const RPC_START_INSTANCE_GET_TIMEOUT_MS = 8e3;
6543
7240
  const BACKUP_MAX_NAME_LENGTH = 256;
6544
7241
  const BACKUP_CONTAINER_DIR = "/var/backups";
6545
7242
  const BACKUP_STORAGE_PREFIX = "backups";
@@ -6949,6 +7646,11 @@ var Sandbox = class Sandbox extends Container {
6949
7646
  port: 3e3,
6950
7647
  logger: this.logger,
6951
7648
  retryTimeoutMs: this.computeRetryTimeoutMs(),
7649
+ getSandboxInfo: () => ({
7650
+ id: this.ctx.id.toString(),
7651
+ name: this.sandboxName ?? void 0
7652
+ }),
7653
+ startContainer: () => this.startContainerForRPC(),
6952
7654
  localMain: this.controlCallback,
6953
7655
  onActivity: () => {
6954
7656
  this.renewActivityTimeout();
@@ -7069,7 +7771,7 @@ var Sandbox = class Sandbox extends Container {
7069
7771
  this.tunnelsHandler = null;
7070
7772
  this.tunnelExitHandler = null;
7071
7773
  this.destroyAllTunnels = null;
7072
- previousClient.disconnect();
7774
+ previousClient.disconnect(this.buildDisconnectCause("runtime_replaced", "The sandbox transport was switched while the operation was pending."));
7073
7775
  }
7074
7776
  if (storedTransport) this.hasStoredTransport = true;
7075
7777
  const storedLabels = await this.ctx.storage.get("labels");
@@ -7154,7 +7856,7 @@ var Sandbox = class Sandbox extends Container {
7154
7856
  this.tunnelsHandler = null;
7155
7857
  this.tunnelExitHandler = null;
7156
7858
  this.destroyAllTunnels = null;
7157
- previousClient.disconnect();
7859
+ previousClient.disconnect(this.buildDisconnectCause("runtime_replaced", "The sandbox transport was switched while the operation was pending."));
7158
7860
  this.renewActivityTimeout();
7159
7861
  this.logger.debug("Transport updated", { transport });
7160
7862
  }
@@ -7844,9 +8546,14 @@ var Sandbox = class Sandbox extends Container {
7844
8546
  }
7845
8547
  await this.ctx.storage.delete("tunnels");
7846
8548
  await this.ctx.storage.delete("tunnels:meta");
7847
- this.client.disconnect();
8549
+ this.client.disconnect(this.buildDisconnectCause("sandbox_lifetime_changed", "The sandbox was destroyed while the operation was pending."));
7848
8550
  outcome = "success";
7849
- await super.destroy();
8551
+ try {
8552
+ await super.destroy();
8553
+ } catch (error) {
8554
+ if (!this.isNoInstanceError(error)) throw error;
8555
+ this.logger.debug("super.destroy() reported no container instance; treating as no-op");
8556
+ }
7850
8557
  } catch (error) {
7851
8558
  caughtError = error instanceof Error ? error : new Error(String(error));
7852
8559
  throw error;
@@ -7937,8 +8644,11 @@ var Sandbox = class Sandbox extends Container {
7937
8644
  versionOutcome: outcome
7938
8645
  }, { successLevel });
7939
8646
  }
7940
- async onStop() {
7941
- this.logger.debug("Sandbox stopped");
8647
+ async onStop(params) {
8648
+ this.logger.debug("Sandbox stopped", {
8649
+ exitCode: params?.exitCode,
8650
+ reason: params?.reason
8651
+ });
7942
8652
  this.containerGeneration++;
7943
8653
  this.defaultSession = null;
7944
8654
  this.defaultSessionInit = null;
@@ -7949,7 +8659,7 @@ var Sandbox = class Sandbox extends Container {
7949
8659
  } catch (error) {
7950
8660
  this.logger.error("Failed to reconcile tunnel storage after container stop", error instanceof Error ? error : new Error(String(error)));
7951
8661
  }
7952
- this.client.disconnect();
8662
+ this.client.disconnect(this.buildDisconnectCause("runtime_replaced", "The sandbox container stopped while the operation was pending.", params));
7953
8663
  let hadR2EgressMount = false;
7954
8664
  let hadCredentialProxyMount = false;
7955
8665
  for (const [, m] of this.activeMounts) if (m.mountType === "local-sync") await m.syncManager.stop().catch(() => {});
@@ -7987,14 +8697,21 @@ var Sandbox = class Sandbox extends Container {
7987
8697
  }
7988
8698
  });
7989
8699
  } catch (e) {
7990
- if (this.isNoInstanceError(e)) {
8700
+ const admissionReason = matchContainerUnavailable(e);
8701
+ if (admissionReason) {
8702
+ const originalMessage = e instanceof Error ? e.message : String(e);
8703
+ const context = {
8704
+ reason: admissionReason,
8705
+ retryable: true,
8706
+ originalMessage
8707
+ };
7991
8708
  const errorBody$1 = {
7992
- code: ErrorCode.INTERNAL_ERROR,
7993
- message: "Container is currently provisioning. This can take several minutes on first deployment.",
7994
- context: { phase: "provisioning" },
8709
+ code: ErrorCode.CONTAINER_UNAVAILABLE,
8710
+ message: originalMessage,
8711
+ context,
7995
8712
  httpStatus: 503,
7996
8713
  timestamp: (/* @__PURE__ */ new Date()).toISOString(),
7997
- suggestion: "This is expected during first deployment. The SDK will retry automatically."
8714
+ suggestion: getSuggestion(ErrorCode.CONTAINER_UNAVAILABLE, context)
7998
8715
  };
7999
8716
  return new Response(JSON.stringify(errorBody$1), {
8000
8717
  status: 503,
@@ -8085,7 +8802,105 @@ var Sandbox = class Sandbox extends Container {
8085
8802
  * This indicates the container VM is still being provisioned.
8086
8803
  */
8087
8804
  isNoInstanceError(error) {
8088
- return error instanceof Error && error.message.toLowerCase().includes("no container instance");
8805
+ return matchContainerUnavailable(error) !== null;
8806
+ }
8807
+ /**
8808
+ * Explicit container-start hook for the RPC transport.
8809
+ *
8810
+ * Runs `startAndWaitForPorts` under a short instance-get budget
8811
+ * ({@link RPC_START_INSTANCE_GET_TIMEOUT_MS}) so a single attempt fails fast
8812
+ * under capacity pressure — the base class throws the classifiable
8813
+ * `NO_CONTAINER_INSTANCE_ERROR` once that budget is exhausted, and the RPC
8814
+ * control connection's own retry loop (with backoff) owns cross-attempt
8815
+ * retries. Port readiness still uses the full `portReadyTimeoutMS` so a
8816
+ * genuinely-booting app isn't cut short once an instance exists.
8817
+ *
8818
+ * Always throws a typed `SandboxError` on failure so the control connection
8819
+ * never surfaces a raw capnweb/transport string to the caller:
8820
+ * - container-admission/capacity failures → retryable
8821
+ * `ContainerUnavailableError` (preserving the platform message so the
8822
+ * connection's `shouldRetryError` still recognizes and retries it);
8823
+ * - anything else (platform-transient DO reset, network loss, unexpected
8824
+ * startup failure) → `INTERNAL_ERROR`-coded SandboxError carrying the
8825
+ * original message.
8826
+ */
8827
+ async startContainerForRPC() {
8828
+ try {
8829
+ await this.startAndWaitForPorts({
8830
+ ports: 3e3,
8831
+ cancellationOptions: {
8832
+ instanceGetTimeoutMS: RPC_START_INSTANCE_GET_TIMEOUT_MS,
8833
+ portReadyTimeoutMS: this.containerTimeouts.portReadyTimeoutMS,
8834
+ waitInterval: this.containerTimeouts.waitIntervalMS
8835
+ }
8836
+ });
8837
+ } catch (error) {
8838
+ throw this.toContainerStartError(error);
8839
+ }
8840
+ }
8841
+ /**
8842
+ * Convert a container-start failure into a typed SandboxError. Preserves an
8843
+ * existing SandboxError as-is; maps platform admission/capacity failures to
8844
+ * a retryable ContainerUnavailableError; and wraps everything else as an
8845
+ * INTERNAL_ERROR carrying the original message (never a raw transport
8846
+ * string).
8847
+ */
8848
+ toContainerStartError(error) {
8849
+ if (error instanceof SandboxError) return error;
8850
+ const originalMessage = error instanceof Error ? error.message : String(error);
8851
+ const admissionReason = matchContainerUnavailable(error);
8852
+ if (admissionReason) {
8853
+ const context$1 = {
8854
+ reason: admissionReason,
8855
+ retryable: true,
8856
+ originalMessage
8857
+ };
8858
+ return createErrorFromResponse({
8859
+ code: ErrorCode.CONTAINER_UNAVAILABLE,
8860
+ message: originalMessage,
8861
+ context: context$1,
8862
+ httpStatus: getHttpStatus(ErrorCode.CONTAINER_UNAVAILABLE),
8863
+ suggestion: getSuggestion(ErrorCode.CONTAINER_UNAVAILABLE, context$1),
8864
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
8865
+ }, { cause: error });
8866
+ }
8867
+ const context = {
8868
+ phase: "startup",
8869
+ error: originalMessage
8870
+ };
8871
+ return createErrorFromResponse({
8872
+ code: ErrorCode.INTERNAL_ERROR,
8873
+ message: `Container failed to start: ${originalMessage}`,
8874
+ context,
8875
+ httpStatus: getHttpStatus(ErrorCode.INTERNAL_ERROR),
8876
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
8877
+ }, { cause: error });
8878
+ }
8879
+ /**
8880
+ * Build a typed lifecycle cause to stamp on the RPC transport when the DO
8881
+ * tears the connection down for its own reasons (container stopped, sandbox
8882
+ * destroyed, transport switched). Handed to `client.disconnect(cause)` so
8883
+ * any RPC calls queued on the transport reject with this actionable reason
8884
+ * instead of the generic capnweb "disposing the main stub" message.
8885
+ */
8886
+ buildDisconnectCause(reason, detail, stopParams) {
8887
+ const context = {
8888
+ reason,
8889
+ operation: "rpc.connect",
8890
+ phase: "connection",
8891
+ admitted: "unknown",
8892
+ retryable: reason === "runtime_replaced",
8893
+ ...stopParams?.exitCode !== void 0 && { containerExitCode: stopParams.exitCode },
8894
+ ...stopParams?.reason !== void 0 && { stopReason: stopParams.reason }
8895
+ };
8896
+ return new OperationInterruptedError({
8897
+ code: ErrorCode.OPERATION_INTERRUPTED,
8898
+ message: detail,
8899
+ context,
8900
+ httpStatus: getHttpStatus(ErrorCode.OPERATION_INTERRUPTED),
8901
+ suggestion: getSuggestion(ErrorCode.OPERATION_INTERRUPTED, context),
8902
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
8903
+ });
8089
8904
  }
8090
8905
  /**
8091
8906
  * Helper: Check if error is a transient startup error that should trigger retry
@@ -10734,4 +11549,4 @@ var Sandbox = class Sandbox extends Container {
10734
11549
 
10735
11550
  //#endregion
10736
11551
  export { PortClient as A, InvalidBackupConfigError as B, collectFile as C, SandboxClient as D, isPlatformTransientError as E, BackupCreateError as F, SessionTerminatedError as G, ProcessExitedBeforeReadyError as H, BackupExpiredError as I, BackupNotFoundError as L, FileClient as M, CommandClient as N, UtilityClient as O, BackupClient as P, BackupRestoreError as R, validateTunnelName as S, isDurableObjectCodeUpdateReset as T, ProcessReadyTimeoutError as U, OperationInterruptedError as V, RPCTransportError as W, responseToAsyncIterable as _, PREVIEW_PROXY_HEADER as a, sanitizeSandboxId as b, PREVIEW_PROXY_SANDBOX_ID_HEADER as c, BucketUnmountError as d, InvalidMountConfigError as f, parseSSEStream as g, asyncIterableToSSEStream as h, proxyTerminal as i, GitClient as j, ProcessClient as k, PREVIEW_PROXY_TOKEN_HEADER as l, S3FSMountError as m, Sandbox as n, PREVIEW_PROXY_HEADERS as o, MissingCredentialsError as p, getSandbox as r, PREVIEW_PROXY_PORT_HEADER as s, ContainerProxy$1 as t, BucketMountError as u, CodeInterpreter as v, streamFile as w, validatePort as x, SandboxSecurityError as y, ContainerUnavailableError as z };
10737
- //# sourceMappingURL=sandbox-CyqG4jca.js.map
11552
+ //# sourceMappingURL=sandbox-FBmZ9mj1.js.map