deepline 0.1.206 → 0.1.207

package/dist/bundling-sources/shared_libs/play-runtime/run-execution-scope.ts

@@ -0,0 +1,256 @@
+import {
+  isRuntimeExecutionCapability,
+  type RuntimeExecutionCapability,
+} from './execution-capabilities';
+
+export type RunExecutionPlacementKind =
+  | 'root'
+  | 'inline_child'
+  | 'scheduled_child';
+
+export type RunExecutionAuthorityInput = {
+  readonly orgId: string;
+  readonly workflowId?: string;
+  readonly billingRunId?: string | null;
+  readonly maxCreditsPerRun?: number | null;
+  readonly integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
+  readonly synthetic?: boolean;
+  readonly capabilities: readonly RuntimeExecutionCapability[];
+};
+
+export type RunExecutionScope = Readonly<{
+  logical: Readonly<{
+    runId: string;
+    playId: string;
+    parentRunId: string | null;
+    parentPlayId: string | null;
+    rootRunId: string;
+    rootPlayId: string;
+  }>;
+  receipt: Readonly<{
+    ownerRunId: string;
+    ownerAttempt: number;
+    namespace: string;
+  }>;
+  placement: Readonly<{
+    kind: RunExecutionPlacementKind;
+    executorRunId: string;
+    executorPlayId: string;
+  }>;
+  /** Persistable runtime authority facts. Bearer tokens and secret names do not belong here. */
+  authority: Readonly<{
+    orgId: string;
+    workflowId: string;
+    runId: string;
+    playId: string;
+    billingRunId: string | null;
+    maxCreditsPerRun: number | null;
+    integrationMode: 'live' | 'eval_stub' | 'fixture' | null;
+    synthetic: boolean;
+    capabilities: readonly RuntimeExecutionCapability[];
+  }>;
+}>;
+
+export type RootRunExecutionScopeInput = {
+  readonly runId: string;
+  readonly playId: string;
+  readonly attempt: number;
+  readonly receiptNamespace: string;
+  readonly authority: RunExecutionAuthorityInput;
+};
+
+export type ChildRunExecutionScopeInput =
+  | {
+      readonly placement: 'inline';
+      readonly runId: string;
+      readonly playId: string;
+      readonly receiptNamespace: string;
+    }
+  | {
+      readonly placement: 'scheduled';
+      readonly runId: string;
+      readonly playId: string;
+      readonly attempt: number;
+      readonly receiptNamespace: string;
+      readonly authority?: Partial<Omit<RunExecutionAuthorityInput, 'orgId'>>;
+    };
+
+function requireIdentity(value: string, field: string): string {
+  if (
+    typeof value !== 'string' ||
+    value.length === 0 ||
+    value !== value.trim()
+  ) {
+    throw new Error(`${field} must be a non-empty, trimmed string.`);
+  }
+  return value;
+}
+
+function requireAttempt(value: number, field: string): number {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new Error(`${field} must be a non-negative safe integer.`);
+  }
+  return value;
+}
+
+function requireMaxCredits(value: number | null | undefined): number | null {
+  if (value == null) return null;
+  if (!Number.isFinite(value) || value < 0) {
+    throw new Error(
+      'authority.maxCreditsPerRun must be a non-negative number.',
+    );
+  }
+  return value;
+}
+
+function requireCapabilities(
+  values: readonly RuntimeExecutionCapability[],
+): readonly RuntimeExecutionCapability[] {
+  if (!Array.isArray(values) || values.length === 0) {
+    throw new Error(
+      'authority.capabilities must contain at least one capability.',
+    );
+  }
+  const capabilities = [...new Set(values)];
+  if (!capabilities.every(isRuntimeExecutionCapability)) {
+    throw new Error('authority.capabilities contains an unknown capability.');
+  }
+  return Object.freeze(capabilities.sort());
+}
+
+function authorityMetadata(input: {
+  runId: string;
+  playId: string;
+  orgId: string;
+  authority: Omit<RunExecutionAuthorityInput, 'orgId'>;
+}): RunExecutionScope['authority'] {
+  return Object.freeze({
+    orgId: requireIdentity(input.orgId, 'authority.orgId'),
+    workflowId: requireIdentity(
+      input.authority.workflowId ?? input.runId,
+      'authority.workflowId',
+    ),
+    runId: input.runId,
+    playId: input.playId,
+    billingRunId:
+      input.authority.billingRunId == null
+        ? null
+        : requireIdentity(
+            input.authority.billingRunId,
+            'authority.billingRunId',
+          ),
+    maxCreditsPerRun: requireMaxCredits(input.authority.maxCreditsPerRun),
+    integrationMode: input.authority.integrationMode ?? null,
+    synthetic: input.authority.synthetic === true,
+    capabilities: requireCapabilities(input.authority.capabilities),
+  });
+}
+
+function freezeScope(input: RunExecutionScope): RunExecutionScope {
+  return Object.freeze({
+    logical: Object.freeze(input.logical),
+    receipt: Object.freeze(input.receipt),
+    placement: Object.freeze(input.placement),
+    authority: input.authority,
+  });
+}
+
+export function createRootRunExecutionScope(
+  input: RootRunExecutionScopeInput,
+): RunExecutionScope {
+  const runId = requireIdentity(input.runId, 'runId');
+  const playId = requireIdentity(input.playId, 'playId');
+  const attempt = requireAttempt(input.attempt, 'attempt');
+  const namespace = requireIdentity(input.receiptNamespace, 'receiptNamespace');
+
+  return freezeScope({
+    logical: {
+      runId,
+      playId,
+      parentRunId: null,
+      parentPlayId: null,
+      rootRunId: runId,
+      rootPlayId: playId,
+    },
+    receipt: { ownerRunId: runId, ownerAttempt: attempt, namespace },
+    placement: { kind: 'root', executorRunId: runId, executorPlayId: playId },
+    authority: authorityMetadata({
+      runId,
+      playId,
+      orgId: input.authority.orgId,
+      authority: input.authority,
+    }),
+  });
+}
+
+export function deriveChildRunExecutionScope(
+  parent: RunExecutionScope,
+  input: ChildRunExecutionScopeInput,
+): RunExecutionScope {
+  const runId = requireIdentity(input.runId, 'runId');
+  const playId = requireIdentity(input.playId, 'playId');
+  const namespace = requireIdentity(input.receiptNamespace, 'receiptNamespace');
+  if (runId === parent.logical.runId) {
+    throw new Error('A child runId must differ from its parent runId.');
+  }
+
+  const logical: RunExecutionScope['logical'] = {
+    runId,
+    playId,
+    parentRunId: parent.logical.runId,
+    parentPlayId: parent.logical.playId,
+    rootRunId: parent.logical.rootRunId,
+    rootPlayId: parent.logical.rootPlayId,
+  };
+
+  if (input.placement === 'inline') {
+    return freezeScope({
+      logical,
+      receipt: {
+        ownerRunId: parent.receipt.ownerRunId,
+        ownerAttempt: parent.receipt.ownerAttempt,
+        namespace,
+      },
+      placement: {
+        kind: 'inline_child',
+        executorRunId: parent.placement.executorRunId,
+        executorPlayId: parent.placement.executorPlayId,
+      },
+      authority: parent.authority,
+    });
+  }
+
+  const attempt = requireAttempt(input.attempt, 'attempt');
+  return freezeScope({
+    logical,
+    receipt: { ownerRunId: runId, ownerAttempt: attempt, namespace },
+    placement: {
+      kind: 'scheduled_child',
+      executorRunId: runId,
+      executorPlayId: playId,
+    },
+    authority: authorityMetadata({
+      runId,
+      playId,
+      orgId: parent.authority.orgId,
+      authority: {
+        workflowId: input.authority?.workflowId,
+        billingRunId:
+          input.authority?.billingRunId === undefined
+            ? parent.authority.billingRunId
+            : input.authority.billingRunId,
+        maxCreditsPerRun:
+          input.authority?.maxCreditsPerRun === undefined
+            ? parent.authority.maxCreditsPerRun
+            : input.authority.maxCreditsPerRun,
+        integrationMode:
+          input.authority?.integrationMode === undefined
+            ? parent.authority.integrationMode
+            : input.authority.integrationMode,
+        synthetic: input.authority?.synthetic ?? parent.authority.synthetic,
+        capabilities:
+          input.authority?.capabilities ?? parent.authority.capabilities,
+      },
+    }),
+  });
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -39,12 +39,15 @@ export type DaytonaStageEmitter = (
 export type AcquiredDaytonaSandbox = {
   sandbox: DaytonaSandbox;
   billingStartedAt: number;
+  billingEndedAt?: number;
 };
 
 export type OneShotDaytonaSandboxLifecycle = {
   startedAt: number;
   acquiredSandboxPromise: Promise<AcquiredDaytonaSandbox>;
   createFreshSandbox: () => Promise<AcquiredDaytonaSandbox>;
+  acquiredSandboxes: () => readonly AcquiredDaytonaSandbox[];
+  settlePendingCreates: () => Promise<void>;
   dispose: () => Promise<void>;
 };
 
@@ -269,8 +272,9 @@ async function acquireOneShotDaytonaSandbox(input: {
     );
   }
   const billingStartedAt = Date.now();
+  const sandbox = result.sandbox;
   input.emitStage('create:done', {
-    sandboxId: result.sandbox.id,
+    sandboxId: sandbox.id,
     attempt: result.attempt,
     elapsedMs: billingStartedAt - input.startedAt,
     attemptElapsedMs: result.attemptElapsedMs,
@@ -278,10 +282,7 @@ async function acquireOneShotDaytonaSandbox(input: {
     memoryGiB: granted.memoryGiB,
     diskGiB: granted.diskGiB,
   });
-  return {
-    sandbox: result.sandbox,
-    billingStartedAt,
-  };
+  return { sandbox, billingStartedAt };
 }
 
 export function createOneShotDaytonaSandboxLifecycle(input: {
@@ -293,6 +294,7 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
   const orgId = validateDaytonaExecutionContext(input.context);
   const startedAt = input.startedAt ?? Date.now();
   let disposed = false;
+  const acquiredSandboxes = new Map<string, AcquiredDaytonaSandbox>();
   let latestAcquiredSandboxPromise: Promise<AcquiredDaytonaSandbox>;
   const createFreshSandbox = () => {
     latestAcquiredSandboxPromise = acquireOneShotDaytonaSandbox({
@@ -301,6 +303,9 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
       context: input.context,
       emitStage: input.emitStage,
       startedAt,
+    }).then((acquired) => {
+      acquiredSandboxes.set(acquired.sandbox.id, acquired);
+      return acquired;
     });
     return latestAcquiredSandboxPromise;
   };
@@ -310,6 +315,13 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
     startedAt,
     acquiredSandboxPromise,
     createFreshSandbox,
+    acquiredSandboxes: () => [...acquiredSandboxes.values()],
+    settlePendingCreates: async () => {
+      await latestAcquiredSandboxPromise.then(
+        () => undefined,
+        () => undefined,
+      );
+    },
     dispose: async () => {
       if (disposed) return;
       disposed = true;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -4,6 +4,7 @@ import type {
   PlayRunnerPreparedExecution,
   PlayRunnerPrepareInput,
 } from '../types';
+import { RuntimeResourceFenceLostError } from '../types';
 import { buildPlayRunnerBundle } from '../bundle';
 import {
   findPlayRunnerResult,
@@ -113,6 +114,8 @@ export async function withDaytonaOperationDeadline<T>(input: {
 }
 const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
   /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
+const RUNTIME_RECEIPT_GATEWAY_TRANSPORT_RETRY_PATTERN =
+  /AppRuntimeApiTransportError: App runtime API transport exhausted action=(?:get|claim|mark|complete|fail|heartbeat|release|skip)_runtime_step_receipts?/i;
 const RUNTIME_API_TRANSPORT_RETRY_PATTERN =
   /\bRuntime API request to .+ failed before receiving a response:\s*(?:fetch failed|connection (?:terminated|timed out|closed|reset)|ECONNRESET|ETIMEDOUT|ECONNREFUSED)\b/i;
 const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =
@@ -498,11 +501,18 @@ function createDaytonaFailedResult(input: {
 
 function retryableRuntimeInitializationFailureReason(
   result: PlayRunnerResult,
-): 'runtime_postgres_connect' | 'runtime_api_transport' | null {
+):
+  | 'runtime_postgres_connect'
+  | 'runtime_api_transport'
+  | 'runtime_receipt_gateway_transport'
+  | null {
   if (result.status !== 'failed') return null;
   if (RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN.test(result.error)) {
     return 'runtime_postgres_connect';
   }
+  if (RUNTIME_RECEIPT_GATEWAY_TRANSPORT_RETRY_PATTERN.test(result.error)) {
+    return 'runtime_receipt_gateway_transport';
+  }
   const rowsProcessed = Number(result.stats?.rowsProcessed ?? 0);
   if (
     rowsProcessed === 0 &&
@@ -617,13 +627,69 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
     };
     let cancellationRequested = false;
     let cancellationCleanupStarted = false;
+    let activeAcquiredResource: {
+      sandbox: DaytonaSandbox;
+      billingStartedAt: number;
+      billingEndedAt?: number;
+    } | null = null;
     const uploadDeadlineBreaches: DaytonaUploadAttemptTiming[] = [];
+    const reportedSandboxIds = new Set<string>();
+    const reportedSandboxEndTimes = new Map<string, number | null>();
+    const runtimeResourceReportErrors = new Set<unknown>();
+    const reportRuntimeResource = async (acquired: {
+      sandbox: DaytonaSandbox;
+      billingStartedAt: number;
+      billingEndedAt?: number;
+    }) => {
+      const sandbox = acquired.sandbox;
+      const billingEndedAt = acquired.billingEndedAt ?? null;
+      if (
+        reportedSandboxIds.has(sandbox.id) &&
+        reportedSandboxEndTimes.get(sandbox.id) === billingEndedAt
+      ) {
+        return;
+      }
+      try {
+        await callbacks?.onRuntimeResourceAcquired?.({
+          kind: 'daytona_sandbox',
+          sandboxId: sandbox.id,
+          billingStartedAt: acquired.billingStartedAt,
+          billingEndedAt,
+          cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
+          memoryGiB: typeof sandbox.memory === 'number' ? sandbox.memory : null,
+          diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
+        });
+      } catch (error) {
+        runtimeResourceReportErrors.add(error);
+        throw error;
+      }
+      reportedSandboxIds.add(sandbox.id);
+      reportedSandboxEndTimes.set(sandbox.id, billingEndedAt);
+    };
+    const reportRetiringRuntimeResource = async (acquired: {
+      sandbox: DaytonaSandbox;
+      billingStartedAt: number;
+      billingEndedAt?: number;
+    }) => {
+      try {
+        await reportRuntimeResource(acquired);
+      } catch (error) {
+        if (error instanceof RuntimeResourceFenceLostError) throw error;
+        console.warn('[play-runner.daytona.resource_report_deferred]', {
+          sandboxId: acquired.sandbox.id,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    };
     let cancelExecution!: (error: Error) => void;
     const cancellationPromise = new Promise<never>((_resolve, reject) => {
       cancelExecution = reject;
     });
     const onCancel = () => {
       cancellationRequested = true;
+      if (activeAcquiredResource && !activeAcquiredResource.billingEndedAt) {
+        activeAcquiredResource.billingEndedAt = Date.now();
+      }
       if (cancellationCleanupStarted) {
         cancelExecution(new Error(DAYTONA_CANCELLED_ERROR));
         return;
@@ -663,28 +729,20 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             acquiredSandboxPromise,
             cancellationPromise,
           ]);
+          activeAcquiredResource = acquiredSandbox;
           sandboxCleanup.activate(acquiredSandbox);
           runtimeTiming.daytonaCreateMs =
             acquiredSandbox.billingStartedAt - startedAt;
           const sandbox = acquiredSandbox.sandbox;
           sandboxForAttempt = sandbox;
-          const resourceAcquiredNotice = callbacks?.onRuntimeResourceAcquired?.(
-            {
-              kind: 'daytona_sandbox',
-              sandboxId: sandbox.id,
-              billingStartedAt: acquiredSandbox.billingStartedAt,
-              cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
-              memoryGiB:
-                typeof sandbox.memory === 'number' ? sandbox.memory : null,
-              diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
-            },
-          );
-          if (resourceAcquiredNotice) {
-            await withDaytonaOperationDeadline({
-              operation: 'onRuntimeResourceAcquired',
-              deadlineMs: DAYTONA_API_CALL_DEADLINE_MS,
-              promise: resourceAcquiredNotice,
-            });
+          await reportRuntimeResource(acquiredSandbox);
+          // Account for every acquired retry sandbox before customer code runs.
+          // This keeps resource persistence failures on the pre-side-effect
+          // side of the execution boundary and makes cancellation billing
+          // complete without a post-terminal write race.
+          await sandboxLifecycle.settlePendingCreates();
+          for (const acquired of sandboxLifecycle.acquiredSandboxes()) {
+            await reportRuntimeResource(acquired);
           }
           throwIfCancellationRequested();
 
@@ -748,6 +806,8 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
                 elapsedMs: Date.now() - startedAt,
               });
               if (executionAttempt < DAYTONA_UPLOAD_MAX_ATTEMPTS) {
+                acquiredSandbox.billingEndedAt = Date.now();
+                await reportRetiringRuntimeResource(acquiredSandbox);
                 sandboxCleanup.stashActiveSandboxForRetry();
                 emitDaytonaStage(callbacks, config.context, 'recreate:start', {
                   previousSandboxId: sandbox.id,
@@ -893,6 +953,8 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
                 error: result.error,
                 elapsedMs: Date.now() - startedAt,
               });
+              acquiredSandbox.billingEndedAt = Date.now();
+              await reportRetiringRuntimeResource(acquiredSandbox);
               sandboxCleanup.stashActiveSandboxForRetry();
               acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
               continue;
@@ -964,6 +1026,8 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               elapsedMs: Date.now() - startedAt,
             });
             if (sandboxForAttempt) {
+              activeAcquiredResource!.billingEndedAt = Date.now();
+              await reportRetiringRuntimeResource(activeAcquiredResource!);
               sandboxCleanup.stashActiveSandboxForRetry();
             }
             acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
@@ -981,6 +1045,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
         }),
       );
     } catch (error) {
+      // Resource persistence belongs to the scheduler control plane. Preserve
+      // its typed capacity/fence errors so Absurd can defer or fence the
+      // attempt; converting them into a runner failure would terminally fail a
+      // play that never began executing customer code.
+      if (runtimeResourceReportErrors.has(error)) throw error;
       emitDaytonaStage(callbacks, config.context, 'execute:error', {
         sandboxId: sandboxCleanup.currentSandbox()?.id ?? null,
         error: formatDaytonaError(error),
@@ -999,6 +1068,22 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
       );
     } finally {
       callbacks?.cancellationSignal?.removeEventListener('abort', onCancel);
+      if (cancellationRequested) {
+        await sandboxLifecycle.settlePendingCreates();
+        for (const acquired of sandboxLifecycle.acquiredSandboxes()) {
+          try {
+            await reportRuntimeResource(acquired);
+          } catch (error) {
+            console.error(
+              '[play-runner.daytona.cancel_resource_report_failed]',
+              {
+                sandboxId: acquired.sandbox.id,
+                error: error instanceof Error ? error.message : String(error),
+              },
+            );
+          }
+        }
+      }
     }
   },
 };

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts

@@ -14,11 +14,19 @@ export type PlayRunnerRuntimeResource =
       kind: 'daytona_sandbox';
       sandboxId: string;
       billingStartedAt: number;
+      billingEndedAt?: number | null;
       cpu?: number | null;
       memoryGiB?: number | null;
       diskGiB?: number | null;
     };
 
+export class RuntimeResourceFenceLostError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'RuntimeResourceFenceLostError';
+  }
+}
+
 export interface PlayRunnerCallbacks {
   onLog?: (event: PlayRunnerLogEvent) => void;
   onCheckpoint?: (checkpoint: PlayCheckpoint) => void;

package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts

@@ -84,7 +84,7 @@ export type RuntimeBillingAction =
       workflowId?: string;
       runId?: string;
       maxCreditsPerRun?: number | null;
-      finalItem: ComputeBillingItem;
+      finalItem?: ComputeBillingItem;
     };
 
 export type RuntimeReceiptAction =

package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts

@@ -45,6 +45,11 @@ const pgRuntimePoolFactory: RuntimePoolFactory = (input): RuntimePool => {
     idleTimeoutMillis: input.idleTimeoutMs ?? 15_000,
     connectionTimeoutMillis: input.connectTimeoutMs ?? 10_000,
   });
+  pool.on('error', (error) => {
+    console.warn(
+      `[warn] runtime Postgres discarded an idle client after an error: ${error.message}`,
+    );
+  });
   return wrapPgPool(pool);
 };
 

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts

@@ -0,0 +1,74 @@
+import {
+  resolveDaytonaSandboxComputeItem,
+  type ComputeBillingItem,
+} from './worker-api-types';
+
+function finiteNonNegative(value: unknown): number | null {
+  return typeof value === 'number' && Number.isFinite(value)
+    ? Math.max(0, value)
+    : null;
+}
+
+/** Project executor resource facts into one idempotent item per real sandbox. */
+export function resolveDaytonaRuntimeComputeItems(input: {
+  resources: readonly unknown[];
+  endedAt: number;
+}): ComputeBillingItem[] {
+  const resourcesBySandboxId = new Map<
+    string,
+    {
+      startedAt: number;
+      endedAt: number | null;
+      cpu: number | null;
+      memoryGiB: number | null;
+      diskGiB: number | null;
+    }
+  >();
+  const items: ComputeBillingItem[] = [];
+  for (const resource of input.resources) {
+    if (!resource || typeof resource !== 'object' || Array.isArray(resource)) {
+      continue;
+    }
+    const value = resource as Record<string, unknown>;
+    const sandboxId =
+      typeof value.sandboxId === 'string' ? value.sandboxId.trim() : '';
+    if (value.kind !== 'daytona_sandbox' || !sandboxId) {
+      continue;
+    }
+    const billingStartedAt = finiteNonNegative(value.billingStartedAt);
+    if (billingStartedAt === null) continue;
+    const billingEndedAt = finiteNonNegative(value.billingEndedAt);
+    const existing = resourcesBySandboxId.get(sandboxId);
+    resourcesBySandboxId.set(sandboxId, {
+      startedAt: Math.min(
+        existing?.startedAt ?? billingStartedAt,
+        billingStartedAt,
+      ),
+      endedAt:
+        billingEndedAt === null
+          ? existing?.endedAt ?? null
+          : Math.max(existing?.endedAt ?? billingEndedAt, billingEndedAt),
+      cpu: finiteNonNegative(value.cpu) ?? existing?.cpu ?? null,
+      memoryGiB:
+        finiteNonNegative(value.memoryGiB) ?? existing?.memoryGiB ?? null,
+      diskGiB: finiteNonNegative(value.diskGiB) ?? existing?.diskGiB ?? null,
+    });
+  }
+  for (const [sandboxId, resource] of resourcesBySandboxId) {
+    const endedAt = resource.endedAt ?? input.endedAt;
+    items.push(
+      resolveDaytonaSandboxComputeItem({
+        itemId: `daytona:${sandboxId}`,
+        source: 'daytona',
+        wallTimeSeconds: Math.max(0, endedAt - resource.startedAt) / 1_000,
+        cpu: resource.cpu ?? 1,
+        memoryGiB: resource.memoryGiB ?? 1,
+        diskGiB: resource.diskGiB ?? 0,
+        sandboxId,
+        startedAt: resource.startedAt,
+        endedAt,
+      }),
+    );
+  }
+  return items;
+}

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -1,3 +1,5 @@
+import { getRuntimeEnv } from '../runtime-env';
+
 export const PLAY_RUNTIME_TEST_FAULT_HEADER = 'x-deepline-test-fault';
 
 export type RuntimeTestFaultName =
@@ -82,15 +84,6 @@ const RUNTIME_TEST_POLICY_WORK_BUDGET_KEYS = new Set([
 const MAX_RUNTIME_TEST_POLICY_MS = 10 * 60_000;
 const MAX_RUNTIME_TEST_POLICY_YIELD_LIMIT = 1_000_000;
 
-function isProductionRuntime(): boolean {
-  const deeplineEnv = process.env.DEEPLINE_ENV?.trim().toLowerCase();
-  if (deeplineEnv === 'production' || deeplineEnv === 'prod') return true;
-  const vercelEnv = process.env.VERCEL_ENV?.trim().toLowerCase();
-  if (vercelEnv === 'production') return true;
-  if (vercelEnv === 'preview' || vercelEnv === 'development') return false;
-  return process.env.NODE_ENV === 'production';
-}
-
 function testRuntimeSeamsEnabled(): boolean {
   return process.env.DEEPLINE_TEST_RUNTIME_SEAMS === '1';
 }
@@ -243,7 +236,7 @@ export function validateRuntimeTestPolicyOverrides(input: {
   if (input.value === undefined || input.value === null) {
     return { ok: true, overrides: null };
   }
-  if (!runtimeTestSeamsAuthorized(input) || isProductionRuntime()) {
+  if (!runtimeTestSeamsAuthorized(input) || getRuntimeEnv() === 'prod') {
     return {
       ok: false,
       status: 403,
@@ -270,7 +263,7 @@ export function readRuntimeTestFaultRegistry(input: {
   if (!headerValue) {
     return { ok: true, registry: null };
   }
-  if (!runtimeTestSeamsAuthorized(input) || isProductionRuntime()) {
+  if (!runtimeTestSeamsAuthorized(input) || getRuntimeEnv() === 'prod') {
     return {
       ok: false,
       status: 403,
@@ -323,7 +316,7 @@ export function parseRuntimeTestFaultCounts(input: {
   if (!headerValue) {
     return { ok: true, counts: null, headerValue: null };
   }
-  if (!runtimeTestSeamsAuthorized(input) || isProductionRuntime()) {
+  if (!runtimeTestSeamsAuthorized(input) || getRuntimeEnv() === 'prod') {
     return {
       ok: false,
       status: 403,