deepline 0.3.23 → 0.3.25

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts

@@ -36,10 +36,50 @@ const MODAL_RUNNER_READY_TIMEOUT_MS =
   RUNTIME_RELIABILITY_POLICY.sandbox.runnerReadyTimeoutMs;
 const MODAL_SANDBOX_CREATE_TIMEOUT_MS =
   RUNTIME_RELIABILITY_POLICY.sandbox.modalSandboxCreateTimeoutMs;
+const MODAL_PAYLOAD_UPLOAD_TIMEOUT_MS = 90_000;
+const MODAL_TERMINATION_TIMEOUT_MS = 10_000;
 const MODAL_CAPACITY_RELEASE_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
 const MODAL_LATE_CREATE_PERSIST_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
 type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
 
+class ModalPayloadUploadTimeoutError extends Error {
+  constructor(input: { sandboxId: string; path: string; bytes: number }) {
+    super(
+      `MODAL_PAYLOAD_UPLOAD_TIMEOUT: Modal payload upload did not complete within ${MODAL_PAYLOAD_UPLOAD_TIMEOUT_MS}ms (sandbox=${input.sandboxId}, path=${input.path}, bytes=${input.bytes}).`,
+    );
+    this.name = 'ModalPayloadUploadTimeoutError';
+  }
+}
+
+async function writeModalFileWithDeadline(input: {
+  sandbox: ModalSandbox;
+  content: Buffer;
+  path: string;
+  cancellation: Promise<never>;
+}): Promise<void> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(() => {
+      reject(
+        new ModalPayloadUploadTimeoutError({
+          sandboxId: input.sandbox.sandboxId,
+          path: input.path,
+          bytes: input.content.byteLength,
+        }),
+      );
+    }, MODAL_PAYLOAD_UPLOAD_TIMEOUT_MS);
+  });
+  try {
+    await Promise.race([
+      input.sandbox.filesystem.writeBytes(input.content, input.path),
+      input.cancellation,
+      deadline,
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
 /**
  * A create RPC that outlives its client deadline is fundamentally ambiguous:
  * Modal may still create the sandbox after the caller stops waiting.  It is
@@ -185,10 +225,22 @@ async function confirmDetachedModalRunnerReady(input: {
 }
 
 async function terminateModalSandbox(sandbox: ModalSandbox): Promise<boolean> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
   try {
     // The default terminate call only acknowledges the request. Capacity may
     // be released only after Modal confirms the sandbox itself has stopped.
-    await sandbox.terminate({ wait: true });
+    await Promise.race([
+      sandbox.terminate({ wait: true }),
+      new Promise<never>((_resolve, reject) => {
+        timer = setTimeout(() => {
+          reject(
+            new Error(
+              `Modal sandbox termination did not settle within ${MODAL_TERMINATION_TIMEOUT_MS}ms.`,
+            ),
+          );
+        }, MODAL_TERMINATION_TIMEOUT_MS);
+      }),
+    ]);
     return true;
   } catch (error) {
     console.warn('[play-runner.modal.terminate_failed]', {
@@ -196,6 +248,8 @@ async function terminateModalSandbox(sandbox: ModalSandbox): Promise<boolean> {
       error: error instanceof Error ? error.message : String(error),
     });
     return false;
+  } finally {
+    if (timer) clearTimeout(timer);
   }
 }
 
@@ -529,7 +583,12 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
             sandbox: {
               id: sandbox.sandboxId,
               uploadFile: (content, path) =>
-                sandbox.filesystem.writeBytes(content, path),
+                writeModalFileWithDeadline({
+                  sandbox,
+                  content,
+                  path,
+                  cancellation: cancellationPromise,
+                }),
             },
             bundlePromise: buildPlayRunnerBundle(),
             config,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts

@@ -1,6 +1,7 @@
 import type { PlayRunnerBackend } from './types';
 import { daytonaPlayRunnerBackend } from './backends/daytona';
 import {
+  daytonaModalCanaryPlayRunnerBackend,
   daytonaModalFallbackPlayRunnerBackend,
   daytonaOnlyPlayRunnerBackend,
   modalOnlyPlayRunnerBackend,
@@ -14,6 +15,8 @@ import {
   type PlayRuntimeBackendId,
 } from '@shared_libs/play-runtime/backend';
 import {
+  isRuntimeSandboxCanaryProduction,
+  RUNTIME_SANDBOX_PLACEMENT_POLICIES,
   runtimeSandboxPlacementPolicyForBackend,
   type RuntimeSandboxPlacementPolicyId,
 } from '@shared_libs/play-runtime/runtime-sandbox-placement-policy';
@@ -32,6 +35,13 @@ export function resolvePlayRunnerBackend(
   );
   const placementPolicy = runtimeSandboxPlacementPolicyForBackend(backend);
   if (placementPolicy) {
+    if (
+      placementPolicy.id ===
+        RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1 &&
+      !isRuntimeSandboxCanaryProduction({ nodeEnv: process.env.NODE_ENV })
+    ) {
+      return daytonaModalFallbackPlayRunnerBackend;
+    }
     return resolveRuntimeSandboxPlacementBackend(placementPolicy.id);
   }
   if (backend === PLAY_RUNTIME_BACKENDS.localProcess) {
@@ -42,6 +52,7 @@ export function resolvePlayRunnerBackend(
 
 export {
   daytonaPlayRunnerBackend,
+  daytonaModalCanaryPlayRunnerBackend,
   daytonaModalFallbackPlayRunnerBackend,
   daytonaOnlyPlayRunnerBackend,
   localProcessPlayRunnerBackend,

package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts

@@ -1,4 +1,6 @@
 import { PLAY_RUNTIME_BACKENDS, type PlayRuntimeBackendId } from './backend';
+import { createHash } from 'node:crypto';
+import { isIsolatedRuntimeSchedulerSchema } from './runtime-scheduler-topology';
 
 export const RUNTIME_SANDBOX_PROVIDERS = {
   daytona: 'daytona',
@@ -15,6 +17,7 @@ export type RuntimeSandboxProvider =
 export const RUNTIME_SANDBOX_PLACEMENT_POLICIES = {
   daytonaOnlyV1: 'daytona_only@1',
   daytonaThenModalV1: 'daytona_then_modal@1',
+  daytonaModalCanaryV1: 'daytona_modal_canary@1',
   modalOnlyV1: 'modal_only@1',
 } as const;
 
@@ -41,8 +44,59 @@ export type RuntimeSandboxPlacementPolicy = Readonly<{
   transitions: readonly RuntimeSandboxPlacementTransition[];
   requiredCredentialEnv: readonly string[];
   cleanupProviders: readonly RuntimeSandboxProvider[];
+  canary?: Readonly<{
+    provider: RuntimeSandboxProvider;
+    basisPoints: number;
+    selector: 'sha256_run_id_basis_points@1';
+    environment: 'production';
+  }>;
 }>;
 
+const CANARY_BUCKET_COUNT = 10_000;
+
+export type RuntimeSandboxCanarySelection = Readonly<{
+  canary: RuntimeSandboxProvider | null;
+  bucket: number | null;
+  basisPoints: number;
+}>;
+
+export function isRuntimeSandboxCanaryProduction(input: {
+  nodeEnv?: string | null;
+  runtimeSchedulerSchema?: string | null;
+}): boolean {
+  return (
+    input.nodeEnv === 'production' &&
+    !isIsolatedRuntimeSchedulerSchema(input.runtimeSchedulerSchema)
+  );
+}
+
+/** Stable across scheduler attempts and worker processes for a durable run id. */
+export function selectRuntimeSandboxCanary(input: {
+  policy: RuntimeSandboxPlacementPolicy;
+  runId?: string | null;
+  production: boolean;
+}): RuntimeSandboxCanarySelection {
+  const canary = input.policy.canary;
+  const runId = input.runId?.trim();
+  if (!canary || !runId || !input.production) {
+    return {
+      canary: null,
+      bucket: null,
+      basisPoints: canary?.basisPoints ?? 0,
+    };
+  }
+  const bucket =
+    createHash('sha256')
+      .update(`runtime-sandbox-placement@1:${runId}`)
+      .digest()
+      .readUInt32BE(0) % CANARY_BUCKET_COUNT;
+  return {
+    canary: bucket < canary.basisPoints ? canary.provider : null,
+    bucket,
+    basisPoints: canary.basisPoints,
+  };
+}
+
 const DAYTONA_THEN_MODAL_V1: RuntimeSandboxPlacementPolicy = {
   id: RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaThenModalV1,
   compatibilityRuntimeBackend: PLAY_RUNTIME_BACKENDS.daytona,
@@ -72,6 +126,23 @@ const DAYTONA_THEN_MODAL_V1: RuntimeSandboxPlacementPolicy = {
   ],
 };
 
+/**
+ * Staged Daytona-compatible policy. Workers learn this policy before launch
+ * writers start persisting it, so an app-first deploy cannot strand runs on an
+ * unsupported policy id. One percent of production runs enter the same Modal
+ * Adapter used by DAYTONA_THEN_MODAL_V1 after a capacity failure.
+ */
+const DAYTONA_MODAL_CANARY_V1: RuntimeSandboxPlacementPolicy = {
+  ...DAYTONA_THEN_MODAL_V1,
+  id: RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1,
+  canary: {
+    provider: RUNTIME_SANDBOX_PROVIDERS.modal,
+    basisPoints: 100,
+    selector: 'sha256_run_id_basis_points@1',
+    environment: 'production',
+  },
+};
+
 /**
  * Frozen compatibility policy for launches written before placement policy ids
  * existed. It is retained until every pre-policy release lane has drained.
@@ -100,6 +171,8 @@ export const RUNTIME_SANDBOX_PLACEMENT_POLICY_CATALOG: Readonly<
   [RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaOnlyV1]: DAYTONA_ONLY_V1,
   [RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaThenModalV1]:
     DAYTONA_THEN_MODAL_V1,
+  [RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1]:
+    DAYTONA_MODAL_CANARY_V1,
   [RUNTIME_SANDBOX_PLACEMENT_POLICIES.modalOnlyV1]: MODAL_ONLY_V1,
 };
 
@@ -129,7 +202,7 @@ export function runtimeSandboxPlacementPolicyForBackend(
   backend: PlayRuntimeBackendId,
 ): RuntimeSandboxPlacementPolicy | null {
   if (backend === PLAY_RUNTIME_BACKENDS.daytona) {
-    return DAYTONA_THEN_MODAL_V1;
+    return DAYTONA_MODAL_CANARY_V1;
   }
   if (backend === PLAY_RUNTIME_BACKENDS.modal) {
     return MODAL_ONLY_V1;

package/dist/bundling-sources/shared_libs/plays/tool-category-descriptions.ts

@@ -42,6 +42,11 @@ export const TOOL_CATEGORY_DESCRIPTIONS: Readonly<Record<string, string>> = {
   free: 'Free tools that do not spend Deepline credits.',
 };
 
+/** Stable UI/CLI enumeration of the well-known canonical category slugs. */
+export const WELL_KNOWN_TOOL_CATEGORIES = Object.freeze(
+  Object.keys(TOOL_CATEGORY_DESCRIPTIONS),
+);
+
 /**
  * One-line definition for a tool category slug, or null when none is declared.
  * Never fabricates a definition for unknown slugs.