deepline 0.3.22 → 0.3.24

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-modal-fallback.ts

@@ -11,10 +11,14 @@ import type {
 import { daytonaPlayRunnerBackend } from './daytona';
 import { DaytonaSandboxAcquisitionUnavailableError } from './daytona-lifecycle';
 import { modalPlayRunnerBackend } from './modal';
+import { isPlayRunRecoveryError } from '@shared_libs/play-runtime/play-run-recovery-policy';
 import {
   canTransitionRuntimeSandboxPlacement,
+  isRuntimeSandboxCanaryProduction,
   resolveRuntimeSandboxPlacementPolicy,
   RUNTIME_SANDBOX_PLACEMENT_POLICIES,
+  selectRuntimeSandboxCanary,
+  type RuntimeSandboxCanarySelection,
   type RuntimeSandboxPlacementPolicyId,
   type RuntimeSandboxPlacementFailureReason,
   type RuntimeSandboxPlacementPolicy,
@@ -41,6 +45,54 @@ type RuntimeSandboxProviderAdapter = {
   ) => RuntimeSandboxPlacementFailureReason | null;
 };
 
+function selectionForRun(
+  policy: RuntimeSandboxPlacementPolicy,
+  runId?: string | null,
+  runtimeSchedulerSchema?: string | null,
+): RuntimeSandboxCanarySelection {
+  return selectRuntimeSandboxCanary({
+    policy,
+    runId,
+    production: isRuntimeSandboxCanaryProduction({
+      nodeEnv: process.env.NODE_ENV,
+      runtimeSchedulerSchema,
+    }),
+  });
+}
+
+function logCanaryOutcome(input: {
+  config: PlayRunnerExecutionConfig;
+  policy: RuntimeSandboxPlacementPolicy;
+  selection: RuntimeSandboxCanarySelection;
+  outcome: string;
+  error?: unknown;
+}): void {
+  const recoveryReason = isPlayRunRecoveryError(input.error)
+    ? input.error.decision.reason
+    : null;
+  console.warn(
+    '[play-runner.sandbox_provider_canary]',
+    JSON.stringify({
+      runId: input.config.context.runId ?? null,
+      workflowId: input.config.context.workflowId ?? null,
+      runAttempt: input.config.context.runAttempt ?? null,
+      policyId: input.policy.id,
+      provider: input.selection.canary,
+      bucket: input.selection.bucket,
+      basisPoints: input.selection.basisPoints,
+      runtimeEnvironment: 'production',
+      outcome: input.outcome,
+      phase: 'startup',
+      // The Modal Adapter also performs scheduler-owned fencing, staging, and
+      // readiness work. Page only for its existing typed provider decisions;
+      // an unclassified setup failure remains observable without blaming Modal.
+      providerHealthFailure: recoveryReason?.startsWith('modal_') ?? false,
+      errorName: input.error instanceof Error ? input.error.name : null,
+      recoveryReason,
+    }),
+  );
+}
+
 function classifyDaytonaAcquisitionFailure(
   error: unknown,
 ): RuntimeSandboxPlacementFailureReason | null {
@@ -70,18 +122,23 @@ export function createRuntimeSandboxPlacementBackend(input: {
     Partial<Record<RuntimeSandboxProvider, RuntimeSandboxProviderAdapter>>
   >;
 }): PlayRunnerBackend {
-  const primaryProvider = input.policy.providers[0];
-  const primaryAdapter = input.adapters[primaryProvider];
-  if (!primaryAdapter) {
-    throw new Error(
-      `Runtime sandbox placement policy ${input.policy.id} has no Adapter for ${primaryProvider}.`,
-    );
-  }
   return {
     async prepare(
       prepareInput: PlayRunnerPrepareInput,
       callbacks?: PlayRunnerCallbacks,
     ): Promise<PlayRunnerPreparedExecution> {
+      const selection = selectionForRun(
+        input.policy,
+        prepareInput.context.runId,
+        prepareInput.context.runtimeSchedulerSchema,
+      );
+      const primaryProvider = selection.canary ?? input.policy.providers[0];
+      const primaryAdapter = input.adapters[primaryProvider];
+      if (!primaryAdapter) {
+        throw new Error(
+          `Runtime sandbox placement policy ${input.policy.id} has no Adapter for ${primaryProvider}.`,
+        );
+      }
       const providerPrepared = await primaryAdapter.backend.prepare?.(
         prepareInput,
         callbacks,
@@ -105,7 +162,15 @@ export function createRuntimeSandboxPlacementBackend(input: {
         isPrepared(prepared) && prepared.policyId === input.policy.id
           ? prepared
           : null;
-      for (const [index, provider] of input.policy.providers.entries()) {
+      const selection = selectionForRun(
+        input.policy,
+        config.context.runId,
+        config.context.runtimeSchedulerSchema,
+      );
+      const providers = selection.canary
+        ? ([selection.canary] as const)
+        : input.policy.providers;
+      for (const [index, provider] of providers.entries()) {
         const adapter = input.adapters[provider];
         if (!adapter) {
           throw new Error(
@@ -113,15 +178,34 @@ export function createRuntimeSandboxPlacementBackend(input: {
           );
         }
         try {
-          return await adapter.backend.execute(
+          const result = await adapter.backend.execute(
             config,
             callbacks,
             index === 0 && policyPrepared?.provider === provider
               ? policyPrepared.providerPrepared
               : undefined,
           );
+          if (selection.canary) {
+            logCanaryOutcome({
+              config,
+              policy: input.policy,
+              selection,
+              outcome: result.status,
+            });
+          }
+          return result;
         } catch (error) {
-          const nextProvider = input.policy.providers[index + 1];
+          if (selection.canary) {
+            logCanaryOutcome({
+              config,
+              policy: input.policy,
+              selection,
+              outcome: 'error',
+              error,
+            });
+            throw error;
+          }
+          const nextProvider = providers[index + 1];
           const reason = adapter.classifyAcquisitionFailure(error);
           if (
             !nextProvider ||
@@ -199,6 +283,11 @@ export function createDaytonaModalFallbackBackend(
 export const daytonaModalFallbackPlayRunnerBackend =
   createDaytonaModalFallbackBackend();
 
+export const daytonaModalCanaryPlayRunnerBackend =
+  createRuntimeSandboxPlacementBackendForPolicy(
+    RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1,
+  );
+
 export const daytonaOnlyPlayRunnerBackend =
   createRuntimeSandboxPlacementBackendForPolicy(
     RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaOnlyV1,
@@ -216,6 +305,8 @@ const DEFAULT_POLICY_BACKENDS: Readonly<
     daytonaOnlyPlayRunnerBackend,
   [RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaThenModalV1]:
     daytonaModalFallbackPlayRunnerBackend,
+  [RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1]:
+    daytonaModalCanaryPlayRunnerBackend,
   [RUNTIME_SANDBOX_PLACEMENT_POLICIES.modalOnlyV1]: modalOnlyPlayRunnerBackend,
 };
 

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts

@@ -36,10 +36,50 @@ const MODAL_RUNNER_READY_TIMEOUT_MS =
   RUNTIME_RELIABILITY_POLICY.sandbox.runnerReadyTimeoutMs;
 const MODAL_SANDBOX_CREATE_TIMEOUT_MS =
   RUNTIME_RELIABILITY_POLICY.sandbox.modalSandboxCreateTimeoutMs;
+const MODAL_PAYLOAD_UPLOAD_TIMEOUT_MS = 90_000;
+const MODAL_TERMINATION_TIMEOUT_MS = 10_000;
 const MODAL_CAPACITY_RELEASE_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
 const MODAL_LATE_CREATE_PERSIST_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
 type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
 
+class ModalPayloadUploadTimeoutError extends Error {
+  constructor(input: { sandboxId: string; path: string; bytes: number }) {
+    super(
+      `MODAL_PAYLOAD_UPLOAD_TIMEOUT: Modal payload upload did not complete within ${MODAL_PAYLOAD_UPLOAD_TIMEOUT_MS}ms (sandbox=${input.sandboxId}, path=${input.path}, bytes=${input.bytes}).`,
+    );
+    this.name = 'ModalPayloadUploadTimeoutError';
+  }
+}
+
+async function writeModalFileWithDeadline(input: {
+  sandbox: ModalSandbox;
+  content: Buffer;
+  path: string;
+  cancellation: Promise<never>;
+}): Promise<void> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(() => {
+      reject(
+        new ModalPayloadUploadTimeoutError({
+          sandboxId: input.sandbox.sandboxId,
+          path: input.path,
+          bytes: input.content.byteLength,
+        }),
+      );
+    }, MODAL_PAYLOAD_UPLOAD_TIMEOUT_MS);
+  });
+  try {
+    await Promise.race([
+      input.sandbox.filesystem.writeBytes(input.content, input.path),
+      input.cancellation,
+      deadline,
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
 /**
  * A create RPC that outlives its client deadline is fundamentally ambiguous:
  * Modal may still create the sandbox after the caller stops waiting.  It is
@@ -185,10 +225,22 @@ async function confirmDetachedModalRunnerReady(input: {
 }
 
 async function terminateModalSandbox(sandbox: ModalSandbox): Promise<boolean> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
   try {
     // The default terminate call only acknowledges the request. Capacity may
     // be released only after Modal confirms the sandbox itself has stopped.
-    await sandbox.terminate({ wait: true });
+    await Promise.race([
+      sandbox.terminate({ wait: true }),
+      new Promise<never>((_resolve, reject) => {
+        timer = setTimeout(() => {
+          reject(
+            new Error(
+              `Modal sandbox termination did not settle within ${MODAL_TERMINATION_TIMEOUT_MS}ms.`,
+            ),
+          );
+        }, MODAL_TERMINATION_TIMEOUT_MS);
+      }),
+    ]);
     return true;
   } catch (error) {
     console.warn('[play-runner.modal.terminate_failed]', {
@@ -196,6 +248,8 @@ async function terminateModalSandbox(sandbox: ModalSandbox): Promise<boolean> {
       error: error instanceof Error ? error.message : String(error),
     });
     return false;
+  } finally {
+    if (timer) clearTimeout(timer);
   }
 }
 
@@ -529,7 +583,12 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
             sandbox: {
               id: sandbox.sandboxId,
               uploadFile: (content, path) =>
-                sandbox.filesystem.writeBytes(content, path),
+                writeModalFileWithDeadline({
+                  sandbox,
+                  content,
+                  path,
+                  cancellation: cancellationPromise,
+                }),
             },
             bundlePromise: buildPlayRunnerBundle(),
             config,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts

@@ -1,6 +1,7 @@
 import type { PlayRunnerBackend } from './types';
 import { daytonaPlayRunnerBackend } from './backends/daytona';
 import {
+  daytonaModalCanaryPlayRunnerBackend,
   daytonaModalFallbackPlayRunnerBackend,
   daytonaOnlyPlayRunnerBackend,
   modalOnlyPlayRunnerBackend,
@@ -14,6 +15,8 @@ import {
   type PlayRuntimeBackendId,
 } from '@shared_libs/play-runtime/backend';
 import {
+  isRuntimeSandboxCanaryProduction,
+  RUNTIME_SANDBOX_PLACEMENT_POLICIES,
   runtimeSandboxPlacementPolicyForBackend,
   type RuntimeSandboxPlacementPolicyId,
 } from '@shared_libs/play-runtime/runtime-sandbox-placement-policy';
@@ -32,6 +35,13 @@ export function resolvePlayRunnerBackend(
   );
   const placementPolicy = runtimeSandboxPlacementPolicyForBackend(backend);
   if (placementPolicy) {
+    if (
+      placementPolicy.id ===
+        RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1 &&
+      !isRuntimeSandboxCanaryProduction({ nodeEnv: process.env.NODE_ENV })
+    ) {
+      return daytonaModalFallbackPlayRunnerBackend;
+    }
     return resolveRuntimeSandboxPlacementBackend(placementPolicy.id);
   }
   if (backend === PLAY_RUNTIME_BACKENDS.localProcess) {
@@ -42,6 +52,7 @@ export function resolvePlayRunnerBackend(
 
 export {
   daytonaPlayRunnerBackend,
+  daytonaModalCanaryPlayRunnerBackend,
   daytonaModalFallbackPlayRunnerBackend,
   daytonaOnlyPlayRunnerBackend,
   localProcessPlayRunnerBackend,

package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts

@@ -1,4 +1,6 @@
 import { PLAY_RUNTIME_BACKENDS, type PlayRuntimeBackendId } from './backend';
+import { createHash } from 'node:crypto';
+import { isIsolatedRuntimeSchedulerSchema } from './runtime-scheduler-topology';
 
 export const RUNTIME_SANDBOX_PROVIDERS = {
   daytona: 'daytona',
@@ -15,6 +17,7 @@ export type RuntimeSandboxProvider =
 export const RUNTIME_SANDBOX_PLACEMENT_POLICIES = {
   daytonaOnlyV1: 'daytona_only@1',
   daytonaThenModalV1: 'daytona_then_modal@1',
+  daytonaModalCanaryV1: 'daytona_modal_canary@1',
   modalOnlyV1: 'modal_only@1',
 } as const;
 
@@ -41,8 +44,59 @@ export type RuntimeSandboxPlacementPolicy = Readonly<{
   transitions: readonly RuntimeSandboxPlacementTransition[];
   requiredCredentialEnv: readonly string[];
   cleanupProviders: readonly RuntimeSandboxProvider[];
+  canary?: Readonly<{
+    provider: RuntimeSandboxProvider;
+    basisPoints: number;
+    selector: 'sha256_run_id_basis_points@1';
+    environment: 'production';
+  }>;
 }>;
 
+const CANARY_BUCKET_COUNT = 10_000;
+
+export type RuntimeSandboxCanarySelection = Readonly<{
+  canary: RuntimeSandboxProvider | null;
+  bucket: number | null;
+  basisPoints: number;
+}>;
+
+export function isRuntimeSandboxCanaryProduction(input: {
+  nodeEnv?: string | null;
+  runtimeSchedulerSchema?: string | null;
+}): boolean {
+  return (
+    input.nodeEnv === 'production' &&
+    !isIsolatedRuntimeSchedulerSchema(input.runtimeSchedulerSchema)
+  );
+}
+
+/** Stable across scheduler attempts and worker processes for a durable run id. */
+export function selectRuntimeSandboxCanary(input: {
+  policy: RuntimeSandboxPlacementPolicy;
+  runId?: string | null;
+  production: boolean;
+}): RuntimeSandboxCanarySelection {
+  const canary = input.policy.canary;
+  const runId = input.runId?.trim();
+  if (!canary || !runId || !input.production) {
+    return {
+      canary: null,
+      bucket: null,
+      basisPoints: canary?.basisPoints ?? 0,
+    };
+  }
+  const bucket =
+    createHash('sha256')
+      .update(`runtime-sandbox-placement@1:${runId}`)
+      .digest()
+      .readUInt32BE(0) % CANARY_BUCKET_COUNT;
+  return {
+    canary: bucket < canary.basisPoints ? canary.provider : null,
+    bucket,
+    basisPoints: canary.basisPoints,
+  };
+}
+
 const DAYTONA_THEN_MODAL_V1: RuntimeSandboxPlacementPolicy = {
   id: RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaThenModalV1,
   compatibilityRuntimeBackend: PLAY_RUNTIME_BACKENDS.daytona,
@@ -72,6 +126,23 @@ const DAYTONA_THEN_MODAL_V1: RuntimeSandboxPlacementPolicy = {
   ],
 };
 
+/**
+ * Staged Daytona-compatible policy. Workers learn this policy before launch
+ * writers start persisting it, so an app-first deploy cannot strand runs on an
+ * unsupported policy id. One percent of production runs enter the same Modal
+ * Adapter used by DAYTONA_THEN_MODAL_V1 after a capacity failure.
+ */
+const DAYTONA_MODAL_CANARY_V1: RuntimeSandboxPlacementPolicy = {
+  ...DAYTONA_THEN_MODAL_V1,
+  id: RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1,
+  canary: {
+    provider: RUNTIME_SANDBOX_PROVIDERS.modal,
+    basisPoints: 100,
+    selector: 'sha256_run_id_basis_points@1',
+    environment: 'production',
+  },
+};
+
 /**
  * Frozen compatibility policy for launches written before placement policy ids
  * existed. It is retained until every pre-policy release lane has drained.
@@ -100,6 +171,8 @@ export const RUNTIME_SANDBOX_PLACEMENT_POLICY_CATALOG: Readonly<
   [RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaOnlyV1]: DAYTONA_ONLY_V1,
   [RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaThenModalV1]:
     DAYTONA_THEN_MODAL_V1,
+  [RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1]:
+    DAYTONA_MODAL_CANARY_V1,
   [RUNTIME_SANDBOX_PLACEMENT_POLICIES.modalOnlyV1]: MODAL_ONLY_V1,
 };
 
@@ -129,7 +202,7 @@ export function runtimeSandboxPlacementPolicyForBackend(
   backend: PlayRuntimeBackendId,
 ): RuntimeSandboxPlacementPolicy | null {
   if (backend === PLAY_RUNTIME_BACKENDS.daytona) {
-    return DAYTONA_THEN_MODAL_V1;
+    return DAYTONA_MODAL_CANARY_V1;
   }
   if (backend === PLAY_RUNTIME_BACKENDS.modal) {
     return MODAL_ONLY_V1;

package/dist/bundling-sources/shared_libs/plays/tool-category-descriptions.ts

@@ -42,6 +42,11 @@ export const TOOL_CATEGORY_DESCRIPTIONS: Readonly<Record<string, string>> = {
   free: 'Free tools that do not spend Deepline credits.',
 };
 
+/** Stable UI/CLI enumeration of the well-known canonical category slugs. */
+export const WELL_KNOWN_TOOL_CATEGORIES = Object.freeze(
+  Object.keys(TOOL_CATEGORY_DESCRIPTIONS),
+);
+
 /**
  * One-line definition for a tool category slug, or null when none is declared.
  * Never fabricates a definition for unknown slugs.