deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts

@@ -1,20 +1,7 @@
 /**
  * HTTP headers attached to every server-to-coordinator request.
- *
- * The coordinator is a Cloudflare Worker (`deepline-play-coordinator-*`) that
- * we publish through Cloudflare's Versions + Deployments primitives. Whenever
- * a deployment is split across two versions (e.g. 10% canary), Cloudflare
- * uses the `Cloudflare-Workers-Version-Key` header to pick a stable version
- * per request. Sending the runId as the key keeps every request for one play
- * run pinned to whichever version started the run, so a partial rollout
- * cannot strand a Workflow's `/submit` and `/result` calls on different
- * code versions.
- *
- * The header is harmless when there is only one active version (the default).
  */
-export const COORDINATOR_VERSION_KEY_HEADER = 'Cloudflare-Workers-Version-Key';
-export const COORDINATOR_VERSION_OVERRIDES_HEADER =
-  'Cloudflare-Workers-Version-Overrides';
+
 /**
  * Shared secret the coordinator uses to authenticate dispatcher (Vercel app)
  * traffic. The coordinator also sanity-checks `x-deepline-run-scope` matches
@@ -88,8 +75,6 @@ export function coordinatorRequestHeaders(input: {
   runId: string;
   contentType?: string | null;
   internalToken?: string | null;
-  runtimeDeployVersion?: string | null;
-  coordinatorWorkerName?: string | null;
   /**
    * When set, the coordinator validates this matches the runId in the URL.
    * Pass the runId on `/cancel` / `/signal` calls so a leaked dispatcher
@@ -102,22 +87,6 @@ export function coordinatorRequestHeaders(input: {
     headers['content-type'] = input.contentType;
   }
   const trimmed = input.runId.trim();
-  if (trimmed) {
-    headers[COORDINATOR_VERSION_KEY_HEADER] = trimmed;
-  }
-  const runtimeDeployVersion = input.runtimeDeployVersion?.trim();
-  const coordinatorWorkerName = input.coordinatorWorkerName?.trim();
-  if (
-    runtimeDeployVersion &&
-    coordinatorWorkerName &&
-    /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
-      runtimeDeployVersion,
-    ) &&
-    /^[A-Za-z0-9_-]+$/.test(coordinatorWorkerName)
-  ) {
-    headers[COORDINATOR_VERSION_OVERRIDES_HEADER] =
-      `${coordinatorWorkerName}="${runtimeDeployVersion}"`;
-  }
   const internalToken =
     input.internalToken?.trim() || resolveInternalCoordinatorToken();
   if (internalToken) {

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -10,8 +10,10 @@ import type { PlayRunContractSnapshot } from '../plays/contracts';
 import type { PlayStructuredDefinition } from '../plays/definition';
 import type { PlayStaticPipeline } from '../plays/static-pipeline';
 import type {
+  AsyncPlayRunRef,
   DurableCallStaleAfterSeconds,
   PlayAuthoringCallOptions,
+  PlayAuthoringAsyncCallOptions,
   PlayAuthoringCsvOptions,
   PlayAuthoringFetchOptions,
   PlayAuthoringRuntimeStepOptions,
@@ -331,6 +333,19 @@ export type PlayCallOptions = Omit<
   timeoutMs?: number;
 };
 
+export type PlayAsyncCallOptions = PlayAuthoringAsyncCallOptions;
+
+export type AsyncPlayLaunchRequest = Readonly<{
+  key: string;
+  invocationKey: string;
+  playRef: string;
+  input: Record<string, unknown>;
+  description: string;
+  schedulerSchema: string | null;
+  /** Governor ancestry at the authored call site, before entering the child. */
+  ancestryPlayIds: readonly string[];
+}>;
+
 export type RuntimeStepOptions = PlayAuthoringRuntimeStepOptions;
 
 export type FetchOptions = PlayAuthoringFetchOptions;
@@ -345,6 +360,16 @@ export interface ResolvedPlayExecution {
   definition?: PlayStructuredDefinition | null;
   staticPipeline?: PlayStaticPipeline | null;
   contractSnapshot?: PlayRunContractSnapshot | null;
+  launchMetadata?: {
+    revisionId?: string | null;
+    artifactStorageKey: string;
+    artifactHash: string;
+    graphHash: string;
+    definitionScope: 'org' | 'system';
+    authoringContract?:
+      | import('../plays/authoring-contract').AdmittedPlayAuthoringContract
+      | null;
+  } | null;
 }
 
 export interface BatchRequest {
@@ -374,6 +399,15 @@ export interface MapExecutionFrame {
   totalRows: number;
   completedRowKeys: string[];
   pendingRowKeys: string[];
+  /**
+   * The admitted inputs that still need to run after a durable suspension.
+   * This prevents a replay from losing work when its authored source query is
+   * stateful and no longer returns rows that were selected before the pause.
+   */
+  suspendedInputRows?: Array<{
+    inputIndex: number;
+    data: Record<string, unknown>;
+  }>;
   completedRowsCount?: number;
   pendingRowsCount?: number;
   failedRowsCount?: number;
@@ -548,6 +582,9 @@ export type IntegrationEventWaitRuntimeContext = {
   workflowId?: string;
   orgId?: string;
   executorToken?: string;
+  slackHitlDefaultTimeoutMs?: number;
+  /** Behavior selected by the admitted Play artifact, never worker version. */
+  slackHitlTimeoutOutcome?: 'park' | 'error';
 };
 
 export type IntegrationEventWaitBoundary = {
@@ -560,6 +597,21 @@ export type IntegrationEventWaitBoundary = {
     channel: string;
     ts: string;
   };
+  /**
+   * Optional control-plane timeout behavior. Missing preserves the legacy
+   * "return the timed_out result" replay behavior for older app deployments.
+   */
+  timeoutPolicy?: {
+    suspendAfterTimeoutMs?: number;
+    finalizeOnTimeout?: boolean;
+    /**
+     * A terminal policy is persisted with the boundary so durable replay never
+     * changes a published Play's timeout behavior.
+     */
+    timeoutOutcome?: 'return_null' | 'error';
+    /** Customer-authored message used only for the terminal error outcome. */
+    timeoutError?: string;
+  };
 };
 
 export type IntegrationEventWaitHandler = {
@@ -584,6 +636,11 @@ export type IntegrationEventWaitHandler = {
     context: IntegrationEventWaitRuntimeContext;
     boundary: IntegrationEventWaitBoundary;
   }) => Promise<void>;
+  /**
+   * When set, timeout finalization is side-effect-only: the Play stays parked
+   * instead of consuming a `timed_out` result.
+   */
+  suspendAfterTimeoutMs?: number;
 };
 
 export interface ContextOptions {
@@ -598,6 +655,8 @@ export interface ContextOptions {
   executionScope?: RunExecutionScope;
   /** Authoring semantics pinned by the immutable artifact. */
   authoringContractEdition?: PlayAuthoringContractEdition;
+  /** Direct/local callers may opt into the new terminal Slack HITL outcome. */
+  slackHitlTimeoutOutcome?: 'park' | 'error';
   /** Error shape pinned by the immutable play artifact; missing preserves legacy schema 0. */
   toolErrorSchemaVersion?: ToolExecutionErrorSchemaVersion;
   /** Execute-result contract pinned by the immutable Play artifact. */
@@ -769,6 +828,8 @@ export interface ContextOptions {
   }) => Promise<void>;
   playId?: string;
   runId?: string;
+  /** Shared governance/billing root. Differs from runId for async children. */
+  rootRunId?: string;
   /** Physical executor run that owns leases for an inline child context. */
   runtimeReceiptOwnerRunId?: string;
   /** Logical invocation scope for receipts created inside an inline child. */
@@ -800,6 +861,10 @@ export interface ContextOptions {
    */
   runtimeSheetBackedMapDatasets?: boolean;
   resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
+  /** Admit an independently scheduled scalar/HITL Play Run. */
+  launchPlayAsync?: (
+    request: AsyncPlayLaunchRequest,
+  ) => Promise<AsyncPlayRunRef>;
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
   getToolProvider?: (toolId: string) => Promise<string | null>;
   getToolOperation?: (toolId: string) => Promise<string | null>;
@@ -811,7 +876,17 @@ export interface ContextOptions {
     requiresExecutionFence?: boolean;
   } | null>;
   getToolActionCacheVersion?: (toolId: string) => Promise<string> | string;
+  /**
+   * Source-declared older action identities that remain safe to read. They
+   * are read-only fallbacks: new receipts and provider calls always use the
+   * primary `getToolActionCacheVersion` value.
+   */
+  getToolActionCacheReadVersions?: (
+    toolId: string,
+  ) => Promise<string[]> | string[];
   getToolAuthScopeDigest?: (toolId: string) => Promise<string> | string;
+  /** Original run for an explicit customer rerun, never an automatic retry. */
+  replayedFromRunId?: string | null;
   getToolRateScope?: (
     toolId: string,
     provider: string,
@@ -842,6 +917,17 @@ export interface ContextOptions {
     | Promise<import('./async-operation').AsyncOperationContractWire | null>
     | import('./async-operation').AsyncOperationContractWire
     | null;
+  /**
+   * Bounded-check policy projected onto a caller-managed async lifecycle
+   * companion. The runtime owns cadence; provider adapters only translate
+   * this budget into their native result/status request.
+   */
+  getToolAsyncOperationPoll?: (
+    toolId: string,
+  ) =>
+    | Promise<import('./async-operation').AsyncOperationPollExecution | null>
+    | import('./async-operation').AsyncOperationPollExecution
+    | null;
   getIntegrationEventWaitHandler?: (
     toolId: string,
   ) =>
@@ -982,6 +1068,11 @@ export interface PlayCheckpoint {
           channel: string;
           ts: string;
         };
+        /**
+         * Control-plane timeout behavior captured when the wait was armed.
+         * Missing fields preserve legacy checkpoint replay behavior.
+         */
+        timeoutPolicy?: IntegrationEventWaitBoundary['timeoutPolicy'];
         output?: unknown;
         completedAt?: number;
       }
@@ -1147,6 +1238,12 @@ export type PlayStep =
       nestedSteps: PlayStep[];
       description?: string;
     }
+  | {
+      type: 'play_launch';
+      playId: string;
+      runId: string;
+      description?: string;
+    }
   | {
       type: 'run_javascript';
       alias: string;
@@ -1187,6 +1284,12 @@ export type PlayDatasetSubstep =
       nestedSteps: PlayStep[];
       description?: string;
     }
+  | {
+      type: 'play_launch';
+      playId: string;
+      runId: string;
+      description?: string;
+    }
   | {
       type: 'run_javascript';
       alias: string;

package/dist/bundling-sources/shared_libs/play-runtime/daytona-payload-contract.ts

@@ -0,0 +1,68 @@
+import {
+  deployed,
+  readDeployedObject,
+  type DeployedDiagnostic,
+} from '@deepline/deployed-contracts';
+import type { PlayRunnerExecutionConfig } from './protocol';
+
+export type DaytonaPayloadEnvelope = {
+  /** Diagnostic only. A missing/different marker cannot block materialization. */
+  schemaVersion?: number;
+  runnerCode: string;
+  artifactBundledCode: string;
+  artifactSourceMap: string;
+  config: PlayRunnerExecutionConfig;
+  crashPusherCode: string;
+};
+
+export const DAYTONA_PAYLOAD_CONTRACT = deployed.object({
+  id: 'daytona-payload',
+  unknown: 'preserve',
+  fields: {
+    runnerCode: deployed.string(),
+    artifactBundledCode: deployed.string(),
+    artifactSourceMap: deployed.string(),
+    config: deployed.record(),
+    crashPusherCode: deployed.string(),
+    schemaVersion: deployed.optional(deployed.diagnostic.number(), {
+      whenAbsent: 'diagnostic unavailable',
+    }),
+  },
+});
+
+export const DAYTONA_PAYLOAD_FILE_FIELDS = [
+  'runnerCode',
+  'artifactBundledCode',
+  'artifactSourceMap',
+  'crashPusherCode',
+] as const;
+
+export function readDaytonaPayloadEnvelope(input: unknown): {
+  value: DaytonaPayloadEnvelope;
+  diagnostics: DeployedDiagnostic[];
+} {
+  const { value, diagnostics } = readDeployedObject(
+    DAYTONA_PAYLOAD_CONTRACT,
+    input,
+  );
+  return {
+    value: {
+      runnerCode: value.runnerCode as string,
+      artifactBundledCode: value.artifactBundledCode as string,
+      artifactSourceMap: value.artifactSourceMap as string,
+      config: value.config as PlayRunnerExecutionConfig,
+      crashPusherCode: value.crashPusherCode as string,
+    },
+    diagnostics,
+  };
+}
+
+/**
+ * The sandbox has only Node built-ins. Generate its tiny entry from the same
+ * contract-owned field list instead of maintaining a second hand-written list
+ * in transport code. This is shipped source, not Function#toString output.
+ */
+export function buildDaytonaPayloadMaterializerSource(): string {
+  const fields = JSON.stringify(DAYTONA_PAYLOAD_FILE_FIELDS);
+  return `const fs=require('node:fs');const zlib=require('node:zlib');const p=JSON.parse(zlib.gunzipSync(fs.readFileSync(process.argv[1])));if(!p||typeof p!=='object'||Array.isArray(p))throw new Error('Daytona payload: expected object');for(const key of ${fields})if(typeof p[key]!=='string')throw new Error('Daytona payload is missing required field '+key);if(!p.config||typeof p.config!=='object'||Array.isArray(p.config))throw new Error('Daytona payload is missing required field config');if(typeof p.schemaVersion==='number'&&p.schemaVersion!==1)console.warn('[daytona] payload schema marker differs; attempting structural materialization');fs.writeFileSync(process.argv[2],p.runnerCode);fs.writeFileSync(process.argv[3],JSON.stringify(p.config));fs.writeFileSync(process.argv[4],p.artifactBundledCode);fs.writeFileSync(process.argv[5],p.artifactSourceMap);fs.writeFileSync(process.argv[6],p.crashPusherCode);`;
+}

package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts

@@ -0,0 +1,103 @@
+import { withRuntimeSchedulerClient } from './/scheduler-backends/postgres-pool';
+import { isIsolatedRuntimeSchedulerSchema } from './/runtime-scheduler-topology';
+
+/** A single controller owns this fleet-wide sweep. Workers clean resources for
+ * their own attempts; they do not each enumerate the shared Daytona account. */
+export const DAYTONA_JANITOR_INTERVAL_MS = 5 * 60 * 1_000;
+
+export type DaytonaSandboxJanitorOptions = {
+  url: string;
+  schema: string;
+  maxClients?: number;
+};
+
+/**
+ * Start the production-only Daytona orphan sweep.
+ *
+ * Sandboxes have a source-scoped label shared by production, previews, and
+ * worktrees. An isolated schema cannot authoritatively distinguish its own
+ * sandboxes from the others, so it must never sweep. The fixed runtime
+ * controller is the one production owner; moving this here makes the Daytona
+ * list/read traffic constant with fleet size.
+ */
+export function startDaytonaSandboxJanitor(
+  options: DaytonaSandboxJanitorOptions,
+): { stop: () => void } {
+  if (!process.env.DAYTONA_API_KEY?.trim()) {
+    console.info('[daytona-janitor.controller] disabled (no DAYTONA_API_KEY)');
+    return { stop: () => undefined };
+  }
+  if (isIsolatedRuntimeSchedulerSchema(options.schema)) {
+    console.info('[daytona-janitor.controller] disabled (isolated schema)', {
+      schema: options.schema,
+    });
+    return { stop: () => undefined };
+  }
+
+  let running = false;
+  const runPass = async (): Promise<void> => {
+    if (running) {
+      console.info('[daytona-janitor.controller] skip (previous pass running)');
+      return;
+    }
+    running = true;
+    try {
+      const [
+        { Daytona },
+        { loadDaytonaRequiredConfig },
+        {
+          createPostgresDaytonaJanitorOwnershipPort,
+          reapOrphanDaytonaSandboxes,
+        },
+      ] = await Promise.all([
+        import('@daytonaio/sdk'),
+        import('.//daytona-runtime-config'),
+        import('./daytona-sandbox-reaper'),
+      ]);
+      const { clientOptions } = loadDaytonaRequiredConfig();
+      const daytona = new Daytona(clientOptions);
+      await withRuntimeSchedulerClient(
+        {
+          url: options.url,
+          applicationName: 'deepline-daytona-janitor',
+          maxClients: options.maxClients ?? 2,
+        },
+        (client) =>
+          reapOrphanDaytonaSandboxes({
+            daytona: {
+              list: (query) => daytona.list(query),
+            },
+            ownership: createPostgresDaytonaJanitorOwnershipPort({
+              client,
+              schema: options.schema,
+            }),
+          }),
+      );
+    } catch (error) {
+      // This is a best-effort resource backstop. A failed sweep must never
+      // terminate the controller that scales and recovers customer work.
+      console.warn('[daytona-janitor.controller] pass failed', {
+        error: error instanceof Error ? error.message : String(error),
+      });
+    } finally {
+      running = false;
+    }
+  };
+
+  const interval = setInterval(() => {
+    void runPass();
+  }, DAYTONA_JANITOR_INTERVAL_MS);
+  interval.unref();
+
+  // Run once immediately. The singleton controller has no fleet-start
+  // stampede to smooth, and an early pass shortens recovery after deploy.
+  void runPass();
+  console.info('[daytona-janitor.controller] scheduled', {
+    intervalMs: DAYTONA_JANITOR_INTERVAL_MS,
+    schema: options.schema,
+  });
+
+  return {
+    stop: () => clearInterval(interval),
+  };
+}