deepline 0.1.21 → 0.1.23

package/dist/repo/sdk/src/client.ts

@@ -34,6 +34,7 @@
  * @module
  */
 import { resolveConfig } from './config.js';
+import { DeeplineError } from './errors.js';
 import { HttpClient } from './http.js';
 import type {
   DeeplineClientOptions,
@@ -66,6 +67,26 @@ import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-mani
 
 const TERMINAL_PLAY_STATUSES = new Set(['completed', 'failed', 'cancelled']);
 const INCLUDE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
+const COMPILE_MANIFEST_RETRY_DELAYS_MS = [250, 1_000];
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isTransientCompileManifestError(error: unknown): boolean {
+  if (error instanceof DeeplineError && typeof error.statusCode === 'number') {
+    return (
+      error.statusCode === 408 ||
+      error.statusCode === 425 ||
+      error.statusCode === 499 ||
+      (error.statusCode >= 500 && error.statusCode < 600)
+    );
+  }
+  const message = error instanceof Error ? error.message : String(error);
+  return /fetch failed|connection (?:closed|reset|terminated)|socket hang up|econnreset|etimedout|eai_again|abort/i.test(
+    message,
+  );
+}
 
 type ExecuteToolRawOptions = {
   includeToolMetadata?: boolean;
@@ -88,11 +109,7 @@ export type RunsListOptions = {
 };
 
 export type RunsTailOptions = {
-  cursor?: string | number;
-  afterLogIndex?: number;
-  waitMs?: number;
-  terminalOnly?: boolean;
-  compact?: boolean;
+  signal?: AbortSignal;
 };
 
 export type RunsLogsOptions = {
@@ -127,11 +144,7 @@ function isRecord(value: unknown): value is Record<string, unknown> {
 
 function normalizePlayStatus(raw: Record<string, unknown>): PlayStatus {
   const status =
-    typeof raw.status === 'string'
-      ? raw.status
-      : typeof raw.temporalStatus === 'string'
-        ? mapLegacyTemporalStatus(raw.temporalStatus)
-        : 'running';
+    typeof raw.status === 'string' ? raw.status : 'running';
   const runId =
     typeof raw.runId === 'string'
       ? raw.runId
@@ -145,24 +158,6 @@ function normalizePlayStatus(raw: Record<string, unknown>): PlayStatus {
   };
 }
 
-function mapLegacyTemporalStatus(status: string): PlayStatus['status'] {
-  switch (status.trim().toUpperCase()) {
-    case 'PENDING':
-      return 'queued';
-    case 'COMPLETED':
-      return 'completed';
-    case 'FAILED':
-      return 'failed';
-    case 'CANCELLED':
-    case 'TERMINATED':
-    case 'TIMED_OUT':
-      return 'cancelled';
-    case 'RUNNING':
-    default:
-      return 'running';
-  }
-}
-
 function decodeBase64Bytes(value: string): Uint8Array {
   const binary = atob(value);
   const bytes = new Uint8Array(binary.length);
@@ -172,6 +167,131 @@ function decodeBase64Bytes(value: string): Uint8Array {
   return bytes;
 }
 
+function readStringArray(value: unknown): string[] {
+  return Array.isArray(value)
+    ? value.filter((line): line is string => typeof line === 'string')
+    : [];
+}
+
+type PlayLiveStatusState = {
+  runId: string;
+  status: PlayStatus['status'];
+  logs: string[];
+  result?: unknown;
+  error?: string;
+  latest: PlayStatus | null;
+};
+
+function getPlayLiveEventPayload(event: PlayLiveEvent): Record<string, unknown> {
+  return event.payload && typeof event.payload === 'object'
+    ? (event.payload as Record<string, unknown>)
+    : {};
+}
+
+function normalizeLiveStatus(value: unknown): PlayStatus['status'] | null {
+  if (
+    value === 'queued' ||
+    value === 'running' ||
+    value === 'waiting' ||
+    value === 'completed' ||
+    value === 'failed' ||
+    value === 'cancelled'
+  ) {
+    return value;
+  }
+  return null;
+}
+
+function updatePlayLiveStatusState(
+  state: PlayLiveStatusState,
+  event: PlayLiveEvent,
+): PlayStatus | null {
+  const payload = getPlayLiveEventPayload(event);
+  if (event.type === 'play.run.log') {
+    state.logs.push(...readStringArray(payload.lines));
+    return null;
+  }
+  if (
+    event.type !== 'play.run.snapshot' &&
+    event.type !== 'play.run.status' &&
+    event.type !== 'play.run.final_status'
+  ) {
+    return null;
+  }
+
+  const runId =
+    typeof payload.runId === 'string' && payload.runId
+      ? payload.runId
+      : state.runId;
+  const status = normalizeLiveStatus(payload.status) ?? state.status;
+  const logs = readStringArray(payload.logs);
+  if (logs.length > 0 || event.type === 'play.run.snapshot') {
+    state.logs = logs;
+  }
+  if ('result' in payload) {
+    state.result = payload.result;
+  }
+  if (typeof payload.error === 'string' && payload.error.trim()) {
+    state.error = payload.error;
+  }
+  state.runId = runId;
+  state.status = status;
+
+  const progressRecord =
+    payload.progress &&
+    typeof payload.progress === 'object' &&
+    !Array.isArray(payload.progress)
+      ? (payload.progress as Record<string, unknown>)
+      : {};
+  const next: PlayStatus = {
+    ...(payload as unknown as Omit<PlayStatus, 'runId' | 'status' | 'progress'>),
+    runId,
+    status,
+    progress: {
+      ...progressRecord,
+      status:
+        typeof progressRecord.status === 'string'
+          ? progressRecord.status
+          : status,
+      logs: state.logs,
+      ...(state.error ? { error: state.error } : {}),
+    },
+    ...('result' in state ? { result: state.result } : {}),
+  };
+  state.latest = next;
+  return next;
+}
+
+function playRunResultFromStatus(
+  status: PlayStatus,
+  startedAt: number,
+  fallbackRunId: string,
+): PlayRunResult {
+  return {
+    success: status.status === 'completed',
+    runId: status.runId || fallbackRunId,
+    result: status.result,
+    logs: status.progress?.logs ?? [],
+    durationMs: Date.now() - startedAt,
+    error:
+      status.progress?.error ??
+      (status.status !== 'completed' ? status.status : undefined),
+  };
+}
+
+function playRunStatusFromState(state: PlayLiveStatusState): PlayStatus {
+  return {
+    runId: state.runId,
+    status: state.status,
+    progress: {
+      status: state.status,
+      logs: state.logs,
+      ...(state.error ? { error: state.error } : {}),
+    },
+    ...('result' in state ? { result: state.result } : {}),
+  };
+}
+
 /**
  * Low-level client for the Deepline REST API.
  *
@@ -339,7 +459,7 @@ export class DeeplineClient {
   /**
    * Search available tools using Deepline's ranked backend search.
    *
-   * This is the same discovery surface used by the legacy CLI: it ranks across
+   * This is the same discovery surface used by the CLI: it ranks across
    * tool metadata, categories, agent guidance, and input schema fields.
    */
   async searchTools(
@@ -453,7 +573,7 @@ export class DeeplineClient {
    * `progress.logs`; they are not part of the user output object.
    *
    * @param request - Play run configuration (name, code, input, etc.)
-   * @returns Workflow metadata including the `workflowId` for status polling
+   * @returns Run metadata including the public `workflowId`
    *
    * @example
    * ```typescript
@@ -659,10 +779,24 @@ export class DeeplineClient {
     artifact: Record<string, unknown>;
     importedPlayDependencies?: PlayCompilerManifest[];
   }): Promise<PlayCompilerManifest> {
-    const response = await this.http.post<{
-      compilerManifest: PlayCompilerManifest;
-    }>('/api/v2/plays/compile-manifest', input);
-    return response.compilerManifest;
+    const retryDelays = COMPILE_MANIFEST_RETRY_DELAYS_MS.slice(
+      0,
+      Math.max(0, this.config.maxRetries),
+    );
+    for (let attempt = 0; ; attempt += 1) {
+      try {
+        const response = await this.http.post<{
+          compilerManifest: PlayCompilerManifest;
+        }>('/api/v2/plays/compile-manifest', input);
+        return response.compilerManifest;
+      } catch (error) {
+        const delayMs = retryDelays[attempt];
+        if (delayMs === undefined || !isTransientCompileManifestError(error)) {
+          throw error;
+        }
+        await sleep(delayMs);
+      }
+    }
   }
 
   /**
@@ -917,9 +1051,6 @@ export class DeeplineClient {
    * Internal/advanced primitive. Public callers should usually prefer
    * {@link runPlay}, {@link PlayJob.get}, or `deepline play run --watch`.
    *
-   * Poll this method until `status` reaches a terminal state:
-   * `'completed'`, `'failed'`, or `'cancelled'`.
-   *
    * @param workflowId - Play-run id from {@link startPlayRun}
    * @returns Current status with progress logs and partial results
    *
@@ -945,43 +1076,11 @@ export class DeeplineClient {
     return normalizePlayStatus(response);
   }
 
-  /**
-   * Get the lightweight tail-polling status for a play execution.
-   *
-   * This is intentionally smaller than {@link getPlayStatus}: it returns the
-   * fields needed for CLI log tailing while the run is in flight, without
-   * forcing the API to rebuild final result views on every poll. Call
-   * {@link getPlayStatus} once after a terminal state for the full result.
-   */
-  async getPlayTailStatus(
-    workflowId: string,
-    options?: {
-      afterLogIndex?: number;
-      waitMs?: number;
-      terminalOnly?: boolean;
-    },
-  ): Promise<PlayStatus> {
-    const params = new URLSearchParams({ mode: 'tail' });
-    if (typeof options?.afterLogIndex === 'number') {
-      params.set('afterLogIndex', String(options.afterLogIndex));
-    }
-    if (typeof options?.waitMs === 'number') {
-      params.set('waitMs', String(options.waitMs));
-    }
-    if (options?.terminalOnly) {
-      params.set('terminalOnly', 'true');
-    }
-    const response = await this.http.get<Record<string, unknown>>(
-      `/api/v2/plays/run/${encodeURIComponent(workflowId)}?${params.toString()}`,
-    );
-    return normalizePlayStatus(response);
-  }
-
   /**
    * Stream semantic play-run events using the same SSE feed as the dashboard.
    *
-   * Consumers should still keep a polling fallback: SSE is the fast live-update
-   * transport, while the status endpoints remain the authoritative recovery path.
+   * The server emits a canonical `play.run.snapshot` event first for every
+   * connection, then incremental live events until terminal state or reconnect.
    */
   async *streamPlayRunEvents(
     workflowId: string,
@@ -1012,7 +1111,7 @@ export class DeeplineClient {
    *
    * Sends a stop request for the run.
    *
-   * @param workflowId - Temporal workflow ID to cancel
+   * @param workflowId - Public Deepline play-run id to cancel
    *
    * @example
    * ```typescript
@@ -1029,7 +1128,7 @@ export class DeeplineClient {
   /**
    * Stop a running play execution, including open HITL waits.
    *
-   * @param workflowId - Temporal workflow ID to stop
+   * @param workflowId - Public Deepline play-run id to stop
    * @param options.reason - Optional audit/debug reason
    */
   async stopPlay(
@@ -1108,39 +1207,42 @@ export class DeeplineClient {
     return response.runs ?? [];
   }
 
-  /**
-   * Fetch the lightweight tail status for a run using the public runs resource model.
-   *
-   * This is the SDK equivalent of:
-   *
-   * ```bash
-   * deepline runs tail <run-id> --json
-   * ```
-   */
+  /** Read the canonical run stream and return the latest run snapshot. */
   async tailRun(runId: string, options?: RunsTailOptions): Promise<PlayStatus> {
-    const afterLogIndex =
-      typeof options?.afterLogIndex === 'number'
-        ? options.afterLogIndex
-        : typeof options?.cursor === 'number'
-          ? options.cursor
-          : typeof options?.cursor === 'string' && options.cursor.trim()
-            ? Number(options.cursor)
-            : undefined;
-    const params = new URLSearchParams();
-    if (Number.isFinite(afterLogIndex)) {
-      params.set('afterLogIndex', String(Number(afterLogIndex)));
+    const state: PlayLiveStatusState = {
+      runId,
+      status: 'running',
+      logs: [],
+      latest: null,
+    };
+    let terminal = false;
+    for await (const event of this.streamPlayRunEvents(runId, {
+      mode: 'cli',
+      signal: options?.signal,
+    })) {
+      const status = updatePlayLiveStatusState(state, event);
+      if (!status) {
+        continue;
+      }
+      terminal = TERMINAL_PLAY_STATUSES.has(status.status);
+      if (terminal) {
+        break;
+      }
     }
-    if (typeof options?.waitMs === 'number') {
-      params.set('waitMs', String(options.waitMs));
+    if (terminal && state.latest) {
+      return await this.getRunStatus(state.latest.runId || runId).catch(
+        () => state.latest ?? playRunStatusFromState(state),
+      );
     }
-    if (options?.terminalOnly) {
-      params.set('terminalOnly', 'true');
+    if (state.latest) {
+      return state.latest;
     }
-    const suffix = params.toString() ? `?${params.toString()}` : '';
-    const response = await this.http.get<Record<string, unknown>>(
-      `/api/v2/runs/${encodeURIComponent(runId)}/tail${suffix}`,
+    throw new DeeplineError(
+      `Run stream for ${runId} ended before the initial snapshot.`,
+      undefined,
+      'PLAY_RUN_STREAM_EMPTY',
+      { runId },
     );
-    return normalizePlayStatus(response);
   }
 
   /**
@@ -1337,11 +1439,11 @@ export class DeeplineClient {
   // ——————————————————————————————————————————————————————————
 
   /**
-   * Run a play end-to-end: submit, poll until terminal, return result.
+   * Run a play end-to-end: submit, stream until terminal, return result.
    *
    * This is the highest-level play execution method. It submits the play,
-   * polls for status updates, and returns a structured result with logs
-   * and timing. Supports cancellation via `AbortSignal`.
+   * reads the canonical run stream for status updates, and returns a structured
+   * result with logs and timing. Supports cancellation via `AbortSignal`.
    *
    * @param code - Source string fallback; pass the bundled artifact in `options.artifact`
    * @param csvPath - Input CSV path, or `null`
@@ -1357,7 +1459,6 @@ export class DeeplineClient {
    *     const logs = status.progress?.logs ?? [];
    *     console.log(`[${status.status}] ${logs.length} log lines`);
    *   },
-   *   pollIntervalMs: 1000,
    * });
    *
    * if (result.success) {
@@ -1383,10 +1484,8 @@ export class DeeplineClient {
     csvPath: string | null,
     name?: string,
     options?: {
-      /** Called on each poll iteration with the current status. */
+      /** Called for each status snapshot emitted by the run stream. */
       onProgress?: (status: PlayStatus) => void;
-      /** Milliseconds between status polls. Default: `500`. */
-      pollIntervalMs?: number;
       /** Abort signal — triggers cancellation and immediate return. */
       signal?: AbortSignal;
       /** Runtime input for the play function. */
@@ -1409,39 +1508,59 @@ export class DeeplineClient {
       packagedFiles: options?.packagedFiles,
       force: options?.force,
     });
-    const pollInterval = options?.pollIntervalMs ?? 500;
     const start = Date.now();
+    const state: PlayLiveStatusState = {
+      runId: workflowId,
+      status: 'running',
+      logs: [],
+      latest: null,
+    };
+
+    if (options?.signal?.aborted) {
+      await this.cancelPlay(workflowId);
+      return {
+        success: false,
+        runId: workflowId,
+        logs: [],
+        durationMs: Date.now() - start,
+        error: 'Cancelled by user',
+      };
+    }
 
-    while (true) {
+    for await (const event of this.streamPlayRunEvents(workflowId, {
+      mode: 'cli',
+      signal: options?.signal,
+    })) {
       if (options?.signal?.aborted) {
         await this.cancelPlay(workflowId);
         return {
           success: false,
           runId: workflowId,
-          logs: [],
+          logs: state.logs,
           durationMs: Date.now() - start,
           error: 'Cancelled by user',
         };
       }
 
-      const status = await this.getPlayStatus(workflowId);
+      const status = updatePlayLiveStatusState(state, event);
+      if (!status) {
+        continue;
+      }
       options?.onProgress?.(status);
 
       if (TERMINAL_PLAY_STATUSES.has(status.status)) {
-        return {
-          success: status.status === 'completed',
-          runId: status.runId || workflowId,
-          result: status.result,
-          logs: status.progress?.logs ?? [],
-          durationMs: Date.now() - start,
-          error:
-            status.progress?.error ??
-            (status.status !== 'completed' ? status.status : undefined),
-        };
+        const finalStatus = await this.getPlayStatus(status.runId || workflowId)
+          .catch(() => status);
+        return playRunResultFromStatus(finalStatus, start, workflowId);
       }
-
-      await new Promise((resolve) => setTimeout(resolve, pollInterval));
     }
+
+    throw new DeeplineError(
+      `Run stream for ${workflowId} ended before the run reached a terminal state.`,
+      undefined,
+      'PLAY_RUN_STREAM_ENDED',
+      { runId: workflowId, workflowId },
+    );
   }
 
   // ——————————————————————————————————————————————————————————

package/dist/repo/sdk/src/types.ts

@@ -200,7 +200,7 @@ export interface ToolMetadata extends ToolDefinition {
 export interface PlayRunResult {
   /** `true` if the play completed successfully (`status === 'completed'`). */
   success: boolean;
-  /** Public play-run identifier — use for `play tail` or status polling. */
+  /** Public play-run identifier. */
   runId: string;
   /** The play's return value. Only present on success. */
   result?: unknown;
@@ -222,8 +222,6 @@ export interface PlayProgressStatus {
   totalRows?: number;
   /** Accumulated log lines from `ctx.log()`. Grows monotonically. */
   logs: string[];
-  /** Zero-based offset of the first returned log line when a tail cursor is used. */
-  logOffset?: number;
   /** Error message if the play has failed. */
   error?: string;
 }
@@ -265,7 +263,7 @@ export interface PlayStatus {
   progress?: PlayProgressStatus;
   /** Partial or final result. Available once the play returns. */
   result?: unknown;
-  /** Temporal-backed run metadata when returned by the status endpoint. */
+  /** Scheduler-backed run metadata when returned by the status endpoint. */
   run?: {
     startTime?: string | null;
     closeTime?: string | null;
@@ -315,9 +313,9 @@ export interface StopPlayRunResult {
  * Summary of a single play run, returned by {@link DeeplineClient.listPlayRuns}.
  */
 export interface PlayRunListItem {
-  /** Temporal workflow ID. */
+  /** Public Deepline play-run id. */
   workflowId: string;
-  /** Temporal run ID (unique per retry of the same workflow). */
+  /** Backend run attempt id, when exposed. */
   runId: string;
   /** Workflow type (typically `'Workflow'`). */
   type: string;
@@ -440,7 +438,7 @@ export interface PlayDefinitionDetail {
   runCount: number;
   /** Primary key column for CSV-based plays. */
   tableNamespace?: string | null;
-  /** Temporal workflow ID of the latest run. */
+  /** Public Deepline id of the latest run. */
   latestRunId?: string | null;
   /** Unix timestamp (ms). */
   createdAt: number;
@@ -559,12 +557,11 @@ export interface ClearPlayHistoryResult {
  * ```typescript
  * const started = await client.startPlayRun({ name: 'my-play', input: { domain: 'stripe.com' } });
  * console.log(`Started: ${started.workflowId}`);
- * console.log(`Status URL: ${started.statusUrl}`);
  * console.log(`Dashboard: ${started.dashboardUrl}`);
  * ```
  */
 export interface PlayRunStart {
-  /** Temporal workflow ID for tracking this execution. */
+  /** Public Deepline play-run id for tracking this execution. */
   workflowId: string;
   /** Public Deepline play-run API version. */
   apiVersion?: number;
@@ -576,8 +573,6 @@ export interface PlayRunStart {
   runtimeBackend?: string;
   /** Canonical run contract compatibility metadata. */
   contract?: Record<string, unknown> | null;
-  /** URL to poll for status updates. */
-  statusUrl?: string;
   /** Dashboard URL for the named play. */
   dashboardUrl?: string;
   /** Terminal status returned when the start request used a short completion wait. */
@@ -589,7 +584,7 @@ export interface PlayRunStart {
  *
  * This is the check-only version of play artifact registration: the server runs
  * the same preflight compiler and static-analysis pass without storing,
- * publishing, or starting a Temporal run.
+ * publishing, or starting a play run.
  */
 export interface PlayCheckResult {
   valid: boolean;
@@ -657,8 +652,7 @@ export interface StartPlayRunRequest {
   waitForCompletionMs?: number;
   /**
    * Per-run execution profile override. The server defaults to `workers_edge`;
-   * tests/benchmarks pass `legacy` or `local` here. Most callers should leave
-   * this unset.
+   * tests can pass `local` here. Most callers should leave this unset.
    */
   profile?: string;
 }

package/dist/repo/sdk/src/version.ts

@@ -1,2 +1,2 @@
-export const SDK_VERSION = "0.1.21";
+export const SDK_VERSION = "0.1.23";
 export const SDK_API_CONTRACT = "2026-05-runs-v2";

package/dist/repo/shared_libs/play-runtime/execution-plan.ts

@@ -7,6 +7,7 @@ import {
 export const EXECUTION_PLAN_DEFAULTS = {
   inlineRowsLimit: 1_000,
   largeMapChunkSize: 5_000,
+  complexMapChunkSize: 1_000,
   workflowSoftStepBudget: 20_000,
   workflowHardStepBudget: 25_000,
   ingestStepCount: 1,
@@ -183,6 +184,10 @@ function extractPlanMaps(
     (substep): substep is Extract<PlayStaticSubstep, { type: 'waterfall' }> =>
       substep.type === 'waterfall',
   );
+  const fallbackStepSuites = substeps.filter(
+    (substep): substep is Extract<PlayStaticSubstep, { type: 'step_suite' }> =>
+      substep.type === 'step_suite',
+  );
   return substeps
     .filter(
       (substep): substep is Extract<PlayStaticSubstep, { type: 'map' }> =>
@@ -200,13 +205,33 @@ function extractPlanMaps(
         waterfallId: waterfall.id ?? waterfall.field,
         stageIds: waterfall.steps?.map((step) => step.id) ?? [],
       }));
+      const stepSuites = fallbackStepSuites.filter((stepSuite) => {
+        if (!mapSubstep.waterfallIds?.length) return true;
+        return mapSubstep.waterfallIds.includes(stepSuite.field);
+      });
+      const stepSuiteStepsPerChunk = stepSuites.reduce(
+        (max, stepSuite) => Math.max(max, stepSuite.steps.length),
+        0,
+      );
+      const waterfallStepsPerChunk = waterfallStages.reduce(
+        (max, waterfall) => Math.max(max, waterfall.stageIds.length),
+        0,
+      );
+      const stepsPerChunk = Math.max(
+        1,
+        waterfallStepsPerChunk,
+        stepSuiteStepsPerChunk,
+      );
       return {
         mapName: mapSubstep.name ?? mapSubstep.field,
         tableNamespace: mapSubstep.tableNamespace ?? mapSubstep.field,
         outputFields: mapSubstep.outputFields ?? [],
         waterfallStages,
-        defaultChunkSize: EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
-        stepsPerChunk: 1,
+        defaultChunkSize:
+          stepsPerChunk > 1
+            ? EXECUTION_PLAN_DEFAULTS.complexMapChunkSize
+            : EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
+        stepsPerChunk,
       };
     });
 }