deepline 0.1.205 → 0.1.207

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -40,6 +40,7 @@ import {
   STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
   STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
 } from '../../../shared_libs/play-runtime/runtime-constants';
+import { childSubmitIdempotencyKey } from '../../../shared_libs/play-runtime/child-run-id';
 import {
   createPlayExecutionGovernor,
   type GovernanceSnapshot,
@@ -3545,18 +3546,18 @@ function resolveSheetContractFromReq(
 }
 
 /**
- * Run-fatal teardown: release the runtime-sheet attempt leases and work-receipt
- * leases this run still holds, so an immediate rerun proceeds without waiting
- * out the 10-minute lease TTL. Every release is attempt/owner fenced downstream,
- * so it can never weaken a different run's live lease. Failures are logged loud
- * and swallowed here — the caller still throws the original run-fatal error.
+ * Release attempt-scoped leases after terminal settlement. Completed and failed
+ * rows are durable; retaining their live fences only delays the next repair run.
  */
-async function releaseRuntimeLeasesOnTeardown(input: {
+async function releaseRuntimeLeasesOnSettlement(input: {
   req: RunRequest;
   leasedSheetNamespaces: Set<string>;
   emit: (event: RunnerEvent) => void;
+  reason: 'completed' | 'run-fatal';
 }): Promise<void> {
-  const { req, leasedSheetNamespaces, emit } = input;
+  const { req, leasedSheetNamespaces, emit, reason } = input;
+  const reasonLabel =
+    reason === 'completed' ? 'completed settlement' : 'run-fatal teardown';
   const scope = runtimeSheetSessionScope(req);
   for (const tableNamespace of leasedSheetNamespaces) {
     const sheetContract = resolveSheetContractFromReq(req, tableNamespace);
@@ -3574,7 +3575,7 @@ async function releaseRuntimeLeasesOnTeardown(input: {
         emit({
           type: 'log',
           level: 'info',
-          message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on run-fatal teardown.`,
+          message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on ${reasonLabel}.`,
           ts: nowMs(),
         });
       }
@@ -3582,7 +3583,7 @@ async function releaseRuntimeLeasesOnTeardown(input: {
       emit({
         type: 'log',
         level: 'warn',
-        message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
+        message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on ${reasonLabel} (rerun will fall back to lease-TTL expiry): ${
           releaseError instanceof Error
             ? releaseError.message
             : String(releaseError)
@@ -3601,7 +3602,7 @@ async function releaseRuntimeLeasesOnTeardown(input: {
       emit({
         type: 'log',
         level: 'info',
-        message: `Released ${released.released} work-receipt lease(s) on run-fatal teardown.`,
+        message: `Released ${released.released} work-receipt lease(s) on ${reasonLabel}.`,
         ts: nowMs(),
       });
     }
@@ -3609,7 +3610,7 @@ async function releaseRuntimeLeasesOnTeardown(input: {
     emit({
       type: 'log',
       level: 'warn',
-      message: `Work-receipt release failed on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
+      message: `Work-receipt release failed on ${reasonLabel} (rerun will fall back to lease-TTL expiry): ${
         releaseError instanceof Error
           ? releaseError.message
           : String(releaseError)
@@ -7046,10 +7047,11 @@ function createMinimalWorkerCtx(
                   options?.timeoutMs == null && !childNeedsWorkflowScheduler,
                 body: {
                   name: resolvedName,
-                  childIdempotencyKey:
-                    wasFailed && receiptContext.leaseId
-                      ? `${receiptKey}:claim:${receiptContext.leaseId}`
-                      : receiptKey,
+                  childIdempotencyKey: childSubmitIdempotencyKey({
+                    receiptKey,
+                    wasFailed,
+                    leaseId: receiptContext.leaseId,
+                  }),
                   input: isRecord(input) ? input : {},
                   orgId: req.orgId,
                   callbackUrl: req.callbackUrl,
@@ -8154,6 +8156,12 @@ async function executeRunRequest(
     if (req.playCallGovernance && !options?.persistResultDatasets) {
       await persistProjectedResultDatasets();
     }
+    await releaseRuntimeLeasesOnSettlement({
+      req,
+      leasedSheetNamespaces,
+      emit: wrappedEmit,
+      reason: 'completed',
+    });
     if (options?.persistResultDatasets) {
       await persistProjectedResultDatasets();
       // Capped runs settle compute billing BEFORE declaring run.completed: a
@@ -8274,10 +8282,11 @@ async function executeRunRequest(
     // blocked for the full lease TTL. Best-effort but LOUD — a release failure
     // is logged and never masks the original run-fatal error. TTL expiry stays
     // the recovery path for true crashes (isolate death) where this never runs.
-    await releaseRuntimeLeasesOnTeardown({
+    await releaseRuntimeLeasesOnSettlement({
       req,
       leasedSheetNamespaces,
       emit: wrappedEmit,
+      reason: 'run-fatal',
     });
     const errorBilling = extractErrorBilling(error);
     const salvage: ReceiptSalvage | null = aborted

package/dist/bundling-sources/sdk/src/release.ts

@@ -106,10 +106,10 @@ export const SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: '0.1.205',
+  version: '0.1.207',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.205',
+    latest: '0.1.207',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -108,9 +108,11 @@ type RuntimeApiRequest =
       action: 'append_run_events';
       playId: string;
       events: PlayRunLedgerEvent[];
+      idempotencyKey?: string;
     }
   | {
       action: 'start_inline_child_run';
+      idempotencyKey?: string;
       playName: string;
       runId: string;
       parentRunId?: string | null;
@@ -127,6 +129,15 @@ type RuntimeApiRequest =
       staticPipeline?: unknown;
       source?: 'published' | 'ad_hoc' | 'draft';
     }
+  | {
+      action: 'settle_inline_child_run';
+      parentRunId: string;
+      childRunId: string;
+      childPlayName: string;
+      status: 'completed' | 'failed' | 'cancelled';
+      result?: unknown;
+      error?: string | null;
+    }
   | ({
       action: 'save_results';
     } & RuntimeSaveResults)
@@ -167,7 +178,7 @@ type RuntimeApiRequest =
       workflowId?: string;
       runId?: string;
       maxCreditsPerRun?: number | null;
-      finalItem: ComputeBillingItem;
+      finalItem?: ComputeBillingItem;
     }
   | ({
       action: 'rate_state_acquire';
@@ -322,6 +333,7 @@ export type WorkerRuntimeApiContext = {
   vercelProtectionBypassToken?: string | null;
   runtimeTestFaultHeader?: string | null;
   fetch?: typeof fetch;
+  requestTimeoutMs?: number | null;
 };
 
 const APP_RUNTIME_API_RETRY_DELAYS_MS = [100, 250, 500, 1_000] as const;
@@ -331,6 +343,7 @@ const APP_RUNTIME_API_RECEIPT_RETRY_DELAYS_MS = [
 const APP_RUNTIME_API_APPEND_RETRY_DELAYS_MS = [
   100, 250, 500, 1_000, 2_000, 4_000, 8_000,
 ] as const;
+const APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
 const RUN_STATUS_LEDGER_SNAPSHOT_CACHE_LIMIT = 1_000;
 const runStatusLedgerSnapshots = new Map<string, PlayRunLedgerSnapshot>();
 // Positional log cursor per run: count of liveLogs lines already forwarded to
@@ -418,11 +431,13 @@ function summarizeAppRuntimeErrorBody(body: string): string {
           ? record.debug_error
           : typeof record.details === 'string'
             ? record.details
-            : typeof record.error === 'string'
-              ? record.error
-              : typeof record.message === 'string'
-                ? record.message
-                : null;
+            : typeof record.detail === 'string'
+              ? record.detail
+              : typeof record.error === 'string'
+                ? record.error
+                : typeof record.message === 'string'
+                  ? record.message
+                  : null;
       if (message?.trim()) {
         return message.trim();
       }
@@ -447,6 +462,13 @@ function isRetryableAppRuntimeFetchError(input: {
   if (!isRetryableAppRuntimeAction(input.action)) {
     return false;
   }
+  // The only signal on the request is AbortSignal.timeout(requestTimeoutMs), so
+  // any abort here is our own per-request deadline firing against a slow app
+  // runtime API (common under heavy map load) — always safe to retry across the
+  // backoff ladder rather than failing the tool call on the first slow write.
+  if (isRequestTimeoutAbort(input.error)) {
+    return true;
+  }
   const message =
     input.error instanceof Error ? input.error.message : String(input.error);
   return /fetch failed|connection (terminated|timeout|timed out|closed|reset)|econnreset|etimedout|econnrefused/i.test(
@@ -454,6 +476,24 @@ function isRetryableAppRuntimeFetchError(input: {
   );
 }
 
+function isRequestTimeoutAbort(error: unknown): boolean {
+  if (!error || typeof error !== 'object') {
+    return false;
+  }
+  // AbortSignal.timeout() — the only signal on the request — rejects with a
+  // DOMException named TimeoutError. Match that (and its message across
+  // runtimes) but NOT a bare AbortError / "the operation was aborted", so a
+  // genuine run cancellation still propagates instead of being retried.
+  if ((error as { name?: unknown }).name === 'TimeoutError') {
+    return true;
+  }
+  const message =
+    error instanceof Error
+      ? error.message
+      : String((error as { message?: unknown }).message ?? '');
+  return /operation was aborted due to timeout|signal timed out/i.test(message);
+}
+
 function isRetryableAppRuntimeAction(
   action: RuntimeApiRequest['action'],
 ): boolean {
@@ -484,7 +524,8 @@ function isRetryableAppRuntimeAction(
     action === 'release_runtime_step_receipt' ||
     action === 'save_results' ||
     action === 'skip_runtime_step_receipt' ||
-    action === 'start_inline_child_run'
+    action === 'start_inline_child_run' ||
+    action === 'settle_inline_child_run'
   );
 }
 
@@ -560,6 +601,27 @@ function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+export class AppRuntimeApiTransportError extends Error {
+  readonly action: RuntimeApiRequest['action'];
+  readonly attempts: number;
+
+  constructor(input: {
+    action: RuntimeApiRequest['action'];
+    attempts: number;
+    cause: unknown;
+  }) {
+    const causeMessage =
+      input.cause instanceof Error ? input.cause.message : String(input.cause);
+    super(
+      `App runtime API transport exhausted action=${input.action} attempts=${input.attempts}: ${causeMessage}`,
+      { cause: input.cause },
+    );
+    this.name = 'AppRuntimeApiTransportError';
+    this.action = input.action;
+    this.attempts = input.attempts;
+  }
+}
+
 function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
   const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
   return `${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`;
@@ -585,6 +647,12 @@ async function postAppRuntimeApi<TResponse>(
   });
 
   const maxAttempts = appRuntimeMaxAttempts(body.action);
+  const requestTimeoutMs = Math.max(
+    1,
+    Math.floor(
+      context.requestTimeoutMs ?? APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS,
+    ),
+  );
 
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     let response: Response;
@@ -604,6 +672,7 @@ async function postAppRuntimeApi<TResponse>(
             : {}),
         },
         body: JSON.stringify(body),
+        signal: AbortSignal.timeout(requestTimeoutMs),
       });
     } catch (error) {
       if (
@@ -613,11 +682,11 @@ async function postAppRuntimeApi<TResponse>(
         await sleep(appRuntimeRetryDelayMs(body.action, attempt));
         continue;
       }
-      const message = error instanceof Error ? error.message : String(error);
-      throw new Error(
-        `App runtime API ${body.action} failed before receiving a response from ${resolveAppRuntimeApiUrl(context)} after ${attempt}/${maxAttempts} attempts: ${message}`,
-        { cause: error },
-      );
+      throw new AppRuntimeApiTransportError({
+        action: body.action,
+        attempts: attempt,
+        cause: error,
+      });
     }
     if (response.ok) {
       return (await response.json()) as TResponse;
@@ -912,7 +981,11 @@ async function updateRunStatusViaAppRuntimeUnlocked(
 
 export async function appendRunEventsViaAppRuntime(
   context: WorkerRuntimeApiContext,
-  input: { playId: string; events: PlayRunLedgerEvent[] },
+  input: {
+    playId: string;
+    events: PlayRunLedgerEvent[];
+    idempotencyKey?: string;
+  },
 ): Promise<void> {
   await postAppRuntimeApi<{ ok: true }>(context, {
     action: 'append_run_events',
@@ -938,6 +1011,7 @@ export async function startRunViaAppRuntime(
     maxCreditsPerRun?: number | null;
     staticPipeline?: unknown;
     source?: 'published' | 'ad_hoc' | 'draft';
+    idempotencyKey?: string;
   },
 ): Promise<void> {
   await postAppRuntimeApi<{ ok: true }>(context, {
@@ -946,6 +1020,23 @@ export async function startRunViaAppRuntime(
   });
 }
 
+export async function settleInlineChildRunViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: {
+    parentRunId: string;
+    childRunId: string;
+    childPlayName: string;
+    status: 'completed' | 'failed' | 'cancelled';
+    result?: unknown;
+    error?: string | null;
+  },
+): Promise<void> {
+  await postAppRuntimeApi<{ ok: true }>(context, {
+    action: 'settle_inline_child_run',
+    ...input,
+  });
+}
+
 export async function saveResultsViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: RuntimeSaveResults,

package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts

@@ -0,0 +1,51 @@
+import {
+  flattenStaticPipeline,
+  type PlayStaticPipeline,
+} from '../plays/static-pipeline';
+
+export type ChildExecutionStrategy = 'inline' | 'scheduled';
+
+export type ChildExecutionDecision = {
+  strategy: ChildExecutionStrategy;
+  reason:
+    | 'scalar_child'
+    | 'missing_static_contract'
+    | 'dataset_child'
+    | 'event_wait_child'
+    | 'explicit_timeout';
+};
+
+/**
+ * Choose execution placement from the child's authored contract. Scalar child
+ * plays are ordinary function composition. Dataset and event-wait children own
+ * independent durable state, so they remain scheduler jobs.
+ */
+export function resolveChildExecutionStrategy(input: {
+  pipeline: PlayStaticPipeline | null | undefined;
+  timeoutMs?: number | null;
+}): ChildExecutionDecision {
+  if (input.timeoutMs != null) {
+    return { strategy: 'scheduled', reason: 'explicit_timeout' };
+  }
+  if (!input.pipeline) {
+    return { strategy: 'scheduled', reason: 'missing_static_contract' };
+  }
+  const substeps = flattenStaticPipeline(input.pipeline);
+  if (
+    input.pipeline.tableNamespace ||
+    input.pipeline.sheetContract ||
+    substeps.some((step) => step.type === 'dataset' || step.type === 'csv')
+  ) {
+    return { strategy: 'scheduled', reason: 'dataset_child' };
+  }
+  if (
+    substeps.some(
+      (step) =>
+        step.type === 'tool' &&
+        (step.isEventWait === true || step.toolId === 'test_wait_for_event'),
+    )
+  ) {
+    return { strategy: 'scheduled', reason: 'event_wait_child' };
+  }
+  return { strategy: 'inline', reason: 'scalar_child' };
+}

package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts

@@ -56,6 +56,22 @@ export interface ChildRunIdInputs {
   graphHash?: string | null;
 }
 
+/**
+ * A failed durable runPlay receipt must launch a fresh child attempt while
+ * ordinary submit retries keep attaching to the original child. The receipt
+ * lease is the attempt fence, so it is the only extra identity required.
+ */
+export function childSubmitIdempotencyKey(input: {
+  receiptKey: string;
+  wasFailed: boolean;
+  leaseId?: string | null;
+}): string {
+  const leaseId = input.leaseId?.trim();
+  return input.wasFailed && leaseId
+    ? `${input.receiptKey}:claim:${leaseId}`
+    : input.receiptKey;
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === 'object' && value !== null && !Array.isArray(value);
 }

package/dist/bundling-sources/shared_libs/play-runtime/child-run-lifecycle.ts

@@ -0,0 +1,70 @@
+import type { RunExecutionScope } from './run-execution-scope';
+
+export type ChildRunPlacement = 'inline' | 'scheduled';
+
+export interface ChildRunLifecycleMetadata {
+  placement: ChildRunPlacement;
+}
+
+export interface ChildRunLifecycleAdapter<
+  Metadata extends ChildRunLifecycleMetadata,
+  Result,
+> {
+  open(scope: RunExecutionScope, metadata: Metadata): Promise<void> | void;
+  started(scope: RunExecutionScope, metadata: Metadata): Promise<void> | void;
+  complete(
+    scope: RunExecutionScope,
+    metadata: Metadata,
+    result: Result,
+  ): Promise<void> | void;
+  fail(
+    scope: RunExecutionScope,
+    metadata: Metadata,
+    error: unknown,
+  ): Promise<void> | void;
+}
+
+export interface ExecuteChildRunLifecycleInput<
+  Metadata extends ChildRunLifecycleMetadata,
+  Result,
+> {
+  scope: RunExecutionScope;
+  metadata: Metadata;
+  adapter: ChildRunLifecycleAdapter<Metadata, Result>;
+  execute(
+    scope: RunExecutionScope,
+    metadata: Metadata,
+  ): Promise<Result> | Result;
+}
+
+/**
+ * Projects one logical child execution through its lifecycle. Placement is
+ * descriptive metadata; inline and scheduled children use the same protocol.
+ */
+export async function executeChildRunLifecycle<
+  Metadata extends ChildRunLifecycleMetadata,
+  Result,
+>(input: ExecuteChildRunLifecycleInput<Metadata, Result>): Promise<Result> {
+  const { adapter, execute, metadata, scope } = input;
+
+  let result: Result;
+  try {
+    await adapter.open(scope, metadata);
+    await adapter.started(scope, metadata);
+    result = await execute(scope, metadata);
+  } catch (executionError) {
+    try {
+      await adapter.fail(scope, metadata, executionError);
+    } catch (failureProjectionError) {
+      throw new AggregateError(
+        [executionError, failureProjectionError],
+        'Child run execution and failure projection both failed',
+        { cause: executionError },
+      );
+    }
+    throw executionError;
+  }
+
+  await adapter.complete(scope, metadata, result);
+  return result;
+}