deepline 0.1.211 → 0.1.213

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -66,6 +66,22 @@ export type RuntimeStatusUpdate = {
    * positional cursor stay correct once the buffer has rotated.
    */
   liveLogTotalCount?: number;
+  /**
+   * Monotonic scheduler attempt that owns the positional log channel. A new
+   * attempt starts at offset zero without colliding with earlier attempts.
+   */
+  liveLogProducerAttempt?: number;
+  datasetLifecycleEvents?: Array<{
+    datasetId: string;
+    path: string;
+    tableNamespace: string;
+    phase: 'registered' | 'available' | 'failed';
+    persistedRows?: number;
+    succeededRows?: number;
+    failedRows?: number;
+    complete?: boolean;
+    at: number;
+  }>;
   /**
    * Explicit terminal-output replay for final runner logs when the caller must
    * keep `status` nonterminal. Direct-worker/Postgres progress finalizers do this so
@@ -75,6 +91,8 @@ export type RuntimeStatusUpdate = {
   terminalLogReplay?: {
     lines: string[];
     totalCount?: number;
+    dedupeMode?: 'semantic' | 'exact';
+    producerAttempt?: number;
   } | null;
   liveTimeline?: PlayRunTimelineEntry[];
   liveNodeProgress?: PlayVisualNodeProgressMap;
@@ -189,11 +207,27 @@ type RuntimeApiRequest =
       runId: string;
       key: string;
     }
+  | {
+      action: 'acquire_runtime_receipt_execution_lock';
+      playName: string;
+      runId: string;
+      key: string;
+      ownerExecutionId: string;
+      ttlMs: number;
+    }
+  | {
+      action: 'release_runtime_receipt_execution_lock';
+      playName: string;
+      runId: string;
+      key: string;
+      ownerExecutionId: string;
+    }
   | {
       action: 'claim_runtime_step_receipt';
       playName: string;
       runId: string;
       key: string;
+      leaseId?: string;
       runAttempt?: number | null;
       leaseAware?: boolean;
       reclaimRunning?: boolean;
@@ -247,6 +281,7 @@ type RuntimeApiRequest =
       playName: string;
       runId: string;
       keys: string[];
+      leaseIds?: string[];
       runAttempt?: number | null;
       leaseAware?: boolean;
       reclaimRunning?: boolean;
@@ -347,12 +382,84 @@ function applyRetryJitter(delayMs: number): number {
 const APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
 const RUN_STATUS_LEDGER_SNAPSHOT_CACHE_LIMIT = 1_000;
 const runStatusLedgerSnapshots = new Map<string, PlayRunLedgerSnapshot>();
-// Positional log cursor per run: count of liveLogs lines already forwarded to
-// the ledger from this process. Process-local on purpose — a fresh process
-// re-sends from offset 0 and Run Log Stream ingestion skips the overlap
-// positionally (exactly-once on the channel, repeated identical lines kept).
+// Positional log cursor per run attempt: count of liveLogs lines already
+// forwarded from this process. A fresh process re-sends from offset 0; durable
+// ingestion skips overlap within that same attempt.
 const runLogChannelSentCounts = new Map<string, number>();
 const runStatusUpdateChains = new Map<string, Promise<void>>();
+const APP_RUNTIME_LOG_EVENT_MAX_JSON_BYTES = 128 * 1024;
+const APP_RUNTIME_EVENT_BATCH_MAX_JSON_BYTES = 512 * 1024;
+const appRuntimeTextEncoder = new TextEncoder();
+
+function splitRunEventForAppRuntime(
+  event: PlayRunLedgerEvent,
+): PlayRunLedgerEvent[] {
+  if (event.type !== 'log.appended' || event.lines.length === 0) return [event];
+  const split: PlayRunLedgerEvent[] = [];
+  let index = 0;
+  while (index < event.lines.length) {
+    const start = index;
+    let bytes = 2;
+    while (index < event.lines.length) {
+      const lineBytes = appRuntimeTextEncoder.encode(
+        JSON.stringify(event.lines[index]!),
+      ).length;
+      if (
+        index > start &&
+        bytes + lineBytes + 1 > APP_RUNTIME_LOG_EVENT_MAX_JSON_BYTES
+      ) {
+        break;
+      }
+      bytes += lineBytes + 1;
+      index += 1;
+    }
+    split.push({
+      ...event,
+      lines: event.lines.slice(start, index),
+      ...(typeof event.channelOffset === 'number'
+        ? { channelOffset: event.channelOffset + start }
+        : {}),
+    });
+  }
+  return split;
+}
+
+export function partitionRunEventsForAppRuntime(
+  events: readonly PlayRunLedgerEvent[],
+): PlayRunLedgerEvent[][] {
+  const batches: PlayRunLedgerEvent[][] = [];
+  let current: PlayRunLedgerEvent[] = [];
+  let currentBytes = 2;
+  for (const event of events.flatMap(splitRunEventForAppRuntime)) {
+    const eventBytes = appRuntimeTextEncoder.encode(
+      JSON.stringify(event),
+    ).length;
+    if (
+      current.length > 0 &&
+      currentBytes + eventBytes + 1 > APP_RUNTIME_EVENT_BATCH_MAX_JSON_BYTES
+    ) {
+      batches.push(current);
+      current = [];
+      currentBytes = 2;
+    }
+    current.push(event);
+    currentBytes += eventBytes + 1;
+  }
+  if (current.length > 0) batches.push(current);
+  return batches;
+}
+
+async function appendRunEventBatchesViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: { playId: string; events: readonly PlayRunLedgerEvent[] },
+): Promise<void> {
+  for (const events of partitionRunEventsForAppRuntime(input.events)) {
+    await appendRunEventsViaAppRuntime(context, {
+      playId: input.playId,
+      events,
+    });
+  }
+}
 
 function isTerminalRuntimeStatus(status: string): boolean {
   return (
@@ -364,6 +471,10 @@ function isTerminalRuntimeStatus(status: string): boolean {
   );
 }
 
+function runLogChannelKey(runId: string, producerAttempt?: number): string {
+  return `${runId}:${producerAttempt ?? 'legacy'}`;
+}
+
 function rememberRunStatusLedgerSnapshot(
   runId: string,
   snapshot: PlayRunLedgerSnapshot,
@@ -376,7 +487,11 @@ function rememberRunStatusLedgerSnapshot(
     const oldestRunId = runStatusLedgerSnapshots.keys().next().value;
     if (!oldestRunId) break;
     runStatusLedgerSnapshots.delete(oldestRunId);
-    runLogChannelSentCounts.delete(oldestRunId);
+    for (const key of runLogChannelSentCounts.keys()) {
+      if (key.startsWith(`${oldestRunId}:`)) {
+        runLogChannelSentCounts.delete(key);
+      }
+    }
     runStatusUpdateChains.delete(oldestRunId);
   }
 }
@@ -399,12 +514,16 @@ function isRetryableAppRuntimeResponse(input: {
   if (input.status === 500 && isTransientAppRuntimeFailureBody(input.body)) {
     return true;
   }
+  return isRetryableAppRuntimeResponseStatus(input.status);
+}
+
+function isRetryableAppRuntimeResponseStatus(status: number): boolean {
   return (
-    input.status === 429 ||
-    (input.status >= 520 && input.status <= 524) ||
-    input.status === 502 ||
-    input.status === 503 ||
-    input.status === 504
+    status === 429 ||
+    (status >= 520 && status <= 524) ||
+    status === 502 ||
+    status === 503 ||
+    status === 504
   );
 }
 
@@ -601,6 +720,38 @@ function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+async function retryAppRuntimeBodyTimeoutOrThrow(input: {
+  action: RuntimeApiRequest['action'];
+  attempt: number;
+  maxAttempts: number;
+  error: unknown;
+  retryAfterMs?: number | null;
+}): Promise<void> {
+  if (!isRequestTimeoutAbort(input.error)) {
+    throw input.error;
+  }
+  if (
+    input.attempt < input.maxAttempts &&
+    isRetryableAppRuntimeFetchError({
+      action: input.action,
+      error: input.error,
+    })
+  ) {
+    await sleep(
+      Math.max(
+        applyRetryJitter(appRuntimeRetryDelayMs(input.action, input.attempt)),
+        input.retryAfterMs ?? 0,
+      ),
+    );
+    return;
+  }
+  throw new AppRuntimeApiTransportError({
+    action: input.action,
+    attempts: input.attempt,
+    cause: input.error,
+  });
+}
+
 export class AppRuntimeApiTransportError extends Error {
   readonly action: RuntimeApiRequest['action'];
   readonly attempts: number;
@@ -691,9 +842,40 @@ async function postAppRuntimeApi<TResponse>(
       });
     }
     if (response.ok) {
-      return (await response.json()) as TResponse;
+      try {
+        return (await response.json()) as TResponse;
+      } catch (error) {
+        await retryAppRuntimeBodyTimeoutOrThrow({
+          action: body.action,
+          attempt,
+          maxAttempts,
+          error,
+        });
+        continue;
+      }
+    }
+    let responseText: string;
+    try {
+      responseText = await response.text();
+    } catch (error) {
+      if (
+        isRequestTimeoutAbort(error) &&
+        !isRetryableAppRuntimeResponseStatus(response.status)
+      ) {
+        throw new Error(
+          `App runtime API ${body.action} failed with status ${response.status}: response body timed out`,
+          { cause: error },
+        );
+      }
+      await retryAppRuntimeBodyTimeoutOrThrow({
+        action: body.action,
+        attempt,
+        maxAttempts,
+        error,
+        retryAfterMs: appRuntimeRetryAfterMs(response, ''),
+      });
+      continue;
     }
-    const responseText = await response.text();
     if (
       attempt < maxAttempts &&
       isRetryableAppRuntimeResponse({
@@ -917,7 +1099,9 @@ async function updateRunStatusViaAppRuntimeUnlocked(
   // durable stream can recover any progress flush that was lost in transit.
   const sentCount = isTerminalRuntimeStatus(update.status)
     ? 0
-    : (runLogChannelSentCounts.get(update.playId) ?? 0);
+    : (runLogChannelSentCounts.get(
+        runLogChannelKey(update.playId, update.liveLogProducerAttempt),
+      ) ?? 0);
   const logSlice = Array.isArray(update.liveLogs)
     ? slicePositionalLogLines({
         bufferLines: update.liveLogs,
@@ -932,6 +1116,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
         ? {
             lines: update.liveLogs,
             totalCount: update.liveLogTotalCount,
+            producerAttempt: update.liveLogProducerAttempt,
           }
         : null;
   const statusPatchLogSlice = terminalLogReplay ? null : logSlice;
@@ -944,6 +1129,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
       lastCheckpointAt: update.lastCheckpointAt ?? null,
       liveLogs: statusPatchLogSlice?.lines ?? null,
       liveLogsChannelOffset: statusPatchLogSlice?.channelOffset ?? null,
+      liveLogsProducerAttempt: update.liveLogProducerAttempt ?? null,
       liveNodeProgress: update.liveNodeProgress ?? null,
       result: update.result,
     },
@@ -951,33 +1137,87 @@ async function updateRunStatusViaAppRuntimeUnlocked(
     now: update.lastCheckpointAt ?? Date.now(),
     source: 'worker',
   });
-  if (terminalLogReplay && terminalLogReplay.lines.length > 0) {
-    events.push(
-      ...buildTerminalLogReplayEvents({
-        runId: update.playId,
-        source: 'worker',
-        occurredAt: update.lastCheckpointAt ?? Date.now(),
-        lines: terminalLogReplay.lines,
-        liveLogTotalCount: terminalLogReplay.totalCount,
-      }),
-    );
+  for (const event of update.datasetLifecycleEvents ?? []) {
+    const current = previousSnapshot.datasetsById[event.datasetId];
+    const changesSnapshot =
+      !current ||
+      current.path !== event.path ||
+      current.tableNamespace !== event.tableNamespace ||
+      current.phase !== event.phase ||
+      (event.persistedRows ?? 0) > current.persistedRows ||
+      (event.succeededRows ?? 0) > current.succeededRows ||
+      (event.failedRows ?? 0) > current.failedRows ||
+      (event.complete === true && current.complete !== true);
+    if (!changesSnapshot) continue;
+    events.push({
+      type: 'dataset.lifecycle',
+      runId: update.playId,
+      source: 'worker',
+      occurredAt: event.at,
+      datasetId: event.datasetId,
+      path: event.path,
+      tableNamespace: event.tableNamespace,
+      phase: event.phase,
+      ...(event.persistedRows === undefined
+        ? {}
+        : { persistedRows: event.persistedRows }),
+      ...(event.succeededRows === undefined
+        ? {}
+        : { succeededRows: event.succeededRows }),
+      ...(event.failedRows === undefined
+        ? {}
+        : { failedRows: event.failedRows }),
+      ...(event.complete === undefined ? {} : { complete: event.complete }),
+    });
   }
-  if (events.length === 0) {
+  const terminalLogEvents =
+    terminalLogReplay && terminalLogReplay.lines.length > 0
+      ? buildTerminalLogReplayEvents({
+          runId: update.playId,
+          source: 'worker',
+          occurredAt: update.lastCheckpointAt ?? Date.now(),
+          lines: terminalLogReplay.lines,
+          liveLogTotalCount: terminalLogReplay.totalCount,
+          dedupeMode: terminalLogReplay.dedupeMode,
+          producerAttempt: terminalLogReplay.producerAttempt,
+        })
+      : [];
+  if (events.length === 0 && terminalLogEvents.length === 0) {
     return;
   }
-  await appendRunEventsViaAppRuntime(context, {
-    playId: update.playId,
-    events,
-  });
+  const combinedEvents = [...events, ...terminalLogEvents];
+  if (partitionRunEventsForAppRuntime(combinedEvents).length === 1) {
+    // Preserve the single-mutation fast path for ordinary bounded updates.
+    await appendRunEventsViaAppRuntime(context, {
+      playId: update.playId,
+      events: combinedEvents,
+    });
+  } else {
+    // Persist terminal lifecycle first so an oversized or interrupted
+    // best-effort log heal can never prevent settlement. Each bounded log
+    // batch is a separate mutation; retries are safe because ingestion owns
+    // positional reconciliation.
+    await appendRunEventBatchesViaAppRuntime(context, {
+      playId: update.playId,
+      events,
+    });
+    await appendRunEventBatchesViaAppRuntime(context, {
+      playId: update.playId,
+      events: terminalLogEvents,
+    });
+  }
   if (statusPatchLogSlice) {
     runLogChannelSentCounts.set(
-      update.playId,
+      runLogChannelKey(update.playId, update.liveLogProducerAttempt),
       statusPatchLogSlice.channelOffset + statusPatchLogSlice.lines.length,
     );
   }
   rememberRunStatusLedgerSnapshot(
     update.playId,
-    reducePlayRunLedgerEvents(previousSnapshot, events),
+    reducePlayRunLedgerEvents(previousSnapshot, [
+      ...events,
+      ...terminalLogEvents,
+    ]),
   );
 }
 
@@ -1109,6 +1349,38 @@ export async function getRuntimeStepReceiptViaAppRuntime(
   });
 }
 
+export async function acquireRuntimeReceiptExecutionLockViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: Omit<
+    Extract<
+      RuntimeApiRequest,
+      { action: 'acquire_runtime_receipt_execution_lock' }
+    >,
+    'action'
+  >,
+): Promise<{ ownerExecutionId: string; expiresAt: string } | null> {
+  return await postAppRuntimeApi(context, {
+    action: 'acquire_runtime_receipt_execution_lock',
+    ...input,
+  });
+}
+
+export async function releaseRuntimeReceiptExecutionLockViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: Omit<
+    Extract<
+      RuntimeApiRequest,
+      { action: 'release_runtime_receipt_execution_lock' }
+    >,
+    'action'
+  >,
+): Promise<boolean> {
+  return await postAppRuntimeApi(context, {
+    action: 'release_runtime_receipt_execution_lock',
+    ...input,
+  });
+}
+
 export async function getRuntimeStepReceiptsViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: Omit<

package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts

@@ -0,0 +1,14 @@
+import type { PlayCallExecution } from './play-call-execution';
+
+export function resolveChildExecutionPlacement(input: {
+  requiresWorkflow: boolean;
+  execution?: PlayCallExecution | null;
+}): { strategy: 'inline' | 'scheduled' } {
+  return {
+    strategy:
+      input.execution === 'child-workflow' ||
+      (input.execution !== 'inline' && input.requiresWorkflow)
+        ? 'scheduled'
+        : 'inline',
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts

@@ -2,6 +2,8 @@ import {
   flattenStaticPipeline,
   type PlayStaticPipeline,
 } from '../plays/static-pipeline';
+import type { PlayCallExecution } from './play-call-execution';
+import { resolveChildExecutionPlacement } from './child-execution-placement';
 
 export type ChildExecutionStrategy = 'inline' | 'scheduled';
 
@@ -12,7 +14,9 @@ export type ChildExecutionDecision = {
     | 'missing_static_contract'
     | 'dataset_child'
     | 'event_wait_child'
-    | 'explicit_timeout';
+    | 'explicit_timeout'
+    | 'explicit_inline'
+    | 'explicit_child_workflow';
 };
 
 /**
@@ -23,31 +27,61 @@ export type ChildExecutionDecision = {
 export function resolveChildExecutionStrategy(input: {
   pipeline: PlayStaticPipeline | null | undefined;
   timeoutMs?: number | null;
+  execution?: PlayCallExecution | null;
+  childPlayName?: string | null;
 }): ChildExecutionDecision {
-  if (input.timeoutMs != null) {
-    return { strategy: 'scheduled', reason: 'explicit_timeout' };
+  if (input.execution === 'child-workflow') {
+    return { strategy: 'scheduled', reason: 'explicit_child_workflow' };
   }
-  if (!input.pipeline) {
-    return { strategy: 'scheduled', reason: 'missing_static_contract' };
+  const steps = input.pipeline ? flattenStaticPipeline(input.pipeline) : [];
+  const authoredDecision: ChildExecutionDecision =
+    input.timeoutMs != null
+      ? { strategy: 'scheduled', reason: 'explicit_timeout' }
+      : !input.pipeline
+        ? { strategy: 'scheduled', reason: 'missing_static_contract' }
+        : input.pipeline.tableNamespace ||
+            input.pipeline.sheetContract ||
+            steps.some((step) => step.type === 'dataset' || step.type === 'csv')
+          ? { strategy: 'scheduled', reason: 'dataset_child' }
+          : steps.some(
+                (step) =>
+                  step.type === 'tool' &&
+                  (step.isEventWait || step.toolId === 'test_wait_for_event'),
+              )
+            ? { strategy: 'scheduled', reason: 'event_wait_child' }
+            : { strategy: 'inline', reason: 'scalar_child' };
+  if (input.execution === 'inline') {
+    if (
+      authoredDecision.strategy === 'scheduled' &&
+      authoredDecision.reason !== 'missing_static_contract'
+    ) {
+      throw new Error(
+        inlineExecutionRequiresScalarChildMessage(
+          input.childPlayName?.trim() || 'child',
+          authoredDecision.reason,
+        ),
+      );
+    }
+    return { strategy: 'inline', reason: 'explicit_inline' };
   }
-  const substeps = flattenStaticPipeline(input.pipeline);
-  if (
-    input.pipeline.tableNamespace ||
-    input.pipeline.sheetContract ||
-    substeps.some((step) => step.type === 'dataset' || step.type === 'csv')
-  ) {
-    return { strategy: 'scheduled', reason: 'dataset_child' };
+  const placement = resolveChildExecutionPlacement({
+    requiresWorkflow: authoredDecision.strategy === 'scheduled',
+    execution: input.execution,
+  });
+  if (placement.strategy === 'inline') {
+    return { strategy: 'inline', reason: 'scalar_child' };
   }
-  if (
-    substeps.some(
-      (step) =>
-        step.type === 'tool' &&
-        (step.isEventWait === true || step.toolId === 'test_wait_for_event'),
-    )
-  ) {
-    return { strategy: 'scheduled', reason: 'event_wait_child' };
-  }
-  return { strategy: 'inline', reason: 'scalar_child' };
+  return authoredDecision;
+}
+
+export function inlineExecutionRequiresScalarChildMessage(
+  childPlayName: string,
+  reason: ChildExecutionDecision['reason'],
+): string {
+  return (
+    `ctx.runPlay("${childPlayName}") requested execution: "inline", but the child requires a child workflow ` +
+    `(${reason}). Remove the explicit execution option or use execution: "child-workflow".`
+  );
 }
 
 /**
@@ -63,7 +97,7 @@ export function inlineChildScheduledInRowMessage(
   return (
     `ctx.runPlay("${childPlayName}") runs per row in the "${rowContextField}" batch, ` +
     'but it is a scheduling-required child (it uses ctx.dataset()/ctx.csv(), waits on an ' +
-    'event, or sets an explicit timeout). Row-scoped children must be scalar. ' +
+    'event, sets an explicit timeout, or explicitly requests a child workflow). Row-scoped children must be scalar. ' +
     'Give the batch ONE dataset child at play level instead of fanning a scheduled run out per row.'
   );
 }