deepline 0.1.181 → 0.1.183

package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts

@@ -20,11 +20,15 @@ export type WorkflowStepLike = {
 };
 
 export type ChildTerminalCoordinator = {
-  readChildTerminalState(
+  readChildTerminalState?(
     parentRunId: string,
     eventKey: string,
     timeoutMs?: number,
   ): Promise<{ data?: unknown } | null>;
+  readRunTerminalState(
+    runId: string,
+    timeoutMs?: number,
+  ): Promise<ChildRunTerminalSnapshotState | null>;
 };
 
 export type ChildPlayTerminalWaitResult = {
@@ -34,6 +38,52 @@ export type ChildPlayTerminalWaitResult = {
   waitMs: number;
 };
 
+export type ChildRunTerminalSnapshotState = {
+  data?: unknown;
+  reason?: string;
+  status?: string;
+  runId?: string;
+  playName?: string;
+  parentRunId?: string | null;
+};
+
+export class ChildPlayTerminalVerificationError extends Error {
+  readonly parentRunId: string;
+  readonly eventKey: string;
+  readonly cause: unknown;
+
+  constructor(input: {
+    parentRunId: string;
+    eventKey: string;
+    cause: unknown;
+  }) {
+    const causeMessage =
+      input.cause instanceof Error
+        ? input.cause.message
+        : String(input.cause ?? '');
+    super(
+      `Child play terminal verification failed for parent run ${input.parentRunId} and event ${input.eventKey}.${
+        causeMessage ? ` Cause: ${causeMessage}` : ''
+      }`,
+    );
+    this.name = 'ChildPlayTerminalVerificationError';
+    this.parentRunId = input.parentRunId;
+    this.eventKey = input.eventKey;
+    this.cause = input.cause;
+  }
+}
+
+const CHILD_TERMINAL_DURABLE_VERIFY_GRACE_MS = 5_000;
+const CHILD_TERMINAL_DURABLE_VERIFY_POLL_MS = 250;
+const CHILD_TERMINAL_DURABLE_VERIFY_READ_TIMEOUT_MS = 3_000;
+const CHILD_TERMINAL_DURABLE_VERIFY_MIN_READ_TIMEOUT_MS = 2_000;
+const CHILD_TERMINAL_PERMANENT_RUNTIME_API_STATUS_RE =
+  /runtime API (400|401|403|404):/i;
+const CHILD_TERMINAL_RETRYABLE_RUNTIME_API_STATUS_RE =
+  /runtime API (408|429|500|502|503|504|530):/i;
+const CHILD_TERMINAL_RETRYABLE_ERROR_RE =
+  /timed out|timeout|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT/i;
+
 export interface AwaitChildTerminalInput {
   parentRunId: string;
   workflowStep: WorkflowStepLike | undefined;
@@ -41,7 +91,7 @@ export interface AwaitChildTerminalInput {
   playName: string;
   key: string;
   timeoutMs: number;
-  /** Coordinator binding for the terminal-state poll. Null disables the poll. */
+  /** Coordinator binding for the terminal-state poll and durable run verification. */
   coordinator: ChildTerminalCoordinator | null;
   now: () => number;
   /** SHA-256 hex digest helper (same canonical hash entry.ts uses). */
@@ -84,6 +134,27 @@ function extractChildPlayOutput(status: Record<string, unknown>): unknown {
   return result ?? null;
 }
 
+function terminalPayloadFromState(
+  state: { data?: unknown } | null | undefined,
+): Record<string, unknown> | null {
+  return isRecord(state?.data) ? state.data : null;
+}
+
+function isRetryableChildTerminalSnapshotError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  if (CHILD_TERMINAL_PERMANENT_RUNTIME_API_STATUS_RE.test(message)) {
+    return false;
+  }
+  return (
+    CHILD_TERMINAL_RETRYABLE_RUNTIME_API_STATUS_RE.test(message) ||
+    CHILD_TERMINAL_RETRYABLE_ERROR_RE.test(message)
+  );
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 async function childPlayEventKey(input: {
   key: string;
   workflowId: string;
@@ -120,9 +191,19 @@ async function pollParentChildTerminalState(input: {
       input.timeoutMs - (input.now() - startedAt),
     );
     const waitMs = Math.min(remainingMs, 30_000);
-    const state = await coordinator
-      .readChildTerminalState(parentRunId, input.eventKey, waitMs)
-      .catch(() => null);
+    let state: { data?: unknown } | null;
+    try {
+      // prettier-ignore
+      state = await coordinator.readChildTerminalState(parentRunId, input.eventKey, waitMs);
+    } catch {
+      // The parent-child cache is a safety net for missed workflow events.
+      // Transient cache read failures must not beat the event wait plus durable
+      // child run verification path in the race.
+      if (remainingMs > 0) {
+        await sleep(Math.min(remainingMs, CHILD_TERMINAL_DURABLE_VERIFY_POLL_MS));
+      }
+      continue;
+    }
     if (isRecord(state?.data)) {
       return {
         source: 'parent_child_terminal_cache',
@@ -134,6 +215,78 @@ async function pollParentChildTerminalState(input: {
   return await new Promise(() => undefined);
 }
 
+async function readVerifiedChildTerminalPayload(input: {
+  coordinator: ChildTerminalCoordinator;
+  workflowId: string;
+  timeoutMs: number;
+}): Promise<{
+  payload: Record<string, unknown> | null;
+  lastState: ChildRunTerminalSnapshotState | null;
+  lastError: unknown;
+}> {
+  const graceMs = Math.max(
+    1,
+    Math.min(input.timeoutMs, CHILD_TERMINAL_DURABLE_VERIFY_GRACE_MS),
+  );
+  const pollMs = Math.max(
+    1,
+    Math.min(
+      CHILD_TERMINAL_DURABLE_VERIFY_POLL_MS,
+      Math.max(1, Math.floor(graceMs / 2)),
+    ),
+  );
+  const deadlineMs = Date.now() + graceMs;
+  let lastState: ChildRunTerminalSnapshotState | null = null;
+  let lastError: unknown = null;
+  const minReadTimeoutMs =
+    graceMs >= CHILD_TERMINAL_DURABLE_VERIFY_MIN_READ_TIMEOUT_MS
+      ? CHILD_TERMINAL_DURABLE_VERIFY_MIN_READ_TIMEOUT_MS
+      : 1;
+  while (Date.now() <= deadlineMs) {
+    const remainingMs = Math.max(0, deadlineMs - Date.now());
+    if (remainingMs < minReadTimeoutMs) break;
+    const readTimeoutMs = Math.max(
+      minReadTimeoutMs,
+      Math.min(remainingMs, CHILD_TERMINAL_DURABLE_VERIFY_READ_TIMEOUT_MS),
+    );
+    try {
+      const state = await input.coordinator.readRunTerminalState(
+        input.workflowId,
+        readTimeoutMs,
+      );
+      lastState = state ?? null;
+      const payload = terminalPayloadFromState(state);
+      if (payload) return { payload, lastState, lastError };
+    } catch (error) {
+      lastError = error;
+      if (!isRetryableChildTerminalSnapshotError(error)) {
+        throw error;
+      }
+    }
+    const sleepMs = Math.min(
+      pollMs,
+      Math.max(0, deadlineMs - Date.now() - minReadTimeoutMs),
+    );
+    if (sleepMs <= 0) break;
+    await sleep(sleepMs);
+  }
+  return { payload: null, lastState, lastError };
+}
+
+function describeSnapshotState(
+  state: ChildRunTerminalSnapshotState | null,
+): string {
+  if (!state) return 'no snapshot response';
+  const parts = [
+    state.reason ? `reason=${state.reason}` : null,
+    state.status ? `status=${state.status}` : null,
+    state.runId ? `runId=${state.runId}` : null,
+    state.playName ? `playName=${state.playName}` : null,
+    state.parentRunId ? `parentRunId=${state.parentRunId}` : null,
+  ].filter((part): part is string => Boolean(part));
+  return parts.length > 0 ? parts.join(' ') : 'empty snapshot response';
+}
+
 /**
  * Block until the child run reaches a terminal state, via the workflow event
  * wait raced against the coordinator terminal-state poll. Throws on a non-
@@ -153,6 +306,15 @@ export async function awaitChildTerminal(
     workflowId: input.workflowId,
     hashJson: input.hashJson,
   });
+  if (!input.coordinator?.readRunTerminalState) {
+    throw new ChildPlayTerminalVerificationError({
+      parentRunId: input.parentRunId,
+      eventKey,
+      cause: new Error(
+        `Child run ${input.workflowId} terminal snapshot reader was not available.`,
+      ),
+    });
+  }
   const eventPromise = input.workflowStep
     .waitForEvent(`child_play_terminal:${eventKey}`, {
       type: integrationEventType(eventKey),
@@ -173,7 +335,42 @@ export async function awaitChildTerminal(
       now: input.now,
     }),
   ]);
-  const payload = terminal.payload;
+  let verified:
+    | {
+        payload: Record<string, unknown> | null;
+        lastState: ChildRunTerminalSnapshotState | null;
+        lastError: unknown;
+      }
+    | null = null;
+  try {
+    verified = await readVerifiedChildTerminalPayload({
+      coordinator: input.coordinator,
+      workflowId: input.workflowId,
+      timeoutMs: input.timeoutMs,
+    });
+  } catch (error) {
+    throw new ChildPlayTerminalVerificationError({
+      parentRunId: input.parentRunId,
+      eventKey,
+      cause: error,
+    });
+  }
+  if (!verified?.payload) {
+    const lastErrorMessage =
+      verified?.lastError instanceof Error
+        ? ` lastError=${verified.lastError.message}`
+        : verified?.lastError
+          ? ` lastError=${String(verified.lastError)}`
+          : '';
+    throw new ChildPlayTerminalVerificationError({
+      parentRunId: input.parentRunId,
+      eventKey,
+      cause: new Error(
+        `Child run ${input.workflowId} terminal snapshot was not available after durable verification grace; ${describeSnapshotState(verified?.lastState ?? null)}.${lastErrorMessage}`,
+      ),
+    });
+  }
+  const payload = verified.payload;
   const status = String(payload.status ?? '').toLowerCase();
   if (status === 'completed') {
     return {

package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts

@@ -142,6 +142,12 @@ export async function submitChildPlayThroughCoordinator(
     input.req.runId,
   )}/submit-child`;
   const serializedBody = JSON.stringify(body);
+  const requestId = input.makeRequestId();
+  const idempotencyKey =
+    typeof body.childIdempotencyKey === 'string' &&
+    body.childIdempotencyKey.trim()
+      ? body.childIdempotencyKey.trim()
+      : requestId;
   let lastError: unknown = null;
 
   for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) {
@@ -150,7 +156,8 @@ export async function submitChildPlayThroughCoordinator(
       response = await fetchFn(url, {
         method: 'POST',
         headers: {
-          'x-deepline-request-id': input.makeRequestId(),
+          'x-deepline-request-id': requestId,
+          'x-deepline-idempotency-key': idempotencyKey,
           ...input.coordinatorRequestHeaders({
             runId: input.req.runId,
             contentType: 'application/json',