deepline 0.3.90 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts

@@ -0,0 +1,625 @@
+import {
+  RUNNER_TERMINAL_CHECKPOINT_DISPOSITION,
+  type PlayRunnerEvent,
+  type PlayRunnerResult,
+} from '../protocol';
+import { parseCtxFetchHttpFailureDiagnostic } from '../log-provenance';
+import {
+  RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES,
+  TERMINAL_RUN_RESULT_MAX_BYTES,
+  jsonByteLength,
+  jsonByteLengthUpTo,
+} from '../output-size-limits';
+import {
+  PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER,
+} from '../runtime-contract';
+import {
+  createRunnerGatewayTransportAttemptId,
+  runnerGatewayTransportErrorCode,
+  type RunnerGatewayTransportEvent,
+} from './runtime-gateway-transport';
+import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
+
+/**
+ * Push-execution terminal push (runs INSIDE the Daytona sandbox, B2-final).
+ *
+ * The worker parks after the detached start; the runner's pushed terminal is
+ * what wakes it. This POSTs the final status-aware terminal projection to the
+ * receipt gateway's `runner_terminal` action, which persists it on the attempt
+ * row (first-write-wins) and emits the attempt-scoped absurd wake event.
+ * Completed and failed projections omit replay payloads; suspended projections
+ * retain the checkpoint required for resume.
+ *
+ * Retries are bounded and loud. A push that ultimately fails does NOT fail the
+ * play by itself: the structured `result` event still lands in the captured
+ * output file, and the parked attempt's ceiling-timeout salvage reads it from
+ * there — slower, but no outcome is lost.
+ */
+
+const TERMINAL_PUSH_MAX_ATTEMPTS =
+  RUNTIME_RELIABILITY_POLICY.gateway.terminalPushMaxAttempts;
+const TERMINAL_PUSH_RETRY_DELAYS_MS = [500, 1_000, 2_000, 4_000] as const;
+const TERMINAL_PUSH_REQUEST_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.gateway.terminalPushRequestTimeoutMs;
+const APP_RUNTIME_COMPAT_PATH = '/api/v2/plays/runtime';
+
+/**
+ * A terminal result reserves its 5 MiB customer budget for the authored
+ * `output`. Runner diagnostics are useful, but they must never make a valid
+ * customer return too large to settle or to recover from Daytona stdout.
+ *
+ * These caps total well below the 1 MiB terminal-envelope headroom. They apply
+ * only to runner-owned diagnostics; returned rows and output values are never
+ * sliced here.
+ */
+const TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES = 128 * 1024;
+const TERMINAL_TRANSPORT_STATS_MAX_BYTES = 96 * 1024;
+const TERMINAL_TRANSPORT_STEPS_MAX_BYTES = 160 * 1024;
+const TERMINAL_TRANSPORT_FAILURE_DETAILS_MAX_BYTES = 96 * 1024;
+const TERMINAL_TRANSPORT_OUTPUT_WARNINGS_MAX_BYTES = 32 * 1024;
+const TERMINAL_TRANSPORT_RUNTIME_DIAGNOSTICS_MAX_BYTES = 16 * 1024;
+const TERMINAL_TRANSPORT_ERROR_MAX_BYTES = 64 * 1024;
+const TERMINAL_TRANSPORT_TABLE_NAMESPACE_MAX_BYTES = 8 * 1024;
+
+type CompletedRunnerResult = Extract<PlayRunnerResult, { status: 'completed' }>;
+type SuspendedRunnerResult = Extract<PlayRunnerResult, { status: 'suspended' }>;
+type FailedRunnerResult = Extract<PlayRunnerResult, { status: 'failed' }>;
+
+/**
+ * Settlement payload carried by a detached runner's gateway POST and stdout
+ * salvage line. Completed and failed executions cannot resume, so their replay
+ * checkpoint is redundant with durable receipts and must not cross the
+ * gateway's bounded request-body transport. Suspended executions still need
+ * the exact checkpoint to resume and remain unchanged.
+ */
+type OmittedTerminalCheckpoint = {
+  checkpointDisposition: typeof RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.OMITTED_TERMINAL_REPLAY;
+  checkpointBytes: number;
+};
+
+type IncludedResumeCheckpoint = {
+  checkpointDisposition: typeof RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.INCLUDED_RESUME_REQUIRED;
+  checkpointBytes: number;
+};
+
+type AbsentCheckpoint = {
+  checkpointDisposition: typeof RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.ABSENT;
+  checkpointBytes: 0;
+};
+
+export type PlayRunnerTerminalTransportResult =
+  | (Omit<CompletedRunnerResult, 'checkpoint'> &
+      (OmittedTerminalCheckpoint | AbsentCheckpoint))
+  | (SuspendedRunnerResult & IncludedResumeCheckpoint)
+  | (Omit<FailedRunnerResult, 'checkpoint'> &
+      (OmittedTerminalCheckpoint | AbsentCheckpoint));
+
+type SettledRunnerTerminalTransportResult = Exclude<
+  PlayRunnerTerminalTransportResult,
+  SuspendedRunnerResult & IncludedResumeCheckpoint
+>;
+
+function isTerminalTransportResult(
+  result: PlayRunnerResult | PlayRunnerTerminalTransportResult,
+): result is PlayRunnerTerminalTransportResult {
+  const disposition = (result as Partial<PlayRunnerTerminalTransportResult>)
+    .checkpointDisposition;
+  return (
+    disposition ===
+      RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.OMITTED_TERMINAL_REPLAY ||
+    disposition ===
+      RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.INCLUDED_RESUME_REQUIRED ||
+    disposition === RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.ABSENT
+  );
+}
+
+function fitsJsonByteLimit(value: unknown, limitBytes: number): boolean {
+  return !jsonByteLengthUpTo(value, limitBytes).exceeded;
+}
+
+function truncateStringForJsonValue(
+  value: string,
+  limitBytes: number,
+): { value: string; truncated: boolean } {
+  if (fitsJsonByteLimit(value, limitBytes)) {
+    return { value, truncated: false };
+  }
+
+  const suffix = '\u2026 [truncated]';
+  let low = 0;
+  let high = value.length;
+  let best = suffix;
+  while (low <= high) {
+    const middle = Math.floor((low + high) / 2);
+    const candidate = `${value.slice(0, middle)}${suffix}`;
+    if (fitsJsonByteLimit(candidate, limitBytes)) {
+      best = candidate;
+      low = middle + 1;
+    } else {
+      high = middle - 1;
+    }
+  }
+  return { value: best, truncated: true };
+}
+
+function retainArrayWithinJsonByteLimit<T>(
+  values: readonly T[],
+  limitBytes: number,
+  direction: 'head' | 'tail',
+): { values: T[]; truncated: boolean } {
+  const retained: T[] = [];
+  const ordered = direction === 'tail' ? [...values].reverse() : values;
+  for (const value of ordered) {
+    const candidate =
+      direction === 'tail' ? [value, ...retained] : [...retained, value];
+    if (!fitsJsonByteLimit(candidate, limitBytes)) {
+      return { values: retained, truncated: true };
+    }
+    if (direction === 'tail') {
+      retained.unshift(value);
+    } else {
+      retained.push(value);
+    }
+  }
+  return { values: retained, truncated: false };
+}
+
+/**
+ * Retain the normal newest log tail, but reserve space for the canonical
+ * ctx.fetch failure diagnostics first. `runtimeDiagnostics` powers the final
+ * warning, while the matching raw log line is the durable evidence customers
+ * see through `runs logs --debug` after progress events could not be pushed.
+ */
+function retainTerminalLogTailWithRuntimeDiagnostics(logs: readonly string[]): {
+  values: string[];
+  truncated: boolean;
+} {
+  const retainedIndexes = new Set<number>();
+  let retainedBytes = 2; // JSON array brackets.
+  const retainIfWithinLimit = (index: number): void => {
+    // String encodings and separators contribute independently to a JSON
+    // array. Re-serializing the retained 128 KiB tail for every historical
+    // line made large completed runs exhaust their terminal-flush deadline.
+    const separatorBytes = retainedIndexes.size ? 1 : 0;
+    const measurement = jsonByteLengthUpTo(
+      logs[index]!,
+      TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES - retainedBytes - separatorBytes,
+    );
+    if (measurement.exceeded) return;
+    retainedBytes += measurement.bytes + separatorBytes;
+    retainedIndexes.add(index);
+  };
+
+  // The newest diagnostics are the most relevant if malformed external output
+  // somehow exceeds the terminal transport's bounded diagnostic allowance.
+  for (let index = logs.length - 1; index >= 0; index -= 1) {
+    const line = logs[index]!;
+    if (!parseCtxFetchHttpFailureDiagnostic(line)) continue;
+    retainIfWithinLimit(index);
+  }
+
+  for (let index = logs.length - 1; index >= 0; index -= 1) {
+    if (retainedIndexes.has(index)) continue;
+    retainIfWithinLimit(index);
+  }
+
+  const indexes = [...retainedIndexes].sort((left, right) => left - right);
+  return {
+    values: indexes.map((index) => logs[index]!),
+    truncated: indexes.length !== logs.length,
+  };
+}
+
+function retainRecordWithinJsonByteLimit(
+  value: Record<string, unknown>,
+  limitBytes: number,
+): { value: Record<string, unknown>; truncated: boolean } {
+  if (fitsJsonByteLimit(value, limitBytes)) {
+    return { value, truncated: false };
+  }
+  const retained: Record<string, unknown> = {};
+  let truncated = false;
+  for (const [key, entry] of Object.entries(value)) {
+    const candidate = { ...retained, [key]: entry };
+    if (fitsJsonByteLimit(candidate, limitBytes)) {
+      retained[key] = entry;
+    } else {
+      truncated = true;
+    }
+  }
+  return { value: retained, truncated };
+}
+
+function appendTerminalDiagnosticsMarker(
+  logs: string[],
+  truncatedFields: string[],
+): string[] {
+  const marker =
+    `[deepline] terminal diagnostic metadata was bounded (${truncatedFields.join(', ')}); ` +
+    'the authored play output is complete.';
+  const marked = [marker, ...logs];
+  while (
+    marked.length > 1 &&
+    !fitsJsonByteLimit(marked, TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES)
+  ) {
+    // Preserve canonical ctx.fetch evidence when a marker itself needs room.
+    // Other tail lines are diagnostic context, but this line is the durable
+    // proof behind the final run warning and `runs logs --debug` explanation.
+    const removableIndex = marked.findIndex(
+      (line, index) =>
+        index > 0 && parseCtxFetchHttpFailureDiagnostic(line) === null,
+    );
+    marked.splice(removableIndex === -1 ? 1 : removableIndex, 1);
+  }
+  return fitsJsonByteLimit(marked, TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES)
+    ? marked
+    : [marker];
+}
+
+/**
+ * `logs`, `stats`, and rendered `steps` are runner diagnostics. Unlike
+ * `output`, they have no customer-return contract and can grow with every map
+ * row. Project them before stdout and gateway settlement so a complete 5 MiB
+ * result is always recoverable after a primary Daytona push failure.
+ */
+function boundSettledTerminalTransportDiagnostics(
+  result: SettledRunnerTerminalTransportResult,
+): SettledRunnerTerminalTransportResult {
+  const logProjection = retainTerminalLogTailWithRuntimeDiagnostics(
+    result.logs,
+  );
+  const statsProjection = retainRecordWithinJsonByteLimit(
+    result.stats,
+    TERMINAL_TRANSPORT_STATS_MAX_BYTES,
+  );
+  const stepsProjection = retainArrayWithinJsonByteLimit(
+    result.steps,
+    TERMINAL_TRANSPORT_STEPS_MAX_BYTES,
+    'head',
+  );
+  const tableNamespace =
+    typeof result.tableNamespace === 'string'
+      ? truncateStringForJsonValue(
+          result.tableNamespace,
+          TERMINAL_TRANSPORT_TABLE_NAMESPACE_MAX_BYTES,
+        )
+      : { value: result.tableNamespace, truncated: false };
+  const runtimeDiagnosticsProjection = retainArrayWithinJsonByteLimit(
+    result.status === 'completed' ? (result.runtimeDiagnostics ?? []) : [],
+    TERMINAL_TRANSPORT_RUNTIME_DIAGNOSTICS_MAX_BYTES,
+    'head',
+  );
+
+  const truncatedFields = [
+    ...(logProjection.truncated ? ['logs'] : []),
+    ...(statsProjection.truncated ? ['stats'] : []),
+    ...(stepsProjection.truncated ? ['steps'] : []),
+    ...(tableNamespace.truncated ? ['tableNamespace'] : []),
+    ...(runtimeDiagnosticsProjection.truncated ? ['runtimeDiagnostics'] : []),
+  ];
+
+  if (result.status === 'completed') {
+    const warningsProjection = retainArrayWithinJsonByteLimit(
+      result.outputWarnings ?? [],
+      TERMINAL_TRANSPORT_OUTPUT_WARNINGS_MAX_BYTES,
+      'head',
+    );
+    if (warningsProjection.truncated) truncatedFields.push('outputWarnings');
+    return {
+      ...result,
+      logs:
+        truncatedFields.length > 0
+          ? appendTerminalDiagnosticsMarker(
+              logProjection.values,
+              truncatedFields,
+            )
+          : logProjection.values,
+      stats: statsProjection.value,
+      steps: stepsProjection.values,
+      ...(result.outputWarnings
+        ? { outputWarnings: warningsProjection.values }
+        : {}),
+      ...(result.runtimeDiagnostics
+        ? { runtimeDiagnostics: runtimeDiagnosticsProjection.values }
+        : {}),
+      ...(typeof tableNamespace.value === 'string'
+        ? { tableNamespace: tableNamespace.value }
+        : {}),
+    };
+  }
+
+  const error = truncateStringForJsonValue(
+    result.error,
+    TERMINAL_TRANSPORT_ERROR_MAX_BYTES,
+  );
+  const errorsProjection = retainArrayWithinJsonByteLimit(
+    result.errors ?? [],
+    TERMINAL_TRANSPORT_FAILURE_DETAILS_MAX_BYTES,
+    'head',
+  );
+  if (error.truncated) truncatedFields.push('error');
+  if (errorsProjection.truncated) truncatedFields.push('errors');
+  return {
+    ...result,
+    error: error.value,
+    logs:
+      truncatedFields.length > 0
+        ? appendTerminalDiagnosticsMarker(logProjection.values, truncatedFields)
+        : logProjection.values,
+    stats: statsProjection.value,
+    steps: stepsProjection.values,
+    ...(result.errors ? { errors: errorsProjection.values } : {}),
+    ...(typeof tableNamespace.value === 'string'
+      ? { tableNamespace: tableNamespace.value }
+      : {}),
+  };
+}
+
+export function runnerTerminalResultForTransport(
+  result: PlayRunnerResult | PlayRunnerTerminalTransportResult,
+): PlayRunnerTerminalTransportResult {
+  let transportResult: PlayRunnerTerminalTransportResult;
+  if (isTerminalTransportResult(result)) {
+    transportResult = result;
+  } else {
+    // `isTerminalTransportResult` separates the optional transport shape from
+    // the runner's canonical result. Keep the raw result type here so a
+    // completed checkpoint remains required by that canonical contract.
+    const rawResult = result as PlayRunnerResult;
+    if (rawResult.status === 'suspended') {
+      transportResult = {
+        ...rawResult,
+        checkpointDisposition:
+          RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.INCLUDED_RESUME_REQUIRED,
+        checkpointBytes: jsonByteLength(rawResult.checkpoint),
+      };
+    } else {
+      const { checkpoint, ...terminal } = rawResult;
+      transportResult =
+        checkpoint === null || checkpoint === undefined
+          ? {
+              ...terminal,
+              checkpointDisposition:
+                RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.ABSENT,
+              checkpointBytes: 0,
+            }
+          : {
+              ...terminal,
+              checkpointDisposition:
+                RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.OMITTED_TERMINAL_REPLAY,
+              checkpointBytes: jsonByteLength(checkpoint),
+            };
+    }
+  }
+  if (transportResult.status === 'suspended') {
+    // A suspended result retains its exact checkpoint because it is needed to
+    // resume. Its checkpoint follows the dedicated receipt transport contract.
+    return transportResult;
+  }
+  const bounded = boundSettledTerminalTransportDiagnostics(transportResult);
+  if (!fitsJsonByteLimit(bounded, TERMINAL_RUN_RESULT_MAX_BYTES)) {
+    throw new Error(
+      'Terminal result exceeded its durable envelope after runner diagnostics were bounded.',
+    );
+  }
+  return bounded;
+}
+
+export interface PushRunnerTerminalOptions {
+  gatewayBaseUrl: string;
+  executorToken: string;
+  runId: string;
+  result: PlayRunnerResult | PlayRunnerTerminalTransportResult;
+  finalProgressEvents?: PlayRunnerEvent[];
+  finalProgressLogOffset?: number;
+  /** Injectable for tests. Defaults to a bearer POST to the gateway. */
+  fetchImpl?: typeof fetch;
+  sleepImpl?: (ms: number) => Promise<void>;
+  nowImpl?: () => number;
+  createTransportAttemptId?: () => string;
+  /** Per-attempt HTTP deadline. A hung gateway request must not suppress the
+   * remaining bounded retries. */
+  requestTimeoutMs?: number;
+  warn?: (message: string, detail: Record<string, unknown>) => void;
+  onTransport?: (event: RunnerGatewayTransportEvent) => void;
+}
+
+type RunnerTerminalRequestBody = {
+  action: 'runner_terminal';
+  runId: string;
+  result: PlayRunnerTerminalTransportResult;
+  finalProgressLogOffset?: number;
+  finalProgressEvents?: PlayRunnerEvent[];
+};
+
+function durableProgressLogCount(events: readonly PlayRunnerEvent[]): number {
+  return events.reduce(
+    (count, event) =>
+      event.type === 'log' && event.source !== 'worker' && event.line.length > 0
+        ? count + 1
+        : count,
+    0,
+  );
+}
+
+/** Keep the newest recovery observations without ever exceeding the gateway. */
+function terminalRequestBody(input: {
+  runId: string;
+  result: PlayRunnerTerminalTransportResult;
+  finalProgressEvents?: PlayRunnerEvent[];
+  finalProgressLogOffset?: number;
+}): { body: RunnerTerminalRequestBody; droppedProgressEvents: number } {
+  const events = input.finalProgressEvents ?? [];
+  const bodyFor = (
+    droppedProgressEvents: number,
+  ): RunnerTerminalRequestBody => {
+    const retainedEvents = events.slice(droppedProgressEvents);
+    return {
+      action: 'runner_terminal',
+      runId: input.runId,
+      result: input.result,
+      ...(input.finalProgressLogOffset !== undefined
+        ? {
+            finalProgressLogOffset:
+              input.finalProgressLogOffset +
+              durableProgressLogCount(events.slice(0, droppedProgressEvents)),
+          }
+        : {}),
+      ...(retainedEvents.length ? { finalProgressEvents: retainedEvents } : {}),
+    };
+  };
+
+  if (
+    !jsonByteLengthUpTo(bodyFor(0), RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES)
+      .exceeded
+  ) {
+    return { body: bodyFor(0), droppedProgressEvents: 0 };
+  }
+
+  // Dropping a prefix is monotonic for ordinary PlayRunnerEvent JSON. Binary
+  // search avoids repeatedly serializing a potentially multi-megabyte tail.
+  let low = 1;
+  let high = events.length;
+  let retained = bodyFor(events.length);
+  let droppedProgressEvents = events.length;
+  while (low <= high) {
+    const middle = Math.floor((low + high) / 2);
+    const candidate = bodyFor(middle);
+    if (
+      jsonByteLengthUpTo(candidate, RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES)
+        .exceeded
+    ) {
+      low = middle + 1;
+    } else {
+      retained = candidate;
+      droppedProgressEvents = middle;
+      high = middle - 1;
+    }
+  }
+  return { body: retained, droppedProgressEvents };
+}
+
+export async function pushRunnerTerminal(
+  options: PushRunnerTerminalOptions,
+): Promise<boolean> {
+  const fetchImpl = options.fetchImpl ?? fetch;
+  const now = options.nowImpl ?? Date.now;
+  const sleepImpl =
+    options.sleepImpl ??
+    ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const warn =
+    options.warn ??
+    ((message, detail) =>
+      console.warn(`[play-runner.terminal-push] ${message}`, detail));
+  const url = `${options.gatewayBaseUrl.replace(/\/$/, '')}${APP_RUNTIME_COMPAT_PATH}`;
+  const result = runnerTerminalResultForTransport(options.result);
+  const request = terminalRequestBody({
+    runId: options.runId,
+    result,
+    finalProgressEvents: options.finalProgressEvents,
+    finalProgressLogOffset: options.finalProgressLogOffset,
+  });
+  if (request.droppedProgressEvents > 0) {
+    warn('terminal_progress_truncated', {
+      runId: options.runId,
+      retainedEvents: request.body.finalProgressEvents?.length ?? 0,
+      droppedEvents: request.droppedProgressEvents,
+      requestBytes: jsonByteLength(request.body),
+    });
+  }
+  const requestBody = JSON.stringify(request.body);
+  let lastError = '';
+  for (let attempt = 1; attempt <= TERMINAL_PUSH_MAX_ATTEMPTS; attempt += 1) {
+    const startedAt = now();
+    const transportAttemptId =
+      options.createTransportAttemptId?.() ??
+      createRunnerGatewayTransportAttemptId();
+    const reportTransport = (
+      outcome: RunnerGatewayTransportEvent['outcome'],
+      detail: Pick<RunnerGatewayTransportEvent, 'status' | 'errorCode'> = {},
+    ) => {
+      const event: RunnerGatewayTransportEvent = {
+        operation: 'runner_terminal',
+        attempt,
+        transportAttemptId,
+        durationMs: Math.max(0, now() - startedAt),
+        outcome,
+        ...detail,
+      };
+      if (outcome !== 'ok') {
+        warn('gateway_transport', { runId: options.runId, ...event });
+      }
+      try {
+        options.onTransport?.(event);
+      } catch {
+        // Diagnostics must not interfere with terminal persistence or salvage.
+      }
+    };
+    const requestController = new AbortController();
+    const requestTimeout = setTimeout(
+      () =>
+        requestController.abort(
+          new Error(
+            `runner_terminal request exceeded ${
+              options.requestTimeoutMs ?? TERMINAL_PUSH_REQUEST_TIMEOUT_MS
+            }ms`,
+          ),
+        ),
+      options.requestTimeoutMs ?? TERMINAL_PUSH_REQUEST_TIMEOUT_MS,
+    );
+    requestTimeout.unref?.();
+    try {
+      const response = await fetchImpl(url, {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          authorization: `Bearer ${options.executorToken}`,
+          [PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER]: transportAttemptId,
+        },
+        body: requestBody,
+        redirect: 'manual',
+        signal: requestController.signal,
+      });
+      if (response.ok) {
+        await response.text().catch(() => '');
+        reportTransport('ok', { status: response.status });
+        return true;
+      }
+      const body = await response.text().catch(() => '');
+      lastError = `runner_terminal failed (${response.status}): ${body}`;
+      reportTransport(
+        response.status >= 500
+          ? 'gateway_5xx'
+          : response.status >= 400
+            ? 'gateway_4xx'
+            : 'redirect',
+        { status: response.status },
+      );
+      // 4xx (scope/fence rejections) are deterministic — retrying cannot help.
+      if (response.status >= 400 && response.status < 500) {
+        warn('terminal_push_rejected', {
+          runId: options.runId,
+          status: response.status,
+          body,
+        });
+        return false;
+      }
+    } catch (error) {
+      lastError = error instanceof Error ? error.message : String(error);
+      reportTransport('network_error', {
+        errorCode: runnerGatewayTransportErrorCode(error),
+      });
+    } finally {
+      clearTimeout(requestTimeout);
+    }
+    if (attempt < TERMINAL_PUSH_MAX_ATTEMPTS) {
+      await sleepImpl(TERMINAL_PUSH_RETRY_DELAYS_MS[attempt - 1] ?? 4_000);
+    }
+  }
+  warn('terminal_push_exhausted', {
+    runId: options.runId,
+    attempts: TERMINAL_PUSH_MAX_ATTEMPTS,
+    lastError,
+  });
+  return false;
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -319,6 +319,7 @@ const RUNTIME_SHEET_ATTEMPT_COLUMNS = [
   '_attempt_seq',
   '_attempt_expires_at',
 ] as const;
+const RUNTIME_SHEET_TIMESTAMP_COLUMNS = ['_created_at', '_updated_at'] as const;
 const RUNTIME_SHEET_ENSURE_CACHE_TTL_MS = 10 * 60_000;
 const RUNTIME_SHEET_READY_AFTER_ENSURE_RETRY_DELAYS_MS = [
   100, 250, 500, 1_000, 2_000,
@@ -1007,9 +1008,28 @@ async function isRuntimeSheetSchemaReady(
         [...RUNTIME_SHEET_ATTEMPT_COLUMNS],
       ],
     );
+    const timestampColumns = await client.query<Record<string, unknown>>(
+      `
+        SELECT count(*)::int AS ready_count
+        FROM information_schema.columns
+        WHERE table_schema = $1
+          AND table_name = $2
+          AND column_name = ANY($3::text[])
+          AND data_type = 'timestamp with time zone'
+          AND is_nullable = 'NO'
+          AND column_default IS NOT NULL
+      `,
+      [
+        session.postgres.schema,
+        session.postgres.sheetTable,
+        [...RUNTIME_SHEET_TIMESTAMP_COLUMNS],
+      ],
+    );
     return (
       Number(attemptColumns.rows[0]?.present_count ?? 0) ===
-      RUNTIME_SHEET_ATTEMPT_COLUMNS.length
+        RUNTIME_SHEET_ATTEMPT_COLUMNS.length &&
+      Number(timestampColumns.rows[0]?.ready_count ?? 0) ===
+        RUNTIME_SHEET_TIMESTAMP_COLUMNS.length
     );
   };
   try {
@@ -1040,9 +1060,14 @@ async function readRuntimeSheetSchemaDiagnostic(
   const query = async (
     client: RuntimeQueryClient,
   ): Promise<Record<string, unknown>> => {
-    const visibleColumns = await client.query<{ column_name: string }>(
+    const visibleColumns = await client.query<{
+      column_name: string;
+      data_type: string;
+      is_nullable: string;
+      column_default: string | null;
+    }>(
       `
-        SELECT column_name
+        SELECT column_name, data_type, is_nullable, column_default
         FROM information_schema.columns
         WHERE table_schema = $1
           AND table_name = $2
@@ -1066,6 +1091,15 @@ async function readRuntimeSheetSchemaDiagnostic(
       ],
     );
     const visible = new Set(visibleColumns.rows.map((row) => row.column_name));
+    const timestampColumns = new Map(
+      visibleColumns.rows
+        .filter((row) =>
+          RUNTIME_SHEET_TIMESTAMP_COLUMNS.includes(
+            row.column_name as (typeof RUNTIME_SHEET_TIMESTAMP_COLUMNS)[number],
+          ),
+        )
+        .map((row) => [row.column_name, row]),
+    );
     return {
       schema: session.postgres.schema,
       table: session.postgres.sheetTable,
@@ -1078,6 +1112,15 @@ async function readRuntimeSheetSchemaDiagnostic(
       missingAttemptColumns: RUNTIME_SHEET_ATTEMPT_COLUMNS.filter(
         (column) => !visible.has(column),
       ),
+      timestampColumnStates: RUNTIME_SHEET_TIMESTAMP_COLUMNS.map((column) => {
+        const state = timestampColumns.get(column);
+        return {
+          column,
+          dataType: state?.data_type ?? null,
+          isNullable: state?.is_nullable ?? null,
+          hasDefault: state?.column_default != null,
+        };
+      }),
     };
   };
   try {