deepline 0.1.182 → 0.1.184

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -2,6 +2,11 @@ import {
   normalizePlayRunFailure,
   PLATFORM_DEPLOY_INTERRUPTED_MESSAGE,
 } from './run-failure';
+import {
+  isAuthoritativePlayRunTerminalSource,
+  nextPlayRunLedgerTerminalSource,
+  normalizePlayRunLedgerTerminalSource,
+} from './run-terminal-source';
 
 export type PlayRunLedgerStatus =
   | 'queued'
@@ -88,6 +93,17 @@ export type PlayRunLedgerSnapshot = {
   resultTableNamespace?: string | null;
   resultSummary?: unknown;
   result?: unknown;
+  /**
+   * Source of the event that made this snapshot terminal. Authoritative
+   * producers ('worker', 'temporal', scheduler-outbox 'system') carry the
+   * play's final result; a 'coordinator'-sourced terminal is a transport
+   * heal whose `result` came from the coordinator's live cache and may be a
+   * partial mid-execution snapshot. Read-side canonical checks treat
+   * 'coordinator'-sourced completeds as provisional (see
+   * `hasCanonicalTerminalRunSnapshot`). Absent on snapshots persisted before
+   * this field existed — those are treated as authoritative.
+   */
+  terminalSource?: PlayRunLedgerEventSource | null;
 };
 
 type PlayRunLedgerBaseEvent = {
@@ -406,6 +422,85 @@ export function normalizePlayRunLedgerSnapshot(
     resultTableNamespace: optionalNullableString(value.resultTableNamespace),
     resultSummary: value.resultSummary,
     result: value.result,
+    terminalSource: normalizePlayRunLedgerTerminalSource(value.terminalSource),
+  };
+}
+
+/**
+ * Terminal-source transition for a (possibly repeated) terminal event.
+ * A transport heal ('coordinator') never overwrites an already-recorded
+ * authoritative source: when the producer's own terminal has landed, a late
+ * heal replay must not demote the snapshot back to provisional. Any
+ * authoritative source (worker/temporal/system/convex) always wins, so the
+ * worker's post-heal `run.completed` flips a healed snapshot to final.
+ */
+function nextTerminalSource(
+  base: PlayRunLedgerSnapshot,
+  eventSource: PlayRunLedgerEventSource,
+): PlayRunLedgerEventSource {
+  return nextPlayRunLedgerTerminalSource({
+    currentSource: base.terminalSource,
+    eventSource,
+  });
+}
+
+function shouldUpgradeFromCoordinatorTerminal(
+  base: PlayRunLedgerSnapshot,
+  eventSource: PlayRunLedgerEventSource,
+): boolean {
+  return (
+    base.terminalSource === 'coordinator' &&
+    isAuthoritativePlayRunTerminalSource(eventSource)
+  );
+}
+
+function shouldIgnoreNonTerminalAfterStickyTerminal(
+  base: PlayRunLedgerSnapshot,
+): boolean {
+  return isTerminalPlayRunLedgerStatus(base.status);
+}
+
+function maxProgressNumber(
+  current: number | undefined,
+  next: number | undefined,
+): number | undefined {
+  if (current === undefined) return next;
+  if (next === undefined) return current;
+  return Math.max(current, next);
+}
+
+function mergeMonotonicStepProgress(input: {
+  current?: PlayRunLedgerStepProgress | null;
+  next: PlayRunLedgerStepProgress;
+}): PlayRunLedgerStepProgress {
+  const current = input.current ?? {};
+  const next = input.next;
+  return {
+    ...current,
+    ...next,
+    ...(maxProgressNumber(current.completed, next.completed) !== undefined
+      ? { completed: maxProgressNumber(current.completed, next.completed) }
+      : {}),
+    ...(maxProgressNumber(current.total, next.total) !== undefined
+      ? { total: maxProgressNumber(current.total, next.total) }
+      : {}),
+    ...(maxProgressNumber(current.failed, next.failed) !== undefined
+      ? { failed: maxProgressNumber(current.failed, next.failed) }
+      : {}),
+    ...(maxProgressNumber(current.startedRows, next.startedRows) !== undefined
+      ? {
+          startedRows: maxProgressNumber(current.startedRows, next.startedRows),
+        }
+      : {}),
+    ...(maxProgressNumber(current.completedRows, next.completedRows) !==
+    undefined
+      ? {
+          completedRows: maxProgressNumber(
+            current.completedRows,
+            next.completedRows,
+          ),
+        }
+      : {}),
   };
 }
 
@@ -601,12 +696,28 @@ function conflictingTerminalSnapshot(
   eventType: keyof typeof TERMINAL_STATUS_BY_EVENT_TYPE,
   occurredAt: number,
   eventError?: string | null,
-  options?: { allowStaleSameStatusPayloadMerge?: boolean },
+  options?: {
+    allowStaleSameStatusPayloadMerge?: boolean;
+    allowCoordinatorTerminalUpgrade?: boolean;
+  },
 ): PlayRunLedgerSnapshot | null {
   if (!isTerminalPlayRunLedgerStatus(base.status)) {
     return null;
   }
   const terminalAt = base.finishedAt ?? base.updatedAt ?? 0;
+  if (base.status === 'cancelled' && eventType !== 'run.cancelled') {
+    if (occurredAt <= terminalAt) {
+      return withTiming(
+        appendLogLines(base, [
+          `[ledger] stale conflicting terminal event ${eventType} ignored; status already ${base.status}`,
+        ]),
+      );
+    }
+    return withTiming(base);
+  }
+  if (options?.allowCoordinatorTerminalUpgrade === true) {
+    return null;
+  }
   if (
     base.status === 'completed' &&
     eventType === 'run.failed' &&
@@ -727,9 +838,15 @@ export function reducePlayRunLedgerEvent(
     ...snapshot,
     updatedAt: Math.max(snapshot.updatedAt ?? 0, occurredAt),
   };
+  const ignoreNonTerminalAfterStickyTerminal =
+    (event.type === 'run.created' || event.type === 'run.started') &&
+    shouldIgnoreNonTerminalAfterStickyTerminal(base);
 
   switch (event.type) {
     case 'run.created': {
+      if (ignoreNonTerminalAfterStickyTerminal) {
+        return withTiming(base);
+      }
       const createdStatus = event.status ?? base.status;
       return withTiming({
         ...base,
@@ -742,6 +859,9 @@ export function reducePlayRunLedgerEvent(
       });
     }
     case 'run.started':
+      if (ignoreNonTerminalAfterStickyTerminal) {
+        return withTiming(base);
+      }
       return withTiming({
         ...base,
         playName: event.playName ?? base.playName ?? null,
@@ -753,7 +873,12 @@ export function reducePlayRunLedgerEvent(
     case 'run.completed':
       return (
         conflictingTerminalSnapshot(base, event.type, occurredAt, null, {
+          allowCoordinatorTerminalUpgrade: shouldUpgradeFromCoordinatorTerminal(
+            base,
+            event.source,
+          ),
           allowStaleSameStatusPayloadMerge:
+            shouldUpgradeFromCoordinatorTerminal(base, event.source) ||
             (base.result === undefined && event.result !== undefined) ||
             (base.resultSummary === undefined &&
               event.resultSummary !== undefined),
@@ -768,6 +893,7 @@ export function reducePlayRunLedgerEvent(
           activeArtifactTableNamespace: null,
           result: event.result ?? base.result,
           resultSummary: event.resultSummary ?? base.resultSummary,
+          terminalSource: nextTerminalSource(base, event.source),
         })
       );
     case 'run.failed':
@@ -775,6 +901,7 @@ export function reducePlayRunLedgerEvent(
         retryablePlatformDeployFailureSnapshot(base, event.error) ??
         conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
           allowStaleSameStatusPayloadMerge:
+            shouldUpgradeFromCoordinatorTerminal(base, event.source) ||
             (base.error == null && event.error != null) ||
             (base.result === undefined && event.result !== undefined),
         }) ??
@@ -790,12 +917,14 @@ export function reducePlayRunLedgerEvent(
           activeStepId: null,
           activeArtifactTableNamespace: null,
           result: event.result ?? base.result,
+          terminalSource: nextTerminalSource(base, event.source),
         })
       );
     case 'run.cancelled':
       return (
         conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
           allowStaleSameStatusPayloadMerge:
+            shouldUpgradeFromCoordinatorTerminal(base, event.source) ||
             (base.error == null && event.error != null) ||
             (base.result === undefined && event.result !== undefined),
         }) ??
@@ -811,6 +940,7 @@ export function reducePlayRunLedgerEvent(
           activeStepId: null,
           activeArtifactTableNamespace: null,
           result: event.result ?? base.result,
+          terminalSource: nextTerminalSource(base, event.source),
         })
       );
     case 'log.appended':
@@ -824,6 +954,7 @@ export function reducePlayRunLedgerEvent(
       );
     case 'step.started': {
       const current = base.stepsById[event.stepId];
+      const runTerminal = isTerminalPlayRunLedgerStatus(base.status);
       const shouldRefineSyntheticStart =
         current?.startedAt != null &&
         current.completedAt != null &&
@@ -855,19 +986,26 @@ export function reducePlayRunLedgerEvent(
         startedAt: base.startedAt ?? occurredAt,
         orderedStepIds: appendOrderedStepId(base, event.stepId),
         stepsById: { ...base.stepsById, [event.stepId]: nextStep },
-        activeStepId:
-          nextStep.status === 'running' ? event.stepId : base.activeStepId,
-        activeArtifactTableNamespace:
-          nextStep.status === 'running'
+        activeStepId: runTerminal
+          ? base.activeStepId
+          : nextStep.status === 'running'
+            ? event.stepId
+            : base.activeStepId,
+        activeArtifactTableNamespace: runTerminal
+          ? (base.activeArtifactTableNamespace ?? null)
+          : nextStep.status === 'running'
             ? (nextStep.artifactTableNamespace ?? null)
             : (base.activeArtifactTableNamespace ?? null),
       });
     }
     case 'step.progress': {
       const current = base.stepsById[event.stepId];
+      const runTerminal = isTerminalPlayRunLedgerStatus(base.status);
       const progress = {
-        ...(current?.progress ?? {}),
-        ...event.progress,
+        ...mergeMonotonicStepProgress({
+          current: current?.progress,
+          next: event.progress,
+        }),
         updatedAt: event.progress.updatedAt ?? occurredAt,
       };
       const inferredStatus =
@@ -924,14 +1062,16 @@ export function reducePlayRunLedgerEvent(
         ...base,
         orderedStepIds: appendOrderedStepId(base, event.stepId),
         stepsById: { ...base.stepsById, [event.stepId]: nextStep },
-        activeStepId:
-          status === 'running'
+        activeStepId: runTerminal
+          ? base.activeStepId
+          : status === 'running'
             ? event.stepId
             : base.activeStepId === event.stepId
               ? null
               : base.activeStepId,
-        activeArtifactTableNamespace:
-          status === 'running'
+        activeArtifactTableNamespace: runTerminal
+          ? (base.activeArtifactTableNamespace ?? null)
+          : status === 'running'
             ? (nextStep.artifactTableNamespace ?? null)
             : base.activeStepId === event.stepId
               ? null
@@ -942,6 +1082,7 @@ export function reducePlayRunLedgerEvent(
     case 'step.failed':
     case 'step.skipped': {
       const current = base.stepsById[event.stepId];
+      const runTerminal = isTerminalPlayRunLedgerStatus(base.status);
       const preserveFailedStatus =
         current?.status === 'failed' && event.type !== 'step.failed';
       const status = preserveFailedStatus
@@ -983,10 +1124,12 @@ export function reducePlayRunLedgerEvent(
         ...base,
         orderedStepIds: appendOrderedStepId(base, event.stepId),
         stepsById: { ...base.stepsById, [event.stepId]: nextStep },
-        activeStepId,
-        activeArtifactTableNamespace: activeStepId
-          ? (base.stepsById[activeStepId]?.artifactTableNamespace ?? null)
-          : null,
+        activeStepId: runTerminal ? base.activeStepId : activeStepId,
+        activeArtifactTableNamespace: runTerminal
+          ? (base.activeArtifactTableNamespace ?? null)
+          : activeStepId
+            ? (base.stepsById[activeStepId]?.artifactTableNamespace ?? null)
+            : null,
       });
     }
     case 'sheet.summary.updated':

package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts

@@ -0,0 +1,45 @@
+import type { PlayRunLedgerEventSource } from './run-ledger';
+
+const PLAY_RUN_LEDGER_EVENT_SOURCES: readonly PlayRunLedgerEventSource[] = [
+  'worker',
+  'temporal',
+  'convex',
+  'coordinator',
+  'system',
+];
+
+/**
+ * Run terminal source state machine.
+ *
+ * Coordinator terminals are transport heals from a live cache and stay
+ * provisional. Producer terminals are authoritative and must win over heals.
+ */
+export function normalizePlayRunLedgerTerminalSource(
+  value: unknown,
+): PlayRunLedgerEventSource | null {
+  return PLAY_RUN_LEDGER_EVENT_SOURCES.includes(
+    value as PlayRunLedgerEventSource,
+  )
+    ? (value as PlayRunLedgerEventSource)
+    : null;
+}
+
+export function isAuthoritativePlayRunTerminalSource(
+  value: unknown,
+): value is Exclude<PlayRunLedgerEventSource, 'coordinator'> {
+  const source = normalizePlayRunLedgerTerminalSource(value);
+  return source !== null && source !== 'coordinator';
+}
+
+export function nextPlayRunLedgerTerminalSource(input: {
+  currentSource: unknown;
+  eventSource: PlayRunLedgerEventSource;
+}): PlayRunLedgerEventSource {
+  if (input.eventSource !== 'coordinator') {
+    return input.eventSource;
+  }
+  const current = normalizePlayRunLedgerTerminalSource(input.currentSource);
+  return isAuthoritativePlayRunTerminalSource(current)
+    ? current
+    : 'coordinator';
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts

@@ -101,6 +101,7 @@ export type RuntimeReceiptAction =
       key: string;
       reclaimRunning?: boolean;
       forceRefresh?: boolean;
+      forceFailedRefresh?: boolean;
     }
   | {
       action: 'complete_runtime_step_receipt';
@@ -109,6 +110,14 @@ export type RuntimeReceiptAction =
       key: string;
       output: unknown;
     }
+  | {
+      action: 'release_runtime_step_receipt';
+      playName: string;
+      runId: string;
+      key: string;
+      runAttempt?: number;
+      leaseId?: string | null;
+    }
   | {
       action: 'fail_runtime_step_receipt';
       playName: string;

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts

@@ -0,0 +1,21 @@
+export const PLAY_RUNTIME_API_COMPAT_PATH = '/api/v2/plays/internal/runtime';
+export const PLAY_RUNTIME_API_CURRENT_PATH = '/api/v2/internal/play-runtime';
+export const PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH =
+  '/api/v2/plays/internal/child-executor-token';
+export const PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_CURRENT_PATH =
+  '/api/v2/internal/play-runtime/child-executor-token';
+export const PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH =
+  '/api/v2/plays/internal/tail-log';
+export const PLAY_RUNTIME_TAIL_LOG_CURRENT_PATH =
+  '/api/v2/internal/play-runtime/tail-log';
+export const PLAY_RUNTIME_EGRESS_FETCH_COMPAT_PATH =
+  '/api/v2/plays/internal/egress-fetch';
+export const PLAY_RUNTIME_EGRESS_FETCH_CURRENT_PATH =
+  '/api/v2/internal/play-runtime/egress-fetch';
+
+export const PLAY_RUNTIME_HARNESS_ALLOWED_PATH_PREFIXES = [
+  PLAY_RUNTIME_API_COMPAT_PATH,
+  PLAY_RUNTIME_API_CURRENT_PATH,
+  '/api/v2/plays/',
+  '/api/v2/integrations/',
+] as const;