deepline 0.1.182 → 0.1.184

package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts

@@ -7,6 +7,15 @@ import {
 import { assertNoSecretTaint } from './secret-capability';
 import type { RuntimeStepReceipt } from './ctx-types';
 import type { DurableReceiptRecoverySource } from './tool-execution-outcome';
+import {
+  isPlayExecutionSuspendedError,
+  isPlayRowExecutionSuspendedError,
+} from './suspension';
+import type { WorkReceiptFailureKind } from './work-receipts';
+import {
+  PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
+  runtimeLeaseHeartbeatIntervalMs,
+} from './lease-policy';
 
 const DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS = 240;
 const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
@@ -21,6 +30,30 @@ export class RuntimeReceiptWaitTimeoutError extends Error {
   }
 }
 
+export class RuntimeReceiptLeaseLostError extends Error {
+  constructor(input: { receiptKey: string; runId: string; leaseId: string }) {
+    super(
+      `Lost durable receipt lease for ${input.receiptKey}. runId=${input.runId} leaseId=${input.leaseId}`,
+    );
+    this.name = 'RuntimeReceiptLeaseLostError';
+  }
+}
+
+function isInFlightRuntimeReceipt(
+  receipt: RuntimeStepReceipt | null | undefined,
+): receipt is RuntimeStepReceipt & { status: 'pending' | 'running' } {
+  return receipt?.status === 'pending' || receipt?.status === 'running';
+}
+
+export function runtimeReceiptFailureKindForError(
+  error: unknown,
+): WorkReceiptFailureKind {
+  return isPlayExecutionSuspendedError(error) ||
+    isPlayRowExecutionSuspendedError(error)
+    ? 'repairable'
+    : 'terminal';
+}
+
 export function resolveRuntimeToolReceiptWaitTimeoutMs(
   requestInput: Record<string, unknown>,
 ): number {
@@ -102,21 +135,107 @@ export type DurableReceiptExecutionStore = {
     runId: string,
     reclaimRunning?: boolean,
     forceRefresh?: boolean,
+    forceFailedRefresh?: boolean,
   ): Promise<RuntimeStepReceipt | null>;
   complete(
     receiptKey: string,
     runId: string,
     output: unknown | null,
+    leaseId?: string | null,
+  ): Promise<RuntimeStepReceipt | null>;
+  release(
+    receiptKey: string,
+    runId: string,
+    leaseId?: string | null,
+  ): Promise<RuntimeStepReceipt | null>;
+  heartbeat?(
+    receiptKey: string,
+    runId: string,
+    leaseId: string,
   ): Promise<RuntimeStepReceipt | null>;
   fail(
     receiptKey: string,
     runId: string,
     error: string,
+    leaseId?: string | null,
+    failureKind?: WorkReceiptFailureKind | null,
   ): Promise<RuntimeStepReceipt | null>;
   canPersistFailure: boolean;
   canPersistCompletion: boolean;
 };
 
+async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
+  receiptKey: string;
+  runId: string;
+  leaseId: string;
+  heartbeatIntervalMs?: number;
+  store: DurableReceiptExecutionStore;
+  execute: () => Promise<T>;
+}): Promise<T> {
+  if (!input.store.heartbeat) {
+    return await input.execute();
+  }
+  const heartbeatIntervalMs =
+    input.heartbeatIntervalMs ??
+    runtimeLeaseHeartbeatIntervalMs(PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS);
+  if (!Number.isFinite(heartbeatIntervalMs) || heartbeatIntervalMs <= 0) {
+    throw new Error('Runtime receipt heartbeat interval must be positive.');
+  }
+
+  let stopped = false;
+  let timeout: ReturnType<typeof setTimeout> | null = null;
+  let rejectLeaseLost!: (error: RuntimeReceiptLeaseLostError) => void;
+  const leaseLost = new Promise<never>((_, reject) => {
+    rejectLeaseLost = reject;
+  });
+  const stop = () => {
+    stopped = true;
+    if (timeout !== null) {
+      clearTimeout(timeout);
+      timeout = null;
+    }
+  };
+  const heartbeatOnce = async () => {
+    const receipt = await input.store.heartbeat!(
+      input.receiptKey,
+      input.runId,
+      input.leaseId,
+    );
+    if (
+      !receipt ||
+      receipt.status !== 'running' ||
+      receipt.leaseId !== input.leaseId
+    ) {
+      throw new RuntimeReceiptLeaseLostError(input);
+    }
+  };
+  const schedule = () => {
+    timeout = setTimeout(() => {
+      void (async () => {
+        try {
+          await heartbeatOnce();
+          if (!stopped) schedule();
+        } catch (error) {
+          stop();
+          rejectLeaseLost(
+            error instanceof RuntimeReceiptLeaseLostError
+              ? error
+              : new RuntimeReceiptLeaseLostError(input),
+          );
+        }
+      })();
+    }, heartbeatIntervalMs);
+  };
+
+  await heartbeatOnce();
+  schedule();
+  try {
+    return await Promise.race([input.execute(), leaseLost]);
+  } finally {
+    stop();
+  }
+}
+
 export function runtimeReceiptOutput<T>(receipt: RuntimeStepReceipt): T {
   return isSerializedToolExecuteResult(receipt.output)
     ? (deserializeToolExecuteResult(receipt.output) as T)
@@ -171,12 +290,14 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     source: DurableReceiptRecoverySource,
   ) => T;
   onClaimedResult?: (output: T, receiptKey: string) => T;
+  shouldPersistFailure?: (error: unknown) => boolean;
+  heartbeatIntervalMs?: number;
   formatError: (error: unknown) => string;
   log: (message: string) => void;
-  execute: () => Promise<T>;
+  execute: (context: { leaseId: string | null }) => Promise<T>;
 }): Promise<T> {
   if (!input.store.enabled) {
-    return await input.execute();
+    return await input.execute({ leaseId: null });
   }
 
   const recoverCompletedReceipt = async (
@@ -199,7 +320,9 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     return recovered;
   };
 
-  const reclaimRunningReceipt = async (): Promise<
+  let ownedLeaseId: string | null = null;
+
+  const reclaimReceipt = async (): Promise<
     { kind: 'recovered'; output: T } | { kind: 'claimed' }
   > => {
     const reclaimed = await input.store.claim(
@@ -219,21 +342,21 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
       );
     }
     if (
-      reclaimed?.status === 'running' &&
+      isInFlightRuntimeReceipt(reclaimed) &&
       reclaimed.claimState === 'existing'
     ) {
       throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
     }
+    ownedLeaseId = reclaimed?.leaseId ?? null;
     return { kind: 'claimed' };
   };
   const shouldRepairSameRunRunningReceipt = (
     receipt: RuntimeStepReceipt,
   ): boolean =>
     input.repairRunningReceiptForSameRun === true &&
-    receipt.status === 'running' &&
+    isInFlightRuntimeReceipt(receipt) &&
     typeof receipt.runId === 'string' &&
     receipt.runId.trim() === input.runId;
-
   const waitForRunningReceipt = async (): Promise<{
     kind: 'recovered';
     output: T;
@@ -257,7 +380,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     receipt: RuntimeStepReceipt,
   ): Promise<{ kind: 'recovered'; output: T } | { kind: 'claimed' }> => {
     if (shouldRepairSameRunRunningReceipt(receipt)) {
-      const recovered = await reclaimRunningReceipt();
+      const recovered = await reclaimReceipt();
       if (recovered.kind === 'recovered') return recovered;
       return { kind: 'claimed' };
     }
@@ -265,13 +388,20 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
       return await waitForRunningReceiptOrTimeout();
     } catch (error) {
       if (error instanceof RuntimeReceiptWaitTimeoutError) {
-        const recovered = await reclaimRunningReceipt();
+        const recovered = await reclaimReceipt();
         if (recovered.kind === 'recovered') return recovered;
         return { kind: 'claimed' };
       }
       throw error;
     }
   };
+  const repairFailedReceipt = async (
+    receipt: RuntimeStepReceipt,
+  ): Promise<never> => {
+    throw new Error(
+      `ctx.${input.operation}(${input.id}): receipt is failed and could not be claimed: ${receipt.error ?? 'unknown error'}.`,
+    );
+  };
 
   const claimed = await input.store.claim(
     input.receiptKey,
@@ -290,42 +420,70 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     if (latest?.status === 'completed' || latest?.status === 'skipped') {
       return await recoverCompletedReceipt(latest);
     }
-    if (latest?.status === 'running') {
+    if (isInFlightRuntimeReceipt(latest)) {
       const recovered = await repairOrWaitForRunningReceipt(latest);
       if (recovered.kind === 'recovered') return recovered.output;
+    } else if (latest?.status === 'failed') {
+      await repairFailedReceipt(latest);
     } else {
-      if (latest?.status === 'failed') {
-        throw new Error(
-          `ctx.${input.operation}(${input.id}): receipt is failed and could not be claimed: ${latest.error ?? 'unknown error'}.`,
-        );
-      }
       throw new Error(
         `ctx.${input.operation}(${input.id}): receipt claim did not return execution ownership.`,
       );
     }
-  } else if (claimed.status === 'running') {
+  } else if (isInFlightRuntimeReceipt(claimed)) {
     if (claimed.claimState === 'existing') {
       const recovered = await repairOrWaitForRunningReceipt(claimed);
       if (recovered.kind === 'recovered') return recovered.output;
+    } else {
+      ownedLeaseId = claimed.leaseId ?? null;
     }
   } else if (claimed.status === 'failed') {
-    throw new Error(
-      `ctx.${input.operation}(${input.id}): receipt is failed and could not be claimed: ${claimed.error ?? 'unknown error'}.`,
-    );
+    await repairFailedReceipt(claimed);
   }
 
   let result: T;
   try {
-    const executed = await input.execute();
+    const executed =
+      ownedLeaseId && input.store.heartbeat
+        ? await executeWithDurableRuntimeReceiptHeartbeat({
+            receiptKey: input.receiptKey,
+            runId: input.runId,
+            leaseId: ownedLeaseId,
+            heartbeatIntervalMs: input.heartbeatIntervalMs,
+            store: input.store,
+            execute: () => input.execute({ leaseId: ownedLeaseId }),
+          })
+        : await input.execute({ leaseId: ownedLeaseId });
     result = input.onClaimedResult
       ? input.onClaimedResult(executed, input.receiptKey)
       : executed;
     assertNoSecretTaint(result, `ctx.${input.operation} result`);
   } catch (error) {
+    if (input.shouldPersistFailure?.(error) === false) {
+      throw error;
+    }
+    if (
+      isPlayExecutionSuspendedError(error) ||
+      isPlayRowExecutionSuspendedError(error)
+    ) {
+      const released = await input.store.release(
+        input.receiptKey,
+        input.runId,
+        ownedLeaseId,
+      );
+      if (!released || released.status !== 'pending') {
+        throw new Error(
+          `ctx.${input.operation}(${input.id}): receipt suspended but ownership could not be released: ${input.formatError(error)}.`,
+        );
+      }
+      throw error;
+    }
     const failed = await input.store.fail(
       input.receiptKey,
       input.runId,
       input.formatError(error),
+      ownedLeaseId,
+      runtimeReceiptFailureKindForError(error),
     );
     if (!failed && input.store.canPersistFailure) {
       throw new Error(
@@ -339,9 +497,12 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     input.receiptKey,
     input.runId,
     isToolExecuteResult(result) ? serializeToolExecuteResult(result) : result,
+    ownedLeaseId,
   );
   if (!completed && input.store.canPersistCompletion) {
-    return result;
+    throw new Error(
+      `ctx.${input.operation}(${input.id}): lost durable receipt ownership before completion.`,
+    );
   }
   if (
     completed &&
@@ -349,5 +510,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
   ) {
     return await recoverCompletedReceipt(completed, 'owner');
   }
+  if (input.store.canPersistCompletion) {
+    throw new Error(
+      `ctx.${input.operation}(${input.id}): lost durable receipt ownership before completion.`,
+    );
+  }
   return result;
 }

package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts

@@ -0,0 +1,2 @@
+export const DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION =
+  'h24-map-dispatch-batch-metadata';

package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts

@@ -0,0 +1,133 @@
+import type {
+  WorkReceipt,
+  WorkReceiptBatchClaimCommand,
+  WorkReceiptBatchCompleteCommand,
+  WorkReceiptBatchFailCommand,
+  WorkReceiptBatchHeartbeatCommand,
+  WorkReceiptClaim,
+  WorkReceiptCommand,
+  WorkReceiptFailureKind,
+  WorkReceiptGetCommand,
+} from './work-receipts';
+
+export type WorkReceiptBatchGetCommand = {
+  playName: string;
+  keys: string[];
+};
+
+export type WorkReceiptCompleteCommand = WorkReceiptCommand & {
+  output: unknown;
+};
+
+export type WorkReceiptFailCommand = WorkReceiptCommand & {
+  error: string;
+  failureKind?: WorkReceiptFailureKind | null;
+};
+
+export type WorkReceiptReleaseCommand = WorkReceiptCommand;
+
+export type WorkReceiptSkipCommand = WorkReceiptCommand & {
+  output?: unknown | null;
+};
+
+export type ProviderRequestStatus =
+  | 'pending'
+  | 'running'
+  | 'completed'
+  | 'failed';
+
+export type ProviderRequestRecord = {
+  key: string;
+  status: ProviderRequestStatus;
+  receiptKey: string;
+  providerIdempotencyKey: string;
+  runId: string;
+  provider: string;
+  operation: string;
+  method: string;
+  endpoint: string;
+  bodyHash: string;
+  output?: unknown;
+  error?: string | null;
+  createdAt?: string | null;
+  updatedAt?: string | null;
+};
+
+export type ProviderRequestClaim =
+  | {
+      disposition: 'claimed';
+      record: ProviderRequestRecord;
+    }
+  | {
+      disposition: 'reused';
+      record: ProviderRequestRecord;
+    }
+  | {
+      disposition: 'running';
+      record: ProviderRequestRecord;
+    }
+  | {
+      disposition: 'failed';
+      record: ProviderRequestRecord;
+    };
+
+export type ProviderRequestClaimCommand = {
+  key: string;
+  receiptKey: string;
+  providerIdempotencyKey: string;
+  runId: string;
+  provider: string;
+  operation: string;
+  method: string;
+  endpoint: string;
+  bodyHash: string;
+};
+
+export type ProviderRequestCompleteCommand = {
+  key: string;
+  runId: string;
+  output?: unknown;
+};
+
+export type ProviderRequestFailCommand = {
+  key: string;
+  runId: string;
+  error: string;
+};
+
+export interface ExecutionLedgerStore {
+  getReceipt(input: WorkReceiptGetCommand): Promise<WorkReceipt | null>;
+  getReceipts(
+    input: WorkReceiptBatchGetCommand,
+  ): Promise<Array<WorkReceipt | null>>;
+  claimReceipt(input: WorkReceiptCommand): Promise<WorkReceiptClaim>;
+  claimReceipts(
+    input: WorkReceiptBatchClaimCommand,
+  ): Promise<WorkReceiptClaim[]>;
+  completeReceipt(
+    input: WorkReceiptCompleteCommand,
+  ): Promise<WorkReceipt | null>;
+  completeReceipts(
+    input: WorkReceiptBatchCompleteCommand,
+  ): Promise<Array<WorkReceipt | null>>;
+  failReceipt(input: WorkReceiptFailCommand): Promise<WorkReceipt | null>;
+  failReceipts(
+    input: WorkReceiptBatchFailCommand,
+  ): Promise<Array<WorkReceipt | null>>;
+  releaseReceipt(
+    input: WorkReceiptReleaseCommand,
+  ): Promise<WorkReceipt | null>;
+  skipReceipt(input: WorkReceiptSkipCommand): Promise<WorkReceipt | null>;
+  heartbeatReceipts(
+    input: WorkReceiptBatchHeartbeatCommand,
+  ): Promise<Array<WorkReceipt | null>>;
+  claimProviderRequest(
+    input: ProviderRequestClaimCommand,
+  ): Promise<ProviderRequestClaim>;
+  completeProviderRequest(
+    input: ProviderRequestCompleteCommand,
+  ): Promise<ProviderRequestRecord | null>;
+  failProviderRequest(
+    input: ProviderRequestFailCommand,
+  ): Promise<ProviderRequestRecord | null>;
+}