deepline 0.1.243 → 0.1.245

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts

@@ -1,570 +0,0 @@
-import {
-  buildScopedWorkReceiptKey,
-  type WorkReceipt,
-  type WorkReceiptBatchMarkRunningCommand,
-  type WorkReceiptClaim,
-  type WorkReceiptCommand,
-  type WorkReceiptGetCommand,
-  type WorkReceiptStatus,
-} from '../../../../shared_libs/play-runtime/work-receipts';
-import type { ExecutionLedgerStore } from '../../../../shared_libs/play-runtime/execution-ledger-store';
-import {
-  PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
-  runtimeLeaseHeartbeatIntervalMs,
-} from '../../../../shared_libs/play-runtime/lease-policy';
-import {
-  isPlayExecutionSuspendedError,
-  isPlayRowExecutionSuspendedError,
-} from '../../../../shared_libs/play-runtime/suspension';
-import { runtimeReceiptFailureKindForError } from '../../../../shared_libs/play-runtime/durable-receipt-execution';
-import type { WorkerWorkBudgetMeter } from './work-budget';
-
-export type RuntimeReceiptStatus = WorkReceiptStatus;
-
-export type WorkerRuntimeReceipt = WorkReceipt;
-
-export type WorkerRuntimeReceiptCommand = WorkReceiptCommand;
-export type WorkerRuntimeReceiptGetCommand = WorkReceiptGetCommand;
-
-export type WorkerRuntimeReceiptClaim = WorkReceiptClaim;
-
-export type WorkerRuntimeReceiptStore = Pick<
-  ExecutionLedgerStore,
-  'claimReceipt' | 'completeReceipt' | 'failReceipt'
-> &
-  Partial<
-    Pick<
-      ExecutionLedgerStore,
-      | 'getReceipt'
-      | 'claimReceipts'
-      | 'completeReceipts'
-      | 'failReceipts'
-      | 'releaseReceipt'
-      | 'heartbeatReceipts'
-    >
-  > & {
-    markReceiptsRunning?: (
-      input: WorkReceiptBatchMarkRunningCommand,
-    ) => Promise<Array<WorkReceipt | null>>;
-  };
-
-export type WorkerRuntimeReceiptMarkRunningCommand =
-  WorkReceiptBatchMarkRunningCommand;
-
-type RuntimeReceiptContext = {
-  orgId?: string | null;
-  playName: string;
-  runId: string;
-  runAttempt?: number | null;
-  key: string;
-  receiptStore: WorkerRuntimeReceiptStore;
-  budgetMeter?: WorkerWorkBudgetMeter;
-};
-
-export type WorkerRuntimeReceiptExecutionContext = {
-  leaseId: string | null;
-};
-
-type WorkerRuntimeReceiptExecute<T> = (
-  context: WorkerRuntimeReceiptExecutionContext,
-  wasFailed: boolean,
-) => Promise<T> | T;
-
-const WORKER_RECEIPT_DEFAULT_WAIT_MS = 300_000;
-const WORKER_RECEIPT_WAIT_DELAY_MS = 250;
-const WORKER_RECEIPT_WAIT_MAX_ATTEMPTS = Math.ceil(
-  WORKER_RECEIPT_DEFAULT_WAIT_MS / WORKER_RECEIPT_WAIT_DELAY_MS,
-);
-const WORKER_RECEIPT_HEARTBEAT_INTERVAL_MS = runtimeLeaseHeartbeatIntervalMs(
-  PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
-);
-
-class RuntimeReceiptWaitTimeoutError extends Error {
-  constructor(key: string) {
-    super(`Timed out waiting for runtime receipt ${key}.`);
-    this.name = 'RuntimeReceiptWaitTimeoutError';
-  }
-}
-
-export class RuntimeLeaseLostError extends Error {
-  readonly keys: string[];
-  readonly runId: string;
-  readonly leaseId: string;
-
-  constructor(input: {
-    keys: string[];
-    runId: string;
-    leaseId: string;
-    cause?: unknown;
-  }) {
-    const keySummary =
-      input.keys.length === 1
-        ? input.keys[0]!
-        : `${input.keys.length} receipts`;
-    super(
-      `Runtime receipt ${keySummary} lost lease ${input.leaseId} before execution finished.`,
-      { cause: input.cause },
-    );
-    this.name = 'RuntimeLeaseLostError';
-    this.keys = input.keys;
-    this.runId = input.runId;
-    this.leaseId = input.leaseId;
-  }
-}
-
-function scopedReceiptKey(input: {
-  orgId?: string | null;
-  playName: string;
-  key: string;
-}): string {
-  const orgId = input.orgId?.trim() || 'org';
-  if (input.key.startsWith(`ctx:${orgId}:`)) {
-    return input.key;
-  }
-  return buildScopedWorkReceiptKey(input);
-}
-
-function receiptOutput<T>(receipt: WorkerRuntimeReceipt): T {
-  return receipt.output as T;
-}
-
-function errorMessage(error: unknown): string {
-  return error instanceof Error ? error.message : String(error);
-}
-
-async function waitForReusableReceipt<T>(input: {
-  key: string;
-  playName: string;
-  receiptStore: WorkerRuntimeReceiptStore;
-  maxAttempts?: number;
-  delayMs?: number;
-}): Promise<T> {
-  if (!input.receiptStore.getReceipt) {
-    throw new Error(
-      `Runtime receipt ${input.key} wait requires read-only receipt lookup.`,
-    );
-  }
-  const maxAttempts = input.maxAttempts ?? WORKER_RECEIPT_WAIT_MAX_ATTEMPTS;
-  const delayMs = input.delayMs ?? WORKER_RECEIPT_WAIT_DELAY_MS;
-  for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
-    if (attempt > 0) {
-      await new Promise((resolve) => setTimeout(resolve, delayMs));
-    }
-    const receipt = await input.receiptStore.getReceipt({
-      playName: input.playName,
-      key: input.key,
-    });
-    if (receipt?.status === 'completed' || receipt?.status === 'skipped') {
-      return receiptOutput<T>(receipt);
-    }
-    if (receipt?.status === 'failed') {
-      throw new Error(
-        `Runtime receipt ${input.key} failed while waiting: ${receipt.error ?? 'unknown error'}`,
-      );
-    }
-  }
-  throw new RuntimeReceiptWaitTimeoutError(input.key);
-}
-
-type WorkerRuntimeReceiptHeartbeatTarget = {
-  key: string | null | undefined;
-  leaseId: string | null | undefined;
-};
-
-function heartbeatGroups(
-  targets: WorkerRuntimeReceiptHeartbeatTarget[],
-): Array<{ leaseId: string; keys: string[] }> {
-  const byLeaseId = new Map<string, string[]>();
-  for (const target of targets) {
-    const key = target.key?.trim();
-    const leaseId = target.leaseId?.trim();
-    if (!key || !leaseId) continue;
-    const keys = byLeaseId.get(leaseId);
-    if (keys) {
-      keys.push(key);
-    } else {
-      byLeaseId.set(leaseId, [key]);
-    }
-  }
-  return Array.from(byLeaseId, ([leaseId, keys]) => ({ leaseId, keys }));
-}
-
-function buildLeaseLostError(input: {
-  keys: string[];
-  runId: string;
-  leaseId: string;
-  cause?: unknown;
-}): RuntimeLeaseLostError {
-  return new RuntimeLeaseLostError(input);
-}
-
-function receiptHeartbeatStillOwnsOrSettled(input: {
-  receipt: WorkerRuntimeReceipt | null | undefined;
-  leaseId: string;
-}): boolean {
-  const { receipt } = input;
-  if (!receipt) return false;
-  if (receipt.status === 'completed' || receipt.status === 'skipped') {
-    return true;
-  }
-  if (receipt.status !== 'running' && receipt.status !== 'queued') {
-    return false;
-  }
-  return receipt.leaseId === input.leaseId;
-}
-
-export async function executeWithWorkerReceiptHeartbeats<T>(input: {
-  playName: string;
-  runId: string;
-  runAttempt?: number | null;
-  receiptStore?: WorkerRuntimeReceiptStore | null;
-  budgetMeter?: WorkerWorkBudgetMeter;
-  targets: WorkerRuntimeReceiptHeartbeatTarget[];
-  heartbeatIntervalMs?: number;
-  execute: () => Promise<T> | T;
-}): Promise<T> {
-  const groups = heartbeatGroups(input.targets);
-  const heartbeatReceipts = input.receiptStore?.heartbeatReceipts;
-  if (groups.length === 0 || !heartbeatReceipts) {
-    return await input.execute();
-  }
-
-  const heartbeatIntervalMs =
-    input.heartbeatIntervalMs ?? WORKER_RECEIPT_HEARTBEAT_INTERVAL_MS;
-  if (!Number.isFinite(heartbeatIntervalMs) || heartbeatIntervalMs <= 0) {
-    throw new Error('Runtime receipt heartbeat interval must be positive.');
-  }
-
-  let stopped = false;
-  let timeout: ReturnType<typeof setTimeout> | null = null;
-  let rejectLeaseLost!: (error: RuntimeLeaseLostError) => void;
-  const leaseLost = new Promise<never>((_, reject) => {
-    rejectLeaseLost = reject;
-  });
-
-  const stop = () => {
-    stopped = true;
-    if (timeout !== null) {
-      clearTimeout(timeout);
-      timeout = null;
-    }
-  };
-
-  const heartbeatOnce = async () => {
-    for (const group of groups) {
-      input.budgetMeter?.count('heartbeat');
-      const receipts = await heartbeatReceipts({
-        playName: input.playName,
-        runId: input.runId,
-        ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-        leaseId: group.leaseId,
-        keys: group.keys,
-      });
-      for (let index = 0; index < group.keys.length; index += 1) {
-        const receipt = receipts[index];
-        if (
-          !receiptHeartbeatStillOwnsOrSettled({
-            receipt,
-            leaseId: group.leaseId,
-          })
-        ) {
-          throw buildLeaseLostError({
-            keys: group.keys,
-            runId: input.runId,
-            leaseId: group.leaseId,
-          });
-        }
-      }
-    }
-  };
-
-  const schedule = () => {
-    timeout = setTimeout(() => {
-      void (async () => {
-        try {
-          await heartbeatOnce();
-        } catch (error) {
-          stop();
-          if (error instanceof RuntimeLeaseLostError) {
-            rejectLeaseLost(error);
-          } else {
-            rejectLeaseLost(
-              buildLeaseLostError({
-                keys: groups.flatMap((group) => group.keys),
-                runId: input.runId,
-                leaseId: groups[0]?.leaseId ?? 'unknown',
-                cause: error,
-              }),
-            );
-          }
-          return;
-        }
-        if (!stopped) {
-          schedule();
-        }
-      })();
-    }, heartbeatIntervalMs);
-  };
-
-  await heartbeatOnce();
-  schedule();
-  const execution = Promise.resolve().then(input.execute);
-  execution.catch(() => {});
-  try {
-    return await Promise.race([execution, leaseLost]);
-  } finally {
-    stop();
-  }
-}
-
-function shouldRepairSameRunRunningReceipt(input: {
-  repairRunningReceiptForSameRun?: boolean;
-  currentRunId: string;
-  receipt: WorkerRuntimeReceipt;
-}): boolean {
-  return (
-    input.repairRunningReceiptForSameRun === true &&
-    input.receipt.status === 'running' &&
-    typeof input.receipt.runId === 'string' &&
-    input.receipt.runId.trim() === input.currentRunId
-  );
-}
-
-async function executeAndPersistReceipt<T>(input: {
-  key: string;
-  playName: string;
-  runId: string;
-  runAttempt?: number | null;
-  leaseId?: string | null;
-  wasFailed?: boolean;
-  execute: WorkerRuntimeReceiptExecute<T>;
-  receiptStore: WorkerRuntimeReceiptStore;
-  ownership: 'claimed' | 'reconciled';
-  budgetMeter?: WorkerWorkBudgetMeter;
-  heartbeatIntervalMs?: number;
-}): Promise<T> {
-  let output: T;
-  try {
-    const leaseId = input.leaseId?.trim() || null;
-    output = await executeWithWorkerReceiptHeartbeats({
-      playName: input.playName,
-      runId: input.runId,
-      ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-      receiptStore: input.receiptStore,
-      budgetMeter: input.budgetMeter,
-      heartbeatIntervalMs: input.heartbeatIntervalMs,
-      targets: [{ key: input.key, leaseId }],
-      execute: () => input.execute({ leaseId }, input.wasFailed === true),
-    });
-  } catch (error) {
-    if (error instanceof RuntimeLeaseLostError) {
-      throw error;
-    }
-    if (
-      isPlayExecutionSuspendedError(error) ||
-      isPlayRowExecutionSuspendedError(error)
-    ) {
-      if (!input.receiptStore.releaseReceipt) {
-        throw new Error(
-          `Runtime receipt ${input.key} suspended but the receipt store cannot release ownership: ${errorMessage(error)}`,
-        );
-      }
-      const released = await input.receiptStore.releaseReceipt({
-        playName: input.playName,
-        runId: input.runId,
-        ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-        key: input.key,
-        ...(input.leaseId ? { leaseId: input.leaseId } : {}),
-      });
-      if (!released || released.status !== 'pending') {
-        throw new Error(
-          `Runtime receipt ${input.key} suspended but failed to release ownership: ${errorMessage(error)}`,
-        );
-      }
-      throw error;
-    }
-    const failed = await input.receiptStore.failReceipt({
-      playName: input.playName,
-      runId: input.runId,
-      ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-      key: input.key,
-      ...(input.leaseId ? { leaseId: input.leaseId } : {}),
-      error: errorMessage(error),
-      failureKind: runtimeReceiptFailureKindForError(error),
-    });
-    if (!failed) {
-      throw new Error(
-        `Runtime receipt ${input.key} ${input.ownership} execution failed and failed receipt could not be persisted: ${errorMessage(error)}`,
-      );
-    }
-    throw error;
-  }
-
-  const completed = await input.receiptStore.completeReceipt({
-    playName: input.playName,
-    runId: input.runId,
-    ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-    key: input.key,
-    ...(input.leaseId ? { leaseId: input.leaseId } : {}),
-    output,
-  });
-  if (
-    !completed ||
-    (completed.status !== 'completed' && completed.status !== 'skipped')
-  ) {
-    throw new Error(
-      `Runtime receipt ${input.key} lost receipt ownership before completion.`,
-    );
-  }
-  if (completed.output !== undefined) {
-    return receiptOutput<T>(completed);
-  }
-  return output;
-}
-
-export async function runWorkerRuntimeReceiptBoundary<T>(
-  input: RuntimeReceiptContext & {
-    execute: WorkerRuntimeReceiptExecute<T>;
-    repairRunningReceiptForSameRun?: boolean;
-    repairRunningReceiptForSameRunAfterWaitTimeout?: boolean;
-    runningReceiptWaitMaxAttempts?: number;
-    runningReceiptWaitDelayMs?: number;
-    runtimeReceiptHeartbeatIntervalMs?: number;
-    runtimeReceiptLeaseTtlMs?: number;
-    reclaimRunning?: boolean;
-    forceFailedRefresh?: boolean;
-  },
-): Promise<T> {
-  const key = scopedReceiptKey(input);
-  const receiptStore = input.receiptStore;
-  const claimed = await receiptStore.claimReceipt({
-    playName: input.playName,
-    runId: input.runId,
-    ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-    key,
-    leaseAware: true,
-    leaseTtlMs: input.runtimeReceiptLeaseTtlMs,
-    ...(input.reclaimRunning === true ? { reclaimRunning: true } : {}),
-    ...(input.forceFailedRefresh === true ? { forceFailedRefresh: true } : {}),
-  });
-  if (claimed.disposition === 'reused') {
-    return receiptOutput<T>(claimed.receipt);
-  }
-  if (
-    claimed.disposition === 'running' ||
-    claimed.disposition === 'blocked_active_lease'
-  ) {
-    if (
-      shouldRepairSameRunRunningReceipt({
-        repairRunningReceiptForSameRun: input.repairRunningReceiptForSameRun,
-        currentRunId: input.runId,
-        receipt: claimed.receipt,
-      })
-    ) {
-      const repaired = await receiptStore.claimReceipt({
-        playName: input.playName,
-        runId: input.runId,
-        ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-        key,
-        leaseAware: true,
-        leaseTtlMs: input.runtimeReceiptLeaseTtlMs,
-        reclaimRunning: true,
-      });
-      if (repaired.disposition === 'reused') {
-        return receiptOutput<T>(repaired.receipt);
-      }
-      if (repaired.disposition === 'failed') {
-        throw new Error(
-          `Runtime receipt ${key} is failed and could not be repaired: ${repaired.receipt.error ?? 'unknown error'}`,
-        );
-      }
-      if (repaired.disposition === 'claimed') {
-        return executeAndPersistReceipt({
-          key,
-          playName: input.playName,
-          runId: input.runId,
-          ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-          leaseId: repaired.receipt.leaseId ?? null,
-          execute: input.execute,
-          receiptStore,
-          ownership: 'claimed',
-          budgetMeter: input.budgetMeter,
-          heartbeatIntervalMs: input.runtimeReceiptHeartbeatIntervalMs,
-        });
-      }
-    }
-    try {
-      return await waitForReusableReceipt<T>({
-        key,
-        playName: input.playName,
-        receiptStore,
-        maxAttempts: input.runningReceiptWaitMaxAttempts,
-        delayMs: input.runningReceiptWaitDelayMs,
-      });
-    } catch (error) {
-      if (
-        input.repairRunningReceiptForSameRunAfterWaitTimeout === true &&
-        error instanceof RuntimeReceiptWaitTimeoutError &&
-        shouldRepairSameRunRunningReceipt({
-          repairRunningReceiptForSameRun: true,
-          currentRunId: input.runId,
-          receipt: claimed.receipt,
-        })
-      ) {
-        const repaired = await receiptStore.claimReceipt({
-          playName: input.playName,
-          runId: input.runId,
-          ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-          key,
-          leaseAware: true,
-          leaseTtlMs: input.runtimeReceiptLeaseTtlMs,
-          reclaimRunning: true,
-        });
-        if (repaired.disposition === 'reused') {
-          return receiptOutput<T>(repaired.receipt);
-        }
-        if (repaired.disposition === 'failed') {
-          throw new Error(
-            `Runtime receipt ${key} is failed and could not be repaired after wait timeout: ${repaired.receipt.error ?? 'unknown error'}`,
-          );
-        }
-        if (repaired.disposition === 'claimed') {
-          return executeAndPersistReceipt({
-            key,
-            playName: input.playName,
-            runId: input.runId,
-            ...(input.runAttempt != null
-              ? { runAttempt: input.runAttempt }
-              : {}),
-            leaseId: repaired.receipt.leaseId ?? null,
-            execute: input.execute,
-            receiptStore,
-            ownership: 'claimed',
-            budgetMeter: input.budgetMeter,
-            heartbeatIntervalMs: input.runtimeReceiptHeartbeatIntervalMs,
-          });
-        }
-      }
-      throw error;
-    }
-  }
-  if (claimed.disposition === 'failed') {
-    throw new Error(
-      `Runtime receipt ${key} is failed and could not be claimed: ${claimed.receipt.error ?? 'unknown error'}`,
-    );
-  }
-
-  return executeAndPersistReceipt({
-    key,
-    playName: input.playName,
-    runId: input.runId,
-    ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
-    leaseId: claimed.receipt.leaseId ?? null,
-    wasFailed: claimed.wasFailed === true,
-    execute: input.execute,
-    receiptStore,
-    ownership: 'claimed',
-    budgetMeter: input.budgetMeter,
-    heartbeatIntervalMs: input.runtimeReceiptHeartbeatIntervalMs,
-  });
-}

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/result-dataset-persistence.ts

@@ -1,50 +0,0 @@
-import { augmentSheetContractWithDatasetFields } from '../../../../shared_libs/play-data-plane/sheet-contract';
-import type { PlaySheetContract } from '../../../../shared_libs/plays/static-pipeline';
-import type { OutputDatasetRow } from './output-datasets';
-
-export type StableResultDatasetPersistenceContract = {
-  sheetContract: PlaySheetContract;
-  cachedChunks: OutputDatasetRow[][] | null;
-  rowCount: number;
-};
-
-export async function collectStableResultDatasetPersistenceContract(input: {
-  contract: PlaySheetContract;
-  chunks:
-    | AsyncIterable<readonly OutputDatasetRow[]>
-    | Iterable<readonly OutputDatasetRow[]>;
-  cacheChunks?: boolean;
-}): Promise<StableResultDatasetPersistenceContract> {
-  const fields = new Set<string>();
-  const cachedChunks: OutputDatasetRow[][] | null = input.cacheChunks
-    ? []
-    : null;
-  let rowCount = 0;
-
-  for await (const chunk of input.chunks) {
-    if (chunk.length === 0) continue;
-    rowCount += chunk.length;
-    for (const row of chunk) {
-      for (const field of Object.keys(row)) {
-        fields.add(field);
-      }
-    }
-    if (cachedChunks) {
-      cachedChunks.push(chunk.map((row) => ({ ...row })));
-    }
-  }
-
-  const unionRow =
-    fields.size === 0
-      ? []
-      : [Object.fromEntries([...fields].map((field) => [field, null]))];
-
-  return {
-    sheetContract: augmentSheetContractWithDatasetFields({
-      contract: input.contract,
-      rows: unionRow,
-    }),
-    cachedChunks,
-    rowCount,
-  };
-}

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts

@@ -1,5 +0,0 @@
-export {
-  WorkflowAbortError,
-  isAbortLikeError,
-  isRowIsolationExemptError,
-} from '../../../../shared_libs/play-runtime/row-isolation';