deepline 0.1.181 → 0.1.183

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -8,12 +8,16 @@ import type {
   PlayDatasetInput,
   PlayDatasetRow,
 } from '../plays/dataset';
-import type { PlayQueueHint } from './governor/rate-state-backend';
+import type {
+  PlayQueueHint,
+  RateStateBackend,
+} from './governor/rate-state-backend';
 import type { GovernanceSnapshot } from './governor/governor';
 import type { AnyBatchOperationStrategy } from './batching-types';
 import type { ToolResultMetadataInput } from './tool-result';
 import type { PreviousCell } from './cell-staleness';
 import type { MapRowOutcome } from './durability-store';
+import type { WorkReceiptFailureKind } from './work-receipts';
 
 export interface RowState {
   results: Map<string, unknown>;
@@ -22,8 +26,12 @@ export interface RowState {
 export interface ToolCallRequest {
   callId: string;
   cacheKey: string;
+  cacheable?: boolean;
   receiptKey?: string | null;
+  executionAuthScopeDigest?: string | null;
+  receiptLeaseId?: string | null;
   force?: boolean;
+  forceFailedRefresh?: boolean;
   rowId: number;
   fieldName?: string;
   toolId: string;
@@ -45,7 +53,12 @@ export interface RuntimeStepReceipt {
   status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
   output?: unknown;
   error?: string;
+  failureKind?: WorkReceiptFailureKind | null;
   runId?: string | null;
+  leaseId?: string | null;
+  leaseOwnerRunId?: string | null;
+  leaseOwnerAttempt?: number | null;
+  leaseExpiresAt?: string | null;
   claimState?: 'claimed' | 'existing';
 }
 
@@ -56,25 +69,42 @@ export interface GetRuntimeStepReceiptInput {
 export interface ClaimRuntimeStepReceiptInput {
   key: string;
   runId: string;
+  runAttempt?: number | null;
+  leaseAware?: boolean;
   reclaimRunning?: boolean;
   forceRefresh?: boolean;
+  forceFailedRefresh?: boolean;
 }
 
 export interface CompleteRuntimeStepReceiptInput {
   key: string;
   runId: string;
+  runAttempt?: number | null;
+  leaseId?: string | null;
   output: unknown | null;
 }
 
+export interface ReleaseRuntimeStepReceiptInput {
+  key: string;
+  runId: string;
+  runAttempt?: number | null;
+  leaseId?: string | null;
+}
+
 export interface FailRuntimeStepReceiptInput {
   key: string;
   runId: string;
+  runAttempt?: number | null;
+  leaseId?: string | null;
   error: string;
+  failureKind?: WorkReceiptFailureKind | null;
 }
 
 export interface SkipRuntimeStepReceiptInput {
   key: string;
   runId: string;
+  runAttempt?: number | null;
+  leaseId?: string | null;
   output?: unknown | null;
 }
 
@@ -85,14 +115,19 @@ export interface GetRuntimeStepReceiptsInput {
 export interface ClaimRuntimeStepReceiptsInput {
   keys: string[];
   runId: string;
+  runAttempt?: number | null;
+  leaseAware?: boolean;
   reclaimRunning?: boolean;
   forceRefresh?: boolean;
+  forceFailedRefresh?: boolean;
 }
 
 export interface CompleteRuntimeStepReceiptsInput {
   receipts: Array<{
     key: string;
     runId: string;
+    runAttempt?: number | null;
+    leaseId?: string | null;
     output: unknown | null;
   }>;
 }
@@ -101,10 +136,20 @@ export interface FailRuntimeStepReceiptsInput {
   receipts: Array<{
     key: string;
     runId: string;
+    runAttempt?: number | null;
+    leaseId?: string | null;
     error: string;
+    failureKind?: WorkReceiptFailureKind | null;
   }>;
 }
 
+export interface HeartbeatRuntimeStepReceiptsInput {
+  keys: string[];
+  runId: string;
+  runAttempt?: number | null;
+  leaseId: string;
+}
+
 export interface CsvOptions {
   description?: string;
   columns?: Record<string, string | readonly string[]>;
@@ -270,6 +315,10 @@ export interface BatchRequest {
 export interface MapStartResult {
   /** Rows that need processing for this run. Previous values may be included for context. */
   pendingRows: Record<string, unknown>[];
+  /** Rows already terminal for this run and safe to count as completed. */
+  completedRows?: Record<string, unknown>[];
+  /** Rows leased by another active attempt and not executable by this attempt. */
+  blockedRows?: Record<string, unknown>[];
   /** Resolved table namespace. */
   tableNamespace: string;
 }
@@ -402,6 +451,7 @@ export interface ContextOptions {
   graphHash?: string | null;
   artifactHash?: string | null;
   convexUrl?: string;
+  runtimeSchedulerSchema?: string | null;
   staticPipeline?: PlayStaticPipeline | null;
   workflowId?: string;
   sessionId?: string;
@@ -410,9 +460,15 @@ export interface ContextOptions {
   checkpoint?: PlayCheckpoint;
   /** Enables durable boundary replay such as ctx.sleep() resumptions. */
   durableBoundaries?: boolean;
-  /** Run-level cache policy. forceToolRefresh bypasses ctx.tools.execute receipts. */
+  /**
+   * Run-level cache policy.
+   * forceToolRefresh bypasses all ctx.tools.execute receipts.
+   * forceFailedToolRefresh reclaims only failed receipts, preserving completed
+   * provider-call idempotency while allowing forced repair runs to make progress.
+   */
   cachePolicy?: {
     forceToolRefresh?: boolean;
+    forceFailedToolRefresh?: boolean;
   };
   /** Called after each durable batch completes — for Temporal heartbeating. */
   onBatchComplete?: (checkpoint: PlayCheckpoint) => void;
@@ -438,6 +494,7 @@ export interface ContextOptions {
       playName?: string;
       playId?: string;
       runId?: string;
+      executorToken?: string;
       staticPipeline?: PlayStaticPipeline | null;
       forceRefresh?: boolean;
     },
@@ -452,6 +509,7 @@ export interface ContextOptions {
     playName?: string;
     playId?: string;
     runId?: string;
+    executorToken?: string;
     tableNamespace: string;
     rows: MapRowOutcome[];
     outputFields: string[];
@@ -459,12 +517,25 @@ export interface ContextOptions {
   }) => Promise<void>;
   playId?: string;
   runId?: string;
+  runAttempt?: number | null;
+  /**
+   * Optional shared provider-rate state for fleet substrates. Local/test
+   * contexts omit this and use the in-process backend.
+   */
+  rateState?: RateStateBackend;
+  /**
+   * Loud guard for production Node substrates: when true, constructing a
+   * context without a shared rate-state backend is a configuration error.
+   */
+  requireSharedRateState?: boolean;
   resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
   getToolRetryPolicy?: (toolId: string) => Promise<{
     retrySafeTransientHttp?: boolean;
   } | null>;
   getToolActionCacheVersion?: (toolId: string) => Promise<string> | string;
+  getToolAuthScopeDigest?: (toolId: string) => Promise<string> | string;
+  invalidateToolAuthScopeDigest?: (toolId: string) => void;
   getToolTargetGetters?: (
     toolId: string,
     output: string,
@@ -501,25 +572,41 @@ export interface ContextOptions {
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
   getRuntimeStepReceipts?: (
     input: GetRuntimeStepReceiptsInput,
-  ) => Promise<RuntimeStepReceipt[]> | RuntimeStepReceipt[];
+  ) =>
+    | Promise<Array<RuntimeStepReceipt | null>>
+    | Array<RuntimeStepReceipt | null>;
   claimRuntimeStepReceipt?: (
     input: ClaimRuntimeStepReceiptInput,
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
   claimRuntimeStepReceipts?: (
     input: ClaimRuntimeStepReceiptsInput,
-  ) => Promise<RuntimeStepReceipt[]> | RuntimeStepReceipt[];
+  ) =>
+    | Promise<Array<RuntimeStepReceipt | null>>
+    | Array<RuntimeStepReceipt | null>;
   completeRuntimeStepReceipt?: (
     input: CompleteRuntimeStepReceiptInput,
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  releaseRuntimeStepReceipt?: (
+    input: ReleaseRuntimeStepReceiptInput,
+  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
   completeRuntimeStepReceipts?: (
     input: CompleteRuntimeStepReceiptsInput,
-  ) => Promise<RuntimeStepReceipt[]> | RuntimeStepReceipt[];
+  ) =>
+    | Promise<Array<RuntimeStepReceipt | null>>
+    | Array<RuntimeStepReceipt | null>;
   failRuntimeStepReceipt?: (
     input: FailRuntimeStepReceiptInput,
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
   failRuntimeStepReceipts?: (
     input: FailRuntimeStepReceiptsInput,
-  ) => Promise<RuntimeStepReceipt[]> | RuntimeStepReceipt[];
+  ) =>
+    | Promise<Array<RuntimeStepReceipt | null>>
+    | Array<RuntimeStepReceipt | null>;
+  heartbeatRuntimeStepReceipts?: (
+    input: HeartbeatRuntimeStepReceiptsInput,
+  ) =>
+    | Promise<Array<RuntimeStepReceipt | null>>
+    | Array<RuntimeStepReceipt | null>;
   skipRuntimeStepReceipt?: (
     input: SkipRuntimeStepReceiptInput,
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
@@ -579,6 +666,15 @@ export interface PlayCheckpoint {
         output?: unknown;
         completedAt?: number;
       }
+    | {
+        kind: 'child_play';
+        childRunId: string;
+        childPlayName: string;
+        status: 'completed' | 'failed' | 'cancelled';
+        output?: unknown;
+        error?: string;
+        completedAt?: number;
+      }
   >;
   /** Per-map execution frames keyed by invocation id. */
   mapFrames?: Record<string, MapExecutionFrame>;
@@ -782,6 +878,10 @@ export interface PlayRowUpdate {
   key: string;
   rowId: number;
   tableNamespace?: string | null;
+  attemptId?: string | null;
+  attemptOwnerRunId?: string | null;
+  attemptSeq?: number | null;
+  attemptExpiresAt?: string | null;
   status?: 'running' | 'completed' | 'failed';
   stage?: string | null;
   provider?: string | null;

package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts

@@ -81,6 +81,7 @@ const testRateLimitBatchStrategy: BatchOperationStrategy<
   batchOperation: 'test_batch_rate_limit',
   kind: 'identifier_batch',
   maxBatchSize: 200,
+  bucketKeyPayloadFields: ['stage'],
   canBatchWith(left, right) {
     return String(left.stage || '') === String(right.stage || '');
   },

package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts

@@ -3,6 +3,7 @@ import type {
   CompleteRuntimeStepReceiptInput,
   FailRuntimeStepReceiptInput,
   GetRuntimeStepReceiptInput,
+  ReleaseRuntimeStepReceiptInput,
   RuntimeStepReceipt,
   SkipRuntimeStepReceiptInput,
 } from './ctx-types';
@@ -50,6 +51,9 @@ export interface PlayDurabilityStore {
   completeRuntimeStepReceipt(
     input: CompleteRuntimeStepReceiptInput,
   ): Promise<RuntimeStepReceipt | null>;
+  releaseRuntimeStepReceipt(
+    input: ReleaseRuntimeStepReceiptInput,
+  ): Promise<RuntimeStepReceipt | null>;
   failRuntimeStepReceipt(
     input: FailRuntimeStepReceiptInput,
   ): Promise<RuntimeStepReceipt | null>;

package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts

@@ -47,37 +47,36 @@ export const durableCallStaleBucket = durableCacheStaleBucket;
 
 export function buildDurableToolCallCacheKey(input: {
   orgId?: string | null;
-  playId: string;
   toolId: string;
   requestInput: Record<string, unknown>;
   authScopeDigest?: string | null;
   providerActionVersion?: string | null;
   cachePolicyVersion?: string | null;
   staleAfterSeconds?: number | null;
+  playLocalScope?: string | null;
 }): string {
   const orgId = input.orgId?.trim() || 'org';
-  const playId = input.playId.trim();
-  if (!playId) {
-    throw new Error('Missing durable tool play id.');
-  }
   const toolId = input.toolId.trim();
   if (!toolId) {
-    throw new Error('Missing durable tool id.');
+    throw new Error('Durable tool call cache key requires a non-empty toolId.');
   }
-  const p = input.providerActionVersion?.trim();
-  if (!p) {
-    throw new Error('Missing durable tool action version.');
+  const providerActionVersion = input.providerActionVersion?.trim();
+  if (!providerActionVersion) {
+    throw new Error(
+      'Durable tool call cache key requires a provider action version.',
+    );
   }
+  const playLocalScope = input.playLocalScope?.trim() || 'play';
   const digest = sha256Hex(
     stableStringify({
       kind: 'tool' satisfies DurableCallKind,
       orgId,
-      playId,
+      playLocalScope,
       toolId,
       normalizedToolId: normalizeTableNamespace(toolId),
       requestInput: input.requestInput,
       authScopeDigest: input.authScopeDigest ?? null,
-      providerActionVersion: p,
+      providerActionVersion,
       cachePolicyVersion:
         input.cachePolicyVersion ?? DURABLE_CALL_CACHE_POLICY_VERSION,
       staleBucket: durableCacheStaleBucket({
@@ -101,9 +100,7 @@ export function buildDurableCtxCallCacheKey(input: {
   const playId = input.playId?.trim() || 'play';
   const id = input.id.trim();
   if (!id) {
-    throw new Error(
-      `Durable ${input.kind} call cache key requires a non-empty id.`,
-    );
+    throw new Error(`Durable ${input.kind} key needs id.`);
   }
   const normalizedId = normalizeTableNamespace(id);
   const digest = sha256Hex(
@@ -124,16 +121,119 @@ export function buildDurableCtxCallCacheKey(input: {
   return `ctx:${orgId}:call:${input.kind}:${normalizedId}:${digest}`;
 }
 
+export function buildDurableRunPlaySemanticKey(input: {
+  childPlayName: string;
+  input: Record<string, unknown>;
+  childRevisionFingerprint?: string | null;
+  rowScope?: Record<string, unknown> | null;
+}): string {
+  return sha256Hex(
+    stableStringify({
+      childPlayName: input.childPlayName.trim(),
+      childRevisionFingerprint: input.childRevisionFingerprint ?? null,
+      input: input.input,
+      rowScope: input.rowScope ?? null,
+    }),
+  );
+}
+
 export function buildDurableToolCallAuthScopeDigest(input: {
   orgId?: string | null;
-  userEmail?: string | null;
   toolId: string;
+  executionAuthScopeDigest?: string | null;
 }): string {
   return sha256Hex(
     stableStringify({
       orgId: input.orgId?.trim() || 'org',
-      actor: input.userEmail?.trim().toLowerCase() || 'workspace',
       toolId: input.toolId.trim(),
+      executionAuthScopeDigest: input.executionAuthScopeDigest?.trim() || null,
     }),
   );
 }
+
+export function buildDurableToolReceiptPrefix(input: {
+  orgId?: string | null;
+  toolId: string;
+}): string {
+  const orgId = input.orgId?.trim() || 'org';
+  const toolId = input.toolId.trim();
+  if (!toolId) {
+    throw new Error('Durable tool receipt prefix requires a non-empty toolId.');
+  }
+  return `ctx:${orgId}:call:tool:${normalizeTableNamespace(toolId)}:`;
+}
+
+export function buildDurableToolProviderIdempotencyKey(input: {
+  receiptKey: string;
+  force?: boolean;
+  receiptLeaseId?: string | null;
+  fallbackAttemptId?: string | null;
+}): string {
+  const receiptKey = input.receiptKey.trim();
+  if (!receiptKey) {
+    throw new Error('Need receipt key.');
+  }
+  if (input.force !== true) {
+    return receiptKey;
+  }
+  const attemptId =
+    input.receiptLeaseId?.trim() || input.fallbackAttemptId?.trim() || null;
+  if (!attemptId) {
+    throw new Error('Forced idempotency needs lease id.');
+  }
+  return `${receiptKey}:force:${attemptId}`;
+}
+
+export function buildDurableToolAggregateReceiptKey(input: {
+  receiptKeys: string[];
+  prefix?: 'batch' | null;
+  aggregateReceiptPrefix?: string | null;
+}): string {
+  const receiptKeys = input.receiptKeys.map((key) => key.trim()).sort();
+  const digest = sha256Hex(stableStringify(receiptKeys));
+  if (input.prefix !== 'batch') {
+    return digest;
+  }
+  const scopedPrefix =
+    input.aggregateReceiptPrefix?.trim() ||
+    commonDurableToolReceiptPrefix(receiptKeys);
+  if (!scopedPrefix) {
+    throw new Error('Batch receipt key needs same-tool members.');
+  }
+  return `${scopedPrefix}batch:${digest}`;
+}
+
+function commonDurableToolReceiptPrefix(
+  receiptKeys: readonly string[],
+): string {
+  let commonPrefix: string | null = null;
+  for (const receiptKey of receiptKeys) {
+    const match = /^(ctx:[^:]+:call:tool:[^:]+:)/.exec(receiptKey);
+    if (!match) {
+      return '';
+    }
+    const prefix = match[1] ?? '';
+    if (!commonPrefix) {
+      commonPrefix = prefix;
+      continue;
+    }
+    if (commonPrefix !== prefix) {
+      return '';
+    }
+  }
+  return commonPrefix ?? '';
+}
+
+export function buildDurableToolAggregateProviderIdempotencyKey(input: {
+  aggregateReceiptKey: string;
+  receiptKeys: string[];
+  providerIdempotencyKeys: string[];
+}): string {
+  const receiptKeys = input.receiptKeys.map((key) => key.trim());
+  const providerIdempotencyKeys = input.providerIdempotencyKeys.map((key) =>
+    key.trim(),
+  );
+  return `${input.aggregateReceiptKey}:ordered:${sha256Hex(
+    stableStringify({ receiptKeys, providerIdempotencyKeys }),
+  )}`;
+}