deepline 0.1.209 → 0.1.211

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts

@@ -54,6 +54,7 @@ import {
   markWorkerToolReceiptResultCached,
   markWorkerToolReceiptResultExecution,
   planWorkerToolReceiptGroups,
+  resolveRunningReceiptWaitMaxAttempts,
   resolveWorkerToolReceiptGroupWaitMaxAttempts,
   resolveWorkerToolRuntimeTimeoutMs,
 } from './tool-receipts';
@@ -1232,10 +1233,17 @@ export class WorkerToolBatchScheduler {
               try {
                 const result = await this.waitForDurableToolReceipt({
                   receiptKey,
-                  maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
-                    group,
-                    (request) => request.receiptWaitMaxAttempts,
-                  ),
+                  maxAttempts: resolveRunningReceiptWaitMaxAttempts({
+                    ownerRunId: claim.receipt.runId,
+                    currentRunId: this.options.req.runId,
+                    updatedAt: claim.receipt.updatedAt,
+                    requestedMaxAttempts:
+                      resolveWorkerToolReceiptGroupWaitMaxAttempts(
+                        group,
+                        (request) => request.receiptWaitMaxAttempts,
+                      ),
+                    nowMs: this.options.nowMs(),
+                  }),
                 });
                 if (groupState.forceDurableRefresh) {
                   return await this.claimForcedDurableToolReceiptGroupAfterWait(
@@ -1992,8 +2000,14 @@ export class WorkerToolBatchScheduler {
     toolId: string,
     requests: WorkerToolBatchRequest[],
   ): Promise<void> {
+    const receiptPrepareStartedAt = this.options.nowMs();
     const { claimedRequests, deferredClaimedRequests } =
       await this.prepareDurableToolRequests(requests);
+    this.options.recordPerfTrace?.({
+      phase: 'runner.tool.receipt_prepare',
+      ms: this.options.nowMs() - receiptPrepareStartedAt,
+      extra: { toolId, requests: requests.length },
+    });
     await this.executeClaimedToolRequests(
       toolId,
       requests.length,
@@ -2063,6 +2077,7 @@ export class WorkerToolBatchScheduler {
         // charges tool budget, holds a global tool-concurrency slot, and
         // applies per-(org,provider) pacing before the call runs.
         let slot: { release: () => void };
+        const admissionStartedAt = this.options.nowMs();
         try {
           slot = await this.options.governor.acquireToolSlot(toolId, {
             signal: this.options.abortSignal,
@@ -2074,6 +2089,11 @@ export class WorkerToolBatchScheduler {
           );
           return;
         }
+        this.options.recordPerfTrace?.({
+          phase: 'runner.tool.admission',
+          ms: this.options.nowMs() - admissionStartedAt,
+          extra: { toolId, requests: 1 },
+        });
         try {
           let result: unknown;
           try {
@@ -2092,59 +2112,68 @@ export class WorkerToolBatchScheduler {
                   claimed,
                 })
               : null;
-            result = await executeWithWorkerReceiptHeartbeats({
-              playName: this.options.req.playName,
-              runId: this.options.req.runId,
-              runAttempt: this.options.req.runAttempt ?? 0,
-              ...(this.options.receiptStore
-                ? { receiptStore: this.options.receiptStore }
-                : {}),
-              budgetMeter: this.options.budgetMeter,
-              heartbeatIntervalMs:
-                this.options.runtimeReceiptHeartbeatIntervalMs,
-              targets: claimed.receiptKey
-                ? [
-                    {
-                      key: claimed.receiptKey,
-                      leaseId: claimed.receiptLeaseId,
-                    },
-                  ]
-                : [],
-              execute: () =>
-                this.options.executeTool({
-                  id: request.id,
-                  toolId,
-                  input: request.input,
-                  workflowStep: request.workflowStep,
-                  callbacks: this.options.callbacks,
-                  onProviderBackpressure: (retryAfterMs) =>
-                    this.reportBackpressure(toolId, retryAfterMs),
-                  onRetryAttempt: () =>
-                    this.options.governor.chargeBudget('retry'),
-                  transientHttpRetrySafe:
-                    toolContract?.retrySafeTransientHttp === true,
-                  ...(durableReceiptKey
-                    ? {
-                        durableCallReceiptKey: durableReceiptKey,
-                        executionAuthScopeDigest:
-                          request.executionAuthScopeDigest,
-                        providerIdempotencyKey:
-                          providerIdempotencyKeyForClaimed({
-                            req: this.options.req,
-                            claimed,
-                          }),
-                      }
-                    : {}),
-                  runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
-                    [claimed],
-                    {
-                      resolveOwnerTimeoutMs: (request) =>
-                        request.runtimeTimeoutMs,
-                    },
-                  ),
-                  directOptions: request.directOptions,
-                }),
-            });
+            const providerStartedAt = this.options.nowMs();
+            try {
+              result = await executeWithWorkerReceiptHeartbeats({
+                playName: this.options.req.playName,
+                runId: this.options.req.runId,
+                runAttempt: this.options.req.runAttempt ?? 0,
+                ...(this.options.receiptStore
+                  ? { receiptStore: this.options.receiptStore }
+                  : {}),
+                budgetMeter: this.options.budgetMeter,
+                heartbeatIntervalMs:
+                  this.options.runtimeReceiptHeartbeatIntervalMs,
+                targets: claimed.receiptKey
+                  ? [
+                      {
+                        key: claimed.receiptKey,
+                        leaseId: claimed.receiptLeaseId,
+                      },
+                    ]
+                  : [],
+                execute: () =>
+                  this.options.executeTool({
+                    id: request.id,
+                    toolId,
+                    input: request.input,
+                    workflowStep: request.workflowStep,
+                    callbacks: this.options.callbacks,
+                    onProviderBackpressure: (retryAfterMs) =>
+                      this.reportBackpressure(toolId, retryAfterMs),
+                    onRetryAttempt: () =>
+                      this.options.governor.chargeBudget('retry'),
+                    transientHttpRetrySafe:
+                      toolContract?.retrySafeTransientHttp === true,
+                    ...(durableReceiptKey
+                      ? {
+                          durableCallReceiptKey: durableReceiptKey,
+                          executionAuthScopeDigest:
+                            request.executionAuthScopeDigest,
+                          providerIdempotencyKey:
+                            providerIdempotencyKeyForClaimed({
+                              req: this.options.req,
+                              claimed,
+                            }),
+                        }
+                      : {}),
+                    runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
+                      [claimed],
+                      {
+                        resolveOwnerTimeoutMs: (request) =>
+                          request.runtimeTimeoutMs,
+                      },
+                    ),
+                    directOptions: request.directOptions,
+                  }),
+              });
+            } finally {
+              this.options.recordPerfTrace?.({
+                phase: 'runner.tool.provider',
+                ms: this.options.nowMs() - providerStartedAt,
+                extra: { toolId, requests: 1 },
+              });
+            }
           } catch (error) {
             if (error instanceof RuntimeLeaseLostError) {
               this.rejectRequests(claimed, error);
@@ -2165,10 +2194,16 @@ export class WorkerToolBatchScheduler {
             return;
           }
           try {
+            const receiptCompleteStartedAt = this.options.nowMs();
             this.settleRequests(
               claimed,
               await this.completeDurableToolRequest(claimed, result),
             );
+            this.options.recordPerfTrace?.({
+              phase: 'runner.tool.receipt_complete',
+              ms: this.options.nowMs() - receiptCompleteStartedAt,
+              extra: { toolId, requests: 1 },
+            });
           } catch (receiptError) {
             this.rejectRequests(claimed, receiptError);
           }
@@ -2253,12 +2288,22 @@ export class WorkerToolBatchScheduler {
         }
         // One provider call per batch -> one tool slot (budget + global
         // concurrency + per-(org,provider) pacing) around the whole batch.
+        const admissionStartedAt = this.options.nowMs();
         const slot = await this.options.governor.acquireToolSlot(
           batch.batchOperation,
           {
             signal: this.options.abortSignal,
           },
         );
+        this.options.recordPerfTrace?.({
+          phase: 'runner.tool.admission',
+          ms: this.options.nowMs() - admissionStartedAt,
+          extra: {
+            toolId: batch.batchOperation,
+            requests: batch.memberRequests.length,
+            batched: true,
+          },
+        });
         try {
           await this.markAdmittedToolRequestsRunning(batch.memberRequests);
           this.options.budgetMeter?.count('provider');
@@ -2273,54 +2318,69 @@ export class WorkerToolBatchScheduler {
             batchOperation: batch.batchOperation,
             receiptKeys,
           });
-          return await executeWithWorkerReceiptHeartbeats({
-            playName: this.options.req.playName,
-            runId: this.options.req.runId,
-            runAttempt: this.options.req.runAttempt ?? 0,
-            ...(this.options.receiptStore
-              ? { receiptStore: this.options.receiptStore }
-              : {}),
-            budgetMeter: this.options.budgetMeter,
-            heartbeatIntervalMs: this.options.runtimeReceiptHeartbeatIntervalMs,
-            targets: batch.memberRequests.map((member) => ({
-              key: member.receiptKey,
-              leaseId: member.receiptLeaseId,
-            })),
-            execute: () =>
-              this.options.executeTool({
-                id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
-                toolId: batch.batchOperation,
-                input: batch.batchPayload,
-                callbacks: this.options.callbacks,
-                onProviderBackpressure: (retryAfterMs) =>
-                  this.reportBackpressure(input.sourceToolId, retryAfterMs),
-                onRetryAttempt: () =>
-                  this.options.governor.chargeBudget('retry'),
-                transientHttpRetrySafe:
-                  toolContract?.retrySafeTransientHttp === true,
-                durableCallReceiptKey: aggregateReceiptKey,
-                executionAuthScopeDigest:
-                  batch.memberRequests[0]?.request.executionAuthScopeDigest ??
-                  null,
-                providerIdempotencyKey: buildBatchProviderIdempotencyKey({
-                  aggregateReceiptKey,
-                  receiptKeys,
-                  providerIdempotencyKeys: batch.memberRequests.map((member) =>
-                    providerIdempotencyKeyForClaimed({
-                      req: this.options.req,
-                      claimed: member,
-                    }),
+          const providerStartedAt = this.options.nowMs();
+          try {
+            return await executeWithWorkerReceiptHeartbeats({
+              playName: this.options.req.playName,
+              runId: this.options.req.runId,
+              runAttempt: this.options.req.runAttempt ?? 0,
+              ...(this.options.receiptStore
+                ? { receiptStore: this.options.receiptStore }
+                : {}),
+              budgetMeter: this.options.budgetMeter,
+              heartbeatIntervalMs:
+                this.options.runtimeReceiptHeartbeatIntervalMs,
+              targets: batch.memberRequests.map((member) => ({
+                key: member.receiptKey,
+                leaseId: member.receiptLeaseId,
+              })),
+              execute: () =>
+                this.options.executeTool({
+                  id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
+                  toolId: batch.batchOperation,
+                  input: batch.batchPayload,
+                  callbacks: this.options.callbacks,
+                  onProviderBackpressure: (retryAfterMs) =>
+                    this.reportBackpressure(input.sourceToolId, retryAfterMs),
+                  onRetryAttempt: () =>
+                    this.options.governor.chargeBudget('retry'),
+                  transientHttpRetrySafe:
+                    toolContract?.retrySafeTransientHttp === true,
+                  durableCallReceiptKey: aggregateReceiptKey,
+                  executionAuthScopeDigest:
+                    batch.memberRequests[0]?.request.executionAuthScopeDigest ??
+                    null,
+                  providerIdempotencyKey: buildBatchProviderIdempotencyKey({
+                    aggregateReceiptKey,
+                    receiptKeys,
+                    providerIdempotencyKeys: batch.memberRequests.map(
+                      (member) =>
+                        providerIdempotencyKeyForClaimed({
+                          req: this.options.req,
+                          claimed: member,
+                        }),
+                    ),
+                  }),
+                  runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
+                    batch.memberRequests,
+                    {
+                      resolveOwnerTimeoutMs: (request) =>
+                        request.runtimeTimeoutMs,
+                    },
                   ),
                 }),
-                runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
-                  batch.memberRequests,
-                  {
-                    resolveOwnerTimeoutMs: (request) =>
-                      request.runtimeTimeoutMs,
-                  },
-                ),
-              }),
-          });
+            });
+          } finally {
+            this.options.recordPerfTrace?.({
+              phase: 'runner.tool.provider',
+              ms: this.options.nowMs() - providerStartedAt,
+              extra: {
+                toolId: batch.batchOperation,
+                requests: batch.memberRequests.length,
+                batched: true,
+              },
+            });
+          }
         } finally {
           slot.release();
         }
@@ -2381,6 +2441,7 @@ export class WorkerToolBatchScheduler {
               : entry.request.memberRequests.map(() => null);
           let completedResults: WorkerToolCompletionResult[];
           try {
+            const receiptCompleteStartedAt = this.options.nowMs();
             completedResults = await this.completeDurableToolRequests(
               entry.request.memberRequests.map((claimed, index) => ({
                 claimed,
@@ -2391,6 +2452,15 @@ export class WorkerToolBatchScheduler {
                 ),
               })),
             );
+            this.options.recordPerfTrace?.({
+              phase: 'runner.tool.receipt_complete',
+              ms: this.options.nowMs() - receiptCompleteStartedAt,
+              extra: {
+                toolId: entry.request.batchOperation,
+                requests: entry.request.memberRequests.length,
+                batched: true,
+              },
+            });
           } catch (receiptError) {
             for (const claimed of entry.request.memberRequests) {
               this.rejectRequests(claimed, receiptError);

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts

@@ -39,106 +39,39 @@ export function canReclaimTimedOutWorkerToolReceipt(input: {
  * conservative constant, not a heartbeat multiple.
  */
 export const RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS = 60_000;
+export const RUNNING_RECEIPT_POLL_INTERVAL_MS = 250;
 
-/**
- * Decide whether a `running` receipt should be reclaimed IMMEDIATELY, skipping
- * the completion poll.
- *
- * The poll (`waitForCompletedRuntimeReceipt`) issues one service-binding
- * subrequest per 250ms tick, up to the receipt's wait budget (~1320 ticks for a
- * 5.5-minute default). On resume, a map can carry many receipts left `running`
- * by a DEAD prior run; polling each one burns its full tick budget waiting on a
- * receipt no live worker will complete, which blows Cloudflare's ~1000
- * subrequests-per-invocation budget and kills the resume with
- * "Too many subrequests by single Worker invocation" (prod resume deaths
- * 20260703t230209 / 20260703t232104).
- *
- * A receipt is a dead-owner orphan when it is owned by a DIFFERENT run (a
- * same-run `running` is this run replaying — its owner may still be live) and
- * its last write is stale (or missing entirely — a live owner would carry a
- * recent claim timestamp). Those reclaim immediately, matching the dispatch
- * code's own contract ("on resume it reclaims and re-executes"). A live or
- * freshly-claimed owner keeps the poll.
- */
-export function shouldReclaimRunningReceiptWithoutPolling(input: {
+/** Bound a follower poll by when a different-run owner becomes reclaimable. */
+export function resolveRunningReceiptWaitMaxAttempts(input: {
   ownerRunId?: string | null;
   currentRunId: string;
   updatedAt?: string | null;
+  requestedMaxAttempts: number;
   nowMs?: number;
   stalenessFloorMs?: number;
-}): boolean {
-  const currentRunId = input.currentRunId.trim();
+  pollIntervalMs?: number;
+}): number {
+  const requested = Math.max(1, Math.floor(input.requestedMaxAttempts));
   const ownerRunId = input.ownerRunId?.trim() ?? '';
-  // Only a receipt owned by a different run can be a dead prior-run orphan.
-  if (!currentRunId || !ownerRunId || ownerRunId === currentRunId) return false;
-  const floorMs =
-    input.stalenessFloorMs ?? RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS;
-  // No freshness signal at all: treat as stale. A live owner would carry a
-  // recent claim timestamp; an orphan from a dead run may carry none.
-  if (!input.updatedAt) return true;
-  const updatedAtMs = Date.parse(input.updatedAt);
-  if (!Number.isFinite(updatedAtMs)) return true;
-  const now = input.nowMs ?? Date.now();
-  return now - updatedAtMs >= floorMs;
-}
-
-/** Metadata a `running` receipt carries about its owner and last write. */
-export interface RunningReceiptOwnershipInput {
-  ownerRunId?: string | null;
-  currentRunId: string;
-  updatedAt?: string | null;
-  nowMs?: number;
-  stalenessFloorMs?: number;
-}
-
-/** Outcome of polling a `running` receipt for completion. */
-export type RunningReceiptPollOutcome =
-  /** The poll saw the receipt complete and already resolved the group. */
-  | { kind: 'completed' }
-  /** The poll timed out; the receipt is still `running`. */
-  | { kind: 'timed_out'; error: unknown }
-  /** The poll failed for a non-timeout reason (e.g. the receipt failed). */
-  | { kind: 'errored'; error: unknown };
-
-/**
- * Orchestrate the resume-time handling of a `running` receipt: reclaim it
- * immediately when it is a dead-owner orphan, otherwise poll for completion and
- * only reclaim after the poll times out. Kept as a pure, dependency-injected
- * function (the poll and reclaim side effects are passed in) so the
- * "skip the poll for a dead owner" decision is unit-testable with a poll counter
- * — the Worker scheduler that owns the real effects imports `cloudflare:workers`
- * and cannot be loaded under vitest.
- *
- * Behavior (must match the scheduler's prior inline logic exactly):
- * - dead-owner + reclaim-eligible → `reclaim` now, never polling;
- * - otherwise `poll`; on `completed` the group is already resolved (return []);
- *   on a non-timeout error `reject` and stop; on timeout, `reclaim` if still
- *   eligible else `reject` with the timeout error.
- */
-export async function reclaimRunningReceiptGroupAfterWait<TClaimed>(input: {
-  ownership: RunningReceiptOwnershipInput;
-  timeoutError: unknown;
-  poll: () => Promise<RunningReceiptPollOutcome>;
-  reclaim: (rejectionOnFallthrough: unknown) => Promise<TClaimed[]>;
-  reject: (error: unknown) => void;
-}): Promise<TClaimed[]> {
-  if (
-    shouldReclaimRunningReceiptWithoutPolling(input.ownership) &&
-    canReclaimTimedOutWorkerToolReceipt(input.ownership)
-  ) {
-    return await input.reclaim(input.timeoutError);
-  }
-  const outcome = await input.poll();
-  if (outcome.kind === 'completed') return [];
-  if (outcome.kind === 'errored') {
-    input.reject(outcome.error);
-    return [];
-  }
-  if (!canReclaimTimedOutWorkerToolReceipt(input.ownership)) {
-    input.reject(outcome.error);
-    return [];
+  const currentRunId = input.currentRunId.trim();
+  if (!ownerRunId || !currentRunId || ownerRunId === currentRunId) {
+    return requested;
   }
-  return await input.reclaim(outcome.error);
+  const updatedAtMs = input.updatedAt ? Date.parse(input.updatedAt) : NaN;
+  if (!Number.isFinite(updatedAtMs)) return 1;
+  const remainingMs = Math.max(
+    0,
+    updatedAtMs +
+      (input.stalenessFloorMs ?? RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS) -
+      (input.nowMs ?? Date.now()),
+  );
+  const attemptsUntilReclaim = Math.max(
+    1,
+    Math.floor(
+      remainingMs / (input.pollIntervalMs ?? RUNNING_RECEIPT_POLL_INTERVAL_MS),
+    ) + 1,
+  );
+  return Math.min(requested, attemptsUntilReclaim);
 }
 
 export function canReclaimFailedWorkerToolReceipt(input: {

package/dist/bundling-sources/sdk/src/play.ts

@@ -173,6 +173,8 @@ export type PlayBindings = {
    * older clients can continue to register revisions during the migration.
    */
   description?: string;
+  /** Allow compilers to bundle this named handler directly without a child run. */
+  inline?: boolean;
   /** Optional per-run billing controls enforced by the runtime. */
   billing?: {
     /** Stop the run before a billed action would push total run credits above this cap. */
@@ -400,7 +402,7 @@ export type StepOptions<Row, Value = unknown> = {
    * Legacy dataset-column recompute flag accepted for older authored plays.
    *
    * Prefer putting freshness on the actual reusable call
-   * (`ctx.tools.execute`, `ctx.step`, `ctx.fetch`, or `ctx.runPlay`).
+   * (`ctx.tools.execute`, `ctx.step`, or `ctx.fetch`).
    */
   readonly recompute?: boolean;
   /** Legacy error-recompute flag accepted for older authored plays. */
@@ -889,7 +891,7 @@ export interface DeeplinePlayRuntimeContext {
     key: string,
     playRef: string | PlayReferenceLike,
     input: Record<string, unknown>,
-    options: { description: string; staleAfterSeconds?: number },
+    options: { description: string },
   ): Promise<TOutput>;
 
   /**

package/dist/bundling-sources/sdk/src/release.ts

@@ -106,10 +106,10 @@ export const SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: '0.1.209',
+  version: '0.1.211',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.209',
+    latest: '0.1.211',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [

package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts

@@ -44,8 +44,7 @@ export interface ChildRunIdInputs {
   key?: string | null;
   /**
    * Canonical durable runPlay semantic key when the launching substrate has one.
-   * In-process runtimes pass the same key used by durable receipts so row-scoped
-   * receipt identity and child run identity cannot drift apart.
+   * This scopes each fresh child invocation to its parent call site and row.
    */
   runPlaySemanticKey?: string | null;
   /** Child input payload (semantic-fallback input). */
@@ -57,9 +56,9 @@ export interface ChildRunIdInputs {
 }
 
 /**
- * A failed durable runPlay receipt must launch a fresh child attempt while
- * ordinary submit retries keep attaching to the original child. The receipt
- * lease is the attempt fence, so it is the only extra identity required.
+ * A failed child launch must use a fresh attempt identity while ordinary
+ * transport retries keep attaching to the original child. The caller-provided
+ * lease is the attempt fence.
  */
 export function childSubmitIdempotencyKey(input: {
   receiptKey: string;