deepline 0.1.210 → 0.1.211

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -166,6 +166,7 @@ import {
   createWorkerWorkBudgetMeter,
   type WorkerWorkBudgetMeter,
 } from './runtime/work-budget';
+import { MapLatencyProfile } from './runtime/map-latency-profile';
 import {
   WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES,
   WorkerRunWorkDispatcher,
@@ -4598,6 +4599,11 @@ function createMinimalWorkerCtx(
     opts?: WorkerMapOptions,
   ): Promise<unknown> => {
     const mapStartedAt = nowMs();
+    const mapLatencyProfile = new MapLatencyProfile(
+      req.testPolicyOverrides?.mapLatencyProfile === true,
+      name,
+      mapStartedAt,
+    );
     const mapNodeId = `map:${name}`;
     const inputRows = rows;
     const rowCountHint = datasetRowCountHint(inputRows);
@@ -4935,6 +4941,7 @@ function createMinimalWorkerCtx(
           blockedRows: prepared.blockedRows.length,
         },
       });
+      mapLatencyProfile.recordPhase('prepare_rows', nowMs() - prepareStartedAt);
       const progressTotalRows = rowCountHint ?? chunkRows.length;
       const preparedCompletedRows = Math.min(
         progressTotalRows,
@@ -5215,7 +5222,10 @@ function createMinimalWorkerCtx(
         deserializeDurableValue: deserializeDurableStepValue,
         nowMs,
         batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
-        recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
+        recordPerfTrace: (trace) => {
+          mapLatencyProfile.recordToolTrace(trace);
+          recordRunnerPerfTrace({ req, ...trace });
+        },
         abortSignal,
         onRequestsSettled: reportSettledToolRequests,
         callbacks,
@@ -5505,9 +5515,16 @@ function createMinimalWorkerCtx(
               if (circuitError) throw circuitError;
               const myIndex = idx++;
               if (myIndex >= rowsToExecute.length) return;
+              const rowQueueStartedAt = mapLatencyProfile.enabled ? nowMs() : 0;
               const rowSlot = await governor.acquireRowSlot({
                 signal: abortSignal,
               });
+              if (mapLatencyProfile.enabled) {
+                mapLatencyProfile.recordRowQueue(nowMs() - rowQueueStartedAt);
+              }
+              const rowExecutionStartedAt = mapLatencyProfile.enabled
+                ? nowMs()
+                : 0;
               let rowMarkedActive = false;
               try {
                 startedExecutedRows += 1;
@@ -5744,6 +5761,11 @@ function createMinimalWorkerCtx(
                   // single clean run failure.
                 }
               } finally {
+                if (mapLatencyProfile.enabled) {
+                  mapLatencyProfile.recordRowExecution(
+                    nowMs() - rowExecutionStartedAt,
+                  );
+                }
                 if (rowMarkedActive) {
                   activeExecutedRows = Math.max(0, activeExecutedRows - 1);
                   reportExecutionHeartbeat(false);
@@ -5796,6 +5818,10 @@ function createMinimalWorkerCtx(
           concurrency,
         },
       });
+      mapLatencyProfile.recordPhase(
+        'worker_execution',
+        nowMs() - workersStartedAt,
+      );
       const buildRowFailureSamples = () =>
         failedRowEntries
           .map((failure, executedIndex) =>
@@ -5923,6 +5949,10 @@ function createMinimalWorkerCtx(
             rowsToExecute: rowsToExecute.length,
           },
         });
+        mapLatencyProfile.recordPhase(
+          'persist_rows',
+          nowMs() - persistRowsStartedAt,
+        );
       } catch (error) {
         if (
           workflowStep &&
@@ -6028,6 +6058,7 @@ function createMinimalWorkerCtx(
         ms: nowMs() - hashStartedAt,
         extra: { mapName: name, chunkIndex, rows: out.length },
       });
+      mapLatencyProfile.recordPhase('hash', nowMs() - hashStartedAt);
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.total',
@@ -6643,6 +6674,27 @@ function createMinimalWorkerCtx(
         chunks: dispatchResult.chunksExecuted,
       },
     });
+    if (mapLatencyProfile.enabled) {
+      const endedAt = nowMs();
+      const summary = mapLatencyProfile.summary(endedAt, {
+        chunks: dispatchResult.chunksExecuted,
+        rows: totalRowsWritten,
+      });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map.profile',
+        ms: endedAt - mapStartedAt,
+        extra: summary,
+      });
+      // Local Absurd does not always forward dynamic-worker perf traces. Keep
+      // one bounded copy in the durable run log for the diagnostic command.
+      emitEvent({
+        type: 'log',
+        level: 'info',
+        message: `[perf] map_latency_profile ${JSON.stringify(summary)}`,
+        ts: endedAt,
+      });
+    }
     return dataset;
   };
 

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts

@@ -337,6 +337,21 @@ function countToolSubstep(substep: PlayStaticSubstep): MapToolStats {
         minProviderBranchCoalescedCount(branchStats),
       ];
     }
+    if (substep.kind === 'loop') {
+      // The static extractor records every switch arm in `steps` when a loop
+      // walks a fixed provider list. Each arm can execute on a different
+      // iteration, so using the conditional's max-one-branch estimate would
+      // budget a seven-provider waterfall as one provider call per row.
+      return substep.steps.reduce<MapToolStats>((stats, step) => {
+        const stepStats =
+          step.type === 'control_flow' &&
+          step.kind === 'conditional' &&
+          step.steps.length > 0
+            ? countToolSubsteps(step.steps)
+            : countToolSubstep(step);
+        return addStats(stats, stepStats);
+      }, [0, 0, 0, 0, 0]);
+    }
     return countToolSubsteps(substep.steps);
   }
 

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts

@@ -0,0 +1,90 @@
+type ToolTrace = {
+  phase: string;
+  ms?: number;
+  extra?: Record<string, unknown>;
+};
+
+type Totals = Record<string, number>;
+
+function add(totals: Totals, key: string, ms: number | undefined): void {
+  if (typeof ms !== 'number' || !Number.isFinite(ms) || ms < 0) return;
+  totals[key] = (totals[key] ?? 0) + ms;
+}
+
+/**
+ * Opt-in map profiler. It aggregates in memory and produces one terminal
+ * trace, so normal runs do not pay per-row/per-tool telemetry cost.
+ */
+export class MapLatencyProfile {
+  private readonly phaseWorkMs: Totals = {};
+  private readonly toolWorkMs: Totals = {};
+  private readonly counts: Totals = {};
+
+  constructor(
+    readonly enabled: boolean,
+    private readonly mapName: string,
+    private readonly startedAtMs: number,
+  ) {}
+
+  recordPhase(phase: string, ms: number, count = 1): void {
+    if (!this.enabled) return;
+    add(this.phaseWorkMs, phase, ms);
+    this.counts[`${phase}_count`] =
+      (this.counts[`${phase}_count`] ?? 0) + count;
+  }
+
+  recordRowQueue(ms: number): void {
+    this.recordPhase('row_slot_queue', ms);
+  }
+
+  recordRowExecution(ms: number): void {
+    this.recordPhase('row_execution', ms);
+  }
+
+  recordToolTrace(trace: ToolTrace): void {
+    if (!this.enabled) return;
+    switch (trace.phase) {
+      case 'runner.tool.request':
+        add(this.toolWorkMs, 'request_end_to_end', trace.ms);
+        this.counts.tool_requests = (this.counts.tool_requests ?? 0) + 1;
+        break;
+      case 'runner.tool.drain':
+        add(this.toolWorkMs, 'drain', trace.ms);
+        this.counts.tool_drains = (this.counts.tool_drains ?? 0) + 1;
+        break;
+      case 'runner.tool.group':
+        add(this.toolWorkMs, 'group', trace.ms);
+        this.counts.tool_groups = (this.counts.tool_groups ?? 0) + 1;
+        break;
+      case 'runner.tool.receipt_prepare':
+        add(this.toolWorkMs, 'receipt_prepare', trace.ms);
+        break;
+      case 'runner.tool.admission':
+        add(this.toolWorkMs, 'admission_wait', trace.ms);
+        break;
+      case 'runner.tool.provider':
+        add(this.toolWorkMs, 'provider_execute', trace.ms);
+        break;
+      case 'runner.tool.receipt_complete':
+        add(this.toolWorkMs, 'receipt_complete', trace.ms);
+        break;
+      default:
+        return;
+    }
+  }
+
+  summary(endedAtMs: number, input: { chunks: number; rows: number }) {
+    return {
+      mapName: this.mapName,
+      mapWallMs: Math.max(0, endedAtMs - this.startedAtMs),
+      chunks: input.chunks,
+      rows: input.rows,
+      // These are summed work durations. They can exceed mapWallMs when rows
+      // or provider calls overlap; the terminal trace must not mislabel them
+      // as a wall-clock decomposition.
+      phaseWorkMs: this.phaseWorkMs,
+      toolWorkMs: this.toolWorkMs,
+      counts: this.counts,
+    };
+  }
+}

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts

@@ -2000,8 +2000,14 @@ export class WorkerToolBatchScheduler {
     toolId: string,
     requests: WorkerToolBatchRequest[],
   ): Promise<void> {
+    const receiptPrepareStartedAt = this.options.nowMs();
     const { claimedRequests, deferredClaimedRequests } =
       await this.prepareDurableToolRequests(requests);
+    this.options.recordPerfTrace?.({
+      phase: 'runner.tool.receipt_prepare',
+      ms: this.options.nowMs() - receiptPrepareStartedAt,
+      extra: { toolId, requests: requests.length },
+    });
     await this.executeClaimedToolRequests(
       toolId,
       requests.length,
@@ -2071,6 +2077,7 @@ export class WorkerToolBatchScheduler {
         // charges tool budget, holds a global tool-concurrency slot, and
         // applies per-(org,provider) pacing before the call runs.
         let slot: { release: () => void };
+        const admissionStartedAt = this.options.nowMs();
         try {
           slot = await this.options.governor.acquireToolSlot(toolId, {
             signal: this.options.abortSignal,
@@ -2082,6 +2089,11 @@ export class WorkerToolBatchScheduler {
           );
           return;
         }
+        this.options.recordPerfTrace?.({
+          phase: 'runner.tool.admission',
+          ms: this.options.nowMs() - admissionStartedAt,
+          extra: { toolId, requests: 1 },
+        });
         try {
           let result: unknown;
           try {
@@ -2100,59 +2112,68 @@ export class WorkerToolBatchScheduler {
                   claimed,
                 })
               : null;
-            result = await executeWithWorkerReceiptHeartbeats({
-              playName: this.options.req.playName,
-              runId: this.options.req.runId,
-              runAttempt: this.options.req.runAttempt ?? 0,
-              ...(this.options.receiptStore
-                ? { receiptStore: this.options.receiptStore }
-                : {}),
-              budgetMeter: this.options.budgetMeter,
-              heartbeatIntervalMs:
-                this.options.runtimeReceiptHeartbeatIntervalMs,
-              targets: claimed.receiptKey
-                ? [
-                    {
-                      key: claimed.receiptKey,
-                      leaseId: claimed.receiptLeaseId,
-                    },
-                  ]
-                : [],
-              execute: () =>
-                this.options.executeTool({
-                  id: request.id,
-                  toolId,
-                  input: request.input,
-                  workflowStep: request.workflowStep,
-                  callbacks: this.options.callbacks,
-                  onProviderBackpressure: (retryAfterMs) =>
-                    this.reportBackpressure(toolId, retryAfterMs),
-                  onRetryAttempt: () =>
-                    this.options.governor.chargeBudget('retry'),
-                  transientHttpRetrySafe:
-                    toolContract?.retrySafeTransientHttp === true,
-                  ...(durableReceiptKey
-                    ? {
-                        durableCallReceiptKey: durableReceiptKey,
-                        executionAuthScopeDigest:
-                          request.executionAuthScopeDigest,
-                        providerIdempotencyKey:
-                          providerIdempotencyKeyForClaimed({
-                            req: this.options.req,
-                            claimed,
-                          }),
-                      }
-                    : {}),
-                  runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
-                    [claimed],
-                    {
-                      resolveOwnerTimeoutMs: (request) =>
-                        request.runtimeTimeoutMs,
-                    },
-                  ),
-                  directOptions: request.directOptions,
-                }),
-            });
+            const providerStartedAt = this.options.nowMs();
+            try {
+              result = await executeWithWorkerReceiptHeartbeats({
+                playName: this.options.req.playName,
+                runId: this.options.req.runId,
+                runAttempt: this.options.req.runAttempt ?? 0,
+                ...(this.options.receiptStore
+                  ? { receiptStore: this.options.receiptStore }
+                  : {}),
+                budgetMeter: this.options.budgetMeter,
+                heartbeatIntervalMs:
+                  this.options.runtimeReceiptHeartbeatIntervalMs,
+                targets: claimed.receiptKey
+                  ? [
+                      {
+                        key: claimed.receiptKey,
+                        leaseId: claimed.receiptLeaseId,
+                      },
+                    ]
+                  : [],
+                execute: () =>
+                  this.options.executeTool({
+                    id: request.id,
+                    toolId,
+                    input: request.input,
+                    workflowStep: request.workflowStep,
+                    callbacks: this.options.callbacks,
+                    onProviderBackpressure: (retryAfterMs) =>
+                      this.reportBackpressure(toolId, retryAfterMs),
+                    onRetryAttempt: () =>
+                      this.options.governor.chargeBudget('retry'),
+                    transientHttpRetrySafe:
+                      toolContract?.retrySafeTransientHttp === true,
+                    ...(durableReceiptKey
+                      ? {
+                          durableCallReceiptKey: durableReceiptKey,
+                          executionAuthScopeDigest:
+                            request.executionAuthScopeDigest,
+                          providerIdempotencyKey:
+                            providerIdempotencyKeyForClaimed({
+                              req: this.options.req,
+                              claimed,
+                            }),
+                        }
+                      : {}),
+                    runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
+                      [claimed],
+                      {
+                        resolveOwnerTimeoutMs: (request) =>
+                          request.runtimeTimeoutMs,
+                      },
+                    ),
+                    directOptions: request.directOptions,
+                  }),
+              });
+            } finally {
+              this.options.recordPerfTrace?.({
+                phase: 'runner.tool.provider',
+                ms: this.options.nowMs() - providerStartedAt,
+                extra: { toolId, requests: 1 },
+              });
+            }
           } catch (error) {
             if (error instanceof RuntimeLeaseLostError) {
               this.rejectRequests(claimed, error);
@@ -2173,10 +2194,16 @@ export class WorkerToolBatchScheduler {
             return;
           }
           try {
+            const receiptCompleteStartedAt = this.options.nowMs();
             this.settleRequests(
               claimed,
               await this.completeDurableToolRequest(claimed, result),
             );
+            this.options.recordPerfTrace?.({
+              phase: 'runner.tool.receipt_complete',
+              ms: this.options.nowMs() - receiptCompleteStartedAt,
+              extra: { toolId, requests: 1 },
+            });
           } catch (receiptError) {
             this.rejectRequests(claimed, receiptError);
           }
@@ -2261,12 +2288,22 @@ export class WorkerToolBatchScheduler {
         }
         // One provider call per batch -> one tool slot (budget + global
         // concurrency + per-(org,provider) pacing) around the whole batch.
+        const admissionStartedAt = this.options.nowMs();
         const slot = await this.options.governor.acquireToolSlot(
           batch.batchOperation,
           {
             signal: this.options.abortSignal,
           },
         );
+        this.options.recordPerfTrace?.({
+          phase: 'runner.tool.admission',
+          ms: this.options.nowMs() - admissionStartedAt,
+          extra: {
+            toolId: batch.batchOperation,
+            requests: batch.memberRequests.length,
+            batched: true,
+          },
+        });
         try {
           await this.markAdmittedToolRequestsRunning(batch.memberRequests);
           this.options.budgetMeter?.count('provider');
@@ -2281,54 +2318,69 @@ export class WorkerToolBatchScheduler {
             batchOperation: batch.batchOperation,
             receiptKeys,
           });
-          return await executeWithWorkerReceiptHeartbeats({
-            playName: this.options.req.playName,
-            runId: this.options.req.runId,
-            runAttempt: this.options.req.runAttempt ?? 0,
-            ...(this.options.receiptStore
-              ? { receiptStore: this.options.receiptStore }
-              : {}),
-            budgetMeter: this.options.budgetMeter,
-            heartbeatIntervalMs: this.options.runtimeReceiptHeartbeatIntervalMs,
-            targets: batch.memberRequests.map((member) => ({
-              key: member.receiptKey,
-              leaseId: member.receiptLeaseId,
-            })),
-            execute: () =>
-              this.options.executeTool({
-                id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
-                toolId: batch.batchOperation,
-                input: batch.batchPayload,
-                callbacks: this.options.callbacks,
-                onProviderBackpressure: (retryAfterMs) =>
-                  this.reportBackpressure(input.sourceToolId, retryAfterMs),
-                onRetryAttempt: () =>
-                  this.options.governor.chargeBudget('retry'),
-                transientHttpRetrySafe:
-                  toolContract?.retrySafeTransientHttp === true,
-                durableCallReceiptKey: aggregateReceiptKey,
-                executionAuthScopeDigest:
-                  batch.memberRequests[0]?.request.executionAuthScopeDigest ??
-                  null,
-                providerIdempotencyKey: buildBatchProviderIdempotencyKey({
-                  aggregateReceiptKey,
-                  receiptKeys,
-                  providerIdempotencyKeys: batch.memberRequests.map((member) =>
-                    providerIdempotencyKeyForClaimed({
-                      req: this.options.req,
-                      claimed: member,
-                    }),
+          const providerStartedAt = this.options.nowMs();
+          try {
+            return await executeWithWorkerReceiptHeartbeats({
+              playName: this.options.req.playName,
+              runId: this.options.req.runId,
+              runAttempt: this.options.req.runAttempt ?? 0,
+              ...(this.options.receiptStore
+                ? { receiptStore: this.options.receiptStore }
+                : {}),
+              budgetMeter: this.options.budgetMeter,
+              heartbeatIntervalMs:
+                this.options.runtimeReceiptHeartbeatIntervalMs,
+              targets: batch.memberRequests.map((member) => ({
+                key: member.receiptKey,
+                leaseId: member.receiptLeaseId,
+              })),
+              execute: () =>
+                this.options.executeTool({
+                  id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
+                  toolId: batch.batchOperation,
+                  input: batch.batchPayload,
+                  callbacks: this.options.callbacks,
+                  onProviderBackpressure: (retryAfterMs) =>
+                    this.reportBackpressure(input.sourceToolId, retryAfterMs),
+                  onRetryAttempt: () =>
+                    this.options.governor.chargeBudget('retry'),
+                  transientHttpRetrySafe:
+                    toolContract?.retrySafeTransientHttp === true,
+                  durableCallReceiptKey: aggregateReceiptKey,
+                  executionAuthScopeDigest:
+                    batch.memberRequests[0]?.request.executionAuthScopeDigest ??
+                    null,
+                  providerIdempotencyKey: buildBatchProviderIdempotencyKey({
+                    aggregateReceiptKey,
+                    receiptKeys,
+                    providerIdempotencyKeys: batch.memberRequests.map(
+                      (member) =>
+                        providerIdempotencyKeyForClaimed({
+                          req: this.options.req,
+                          claimed: member,
+                        }),
+                    ),
+                  }),
+                  runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
+                    batch.memberRequests,
+                    {
+                      resolveOwnerTimeoutMs: (request) =>
+                        request.runtimeTimeoutMs,
+                    },
                   ),
                 }),
-                runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
-                  batch.memberRequests,
-                  {
-                    resolveOwnerTimeoutMs: (request) =>
-                      request.runtimeTimeoutMs,
-                  },
-                ),
-              }),
-          });
+            });
+          } finally {
+            this.options.recordPerfTrace?.({
+              phase: 'runner.tool.provider',
+              ms: this.options.nowMs() - providerStartedAt,
+              extra: {
+                toolId: batch.batchOperation,
+                requests: batch.memberRequests.length,
+                batched: true,
+              },
+            });
+          }
         } finally {
           slot.release();
         }
@@ -2389,6 +2441,7 @@ export class WorkerToolBatchScheduler {
               : entry.request.memberRequests.map(() => null);
           let completedResults: WorkerToolCompletionResult[];
           try {
+            const receiptCompleteStartedAt = this.options.nowMs();
             completedResults = await this.completeDurableToolRequests(
               entry.request.memberRequests.map((claimed, index) => ({
                 claimed,
@@ -2399,6 +2452,15 @@ export class WorkerToolBatchScheduler {
                 ),
               })),
             );
+            this.options.recordPerfTrace?.({
+              phase: 'runner.tool.receipt_complete',
+              ms: this.options.nowMs() - receiptCompleteStartedAt,
+              extra: {
+                toolId: entry.request.batchOperation,
+                requests: entry.request.memberRequests.length,
+                batched: true,
+              },
+            });
           } catch (receiptError) {
             for (const claimed of entry.request.memberRequests) {
               this.rejectRequests(claimed, receiptError);

package/dist/bundling-sources/sdk/src/play.ts

@@ -173,6 +173,8 @@ export type PlayBindings = {
    * older clients can continue to register revisions during the migration.
    */
   description?: string;
+  /** Allow compilers to bundle this named handler directly without a child run. */
+  inline?: boolean;
   /** Optional per-run billing controls enforced by the runtime. */
   billing?: {
     /** Stop the run before a billed action would push total run credits above this cap. */

package/dist/bundling-sources/sdk/src/release.ts

@@ -106,10 +106,10 @@ export const SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: '0.1.210',
+  version: '0.1.211',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.210',
+    latest: '0.1.211',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [