deepline 0.1.209 → 0.1.211

package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts

@@ -2445,8 +2445,6 @@ function runRequestFromPlayWorkflowParams(
     preloadedDbSessions: params.preloadedDbSessions ?? null,
     runtimeTestFaultHeader: params.runtimeTestFaultHeader ?? null,
     testPolicyOverrides: params.testPolicyOverrides ?? null,
-    inlineChildRunRegistered:
-      params.runtimeBackend === 'cf_workflows_dynamic_worker_inline_child',
     coordinatorUrl: params.coordinatorUrl ?? null,
     coordinatorInternalToken: params.coordinatorInternalToken ?? null,
     totalRows: params.totalRows,

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -32,6 +32,7 @@ import {
   type WorkflowEvent,
   type WorkflowStep,
 } from 'cloudflare:workers';
+import { AsyncLocalStorage } from 'node:async_hooks';
 import {
   deterministicMapChunkStepName,
   type ExecutionPlan,
@@ -40,7 +41,6 @@ import {
   STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
   STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
 } from '../../../shared_libs/play-runtime/runtime-constants';
-import { childSubmitIdempotencyKey } from '../../../shared_libs/play-runtime/child-run-id';
 import {
   createPlayExecutionGovernor,
   type GovernanceSnapshot,
@@ -104,7 +104,7 @@ import {
 import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
 import {
   buildDurableCtxCallCacheKey,
-  buildDurableRunPlaySemanticKey,
+  buildDurableRunPlayInvocationScope,
 } from '../../../shared_libs/play-runtime/durable-call-cache';
 import {
   resolveRuntimeToolReceiptWaitMaxAttempts,
@@ -166,6 +166,7 @@ import {
   createWorkerWorkBudgetMeter,
   type WorkerWorkBudgetMeter,
 } from './runtime/work-budget';
+import { MapLatencyProfile } from './runtime/map-latency-profile';
 import {
   WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES,
   WorkerRunWorkDispatcher,
@@ -368,8 +369,6 @@ type RunRequest = {
   /** Internal ctx.runPlay lineage. Public SDK/users never see this. */
   playCallGovernance?: PlayCallGovernanceSnapshot | null;
   preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
-  /** Coordinator already created the child run row before invoking /run-inline. */
-  inlineChildRunRegistered?: boolean | null;
   /** Cloudflare coordinator URL for direct Workflow control-plane signals. */
   coordinatorUrl?: string | null;
   /** Request-scoped coordinator auth token for preview/dev direct control calls. */
@@ -1148,11 +1147,29 @@ async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
 
 let workerRequestSequence = 0;
 
+const workerRunPlayRowContext = new AsyncLocalStorage<{
+  tableNamespace: string;
+  rowKey: string;
+  fieldName: string;
+}>();
+
 function makeRequestId(): string {
   workerRequestSequence = (workerRequestSequence + 1) % Number.MAX_SAFE_INTEGER;
   return `worker-request-${workerRequestSequence}`;
 }
 
+async function executeFreshChildInvocation<T>(
+  input: {
+    parentRunId: string;
+    key: string;
+    invocationScope: string;
+  },
+  execute: (invocationId: string) => Promise<T>,
+): Promise<T> {
+  const invocationId = `${input.parentRunId}:runPlay:${input.key}:${input.invocationScope}`;
+  return await execute(invocationId);
+}
+
 function makeRuntimeSheetAttempt(input: {
   runId: string;
   runAttempt?: number | null;
@@ -1665,27 +1682,6 @@ async function hashChildPlayEventKey(input: unknown): Promise<string> {
   return (await hashJson(input)).slice(0, 32);
 }
 
-function workerChildRevisionFingerprint(input: {
-  playName: string;
-  manifest: PlayRuntimeManifest | null | undefined;
-}): string | null {
-  const manifest = input.manifest;
-  if (!manifest) return null;
-  return sha256Hex(
-    stableStringify({
-      playId: manifest.playName?.trim() || input.playName,
-      codeFormat: 'cjs_module',
-      artifactHash: manifest.artifactHash ?? null,
-      graphHash: manifest.graphHash ?? null,
-      sourceHash: null,
-      sourceCodeHash:
-        typeof manifest.sourceCode === 'string'
-          ? sha256Hex(manifest.sourceCode)
-          : null,
-    }),
-  );
-}
-
 type RuntimeResolvedPlayResponse = {
   manifest?: PlayRuntimeManifest | null;
 };
@@ -4603,6 +4599,11 @@ function createMinimalWorkerCtx(
     opts?: WorkerMapOptions,
   ): Promise<unknown> => {
     const mapStartedAt = nowMs();
+    const mapLatencyProfile = new MapLatencyProfile(
+      req.testPolicyOverrides?.mapLatencyProfile === true,
+      name,
+      mapStartedAt,
+    );
     const mapNodeId = `map:${name}`;
     const inputRows = rows;
     const rowCountHint = datasetRowCountHint(inputRows);
@@ -4940,6 +4941,7 @@ function createMinimalWorkerCtx(
           blockedRows: prepared.blockedRows.length,
         },
       });
+      mapLatencyProfile.recordPhase('prepare_rows', nowMs() - prepareStartedAt);
       const progressTotalRows = rowCountHint ?? chunkRows.length;
       const preparedCompletedRows = Math.min(
         progressTotalRows,
@@ -5220,7 +5222,10 @@ function createMinimalWorkerCtx(
         deserializeDurableValue: deserializeDurableStepValue,
         nowMs,
         batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
-        recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
+        recordPerfTrace: (trace) => {
+          mapLatencyProfile.recordToolTrace(trace);
+          recordRunnerPerfTrace({ req, ...trace });
+        },
         abortSignal,
         onRequestsSettled: reportSettledToolRequests,
         callbacks,
@@ -5510,9 +5515,16 @@ function createMinimalWorkerCtx(
               if (circuitError) throw circuitError;
               const myIndex = idx++;
               if (myIndex >= rowsToExecute.length) return;
+              const rowQueueStartedAt = mapLatencyProfile.enabled ? nowMs() : 0;
               const rowSlot = await governor.acquireRowSlot({
                 signal: abortSignal,
               });
+              if (mapLatencyProfile.enabled) {
+                mapLatencyProfile.recordRowQueue(nowMs() - rowQueueStartedAt);
+              }
+              const rowExecutionStartedAt = mapLatencyProfile.enabled
+                ? nowMs()
+                : 0;
               let rowMarkedActive = false;
               try {
                 startedExecutedRows += 1;
@@ -5580,37 +5592,49 @@ function createMinimalWorkerCtx(
                             })
                           : null,
                     });
-                    const resolved = await executeWorkerStepResolver(
-                      value,
-                      enriched,
-                      rowCtx,
-                      absoluteIndex,
-                      previousCell,
-                      isWorkerStepProgram(value)
-                        ? {
-                            parentField: key,
-                            path: [],
-                            outputs: stepProgramOutputs,
-                            onOutput: async (stepOutput) => {
-                              generatedOutputFields.add(stepOutput.columnName);
-                              const status = stepOutput.status ?? 'completed';
-                              if (status === 'skipped') stepCellsSkipped += 1;
-                              else stepCellsCompleted += 1;
-                              await enqueueLiveRowUpdate({
-                                key: entry.rowKey,
-                                dataPatch: {
-                                  [stepOutput.columnName]: stepOutput.value,
-                                },
-                                cellMetaPatch: {
-                                  [stepOutput.columnName]: {
-                                    status,
-                                    stage: stepOutput.stepId,
-                                  },
+                    const resolved = await workerRunPlayRowContext.run(
+                      {
+                        tableNamespace: name,
+                        rowKey: entry.rowKey,
+                        fieldName: key,
+                      },
+                      () =>
+                        executeWorkerStepResolver(
+                          value,
+                          enriched,
+                          rowCtx,
+                          absoluteIndex,
+                          previousCell,
+                          isWorkerStepProgram(value)
+                            ? {
+                                parentField: key,
+                                path: [],
+                                outputs: stepProgramOutputs,
+                                onOutput: async (stepOutput) => {
+                                  generatedOutputFields.add(
+                                    stepOutput.columnName,
+                                  );
+                                  const status =
+                                    stepOutput.status ?? 'completed';
+                                  if (status === 'skipped')
+                                    stepCellsSkipped += 1;
+                                  else stepCellsCompleted += 1;
+                                  await enqueueLiveRowUpdate({
+                                    key: entry.rowKey,
+                                    dataPatch: {
+                                      [stepOutput.columnName]: stepOutput.value,
+                                    },
+                                    cellMetaPatch: {
+                                      [stepOutput.columnName]: {
+                                        status,
+                                        stage: stepOutput.stepId,
+                                      },
+                                    },
+                                  });
                                 },
-                              });
-                            },
-                          }
-                        : undefined,
+                              }
+                            : undefined,
+                        ),
                     );
                     enriched[key] = resolved.value;
                     fieldOutputs[key] = resolved.value;
@@ -5737,6 +5761,11 @@ function createMinimalWorkerCtx(
                   // single clean run failure.
                 }
               } finally {
+                if (mapLatencyProfile.enabled) {
+                  mapLatencyProfile.recordRowExecution(
+                    nowMs() - rowExecutionStartedAt,
+                  );
+                }
                 if (rowMarkedActive) {
                   activeExecutedRows = Math.max(0, activeExecutedRows - 1);
                   reportExecutionHeartbeat(false);
@@ -5789,6 +5818,10 @@ function createMinimalWorkerCtx(
           concurrency,
         },
       });
+      mapLatencyProfile.recordPhase(
+        'worker_execution',
+        nowMs() - workersStartedAt,
+      );
       const buildRowFailureSamples = () =>
         failedRowEntries
           .map((failure, executedIndex) =>
@@ -5916,6 +5949,10 @@ function createMinimalWorkerCtx(
             rowsToExecute: rowsToExecute.length,
           },
         });
+        mapLatencyProfile.recordPhase(
+          'persist_rows',
+          nowMs() - persistRowsStartedAt,
+        );
       } catch (error) {
         if (
           workflowStep &&
@@ -6021,6 +6058,7 @@ function createMinimalWorkerCtx(
         ms: nowMs() - hashStartedAt,
         extra: { mapName: name, chunkIndex, rows: out.length },
       });
+      mapLatencyProfile.recordPhase('hash', nowMs() - hashStartedAt);
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.total',
@@ -6636,6 +6674,27 @@ function createMinimalWorkerCtx(
         chunks: dispatchResult.chunksExecuted,
       },
     });
+    if (mapLatencyProfile.enabled) {
+      const endedAt = nowMs();
+      const summary = mapLatencyProfile.summary(endedAt, {
+        chunks: dispatchResult.chunksExecuted,
+        rows: totalRowsWritten,
+      });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map.profile',
+        ms: endedAt - mapStartedAt,
+        extra: summary,
+      });
+      // Local Absurd does not always forward dynamic-worker perf traces. Keep
+      // one bounded copy in the durable run log for the diagnostic command.
+      emitEvent({
+        type: 'log',
+        level: 'info',
+        message: `[perf] map_latency_profile ${JSON.stringify(summary)}`,
+        ts: endedAt,
+      });
+    }
     return dataset;
   };
 
@@ -6933,7 +6992,6 @@ function createMinimalWorkerCtx(
       options?: {
         description?: string;
         timeoutMs?: number;
-        staleAfterSeconds?: number;
       },
     ): Promise<unknown> {
       const normalizedKey = normalizeContextKey(key, 'runPlay');
@@ -6947,34 +7005,49 @@ function createMinimalWorkerCtx(
         playRef: resolvedName,
         cache: childManifestCache,
       });
-      const receiptKey = buildDurableCtxCallCacheKey({
-        orgId: req.orgId,
-        playId: req.playName,
-        kind: 'runPlay',
-        id: normalizedKey,
-        semanticKey: buildDurableRunPlaySemanticKey({
-          childPlayName: resolvedName,
-          childRevisionFingerprint: workerChildRevisionFingerprint({
-            playName: resolvedName,
-            manifest: childManifest,
-          }),
-          input,
-        }),
-        staleAfterSeconds: options?.staleAfterSeconds,
+      const rowScope = workerRunPlayRowContext.getStore();
+      const invocationScope = buildDurableRunPlayInvocationScope({
+        childPlayName: resolvedName,
+        input,
+        rowScope: rowScope
+          ? {
+              fieldName: rowScope.fieldName,
+              rowKey: rowScope.rowKey,
+              tableNamespace: rowScope.tableNamespace,
+            }
+          : null,
       });
-      return await executeWithRuntimeReceipt(
-        receiptKey,
-        async (receiptContext, wasFailed) => {
+      return await executeFreshChildInvocation(
+        {
+          parentRunId: req.runId,
+          key: normalizedKey,
+          invocationScope,
+        },
+        async (childInvocationId) => {
           // The Governor owns the play-call lineage: forkChild does the cycle
           // guard, depth/per-parent/playCall/descendant budget charges, and
           // returns the snapshot to thread into the child so budgets accumulate
-          // across isolates. Charged inside the receipt boundary so a replay
-          // (cache hit) never double-charges.
-          const childRunId = `${req.runId}:child:${normalizedKey}`;
-          const childGovernance = await governor.forkChild({
-            childPlayName: resolvedName,
-            childRunId,
-          });
+          // across isolates. Every ctx.runPlay call is a fresh child invocation.
+          const childRunId = `${req.runId}:child:${childInvocationId}`;
+          // Checkpoint governance admission separately from the child result.
+          // Workflow replay reuses this admission and the coordinator's stable
+          // child launch identity, but ctx.runPlay never caches child output.
+          const forkChild = () =>
+            governor.forkChild({
+              childPlayName: resolvedName,
+              childRunId,
+            });
+          const childGovernance = workflowStep
+            ? await (
+                workflowStep.do as unknown as <T>(
+                  name: string,
+                  callback: () => Promise<T>,
+                ) => Promise<T>
+              )(
+                `runPlay-governance:${sha256Hex(childInvocationId).slice(0, 24)}`,
+                forkChild,
+              )
+            : await forkChild();
           const nextDepth = childGovernance.callDepth;
           const nextParentCalls =
             governor.snapshot().parentChildCalls[req.playName] ?? 0;
@@ -7047,11 +7120,7 @@ function createMinimalWorkerCtx(
                   options?.timeoutMs == null && !childNeedsWorkflowScheduler,
                 body: {
                   name: resolvedName,
-                  childIdempotencyKey: childSubmitIdempotencyKey({
-                    receiptKey,
-                    wasFailed,
-                    leaseId: receiptContext.leaseId,
-                  }),
+                  childIdempotencyKey: childInvocationId,
                   input: isRecord(input) ? input : {},
                   orgId: req.orgId,
                   callbackUrl: req.callbackUrl,
@@ -7329,12 +7398,6 @@ function createMinimalWorkerCtx(
             childPlaySlot?.release();
           }
         },
-        {
-          forceFailedRefresh: true,
-          repairRunningReceiptForSameRunAfterWaitTimeout: true,
-          runningReceiptWaitMaxAttempts:
-            resolveRuntimeToolReceiptWaitMaxAttempts(input),
-        },
       );
     },
     async fetch(
@@ -7615,13 +7678,6 @@ async function handleRunInline(
     const probeStartedAt = nowMs();
     await probeHarnessOnce(env, runPrefix);
     traceInline('inline.probe_harness', probeStartedAt);
-    if (!req.inlineChildRunRegistered) {
-      const registerStartedAt = nowMs();
-      await registerInlineChildRun(req);
-      traceInline('inline.register_child_run', registerStartedAt);
-    } else {
-      traceInline('inline.register_child_run', nowMs(), { skipped: true });
-    }
     const executeStartedAt = nowMs();
     const output = await executeRunRequest(
       req,
@@ -7667,46 +7723,6 @@ async function handleRunInline(
   }
 }
 
-async function registerInlineChildRun(req: RunRequest): Promise<void> {
-  const snapshot = isRecord(req.contractSnapshot) ? req.contractSnapshot : {};
-  const artifactMetadata = isRecord(snapshot.artifactMetadata)
-    ? snapshot.artifactMetadata
-    : {};
-  const governance = req.playCallGovernance;
-  await postRuntimeApi(req.baseUrl, req.executorToken, {
-    action: 'start_inline_child_run',
-    playName: req.playName,
-    runId: req.runId,
-    parentRunId: governance?.parentRunId,
-    rootRunId: governance?.rootRunId,
-    workflowFamilyKey:
-      governance?.rootRunId ?? governance?.parentRunId ?? req.runId,
-    artifactStorageKey:
-      typeof artifactMetadata.storageKey === 'string'
-        ? artifactMetadata.storageKey
-        : undefined,
-    artifactHash:
-      typeof artifactMetadata.artifactHash === 'string'
-        ? artifactMetadata.artifactHash
-        : undefined,
-    graphHash:
-      typeof artifactMetadata.graphHash === 'string'
-        ? artifactMetadata.graphHash
-        : undefined,
-    runtimeBackend: 'workers_edge',
-    schedulerBackend: 'inline_child',
-    executionProfile: 'workers_edge',
-    maxCreditsPerRun: extractMaxCreditsPerRun(req.contractSnapshot),
-    staticPipeline: snapshot.staticPipeline ?? null,
-    source:
-      snapshot.source === 'published' ||
-      snapshot.source === 'ad_hoc' ||
-      snapshot.source === 'draft'
-        ? snapshot.source
-        : 'published',
-  });
-}
-
 /** Cap on run log lines retained in the terminal output compatibility shape. */
 const RUN_LOG_BUFFER_LIMIT = 500;
 /** Min wall-clock interval between live run-ledger flushes during a run. */

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts

@@ -337,6 +337,21 @@ function countToolSubstep(substep: PlayStaticSubstep): MapToolStats {
         minProviderBranchCoalescedCount(branchStats),
       ];
     }
+    if (substep.kind === 'loop') {
+      // The static extractor records every switch arm in `steps` when a loop
+      // walks a fixed provider list. Each arm can execute on a different
+      // iteration, so using the conditional's max-one-branch estimate would
+      // budget a seven-provider waterfall as one provider call per row.
+      return substep.steps.reduce<MapToolStats>((stats, step) => {
+        const stepStats =
+          step.type === 'control_flow' &&
+          step.kind === 'conditional' &&
+          step.steps.length > 0
+            ? countToolSubsteps(step.steps)
+            : countToolSubstep(step);
+        return addStats(stats, stepStats);
+      }, [0, 0, 0, 0, 0]);
+    }
     return countToolSubsteps(substep.steps);
   }
 

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts

@@ -0,0 +1,90 @@
+type ToolTrace = {
+  phase: string;
+  ms?: number;
+  extra?: Record<string, unknown>;
+};
+
+type Totals = Record<string, number>;
+
+function add(totals: Totals, key: string, ms: number | undefined): void {
+  if (typeof ms !== 'number' || !Number.isFinite(ms) || ms < 0) return;
+  totals[key] = (totals[key] ?? 0) + ms;
+}
+
+/**
+ * Opt-in map profiler. It aggregates in memory and produces one terminal
+ * trace, so normal runs do not pay per-row/per-tool telemetry cost.
+ */
+export class MapLatencyProfile {
+  private readonly phaseWorkMs: Totals = {};
+  private readonly toolWorkMs: Totals = {};
+  private readonly counts: Totals = {};
+
+  constructor(
+    readonly enabled: boolean,
+    private readonly mapName: string,
+    private readonly startedAtMs: number,
+  ) {}
+
+  recordPhase(phase: string, ms: number, count = 1): void {
+    if (!this.enabled) return;
+    add(this.phaseWorkMs, phase, ms);
+    this.counts[`${phase}_count`] =
+      (this.counts[`${phase}_count`] ?? 0) + count;
+  }
+
+  recordRowQueue(ms: number): void {
+    this.recordPhase('row_slot_queue', ms);
+  }
+
+  recordRowExecution(ms: number): void {
+    this.recordPhase('row_execution', ms);
+  }
+
+  recordToolTrace(trace: ToolTrace): void {
+    if (!this.enabled) return;
+    switch (trace.phase) {
+      case 'runner.tool.request':
+        add(this.toolWorkMs, 'request_end_to_end', trace.ms);
+        this.counts.tool_requests = (this.counts.tool_requests ?? 0) + 1;
+        break;
+      case 'runner.tool.drain':
+        add(this.toolWorkMs, 'drain', trace.ms);
+        this.counts.tool_drains = (this.counts.tool_drains ?? 0) + 1;
+        break;
+      case 'runner.tool.group':
+        add(this.toolWorkMs, 'group', trace.ms);
+        this.counts.tool_groups = (this.counts.tool_groups ?? 0) + 1;
+        break;
+      case 'runner.tool.receipt_prepare':
+        add(this.toolWorkMs, 'receipt_prepare', trace.ms);
+        break;
+      case 'runner.tool.admission':
+        add(this.toolWorkMs, 'admission_wait', trace.ms);
+        break;
+      case 'runner.tool.provider':
+        add(this.toolWorkMs, 'provider_execute', trace.ms);
+        break;
+      case 'runner.tool.receipt_complete':
+        add(this.toolWorkMs, 'receipt_complete', trace.ms);
+        break;
+      default:
+        return;
+    }
+  }
+
+  summary(endedAtMs: number, input: { chunks: number; rows: number }) {
+    return {
+      mapName: this.mapName,
+      mapWallMs: Math.max(0, endedAtMs - this.startedAtMs),
+      chunks: input.chunks,
+      rows: input.rows,
+      // These are summed work durations. They can exceed mapWallMs when rows
+      // or provider calls overlap; the terminal trace must not mislabel them
+      // as a wall-clock decomposition.
+      phaseWorkMs: this.phaseWorkMs,
+      toolWorkMs: this.toolWorkMs,
+      counts: this.counts,
+    };
+  }
+}