deepline 0.1.110 → 0.1.112

package/dist/index.js

@@ -274,10 +274,10 @@ var SDK_RELEASE = {
   // skill on the sdk sync surface, and the people-search-to-email prebuilt.
   // 0.1.108 ships explicit dataset column/tool recompute policy and removes
   // the SDK enrich generator's one-second stale policy.
-  version: "0.1.110",
+  version: "0.1.112",
   apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
   supportPolicy: {
-    latest: "0.1.110",
+    latest: "0.1.112",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [

package/dist/index.mjs

@@ -196,10 +196,10 @@ var SDK_RELEASE = {
   // skill on the sdk sync surface, and the people-search-to-email prebuilt.
   // 0.1.108 ships explicit dataset column/tool recompute policy and removes
   // the SDK enrich generator's one-second stale policy.
-  version: "0.1.110",
+  version: "0.1.112",
   apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
   supportPolicy: {
-    latest: "0.1.110",
+    latest: "0.1.112",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -2898,150 +2898,181 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
       },
     });
 
-    return dispatchWorkflow(
-      { env: this.env, ctx: this.ctx },
-      dispatchedEvent as Parameters<typeof dispatchWorkflow>[1],
-      step as Parameters<typeof dispatchWorkflow>[2],
-      async ({ metadata, env }) => {
-        const graphHash = readMetadataString(metadata, 'graphHash');
-        const artifactStorageKey = readMetadataString(
-          metadata,
-          'artifactStorageKey',
-        );
-        const runIdForTrace =
-          typeof (metadata as Record<string, unknown>).runId === 'string'
-            ? ((metadata as Record<string, unknown>).runId as string)
-            : graphHash;
-        const loaderStartedAt = Date.now();
-        trace({
-          runId: runIdForTrace,
-          phase: 'coordinator.loader_callback_entry',
-          ms: 0,
-          graphHash,
-          extra: { artifactStorageKey },
-        });
-        const stub = loadDynamicPlayWorkerSync(
-          env,
-          {
+    const dispatchWorkflowStartedAt = Date.now();
+    try {
+      return await dispatchWorkflow(
+        { env: this.env, ctx: this.ctx },
+        dispatchedEvent as Parameters<typeof dispatchWorkflow>[1],
+        step as Parameters<typeof dispatchWorkflow>[2],
+        async ({ metadata, env }) => {
+          const graphHash = readMetadataString(metadata, 'graphHash');
+          const artifactStorageKey = readMetadataString(
+            metadata,
+            'artifactStorageKey',
+          );
+          const runIdForTrace =
+            typeof (metadata as Record<string, unknown>).runId === 'string'
+              ? ((metadata as Record<string, unknown>).runId as string)
+              : graphHash;
+          const loaderStartedAt = Date.now();
+          trace({
             runId: runIdForTrace,
+            phase: 'coordinator.loader_callback_entry',
+            ms: 0,
             graphHash,
-            artifactStorageKey,
-            artifactHash:
-              typeof metadata.artifactHash === 'string'
-                ? metadata.artifactHash
-                : null,
-            dynamicWorkerCode:
-              typeof metadata.dynamicWorkerCode === 'string'
-                ? metadata.dynamicWorkerCode
-                : null,
-            packagedFiles: normalizePackagedFiles(metadata.packagedFiles),
-          },
-          trace,
-        );
-        const entrypoint = stub.getEntrypoint(
-          'TenantWorkflow',
-        ) as unknown as WorkflowRunner;
-        trace({
-          runId: runIdForTrace,
-          phase: 'coordinator.loader_compile',
-          ms: Date.now() - loaderStartedAt,
-          graphHash,
-        });
-        // Wrap the entrypoint so its run() failure surfaces here rather
-        // than disappearing into the framework's silent rpcMethod=run
-        // exception path.
-        return {
-          run: async (innerEvent: unknown, innerStep: unknown) => {
-            const innerStartedAt = Date.now();
-            trace({
+            extra: { artifactStorageKey },
+          });
+          const stub = loadDynamicPlayWorkerSync(
+            env,
+            {
               runId: runIdForTrace,
-              phase: 'coordinator.runner_run_start',
-              ms: 0,
               graphHash,
-            });
-            try {
-              const result = await (
-                entrypoint as unknown as {
-                  run(e: unknown, s: unknown): Promise<unknown>;
-                }
-              ).run(innerEvent, innerStep);
-              const output = isRecord(result) ? result : null;
-              try {
-                await writeCoordinatorTerminalState(env, {
-                  runId: runIdForTrace,
-                  status: 'completed',
-                  result: output?.result ?? result,
-                  totalRows: output?.totalRows ?? output?.outputRows ?? null,
-                  durationMs: output?.durationMs ?? null,
-                  playName:
-                    typeof output?.playName === 'string'
-                      ? output.playName
-                      : null,
-                  liveLogs: sanitizeLiveLogLines(output?.liveLogs),
-                  liveNodeProgress: output?.liveNodeProgress ?? null,
-                });
-              } catch (terminalError) {
-                console.warn(
-                  '[coordinator] completed terminal cache write failed; preserving completed run',
-                  {
-                    graphHash,
-                    runId: runIdForTrace,
-                    message:
-                      terminalError instanceof Error
-                        ? terminalError.message
-                        : String(terminalError),
-                  },
-                );
-              }
+              artifactStorageKey,
+              artifactHash:
+                typeof metadata.artifactHash === 'string'
+                  ? metadata.artifactHash
+                  : null,
+              dynamicWorkerCode:
+                typeof metadata.dynamicWorkerCode === 'string'
+                  ? metadata.dynamicWorkerCode
+                  : null,
+              packagedFiles: normalizePackagedFiles(metadata.packagedFiles),
+            },
+            trace,
+          );
+          const entrypointStartedAt = Date.now();
+          const entrypoint = stub.getEntrypoint(
+            'TenantWorkflow',
+          ) as unknown as WorkflowRunner;
+          trace({
+            runId: runIdForTrace,
+            phase: 'coordinator.loader_get_entrypoint',
+            ms: Date.now() - entrypointStartedAt,
+            graphHash,
+          });
+          trace({
+            runId: runIdForTrace,
+            phase: 'coordinator.loader_compile',
+            ms: Date.now() - loaderStartedAt,
+            graphHash,
+          });
+          // Wrap the entrypoint so its run() failure surfaces here rather
+          // than disappearing into the framework's silent rpcMethod=run
+          // exception path.
+          return {
+            run: async (innerEvent: unknown, innerStep: unknown) => {
+              const innerStartedAt = Date.now();
               trace({
                 runId: runIdForTrace,
-                phase: 'coordinator.runner_run',
-                ms: Date.now() - innerStartedAt,
+                phase: 'coordinator.runner_run_start',
+                ms: 0,
                 graphHash,
               });
-              return result;
-            } catch (innerError) {
-              const failure = normalizePlayRunFailure(innerError);
-              console.error('[coordinator] DynamicWorkflow runner.run threw', {
-                graphHash,
-                message:
-                  innerError instanceof Error
-                    ? innerError.message
-                    : String(innerError),
-                name: innerError instanceof Error ? innerError.name : null,
-                stack:
-                  innerError instanceof Error &&
-                  typeof innerError.stack === 'string'
-                    ? innerError.stack.split('\n').slice(0, 12).join('\n')
-                    : null,
-              });
-              await markWorkflowRuntimeFailure({
-                env,
-                event: innerEvent,
-                error: innerError,
-              }).catch((markError) => {
+              try {
+                const result = await (
+                  entrypoint as unknown as {
+                    run(e: unknown, s: unknown): Promise<unknown>;
+                  }
+                ).run(innerEvent, innerStep);
+                const output = isRecord(result) ? result : null;
+                const terminalStartedAt = Date.now();
+                try {
+                  await writeCoordinatorTerminalState(env, {
+                    runId: runIdForTrace,
+                    status: 'completed',
+                    result: output?.result ?? result,
+                    totalRows: output?.totalRows ?? output?.outputRows ?? null,
+                    durationMs: output?.durationMs ?? null,
+                    playName:
+                      typeof output?.playName === 'string'
+                        ? output.playName
+                        : null,
+                    liveLogs: sanitizeLiveLogLines(output?.liveLogs),
+                    liveNodeProgress: output?.liveNodeProgress ?? null,
+                  });
+                  trace({
+                    runId: runIdForTrace,
+                    phase: 'coordinator.terminal_state_write',
+                    ms: Date.now() - terminalStartedAt,
+                    graphHash,
+                    extra: { status: 'completed' },
+                  });
+                } catch (terminalError) {
+                  console.warn(
+                    '[coordinator] completed terminal cache write failed; preserving completed run',
+                    {
+                      graphHash,
+                      runId: runIdForTrace,
+                      message:
+                        terminalError instanceof Error
+                          ? terminalError.message
+                          : String(terminalError),
+                    },
+                  );
+                }
+                trace({
+                  runId: runIdForTrace,
+                  phase: 'coordinator.runner_run',
+                  ms: Date.now() - innerStartedAt,
+                  graphHash,
+                });
+                return result;
+              } catch (innerError) {
+                const failure = normalizePlayRunFailure(innerError);
                 console.error(
-                  '[coordinator] failed to forward DynamicWorkflow runner error',
+                  '[coordinator] DynamicWorkflow runner.run threw',
                   {
                     graphHash,
                     message:
-                      markError instanceof Error
-                        ? markError.message
-                        : String(markError),
+                      innerError instanceof Error
+                        ? innerError.message
+                        : String(innerError),
+                    name: innerError instanceof Error ? innerError.name : null,
+                    stack:
+                      innerError instanceof Error &&
+                      typeof innerError.stack === 'string'
+                        ? innerError.stack.split('\n').slice(0, 12).join('\n')
+                        : null,
                   },
                 );
-              });
-              await writeCoordinatorTerminalState(env, {
-                runId: runIdForTrace,
-                status: 'failed',
-                error: failure.message,
-              }).catch(() => undefined);
-              throw innerError;
-            }
-          },
-        };
-      },
-    );
+                await markWorkflowRuntimeFailure({
+                  env,
+                  event: innerEvent,
+                  error: innerError,
+                }).catch((markError) => {
+                  console.error(
+                    '[coordinator] failed to forward DynamicWorkflow runner error',
+                    {
+                      graphHash,
+                      message:
+                        markError instanceof Error
+                          ? markError.message
+                          : String(markError),
+                    },
+                  );
+                });
+                await writeCoordinatorTerminalState(env, {
+                  runId: runIdForTrace,
+                  status: 'failed',
+                  error: failure.message,
+                }).catch(() => undefined);
+                throw innerError;
+              }
+            },
+          };
+        },
+      );
+    } finally {
+      trace({
+        runId: dispatchTrace.runId,
+        phase: 'coordinator.dispatch_workflow_total',
+        ms: Date.now() - dispatchWorkflowStartedAt,
+        graphHash: dispatchTrace.graphHash,
+        extra: {
+          instanceId: dispatchTrace.instanceId,
+        },
+      });
+    }
   }
 }
 
@@ -3426,6 +3457,18 @@ async function handleWorkflowRoute(input: {
     const parseStartedAt = Date.now();
     const params = (await request.json()) as PlayWorkflowParams;
     submittedRunId = params.runId ?? runId;
+    recordSubmitTiming({
+      phase: 'coordinator.submit_received',
+      ms: Date.now() - submitStartedAt,
+      graphHash: params.graphHash ?? null,
+      extra: {
+        hasDynamicWorkerCode: Boolean(params.dynamicWorkerCode),
+        dynamicWorkerBytes:
+          typeof params.dynamicWorkerCode === 'string'
+            ? params.dynamicWorkerCode.length
+            : 0,
+      },
+    });
     recordSubmitTiming({
       phase: 'coordinator.submit_parse_body',
       ms: Date.now() - parseStartedAt,
@@ -3555,6 +3598,12 @@ async function handleWorkflowRoute(input: {
     try {
       const dispatchStartedAt = Date.now();
       const createStartedAt = Date.now();
+      recordSubmitTiming({
+        phase: 'coordinator.workflow_create_start',
+        ms: 0,
+        graphHash: params.graphHash ?? null,
+        extra: { instanceId: defaultInstanceId },
+      });
       const createResult = await createOrAttachWorkflowInstance({
         create: () =>
           createDynamicWorkflowInstance({

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -401,6 +401,9 @@ function captureRuntimeApiBinding(env: WorkerEnv): void {
 }
 
 let cachedCoordinatorBinding: WorkerEnv['COORDINATOR'] | null = null;
+const TRACE_FLUSH_MS = 1_000;
+const pendingTraceForwardsByRun = new Map<string, Promise<void>>();
+
 function captureCoordinatorBinding(env: WorkerEnv): void {
   cachedCoordinatorBinding = env.COORDINATOR ?? null;
 }
@@ -685,32 +688,58 @@ function recordRunnerPerfTrace(input: {
   ms?: number;
   extra?: Record<string, unknown>;
 }): void {
+  // Benchmark note: these runner spans decompose the server watch's terminal
+  // wait. They are logged locally and forwarded to the coordinator so
+  // `/api/v2/plays/run --watch` benchmark exports can join them with
+  // `server.stream_scheduler_terminal_event` by runId.
   if (!input.req.runId || !input.phase) return;
+  const phase = input.phase.startsWith('runner.')
+    ? input.phase
+    : `runner.${input.phase}`;
   // Tool-level traces can fire once per row/provider step. Forwarding each one
   // through the coordinator binding can consume Cloudflare's subrequest budget
   // before large batched maps finish.
-  if (input.phase.startsWith('runner.tool.')) {
+  if (phase.startsWith('runner.tool.')) {
     return;
   }
   const payload = {
     ts: Date.now(),
     source: 'dynamic_worker' as const,
     runId: input.req.runId,
-    phase: `runner.${input.phase}`,
+    phase,
     ms: input.ms ?? 0,
     ...(input.extra ?? {}),
   };
   console.log(
     `[deepline-run:${input.req.runId}] [perf-trace] ${JSON.stringify(payload)}`,
   );
-  cachedCoordinatorBinding
-    ?.recordPerfTrace(input.req.runId, payload)
-    .catch((error: unknown) => {
-      const message = error instanceof Error ? error.message : String(error);
-      console.warn(
-        `[deepline-run:${input.req.runId}] failed to forward runner perf trace: ${message}`,
-      );
-    });
+  const binding = cachedCoordinatorBinding;
+  if (!binding) return;
+  const forward = binding
+    .recordPerfTrace(input.req.runId, payload)
+    .catch(() => undefined);
+  const previous = pendingTraceForwardsByRun.get(input.req.runId);
+  const pending = previous
+    ? previous.then(
+        () => forward,
+        () => forward,
+      )
+    : forward;
+  pendingTraceForwardsByRun.set(input.req.runId, pending);
+  void pending.finally(() => {
+    if (pendingTraceForwardsByRun.get(input.req.runId) === pending) {
+      pendingTraceForwardsByRun.delete(input.req.runId);
+    }
+  });
+}
+
+async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
+  const pending = pendingTraceForwardsByRun.get(req.runId);
+  if (!pending) return;
+  await Promise.race([
+    pending,
+    new Promise((resolve) => setTimeout(resolve, TRACE_FLUSH_MS)),
+  ]);
 }
 
 function makeRequestId(): string {
@@ -5962,7 +5991,17 @@ async function executeRunRequest(
     onToolFailed: (toolId, at) => stepLifecycle?.onToolFailed(toolId, at),
   };
 
+  let hasEmittedRunnerEvent = false;
   const wrappedEmit = (event: RunnerEvent) => {
+    if (!hasEmittedRunnerEvent) {
+      hasEmittedRunnerEvent = true;
+      recordRunnerPerfTrace({
+        req,
+        phase: 'first_event',
+        ms: nowMs() - startedAt,
+        extra: { eventType: event.type },
+      });
+    }
     if (event.type === 'log') {
       appendRunLogLine(event.message);
       flushLedgerEvents(false);
@@ -6133,6 +6172,14 @@ async function executeRunRequest(
       phase: 'runner.execute_total',
       ms: nowMs() - startedAt,
     });
+    // The server-side watch path reads coordinator-buffered perf traces from
+    // the same tail response that carries the terminal event. Runner traces are
+    // forwarded asynchronously during execution so normal play latency is not
+    // gated on observability writes; before returning terminal output, wait a
+    // bounded interval for those writes to land. This keeps benchmark exports
+    // able to decompose "terminal wait" into runner/dataset/ledger phases
+    // without turning trace delivery into a correctness dependency.
+    await drainRunnerPerfTraces(req);
     return {
       playName: req.playName,
       result: serializedResult,
@@ -6160,6 +6207,7 @@ async function executeRunRequest(
       appendRunLogLine(
         `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(message)}`,
       );
+      const terminalUpdateStartedAt = nowMs();
       await flushTerminalLedgerEvents({
         type: aborted ? 'run.cancelled' : 'run.failed',
         runId: req.runId,
@@ -6184,25 +6232,55 @@ async function executeRunRequest(
               ],
             },
       });
+      recordRunnerPerfTrace({
+        req,
+        phase: aborted
+          ? 'runner.terminal_ledger_append_cancelled'
+          : 'runner.terminal_ledger_append_failed',
+        ms: nowMs() - terminalUpdateStartedAt,
+        extra: {
+          errorCode: failure.code,
+          errorPhase: failure.phase,
+        },
+      });
+      const billingStartedAt = nowMs();
       await finalizeWorkerComputeBilling({
         req,
         success: false,
         actionEstimate: 4,
-      }).catch((finalizeError) => {
-        console.error(
-          `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
-            finalizeError instanceof Error
-              ? finalizeError.message
-              : String(finalizeError)
-          }`,
-        );
-      });
+      })
+        .catch((finalizeError) => {
+          console.error(
+            `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
+              finalizeError instanceof Error
+                ? finalizeError.message
+                : String(finalizeError)
+            }`,
+          );
+        })
+        .finally(() => {
+          recordRunnerPerfTrace({
+            req,
+            phase: 'runner.compute_billing_finalize_failed',
+            ms: nowMs() - billingStartedAt,
+          });
+        });
     }
     await signalParentPlayTerminal({
       req,
       status: aborted ? 'cancelled' : 'failed',
       error: message,
     }).catch(() => null);
+    recordRunnerPerfTrace({
+      req,
+      phase: aborted ? 'runner.execute_cancelled' : 'runner.execute_failed',
+      ms: nowMs() - startedAt,
+      extra: {
+        errorCode: failure.code,
+        errorPhase: failure.phase,
+      },
+    });
+    await drainRunnerPerfTraces(req);
     throw error;
   } finally {
     clearTimeout(runtimeDeadlineTimer);
@@ -6631,14 +6709,29 @@ export class TenantWorkflow extends WorkflowEntrypoint<
     // Must run BEFORE any SDK call site that would reach into HARNESS,
     // i.e. before user play code is invoked. Idempotent within a run.
     captureHarnessBinding(this.env);
+    recordRunnerPerfTrace({
+      req,
+      phase: 'tenant_workflow_entry',
+      ms: 0,
+      extra: {
+        hasWorkflowStep: true,
+      },
+    });
     // Fire the one-time wiring probe (deduplicated across runs in the
     // same isolate). Awaited so the result is in the log before user code
     // begins. A missing or unhealthy HARNESS fails the run before user code
     // can accidentally take a slower fallback path.
+    const probeStartedAt = nowMs();
     await probeHarnessOnce(this.env, runPrefix);
+    recordRunnerPerfTrace({
+      req,
+      phase: 'tenant_workflow_probe_harness',
+      ms: nowMs() - probeStartedAt,
+    });
     const abortController = new AbortController();
     try {
-      return (await executeRunRequest(
+      const executeStartedAt = nowMs();
+      const output = (await executeRunRequest(
         req,
         this.env,
         (runnerEvent) => {
@@ -6669,6 +6762,12 @@ export class TenantWorkflow extends WorkflowEntrypoint<
           waitUntil: (promise) => this.ctx.waitUntil(promise),
         },
       )) as Record<string, unknown>;
+      recordRunnerPerfTrace({
+        req,
+        phase: 'tenant_workflow_execute_request',
+        ms: nowMs() - executeStartedAt,
+      });
+      return output;
     } catch (error) {
       // CF Workflows + the dynamic-workflows framework swallow the error
       // message and surface only "internal error; reference = <id>" via

package/dist/repo/sdk/src/release.ts

@@ -99,10 +99,10 @@ export const SDK_RELEASE = {
   // skill on the sdk sync surface, and the people-search-to-email prebuilt.
   // 0.1.108 ships explicit dataset column/tool recompute policy and removes
   // the SDK enrich generator's one-second stale policy.
-  version: '0.1.110',
+  version: '0.1.112',
   apiContract: '2026-06-dataset-column-cell-stale-hard-cutover',
   supportPolicy: {
-    latest: '0.1.110',
+    latest: '0.1.112',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [