deepline 0.1.210 → 0.1.212

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -68,6 +68,7 @@ import {
 } from './child-play-await';
 import { submitChildPlayThroughCoordinator } from './child-play-submit';
 import { isRetryableWorkerRuntimeApiError } from './runtime-api-retry';
+import { formatCustomerConsoleValue } from './runtime/customer-console';
 import {
   attachToolResultListDataset,
   createToolExecuteResult,
@@ -116,6 +117,7 @@ import {
   isQueryResultDatasetTool,
 } from '../../../shared_libs/play-runtime/query-result-dataset';
 import { DEDUPE_DUPLICATE_KEY_SAMPLE_CAP } from '../../../shared_libs/play-runtime/map-row-identity';
+import { createRuntimeDatasetId } from '../../../shared_libs/play-runtime/dataset-id';
 import {
   getTopLevelPipelineSubsteps,
   getCompiledPipelineSubsteps,
@@ -166,6 +168,7 @@ import {
   createWorkerWorkBudgetMeter,
   type WorkerWorkBudgetMeter,
 } from './runtime/work-budget';
+import { MapLatencyProfile } from './runtime/map-latency-profile';
 import {
   WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES,
   WorkerRunWorkDispatcher,
@@ -299,7 +302,7 @@ import type {
   LiveNodeProgressMap,
   LiveNodeProgressSnapshot,
 } from './runtime/live-progress';
-import { extractErrorBilling } from './runtime/tool-http-errors';
+import { extractErrorBilling, ToolHttpError } from './runtime/tool-http-errors';
 import {
   WorkflowAbortError,
   isAbortLikeError,
@@ -323,6 +326,55 @@ import type { PlayRunInputPayload } from '../../../shared_libs/play-runtime/sche
 // @ts-ignore - virtual module substituted by esbuild plugin at bundle time.
 import playFn from 'deepline-play-entry';
 
+const customerConsoleStorage = new AsyncLocalStorage<
+  (message: string) => void
+>();
+
+type CustomerConsoleGlobal = typeof globalThis & {
+  __deeplineCaptureCustomerConsole?: (
+    method: string,
+    args: unknown[],
+  ) => unknown;
+};
+
+(globalThis as CustomerConsoleGlobal).__deeplineCaptureCustomerConsole = (
+  method,
+  args,
+) => {
+  const log = customerConsoleStorage.getStore();
+  if (log) {
+    assertNoSecretTaint(args, `console.${method}`);
+    const message = args.map(formatCustomerConsoleValue).join(' ');
+    log(`[console.${method}] ${message}`.slice(0, 2_000));
+  }
+};
+
+function runCustomerPlay(
+  ctx: unknown,
+  input: PlayRunInputPayload,
+): Promise<unknown> {
+  return customerConsoleStorage.run(
+    (ctx as { log(message: string): void }).log,
+    () => (playFn as typeof runCustomerPlay)(ctx, input),
+  );
+}
+
+function runtimeDatasetIdentity(playName: string, tableNamespace: string) {
+  return {
+    datasetId: createRuntimeDatasetId(playName, tableNamespace),
+    path: `datasets.${tableNamespace}`,
+    tableNamespace,
+  };
+}
+
+function datasetAvailableLog(
+  runId: string,
+  dataset: ReturnType<typeof runtimeDatasetIdentity>,
+  count: number,
+): string {
+  return `Dataset ${dataset.path} available: ${count} persisted rows; export: deepline runs export ${runId} --dataset ${dataset.path} --out ${dataset.tableNamespace}.csv`;
+}
+
 type RunRequest = {
   runId: string;
   callbackUrl: string;
@@ -998,6 +1050,17 @@ type WorkerCtxCallbacks = {
   }) => void | Promise<void>;
   onMapStarted?: (nodeId: string, at?: number) => void;
   onMapCompleted?: (nodeId: string, at?: number) => void;
+  onDatasetLifecycle?: (event: {
+    datasetId: string;
+    path: string;
+    tableNamespace: string;
+    phase: 'registered' | 'available' | 'failed';
+    persistedRows?: number;
+    succeededRows?: number;
+    failedRows?: number;
+    complete?: boolean;
+    at: number;
+  }) => void | Promise<void>;
   onToolCalled?: (toolId: string, at?: number) => void;
   onToolFailed?: (toolId: string, at?: number) => void;
 };
@@ -2072,6 +2135,7 @@ async function callToolDirect(
   while (true) {
     requestAttempt += 1;
     let res: Response;
+    const providerCallStartedAt = nowMs();
     try {
       const runtimeApiTimeout = resolveToolRuntimeApiTimeout({
         requestInput: input,
@@ -2160,8 +2224,11 @@ async function callToolDirect(
     } catch (error) {
       transportAttempt += 1;
       const message = error instanceof Error ? error.message : String(error);
-      lastError = new Error(
+      lastError = new ToolHttpError(
         `Tool ${toolId} transport failed calling ${path} for run ${req.runId} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}: ${message}`,
+        null,
+        0,
+        'repairable',
       );
       if (
         transportAttempt >= TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS ||
@@ -2211,6 +2278,7 @@ async function callToolDirect(
       bodyText: text,
       retryAfterHeader: res.headers.get('retry-after'),
       transientHttpRetrySafe,
+      providerLatencyMs: nowMs() - providerCallStartedAt,
     });
     lastError = failure.error;
     if (failure.backpressureDelayMs !== null) {
@@ -4598,6 +4666,11 @@ function createMinimalWorkerCtx(
     opts?: WorkerMapOptions,
   ): Promise<unknown> => {
     const mapStartedAt = nowMs();
+    const mapLatencyProfile = new MapLatencyProfile(
+      req.testPolicyOverrides?.mapLatencyProfile === true,
+      name,
+      mapStartedAt,
+    );
     const mapNodeId = `map:${name}`;
     const inputRows = rows;
     const rowCountHint = datasetRowCountHint(inputRows);
@@ -4859,6 +4932,7 @@ function createMinimalWorkerCtx(
     };
 
     let totalRowsWritten = 0;
+    let datasetLifecycleRegistered = false;
 
     const volatileWorkflowChunkRows = new Map<
       number,
@@ -4909,6 +4983,22 @@ function createMinimalWorkerCtx(
         budgetMeter: workBudgetMeter,
         force: !!req.force,
       });
+      if (!datasetLifecycleRegistered) {
+        datasetLifecycleRegistered = true;
+        const dataset = runtimeDatasetIdentity(req.playName, name);
+        await callbacks?.onDatasetLifecycle?.({
+          ...dataset,
+          phase: 'registered',
+          complete: false,
+          at: nowMs(),
+        });
+        emitEvent({
+          type: 'log',
+          level: 'info',
+          message: `Dataset ${dataset.path} registered`,
+          ts: nowMs(),
+        });
+      }
       const activeRowAttempt = {
         attemptId: prepared.attemptId ?? rowAttempt.attemptId,
         attemptOwnerRunId:
@@ -4935,6 +5025,7 @@ function createMinimalWorkerCtx(
           blockedRows: prepared.blockedRows.length,
         },
       });
+      mapLatencyProfile.recordPhase('prepare_rows', nowMs() - prepareStartedAt);
       const progressTotalRows = rowCountHint ?? chunkRows.length;
       const preparedCompletedRows = Math.min(
         progressTotalRows,
@@ -5215,7 +5306,10 @@ function createMinimalWorkerCtx(
         deserializeDurableValue: deserializeDurableStepValue,
         nowMs,
         batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
-        recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
+        recordPerfTrace: (trace) => {
+          mapLatencyProfile.recordToolTrace(trace);
+          recordRunnerPerfTrace({ req, ...trace });
+        },
         abortSignal,
         onRequestsSettled: reportSettledToolRequests,
         callbacks,
@@ -5505,9 +5599,16 @@ function createMinimalWorkerCtx(
               if (circuitError) throw circuitError;
               const myIndex = idx++;
               if (myIndex >= rowsToExecute.length) return;
+              const rowQueueStartedAt = mapLatencyProfile.enabled ? nowMs() : 0;
               const rowSlot = await governor.acquireRowSlot({
                 signal: abortSignal,
               });
+              if (mapLatencyProfile.enabled) {
+                mapLatencyProfile.recordRowQueue(nowMs() - rowQueueStartedAt);
+              }
+              const rowExecutionStartedAt = mapLatencyProfile.enabled
+                ? nowMs()
+                : 0;
               let rowMarkedActive = false;
               try {
                 startedExecutedRows += 1;
@@ -5744,6 +5845,11 @@ function createMinimalWorkerCtx(
                   // single clean run failure.
                 }
               } finally {
+                if (mapLatencyProfile.enabled) {
+                  mapLatencyProfile.recordRowExecution(
+                    nowMs() - rowExecutionStartedAt,
+                  );
+                }
                 if (rowMarkedActive) {
                   activeExecutedRows = Math.max(0, activeExecutedRows - 1);
                   reportExecutionHeartbeat(false);
@@ -5796,6 +5902,10 @@ function createMinimalWorkerCtx(
           concurrency,
         },
       });
+      mapLatencyProfile.recordPhase(
+        'worker_execution',
+        nowMs() - workersStartedAt,
+      );
       const buildRowFailureSamples = () =>
         failedRowEntries
           .map((failure, executedIndex) =>
@@ -5923,6 +6033,10 @@ function createMinimalWorkerCtx(
             rowsToExecute: rowsToExecute.length,
           },
         });
+        mapLatencyProfile.recordPhase(
+          'persist_rows',
+          nowMs() - persistRowsStartedAt,
+        );
       } catch (error) {
         if (
           workflowStep &&
@@ -6028,6 +6142,7 @@ function createMinimalWorkerCtx(
         ms: nowMs() - hashStartedAt,
         extra: { mapName: name, chunkIndex, rows: out.length },
       });
+      mapLatencyProfile.recordPhase('hash', nowMs() - hashStartedAt);
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.total',
@@ -6230,6 +6345,23 @@ function createMinimalWorkerCtx(
                 finalizedRowsWritten,
               ),
       });
+      const dataset = runtimeDatasetIdentity(req.playName, name);
+      const persistedRows = finalizedRowsWritten + totalRowsFailed;
+      await callbacks?.onDatasetLifecycle?.({
+        ...dataset,
+        phase: 'available',
+        persistedRows,
+        succeededRows: finalizedRowsWritten,
+        failedRows: totalRowsFailed,
+        complete: true,
+        at: completedAt,
+      });
+      emitEvent({
+        type: 'log',
+        level: totalRowsFailed > 0 ? 'warn' : 'info',
+        message: datasetAvailableLog(req.runId, dataset, persistedRows),
+        ts: completedAt,
+      });
       callbacks?.onMapCompleted?.(mapNodeId, completedAt);
       emitEvent({
         type: 'log',
@@ -6503,7 +6635,8 @@ function createMinimalWorkerCtx(
           if (chunkResult.fatalError) {
             throw new Error(
               `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
-                `outside the retryable chunk step. Provider calls already executed for ` +
+                `outside the retryable chunk step. Circuit breaker prevented queued row ` +
+                `dispatch. Provider calls already executed for ` +
                 `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
                 `Fix the runtime persistence cause and re-run to resume. ` +
                 `First error: ${chunkResult.fatalError}`,
@@ -6643,6 +6776,27 @@ function createMinimalWorkerCtx(
         chunks: dispatchResult.chunksExecuted,
       },
     });
+    if (mapLatencyProfile.enabled) {
+      const endedAt = nowMs();
+      const summary = mapLatencyProfile.summary(endedAt, {
+        chunks: dispatchResult.chunksExecuted,
+        rows: totalRowsWritten,
+      });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map.profile',
+        ms: endedAt - mapStartedAt,
+        extra: summary,
+      });
+      // Local Absurd does not always forward dynamic-worker perf traces. Keep
+      // one bounded copy in the durable run log for the diagnostic command.
+      emitEvent({
+        type: 'log',
+        level: 'info',
+        message: `[perf] map_latency_profile ${JSON.stringify(summary)}`,
+        ts: endedAt,
+      });
+    }
     return dataset;
   };
 
@@ -7555,9 +7709,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
       captureHarnessBinding(env);
       await probeHarnessOnce(env, runPrefix);
       const ctx = createMinimalWorkerCtx(req, emit, env);
-      const result = await (
-        playFn as (ctx: unknown, input: PlayRunInputPayload) => Promise<unknown>
-      )(ctx, req.runtimeInput);
+      const result = await runCustomerPlay(ctx, req.runtimeInput);
       emit({
         type: 'result',
         result,
@@ -7971,6 +8123,17 @@ async function executeRunRequest(
     },
     onMapStarted: (nodeId, at) => stepLifecycle?.onMapStarted(nodeId, at),
     onMapCompleted: (nodeId, at) => stepLifecycle?.onMapCompleted(nodeId, at),
+    onDatasetLifecycle: async (event) => {
+      const { at, ...dataset } = event;
+      pendingLedgerEvents.push({
+        ...dataset,
+        type: 'dataset.lifecycle',
+        runId: req.runId,
+        source: 'worker',
+        occurredAt: at,
+      });
+      await flushLedgerEvents(event.complete === true);
+    },
     onToolCalled: (toolId, at) => stepLifecycle?.onToolCalled(toolId, at),
     onToolFailed: (toolId, at) => stepLifecycle?.onToolFailed(toolId, at),
   };
@@ -8037,9 +8200,7 @@ async function executeRunRequest(
       : null;
   try {
     const playStartedAt = nowMs();
-    const result = await (
-      playFn as (ctx: unknown, input: PlayRunInputPayload) => Promise<unknown>
-    )(ctx, req.runtimeInput);
+    const result = await runCustomerPlay(ctx, req.runtimeInput);
     recordRunnerPerfTrace({
       req,
       phase: 'runner.play_function',
@@ -8083,7 +8244,33 @@ async function executeRunRequest(
         ms: nowMs() - ledgerFlushWaitStartedAt,
       });
       const resultDatasetStartedAt = nowMs();
-      await persistResultDatasets(req, promoted.handles, serializedResult);
+      const persistedResultCounts = await persistResultDatasets(
+        req,
+        promoted.handles,
+        serializedResult,
+      );
+      for (const dataset of promoted.handles) {
+        if (dataset.datasetKind === 'map') continue;
+        const identity = runtimeDatasetIdentity(
+          req.playName,
+          dataset.tableNamespace,
+        );
+        const count = persistedResultCounts.get(dataset.tableNamespace) ?? 0;
+        pendingLedgerEvents.push({
+          ...identity,
+          type: 'dataset.lifecycle',
+          runId: req.runId,
+          source: 'worker',
+          occurredAt: nowMs(),
+          phase: 'available',
+          persistedRows: count,
+          succeededRows: count,
+          failedRows: 0,
+          complete: true,
+        });
+        appendRunLogLine(datasetAvailableLog(req.runId, identity, count));
+      }
+      await flushLedgerEvents(true);
       resultDatasetsPersisted = true;
       recordRunnerPerfTrace({
         req,
@@ -8547,8 +8734,9 @@ async function persistResultDatasets(
   req: RunRequest,
   resultDatasets: ProjectedResultDatasetHandle[],
   serializedResult: unknown,
-): Promise<void> {
+): Promise<Map<string, number>> {
   const persistedNamespaces = new Set<string>();
+  const persistedCounts = new Map<string, number>();
   for (const dataset of resultDatasets) {
     if (dataset.datasetKind === 'map') continue;
     const handle = dataset.handle as WorkerDatasetHandle<
@@ -8587,6 +8775,7 @@ async function persistResultDatasets(
       inputOffset += chunk.length;
     }
     persistedNamespaces.add(dataset.tableNamespace);
+    persistedCounts.set(dataset.tableNamespace, inputOffset);
   }
 
   const datasets = collectDatasetEnvelopes(serializedResult);
@@ -8620,7 +8809,9 @@ async function persistResultDatasets(
       attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
       userEmail: req.userEmail,
     });
+    persistedCounts.set(dataset.tableNamespace, dataset.rows.length);
   }
+  return persistedCounts;
 }
 
 const RESULT_DATASET_PERSIST_CHUNK_ROWS = 5_000;

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts

@@ -0,0 +1,23 @@
+export function formatCustomerConsoleValue(value: unknown): string {
+  if (typeof value === 'string') return value;
+  try {
+    if (value instanceof Error) return value.stack ?? value.message;
+    if (value && typeof value === 'object') {
+      const seen = new WeakSet<object>();
+      const serialized = JSON.stringify(value, (_key, nested) => {
+        if (typeof nested === 'symbol' || typeof nested === 'bigint') {
+          return nested.toString();
+        }
+        if (nested && typeof nested === 'object') {
+          if (seen.has(nested)) return '[Circular]';
+          seen.add(nested);
+        }
+        return nested;
+      });
+      if (serialized !== undefined) return serialized;
+    }
+    return String(value);
+  } catch {
+    return '[Unprintable]';
+  }
+}

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts

@@ -337,6 +337,21 @@ function countToolSubstep(substep: PlayStaticSubstep): MapToolStats {
         minProviderBranchCoalescedCount(branchStats),
       ];
     }
+    if (substep.kind === 'loop') {
+      // The static extractor records every switch arm in `steps` when a loop
+      // walks a fixed provider list. Each arm can execute on a different
+      // iteration, so using the conditional's max-one-branch estimate would
+      // budget a seven-provider waterfall as one provider call per row.
+      return substep.steps.reduce<MapToolStats>((stats, step) => {
+        const stepStats =
+          step.type === 'control_flow' &&
+          step.kind === 'conditional' &&
+          step.steps.length > 0
+            ? countToolSubsteps(step.steps)
+            : countToolSubstep(step);
+        return addStats(stats, stepStats);
+      }, [0, 0, 0, 0, 0]);
+    }
     return countToolSubsteps(substep.steps);
   }
 

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts

@@ -0,0 +1,90 @@
+type ToolTrace = {
+  phase: string;
+  ms?: number;
+  extra?: Record<string, unknown>;
+};
+
+type Totals = Record<string, number>;
+
+function add(totals: Totals, key: string, ms: number | undefined): void {
+  if (typeof ms !== 'number' || !Number.isFinite(ms) || ms < 0) return;
+  totals[key] = (totals[key] ?? 0) + ms;
+}
+
+/**
+ * Opt-in map profiler. It aggregates in memory and produces one terminal
+ * trace, so normal runs do not pay per-row/per-tool telemetry cost.
+ */
+export class MapLatencyProfile {
+  private readonly phaseWorkMs: Totals = {};
+  private readonly toolWorkMs: Totals = {};
+  private readonly counts: Totals = {};
+
+  constructor(
+    readonly enabled: boolean,
+    private readonly mapName: string,
+    private readonly startedAtMs: number,
+  ) {}
+
+  recordPhase(phase: string, ms: number, count = 1): void {
+    if (!this.enabled) return;
+    add(this.phaseWorkMs, phase, ms);
+    this.counts[`${phase}_count`] =
+      (this.counts[`${phase}_count`] ?? 0) + count;
+  }
+
+  recordRowQueue(ms: number): void {
+    this.recordPhase('row_slot_queue', ms);
+  }
+
+  recordRowExecution(ms: number): void {
+    this.recordPhase('row_execution', ms);
+  }
+
+  recordToolTrace(trace: ToolTrace): void {
+    if (!this.enabled) return;
+    switch (trace.phase) {
+      case 'runner.tool.request':
+        add(this.toolWorkMs, 'request_end_to_end', trace.ms);
+        this.counts.tool_requests = (this.counts.tool_requests ?? 0) + 1;
+        break;
+      case 'runner.tool.drain':
+        add(this.toolWorkMs, 'drain', trace.ms);
+        this.counts.tool_drains = (this.counts.tool_drains ?? 0) + 1;
+        break;
+      case 'runner.tool.group':
+        add(this.toolWorkMs, 'group', trace.ms);
+        this.counts.tool_groups = (this.counts.tool_groups ?? 0) + 1;
+        break;
+      case 'runner.tool.receipt_prepare':
+        add(this.toolWorkMs, 'receipt_prepare', trace.ms);
+        break;
+      case 'runner.tool.admission':
+        add(this.toolWorkMs, 'admission_wait', trace.ms);
+        break;
+      case 'runner.tool.provider':
+        add(this.toolWorkMs, 'provider_execute', trace.ms);
+        break;
+      case 'runner.tool.receipt_complete':
+        add(this.toolWorkMs, 'receipt_complete', trace.ms);
+        break;
+      default:
+        return;
+    }
+  }
+
+  summary(endedAtMs: number, input: { chunks: number; rows: number }) {
+    return {
+      mapName: this.mapName,
+      mapWallMs: Math.max(0, endedAtMs - this.startedAtMs),
+      chunks: input.chunks,
+      rows: input.rows,
+      // These are summed work durations. They can exceed mapWallMs when rows
+      // or provider calls overlap; the terminal trace must not mislabel them
+      // as a wall-clock decomposition.
+      phaseWorkMs: this.phaseWorkMs,
+      toolWorkMs: this.toolWorkMs,
+      counts: this.counts,
+    };
+  }
+}