deepline 0.1.80 → 0.1.81

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -61,10 +61,6 @@ import {
   type WorkflowStepLike,
 } from './child-play-await';
 import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
-import {
-  createToolBatchExecutor,
-  type ToolBatchRequest,
-} from '../../../shared_libs/play-runtime/tool-batch-executor';
 import {
   adaptV2ExecuteResponseToToolResult,
   createToolExecuteResult,
@@ -137,7 +133,6 @@ import {
 import { createHarnessWorkerReceiptStore } from './runtime/harness-receipt-store';
 import {
   applyCsvRenameProjection,
-  stripCsvProjectedFields,
   stripCsvProjectionMetadata,
   cloneCsvAliasedRow,
   type CsvRenameOptions,
@@ -162,7 +157,6 @@ import type {
   LiveNodeProgressSnapshot,
 } from './runtime/live-progress';
 import {
-  ToolHttpError,
   extractErrorBilling,
   isHardBillingToolHttpError,
   normalizeToolHttpErrorMessage,
@@ -597,6 +591,7 @@ type WorkerCtxCallbacks = {
   onNodeProgress?: (input: {
     nodeId: string;
     progress: LiveNodeProgressSnapshot;
+    forceFlush?: boolean;
   }) => void;
   onMapStarted?: (nodeId: string, at?: number) => void;
   onMapCompleted?: (nodeId: string, at?: number) => void;
@@ -685,12 +680,17 @@ function makeRequestId(): string {
 }
 
 function publicCsvInputRow<T extends Record<string, unknown>>(row: T): T {
-  const stripped = stripCsvProjectedFields(row) as Record<string, unknown>;
-  return Object.fromEntries(
-    Object.entries(stripped).filter(
-      ([fieldName]) => !fieldName.startsWith('__deepline'),
-    ),
-  ) as T;
+  const restored = stripCsvProjectionMetadata(row) as Record<string, unknown>;
+  const publicRow: Record<string, unknown> = {};
+  for (const fieldName of Reflect.ownKeys(restored)) {
+    if (typeof fieldName === 'string' && fieldName.startsWith('__deepline')) {
+      continue;
+    }
+    const descriptor = Object.getOwnPropertyDescriptor(restored, fieldName);
+    if (!descriptor) continue;
+    Object.defineProperty(publicRow, fieldName, descriptor);
+  }
+  return publicRow as T;
 }
 
 function publicCsvOutputRow<T extends Record<string, unknown>>(row: T): T {
@@ -707,6 +707,27 @@ function publicCsvOutputRow<T extends Record<string, unknown>>(row: T): T {
   return publicRow as T;
 }
 
+function publicCsvStorageRow<T extends Record<string, unknown>>(row: T): T {
+  const publicRow = publicCsvInputRow(row) as Record<string, unknown>;
+  const storageRow: Record<string, unknown> = {};
+  for (const fieldName of Reflect.ownKeys(publicRow)) {
+    if (typeof fieldName !== 'string') continue;
+    const descriptor = Object.getOwnPropertyDescriptor(publicRow, fieldName);
+    if (!descriptor) continue;
+    storageRow[fieldName] =
+      'value' in descriptor ? descriptor.value : publicRow[fieldName];
+  }
+  for (const runtimeField of [
+    '__deeplineRowKey',
+    '__deeplineCellMetaPatch',
+  ]) {
+    if (Object.prototype.hasOwnProperty.call(row, runtimeField)) {
+      storageRow[runtimeField] = row[runtimeField];
+    }
+  }
+  return storageRow as T;
+}
+
 /**
  * Strip credentials and JWT-shaped tokens from any string before it lands in
  * a log buffer or upstream error message. The harness routinely echoes
@@ -1245,50 +1266,11 @@ async function callToolDirect(
   onRetryAttempt?: () => void,
 ): Promise<ToolExecuteResult> {
   const { id, toolId, input } = args;
-  if (toolId === 'test_rate_limit') {
-    return wrapWorkerToolResult(
-      toolId,
-      executeSyntheticTestRateLimit(input),
-      syntheticToolMetadata(toolId),
-    );
-  }
-  if (toolId === 'test_batch_rate_limit') {
-    return wrapWorkerToolResult(
-      toolId,
-      await executeSyntheticTestRateLimitBatch(req, input),
-      syntheticToolMetadata(toolId),
-    );
-  }
   const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
   const maxAttempts = 3;
   let lastError: Error | null = null;
 
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
-    if (toolId === 'test_transient_500' || toolId === 'test_transient_429') {
-      const syntheticResult = executeSyntheticTransientRetry(
-        toolId,
-        input,
-        attempt,
-      );
-      if (syntheticResult.ok) {
-        return wrapWorkerToolResult(
-          toolId,
-          syntheticResult.result,
-          syntheticToolMetadata(toolId),
-        );
-      }
-      lastError = new Error(
-        `tool ${toolId} ${syntheticResult.status} attempt ${attempt}/${maxAttempts}: ${syntheticResult.message}`,
-      );
-      if (attempt >= maxAttempts) {
-        throw lastError;
-      }
-      // Charge the retry budget per attempt, matching the cjs runner.
-      onRetryAttempt?.();
-      await new Promise((resolve) => setTimeout(resolve, 1_000));
-      continue;
-    }
-
     const res = await fetchRuntimeApi(req.baseUrl, path, {
       method: 'POST',
       headers: {
@@ -1466,7 +1448,7 @@ function parseStringArray(value: unknown): string[] {
     .filter(Boolean);
 }
 
-function syntheticToolMetadata(toolId: string): ToolResultMetadataInput {
+function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
   if (toolId === 'test_rate_limit') {
     return {
       toolId,
@@ -1511,193 +1493,6 @@ function wrapWorkerToolResult(
   });
 }
 
-async function executeSyntheticTestRateLimitBatch(
-  req: RunRequest,
-  input: Record<string, unknown>,
-): Promise<Record<string, unknown>> {
-  const delayMs =
-    typeof input.simulated_delay_ms === 'number' &&
-    Number.isInteger(input.simulated_delay_ms) &&
-    input.simulated_delay_ms > 0
-      ? input.simulated_delay_ms
-      : 0;
-  if (delayMs > 0) {
-    await new Promise((resolve) => setTimeout(resolve, delayMs));
-  }
-  const rawItems = Array.isArray(input.items) ? input.items : [];
-  const items = rawItems
-    .filter((item): item is Record<string, unknown> =>
-      Boolean(item && typeof item === 'object' && !Array.isArray(item)),
-    )
-    .map((item, index) => {
-      const itemKey =
-        typeof item.itemKey === 'string' && item.itemKey.trim()
-          ? item.itemKey.trim()
-          : `item-${index}`;
-      const payload =
-        item.payload &&
-        typeof item.payload === 'object' &&
-        !Array.isArray(item.payload)
-          ? (item.payload as Record<string, unknown>)
-          : {};
-      return { itemKey, payload };
-    });
-  const batchRequest: ToolBatchRequest = {
-    runId: req.runId,
-    orgId: req.orgId,
-    toolId: 'test_rate_limit',
-    operation: 'test_batch_rate_limit',
-    provider: 'test',
-    items,
-    waterfallId:
-      typeof input.waterfall_id === 'string' ? input.waterfall_id : null,
-    stageId: typeof input.stage === 'string' ? input.stage : null,
-    fieldName: typeof input.field_name === 'string' ? input.field_name : null,
-    mapName: typeof input.map_name === 'string' ? input.map_name : null,
-    chunkIndex:
-      typeof input.chunk_index === 'number' ? input.chunk_index : null,
-    userProvidedRateLimitKey:
-      typeof input.rate_limit_key === 'string' ? input.rate_limit_key : null,
-    providerBatchSize: 200,
-  };
-  const executor = createToolBatchExecutor({
-    async executeProviderBatch({ items: providerItems }) {
-      return providerItems.map((item) => ({
-        itemKey: item.itemKey,
-        result: executeSyntheticTestRateLimit(item.payload),
-      }));
-    },
-  });
-  const result = await executor.executeToolBatch(batchRequest);
-  return {
-    status: 'completed',
-    key: String(input.key ?? 'batch'),
-    provider: 'test',
-    batch: true,
-    batch_size: result.itemCount,
-    provider_batch_count: result.batchCount,
-    items: result.results.map((item) => ({
-      itemKey: item.itemKey,
-      result: item.result,
-    })),
-  };
-}
-
-type SyntheticTransientRetryResult =
-  | { ok: true; result: Record<string, unknown> }
-  | { ok: false; status: number; message: string };
-
-function executeSyntheticTransientRetry(
-  toolId: string,
-  input: Record<string, unknown>,
-  attempt: number,
-): SyntheticTransientRetryResult {
-  const failuresBeforeSuccess =
-    typeof input.failures_before_success === 'number' &&
-    Number.isInteger(input.failures_before_success) &&
-    input.failures_before_success >= 0
-      ? input.failures_before_success
-      : 1;
-  if (attempt <= failuresBeforeSuccess) {
-    const status = toolId === 'test_transient_429' ? 429 : 502;
-    return {
-      ok: false,
-      status,
-      message: `Synthetic transient ${status} for attempt ${attempt}`,
-    };
-  }
-  return {
-    ok: true,
-    result: {
-      status: 'completed',
-      provider: 'test',
-      key: String(input.key ?? 'transient'),
-      attempts: attempt,
-      recovered: attempt > 1,
-    },
-  };
-}
-
-function executeSyntheticTestRateLimit(
-  input: Record<string, unknown>,
-): Record<string, unknown> {
-  if (
-    typeof input.key === 'string' &&
-    input.key.startsWith('public-error-message-regression')
-  ) {
-    throw new ToolHttpError(
-      [
-        'tool test_rate_limit 422 attempt 1/1:',
-        'Synthetic public test error with a redacted token=[REDACTED].',
-        'code=TEST_PUBLIC_ERROR.',
-        'failure_description=The fake test provider intentionally raised a typed public error so V2 runner output preserves actionable details.',
-        'operator_hint=Use this no-bill test provider fixture when verifying play runner error rendering.',
-      ].join(' '),
-      null,
-    );
-  }
-  const rowNumber =
-    typeof input.row_number === 'number' && Number.isInteger(input.row_number)
-      ? input.row_number
-      : null;
-  const leadId = typeof input.lead_id === 'string' ? input.lead_id : null;
-  const matchedDomain =
-    typeof input.matched_domain === 'string' && input.matched_domain.trim()
-      ? input.matched_domain.trim()
-      : 'example.com';
-  const matchedPrefix =
-    typeof input.matched_prefix === 'string' && input.matched_prefix.trim()
-      ? input.matched_prefix.trim()
-      : (leadId ??
-        (rowNumber !== null
-          ? `row${String(rowNumber).padStart(3, '0')}`
-          : 'match'));
-  const matched = syntheticMatchWindow(input, rowNumber);
-  const matchedEmail = matched ? `${matchedPrefix}@${matchedDomain}` : null;
-  const securityGateway =
-    input.emit_security_gateway === true
-      ? { email_status: 'valid', mx_security_gateway: true }
-      : {};
-  return {
-    status: 'completed',
-    key: String(input.key || ''),
-    provider: 'test',
-    lead_id: leadId,
-    row_number: rowNumber,
-    matched_result: matchedEmail,
-    email: matchedEmail,
-    value: matchedEmail,
-    batch: false,
-    ...securityGateway,
-  };
-}
-
-function syntheticMatchWindow(
-  input: Record<string, unknown>,
-  rowNumber: number | null,
-): boolean {
-  const min =
-    typeof input.match_rows_min === 'number' ? input.match_rows_min : null;
-  const max =
-    typeof input.match_rows_max === 'number' ? input.match_rows_max : null;
-  if (rowNumber === null) return min === null && max === null;
-  if (min !== null && rowNumber < min) return false;
-  if (max !== null && rowNumber > max) return false;
-  const moduloBase =
-    typeof input.match_modulo_base === 'number' && input.match_modulo_base > 0
-      ? input.match_modulo_base
-      : null;
-  if (moduloBase !== null) {
-    const equals = Array.isArray(input.match_modulo_equals)
-      ? input.match_modulo_equals
-          .filter((entry): entry is number => typeof entry === 'number')
-          .map((entry) => entry % moduloBase)
-      : [];
-    return equals.length > 0 && equals.includes(rowNumber % moduloBase);
-  }
-  return true;
-}
-
 function isRecordLike(value: unknown): value is Record<string, unknown> {
   return value != null && typeof value === 'object' && !Array.isArray(value);
 }
@@ -1791,7 +1586,7 @@ type WorkerToolBatchRequest = {
   reject: (error: unknown) => void;
 };
 
-const WORKER_TOOL_BATCH_GRACE_MS = 15;
+const WORKER_TOOL_BATCH_GRACE_MS = 250;
 // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
 // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
 // their previous batching behavior; declared providers tighten via the
@@ -1812,6 +1607,7 @@ class WorkerToolBatchScheduler {
     private readonly governor: PlayExecutionGovernor,
     private readonly resolvePacing: WorkerPacingResolver,
     private readonly abortSignal?: AbortSignal,
+    private readonly onRequestsSettled?: (count: number) => void,
   ) {}
 
   /**
@@ -1933,6 +1729,7 @@ class WorkerToolBatchScheduler {
           } catch (error) {
             request.reject(error);
           } finally {
+            this.onRequestsSettled?.(1);
             slot.release();
           }
         }),
@@ -1959,6 +1756,7 @@ class WorkerToolBatchScheduler {
       abortSignal: this.abortSignal,
       reportBackpressure: (retryAfterMs) =>
         this.reportBackpressure(toolId, retryAfterMs),
+      onRequestsSettled: this.onRequestsSettled,
     });
     recordRunnerPerfTrace({
       req: this.req,
@@ -1992,12 +1790,25 @@ async function executeBatchedWorkerToolGroup(input: {
   suggestedParallelism: number;
   abortSignal?: AbortSignal;
   reportBackpressure: (retryAfterMs: number) => void;
+  onRequestsSettled?: (count: number) => void;
 }): Promise<void> {
   const compiledBatches = compileRequestsWithStrategy({
     requests: input.requests,
     strategy: input.strategy,
     getPayload: (request) => request.input,
   });
+  recordRunnerPerfTrace({
+    req: input.req,
+    phase: 'runner.tool.batch.compile',
+    ms: 0,
+    extra: {
+      sourceOperation: input.strategy.sourceOperation,
+      batchOperation: input.strategy.batchOperation,
+      requests: input.requests.length,
+      batches: compiledBatches.length,
+      batchSizes: compiledBatches.map((batch) => batch.memberRequests.length),
+    },
+  });
 
   await executeChunkedRequests({
     requests: compiledBatches,
@@ -2052,11 +1863,18 @@ async function executeBatchedWorkerToolGroup(input: {
             wrapWorkerToolResult(
               request.toolId,
               splitResults[index] ?? null,
-              syntheticToolMetadata(request.toolId),
+              toolMetadataFallback(request.toolId),
             ),
           );
         }
       }
+      const settledMembers = chunkResults.reduce(
+        (total, entry) => total + entry.request.memberRequests.length,
+        0,
+      );
+      if (settledMembers > 0) {
+        input.onRequestsSettled?.(settledMembers);
+      }
     },
   }).catch((error) => {
     for (const request of input.requests) {
@@ -3172,10 +2990,10 @@ async function persistCompletedMapRows(input: {
     tableNamespace: input.tableNamespace,
     sheetContract: augmentSheetContractWithDatasetFields({
       contract: requireSheetContract(input.req, input.tableNamespace),
-      rows: input.rows,
+      rows: input.rows.map((row) => publicCsvStorageRow(row)),
       outputFields,
     }),
-    rows: input.rows,
+    rows: input.rows.map((row) => publicCsvStorageRow(row)),
     outputFields,
     runId: input.req.runId,
     userEmail: input.req.userEmail,
@@ -3206,10 +3024,10 @@ async function prepareMapRows(input: {
     tableNamespace: input.tableNamespace,
     sheetContract: augmentSheetContractWithDatasetFields({
       contract: requireSheetContract(input.req, input.tableNamespace),
-      rows: input.rows,
+      rows: input.rows.map((row) => publicCsvStorageRow(row)),
       outputFields: input.outputFields,
     }),
-    rows: input.rows.map((row) => ({ ...row })),
+    rows: input.rows.map((row) => publicCsvStorageRow(row)),
     runId: input.req.runId,
     userEmail: input.req.userEmail,
     cellPolicies: input.cellPolicies,
@@ -3334,18 +3152,65 @@ function childPipelineNeedsWorkflowScheduler(
  * fail OPEN — grant immediately — matching customer-rate-limiter semantics so a
  * miswired binding degrades pacing without stalling the run.
  */
-function createCoordinatorRatePort(): CoordinatorRatePort {
+function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
   return {
     async rateAcquire(input) {
       const binding = cachedCoordinatorBinding;
       if (!binding?.rateAcquire) {
-        return { granted: input.requested, waitMs: 0 };
+        const coordinatorUrl = req.coordinatorUrl?.trim();
+        if (!coordinatorUrl) {
+          throw new Error('Coordinator rate acquire is unavailable.');
+        }
+        const res = await fetch(`${coordinatorUrl.replace(/\/$/, '')}/rate-acquire`, {
+          method: 'POST',
+          headers: {
+            'x-deepline-request-id': makeRequestId(),
+            ...coordinatorRequestHeaders({
+              runId: req.runId,
+              contentType: 'application/json',
+              internalToken: req.coordinatorInternalToken,
+            }),
+          },
+          body: JSON.stringify(input),
+        });
+        if (!res.ok) {
+          const text = await res.text().catch(() => '');
+          throw new Error(
+            `Coordinator rate acquire failed (${res.status}): ${text}`,
+          );
+        }
+        return (await res.json()) as { granted: number; waitMs: number };
       }
       return await binding.rateAcquire(input);
     },
     async ratePenalize(input) {
       const binding = cachedCoordinatorBinding;
-      if (!binding?.ratePenalize) return;
+      if (!binding?.ratePenalize) {
+        const coordinatorUrl = req.coordinatorUrl?.trim();
+        if (!coordinatorUrl) return;
+        const res = await fetch(
+          `${coordinatorUrl.replace(/\/$/, '')}/rate-penalize`,
+          {
+            method: 'POST',
+            headers: {
+              'x-deepline-request-id': makeRequestId(),
+              ...coordinatorRequestHeaders({
+                runId: req.runId,
+                contentType: 'application/json',
+                internalToken: req.coordinatorInternalToken,
+              }),
+            },
+            body: JSON.stringify(input),
+          },
+        );
+        if (!res.ok) {
+          const text = await res.text().catch(() => '');
+          throw new Error(
+            `Coordinator rate penalize failed (${res.status}): ${text}`,
+          );
+        }
+        return;
+      }
       await binding.ratePenalize(input);
     },
   };
@@ -3477,7 +3342,7 @@ function createGovernorForRun(req: RunRequest): {
       orgId: req.orgId,
       rootRunId: req.playCallGovernance?.rootRunId ?? req.runId,
     },
-    rateState: new CoordinatorRateStateBackend(createCoordinatorRatePort()),
+    rateState: new CoordinatorRateStateBackend(createCoordinatorRatePort(req)),
     resolvePacing,
     resume: resumeGovernanceFromRequest(req),
   });
@@ -3660,6 +3525,7 @@ function createMinimalWorkerCtx(
           ...progress,
           updatedAt: progress.updatedAt ?? nowMs(),
         },
+        forceFlush: true,
       });
     };
     const formatMapProgressMessage = (completed: number, total?: number) =>
@@ -3789,6 +3655,18 @@ function createMinimalWorkerCtx(
           completedRows: prepared.completedRows.length,
         },
       });
+      updateMapProgress({
+        completed: prepared.completedRows.length,
+        total: chunkRows.length,
+        startedAt: mapStartedAt,
+        message:
+          prepared.pendingRows.length > 0
+            ? `${prepared.pendingRows.length.toLocaleString()} rows queued`
+            : formatMapProgressMessage(
+                prepared.completedRows.length,
+                chunkRows.length,
+              ),
+      });
       const pendingKeys = new Set<string>();
       const pendingRowsByKey = new Map<string, Record<string, unknown>>();
       const completedKeys = new Set<string>();
@@ -3836,6 +3714,34 @@ function createMinimalWorkerCtx(
         0,
         prepared.skipped - missingPreparedRows.length,
       );
+      let settledToolRequests = 0;
+      let lastToolProgressAt = 0;
+      const reportSettledToolRequests = (count: number) => {
+        if (count <= 0) return;
+        settledToolRequests += count;
+        const now = nowMs();
+        const estimatedCompleted = Math.min(
+          chunkRows.length,
+          prepared.completedRows.length + settledToolRequests,
+        );
+        const isTerminalEstimate = estimatedCompleted >= chunkRows.length;
+        if (
+          !isTerminalEstimate &&
+          now - lastToolProgressAt < RUN_LEDGER_FLUSH_INTERVAL_MS
+        ) {
+          return;
+        }
+        lastToolProgressAt = now;
+        updateMapProgress({
+          completed: estimatedCompleted,
+          total: chunkRows.length,
+          startedAt: mapStartedAt,
+          message: formatMapProgressMessage(
+            estimatedCompleted,
+            chunkRows.length,
+          ),
+        });
+      };
       // Row concurrency comes from the Governor: an explicit map concurrency is
       // clamped to the policy row-max, otherwise the policy default. Each row
       // body additionally acquires a global row slot (the Governor's rowMax
@@ -3863,6 +3769,7 @@ function createMinimalWorkerCtx(
         governor,
         resolveToolPacing,
         abortSignal,
+        reportSettledToolRequests,
       );
       const generatedOutputFields = new Set<string>();
       let idx = 0;
@@ -5398,6 +5305,100 @@ async function executeRunRequest(
 
   const stepProgressSnapshot = () => ({ ...stepProgressByNodeId });
 
+  const publishCoordinatorProgressEvent = async (
+    occurredAt: number,
+  ): Promise<void> => {
+    const coordinatorUrl = req.coordinatorUrl?.trim();
+    if (!coordinatorUrl) {
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.coordinator_progress_publish',
+        ms: 0,
+        extra: { status: 'skipped_no_url' },
+      });
+      return;
+    }
+    const publishStartedAt = nowMs();
+    const liveNodeProgress = stepProgressSnapshot();
+    const activeEntry =
+      Object.entries(liveNodeProgress).find(
+        ([, progress]) => typeof progress.completedAt !== 'number',
+      ) ?? Object.entries(liveNodeProgress).at(-1);
+    const activeNodeId = activeEntry?.[0] ?? null;
+    const activeProgress = activeEntry?.[1] ?? null;
+    const activeArtifactTableNamespace =
+      typeof activeProgress?.artifactTableNamespace === 'string'
+        ? activeProgress.artifactTableNamespace
+        : null;
+    const activeCompleted =
+      typeof activeProgress?.completed === 'number'
+        ? activeProgress.completed
+        : null;
+    const activeTotal =
+      typeof activeProgress?.total === 'number' ? activeProgress.total : null;
+    const activeMessage =
+      typeof activeProgress?.message === 'string'
+        ? activeProgress.message
+        : null;
+    const response = await fetch(
+      `${coordinatorUrl.replace(/\/$/, '')}/dedup/${encodeURIComponent(
+        req.runId,
+      )}/event-add`,
+      {
+        method: 'POST',
+        headers: {
+          'x-deepline-request-id': makeRequestId(),
+          ...coordinatorRequestHeaders({
+            runId: req.runId,
+            contentType: 'application/json',
+            internalToken: req.coordinatorInternalToken,
+          }),
+        },
+        body: JSON.stringify({
+          runId: req.runId,
+          type: 'progress',
+          status: 'running',
+          ts: occurredAt,
+          logs: runLogBuffer,
+          activeNodeId,
+          activeArtifactTableNamespace,
+          updatedAt: occurredAt,
+          liveNodeProgress,
+        }),
+      },
+    );
+    if (!response.ok) {
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.coordinator_progress_publish',
+        ms: nowMs() - publishStartedAt,
+        extra: {
+          status: 'failed',
+          httpStatus: response.status,
+          activeNodeId,
+          activeArtifactTableNamespace,
+          activeCompleted,
+          activeTotal,
+          activeMessage,
+        },
+      });
+      throw new Error(`coordinator progress event failed ${response.status}`);
+    }
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.coordinator_progress_publish',
+      ms: nowMs() - publishStartedAt,
+      extra: {
+        status: 'ok',
+        activeNodeId,
+        activeArtifactTableNamespace,
+        activeCompleted,
+        activeTotal,
+        activeMessage,
+      },
+    });
+  };
+
   const appendStepLifecycleEvent = (event: PlayStepLifecycleEvent) => {
     updateStepProgress({
       nodeId: event.nodeId,
@@ -5465,6 +5466,12 @@ async function executeRunRequest(
           progress.artifactTableNamespace === null
             ? { artifactTableNamespace: progress.artifactTableNamespace }
             : {}),
+          ...(typeof progress.startedAt === 'number'
+            ? { startedAt: progress.startedAt }
+            : {}),
+          ...(typeof progress.completedAt === 'number'
+            ? { completedAt: progress.completedAt }
+            : {}),
           updatedAt:
             typeof progress.updatedAt === 'number'
               ? progress.updatedAt
@@ -5513,6 +5520,7 @@ async function executeRunRequest(
           pendingLedgerEvents = [...events, ...pendingLedgerEvents];
           throw new Error('runtime run-ledger append failed');
         }
+        await publishCoordinatorProgressEvent(now).catch(() => undefined);
       })
       .catch(() => undefined);
   };
@@ -5556,7 +5564,7 @@ async function executeRunRequest(
   const workerCallbacks: WorkerCtxCallbacks = {
     onNodeProgress: (input) => {
       updateStepProgress(input);
-      flushLedgerEvents(false);
+      flushLedgerEvents(Boolean(input.forceFlush));
     },
     onMapStarted: (nodeId, at) => stepLifecycle?.onMapStarted(nodeId, at),
     onMapCompleted: (nodeId, at) => stepLifecycle?.onMapCompleted(nodeId, at),