deepline 0.3.237 → 0.3.239

package/dist/bundling-sources/sdk/src/stream-reconnect.ts

@@ -1,13 +1,9 @@
 import { DeeplineError } from './errors.js';
 
 /**
- * Shared reconnect policy for the canonical run SSE stream.
- *
- * Server stream windows are finite: the platform ends them cleanly at the
- * function ceiling even while the run keeps executing. Consumers that wait for
- * a terminal status (`plays run --watch`, `runs tail`, `client.runs.tail`)
- * therefore reconnect with full-jitter exponential backoff instead of treating
- * a window end as a failure.
+ * Retry policy for the legacy run SSE compatibility method. Ordinary run
+ * watching uses a fixed REST snapshot cadence and shares only the transient
+ * read-error classification below.
  */
 
 /** Base delay for the full-jitter exponential backoff between reconnects. */
@@ -29,7 +25,7 @@ export function streamReconnectDelayMs(attempt: number): number {
   return Math.max(1, Math.floor(Math.random() * (cappedExponentialMs + 1)));
 }
 
-export function isTransientPlayStreamError(error: unknown): boolean {
+export function isTransientPlayRunReadError(error: unknown): boolean {
   if (
     error instanceof DeeplineError &&
     error.code === 'PLAY_STREAM_NETWORK_ABORTED'
@@ -37,10 +33,8 @@ export function isTransientPlayStreamError(error: unknown): boolean {
     return true;
   }
   if (error instanceof DeeplineError && typeof error.statusCode === 'number') {
-    // Server-shaped errors with a definite status code are NOT transient by
-    // pattern — only network-level failures are. 5xx counts as transient
-    // since the server may recover, but 4xx (especially 404 = run gone) is
-    // terminal and should not be hidden behind a silent retry loop.
+    // 5xx responses and network-level failures may recover. Client errors,
+    // especially 404, stay visible instead of disappearing into a retry loop.
     return error.statusCode >= 500 && error.statusCode < 600;
   }
   const text = error instanceof Error ? error.message : String(error);
@@ -48,3 +42,6 @@ export function isTransientPlayStreamError(error: unknown): boolean {
     text,
   );
 }
+
+/** @deprecated Use `isTransientPlayRunReadError`. */
+export const isTransientPlayStreamError = isTransientPlayRunReadError;

package/dist/bundling-sources/shared_libs/play-runtime/activity-observation.ts

@@ -263,10 +263,40 @@ function targetLabel(target: PlayActivityTarget): string {
     case 'step':
       return target.label ?? target.stepId;
     case 'capacity':
-      return target.label ?? `${target.pool} capacity`;
+      return target.pool === 'provider'
+        ? providerCapacityLabel(target.label)
+        : (target.label ?? `${target.pool} capacity`);
   }
 }
 
+/** Older waits carried a rate scope rather than a customer-facing provider. */
+export function providerCapacityLabel(label?: string): string {
+  if (!label) return 'The provider';
+  if (label.startsWith('org:')) return label.split(':')[1] || 'The provider';
+  return label.includes(':') ? 'The provider' : label;
+}
+
+function publicProviderCapacityObservation(
+  observation: PlayActivityObservation,
+): PlayActivityObservation {
+  if (
+    observation.target.kind !== 'capacity' ||
+    observation.target.pool !== 'provider'
+  )
+    return observation;
+  if (
+    !observation.target.label?.includes(':') &&
+    !observation.activityId.startsWith('capacity:provider:org:')
+  )
+    return observation;
+  const label = providerCapacityLabel(observation.target.label);
+  return {
+    ...observation,
+    activityId: `capacity:provider:${label}`,
+    target: { ...observation.target, label },
+  };
+}
+
 export function summarizePlayActivity(
   observation: PlayActivityObservation,
 ): string {
@@ -297,10 +327,10 @@ export function summarizePlayActivity(
       if (state.reason.kind === 'lease') {
         return `Contended: waiting for ${state.reason.resourceKind} lease ${state.reason.resourceLabel}.`;
       }
-      return `Paused: ${label}; the legacy producer did not report a typed reason.`;
+      return `Waiting: ${label}.`;
     case 'retrying':
       if (state.reason === 'rate_limit') {
-        return `Waiting for ${label} request capacity; the run will retry automatically.`;
+        return `Waiting for ${label}'s rate limit to allow more calls. Completed work is saved; remaining calls will continue automatically.`;
       }
       return `Delayed: ${label} will retry automatically.`;
     case 'scheduled':
@@ -385,6 +415,7 @@ export function projectPlayRunActivity(input: {
   status: string;
   updatedAt?: number | null;
   waitKind?: string | null;
+  waitBoundaryId?: string | null;
   waitUntil?: number | null;
   eventKey?: string | null;
   runtimeBackend?: string | null;
@@ -410,17 +441,24 @@ export function projectPlayRunActivity(input: {
   const latestExplicit = [...(input.explicit ?? [])]
     .filter((candidate) => !isTerminalActivityState(candidate.state))
     .sort((left, right) => right.observedAt - left.observedAt)[0];
-  // A provider wait is an observation, not a permanent run state. Once its
-  // deadline has passed, use the current lifecycle/step facts until the next
-  // producer observation arrives instead of claiming the old hold persists.
+  // Retry-After is the earliest retry, not evidence that execution resumed.
+  // Retain the reason while its provider boundary is still waiting, including
+  // lagging readers with no wait projection. A different durable wait wins.
+  const stillWaitingForProvider =
+    normalizedStatus === 'waiting' &&
+    (!input.waitKind ||
+      (input.waitKind === 'sleep' &&
+        input.waitBoundaryId?.includes(':provider-backpressure-') === true));
   const explicit =
+    !stillWaitingForProvider &&
     latestExplicit?.target.kind === 'capacity' &&
     latestExplicit.target.pool === 'provider' &&
     latestExplicit.state.kind === 'retrying' &&
     latestExplicit.state.retryAt < now
       ? null
       : latestExplicit;
-  if (explicit) return projection(explicit, now);
+  if (explicit)
+    return projection(publicProviderCapacityObservation(explicit), now);
 
   if (input.waitKind === 'detached_runner') {
     return projection(

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -57,6 +57,7 @@ import type {
 } from './governor/rate-state-backend';
 import type { MapRowOutcome } from './durability-store';
 import { stringifyPostgresJson } from './postgres-json';
+import { providerCapacityLabel } from './activity-observation';
 import { RUNTIME_ROW_OUTPUT_NOT_JSON_SERIALIZABLE } from './runtime-sheet-row-transition';
 import {
   RuntimeSheetRowWriter,
@@ -204,7 +205,7 @@ import {
   TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE,
   TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER,
   TOOL_EXECUTE_PROVIDER_RATE_RETRY_SNAPSHOT_HEADER,
-  TOOL_EXECUTE_PROVIDER_THROTTLE_HEADER,
+  isAttestedProvider429Response,
   classifyToolExecuteHttpFailure,
   createToolExecuteHttpFailureAttemptTracker,
   parseToolExecuteRetryAfterMs,
@@ -9143,6 +9144,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
       successfulCount = Math.max(0, successfulCount - staleCompletedKeys.size);
       failedCount = Math.max(0, failedCount - staleFailedKeys.size);
+      this.#options.onMapRowsSettled?.({
+        playName: this.#options.playName,
+        runId: this.#options.runId,
+        tableNamespace: resolvedTableNamespace,
+        succeededRows: successfulCount,
+        failedRows: failedCount,
+      });
 
       // Isolated failures are a partial result only when at least one admitted
       // row completed. A fully failed admission run is systemic: preserve the
@@ -9995,6 +10003,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       });
     }
 
+    this.#options.onMapRowsSettled?.({
+      playName: this.#options.playName,
+      runId: this.#options.runId,
+      tableNamespace: resolvedTableNamespace,
+      succeededRows:
+        previouslyCompletedRowKeys.length + visibleCompletedRows.length,
+      failedRows: durableFailedRows,
+    });
+
     // This map's terminal rows have already crossed the persistence barrier
     // above. Keep that verified snapshot as the authored return value instead
     // of immediately re-reading the Runtime Sheet projection. A second read
@@ -14532,11 +14549,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     retryAtMs: number,
   ): void {
     const observedAt = Date.now();
+    const label = providerCapacityLabel(provider);
     this.emitExecutionEvent({
       type: 'activity.observed',
       observation: {
-        activityId: `capacity:provider:${provider}`,
-        target: { kind: 'capacity', pool: 'provider', label: provider },
+        activityId: `capacity:provider:${label}`,
+        target: { kind: 'capacity', pool: 'provider', label },
         state: {
           kind: 'retrying',
           reason: 'rate_limit',
@@ -19546,12 +19564,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                               : undefined;
                           }
                           if (
-                            dispatchedResponse.status === 429 &&
-                            dispatchedResponse.headers.get(
-                              TOOL_EXECUTE_PROVIDER_THROTTLE_HEADER,
-                            ) === '1'
+                            isAttestedProvider429Response(dispatchedResponse)
                           ) {
                             this.resourceGovernor.recordProvider429();
+                          }
+                          if (
+                            dispatchedResponse.status === 429 &&
+                            isAttestedProvider429Response(dispatchedResponse)
+                          ) {
                             provider429RetryDiagnosticStartedAt =
                               performance.now();
                             provider429RetryDiagnosticNextInvocationAttempt =

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -931,6 +931,14 @@ export interface ContextOptions {
      */
     supersededKeys?: string[];
   }>;
+  /** Distinct terminal rows after the map's persistence barrier, across resumes. */
+  onMapRowsSettled?: (input: {
+    playName?: string;
+    runId?: string;
+    tableNamespace: string;
+    succeededRows: number;
+    failedRows: number;
+  }) => void;
   /**
    * Persists coalesced, non-terminal row patches before a map suspends.
    * Terminal rows use onMapRowsCompleted; this narrow seam only preserves

package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts

@@ -30,9 +30,9 @@ import { stripProvenanceTag } from './log-provenance';
  * The canonical client-side projection of a persisted Play Run document into
  * the live snapshot shape, plus the differ that turns successive snapshots
  * into incremental `play.*` live events. This module is shared between the
- * Vercel app (legacy SSE shim, dashboard) and the SDK subscription transport
- * (`packages/sdk/src/runs/observe-transport.ts`), so both watchers render identical
- * event streams from the same Convex Run Snapshot. See ADR-0008.
+ * Vercel app (dashboard and legacy SSE compatibility) and the SDK REST polling
+ * watcher (`packages/sdk/src/runs/watch-polling.ts`), so both use the same
+ * event projection from the persisted Run Snapshot. See ADR-0008.
  */
 
 export type PlayRunLiveStatus = PlayRunLifecycleStatus;
@@ -328,6 +328,7 @@ export function buildPlayRunStatusSnapshot(input: {
     status: projectedSnapshot.status,
     updatedAt: projectedSnapshot.updatedAt,
     waitKind: input.run.waitKind ?? input.run.wait?.kind ?? null,
+    waitBoundaryId: input.run.wait?.boundaryId ?? input.run.activeBoundaryId,
     waitUntil: input.run.waitUntil,
     eventKey: input.run.wait?.eventKey ?? null,
     runtimeBackend: input.run.runtimeBackend,
@@ -551,7 +552,7 @@ function diffLogLines(input: {
   const missingCount = tailFirstSeq - 1 - input.lastLogSeq;
   return {
     lines: [
-      `[stream] ${missingCount} log lines not retained in the live window; full logs via runs logs`,
+      `[watch] ${missingCount} log lines not retained in the live window; full logs via runs logs`,
       ...logs,
     ],
     lastLogSeq: totalLogCount,

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts

@@ -496,6 +496,20 @@ export function runtimeSheetTerminalWriteCounts(input: {
   };
 }
 
+/** Distinct settled counts include saved siblings, never skipped input duplicates. */
+export function runtimeSheetTerminalMapCounts(
+  current: { persistedRows: number; succeededRows: number; failedRows: number },
+  map: { succeededRows: number; failedRows: number },
+): { persistedRows: number; succeededRows: number; failedRows: number } {
+  const succeededRows = Math.max(current.succeededRows, map.succeededRows);
+  const failedRows = Math.max(current.failedRows, map.failedRows);
+  return {
+    persistedRows: Math.max(current.persistedRows, succeededRows + failedRows),
+    succeededRows,
+    failedRows,
+  };
+}
+
 async function vercelBypassHeaders(
   config: PlayRunnerExecutionConfig,
 ): Promise<Record<string, string>> {
@@ -2308,6 +2322,35 @@ async function run(
       const MAP_ROW_UPDATED_EMIT_INTERVAL_MS = 200;
       const MAP_ROW_UPDATED_EMIT_SAMPLE_RATE = 100;
       return (event: PlayExecutionEvent) => {
+        if (event.type === 'map.suspended') {
+          const datasetId = createRuntimeDatasetId(
+            config.context.playName!,
+            event.artifactTableNamespace,
+          );
+          const settlement = runtimeMapDatasetSettlements.get(datasetId);
+          if (settlement) {
+            // This event follows the map writer's durability barrier. Only
+            // report accepted writes, not progress (which includes duplicates).
+            emitRunnerEvent(
+              {
+                type: 'execution_event',
+                event: {
+                  type: 'dataset.lifecycle',
+                  datasetId,
+                  path: settlement.path,
+                  tableNamespace: settlement.tableNamespace,
+                  phase: 'registered',
+                  persistedRows: settlement.persistedRows,
+                  succeededRows: settlement.succeededRows,
+                  failedRows: settlement.failedRows,
+                  complete: false,
+                  at: event.at,
+                },
+              },
+              options,
+            );
+          }
+        }
         if (event.type === 'map.row.updated') {
           mapRowUpdatedCount++;
           const now = Date.now();
@@ -2638,6 +2681,19 @@ async function run(
           ),
       });
     },
+    onMapRowsSettled: (input) => {
+      const datasetId = createRuntimeDatasetId(
+        input.playName ?? config.context.playName!,
+        input.tableNamespace,
+      );
+      const settlement = runtimeMapDatasetSettlements.get(datasetId);
+      if (settlement && (!input.runId || settlement.runId === input.runId)) {
+        Object.assign(
+          settlement,
+          runtimeSheetTerminalMapCounts(settlement, input),
+        );
+      }
+    },
     onMapRowsCompleted: async (input) => {
       // Persist executed rows (data + cell meta) into the tenant sheet —
       // the sheet is the source of truth for dataset state, and the

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -86,6 +86,23 @@ export const TOOL_EXECUTE_PROVIDER_THROTTLE_HEADER =
   'x-deepline-provider-throttle';
 export const TOOL_EXECUTE_PROVIDER_PACING_WAIT_CODE = 'PROVIDER_PACING_WAIT';
 
+/** A trusted provider 429 remains observable if the gateway cannot commit its body. */
+export function isAttestedProvider429Response(response: {
+  status: number;
+  headers: Pick<Headers, 'get'>;
+}): boolean {
+  if (response.headers.get(TOOL_EXECUTE_PROVIDER_THROTTLE_HEADER) !== '1') {
+    return false;
+  }
+  if (response.status === 429) return true;
+  const code = response.headers.get('x-deepline-error-code');
+  return (
+    response.status === 409 &&
+    (code === 'durable_invocation_outcome_unknown' ||
+      code === 'durable_invocation_commit_failed')
+  );
+}
+
 /**
  * Parse the advertisement as a comma-separated token list so a later runner can
  * add a capability without a second header. An absent or unparsable value means