deepline 0.3.257 → 0.3.259

package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts

@@ -10,12 +10,14 @@ import {
   type PlayRunRowOutcomeSummary,
 } from './run-ledger';
 import {
+  decodePlayRunPublicStatus,
   isActivePlayRunLifecycleStatus,
   isTerminalPlayRunLifecycleStatus,
   normalizePlayRunLifecycleStatus,
   type PlayRunLifecycleStatus,
 } from './run-lifecycle-policy';
 import type { PlayDocflowNodeIoState } from './docflow-node-io';
+import type { RunStopCause } from './run-execution-state';
 import type { PlayVisualNodeUsageSnapshot } from './live-state-contract';
 import {
   projectPlayRunActivity,
@@ -57,6 +59,8 @@ export type PlayRunStreamNodeState = {
   nodeId: string;
   status: 'idle' | 'running' | 'completed' | 'failed' | 'skipped';
   error?: string | null;
+  /** The run's terminal event settled this step in flight (ledger fact). */
+  stoppedBy?: RunStopCause;
   artifactTableNamespace?: string | null;
   progress?: PlayRunStreamNodeProgress | null;
   startedAt?: number | null;
@@ -193,6 +197,28 @@ function extractTerminalRunLogTail(result: unknown): {
   };
 }
 
+/**
+ * Whether the run stopped under this step rather than the step failing. The
+ * ledger records it (`stoppedBy`) since run-execution-state; a snapshot written
+ * before that settled every in-flight step `failed` on the run's terminal
+ * event WITHOUT an error of its own, while a step that failed itself carries
+ * one (`step.failed` with its error). Projected here, once, so every reader —
+ * the run read, the observation, the CLI — gets the same fact.
+ */
+function stepStopCause(
+  step: PlayRunLedgerStepSnapshot,
+  runStatus: PlayRunLedgerSnapshot['status'],
+): { stoppedBy?: RunStopCause } {
+  if (step.status !== 'failed') return {};
+  if (step.stoppedBy) return { stoppedBy: step.stoppedBy };
+  const run = decodePlayRunPublicStatus(runStatus);
+  if (run === 'cancelled') return { stoppedBy: 'run_cancelled' };
+  if (run === 'failed' && !step.error?.trim()) {
+    return { stoppedBy: 'run_failed' };
+  }
+  return {};
+}
+
 function buildSnapshotFromLedger(
   snapshot: PlayRunLedgerSnapshot,
 ): PlayRunLiveSnapshot {
@@ -203,6 +229,7 @@ function buildSnapshotFromLedger(
       nodeId: step.stepId,
       status: step.status,
       error: step.error ?? null,
+      ...stepStopCause(step, snapshot.status),
       artifactTableNamespace: step.artifactTableNamespace ?? null,
       progress: step.progress
         ? {
@@ -327,9 +354,15 @@ export function buildPlayRunStatusSnapshot(input: {
     playName: input.run.name,
     status: projectedSnapshot.status,
     updatedAt: projectedSnapshot.updatedAt,
-    waitKind: input.run.waitKind ?? input.run.wait?.kind ?? null,
+    // The typed wait is recorded on the durable `run.waiting` ledger event;
+    // run-record fields are an older projection that may not carry it.
+    waitKind:
+      input.run.waitKind ??
+      input.run.wait?.kind ??
+      ledgerSnapshot.waitKind ??
+      null,
     waitBoundaryId: input.run.wait?.boundaryId ?? input.run.activeBoundaryId,
-    waitUntil: input.run.waitUntil,
+    waitUntil: input.run.waitUntil ?? ledgerSnapshot.waitUntil ?? null,
     eventKey: input.run.wait?.eventKey ?? null,
     runtimeBackend: input.run.runtimeBackend,
     activeNodeId: input.run.activeBoundaryId ?? projectedSnapshot.activeNodeId,

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts

@@ -43,6 +43,10 @@ import {
   runtimeLimitExceededFailure,
 } from '../run-failure';
 import { createPlayContext } from '../context';
+import {
+  isHtmlErrorBody,
+  summarizeHtmlErrorBody,
+} from '../tool-http-errors';
 import { augmentSheetContractWithDatasetFields } from '../../play-data-plane/sheet-contract';
 import { nextWorkCounterInvocationSeq } from '../runtime-work-counters';
 import type { RuntimeApiResponseMeta } from '../app-runtime-api';
@@ -688,9 +692,11 @@ function createRuntimeToolMetadataResolver(config: PlayRunnerExecutionConfig) {
         },
       });
       if (!response.ok) {
-        throw new Error(
-          `Runtime metadata lookup for tool ${normalizedToolId} failed (${response.status}): ${await response.text()}`,
-        );
+        throw await runtimeControlHttpError({
+          description: `Runtime metadata lookup for tool ${normalizedToolId}`,
+          toolId: normalizedToolId,
+          response,
+        });
       }
       return (await response.json()) as RuntimeToolMetadata;
     })();
@@ -730,9 +736,11 @@ function createAppRuntimeToolAuthScopeDigestResolver(
         },
       });
       if (!response.ok) {
-        throw new Error(
-          `Runtime auth scope lookup for tool ${normalizedToolId} failed (${response.status}): ${await response.text()}`,
-        );
+        throw await runtimeControlHttpError({
+          description: `Runtime auth scope lookup for tool ${normalizedToolId}`,
+          toolId: normalizedToolId,
+          response,
+        });
       }
       const body = (await response.json()) as RuntimeToolAuthScope;
       const digest = typeof body.digest === 'string' ? body.digest.trim() : '';
@@ -952,11 +960,62 @@ export function formatRuntimeControlFailure(input: {
       message = [code, error, operatorHint].filter(Boolean).join(': ');
     }
   } catch {
-    // Preserve the original runtime-control response for unstructured errors.
+    // Not JSON. An edge proxy's HTML error page (a tunnel or gateway that
+    // never reached the app) is kilobytes of markup with nothing in it but a
+    // title and a Ray ID: name the failure and keep those, never the page —
+    // it was stored verbatim as every failed row's error (6.7 KB a row).
+    message = isHtmlErrorBody(input.bodyText)
+      ? `${runtimeControlStatusLabel(input.status)} — ${summarizeHtmlErrorBody(input.bodyText)}`
+      : input.bodyText.replace(/\s+/g, ' ').trim().slice(0, 500);
   }
   return `${input.description} failed (${input.status}): ${message}`;
 }
 
+/** What a gateway status means, for a reader who sees only the message. */
+function runtimeControlStatusLabel(status: number): string {
+  switch (status) {
+    case 502:
+      return 'Bad gateway';
+    case 503:
+      return 'Service unavailable';
+    case 504:
+      return 'Gateway timeout';
+    case 520:
+    case 521:
+    case 522:
+    case 523:
+    case 524:
+    case 525:
+    case 526:
+    case 530:
+      return 'Origin unreachable';
+    default:
+      return `HTTP ${status}`;
+  }
+}
+
+/**
+ * A failed runtime-control call as an Error that says WHICH call, HOW it
+ * failed (`status`) and for WHICH tool — never the raw response body.
+ */
+async function runtimeControlHttpError(input: {
+  description: string;
+  toolId: string;
+  response: Response;
+}): Promise<Error> {
+  const bodyText = await input.response.text().catch(() => '');
+  return Object.assign(
+    new Error(
+      formatRuntimeControlFailure({
+        description: input.description,
+        status: input.response.status,
+        bodyText,
+      }),
+    ),
+    { status: input.response.status, toolId: input.toolId },
+  );
+}
+
 function runtimeControlSlackHitlTimeoutError(input: {
   toolId: string;
   phase: 'prepare' | 'arm' | 'timeout';

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -3118,35 +3118,6 @@ function mapRuntimePostgresRow(input: {
   };
 }
 
-function completedRuntimeCellMetaPatch(input: {
-  runId: string;
-  outputFields: readonly string[];
-  rowPatch?: Record<string, unknown>;
-}): Record<string, unknown> {
-  const patch: Record<string, unknown> = {};
-  const completedAt = Date.now();
-  for (const field of input.outputFields) {
-    const existing =
-      input.rowPatch?.[field] &&
-      typeof input.rowPatch[field] === 'object' &&
-      !Array.isArray(input.rowPatch[field])
-        ? (input.rowPatch[field] as Record<string, unknown>)
-        : {};
-    patch[field] = {
-      status: 'completed',
-      runId: input.runId,
-      completedAt,
-      ...existing,
-    };
-  }
-  for (const [field, meta] of Object.entries(input.rowPatch ?? {})) {
-    if (!Object.hasOwn(patch, field)) {
-      patch[field] = meta;
-    }
-  }
-  return patch;
-}
-
 function mergeRuntimeCellMetaPatchSql(
   targetExpression: string,
   patchExpression: string,

package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts

@@ -49,9 +49,15 @@ export function completedRuntimeCellMetaPatch(input: {
         : {};
     patch[field] = {
       status: 'completed',
-      runId: input.runId,
       completedAt,
       ...existing,
+      // This run wrote the row, so this run is the cell's run — including a
+      // cell it REUSED (from a receipt or an earlier run's row), whose meta
+      // can still carry the run that first produced it. Spread last so a
+      // carried stamp never wins: a reused row kept the first run's stamp,
+      // the reusing run's column counts held none of its cells, and the
+      // canvas lent the step its table's count instead.
+      runId: input.runId,
     };
   }
   for (const [field, meta] of Object.entries(input.rowPatch ?? {})) {
@@ -62,6 +68,41 @@ export function completedRuntimeCellMetaPatch(input: {
   return patch;
 }
 
+/**
+ * Cell meta for a failed row's terminal write.
+ *
+ * The durable merge is per cell and shallow (`existing || patch`), so a cell
+ * patch that omits `runId` keeps whatever run stamped the cell before. A failed
+ * row re-executed by a later run therefore kept the earlier run's stamp on the
+ * very cell that failed now, and every per-run column count (which keys on the
+ * cell's `runId`) dropped this run's failure.
+ *
+ * Every output cell this run touched — the ones it carries meta for — is
+ * stamped with the current run, exactly as a completed row stamps its cells; the
+ * cell's own status (`failed` for the column that threw, `completed` for the
+ * ones that resolved before it) is kept. An output cell this run never reached
+ * keeps its earlier meta and value: that is still the earlier run's fact, and
+ * the row's failure is attributed through the cell that failed.
+ */
+export function failedRuntimeCellMetaPatch(input: {
+  runId: string;
+  outputFields: readonly string[];
+  rowPatch?: Record<string, unknown>;
+}): Record<string, unknown> {
+  const patch: Record<string, unknown> = { ...(input.rowPatch ?? {}) };
+  for (const field of input.outputFields) {
+    const existing = patch[field];
+    if (!existing || typeof existing !== 'object' || Array.isArray(existing)) {
+      continue;
+    }
+    patch[field] = {
+      ...(existing as Record<string, unknown>),
+      runId: input.runId,
+    };
+  }
+  return patch;
+}
+
 function serializeRuntimeRowPatch(value: Record<string, unknown>): string {
   const serialized = stringifyPostgresJson(value);
   if (serialized === undefined) {
@@ -126,7 +167,11 @@ export function prepareRuntimeSheetRowTransitions(input: {
     try {
       cellMetaPatch =
         row.status === 'failed'
-          ? (row.cellMetaPatch ?? {})
+          ? failedRuntimeCellMetaPatch({
+              runId: input.runId,
+              outputFields: input.outputFields,
+              rowPatch: row.cellMetaPatch,
+            })
           : completedRuntimeCellMetaPatch({
               runId: input.runId,
               outputFields: input.outputFields,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -170,6 +170,12 @@ export type PlaySchedulerSubmitInput = RetainedPipelineCarrier & {
   source?: 'published' | 'ad_hoc' | 'draft';
   /** Invocation origin, distinct from the artifact source above. */
   triggerSource?: 'api' | 'webhook' | 'cron' | 'sql_listener' | 'async_child';
+  /**
+   * Display-only client surface of a person-started run (`ui` | `cli` |
+   * `api`). Optional: older launchers and automated triggers omit it, and
+   * readers show nothing rather than guess.
+   */
+  launchSurface?: 'ui' | 'cli' | 'api' | null;
   /** Null lets a durable trigger wait for capacity; a number bounds queue delay. */
   queueMaxDelaySeconds?: number | null;
   /**