deepline 0.3.290 → 0.3.292

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -40,7 +40,7 @@ const MAX_TIMEOUT_MS = 2_147_483_647;
 /**
  * One projector can keep this many independent Run Ledger deliveries in
  * flight after it has found durable work. Each delivery owns only short
- * Postgres checkouts: claim, token-context read, and acknowledgement. It does
+ * Postgres checkouts: claim with routing, renewal, and acknowledgement. It does
  * not hold a scheduler connection across either gateway HTTP request. This is
  * therefore a network/projection concurrency budget, deliberately independent
  * of the projector's much smaller Postgres pool.

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts

@@ -3,8 +3,12 @@
  *
  * After reader-first cutover, triggers persist raw input in the existing
  * outbox. The leased dispatcher projects and compacts it before delivery.
- * Receipt output remains exclusively in the authoritative receipt store.
- * The API may read only through fixed, tenant-scoped queries in this module.
+ * PlanetScale runtime_observability.events is the single log/event/receipt
+ * discovery surface. JSONPath runs natively against its indexed search_doc;
+ * exact identities use indexed columns on that same table. Full saved results
+ * resolve through the scoped fetch_receipt capability, never a second search
+ * store or a CLI-side scan. Receipt output remains authoritative in its store.
+ * The API uses only the fixed, tenant-scoped readers in this module.
  */
 import { normalizeRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
 import type { PostgresSchedulerQueryClient } from './postgres';
@@ -57,7 +61,7 @@ const RUNTIME_EVENT_LOG_POSITION_EXPRESSION = `jsonb_set(
 // Migration readiness tracks this projection body independently of scheduler
 // wire and schema versions. Bump when a changed trigger must be reinstalled.
 export const RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER =
-  'runtime-event-search-origin-batches-independent-progress';
+  'runtime-event-search-origin-batches-log-metadata';
 /** Trusted trigger-authored state; absent on already-projected legacy rows. */
 export const RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD =
   '_runtimeSearchProjectionPending';
@@ -177,7 +181,7 @@ function parseRuntimeEventJsonpath(value: string): RuntimeEventPredicate[] {
     );
   }
   const clause =
-    /\$\.(orgId|runId|playName|runtimeScope|producerAttempt|kind|status|source|stepId|stepKind|datasetId|phase|provider|operation|context\.[A-Za-z_][A-Za-z0-9_]{0,63})\s*==\s*("(?:\\.|[^"\\])*"|true|false|-?\d+(?:\.\d+)?)/y;
+    /\$\.(eventId|orgId|runId|playName|runtimeScope|producerAttempt|kind|status|source|level|stepId|stepKind|datasetId|phase|provider|operation|context\.[A-Za-z_][A-Za-z0-9_]{0,63})\s*==\s*("(?:\\.|[^"\\])*"|true|false|-?\d+(?:\.\d+)?)/y;
   let position = 0;
   let clauses = 0;
   const predicates: RuntimeEventPredicate[] = [];
@@ -214,7 +218,9 @@ function parseRuntimeEventJsonpath(value: string): RuntimeEventPredicate[] {
       );
     }
     if (
-      ['orgId', 'runId', 'playName', 'runtimeScope'].includes(match[1]) &&
+      ['eventId', 'orgId', 'runId', 'playName', 'runtimeScope'].includes(
+        match[1],
+      ) &&
       typeof parsed !== 'string'
     ) {
       throw new RuntimeEventInputError(
@@ -231,6 +237,14 @@ function parseRuntimeEventJsonpath(value: string): RuntimeEventPredicate[] {
         'Runtime event producerAttempt requires a positive integer.',
       );
     }
+    if (
+      match[1] === 'level' &&
+      !['debug', 'info', 'warn', 'error'].includes(String(parsed))
+    ) {
+      throw new RuntimeEventInputError(
+        'Runtime event level requires debug, info, warn or error.',
+      );
+    }
     predicates.push({ path: match[1], literal: match[2], value: parsed });
     if (
       match[1].startsWith('context.') &&
@@ -348,6 +362,7 @@ export async function searchRuntimeEvents(
   addExact('kind', input.kind);
   if (input.jsonpath !== undefined) {
     const identityColumns = {
+      eventId: 'event_id',
       orgId: 'org_id',
       runId: 'run_id',
       playName: 'play_name',
@@ -1170,7 +1185,19 @@ export async function attachRuntimeEventSearchTriggers(
         NEW.updated_at, NEW.run_id, play, 'receipt.' || NEW.status,
         ref, NEW.k, jsonb_strip_nulls(jsonb_build_object(
           'kind', 'receipt.' || NEW.status, 'status', NEW.status,
-          'source', 'receipt', 'producerAttempt', NEW.lease_owner_attempt
+          'source', 'receipt', 'producerAttempt', NEW.lease_owner_attempt,
+          -- Only the reserved runtime diagnostic record supplies this flat
+          -- context. Ordinary authored receipt outputs are never indexed.
+          'context', CASE WHEN NEW.k LIKE 'tool-evidence:%'
+            AND NEW.output_json->>'kind' = 'tool_call_evidence' THEN jsonb_strip_nulls(jsonb_build_object(
+              'callKey', NEW.output_json#>'{context,callKey}',
+              'toolId', NEW.output_json#>'{context,toolId}',
+              'callStatus', NEW.output_json#>'{context,callStatus}',
+              'stepId', NEW.output_json#>'{context,stepId}',
+              'dataset', NEW.output_json#>'{context,dataset}',
+              'column', NEW.output_json#>'{context,column}',
+              'rowKey', NEW.output_json#>'{context,rowKey}'
+            )) END
         ))
       );
       RETURN NEW;
@@ -1189,6 +1216,41 @@ export async function attachRuntimeEventSearchTriggers(
         FOR EACH ROW EXECUTE FUNCTION ${schema}.search_receipt_event();
     END IF;
   END $create$`);
+  await client.query(`
+    CREATE OR REPLACE FUNCTION ${schema}.run_log_metadata(line text, metadata jsonb, fallback_at timestamptz) RETURNS jsonb
+    LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog AS $function$
+    DECLARE clean text; message text; level text; at_time timestamptz := fallback_at;
+    BEGIN
+      clean := regexp_replace(line, '^' || chr(1) || 'prov:[^' || chr(1) || ']*' || chr(1), '');
+      IF metadata IS NOT NULL AND metadata <> 'null'::jsonb THEN
+        IF jsonb_typeof(metadata) IS DISTINCT FROM 'object'
+          OR NOT coalesce(metadata->>'level' IN ('debug','info','warn','error'), false)
+          OR jsonb_typeof(metadata->'occurredAt') IS DISTINCT FROM 'number'
+          OR NOT coalesce(metadata->>'occurredAt' ~ '^[1-9][0-9]{0,15}$', false)
+          OR (metadata ? 'stepId' AND (jsonb_typeof(metadata->'stepId') IS DISTINCT FROM 'string'
+            OR length(metadata->>'stepId') NOT BETWEEN 1 AND 256)) THEN
+          RAISE EXCEPTION 'Run log metadata is invalid';
+        END IF;
+        RETURN jsonb_strip_nulls(jsonb_build_object('level', metadata->'level',
+          'occurredAt', metadata->'occurredAt', 'stepId', metadata->'stepId'));
+      END IF;
+      -- Historical lines already carry the original ISO time and severity.
+      IF clean ~ '^\\[[-0-9]{10}T[0-9:.]+Z\\]' THEN
+        BEGIN at_time := substring(clean from '^\\[([^]]+)\\]')::timestamptz;
+        EXCEPTION WHEN invalid_datetime_format OR datetime_field_overflow THEN at_time := fallback_at; END;
+      END IF;
+      message := regexp_replace(clean, '^\\[[-0-9]{10}T[0-9:.]+Z\\][ ]*', '');
+      level := CASE
+        WHEN message ~* '^\\[(error|console.error)\\]' THEN 'error'
+        WHEN message ~* '^\\[(warn|console.warn)\\]' THEN 'warn'
+        WHEN message ~* '^\\[info\\]' THEN 'info'
+        WHEN message ~* '^\\[(debug|console.debug|perf|runtime.(phase_summary|logs|log_transport))\\]'
+          OR line ~ ('^' || chr(1) || 'prov:(infra|receipt|replay)' || chr(1)) THEN 'debug'
+        ELSE 'info' END;
+      RETURN jsonb_build_object('level', level, 'occurredAt', floor(extract(epoch FROM at_time) * 1000)::bigint);
+    END
+    $function$
+  `);
   await client.query(`
     CREATE OR REPLACE FUNCTION ${schema}.project_origin_log_batch(observation ${schema}.outbox, item jsonb, run_org text, play text, compact_logs boolean, canonical_events boolean, at_time timestamptz, doc jsonb, closed boolean)
     RETURNS text[] LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
@@ -1221,7 +1283,8 @@ export async function attachRuntimeEventSearchTriggers(
       WITH entries AS MATERIALIZED (
         SELECT value #>> '{}' AS body, ordinality AS ordinal,
           'log-origin:' || observation.run_id || ':' || (item->'lineIds'->>(ordinality-1)::int) AS eid,
-          NULLIF(item->'lineContexts'->(ordinality-1)::int, 'null'::jsonb) AS ctx
+          NULLIF(item->'lineContexts'->(ordinality-1)::int, 'null'::jsonb) AS ctx,
+          ${schema}.run_log_metadata(value #>> '{}', item->'lineMetadata'->(ordinality-1)::int, at_time) AS metadata
         FROM jsonb_array_elements(item->'lines') WITH ORDINALITY
       ), inserted AS (
         INSERT INTO runtime_observability.events (
@@ -1229,16 +1292,16 @@ export async function attachRuntimeEventSearchTriggers(
           run_id, play_name, kind, level, message, log_body, source_digest, log_text_digest, search_doc
         ) SELECT ${quoteLiteral(scope)}, run_org, eid,
           CASE WHEN canonical_events THEN nextval('runtime_observability.runtime_event_order_seq') END,
-          CASE WHEN canonical_events THEN jsonb_strip_nulls((item - 'lines' - 'lineContexts' - 'lineIds') || jsonb_build_object(
+          CASE WHEN canonical_events THEN jsonb_strip_nulls((item - 'lines' - 'lineContexts' - 'lineIds' - 'lineMetadata') || jsonb_build_object(
             'type', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'runId', observation.run_id,
-            'occurredAt', floor(extract(epoch FROM at_time)*1000)::bigint, 'lineOrdinal', ordinal,
+            'occurredAt', metadata->'occurredAt', 'level', metadata->'level', 'stepId', metadata->'stepId', 'lineOrdinal', ordinal,
             'entryId', item->'lineIds'->(ordinal-1)::int, 'context', ctx)) END,
-          at_time, observation.run_id, play, ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, NULL,
+          to_timestamp((metadata->>'occurredAt')::double precision / 1000), observation.run_id, play, ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, metadata->>'level',
           CASE WHEN compact_logs THEN NULL ELSE left(body, 2048) END, body,
           CASE WHEN ctx IS NULL THEN encode(sha256(convert_to(body,'UTF8')),'hex')
             ELSE encode(sha256(convert_to(jsonb_build_array(body,ctx)::text,'UTF8')),'hex') END,
           encode(sha256(convert_to(btrim(body),'UTF8')),'hex'),
-          doc || jsonb_strip_nulls(jsonb_build_object('entryId', item->'lineIds'->(ordinal-1)::int, 'context', ctx))
+          doc || (metadata - 'occurredAt') || jsonb_strip_nulls(jsonb_build_object('entryId', item->'lineIds'->(ordinal-1)::int, 'context', ctx))
         FROM entries WHERE NOT closed ORDER BY ordinal
         ON CONFLICT (runtime_scope, org_id, event_id) DO NOTHING RETURNING event_id
       ) SELECT coalesce(array_agg(inserted.event_id ORDER BY entries.ordinal), ARRAY[]::text[])
@@ -1264,7 +1327,7 @@ export async function attachRuntimeEventSearchTriggers(
     LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
     DECLARE run_org text; play text; compact_logs boolean; canonical_events boolean; run_status text; item jsonb; line jsonb; event_batch jsonb;
             ordinal bigint; line_ordinal bigint; kind text; at_time timestamptz;
-            doc jsonb; line_doc jsonb; line_context jsonb;
+            doc jsonb; line_doc jsonb; line_context jsonb; line_metadata jsonb; line_at timestamptz;
             eid text; existing_digest text; digest text; assigned_seq bigint; terminal_progress_identity boolean;
             log_channel text; log_position bigint; matched_event_id text; matched_doc jsonb;
             legacy_position_prefix text; position_event_ids text[];
@@ -1349,6 +1412,7 @@ export async function attachRuntimeEventSearchTriggers(
             'producerAttempt', envelope_attempt, 'channelOffset', 0,
             'lines', observation.payload_json->'terminalLogLines',
             'lineContexts', observation.payload_json->'terminalLogContexts',
+            'lineMetadata', observation.payload_json->'terminalLogMetadata',
             'lineIds', NULLIF(observation.payload_json->'terminalLogIds', 'null'::jsonb)
           ))
         );
@@ -1453,6 +1517,11 @@ export async function attachRuntimeEventSearchTriggers(
                  OR jsonb_array_length(item->'lineContexts') <> jsonb_array_length(item->'lines')) THEN
             RAISE EXCEPTION 'Search projection requires lineContexts aligned to log lines';
           END IF;
+          IF kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND item ? 'lineMetadata'
+            AND (jsonb_typeof(item->'lineMetadata') IS DISTINCT FROM 'array'
+              OR jsonb_array_length(item->'lineMetadata') <> jsonb_array_length(item->'lines')) THEN
+            RAISE EXCEPTION 'Run log metadata must align with lines';
+          END IF;
           IF kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND item ? 'lineIds' THEN
             IF jsonb_typeof(item->'lineIds') IS DISTINCT FROM 'array' OR jsonb_array_length(item->'lineIds') <> jsonb_array_length(item->'lines') THEN
               RAISE EXCEPTION 'Origin log identities must align with lines';
@@ -1524,8 +1593,10 @@ export async function attachRuntimeEventSearchTriggers(
                   RAISE EXCEPTION 'Search projection log context has an unsupported key or value';
                 END IF;
               END IF;
-              line_doc := CASE WHEN line_context IS NULL THEN doc
-                ELSE doc || jsonb_build_object('context', line_context) END;
+              line_metadata := ${schema}.run_log_metadata(line #>> '{}', item->'lineMetadata'->((line_ordinal - 1)::int), at_time);
+              line_at := to_timestamp((line_metadata->>'occurredAt')::double precision / 1000);
+              line_doc := (CASE WHEN line_context IS NULL THEN doc
+                ELSE doc || jsonb_build_object('context', line_context) END) || (line_metadata - 'occurredAt');
               eid := CASE WHEN (item->>'channelOffset') ~ '^[0-9]{1,12}$'
                 AND (item->>'producerAttempt') ~ '^[0-9]{1,6}$'
                 THEN 'log:' || observation.run_id || ':' || (item->>'producerAttempt') || ':' ||
@@ -1673,14 +1744,14 @@ export async function attachRuntimeEventSearchTriggers(
                 ${quoteLiteral(scope)}, run_org, eid,
                 CASE WHEN canonical_events THEN nextval('runtime_observability.runtime_event_order_seq') END,
                 CASE WHEN canonical_events THEN jsonb_strip_nulls(
-                  (item - 'lines' - 'lineContexts' - 'lineIds') || jsonb_build_object(
+                  (item - 'lines' - 'lineContexts' - 'lineMetadata' - 'lineIds') || jsonb_build_object(
                     'type', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'runId', observation.run_id,
-                    'occurredAt', floor(extract(epoch FROM at_time) * 1000)::bigint,
+                    'occurredAt', line_metadata->'occurredAt', 'level', line_metadata->'level', 'stepId', line_metadata->'stepId',
                     'lineOrdinal', line_ordinal,
                     'context', line_context
                   )) END,
-                at_time, observation.run_id, play,
-                ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, NULL, CASE WHEN compact_logs THEN NULL ELSE left(line #>> '{}', 2048) END,
+                line_at, observation.run_id, play,
+                ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, line_metadata->>'level', CASE WHEN compact_logs THEN NULL ELSE left(line #>> '{}', 2048) END,
                 line #>> '{}', digest,
                 encode(sha256(convert_to(btrim(line #>> '{}'), 'UTF8')), 'hex'), line_doc
               ) ON CONFLICT DO NOTHING;
@@ -1760,6 +1831,15 @@ export async function attachRuntimeEventSearchTriggers(
           AND NOT (observation.payload_json ? 'terminalLogLines') THEN
           RAISE EXCEPTION 'Search projection requires terminalLogLines for terminalLogContexts';
         END IF;
+        IF observation.payload_json ? 'terminalLogMetadata'
+          AND NOT (observation.payload_json ? 'terminalLogLines') THEN
+          RAISE EXCEPTION 'Search projection requires terminalLogLines for terminalLogMetadata';
+        END IF;
+        IF observation.payload_json ? 'terminalLogMetadata'
+          AND (jsonb_typeof(observation.payload_json->'terminalLogMetadata') IS DISTINCT FROM 'array'
+            OR jsonb_array_length(observation.payload_json->'terminalLogMetadata') <> jsonb_array_length(observation.payload_json->'terminalLogLines')) THEN
+          RAISE EXCEPTION 'Search projection requires terminalLogMetadata aligned to terminalLogLines';
+        END IF;
         IF observation.event_type IN (${RUN_EVENT_TERMINAL_LOG_CARRIER_SQL_LIST})
           AND observation.payload_json ? 'terminalLogLines'
           AND NOT coalesce((envelope_attempt #>> '{}') ~ '^[1-9][0-9]{0,5}$', false) THEN
@@ -1852,7 +1932,9 @@ export async function attachRuntimeEventSearchTriggers(
                 RAISE EXCEPTION 'Search projection terminal log context has an unsupported key or value';
               END IF;
             END IF;
-            line_doc := jsonb_strip_nulls(jsonb_build_object(
+            line_metadata := ${schema}.run_log_metadata(line #>> '{}', observation.payload_json->'terminalLogMetadata'->((line_ordinal - 1)::int), observation.created_at);
+            line_at := to_timestamp((line_metadata->>'occurredAt')::double precision / 1000);
+            line_doc := (line_metadata - 'occurredAt') || jsonb_strip_nulls(jsonb_build_object(
               'kind', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'source', 'terminal',
               'producerAttempt', observation.payload_json->'producerAttempt'
             ));
@@ -1868,11 +1950,11 @@ export async function attachRuntimeEventSearchTriggers(
               CASE WHEN canonical_events THEN nextval('runtime_observability.runtime_event_order_seq') END,
               CASE WHEN canonical_events THEN jsonb_strip_nulls(jsonb_build_object(
                 'type', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'runId', observation.run_id,
-                'occurredAt', floor(extract(epoch FROM observation.created_at) * 1000)::bigint,
+                'occurredAt', line_metadata->'occurredAt', 'level', line_metadata->'level', 'stepId', line_metadata->'stepId',
                 'source', 'terminal', 'producerAttempt', observation.payload_json->'producerAttempt',
                 'lineOrdinal', line_ordinal, 'context', line_context
               )) END,
-              observation.created_at, observation.run_id, play, ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, NULL,
+              line_at, observation.run_id, play, ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, line_metadata->>'level',
               CASE WHEN compact_logs THEN NULL ELSE left(regexp_replace(line #>> '{}',
                 '^' || chr(1) || 'prov:[^' || chr(1) || ']*' || chr(1), ''), 2048) END,
               line #>> '{}',
@@ -2019,7 +2101,7 @@ export async function attachRuntimeEventSearchTriggers(
     LANGUAGE sql IMMUTABLE SET search_path = pg_catalog AS $function$
       WITH envelope AS (
         SELECT CASE WHEN jsonb_typeof(value) = 'object' THEN
-          (value - 'logContexts' - 'logIds') || CASE WHEN value ? 'logs'
+          (value - 'logContexts' - 'logMetadata' - 'logIds') || CASE WHEN value ? 'logs'
             THEN jsonb_build_object('logs', '[]'::jsonb) ELSE '{}'::jsonb END
           ELSE value END AS result
       )
@@ -2037,7 +2119,7 @@ export async function attachRuntimeEventSearchTriggers(
         INTO compact_logs, canonical_events
         FROM ${schema}.runs WHERE run_id = observation.run_id;
       IF compact_logs OR canonical_events THEN
-        observation.payload_json := observation.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds';
+        observation.payload_json := observation.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds' - 'terminalLogMetadata';
         IF observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
           observation.payload_json := jsonb_set(observation.payload_json, '{events}',
             ${schema}.without_log_events(observation.payload_json->'events'))
@@ -2110,6 +2192,16 @@ export async function attachRuntimeEventSearchTriggers(
       NEW.payload_json := NEW.payload_json - '${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' - '_runtimeSearchRunClosed';
       PERFORM ${schema}.project_outbox_event(NEW);
       NEW.payload_json := ${schema}.compact_runtime_outbox_payload(NEW);
+      -- The search rows above are committed atomically with this outbox row.
+      -- When compaction leaves no ledger event to project, settle the no-op
+      -- receipt here so a delayed log-search fact cannot hold up run terminal
+      -- projection behind the scheduler outbox.
+      IF NEW.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
+        AND NEW.payload_json->>'logBodyStore' = 'planet_scale'
+        AND jsonb_typeof(NEW.payload_json->'events') = 'array'
+        AND jsonb_array_length(NEW.payload_json->'events') = 0 THEN
+        NEW.delivered_at := coalesce(NEW.delivered_at, clock_timestamp());
+      END IF;
       RETURN NEW;
     END
     $function$
@@ -2295,7 +2387,7 @@ export async function attachRuntimeEventSearchTriggers(
     ${schema}.accept_runner_log_payload(), ${schema}.search_outbox_event(),
     ${schema}.compact_run_log_result(), ${schema}.without_log_events(jsonb),
     ${schema}.convex_projection_event(jsonb), ${schema}.convex_projection_events(jsonb),
-    ${schema}.compact_runner_log_envelope(jsonb), ${schema}.without_runtime_log_tail(jsonb)
+    ${schema}.run_log_metadata(text, jsonb, timestamptz), ${schema}.compact_runner_log_envelope(jsonb), ${schema}.without_runtime_log_tail(jsonb)
     FROM PUBLIC`);
   // Publish readiness last, after all acceptance/compaction functions exist.
   await client.query(

package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts

@@ -45,6 +45,8 @@ export type PlayVisualNodeProgress = {
   completedAt?: number | null;
   artifactTableNamespace?: string | null;
   nodeIo?: PlayDocflowNodeIoState;
+  /** Error reported by this step, distinct from a run stopping other work. */
+  error?: string;
   conditionCounts?: PlayConditionCounts;
 };
 
@@ -193,6 +195,9 @@ export function normalizePlayVisualNodeProgressMap(
       ...(startedAt !== undefined ? { startedAt } : {}),
       ...(completedAt !== undefined ? { completedAt } : {}),
       ...(message !== undefined ? { message } : {}),
+      ...(typeof rawProgress.error === 'string' && rawProgress.error.trim()
+        ? { error: rawProgress.error.trim() }
+        : {}),
       ...(artifactTableNamespace !== undefined
         ? { artifactTableNamespace }
         : {}),
@@ -305,7 +310,7 @@ export function applyVisualNodeProgress(input: {
     const completed = Number(progress.completed ?? 0);
     const total = Number(progress.total ?? 0);
     const progressStatus =
-      nodeState.status === 'failed'
+      nodeState.status === 'failed' || progress.error || progress.nodeIo?.error
         ? 'failed'
         : Number.isFinite(total) && total > 0
           ? Number.isFinite(completed) && completed >= total

package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts

@@ -349,10 +349,14 @@ export function createRuntimeStepToolResultReceipts(
         // exists to cap how many stored payloads expand at once; holding one
         // of its slots across this gateway round trip only queued unrelated
         // rows behind this row's write, so the mutation runs outside it.
+        const storedResult = stampResultReceipt(
+          candidate.result,
+          record.resultReceiptKey,
+        );
         const persisted = await input.store.complete(
           record.resultReceiptKey,
           ownerRunId,
-          serializeToolExecuteResult(candidate.result),
+          serializeToolExecuteResult(storedResult),
           COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
         );
         // Publication is immutable, but an independent miss keeps the
@@ -369,10 +373,10 @@ export function createRuntimeStepToolResultReceipts(
           }
         }
         await input.onStored?.({
-          result: candidate.result,
+          result: storedResult,
           resultReceiptKey: record.resultReceiptKey,
         });
-        return { kind: 'stored', result: candidate.result };
+        return { kind: 'stored', result: storedResult };
       } finally {
         await stopHeartbeat(record);
         await releaseExecutionLock(record);
@@ -394,6 +398,16 @@ export function createRuntimeStepToolResultReceipts(
   };
 }
 
+function stampResultReceipt(
+  result: ToolExecuteResult,
+  resultReceiptKey: string,
+): ToolExecuteResult {
+  return deserializeToolExecuteResult({
+    ...serializeToolExecuteResult(result),
+    execution: { ...result._metadata.execution, resultReceiptKey },
+  });
+}
+
 function completedResult(
   receipt: RuntimeStepReceipt | null,
   resultReceiptKey: string,
@@ -402,10 +416,13 @@ function completedResult(
     return null;
   }
   if (isSerializedToolExecuteResult(receipt.output)) {
-    return deserializeToolExecuteResult(receipt.output);
+    return stampResultReceipt(
+      deserializeToolExecuteResult(receipt.output),
+      resultReceiptKey,
+    );
   }
   if (isToolExecuteResult(receipt.output)) {
-    return receipt.output;
+    return stampResultReceipt(receipt.output, resultReceiptKey);
   }
   throw new Error(
     `Completed Tool Result Receipt ${resultReceiptKey} has no valid serialized ToolExecuteResult.`,

package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts

@@ -12,6 +12,41 @@ import {
 } from '../plays/tool-execution-error';
 
 const TOOL_HTTP_ERROR_BRAND = Symbol.for('deepline.tool-http-error.v1');
+const callEvidenceByError = new WeakMap<object, unknown>();
+
+/** Payload evidence stays out of the scalar portable error/status contract. */
+export function attachToolCallFailureEvidence(
+  error: object,
+  evidence: unknown,
+): void {
+  callEvidenceByError.set(error, evidence);
+}
+
+/** Read the already consumed error body without changing failure classification. */
+export function attachToolCallFailureEvidenceFromResponse(
+  error: object,
+  body: string,
+): void {
+  let response: unknown;
+  try {
+    response = JSON.parse(body);
+  } catch {
+    return;
+  }
+  if (!response || typeof response !== 'object') return;
+  const meta = (response as Record<string, unknown>).meta;
+  if (!meta || typeof meta !== 'object') return;
+  attachToolCallFailureEvidence(
+    error,
+    (meta as Record<string, unknown>).callEvidence,
+  );
+}
+
+export function readToolCallFailureEvidence(error: unknown): unknown {
+  return error && typeof error === 'object'
+    ? callEvidenceByError.get(error)
+    : undefined;
+}
 
 type ToolHttpErrorConstructorInput = {
   message: string;

package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts

@@ -1,3 +1,7 @@
+import {
+  jsonByteLengthUpTo,
+  RUNTIME_RECEIPT_OUTPUT_MAX_BYTES,
+} from './output-size-limits';
 export type {
   SerializedToolExecuteResult,
   ToolExecuteResult,
@@ -83,6 +87,11 @@ const relayPublishedResultReferences = new WeakMap<
   RuntimeReceiptValueReference
 >();
 
+// Call evidence belongs to the saved per-call payload. Keep it out of the
+// enumerable Play result so returning a tool result or storing it in a row
+// does not enlarge customer output with diagnostic request bodies.
+const savedToolCallEvidence = new WeakMap<object, unknown>();
+
 export function attachRelayPublishedToolResultReference(
   result: ToolExecuteResult,
   reference: RuntimeReceiptValueReference,
@@ -1271,6 +1280,7 @@ export function createToolExecuteResult<TResult = unknown>(input: {
     meta?: Record<string, unknown>;
   };
 }): ToolExecuteResult<TResult> {
+  const { callEvidence, ...publicMeta } = input.meta ?? {};
   const result = toResultEnvelope(input.result);
   const resultRoot = {
     toolResponse: {
@@ -1320,7 +1330,7 @@ export function createToolExecuteResult<TResult = unknown>(input: {
   const wrapper = {
     status: input.status,
     ...(input.jobId ? { job_id: input.jobId } : {}),
-    ...(input.meta ? { meta: input.meta } : {}),
+    ...(input.meta ? { meta: publicMeta } : {}),
     ...(input.billing ? { billing: input.billing } : {}),
     toolResponse,
     extractedValues,
@@ -1336,6 +1346,9 @@ export function createToolExecuteResult<TResult = unknown>(input: {
     value: metadata,
     enumerable: false,
   });
+  if (callEvidence !== undefined) {
+    savedToolCallEvidence.set(wrapper, callEvidence);
+  }
   return wrapper;
 }
 
@@ -1976,9 +1989,50 @@ function rawV2WithSerializedListRows(input: {
   });
 }
 
+/** Optional call evidence cannot turn a valid provider result into a retry. */
+function fitSavedCallEvidence(
+  value: SerializedToolExecuteResult,
+): SerializedToolExecuteResult {
+  if (
+    !value.meta?.callEvidence ||
+    !jsonByteLengthUpTo(value, RUNTIME_RECEIPT_OUTPUT_MAX_BYTES).exceeded
+  )
+    return value;
+  const { callEvidence: _evidence, ...meta } = value.meta;
+  const withoutEvidence = { ...value, meta };
+  // Preserve the original oversized-output failure if the result itself is
+  // invalid. Only newly added diagnostic evidence may be omitted.
+  if (
+    jsonByteLengthUpTo(withoutEvidence, RUNTIME_RECEIPT_OUTPUT_MAX_BYTES)
+      .exceeded
+  )
+    return value;
+  const reason =
+    'Call evidence exceeds the saved-result size limit; the complete tool output is retained.';
+  const unavailable = {
+    ...value,
+    meta: { ...meta, callEvidence: { unavailable: reason } },
+  };
+  if (
+    !jsonByteLengthUpTo(unavailable, RUNTIME_RECEIPT_OUTPUT_MAX_BYTES).exceeded
+  )
+    return unavailable;
+  // An output exactly at the historical limit has no room for the marker.
+  // Its original bytes remain legal; the runner retains the warning instead.
+  console.warn(reason);
+  return withoutEvidence;
+}
+
 export function serializeToolExecuteResult(
   value: ToolExecuteResult,
+  options?: { includeCallEvidence?: boolean },
 ): SerializedToolExecuteResult {
+  const callEvidence =
+    options?.includeCallEvidence === false
+      ? undefined
+      : savedToolCallEvidence.get(value);
+  const meta =
+    callEvidence !== undefined ? { ...value.meta, callEvidence } : value.meta;
   const metadata = metadataInputFromToolExecuteResult(value);
   const listDatasets = serializedListDatasetsFromResult(value);
   const listRows = serializedListRowsFromResult({
@@ -2007,11 +2061,11 @@ export function serializeToolExecuteResult(
       view,
       meta: value.toolResponse.meta,
     });
-    return {
+    return fitSavedCallEvidence({
       __kind: SERIALIZED_TOOL_EXECUTE_RESULT_V2_KIND,
       status: value.status,
       ...(typeof value.job_id === 'string' ? { job_id: value.job_id } : {}),
-      ...(isRecord(value.meta) ? { meta: value.meta } : {}),
+      ...(isRecord(meta) ? { meta } : {}),
       ...(isRecord(value.billing) ? { billing: value.billing } : {}),
       toolResponse: {
         // rawV2 is the canonical complete provider response. Only the legacy
@@ -2035,13 +2089,13 @@ export function serializeToolExecuteResult(
       ...(targetValues ? { targetValues } : {}),
       metadata,
       execution: value._metadata.execution,
-    };
+    });
   }
-  return {
+  return fitSavedCallEvidence({
     __kind: SERIALIZED_TOOL_EXECUTE_RESULT_V1_KIND,
     status: value.status,
     ...(typeof value.job_id === 'string' ? { job_id: value.job_id } : {}),
-    ...(isRecord(value.meta) ? { meta: value.meta } : {}),
+    ...(isRecord(meta) ? { meta } : {}),
     ...(isRecord(value.billing) ? { billing: value.billing } : {}),
     toolResponse: {
       raw: serializedRaw,
@@ -2052,7 +2106,7 @@ export function serializeToolExecuteResult(
     ...(targetValues ? { targetValues } : {}),
     metadata,
     execution: value._metadata.execution,
-  };
+  });
 }
 
 export function isSerializedToolExecuteResult(
@@ -2156,7 +2210,13 @@ export function cloneToolExecuteResultWithExecution<TResult>(
 ): ToolExecuteResult<TResult> {
   return deserializeToolExecuteResult({
     ...serializeToolExecuteResult(value),
-    execution,
+    execution: {
+      ...execution,
+      ...(!execution.resultReceiptKey &&
+      value._metadata.execution.resultReceiptKey
+        ? { resultReceiptKey: value._metadata.execution.resultReceiptKey }
+        : {}),
+    },
   }) as ToolExecuteResult<TResult>;
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts

@@ -34,6 +34,8 @@ export type PlayVisualNodeProgress = {
   completedAt?: number;
   artifactTableNamespace?: string | null;
   nodeIo?: PlayDocflowNodeIoState;
+  /** Error reported by this step, distinct from a run stopping other work. */
+  error?: string;
   conditionCounts?: PlayConditionCounts;
 };
 

package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts

@@ -37,6 +37,8 @@ export type ToolResultExecutionMetadata = {
   cacheKey?: string;
   receiptRole?: 'owner' | 'follower';
   receiptKey?: string;
+  /** Canonical saved result identity; receiptKey may name a transport alias. */
+  resultReceiptKey?: string;
   attachedToReceiptKey?: string;
 };