deepline 0.3.362 → 0.3.364

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.362',
+  version: '0.3.364',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -1,3 +1,7 @@
+import {
+  readRowFailureCounts,
+  settledRowFailureKind,
+} from './row-failure-summary';
 import { RUN_EVENT_KIND } from './run-event-catalog';
 import {
   MapExecutionFrameStore,
@@ -8740,6 +8744,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           completedRowsCount: settledBeforePool.completedRows,
           pendingRowsCount: Math.max(0, totalInputCount - processedCount),
           failedRowsCount: settledBeforePool.failedRows,
+          failureCounts: readRowFailureCounts(
+            this.checkpoint.mapFrames?.[mapScope.mapInvocationId]
+              ?.failureCounts,
+            settledBeforePool.failedRows,
+          ),
           supersededRows: visibility.supersededRows,
           startedAt:
             this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ??
@@ -9006,6 +9015,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           completedRowsCount: completedRows,
           pendingRowsCount: 0,
           failedRowsCount: failedCount,
+          failureCounts: readRowFailureCounts(
+            this.checkpoint.mapFrames?.[mapScope.mapInvocationId]
+              ?.failureCounts,
+            failedCount,
+          ),
           supersededRows: visibility.supersededRows,
           startedAt:
             this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ??
@@ -9044,6 +9058,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         completedRowsCount: terminalProgress.completedRows,
         pendingRowsCount: 0,
         failedRowsCount: failedCount,
+        failureCounts: readRowFailureCounts(
+          this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.failureCounts,
+          failedCount,
+        ),
         supersededRows: visibility.supersededRows,
         startedAt:
           this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ??
@@ -10535,6 +10553,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   ? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
                   : {}),
                 error: formattedError,
+                failureKind: settledRowFailureKind(error),
               });
               // This row has not entered incremental persistence because the
               // fail-fast error is about to unwind the resolver. Retain this
@@ -10558,6 +10577,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               data: failedData,
               ...(cellMetaPatch ? { cellMetaPatch } : {}),
               error: formattedError,
+              failureKind: settledRowFailureKind(error),
             });
             failedRowCount += 1;
             if (
@@ -11295,6 +11315,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     // optional source-run pointer may share a later writer batch. A map must
     // not announce a terminal state with that bounded egress still retained.
     await this.runtimeStepReceiptCompletionWriter.flush();
+    mapProgress.flushCommitted();
     if (emitTerminalEvent) {
       mapProgress.update({
         status: 'completed',

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -507,6 +507,8 @@ export interface MapExecutionFrame {
   completedRowsCount?: number;
   pendingRowsCount?: number;
   failedRowsCount?: number;
+  /** Bounded final-row causes, additive for checkpoint compatibility. */
+  failureCounts?: import('./row-failure-summary').RowFailureCounts;
   /** Terminal rows omitted from this run's latest view because a newer write won. */
   supersededRows?: number;
   activeBoundaryId?: string | null;

package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts

@@ -8,6 +8,7 @@ import type {
   SkipRuntimeStepReceiptInput,
 } from './ctx-types';
 import type { PlayRunLedgerEvent } from './run-ledger';
+import type { RowFailureKind } from './row-failure-summary';
 
 export type MapRowOutcomeStatus = 'completed' | 'failed';
 
@@ -28,6 +29,8 @@ export type MapRowOutcome = {
   status?: MapRowOutcomeStatus;
   /** Row-level error message persisted when `status` is failed. */
   error?: string | null;
+  /** Final resolver error class; successful retries never produce this fact. */
+  failureKind?: RowFailureKind;
 };
 
 /**

package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts

@@ -7,10 +7,18 @@ import type {
 import type { MapRowOutcome, MapRowOutcomeStatus } from './durability-store';
 import { cloneCsvAliasedRow } from './csv-rename';
 import { RuntimeSheetAttemptSupersededError } from './runtime-sheet-errors';
+import {
+  emptyRowFailureCounts,
+  readRowFailureCounts,
+  type RowFailureCounts,
+} from './row-failure-summary';
 
 export function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
   return {
     ...frame,
+    ...(frame.failureCounts
+      ? { failureCounts: { ...frame.failureCounts } }
+      : {}),
     completedRowKeys: [...frame.completedRowKeys],
     pendingRowKeys: [...frame.pendingRowKeys],
     ...(frame.suspendedInputRows
@@ -278,6 +286,7 @@ type ProgressChange = {
   failedRowKey?: string | null;
   failedRowKeys?: readonly string[];
   failedDelta?: number;
+  failureCountsDelta?: RowFailureCounts;
   activeBoundaryId?: string | null;
   emitEventType?: PlayExecutionEvent['type'];
 };
@@ -298,6 +307,8 @@ export class MapExecutionProgress {
   private readonly completed: Set<string>;
   private readonly pending: Set<string>;
   private failed = 0;
+  private readonly failureCounts: RowFailureCounts;
+  private readonly initialSuperseded: number;
   private rowsSinceFlush = 0;
   private lastFlushAt = 0;
   private started = 0;
@@ -330,6 +341,11 @@ export class MapExecutionProgress {
     // Freeze settled admission once: live superseded facts can include a
     // resolver already counted in started, whereas skipped inputs are new.
     const baseline = this.settlement();
+    this.failureCounts = readRowFailureCounts(
+      frame?.failureCounts,
+      baseline.failedRows,
+    );
+    this.initialSuperseded = source.visibility.supersededRows;
     this.initiallyStarted =
       baseline.completedRows + baseline.failedRows + baseline.supersededRows;
     this.initiallySkipped = readCount(source.skippedInputs);
@@ -404,7 +420,14 @@ export class MapExecutionProgress {
       const key = raw?.trim();
       if (key) this.pending.delete(key);
     }
-    this.failed += Math.max(0, input.failedDelta ?? 0);
+    const failedDelta = Math.max(0, input.failedDelta ?? 0);
+    this.failed += failedDelta;
+    if (input.failureCountsDelta) {
+      for (const key of Object.keys(this.failureCounts) as Array<
+        keyof RowFailureCounts
+      >)
+        this.failureCounts[key] += input.failureCountsDelta[key];
+    } else this.failureCounts.unknown += failedDelta;
     this.rowsSinceFlush += this.completed.size - before;
     this.status = input.status ?? this.status;
     if (input.activeBoundaryId !== undefined)
@@ -430,18 +453,27 @@ export class MapExecutionProgress {
   recordCommittedRows(rows: readonly MapRowOutcome[]): void {
     const completed: string[] = [];
     const failed: string[] = [];
+    const failureCountsDelta = emptyRowFailureCounts();
     for (const row of rows) {
       if (typeof row.inputIndex === 'number')
         this.suspendedInputs?.delete(row.inputIndex);
       (row.status === 'failed' ? failed : completed).push(row.key);
+      if (row.status === 'failed' && this.source.visibility.isVisible(row.key))
+        failureCountsDelta[row.failureKind ?? 'unknown'] += 1;
     }
     this.update({
       completedRowKeys: completed,
       failedRowKeys: failed,
       failedDelta: failed.length,
+      failureCountsDelta,
     });
   }
 
+  /** Publish a settled pool after its writer barrier, even within the progress throttle. */
+  flushCommitted(): void {
+    this.flush(Date.now());
+  }
+
   private flush(now: number): void {
     const frame = this.frame();
     if (!frame) return;
@@ -465,6 +497,12 @@ export class MapExecutionProgress {
         : {}),
       completedRowsCount: progress.completedRows,
       failedRowsCount: progress.failedRows,
+      // A later superseding write can remove a previously counted cause. Its
+      // row identity remains in the sheet; do not guess which cause survived.
+      failureCounts:
+        this.source.visibility.supersededRows === this.initialSuperseded
+          ? readRowFailureCounts(this.failureCounts, progress.failedRows)
+          : readRowFailureCounts(undefined, progress.failedRows),
       supersededRows: progress.supersededRows,
       ...(this.suspendedInputs !== undefined
         ? { suspendedInputRows: [...this.suspendedInputs.values()] }

package/dist/bundling-sources/shared_libs/play-runtime/map-row-outcome.ts

@@ -1,4 +1,5 @@
 import type { MapRowOutcome, MapRowOutcomeStatus } from './durability-store';
+import type { RowFailureKind } from './row-failure-summary';
 
 export const MAP_ROW_OUTCOME_RUNTIME_FIELDS = {
   rowKey: '__deeplineRowKey',
@@ -75,11 +76,13 @@ export function failedMapRowOutcome(input: {
   data: Record<string, unknown>;
   cellMetaPatch?: Record<string, unknown> | null;
   error?: string | null;
+  failureKind?: RowFailureKind;
 }): MapRowOutcome {
   return {
     ...completedMapRowOutcome(input),
     status: 'failed',
     error: input.error ?? null,
+    ...(input.failureKind ? { failureKind: input.failureKind } : {}),
   };
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -480,6 +480,7 @@ export type PlayRunnerRowOutcomeSummary = {
   totalRows: number;
   /** Rows omitted because a newer Runtime Sheet write took precedence. */
   supersededRows?: number;
+  failureCounts?: import('./row-failure-summary').RowFailureCounts;
 };
 
 export const RETAINED_PLAY_PIPELINE_CONTRACT = deployed.object({

package/dist/bundling-sources/shared_libs/play-runtime/row-failure-summary.ts

@@ -0,0 +1,139 @@
+import {
+  isProviderTransientFailure,
+  normalizeToolExecutionCategory,
+  normalizeToolExecutionOrigin,
+} from '../plays/tool-execution-error';
+/** Fixed-size facts about final failed rows, never about retry attempts. */
+export const ROW_FAILURE_KINDS = [
+  'billingUnavailable',
+  'runtimeInternal',
+  'providerTransient',
+  'billingConstraint',
+  'inputOrAuth',
+  'resourceLimit',
+  'unknown',
+] as const;
+export type RowFailureKind = (typeof ROW_FAILURE_KINDS)[number];
+export type RowFailureCounts = Record<RowFailureKind, number>;
+
+export function emptyRowFailureCounts(): RowFailureCounts {
+  return Object.fromEntries(
+    ROW_FAILURE_KINDS.map((key) => [key, 0]),
+  ) as RowFailureCounts;
+}
+
+/** Preserve old checkpoints as unknown, rather than manufacturing zero errors. */
+export function readRowFailureCounts(
+  value: unknown,
+  failedRows: number,
+): RowFailureCounts {
+  if (!Number.isSafeInteger(failedRows) || failedRows < 0)
+    throw new Error('Invalid final failed-row count.');
+  if (value === undefined || value === null)
+    return { ...emptyRowFailureCounts(), unknown: failedRows };
+  if (typeof value !== 'object' || Array.isArray(value))
+    throw new Error('Invalid final row failure summary.');
+  const record = value as Record<string, unknown>;
+  const counts = emptyRowFailureCounts();
+  for (const key of ROW_FAILURE_KINDS) {
+    const count = record[key];
+    if (typeof count !== 'number' || !Number.isSafeInteger(count) || count < 0)
+      throw new Error(`Invalid final row failure count: ${key}.`);
+    counts[key] = count;
+  }
+  if (Object.values(counts).reduce((sum, n) => sum + n, 0) !== failedRows)
+    throw new Error(
+      'Final row failure causes do not reconcile with failed rows.',
+    );
+  return counts;
+}
+
+/** Only structured terminal error metadata establishes a cause. */
+export function settledRowFailureKind(error: unknown): RowFailureKind {
+  if (!error || typeof error !== 'object') return 'unknown';
+  const e = error as {
+    code?: unknown;
+    name?: unknown;
+    status?: unknown;
+    category?: unknown;
+    origin?: unknown;
+  };
+  const code = typeof e.code === 'string' ? e.code : '';
+  if (code === 'BILLING_UNAVAILABLE') return 'billingUnavailable';
+  if (
+    [
+      'INSUFFICIENT_CREDITS',
+      'RUN_BILLING_CAP_EXCEEDED',
+      'RUN_CREDIT_CAP_EXCEEDED',
+      'MONTHLY_BILLING_LIMIT_EXCEEDED',
+      'MONTHLY_CREDIT_CAP_EXCEEDED',
+      'billing_cap_exceeded',
+    ].includes(code)
+  )
+    return 'billingConstraint';
+  if (
+    [
+      'RUNTIME_LIMIT_EXCEEDED',
+      'OUTPUT_TOO_LARGE',
+      'RUNTIME_SANDBOX_MEMORY_ABORT',
+    ].includes(code)
+  )
+    return 'resourceLimit';
+  if (
+    [
+      'INTEGRATION_CREDENTIALS_MISSING',
+      'UPSTREAM_BAD_INPUT',
+      'UPSTREAM_VALIDATION_ERROR',
+      'VALIDATION_ERROR',
+      'UPSTREAM_NOT_FOUND',
+      'PROVIDER_AUTHORIZATION_FAILED',
+      'PROVIDER_AUTHENTICATION_FAILED',
+      'PROVIDER_ACCOUNT_CAPACITY',
+    ].includes(code)
+  )
+    return 'inputOrAuth';
+  if (
+    [
+      'INTEGRATION_EXECUTION_ERROR',
+      'INTERNAL_ERROR',
+      'runtime_durability_internal',
+      'DURABLE_INVOCATION_OUTCOME_UNKNOWN',
+      'RUNTIME_CONTROL_PLANE_UNAVAILABLE',
+      'RUNTIME_RUNNER_LOST',
+      'RUNTIME_SANDBOX_START_FAILED',
+      'RUNTIME_SCHEDULER_UNAVAILABLE',
+    ].includes(code)
+  )
+    return 'runtimeInternal';
+  if (
+    [
+      'RuntimeMapRowFeedStalledError',
+      'RuntimePersistenceCircuitOpenError',
+      'RuntimeReceiptLeaseLostError',
+      'RuntimeReceiptWaitTimeoutError',
+      'RuntimeReceiptWriterRetryDeadlineError',
+      'ToolCallReceiptCohortWaitTimeoutError',
+    ].includes(String(e.name))
+  )
+    return 'runtimeInternal';
+  if (
+    [
+      'UPSTREAM_NETWORK_ERROR',
+      'UPSTREAM_RATE_LIMIT',
+      'CTX_FETCH_TIMEOUT',
+    ].includes(code) ||
+    e.name === 'CtxFetchTimeoutError'
+  )
+    return 'providerTransient';
+  if (
+    isProviderTransientFailure({
+      origin: normalizeToolExecutionOrigin(e.origin),
+      category: normalizeToolExecutionCategory(e.category),
+      retryable: false,
+    })
+  )
+    return 'providerTransient';
+  if (e.origin === 'deepline' && e.category === 'internal')
+    return 'runtimeInternal';
+  return 'unknown';
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts

@@ -1,3 +1,8 @@
+import {
+  emptyRowFailureCounts,
+  readRowFailureCounts,
+  type RowFailureCounts,
+} from '../row-failure-summary';
 import { RUN_EVENT_KIND } from '../run-event-catalog';
 import { hydratePlayRunnerConfig } from '../protocol';
 import { appendFileSync, createReadStream, writeFileSync } from 'node:fs';
@@ -2032,6 +2037,7 @@ function rowOutcomesFromCheckpoint(checkpoint: {
       completedRowsCount?: number;
       completedRowKeys?: string[];
       failedRowsCount?: number;
+      failureCounts?: RowFailureCounts;
       totalRows?: number;
       supersededRows?: number;
     }
@@ -2041,6 +2047,7 @@ function rowOutcomesFromCheckpoint(checkpoint: {
   failedRows: number;
   totalRows: number;
   supersededRows?: number;
+  failureCounts: RowFailureCounts;
 } | null {
   const frames = Object.values(checkpoint.mapFrames ?? {});
   if (frames.length === 0) return null;
@@ -2048,6 +2055,7 @@ function rowOutcomesFromCheckpoint(checkpoint: {
   let failedRows = 0;
   let totalRows = 0;
   let supersededRows = 0;
+  const failureCounts = emptyRowFailureCounts();
   for (const frame of frames) {
     const completed = Number.isFinite(frame.completedRowsCount)
       ? Math.max(0, Math.floor(frame.completedRowsCount ?? 0))
@@ -2060,6 +2068,11 @@ function rowOutcomesFromCheckpoint(checkpoint: {
       : 0;
     completedRows += completed;
     failedRows += failed;
+    const frameCounts = readRowFailureCounts(frame.failureCounts, failed);
+    for (const key of Object.keys(failureCounts) as Array<
+      keyof RowFailureCounts
+    >)
+      failureCounts[key] += frameCounts[key];
     supersededRows += superseded;
     totalRows +=
       Number.isFinite(frame.totalRows) && (frame.totalRows ?? 0) >= 0
@@ -2071,6 +2084,7 @@ function rowOutcomesFromCheckpoint(checkpoint: {
     failedRows,
     totalRows,
     supersededRows,
+    failureCounts,
   };
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -1,3 +1,7 @@
+import {
+  readRowFailureCounts,
+  type RowFailureCounts,
+} from '../row-failure-summary';
 import {
   parseRunLedgerDispatchLaunch,
   runLedgerDispatchLaunchSql,
@@ -14548,7 +14552,13 @@ function finiteRunnerNumber(value: unknown): number | null {
 }
 
 function terminalRowOutcomesSummary(value: unknown): {
-  rowOutcomes: Record<string, number>;
+  rowOutcomes: {
+    completedRows?: number;
+    failedRows?: number;
+    totalRows?: number;
+    supersededRows?: number;
+    failureCounts?: RowFailureCounts;
+  };
 } | null {
   if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
   const record = value as Record<string, unknown>;
@@ -14556,11 +14566,25 @@ function terminalRowOutcomesSummary(value: unknown): {
   const failedRows = finiteRunnerNumber(record.failedRows);
   const totalRows = finiteRunnerNumber(record.totalRows);
   const supersededRows = finiteRunnerNumber(record.supersededRows);
-  const rowOutcomes: Record<string, number> = {};
+  const rowOutcomes: {
+    completedRows?: number;
+    failedRows?: number;
+    totalRows?: number;
+    supersededRows?: number;
+    failureCounts?: RowFailureCounts;
+  } = {};
   if (completedRows !== null) rowOutcomes.completedRows = completedRows;
   if (failedRows !== null) rowOutcomes.failedRows = failedRows;
   if (totalRows !== null) rowOutcomes.totalRows = totalRows;
   if (supersededRows !== null) rowOutcomes.supersededRows = supersededRows;
+  if (record.failureCounts !== undefined) {
+    if (failedRows === null)
+      throw new Error('Final row causes require a failed-row count.');
+    rowOutcomes.failureCounts = readRowFailureCounts(
+      record.failureCounts,
+      failedRows,
+    );
+  }
   return Object.keys(rowOutcomes).length > 0 ? { rowOutcomes } : null;
 }
 
@@ -18788,6 +18812,8 @@ export async function readPostgresSchedulerOutboxNextDueAt(
   client: PostgresSchedulerQueryClient,
   options?: Pick<PostgresSchedulerOptions, 'schema'> & {
     lane?: 'lifecycle' | 'observability';
+    /** Read one notified run's clock instead of enumerating every pending run. */
+    runId?: string;
   },
 ): Promise<Date | null> {
   const lanes =
@@ -18796,10 +18822,18 @@ export async function readPostgresSchedulerOutboxNextDueAt(
       : options?.lane === 'observability'
         ? [true]
         : [false, true];
+  const runId = options?.runId?.trim() || null;
   const result = await client.query<{ due_at: Date | string | null }>(
     `
       -- Seek distinct run identities without reading their pending fact tails.
-      WITH RECURSIVE ${runLedgerPendingRunsSql(options, options?.lane === undefined ? undefined : lanes[0])}, heads AS MATERIALIZED (
+      WITH RECURSIVE ${
+        runId
+          ? 'pending_runs AS (SELECT $1::text AS run_id)'
+          : runLedgerPendingRunsSql(
+              options,
+              options?.lane === undefined ? undefined : lanes[0],
+            )
+      }, heads AS MATERIALIZED (
         ${lanes
           .map(
             (
@@ -18836,6 +18870,7 @@ export async function readPostgresSchedulerOutboxNextDueAt(
       -- A failed head quarantines its lane; never wake on its successor.
       WHERE candidate.failed_at IS NULL
     `,
+    runId ? [runId] : undefined,
   );
   const value = result.rows[0]?.due_at;
   if (value instanceof Date) return value;