deepline 0.1.140 → 0.1.142

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -53,6 +53,14 @@ import {
   type ToolExecuteResult,
   type ToolResultMetadataInput,
 } from './tool-result';
+import {
+  TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
+  decideToolExecuteHttpRetry,
+} from './tool-execute-retry-policy';
+import {
+  isHardBillingToolHttpError,
+  normalizeToolHttpErrorMessage,
+} from './tool-http-errors';
 import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
 import { createRuntimeDatasetId } from './dataset-id';
 import { dedupeExplicitMapKeyRows } from './map-row-identity';
@@ -179,7 +187,6 @@ const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 100;
 const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
 const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000;
 const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
-const TOOL_TRANSIENT_HTTP_MAX_ATTEMPTS = 3;
 const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
 const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
 type SafeFetchModule = typeof import('@shared_libs/security/safe-fetch');
@@ -5561,7 +5568,7 @@ export class PlayContextImpl {
               transportAttempt += 1;
               const message =
                 error instanceof Error ? error.message : String(error);
-              if (transportAttempt < TOOL_TRANSIENT_HTTP_MAX_ATTEMPTS) {
+              if (transportAttempt < TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS) {
                 this.governor.chargeBudget('retry');
                 const retryAfterMs =
                   TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
@@ -5570,13 +5577,13 @@ export class PlayContextImpl {
                   transportAttempt,
                 );
                 this.log(
-                  `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_TRANSIENT_HTTP_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
+                  `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
                 );
                 await this.sleepWithCheckpointHeartbeat(retryAfterMs);
                 continue;
               }
               throw new Error(
-                `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_TRANSIENT_HTTP_MAX_ATTEMPTS} attempts: ${message}`,
+                `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS} attempts: ${message}`,
               );
             }
 
@@ -5584,6 +5591,29 @@ export class PlayContextImpl {
 
             if (response.status === 429) {
               rateLimitAttempt += 1;
+              const text = await response.text();
+              const initialRetryDecision = decideToolExecuteHttpRetry({
+                toolId,
+                status: response.status,
+              });
+              const error = normalizeToolHttpErrorMessage({
+                toolId,
+                status: response.status,
+                attempt: rateLimitAttempt,
+                maxAttempts: initialRetryDecision.attemptCap,
+                bodyText: text,
+              });
+              const retryDecision = decideToolExecuteHttpRetry({
+                toolId,
+                status: response.status,
+                hardBillingFailure: isHardBillingToolHttpError(error),
+              });
+              if (
+                !retryDecision.retryable ||
+                rateLimitAttempt >= retryDecision.attemptCap
+              ) {
+                throw error;
+              }
               this.governor.chargeBudget('retry');
               const retryAfterMs = parseRetryAfterMs(
                 response.headers.get('retry-after'),
@@ -5605,10 +5635,25 @@ export class PlayContextImpl {
 
             if (!response.ok) {
               const text = await response.text();
+              const initialRetryDecision = decideToolExecuteHttpRetry({
+                toolId,
+                status: response.status,
+              });
+              const error = normalizeToolHttpErrorMessage({
+                toolId,
+                status: response.status,
+                attempt: rateLimitAttempt + 1,
+                maxAttempts: initialRetryDecision.attemptCap,
+                bodyText: text,
+              });
+              const retryDecision = decideToolExecuteHttpRetry({
+                toolId,
+                status: response.status,
+                hardBillingFailure: isHardBillingToolHttpError(error),
+              });
               if (
-                response.status >= 500 &&
-                response.status < 600 &&
-                rateLimitAttempt + 1 < TOOL_TRANSIENT_HTTP_MAX_ATTEMPTS
+                retryDecision.retryable &&
+                rateLimitAttempt + 1 < retryDecision.attemptCap
               ) {
                 rateLimitAttempt += 1;
                 this.governor.chargeBudget('retry');
@@ -5629,33 +5674,8 @@ export class PlayContextImpl {
                 await this.sleepWithCheckpointHeartbeat(retryAfterMs);
                 continue;
               }
-              const failureDetail = (() => {
-                try {
-                  const parsed = JSON.parse(text) as Record<string, unknown>;
-                  const detail =
-                    typeof parsed.message === 'string'
-                      ? parsed.message
-                      : typeof parsed.error === 'string'
-                        ? parsed.error
-                        : text;
-                  const code =
-                    typeof parsed.code === 'string'
-                      ? ` code=${parsed.code}`
-                      : '';
-                  const requestId =
-                    typeof parsed.request_id === 'string'
-                      ? ` requestId=${parsed.request_id}`
-                      : typeof parsed.requestId === 'string'
-                        ? ` requestId=${parsed.requestId}`
-                        : '';
-                  return `${detail}${code}${requestId}`;
-                } catch {
-                  return text;
-                }
-              })();
-              const message = `Tool ${toolId} failed (${response.status}): ${failureDetail}`;
-              this.log(message);
-              throw new Error(message);
+              this.log(error.message);
+              throw error;
             }
 
             const data = (await response.json()) as Record<string, unknown>;

package/dist/bundling-sources/shared_libs/play-runtime/live-events.ts

@@ -79,6 +79,10 @@ export type PlayStepProgressEventPayload = {
   completed?: number;
   total?: number;
   failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
   message?: string;
   artifactTableNamespace?: string | null;
 } & PlayRuntimeTimingWindow;

package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts

@@ -21,6 +21,10 @@ export type PlayVisualNodeProgressSnapshot = {
   completed?: number;
   total?: number;
   failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
   message?: string;
   updatedAt?: number | null;
   startedAt?: number | null;

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -28,6 +28,10 @@ export type PlayRunLedgerStepProgress = {
   completed?: number;
   total?: number;
   failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
   message?: string;
   artifactTableNamespace?: string | null;
   startedAt?: number | null;
@@ -430,6 +434,18 @@ function normalizeStepProgress(
     ...(optionalFiniteNumber(value.failed) !== undefined
       ? { failed: optionalFiniteNumber(value.failed) }
       : {}),
+    ...(optionalFiniteNumber(value.startedRows) !== undefined
+      ? { startedRows: optionalFiniteNumber(value.startedRows) }
+      : {}),
+    ...(optionalFiniteNumber(value.activeRows) !== undefined
+      ? { activeRows: optionalFiniteNumber(value.activeRows) }
+      : {}),
+    ...(optionalFiniteNumber(value.waitingRows) !== undefined
+      ? { waitingRows: optionalFiniteNumber(value.waitingRows) }
+      : {}),
+    ...(optionalFiniteNumber(value.completedRows) !== undefined
+      ? { completedRows: optionalFiniteNumber(value.completedRows) }
+      : {}),
     ...(optionalString(value.message)
       ? { message: optionalString(value.message) }
       : {}),
@@ -933,6 +949,10 @@ function progressSignature(
     completed: progress?.completed ?? null,
     total: progress?.total ?? null,
     failed: progress?.failed ?? null,
+    startedRows: progress?.startedRows ?? null,
+    activeRows: progress?.activeRows ?? null,
+    waitingRows: progress?.waitingRows ?? null,
+    completedRows: progress?.completedRows ?? null,
     message: progress?.message ?? null,
     artifactTableNamespace: progress?.artifactTableNamespace ?? null,
     startedAt: progress?.startedAt ?? null,
@@ -1152,6 +1172,18 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
         : {}),
       ...(progress.total !== undefined ? { total: progress.total } : {}),
       ...(progress.failed !== undefined ? { failed: progress.failed } : {}),
+      ...(progress.startedRows !== undefined
+        ? { startedRows: progress.startedRows }
+        : {}),
+      ...(progress.activeRows !== undefined
+        ? { activeRows: progress.activeRows }
+        : {}),
+      ...(progress.waitingRows !== undefined
+        ? { waitingRows: progress.waitingRows }
+        : {}),
+      ...(progress.completedRows !== undefined
+        ? { completedRows: progress.completedRows }
+        : {}),
       ...(progress.message !== undefined ? { message: progress.message } : {}),
       ...(progress.artifactTableNamespace !== undefined
         ? { artifactTableNamespace: progress.artifactTableNamespace }

package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts

@@ -31,6 +31,10 @@ export type PlayRunStreamNodeProgress = {
   completed?: number;
   total?: number;
   failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
   message?: string;
   updatedAt?: number | null;
   startedAt?: number | null;
@@ -192,6 +196,10 @@ function buildSnapshotFromLedger(
             completed: step.progress.completed,
             total: step.progress.total,
             failed: step.progress.failed,
+            startedRows: step.progress.startedRows,
+            activeRows: step.progress.activeRows,
+            waitingRows: step.progress.waitingRows,
+            completedRows: step.progress.completedRows,
             message: step.progress.message,
             artifactTableNamespace:
               step.progress.artifactTableNamespace ??
@@ -544,6 +552,10 @@ export function diffPlayRunStreamEvents(input: {
             completed: state.progress.completed,
             total: state.progress.total,
             failed: state.progress.failed,
+            startedRows: state.progress.startedRows,
+            activeRows: state.progress.activeRows,
+            waitingRows: state.progress.waitingRows,
+            completedRows: state.progress.completedRows,
             message: state.progress.message,
             artifactTableNamespace:
               state.progress.artifactTableNamespace ??

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -0,0 +1,55 @@
+export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 3;
+export const TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS = 8;
+export const TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS = 3;
+export const TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS = 1_000;
+
+export type ToolExecuteHttpRetryDecision = {
+  retryable: boolean;
+  attemptCap: number;
+  reason:
+    | 'rate_limit'
+    | 'retry_safe_transient_5xx'
+    | 'hard_billing_error'
+    | 'unsafe_transient_5xx'
+    | 'non_retryable_status';
+};
+
+export function decideToolExecuteHttpRetry(input: {
+  toolId: string;
+  status: number;
+  hardBillingFailure?: boolean;
+}): ToolExecuteHttpRetryDecision {
+  if (input.status === 429) {
+    if (input.hardBillingFailure) {
+      return {
+        retryable: false,
+        attemptCap: TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
+        reason: 'hard_billing_error',
+      };
+    }
+    return {
+      retryable: true,
+      attemptCap: TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
+      reason: 'rate_limit',
+    };
+  }
+  if (input.status >= 500 && input.status < 600) {
+    if (input.toolId === 'test_transient_500') {
+      return {
+        retryable: true,
+        attemptCap: TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
+        reason: 'retry_safe_transient_5xx',
+      };
+    }
+    return {
+      retryable: false,
+      attemptCap: 1,
+      reason: 'unsafe_transient_5xx',
+    };
+  }
+  return {
+    retryable: false,
+    attemptCap: 1,
+    reason: 'non_retryable_status',
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts

@@ -0,0 +1,248 @@
+export class ToolHttpError extends Error {
+  readonly billing: Record<string, unknown> | null;
+  /** HTTP status of the failed tool-execute response (e.g. 429, 502). */
+  readonly status: number;
+
+  constructor(
+    message: string,
+    billing: Record<string, unknown> | null,
+    status: number,
+  ) {
+    super(message);
+    this.name = 'ToolHttpError';
+    this.billing = billing;
+    this.status = status;
+  }
+}
+
+function formatCreditAmount(value: unknown): string {
+  if (typeof value !== 'number' || !Number.isFinite(value)) {
+    return String(value ?? '-');
+  }
+  return Number(value.toFixed(8)).toString();
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function getStringField(value: unknown, key: string): string | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return typeof field === 'string' && field.trim() ? field : null;
+}
+
+function getObjectField(
+  value: unknown,
+  key: string,
+): Record<string, unknown> | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return isRecord(field) ? field : null;
+}
+
+function isInsufficientCreditsBilling(
+  billing: Record<string, unknown> | null,
+): billing is Record<string, unknown> {
+  return billing?.kind === 'insufficient_credits';
+}
+
+function isHardBillingFailurePayload(
+  payload: Record<string, unknown> | null,
+): payload is Record<string, unknown> {
+  if (!payload) return false;
+  const category = String(
+    payload.error_category ?? payload.errorCategory ?? '',
+  ).toLowerCase();
+  const code = String(payload.code ?? payload.error_code ?? '').toUpperCase();
+  const message = String(
+    payload.error ?? payload.message ?? payload.failure_description ?? '',
+  ).toLowerCase();
+  if (category === 'billing') return true;
+  if (
+    code === 'INSUFFICIENT_CREDITS' ||
+    code === 'BILLING_CAP_EXCEEDED' ||
+    code === 'MONTHLY_BILLING_LIMIT_EXCEEDED'
+  ) {
+    return true;
+  }
+  return (
+    (message.includes('billing cap') ||
+      message.includes('monthly billing limit') ||
+      message.includes('rolling 30-day organization billing cap') ||
+      message.includes('insufficient credits')) &&
+    !message.includes('rate limit')
+  );
+}
+
+function normalizeHardBillingPayload(
+  payload: Record<string, unknown>,
+): Record<string, unknown> {
+  return {
+    kind: 'billing_cap_exceeded',
+    code: getStringField(payload, 'code') ?? 'MONTHLY_BILLING_LIMIT_EXCEEDED',
+    error_category: 'billing',
+    failure_origin:
+      getStringField(payload, 'failure_origin') ?? 'deepline_billing',
+    message:
+      getStringField(payload, 'error') ??
+      getStringField(payload, 'message') ??
+      'Deepline billing cap exceeded.',
+    ...payload,
+  };
+}
+
+function formatHardBillingFailureMessage(input: {
+  billing: Record<string, unknown>;
+  toolId: string;
+  status: number;
+  attempt: number;
+  maxAttempts: number;
+}): string {
+  const code = getStringField(input.billing, 'code');
+  const message =
+    getStringField(input.billing, 'message') ??
+    getStringField(input.billing, 'error') ??
+    'Deepline billing cap exceeded.';
+  return `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: Deepline billing cap exceeded. Run halted before marking remaining rows processed. ${code ? `code=${code}. ` : ''}${message}`;
+}
+
+function formatInsufficientCreditsMessage(input: {
+  billing: Record<string, unknown>;
+  toolId: string;
+}): string {
+  const operation =
+    getStringField(input.billing, 'operation_id') ??
+    getStringField(input.billing, 'operation') ??
+    input.toolId;
+  const balance = formatCreditAmount(input.billing.balance_credits);
+  const required = formatCreditAmount(input.billing.required_credits);
+  const recommended = formatCreditAmount(
+    input.billing.recommended_add_credits ?? input.billing.needed_credits,
+  );
+  const billingUrl = getStringField(input.billing, 'billing_url');
+  const addSuffix =
+    billingUrl && recommended !== '-'
+      ? ` Add >=${recommended} at ${billingUrl}.`
+      : billingUrl
+        ? ` Add credits at ${billingUrl}.`
+        : '';
+  return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`;
+}
+
+function formatPublicToolErrorPayload(input: {
+  parsed: Record<string, unknown> | null;
+  bodyText: string;
+}): string {
+  if (!input.parsed) {
+    return input.bodyText.slice(0, 500);
+  }
+
+  const selected: Record<string, unknown> = {};
+  for (const key of [
+    'error',
+    'message',
+    'code',
+    'failure_origin',
+    'error_category',
+    'failure_description',
+    'operator_hint',
+    'failure_hint',
+    'details',
+    'provider',
+    'operation',
+    'request_id',
+    'requestId',
+    'credential_source',
+    'credential_owner',
+  ]) {
+    const value = input.parsed[key];
+    if (typeof value === 'string' && value.trim()) {
+      selected[key] = value;
+    }
+  }
+
+  return JSON.stringify(
+    Object.keys(selected).length > 0 ? selected : input.parsed,
+  ).slice(0, 1_500);
+}
+
+export function normalizeToolHttpErrorMessage(input: {
+  toolId: string;
+  status: number;
+  attempt: number;
+  maxAttempts: number;
+  bodyText: string;
+}): ToolHttpError {
+  let parsed: Record<string, unknown> | null = null;
+  try {
+    const candidate = JSON.parse(input.bodyText);
+    parsed = isRecord(candidate) ? candidate : null;
+  } catch {
+    parsed = null;
+  }
+  const billing = getObjectField(parsed, 'billing');
+  if (isInsufficientCreditsBilling(billing)) {
+    return new ToolHttpError(
+      `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
+        {
+          billing,
+          toolId: input.toolId,
+        },
+      )}`,
+      billing,
+      input.status,
+    );
+  }
+  const hardBillingPayload = isHardBillingFailurePayload(billing)
+    ? normalizeHardBillingPayload(billing)
+    : isHardBillingFailurePayload(parsed)
+      ? normalizeHardBillingPayload(parsed)
+      : null;
+  if (hardBillingPayload) {
+    return new ToolHttpError(
+      formatHardBillingFailureMessage({
+        billing: hardBillingPayload,
+        toolId: input.toolId,
+        status: input.status,
+        attempt: input.attempt,
+        maxAttempts: input.maxAttempts,
+      }),
+      hardBillingPayload,
+      input.status,
+    );
+  }
+  return new ToolHttpError(
+    `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatPublicToolErrorPayload(
+      {
+        parsed,
+        bodyText: input.bodyText,
+      },
+    )}`,
+    billing,
+    input.status,
+  );
+}
+
+export function extractErrorBilling(
+  error: unknown,
+): Record<string, unknown> | null {
+  return error instanceof ToolHttpError ? error.billing : null;
+}
+
+export function isHardBillingToolHttpError(error: unknown): boolean {
+  return (
+    error instanceof ToolHttpError &&
+    (isInsufficientCreditsBilling(error.billing) ||
+      isHardBillingFailurePayload(error.billing))
+  );
+}
+
+/**
+ * A tool call that ultimately failed with HTTP 429 — provider or
+ * Deepline-internal rate-limit pushback that survived the in-process retry
+ * budget. This is run-level throughput pressure, never a row-specific defect.
+ */
+export function isRateLimitToolHttpError(error: unknown): boolean {
+  return error instanceof ToolHttpError && error.status === 429;
+}

package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts

@@ -12,6 +12,10 @@ export type PlayVisualNodeProgress = {
   completed?: number;
   total?: number;
   failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
   message?: string;
   updatedAt?: number;
   startedAt?: number;