deepline 0.1.140 → 0.1.141

package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts

@@ -21,6 +21,10 @@ export type PlayVisualNodeProgressSnapshot = {
   completed?: number;
   total?: number;
   failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
   message?: string;
   updatedAt?: number | null;
   startedAt?: number | null;

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -28,6 +28,10 @@ export type PlayRunLedgerStepProgress = {
   completed?: number;
   total?: number;
   failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
   message?: string;
   artifactTableNamespace?: string | null;
   startedAt?: number | null;
@@ -430,6 +434,18 @@ function normalizeStepProgress(
     ...(optionalFiniteNumber(value.failed) !== undefined
       ? { failed: optionalFiniteNumber(value.failed) }
       : {}),
+    ...(optionalFiniteNumber(value.startedRows) !== undefined
+      ? { startedRows: optionalFiniteNumber(value.startedRows) }
+      : {}),
+    ...(optionalFiniteNumber(value.activeRows) !== undefined
+      ? { activeRows: optionalFiniteNumber(value.activeRows) }
+      : {}),
+    ...(optionalFiniteNumber(value.waitingRows) !== undefined
+      ? { waitingRows: optionalFiniteNumber(value.waitingRows) }
+      : {}),
+    ...(optionalFiniteNumber(value.completedRows) !== undefined
+      ? { completedRows: optionalFiniteNumber(value.completedRows) }
+      : {}),
     ...(optionalString(value.message)
       ? { message: optionalString(value.message) }
       : {}),
@@ -933,6 +949,10 @@ function progressSignature(
     completed: progress?.completed ?? null,
     total: progress?.total ?? null,
     failed: progress?.failed ?? null,
+    startedRows: progress?.startedRows ?? null,
+    activeRows: progress?.activeRows ?? null,
+    waitingRows: progress?.waitingRows ?? null,
+    completedRows: progress?.completedRows ?? null,
     message: progress?.message ?? null,
     artifactTableNamespace: progress?.artifactTableNamespace ?? null,
     startedAt: progress?.startedAt ?? null,
@@ -1152,6 +1172,18 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
         : {}),
       ...(progress.total !== undefined ? { total: progress.total } : {}),
       ...(progress.failed !== undefined ? { failed: progress.failed } : {}),
+      ...(progress.startedRows !== undefined
+        ? { startedRows: progress.startedRows }
+        : {}),
+      ...(progress.activeRows !== undefined
+        ? { activeRows: progress.activeRows }
+        : {}),
+      ...(progress.waitingRows !== undefined
+        ? { waitingRows: progress.waitingRows }
+        : {}),
+      ...(progress.completedRows !== undefined
+        ? { completedRows: progress.completedRows }
+        : {}),
       ...(progress.message !== undefined ? { message: progress.message } : {}),
       ...(progress.artifactTableNamespace !== undefined
         ? { artifactTableNamespace: progress.artifactTableNamespace }

package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts

@@ -31,6 +31,10 @@ export type PlayRunStreamNodeProgress = {
   completed?: number;
   total?: number;
   failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
   message?: string;
   updatedAt?: number | null;
   startedAt?: number | null;
@@ -192,6 +196,10 @@ function buildSnapshotFromLedger(
             completed: step.progress.completed,
             total: step.progress.total,
             failed: step.progress.failed,
+            startedRows: step.progress.startedRows,
+            activeRows: step.progress.activeRows,
+            waitingRows: step.progress.waitingRows,
+            completedRows: step.progress.completedRows,
             message: step.progress.message,
             artifactTableNamespace:
               step.progress.artifactTableNamespace ??
@@ -544,6 +552,10 @@ export function diffPlayRunStreamEvents(input: {
             completed: state.progress.completed,
             total: state.progress.total,
             failed: state.progress.failed,
+            startedRows: state.progress.startedRows,
+            activeRows: state.progress.activeRows,
+            waitingRows: state.progress.waitingRows,
+            completedRows: state.progress.completedRows,
             message: state.progress.message,
             artifactTableNamespace:
               state.progress.artifactTableNamespace ??

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -0,0 +1,55 @@
+export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 3;
+export const TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS = 8;
+export const TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS = 3;
+export const TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS = 1_000;
+
+export type ToolExecuteHttpRetryDecision = {
+  retryable: boolean;
+  attemptCap: number;
+  reason:
+    | 'rate_limit'
+    | 'retry_safe_transient_5xx'
+    | 'hard_billing_error'
+    | 'unsafe_transient_5xx'
+    | 'non_retryable_status';
+};
+
+export function decideToolExecuteHttpRetry(input: {
+  toolId: string;
+  status: number;
+  hardBillingFailure?: boolean;
+}): ToolExecuteHttpRetryDecision {
+  if (input.status === 429) {
+    if (input.hardBillingFailure) {
+      return {
+        retryable: false,
+        attemptCap: TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
+        reason: 'hard_billing_error',
+      };
+    }
+    return {
+      retryable: true,
+      attemptCap: TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
+      reason: 'rate_limit',
+    };
+  }
+  if (input.status >= 500 && input.status < 600) {
+    if (input.toolId === 'test_transient_500') {
+      return {
+        retryable: true,
+        attemptCap: TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
+        reason: 'retry_safe_transient_5xx',
+      };
+    }
+    return {
+      retryable: false,
+      attemptCap: 1,
+      reason: 'unsafe_transient_5xx',
+    };
+  }
+  return {
+    retryable: false,
+    attemptCap: 1,
+    reason: 'non_retryable_status',
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts

@@ -0,0 +1,248 @@
+export class ToolHttpError extends Error {
+  readonly billing: Record<string, unknown> | null;
+  /** HTTP status of the failed tool-execute response (e.g. 429, 502). */
+  readonly status: number;
+
+  constructor(
+    message: string,
+    billing: Record<string, unknown> | null,
+    status: number,
+  ) {
+    super(message);
+    this.name = 'ToolHttpError';
+    this.billing = billing;
+    this.status = status;
+  }
+}
+
+function formatCreditAmount(value: unknown): string {
+  if (typeof value !== 'number' || !Number.isFinite(value)) {
+    return String(value ?? '-');
+  }
+  return Number(value.toFixed(8)).toString();
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function getStringField(value: unknown, key: string): string | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return typeof field === 'string' && field.trim() ? field : null;
+}
+
+function getObjectField(
+  value: unknown,
+  key: string,
+): Record<string, unknown> | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return isRecord(field) ? field : null;
+}
+
+function isInsufficientCreditsBilling(
+  billing: Record<string, unknown> | null,
+): billing is Record<string, unknown> {
+  return billing?.kind === 'insufficient_credits';
+}
+
+function isHardBillingFailurePayload(
+  payload: Record<string, unknown> | null,
+): payload is Record<string, unknown> {
+  if (!payload) return false;
+  const category = String(
+    payload.error_category ?? payload.errorCategory ?? '',
+  ).toLowerCase();
+  const code = String(payload.code ?? payload.error_code ?? '').toUpperCase();
+  const message = String(
+    payload.error ?? payload.message ?? payload.failure_description ?? '',
+  ).toLowerCase();
+  if (category === 'billing') return true;
+  if (
+    code === 'INSUFFICIENT_CREDITS' ||
+    code === 'BILLING_CAP_EXCEEDED' ||
+    code === 'MONTHLY_BILLING_LIMIT_EXCEEDED'
+  ) {
+    return true;
+  }
+  return (
+    (message.includes('billing cap') ||
+      message.includes('monthly billing limit') ||
+      message.includes('rolling 30-day organization billing cap') ||
+      message.includes('insufficient credits')) &&
+    !message.includes('rate limit')
+  );
+}
+
+function normalizeHardBillingPayload(
+  payload: Record<string, unknown>,
+): Record<string, unknown> {
+  return {
+    kind: 'billing_cap_exceeded',
+    code: getStringField(payload, 'code') ?? 'MONTHLY_BILLING_LIMIT_EXCEEDED',
+    error_category: 'billing',
+    failure_origin:
+      getStringField(payload, 'failure_origin') ?? 'deepline_billing',
+    message:
+      getStringField(payload, 'error') ??
+      getStringField(payload, 'message') ??
+      'Deepline billing cap exceeded.',
+    ...payload,
+  };
+}
+
+function formatHardBillingFailureMessage(input: {
+  billing: Record<string, unknown>;
+  toolId: string;
+  status: number;
+  attempt: number;
+  maxAttempts: number;
+}): string {
+  const code = getStringField(input.billing, 'code');
+  const message =
+    getStringField(input.billing, 'message') ??
+    getStringField(input.billing, 'error') ??
+    'Deepline billing cap exceeded.';
+  return `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: Deepline billing cap exceeded. Run halted before marking remaining rows processed. ${code ? `code=${code}. ` : ''}${message}`;
+}
+
+function formatInsufficientCreditsMessage(input: {
+  billing: Record<string, unknown>;
+  toolId: string;
+}): string {
+  const operation =
+    getStringField(input.billing, 'operation_id') ??
+    getStringField(input.billing, 'operation') ??
+    input.toolId;
+  const balance = formatCreditAmount(input.billing.balance_credits);
+  const required = formatCreditAmount(input.billing.required_credits);
+  const recommended = formatCreditAmount(
+    input.billing.recommended_add_credits ?? input.billing.needed_credits,
+  );
+  const billingUrl = getStringField(input.billing, 'billing_url');
+  const addSuffix =
+    billingUrl && recommended !== '-'
+      ? ` Add >=${recommended} at ${billingUrl}.`
+      : billingUrl
+        ? ` Add credits at ${billingUrl}.`
+        : '';
+  return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`;
+}
+
+function formatPublicToolErrorPayload(input: {
+  parsed: Record<string, unknown> | null;
+  bodyText: string;
+}): string {
+  if (!input.parsed) {
+    return input.bodyText.slice(0, 500);
+  }
+
+  const selected: Record<string, unknown> = {};
+  for (const key of [
+    'error',
+    'message',
+    'code',
+    'failure_origin',
+    'error_category',
+    'failure_description',
+    'operator_hint',
+    'failure_hint',
+    'details',
+    'provider',
+    'operation',
+    'request_id',
+    'requestId',
+    'credential_source',
+    'credential_owner',
+  ]) {
+    const value = input.parsed[key];
+    if (typeof value === 'string' && value.trim()) {
+      selected[key] = value;
+    }
+  }
+
+  return JSON.stringify(
+    Object.keys(selected).length > 0 ? selected : input.parsed,
+  ).slice(0, 1_500);
+}
+
+export function normalizeToolHttpErrorMessage(input: {
+  toolId: string;
+  status: number;
+  attempt: number;
+  maxAttempts: number;
+  bodyText: string;
+}): ToolHttpError {
+  let parsed: Record<string, unknown> | null = null;
+  try {
+    const candidate = JSON.parse(input.bodyText);
+    parsed = isRecord(candidate) ? candidate : null;
+  } catch {
+    parsed = null;
+  }
+  const billing = getObjectField(parsed, 'billing');
+  if (isInsufficientCreditsBilling(billing)) {
+    return new ToolHttpError(
+      `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
+        {
+          billing,
+          toolId: input.toolId,
+        },
+      )}`,
+      billing,
+      input.status,
+    );
+  }
+  const hardBillingPayload = isHardBillingFailurePayload(billing)
+    ? normalizeHardBillingPayload(billing)
+    : isHardBillingFailurePayload(parsed)
+      ? normalizeHardBillingPayload(parsed)
+      : null;
+  if (hardBillingPayload) {
+    return new ToolHttpError(
+      formatHardBillingFailureMessage({
+        billing: hardBillingPayload,
+        toolId: input.toolId,
+        status: input.status,
+        attempt: input.attempt,
+        maxAttempts: input.maxAttempts,
+      }),
+      hardBillingPayload,
+      input.status,
+    );
+  }
+  return new ToolHttpError(
+    `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatPublicToolErrorPayload(
+      {
+        parsed,
+        bodyText: input.bodyText,
+      },
+    )}`,
+    billing,
+    input.status,
+  );
+}
+
+export function extractErrorBilling(
+  error: unknown,
+): Record<string, unknown> | null {
+  return error instanceof ToolHttpError ? error.billing : null;
+}
+
+export function isHardBillingToolHttpError(error: unknown): boolean {
+  return (
+    error instanceof ToolHttpError &&
+    (isInsufficientCreditsBilling(error.billing) ||
+      isHardBillingFailurePayload(error.billing))
+  );
+}
+
+/**
+ * A tool call that ultimately failed with HTTP 429 — provider or
+ * Deepline-internal rate-limit pushback that survived the in-process retry
+ * budget. This is run-level throughput pressure, never a row-specific defect.
+ */
+export function isRateLimitToolHttpError(error: unknown): boolean {
+  return error instanceof ToolHttpError && error.status === 429;
+}

package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts

@@ -12,6 +12,10 @@ export type PlayVisualNodeProgress = {
   completed?: number;
   total?: number;
   failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
   message?: string;
   updatedAt?: number;
   startedAt?: number;