deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -14,6 +14,8 @@ const VERCEL_RUNTIME_API_DEPLOYMENT_MISSING_RE =
   /runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i;
 const RUNTIME_LIMIT_EXCEEDED_RE = /\bRUNTIME_LIMIT_EXCEEDED\b/i;
 const SANDBOX_CAPACITY_EXCEEDED_RE = /\bSANDBOX_CAPACITY_EXCEEDED\b/i;
+const RUNTIME_CONTROL_PLANE_UNAVAILABLE_RE =
+  /\bRUNTIME_CONTROL_PLANE_UNAVAILABLE\b/i;
 const RUNTIME_RUNNER_LOST_RE = /\bRUNTIME_RUNNER_LOST\b/i;
 const RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_RE =
   /\bRUNTIME_SANDBOX_INSPECTION_UNAVAILABLE\b/i;
@@ -21,6 +23,7 @@ const RUNTIME_SANDBOX_LOST_RE = /\bRUNTIME_SANDBOX_LOST\b/i;
 const RUNTIME_SANDBOX_START_FAILED_RE = /\bRUNTIME_SANDBOX_START_FAILED\b/i;
 const RUNTIME_SANDBOX_OOM_RE =
   /\bRUNTIME_SANDBOX_OOM\b|(?:javascript heap out of memory|fatal error:.*(?:heap|allocation).*memory)/i;
+const RUNTIME_SANDBOX_MEMORY_ABORT_RE = /\bRUNTIME_SANDBOX_MEMORY_ABORT\b/i;
 const RUNTIME_SANDBOX_KILLED_RE = /\bRUNTIME_SANDBOX_KILLED\b/i;
 const MODAL_PAYLOAD_UPLOAD_TIMEOUT_RE = /\bMODAL_PAYLOAD_UPLOAD_TIMEOUT\b/i;
 const OUTPUT_TOO_LARGE_RE = /\b(?:OUTPUT_TOO_LARGE|OutputTooLarge)\b/;
@@ -38,12 +41,25 @@ export const RUNTIME_SANDBOX_LOST_MESSAGE =
   'The execution sandbox became unreachable before returning a terminal result. Re-run the same command; if this keeps happening, contact Deepline support with the run ID.';
 export const RUNTIME_SANDBOX_OOM_MESSAGE =
   'The execution sandbox ran out of memory. Completed work is durably recorded. Deepline does not automatically retry this run because the same resource profile is unlikely to succeed. Reduce the batch size or row payload before starting a new run.';
+/**
+ * Controlled counterpart to RUNTIME_SANDBOX_OOM. The external watchdog stops
+ * the runner while the sandbox is still under its cgroup limit, so the run
+ * reports a diagnosis (observed usage/limit plus the runtime's byte ledger)
+ * instead of vanishing into a kernel OOM kill. Same customer-safe shape and
+ * same non-retryable disposition as the OOM: the resource profile, not the
+ * attempt, is what has to change.
+ */
+export const RUNTIME_SANDBOX_MEMORY_ABORT_CODE = 'RUNTIME_SANDBOX_MEMORY_ABORT';
+export const RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE =
+  'The execution sandbox reached its memory limit, so Deepline stopped this run before the sandbox was killed. Completed work is durably recorded. Deepline does not automatically retry this run because the same resource profile is unlikely to succeed. Reduce the batch size or row payload before starting a new run.';
 export const RUNTIME_SANDBOX_KILLED_MESSAGE =
   'The execution sandbox was killed before it could return a result, and the kill reason is unavailable. Completed work is durably recorded. Retry the same command safely; Deepline reuses completed steps and does not repeat their provider calls.';
 export const RUNTIME_LIMIT_EXCEEDED_MESSAGE =
-  'The play reached its 30 minute runtime limit and was stopped. Completed row state was preserved; run a smaller batch or continue from the persisted rows.';
+  'The play reached its 30 minute runtime limit and was stopped before returning. Saved datasets remain available: inspect and export them before rerunning unfinished work. Use fewer rows or increase runtime.timeout in the Play; remove the override to use the 30-minute default. Compatible completed work can be reused; an in-flight response is not proof of a saved result.';
 export const SANDBOX_CAPACITY_EXCEEDED_MESSAGE =
   'Your organization has reached its concurrency limit, so this play did not start. Try again after another play finishes, or contact Deepline to request a higher limit.';
+export const RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE =
+  'Deepline could not reach the runtime control plane long enough to safely continue this run, so it stopped the execution sandbox. Completed work is preserved. Inspect and export saved datasets before deciding whether to rerun unfinished work; in-flight provider calls are not automatically replayed.';
 export const RUNTIME_RUNNER_LOST_MESSAGE =
   'The execution runner stopped reporting liveness before returning a terminal result. Completed work is preserved, but Deepline did not automatically replay the run because provider side effects may already exist. Re-run only when it is safe to do so.';
 export const RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_MESSAGE =
@@ -326,7 +342,11 @@ export function runtimeLimitExceededCause(timeoutSeconds: number): string {
 
 export function runtimeLimitExceededMessage(timeoutSeconds: number): string {
   if (timeoutSeconds === 30 * 60) return RUNTIME_LIMIT_EXCEEDED_MESSAGE;
-  return `The play reached its configured runtime limit of ${formatRuntimeLimitDuration(timeoutSeconds)} and was stopped. Completed row state was preserved; run a smaller batch or continue from the persisted rows.`;
+  const guidance =
+    timeoutSeconds >= 4 * 60 * 60
+      ? 'This Play already uses the maximum 4-hour runtime.timeout; split unfinished work into smaller batches.'
+      : 'Use fewer rows or increase runtime.timeout in the Play; remove the override to use the 30-minute default.';
+  return `The play reached its configured runtime limit of ${formatRuntimeLimitDuration(timeoutSeconds)} and was stopped before returning. Saved datasets remain available: inspect and export them before rerunning unfinished work. ${guidance} Compatible completed work can be reused; an in-flight response is not proof of a saved result.`;
 }
 
 export function runtimeLimitExceededFailure(
@@ -466,6 +486,20 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
       cause,
     };
   }
+  if (RUNTIME_CONTROL_PLANE_UNAVAILABLE_RE.test(rawCause)) {
+    const stack = error instanceof Error ? boundedFailureStack(error) : null;
+    const causes = error instanceof Error ? failureCauseTexts(error) : [];
+    return {
+      code: 'RUNTIME_CONTROL_PLANE_UNAVAILABLE',
+      phase: 'infrastructure',
+      message: RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE,
+      retryable: false,
+      cause,
+      ...(error instanceof Error ? { name: error.name || 'Error' } : {}),
+      ...(stack ? { stack } : {}),
+      ...(causes.length > 0 ? { causes } : {}),
+    };
+  }
   if (RUNTIME_RUNNER_LOST_RE.test(rawCause)) {
     const stack = error instanceof Error ? boundedFailureStack(error) : null;
     const causes = error instanceof Error ? failureCauseTexts(error) : [];
@@ -519,6 +553,19 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
       cause,
     };
   }
+  // Ordered before the OOM branch: a controlled abort is the diagnosed case
+  // and must never collapse into the generic out-of-memory classification.
+  if (RUNTIME_SANDBOX_MEMORY_ABORT_RE.test(rawCause)) {
+    return {
+      code: RUNTIME_SANDBOX_MEMORY_ABORT_CODE,
+      phase: 'infrastructure',
+      message: RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE,
+      // The observed usage, limit, and byte ledger ride on `cause`, which is
+      // the failure packet's verbatim evidence channel.
+      retryable: false,
+      cause,
+    };
+  }
   if (RUNTIME_SANDBOX_OOM_RE.test(rawCause)) {
     return {
       code: 'RUNTIME_SANDBOX_OOM',

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -679,7 +679,12 @@ export function normalizePlayRunLedgerSnapshot(
       finiteNumber(value.totalLogCount) ?? logTail.length,
       logTail.length,
     ),
-    logsTruncated: value.logsTruncated === true,
+    // A persisted snapshot carrying more lines than the tail keeps is
+    // truncated here, whatever the stored flag says. Reporting `false` while
+    // dropping lines is silent loss: a reader would take the tail for the
+    // whole log.
+    logsTruncated:
+      value.logsTruncated === true || logTail.length > LOG_TAIL_LIMIT,
     activeStepId: optionalNullableString(value.activeStepId),
     activeActivityId: optionalNullableString(value.activeActivityId),
     activeArtifactTableNamespace: optionalNullableString(

package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts

@@ -0,0 +1,55 @@
+/** Limits apply before logs enter runner memory or a transport request. */
+export const RUNNER_LOG_MAX_CHARS = 2_000;
+export const RUNNER_LOG_MAX_BYTES = 5 * 1024 * 1024;
+export const RUNNER_LOG_MAX_LINES = 25_000;
+const TERMINAL_RESERVE_BYTES = 64 * 1024;
+const TERMINAL_RESERVE_LINES = 32;
+const encoder = new TextEncoder();
+
+export function boundRunLogLine(line: string): string {
+  if (line.length <= RUNNER_LOG_MAX_CHARS) return line;
+  const suffix =
+    '… [log shortened; log identifiers/counts, or save a dataset and use runs export for full values]';
+  return line.slice(0, RUNNER_LOG_MAX_CHARS - suffix.length) + suffix;
+}
+
+/**
+ * Append-only so existing positional replay cursors remain valid. Ordinary
+ * logs share the durable stream's budget; a small reserve lets the runner
+ * explain a late timeout after a customer has exhausted that budget.
+ */
+export class RunLogBuffer {
+  readonly lines: string[] = [];
+  private bytes = 0;
+  private omitted = 0;
+  private marked = false;
+
+  append(raw: string, terminal = false): string[] {
+    const line = boundRunLogLine(raw);
+    const bytes = encoder.encode(line).byteLength;
+    const byteLimit =
+      RUNNER_LOG_MAX_BYTES - (terminal ? 0 : TERMINAL_RESERVE_BYTES);
+    const lineLimit =
+      RUNNER_LOG_MAX_LINES - (terminal ? 0 : TERMINAL_RESERVE_LINES);
+    if (this.bytes + bytes > byteLimit || this.lines.length >= lineLimit) {
+      this.omitted++;
+      if (this.marked) return [];
+      this.marked = true;
+      return this.append(
+        '[warn] [logs limited: runner log budget reached; further lines may be omitted. Remove per-row logging; save rows in a dataset and use deepline runs export. Run status and saved datasets remain available.]',
+        true,
+      );
+    }
+    this.lines.push(line);
+    this.bytes += bytes;
+    return [line];
+  }
+
+  snapshot() {
+    return {
+      retainedBytes: this.bytes,
+      retainedLines: this.lines.length,
+      omittedLines: this.omitted,
+    };
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts

@@ -0,0 +1,221 @@
+import * as diagnosticsChannel from 'node:diagnostics_channel';
+
+import {
+  EGRESS_BANDWIDTH_PROBE_BUDGET_MS,
+  formatEgressBandwidthLine,
+  probeEgressBandwidth,
+} from '../egress-bandwidth-probe';
+
+export type EgressProbeTarget = 'gateway' | 'relay';
+
+export type EgressProbeObservation = {
+  target: EgressProbeTarget;
+  timingsMs: number[];
+  attempted: number;
+  errorClass: string | null;
+  connects: number | null;
+  requestCreates: number | null;
+};
+
+function median(values: readonly number[]): number | null {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 1
+    ? sorted[middle]!
+    : Math.round(((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2);
+}
+
+export function egressProbeConnectionReuse(input: {
+  connects: number | null;
+  requestCreates: number | null;
+  attempted: number;
+}): 'true' | 'false' | 'unknown' {
+  const { connects, requestCreates, attempted } = input;
+  if (connects === null || requestCreates === null) return 'unknown';
+  if (attempted <= 1 || requestCreates < attempted) return 'unknown';
+  return connects < attempted ? 'true' : 'false';
+}
+
+export function formatEgressProbeLine(
+  observation: EgressProbeObservation,
+): string {
+  const [first, ...rest] = observation.timingsMs;
+  const restP50 = median(rest);
+  const restMax = rest.length > 0 ? Math.max(...rest) : null;
+  const number = (value: number | null | undefined) =>
+    value === null || value === undefined ? 'none' : String(Math.round(value));
+  const reused = egressProbeConnectionReuse({
+    connects: observation.connects,
+    requestCreates: observation.requestCreates,
+    attempted: observation.attempted,
+  });
+  return (
+    `[perf] egress_probe target=${observation.target} ` +
+    `first_ms=${number(first)} rest_p50_ms=${number(restP50)} ` +
+    `rest_max_ms=${number(restMax)} reused_connection=${reused} ` +
+    `samples=${observation.timingsMs.length}/${observation.attempted}` +
+    (observation.errorClass ? ` error=${observation.errorClass}` : '')
+  );
+}
+
+export function formatEgressProbeRegionLine(
+  env: Record<string, string | undefined>,
+): string {
+  const hint = (name: string) => env[name]?.trim() || 'none';
+  return (
+    `[perf] egress_probe_region daytona_target=${hint('DAYTONA_TARGET')} ` +
+    `sandbox_target=${hint('DEEPLINE_SANDBOX_TARGET')} ` +
+    `fly_region=${hint('FLY_REGION')}`
+  );
+}
+
+export async function probeEgressTarget(input: {
+  target: EgressProbeTarget;
+  baseUrl: string;
+  fetchImpl: typeof fetch;
+  requests: number;
+  deadlineAtMs: number;
+  now?: () => number;
+}): Promise<EgressProbeObservation> {
+  const now = input.now ?? (() => Date.now());
+  const url = `${input.baseUrl.replace(/\/$/, '')}/healthz`;
+  const timingsMs: number[] = [];
+  let attempted = 0;
+  let errorClass: string | null = null;
+  const counters = subscribeEgressProbeCounters();
+  try {
+    for (let index = 0; index < input.requests; index += 1) {
+      const remainingMs = input.deadlineAtMs - now();
+      if (remainingMs <= 0) break;
+      attempted += 1;
+      const startedAt = now();
+      try {
+        const response = await input.fetchImpl(url, {
+          method: 'GET',
+          redirect: 'manual',
+          signal: AbortSignal.timeout(remainingMs),
+        });
+        await response.text().catch(() => '');
+        timingsMs.push(now() - startedAt);
+      } catch (error) {
+        errorClass ??=
+          error instanceof Error ? error.constructor.name : typeof error;
+      }
+    }
+  } finally {
+    counters.stop();
+  }
+  return {
+    target: input.target,
+    timingsMs,
+    attempted,
+    errorClass,
+    connects: counters.connects,
+    requestCreates: counters.requestCreates,
+  };
+}
+
+function subscribeEgressProbeCounters(): {
+  readonly connects: number | null;
+  readonly requestCreates: number | null;
+  stop: () => void;
+} {
+  let connects = 0;
+  let requestCreates = 0;
+  let live = false;
+  let unsubscribe = () => {};
+  try {
+    const onConnect = () => {
+      connects += 1;
+    };
+    const onCreate = () => {
+      requestCreates += 1;
+    };
+    diagnosticsChannel.subscribe('undici:client:connected', onConnect);
+    diagnosticsChannel.subscribe('undici:request:create', onCreate);
+    live = true;
+    unsubscribe = () => {
+      diagnosticsChannel.unsubscribe('undici:client:connected', onConnect);
+      diagnosticsChannel.unsubscribe('undici:request:create', onCreate);
+    };
+  } catch {
+    live = false;
+  }
+  return {
+    get connects() {
+      return live ? connects : null;
+    },
+    get requestCreates() {
+      return live ? requestCreates : null;
+    },
+    stop: () => {
+      try {
+        unsubscribe();
+      } catch {
+        // Diagnostics teardown must not affect a customer run.
+      }
+    },
+  };
+}
+
+export const EGRESS_PROBE_BUDGET_MS = 3_000;
+const EGRESS_PROBE_REQUESTS_PER_TARGET = 6;
+
+export async function collectStartupEgressProbeLines(input: {
+  receiptGatewayBaseUrl: string | null;
+  executionRelayBaseUrl: string | null;
+  /** Bearer credential for the gateway's authenticated bandwidth sink. */
+  executorToken?: string | null;
+}): Promise<string[]> {
+  const gateway = input.receiptGatewayBaseUrl?.trim().replace(/\/$/, '') || '';
+  const relay = input.executionRelayBaseUrl?.trim().replace(/\/$/, '') || '';
+  const targets: Array<{ target: EgressProbeTarget; baseUrl: string }> = [];
+  if (gateway) targets.push({ target: 'gateway', baseUrl: gateway });
+  if (relay && relay !== gateway) {
+    targets.push({ target: 'relay', baseUrl: relay });
+  }
+  if (targets.length === 0) return [];
+  const lines: string[] = [formatEgressProbeRegionLine(process.env)];
+  const deadlineAtMs = Date.now() + EGRESS_PROBE_BUDGET_MS;
+  for (const target of targets) {
+    try {
+      const observation = await probeEgressTarget({
+        ...target,
+        fetchImpl: globalThis.fetch,
+        requests: EGRESS_PROBE_REQUESTS_PER_TARGET,
+        deadlineAtMs,
+      });
+      lines.push(formatEgressProbeLine(observation));
+    } catch (error) {
+      lines.push(
+        `[perf] egress_probe target=${target.target} failed error=${
+          error instanceof Error ? error.constructor.name : typeof error
+        }`,
+      );
+    }
+  }
+  const executorToken = input.executorToken?.trim() || null;
+  if (gateway && executorToken) {
+    try {
+      lines.push(
+        formatEgressBandwidthLine(
+          await probeEgressBandwidth({
+            target: 'gateway',
+            baseUrl: gateway,
+            fetchImpl: globalThis.fetch,
+            headers: { Authorization: `Bearer ${executorToken}` },
+            deadlineAtMs: Date.now() + EGRESS_BANDWIDTH_PROBE_BUDGET_MS,
+          }),
+        ),
+      );
+    } catch (error) {
+      lines.push(
+        `[perf] egress_bandwidth target=gateway failed error=${
+          error instanceof Error ? error.constructor.name : typeof error
+        }`,
+      );
+    }
+  }
+  return lines;
+}