deepline 0.3.461 → 0.3.462

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts

@@ -17,7 +17,7 @@ import { RUNTIME_RUNNER_TERMINAL_PROGRESS_MAX_EVENTS } from '../runtime-contract
  * Progress is best-effort observability: the authoritative terminal result still
  * flows through the runner's stdout `result` event and the gateway's receipt
  * terminal. A push failure must therefore never fail the play — it is logged and
- * dropped.
+ * retained for the next cadence or the terminal handoff.
  */
 
 export const RUNNER_PROGRESS_PUSH_BATCH_MS = 1_000;
@@ -70,6 +70,16 @@ export interface RunnerProgressPushOptions {
   stopTimeoutMs?: number;
 }
 
+class RunnerProgressHttpError extends Error {
+  readonly retryable: boolean;
+
+  constructor(status: number, body: string) {
+    super(`runner_progress_events failed (${status}): ${body}`);
+    this.retryable =
+      status === 408 || status === 429 || (status >= 500 && status <= 599);
+  }
+}
+
 async function defaultSendBatch(input: {
   gatewayBaseUrl: string;
   executorToken: string;
@@ -98,9 +108,7 @@ async function defaultSendBatch(input: {
   });
   if (!response.ok) {
     const body = await response.text().catch(() => '');
-    throw new Error(
-      `runner_progress_events failed (${response.status}): ${body}`,
-    );
+    throw new RunnerProgressHttpError(response.status, body);
   }
   await response.text().catch(() => '');
 }
@@ -142,7 +150,7 @@ export function createRunnerProgressPusher(
   let omittedEvents = 0;
   let stopped = false;
   let abandoned = false;
-  let deliveryBroken = false;
+  let deliveryRefused = false;
   let controller: AbortController | null = null;
   let inFlight: Promise<void> | null = null;
 
@@ -151,7 +159,7 @@ export function createRunnerProgressPusher(
   };
   const enforceBudget = () => {
     let dropped = 0;
-    // Never evict the active request: its ambiguous outcome must remain owned.
+    // Never evict an active or failed request: its ambiguous outcome must remain owned.
     // Evict only a prefix, preserving absolute log offsets for the existing
     // durable ingestion gap marker.
     while (
@@ -159,7 +167,7 @@ export function createRunnerProgressPusher(
       active.length + failed.length + pending.length >
         RUNNER_PROGRESS_MAX_RETAINED_EVENTS
     ) {
-      const entry = failed.shift() ?? pending.shift();
+      const entry = pending.shift();
       if (!entry) break;
       retainedBytes -= entry.bytes;
       dropped++;
@@ -174,9 +182,14 @@ export function createRunnerProgressPusher(
     }
   };
 
-  const flush = (): Promise<void> => {
+  const flush = (retryFailed = false): Promise<void> => {
     if (inFlight) return inFlight;
-    if (abandoned || deliveryBroken || !pending.length)
+    if (
+      abandoned ||
+      deliveryRefused ||
+      (failed.length > 0 && !retryFailed) ||
+      (!failed.length && !pending.length)
+    )
       return Promise.resolve();
     let bytes = 0;
     let count = 0;
@@ -193,7 +206,9 @@ export function createRunnerProgressPusher(
       bytes += next.bytes;
       count++;
     }
-    active = pending.splice(0, count);
+    // A lost acknowledgement must replay the same batch before later offsets.
+    // Only the existing cadence retries failures, never enqueue or this pump.
+    active = failed.length ? failed.splice(0) : pending.splice(0, count);
     const batch = active;
     const requestController = new AbortController();
     controller = requestController;
@@ -222,7 +237,8 @@ export function createRunnerProgressPusher(
       } catch (error) {
         if (!abandoned) {
           failed = batch;
-          deliveryBroken = true;
+          deliveryRefused =
+            error instanceof RunnerProgressHttpError && !error.retryable;
           warn('batch_failed', {
             runId: options.runId,
             count: batch.length,
@@ -235,13 +251,14 @@ export function createRunnerProgressPusher(
         controller = null;
         inFlight = null;
         enforceBudget();
-        if (!stopped && pending.length >= maxBatch) void flush();
+        if (!stopped && failed.length === 0 && pending.length >= maxBatch)
+          void flush();
       }
     });
     return inFlight;
   };
   const timer = setIntervalImpl(() => {
-    void flush();
+    void flush(true);
   }, options.batchMs ?? RUNNER_PROGRESS_PUSH_BATCH_MS);
   timer.unref?.();
 
@@ -320,7 +337,11 @@ export function createRunnerProgressPusher(
       clearIntervalImpl(timer);
       let timeout: ReturnType<typeof setTimeout> | undefined;
       const drain = async () => {
-        while (!abandoned && !deliveryBroken && (pending.length || inFlight))
+        while (
+          !abandoned &&
+          failed.length === 0 &&
+          (pending.length || inFlight)
+        )
           await flush();
       };
       const outcome = await Promise.race([

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-sheet-persistence-retry.ts

@@ -2,6 +2,7 @@ import { isAppRuntimeApiCapacityError } from '../app-runtime-api';
 import { isRuntimeApiCapacityError } from '../runtime-api';
 
 const RUNTIME_SHEET_CONTENTION_MAX_ATTEMPTS = 4;
+const RUNTIME_SHEET_CAPACITY_BACKOFF_MAX_MS = 8_000;
 
 function persistenceCapacityRetryAfterMs(error: unknown): number | null {
   if (isAppRuntimeApiCapacityError(error) || isRuntimeApiCapacityError(error)) {
@@ -71,6 +72,7 @@ export async function retryRuntimeSheetPersistence<T>(input: {
   const sleep = input.sleep ?? abortableSleep;
   let contentionAttempt = 0;
   let capacityAttempt = 0;
+  let capacityBackoffMs = 0;
 
   for (;;) {
     throwIfAborted(input.signal);
@@ -81,7 +83,16 @@ export async function retryRuntimeSheetPersistence<T>(input: {
       const retryAfterMs = persistenceCapacityRetryAfterMs(error);
       if (retryAfterMs !== null) {
         capacityAttempt += 1;
-        const delayMs = capacityRetryDelayMs(retryAfterMs, random);
+        // Repeated refusals must not resend a frozen large write every second.
+        // Cap client-added growth, while always honoring a longer server hint.
+        capacityBackoffMs = Math.max(
+          retryAfterMs,
+          Math.min(
+            RUNTIME_SHEET_CAPACITY_BACKOFF_MAX_MS,
+            capacityBackoffMs > 0 ? capacityBackoffMs * 2 : retryAfterMs,
+          ),
+        );
+        const delayMs = capacityRetryDelayMs(capacityBackoffMs, random);
         input.log(
           `[warn] ${input.description} waiting for persistence capacity attempt=${capacityAttempt} delay_ms=${delayMs}: ${
             error instanceof Error ? error.message : String(error)

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-control-transport.ts

@@ -124,6 +124,10 @@ export function createDaytonaControlClient(
       sandboxData.set(sandbox, data);
       return sandbox;
     },
+    /** The provider's own record, including fields the SDK class drops. */
+    providerData(sandbox: Sandbox): SandboxData | undefined {
+      return sandboxData.get(sandbox);
+    },
     /**
      * Full captured stdout is a legacy data-plane artifact, not fixed-size
      * metadata. Preserve its SDK file-download timeout independently of this

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-labels.ts

@@ -1,2 +1,38 @@
 /** Label that identifies sandboxes owned by the play runner fleet. */
 export const DAYTONA_PLAY_RUNNER_LABEL_SOURCE = 'deepline-play-runner';
+
+function normalizeDaytonaLabelValue(value: string | null | undefined): string | null {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed.slice(0, 63) : null;
+}
+
+/**
+ * Confirm a named Daytona sandbox belongs to the exact reserved Play operation
+ * before adopting it or recording cleanup against it.
+ */
+export function daytonaSandboxMatchesOperationIdentity(input: {
+  sandbox: {
+    name?: string | null;
+    organizationId?: string | null;
+    labels?: Record<string, string | undefined> | null;
+  };
+  sandboxName: string;
+  providerOperationToken: string;
+  orgId: string;
+  runId?: string | null;
+  configuredOrganizationId?: string | null;
+}): boolean {
+  const organizationId = input.sandbox.organizationId?.trim();
+  const configuredOrganizationId = input.configuredOrganizationId?.trim();
+  const runId = normalizeDaytonaLabelValue(input.runId);
+  return (
+    input.sandbox.name === input.sandboxName &&
+    input.sandbox.labels?.source === DAYTONA_PLAY_RUNNER_LABEL_SOURCE &&
+    input.sandbox.labels?.operationKey === input.providerOperationToken &&
+    input.sandbox.labels?.orgId === normalizeDaytonaLabelValue(input.orgId) &&
+    (!runId || input.sandbox.labels?.runId === runId) &&
+    Boolean(organizationId) &&
+    (!configuredOrganizationId ||
+      organizationId === configuredOrganizationId)
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts

@@ -23,6 +23,10 @@ import {
   DAYTONA_CONTROL_TIMEOUT_MS,
   DaytonaControlTimeoutError,
 } from './daytona-control-transport';
+import {
+  boundedProviderTextForLog as boundedDaytonaProviderTextForLog,
+  redactProviderErrorDetail as redactDaytonaErrorDetail,
+} from '../../provider-text-for-log';
 
 // Diagnostics below are log-only. They never select an error, a retry, or a
 // recovery decision, and they stay out of every persisted run document.
@@ -63,7 +67,6 @@ const DAYTONA_LOGGED_ERROR_NAMES: ReadonlySet<string> = new Set([
   'TypeError',
   'Error',
 ]);
-const DAYTONA_LOGGED_PROVIDER_TEXT_MAX_CHARS = 160;
 // Deliberately unanchored: `insufficient_credits` and `billingError` must match,
 // and withholding an innocent word costs only one message. `suspended` is a
 // sandbox state and `load balancer` is infrastructure, so they stay readable.
@@ -98,6 +101,50 @@ export function daytonaSandboxStateForLog(state: unknown): string | null {
     : 'unrecognized';
 }
 
+const DAYTONA_INFRASTRUCTURE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
+
+/**
+ * A Daytona id is only useful whole: free-text redaction turns every id into
+ * the same value, which cannot be grouped or quoted back to Daytona. Anything
+ * that is not plainly an id is dropped rather than logged.
+ */
+function daytonaInfrastructureIdForLog(value: unknown): string | null {
+  return typeof value === 'string' &&
+    DAYTONA_INFRASTRUCTURE_ID_PATTERN.test(value)
+    ? value
+    : null;
+}
+
+/**
+ * Where a sandbox runs and what Daytona believes it is doing. Grouping failures
+ * by host is the only way to tell one bad machine from many unrelated faults,
+ * and these facts are gone once the sandbox is deleted.
+ */
+export function daytonaSandboxPlacementForLog(
+  data: Partial<SandboxData> | undefined,
+): Record<string, string | boolean | null> {
+  // Newer API versions add these. The pinned client has no types for them, but
+  // they arrive in the same JSON and are exactly what a host fault would change.
+  const newerFields = (data ?? {}) as Record<string, unknown>;
+  return {
+    target: boundedDaytonaProviderTextForLog(data?.target),
+    runnerId: daytonaInfrastructureIdForLog(data?.runnerId),
+    desiredState: daytonaSandboxStateForLog(data?.desiredState),
+    backupState: boundedDaytonaProviderTextForLog(data?.backupState),
+    backupCreatedAt: boundedDaytonaProviderTextForLog(data?.backupCreatedAt),
+    errorReason: boundedDaytonaProviderTextForLog(data?.errorReason),
+    recoverable: data?.recoverable ?? null,
+    lastActivityAt: boundedDaytonaProviderTextForLog(data?.lastActivityAt),
+    updatedAt: boundedDaytonaProviderTextForLog(data?.updatedAt),
+    daemonVersion: boundedDaytonaProviderTextForLog(newerFields.daemonVersion),
+    sandboxClass: boundedDaytonaProviderTextForLog(newerFields.sandboxClass),
+    kvm: typeof newerFields.kvm === 'boolean' ? newerFields.kvm : null,
+    warmPoolId: daytonaInfrastructureIdForLog(newerFields.warmPoolId),
+    spot: typeof newerFields.spot === 'boolean' ? newerFields.spot : null,
+    spotEvictedAt: boundedDaytonaProviderTextForLog(newerFields.spotEvictedAt),
+  };
+}
+
 /** A known error class name, or `other`; never a provider-authored string. */
 export function daytonaErrorClassForLog(error: unknown): string {
   const name = error instanceof Error ? error.name : null;
@@ -130,69 +177,7 @@ export function daytonaErrorCodeForLog(error: unknown): string | null {
     : null;
 }
 
-/**
- * Credential redaction for a Daytona control-plane error message. Shared by
- * the backend's lookup-failure description and the provider text below.
- */
-export function redactDaytonaErrorDetail(value: string): string {
-  return value
-    .replace(
-      /authorization\s*[=:]\s*bearer\s+[^\s,;]+/gi,
-      'Authorization=Bearer [redacted]',
-    )
-    .replace(
-      /(authorization\s*[=:]\s*)(?!bearer\s+\[redacted\])[^\s,;]+/gi,
-      '$1[redacted]',
-    )
-    .replace(/(bearer\s+)[^\s,;]+/gi, '$1[redacted]')
-    .replace(/((?:api[_-]?key|token)\s*[=:]\s*)[^\s,;]+/gi, '$1[redacted]')
-    .replace(/\s+/g, ' ')
-    .trim()
-    .slice(0, 1_000);
-}
-
-/**
- * Provider free text (for example a sandbox `errorReason`) for logs. This
- * field is written by the provider, so it is redacted more strictly than the
- * shared function alone: URL userinfo, any authorization scheme, password,
- * secret, token, signature, credential and key pairs in query, header or JSON
- * form, email addresses, and opaque identifiers such as organization ids. It
- * is then cut to one short fixed bound so an upstream response is never
- * copied wholesale.
- */
-export function boundedDaytonaProviderTextForLog(
-  value: unknown,
-): string | null {
-  if (typeof value !== 'string') return null;
-  const strict = value
-    .replace(/(\b[a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, '$1[redacted]@')
-    .replace(
-      /(authorization\s*[=:]\s*["']?)(?:[a-z]+\s+)?[^\s,;"']+/gi,
-      '$1[redacted]',
-    )
-    .replace(/\b(basic|bearer)\s+[A-Za-z0-9+/=._~-]{8,}/gi, '$1 [redacted]')
-    .replace(
-      /(["']?[\w.-]*(?:password|passwd|secret|token|signature|credential|api[_-]?key|access[_-]?key)[\w.-]*["']?\s*[=:]\s*)(?:"[^"]*"|'[^']*'|[^\s&,;"'}\]]+)/gi,
-      '$1[redacted]',
-    )
-    .replace(
-      /[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+/g,
-      '[redacted-email]',
-    )
-    .replace(
-      /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi,
-      '[redacted-id]',
-    )
-    .replace(
-      /\b(?=[A-Za-z0-9_-]*\d)(?=[A-Za-z0-9_-]*[A-Za-z])[A-Za-z0-9_-]{24,}\b/g,
-      '[redacted-id]',
-    );
-  const bounded = redactDaytonaErrorDetail(strict).slice(
-    0,
-    DAYTONA_LOGGED_PROVIDER_TEXT_MAX_CHARS,
-  );
-  return bounded || null;
-}
+export { boundedDaytonaProviderTextForLog, redactDaytonaErrorDetail };
 
 /**
  * Provider text for a run's error, which customers read. Same bounds as the
@@ -325,7 +310,9 @@ function providerError(error: unknown): unknown {
       ? data
       : typeof data?.message === 'string'
         ? data.message
-        : error.message;
+        : status === undefined
+          ? error.message
+          : '';
   // Not axios's "Request failed with status code" wording: retry classification
   // matches that text, and this change only explains the failure.
   const message =
@@ -477,15 +464,73 @@ export function createDaytonaLaunchClient(
       },
     });
     const api = new SandboxApi(scopedConfiguration);
+    const transport = createTransport(signal);
     const sandbox = new Sandbox(
       data,
       new Configuration({
         basePath: configuration.basePath,
         baseOptions: { ...configuration.baseOptions },
       }),
-      createTransport(signal),
+      transport,
       api,
     );
+    const sdkUploadFile = sandbox.fs.uploadFile.bind(sandbox.fs);
+    sandbox.fs.uploadFile = async (source, destination, timeout = 30 * 60) => {
+      // The SDK's serverless upload path uses fetch but does not inspect
+      // Response.ok. Keep Buffer uploads on the scoped Axios transport so
+      // provider HTTP failures reach the same Daytona error classifier as
+      // create, read and delete calls. File-path uploads retain the SDK's
+      // streaming implementation.
+      if (typeof source === 'string') {
+        await sdkUploadFile(source, destination, timeout);
+        return;
+      }
+
+      const form = new FormData();
+      form.append('files[0].path', destination);
+      form.append(
+        'files[0].file',
+        new Blob([source.slice()], { type: 'application/octet-stream' }),
+        destination,
+      );
+      const toolboxBase = data.toolboxProxyUrl.endsWith('/')
+        ? data.toolboxProxyUrl
+        : `${data.toolboxProxyUrl}/`;
+      const response = await transport.post<unknown>(
+        `${toolboxBase}${encodeURIComponent(data.id)}/files/bulk-upload`,
+        form,
+        {
+          headers: configuration.baseOptions.headers,
+          maxRedirects: 0,
+          timeout: timeout * 1_000,
+          signal: signal(),
+        },
+      );
+      const responseData = response.data;
+      const errors =
+        responseData &&
+        typeof responseData === 'object' &&
+        'errors' in responseData &&
+        Array.isArray(responseData.errors)
+          ? responseData.errors.filter(
+              (value): value is string => typeof value === 'string',
+            )
+          : [];
+      if (errors.length > 0) {
+        const details = errors
+          .map(boundedDaytonaProviderTextForLog)
+          .filter((value): value is string => value !== null)
+          .join('; ');
+        throw new DaytonaError(
+          details
+            ? `Daytona upload failed: ${details}`
+            : 'Daytona reported a partial upload failure.',
+          response.status,
+          undefined,
+          'DAYTONA_UPLOAD_FAILED',
+        );
+      }
+    };
     dataBySandbox.set(sandbox, data);
     return sandbox;
   };
@@ -541,6 +586,7 @@ export function createDaytonaLaunchClient(
           onCreateStage?.('create:response', {
             createRequestMs: performance.now() - requestStartedAt,
             initialState: daytonaSandboxStateForLog(data.state),
+            ...daytonaSandboxPlacementForLog(data),
           });
         } catch {
           // An advisory observer cannot select provider recovery or skip readiness.
@@ -605,6 +651,9 @@ export function createDaytonaLaunchClient(
         phase.dispose();
       }
     },
+    placementForLog(sandbox: Sandbox) {
+      return daytonaSandboxPlacementForLog(dataBySandbox.get(sandbox));
+    },
     async delete(
       sandbox: Pick<Sandbox, 'id'>,
       timeoutSeconds = 30,
@@ -626,11 +675,19 @@ export function createDaytonaLaunchClient(
           phase.signal.throwIfAborted();
           if (!absenceOnlyConflict) {
             try {
-              await api.deleteSandbox(sandbox.id, undefined, {
-                timeout: timeoutSeconds * 1_000,
-              });
+              await api.deleteSandbox(sandbox.id);
               return;
             } catch (caught) {
+              // The phase signal is the authoritative local deadline. Axios
+              // reports its cancellation as ERR_CANCELED, which loses the
+              // distinction between an ambiguous local timeout and a provider
+              // rejection unless we preserve the signal's typed reason here.
+              if (
+                phase.signal.aborted &&
+                phase.signal.reason instanceof DaytonaControlTimeoutError
+              ) {
+                throw phase.signal.reason;
+              }
               const error = providerError(caught);
               // A prior accepted delete can already have removed this exact ID.
               if (retry && error instanceof DaytonaNotFoundError) return;
@@ -660,7 +717,10 @@ export function createDaytonaLaunchClient(
           // backoff before observing absence, so a closed resource need not
           // be re-reported just to learn that deletion succeeded.
           await delay(backoffMs, undefined, { signal: phase.signal });
-          backoffMs = Math.min(backoffMs * 2, DAYTONA_RETIREMENT_BACKOFF_MAX_MS);
+          backoffMs = Math.min(
+            backoffMs * 2,
+            DAYTONA_RETIREMENT_BACKOFF_MAX_MS,
+          );
           // Reconcile only the acquired ID. Never resolve a name, start it, or
           // infer absence from a failed/ambiguous provider read.
           try {
@@ -693,6 +753,12 @@ export function createDaytonaLaunchClient(
         // Absence never arrived, so the provider's conflict is the real cause.
         if (absenceOnlyConflict && phase.signal.aborted)
           throw absenceOnlyConflict;
+        if (
+          phase.signal.aborted &&
+          phase.signal.reason instanceof DaytonaControlTimeoutError
+        ) {
+          throw phase.signal.reason;
+        }
         throw providerError(error);
       } finally {
         phase.dispose();