deepline 0.3.90 → 0.3.92

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts

@@ -0,0 +1,321 @@
+import type { PlayRunnerEvent } from '../protocol';
+import {
+  RUNTIME_RUNNER_TERMINAL_PROGRESS_MAX_EVENTS,
+} from '../runtime-contract';
+
+/**
+ * Push-execution live progress channel (runs INSIDE the Daytona sandbox).
+ *
+ * The babysitting model streamed progress by having the WORKER poll a progress
+ * JSONL sidecar file out of the sandbox every second. Push execution deletes
+ * that file poll: the runner instead PUSHES the same progress events to the
+ * receipt gateway's `runner_progress_events` action on a small batched cadence.
+ * The gateway folds them into the durable run-log/progress path (the same
+ * postgres-progress reporter the worker used), so live progress survives without
+ * per-second sandbox filesystem reads.
+ *
+ * Progress is best-effort observability: the authoritative terminal result still
+ * flows through the runner's stdout `result` event and the gateway's receipt
+ * terminal. A push failure must therefore never fail the play — it is logged and
+ * dropped.
+ */
+
+export const RUNNER_PROGRESS_PUSH_BATCH_MS = 1_000;
+export const RUNNER_PROGRESS_PUSH_MAX_BATCH = 500;
+/**
+ * Progress is best-effort. Never let a stalled observability upload consume the
+ * runner's terminal-settlement grace or delay the authoritative result record.
+ */
+export const RUNNER_PROGRESS_PUSH_STOP_TIMEOUT_MS = 5_000;
+
+const APP_RUNTIME_COMPAT_PATH = '/api/v2/plays/runtime';
+
+export interface RunnerProgressPusher {
+  /** Enqueue one progress event for the next batch flush. */
+  enqueue: (event: PlayRunnerEvent) => void;
+  /** Stop delivery and return all observations not known to be delivered. */
+  sealForTerminal: () => {
+    events: PlayRunnerEvent[];
+    liveLogOffset: number;
+  };
+  /** Flush any pending events and stop the batch timer. */
+  stop: () => Promise<void>;
+}
+
+export interface RunnerProgressPushOptions {
+  gatewayBaseUrl: string;
+  executorToken: string;
+  runId: string;
+  playName: string;
+  /** Injectable for tests. Defaults to a bearer POST to the gateway. */
+  sendBatch?: (input: {
+    gatewayBaseUrl: string;
+    executorToken: string;
+    runId: string;
+    playName: string;
+    events: PlayRunnerEvent[];
+    liveLogOffset: number;
+    signal: AbortSignal;
+  }) => Promise<void>;
+  setIntervalImpl?: (fn: () => void, ms: number) => { unref?: () => void };
+  clearIntervalImpl?: (handle: unknown) => void;
+  warn?: (message: string, detail: Record<string, unknown>) => void;
+  batchMs?: number;
+  maxBatch?: number;
+  stopTimeoutMs?: number;
+}
+
+async function defaultSendBatch(input: {
+  gatewayBaseUrl: string;
+  executorToken: string;
+  runId: string;
+  playName: string;
+  events: PlayRunnerEvent[];
+  liveLogOffset: number;
+  signal: AbortSignal;
+}): Promise<void> {
+  const url = `${input.gatewayBaseUrl.replace(/\/$/, '')}${APP_RUNTIME_COMPAT_PATH}`;
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      authorization: `Bearer ${input.executorToken}`,
+    },
+    body: JSON.stringify({
+      action: 'runner_progress_events',
+      runId: input.runId,
+      playName: input.playName,
+      events: input.events,
+      liveLogOffset: input.liveLogOffset,
+    }),
+    signal: input.signal,
+    redirect: 'manual',
+  });
+  if (!response.ok) {
+    const body = await response.text().catch(() => '');
+    throw new Error(
+      `runner_progress_events failed (${response.status}): ${body}`,
+    );
+  }
+  await response.text().catch(() => '');
+}
+
+type ProgressBatch = {
+  events: PlayRunnerEvent[];
+  /** Number of durable play-authored log lines before this batch. */
+  liveLogOffset: number;
+};
+
+function durableLogCount(events: readonly PlayRunnerEvent[]): number {
+  return events.reduce(
+    (count, event) =>
+      event.type === 'log' && event.source !== 'worker' && event.line.length > 0
+        ? count + 1
+        : count,
+    0,
+  );
+}
+
+export function createRunnerProgressPusher(
+  options: RunnerProgressPushOptions,
+): RunnerProgressPusher {
+  const sendBatch = options.sendBatch ?? defaultSendBatch;
+  const setIntervalImpl =
+    options.setIntervalImpl ??
+    ((fn, ms) => setInterval(fn, ms) as unknown as { unref?: () => void });
+  const clearIntervalImpl =
+    options.clearIntervalImpl ??
+    ((handle) => clearInterval(handle as ReturnType<typeof setInterval>));
+  const warn =
+    options.warn ??
+    ((message, detail) =>
+      console.warn(`[play-runner.progress-push] ${message}`, detail));
+  const batchMs = options.batchMs ?? RUNNER_PROGRESS_PUSH_BATCH_MS;
+  const maxBatch = options.maxBatch ?? RUNNER_PROGRESS_PUSH_MAX_BATCH;
+  const stopTimeoutMs =
+    options.stopTimeoutMs ?? RUNNER_PROGRESS_PUSH_STOP_TIMEOUT_MS;
+
+  const pending: PlayRunnerEvent[] = [];
+  const queuedBatches: ProgressBatch[] = [];
+  // A gateway acknowledgement is the durability boundary. Preserve any batch
+  // that failed before that boundary so the terminal request can
+  // carry it to whichever healthy replica Fly selects next.
+  const failedBatches: ProgressBatch[] = [];
+  const activeBatches = new Map<AbortController, ProgressBatch>();
+  let nextBatchLiveLogOffset = 0;
+  let deliveryBroken = false;
+  let stopped = false;
+  let abandoned = false;
+  let suppressWarnings = false;
+  let inFlight: Promise<void> = Promise.resolve();
+  let handle: { unref?: () => void } | null = null;
+
+  const flush = (): Promise<void> => {
+    if (abandoned) {
+      pending.length = 0;
+      return inFlight;
+    }
+    if (pending.length === 0 || deliveryBroken) return inFlight;
+    const events = pending.splice(0, Math.min(pending.length, maxBatch));
+    const batch: ProgressBatch = {
+      events,
+      liveLogOffset: nextBatchLiveLogOffset,
+    };
+    nextBatchLiveLogOffset += durableLogCount(events);
+    queuedBatches.push(batch);
+    inFlight = inFlight
+      .catch(() => {})
+      .then(async () => {
+        // stop() can time out while this batch waits behind an earlier upload.
+        // Do not start it after the authoritative terminal has been emitted.
+        if (abandoned) return;
+        if (deliveryBroken) {
+          const queuedIndex = queuedBatches.indexOf(batch);
+          if (queuedIndex >= 0) queuedBatches.splice(queuedIndex, 1);
+          failedBatches.push(batch);
+          return;
+        }
+        const controller = new AbortController();
+        const queuedIndex = queuedBatches.indexOf(batch);
+        if (queuedIndex >= 0) queuedBatches.splice(queuedIndex, 1);
+        activeBatches.set(controller, batch);
+        try {
+          await sendBatch({
+            gatewayBaseUrl: options.gatewayBaseUrl,
+            executorToken: options.executorToken,
+            runId: options.runId,
+            playName: options.playName,
+            events: batch.events,
+            liveLogOffset: batch.liveLogOffset,
+            signal: controller.signal,
+          });
+        } catch (error) {
+          if (!abandoned) {
+            failedBatches.push(batch);
+            // Offsets are positional. After one unknown delivery outcome, keep
+            // the remaining contiguous suffix for the terminal request rather
+            // than sending holes that another replica could misinterpret.
+            deliveryBroken = true;
+          }
+          if (!suppressWarnings) {
+            warn('batch_failed', {
+              runId: options.runId,
+              count: batch.events.length,
+              error: error instanceof Error ? error.message : String(error),
+            });
+          }
+        } finally {
+          activeBatches.delete(controller);
+        }
+      });
+    return inFlight;
+  };
+
+  handle = setIntervalImpl(() => {
+    void flush();
+  }, batchMs);
+  handle.unref?.();
+
+  return {
+    enqueue(event) {
+      if (stopped) return;
+      pending.push(event);
+      if (pending.length >= maxBatch) {
+        void flush();
+      }
+    },
+    sealForTerminal() {
+      if (stopped) return { events: [], liveLogOffset: nextBatchLiveLogOffset };
+      stopped = true;
+      if (handle) {
+        clearIntervalImpl(handle);
+        handle = null;
+      }
+      // Preserve every batch removed from `pending`, including batches queued
+      // behind an earlier upload. The terminal request is the final owner of
+      // these observations; do not drop a burst merely because it has not yet
+      // acquired an AbortController.
+      const retainedBatches: ProgressBatch[] = [
+        ...failedBatches,
+        ...activeBatches.values(),
+        ...queuedBatches,
+        ...(pending.length
+          ? [{ events: [...pending], liveLogOffset: nextBatchLiveLogOffset }]
+          : []),
+      ];
+      const unboundedTail = retainedBatches.flatMap((batch) => batch.events);
+      const droppedEvents = Math.max(
+        0,
+        unboundedTail.length - RUNTIME_RUNNER_TERMINAL_PROGRESS_MAX_EVENTS,
+      );
+      const droppedPrefix = unboundedTail.slice(0, droppedEvents);
+      const tail = unboundedTail.slice(
+        -RUNTIME_RUNNER_TERMINAL_PROGRESS_MAX_EVENTS,
+      );
+      const liveLogOffset =
+        (retainedBatches[0]?.liveLogOffset ?? nextBatchLiveLogOffset) +
+        durableLogCount(droppedPrefix);
+      if (tail.length < unboundedTail.length) {
+        warn('terminal_tail_truncated', {
+          runId: options.runId,
+          retainedEvents: tail.length,
+          droppedEvents: unboundedTail.length - tail.length,
+        });
+      }
+      failedBatches.length = 0;
+      queuedBatches.length = 0;
+      pending.length = 0;
+      abandoned = true;
+      suppressWarnings = true;
+      for (const controller of activeBatches.keys()) controller.abort();
+      return { events: tail, liveLogOffset };
+    },
+    async stop() {
+      if (stopped) return;
+      stopped = true;
+      if (handle) {
+        clearIntervalImpl(handle);
+        handle = null;
+      }
+      const drain = async () => {
+        // Drain everything still buffered.
+        while (pending.length > 0 && !deliveryBroken) {
+          await flush();
+        }
+        await inFlight.catch(() => {});
+        if (deliveryBroken) pending.length = 0;
+      };
+      let stopTimeoutHandle: ReturnType<typeof setTimeout> | null = null;
+      const outcome = await Promise.race([
+        drain().then(() => 'drained' as const),
+        new Promise<'timed_out'>((resolve) => {
+          stopTimeoutHandle = setTimeout(
+            () => resolve('timed_out'),
+            stopTimeoutMs,
+          );
+        }),
+      ]);
+      if (stopTimeoutHandle) clearTimeout(stopTimeoutHandle);
+      if (outcome === 'drained') return;
+
+      // The authoritative terminal follows this method. Abort ordinary fetches
+      // and drop non-cooperative test/transport promises without letting their
+      // later rejection write a post-terminal diagnostic.
+      abandoned = true;
+      suppressWarnings = true;
+      const pendingEvents = pending.length;
+      pending.length = 0;
+      const inFlightEvents = [...activeBatches.values()].reduce(
+        (sum, batch) => sum + batch.events.length,
+        0,
+      );
+      for (const controller of activeBatches.keys()) controller.abort();
+      warn('drain_timed_out', {
+        runId: options.runId,
+        timeoutMs: stopTimeoutMs,
+        pendingEvents,
+        inFlightEvents,
+      });
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-sheet-persistence-retry.ts

@@ -0,0 +1,110 @@
+import { isAppRuntimeApiCapacityError } from '../app-runtime-api';
+import { isRuntimeApiCapacityError } from '../runtime-api';
+
+const RUNTIME_SHEET_CONTENTION_MAX_ATTEMPTS = 4;
+
+function persistenceCapacityRetryAfterMs(error: unknown): number | null {
+  if (isAppRuntimeApiCapacityError(error) || isRuntimeApiCapacityError(error)) {
+    return Math.max(0, Math.ceil(error.retryAfterMs));
+  }
+  return null;
+}
+
+function capacityRetryDelayMs(
+  retryAfterMs: number,
+  random: () => number,
+): number {
+  // retryAfterMs is the server's lower bound. Add one-sided jitter so a
+  // synchronized cohort spreads out without retrying before capacity is
+  // expected to be available.
+  const jitter = Math.max(0, Math.min(1, random())) * 0.2 + 1;
+  return Math.ceil(retryAfterMs * jitter);
+}
+
+function isRetryableRuntimePersistenceContention(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return /deadlock detected|Runtime Postgres connection timed out|could not serialize access|tuple concurrently updated/i.test(
+    message,
+  );
+}
+
+function throwIfAborted(signal: AbortSignal | undefined): void {
+  if (!signal?.aborted) return;
+  if (signal.reason !== undefined) throw signal.reason;
+  const error = new Error('Runtime Sheet persistence retry was aborted.');
+  error.name = 'AbortError';
+  throw error;
+}
+
+async function abortableSleep(
+  ms: number,
+  signal: AbortSignal | undefined,
+): Promise<void> {
+  throwIfAborted(signal);
+  await new Promise<void>((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      signal?.removeEventListener('abort', onAbort);
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timeout);
+      signal?.removeEventListener('abort', onAbort);
+      try {
+        throwIfAborted(signal);
+      } catch (error) {
+        reject(error);
+      }
+    };
+    signal?.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
+export async function retryRuntimeSheetPersistence<T>(input: {
+  description: string;
+  log: (line: string) => void;
+  run: () => Promise<T>;
+  signal: AbortSignal;
+  random?: () => number;
+  sleep?: (ms: number, signal: AbortSignal | undefined) => Promise<void>;
+}): Promise<T> {
+  const random = input.random ?? Math.random;
+  const sleep = input.sleep ?? abortableSleep;
+  let contentionAttempt = 0;
+  let capacityAttempt = 0;
+
+  for (;;) {
+    throwIfAborted(input.signal);
+    try {
+      return await input.run();
+    } catch (error) {
+      throwIfAborted(input.signal);
+      const retryAfterMs = persistenceCapacityRetryAfterMs(error);
+      if (retryAfterMs !== null) {
+        capacityAttempt += 1;
+        const delayMs = capacityRetryDelayMs(retryAfterMs, random);
+        input.log(
+          `[warn] ${input.description} waiting for persistence capacity attempt=${capacityAttempt} delay_ms=${delayMs}: ${
+            error instanceof Error ? error.message : String(error)
+          }`,
+        );
+        await sleep(delayMs, input.signal);
+        continue;
+      }
+
+      contentionAttempt += 1;
+      if (
+        contentionAttempt >= RUNTIME_SHEET_CONTENTION_MAX_ATTEMPTS ||
+        !isRetryableRuntimePersistenceContention(error)
+      ) {
+        throw error;
+      }
+      const delayMs = 150 * contentionAttempt;
+      input.log(
+        `[warn] ${input.description} retryable persistence contention attempt=${contentionAttempt}/${RUNTIME_SHEET_CONTENTION_MAX_ATTEMPTS} delay_ms=${delayMs}: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+      await sleep(delayMs, input.signal);
+    }
+  }
+}