@parall/agent-core 1.45.0 → 1.46.0

package/src/step-persister.ts

@@ -0,0 +1,161 @@
+import type { AgentStep, CreateAgentStepRequest, ParallClient } from '@parall/sdk';
+import { ApiError } from '@parall/sdk';
+import { type StepDrainOutcome, StepRetryQueue } from './step-retry-queue.js';
+
+/**
+ * AgentStep write path: one place decides inline-write vs park-for-retry.
+ *
+ * Extracted from the gateway so step persistence is a nameable collaborator
+ * instead of more growth in gateway-base.ts. Behavior contract (see
+ * AGENTS.md § Step write resilience): every request carries an
+ * idempotency_key so retries are replay-safe; transient failures park in the
+ * per-session FIFO StepRetryQueue; while a session has parked writes, new
+ * writes queue behind them to preserve ledger order; session-stale errors
+ * propagate to the caller (the gateway's existing recovery contract).
+ */
+export type StepPersisterOpts = {
+  client: ParallClient;
+  orgId: string;
+  agentUserId: string;
+  log?: { warn: (msg: string) => void };
+  /** Session is dead (SESSION_NOT_LIVE / INVALID_TRANSITION 409). */
+  isSessionStale: (err: unknown) => boolean;
+  /** Retry backoff override (tests). Default: StepRetryQueue's schedule. */
+  retryDelaysMs?: number[];
+};
+
+export function isRetryableStepError(err: unknown): boolean {
+  // ApiError status 0 = fetch-level failure (timeout / network); 5xx =
+  // server-side transient. 4xx are contract errors a retry cannot fix.
+  return err instanceof ApiError && (err.status === 0 || err.status >= 500);
+}
+
+export class StepPersister {
+  private readonly queue: StepRetryQueue;
+  // Sessions whose producer end is closed (finalization in progress) — new
+  // writes are refused instead of racing the drain/close sequence.
+  private readonly sealed = new Set<string>();
+  private disposed = false;
+
+  constructor(private readonly opts: StepPersisterOpts) {
+    this.queue = new StepRetryQueue({
+      log: opts.log,
+      isRetryable: isRetryableStepError,
+      isSessionStale: opts.isSessionStale,
+      retryDelaysMs: opts.retryDelaysMs,
+      // Inline step writes already drive full stale recovery; a background
+      // retry discovering it just drops the dead session's queue — the next
+      // dispatch hits the same 409 inline and heals.
+      onSessionStale: (sessionId, err) =>
+        opts.log?.warn(`queued step hit stale session ${sessionId}: ${String(err)}`),
+    });
+  }
+
+  /**
+   * Write one AgentStep, riding the retry queue on transient failure.
+   * Returns the created step, or null when the write was queued or dropped.
+   */
+  async persist(
+    sessionId: string,
+    label: string,
+    req: CreateAgentStepRequest,
+  ): Promise<AgentStep | null> {
+    if (this.disposed) {
+      // Absorbing, like the queue and the lifecycle coordinator: a dispatch
+      // still draining past the shutdown deadline must not race the exiting
+      // process with fresh inline writes — its WorkItem is un-acked, so the
+      // re-drive on the replacement pod rewrites these steps idempotently.
+      this.opts.log?.warn(`refusing ${label} step after dispose (shutting down)`);
+      return null;
+    }
+    if (this.sealed.has(sessionId)) {
+      this.opts.log?.warn(
+        `refusing ${label} step for sealed session ${sessionId} (finalization in progress)`,
+      );
+      return null;
+    }
+    if (this.queue.hasPending(sessionId)) {
+      // Preserve per-session FIFO while degraded.
+      this.enqueue(sessionId, label, req);
+      return null;
+    }
+    try {
+      return await this.opts.client.createAgentStep(
+        this.opts.orgId,
+        this.opts.agentUserId,
+        sessionId,
+        req,
+      );
+    } catch (err) {
+      if (this.opts.isSessionStale(err)) throw err;
+      if (isRetryableStepError(err)) {
+        this.opts.log?.warn(`failed to create ${label} step (queued for retry): ${String(err)}`);
+        this.enqueue(sessionId, label, req);
+      } else {
+        this.opts.log?.warn(`failed to create ${label} step: ${String(err)}`);
+      }
+      return null;
+    }
+  }
+
+  /**
+   * Close the producer end for a session (finalization began): subsequent
+   * persist() calls are refused with a warn. Parked/in-flight writes keep
+   * retrying — seal gates NEW writes only.
+   */
+  seal(sessionId: string): void {
+    this.sealed.add(sessionId);
+  }
+
+  /**
+   * Reclaim a seal once its finalization released the session (the seal set
+   * must not grow forever on a daemon churning fork sessions). After the
+   * session is closed server-side, any late write is rejected there (409).
+   */
+  unseal(sessionId: string): void {
+    this.sealed.delete(sessionId);
+  }
+
+  /**
+   * Resolves when the session's step queue reaches quiescence — see
+   * StepRetryQueue.awaitSessionDrained. Non-destructive: parked writes keep
+   * retrying on their normal schedule while awaited.
+   */
+  drainSession(sessionId: string): Promise<StepDrainOutcome> {
+    return this.queue.awaitSessionDrained(sessionId);
+  }
+
+  /** DESTRUCTIVE: discards the session's parked writes (server-terminal only). */
+  dropSession(sessionId: string): void {
+    this.sealed.delete(sessionId);
+    this.queue.dropSession(sessionId);
+  }
+
+  /** Every session is dead (server closed them all — new_session). */
+  dropAllSessions(): void {
+    this.sealed.clear();
+    this.queue.dropAllSessions();
+  }
+
+  /** Best-effort single pass over parked writes; returns items still parked. */
+  flush(deadlineMs: number): Promise<number> {
+    return this.queue.flush(deadlineMs);
+  }
+
+  pendingTotal(): number {
+    return this.queue.pendingTotal();
+  }
+
+  dispose(): void {
+    this.disposed = true;
+    this.queue.dispose();
+  }
+
+  private enqueue(sessionId: string, label: string, req: CreateAgentStepRequest): void {
+    this.queue.enqueue(sessionId, label, () =>
+      this.opts.client
+        .createAgentStep(this.opts.orgId, this.opts.agentUserId, sessionId, req)
+        .then(() => undefined),
+    );
+  }
+}

package/src/step-retry-queue.ts

@@ -0,0 +1,296 @@
+/**
+ * Per-session FIFO retry queue for AgentStep writes.
+ *
+ * Step creation runs serially inside the dispatch drain loop, so a blocking
+ * retry there would stall healthy runtime work behind a degraded api link
+ * (the 2026-07-10 window: model calls fine, every fresh parall HTTPS
+ * handshake timing out). Instead, a failed write is parked here and retried
+ * in the background with backoff; while a session has parked items, new
+ * writes for that session are appended behind them so ledger order is
+ * preserved. All step requests carry an `idempotency_key`, which makes
+ * retries safe even when the original attempt landed server-side but its
+ * response was lost.
+ *
+ * In-memory only — a process restart drops parked items (the ledger hole is
+ * then no worse than the pre-queue behavior). Items are dropped, loudly, when
+ * they exceed `maxAgeMs` or the per-session cap.
+ */
+
+export type StepRetryQueueOpts = {
+  log?: { warn: (msg: string) => void };
+  /** Transient failures worth retrying (timeouts, network, 5xx). */
+  isRetryable: (err: unknown) => boolean;
+  /** Session is dead (SESSION_NOT_LIVE / INVALID_TRANSITION) — drop its queue. */
+  isSessionStale: (err: unknown) => boolean;
+  /** Invoked once when a retry discovers a stale session. */
+  onSessionStale?: (sessionId: string, err: unknown) => void;
+  /** Drop an item once it has been parked this long. Default 10 minutes. */
+  maxAgeMs?: number;
+  /** Per-session cap; enqueue beyond it drops the NEW item. Default 200. */
+  maxQueueLength?: number;
+  /** Backoff schedule override (tests). Default 5s→10s→20s→40s→60s. */
+  retryDelaysMs?: number[];
+};
+
+type QueuedItem = {
+  label: string;
+  exec: () => Promise<void>;
+  enqueuedAt: number;
+  attempts: number;
+};
+
+/**
+ * How a session's queue reached quiescence: `drained` — every parked item
+ * settled (written, or dropped loudly by age/cap/non-retryable policy);
+ * `dropped` — the queue was discarded because the server considers the
+ * session terminal (stale 409, `agent.new_session`); `disposed` — the queue
+ * was shut down mid-wait.
+ */
+export type StepDrainOutcome = 'drained' | 'dropped' | 'disposed';
+
+const DEFAULT_RETRY_DELAYS_MS = [5_000, 10_000, 20_000, 40_000, 60_000];
+
+async function raceWithDeadline<T>(work: Promise<T>, ms: number): Promise<T | 'timeout'> {
+  let timer: NodeJS.Timeout | undefined;
+  const timeout = new Promise<'timeout'>((resolve) => {
+    timer = setTimeout(() => resolve('timeout'), Math.max(0, ms));
+  });
+  try {
+    return await Promise.race([work, timeout]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export class StepRetryQueue {
+  private readonly queues = new Map<string, QueuedItem[]>();
+  private readonly timers = new Map<string, NodeJS.Timeout>();
+  // Per-session in-flight drain promise. At most ONE drain may be active per
+  // session: a second one would re-run the same head and the two shift()
+  // calls would silently discard the next queued item. schedule() is a no-op
+  // while a drain is in flight; a concurrent drain() call JOINS the in-flight
+  // promise (flush must be able to await a background drain instead of
+  // misreading it as "no progress"); the drain's finally re-arms from live
+  // map state.
+  private readonly drains = new Map<string, Promise<void>>();
+  // Callers awaiting a session's quiescence (awaitSessionDrained).
+  private readonly drainWaiters = new Map<string, Array<(outcome: StepDrainOutcome) => void>>();
+  private readonly retryDelays: number[];
+  private disposed = false;
+
+  constructor(private readonly opts: StepRetryQueueOpts) {
+    this.retryDelays = opts.retryDelaysMs?.length ? opts.retryDelaysMs : DEFAULT_RETRY_DELAYS_MS;
+  }
+
+  hasPending(sessionId: string): boolean {
+    return (this.queues.get(sessionId)?.length ?? 0) > 0;
+  }
+
+  pendingCount(sessionId: string): number {
+    return this.queues.get(sessionId)?.length ?? 0;
+  }
+
+  enqueue(sessionId: string, label: string, exec: () => Promise<void>): void {
+    if (this.disposed) return;
+    const queue = this.queues.get(sessionId) ?? [];
+    const cap = this.opts.maxQueueLength ?? 200;
+    if (queue.length >= cap) {
+      this.opts.log?.warn(
+        `step retry queue full for session ${sessionId} (${cap}); dropping ${label} step permanently`,
+      );
+      return;
+    }
+    queue.push({ label, exec, enqueuedAt: Date.now(), attempts: 0 });
+    this.queues.set(sessionId, queue);
+    this.schedule(sessionId, this.retryDelays[0] ?? 5_000);
+  }
+
+  dropSession(sessionId: string): void {
+    const queue = this.queues.get(sessionId);
+    if (queue?.length) {
+      this.opts.log?.warn(`dropping ${queue.length} queued step(s) for session ${sessionId}`);
+    }
+    this.queues.delete(sessionId);
+    const timer = this.timers.get(sessionId);
+    if (timer) clearTimeout(timer);
+    this.timers.delete(sessionId);
+    this.resolveDrainWaiters(sessionId, 'dropped');
+  }
+
+  /**
+   * Resolves when this session's queue reaches quiescence: every parked item
+   * settled (`drained` — including items dropped loudly by the age/cap/
+   * non-retryable policy), the queue was discarded because the server
+   * considers the session terminal (`dropped`), or the queue was disposed
+   * (`disposed`). Retries keep riding the normal backoff schedule while a
+   * caller waits — this only OBSERVES the queue, it never accelerates or
+   * abandons it.
+   */
+  awaitSessionDrained(sessionId: string): Promise<StepDrainOutcome> {
+    if (this.disposed) return Promise.resolve('disposed');
+    if (this.pendingCount(sessionId) === 0 && !this.drains.has(sessionId)) {
+      return Promise.resolve('drained');
+    }
+    return new Promise((resolve) => {
+      const waiters = this.drainWaiters.get(sessionId) ?? [];
+      waiters.push(resolve);
+      this.drainWaiters.set(sessionId, waiters);
+    });
+  }
+
+  pendingTotal(): number {
+    let total = 0;
+    for (const queue of this.queues.values()) total += queue.length;
+    return total;
+  }
+
+  /**
+   * One best-effort pass over every parked item — no backoff waits. Used at
+   * graceful shutdown: parked items are process-local (their WorkItems
+   * already resolved, so restart catch-up will NOT re-drive them), and the
+   * common shutdown (idle-stop, deploy) happens on a healthy network where
+   * the writes just succeed. A session whose head still fails retryably is
+   * skipped (its remaining items would fail the same way). A drain already in
+   * flight when flush arrives is JOINED (drain() returns the shared promise),
+   * so flush waits for its settle within the deadline instead of misreading
+   * "no progress" and giving up early. `deadlineMs` is a HARD cap: a write
+   * still in flight at the deadline is abandoned to the background (it is
+   * idempotent; dispose() right after clears any re-arm) and flush returns.
+   * Returns the number of items still parked.
+   */
+  async flush(deadlineMs: number): Promise<number> {
+    const deadline = Date.now() + deadlineMs;
+    const timeLeft = () => deadline - Date.now();
+    let timedOut = false;
+    for (const sessionId of [...this.queues.keys()]) {
+      while (!this.disposed && !timedOut && timeLeft() > 0) {
+        const before = this.pendingCount(sessionId);
+        if (before === 0) break;
+        const outcome = await raceWithDeadline(
+          this.drain(sessionId).then(() => 'drained' as const),
+          timeLeft(),
+        );
+        if (outcome === 'timeout') {
+          timedOut = true;
+          break;
+        }
+        if (this.pendingCount(sessionId) >= before) break;
+      }
+      if (this.disposed || timedOut || timeLeft() <= 0) break;
+    }
+    return this.pendingTotal();
+  }
+
+  /** Every tracked session is dead (server closed them all — new_session). */
+  dropAllSessions(): void {
+    for (const sessionId of [...this.queues.keys()]) this.dropSession(sessionId);
+  }
+
+  dispose(): void {
+    this.disposed = true;
+    for (const timer of this.timers.values()) clearTimeout(timer);
+    this.timers.clear();
+    this.queues.clear();
+    for (const sessionId of [...this.drainWaiters.keys()]) {
+      this.resolveDrainWaiters(sessionId, 'disposed');
+    }
+  }
+
+  private schedule(sessionId: string, delayMs: number): void {
+    // No timer while a drain is in flight — its finally re-arms from live
+    // state, so arming here would start a second concurrent drain.
+    if (this.disposed || this.timers.has(sessionId) || this.drains.has(sessionId)) return;
+    const timer = setTimeout(() => {
+      this.timers.delete(sessionId);
+      void this.drain(sessionId);
+    }, delayMs);
+    timer.unref?.();
+    this.timers.set(sessionId, timer);
+  }
+
+  // At most one drain per session; a concurrent call joins the in-flight
+  // promise so callers (flush) await the real settle instead of a no-op.
+  private drain(sessionId: string): Promise<void> {
+    const inFlight = this.drains.get(sessionId);
+    if (inFlight) return inFlight;
+    const run = (async () => {
+      try {
+        await this.drainOnce(sessionId);
+      } finally {
+        this.drains.delete(sessionId);
+        this.rearm(sessionId);
+      }
+    })();
+    this.drains.set(sessionId, run);
+    return run;
+  }
+
+  private async drainOnce(sessionId: string): Promise<void> {
+    const queue = this.queues.get(sessionId);
+    const head = queue?.[0];
+    if (!queue || !head) {
+      if (queue) this.queues.delete(sessionId);
+      return;
+    }
+    const maxAgeMs = this.opts.maxAgeMs ?? 10 * 60_000;
+    if (Date.now() - head.enqueuedAt > maxAgeMs) {
+      this.opts.log?.warn(
+        `giving up on ${head.label} step for session ${sessionId} after ${head.attempts} retries (parked > ${Math.round(maxAgeMs / 1000)}s); step permanently lost`,
+      );
+      queue.shift();
+      return;
+    }
+    head.attempts += 1;
+    try {
+      await head.exec();
+      // The session may have been dropped (or the queue replaced) while
+      // exec was in flight — only mutate the queue we started with if it
+      // is still the live one.
+      if (this.queues.get(sessionId) === queue) queue.shift();
+    } catch (err) {
+      if (this.opts.isSessionStale(err)) {
+        this.dropSession(sessionId);
+        this.opts.onSessionStale?.(sessionId, err);
+        return;
+      }
+      if (this.queues.get(sessionId) !== queue) return;
+      if (!this.opts.isRetryable(err)) {
+        this.opts.log?.warn(
+          `dropping ${head.label} step for session ${sessionId} (non-retryable): ${String(err)}`,
+        );
+        queue.shift();
+      }
+      // Retryable: keep the head in place for the next pass.
+    }
+  }
+
+  // Single re-arm point after a drain settles, reading LIVE map state (the
+  // queue may have been dropped, or grown, while exec was awaited). Also the
+  // single point that observes quiescence for awaitSessionDrained: it runs
+  // after EVERY drain settle, so an emptied (or already-dropped) queue always
+  // resolves its waiters here.
+  private rearm(sessionId: string): void {
+    const queue = this.queues.get(sessionId);
+    if (!queue || queue.length === 0) {
+      if (queue) this.queues.delete(sessionId);
+      this.resolveDrainWaiters(sessionId, 'drained');
+      return;
+    }
+    const head = queue[0];
+    if (!head) return;
+    // A fresh head (previous item succeeded or was dropped) drains quickly;
+    // a head that just failed a retryable attempt backs off by its attempts.
+    const delay =
+      head.attempts === 0
+        ? Math.min(250, this.retryDelays[0] ?? 250)
+        : (this.retryDelays[Math.min(head.attempts, this.retryDelays.length - 1)] ?? 60_000);
+    this.schedule(sessionId, delay);
+  }
+
+  private resolveDrainWaiters(sessionId: string, outcome: StepDrainOutcome): void {
+    const waiters = this.drainWaiters.get(sessionId);
+    if (!waiters?.length) return;
+    this.drainWaiters.delete(sessionId);
+    for (const resolve of waiters) resolve(outcome);
+  }
+}