cartethyia 2.6.5 → 2.6.7

package/src/transport/dispatch/streaming/finalize.ts

@@ -96,6 +96,17 @@ export interface StreamFinalizeInput {
  * stream back into the error path and emitting duplicate telemetry rows (and
  * a second resource release) for one request.
  */
+
+/**
+ * How long a deferred release waits for the reader it was handed to.
+ *
+ * Only a backstop: the pull or `attachController` it defers to normally
+ * releases first. Long enough that a reader mid-write is never cut off, short
+ * enough that a reader which never runs does not leave the flight registered
+ * indefinitely.
+ */
+const DEFERRED_RELEASE_BACKSTOP_MS = 30_000;
+
 export class StreamFinalizer {
   /**
    * True only while a `pull()` is awaiting the upstream iterator. A watchdog
@@ -105,6 +116,8 @@ export class StreamFinalizer {
    * abort handler instead.
    */
   pullActive = false;
+  /** Backstop timer for an abort whose release was deferred to another reader. */
+  private releaseBackstop: ReturnType<typeof setTimeout> | undefined;
 
   /**
    * Max time between upstream events before the stream is declared stalled,
@@ -183,6 +196,29 @@ export class StreamFinalizer {
     this.stallTimer = undefined;
   }
 
+  /**
+   * Guarantees {@link releaseStreamResources} runs after a deferred abort.
+   *
+   * Two branches of `onStreamAbort` hand the release to some other reader — an
+   * active pull, or `attachController` — and return. When that reader never
+   * runs, nothing releases: the flight stays in the in-flight registry and the
+   * gauge never returns to zero, which is indistinguishable from permanent
+   * load. A server-side abort (deadline, 504) is the case that reaches them,
+   * because only a client disconnect is handled inline.
+   *
+   * The timer is unref'd so it cannot keep the process alive during shutdown,
+   * and `releaseStreamResources` is idempotent (`streamReleased`), so a pull
+   * that does settle first simply makes this a no-op.
+   */
+  private armDeferredReleaseBackstop(): void {
+    if (this.releaseBackstop !== undefined) return;
+    this.releaseBackstop = setTimeout(() => {
+      this.releaseBackstop = undefined;
+      void this.releaseStreamResources();
+    }, DEFERRED_RELEASE_BACKSTOP_MS);
+    this.releaseBackstop.unref?.();
+  }
+
   /**
    * Attaches the stream controller so an abort can settle even when no pull is
    * currently pending (for example, the client stopped reading between chunks).
@@ -257,10 +293,28 @@ export class StreamFinalizer {
     // Let an active pull emit the terminal frame for a server drain or
     // deadline. With no pull, no reader will handle it, so this listener must
     // emit and close through the attached controller itself.
-    if (this.pullActive && !clientDisconnect) return;
+    // Deferring to the pull assumes the pull will settle and release. It may
+    // not: a pull awaits the upstream iterator, and an upstream that went
+    // quiet after the abort leaves that await pending — so nothing ever
+    // reaches `releaseStreamResources()` and the flight stays registered
+    // forever. A server-side abort (deadline, 504) is exactly the case, since
+    // only a client disconnect takes the path below.
+    //
+    // Arm a backstop instead of returning bare. If the pull does settle it
+    // releases first and this becomes a no-op, so the normal path is
+    // unchanged; if it never does, the flight is still reaped.
+    if (this.pullActive && !clientDisconnect) {
+      this.armDeferredReleaseBackstop();
+      return;
+    }
     // Before ReadableStream.start attaches its controller, defer server-side
     // failures so attachController can deliver the typed terminal frame.
-    if (!clientDisconnect && this.streamController === undefined) return;
+    if (!clientDisconnect && this.streamController === undefined) {
+      // Same assumption, same consequence: attachController is expected to
+      // deliver the frame and release. Nothing guarantees it runs.
+      this.armDeferredReleaseBackstop();
+      return;
+    }
     if (clientDisconnect) {
       const { candidate } = this.input;
       if (!state.outcome) {
@@ -336,6 +390,14 @@ export class StreamFinalizer {
     if (this.streamReleased) return;
     this.streamReleased = true;
     const { state, deps, prepared, lease, reservation, proxySlot } = this.input;
+    // The deferred-release backstop has done its job — or the reader it
+    // deferred to beat it here. Either way the timer must not outlive the
+    // release, and an unref'd timer left armed would fire into a released
+    // finalizer.
+    if (this.releaseBackstop !== undefined) {
+      clearTimeout(this.releaseBackstop);
+      this.releaseBackstop = undefined;
+    }
     state.abortController.signal.removeEventListener("abort", this.handleAbort);
     this.streamController = undefined;
     this.clearClientKeepaliveTimer();

package/src/workers/scheduler.ts

@@ -72,6 +72,37 @@ export async function runGrowingWaves<T>(
 
 export const DEFAULT_INITIAL_BATCH_SIZE = 2;
 
+/**
+ * How long a tick may wait for the maintenance lease.
+ *
+ * `tryAcquire` is documented as one attempt that skips rather than waits, but
+ * nothing bounded it: a Redis call that never answered held the tick open
+ * indefinitely. This restores the documented behaviour by force-skipping, so a
+ * stalled lease store costs one interval instead of every future one.
+ */
+const LEASE_ACQUIRE_TIMEOUT_MS = 5_000;
+
+/** Resolves with the value, or rejects if `promise` has not settled in time. */
+async function withDeadline<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+  // `clearTimeout` no-ops for an absent handle, so it needs no guard. A live
+  // timer would hold the process open, hence the unref.
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      promise,
+      new Promise<never>((_, reject) => {
+        timer = setTimeout(
+          () => reject(new Error(`lease acquisition did not settle within ${timeoutMs}ms`)),
+          timeoutMs,
+        );
+        timer.unref?.();
+      }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
 /**
  * Tasks may be registered at any time: before `start()`, they are scheduled
  * when `start()` runs; after `start()`, each `register()` immediately schedules
@@ -180,6 +211,18 @@ export class ScheduledTaskRegistry {
     return this.running.has(name);
   }
 
+  /**
+   * Clears the shared gate so other names may run.
+   *
+   * Every path that latches the gate and then does not run the task must reach
+   * this: a latch that is never cleared stops every later tick of every task,
+   * because the gate is process-wide rather than per-name.
+   */
+  private releaseGate(name: string): void {
+    this.running.delete(name);
+    if (this.busy === name) this.busy = undefined;
+  }
+
   /** Runs one named task immediately, bypassing its interval — for tests and manual triggers. */
   async runNow(name: string): Promise<boolean> {
     const task = this.tasks.find((t) => t.name === name);
@@ -220,13 +263,37 @@ export class ScheduledTaskRegistry {
     if (this.busy !== undefined) return;
     this.running.add(task.name);
     this.busy = task.name;
-    const lease =
-      this.lease !== undefined && task.shared === true
-        ? await this.lease.tryAcquire(task.name, leaseTtlMs(task.intervalMs))
-        : undefined;
+    let lease: Awaited<ReturnType<MaintenanceLeaseStore["tryAcquire"]>> | undefined;
+    try {
+      // The gate above is latched before this await so a concurrent tick cannot
+      // slip past while the lease is in flight. It is bounded: a lease store
+      // that never answers used to latch `busy` forever, and since the gate is
+      // shared across every task name, one stalled acquisition stopped the
+      // whole registry — including the sweep whose job is to clean up after
+      // something that never finishes.
+      lease =
+        this.lease !== undefined && task.shared === true
+          ? await withDeadline(
+              this.lease.tryAcquire(task.name, leaseTtlMs(task.intervalMs)),
+              LEASE_ACQUIRE_TIMEOUT_MS,
+            )
+          : undefined;
+    } catch (error) {
+      // A lease store that cannot answer is not this task's failure to run;
+      // without this the error propagated out of `runOnce` and the tick was
+      // lost with it.
+      log.error(`[scheduled-task:${task.name}] lease acquisition failed`, error as Error);
+    }
+    if (lease === undefined && this.lease !== undefined && task.shared === true) {
+      // Could not establish a lease: skip this tick rather than run the task
+      // unguarded, which is what sharing the lease was for.
+      this.skippedByLease.set(task.name, (this.skippedByLease.get(task.name) ?? 0) + 1);
+      this.lastSkipAt.set(task.name, this.now());
+      this.releaseGate(task.name);
+      return;
+    }
     if (lease !== undefined && !lease.held) {
-      this.running.delete(task.name);
-      this.busy = undefined;
+      this.releaseGate(task.name);
       this.skippedByLease.set(task.name, (this.skippedByLease.get(task.name) ?? 0) + 1);
       this.lastSkipAt.set(task.name, this.now());
       return;