@frockbot/provider-openai-compatible 0.3.5 → 0.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/provider-openai-compatible",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "private": false,
   "type": "module",
   "exports": {
@@ -11,8 +11,8 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/kernel-contracts": "0.3.5",
-    "@frockbot/plugin-models": "0.3.5",
+    "@frockbot/kernel-contracts": "0.3.6",
+    "@frockbot/plugin-models": "0.3.6",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {

package/src/deadlines.test.ts

@@ -0,0 +1,192 @@
+// A Turn once hung for seventeen minutes with nothing on screen: there was no
+// time bound anywhere on a model request, so a provider that accepted the call
+// and then went quiet held the socket — and the Turn — open indefinitely.
+//
+// The timer is injected, so these tests assert the behaviour rather than
+// waiting two minutes for it.
+import { describe, expect, test } from "bun:test";
+import {
+  ModelRequestDeadlineError,
+  type NormalizedModelRequest,
+} from "@frockbot/kernel-contracts";
+import { OpenAICompatibleProvider } from "./index.js";
+
+const request: NormalizedModelRequest = {
+  requestId: "request-1",
+  provider: "openai-compatible",
+  model: "test-model",
+  system: "Be useful.",
+  messages: [{ role: "user", content: "hello" }],
+  tools: [],
+};
+
+/** A clock the test advances by hand. */
+function manualClock() {
+  const pending = new Map<number, { run: () => void; due: number }>();
+  let next = 1;
+  let now = 0;
+  return {
+    schedule(run: () => void, milliseconds: number): () => void {
+      const id = next++;
+      pending.set(id, { run, due: now + milliseconds });
+      return () => pending.delete(id);
+    },
+    /** Fire everything due at or before `now + milliseconds`. */
+    advance(milliseconds: number): void {
+      now += milliseconds;
+      for (const [id, timer] of [...pending]) {
+        if (timer.due <= now) {
+          pending.delete(id);
+          timer.run();
+        }
+      }
+    },
+    get armed(): number {
+      return pending.size;
+    },
+  };
+}
+
+/** Let the generator's own pump run: the clock here is manual, real time is not. */
+async function settle(): Promise<void> {
+  for (let tick = 0; tick < 10; tick += 1) {
+    await new Promise((resolve) => setTimeout(resolve, 0));
+  }
+}
+
+async function collect(iterable: AsyncIterable<unknown>): Promise<unknown[]> {
+  const events: unknown[] = [];
+  for await (const event of iterable) events.push(event);
+  return events;
+}
+
+describe("a model request that produces nothing", () => {
+  test("is abandoned at the first-byte deadline, naming the deadline as the reason", async () => {
+    const clock = manualClock();
+    const provider = new OpenAICompatibleProvider({
+      baseUrl: "https://models.example",
+      deadlines: { firstByteMs: 120_000, idleMs: 120_000 },
+      schedule: clock.schedule,
+      fetch: (_input, init) =>
+        new Promise((_resolve, reject) => {
+          init?.signal?.addEventListener("abort", () =>
+            reject(new Error("aborted")),
+          );
+        }),
+    });
+
+    const streaming = collect(
+      provider.stream(request, new AbortController().signal),
+    );
+    // The provider has been sent the request and has said nothing.
+    clock.advance(120_000);
+
+    const failure = await streaming.then(
+      () => undefined,
+      (error: unknown) => error,
+    );
+    expect(failure).toBeInstanceOf(ModelRequestDeadlineError);
+    expect((failure as ModelRequestDeadlineError).phase).toBe("first-byte");
+    expect((failure as Error).message).toBe(
+      "Model request produced nothing within 120s",
+    );
+  });
+});
+
+describe("a model response that stalls mid-answer", () => {
+  test("is abandoned at the idle deadline, not at the first-byte one", async () => {
+    const clock = manualClock();
+    let push: ((chunk: string) => void) | undefined;
+    const body = new ReadableStream<Uint8Array>({
+      start(controller) {
+        push = (chunk) => controller.enqueue(new TextEncoder().encode(chunk));
+      },
+    });
+    const provider = new OpenAICompatibleProvider({
+      baseUrl: "https://models.example",
+      deadlines: { firstByteMs: 10_000, idleMs: 60_000 },
+      schedule: clock.schedule,
+      fetch: () => Promise.resolve(new Response(body, { status: 200 })),
+    });
+
+    const events: unknown[] = [];
+    const streaming = (async () => {
+      for await (const event of provider.stream(
+        request,
+        new AbortController().signal,
+      )) {
+        events.push(event);
+      }
+    })();
+
+    push?.(
+      'data: {"choices":[{"delta":{"content":"Half a "}}]}\n\ndata: {"choices":[{"delta":{"content":"thought"}}]}\n\n',
+    );
+    await settle();
+
+    // Past the first-byte allowance, inside the idle one: the answer started,
+    // so the clock it is being held to is the idle one.
+    clock.advance(10_000);
+    await settle();
+    clock.advance(60_000);
+
+    const failure = await streaming.then(
+      () => undefined,
+      (error: unknown) => error,
+    );
+    expect(failure).toBeInstanceOf(ModelRequestDeadlineError);
+    expect((failure as ModelRequestDeadlineError).phase).toBe("idle");
+    expect(events.length).toBeGreaterThan(0);
+  });
+});
+
+describe("a model request that finishes", () => {
+  test("leaves no timer armed", async () => {
+    const clock = manualClock();
+    const provider = new OpenAICompatibleProvider({
+      baseUrl: "https://models.example",
+      schedule: clock.schedule,
+      fetch: () =>
+        Promise.resolve(
+          new Response(
+            'data: {"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n',
+            { status: 200 },
+          ),
+        ),
+    });
+
+    const events = await collect(
+      provider.stream(request, new AbortController().signal),
+    );
+
+    expect(events.length).toBeGreaterThan(0);
+    // A live timer in a Worker isolate holds the request open long after
+    // anybody is listening for it.
+    expect(clock.armed).toBe(0);
+  });
+
+  test("a caller's own Stop still reports the cancellation, not a deadline", async () => {
+    const clock = manualClock();
+    const caller = new AbortController();
+    const provider = new OpenAICompatibleProvider({
+      baseUrl: "https://models.example",
+      schedule: clock.schedule,
+      fetch: (_input, init) =>
+        new Promise((_resolve, reject) => {
+          init?.signal?.addEventListener("abort", () =>
+            reject(new Error("aborted by caller")),
+          );
+        }),
+    });
+
+    const streaming = collect(provider.stream(request, caller.signal));
+    caller.abort(new Error("Stop"));
+
+    const failure = await streaming.then(
+      () => undefined,
+      (error: unknown) => error,
+    );
+    expect(failure).not.toBeInstanceOf(ModelRequestDeadlineError);
+    expect(clock.armed).toBe(0);
+  });
+});

package/src/index.ts

@@ -2,6 +2,9 @@ import {
   type LlmMessage,
   type LlmProvider,
   type LlmStreamEvent,
+  MODEL_REQUEST_DEADLINES_V1,
+  type ModelRequestDeadlinesV1,
+  ModelRequestDeadlineError,
   type NormalizedModelRequest,
 } from "@frockbot/kernel-contracts";
 import type { Plugin } from "cordis";
@@ -32,6 +35,82 @@ export interface OpenAICompatibleConfig {
    * model id decides through {@link modelAcceptsImagesV1}.
    */
   acceptsImages?: boolean;
+  /** Overrides {@link MODEL_REQUEST_DEADLINES_V1}. */
+  deadlines?: Partial<ModelRequestDeadlinesV1>;
+  /**
+   * Timer seam, so a deadline test does not have to wait two minutes for one.
+   * Defaults to `setTimeout`.
+   */
+  schedule?: (run: () => void, milliseconds: number) => () => void;
+}
+
+/**
+ * A request's clock, watching for silence.
+ *
+ * One controller, aborted with a {@link ModelRequestDeadlineError} when the
+ * provider says nothing for too long, and rearmed on every stream event. It
+ * chains the caller's signal so a Stop still cancels immediately, and it is
+ * always disarmed in a `finally`: a live timer in a Worker isolate holds the
+ * request open long after anyone is listening.
+ */
+class ModelRequestClockV1 {
+  readonly #controller = new AbortController();
+  readonly #deadlines: ModelRequestDeadlinesV1;
+  readonly #schedule: (run: () => void, milliseconds: number) => () => void;
+  #cancelTimer: (() => void) | undefined;
+  #disarmed = false;
+
+  constructor(
+    caller: AbortSignal,
+    deadlines: ModelRequestDeadlinesV1,
+    schedule: (run: () => void, milliseconds: number) => () => void,
+  ) {
+    this.#deadlines = deadlines;
+    this.#schedule = schedule;
+    if (caller.aborted) this.#controller.abort(caller.reason);
+    else {
+      caller.addEventListener(
+        "abort",
+        () => this.#controller.abort(caller.reason),
+        { once: true },
+      );
+    }
+    this.#arm("first-byte");
+  }
+
+  get signal(): AbortSignal {
+    return this.#controller.signal;
+  }
+
+  /** A stream event arrived: the clock restarts on the idle allowance. */
+  progressed(): void {
+    this.#arm("idle");
+  }
+
+  disarm(): void {
+    this.#disarmed = true;
+    this.#cancelTimer?.();
+    this.#cancelTimer = undefined;
+  }
+
+  #arm(phase: "first-byte" | "idle"): void {
+    if (this.#disarmed) return;
+    this.#cancelTimer?.();
+    const milliseconds =
+      phase === "first-byte"
+        ? this.#deadlines.firstByteMs
+        : this.#deadlines.idleMs;
+    this.#cancelTimer = this.#schedule(() => {
+      this.#controller.abort(
+        new ModelRequestDeadlineError(phase, milliseconds),
+      );
+    }, milliseconds);
+  }
+}
+
+function defaultScheduleV1(run: () => void, milliseconds: number): () => void {
+  const timer = setTimeout(run, milliseconds);
+  return () => clearTimeout(timer);
 }
 
 interface ToolAccumulator {
@@ -363,26 +442,59 @@ export class OpenAICompatibleProvider implements LlmProvider {
     };
     if (this.config.apiKey)
       headers.authorization = `Bearer ${this.config.apiKey}`;
-    const response = await fetcher(`${this.config.baseUrl}/chat/completions`, {
-      method: "POST",
-      headers,
-      body: JSON.stringify(
-        requestToWire(request, {
-          ...(this.config.acceptsImages === undefined
-            ? {}
-            : { acceptsImages: this.config.acceptsImages }),
-        }),
-      ),
+    // Nothing here used to have a time bound: a provider that accepted the
+    // request and then went quiet held the Turn open for as long as the socket
+    // stayed up — seventeen minutes, in the incident this exists for, with
+    // nothing on the person's screen the whole time.
+    const clock = new ModelRequestClockV1(
       signal,
-    });
-    if (!response.ok) {
-      await response.body?.cancel();
-      throw new OpenAICompatibleHttpError(response.status);
-    }
-    if (!response.body)
-      throw new Error("Model response did not include a stream");
+      { ...MODEL_REQUEST_DEADLINES_V1, ...this.config.deadlines },
+      this.config.schedule ?? defaultScheduleV1,
+    );
+    try {
+      const response = await fetcher(
+        `${this.config.baseUrl}/chat/completions`,
+        {
+          method: "POST",
+          headers,
+          body: JSON.stringify(
+            requestToWire(request, {
+              ...(this.config.acceptsImages === undefined
+                ? {}
+                : { acceptsImages: this.config.acceptsImages }),
+            }),
+          ),
+          signal: clock.signal,
+        },
+      );
+      if (!response.ok) {
+        await response.body?.cancel();
+        throw new OpenAICompatibleHttpError(response.status);
+      }
+      if (!response.body)
+        throw new Error("Model response did not include a stream");
 
-    yield* streamOpenAICompatibleBody(response.body, signal);
+      for await (const event of streamOpenAICompatibleBody(
+        response.body,
+        clock.signal,
+      )) {
+        clock.progressed();
+        yield event;
+      }
+    } catch (error) {
+      // The abort reason is the real failure; `AbortError` is only how it
+      // reached us. Without this the Turn reports a cancellation nobody asked
+      // for instead of the deadline it actually hit.
+      if (
+        clock.signal.reason instanceof ModelRequestDeadlineError &&
+        !signal.aborted
+      ) {
+        throw clock.signal.reason;
+      }
+      throw error;
+    } finally {
+      clock.disarm();
+    }
   }
 }