@remnic/core 10.0.25 → 10.0.27

package/src/local-llm-helpers.ts

@@ -1,10 +1,11 @@
+import type { ResponseFormatJSONSchema } from "openai/resources/shared";
 import { log } from "./logger.js";
 import type { LocalLlmRequestPriority } from "./local-llm.js";
 
 export interface LocalLlmChatCompletionOptions {
   temperature?: number;
   maxTokens?: number;
-  responseFormat?: { type: string };
+  responseFormat?: { type: string } | ResponseFormatJSONSchema;
   timeoutMs?: number;
   operation?: string;
   forceDisableThinking?: boolean;

package/src/local-llm-json-schema-fallback.test.ts

@@ -0,0 +1,278 @@
+import assert from "node:assert/strict";
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+import test, { mock } from "node:test";
+
+import { LocalLlmClient } from "./local-llm.js";
+import { ChatTransport } from "./local-llm-transport.js";
+import { parseConfig } from "./config.js";
+import type { LocalLlmChatCompletionOptions } from "./local-llm-helpers.js";
+import type { PluginConfig } from "./types.js";
+
+function createConfig(overrides: Partial<PluginConfig> = {}): PluginConfig {
+  return parseConfig({
+    localLlmEnabled: true,
+    localLlmModel: "test-local-model",
+    localLlmUrl: "http://127.0.0.1:1234",
+    localLlmTimeoutMs: 5_000,
+    localLlmRetry5xxCount: 0,
+    localLlmRateLimitRetryCount: 0,
+    localLlmRetryBackoffMs: 1,
+    localLlmHeaders: { "X-Test": "yes" },
+    localLlmApiKey: undefined,
+    localLlmAuthHeader: false,
+    localLlm400TripThreshold: 50,
+    localLlm400CooldownMs: 60_000,
+    debug: false,
+    localLlmReasoningEffort: "none",
+    slowLogEnabled: false,
+    slowLogThresholdMs: 60_000,
+    ...overrides,
+  });
+}
+
+function primeClient(client: LocalLlmClient): void {
+  mock.method(client, "checkAvailability", async () => true);
+}
+
+interface RecordedRequest {
+  headers: http.IncomingHttpHeaders;
+  body: { model: string; response_format?: LocalLlmChatCompletionOptions["responseFormat"] };
+}
+
+async function startRecordingServer(
+  respond: (req: http.IncomingMessage, requestCount: number) => {
+    status: number;
+    body: unknown;
+  },
+): Promise<{
+  url: string;
+  requests: RecordedRequest[];
+  close: () => Promise<void>;
+}> {
+  const requests: RecordedRequest[] = [];
+  const server = http.createServer((req, res) => {
+    let raw = "";
+    req.setEncoding("utf8");
+    req.on("data", (chunk: string) => {
+      raw += chunk;
+    });
+    req.on("end", () => {
+      requests.push({ headers: req.headers, body: JSON.parse(raw) });
+      const { status, body } = respond(req, requests.length);
+      res.writeHead(status, { "content-type": "application/json" });
+      res.end(JSON.stringify(body));
+    });
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  // SAFETY: listen(0, "127.0.0.1") above binds TCP, so its address is AddressInfo.
+  const { port } = server.address() as AddressInfo;
+  return {
+    url: `http://127.0.0.1:${port}/v1`,
+    requests,
+    close: async () => {
+      await new Promise<void>((resolve) => server.close(() => resolve()));
+    },
+  };
+}
+
+const okCompletion = {
+  choices: [{ message: { content: "done" } }],
+  usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+};
+
+const responseFormat = {
+  type: "json_schema",
+  json_schema: {
+    name: "result",
+    schema: { type: "object", properties: { ok: { type: "boolean" } } },
+  },
+};
+
+test("forwards json_schema response_format when the backend accepts it", async () => {
+  const server = await startRecordingServer(() => ({ status: 200, body: okCompletion }));
+  try {
+    const client = new LocalLlmClient(createConfig({ localLlmUrl: server.url }));
+    primeClient(client);
+    const result = await client.chatCompletion([{ role: "user", content: "hello" }], {
+      responseFormat,
+    });
+    assert.equal(result?.content, "done");
+    assert.equal(server.requests.length, 1);
+    assert.deepEqual(server.requests[0].body.response_format, responseFormat);
+  } finally {
+    await server.close();
+  }
+});
+
+test("400 rejecting json_schema triggers exactly one retry without the format", async (t) => {
+  const server = await startRecordingServer((_req, count) =>
+    count === 1
+      ? {
+          status: 400,
+          body: {
+            error: { message: "response_format type json_schema is not supported" },
+          },
+        }
+      : { status: 200, body: okCompletion },
+  );
+  try {
+    const client = new LocalLlmClient(createConfig({ localLlmUrl: server.url }));
+    primeClient(client);
+    const post = ChatTransport.prototype.post;
+    const attempts: Array<Parameters<ChatTransport["post"]>[0]> = [];
+    t.mock.method(ChatTransport.prototype, "post", async function (this: ChatTransport, request: Parameters<ChatTransport["post"]>[0]) {
+      attempts.push(request);
+      return post.call(this, request);
+    });
+    const result = await client.chatCompletion([{ role: "user", content: "hello" }], {
+      responseFormat,
+    });
+    assert.equal(result?.content, "done");
+    assert.equal(server.requests.length, 2, "exactly one strip-and-retry");
+    assert.deepEqual(server.requests[0].body.response_format, responseFormat);
+    assert.equal(server.requests[1].body.response_format, undefined);
+    // headers/model/deadline preserved on the retry
+    assert.equal(server.requests[1].headers["x-test"], "yes");
+    assert.equal(server.requests[1].body.model, "test-local-model");
+    assert.equal(attempts[0].signal, attempts[1].signal, "retry shares the attempt deadline and caller abort");
+    assert.equal(attempts[0].budgetMs, attempts[1].budgetMs);
+    assert.deepEqual(attempts[0].headers, attempts[1].headers);
+  } finally {
+    await server.close();
+  }
+});
+
+test("timeout before strip-retry headers uses the configured transport retry", async (t) => {
+  const bodies: RecordedRequest["body"][] = [];
+  t.mock.method(ChatTransport.prototype, "post", async (request: Parameters<ChatTransport["post"]>[0]) => {
+    bodies.push(JSON.parse(request.body));
+    if (bodies.length === 1) return new Response("json_schema is not supported", { status: 400 });
+    if (bodies.length === 2) throw new DOMException("Timed out before headers", "AbortError");
+    return new Response(JSON.stringify(okCompletion), { status: 200 });
+  });
+  const client = new LocalLlmClient(createConfig({ localLlmRetry5xxCount: 1, localLlm400TripThreshold: 1 }));
+  primeClient(client);
+  const result = await client.chatCompletion([{ role: "user", content: "hello" }], { responseFormat });
+  assert.equal(result?.content, "done");
+  assert.equal(bodies.length, 3);
+  assert.deepEqual(bodies[0].response_format, responseFormat);
+  assert.equal(bodies[1].response_format, undefined);
+  assert.equal(bodies[2].response_format, undefined);
+});
+
+test("a diagnostic echo of the format is not a schema capability rejection", async () => {
+  for (const message of [
+    "Context length exceeded; request included response_format json_schema",
+    "Unsupported parameter temperature; request included response_format json_schema",
+  ]) {
+    const server = await startRecordingServer(() => ({ status: 400, body: { error: { message } } }));
+    try {
+      const client = new LocalLlmClient(createConfig({ localLlmUrl: server.url }));
+      primeClient(client);
+      assert.equal(await client.chatCompletion([{ role: "user", content: "hello" }], { responseFormat }), null);
+      assert.equal(server.requests.length, 1, "unrelated provider errors must not strip the format");
+    } finally {
+      await server.close();
+    }
+  }
+});
+
+test("an explicit unknown response_format field retries without the format", async () => {
+  const server = await startRecordingServer((_req, count) => count === 1
+    ? { status: 400, body: { error: { message: "Unknown field response_format" } } }
+    : { status: 200, body: okCompletion });
+  try {
+    const client = new LocalLlmClient(createConfig({ localLlmUrl: server.url }));
+    primeClient(client);
+    assert.equal((await client.chatCompletion([{ role: "user", content: "hello" }], { responseFormat }))?.content, "done");
+    assert.equal(server.requests.length, 2);
+    assert.deepEqual(server.requests[0].body.response_format, responseFormat);
+    assert.equal(server.requests[1].body.response_format, undefined);
+  } finally {
+    await server.close();
+  }
+});
+
+test("format removal persists for later 5xx attempts", async () => {
+  const server = await startRecordingServer((_req, count) =>
+    count === 1
+      ? { status: 400, body: { error: { message: "json_schema unsupported by backend" } } }
+      : count === 2
+        ? { status: 500, body: { error: { message: "boom" } } }
+        : { status: 200, body: okCompletion },
+  );
+  try {
+    const client = new LocalLlmClient(
+      createConfig({ localLlmUrl: server.url, localLlmRetry5xxCount: 2 }),
+    );
+    primeClient(client);
+    const result = await client.chatCompletion([{ role: "user", content: "hello" }], {
+      responseFormat,
+    });
+    assert.equal(result?.content, "done");
+    assert.equal(server.requests.length, 3);
+    assert.equal(server.requests[2].body.response_format, undefined);
+  } finally {
+    await server.close();
+  }
+});
+
+test("unrelated 400 and 429 are not retried with a stripped format", async () => {
+  for (const status of [400, 429]) {
+    const server = await startRecordingServer(() => ({
+      status,
+      body: { error: { message: "some other problem" } },
+    }));
+    try {
+      const client = new LocalLlmClient(createConfig({ localLlmUrl: server.url }));
+      primeClient(client);
+      const result = await client.chatCompletion([{ role: "user", content: "hello" }], {
+        responseFormat,
+      });
+      assert.equal(result, null);
+      assert.equal(server.requests.length, 1);
+      assert.ok(server.requests[0].body.response_format, "format still present");
+    } finally {
+      await server.close();
+    }
+  }
+});
+
+test("caller abort still fails closed during the strip retry", async () => {
+  const controller = new AbortController();
+  const server = await startRecordingServer((_req, count) => {
+    if (count === 2) controller.abort();
+    return count === 1
+      ? { status: 400, body: { error: { message: "response_format json_schema is not supported" } } }
+      : { status: 200, body: okCompletion };
+  });
+  try {
+    const client = new LocalLlmClient(createConfig({ localLlmUrl: server.url }));
+    primeClient(client);
+    const pending = client.chatCompletion([{ role: "user", content: "hello" }], {
+      responseFormat,
+      signal: controller.signal,
+    });
+    const result = await pending;
+    assert.equal(result, null);
+    assert.equal(server.requests.length, 2, "caller abort happens during the strip retry, not before the first request");
+  } finally {
+    await server.close();
+  }
+});
+
+test("request without response_format is unchanged", async () => {
+  const server = await startRecordingServer(() => ({ status: 200, body: okCompletion }));
+  try {
+    const client = new LocalLlmClient(createConfig({ localLlmUrl: server.url }));
+    primeClient(client);
+    const result = await client.chatCompletion([{ role: "user", content: "hello" }]);
+    assert.equal(result?.content, "done");
+    assert.equal(server.requests.length, 1);
+    assert.equal(server.requests[0].body.response_format, undefined);
+  } finally {
+    await server.close();
+  }
+});
+

package/src/local-llm-rate-limit.test.ts

@@ -0,0 +1,406 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { parseConfig } from "./config.js";
+import type { LocalLlmChatCompletionOptions } from "./local-llm-helpers.js";
+import {
+  LocalLlmConcurrencyLimiter,
+  LocalLlmRateLimitSession,
+  computeFullJitterBackoffMs,
+  computeRateLimitDelayMs,
+  endpointBackoffRemainingMs,
+  extendEndpointBackoff,
+  isRateLimitedResponse,
+  parseRetryAfterMs,
+  parseTryAgainHintMs,
+  resetLocalLlmRateLimitStateForTests,
+  resolveRateLimitSettings,
+} from "./local-llm-rate-limit.js";
+import { LocalLlmClient } from "./local-llm.js";
+import type { PluginConfig } from "./types.js";
+
+/**
+ * COS-447: a saturated LiteLLM proxy answered `429 No deployments available`
+ * and the client returned at once, so caller loops re-fired immediately
+ * (2,400+ calls/h). These tests pin backoff, Retry-After, the retry cap, the
+ * process-wide endpoint deadline, and the concurrency cap.
+ */
+
+const LITELLM_429 = JSON.stringify({
+  error: {
+    message:
+      "No deployments available for selected model, Try again in 60 seconds. Passed model=m. pre-call-checks=True, cooldown_list=['a', 'b']",
+  },
+});
+
+type RateLimitTestOverrides = Partial<Record<"localLlmUrl", string>> &
+  Partial<
+    Record<
+      | "localLlmRetry5xxCount"
+      | "localLlmRateLimitRetryCount"
+      | "localLlmRateLimitBackoffMaxMs"
+      | "localLlmMaxConcurrency",
+      number
+    >
+  >;
+
+function createConfig(overrides: RateLimitTestOverrides = {}): PluginConfig {
+  return parseConfig({
+    memoryDir: "/tmp/remnic-cos-447-rate-limit-test",
+    localLlmEnabled: true,
+    localLlmModel: "test-local-model",
+    localLlmUrl: "http://127.0.0.1:1234",
+    localLlmTimeoutMs: 5_000,
+    localLlmRetry5xxCount: 0,
+    localLlmRetryBackoffMs: 1,
+    localLlmAuthHeader: false,
+    localLlmRateLimitRetryCount: 3,
+    localLlmRateLimitBackoffBaseMs: 1,
+    localLlmRateLimitBackoffMaxMs: 20,
+    localLlmMaxConcurrency: 4,
+    ...overrides,
+  });
+}
+
+function ok(content = "ok"): Response {
+  return new Response(JSON.stringify({ choices: [{ message: { content } }] }), {
+    status: 200,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+const DEFAULT_429_BODY = JSON.stringify({ error: { message: "No deployments available for selected model." } });
+
+function rateLimited(status = 429, headers: Record<string, string> = {}, body = DEFAULT_429_BODY): Response {
+  return new Response(body, { status, headers: { "content-type": "application/json", ...headers } });
+}
+
+function modelsResponse(): Response {
+  return new Response(JSON.stringify({ object: "list", data: [{ id: "test-local-model", object: "model" }] }), {
+    status: 200,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+/** Route chat completions to `chat`; answer availability probes with a model list. */
+async function withFetch(chat: () => Promise<Response>, fn: () => Promise<void>): Promise<void> {
+  const original = globalThis.fetch;
+  globalThis.fetch = async (input: string | URL | Request) => {
+    const url = input instanceof Request ? input.url : String(input);
+    return url.includes("/chat/completions") ? chat() : modelsResponse();
+  };
+  resetLocalLlmRateLimitStateForTests();
+  try {
+    await fn();
+  } finally {
+    globalThis.fetch = original;
+    resetLocalLlmRateLimitStateForTests();
+  }
+}
+
+function failureStatus(diag: LocalLlmChatCompletionOptions["failureDiag"]): number | undefined {
+  const err = diag?.lastError;
+  return err instanceof Error && "status" in err && Number.isInteger(err.status) ? Number(err.status) : undefined;
+}
+
+// ---- pure helpers ----------------------------------------------------------
+
+test("classifies 429/503/408 and LiteLLM cooldown bodies as rate limits, not 400s", () => {
+  assert.equal(isRateLimitedResponse(429, ""), true);
+  assert.equal(isRateLimitedResponse(503, ""), true);
+  assert.equal(isRateLimitedResponse(408, ""), true);
+  assert.equal(isRateLimitedResponse(500, "No deployments available for selected model"), true);
+  assert.equal(isRateLimitedResponse(500, "internal error"), false);
+  assert.equal(isRateLimitedResponse(400, "Context Window exceeded ... fallbacks cooldown_list"), false);
+  assert.equal(isRateLimitedResponse(502, ""), false);
+});
+
+test("parses Retry-After as delta-seconds or HTTP-date", () => {
+  assert.equal(parseRetryAfterMs("5"), 5_000);
+  assert.equal(parseRetryAfterMs(" 1.5 "), 1_500);
+  const now = Date.parse("2026-10-26T00:00:00Z");
+  assert.equal(parseRetryAfterMs("Mon, 26 Oct 2026 00:00:30 GMT", now), 30_000);
+  assert.equal(parseRetryAfterMs("Sun, 25 Oct 2026 23:00:00 GMT", now), 0);
+  assert.equal(parseRetryAfterMs("soon"), null);
+  assert.equal(parseRetryAfterMs(""), null);
+  assert.equal(parseRetryAfterMs(null), null);
+  assert.equal(parseRetryAfterMs("999999"), 10 * 60_000, "clamped to the sanity bound");
+});
+
+test("parses LiteLLM 'Try again in N seconds' body hint", () => {
+  assert.equal(parseTryAgainHintMs(LITELLM_429), 60_000);
+  assert.equal(parseTryAgainHintMs("try again in 250ms"), 250);
+  assert.equal(parseTryAgainHintMs("Try again in 2 minutes"), 120_000);
+  assert.equal(parseTryAgainHintMs("busy"), null);
+});
+
+test("full-jitter backoff grows exponentially and is capped", () => {
+  const max = () => 0.999999;
+  assert.equal(computeFullJitterBackoffMs(1, 100, 10_000, max), 99);
+  assert.equal(computeFullJitterBackoffMs(2, 100, 10_000, max), 199);
+  assert.equal(computeFullJitterBackoffMs(4, 100, 10_000, max), 799);
+  assert.equal(computeFullJitterBackoffMs(20, 100, 10_000, max), 9_999);
+  assert.equal(
+    computeFullJitterBackoffMs(3, 100, 10_000, () => 0),
+    0
+  );
+  for (let i = 0; i < 200; i += 1) {
+    const v = computeFullJitterBackoffMs(3, 100, 10_000);
+    assert.ok(v >= 0 && v < 400);
+  }
+});
+
+test("server hint is a floor on the backoff delay", () => {
+  const settings = { backoffBaseMs: 100, backoffMaxMs: 1_000 };
+  assert.equal(
+    computeRateLimitDelayMs(1, settings, 5_000, () => 0.5),
+    5_000
+  );
+  assert.equal(
+    computeRateLimitDelayMs(4, settings, 10, () => 0.5),
+    400
+  );
+  assert.equal(
+    computeRateLimitDelayMs(4, settings, null, () => 0.5),
+    400
+  );
+});
+
+test("settings default sanely and reject invalid values", () => {
+  assert.deepEqual(resolveRateLimitSettings({}), {
+    retryCount: 3,
+    backoffBaseMs: 1_000,
+    backoffMaxMs: 60_000,
+    maxConcurrency: 4,
+  });
+  const s = resolveRateLimitSettings({
+    localLlmRateLimitRetryCount: -1,
+    localLlmRateLimitBackoffBaseMs: 5_000,
+    localLlmRateLimitBackoffMaxMs: 10,
+    localLlmMaxConcurrency: 0,
+  });
+  assert.equal(s.retryCount, 3);
+  assert.equal(s.backoffMaxMs, 5_000, "max never below base");
+  assert.equal(s.maxConcurrency, 0);
+});
+
+test("limiter caps concurrency, serves high priority first, and drops aborted waiters", async () => {
+  const limiter = new LocalLlmConcurrencyLimiter(1);
+  const first = await limiter.acquire();
+  assert.ok(first);
+  const order: string[] = [];
+  const low = limiter.acquire().then((r) => {
+    order.push("low");
+    return r;
+  });
+  const aborter = new AbortController();
+  const aborted = limiter.acquire({ signal: aborter.signal });
+  const high = limiter.acquire({ highPriority: true }).then((r) => {
+    order.push("high");
+    return r;
+  });
+  assert.equal(limiter.queued, 3);
+  aborter.abort();
+  assert.equal(await aborted, null);
+  assert.equal(limiter.queued, 2);
+  first();
+  const highRelease = await high;
+  assert.deepEqual(order, ["high"]);
+  assert.equal(limiter.inFlight, 1);
+  highRelease?.();
+  (await low)?.();
+  assert.deepEqual(order, ["high", "low"]);
+  assert.equal(limiter.inFlight, 0);
+});
+
+// ---- client integration ----------------------------------------------------
+
+test("429 is retried with backoff and then succeeds", async () => {
+  let calls = 0;
+  await withFetch(
+    async () => {
+      calls += 1;
+      return calls === 1 ? rateLimited(429) : ok("after backoff");
+    },
+    async () => {
+      const client = new LocalLlmClient(createConfig({ localLlmUrl: "http://127.0.0.1:1001" }));
+      const result = await client.chatCompletion([{ role: "user", content: "x" }], { operation: "lcm-summarize" });
+      assert.equal(result?.content, "after backoff");
+      assert.equal(calls, 2);
+    }
+  );
+});
+
+test("Retry-After header delays the retry", async () => {
+  const stamps: number[] = [];
+  await withFetch(
+    async () => {
+      stamps.push(Date.now());
+      return stamps.length === 1 ? rateLimited(503, { "retry-after": "1" }) : ok();
+    },
+    async () => {
+      const client = new LocalLlmClient(
+        createConfig({ localLlmUrl: "http://127.0.0.1:1002", localLlmRateLimitBackoffMaxMs: 5_000 })
+      );
+      const result = await client.chatCompletion([{ role: "user", content: "x" }]);
+      assert.equal(result?.content, "ok");
+      assert.equal(stamps.length, 2);
+      assert.ok(stamps[1] - stamps[0] >= 950, `waited ${stamps[1] - stamps[0]}ms, expected >= 1000ms`);
+    }
+  );
+});
+
+test("rate-limit retries are capped and report status 429", async () => {
+  let calls = 0;
+  await withFetch(
+    async () => {
+      calls += 1;
+      return rateLimited(429);
+    },
+    async () => {
+      const client = new LocalLlmClient(
+        createConfig({ localLlmUrl: "http://127.0.0.1:1003", localLlmRateLimitRetryCount: 2 })
+      );
+      const options: LocalLlmChatCompletionOptions = { failureDiag: {} };
+      const result = await client.chatCompletion([{ role: "user", content: "x" }], options);
+      assert.equal(result, null);
+      assert.equal(calls, 3, "one initial attempt + 2 rate-limit retries");
+      assert.equal(failureStatus(options.failureDiag), 429);
+    }
+  );
+});
+
+test("a server hint beyond the max skips the retry and gates every client on that endpoint", async () => {
+  let calls = 0;
+  await withFetch(
+    async () => {
+      calls += 1;
+      return rateLimited(429, {}, LITELLM_429);
+    },
+    async () => {
+      const url = "http://127.0.0.1:1004";
+      const a = new LocalLlmClient(createConfig({ localLlmUrl: url }));
+      const b = new LocalLlmClient(createConfig({ localLlmUrl: url }));
+      assert.equal(await a.chatCompletion([{ role: "user", content: "x" }]), null);
+      assert.equal(calls, 1, "60s hint > 20ms max: no immediate retry");
+      assert.ok(endpointBackoffRemainingMs(`${url}/v1/chat/completions`) > 50_000);
+      const options: LocalLlmChatCompletionOptions = { failureDiag: {} };
+      assert.equal(await b.chatCompletion([{ role: "user", content: "y" }], options), null);
+      assert.equal(calls, 1, "second client must not hit the proxy during the shared backoff");
+      assert.equal(failureStatus(options.failureDiag), 429);
+    }
+  );
+});
+
+test("503 rate-limit retries do not consume the 5xx retry budget", async () => {
+  let calls = 0;
+  await withFetch(
+    async () => {
+      calls += 1;
+      return calls < 3 ? rateLimited(503) : ok();
+    },
+    async () => {
+      const client = new LocalLlmClient(
+        createConfig({ localLlmUrl: "http://127.0.0.1:1005", localLlmRetry5xxCount: 0 })
+      );
+      assert.equal((await client.chatCompletion([{ role: "user", content: "x" }]))?.content, "ok");
+      assert.equal(calls, 3);
+    }
+  );
+});
+
+test("400 responses are not treated as rate limits", async () => {
+  let calls = 0;
+  await withFetch(
+    async () => {
+      calls += 1;
+      return new Response(JSON.stringify({ error: { message: "Context Window exceeded" } }), { status: 400 });
+    },
+    async () => {
+      const client = new LocalLlmClient(createConfig({ localLlmUrl: "http://127.0.0.1:1006" }));
+      assert.equal(await client.chatCompletion([{ role: "user", content: "x" }]), null);
+      assert.equal(calls, 1);
+    }
+  );
+});
+
+test("process-wide concurrency cap bounds in-flight requests across clients", async () => {
+  let inFlight = 0;
+  let peak = 0;
+  await withFetch(
+    async () => {
+      inFlight += 1;
+      peak = Math.max(peak, inFlight);
+      await new Promise((r) => setTimeout(r, 25));
+      inFlight -= 1;
+      return ok();
+    },
+    async () => {
+      const config = createConfig({ localLlmUrl: "http://127.0.0.1:1007", localLlmMaxConcurrency: 2 });
+      const clients = [new LocalLlmClient(config), new LocalLlmClient(config), new LocalLlmClient(config)];
+      const results = await Promise.all(
+        Array.from({ length: 7 }, (_, i) => clients[i % 3].chatCompletion([{ role: "user", content: `q${i}` }]))
+      );
+      assert.equal(results.filter((r) => r?.content === "ok").length, 7);
+      assert.equal(peak, 2);
+    }
+  );
+});
+
+test("rate-limit waits never exceed the caller's request timeout", async () => {
+  let calls = 0;
+  await withFetch(
+    async () => {
+      calls += 1;
+      return rateLimited(429, { "retry-after": "2" });
+    },
+    async () => {
+      const client = new LocalLlmClient(
+        createConfig({ localLlmUrl: "http://127.0.0.1:1008", localLlmRateLimitBackoffMaxMs: 5_000 })
+      );
+      const started = Date.now();
+      const options: LocalLlmChatCompletionOptions = { timeoutMs: 300, failureDiag: {} };
+      assert.equal(await client.chatCompletion([{ role: "user", content: "x" }], options), null);
+      assert.equal(await client.chatCompletion([{ role: "user", content: "y" }], { timeoutMs: 300 }), null);
+      assert.ok(Date.now() - started < 1_000, `took ${Date.now() - started}ms despite a 300ms budget`);
+      assert.equal(calls, 1, "a 2s hint beyond the 300ms budget is neither retried nor re-sent");
+      assert.equal(failureStatus(options.failureDiag), 429);
+    }
+  );
+});
+
+test("a queued attempt rechecks a cooldown recorded while it waited for a slot", async () => {
+  await withFetch(
+    async () => ok(),
+    async () => {
+      const endpoint = "http://127.0.0.1:1009/v1/chat/completions";
+      const settings = resolveRateLimitSettings(
+        createConfig({ localLlmMaxConcurrency: 1, localLlmRateLimitBackoffMaxMs: 1_000 })
+      );
+      const holder = await new LocalLlmRateLimitSession(endpoint, settings, false).beforeAttempt();
+      assert.equal(holder.kind, "go");
+      const queued = new LocalLlmRateLimitSession(endpoint, settings, false, 5_000).beforeAttempt();
+      extendEndpointBackoff(endpoint, 60_000);
+      if (holder.kind === "go") holder.release();
+      const gate = await queued;
+      assert.equal(gate.kind, "skip");
+      assert.equal(gate.kind === "skip" ? gate.reason : null, "backoff");
+    }
+  );
+});
+
+test("waiting for a concurrency slot is bounded by the wait budget", async () => {
+  await withFetch(
+    async () => ok(),
+    async () => {
+      const endpoint = "http://127.0.0.1:1010/v1/chat/completions";
+      const settings = resolveRateLimitSettings(createConfig({ localLlmMaxConcurrency: 1 }));
+      const holder = await new LocalLlmRateLimitSession(endpoint, settings, false).beforeAttempt();
+      const started = Date.now();
+      const gate = await new LocalLlmRateLimitSession(endpoint, settings, false, 50).beforeAttempt();
+      assert.equal(gate.kind === "skip" ? gate.reason : gate.kind, "queue");
+      assert.ok(Date.now() - started < 1_000);
+      if (holder.kind === "go") holder.release();
+    }
+  );
+});