deepdebug-local-agent 1.9.0 → 1.10.0

package/test/la-protocol-v2/llm-client.test.js

@@ -0,0 +1,380 @@
+// F3-10 (1): the Local Agent's LLM client against a FAKE OpenAI-compatible server (real HTTP, real SSE). What it proves: the DLP masks before anything
+// leaves, the key travels only in the Authorization header and is never in a body, an audit record, a stat or an error, limits and timeouts and retries
+// behave, errors are codes only, and with the flag off there is no client at all. Not proven here: the real LiteLLM (only a fake server).
+import { test, after } from "node:test";
+import assert from "node:assert/strict";
+import util from "node:util";
+import { createLlmKeyHolder } from "../../src/la-protocol-v2/llm-key/store.js";
+import { createLlmClient, LlmClientError, LLM_ERROR_CODES } from "../../src/la-protocol-v2/llm/client.js";
+import { resolveLlmClientConfig } from "../../src/la-protocol-v2/llm/config.js";
+import { startFakeLlm, closeAllFakes, okJson, okSse, status, mkKey, keySet } from "./helpers/fake-llm.js";
+
+after(closeAllFakes);
+
+const noop = () => {};
+const GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8";
+const PEM = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n-----END RSA PRIVATE KEY-----";
+const ENFORCE = { LOCAL_AGENT_DLP_MODE: "enforce" };
+const TENANT = "3f2b8c1e-5a4d-4e7b-9c21-0a1b2c3d4e5f";
+const AGENT = "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d";
+const JOB = "9d1f6a2c-1111-4222-8333-444455556666";
+const msg = (content, role = "user") => ({ role, content });
+
+async function rig({ behavior, env = ENFORCE, cfg = {}, key = mkKey(), holderOpts = {}, setKey = true } = {}) {
+    const fake = await startFakeLlm(behavior);
+    const holder = createLlmKeyHolder({ log: { warn: noop }, ...holderOpts });
+    if (setKey) assert.equal(holder.set(keySet(key)).ok, true);
+    const config = { ...resolveLlmClientConfig({ LOCAL_AGENT_LLM_CLIENT: "true" }, noop), backoffMs: 0, ...cfg };
+    const client = createLlmClient({ holder: { request: (p, i, o) => holder.request(p, i, o), describe: () => holder.describe(), get held() { return holder.held; } },
+        config, tenantId: TENANT, agentId: AGENT, env, fetchImpl: fake.fetchImpl, sleep: async () => {} });
+    return { fake, holder, client, key, config, close: () => fake.close() };
+}
+
+async function codeOf(p) {
+    try { await p; } catch (e) { assert.ok(e instanceof LlmClientError, "only LlmClientError escapes"); assert.ok(LLM_ERROR_CODES.includes(e.code)); assert.equal(e.message, e.code, "the message is the code, nothing else"); return e.code; }
+    return null;
+}
+
+// ------------------------------------------------------------------------------------------------------ config
+
+test("config: off by default (unset, empty, unknown value -> off, unknown warns); on only with true/1/yes/on", () => {
+    for (const v of [undefined, "", "false", "0", "off", "nope"]) {
+        const warns = [];
+        assert.deepEqual(resolveLlmClientConfig({ LOCAL_AGENT_LLM_CLIENT: v }, (m) => warns.push(m)), { enabled: false });
+        assert.equal(warns.length, v === "nope" ? 1 : 0);
+    }
+    for (const v of ["true", "1", "YES", " on "]) assert.equal(resolveLlmClientConfig({ LOCAL_AGENT_LLM_CLIENT: v }, noop).enabled, true);
+});
+
+test("config: every number is clamped, a bad one falls back to the default (never unlimited); the path must be a plain absolute path", () => {
+    const c = resolveLlmClientConfig({ LOCAL_AGENT_LLM_CLIENT: "true", LOCAL_AGENT_LLM_TIMEOUT_MS: "1", LOCAL_AGENT_LLM_MAX_RETRIES: "99", LOCAL_AGENT_LLM_MAX_PROMPT_BYTES: "abc",
+        LOCAL_AGENT_LLM_MAX_OUTPUT_TOKENS: "0", LOCAL_AGENT_LLM_MAX_CONCURRENT: "1000", LOCAL_AGENT_LLM_MAX_RESPONSE_BYTES: "-1" }, noop);
+    assert.equal(c.timeoutMs, 60_000); assert.equal(c.maxRetries, 2); assert.equal(c.maxPromptBytes, 256 * 1024); assert.equal(c.maxOutputTokens, 4096);
+    assert.equal(c.maxConcurrent, 4); assert.equal(c.maxResponseBytes, 1024 * 1024); assert.equal(c.tags, true); assert.equal(c.path, "/v1/chat/completions");
+    const warns = [];
+    for (const bad of ["v1/chat", "/../admin", "/a b", "//x", "https://evil/x", "/a?b=1"]) {
+        assert.equal(resolveLlmClientConfig({ LOCAL_AGENT_LLM_CLIENT: "true", LOCAL_AGENT_LLM_PATH: bad }, (m) => warns.push(m)).path, "/v1/chat/completions", bad);
+    }
+    assert.equal(warns.length, 6);
+    assert.equal(resolveLlmClientConfig({ LOCAL_AGENT_LLM_CLIENT: "true", LOCAL_AGENT_LLM_PATH: "/chat/completions", LOCAL_AGENT_LLM_TAGS: "false" }, noop).tags, false);
+});
+
+// ------------------------------------------------------------------------------------------------------ the call
+
+test("chat: OpenAI-compatible request (model, masked messages, max_tokens, tags), key ONLY in the Authorization header, usage and cost parsed", async () => {
+    const r = await rig({ behavior: okJson("a summary", { promptTokens: 30, completionTokens: 9, cost: "0.0123" }) });
+    const out = await r.client.chat({ model: "vh-claude-sonnet", messages: [msg("be brief", "system"), msg("what is this?")], maxTokens: 100, temperature: 0, job: { jobId: JOB, agent: "smoke" } });
+    assert.equal(out.text, "a summary");
+    assert.deepEqual(out.usage, { promptTokens: 30, completionTokens: 9, estimated: false });
+    assert.equal(out.costUsd, 0.0123);
+    assert.equal(out.finishReason, "stop");
+    assert.equal(out.attempts, 1);
+    const q = r.fake.requests[0];
+    assert.equal(q.method, "POST");
+    assert.equal(q.url, "/litellm/v1/chat/completions", "endpoint of the key + the configured path");
+    assert.equal(q.headers.authorization, `Bearer ${r.key}`);
+    assert.equal(q.headers["content-type"], "application/json");
+    assert.deepEqual(q.body, { model: "vh-claude-sonnet", messages: [msg("be brief", "system"), msg("what is this?")], max_tokens: 100, stream: false, temperature: 0,
+        metadata: { tags: [`vh-tenant:${TENANT}`, `vh-la:${AGENT}`, `vh-job:${JOB}`, "vh-agent:smoke"] } });
+    assert.ok(!q.raw.includes(r.key), "the key is never in a body");
+    await r.close();
+});
+
+test("chat: tags off (LOCAL_AGENT_LLM_TAGS=false) = no metadata; the output is capped by LOCAL_AGENT_LLM_MAX_OUTPUT_TOKENS", async () => {
+    const r = await rig({ cfg: { tags: false, maxOutputTokens: 50 } });
+    await r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")], maxTokens: 99_999 });
+    assert.equal(r.fake.requests[0].body.metadata, undefined);
+    assert.equal(r.fake.requests[0].body.max_tokens, 50);
+    await r.close();
+});
+
+test("chat: without usage in the answer the usage is ESTIMATED from the bytes (a budget must still count)", async () => {
+    const r = await rig({ behavior: (q, s) => { s.writeHead(200, { "content-type": "application/json" }); s.end(JSON.stringify({ choices: [{ message: { content: "x".repeat(300) } }] })); } });
+    const out = await r.client.chat({ model: "vh-claude-sonnet", messages: [msg("y".repeat(90))] });
+    assert.deepEqual(out.usage, { promptTokens: 30, completionTokens: 100, estimated: true });
+    assert.equal(out.costUsd, null);
+    await r.close();
+});
+
+test("chat: streaming - deltas in order, usage from the last chunk (stream_options.include_usage asked), text assembled", async () => {
+    const r = await rig({ behavior: okSse(["Hel", "lo ", "wor", "ld"], { promptTokens: 21, completionTokens: 4 }) });
+    const seen = [];
+    const out = await r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")], stream: true, onDelta: (d) => seen.push(d) });
+    assert.deepEqual(seen, ["Hel", "lo ", "wor", "ld"]);
+    assert.equal(out.text, "Hello world");
+    assert.deepEqual(out.usage, { promptTokens: 21, completionTokens: 4, estimated: false });
+    assert.equal(r.fake.requests[0].body.stream, true);
+    assert.deepEqual(r.fake.requests[0].body.stream_options, { include_usage: true });
+    assert.equal(r.fake.requests[0].headers.accept, "text/event-stream");
+    await r.close();
+});
+
+test("chat: streaming - a consumer that throws in onDelta does not break the call; a half line / CRLF framing still parses", async () => {
+    const r = await rig({ behavior: (q, s) => {
+        s.writeHead(200, { "content-type": "text/event-stream" });
+        s.write('data: {"choices":[{"delta":{"content":"ab"}}]}\r\n\r\nda');
+        setTimeout(() => { s.write('ta: {"choices":[{"delta":{"content":"cd"}}]}\r\n\r\n'); s.end("data: [DONE]\n\n"); }, 20);
+    } });
+    const out = await r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")], stream: true, onDelta: () => { throw new Error("consumer bug"); } });
+    assert.equal(out.text, "abcd");
+    assert.equal(out.usage.estimated, true);
+    await r.close();
+});
+
+// ------------------------------------------------------------------------------------------------------ DLP
+
+test("DLP: a secret in ANY message is masked BEFORE the request leaves; the server never sees it; the audit has counts only", async () => {
+    const r = await rig();
+    await r.client.chat({ model: "vh-claude-sonnet", messages: [msg(`use this token ${GH} to push`, "system"), msg(`and again ${GH}`)], job: { jobId: JOB } });
+    const sent = r.fake.requests[0].raw;
+    assert.ok(!sent.includes(GH), "the secret never reaches the wire");
+    assert.match(sent, /\[REDACTED:[a-z0-9_]+\]/);
+    const rec = r.client.audit.tail(5).find((a) => a.event === "llm_call");
+    assert.equal(rec.ok, true);
+    assert.ok(Object.values(rec.redactions).reduce((a, b) => a + b, 0) >= 2, "counted per rule");
+    assert.ok(!JSON.stringify(r.client.audit.tail(50)).includes(GH));
+    await r.close();
+});
+
+test("DLP: a hit of a blocked class refuses the call (dlp_blocked) and NOTHING is sent", async () => {
+    const r = await rig({ env: { ...ENFORCE, LOCAL_AGENT_DLP_BLOCK_CLASSES: "private-key" } });
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg(`here: ${PEM}`)] })), "dlp_blocked");
+    assert.equal(r.fake.requests.length, 0);
+    await r.close();
+});
+
+test("DLP: off / warn / unset = dlp_not_enforced (per call, no opt-out) and nothing is sent; an oversize input is dlp_denied", async () => {
+    for (const mode of [undefined, "off", "warn"]) {
+        const r = await rig({ env: mode ? { LOCAL_AGENT_DLP_MODE: mode } : {} });
+        assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg(`x ${GH}`)] })), "dlp_not_enforced", String(mode));
+        assert.equal(r.fake.requests.length, 0);
+        await r.close();
+    }
+    const r = await rig({ env: { ...ENFORCE, LOCAL_AGENT_DLP_MAX_BYTES: "100" } });
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("z".repeat(500))] })), "dlp_denied");
+    assert.equal(r.fake.requests.length, 0);
+    await r.close();
+});
+
+test("DLP: the mode is read at every call (switching it off mid-life closes the door)", async () => {
+    const env = { ...ENFORCE };
+    const r = await rig({ env });
+    await r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] });
+    env.LOCAL_AGENT_DLP_MODE = "off";
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), "dlp_not_enforced");
+    assert.equal(r.fake.requests.length, 1);
+    await r.close();
+});
+
+// ------------------------------------------------------------------------------------------------------ key and model
+
+test("no key / expired key / alias not in the key's models: refused locally with a code, nothing is sent", async () => {
+    let t = 1_700_000_000_000;
+    const r = await rig({ setKey: false, holderOpts: { now: () => t } });
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), "no_key");
+    r.holder.set(keySet(r.key, { expiresAt: new Date(t + 60_000).toISOString() }));
+    assert.equal(await codeOf(r.client.chat({ model: "vh-drael", messages: [msg("hi")] })), "model_not_allowed");
+    t += 120_000;
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), "no_key", "an expired key is dropped by the holder: no key");
+    assert.equal(r.fake.requests.length, 0);
+    await r.close();
+});
+
+test("bad request shapes: model name, roles, non-string content, too many messages, bad temperature -> bad_request, nothing sent", async () => {
+    const r = await rig();
+    const bad = [
+        { model: "x y", messages: [msg("hi")] }, { model: "vh-claude-sonnet", messages: [] }, { model: "vh-claude-sonnet", messages: [{ role: "tool", content: "x" }] },
+        { model: "vh-claude-sonnet", messages: [{ role: "user", content: 42 }] }, { model: "vh-claude-sonnet", messages: Array.from({ length: 65 }, () => msg("x")) },
+        { model: "vh-claude-sonnet", messages: [msg("x")], temperature: 9 }, {},
+    ];
+    for (const o of bad) assert.equal(await codeOf(r.client.chat(o)), "bad_request");
+    assert.equal(r.fake.requests.length, 0);
+    await r.close();
+});
+
+// ------------------------------------------------------------------------------------------------------ errors, retries, timeouts
+
+test("HTTP errors map to codes; 401/403 and other 4xx are NOT retried; the body of an error response is never read or echoed", async () => {
+    const CANARY = "PROMPT-ECHO-CANARY-7731";
+    for (const [st, code, calls] of [[401, "key_rejected", 1], [403, "key_rejected", 1], [400, "bad_request", 1], [404, "bad_request", 1], [413, "prompt_too_large", 1]]) {
+        const r = await rig({ behavior: status(st, JSON.stringify({ error: { message: `you sent ${CANARY}` } })) });
+        let err;
+        try { await r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] }); } catch (e) { err = e; }
+        assert.equal(err.code, code, String(st));
+        assert.equal(r.fake.requests.length, calls, `no retry on ${st}`);
+        assert.ok(!util.inspect(err, { depth: 5 }).includes(CANARY) && !err.stack.includes(CANARY));
+        assert.ok(!JSON.stringify(r.client.audit.tail(10)).includes(CANARY));
+        await r.close();
+    }
+});
+
+test("retries: 5xx / 429 / 408 / dropped connection are retried up to maxRetries (then the code); Retry-After is honoured and capped; success after a retry", async () => {
+    const r = await rig({ behavior: [status(503), status(500), okJson("finally")], cfg: { maxRetries: 2 } });
+    const waits = [];
+    const client = createLlmClient({ holder: { request: (p, i, o) => r.holder.request(p, i, o), describe: () => r.holder.describe(), get held() { return r.holder.held; } },
+        config: { ...r.config, backoffMs: 100 }, tenantId: TENANT, agentId: AGENT, env: ENFORCE, fetchImpl: r.fake.fetchImpl, sleep: async (ms) => { waits.push(ms); } });
+    const out = await client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] });
+    assert.equal(out.text, "finally"); assert.equal(out.attempts, 3); assert.deepEqual(waits, [100, 200], "exponential backoff");
+    assert.equal(client.stats.retries, 2);
+    await r.close();
+
+    const r2 = await rig({ behavior: status(500), cfg: { maxRetries: 2 } });
+    assert.equal(await codeOf(r2.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), "upstream_error");
+    assert.equal(r2.fake.requests.length, 3, "1 + maxRetries, never more");
+    await r2.close();
+
+    const r3 = await rig({ behavior: [(q, s) => { s.writeHead(429, { "retry-after": "999" }); s.end(); }, okJson("ok")] });
+    const w3 = [];
+    const c3 = createLlmClient({ holder: { request: (p, i, o) => r3.holder.request(p, i, o), describe: () => r3.holder.describe(), get held() { return r3.holder.held; } },
+        config: r3.config, tenantId: TENANT, agentId: AGENT, env: ENFORCE, fetchImpl: r3.fake.fetchImpl, sleep: async (ms) => { w3.push(ms); } });
+    assert.equal((await c3.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })).text, "ok");
+    assert.deepEqual(w3, [10_000], "Retry-After capped");
+    await r3.close();
+
+    const r4 = await rig({ behavior: [(q, s) => s.destroy(), (q, s) => s.destroy(), (q, s) => s.destroy()], cfg: { maxRetries: 2 } });
+    assert.equal(await codeOf(r4.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), "network_error");
+    assert.equal(r4.fake.requests.length, 3);
+    await r4.close();
+
+    const r5 = await rig({ behavior: [status(408), status(429), okJson("fine")], cfg: { maxRetries: 2 } });
+    assert.equal((await r5.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })).text, "fine");
+    await r5.close();
+});
+
+test("timeouts: a server that never answers -> timeout (retried, then the code); the total deadline stops the retries", async () => {
+    const r = await rig({ behavior: () => new Promise(() => {}), cfg: { timeoutMs: 80, maxRetries: 1 } });
+    const t0 = Date.now();
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), "timeout");
+    assert.equal(r.fake.requests.length, 2);
+    assert.ok(Date.now() - t0 < 2000);
+    await r.close();
+    const r2 = await rig({ behavior: () => new Promise(() => {}), cfg: { timeoutMs: 80, maxRetries: 5, totalMs: 100, backoffMs: 60 } });
+    assert.equal(await codeOf(r2.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), "timeout");
+    assert.ok(r2.fake.requests.length <= 2, "the deadline of the whole call cut the retries");
+    await r2.close();
+});
+
+test("streaming: idle timeout between chunks -> timeout; no retry once a delta was delivered (it would repeat output)", async () => {
+    const stall = (q, s) => { s.writeHead(200, { "content-type": "text/event-stream" }); s.write('data: {"choices":[{"delta":{"content":"par"}}]}\n\n'); /* then silence */ };
+    const r = await rig({ behavior: stall, cfg: { streamIdleMs: 80, maxRetries: 3 } });
+    const seen = [];
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")], stream: true, onDelta: (d) => seen.push(d) })), "timeout");
+    assert.deepEqual(seen, ["par"]);
+    assert.equal(r.fake.requests.length, 1, "no retry after output was delivered");
+    await r.close();
+});
+
+test("a redirect is refused (a 3xx would carry the key elsewhere): network_error, and the second host is never called", async () => {
+    const other = await startFakeLlm(okJson("leak"));
+    const r = await rig({ behavior: (q, s) => { s.writeHead(302, { location: `http://127.0.0.1:${other.port}/steal` }); s.end(); }, cfg: { maxRetries: 0 } });
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), "network_error");
+    assert.equal(other.requests.length, 0);
+    await r.close(); await other.close();
+});
+
+test("cancellation: an aborted signal ends the call with `cancelled` (before and during the request), no retry", async () => {
+    const r = await rig({ behavior: () => new Promise(() => {}) });
+    const ac = new AbortController();
+    const p = r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")], signal: ac.signal });
+    setTimeout(() => ac.abort(), 40);
+    assert.equal(await codeOf(p), "cancelled");
+    assert.equal(r.fake.requests.length, 1);
+    const pre = new AbortController(); pre.abort();
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")], signal: pre.signal })), "cancelled");
+    assert.equal(r.fake.requests.length, 1);
+    assert.equal(r.client.active, 0, "the slot is released");
+    await r.close();
+});
+
+// ------------------------------------------------------------------------------------------------------ limits
+
+test("limits: prompt above maxPromptBytes (after masking) -> prompt_too_large; response above maxResponseBytes -> response_too_large (JSON and SSE); nothing partial returned", async () => {
+    const r = await rig({ behavior: okJson("fine"), cfg: { maxPromptBytes: 1024 } });
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("a".repeat(600)), msg("b".repeat(600))] })), "prompt_too_large");
+    assert.equal(r.fake.requests.length, 0);
+    await r.close();
+    const big = await rig({ behavior: okJson("x".repeat(20_000)), cfg: { maxResponseBytes: 4096 } });
+    assert.equal(await codeOf(big.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), "response_too_large");
+    await big.close();
+    const bigSse = await rig({ behavior: okSse(Array.from({ length: 40 }, () => "y".repeat(500))), cfg: { maxResponseBytes: 4096 } });
+    const seen = [];
+    assert.equal(await codeOf(bigSse.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")], stream: true, onDelta: (d) => seen.push(d) })), "response_too_large");
+    assert.equal(bigSse.fake.requests.length, 1, "not retried");
+    await bigSse.close();
+});
+
+test("token budget: the prompt estimate + max_tokens must fit tokensLeft (max_tokens is shrunk to what is left; none left -> budget_exceeded, nothing sent)", async () => {
+    const r = await rig();
+    await r.client.chat({ model: "vh-claude-sonnet", messages: [msg("x".repeat(30))], maxTokens: 1000, tokensLeft: 60 });
+    assert.equal(r.fake.requests[0].body.max_tokens, 50, "60 left - 10 estimated prompt tokens");
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("x".repeat(30))], tokensLeft: 10 })), "budget_exceeded");
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("x")], tokensLeft: 0 })), "budget_exceeded");
+    assert.equal(r.fake.requests.length, 1);
+    await r.close();
+});
+
+test("concurrency: above maxConcurrent in flight -> busy at once (no queue, no unbounded fan-out); the slot is released after each call", async () => {
+    let release;
+    const gate = new Promise((r) => { release = r; });
+    const r = await rig({ behavior: async (q, s, e) => { await gate; okJson("late")(q, s, e); }, cfg: { maxConcurrent: 2 } });
+    const a = r.client.chat({ model: "vh-claude-sonnet", messages: [msg("1")] });
+    const b = r.client.chat({ model: "vh-claude-sonnet", messages: [msg("2")] });
+    await new Promise((x) => setTimeout(x, 30));
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("3")] })), "busy");
+    release();
+    await Promise.all([a, b]);
+    assert.equal(r.client.active, 0);
+    await r.client.chat({ model: "vh-claude-sonnet", messages: [msg("4")] });
+    await r.close();
+});
+
+test("bad responses: not JSON, no choices, an error event in the stream -> bad_response / upstream_error (codes only)", async () => {
+    for (const [behavior, code] of [
+        [(q, s) => { s.writeHead(200); s.end("<html>nope</html>"); }, "bad_response"],
+        [(q, s) => { s.writeHead(200); s.end(JSON.stringify({ choices: [] })); }, "bad_response"],
+        [(q, s) => { s.writeHead(200); s.end(JSON.stringify({ choices: [{ message: { content: 5 } }] })); }, "bad_response"],
+    ]) {
+        const r = await rig({ behavior });
+        assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")] })), code);
+        await r.close();
+    }
+    const r = await rig({ behavior: (q, s) => { s.writeHead(200, { "content-type": "text/event-stream" }); s.end('data: {"error":{"message":"boom"}}\n\n'); } });
+    assert.equal(await codeOf(r.client.chat({ model: "vh-claude-sonnet", messages: [msg("hi")], stream: true })), "upstream_error");
+    await r.close();
+});
+
+// ------------------------------------------------------------------------------------------------------ nothing leaks
+
+test("SWEEP: across success, every error and streaming, the key, its id, the prompt and the answer are in NO audit record, stat, error or inspected object", async () => {
+    const key = mkKey("SWEEP");
+    const PROMPT = "PROMPT-CANARY-5521";
+    const ANSWER = "ANSWER-CANARY-8842";
+    const r = await rig({ key, behavior: [okJson(ANSWER), okSse([ANSWER]), status(401), status(500), status(400), (q, s) => s.destroy()], cfg: { maxRetries: 0 } });
+    const errors = [];
+    for (const o of [{}, { stream: true }, {}, {}, {}, {}]) {
+        try { await r.client.chat({ model: "vh-claude-sonnet", messages: [msg(`${PROMPT} ${GH}`)], job: { jobId: JOB }, ...o }); } catch (e) { errors.push(e); }
+    }
+    assert.equal(errors.length, 4);
+    const dump = JSON.stringify([r.client.audit.tail(100), r.client.stats, errors.map((e) => [e.message, e.code, e.stack, util.inspect(e, { depth: 6 })]), util.inspect(r.client, { depth: 6, showHidden: true })]);
+    for (const secret of [key, (await import("./helpers/fake-llm.js")).sha(key), PROMPT, ANSWER, GH]) assert.ok(!dump.includes(secret), `leaked: ${secret.slice(0, 12)}`);
+    for (const q of r.fake.requests) { assert.ok(!q.raw.includes(key) && !q.raw.includes(GH)); assert.equal(q.headers.authorization, `Bearer ${key}`); }
+    for (const rec of r.client.audit.tail(100)) assert.ok(Object.keys(rec).every((k) => ["ts", "event", "ok", "code", "model", "jobShort", "promptBytes", "outBytes", "tokensIn", "tokensOut", "estimated", "attempts", "redactions", "ms", "stream"].includes(k)), `unexpected audit field in ${Object.keys(rec)}`);
+    await r.close();
+});
+
+test("the error codes are a closed list (an unknown code can not be created)", () => {
+    assert.equal(new LlmClientError("whatever text with a secret").code, "internal");
+    assert.equal(new LlmClientError("whatever text with a secret").message, "internal");
+    for (const c of LLM_ERROR_CODES) assert.match(c, /^[a-z_]+$/);
+});
+
+test("holder.matches(ref): true only for the held key's id or its 8..64-hex prefix; a boolean, never the id; false without a key or for a non-hex / short ref", async () => {
+    const r = await rig();
+    const id = (await import("./helpers/fake-llm.js")).sha(r.key);
+    assert.equal(r.holder.matches(id), true);
+    assert.equal(r.holder.matches(id.slice(0, 8)), true);
+    for (const bad of [id.slice(0, 7), id.slice(0, 7) + "z", "0".repeat(64), "", undefined, 5, id.toUpperCase()]) assert.equal(r.holder.matches(bad), false, String(bad));
+    r.holder.revoke(id);
+    assert.equal(r.holder.matches(id), false);
+    await r.close();
+});