deepdebug-local-agent 1.13.0 → 1.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/la-jobs-and-llm.md +1 -1
- package/package.json +1 -1
- package/src/la-protocol-v2/jobs/manager.js +1 -1
- package/src/la-protocol-v2/llm/client.js +21 -1
- package/test/la-protocol-v2/agent2-runner.test.js +3 -1
- package/test/la-protocol-v2/jobs-e2e.test.js +2 -1
- package/test/la-protocol-v2/llm-client.test.js +1 -1
- package/test/la-protocol-v2/llm-messages.test.js +2 -0
- package/test/la-protocol-v2/llm-scope-ids.test.js +129 -0
package/docs/la-jobs-and-llm.md
CHANGED
|
@@ -13,7 +13,7 @@ the gateway's, synced by `scripts/sync-la-protocol-v2-contract.mjs`). The gatewa
|
|
|
13
13
|
- Order: shape -> DLP enforce -> key held and alias in the key's models -> mask EVERY message (blocked class / DLP failure / oversize = refused, nothing sent) -> size and token budget -> concurrency slot -> call.
|
|
14
14
|
- Limits: timeout per attempt, stream idle timeout, total deadline, <= 5 retries (network error, 408, 429, 5xx; `Retry-After` honoured, capped 10 s; never a 4xx, never after a delta was delivered), prompt bytes, output
|
|
15
15
|
tokens, response bytes, concurrent calls (`busy`). Streaming (`stream:true`, `onDelta`) with `stream_options.include_usage`.
|
|
16
|
-
-
|
|
16
|
+
- Scope ids and tags (both routes, chat/completions and /v1/messages): request `metadata` = `vh_tenant_id` (this agent's session tenant), `vh_workspace_id` (the job's workspace, only when it is a plain id) and `tags` = `vh-tenant:<id>`, `vh-ws:<id>`, `vh-la:<agent>`, `vh-job:<id>`, `vh-agent:<runner>`, unique by value (`LOCAL_AGENT_LLM_TAGS=false` omits all of it). vh-litellm's scope-tag guard (enforce) rejects a `vh-tenant:` tag whose `vh_tenant_id` is missing or different (`tenant_tag_mismatch`), so the ids and the tags are always sent together. They never come from agent2 or a job payload. The key also carries `vh-tenant:`/`vh-la:` at the proxy, so LiteLLM sees the tenant tag twice (same value: the guard treats it as one). Whether LiteLLM merges as assumed is NOT verified (fake server with a port of the guard only).
|
|
17
17
|
- Errors are codes only (`LlmClientError.code`, closed list in `client.js`); the body of an error response is never read. Usage comes from `usage`; without it the tokens are estimated (bytes/3) so a budget still counts. Cost from `x-litellm-response-cost` when present.
|
|
18
18
|
- Local audit (ring, optional `LOCAL_AGENT_LLM_AUDIT_FILE`): alias, byte and token counts, per-rule redaction counts, short hash of the job id. Never a prompt, answer or key.
|
|
19
19
|
|
package/package.json
CHANGED
|
@@ -158,7 +158,7 @@ export function createJobManager({ session, tenantId, resolveWorkspace, material
|
|
|
158
158
|
}
|
|
159
159
|
async function llmCall(kind, opts) {
|
|
160
160
|
if (rec.abortReason || rec.finished) throw new LlmClientError("cancelled");
|
|
161
|
-
const r = await llm.client[kind]({ ...opts, model: opts.model ?? rec.job.llm.alias, signal: rec.ac.signal, job: { jobId: rec.jobId, agent: theRunner.name }, tokensLeft: Math.max(0, rec.budget.tokens - tokensUsed(rec)) });
|
|
161
|
+
const r = await llm.client[kind]({ ...opts, model: opts.model ?? rec.job.llm.alias, signal: rec.ac.signal, job: { jobId: rec.jobId, agent: theRunner.name, workspaceId: rec.workspaceId }, tokensLeft: Math.max(0, rec.budget.tokens - tokensUsed(rec)) });
|
|
162
162
|
rec.tokensIn += r.usage.promptTokens;
|
|
163
163
|
rec.tokensOut += r.usage.completionTokens;
|
|
164
164
|
if (r.costUsd !== null && r.costUsd !== undefined) rec.costUsd = (rec.costUsd || 0) + r.costUsd;
|
|
@@ -281,12 +281,32 @@ export function createLlmClient({ holder, config, tenantId, agentId, env = proce
|
|
|
281
281
|
}));
|
|
282
282
|
}
|
|
283
283
|
|
|
284
|
+
const SAFE_ID = /^[A-Za-z0-9_-]{1,64}$/; // same rule as the agents' LiteLlmScopeTags: a crafted id must never smuggle a second tag
|
|
285
|
+
const SAFE_TAG_VALUE = /^[A-Za-z0-9._-]{1,64}$/;
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* The `metadata` of every LiteLLM request (chat/completions AND /v1/messages). vh-litellm's scope-tag guard needs the caller ids IN the body
|
|
289
|
+
* (`vh_tenant_id`, `vh_workspace_id`) and the scope tags to agree with them. The tenant is this agent's own (session); the workspace is the job's
|
|
290
|
+
* (the one this agent serves); nothing here comes from agent2 or a job payload. Tags are unique by value. null = nothing to send.
|
|
291
|
+
*/
|
|
292
|
+
function requestMetadata(job) {
|
|
293
|
+
const md = {};
|
|
294
|
+
const tags = new Set();
|
|
295
|
+
if (SAFE_ID.test(String(tenantId))) { md.vh_tenant_id = String(tenantId); tags.add(`vh-tenant:${tenantId}`); }
|
|
296
|
+
const ws = typeof job?.workspaceId === "string" ? job.workspaceId.trim() : "";
|
|
297
|
+
if (md.vh_tenant_id && SAFE_ID.test(ws)) { md.vh_workspace_id = ws; tags.add(`vh-ws:${ws}`); }
|
|
298
|
+
if (SAFE_TAG_VALUE.test(String(agentId))) tags.add(`vh-la:${agentId}`);
|
|
299
|
+
if (job?.jobId && SAFE_TAG_VALUE.test(String(job.jobId))) tags.add(`vh-job:${job.jobId}`);
|
|
300
|
+
if (job?.agent && /^[a-z0-9][a-z0-9._-]{0,31}$/.test(job.agent)) tags.add(`vh-agent:${job.agent}`);
|
|
301
|
+
return { ...md, tags: [...tags] };
|
|
302
|
+
}
|
|
303
|
+
|
|
284
304
|
/** The slot, the HTTP call with its retries and the bookkeeping, shared by chat() and messages(). `hooks` = { path, parseJson } of the API in use. */
|
|
285
305
|
async function dispatch(o, ctx, hooks, makeBody) {
|
|
286
306
|
if (active >= config.maxConcurrent) return fail("busy", ctx);
|
|
287
307
|
active++;
|
|
288
308
|
try {
|
|
289
|
-
const tags = config.tags ?
|
|
309
|
+
const tags = config.tags ? requestMetadata(o.job) : null;
|
|
290
310
|
const bodyText = makeBody(tags);
|
|
291
311
|
stats.bytesOut += byteLength(bodyText);
|
|
292
312
|
let attempts = 0;
|
|
@@ -84,7 +84,9 @@ test("job end to end: POST /jobs (local workspace, per-job token, options), the
|
|
|
84
84
|
assert.equal(q.body.system[0].cache_control.type, "ephemeral", "prompt-cache markers go through");
|
|
85
85
|
assert.ok(!q.raw.includes(GH), "DLP masked the token before it left");
|
|
86
86
|
assert.ok(!q.raw.includes("vh_tier") && !q.headers["x-litellm-tags"], "agent2's own metadata and tags are not forwarded: the tags are the LA's");
|
|
87
|
-
assert.deepEqual(q.body.metadata.tags, [`vh-tenant:${TENANT}`, `vh-la:${r.link.session.agentId}`, `vh-job:${a.p.jobId}`, "vh-agent:agent2"]);
|
|
87
|
+
assert.deepEqual(q.body.metadata.tags, [`vh-tenant:${TENANT}`, "vh-ws:ws-1", `vh-la:${r.link.session.agentId}`, `vh-job:${a.p.jobId}`, "vh-agent:agent2"]);
|
|
88
|
+
assert.equal(q.body.metadata.vh_tenant_id, TENANT);
|
|
89
|
+
assert.equal(q.body.metadata.vh_workspace_id, "ws-1");
|
|
88
90
|
assert.equal(job.record.llm.json.model, "vh-claude-sonnet", "the upstream model name is not handed to agent2");
|
|
89
91
|
// what the control plane saw
|
|
90
92
|
const evs = r.frames(a.p.jobId, "job.event");
|
|
@@ -120,7 +120,8 @@ test("smoke job: ack accepted -> job.event running -> LLM call (file masked, key
|
|
|
120
120
|
const q = r.fake.requests[0];
|
|
121
121
|
assert.equal(q.headers.authorization, `Bearer ${r.key}`);
|
|
122
122
|
assert.equal(q.body.model, "vh-claude-sonnet");
|
|
123
|
-
assert.
|
|
123
|
+
assert.equal(q.body.metadata.vh_workspace_id, "ws-1");
|
|
124
|
+
assert.deepEqual(q.body.metadata.tags, [`vh-tenant:${TENANT}`, "vh-ws:ws-1", `vh-la:${AGENT}`, `vh-job:${a.p.jobId}`, "vh-agent:smoke"]);
|
|
124
125
|
const prompt = q.body.messages.map((m) => m.content).join("\n");
|
|
125
126
|
assert.ok(prompt.includes(SMOKE_QUESTION) && prompt.includes(FILE_CANARY), "the (allowed) file is the subject of the fixed question");
|
|
126
127
|
assert.ok(!q.raw.includes(GH), "the token inside the file was masked BEFORE leaving");
|
|
@@ -75,7 +75,7 @@ test("chat: OpenAI-compatible request (model, masked messages, max_tokens, tags)
|
|
|
75
75
|
assert.equal(q.headers.authorization, `Bearer ${r.key}`);
|
|
76
76
|
assert.equal(q.headers["content-type"], "application/json");
|
|
77
77
|
assert.deepEqual(q.body, { model: "vh-claude-sonnet", messages: [msg("be brief", "system"), msg("what is this?")], max_tokens: 100, stream: false, temperature: 0,
|
|
78
|
-
metadata: { tags: [`vh-tenant:${TENANT}`, `vh-la:${AGENT}`, `vh-job:${JOB}`, "vh-agent:smoke"] } });
|
|
78
|
+
metadata: { vh_tenant_id: TENANT, tags: [`vh-tenant:${TENANT}`, `vh-la:${AGENT}`, `vh-job:${JOB}`, "vh-agent:smoke"] } });
|
|
79
79
|
assert.ok(!q.raw.includes(r.key), "the key is never in a body");
|
|
80
80
|
await r.close();
|
|
81
81
|
});
|
|
@@ -41,6 +41,8 @@ test("messages(): the request is /v1/messages with the key in the header, tags i
|
|
|
41
41
|
assert.ok(!q.raw.includes(r.key));
|
|
42
42
|
assert.deepEqual([q.body.model, q.body.max_tokens, q.body.temperature, q.body.tool_choice], ["vh-claude-sonnet", 100, 0, { type: "auto" }]);
|
|
43
43
|
assert.deepEqual(q.body.system, [{ type: "text", text: "sys", cache_control: { type: "ephemeral" } }]);
|
|
44
|
+
assert.equal(q.body.metadata.vh_tenant_id, TENANT);
|
|
45
|
+
assert.equal(q.body.metadata.vh_workspace_id, undefined, "no workspace given: none claimed");
|
|
44
46
|
assert.deepEqual(q.body.metadata.tags, [`vh-tenant:${TENANT}`, `vh-la:${AGENT}`, `vh-job:${JOB}`, "vh-agent:agent2"]);
|
|
45
47
|
assert.deepEqual(out.usage, { promptTokens: 10, completionTokens: 4, estimated: false });
|
|
46
48
|
assert.equal(out.costUsd, 0.002);
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
// QA 2026-11-05: vh-litellm's scope-tag guard (enforce) rejected every Local Agent chat call with tenant_tag_mismatch: the LA sent the tag vh-tenant:<id>
|
|
2
|
+
// but NOT vh_tenant_id / vh_workspace_id in the body. `guard()` below is a JS port of vh_plugins/scope_tags.py (collect + check); the fake gateway applies it to
|
|
3
|
+
// what really arrives, on both routes. Not proven here: the real LiteLLM (how it merges key tags / request tags / litellm_metadata).
|
|
4
|
+
import { test, after } from "node:test";
|
|
5
|
+
import assert from "node:assert/strict";
|
|
6
|
+
import { createLlmKeyHolder } from "../../src/la-protocol-v2/llm-key/store.js";
|
|
7
|
+
import { createLlmClient } from "../../src/la-protocol-v2/llm/client.js";
|
|
8
|
+
import { resolveLlmClientConfig } from "../../src/la-protocol-v2/llm/config.js";
|
|
9
|
+
import { startFakeLlm, closeAllFakes, okJson, okMessages, byPath, mkKey, keySet } from "./helpers/fake-llm.js";
|
|
10
|
+
import { agent2Rig, TENANT, waitFor } from "./helpers/agent2-rig.js";
|
|
11
|
+
|
|
12
|
+
after(closeAllFakes);
|
|
13
|
+
const noop = () => {};
|
|
14
|
+
const ENFORCE = { LOCAL_AGENT_DLP_MODE: "enforce" };
|
|
15
|
+
const AGENT = "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d";
|
|
16
|
+
|
|
17
|
+
// ---- port of vh_plugins/scope_tags.py
|
|
18
|
+
const blank = (v) => v === null || v === undefined || (typeof v === "string" && !v.trim());
|
|
19
|
+
const vals = (tags, prefix) => [...new Set(tags.filter((t) => typeof t === "string" && t.startsWith(prefix)).map((t) => t.slice(prefix.length).trim()))].sort();
|
|
20
|
+
export function check(tags, tenantId, workspaceId) {
|
|
21
|
+
const tt = vals(tags, "vh-tenant:"), wt = vals(tags, "vh-ws:");
|
|
22
|
+
if (tt.length > 1) return "multiple_tenant_tags";
|
|
23
|
+
if (wt.length > 1) return "multiple_workspace_tags";
|
|
24
|
+
if (tt.length && (blank(tenantId) || tt[0] !== String(tenantId).trim())) return "tenant_tag_mismatch";
|
|
25
|
+
if (wt.length && (blank(workspaceId) || wt[0] !== String(workspaceId).trim())) return "workspace_tag_mismatch";
|
|
26
|
+
if (!blank(tenantId) && !tt.length) return "missing_tenant_tag";
|
|
27
|
+
if (!blank(workspaceId) && !wt.length) return "missing_workspace_tag";
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
export function collect(data) {
|
|
31
|
+
const tags = []; const ids = {};
|
|
32
|
+
for (const f of ["metadata", "litellm_metadata"]) {
|
|
33
|
+
const md = data[f];
|
|
34
|
+
if (!md || typeof md !== "object" || Array.isArray(md)) continue;
|
|
35
|
+
if (Array.isArray(md.tags)) tags.push(...md.tags);
|
|
36
|
+
for (const k of ["vh_tenant_id", "vh_workspace_id"]) if (blank(ids[k]) && !blank(md[k])) ids[k] = md[k];
|
|
37
|
+
}
|
|
38
|
+
if (Array.isArray(data.tags)) tags.push(...data.tags);
|
|
39
|
+
return [tags, ids.vh_tenant_id, ids.vh_workspace_id];
|
|
40
|
+
}
|
|
41
|
+
/** What the proxy does for /v1/messages: ids stay in `metadata`, the tags are also present in `litellm_metadata` (merged by the proxy), and the key's own tenant tag is added too (the duplicate seen in QA). */
|
|
42
|
+
const asProxySees = (body, route, keyTags = []) => (route === "messages"
|
|
43
|
+
? { ...body, litellm_metadata: { tags: [...(body.metadata?.tags || []), ...keyTags] } }
|
|
44
|
+
: { ...body, metadata: { ...body.metadata, tags: [...(body.metadata?.tags || []), ...keyTags] } });
|
|
45
|
+
export const guard = (body, route, keyTags) => check(...collect(asProxySees(body, route, keyTags)));
|
|
46
|
+
const KEY_TAGS = [`vh-tenant:${TENANT}`, `vh-la:${AGENT}`, "vh-dpm:x"];
|
|
47
|
+
|
|
48
|
+
async function rig(tenantId = TENANT) {
|
|
49
|
+
const fake = await startFakeLlm(byPath({ chat: okJson("hi"), messages: okMessages("hi") }));
|
|
50
|
+
const key = mkKey();
|
|
51
|
+
const holder = createLlmKeyHolder({ log: { warn: noop } });
|
|
52
|
+
holder.set(keySet(key));
|
|
53
|
+
const config = { ...resolveLlmClientConfig({ LOCAL_AGENT_LLM_CLIENT: "true" }, noop), backoffMs: 0 };
|
|
54
|
+
const client = createLlmClient({ holder: { request: (p, i, o) => holder.request(p, i, o), describe: () => holder.describe(), get held() { return holder.held; } }, config, tenantId, agentId: AGENT, env: ENFORCE, fetchImpl: fake.fetchImpl });
|
|
55
|
+
return { fake, client };
|
|
56
|
+
}
|
|
57
|
+
const msgs = [{ role: "user", content: "go" }];
|
|
58
|
+
|
|
59
|
+
test("the guard port rejects what QA rejected (tag without ids) and accepts the agreeing request", () => {
|
|
60
|
+
assert.equal(check([`vh-tenant:${TENANT}`, `vh-tenant:${TENANT}`], null, null), "tenant_tag_mismatch");
|
|
61
|
+
assert.equal(check([`vh-tenant:${TENANT}`, `vh-tenant:${TENANT}`], TENANT, null), null, "the same tag twice is one value");
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
for (const route of ["chat", "messages"]) {
|
|
65
|
+
test(`${route}: the request carries vh_tenant_id and vh_workspace_id and ONE tag each; the scope guard passes (with the key's tenant tag duplicated on top)`, async () => {
|
|
66
|
+
const r = await rig();
|
|
67
|
+
await r.client[route]({ model: "vh-claude-sonnet", messages: msgs, job: { jobId: "job-1", agent: "agent2", workspaceId: "ws-1" } });
|
|
68
|
+
const body = r.fake.requests[0].body;
|
|
69
|
+
assert.equal(body.metadata.vh_tenant_id, TENANT);
|
|
70
|
+
assert.equal(body.metadata.vh_workspace_id, "ws-1");
|
|
71
|
+
assert.equal(body.metadata.tags.filter((t) => t.startsWith("vh-tenant:")).length, 1);
|
|
72
|
+
assert.equal(body.metadata.tags.filter((t) => t.startsWith("vh-ws:")).length, 1);
|
|
73
|
+
assert.equal(new Set(body.metadata.tags).size, body.metadata.tags.length, "no duplicates");
|
|
74
|
+
assert.equal(guard(body, route, KEY_TAGS), null);
|
|
75
|
+
assert.equal(guard(body, route, []), null);
|
|
76
|
+
// before the fix: tags only, no ids
|
|
77
|
+
const old = { ...body, metadata: { tags: body.metadata.tags } };
|
|
78
|
+
assert.equal(guard(old, route, KEY_TAGS), "tenant_tag_mismatch");
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test(`${route}: no workspace (or an unsafe one): the tenant is claimed, the workspace is not, no ws tag`, async () => {
|
|
82
|
+
const r = await rig();
|
|
83
|
+
await r.client[route]({ model: "vh-claude-sonnet", messages: msgs, job: { jobId: "job-1" } });
|
|
84
|
+
await r.client[route]({ model: "vh-claude-sonnet", messages: msgs, job: { jobId: "job-1", workspaceId: "ws-1,vh-tenant:evil" } });
|
|
85
|
+
for (const q of r.fake.requests) {
|
|
86
|
+
assert.equal(q.body.metadata.vh_tenant_id, TENANT);
|
|
87
|
+
assert.equal(q.body.metadata.vh_workspace_id, undefined);
|
|
88
|
+
assert.ok(!q.body.metadata.tags.some((t) => t.startsWith("vh-ws:") || t.includes("evil")));
|
|
89
|
+
assert.equal(guard(q.body, route, KEY_TAGS), null);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
test("agent2 hostile through the bridge: its metadata, tags and headers (another tenant/workspace) are overwritten by the LA's own, on /v1/messages and /v1/chat/completions", async () => {
|
|
95
|
+
const r = await agent2Rig({ scenario: "stuck" });
|
|
96
|
+
const a = r.assign();
|
|
97
|
+
await a.acked();
|
|
98
|
+
await waitFor(() => r.fake.jobs.size === 1);
|
|
99
|
+
const job = r.fake.jobs.get(a.p.jobId);
|
|
100
|
+
const evil = { vh_tenant_id: "11111111-2222-3333-4444-555555555555", vh_workspace_id: "other-ws", tags: ["vh-tenant:11111111-2222-3333-4444-555555555555", "vh-ws:other-ws", "vh-tenant:" + TENANT] };
|
|
101
|
+
const headers = { authorization: `Bearer ${job.llm.key}`, "content-type": "application/json", "x-litellm-tags": "vh-tenant:11111111-2222-3333-4444-555555555555", "x-vh-tenant-id": "evil" };
|
|
102
|
+
const before = r.fakeLlm.requests.length;
|
|
103
|
+
const m = await fetch(`${r.bridgeBase}/v1/messages`, { method: "POST", headers, body: JSON.stringify({ model: "claude-x", max_tokens: 50, messages: msgs, metadata: evil, litellm_metadata: evil, tags: evil.tags }) });
|
|
104
|
+
const c = await fetch(`${r.bridgeBase}/v1/chat/completions`, { method: "POST", headers, body: JSON.stringify({ model: "gpt-x", messages: msgs, metadata: evil, tags: evil.tags }) });
|
|
105
|
+
assert.equal(m.status, 200); assert.equal(c.status, 200);
|
|
106
|
+
const sent = r.fakeLlm.requests.slice(before);
|
|
107
|
+
assert.equal(sent.length, 2);
|
|
108
|
+
for (const q of sent) {
|
|
109
|
+
const route = /\/messages$/.test(q.url) ? "messages" : "chat";
|
|
110
|
+
assert.equal(q.body.metadata.vh_tenant_id, TENANT);
|
|
111
|
+
assert.equal(q.body.metadata.vh_workspace_id, "ws-1");
|
|
112
|
+
assert.ok(!q.raw.includes("11111111-2222") && !q.raw.includes("other-ws") && !q.raw.includes("evil"), "nothing of agent2's claim leaves");
|
|
113
|
+
assert.equal(q.headers["x-litellm-tags"], undefined);
|
|
114
|
+
assert.equal(q.body.litellm_metadata, undefined);
|
|
115
|
+
assert.equal(q.body.tags, undefined);
|
|
116
|
+
assert.equal(new Set(q.body.metadata.tags).size, q.body.metadata.tags.length);
|
|
117
|
+
assert.equal(guard(q.body, route, KEY_TAGS), null);
|
|
118
|
+
}
|
|
119
|
+
await r.stop();
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("an unsafe tenant id claims nothing and tags nothing (never a smuggled tag)", async () => {
|
|
123
|
+
const r = await rig("t1,vh-tenant:evil");
|
|
124
|
+
await r.client.chat({ model: "vh-claude-sonnet", messages: msgs, job: { jobId: "j", workspaceId: "ws-1" } });
|
|
125
|
+
const md = r.fake.requests[0].body.metadata;
|
|
126
|
+
assert.equal(md.vh_tenant_id, undefined);
|
|
127
|
+
assert.equal(md.vh_workspace_id, undefined);
|
|
128
|
+
assert.ok(!md.tags.some((t) => t.startsWith("vh-tenant:") || t.startsWith("vh-ws:") || t.includes("evil")));
|
|
129
|
+
});
|