deepdebug-local-agent 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CLAUDE.md +1 -1
- package/docs/la-jobs-and-llm.md +56 -0
- package/docs/la-protocol-v2.md +3 -3
- package/env.example +6 -0
- package/package.json +1 -1
- package/src/la-protocol-v2/content-handlers/install.js +27 -1
- package/src/la-protocol-v2/contract/CONTRACT_SOURCE.json +4 -4
- package/src/la-protocol-v2/contract/README.md +25 -1
- package/src/la-protocol-v2/contract/job.assign.schema.json +31 -0
- package/src/la-protocol-v2/jobs/config.js +60 -0
- package/src/la-protocol-v2/jobs/events.js +106 -0
- package/src/la-protocol-v2/jobs/install.js +37 -0
- package/src/la-protocol-v2/jobs/manager.js +276 -0
- package/src/la-protocol-v2/jobs/options.js +56 -0
- package/src/la-protocol-v2/jobs/smoke-runner.js +39 -0
- package/src/la-protocol-v2/llm/client.js +311 -0
- package/src/la-protocol-v2/llm/config.js +61 -0
- package/src/la-protocol-v2/llm/install.js +17 -0
- package/src/la-protocol-v2/llm-key/store.js +10 -0
- package/src/la-protocol-v2/runtime.js +35 -2
- package/test/contract/la-protocol-v2/vectors/job.assign.vectors.json +1538 -0
- package/test/la-protocol-v2/contract.test.js +4 -4
- package/test/la-protocol-v2/helpers/fake-llm.js +64 -0
- package/test/la-protocol-v2/jobs-e2e.test.js +405 -0
- package/test/la-protocol-v2/jobs-manager.test.js +370 -0
- package/test/la-protocol-v2/jobs-options.test.js +288 -0
- package/test/la-protocol-v2/llm-client.test.js +380 -0
|
@@ -82,10 +82,10 @@ for (const file of vectorFiles) {
|
|
|
82
82
|
}
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
-
test("vectors: counts (
|
|
86
|
-
assert.equal(valid + invalid,
|
|
87
|
-
assert.equal(valid,
|
|
88
|
-
assert.equal(invalid,
|
|
85
|
+
test("vectors: counts (317 frames = 72 valid + 245 invalid in this copy (contract 2.1, with job.assign options), plus the bodies of file.list)", () => {
|
|
86
|
+
assert.equal(valid + invalid, 317);
|
|
87
|
+
assert.equal(valid, 72);
|
|
88
|
+
assert.equal(invalid, 245);
|
|
89
89
|
assert.equal(bodyVectors, 54, "file.list.body.vectors.json (F3-07c): 54 bodies");
|
|
90
90
|
});
|
|
91
91
|
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// A fake OpenAI-compatible server (chat/completions, JSON and SSE) on 127.0.0.1, plus the fetch that points the https endpoint of the key at it.
|
|
2
|
+
// It records what it was sent (method, url, headers, parsed body) so the tests can prove what left the agent and what did not.
|
|
3
|
+
import http from "node:http";
|
|
4
|
+
import crypto from "node:crypto";
|
|
5
|
+
|
|
6
|
+
export const sha = (s) => crypto.createHash("sha256").update(Buffer.from(s, "utf8")).digest("hex");
|
|
7
|
+
export const mkKey = (tag = "A") => `sk-CANARY-LLMKEY-${tag}-${crypto.randomBytes(12).toString("hex")}`;
|
|
8
|
+
export const FUTURE = (ms = 3_600_000) => new Date(Date.now() + ms).toISOString();
|
|
9
|
+
export const keySet = (key, over = {}) => ({ op: "set", keyId: sha(key), endpoint: "https://ai.visionhammer.io/litellm", key, models: ["vh-claude-sonnet", "vh-embed"], expiresAt: FUTURE(), ...over });
|
|
10
|
+
|
|
11
|
+
export const okJson = (content = "ok", { promptTokens = 12, completionTokens = 5, cost, model = "vh-claude-sonnet" } = {}) => (req, res) => {
|
|
12
|
+
const body = JSON.stringify({ id: "cmpl-1", object: "chat.completion", model, choices: [{ index: 0, message: { role: "assistant", content }, finish_reason: "stop" }], usage: { prompt_tokens: promptTokens, completion_tokens: completionTokens, total_tokens: promptTokens + completionTokens } });
|
|
13
|
+
res.writeHead(200, { "content-type": "application/json", ...(cost !== undefined ? { "x-litellm-response-cost": String(cost) } : {}) });
|
|
14
|
+
res.end(body);
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
export const okSse = (deltas, { promptTokens = 20, completionTokens = 7, gapMs = 0 } = {}) => async (req, res) => {
|
|
18
|
+
res.writeHead(200, { "content-type": "text/event-stream" });
|
|
19
|
+
for (const d of deltas) {
|
|
20
|
+
res.write(`data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: d } }] })}\n\n`);
|
|
21
|
+
if (gapMs) await new Promise((r) => setTimeout(r, gapMs));
|
|
22
|
+
}
|
|
23
|
+
res.write(`data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`);
|
|
24
|
+
res.write(`data: ${JSON.stringify({ choices: [], usage: { prompt_tokens: promptTokens, completion_tokens: completionTokens } })}\n\n`);
|
|
25
|
+
res.end("data: [DONE]\n\n");
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export const status = (code, body = "") => (req, res) => { res.writeHead(code, { "content-type": "application/json" }); res.end(body); };
|
|
29
|
+
|
|
30
|
+
/** @param {Function|Function[]} behavior one handler for every request, or a list used in order (the last one repeats) */
|
|
31
|
+
const OPEN = new Set();
|
|
32
|
+
/** Closes every fake server a failed test left open (so a red test never hangs the run). */
|
|
33
|
+
export async function closeAllFakes() { await Promise.all([...OPEN].map((f) => f.close())); }
|
|
34
|
+
|
|
35
|
+
export async function startFakeLlm(behavior = okJson()) {
|
|
36
|
+
const requests = [];
|
|
37
|
+
let list = Array.isArray(behavior) ? behavior : [behavior];
|
|
38
|
+
const sockets = new Set();
|
|
39
|
+
const server = http.createServer((req, res) => {
|
|
40
|
+
const chunks = [];
|
|
41
|
+
req.on("data", (c) => chunks.push(c));
|
|
42
|
+
req.on("end", () => {
|
|
43
|
+
const raw = Buffer.concat(chunks).toString("utf8");
|
|
44
|
+
let body = null;
|
|
45
|
+
try { body = JSON.parse(raw); } catch { /* keep null */ }
|
|
46
|
+
const entry = { method: req.method, url: req.url, headers: req.headers, raw, body };
|
|
47
|
+
requests.push(entry);
|
|
48
|
+
const h = list[Math.min(requests.length - 1, list.length - 1)];
|
|
49
|
+
Promise.resolve(h(req, res, entry)).catch(() => { try { res.destroy(); } catch { /* gone */ } });
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
server.on("connection", (s) => { sockets.add(s); s.on("close", () => sockets.delete(s)); });
|
|
53
|
+
await new Promise((r) => server.listen(0, "127.0.0.1", r));
|
|
54
|
+
const port = server.address().port;
|
|
55
|
+
const fake = {
|
|
56
|
+
port, requests,
|
|
57
|
+
set(b) { list = Array.isArray(b) ? b : [b]; },
|
|
58
|
+
/** the fetch the holder uses: the https origin of the key's endpoint is pointed at this server, everything else is untouched */
|
|
59
|
+
fetchImpl: (url, init) => fetch(String(url).replace(/^https:\/\/[^/]+/, `http://127.0.0.1:${port}`), init),
|
|
60
|
+
async close() { OPEN.delete(fake); for (const s of sockets) s.destroy(); await new Promise((r) => server.close(r)); },
|
|
61
|
+
};
|
|
62
|
+
OPEN.add(fake);
|
|
63
|
+
return fake;
|
|
64
|
+
}
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
// F3-10 end to end, in process: the REAL v2 session (signed frames, schemas, acks) against the ReferenceCp, buildV2Link with every flag, the content
|
|
2
|
+
// registry, the LLM key holder, the LLM client and the smoke runner, talking to a FAKE OpenAI-compatible server. What it proves: job.assign ->
|
|
3
|
+
// ack -> job.event running ... succeeded + job.result, metadata only; the file goes to the LLM masked and the key only in a header; cancel reaches the
|
|
4
|
+
// in-flight LLM call; budget, DLP, file gate and workspace/tenant isolation; flags off = nothing. NOT proven: the real LiteLLM, a real CP, the real
|
|
5
|
+
// gateway dispatch (F3-10 gateway side) - only the reference CP and a fake LLM server.
|
|
6
|
+
import { test, after } from "node:test";
|
|
7
|
+
import assert from "node:assert/strict";
|
|
8
|
+
import fs from "node:fs";
|
|
9
|
+
import { ReferenceCp, TENANT, AGENT, uuid, waitFor } from "./helpers/reference-cp.js";
|
|
10
|
+
import { signingFor } from "./helpers/la-rig.js";
|
|
11
|
+
import { write, tmpDir } from "./helpers/content-rig.js";
|
|
12
|
+
import { startFakeLlm, closeAllFakes, okJson, okSse, status, sha, mkKey, keySet } from "./helpers/fake-llm.js";
|
|
13
|
+
import { buildV2Link } from "../../src/la-protocol-v2/runtime.js";
|
|
14
|
+
import { SMOKE_QUESTION } from "../../src/la-protocol-v2/jobs/smoke-runner.js";
|
|
15
|
+
|
|
16
|
+
after(closeAllFakes);
|
|
17
|
+
|
|
18
|
+
const GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8";
|
|
19
|
+
const FILE_CANARY = "FILE-CANARY-9931";
|
|
20
|
+
const INLINE_CANARY = "TICKET-CANARY-4417";
|
|
21
|
+
const ANSWER = "ANSWER-CANARY-2208 this file prints a greeting.";
|
|
22
|
+
const ALL_ON = { LOCAL_AGENT_DLP_MODE: "enforce", LOCAL_AGENT_CONTENT_HANDLERS: "true", LOCAL_AGENT_LLM_KEY: "true", LOCAL_AGENT_LLM_CLIENT: "true", LOCAL_AGENT_JOBS: "true",
|
|
23
|
+
LOCAL_AGENT_JOBS_RUNNER: "smoke", LOCAL_AGENT_LLM_BACKOFF_MS: "0", LOCAL_AGENT_JOBS_EVENT_MIN_MS: "50", LOCAL_AGENT_JOBS_CANCEL_GRACE_MS: "500" };
|
|
24
|
+
|
|
25
|
+
async function rig({ env = ALL_ON, behavior = okJson(ANSWER, { promptTokens: 40, completionTokens: 11, cost: "0.0042" }), files, tenantWs, setKey = true, jobRunner } = {}) {
|
|
26
|
+
const root = tmpDir("f310-");
|
|
27
|
+
write(root, "src/hello.js", files?.hello ?? `// ${FILE_CANARY}\nconst token = "${GH}";\nconsole.log("hello");\n`);
|
|
28
|
+
write(root, ".env", "DB_PASSWORD=hunter2-CANARY\n");
|
|
29
|
+
const other = tmpDir("f310b-");
|
|
30
|
+
write(other, "src/other.js", "OTHER-WS-CANARY\n");
|
|
31
|
+
const fake = await startFakeLlm(behavior);
|
|
32
|
+
const cp = new ReferenceCp();
|
|
33
|
+
const logs = [];
|
|
34
|
+
const errors = [];
|
|
35
|
+
const log = { info: (m) => logs.push(m), log: (m) => logs.push(m), warn: (m) => logs.push(m), error: (m) => errors.push(m) };
|
|
36
|
+
const wsManager = { list: () => [{ id: "ws-1", root, tenantId: tenantWs ?? TENANT }, { id: "ws-2", root: other, tenantId: TENANT }, { id: "ws-x", root: other, tenantId: "9d1f6a2c-0000-4000-8000-000000000000" }] };
|
|
37
|
+
const results = [];
|
|
38
|
+
const link = await buildV2Link({ tenantId: TENANT, agentId: AGENT, signing: signingFor({ keyId: cp.keyId, publicKeyRaw: cp.publicKeyRaw }, "enforce"), wsManager,
|
|
39
|
+
getWorkspaceRoot: () => root, log, env, llmFetch: fake.fetchImpl, onJobResult: (r) => results.push(r), ...(jobRunner ? { jobRunner } : {}) });
|
|
40
|
+
const transport = { send: (t) => cp.feed(t), close() {}, bufferedAmount: () => 0 };
|
|
41
|
+
cp.attach((t) => queueMicrotask(() => link.session.handleText(t, transport)), () => {});
|
|
42
|
+
link.session.attach(transport);
|
|
43
|
+
await waitFor(() => link.session.state === "ready");
|
|
44
|
+
const key = mkKey("E2E");
|
|
45
|
+
if (setKey && link.llm) {
|
|
46
|
+
cp.sendData("config.update", { llmKey: keySet(key) });
|
|
47
|
+
await waitFor(() => cp.acksFromLa.length >= 1);
|
|
48
|
+
}
|
|
49
|
+
const refOf = (rel, workspaceId = "ws-1") => link.content?.mintRef({ workspaceId, kind: "file", rel })?.id ?? "A".repeat(43);
|
|
50
|
+
const ack0 = () => cp.acksFromLa.length;
|
|
51
|
+
const r = {
|
|
52
|
+
cp, link, fake, root, other, key, logs, errors, results, refOf,
|
|
53
|
+
assign(over = {}, workspaceId = "ws-1") {
|
|
54
|
+
const p = { jobId: uuid(), kind: "quality", repoRef: { provider: "github", org: "acme", repo: "app", branch: "main" }, policy: { operationMode: "assist", automationLevel: 2, ceiling: "assist" },
|
|
55
|
+
budget: { tokens: 50_000, seconds: 60 }, llm: { alias: "vh-claude-sonnet", keyRef: sha(key) }, inputs: [{ ref: refOf("src/hello.js") }], ...over };
|
|
56
|
+
const n = ack0();
|
|
57
|
+
cp.sendData("job.assign", p, { workspaceId });
|
|
58
|
+
return { p, acked: () => waitFor(() => cp.acksFromLa.length > n).then(() => cp.acksFromLa[n]) };
|
|
59
|
+
},
|
|
60
|
+
frames: (jobId, type) => cp.received.filter((f) => (!type || f.type === type) && f.payload.jobId === jobId),
|
|
61
|
+
done: (jobId) => waitFor(() => r.frames(jobId, "job.result").length === 1, 6000),
|
|
62
|
+
everything: () => JSON.stringify([cp.received, logs, errors, link.jobs?.audit.tail(1000), link.llm?.client?.audit.tail(1000), link.llm?.audit.tail(1000), link.llm?.client?.stats, link.jobs?.stats]),
|
|
63
|
+
async stop() { link.close(); await fake.close(); fs.rmSync(root, { recursive: true, force: true }); fs.rmSync(other, { recursive: true, force: true }); },
|
|
64
|
+
};
|
|
65
|
+
return r;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// ------------------------------------------------------------------------------------------------------ flags
|
|
69
|
+
|
|
70
|
+
test("flags off (default, or any LLM/JOBS combination that is not complete): no job capability, no client, job.assign is acked `rejected protocol_unsupported`", async () => {
|
|
71
|
+
const cases = [
|
|
72
|
+
{ env: {} },
|
|
73
|
+
{ env: { LOCAL_AGENT_JOBS: "true", LOCAL_AGENT_DLP_MODE: "enforce" } }, // no runner
|
|
74
|
+
{ env: { ...ALL_ON, LOCAL_AGENT_DLP_MODE: "warn" }, err: /DLP_MODE is not enforce/ }, // DLP not enforce
|
|
75
|
+
{ env: { ...ALL_ON, LOCAL_AGENT_LLM_KEY: "false" }, err: /LOCAL_AGENT_LLM_KEY is not true/ }, // client without a holder -> jobs without a client
|
|
76
|
+
{ env: { ...ALL_ON, LOCAL_AGENT_LLM_CLIENT: "false" }, err: /needs the LLM client/ }, // smoke without the client
|
|
77
|
+
];
|
|
78
|
+
for (const c of cases) {
|
|
79
|
+
const r = await rig({ ...c, setKey: false });
|
|
80
|
+
assert.equal(r.link.jobs, undefined, JSON.stringify(c.env));
|
|
81
|
+
assert.equal(r.link.llm?.client, undefined);
|
|
82
|
+
assert.ok(!r.cp.registers[0].capabilities.includes("job.assign"));
|
|
83
|
+
assert.ok(!r.cp.registers[0].capabilities.includes("job.options"), "no job.options without job.assign");
|
|
84
|
+
if (c.err) assert.ok(r.errors.some((e) => c.err.test(e)), `expected an ERROR log for ${JSON.stringify(c.env)}`);
|
|
85
|
+
const a = r.assign({ inputs: [] });
|
|
86
|
+
const ack = await a.acked();
|
|
87
|
+
assert.deepEqual([ack.status, ack.errorCode], ["rejected", "protocol_unsupported"], JSON.stringify(c.env));
|
|
88
|
+
assert.equal(r.fake.requests.length, 0);
|
|
89
|
+
await r.stop();
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("flags on: `job.assign` is announced (only with a runner), nothing else changes in register/hello", async () => {
|
|
94
|
+
const r = await rig();
|
|
95
|
+
assert.ok(r.cp.registers[0].capabilities.includes("job.assign"));
|
|
96
|
+
assert.ok(r.cp.hellos[0].capabilities.includes("job.assign"));
|
|
97
|
+
assert.ok(r.cp.registers[0].capabilities.includes("job.options") && r.cp.hellos[0].capabilities.includes("job.options"), "the smoke runner understands options");
|
|
98
|
+
assert.ok(r.link.llm.client && r.link.jobs && r.link.jobs.runner.name === "smoke");
|
|
99
|
+
await r.stop();
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
// ------------------------------------------------------------------------------------------------------ the smoke job, end to end
|
|
103
|
+
|
|
104
|
+
test("smoke job: ack accepted -> job.event running -> LLM call (file masked, key in the header only, tags) -> job.event succeeded (tokens, cost) -> job.result no_change; the answer stays local", async () => {
|
|
105
|
+
const r = await rig();
|
|
106
|
+
const a = r.assign({ inputs: [{ ref: r.refOf("src/hello.js") }, { inline: `${INLINE_CANARY} please look`, transient: true }] });
|
|
107
|
+
const ack = await a.acked();
|
|
108
|
+
assert.equal(ack.status, "accepted");
|
|
109
|
+
assert.equal(ack.errorCode, undefined);
|
|
110
|
+
await r.done(a.p.jobId);
|
|
111
|
+
const evs = r.frames(a.p.jobId, "job.event");
|
|
112
|
+
assert.equal(evs[0].payload.state, "running");
|
|
113
|
+
const last = evs.at(-1).payload;
|
|
114
|
+
assert.deepEqual({ state: last.state, phase: last.phase, errorCode: last.errorCode, tokensIn: last.tokensIn, tokensOut: last.tokensOut, costUsd: last.costUsd, filesRead: last.filesRead },
|
|
115
|
+
{ state: "succeeded", phase: "done", errorCode: "none", tokensIn: 40, tokensOut: 11, costUsd: 0.0042, filesRead: 1 });
|
|
116
|
+
assert.ok(Number.isInteger(last.durationMs));
|
|
117
|
+
assert.deepEqual(r.frames(a.p.jobId, "job.result")[0].payload, { jobId: a.p.jobId, outcome: "no_change" });
|
|
118
|
+
// what reached the AI gateway
|
|
119
|
+
assert.equal(r.fake.requests.length, 1);
|
|
120
|
+
const q = r.fake.requests[0];
|
|
121
|
+
assert.equal(q.headers.authorization, `Bearer ${r.key}`);
|
|
122
|
+
assert.equal(q.body.model, "vh-claude-sonnet");
|
|
123
|
+
assert.deepEqual(q.body.metadata.tags, [`vh-tenant:${TENANT}`, `vh-la:${AGENT}`, `vh-job:${a.p.jobId}`, "vh-agent:smoke"]);
|
|
124
|
+
const prompt = q.body.messages.map((m) => m.content).join("\n");
|
|
125
|
+
assert.ok(prompt.includes(SMOKE_QUESTION) && prompt.includes(FILE_CANARY), "the (allowed) file is the subject of the fixed question");
|
|
126
|
+
assert.ok(!q.raw.includes(GH), "the token inside the file was masked BEFORE leaving");
|
|
127
|
+
assert.match(prompt, /\[REDACTED:/);
|
|
128
|
+
assert.ok(!q.raw.includes(r.key) && !q.raw.includes(INLINE_CANARY));
|
|
129
|
+
// the answer is local
|
|
130
|
+
assert.equal(r.results.length, 1);
|
|
131
|
+
assert.equal(r.results[0].answer, ANSWER);
|
|
132
|
+
assert.equal(r.results[0].state, "succeeded");
|
|
133
|
+
// nothing client-side on the wire, in logs or audits
|
|
134
|
+
const dump = r.everything();
|
|
135
|
+
for (const secret of [r.key, sha(r.key), FILE_CANARY, GH, INLINE_CANARY, ANSWER, SMOKE_QUESTION, r.root, "hello.js", "hunter2"]) assert.ok(!dump.includes(secret), `leaked: ${secret.slice(0, 14)}`);
|
|
136
|
+
for (const f of r.cp.received.filter((x) => x.type === "job.event")) assert.ok(Object.keys(f.payload).every((k) => ["jobId", "state", "phase", "iteration", "filesRead", "filesChanged", "tokensIn", "tokensOut", "costUsd", "durationMs", "errorCode"].includes(k)));
|
|
137
|
+
await r.stop();
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test("a gateway that answers the non-streaming smoke request with an SSE stream is a bad_response: the job fails llm_error (no partial result)", async () => {
|
|
141
|
+
const r = await rig({ behavior: okSse(["x"]) });
|
|
142
|
+
const a = r.assign();
|
|
143
|
+
await r.done(a.p.jobId);
|
|
144
|
+
assert.equal(r.frames(a.p.jobId, "job.event").at(-1).payload.state, "failed");
|
|
145
|
+
assert.equal(r.frames(a.p.jobId, "job.event").at(-1).payload.errorCode, "llm_error");
|
|
146
|
+
await r.stop();
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("the file gate: a denied file (.env), an id of ANOTHER workspace, an unknown id -> the job fails, NO LLM call is made, nothing of the file is anywhere", async () => {
|
|
150
|
+
const r = await rig();
|
|
151
|
+
const cases = [
|
|
152
|
+
[r.refOf(".env"), null], // the registry itself refuses to mint an id for a sensitive path
|
|
153
|
+
[r.refOf("src/other.js", "ws-2"), "ws-2 id"], // an id of a sibling workspace, used in a job for ws-1
|
|
154
|
+
["A".repeat(43), "unknown id"],
|
|
155
|
+
];
|
|
156
|
+
for (const [ref] of cases) {
|
|
157
|
+
const a = r.assign({ inputs: [{ ref: ref ?? "B".repeat(43) }] });
|
|
158
|
+
assert.equal((await a.acked()).status, "accepted");
|
|
159
|
+
await r.done(a.p.jobId);
|
|
160
|
+
assert.equal(r.frames(a.p.jobId, "job.event").at(-1).payload.state, "failed");
|
|
161
|
+
assert.equal(r.frames(a.p.jobId, "job.result")[0].payload.outcome, "failed");
|
|
162
|
+
}
|
|
163
|
+
assert.equal(r.fake.requests.length, 0);
|
|
164
|
+
const noInput = r.assign({ inputs: [] });
|
|
165
|
+
await r.done(noInput.p.jobId);
|
|
166
|
+
assert.equal(r.frames(noInput.p.jobId, "job.event").at(-1).payload.errorCode, "unknown");
|
|
167
|
+
assert.ok(!r.everything().includes("OTHER-WS-CANARY") && !r.everything().includes("hunter2"));
|
|
168
|
+
await r.stop();
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("isolation: a workspace this agent does not serve, or one that belongs to ANOTHER tenant, is rejected `unauthorized` at the ack; nothing runs", async () => {
|
|
172
|
+
const r = await rig();
|
|
173
|
+
for (const ws of ["ws-unknown", "ws-x"]) {
|
|
174
|
+
const a = r.assign({ inputs: [] }, ws);
|
|
175
|
+
const ack = await a.acked();
|
|
176
|
+
assert.equal(ack.status, "rejected", ws);
|
|
177
|
+
assert.equal(ack.errorCode, "unauthorized", ws);
|
|
178
|
+
}
|
|
179
|
+
assert.equal(r.link.jobs.stats.accepted, 0);
|
|
180
|
+
assert.equal(r.fake.requests.length, 0);
|
|
181
|
+
await r.stop();
|
|
182
|
+
const t = await rig({ tenantWs: "9d1f6a2c-0000-4000-8000-000000000000" });
|
|
183
|
+
const a = t.assign({ inputs: [] });
|
|
184
|
+
assert.equal((await a.acked()).errorCode, "unauthorized");
|
|
185
|
+
await t.stop();
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
test("the job names a key that is not the held one (or there is no key): accepted, then failed llm_error; no LLM call", async () => {
|
|
189
|
+
const r = await rig();
|
|
190
|
+
const a = r.assign({ llm: { alias: "vh-claude-sonnet", keyRef: sha("another key") } });
|
|
191
|
+
assert.equal((await a.acked()).status, "accepted");
|
|
192
|
+
await r.done(a.p.jobId);
|
|
193
|
+
assert.equal(r.frames(a.p.jobId, "job.event").at(-1).payload.errorCode, "llm_error");
|
|
194
|
+
const b = r.assign({ llm: { alias: "vh-claude-sonnet", keyRef: sha(r.key).slice(0, 8) } });
|
|
195
|
+
await r.done(b.p.jobId);
|
|
196
|
+
assert.equal(r.frames(b.p.jobId, "job.event").at(-1).payload.state, "succeeded", "the 8-hex key_short names the held key");
|
|
197
|
+
assert.equal(r.fake.requests.length, 1);
|
|
198
|
+
await r.stop();
|
|
199
|
+
const n = await rig({ setKey: false });
|
|
200
|
+
const c = n.assign();
|
|
201
|
+
await n.done(c.p.jobId);
|
|
202
|
+
assert.equal(n.frames(c.p.jobId, "job.event").at(-1).payload.errorCode, "llm_error");
|
|
203
|
+
await n.stop();
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
test("an alias outside the key's models is policy_denied (no call); a gateway outage is llm_error after the bounded retries (3 calls)", async () => {
|
|
207
|
+
const r = await rig();
|
|
208
|
+
const a = r.assign({ llm: { alias: "vh-drael", keyRef: sha(r.key) } });
|
|
209
|
+
await r.done(a.p.jobId);
|
|
210
|
+
assert.equal(r.frames(a.p.jobId, "job.event").at(-1).payload.errorCode, "policy_denied");
|
|
211
|
+
assert.equal(r.fake.requests.length, 0);
|
|
212
|
+
await r.stop();
|
|
213
|
+
const down = await rig({ behavior: status(503) });
|
|
214
|
+
const b = down.assign();
|
|
215
|
+
await down.done(b.p.jobId);
|
|
216
|
+
assert.equal(down.frames(b.p.jobId, "job.event").at(-1).payload.errorCode, "llm_error");
|
|
217
|
+
assert.equal(down.fake.requests.length, 3, "1 + 2 retries, then the code");
|
|
218
|
+
await down.stop();
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test("budget: a token budget too small for the prompt fails the job budget_exceeded BEFORE any call; the local ceiling clamps what the job asked", async () => {
|
|
222
|
+
const r = await rig({ env: { ...ALL_ON, LOCAL_AGENT_JOBS_MAX_TOKENS: "20" } });
|
|
223
|
+
const a = r.assign({ budget: { tokens: 9_000_000, seconds: 60 } }); // clamped to 20 tokens: the prompt alone does not fit
|
|
224
|
+
await r.done(a.p.jobId);
|
|
225
|
+
assert.equal(r.frames(a.p.jobId, "job.event").at(-1).payload.errorCode, "budget_exceeded");
|
|
226
|
+
assert.equal(r.fake.requests.length, 0);
|
|
227
|
+
await r.stop();
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test("cancel: job.cancel while the LLM call is in flight -> the call is aborted, job.event cancelled + job.result cancelled, the slot is free for the next job", async () => {
|
|
231
|
+
let hangs = true;
|
|
232
|
+
const r = await rig({ behavior: (q, s, e) => (hangs ? new Promise(() => {}) : okJson("fine")(q, s, e)) });
|
|
233
|
+
const a = r.assign();
|
|
234
|
+
assert.equal((await a.acked()).status, "accepted");
|
|
235
|
+
await waitFor(() => r.fake.requests.length === 1);
|
|
236
|
+
r.cp.sendData("job.cancel", { jobId: a.p.jobId, reason: "user" });
|
|
237
|
+
await r.done(a.p.jobId);
|
|
238
|
+
assert.equal(r.frames(a.p.jobId, "job.event").at(-1).payload.state, "cancelled");
|
|
239
|
+
assert.equal(r.frames(a.p.jobId, "job.result")[0].payload.outcome, "cancelled");
|
|
240
|
+
assert.equal(r.link.llm.client.active, 0, "the in-flight call was released");
|
|
241
|
+
hangs = false;
|
|
242
|
+
const b = r.assign();
|
|
243
|
+
await r.done(b.p.jobId);
|
|
244
|
+
assert.equal(r.frames(b.p.jobId, "job.event").at(-1).payload.state, "succeeded");
|
|
245
|
+
assert.equal(r.cp.acksFromLa.filter((x) => x.status === "rejected").length, 0, "job.cancel was acked accepted");
|
|
246
|
+
await r.stop();
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
test("limits: concurrency 1 / queue 0 -> the 2nd job.assign is rejected rate_limited while the 1st runs; duplicates of the same job id are accepted and run once", async () => {
|
|
250
|
+
let release;
|
|
251
|
+
const gate = new Promise((x) => { release = x; });
|
|
252
|
+
const r = await rig({ env: { ...ALL_ON, LOCAL_AGENT_JOBS_MAX_QUEUED: "0" }, behavior: async (q, s, e) => { await gate; okJson("late")(q, s, e); } });
|
|
253
|
+
const a = r.assign();
|
|
254
|
+
assert.equal((await a.acked()).status, "accepted");
|
|
255
|
+
await waitFor(() => r.fake.requests.length === 1);
|
|
256
|
+
const b = r.assign();
|
|
257
|
+
const ackB = await b.acked();
|
|
258
|
+
assert.deepEqual([ackB.status, ackB.errorCode], ["rejected", "rate_limited"]);
|
|
259
|
+
const dup = r.assign({ jobId: a.p.jobId });
|
|
260
|
+
assert.equal((await dup.acked()).status, "accepted");
|
|
261
|
+
release();
|
|
262
|
+
await r.done(a.p.jobId);
|
|
263
|
+
assert.equal(r.fake.requests.length, 1, "the duplicate did not start a second run");
|
|
264
|
+
assert.equal(r.frames(a.p.jobId, "job.result").length, 1);
|
|
265
|
+
await r.stop();
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
test("the contract is closed: a git token (or any extra field) in job.assign is a schema violation at the session - no runner, no call, no token in a log", async () => {
|
|
269
|
+
const r = await rig();
|
|
270
|
+
const evil = "ghp_" + "Z".repeat(36);
|
|
271
|
+
r.cp.sendData("job.assign", { jobId: uuid(), kind: "fix", repoRef: { provider: "github", org: "a", repo: "b", branch: "main" }, policy: { operationMode: "assist", automationLevel: 2, ceiling: "assist" },
|
|
272
|
+
budget: { tokens: 1000, seconds: 60 }, llm: { alias: "vh-claude-sonnet", keyRef: sha(r.key) }, gitToken: evil });
|
|
273
|
+
await waitFor(() => r.cp.peerErrors.length >= 1);
|
|
274
|
+
assert.deepEqual(r.cp.peerErrors, ["schema_violation"]);
|
|
275
|
+
assert.equal(r.link.jobs.stats.received, 0);
|
|
276
|
+
assert.equal(r.fake.requests.length, 0);
|
|
277
|
+
assert.ok(!r.everything().includes(evil));
|
|
278
|
+
await r.stop();
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
test("a pluggable runner: an injected one replaces the smoke runner (job.assign needs no LOCAL_AGENT_JOBS_RUNNER), gets the job + ctx, and its metadata goes back", async () => {
|
|
282
|
+
const seen = [];
|
|
283
|
+
const runner = { name: "loop", needsLlm: false, cancel() {}, async start(job, ctx) { seen.push(job.kind, job.inputs.length, Object.keys(ctx).sort()); ctx.progress({ phase: "edit", filesChanged: 2 }); return { outcome: "pr_opened", prUrl: "https://git.example/a/b/pull/1", branch: "vh/x", commitSha: "abcdef1234", filesChangedCount: 2, answer: "local" }; } };
|
|
284
|
+
const { LOCAL_AGENT_JOBS_RUNNER, LOCAL_AGENT_LLM_CLIENT, ...env } = ALL_ON;
|
|
285
|
+
const r = await rig({ env, jobRunner: runner, setKey: false });
|
|
286
|
+
assert.ok(r.cp.registers[0].capabilities.includes("job.assign"));
|
|
287
|
+
const a = r.assign({ inputs: [{ inline: INLINE_CANARY, transient: true }] });
|
|
288
|
+
await r.done(a.p.jobId);
|
|
289
|
+
assert.deepEqual(seen, ["quality", 1, ["budget", "llm", "progress", "readFile", "signal", "workspace"]]);
|
|
290
|
+
assert.deepEqual(r.frames(a.p.jobId, "job.result")[0].payload, { jobId: a.p.jobId, outcome: "pr_opened", prUrl: "https://git.example/a/b/pull/1", branch: "vh/x", commitSha: "abcdef1234", filesChangedCount: 2 });
|
|
291
|
+
assert.ok(!r.everything().includes(INLINE_CANARY));
|
|
292
|
+
await r.stop();
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
// ------------------------------------------------------------------------------------------------------ job.options (contract 2.1) and the cancel rules
|
|
296
|
+
|
|
297
|
+
test("job.options: announced only for a runner that declared supportsOptions (the smoke one does); an injected runner without it is not offered options and refuses them", async () => {
|
|
298
|
+
const { LOCAL_AGENT_JOBS_RUNNER, LOCAL_AGENT_LLM_CLIENT, ...env } = ALL_ON;
|
|
299
|
+
const plain = { name: "plain", needsLlm: false, cancel() {}, async start() { return { outcome: "no_change" }; } };
|
|
300
|
+
const r = await rig({ env, jobRunner: plain, setKey: false });
|
|
301
|
+
assert.ok(r.cp.registers[0].capabilities.includes("job.assign"));
|
|
302
|
+
assert.ok(!r.cp.registers[0].capabilities.includes("job.options") && !r.cp.hellos[0].capabilities.includes("job.options"));
|
|
303
|
+
const a = r.assign({ options: { dryRun: true } });
|
|
304
|
+
const ack = await a.acked();
|
|
305
|
+
assert.deepEqual([ack.status, ack.errorCode], ["rejected", "protocol_unsupported"], "a dry run is never run as a real job");
|
|
306
|
+
assert.equal(r.link.jobs.stats.optionsUnsupported, 1);
|
|
307
|
+
await r.stop();
|
|
308
|
+
const seen = [];
|
|
309
|
+
const aware = { name: "aware", needsLlm: false, supportsOptions: true, cancel() {}, async start(job) { seen.push(job.options); return { outcome: "no_change" }; } };
|
|
310
|
+
const r2 = await rig({ env, jobRunner: aware, setKey: false });
|
|
311
|
+
assert.ok(r2.cp.registers[0].capabilities.includes("job.options") && r2.cp.hellos[0].capabilities.includes("job.options"));
|
|
312
|
+
const b = r2.assign({ options: { branchName: "vh/KAN-28-fix", createPullRequest: false, maxIterations: 4 } });
|
|
313
|
+
assert.equal((await b.acked()).status, "accepted");
|
|
314
|
+
await r2.done(b.p.jobId);
|
|
315
|
+
assert.deepEqual(seen, [{ branchName: "vh/KAN-28-fix", createBranch: true, createPullRequest: false, dryRun: false, maxIterations: 4, effects: { createBranch: true, push: true, openPullRequest: false } }]);
|
|
316
|
+
await r2.stop();
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
test("smoke job with options (dryRun, createPullRequest:false, maxIterations): accepted, one LLM call, no_change, no branch/PR in the result, the option values are on no frame, log or audit", async () => {
|
|
320
|
+
const r = await rig();
|
|
321
|
+
const BRANCH = "vh/OPT-CANARY-branch";
|
|
322
|
+
const a = r.assign({ options: { ticketKey: "KAN-OPT", branchName: BRANCH, dryRun: true, createPullRequest: false, maxIterations: 1 } });
|
|
323
|
+
assert.equal((await a.acked()).status, "accepted");
|
|
324
|
+
await r.done(a.p.jobId);
|
|
325
|
+
assert.deepEqual(r.frames(a.p.jobId, "job.result")[0].payload, { jobId: a.p.jobId, outcome: "no_change" });
|
|
326
|
+
assert.equal(r.fake.requests.length, 1, "a dry run still runs the loop (one question)");
|
|
327
|
+
assert.equal(Math.max(...r.frames(a.p.jobId, "job.event").map((f) => f.payload.iteration ?? 0)), 1);
|
|
328
|
+
const dump = r.everything();
|
|
329
|
+
for (const v of [BRANCH, "KAN-OPT"]) assert.ok(!dump.includes(v), `leaked: ${v}`);
|
|
330
|
+
assert.ok(!r.fake.requests[0].raw.includes(BRANCH));
|
|
331
|
+
await r.stop();
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
test("options the contract refuses never reach the receiver: a token, an unknown key, an empty object, a bad branch name are schema violations at the session (fatal for that connection, as for any closed-schema breach)", async () => {
|
|
335
|
+
for (const options of [{ token: "ghp_" + "Z".repeat(36) }, {}, { dryRun: "true" }, { branchName: "a/../b" }, { maxIterations: 101 }, { createBranch: true, extra: 1 }]) {
|
|
336
|
+
const r = await rig();
|
|
337
|
+
r.cp.sendData("job.assign", { jobId: uuid(), kind: "fix", repoRef: { provider: "github", org: "a", repo: "b", branch: "main" }, policy: { operationMode: "assist", automationLevel: 2, ceiling: "assist" },
|
|
338
|
+
budget: { tokens: 1000, seconds: 60 }, llm: { alias: "vh-claude-sonnet", keyRef: sha(r.key) }, options });
|
|
339
|
+
await waitFor(() => r.cp.peerErrors.length >= 1);
|
|
340
|
+
assert.deepEqual(r.cp.peerErrors, ["schema_violation"], JSON.stringify(options));
|
|
341
|
+
assert.equal(r.link.jobs.stats.received, 0);
|
|
342
|
+
assert.equal(r.fake.requests.length, 0);
|
|
343
|
+
assert.ok(!r.everything().includes("ZZZZZZ"));
|
|
344
|
+
await r.stop();
|
|
345
|
+
}
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
test("cancel with an id that is not a UUID: a schema_violation error from the session (clear, nothing echoed), no lookup, the running job is untouched", async () => {
|
|
349
|
+
for (const jobId of ["../../etc/passwd-ECHO", "not-a-uuid", "", "0".repeat(37)]) {
|
|
350
|
+
const r = await rig({ behavior: () => new Promise(() => {}) });
|
|
351
|
+
const a = r.assign();
|
|
352
|
+
assert.equal((await a.acked()).status, "accepted");
|
|
353
|
+
await waitFor(() => r.fake.requests.length === 1);
|
|
354
|
+
r.cp.sendData("job.cancel", { jobId, reason: "timeout" });
|
|
355
|
+
await waitFor(() => r.cp.peerErrors.length >= 1);
|
|
356
|
+
assert.equal(r.cp.peerErrors.length, 1);
|
|
357
|
+
assert.match(r.cp.peerErrors[0], /^(schema_violation|payload_too_large)$/, JSON.stringify(jobId)); // an over-long id breaks the size limit first
|
|
358
|
+
assert.equal(r.link.jobs.running, 1, "the job did not notice");
|
|
359
|
+
assert.equal(r.link.jobs.stats.cancels, 0, "no lookup");
|
|
360
|
+
assert.equal(r.link.jobs.stats.cancelInvalid, 0, "the session refused it first");
|
|
361
|
+
assert.ok(!r.everything().includes("ECHO"));
|
|
362
|
+
await r.stop();
|
|
363
|
+
}
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
test("cancel of an unknown or a finished job is acked accepted and does nothing; of a running one it cancels it", async () => {
|
|
367
|
+
let hangs = false;
|
|
368
|
+
const r = await rig({ behavior: (q, s, e) => (hangs ? new Promise(() => {}) : okJson("fine")(q, s, e)) });
|
|
369
|
+
const done = r.assign();
|
|
370
|
+
await r.done(done.p.jobId);
|
|
371
|
+
hangs = true;
|
|
372
|
+
const live = r.assign();
|
|
373
|
+
assert.equal((await live.acked()).status, "accepted");
|
|
374
|
+
await waitFor(() => r.fake.requests.length === 2);
|
|
375
|
+
const frames = r.cp.received.length;
|
|
376
|
+
for (const jobId of [uuid(), done.p.jobId]) {
|
|
377
|
+
const n = r.cp.acksFromLa.length;
|
|
378
|
+
r.cp.sendData("job.cancel", { jobId, reason: "timeout" });
|
|
379
|
+
await waitFor(() => r.cp.acksFromLa.length > n);
|
|
380
|
+
assert.equal(r.cp.acksFromLa[n].status, "accepted");
|
|
381
|
+
}
|
|
382
|
+
await new Promise((x) => setTimeout(x, 60));
|
|
383
|
+
assert.equal(r.link.jobs.running, 1);
|
|
384
|
+
assert.ok(r.cp.received.slice(frames).filter((f) => f.payload.jobId === live.p.jobId).every((f) => f.type === "job.event" && f.payload.state === "running"), "the live job saw nothing but its own progress");
|
|
385
|
+
r.cp.sendData("job.cancel", { jobId: live.p.jobId, reason: "user" });
|
|
386
|
+
await r.done(live.p.jobId);
|
|
387
|
+
assert.equal(r.frames(live.p.jobId, "job.result")[0].payload.outcome, "cancelled");
|
|
388
|
+
assert.equal(r.frames(done.p.jobId, "job.result").length, 1, "the finished one has no second result");
|
|
389
|
+
await r.stop();
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
test("an orphan the CP gave up (job.cancel first, job.assign later): the assign is acked accepted and no LLM call, no event, no result", async () => {
|
|
393
|
+
const r = await rig();
|
|
394
|
+
const jobId = uuid();
|
|
395
|
+
const n = r.cp.acksFromLa.length;
|
|
396
|
+
r.cp.sendData("job.cancel", { jobId, reason: "timeout" });
|
|
397
|
+
await waitFor(() => r.cp.acksFromLa.length > n);
|
|
398
|
+
const a = r.assign({ jobId });
|
|
399
|
+
assert.equal((await a.acked()).status, "accepted");
|
|
400
|
+
await new Promise((x) => setTimeout(x, 100));
|
|
401
|
+
assert.equal(r.fake.requests.length, 0);
|
|
402
|
+
assert.equal(r.frames(jobId).length, 0);
|
|
403
|
+
assert.equal(r.link.jobs.activeCount, 0);
|
|
404
|
+
await r.stop();
|
|
405
|
+
});
|