deepdebug-local-agent 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/deploy-gcp.yml +58 -26
- package/CHANGELOG.md +13 -0
- package/docs/la-protocol-v2.md +68 -0
- package/docs/ws-metadata-only.md +44 -0
- package/env.example +7 -0
- package/package.json +2 -1
- package/scripts/sync-la-protocol-v2-contract.mjs +58 -0
- package/src/la-protocol-v2/chunks.js +45 -0
- package/src/la-protocol-v2/config.js +27 -0
- package/src/la-protocol-v2/content.js +57 -0
- package/src/la-protocol-v2/contract/CONTRACT_SOURCE.json +81 -0
- package/src/la-protocol-v2/contract/README.md +103 -0
- package/src/la-protocol-v2/contract/ack.schema.json +51 -0
- package/src/la-protocol-v2/contract/alert.schema.json +62 -0
- package/src/la-protocol-v2/contract/approval.decision.schema.json +57 -0
- package/src/la-protocol-v2/contract/audit.event.schema.json +86 -0
- package/src/la-protocol-v2/contract/config.update.schema.json +73 -0
- package/src/la-protocol-v2/contract/content.request.schema.json +110 -0
- package/src/la-protocol-v2/contract/content.response.schema.json +90 -0
- package/src/la-protocol-v2/contract/defs.schema.json +191 -0
- package/src/la-protocol-v2/contract/envelope.schema.json +89 -0
- package/src/la-protocol-v2/contract/error.schema.json +59 -0
- package/src/la-protocol-v2/contract/heartbeat.schema.json +100 -0
- package/src/la-protocol-v2/contract/hello.schema.json +111 -0
- package/src/la-protocol-v2/contract/incident.candidate.schema.json +102 -0
- package/src/la-protocol-v2/contract/index.json +166 -0
- package/src/la-protocol-v2/contract/job.assign.schema.json +212 -0
- package/src/la-protocol-v2/contract/job.cancel.schema.json +52 -0
- package/src/la-protocol-v2/contract/job.event.schema.json +115 -0
- package/src/la-protocol-v2/contract/job.result.schema.json +100 -0
- package/src/la-protocol-v2/contract/log.stats.schema.json +79 -0
- package/src/la-protocol-v2/contract/ready.schema.json +106 -0
- package/src/la-protocol-v2/contract/register.schema.json +92 -0
- package/src/la-protocol-v2/contract/scan.upload.schema.json +59 -0
- package/src/la-protocol-v2/contract/scan.uploaded.schema.json +51 -0
- package/src/la-protocol-v2/contract/webhook.forward.schema.json +69 -0
- package/src/la-protocol-v2/contract.js +90 -0
- package/src/la-protocol-v2/errors.js +34 -0
- package/src/la-protocol-v2/index.js +8 -0
- package/src/la-protocol-v2/link.js +93 -0
- package/src/la-protocol-v2/runtime.js +72 -0
- package/src/la-protocol-v2/session.js +650 -0
- package/src/la-protocol-v2/strict-json.js +47 -0
- package/src/mcp-http-server.js +5 -3
- package/src/server.js +70 -9
- package/src/utils/exec-policy.js +8 -1
- package/src/ws-channel/allowlist.js +143 -0
- package/src/ws-channel/config.js +22 -0
- package/src/ws-channel/connection-state.js +18 -0
- package/src/ws-channel/index.js +6 -0
- package/src/ws-channel/metadata-guard.js +78 -0
- package/src/ws-channel/policy-offline.js +40 -0
- package/src/ws-channel/transit-guard.js +55 -0
- package/test/contract/la-protocol-v2/vectors/ack.vectors.json +149 -0
- package/test/contract/la-protocol-v2/vectors/alert.vectors.json +101 -0
- package/test/contract/la-protocol-v2/vectors/approval.decision.vectors.json +165 -0
- package/test/contract/la-protocol-v2/vectors/audit.event.vectors.json +156 -0
- package/test/contract/la-protocol-v2/vectors/config.update.vectors.json +189 -0
- package/test/contract/la-protocol-v2/vectors/content.request.vectors.json +325 -0
- package/test/contract/la-protocol-v2/vectors/content.response.vectors.json +413 -0
- package/test/contract/la-protocol-v2/vectors/cross-type.vectors.json +260 -0
- package/test/contract/la-protocol-v2/vectors/error.vectors.json +179 -0
- package/test/contract/la-protocol-v2/vectors/heartbeat.vectors.json +109 -0
- package/test/contract/la-protocol-v2/vectors/hello.vectors.json +224 -0
- package/test/contract/la-protocol-v2/vectors/incident.candidate.vectors.json +186 -0
- package/test/contract/la-protocol-v2/vectors/job.assign.vectors.json +631 -0
- package/test/contract/la-protocol-v2/vectors/job.cancel.vectors.json +133 -0
- package/test/contract/la-protocol-v2/vectors/job.event.vectors.json +233 -0
- package/test/contract/la-protocol-v2/vectors/job.result.vectors.json +137 -0
- package/test/contract/la-protocol-v2/vectors/log.stats.vectors.json +137 -0
- package/test/contract/la-protocol-v2/vectors/ready.vectors.json +177 -0
- package/test/contract/la-protocol-v2/vectors/register.vectors.json +399 -0
- package/test/contract/la-protocol-v2/vectors/scan.upload.vectors.json +137 -0
- package/test/contract/la-protocol-v2/vectors/scan.uploaded.vectors.json +101 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-agent-mismatch.json +140 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-chunk-after-last.json +200 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-chunk-byte-length-mismatch.json +174 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-chunk-last-flag.json +200 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-chunk-out-of-order.json +200 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-chunk-total-changes.json +200 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-clock-skew.json +46 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-event-before-ready.json +72 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-first-frame-not-register.json +48 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-frame-over-48kib.json +141 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-la-sends-job-assign.json +173 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-params-raw-differs-from-payload.json +172 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-payload-above-limit.json +174 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-resume-restarts-seq.json +144 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-seq-gap.json +166 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-seq-repeated.json +166 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-sig-tenant-mismatch.json +172 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-signature-warn-with-enforce-required.json +77 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-tenant-mismatch.json +140 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-type-outside-allowlist.json +140 -0
- package/test/contract/la-protocol-v2/vectors/traces/valid-content-chunks-in-order.json +224 -0
- package/test/contract/la-protocol-v2/vectors/traces/valid-handshake-and-job.json +266 -0
- package/test/contract/la-protocol-v2/vectors/traces/valid-resume-continues-seq.json +168 -0
- package/test/contract/la-protocol-v2/vectors/traces/valid-retransmit-is-idempotent.json +193 -0
- package/test/contract/la-protocol-v2/vectors/webhook.forward.vectors.json +170 -0
- package/test/la-protocol-v2/contract.test.js +131 -0
- package/test/la-protocol-v2/helpers/la-rig.js +83 -0
- package/test/la-protocol-v2/helpers/reference-cp.js +349 -0
- package/test/la-protocol-v2/helpers/spawn-agent.js +35 -0
- package/test/la-protocol-v2/link.test.js +141 -0
- package/test/la-protocol-v2/session-inbound.test.js +279 -0
- package/test/la-protocol-v2/session-outbound.test.js +382 -0
- package/test/la-protocol-v2/traces.test.js +270 -0
- package/test/la-protocol-v2/tunnel-e2e.test.js +124 -0
- package/test/ws-channel/allowlist.test.js +67 -0
- package/test/ws-channel/metadata-guard.test.js +104 -0
- package/test/ws-channel/policy-offline.test.js +99 -0
- package/test/ws-channel/tunnel-e2e.test.js +88 -0
- package/tools/graphify-validation/extras/README.md +19 -0
- package/tools/graphify-validation/extras/bymethod.py +35 -0
- package/tools/graphify-validation/extras/docker/Dockerfile +16 -0
- package/tools/graphify-validation/extras/docker/inner.sh +7 -0
- package/tools/graphify-validation/extras/g10.py +59 -0
- package/tools/graphify-validation/extras/g3aug.py +49 -0
- package/tools/graphify-validation/extras/hybrid.py +34 -0
- package/tools/graphify-validation/extras/truth2.py +44 -0
- package/tools/interop/LaV2NodeInteropTest.java +266 -0
- package/tools/interop/README.md +21 -0
- package/tools/interop/la-interop-client.mjs +97 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
// F3-05b: what the Local Agent does with a frame from the control plane. No sockets: the session is wired to a
|
|
2
|
+
// ReferenceCp in memory, so the order of events is deterministic. Every refusal must carry only the wire code.
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import crypto from "node:crypto";
|
|
6
|
+
import { ReferenceCp, TENANT, AGENT, uuid, waitFor, newKeyPair } from "./helpers/reference-cp.js";
|
|
7
|
+
import { inMemory, signingFor } from "./helpers/la-rig.js";
|
|
8
|
+
import { closeCodeOf } from "../../src/la-protocol-v2/errors.js";
|
|
9
|
+
|
|
10
|
+
const SECRET = "CLIENT_SECRET_MARKER_5521";
|
|
11
|
+
const JOB = "c3d4e5f6-a7b8-4c9d-8e0f-2a3b4c5d6e7f";
|
|
12
|
+
|
|
13
|
+
async function ready(opts = {}) {
|
|
14
|
+
const cp = new ReferenceCp(opts.cp);
|
|
15
|
+
const env = inMemory(cp, opts.la).start();
|
|
16
|
+
await waitFor(() => env.session.state === "ready");
|
|
17
|
+
return { cp, ...env };
|
|
18
|
+
}
|
|
19
|
+
const errorsFromLa = (env) => env.wire.toCp.map((t) => JSON.parse(t)).filter((f) => f.type === "error").map((f) => f.payload);
|
|
20
|
+
const sentText = (env) => env.wire.toCp.join("\n");
|
|
21
|
+
const close = (env) => env.session.close();
|
|
22
|
+
|
|
23
|
+
test("handshake: register then hello, a signed ready is verified (enforce) and the session is ready", async () => {
|
|
24
|
+
const env = await ready({ cp: { requireEnforce: true } });
|
|
25
|
+
assert.equal(env.session.state, "ready");
|
|
26
|
+
assert.equal(env.cp.registers[0].signatureMode, "enforce");
|
|
27
|
+
assert.equal(env.cp.hellos[0].signatureMode, "enforce");
|
|
28
|
+
assert.equal(env.session.sessionId, env.cp.sessionId);
|
|
29
|
+
close(env);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const BAD_FRAMES = [
|
|
33
|
+
// [name, build(cp) -> text, expected code]
|
|
34
|
+
["a type outside the allowlist", (cp) => JSON.stringify({ ...cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }), type: "shell.exec" }), "unknown_type"],
|
|
35
|
+
["a v1 command name used as a type", (cp) => JSON.stringify({ ...cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }), type: "mcp.execute-command" }), "unknown_type"],
|
|
36
|
+
["a la->cp type sent by the CP", (cp) => JSON.stringify({ schemaVersion: 2, type: "job.event", msgId: uuid(), seq: 1, tenantId: TENANT, agentId: AGENT, workspaceId: "", ts: new Date().toISOString(), payload: { jobId: JOB, state: "running" } }), "wrong_direction"],
|
|
37
|
+
["JSON that does not parse", () => "{not json " + SECRET, "bad_frame"],
|
|
38
|
+
["a duplicate key", () => `{"schemaVersion":2,"schemaVersion":2,"note":"${SECRET}"}`, "bad_frame"],
|
|
39
|
+
["a JSON array", () => `["${SECRET}"]`, "bad_frame"],
|
|
40
|
+
["a frame above 48 KiB", (cp) => { const f = cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }); return JSON.stringify(f) + " ".repeat(50_000); }, "frame_too_large"],
|
|
41
|
+
["an extra top-level field", (cp) => JSON.stringify({ ...cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }), extra: SECRET }), "schema_violation"],
|
|
42
|
+
["an extra payload field", (cp) => { const f = cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }); f.payload.note = SECRET; return JSON.stringify(f); }, "schema_violation"],
|
|
43
|
+
["schemaVersion 1", (cp) => JSON.stringify({ ...cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }), schemaVersion: 1 }), "schema_violation"],
|
|
44
|
+
["another tenant", (cp) => JSON.stringify({ ...cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }), tenantId: "99999999-5a4d-4e7b-9c21-0a1b2c3d4e5f" }), "tenant_mismatch"],
|
|
45
|
+
["another agent", (cp) => JSON.stringify({ ...cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }), agentId: "99999999-e5f6-4a7b-8c9d-0e1f2a3b4c5d" }), "agent_mismatch"],
|
|
46
|
+
["paramsRaw different from payload", (cp) => { const f = cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }); f.paramsRaw = JSON.stringify({ jobId: JOB, reason: "ops" }); return JSON.stringify(f); }, "params_mismatch"],
|
|
47
|
+
["paramsRaw with a duplicate key", (cp) => { const f = cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }); f.paramsRaw = `{"jobId":"${JOB}","reason":"user","reason":"user"}`; return JSON.stringify(f); }, "params_mismatch"],
|
|
48
|
+
["sig bound to another tenant", (cp) => { const f = cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }); f.sig.tenantId = "99999999-5a4d-4e7b-9c21-0a1b2c3d4e5f"; return JSON.stringify(f); }, "tenant_mismatch"],
|
|
49
|
+
["sig bound to another workspace", (cp) => { const f = cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }); f.sig.workspaceId = "other-ws"; return JSON.stringify(f); }, "tenant_mismatch"],
|
|
50
|
+
["a bad signature (enforce)", (cp) => { const f = cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }); f.sig.signature = Buffer.alloc(64, 7).toString("base64"); return JSON.stringify(f); }, "unauthorized"],
|
|
51
|
+
["a signature of another key id", (cp) => { const f = cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }); f.sig.keyId = "unknown-key"; return JSON.stringify(f); }, "unauthorized"],
|
|
52
|
+
["a seq gap", (cp) => { cp.cpSeq += 4; return JSON.stringify(cp.frameWith("job.cancel", { jobId: JOB, reason: "user" })); }, "seq_gap"],
|
|
53
|
+
["a second ready", (cp) => { const f = cp.frameWith("config.update", { retentionDays: 7 }); f.type = "ready"; f.seq = 0; f.payload = { protocol: 2, sessionId: uuid(), resumed: false, peerLastSeq: 0, limits: { maxFrameBytes: 49152, chunkRawBytes: 32768, maxChunks: 64, ackTimeoutMs: 5000, maxRetries: 3, heartbeatSeconds: 30 } }; f.paramsRaw = JSON.stringify(f.payload); return JSON.stringify(f); }, "handshake_violation"],
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
for (const [name, build, code] of BAD_FRAMES) {
|
|
57
|
+
test(`inbound: ${name} -> error ${code}, close ${closeCodeOf(code)}, nothing of the content comes back`, async () => {
|
|
58
|
+
const env = await ready();
|
|
59
|
+
env.cp.sendRaw(build(env.cp));
|
|
60
|
+
await waitFor(() => errorsFromLa(env).length > 0);
|
|
61
|
+
const err = errorsFromLa(env)[0];
|
|
62
|
+
assert.equal(err.code, code);
|
|
63
|
+
assert.equal(err.closeCode, closeCodeOf(code));
|
|
64
|
+
assert.deepEqual(Object.keys(err).filter((k) => !["code", "retryable", "closeCode", "refSeq", "refMsgId"].includes(k)), []);
|
|
65
|
+
assert.equal(sentText(env).includes(SECRET), false, "the error frame echoed content");
|
|
66
|
+
assert.equal(env.logs.join("\n").includes(SECRET), false, "the log carries content");
|
|
67
|
+
await waitFor(() => env.wire.closed.length > 0);
|
|
68
|
+
assert.equal(env.wire.closed[0][0], closeCodeOf(code));
|
|
69
|
+
close(env);
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
test("inbound: a replayed nonce is refused (unauthorized): the same signed bytes cannot be sent twice", async () => {
|
|
74
|
+
const env = await ready();
|
|
75
|
+
const f = env.cp.frameWith("job.cancel", { jobId: JOB, reason: "user" });
|
|
76
|
+
env.session.setHandler("job.cancel", () => ({ accept: true }));
|
|
77
|
+
env.cp.sendRaw(JSON.stringify(f));
|
|
78
|
+
await waitFor(() => env.cp.acksFromLa.length >= 1);
|
|
79
|
+
// same signature/nonce on a NEW seq: the nonce was already consumed
|
|
80
|
+
const g = { ...f, seq: f.seq + 1, msgId: uuid() };
|
|
81
|
+
env.cp.sendRaw(JSON.stringify(g));
|
|
82
|
+
await waitFor(() => errorsFromLa(env).length > 0);
|
|
83
|
+
assert.equal(errorsFromLa(env)[0].code, "unauthorized");
|
|
84
|
+
close(env);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("inbound: seq_replayed (same seq, another msgId) carries refSeq and refMsgId", async () => {
|
|
88
|
+
const env = await ready();
|
|
89
|
+
env.session.setHandler("job.cancel", () => ({ accept: true }));
|
|
90
|
+
const f = env.cp.frameWith("job.cancel", { jobId: JOB, reason: "user" });
|
|
91
|
+
env.cp.sendRaw(JSON.stringify(f));
|
|
92
|
+
await waitFor(() => env.cp.acksFromLa.length >= 1);
|
|
93
|
+
const other = env.cp.frameWith("job.cancel", { jobId: JOB, reason: "user" });
|
|
94
|
+
other.seq = f.seq; // same seq, another msgId
|
|
95
|
+
env.cp.sendRaw(JSON.stringify(other));
|
|
96
|
+
await waitFor(() => errorsFromLa(env).length > 0);
|
|
97
|
+
const e = errorsFromLa(env)[0];
|
|
98
|
+
assert.equal(e.code, "seq_replayed");
|
|
99
|
+
assert.equal(e.refSeq, f.seq);
|
|
100
|
+
assert.equal(e.refMsgId, other.msgId);
|
|
101
|
+
close(env);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("inbound: before ready only ready/error/ack are accepted (not_ready)", async () => {
|
|
105
|
+
const cp = new ReferenceCp();
|
|
106
|
+
const env = inMemory(cp);
|
|
107
|
+
// attach without letting the CP answer: use a transport that swallows the register/hello
|
|
108
|
+
const swallow = { send() {}, close() {}, bufferedAmount: () => 0 };
|
|
109
|
+
env.session.attach(swallow);
|
|
110
|
+
env.session.handleText(JSON.stringify(cp.frameWith("job.cancel", { jobId: JOB, reason: "user" })), swallow);
|
|
111
|
+
await waitFor(() => env.session.stats.rejectedIn.not_ready === 1);
|
|
112
|
+
env.session.close();
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
// ------------------------------------------------------------------------------------------ signature modes
|
|
116
|
+
|
|
117
|
+
test("signature modes: warn and off accept a bad signature, enforce refuses it; hello reports the mode", async () => {
|
|
118
|
+
for (const [mode, accepted] of [["warn", true], ["off", true], ["enforce", false]]) {
|
|
119
|
+
const env = await ready({ la: { mode } });
|
|
120
|
+
assert.equal(env.cp.hellos[0].signatureMode, mode);
|
|
121
|
+
let ran = 0;
|
|
122
|
+
env.session.setHandler("job.cancel", () => { ran++; return { accept: true }; });
|
|
123
|
+
const f = env.cp.frameWith("job.cancel", { jobId: JOB, reason: "user" });
|
|
124
|
+
f.sig.signature = Buffer.alloc(64, 9).toString("base64");
|
|
125
|
+
env.cp.sendRaw(JSON.stringify(f));
|
|
126
|
+
if (accepted) {
|
|
127
|
+
await waitFor(() => ran === 1);
|
|
128
|
+
assert.equal(errorsFromLa(env).length, 0, mode);
|
|
129
|
+
} else {
|
|
130
|
+
await waitFor(() => errorsFromLa(env).length === 1);
|
|
131
|
+
assert.equal(ran, 0);
|
|
132
|
+
}
|
|
133
|
+
close(env);
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
test("signature: the signed text is what runs (handler gets the parsed paramsRaw)", async () => {
|
|
138
|
+
const env = await ready();
|
|
139
|
+
let got;
|
|
140
|
+
env.session.setHandler("job.cancel", (ctx) => { got = ctx; return { accept: true }; });
|
|
141
|
+
env.cp.sendRaw(JSON.stringify(env.cp.frameWith("job.cancel", { jobId: JOB, reason: "user" }, { requestId: uuid() })));
|
|
142
|
+
await waitFor(() => got);
|
|
143
|
+
assert.deepEqual(got.params, { jobId: JOB, reason: "user" });
|
|
144
|
+
assert.equal(got.type, "job.cancel");
|
|
145
|
+
assert.equal(got.seq, 1);
|
|
146
|
+
assert.throws(() => { got.type = "x"; }, TypeError, "the context handed to a handler is frozen");
|
|
147
|
+
close(env);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
test("signature: a key the LA does not trust (rotation) is refused in the ready itself", async () => {
|
|
151
|
+
const cp = new ReferenceCp();
|
|
152
|
+
const other = newKeyPair();
|
|
153
|
+
const env = inMemory(cp, { signing: signingFor({ keyId: cp.keyId, publicKeyRaw: other.publicKeyRaw }, "enforce") }).start();
|
|
154
|
+
await waitFor(() => cp.peerErrors.length > 0);
|
|
155
|
+
assert.deepEqual(cp.peerErrors, ["unauthorized"]);
|
|
156
|
+
assert.notEqual(env.session.state, "ready");
|
|
157
|
+
env.session.close();
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
// ------------------------------------------------------------------------------------------ acks and handlers
|
|
161
|
+
|
|
162
|
+
test("ack: no handler -> rejected protocol_unsupported; the seq is still consumed", async () => {
|
|
163
|
+
const env = await ready();
|
|
164
|
+
env.cp.sendData("job.assign", { jobId: JOB, kind: "fix", repoRef: { provider: "github", org: "acme", repo: "billing", branch: "main" },
|
|
165
|
+
policy: { operationMode: "assist", automationLevel: 2, ceiling: "autonomous" }, budget: { tokens: 1000, seconds: 60 }, llm: { alias: "vh-coder", keyRef: "key-la-1" } });
|
|
166
|
+
env.cp.sendData("job.cancel", { jobId: JOB, reason: "user" });
|
|
167
|
+
await waitFor(() => env.cp.acksFromLa.length >= 2);
|
|
168
|
+
assert.deepEqual(env.cp.acksFromLa.map((a) => [a.ackSeq, a.status, a.errorCode]), [[1, "rejected", "protocol_unsupported"], [2, "rejected", "protocol_unsupported"]]);
|
|
169
|
+
assert.equal(env.session.lastRecvSeq, 2);
|
|
170
|
+
assert.equal(env.cp.unacked.size, 0, "a rejected ack settles the frame at the CP");
|
|
171
|
+
close(env);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test("ack: a handler decides accepted / rejected / throws / is slow; every verdict is one ack", async () => {
|
|
175
|
+
const env = await ready({ la: { options: { handlerAckMs: 80 } } });
|
|
176
|
+
const verdicts = [{ accept: true }, { reject: "rate_limited" }, "throw", "slow"];
|
|
177
|
+
let i = 0;
|
|
178
|
+
env.session.setHandler("job.cancel", () => {
|
|
179
|
+
const v = verdicts[i++];
|
|
180
|
+
if (v === "throw") throw new Error("boom " + SECRET);
|
|
181
|
+
if (v === "slow") return new Promise((r) => setTimeout(() => r({ reject: "internal" }), 400));
|
|
182
|
+
return v;
|
|
183
|
+
});
|
|
184
|
+
for (let n = 0; n < 4; n++) env.cp.sendData("job.cancel", { jobId: JOB, reason: "user" });
|
|
185
|
+
await waitFor(() => env.cp.acksFromLa.length >= 4);
|
|
186
|
+
assert.deepEqual(env.cp.acksFromLa.map((a) => [a.ackSeq, a.status, a.errorCode]),
|
|
187
|
+
[[1, "accepted", undefined], [2, "rejected", "rate_limited"], [3, "rejected", "internal"], [4, "accepted", undefined]]);
|
|
188
|
+
assert.equal(sentText(env).includes(SECRET), false);
|
|
189
|
+
close(env);
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
test("dedupe: a retransmission (same seq and msgId, fresh signature) is re-acked as duplicate and NEVER runs twice", async () => {
|
|
193
|
+
const env = await ready();
|
|
194
|
+
let ran = 0;
|
|
195
|
+
env.session.setHandler("job.cancel", () => { ran++; return { accept: true }; });
|
|
196
|
+
const seq = env.cp.sendData("job.cancel", { jobId: JOB, reason: "user" });
|
|
197
|
+
await waitFor(() => env.cp.acksFromLa.length >= 1);
|
|
198
|
+
// the CP never saw the ack: it sends the frame again (what its retransmission timer does, re-signed)
|
|
199
|
+
const u = [...env.wire.toLa].map((t) => JSON.parse(t)).find((f) => f.type === "job.cancel");
|
|
200
|
+
env.cp.cpSeq = seq; // keep numbering; re-render the same seq/msgId with a fresh nonce
|
|
201
|
+
const again = env.cp.frameWith("job.cancel", { jobId: JOB, reason: "user" });
|
|
202
|
+
again.seq = u.seq; again.msgId = u.msgId;
|
|
203
|
+
env.cp.sendRaw(JSON.stringify(again));
|
|
204
|
+
await waitFor(() => env.cp.acksFromLa.length >= 2);
|
|
205
|
+
assert.equal(ran, 1);
|
|
206
|
+
assert.deepEqual(env.cp.acksFromLa.map((a) => a.status), ["accepted", "duplicate"]);
|
|
207
|
+
assert.equal(env.session.stats.duplicatesIn, 1);
|
|
208
|
+
assert.equal(errorsFromLa(env).length, 0);
|
|
209
|
+
close(env);
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
test("dedupe: the last 512 seq->msgId pairs are kept (the 513th old one is a replay)", async () => {
|
|
213
|
+
const env = await ready();
|
|
214
|
+
env.session.setHandler("job.cancel", () => ({ accept: true }));
|
|
215
|
+
for (let n = 0; n < 520; n++) env.cp.sendData("job.cancel", { jobId: JOB, reason: "user" });
|
|
216
|
+
await waitFor(() => env.session.lastRecvSeq === 520, 8000);
|
|
217
|
+
assert.equal(env.session.recvRing.size, 512);
|
|
218
|
+
close(env);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test("a piggy-backed ack field on a CP frame settles our frames", async () => {
|
|
222
|
+
const env = await ready();
|
|
223
|
+
env.cp.silent = true; // the CP does not send ack frames
|
|
224
|
+
env.session.setHandler("job.cancel", () => ({ accept: true }));
|
|
225
|
+
const h = await env.session.enqueue("job.event", { jobId: JOB, state: "running" });
|
|
226
|
+
assert.equal(env.session.unacked.size, 1);
|
|
227
|
+
env.cp.laSeq = 1; // CP has the frame
|
|
228
|
+
env.cp.sendData("job.cancel", { jobId: JOB, reason: "user" }); // carries ack:1
|
|
229
|
+
await h.acked;
|
|
230
|
+
assert.equal(env.session.unacked.size, 0);
|
|
231
|
+
close(env);
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
test("a rejected ack from the CP fails that frame with the code and releases the others", async () => {
|
|
235
|
+
const env = await ready();
|
|
236
|
+
env.cp.silent = true;
|
|
237
|
+
const a = await env.session.enqueue("job.event", { jobId: JOB, state: "running" });
|
|
238
|
+
const b = await env.session.enqueue("job.event", { jobId: JOB, state: "running" });
|
|
239
|
+
const c = await env.session.enqueue("job.event", { jobId: JOB, state: "succeeded" });
|
|
240
|
+
const f = env.cp.frameWith("ack", { ackSeq: 2, status: "rejected", errorCode: "rate_limited" });
|
|
241
|
+
f.type = "ack"; f.seq = 0; delete f.paramsRaw; delete f.sig; f.workspaceId = "";
|
|
242
|
+
env.cp.sendRaw(JSON.stringify(f));
|
|
243
|
+
await a.acked;
|
|
244
|
+
await assert.rejects(b.acked, (e) => e.code === "rate_limited");
|
|
245
|
+
assert.equal(env.session.unacked.has(3), true);
|
|
246
|
+
void c;
|
|
247
|
+
close(env);
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
test("an ack for a frame we never sent is ignored", async () => {
|
|
251
|
+
const env = await ready();
|
|
252
|
+
const f = { schemaVersion: 2, type: "ack", msgId: uuid(), seq: 0, tenantId: TENANT, agentId: AGENT, workspaceId: "", ts: new Date().toISOString(), payload: { ackSeq: 99, status: "accepted" } };
|
|
253
|
+
env.cp.sendRaw(JSON.stringify(f));
|
|
254
|
+
await new Promise((r) => setTimeout(r, 30));
|
|
255
|
+
assert.equal(errorsFromLa(env).length, 0);
|
|
256
|
+
assert.equal(env.session.state, "ready");
|
|
257
|
+
close(env);
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
test("inbound queue is bounded (maxInboundQueue): the overflow is answered rate_limited, the session survives", async () => {
|
|
261
|
+
const env = await ready({ la: { options: { maxInboundQueue: 2 } } });
|
|
262
|
+
env.session.setHandler("job.cancel", () => new Promise((r) => setTimeout(() => r({ accept: true }), 50)));
|
|
263
|
+
for (let n = 0; n < 6; n++) env.cp.sendRaw(JSON.stringify(env.cp.frameWith("job.cancel", { jobId: JOB, reason: "user" })));
|
|
264
|
+
await waitFor(() => errorsFromLa(env).some((e) => e.code === "rate_limited"));
|
|
265
|
+
assert.equal(env.wire.closed.length, 0, "rate_limited is not fatal");
|
|
266
|
+
close(env);
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
test("binary frames from the CP are dropped and counted, never parsed", async () => {
|
|
270
|
+
const env = await ready();
|
|
271
|
+
env.session.handleText("\u0000\u0001", env.transport, true);
|
|
272
|
+
assert.equal(env.session.stats.nonText, 1);
|
|
273
|
+
assert.equal(errorsFromLa(env).length, 0);
|
|
274
|
+
close(env);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
test("sanity: the helper crypto is real (an Ed25519 signature over the v1 canonical string)", () => {
|
|
278
|
+
assert.equal(crypto.getHashes().includes("sha256"), true);
|
|
279
|
+
});
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
// F3-05b: what the Local Agent sends, and what it does about acks, loss, backpressure and resume.
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import assert from "node:assert/strict";
|
|
4
|
+
import crypto from "node:crypto";
|
|
5
|
+
import { ReferenceCp, ReferenceCpServer, TENANT, AGENT, uuid, waitFor } from "./helpers/reference-cp.js";
|
|
6
|
+
import { inMemory, startRig } from "./helpers/la-rig.js";
|
|
7
|
+
import { sendContent, sendContentFailure } from "../../src/la-protocol-v2/content.js";
|
|
8
|
+
import { loadContract } from "../../src/la-protocol-v2/contract.js";
|
|
9
|
+
|
|
10
|
+
const JOB = "c3d4e5f6-a7b8-4c9d-8e0f-2a3b4c5d6e7f";
|
|
11
|
+
const REQ = "d4e5f6a7-b8c9-4d0e-9f1a-3b4c5d6e7f80";
|
|
12
|
+
const ev = (state = "running", extra = {}) => ({ jobId: JOB, state, ...extra });
|
|
13
|
+
const contract = loadContract();
|
|
14
|
+
|
|
15
|
+
async function ready(opts = {}) {
|
|
16
|
+
const cp = new ReferenceCp(opts.cp);
|
|
17
|
+
const env = inMemory(cp, opts.la).start();
|
|
18
|
+
await waitFor(() => env.session.state === "ready");
|
|
19
|
+
return { cp, ...env };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// ------------------------------------------------------------------------------------------ envelope and numbering
|
|
23
|
+
|
|
24
|
+
test("outbound: every frame is closed, schema-valid, carries the configured tenant/agent and consecutive seq", async () => {
|
|
25
|
+
const env = await ready();
|
|
26
|
+
for (let i = 0; i < 5; i++) await env.session.send("job.event", ev("running", { iteration: i }));
|
|
27
|
+
await env.session.send("alert", { code: "disk_low", severity: "warn" }).catch(() => {});
|
|
28
|
+
const frames = env.wire.toCp.map((t) => JSON.parse(t));
|
|
29
|
+
for (const f of frames) {
|
|
30
|
+
assert.deepEqual(contract.validateFrame(f), { ok: true }, f.type);
|
|
31
|
+
assert.equal(f.tenantId, TENANT);
|
|
32
|
+
assert.equal(f.agentId, AGENT);
|
|
33
|
+
assert.equal(f.schemaVersion, 2);
|
|
34
|
+
assert.match(f.msgId, /^[0-9a-f-]{36}$/);
|
|
35
|
+
assert.equal("sig" in f || "paramsRaw" in f, false, "LA frames are never signed");
|
|
36
|
+
}
|
|
37
|
+
assert.deepEqual(frames.filter((f) => f.type === "register" || f.type === "hello").map((f) => f.seq), [0, 0]);
|
|
38
|
+
assert.deepEqual(env.cp.received.filter((f) => f.type === "job.event").map((f) => f.seq), [1, 2, 3, 4, 5]);
|
|
39
|
+
assert.equal(new Set(env.cp.received.map((f) => f.msgId)).size, env.cp.received.length, "msgId is unique per frame");
|
|
40
|
+
env.session.close();
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("outbound: our frames piggyback ack = highest contiguous CP seq received", async () => {
|
|
44
|
+
const env = await ready();
|
|
45
|
+
env.session.setHandler("job.cancel", () => ({ accept: true }));
|
|
46
|
+
env.cp.sendData("job.cancel", { jobId: JOB, reason: "user" });
|
|
47
|
+
env.cp.sendData("job.cancel", { jobId: JOB, reason: "user" });
|
|
48
|
+
await waitFor(() => env.session.lastRecvSeq === 2);
|
|
49
|
+
await env.session.send("job.event", ev());
|
|
50
|
+
assert.equal(env.cp.received.at(-1).ack, 2);
|
|
51
|
+
env.session.close();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("outbound: heartbeat frames go out on their own, with mode/clock/disk/queue (no paths, no content)", async () => {
|
|
55
|
+
const env = await ready({ la: { options: { heartbeatMs: 40 } } });
|
|
56
|
+
await waitFor(() => env.cp.received.filter((f) => f.type === "heartbeat").length >= 2);
|
|
57
|
+
const hb = env.cp.received.find((f) => f.type === "heartbeat");
|
|
58
|
+
assert.deepEqual(Object.keys(hb.payload).sort(), ["clockEpochMs", "diskFreeBytes", "inflight", "queueDepth", "signatureMode"]);
|
|
59
|
+
assert.equal(hb.payload.diskFreeBytes, 1234);
|
|
60
|
+
assert.equal(hb.payload.signatureMode, "enforce");
|
|
61
|
+
env.session.close();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
// ------------------------------------------------------------------------------------------ refusals before a byte is written
|
|
65
|
+
|
|
66
|
+
const REFUSED = [
|
|
67
|
+
["a type outside the allowlist", "shell.exec", {}, "unknown_type"],
|
|
68
|
+
["a v1 command name", "mcp.execute-command", {}, "unknown_type"],
|
|
69
|
+
["a cp->la type", "job.assign", {}, "wrong_direction"],
|
|
70
|
+
["a session type (register)", "register", {}, "wrong_direction"],
|
|
71
|
+
["a field the schema does not know", "job.event", ev("running", { stack: "x" }), "schema_violation"],
|
|
72
|
+
["a state outside the enum", "job.event", ev("exploded"), "schema_violation"],
|
|
73
|
+
["a string above its maxLength", "job.result", { jobId: JOB, outcome: "pr_opened", prUrl: "https://example.com/" + "a".repeat(600) }, "payload_too_large"],
|
|
74
|
+
];
|
|
75
|
+
for (const [name, type, payload, code] of REFUSED) {
|
|
76
|
+
test(`outbound refusal: ${name} -> ${code}, nothing is written`, async () => {
|
|
77
|
+
const env = await ready();
|
|
78
|
+
const before = env.wire.toCp.length;
|
|
79
|
+
await assert.rejects(env.session.enqueue(type, payload), (e) => e.code === code);
|
|
80
|
+
assert.equal(env.wire.toCp.length, before);
|
|
81
|
+
assert.equal(env.session.sendSeq, 0, "a refused frame does not consume a seq");
|
|
82
|
+
env.session.close();
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
test("outbound refusal: before the first ready nothing can be sent (not_ready)", async () => {
|
|
87
|
+
const cp = new ReferenceCp();
|
|
88
|
+
const env = inMemory(cp);
|
|
89
|
+
await assert.rejects(env.session.enqueue("job.event", ev()), (e) => e.code === "not_ready");
|
|
90
|
+
env.session.close();
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("outbound refusal: the send queue is bounded (rate_limited)", async () => {
|
|
94
|
+
const env = await ready({ cp: { }, la: { options: { maxInFlight: 1, maxQueued: 3 } } });
|
|
95
|
+
env.cp.silent = true;
|
|
96
|
+
const ps = [];
|
|
97
|
+
for (let i = 0; i < 6; i++) ps.push(env.session.enqueue("job.event", ev()).then(() => "ok", (e) => e.code));
|
|
98
|
+
const res = await Promise.race([Promise.all(ps), new Promise((r) => setTimeout(() => r(null), 100))]);
|
|
99
|
+
assert.equal(res, null, "waiting frames keep waiting");
|
|
100
|
+
const refused = await Promise.all(ps.slice(-2));
|
|
101
|
+
assert.deepEqual(refused, ["rate_limited", "rate_limited"]);
|
|
102
|
+
env.session.close();
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
// ------------------------------------------------------------------------------------------ acks, loss, backpressure
|
|
106
|
+
|
|
107
|
+
test("backpressure: at most maxInFlight frames unacked, the rest wait; when acks resume everything arrives once and in order", async () => {
|
|
108
|
+
const env = await ready({ cp: { limits: { ackTimeoutMs: 150, maxRetries: 3 } }, la: { options: { maxInFlight: 3 } } });
|
|
109
|
+
env.cp.silent = true; // acks lost
|
|
110
|
+
const all = [];
|
|
111
|
+
for (let i = 0; i < 10; i++) all.push(env.session.send("job.event", ev("running", { iteration: i })));
|
|
112
|
+
await waitFor(() => env.cp.received.length === 3);
|
|
113
|
+
await new Promise((r) => setTimeout(r, 40));
|
|
114
|
+
assert.equal(env.cp.received.length, 3, "never more than maxInFlight on the wire");
|
|
115
|
+
assert.equal(env.session.snapshot.waiting, 7);
|
|
116
|
+
assert.equal(env.session.snapshot.unacked, 3);
|
|
117
|
+
env.cp.silent = false; // the retransmissions of 1..3 are now acked (as duplicates)
|
|
118
|
+
await Promise.all(all);
|
|
119
|
+
assert.deepEqual(env.cp.received.map((f) => f.payload.iteration), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
|
|
120
|
+
assert.deepEqual(env.cp.received.map((f) => f.seq), [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
|
|
121
|
+
assert.ok(env.session.stats.retransmits >= 1, "the lost acks forced a retransmission (cumulative acks settle the rest)");
|
|
122
|
+
env.session.close();
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("ack lost: the SAME bytes (same seq and msgId) are sent again, the CP sees a duplicate, the caller's ack resolves", async () => {
|
|
126
|
+
const env = await ready({ cp: { limits: { ackTimeoutMs: 100 } } });
|
|
127
|
+
env.cp.dropAcks = 1;
|
|
128
|
+
await env.session.send("job.event", ev());
|
|
129
|
+
const copies = env.wire.toCp.filter((t) => t.includes('"job.event"'));
|
|
130
|
+
assert.equal(copies.length, 2);
|
|
131
|
+
assert.equal(copies[0], copies[1], "a retransmission is byte-identical");
|
|
132
|
+
assert.deepEqual(env.cp.duplicates, [1]);
|
|
133
|
+
assert.equal(env.cp.received.length, 1);
|
|
134
|
+
assert.equal(env.session.stats.retransmits, 1);
|
|
135
|
+
env.session.close();
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("retries exhausted: the caller gets timeout, the connection is closed with 4420, the frame is kept for a resume", async () => {
|
|
139
|
+
const env = await ready({ cp: { limits: { ackTimeoutMs: 100, maxRetries: 2 } } });
|
|
140
|
+
env.cp.silent = true;
|
|
141
|
+
const h = await env.session.enqueue("job.event", ev());
|
|
142
|
+
await assert.rejects(h.acked, (e) => e.code === "timeout");
|
|
143
|
+
assert.deepEqual(env.wire.closed[0], [4420, "ack_timeout"]);
|
|
144
|
+
assert.equal(env.session.stats.retransmits, 2);
|
|
145
|
+
assert.equal(env.session.state, "detached");
|
|
146
|
+
assert.equal(env.session.unacked.size, 1, "kept for the resume");
|
|
147
|
+
env.session.close();
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
// ------------------------------------------------------------------------------------------ content (chunking)
|
|
151
|
+
|
|
152
|
+
const bytesOf = (n) => crypto.randomBytes(n);
|
|
153
|
+
|
|
154
|
+
test("chunking: 100 KiB goes out as 4 ordered chunks, each <= 32 KiB raw, each frame <= 48 KiB, and the CP rebuilds it exactly", async () => {
|
|
155
|
+
const env = await ready();
|
|
156
|
+
const data = bytesOf(100 * 1024);
|
|
157
|
+
const r = await sendContent(env.session, { requestId: REQ, source: data, contentType: "application/octet-stream" });
|
|
158
|
+
assert.deepEqual(r, { chunks: 4, bytes: data.length });
|
|
159
|
+
const frames = env.cp.received.filter((f) => f.type === "content.response");
|
|
160
|
+
assert.deepEqual(frames.map((f) => f.chunk), [0, 1, 2, 3].map((index) => ({ index, total: 4, last: index === 3 })));
|
|
161
|
+
assert.deepEqual(frames.map((f) => f.payload.byteLength), [32768, 32768, 32768, 4096]);
|
|
162
|
+
assert.ok(env.cp.rawIn.every((t) => Buffer.byteLength(t) <= 49152));
|
|
163
|
+
assert.ok(Math.max(...env.cp.rawIn.map((t) => Buffer.byteLength(t))) > 43692, "a full chunk really is that big");
|
|
164
|
+
assert.equal(frames[0].requestId, REQ);
|
|
165
|
+
assert.ok(frames.every((f) => f.payload.totalBytes === data.length && f.payload.encoding === "base64"));
|
|
166
|
+
const c = env.cp.contentRequests.get(REQ);
|
|
167
|
+
assert.equal(c.done, true);
|
|
168
|
+
assert.ok(Buffer.concat(c.chunks).equals(data));
|
|
169
|
+
assert.deepEqual(env.cp.errorsSent, []);
|
|
170
|
+
env.session.close();
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test("chunking: exactly 2 MiB = 64 chunks is accepted, 2 MiB + 1 is refused before a single chunk is sent", async () => {
|
|
174
|
+
const env = await ready();
|
|
175
|
+
const max = bytesOf(2 * 1024 * 1024);
|
|
176
|
+
const r = await sendContent(env.session, { requestId: REQ, source: max });
|
|
177
|
+
assert.equal(r.chunks, 64);
|
|
178
|
+
assert.ok(Buffer.concat(env.cp.contentRequests.get(REQ).chunks).equals(max));
|
|
179
|
+
const before = env.wire.toCp.length;
|
|
180
|
+
await assert.rejects(sendContent(env.session, { requestId: uuid(), source: bytesOf(2 * 1024 * 1024 + 1) }), (e) => e.code === "payload_too_large");
|
|
181
|
+
assert.equal(env.wire.toCp.length, before);
|
|
182
|
+
env.session.close();
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
test("chunking: empty content is one empty chunk; a failure answer is one empty chunk with the status", async () => {
|
|
186
|
+
const env = await ready();
|
|
187
|
+
await sendContent(env.session, { requestId: REQ, source: Buffer.alloc(0), contentType: "text/plain" });
|
|
188
|
+
const rid2 = uuid();
|
|
189
|
+
await sendContentFailure(env.session, { requestId: rid2, status: "not_found", errorCode: "internal" });
|
|
190
|
+
const [a, b] = env.cp.received;
|
|
191
|
+
assert.deepEqual(a.chunk, { index: 0, total: 1, last: true });
|
|
192
|
+
assert.equal(a.payload.data, "");
|
|
193
|
+
assert.equal(b.payload.status, "not_found");
|
|
194
|
+
assert.deepEqual(b.chunk, { index: 0, total: 1, last: true });
|
|
195
|
+
env.session.close();
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
test("chunking + backpressure: with a slow CP never more than maxInFlight chunks are unacked", async () => {
|
|
199
|
+
const env = await ready({ cp: { }, la: { options: { maxInFlight: 3 } } });
|
|
200
|
+
env.cp.ackDelayMs = 15;
|
|
201
|
+
let maxUnacked = 0;
|
|
202
|
+
env.cp.onReceived = () => { maxUnacked = Math.max(maxUnacked, env.session.unacked.size); };
|
|
203
|
+
const data = bytesOf(40 * 32768);
|
|
204
|
+
await sendContent(env.session, { requestId: REQ, source: data });
|
|
205
|
+
assert.ok(maxUnacked <= 3 && maxUnacked >= 2, `max unacked ${maxUnacked}`);
|
|
206
|
+
assert.ok(Buffer.concat(env.cp.contentRequests.get(REQ).chunks).equals(data));
|
|
207
|
+
assert.equal(env.session.stats.retransmits, 0);
|
|
208
|
+
env.session.close();
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
test("chunking is lazy: the next chunk is not read until a slot opens (bounded memory)", async () => {
|
|
212
|
+
const env = await ready({ la: { options: { maxInFlight: 2 } } });
|
|
213
|
+
env.cp.silent = true;
|
|
214
|
+
let reads = 0;
|
|
215
|
+
const source = { size: 10 * 32768, async read(offset, length) { reads++; return Buffer.alloc(length, 1); } };
|
|
216
|
+
const p = sendContent(env.session, { requestId: REQ, source });
|
|
217
|
+
p.catch(() => {});
|
|
218
|
+
await waitFor(() => env.cp.received.length === 2);
|
|
219
|
+
await new Promise((r) => setTimeout(r, 60));
|
|
220
|
+
assert.ok(reads <= 3, `read ${reads} chunks while only 2 can be in flight`);
|
|
221
|
+
env.session.close();
|
|
222
|
+
await assert.rejects(p);
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
test("chunking: cancelling (AbortSignal) stops reading and drops the chunks still waiting", async () => {
|
|
226
|
+
const env = await ready({ la: { options: { maxInFlight: 2 } } });
|
|
227
|
+
env.cp.ackDelayMs = 30;
|
|
228
|
+
const ac = new AbortController();
|
|
229
|
+
let reads = 0;
|
|
230
|
+
const source = { size: 20 * 32768, async read(o, l) { reads++; if (reads === 4) ac.abort(); return Buffer.alloc(l, 2); } };
|
|
231
|
+
await assert.rejects(sendContent(env.session, { requestId: REQ, source, signal: ac.signal }), (e) => e.code === "agent_offline");
|
|
232
|
+
assert.ok(reads <= 5);
|
|
233
|
+
assert.equal(env.session.snapshot.waiting, 0);
|
|
234
|
+
env.session.close();
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
// the chunk rules of the contract applied to what we send (the invalid-chunk-* traces, from the sender's side)
|
|
238
|
+
const CHUNK = (data = Buffer.alloc(26, 1)) => ({ status: "ok", contentType: "text/plain", encoding: "base64", data: data.toString("base64"), byteLength: data.length });
|
|
239
|
+
const BAD_CHUNKS = [
|
|
240
|
+
["out of order (0 then 2)", [[0, 3, false], [2, 3, true]], "chunk_out_of_order"],
|
|
241
|
+
["after last", [[0, 1, true], [1, 2, true]], "chunk_out_of_order"],
|
|
242
|
+
["total changes mid-stream", [[0, 3, false], [1, 4, false]], "chunk_inconsistent"],
|
|
243
|
+
["last on the wrong index", [[0, 3, false], [1, 3, true]], "chunk_inconsistent"],
|
|
244
|
+
["last missing on the last index", [[0, 1, false]], "chunk_inconsistent"],
|
|
245
|
+
["a stream that does not start at 0", [[1, 3, false]], "chunk_out_of_order"],
|
|
246
|
+
];
|
|
247
|
+
for (const [name, steps, code] of BAD_CHUNKS) {
|
|
248
|
+
test(`chunk rules on send: ${name} -> ${code}`, async () => {
|
|
249
|
+
const env = await ready();
|
|
250
|
+
let got = null;
|
|
251
|
+
for (const [index, total, last] of steps) {
|
|
252
|
+
try { await env.session.send("content.response", CHUNK(), { requestId: uuid() === "" ? "" : REQ, chunk: { index, total, last } }); }
|
|
253
|
+
catch (e) { got = e.code; break; }
|
|
254
|
+
}
|
|
255
|
+
assert.equal(got, code);
|
|
256
|
+
env.session.close();
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
test("chunk rules on send: byteLength must equal the decoded size; data above 32 KiB raw is refused", async () => {
|
|
261
|
+
const env = await ready();
|
|
262
|
+
const p = CHUNK();
|
|
263
|
+
p.byteLength = 27;
|
|
264
|
+
await assert.rejects(env.session.send("content.response", p, { requestId: REQ, chunk: { index: 0, total: 1, last: true } }), (e) => e.code === "chunk_inconsistent");
|
|
265
|
+
const big = bytesOf(32769);
|
|
266
|
+
await assert.rejects(env.session.send("content.response", { ...CHUNK(big) }, { requestId: uuid(), chunk: { index: 0, total: 1, last: true } }),
|
|
267
|
+
(e) => ["payload_too_large", "schema_violation", "chunk_inconsistent"].includes(e.code));
|
|
268
|
+
assert.equal(env.cp.received.length, 0);
|
|
269
|
+
env.session.close();
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
// ------------------------------------------------------------------------------------------ resume (real sockets)
|
|
273
|
+
|
|
274
|
+
// every ws test runs inside withRig: sockets and servers are closed even when an assertion throws
|
|
275
|
+
async function withRig(opts, body) {
|
|
276
|
+
const srv = await new ReferenceCpServer({ requireEnforce: true, ...(opts.cp || {}) }).start();
|
|
277
|
+
const rig = startRig(srv, opts.rig);
|
|
278
|
+
try {
|
|
279
|
+
await rig.connect();
|
|
280
|
+
await waitFor(() => rig.session.state === "ready");
|
|
281
|
+
await body({ srv, rig });
|
|
282
|
+
} finally {
|
|
283
|
+
await rig.stop();
|
|
284
|
+
await srv.stop();
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
test("resume: after a dropped socket the LA offers {sessionId,lastRecvSeq}, continues numbering, and the CP gets one copy of everything", async () => {
|
|
289
|
+
await withRig({}, async ({ srv, rig }) => {
|
|
290
|
+
rig.session.setHandler("job.cancel", () => ({ accept: true }));
|
|
291
|
+
await rig.session.send("job.event", ev("running", { iteration: 1 }));
|
|
292
|
+
await rig.session.send("job.event", ev("running", { iteration: 2 }));
|
|
293
|
+
srv.cp.sendData("job.cancel", { jobId: JOB, reason: "user" });
|
|
294
|
+
await waitFor(() => rig.session.lastRecvSeq === 1);
|
|
295
|
+
const sid = rig.session.sessionId;
|
|
296
|
+
rig.drop();
|
|
297
|
+
await waitFor(() => rig.session.state === "detached");
|
|
298
|
+
const queued = rig.session.send("job.event", ev("running", { iteration: 3 })); // queued while detached
|
|
299
|
+
await rig.connect();
|
|
300
|
+
await queued;
|
|
301
|
+
const reg = srv.cp.registers.at(-1);
|
|
302
|
+
assert.deepEqual(reg.resume, { sessionId: sid, lastRecvSeq: 1 });
|
|
303
|
+
assert.equal(rig.session.sessionId, sid);
|
|
304
|
+
assert.deepEqual(srv.cp.received.filter((f) => f.type === "job.event").map((f) => [f.seq, f.payload.iteration]), [[1, 1], [2, 2], [3, 3]]);
|
|
305
|
+
assert.equal(srv.cp.hellos.length, 2);
|
|
306
|
+
});});
|
|
307
|
+
|
|
308
|
+
test("resume: frames the CP never saw are sent again (same seq and msgId) and are acked after the reconnect", async () => {
|
|
309
|
+
await withRig({ cp: { limits: { ackTimeoutMs: 5000 } } }, async ({ srv, rig }) => {
|
|
310
|
+
srv.cp.silent = true;
|
|
311
|
+
const h = await rig.session.enqueue("job.event", ev("running", { iteration: 1 }));
|
|
312
|
+
await waitFor(() => srv.cp.received.length === 1);
|
|
313
|
+
rig.drop();
|
|
314
|
+
await waitFor(() => rig.session.state === "detached");
|
|
315
|
+
srv.cp.silent = false;
|
|
316
|
+
await rig.connect();
|
|
317
|
+
await h.acked; // ready{peerLastSeq:1} settles it
|
|
318
|
+
assert.equal(srv.cp.received.length, 1, "no second copy");
|
|
319
|
+
assert.equal(rig.session.unacked.size, 0);
|
|
320
|
+
});});
|
|
321
|
+
|
|
322
|
+
test("resume: a frame lost with the socket is delivered after the reconnect (retransmitted once, not twice)", async () => {
|
|
323
|
+
await withRig({}, async ({ srv, rig }) => {
|
|
324
|
+
// write the frame, then lose the connection before the CP reads it
|
|
325
|
+
srv.ws.pause?.();
|
|
326
|
+
rig.ws.terminate();
|
|
327
|
+
await waitFor(() => rig.session.state === "detached");
|
|
328
|
+
const p = rig.session.send("job.event", ev("running", { iteration: 9 }));
|
|
329
|
+
await rig.connect();
|
|
330
|
+
await p;
|
|
331
|
+
assert.deepEqual(srv.cp.received.map((f) => f.payload.iteration), [9]);
|
|
332
|
+
});});
|
|
333
|
+
|
|
334
|
+
test("resumed:false (the CP lost the session): numbering restarts at 1 and frames that were in flight fail with agent_offline", async () => {
|
|
335
|
+
await withRig({}, async ({ srv, rig }) => {
|
|
336
|
+
srv.cp.silent = true;
|
|
337
|
+
const a = await rig.session.enqueue("job.event", ev("running", { iteration: 1 }));
|
|
338
|
+
const b = await rig.session.enqueue("job.event", ev("running", { iteration: 2 }));
|
|
339
|
+
await waitFor(() => srv.cp.received.length === 2);
|
|
340
|
+
const old = rig.session.sessionId;
|
|
341
|
+
rig.drop();
|
|
342
|
+
await waitFor(() => rig.session.state === "detached");
|
|
343
|
+
srv.cp.sessionId = uuid(); // the CP has no memory of this session
|
|
344
|
+
srv.cp.silent = false;
|
|
345
|
+
await rig.connect();
|
|
346
|
+
await assert.rejects(a.acked, (e) => e.code === "agent_offline");
|
|
347
|
+
await assert.rejects(b.acked, (e) => e.code === "agent_offline");
|
|
348
|
+
assert.notEqual(rig.session.sessionId, old);
|
|
349
|
+
await rig.session.send("job.event", ev("running", { iteration: 3 }));
|
|
350
|
+
assert.equal(srv.cp.received.at(-1).seq, 1, "both sides restart at 1");
|
|
351
|
+
});});
|
|
352
|
+
|
|
353
|
+
test("resume window: after resumeWindowMs a dropped session is forgotten (no resume offered, in-flight fail)", async () => {
|
|
354
|
+
await withRig({ rig: { options: { resumeWindowMs: 80 } } }, async ({ srv, rig }) => {
|
|
355
|
+
srv.cp.silent = true;
|
|
356
|
+
const h = await rig.session.enqueue("job.event", ev());
|
|
357
|
+
await waitFor(() => srv.cp.received.length === 1);
|
|
358
|
+
rig.drop();
|
|
359
|
+
await waitFor(() => rig.session.state === "idle");
|
|
360
|
+
await assert.rejects(h.acked, (e) => e.code === "agent_offline");
|
|
361
|
+
srv.cp.silent = false;
|
|
362
|
+
await rig.connect();
|
|
363
|
+
await waitFor(() => rig.session.state === "ready");
|
|
364
|
+
assert.equal(srv.cp.registers.at(-1).resume, undefined);
|
|
365
|
+
});});
|
|
366
|
+
|
|
367
|
+
test("resume: a content stream does not survive a reset (its chunks would reach a CP that never saw chunk 0)", async () => {
|
|
368
|
+
await withRig({ rig: { options: { maxInFlight: 1 } } }, async ({ srv, rig }) => {
|
|
369
|
+
srv.cp.silent = true;
|
|
370
|
+
const p = sendContent(rig.session, { requestId: REQ, source: bytesOf(5 * 32768) });
|
|
371
|
+
p.catch(() => {});
|
|
372
|
+
await waitFor(() => srv.cp.received.length === 1);
|
|
373
|
+
rig.drop();
|
|
374
|
+
await waitFor(() => rig.session.state === "detached");
|
|
375
|
+
srv.cp.sessionId = uuid();
|
|
376
|
+
srv.cp.silent = false;
|
|
377
|
+
await rig.connect();
|
|
378
|
+
await assert.rejects(p, (e) => e.code === "agent_offline");
|
|
379
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
380
|
+
assert.equal(srv.cp.received.filter((f) => f.type === "content.response").length, 1);
|
|
381
|
+
assert.deepEqual(srv.cp.errorsSent, []);
|
|
382
|
+
});});
|