deepdebug-local-agent 1.8.0 → 1.10.0

package/test/la-protocol-v2/webhook-receiver.test.js

@@ -0,0 +1,702 @@
+// F3-12 consumer (contract 2.1 (b)): the webhook.forward receiver of the Local Agent. Chunks in order / out of order / duplicated / wrong
+// sha / over every limit, the ack AFTER the local delivery, the 8 traces of the contract through the REAL session, the flag off, and an
+// end-to-end through buildV2Link + the incident detector with the canaries searched in everything the CP receives.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { ReferenceCp, TENANT, AGENT, waitFor, uuid } from "./helpers/reference-cp.js";
+import { inMemory, signingFor } from "./helpers/la-rig.js";
+import { loadContract } from "../../src/la-protocol-v2/contract.js";
+import { buildV2Link } from "../../src/la-protocol-v2/runtime.js";
+import { resolveWebhookConfig } from "../../src/la-protocol-v2/webhooks/config.js";
+import { createWebhookAssembler } from "../../src/la-protocol-v2/webhooks/assembler.js";
+import { createWebhookHandler } from "../../src/la-protocol-v2/webhooks/handler.js";
+import { entriesFromWebhook } from "../../src/incident-detector/sources/webhook-source.js";
+
+const LIMITS = loadContract().limits;
+const RAW = LIMITS.webhookChunkRawBytes;
+const noop = () => {};
+const cfg = (over = {}) => ({ ...resolveWebhookConfig({ LOCAL_AGENT_WEBHOOK_FORWARD: "true" }, noop, LIMITS), ...over });
+const sha = (b) => crypto.createHash("sha256").update(b).digest("hex");
+const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), "la-wh-"));
+const TRACES = path.join(path.dirname(fileURLToPath(import.meta.url)), "../contract/la-protocol-v2/vectors/traces");
+
+/** The chunks of a body exactly as the contract has them: 16384 raw bytes each, the last one the rest. */
+function chunksOf(body, { webhookId = uuid(), source = "sentry", contentType = "application/json" } = {}) {
+    const total = Math.max(1, Math.ceil(body.length / RAW));
+    const bodySha256 = sha(body);
+    return Array.from({ length: total }, (_, index) => ({
+        params: { webhookId, source, contentType, bodyB64: body.subarray(index * RAW, (index + 1) * RAW).toString("base64"), bodyBytes: body.length, bodySha256 },
+        chunk: { index, total, last: index === total - 1 },
+    }));
+}
+const bodyOf = (n) => Buffer.from(Array.from({ length: n }, (_, i) => (i * 31 + 7) & 0xff));
+
+// ------------------------------------------------------------------------------------------ the assembler
+
+test("assembler: chunks in order rebuild the body, the sha256 is verified, the webhook is handed over once", () => {
+    const a = createWebhookAssembler({ limits: LIMITS, config: cfg() });
+    const body = bodyOf(RAW * 2 + 100);
+    const fr = chunksOf(body);
+    assert.equal(fr.length, 3);
+    assert.deepEqual(a.accept(fr[0]), { ok: true, done: false });
+    assert.equal(a.openCount, 1);
+    assert.deepEqual(a.accept(fr[1]), { ok: true, done: false });
+    const r = a.accept(fr[2]);
+    assert.equal(r.ok && r.done, true);
+    assert.ok(r.webhook.body.equals(body));
+    assert.equal(r.webhook.source, "sentry");
+    assert.equal(a.openCount, 0);
+    assert.equal(a.stats.bytesHeld, 0);
+});
+
+test("assembler: a 1 MiB body in 64 chunks, and a body of exactly 1 chunk (total 1)", () => {
+    const a = createWebhookAssembler({ limits: LIMITS, config: cfg() });
+    const big = bodyOf(1048576);
+    const fr = chunksOf(big);
+    assert.equal(fr.length, 64);
+    let last;
+    for (const f of fr) last = a.accept(f);
+    assert.ok(last.done && last.webhook.body.equals(big));
+    const small = chunksOf(bodyOf(100));
+    assert.equal(small.length, 1);
+    assert.ok(a.accept(small[0]).done);
+});
+
+test("assembler: out of order (the first chunk seen is not index 0; index skipped; index repeated) -> chunk_out_of_order, the assembly is dropped", () => {
+    const body = bodyOf(RAW * 3);
+    const mk = () => createWebhookAssembler({ limits: LIMITS, config: cfg() });
+    let a = mk();
+    let fr = chunksOf(body);
+    assert.deepEqual(a.accept(fr[1]), { ok: false, code: "chunk_out_of_order" });
+    a = mk(); fr = chunksOf(body);
+    a.accept(fr[0]);
+    assert.deepEqual(a.accept(fr[2]), { ok: false, code: "chunk_out_of_order" });
+    assert.equal(a.openCount, 0, "dropped");
+    a = mk(); fr = chunksOf(body);
+    a.accept(fr[0]); a.accept(fr[1]);
+    assert.deepEqual(a.accept(fr[1]), { ok: false, code: "chunk_out_of_order" }, "the same index twice (duplicated with a new seq)");
+    assert.equal(a.accept(fr[2]).code, "chunk_out_of_order", "and then the assembly is gone");
+});
+
+test("assembler: a chunk after the one flagged last -> chunk_out_of_order (the assembly is already delivered)", () => {
+    const a = createWebhookAssembler({ limits: LIMITS, config: cfg() });
+    const fr = chunksOf(bodyOf(40));
+    assert.ok(a.accept(fr[0]).done);
+    assert.deepEqual(a.accept({ params: fr[0].params, chunk: { index: 1, total: 1, last: true } }), { ok: false, code: "chunk_out_of_order" });
+});
+
+test("assembler: sha256 that is not the hash of the rebuilt body -> chunk_inconsistent on the last chunk", () => {
+    const a = createWebhookAssembler({ limits: LIMITS, config: cfg() });
+    const body = bodyOf(RAW + 50);
+    const fr = chunksOf(body);
+    for (const f of fr) f.params.bodySha256 = sha(Buffer.from("something else"));
+    assert.deepEqual(a.accept(fr[0]), { ok: true, done: false });
+    assert.deepEqual(a.accept(fr[1]), { ok: false, code: "chunk_inconsistent" });
+    assert.equal(a.openCount, 0);
+});
+
+test("assembler: what must not change between chunks (total, source, contentType, bodyBytes, bodySha256) -> chunk_inconsistent", () => {
+    const body = bodyOf(RAW * 2 + 5);
+    for (const mutate of [
+        (f) => { f.chunk.total = 4; },
+        (f) => { f.params.source = "sonar"; },
+        (f) => { f.params.contentType = "text/plain"; },
+        (f) => { f.params.bodySha256 = "0".repeat(64); },
+        (f) => { f.params.bodyBytes = body.length + 1; },
+    ]) {
+        const a = createWebhookAssembler({ limits: LIMITS, config: cfg() });
+        const fr = chunksOf(body);
+        a.accept(fr[0]);
+        mutate(fr[1]);
+        assert.equal(a.accept(fr[1]).code, "chunk_inconsistent", mutate.toString());
+    }
+});
+
+test("assembler: last flag wrong, and the size rule (non-last too short, last of the wrong size) -> chunk_inconsistent", () => {
+    const body = bodyOf(RAW + 10);
+    let a = createWebhookAssembler({ limits: LIMITS, config: cfg() });
+    let fr = chunksOf(body);
+    a.accept(fr[0]);
+    fr[1].chunk.last = false;
+    assert.equal(a.accept(fr[1]).code, "chunk_inconsistent");
+    a = createWebhookAssembler({ limits: LIMITS, config: cfg() });
+    fr = chunksOf(body);
+    fr[0].params.bodyB64 = Buffer.alloc(RAW - 1).toString("base64");
+    assert.equal(a.accept(fr[0]).code, "chunk_inconsistent", "a non-last chunk must be exactly 16384 bytes");
+    a = createWebhookAssembler({ limits: LIMITS, config: cfg() });
+    fr = chunksOf(body);
+    a.accept(fr[0]);
+    fr[1].params.bodyB64 = Buffer.alloc(11).toString("base64");
+    assert.equal(a.accept(fr[1]).code, "chunk_inconsistent", "the last chunk is the rest");
+});
+
+test("assembler limits: open assemblies, bytes buffered, body above the cap, 30 s timeout -> rate_limited / payload_too_large / dropped and zeroed", () => {
+    // open assemblies
+    let a = createWebhookAssembler({ limits: LIMITS, config: cfg({ maxOpen: 2 }) });
+    const big = bodyOf(RAW * 2);
+    for (let i = 0; i < 2; i++) assert.ok(a.accept(chunksOf(big)[0]).ok);
+    assert.deepEqual(a.accept(chunksOf(big)[0]), { ok: false, code: "rate_limited" });
+    assert.equal(a.openCount, 2, "the refused one is not kept and the open ones survive");
+    // bytes buffered
+    a = createWebhookAssembler({ limits: LIMITS, config: cfg({ maxBufferedBytes: RAW * 3 }) });
+    assert.ok(a.accept(chunksOf(big)[0]).ok);
+    assert.equal(a.accept(chunksOf(big)[0]).code, "rate_limited");
+    assert.equal(a.stats.bytesHeld, RAW * 2);
+    // body above the configured cap
+    a = createWebhookAssembler({ limits: LIMITS, config: cfg({ maxBodyBytes: RAW }) });
+    assert.equal(a.accept(chunksOf(big)[0]).code, "payload_too_large");
+    // timeout (fake clock + fake timers)
+    let t = 1_000_000;
+    const pending = [];
+    const timers = { setTimeout: (fn, ms) => { const h = { fn, ms, cleared: false }; pending.push(h); return h; }, clearTimeout: (h) => { if (h) h.cleared = true; } };
+    a = createWebhookAssembler({ limits: LIMITS, config: cfg(), now: () => t, timers });
+    const fr = chunksOf(big);
+    a.accept(fr[0]);
+    assert.equal(pending[0].ms, 30_000);
+    t += 30_001;
+    pending[0].fn();
+    assert.equal(a.openCount, 0);
+    assert.equal(a.stats.expired, 1);
+    assert.equal(a.accept(fr[1]).code, "chunk_out_of_order", "a late chunk finds nothing");
+    // lazy expiry even if the timer never ran (clock check at accept time)
+    a = createWebhookAssembler({ limits: LIMITS, config: cfg(), now: () => t, timers });
+    const g = chunksOf(big);
+    a.accept(g[0]);
+    t += 31_000;
+    assert.equal(a.accept(g[1]).code, "chunk_out_of_order");
+});
+
+test("assembler: a refusal and a clear() zero the buffer (nothing of the body stays in the dropped assembly)", () => {
+    const a = createWebhookAssembler({ limits: LIMITS, config: cfg() });
+    const body = Buffer.alloc(RAW * 2, 0x41);
+    const fr = chunksOf(body);
+    a.accept(fr[0]);
+    assert.equal(a.openCount, 1);
+    a.clear();
+    assert.equal(a.openCount, 0);
+    assert.equal(a.stats.bytesHeld, 0);
+});
+
+// ------------------------------------------------------------------------------------------ the handler on the real session
+
+async function rig({ sink, config = cfg(), sessionOptions = {}, mode = "enforce" } = {}) {
+    const cp = new ReferenceCp({ capabilities: ["incident.candidate.size-model"] });
+    const env = inMemory(cp, { mode, options: { handlerAckMs: 5000, ...sessionOptions } }).start();
+    await waitFor(() => env.session.state === "ready");
+    const delivered = [];
+    const handler = createWebhookHandler({ config, getSink: () => (sink === undefined ? async (w) => { delivered.push({ ...w, body: Buffer.from(w.body) }); } : sink) });
+    env.session.setHandler("webhook.forward", (ctx) => handler.handle(ctx));
+    return { cp, env, handler, delivered, session: env.session };
+}
+const send = (cp, f) => cp.sendData("webhook.forward", f.params, f.chunk ? { chunk: f.chunk } : {});
+
+test("a webhook of at most 9216 bytes in ONE frame (no chunk) is delivered and acked accepted", async () => {
+    const { cp, delivered, session } = await rig();
+    const body = Buffer.from(JSON.stringify({ hello: "world" }));
+    send(cp, { params: { webhookId: uuid(), source: "other", contentType: "application/json", bodyB64: body.toString("base64") } });
+    await waitFor(() => cp.acksFromLa.length === 1);
+    assert.deepEqual(cp.acksFromLa[0], { ackSeq: 1, status: "accepted", errorCode: undefined });
+    assert.equal(delivered.length, 1);
+    assert.ok(delivered[0].body.equals(body));
+    session.close();
+});
+
+test("chunked: every chunk is acked accepted in order, the LAST ack only after the sink returned (ack AFTER the local delivery)", async () => {
+    let release;
+    const gate = new Promise((r) => { release = r; });
+    let sinkSaw = null;
+    const { cp, handler, session } = await rig({ sink: async (w) => { sinkSaw = Buffer.from(w.body); await gate; } });
+    const body = bodyOf(RAW * 2 + 77);
+    const fr = chunksOf(body);
+    for (const f of fr) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 2);
+    assert.deepEqual(cp.acksFromLa.map((a) => [a.ackSeq, a.status]), [[1, "accepted"], [2, "accepted"]]);
+    await waitFor(() => sinkSaw !== null);
+    await new Promise((r) => setTimeout(r, 120));
+    assert.equal(cp.acksFromLa.length, 2, "the ack of the last chunk is held while the local processing runs");
+    release();
+    await waitFor(() => cp.acksFromLa.length === 3);
+    assert.deepEqual(cp.acksFromLa[2], { ackSeq: 3, status: "accepted", errorCode: undefined });
+    assert.ok(sinkSaw.equals(body));
+    assert.equal(handler.stats.delivered, 1);
+    session.close();
+});
+
+test("the body is zeroed after delivery (the sink cannot keep it) and nothing is left open", async () => {
+    let kept = null;
+    const { cp, handler, session } = await rig({ sink: async (w) => { kept = w.body; } });
+    for (const f of chunksOf(Buffer.alloc(RAW + 5, 0x42))) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 2);
+    assert.ok(kept.every((b) => b === 0));
+    assert.equal(handler.assembler.openCount, 0);
+    session.close();
+});
+
+test("refusals are non-fatal `rejected` acks with the wire code (the session goes on): out of order, wrong sha, no sink, a sink that throws", async () => {
+    const { cp, session, handler } = await rig();
+    const body = bodyOf(RAW + 9);
+    const fr = chunksOf(body);
+    send(cp, fr[1]);                                  // first chunk seen is index 1
+    await waitFor(() => cp.acksFromLa.length === 1);
+    assert.deepEqual(cp.acksFromLa[0], { ackSeq: 1, status: "rejected", errorCode: "chunk_out_of_order" });
+    const bad = chunksOf(body);
+    for (const f of bad) f.params.bodySha256 = "1".repeat(64);
+    send(cp, bad[0]); send(cp, bad[1]);
+    await waitFor(() => cp.acksFromLa.length === 3);
+    assert.deepEqual(cp.acksFromLa[2], { ackSeq: 3, status: "rejected", errorCode: "chunk_inconsistent" });
+    assert.equal(session.state, "ready", "not fatal: the connection stays");
+    assert.deepEqual(cp.errorsSent, []);
+    assert.equal(handler.assembler.openCount, 0);
+    session.close();
+    // no sink: protocol_unsupported (the LA never says "delivered" when nothing took it)
+    const r2 = await rig({ sink: null });
+    send(r2.cp, { params: { webhookId: uuid(), source: "other", contentType: "text/plain", bodyB64: Buffer.from("x").toString("base64") } });
+    await waitFor(() => r2.cp.acksFromLa.length === 1);
+    assert.equal(r2.cp.acksFromLa[0].errorCode, "protocol_unsupported");
+    r2.session.close();
+    // a sink that throws: internal; a sink that says rate_limited: rate_limited
+    const r3 = await rig({ sink: async () => { throw new Error("boom with CANARY-TEXT"); } });
+    send(r3.cp, { params: { webhookId: uuid(), source: "other", contentType: "text/plain", bodyB64: Buffer.from("x").toString("base64") } });
+    await waitFor(() => r3.cp.acksFromLa.length === 1);
+    assert.deepEqual(r3.cp.acksFromLa[0], { ackSeq: 1, status: "rejected", errorCode: "internal" });
+    assert.ok(!JSON.stringify(r3.env.wire.toCp).includes("CANARY-TEXT"), "the error text never goes out");
+    r3.session.close();
+    const r4 = await rig({ sink: async () => { const e = new Error("full"); e.code = "rate_limited"; throw e; } });
+    send(r4.cp, { params: { webhookId: uuid(), source: "other", contentType: "text/plain", bodyB64: Buffer.from("x").toString("base64") } });
+    await waitFor(() => r4.cp.acksFromLa.length === 1);
+    assert.equal(r4.cp.acksFromLa[0].errorCode, "rate_limited");
+    r4.session.close();
+});
+
+test("a retransmitted chunk (same seq and msgId: the ack was lost) is the session's idempotent duplicate: delivered once, re-acked", async () => {
+    // mode off: the gateway re-signs a retransmission (new nonce); a byte-identical replay would be refused by the nonce cache before the dedupe
+    const { cp, env, session, delivered } = await rig({ mode: "off" });
+    const fr = chunksOf(Buffer.from('{"a":1}'));
+    send(cp, fr[0]);
+    await waitFor(() => cp.acksFromLa.length === 1);
+    const original = env.wire.toLa.find((t) => JSON.parse(t).type === "webhook.forward");
+    session.handleText(original, session.transport);   // byte for byte the same frame again
+    await waitFor(() => cp.acksFromLa.length === 2);
+    assert.equal(delivered.length, 1, "the second copy of the same seq+msgId did not run the handler again");
+    assert.equal(cp.acksFromLa[1].status, "duplicate");
+    session.close();
+});
+
+test("a chunk of the wrong size is refused by the session as chunk_inconsistent only AFTER the state rules (the handler answers, non-fatal)", async () => {
+    const { cp, session } = await rig();
+    const fr = chunksOf(bodyOf(RAW * 2));
+    fr[0].params.bodyB64 = Buffer.alloc(100).toString("base64");   // a non-last chunk of 100 bytes
+    send(cp, fr[0]);
+    await waitFor(() => cp.acksFromLa.length === 1);
+    assert.deepEqual(cp.acksFromLa[0], { ackSeq: 1, status: "rejected", errorCode: "chunk_inconsistent" });
+    assert.equal(session.state, "ready");
+    session.close();
+});
+
+test("another tenant's chunk never reaches the receiver: tenant_mismatch before any handler (cross-tenant leak)", async () => {
+    const { cp, session, delivered, handler } = await rig();
+    const f = chunksOf(Buffer.from("secret"))[0];
+    const frame = cp.frameWith("webhook.forward", f.params, { frame: { chunk: f.chunk } });
+    frame.tenantId = "99999999-5a4d-4e7b-9c21-0a1b2c3d4e5f";
+    session.handleText(JSON.stringify(frame), session.transport);
+    await waitFor(() => cp.peerErrors.length > 0);
+    assert.ok(["tenant_mismatch", "unauthorized", "params_mismatch"].includes(cp.peerErrors[0]), cp.peerErrors[0]);
+    assert.equal(delivered.length, 0);
+    assert.equal(handler.stats.frames, 0);
+    session.close();
+});
+
+test("two links (two tenants) never share an assembly: the same webhookId in two handlers builds two separate bodies", async () => {
+    const a = await rig();
+    const b = await rig();
+    const id = uuid();
+    const bodyA = bodyOf(RAW + 1);
+    const bodyB = bodyOf(RAW + 2);
+    const fa = chunksOf(bodyA, { webhookId: id });
+    const fb = chunksOf(bodyB, { webhookId: id });
+    send(a.cp, fa[0]); send(b.cp, fb[0]); send(b.cp, fb[1]); send(a.cp, fa[1]);
+    await waitFor(() => a.delivered.length === 1 && b.delivered.length === 1);
+    assert.ok(a.delivered[0].body.equals(bodyA));
+    assert.ok(b.delivered[0].body.equals(bodyB));
+    a.session.close(); b.session.close();
+});
+
+// ------------------------------------------------------------------------------------------ the 8 traces of the contract
+
+const traceFiles = fs.readdirSync(TRACES).filter((f) => /webhook-chunk/.test(f)).sort();
+test("the contract has the 8 webhook traces (1 valid, 7 invalid)", () => {
+    assert.equal(traceFiles.length, 8);
+});
+
+for (const file of traceFiles) {
+    const t = JSON.parse(fs.readFileSync(path.join(TRACES, file), "utf8"));
+    test(`trace ${t.name}: the REAL session + receiver ${t.expect.ok ? "accept every chunk and deliver a body with the declared sha256" : `answer ${t.expect.error} at frame ${t.expect.atFrame}`}`, async () => {
+        const cp = new ReferenceCp({ tenantId: t.session.tenantId, agentId: t.session.agentId });
+        const env = inMemory(cp, { mode: "off", tenantId: t.session.tenantId, agentId: t.session.agentId, options: { handlerAckMs: 5000 } }).start();   // the trace signatures are placeholders
+        await waitFor(() => env.session.state === "ready");
+        const delivered = [];
+        const handler = createWebhookHandler({ config: cfg(), getSink: () => async (w) => { delivered.push(Buffer.from(w.body)); } });
+        env.session.setHandler("webhook.forward", (ctx) => handler.handle(ctx));
+        const cpFrames = t.frames.map((f, i) => ({ ...f, i })).filter((f) => f.dir === "cp->la" && f.frame.type === "webhook.forward");
+        for (const fr of cpFrames) env.session.handleText(JSON.stringify(fr.frame), env.transport);
+        await waitFor(() => cp.acksFromLa.length >= cpFrames.length || cp.peerErrors.length > 0);
+        if (t.expect.ok) {
+            assert.deepEqual(cp.acksFromLa.map((a) => a.status), cpFrames.map(() => "accepted"));
+            assert.equal(delivered.length, 1);
+            assert.equal(sha(delivered[0]), cpFrames[0].frame.payload.bodySha256);
+            assert.equal(delivered[0].length, cpFrames[0].frame.payload.bodyBytes);
+            assert.deepEqual(cp.peerErrors, []);
+        } else {
+            const at = cpFrames.findIndex((f) => f.i === t.expect.atFrame);
+            assert.ok(at >= 0);
+            assert.deepEqual(cp.acksFromLa.slice(0, at).map((a) => a.status), cpFrames.slice(0, at).map(() => "accepted"), "everything before the bad frame is accepted");
+            assert.deepEqual(cp.acksFromLa[at], { ackSeq: cpFrames[at].frame.seq, status: "rejected", errorCode: t.expect.error });
+            // only "a chunk after the one flagged last" has a complete webhook before the bad frame (delivered once, legitimately)
+            assert.equal(delivered.length, t.name === "invalid-webhook-chunk-after-last" ? 1 : 0, "an invalid chunked webhook is never delivered");
+            assert.equal(env.session.state, "ready");
+        }
+        env.session.close();
+    });
+}
+
+// ------------------------------------------------------------------------------------------ the mappers of the local consumer
+
+test("local consumer: text/plain lines, an Alertmanager group, a Sentry event and an unmapped body", () => {
+    const text = entriesFromWebhook({ source: "other", contentType: "text/plain", body: Buffer.from("2026-11-03 10:00:00 ERROR java.lang.IllegalStateException: boom\n\tat a.B.c(B.java:1)\n\tat a.B.d(B.java:2)\n") });
+    assert.equal(text.entries.length, 1);
+    assert.match(text.entries[0].stacktrace, /B\.java:1/);
+    const am = entriesFromWebhook({ source: "alertmanager", contentType: "application/json", body: Buffer.from(JSON.stringify({ alerts: [
+        { status: "firing", labels: { alertname: "HighErrorRate", severity: "critical", service: "billing" }, annotations: { summary: "5xx above 5%" }, startsAt: "2026-11-03T10:00:00Z" },
+        { status: "resolved", labels: { alertname: "Old" } },
+        { status: "firing", labels: { alertname: "Warn", severity: "warning" } },
+    ] })) });
+    assert.equal(am.entries.length, 2);
+    assert.deepEqual([am.entries[0].severity, am.entries[0].service, am.entries[0].message], ["CRITICAL", "billing", "HighErrorRate: 5xx above 5%"]);
+    assert.equal(am.entries[1].severity, "WARN", "a warning alert is not an incident for the detector");
+    const se = entriesFromWebhook({ source: "sentry", contentType: "application/json", body: Buffer.from(JSON.stringify({ project_name: "checkout", data: { event: { level: "error", timestamp: 1793268000, exception: { values: [{ type: "TypeError", value: "x is undefined", stacktrace: { frames: [{ function: "a", filename: "a.js", lineno: 3 }, { function: "b", filename: "b.js", lineno: 9 }] } }] } } } })) });
+    assert.equal(se.entries.length, 1);
+    assert.equal(se.entries[0].message, "TypeError: x is undefined");
+    assert.equal(se.entries[0].service, "checkout");
+    assert.match(se.entries[0].stacktrace, /at b \(b\.js:9\)\n\s+at a \(a\.js:3\)/, "frames are listed innermost first");
+    assert.deepEqual(entriesFromWebhook({ source: "snyk", contentType: "application/json", body: Buffer.from("[1,2]") }), { entries: [], unmapped: true });
+    assert.deepEqual(entriesFromWebhook({ source: "other", contentType: "application/json", body: Buffer.from("not json") }), { entries: [], unmapped: true });
+});
+
+// ------------------------------------------------------------------------------------------ flags, capabilities, end to end
+
+async function linkFor(cp, env, root = tmp()) {
+    const logs = [];
+    const log = { info: (m) => logs.push(m), log: (m) => logs.push(m), warn: (m) => logs.push(m), error: (m) => logs.push(m) };
+    const wsManager = { list: () => [{ id: "ws-1", root, tenantId: TENANT }] };
+    const link = await buildV2Link({ tenantId: TENANT, agentId: AGENT, signing: signingFor({ keyId: cp.keyId, publicKeyRaw: cp.publicKeyRaw }, "enforce"), wsManager, getWorkspaceRoot: () => root, log, env });
+    const transport = { send: (t) => cp.feed(t), close() {}, bufferedAmount: () => 0 };
+    cp.attach((t) => queueMicrotask(() => link.session.handleText(t, transport)), () => {});
+    link.session.attach(transport);
+    await waitFor(() => link.session.state === "ready");
+    return { link, logs, transport };
+}
+
+test("flag off (unset or any non-true value): no capability, no handler (webhook.forward acked protocol_unsupported), register is exactly the baseline", async () => {
+    for (const v of [undefined, "false", "maybe"]) {
+        const cp = new ReferenceCp({ capabilities: ["incident.candidate.size-model"] });
+        const { link } = await linkFor(cp, { LOCAL_AGENT_WEBHOOK_FORWARD: v });
+        assert.deepEqual([...cp.registers[0].capabilities].sort(), ["chunk.base64", "protocol.v2", "v1.commands"]);
+        assert.equal(link.webhooks, undefined);
+        cp.sendData("webhook.forward", { webhookId: uuid(), source: "other", contentType: "text/plain", bodyB64: Buffer.from("x").toString("base64") });
+        await waitFor(() => cp.acksFromLa.length === 1);
+        assert.deepEqual(cp.acksFromLa[0], { ackSeq: 1, status: "rejected", errorCode: "protocol_unsupported" });
+        link.close();
+    }
+});
+
+test("flag on WITHOUT the incident detector: there is no local consumer, the receiver is not registered and no webhook capability is announced", async () => {
+    const cp = new ReferenceCp();
+    const { link, logs } = await linkFor(cp, { LOCAL_AGENT_WEBHOOK_FORWARD: "true" });
+    assert.ok(!cp.registers[0].capabilities.some((c) => c.startsWith("webhook.")));
+    assert.ok(!cp.registers[0].capabilities.includes("contract.v2.1"));
+    assert.equal(link.webhooks, undefined);
+    assert.ok(logs.some((l) => /no local consumer/.test(l)));
+    link.close();
+});
+
+const GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8";
+const CANARIES = ["CANARY-WH-5521", "CanaryWebhookVault", "/home/customer/wh-secret", "order-7777-detail", GH];
+
+test("end to end (flags on): a 40 KB chunked webhook -> reassembled -> detector -> a candidate with metadata only (+ stackBytes and model); the ack of the last chunk comes after; canaries never on the wire", async () => {
+    const dir = tmp();
+    const cp = new ReferenceCp({ capabilities: ["incident.candidate.size-model"] });
+    const { link } = await linkFor(cp, {
+        LOCAL_AGENT_PROTOCOL_V2: "true", LOCAL_AGENT_INCIDENT_DETECTOR: "true", LOCAL_AGENT_WEBHOOK_FORWARD: "true", LOCAL_AGENT_DLP_MODE: "enforce",
+        LOCAL_AGENT_INCIDENT_STORE_DIR: path.join(dir, "store"), LOCAL_AGENT_INCIDENT_SERVICE: "billing-api", LOCAL_AGENT_INCIDENT_FLUSH_MS: "300",
+    });
+    for (const c of ["webhook.forward", "webhook.chunked", "incident.candidate", "contract.v2.1"]) assert.ok(cp.registers[0].capabilities.includes(c), c);
+    assert.ok(!cp.registers[0].capabilities.includes("llm.key"));
+    let stack = "";
+    for (let i = 0; i < 180; i++) stack += `\tat com.acme.CanaryWebhookVault.step${i}(CanaryWebhookVault.java:${i}) /home/customer/wh-secret order-7777-detail\n`;
+    const text = `2026-11-03 10:00:00 ERROR c.a.Order : java.lang.IllegalStateException: CANARY-WH-5521 token ${GH}\n${stack}`;
+    const body = Buffer.from(text, "utf8");
+    assert.ok(body.length > 9216 && body.length < 60_000, `the body must need chunks: ${body.length}`);
+    const fr = chunksOf(body, { source: "other", contentType: "text/plain" });
+    assert.ok(fr.length >= 2);
+    for (const f of fr) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length >= fr.length);
+    assert.deepEqual(cp.acksFromLa.slice(0, fr.length).map((a) => a.status), fr.map(() => "accepted"));
+    const cands = () => cp.received.filter((f) => f.type === "incident.candidate");
+    await waitFor(() => cands().length >= 1, 5000);
+    const c = cands()[0].payload;
+    assert.deepEqual(Object.keys(c).sort(), ["count", "environment", "errorHash", "fingerprint", "firstSeen", "lastSeen", "logCategory", "model", "service", "severity", "stackBytes"]);
+    assert.equal(c.service, "billing-api");
+    assert.match(c.model, /^stack-detector-\d+\.\d+\.\d+$/);
+    assert.equal(c.stackBytes, Buffer.byteLength(stack.replace(/\n$/, "")), "the size of the stack as received");
+    const wire = cp.rawIn.join("\n");
+    for (const w of CANARIES) assert.ok(!wire.includes(w), `the wire must not carry ${w.slice(0, 12)}...`);
+    const detail = await link.incidents.getDetail(c.errorHash);
+    assert.ok(detail.length >= 1 && detail[0].message.includes("CANARY-WH-5521"), "the text stayed local");
+    assert.ok(!JSON.stringify(detail).includes(GH), "the DLP masked the token in the local store");
+    assert.ok(!JSON.stringify(link.incidents.audit.tail(100)).includes("CANARY"));
+    assert.equal(link.webhooks.handler.assembler.openCount, 0);
+    // nothing of the body was written anywhere but the (masked) local store: no temp file in the store dir other than the jsonl
+    assert.deepEqual(fs.readdirSync(path.join(dir, "store")).sort(), ["incident-detail.jsonl"]);
+    link.close();
+});
+
+test("end to end: a small Alertmanager webhook in ONE frame goes through the same pipeline", async () => {
+    const dir = tmp();
+    const cp = new ReferenceCp({ capabilities: ["incident.candidate.size-model"] });
+    const { link } = await linkFor(cp, {
+        LOCAL_AGENT_INCIDENT_DETECTOR: "true", LOCAL_AGENT_WEBHOOK_FORWARD: "true", LOCAL_AGENT_DLP_MODE: "enforce",
+        LOCAL_AGENT_INCIDENT_STORE_DIR: path.join(dir, "store"), LOCAL_AGENT_INCIDENT_FLUSH_MS: "300",
+    });
+    const body = Buffer.from(JSON.stringify({ alerts: [{ status: "firing", labels: { alertname: "HighLatencyException", severity: "critical", service: "checkout" }, annotations: { summary: "java.lang.IllegalStateException: pool exhausted" }, startsAt: "2026-11-03T10:00:00Z" }] }));
+    cp.sendData("webhook.forward", { webhookId: uuid(), source: "alertmanager", contentType: "application/json", bodyB64: body.toString("base64") });
+    await waitFor(() => cp.acksFromLa.length === 1);
+    assert.equal(cp.acksFromLa[0].status, "accepted");
+    await waitFor(() => cp.received.some((f) => f.type === "incident.candidate"), 5000);
+    const c = cp.received.find((f) => f.type === "incident.candidate").payload;
+    assert.equal(c.service, "checkout");
+    assert.equal(c.severity, "critical");
+    link.close();
+});
+
+// ------------------------------------------------------------------------------------------ L2: final ack after the local delivery, resends
+
+import { installWebhookReceiver } from "../../src/la-protocol-v2/webhooks/install.js";
+
+/** The receiver installed the way runtime.js does it (setHandler with the delivery deadline), on a session whose OWN handlerAckMs is tiny. */
+async function installedRig({ sink, config = cfg({ deliveryMs: 1500 }), now } = {}) {
+    const cp = new ReferenceCp({ capabilities: ["incident.candidate.size-model"] });
+    const env = inMemory(cp, { mode: "enforce", options: { handlerAckMs: 100 } }).start();
+    await waitFor(() => env.session.state === "ready");
+    const link = { session: env.session, close() {} };
+    const rx = installWebhookReceiver({ link, config: now ? { ...config } : config });
+    const calls = [];
+    rx.setSink(sink || (async (w) => { calls.push({ id: w.webhookId, body: Buffer.from(w.body) }); }));
+    return { cp, env, rx, calls, session: env.session };
+}
+const one = (body, webhookId = uuid()) => ({ params: { webhookId, source: "other", contentType: "text/plain", bodyB64: Buffer.from(body).toString("base64") } });
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+test("L2: the ack of the LAST chunk waits for a slow local delivery (longer than the session's handlerAckMs) and says `accepted` only after it", async () => {
+    let finished = false;
+    const { cp, session, rx } = await installedRig({ sink: async () => { await sleep(450); finished = true; } });
+    const fr = chunksOf(bodyOf(RAW + 50));
+    for (const f of fr) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 1);                 // chunk 0: stored
+    await sleep(300);                                                 // > handlerAckMs (100) and < the sink (450)
+    assert.equal(cp.acksFromLa.length, 1, "no ack for the last chunk while the sink runs, although the session's own limit passed");
+    await waitFor(() => cp.acksFromLa.length === 2);
+    assert.equal(finished, true, "the ack came after the delivery, not before");
+    assert.deepEqual(cp.acksFromLa[1], { ackSeq: 2, status: "accepted", errorCode: undefined });
+    assert.equal(rx.handler.stats.delivered, 1);
+    session.close();
+});
+
+test("L2: a single-frame webhook is acked after the sink too", async () => {
+    let finished = false;
+    const { cp, session } = await installedRig({ sink: async () => { await sleep(300); finished = true; } });
+    send(cp, one("hello"));
+    await sleep(200);
+    assert.equal(cp.acksFromLa.length, 0);
+    await waitFor(() => cp.acksFromLa.length === 1);
+    assert.ok(finished);
+    assert.equal(cp.acksFromLa[0].status, "accepted");
+    session.close();
+});
+
+test("L2: a sink slower than the deadline: the last chunk is `rejected timeout` (clear error), the body is NOT zeroed under the running sink, zeroed once it settles, and a late success is remembered", async () => {
+    let release; let kept = null; let runs = 0;
+    const gate = new Promise((r) => { release = r; });
+    const { cp, session, rx } = await installedRig({ config: cfg({ deliveryMs: 250 }), sink: async (w) => { runs++; kept = w.body; await gate; } });
+    const body = bodyOf(RAW + 9);
+    const fr = chunksOf(body);
+    for (const f of fr) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 2, 3000);
+    assert.deepEqual(cp.acksFromLa[1], { ackSeq: 2, status: "rejected", errorCode: "timeout" });
+    assert.equal(session.state, "ready", "non-fatal");
+    assert.deepEqual(cp.errorsSent, []);
+    assert.ok(kept.equals(body), "the running sink still sees its body");
+    assert.equal(rx.handler.stats.deliveryTimeouts, 1);
+    // a retry while the first attempt is still running is told to come back later, not processed twice
+    for (const f of chunksOf(body, { webhookId: fr[0].params.webhookId })) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 4);
+    assert.deepEqual(cp.acksFromLa[3], { ackSeq: 4, status: "rejected", errorCode: "rate_limited" });
+    assert.equal(runs, 1);
+    release();
+    await waitFor(() => kept.every((b) => b === 0));
+    // the first attempt finished late: the webhook WAS delivered, so a retry now is acked without running the sink again
+    for (const f of chunksOf(body, { webhookId: fr[0].params.webhookId })) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 6);
+    assert.deepEqual(cp.acksFromLa.slice(4).map((a) => a.status), ["accepted", "accepted"]);
+    assert.equal(runs, 1, "not processed again");
+    session.close();
+});
+
+test("L2: a single frame past the deadline is `rejected timeout` too; the deadline can only be tuned inside 100 ms .. 14 s", () => {
+    assert.equal(cfg().deliveryMs, 10_000);
+    for (const [v, want] of [["50", 10_000], ["15000", 10_000], ["abc", 10_000], ["100", 100], ["14000", 14_000], ["2500", 2500]]) {
+        assert.equal(resolveWebhookConfig({ LOCAL_AGENT_WEBHOOK_FORWARD: "true", LOCAL_AGENT_WEBHOOK_DELIVERY_MS: v }, noop, LIMITS).deliveryMs, want, v);
+    }
+    assert.deepEqual([cfg().dedupeMs, cfg().dedupeMax], [600_000, 1024]);
+    assert.equal(resolveWebhookConfig({ LOCAL_AGENT_WEBHOOK_FORWARD: "true", LOCAL_AGENT_WEBHOOK_DEDUPE_MAX: "3" }, noop, LIMITS).dedupeMax, 1024, "below the minimum: the default");
+});
+
+test("L2: single-frame deadline", async () => {
+    const { cp, session } = await installedRig({ config: cfg({ deliveryMs: 200 }), sink: () => new Promise(() => {}) });
+    send(cp, one("never done"));
+    await waitFor(() => cp.acksFromLa.length === 1, 3000);
+    assert.deepEqual(cp.acksFromLa[0], { ackSeq: 1, status: "rejected", errorCode: "timeout" });
+    session.close();
+});
+
+test("L2: a resend of a delivered single-frame webhook (same id, same body) is acked accepted WITHOUT reaching the sink; another body under the same id is a new webhook", async () => {
+    const { cp, session, rx, calls } = await installedRig();
+    const id = uuid();
+    send(cp, one("first body", id));
+    send(cp, one("first body", id));
+    send(cp, one("another body", id));
+    send(cp, one("first body", uuid()));
+    await waitFor(() => cp.acksFromLa.length === 4);
+    assert.deepEqual(cp.acksFromLa.map((a) => a.status), ["accepted", "accepted", "accepted", "accepted"]);
+    assert.equal(calls.length, 3, "the second copy was not processed");
+    assert.equal(rx.handler.stats.deduped, 1);
+    session.close();
+});
+
+test("L2: a resend of a delivered CHUNKED webhook: every chunk acked in order, the sink is not called again; the count of the detector is not inflated", async () => {
+    const { cp, session, rx, calls } = await installedRig();
+    const body = bodyOf(RAW * 2 + 33);
+    const id = uuid();
+    for (const f of chunksOf(body, { webhookId: id })) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 3);
+    assert.equal(calls.length, 1);
+    for (const f of chunksOf(body, { webhookId: id })) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 6);
+    assert.deepEqual(cp.acksFromLa.slice(3).map((a) => a.status), ["accepted", "accepted", "accepted"]);
+    assert.equal(calls.length, 1, "still one delivery");
+    assert.equal(rx.handler.stats.deduped, 1);
+    assert.equal(rx.handler.assembler.openCount, 0, "a resend is never buffered");
+    // and the same id with a different body is processed
+    for (const f of chunksOf(bodyOf(RAW + 1), { webhookId: id })) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 8);
+    assert.equal(calls.length, 2);
+    session.close();
+});
+
+test("L2: the resend path keeps the contract's order rules: a chunk after the last one, a skipped index, a wrong size are refused as before", async () => {
+    const { cp, session, calls } = await installedRig();
+    const body = bodyOf(RAW * 2 + 5);
+    const id = uuid();
+    const fr = chunksOf(body, { webhookId: id });
+    for (const f of fr) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 3);
+    send(cp, fr[2]);                                                  // a chunk after the last one (not a resend: it does not start at 0)
+    await waitFor(() => cp.acksFromLa.length === 4);
+    assert.deepEqual(cp.acksFromLa[3], { ackSeq: 4, status: "rejected", errorCode: "chunk_out_of_order" });
+    send(cp, fr[0]); send(cp, fr[2]);                                 // a resend that skips index 1
+    await waitFor(() => cp.acksFromLa.length === 6);
+    assert.deepEqual(cp.acksFromLa.slice(4).map((a) => [a.status, a.errorCode]), [["accepted", undefined], ["rejected", "chunk_out_of_order"]]);
+    const bad = chunksOf(body, { webhookId: id });
+    bad[0].params.bodyB64 = Buffer.alloc(100).toString("base64");     // a resend whose first chunk has the wrong size
+    send(cp, bad[0]);
+    await waitFor(() => cp.acksFromLa.length === 7);
+    assert.deepEqual(cp.acksFromLa[6], { ackSeq: 7, status: "rejected", errorCode: "chunk_inconsistent" });
+    assert.equal(calls.length, 1);
+    assert.equal(session.state, "ready");
+    session.close();
+});
+
+test("L2: only a SUCCESSFUL delivery is remembered (a sink that threw is retried), the window is 10 minutes, and the memory is bounded", async () => {
+    let t = 5_000_000;
+    let fail = true;
+    const calls = [];
+    const handler = createWebhookHandler({ config: cfg({ dedupeMs: 600_000, dedupeMax: 3 }), now: () => t, getSink: () => async (w) => { calls.push(w.webhookId); if (fail) throw new Error("boom"); } });
+    const ctx = (f) => ({ params: f.params, chunk: undefined });
+    const a = one("x", uuid());
+    assert.deepEqual(await handler.handle(ctx(a)), { reject: "internal" });
+    fail = false;
+    assert.deepEqual(await handler.handle(ctx(a)), { accept: true });
+    assert.equal(calls.length, 2, "the failed delivery was retried");
+    assert.deepEqual(await handler.handle(ctx(a)), { accept: true });
+    assert.equal(calls.length, 2, "now remembered");
+    t += 599_000;
+    await handler.handle(ctx(a));
+    assert.equal(calls.length, 2, "inside the window");
+    t += 2_000;
+    await handler.handle(ctx(a));
+    assert.equal(calls.length, 3, "10 minutes later it is processed again");
+    // bounded: dedupeMax 3 -> the oldest id is forgotten
+    const ids = [one("1", uuid()), one("2", uuid()), one("3", uuid()), one("4", uuid())];
+    for (const f of ids) await handler.handle(ctx(f));
+    const before = calls.length;
+    await handler.handle(ctx(ids[3]));
+    assert.equal(calls.length, before, "the newest is still remembered");
+    await handler.handle(ctx(ids[0]));
+    assert.equal(calls.length, before + 1, "the oldest was evicted");
+});
+
+test("L2: a new session (not resumed) forgets the chunks in progress AND the resends in progress, but not the delivered ids (they are the point)", async () => {
+    const { cp, session, rx, calls } = await installedRig();
+    const body = bodyOf(RAW * 2 + 5);
+    const id = uuid();
+    const fr = chunksOf(body, { webhookId: id });
+    for (const f of fr) send(cp, f);
+    await waitFor(() => cp.acksFromLa.length === 3);
+    send(cp, fr[0]);                                                  // a resend starts...
+    await waitFor(() => cp.acksFromLa.length === 4);
+    rx.handler.clear();                                               // ...and a new session clears it
+    send(cp, fr[1]);                                                  // chunk 1 of that resend: nothing in progress, not index 0
+    await waitFor(() => cp.acksFromLa.length === 5);
+    assert.equal(cp.acksFromLa[4].errorCode, "chunk_out_of_order");
+    send(cp, fr[0]);
+    await waitFor(() => cp.acksFromLa.length === 6);
+    assert.equal(cp.acksFromLa[5].status, "accepted", "the delivered id is still remembered");
+    assert.equal(calls.length, 1);
+    session.close();
+});
+
+test("L2: end to end with the incident detector: the same Alertmanager webhook sent twice produces ONE count", async () => {
+    const dir = tmp();
+    const cp = new ReferenceCp({ capabilities: ["incident.candidate.size-model"] });
+    const { link } = await linkFor(cp, {
+        LOCAL_AGENT_INCIDENT_DETECTOR: "true", LOCAL_AGENT_WEBHOOK_FORWARD: "true", LOCAL_AGENT_DLP_MODE: "enforce",
+        LOCAL_AGENT_INCIDENT_STORE_DIR: path.join(dir, "store"), LOCAL_AGENT_INCIDENT_FLUSH_MS: "300",
+    });
+    const body = Buffer.from(JSON.stringify({ alerts: [{ status: "firing", labels: { alertname: "Dup", severity: "critical", service: "checkout" }, annotations: { summary: "java.lang.IllegalStateException: dup pool" }, startsAt: "2026-11-03T10:00:00Z" }] }));
+    const id = uuid();
+    const frame = { webhookId: id, source: "alertmanager", contentType: "application/json", bodyB64: body.toString("base64") };
+    cp.sendData("webhook.forward", frame);
+    cp.sendData("webhook.forward", frame);
+    await waitFor(() => cp.acksFromLa.length === 2);
+    assert.deepEqual(cp.acksFromLa.map((a) => a.status), ["accepted", "accepted"]);
+    assert.equal(link.webhooks.handler.stats.deduped, 1);
+    await waitFor(() => cp.received.some((f) => f.type === "incident.candidate"), 5000);
+    await sleep(700);
+    const total = cp.received.filter((f) => f.type === "incident.candidate").reduce((n, f) => n + f.payload.count, 0);
+    assert.equal(total, 1, "one occurrence, not two");
+    link.close();
+});