deepdebug-local-agent 1.7.0 → 1.9.0

package/test/incident-detector/units.test.js

@@ -0,0 +1,430 @@
+// F3-11b: unit tests of the pieces (config, parse, categorize, aggregator, store, emitter) with fake clocks and a fake session.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { EventEmitter } from "node:events";
+import { resolveIncidentDetectorConfig } from "../../src/incident-detector/config.js";
+import { parseRecord, EntryAssembler, parseTime } from "../../src/incident-detector/parse.js";
+import { severityOf, maxSeverity, inferEnvironment, sanitizeService, sanitizeEnvironment, logCategoryOf } from "../../src/incident-detector/categorize.js";
+import { createAggregator } from "../../src/incident-detector/aggregator.js";
+import { createDetailStore } from "../../src/incident-detector/store.js";
+import { createCandidateEmitter } from "../../src/incident-detector/emitter.js";
+import { ProtocolError } from "../../src/la-protocol-v2/errors.js";
+
+const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), "la-inc-"));
+const H = (n) => n.toString(16).padStart(32, "0");
+
+// ------------------------------------------------------------------------------------------ config
+
+test("config: only true/1/yes/on enable it; unset, empty, false and unknown values keep it off (unknown warns)", () => {
+    for (const v of [undefined, "", "false", "0", "no", "off", " OFF ", "maybe", "2", "enabled"]) {
+        const warns = [];
+        assert.deepEqual(resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: v }, (m) => warns.push(m)), { enabled: false }, String(v));
+        assert.equal(warns.length, ["maybe", "2", "enabled"].includes(v) ? 1 : 0, String(v));
+    }
+    for (const v of ["true", "TRUE", "1", "yes", "on", " On "]) assert.equal(resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: v }, () => {}).enabled, true, v);
+});
+
+test("config: numbers are clamped, a bad value falls back to the default (never unlimited); sources and Loki validation", () => {
+    const warns = [];
+    const c = resolveIncidentDetectorConfig({
+        LOCAL_AGENT_INCIDENT_DETECTOR: "true", LOCAL_AGENT_INCIDENT_MAX_TRACKED: "99999999", LOCAL_AGENT_INCIDENT_MAX_PENDING: "0", LOCAL_AGENT_INCIDENT_FLUSH_MS: "abc",
+        LOCAL_AGENT_INCIDENT_MAX_EMITS_PER_MINUTE: "-1", LOCAL_AGENT_INCIDENT_LOG_FILES: " /a.log , ,/b.log", LOCAL_AGENT_INCIDENT_STDIN: "yes",
+        LOCAL_AGENT_INCIDENT_LOKI_URL: "ftp://x", LOCAL_AGENT_INCIDENT_MAX_ENTRY_BYTES: "1",
+    }, (m) => warns.push(m));
+    assert.equal(c.maxTracked, 5_000);
+    assert.equal(c.maxPending, 500);
+    assert.equal(c.flushMs, 30_000);
+    assert.equal(c.maxEmitsPerMinute, 60);
+    assert.equal(c.maxEntryBytes, 64 * 1024);
+    assert.deepEqual(c.files, ["/a.log", "/b.log"]);
+    assert.equal(c.stdin, true);
+    assert.equal(c.loki, null);
+    assert.equal(warns.length, 1);
+    const l = resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: "1", LOCAL_AGENT_INCIDENT_LOKI_URL: "https://loki.internal:3100/", LOCAL_AGENT_INCIDENT_LOKI_QUERY: '{app="x"} |= "error"', LOCAL_AGENT_INCIDENT_LOKI_TOKEN: "t" }, () => {});
+    assert.equal(l.loki.url, "https://loki.internal:3100");
+    assert.equal(l.loki.token, "t");
+    const noQuery = [];
+    assert.equal(resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: "1", LOCAL_AGENT_INCIDENT_LOKI_URL: "http://l" }, (m) => noQuery.push(m)).loki, null);
+    assert.equal(noQuery.length, 1);
+});
+
+// ------------------------------------------------------------------------------------------ parse
+
+test("parse: JSON lines with aliases, separate stack field, docker json log, bad JSON as text", () => {
+    const a = parseRecord('{"level":"error","msg":"boom 1","stack_trace":"a.B: x\\n\\tat c.D.e(D.java:1)","service":"api","time":"2026-10-29T10:00:00Z"}');
+    assert.deepEqual([a.severity, a.message, a.service, a.tsMs], ["error", "boom 1", "api", Date.parse("2026-10-29T10:00:00Z")]);
+    assert.match(a.stacktrace, /^a\.B: x/);
+    const d = parseRecord('{"log":"ERROR something failed\\n","stream":"stderr","time":"2026-10-29T10:00:00.5Z"}');
+    assert.equal(d.severity, "ERROR");
+    assert.equal(d.message, "ERROR something failed");
+    const bad = parseRecord("{not json ERROR x");
+    assert.equal(bad.severity, "ERROR");
+    assert.equal(parseRecord(""), null);
+    assert.equal(parseRecord('{"nomsg":1}'), null);
+    const multi = parseRecord("java.lang.X: boom\n\tat a.B.c(B.java:1)");
+    assert.equal(multi.message, "java.lang.X: boom");
+    assert.equal(multi.stacktrace, "\tat a.B.c(B.java:1)");
+});
+
+test("parse: times from ISO, epoch s/ms/us/ns", () => {
+    const ms = Date.parse("2026-10-29T10:00:00Z");
+    assert.equal(parseTime("2026-10-29T10:00:00Z"), ms);
+    assert.equal(parseTime("2026-10-29 10:00:00,250"), ms + 250);
+    assert.equal(parseTime(ms), ms);
+    assert.equal(parseTime(ms / 1000), ms);
+    assert.equal(parseTime(ms * 1000), ms);
+    assert.equal(parseTime(ms * 1_000_000), ms);
+    assert.equal(parseTime("nope"), null);
+    assert.equal(parseTime(null), null);
+});
+
+test("assembler: a record owns the lines that do not start a new one; JSON lines stand alone; caps are enforced and counted", () => {
+    const a = new EntryAssembler({ maxEntryBytes: 200, maxStackLines: 3 });
+    const out = [];
+    for (const l of ["2026-10-29 10:00:00 ERROR c.a.Foo : failed 42", "java.lang.IllegalStateException: x", "\tat a.B.c(B.java:1)", "\tat a.B.d(B.java:2)", "\tat a.B.e(B.java:3)", "\tat a.B.f(B.java:4)",
+        "2026-10-29 10:00:01 INFO fine", '{"level":"error","msg":"json one"}', "2026-10-29 10:00:02 ERROR " + "z".repeat(500)]) out.push(...a.push(l));
+    out.push(...a.flush());
+    assert.equal(out.length, 4);
+    assert.equal(out[0].severity, "ERROR");
+    assert.equal(out[0].stacktrace.split("\n").length, 3, "maxStackLines");
+    assert.equal(out[1].severity, "INFO");
+    assert.equal(out[2].message, "json one");
+    assert.ok(out[3].message.length <= 200, "maxEntryBytes");
+    assert.equal(a.truncated, 2);
+});
+
+// ------------------------------------------------------------------------------------------ categorize
+
+test("categorize: severity, environment, sanitised labels fit the contract patterns, categories", () => {
+    assert.deepEqual(["FATAL", "critical", "alert", "emerg", "ERROR", "severe", "WARN", null].map(severityOf), ["critical", "critical", "critical", "critical", "high", "high", "medium", "medium"]);
+    assert.equal(maxSeverity("high", "critical"), "critical");
+    assert.equal(maxSeverity("critical", "high"), "critical");
+    assert.deepEqual(["api-qa", "api-qa-2", "x-demo", "x-staging", "x-stg", "api", "", null].map(inferEnvironment), ["qa", "qa", "demo", "staging", "staging", "production", "production", "production"]);
+    for (const raw of ["my service!", "a".repeat(300), "", null, "ok.name:v1/x-y", "é💥"]) assert.match(sanitizeService(raw), /^[A-Za-z0-9._:/-]{1,128}$/, String(raw));
+    for (const raw of ["Prod!", "-x", "", null, "B".repeat(99), "staging-eu"]) assert.match(sanitizeEnvironment(raw), /^[a-z0-9][a-z0-9_-]{0,31}$/, String(raw));
+    const cat = (m, s = null, t = null) => logCategoryOf(m, s, t);
+    assert.equal(cat("java.lang.OutOfMemoryError: Java heap space timed out"), "oom");
+    assert.equal(cat("java.util.concurrent.TimeoutException: x"), "timeout");
+    assert.equal(cat("org.postgresql.util.PSQLException: boom"), "db");
+    assert.equal(cat("x", null, "PG_ERROR"), "db");
+    assert.equal(cat("AccessDeniedException: no"), "security");
+    assert.equal(cat("request failed HTTP 503"), "http_5xx");
+    assert.equal(cat("java.lang.IllegalStateException: x", null, "java.lang.IllegalStateException"), "exception");
+    assert.equal(cat("plain failure"), "other");
+});
+
+// ------------------------------------------------------------------------------------------ aggregator
+
+function aggRig(opts = {}) {
+    const clock = { t: 1_000_000 };
+    const sent = [];
+    const agg = createAggregator({ windowMs: 10_000, flushMs: 2_000, maxTracked: 3, now: () => clock.t, onCandidate: (s) => sent.push(s), ...opts });
+    const obs = (hash, extra = {}) => agg.observe({ errorHash: hash, fingerprint: "f".repeat(64), severity: "high", service: "api", environment: "qa", logCategory: "exception", tsMs: clock.t, ...extra });
+    return { clock, sent, agg, obs };
+}
+
+test("aggregator: leading edge at once, then batched updates with growing count and a stable window identity", () => {
+    const { clock, sent, obs, agg } = aggRig();
+    assert.equal(obs(H(1)).opened, true);
+    assert.equal(sent.length, 1);
+    assert.equal(sent[0].count, 1);
+    clock.t += 100; obs(H(1)); clock.t += 100; obs(H(1));
+    assert.equal(sent.length, 1, "within flushMs nothing more is sent");
+    clock.t += 2_000; obs(H(1));
+    assert.equal(sent.length, 2);
+    assert.equal(sent[1].count, 4);
+    assert.equal(sent[1].firstSeen, sent[0].firstSeen, "same window: same firstSeen");
+    assert.equal(sent[1].windowStart, sent[0].windowStart);
+    clock.t += 500; obs(H(1));
+    clock.t += 2_000; agg.tick();
+    assert.equal(sent.length, 3, "tick sends the due update");
+    assert.equal(sent[2].count, 5);
+    clock.t += 2_000; agg.tick();
+    assert.equal(sent.length, 3, "nothing new: nothing sent");
+});
+
+test("aggregator: the window closes with a final update and the next occurrence opens a NEW window (new firstSeen, count restarts)", () => {
+    const { clock, sent, obs, agg } = aggRig();
+    obs(H(1)); clock.t += 500; obs(H(1));          // count 2, 1 unsent
+    clock.t += 10_000; agg.tick();                  // window over: final update
+    assert.deepEqual(sent.map((s) => s.count), [1, 2]);
+    clock.t += 1; const r = obs(H(1));
+    assert.equal(r.opened, true);
+    assert.equal(sent.at(-1).count, 1);
+    assert.ok(sent.at(-1).windowStart > sent[0].windowStart);
+    assert.equal(agg.stats.windows, 2);
+});
+
+test("aggregator: an occurrence that arrives after the window ended (no tick yet) closes it itself: final update, then a new window", () => {
+    const { clock, sent, obs } = aggRig();
+    obs(H(1)); clock.t += 100; obs(H(1)); obs(H(1));       // count 3, 2 unsent
+    clock.t += 10_000;
+    obs(H(1));
+    assert.deepEqual(sent.map((s) => s.count), [1, 3, 1]);
+    assert.ok(sent[2].windowStart > sent[0].windowStart);
+});
+
+test("aggregator: highest severity of the window, first fingerprint/category win, firstSeen never in the future, an old line keeps the window identity", () => {
+    const { clock, sent, obs } = aggRig();
+    obs(H(1), { severity: "high", fingerprint: "a".repeat(64), logCategory: "db", tsMs: clock.t + 999_999 });
+    assert.equal(sent[0].firstSeen, clock.t, "clamped to now");
+    clock.t += 3_000;
+    obs(H(1), { severity: "critical", fingerprint: "b".repeat(64), logCategory: "oom", tsMs: 5 });
+    assert.equal(sent[1].severity, "critical");
+    assert.equal(sent[1].fingerprint, "a".repeat(64));
+    assert.equal(sent[1].logCategory, "db");
+    assert.equal(sent[1].firstSeen, sent[0].firstSeen);
+});
+
+test("aggregator: maxTracked evicts the least recently seen hash (with its pending update) and counts it", () => {
+    const { clock, sent, obs, agg } = aggRig();
+    obs(H(1)); obs(H(2)); obs(H(3));
+    clock.t += 100; obs(H(1));                     // 1 is now the most recent; 2 is the oldest; 1 has an unsent update
+    obs(H(4));                                     // evicts 2 (nothing pending for it)
+    assert.equal(agg.tracked, 3);
+    assert.equal(agg.stats.evicted, 1);
+    clock.t += 100; obs(H(3)); obs(H(5));          // evicts 1 (it has count 2 unsent: a final update goes out)
+    const finalOf1 = sent.filter((s) => s.errorHash === H(1));
+    assert.deepEqual(finalOf1.map((s) => s.count), [1, 2]);
+});
+
+test("aggregator: drain() sends what is pending and forgets the windows", () => {
+    const { clock, sent, obs, agg } = aggRig();
+    obs(H(1)); clock.t += 10; obs(H(1)); obs(H(2));
+    agg.drain();
+    assert.equal(agg.tracked, 0);
+    assert.deepEqual(sent.map((s) => [s.errorHash, s.count]), [[H(1), 1], [H(2), 1], [H(1), 2]]);
+});
+
+// ------------------------------------------------------------------------------------------ store
+
+function storeRig(extra = {}) {
+    const dir = path.join(tmp(), "inc");
+    const store = createDetailStore({ dir, maxBytes: 4096, maxQueue: 5, messageChars: 200, stackChars: 300, samplesPerWindow: 2, env: { LOCAL_AGENT_DLP_MODE: "enforce" }, ...extra });
+    return { dir, store, file: path.join(dir, "incident-detail.jsonl") };
+}
+const rec = (n, extra = {}) => ({ errorHash: H(n), fingerprint: "f".repeat(64), service: "api", severity: "high", windowStart: 1, message: `msg ${n}`, stack: `stack ${n}`, ...extra });
+
+test("store: 0700 dir, 0600 file, one JSON per line, the detail is readable back (newest first), samples per window are capped", async () => {
+    const { store, dir, file } = storeRig();
+    for (let i = 0; i < 4; i++) await store.add(rec(1, { message: `first ${i}` }));
+    await store.add(rec(2, { windowStart: 2 }));
+    await store.flush();
+    assert.equal(store.stats.written, 3);
+    assert.equal(store.stats.sampledOut, 2);
+    assert.equal(fs.statSync(file).mode & 0o777, 0o600);
+    assert.equal(fs.statSync(dir).mode & 0o777, 0o700);
+    const got = await store.getDetail(H(1), { limit: 5 });
+    assert.deepEqual(got.map((r) => r.message), ["first 1", "first 0"]);
+    assert.equal(got[0].stackBytes, 7);
+    assert.deepEqual(await store.getDetail(H(9)), []);
+});
+
+test("store: the DLP masks secrets before they are written; text and stack are cut; a denied value is never written", async () => {
+    const { store, file } = storeRig();
+    const secret = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8";
+    await store.add(rec(1, { message: `token ${secret} leaked ${"x".repeat(500)}`, stack: `s ${secret}\n` + "y".repeat(900) }));
+    await store.flush();
+    const raw = fs.readFileSync(file, "utf8");
+    assert.ok(!raw.includes(secret), "the secret is not on disk");
+    const r = JSON.parse(raw);
+    assert.equal(r.truncated, true);
+    assert.ok(r.message.length <= 200 + 40 && r.stack.length <= 300 + 40);
+    assert.ok(store.stats.redactions >= 1);
+});
+
+test("store: rotates at maxBytes to .1 (one generation) and reads both", async () => {
+    const { store, file } = storeRig({ maxBytes: 1500 });
+    for (let i = 0; i < 12; i++) await store.add(rec(100 + i, { message: "m".repeat(100) }));
+    await store.flush();
+    assert.ok(store.stats.rotations >= 1);
+    assert.ok(fs.existsSync(`${file}.1`));
+    assert.ok(fs.statSync(file).size <= 1500 + 600);
+    assert.equal((await store.getDetail(H(100 + 11))).length, 1, "newest is in the current file");
+});
+
+test("store: a broken disk disables the store (counted, no throw) and add() resolves false; the queue is bounded", async () => {
+    const dir = tmp();
+    fs.writeFileSync(path.join(dir, "blocker"), "x");
+    const store = createDetailStore({ dir: path.join(dir, "blocker", "sub"), maxBytes: 4096, maxQueue: 2, messageChars: 100, stackChars: 100, samplesPerWindow: 5, env: {} });
+    assert.equal(await store.add(rec(1)), true, "queued before the failure is known");
+    await store.flush();
+    assert.equal(store.broken, true);
+    assert.equal(await store.add(rec(2)), false);
+    assert.ok(store.stats.errors >= 1);
+    assert.ok(store.stats.dropped >= 1);
+});
+
+test("store: add() waits (backpressure) while the queue is full instead of growing it", async () => {
+    const { store } = storeRig({ maxQueue: 2 });
+    let max = 0;
+    const ps = [];
+    for (let i = 0; i < 20; i++) { ps.push(store.add(rec(200 + i, { windowStart: i }))); max = Math.max(max, store.queued); }
+    await Promise.all(ps);
+    await store.flush();
+    assert.ok(max <= 2, `queue never above maxQueue (saw ${max})`);
+    assert.equal(store.stats.written, 20);
+});
+
+// ------------------------------------------------------------------------------------------ emitter (fake session)
+
+class FakeSession extends EventEmitter {
+    constructor() { super(); this.state = "ready"; this.sent = []; this.script = []; }
+    async send(type, payload, opts) {
+        const next = this.script.shift();
+        if (next) throw new ProtocolError(next);
+        this.sent.push({ type, payload, opts });
+    }
+}
+const cfg = (extra = {}) => ({ maxPending: 3, maxEmitsPerMinute: 60, emitBurst: 10, ...extra });
+const snap = (n, extra = {}) => ({ errorHash: H(n), fingerprint: "f".repeat(64), severity: "high", service: "api", environment: "qa", count: 1, firstSeen: 1_700_000_000_000, lastSeen: 1_700_000_001_000, logCategory: "exception", windowStart: 1, ...extra });
+const quietLog = { warn() {}, info() {}, log() {} };
+
+test("emitter: exactly the 9 contract fields, ISO times, the workspace of the session; nothing else", async () => {
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg(), workspaceId: "ws-1", env: {}, log: quietLog });
+    e.submit(snap(1));
+    await e.flush();
+    assert.equal(s.sent.length, 1);
+    assert.deepEqual(Object.keys(s.sent[0].payload).sort(), ["count", "environment", "errorHash", "fingerprint", "firstSeen", "lastSeen", "logCategory", "service", "severity"]);
+    assert.equal(s.sent[0].payload.firstSeen, "2023-11-14T22:13:20.000Z");
+    assert.equal(s.sent[0].opts.workspaceId, "ws-1");
+    assert.equal(s.sent[0].type, "incident.candidate");
+    e.close();
+});
+
+test("emitter: updates of the same window coalesce (the newest count wins); a full queue drops NEW candidates but still takes updates", async () => {
+    const s = new FakeSession(); s.state = "not_ready";   // nothing leaves
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: {}, log: quietLog });
+    e.submit(snap(1)); e.submit(snap(2)); e.submit(snap(3));
+    assert.equal(e.submit(snap(4)), false, "full: a new candidate is dropped");
+    assert.equal(e.stats.droppedFull, 1);
+    e.submit(snap(1, { count: 9 }));
+    assert.equal(e.pending, 3);
+    assert.equal(e.stats.coalesced, 1);
+    s.state = "ready"; s.emit("ready", {});
+    await e.flush();
+    assert.equal(s.sent.length, 3);
+    assert.equal(s.sent[0].payload.count, 9);
+    e.close();
+});
+
+test("emitter: rate limit (token bucket) holds the rest back; the bucket refills with time", async () => {
+    const clock = { t: 0 };
+    const timers = { setTimeout: (fn, ms) => ({ fn, ms, unref() {} }), clearTimeout() {} };
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg({ maxPending: 50, maxEmitsPerMinute: 60, emitBurst: 2 }), env: {}, log: quietLog, now: () => clock.t, timers });
+    for (let i = 1; i <= 5; i++) e.submit(snap(i, { windowStart: i }));
+    await e.flush();
+    assert.equal(s.sent.length, 2, "burst of 2");
+    assert.equal(e.pending, 3);
+    clock.t += 3_000;                                // 3 s at 60/min = 3 tokens (capped at the burst of 2)
+    await e.flush();
+    assert.equal(s.sent.length, 4);
+    e.close();
+});
+
+test("emitter: rate_limited keeps everything and backs off; protocol_unsupported forgets the queue until the next ready; invalid is dropped", async () => {
+    const clock = { t: 0 };
+    const timers = { setTimeout: () => ({ unref() {} }), clearTimeout() {} };
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg({ maxPending: 10 }), env: {}, log: quietLog, now: () => clock.t, timers });
+    s.script.push("rate_limited");
+    e.submit(snap(1)); e.submit(snap(2));
+    await e.flush();
+    assert.equal(s.sent.length, 0);
+    assert.equal(e.pending, 2, "kept");
+    await e.flush();
+    assert.equal(s.sent.length, 0, "still inside the backoff window");
+    clock.t += 31_000;
+    await e.flush();
+    assert.equal(s.sent.length, 2);
+    s.script.push("protocol_unsupported");
+    e.submit(snap(3, { windowStart: 3 }));
+    await e.flush();
+    assert.equal(e.pending, 0, "dropped: this gateway takes none");
+    e.submit(snap(4, { windowStart: 4 }));
+    await e.flush();
+    assert.equal(s.sent.length, 2, "quiet");
+    s.emit("ready", {});
+    e.submit(snap(5, { windowStart: 5 }));
+    await e.flush();
+    assert.equal(s.sent.length, 3, "after ready it talks again");
+    s.script.push("schema_violation");
+    e.submit(snap(6, { windowStart: 6 }));
+    await e.flush();
+    assert.equal(e.stats.droppedInvalid, 1);
+    e.close();
+});
+
+test("emitter: internal/timeout retried, given up after 5 attempts in a row (counted)", async () => {
+    const timers = { setTimeout: () => ({ unref() {} }), clearTimeout() {} };
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: {}, log: quietLog, timers });
+    s.script.push("internal", "timeout", "internal", "timeout", "internal");
+    e.submit(snap(1));
+    for (let i = 0; i < 6; i++) await e.flush();
+    assert.equal(e.stats.droppedGaveUp, 1);
+    assert.equal(e.pending, 0);
+    e.close();
+});
+
+test("emitter: a label the DLP would change is not sent (the candidate is dropped, counted); with DLP off the label goes as is", async () => {
+    const secret = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8";
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: { LOCAL_AGENT_DLP_MODE: "enforce" }, log: quietLog });
+    e.submit(snap(1, { service: `svc-${secret}` }));
+    await e.flush();
+    assert.equal(s.sent.length, 0);
+    assert.equal(e.stats.droppedDlp, 1);
+    e.submit(snap(2, { service: "billing-api" }));
+    await e.flush();
+    assert.equal(s.sent.length, 1);
+    e.close();
+});
+
+test("emitter: not ready keeps the candidates; the next ready sends them (one at a time, in order)", async () => {
+    const s = new FakeSession(); s.state = "detached";
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: {}, log: quietLog });
+    e.submit(snap(1)); e.submit(snap(2));
+    await e.flush();
+    assert.equal(s.sent.length, 0);
+    s.state = "ready"; s.emit("ready", { resumed: true });
+    await e.flush();
+    assert.deepEqual(s.sent.map((x) => x.payload.errorHash), [H(1), H(2)]);
+    e.close();
+});
+
+// ------------------------------------------------------------------------------------------ untrusted timestamps (review of #65)
+
+test("an absurd timestamp in a log line is not a time: parse gives null, the aggregator uses the clock, the emitter never blocks", async () => {
+    for (const v of [-1e30, 1e30, "-99999-01-01T00:00:00Z"]) {
+        const t = parseTime(v);
+        assert.ok(t === null || Math.abs(t) <= 8.64e15, `parseTime(${v}) = ${t}: it must be null or a valid Date value`);
+    }
+    assert.equal(parseTime(-1e30), null);
+    assert.equal(parseTime(1e300), null);
+    // the aggregator ignores a negative or non-finite time: firstSeen is the clock, never -1e33
+    const { clock, sent, obs } = aggRig();
+    obs(H(7), { tsMs: -1e33 });
+    assert.equal(sent[0].firstSeen, clock.t);
+    obs(H(8), { tsMs: -5 });
+    assert.equal(sent[1].firstSeen, clock.t);
+    // even if a bad snapshot reached the emitter, it is dropped (invalid) and the next candidate still goes out
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: {}, log: quietLog });
+    e.submit(snap(1, { firstSeen: -1e33, lastSeen: -1e33 }));
+    await e.flush();
+    assert.equal(s.sent.length, 0, "the bad candidate is not sent");
+    assert.equal(e.pending, 0, "and it no longer sits at the head of the queue (it used to block it for ever)");
+    e.submit(snap(2));
+    await e.flush();
+    assert.equal(s.sent.length, 1, "the good one goes out");
+    assert.equal(s.sent[0].payload.errorHash, H(2));
+    assert.equal(e.stats.droppedInvalid >= 1, true);
+    assert.equal(e.stats.droppedDlp, 0, "it is not a DLP drop");
+    e.close();
+});

package/test/incident-detector/wiring.test.js

@@ -0,0 +1,165 @@
+// F3-11b: the detector wired on the REAL v2 session (buildV2Link) against the in-memory reference control plane, with real files.
+// What the CP receives is checked on the wire: 9 metadata fields, and never a word of the error text.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { ReferenceCp, TENANT, AGENT, waitFor } from "../la-protocol-v2/helpers/reference-cp.js";
+import { signingFor } from "../la-protocol-v2/helpers/la-rig.js";
+import { buildV2Link } from "../../src/la-protocol-v2/runtime.js";
+import { createIncidentDetector } from "../../src/incident-detector/detector.js";
+import { resolveIncidentDetectorConfig } from "../../src/incident-detector/config.js";
+
+const VECTORS = JSON.parse(fs.readFileSync(path.join(path.dirname(new URL(import.meta.url).pathname), "parity.vectors.json"), "utf8"));
+const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), "la-incw-"));
+const GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8";
+const CANARIES = ["CANARY-MSG-7731", "CanaryVault", "/home/customer/secret-app", "order-9001-detail", GH];
+
+async function linkFor(cp, env, root = tmp()) {
+    const logs = [];
+    const log = { info: (m) => logs.push(m), log: (m) => logs.push(m), warn: (m) => logs.push(m), error: (m) => logs.push(m) };
+    const wsManager = { list: () => [{ id: "ws-1", root, tenantId: TENANT }] };
+    const link = await buildV2Link({ tenantId: TENANT, agentId: AGENT, signing: signingFor({ keyId: cp.keyId, publicKeyRaw: cp.publicKeyRaw }, "enforce"), wsManager, getWorkspaceRoot: () => root, log, env });
+    const transport = { send: (t) => cp.feed(t), close() {}, bufferedAmount: () => 0 };
+    cp.attach((t) => queueMicrotask(() => link.session.handleText(t, transport)), () => {});
+    link.session.attach(transport);
+    await waitFor(() => link.session.state === "ready");
+    return { link, logs };
+}
+const candidates = (cp) => cp.received.filter((f) => f.type === "incident.candidate");
+const envFor = (dir, extra = {}) => ({
+    LOCAL_AGENT_INCIDENT_DETECTOR: "true", LOCAL_AGENT_INCIDENT_LOG_FILES: path.join(dir, "billing-api.log"), LOCAL_AGENT_INCIDENT_STORE_DIR: path.join(dir, "store"),
+    LOCAL_AGENT_INCIDENT_FILE_POLL_MS: "50", LOCAL_AGENT_INCIDENT_FLUSH_MS: "300", LOCAL_AGENT_DLP_MODE: "enforce", ...extra,
+});
+
+test("flag off (unset or any non-true value): no capability, nothing started, the log file is not read, no store dir; flag on without a source warns and starts nothing", async () => {
+    for (const v of [undefined, "false", "maybe"]) {
+        const dir = tmp();
+        fs.writeFileSync(path.join(dir, "billing-api.log"), "");
+        const cp = new ReferenceCp();
+        const { link } = await linkFor(cp, { LOCAL_AGENT_INCIDENT_DETECTOR: v, LOCAL_AGENT_INCIDENT_LOG_FILES: path.join(dir, "billing-api.log"), LOCAL_AGENT_INCIDENT_STORE_DIR: path.join(dir, "store") });
+        fs.appendFileSync(path.join(dir, "billing-api.log"), "2026-10-29 10:00:00 ERROR java.lang.IllegalStateException: boom\n");
+        await new Promise((r) => setTimeout(r, 150));
+        assert.equal(link.incidents, undefined);
+        assert.ok(!cp.registers[0].capabilities.includes("incident.candidate"));
+        assert.equal(candidates(cp).length, 0);
+        assert.ok(!fs.existsSync(path.join(dir, "store")), "no store dir");
+        link.close();
+    }
+    const cp = new ReferenceCp();
+    const { link, logs } = await linkFor(cp, { LOCAL_AGENT_INCIDENT_DETECTOR: "true" });
+    assert.equal(link.incidents, undefined);
+    assert.ok(logs.some((l) => /no source is configured/.test(l)));
+    assert.ok(!cp.registers[0].capabilities.includes("incident.candidate"));
+    link.close();
+});
+
+test("flag on + file source: candidates carry the 9 metadata fields and nothing of the text; the detail is in the local store; the capability is announced", async () => {
+    const dir = tmp(); const file = path.join(dir, "billing-api.log");
+    fs.writeFileSync(file, "");
+    const cp = new ReferenceCp();
+    const { link } = await linkFor(cp, envFor(dir));
+    assert.ok(cp.registers[0].capabilities.includes("incident.candidate"));
+    await new Promise((r) => setTimeout(r, 120));   // the tailer has taken its offset
+    const stack = `\tat com.acme.CanaryVault.open(CanaryVault.java:44)\n\tat com.acme.Main.run(Main.java:9)\n`;
+    fs.appendFileSync(file,
+        `2026-10-29 10:00:00 ERROR c.a.Order : java.lang.IllegalStateException: CANARY-MSG-7731 order 9001 not found\n`
+        + `java.lang.IllegalStateException: CANARY-MSG-7731 order-9001-detail at /home/customer/secret-app token ${GH}\n` + stack
+        + "2026-10-29 10:00:01 INFO nothing to see\n");
+    await waitFor(() => candidates(cp).length >= 1, 4000);
+    const c = candidates(cp)[0];
+    assert.deepEqual(Object.keys(c.payload).sort(), ["count", "environment", "errorHash", "fingerprint", "firstSeen", "lastSeen", "logCategory", "service", "severity"]);
+    assert.equal(c.payload.service, "billing-api");
+    assert.equal(c.payload.environment, "production");
+    assert.equal(c.payload.severity, "high");
+    assert.equal(c.payload.logCategory, "exception");
+    assert.equal(c.payload.count, 1);
+    assert.match(c.payload.errorHash, /^[0-9a-f]{32}$/);
+    assert.match(c.payload.fingerprint, /^[0-9a-f]{64}$/);
+    assert.equal(c.workspaceId, "");
+    // NOTHING of the text on the wire (every raw frame of the session, both the handshake and the data)
+    const wire = cp.rawIn.join("\n");
+    for (const w of CANARIES) assert.ok(!wire.includes(w), `the wire must not carry ${w.slice(0, 12)}...`);
+    // the detail stayed local, masked
+    const detail = await link.incidents.getDetail(c.payload.errorHash);
+    assert.equal(detail.length, 1);
+    assert.ok(detail[0].message.includes("CANARY-MSG-7731"));
+    assert.ok(!JSON.stringify(detail).includes(GH), "the DLP masked the token before it was stored");
+    assert.ok(detail[0].stack.includes("CanaryVault"));
+    assert.equal(detail[0].stackBytes, Buffer.byteLength(`java.lang.IllegalStateException: CANARY-MSG-7731 order-9001-detail at /home/customer/secret-app token ${GH}\n` + stack.replace(/\n$/, "")), "the size of the stack is local metadata");
+    assert.equal(fs.statSync(path.join(dir, "store", "incident-detail.jsonl")).mode & 0o777, 0o600);
+    // the same error with other ids: same errorHash, ONE window: the count grows in a batched update
+    for (let i = 0; i < 5; i++) fs.appendFileSync(file, `2026-10-29 10:00:0${2 + i} ERROR c.a.Order : java.lang.IllegalStateException: CANARY-MSG-7731 order ${9100 + i} not found\n`);
+    await waitFor(() => candidates(cp).length >= 2, 4000);
+    const u = candidates(cp).at(-1).payload;
+    assert.equal(u.firstSeen, c.payload.firstSeen, "same window");
+    assert.ok(u.count >= 2);
+    const wire2 = cp.rawIn.join("\n");
+    for (const w of CANARIES) assert.ok(!wire2.includes(w));
+    // the audit has no text either
+    assert.ok(!JSON.stringify(link.incidents.audit.tail(100)).includes("CANARY"));
+    link.close();
+});
+
+test("flag on: noise, INFO/WARN, duplicates and an entry without a message produce no candidate", async () => {
+    const dir = tmp(); const file = path.join(dir, "billing-api.log");
+    fs.writeFileSync(file, "");
+    const cp = new ReferenceCp();
+    const { link } = await linkFor(cp, envFor(dir));
+    await new Promise((r) => setTimeout(r, 120));
+    fs.appendFileSync(file, [
+        '2026-10-29 10:00:00 ERROR {code=9, message=Ready condition status changed to False}',
+        "2026-10-29 10:00:01 WARN java.lang.RuntimeException: not an error level",
+        "2026-10-29 10:00:02 ERROR GET /actuator/health Exception",
+        "2026-10-29 10:00:03 ERROR plain failure without any marker",
+        "2026-10-29 10:00:04 ERROR org.apache.maven.lifecycle.LifecycleExecutionException: build",
+        "",
+    ].join("\n"));
+    await new Promise((r) => setTimeout(r, 500));
+    assert.equal(candidates(cp).length, 0);
+    const s = link.incidents.detector.stats;
+    assert.equal(s.noise, 1, "the Maven build failure is significant to ErrorDetector but infra/build noise to LogNoiseClassifier");
+    assert.equal(s.insignificant, 4, JSON.stringify(s));
+    assert.equal(s.detected, 0);
+    link.close();
+});
+
+// ------------------------------------------------------------------------------------------ the Java vectors through the whole pipeline
+
+test("parity end to end: every significant vector goes through detector -> aggregator -> emitter and the candidate carries the Java errorHash and fingerprint", async () => {
+    const sent = [];
+    const session = { state: "ready", on() {}, off() {}, async send(type, payload) { sent.push(payload); } };
+    const { createCandidateEmitter } = await import("../../src/incident-detector/emitter.js");
+    const { createDetailStore } = await import("../../src/incident-detector/store.js");
+    const config = resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: "1", LOCAL_AGENT_INCIDENT_STORE_DIR: path.join(tmp(), "s"), LOCAL_AGENT_INCIDENT_MAX_PENDING: "5000", LOCAL_AGENT_INCIDENT_MAX_EMITS_PER_MINUTE: "600", LOCAL_AGENT_INCIDENT_MAX_TRACKED: "5000" }, () => {});
+    config.emitBurst = 100_000;
+    const emitter = createCandidateEmitter({ session, config, env: {}, log: { warn() {}, info() {}, log() {} } });
+    const store = createDetailStore({ dir: config.storeDir, maxBytes: 8 << 20, maxQueue: 100, messageChars: 1000, stackChars: 1000, samplesPerWindow: 1, env: {} });
+    const det = createIncidentDetector({ config, emitter, store, log: console });
+    let expected = 0;
+    const want = new Map();
+    for (const c of VECTORS.cases) {
+        const e = c.expect;
+        if (!e.significant || e.noiseClass !== "app_error" || e.fingerprint == null) continue;
+        const svc = `svc-${expected}`;
+        await det.ingest({ severity: c.severity, message: c.message, stacktrace: c.stack, service: svc, tsMs: 1_700_000_000_000 + expected });
+        want.set(svc, e);
+        expected++;
+    }
+    await emitter.flush();
+    assert.ok(expected >= 40, `${expected} significant vectors`);
+    // errorHash is shared by cases with the same normalised first line: one window per errorHash, so compare per hash
+    const byHash = new Map(sent.map((p) => [p.errorHash, p]));
+    const hashes = new Set([...want.values()].map((e) => e.groupHash));
+    assert.equal(byHash.size, hashes.size, "one window per Java errorHash");
+    for (const e of want.values()) {
+        const p = byHash.get(e.groupHash);
+        assert.ok(p, "candidate for " + e.groupHash);
+    }
+    // the first occurrence of each hash carries ITS fingerprint (first one wins, as in incident_dedup_log)
+    const firstOfHash = new Map();
+    for (const e of want.values()) if (!firstOfHash.has(e.groupHash)) firstOfHash.set(e.groupHash, e);
+    for (const [h, e] of firstOfHash) assert.equal(byHash.get(h).fingerprint, e.fingerprint, "Java fingerprint of the first occurrence of " + h);
+    emitter.close();
+});