deepdebug-local-agent 1.7.0 → 1.8.0

package/test/incident-detector/sources.test.js

@@ -0,0 +1,179 @@
+// F3-11b: the log sources (file tail, Loki, stream) with real files, a fake fetch and real streams.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { PassThrough, Readable } from "node:stream";
+import { createFileSource } from "../../src/incident-detector/sources/file-source.js";
+import { createLokiSource } from "../../src/incident-detector/sources/loki-source.js";
+import { createStreamSource } from "../../src/incident-detector/sources/stream-source.js";
+
+const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), "la-src-"));
+const collect = () => { const got = []; return { got, handler: async (entries) => { got.push(...entries); } }; };
+
+// ------------------------------------------------------------------------------------------ file
+
+test("file: only NEW lines by default; fromStart reads what is there; a record spanning appends is one entry; the file name is the service", async () => {
+    const dir = tmp(); const f = path.join(dir, "billing-api.log");
+    fs.writeFileSync(f, "2026-10-29 10:00:00 ERROR old one\n");
+    const a = collect();
+    const src = createFileSource({ filePath: f, idleFlushMs: 60_000 });
+    src.start(a.handler); await src.poll();
+    assert.equal(a.got.length, 0, "history is not replayed");
+    fs.appendFileSync(f, "2026-10-29 10:00:01 ERROR c.a.Foo : failed 7\njava.lang.IllegalStateException: x\n");
+    await src.poll();
+    fs.appendFileSync(f, "\tat com.x.A.m(A.java:1)\n2026-10-29 10:00:02 INFO ok\n");
+    await src.poll();
+    src.stop();
+    assert.equal(a.got.length, 1, "the INFO line closed the ERROR record; the INFO record is still in progress");
+    assert.equal(a.got[0].service, "billing-api");
+    assert.match(a.got[0].stacktrace, /IllegalStateException[\s\S]*A\.java:1/);
+    const b = collect();
+    const s2 = createFileSource({ filePath: f, fromStart: true, idleFlushMs: 60_000 });
+    s2.start(b.handler); await s2.poll(); s2.stop();
+    assert.equal(b.got.length, 2);
+});
+
+test("file: rotation (new inode) and truncation restart at 0; a missing file is waited for; a partial last line waits for its newline; CRLF and multibyte are intact", async () => {
+    const dir = tmp(); const f = path.join(dir, "x.log");
+    const a = collect();
+    const src = createFileSource({ filePath: f, fromStart: true, idleFlushMs: 0 });
+    src.start(a.handler);
+    await src.poll();                                   // no file yet
+    fs.writeFileSync(f, "2026-10-29 10:00:00 ERROR one é💥\r\n2026-10-29 10:00:01 ERROR par");
+    await src.poll();
+    assert.deepEqual(a.got.map((e) => e.message), ["2026-10-29 10:00:00 ERROR one é💥"], "the partial line is not an entry yet");
+    fs.appendFileSync(f, "tial\n");
+    await src.poll();
+    assert.equal(a.got.at(-1).message, "2026-10-29 10:00:01 ERROR partial");
+    fs.renameSync(f, f + ".1");
+    fs.writeFileSync(f, "2026-10-29 10:00:02 ERROR after rotation\n");
+    await src.poll();
+    assert.equal(a.got.at(-1).message, "2026-10-29 10:00:02 ERROR after rotation");
+    fs.writeFileSync(f, "2026-10-29 10:00:03 ERROR t\n");   // truncate + rewrite (shorter than the offset)
+    await src.poll();
+    assert.equal(a.got.at(-1).message, "2026-10-29 10:00:03 ERROR t");
+    src.stop();
+});
+
+test("file: an over-long line is cut at maxLineBytes (the rest to its newline is discarded and counted); reading is bounded per tick", async () => {
+    const dir = tmp(); const f = path.join(dir, "y.log");
+    fs.writeFileSync(f, "");
+    const a = collect(); const stats = {};
+    const src = createFileSource({ filePath: f, fromStart: true, idleFlushMs: 0, maxLineBytes: 100, maxReadBytes: 64 * 1024, stats });
+    src.start(a.handler); await src.poll();
+    fs.appendFileSync(f, "2026-10-29 10:00:00 ERROR " + "z".repeat(10_000) + "\n2026-10-29 10:00:01 ERROR next\n");
+    await src.poll(); await src.poll();
+    src.stop();
+    assert.equal(stats.longLines, 1);
+    assert.ok(a.got[0].message.length <= 100);
+    assert.equal(a.got.at(-1).message, "2026-10-29 10:00:01 ERROR next");
+    // bounded per tick: 300 KiB are read over several ticks, never more than maxReadBytes (+ one chunk) at once
+    const g = path.join(dir, "big.log");
+    fs.writeFileSync(g, "");
+    const sizes = [];
+    const big = createFileSource({ filePath: g, fromStart: true, idleFlushMs: 0, maxReadBytes: 64 * 1024 });
+    big.start(async (entries) => { sizes.push(entries.length); });
+    await big.poll();
+    fs.appendFileSync(g, "2026-10-29 10:00:00 ERROR line\n".repeat(10_000));
+    await big.poll();
+    const afterOne = sizes.reduce((x, y) => x + y, 0);
+    assert.ok(afterOne > 0 && afterOne < 10_000, `one tick reads a bounded slice (${afterOne})`);
+    big.stop();
+});
+
+// ------------------------------------------------------------------------------------------ Loki
+
+const nsOf = (ms) => (BigInt(ms) * 1_000_000n).toString();
+
+test("loki: query_range with the cursor, bearer + org headers, service from the stream labels, Loki timestamp wins, cursor advances", async () => {
+    const calls = [];
+    const t0 = Date.parse("2026-10-29T10:00:00Z");
+    const fetchImpl = async (url, init) => {
+        calls.push({ url: new URL(url), init });
+        return { ok: true, status: 200, body: Readable.toWeb(Readable.from([Buffer.from(JSON.stringify({ data: { result: [
+            { stream: { service_name: "checkout", level: "error" }, values: [[nsOf(t0 + 5), "boom 2"], [nsOf(t0 + 1), '{"level":"error","msg":"json boom"}']] },
+        ] } }))])) };
+    };
+    const a = collect();
+    const src = createLokiSource({ url: "https://loki.internal:3100", query: '{app="x"}', token: "tok", org: "tenant-1", fetchImpl, now: () => t0 + 1000, lookbackMs: 60_000, limit: 500 });
+    src.start(a.handler); await src.poll();
+    const c = calls[0];
+    assert.equal(c.url.pathname, "/loki/api/v1/query_range");
+    assert.equal(c.url.searchParams.get("query"), '{app="x"}');
+    assert.equal(c.url.searchParams.get("direction"), "forward");
+    assert.equal(c.url.searchParams.get("start"), nsOf(t0 + 1000 - 60_000));
+    assert.equal(c.init.headers.Authorization, "Bearer tok");
+    assert.equal(c.init.headers["X-Scope-OrgID"], "tenant-1");
+    assert.equal(c.init.redirect, "error");
+    assert.deepEqual(a.got.map((e) => [e.message, e.service]), [["json boom", "checkout"], ["boom 2", "checkout"]], "sorted by time");
+    assert.equal(a.got[0].tsNs, BigInt(nsOf(t0 + 1)));
+    await src.poll();
+    src.stop();
+    assert.equal(calls[1].url.searchParams.get("start"), (BigInt(nsOf(t0 + 5)) + 1n).toString(), "cursor = last ns + 1");
+});
+
+test("loki: a full page continues at once (up to maxPages); an error is counted, never thrown, and carries nothing to the log", async () => {
+    let n = 0;
+    const t0 = Date.parse("2026-10-29T10:00:00Z");
+    const fetchImpl = async () => {
+        n++;
+        const values = [[nsOf(t0 + n * 10), `ERROR e${n}`], [nsOf(t0 + n * 10 + 1), `ERROR f${n}`]];
+        return { ok: true, status: 200, body: Readable.toWeb(Readable.from([Buffer.from(JSON.stringify({ data: { result: [{ stream: { app: "a" }, values }] } }))])) };
+    };
+    const a = collect(); const stats = {};
+    const src = createLokiSource({ url: "http://l", query: "q", fetchImpl, limit: 2, maxPages: 3, now: () => t0, stats });
+    src.start(a.handler); await src.poll(); src.stop();
+    assert.equal(n, 3, "maxPages");
+    assert.equal(a.got.length, 6);
+    const bad = createLokiSource({ url: "http://l", query: "q", fetchImpl: async () => { throw new Error("connect ECONNREFUSED http://secret-host:3100"); }, stats: {} });
+    bad.start(async () => {});
+    await new Promise((r) => setTimeout(r, 30));
+    assert.equal(bad.stats.errors, 1);
+    bad.stop();
+});
+
+test("loki: a response over maxResponseBytes, a non-200 and malformed rows are survived", async () => {
+    const mk = (body, status = 200) => async () => ({ ok: status === 200, status, body: Readable.toWeb(Readable.from([Buffer.from(body)])) });
+    const a = collect();
+    const big = createLokiSource({ url: "http://l", query: "q", fetchImpl: mk("x".repeat(5000)), maxResponseBytes: 1000 });
+    big.start(a.handler);
+    await assert.rejects(big.poll(), /response_too_large/);
+    big.stop();
+    const bad = createLokiSource({ url: "http://l", query: "q", fetchImpl: mk("{}", 500) });
+    bad.start(a.handler);
+    await assert.rejects(bad.poll(), /http_500/);
+    bad.stop();
+    const odd = createLokiSource({ url: "http://l", query: "q", fetchImpl: mk(JSON.stringify({ data: { result: [{ stream: null, values: [["x", "y"], [1, 2], ["5", "ERROR ok"], "z"] }, null] } })) });
+    odd.start(a.handler); await odd.poll(); odd.stop();
+    assert.deepEqual(a.got.map((e) => e.message), ["ERROR ok"]);
+});
+
+// ------------------------------------------------------------------------------------------ stream
+
+test("stream: lines become entries, the service label is the default, the record in progress is flushed at the end", async () => {
+    const input = new PassThrough();
+    const a = collect();
+    const src = createStreamSource({ stream: input, service: "container-1", idleFlushMs: 20 });
+    src.start(a.handler);
+    input.write("2026-10-29 10:00:00 ERROR boom\n\tat a.B.c(B.java:1)\n");
+    await new Promise((r) => setTimeout(r, 80));
+    assert.equal(a.got.length, 1, "idle flush");
+    input.end("2026-10-29 10:00:01 ERROR last");
+    await new Promise((r) => setTimeout(r, 50));
+    assert.deepEqual(a.got.map((e) => e.service), ["container-1", "container-1"]);
+    src.stop();
+});
+
+test("stream: backpressure, a slow consumer pauses the stream instead of buffering the world", async () => {
+    const input = new PassThrough({ highWaterMark: 1024 });
+    let consumed = 0;
+    const src = createStreamSource({ stream: input, idleFlushMs: 5 });
+    src.start(async (entries) => { consumed += entries.length; await new Promise((r) => setTimeout(r, 30)); });
+    let written = 0; let backpressured = false;
+    for (let i = 0; i < 2000; i++) { const ok = input.write("2026-10-29 10:00:00 ERROR line " + i + "\n"); written++; if (!ok) { backpressured = true; break; } }
+    assert.equal(backpressured, true, "the writer is told to stop long before 2000 lines");
+    assert.ok(written < 2000);
+    src.stop(); input.destroy();
+});

package/test/incident-detector/units.test.js

@@ -0,0 +1,430 @@
+// F3-11b: unit tests of the pieces (config, parse, categorize, aggregator, store, emitter) with fake clocks and a fake session.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { EventEmitter } from "node:events";
+import { resolveIncidentDetectorConfig } from "../../src/incident-detector/config.js";
+import { parseRecord, EntryAssembler, parseTime } from "../../src/incident-detector/parse.js";
+import { severityOf, maxSeverity, inferEnvironment, sanitizeService, sanitizeEnvironment, logCategoryOf } from "../../src/incident-detector/categorize.js";
+import { createAggregator } from "../../src/incident-detector/aggregator.js";
+import { createDetailStore } from "../../src/incident-detector/store.js";
+import { createCandidateEmitter } from "../../src/incident-detector/emitter.js";
+import { ProtocolError } from "../../src/la-protocol-v2/errors.js";
+
+const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), "la-inc-"));
+const H = (n) => n.toString(16).padStart(32, "0");
+
+// ------------------------------------------------------------------------------------------ config
+
+test("config: only true/1/yes/on enable it; unset, empty, false and unknown values keep it off (unknown warns)", () => {
+    for (const v of [undefined, "", "false", "0", "no", "off", " OFF ", "maybe", "2", "enabled"]) {
+        const warns = [];
+        assert.deepEqual(resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: v }, (m) => warns.push(m)), { enabled: false }, String(v));
+        assert.equal(warns.length, ["maybe", "2", "enabled"].includes(v) ? 1 : 0, String(v));
+    }
+    for (const v of ["true", "TRUE", "1", "yes", "on", " On "]) assert.equal(resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: v }, () => {}).enabled, true, v);
+});
+
+test("config: numbers are clamped, a bad value falls back to the default (never unlimited); sources and Loki validation", () => {
+    const warns = [];
+    const c = resolveIncidentDetectorConfig({
+        LOCAL_AGENT_INCIDENT_DETECTOR: "true", LOCAL_AGENT_INCIDENT_MAX_TRACKED: "99999999", LOCAL_AGENT_INCIDENT_MAX_PENDING: "0", LOCAL_AGENT_INCIDENT_FLUSH_MS: "abc",
+        LOCAL_AGENT_INCIDENT_MAX_EMITS_PER_MINUTE: "-1", LOCAL_AGENT_INCIDENT_LOG_FILES: " /a.log , ,/b.log", LOCAL_AGENT_INCIDENT_STDIN: "yes",
+        LOCAL_AGENT_INCIDENT_LOKI_URL: "ftp://x", LOCAL_AGENT_INCIDENT_MAX_ENTRY_BYTES: "1",
+    }, (m) => warns.push(m));
+    assert.equal(c.maxTracked, 5_000);
+    assert.equal(c.maxPending, 500);
+    assert.equal(c.flushMs, 30_000);
+    assert.equal(c.maxEmitsPerMinute, 60);
+    assert.equal(c.maxEntryBytes, 64 * 1024);
+    assert.deepEqual(c.files, ["/a.log", "/b.log"]);
+    assert.equal(c.stdin, true);
+    assert.equal(c.loki, null);
+    assert.equal(warns.length, 1);
+    const l = resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: "1", LOCAL_AGENT_INCIDENT_LOKI_URL: "https://loki.internal:3100/", LOCAL_AGENT_INCIDENT_LOKI_QUERY: '{app="x"} |= "error"', LOCAL_AGENT_INCIDENT_LOKI_TOKEN: "t" }, () => {});
+    assert.equal(l.loki.url, "https://loki.internal:3100");
+    assert.equal(l.loki.token, "t");
+    const noQuery = [];
+    assert.equal(resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: "1", LOCAL_AGENT_INCIDENT_LOKI_URL: "http://l" }, (m) => noQuery.push(m)).loki, null);
+    assert.equal(noQuery.length, 1);
+});
+
+// ------------------------------------------------------------------------------------------ parse
+
+test("parse: JSON lines with aliases, separate stack field, docker json log, bad JSON as text", () => {
+    const a = parseRecord('{"level":"error","msg":"boom 1","stack_trace":"a.B: x\\n\\tat c.D.e(D.java:1)","service":"api","time":"2026-10-29T10:00:00Z"}');
+    assert.deepEqual([a.severity, a.message, a.service, a.tsMs], ["error", "boom 1", "api", Date.parse("2026-10-29T10:00:00Z")]);
+    assert.match(a.stacktrace, /^a\.B: x/);
+    const d = parseRecord('{"log":"ERROR something failed\\n","stream":"stderr","time":"2026-10-29T10:00:00.5Z"}');
+    assert.equal(d.severity, "ERROR");
+    assert.equal(d.message, "ERROR something failed");
+    const bad = parseRecord("{not json ERROR x");
+    assert.equal(bad.severity, "ERROR");
+    assert.equal(parseRecord(""), null);
+    assert.equal(parseRecord('{"nomsg":1}'), null);
+    const multi = parseRecord("java.lang.X: boom\n\tat a.B.c(B.java:1)");
+    assert.equal(multi.message, "java.lang.X: boom");
+    assert.equal(multi.stacktrace, "\tat a.B.c(B.java:1)");
+});
+
+test("parse: times from ISO, epoch s/ms/us/ns", () => {
+    const ms = Date.parse("2026-10-29T10:00:00Z");
+    assert.equal(parseTime("2026-10-29T10:00:00Z"), ms);
+    assert.equal(parseTime("2026-10-29 10:00:00,250"), ms + 250);
+    assert.equal(parseTime(ms), ms);
+    assert.equal(parseTime(ms / 1000), ms);
+    assert.equal(parseTime(ms * 1000), ms);
+    assert.equal(parseTime(ms * 1_000_000), ms);
+    assert.equal(parseTime("nope"), null);
+    assert.equal(parseTime(null), null);
+});
+
+test("assembler: a record owns the lines that do not start a new one; JSON lines stand alone; caps are enforced and counted", () => {
+    const a = new EntryAssembler({ maxEntryBytes: 200, maxStackLines: 3 });
+    const out = [];
+    for (const l of ["2026-10-29 10:00:00 ERROR c.a.Foo : failed 42", "java.lang.IllegalStateException: x", "\tat a.B.c(B.java:1)", "\tat a.B.d(B.java:2)", "\tat a.B.e(B.java:3)", "\tat a.B.f(B.java:4)",
+        "2026-10-29 10:00:01 INFO fine", '{"level":"error","msg":"json one"}', "2026-10-29 10:00:02 ERROR " + "z".repeat(500)]) out.push(...a.push(l));
+    out.push(...a.flush());
+    assert.equal(out.length, 4);
+    assert.equal(out[0].severity, "ERROR");
+    assert.equal(out[0].stacktrace.split("\n").length, 3, "maxStackLines");
+    assert.equal(out[1].severity, "INFO");
+    assert.equal(out[2].message, "json one");
+    assert.ok(out[3].message.length <= 200, "maxEntryBytes");
+    assert.equal(a.truncated, 2);
+});
+
+// ------------------------------------------------------------------------------------------ categorize
+
+test("categorize: severity, environment, sanitised labels fit the contract patterns, categories", () => {
+    assert.deepEqual(["FATAL", "critical", "alert", "emerg", "ERROR", "severe", "WARN", null].map(severityOf), ["critical", "critical", "critical", "critical", "high", "high", "medium", "medium"]);
+    assert.equal(maxSeverity("high", "critical"), "critical");
+    assert.equal(maxSeverity("critical", "high"), "critical");
+    assert.deepEqual(["api-qa", "api-qa-2", "x-demo", "x-staging", "x-stg", "api", "", null].map(inferEnvironment), ["qa", "qa", "demo", "staging", "staging", "production", "production", "production"]);
+    for (const raw of ["my service!", "a".repeat(300), "", null, "ok.name:v1/x-y", "é💥"]) assert.match(sanitizeService(raw), /^[A-Za-z0-9._:/-]{1,128}$/, String(raw));
+    for (const raw of ["Prod!", "-x", "", null, "B".repeat(99), "staging-eu"]) assert.match(sanitizeEnvironment(raw), /^[a-z0-9][a-z0-9_-]{0,31}$/, String(raw));
+    const cat = (m, s = null, t = null) => logCategoryOf(m, s, t);
+    assert.equal(cat("java.lang.OutOfMemoryError: Java heap space timed out"), "oom");
+    assert.equal(cat("java.util.concurrent.TimeoutException: x"), "timeout");
+    assert.equal(cat("org.postgresql.util.PSQLException: boom"), "db");
+    assert.equal(cat("x", null, "PG_ERROR"), "db");
+    assert.equal(cat("AccessDeniedException: no"), "security");
+    assert.equal(cat("request failed HTTP 503"), "http_5xx");
+    assert.equal(cat("java.lang.IllegalStateException: x", null, "java.lang.IllegalStateException"), "exception");
+    assert.equal(cat("plain failure"), "other");
+});
+
+// ------------------------------------------------------------------------------------------ aggregator
+
+function aggRig(opts = {}) {
+    const clock = { t: 1_000_000 };
+    const sent = [];
+    const agg = createAggregator({ windowMs: 10_000, flushMs: 2_000, maxTracked: 3, now: () => clock.t, onCandidate: (s) => sent.push(s), ...opts });
+    const obs = (hash, extra = {}) => agg.observe({ errorHash: hash, fingerprint: "f".repeat(64), severity: "high", service: "api", environment: "qa", logCategory: "exception", tsMs: clock.t, ...extra });
+    return { clock, sent, agg, obs };
+}
+
+test("aggregator: leading edge at once, then batched updates with growing count and a stable window identity", () => {
+    const { clock, sent, obs, agg } = aggRig();
+    assert.equal(obs(H(1)).opened, true);
+    assert.equal(sent.length, 1);
+    assert.equal(sent[0].count, 1);
+    clock.t += 100; obs(H(1)); clock.t += 100; obs(H(1));
+    assert.equal(sent.length, 1, "within flushMs nothing more is sent");
+    clock.t += 2_000; obs(H(1));
+    assert.equal(sent.length, 2);
+    assert.equal(sent[1].count, 4);
+    assert.equal(sent[1].firstSeen, sent[0].firstSeen, "same window: same firstSeen");
+    assert.equal(sent[1].windowStart, sent[0].windowStart);
+    clock.t += 500; obs(H(1));
+    clock.t += 2_000; agg.tick();
+    assert.equal(sent.length, 3, "tick sends the due update");
+    assert.equal(sent[2].count, 5);
+    clock.t += 2_000; agg.tick();
+    assert.equal(sent.length, 3, "nothing new: nothing sent");
+});
+
+test("aggregator: the window closes with a final update and the next occurrence opens a NEW window (new firstSeen, count restarts)", () => {
+    const { clock, sent, obs, agg } = aggRig();
+    obs(H(1)); clock.t += 500; obs(H(1));          // count 2, 1 unsent
+    clock.t += 10_000; agg.tick();                  // window over: final update
+    assert.deepEqual(sent.map((s) => s.count), [1, 2]);
+    clock.t += 1; const r = obs(H(1));
+    assert.equal(r.opened, true);
+    assert.equal(sent.at(-1).count, 1);
+    assert.ok(sent.at(-1).windowStart > sent[0].windowStart);
+    assert.equal(agg.stats.windows, 2);
+});
+
+test("aggregator: an occurrence that arrives after the window ended (no tick yet) closes it itself: final update, then a new window", () => {
+    const { clock, sent, obs } = aggRig();
+    obs(H(1)); clock.t += 100; obs(H(1)); obs(H(1));       // count 3, 2 unsent
+    clock.t += 10_000;
+    obs(H(1));
+    assert.deepEqual(sent.map((s) => s.count), [1, 3, 1]);
+    assert.ok(sent[2].windowStart > sent[0].windowStart);
+});
+
+test("aggregator: highest severity of the window, first fingerprint/category win, firstSeen never in the future, an old line keeps the window identity", () => {
+    const { clock, sent, obs } = aggRig();
+    obs(H(1), { severity: "high", fingerprint: "a".repeat(64), logCategory: "db", tsMs: clock.t + 999_999 });
+    assert.equal(sent[0].firstSeen, clock.t, "clamped to now");
+    clock.t += 3_000;
+    obs(H(1), { severity: "critical", fingerprint: "b".repeat(64), logCategory: "oom", tsMs: 5 });
+    assert.equal(sent[1].severity, "critical");
+    assert.equal(sent[1].fingerprint, "a".repeat(64));
+    assert.equal(sent[1].logCategory, "db");
+    assert.equal(sent[1].firstSeen, sent[0].firstSeen);
+});
+
+test("aggregator: maxTracked evicts the least recently seen hash (with its pending update) and counts it", () => {
+    const { clock, sent, obs, agg } = aggRig();
+    obs(H(1)); obs(H(2)); obs(H(3));
+    clock.t += 100; obs(H(1));                     // 1 is now the most recent; 2 is the oldest; 1 has an unsent update
+    obs(H(4));                                     // evicts 2 (nothing pending for it)
+    assert.equal(agg.tracked, 3);
+    assert.equal(agg.stats.evicted, 1);
+    clock.t += 100; obs(H(3)); obs(H(5));          // evicts 1 (it has count 2 unsent: a final update goes out)
+    const finalOf1 = sent.filter((s) => s.errorHash === H(1));
+    assert.deepEqual(finalOf1.map((s) => s.count), [1, 2]);
+});
+
+test("aggregator: drain() sends what is pending and forgets the windows", () => {
+    const { clock, sent, obs, agg } = aggRig();
+    obs(H(1)); clock.t += 10; obs(H(1)); obs(H(2));
+    agg.drain();
+    assert.equal(agg.tracked, 0);
+    assert.deepEqual(sent.map((s) => [s.errorHash, s.count]), [[H(1), 1], [H(2), 1], [H(1), 2]]);
+});
+
+// ------------------------------------------------------------------------------------------ store
+
+function storeRig(extra = {}) {
+    const dir = path.join(tmp(), "inc");
+    const store = createDetailStore({ dir, maxBytes: 4096, maxQueue: 5, messageChars: 200, stackChars: 300, samplesPerWindow: 2, env: { LOCAL_AGENT_DLP_MODE: "enforce" }, ...extra });
+    return { dir, store, file: path.join(dir, "incident-detail.jsonl") };
+}
+const rec = (n, extra = {}) => ({ errorHash: H(n), fingerprint: "f".repeat(64), service: "api", severity: "high", windowStart: 1, message: `msg ${n}`, stack: `stack ${n}`, ...extra });
+
+test("store: 0700 dir, 0600 file, one JSON per line, the detail is readable back (newest first), samples per window are capped", async () => {
+    const { store, dir, file } = storeRig();
+    for (let i = 0; i < 4; i++) await store.add(rec(1, { message: `first ${i}` }));
+    await store.add(rec(2, { windowStart: 2 }));
+    await store.flush();
+    assert.equal(store.stats.written, 3);
+    assert.equal(store.stats.sampledOut, 2);
+    assert.equal(fs.statSync(file).mode & 0o777, 0o600);
+    assert.equal(fs.statSync(dir).mode & 0o777, 0o700);
+    const got = await store.getDetail(H(1), { limit: 5 });
+    assert.deepEqual(got.map((r) => r.message), ["first 1", "first 0"]);
+    assert.equal(got[0].stackBytes, 7);
+    assert.deepEqual(await store.getDetail(H(9)), []);
+});
+
+test("store: the DLP masks secrets before they are written; text and stack are cut; a denied value is never written", async () => {
+    const { store, file } = storeRig();
+    const secret = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8";
+    await store.add(rec(1, { message: `token ${secret} leaked ${"x".repeat(500)}`, stack: `s ${secret}\n` + "y".repeat(900) }));
+    await store.flush();
+    const raw = fs.readFileSync(file, "utf8");
+    assert.ok(!raw.includes(secret), "the secret is not on disk");
+    const r = JSON.parse(raw);
+    assert.equal(r.truncated, true);
+    assert.ok(r.message.length <= 200 + 40 && r.stack.length <= 300 + 40);
+    assert.ok(store.stats.redactions >= 1);
+});
+
+test("store: rotates at maxBytes to .1 (one generation) and reads both", async () => {
+    const { store, file } = storeRig({ maxBytes: 1500 });
+    for (let i = 0; i < 12; i++) await store.add(rec(100 + i, { message: "m".repeat(100) }));
+    await store.flush();
+    assert.ok(store.stats.rotations >= 1);
+    assert.ok(fs.existsSync(`${file}.1`));
+    assert.ok(fs.statSync(file).size <= 1500 + 600);
+    assert.equal((await store.getDetail(H(100 + 11))).length, 1, "newest is in the current file");
+});
+
+test("store: a broken disk disables the store (counted, no throw) and add() resolves false; the queue is bounded", async () => {
+    const dir = tmp();
+    fs.writeFileSync(path.join(dir, "blocker"), "x");
+    const store = createDetailStore({ dir: path.join(dir, "blocker", "sub"), maxBytes: 4096, maxQueue: 2, messageChars: 100, stackChars: 100, samplesPerWindow: 5, env: {} });
+    assert.equal(await store.add(rec(1)), true, "queued before the failure is known");
+    await store.flush();
+    assert.equal(store.broken, true);
+    assert.equal(await store.add(rec(2)), false);
+    assert.ok(store.stats.errors >= 1);
+    assert.ok(store.stats.dropped >= 1);
+});
+
+test("store: add() waits (backpressure) while the queue is full instead of growing it", async () => {
+    const { store } = storeRig({ maxQueue: 2 });
+    let max = 0;
+    const ps = [];
+    for (let i = 0; i < 20; i++) { ps.push(store.add(rec(200 + i, { windowStart: i }))); max = Math.max(max, store.queued); }
+    await Promise.all(ps);
+    await store.flush();
+    assert.ok(max <= 2, `queue never above maxQueue (saw ${max})`);
+    assert.equal(store.stats.written, 20);
+});
+
+// ------------------------------------------------------------------------------------------ emitter (fake session)
+
+class FakeSession extends EventEmitter {
+    constructor() { super(); this.state = "ready"; this.sent = []; this.script = []; }
+    async send(type, payload, opts) {
+        const next = this.script.shift();
+        if (next) throw new ProtocolError(next);
+        this.sent.push({ type, payload, opts });
+    }
+}
+const cfg = (extra = {}) => ({ maxPending: 3, maxEmitsPerMinute: 60, emitBurst: 10, ...extra });
+const snap = (n, extra = {}) => ({ errorHash: H(n), fingerprint: "f".repeat(64), severity: "high", service: "api", environment: "qa", count: 1, firstSeen: 1_700_000_000_000, lastSeen: 1_700_000_001_000, logCategory: "exception", windowStart: 1, ...extra });
+const quietLog = { warn() {}, info() {}, log() {} };
+
+test("emitter: exactly the 9 contract fields, ISO times, the workspace of the session; nothing else", async () => {
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg(), workspaceId: "ws-1", env: {}, log: quietLog });
+    e.submit(snap(1));
+    await e.flush();
+    assert.equal(s.sent.length, 1);
+    assert.deepEqual(Object.keys(s.sent[0].payload).sort(), ["count", "environment", "errorHash", "fingerprint", "firstSeen", "lastSeen", "logCategory", "service", "severity"]);
+    assert.equal(s.sent[0].payload.firstSeen, "2023-11-14T22:13:20.000Z");
+    assert.equal(s.sent[0].opts.workspaceId, "ws-1");
+    assert.equal(s.sent[0].type, "incident.candidate");
+    e.close();
+});
+
+test("emitter: updates of the same window coalesce (the newest count wins); a full queue drops NEW candidates but still takes updates", async () => {
+    const s = new FakeSession(); s.state = "not_ready";   // nothing leaves
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: {}, log: quietLog });
+    e.submit(snap(1)); e.submit(snap(2)); e.submit(snap(3));
+    assert.equal(e.submit(snap(4)), false, "full: a new candidate is dropped");
+    assert.equal(e.stats.droppedFull, 1);
+    e.submit(snap(1, { count: 9 }));
+    assert.equal(e.pending, 3);
+    assert.equal(e.stats.coalesced, 1);
+    s.state = "ready"; s.emit("ready", {});
+    await e.flush();
+    assert.equal(s.sent.length, 3);
+    assert.equal(s.sent[0].payload.count, 9);
+    e.close();
+});
+
+test("emitter: rate limit (token bucket) holds the rest back; the bucket refills with time", async () => {
+    const clock = { t: 0 };
+    const timers = { setTimeout: (fn, ms) => ({ fn, ms, unref() {} }), clearTimeout() {} };
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg({ maxPending: 50, maxEmitsPerMinute: 60, emitBurst: 2 }), env: {}, log: quietLog, now: () => clock.t, timers });
+    for (let i = 1; i <= 5; i++) e.submit(snap(i, { windowStart: i }));
+    await e.flush();
+    assert.equal(s.sent.length, 2, "burst of 2");
+    assert.equal(e.pending, 3);
+    clock.t += 3_000;                                // 3 s at 60/min = 3 tokens (capped at the burst of 2)
+    await e.flush();
+    assert.equal(s.sent.length, 4);
+    e.close();
+});
+
+test("emitter: rate_limited keeps everything and backs off; protocol_unsupported forgets the queue until the next ready; invalid is dropped", async () => {
+    const clock = { t: 0 };
+    const timers = { setTimeout: () => ({ unref() {} }), clearTimeout() {} };
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg({ maxPending: 10 }), env: {}, log: quietLog, now: () => clock.t, timers });
+    s.script.push("rate_limited");
+    e.submit(snap(1)); e.submit(snap(2));
+    await e.flush();
+    assert.equal(s.sent.length, 0);
+    assert.equal(e.pending, 2, "kept");
+    await e.flush();
+    assert.equal(s.sent.length, 0, "still inside the backoff window");
+    clock.t += 31_000;
+    await e.flush();
+    assert.equal(s.sent.length, 2);
+    s.script.push("protocol_unsupported");
+    e.submit(snap(3, { windowStart: 3 }));
+    await e.flush();
+    assert.equal(e.pending, 0, "dropped: this gateway takes none");
+    e.submit(snap(4, { windowStart: 4 }));
+    await e.flush();
+    assert.equal(s.sent.length, 2, "quiet");
+    s.emit("ready", {});
+    e.submit(snap(5, { windowStart: 5 }));
+    await e.flush();
+    assert.equal(s.sent.length, 3, "after ready it talks again");
+    s.script.push("schema_violation");
+    e.submit(snap(6, { windowStart: 6 }));
+    await e.flush();
+    assert.equal(e.stats.droppedInvalid, 1);
+    e.close();
+});
+
+test("emitter: internal/timeout retried, given up after 5 attempts in a row (counted)", async () => {
+    const timers = { setTimeout: () => ({ unref() {} }), clearTimeout() {} };
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: {}, log: quietLog, timers });
+    s.script.push("internal", "timeout", "internal", "timeout", "internal");
+    e.submit(snap(1));
+    for (let i = 0; i < 6; i++) await e.flush();
+    assert.equal(e.stats.droppedGaveUp, 1);
+    assert.equal(e.pending, 0);
+    e.close();
+});
+
+test("emitter: a label the DLP would change is not sent (the candidate is dropped, counted); with DLP off the label goes as is", async () => {
+    const secret = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8";
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: { LOCAL_AGENT_DLP_MODE: "enforce" }, log: quietLog });
+    e.submit(snap(1, { service: `svc-${secret}` }));
+    await e.flush();
+    assert.equal(s.sent.length, 0);
+    assert.equal(e.stats.droppedDlp, 1);
+    e.submit(snap(2, { service: "billing-api" }));
+    await e.flush();
+    assert.equal(s.sent.length, 1);
+    e.close();
+});
+
+test("emitter: not ready keeps the candidates; the next ready sends them (one at a time, in order)", async () => {
+    const s = new FakeSession(); s.state = "detached";
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: {}, log: quietLog });
+    e.submit(snap(1)); e.submit(snap(2));
+    await e.flush();
+    assert.equal(s.sent.length, 0);
+    s.state = "ready"; s.emit("ready", { resumed: true });
+    await e.flush();
+    assert.deepEqual(s.sent.map((x) => x.payload.errorHash), [H(1), H(2)]);
+    e.close();
+});
+
+// ------------------------------------------------------------------------------------------ untrusted timestamps (review of #65)
+
+test("an absurd timestamp in a log line is not a time: parse gives null, the aggregator uses the clock, the emitter never blocks", async () => {
+    for (const v of [-1e30, 1e30, "-99999-01-01T00:00:00Z"]) {
+        const t = parseTime(v);
+        assert.ok(t === null || Math.abs(t) <= 8.64e15, `parseTime(${v}) = ${t}: it must be null or a valid Date value`);
+    }
+    assert.equal(parseTime(-1e30), null);
+    assert.equal(parseTime(1e300), null);
+    // the aggregator ignores a negative or non-finite time: firstSeen is the clock, never -1e33
+    const { clock, sent, obs } = aggRig();
+    obs(H(7), { tsMs: -1e33 });
+    assert.equal(sent[0].firstSeen, clock.t);
+    obs(H(8), { tsMs: -5 });
+    assert.equal(sent[1].firstSeen, clock.t);
+    // even if a bad snapshot reached the emitter, it is dropped (invalid) and the next candidate still goes out
+    const s = new FakeSession();
+    const e = createCandidateEmitter({ session: s, config: cfg(), env: {}, log: quietLog });
+    e.submit(snap(1, { firstSeen: -1e33, lastSeen: -1e33 }));
+    await e.flush();
+    assert.equal(s.sent.length, 0, "the bad candidate is not sent");
+    assert.equal(e.pending, 0, "and it no longer sits at the head of the queue (it used to block it for ever)");
+    e.submit(snap(2));
+    await e.flush();
+    assert.equal(s.sent.length, 1, "the good one goes out");
+    assert.equal(s.sent[0].payload.errorHash, H(2));
+    assert.equal(e.stats.droppedInvalid >= 1, true);
+    assert.equal(e.stats.droppedDlp, 0, "it is not a DLP drop");
+    e.close();
+});