deepdebug-local-agent 1.7.0 → 1.9.0

package/test/incident-detector/size-model.test.js

@@ -0,0 +1,127 @@
+// Contract 2.1 (c) on the detector side (F3-11b): incident.candidate carries `stackBytes` and `model` ONLY when the control plane announced
+// `incident.candidate.size-model` in ready.capabilities. Toward a 2.0 control plane the frame is exactly what it was (9 fields).
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { ReferenceCp, TENANT, AGENT, waitFor } from "../la-protocol-v2/helpers/reference-cp.js";
+import { signingFor } from "../la-protocol-v2/helpers/la-rig.js";
+import { buildV2Link } from "../../src/la-protocol-v2/runtime.js";
+import { createAggregator } from "../../src/incident-detector/aggregator.js";
+import { createCandidateEmitter } from "../../src/incident-detector/emitter.js";
+import { resolveIncidentDetectorConfig } from "../../src/incident-detector/config.js";
+
+const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), "la-size-"));
+const NINE = ["count", "environment", "errorHash", "fingerprint", "firstSeen", "lastSeen", "logCategory", "service", "severity"];
+const base = { errorHash: "a".repeat(32), fingerprint: "b".repeat(64), severity: "high", service: "svc", environment: "production", logCategory: "exception", tsMs: Date.now() };
+
+test("aggregator: stackBytes of a window is the biggest seen, saturating at 16 MiB; no stack = 0; it travels in every snapshot of the window", () => {
+    const out = [];
+    const agg = createAggregator({ windowMs: 60_000, flushMs: 0, maxTracked: 10, now: () => 1_000, onCandidate: (s) => out.push(s) });
+    agg.observe({ ...base, stackBytes: 100 });
+    agg.observe({ ...base, stackBytes: 5000 });
+    agg.observe({ ...base, stackBytes: 40 });
+    agg.observe({ ...base });                                   // no stack at all
+    assert.deepEqual(out.map((s) => s.stackBytes), [100, 5000, 5000, 5000]);
+    const big = createAggregator({ windowMs: 60_000, flushMs: 0, maxTracked: 10, now: () => 1_000, onCandidate: (s) => out.push(s) });
+    big.observe({ ...base, errorHash: "c".repeat(32), stackBytes: 2 ** 40 });
+    assert.equal(out.at(-1).stackBytes, 16 * 1024 * 1024);
+    for (const bad of [NaN, -5, Infinity, "12", null, undefined]) {
+        big.observe({ ...base, errorHash: "d".repeat(32), stackBytes: bad });
+    }
+    assert.equal(out.at(-1).stackBytes, 0);
+});
+
+function emitterFor({ caps = [], model = "stack-detector-1.8.0", snap = {} } = {}) {
+    const sent = [];
+    const session = { state: "ready", on() {}, off() {}, peerHas: (c) => caps.includes(c), async send(type, payload) { sent.push(payload); } };
+    const config = resolveIncidentDetectorConfig({ LOCAL_AGENT_INCIDENT_DETECTOR: "1", LOCAL_AGENT_INCIDENT_STORE_DIR: path.join(tmp(), "s") }, () => {});
+    const emitter = createCandidateEmitter({ session, config, model, env: {}, log: { warn() {}, info() {}, log() {} } });
+    emitter.submit({ ...base, count: 1, firstSeen: 1_000, lastSeen: 2_000, windowStart: 1_000, stackBytes: 123, ...snap });
+    return { sent, emitter };
+}
+
+test("emitter: toward a 2.0 control plane (no size-model capability) the payload is exactly the 9 fields; with it, + stackBytes and model", async () => {
+    let r = emitterFor();
+    await r.emitter.flush();
+    assert.deepEqual(Object.keys(r.sent[0]).sort(), NINE);
+    r = emitterFor({ caps: ["incident.candidate.size-model"] });
+    await r.emitter.flush();
+    assert.deepEqual(Object.keys(r.sent[0]).sort(), [...NINE, "model", "stackBytes"].sort());
+    assert.equal(r.sent[0].stackBytes, 123);
+    assert.equal(r.sent[0].model, "stack-detector-1.8.0");
+    // a capability that is not the size-model one changes nothing
+    r = emitterFor({ caps: ["something.else"] });
+    await r.emitter.flush();
+    assert.deepEqual(Object.keys(r.sent[0]).sort(), NINE);
+});
+
+test("emitter: a model label that the schema would refuse (space, slash, too long, empty) is not sent; stackBytes is clamped to 0..16 MiB", async () => {
+    for (const model of ["has space", "a/b", "x".repeat(49), "", null, "-lead"]) {
+        const r = emitterFor({ caps: ["incident.candidate.size-model"], model });
+        await r.emitter.flush();
+        assert.ok(!("model" in r.sent[0]), String(model));
+        assert.equal(r.sent[0].stackBytes, 123);
+    }
+    let r = emitterFor({ caps: ["incident.candidate.size-model"], snap: { stackBytes: 2 ** 40 } });
+    await r.emitter.flush();
+    assert.equal(r.sent[0].stackBytes, 16777216);
+    r = emitterFor({ caps: ["incident.candidate.size-model"], snap: { stackBytes: -3 } });
+    await r.emitter.flush();
+    assert.equal(r.sent[0].stackBytes, 0);
+    r = emitterFor({ caps: ["incident.candidate.size-model"], snap: { stackBytes: undefined } });
+    await r.emitter.flush();
+    assert.ok(!("stackBytes" in r.sent[0]), "no number, no field");
+});
+
+async function linkFor(cp, env, root = tmp()) {
+    const logs = [];
+    const log = { info: (m) => logs.push(m), log: (m) => logs.push(m), warn: (m) => logs.push(m), error: (m) => logs.push(m) };
+    const wsManager = { list: () => [{ id: "ws-1", root, tenantId: TENANT }] };
+    const link = await buildV2Link({ tenantId: TENANT, agentId: AGENT, signing: signingFor({ keyId: cp.keyId, publicKeyRaw: cp.publicKeyRaw }, "enforce"), wsManager, getWorkspaceRoot: () => root, log, env });
+    const transport = { send: (t) => cp.feed(t), close() {}, bufferedAmount: () => 0 };
+    cp.attach((t) => queueMicrotask(() => link.session.handleText(t, transport)), () => {});
+    link.session.attach(transport);
+    await waitFor(() => link.session.state === "ready");
+    return { link, logs };
+}
+
+for (const [label, caps] of [["a 2.1 control plane (announces size-model)", ["incident.candidate.size-model"]], ["a 2.0 control plane (announces nothing)", []]]) {
+    test(`end to end with ${label}: the candidate is closed-schema valid; stackBytes/model only with the capability; the text never on the wire`, async () => {
+        const dir = tmp(); const file = path.join(dir, "billing-api.log");
+        fs.writeFileSync(file, "");
+        const cp = new ReferenceCp({ capabilities: caps });
+        const { link } = await linkFor(cp, {
+            LOCAL_AGENT_INCIDENT_DETECTOR: "true", LOCAL_AGENT_INCIDENT_LOG_FILES: file, LOCAL_AGENT_INCIDENT_STORE_DIR: path.join(dir, "store"),
+            LOCAL_AGENT_INCIDENT_FILE_POLL_MS: "50", LOCAL_AGENT_INCIDENT_FLUSH_MS: "300", LOCAL_AGENT_DLP_MODE: "enforce",
+        });
+        try {
+            assert.equal(cp.registers[0].capabilities.includes("contract.v2.1"), true, "the LA announces it can read ready.capabilities");
+            await new Promise((r) => setTimeout(r, 120));
+            const stack = "\tat com.acme.SizeCanary.open(SizeCanary.java:44)\n\tat com.acme.Main.run(Main.java:9)";
+            fs.appendFileSync(file, `2026-11-03 10:00:00 ERROR c.a.Order : java.lang.IllegalStateException: SIZE-CANARY-1 order 1 not found\n${stack}\n2026-11-03 10:00:01 INFO nothing\n`);
+            await waitFor(() => cp.received.some((f) => f.type === "incident.candidate"), 5000);
+            assert.deepEqual(cp.errorsSent, [], "the CP's closed schema accepted every frame");
+            const p = cp.received.find((f) => f.type === "incident.candidate").payload;
+            if (caps.length) {
+                assert.deepEqual(Object.keys(p).sort(), [...NINE, "model", "stackBytes"].sort());
+                assert.equal(p.stackBytes, Buffer.byteLength(stack), "size of the stack as the source gave it");
+                assert.match(p.model, /^stack-detector-\d+\.\d+\.\d+$/);
+            } else {
+                assert.deepEqual(Object.keys(p).sort(), NINE, "identical to before 2.1");
+            }
+            const wire = cp.rawIn.join("\n");
+            for (const w of ["SIZE-CANARY-1", "SizeCanary", "SizeCanary.java"]) assert.ok(!wire.includes(w), w);
+        } finally { link.close(); }
+    });
+}
+
+test("every flag off: register/hello are byte-for-byte the baseline capabilities (no contract.v2.1, no llm.key, no webhook.*, no incident.candidate)", async () => {
+    const cp = new ReferenceCp({ capabilities: ["incident.candidate.size-model"] });
+    const { link } = await linkFor(cp, {});
+    assert.deepEqual([...cp.registers[0].capabilities].sort(), ["chunk.base64", "protocol.v2", "v1.commands"]);
+    assert.deepEqual([...cp.hellos[0].capabilities].sort(), ["chunk.base64", "protocol.v2", "v1.commands"]);
+    assert.deepEqual([...link.session.peerCapabilities], [], "a ready without capabilities (the LA did not announce contract.v2.1)");
+    link.close();
+});

package/test/incident-detector/sources.test.js

@@ -0,0 +1,179 @@
+// F3-11b: the log sources (file tail, Loki, stream) with real files, a fake fetch and real streams.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { PassThrough, Readable } from "node:stream";
+import { createFileSource } from "../../src/incident-detector/sources/file-source.js";
+import { createLokiSource } from "../../src/incident-detector/sources/loki-source.js";
+import { createStreamSource } from "../../src/incident-detector/sources/stream-source.js";
+
+const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), "la-src-"));
+const collect = () => { const got = []; return { got, handler: async (entries) => { got.push(...entries); } }; };
+
+// ------------------------------------------------------------------------------------------ file
+
+test("file: only NEW lines by default; fromStart reads what is there; a record spanning appends is one entry; the file name is the service", async () => {
+    const dir = tmp(); const f = path.join(dir, "billing-api.log");
+    fs.writeFileSync(f, "2026-10-29 10:00:00 ERROR old one\n");
+    const a = collect();
+    const src = createFileSource({ filePath: f, idleFlushMs: 60_000 });
+    src.start(a.handler); await src.poll();
+    assert.equal(a.got.length, 0, "history is not replayed");
+    fs.appendFileSync(f, "2026-10-29 10:00:01 ERROR c.a.Foo : failed 7\njava.lang.IllegalStateException: x\n");
+    await src.poll();
+    fs.appendFileSync(f, "\tat com.x.A.m(A.java:1)\n2026-10-29 10:00:02 INFO ok\n");
+    await src.poll();
+    src.stop();
+    assert.equal(a.got.length, 1, "the INFO line closed the ERROR record; the INFO record is still in progress");
+    assert.equal(a.got[0].service, "billing-api");
+    assert.match(a.got[0].stacktrace, /IllegalStateException[\s\S]*A\.java:1/);
+    const b = collect();
+    const s2 = createFileSource({ filePath: f, fromStart: true, idleFlushMs: 60_000 });
+    s2.start(b.handler); await s2.poll(); s2.stop();
+    assert.equal(b.got.length, 2);
+});
+
+test("file: rotation (new inode) and truncation restart at 0; a missing file is waited for; a partial last line waits for its newline; CRLF and multibyte are intact", async () => {
+    const dir = tmp(); const f = path.join(dir, "x.log");
+    const a = collect();
+    const src = createFileSource({ filePath: f, fromStart: true, idleFlushMs: 0 });
+    src.start(a.handler);
+    await src.poll();                                   // no file yet
+    fs.writeFileSync(f, "2026-10-29 10:00:00 ERROR one é💥\r\n2026-10-29 10:00:01 ERROR par");
+    await src.poll();
+    assert.deepEqual(a.got.map((e) => e.message), ["2026-10-29 10:00:00 ERROR one é💥"], "the partial line is not an entry yet");
+    fs.appendFileSync(f, "tial\n");
+    await src.poll();
+    assert.equal(a.got.at(-1).message, "2026-10-29 10:00:01 ERROR partial");
+    fs.renameSync(f, f + ".1");
+    fs.writeFileSync(f, "2026-10-29 10:00:02 ERROR after rotation\n");
+    await src.poll();
+    assert.equal(a.got.at(-1).message, "2026-10-29 10:00:02 ERROR after rotation");
+    fs.writeFileSync(f, "2026-10-29 10:00:03 ERROR t\n");   // truncate + rewrite (shorter than the offset)
+    await src.poll();
+    assert.equal(a.got.at(-1).message, "2026-10-29 10:00:03 ERROR t");
+    src.stop();
+});
+
+test("file: an over-long line is cut at maxLineBytes (the rest to its newline is discarded and counted); reading is bounded per tick", async () => {
+    const dir = tmp(); const f = path.join(dir, "y.log");
+    fs.writeFileSync(f, "");
+    const a = collect(); const stats = {};
+    const src = createFileSource({ filePath: f, fromStart: true, idleFlushMs: 0, maxLineBytes: 100, maxReadBytes: 64 * 1024, stats });
+    src.start(a.handler); await src.poll();
+    fs.appendFileSync(f, "2026-10-29 10:00:00 ERROR " + "z".repeat(10_000) + "\n2026-10-29 10:00:01 ERROR next\n");
+    await src.poll(); await src.poll();
+    src.stop();
+    assert.equal(stats.longLines, 1);
+    assert.ok(a.got[0].message.length <= 100);
+    assert.equal(a.got.at(-1).message, "2026-10-29 10:00:01 ERROR next");
+    // bounded per tick: 300 KiB are read over several ticks, never more than maxReadBytes (+ one chunk) at once
+    const g = path.join(dir, "big.log");
+    fs.writeFileSync(g, "");
+    const sizes = [];
+    const big = createFileSource({ filePath: g, fromStart: true, idleFlushMs: 0, maxReadBytes: 64 * 1024 });
+    big.start(async (entries) => { sizes.push(entries.length); });
+    await big.poll();
+    fs.appendFileSync(g, "2026-10-29 10:00:00 ERROR line\n".repeat(10_000));
+    await big.poll();
+    const afterOne = sizes.reduce((x, y) => x + y, 0);
+    assert.ok(afterOne > 0 && afterOne < 10_000, `one tick reads a bounded slice (${afterOne})`);
+    big.stop();
+});
+
+// ------------------------------------------------------------------------------------------ Loki
+
+const nsOf = (ms) => (BigInt(ms) * 1_000_000n).toString();
+
+test("loki: query_range with the cursor, bearer + org headers, service from the stream labels, Loki timestamp wins, cursor advances", async () => {
+    const calls = [];
+    const t0 = Date.parse("2026-10-29T10:00:00Z");
+    const fetchImpl = async (url, init) => {
+        calls.push({ url: new URL(url), init });
+        return { ok: true, status: 200, body: Readable.toWeb(Readable.from([Buffer.from(JSON.stringify({ data: { result: [
+            { stream: { service_name: "checkout", level: "error" }, values: [[nsOf(t0 + 5), "boom 2"], [nsOf(t0 + 1), '{"level":"error","msg":"json boom"}']] },
+        ] } }))])) };
+    };
+    const a = collect();
+    const src = createLokiSource({ url: "https://loki.internal:3100", query: '{app="x"}', token: "tok", org: "tenant-1", fetchImpl, now: () => t0 + 1000, lookbackMs: 60_000, limit: 500 });
+    src.start(a.handler); await src.poll();
+    const c = calls[0];
+    assert.equal(c.url.pathname, "/loki/api/v1/query_range");
+    assert.equal(c.url.searchParams.get("query"), '{app="x"}');
+    assert.equal(c.url.searchParams.get("direction"), "forward");
+    assert.equal(c.url.searchParams.get("start"), nsOf(t0 + 1000 - 60_000));
+    assert.equal(c.init.headers.Authorization, "Bearer tok");
+    assert.equal(c.init.headers["X-Scope-OrgID"], "tenant-1");
+    assert.equal(c.init.redirect, "error");
+    assert.deepEqual(a.got.map((e) => [e.message, e.service]), [["json boom", "checkout"], ["boom 2", "checkout"]], "sorted by time");
+    assert.equal(a.got[0].tsNs, BigInt(nsOf(t0 + 1)));
+    await src.poll();
+    src.stop();
+    assert.equal(calls[1].url.searchParams.get("start"), (BigInt(nsOf(t0 + 5)) + 1n).toString(), "cursor = last ns + 1");
+});
+
+test("loki: a full page continues at once (up to maxPages); an error is counted, never thrown, and carries nothing to the log", async () => {
+    let n = 0;
+    const t0 = Date.parse("2026-10-29T10:00:00Z");
+    const fetchImpl = async () => {
+        n++;
+        const values = [[nsOf(t0 + n * 10), `ERROR e${n}`], [nsOf(t0 + n * 10 + 1), `ERROR f${n}`]];
+        return { ok: true, status: 200, body: Readable.toWeb(Readable.from([Buffer.from(JSON.stringify({ data: { result: [{ stream: { app: "a" }, values }] } }))])) };
+    };
+    const a = collect(); const stats = {};
+    const src = createLokiSource({ url: "http://l", query: "q", fetchImpl, limit: 2, maxPages: 3, now: () => t0, stats });
+    src.start(a.handler); await src.poll(); src.stop();
+    assert.equal(n, 3, "maxPages");
+    assert.equal(a.got.length, 6);
+    const bad = createLokiSource({ url: "http://l", query: "q", fetchImpl: async () => { throw new Error("connect ECONNREFUSED http://secret-host:3100"); }, stats: {} });
+    bad.start(async () => {});
+    await new Promise((r) => setTimeout(r, 30));
+    assert.equal(bad.stats.errors, 1);
+    bad.stop();
+});
+
+test("loki: a response over maxResponseBytes, a non-200 and malformed rows are survived", async () => {
+    const mk = (body, status = 200) => async () => ({ ok: status === 200, status, body: Readable.toWeb(Readable.from([Buffer.from(body)])) });
+    const a = collect();
+    const big = createLokiSource({ url: "http://l", query: "q", fetchImpl: mk("x".repeat(5000)), maxResponseBytes: 1000 });
+    big.start(a.handler);
+    await assert.rejects(big.poll(), /response_too_large/);
+    big.stop();
+    const bad = createLokiSource({ url: "http://l", query: "q", fetchImpl: mk("{}", 500) });
+    bad.start(a.handler);
+    await assert.rejects(bad.poll(), /http_500/);
+    bad.stop();
+    const odd = createLokiSource({ url: "http://l", query: "q", fetchImpl: mk(JSON.stringify({ data: { result: [{ stream: null, values: [["x", "y"], [1, 2], ["5", "ERROR ok"], "z"] }, null] } })) });
+    odd.start(a.handler); await odd.poll(); odd.stop();
+    assert.deepEqual(a.got.map((e) => e.message), ["ERROR ok"]);
+});
+
+// ------------------------------------------------------------------------------------------ stream
+
+test("stream: lines become entries, the service label is the default, the record in progress is flushed at the end", async () => {
+    const input = new PassThrough();
+    const a = collect();
+    const src = createStreamSource({ stream: input, service: "container-1", idleFlushMs: 20 });
+    src.start(a.handler);
+    input.write("2026-10-29 10:00:00 ERROR boom\n\tat a.B.c(B.java:1)\n");
+    await new Promise((r) => setTimeout(r, 80));
+    assert.equal(a.got.length, 1, "idle flush");
+    input.end("2026-10-29 10:00:01 ERROR last");
+    await new Promise((r) => setTimeout(r, 50));
+    assert.deepEqual(a.got.map((e) => e.service), ["container-1", "container-1"]);
+    src.stop();
+});
+
+test("stream: backpressure, a slow consumer pauses the stream instead of buffering the world", async () => {
+    const input = new PassThrough({ highWaterMark: 1024 });
+    let consumed = 0;
+    const src = createStreamSource({ stream: input, idleFlushMs: 5 });
+    src.start(async (entries) => { consumed += entries.length; await new Promise((r) => setTimeout(r, 30)); });
+    let written = 0; let backpressured = false;
+    for (let i = 0; i < 2000; i++) { const ok = input.write("2026-10-29 10:00:00 ERROR line " + i + "\n"); written++; if (!ok) { backpressured = true; break; } }
+    assert.equal(backpressured, true, "the writer is told to stop long before 2000 lines");
+    assert.ok(written < 2000);
+    src.stop(); input.destroy();
+});