deepdebug-local-agent 1.7.0 → 1.8.0

package/src/incident-detector/normalize.js

@@ -0,0 +1,209 @@
+/**
+ * normalize.js - the Java detection rules, ported (F3-11b). The Java code is the source of truth; this file must give
+ * byte-identical results (parity.vectors.json is generated from it by scripts/incident-parity/gen.sh):
+ *
+ *   isSignificant      monitor-service ErrorDetector.isSignificant           (severity gate + noise + stack/HTTP patterns)
+ *   groupHash/Key      monitor-service MonitorPollScheduler.computeGroupHash  (sha256 of the normalised FIRST LINE, first 16 bytes = 32 hex)
+ *                      = the candidate's `errorHash` (it is log_entries.error_hash and incident_dedup_log.error_hash)
+ *   fingerprint        shared IncidentFingerprintGenerator.compute(extractErrorType, extractFirstUserFrame, message)  (sha256, 64 hex)
+ *   classifyNoise      shared LogNoiseClassifier.classify  (infra/build noise is persisted by Java but never promoted)
+ *   naturalKey         monitor-service LogEntryDedup.key  (service|epoch micros|md5(message)): a re-read of the same entry
+ *
+ * NOT ported: ErrorDetector.hash (the legacy `service|message` hash). Java only uses it when computeGroupHash throws, which it
+ * cannot do on a String (parity vectors still carry it as `legacyHash`).
+ */
+import { SP, SP_CLASS, NOT_SP_CLASS, javaTrim, javaIsBlank, javaStripLeading, sha256Hex, sha256Prefix16, md5Hex } from "./java-compat.js";
+
+// ---------------------------------------------------------------------------------------------- group hash (errorHash)
+
+const GROUP_URL = new RegExp(`https?://${NOT_SP_CLASS}+`, "g");
+const GROUP_UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;
+const GROUP_ADDR = /0x[0-9a-f]+/g;
+const GROUP_HEX = /\b[0-9a-f]{12,}\b/g;
+const GROUP_VER = /\d+(\.\d+)+/g;
+const GROUP_DIGITS = /\d+/g;
+const GROUP_WS = new RegExp(`${SP_CLASS}+`, "g");
+
+/** Order matters (as in Java): URL first (may contain uuid/digits), then uuid, 0x-addr, long hex, versions, digits. */
+export function normalizeGroupKey(firstLine) {
+    return javaTrim(String(firstLine).toLowerCase()
+        .replace(GROUP_URL, "<url>")
+        .replace(GROUP_UUID, "<uuid>")
+        .replace(GROUP_ADDR, "<addr>")
+        .replace(GROUP_HEX, "<hex>")
+        .replace(GROUP_VER, "<ver>")
+        .replace(GROUP_DIGITS, "<n>")
+        .replace(GROUP_WS, " "));
+}
+
+/** errorHash: 32 hex. Only the FIRST line of the message counts (the stack trace never does). */
+export function computeGroupHash(message) {
+    const msg = message == null ? "" : String(message);
+    const nl = msg.indexOf("\n");
+    return sha256Prefix16(normalizeGroupKey(nl >= 0 ? msg.slice(0, nl) : msg));
+}
+
+// ---------------------------------------------------------------------------------------------- fingerprint
+
+const PG_LOG_PREAMBLE = new RegExp(
+    `^${SP_CLASS}*\\d{4}-\\d{2}-\\d{2}${SP_CLASS}+\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?${SP_CLASS}+\\w+${SP_CLASS}+\\[\\d+\\]:${SP_CLASS}+\\[[^\\]]+\\]${SP_CLASS}+(?:db=[^,]+,user=${NOT_SP_CLASS}+${SP_CLASS}+)?(?:ERROR|WARNING|FATAL|LOG|NOTICE|INFO|DEBUG):?${SP_CLASS}*`,
+    "i");
+const ISO_TIMESTAMP = new RegExp(`\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:${SP_CLASS}+[A-Z]{2,5}|[+-]\\d{2}:?\\d{2}|Z)?`, "g");
+const UUID_PAT = /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/g;
+const HEX_ADDR = /(?:0x|@)[0-9a-fA-F]{4,}/g;
+const BRACKET_PID = /\[\d{2,}\]/g;
+const LINE_NUM_TAIL = /(\.java|\.kt|\.swift|\.py|\.js|\.ts|\.rb|\.go):\d+/g;
+const LONG_INT = /(?<![\w-])\d{5,}(?![\w-])/g;
+const WS = new RegExp(`${SP_CLASS}+`, "g");
+
+export function normaliseMessage(raw) {
+    if (raw == null) return null;
+    let s = String(raw);
+    s = s.replace(PG_LOG_PREAMBLE, "");   // anchored with ^ and no /g: at most one match, as in Java
+    s = s.replace(ISO_TIMESTAMP, "<ts>");
+    s = s.replace(UUID_PAT, "<uuid>");
+    s = s.replace(HEX_ADDR, "<addr>");
+    s = s.replace(BRACKET_PID, "[<pid>]");
+    s = s.replace(LONG_INT, "<n>");
+    s = javaTrim(s.replace(WS, " "));
+    return s.length > 500 ? s.slice(0, 500) : s;
+}
+
+export function normaliseFrame(raw) {
+    if (raw == null) return "";
+    let s = javaTrim(String(raw));
+    if (s.startsWith("at ")) s = s.slice(3);
+    s = s.replace(LINE_NUM_TAIL, "$1:<line>");
+    s = s.replace(HEX_ADDR, "<addr>");
+    return s.length > 200 ? s.slice(0, 200) : s;
+}
+
+const LINE_SPLIT = /\r?\n/;
+
+/** IncidentFingerprintGenerator.extractErrorType (quirks included: "Caused by: X: y" gives "Caused by"). */
+export function extractErrorType(message, stacktrace) {
+    if (stacktrace != null && !javaIsBlank(stacktrace)) {
+        const m = LINE_SPLIT.exec(stacktrace);
+        const first = javaTrim(m ? stacktrace.slice(0, m.index) : stacktrace);
+        const colon = first.indexOf(":");
+        const space = first.indexOf(" ");
+        if ((colon > 0 && space < 0) || (colon > 0 && colon < space)) return javaTrim(first.slice(0, colon));
+        if (first.startsWith("Caused by:")) {
+            const rest = javaTrim(first.slice("Caused by:".length));
+            const colon2 = rest.indexOf(":");
+            return colon2 > 0 ? javaTrim(rest.slice(0, colon2)) : rest;
+        }
+    }
+    if (message != null) {
+        const m = String(message);
+        if (PG_LOG_PREAMBLE.test(m)) {
+            if (m.includes("ERROR:")) return "PG_ERROR";
+            if (m.includes("FATAL:")) return "PG_FATAL";
+            if (m.includes("WARNING:")) return "PG_WARNING";
+        }
+        const colon = m.indexOf(":");
+        const space = m.indexOf(" ");
+        if (colon > 0 && (space < 0 || colon < space) && m.slice(0, colon).includes(".")) return javaTrim(m.slice(0, colon));
+    }
+    return null;
+}
+
+const FRAMEWORK_SKIP = ["at org.springframework.", "at reactor.", "at io.netty.", "at io.r2dbc.", "at org.apache.", "at com.fasterxml.", "at java.", "at javax.", "at jakarta."];
+const FRAMEWORK_CONTAINS = ["java.lang.reflect", "sun.reflect", "jdk.internal", "$Proxy"];
+const cut200 = (l) => (l.length > 200 ? l.slice(0, 200) : l);
+
+export function extractFirstUserFrame(stacktrace) {
+    if (stacktrace == null) return "";
+    const lines = String(stacktrace).split(LINE_SPLIT).map(javaTrim);
+    for (const line of lines) {
+        if (!line.startsWith("at ")) continue;
+        if (FRAMEWORK_CONTAINS.some((c) => line.includes(c)) || FRAMEWORK_SKIP.some((p) => line.startsWith(p))) continue;
+        return cut200(line);
+    }
+    for (const line of lines) if (line.startsWith("at ")) return cut200(line);
+    return "";
+}
+
+/** IncidentFingerprintGenerator.compute: sha256 (64 hex) or null when the normalised message is empty/blank. */
+export function computeFingerprintOf(errorType, firstFrame, message) {
+    const nm = normaliseMessage(message);
+    if (nm == null || javaIsBlank(nm)) return null;
+    return sha256Hex(`${javaTrim(errorType == null ? "" : String(errorType))}|${normaliseFrame(firstFrame)}|${nm}`);
+}
+
+/** What the gateway/orchestrator do with a (message, stack) pair. */
+export function computeFingerprint(message, stacktrace) {
+    return computeFingerprintOf(extractErrorType(message, stacktrace), extractFirstUserFrame(stacktrace), message);
+}
+
+// ---------------------------------------------------------------------------------------------- significance and noise
+
+const ERROR_LEVELS = new Set(["ERROR", "CRITICAL", "FATAL", "EMERG", "ALERT", "SEVERE"]);
+const NOISE_PATTERNS = [
+    /healthcheck/i, /\/actuator\/health/i, /broken pipe/i, /connection reset by peer/i, /client disconnected/i, /favicon\.ico/i,
+    new RegExp(`/ping${SP_CLASS}`, "i"),
+];
+const STACKTRACE_PATTERNS = [
+    /\bat [\w$.]+\([\w.]+:\d+\)/,
+    /[\w]+\.swift:\d+/,
+    /\(\/[^)]*\.(?:js|ts|mjs|cjs):\d+:\d+\)/,
+    /File "[^"]+\.py", line \d+/,
+    new RegExp(`file:///[^${SP})]*\\.(?:js|ts|mjs|cjs|jsx|tsx):\\d+(?::\\d+)?`),
+    /Exception|Error: /i,
+];
+const HTTP_ERROR_PATTERN = new RegExp(`\\bHTTP${SP_CLASS}+([45]\\d\\d)\\b`, "i");
+const safeUpper = (s) => (s == null ? "" : String(s).toUpperCase());
+
+/** ErrorDetector.isSignificant: `entry` = {severity, message, stacktrace}. */
+export function isSignificant(entry) {
+    if (!entry || entry.message == null) return false;
+    const message = String(entry.message);
+    if (!ERROR_LEVELS.has(safeUpper(entry.severity))) return false;
+    for (const p of NOISE_PATTERNS) if (p.test(message)) return false;
+    const stack = entry.stacktrace == null ? "" : String(entry.stacktrace);
+    for (const p of STACKTRACE_PATTERNS) if (p.test(message) || p.test(stack)) return true;
+    if (HTTP_ERROR_PATTERN.test(message)) return true;
+    const sev = safeUpper(entry.severity);
+    return sev === "FATAL" || sev === "CRITICAL";
+}
+
+const PLATFORM_PROBE = /\b(STARTUP|LIVENESS|READINESS) (TCP|HTTP|GRPC) probe failed/;
+
+/** LogNoiseClassifier.classify: "app_error" | "infra" | "build". */
+export function classifyNoise(message) {
+    if (message == null || javaIsBlank(String(message))) return "app_error";
+    const m = javaStripLeading(String(message));
+    if (m.startsWith("{code=")) return "infra";
+    if (m.includes("Ready condition status changed")) return "infra";
+    if (m.startsWith("Memory limit of")) return "infra";
+    if (m.startsWith("Uncaught signal")) return "infra";
+    if (m.startsWith("The request failed because either the HTTP response was malformed")) return "infra";
+    if (PLATFORM_PROBE.test(m)) return "infra";
+    if (m.includes("AUDIT EVENT: method=")) return "infra";
+    if (m.startsWith("{@type=type.googleapis.com/google.cloud.audit.AuditLog")) return "infra";
+    if (m.includes("MongoSocketWriteException") || m.includes("MongoSocketReadException")) return "infra";
+    if (m.includes("LifecycleExecutionException")) return "build";
+    if (m.includes("Failed to execute goal")) return "build";
+    if (m.includes("Execution failed for task")) return "build";
+    if (m.startsWith("FAILURE: Build failed with an exception")) return "build";
+    return "app_error";
+}
+
+export const isNoiseCategory = (c) => c === "infra" || c === "build";
+
+// ---------------------------------------------------------------------------------------------- natural key (re-read of the same entry)
+
+export const MAX_MESSAGE_CHARS = 8000;
+
+/** Epoch microseconds, truncated (floor) like Instant.truncatedTo(MICROS). `nanos` is a BigInt of epoch nanoseconds. */
+export const epochMicros = (nanos) => {
+    const q = nanos / 1000n;
+    return nanos % 1000n < 0n ? q - 1n : q;
+};
+
+/** LogEntryDedup.key: null when the entry has no timestamp. `epochNanos` is a BigInt or null. */
+export function naturalKey(service, epochNanos, message) {
+    if (epochNanos == null) return null;
+    const stored = message == null ? "" : String(message).length <= MAX_MESSAGE_CHARS ? String(message) : String(message).slice(0, MAX_MESSAGE_CHARS);
+    return `${service == null ? "unknown" : service}|${epochMicros(epochNanos)}|${md5Hex(stored)}`;
+}

package/src/incident-detector/parse.js

@@ -0,0 +1,114 @@
+/**
+ * parse.js - from log lines to entries {severity, message, stacktrace, service, tsMs} (F3-11b).
+ *
+ *   JSON line    {"level":"error","message":"...","stack_trace":"..."} (and the usual aliases: msg/log/textPayload, severity/lvl,
+ *                service/service_name/app, time/timestamp/@timestamp/ts, stack/stacktrace/exception.stacktrace/error.stack)
+ *   plain text   a record starts at a line with a timestamp or a level token; the lines that follow (indented, "Caused by:",
+ *                "... n more", python Traceback/File lines, or any line that does not look like a new record) are its stack trace.
+ *
+ * message = the header line (for Java parity only its first line enters errorHash; the fingerprint also sees the stack's first
+ * line, as the Java code does). Entries are capped (maxEntryBytes, maxStackLines): the excess is dropped and counted.
+ */
+const LEVEL = /\b(FATAL|CRITICAL|EMERG(?:ENCY)?|ALERT|SEVERE|ERROR|ERR|WARN(?:ING)?|NOTICE|INFO|DEBUG|TRACE)\b/i;
+const RECORD_START = /^\s*(?:\[?\d{4}-\d{2}-\d{2}|\[?\d{2}:\d{2}:\d{2}|[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}|\[?(?:FATAL|CRITICAL|SEVERE|ERROR|ERR|WARN|WARNING|INFO|DEBUG|TRACE)\b)/;
+const ISO_PREFIX = /^\s*\[?(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d{1,9})?(?:Z|[+-]\d{2}:?\d{2})?)/;
+
+const pick = (o, keys) => { for (const k of keys) { const v = o[k]; if (v !== undefined && v !== null && v !== "") return v; } return undefined; };
+const str = (v) => (v == null ? undefined : typeof v === "string" ? v : typeof v === "number" || typeof v === "boolean" ? String(v) : undefined);
+
+/** @returns {number|null} epoch ms */
+export function parseTime(v) {
+    if (v == null) return null;
+    if (typeof v === "number") {
+        const ms = v > 1e17 ? v / 1e6 : v > 1e14 ? v / 1e3 : v > 1e11 ? v : v * 1000;   // ns, us, ms, s
+        return Number.isFinite(ms) && Math.abs(ms) <= 8.64e15 ? Math.round(ms) : null;   // outside the Date range: not a timestamp
+    }
+    if (typeof v !== "string") return null;
+    const m = ISO_PREFIX.exec(v) || null;
+    const iso = m ? m[1].replace(" ", "T").replace(",", ".") : v;
+    const t = Date.parse(/(?:Z|[+-]\d{2}:?\d{2})$/.test(iso) ? iso : `${iso}Z`);
+    return Number.isFinite(t) ? t : null;
+}
+
+const levelOf = (message) => { const m = LEVEL.exec(message.slice(0, 300)); return m ? m[1] : null; };
+
+function splitMessage(text) {
+    const i = text.indexOf("\n");
+    return i < 0 ? { message: text, stack: null } : { message: text.slice(0, i).replace(/\r$/, ""), stack: text.slice(i + 1) || null };
+}
+
+/** One complete line or block (Loki entries, JSON lines). @returns {object|null} */
+export function parseRecord(text) {
+    if (typeof text !== "string" || text === "") return null;
+    const t = text.trimStart();
+    if (t.startsWith("{")) {
+        let o = null;
+        try { o = JSON.parse(t); } catch { o = null; }
+        if (o && typeof o === "object" && !Array.isArray(o)) {
+            const msgRaw = str(pick(o, ["message", "msg", "log", "textPayload", "text", "event"]))
+                ?? str(o.jsonPayload && typeof o.jsonPayload === "object" ? o.jsonPayload.message : undefined);
+            if (msgRaw === undefined) return null;
+            const err = o.error && typeof o.error === "object" ? o.error : {};
+            const exc = o.exception && typeof o.exception === "object" ? o.exception : {};
+            const stack = str(pick(o, ["stack_trace", "stacktrace", "stackTrace", "stack"])) ?? str(pick(exc, ["stacktrace", "stack_trace"])) ?? str(pick(err, ["stack", "stack_trace"]));
+            const { message, stack: rest } = splitMessage(msgRaw);
+            return {
+                severity: str(pick(o, ["level", "severity", "lvl", "levelname"])) ?? levelOf(message),
+                message: stack ? msgRaw : message,   // a separate stack field: the message is the whole text (its first line is what errorHash uses)
+                stacktrace: stack ?? rest,
+                service: str(pick(o, ["service", "service_name", "serviceName", "app", "application", "container_name"])) ?? null,
+                tsMs: parseTime(pick(o, ["timestamp", "time", "@timestamp", "ts", "t"])),
+            };
+        }
+    }
+    const { message, stack } = splitMessage(text);
+    const tm = ISO_PREFIX.exec(message);
+    return { severity: levelOf(message), message, stacktrace: stack, service: null, tsMs: tm ? parseTime(tm[1]) : null };
+}
+
+/**
+ * Joins the lines of a plain-text file/stream into entries. push(line) returns the entries completed by that line; flush() returns
+ * the one in progress (call it on a short idle timer and at the end). JSON lines are entries by themselves.
+ */
+export class EntryAssembler {
+    constructor({ maxEntryBytes = 64 * 1024, maxStackLines = 200 } = {}) {
+        this.maxEntryBytes = maxEntryBytes;
+        this.maxStackLines = maxStackLines;
+        this.cur = null;
+        this.truncated = 0;
+    }
+    #finish() {
+        const c = this.cur;
+        this.cur = null;
+        if (!c) return null;
+        const entry = parseRecord(c.lines.join("\n"));
+        if (entry && c.truncated) this.truncated++;
+        return entry;
+    }
+    push(line) {
+        const out = [];
+        if (line === "") return out;
+        const isJson = line.trimStart().startsWith("{");
+        // anything that does not look like the start of a record belongs to the one in progress (its message continuation or its stack)
+        if (this.cur && !isJson && !RECORD_START.test(line)) {
+            const c = this.cur;
+            if (c.lines.length - 1 >= this.maxStackLines || c.bytes + line.length > this.maxEntryBytes) { c.truncated = true; return out; }
+            c.lines.push(line);
+            c.bytes += line.length + 1;
+            return out;
+        }
+        const done = this.#finish();
+        if (done) out.push(done);
+        if (isJson) {
+            const e = parseRecord(line);
+            if (e) out.push(e);
+        } else {
+            this.cur = { lines: [line.length > this.maxEntryBytes ? line.slice(0, this.maxEntryBytes) : line], bytes: Math.min(line.length, this.maxEntryBytes), truncated: line.length > this.maxEntryBytes };
+        }
+        return out;
+    }
+    flush() {
+        const e = this.#finish();
+        return e ? [e] : [];
+    }
+}

package/src/incident-detector/sources/file-source.js

@@ -0,0 +1,113 @@
+/**
+ * file-source.js - tails log files (F3-11b).
+ *
+ *   - by default only NEW lines are read (the agent starting must not turn a week of old errors into candidates);
+ *     fromStart=true reads what is already there, once
+ *   - rotation (new inode) and truncation (size < offset) restart at 0; a file that does not exist yet is waited for
+ *   - pull based: at most `maxReadBytes` per file per tick and nothing is read while the previous batch is being processed
+ *     (backpressure: a slow consumer slows the reading down, it never grows a queue)
+ *   - one line is at most `maxLineBytes`: the rest of a longer line is discarded up to its newline (and counted)
+ *   - the file name is used as the service label unless `service` is given; the PATH is never logged or sent
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { StringDecoder } from "node:string_decoder";
+import { EntryAssembler } from "../parse.js";
+
+const READ_CHUNK = 64 * 1024;
+
+export function createFileSource({ filePath, service = null, fromStart = false, pollMs = 1000, idleFlushMs = 300, maxReadBytes = 256 * 1024, maxLineBytes = 16 * 1024,
+    maxEntryBytes, maxStackLines, timers = { setTimeout, clearTimeout }, stats = {} }) {
+    const label = service || path.basename(filePath).replace(/\.(log|txt|out)$/i, "") || "unknown";
+    let timer = null;
+    let stopped = false;
+    let onEntries = null;
+    let inode = null;
+    let offset = 0;
+    let first = true;
+    let partial = "";
+    let discarding = false;     // inside a line that exceeded maxLineBytes
+    let decoder = new StringDecoder("utf8");
+    let lastActivity = 0;
+    const asm = new EntryAssembler({ maxEntryBytes, maxStackLines });
+    stats.longLines = stats.longLines || 0;
+    stats.readErrors = stats.readErrors || 0;
+    const unref = (t) => { if (t && typeof t.unref === "function") t.unref(); return t; };
+
+    const emit = async (entries) => {
+        if (entries.length === 0) return;
+        for (const e of entries) if (!e.service) e.service = label;
+        await onEntries(entries);
+    };
+
+    function takeLines(text) {
+        const lines = [];
+        let buf = partial + text;
+        let i;
+        while ((i = buf.indexOf("\n")) >= 0) {
+            let line = buf.slice(0, i);
+            buf = buf.slice(i + 1);
+            if (discarding) { discarding = false; continue; }   // the rest of an over-long line
+            if (line.endsWith("\r")) line = line.slice(0, -1);
+            if (line.length > maxLineBytes) { stats.longLines++; line = line.slice(0, maxLineBytes); }
+            lines.push(line);
+        }
+        if (buf.length > maxLineBytes) {
+            if (!discarding) { stats.longLines++; lines.push(buf.slice(0, maxLineBytes)); }
+            discarding = true;
+            buf = "";
+        }
+        partial = buf;
+        return lines;
+    }
+
+    async function tick() {
+        timer = null;
+        if (stopped) return;
+        let delay = pollMs;
+        try {
+            const fh = await fs.promises.open(filePath, "r").catch(() => null);
+            if (fh) {
+                try {
+                    const st = await fh.stat();
+                    if (st.isFile()) {
+                        if (first) { first = false; inode = st.ino; offset = fromStart ? 0 : st.size; }
+                        else if (st.ino !== inode || st.size < offset) { inode = st.ino; offset = 0; partial = ""; discarding = false; decoder = new StringDecoder("utf8"); }
+                        let readNow = 0;
+                        const buf = Buffer.allocUnsafe(READ_CHUNK);
+                        while (!stopped && offset < st.size && readNow < maxReadBytes) {
+                            const { bytesRead } = await fh.read(buf, 0, Math.min(READ_CHUNK, st.size - offset), offset);
+                            if (bytesRead === 0) break;
+                            offset += bytesRead;
+                            readNow += bytesRead;
+                            const lines = takeLines(decoder.write(buf.subarray(0, bytesRead)));
+                            const entries = [];
+                            for (const l of lines) entries.push(...asm.push(l));
+                            await emit(entries);
+                            lastActivity = Date.now();
+                        }
+                        if (offset < st.size) delay = 0;   // more to read: come back at once (after yielding)
+                    }
+                } finally { await fh.close().catch(() => {}); }
+            }
+            // a record in progress with nothing after it: the next line may never come
+            if (asm.cur && Date.now() - lastActivity >= idleFlushMs) await emit(asm.flush());
+        } catch {
+            stats.readErrors++;   // the reason can carry a path: only counted
+        }
+        if (!stopped) timer = unref(timers.setTimeout(tick, asm.cur ? Math.min(delay, idleFlushMs) : delay));
+    }
+
+    return {
+        name: "file",
+        stats,
+        start(handler) {
+            onEntries = handler;
+            stopped = false;
+            timer = unref(timers.setTimeout(tick, 0));
+        },
+        /** Reads once, now (tests, shutdown). */
+        async poll() { if (timer) { timers.clearTimeout(timer); timer = null; } await tick(); },
+        stop() { stopped = true; if (timer) timers.clearTimeout(timer); timer = null; },
+    };
+}

package/src/incident-detector/sources/loki-source.js

@@ -0,0 +1,101 @@
+/**
+ * loki-source.js - polls a Loki (the customer's own) with a LogQL query (F3-11b).
+ *
+ *   GET {url}/loki/api/v1/query_range?query=..&start=<ns>&end=<ns>&limit=<n>&direction=forward
+ *   - cursor = the timestamp of the last entry seen (+1 ns); a poll that returns a full page continues at once, up to maxPages
+ *   - the response body is read with a byte limit (maxResponseBytes) and a timeout; a failure is counted and retried next poll
+ *   - the credentials (token, org id) go in headers only; the URL, the query and the response are never logged
+ *   - the service label comes from the stream labels (service_name, service, app, job, container); the labels are not sent anywhere
+ * Re-reads (the cursor overlaps on purpose by 0; an instance restart looks back lookbackMs) are absorbed by the detector's natural-key dedup.
+ */
+import { parseRecord } from "../parse.js";
+
+const SERVICE_LABELS = ["service_name", "service", "app", "application", "container", "container_name", "job"];
+
+async function readLimited(res, maxBytes) {
+    const reader = res.body.getReader();
+    const chunks = [];
+    let total = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        total += value.byteLength;
+        if (total > maxBytes) { try { await reader.cancel(); } catch { /* ignore */ } throw new Error("response_too_large"); }
+        chunks.push(value);
+    }
+    return Buffer.concat(chunks.map((c) => Buffer.from(c.buffer, c.byteOffset, c.byteLength))).toString("utf8");
+}
+
+export function createLokiSource({ url, query, token = null, org = null, pollMs = 15_000, limit = 500, maxPages = 5, timeoutMs = 10_000, maxResponseBytes = 4 * 1024 * 1024,
+    lookbackMs = 60_000, service = null, fetchImpl = globalThis.fetch, now = Date.now, timers = { setTimeout, clearTimeout }, stats = {} }) {
+    let timer = null;
+    let stopped = false;
+    let onEntries = null;
+    let cursorNs = null;   // BigInt
+    stats.polls = stats.polls || 0;
+    stats.errors = stats.errors || 0;
+    stats.pages = stats.pages || 0;
+    const unref = (t) => { if (t && typeof t.unref === "function") t.unref(); return t; };
+
+    async function page(startNs, endNs) {
+        const q = new URLSearchParams({ query, start: startNs.toString(), end: endNs.toString(), limit: String(limit), direction: "forward" });
+        const headers = { Accept: "application/json" };
+        if (token) headers.Authorization = `Bearer ${token}`;
+        if (org) headers["X-Scope-OrgID"] = org;
+        const ctl = new AbortController();
+        const to = setTimeout(() => ctl.abort(), timeoutMs);
+        try {
+            const res = await fetchImpl(`${url}/loki/api/v1/query_range?${q}`, { headers, signal: ctl.signal, redirect: "error" });
+            if (!res.ok) throw new Error(`http_${res.status}`);
+            const body = JSON.parse(await readLimited(res, maxResponseBytes));
+            return Array.isArray(body?.data?.result) ? body.data.result : [];
+        } finally { clearTimeout(to); }
+    }
+
+    async function pollOnce() {
+        stats.polls++;
+        const endNs = BigInt(now()) * 1_000_000n;
+        if (cursorNs == null) cursorNs = endNs - BigInt(lookbackMs) * 1_000_000n;
+        for (let p = 0; p < maxPages && !stopped; p++) {
+            const result = await page(cursorNs, endNs);
+            stats.pages++;
+            const rows = [];
+            for (const s of result) {
+                const labels = s && typeof s.stream === "object" && s.stream ? s.stream : {};
+                const svc = service || SERVICE_LABELS.map((k) => labels[k]).find((v) => typeof v === "string" && v) || "unknown";
+                for (const v of Array.isArray(s?.values) ? s.values : []) {
+                    if (!Array.isArray(v) || typeof v[0] !== "string" || typeof v[1] !== "string" || !/^\d{1,20}$/.test(v[0])) continue;
+                    rows.push({ ns: BigInt(v[0]), line: v[1], svc });
+                }
+            }
+            rows.sort((a, b) => (a.ns < b.ns ? -1 : a.ns > b.ns ? 1 : 0));
+            const entries = [];
+            for (const r of rows) {
+                const e = parseRecord(r.line);
+                if (!e) continue;
+                e.service = e.service || r.svc;
+                e.tsNs = r.ns;                       // the Loki timestamp wins: it is what the natural key is made of
+                e.tsMs = Number(r.ns / 1_000_000n);
+                entries.push(e);
+            }
+            if (entries.length) await onEntries(entries);
+            if (rows.length) cursorNs = rows[rows.length - 1].ns + 1n;
+            if (rows.length < limit) break;
+        }
+    }
+
+    async function tick() {
+        timer = null;
+        if (stopped) return;
+        try { await pollOnce(); } catch { stats.errors++; }   // the message of a fetch error can carry the URL: only counted
+        if (!stopped) timer = unref(timers.setTimeout(tick, pollMs));
+    }
+
+    return {
+        name: "loki",
+        stats,
+        start(handler) { onEntries = handler; stopped = false; timer = unref(timers.setTimeout(tick, 0)); },
+        async poll() { if (timer) { timers.clearTimeout(timer); timer = null; } await pollOnce(); },
+        stop() { stopped = true; if (timer) timers.clearTimeout(timer); timer = null; },
+    };
+}

package/src/incident-detector/sources/stream-source.js

@@ -0,0 +1,39 @@
+/**
+ * stream-source.js - reads log lines from any Readable: the agent's stdin (a container or process piped into it) or a test
+ * stream (F3-11b). Pull based: the loop awaits the consumer before taking the next line, so a slow consumer pauses the stream
+ * (Node stops reading from the pipe: backpressure reaches the writer instead of growing memory).
+ */
+import readline from "node:readline";
+import { EntryAssembler } from "../parse.js";
+
+export function createStreamSource({ stream, service = "stdin", idleFlushMs = 300, maxLineBytes = 16 * 1024, maxEntryBytes, maxStackLines, stats = {} }) {
+    let rl = null;
+    let stopped = false;
+    stats.longLines = stats.longLines || 0;
+    return {
+        name: "stream",
+        stats,
+        start(onEntries) {
+            const asm = new EntryAssembler({ maxEntryBytes, maxStackLines });
+            rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+            const label = (entries) => { for (const e of entries) if (!e.service) e.service = service; return entries; };
+            (async () => {
+                try {
+                    let idle = null;
+                    for await (let line of rl) {
+                        if (stopped) break;
+                        if (idle) { clearTimeout(idle); idle = null; }
+                        if (line.length > maxLineBytes) { stats.longLines++; line = line.slice(0, maxLineBytes); }
+                        const out = label(asm.push(line));
+                        if (out.length) await onEntries(out);
+                        if (asm.cur) { idle = setTimeout(() => { const f = label(asm.flush()); if (f.length) void onEntries(f); }, idleFlushMs); if (idle.unref) idle.unref(); }
+                    }
+                    if (idle) clearTimeout(idle);
+                    const rest = label(asm.flush());
+                    if (rest.length) await onEntries(rest);
+                } catch { stats.readErrors = (stats.readErrors || 0) + 1; }
+            })();
+        },
+        stop() { stopped = true; try { rl?.close(); } catch { /* closed */ } },
+    };
+}