deepdebug-local-agent 1.7.0 → 1.9.0

package/src/incident-detector/categorize.js

@@ -0,0 +1,48 @@
+/**
+ * categorize.js - the small mappings between a log entry and the metadata of `incident.candidate` (F3-11b).
+ * Nothing here returns text from the entry: only enum values and sanitised labels.
+ */
+
+/** Gateway SeverityNormalizer for the levels ErrorDetector lets through (SEVERE is not in the gateway table; it counts as an error). */
+export function severityOf(level) {
+    switch (String(level ?? "").trim().toUpperCase()) {
+        case "CRITICAL": case "ALERT": case "EMERG": case "EMERGENCY": case "FATAL": return "critical";
+        case "ERROR": case "ERR": case "SEVERE": return "high";
+        default: return "medium";
+    }
+}
+const RANK = { info: 0, low: 1, medium: 2, high: 3, critical: 4 };
+export const maxSeverity = (a, b) => (RANK[b] > RANK[a] ? b : a);
+
+/** AutoIncidentCreator.inferEnvironment: the platform convention for the service name suffix. */
+export function inferEnvironment(service) {
+    if (service == null || String(service).trim() === "") return "production";
+    const s = String(service).toLowerCase();
+    if (s.endsWith("-qa") || s.includes("-qa-")) return "qa";
+    if (s.endsWith("-demo") || s.includes("-demo-")) return "demo";
+    if (s.endsWith("-staging") || s.includes("-staging-") || s.endsWith("-stg")) return "staging";
+    return "production";
+}
+
+/** Schema: ^[A-Za-z0-9._:/-]{1,128}$ */
+export function sanitizeService(raw) {
+    const s = String(raw ?? "").replace(/[^A-Za-z0-9._:/-]/g, "_").slice(0, 128);
+    return s === "" ? "unknown" : s;
+}
+/** Schema: ^[a-z0-9][a-z0-9_-]{0,31}$ */
+export function sanitizeEnvironment(raw) {
+    const s = String(raw ?? "").toLowerCase().replace(/[^a-z0-9_-]/g, "-").replace(/^[^a-z0-9]+/, "").slice(0, 32);
+    return s === "" ? "production" : s;
+}
+
+/** logCategory enum of the contract. The order is the priority: an OOM that mentions a timeout is an oom. */
+export function logCategoryOf(message, stacktrace, errorType) {
+    const t = `${message ?? ""}\n${stacktrace ?? ""}`;
+    if (/OutOfMemoryError|out of memory|Memory limit of|\bOOM\b/i.test(t)) return "oom";
+    if (/TimeoutException|timed? ?out|deadline exceeded/i.test(t)) return "timeout";
+    if (/SQLException|PSQLException|SQLSTATE|duplicate key|deadlock|MongoException|R2dbc|JDBC|relation "[^"]*" does not exist/i.test(t) || /^PG_/.test(errorType ?? "")) return "db";
+    if (/AccessDenied|Unauthorized|Forbidden|SecurityException|authentication failed|invalid token|\bCSRF\b/i.test(t)) return "security";
+    if (/\bHTTP\s+5\d\d\b|\bstatus[= :]+5\d\d\b/i.test(t)) return "http_5xx";
+    if (errorType || /Exception|Traceback|Error:/.test(t)) return "exception";
+    return "other";
+}

package/src/incident-detector/config.js

@@ -0,0 +1,95 @@
+/**
+ * config.js - LOCAL_AGENT_INCIDENT_DETECTOR flag and limits (F3-11b).
+ *
+ *   false (default)  only this file is loaded (to read the flag): no source is opened, nothing is read, nothing is sent.
+ *   true             the agent watches the configured log sources, detects errors with the SAME rules as the Java monitor
+ *                    (parity vectors) and sends `incident.candidate` frames (metadata only) on the v2 session. Needs
+ *                    LOCAL_AGENT_PROTOCOL_V2=true as well (without a v2 session there is nobody to send to: the flag does nothing).
+ *
+ * Sources (at least one, or the detector does not start):
+ *   LOCAL_AGENT_INCIDENT_LOG_FILES   comma separated file paths, tailed (new lines only; rotation/truncation followed)
+ *   LOCAL_AGENT_INCIDENT_STDIN       true = read log lines from the agent's own stdin (pipe a container/process into it)
+ *   LOCAL_AGENT_INCIDENT_LOKI_URL    base URL of a Loki (http/https), polled with LOCAL_AGENT_INCIDENT_LOKI_QUERY (LogQL, required)
+ *                                    optional: LOCAL_AGENT_INCIDENT_LOKI_TOKEN (Bearer), LOCAL_AGENT_INCIDENT_LOKI_ORG (X-Scope-OrgID)
+ *   Alloy / OTLP / cloud providers: documented hooks (see README.md in this folder), not built here.
+ * Every number is clamped: a bad value falls back to the default, never to "unlimited".
+ */
+import os from "node:os";
+import path from "node:path";
+
+export const INCIDENT_DETECTOR_FLAG = "LOCAL_AGENT_INCIDENT_DETECTOR";
+
+const TRUE = new Set(["true", "1", "yes", "on"]);
+const FALSE = new Set(["", "false", "0", "no", "off"]);
+const KIB = 1024;
+const MIB = 1024 * 1024;
+
+function int(raw, def, min, max) {
+    const n = Number.parseInt(String(raw ?? ""), 10);
+    return Number.isFinite(n) && n >= min && n <= max ? n : def;
+}
+const bool = (raw) => TRUE.has(String(raw ?? "").trim().toLowerCase());
+const list = (raw, max) => String(raw ?? "").split(",").map((s) => s.trim()).filter(Boolean).slice(0, max);
+
+export function resolveIncidentDetectorConfig(env = process.env, warn = console.warn) {
+    const raw = String(env[INCIDENT_DETECTOR_FLAG] ?? "").trim().toLowerCase();
+    if (!TRUE.has(raw)) {
+        if (!FALSE.has(raw)) warn(`[la-incidents] unrecognised ${INCIDENT_DETECTOR_FLAG} value; the incident detector stays off`);
+        return { enabled: false };
+    }
+    let lokiUrl = null;
+    const lokiRaw = String(env.LOCAL_AGENT_INCIDENT_LOKI_URL ?? "").trim();
+    if (lokiRaw) {
+        try {
+            const u = new URL(lokiRaw);
+            if (u.protocol === "http:" || u.protocol === "https:") lokiUrl = u.origin + u.pathname.replace(/\/+$/, "");
+            else warn("[la-incidents] LOCAL_AGENT_INCIDENT_LOKI_URL must be http(s); the Loki source is off");
+        } catch { warn("[la-incidents] LOCAL_AGENT_INCIDENT_LOKI_URL is not a URL; the Loki source is off"); }
+    }
+    const lokiQuery = String(env.LOCAL_AGENT_INCIDENT_LOKI_QUERY ?? "").trim();
+    if (lokiUrl && !lokiQuery) { warn("[la-incidents] LOCAL_AGENT_INCIDENT_LOKI_URL without LOCAL_AGENT_INCIDENT_LOKI_QUERY; the Loki source is off"); lokiUrl = null; }
+    const maxEmitsPerMinute = int(env.LOCAL_AGENT_INCIDENT_MAX_EMITS_PER_MINUTE, 60, 1, 600);
+    const flushMs = int(env.LOCAL_AGENT_INCIDENT_FLUSH_MS, 30_000, 200, 600_000);
+    return {
+        enabled: true,
+        files: list(env.LOCAL_AGENT_INCIDENT_LOG_FILES, 32),
+        stdin: bool(env.LOCAL_AGENT_INCIDENT_STDIN),
+        loki: lokiUrl ? {
+            url: lokiUrl,
+            query: lokiQuery.slice(0, 2000),
+            token: String(env.LOCAL_AGENT_INCIDENT_LOKI_TOKEN ?? "").trim() || null,
+            org: String(env.LOCAL_AGENT_INCIDENT_LOKI_ORG ?? "").trim().slice(0, 128) || null,
+            pollMs: int(env.LOCAL_AGENT_INCIDENT_LOKI_POLL_MS, 15_000, 1_000, 600_000),
+            limit: int(env.LOCAL_AGENT_INCIDENT_LOKI_LIMIT, 500, 1, 5_000),
+            maxPages: int(env.LOCAL_AGENT_INCIDENT_LOKI_MAX_PAGES, 5, 1, 50),
+            timeoutMs: int(env.LOCAL_AGENT_INCIDENT_LOKI_TIMEOUT_MS, 10_000, 500, 60_000),
+            maxResponseBytes: int(env.LOCAL_AGENT_INCIDENT_LOKI_MAX_RESPONSE_BYTES, 4 * MIB, 64 * KIB, 32 * MIB),
+            lookbackMs: int(env.LOCAL_AGENT_INCIDENT_LOKI_LOOKBACK_MS, 60_000, 0, 24 * 3_600_000),   // first poll looks this far back
+        } : null,
+        service: String(env.LOCAL_AGENT_INCIDENT_SERVICE ?? "").trim() || null,
+        environment: String(env.LOCAL_AGENT_INCIDENT_ENVIRONMENT ?? "").trim().toLowerCase() || null,
+        workspaceId: String(env.LOCAL_AGENT_INCIDENT_WORKSPACE ?? "").trim() || null,
+        filePollMs: int(env.LOCAL_AGENT_INCIDENT_FILE_POLL_MS, 1_000, 50, 60_000),
+        fromStart: bool(env.LOCAL_AGENT_INCIDENT_FROM_START),            // files: read the existing content on the first open (default: only new lines)
+        // detection / dedup
+        windowMs: int(env.LOCAL_AGENT_INCIDENT_WINDOW_MS, 300_000, 1_000, 3_600_000),   // one candidate window per errorHash
+        flushMs,                                                                          // at most one update per errorHash per flushMs
+        maxTracked: int(env.LOCAL_AGENT_INCIDENT_MAX_TRACKED, 5_000, 10, 100_000),        // errorHashes tracked at once (LRU)
+        maxSeenKeys: int(env.LOCAL_AGENT_INCIDENT_MAX_SEEN_KEYS, 20_000, 100, 500_000),   // natural keys remembered (re-read protection)
+        // bounds
+        maxLineBytes: int(env.LOCAL_AGENT_INCIDENT_MAX_LINE_BYTES, 16 * KIB, 512, 256 * KIB),
+        maxEntryBytes: int(env.LOCAL_AGENT_INCIDENT_MAX_ENTRY_BYTES, 64 * KIB, 1 * KIB, 1 * MIB),   // message + stack of one entry
+        maxStackLines: int(env.LOCAL_AGENT_INCIDENT_MAX_STACK_LINES, 200, 1, 5_000),
+        maxPending: int(env.LOCAL_AGENT_INCIDENT_MAX_PENDING, 500, 1, 50_000),            // candidates waiting to be sent
+        maxEmitsPerMinute,
+        emitBurst: Math.min(maxEmitsPerMinute, 20),
+        // local detail store (JSONL, 0600): the text of the error stays HERE
+        storeDir: String(env.LOCAL_AGENT_INCIDENT_STORE_DIR ?? "").trim() || path.join(os.homedir(), ".deepdebug", "incidents"),
+        storeMaxBytes: int(env.LOCAL_AGENT_INCIDENT_STORE_MAX_BYTES, 8 * MIB, 64 * KIB, 512 * MIB),   // per file; at the limit it rotates to .1 (so 2x on disk)
+        storeMaxQueue: int(env.LOCAL_AGENT_INCIDENT_STORE_MAX_QUEUE, 200, 1, 10_000),
+        samplesPerWindow: int(env.LOCAL_AGENT_INCIDENT_SAMPLES_PER_WINDOW, 3, 1, 50),     // occurrences kept per errorHash and window
+        storeMessageChars: int(env.LOCAL_AGENT_INCIDENT_STORE_MESSAGE_CHARS, 8_000, 100, 64_000),
+        storeStackChars: int(env.LOCAL_AGENT_INCIDENT_STORE_STACK_CHARS, 16_000, 100, 128_000),
+        auditFile: String(env.LOCAL_AGENT_INCIDENT_AUDIT_FILE ?? "").trim() || null,
+    };
+}

package/src/incident-detector/detector.js

@@ -0,0 +1,93 @@
+/**
+ * detector.js - the incident detector of the Local Agent (F3-11b): sources -> entries -> the Java rules -> local store +
+ * candidates (metadata only) -> v2 session.
+ *
+ * For each entry (one error record of a log source):
+ *   1. an entry already seen (same service|timestamp|md5(message), the Java LogEntryDedup key: a Loki overlap, a restart) is skipped
+ *   2. ErrorDetector.isSignificant (severity ERROR/CRITICAL/FATAL/EMERG/ALERT/SEVERE + noise + stack/HTTP patterns); LogNoiseClassifier
+ *      infra/build entries are skipped (the Java monitor persists them but never promotes them)
+ *   3. errorHash = group hash of the first line, fingerprint = IncidentFingerprintGenerator; no fingerprint (empty message) = skipped
+ *   4. the detail (message, stack) goes to the local store (masked by the DLP); the aggregator decides what is sent and when
+ *   5. the candidate frame carries metadata only (emitter.js)
+ * The text of the entry never leaves this process except into the local store. Counters only in logs and audit.
+ */
+import { classifyNoise, computeFingerprint, computeGroupHash, extractErrorType, isNoiseCategory, isSignificant, naturalKey } from "./normalize.js";
+import { inferEnvironment, logCategoryOf, severityOf } from "./categorize.js";
+import { createAggregator } from "./aggregator.js";
+
+export function createIncidentDetector({ config, emitter, store, audit = null, now = Date.now, timers = { setInterval, clearInterval }, sources = [], log = console }) {
+    const stats = { seen: 0, duplicates: 0, insignificant: 0, noise: 0, noFingerprint: 0, detected: 0, stored: 0, truncated: 0 };
+    const seenKeys = new Set();   // natural keys (insertion order = oldest first)
+    let tickTimer = null;
+    let started = false;
+    const note = (event, extra = {}) => { try { audit?.record({ event, ...extra }); } catch { /* never breaks the data path */ } };
+
+    const aggregator = createAggregator({
+        windowMs: config.windowMs, flushMs: config.flushMs, maxTracked: config.maxTracked, now,
+        onCandidate: (snap) => emitter.submit(snap),
+    });
+
+    function remember(key) {
+        if (key == null) return false;
+        if (seenKeys.has(key)) return true;
+        seenKeys.add(key);
+        if (seenKeys.size > config.maxSeenKeys) seenKeys.delete(seenKeys.values().next().value);
+        return false;
+    }
+
+    /** One entry: {severity, message, stacktrace, service, tsMs, tsNs?}. Resolves when its detail is queued (backpressure). */
+    async function ingest(entry) {
+        stats.seen++;
+        if (!entry || typeof entry.message !== "string") { stats.insignificant++; return; }
+        let message = entry.message;
+        let stack = typeof entry.stacktrace === "string" ? entry.stacktrace : null;
+        if (message.length + (stack ? stack.length : 0) > config.maxEntryBytes) {
+            stats.truncated++;
+            message = message.slice(0, config.maxEntryBytes);
+            stack = stack ? stack.slice(0, Math.max(0, config.maxEntryBytes - message.length)) : null;
+        }
+        const service = entry.service || config.service || "unknown";
+        const nanos = entry.tsNs != null ? entry.tsNs : Number.isFinite(entry.tsMs) ? BigInt(Math.trunc(entry.tsMs)) * 1_000_000n : null;
+        if (remember(naturalKey(service, nanos, message))) { stats.duplicates++; return; }
+        if (!isSignificant({ severity: entry.severity, message, stacktrace: stack })) { stats.insignificant++; return; }
+        if (isNoiseCategory(classifyNoise(message))) { stats.noise++; return; }
+        const fingerprint = computeFingerprint(message, stack);
+        if (fingerprint == null) { stats.noFingerprint++; return; }
+        const errorHash = computeGroupHash(message);
+        const severity = severityOf(entry.severity);
+        const environment = config.environment || inferEnvironment(service);
+        const logCategory = logCategoryOf(message, stack, extractErrorType(message, stack));
+        stats.detected++;
+        const stackBytes = typeof entry.stacktrace === "string" ? Buffer.byteLength(entry.stacktrace) : 0;   // as the source gave it, before the entry cap
+        const { windowStart } = aggregator.observe({ errorHash, fingerprint, severity, service, environment, logCategory, tsMs: entry.tsMs, stackBytes });
+        // the detail of the occurrence, tied to its window (the store keeps a few samples per window); resolves when queued
+        if (await store.add({ errorHash, fingerprint, service, severity, windowStart, message, stack })) stats.stored++;
+    }
+
+    const onEntries = async (entries) => { for (const e of entries) await ingest(e); };
+
+    return {
+        stats,
+        aggregator,
+        ingest,
+        onEntries,
+        start() {
+            if (started) return;
+            started = true;
+            tickTimer = timers.setInterval(() => aggregator.tick(), Math.min(config.flushMs, 5_000));
+            if (tickTimer && typeof tickTimer.unref === "function") tickTimer.unref();
+            for (const s of sources) s.start(onEntries);
+            note("start", { sources: sources.map((s) => s.name) });
+        },
+        async stop() {
+            if (!started) return;
+            started = false;
+            for (const s of sources) { try { s.stop(); } catch { /* ignore */ } }
+            if (tickTimer) timers.clearInterval(tickTimer);
+            tickTimer = null;
+            aggregator.drain();
+            note("stop", {});
+        },
+        getDetail: (errorHash, opts) => store.getDetail(errorHash, opts),
+    };
+}

package/src/incident-detector/emitter.js

@@ -0,0 +1,164 @@
+/**
+ * emitter.js - sends `incident.candidate` frames (F3-11b, contract F3-22: la->cp, metadata only).
+ *
+ * The payload has the 9 contract fields: errorHash, fingerprint, severity, service, environment, count, firstSeen,
+ * lastSeen, logCategory and, ONLY when the CP announced `incident.candidate.size-model` in `ready.capabilities` (contract 2.1 (c)), two more
+ * that carry no text: `stackBytes` (UTF-8 size of the biggest stack of the window) and `model` (detector version, e.g. stack-detector-1.8.0). There is no field for text, a stack, a path or a message, and the session validates every frame against
+ * the closed schema before it is written, so even a bug here cannot add one. (window = firstSeen..lastSeen.)
+ *
+ *   DLP       service and environment are labels of the customer's own logs; they go through the DLP and a label it would change
+ *             (or deny) is not sent at all: the candidate is dropped and counted (`dlp`).
+ *   bounds    pending candidates are coalesced by (errorHash, window): a newer count replaces the older one; at most maxPending;
+ *             when full a NEW candidate is dropped (counted), an update of a known one is still accepted.
+ *             Rate: a token bucket of maxEmitsPerMinute (burst emitBurst). ONE frame in flight; the ack is the answer.
+ *   the ack   accepted = done. rate_limited: back off 30 s doubling to 5 min, keep everything. protocol_unsupported: this gateway
+ *             does not take candidates: forget the queue and stay quiet until the next `ready`. internal/timeout: send again after a
+ *             short pause, at most 5 attempts in a row (then that candidate is dropped, counted). Frames the session refuses before
+ *             writing (schema, size): dropped, counted (`invalid`). Not ready / reset / offline: kept; the next `ready` sends them.
+ *   never     logs or audits a message, a path or a label: counters and codes only.
+ */
+import { ProtocolError } from "../la-protocol-v2/errors.js";
+import { protectEgress } from "../dlp/index.js";
+import { sanitizeEnvironment, sanitizeService } from "./categorize.js";
+
+const BACKOFF_MIN_MS = 30_000;
+const BACKOFF_MAX_MS = 300_000;
+const RETRY_MS = 5_000;
+const MAX_ATTEMPTS = 5;
+const NON_RETRYABLE_REFUSALS = new Set(["schema_violation", "frame_too_large", "payload_too_large", "unknown_type", "wrong_direction"]);
+
+const SIZE_MODEL_CAPABILITY = "incident.candidate.size-model";
+const MODEL = /^[A-Za-z0-9][A-Za-z0-9._:+-]{0,47}$/;
+const MAX_STACK_BYTES = 16 * 1024 * 1024;
+
+export function createCandidateEmitter({ session, config, workspaceId = "", model = null, env = process.env, audit = null, log = console, now = Date.now, timers = { setTimeout, clearTimeout } }) {
+    const warn = (m) => (log.warn || log.log).call(log, m);
+    const pending = new Map();   // `${errorHash}|${windowStart}` -> snapshot
+    const stats = { submitted: 0, sent: 0, coalesced: 0, droppedFull: 0, droppedDlp: 0, droppedInvalid: 0, droppedGaveUp: 0, droppedStopped: 0, rejected: 0, byCode: {} };
+    let tokens = config.emitBurst;
+    let refillAt = now();
+    let timer = null;
+    let running = null;
+    let stopped = false;       // protocol_unsupported: quiet until the next ready
+    let closed = false;
+    let attempts = 0;
+    let backoffMs = 0;
+    let notBefore = 0;
+    const unref = (t) => { if (t && typeof t.unref === "function") t.unref(); return t; };
+    const note = (event, extra = {}) => { try { audit?.record({ event, ...extra }); } catch { /* the audit never breaks the data path */ } };
+
+    function refill() {
+        const t = now();
+        const add = ((t - refillAt) * config.maxEmitsPerMinute) / 60_000;
+        if (add > 0) { tokens = Math.min(config.emitBurst, tokens + add); refillAt = t; }
+    }
+
+    const iso = (ms) => new Date(ms).toISOString();
+
+    /** The 9 fields, or null when a label does not survive the DLP unchanged. */
+    function payloadOf(s) {
+        const service = sanitizeService(s.service);
+        const environment = sanitizeEnvironment(s.environment);
+        for (const label of [service, environment]) {
+            const r = protectEgress(label, { env, source: "incident-candidate" });
+            if (r.denied || r.text !== label) return null;
+        }
+        const payload = { errorHash: s.errorHash, fingerprint: s.fingerprint, severity: s.severity, service, environment, count: s.count, firstSeen: iso(s.firstSeen), lastSeen: iso(s.lastSeen), logCategory: s.logCategory };
+        // contract 2.1 (c): only when the CP said (ready.capabilities) it takes them: a 2.0 CP answers the closed schema with schema_violation
+        if (typeof session.peerHas === "function" && session.peerHas(SIZE_MODEL_CAPABILITY)) {
+            if (Number.isFinite(s.stackBytes)) payload.stackBytes = Math.max(0, Math.min(Math.trunc(s.stackBytes), MAX_STACK_BYTES));
+            if (typeof model === "string" && MODEL.test(model)) payload.model = model;
+        }
+        return payload;
+    }
+
+    function schedule(ms) {
+        if (closed || timer) return;
+        timer = unref(timers.setTimeout(() => { timer = null; void pump(); }, ms));
+    }
+
+    function pump() {
+        if (running) return running;
+        if (closed || stopped || pending.size === 0 || session.state !== "ready") return Promise.resolve();
+        if (notBefore > now()) { schedule(notBefore - now()); return Promise.resolve(); }
+        running = drain().finally(() => { running = null; });
+        return running;
+    }
+
+    async function drain() {
+        try {
+            while (!closed && !stopped && pending.size > 0 && session.state === "ready") {
+                refill();
+                if (tokens < 1) { schedule(Math.ceil(((1 - tokens) * 60_000) / config.maxEmitsPerMinute)); return; }
+                const [key, snap] = pending.entries().next().value;
+                let payload = null;
+                let unbuildable = false;
+                try { payload = payloadOf(snap); } catch { unbuildable = true; }   // e.g. an invalid date from an untrusted log line: dropped, it never blocks the queue
+                if (!payload) {
+                    pending.delete(key);
+                    if (unbuildable) { stats.droppedInvalid++; note("drop", { reason: "invalid", errorHash: snap.errorHash }); }
+                    else { stats.droppedDlp++; note("drop", { reason: "dlp", errorHash: snap.errorHash }); }
+                    continue;
+                }
+                tokens -= 1;
+                try {
+                    await session.send("incident.candidate", payload, { workspaceId: workspaceId || undefined });
+                } catch (e) {
+                    const code = e instanceof ProtocolError ? e.code : "internal";
+                    stats.rejected++;
+                    stats.byCode[code] = (stats.byCode[code] || 0) + 1;
+                    tokens += 1;   // nothing was spent on the wire for a refusal before writing; for the others the bucket stays honest enough
+                    if (NON_RETRYABLE_REFUSALS.has(code)) {
+                        pending.delete(key); stats.droppedInvalid++; note("drop", { reason: "invalid", code, errorHash: snap.errorHash });
+                        continue;
+                    }
+                    if (code === "protocol_unsupported") {
+                        stopped = true; pending.clear();
+                        warn("[la-incidents] the gateway does not take incident candidates (protocol_unsupported); quiet until the next connection");
+                    } else if (code === "rate_limited") {
+                        backoffMs = Math.min(BACKOFF_MAX_MS, backoffMs ? backoffMs * 2 : BACKOFF_MIN_MS);
+                        notBefore = now() + backoffMs;
+                        warn(`[la-incidents] candidates rate limited; sending again in ${Math.round(backoffMs / 1000)} s`);
+                        schedule(backoffMs);
+                    } else if (code === "internal" || code === "timeout") {
+                        if (++attempts >= MAX_ATTEMPTS) {
+                            attempts = 0; pending.delete(key); stats.droppedGaveUp++; note("drop", { reason: "gave_up", code, errorHash: snap.errorHash });
+                            warn(`[la-incidents] one candidate not accepted (${code}); gave up after ${MAX_ATTEMPTS} attempts`);
+                            if (pending.size > 0) schedule(RETRY_MS);
+                        } else {
+                            warn(`[la-incidents] candidate not accepted (${code}); trying again shortly`);
+                            schedule(RETRY_MS);
+                        }
+                    }
+                    return;   // anything else (not_ready, agent_offline, reset): kept; the next `ready` sends again
+                }
+                attempts = 0; backoffMs = 0; notBefore = 0;
+                stats.sent++;
+                note("candidate", { errorHash: snap.errorHash, severity: snap.severity, count: snap.count, logCategory: snap.logCategory });
+                if (pending.get(key) === snap) pending.delete(key);   // a newer count that arrived meanwhile stays for the next round
+            }
+        } catch { /* nothing here may throw into the data path */ }
+    }
+
+    const onReady = () => { stopped = false; backoffMs = 0; notBefore = 0; attempts = 0; void pump(); };
+    session.on("ready", onReady);
+
+    return {
+        stats,
+        get pending() { return pending.size; },
+        submit(snap) {
+            if (closed) return false;
+            stats.submitted++;
+            if (stopped) { stats.droppedStopped++; return false; }   // this gateway takes no candidates (until the next ready)
+            const key = `${snap.errorHash}|${snap.windowStart}`;
+            if (pending.has(key)) { pending.set(key, snap); stats.coalesced++; }
+            else if (pending.size >= config.maxPending) { stats.droppedFull++; note("drop", { reason: "queue_full", errorHash: snap.errorHash }); return false; }
+            else pending.set(key, snap);
+            void pump();
+            return true;
+        },
+        /** Waits until nothing is pending or in flight, or the session is not ready (tests, shutdown). */
+        async flush() { if (timer) { timers.clearTimeout(timer); timer = null; } await pump(); if (running) await running; },
+        close() { closed = true; if (timer) timers.clearTimeout(timer); timer = null; pending.clear(); session.off("ready", onReady); },
+    };
+}

package/src/incident-detector/index.js

@@ -0,0 +1,44 @@
+/**
+ * index.js - installs the incident detector on a v2 link (F3-11b). Called by la-protocol-v2/runtime.js only when
+ * LOCAL_AGENT_INCIDENT_DETECTOR=true (and v2 is on). Returns the detector (link.incidents), or null when there is nothing to start.
+ */
+import { createAudit } from "../la-protocol-v2/content-handlers/audit.js";
+import { createDetailStore } from "./store.js";
+import { createCandidateEmitter } from "./emitter.js";
+import { createIncidentDetector } from "./detector.js";
+import { createFileSource } from "./sources/file-source.js";
+import { createLokiSource } from "./sources/loki-source.js";
+import { createStreamSource } from "./sources/stream-source.js";
+
+export { INCIDENT_DETECTOR_FLAG, resolveIncidentDetectorConfig } from "./config.js";
+
+const WORKSPACE_ID = /^[A-Za-z0-9_.-]{0,64}$/;
+
+export function buildSources(config, { stdin = process.stdin } = {}) {
+    const common = { maxLineBytes: config.maxLineBytes, maxEntryBytes: config.maxEntryBytes, maxStackLines: config.maxStackLines };
+    const sources = [];
+    for (const filePath of config.files) sources.push(createFileSource({ filePath, service: config.service, fromStart: config.fromStart, pollMs: config.filePollMs, ...common }));
+    if (config.stdin) sources.push(createStreamSource({ stream: stdin, service: config.service || "stdin", ...common }));
+    if (config.loki) sources.push(createLokiSource({ ...config.loki, service: config.service }));
+    return sources;
+}
+
+export function installIncidentDetector({ link, workspaceId = "", config, env = process.env, log = console, sources = null, extraSources = [], model = null }) {
+    const list = sources || [...buildSources(config), ...extraSources];   // extraSources: push sources owned by someone else (the webhook receiver, F3-12 consumer)
+    if (list.length === 0) {
+        log.warn("[la-incidents] LOCAL_AGENT_INCIDENT_DETECTOR=true but no source is configured (LOCAL_AGENT_INCIDENT_LOG_FILES, LOCAL_AGENT_INCIDENT_STDIN or LOCAL_AGENT_INCIDENT_LOKI_URL): the detector does not start");
+        return null;
+    }
+    const ws = WORKSPACE_ID.test(config.workspaceId || "") && config.workspaceId ? config.workspaceId : workspaceId;
+    const audit = createAudit({ file: config.auditFile });
+    const store = createDetailStore({ dir: config.storeDir, maxBytes: config.storeMaxBytes, maxQueue: config.storeMaxQueue, messageChars: config.storeMessageChars,
+        stackChars: config.storeStackChars, samplesPerWindow: config.samplesPerWindow, env });
+    const emitter = createCandidateEmitter({ session: link.session, config, workspaceId: ws, model, env, audit, log });
+    const detector = createIncidentDetector({ config, emitter, store, audit, sources: list, log });
+    const closeLink = link.close.bind(link);
+    link.close = () => { void detector.stop().finally(() => { emitter.close(); }); closeLink(); };
+    link.incidents = { detector, emitter, store, audit, getDetail: (errorHash, opts) => store.getDetail(errorHash, opts) };
+    detector.start();
+    log.info(`[la-incidents] incident detector on (${list.map((s) => s.name).join(", ")}); candidates carry metadata only, the detail stays in the local store`);
+    return link.incidents;
+}

package/src/incident-detector/java-compat.js

@@ -0,0 +1,66 @@
+/**
+ * java-compat.js - the few java.lang.String behaviours the parity needs (F3-11b).
+ *
+ * The detector must produce the SAME hashes as the Java code (monitor-service, shared/). JS and Java differ in small,
+ * silent ways that change a hash for exactly the messages nobody tests by hand:
+ *   - `\s` / `\S`        Java: [ \t\n\x0B\f\r] only. JS also matches NBSP, U+2003, U+2028... -> use SP_CLASS / NOT_SP_CLASS.
+ *   - String.trim()      Java removes every char <= U+0020. JS trim() removes Unicode whitespace (and not U+0001..U+001F).
+ *   - String.isBlank()   Java: Character.isWhitespace (no NBSP / U+2007 / U+202F). JS: Unicode White_Space.
+ *   - stripLeading()     same rule as isBlank.
+ *   - getBytes(UTF_8)    a lone surrogate becomes '?'. JS Buffer.from() writes U+FFFD (EF BF BD).
+ * Everything here is proven against the vectors generated from the Java code (test/incident-detector/parity.vectors.json).
+ */
+import crypto from "node:crypto";
+
+/** Java's `\s` as a character-class body (use inside [...]) and as a standalone class. */
+export const SP = " \\t\\n\\x0B\\f\\r";
+export const SP_CLASS = `[${SP}]`;
+export const NOT_SP_CLASS = `[^${SP}]`;
+
+/** String.trim(): strips every leading/trailing char <= U+0020. */
+export function javaTrim(s) {
+    let a = 0;
+    let b = s.length;
+    while (a < b && s.charCodeAt(a) <= 0x20) a++;
+    while (b > a && s.charCodeAt(b - 1) <= 0x20) b--;
+    return a === 0 && b === s.length ? s : s.slice(a, b);
+}
+
+/** Character.isWhitespace(char) as used by String.isBlank()/stripLeading(). */
+export function javaIsWhitespace(c) {
+    if (c === 0x20 || (c >= 0x09 && c <= 0x0d) || (c >= 0x1c && c <= 0x1f)) return true;
+    if (c < 0x80) return false;
+    if (c === 0x00a0 || c === 0x2007 || c === 0x202f) return false;   // non-breaking spaces are excluded
+    return c === 0x1680 || (c >= 0x2000 && c <= 0x200a) || c === 0x2028 || c === 0x2029 || c === 0x205f || c === 0x3000;
+}
+
+export function javaIsBlank(s) {
+    for (let i = 0; i < s.length; i++) if (!javaIsWhitespace(s.charCodeAt(i))) return false;
+    return true;
+}
+
+export function javaStripLeading(s) {
+    let i = 0;
+    while (i < s.length && javaIsWhitespace(s.charCodeAt(i))) i++;
+    return i === 0 ? s : s.slice(i);
+}
+
+/** s.getBytes(StandardCharsets.UTF_8): unpaired surrogates are replaced by '?' (0x3F), not by U+FFFD. */
+export function javaUtf8(s) {
+    // fast path: no surrogate at all
+    if (!/[\ud800-\udfff]/.test(s)) return Buffer.from(s, "utf8");
+    let out = "";
+    for (let i = 0; i < s.length; i++) {
+        const c = s.charCodeAt(i);
+        if (c >= 0xd800 && c <= 0xdbff && i + 1 < s.length) {
+            const d = s.charCodeAt(i + 1);
+            if (d >= 0xdc00 && d <= 0xdfff) { out += s[i] + s[i + 1]; i++; continue; }
+        }
+        out += c >= 0xd800 && c <= 0xdfff ? "?" : s[i];
+    }
+    return Buffer.from(out, "utf8");
+}
+
+export const sha256Hex = (s) => crypto.createHash("sha256").update(javaUtf8(s)).digest("hex");
+export const sha256Prefix16 = (s) => crypto.createHash("sha256").update(javaUtf8(s)).digest().subarray(0, 16).toString("hex");
+export const md5Hex = (s) => crypto.createHash("md5").update(javaUtf8(s)).digest("hex");