@bli-cockpit/log-digest 0.0.0-stage → 0.1.0

package/dist/digest.js

@@ -0,0 +1,259 @@
+/**
+ * The digest accumulator (port of Kaiben's `log_digest.Digest`).
+ *
+ * Keeps only the lines that can mean trouble, masks their variable parts into
+ * templates, groups equal templates with a count, first/last time and one
+ * example, and turns routine lines into counts per event type. Routine runs are
+ * collapsed into facts true of every collapsed line: a small value set is
+ * enumerated with exact counts, a large one becomes "N distinct" (Caveman).
+ *
+ * Output is deterministic and order-stable: the same input renders
+ * byte-identically.
+ *
+ * Not ported: the circuit-breaker facts (`breaker`) that read Kaiben's own
+ * deploy modules. The result still carries `breaker: {}` so the shape matches.
+ */
+import { DECLARED_WARNING, LONG_LINE_CHARS, judgeJson, jsonRecords, looksImportant, pythonJson, unitGroup } from "./judge.js";
+import { EXAMPLE_CHARS, cut, mask, redact, squash, templateId } from "./mask.js";
+import { TEMPLATE_CHARS } from "./mask.js";
+import { judgeTagged, parseTagged, withoutZeroCounters } from "./tagged.js";
+export const SUBJECTS_TRACKED = 6; // Caveman: enumerate a field only while its value set stays small
+/** Units that only ever say "someone logged in" or "a package timer ran": dropped and counted as noise. */
+export const DEFAULT_NOISE = [
+    "^ssh(d)?(\\.service)?$", "^sshd-session$", "^systemd-logind(\\.service)?$", "^session-\\d+\\.scope$",
+    "^user@\\d+\\.service$", "^user-runtime-dir@\\d+\\.service$", "^user-\\d+\\.slice$", "^run-user-\\d+\\.mount$",
+    "^(cron|CRON)(\\.service)?$", "^dbus(\\.service)?$", "^\\(sd-pam\\)$", "^packagekit(d)?(\\.service)?$",
+    "^(apt-news|esm-cache|apt-daily|apt-daily-upgrade|motd-news|man-db|logrotate|dpkg-db-backup|e2scrub_all|fstrim|"
+        + "systemd-tmpfiles-clean|update-notifier-download|update-notifier-motd|fwupd-refresh|fwupd|snapd|"
+        + "unattended-upgrades|systemd-timedated|sysstat-collect|sysstat-summary|polkit|systemd-hostnamed)(\\.service)?$",
+];
+// systemd's own routine words about a unit; the rest of such a line is the unit's description, which can say anything
+const SYSTEMD_ROUTINE = /^(Starting|Started|Finished|Stopping|Stopped|Reloading|Reloaded|Created slice|Removed slice|Listening|Reached target|Closed|Queued|Activating|Deactivated|Consumed|\S+: (Deactivated|Consumed|Succeeded|Main process exited, code=exited, status=0\/SUCCESS))/;
+const EXCEPTION_LINE = /^([A-Za-z_][\w.]*(?:Error|Exception|Interrupt|Exit|Warning|Failure))(?::\s?(.*))?$/;
+const FRAME_LINE = /^\s+File "([^"]+)", line \d+, in (\S+)/;
+function compileAll(patterns) {
+    return patterns.map((pattern) => new RegExp(pattern));
+}
+function emptySubjects() {
+    return Object.create(null);
+}
+export class Digest {
+    groups = new Map();
+    routine = new Map();
+    lines = 0;
+    noiseLines = 0;
+    firstTs = null;
+    lastTs = null;
+    firstCursor = null;
+    lastCursor = null;
+    noise;
+    quiet;
+    routineUnits;
+    taggedLines;
+    openTracebacks = new Map();
+    constructor(options = {}) {
+        this.taggedLines = options.taggedLines ?? false;
+        this.noise = compileAll(options.noise ?? DEFAULT_NOISE);
+        this.quiet = compileAll(options.quietUnits ?? []);
+        this.routineUnits = new Set(options.routineUnits ?? []);
+    }
+    // -- input
+    isNoise(unit, identifier) {
+        return [unit, identifier].some((name) => name && this.noise.some((pattern) => pattern.test(name)));
+    }
+    /** One `journalctl -o json` entry. */
+    addEntry(entry) {
+        let message = entry.MESSAGE;
+        if (Array.isArray(message)) {
+            // journald stores non-UTF-8 messages as a byte list
+            const bytes = message.filter((value) => Number.isInteger(value) && value >= 0 && value < 256);
+            message = Buffer.from(bytes).toString("utf8");
+        }
+        if (typeof message !== "string")
+            return;
+        if (entry.__CURSOR) {
+            this.firstCursor = this.firstCursor ?? entry.__CURSOR;
+            this.lastCursor = entry.__CURSOR;
+        }
+        const parsed = Number.parseInt(String(entry.__REALTIME_TIMESTAMP ?? 0), 10);
+        const ts = Number.isFinite(parsed) ? parsed / 1e6 : 0;
+        const identifier = entry.SYSLOG_IDENTIFIER;
+        let unit = entry.UNIT || entry._SYSTEMD_UNIT;
+        if (!unit)
+            unit = entry._TRANSPORT === "kernel" ? "kernel" : (identifier || "?");
+        const ownUnit = entry._SYSTEMD_UNIT ?? "";
+        if (this.isNoise(unit, identifier)
+            || this.isNoise(ownUnit, undefined)
+            || (!entry.UNIT && this.quiet.some((pattern) => pattern.test(ownUnit)))) {
+            this.noiseLines += 1;
+            return;
+        }
+        this.addLine(unit, message, ts, { manager: Boolean(entry.UNIT) });
+    }
+    /** One log line from any source. `ts` is epoch seconds (0 when the source has none). */
+    addLine(unit, message, ts, facts = {}) {
+        const manager = facts.manager ?? false;
+        this.lines += 1;
+        if (ts) {
+            this.firstTs = this.firstTs === null ? ts : Math.min(this.firstTs, ts);
+            this.lastTs = this.lastTs === null ? ts : Math.max(this.lastTs, ts);
+        }
+        const [group, instance] = unitGroup(unit);
+        const bare = unit.endsWith(".service") ? unit.slice(0, -".service".length) : unit;
+        if (!manager && this.routineUnits.has(bare) && (message.startsWith("FLAG ") || message.startsWith("NOTE "))) {
+            this.routineCount(group, "relay", null); // watchdog and breaker flags: ingested from their own log files
+            return;
+        }
+        if (this.foldTraceback(unit, group, instance, message, ts))
+            return;
+        const stripped = message.trim();
+        if (stripped.startsWith("{") || Array.from(stripped).length > LONG_LINE_CHARS) {
+            const records = jsonRecords(stripped);
+            if (records === null) {
+                // journald splits a huge JSON line into pieces; a piece is not a sentence, and words like
+                // "archiveFailed" or "errors" inside it are key names, not failures.
+                this.routineCount(group, "json_fragment", null);
+                return;
+            }
+            for (const data of records) {
+                if (this.routineUnits.has(bare)) {
+                    this.routineCount(group, "relay", null); // watchdog and breaker records: ingested from their own files
+                    continue;
+                }
+                const judged = judgeJson(data);
+                if (judged.important && judged.template !== null) {
+                    this.addGroup(group, instance, judged.event, judged.template, ts, cut(pythonJson(data), EXAMPLE_CHARS * 2));
+                }
+                else {
+                    this.routineCount(group, judged.event, judged.subject);
+                }
+            }
+            return;
+        }
+        if (FRAME_LINE.test(message) || message.startsWith("    ")) {
+            this.routineCount(group, "traceback_frame", null); // a frame or code line whose traceback header was interleaved away
+            return;
+        }
+        if (this.taggedLines && !manager) {
+            const parsed = parseTagged(message);
+            if (parsed) {
+                const judged = judgeTagged(parsed, facts.level);
+                if (!judged.important) {
+                    this.routineCount(group, judged.template, null);
+                }
+                else {
+                    const template = judged.declaredWarning ? cut(judged.template, TEMPLATE_CHARS - DECLARED_WARNING.length) + DECLARED_WARNING : judged.template;
+                    this.addGroup(group, instance, "tagged", template, ts, message);
+                }
+                return;
+            }
+        }
+        const event = manager ? "systemd" : unit === "kernel" ? "kernel" : "log";
+        // The source's own level is NOT a verdict on a plain line: Tower writes ordinary output to stderr, which Vercel
+        // and Railway both mark "error" (measured 2026-10-07). Words and fields decide; see tagged.ts.
+        const judgedText = this.taggedLines ? withoutZeroCounters(message) : message;
+        if (looksImportant(judgedText) && !(manager && SYSTEMD_ROUTINE.test(message))) {
+            this.addGroup(group, instance, event, mask(message), ts, message);
+            return;
+        }
+        const word = message.split(" ");
+        const routineEvent = manager
+            ? `systemd ${cut((word.length > 1 && word[0].endsWith(":") ? word[1] : word[0]).replace(/^[.:]+|[.:]+$/g, ""), 20)}`
+            : "text";
+        this.routineCount(group, routineEvent, null);
+    }
+    // -- internals
+    routineCount(group, event, subject) {
+        let events = this.routine.get(group);
+        if (!events)
+            this.routine.set(group, (events = new Map()));
+        let record = events.get(event);
+        if (!record)
+            events.set(event, (record = [0, emptySubjects()]));
+        record[0] += 1;
+        if (subject !== null) {
+            const subjects = record[1];
+            if (subject in subjects || Object.keys(subjects).length < SUBJECTS_TRACKED)
+                subjects[subject] = (subjects[subject] ?? 0) + 1;
+            else
+                subjects["…"] = (subjects["…"] ?? 0) + 1;
+        }
+    }
+    addGroup(group, instance, event, template, ts, message) {
+        const tid = templateId(group, event, template);
+        let record = this.groups.get(tid);
+        if (!record) {
+            record = {
+                tid, unit: group, event, template, count: 0,
+                first: ts || null, last: ts || null,
+                example: cut(redact(squash(message)), EXAMPLE_CHARS),
+                instances: [],
+            };
+            this.groups.set(tid, record);
+        }
+        record.count += 1;
+        if (ts) {
+            record.first = Math.min(record.first ?? ts, ts);
+            record.last = Math.max(record.last ?? ts, ts);
+        }
+        if (instance && !record.instances.includes(instance) && record.instances.length < 64)
+            record.instances.push(instance);
+    }
+    /** Python tracebacks span many lines: fold each into one group named by its exception and last frame. */
+    foldTraceback(unit, group, instance, message, ts) {
+        const state = this.openTracebacks.get(unit);
+        if (message.startsWith("Traceback (most recent call last)")) {
+            this.openTracebacks.set(unit, { frame: null });
+            return true;
+        }
+        if (!state)
+            return false;
+        const frame = FRAME_LINE.exec(message);
+        if (frame) {
+            state.frame = `${frame[1].split("/").pop()}:${frame[2]}`;
+            return true;
+        }
+        if (/^[ \t]/.test(message) || message.startsWith("During handling") || message.startsWith("The above exception") || !message.trim()) {
+            return true;
+        }
+        if (message.trimStart().startsWith("{"))
+            return false; // other threads keep logging JSON mid-traceback: judge it, keep it open
+        const exception = EXCEPTION_LINE.exec(message.trim());
+        if (exception) {
+            const where = state.frame ? ` @${state.frame}` : "";
+            const named = `${exception[1]}: ${exception[2] ?? ""}`.replace(/[: ]+$/, "");
+            const template = mask(named) + where;
+            this.addGroup(group, instance, "traceback", cut(template, TEMPLATE_CHARS), ts, message);
+            state.frame = null; // a chained exception may follow; its own frames come first
+            return true;
+        }
+        this.openTracebacks.delete(unit); // anything else ends the traceback; judge the line normally
+        return false;
+    }
+    // -- output
+    /** The result, sorted so equal input renders byte-identically: groups by count then tid, routine by name. */
+    result() {
+        const routine = Object.create(null);
+        for (const group of [...this.routine.keys()].sort()) {
+            const events = Object.create(null);
+            for (const event of [...this.routine.get(group).keys()].sort()) {
+                const [count, subjects] = this.routine.get(group).get(event);
+                const sorted = emptySubjects();
+                for (const name of Object.keys(subjects).sort())
+                    sorted[name] = subjects[name];
+                events[event] = [count, sorted];
+            }
+            routine[group] = events;
+        }
+        const groups = [...this.groups.values()].sort((a, b) => b.count - a.count || (a.tid < b.tid ? -1 : a.tid > b.tid ? 1 : 0));
+        return {
+            lines: this.lines,
+            noise: this.noiseLines,
+            span: [this.firstTs, this.lastTs],
+            cursors: [this.firstCursor, this.lastCursor],
+            groups,
+            routine,
+            breaker: {},
+        };
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,15 @@
+export { mask, redact, templateId, cut, squash, TEMPLATE_CHARS, EXAMPLE_CHARS } from "./mask.js";
+export { DECLARED_WARNING, LONG_LINE_CHARS, judgeJson, jsonEvent, jsonRecords, looksImportant, pythonJson, unitGroup, type JsonJudgement, type JsonObject, } from "./judge.js";
+export { DEFAULT_NOISE, Digest, SUBJECTS_TRACKED, type DigestGroup, type DigestOptions, type DigestResult, type JournalEntry, type LineFacts, type RoutineEvents, } from "./digest.js";
+export { mergeResults, type MergeInput, type MergedResult } from "./merge.js";
+export { CLASS_ORDER, ClassTable, type Classify, type ClassInfo, type ClassTableData, type Severity, type TablePattern, type Verdict, } from "./classify.js";
+export { clock, fleetView, numberText, renderCapped, renderFleet, renderLocal, routineText, subjectsText, timeRange, type CappedOptions, type FleetView, } from "./render.js";
+export { TOWER_CLASSES, TOWER_PATTERNS, TOWER_TABLE, towerClassTable } from "./tower-classes.js";
+export { WindowError, isoOf, parseWindow, relativeSeconds, type LogWindow } from "./window.js";
+export { DEFAULT_MAX_BYTES, SOURCE_NAMES, digestSource, sourceFamily, type DigestAnswer, type DigestEnvironment, type DigestOutcome, type DigestRequest, type DigestTemplate, type RawAnswer, type Refusal, } from "./digest-source.js";
+export { SourceRefusal, type CliRun, type LogFn, type RunCli, type SourceContext, type SourceLine, type SourceRead } from "./sources/types.js";
+export { nodeRunCli } from "./sources/process.js";
+export { readCollector, receiptLine, streamLines, windowShare } from "./sources/collector.js";
+export { RAILWAY_ENVIRONMENT, RAILWAY_PROJECT_ID, railwayRefusal, readRailway } from "./sources/railway.js";
+export { VERCEL_PROJECT, VERCEL_SCOPE, readDashboard, routeTemplate, vercelRefusal } from "./sources/vercel.js";
+export { readStdin } from "./sources/stdin.js";

package/dist/index.js

@@ -0,0 +1,15 @@
+export { mask, redact, templateId, cut, squash, TEMPLATE_CHARS, EXAMPLE_CHARS } from "./mask.js";
+export { DECLARED_WARNING, LONG_LINE_CHARS, judgeJson, jsonEvent, jsonRecords, looksImportant, pythonJson, unitGroup, } from "./judge.js";
+export { DEFAULT_NOISE, Digest, SUBJECTS_TRACKED, } from "./digest.js";
+export { mergeResults } from "./merge.js";
+export { CLASS_ORDER, ClassTable, } from "./classify.js";
+export { clock, fleetView, numberText, renderCapped, renderFleet, renderLocal, routineText, subjectsText, timeRange, } from "./render.js";
+export { TOWER_CLASSES, TOWER_PATTERNS, TOWER_TABLE, towerClassTable } from "./tower-classes.js";
+export { WindowError, isoOf, parseWindow, relativeSeconds } from "./window.js";
+export { DEFAULT_MAX_BYTES, SOURCE_NAMES, digestSource, sourceFamily, } from "./digest-source.js";
+export { SourceRefusal } from "./sources/types.js";
+export { nodeRunCli } from "./sources/process.js";
+export { readCollector, receiptLine, streamLines, windowShare } from "./sources/collector.js";
+export { RAILWAY_ENVIRONMENT, RAILWAY_PROJECT_ID, railwayRefusal, readRailway } from "./sources/railway.js";
+export { VERCEL_PROJECT, VERCEL_SCOPE, readDashboard, routeTemplate, vercelRefusal } from "./sources/vercel.js";
+export { readStdin } from "./sources/stdin.js";

package/dist/judge.d.ts

@@ -0,0 +1,33 @@
+export declare const LONG_LINE_CHARS = 4000;
+/**
+ * Appended to a JSON line's template when the line says `"level": "WARNING"`.
+ * The classifier strips it before matching patterns, and an otherwise unknown
+ * template with it is counted (WARNING), not escalated.
+ */
+export declare const DECLARED_WARNING = " level=WARNING";
+export type JsonObject = Record<string, unknown>;
+/** True when the line contains a trouble word (literal pre-filter, then the regex). */
+export declare function looksImportant(text: string): boolean;
+/**
+ * Every JSON object on one line: unbuffered writers sometimes glue two records
+ * together (`{...}{...}`). Null when the line is not entirely JSON objects.
+ */
+export declare function jsonRecords(text: string): JsonObject[] | null;
+/** `rs-gap@caseys.service` becomes (`rs-gap@*`, `caseys`); `kaiben-shard.service` becomes (`kaiben-shard`, null). */
+export declare function unitGroup(unit: string | null | undefined): [string, string | null];
+/** The event name of one JSON log line: its "event" field, else its first key that is not a timestamp. */
+export declare function jsonEvent(data: JsonObject): [string, boolean];
+export interface JsonJudgement {
+    event: string;
+    important: boolean;
+    template: string | null;
+    subject: string | null;
+}
+/** (event, important, template, subject) for one JSON log line. */
+export declare function judgeJson(data: JsonObject): JsonJudgement;
+/**
+ * `json.dumps` as Python writes it (`", "` and `": "` separators, non-ASCII as
+ * `\uXXXX`), so a stored example matches the Python digest byte for byte. A
+ * float with a zero fraction prints as an integer here; JS cannot tell them apart.
+ */
+export declare function pythonJson(value: unknown): string;

package/dist/judge.js

@@ -0,0 +1,198 @@
+/**
+ * Which lines can mean trouble, and what a JSON log line says (port of the
+ * important-line half of Kaiben's `log_digest.py`).
+ *
+ * Caveman's cheap literal pre-filter first (most lines contain none of the
+ * literals, so the regex runs on few lines), then the regex; JSON lines are
+ * judged by their event name and state fields, not by words inside key names.
+ */
+import { TEMPLATE_CHARS, mask } from "./mask.js";
+export const LONG_LINE_CHARS = 4000;
+const IMPORTANT_LITERALS = [
+    "error", "fail", "traceback", "exception", "refused", "denied", "timeout", "timed out", "warn",
+    "fatal", "panic", "crit", "killed", "out of memory", "oom", "no space", "unreachable", "captcha",
+    "forbidden", "too many requests", "403", "429", "50", "scheduled restart", "abort", "blocked",
+];
+const IMPORTANT_TEXT = /\b(error|errors|fatal|panic|exception|traceback|fail|failed|failure|warn|warning|crit|critical|alert|emerg|refused|denied|timeout|timed out|unreachable|killed|oom-kill|out of memory|no space left|captcha|forbidden|too many requests|aborted|blocked)\b|\bHTTP(?: Error)?[ /:](?:403|429|5\d\d)\b|\bhttp[=: ](?:403|429|5\d\d)\b|Scheduled restart job|status=\d+\/[A-Z]+/i;
+// Event names are judged by whole words (camelCase and snake_case split):
+// "newsroom" must not match "oom".
+const IMPORTANT_EVENT_WORDS = new Set([
+    "fail", "failed", "failure", "error", "errors", "exception", "traceback", "refused", "denied",
+    "blocked", "captcha", "timeout", "timed", "exhausted", "fatal", "panic", "crash", "crashed", "oom",
+    "killed", "abort", "aborted", "limited", "throttled", "warn", "warning", "retry", "stalled",
+    "skipped",
+]);
+const IMPORTANT_COMPOUND_EVENTS = new Set(["circuit_open", "needs_human", "stop_marker"]);
+const EVENT_WORD = /[A-Z]?[a-z]+|[A-Z]+(?![a-z])|\d+/g;
+const BAD_STATE = /^(fail|failed|failure|error|errored|fatal|blocked|refused|timeout|timed_out|crash|crashed|killed|aborted|needs_human|stopped)$/i;
+const STATE_FIELDS = ["status", "outcome", "result", "level", "verdict", "state"];
+/**
+ * Appended to a JSON line's template when the line says `"level": "WARNING"`.
+ * The classifier strips it before matching patterns, and an otherwise unknown
+ * template with it is counted (WARNING), not escalated.
+ */
+export const DECLARED_WARNING = " level=WARNING";
+const EXPLAIN_FIELDS = [
+    "status", "outcome", "result", "http", "code", "step", "what", "reason", "error", "exception",
+    "message", "msg", "detail",
+];
+const META_KEYS = new Set(["ts", "at", "time", "timestamp", "level", "t"]);
+/** True when the line contains a trouble word (literal pre-filter, then the regex). */
+export function looksImportant(text) {
+    const lower = text.toLowerCase();
+    if (!IMPORTANT_LITERALS.some((literal) => lower.includes(literal)))
+        return false;
+    return IMPORTANT_TEXT.test(text);
+}
+/** The end index (exclusive) of the JSON object starting at `start`, or -1 when it never closes. */
+function objectEnd(text, start) {
+    let depth = 0;
+    let inString = false;
+    for (let index = start; index < text.length; index += 1) {
+        const char = text[index];
+        if (inString) {
+            if (char === "\\")
+                index += 1;
+            else if (char === '"')
+                inString = false;
+            continue;
+        }
+        if (char === '"')
+            inString = true;
+        else if (char === "{")
+            depth += 1;
+        else if (char === "}") {
+            depth -= 1;
+            if (depth === 0)
+                return index + 1;
+        }
+    }
+    return -1;
+}
+/**
+ * Every JSON object on one line: unbuffered writers sometimes glue two records
+ * together (`{...}{...}`). Null when the line is not entirely JSON objects.
+ */
+export function jsonRecords(text) {
+    const records = [];
+    let position = 0;
+    while (position < text.length) {
+        if (text[position] !== "{")
+            return null;
+        const end = objectEnd(text, position);
+        if (end < 0)
+            return null;
+        try {
+            const data = JSON.parse(text.slice(position, end));
+            if (data === null || typeof data !== "object" || Array.isArray(data))
+                return null;
+            records.push(data);
+        }
+        catch {
+            return null;
+        }
+        position = end;
+        while (position < text.length && " \t,".includes(text[position]))
+            position += 1;
+    }
+    return records.length > 0 ? records : null;
+}
+/** `rs-gap@caseys.service` becomes (`rs-gap@*`, `caseys`); `kaiben-shard.service` becomes (`kaiben-shard`, null). */
+export function unitGroup(unit) {
+    let name = unit || "?";
+    if (name.endsWith(".service"))
+        name = name.slice(0, -".service".length);
+    const at = name.indexOf("@");
+    if (at >= 0)
+        return [`${name.slice(0, at)}@*`, name.slice(at + 1)];
+    return [name, null];
+}
+/** The event name of one JSON log line: its "event" field, else its first key that is not a timestamp. */
+export function jsonEvent(data) {
+    const event = data["event"];
+    if (typeof event === "string" && event)
+        return [event, false];
+    for (const key of Object.keys(data)) {
+        if (!META_KEYS.has(key))
+            return [key, true];
+    }
+    return ["json", false];
+}
+function shortValue(value) {
+    if (typeof value === "boolean" || value === null || value === undefined)
+        return JSON.stringify(value ?? null);
+    if (typeof value === "number") {
+        if (Number.isInteger(value) && value >= 100 && value <= 599)
+            return String(value); // an HTTP-like status
+        return Number.isInteger(value) && value >= -9 && value <= 9 ? String(value) : "#";
+    }
+    if (Array.isArray(value))
+        return "<list>";
+    if (typeof value === "object")
+        return "<dict>";
+    return mask(value);
+}
+function isEmptyError(value) {
+    if (value === null || value === undefined || value === "" || value === 0 || value === false)
+        return true;
+    if (Array.isArray(value))
+        return value.length === 0;
+    if (typeof value === "object")
+        return Object.keys(value).length === 0;
+    return false;
+}
+/** (event, important, template, subject) for one JSON log line. */
+export function judgeJson(data) {
+    const [event, eventIsKey] = jsonEvent(data);
+    const subjectValue = eventIsKey ? data[event] : (data["kind"] || data["store"]);
+    const words = new Set((event.match(EVENT_WORD) ?? []).map((word) => word.toLowerCase()));
+    let important = [...words].some((word) => IMPORTANT_EVENT_WORDS.has(word)) || IMPORTANT_COMPOUND_EVENTS.has(event);
+    for (const field of STATE_FIELDS) {
+        const value = data[field];
+        if (typeof value === "string" && BAD_STATE.test(value))
+            important = true;
+    }
+    for (const field of ["http", "code", "status"]) {
+        const value = data[field];
+        if (typeof value === "number" && Number.isInteger(value) && (value === 403 || value === 429 || (value >= 500 && value <= 599))) {
+            important = true;
+        }
+    }
+    if (!isEmptyError(data["error"]))
+        important = true;
+    if (!important) {
+        const subject = typeof subjectValue === "string" && Array.from(subjectValue).length <= 24 ? subjectValue : null;
+        return { event, important: false, template: null, subject };
+    }
+    const parts = [event];
+    if (eventIsKey && typeof subjectValue === "string" && Array.from(subjectValue).length <= 40)
+        parts.push(mask(subjectValue));
+    for (const field of EXPLAIN_FIELDS) {
+        const value = data[field];
+        if (field in data && field !== event && value !== null && value !== undefined && value !== "") {
+            parts.push(`${field}=${shortValue(value)}`);
+        }
+    }
+    let template = Array.from(parts.join(" ")).slice(0, TEMPLATE_CHARS - DECLARED_WARNING.length).join("");
+    const level = String(data["level"] ?? "").trim().toLowerCase();
+    if (level === "warning" || level === "warn")
+        template += DECLARED_WARNING;
+    return { event, important: true, template, subject: null };
+}
+/**
+ * `json.dumps` as Python writes it (`", "` and `": "` separators, non-ASCII as
+ * `\uXXXX`), so a stored example matches the Python digest byte for byte. A
+ * float with a zero fraction prints as an integer here; JS cannot tell them apart.
+ */
+export function pythonJson(value) {
+    if (value === null || value === undefined)
+        return "null";
+    if (typeof value === "string")
+        return JSON.stringify(value).replace(/[\u007f-\uffff]/g, (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`);
+    if (typeof value === "number" || typeof value === "boolean")
+        return String(value);
+    if (Array.isArray(value))
+        return `[${value.map(pythonJson).join(", ")}]`;
+    const entries = Object.entries(value).map(([key, item]) => `${pythonJson(key)}: ${pythonJson(item)}`);
+    return `{${entries.join(", ")}}`;
+}

package/dist/mask.d.ts

@@ -0,0 +1,12 @@
+export declare const TEMPLATE_CHARS = 160;
+export declare const EXAMPLE_CHARS = 200;
+/** First `limit` code points, like Python's `text[:limit]`. */
+export declare function cut(text: string, limit: number): string;
+/** Python's `" ".join(text.split())`: whitespace runs become one space. */
+export declare function squash(text: string): string;
+/** Tokens and keys masked; the rest untouched. Applied to every example before it is stored or printed. */
+export declare function redact(text: string): string;
+/** The template of one message: every variable part replaced by a placeholder, then cut to 160 characters. */
+export declare function mask(input: unknown): string;
+/** Eight hex characters of sha1(unit group, event, template). Equal problems on two machines share a tid. */
+export declare function templateId(unitGroup: string, event: string, template: string): string;

package/dist/mask.js

@@ -0,0 +1,106 @@
+/**
+ * Masking: the template of one log message (port of Kaiben's `log_digest.mask`).
+ *
+ * Plain words: every part of a line that changes from one occurrence to the next
+ * (an address, an id, a time, a number) becomes a placeholder, so two lines that
+ * say the same thing become the same template and can be counted instead of read.
+ *
+ * Fixed masking order, so a memory address can never fake a status code
+ * (`0x7f...403...`): memory addresses, URLs, UUIDs, timestamps, IPs, long hex,
+ * long ids, absolute paths, long or numeric quoted strings, then every other
+ * number except a status code in a status position (`HTTP 403`, `status=500`,
+ * `[Errno 28]`).
+ *
+ * Known difference from the Python original: `\b`, `\w` and `\d` here are ASCII
+ * only, Python's are Unicode. A template next to a non-ASCII letter can differ.
+ */
+import { createHash } from "node:crypto";
+export const TEMPLATE_CHARS = 160;
+export const EXAMPLE_CHARS = 200;
+const ANSI = /\x1b\[[0-9;]*[A-Za-z]/g;
+const SECRET_PARAMETER = /\b(token|key|apikey|api_key|secret|signature|password|sig|x-amz-[a-z-]+)=([^&\s"']+)/gi;
+const BEARER = /(bearer\s+)\S+/gi;
+// A secret written as a JSON field, which the `name=value` rule cannot see.
+const SECRET_JSON_FIELD = /("(?:token|secret|password|passwd|api_key|apikey|authorization|access_token|refresh_token|client_secret)"\s*:\s*")[^"]*/gi;
+// Token-shaped strings by their well-known prefixes, wherever they appear.
+const TOKEN_SHAPES = [
+    /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}/g, // JWT
+    /\b(?:sk|pk|rk)-[A-Za-z0-9_-]{16,}/g, // OpenAI-style keys
+    /\bgh[pousr]_[A-Za-z0-9]{20,}/g, // GitHub tokens
+    /\bgithub_pat_[A-Za-z0-9_]{20,}/g,
+    /\bsb_(?:secret|publishable)_[A-Za-z0-9_-]{8,}/g, // Supabase keys
+    /\bxox[abprs]-[A-Za-z0-9-]{10,}/g, // Slack tokens
+    /\bAKIA[0-9A-Z]{16}\b/g, // AWS access key id
+    /\bnpm_[A-Za-z0-9]{30,}/g,
+];
+const MEMORY_ADDRESS = /\b0x[0-9a-fA-F]+\b/g;
+const URL_PATTERN = /\b[a-zA-Z][a-zA-Z0-9+.-]*:\/\/([^/\s"'<>?#:]+)[^\s"'<>]*/g;
+const UUID = /\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g;
+const TIMESTAMP = /\b\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:[.,]\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?\b|\b\d{2}:\d{2}:\d{2}(?:[.,]\d+)?\b/g;
+const IP_ADDRESS = /\b\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?\b/g;
+const LONG_HEX = /\b(?=[0-9a-fA-F]*[0-9])(?=[0-9a-fA-F]*[a-fA-F])[0-9a-fA-F]{8,}\b/g;
+// 4+ digits: an id, not a unit name.
+const LONG_ID = /\b(?=(?:[A-Za-z_-]*\d){4})(?=[A-Za-z0-9_-]*[A-Za-z])[A-Za-z0-9_-]{20,}={0,2}/g;
+// A ':' or ',' inside a segment, never a trailing one.
+const PATH = /(?<![\w.<>/])(?:\/(?:[\w.@%+~-]|[:,](?=\w))+){2,}\/?/g;
+const QUOTED = /'([^'\n]{0,300})'|"([^"\n]{0,300})"/g;
+const NUMBER = /(?<![A-Za-z_\d#])\d+(?:[.,]\d+)*/g;
+// A number right after one of these is a status, not a quantity: kept so
+// "HTTP 403" and "HTTP 404" stay different templates.
+const STATUS_BEFORE = /(?:\bHTTP(?: Error)?[ /:]|\bHTTP\/\d\.\d |\bhttp[=: ]|\bstatus[=: ]|\bcode[=: ]|\[Errno |\bexit(?: code| status)? |\bsignal |\bresponse |\breturned )$/i;
+// The longest alternative above is 12 characters, so 17 characters of context
+// (one more than Python's 16, so `\b` sees the real previous character) decide it.
+const STATUS_WINDOW = 17;
+/** First `limit` code points, like Python's `text[:limit]`. */
+export function cut(text, limit) {
+    if (text.length <= limit)
+        return text;
+    return Array.from(text).slice(0, limit).join("");
+}
+/** Python's `" ".join(text.split())`: whitespace runs become one space. */
+export function squash(text) {
+    return text.split(/\s+/).filter(Boolean).join(" ");
+}
+/** Tokens and keys masked; the rest untouched. Applied to every example before it is stored or printed. */
+export function redact(text) {
+    let out = text.replace(BEARER, "$1<redacted>").replace(SECRET_PARAMETER, "$1=<redacted>");
+    out = out.replace(SECRET_JSON_FIELD, "$1<redacted>");
+    for (const shape of TOKEN_SHAPES)
+        out = out.replace(shape, "<redacted>");
+    return out;
+}
+function keepStatusNumbers(match, offset, whole) {
+    const before = whole.slice(Math.max(0, offset - STATUS_WINDOW), offset);
+    return STATUS_BEFORE.test(before) ? match : "#";
+}
+function maskPath(path) {
+    const name = path.replace(/\/+$/, "").split("/").pop() ?? "";
+    return name.endsWith(".py") ? `<path>/${name}` : "<path>";
+}
+function maskQuoted(match, single, double) {
+    const inner = single !== undefined ? single : (double ?? "");
+    const quote = single !== undefined ? "'" : '"';
+    if (Array.from(inner).length > 40 || /\d|\/|</.test(inner))
+        return `${quote}<q>${quote}`;
+    return match;
+}
+/** The template of one message: every variable part replaced by a placeholder, then cut to 160 characters. */
+export function mask(input) {
+    let text = String(input).replace(ANSI, "");
+    text = redact(text);
+    text = text.replace(MEMORY_ADDRESS, "<addr>");
+    text = text.replace(URL_PATTERN, (_m, host) => `<url:${host.toLowerCase()}>`);
+    text = text.replace(UUID, "<uuid>");
+    text = text.replace(TIMESTAMP, "<ts>");
+    text = text.replace(IP_ADDRESS, "<ip>");
+    text = text.replace(LONG_HEX, "<hex>");
+    text = text.replace(LONG_ID, "<id>");
+    text = text.replace(PATH, maskPath);
+    text = text.replace(QUOTED, maskQuoted);
+    text = text.replace(NUMBER, keepStatusNumbers);
+    return cut(squash(text), TEMPLATE_CHARS);
+}
+/** Eight hex characters of sha1(unit group, event, template). Equal problems on two machines share a tid. */
+export function templateId(unitGroup, event, template) {
+    return createHash("sha1").update(`${unitGroup}\x1f${event}\x1f${template}`, "utf8").digest("hex").slice(0, 8);
+}

package/dist/merge.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Several digest results as one (port of Kaiben's `merge_results`): counts
+ * summed, times widened, instance sets united. The next slice, a fleet view of
+ * every machine's collector digest, merges per-machine results with this.
+ */
+import { type DigestGroup, type DigestResult } from "./digest.js";
+/** A result whose groups may be a list (probe output) or keyed by tid (a stored bucket). */
+export type MergeInput = Omit<Partial<DigestResult>, "groups"> & {
+    groups?: DigestGroup[] | Record<string, DigestGroup>;
+};
+export type MergedResult = DigestResult;
+export declare function mergeResults(results: readonly MergeInput[], registry?: Record<string, Partial<DigestGroup>>): MergedResult;