@bli-cockpit/log-digest 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,31 @@
-# Temporary Holding Version
+# @bli-cockpit/log-digest
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+A token-cheap digest of log lines, shared by `cockpit logs` and the `logs_digest` MCP tool.
+Node standard library only, no runtime dependencies.
+
+Plain words: a busy log is tens of thousands of lines and almost all of it is routine. This keeps the
+lines that can mean trouble, masks their variable parts (addresses, ids, times, numbers) into a
+**template**, counts equal templates with first and last time and one example, turns routine runs into
+counts, and **classifies** every template so harmless ones are counted and never alerted.
+
+It is a TypeScript port of Kaiben's `log_digest.py` and `operator_classify.py` with the same output
+contract: the masking order is fixed (memory addresses, URLs, UUIDs, timestamps, IPs, long hex, long ids,
+paths, quoted strings, numbers except a status in a status position), a template id is the first 8 hex of
+`sha1(unit group, event, template)`, output is deterministic and order-stable, and token-shaped strings are
+redacted from every example. `src/python-contract.test.ts` pins it to the Python on Kaiben's real fixture.
+
+```ts
+import { digestSource } from "@bli-cockpit/log-digest";
+
+const outcome = await digestSource({ source: "railway:ladder", since: "1h" }, { stateDir });
+if (outcome.ok && outcome.kind === "digest") console.log(outcome.text); // about 2 KB, our_bug and unknown first
+else if (!outcome.ok) console.error(outcome.reason, outcome.detail);    // a named refusal, never a stack trace
+```
+
+Sources: `collector`, `dashboard` (Vercel), `railway`, `railway:<service>`, `-` (stdin text).
+Every source is read-only. How to run it, the source inventory, what each source cannot see and how to
+teach the class table a new pattern: `docs/runbooks/tower-logs.md` in the repository.
+
+Known differences from the Python original: `\b`, `\w` and `\d` are ASCII only here (Python's are
+Unicode); the circuit-breaker facts that read Kaiben's own deploy modules are not ported (`breaker` stays
+`{}`); and `taggedLines` (on for Tower, off by default) judges `[tag] sentence {json}` lines by their parts.

package/dist/classify.d.ts

@@ -0,0 +1,48 @@
+export type Severity = "ERROR" | "hiccup" | "WARNING";
+export interface ClassInfo {
+    severity: Severity;
+    about: string;
+}
+export interface TablePattern {
+    id: string;
+    class: string;
+    /** Regex source on the masked template. */
+    match: string;
+    /** Regex source on the unit group. */
+    unit?: string;
+    /** Regex source on the source name (`dashboard`, `railway:ladder`, `collector`...). */
+    source?: string;
+    /** Overrides the class's severity (a hiccup whose retries ran out is a WARNING). */
+    severity?: Severity;
+    note: string;
+    /** Where the verdict came from: a ticket, a commit, a measured window. */
+    basis: string;
+}
+export interface ClassTableData {
+    version: number;
+    classes: Record<string, ClassInfo>;
+    patterns: readonly TablePattern[];
+    /** Regexes on a unit name; a match drops the line and counts it as noise. */
+    noiseUnits?: readonly string[];
+}
+export interface Verdict {
+    class: string;
+    severity: Severity;
+    pattern: string | null;
+    note: string;
+}
+export type Classify = (unit: string, template: string, source?: string) => Verdict;
+export declare class ClassTable {
+    readonly classes: Record<string, ClassInfo>;
+    readonly noiseUnits: readonly string[];
+    private readonly patterns;
+    private readonly cache;
+    constructor(data: ClassTableData);
+    /** A table read from its JSON text (the `--table` file a person writes to try a pattern). */
+    static fromJson(text: string): ClassTable;
+    classify: Classify;
+    private decide;
+    private verdict;
+}
+/** Display and sort order: what needs a person first, what is counted last. */
+export declare const CLASS_ORDER: Record<string, number>;

package/dist/classify.js

@@ -0,0 +1,81 @@
+/**
+ * The error table, applied (port of Kaiben's `operator_classify.ClassTable`).
+ *
+ * A table maps a masked log template to a class, and a class to a severity word:
+ *
+ *   ERROR    our_bug, config_drift, capacity, unknown   something we must fix
+ *   hiccup   hiccup, rate_pushback                       retried with back-off
+ *   WARNING  dead_source, benign                         the world changed, nothing to fix: counted, never alerted
+ *
+ * A template no pattern matches is class `unknown`, so a new kind of problem
+ * always reaches someone. First match wins: specific patterns sit above generic
+ * ones. A pattern's `match` is a JavaScript regex on the MASKED template; the
+ * optional `unit` regex narrows it to a unit group (a Vercel route family, a
+ * Railway service, a collector tag) and `source` to one log source.
+ */
+import { DECLARED_WARNING } from "./judge.js";
+export class ClassTable {
+    classes;
+    noiseUnits;
+    patterns;
+    cache = new Map();
+    constructor(data) {
+        if (!data.classes["unknown"])
+            throw new Error("a class table needs an `unknown` class: it is where a new problem lands");
+        this.classes = data.classes;
+        this.noiseUnits = data.noiseUnits ?? [];
+        this.patterns = data.patterns.map((spec) => {
+            if (!data.classes[spec.class])
+                throw new Error(`pattern ${spec.id} names the unknown class ${spec.class}`);
+            return {
+                spec,
+                match: new RegExp(spec.match),
+                unit: spec.unit ? new RegExp(spec.unit) : null,
+                source: spec.source ? new RegExp(spec.source) : null,
+            };
+        });
+    }
+    /** A table read from its JSON text (the `--table` file a person writes to try a pattern). */
+    static fromJson(text) {
+        return new ClassTable(JSON.parse(text));
+    }
+    classify = (unit, template, source) => {
+        const key = `${source ?? ""}\x1f${unit}\x1f${template}`;
+        let verdict = this.cache.get(key);
+        if (!verdict) {
+            verdict = this.decide(unit || "", template || "", source || "");
+            this.cache.set(key, verdict);
+        }
+        return verdict;
+    };
+    decide(unit, template, source) {
+        const declaredWarning = template.endsWith(DECLARED_WARNING);
+        const bare = declaredWarning ? template.slice(0, -DECLARED_WARNING.length) : template; // patterns see the template before the suffix
+        for (const pattern of this.patterns) {
+            if (pattern.unit && !pattern.unit.test(unit))
+                continue;
+            if (pattern.source && !pattern.source.test(source))
+                continue;
+            if (pattern.match.test(bare))
+                return this.verdict(pattern.spec.class, pattern.spec);
+        }
+        if (declaredWarning) {
+            // The unit logged this as WARNING on purpose and no pattern says otherwise: count it, never escalate.
+            return { class: "unknown", severity: "WARNING", pattern: "declared-warning", note: "the unit logged it with level WARNING; counted, not escalated" };
+        }
+        return this.verdict("unknown", null);
+    }
+    verdict(name, pattern) {
+        const info = this.classes[name] ?? this.classes["unknown"];
+        return {
+            class: name,
+            severity: pattern?.severity ?? info.severity,
+            pattern: pattern ? pattern.id : null,
+            note: pattern?.note ?? "",
+        };
+    }
+}
+/** Display and sort order: what needs a person first, what is counted last. */
+export const CLASS_ORDER = {
+    unknown: 0, our_bug: 1, config_drift: 2, capacity: 3, rate_pushback: 4, hiccup: 5, transient: 5, dead_source: 6, benign: 7,
+};

package/dist/digest-source.d.ts

@@ -0,0 +1,99 @@
+/**
+ * One call that does the whole job for any source: read it, digest it, classify
+ * it, render it. The CLI (`cockpit logs`) and the MCP tool (`logs_digest`) are
+ * both thin callers of `digestSource`, so they print the same thing.
+ */
+import { ClassTable } from "./classify.js";
+import { type RoutineEvents } from "./digest.js";
+import { type LogFn, type RunCli, type SourceContext } from "./sources/types.js";
+export declare const SOURCE_NAMES: readonly ["collector", "dashboard", "railway", "railway:<service>", "-"];
+/** A Railway service name: validated before anything is spawned, because a Windows run goes through cmd.exe. */
+export declare const SERVICE_NAME: RegExp;
+export declare const DEFAULT_MAX_BYTES = 2048;
+export interface DigestRequest {
+    /** `collector`, `dashboard`, `railway`, `railway:<service>` or `-`. */
+    source: string;
+    /** `15m`, `1h`, `2d` or an ISO time. Default 15m. */
+    since?: string;
+    /** A template id: print the original lines behind it instead of the digest. */
+    raw?: string;
+    /** How many original lines `raw` prints (newest last). */
+    rawLimit?: number;
+    /** The class table; the Tower table by default. */
+    table?: ClassTable;
+    /** Ceiling on the printed text. About 2 KB by default. */
+    maxBytes?: number;
+    /** `dashboard` only: read info-only requests too (slow). */
+    allLevels?: boolean;
+}
+export interface DigestEnvironment {
+    /** Where the collector keeps its state on this machine. */
+    stateDir?: string;
+    stdinText?: string;
+    stdinIsTerminal?: boolean;
+    run?: RunCli;
+    log?: LogFn;
+    nowMs?: number;
+    limits?: SourceContext["limits"];
+}
+export interface DigestTemplate {
+    tid: string;
+    unit: string;
+    class: string;
+    severity: string;
+    pattern: string | null;
+    count: number;
+    first: number | null;
+    last: number | null;
+    template: string;
+    example: string;
+    instances: string[];
+}
+export interface DigestAnswer {
+    ok: true;
+    kind: "digest";
+    source: string;
+    window: {
+        from: string;
+        to: string;
+        label: string;
+    };
+    lines: number;
+    noise: number;
+    notes: string[];
+    templates: DigestTemplate[];
+    routine: Record<string, RoutineEvents>;
+    /** The capped human text, the same thing `cockpit logs` prints. */
+    text: string;
+    /** What the digest saved: characters of the log lines read vs characters printed (tokens are about chars / 4). */
+    measure: {
+        raw_chars: number;
+        digest_chars: number;
+        basis: "chars/4";
+    };
+}
+export interface RawAnswer {
+    ok: true;
+    kind: "raw";
+    source: string;
+    tid: string;
+    window: {
+        from: string;
+        to: string;
+        label: string;
+    };
+    matched: number;
+    lines: string[];
+    notes: string[];
+    text: string;
+}
+export interface Refusal {
+    ok: false;
+    reason: string;
+    detail: string;
+}
+export type DigestOutcome = DigestAnswer | RawAnswer | Refusal;
+/** `railway:ladder` becomes `railway`: the source family a table pattern's `source` regex is matched against. */
+export declare function sourceFamily(source: string): string;
+/** Reads, digests, classifies and renders one source. Never throws for a refusal: it is returned, named. */
+export declare function digestSource(request: DigestRequest, env?: DigestEnvironment): Promise<DigestOutcome>;

package/dist/digest-source.js

@@ -0,0 +1,130 @@
+/**
+ * One call that does the whole job for any source: read it, digest it, classify
+ * it, render it. The CLI (`cockpit logs`) and the MCP tool (`logs_digest`) are
+ * both thin callers of `digestSource`, so they print the same thing.
+ */
+import { CLASS_ORDER } from "./classify.js";
+import { Digest } from "./digest.js";
+import { redact, squash, cut } from "./mask.js";
+import { renderCapped } from "./render.js";
+import { nodeRunCli } from "./sources/process.js";
+import { readCollector } from "./sources/collector.js";
+import { readRailway } from "./sources/railway.js";
+import { readStdin } from "./sources/stdin.js";
+import { readDashboard } from "./sources/vercel.js";
+import { SourceRefusal } from "./sources/types.js";
+import { towerClassTable } from "./tower-classes.js";
+import { WindowError, parseWindow } from "./window.js";
+export const SOURCE_NAMES = ["collector", "dashboard", "railway", "railway:<service>", "-"];
+/** A Railway service name: validated before anything is spawned, because a Windows run goes through cmd.exe. */
+export const SERVICE_NAME = /^railway:([A-Za-z0-9][A-Za-z0-9._-]{0,63})$/;
+export const DEFAULT_MAX_BYTES = 2048;
+const RAW_LINE_CHARS = 400;
+const RAW_DEFAULT_LIMIT = 20;
+const defaultLog = (message, fields) => console.error(message, JSON.stringify(fields));
+/** `railway:ladder` becomes `railway`: the source family a table pattern's `source` regex is matched against. */
+export function sourceFamily(source) {
+    return source.startsWith("railway:") ? "railway" : source;
+}
+function reader(source) {
+    if (source === "collector")
+        return readCollector;
+    if (source === "dashboard")
+        return readDashboard;
+    if (source === "-")
+        return readStdin;
+    if (source === "railway")
+        return (context) => readRailway(context);
+    const service = SERVICE_NAME.exec(source)?.[1];
+    if (service)
+        return (context) => readRailway(context, service);
+    return null;
+}
+const iso = (epoch) => new Date(epoch * 1000).toISOString();
+function windowFacts(window) {
+    return { from: iso(window.fromEpoch), to: iso(window.toEpoch), label: window.label };
+}
+function templates(result, table, family) {
+    const rows = result.groups.map((record) => {
+        const verdict = table.classify(record.unit, record.template, family);
+        return {
+            tid: record.tid, unit: record.unit, class: verdict.class, severity: verdict.severity, pattern: verdict.pattern,
+            count: record.count, first: record.first, last: record.last, template: record.template, example: record.example, instances: record.instances,
+        };
+    });
+    // What needs a person first, then by count: the same order the text view uses.
+    return rows.sort((a, b) => (CLASS_ORDER[a.class] ?? 0) - (CLASS_ORDER[b.class] ?? 0) || b.count - a.count || (a.tid < b.tid ? -1 : 1));
+}
+/** Reads, digests, classifies and renders one source. Never throws for a refusal: it is returned, named. */
+export async function digestSource(request, env = {}) {
+    const log = env.log ?? defaultLog;
+    const family = sourceFamily(request.source);
+    const read = reader(request.source);
+    if (!read && request.source.startsWith("railway:")) {
+        log("[logs digest] refused", { reason: "invalid_service_name", source_family: "railway" });
+        return { ok: false, reason: "invalid_service_name", detail: "a Railway service name is letters, digits, dot, dash and underscore, starting with a letter or digit, at most 64 characters" };
+    }
+    if (!read) {
+        log("[logs digest] refused", { reason: "unknown_source", source_family: family });
+        return { ok: false, reason: "unknown_source", detail: `source must be one of ${SOURCE_NAMES.join(", ")}; got ${JSON.stringify(request.source.slice(0, 40))}` };
+    }
+    let window;
+    try {
+        window = parseWindow(request.since, env.nowMs);
+    }
+    catch (error) {
+        if (!(error instanceof WindowError))
+            throw error;
+        log("[logs digest] refused", { reason: "invalid_since", source: family });
+        return { ok: false, reason: error.reason, detail: error.message };
+    }
+    if (request.raw !== undefined && !/^[0-9a-f]{8}$/.test(request.raw)) {
+        return { ok: false, reason: "invalid_template_id", detail: "--raw wants the 8 hex characters a digest line starts with" };
+    }
+    const table = request.table ?? towerClassTable();
+    const context = {
+        window, run: env.run ?? nodeRunCli, log, stateDir: env.stateDir, stdinText: env.stdinText, stdinIsTerminal: env.stdinIsTerminal, limits: env.limits, allLevels: request.allLevels,
+    };
+    let source;
+    try {
+        source = await read(context);
+    }
+    catch (error) {
+        if (error instanceof SourceRefusal) {
+            log("[logs digest] refused", { reason: error.reason, source: family });
+            return { ok: false, reason: error.reason, detail: error.detail };
+        }
+        throw error;
+    }
+    const digest = new Digest({ noise: table.noiseUnits, taggedLines: true });
+    const raw = [];
+    const wanted = request.raw;
+    let matched = 0;
+    for (const line of source.lines) {
+        const before = wanted ? digest.groups.get(wanted)?.count ?? 0 : 0;
+        digest.addLine(line.unit, line.message, line.ts, { level: line.level });
+        if (wanted && (digest.groups.get(wanted)?.count ?? 0) !== before) {
+            matched += 1;
+            const stamp = line.ts ? `${new Date(line.ts * 1000).toISOString().slice(11, 19)}Z ` : "";
+            raw.push(`${stamp}${line.unit}: ${cut(redact(squash(line.message)), RAW_LINE_CHARS)}`);
+        }
+    }
+    const result = digest.result();
+    const facts = windowFacts(window);
+    if (wanted) {
+        const limit = request.rawLimit ?? RAW_DEFAULT_LIMIT;
+        const lines = raw.slice(-limit);
+        log("[logs digest] raw lines", { source: family, tid: wanted, matched, printed: lines.length });
+        if (matched === 0) {
+            return { ok: false, reason: "template_not_found", detail: `no line in this ${request.source} window (${window.label}) has template id ${wanted}; use the same source and --since as the digest that printed it` };
+        }
+        const head = `RAW ${wanted} ${request.source} ${window.label}: ${matched} line${matched === 1 ? "" : "s"}${matched > lines.length ? `, newest ${lines.length}` : ""}`;
+        return { ok: true, kind: "raw", source: request.source, tid: wanted, window: facts, matched, lines, notes: source.notes, text: [head, ...lines].join("\n") };
+    }
+    const text = renderCapped(result, { source: request.source, window: window.label, classify: (unit, template) => table.classify(unit, template, family), maxBytes: request.maxBytes ?? DEFAULT_MAX_BYTES, notes: source.notes });
+    const all = templates(result, table, family);
+    const rawChars = source.rawChars ?? source.lines.reduce((sum, line) => sum + line.message.length + 1, 0);
+    const errors = all.filter((entry) => entry.severity === "ERROR").length;
+    log("[logs digest] digested", { source: family, window: window.label, lines: result.lines, noise: result.noise, templates: all.length, error_templates: errors, printed_bytes: Buffer.byteLength(text, "utf8"), raw_chars: rawChars });
+    return { ok: true, kind: "digest", source: request.source, window: facts, lines: result.lines, noise: result.noise, notes: source.notes, templates: all, routine: result.routine, text, measure: { raw_chars: rawChars, digest_chars: text.length, basis: "chars/4" } };
+}

package/dist/digest.d.ts

@@ -0,0 +1,81 @@
+export declare const SUBJECTS_TRACKED = 6;
+/** Units that only ever say "someone logged in" or "a package timer ran": dropped and counted as noise. */
+export declare const DEFAULT_NOISE: readonly string[];
+/** One `journalctl -o json` entry (the contract Kaiben's digest takes), or any source shaped like it. */
+export interface JournalEntry {
+    MESSAGE?: unknown;
+    __CURSOR?: string;
+    __REALTIME_TIMESTAMP?: string | number;
+    SYSLOG_IDENTIFIER?: string;
+    UNIT?: string;
+    _SYSTEMD_UNIT?: string;
+    _TRANSPORT?: string;
+}
+export interface DigestGroup {
+    tid: string;
+    unit: string;
+    event: string;
+    template: string;
+    count: number;
+    first: number | null;
+    last: number | null;
+    example: string;
+    instances: string[];
+}
+/** event -> [count, subject -> count] */
+export type RoutineEvents = Record<string, [number, Record<string, number>]>;
+export interface DigestResult {
+    lines: number;
+    noise: number;
+    span: [number | null, number | null];
+    cursors: [string | null, string | null];
+    groups: DigestGroup[];
+    routine: Record<string, RoutineEvents>;
+    breaker: Record<string, never>;
+}
+export interface DigestOptions {
+    /** Regexes (source text) on a unit or identifier name; a match drops the line and counts it as noise. */
+    noise?: readonly string[];
+    /** Units whose FLAG/NOTE lines and JSON records are relays of another log: counted, never judged. */
+    routineUnits?: readonly string[];
+    /** Units whose own output is dropped while systemd's lines about them are kept. */
+    quietUnits?: readonly string[];
+    /**
+     * Judge `[tag] sentence {json}` lines (Tower's log shape) by their parts instead of by words anywhere in the line.
+     * Off by default so Kaiben's journals digest exactly as the Python does.
+     */
+    taggedLines?: boolean;
+}
+export interface LineFacts {
+    /** systemd's own words about a unit rather than the unit's output. */
+    manager?: boolean;
+    /** The source's own severity word (`error`, `warn`, `info`...). Only `warn` on a tagged line counts: see tagged.ts. */
+    level?: string;
+}
+export declare class Digest {
+    readonly groups: Map<string, DigestGroup>;
+    readonly routine: Map<string, Map<string, [number, Record<string, number>]>>;
+    lines: number;
+    noiseLines: number;
+    firstTs: number | null;
+    lastTs: number | null;
+    firstCursor: string | null;
+    lastCursor: string | null;
+    private readonly noise;
+    private readonly quiet;
+    private readonly routineUnits;
+    private readonly taggedLines;
+    private readonly openTracebacks;
+    constructor(options?: DigestOptions);
+    isNoise(unit: string | undefined, identifier: string | undefined): boolean;
+    /** One `journalctl -o json` entry. */
+    addEntry(entry: JournalEntry): void;
+    /** One log line from any source. `ts` is epoch seconds (0 when the source has none). */
+    addLine(unit: string, message: string, ts: number, facts?: LineFacts): void;
+    private routineCount;
+    private addGroup;
+    /** Python tracebacks span many lines: fold each into one group named by its exception and last frame. */
+    private foldTraceback;
+    /** The result, sorted so equal input renders byte-identically: groups by count then tid, routine by name. */
+    result(): DigestResult;
+}