@m1kad0/hannah-logging 0.0.0

package/README.md

@@ -0,0 +1,99 @@
+# @m1kad0/hannah-logging (TypeScript)
+
+Ships the logs of a Hannah component to the Hannah log collector, so a user can download
+one archive of all components' logs from the WebUI.
+
+It works with whatever the component logs with. The component's own output stays exactly
+as it is; the library additionally buffers a copy for the collector.
+
+## Installation
+
+```sh
+npm install @m1kad0/hannah-logging
+```
+
+`@m1kad0/hannah-proto` and `@grpc/grpc-js` are peer dependencies, so the library uses the
+component's own copies. Every call to Hannah Core has to carry the protocol version of the
+`hannah-proto` the component uses.
+
+## Usage
+
+Create the shipper as early as possible; everything from then on is buffered. Connect it
+once the config is loaded.
+
+```ts
+import { LogShipper, hookConsole, wrapLogger } from '@m1kad0/hannah-logging';
+
+const shipper = new LogShipper({
+    component: 'msteams',
+    version,
+    secrets: [config.botPassword],              // masked wherever they appear
+});
+
+// Pick the way in that fits the component:
+hookConsole(shipper);                           // console.log/info/warn/error/debug
+const log = wrapLogger(this.log, shipper);      // ioBroker's this.log, or anything with debug/info/warn/error
+myLogger.onLine(shipper.hook);                  // any other logger: hook(level, message, meta?)
+
+// Once the config is loaded:
+shipper.connect({ hannahAddress: 'localhost:50051' });   // optional: collectorAddress as fallback
+// On shutdown:
+await shipper.close();                          // sends what is still buffered (up to 2 s)
+```
+
+### The hook
+
+`shipper.hook(level, message, meta?)` takes one log line:
+
+- `level`: `'silly' | 'trace' | 'debug' | 'info' | 'log' | 'warn' | 'warning' | 'error' | 'fatal' | 'critical'`
+- `meta.logger`: module or subsystem name, used for `loggerCategories`
+- `meta.category`: `Category.GENERAL`, `Category.TRANSCRIPT` or `Category.METADATA`
+- `meta.timestamp`: Unix ms, defaults to now
+
+It is a bound function, so it can be handed to a pino hook or a winston transport as it is.
+
+### wrapLogger and hookConsole
+
+- `wrapLogger(target, shipper, meta?)` returns an object with the same interface as
+  `target`. Level method calls go to `target` unchanged and are shipped as well.
+  Everything else (for example ioBroker's `log.level`) passes through.
+- `hookConsole(shipper, meta?)` replaces `console.debug/info/log/warn/error` with versions
+  that print as before and ship as well. Arguments are formatted like Node does, so an
+  `Error` keeps its stack trace. It returns a function that restores the original console.
+
+Both report the library's own connection problems to the unwrapped logger or the original
+console. Those messages are never shipped.
+
+### Categories
+
+Each line is tagged `GENERAL`, `TRANSCRIPT` (speech transcripts, user utterances) or
+`METADATA` (room and device names, presence). Exports can leave out the last two, and the
+WebUI does so by default. Tag by logger name with
+`loggerCategories: { stt: Category.TRANSCRIPT }` (which also covers `stt.whisper`), or per
+line with `meta.category`.
+
+### Secrets
+
+Before a line leaves the process, the library masks:
+
+- values of keys such as `password`, `token`, `api_key`, `secret`, `psk`
+- `Bearer` tokens, JWTs, credentials in URLs (`mqtt://user:pw@host`), PEM private keys
+- Telegram bot tokens and well-known token prefixes (`glpat-`, `ghp_`, …)
+- every value in `secrets` or added later with `shipper.addSecret(…)`
+
+Extra regexes go in `secretPatterns`. The component's own output still shows the unmasked
+line.
+
+## Behaviour
+
+- **Buffer:** 4 MiB by default (`maxBufferBytes`). When it is full, the oldest lines are
+  dropped and reported to the collector as a gap. While no collector is known, the buffer
+  simply keeps running as a ring.
+- **Timestamps** are taken when a line is logged, not when it is sent.
+- **Discovery:** subscribes to Hannah Core's infrastructure announcements and follows the
+  log collector when it moves. If Core is unreachable, the last known collector is kept.
+  The static `collectorAddress` is used while none is announced.
+- **Never blocks the component:** logging only appends to the buffer. Sending is
+  asynchronous, connection problems are reported once and retried with backoff.
+- **Process exit:** an open connection keeps Node's event loop alive. Call
+  `shipper.close()` on shutdown.

package/dist/buffer.d.ts

@@ -0,0 +1,40 @@
+/** Bounded in-memory buffer between the hook and the shipper. */
+export interface Entry {
+    timestampMs: number;
+    level: number;
+    logger: string;
+    message: string;
+    category: number;
+}
+export interface Gap {
+    dropped: number;
+    fromMs: number;
+    toMs: number;
+}
+/**
+ * Ring buffer limited by size. When full, the oldest entries are dropped and recorded
+ * as a gap, which is shipped before the remaining entries.
+ */
+export declare class LogBuffer {
+    private readonly maxBytes;
+    private entries;
+    private head;
+    private bytes;
+    private gap;
+    private waiters;
+    constructor(maxBytes: number);
+    put(entry: Entry): void;
+    private dropOldest;
+    /** Removes and returns the pending gap (if any) and up to `maxItems` entries, oldest first. */
+    take(maxItems: number): {
+        gap: Gap | null;
+        entries: Entry[];
+    };
+    /** Resolves when data arrives, on `wake()`, or after `timeoutMs`. */
+    wait(timeoutMs: number): Promise<void>;
+    /** Wakes a pending `wait`, e.g. to react to a new collector address or shutdown. */
+    wake(): void;
+    get length(): number;
+    get empty(): boolean;
+    private compact;
+}

package/dist/buffer.js

@@ -0,0 +1,94 @@
+"use strict";
+/** Bounded in-memory buffer between the hook and the shipper. */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LogBuffer = void 0;
+// Rough per-entry overhead on top of message + logger name, so a flood of empty
+// messages still counts against the limit.
+const ENTRY_OVERHEAD = 64;
+function size(e) {
+    return e.message.length + e.logger.length + ENTRY_OVERHEAD;
+}
+/**
+ * Ring buffer limited by size. When full, the oldest entries are dropped and recorded
+ * as a gap, which is shipped before the remaining entries.
+ */
+class LogBuffer {
+    maxBytes;
+    entries = [];
+    head = 0; // index of the oldest entry; compacted lazily
+    bytes = 0;
+    gap = null;
+    waiters = [];
+    constructor(maxBytes) {
+        this.maxBytes = maxBytes;
+    }
+    put(entry) {
+        this.entries.push(entry);
+        this.bytes += size(entry);
+        while (this.bytes > this.maxBytes && this.length > 0) {
+            this.dropOldest();
+        }
+        this.wake();
+    }
+    dropOldest() {
+        const old = this.entries[this.head++];
+        this.bytes -= size(old);
+        if (this.gap === null) {
+            this.gap = { dropped: 1, fromMs: old.timestampMs, toMs: old.timestampMs };
+        }
+        else {
+            this.gap.dropped++;
+            this.gap.toMs = Math.max(this.gap.toMs, old.timestampMs);
+        }
+        this.compact();
+    }
+    /** Removes and returns the pending gap (if any) and up to `maxItems` entries, oldest first. */
+    take(maxItems) {
+        const gap = this.gap;
+        this.gap = null;
+        const entries = this.entries.slice(this.head, this.head + maxItems);
+        this.head += entries.length;
+        for (const e of entries) {
+            this.bytes -= size(e);
+        }
+        this.compact();
+        return { gap, entries };
+    }
+    /** Resolves when data arrives, on `wake()`, or after `timeoutMs`. */
+    wait(timeoutMs) {
+        if (!this.empty) {
+            return Promise.resolve();
+        }
+        return new Promise(resolve => {
+            const done = () => {
+                clearTimeout(timer);
+                this.waiters = this.waiters.filter(w => w !== done);
+                resolve();
+            };
+            const timer = setTimeout(done, timeoutMs);
+            timer.unref();
+            this.waiters.push(done);
+        });
+    }
+    /** Wakes a pending `wait`, e.g. to react to a new collector address or shutdown. */
+    wake() {
+        const waiters = this.waiters;
+        this.waiters = [];
+        for (const w of waiters) {
+            w();
+        }
+    }
+    get length() {
+        return this.entries.length - this.head;
+    }
+    get empty() {
+        return this.length === 0 && this.gap === null;
+    }
+    compact() {
+        if (this.head > 1024 && this.head * 2 > this.entries.length) {
+            this.entries = this.entries.slice(this.head);
+            this.head = 0;
+        }
+    }
+}
+exports.LogBuffer = LogBuffer;

package/dist/hooks.d.ts

@@ -0,0 +1,15 @@
+import type { LogMeta, LogShipper } from './shipper';
+/**
+ * Wraps an object with level methods (`debug`, `info`, `warn`, `error`, … — e.g.
+ * ioBroker's `this.log`). The result has the same interface and forwards every call
+ * unchanged; each call is also shipped. Everything else on the object passes through.
+ *
+ * The library reports its own connection problems to the unwrapped object.
+ */
+export declare function wrapLogger<T extends object>(target: T, shipper: LogShipper, meta?: LogMeta): T;
+/**
+ * Hooks into `console.debug/info/log/warn/error`: output stays as it is, each call is
+ * also shipped. Arguments are formatted like Node does (`util.format`), so an Error
+ * argument keeps its stack trace. Returns a function that restores the original console.
+ */
+export declare function hookConsole(shipper: LogShipper, meta?: LogMeta): () => void;

package/dist/hooks.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.wrapLogger = wrapLogger;
+exports.hookConsole = hookConsole;
+const node_util_1 = require("node:util");
+const LOGGER_METHODS = ['silly', 'trace', 'debug', 'info', 'log', 'warn', 'error', 'fatal'];
+/**
+ * Wraps an object with level methods (`debug`, `info`, `warn`, `error`, … — e.g.
+ * ioBroker's `this.log`). The result has the same interface and forwards every call
+ * unchanged; each call is also shipped. Everything else on the object passes through.
+ *
+ * The library reports its own connection problems to the unwrapped object.
+ */
+function wrapLogger(target, shipper, meta = {}) {
+    const t = target;
+    if (typeof t.info === 'function' && typeof t.warn === 'function') {
+        shipper.setInternalLog({
+            info: m => t.info.call(target, m),
+            warn: m => t.warn.call(target, m),
+        });
+    }
+    return new Proxy(target, {
+        get(obj, prop, receiver) {
+            const value = Reflect.get(obj, prop, receiver);
+            if (typeof value !== 'function' || !LOGGER_METHODS.includes(prop)) {
+                return value;
+            }
+            return (...args) => {
+                const result = value.apply(obj, args);
+                shipper.log(prop, (0, node_util_1.format)(...args), meta);
+                return result;
+            };
+        },
+    });
+}
+const CONSOLE_METHODS = ['debug', 'info', 'log', 'warn', 'error'];
+/**
+ * Hooks into `console.debug/info/log/warn/error`: output stays as it is, each call is
+ * also shipped. Arguments are formatted like Node does (`util.format`), so an Error
+ * argument keeps its stack trace. Returns a function that restores the original console.
+ */
+function hookConsole(shipper, meta = {}) {
+    const originals = CONSOLE_METHODS.map(m => [m, console[m]]);
+    // The library's own messages go to the original console, not back through the hook.
+    const original = Object.fromEntries(originals);
+    shipper.setInternalLog({
+        info: m => original.info.call(console, m),
+        warn: m => original.warn.call(console, m),
+    });
+    for (const [method, fn] of originals) {
+        console[method] = (...args) => {
+            fn.apply(console, args);
+            shipper.log(method, (0, node_util_1.format)(...args), meta);
+        };
+    }
+    return () => {
+        for (const [method, fn] of originals) {
+            console[method] = fn;
+        }
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Ships the logs of a Hannah component to the Hannah log collector.
+ *
+ *     const shipper = new LogShipper({ component: 'msteams', version });  // as early as possible
+ *     hookConsole(shipper);                                                // or wrapLogger / shipper.hook
+ *     ...
+ *     shipper.connect({ hannahAddress: 'localhost:50051' });              // once the config is loaded
+ */
+export { LogShipper, Category, DEFAULT_MAX_BUFFER_BYTES, toLogLevel } from './shipper';
+export type { Level, LogMeta, InternalLog, ShipperOptions, ConnectOptions } from './shipper';
+export { wrapLogger, hookConsole } from './hooks';
+export { SecretFilter, MASK } from './secrets';

package/dist/index.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MASK = exports.SecretFilter = exports.hookConsole = exports.wrapLogger = exports.toLogLevel = exports.DEFAULT_MAX_BUFFER_BYTES = exports.Category = exports.LogShipper = void 0;
+/**
+ * Ships the logs of a Hannah component to the Hannah log collector.
+ *
+ *     const shipper = new LogShipper({ component: 'msteams', version });  // as early as possible
+ *     hookConsole(shipper);                                                // or wrapLogger / shipper.hook
+ *     ...
+ *     shipper.connect({ hannahAddress: 'localhost:50051' });              // once the config is loaded
+ */
+var shipper_1 = require("./shipper");
+Object.defineProperty(exports, "LogShipper", { enumerable: true, get: function () { return shipper_1.LogShipper; } });
+Object.defineProperty(exports, "Category", { enumerable: true, get: function () { return shipper_1.Category; } });
+Object.defineProperty(exports, "DEFAULT_MAX_BUFFER_BYTES", { enumerable: true, get: function () { return shipper_1.DEFAULT_MAX_BUFFER_BYTES; } });
+Object.defineProperty(exports, "toLogLevel", { enumerable: true, get: function () { return shipper_1.toLogLevel; } });
+var hooks_1 = require("./hooks");
+Object.defineProperty(exports, "wrapLogger", { enumerable: true, get: function () { return hooks_1.wrapLogger; } });
+Object.defineProperty(exports, "hookConsole", { enumerable: true, get: function () { return hooks_1.hookConsole; } });
+var secrets_1 = require("./secrets");
+Object.defineProperty(exports, "SecretFilter", { enumerable: true, get: function () { return secrets_1.SecretFilter; } });
+Object.defineProperty(exports, "MASK", { enumerable: true, get: function () { return secrets_1.MASK; } });

package/dist/secrets.d.ts

@@ -0,0 +1,14 @@
+/** Masks secrets in a log message before it leaves the process. */
+export declare const MASK = "***";
+/**
+ * Masks well-known secret shapes plus literal values the component knows are secret
+ * (its own tokens and passwords — the most reliable way to catch them).
+ */
+export declare class SecretFilter {
+    private literals;
+    private readonly patterns;
+    /** @throws SyntaxError for an invalid entry in `patterns` */
+    constructor(literals?: Iterable<string>, patterns?: Iterable<string>);
+    addLiteral(secret: string): void;
+    apply(message: string): string;
+}

package/dist/secrets.js

@@ -0,0 +1,70 @@
+"use strict";
+/** Masks secrets in a log message before it leaves the process. */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SecretFilter = exports.MASK = void 0;
+exports.MASK = '***';
+// Key names whose value is treated as secret. The key has to *end* in one of them,
+// so "bot_token=…" is masked but "tokens=512" or "token_count=3" is not.
+const SECRET_KEYS = 'password|passwd|pwd|passphrase|secret|client[_-]?secret|token|access[_-]?token|refresh[_-]?token' +
+    '|api[_-]?key|apikey|access[_-]?key|private[_-]?key|psk|auth|authorization|credentials?';
+function quotedMask(value) {
+    const q = value[0];
+    if (value.length >= 2 && (q === '"' || q === "'") && value[value.length - 1] === q) {
+        return q + exports.MASK + q;
+    }
+    return exports.MASK;
+}
+const BUILTIN_PATTERNS = [
+    // PEM blocks (private keys).
+    [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, exports.MASK],
+    // "Authorization: Bearer abc", "Bearer abc"
+    [/\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]+/gi, `$1 ${exports.MASK}`],
+    // key=value / key: value / "key": "value"
+    [
+        new RegExp(`(["']?[\\w.-]*(?:${SECRET_KEYS})["']?)(\\s*[:=]\\s*)("[^"]*"|'[^']*'|[^\\s,;&)}\\]]+)`, 'gi'),
+        (_m, key, sep, value) => key + sep + quotedMask(value),
+    ],
+    // Credentials in URLs: scheme://user:password@host
+    [/\b([a-z][a-z0-9+.-]*:\/\/[^/\s:@]+):[^/\s@]+@/gi, `$1:${exports.MASK}@`],
+    // JWTs
+    [/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, exports.MASK],
+    // Telegram bot tokens
+    [/\b\d{8,10}:[A-Za-z0-9_-]{35}\b/g, exports.MASK],
+    // Well-known token prefixes (GitLab, GitHub, npm, PyPI, Slack, OpenAI-style)
+    [/\b(?:glpat|gldt|glrt|ghp|gho|ghu|ghs|ghr|github_pat|npm|pypi|xox[abpr]|sk)[-_][A-Za-z0-9_-]{16,}/g, exports.MASK],
+];
+/**
+ * Masks well-known secret shapes plus literal values the component knows are secret
+ * (its own tokens and passwords — the most reliable way to catch them).
+ */
+class SecretFilter {
+    literals = [];
+    patterns;
+    /** @throws SyntaxError for an invalid entry in `patterns` */
+    constructor(literals = [], patterns = []) {
+        this.patterns = [...BUILTIN_PATTERNS, ...[...patterns].map((p) => [new RegExp(p, 'g'), exports.MASK])];
+        for (const l of literals) {
+            this.addLiteral(l);
+        }
+    }
+    addLiteral(secret) {
+        // A 1–3 character "secret" would mangle ordinary text.
+        if (!secret || secret.length < 4 || this.literals.includes(secret)) {
+            return;
+        }
+        this.literals.push(secret);
+        // Longest first, so a secret that contains another one is masked as a whole.
+        this.literals.sort((a, b) => b.length - a.length);
+    }
+    apply(message) {
+        for (const literal of this.literals) {
+            message = message.split(literal).join(exports.MASK);
+        }
+        for (const [re, replacement] of this.patterns) {
+            message =
+                typeof replacement === 'string' ? message.replace(re, replacement) : message.replace(re, replacement);
+        }
+        return message;
+    }
+}
+exports.SecretFilter = SecretFilter;

package/dist/shipper.d.ts

@@ -0,0 +1,97 @@
+import { logging } from '@m1kad0/hannah-proto';
+export declare const DEFAULT_MAX_BUFFER_BYTES: number;
+export declare const Category: {
+    readonly GENERAL: logging.LogCategory.LOG_CATEGORY_GENERAL;
+    /** speech transcripts / user utterances */
+    readonly TRANSCRIPT: logging.LogCategory.LOG_CATEGORY_TRANSCRIPT;
+    /** room/device names, presence and similar */
+    readonly METADATA: logging.LogCategory.LOG_CATEGORY_METADATA;
+};
+/**
+ * Level names as used by common loggers (ioBroker, console, pino, winston), or a
+ * LogLevel value. Unknown names count as info.
+ */
+export type Level = 'silly' | 'trace' | 'debug' | 'info' | 'log' | 'warn' | 'warning' | 'error' | 'fatal' | 'critical' | logging.LogLevel;
+export interface LogMeta {
+    /** e.g. the module or subsystem; used for `loggerCategories` */
+    logger?: string;
+    category?: logging.LogCategory;
+    /** Unix ms; defaults to now */
+    timestamp?: number;
+}
+/** Where the library reports its own connection problems. Never shipped. */
+export interface InternalLog {
+    info(message: string): void;
+    warn(message: string): void;
+}
+export interface ShipperOptions {
+    /** e.g. "iobroker", "msteams" — the name shown in the export */
+    component: string;
+    /** component version, shown in the export manifest */
+    version?: string;
+    /** distinguishes several instances; defaults to the hostname */
+    instance?: string;
+    /** literal values masked wherever they appear (the component's own tokens and passwords) */
+    secrets?: string[];
+    /** extra regexes to mask, on top of the built-in ones */
+    secretPatterns?: string[];
+    /** logger name prefix -> category, e.g. { stt: Category.TRANSCRIPT } also covers "stt.whisper" */
+    loggerCategories?: Record<string, logging.LogCategory>;
+    maxBufferBytes?: number;
+    /** defaults to console */
+    internalLog?: InternalLog;
+}
+export interface ConnectOptions {
+    /** host:port of Hannah Core — enables discovery of the collector */
+    hannahAddress?: string;
+    /** host:port of the collector — fallback while none is announced */
+    collectorAddress?: string;
+}
+export declare function toLogLevel(level: Level): logging.LogLevel;
+export declare function formatAddress(host: string, port: number): string;
+/**
+ * Buffers log lines from the moment it is created and ships them to the log collector
+ * once one is known. Feed it through `hook` (any logger), `wrapLogger` or `hookConsole`.
+ */
+export declare class LogShipper {
+    /** The hook: call it for every log line, e.g. from a pino hook or a winston transport. */
+    readonly hook: (level: Level, message: string, meta?: LogMeta) => void;
+    private readonly buffer;
+    private readonly secrets;
+    private readonly categories;
+    private readonly hello;
+    private internalLog;
+    private reportingInternally;
+    private connected;
+    private stopping;
+    private staticAddress;
+    private discovered;
+    private shipperDone;
+    private discoveryCall;
+    private sleepers;
+    constructor(opts: ShipperOptions);
+    /** Buffers one log line. Never throws, never blocks. */
+    log(level: Level, message: string, meta?: LogMeta): void;
+    /** Masks this literal value in everything logged from now on. */
+    addSecret(secret: string): void;
+    /** Where the library reports its own connection problems (default: console). */
+    setInternalLog(log: InternalLog): void;
+    /** Starts shipping. Only the first call has an effect. */
+    connect(opts: ConnectOptions): void;
+    /** Stops discovery and gives the shipper up to `timeoutMs` to send what is still buffered. */
+    close(timeoutMs?: number): Promise<void>;
+    private categoryFor;
+    private report;
+    private target;
+    /**
+     * Resolves after `ms`, or earlier on close or when the collector address changes.
+     * Returns false once closing.
+     */
+    private sleep;
+    private wakeSleepers;
+    private runDiscovery;
+    private subscribeOnce;
+    private setDiscovered;
+    private runShipper;
+    private shipOnce;
+}

package/dist/shipper.js

@@ -0,0 +1,422 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LogShipper = exports.Category = exports.DEFAULT_MAX_BUFFER_BYTES = void 0;
+exports.toLogLevel = toLogLevel;
+exports.formatAddress = formatAddress;
+const node_events_1 = require("node:events");
+const node_os_1 = require("node:os");
+const grpc = __importStar(require("@grpc/grpc-js"));
+const hannah_proto_1 = require("@m1kad0/hannah-proto");
+const buffer_1 = require("./buffer");
+const secrets_1 = require("./secrets");
+exports.DEFAULT_MAX_BUFFER_BYTES = 4 * 1024 * 1024;
+const MAX_BACKOFF_MS = 30_000;
+const CONNECT_TIMEOUT_MS = 5_000;
+const BATCH_SIZE = 200;
+exports.Category = {
+    GENERAL: hannah_proto_1.logging.LogCategory.LOG_CATEGORY_GENERAL,
+    /** speech transcripts / user utterances */
+    TRANSCRIPT: hannah_proto_1.logging.LogCategory.LOG_CATEGORY_TRANSCRIPT,
+    /** room/device names, presence and similar */
+    METADATA: hannah_proto_1.logging.LogCategory.LOG_CATEGORY_METADATA,
+};
+const LEVELS = {
+    silly: hannah_proto_1.logging.LogLevel.LOG_LEVEL_DEBUG,
+    trace: hannah_proto_1.logging.LogLevel.LOG_LEVEL_DEBUG,
+    debug: hannah_proto_1.logging.LogLevel.LOG_LEVEL_DEBUG,
+    info: hannah_proto_1.logging.LogLevel.LOG_LEVEL_INFO,
+    log: hannah_proto_1.logging.LogLevel.LOG_LEVEL_INFO,
+    warn: hannah_proto_1.logging.LogLevel.LOG_LEVEL_WARNING,
+    warning: hannah_proto_1.logging.LogLevel.LOG_LEVEL_WARNING,
+    error: hannah_proto_1.logging.LogLevel.LOG_LEVEL_ERROR,
+    fatal: hannah_proto_1.logging.LogLevel.LOG_LEVEL_CRITICAL,
+    critical: hannah_proto_1.logging.LogLevel.LOG_LEVEL_CRITICAL,
+};
+function toLogLevel(level) {
+    return typeof level === 'number' ? level : (LEVELS[level] ?? hannah_proto_1.logging.LogLevel.LOG_LEVEL_INFO);
+}
+const protocolVersionInterceptor = (options, nextCall) => new grpc.InterceptingCall(nextCall(options), {
+    start(metadata, listener, next) {
+        metadata.set('x-proto-version', String(hannah_proto_1.PROTO_VERSION));
+        next(metadata, listener);
+    },
+});
+const CLIENT_OPTIONS = {
+    interceptors: [protocolVersionInterceptor, hannah_proto_1.compat_interceptor.compatVersionInterceptor],
+};
+function formatAddress(host, port) {
+    if (host.includes(':') && !host.startsWith('[')) {
+        host = `[${host}]`; // IPv6
+    }
+    return `${host}:${port}`;
+}
+function errorText(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+/**
+ * Buffers log lines from the moment it is created and ships them to the log collector
+ * once one is known. Feed it through `hook` (any logger), `wrapLogger` or `hookConsole`.
+ */
+class LogShipper {
+    /** The hook: call it for every log line, e.g. from a pino hook or a winston transport. */
+    hook;
+    buffer;
+    secrets;
+    categories;
+    hello;
+    internalLog;
+    reportingInternally = false;
+    connected = false;
+    stopping = false;
+    staticAddress = '';
+    discovered = '';
+    shipperDone = Promise.resolve();
+    discoveryCall = null;
+    sleepers = [];
+    constructor(opts) {
+        if (!opts.component) {
+            throw new Error('hannah-logging: component is required');
+        }
+        this.buffer = new buffer_1.LogBuffer(opts.maxBufferBytes ?? exports.DEFAULT_MAX_BUFFER_BYTES);
+        this.secrets = new secrets_1.SecretFilter(opts.secrets, opts.secretPatterns);
+        this.categories = Object.entries(opts.loggerCategories ?? {}).sort((a, b) => b[0].length - a[0].length);
+        this.hello = { component: opts.component, instance: opts.instance || (0, node_os_1.hostname)(), version: opts.version ?? '' };
+        this.internalLog = opts.internalLog ?? console;
+        this.hook = (level, message, meta) => this.log(level, message, meta);
+    }
+    /** Buffers one log line. Never throws, never blocks. */
+    log(level, message, meta = {}) {
+        if (this.reportingInternally) {
+            return; // the library's own message, e.g. through a hooked console
+        }
+        try {
+            const logger = meta.logger ?? '';
+            this.buffer.put({
+                timestampMs: meta.timestamp ?? Date.now(),
+                level: toLogLevel(level),
+                logger,
+                message: this.secrets.apply(String(message)),
+                category: meta.category || this.categoryFor(logger),
+            });
+        }
+        catch {
+            // never let logging break the component
+        }
+    }
+    /** Masks this literal value in everything logged from now on. */
+    addSecret(secret) {
+        this.secrets.addLiteral(secret);
+    }
+    /** Where the library reports its own connection problems (default: console). */
+    setInternalLog(log) {
+        this.internalLog = log;
+    }
+    /** Starts shipping. Only the first call has an effect. */
+    connect(opts) {
+        if (this.connected || this.stopping) {
+            return;
+        }
+        this.connected = true;
+        this.staticAddress = opts.collectorAddress ?? '';
+        this.shipperDone = this.runShipper();
+        if (opts.hannahAddress) {
+            void this.runDiscovery(opts.hannahAddress);
+        }
+    }
+    /** Stops discovery and gives the shipper up to `timeoutMs` to send what is still buffered. */
+    async close(timeoutMs = 2000) {
+        if (this.stopping) {
+            return;
+        }
+        this.stopping = true;
+        this.discoveryCall?.cancel();
+        this.wakeSleepers();
+        this.buffer.wake();
+        let timer;
+        await Promise.race([this.shipperDone, new Promise(r => (timer = setTimeout(r, timeoutMs)))]);
+        clearTimeout(timer);
+    }
+    categoryFor(logger) {
+        for (const [prefix, category] of this.categories) {
+            if (logger === prefix || logger.startsWith(`${prefix}.`)) {
+                return category;
+            }
+        }
+        return exports.Category.GENERAL;
+    }
+    report(level, message) {
+        this.reportingInternally = true;
+        try {
+            this.internalLog[level](`[hannah-logging] ${message}`);
+        }
+        catch {
+            // ignore
+        }
+        finally {
+            this.reportingInternally = false;
+        }
+    }
+    target() {
+        return this.discovered || this.staticAddress;
+    }
+    /**
+     * Resolves after `ms`, or earlier on close or when the collector address changes.
+     * Returns false once closing.
+     */
+    sleep(ms) {
+        if (this.stopping) {
+            return Promise.resolve(false);
+        }
+        return new Promise(resolve => {
+            const done = () => {
+                clearTimeout(timer);
+                this.sleepers = this.sleepers.filter(w => w !== done);
+                resolve(!this.stopping);
+            };
+            const timer = setTimeout(done, ms);
+            timer.unref();
+            this.sleepers.push(done);
+        });
+    }
+    wakeSleepers() {
+        for (const w of this.sleepers.slice()) {
+            w();
+        }
+    }
+    // ------------------------------------------------------------------
+    // Discovery: follows Hannah Core's infrastructure broadcast. If Core becomes
+    // unreachable, the last known collector is kept — the collector doesn't depend on
+    // Core, so logs keep flowing. The snapshot after a reconnect replaces the known state.
+    // ------------------------------------------------------------------
+    async runDiscovery(address) {
+        let backoff = 1000;
+        let failing = false;
+        const collectors = new Map(); // instance -> address
+        while (!this.stopping) {
+            const err = await this.subscribeOnce(address, collectors, () => {
+                if (failing) {
+                    this.report('info', `log collector discovery: connected to Hannah at ${address}`);
+                    failing = false;
+                }
+                backoff = 1000;
+            });
+            if (this.stopping) {
+                return;
+            }
+            if (!failing) {
+                this.report('warn', `log collector discovery: Hannah at ${address} unreachable (${err}), retrying`);
+                failing = true;
+            }
+            if (!(await this.sleep(backoff))) {
+                return;
+            }
+            backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
+        }
+    }
+    subscribeOnce(address, collectors, onMessage) {
+        return new Promise(resolve => {
+            const client = new hannah_proto_1.hannah.HannahServiceClient(address, grpc.credentials.createInsecure(), CLIENT_OPTIONS);
+            const call = client.subscribeInfrastructure({
+                kinds: [hannah_proto_1.infrastructure.ServiceKind.SERVICE_KIND_LOG_COLLECTOR],
+            });
+            this.discoveryCall = call;
+            let settled = false;
+            const finish = (reason) => {
+                if (settled) {
+                    return;
+                }
+                settled = true;
+                this.discoveryCall = null;
+                client.close();
+                resolve(reason);
+            };
+            call.on('data', (msg) => {
+                onMessage();
+                if (applyInfrastructure(msg, collectors)) {
+                    this.setDiscovered(pickCollector(collectors));
+                }
+            });
+            call.on('error', (err) => finish(errorText(err)));
+            call.on('end', () => finish('stream ended'));
+        });
+    }
+    setDiscovered(address) {
+        if (address === this.discovered) {
+            return;
+        }
+        this.discovered = address;
+        if (address) {
+            this.report('info', `log collector discovered at ${address}`);
+        }
+        this.wakeSleepers();
+        this.buffer.wake();
+    }
+    // ------------------------------------------------------------------
+    // Shipper: streams the buffer to the collector over LogService.Ship. Only opens the
+    // stream once the channel is ready, so nothing leaves the buffer while the collector
+    // is unreachable. Ends the stream when the collector address changes.
+    // ------------------------------------------------------------------
+    async runShipper() {
+        let backoff = 1000;
+        let failing = false;
+        while (!this.stopping) {
+            const target = this.target();
+            if (!target) {
+                // No collector known: the buffer keeps running as a ring. Wait for an
+                // address (setDiscovered wakes the sleep) — not for data, which may already
+                // be buffered and would turn this into a busy loop.
+                await this.sleep(1000);
+                continue;
+            }
+            try {
+                await this.shipOnce(target, () => {
+                    if (failing) {
+                        this.report('info', `log collector at ${target} reachable again`);
+                        failing = false;
+                    }
+                });
+                backoff = 1000;
+                continue;
+            }
+            catch (err) {
+                if (this.stopping) {
+                    return;
+                }
+                if (!failing) {
+                    this.report('warn', `log collector at ${target}: ${errorText(err)} — buffering, retrying`);
+                    failing = true;
+                }
+            }
+            if (!(await this.sleep(backoff))) {
+                return;
+            }
+            backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
+        }
+    }
+    async shipOnce(target, onConnected) {
+        const client = new hannah_proto_1.logging.LogServiceClient(target, grpc.credentials.createInsecure(), CLIENT_OPTIONS);
+        try {
+            await new Promise((resolve, reject) => client.waitForReady(Date.now() + CONNECT_TIMEOUT_MS, err => (err ? reject(new Error('not reachable')) : resolve())));
+            onConnected();
+            // Settles when the collector answers (ShipAck) or the call fails.
+            let call;
+            const callDone = new Promise((resolve, reject) => {
+                call = client.ship(err => (err ? reject(err) : resolve()));
+            });
+            let settled = false;
+            callDone.then(() => (settled = true), () => (settled = true));
+            const write = async (msg) => {
+                if (!call.write(msg)) {
+                    await Promise.race([(0, node_events_1.once)(call, 'drain'), callDone]);
+                }
+            };
+            await write({ hello: this.hello });
+            for (;;) {
+                const { gap, entries } = this.buffer.take(BATCH_SIZE);
+                if (gap) {
+                    await write({
+                        gap: { dropped: BigInt(gap.dropped), fromMs: BigInt(gap.fromMs), toMs: BigInt(gap.toMs) },
+                    });
+                }
+                for (const e of entries) {
+                    await write({
+                        entry: {
+                            timestampMs: BigInt(e.timestampMs),
+                            level: e.level,
+                            logger: e.logger,
+                            message: e.message,
+                            category: e.category,
+                        },
+                    });
+                }
+                const drained = !gap && entries.length === 0;
+                // Collector moved: what's left in the buffer goes to the new one.
+                // Shutdown: close the stream only once the buffer is drained.
+                if (this.target() !== target || (this.stopping && drained)) {
+                    call.end();
+                    await callDone;
+                    return;
+                }
+                if (drained) {
+                    await Promise.race([this.buffer.wait(1000), callDone]);
+                }
+                if (settled) {
+                    // The call ended without us closing it: rethrow its error, if any.
+                    await callDone;
+                    throw new Error('collector closed the stream');
+                }
+            }
+        }
+        finally {
+            client.close();
+        }
+    }
+}
+exports.LogShipper = LogShipper;
+/** Applies `msg` to `collectors` and reports whether it was relevant. */
+function applyInfrastructure(msg, collectors) {
+    const LOG_COLLECTOR = hannah_proto_1.infrastructure.ServiceKind.SERVICE_KIND_LOG_COLLECTOR;
+    if (msg.snapshot) {
+        collectors.clear();
+        for (const s of msg.snapshot.services) {
+            if (s.kind === LOG_COLLECTOR) {
+                collectors.set(s.instance, formatAddress(s.host, s.port));
+            }
+        }
+        return true;
+    }
+    if (msg.available?.service) {
+        const s = msg.available.service;
+        if (s.kind !== LOG_COLLECTOR) {
+            return false;
+        }
+        collectors.set(s.instance, formatAddress(s.host, s.port));
+        return true;
+    }
+    if (msg.unavailable) {
+        if (msg.unavailable.kind !== LOG_COLLECTOR) {
+            return false;
+        }
+        collectors.delete(msg.unavailable.instance);
+        return true;
+    }
+    return false;
+}
+/** Chooses one collector deterministically; several are unusual. */
+function pickCollector(collectors) {
+    const instances = [...collectors.keys()].sort();
+    return instances.length ? collectors.get(instances[0]) : '';
+}

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@m1kad0/hannah-logging",
+  "version": "0.0.0",
+  "description": "Ships the logs of a Hannah component to the Hannah log collector",
+  "license": "UNLICENSED",
+  "repository": {
+    "type": "git",
+    "url": "https://gitlab.com/gessinger/hannah-logging-libs.git",
+    "directory": "ts"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">= 22"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "tsc -p tsconfig.test.json && node --test --test-timeout=30000 \"build-test/**/*.test.js\""
+  },
+  "peerDependencies": {
+    "@grpc/grpc-js": "^1.14.4",
+    "@m1kad0/hannah-proto": "^4.5.0"
+  },
+  "devDependencies": {
+    "@grpc/grpc-js": "1.14.4",
+    "@m1kad0/hannah-proto": "4.5.0",
+    "@types/node": "^22.0.0",
+    "typescript": "~6.0.3"
+  }
+}