@omnitron-dev/omnitron 0.2.0 → 0.3.1

package/dist/services/log-collector.service.js

@@ -7,19 +7,51 @@
  * Designed to be wired into orchestrator.onAppLog() to capture all child
  * process log lines, parse pino JSON, and persist to the logs table.
  */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 import { randomUUID } from 'node:crypto';
+import {} from 'kysely';
+import { planRetention, batchesPerPass } from './log-retention.js';
+import { jsonContains, dialectOf } from '../database/dialect.js';
 import { EventEmitter } from 'node:events';
+import { Injectable, Inject } from '@omnitron-dev/titan/decorators';
+import { OMNITRON_DB_TOKEN } from '../shared/tokens.js';
+import { LineAssembler } from '../monitoring/log-line.js';
 // =============================================================================
-// Pino Level Map (numeric → string)
+// Capture order within a millisecond
 // =============================================================================
-const PINO_LEVELS = {
-    10: 'trace',
-    20: 'debug',
-    30: 'info',
-    40: 'warn',
-    50: 'error',
-    60: 'fatal',
-};
+/**
+ * A millisecond's capture sequence, written as its microseconds.
+ *
+ * Records captured in one millisecond were stored with equal timestamps and
+ * read back in whatever order the database chose for the tie — on the master
+ * a uuid's, which is to say none — so the lines of one dump, and bursts of
+ * records, came back shuffled. The column is `timestamptz`, which keeps
+ * microseconds, and no source stamps finer than a millisecond: so the
+ * microseconds carry the order the records were captured in, 0–999 within
+ * their millisecond. `ORDER BY timestamp` then IS capture order, with no
+ * migration and no second sort key — and the string travels to the master
+ * inside the replicated entry, whose ingest passes it through unchanged.
+ *
+ * The millisecond itself stays the record's own. Past 999 records in one
+ * millisecond the rest share .999 — a tie, never a later time.
+ */
+export function timestampWithSequence(ms, sequence) {
+    const iso = new Date(ms).toISOString();
+    return `${iso.slice(0, -1)}${String(Math.min(sequence, 999)).padStart(3, '0')}Z`;
+}
+/** Milliseconds whose sequence is remembered — far more than one flush spans. */
+const SEQUENCED_MILLISECONDS = 4096;
 // =============================================================================
 // Log Collector Service
 // =============================================================================
@@ -27,27 +59,244 @@ const PINO_LEVELS = {
 function escapeLike(str) {
     return str.replace(/[%_\\]/g, '\\$&');
 }
+/** A JSON column read back as text (SQLite), or nothing if it is not an object. */
+function safeJson(text) {
+    try {
+        const value = JSON.parse(text);
+        return value !== null && typeof value === 'object' && !Array.isArray(value) ? value : null;
+    }
+    catch {
+        return null;
+    }
+}
 const FLUSH_INTERVAL_MS = 1_000;
+/**
+ * How often to prune old log rows. Hourly: the table grows over days, so a
+ * pass more often than this is work for its own sake, and one much less often
+ * lets a burst accumulate between passes.
+ */
+const RETENTION_INTERVAL_MS = 60 * 60 * 1_000;
+/**
+ * How long to wait before the FIRST retention pass.
+ *
+ * Long enough for migrations to have created the table. See
+ * `setRetentionDays` for what firing at zero produced.
+ */
+const RETENTION_FIRST_PASS_DELAY_MS = 60_000;
+/** Cadence while a backlog is still draining — see `scheduleRetention`. */
+const RETENTION_BACKLOG_INTERVAL_MS = 60 * 1_000;
 const FLUSH_THRESHOLD = 100;
-export class LogCollectorService extends EventEmitter {
+/**
+ * Hard ceiling on buffered entries. During a DB outage every flush fails and
+ * requeues while new entries keep arriving — without a ceiling the buffer
+ * grows for as long as the outage lasts. The 2026-07-11 daemon crash was
+ * exactly this: hours of omnitron-pg downtime grew the buffer into the
+ * hundreds of thousands, and the then-`unshift(...batch)` requeue blew the
+ * argument-spread call-stack limit (RangeError → unhandled rejection →
+ * daemon exit). Newest entries win; the oldest overflow is counted and
+ * dropped.
+ */
+const MAX_BUFFER = 50_000;
+/** Per-flush insert ceiling — bounds both the INSERT payload and the requeue. */
+const FLUSH_BATCH_MAX = 5_000;
+let LogCollectorService = class LogCollectorService extends EventEmitter {
     db;
     buffer = [];
     flushTimer = null;
     flushing = false;
     disposed = false;
+    /** Entries dropped to the MAX_BUFFER ceiling since the last successful flush. */
+    droppedSinceFlush = 0;
+    /**
+     * Lifetime counters, in memory.
+     *
+     * The console's "Log Ingestion" card was a hardcoded `0 lines/s`, and the
+     * only alternative source is `getLogStats`, which runs three `count(*)`
+     * over the whole table — 22.5 million rows on this host, so not something
+     * to poll. A counter costs an addition per flush and answers the question
+     * the card was asking.
+     *
+     * `droppedTotal` matters more than the rate. The buffer sheds its oldest
+     * entries at the MAX_BUFFER ceiling during a database outage, and until
+     * now that happened with no counter anyone could read — logs vanished and
+     * the only trace was an event nobody was listening to.
+     */
+    ingestedTotal = 0;
+    droppedTotal = 0;
+    /**
+     * Where persisted batches also go, on a slave.
+     *
+     * A slave keeps its logs locally and replicates them to the master; the
+     * replication half was fully built — a WAL, a dedup ledger on the master,
+     * backoff, eviction — and `SyncService.buffer`/`bufferBatch` had **zero
+     * callers**. Everything downstream of them worked and moved nothing,
+     * because nothing upstream ever enqueued. This is the missing producer.
+     *
+     * A sink rather than a constructor dependency, for the reason the
+     * retention setter below gives: this service is registered with `useClass`
+     * and takes the database as its only injected argument, and adding a
+     * second is a change to module wiring that fails at runtime rather than at
+     * compile time.
+     *
+     * Set only on a slave. `bufferBatch` is itself a no-op for any other role,
+     * so a mis-wire costs nothing, but not wiring it on a master also avoids
+     * the work.
+     */
+    syncSink = null;
+    retentionTimer = null;
+    /** Days of history to keep in the table. Zero or less disables pruning. */
+    retentionDays = 0;
+    /**
+     * Set alongside retention rather than injected.
+     *
+     * The service is registered with `useClass` and takes the database as its
+     * only constructor dependency; adding a second is a change to the module's
+     * wiring, and a wrong one there fails at runtime rather than at compile
+     * time — twice today already. One setter, one call site, nothing to get out
+     * of order.
+     */
+    logger = null;
+    /** Lines become records here exactly as they do for the files — `log-line.ts`. */
+    assembler = new LineAssembler((app, record) => this.ingestRecord(app, record));
+    /** Next capture sequence per millisecond — see `timestampWithSequence`. */
+    sequenceByMs = new Map();
+    dialect = null;
     constructor(db) {
         super();
         this.db = db;
         this.startFlushTimer();
     }
+    /**
+     * Turn on table retention.
+     *
+     * Off until called, because the number belongs to the daemon config and
+     * this service is constructed before it is read. A default chosen here
+     * would be a policy decision made in the wrong place — and a wrong one
+     * deletes an operator's history.
+     */
+    /**
+     * Replicate persisted batches to the master (slave role only).
+     *
+     * Called with the entries that REACHED the table, not the ones that were
+     * offered: a flush that fails puts its batch back at the front of the
+     * buffer and retries, and queueing on the attempt would send every retried
+     * line twice.
+     */
+    setSyncSink(sink) {
+        this.syncSink = sink;
+    }
+    setRetentionDays(days, logger) {
+        this.retentionDays = days;
+        if (logger)
+            this.logger = logger;
+        if (this.retentionTimer) {
+            clearTimeout(this.retentionTimer);
+            this.retentionTimer = null;
+        }
+        if (planRetention(days) === null) {
+            this.logger?.info?.({ days }, 'Log table retention disabled');
+            return;
+        }
+        // Not zero. A pass fired the instant the collector is configured races
+        // the migrations that create the table it reads: measured on a daemon
+        // start, `Log retention pass failed — relation "logs" does not exist`,
+        // once, at 03:27:03. Harmless in itself — the next pass an hour later
+        // succeeds — but it is an error-level line about a subsystem that is
+        // fine, and an error nobody can act on is one they learn to scroll past.
+        //
+        // The first pass has nothing urgent to do anyway: rows become eligible
+        // for deletion by ageing past the window, so a minute's delay costs
+        // nothing and a backlog is still drained at the faster cadence once the
+        // first pass sees one.
+        this.scheduleRetention(RETENTION_FIRST_PASS_DELAY_MS);
+    }
+    /**
+     * Arm the next retention pass.
+     *
+     * The per-pass ceiling exists to keep each statement short; it should not
+     * also decide how long a backlog takes to clear. A pass that hits its
+     * ceiling means there is more waiting, so the next one comes in a minute
+     * rather than an hour — thirteen million stale rows drain in half an hour
+     * instead of a day, with every individual statement still bounded. A pass
+     * that finishes early returns to the hourly cadence.
+     *
+     * `setTimeout` chained rather than `setInterval`, so the cadence is a
+     * decision made after each pass with its result in hand.
+     */
+    scheduleRetention(delayMs) {
+        if (this.retentionTimer)
+            clearTimeout(this.retentionTimer);
+        if (this.disposed)
+            return;
+        this.retentionTimer = setTimeout(() => {
+            void this.pruneOldLogs().then((removed) => {
+                const plan = planRetention(this.retentionDays);
+                if (!plan || this.disposed)
+                    return;
+                this.scheduleRetention(removed >= plan.maxThisPass ? RETENTION_BACKLOG_INTERVAL_MS : RETENTION_INTERVAL_MS);
+            });
+        }, delayMs);
+        this.retentionTimer.unref();
+    }
+    /**
+     * Delete rows past the retention window, in bounded batches.
+     *
+     * Batched because a single `DELETE` over 13 GB holds its transaction for
+     * the duration and blocks the flush path behind it — a retention pass would
+     * show up as the log pipeline stalling, which is the opposite of the point.
+     */
+    async pruneOldLogs() {
+        const plan = planRetention(this.retentionDays);
+        if (!plan || this.disposed)
+            return 0;
+        let removed = 0;
+        try {
+            for (let i = 0; i < batchesPerPass(plan); i++) {
+                if (this.disposed)
+                    break;
+                const doomed = await this.db
+                    .selectFrom('logs')
+                    .select('id')
+                    .where('timestamp', '<', plan.cutoff)
+                    .limit(plan.batchSize)
+                    .execute();
+                if (doomed.length === 0)
+                    break;
+                await this.db
+                    .deleteFrom('logs')
+                    .where('id', 'in', doomed.map((r) => String(r.id)))
+                    .execute();
+                removed += doomed.length;
+            }
+            if (removed > 0) {
+                this.logger?.info?.({ removed, cutoff: plan.cutoff.toISOString(), retentionDays: this.retentionDays }, 'Pruned log rows past retention');
+            }
+        }
+        catch (err) {
+            // Reported rather than swallowed: a retention pass that keeps failing
+            // is the disk filling up in slow motion, and the only warning of it.
+            this.logger?.error?.({ error: err.message, removed }, 'Log retention pass failed');
+        }
+        return removed;
+    }
     // ===========================================================================
     // Ingestion
     // ===========================================================================
+    /** Drop the oldest overflow so the buffer never exceeds MAX_BUFFER. */
+    enforceCap() {
+        const overflow = this.buffer.length - MAX_BUFFER;
+        if (overflow > 0) {
+            this.buffer.splice(0, overflow);
+            this.droppedSinceFlush += overflow;
+            this.droppedTotal += overflow;
+        }
+    }
     /** Ingest a single log entry into the buffer */
     ingestLog(entry) {
         if (this.disposed)
             return;
         this.buffer.push(entry);
+        this.enforceCap();
         if (this.buffer.length >= FLUSH_THRESHOLD) {
             void this.flush();
         }
@@ -56,119 +305,178 @@ export class LogCollectorService extends EventEmitter {
     ingestBatch(entries) {
         if (this.disposed)
             return;
-        this.buffer.push(...entries);
+        // concat, not push(...entries) — spreading a large array as arguments
+        // risks the same call-stack blowup the flush requeue hit.
+        this.buffer = this.buffer.concat(entries);
+        this.enforceCap();
         if (this.buffer.length >= FLUSH_THRESHOLD) {
             void this.flush();
         }
     }
     /**
-     * Parse a raw pino JSON log line from a child process and ingest it.
+     * Take one line of an app's output (or of the daemon's own, as `omnitron`).
      * This is the primary integration point with orchestrator.onAppLog().
+     *
+     * The line is read by the one classifier (`log-line.ts`) and joined into a
+     * record there. What the table held before: a line that was not JSON became
+     * a row at level `info`, and every indented line of a dump a row of its own
+     * — on the master, payments from the test node: 3 `error` rows, 1 264 995
+     * `info`, 1 216 292 of those dump fragments.
      */
     ingestPinoLine(appName, line) {
-        try {
-            const parsed = JSON.parse(line);
-            const level = typeof parsed.level === 'number'
-                ? (PINO_LEVELS[parsed.level] ?? 'info')
-                : (parsed.level ?? 'info');
-            const message = parsed.msg ?? parsed.message ?? '';
-            // Extract well-known fields, put the rest into metadata
-            const { level: _l, msg: _m, message: _msg, time, pid: _pid, hostname: _hostname, ...rest } = parsed;
-            // Extract labels if present
-            const labels = parsed.labels ?? undefined;
-            const traceId = parsed.traceId ?? parsed.trace_id ?? undefined;
-            const spanId = parsed.spanId ?? parsed.span_id ?? undefined;
-            // Everything else goes into metadata
-            const metadata = Object.keys(rest).length > 0 ? rest : undefined;
-            this.ingestLog({
-                app: appName,
-                level,
-                message: String(message),
-                timestamp: time ? new Date(time) : new Date(),
-                labels,
-                traceId,
-                spanId,
-                metadata,
-            });
+        if (this.disposed)
+            return;
+        this.assembler.push(appName, line);
+    }
+    /** One record, as a row. */
+    ingestRecord(appName, record) {
+        const timestamp = this.sequenced(record.time);
+        if (record.kind !== 'json' || !record.data) {
+            this.ingestLog({ app: appName, level: record.level, message: record.message, timestamp });
+            return;
         }
-        catch {
-            // Not valid JSON — ingest as raw text
-            this.ingestLog({
-                app: appName,
-                level: 'info',
-                message: line,
-                timestamp: new Date(),
-            });
+        const parsed = record.data;
+        // Extract well-known fields, put the rest into metadata
+        const { level: _l, msg: _m, message: _msg, time: _t, pid: _pid, hostname: _hostname, ...rest } = parsed;
+        // Extract labels if present
+        const labels = parsed['labels'] ?? undefined;
+        const traceId = (parsed['traceId'] ?? parsed['trace_id'] ?? undefined);
+        const spanId = (parsed['spanId'] ?? parsed['span_id'] ?? undefined);
+        // Everything else goes into metadata
+        const metadata = Object.keys(rest).length > 0 ? rest : undefined;
+        this.ingestLog({
+            app: appName,
+            level: record.level,
+            message: record.message,
+            timestamp,
+            ...(labels !== undefined ? { labels } : {}),
+            ...(traceId !== undefined ? { traceId } : {}),
+            ...(spanId !== undefined ? { spanId } : {}),
+            ...(metadata !== undefined ? { metadata } : {}),
+        });
+    }
+    /** The record's millisecond, and its place among the records captured in it. */
+    sequenced(ms) {
+        const at = Number.isFinite(ms) ? Math.floor(ms) : Date.now();
+        const next = this.sequenceByMs.get(at) ?? 0;
+        this.sequenceByMs.set(at, next + 1);
+        if (this.sequenceByMs.size > SEQUENCED_MILLISECONDS) {
+            const oldest = this.sequenceByMs.keys().next().value;
+            if (oldest !== undefined)
+                this.sequenceByMs.delete(oldest);
         }
+        return timestampWithSequence(at, next);
     }
     // ===========================================================================
     // Querying
     // ===========================================================================
+    /**
+     * The two filters the schema was built for and no query used.
+     *
+     * `logs.nodeId` is a uuid column that `SyncService.ingestLog` fills for every
+     * entry a slave sends up, and `logs.labels` is jsonb carrying a GIN index
+     * that `001_initial_schema` created with the intended query written in its
+     * own comment: `WHERE labels @> '{"env":"prod"}'`. Neither `queryLogs` nor
+     * `getRecentLogs` referenced either one.
+     *
+     * The visible consequence was in the console: its cluster node filter put the
+     * node id inside `labels`, sent it, and the daemon dropped the whole field —
+     * so selecting a node narrowed nothing while the chip stayed lit. A filter
+     * that silently does nothing is worse than one that is missing, because the
+     * operator reads the unnarrowed result as an answer about that node.
+     *
+     * Applied in BOTH paths deliberately. A filter honoured by the paginated
+     * query and not by the live tail is how a viewer changes its answer when you
+     * press Live.
+     */
     async queryLogs(filter) {
         const limit = Math.min(filter.limit ?? 100, 1000);
         const offset = filter.offset ?? 0;
-        let query = this.db.selectFrom('logs').selectAll();
-        // Apply filters
-        if (filter.app) {
-            query = query.where('app', '=', filter.app);
-        }
-        if (filter.level) {
-            if (Array.isArray(filter.level)) {
-                query = query.where('level', 'in', filter.level);
+        // One definition of "which rows", used by both queries.
+        //
+        // There were two, and they had drifted: the rows honoured `nodeId` and
+        // `labels`, the count did not. So a log view filtered to one node showed
+        // that node's lines under a total counting every node's — a pager
+        // offering pages that hold nothing, and a number an operator reads as
+        // how much that node said. The two lists were written together and
+        // maintained apart, which is the failure mode of writing them twice.
+        const matching = (eb) => {
+            const clauses = [];
+            if (filter.app)
+                clauses.push(eb('app', '=', filter.app));
+            if (filter.level) {
+                clauses.push(Array.isArray(filter.level)
+                    ? eb('level', 'in', filter.level)
+                    : eb('level', '=', filter.level));
             }
-            else {
-                query = query.where('level', '=', filter.level);
+            if (filter.search)
+                clauses.push(eb('message', 'like', `%${escapeLike(filter.search)}%`));
+            if (filter.traceId)
+                clauses.push(eb('traceId', '=', filter.traceId));
+            if (filter.nodeId)
+                clauses.push(eb('nodeId', '=', filter.nodeId));
+            if (filter.labels && Object.keys(filter.labels).length > 0) {
+                // Containment, asked the way each engine understands it. Written as
+                // `labels @> ?::jsonb`, this was a syntax error on every slave —
+                // SQLite reads `@` as an unrecognized token and refuses the whole
+                // statement — so a log query carrying a label filter threw on the
+                // exact nodes whose logs you go looking for. See `jsonContains`.
+                clauses.push(jsonContains(this.db, 'labels', filter.labels));
             }
-        }
-        if (filter.search) {
-            query = query.where('message', 'like', `%${escapeLike(filter.search)}%`);
-        }
-        if (filter.traceId) {
-            query = query.where('traceId', '=', filter.traceId);
-        }
-        if (filter.from) {
-            const fromDate = typeof filter.from === 'string' ? new Date(filter.from) : filter.from;
-            query = query.where('timestamp', '>=', fromDate);
-        }
-        if (filter.to) {
-            const toDate = typeof filter.to === 'string' ? new Date(filter.to) : filter.to;
-            query = query.where('timestamp', '<=', toDate);
-        }
-        // Get total count for pagination
-        let countQuery = this.db.selectFrom('logs').select(this.db.fn.countAll().as('count'));
-        if (filter.app)
-            countQuery = countQuery.where('app', '=', filter.app);
-        if (filter.level) {
-            if (Array.isArray(filter.level)) {
-                countQuery = countQuery.where('level', 'in', filter.level);
+            if (filter.from) {
+                const fromDate = typeof filter.from === 'string' ? new Date(filter.from) : filter.from;
+                clauses.push(eb('timestamp', '>=', fromDate));
             }
-            else {
-                countQuery = countQuery.where('level', '=', filter.level);
+            if (filter.to) {
+                const toDate = typeof filter.to === 'string' ? new Date(filter.to) : filter.to;
+                clauses.push(eb('timestamp', '<=', toDate));
             }
-        }
-        if (filter.search)
-            countQuery = countQuery.where('message', 'like', `%${escapeLike(filter.search)}%`);
-        if (filter.traceId)
-            countQuery = countQuery.where('traceId', '=', filter.traceId);
-        if (filter.from) {
-            const fromDate = typeof filter.from === 'string' ? new Date(filter.from) : filter.from;
-            countQuery = countQuery.where('timestamp', '>=', fromDate);
-        }
-        if (filter.to) {
-            const toDate = typeof filter.to === 'string' ? new Date(filter.to) : filter.to;
-            countQuery = countQuery.where('timestamp', '<=', toDate);
-        }
-        const [entries, countResult] = await Promise.all([
+            // No filter at all is not "match nothing" — the unfiltered view is
+            // the default one. `lit`, not `val`: a bound `true` is a JS boolean,
+            // and better-sqlite3 binds "numbers, strings, bigints, buffers, and
+            // null", so a parameter here would refuse every unfiltered log query
+            // on every slave. `true` as a keyword is understood by both engines.
+            return clauses.length > 0 ? eb.and(clauses) : eb.lit(true);
+        };
+        const query = this.db.selectFrom('logs').selectAll().where(matching);
+        // The total and its breakdown by level, counted by one query: the total
+        // is their sum. The console printed the total beside `getLogStats`'s
+        // levels — the whole table's, filtered by nothing — so on the master
+        // (2026-09-23) «Showing 100 of 48 991» sat beside «error: 42 650» for an
+        // hour that held 149 errors among 9 159 730 rows since 09-09.
+        const levelsQuery = this.db
+            .selectFrom('logs')
+            .select(['level', this.db.fn.countAll().as('count')])
+            .where(matching)
+            .groupBy('level');
+        const [entries, levels] = await Promise.all([
             query.orderBy('timestamp', 'desc').limit(limit).offset(offset).execute(),
-            countQuery.executeTakeFirst(),
+            levelsQuery.execute(),
         ]);
-        const total = Number(countResult?.count ?? 0);
+        const byLevel = levels.map((r) => ({ level: r.level, count: Number(r.count) }));
+        const total = byLevel.reduce((sum, { count }) => sum + count, 0);
         return {
             entries: entries,
             total,
+            byLevel,
             hasMore: offset + limit < total,
         };
     }
+    /**
+     * In-memory counters. No database work, so it is safe to poll.
+     *
+     * `ingestedTotal` counts rows actually written, not rows accepted: a caller
+     * sampling it twice gets a real write rate, and a stalled flush shows as a
+     * rate of zero with a growing `bufferSize` beside it — which is the state
+     * an operator needs to be able to tell from "nothing is being logged".
+     */
+    getIngestionStats() {
+        return {
+            ingestedTotal: this.ingestedTotal,
+            droppedTotal: this.droppedTotal,
+            bufferSize: this.buffer.length,
+        };
+    }
     async getLogStats() {
         const [byApp, byLevel, bounds] = await Promise.all([
             this.db
@@ -220,6 +528,16 @@ export class LogCollectorService extends EventEmitter {
         }
         if (filter.search)
             query = query.where('message', 'like', `%${escapeLike(filter.search)}%`);
+        if (filter.traceId)
+            query = query.where('traceId', '=', filter.traceId);
+        if (filter.nodeId)
+            query = query.where('nodeId', '=', filter.nodeId);
+        if (filter.labels && Object.keys(filter.labels).length > 0) {
+            // Both paths, deliberately: a filter honoured by the paginated query
+            // and not by the live tail is how a viewer changes its answer when you
+            // press Live.
+            query = query.where(jsonContains(this.db, 'labels', filter.labels));
+        }
         // Range filters — critical for efficient live polling (since parameter)
         if (filter.from) {
             const from = typeof filter.from === 'string' ? new Date(filter.from) : filter.from;
@@ -236,6 +554,28 @@ export class LogCollectorService extends EventEmitter {
             .execute();
         return entries.reverse();
     }
+    /**
+     * The last `lines` stored records of one app, as the daemon's `getLogs`
+     * answers — oldest first, each saying which machine it came from.
+     *
+     * `sourceNode` is the row's `nodeId`: the node that synced it, or `null` for
+     * a line this machine captured itself. Under its own key in `data` because
+     * the rest of `data` is the record's fields, and records carry `nodeId`s of
+     * their own (the daemon logs them).
+     */
+    async storedEntries(app, lines) {
+        const rows = await this.getRecentLogs({ app, tail: Math.max(1, Math.min(lines, 10_000)) });
+        return rows.map((row) => {
+            const metadata = typeof row.metadata === 'string' ? safeJson(row.metadata) : row.metadata;
+            return {
+                timestamp: new Date(row.timestamp).getTime(),
+                app: row.app,
+                level: row.level,
+                message: row.message,
+                data: { ...(metadata ?? {}), sourceNode: row.nodeId ?? null },
+            };
+        });
+    }
     // ===========================================================================
     // Buffer Flush
     // ===========================================================================
@@ -244,14 +584,27 @@ export class LogCollectorService extends EventEmitter {
         if (this.flushing || this.buffer.length === 0)
             return;
         this.flushing = true;
-        const batch = this.buffer.splice(0);
+        // Bounded batch — caps the INSERT payload AND the requeue-on-failure so
+        // an outage backlog drains in chunks instead of one giant statement.
+        // The 1s flush timer + threshold-triggered flushes drain the remainder.
+        const batch = this.buffer.splice(0, FLUSH_BATCH_MAX);
         try {
             if (batch.length === 0)
                 return;
+            // Postgres keeps the microseconds that carry capture order, so it gets
+            // the string as sequenced. SQLite (a slave) stores this column as TEXT
+            // and compares it as text, where `.123004Z` sorts BEFORE `.123Z` — its
+            // rows keep the millisecond form every other writer and every bound
+            // there uses. The order still reaches the master: the replicated entry
+            // carries the sequenced string, not this row.
+            this.dialect ??= dialectOf(this.db);
+            const keepsSequence = this.dialect === 'postgres';
             const rows = batch.map((entry) => ({
                 id: randomUUID(),
                 timestamp: entry.timestamp
-                    ? (typeof entry.timestamp === 'string' ? new Date(entry.timestamp) : entry.timestamp)
+                    ? (typeof entry.timestamp === 'string'
+                        ? (keepsSequence ? entry.timestamp : new Date(entry.timestamp))
+                        : entry.timestamp)
                     : new Date(),
                 nodeId: entry.nodeId ?? null,
                 app: entry.app,
@@ -263,14 +616,38 @@ export class LogCollectorService extends EventEmitter {
                 metadata: entry.metadata ? JSON.stringify(entry.metadata) : null,
             }));
             await this.db.insertInto('logs').values(rows).execute();
+            // After the write, never instead of it: what the master receives is
+            // what this node actually holds. Guarded because a sink that throws
+            // must not turn a successful flush into a failed one — the batch is
+            // already persisted and re-queueing it would duplicate the rows.
+            if (this.syncSink) {
+                try {
+                    // The BATCH, not `rows`. `rows` has `labels`/`metadata` already
+                    // serialised for this table, and the master's `ingestLog`
+                    // serialises what it is given — so shipping rows would store
+                    // `"{\"a\":1}"` as a label object on the other side. `LogEntry`
+                    // is the unencoded shape both ends already agree on.
+                    this.syncSink(batch);
+                }
+                catch (err) {
+                    this.emit('sync_error', err);
+                }
+            }
+            if (this.droppedSinceFlush > 0) {
+                this.emit('dropped', this.droppedSinceFlush);
+                this.droppedSinceFlush = 0;
+            }
+            this.ingestedTotal += batch.length;
             this.emit('flushed', batch.length);
         }
         catch (err) {
-            // On failure, put entries back at the front of the buffer
-            // (drop if buffer is too large to avoid unbounded growth)
-            if (this.buffer.length < 10_000) {
-                this.buffer.unshift(...batch);
-            }
+            // On failure, put the batch back at the FRONT of the buffer so order
+            // is preserved. concat (never unshift(...batch)) — spreading a large
+            // batch as arguments overflows the call stack, which is precisely how
+            // the 2026-07-11 daemon crash happened. The ceiling then drops the
+            // oldest overflow.
+            this.buffer = batch.concat(this.buffer);
+            this.enforceCap();
             this.emit('flush_error', err);
         }
         finally {
@@ -287,11 +664,18 @@ export class LogCollectorService extends EventEmitter {
         this.flushTimer.unref();
     }
     async dispose() {
+        // Held records (a dump waiting for its last lines) go into the buffer
+        // before the final flush, not out with the process.
+        this.assembler.dispose();
         this.disposed = true;
         if (this.flushTimer) {
             clearInterval(this.flushTimer);
             this.flushTimer = null;
         }
+        if (this.retentionTimer) {
+            clearTimeout(this.retentionTimer);
+            this.retentionTimer = null;
+        }
         // Wait for any in-progress flush to complete before final flush
         while (this.flushing) {
             await new Promise((r) => setTimeout(r, 50));
@@ -299,5 +683,11 @@ export class LogCollectorService extends EventEmitter {
         // Final flush of remaining buffered entries
         await this.flush();
     }
-}
+};
+LogCollectorService = __decorate([
+    Injectable(),
+    __param(0, Inject(OMNITRON_DB_TOKEN)),
+    __metadata("design:paramtypes", [Function])
+], LogCollectorService);
+export { LogCollectorService };
 //# sourceMappingURL=log-collector.service.js.map