@omnitron-dev/omnitron 0.2.0 → 0.3.1

package/dist/services/log-collector.service.d.ts

@@ -7,78 +7,186 @@
  * Designed to be wired into orchestrator.onAppLog() to capture all child
  * process log lines, parse pino JSON, and persist to the logs table.
  */
-import type { Kysely } from 'kysely';
+import { type Kysely } from 'kysely';
 import type { OmnitronDatabase } from '../database/schema.js';
 import { EventEmitter } from 'node:events';
-export interface LogEntry {
-    app: string;
-    level: string;
-    message: string;
-    timestamp?: Date | string;
-    nodeId?: string;
-    labels?: Record<string, unknown>;
-    traceId?: string;
-    spanId?: string;
-    metadata?: Record<string, unknown>;
-}
-export interface LogQueryFilter {
-    app?: string | undefined;
-    level?: string | string[] | undefined;
-    search?: string | undefined;
-    labels?: Record<string, string> | undefined;
-    traceId?: string | undefined;
-    from?: Date | string | undefined;
-    to?: Date | string | undefined;
-    limit?: number | undefined;
-    offset?: number | undefined;
-}
-export interface LogQueryResult {
-    entries: LogEntryRow[];
-    total: number;
-    hasMore: boolean;
-}
-export interface LogEntryRow {
-    id: string;
-    timestamp: Date;
-    nodeId: string | null;
-    app: string;
-    level: string;
-    message: string;
-    labels: Record<string, unknown> | null;
-    traceId: string | null;
-    spanId: string | null;
-    metadata: Record<string, unknown> | null;
-}
-export interface LogStats {
-    byApp: Array<{
-        app: string;
-        count: number;
-    }>;
-    byLevel: Array<{
-        level: string;
-        count: number;
-    }>;
-    totalCount: number;
-    oldestEntry: Date | null;
-    newestEntry: Date | null;
-}
+import type { LogEntry, LogQueryFilter, LogEntryRow, LogQueryResult, LogStats } from '../shared/dto/logs.js';
+import type { LogEntryDto } from '../config/types.js';
+export type { LogEntry, LogQueryFilter, LogEntryRow, LogQueryResult, LogStats } from '../shared/dto/logs.js';
+/**
+ * A millisecond's capture sequence, written as its microseconds.
+ *
+ * Records captured in one millisecond were stored with equal timestamps and
+ * read back in whatever order the database chose for the tie — on the master
+ * a uuid's, which is to say none — so the lines of one dump, and bursts of
+ * records, came back shuffled. The column is `timestamptz`, which keeps
+ * microseconds, and no source stamps finer than a millisecond: so the
+ * microseconds carry the order the records were captured in, 0–999 within
+ * their millisecond. `ORDER BY timestamp` then IS capture order, with no
+ * migration and no second sort key — and the string travels to the master
+ * inside the replicated entry, whose ingest passes it through unchanged.
+ *
+ * The millisecond itself stays the record's own. Past 999 records in one
+ * millisecond the rest share .999 — a tie, never a later time.
+ */
+export declare function timestampWithSequence(ms: number, sequence: number): string;
 export declare class LogCollectorService extends EventEmitter {
     private readonly db;
     private buffer;
     private flushTimer;
     private flushing;
     private disposed;
+    /** Entries dropped to the MAX_BUFFER ceiling since the last successful flush. */
+    private droppedSinceFlush;
+    /**
+     * Lifetime counters, in memory.
+     *
+     * The console's "Log Ingestion" card was a hardcoded `0 lines/s`, and the
+     * only alternative source is `getLogStats`, which runs three `count(*)`
+     * over the whole table — 22.5 million rows on this host, so not something
+     * to poll. A counter costs an addition per flush and answers the question
+     * the card was asking.
+     *
+     * `droppedTotal` matters more than the rate. The buffer sheds its oldest
+     * entries at the MAX_BUFFER ceiling during a database outage, and until
+     * now that happened with no counter anyone could read — logs vanished and
+     * the only trace was an event nobody was listening to.
+     */
+    private ingestedTotal;
+    private droppedTotal;
+    /**
+     * Where persisted batches also go, on a slave.
+     *
+     * A slave keeps its logs locally and replicates them to the master; the
+     * replication half was fully built — a WAL, a dedup ledger on the master,
+     * backoff, eviction — and `SyncService.buffer`/`bufferBatch` had **zero
+     * callers**. Everything downstream of them worked and moved nothing,
+     * because nothing upstream ever enqueued. This is the missing producer.
+     *
+     * A sink rather than a constructor dependency, for the reason the
+     * retention setter below gives: this service is registered with `useClass`
+     * and takes the database as its only injected argument, and adding a
+     * second is a change to module wiring that fails at runtime rather than at
+     * compile time.
+     *
+     * Set only on a slave. `bufferBatch` is itself a no-op for any other role,
+     * so a mis-wire costs nothing, but not wiring it on a master also avoids
+     * the work.
+     */
+    private syncSink;
+    private retentionTimer;
+    /** Days of history to keep in the table. Zero or less disables pruning. */
+    private retentionDays;
+    /**
+     * Set alongside retention rather than injected.
+     *
+     * The service is registered with `useClass` and takes the database as its
+     * only constructor dependency; adding a second is a change to the module's
+     * wiring, and a wrong one there fails at runtime rather than at compile
+     * time — twice today already. One setter, one call site, nothing to get out
+     * of order.
+     */
+    private logger;
+    /** Lines become records here exactly as they do for the files — `log-line.ts`. */
+    private readonly assembler;
+    /** Next capture sequence per millisecond — see `timestampWithSequence`. */
+    private readonly sequenceByMs;
+    private dialect;
     constructor(db: Kysely<OmnitronDatabase>);
+    /**
+     * Turn on table retention.
+     *
+     * Off until called, because the number belongs to the daemon config and
+     * this service is constructed before it is read. A default chosen here
+     * would be a policy decision made in the wrong place — and a wrong one
+     * deletes an operator's history.
+     */
+    /**
+     * Replicate persisted batches to the master (slave role only).
+     *
+     * Called with the entries that REACHED the table, not the ones that were
+     * offered: a flush that fails puts its batch back at the front of the
+     * buffer and retries, and queueing on the attempt would send every retried
+     * line twice.
+     */
+    setSyncSink(sink: ((entries: LogEntry[]) => void) | null): void;
+    setRetentionDays(days: number, logger?: LogCollectorService['logger']): void;
+    /**
+     * Arm the next retention pass.
+     *
+     * The per-pass ceiling exists to keep each statement short; it should not
+     * also decide how long a backlog takes to clear. A pass that hits its
+     * ceiling means there is more waiting, so the next one comes in a minute
+     * rather than an hour — thirteen million stale rows drain in half an hour
+     * instead of a day, with every individual statement still bounded. A pass
+     * that finishes early returns to the hourly cadence.
+     *
+     * `setTimeout` chained rather than `setInterval`, so the cadence is a
+     * decision made after each pass with its result in hand.
+     */
+    private scheduleRetention;
+    /**
+     * Delete rows past the retention window, in bounded batches.
+     *
+     * Batched because a single `DELETE` over 13 GB holds its transaction for
+     * the duration and blocks the flush path behind it — a retention pass would
+     * show up as the log pipeline stalling, which is the opposite of the point.
+     */
+    pruneOldLogs(): Promise<number>;
+    /** Drop the oldest overflow so the buffer never exceeds MAX_BUFFER. */
+    private enforceCap;
     /** Ingest a single log entry into the buffer */
     ingestLog(entry: LogEntry): void;
     /** Ingest a batch of log entries */
     ingestBatch(entries: LogEntry[]): void;
     /**
-     * Parse a raw pino JSON log line from a child process and ingest it.
+     * Take one line of an app's output (or of the daemon's own, as `omnitron`).
      * This is the primary integration point with orchestrator.onAppLog().
+     *
+     * The line is read by the one classifier (`log-line.ts`) and joined into a
+     * record there. What the table held before: a line that was not JSON became
+     * a row at level `info`, and every indented line of a dump a row of its own
+     * — on the master, payments from the test node: 3 `error` rows, 1 264 995
+     * `info`, 1 216 292 of those dump fragments.
      */
     ingestPinoLine(appName: string, line: string): void;
+    /** One record, as a row. */
+    private ingestRecord;
+    /** The record's millisecond, and its place among the records captured in it. */
+    private sequenced;
+    /**
+     * The two filters the schema was built for and no query used.
+     *
+     * `logs.nodeId` is a uuid column that `SyncService.ingestLog` fills for every
+     * entry a slave sends up, and `logs.labels` is jsonb carrying a GIN index
+     * that `001_initial_schema` created with the intended query written in its
+     * own comment: `WHERE labels @> '{"env":"prod"}'`. Neither `queryLogs` nor
+     * `getRecentLogs` referenced either one.
+     *
+     * The visible consequence was in the console: its cluster node filter put the
+     * node id inside `labels`, sent it, and the daemon dropped the whole field —
+     * so selecting a node narrowed nothing while the chip stayed lit. A filter
+     * that silently does nothing is worse than one that is missing, because the
+     * operator reads the unnarrowed result as an answer about that node.
+     *
+     * Applied in BOTH paths deliberately. A filter honoured by the paginated
+     * query and not by the live tail is how a viewer changes its answer when you
+     * press Live.
+     */
     queryLogs(filter: LogQueryFilter): Promise<LogQueryResult>;
+    /**
+     * In-memory counters. No database work, so it is safe to poll.
+     *
+     * `ingestedTotal` counts rows actually written, not rows accepted: a caller
+     * sampling it twice gets a real write rate, and a stalled flush shows as a
+     * rate of zero with a growing `bufferSize` beside it — which is the state
+     * an operator needs to be able to tell from "nothing is being logged".
+     */
+    getIngestionStats(): {
+        ingestedTotal: number;
+        droppedTotal: number;
+        bufferSize: number;
+    };
     getLogStats(): Promise<LogStats>;
     /**
      * Get recent logs for real-time tailing.
@@ -88,6 +196,16 @@ export declare class LogCollectorService extends EventEmitter {
     getRecentLogs(filter: LogQueryFilter & {
         tail?: number;
     }): Promise<LogEntryRow[]>;
+    /**
+     * The last `lines` stored records of one app, as the daemon's `getLogs`
+     * answers — oldest first, each saying which machine it came from.
+     *
+     * `sourceNode` is the row's `nodeId`: the node that synced it, or `null` for
+     * a line this machine captured itself. Under its own key in `data` because
+     * the rest of `data` is the record's fields, and records carry `nodeId`s of
+     * their own (the daemon logs them).
+     */
+    storedEntries(app: string, lines: number): Promise<LogEntryDto[]>;
     /** Flush buffered entries to the database */
     flush(): Promise<void>;
     private startFlushTimer;

package/dist/services/log-collector.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"log-collector.service.d.ts","sourceRoot":"","sources":["../../src/services/log-collector.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,MAAM,WAAW,QAAQ;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IAC5C,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,IAAI,CAAC,EAAE,IAAI,GAAG,MAAM,GAAG,SAAS,CAAC;IACjC,EAAE,CAAC,EAAE,IAAI,GAAG,MAAM,GAAG,SAAS,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,IAAI,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACvC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1C;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC7C,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;CAC1B;AA2BD,qBAAa,mBAAoB,SAAQ,YAAY;IAMvC,OAAO,CAAC,QAAQ,CAAC,EAAE;IAL/B,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;gBAEI,EAAE,EAAE,MAAM,CAAC,gBAAgB,CAAC;IASzD,gDAAgD;IAChD,SAAS,CAAC,KAAK,EAAE,QAAQ,GAAG,IAAI;IAUhC,oCAAoC;IACpC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI;IAUtC;;;OAGG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IA8C7C,SAAS,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;IA2E1D,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAmCtC;;;;OAIG;IACG,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAsCvF,6CAA6C;IACvC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA2C5B,OAAO,CAAC,eAAe;IAOjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAgB/B"}
+{"version":3,"file":"log-collector.service.d.ts","sourceRoot":"","sources":["../../src/services/log-collector.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,KAAK,MAAM,EAAyD,MAAM,QAAQ,CAAC;AAI5F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAC7G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAOtD,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAM7G;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAG1E;AA2DD,qBACa,mBAAoB,SAAQ,YAAY;IAoEZ,OAAO,CAAC,QAAQ,CAAC,EAAE;IAnE1D,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,iFAAiF;IACjF,OAAO,CAAC,iBAAiB,CAAK;IAC9B;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,YAAY,CAAK;IAEzB;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,QAAQ,CAAgD;IAEhE,OAAO,CAAC,cAAc,CAA+B;IACrD,2EAA2E;IAC3E,OAAO,CAAC,aAAa,CAAK;IAC1B;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM,CAGE;IAEhB,kFAAkF;IAClF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsE;IAChG,2EAA2E;IAC3E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA6B;IAC1D,OAAO,CAAC,OAAO,CAA2B;IAE1C,YAAwD,EAAE,EAAE,MAAM,CAAC,gBAAgB,CAAC,EAGnF;IAED;;;;;;;OAOG;IACH;;;;;;;OAOG;IACH,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAE9D;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,mBAAmB,CAAC,QAAQ,CAAC,GAAG,IAAI,CAuB3E;IAED;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB;IAgBzB;;;;;;OAMG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAmCpC;IAMD,uEAAuE;IACvE,OAAO,CAAC,UAAU;IASlB,gDAAgD;IAChD,SAAS,CAAC,KAAK,EAAE,QAAQ,GAAG,IAAI,CAS/B;IAED,oCAAoC;IACpC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI,CAWrC;IAED;;;;;;;;;OASG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAGlD;IAED,4BAA4B;IAC5B,OAAO,CAAC,YAAY;IAgCpB,gFAAgF;IAChF,OAAO,CAAC,SAAS;IAejB;;;;;;;;;;;;;;;;;;OAkBG;IACG,SAAS,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAkF/D;IAED;;;;;;;OAOG;IACH,iBAAiB,IAAI;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAMvF;IAEK,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC,CAiCrC;IAED;;;;OAIG;IACG,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAyCtF;IAED;;;;;;;;OAQG;IACG,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAYtE;IAMD,6CAA6C;IACvC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA2E3B;IAMD,OAAO,CAAC,eAAe;IAOjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAuB7B;CACF"}