@omnitron-dev/omnitron 0.2.0 → 0.3.0

package/dist/services/sync.service.d.ts

@@ -5,26 +5,58 @@
  * - Every slave daemon collects ALL metrics, logs, events, alerts locally
  * - Data is buffered in a local append-only WAL (write-ahead log) in omnitron-pg
  * - Periodically, slave attempts to push buffered data to master via Netron TCP
- * - On success: WAL entries marked as synced
- * - On failure: exponential backoff, retry later — data is NEVER lost
- * - All sync batches are idempotent (safe to retry)
+ * - On success: the entries the master confirms are marked as synced
+ * - On failure: exponential backoff, retry later — the entries stay pending
+ * - Sync batches are idempotent: the master keeps a ledger of what it took
  *
  * Guarantees:
- * 1. Zero data loss — local WAL persists through daemon restarts
+ * 1. No loss on delivery failure — an entry is released only once the master
+ *    has confirmed it holds it. Bounded by (5): when the buffer is over
+ *    budget, undelivered entries are dropped oldest-first, loudly.
  * 2. Autonomous operation — slave works normally without master
  * 3. Eventual consistency — master sees all data when connectivity restores
  * 4. Conflict-free — operational data is append-only, config uses master-wins
  * 5. Bounded buffer — oldest entries evicted when maxBufferSize reached
  *
+ * Four of these were stated here long before anything implemented them, and
+ * every one of the failures was silent: replication that drops data still
+ * reports a successful cycle, because what would have noticed is what was
+ * missing. Guarantee 1 said "zero data loss" while both transports discarded
+ * whatever the master rejected; the batches were called idempotent with
+ * `batchId` used only inside log strings; and `maxBufferSize` was read from
+ * config, defaulted to 500 MB, and never referenced again.
+ *
+ * Guarantee 1 is now stated as what it can be. "Zero data loss" was not
+ * achievable alongside a bounded buffer, and writing it down did not make it
+ * so — a slave that cannot reach its master eventually has to choose between
+ * dropping data and filling the disk. It drops, oldest first, at warning
+ * level with a count. See `sync-policy.ts`.
+ *
  * WAL table: `sync_buffer` in slave's local omnitron-pg
  * Each entry: { id, category, payload (jsonb), createdAt, syncedAt (null until synced) }
+ * Dedup ledger (master side): `sync_ingested`, keyed on (nodeId, slave entry id)
  */
-import type { Kysely } from 'kysely';
+import type { Kysely, Transaction } from 'kysely';
 import type { ILogger } from '@omnitron-dev/titan/module/logger';
 import type { OmnitronDatabase } from '../database/schema.js';
 import type { ISyncConfig, DaemonRole } from '../config/types.js';
 import type { ISyncStatus } from '../shared/dto/project.js';
+import { type RetentionPlan } from './log-retention.js';
 export type SyncCategory = 'metrics' | 'logs' | 'events' | 'alerts' | 'traces' | 'state';
+/**
+ * Records one ingested remote metric into the store the console reads.
+ *
+ * `node` is separate from `labels` in the signature so it cannot be left out
+ * by a caller building the label map — which is the failure this exists to
+ * prevent.
+ */
+export type MetricsSink = (sample: {
+    node: string;
+    name: string;
+    app: string;
+    labels: Record<string, string>;
+    value: number;
+}) => void;
 export interface SyncEntry {
     category: SyncCategory;
     payload: Record<string, unknown>;
@@ -41,13 +73,49 @@ export interface SyncBatch {
         createdAt: string;
     }>;
 }
+/**
+ * Either the pooled connection or a transaction handle. Ingestion runs
+ * inside the transaction that claims the entry, so it cannot take `this.db`.
+ */
+export type SyncExecutor = Kysely<OmnitronDatabase> | Transaction<OmnitronDatabase>;
+/**
+ * What the master did with a batch.
+ *
+ * `accepted` stays a number so the existing RPC contract is unchanged; the
+ * id lists are the addition. Without them the slave could only learn *how
+ * many* entries survived, never *which* — and so marked all of them
+ * delivered, which is how the "zero data loss" guarantee was losing data.
+ */
+export interface IngestBatchResult {
+    accepted: number;
+    duplicates: number;
+    acceptedIds: string[];
+    duplicateIds: string[];
+    failedIds: string[];
+    /**
+     * Entries the master will never accept, and the slave should release.
+     *
+     * Distinct from `failedIds`, which are worth offering again. An entry the
+     * database refuses on its content — a malformed uuid, a foreign key the
+     * master does not have — answers the same way every time, and leaving it
+     * unacknowledged parks it at the head of the buffer with everything behind
+     * it waiting.
+     */
+    discardedIds: string[];
+}
 export declare class SyncService {
     private readonly db;
     private readonly logger;
     private readonly nodeId;
     private readonly role;
+    /** Set on a master; see `setMetricsSink`. */
+    private metricsSink;
     private readonly config;
     private syncTimer;
+    /** The master's ledger pruning, re-armed after each pass. See `pruneLedger`. */
+    private ledgerTimer;
+    /** Whether this database keeps the buffer's figures; null until asked. See `keptBufferStats`. */
+    private bufferStatsKept;
     private isSyncing;
     private backoff;
     private lastSyncAt;
@@ -60,7 +128,11 @@ export declare class SyncService {
     constructor(db: Kysely<OmnitronDatabase>, logger: ILogger, nodeId: string, role: DaemonRole, syncConfig?: ISyncConfig);
     /**
      * Start the sync loop.
-     * Only runs on slave daemons — master is a no-op.
+     *
+     * On a slave, the buffer's pass: bound it, then try to drain it. On a
+     * master, the ledger's: `sync_ingested` pruned to its dedup window. The
+     * master used to return here, and its half of the pipeline was never
+     * maintained at all.
      */
     start(): void;
     /**
@@ -82,37 +154,297 @@ export declare class SyncService {
      * Returns immediately — never blocks the caller.
      */
     buffer(entry: SyncEntry): Promise<void>;
+    /**
+     * Where ingested remote metrics are recorded, on a master.
+     *
+     * Set by the daemon to `MetricsService.recordTyped` — the store the
+     * console's charts read, with the node as a label. It is the only place a
+     * remote metric goes: the `metrics_raw` table it was also written to had
+     * no reader and no pruning (migration 010). Without a sink, remote metrics
+     * are claimed and go nowhere.
+     *
+     * A sink rather than a constructor dependency: this service is built by
+     * hand in `daemon.ts` with five positional arguments, and a sixth is a
+     * change every caller has to get right at runtime rather than at compile
+     * time.
+     */
+    setMetricsSink(sink: MetricsSink | null): void;
     /**
      * Buffer multiple entries at once (batch optimization).
      */
     bufferBatch(entries: SyncEntry[]): Promise<void>;
     /**
-     * Drain pending buffer entries — called by master via RPC.
-     * Returns a batch of unsynced entries and marks them as synced.
-     * Idempotent: entries are only drained once.
+     * Hand pending entries to the master — called by the master via RPC.
+     *
+     * This used to mark the entries synced before returning them, and called
+     * that idempotent. It is the opposite: a response lost in transit, a
+     * checksum mismatch or a rate-limit rejection left the data marked
+     * delivered on the slave and absent on the master, to be evicted twenty-
+     * four hours later. The caller logged the failure at debug level.
+     *
+     * Entries are released only by `ackDrained`, after the master has them.
+     * Re-draining before that returns the same entries, which is correct and
+     * is what the deduplication ledger on the master side is for.
      */
     drainBuffer(limit?: number): Promise<SyncBatch>;
+    /**
+     * Release entries the master has confirmed it holds.
+     *
+     * Called after `receiveBatch` succeeds. Split from `drainBuffer` so that
+     * everything between the two — the network, the checksum, the ingestion —
+     * can fail without the slave forgetting the data.
+     */
+    ackDrained(ids: string[]): Promise<{
+        released: number;
+    }>;
+    /**
+     * One scheduled pass: bound the buffer, then try to drain it.
+     *
+     * The order is the whole point. `enforceBufferBounds` used to be the last
+     * statement of `syncCycle`'s `try`, which meant it ran on a SUCCESSFUL
+     * cycle and on nothing else — and a successful cycle is the one case where
+     * the buffer is being drained and needs no bound. Every path that leaves it
+     * growing skipped it:
+     *
+     *   - no master configured at all → `syncCycle` returns at its second line;
+     *   - master unreachable → the push throws and control leaves via `catch`;
+     *   - and then the backoff, which saturates at five minutes, returns at the
+     *     third line for most ticks after that.
+     *
+     * So the guarantee in this file's header — "bounded buffer, oldest entries
+     * evicted when maxBufferSize is reached" — held only while it was not
+     * needed. A slave that lost its master still buffered until the disk filled,
+     * which is the exact failure the bound was written to prevent and the
+     * reason it is not read as a replication bug when it happens: what fills is
+     * the disk, and what breaks is everything else sharing it.
+     *
+     * The bound is a local database operation. It needs no master, so it does
+     * not belong behind a check for one.
+     */
+    private syncTick;
     private syncCycle;
+    /**
+     * Grow the retry delay and record why.
+     *
+     * Shared by both ways a cycle can fail — a master that cannot be reached,
+     * and a master that answers no — because the buffer does not care which
+     * one it was, and neither does the disk it is filling.
+     */
+    private noteSyncFailure;
     private fetchPendingBatch;
     private computeChecksum;
+    /**
+     * Push a batch and report what the master kept.
+     *
+     * The return value used to be discarded, and `receiveBatch` swallows
+     * per-entry ingestion failures into a warning — so the caller marked every
+     * entry delivered whether or not any of them arrived. A master response
+     * without the id lists is treated as "all delivered", which is what an
+     * older master would mean by it; that is the only reading that does not
+     * silently retry everything against a peer that cannot tell us otherwise.
+     */
     private pushBatch;
     private markSynced;
     /**
-     * Evict synced entries older than 24h to keep WAL bounded.
+     * Keep the WAL inside `maxBufferSize`.
+     *
+     * Guarantee 5 in the header of this file — "bounded buffer, oldest entries
+     * evicted when maxBufferSize reached" — had no implementation:
+     * `maxBufferSize` was read from config, defaulted to 500 MB, and never
+     * referenced again. The only eviction was of *synced* rows older than a
+     * day, so a slave that lost its master buffered until the disk filled. The
+     * failure lands on everything else sharing that disk, which is why it
+     * would never have been read as a replication bug.
+     *
+     * Two passes. The routine one removes delivered entries past their
+     * retention. The overflow one runs only when the table is still over
+     * budget and removes oldest-first regardless of delivery — at which point
+     * data is being dropped, so it is said at warning level with the count.
      */
-    private evictSynced;
+    private enforceBufferBounds;
+    /**
+     * Delete up to `limit` of the oldest rows of one kind; say how many went.
+     *
+     * By a subquery, not by a list. The ids used to be read into memory and
+     * deleted with `WHERE id IN (…)` — one bound parameter per row, where
+     * SQLite takes 32 766. So the pass worked for a small overflow and refused
+     * exactly the large one, which is the only kind a buffer cut off from its
+     * master produces. Measured on the test node, 2026-09-22: «too many SQL
+     * variables» on 558 passes in a row, 13:03–17:42 UTC, against 4 019 288
+     * rows and 555 MB of payload over a 500 MB budget. The bound bounded
+     * nothing, and said so every thirty seconds in a log nobody on the master
+     * reads.
+     *
+     * Two queries rather than one ordered `syncedAt DESC`, which put NULLs —
+     * the undelivered rows — last on SQLite and FIRST on Postgres, and among
+     * the delivered took the most recently delivered first, against the
+     * comment above it. Both of these are served by `idx_sync_pending
+     * (syncedAt, createdAt)`: delivered rows in the order they were delivered,
+     * which is the order the routine pass retires them in, only sooner; and
+     * undelivered ones in the order they were buffered.
+     */
+    private dropOldest;
+    /**
+     * Size and row counts of the WAL, or null when the database cannot say.
+     *
+     * This asked `pg_total_relation_size('sync_buffer')` — a Postgres function,
+     * about the one table that only ever exists where the database is SQLite.
+     * A slave buffers and a master receives, so the measurement the bound above
+     * depends on threw `no such function: pg_total_relation_size` on every
+     * pass, on every slave. Observed on a provisioned node, every thirty
+     * seconds, immediately behind the `Date`-binding fix that let the pass get
+     * this far:
+     *
+     *     05:58:48  Sync buffer retention pass failed
+     *       error: "no such function: pg_total_relation_size"
+     *
+     * Three defects stacked on one guarantee: the bound ran only on the cycle
+     * that did not need it, then could not bind its own cutoff, and then could
+     * not measure. Each fix revealed the next, and only the last one makes
+     * "bounded buffer" true. The routine delete of synced rows sits before this
+     * call and was unaffected throughout.
+     *
+     * The replacement measures the same quantity in the same terms on both
+     * dialects: the bytes of the buffered payloads. That is deliberately not
+     * the relation's on-disk footprint — it excludes indexes, row headers and
+     * Postgres's TOAST compression — because one definition of `maxBufferSize`
+     * that means the same thing on a master and on a slave is worth more than
+     * two that each drift by whatever their storage engine adds. The payload is
+     * the term that grows.
+     *
+     * `cast(payload as text)` is required by both: the column is `jsonb` on
+     * Postgres, where `length()` has no jsonb form, and TEXT on SQLite, where
+     * the cast costs nothing. `octet_length` rather than `length` because the
+     * second counts CHARACTERS — a buffer of Cyrillic log lines measures at
+     * half its size, and a budget is about bytes. Verified against both engines
+     * (postgres:17-alpine, better-sqlite3 13.0.3 / SQLite 3.53.4).
+     */
+    private bufferStats;
+    /**
+     * The buffer's figures as its triggers keep them, or null when this
+     * database keeps none (see `SlaveStorageService.createBufferStats`).
+     *
+     * One row instead of a pass over the table: 395–399 ms on the test node's
+     * 4 million rows, every thirty seconds, on the daemon's event loop. Null
+     * sends `bufferStats` back to that pass, and is remembered, so a database
+     * without the row — one this service did not create — costs one refused
+     * query per process rather than one per pass.
+     */
+    private keptBufferStats;
     /**
      * Receive a sync batch from a slave.
      * Called on the master daemon via RPC.
      * Ingests data into the master's omnitron-pg.
      */
-    receiveBatch(batch: SyncBatch): Promise<{
-        accepted: number;
-    }>;
-    private ingestMetric;
-    private ingestLog;
-    private ingestAlert;
-    private ingestTrace;
+    receiveBatch(batch: SyncBatch): Promise<IngestBatchResult>;
+    /**
+     * Take a batch: as one transaction, or — when the database refuses it as
+     * a whole — one entry at a time, which is what finds the entry it refused.
+     *
+     * Every batch used to take the second road. Each entry was claimed and
+     * ingested in a transaction of its own, so that one entry the database
+     * refuses costs that entry and not its thousand neighbours — and so every
+     * batch cost a thousand `BEGIN … COMMIT`s, on a master whose omnitron-pg
+     * runs `synchronous_commit = on`: a WAL flush per entry, a thousand per
+     * pull. While the machine was idle that was 800–1 100 entries a second
+     * against 38 produced on the test node. Under load a pull outlasted the
+     * 15 s heartbeat, and the next heartbeat started another over the same
+     * entries (see `pullSyncData`). Measured 2026-09-22: nine pulls of the
+     * same 7 918 entries finishing within 122 ms of each other.
+     *
+     * The isolation is kept, and paid for only where it buys something: a
+     * batch the database refuses — a malformed uuid, a foreign key the master
+     * does not have — is retried entry by entry, and that path classifies
+     * each refusal exactly as it always has. The fast path is never worse
+     * than the old one; it falls back to it.
+     */
+    private ingest;
+    /**
+     * The whole batch in one transaction: the claims in one statement, each
+     * destination table's rows in one more.
+     *
+     * An id that appears twice in a batch is taken once and its second
+     * occurrence reported a duplicate — what the per-entry path does, because
+     * there the first occurrence's claim is committed before the second's is
+     * tried.
+     */
+    private ingestAsOne;
+    /**
+     * One transaction per entry: the road for a batch the database refused.
+     *
+     * Slow on purpose. Each entry succeeds or fails on its own, so the one the
+     * database will never store is named, discarded and released, and a
+     * failure that may pass is left for the next pull — while everything else
+     * in the batch is taken.
+     */
+    private ingestEntryByEntry;
+    /**
+     * Claim entries for this node; the ids that come back are the new ones.
+     *
+     * Sorted, so that two transactions claiming overlapping batches take the
+     * ledger's rows in the same order and cannot deadlock on each other.
+     */
+    private claimAll;
+    /**
+     * Store claimed entries in the tables the console reads, one statement per
+     * table (per `MAX_BOUND_PARAMETERS` rows). Both roads write through here,
+     * so an entry lands as the same row whichever one took it.
+     */
+    private writeRows;
+    /**
+     * Hand claimed remote metrics to the store the console reads.
+     *
+     * The sink records each sample into titan-metrics, which is what the
+     * console's `getSnapshot` and `querySeries` read, tagged with `node` so a
+     * remote reading is a label dimension rather than a second table. The
+     * `metrics_raw` table these were also written to was kept «until the
+     * console is observed showing the label»; it was observed (2026-09-23: 19
+     * series labelled with the test node), and the table — written ~0.93
+     * million rows a day, read and pruned by nothing — is gone (migration 010).
+     * History now lives in titan-metrics' store (in memory, 7 days, lost with a
+     * master restart); how long it should be kept is that store's question.
+     *
+     * AFTER the commit. The sink is not part of the transaction, and it used to
+     * be called inside it, before the row was written: an entry whose write
+     * then failed was recorded anyway, and recorded again by the retry that
+     * stored it. With a batch rolled back and retried entry by entry, that
+     * would have been every metric in it.
+     */
+    private recordMetrics;
+    /**
+     * Record the entry as taken and ingest it, atomically.
+     *
+     * Returns `'duplicate'` when the master already holds it — a retry after a
+     * lost acknowledgement, which is delivery, not failure. Throws when
+     * ingestion fails, leaving nothing recorded.
+     */
+    private claimAndIngest;
+    private claimAndIngestOnce;
+    /**
+     * Arm the next ledger pass: in an hour, or in a minute when the last one
+     * hit its ceiling and more is waiting. Chained rather than an interval, so
+     * the cadence is decided with the last pass's result in hand.
+     */
+    private scheduleLedgerPrune;
+    /**
+     * Forget claims older than the dedup window (`LEDGER_WINDOW_DAYS`), in
+     * bounded batches, and say how many went.
+     *
+     * `sync_ingested` was written on every entry and pruned by nothing: 9.9
+     * million rows, 2.2 GB, and ~3.3 million more a day per node, beside an
+     * index on `ingestedAt` whose migration says it is there «for pruning». A
+     * claim is needed only while its node may offer the entry again — see the
+     * window's own comment for why a day covers that.
+     *
+     * By a subquery on the key, like the buffer's eviction, so no statement
+     * binds a parameter per row; `batchSize` rows at a time, so no statement
+     * holds the ingest path's inserts behind it for long.
+     *
+     * Said at info whenever it removed something, with the oldest claim left:
+     * a month from now, «the ledger is stable» must be tellable from «the
+     * pruner never ran» by reading the log rather than the code.
+     */
+    pruneLedger(plan?: RetentionPlan | null): Promise<number>;
     private checkRateLimit;
     /** Remove rate limit entries older than 2 minutes to prevent memory leaks. */
     private cleanupRateLimits;

package/dist/services/sync.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sync.service.d.ts","sourceRoot":"","sources":["../../src/services/sync.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAM5D,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEzF,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,YAAY,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7G;AA8BD,qBAAa,WAAW;IAgBpB,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAlBvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAoD;IACnE,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,QAAQ,CAAS;IAEzB,4DAA4D;IAC5D,OAAO,CAAC,YAAY,CAAyF;IAE7G,kDAAkD;IAClD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqC;gBAG7C,EAAE,EAAE,MAAM,CAAC,gBAAgB,CAAC,EAC5B,MAAM,EAAE,OAAO,EACf,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,UAAU,EACjC,UAAU,CAAC,EAAE,WAAW;IAoB1B;;;OAGG;IACH,KAAK,IAAI,IAAI;IAsBb;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB3B;;;OAGG;IACH,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI;IAMzG;;OAEG;IACH,qBAAqB,IAAI,IAAI;IAS7B;;;;OAIG;IACG,MAAM,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAc7C;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBtD;;;;OAIG;IACG,WAAW,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;YAavC,SAAS;YAyDT,iBAAiB;YAwBjB,eAAe;YAOf,SAAS;YAMT,UAAU;IASxB;;OAEG;YACW,WAAW;IAiBzB;;;;OAIG;IACG,YAAY,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;YA+DrD,YAAY;YAWZ,SAAS;YAcT,WAAW;YAaX,WAAW;IAoBzB,OAAO,CAAC,cAAc;IActB,8EAA8E;IAC9E,OAAO,CAAC,iBAAiB;IAanB,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC;CA8BxC"}
+{"version":3,"file":"sync.service.d.ts","sourceRoot":"","sources":["../../src/services/sync.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAc,MAAM,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAe5D,OAAO,EAAiC,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAMvF,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEzF;;;;;;GAMG;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,MAAM,EAAE;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;CACf,KAAK,IAAI,CAAC;AAEX,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,YAAY,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7G;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,gBAAgB,CAAC,GAAG,WAAW,CAAC,gBAAgB,CAAC,CAAC;AAEpF;;;;;;;GAOG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB;;;;;;;;OAQG;IACH,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAiID,qBAAa,WAAW;IAuBpB,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAzBvB,6CAA6C;IAC7C,OAAO,CAAC,WAAW,CAA4B;IAE/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,SAAS,CAA+B;IAChD,gFAAgF;IAChF,OAAO,CAAC,WAAW,CAA+B;IAClD,iGAAiG;IACjG,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAoD;IACnE,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,QAAQ,CAAS;IAEzB,4DAA4D;IAC5D,OAAO,CAAC,YAAY,CAAyF;IAE7G,kDAAkD;IAClD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqC;IAEhE,YACmB,EAAE,EAAE,MAAM,CAAC,gBAAgB,CAAC,EAC5B,MAAM,EAAE,OAAO,EACf,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,UAAU,EACjC,UAAU,CAAC,EAAE,WAAW,EAczB;IAMD;;;;;;;OAOG;IACH,KAAK,IAAI,IAAI,CAyBZ;IAED;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAoB1B;IAED;;;OAGG;IACH,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAIxG;IAED;;OAEG;IACH,qBAAqB,IAAI,IAAI,CAG5B;IAMD;;;;OAIG;IACG,MAAM,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAe5C;IAED;;;;;;;;;;;;;OAaG;IACH,cAAc,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI,GAAG,IAAI,CAE7C;IAED;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBrD;IAMD;;;;;;;;;;;;OAYG;IACG,WAAW,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAEpD;IAED;;;;;;OAMG;IACG,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAK7D;IAMD;;;;;;;;;;;;;;;;;;;;;;;OAuBG;YACW,QAAQ;YAMR,SAAS;IA0EvB;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;YAgBT,iBAAiB;YA0BjB,eAAe;IAO7B;;;;;;;;;OASG;YACW,SAAS;YAqBT,UAAU;IASxB;;;;;;;;;;;;;;;OAeG;YACW,mBAAmB;IAiDjC;;;;;;;;;;;;;;;;;;;;OAoBG;YACW,UAAU;IAqBxB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkCG;YACW,WAAW;IAqBzB;;;;;;;;;OASG;YACW,eAAe;IA2B7B;;;;OAIG;IACG,YAAY,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkE/D;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;YACW,MAAM;IAiBpB;;;;;;;;OAQG;YACW,WAAW;IAqBzB;;;;;;;OAOG;YACW,kBAAkB;IA6ChC;;;;;OAKG;YACW,QAAQ;IAetB;;;;OAIG;YACW,SAAS;IAwDvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,aAAa;IAoBrB;;;;;;OAMG;YACW,cAAc;YAkCd,kBAAkB;IAuBhC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IAc3B;;;;;;;;;;;;;;;;;OAiBG;IACG,WAAW,CAAC,IAAI,GAAE,aAAa,GAAG,IAAwC,GAAG,OAAO,CAAC,MAAM,CAAC,CA2CjG;IAMD,OAAO,CAAC,cAAc;IActB,8EAA8E;IAC9E,OAAO,CAAC,iBAAiB;IAanB,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,CA6BtC;CACF"}