@omnitron-dev/omnitron 0.3.0 → 0.3.2

package/dist/services/sync.service.js

@@ -1,45 +1,12 @@
-/**
- * SyncService — Ultra-reliable slave→master data replication
- *
- * Architecture:
- * - Every slave daemon collects ALL metrics, logs, events, alerts locally
- * - Data is buffered in a local append-only WAL (write-ahead log) in omnitron-pg
- * - Periodically, slave attempts to push buffered data to master via Netron TCP
- * - On success: the entries the master confirms are marked as synced
- * - On failure: exponential backoff, retry later — the entries stay pending
- * - Sync batches are idempotent: the master keeps a ledger of what it took
- *
- * Guarantees:
- * 1. No loss on delivery failure — an entry is released only once the master
- *    has confirmed it holds it. Bounded by (5): when the buffer is over
- *    budget, undelivered entries are dropped oldest-first, loudly.
- * 2. Autonomous operation — slave works normally without master
- * 3. Eventual consistency — master sees all data when connectivity restores
- * 4. Conflict-free — operational data is append-only, config uses master-wins
- * 5. Bounded buffer — oldest entries evicted when maxBufferSize reached
- *
- * Four of these were stated here long before anything implemented them, and
- * every one of the failures was silent: replication that drops data still
- * reports a successful cycle, because what would have noticed is what was
- * missing. Guarantee 1 said "zero data loss" while both transports discarded
- * whatever the master rejected; the batches were called idempotent with
- * `batchId` used only inside log strings; and `maxBufferSize` was read from
- * config, defaulted to 500 MB, and never referenced again.
- *
- * Guarantee 1 is now stated as what it can be. "Zero data loss" was not
- * achievable alongside a bounded buffer, and writing it down did not make it
- * so — a slave that cannot reach its master eventually has to choose between
- * dropping data and filling the disk. It drops, oldest first, at warning
- * level with a count. See `sync-policy.ts`.
- *
- * WAL table: `sync_buffer` in slave's local omnitron-pg
- * Each entry: { id, category, payload (jsonb), createdAt, syncedAt (null until synced) }
- * Dedup ledger (master side): `sync_ingested`, keyed on (nodeId, slave entry id)
- */
+import { assertLogDurability } from '../database/log-durability.js';
+import {} from '@omnitron-dev/titan-metrics';
 import { toIsoUtc } from '../database/sqlite-date-binding.js';
 import { describeError } from '../shared/describe-error.js';
-import { classifyIngestFailure, deliveredIds, sweepMadeProgress, planEviction, SYNCED_RETENTION_MS, LEDGER_WINDOW_DAYS, LEDGER_FIRST_PASS_DELAY_MS, LEDGER_INTERVAL_MS, LEDGER_BACKLOG_INTERVAL_MS, } from './sync-policy.js';
+import { readReplicationEvidence, receiptGroups } from './replication-evidence.js';
+import { acknowledgeReconciliation, readUnverifiedMetrics, reconcileMetricPage, reconciliationSummaries, reconciledMetricIds, replicatedMetric } from './replication-reconciliation.js';
+import { classifyIngestFailure, deliveredIds, planEviction, SYNCED_RETENTION_MS, LEDGER_WINDOW_DAYS, LEDGER_FIRST_PASS_DELAY_MS, LEDGER_INTERVAL_MS, LEDGER_BACKLOG_INTERVAL_MS, } from './sync-policy.js';
 import { planRetention, batchesPerPass } from './log-retention.js';
+import { historyCoverage } from './replication-history.js';
 /** Max batches per node per minute */
 const RATE_LIMIT_PER_MINUTE = 60;
 /**
@@ -137,21 +104,21 @@ export class SyncService {
     logger;
     nodeId;
     role;
-    /** Set on a master; see `setMetricsSink`. */
-    metricsSink = null;
+    metricsStorage = null;
+    metricsHistoryMaxAgeMs = 7 * 86400_000;
+    metricsRegistry = null;
     config;
     syncTimer = null;
+    outboxMaintenance = null;
     /** The master's ledger pruning, re-armed after each pass. See `pruneLedger`. */
     ledgerTimer = null;
     /** Whether this database keeps the buffer's figures; null until asked. See `keptBufferStats`. */
     bufferStatsKept = null;
-    isSyncing = false;
-    backoff = { attempt: 0, nextRetryAt: 0 };
+    evidenceKept = null;
     lastSyncAt = null;
-    lastError = null;
+    /** At most 60 one-second buckets, independent of batch size and polling rate. */
+    deliveries = new Map();
     disposed = false;
-    /** Master RPC invoke function — injected at connect time */
-    masterInvoke = null;
     /** Rate limiter state per nodeId (master-side) */
     rateLimits = new Map();
     constructor(db, logger, nodeId, role, syncConfig) {
@@ -162,28 +129,17 @@ export class SyncService {
         this.config = {
             interval: syncConfig?.interval ?? 30_000,
             batchSize: syncConfig?.batchSize ?? 1000,
-            backoff: {
-                initial: syncConfig?.backoff?.initial ?? 5_000,
-                max: syncConfig?.backoff?.max ?? 300_000, // 5 min max
-                factor: syncConfig?.backoff?.factor ?? 2,
-            },
             categories: syncConfig?.categories ?? ['metrics', 'logs', 'events', 'alerts', 'traces', 'state'],
-            bufferPath: syncConfig?.bufferPath ?? '~/.omnitron/sync-buffer/',
             maxBufferSize: syncConfig?.maxBufferSize ?? 500 * 1024 * 1024, // 500MB
         };
     }
     // ===========================================================================
     // Lifecycle
     // ===========================================================================
-    /**
-     * Start the sync loop.
-     *
-     * On a slave, the buffer's pass: bound it, then try to drain it. On a
-     * master, the ledger's: `sync_ingested` pruned to its dedup window. The
-     * master used to return here, and its half of the pipeline was never
-     * maintained at all.
-     */
+    /** Bound the worker outbox even while offline. The master periodically prunes its deduplication ledger. */
     start() {
+        if (this.disposed || this.syncTimer || this.ledgerTimer)
+            return;
         if (this.role === 'master') {
             this.scheduleLedgerPrune(LEDGER_FIRST_PASS_DELAY_MS);
             return;
@@ -191,21 +147,14 @@ export class SyncService {
         this.logger.info({ interval: this.config.interval, batchSize: this.config.batchSize, categories: this.config.categories }, 'Starting sync service');
         this.syncTimer = setInterval(() => {
             this.syncTick().catch((err) => {
-                this.logger.error({ error: err.message }, 'Sync cycle failed');
+                this.logger.error({ error: err.message }, 'Outbox maintenance failed');
             });
         }, this.config.interval);
         this.syncTimer.unref();
-        // Initial sync attempt — reported like every other one. The recurring
-        // call four lines above logs its failure; this one swallowed it, and it
-        // is the more informative of the two: it is the first evidence that a
-        // slave can reach its master at all.
         this.syncTick().catch((err) => {
-            this.logger.error({ error: err.message }, 'Initial sync cycle failed');
+            this.logger.error({ error: err.message }, 'Initial outbox maintenance failed');
         });
     }
-    /**
-     * Stop the sync loop and flush remaining buffer.
-     */
     async stop() {
         this.disposed = true;
         if (this.syncTimer) {
@@ -216,31 +165,7 @@ export class SyncService {
             clearTimeout(this.ledgerTimer);
             this.ledgerTimer = null;
         }
-        // Final sync attempt before shutdown
-        if (this.role === 'slave' && this.masterInvoke) {
-            try {
-                await this.syncCycle();
-            }
-            catch {
-                this.logger.warn('Final sync attempt failed — data preserved in local WAL');
-            }
-        }
-    }
-    /**
-     * Set the master RPC connection.
-     * Called when slave establishes Netron TCP connection to master.
-     */
-    setMasterConnection(invoke) {
-        this.masterInvoke = invoke;
-        this.backoff = { attempt: 0, nextRetryAt: 0 };
-        this.logger.info('Master connection established — sync enabled');
-    }
-    /**
-     * Clear master connection (disconnect/failure).
-     */
-    clearMasterConnection() {
-        this.masterInvoke = null;
-        this.logger.warn('Master connection lost — buffering locally');
+        await this.outboxMaintenance;
     }
     // ===========================================================================
     // Buffer Operations (write to local WAL)
@@ -265,34 +190,45 @@ export class SyncService {
             this.logger.error({ nodeId: this.nodeId, category: entry.category, error: err.message }, 'Failed to buffer sync entry — it will not reach the master');
         }
     }
-    /**
-     * Where ingested remote metrics are recorded, on a master.
-     *
-     * Set by the daemon to `MetricsService.recordTyped` — the store the
-     * console's charts read, with the node as a label. It is the only place a
-     * remote metric goes: the `metrics_raw` table it was also written to had
-     * no reader and no pruning (migration 010). Without a sink, remote metrics
-     * are claimed and go nowhere.
-     *
-     * A sink rather than a constructor dependency: this service is built by
-     * hand in `daemon.ts` with five positional arguments, and a sixth is a
-     * change every caller has to get right at runtime rather than at compile
-     * time.
-     */
-    setMetricsSink(sink) {
-        this.metricsSink = sink;
+    /** Optional post-commit exposition observer. It is never the durable destination. */
+    setMetricsRegistry(registry) {
+        this.metricsRegistry = registry;
+    }
+    /** Required for metric ingestion. The adapter writes inside the receipt transaction. */
+    setMetricsStorage(storage, historyMaxAgeMs = 7 * 86400_000) {
+        if (!Number.isSafeInteger(historyMaxAgeMs) || historyMaxAgeMs < 1)
+            throw new Error('Invalid metrics history horizon');
+        this.metricsStorage = storage;
+        this.metricsHistoryMaxAgeMs = historyMaxAgeMs;
+    }
+    getReconciliationSummaries() { return reconciliationSummaries(this.db); }
+    getHistoryCoverage() { return historyCoverage(this.db); }
+    readUnverifiedMetrics(options) {
+        if (this.role !== 'slave')
+            throw new Error('Only a node owns retained replication entries');
+        return readUnverifiedMetrics(this.db, this.nodeId, options);
+    }
+    reconcileMetrics(page, restore) {
+        if (this.role !== 'master' || !this.metricsStorage)
+            throw new Error('Reconciliation requires a master with persistent metrics');
+        return reconcileMetricPage(this.db, this.metricsStorage, page, { restore, maxAgeMs: this.metricsHistoryMaxAgeMs });
+    }
+    acknowledgeReconciliation(proofs) {
+        if (this.role !== 'slave')
+            throw new Error('Only a node can settle its replication evidence');
+        return acknowledgeReconciliation(this.db, this.nodeId, proofs);
     }
     /**
      * Buffer multiple entries at once (batch optimization).
      */
-    async bufferBatch(entries) {
+    async bufferBatch(entries, transaction) {
         if (this.role !== 'slave' || entries.length === 0)
             return;
         const filteredEntries = entries.filter((e) => this.config.categories.includes(e.category));
         if (filteredEntries.length === 0)
             return;
         try {
-            await this.db.insertInto('sync_buffer').values(filteredEntries.map((e) => ({
+            await (transaction ?? this.db).insertInto('sync_buffer').values(filteredEntries.map((e) => ({
                 category: e.category,
                 payload: JSON.stringify(e.payload),
             }))).execute();
@@ -303,6 +239,7 @@ export class SyncService {
             // like a node that had nothing to say — someone reads an empty screen
             // and draws a conclusion about the system rather than about the pipe.
             this.logger.error({ nodeId: this.nodeId, count: filteredEntries.length, category: filteredEntries[0]?.category, error: err.message }, 'Failed to buffer sync batch — these entries will not reach the master');
+            throw err;
         }
     }
     // ===========================================================================
@@ -331,139 +268,35 @@ export class SyncService {
      * everything between the two — the network, the checksum, the ingestion —
      * can fail without the slave forgetting the data.
      */
-    async ackDrained(ids) {
-        if (ids.length === 0)
-            return { released: 0 };
-        await this.markSynced(ids);
-        this.lastSyncAt = Date.now();
-        return { released: ids.length };
+    async ackDrained(ids, receipt) {
+        return { released: await this.markSynced(ids, receipt) };
     }
     // ===========================================================================
-    // Sync Cycle (push to master — legacy, kept for backward compatibility)
+    // Worker outbox maintenance and delivery
     // ===========================================================================
-    /**
-     * One scheduled pass: bound the buffer, then try to drain it.
-     *
-     * The order is the whole point. `enforceBufferBounds` used to be the last
-     * statement of `syncCycle`'s `try`, which meant it ran on a SUCCESSFUL
-     * cycle and on nothing else — and a successful cycle is the one case where
-     * the buffer is being drained and needs no bound. Every path that leaves it
-     * growing skipped it:
-     *
-     *   - no master configured at all → `syncCycle` returns at its second line;
-     *   - master unreachable → the push throws and control leaves via `catch`;
-     *   - and then the backoff, which saturates at five minutes, returns at the
-     *     third line for most ticks after that.
-     *
-     * So the guarantee in this file's header — "bounded buffer, oldest entries
-     * evicted when maxBufferSize is reached" — held only while it was not
-     * needed. A slave that lost its master still buffered until the disk filled,
-     * which is the exact failure the bound was written to prevent and the
-     * reason it is not read as a replication bug when it happens: what fills is
-     * the disk, and what breaks is everything else sharing it.
-     *
-     * The bound is a local database operation. It needs no master, so it does
-     * not belong behind a check for one.
-     */
-    async syncTick() {
+    syncTick() {
         if (this.disposed)
-            return;
-        await this.enforceBufferBounds();
-        await this.syncCycle();
-    }
-    async syncCycle() {
-        if (this.disposed || this.isSyncing)
-            return;
-        if (!this.masterInvoke)
-            return; // No master connection — skip
-        // Respect backoff
-        if (Date.now() < this.backoff.nextRetryAt)
-            return;
-        this.isSyncing = true;
-        try {
-            let totalSynced = 0;
-            let totalRefused = 0;
-            const seenThisCycle = new Set();
-            // Sync in batches until no more pending entries
-            while (true) {
-                const batch = await this.fetchPendingBatch();
-                if (batch.entries.length === 0)
-                    break;
-                const outcome = await this.pushBatch(batch);
-                const delivered = deliveredIds(outcome);
-                await this.markSynced(delivered);
-                totalSynced += delivered.length;
-                totalRefused += outcome.failed.length;
-                // Entries the master rejected stay pending, so the next fetch
-                // returns them again. Stop the sweep when a page yields nothing new
-                // — otherwise a permanently unacceptable entry loops here forever.
-                if (!sweepMadeProgress(seenThisCycle, batch.entries.map((e) => e.id))) {
-                    this.logger.warn({ nodeId: this.nodeId, pending: batch.entries.length }, 'Sync sweep stalled — the master is rejecting the oldest pending entries');
-                    break;
-                }
-                for (const e of batch.entries)
-                    seenThisCycle.add(e.id);
-                // Don't monopolize — yield after each batch
-                if (totalSynced >= this.config.batchSize * 5)
-                    break;
-            }
-            if (totalSynced > 0) {
-                this.logger.info({ synced: totalSynced }, 'Sync cycle completed');
-            }
-            // A cycle that delivered NOTHING and was refused everything is not a
-            // success, even though nothing threw.
-            //
-            // Only one of the two ways to fail used to reach the backoff. An
-            // unreachable master makes the push throw, and the `catch` below grows
-            // the delay to five minutes. But a master that ANSWERS and says no to
-            // every entry returns normally — and control arrived here, at a reset
-            // of a backoff that had never been set. That is how 1000 entries were
-            // offered 94 times in 37 minutes while the master's own Postgres was
-            // down behind a stopped Docker: the RPC succeeded on every attempt,
-            // and only the ingest failed. The configured delay (5 s, doubling, to
-            // 300 s) would have made that about a dozen attempts; it was never
-            // consulted.
-            if (totalSynced === 0 && totalRefused > 0) {
-                this.noteSyncFailure(`master refused every entry (${totalRefused})`);
-                return;
-            }
-            // Reset backoff on success
-            this.backoff = { attempt: 0, nextRetryAt: 0 };
-            this.lastSyncAt = Date.now();
-            this.lastError = null;
-        }
-        catch (err) {
-            this.noteSyncFailure(describeError(err));
-        }
-        finally {
-            this.isSyncing = false;
-        }
-    }
-    /**
-     * Grow the retry delay and record why.
-     *
-     * Shared by both ways a cycle can fail — a master that cannot be reached,
-     * and a master that answers no — because the buffer does not care which
-     * one it was, and neither does the disk it is filling.
-     */
-    noteSyncFailure(message) {
-        this.lastError = message;
-        this.backoff.attempt++;
-        const delay = Math.min(this.config.backoff.initial * Math.pow(this.config.backoff.factor, this.backoff.attempt), this.config.backoff.max);
-        this.backoff.nextRetryAt = Date.now() + delay;
-        this.logger.warn({ error: message, attempt: this.backoff.attempt, nextRetryMs: delay }, 'Sync failed — will retry');
+            return Promise.resolve();
+        this.outboxMaintenance ??= this.enforceBufferBounds().finally(() => { this.outboxMaintenance = null; });
+        return this.outboxMaintenance;
     }
     async fetchPendingBatch(limit) {
+        const requested = limit ?? this.config.batchSize;
+        if (!Number.isSafeInteger(requested) || requested < 1 || requested > 10_000)
+            throw new Error('Replication batch limit must be between 1 and 10000');
+        const tracked = await this.replicationEvidence();
+        const { sql } = await import('kysely');
         const rows = await this.db
             .selectFrom('sync_buffer')
-            .selectAll()
+            .select(['category', 'payload', 'createdAt'])
+            .select(sql `cast(id as text)`.as('id'))
             .where('syncedAt', 'is', null)
             .orderBy('createdAt', 'asc')
-            .limit(limit ?? this.config.batchSize)
+            .limit(requested)
             .execute();
         const batchId = `${this.nodeId}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
         const entries = rows.map((r) => ({
-            id: String(r.id),
+            id: tracked && BigInt(r.id) >= BigInt(tracked.namespaceFrom) ? `${tracked.evidence.epoch}:${r.id}` : String(r.id),
             category: r.category,
             payload: (typeof r.payload === 'string' ? JSON.parse(r.payload) : r.payload),
             // Whatever spelling the local column holds, the master receives a
@@ -480,47 +313,88 @@ export class SyncService {
         hash.update(JSON.stringify(entries));
         return hash.digest('hex');
     }
-    /**
-     * Push a batch and report what the master kept.
-     *
-     * The return value used to be discarded, and `receiveBatch` swallows
-     * per-entry ingestion failures into a warning — so the caller marked every
-     * entry delivered whether or not any of them arrived. A master response
-     * without the id lists is treated as "all delivered", which is what an
-     * older master would mean by it; that is the only reading that does not
-     * silently retry everything against a peer that cannot tell us otherwise.
-     */
-    async pushBatch(batch) {
-        if (!this.masterInvoke)
-            throw new Error('No master connection');
-        const response = (await this.masterInvoke('OmnitronSync', 'receiveBatch', [batch]));
-        const allIds = batch.entries.map((e) => e.id);
-        if (!response || !Array.isArray(response.acceptedIds)) {
-            return { accepted: allIds, duplicates: [], failed: [], discarded: [] };
+    async replicationEvidence() {
+        if (this.evidenceKept === false)
+            return null;
+        try {
+            const reading = await readReplicationEvidence(this.db);
+            this.evidenceKept = true;
+            return reading;
+        }
+        catch (error) {
+            // An older schema is compatible. Corrupt state and I/O errors are not
+            // evidence of an empty queue and must reach the caller.
+            if (/no such table: sync_replication_state|relation ["']?sync_replication_state["']? does not exist/i.test(describeError(error))) {
+                this.evidenceKept = false;
+                return null;
+            }
+            throw error;
         }
-        return {
-            accepted: response.acceptedIds,
-            duplicates: Array.isArray(response.duplicateIds) ? response.duplicateIds : [],
-            failed: Array.isArray(response.failedIds) ? response.failedIds : [],
-            // An older master answers without this field. Treating a missing list
-            // as an empty one is right: it discarded nothing.
-            discarded: Array.isArray(response.discardedIds) ? response.discardedIds : [],
-        };
     }
-    async markSynced(ids) {
+    async markSynced(ids, receipt) {
+        const groups = receiptGroups(ids, receipt);
         if (ids.length === 0)
-            return;
-        await this.db
-            .updateTable('sync_buffer')
-            .set({ syncedAt: new Date() })
-            .where('id', 'in', ids)
-            .execute();
+            return 0;
+        const now = Date.now();
+        const tracked = await this.replicationEvidence();
+        let released;
+        if (tracked) {
+            // Classification, queue release and durable evidence commit together.
+            released = await this.db.transaction().execute(async (transaction) => {
+                let count = 0;
+                for (const [outcome, wireIds] of Object.entries(groups)) {
+                    if (!wireIds.length)
+                        continue;
+                    const values = wireIds.map(id => {
+                        if (!id.includes(':')) {
+                            if (!/^[1-9]\d*$/.test(id) || BigInt(id) >= BigInt(tracked.namespaceFrom)) {
+                                throw new Error('Replication receipt is missing its buffer epoch');
+                            }
+                            return id; // IDs offered before migration retain their original wire representation.
+                        }
+                        const [epoch, sequence, extra] = id.split(':');
+                        if (epoch !== tracked.evidence.epoch || !sequence || !/^[1-9]\d*$/.test(sequence) || extra !== undefined ||
+                            BigInt(sequence) < BigInt(tracked.namespaceFrom) || BigInt(sequence) > 9223372036854775807n) {
+                            throw new Error('Replication receipt belongs to a different buffer epoch');
+                        }
+                        return sequence;
+                    });
+                    const result = await transaction.updateTable('sync_buffer')
+                        .set({ syncedAt: new Date(now), syncOutcome: outcome })
+                        .where('id', 'in', values).where('syncedAt', 'is', null).executeTakeFirst();
+                    count += Number(result.numUpdatedRows);
+                }
+                return count;
+            });
+        }
+        else {
+            const result = await this.db.updateTable('sync_buffer').set({ syncedAt: new Date(now) })
+                .where('id', 'in', [...new Set(ids)]).where('syncedAt', 'is', null).executeTakeFirst();
+            released = Number(result.numUpdatedRows);
+        }
+        if (released > 0) {
+            this.lastSyncAt = now;
+            const second = Math.floor(now / 1000);
+            this.deliveries.set(second, (this.deliveries.get(second) ?? 0) + released);
+            this.deliveryRate(now);
+        }
+        return released;
+    }
+    deliveryRate(now) {
+        const cutoff = Math.floor(now / 1000) - 59;
+        let count = 0;
+        for (const [second, entries] of this.deliveries) {
+            if (second < cutoff)
+                this.deliveries.delete(second);
+            else
+                count += entries;
+        }
+        return count / 60;
     }
     /**
      * Keep the WAL inside `maxBufferSize`.
      *
-     * Guarantee 5 in the header of this file — "bounded buffer, oldest entries
-     * evicted when maxBufferSize reached" — had no implementation:
+     * The original bounded-buffer promise had no implementation:
      * `maxBufferSize` was read from config, defaulted to 500 MB, and never
      * referenced again. The only eviction was of *synced* rows older than a
      * day, so a slave that lost its master buffered until the disk filled. The
@@ -728,6 +602,10 @@ export class SyncService {
         if (typeof batch.nodeId !== 'string' || batch.nodeId.trim() === '') {
             throw new Error('Sync batch rejected: `nodeId` is required and must be a non-empty string.');
         }
+        if (!Array.isArray(batch.entries) || batch.entries.length > 10_000 || batch.entries.some(entry => !entry || typeof entry.id !== 'string' || !entry.id.length || entry.id.length > 64 ||
+            typeof entry.category !== 'string' || !entry.payload || typeof entry.payload !== 'object' || Array.isArray(entry.payload))) {
+            throw new Error('Sync batch rejected: at most 10000 entries with bounded IDs and object payloads are required');
+        }
         // Rate limiting — prevent flood from misbehaving slaves
         if (!this.checkRateLimit(batch.nodeId)) {
             throw new Error(`Rate limit exceeded for node ${batch.nodeId}. Max ${RATE_LIMIT_PER_MINUTE} batches/min.`);
@@ -756,18 +634,28 @@ export class SyncService {
                 total: batch.entries.length,
             }, 'Sync batch received');
         }
+        const released = deliveredIds(outcome);
+        const receipts = released.length ? await this.db.selectFrom('sync_ingested').select(['entryId', 'outcome'])
+            .where('nodeId', '=', batch.nodeId).where('entryId', 'in', released).execute() : [];
+        const discarded = new Set(receipts.filter(row => row.outcome === 'discarded').map(row => row.entryId));
+        const unverified = new Set(receipts.filter(row => row.outcome === 'unverified').map(row => row.entryId));
+        const reconciled = await reconciledMetricIds(this.db, batch.nodeId, batch.entries.filter(entry => unverified.has(entry.id)));
+        const acceptedIds = outcome.accepted.filter(id => !discarded.has(id));
+        const duplicateIds = outcome.duplicates.filter(id => !discarded.has(id));
         return {
+            receiptVersion: 1,
+            committedIds: [...receipts.filter(row => row.outcome === 'committed').map(row => row.entryId), ...reconciled],
             // Kept as a number for the existing RPC contract; the id lists are
             // what the slave needs to know which entries it may release.
-            accepted: outcome.accepted.length,
-            duplicates: outcome.duplicates.length,
-            acceptedIds: outcome.accepted,
-            duplicateIds: outcome.duplicates,
+            accepted: acceptedIds.length,
+            duplicates: duplicateIds.length,
+            acceptedIds,
+            duplicateIds,
             failedIds: outcome.failed,
             // Released, not stored. The slave must let these go or it offers them
             // forever; the ERROR beside each one is the only record that they
             // existed.
-            discardedIds: outcome.discarded,
+            discardedIds: [...discarded],
         };
     }
     /**
@@ -815,7 +703,7 @@ export class SyncService {
         const outcome = { accepted: [], duplicates: [], failed: [], discarded: [] };
         const taken = [];
         await this.db.transaction().execute(async (trx) => {
-            const claimed = await this.claimAll(trx, nodeId, entries.map((e) => e.id));
+            const claimed = await this.claimAll(trx, nodeId, entries);
             for (const entry of entries) {
                 if (claimed.delete(entry.id)) {
                     outcome.accepted.push(entry.id);
@@ -883,8 +771,13 @@ export class SyncService {
      * Sorted, so that two transactions claiming overlapping batches take the
      * ledger's rows in the same order and cannot deadlock on each other.
      */
-    async claimAll(db, nodeId, ids) {
-        const rows = [...new Set(ids)].sort().map((entryId) => ({ nodeId, entryId }));
+    async claimAll(db, nodeId, entries) {
+        const byId = new Map();
+        for (const entry of entries)
+            if (!byId.has(entry.id))
+                byId.set(entry.id, entry);
+        const rows = [...byId.values()].sort((a, b) => a.id.localeCompare(b.id))
+            .map(entry => ({ nodeId, entryId: entry.id, outcome: this.receiptOutcome(entry.category) }));
         const claimed = new Set();
         for (const part of statementsOf(rows)) {
             const taken = await db
@@ -904,6 +797,7 @@ export class SyncService {
      * so an entry lands as the same row whichever one took it.
      */
     async writeRows(db, nodeId, entries) {
+        const metrics = [];
         const logs = [];
         const alerts = [];
         const traces = [];
@@ -911,9 +805,7 @@ export class SyncService {
         for (const entry of entries) {
             switch (entry.category) {
                 case 'metrics':
-                    // No row: a metric's only store is the sink, after the claim
-                    // commits (`recordMetrics`). Its claim is what stops a retry from
-                    // recording it twice.
+                    metrics.push(this.metricSample(nodeId, entry));
                     break;
                 case 'logs':
                     logs.push(logRow(nodeId, entry));
@@ -936,12 +828,19 @@ export class SyncService {
                     break;
             }
         }
+        if (logs.length)
+            await assertLogDurability(db);
         for (const part of statementsOf(logs))
             await db.insertInto('logs').values(part).execute();
         for (const part of statementsOf(alerts))
             await db.insertInto('alert_events').values(part).execute();
         for (const part of statementsOf(traces))
             await db.insertInto('traces').values(part).execute();
+        if (metrics.length) {
+            if (!this.metricsStorage)
+                throw new Error('Persistent metric storage is unavailable; payloads remain pending on the node');
+            await this.metricsStorage.writeInTransaction(db, metrics);
+        }
         if (unknown.length > 0) {
             // An unknown category is not a failure to retry — retrying cannot make
             // it known. It is claimed and dropped, so the slave releases it, and
@@ -955,45 +854,20 @@ export class SyncService {
             }, 'Unknown sync category — entries discarded');
         }
     }
-    /**
-     * Hand claimed remote metrics to the store the console reads.
-     *
-     * The sink records each sample into titan-metrics, which is what the
-     * console's `getSnapshot` and `querySeries` read, tagged with `node` so a
-     * remote reading is a label dimension rather than a second table. The
-     * `metrics_raw` table these were also written to was kept «until the
-     * console is observed showing the label»; it was observed (2026-09-23: 19
-     * series labelled with the test node), and the table — written ~0.93
-     * million rows a day, read and pruned by nothing — is gone (migration 010).
-     * History now lives in titan-metrics' store (in memory, 7 days, lost with a
-     * master restart); how long it should be kept is that store's question.
-     *
-     * AFTER the commit. The sink is not part of the transaction, and it used to
-     * be called inside it, before the row was written: an entry whose write
-     * then failed was recorded anyway, and recorded again by the retry that
-     * stored it. With a batch rolled back and retried entry by entry, that
-     * would have been every metric in it.
-     */
+    /** Update live exposition after the payload and receipt commit atomically. */
     recordMetrics(nodeId, entries) {
-        if (!this.metricsSink)
+        if (!this.metricsRegistry)
             return;
-        try {
-            for (const entry of entries) {
-                if (entry.category !== 'metrics')
-                    continue;
-                this.metricsSink({
-                    node: nodeId,
-                    name: String(entry.payload['name'] ?? 'unknown'),
-                    app: String(entry.payload['app'] ?? 'unknown'),
-                    labels: (entry.payload['labels'] ?? {}),
-                    value: Number(entry.payload['value'] ?? 0),
-                });
+        for (const entry of entries) {
+            if (entry.category !== 'metrics')
+                continue;
+            try {
+                this.metricsRegistry.record(this.metricSample(nodeId, entry));
+            }
+            catch (err) {
+                // The sample and receipt have already committed. Projection cannot undo that or skip later samples.
+                this.logger.warn({ nodeId, entryId: entry.id, error: describeError(err) }, 'Failed to expose a committed remote metric');
             }
-        }
-        catch (err) {
-            // The entries are stored and will be acknowledged; a sink that throws
-            // must not turn that into a retry the ledger then calls a duplicate.
-            this.logger.warn({ nodeId, error: describeError(err) }, 'Stored remote metrics could not be recorded for the console');
         }
     }
     /**
@@ -1003,6 +877,18 @@ export class SyncService {
      * lost acknowledgement, which is delivery, not failure. Throws when
      * ingestion fails, leaving nothing recorded.
      */
+    receiptOutcome(category) {
+        return ['metrics', 'logs', 'alerts', 'traces', 'events', 'state'].includes(category) ? 'committed' : 'discarded';
+    }
+    metricSample(nodeId, entry) {
+        try {
+            return replicatedMetric(nodeId, entry);
+        }
+        catch (cause) {
+            // Malformed telemetry is a permanent refusal; database/network failures are retryable.
+            throw Object.assign(new Error(`Invalid replicated metric: ${describeError(cause)}`), { code: '22023' });
+        }
+    }
     async claimAndIngest(nodeId, entry) {
         try {
             return await this.claimAndIngestOnce(nodeId, entry);
@@ -1018,12 +904,13 @@ export class SyncService {
             // The claim is made in its own transaction because the one above rolled
             // back with the failure, taking the claim with it. Claiming it here is
             // what stops it coming back.
-            await this.db
+            const recorded = await this.db
                 .insertInto('sync_ingested')
-                .values({ nodeId, entryId: entry.id })
+                .values({ nodeId, entryId: entry.id, outcome: 'discarded' })
                 .onConflict((oc) => oc.columns(['nodeId', 'entryId']).doNothing())
-                .execute()
-                .catch(() => undefined);
+                .returning('entryId').executeTakeFirst();
+            if (!recorded)
+                return 'duplicate'; // The canonical receipt decides which outcome won.
             // ERROR, with everything needed to find the entry: this is data that
             // will never arrive, and the node has been told to forget it.
             this.logger.error({ nodeId, entryId: entry.id, category: entry.category, error: err.message }, 'Sync entry rejected permanently — discarded so the buffer can drain');
@@ -1034,7 +921,7 @@ export class SyncService {
         return this.db.transaction().execute(async (trx) => {
             const claim = await trx
                 .insertInto('sync_ingested')
-                .values({ nodeId, entryId: entry.id })
+                .values({ nodeId, entryId: entry.id, outcome: this.receiptOutcome(entry.category) })
                 .onConflict((oc) => oc.columns(['nodeId', 'entryId']).doNothing())
                 .returning('entryId')
                 .executeTakeFirst();
@@ -1158,30 +1045,45 @@ export class SyncService {
     async getStatus() {
         let pendingItems = 0;
         let bufferSize = 0;
+        let oldestPendingAgeMs = null;
+        let lastSyncAt = this.lastSyncAt;
+        let tracked = null;
+        let reconciliationError;
         if (this.role === 'slave') {
-            try {
-                const result = await this.db
-                    .selectFrom('sync_buffer')
-                    .select((eb) => [
-                    eb.fn.count('id').as('count'),
-                ])
-                    .where('syncedAt', 'is', null)
-                    .executeTakeFirst();
-                pendingItems = Number(result?.count ?? 0);
-                // Estimate buffer size: pending rows * ~256 bytes average row size
-                bufferSize = pendingItems * 256;
+            tracked = await this.replicationEvidence();
+            if (tracked) {
+                ({ pendingItems, bufferSize, oldestPendingAgeMs, lastSyncAt } = tracked);
+                try {
+                    const reconciled = (await this.getReconciliationSummaries()).find(r => r.nodeId === this.nodeId && r.epoch === tracked.evidence.epoch);
+                    if (reconciled)
+                        tracked.evidence.reconciliation = { ...reconciled, scope: 'source' };
+                }
+                catch (error) {
+                    reconciliationError = describeError(error);
+                }
             }
-            catch {
-                // Table may not exist yet
+            else {
+                const { sql } = await import('kysely');
+                const result = await this.db.selectFrom('sync_buffer').select(eb => [
+                    eb.fn.count('id').as('count'), eb.fn.min('createdAt').as('oldest'),
+                    sql `coalesce(sum(octet_length(cast(payload as text))), 0)`.as('bytes'),
+                ]).where('syncedAt', 'is', null).executeTakeFirst();
+                pendingItems = Number(result?.count ?? 0);
+                bufferSize = Number(result?.bytes ?? 0);
+                if (result?.oldest != null)
+                    oldestPendingAgeMs = Math.max(0, Date.now() - Date.parse(toIsoUtc(result.oldest)));
             }
         }
         return {
-            connected: this.masterInvoke !== null,
-            lastSyncAt: this.lastSyncAt,
-            pendingItems,
-            bufferSize,
-            lastError: this.lastError,
-            failedAttempts: this.backoff.attempt,
+            connected: false, // Legacy reverse-channel field. Connectivity is observed by the controller.
+            lastSyncAt, pendingItems, oldestPendingAgeMs,
+            lastSyncAgeMs: lastSyncAt === null ? null : Math.max(0, Date.now() - lastSyncAt),
+            deliveryRatePerSecond: this.deliveryRate(Date.now()),
+            bufferSize, bufferSizeAccuracy: 'measured',
+            ...(tracked && { evidence: tracked.evidence, retainedBufferSize: tracked.retainedBufferSize }),
+            ...(reconciliationError && { reconciliationError }),
+            lastError: null,
+            failedAttempts: 0,
         };
     }
 }