instar 1.3.338 → 1.3.340

package/dist/core/JournalSyncApplier.d.ts

@@ -0,0 +1,272 @@
+/**
+ * JournalSyncApplier — the RECEIVE (and own-stream SERVE) side of coherence
+ * journal replication (P1.3 of multi-machine coherence).
+ *
+ * Spec: docs/specs/COHERENCE-JOURNAL-SPEC.md §3.4 (replication trust model —
+ * ALL seven rules), §3.1 (stream/meta layout), §5 (trust model).
+ *
+ * This class is PURE and MESH-INDEPENDENT. It owns disk state only; another
+ * workstream wires the `journal-sync` MeshRpc verb, the `session-status`
+ * advert, and the delta-request loop on top of it. It NEVER imports MeshRpc,
+ * the server, or the mesh dispatcher — it is the engine those call.
+ *
+ * Replica layout (§3.1): peer copies of machine M's stream of kind K live at
+ *   <stateDir>/state/coherence-journal/peers/<safeMachineId(M)>.<kind>.jsonl
+ * with one sidecar
+ *   <stateDir>/state/coherence-journal/peers/<safeMachineId(M)>.meta.json
+ * holding the stream-set incarnation token (top-level, mirroring the writer's
+ * own meta so `CoherenceJournalReader.readIncarnation` parses it) plus per-kind
+ * apply bookkeeping (lastHeldSeq, status, suspect/flap counters, gap sentinels)
+ * and the bounded quarantine record.
+ *
+ * The receive-side §4.1 durability rule lives HERE: a replica append fsyncs
+ * BEFORE the entries are reported `applied`, so a journal-sync ack is an
+ * ack-after-durable-commit. Author-side durability is the writer's separate
+ * best-effort flush window — this is the receiver's contract.
+ *
+ * Trust model enforced (§3.4):
+ *  1. FIRST-HOP SENDER BINDING — every entry's `machine` must === the
+ *     authenticated sender; the target file derives from the sender, NEVER a
+ *     payload field. Forged entries are rejected + counted, never appended.
+ *  2. SCHEMA-VALIDATED APPLY — per entry: parseable + size-cap, seq exactly
+ *     lastHeldSeq+1, ts parses, kind known-or-ignorable, data passes the kind's
+ *     typed schema. ANY failure marks the stream `suspect` and STOPS the batch
+ *     at the last valid line. `suspect` self-clears after K=20 consecutive
+ *     valid in-order applies.
+ *  3. INCARNATION FENCING — a known stream arriving with a NEW incarnation
+ *     quarantines the old replica (rename aside, ≤2 kept per stream), starts
+ *     fresh, and surfaces a coalesced divergence signal (per-machine 10min
+ *     window); >3 flips in the window → status `reset-flapping`, surfaced once.
+ *  4. TRUNCATION SIGNALS — a batch may carry `oldestRetainedSeq`; when
+ *     lastHeldSeq+1 < oldestRetainedSeq the receiver records a gap sentinel in
+ *     the replica meta (NOT a fake journal line), fast-forwards lastHeldSeq to
+ *     oldestRetainedSeq-1, and marks the stream `gapped` until the next clean
+ *     apply. (Gap re-request PACING is the caller's job; this exposes status.)
+ *
+ * All replica writes go through the injected guardWrite seam (same seam as the
+ * writer) and O_APPEND single-line writes. The applier is tolerant + bounded
+ * everywhere and NEVER throws into its caller; degradation is surfaced via
+ * counters and per-stream status.
+ */
+import { type JournalEntry, type JournalKind, type JournalFs } from './CoherenceJournal.js';
+/** Per-entry size cap on the receive side (§3.4 rule 2). Mirrors the writer's
+ *  DEFAULT_MAX_ENTRY_BYTES (source of truth: CoherenceJournal.ts). */
+export declare const APPLIER_MAX_ENTRY_BYTES: number;
+/** Consecutive valid in-order applies that self-clear `suspect` (§3.4 rule 2). */
+export declare const SUSPECT_CLEAR_THRESHOLD = 20;
+/** Quarantined replica files kept per stream before evicting the oldest (rule 3). */
+export declare const MAX_QUARANTINE_PER_STREAM = 2;
+/** Incarnation-flap coalescing window per machine (rule 3). */
+export declare const FLAP_WINDOW_MS: number;
+/** Flips within the window past this → `reset-flapping`, surfaced once (rule 3). */
+export declare const FLAP_RESET_THRESHOLD = 3;
+/** Default serve-batch size cap (§3.4 rule 5 — journalSyncMaxBatchBytes). */
+export declare const DEFAULT_MAX_BATCH_BYTES = 262144;
+/** Per-stream replication status surfaced to the caller (§3.4 rule 4). */
+export type StreamReplStatus = 'current' | 'behind' | 'gapped' | 'suspect' | 'reset-flapping';
+/** One stream's apply bookkeeping, persisted in the peer meta sidecar. */
+export interface PeerKindState {
+    /** Highest contiguously-applied seq held locally for this peer+kind. */
+    lastHeldSeq: number;
+    status: StreamReplStatus;
+    /** Consecutive valid in-order applies since the stream last went `suspect`. */
+    consecutiveValid: number;
+    /** Gap sentinels recorded by truncation fast-forwards (§3.4 rule 4). */
+    gaps: GapSentinel[];
+}
+/** A recorded hole in a replica stream (rule 4) — meta bookkeeping, NOT a line. */
+export interface GapSentinel {
+    /** First seq known-missing (the old lastHeldSeq+1). */
+    fromSeq: number;
+    /** Last seq known-missing (oldestRetainedSeq-1). */
+    toSeq: number;
+    /** When this machine recorded the gap. */
+    recordedAt: string;
+}
+/** The peer meta sidecar (§3.1) — incarnation top-level (reader-compatible). */
+export interface PeerMeta {
+    incarnation: string;
+    /** Per-kind apply bookkeeping. */
+    kinds: Partial<Record<JournalKind, PeerKindState>>;
+    /** Incarnation flips observed (timestamps ms) for the flap window. */
+    flipsMs: number[];
+    /** Sticky once a machine has been flagged reset-flapping (surfaced once). */
+    resetFlapping: boolean;
+}
+/** A divergence signal coalesced per machine within FLAP_WINDOW_MS (rule 3). */
+export interface DivergenceSignal {
+    machineId: string;
+    kind: JournalKind;
+    oldIncarnation: string;
+    newIncarnation: string;
+    at: string;
+}
+/** One stream's slice of an inbound batch. */
+export interface ApplyBatchStream {
+    kind: JournalKind;
+    incarnation: string;
+    entries: JournalEntry[];
+    /** Optional truncation watermark (§3.4 rule 4). */
+    oldestRetainedSeq?: number;
+}
+/** Result of an apply() call — never thrown, fully observable. */
+export interface ApplyResult {
+    /** Total entries durably appended across all streams in this call. */
+    applied: number;
+    /** Entries rejected by first-hop sender binding (rule 1). */
+    forgedEntries: number;
+    /** Entries dropped as duplicates (seq <= lastHeldSeq) — silent (rule 2). */
+    duplicates: number;
+    /** Entries failing schema/seq/size/ts validation (rule 2). */
+    invalidEntries: number;
+    /** Streams marked suspect in this call (rule 2). */
+    suspectStreams: number;
+    /** Streams quarantined for a new incarnation in this call (rule 3). */
+    quarantined: number;
+    /** Truncation gaps recorded in this call (rule 4). */
+    gapsRecorded: number;
+    /** Append batches skipped because guardWrite refused (never thrown). */
+    guardSkips: number;
+    /** Divergence signals emitted (coalesced) in this call (rule 3). */
+    signals: DivergenceSignal[];
+    /** Per-stream final status after this call. */
+    statuses: Record<string, StreamReplStatus>;
+}
+/** Aggregate degradation counters (lifetime). */
+export interface ApplierDegradation {
+    applied: number;
+    forgedEntries: number;
+    duplicates: number;
+    invalidEntries: number;
+    suspectMarks: number;
+    quarantines: number;
+    gapsRecorded: number;
+    guardSkips: number;
+    appendErrors: number;
+    signals: number;
+}
+export interface JournalSyncApplierConfig {
+    /** Absolute path to the agent's `.instar/` directory (the stateDir). */
+    stateDir: string;
+    /**
+     * Injected standby read-only guard (§3.1). Called with the target replica
+     * file path before each append batch; throwing skips the batch + counts.
+     * Same seam the writer uses (`StateManager.guardJournalWrite`).
+     */
+    guardWrite?: (filePath: string) => void;
+    /** Optional clock override (tests). */
+    now?: () => Date;
+    /** Optional logger; called once per failure class. */
+    logger?: (msg: string) => void;
+    /** Optional fs seam for fault-injection tests. Defaults to node:fs. */
+    fsImpl?: JournalFs;
+    /** Per-entry size cap override. Default APPLIER_MAX_ENTRY_BYTES. */
+    maxEntryBytes?: number;
+}
+export declare class JournalSyncApplier {
+    private readonly stateDir;
+    private readonly guardWrite?;
+    private readonly now;
+    private readonly logger?;
+    private readonly io;
+    private readonly maxEntryBytes;
+    /** In-memory meta cache per peer machine (keyed by RAW machineId). */
+    private metaCache;
+    private degradation;
+    private loggedClasses;
+    constructor(config: JournalSyncApplierConfig);
+    getDegradation(): Readonly<ApplierDegradation>;
+    /**
+     * What this machine holds per peer stream — `{ machineId → { kind →
+     * { incarnation, lastSeq } } }` — so the caller can compare against a peer's
+     * advert and request only the missing ranges (§3.4 rule 5). Sourced from the
+     * persisted peer meta (incarnation + per-kind lastHeldSeq).
+     */
+    getAdvertState(): Record<string, Record<string, {
+        incarnation: string;
+        lastSeq: number;
+    }>>;
+    /** Per-stream replication status (§3.4 rule 4 — caller drives gap pacing). */
+    getStreamStatus(): Record<string, StreamReplStatus>;
+    /**
+     * Apply an inbound batch from `senderMachineId` (the AUTHENTICATED envelope
+     * identity). Implements §3.4 rules 1-4. Never throws; the result is fully
+     * observable. The target replica files derive from `senderMachineId` only —
+     * NEVER from any payload field.
+     */
+    apply(senderMachineId: string, batch: ApplyBatchStream[]): ApplyResult;
+    private applyStream;
+    /**
+     * Per-entry verdict (§3.4 rules 1 + 2). `expectedSeq` is lastHeldSeq + count
+     * already queued this batch, so seq must be exactly contiguous.
+     */
+    private validateEntry;
+    /**
+     * §3.2 typed schema validation — MIRRORED from CoherenceJournal.validate()
+     * (source of truth: src/core/CoherenceJournal.ts). The applier validates
+     * STRICTLY (reject on unknown/extra fields too) so a peer's buggy/forged
+     * writer cannot smuggle free text past the receiver. Returns true if the
+     * data is a well-formed instance of the kind's schema.
+     */
+    private validateData;
+    private handleIncarnationFlip;
+    /**
+     * Rename a peer replica file aside as `<file>.quarantine.<stamp>`, keeping at
+     * most MAX_QUARANTINE_PER_STREAM per stream (oldest evicted). Bounded disk.
+     */
+    private quarantineReplica;
+    private pruneQuarantine;
+    /**
+     * Append validated lines to the peer replica file with O_APPEND single-line
+     * writes, then fdatasync BEFORE returning the applied count (§4.1 — the
+     * receiver's durability-before-ack rule lives HERE). Returns the number of
+     * lines durably committed (0 if the guard refused or the open/write failed
+     * before any byte landed). Never throws.
+     */
+    private durablyAppend;
+    /**
+     * Build a delta batch from THIS machine's OWN stream for a peer that is
+     * behind (§3.4 rule 5). Reads the FILE (never any queue) so only
+     * durably-flushed entries are served — a §3.4-rule-3 corollary the writer
+     * guarantees by advertising only flushed seqs. First-hop only: this serves
+     * exactly one own stream of one kind.
+     *
+     * @param kind          which own stream to serve.
+     * @param fromSeq       serve entries with seq > fromSeq (the peer's lastSeq).
+     * @param maxBatchBytes total serialized byte cap (default DEFAULT_MAX_BATCH_BYTES).
+     * @param ownMachineId  this machine's id (the producer of the served stream).
+     */
+    buildServeBatch(kind: JournalKind, fromSeq: number, ownMachineId: string, maxBatchBytes?: number): {
+        kind: JournalKind;
+        incarnation: string;
+        entries: JournalEntry[];
+        oldestRetainedSeq?: number;
+    };
+    private ensureKind;
+    private markSuspect;
+    private loadMeta;
+    private normalizeKinds;
+    private persistMeta;
+    private mintIncarnation;
+    /** Discover peer machine ids from the peers/ meta sidecars + replica files. */
+    private discoverPeerMachines;
+    /**
+     * The peer file grammar uses the SANITIZED machine id as the literal. The raw
+     * id is only needed for first-hop binding (which uses the sender id directly,
+     * not a derived one) — for advert/status keys we surface the on-disk literal
+     * (already sanitized + injective), so reverse-mapping is unnecessary and we
+     * keep the literal as the machine key. (sanitizeMachineId is injective, so
+     * the literal is a faithful, stable handle.)
+     */
+    private unsanitizeBestEffort;
+    /** Own current + archive stream files (newest-first) for serve reads. */
+    private ownStreamFilesNewestFirst;
+    private readOwnIncarnation;
+    private dirPath;
+    private peersDirPath;
+    private replicaFilePath;
+    private metaPath;
+    private ensurePeersDir;
+    private log;
+}
+//# sourceMappingURL=JournalSyncApplier.d.ts.map

package/dist/core/JournalSyncApplier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JournalSyncApplier.d.ts","sourceRoot":"","sources":["../../src/core/JournalSyncApplier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAMH,OAAO,EAIL,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,SAAS,EAIf,MAAM,uBAAuB,CAAC;AAK/B;sEACsE;AACtE,eAAO,MAAM,uBAAuB,QAAW,CAAC;AAChD,kFAAkF;AAClF,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAC1C,qFAAqF;AACrF,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAC3C,+DAA+D;AAC/D,eAAO,MAAM,cAAc,QAAiB,CAAC;AAC7C,oFAAoF;AACpF,eAAO,MAAM,oBAAoB,IAAI,CAAC;AACtC,6EAA6E;AAC7E,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAM9C,0EAA0E;AAC1E,MAAM,MAAM,gBAAgB,GAAG,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,gBAAgB,CAAC;AAE9F,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B,wEAAwE;IACxE,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,gBAAgB,CAAC;IACzB,+EAA+E;IAC/E,gBAAgB,EAAE,MAAM,CAAC;IACzB,wEAAwE;IACxE,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IAC1B,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,gFAAgF;AAChF,MAAM,WAAW,QAAQ;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;IACnD,sEAAsE;IACtE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,6EAA6E;IAC7E,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,gFAAgF;AAChF,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,WAAW,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,8CAA8C;AAC9C,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,mDAAmD;IACnD,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,kEAAkE;AAClE,MAAM,WAAW,WAAW;IAC1B,sEAAsE;IACtE,OAAO,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,cAAc,EAAE,MAAM,CAAC;IACvB,uEAAuE;IACvE,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,YAAY,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;CAC5C;AAED,iDAAiD;AACjD,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB;IACvC,wEAAwE;IACxE,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,uCAAuC;IACvC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,sDAAsD;IACtD,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAC/B,uEAAuE;IACvE,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,oEAAoE;IACpE,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAwBD,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;IACzD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAa;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IAEvC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA+B;IAEhD,OAAO,CAAC,WAAW,CAWjB;IACF,OAAO,CAAC,aAAa,CAAqB;gBAE9B,MAAM,EAAE,wBAAwB;IAS5C,cAAc,IAAI,QAAQ,CAAC,kBAAkB,CAAC;IAM9C;;;;;OAKG;IACH,cAAc,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAkB1F,8EAA8E;IAC9E,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAuBnD;;;;;OAKG;IACH,KAAK,CAAC,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,GAAG,WAAW;IA0CtE,OAAO,CAAC,WAAW;IAmHnB;;;OAGG;IACH,OAAO,CAAC,aAAa;IA8CrB;;;;;;OAMG;IACH,OAAO,CAAC,YAAY;IAqCpB,OAAO,CAAC,qBAAqB;IA6D7B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAgBzB,OAAO,CAAC,eAAe;IAgCvB;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IA2DrB;;;;;;;;;;;OAWG;IACH,eAAe,CACb,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,EACpB,aAAa,SAA0B,GACtC;QAAE,IAAI,EAAE,WAAW,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,YAAY,EAAE,CAAC;QAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;KAAE;IAmDlG,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,WAAW;IAKnB,OAAO,CAAC,QAAQ;IA0BhB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,WAAW;IAanB,OAAO,CAAC,eAAe;IAMvB,+EAA+E;IAC/E,OAAO,CAAC,oBAAoB;IA4B5B;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAI5B,yEAAyE;IACzE,OAAO,CAAC,yBAAyB;IAuBjC,OAAO,CAAC,kBAAkB;IAe1B,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,cAAc;IAKtB,OAAO,CAAC,GAAG;CAKZ"}