@voltro/database 0.55.0 → 0.56.0

package/dist/index.d.ts

@@ -621,6 +621,11 @@ export declare interface CdcConfig {
     readonly exclusionRefreshFollows?: boolean;
 }
 
+/** Env var that tunes how long a change-claim row survives. Declared here rather
+ *  than at the retention registration, because the OCCURRENCE COUNTER's window
+ *  has to be derived from the same number and lives in this file. */
+export declare const CHANGE_CLAIM_TTL_ENV = "VOLTRO_CHANGE_CLAIMS_TTL_HOURS";
+
 /** Maximum `onChange` listeners a DataStore's change bus accepts before Node
  *  warns. See this file's header for why it is 512 and why it is not `0`. */
 export declare const CHANGE_LISTENER_CEILING = 512;
@@ -638,6 +643,17 @@ export declare interface ChangeCaptureExclusionRefresh {
 /** A row was removed from the table. `new` is null, always. */
 export declare const changeDelete: (table: string, row: Row, meta?: ChangeEventMeta) => ChangeEvent;
 
+/** The content half of a change identity — equal iff two replicas are looking
+ *  at the same change (or at two changes with byte-identical images, which the
+ *  occurrence counter then separates). */
+export declare const changeDigest: (input: {
+    readonly pipe: string;
+    readonly op: "insert" | "update" | "delete";
+    readonly key: string;
+    readonly next: Record<string, unknown> | null;
+    readonly prev: Record<string, unknown> | null;
+}) => string;
+
 /**
  * A row-change event emitted by the data store (in-memory) or the CDC
  * consumer (Postgres logical replication). Drives the reactive engine.
@@ -766,6 +782,22 @@ export declare const changeSoftDelete: (table: string, row: Row, opts?: {
  */
 export declare type ChangeStrategy = 'inline' | 'cdc';
 
+/**
+ * A proven interruption in a store's change stream.
+ *
+ * Deliberately NOT a list of what was missed. A pub/sub transport keeps no log,
+ * so the missed changes are gone — the only honest recovery is to re-derive
+ * everything downstream, and a consumer that were handed a partial list would be
+ * tempted to patch instead. `reason` is for the log line; the instruction is the
+ * event itself.
+ */
+export declare interface ChangeStreamGap {
+    /** Human-readable cause — goes into the refresh's log line. */
+    readonly reason: string;
+    /** Wall-clock ms the stream was known to be down, when it is known. */
+    readonly downForMs?: number;
+}
+
 /** A row changed. Both sides are present, because an update has both. */
 export declare const changeUpdate: (table: string, before: Row, after: Row, meta?: ChangeEventMeta) => ChangeEvent;
 
@@ -1772,6 +1804,9 @@ export declare const compileRawFragment: <Row extends object = Record<string, un
 
 export declare const compileSelect: <Row extends object>(descriptor: QueryDescriptor, sql: SqlConstructor, namespace?: QueryNamespace) => Statement.Statement<Row>;
 
+/** Compose the durable identity written to `_voltro_cdcout_claims.changeKey`. */
+export declare const composeChangeKey: (digest: string, occurrence: number) => string;
+
 /**
  * Compute the embedded row from a {@link VectorEmbeddingContext}. Returns
  * the patch to merge into the row: `{ [as]: <vector> }`. When the source
@@ -2298,6 +2333,23 @@ export declare interface DataStore {
      * must elect a single worker instead of filtering by origin.
      */
     readonly changeScope?: 'local' | 'fleet';
+    /**
+     * Subscribe to CHANGE-STREAM GAPS — moments where this process proved it
+     * stopped receiving changes for a while and has now started again.
+     *
+     * A gap is not an error to show a user; it is an instruction to the reactive
+     * layer: everything derived from this store may be stale, so re-run it. A
+     * live query is idempotent, which is what makes "re-run all of them" both
+     * safe and the complete recovery. `voltro dev` and `voltro serve` wire this
+     * to the dispatcher's `refreshAll`, the same recovery the broadcast bus's
+     * peer-serial gap uses.
+     *
+     * Optional because only a store whose changes arrive over a TRANSPORT can
+     * have one. An inline store emits its own writes in-process — there is no
+     * connection to lose — and correctly does not implement this. Absent
+     * therefore means "gaps are impossible here", never "gaps are not reported".
+     */
+    readonly onChangeStreamGap?: (listener: (gap: ChangeStreamGap) => void) => () => void;
     /**
      * Bind a per-request tenant NAMESPACE, returning a store view whose
      * every read + write is physically isolated to that namespace
@@ -2724,6 +2776,10 @@ export declare const decryptFieldsOnRead: (rows: ReadonlyArray<Row>, table: Tabl
  */
 export declare const DEFAULT_ACQUIRE_TIMEOUT_MS = 10000;
 
+/** Default lifetime of a change-claim row, in hours. A lock ledger, not a
+ *  history — nobody reads yesterday's claim. */
+export declare const DEFAULT_CHANGE_CLAIM_TTL_HOURS = 1;
+
 /**
  * How often a persistent fallback is allowed to say so.
  *
@@ -3498,7 +3554,6 @@ export declare const formatSubjectGraph: (graph: SubjectGraph) => string;
  */
 export declare const FRAMEWORK_LIVE_TABLES: ReadonlySet<string>;
 
-/** All framework tables as an array — useful for bootstrap DDL emission. */
 export declare const frameworkTables: readonly [TableLike, TableLike, TableLike];
 
 declare type FullTextIndexMeta = {
@@ -4601,6 +4656,67 @@ export declare const numeric: (precision: number, scale?: number) => ColumnBuild
  */
 export declare const observeDbOp: <T>(dialect: DialectId | string, op: DbOp, run: () => Promise<T>) => Promise<T>;
 
+/**
+ * How many times this replica has already seen each digest inside the dedup
+ * window. Bounded on BOTH axes — a per-change map with no ceiling is an
+ * unbounded table in memory:
+ *   * by time — entries untouched for `windowMs` are pruned, and
+ *   * by size — `maxEntries` oldest-first, so a burst of unique changes can't
+ *     grow it without limit.
+ *
+ * `windowMs` MUST be longer than the lifetime of a claim row in the database.
+ * That ordering is the invariant: if a claim outlived the counter that
+ * produced it, the next sighting of that digest would restart at occurrence 0,
+ * collide with the retained claim, and be dropped as a duplicate it is not.
+ *
+ * **`maxEntries` is the SAME invariant reached by size, and it is worth saying
+ * plainly rather than reassuringly.** An evicted digest also restarts at 0, so
+ * an eviction is only harmless while no claim for that digest is still retained
+ * — which is not the same thing as "the position a freshly-booted replica is
+ * in", because a fresh replica has no claims of its own to collide with. The
+ * exposure is narrow (it needs more than `maxEntries` distinct digests inside
+ * the claim TTL *and* a content-identical repeat of an evicted one), and it is
+ * real. Size the map against the change rate the gated tables actually carry,
+ * the way `plugin-cdc-out` sizes it against its handoff buffer.
+ */
+export declare class OccurrenceCounter {
+    #private;
+    constructor(opts: {
+        readonly windowMs: number;
+        readonly maxEntries: number;
+    });
+    get size(): number;
+    /** Record a sighting and return its occurrence index (0 for the first). */
+    next(digest: string, nowMs: number): number;
+    /**
+     * Align this replica with the fleet from the claims already recorded in the
+     * database — the counters of a process that booted mid-window would
+     * otherwise restart at 0 and re-key changes the incumbents have already
+     * keyed. Call ONCE, before the first observed change is keyed.
+     */
+    seedFromChangeKeys(changeKeys: Iterable<string>, nowMs: number): void;
+    /** Drop entries whose last sighting is older than the window. */
+    prune(nowMs: number): void;
+}
+
+/**
+ * The counter window that keeps {@link OccurrenceCounter}'s invariant true for
+ * the claim TTL this deployment actually runs with.
+ *
+ * The invariant is stated on the class: the window MUST be longer than the
+ * lifetime of a claim row, or a digest whose counter entry was pruned restarts
+ * at occurrence 0, collides with the retained claim, and is dropped as a
+ * duplicate it is not. Three call sites hard-coded two hours against a
+ * one-hour default — correct until an operator raises the TTL, at which point
+ * the invariant breaks SILENTLY and in the direction that loses effects.
+ *
+ * So it is derived, not written down: twice the TTL, from the same env var the
+ * retention policy reads.
+ */
+export declare const occurrenceWindowMs: (env?: {
+    readonly [key: string]: string | undefined;
+}) => number;
+
 /**
  * What a decrypt failure does. `'throw'` (default, and the ONLY safe production
  * behaviour) surfaces a typed `FieldDecryptionError`. `'null'` degrades the one
@@ -4732,6 +4848,12 @@ export declare const paginateBy: (descriptor: QueryDescriptor, column: string, c
  */
 export declare const paginateById: (descriptor: QueryDescriptor, cursor: string | undefined, limit: number) => QueryDescriptor;
 
+/** Split a stored `changeKey` back into its two halves (`null` when malformed). */
+export declare const parseChangeKey: (changeKey: string) => {
+    digest: string;
+    occurrence: number;
+} | null;
+
 /** Test seam — how many registrations are outstanding. */
 export declare const pendingAttributionCount: () => number;
 
@@ -8050,8 +8172,20 @@ export declare const _voltroDataTransfersTable: TableLike;
  * (the `Idempotency-Key` header on REST routes). `scope` namespaces the key
  * per tenant + method + path; the UNIQUE on (scope, key) is the atomic claim
  * arbiter (INSERT/upsert-on-conflict). `response` holds the cached
- * `{ status, body }` for replay. Created only when `idempotency` is set in
- * `app.config.ts`. A periodic sweep / lazy-TTL drops rows past their window.
+ * `{ status, body }` for replay.
+ *
+ * Created for EVERY sql app (`when: 'always'` in the cli's registry), empty
+ * unless `idempotency` is set in `app.config.ts`. This used to say "created only
+ * when idempotency is set", which is what the config gates — not what the table
+ * registry does.
+ *
+ * Two things clean it up, and only one of them existed for most of this table's
+ * life. The LAZY TTL drops a stale row when the SAME key is claimed again — and
+ * a key is used once by definition, so a row whose key never returns was never
+ * read and never deleted. The retention SWEEP is the half that bounds it
+ * (`retentionSweep.ts`), and its window is floored by the app's own dedup window:
+ * a record dropped while still inside that window lets the duplicate request it
+ * exists to stop execute a second time.
  */
 export declare const _voltroIdempotencyTable: TableLike;
 
@@ -8104,6 +8238,43 @@ export declare const _voltroMssqlCdcOffsetsTable: TableLike;
  */
 export declare const _voltroReplaceInProgressTable: TableLike;
 
+/**
+ * `_voltro_replica_observations` — what each replica knows about itself, put
+ * somewhere every replica can read it.
+ *
+ * ── Why the shared store and not a fan-out ────────────────────────────────
+ *
+ * The fleet-wide questions an operator asks — "how many subscriptions is each
+ * pod holding", "which of my queries can resume anywhere" — are answers each
+ * process holds privately. Asking the others at read time means inventing
+ * request/response over a pub/sub bus: a correlation id, a reply channel, a
+ * deadline to guess, and a partial-result protocol. And it puts the DIAGNOSTIC
+ * on the failure path, so the moment you most need it is the moment it times
+ * out.
+ *
+ * Writing instead inverts that. The transport is the database — the one thing
+ * that must be up for anything to work — and a replica that dies goes STALE
+ * rather than silent: staleness is a number, and a number can be reported.
+ * Paired with `instanceMembership`'s roster, "old" becomes "missing", which is
+ * the honest partial result the fan-out would have had to build by hand.
+ *
+ * `coordinationState.recentReplicaIds` already answers a fleet question this
+ * way, by reading `_voltro_schedule_runs`. This generalises that one case.
+ *
+ * ── The shape, and what it must not become ────────────────────────────────
+ *
+ * One row per `(replicaId, kind)`, upserted. So the table is bounded by fleet
+ * size times the number of kinds, not by time — there is no history here and
+ * there must not be: a history would need a retention policy, and this is a
+ * cache of the present, not a log of the past.
+ *
+ * `payload` is a JSON STRING and small on purpose. Anything big enough to
+ * warrant paging belongs on the process that holds it, reachable per replica —
+ * writing a full subscription list here every heartbeat would be write
+ * amplification for state nobody is reading.
+ */
+export declare const _voltroReplicaObservationsTable: TableLike;
+
 /**
  * `_voltro_seeds` — one row per registered seed. Written to by the
  * seed runner as each seed's lifecycle changes. Unlike migrations,