@voltro/database 0.54.0 → 0.56.0

package/dist/index.d.ts

@@ -621,6 +621,11 @@ export declare interface CdcConfig {
     readonly exclusionRefreshFollows?: boolean;
 }
 
+/** Env var that tunes how long a change-claim row survives. Declared here rather
+ *  than at the retention registration, because the OCCURRENCE COUNTER's window
+ *  has to be derived from the same number and lives in this file. */
+export declare const CHANGE_CLAIM_TTL_ENV = "VOLTRO_CHANGE_CLAIMS_TTL_HOURS";
+
 /** Maximum `onChange` listeners a DataStore's change bus accepts before Node
  *  warns. See this file's header for why it is 512 and why it is not `0`. */
 export declare const CHANGE_LISTENER_CEILING = 512;
@@ -638,6 +643,17 @@ export declare interface ChangeCaptureExclusionRefresh {
 /** A row was removed from the table. `new` is null, always. */
 export declare const changeDelete: (table: string, row: Row, meta?: ChangeEventMeta) => ChangeEvent;
 
+/** The content half of a change identity — equal iff two replicas are looking
+ *  at the same change (or at two changes with byte-identical images, which the
+ *  occurrence counter then separates). */
+export declare const changeDigest: (input: {
+    readonly pipe: string;
+    readonly op: "insert" | "update" | "delete";
+    readonly key: string;
+    readonly next: Record<string, unknown> | null;
+    readonly prev: Record<string, unknown> | null;
+}) => string;
+
 /**
  * A row-change event emitted by the data store (in-memory) or the CDC
  * consumer (Postgres logical replication). Drives the reactive engine.
@@ -766,6 +782,22 @@ export declare const changeSoftDelete: (table: string, row: Row, opts?: {
  */
 export declare type ChangeStrategy = 'inline' | 'cdc';
 
+/**
+ * A proven interruption in a store's change stream.
+ *
+ * Deliberately NOT a list of what was missed. A pub/sub transport keeps no log,
+ * so the missed changes are gone — the only honest recovery is to re-derive
+ * everything downstream, and a consumer that were handed a partial list would be
+ * tempted to patch instead. `reason` is for the log line; the instruction is the
+ * event itself.
+ */
+export declare interface ChangeStreamGap {
+    /** Human-readable cause — goes into the refresh's log line. */
+    readonly reason: string;
+    /** Wall-clock ms the stream was known to be down, when it is known. */
+    readonly downForMs?: number;
+}
+
 /** A row changed. Both sides are present, because an update has both. */
 export declare const changeUpdate: (table: string, before: Row, after: Row, meta?: ChangeEventMeta) => ChangeEvent;
 
@@ -1772,6 +1804,9 @@ export declare const compileRawFragment: <Row extends object = Record<string, un
 
 export declare const compileSelect: <Row extends object>(descriptor: QueryDescriptor, sql: SqlConstructor, namespace?: QueryNamespace) => Statement.Statement<Row>;
 
+/** Compose the durable identity written to `_voltro_cdcout_claims.changeKey`. */
+export declare const composeChangeKey: (digest: string, occurrence: number) => string;
+
 /**
  * Compute the embedded row from a {@link VectorEmbeddingContext}. Returns
  * the patch to merge into the row: `{ [as]: <vector> }`. When the source
@@ -2298,6 +2333,23 @@ export declare interface DataStore {
      * must elect a single worker instead of filtering by origin.
      */
     readonly changeScope?: 'local' | 'fleet';
+    /**
+     * Subscribe to CHANGE-STREAM GAPS — moments where this process proved it
+     * stopped receiving changes for a while and has now started again.
+     *
+     * A gap is not an error to show a user; it is an instruction to the reactive
+     * layer: everything derived from this store may be stale, so re-run it. A
+     * live query is idempotent, which is what makes "re-run all of them" both
+     * safe and the complete recovery. `voltro dev` and `voltro serve` wire this
+     * to the dispatcher's `refreshAll`, the same recovery the broadcast bus's
+     * peer-serial gap uses.
+     *
+     * Optional because only a store whose changes arrive over a TRANSPORT can
+     * have one. An inline store emits its own writes in-process — there is no
+     * connection to lose — and correctly does not implement this. Absent
+     * therefore means "gaps are impossible here", never "gaps are not reported".
+     */
+    readonly onChangeStreamGap?: (listener: (gap: ChangeStreamGap) => void) => () => void;
     /**
      * Bind a per-request tenant NAMESPACE, returning a store view whose
      * every read + write is physically isolated to that namespace
@@ -2478,6 +2530,38 @@ export declare interface DataStore {
      * callers guard with `if (store.refreshChangeCaptureExclusions)`.
      */
     refreshChangeCaptureExclusions?(): Promise<ChangeCaptureExclusionRefresh>;
+    /**
+     * Run an Effect that needs this store's own `SqlClient` — the raw-SQL seam.
+     *
+     * The capability the STAGED SWAP is built on. `--mode replace` loads into
+     * `_voltro_staging_<t>` tables and then swaps the CONTENT in one short
+     * transaction (`DELETE FROM t; INSERT INTO t (…) SELECT … FROM
+     * _voltro_staging_t`), which moves the long, network-bound load OUT of the
+     * destructive transaction. None of that is expressible through the typed row
+     * API: it is DDL plus a server-side `INSERT … SELECT`, and the whole point is
+     * that no row crosses the wire.
+     *
+     * OPTIONAL on the interface, and declared here rather than duck-typed at the
+     * call site for the reason the house rule gives: the four dialect stores are
+     * four hand-written copies of one interface, and a capability nothing declares
+     * is a capability nothing can be red about. A store that loses it now fails to
+     * satisfy this member instead of silently falling out of `canStage()` and
+     * taking the weaker path forever, on one dialect, in silence.
+     *
+     * The DECISION does not live per store — that would be the four-copies trap
+     * wearing a different hat. `@voltro/data-transfer`'s `stagedSwap.ts` writes
+     * the per-dialect SQL ONCE and sends it down this seam; the store contributes
+     * a connection, not a policy.
+     *
+     * Callers guard with `if (store.run)` (or `canStage(store)`), and a store
+     * without it keeps the path it already has: the in-memory store has no SQL
+     * and no catalog, and neither does a bare `DataStore` view.
+     *
+     * `R` is deliberately `SqlClient.SqlClient`: the effect asks for the client
+     * and the store provides its own, which is what makes this a SEAM rather than
+     * a way to hand a store somebody else's connection.
+     */
+    run?<A, E>(effect: Effect.Effect<A, E, SqlClient.SqlClient>): Promise<A>;
 }
 
 /** One transfer, as the inspector reads it back. */
@@ -2692,6 +2776,10 @@ export declare const decryptFieldsOnRead: (rows: ReadonlyArray<Row>, table: Tabl
  */
 export declare const DEFAULT_ACQUIRE_TIMEOUT_MS = 10000;
 
+/** Default lifetime of a change-claim row, in hours. A lock ledger, not a
+ *  history — nobody reads yesterday's claim. */
+export declare const DEFAULT_CHANGE_CLAIM_TTL_HOURS = 1;
+
 /**
  * How often a persistent fallback is allowed to say so.
  *
@@ -3466,7 +3554,6 @@ export declare const formatSubjectGraph: (graph: SubjectGraph) => string;
  */
 export declare const FRAMEWORK_LIVE_TABLES: ReadonlySet<string>;
 
-/** All framework tables as an array — useful for bootstrap DDL emission. */
 export declare const frameworkTables: readonly [TableLike, TableLike, TableLike];
 
 declare type FullTextIndexMeta = {
@@ -3828,6 +3915,13 @@ export declare interface InterruptedReplace {
     readonly tables: number | null;
     readonly capture: string | null;
     readonly via: string;
+    /** The staging tables the run created. Empty for every kind but `staging`. */
+    readonly staging: ReadonlyArray<string>;
+    /** Last sign of life from the run, or `null` when it never wrote one. */
+    readonly heartbeatAt: Date | null;
+    /** `true` when the run was started `--no-atomic` — its staging is a resume
+     *  point, not a leftover. */
+    readonly resumable: boolean;
 }
 
 /**
@@ -3972,6 +4066,41 @@ export declare const isSeedDefinition: (value: unknown) => value is SeedDefiniti
  */
 export declare const isSqliteFamily: (dialect: DialectId) => boolean;
 
+/**
+ * True when `name` is a `--mode replace` staging table.
+ *
+ * ── This is a THIRD rule, and it must not be folded into the other two ─────
+ *
+ * The framework-table filter above is deliberately ASYMMETRIC: a framework
+ * table nobody declares is never planned for a drop, and one we DO declare
+ * diffs like any other table. Collapsing those two into one filter is what
+ * produced a postgres-only evolution path for the framework's own tables, and
+ * the comment in `boot.ts` exists to stop it being collapsed again.
+ *
+ * A staging table satisfies NEITHER half. It is not a schema fact at all — it is
+ * a scratch copy of a target table that exists for the length of one import and
+ * is dropped after the swap. So:
+ *
+ *   - the DROP direction is already covered, because `_voltro_staging_` starts
+ *     with `_voltro_` and `isFrameworkOwnedLiveTable` therefore says "never
+ *     drop it". Correct: a `replace` in flight on another replica is loading
+ *     into it right now.
+ *   - the CREATE direction is NOT, and that is the gap this closes. `--mode
+ *     replace` REGISTERS each staging table as a clone of its target for the
+ *     duration of the load — it has to, or a typed write cannot resolve the
+ *     column types and a `json()` column is written unencoded. A registered
+ *     table is a DECLARED table, and by the asymmetry above a declared
+ *     framework table diffs like any other: a plan computed while an import is
+ *     in flight proposes `create-table _voltro_staging_notes` and, once
+ *     applied, a table nothing will ever declare again is kept forever.
+ *
+ * So staging is excluded from the DECLARED side (in `declaredSnapshot`, which is
+ * the one input both the plan and its fingerprint are built from) and left to
+ * the existing prefix rule on the live side. Two different rules, two different
+ * places, on purpose.
+ */
+export declare const isStagingTableName: (name: string) => boolean;
+
 /**
  * Does this table emit change events at all?
  *
@@ -4527,6 +4656,67 @@ export declare const numeric: (precision: number, scale?: number) => ColumnBuild
  */
 export declare const observeDbOp: <T>(dialect: DialectId | string, op: DbOp, run: () => Promise<T>) => Promise<T>;
 
+/**
+ * How many times this replica has already seen each digest inside the dedup
+ * window. Bounded on BOTH axes — a per-change map with no ceiling is an
+ * unbounded table in memory:
+ *   * by time — entries untouched for `windowMs` are pruned, and
+ *   * by size — `maxEntries` oldest-first, so a burst of unique changes can't
+ *     grow it without limit.
+ *
+ * `windowMs` MUST be longer than the lifetime of a claim row in the database.
+ * That ordering is the invariant: if a claim outlived the counter that
+ * produced it, the next sighting of that digest would restart at occurrence 0,
+ * collide with the retained claim, and be dropped as a duplicate it is not.
+ *
+ * **`maxEntries` is the SAME invariant reached by size, and it is worth saying
+ * plainly rather than reassuringly.** An evicted digest also restarts at 0, so
+ * an eviction is only harmless while no claim for that digest is still retained
+ * — which is not the same thing as "the position a freshly-booted replica is
+ * in", because a fresh replica has no claims of its own to collide with. The
+ * exposure is narrow (it needs more than `maxEntries` distinct digests inside
+ * the claim TTL *and* a content-identical repeat of an evicted one), and it is
+ * real. Size the map against the change rate the gated tables actually carry,
+ * the way `plugin-cdc-out` sizes it against its handoff buffer.
+ */
+export declare class OccurrenceCounter {
+    #private;
+    constructor(opts: {
+        readonly windowMs: number;
+        readonly maxEntries: number;
+    });
+    get size(): number;
+    /** Record a sighting and return its occurrence index (0 for the first). */
+    next(digest: string, nowMs: number): number;
+    /**
+     * Align this replica with the fleet from the claims already recorded in the
+     * database — the counters of a process that booted mid-window would
+     * otherwise restart at 0 and re-key changes the incumbents have already
+     * keyed. Call ONCE, before the first observed change is keyed.
+     */
+    seedFromChangeKeys(changeKeys: Iterable<string>, nowMs: number): void;
+    /** Drop entries whose last sighting is older than the window. */
+    prune(nowMs: number): void;
+}
+
+/**
+ * The counter window that keeps {@link OccurrenceCounter}'s invariant true for
+ * the claim TTL this deployment actually runs with.
+ *
+ * The invariant is stated on the class: the window MUST be longer than the
+ * lifetime of a claim row, or a digest whose counter entry was pruned restarts
+ * at occurrence 0, collides with the retained claim, and is dropped as a
+ * duplicate it is not. Three call sites hard-coded two hours against a
+ * one-hour default — correct until an operator raises the TTL, at which point
+ * the invariant breaks SILENTLY and in the direction that loses effects.
+ *
+ * So it is derived, not written down: twice the TTL, from the same env var the
+ * retention policy reads.
+ */
+export declare const occurrenceWindowMs: (env?: {
+    readonly [key: string]: string | undefined;
+}) => number;
+
 /**
  * What a decrypt failure does. `'throw'` (default, and the ONLY safe production
  * behaviour) surfaces a typed `FieldDecryptionError`. `'null'` degrades the one
@@ -4658,6 +4848,12 @@ export declare const paginateBy: (descriptor: QueryDescriptor, column: string, c
  */
 export declare const paginateById: (descriptor: QueryDescriptor, cursor: string | undefined, limit: number) => QueryDescriptor;
 
+/** Split a stored `changeKey` back into its two halves (`null` when malformed). */
+export declare const parseChangeKey: (changeKey: string) => {
+    digest: string;
+    occurrence: number;
+} | null;
+
 /** Test seam — how many registrations are outstanding. */
 export declare const pendingAttributionCount: () => number;
 
@@ -6593,6 +6789,86 @@ export declare interface SqlDialect {
     readonly retryFilter: (error: unknown) => RetryDecision;
 }
 
+/**
+ * How long a staged run's silence has to last before its scratch tables are
+ * treated as abandoned.
+ *
+ * Generous on purpose. The beat is written every couple of seconds while rows
+ * land, so this is not "how long a load may take" — it is how long a load may
+ * be stalled, or its bookkeeping write failing, before a boot on another
+ * replica decides nobody is coming back. The cost of being wrong here is that
+ * an in-flight import's swap fails with a missing table and the operator
+ * re-runs it; the target is untouched either way, which is what makes an
+ * automatic sweep affordable at all.
+ *
+ * Declared as `dataTransfer.stagingStaleMinutes` in `app.config.ts`;
+ * `VOLTRO_STAGING_STALE_MINUTES` overrides that in turn, on the same rule every
+ * other tunable here follows — an operator acting on a running deployment
+ * outranks what the project declared.
+ */
+export declare const STAGING_STALE_MINUTES = 30;
+
+/**
+ * The prefix a STAGING table carries: `_voltro_staging_<target>`.
+ *
+ * Defined in this leaf module, not in `@voltro/data-transfer` where the swap
+ * lives, because the differ has to know it and `@voltro/database` cannot depend
+ * on the package that imports it. One definition; `stagedSwap.ts` re-exports
+ * this one rather than spelling the prefix a second time.
+ */
+export declare const STAGING_TABLE_PREFIX = "_voltro_staging_";
+
+/**
+ * What a boot says about them.
+ *
+ * A NOTE, never a refusal — the target of a staged run is untouched by
+ * definition. But it is said out loud, because the alternative is what shipped:
+ * a set of tables holding a whole bundle that only an introspection could find,
+ * so nobody knew they existed until a disk filled up.
+ */
+export declare const stagingLeftoverReport: (leftovers: StagingLeftovers, now: number) => string | undefined;
+
+/** One staged run's leftovers, and what a boot may do about them. */
+export declare interface StagingLeftovers {
+    /** Every `kind: 'staging'` row, for the report. */
+    readonly all: ReadonlyArray<InterruptedReplace>;
+    /**
+     * The rows whose tables a boot may DROP: the beat is stale AND the run was
+     * not resumable.
+     *
+     * Both halves are load-bearing and neither is enough alone. A fresh beat is
+     * an import in flight — on this pod or another replica — and dropping its
+     * staging would destroy a load in progress. A resumable run's staging is a
+     * RESUME POINT that is meant to outlive the process, so no amount of silence
+     * makes it collectable.
+     */
+    readonly droppable: ReadonlyArray<InterruptedReplace>;
+}
+
+/**
+ * Decide, from the marker rows alone, which staging sets a boot may collect.
+ *
+ * Pure, and separated from the boot so the DECISION can be tested without a
+ * database — the rule is three lines and every one of them has a way to be
+ * quietly wrong in the direction that destroys someone's work.
+ */
+export declare const stagingLeftovers: (rows: ReadonlyArray<InterruptedReplace>, now: number, staleAfterMs: number) => StagingLeftovers;
+
+/** Split a `kind: 'staging'` row's comma-separated name list. */
+export declare const stagingNamesOf: (raw: unknown) => ReadonlyArray<string>;
+
+/**
+ * The threshold in ms: the env override, else what the app declared, else the
+ * default above.
+ *
+ * A non-positive or unparseable value at EITHER level falls through to the next
+ * one rather than failing. This runs on a boot path whose whole contract is that
+ * scratch-table housekeeping cannot stop a boot, so a typo in a tunable must not
+ * be the thing that does — and the fall-through lands on a threshold that is
+ * generous in the safe direction.
+ */
+export declare const stagingStaleAfterMs: (env: Record<string, string | undefined>, declaredMinutes?: number) => number;
+
 /**
  * Fill `row.id` from the table's declared scheme when it is absent.
  *
@@ -7896,8 +8172,20 @@ export declare const _voltroDataTransfersTable: TableLike;
  * (the `Idempotency-Key` header on REST routes). `scope` namespaces the key
  * per tenant + method + path; the UNIQUE on (scope, key) is the atomic claim
  * arbiter (INSERT/upsert-on-conflict). `response` holds the cached
- * `{ status, body }` for replay. Created only when `idempotency` is set in
- * `app.config.ts`. A periodic sweep / lazy-TTL drops rows past their window.
+ * `{ status, body }` for replay.
+ *
+ * Created for EVERY sql app (`when: 'always'` in the cli's registry), empty
+ * unless `idempotency` is set in `app.config.ts`. This used to say "created only
+ * when idempotency is set", which is what the config gates — not what the table
+ * registry does.
+ *
+ * Two things clean it up, and only one of them existed for most of this table's
+ * life. The LAZY TTL drops a stale row when the SAME key is claimed again — and
+ * a key is used once by definition, so a row whose key never returns was never
+ * read and never deleted. The retention SWEEP is the half that bounds it
+ * (`retentionSweep.ts`), and its window is floored by the app's own dedup window:
+ * a record dropped while still inside that window lets the duplicate request it
+ * exists to stop execute a second time.
  */
 export declare const _voltroIdempotencyTable: TableLike;
 
@@ -7950,6 +8238,43 @@ export declare const _voltroMssqlCdcOffsetsTable: TableLike;
  */
 export declare const _voltroReplaceInProgressTable: TableLike;
 
+/**
+ * `_voltro_replica_observations` — what each replica knows about itself, put
+ * somewhere every replica can read it.
+ *
+ * ── Why the shared store and not a fan-out ────────────────────────────────
+ *
+ * The fleet-wide questions an operator asks — "how many subscriptions is each
+ * pod holding", "which of my queries can resume anywhere" — are answers each
+ * process holds privately. Asking the others at read time means inventing
+ * request/response over a pub/sub bus: a correlation id, a reply channel, a
+ * deadline to guess, and a partial-result protocol. And it puts the DIAGNOSTIC
+ * on the failure path, so the moment you most need it is the moment it times
+ * out.
+ *
+ * Writing instead inverts that. The transport is the database — the one thing
+ * that must be up for anything to work — and a replica that dies goes STALE
+ * rather than silent: staleness is a number, and a number can be reported.
+ * Paired with `instanceMembership`'s roster, "old" becomes "missing", which is
+ * the honest partial result the fan-out would have had to build by hand.
+ *
+ * `coordinationState.recentReplicaIds` already answers a fleet question this
+ * way, by reading `_voltro_schedule_runs`. This generalises that one case.
+ *
+ * ── The shape, and what it must not become ────────────────────────────────
+ *
+ * One row per `(replicaId, kind)`, upserted. So the table is bounded by fleet
+ * size times the number of kinds, not by time — there is no history here and
+ * there must not be: a history would need a retention policy, and this is a
+ * cache of the present, not a log of the past.
+ *
+ * `payload` is a JSON STRING and small on purpose. Anything big enough to
+ * warrant paging belongs on the process that holds it, reachable per replica —
+ * writing a full subscription list here every heartbeat would be write
+ * amplification for state nobody is reading.
+ */
+export declare const _voltroReplicaObservationsTable: TableLike;
+
 /**
  * `_voltro_seeds` — one row per registered seed. Written to by the
  * seed runner as each seed's lifecycle changes. Unlike migrations,