@happyvertical/smrt-core 0.37.8 → 0.37.10

package/AGENTS.md

@@ -73,6 +73,16 @@ admin auth.
 - Tables: `_smrt_dispatch`, `_smrt_dispatch_subscriptions`
 - Status: `pending → processing → completed` (or `failed`)
 
+## Change Feed (#1758)
+
+Adapter-agnostic change-observation spine (`src/change-feed.ts`) — the server half of the client/mobile sync contract (PRD #1755):
+
+- `_smrt_changes` system table: one append per framework save/delete via a GlobalInterceptors writer registered at framework init. Deletes are tombstones (`operation: 'delete'`). `_smrt_*` tables are skipped. Feed-append failures log and never fail the user's write. No dirty-check: a field-unchanged `.save()` appends a spurious `update` entry (diff-aware paths like `getOrUpsert`/sync-apply short-circuit before `save()` and append nothing); subscribers must tolerate spurious entries — they are convergent.
+- Sequences: allocated as `MAX(seq)+1` inside the INSERT with conflict retry — committed rows stay contiguous, so commit order == seq order on SQLite/Postgres/DuckDB (deliberately NOT identity/serial: those allocate before commit and break the cursor guarantee under concurrent writers).
+- `getChangesSince(db, { since, tables?, tenantId?, limit? }) → { changes, cursor, resyncRequired? }`: strictly monotonic cursor; polling with returned cursors misses no committed change and never repeats one. A cursor that cannot be served incrementally — pruned below the retained `[floor..horizon]` run, or foreign/ahead of the horizon — gets `resyncRequired: true` with empty `changes` and an unadvanced cursor; detection runs on the UNFILTERED log so `tables`/`tenantId` filters never trigger or mask it. `getTenantScopedChangesSince()` resolves tenant via the DispatchBus resolver hook (fail-closed: tenancy on + no context → global rows only; tenant `T` sees `T` + global rows, never another tenant).
+- Generated `_changes` routes: REST (`GET {basePath}/_changes`, requires `authMiddleware`, otherwise 401 — per-model `api.public` does NOT apply) and SvelteKit (`{routesDir}/_changes/+server.ts`, requires an authenticated principal on `locals`; opt out via `sveltekit.changesRoute.enabled: false`). Query params: `since`, `tables` (comma-separated), `limit`. Responses stay HTTP 200 in the resync state — `resyncRequired` is protocol state, not an error.
+- Retention: `pruneChangeFeed(db, { maxAgeMs?, maxRows? })` — schedule it. Pruning deletes oldest-first and always retains the newest entry (a non-empty feed is never emptied), which is what makes pruned-cursor detection provable and keeps caught-up consumers polling normally. Raw-SQL writes are invisible to the feed (same documented gap as the #1499 cache); `bumpChangeFeed(db, { table, rowId? })` is the manual escape hatch.
+
 ## Single Table Inheritance (STI)
 
 - Base: `@smrt({ tableStrategy: 'sti' })` — children inherit, share one table

package/dist/change-feed.d.ts

@@ -0,0 +1,261 @@
+import { DatabaseInterface } from '@happyvertical/sql';
+/** Name of the append-only change-feed system table. */
+export declare const CHANGE_FEED_TABLE = "_smrt_changes";
+/** Interceptor name of the framework's change-feed writer. */
+export declare const CHANGE_FEED_INTERCEPTOR_NAME = "smrt-change-feed";
+/**
+ * Change operations recorded in the feed. Deletes are tombstones —
+ * consumers can distinguish "row changed" from "row is gone" without
+ * consulting the source table.
+ */
+export type ChangeOperation = 'create' | 'update' | 'delete';
+/** One entry of the change feed. */
+export interface ChangeFeedEntry {
+    /** Strictly monotonic per-database sequence (the cursor dimension). */
+    seq: number;
+    /** Physical table the change happened in (STI children report the shared base table). */
+    table: string;
+    /**
+     * Primary key of the changed row, or `null` for table-level synthetic
+     * bumps recorded via {@link bumpChangeFeed} without a row id.
+     */
+    rowId: string | null;
+    /** What happened. `'delete'` entries double as tombstones. */
+    operation: ChangeOperation;
+    /** Tenant the changed row belongs to, or `null` for global/non-tenant rows. */
+    tenantId: string | null;
+    /** ISO-8601 timestamp recorded when the change was appended. */
+    timestamp: string;
+}
+/** Options for {@link getChangesSince}. */
+export interface GetChangesOptions {
+    /**
+     * Cursor to read after. Only rows with `seq` strictly greater than `since`
+     * are returned; pass a previously returned {@link ChangeFeedPage.cursor} to
+     * poll.
+     *
+     * `0` reads from the start of the log only while it has not been pruned past
+     * the beginning. Once retention has raised the retained floor above the
+     * start, `since: 0` (like any cursor older than the retained window) can no
+     * longer be served incrementally — the read returns
+     * {@link ChangeFeedPage.resyncRequired} and the caller must do a full
+     * resync.
+     */
+    since: number;
+    /** Restrict to these physical table names. Empty/omitted → all tables. */
+    tables?: string[];
+    /**
+     * Tenant visibility filter:
+     * - omitted/`undefined` → no tenant filter (all rows).
+     * - `null` → only global rows (`tenant_id IS NULL`).
+     * - `'<tenantId>'` → that tenant's rows **plus** global rows, matching the
+     *   DispatchBus read rule (`tenant_id = T OR tenant_id IS NULL`). A tenant
+     *   never sees another tenant's changes.
+     */
+    tenantId?: string | null;
+    /**
+     * Page size (default {@link DEFAULT_CHANGES_LIMIT}, capped at
+     * {@link MAX_CHANGES_LIMIT}). When a page fills up, the returned cursor
+     * stops at the last returned row so the next poll continues seamlessly.
+     */
+    limit?: number;
+}
+/** Result page of {@link getChangesSince}. */
+export interface ChangeFeedPage {
+    /** Matching changes ordered by ascending `seq`. */
+    changes: ChangeFeedEntry[];
+    /**
+     * The next cursor. Monotonic: never lower than the `since` it was derived
+     * from. Equal to the committed horizon when the page was exhaustive, or to
+     * the last returned `seq` when the page hit `limit`. Feed the value back
+     * as `since` to observe every later change exactly once.
+     */
+    cursor: number;
+    /**
+     * Present (and `true`) when the supplied cursor cannot be served
+     * incrementally and the consumer must fall back to a full resync:
+     *
+     * - the cursor predates the retained window (entries at or below it were
+     *   pruned away — the changes between it and the retained floor are gone
+     *   for good), or
+     * - the cursor is ahead of the committed horizon / unknown to this
+     *   database (a foreign or reset cursor).
+     *
+     * When set, `changes` is empty and `cursor` echoes `since` unchanged —
+     * the consumer must re-fetch its data in full and restart polling from
+     * the cursor returned by its post-resync read. Detection is computed on
+     * the **unfiltered** log: `tables`/`tenantId` filters legitimately hide
+     * rows and never trigger (or mask) a resync signal.
+     */
+    resyncRequired?: boolean;
+}
+/** Input for {@link appendChange} / {@link bumpChangeFeed}. */
+export interface AppendChangeInput {
+    /** Physical table name the change refers to. */
+    table: string;
+    /** Changed row's primary key; `null`/omitted records a table-level change. */
+    rowId?: string | null;
+    /** Operation to record (default `'update'`). */
+    operation?: ChangeOperation;
+    /** Tenant the change belongs to (default `null` = global). */
+    tenantId?: string | null;
+}
+/** Retention bounds for {@link pruneChangeFeed}. At least one is required. */
+export interface ChangeFeedRetention {
+    /** Prune entries older than this many milliseconds. */
+    maxAgeMs?: number;
+    /** Keep at most this many newest entries (by sequence). */
+    maxRows?: number;
+}
+/** Default page size for {@link getChangesSince}. */
+export declare const DEFAULT_CHANGES_LIMIT = 500;
+/** Hard cap on the page size for {@link getChangesSince}. */
+export declare const MAX_CHANGES_LIMIT = 5000;
+export declare function ensureChangeFeedTable(db: DatabaseInterface): Promise<void>;
+/**
+ * Append one change entry with a database-allocated, strictly monotonic
+ * sequence.
+ *
+ * The sequence is allocated inside the INSERT itself
+ * (`COALESCE(MAX(seq), 0) + 1`) and retried on primary-key conflict, which
+ * keeps committed sequences contiguous and makes commit order equal
+ * sequence order — the property the cursor guarantee rests on (see the
+ * module docs). Throws after {@link MAX_APPEND_ATTEMPTS} consecutive
+ * conflicts or on any non-conflict database error; the framework's
+ * interceptor catches and logs instead of failing the user's write.
+ *
+ * **Transaction assumption.** The conflict retry assumes an autocommit
+ * handle, which is the default framework path: `save()`/`delete()` run their
+ * statements as independent autocommit calls, so a conflicting append aborts
+ * nothing and the retry recomputes `MAX(seq)` against the committed head. If a
+ * caller instead wraps `save()` in its own transaction on an MVCC engine
+ * (e.g. Postgres), a duplicate-key error can abort the *surrounding*
+ * transaction — the retry is then futile and the caller's write rolls back.
+ * Callers wrapping mutations in a transaction under genuine seq-head write
+ * contention should disable the feed writer for that path or accept that
+ * risk. Savepoint-scoped protection is a possible follow-up; it is
+ * intentionally out of scope for the v1 spine.
+ */
+export declare function appendChange(db: DatabaseInterface, input: AppendChangeInput): Promise<void>;
+/**
+ * Manual bump escape hatch for out-of-band writers.
+ *
+ * Framework mutation paths feed the log automatically, but raw SQL issued
+ * outside `save()`/`delete()` is invisible to it (documented gap, shared
+ * with the #1499 collection cache). Call this after such a write so feed
+ * consumers observe the change. Omitting `rowId` records a table-level
+ * change (`rowId: null`), which consumers should treat as "anything in this
+ * table may have changed".
+ *
+ * @example
+ * ```typescript
+ * await db.query(`UPDATE products SET price = price * 1.1`);
+ * await bumpChangeFeed(db, { table: 'products' });
+ * ```
+ */
+export declare function bumpChangeFeed(db: DatabaseInterface, input: AppendChangeInput): Promise<void>;
+/**
+ * Read committed changes after a cursor.
+ *
+ * Returns every committed change with `since < seq <= cursor` that matches
+ * the filters, ordered by ascending `seq`. The returned cursor is safe to
+ * persist and poll with: committed sequences are contiguous (see module
+ * docs), so nothing can commit at or below the observed horizon afterwards —
+ * reads miss no committed change under concurrent writers and never return
+ * the same change twice. When `since` is already at the horizon, returns an
+ * empty page with `cursor: since`.
+ *
+ * ## Resync detection (pruned / foreign cursors)
+ *
+ * A cursor that cannot be served incrementally is flagged with
+ * `resyncRequired: true` (empty `changes`, `cursor` echoed unchanged) so
+ * pollers never go silently, permanently stale:
+ *
+ * - **Pruned gap**: retained sequences always form a contiguous run
+ *   `[floor..horizon]` and {@link pruneChangeFeed} deletes oldest-first
+ *   while always retaining the newest entry, so `since < floor - 1` proves
+ *   changes between the cursor and the retained window were pruned away.
+ * - **Foreign/reset cursor**: `since > horizon` (ahead of anything this
+ *   database ever allocated), including any `since > 0` against a feed
+ *   with no entries.
+ *
+ * Detection runs on the **unfiltered** log — `tables`/`tenantId` filters
+ * legitimately hide rows and never trigger (or mask) the signal. A caught-up
+ * consumer (`since === horizon`) is never asked to resync, even when
+ * retention has pruned everything older.
+ *
+ * Filters (`tables`, `tenantId`) affect which rows are *returned*, never how
+ * the cursor advances — an exhausted filtered page still advances to the
+ * horizon so pollers do not rescan filtered-out rows.
+ */
+export declare function getChangesSince(db: DatabaseInterface, options: GetChangesOptions): Promise<ChangeFeedPage>;
+/**
+ * {@link getChangesSince} scoped by the active tenant context.
+ *
+ * Resolves the tenant through the same dependency-inversion hook the
+ * DispatchBus uses ({@link resolveDispatchTenantScope}), so it works without
+ * core depending on `@happyvertical/smrt-tenancy`:
+ *
+ * - Tenancy disabled (no resolver registered) → no tenant filter.
+ * - Tenancy enabled with an active tenant `T` → `T`'s rows plus global rows.
+ * - Tenancy enabled with **no** active tenant → global rows only
+ *   (**fail-closed**: a missing context never widens visibility to all
+ *   tenants).
+ *
+ * This is the read the generated `_changes` routes call after establishing
+ * tenant context from the authenticated principal.
+ */
+export declare function getTenantScopedChangesSince(db: DatabaseInterface, options: Omit<GetChangesOptions, 'tenantId'>): Promise<ChangeFeedPage>;
+/**
+ * Prune the change feed to bound its growth.
+ *
+ * Applies whichever bounds are provided (at least one is required):
+ * - `maxRows`: keep only the newest N entries by sequence.
+ * - `maxAgeMs`: drop entries older than the cutoff.
+ *
+ * Pruning deletes oldest-first, never renumbers surviving entries, and
+ * **always retains the newest entry** (a non-empty feed is never emptied,
+ * whatever the bounds say). That invariant anchors pruned-cursor detection:
+ * retained sequences stay a contiguous run `[floor..horizon]`, so
+ * {@link getChangesSince} can prove a cursor predates the retained window
+ * (`resyncRequired`) — and a fully caught-up consumer keeps polling
+ * normally even after everything older was pruned.
+ *
+ * Cursors within the retained window keep working. Schedule pruning (e.g.
+ * via `@happyvertical/smrt-jobs`) with a retention window comfortably
+ * larger than the slowest consumer's polling interval; consumers whose
+ * cursor falls out of it are told to full-resync via `resyncRequired`.
+ *
+ * @returns The number of entries pruned (approximate under concurrent prunes).
+ */
+export declare function pruneChangeFeed(db: DatabaseInterface, retention: ChangeFeedRetention): Promise<{
+    pruned: number;
+}>;
+/**
+ * Register the change-feed writer with {@link GlobalInterceptors}.
+ *
+ * Called automatically during framework initialization (every
+ * `SmrtClass.initialize()` passes through it), so applications never need
+ * to call it directly; it is exported for tests and for re-registering
+ * after `GlobalInterceptors.clear()`. Idempotent — a second call while the
+ * writer is registered is a no-op.
+ *
+ * The writer observes the same hooks the reports scheduler and tenancy
+ * interceptors use:
+ * - `beforeSave` stashes whether the instance was already persisted (this
+ *   is what distinguishes `create` from `update` in the feed).
+ * - `afterSave`/`afterDelete` append exactly one change entry per framework
+ *   save/delete. `_smrt_*` system tables are skipped — the feed observes
+ *   application data, not framework bookkeeping (and never itself).
+ *
+ * Failure policy: appends run after the user's write succeeded and must not
+ * un-succeed it — failures are logged (deduped per database) and swallowed.
+ */
+export declare function registerChangeFeedWriter(): void;
+/** Unregister the change-feed writer (test helper). */
+export declare function unregisterChangeFeedWriter(): boolean;
+/**
+ * Reset the append-failure warning dedup (test helper).
+ */
+export declare function resetChangeFeedWarnings(): void;
+//# sourceMappingURL=change-feed.d.ts.map

package/dist/change-feed.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-feed.d.ts","sourceRoot":"","sources":["../src/change-feed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsFG;AAGH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAU5D,wDAAwD;AACxD,eAAO,MAAM,iBAAiB,kBAAkB,CAAC;AAEjD,8DAA8D;AAC9D,eAAO,MAAM,4BAA4B,qBAAqB,CAAC;AAE/D;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE7D,oCAAoC;AACpC,MAAM,WAAW,eAAe;IAC9B,uEAAuE;IACvE,GAAG,EAAE,MAAM,CAAC;IACZ,yFAAyF;IACzF,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,8DAA8D;IAC9D,SAAS,EAAE,eAAe,CAAC;IAC3B,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,2CAA2C;AAC3C,MAAM,WAAW,iBAAiB;IAChC;;;;;;;;;;;OAWG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8CAA8C;AAC9C,MAAM,WAAW,cAAc;IAC7B,mDAAmD;IACnD,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B;;;;;OAKG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;;;;;;;;;;;;;OAeG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,+DAA+D;AAC/D,MAAM,WAAW,iBAAiB;IAChC,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,gDAAgD;IAChD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,8EAA8E;AAC9E,MAAM,WAAW,mBAAmB;IAClC,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qDAAqD;AACrD,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,6DAA6D;AAC7D,eAAO,MAAM,iBAAiB,OAAQ,CAAC;AAwEvC,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,iBAAiB,GACpB,OAAO,CAAC,IAAI,CAAC,CASf;AAMD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,iBAAiB,EACrB,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,IAAI,CAAC,CAwCf;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,iBAAiB,EACrB,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,IAAI,CAAC,CAGf;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,iBAAiB,EACrB,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,cAAc,CAAC,CAyFzB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,2BAA2B,CAC/C,EAAE,EAAE,iBAAiB,EACrB,OAAO,EAAE,IAAI,CAAC,iBAAiB,EAAE,UAAU,CAAC,GAC3C,OAAO,CAAC,cAAc,CAAC,CAMzB;AA4BD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,iBAAiB,EACrB,SAAS,EAAE,mBAAmB,GAC7B,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CA2C7B;AAgCD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,wBAAwB,IAAI,IAAI,CAuC/C;AAED,uDAAuD;AACvD,wBAAgB,0BAA0B,IAAI,OAAO,CAEpD;AAuDD;;GAEG;AACH,wBAAgB,uBAAuB,IAAI,IAAI,CAE9C"}