@jarenjs/db 0.66.1 → 0.67.0

package/src/store.js

@@ -29,16 +29,21 @@ import { chain, toPromise, isThenable, attempt } from './driver.js';
 import { planCollection, planEntity, planJoinTable, verifyShape } from './ddl.js';
 import { translatePatch } from './patch-sql.js';
 import { createQueryEngine, createQueryState, createEntityQueryEngine, createLoadEngine } from './query.js';
-import { admitCursor, admitSyncCursor } from './cursor.js';
+import { admitCursor, admitSyncCursor, createCursor, drainPage, utf8Length } from './cursor.js';
 import { refuseUnsupportedPragmaKeys, resolvePragmaRequests, configurePragmas } from './pragmas.js';
 import { createMaintenance } from './maintenance.js';
 import { createBackup } from './backup.js';
 import { normalizeProfile, assertProfileRoots } from './profile.js';
-import { normalizeEntities, explainMapping } from './model.js';
+import { normalizeEntities, explainMapping, joinTableRoots } from './model.js';
 import { entityCore } from './entity.js';
 import { createTracker, membershipKeys } from './tracker.js';
 import { createCaptureEngine, DEFAULT_RETENTION } from './capture.js';
+import { createReplicationEngine } from './replication.js';
+import { REPLICATION_DEFAULTS } from './replication-format.js';
+import { createLogicalRows } from './logical-rows.js';
+import { shapeHash } from './migrate.js';
 import { createLiveRegistry, classifyLiveQuery, LIVE_DEFAULTS } from './live.js';
+import { classifyEntityLive } from './live-join.js';
 import { normalizeEventTime } from './live-time.js';
 import { createJobEngine } from './jobs.js';
 import { introspectModel } from './introspect.js';
@@ -961,6 +966,9 @@ export function openStore(model, options) {
     collections = normalizeModel(model, options.expressions);
     entities = normalizeEntities(model);
     mapping = entities.size > 0 ? explainMapping(model) : null;
+    if (options.replication !== undefined && [...collections.keys(), ...entities.keys()]
+      .some((name) => name.toLowerCase().startsWith('_jaren_replica')))
+      throw new DbCompileError('JD0060', 'replication reserves table names beginning with _jaren_replica');
     if (collections.size === 0 && entities.size === 0) {
       throw modelError('JD0005',
         'the model must declare at least one collection or entity', '');
@@ -1439,10 +1447,13 @@ export function openStore(model, options) {
           };
 
           // ————— change capture (LIVE-FORMAT §§1–6) —————
-          const captureRequested = options.capture === true
+          const captureOption = options.capture ?? (options.replication === undefined ? undefined : true);
+          const captureRequested = captureOption === true
             ? {}
-            : (options.capture === undefined || options.capture === false
-              ? null : options.capture);
+            : (captureOption === undefined || captureOption === false
+              ? null : captureOption);
+          if (options.replication !== undefined && (captureRequested === null || readOnly))
+            throw new TypeError('replication requires a writable store with capture enabled');
           let captureMode = 'none';
           if (captureRequested !== null) {
             const wanted = captureRequested.mode ?? 'auto';
@@ -1468,6 +1479,9 @@ export function openStore(model, options) {
               ? (hasSessions ? 'session' : 'journal')
               : wanted;
           }
+          if (options.replication !== undefined && captureMode === 'journal'
+            && Object.values(mapping?.entities ?? {}).some((entity) => entity.foreignKeys.some((fk) => fk.onDelete !== 'restrict')))
+            throw new DbCompileError('JD0051', 'journal replication cannot capture cascading or set-null child relations; use session capture');
           const captureShapes = new Map();
           if (captureMode !== 'none') {
             for (const [collectionName, plan] of plans) {
@@ -1525,6 +1539,7 @@ export function openStore(model, options) {
           // a column upgrade; a read-only store creates nothing and takes
           // no lock
           const firstOpen = readOnly ? (fn) => fn() : (fn) => immediately(connection, fn);
+          let replicationEngine = null;
           const capture = captureMode === 'none' ? null : createCaptureEngine({
             connection,
             bracket: firstOpen,
@@ -1534,6 +1549,7 @@ export function openStore(model, options) {
               || (captureRequested.log !== undefined && captureRequested.log !== false),
             retention: captureRequested.log?.retention ?? DEFAULT_RETENTION,
             now: runtime.now,
+            beforeCommit: (patch, context) => replicationEngine?.commit(patch, context),
           });
           // the capture scope around a write runs statements of its own
           // (a session's changeset read, the journal's old-row read, the
@@ -1562,6 +1578,7 @@ export function openStore(model, options) {
           const liveRegistry = capture === null ? null : createLiveRegistry({
             maxQueries: options.live?.maxQueries ?? LIVE_DEFAULTS.maxQueries,
             maxMaintained: options.live?.maxMaintained ?? LIVE_DEFAULTS.maxMaintained,
+            maxBytes: options.live?.maxBytes ?? LIVE_DEFAULTS.maxBytes,
           });
           if (capture !== null) {
             capture.observe((record) => /** @type {any} */ (liveRegistry).deliver(record));
@@ -1613,6 +1630,8 @@ export function openStore(model, options) {
               classification,
               execute: (doc, executeOptions) => core.execute(doc, executeOptions),
               readRow: (token) => core.get(token),
+              rowPosition: (token) => createLogicalRows({ connection, shapes: captureShapes, capture,
+                collectionCore: coreFor, entityCore: entityCoreFor }).position(core.model.name, token),
               keyOf: (doc) => String(extractKey(doc, core.model.keySegments,
                 core.model.key, core.model.name, core.model.docPath)),
             }));
@@ -1655,13 +1674,24 @@ export function openStore(model, options) {
                 const sql = `SELECT ${columns.map(dialect.quoteIdentifier).join(', ')} `
                   + `FROM ${dialect.quoteIdentifier(joinName)} `
                   + `WHERE ${dialect.quoteIdentifier(own.column)} = ${dialect.parameterRef(1, 'v')}`;
-                return chain(connection.prepare(sql), (statement) =>
-                  chain(statement.all([keyParts[0]]), (rows) => {
+                const boundedRows = () => {
+                  const { maxOperations = REPLICATION_DEFAULTS.maxOperations, maxBytes = REPLICATION_DEFAULTS.maxBytes } = options.replication;
+                  const cursor = createCursor({ streaming: 'row', barrier: null,
+                    open: () => chain(connection.prepare(`${sql} LIMIT ?`), (statement) => statement.iterate([keyParts[0], maxOperations + 1])),
+                    items: (row) => [row] });
+                  return chain(drainPage(cursor, { limit: maxOperations, maxBytes,
+                    sizeOf: (row) => utf8Length(JSON.stringify(row)), continuationOf: () => null }), (page) => {
+                    if (page.hasMore) throw new DbRuntimeError('JD2106', 'membership cascade exceeds replication capacity');
+                    return page.items;
+                  });
+                };
+                return chain(options.replication === undefined
+                  ? chain(connection.prepare(sql), (statement) => statement.all([keyParts[0]])) : boundedRows(), (rows) => {
                     for (const row of rows) {
                       capture.record(joinName, columns.map((column) => row[column]), undefined, null);
                     }
                     return nextJoin(i + 1);
-                  }));
+                  });
               };
               return nextJoin(0);
             };
@@ -2170,23 +2200,27 @@ export function openStore(model, options) {
                 'live eventTime maintains a collection view — an entity document re-runs, '
                 + 'so a watermark would describe nothing (LIVE-FORMAT §13)');
             }
-            const roots = collectEntityRoots(document, entities);
+            const roots = collectEntityRoots(document, new Map([...entities, ...joinTableRoots(entities, mapping).entities]));
             if (roots.size === 0) {
               throw new TypeError(
                 'store.live takes an entity-root document — for a collection, '
                 + 'use store.collection(name).live');
             }
+            const logicalRows = createLogicalRows({ connection, shapes: captureShapes, capture,
+              collectionCore: coreFor, entityCore: entityCoreFor });
             return closeOnRollback(liveRegistry.register({
               name: [...roots].join('+'),
               tables: roots,
               document,
               externals: liveOptions?.externals ?? {},
               demanded: liveOptions?.mode,
-              classification: {
+              classification: liveOptions?.mode === 'rerun' ? {
                 strategy: 'rerun',
-                reason: 'entity queries re-run in this version',
-              },
+                reason: 're-run mode was explicitly requested',
+              } : classifyEntityLive(document, entities, mapping, operators),
               execute: (doc, executeOptions) => entityEngine.execute(doc, executeOptions),
+              readDependency: logicalRows.read,
+              dependencyPosition: logicalRows.position,
               readRow: null,
               keyOf: null,
             }));
@@ -2754,6 +2788,7 @@ export function openStore(model, options) {
               // member is ABSENT rather than a second way to close the
               // raw connection under its own savepoint
               close: override(undefined),
+              replication: override(undefined),
               // nor does it run maintenance: a checkpoint inside an open
               // transaction is a no-op the engine answers quietly, and
               // the other three are store-level operations — ABSENT here
@@ -3010,8 +3045,26 @@ export function openStore(model, options) {
             });
           }
           return chain(capture === null ? null : capture.ready,
-            () => chain(jobsEngine === null ? null : jobsEngine.ready,
-              () => Object.freeze(store)));
+            () => chain(jobsEngine === null ? null : jobsEngine.ready, () => {
+          if (options.replication !== undefined) {
+            replicationEngine = createReplicationEngine({ connection, capture,
+              config: options.replication, model: shapeHash(model), now: runtime.now, bracket: firstOpen,
+              rows: createLogicalRows({ connection, shapes: captureShapes, capture,
+                collectionCore: coreFor, entityCore: entityCoreFor, captureJoinDelete }),
+            });
+            store.replication = Object.freeze({
+              frontier: lift(() => gated(() => replicationEngine.frontier())),
+              page: lift((request) => topLevelTransaction(() => replicationEngine.page(request), request?.signal, undefined, 'immediate')),
+              conflicts: lift((request) => gated(() => replicationEngine.conflicts(request), 'replication conflict read', request?.signal)),
+              snapshot: lift((request) => topLevelTransaction(() => replicationEngine.snapshot(request), request?.signal, undefined, 'immediate')),
+              reset: lift((snapshot, request) => replicationEngine.reset(snapshot, request,
+                (fn) => topLevelTransaction(fn, request?.signal, createUnitOfWork(), 'immediate'))),
+              apply: lift((envelope, request) => replicationEngine.apply(envelope, request,
+                (fn) => topLevelTransaction(fn, request?.signal, createUnitOfWork(), 'immediate'))),
+            });
+          }
+          return chain(replicationEngine === null ? null : replicationEngine.ready, () => Object.freeze(store));
+          }));
         })))));
 
       /**

package/types/index.d.ts

@@ -709,6 +709,7 @@ export interface SyncStore {
 }
 
 export interface Store {
+  readonly replication?: Replication;
   readonly capabilities: StoreCapabilities;
   readonly dialect: Dialect;
   stats(): StoreStats;
@@ -867,7 +868,7 @@ export interface TransactionSyncStore extends SyncStore {
  */
 export interface TransactionStore extends Omit<Store,
   'close' | 'transaction' | 'sync' | 'checkpoint' | 'integrityCheck' | 'foreignKeyCheck' | 'optimize' | 'backupTo'
-  | 'jobs'> {
+  | 'jobs' | 'replication'> {
   /** The transactional outbox (JOBS-FORMAT §3): no administration here —
    * an admin operation is a root call. */
   readonly jobs?: JobsApi;
@@ -978,7 +979,7 @@ export interface LiveEventTime {
 
 export interface LiveMode {
   readonly strategy: 'rows' | 'window' | 'accumulator' | 'group'
-    | 'bucket' | 'rolling' | 'rerun';
+    | 'bucket' | 'rolling' | 'join' | 'graph' | 'nested-group' | 'rerun';
   readonly mode: 'incremental' | 'rerun';
   /** Present exactly when the strategy is 'rerun': the named reason. */
   readonly reason?: string;
@@ -1004,6 +1005,9 @@ export interface LiveEvent {
 }
 
 export interface LiveStats {
+  dependencyReads?: number;
+  refreshedRoots?: number;
+  refreshedGroups?: number;
   records: number;
   matched: number;
   emissions: number;
@@ -1037,6 +1041,8 @@ export interface LiveQuery {
 }
 
 export interface LiveBounds {
+  /** Serialized input/output cache credit for join, graph and nested-group strategies. */
+  maxBytes?: number;
   /** Registrations beyond it are JD0052 (default 64). */
   maxQueries?: number;
   /** Per-query ceiling on maintained entries — rows, window entries
@@ -1045,6 +1051,7 @@ export interface LiveBounds {
 }
 
 export interface OpenStoreOptions {
+  replication?: ReplicationOptions;
   driver: Driver;
   path?: string;
   /**
@@ -1605,7 +1612,7 @@ export declare function createLiveRegistry(
   bounds: { maxQueries: number; maxMaintained: number }): unknown;
 export declare function diffRows(oldRows: readonly unknown[], newRows: readonly unknown[]):
   Array<{ op: string; path: string; value?: unknown }>;
-export declare const LIVE_DEFAULTS: { maxQueries: number; maxMaintained: number };
+export declare const LIVE_DEFAULTS: { maxQueries: number; maxMaintained: number; maxBytes: number };
 export declare function createSortedWindow(
   terms: unknown[], limit: number | null): unknown;
 export declare function compareCodepoint(a: string, b: string): number;
@@ -1898,3 +1905,55 @@ export declare const JOB_DEFAULTS: Readonly<{
 export declare function describeValue(value: unknown): string;
 /** A job result as the queue stores it: JSON text, or the reason it could not be. */
 export declare function serializeResult(value: unknown): unknown;
+
+/** Transport-neutral, net logical operations in a single transaction. */
+export interface ReplicationOperation {
+  table: string;
+  key: string;
+  before: Record<string, unknown> | null;
+  after: Record<string, unknown> | null;
+}
+export type ReplicationFrontier = Record<string, number>;
+export interface ReplicationEnvelope {
+  $replication: '0.1'; replica: string; seq: number; model: string;
+  frontier: ReplicationFrontier; operations: ReplicationOperation[];
+}
+export interface ReplicationConflict {
+  envelope: string; table: string; key: string;
+  base: Record<string, unknown> | null;
+  local: { value: Record<string, unknown> | null; frontier: ReplicationFrontier };
+  remote: { value: Record<string, unknown> | null; replica: string; seq: number; frontier: ReplicationFrontier };
+  resolver: string | null;
+  resolution: { action: 'local' | 'remote' | 'merged'; value: Record<string, unknown> | null } | null;
+}
+export interface ReplicationOptions {
+  replica: string; retention?: number; maxOperations?: number; maxBytes?: number;
+  resolver?: { id: string; resolve(conflict: Readonly<ReplicationConflict>):
+    { action: 'local' | 'remote' } | { action: 'merged'; value: Record<string, unknown> | null } };
+}
+export interface ReplicationRequest { signal?: AbortSignal; deadline?: number }
+export interface Replication {
+  snapshot(request?: ReplicationRequest): Promise<ReplicationSnapshot>;
+  reset(snapshot: ReplicationSnapshot, request?: ReplicationRequest): Promise<{ status: 'reset'; frontier: ReplicationFrontier }>;
+  frontier(): Promise<ReplicationFrontier>;
+  apply(envelope: ReplicationEnvelope, request?: ReplicationRequest): Promise<{
+    status: 'applied' | 'duplicate' | 'conflict'; frontier: ReplicationFrontier; conflicts: ReplicationConflict[];
+  }>;
+  page(request?: ReplicationRequest & { after?: number; limit?: number; maxBytes?: number }): Promise<{
+    items: ReplicationEnvelope[]; earliestAvailable: number | null; highWatermark: number;
+    next?: number; bytes?: number; hasMore: boolean; resetRequired: boolean;
+  }>;
+  conflicts(request?: ReplicationRequest & { limit?: number; maxBytes?: number }): Promise<ReplicationConflict[]>;
+}
+export declare const REPLICATION_VERSION: '0.1';
+export declare const REPLICATION_DEFAULTS: Readonly<{ retention: number; maxOperations: number; maxBytes: number }>;
+export declare function normalizeFrontier(value: unknown): ReplicationFrontier;
+export declare function replicationIdentity(replica: string, seq: number): string;
+export declare function normalizeReplication(document: unknown): ReplicationEnvelope;
+export declare function encodeReplication(document: unknown): string;
+export interface ReplicationSnapshot {
+  $replicationSnapshot: '0.1'; model: string; frontier: ReplicationFrontier;
+  rows: { table: string; key: string; value: Record<string, unknown> | null; frontier: ReplicationFrontier }[];
+  receipts: ReplicationEnvelope[];
+}
+export declare function normalizeReplicationSnapshot(document: unknown): ReplicationSnapshot;

package/types/typed.d.ts

@@ -17,7 +17,7 @@ import type {
   StoreStats, Collection, ExecuteOptions, SequenceResult, ValueOrPromise,
   Dialect, ChangeRecord, LiveOptions, LiveQuery, JobsApi, SyncStore,
   EntityScope, RelationEntry, RelationTable, EntityCursorOptions, QueryCursor,
-  LoadContinuation, PageOptions, Page, ChangesReader,
+  LoadContinuation, PageOptions, Page, ChangesReader, Replication, TransactionStore,
 } from '@jarenjs/db';
 
 /** The self-referential constraint an interface can satisfy: generated
@@ -164,7 +164,7 @@ export interface TypedStore<E extends MetaMap<E>> {
   /** The relation tables of every entity, keyed by entity name. */
   readonly relations?: Readonly<Record<keyof E & string, RelationTable>>;
   saveChanges?(): Promise<SaveReport>;
-  transaction<R>(fn: (store: Store) => R | Promise<R>): Promise<Awaited<R>>;
+  transaction<R>(fn: (store: TransactionStore) => R | Promise<R>): Promise<Awaited<R>>;
   observe(fn: (record: ChangeRecord) => void): () => void;
   /** Unbounded, and unsafe for a reconnecting consumer: `changes.page()`
    * is the supported path (LIVE-FORMAT §5). */
@@ -174,6 +174,7 @@ export interface TypedStore<E extends MetaMap<E>> {
   live?(document: unknown, options?: LiveOptions): Promise<LiveQuery>;
   close(options?: { graceMs?: number }): Promise<void>;
   readonly jobs?: JobsApi;
+  readonly replication?: Replication;
   readonly sync?: SyncStore;
 }