@jarenjs/db 0.56.0 → 0.67.0

package/src/capture.js

@@ -32,13 +32,23 @@
 import { createJSONPatch } from '@jarenjs/json/patch';
 import { encodeJSONPointerSegment, decodeJSONPointerSegment } from '@jarenjs/json/pointer';
 
-import { DbCompileError, DbRuntimeError } from './errors.js';
+import { DbCompileError, DbRuntimeError, wrapDriverError } from './errors.js';
 import { chain, attempt } from './driver.js';
+import { createCursor, drainPage, utf8Length, PAGE_LIMIT_DEFAULT } from './cursor.js';
 
 /** The persisted change log (LIVE-FORMAT §5). */
 export const CHANGES_TABLE = '_jaren_changes';
+/**
+ * The log's durable state: one row holding the highest sequence ever
+ * allocated for this file, so the high watermark survives a log that
+ * retention or maintenance has emptied (LIVE-FORMAT §5).
+ */
+export const CHANGES_STATE_TABLE = '_jaren_changes_state';
 export const DEFAULT_RETENTION = 1000;
 
+/** A driver failure met by the change reader, classified — never raw. */
+const captureFailure = (error) => wrapDriverError(error, { docPath: '/capture' });
+
 //#region the binary changeset parser
 
 const OP_INSERT = 18;
@@ -316,11 +326,19 @@ export function translateOperations(connection, shapes, operations) {
 /**
  * @param {{ connection: any, shapes: Map<string, TableShape>,
  *   mode: 'session' | 'journal',
- *   log: boolean, retention: number }} options
+ *   log: boolean, retention: number,
+ *   now?: () => number,
+ *   bracket?: (fn: () => any) => any,
+ *   beforeCommit?: (patch: any[], context: any) => any }} options - `now` is the clock a
+ *   delivery is stamped with (the store's runtime record); the platform's
+ *   when absent. `bracket` runs the first-open DDL (the store's immediate
+ *   transaction on a writable store); bare when absent
  * @returns {any}
  */
 export function createCaptureEngine(options) {
   const { connection, shapes, mode } = options;
+  const clock = options.now ?? Date.now;
+  const bracket = options.bracket ?? ((/** @type {() => any} */ fn) => fn());
   const dialect = connection.dialect;
   const q = dialect.quoteIdentifier;
   if (options.log && !(Number.isInteger(options.retention) && options.retention >= 1)) {
@@ -331,8 +349,12 @@ export function createCaptureEngine(options) {
 
   /** @type {Set<Function>} */
   const observers = new Set();
+  // the last sequence THIS process delivered: the per-process sequence
+  // when no log is kept, and otherwise the file's allocation read back
+  // from the durable row — never an answer to a watermark question
   let seq = 0;
   let depth = 0;
+  let context = null;
   /** @type {any} */
   let session = null;
   /** @type {any[]} */
@@ -350,33 +372,70 @@ export function createCaptureEngine(options) {
         { name: 'patch', type: dialect.typeFor('string', 'key') },
       ],
     }),
-    // the sequence is allocated by the STATEMENT, inside the write's
-    // own transaction: a counter seeded once at open collided with
-    // another store's writes to the same file and rolled the user's
-    // write back with a raw UNIQUE failure
+    // the durable high watermark: a singleton row the file keeps, so an
+    // emptied log still knows the highest sequence it ever allocated
+    createState: dialect.ddl.createPlainTable({
+      table: CHANGES_STATE_TABLE,
+      columns: [
+        { name: 'id', type: dialect.typeFor('integer', 'key'), primaryKey: true },
+        { name: 'high', type: dialect.typeFor('integer', 'key') },
+      ],
+    }),
+    // idempotent: an existing file is upgraded ONCE from its surviving
+    // maximum (a file that never held a row seeds 0); a second open, or
+    // a second store on the same file, changes nothing
+    hasState: `SELECT 1 AS ${q('present')} FROM ${q(CHANGES_STATE_TABLE)} WHERE ${q('id')} = 1`,
+    seedState: `INSERT INTO ${q(CHANGES_STATE_TABLE)} (${q('id')}, ${q('high')}) `
+      + `SELECT 1, (SELECT COALESCE(MAX(${q('seq')}), 0) FROM ${q(CHANGES_TABLE)}) `
+      + `WHERE NOT EXISTS (SELECT 1 FROM ${q(CHANGES_STATE_TABLE)} WHERE ${q('id')} = 1)`,
+    // the sequence is allocated by ONE statement against the durable
+    // row, inside the write's own transaction: the next value is one
+    // past the higher of the durable watermark and whatever survives in
+    // the log (so a state row that went missing can never lower it), and
+    // a rolled-back write takes its allocation back with the row. A
+    // process counter seeded once at open collided with another store's
+    // writes to the same file; the durable row is the file's fact, so two
+    // stores allocate distinct, increasing sequences.
+    allocate: `INSERT INTO ${q(CHANGES_STATE_TABLE)} (${q('id')}, ${q('high')}) `
+      + `VALUES (1, (SELECT COALESCE(MAX(${q('seq')}), 0) + 1 FROM ${q(CHANGES_TABLE)})) `
+      + `ON CONFLICT (${q('id')}) DO UPDATE SET ${q('high')} = CASE `
+      + `WHEN ${q(CHANGES_STATE_TABLE)}.${q('high')} + 1 > ${dialect.excludedRef(q('high'))} `
+      + `THEN ${q(CHANGES_STATE_TABLE)}.${q('high')} + 1 ELSE ${dialect.excludedRef(q('high'))} END `
+      + `RETURNING ${q('high')} AS ${q('seq')}`,
     insert: `INSERT INTO ${q(CHANGES_TABLE)} `
       + `(${['seq', 'at', 'source', 'patch'].map(q).join(', ')}) `
-      + `VALUES ((SELECT COALESCE(MAX(${q('seq')}), 0) + 1 FROM ${q(CHANGES_TABLE)}), `
-      + `${[1, 2, 3].map((i) => dialect.parameterRef(i, 'v')).join(', ')}) `
-      + `RETURNING ${q('seq')} AS ${q('seq')}`,
+      + `VALUES (${[1, 2, 3, 4].map((i) => dialect.parameterRef(i, 'v')).join(', ')})`,
     prune: `DELETE FROM ${q(CHANGES_TABLE)} WHERE ${q('seq')} <= ${dialect.parameterRef(1, 'v')}`,
-    highest: `SELECT MAX(${q('seq')}) AS ${q('n')} FROM ${q(CHANGES_TABLE)}`,
     read: `SELECT ${['seq', 'at', 'source', 'patch'].map(q).join(', ')} `
       + `FROM ${q(CHANGES_TABLE)} WHERE ${q('seq')} > ${dialect.parameterRef(1, 'v')} `
       + `ORDER BY ${q('seq')}`,
+    // the bounded read: a page of records after a cursor, one more than
+    // the page so `hasMore` is a fact and not a guess
+    readPage: `SELECT ${['seq', 'at', 'source', 'patch'].map(q).join(', ')} `
+      + `FROM ${q(CHANGES_TABLE)} WHERE ${q('seq')} > ${dialect.parameterRef(1, 'v')} `
+      + `ORDER BY ${q('seq')} LIMIT ${dialect.parameterRef(2, 'v')}`,
+    // the two watermarks are the FILE's facts: the earliest surviving
+    // row, and the durable high (falling back to the surviving maximum
+    // only for a file whose state row is absent — never to this process)
+    bounds: `SELECT (SELECT MIN(${q('seq')}) FROM ${q(CHANGES_TABLE)}) AS ${q('lo')}, `
+      + `COALESCE((SELECT ${q('high')} FROM ${q(CHANGES_STATE_TABLE)} WHERE ${q('id')} = 1), `
+      + `(SELECT MAX(${q('seq')}) FROM ${q(CHANGES_TABLE)}), 0) AS ${q('hi')}`,
   } : null;
 
   const ready = logStatements === null
     ? null
-    : chain(attempt(() => connection.exec(logStatements.create), (error) => new DbCompileError('JD0002',
+    // the seed is written only when the row is absent: a read-only open
+    // of an already-upgraded file reads the row and writes nothing
+    : chain(attempt(() => bracket(() => chain(connection.exec(logStatements.create), () =>
+      chain(connection.exec(logStatements.createState), () =>
+        chain(connection.prepare(logStatements.hasState), (probe) => chain(probe.get([]), (row) =>
+          (row === undefined || row === null
+            ? chain(connection.prepare(logStatements.seedState), (statement) => statement.run([]))
+            : null)))))),
+    (error) => new DbCompileError('JD0002',
       `the change log table could not be created (${error?.message ?? String(error)}) — `
       + 'a read-only store creates nothing; open it read-write once, or without capture.log',
-      '/capture', error)), () =>
-      chain(connection.prepare(logStatements.highest), (statement) =>
-        chain(statement.get([]), (row) => {
-          seq = Number(row?.n ?? 0) || 0;
-          return null;
-        })));
+      '/capture', error)), () => null);
 
   /** A patch value must be JSON: a `{ ...doc, m: undefined }` write
    * stores the member ABSENT, and the journal must record the JSON
@@ -457,11 +516,13 @@ export function createCaptureEngine(options) {
 
   const persist = (patch, at) => {
     if (logStatements === null || patch.length === 0) return null;
-    return chain(connection.prepare(logStatements.insert), (insert) =>
-      chain(insert.get([at, mode, JSON.stringify(patch)]), (row) => {
+    return chain(connection.prepare(logStatements.allocate), (allocate) =>
+      chain(allocate.get([]), (row) => {
         seq = Number(row.seq);
-        return chain(connection.prepare(logStatements.prune), (prune) =>
-          chain(prune.run([seq - options.retention]), () => null));
+        return chain(connection.prepare(logStatements.insert), (insert) =>
+          chain(insert.run([seq, at, mode, JSON.stringify(patch)]), () =>
+            chain(connection.prepare(logStatements.prune), (prune) =>
+              chain(prune.run([seq - options.retention]), () => null))));
       }));
   };
 
@@ -502,14 +563,18 @@ export function createCaptureEngine(options) {
    * Run `fn` inside the capture scope: the OUTERMOST scope opens a
    * session (or journal buffer) plus a transaction, translates and
    * persists inside it, and delivers to observers after commit.
+   *
+   * The transaction's scope arguments are FORWARDED to `fn`: the store
+   * pins a transaction view to the exact scope its callback runs in,
+   * and with capture that is the scope this wrap opens, not the one
+   * around it. Callers that need no scope simply ignore the arguments.
    */
   const wrap = (fn) => {
     if (depth > 0) return fn();
     depth = 1;
-    if (mode === 'session') session = connection.session();
-    else journal = [];
     const cleanupFailure = () => {
       depth = 0;
+      context = null;
       if (session !== null) {
         session.close();
         session = null;
@@ -518,11 +583,13 @@ export function createCaptureEngine(options) {
     };
     let outcome;
     try {
-      outcome = connection.transaction(() =>
-        chain(fn(), (result) =>
-          chain(collect(), (patch) => {
+      if (mode === 'session') session = connection.session();
+      else journal = [];
+      outcome = connection.transaction((...scopeArgs) =>
+        chain(fn(...scopeArgs), (result) =>
+          chain(collect(), (patch) => chain(options.beforeCommit?.(patch, context), () => {
             if (patch.length === 0) return { result, delivery: null };
-            const at = Date.now();
+            const at = clock();
             return chain(persist(patch, at), () => ({
               result,
               delivery: {
@@ -533,7 +600,7 @@ export function createCaptureEngine(options) {
                 patch,
               },
             }));
-          })));
+          }))));
     }
     catch (error) {
       cleanupFailure();
@@ -541,6 +608,7 @@ export function createCaptureEngine(options) {
     }
     const finish = (bundle) => {
       depth = 0;
+      context = null;
       if (bundle.delivery !== null) pendingDeliveries.push(bundle.delivery);
       deliver();
       return bundle.result;
@@ -582,41 +650,156 @@ export function createCaptureEngine(options) {
     return outcome;
   };
 
+  /**
+   * The journal's checkpoint pair for NAMED partial rollback
+   * (MODEL-FORMAT §5.2). A named `SAVEPOINT` takes a mark; a
+   * `ROLLBACK TO` truncates the buffer to it, so records the engine
+   * undid vanish from the commit's patch exactly as a session drops
+   * rows undone by `ROLLBACK TO`. Session mode needs neither half —
+   * SQLite's own changeset already excludes the undone rows — and
+   * answers `null` so the caller stores nothing.
+   */
+  const mark = () => (mode === 'journal' ? journal.length : null);
+  /** @param {number | null} at - a value {@link mark} answered */
+  const truncate = (at) => {
+    if (mode === 'journal' && at !== null && journal.length > at) journal.length = at;
+  };
+
   return {
     mode,
     ready,
     wrap,
     nest,
+    mark,
+    truncate,
     record,
+    // Metadata belongs to this capture transaction and is cleared on every
+    // settlement, including a failure while collecting or persisting changes.
+    setContext(value) {
+      if (depth === 0) throw new TypeError('capture context needs an active transaction');
+      context = value;
+    },
     observe(fn) {
       if (typeof fn !== 'function')
         throw new TypeError('observe needs a function');
       observers.add(fn);
       return () => observers.delete(fn);
     },
+    /**
+     * Read the persisted log forward from `after` — EVERY surviving
+     * record, in one array, with no bound and no watermark: a
+     * reconnecting consumer whose cursor fell below the retention floor
+     * receives the surviving suffix and cannot tell it from the whole.
+     * Kept for its callers; `changes.page()` is the bounded reader that
+     * reports the gap instead (LIVE-FORMAT §5).
+     * @param {number} after - the last seq seen
+     */
     changesSince(after) {
-      if (logStatements === null) {
-        throw new DbRuntimeError('JD2051',
-          'the change log is not enabled — open the store with capture.log');
-      }
-      if (typeof after !== 'number' || !Number.isFinite(after)) {
-        throw new TypeError(`changesSince(after) takes the last seq seen as a number, got ${
-          after === undefined ? 'undefined' : JSON.stringify(after)}`);
-      }
+      requireLog('changesSince');
+      requireCursor(after, 'changesSince');
       return chain(connection.prepare(logStatements.read), (statement) =>
-        chain(statement.all([after]), (rows) => rows.map((row) => {
-          const patch = JSON.parse(row.patch);
-          // the same record shape observers receive: `collections` too
-          return {
-            seq: Number(row.seq),
-            at: Number(row.at),
-            source: String(row.source),
-            collections: collectionsOf(patch),
-            patch,
-          };
-        })));
+        chain(attempt(() => statement.all([after]), captureFailure), (rows) => rows.map(recordOf)));
     },
+    /** Whether the persisted log exists — what decides whether the
+     * bounded reader is offered at all. */
+    logged: logStatements !== null,
+    bounds: () => readBounds(),
+    page: (options) => readPage(options),
   };
+
+  /** @param {string} member */
+  function requireLog(member) {
+    if (logStatements !== null) return;
+    throw new DbRuntimeError('JD2051',
+      `${member}: the change log is not enabled — open the store with capture.log`);
+  }
+  /** @param {any} after @param {string} member */
+  function requireCursor(after, member) {
+    if (typeof after === 'number' && Number.isFinite(after)) return;
+    throw new TypeError(`${member} takes the last seq seen as a number (after), got ${
+      after === undefined ? 'undefined' : JSON.stringify(after)}`);
+  }
+  /** The record shape observers receive, `collections` included. */
+  function recordOf(row) {
+    const patch = JSON.parse(row.patch);
+    return {
+      seq: Number(row.seq),
+      at: Number(row.at),
+      source: String(row.source),
+      collections: collectionsOf(patch),
+      patch,
+    };
+  }
+  /**
+   * The log's two watermarks: the earliest surviving sequence (`null`
+   * when nothing survives) and the highest ever allocated, read from the
+   * durable state row. Both are the FILE's facts, never this process's:
+   * an emptied log, a reopened file and a second store over the same
+   * file all answer the same high watermark, so a stale consumer meets a
+   * reset rather than a plausible empty history.
+   */
+  function readBounds() {
+    requireLog('changes.bounds');
+    return chain(connection.prepare(logStatements.bounds), (statement) =>
+      chain(attempt(() => statement.get([]), captureFailure), (row) => ({
+        earliestAvailable: row?.lo === null || row?.lo === undefined ? null : Number(row.lo),
+        highWatermark: Number(row?.hi ?? 0),
+      })));
+  }
+  /**
+   * One bounded page of the log after `after`: never more than `limit`
+   * records or `maxBytes` serialised patch bytes (the one page drain,
+   * cursor.js — a record larger than `maxBytes` is its `JD2074`, the
+   * cursor not advanced), `hasMore` by one peek, `signal` honoured at a
+   * record boundary. The watermarks are read AFTER the rows: a floor
+   * that rose during the read can only make the reset verdict stricter,
+   * never let a pruned gap pass as a continuation. A cursor below the
+   * floor — the record after `after` no longer survives — is a TOTAL
+   * refusal: `resetRequired: true`, no items, no `next`, because a
+   * partial suffix beside a reset flag invites a consumer to use both.
+   * @param {{ after: number, limit?: number, maxBytes?: number | null,
+   *   signal?: AbortSignal }} options
+   */
+  function readPage(options) {
+    requireLog('changes.page');
+    const after = options?.after;
+    requireCursor(after, 'changes.page');
+    const limit = options?.limit ?? PAGE_LIMIT_DEFAULT;
+    if (!Number.isSafeInteger(limit) || limit < 1)
+      throw new TypeError('changes.page: limit must be a positive integer');
+    const declaredBytes = options?.maxBytes;
+    const maxBytes = declaredBytes === undefined || declaredBytes === null || declaredBytes === Infinity
+      ? null : declaredBytes;
+    if (maxBytes !== null && !(Number.isSafeInteger(maxBytes) && maxBytes >= 1))
+      throw new TypeError('changes.page: maxBytes must be a positive integer, or Infinity for no byte bound');
+    const deadline = options?.deadline;
+    if (deadline !== undefined && (typeof deadline !== 'number' || !Number.isFinite(deadline)))
+      throw new TypeError('changes.page: deadline is an epoch-millisecond number');
+    // the deadline is read against the store's clock at every record
+    // boundary, as on every other page (JD2075)
+    const cursor = createCursor({ streaming: 'row', barrier: null, signal: options?.signal, deadline, now: clock,
+      open: () => chain(connection.prepare(logStatements.readPage),
+        (statement) => statement.iterate([after, limit + 1])),
+      wrap: captureFailure,
+      items: (row) => [{ record: recordOf(row), bytes: utf8Length(String(row.patch)) }] });
+    return drainPage(cursor, {
+      limit, maxBytes, after,
+      sizeOf: (item) => item.bytes,
+      continuationOf: (item) => item.record.seq,
+    }).then((page) => chain(readBounds(), (bounds) => {
+      const first = bounds.earliestAvailable ?? bounds.highWatermark + 1;
+      if (after + 1 < first) {
+        return { items: [], ...bounds, hasMore: false, resetRequired: true };
+      }
+      return {
+        items: page.items.map((item) => item.record),
+        next: page.continuation ?? after,
+        ...bounds,
+        hasMore: page.hasMore,
+        resetRequired: false,
+      };
+    }));
+  }
 }
 
 //#endregion