@rocicorp/zero 0.0.202410041722 → 0.1.2024100802

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "@rocicorp/zero",
-  "version": "0.0.202410041722+d0f469",
+  "version": "0.1.2024100802+9b60ed",
   "scripts": {
     "prepack": "rm -rf deps && mkdir -p deps/sqlite3 && cp -r ../../deps/sqlite3/* deps/sqlite3/",
     "preinstall": "node tool/install-sqlite3.js",
     "build": "rm -rf out && npm run build-server && npm run build-client",
     "build-client": "tsc -p tsconfig.client.json && tsc-alias -p tsconfig.client.json && node tool/build.js",
-    "build-server": "tsc -p tsconfig.server.json && tsc-alias -p tsconfig.server.json",
-    "check-types": "tsc --noEmit",
-    "check-types:watch": "tsc --noEmit --watch",
+    "build-server": "tsc -p tsconfig.server.json && tsc-alias -p tsconfig.server.json && chmod +x out/zero/src/cli.js",
+    "check-types": "tsc",
+    "check-types:watch": "tsc --watch",
     "format": "prettier --write .",
     "check-format": "prettier --check .",
     "lint": "eslint --ext .ts,.tsx,.js,.jsx src/"
@@ -75,7 +75,7 @@
     "tool/install-sqlite3.js"
   ],
   "eslintConfig": {
-    "extends": "@rocicorp/eslint-config"
+    "extends": "../../eslint-config.json"
   },
   "prettier": "@rocicorp/prettier-config"
 }

package/out/zero-cache/src/services/replicator/checkpointer.d.ts

@@ -1,79 +0,0 @@
-import { LogContext } from '@rocicorp/logger';
-import { Notifier } from './notifier.js';
-/**
- * A `Checkpointer` is consulted by the Replicator after each commit and given
- * the opportunity to perform a blocking WAL checkpoint before the Replicator
- * continues processing the replication stream.
- */
-export interface Checkpointer {
-    maybeCheckpoint(numCommittedChanges: number, notifier: Notifier): Promise<void>;
-    stop(): void;
-}
-/**
- * The `NULL_CHECKPOINTER` is suitable for an environment in which checkpoints are
- * handled by an external entity, such as a `litestream replicate` process.
- */
-export declare const NULL_CHECKPOINTER: Checkpointer;
-export type CheckpointerConfig = {
-    /**
-     * The number of outstanding frames or changes at which a blocking
-     * checkpoint is triggered.
-     *
-     * Defaults to 200.
-     */
-    threshold?: number;
-    /**
-     * The base timeout to block on an active checkpoint. This timeout is increased
-     * in proportion to the size of WAL divided by the `threshold`. For example, if
-     * the size of the WAL is twice the `threshold`, the timeout will be doubled.
-     *
-     * This dynamic timeout algorithm reduces the amount of system-wide pausing in
-     * the common case, even in the presence of occasional, long-held locks due to
-     * events like a long hydration. If checkpoint is not able to start within the
-     * timeout, the checkpointer gives up and allows the system to progress. As the
-     * size of the WAL grows, the checkpointer will wait for longer timeouts to ensure
-     * that the checkpointing eventually succeeds.
-     *
-     * Defaults to 200 milliseconds.
-     */
-    baseCheckpointTimeoutMs?: number;
-    /**
-     * A regular interval at which passive checkpoints are attempted. This allows an
-     * idle task (i.e. no sync connections) to clean up outside of the
-     * replication path.
-     *
-     * Defaults to one minute.
-     */
-    passiveCheckpointPeriodMs?: number;
-};
-/**
- * The `WALCheckpointer` executes checkpoints when the number of WAL log entries
- * exceeds a configurable threshold. It does this by:
- *
- * 1. Broadcasting a `maintenance` ReplicaState message to signal view-syncers to
- *    release their locks.
- * 2. Executing `wal_checkpoint(TRUNCATE)` with a busy_timeout proportional to
- *    the outstanding log size.
- * 3. Broadcasting a `version-ready` ReplicaState message to signal view-syncers to
- *    reestablish their locks.
- *
- * Points of interest:
- *
- * * The checkpointer waits for the `maintenance` broadcast to be sent to over
- *   IPC channels (in-process blocking), but does not request or wait for ACKs from
- *   the view syncers. This is because the `wal_checkpoint(TRUNCATE)` command itself
- *   will block until all read locks are released, up to the configured `busy_timeout`.
- *
- * * After performing the checkpoint, however, the checkpointer _does_ request ACKs
- *   from view-syncers to confirm that they have reestablished their snapshots. There
- *   is otherwise no way to know when it is safe to continue replication. This is
- *   expected to be very fast, but to avoid pathological cases the wait is capped at
- *   100ms.
- */
-export declare class WALCheckpointer implements Checkpointer {
-    #private;
-    constructor(lc: LogContext, replicaDbFile: string, cfg?: CheckpointerConfig);
-    stop(): void;
-    maybeCheckpoint(changes: number, notifier: Notifier): Promise<void>;
-}
-//# sourceMappingURL=checkpointer.d.ts.map

package/out/zero-cache/src/services/replicator/checkpointer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"checkpointer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/checkpointer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAM5C,OAAO,EAAC,QAAQ,EAAC,MAAM,eAAe,CAAC;AAEvC;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,eAAe,CACb,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,GACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,IAAI,IAAI,IAAI,CAAC;CACd;AAED;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,YAGtB,CAAC;AAEX,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;;;;;;OAaG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;OAMG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACpC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBAAa,eAAgB,YAAW,YAAY;;gBAWhD,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,GAAG,GAAE,kBAAuB;IA4B9B,IAAI;IAKE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;CAwE1D"}

package/out/zero-cache/src/services/replicator/checkpointer.js

@@ -1,124 +0,0 @@
-import { LogContext } from '@rocicorp/logger';
-import { assert } from '../../../../shared/src/asserts.js';
-import { randInt } from '../../../../shared/src/rand.js';
-import { promiseVoid } from '../../../../shared/src/resolved-promises.js';
-import { orTimeout } from '../../../../zero-cache/src/types/timeout.js';
-import { Database } from '../../../../zqlite/src/db.js';
-import { Notifier } from './notifier.js';
-/**
- * The `NULL_CHECKPOINTER` is suitable for an environment in which checkpoints are
- * handled by an external entity, such as a `litestream replicate` process.
- */
-export const NULL_CHECKPOINTER = {
-    maybeCheckpoint: () => promiseVoid,
-    stop: () => { },
-};
-/**
- * The `WALCheckpointer` executes checkpoints when the number of WAL log entries
- * exceeds a configurable threshold. It does this by:
- *
- * 1. Broadcasting a `maintenance` ReplicaState message to signal view-syncers to
- *    release their locks.
- * 2. Executing `wal_checkpoint(TRUNCATE)` with a busy_timeout proportional to
- *    the outstanding log size.
- * 3. Broadcasting a `version-ready` ReplicaState message to signal view-syncers to
- *    reestablish their locks.
- *
- * Points of interest:
- *
- * * The checkpointer waits for the `maintenance` broadcast to be sent to over
- *   IPC channels (in-process blocking), but does not request or wait for ACKs from
- *   the view syncers. This is because the `wal_checkpoint(TRUNCATE)` command itself
- *   will block until all read locks are released, up to the configured `busy_timeout`.
- *
- * * After performing the checkpoint, however, the checkpointer _does_ request ACKs
- *   from view-syncers to confirm that they have reestablished their snapshots. There
- *   is otherwise no way to know when it is safe to continue replication. This is
- *   expected to be very fast, but to avoid pathological cases the wait is capped at
- *   100ms.
- */
-export class WALCheckpointer {
-    #lc;
-    #db;
-    #threshold;
-    #baseCheckpointTimeoutMs;
-    #passiveCheckpointTimer;
-    #logSize = 0;
-    #outstandingChanges = 0;
-    constructor(lc, replicaDbFile, cfg = {}) {
-        const { threshold = 200, baseCheckpointTimeoutMs = 10, passiveCheckpointPeriodMs = 60_000, } = cfg;
-        assert(threshold > 0 &&
-            baseCheckpointTimeoutMs > 0 &&
-            passiveCheckpointPeriodMs > 0, `Invalid config ${JSON.stringify(cfg)}`);
-        const db = new Database(lc, replicaDbFile);
-        db.pragma('journal_mode = WAL');
-        this.#lc = lc.withContext('component', 'wal-checkpointer');
-        this.#db = db;
-        this.#threshold = threshold;
-        this.#baseCheckpointTimeoutMs = baseCheckpointTimeoutMs;
-        this.#passiveCheckpointTimer = setInterval(() => this.#checkpoint('PASSIVE'), passiveCheckpointPeriodMs);
-    }
-    stop() {
-        clearTimeout(this.#passiveCheckpointTimer);
-        this.#lc.info?.('stopped');
-    }
-    async maybeCheckpoint(changes, notifier) {
-        this.#outstandingChanges += changes;
-        // Simplification: changes and frames equivalently w.r.t. the threshold.
-        if (Math.max(this.#outstandingChanges, this.#logSize) >= this.#threshold) {
-            // If no read locks are held, a PASSIVE checkpoint may suffice.
-            // Regardless of success, #logSize gets updated to determine if a
-            // blocking checkpoint is warranted.
-            this.#checkpoint('PASSIVE');
-        }
-        if (this.#logSize >= this.#threshold) {
-            const logSize = this.#logSize;
-            const t0 = Date.now();
-            await this.#enterMaintenanceMode(notifier);
-            const t1 = Date.now();
-            // The timeout is proportional to the size of the log compared to the threshold.
-            const timeout = (this.#logSize / this.#threshold) * this.#baseCheckpointTimeoutMs;
-            const result = this.#checkpoint('TRUNCATE', timeout);
-            const t2 = Date.now();
-            await this.#exitMaintenanceMode(notifier);
-            const t3 = Date.now();
-            this.#lc.info?.(`WAL(busy=${timeout}ms): pre=${t1 - t0}ms checkpoint=${t2 - t1}ms post=${t3 - t2}ms logSize=${logSize} result`, result);
-        }
-    }
-    async #enterMaintenanceMode(notifier) {
-        await Promise.all(notifier.notifySubscribers({ state: 'maintenance' }));
-    }
-    async #exitMaintenanceMode(notifier) {
-        // Request an ACK when exiting maintenance mode to maximize the chance of
-        // view syncers re-establishing snapshots at the same version. This is expected
-        // to be very fast (single-digit milliseconds).
-        //
-        // However, this is also the chance for view syncers to invoke an occasional
-        // `PRAGMA optimize` call, for which latency may be variable. To avoid holding
-        // up the replication stream indefinitely, we cap the wait at 100ms.
-        const result = await orTimeout(Promise.all(notifier.notifySubscribers({
-            state: 'version-ready',
-            ack: randomACK(),
-        })), 100);
-        if (result === 'timed-out') {
-            this.#lc.info?.('timed out waiting for view-syncer resumption');
-        }
-    }
-    #checkpoint(mode, timeoutMs) {
-        if (timeoutMs) {
-            this.#db.pragma(`busy_timeout = ${timeoutMs}`);
-        }
-        const result = checkpoint(this.#db, mode);
-        this.#logSize = result.log;
-        this.#outstandingChanges = 0;
-        return result;
-    }
-}
-function checkpoint(db, mode) {
-    const result = db.pragma(`wal_checkpoint(${mode})`);
-    return result[0];
-}
-function randomACK() {
-    return randInt(Number.MIN_SAFE_INTEGER, Number.MAX_SAFE_INTEGER);
-}
-//# sourceMappingURL=checkpointer.js.map

package/out/zero-cache/src/services/replicator/checkpointer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"checkpointer.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/checkpointer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAC7C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAC,WAAW,EAAC,MAAM,iCAAiC,CAAC;AAC5D,OAAO,EAAC,SAAS,EAAC,MAAM,iCAAiC,CAAC;AAC1D,OAAO,EAAC,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,eAAe,CAAC;AAgBvC;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAiB;IAC7C,eAAe,EAAE,GAAG,EAAE,CAAC,WAAW;IAClC,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC;CACN,CAAC;AAqCX;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,OAAO,eAAe;IACjB,GAAG,CAAa;IAChB,GAAG,CAAW;IACd,UAAU,CAAS;IACnB,wBAAwB,CAAS;IACjC,uBAAuB,CAAgC;IAEhE,QAAQ,GAAG,CAAC,CAAC;IACb,mBAAmB,GAAG,CAAC,CAAC;IAExB,YACE,EAAc,EACd,aAAqB,EACrB,MAA0B,EAAE;QAE5B,MAAM,EACJ,SAAS,GAAG,GAAG,EACf,uBAAuB,GAAG,EAAE,EAC5B,yBAAyB,GAAG,MAAM,GACnC,GAAG,GAAG,CAAC;QAER,MAAM,CACJ,SAAS,GAAG,CAAC;YACX,uBAAuB,GAAG,CAAC;YAC3B,yBAAyB,GAAG,CAAC,EAC/B,kBAAkB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CACxC,CAAC;QAEF,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;QAC3C,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAEhC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QAC3D,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,wBAAwB,GAAG,uBAAuB,CAAC;QACxD,IAAI,CAAC,uBAAuB,GAAG,WAAW,CACxC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,EACjC,yBAAyB,CAC1B,CAAC;IACJ,CAAC;IAED,IAAI;QACF,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,EAAE,QAAkB;QACvD,IAAI,CAAC,mBAAmB,IAAI,OAAO,CAAC;QAEpC,wEAAwE;QACxE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACzE,+DAA+D;YAC/D,iEAAiE;YACjE,oCAAoC;YACpC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAC9B,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC;YAE9B,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YAC3C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEtB,gFAAgF;YAChF,MAAM,OAAO,GACX,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC;YACpE,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAErD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEtB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,YAAY,OAAO,YAAY,EAAE,GAAG,EAAE,iBACpC,EAAE,GAAG,EACP,WAAW,EAAE,GAAG,EAAE,cAAc,OAAO,SAAS,EAChD,MAAM,CACP,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,QAAkB;QAC5C,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAC,KAAK,EAAE,aAAa,EAAC,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,QAAkB;QAC3C,yEAAyE;QACzE,+EAA+E;QAC/E,+CAA+C;QAC/C,EAAE;QACF,4EAA4E;QAC5E,8EAA8E;QAC9E,oEAAoE;QACpE,MAAM,MAAM,GAAG,MAAM,SAAS,CAC5B,OAAO,CAAC,GAAG,CACT,QAAQ,CAAC,iBAAiB,CAAC;YACzB,KAAK,EAAE,eAAe;YACtB,GAAG,EAAE,SAAS,EAAE;SACjB,CAAC,CACH,EACD,GAAG,CACJ,CAAC;QACF,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,WAAW,CAAC,IAAoB,EAAE,SAAkB;QAClD,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,kBAAkB,SAAS,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE1C,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAUD,SAAS,UAAU,CAAC,EAAY,EAAE,IAAoB;IACpD,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,IAAI,GAAG,CAAuB,CAAC;IAC1E,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;AACnE,CAAC"}