@rocicorp/zero 0.25.0-canary.1 → 0.25.0-canary.10

package/out/zero-cache/src/services/limiter/sliding-window-limiter.js

@@ -1,137 +1,83 @@
 import { assert } from "../../../../shared/src/asserts.js";
-/**
- * The limiter is a sliding window rate limiter that allows a maximum
- * number of mutations per window.
- *
- * The window is divided into two parts: the prev window and the next window.
- * The prev window is the part of the window that has already passed, and the
- * next window is the part of the window that is still in the future.
- *
- * The limiter keeps track of the number of mutations in each window. When the
- * current time moves completely past the next window, the limiter creates new
- * windows (this only happens if the limiter is not called for a long time).
- * When the current time moves completely into the next window, the
- * limiter rotates the windows.
- *
- * Case 1: need new windows
- * |----|----|
- *             |----| (sliding window)
- *
- * Case 2: rotate windows
- * |----|----|
- *      |----| (sliding window)
- *
- * or
- *
- * |----|----|
- *         |----| (sliding window)
- *
- * The leading edge of the sliding window is current time.
- *
- * The limiter increments the count for the prev window if the current time is
- * in the prior window. Otherwise, it increments the count for the next
- * window.
- *
- * The limiter computes the total mutations by: taking the number of mutation in the next window and
- * adding the number of mutation from the prev window, weighted by
- * the fraction of the window that has passed. The total mutations must be less
- * than the maximum number of mutations allowed per window.
- *
- * |----|----|
- *    |----| (sliding window)
- * ^-- the sliding window covers 25% of the prior window. Only 25% of that window's count counts.
- * The entirety of the next window's count counts.
- */
-export class SlidingWindowLimiter {
-    #windowSizeMs;
-    #maxMutations;
-    #priorWindow;
-    #nextWindow;
-    constructor(windowSizeMs, maxMutations) {
-        this.#windowSizeMs = windowSizeMs;
-        this.#maxMutations = maxMutations;
-        [this.#priorWindow, this.#nextWindow] = this.#newWindows();
+class SlidingWindowLimiter {
+  #windowSizeMs;
+  #maxMutations;
+  #priorWindow;
+  #nextWindow;
+  constructor(windowSizeMs, maxMutations) {
+    this.#windowSizeMs = windowSizeMs;
+    this.#maxMutations = maxMutations;
+    [this.#priorWindow, this.#nextWindow] = this.#newWindows();
+  }
+  canDo() {
+    const now = Date.now();
+    if (now - this.#windowSizeMs > this.#nextWindow.start + this.#windowSizeMs) {
+      [this.#priorWindow, this.#nextWindow] = this.#newWindows();
     }
-    canDo() {
-        const now = Date.now();
-        // If the current sliding window is completely past the next window, we need new windows.
-        if (now - this.#windowSizeMs >
-            this.#nextWindow.start + this.#windowSizeMs) {
-            [this.#priorWindow, this.#nextWindow] = this.#newWindows();
-        }
-        // Has the current sliding window moved completely into the next window?
-        // Then rotate the windows.
-        if (now - this.#windowSizeMs >= this.#nextWindow.start) {
-            this.#rotateWindows();
-        }
-        // Now compute the total mutations in the current window.
-        // Weighted by the fraction of the window that has passed.
-        const totalCalls = this.totalCallsForTime(now);
-        const canDo = totalCalls < this.#maxMutations;
-        // We don't increment counts if we throttled the user. This is so the user can
-        // eventually do the action they are trying to do. If we incremented on every
-        // attempt they can end up in a case where their excessive retries lock them out
-        // continuously.
-        if (canDo) {
-            // If "now" in the prior or next window? Increment the count for the correct window.
-            if (now < this.#nextWindow.start) {
-                this.#priorWindow.count++;
-            }
-            else {
-                this.#nextWindow.count++;
-            }
-        }
-        return canDo;
+    if (now - this.#windowSizeMs >= this.#nextWindow.start) {
+      this.#rotateWindows();
     }
-    totalCallsForTime(now) {
-        let fraction;
-        if (now < this.#priorWindow.start + this.#windowSizeMs) {
-            fraction = 1;
-        }
-        else {
-            fraction =
-                (this.#priorWindow.start +
-                    (this.#windowSizeMs - 1) -
-                    (now - this.#windowSizeMs)) /
-                    this.#windowSizeMs;
-        }
-        if (fraction < 0) {
-            fraction = 0;
-        }
-        assert(fraction <= 1, 'The past cannot contribute more than the a full window.');
-        const totalCalls = this.#priorWindow.count * fraction + this.#nextWindow.count;
-        return totalCalls;
+    const totalCalls = this.totalCallsForTime(now);
+    const canDo = totalCalls < this.#maxMutations;
+    if (canDo) {
+      if (now < this.#nextWindow.start) {
+        this.#priorWindow.count++;
+      } else {
+        this.#nextWindow.count++;
+      }
     }
-    get priorWindow() {
-        return this.#priorWindow;
+    return canDo;
+  }
+  totalCallsForTime(now) {
+    let fraction;
+    if (now < this.#priorWindow.start + this.#windowSizeMs) {
+      fraction = 1;
+    } else {
+      fraction = (this.#priorWindow.start + (this.#windowSizeMs - 1) - (now - this.#windowSizeMs)) / this.#windowSizeMs;
     }
-    get nextWindow() {
-        return this.#nextWindow;
-    }
-    /**
-     * If there were no mutations in over `windowSizeMs` we need new windows.
-     */
-    #newWindows() {
-        const now = Date.now();
-        const start = now - (now % this.#windowSizeMs);
-        return [
-            {
-                // clamp to a `windowSizeMs` boundary to start the prior window.
-                start,
-                count: 0,
-            },
-            {
-                start: start + this.#windowSizeMs,
-                count: 0,
-            },
-        ];
-    }
-    #rotateWindows() {
-        this.#priorWindow = this.#nextWindow;
-        this.#nextWindow = {
-            start: this.#priorWindow.start + this.#windowSizeMs,
-            count: 0,
-        };
+    if (fraction < 0) {
+      fraction = 0;
     }
+    assert(
+      fraction <= 1,
+      "The past cannot contribute more than the a full window."
+    );
+    const totalCalls = this.#priorWindow.count * fraction + this.#nextWindow.count;
+    return totalCalls;
+  }
+  get priorWindow() {
+    return this.#priorWindow;
+  }
+  get nextWindow() {
+    return this.#nextWindow;
+  }
+  /**
+   * If there were no mutations in over `windowSizeMs` we need new windows.
+   */
+  #newWindows() {
+    const now = Date.now();
+    const start = now - now % this.#windowSizeMs;
+    return [
+      {
+        // clamp to a `windowSizeMs` boundary to start the prior window.
+        start,
+        count: 0
+      },
+      {
+        start: start + this.#windowSizeMs,
+        count: 0
+      }
+    ];
+  }
+  #rotateWindows() {
+    this.#priorWindow = this.#nextWindow;
+    this.#nextWindow = {
+      start: this.#priorWindow.start + this.#windowSizeMs,
+      count: 0
+    };
+  }
 }
-//# sourceMappingURL=sliding-window-limiter.js.map
+export {
+  SlidingWindowLimiter
+};
+//# sourceMappingURL=sliding-window-limiter.js.map

package/out/zero-cache/src/services/limiter/sliding-window-limiter.js.map

@@ -1 +1 @@
-{"version":3,"file":"sliding-window-limiter.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/limiter/sliding-window-limiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,mCAAmC,CAAC;AAOzD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,OAAO,oBAAoB;IACtB,aAAa,CAAS;IACtB,aAAa,CAAS;IAE/B,YAAY,CAAS;IACrB,WAAW,CAAS;IAEpB,YAAY,YAAoB,EAAE,YAAoB;QACpD,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAElC,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7D,CAAC;IAED,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,yFAAyF;QACzF,IACE,GAAG,GAAG,IAAI,CAAC,aAAa;YACxB,IAAI,CAAC,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,EAC3C,CAAC;YACD,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7D,CAAC;QAED,wEAAwE;QACxE,2BAA2B;QAC3B,IAAI,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YACvD,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,CAAC;QAED,yDAAyD;QACzD,0DAA0D;QAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC;QAE9C,8EAA8E;QAC9E,6EAA6E;QAC7E,gFAAgF;QAChF,gBAAgB;QAChB,IAAI,KAAK,EAAE,CAAC;YACV,oFAAoF;YACpF,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,iBAAiB,CAAC,GAAW;QAC3B,IAAI,QAAgB,CAAC;QACrB,IAAI,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACvD,QAAQ,GAAG,CAAC,CAAC;QACf,CAAC;aAAM,CAAC;YACN,QAAQ;gBACN,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK;oBACtB,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;oBACxB,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC7B,IAAI,CAAC,aAAa,CAAC;QACvB,CAAC;QACD,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,QAAQ,GAAG,CAAC,CAAC;QACf,CAAC;QACD,MAAM,CACJ,QAAQ,IAAI,CAAC,EACb,yDAAyD,CAC1D,CAAC;QACF,MAAM,UAAU,GACd,IAAI,CAAC,YAAY,CAAC,KAAK,GAAG,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;QAC9D,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;QAC/C,OAAO;YACL;gBACE,gEAAgE;gBAChE,KAAK;gBACL,KAAK,EAAE,CAAC;aACT;YACD;gBACE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,aAAa;gBACjC,KAAK,EAAE,CAAC;aACT;SACF,CAAC;IACJ,CAAC;IAED,cAAc;QACZ,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC;QACrC,IAAI,CAAC,WAAW,GAAG;YACjB,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa;YACnD,KAAK,EAAE,CAAC;SACT,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"sliding-window-limiter.js","sources":["../../../../../../zero-cache/src/services/limiter/sliding-window-limiter.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\n\ntype Window = {\n  start: number;\n  count: number;\n};\n\n/**\n * The limiter is a sliding window rate limiter that allows a maximum\n * number of mutations per window.\n *\n * The window is divided into two parts: the prev window and the next window.\n * The prev window is the part of the window that has already passed, and the\n * next window is the part of the window that is still in the future.\n *\n * The limiter keeps track of the number of mutations in each window. When the\n * current time moves completely past the next window, the limiter creates new\n * windows (this only happens if the limiter is not called for a long time).\n * When the current time moves completely into the next window, the\n * limiter rotates the windows.\n *\n * Case 1: need new windows\n * |----|----|\n *             |----| (sliding window)\n *\n * Case 2: rotate windows\n * |----|----|\n *      |----| (sliding window)\n *\n * or\n *\n * |----|----|\n *         |----| (sliding window)\n *\n * The leading edge of the sliding window is current time.\n *\n * The limiter increments the count for the prev window if the current time is\n * in the prior window. Otherwise, it increments the count for the next\n * window.\n *\n * The limiter computes the total mutations by: taking the number of mutation in the next window and\n * adding the number of mutation from the prev window, weighted by\n * the fraction of the window that has passed. The total mutations must be less\n * than the maximum number of mutations allowed per window.\n *\n * |----|----|\n *    |----| (sliding window)\n * ^-- the sliding window covers 25% of the prior window. Only 25% of that window's count counts.\n * The entirety of the next window's count counts.\n */\nexport class SlidingWindowLimiter {\n  readonly #windowSizeMs: number;\n  readonly #maxMutations: number;\n\n  #priorWindow: Window;\n  #nextWindow: Window;\n\n  constructor(windowSizeMs: number, maxMutations: number) {\n    this.#windowSizeMs = windowSizeMs;\n    this.#maxMutations = maxMutations;\n\n    [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n  }\n\n  canDo(): boolean {\n    const now = Date.now();\n\n    // If the current sliding window is completely past the next window, we need new windows.\n    if (\n      now - this.#windowSizeMs >\n      this.#nextWindow.start + this.#windowSizeMs\n    ) {\n      [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n    }\n\n    // Has the current sliding window moved completely into the next window?\n    // Then rotate the windows.\n    if (now - this.#windowSizeMs >= this.#nextWindow.start) {\n      this.#rotateWindows();\n    }\n\n    // Now compute the total mutations in the current window.\n    // Weighted by the fraction of the window that has passed.\n    const totalCalls = this.totalCallsForTime(now);\n    const canDo = totalCalls < this.#maxMutations;\n\n    // We don't increment counts if we throttled the user. This is so the user can\n    // eventually do the action they are trying to do. If we incremented on every\n    // attempt they can end up in a case where their excessive retries lock them out\n    // continuously.\n    if (canDo) {\n      // If \"now\" in the prior or next window? Increment the count for the correct window.\n      if (now < this.#nextWindow.start) {\n        this.#priorWindow.count++;\n      } else {\n        this.#nextWindow.count++;\n      }\n    }\n\n    return canDo;\n  }\n\n  totalCallsForTime(now: number): number {\n    let fraction: number;\n    if (now < this.#priorWindow.start + this.#windowSizeMs) {\n      fraction = 1;\n    } else {\n      fraction =\n        (this.#priorWindow.start +\n          (this.#windowSizeMs - 1) -\n          (now - this.#windowSizeMs)) /\n        this.#windowSizeMs;\n    }\n    if (fraction < 0) {\n      fraction = 0;\n    }\n    assert(\n      fraction <= 1,\n      'The past cannot contribute more than the a full window.',\n    );\n    const totalCalls =\n      this.#priorWindow.count * fraction + this.#nextWindow.count;\n    return totalCalls;\n  }\n\n  get priorWindow(): Window {\n    return this.#priorWindow;\n  }\n\n  get nextWindow(): Window {\n    return this.#nextWindow;\n  }\n\n  /**\n   * If there were no mutations in over `windowSizeMs` we need new windows.\n   */\n  #newWindows() {\n    const now = Date.now();\n    const start = now - (now % this.#windowSizeMs);\n    return [\n      {\n        // clamp to a `windowSizeMs` boundary to start the prior window.\n        start,\n        count: 0,\n      },\n      {\n        start: start + this.#windowSizeMs,\n        count: 0,\n      },\n    ];\n  }\n\n  #rotateWindows() {\n    this.#priorWindow = this.#nextWindow;\n    this.#nextWindow = {\n      start: this.#priorWindow.start + this.#windowSizeMs,\n      count: 0,\n    };\n  }\n}\n"],"names":[],"mappings":";AAkDO,MAAM,qBAAqB;AAAA,EACvB;AAAA,EACA;AAAA,EAET;AAAA,EACA;AAAA,EAEA,YAAY,cAAsB,cAAsB;AACtD,SAAK,gBAAgB;AACrB,SAAK,gBAAgB;AAErB,KAAC,KAAK,cAAc,KAAK,WAAW,IAAI,KAAK,YAAA;AAAA,EAC/C;AAAA,EAEA,QAAiB;AACf,UAAM,MAAM,KAAK,IAAA;AAGjB,QACE,MAAM,KAAK,gBACX,KAAK,YAAY,QAAQ,KAAK,eAC9B;AACA,OAAC,KAAK,cAAc,KAAK,WAAW,IAAI,KAAK,YAAA;AAAA,IAC/C;AAIA,QAAI,MAAM,KAAK,iBAAiB,KAAK,YAAY,OAAO;AACtD,WAAK,eAAA;AAAA,IACP;AAIA,UAAM,aAAa,KAAK,kBAAkB,GAAG;AAC7C,UAAM,QAAQ,aAAa,KAAK;AAMhC,QAAI,OAAO;AAET,UAAI,MAAM,KAAK,YAAY,OAAO;AAChC,aAAK,aAAa;AAAA,MACpB,OAAO;AACL,aAAK,YAAY;AAAA,MACnB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,kBAAkB,KAAqB;AACrC,QAAI;AACJ,QAAI,MAAM,KAAK,aAAa,QAAQ,KAAK,eAAe;AACtD,iBAAW;AAAA,IACb,OAAO;AACL,kBACG,KAAK,aAAa,SAChB,KAAK,gBAAgB,MACrB,MAAM,KAAK,kBACd,KAAK;AAAA,IACT;AACA,QAAI,WAAW,GAAG;AAChB,iBAAW;AAAA,IACb;AACA;AAAA,MACE,YAAY;AAAA,MACZ;AAAA,IAAA;AAEF,UAAM,aACJ,KAAK,aAAa,QAAQ,WAAW,KAAK,YAAY;AACxD,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc;AACZ,UAAM,MAAM,KAAK,IAAA;AACjB,UAAM,QAAQ,MAAO,MAAM,KAAK;AAChC,WAAO;AAAA,MACL;AAAA;AAAA,QAEE;AAAA,QACA,OAAO;AAAA,MAAA;AAAA,MAET;AAAA,QACE,OAAO,QAAQ,KAAK;AAAA,QACpB,OAAO;AAAA,MAAA;AAAA,IACT;AAAA,EAEJ;AAAA,EAEA,iBAAiB;AACf,SAAK,eAAe,KAAK;AACzB,SAAK,cAAc;AAAA,MACjB,OAAO,KAAK,aAAa,QAAQ,KAAK;AAAA,MACtC,OAAO;AAAA,IAAA;AAAA,EAEX;AACF;"}

package/out/zero-cache/src/services/litestream/commands.d.ts

@@ -1,5 +1,5 @@
 import type { LogContext } from '@rocicorp/logger';
-import { ChildProcess } from 'node:child_process';
+import type { ChildProcess } from 'node:child_process';
 import type { ZeroConfig } from '../../config/zero-config.ts';
 /**
  * @returns The time at which the last restore started

package/out/zero-cache/src/services/litestream/commands.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAC,YAAY,EAAQ,MAAM,oBAAoB,CAAC;AAKvD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAW5D;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAwBf;AAgHD,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,CAO1E"}
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAOrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAwBf;AA4JD,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,CAO1E"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -1,173 +1,220 @@
-import { resolver } from '@rocicorp/resolver';
-import { ChildProcess, spawn } from 'node:child_process';
-import { existsSync } from 'node:fs';
+import { resolver } from "@rocicorp/resolver";
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
 import { must } from "../../../../shared/src/must.js";
 import { sleep } from "../../../../shared/src/sleep.js";
+import { Database } from "../../../../zqlite/src/db.js";
 import { assertNormalized } from "../../config/normalize.js";
+import { deleteLiteDB } from "../../db/delete-lite-db.js";
+import { StatementRunner } from "../../db/statements.js";
 import { getShardConfig } from "../../types/shards.js";
 import { ChangeStreamerHttpClient } from "../change-streamer/change-streamer-http.js";
-// Retry for up to 3 minutes (60 times with 3 second delay).
-// Beyond that, let the container runner restart the task.
+import { getSubscriptionState } from "../replicator/schema/replication-state.js";
 const MAX_RETRIES = 60;
-const RETRY_INTERVAL_MS = 3000;
-/**
- * @returns The time at which the last restore started
- *          (i.e. not counting failed attempts).
- */
-export async function restoreReplica(lc, config) {
-    const { changeStreamer } = config;
-    for (let i = 0; i < MAX_RETRIES; i++) {
-        if (i > 0) {
-            lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`);
-            await sleep(RETRY_INTERVAL_MS);
-        }
-        const start = new Date();
-        const restored = await tryRestore(lc, config);
-        if (restored) {
-            return start;
-        }
-        if (changeStreamer.mode === 'dedicated' &&
-            changeStreamer.uri === undefined) {
-            lc.info?.('no litestream backup found');
-            return start;
-        }
+const RETRY_INTERVAL_MS = 3e3;
+async function restoreReplica(lc, config) {
+  const { changeStreamer } = config;
+  for (let i = 0; i < MAX_RETRIES; i++) {
+    if (i > 0) {
+      lc.info?.(
+        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`
+      );
+      await sleep(RETRY_INTERVAL_MS);
+    }
+    const start = /* @__PURE__ */ new Date();
+    const restored = await tryRestore(lc, config);
+    if (restored) {
+      return start;
     }
-    throw new Error(`max attempts exceeded restoring replica`);
+    if (changeStreamer.mode === "dedicated" && changeStreamer.uri === void 0) {
+      lc.info?.("no litestream backup found");
+      return start;
+    }
+  }
+  throw new Error(`max attempts exceeded restoring replica`);
 }
 function getLitestream(config, logLevelOverride, backupURLOverride) {
-    const { executable, backupURL, logLevel, configPath, port = config.port + 2, checkpointThresholdMB, incrementalBackupIntervalMinutes, snapshotBackupIntervalHours, multipartConcurrency, multipartSize, } = config.litestream;
-    // Set the snapshot interval to something smaller than x hours so that
-    // the hourly check triggers on the hour, rather than the hour after.
-    const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;
-    const minCheckpointPageCount = checkpointThresholdMB * 250; // SQLite page size is 4k
-    const maxCheckpointPageCount = minCheckpointPageCount * 10;
-    return {
-        litestream: must(executable, `Missing --litestream-executable`),
-        env: {
-            ...process.env,
-            ['ZERO_REPLICA_FILE']: config.replica.file,
-            ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),
-            ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(minCheckpointPageCount),
-            ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(maxCheckpointPageCount),
-            ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(incrementalBackupIntervalMinutes),
-            ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,
-            ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(snapshotBackupIntervalMinutes),
-            ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),
-            ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),
-            ['ZERO_LOG_FORMAT']: config.log.format,
-            ['LITESTREAM_CONFIG']: configPath,
-            ['LITESTREAM_PORT']: String(port),
-        },
-    };
+  const {
+    executable,
+    backupURL,
+    logLevel,
+    configPath,
+    port = config.port + 2,
+    checkpointThresholdMB,
+    incrementalBackupIntervalMinutes,
+    snapshotBackupIntervalHours,
+    multipartConcurrency,
+    multipartSize
+  } = config.litestream;
+  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;
+  const minCheckpointPageCount = checkpointThresholdMB * 250;
+  const maxCheckpointPageCount = minCheckpointPageCount * 10;
+  return {
+    litestream: must(executable, `Missing --litestream-executable`),
+    env: {
+      ...process.env,
+      ["ZERO_REPLICA_FILE"]: config.replica.file,
+      ["ZERO_LITESTREAM_BACKUP_URL"]: must(backupURLOverride ?? backupURL),
+      ["ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT"]: String(
+        minCheckpointPageCount
+      ),
+      ["ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT"]: String(
+        maxCheckpointPageCount
+      ),
+      ["ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES"]: String(
+        incrementalBackupIntervalMinutes
+      ),
+      ["ZERO_LITESTREAM_LOG_LEVEL"]: logLevelOverride ?? logLevel,
+      ["ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES"]: String(
+        snapshotBackupIntervalMinutes
+      ),
+      ["ZERO_LITESTREAM_MULTIPART_CONCURRENCY"]: String(multipartConcurrency),
+      ["ZERO_LITESTREAM_MULTIPART_SIZE"]: String(multipartSize),
+      ["ZERO_LOG_FORMAT"]: config.log.format,
+      ["LITESTREAM_CONFIG"]: configPath,
+      ["LITESTREAM_PORT"]: String(port)
+    }
+  };
 }
 async function tryRestore(lc, config) {
-    const { changeStreamer } = config;
-    const isViewSyncer = changeStreamer.mode === 'discover' || changeStreamer.uri !== undefined;
-    // Fire off a snapshot reservation to the current replication-manager
-    // (if there is one).
-    const backupURL = reserveAndGetSnapshotLocation(lc, config, isViewSyncer);
-    let backupURLOverride;
-    if (isViewSyncer) {
-        // The return value is required by view-syncers ...
-        backupURLOverride = await backupURL;
-        lc.info?.(`restoring backup from ${backupURLOverride}`);
+  const { changeStreamer } = config;
+  const isViewSyncer = changeStreamer.mode === "discover" || changeStreamer.uri !== void 0;
+  const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);
+  let snapshotStatus;
+  if (isViewSyncer) {
+    snapshotStatus = await firstMessage;
+    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);
+  } else {
+    void firstMessage.catch((e) => lc.debug?.(e));
+  }
+  const { litestream, env } = getLitestream(
+    config,
+    "debug",
+    // Include all output from `litestream restore`, as it's minimal.
+    snapshotStatus?.backupURL
+  );
+  const { restoreParallelism: parallelism } = config.litestream;
+  const proc = spawn(
+    litestream,
+    [
+      "restore",
+      "-if-db-not-exists",
+      "-if-replica-exists",
+      "-parallelism",
+      String(parallelism),
+      config.replica.file
+    ],
+    { env, stdio: "inherit", windowsHide: true }
+  );
+  const { promise, resolve, reject } = resolver();
+  proc.on("error", reject);
+  proc.on("close", (code, signal) => {
+    if (signal) {
+      reject(`litestream killed with ${signal}`);
+    } else if (code !== 0) {
+      reject(`litestream exited with code ${code}`);
+    } else {
+      resolve();
     }
-    else {
-        // but it is also useful to pause change-log cleanup when a new
-        // replication-manager is starting up. In this case, the request is
-        // best-effort. In particular, there may not be a previous
-        // replication-manager running at all.
-        void backupURL.catch(e => lc.debug?.(e));
+  });
+  await promise;
+  if (!existsSync(config.replica.file)) {
+    return false;
+  }
+  if (snapshotStatus && !replicaIsValid(lc, config.replica.file, snapshotStatus)) {
+    lc.info?.(`Deleting local replica and retrying restore`);
+    deleteLiteDB(config.replica.file);
+    return false;
+  }
+  return true;
+}
+function replicaIsValid(lc, replica, snapshot) {
+  const db = new Database(lc, replica);
+  try {
+    const { replicaVersion, watermark } = getSubscriptionState(
+      new StatementRunner(db)
+    );
+    if (replicaVersion !== snapshot.replicaVersion) {
+      lc.warn?.(
+        `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,
+        snapshot
+      );
+      return false;
     }
-    const { litestream, env } = getLitestream(config, 'debug', // Include all output from `litestream restore`, as it's minimal.
-    backupURLOverride);
-    const { restoreParallelism: parallelism } = config.litestream;
-    const proc = spawn(litestream, [
-        'restore',
-        '-if-db-not-exists',
-        '-if-replica-exists',
-        '-parallelism',
-        String(parallelism),
-        config.replica.file,
-    ], { env, stdio: 'inherit', windowsHide: true });
-    const { promise, resolve, reject } = resolver();
-    proc.on('error', reject);
-    proc.on('close', (code, signal) => {
-        if (signal) {
-            reject(`litestream killed with ${signal}`);
-        }
-        else if (code !== 0) {
-            reject(`litestream exited with code ${code}`);
-        }
-        else {
-            resolve();
-        }
-    });
-    await promise;
-    return existsSync(config.replica.file);
+    if (watermark < snapshot.minWatermark) {
+      lc.warn?.(
+        `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`
+      );
+      return false;
+    }
+    lc.info?.(
+      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,
+      snapshot
+    );
+    return true;
+  } finally {
+    db.close();
+  }
 }
-export function startReplicaBackupProcess(config) {
-    const { litestream, env } = getLitestream(config);
-    return spawn(litestream, ['replicate'], {
-        env,
-        stdio: 'inherit',
-        windowsHide: true,
-    });
+function startReplicaBackupProcess(config) {
+  const { litestream, env } = getLitestream(config);
+  return spawn(litestream, ["replicate"], {
+    env,
+    stdio: "inherit",
+    windowsHide: true
+  });
 }
-function reserveAndGetSnapshotLocation(lc, config, isViewSyncer) {
-    const { promise: backupURL, resolve, reject } = resolver();
-    void (async function () {
-        const abort = new AbortController();
-        process.on('SIGINT', () => abort.abort());
-        process.on('SIGTERM', () => abort.abort());
-        for (let i = 0;; i++) {
-            let err = '';
-            try {
-                let resolved = false;
-                const stream = await reserveSnapshot(lc, config);
-                for await (const msg of stream) {
-                    // Capture the value of the status message that the change-streamer
-                    // (i.e. BackupMonitor) returns, and hold the connection open to
-                    // "reserve" the snapshot and prevent change log cleanup.
-                    resolve(msg[1].backupURL);
-                    resolved = true;
-                }
-                // The change-streamer itself closes the connection when the
-                // subscription is started (or the reservation retried).
-                if (resolved) {
-                    break;
-                }
-            }
-            catch (e) {
-                err = e;
-            }
-            if (!isViewSyncer) {
-                return reject(err);
-            }
-            // Retry in the view-syncer since it cannot proceed until it connects
-            // to a (compatible) replication-manager. In particular, a
-            // replication-manager that does not support the view-syncer's
-            // change-streamer protocol will close the stream with an error; this
-            // retry logic essentially delays the startup of a view-syncer until
-            // a compatible replication-manager has been rolled out, allowing
-            // replication-manager and view-syncer services to be updated in
-            // parallel.
-            lc.warn?.(`Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`, String(err));
-            try {
-                await sleep(5000, abort.signal);
-            }
-            catch (e) {
-                return reject(e);
-            }
+function reserveAndGetSnapshotStatus(lc, config, isViewSyncer) {
+  const { promise: status, resolve, reject } = resolver();
+  void (async function() {
+    const abort = new AbortController();
+    process.on("SIGINT", () => abort.abort());
+    process.on("SIGTERM", () => abort.abort());
+    for (let i = 0; ; i++) {
+      let err = "";
+      try {
+        let resolved = false;
+        const stream = await reserveSnapshot(lc, config);
+        for await (const msg of stream) {
+          resolve(msg[1]);
+          resolved = true;
+        }
+        if (resolved) {
+          break;
         }
-    })();
-    return backupURL;
+      } catch (e) {
+        err = e;
+      }
+      if (!isViewSyncer) {
+        return reject(err);
+      }
+      lc.warn?.(
+        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,
+        String(err)
+      );
+      try {
+        await sleep(5e3, abort.signal);
+      } catch (e) {
+        return reject(e);
+      }
+    }
+  })();
+  return status;
 }
 function reserveSnapshot(lc, config) {
-    assertNormalized(config);
-    const { taskID, change, changeStreamer } = config;
-    const shardID = getShardConfig(config);
-    const changeStreamerClient = new ChangeStreamerHttpClient(lc, shardID, change.db, changeStreamer.uri);
-    return changeStreamerClient.reserveSnapshot(taskID);
+  assertNormalized(config);
+  const { taskID, change, changeStreamer } = config;
+  const shardID = getShardConfig(config);
+  const changeStreamerClient = new ChangeStreamerHttpClient(
+    lc,
+    shardID,
+    change.db,
+    changeStreamer.uri
+  );
+  return changeStreamerClient.reserveSnapshot(taskID);
 }
-//# sourceMappingURL=commands.js.map
+export {
+  restoreReplica,
+  startReplicaBackupProcess
+};
+//# sourceMappingURL=commands.js.map