@pylonsync/sync 0.3.364 → 0.3.366
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +22 -13
- package/dist/test-harness/server.d.ts +5 -0
- package/package.json +1 -1
- package/src/index.ts +115 -70
- package/src/reconcile.test.ts +3 -2
- package/src/scenarios.test.ts +129 -0
- package/src/test-harness/server.ts +5 -0
- package/src/test-harness/transport.ts +17 -0
package/dist/index.d.ts
CHANGED
|
@@ -249,20 +249,29 @@ export declare class SyncEngine {
|
|
|
249
249
|
*/
|
|
250
250
|
private applyQueue;
|
|
251
251
|
/**
|
|
252
|
-
* Live-event hold buffer, active
|
|
253
|
-
*
|
|
254
|
-
*
|
|
255
|
-
*
|
|
256
|
-
*
|
|
257
|
-
*
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
*
|
|
262
|
-
*
|
|
263
|
-
*
|
|
252
|
+
* Live-event hold buffer, active while ANY pull is in flight.
|
|
253
|
+
*
|
|
254
|
+
* From-zero snapshot: a snapshot is full state as-of `snapshot_seq` S; its
|
|
255
|
+
* rows arrive tagged `seq = S`. If a live WS frame (or a tab broadcast) at
|
|
256
|
+
* `seq = S+k` applies FIRST — during the snapshot's (possibly multi-page)
|
|
257
|
+
* HTTP fetch — it advances the cursor past S, and then EVERY snapshot row
|
|
258
|
+
* (seq ≤ S) is dropped by the monotonic filter in enqueueApply, leaving a
|
|
259
|
+
* near-empty replica with the cursor persisted ahead (no 410, no heal until
|
|
260
|
+
* a reconcile happens to fire).
|
|
261
|
+
*
|
|
262
|
+
* Delta catch-up has the SAME leapfrog: a live frame at seq S+k landing
|
|
263
|
+
* mid-catch-up advances the cursor past the not-yet-pulled gap (cursor, S+k),
|
|
264
|
+
* and when the pull then delivers those gap events the monotonic filter
|
|
265
|
+
* drops them — silently missing rows until the next reconcile sweeps them.
|
|
266
|
+
*
|
|
267
|
+
* The store has no per-row seq guard, so we can't just apply out-of-order —
|
|
268
|
+
* an older row would clobber a newer live update. So we ORDER them: hold
|
|
269
|
+
* live/broadcast applies here while pulling, then replay them (seq-filtered)
|
|
270
|
+
* AFTER the pull lands. On a failed pull the buffer is DISCARDED — the
|
|
271
|
+
* cursor didn't reach the held seqs, so the next pull re-fetches those
|
|
272
|
+
* events from the log. null = no pull in flight → normal apply.
|
|
264
273
|
*/
|
|
265
|
-
private
|
|
274
|
+
private pullHold;
|
|
266
275
|
/**
|
|
267
276
|
* Serialized channel for outbound network ops (pull, push, reconcile,
|
|
268
277
|
* refresh, resetReplica). Replaces the per-op `inFlightX` mutexes +
|
|
@@ -73,6 +73,11 @@ export declare class TestServer {
|
|
|
73
73
|
/** Count of snapshot pulls served (since = 0). The egress storm was a
|
|
74
74
|
* runaway count here; the regression test bounds it. */
|
|
75
75
|
snapshotPullCount: number;
|
|
76
|
+
/** When set, delta pulls (since > 0) page their response to this many
|
|
77
|
+
* events per request with a real per-page cursor + has_more — models
|
|
78
|
+
* the production DELTA_BATCH_LIMIT so catch-up pagination and
|
|
79
|
+
* fetch/apply pipelining can be exercised. */
|
|
80
|
+
deltaPageSize: number | null;
|
|
76
81
|
/** Count of /api/sync/push requests received. Lets a test assert the
|
|
77
82
|
* engine actually shipped a batch (e.g. hydrated offline writes that
|
|
78
83
|
* must drain once leader-elected), independent of the no-op push
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -375,20 +375,29 @@ export class SyncEngine {
|
|
|
375
375
|
private applyQueue: Promise<void> = Promise.resolve();
|
|
376
376
|
|
|
377
377
|
/**
|
|
378
|
-
* Live-event hold buffer, active
|
|
379
|
-
*
|
|
380
|
-
*
|
|
381
|
-
*
|
|
382
|
-
*
|
|
383
|
-
*
|
|
384
|
-
*
|
|
385
|
-
*
|
|
386
|
-
*
|
|
387
|
-
*
|
|
388
|
-
*
|
|
389
|
-
*
|
|
378
|
+
* Live-event hold buffer, active while ANY pull is in flight.
|
|
379
|
+
*
|
|
380
|
+
* From-zero snapshot: a snapshot is full state as-of `snapshot_seq` S; its
|
|
381
|
+
* rows arrive tagged `seq = S`. If a live WS frame (or a tab broadcast) at
|
|
382
|
+
* `seq = S+k` applies FIRST — during the snapshot's (possibly multi-page)
|
|
383
|
+
* HTTP fetch — it advances the cursor past S, and then EVERY snapshot row
|
|
384
|
+
* (seq ≤ S) is dropped by the monotonic filter in enqueueApply, leaving a
|
|
385
|
+
* near-empty replica with the cursor persisted ahead (no 410, no heal until
|
|
386
|
+
* a reconcile happens to fire).
|
|
387
|
+
*
|
|
388
|
+
* Delta catch-up has the SAME leapfrog: a live frame at seq S+k landing
|
|
389
|
+
* mid-catch-up advances the cursor past the not-yet-pulled gap (cursor, S+k),
|
|
390
|
+
* and when the pull then delivers those gap events the monotonic filter
|
|
391
|
+
* drops them — silently missing rows until the next reconcile sweeps them.
|
|
392
|
+
*
|
|
393
|
+
* The store has no per-row seq guard, so we can't just apply out-of-order —
|
|
394
|
+
* an older row would clobber a newer live update. So we ORDER them: hold
|
|
395
|
+
* live/broadcast applies here while pulling, then replay them (seq-filtered)
|
|
396
|
+
* AFTER the pull lands. On a failed pull the buffer is DISCARDED — the
|
|
397
|
+
* cursor didn't reach the held seqs, so the next pull re-fetches those
|
|
398
|
+
* events from the log. null = no pull in flight → normal apply.
|
|
390
399
|
*/
|
|
391
|
-
private
|
|
400
|
+
private pullHold: ChangeEvent[] | null = null;
|
|
392
401
|
|
|
393
402
|
/**
|
|
394
403
|
* Serialized channel for outbound network ops (pull, push, reconcile,
|
|
@@ -1305,14 +1314,15 @@ export class SyncEngine {
|
|
|
1305
1314
|
targetCursor?: SyncCursor,
|
|
1306
1315
|
opts: { fromBroadcast?: boolean; isPull?: boolean } = {},
|
|
1307
1316
|
): Promise<void> {
|
|
1308
|
-
//
|
|
1309
|
-
//
|
|
1310
|
-
//
|
|
1311
|
-
//
|
|
1312
|
-
// in arrival (≈seq) order once the
|
|
1313
|
-
// queue chain so held events never interleave into the
|
|
1314
|
-
|
|
1315
|
-
|
|
1317
|
+
// Pull fence: while any pull is in flight, hold live WS frames + tab
|
|
1318
|
+
// broadcasts so they can't advance the cursor past rows the pull hasn't
|
|
1319
|
+
// delivered yet and get them filtered out (see `pullHold`). The pull's
|
|
1320
|
+
// own applies (`isPull`) are exempt — they ARE the pull. Held events are
|
|
1321
|
+
// replayed in arrival (≈seq) order once the pull lands. Synchronous +
|
|
1322
|
+
// before the queue chain so held events never interleave into the
|
|
1323
|
+
// applyQueue.
|
|
1324
|
+
if (this.pullHold !== null && !opts.isPull) {
|
|
1325
|
+
this.pullHold.push(...changes);
|
|
1316
1326
|
return Promise.resolve();
|
|
1317
1327
|
}
|
|
1318
1328
|
const prev = this.applyQueue;
|
|
@@ -1708,22 +1718,48 @@ export class SyncEngine {
|
|
|
1708
1718
|
// bootstrap reconcile (the snapshot path already returned every
|
|
1709
1719
|
// policy-visible row, per-entity refetch right after is waste).
|
|
1710
1720
|
const startedFromZero = this.cursor.last_seq === 0;
|
|
1711
|
-
//
|
|
1712
|
-
//
|
|
1713
|
-
//
|
|
1714
|
-
//
|
|
1715
|
-
//
|
|
1716
|
-
|
|
1721
|
+
// Open the live-event hold for the pull's whole (possibly multi-page)
|
|
1722
|
+
// duration so a racing WS frame can't leapfrog the cursor — past the
|
|
1723
|
+
// snapshot's rows on a from-zero pull, or past the not-yet-pulled gap
|
|
1724
|
+
// on a delta catch-up (see `pullHold`). The pull's own applies pass
|
|
1725
|
+
// `isPull` so they're never held. The 410 handler discards this
|
|
1726
|
+
// buffer before recursing, so the nested pull always opens (and owns)
|
|
1727
|
+
// a fresh hold.
|
|
1728
|
+
this.pullHold = [];
|
|
1729
|
+
// Declared outside the try: on a mid-loop fetch error the previous
|
|
1730
|
+
// page's apply may still be in flight, and the catch (410 reset →
|
|
1731
|
+
// re-pull) MUST let it settle first — otherwise the orphaned apply
|
|
1732
|
+
// races the replica wipe and re-writes stale rows into the fresh
|
|
1733
|
+
// store (and drags the cursor forward under the recursive pull).
|
|
1734
|
+
let pendingApply: Promise<void> | null = null;
|
|
1717
1735
|
try {
|
|
1718
1736
|
// Snapshot pagination: when the cursor is 0 and the server's
|
|
1719
1737
|
// table is larger than a single batch, the response carries
|
|
1720
|
-
// `snapshot_after` for the next page.
|
|
1721
|
-
//
|
|
1722
|
-
//
|
|
1723
|
-
// for the whole
|
|
1738
|
+
// `snapshot_after` for the next page. The change-log tail
|
|
1739
|
+
// paginates via `has_more`. Both loop until exhausted BEFORE
|
|
1740
|
+
// returning so a fresh client always observes a consistent full
|
|
1741
|
+
// snapshot (not a 1k-row prefix it mistakes for the whole
|
|
1742
|
+
// replica) and a catching-up client drains the whole tail.
|
|
1743
|
+
//
|
|
1744
|
+
// Fetch/apply pipelining: applying a page (IndexedDB writes) and
|
|
1745
|
+
// fetching the next one are independent, so page N's apply runs
|
|
1746
|
+
// WHILE page N+1 is in flight — catch-up latency is
|
|
1747
|
+
// max(network, apply) per page instead of their sum. Ordering is
|
|
1748
|
+
// safe because enqueueApply chains every batch onto the same
|
|
1749
|
+
// applyQueue in call order; page N+1 is only ENQUEUED after page
|
|
1750
|
+
// N's apply resolved, so a failed apply aborts the loop instead
|
|
1751
|
+
// of letting later pages advance the cursor over a hole.
|
|
1752
|
+
//
|
|
1753
|
+
// The next `since` comes from each RESPONSE's cursor, not
|
|
1754
|
+
// `this.cursor` — the apply is what advances `this.cursor`, and
|
|
1755
|
+
// waiting on it is exactly what pipelining removes. Mid-snapshot
|
|
1756
|
+
// the server pins the response cursor at 0, which keeps routing
|
|
1757
|
+
// to the snapshot path.
|
|
1724
1758
|
let snapshotAfter: string | undefined;
|
|
1759
|
+
let hasMore = false;
|
|
1725
1760
|
let firstPass = true;
|
|
1726
|
-
|
|
1761
|
+
let nextSince = this.cursor.last_seq;
|
|
1762
|
+
while (firstPass || snapshotAfter || hasMore) {
|
|
1727
1763
|
firstPass = false;
|
|
1728
1764
|
// `snapshot_after` is an OPAQUE cursor the server already URL-encoded
|
|
1729
1765
|
// (it `url_encode`s the JSON payload). It MUST be appended raw — running
|
|
@@ -1733,33 +1769,28 @@ export class SyncEngine {
|
|
|
1733
1769
|
// restart the snapshot from row 0 — an infinite re-snapshot loop for any
|
|
1734
1770
|
// table larger than one page (SNAPSHOT_BATCH_LIMIT rows). `since` is a
|
|
1735
1771
|
// plain integer, so it's safe to inline.
|
|
1736
|
-
let query = `since=${
|
|
1772
|
+
let query = `since=${nextSince}`;
|
|
1737
1773
|
if (snapshotAfter) {
|
|
1738
1774
|
query += `&snapshot_after=${snapshotAfter}`;
|
|
1739
1775
|
}
|
|
1740
1776
|
const resp = await this.request<
|
|
1741
1777
|
PullResponse & { snapshot_after?: string | null }
|
|
1742
1778
|
>("GET", `/api/sync/pull?${query}`);
|
|
1743
|
-
|
|
1779
|
+
if (pendingApply) await pendingApply;
|
|
1780
|
+
pendingApply = this.enqueueApply(resp.changes, resp.cursor, {
|
|
1781
|
+
isPull: true,
|
|
1782
|
+
});
|
|
1783
|
+
// Guard against a server that reports more pages without
|
|
1784
|
+
// advancing the cursor (a transient no-progress page): break
|
|
1785
|
+
// rather than refetch the same page forever. The next poll /
|
|
1786
|
+
// change event re-drives the pull.
|
|
1787
|
+
const advanced = resp.cursor.last_seq > nextSince;
|
|
1788
|
+
nextSince = resp.cursor.last_seq;
|
|
1744
1789
|
// `snapshot_after` is only set when the server is mid-snapshot.
|
|
1745
|
-
// Continue paginating in the same loop iteration so we don't
|
|
1746
|
-
// leave a fresh client with a partial replica.
|
|
1747
1790
|
snapshotAfter = resp.snapshot_after ?? undefined;
|
|
1748
|
-
|
|
1749
|
-
// by recursing into `pullInner` directly. We are INSIDE the
|
|
1750
|
-
// `pull` op-queue slot right now; calling the public `pull()`
|
|
1751
|
-
// would re-enqueue under the same "pull" key, which coalesces
|
|
1752
|
-
// to the promise we're currently running inside (op-queue.ts
|
|
1753
|
-
// deletes the key only after `fn` resolves) and `await` it →
|
|
1754
|
-
// permanent self-deadlock that bricks the entire pull path for
|
|
1755
|
-
// the session. This is the exact hazard the 410 handler avoids;
|
|
1756
|
-
// `pullInner` re-reads `this.cursor.last_seq` (already advanced
|
|
1757
|
-
// by enqueueApply) so the recursion resumes at the right cursor.
|
|
1758
|
-
if (!snapshotAfter && resp.has_more) {
|
|
1759
|
-
await this.pullInner();
|
|
1760
|
-
break;
|
|
1761
|
-
}
|
|
1791
|
+
hasMore = !snapshotAfter && resp.has_more && advanced;
|
|
1762
1792
|
}
|
|
1793
|
+
if (pendingApply) await pendingApply;
|
|
1763
1794
|
// Clear the resync circuit breaker ONLY on a successful DELTA
|
|
1764
1795
|
// pull — one that started from a real, non-zero cursor the server
|
|
1765
1796
|
// honored. A snapshot pull from cursor=0 succeeding does NOT prove
|
|
@@ -1784,17 +1815,23 @@ export class SyncEngine {
|
|
|
1784
1815
|
// truth. Record it so onConnected skips the reconcile that would
|
|
1785
1816
|
// otherwise re-fetch every entity via cursor pagination.
|
|
1786
1817
|
this.lastPullStartedFromZero = startedFromZero;
|
|
1787
|
-
//
|
|
1788
|
-
// arrival (≈seq) order. They filter against the now-
|
|
1789
|
-
//
|
|
1790
|
-
// (already
|
|
1791
|
-
// means this replay applies normally (it isn't re-held).
|
|
1792
|
-
if (
|
|
1793
|
-
const held = this.
|
|
1794
|
-
this.
|
|
1818
|
+
// Pull landed cleanly → replay the live events we held during it, in
|
|
1819
|
+
// arrival (≈seq) order. They filter against the now-advanced cursor,
|
|
1820
|
+
// so events newer than what the pull delivered apply and older ones
|
|
1821
|
+
// (already delivered by the pull) are deduped. Clearing `pullHold`
|
|
1822
|
+
// first means this replay applies normally (it isn't re-held).
|
|
1823
|
+
if (this.pullHold) {
|
|
1824
|
+
const held = this.pullHold;
|
|
1825
|
+
this.pullHold = null;
|
|
1795
1826
|
if (held.length > 0) await this.enqueueApply(held);
|
|
1796
1827
|
}
|
|
1797
1828
|
} catch (err) {
|
|
1829
|
+
// Settle any in-flight page apply before acting on the error —
|
|
1830
|
+
// the 410 path below wipes the replica, and an apply landing
|
|
1831
|
+
// after the wipe would resurrect stale rows and advance the
|
|
1832
|
+
// cursor under the recursive re-pull. Failures are already
|
|
1833
|
+
// handled batch-locally; only settlement matters here.
|
|
1834
|
+
if (pendingApply) await pendingApply.catch(() => {});
|
|
1798
1835
|
// Swallow network + transient errors so the poll/reconnect loop
|
|
1799
1836
|
// keeps trying — but on 429 bump the backoff counter so the next
|
|
1800
1837
|
// reconnect waits noticeably longer. Without this, a rate-limited
|
|
@@ -1831,6 +1868,10 @@ export class SyncEngine {
|
|
|
1831
1868
|
// First resync of the episode — snapshot now. Bypass the queue
|
|
1832
1869
|
// (we ARE the pull op holding the slot; the public pull()
|
|
1833
1870
|
// would re-enqueue and share our own promise back → deadlock).
|
|
1871
|
+
// Discard this failed episode's held events first so the
|
|
1872
|
+
// nested pull opens (and owns) a fresh hold; the from-zero
|
|
1873
|
+
// snapshot it runs covers everything the buffer held.
|
|
1874
|
+
this.pullHold = null;
|
|
1834
1875
|
await this.resetReplicaInner();
|
|
1835
1876
|
await this.pullInner();
|
|
1836
1877
|
} else {
|
|
@@ -1849,14 +1890,14 @@ export class SyncEngine {
|
|
|
1849
1890
|
}
|
|
1850
1891
|
}
|
|
1851
1892
|
} finally {
|
|
1852
|
-
//
|
|
1853
|
-
//
|
|
1854
|
-
// so the
|
|
1855
|
-
//
|
|
1856
|
-
// the
|
|
1857
|
-
//
|
|
1858
|
-
// the
|
|
1859
|
-
if (
|
|
1893
|
+
// Pull failed (network error / 410 mid-fetch): DISCARD any still-held
|
|
1894
|
+
// live events rather than applying them. The cursor never reached the
|
|
1895
|
+
// held seqs, so the next pull re-fetches those events from the log;
|
|
1896
|
+
// applying them here would advance the cursor over the un-pulled gap
|
|
1897
|
+
// (the exact leapfrog the hold exists to prevent). On success the try
|
|
1898
|
+
// already drained + nulled the hold, so this is a no-op there — and
|
|
1899
|
+
// the 410 recursion drained or re-owned it before we got here.
|
|
1900
|
+
if (this.pullHold !== null) this.pullHold = null;
|
|
1860
1901
|
}
|
|
1861
1902
|
}
|
|
1862
1903
|
|
|
@@ -2128,16 +2169,20 @@ export class SyncEngine {
|
|
|
2128
2169
|
): Promise<{ rows: Row[]; truncated: boolean }> {
|
|
2129
2170
|
const out: Row[] = [];
|
|
2130
2171
|
let cursor: string | null = null;
|
|
2131
|
-
//
|
|
2172
|
+
// 20 pages × 1000 per page = 20k rows. A `sync:true` entity larger than
|
|
2132
2173
|
// this should switch to `sync: false` + search/by-id — see useInfiniteQuery.
|
|
2133
|
-
|
|
2174
|
+
// 1000/page matches the sync-pull batch limits; the server only honors it
|
|
2175
|
+
// on replication fetches (`sync=1`), and each page costs a full round
|
|
2176
|
+
// trip, so the page size directly divides sweep latency (a 20k-row
|
|
2177
|
+
// entity was 200 sequential requests; now 20).
|
|
2178
|
+
for (let page = 0; page < 20; page++) {
|
|
2134
2179
|
// `sync=1` marks this as a REPLICATION fetch, which is what makes an
|
|
2135
2180
|
// entity's `sync` scope apply. Without it the server treats the request
|
|
2136
2181
|
// as an ordinary app read and returns unscoped rows — correct for a
|
|
2137
2182
|
// direct read, wrong here, since these rows land in the replica.
|
|
2138
2183
|
const qs: string = cursor
|
|
2139
|
-
? `?limit=
|
|
2140
|
-
: `?limit=
|
|
2184
|
+
? `?limit=1000&sync=1&after=${encodeURIComponent(cursor)}`
|
|
2185
|
+
: `?limit=1000&sync=1`;
|
|
2141
2186
|
const resp: {
|
|
2142
2187
|
data: Row[];
|
|
2143
2188
|
next_cursor: string | null;
|
package/src/reconcile.test.ts
CHANGED
|
@@ -301,8 +301,9 @@ describe("SyncEngine.reconcile", () => {
|
|
|
301
301
|
|
|
302
302
|
await engine.reconcile(["Event"]);
|
|
303
303
|
|
|
304
|
-
// The fetch hit the
|
|
305
|
-
|
|
304
|
+
// The fetch hit the 20-page safety cap (20 × 1000/page = the same 20k
|
|
305
|
+
// row ceiling) — proof the set was truncated.
|
|
306
|
+
expect(pages).toBe(20);
|
|
306
307
|
// The un-fetched local rows are NOT deleted (the bug would drop them).
|
|
307
308
|
expect(engine.store.get("Event", "beyond_1")).not.toBeNull();
|
|
308
309
|
expect(engine.store.get("Event", "beyond_2")).not.toBeNull();
|
package/src/scenarios.test.ts
CHANGED
|
@@ -555,6 +555,135 @@ describe("sync scenarios", () => {
|
|
|
555
555
|
expect(env.engine.store.get("Note", "n2")).not.toBeNull();
|
|
556
556
|
});
|
|
557
557
|
|
|
558
|
+
// CATCH-UP PIPELINING (pins the pullInner fetch/apply overlap). A
|
|
559
|
+
// multi-page delta catch-up used to serialize fetch page N → apply
|
|
560
|
+
// page N → fetch page N+1, paying network + apply per page in SUM.
|
|
561
|
+
// The pipelined loop derives the next `since` from each RESPONSE's
|
|
562
|
+
// cursor and starts the next fetch while the previous page is still
|
|
563
|
+
// applying, so per-page cost is max(network, apply). This test slows
|
|
564
|
+
// applies down and asserts (a) a later page's fetch arrives at the
|
|
565
|
+
// server WHILE an apply is still in flight, and (b) every page still
|
|
566
|
+
// lands, in order, with nothing skipped.
|
|
567
|
+
test("multi-page delta catch-up overlaps fetch with apply and drains completely", async () => {
|
|
568
|
+
let deltaFetches = 0;
|
|
569
|
+
let applyEnds = 0;
|
|
570
|
+
let overlapped = false;
|
|
571
|
+
env = createTestEnv({
|
|
572
|
+
transport: "poll",
|
|
573
|
+
beforePull: (_auth, since) => {
|
|
574
|
+
if (since > 0) {
|
|
575
|
+
deltaFetches++;
|
|
576
|
+
// The serialized loop finishes apply k-1 BEFORE issuing
|
|
577
|
+
// fetch k, so it always arrives with applyEnds == k-1. A
|
|
578
|
+
// fetch arriving with fewer applies completed is the
|
|
579
|
+
// pipeline overlap.
|
|
580
|
+
if (deltaFetches >= 2 && applyEnds < deltaFetches - 1) {
|
|
581
|
+
overlapped = true;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
},
|
|
585
|
+
});
|
|
586
|
+
env.signIn({ userId: "u1" });
|
|
587
|
+
env.server.seed("Note", [{ id: "n0", title: "seed" }]);
|
|
588
|
+
await env.start();
|
|
589
|
+
await env.flush();
|
|
590
|
+
|
|
591
|
+
// 30 changes behind, served 10 per page → a 3-page catch-up.
|
|
592
|
+
for (let i = 1; i <= 30; i++) {
|
|
593
|
+
env.server.insert("Note", { id: `n${i}`, title: `t${i}` });
|
|
594
|
+
}
|
|
595
|
+
env.server.deltaPageSize = 10;
|
|
596
|
+
|
|
597
|
+
// Make applies observably slow so the overlap window is real.
|
|
598
|
+
const store = env.engine.store as unknown as {
|
|
599
|
+
applyChangesAsync: (c: unknown[]) => Promise<boolean>;
|
|
600
|
+
};
|
|
601
|
+
const realApply = store.applyChangesAsync.bind(store);
|
|
602
|
+
store.applyChangesAsync = async (changes) => {
|
|
603
|
+
await new Promise((r) => setTimeout(r, 25));
|
|
604
|
+
const out = await realApply(changes);
|
|
605
|
+
applyEnds++;
|
|
606
|
+
return out;
|
|
607
|
+
};
|
|
608
|
+
|
|
609
|
+
await env.engine.pull();
|
|
610
|
+
await env.flush();
|
|
611
|
+
|
|
612
|
+
// Every page landed — first, middle, and last row all present.
|
|
613
|
+
expect(env.engine.store.get("Note", "n1")).not.toBeNull();
|
|
614
|
+
expect(env.engine.store.get("Note", "n15")).not.toBeNull();
|
|
615
|
+
expect(env.engine.store.get("Note", "n30")).not.toBeNull();
|
|
616
|
+
// It really paginated (≥3 delta pages), and at least one later
|
|
617
|
+
// fetch overlapped an in-flight apply. A serialized loop never
|
|
618
|
+
// trips `overlapped` because each fetch waits for the apply.
|
|
619
|
+
const deltaPulls = env.server.pullUrls.filter(
|
|
620
|
+
(u) => !u.includes("since=0"),
|
|
621
|
+
).length;
|
|
622
|
+
expect(deltaPulls).toBeGreaterThanOrEqual(3);
|
|
623
|
+
expect(overlapped).toBe(true);
|
|
624
|
+
});
|
|
625
|
+
|
|
626
|
+
// WS LEAPFROG (pins the pullHold extension to delta pulls). A live WS
|
|
627
|
+
// frame landing MID-catch-up used to apply immediately and advance the
|
|
628
|
+
// cursor past the not-yet-pulled gap; when the pull then delivered the
|
|
629
|
+
// gap events, the monotonic seq filter dropped them — silently missing
|
|
630
|
+
// rows until a reconcile happened to sweep them. The hold buffers live
|
|
631
|
+
// frames for the pull's duration and replays them (seq-filtered) after
|
|
632
|
+
// it lands, so BOTH the gap rows and the live frame's row survive.
|
|
633
|
+
test("a live event mid-catch-up cannot leapfrog the cursor past unapplied pages", async () => {
|
|
634
|
+
let deltaFetches = 0;
|
|
635
|
+
let engineRef: SyncEngine | null = null;
|
|
636
|
+
env = createTestEnv({
|
|
637
|
+
transport: "poll",
|
|
638
|
+
beforePull: (_auth, since) => {
|
|
639
|
+
if (since > 0) {
|
|
640
|
+
deltaFetches += 1;
|
|
641
|
+
// Mid-catch-up (page 2 of 3), a "live WS frame" with a far
|
|
642
|
+
// newer seq arrives — exactly what onChangeEvent feeds into
|
|
643
|
+
// enqueueApply. Pre-fix this applied instantly, advanced the
|
|
644
|
+
// cursor to 10000, and pages 2–3 were filtered out on apply.
|
|
645
|
+
if (deltaFetches === 2 && engineRef) {
|
|
646
|
+
void (
|
|
647
|
+
engineRef as unknown as {
|
|
648
|
+
enqueueApply(c: unknown[]): Promise<void>;
|
|
649
|
+
}
|
|
650
|
+
).enqueueApply([
|
|
651
|
+
{
|
|
652
|
+
seq: 10_000,
|
|
653
|
+
entity: "Note",
|
|
654
|
+
row_id: "live_1",
|
|
655
|
+
kind: "insert",
|
|
656
|
+
data: { id: "live_1", title: "live frame" },
|
|
657
|
+
timestamp: "",
|
|
658
|
+
},
|
|
659
|
+
]);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
},
|
|
663
|
+
});
|
|
664
|
+
env.signIn({ userId: "u1" });
|
|
665
|
+
env.server.seed("Note", [{ id: "n0", title: "seed" }]);
|
|
666
|
+
await env.start();
|
|
667
|
+
await env.flush();
|
|
668
|
+
engineRef = env.engine;
|
|
669
|
+
|
|
670
|
+
for (let i = 1; i <= 30; i++) {
|
|
671
|
+
env.server.insert("Note", { id: `n${i}`, title: `t${i}` });
|
|
672
|
+
}
|
|
673
|
+
env.server.deltaPageSize = 10;
|
|
674
|
+
|
|
675
|
+
await env.engine.pull();
|
|
676
|
+
await env.flush();
|
|
677
|
+
|
|
678
|
+
// Every gap page landed — including rows from the pages AFTER the
|
|
679
|
+
// live frame arrived (pre-fix these were silently dropped).
|
|
680
|
+
expect(env.engine.store.get("Note", "n15")).not.toBeNull();
|
|
681
|
+
expect(env.engine.store.get("Note", "n25")).not.toBeNull();
|
|
682
|
+
expect(env.engine.store.get("Note", "n30")).not.toBeNull();
|
|
683
|
+
// And the held live frame itself replayed after the pull.
|
|
684
|
+
expect(env.engine.store.get("Note", "live_1")).not.toBeNull();
|
|
685
|
+
});
|
|
686
|
+
|
|
558
687
|
// OFFLINE WRITES (pins the transient/permanent split in pushInner).
|
|
559
688
|
// A push that fails with a NETWORK error (offline — fetch rejects, no
|
|
560
689
|
// HTTP status) must keep the mutation `pending` and the optimistic
|
|
@@ -102,6 +102,11 @@ export class TestServer {
|
|
|
102
102
|
/** Count of snapshot pulls served (since = 0). The egress storm was a
|
|
103
103
|
* runaway count here; the regression test bounds it. */
|
|
104
104
|
snapshotPullCount = 0;
|
|
105
|
+
/** When set, delta pulls (since > 0) page their response to this many
|
|
106
|
+
* events per request with a real per-page cursor + has_more — models
|
|
107
|
+
* the production DELTA_BATCH_LIMIT so catch-up pagination and
|
|
108
|
+
* fetch/apply pipelining can be exercised. */
|
|
109
|
+
deltaPageSize: number | null = null;
|
|
105
110
|
/** Count of /api/sync/push requests received. Lets a test assert the
|
|
106
111
|
* engine actually shipped a batch (e.g. hydrated offline writes that
|
|
107
112
|
* must drain once leader-elected), independent of the no-op push
|
|
@@ -278,6 +278,23 @@ async function handle(
|
|
|
278
278
|
// terminating snapshot (no snapshot_after → client exits the loop).
|
|
279
279
|
}
|
|
280
280
|
const resp = await server.pull(token, since);
|
|
281
|
+
// Delta paging sim: slice to `deltaPageSize` events with a real
|
|
282
|
+
// per-page cursor + has_more, like the production DELTA_BATCH_LIMIT.
|
|
283
|
+
if (
|
|
284
|
+
since > 0 &&
|
|
285
|
+
server.deltaPageSize != null &&
|
|
286
|
+
resp.changes.length > server.deltaPageSize
|
|
287
|
+
) {
|
|
288
|
+
const page = resp.changes.slice(0, server.deltaPageSize);
|
|
289
|
+
return {
|
|
290
|
+
status: 200,
|
|
291
|
+
body: {
|
|
292
|
+
changes: page,
|
|
293
|
+
cursor: { last_seq: page[page.length - 1]!.seq },
|
|
294
|
+
has_more: true,
|
|
295
|
+
},
|
|
296
|
+
};
|
|
297
|
+
}
|
|
281
298
|
// One-shot has_more on a delta pull → drives the tail-pull recursion.
|
|
282
299
|
if (since > 0 && server.consumeNextPullHasMore()) {
|
|
283
300
|
return { status: 200, body: { ...resp, has_more: true } };
|