@spooky-sync/core 0.0.1-canary.196 → 0.0.1-canary.198

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -693,6 +693,28 @@ declare class DataModule<S extends SchemaStructure> {
693
693
  * behavior for windows and is preserved.
694
694
  */
695
695
  private resolveMembership;
696
+ private readonly settledWrites;
697
+ private readonly settledDeletes;
698
+ /**
699
+ * Grace period for a settled write. Long enough to cover an SSP round trip
700
+ * that is running slowly (seconds, not milliseconds, when the edge path is
701
+ * backed up), short enough that a write the server silently dropped cannot
702
+ * linger misleadingly.
703
+ *
704
+ * The rejection case does NOT rely on this expiring: an application error
705
+ * rolls the mutation back and never reports it settled, so it vanishes at
706
+ * once. This deadline only bounds the case where the write succeeded and its
707
+ * membership never arrived at all.
708
+ */
709
+ private static readonly SETTLED_WRITE_GRACE_MS;
710
+ /**
711
+ * Report that a mutation was accepted by the server and its outbox row
712
+ * removed. Called only on the SUCCESS path — a rolled-back mutation must
713
+ * disappear immediately, which is what makes this safe.
714
+ */
715
+ noteWriteSettled(recordId: string, mutationType: string): void;
716
+ /** Drop entries past their deadline. */
717
+ private pruneSettled;
696
718
  /** Apply the pending-write union and pending-delete subtraction, and map to
697
719
  * RecordIds for the engines' id-set path. */
698
720
  private buildRenderIds;
@@ -1341,6 +1363,18 @@ declare class Sp00kySync<S extends SchemaStructure> {
1341
1363
  */
1342
1364
  private withPushTimeout;
1343
1365
  private processUpEvent;
1366
+ /**
1367
+ * A mutation the server accepted, reported once its outbox row is gone.
1368
+ *
1369
+ * Keeps the written row in the render set until its membership arrives.
1370
+ * Without this the row is briefly in neither term of
1371
+ * `(membership ∪ pendingWrites) − pendingDeletes` — the outbox delete is
1372
+ * tied to the push, while membership waits on the SSP ingesting the row,
1373
+ * materializing the view, writing the `_00_list_ref` edge and this client
1374
+ * reading it back. The writer therefore watched its own comment appear,
1375
+ * vanish, and return, while every other client showed it throughout.
1376
+ */
1377
+ private handleMutationSettled;
1344
1378
  private handleRollback;
1345
1379
  private processDownEvent;
1346
1380
  /**
package/dist/index.js CHANGED
@@ -3255,7 +3255,7 @@ function phaseStatOf(samples, lastMs) {
3255
3255
  * Merges the functionality of QueryManager and MutationManager.
3256
3256
  * Uses CacheModule for all storage operations.
3257
3257
  */
3258
- var DataModule = class {
3258
+ var DataModule = class DataModule {
3259
3259
  /** Tab identity baked into mutation ids (shared-tabs rollback routing);
3260
3260
  * undefined in solo mode, where mutation-id falls back to a session id. */
3261
3261
  tabId;
@@ -3583,17 +3583,55 @@ var DataModule = class {
3583
3583
  if (buildWindowMaterialization(config.surql) !== null) return config.remoteArray?.length && config.remoteArray || sspArray?.length && sspArray || config.localArray || [];
3584
3584
  return null;
3585
3585
  }
3586
+ settledWrites = /* @__PURE__ */ new Map();
3587
+ settledDeletes = /* @__PURE__ */ new Map();
3588
+ /**
3589
+ * Grace period for a settled write. Long enough to cover an SSP round trip
3590
+ * that is running slowly (seconds, not milliseconds, when the edge path is
3591
+ * backed up), short enough that a write the server silently dropped cannot
3592
+ * linger misleadingly.
3593
+ *
3594
+ * The rejection case does NOT rely on this expiring: an application error
3595
+ * rolls the mutation back and never reports it settled, so it vanishes at
3596
+ * once. This deadline only bounds the case where the write succeeded and its
3597
+ * membership never arrived at all.
3598
+ */
3599
+ static SETTLED_WRITE_GRACE_MS = 1e4;
3600
+ /**
3601
+ * Report that a mutation was accepted by the server and its outbox row
3602
+ * removed. Called only on the SUCCESS path — a rolled-back mutation must
3603
+ * disappear immediately, which is what makes this safe.
3604
+ */
3605
+ noteWriteSettled(recordId, mutationType) {
3606
+ const until = Date.now() + DataModule.SETTLED_WRITE_GRACE_MS;
3607
+ if (mutationType === "delete") this.settledDeletes.set(recordId, until);
3608
+ else this.settledWrites.set(recordId, until);
3609
+ }
3610
+ /** Drop entries past their deadline. */
3611
+ pruneSettled(now) {
3612
+ for (const [id, until] of this.settledWrites) if (until <= now) this.settledWrites.delete(id);
3613
+ for (const [id, until] of this.settledDeletes) if (until <= now) this.settledDeletes.delete(id);
3614
+ }
3586
3615
  /** Apply the pending-write union and pending-delete subtraction, and map to
3587
3616
  * RecordIds for the engines' id-set path. */
3588
3617
  async buildRenderIds(config, membership, sspArray) {
3589
3618
  const { writes, deletes } = await this.getPendingRecordIds();
3619
+ const now = Date.now();
3620
+ this.pruneSettled(now);
3621
+ if (this.settledWrites.size > 0) for (const id of this.settledWrites.keys()) writes.add(id);
3622
+ if (this.settledDeletes.size > 0) for (const id of this.settledDeletes.keys()) deletes.add(id);
3590
3623
  const ordered = [];
3591
3624
  const seen = /* @__PURE__ */ new Set();
3592
3625
  for (const [id] of membership) {
3626
+ if (this.settledWrites.size > 0) this.settledWrites.delete(id);
3593
3627
  if (deletes.has(id) || seen.has(id)) continue;
3594
3628
  seen.add(id);
3595
3629
  ordered.push(id);
3596
3630
  }
3631
+ if (this.settledDeletes.size > 0) {
3632
+ const stillListed = new Set(membership.map(([id]) => id));
3633
+ for (const id of [...this.settledDeletes.keys()]) if (!stillListed.has(id)) this.settledDeletes.delete(id);
3634
+ }
3597
3635
  if (writes.size > 0) {
3598
3636
  const localView = sspArray?.length && sspArray || config.localArray || [];
3599
3637
  for (const [id] of localView) {
@@ -4803,7 +4841,14 @@ var UpQueue = class {
4803
4841
  for (const { timer } of this.debouncedMutations.values()) clearTimeout(timer);
4804
4842
  this.debouncedMutations.clear();
4805
4843
  }
4806
- async next(fn, onRollback) {
4844
+ /**
4845
+ * @param onSettled Reports a mutation the server ACCEPTED, after its outbox
4846
+ * row is gone. Deliberately not called on the rollback path: a rejected
4847
+ * mutation must stop being rendered immediately, while an accepted one has
4848
+ * to stay visible until its membership arrives (see
4849
+ * `DataModule.noteWriteSettled`).
4850
+ */
4851
+ async next(fn, onRollback, onSettled) {
4807
4852
  const event = this.queue.shift();
4808
4853
  if (event) {
4809
4854
  try {
@@ -4856,6 +4901,15 @@ var UpQueue = class {
4856
4901
  Category: "sp00ky-client::UpQueue::next"
4857
4902
  }, "Failed to remove mutation from database after successful processing");
4858
4903
  }
4904
+ if (onSettled) try {
4905
+ onSettled(event);
4906
+ } catch (error) {
4907
+ this.logger.error({
4908
+ error,
4909
+ event,
4910
+ Category: "sp00ky-client::UpQueue::next"
4911
+ }, "Settled-write handler failed");
4912
+ }
4859
4913
  this._events.addEvent({
4860
4914
  type: SyncQueueEventTypes.MutationDequeued,
4861
4915
  payload: { queueSize: this.queue.length }
@@ -5407,7 +5461,7 @@ var SyncScheduler = class {
5407
5461
  downRetryTimer;
5408
5462
  upRetryAttempt = 0;
5409
5463
  downRetryAttempt = 0;
5410
- constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback, onSyncOutcome) {
5464
+ constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback, onSyncOutcome, onSettled) {
5411
5465
  this.upQueue = upQueue;
5412
5466
  this.downQueue = downQueue;
5413
5467
  this.onProcessUp = onProcessUp;
@@ -5415,6 +5469,7 @@ var SyncScheduler = class {
5415
5469
  this.logger = logger;
5416
5470
  this.onRollback = onRollback;
5417
5471
  this.onSyncOutcome = onSyncOutcome;
5472
+ this.onSettled = onSettled;
5418
5473
  }
5419
5474
  async init(opts = {}) {
5420
5475
  if (opts.loadOutbox !== false) await this.upQueue.loadFromDatabase();
@@ -5508,7 +5563,7 @@ var SyncScheduler = class {
5508
5563
  let processedAny = false;
5509
5564
  try {
5510
5565
  while (this.upQueue.size > 0 && !this.paused) {
5511
- await this.upQueue.next(this.onProcessUp, this.onRollback);
5566
+ await this.upQueue.next(this.onProcessUp, this.onRollback, this.onSettled);
5512
5567
  processedAny = true;
5513
5568
  }
5514
5569
  if (processedAny) this.onSyncOutcome?.(true);
@@ -5801,7 +5856,7 @@ var Sp00kySync = class Sp00kySync {
5801
5856
  this.upQueue = new UpQueue(this.local, this.logger, (dropped) => this.onMutationDropped(dropped));
5802
5857
  this.downQueue = new DownQueue(this.local, this.logger);
5803
5858
  this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
5804
- this.scheduler = new SyncScheduler(this.upQueue, this.downQueue, this.processUpEvent.bind(this), this.processDownEvent.bind(this), this.logger, this.handleRollback.bind(this), this.recordSyncOutcome.bind(this));
5859
+ this.scheduler = new SyncScheduler(this.upQueue, this.downQueue, this.processUpEvent.bind(this), this.processDownEvent.bind(this), this.logger, this.handleRollback.bind(this), this.recordSyncOutcome.bind(this), this.handleMutationSettled.bind(this));
5805
5860
  this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
5806
5861
  this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
5807
5862
  this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
@@ -6425,6 +6480,20 @@ var Sp00kySync = class Sp00kySync {
6425
6480
  return;
6426
6481
  }
6427
6482
  }
6483
+ /**
6484
+ * A mutation the server accepted, reported once its outbox row is gone.
6485
+ *
6486
+ * Keeps the written row in the render set until its membership arrives.
6487
+ * Without this the row is briefly in neither term of
6488
+ * `(membership ∪ pendingWrites) − pendingDeletes` — the outbox delete is
6489
+ * tied to the push, while membership waits on the SSP ingesting the row,
6490
+ * materializing the view, writing the `_00_list_ref` edge and this client
6491
+ * reading it back. The writer therefore watched its own comment appear,
6492
+ * vanish, and return, while every other client showed it throughout.
6493
+ */
6494
+ handleMutationSettled(event) {
6495
+ this.dataModule.noteWriteSettled(encodeRecordId(event.record_id), event.type);
6496
+ }
6428
6497
  async handleRollback(event, error) {
6429
6498
  const recordId = encodeRecordId(event.record_id);
6430
6499
  const tableName = event.type === "create" && event.tableName ? event.tableName : extractTablePart(recordId);
@@ -6677,7 +6746,18 @@ var Sp00kySync = class Sp00kySync {
6677
6746
  }, "Query to register not found");
6678
6747
  throw new Error("Query to register not found");
6679
6748
  }
6680
- await this.remote.query("fn::query::heartbeat($id)", { id: queryState.config.id });
6749
+ const result = await this.remote.query("fn::query::heartbeat($id)", { id: queryState.config.id });
6750
+ const updated = Array.isArray(result) ? result[0] : void 0;
6751
+ if (!(Array.isArray(updated) && updated.length === 0)) return;
6752
+ this.logger.warn({
6753
+ queryHash,
6754
+ id: String(queryState.config.id),
6755
+ Category: "sp00ky-client::Sp00kySync::heartbeatQuery"
6756
+ }, "Query row was reclaimed while still in use; re-registering");
6757
+ this.enqueueDownEvent({
6758
+ type: "register",
6759
+ payload: { hash: queryHash }
6760
+ });
6681
6761
  }
6682
6762
  async cleanupQuery(queryHash) {
6683
6763
  const queryState = this.dataModule.getQueryByHash(queryHash);
@@ -7012,8 +7092,8 @@ function selfAllowlistedVariant(flag, userId) {
7012
7092
 
7013
7093
  //#endregion
7014
7094
  //#region src/modules/devtools/index.ts
7015
- const CORE_VERSION = "0.0.1-canary.196";
7016
- const WASM_VERSION = "0.0.1-canary.196";
7095
+ const CORE_VERSION = "0.0.1-canary.198";
7096
+ const WASM_VERSION = "0.0.1-canary.198";
7017
7097
  const SURREAL_VERSION = "3.0.3";
7018
7098
  var DevToolsService = class DevToolsService {
7019
7099
  eventsHistory = [];
@@ -11444,7 +11524,7 @@ var Sp00kyClient = class {
11444
11524
  return new TabsCoordinator({
11445
11525
  tabId,
11446
11526
  fingerprint: computeTabsFingerprint({
11447
- coreVersion: "0.0.1-canary.196",
11527
+ coreVersion: "0.0.1-canary.198",
11448
11528
  schemaHash: hash53(this.config.schemaSurql),
11449
11529
  endpoint: this.config.database.endpoint ?? "",
11450
11530
  namespace: this.config.database.namespace,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@spooky-sync/core",
3
- "version": "0.0.1-canary.196",
3
+ "version": "0.0.1-canary.198",
4
4
  "type": "module",
5
5
  "sideEffects": false,
6
6
  "main": "./dist/index.js",
@@ -60,8 +60,8 @@
60
60
  }
61
61
  },
62
62
  "dependencies": {
63
- "@spooky-sync/query-builder": "0.0.1-canary.196",
64
- "@spooky-sync/ssp-wasm": "0.0.1-canary.196",
63
+ "@spooky-sync/query-builder": "0.0.1-canary.198",
64
+ "@spooky-sync/ssp-wasm": "0.0.1-canary.198",
65
65
  "@sqlite.org/sqlite-wasm": "3.53.0-build1",
66
66
  "@surrealdb/wasm": "^3.0.3",
67
67
  "fast-json-patch": "^3.1.1",
@@ -0,0 +1,206 @@
1
+ import { describe, it, expect, vi, afterEach } from 'vitest';
2
+ import { RecordId } from 'surrealdb';
3
+ import { DataModule } from './index';
4
+ import type { QueryPlan } from '@spooky-sync/query-builder';
5
+ import type { QueryState, RecordVersionArray } from '../../types';
6
+
7
+ /**
8
+ * A write is visible before the server confirms it because it sits in the
9
+ * outbox: the render set is
10
+ * `(membership ∪ (pendingWrites ∩ localArray)) − pendingDeletes`.
11
+ *
12
+ * The outbox row is deleted the instant the push succeeds, but the row does not
13
+ * enter membership until the SSP has ingested it, materialized the view, written
14
+ * the `_00_list_ref` edge, and this client has read that back. In between it is
15
+ * in NEITHER term, so it renders, disappears, and returns.
16
+ *
17
+ * Reported from production as "the comment shows up on every other client in
18
+ * realtime but not on the one that wrote it" — other clients don't blink because
19
+ * a client that never established membership renders from the predicate scan
20
+ * instead. The gap is invisible when the round trip is fast and glaring when the
21
+ * edge path is backed up.
22
+ */
23
+
24
+ const noop = () => {};
25
+
26
+ function makeLogger(): any {
27
+ const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
28
+ logger.child = () => logger;
29
+ return logger;
30
+ }
31
+
32
+ const schema = { tables: [{ name: 'comment', columns: {} }] } as any;
33
+ const plan: QueryPlan = { table: 'comment', where: [['game', '=', 'game:g1']] } as any;
34
+
35
+ function makeQueryState(
36
+ hash: string,
37
+ opts: { remoteArray?: RecordVersionArray; localArray?: RecordVersionArray } = {}
38
+ ): QueryState {
39
+ return {
40
+ config: {
41
+ id: new RecordId('_00_query', hash),
42
+ surql: 'SELECT * FROM comment WHERE game = $game;',
43
+ plan,
44
+ params: { game: 'game:g1' },
45
+ localArray: opts.localArray ?? [],
46
+ remoteArray: opts.remoteArray ?? [],
47
+ membershipKnown: true,
48
+ ttl: '10m',
49
+ lastActiveAt: new Date(),
50
+ tableName: 'comment',
51
+ },
52
+ records: [],
53
+ ttlTimer: null,
54
+ ttlDurationMs: 0,
55
+ updateCount: 0,
56
+ lastUpdatedAt: null,
57
+ materializationSamples: [],
58
+ lastIngestLatencyMs: null,
59
+ errorCount: 0,
60
+ status: 'idle',
61
+ phaseSamples: {},
62
+ phaseLast: {},
63
+ registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
64
+ } as QueryState;
65
+ }
66
+
67
+ function setup(
68
+ stateOpts: Parameters<typeof makeQueryState>[1] = {},
69
+ pendingRows: Array<{ recordId: RecordId; mutationType: string }> = []
70
+ ) {
71
+ const bodies = new Map(
72
+ ['old', 'new'].map((k) => [`comment:${k}`, { id: new RecordId('comment', k), body: k }])
73
+ );
74
+ const local: any = {
75
+ epoch: 1,
76
+ select: vi.fn(async (p: QueryPlan) => {
77
+ if ((p as any).ids) {
78
+ return ((p as any).ids as RecordId[])
79
+ .map((id) => bodies.get(id.toString()))
80
+ .filter((r): r is NonNullable<typeof r> => r !== undefined);
81
+ }
82
+ return Array.from(bodies.values());
83
+ }),
84
+ query: vi.fn(async (sql: string) => {
85
+ if (sql.includes('_00_pending_mutations')) return [pendingRows];
86
+ if (sql.includes('FROM ONLY')) return [null];
87
+ return [Array.from(bodies.values())];
88
+ }),
89
+ getById: vi.fn(async () => null),
90
+ upsert: vi.fn(async () => {}),
91
+ };
92
+ const dm = new DataModule({ saveBatch: async () => {} } as any, local, schema, makeLogger(), 100);
93
+ const state = makeQueryState('h1', stateOpts);
94
+ (dm as any).activeQueries.set('h1', state);
95
+ return { dm, state };
96
+ }
97
+
98
+ const ids = (rows: Array<Record<string, any>>) => rows.map((r) => String(r.id));
99
+ const materialize = (dm: DataModule<any>, state: QueryState, ssp?: RecordVersionArray) =>
100
+ (dm as any).materializeRecords(state, ssp) as Promise<Record<string, any>[]>;
101
+
102
+ describe('settled-write grace window', () => {
103
+ afterEach(() => vi.useRealTimers());
104
+
105
+ it('keeps an accepted write rendered while membership has not caught up', async () => {
106
+ // Membership still only knows the old comment; the outbox row for the new
107
+ // one is already gone because the push succeeded.
108
+ const { dm, state } = setup({
109
+ remoteArray: [['comment:old', 1]],
110
+ localArray: [
111
+ ['comment:old', 1],
112
+ ['comment:new', 1],
113
+ ],
114
+ });
115
+
116
+ dm.noteWriteSettled('comment:new', 'create');
117
+
118
+ expect(ids(await materialize(dm, state))).toEqual(['comment:new', 'comment:old']);
119
+ });
120
+
121
+ it('renders nothing extra for a write the server never accepted', async () => {
122
+ // The rollback path never reports a mutation settled, so a rejected write
123
+ // has no grace at all. This is what keeps the window from showing rows the
124
+ // server refused.
125
+ const { dm, state } = setup({
126
+ remoteArray: [['comment:old', 1]],
127
+ localArray: [
128
+ ['comment:old', 1],
129
+ ['comment:new', 1],
130
+ ],
131
+ });
132
+
133
+ expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
134
+ });
135
+
136
+ it('stops granting grace once membership names the row', async () => {
137
+ const { dm, state } = setup({
138
+ remoteArray: [['comment:old', 1]],
139
+ localArray: [
140
+ ['comment:old', 1],
141
+ ['comment:new', 1],
142
+ ],
143
+ });
144
+ dm.noteWriteSettled('comment:new', 'create');
145
+ await materialize(dm, state);
146
+
147
+ // Membership catches up; the id must not be double-counted…
148
+ state.config.remoteArray = [
149
+ ['comment:old', 1],
150
+ ['comment:new', 1],
151
+ ];
152
+ expect(ids(await materialize(dm, state))).toEqual(['comment:new', 'comment:old']);
153
+ expect((dm as any).settledWrites.size).toBe(0);
154
+
155
+ // …and once the server later drops it from the window, it goes away rather
156
+ // than being resurrected by a stale grace entry.
157
+ state.config.remoteArray = [['comment:old', 1]];
158
+ expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
159
+ });
160
+
161
+ it('expires the grace so a write whose membership never arrives cannot linger', async () => {
162
+ vi.useFakeTimers();
163
+ const { dm, state } = setup({
164
+ remoteArray: [['comment:old', 1]],
165
+ localArray: [
166
+ ['comment:old', 1],
167
+ ['comment:new', 1],
168
+ ],
169
+ });
170
+ dm.noteWriteSettled('comment:new', 'create');
171
+ expect(ids(await materialize(dm, state))).toContain('comment:new');
172
+
173
+ vi.advanceTimersByTime(11_000);
174
+ expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
175
+ });
176
+
177
+ it('keeps an accepted delete subtracted until membership drops the row', async () => {
178
+ // The mirror case: membership still lists the row until the SSP publishes
179
+ // its removal, so without this the deleted row flashes back.
180
+ const { dm, state } = setup({
181
+ remoteArray: [
182
+ ['comment:old', 1],
183
+ ['comment:new', 1],
184
+ ],
185
+ localArray: [['comment:old', 1]],
186
+ });
187
+
188
+ dm.noteWriteSettled('comment:new', 'delete');
189
+
190
+ expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
191
+ });
192
+
193
+ it('only grants grace to rows the local view still says match', async () => {
194
+ // `localArray` is the local SSP's answer to "does this row match the
195
+ // predicate". A settled write that does NOT match must not be forced into
196
+ // the render set — same rule the pending-write union already follows.
197
+ const { dm, state } = setup({
198
+ remoteArray: [['comment:old', 1]],
199
+ localArray: [['comment:old', 1]],
200
+ });
201
+
202
+ dm.noteWriteSettled('comment:new', 'create');
203
+
204
+ expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
205
+ });
206
+ });
@@ -565,6 +565,62 @@ export class DataModule<S extends SchemaStructure> {
565
565
  return null;
566
566
  }
567
567
 
568
+ // ---- Settled-write grace window ----------------------------------------
569
+ //
570
+ // A local write is visible because it is in the outbox: the render set is
571
+ // `(membership ∪ (pendingWrites ∩ localArray)) − pendingDeletes`. The outbox
572
+ // row is deleted the moment the remote push succeeds (see
573
+ // `UpQueue.next` / `SyncScheduler` — the delete is deliberately tied to the
574
+ // push, not to anything downstream), but the row does not enter `membership`
575
+ // until the SSP has ingested it, materialized the view and written the
576
+ // `_00_list_ref` edge, and this client has read that back.
577
+ //
578
+ // Between those two moments the row is in neither term, so it is rendered,
579
+ // then disappears, then returns — reported as "the comment shows on every
580
+ // other client but not on the one that wrote it". Other clients don't blink
581
+ // because a client that never established membership renders from the local
582
+ // predicate scan instead.
583
+ //
584
+ // So a settled write keeps its place in the union for a short grace period,
585
+ // until membership catches up or the deadline passes. Kept in memory only:
586
+ // it covers a round trip, and a reload re-derives membership anyway.
587
+ private readonly settledWrites = new Map<string, number>();
588
+ private readonly settledDeletes = new Map<string, number>();
589
+
590
+ /**
591
+ * Grace period for a settled write. Long enough to cover an SSP round trip
592
+ * that is running slowly (seconds, not milliseconds, when the edge path is
593
+ * backed up), short enough that a write the server silently dropped cannot
594
+ * linger misleadingly.
595
+ *
596
+ * The rejection case does NOT rely on this expiring: an application error
597
+ * rolls the mutation back and never reports it settled, so it vanishes at
598
+ * once. This deadline only bounds the case where the write succeeded and its
599
+ * membership never arrived at all.
600
+ */
601
+ private static readonly SETTLED_WRITE_GRACE_MS = 10_000;
602
+
603
+ /**
604
+ * Report that a mutation was accepted by the server and its outbox row
605
+ * removed. Called only on the SUCCESS path — a rolled-back mutation must
606
+ * disappear immediately, which is what makes this safe.
607
+ */
608
+ noteWriteSettled(recordId: string, mutationType: string): void {
609
+ const until = Date.now() + DataModule.SETTLED_WRITE_GRACE_MS;
610
+ if (mutationType === 'delete') this.settledDeletes.set(recordId, until);
611
+ else this.settledWrites.set(recordId, until);
612
+ }
613
+
614
+ /** Drop entries past their deadline. */
615
+ private pruneSettled(now: number): void {
616
+ for (const [id, until] of this.settledWrites) {
617
+ if (until <= now) this.settledWrites.delete(id);
618
+ }
619
+ for (const [id, until] of this.settledDeletes) {
620
+ if (until <= now) this.settledDeletes.delete(id);
621
+ }
622
+ }
623
+
568
624
  /** Apply the pending-write union and pending-delete subtraction, and map to
569
625
  * RecordIds for the engines' id-set path. */
570
626
  private async buildRenderIds(
@@ -573,13 +629,39 @@ export class DataModule<S extends SchemaStructure> {
573
629
  sspArray?: Array<[string, number]>
574
630
  ): Promise<unknown[]> {
575
631
  const { writes, deletes } = await this.getPendingRecordIds();
632
+ const now = Date.now();
633
+ this.pruneSettled(now);
634
+ // A settled write counts as pending until membership catches up. Merged
635
+ // into the same sets so the union/subtraction below is unchanged — the
636
+ // grace window changes WHEN an id leaves the render set, never how the
637
+ // set is composed.
638
+ if (this.settledWrites.size > 0) {
639
+ for (const id of this.settledWrites.keys()) writes.add(id);
640
+ }
641
+ if (this.settledDeletes.size > 0) {
642
+ for (const id of this.settledDeletes.keys()) deletes.add(id);
643
+ }
644
+
576
645
  const ordered: string[] = [];
577
646
  const seen = new Set<string>();
578
647
  for (const [id] of membership) {
648
+ // Membership has caught up with this write: the grace window has done
649
+ // its job and ends here rather than at its deadline. Doing it inline
650
+ // keeps the common case (nothing settled) free of extra passes.
651
+ if (this.settledWrites.size > 0) this.settledWrites.delete(id);
579
652
  if (deletes.has(id) || seen.has(id)) continue;
580
653
  seen.add(id);
581
654
  ordered.push(id);
582
655
  }
656
+ // A settled DELETE is the mirror case: membership still lists the row
657
+ // until the SSP publishes its removal, so the id stays subtracted until
658
+ // membership stops naming it.
659
+ if (this.settledDeletes.size > 0) {
660
+ const stillListed = new Set(membership.map(([id]) => id));
661
+ for (const id of [...this.settledDeletes.keys()]) {
662
+ if (!stillListed.has(id)) this.settledDeletes.delete(id);
663
+ }
664
+ }
583
665
  if (writes.size > 0) {
584
666
  // Only pending writes the SSP agrees currently match this query — see the
585
667
  // formula in `materializeRecords`. `sspArray` is the fresher signal when a
@@ -175,7 +175,18 @@ export class UpQueue {
175
175
  this.debouncedMutations.clear();
176
176
  }
177
177
 
178
- async next(fn: (event: UpEvent) => Promise<void>, onRollback?: RollbackCallback): Promise<void> {
178
+ /**
179
+ * @param onSettled Reports a mutation the server ACCEPTED, after its outbox
180
+ * row is gone. Deliberately not called on the rollback path: a rejected
181
+ * mutation must stop being rendered immediately, while an accepted one has
182
+ * to stay visible until its membership arrives (see
183
+ * `DataModule.noteWriteSettled`).
184
+ */
185
+ async next(
186
+ fn: (event: UpEvent) => Promise<void>,
187
+ onRollback?: RollbackCallback,
188
+ onSettled?: (event: UpEvent) => void
189
+ ): Promise<void> {
179
190
  const event = this.queue.shift();
180
191
  if (event) {
181
192
  try {
@@ -229,6 +240,19 @@ export class UpQueue {
229
240
  'Failed to remove mutation from database after successful processing'
230
241
  );
231
242
  }
243
+ // Report AFTER the outbox row is gone: that delete is exactly what drops
244
+ // the row out of the render set, so this is the moment the grace window
245
+ // has to start covering.
246
+ if (onSettled) {
247
+ try {
248
+ onSettled(event);
249
+ } catch (error) {
250
+ this.logger.error(
251
+ { error, event, Category: 'sp00ky-client::UpQueue::next' },
252
+ 'Settled-write handler failed'
253
+ );
254
+ }
255
+ }
232
256
  this._events.addEvent({
233
257
  type: SyncQueueEventTypes.MutationDequeued,
234
258
  payload: { queueSize: this.queue.length },
@@ -37,7 +37,10 @@ export class SyncScheduler {
37
37
  // that actually processed ≥1 item): `ok=true` on a clean drain, `ok=false`
38
38
  // with the error when the round halted on a failure. Drives the consumer's
39
39
  // sync-health tracking; empty/no-op rounds report nothing.
40
- private onSyncOutcome?: (ok: boolean, error?: unknown) => void
40
+ private onSyncOutcome?: (ok: boolean, error?: unknown) => void,
41
+ // Reports each mutation the server accepted, once its outbox row is gone.
42
+ // Lets the consumer keep the row rendered until its membership arrives.
43
+ private onSettled?: (event: UpEvent) => void
41
44
  ) {}
42
45
 
43
46
  async init(opts: { loadOutbox?: boolean } = {}) {
@@ -151,7 +154,7 @@ export class SyncScheduler {
151
154
  let processedAny = false;
152
155
  try {
153
156
  while (this.upQueue.size > 0 && !this.paused) {
154
- await this.upQueue.next(this.onProcessUp, this.onRollback);
157
+ await this.upQueue.next(this.onProcessUp, this.onRollback, this.onSettled);
155
158
  processedAny = true;
156
159
  }
157
160
  if (processedAny) this.onSyncOutcome?.(true);
@@ -0,0 +1,80 @@
1
+ import { describe, it, expect, vi, beforeEach } from 'vitest';
2
+ import { RecordId } from 'surrealdb';
3
+ import { Sp00kySync } from './sync';
4
+
5
+ /**
6
+ * `fn::query::heartbeat` is an `UPDATE $id SET ...`. Against a record that no
7
+ * longer exists it matches nothing and returns an empty array — it does NOT
8
+ * recreate the row. Verified against the deployed function:
9
+ *
10
+ * RETURN fn::query::heartbeat(_00_query:definitely_not_a_real_row_xyz);
11
+ * -- (0 rows)
12
+ *
13
+ * So an unchecked heartbeat cannot tell "refreshed" from "the row I am
14
+ * refreshing is gone", and a client whose row was reclaimed by the TTL sweep
15
+ * beats against nothing forever: no membership, no edges, no re-registration.
16
+ * The page then renders as though the data had been deleted — reported as
17
+ * "Game not found" on a game that was open and working.
18
+ *
19
+ * This is reachable in ordinary use: the sweep expires on `lastActiveAt + ttl`
20
+ * while the heartbeat runs on a timer browsers throttle hard in background
21
+ * tabs, so a second window left idle past its TTL is the normal way in.
22
+ */
23
+
24
+ function makeSync(heartbeatResult: unknown) {
25
+ const logger: any = {
26
+ child: () => logger,
27
+ debug: () => {}, info: () => {}, warn: () => {}, error: () => {}, trace: () => {},
28
+ };
29
+ const remote: any = { query: vi.fn().mockResolvedValue(heartbeatResult) };
30
+ const queryState: any = { config: { id: new RecordId('_00_query', 'h1') } };
31
+ const dataModule: any = { getQueryByHash: vi.fn().mockReturnValue(queryState) };
32
+
33
+ const sync = new Sp00kySync({} as any, remote, {} as any, dataModule, {} as any, logger);
34
+ const enqueueDownEvent = vi.fn();
35
+ (sync as any).enqueueDownEvent = enqueueDownEvent;
36
+
37
+ return { sync, remote, dataModule, enqueueDownEvent };
38
+ }
39
+
40
+ describe('heartbeatQuery — noticing a reclaimed row', () => {
41
+ beforeEach(() => vi.clearAllMocks());
42
+
43
+ it('re-registers when the row it beat against is gone', async () => {
44
+ // `UPDATE` on a deleted record: one statement, zero updated records.
45
+ const { sync, enqueueDownEvent } = makeSync([[]]);
46
+
47
+ await sync.heartbeatQuery('h1');
48
+
49
+ expect(enqueueDownEvent).toHaveBeenCalledWith({
50
+ type: 'register',
51
+ payload: { hash: 'h1' },
52
+ });
53
+ });
54
+
55
+ it('does nothing extra on a healthy heartbeat', async () => {
56
+ const { sync, enqueueDownEvent } = makeSync([[{ id: 'x', lastActiveAt: 'now' }]]);
57
+
58
+ await sync.heartbeatQuery('h1');
59
+
60
+ expect(enqueueDownEvent).not.toHaveBeenCalled();
61
+ });
62
+
63
+ it('does not re-register on an unrecognised result shape', async () => {
64
+ // Only an explicitly EMPTY update result means "the row is gone". Anything
65
+ // else — a driver returning null, a shape change — must not be read as
66
+ // deletion, or every heartbeat would re-register the whole working set.
67
+ for (const shape of [null, undefined, [], [null], ['unexpected']]) {
68
+ const { sync, enqueueDownEvent } = makeSync(shape);
69
+ await sync.heartbeatQuery('h1');
70
+ expect(enqueueDownEvent, `shape ${JSON.stringify(shape)}`).not.toHaveBeenCalled();
71
+ }
72
+ });
73
+
74
+ it('still throws for a query that is no longer registered locally', async () => {
75
+ const { sync, dataModule } = makeSync([[]]);
76
+ dataModule.getQueryByHash.mockReturnValue(undefined);
77
+
78
+ await expect(sync.heartbeatQuery('gone')).rejects.toThrow();
79
+ });
80
+ });
@@ -454,7 +454,8 @@ export class Sp00kySync<S extends SchemaStructure> {
454
454
  this.processDownEvent.bind(this),
455
455
  this.logger,
456
456
  this.handleRollback.bind(this),
457
- this.recordSyncOutcome.bind(this)
457
+ this.recordSyncOutcome.bind(this),
458
+ this.handleMutationSettled.bind(this)
458
459
  );
459
460
  this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
460
461
  this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
@@ -1362,6 +1363,21 @@ export class Sp00kySync<S extends SchemaStructure> {
1362
1363
  }
1363
1364
  }
1364
1365
 
1366
+ /**
1367
+ * A mutation the server accepted, reported once its outbox row is gone.
1368
+ *
1369
+ * Keeps the written row in the render set until its membership arrives.
1370
+ * Without this the row is briefly in neither term of
1371
+ * `(membership ∪ pendingWrites) − pendingDeletes` — the outbox delete is
1372
+ * tied to the push, while membership waits on the SSP ingesting the row,
1373
+ * materializing the view, writing the `_00_list_ref` edge and this client
1374
+ * reading it back. The writer therefore watched its own comment appear,
1375
+ * vanish, and return, while every other client showed it throughout.
1376
+ */
1377
+ private handleMutationSettled(event: UpEvent): void {
1378
+ this.dataModule.noteWriteSettled(encodeRecordId(event.record_id), event.type);
1379
+ }
1380
+
1365
1381
  private async handleRollback(event: UpEvent, error: Error): Promise<void> {
1366
1382
  const recordId = encodeRecordId(event.record_id);
1367
1383
  const tableName =
@@ -1752,9 +1768,40 @@ export class Sp00kySync<S extends SchemaStructure> {
1752
1768
  );
1753
1769
  throw new Error('Query to register not found');
1754
1770
  }
1755
- await this.remote.query('fn::query::heartbeat($id)', {
1771
+ // `fn::query::heartbeat` is an `UPDATE $id SET ...`. On a record that no
1772
+ // longer exists that matches nothing and returns an empty array — it does
1773
+ // NOT recreate the row. So an unchecked heartbeat is indistinguishable from
1774
+ // a successful one, and a client whose row was reclaimed keeps beating
1775
+ // against nothing forever: no membership, no edges, no re-registration.
1776
+ // The page renders as if the data were deleted ("Game not found").
1777
+ //
1778
+ // A live query's row is reclaimed more easily than it looks. The sweep
1779
+ // expires on `lastActiveAt + ttl`, and this heartbeat runs on a timer that
1780
+ // browsers throttle hard in background tabs — so a second window left idle
1781
+ // past its TTL is the ordinary way to get here, not an edge case. Until
1782
+ // canary.194 the sweep could not actually remove the in-memory view (it
1783
+ // looked it up under the other of the two query-id spellings), which masked
1784
+ // this: the view survived its own row. Now reclamation is real, so the
1785
+ // client has to notice and rebuild.
1786
+ const result = await this.remote.query('fn::query::heartbeat($id)', {
1756
1787
  id: queryState.config.id,
1757
1788
  });
1789
+ const updated = Array.isArray(result) ? result[0] : undefined;
1790
+ const rowGone = Array.isArray(updated) && updated.length === 0;
1791
+ if (!rowGone) return;
1792
+
1793
+ this.logger.warn(
1794
+ {
1795
+ queryHash,
1796
+ id: String(queryState.config.id),
1797
+ Category: 'sp00ky-client::Sp00kySync::heartbeatQuery',
1798
+ },
1799
+ 'Query row was reclaimed while still in use; re-registering'
1800
+ );
1801
+ // Re-register rather than recreate the row here: the row alone is useless
1802
+ // without the SSP view behind it, and only registration rebuilds the view,
1803
+ // republishes `_00_list_ref` and writes `rowCount`.
1804
+ this.enqueueDownEvent({ type: 'register', payload: { hash: queryHash } });
1758
1805
  }
1759
1806
 
1760
1807
  // Eager teardown of a deregistered query's remote `_00_query` view (opt-in,