@spooky-sync/core 0.0.1-canary.208 → 0.0.1-canary.209
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +69 -6
- package/dist/index.js +206 -57
- package/dist/sqlite-worker.js +22 -7
- package/dist/types.d.ts +10 -0
- package/package.json +3 -3
- package/src/index.ts +1 -0
- package/src/modules/data/data.pending-ids.test.ts +74 -0
- package/src/modules/data/data.rematerialize.test.ts +114 -0
- package/src/modules/data/index.ts +151 -61
- package/src/modules/devtools/index.ts +65 -18
- package/src/modules/devtools/notify-throttle.test.ts +6 -1
- package/src/modules/devtools/state-shape.test.ts +146 -0
- package/src/modules/sync/sync.live-removal.test.ts +1 -0
- package/src/modules/sync/sync.ts +10 -0
- package/src/services/database/database.ts +14 -5
- package/src/services/database/errors.ts +34 -0
- package/src/services/database/local.ts +6 -0
- package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
- package/src/services/database/sqlite-cache-engine.ts +27 -1
- package/src/services/database/sqlite-lock-verify.test.ts +33 -0
- package/src/services/database/sqlite-lock-verify.ts +45 -0
- package/src/services/database/sqlite-transport.ts +5 -2
- package/src/services/database/sqlite-worker.ts +11 -11
- package/src/services/stream-processor/index.ts +8 -0
- package/src/types.ts +10 -0
- package/src/utils/index.ts +8 -2
|
@@ -63,6 +63,10 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
63
63
|
private static readonly NOTIFY_MIN_INTERVAL_MS = 250;
|
|
64
64
|
private notifyTimer: ReturnType<typeof setTimeout> | null = null;
|
|
65
65
|
private lastNotifyAt = 0;
|
|
66
|
+
/** How many ids a pushed state carries per view; the rest is on demand. */
|
|
67
|
+
private static readonly STATE_IDS_CAP = 200;
|
|
68
|
+
/** devtools numeric hash -> the query's `_00_query` id, for on-demand rows. */
|
|
69
|
+
private hashToQuery = new Map<number, unknown>();
|
|
66
70
|
|
|
67
71
|
/** Shared-tabs snapshot for the panel, wired by Sp00kyClient whenever the
|
|
68
72
|
* feature was REQUESTED (so an inactive/degraded tab still reports why). */
|
|
@@ -123,6 +127,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
123
127
|
const type = (e.data as { type?: string } | undefined)?.type;
|
|
124
128
|
if (type === 'SP00KY_DEVTOOLS_CONNECT') {
|
|
125
129
|
this.enabled = true;
|
|
130
|
+
this.refreshLocalTables();
|
|
126
131
|
this.notifyDevTools();
|
|
127
132
|
} else if (type === 'SP00KY_DEVTOOLS_DISCONNECT') {
|
|
128
133
|
this.enabled = false;
|
|
@@ -189,6 +194,9 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
189
194
|
q.config.lastActiveAt instanceof Date
|
|
190
195
|
? q.config.lastActiveAt.getTime()
|
|
191
196
|
: new Date(q.config.lastActiveAt || Date.now()).getTime();
|
|
197
|
+
this.hashToQuery.set(queryHash, q.config.id);
|
|
198
|
+
const localArray = q.config.localArray ?? [];
|
|
199
|
+
const remoteArray = q.config.remoteArray ?? [];
|
|
192
200
|
result.set(queryHash, {
|
|
193
201
|
queryHash,
|
|
194
202
|
status: 'active',
|
|
@@ -205,9 +213,18 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
205
213
|
query: q.config.surql,
|
|
206
214
|
variables: q.config.params || {},
|
|
207
215
|
dataSize: q.records?.length || 0,
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
216
|
+
// Counts and (capped) ids only. The rows themselves used to ride along
|
|
217
|
+
// here: every push then deep-cloned every view's records twice
|
|
218
|
+
// (serialize + postMessage), on a 4k-row client dataset up to four
|
|
219
|
+
// times a second, from inside the ingest call stack. The panel pulls
|
|
220
|
+
// rows on demand through `getQueryRows` instead.
|
|
221
|
+
localCount: localArray.length,
|
|
222
|
+
remoteCount: remoteArray.length,
|
|
223
|
+
localIds: localArray.slice(0, DevToolsService.STATE_IDS_CAP).map(([id]) => id),
|
|
224
|
+
remoteIds: remoteArray.slice(0, DevToolsService.STATE_IDS_CAP).map(([id]) => id),
|
|
225
|
+
idsTruncated:
|
|
226
|
+
localArray.length > DevToolsService.STATE_IDS_CAP ||
|
|
227
|
+
remoteArray.length > DevToolsService.STATE_IDS_CAP,
|
|
211
228
|
// Membership state, so "why is this list empty" is answerable from
|
|
212
229
|
// the panel: is the server's set known, has a non-empty one been seen
|
|
213
230
|
// this session, how many empty reads were ignored.
|
|
@@ -250,18 +267,28 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
250
267
|
|
|
251
268
|
this.addEvent('QUERY_UPDATED', {
|
|
252
269
|
queryHash,
|
|
253
|
-
|
|
270
|
+
recordCount: Array.isArray(payload.records) ? payload.records.length : 0,
|
|
254
271
|
});
|
|
255
272
|
this.notifyDevTools();
|
|
256
273
|
}
|
|
257
274
|
|
|
258
275
|
public onStreamUpdate(update: StreamUpdate) {
|
|
276
|
+
// A synthetic re-materialize is not an ingest (DataModule.scheduleRematerialize).
|
|
277
|
+
if (update.synthetic) return;
|
|
259
278
|
this.logger.debug(
|
|
260
|
-
{ update, Category: 'sp00ky-client::DevToolsService::onStreamUpdate' },
|
|
279
|
+
{ queryHash: update.queryHash, Category: 'sp00ky-client::DevToolsService::onStreamUpdate' },
|
|
261
280
|
'StreamUpdate'
|
|
262
281
|
);
|
|
282
|
+
// Counts and timings, not the `localArray` itself: that is one [id, version]
|
|
283
|
+
// pair per row of the view, serialized on every single update.
|
|
263
284
|
this.addEvent('STREAM_UPDATE', {
|
|
264
|
-
|
|
285
|
+
queryHash: update.queryHash,
|
|
286
|
+
op: update.op,
|
|
287
|
+
localCount: update.localArray?.length ?? 0,
|
|
288
|
+
materializationTimeMs: update.materializationTimeMs,
|
|
289
|
+
storeApplyMs: update.storeApplyMs,
|
|
290
|
+
circuitStepMs: update.circuitStepMs,
|
|
291
|
+
transformMs: update.transformMs,
|
|
265
292
|
});
|
|
266
293
|
this.notifyDevTools();
|
|
267
294
|
}
|
|
@@ -271,8 +298,10 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
271
298
|
payloads.forEach((p) => {
|
|
272
299
|
this.addEvent('MUTATION_REQUEST_EXECUTION', {
|
|
273
300
|
mutation: {
|
|
274
|
-
type: 'create',
|
|
275
|
-
|
|
301
|
+
type: p.type ?? 'create',
|
|
302
|
+
// Field names only; a payload body (a PGN, a document) is not
|
|
303
|
+
// something to clone on every write.
|
|
304
|
+
fields: 'data' in p && p.data && typeof p.data === 'object' ? Object.keys(p.data) : [],
|
|
276
305
|
selector: encodeRecordId(p.record_id),
|
|
277
306
|
},
|
|
278
307
|
});
|
|
@@ -325,7 +354,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
325
354
|
private refreshLocalTables(): void {
|
|
326
355
|
if (this.localTablesFetching) return;
|
|
327
356
|
const now = Date.now();
|
|
328
|
-
if (now - this.localTablesAt <
|
|
357
|
+
if (now - this.localTablesAt < 30_000) return;
|
|
329
358
|
this.localTablesFetching = true;
|
|
330
359
|
void this.databaseService
|
|
331
360
|
.query<any>('INFO FOR DB')
|
|
@@ -351,9 +380,12 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
351
380
|
});
|
|
352
381
|
}
|
|
353
382
|
|
|
354
|
-
private getState() {
|
|
355
|
-
//
|
|
356
|
-
|
|
383
|
+
private getState(opts: { refreshTables?: boolean } = {}) {
|
|
384
|
+
// The local-table list (`INFO FOR DB`, a local round trip) is refreshed on
|
|
385
|
+
// an explicit pull only, never by the push path: pushes follow every sync
|
|
386
|
+
// event, and a local query per push competed with the app's own writes for
|
|
387
|
+
// the single local op queue.
|
|
388
|
+
if (opts.refreshTables) this.refreshLocalTables();
|
|
357
389
|
return this.serializeForDevTools({
|
|
358
390
|
eventsHistory: [...this.eventsHistory],
|
|
359
391
|
activeQueries: Object.fromEntries(this.getActiveQueries()),
|
|
@@ -481,16 +513,17 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
481
513
|
// A trailing push is already queued; it will carry this change too.
|
|
482
514
|
if (this.notifyTimer !== null) return;
|
|
483
515
|
|
|
516
|
+
// Always a macrotask, never inline: this is called from inside the ingest
|
|
517
|
+
// and mutation call stacks (onStreamUpdate / onMutation), i.e. inside the
|
|
518
|
+
// `await db.create(...)` the app is waiting on. A push that serializes the
|
|
519
|
+
// state right there charged every write for the panel's refresh.
|
|
484
520
|
const waited = Date.now() - this.lastNotifyAt;
|
|
485
|
-
|
|
486
|
-
this.flushNotify();
|
|
487
|
-
return;
|
|
488
|
-
}
|
|
521
|
+
const delay = Math.max(0, DevToolsService.NOTIFY_MIN_INTERVAL_MS - waited);
|
|
489
522
|
this.notifyTimer = setTimeout(() => {
|
|
490
523
|
this.notifyTimer = null;
|
|
491
524
|
// Still gated on `enabled`: the panel may have disconnected while queued.
|
|
492
525
|
if (this.enabled) this.flushNotify();
|
|
493
|
-
},
|
|
526
|
+
}, delay);
|
|
494
527
|
}
|
|
495
528
|
|
|
496
529
|
private flushNotify() {
|
|
@@ -569,7 +602,21 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
569
602
|
if (typeof window !== 'undefined') {
|
|
570
603
|
(window as any).__00__ = {
|
|
571
604
|
version: this.version,
|
|
572
|
-
getState: () => this.getState(),
|
|
605
|
+
getState: () => this.getState({ refreshTables: true }),
|
|
606
|
+
// The rows of ONE view, on demand. The pushed state carries counts and
|
|
607
|
+
// capped ids only (see getActiveQueries); the panel's Data tab and the
|
|
608
|
+
// MCP fetch the rows here when somebody actually looks at them.
|
|
609
|
+
getQueryRows: (queryHash: number) => {
|
|
610
|
+
const id = this.hashToQuery.get(Number(queryHash));
|
|
611
|
+
const q = id !== undefined ? this.dataManager?.getQueryById(id as any) : undefined;
|
|
612
|
+
if (!q) return null;
|
|
613
|
+
return this.serializeForDevTools({
|
|
614
|
+
queryHash: Number(queryHash),
|
|
615
|
+
data: q.records,
|
|
616
|
+
localArray: q.config.localArray,
|
|
617
|
+
remoteArray: q.config.remoteArray,
|
|
618
|
+
});
|
|
619
|
+
},
|
|
573
620
|
// ---- Feature flags (Access tab) --------------------------------
|
|
574
621
|
// Remote reads/writes are admin-gated by SurrealDB, not here: a
|
|
575
622
|
// non-admin gets an empty flag list, and the `fn::feature::*` calls
|
|
@@ -74,9 +74,13 @@ afterEach(() => {
|
|
|
74
74
|
});
|
|
75
75
|
|
|
76
76
|
describe('DevToolsService state-push coalescing', () => {
|
|
77
|
-
it('pushes once
|
|
77
|
+
it('pushes once on connect, on the next macrotask', () => {
|
|
78
78
|
vi.useFakeTimers();
|
|
79
79
|
const { statePushes } = harness();
|
|
80
|
+
// Never inline: a push serializes the state, and the request can come
|
|
81
|
+
// from inside an ingest or a mutation the app is awaiting.
|
|
82
|
+
expect(statePushes().length).toBe(0);
|
|
83
|
+
vi.advanceTimersByTime(0);
|
|
80
84
|
expect(statePushes().length).toBe(1);
|
|
81
85
|
});
|
|
82
86
|
|
|
@@ -117,6 +121,7 @@ describe('DevToolsService state-push coalescing', () => {
|
|
|
117
121
|
// straight away, so the panel never waits on a quiet app.
|
|
118
122
|
vi.advanceTimersByTime(1000);
|
|
119
123
|
(service as any).logEvent('C', {});
|
|
124
|
+
vi.advanceTimersByTime(0);
|
|
120
125
|
expect(statePushes().length).toBe(before + 3);
|
|
121
126
|
});
|
|
122
127
|
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { describe, it, expect, vi, afterEach } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { DevToolsService } from './index';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* The pushed state used to carry every view's full record set (and both
|
|
7
|
+
* membership arrays), deep-cloned once by the serializer and again by
|
|
8
|
+
* postMessage, on every sync event, from inside the ingest call stack. On a
|
|
9
|
+
* client holding a few thousand rows that is what the main thread was doing
|
|
10
|
+
* instead of finishing the write the app was awaiting. Pushes now carry counts
|
|
11
|
+
* and capped ids; rows are pulled per view on demand.
|
|
12
|
+
*/
|
|
13
|
+
function harness(recordCount = 500) {
|
|
14
|
+
const posted: any[] = [];
|
|
15
|
+
const listeners: ((e: any) => void)[] = [];
|
|
16
|
+
const fakeWindow: any = {
|
|
17
|
+
postMessage: (msg: any) => posted.push(msg),
|
|
18
|
+
addEventListener: (_type: string, cb: (e: any) => void) => listeners.push(cb),
|
|
19
|
+
dispatchEvent: () => true,
|
|
20
|
+
};
|
|
21
|
+
fakeWindow.self = fakeWindow;
|
|
22
|
+
vi.stubGlobal('window', fakeWindow);
|
|
23
|
+
vi.stubGlobal('CustomEvent', class {
|
|
24
|
+
type: string;
|
|
25
|
+
constructor(type: string) {
|
|
26
|
+
this.type = type;
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
const noop = () => {};
|
|
30
|
+
const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
31
|
+
logger.child = () => logger;
|
|
32
|
+
const infoQueries: string[] = [];
|
|
33
|
+
const local: any = {
|
|
34
|
+
query: async (sql: string) => {
|
|
35
|
+
infoQueries.push(sql);
|
|
36
|
+
return [];
|
|
37
|
+
},
|
|
38
|
+
getConfig: () => ({ store: 'memory' }),
|
|
39
|
+
currentBucketId: 'anon',
|
|
40
|
+
storageHealth: { status: 'memory', fallback: false },
|
|
41
|
+
};
|
|
42
|
+
const remote: any = { query: async () => [] };
|
|
43
|
+
const auth: any = { isAuthenticated: false, currentUser: undefined, eventSystem: { subscribe: noop } };
|
|
44
|
+
const records = Array.from({ length: recordCount }, (_, i) => ({ id: `game:${i}`, pgn: 'x' }));
|
|
45
|
+
const localArray = records.map((r) => [r.id, 1] as [string, number]);
|
|
46
|
+
const id = new RecordId('_00_query', 'q1');
|
|
47
|
+
const query = {
|
|
48
|
+
config: { id, params: {}, localArray, remoteArray: localArray, surql: 'SELECT * FROM game' },
|
|
49
|
+
status: 'idle',
|
|
50
|
+
records,
|
|
51
|
+
updateCount: 1,
|
|
52
|
+
};
|
|
53
|
+
const dataManager: any = {
|
|
54
|
+
getActiveQueries: () => [query],
|
|
55
|
+
getQueryById: (rid: RecordId<string>) => (String(rid) === String(id) ? query : undefined),
|
|
56
|
+
phaseTimings: () => ({}),
|
|
57
|
+
};
|
|
58
|
+
const service = new DevToolsService(local, remote, logger, { tables: [] } as any, auth, dataManager);
|
|
59
|
+
for (const cb of listeners) cb({ source: fakeWindow, data: { type: 'SP00KY_DEVTOOLS_CONNECT' } });
|
|
60
|
+
const statePushes = () => posted.filter((m) => m.type === 'SP00KY_STATE_CHANGED');
|
|
61
|
+
return { service, statePushes, posted, fakeWindow, infoQueries };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
describe('DevTools pushed state shape', () => {
|
|
65
|
+
afterEach(() => {
|
|
66
|
+
vi.unstubAllGlobals();
|
|
67
|
+
vi.useRealTimers();
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it('carries counts and capped ids, never the rows', async () => {
|
|
71
|
+
vi.useFakeTimers();
|
|
72
|
+
const { service, statePushes } = harness(500);
|
|
73
|
+
service.onQueryUpdated({ queryId: 'x', records: [] });
|
|
74
|
+
await vi.advanceTimersByTimeAsync(300);
|
|
75
|
+
const push = statePushes().at(-1);
|
|
76
|
+
expect(push).toBeDefined();
|
|
77
|
+
const q: any = Object.values(push.state.activeQueries)[0];
|
|
78
|
+
expect(q.data).toBeUndefined();
|
|
79
|
+
expect(q.localArray).toBeUndefined();
|
|
80
|
+
expect(q.remoteArray).toBeUndefined();
|
|
81
|
+
expect(q.dataSize).toBe(500);
|
|
82
|
+
expect(q.localCount).toBe(500);
|
|
83
|
+
expect(q.remoteCount).toBe(500);
|
|
84
|
+
expect(q.localIds).toHaveLength(200);
|
|
85
|
+
expect(q.idsTruncated).toBe(true);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('never pushes synchronously inside the call that requested it', async () => {
|
|
89
|
+
vi.useFakeTimers();
|
|
90
|
+
const { service, statePushes } = harness(5);
|
|
91
|
+
// Connect already queued one push; drain it so the next request is "idle".
|
|
92
|
+
await vi.advanceTimersByTimeAsync(300);
|
|
93
|
+
const before = statePushes().length;
|
|
94
|
+
service.onStreamUpdate({ queryHash: 'q1', localArray: [], op: 'CREATE' });
|
|
95
|
+
expect(statePushes().length).toBe(before);
|
|
96
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
97
|
+
expect(statePushes().length).toBe(before + 1);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('serves the rows of one view on demand', () => {
|
|
101
|
+
const { service, fakeWindow } = harness(3);
|
|
102
|
+
void service;
|
|
103
|
+
const state = fakeWindow.__00__.getState();
|
|
104
|
+
const hash = Number(Object.keys(state.activeQueries)[0]);
|
|
105
|
+
const rows = fakeWindow.__00__.getQueryRows(hash);
|
|
106
|
+
expect(rows.data).toHaveLength(3);
|
|
107
|
+
expect(rows.localArray).toHaveLength(3);
|
|
108
|
+
expect(fakeWindow.__00__.getQueryRows(12345)).toBeNull();
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('records a stream update as counts, not the membership array', async () => {
|
|
112
|
+
vi.useFakeTimers();
|
|
113
|
+
const { service, statePushes } = harness(2);
|
|
114
|
+
service.onStreamUpdate({ queryHash: 'q1', localArray: [['a', 1], ['b', 1]], op: 'UPDATE' });
|
|
115
|
+
await vi.advanceTimersByTimeAsync(300);
|
|
116
|
+
const push = statePushes().at(-1);
|
|
117
|
+
const ev = push.state.eventsHistory.find((e: any) => e.eventType === 'STREAM_UPDATE');
|
|
118
|
+
expect(ev.payload.localCount).toBe(2);
|
|
119
|
+
expect(ev.payload.updates).toBeUndefined();
|
|
120
|
+
expect(ev.payload.localArray).toBeUndefined();
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('ignores a synthetic re-materialize', async () => {
|
|
124
|
+
vi.useFakeTimers();
|
|
125
|
+
const { service, statePushes } = harness(1);
|
|
126
|
+
await vi.advanceTimersByTimeAsync(300);
|
|
127
|
+
const before = statePushes().length;
|
|
128
|
+
service.onStreamUpdate({ queryHash: 'q1', localArray: [], op: 'UPDATE', synthetic: true });
|
|
129
|
+
await vi.advanceTimersByTimeAsync(300);
|
|
130
|
+
expect(statePushes().length).toBe(before);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it('refreshes the table list on an explicit pull, not on a push', async () => {
|
|
134
|
+
vi.useFakeTimers();
|
|
135
|
+
const { service, fakeWindow, infoQueries } = harness(1);
|
|
136
|
+
const connectRefreshes = infoQueries.filter((q) => q.includes('INFO FOR DB')).length;
|
|
137
|
+
for (let i = 0; i < 10; i++) {
|
|
138
|
+
service.onStreamUpdate({ queryHash: 'q1', localArray: [], op: 'UPDATE' });
|
|
139
|
+
await vi.advanceTimersByTimeAsync(300);
|
|
140
|
+
}
|
|
141
|
+
expect(infoQueries.filter((q) => q.includes('INFO FOR DB')).length).toBe(connectRefreshes);
|
|
142
|
+
vi.setSystemTime(Date.now() + 60_000);
|
|
143
|
+
fakeWindow.__00__.getState();
|
|
144
|
+
expect(infoQueries.filter((q) => q.includes('INFO FOR DB')).length).toBe(connectRefreshes + 1);
|
|
145
|
+
});
|
|
146
|
+
});
|
|
@@ -44,6 +44,7 @@ function makeSync(opts: { membershipKnown?: boolean } = {}) {
|
|
|
44
44
|
});
|
|
45
45
|
const notifyQuerySynced = vi.fn().mockResolvedValue(undefined);
|
|
46
46
|
const dataModule: any = {
|
|
47
|
+
scheduleRematerialize: vi.fn(),
|
|
47
48
|
getQueryById: vi.fn().mockReturnValue(queryState),
|
|
48
49
|
getQueryByHash: vi.fn().mockReturnValue(queryState),
|
|
49
50
|
updateQueryRemoteArray,
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -1017,6 +1017,11 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
1017
1017
|
'syncQuery failed during poll'
|
|
1018
1018
|
);
|
|
1019
1019
|
}
|
|
1020
|
+
// Membership moved but a row may have needed no fetch (this client wrote
|
|
1021
|
+
// it, so the engine already holds it at the published version): nothing
|
|
1022
|
+
// then re-materializes the query. Ask for one; it is a no-op behind a
|
|
1023
|
+
// stream update that a fetch above already queued.
|
|
1024
|
+
if (changed) this.dataModule.scheduleRematerialize(queryHash);
|
|
1020
1025
|
// Cross-session fallback for `.related()` child rows: the LIVE-permission
|
|
1021
1026
|
// gap can drop child-edge notifications, so converge their bodies on the
|
|
1022
1027
|
// poll too (idempotent — no-op when nothing changed).
|
|
@@ -1351,6 +1356,11 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
1351
1356
|
// path already does for its own removals (`refetchListRefForQuery`).
|
|
1352
1357
|
if (diff.removed.length > 0 && diff.added.length === 0 && diff.updated.length === 0) {
|
|
1353
1358
|
await this.dataModule.notifyQuerySynced(hash);
|
|
1359
|
+
} else if (diff.added.length === 0 && diff.updated.length === 0) {
|
|
1360
|
+
// Empty diff: the circuit already holds the row at this version (this
|
|
1361
|
+
// tab wrote it), so no fetch and no stream update - but the membership
|
|
1362
|
+
// recorded above is new, and the subscribers have to hear about it.
|
|
1363
|
+
this.dataModule.scheduleRematerialize(hash);
|
|
1354
1364
|
}
|
|
1355
1365
|
}
|
|
1356
1366
|
|
|
@@ -11,12 +11,21 @@ export abstract class AbstractDatabaseService {
|
|
|
11
11
|
protected logger: Logger;
|
|
12
12
|
protected events: DatabaseEventSystem;
|
|
13
13
|
/**
|
|
14
|
-
* Per-query deadline in ms; `0` disables.
|
|
15
|
-
* (see `RemoteDatabaseService`)
|
|
16
|
-
*
|
|
17
|
-
*
|
|
14
|
+
* Per-query deadline in ms; `0` disables. The remote service sets it from
|
|
15
|
+
* `queryTimeoutMs` (see `RemoteDatabaseService`), the local one from
|
|
16
|
+
* `localOpTimeoutMs` (see `LocalDatabaseService`): a local query can be
|
|
17
|
+
* legitimately slow, but it must never be endless - every query waits on the
|
|
18
|
+
* previous link of {@link query}'s chain, and one that never settled wedged
|
|
19
|
+
* every later local op behind it.
|
|
18
20
|
*/
|
|
19
21
|
protected queryTimeoutMs = 0;
|
|
22
|
+
|
|
23
|
+
/** The error a deadline expiry rejects with; the local service substitutes
|
|
24
|
+
* its typed `LocalOpTimeoutError`. "timed out" in the message is
|
|
25
|
+
* load-bearing either way: `classifySyncError` keys off it. */
|
|
26
|
+
protected timeoutError(_query: string): Error {
|
|
27
|
+
return new Error(`Remote query timed out after ${this.queryTimeoutMs}ms`);
|
|
28
|
+
}
|
|
20
29
|
protected abstract eventType:
|
|
21
30
|
| typeof DatabaseEventTypes.LocalQuery
|
|
22
31
|
| typeof DatabaseEventTypes.RemoteQuery;
|
|
@@ -71,7 +80,7 @@ export abstract class AbstractDatabaseService {
|
|
|
71
80
|
const result = (await withTimeout(
|
|
72
81
|
pending as unknown as Promise<T>,
|
|
73
82
|
this.queryTimeoutMs,
|
|
74
|
-
|
|
83
|
+
() => this.timeoutError(query)
|
|
75
84
|
)) as T;
|
|
76
85
|
const duration = performance.now() - startTime;
|
|
77
86
|
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A local-store operation that did not answer within its deadline.
|
|
3
|
+
*
|
|
4
|
+
* The local write path (`db.create` / `db.update` / `db.delete`, every local
|
|
5
|
+
* query behind them) used to have no deadline anywhere: the SQLite worker
|
|
6
|
+
* transport parks a call until the worker replies, the surrealdb engine's
|
|
7
|
+
* query chain waits on the previous link, and `withRetry` retries without a
|
|
8
|
+
* clock. One op that never settled (a worker starved behind a long select, a
|
|
9
|
+
* lock verification awaiting `navigator.locks.query()` forever) left the
|
|
10
|
+
* caller's promise pending for the tab's lifetime - a chat composer that never
|
|
11
|
+
* re-enabled, a call that never got past "Connecting".
|
|
12
|
+
*
|
|
13
|
+
* The message says "timed out" on purpose: `classifySyncError` keys off it and
|
|
14
|
+
* treats the failure as transient (re-queue), never as an application error
|
|
15
|
+
* that rolls the mutation back. `retryable: false` keeps `withRetry` from
|
|
16
|
+
* spinning on it: the op is still running in the engine, retrying queues a
|
|
17
|
+
* second copy behind it.
|
|
18
|
+
*/
|
|
19
|
+
export class LocalOpTimeoutError extends Error {
|
|
20
|
+
override readonly name = 'LocalOpTimeoutError';
|
|
21
|
+
readonly retryable = false;
|
|
22
|
+
readonly op: string;
|
|
23
|
+
readonly timeoutMs: number;
|
|
24
|
+
|
|
25
|
+
constructor(op: string, timeoutMs: number) {
|
|
26
|
+
super(`Local database operation timed out after ${timeoutMs}ms (${op})`);
|
|
27
|
+
this.op = op;
|
|
28
|
+
this.timeoutMs = timeoutMs;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Default deadline for one local-store operation. Generous: a cold 4k-row
|
|
33
|
+
* select on a throttled tab is seconds, not tens of seconds. */
|
|
34
|
+
export const DEFAULT_LOCAL_OP_TIMEOUT_MS = 30_000;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Diagnostic} from 'surrealdb';
|
|
2
|
+
import { DEFAULT_LOCAL_OP_TIMEOUT_MS, LocalOpTimeoutError } from './errors';
|
|
2
3
|
import { applyDiagnostics, DateTime, RecordId, Surreal } from 'surrealdb';
|
|
3
4
|
import type { Sp00kyConfig } from '../../types';
|
|
4
5
|
import type { Logger } from '../logger/index';
|
|
@@ -110,6 +111,11 @@ export class LocalDatabaseService extends AbstractDatabaseService {
|
|
|
110
111
|
// engine-backed client (built lazily) before any query is issued.
|
|
111
112
|
super(createBareSurrealClient(), logger, events);
|
|
112
113
|
this.config = config;
|
|
114
|
+
this.queryTimeoutMs = Math.max(0, config.localOpTimeoutMs ?? DEFAULT_LOCAL_OP_TIMEOUT_MS);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
protected override timeoutError(query: string): Error {
|
|
118
|
+
return new LocalOpTimeoutError(query.slice(0, 80), this.queryTimeoutMs);
|
|
113
119
|
}
|
|
114
120
|
|
|
115
121
|
getConfig(): Sp00kyConfig<any>['database'] {
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { describe, it, expect, vi, afterEach } from 'vitest';
|
|
2
|
+
import { SqliteCacheEngine } from './sqlite-cache-engine';
|
|
3
|
+
import { stubTransport } from './sqlite-transport.fixture';
|
|
4
|
+
import { LocalOpTimeoutError } from './errors';
|
|
5
|
+
import { withRetry } from '../../utils/index';
|
|
6
|
+
|
|
7
|
+
function makeLogger(): any {
|
|
8
|
+
const noop = () => {};
|
|
9
|
+
const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
10
|
+
logger.child = () => logger;
|
|
11
|
+
return logger;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The worker transport parks a call until the worker replies. A worker that
|
|
16
|
+
* never did (starved, or wedged on an unbounded lock check) left every caller
|
|
17
|
+
* waiting forever. Each round trip now has a deadline.
|
|
18
|
+
*/
|
|
19
|
+
describe('SqliteCacheEngine local op deadline', () => {
|
|
20
|
+
afterEach(() => vi.useRealTimers());
|
|
21
|
+
|
|
22
|
+
async function openEngine(handler: (type: string, payload: any) => unknown) {
|
|
23
|
+
const engine = new SqliteCacheEngine(
|
|
24
|
+
{ namespace: 'n', database: 'd', localOpTimeoutMs: 50 } as any,
|
|
25
|
+
makeLogger()
|
|
26
|
+
);
|
|
27
|
+
stubTransport(engine, handler);
|
|
28
|
+
await engine.connect('bucket-a');
|
|
29
|
+
return engine;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
it('rejects a call that never answers with LocalOpTimeoutError and keeps the queue moving', async () => {
|
|
33
|
+
vi.useFakeTimers();
|
|
34
|
+
let hang = false;
|
|
35
|
+
const engine = await openEngine((type) => {
|
|
36
|
+
if (hang && type !== 'open') return new Promise(() => {});
|
|
37
|
+
return { rows: [], result: [] };
|
|
38
|
+
});
|
|
39
|
+
hang = true;
|
|
40
|
+
const stuck = engine.query('SELECT * FROM game');
|
|
41
|
+
const settled = stuck.then(
|
|
42
|
+
() => 'resolved',
|
|
43
|
+
(e) => e
|
|
44
|
+
);
|
|
45
|
+
await vi.advanceTimersByTimeAsync(60);
|
|
46
|
+
const err = await settled;
|
|
47
|
+
expect(err).toBeInstanceOf(LocalOpTimeoutError);
|
|
48
|
+
expect((err as Error).message).toMatch(/timed out/);
|
|
49
|
+
// The next op runs; the transport is not torn down for a slow worker.
|
|
50
|
+
hang = false;
|
|
51
|
+
await expect(engine.query('SELECT * FROM game')).resolves.toBeDefined();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('is not retried by withRetry', async () => {
|
|
55
|
+
const attempts = vi.fn(async () => {
|
|
56
|
+
throw new LocalOpTimeoutError('run', 5);
|
|
57
|
+
});
|
|
58
|
+
await expect(withRetry(makeLogger(), attempts)).rejects.toBeInstanceOf(LocalOpTimeoutError);
|
|
59
|
+
expect(attempts).toHaveBeenCalledTimes(1);
|
|
60
|
+
});
|
|
61
|
+
});
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { applyPatch, type Operation } from 'fast-json-patch';
|
|
2
|
+
import { DEFAULT_LOCAL_OP_TIMEOUT_MS, LocalOpTimeoutError } from './errors';
|
|
3
|
+
import { withTimeout } from '../../utils/index';
|
|
2
4
|
import type { QueryPlan, RelationPlan, WhereNode } from '@spooky-sync/query-builder';
|
|
3
5
|
import {
|
|
4
6
|
renderOrderSql,
|
|
@@ -117,6 +119,8 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
117
119
|
private workerSelectConfigured: boolean;
|
|
118
120
|
private events: DatabaseEventSystem = createDatabaseEventSystem();
|
|
119
121
|
private bucketId = 'anon';
|
|
122
|
+
/** Deadline for one worker round trip; see `localOpTimeoutMs`. */
|
|
123
|
+
private readonly localOpTimeoutMs: number;
|
|
120
124
|
/** Durability of the local store, set on every open. A plain Set of callbacks
|
|
121
125
|
* rather than a `DatabaseEventSystem` event: this changes at most once per
|
|
122
126
|
* open, and the typed event map is about query traffic. */
|
|
@@ -140,6 +144,7 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
140
144
|
private logger: Logger,
|
|
141
145
|
opts: { useOpfs?: boolean; workerSelect?: boolean; shared?: boolean } = {}
|
|
142
146
|
) {
|
|
147
|
+
this.localOpTimeoutMs = Math.max(0, config.localOpTimeoutMs ?? DEFAULT_LOCAL_OP_TIMEOUT_MS);
|
|
143
148
|
this.useOpfs = opts.useOpfs ?? true;
|
|
144
149
|
this.workerSelect = opts.workerSelect ?? config.workerSelect ?? true;
|
|
145
150
|
this.workerSelectConfigured = this.workerSelect;
|
|
@@ -322,7 +327,19 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
322
327
|
this.inFlightCalls.delete(tracked);
|
|
323
328
|
settle();
|
|
324
329
|
};
|
|
325
|
-
|
|
330
|
+
// Deadline per op. The transport parks a call until the worker replies,
|
|
331
|
+
// and a worker starved behind a long op (or wedged on an unbounded lock
|
|
332
|
+
// check) answered never; every caller up the stack then waited forever.
|
|
333
|
+
// Reject the CALLER only: the op is still queued in the worker and its
|
|
334
|
+
// late reply is dropped by the transport (no pending entry left to
|
|
335
|
+
// resolve). Deliberately no teardown here - a slow worker is not a dead
|
|
336
|
+
// one, and killing it mid-write would cost more than the wait.
|
|
337
|
+
const timeoutMs = this.localOpTimeoutMs;
|
|
338
|
+
const result = withTimeout(
|
|
339
|
+
this.transport.call<T>(type, payload),
|
|
340
|
+
timeoutMs,
|
|
341
|
+
() => new LocalOpTimeoutError(type, timeoutMs)
|
|
342
|
+
).then(
|
|
326
343
|
(v: T) => {
|
|
327
344
|
finish();
|
|
328
345
|
s.inFlight--;
|
|
@@ -338,6 +355,13 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
338
355
|
(e: unknown) => {
|
|
339
356
|
finish();
|
|
340
357
|
s.inFlight--;
|
|
358
|
+
if (e instanceof LocalOpTimeoutError) {
|
|
359
|
+
s.timeouts = (s.timeouts ?? 0) + 1;
|
|
360
|
+
this.logger.error(
|
|
361
|
+
{ type, timeoutMs, Category: 'sp00ky-client::SqliteCacheEngine::rawCall' },
|
|
362
|
+
'Local store op did not answer within its deadline; rejecting the caller'
|
|
363
|
+
);
|
|
364
|
+
}
|
|
341
365
|
throw e;
|
|
342
366
|
}
|
|
343
367
|
);
|
|
@@ -1224,6 +1248,8 @@ interface SqliteStats {
|
|
|
1224
1248
|
inFlight: number;
|
|
1225
1249
|
maxInFlight: number;
|
|
1226
1250
|
byType: Record<string, number>;
|
|
1251
|
+
/** Worker round trips that hit `localOpTimeoutMs` (see rawCall). */
|
|
1252
|
+
timeouts?: number;
|
|
1227
1253
|
/** Time ops spent waiting behind the opQueue before dispatch. */
|
|
1228
1254
|
queueWaitMs: number;
|
|
1229
1255
|
/** Time inside the worker's message handler (actual SQLite work). */
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { verifyLockStillHeld } from './sqlite-lock-verify';
|
|
3
|
+
|
|
4
|
+
describe('verifyLockStillHeld', () => {
|
|
5
|
+
it('fences when the lock is no longer held', async () => {
|
|
6
|
+
const fence = vi.fn(async () => {});
|
|
7
|
+
await verifyLockStillHeld({ query: async () => ({ held: [] }) }, 'lock-a', fence, 50);
|
|
8
|
+
expect(fence).toHaveBeenCalledWith('lock missing after suspected freeze');
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it('does nothing when the lock is still held', async () => {
|
|
12
|
+
const fence = vi.fn(async () => {});
|
|
13
|
+
await verifyLockStillHeld({ query: async () => ({ held: [{ name: 'lock-a' }] }) }, 'lock-a', fence, 50);
|
|
14
|
+
expect(fence).not.toHaveBeenCalled();
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it('resolves within the bound when the lock query never answers, without fencing', async () => {
|
|
18
|
+
const fence = vi.fn(async () => {});
|
|
19
|
+
const warn = vi.fn();
|
|
20
|
+
const started = Date.now();
|
|
21
|
+
await verifyLockStillHeld({ query: () => new Promise(() => {}) }, 'lock-a', fence, 30, warn);
|
|
22
|
+
expect(Date.now() - started).toBeLessThan(1_000);
|
|
23
|
+
expect(fence).not.toHaveBeenCalled();
|
|
24
|
+
expect(warn).toHaveBeenCalledTimes(1);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it('is a no-op without a lock or a query API', async () => {
|
|
28
|
+
const fence = vi.fn(async () => {});
|
|
29
|
+
await verifyLockStillHeld(null, 'x', fence);
|
|
30
|
+
await verifyLockStillHeld({ query: async () => ({}) }, undefined, fence);
|
|
31
|
+
expect(fence).not.toHaveBeenCalled();
|
|
32
|
+
});
|
|
33
|
+
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "Do we still hold the OPFS lock?" - asked by the SQLite worker after a gap
|
|
3
|
+
* in its ticks long enough that the tab may have been frozen and the broker
|
|
4
|
+
* may have handed the lock to another tab.
|
|
5
|
+
*
|
|
6
|
+
* Bounded. `navigator.locks.query()` has no deadline of its own, and every op
|
|
7
|
+
* on the worker's chain awaits this verification: one query that never
|
|
8
|
+
* resolves stopped every reply to every client for good. On timeout we resolve
|
|
9
|
+
* WITHOUT fencing - the steal callback registered on the lock request remains
|
|
10
|
+
* the backstop that fences a genuinely stolen lock - and say so.
|
|
11
|
+
*/
|
|
12
|
+
export interface LockQueryLike {
|
|
13
|
+
query(): Promise<{ held?: { name?: string }[] }>;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export const LOCK_VERIFY_TIMEOUT_MS = 2_000;
|
|
17
|
+
|
|
18
|
+
export async function verifyLockStillHeld(
|
|
19
|
+
locks: LockQueryLike | null | undefined,
|
|
20
|
+
name: string | undefined,
|
|
21
|
+
fence: (reason: string) => Promise<void> | void,
|
|
22
|
+
timeoutMs = LOCK_VERIFY_TIMEOUT_MS,
|
|
23
|
+
warn: (msg: string) => void = () => {}
|
|
24
|
+
): Promise<void> {
|
|
25
|
+
if (!locks || !name || typeof locks.query !== 'function') return;
|
|
26
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
27
|
+
try {
|
|
28
|
+
const state = await Promise.race([
|
|
29
|
+
locks.query(),
|
|
30
|
+
new Promise<null>((resolve) => {
|
|
31
|
+
timer = setTimeout(() => resolve(null), timeoutMs);
|
|
32
|
+
}),
|
|
33
|
+
]);
|
|
34
|
+
if (state === null) {
|
|
35
|
+
warn(`lock verification did not answer within ${timeoutMs}ms; trusting the steal callback`);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const stillHeld = (state.held ?? []).some((l) => l.name === name);
|
|
39
|
+
if (!stillHeld) await fence('lock missing after suspected freeze');
|
|
40
|
+
} catch {
|
|
41
|
+
/* query unavailable: fall back to the steal callback */
|
|
42
|
+
} finally {
|
|
43
|
+
if (timer) clearTimeout(timer);
|
|
44
|
+
}
|
|
45
|
+
}
|