@spooky-sync/core 0.0.1-canary.234 → 0.0.1-canary.236
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -2
- package/dist/index.d.ts +109 -24
- package/dist/index.js +377 -105
- package/dist/types.d.ts +5 -0
- package/package.json +3 -3
- package/src/modules/data/index.ts +7 -0
- package/src/modules/sync/sync.health.test.ts +34 -26
- package/src/modules/sync/sync.ts +211 -30
- package/src/modules/sync/utils.test.ts +53 -0
- package/src/modules/sync/utils.ts +77 -0
- package/src/services/database/database.query-timeout.test.ts +58 -1
- package/src/services/database/database.ts +106 -67
- package/src/services/database/remote.ts +30 -0
- package/src/sp00ky.init-query.test.ts +10 -10
- package/src/sp00ky.ts +47 -28
- package/src/types.ts +5 -0
package/dist/index.js
CHANGED
|
@@ -311,60 +311,102 @@ var AbstractDatabaseService = class {
|
|
|
311
311
|
tx() {
|
|
312
312
|
return this.client.beginTransaction();
|
|
313
313
|
}
|
|
314
|
-
queryQueue = Promise.resolve();
|
|
315
314
|
/**
|
|
316
|
-
*
|
|
315
|
+
* How many statements may be in flight at once.
|
|
317
316
|
*
|
|
318
|
-
*
|
|
319
|
-
*
|
|
320
|
-
*
|
|
321
|
-
*
|
|
317
|
+
* `1` serializes every statement behind the previous one. The local engines
|
|
318
|
+
* need that: overlapping ops trip the WASM / SQLite transaction layer. The
|
|
319
|
+
* remote service raises it (`RemoteDatabaseService`): a SurrealDB socket
|
|
320
|
+
* multiplexes requests by id, and serializing them capped the client at one
|
|
321
|
+
* statement per round trip (~5/s at 170ms RTT), so the `_00_list_ref` poll
|
|
322
|
+
* alone could hold every registration behind it and one slow one-shot read
|
|
323
|
+
* stalled the whole client. The scheduler's `MAX_CONCURRENT_DOWN` only means
|
|
324
|
+
* something when this is above 1.
|
|
325
|
+
*/
|
|
326
|
+
maxConcurrentQueries = 1;
|
|
327
|
+
inFlightQueries = 0;
|
|
328
|
+
queryWaiters = [];
|
|
329
|
+
/** FIFO slot acquisition, so `maxConcurrentQueries = 1` keeps strict order. */
|
|
330
|
+
acquireQuerySlot() {
|
|
331
|
+
if (this.inFlightQueries < this.maxConcurrentQueries) {
|
|
332
|
+
this.inFlightQueries++;
|
|
333
|
+
return Promise.resolve();
|
|
334
|
+
}
|
|
335
|
+
return new Promise((resolve) => {
|
|
336
|
+
this.queryWaiters.push(() => {
|
|
337
|
+
this.inFlightQueries++;
|
|
338
|
+
resolve();
|
|
339
|
+
});
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
releaseQuerySlot() {
|
|
343
|
+
this.inFlightQueries--;
|
|
344
|
+
const next = this.queryWaiters.shift();
|
|
345
|
+
if (next) next();
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Hook run right before a statement is sent, inside its slot. The remote
|
|
349
|
+
* service uses it to hold statements while a connect is in flight, so a
|
|
350
|
+
* query never runs on a socket that is open but has not had its namespace
|
|
351
|
+
* and token applied yet. The default does nothing.
|
|
352
|
+
*/
|
|
353
|
+
async beforeQuery() {}
|
|
354
|
+
/**
|
|
355
|
+
* Execute a statement under {@link maxConcurrentQueries}.
|
|
356
|
+
*
|
|
357
|
+
* A statement that never settles would hold its slot forever, and with a
|
|
358
|
+
* limit of 1 that blocks every later statement behind it. {@link
|
|
359
|
+
* queryTimeoutMs} bounds each one: on expiry this promise rejects and the
|
|
360
|
+
* slot is released, even though the underlying RPC is still parked in the
|
|
361
|
+
* SDK's pending map.
|
|
322
362
|
*/
|
|
323
363
|
async query(query, vars) {
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
}
|
|
367
|
-
}
|
|
364
|
+
await this.acquireQuerySlot();
|
|
365
|
+
try {
|
|
366
|
+
await this.beforeQuery();
|
|
367
|
+
const startTime = performance.now();
|
|
368
|
+
try {
|
|
369
|
+
this.logger.debug({
|
|
370
|
+
query,
|
|
371
|
+
vars,
|
|
372
|
+
Category: "sp00ky-client::Database::query"
|
|
373
|
+
}, "Executing query");
|
|
374
|
+
const result = await withTimeout(this.client.query(query, vars), this.queryTimeoutMs, () => this.timeoutError(query));
|
|
375
|
+
const duration = performance.now() - startTime;
|
|
376
|
+
this.events.emit(this.eventType, {
|
|
377
|
+
query,
|
|
378
|
+
vars,
|
|
379
|
+
duration,
|
|
380
|
+
success: true,
|
|
381
|
+
timestamp: Date.now()
|
|
382
|
+
});
|
|
383
|
+
this.logger.trace({
|
|
384
|
+
query,
|
|
385
|
+
result,
|
|
386
|
+
Category: "sp00ky-client::Database::query"
|
|
387
|
+
}, "Query executed successfully");
|
|
388
|
+
return result;
|
|
389
|
+
} catch (err) {
|
|
390
|
+
const duration = performance.now() - startTime;
|
|
391
|
+
this.events.emit(this.eventType, {
|
|
392
|
+
query,
|
|
393
|
+
vars,
|
|
394
|
+
duration,
|
|
395
|
+
success: false,
|
|
396
|
+
error: err instanceof Error ? err.message : String(err),
|
|
397
|
+
timestamp: Date.now()
|
|
398
|
+
});
|
|
399
|
+
this.logger.error({
|
|
400
|
+
query,
|
|
401
|
+
vars,
|
|
402
|
+
err,
|
|
403
|
+
Category: "sp00ky-client::Database::query"
|
|
404
|
+
}, "Query execution failed");
|
|
405
|
+
throw err;
|
|
406
|
+
}
|
|
407
|
+
} finally {
|
|
408
|
+
this.releaseQuerySlot();
|
|
409
|
+
}
|
|
368
410
|
}
|
|
369
411
|
async execute(query, vars) {
|
|
370
412
|
const raw = await this.query(query.sql, vars);
|
|
@@ -994,6 +1036,12 @@ function resolveReconnectConfig(input) {
|
|
|
994
1036
|
...input
|
|
995
1037
|
};
|
|
996
1038
|
}
|
|
1039
|
+
/**
|
|
1040
|
+
* Statements the remote service keeps in flight at once. Enough for the sync
|
|
1041
|
+
* scheduler's `MAX_CONCURRENT_DOWN` registrations plus the list_ref poll and
|
|
1042
|
+
* an app's own one-shot reads, small enough to stay polite to the server.
|
|
1043
|
+
*/
|
|
1044
|
+
const REMOTE_MAX_CONCURRENT_QUERIES = 6;
|
|
997
1045
|
var RemoteDatabaseService = class extends AbstractDatabaseService {
|
|
998
1046
|
config;
|
|
999
1047
|
eventType = DatabaseEventTypes.RemoteQuery;
|
|
@@ -1039,6 +1087,7 @@ var RemoteDatabaseService = class extends AbstractDatabaseService {
|
|
|
1039
1087
|
this.config = config;
|
|
1040
1088
|
this.reconnectConfig = resolveReconnectConfig(config.reconnect);
|
|
1041
1089
|
this.queryTimeoutMs = Math.max(0, config.queryTimeoutMs ?? 6e4);
|
|
1090
|
+
this.maxConcurrentQueries = REMOTE_MAX_CONCURRENT_QUERIES;
|
|
1042
1091
|
}
|
|
1043
1092
|
getConfig() {
|
|
1044
1093
|
return this.config;
|
|
@@ -1121,6 +1170,25 @@ var RemoteDatabaseService = class extends AbstractDatabaseService {
|
|
|
1121
1170
|
}
|
|
1122
1171
|
}
|
|
1123
1172
|
/**
|
|
1173
|
+
* Hold statements while a connect is in flight.
|
|
1174
|
+
*
|
|
1175
|
+
* `connect()` opens the socket, then selects the namespace/database, then
|
|
1176
|
+
* authenticates - three round trips. The SDK considers the connection ready
|
|
1177
|
+
* after the first, so a query queued during the other two ran on a socket
|
|
1178
|
+
* with no namespace ("Specify a namespace to use") or, worse, no identity.
|
|
1179
|
+
* Seen 2026-09-06 after the supervisor rebuilt a socket: a collection
|
|
1180
|
+
* recount failed with exactly that error. Waiting on the in-flight connect
|
|
1181
|
+
* costs nothing when no connect is running; a failed connect is not
|
|
1182
|
+
* swallowed here - the query then fails on its own, as before.
|
|
1183
|
+
*/
|
|
1184
|
+
async beforeQuery() {
|
|
1185
|
+
const inflight = this.connecting;
|
|
1186
|
+
if (!inflight) return;
|
|
1187
|
+
try {
|
|
1188
|
+
await inflight;
|
|
1189
|
+
} catch {}
|
|
1190
|
+
}
|
|
1191
|
+
/**
|
|
1124
1192
|
* Open (or re-open) the remote connection.
|
|
1125
1193
|
*
|
|
1126
1194
|
* Safe to call repeatedly: concurrent calls share the in-flight attempt, and
|
|
@@ -4160,11 +4228,13 @@ var DataModule = class DataModule {
|
|
|
4160
4228
|
const queryState = this.activeQueries.get(hash);
|
|
4161
4229
|
if (!queryState) return;
|
|
4162
4230
|
queryState.hydrated = true;
|
|
4231
|
+
if (queryState.serverMembership) return;
|
|
4163
4232
|
if (rows.length === 0) return;
|
|
4164
4233
|
const epoch = this.local.epoch;
|
|
4165
4234
|
const tableName = queryState.config.tableName;
|
|
4166
4235
|
await this.buildAndSaveCacheBatch(tableName, rows);
|
|
4167
4236
|
if (epoch !== this.local.epoch) return;
|
|
4237
|
+
if (queryState.serverMembership) return;
|
|
4168
4238
|
queryState.config.remoteArray = rows.map((r) => [encodeRecordId(r.id), r._00_rv || 1]);
|
|
4169
4239
|
queryState.config.membershipKnown = true;
|
|
4170
4240
|
queryState.records = await this.materializeRecords(queryState);
|
|
@@ -4461,6 +4531,7 @@ var DataModule = class DataModule {
|
|
|
4461
4531
|
const epoch = this.local.epoch;
|
|
4462
4532
|
queryState.config.remoteArray = remoteArray;
|
|
4463
4533
|
queryState.config.membershipKnown = true;
|
|
4534
|
+
queryState.serverMembership = true;
|
|
4464
4535
|
if (remoteArray.length > 0) {
|
|
4465
4536
|
queryState.config.remoteSeen = true;
|
|
4466
4537
|
queryState.config.emptyReads = 0;
|
|
@@ -4524,6 +4595,7 @@ var DataModule = class DataModule {
|
|
|
4524
4595
|
}
|
|
4525
4596
|
}
|
|
4526
4597
|
queryState.hydrated = false;
|
|
4598
|
+
queryState.serverMembership = false;
|
|
4527
4599
|
queryState.syncNotified = false;
|
|
4528
4600
|
queryState.records = [];
|
|
4529
4601
|
this.setQueryStatus(hash, "fetching");
|
|
@@ -5680,6 +5752,59 @@ function buildSubqueryListRefSelect(table) {
|
|
|
5680
5752
|
return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NOT NONE`;
|
|
5681
5753
|
}
|
|
5682
5754
|
/**
|
|
5755
|
+
* One round trip for the edges of MANY queries: primary and subquery rows
|
|
5756
|
+
* together, split client-side on `parent`. Replaces two selects per query per
|
|
5757
|
+
* poll tick, which on a page with 20+ live queries was ~50 serialized round
|
|
5758
|
+
* trips per cycle and kept the remote busy full-time.
|
|
5759
|
+
*/
|
|
5760
|
+
function buildListRefBatchSelect(table) {
|
|
5761
|
+
return `SELECT in, out, version, parent FROM ${table} WHERE in IN $ins`;
|
|
5762
|
+
}
|
|
5763
|
+
/**
|
|
5764
|
+
* `rowCount` for many `_00_query` rows at once, keyed by id so the caller does
|
|
5765
|
+
* not depend on the array order. Same semantics as
|
|
5766
|
+
* {@link buildQueryRowCountSelect}: a row the session cannot read comes back
|
|
5767
|
+
* without an id and must be treated as "unknown".
|
|
5768
|
+
*/
|
|
5769
|
+
function buildQueryRowCountBatchSelect() {
|
|
5770
|
+
return "SELECT VALUE { id: id, rowCount: rowCount } FROM $ins";
|
|
5771
|
+
}
|
|
5772
|
+
/** Known edges one poll round trip may carry before the cycle is split. */
|
|
5773
|
+
const LIST_REF_POLL_ROW_BUDGET = 1500;
|
|
5774
|
+
/** A view at or past this many edges is "large": it rides on LIVE and is only
|
|
5775
|
+
* re-polled every {@link LIST_REF_POLL_LARGE_VIEW_MIN_AGE_MS}. */
|
|
5776
|
+
const LIST_REF_POLL_LARGE_VIEW_ROWS = 1e3;
|
|
5777
|
+
const LIST_REF_POLL_LARGE_VIEW_MIN_AGE_MS = 15e3;
|
|
5778
|
+
/**
|
|
5779
|
+
* Split the active queries into the round trips of one poll cycle.
|
|
5780
|
+
*
|
|
5781
|
+
* Every query that is due is refreshed once per cycle, oldest refresh first,
|
|
5782
|
+
* packed greedily into chunks of at most `rowBudget` known edges so one
|
|
5783
|
+
* response never carries a whole page's worth of ids at once. A view with
|
|
5784
|
+
* `largeViewRows` or more edges is only due once `largeViewMinAgeMs` has
|
|
5785
|
+
* passed since its last refresh (LIVE remains its primary path); a chunk
|
|
5786
|
+
* always holds at least one query, so a single huge view still refreshes.
|
|
5787
|
+
*/
|
|
5788
|
+
function planListRefPollChunks(candidates, opts) {
|
|
5789
|
+
const { now, rowBudget = LIST_REF_POLL_ROW_BUDGET, largeViewRows = LIST_REF_POLL_LARGE_VIEW_ROWS, largeViewMinAgeMs = LIST_REF_POLL_LARGE_VIEW_MIN_AGE_MS } = opts;
|
|
5790
|
+
const due = candidates.filter((c) => c.rows < largeViewRows || now - c.lastPolledAt >= largeViewMinAgeMs).sort((a, b) => a.lastPolledAt - b.lastPolledAt || (a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0));
|
|
5791
|
+
const chunks = [];
|
|
5792
|
+
let current = [];
|
|
5793
|
+
let currentRows = 0;
|
|
5794
|
+
for (const c of due) {
|
|
5795
|
+
const cost = Math.max(1, c.rows);
|
|
5796
|
+
if (current.length > 0 && currentRows + cost > rowBudget) {
|
|
5797
|
+
chunks.push(current);
|
|
5798
|
+
current = [];
|
|
5799
|
+
currentRows = 0;
|
|
5800
|
+
}
|
|
5801
|
+
current.push(c.hash);
|
|
5802
|
+
currentRows += cost;
|
|
5803
|
+
}
|
|
5804
|
+
if (current.length > 0) chunks.push(current);
|
|
5805
|
+
return chunks;
|
|
5806
|
+
}
|
|
5807
|
+
/**
|
|
5683
5808
|
* Resolve the effective list-ref poll interval. Negative or zero
|
|
5684
5809
|
* values fall back to the default — accepting them would either
|
|
5685
5810
|
* disable polling silently or busy-loop the event loop.
|
|
@@ -6138,6 +6263,7 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6138
6263
|
listRefPollInFlight = null;
|
|
6139
6264
|
refSyncIntervalMs;
|
|
6140
6265
|
listRefIdleStreak = 0;
|
|
6266
|
+
listRefPolledAt = /* @__PURE__ */ new Map();
|
|
6141
6267
|
stillRemoteStreaks = /* @__PURE__ */ new Map();
|
|
6142
6268
|
lastLiveEventAt = null;
|
|
6143
6269
|
_liveRetryCount = 0;
|
|
@@ -6636,6 +6762,7 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6636
6762
|
this.listRefPollTimer = setTimeout(async () => {
|
|
6637
6763
|
if (!this.listRefPollRunning) return;
|
|
6638
6764
|
let changed = false;
|
|
6765
|
+
const startedAt = Date.now();
|
|
6639
6766
|
const tick = (async () => {
|
|
6640
6767
|
changed = await this.pollListRefForActiveQueries();
|
|
6641
6768
|
})();
|
|
@@ -6647,10 +6774,11 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6647
6774
|
if (!this.listRefPollRunning) return;
|
|
6648
6775
|
const awaitingMembership = this.dataModule.hasSettledWritesPending();
|
|
6649
6776
|
this.listRefIdleStreak = changed || awaitingMembership ? 0 : this.listRefIdleStreak + 1;
|
|
6650
|
-
|
|
6777
|
+
const cycleMs = Date.now() - startedAt;
|
|
6778
|
+
schedule(Math.max(listRefPollDelayMs({
|
|
6651
6779
|
idleStreak: this.listRefIdleStreak,
|
|
6652
6780
|
baseIntervalMs: this.refSyncIntervalMs
|
|
6653
|
-
}));
|
|
6781
|
+
}), cycleMs));
|
|
6654
6782
|
}
|
|
6655
6783
|
}, delayMs);
|
|
6656
6784
|
};
|
|
@@ -6664,9 +6792,10 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6664
6792
|
}
|
|
6665
6793
|
}
|
|
6666
6794
|
/**
|
|
6667
|
-
* One poll cycle: refetch `_00_list_ref` for
|
|
6668
|
-
*
|
|
6669
|
-
*
|
|
6795
|
+
* One poll cycle: refetch `_00_list_ref` for the active queries that are due,
|
|
6796
|
+
* one round trip per chunk (see `planListRefPollChunks`). Returns whether ANY
|
|
6797
|
+
* query's remoteArray actually changed — the scheduler uses this to drive the
|
|
6798
|
+
* adaptive idle backoff.
|
|
6670
6799
|
*
|
|
6671
6800
|
* Also the ONLY health signal that runs while the page is idle. Sync health is
|
|
6672
6801
|
* otherwise activity-driven (mutations/registrations via the scheduler,
|
|
@@ -6688,41 +6817,156 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6688
6817
|
}
|
|
6689
6818
|
return false;
|
|
6690
6819
|
}
|
|
6820
|
+
const now = Date.now();
|
|
6821
|
+
const active = new Set(hashes);
|
|
6822
|
+
for (const hash of this.listRefPolledAt.keys()) if (!active.has(hash)) this.listRefPolledAt.delete(hash);
|
|
6823
|
+
const chunks = planListRefPollChunks(hashes.map((hash) => ({
|
|
6824
|
+
hash,
|
|
6825
|
+
rows: this.dataModule.getQueryByHash(hash)?.config.remoteArray?.length ?? 0,
|
|
6826
|
+
lastPolledAt: this.listRefPolledAt.get(hash) ?? 0
|
|
6827
|
+
})), { now });
|
|
6691
6828
|
let anyChanged = false;
|
|
6692
6829
|
let reached = false;
|
|
6693
6830
|
let firstNetworkErr;
|
|
6694
|
-
for (const
|
|
6695
|
-
|
|
6696
|
-
|
|
6697
|
-
|
|
6698
|
-
|
|
6699
|
-
|
|
6700
|
-
|
|
6701
|
-
|
|
6702
|
-
|
|
6703
|
-
|
|
6704
|
-
|
|
6705
|
-
|
|
6831
|
+
for (const chunk of chunks) {
|
|
6832
|
+
let snapshots;
|
|
6833
|
+
try {
|
|
6834
|
+
snapshots = await this.fetchListRefSnapshots(chunk);
|
|
6835
|
+
reached = true;
|
|
6836
|
+
} catch (err) {
|
|
6837
|
+
if (classifySyncError(err) === "network") {
|
|
6838
|
+
if (firstNetworkErr === void 0) firstNetworkErr = err;
|
|
6839
|
+
} else reached = true;
|
|
6840
|
+
this.logger.debug({
|
|
6841
|
+
err: err?.message ?? err,
|
|
6842
|
+
hashes: chunk,
|
|
6843
|
+
Category: "sp00ky-client::Sp00kySync::pollListRefForActiveQueries"
|
|
6844
|
+
}, "list_ref poll round trip failed");
|
|
6845
|
+
continue;
|
|
6846
|
+
}
|
|
6847
|
+
for (const hash of chunk) {
|
|
6848
|
+
const snapshot = snapshots.get(hash);
|
|
6849
|
+
if (!snapshot) continue;
|
|
6850
|
+
this.listRefPolledAt.set(hash, now);
|
|
6851
|
+
try {
|
|
6852
|
+
if (await this.applyListRefSnapshot(hash, snapshot)) anyChanged = true;
|
|
6853
|
+
} catch (err) {
|
|
6854
|
+
this.logger.debug({
|
|
6855
|
+
err: err?.message ?? err,
|
|
6856
|
+
hash,
|
|
6857
|
+
Category: "sp00ky-client::Sp00kySync::pollListRefForActiveQueries"
|
|
6858
|
+
}, "Per-query list_ref poll apply failed");
|
|
6859
|
+
}
|
|
6860
|
+
}
|
|
6706
6861
|
}
|
|
6707
6862
|
if (reached) this.recordSyncOutcome(true);
|
|
6708
6863
|
else if (firstNetworkErr !== void 0) this.recordSyncOutcome(false, firstNetworkErr);
|
|
6709
6864
|
return anyChanged;
|
|
6710
6865
|
}
|
|
6711
6866
|
/**
|
|
6712
|
-
*
|
|
6713
|
-
*
|
|
6714
|
-
*
|
|
6715
|
-
*
|
|
6716
|
-
*
|
|
6717
|
-
*
|
|
6867
|
+
* The server's current `_00_list_ref` state for a chunk of queries in ONE
|
|
6868
|
+
* round trip: primary edges, subquery-child edges and `_00_query.rowCount`.
|
|
6869
|
+
* Every hash whose query still exists gets an entry (empty arrays and
|
|
6870
|
+
* `rowCount: null` when nothing was readable), so a caller can tell "no
|
|
6871
|
+
* edges" from "not asked". A hash is left out only when the select did not
|
|
6872
|
+
* come back as an array at all — the old per-query code skipped the apply in
|
|
6873
|
+
* that case too, since a non-array is not a membership set.
|
|
6874
|
+
*
|
|
6875
|
+
* Guarded against a batch that silently matches nothing: if the select
|
|
6876
|
+
* returned no edge at all while the client holds edges for a query the
|
|
6877
|
+
* server still reports rows for, the batch is not trusted and the chunk is
|
|
6878
|
+
* re-read one query at a time with the single-query select. An empty set
|
|
6879
|
+
* taken at face value would be recorded as the removal of every row.
|
|
6718
6880
|
*/
|
|
6719
|
-
async
|
|
6881
|
+
async fetchListRefSnapshots(hashes) {
|
|
6882
|
+
const listRefTbl = this.listRefTable();
|
|
6883
|
+
const ids = [];
|
|
6884
|
+
const hashById = /* @__PURE__ */ new Map();
|
|
6885
|
+
const out = /* @__PURE__ */ new Map();
|
|
6886
|
+
for (const hash of hashes) {
|
|
6887
|
+
const queryState = this.dataModule.getQueryByHash(hash);
|
|
6888
|
+
if (!queryState) continue;
|
|
6889
|
+
ids.push(queryState.config.id);
|
|
6890
|
+
hashById.set(encodeRecordId(queryState.config.id), hash);
|
|
6891
|
+
}
|
|
6892
|
+
if (ids.length === 0) return out;
|
|
6893
|
+
if (ids.length === 1) {
|
|
6894
|
+
const hash = hashById.get(encodeRecordId(ids[0]));
|
|
6895
|
+
const single = await this.fetchListRefSnapshot(ids[0]);
|
|
6896
|
+
if (single) out.set(hash, single);
|
|
6897
|
+
return out;
|
|
6898
|
+
}
|
|
6899
|
+
const [edges, counts] = await this.remote.query(`${buildListRefBatchSelect(listRefTbl)};\n${buildQueryRowCountBatchSelect()}`, { ins: ids });
|
|
6900
|
+
if (!Array.isArray(edges)) return out;
|
|
6901
|
+
for (const hash of hashById.values()) out.set(hash, {
|
|
6902
|
+
primary: [],
|
|
6903
|
+
subquery: [],
|
|
6904
|
+
rowCount: null
|
|
6905
|
+
});
|
|
6906
|
+
for (const row of edges) {
|
|
6907
|
+
const hash = hashById.get(encodeRecordId(row.in));
|
|
6908
|
+
if (!hash) continue;
|
|
6909
|
+
const snapshot = out.get(hash);
|
|
6910
|
+
const pair = [encodeRecordId(row.out), row.version];
|
|
6911
|
+
if (row.parent == null) snapshot.primary.push(pair);
|
|
6912
|
+
else snapshot.subquery.push(pair);
|
|
6913
|
+
}
|
|
6914
|
+
for (const count of Array.isArray(counts) ? counts : []) {
|
|
6915
|
+
if (!count || !count.id) continue;
|
|
6916
|
+
const hash = hashById.get(encodeRecordId(count.id));
|
|
6917
|
+
if (!hash) continue;
|
|
6918
|
+
out.get(hash).rowCount = typeof count.rowCount === "number" ? count.rowCount : null;
|
|
6919
|
+
}
|
|
6920
|
+
if (edges.length === 0 && Array.from(out.entries()).some(([hash, snapshot]) => {
|
|
6921
|
+
return (this.dataModule.getQueryByHash(hash)?.config.remoteArray?.length ?? 0) > 0 && (snapshot.rowCount ?? 0) > 0;
|
|
6922
|
+
})) {
|
|
6923
|
+
this.logger.warn({
|
|
6924
|
+
hashes,
|
|
6925
|
+
Category: "sp00ky-client::Sp00kySync::fetchListRefSnapshots"
|
|
6926
|
+
}, "Batched list_ref select returned no edges for queries that hold rows; re-reading one query at a time");
|
|
6927
|
+
for (const hash of Array.from(out.keys())) {
|
|
6928
|
+
const queryState = this.dataModule.getQueryByHash(hash);
|
|
6929
|
+
if (!queryState) {
|
|
6930
|
+
out.delete(hash);
|
|
6931
|
+
continue;
|
|
6932
|
+
}
|
|
6933
|
+
const single = await this.fetchListRefSnapshot(queryState.config.id);
|
|
6934
|
+
if (single) out.set(hash, single);
|
|
6935
|
+
else out.delete(hash);
|
|
6936
|
+
}
|
|
6937
|
+
}
|
|
6938
|
+
return out;
|
|
6939
|
+
}
|
|
6940
|
+
/**
|
|
6941
|
+
* Single-query form of {@link fetchListRefSnapshots}: the selects the
|
|
6942
|
+
* registration path has always used, in one statement batch. `null` when
|
|
6943
|
+
* the primary select did not come back as an array.
|
|
6944
|
+
*/
|
|
6945
|
+
async fetchListRefSnapshot(id) {
|
|
6946
|
+
const listRefTbl = this.listRefTable();
|
|
6947
|
+
const [items, serverRowCount, children] = await this.remote.query(`${buildListRefSelect(listRefTbl)};\n${buildQueryRowCountSelect()};\n${buildSubqueryListRefSelect(listRefTbl)}`, { in: id });
|
|
6948
|
+
if (!Array.isArray(items)) return null;
|
|
6949
|
+
const toPairs = (rows) => Array.isArray(rows) ? rows.map((item) => [encodeRecordId(item.out), item.version]) : [];
|
|
6950
|
+
return {
|
|
6951
|
+
primary: toPairs(items),
|
|
6952
|
+
subquery: toPairs(children),
|
|
6953
|
+
rowCount: typeof serverRowCount === "number" ? serverRowCount : null
|
|
6954
|
+
};
|
|
6955
|
+
}
|
|
6956
|
+
/**
|
|
6957
|
+
* Land one poll snapshot for `queryHash`: diff the server's edges against the
|
|
6958
|
+
* cached `remoteArray`, sync any added/updated rows through the SyncEngine,
|
|
6959
|
+
* persist the new remoteArray and converge the subquery children. This is
|
|
6960
|
+
* the same shape `createRemoteQuery` does for its initial fetch and what
|
|
6961
|
+
* `handleRemoteListRefChange` does per-LIVE-event — we reuse it on a timer
|
|
6962
|
+
* as a fallback for missed LIVE notifications. Returns whether membership
|
|
6963
|
+
* changed.
|
|
6964
|
+
*/
|
|
6965
|
+
async applyListRefSnapshot(queryHash, snapshot) {
|
|
6720
6966
|
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
6721
6967
|
if (!queryState) return false;
|
|
6722
|
-
const
|
|
6723
|
-
const
|
|
6724
|
-
if (!Array.isArray(items)) return false;
|
|
6725
|
-
const fresh = items.map((item) => [encodeRecordId(item.out), item.version]);
|
|
6968
|
+
const fresh = snapshot.primary;
|
|
6969
|
+
const serverRowCount = snapshot.rowCount;
|
|
6726
6970
|
const prevRemote = queryState.config.remoteArray ?? [];
|
|
6727
6971
|
const freshIds = new Set(fresh.map(([id]) => id));
|
|
6728
6972
|
const removedIds = prevRemote.filter(([id]) => !freshIds.has(id)).map(([id]) => id);
|
|
@@ -6734,15 +6978,15 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6734
6978
|
this.logger.info({
|
|
6735
6979
|
err: err?.message ?? err,
|
|
6736
6980
|
queryHash,
|
|
6737
|
-
Category: "sp00ky-client::Sp00kySync::
|
|
6981
|
+
Category: "sp00ky-client::Sp00kySync::applyListRefSnapshot"
|
|
6738
6982
|
}, "syncQuery failed during poll");
|
|
6739
6983
|
}
|
|
6740
6984
|
if (changed) this.dataModule.scheduleRematerialize(queryHash);
|
|
6741
|
-
await this.
|
|
6985
|
+
await this.applySubqueryChildren(queryHash, snapshot.subquery).catch((err) => {
|
|
6742
6986
|
this.logger.info({
|
|
6743
6987
|
err: err?.message ?? err,
|
|
6744
6988
|
queryHash,
|
|
6745
|
-
Category: "sp00ky-client::Sp00kySync::
|
|
6989
|
+
Category: "sp00ky-client::Sp00kySync::applyListRefSnapshot"
|
|
6746
6990
|
}, "Subquery child sync failed during poll");
|
|
6747
6991
|
});
|
|
6748
6992
|
if (removedIds.length > 0) try {
|
|
@@ -6751,7 +6995,7 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6751
6995
|
this.logger.info({
|
|
6752
6996
|
err: err?.message ?? err,
|
|
6753
6997
|
queryHash,
|
|
6754
|
-
Category: "sp00ky-client::Sp00kySync::
|
|
6998
|
+
Category: "sp00ky-client::Sp00kySync::applyListRefSnapshot"
|
|
6755
6999
|
}, "notifyQuerySynced failed during poll-removal re-render");
|
|
6756
7000
|
}
|
|
6757
7001
|
return changed;
|
|
@@ -7399,6 +7643,17 @@ var Sp00kySync = class Sp00kySync {
|
|
|
7399
7643
|
const [items] = await this.remote.query(buildSubqueryListRefSelect(listRefTbl), { in: queryState.config.id });
|
|
7400
7644
|
if (!Array.isArray(items)) return;
|
|
7401
7645
|
const fresh = items.map((item) => [encodeRecordId(item.out), item.version]);
|
|
7646
|
+
await this.applySubqueryChildren(queryHash, fresh);
|
|
7647
|
+
}
|
|
7648
|
+
/**
|
|
7649
|
+
* Converge a `.related()` query's child bodies onto `fresh` (the server's
|
|
7650
|
+
* current child edges). Shared by the registration path above and the poll,
|
|
7651
|
+
* which reads the child edges in its batched round trip. See
|
|
7652
|
+
* {@link syncSubqueryChildren} for the deletion-safety rules.
|
|
7653
|
+
*/
|
|
7654
|
+
async applySubqueryChildren(queryHash, fresh) {
|
|
7655
|
+
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
7656
|
+
if (!queryState) return;
|
|
7402
7657
|
const prev = queryState.config.subqueryRemoteArray ?? [];
|
|
7403
7658
|
if (recordVersionArraysEqual(fresh, prev)) return;
|
|
7404
7659
|
const diff = diffRecordVersionArray(prev, fresh);
|
|
@@ -7764,8 +8019,8 @@ function selfAllowlistedVariant(flag, userId) {
|
|
|
7764
8019
|
|
|
7765
8020
|
//#endregion
|
|
7766
8021
|
//#region src/modules/devtools/index.ts
|
|
7767
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
7768
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
8022
|
+
const CORE_VERSION = "0.0.1-canary.236";
|
|
8023
|
+
const WASM_VERSION = "0.0.1-canary.236";
|
|
7769
8024
|
const SURREAL_VERSION = "3.0.3";
|
|
7770
8025
|
var DevToolsService = class DevToolsService {
|
|
7771
8026
|
eventsHistory = [];
|
|
@@ -12780,7 +13035,7 @@ var Sp00kyClient = class {
|
|
|
12780
13035
|
return new TabsCoordinator({
|
|
12781
13036
|
tabId,
|
|
12782
13037
|
fingerprint: computeTabsFingerprint({
|
|
12783
|
-
coreVersion: "0.0.1-canary.
|
|
13038
|
+
coreVersion: "0.0.1-canary.236",
|
|
12784
13039
|
schemaHash: hash53(this.config.schemaSurql),
|
|
12785
13040
|
endpoint: this.config.database.endpoint ?? "",
|
|
12786
13041
|
namespace: this.config.database.namespace,
|
|
@@ -13287,16 +13542,34 @@ var Sp00kyClient = class {
|
|
|
13287
13542
|
return hash;
|
|
13288
13543
|
}
|
|
13289
13544
|
/**
|
|
13290
|
-
* Background tail of {@link initQuery}:
|
|
13291
|
-
* `config.instantHydrate`, and only when
|
|
13292
|
-
*
|
|
13293
|
-
*
|
|
13294
|
-
*
|
|
13295
|
-
*
|
|
13296
|
-
*
|
|
13297
|
-
*
|
|
13545
|
+
* Background tail of {@link initQuery}: enqueue the `register` down-event,
|
|
13546
|
+
* then instant-hydrate (opt-in via `config.instantHydrate`, and only when
|
|
13547
|
+
* the query is cold). Never rejects — both halves catch and log, so
|
|
13548
|
+
* `void`-ing the returned promise can't produce an unhandled rejection. By
|
|
13549
|
+
* default (hydrate off) the register lifecycle is the single freshness path;
|
|
13550
|
+
* the one-shot fetch is an optimization apps enable explicitly, and it runs
|
|
13551
|
+
* regardless of preload state — cache-first delivery never depends on WHY
|
|
13552
|
+
* rows are cached.
|
|
13553
|
+
*
|
|
13554
|
+
* Register goes first on purpose. The hydrate used to run strictly before
|
|
13555
|
+
* the enqueue, so a windowed list's one-shot read (a full-table scan when
|
|
13556
|
+
* the app has no matching index) held its own registration back by seconds
|
|
13557
|
+
* per window. Now whichever answer lands first paints; a hydrate that
|
|
13558
|
+
* arrives after the registration is dropped (`applyHydration`).
|
|
13298
13559
|
*/
|
|
13299
13560
|
async finishQueryInit(hash, q, params) {
|
|
13561
|
+
try {
|
|
13562
|
+
await this.sync.enqueueDownEvent({
|
|
13563
|
+
type: "register",
|
|
13564
|
+
payload: { hash }
|
|
13565
|
+
});
|
|
13566
|
+
} catch (err) {
|
|
13567
|
+
this.logger.error({
|
|
13568
|
+
err,
|
|
13569
|
+
hash,
|
|
13570
|
+
Category: "sp00ky-client::Sp00kyClient::initQuery"
|
|
13571
|
+
}, "Failed to enqueue register down-event");
|
|
13572
|
+
}
|
|
13300
13573
|
if (this.config.instantHydrate === true && this.dataModule.isCold(hash)) try {
|
|
13301
13574
|
const epoch = this.local.epoch;
|
|
13302
13575
|
const [rows] = await this.remote.query(q.selectQuery.query, params);
|
|
@@ -13312,18 +13585,6 @@ var Sp00kyClient = class {
|
|
|
13312
13585
|
Category: "sp00ky-client::Sp00kyClient::instantHydrate"
|
|
13313
13586
|
}, "Instant hydrate failed; proceeding with registration");
|
|
13314
13587
|
}
|
|
13315
|
-
try {
|
|
13316
|
-
await this.sync.enqueueDownEvent({
|
|
13317
|
-
type: "register",
|
|
13318
|
-
payload: { hash }
|
|
13319
|
-
});
|
|
13320
|
-
} catch (err) {
|
|
13321
|
-
this.logger.error({
|
|
13322
|
-
err,
|
|
13323
|
-
hash,
|
|
13324
|
-
Category: "sp00ky-client::Sp00kyClient::initQuery"
|
|
13325
|
-
}, "Failed to enqueue register down-event");
|
|
13326
|
-
}
|
|
13327
13588
|
}
|
|
13328
13589
|
/**
|
|
13329
13590
|
* Smart, awaitable preload/prewarm into the LOCAL cache — without registering a
|
|
@@ -13464,6 +13725,17 @@ var Sp00kyClient = class {
|
|
|
13464
13725
|
return fn(this.remote.getClient());
|
|
13465
13726
|
}
|
|
13466
13727
|
/**
|
|
13728
|
+
* Run one raw SurrealQL statement on the remote through the client's own
|
|
13729
|
+
* query path: the connect gate (it never runs on a socket whose namespace
|
|
13730
|
+
* and token are not applied yet), the per-statement timeout and the
|
|
13731
|
+
* concurrency limit. Prefer this over {@link useRemote} for one-shot reads
|
|
13732
|
+
* such as counts; `useRemote` hands out the bare SDK client and raced a
|
|
13733
|
+
* reconnect with "Specify a namespace to use".
|
|
13734
|
+
*/
|
|
13735
|
+
async remoteQuery(sql, vars) {
|
|
13736
|
+
return this.remote.query(sql, vars);
|
|
13737
|
+
}
|
|
13738
|
+
/**
|
|
13467
13739
|
* Mint the salt used for query-id hashing, so two sessions registering the
|
|
13468
13740
|
* same logical query get distinct `_00_query` rows.
|
|
13469
13741
|
*
|
package/dist/types.d.ts
CHANGED
|
@@ -956,6 +956,11 @@ interface QueryState {
|
|
|
956
956
|
/** Set once `applyHydration` has run for this query, so the cold instant-hydrate
|
|
957
957
|
* path fires at most once per query (see DataModule.isCold/applyHydration). */
|
|
958
958
|
hydrated?: boolean;
|
|
959
|
+
/** Set once a server-side membership set has been accepted this session
|
|
960
|
+
* (`updateQueryRemoteArray`). The `register` down-event is enqueued before
|
|
961
|
+
* the instant-hydrate, so a hydrate that finishes later must not prime
|
|
962
|
+
* membership over the authoritative set. Ephemeral. */
|
|
963
|
+
serverMembership?: boolean;
|
|
959
964
|
/** Set once `notifyQuerySynced` has emitted for this registration lifetime.
|
|
960
965
|
* Ephemeral (unlike the persisted `updateCount`), so a re-registered query
|
|
961
966
|
* always emits at least once even when its records are unchanged — otherwise
|