@spooky-sync/core 0.0.1-canary.235 → 0.0.1-canary.237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -2
- package/dist/index.d.ts +95 -30
- package/dist/index.js +356 -111
- package/dist/types.d.ts +5 -0
- package/package.json +3 -3
- package/src/modules/data/index.ts +7 -0
- package/src/modules/sync/sync.health.test.ts +34 -26
- package/src/modules/sync/sync.ts +211 -30
- package/src/modules/sync/utils.test.ts +53 -0
- package/src/modules/sync/utils.ts +77 -0
- package/src/services/database/database.query-timeout.test.ts +58 -1
- package/src/services/database/database.ts +103 -73
- package/src/services/database/remote.ts +8 -0
- package/src/sp00ky.init-query.test.ts +10 -10
- package/src/sp00ky.ts +47 -28
- package/src/types.ts +5 -0
package/dist/index.js
CHANGED
|
@@ -311,68 +311,102 @@ var AbstractDatabaseService = class {
|
|
|
311
311
|
tx() {
|
|
312
312
|
return this.client.beginTransaction();
|
|
313
313
|
}
|
|
314
|
-
queryQueue = Promise.resolve();
|
|
315
314
|
/**
|
|
316
|
-
*
|
|
315
|
+
* How many statements may be in flight at once.
|
|
317
316
|
*
|
|
318
|
-
*
|
|
319
|
-
*
|
|
320
|
-
*
|
|
321
|
-
*
|
|
322
|
-
|
|
317
|
+
* `1` serializes every statement behind the previous one. The local engines
|
|
318
|
+
* need that: overlapping ops trip the WASM / SQLite transaction layer. The
|
|
319
|
+
* remote service raises it (`RemoteDatabaseService`): a SurrealDB socket
|
|
320
|
+
* multiplexes requests by id, and serializing them capped the client at one
|
|
321
|
+
* statement per round trip (~5/s at 170ms RTT), so the `_00_list_ref` poll
|
|
322
|
+
* alone could hold every registration behind it and one slow one-shot read
|
|
323
|
+
* stalled the whole client. The scheduler's `MAX_CONCURRENT_DOWN` only means
|
|
324
|
+
* something when this is above 1.
|
|
325
|
+
*/
|
|
326
|
+
maxConcurrentQueries = 1;
|
|
327
|
+
inFlightQueries = 0;
|
|
328
|
+
queryWaiters = [];
|
|
329
|
+
/** FIFO slot acquisition, so `maxConcurrentQueries = 1` keeps strict order. */
|
|
330
|
+
acquireQuerySlot() {
|
|
331
|
+
if (this.inFlightQueries < this.maxConcurrentQueries) {
|
|
332
|
+
this.inFlightQueries++;
|
|
333
|
+
return Promise.resolve();
|
|
334
|
+
}
|
|
335
|
+
return new Promise((resolve) => {
|
|
336
|
+
this.queryWaiters.push(() => {
|
|
337
|
+
this.inFlightQueries++;
|
|
338
|
+
resolve();
|
|
339
|
+
});
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
releaseQuerySlot() {
|
|
343
|
+
this.inFlightQueries--;
|
|
344
|
+
const next = this.queryWaiters.shift();
|
|
345
|
+
if (next) next();
|
|
346
|
+
}
|
|
323
347
|
/**
|
|
324
|
-
* Hook run
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
*
|
|
348
|
+
* Hook run right before a statement is sent, inside its slot. The remote
|
|
349
|
+
* service uses it to hold statements while a connect is in flight, so a
|
|
350
|
+
* query never runs on a socket that is open but has not had its namespace
|
|
351
|
+
* and token applied yet. The default does nothing.
|
|
328
352
|
*/
|
|
329
353
|
async beforeQuery() {}
|
|
354
|
+
/**
|
|
355
|
+
* Execute a statement under {@link maxConcurrentQueries}.
|
|
356
|
+
*
|
|
357
|
+
* A statement that never settles would hold its slot forever, and with a
|
|
358
|
+
* limit of 1 that blocks every later statement behind it. {@link
|
|
359
|
+
* queryTimeoutMs} bounds each one: on expiry this promise rejects and the
|
|
360
|
+
* slot is released, even though the underlying RPC is still parked in the
|
|
361
|
+
* SDK's pending map.
|
|
362
|
+
*/
|
|
330
363
|
async query(query, vars) {
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
364
|
+
await this.acquireQuerySlot();
|
|
365
|
+
try {
|
|
366
|
+
await this.beforeQuery();
|
|
367
|
+
const startTime = performance.now();
|
|
368
|
+
try {
|
|
369
|
+
this.logger.debug({
|
|
370
|
+
query,
|
|
371
|
+
vars,
|
|
372
|
+
Category: "sp00ky-client::Database::query"
|
|
373
|
+
}, "Executing query");
|
|
374
|
+
const result = await withTimeout(this.client.query(query, vars), this.queryTimeoutMs, () => this.timeoutError(query));
|
|
375
|
+
const duration = performance.now() - startTime;
|
|
376
|
+
this.events.emit(this.eventType, {
|
|
377
|
+
query,
|
|
378
|
+
vars,
|
|
379
|
+
duration,
|
|
380
|
+
success: true,
|
|
381
|
+
timestamp: Date.now()
|
|
382
|
+
});
|
|
383
|
+
this.logger.trace({
|
|
384
|
+
query,
|
|
385
|
+
result,
|
|
386
|
+
Category: "sp00ky-client::Database::query"
|
|
387
|
+
}, "Query executed successfully");
|
|
388
|
+
return result;
|
|
389
|
+
} catch (err) {
|
|
390
|
+
const duration = performance.now() - startTime;
|
|
391
|
+
this.events.emit(this.eventType, {
|
|
392
|
+
query,
|
|
393
|
+
vars,
|
|
394
|
+
duration,
|
|
395
|
+
success: false,
|
|
396
|
+
error: err instanceof Error ? err.message : String(err),
|
|
397
|
+
timestamp: Date.now()
|
|
398
|
+
});
|
|
399
|
+
this.logger.error({
|
|
400
|
+
query,
|
|
401
|
+
vars,
|
|
402
|
+
err,
|
|
403
|
+
Category: "sp00ky-client::Database::query"
|
|
404
|
+
}, "Query execution failed");
|
|
405
|
+
throw err;
|
|
406
|
+
}
|
|
407
|
+
} finally {
|
|
408
|
+
this.releaseQuerySlot();
|
|
409
|
+
}
|
|
376
410
|
}
|
|
377
411
|
async execute(query, vars) {
|
|
378
412
|
const raw = await this.query(query.sql, vars);
|
|
@@ -1002,6 +1036,12 @@ function resolveReconnectConfig(input) {
|
|
|
1002
1036
|
...input
|
|
1003
1037
|
};
|
|
1004
1038
|
}
|
|
1039
|
+
/**
|
|
1040
|
+
* Statements the remote service keeps in flight at once. Enough for the sync
|
|
1041
|
+
* scheduler's `MAX_CONCURRENT_DOWN` registrations plus the list_ref poll and
|
|
1042
|
+
* an app's own one-shot reads, small enough to stay polite to the server.
|
|
1043
|
+
*/
|
|
1044
|
+
const REMOTE_MAX_CONCURRENT_QUERIES = 6;
|
|
1005
1045
|
var RemoteDatabaseService = class extends AbstractDatabaseService {
|
|
1006
1046
|
config;
|
|
1007
1047
|
eventType = DatabaseEventTypes.RemoteQuery;
|
|
@@ -1047,6 +1087,7 @@ var RemoteDatabaseService = class extends AbstractDatabaseService {
|
|
|
1047
1087
|
this.config = config;
|
|
1048
1088
|
this.reconnectConfig = resolveReconnectConfig(config.reconnect);
|
|
1049
1089
|
this.queryTimeoutMs = Math.max(0, config.queryTimeoutMs ?? 6e4);
|
|
1090
|
+
this.maxConcurrentQueries = REMOTE_MAX_CONCURRENT_QUERIES;
|
|
1050
1091
|
}
|
|
1051
1092
|
getConfig() {
|
|
1052
1093
|
return this.config;
|
|
@@ -4187,11 +4228,13 @@ var DataModule = class DataModule {
|
|
|
4187
4228
|
const queryState = this.activeQueries.get(hash);
|
|
4188
4229
|
if (!queryState) return;
|
|
4189
4230
|
queryState.hydrated = true;
|
|
4231
|
+
if (queryState.serverMembership) return;
|
|
4190
4232
|
if (rows.length === 0) return;
|
|
4191
4233
|
const epoch = this.local.epoch;
|
|
4192
4234
|
const tableName = queryState.config.tableName;
|
|
4193
4235
|
await this.buildAndSaveCacheBatch(tableName, rows);
|
|
4194
4236
|
if (epoch !== this.local.epoch) return;
|
|
4237
|
+
if (queryState.serverMembership) return;
|
|
4195
4238
|
queryState.config.remoteArray = rows.map((r) => [encodeRecordId(r.id), r._00_rv || 1]);
|
|
4196
4239
|
queryState.config.membershipKnown = true;
|
|
4197
4240
|
queryState.records = await this.materializeRecords(queryState);
|
|
@@ -4488,6 +4531,7 @@ var DataModule = class DataModule {
|
|
|
4488
4531
|
const epoch = this.local.epoch;
|
|
4489
4532
|
queryState.config.remoteArray = remoteArray;
|
|
4490
4533
|
queryState.config.membershipKnown = true;
|
|
4534
|
+
queryState.serverMembership = true;
|
|
4491
4535
|
if (remoteArray.length > 0) {
|
|
4492
4536
|
queryState.config.remoteSeen = true;
|
|
4493
4537
|
queryState.config.emptyReads = 0;
|
|
@@ -4551,6 +4595,7 @@ var DataModule = class DataModule {
|
|
|
4551
4595
|
}
|
|
4552
4596
|
}
|
|
4553
4597
|
queryState.hydrated = false;
|
|
4598
|
+
queryState.serverMembership = false;
|
|
4554
4599
|
queryState.syncNotified = false;
|
|
4555
4600
|
queryState.records = [];
|
|
4556
4601
|
this.setQueryStatus(hash, "fetching");
|
|
@@ -5707,6 +5752,59 @@ function buildSubqueryListRefSelect(table) {
|
|
|
5707
5752
|
return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NOT NONE`;
|
|
5708
5753
|
}
|
|
5709
5754
|
/**
|
|
5755
|
+
* One round trip for the edges of MANY queries: primary and subquery rows
|
|
5756
|
+
* together, split client-side on `parent`. Replaces two selects per query per
|
|
5757
|
+
* poll tick, which on a page with 20+ live queries was ~50 serialized round
|
|
5758
|
+
* trips per cycle and kept the remote busy full-time.
|
|
5759
|
+
*/
|
|
5760
|
+
function buildListRefBatchSelect(table) {
|
|
5761
|
+
return `SELECT in, out, version, parent FROM ${table} WHERE in IN $ins`;
|
|
5762
|
+
}
|
|
5763
|
+
/**
|
|
5764
|
+
* `rowCount` for many `_00_query` rows at once, keyed by id so the caller does
|
|
5765
|
+
* not depend on the array order. Same semantics as
|
|
5766
|
+
* {@link buildQueryRowCountSelect}: a row the session cannot read comes back
|
|
5767
|
+
* without an id and must be treated as "unknown".
|
|
5768
|
+
*/
|
|
5769
|
+
function buildQueryRowCountBatchSelect() {
|
|
5770
|
+
return "SELECT VALUE { id: id, rowCount: rowCount } FROM $ins";
|
|
5771
|
+
}
|
|
5772
|
+
/** Known edges one poll round trip may carry before the cycle is split. */
|
|
5773
|
+
const LIST_REF_POLL_ROW_BUDGET = 1500;
|
|
5774
|
+
/** A view at or past this many edges is "large": it rides on LIVE and is only
|
|
5775
|
+
* re-polled every {@link LIST_REF_POLL_LARGE_VIEW_MIN_AGE_MS}. */
|
|
5776
|
+
const LIST_REF_POLL_LARGE_VIEW_ROWS = 1e3;
|
|
5777
|
+
const LIST_REF_POLL_LARGE_VIEW_MIN_AGE_MS = 15e3;
|
|
5778
|
+
/**
|
|
5779
|
+
* Split the active queries into the round trips of one poll cycle.
|
|
5780
|
+
*
|
|
5781
|
+
* Every query that is due is refreshed once per cycle, oldest refresh first,
|
|
5782
|
+
* packed greedily into chunks of at most `rowBudget` known edges so one
|
|
5783
|
+
* response never carries a whole page's worth of ids at once. A view with
|
|
5784
|
+
* `largeViewRows` or more edges is only due once `largeViewMinAgeMs` has
|
|
5785
|
+
* passed since its last refresh (LIVE remains its primary path); a chunk
|
|
5786
|
+
* always holds at least one query, so a single huge view still refreshes.
|
|
5787
|
+
*/
|
|
5788
|
+
function planListRefPollChunks(candidates, opts) {
|
|
5789
|
+
const { now, rowBudget = LIST_REF_POLL_ROW_BUDGET, largeViewRows = LIST_REF_POLL_LARGE_VIEW_ROWS, largeViewMinAgeMs = LIST_REF_POLL_LARGE_VIEW_MIN_AGE_MS } = opts;
|
|
5790
|
+
const due = candidates.filter((c) => c.rows < largeViewRows || now - c.lastPolledAt >= largeViewMinAgeMs).sort((a, b) => a.lastPolledAt - b.lastPolledAt || (a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0));
|
|
5791
|
+
const chunks = [];
|
|
5792
|
+
let current = [];
|
|
5793
|
+
let currentRows = 0;
|
|
5794
|
+
for (const c of due) {
|
|
5795
|
+
const cost = Math.max(1, c.rows);
|
|
5796
|
+
if (current.length > 0 && currentRows + cost > rowBudget) {
|
|
5797
|
+
chunks.push(current);
|
|
5798
|
+
current = [];
|
|
5799
|
+
currentRows = 0;
|
|
5800
|
+
}
|
|
5801
|
+
current.push(c.hash);
|
|
5802
|
+
currentRows += cost;
|
|
5803
|
+
}
|
|
5804
|
+
if (current.length > 0) chunks.push(current);
|
|
5805
|
+
return chunks;
|
|
5806
|
+
}
|
|
5807
|
+
/**
|
|
5710
5808
|
* Resolve the effective list-ref poll interval. Negative or zero
|
|
5711
5809
|
* values fall back to the default — accepting them would either
|
|
5712
5810
|
* disable polling silently or busy-loop the event loop.
|
|
@@ -6165,6 +6263,7 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6165
6263
|
listRefPollInFlight = null;
|
|
6166
6264
|
refSyncIntervalMs;
|
|
6167
6265
|
listRefIdleStreak = 0;
|
|
6266
|
+
listRefPolledAt = /* @__PURE__ */ new Map();
|
|
6168
6267
|
stillRemoteStreaks = /* @__PURE__ */ new Map();
|
|
6169
6268
|
lastLiveEventAt = null;
|
|
6170
6269
|
_liveRetryCount = 0;
|
|
@@ -6663,6 +6762,7 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6663
6762
|
this.listRefPollTimer = setTimeout(async () => {
|
|
6664
6763
|
if (!this.listRefPollRunning) return;
|
|
6665
6764
|
let changed = false;
|
|
6765
|
+
const startedAt = Date.now();
|
|
6666
6766
|
const tick = (async () => {
|
|
6667
6767
|
changed = await this.pollListRefForActiveQueries();
|
|
6668
6768
|
})();
|
|
@@ -6674,10 +6774,11 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6674
6774
|
if (!this.listRefPollRunning) return;
|
|
6675
6775
|
const awaitingMembership = this.dataModule.hasSettledWritesPending();
|
|
6676
6776
|
this.listRefIdleStreak = changed || awaitingMembership ? 0 : this.listRefIdleStreak + 1;
|
|
6677
|
-
|
|
6777
|
+
const cycleMs = Date.now() - startedAt;
|
|
6778
|
+
schedule(Math.max(listRefPollDelayMs({
|
|
6678
6779
|
idleStreak: this.listRefIdleStreak,
|
|
6679
6780
|
baseIntervalMs: this.refSyncIntervalMs
|
|
6680
|
-
}));
|
|
6781
|
+
}), cycleMs));
|
|
6681
6782
|
}
|
|
6682
6783
|
}, delayMs);
|
|
6683
6784
|
};
|
|
@@ -6691,9 +6792,10 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6691
6792
|
}
|
|
6692
6793
|
}
|
|
6693
6794
|
/**
|
|
6694
|
-
* One poll cycle: refetch `_00_list_ref` for
|
|
6695
|
-
*
|
|
6696
|
-
*
|
|
6795
|
+
* One poll cycle: refetch `_00_list_ref` for the active queries that are due,
|
|
6796
|
+
* one round trip per chunk (see `planListRefPollChunks`). Returns whether ANY
|
|
6797
|
+
* query's remoteArray actually changed — the scheduler uses this to drive the
|
|
6798
|
+
* adaptive idle backoff.
|
|
6697
6799
|
*
|
|
6698
6800
|
* Also the ONLY health signal that runs while the page is idle. Sync health is
|
|
6699
6801
|
* otherwise activity-driven (mutations/registrations via the scheduler,
|
|
@@ -6715,41 +6817,156 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6715
6817
|
}
|
|
6716
6818
|
return false;
|
|
6717
6819
|
}
|
|
6820
|
+
const now = Date.now();
|
|
6821
|
+
const active = new Set(hashes);
|
|
6822
|
+
for (const hash of this.listRefPolledAt.keys()) if (!active.has(hash)) this.listRefPolledAt.delete(hash);
|
|
6823
|
+
const chunks = planListRefPollChunks(hashes.map((hash) => ({
|
|
6824
|
+
hash,
|
|
6825
|
+
rows: this.dataModule.getQueryByHash(hash)?.config.remoteArray?.length ?? 0,
|
|
6826
|
+
lastPolledAt: this.listRefPolledAt.get(hash) ?? 0
|
|
6827
|
+
})), { now });
|
|
6718
6828
|
let anyChanged = false;
|
|
6719
6829
|
let reached = false;
|
|
6720
6830
|
let firstNetworkErr;
|
|
6721
|
-
for (const
|
|
6722
|
-
|
|
6723
|
-
|
|
6724
|
-
|
|
6725
|
-
|
|
6726
|
-
|
|
6727
|
-
|
|
6728
|
-
|
|
6729
|
-
|
|
6730
|
-
|
|
6731
|
-
|
|
6732
|
-
|
|
6831
|
+
for (const chunk of chunks) {
|
|
6832
|
+
let snapshots;
|
|
6833
|
+
try {
|
|
6834
|
+
snapshots = await this.fetchListRefSnapshots(chunk);
|
|
6835
|
+
reached = true;
|
|
6836
|
+
} catch (err) {
|
|
6837
|
+
if (classifySyncError(err) === "network") {
|
|
6838
|
+
if (firstNetworkErr === void 0) firstNetworkErr = err;
|
|
6839
|
+
} else reached = true;
|
|
6840
|
+
this.logger.debug({
|
|
6841
|
+
err: err?.message ?? err,
|
|
6842
|
+
hashes: chunk,
|
|
6843
|
+
Category: "sp00ky-client::Sp00kySync::pollListRefForActiveQueries"
|
|
6844
|
+
}, "list_ref poll round trip failed");
|
|
6845
|
+
continue;
|
|
6846
|
+
}
|
|
6847
|
+
for (const hash of chunk) {
|
|
6848
|
+
const snapshot = snapshots.get(hash);
|
|
6849
|
+
if (!snapshot) continue;
|
|
6850
|
+
this.listRefPolledAt.set(hash, now);
|
|
6851
|
+
try {
|
|
6852
|
+
if (await this.applyListRefSnapshot(hash, snapshot)) anyChanged = true;
|
|
6853
|
+
} catch (err) {
|
|
6854
|
+
this.logger.debug({
|
|
6855
|
+
err: err?.message ?? err,
|
|
6856
|
+
hash,
|
|
6857
|
+
Category: "sp00ky-client::Sp00kySync::pollListRefForActiveQueries"
|
|
6858
|
+
}, "Per-query list_ref poll apply failed");
|
|
6859
|
+
}
|
|
6860
|
+
}
|
|
6733
6861
|
}
|
|
6734
6862
|
if (reached) this.recordSyncOutcome(true);
|
|
6735
6863
|
else if (firstNetworkErr !== void 0) this.recordSyncOutcome(false, firstNetworkErr);
|
|
6736
6864
|
return anyChanged;
|
|
6737
6865
|
}
|
|
6738
6866
|
/**
|
|
6739
|
-
*
|
|
6740
|
-
*
|
|
6741
|
-
*
|
|
6742
|
-
*
|
|
6743
|
-
*
|
|
6744
|
-
*
|
|
6867
|
+
* The server's current `_00_list_ref` state for a chunk of queries in ONE
|
|
6868
|
+
* round trip: primary edges, subquery-child edges and `_00_query.rowCount`.
|
|
6869
|
+
* Every hash whose query still exists gets an entry (empty arrays and
|
|
6870
|
+
* `rowCount: null` when nothing was readable), so a caller can tell "no
|
|
6871
|
+
* edges" from "not asked". A hash is left out only when the select did not
|
|
6872
|
+
* come back as an array at all — the old per-query code skipped the apply in
|
|
6873
|
+
* that case too, since a non-array is not a membership set.
|
|
6874
|
+
*
|
|
6875
|
+
* Guarded against a batch that silently matches nothing: if the select
|
|
6876
|
+
* returned no edge at all while the client holds edges for a query the
|
|
6877
|
+
* server still reports rows for, the batch is not trusted and the chunk is
|
|
6878
|
+
* re-read one query at a time with the single-query select. An empty set
|
|
6879
|
+
* taken at face value would be recorded as the removal of every row.
|
|
6880
|
+
*/
|
|
6881
|
+
async fetchListRefSnapshots(hashes) {
|
|
6882
|
+
const listRefTbl = this.listRefTable();
|
|
6883
|
+
const ids = [];
|
|
6884
|
+
const hashById = /* @__PURE__ */ new Map();
|
|
6885
|
+
const out = /* @__PURE__ */ new Map();
|
|
6886
|
+
for (const hash of hashes) {
|
|
6887
|
+
const queryState = this.dataModule.getQueryByHash(hash);
|
|
6888
|
+
if (!queryState) continue;
|
|
6889
|
+
ids.push(queryState.config.id);
|
|
6890
|
+
hashById.set(encodeRecordId(queryState.config.id), hash);
|
|
6891
|
+
}
|
|
6892
|
+
if (ids.length === 0) return out;
|
|
6893
|
+
if (ids.length === 1) {
|
|
6894
|
+
const hash = hashById.get(encodeRecordId(ids[0]));
|
|
6895
|
+
const single = await this.fetchListRefSnapshot(ids[0]);
|
|
6896
|
+
if (single) out.set(hash, single);
|
|
6897
|
+
return out;
|
|
6898
|
+
}
|
|
6899
|
+
const [edges, counts] = await this.remote.query(`${buildListRefBatchSelect(listRefTbl)};\n${buildQueryRowCountBatchSelect()}`, { ins: ids });
|
|
6900
|
+
if (!Array.isArray(edges)) return out;
|
|
6901
|
+
for (const hash of hashById.values()) out.set(hash, {
|
|
6902
|
+
primary: [],
|
|
6903
|
+
subquery: [],
|
|
6904
|
+
rowCount: null
|
|
6905
|
+
});
|
|
6906
|
+
for (const row of edges) {
|
|
6907
|
+
const hash = hashById.get(encodeRecordId(row.in));
|
|
6908
|
+
if (!hash) continue;
|
|
6909
|
+
const snapshot = out.get(hash);
|
|
6910
|
+
const pair = [encodeRecordId(row.out), row.version];
|
|
6911
|
+
if (row.parent == null) snapshot.primary.push(pair);
|
|
6912
|
+
else snapshot.subquery.push(pair);
|
|
6913
|
+
}
|
|
6914
|
+
for (const count of Array.isArray(counts) ? counts : []) {
|
|
6915
|
+
if (!count || !count.id) continue;
|
|
6916
|
+
const hash = hashById.get(encodeRecordId(count.id));
|
|
6917
|
+
if (!hash) continue;
|
|
6918
|
+
out.get(hash).rowCount = typeof count.rowCount === "number" ? count.rowCount : null;
|
|
6919
|
+
}
|
|
6920
|
+
if (edges.length === 0 && Array.from(out.entries()).some(([hash, snapshot]) => {
|
|
6921
|
+
return (this.dataModule.getQueryByHash(hash)?.config.remoteArray?.length ?? 0) > 0 && (snapshot.rowCount ?? 0) > 0;
|
|
6922
|
+
})) {
|
|
6923
|
+
this.logger.warn({
|
|
6924
|
+
hashes,
|
|
6925
|
+
Category: "sp00ky-client::Sp00kySync::fetchListRefSnapshots"
|
|
6926
|
+
}, "Batched list_ref select returned no edges for queries that hold rows; re-reading one query at a time");
|
|
6927
|
+
for (const hash of Array.from(out.keys())) {
|
|
6928
|
+
const queryState = this.dataModule.getQueryByHash(hash);
|
|
6929
|
+
if (!queryState) {
|
|
6930
|
+
out.delete(hash);
|
|
6931
|
+
continue;
|
|
6932
|
+
}
|
|
6933
|
+
const single = await this.fetchListRefSnapshot(queryState.config.id);
|
|
6934
|
+
if (single) out.set(hash, single);
|
|
6935
|
+
else out.delete(hash);
|
|
6936
|
+
}
|
|
6937
|
+
}
|
|
6938
|
+
return out;
|
|
6939
|
+
}
|
|
6940
|
+
/**
|
|
6941
|
+
* Single-query form of {@link fetchListRefSnapshots}: the selects the
|
|
6942
|
+
* registration path has always used, in one statement batch. `null` when
|
|
6943
|
+
* the primary select did not come back as an array.
|
|
6944
|
+
*/
|
|
6945
|
+
async fetchListRefSnapshot(id) {
|
|
6946
|
+
const listRefTbl = this.listRefTable();
|
|
6947
|
+
const [items, serverRowCount, children] = await this.remote.query(`${buildListRefSelect(listRefTbl)};\n${buildQueryRowCountSelect()};\n${buildSubqueryListRefSelect(listRefTbl)}`, { in: id });
|
|
6948
|
+
if (!Array.isArray(items)) return null;
|
|
6949
|
+
const toPairs = (rows) => Array.isArray(rows) ? rows.map((item) => [encodeRecordId(item.out), item.version]) : [];
|
|
6950
|
+
return {
|
|
6951
|
+
primary: toPairs(items),
|
|
6952
|
+
subquery: toPairs(children),
|
|
6953
|
+
rowCount: typeof serverRowCount === "number" ? serverRowCount : null
|
|
6954
|
+
};
|
|
6955
|
+
}
|
|
6956
|
+
/**
|
|
6957
|
+
* Land one poll snapshot for `queryHash`: diff the server's edges against the
|
|
6958
|
+
* cached `remoteArray`, sync any added/updated rows through the SyncEngine,
|
|
6959
|
+
* persist the new remoteArray and converge the subquery children. This is
|
|
6960
|
+
* the same shape `createRemoteQuery` does for its initial fetch and what
|
|
6961
|
+
* `handleRemoteListRefChange` does per-LIVE-event — we reuse it on a timer
|
|
6962
|
+
* as a fallback for missed LIVE notifications. Returns whether membership
|
|
6963
|
+
* changed.
|
|
6745
6964
|
*/
|
|
6746
|
-
async
|
|
6965
|
+
async applyListRefSnapshot(queryHash, snapshot) {
|
|
6747
6966
|
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
6748
6967
|
if (!queryState) return false;
|
|
6749
|
-
const
|
|
6750
|
-
const
|
|
6751
|
-
if (!Array.isArray(items)) return false;
|
|
6752
|
-
const fresh = items.map((item) => [encodeRecordId(item.out), item.version]);
|
|
6968
|
+
const fresh = snapshot.primary;
|
|
6969
|
+
const serverRowCount = snapshot.rowCount;
|
|
6753
6970
|
const prevRemote = queryState.config.remoteArray ?? [];
|
|
6754
6971
|
const freshIds = new Set(fresh.map(([id]) => id));
|
|
6755
6972
|
const removedIds = prevRemote.filter(([id]) => !freshIds.has(id)).map(([id]) => id);
|
|
@@ -6761,15 +6978,15 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6761
6978
|
this.logger.info({
|
|
6762
6979
|
err: err?.message ?? err,
|
|
6763
6980
|
queryHash,
|
|
6764
|
-
Category: "sp00ky-client::Sp00kySync::
|
|
6981
|
+
Category: "sp00ky-client::Sp00kySync::applyListRefSnapshot"
|
|
6765
6982
|
}, "syncQuery failed during poll");
|
|
6766
6983
|
}
|
|
6767
6984
|
if (changed) this.dataModule.scheduleRematerialize(queryHash);
|
|
6768
|
-
await this.
|
|
6985
|
+
await this.applySubqueryChildren(queryHash, snapshot.subquery).catch((err) => {
|
|
6769
6986
|
this.logger.info({
|
|
6770
6987
|
err: err?.message ?? err,
|
|
6771
6988
|
queryHash,
|
|
6772
|
-
Category: "sp00ky-client::Sp00kySync::
|
|
6989
|
+
Category: "sp00ky-client::Sp00kySync::applyListRefSnapshot"
|
|
6773
6990
|
}, "Subquery child sync failed during poll");
|
|
6774
6991
|
});
|
|
6775
6992
|
if (removedIds.length > 0) try {
|
|
@@ -6778,7 +6995,7 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6778
6995
|
this.logger.info({
|
|
6779
6996
|
err: err?.message ?? err,
|
|
6780
6997
|
queryHash,
|
|
6781
|
-
Category: "sp00ky-client::Sp00kySync::
|
|
6998
|
+
Category: "sp00ky-client::Sp00kySync::applyListRefSnapshot"
|
|
6782
6999
|
}, "notifyQuerySynced failed during poll-removal re-render");
|
|
6783
7000
|
}
|
|
6784
7001
|
return changed;
|
|
@@ -7426,6 +7643,17 @@ var Sp00kySync = class Sp00kySync {
|
|
|
7426
7643
|
const [items] = await this.remote.query(buildSubqueryListRefSelect(listRefTbl), { in: queryState.config.id });
|
|
7427
7644
|
if (!Array.isArray(items)) return;
|
|
7428
7645
|
const fresh = items.map((item) => [encodeRecordId(item.out), item.version]);
|
|
7646
|
+
await this.applySubqueryChildren(queryHash, fresh);
|
|
7647
|
+
}
|
|
7648
|
+
/**
|
|
7649
|
+
* Converge a `.related()` query's child bodies onto `fresh` (the server's
|
|
7650
|
+
* current child edges). Shared by the registration path above and the poll,
|
|
7651
|
+
* which reads the child edges in its batched round trip. See
|
|
7652
|
+
* {@link syncSubqueryChildren} for the deletion-safety rules.
|
|
7653
|
+
*/
|
|
7654
|
+
async applySubqueryChildren(queryHash, fresh) {
|
|
7655
|
+
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
7656
|
+
if (!queryState) return;
|
|
7429
7657
|
const prev = queryState.config.subqueryRemoteArray ?? [];
|
|
7430
7658
|
if (recordVersionArraysEqual(fresh, prev)) return;
|
|
7431
7659
|
const diff = diffRecordVersionArray(prev, fresh);
|
|
@@ -7791,8 +8019,8 @@ function selfAllowlistedVariant(flag, userId) {
|
|
|
7791
8019
|
|
|
7792
8020
|
//#endregion
|
|
7793
8021
|
//#region src/modules/devtools/index.ts
|
|
7794
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
7795
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
8022
|
+
const CORE_VERSION = "0.0.1-canary.237";
|
|
8023
|
+
const WASM_VERSION = "0.0.1-canary.237";
|
|
7796
8024
|
const SURREAL_VERSION = "3.0.3";
|
|
7797
8025
|
var DevToolsService = class DevToolsService {
|
|
7798
8026
|
eventsHistory = [];
|
|
@@ -12807,7 +13035,7 @@ var Sp00kyClient = class {
|
|
|
12807
13035
|
return new TabsCoordinator({
|
|
12808
13036
|
tabId,
|
|
12809
13037
|
fingerprint: computeTabsFingerprint({
|
|
12810
|
-
coreVersion: "0.0.1-canary.
|
|
13038
|
+
coreVersion: "0.0.1-canary.237",
|
|
12811
13039
|
schemaHash: hash53(this.config.schemaSurql),
|
|
12812
13040
|
endpoint: this.config.database.endpoint ?? "",
|
|
12813
13041
|
namespace: this.config.database.namespace,
|
|
@@ -13314,16 +13542,34 @@ var Sp00kyClient = class {
|
|
|
13314
13542
|
return hash;
|
|
13315
13543
|
}
|
|
13316
13544
|
/**
|
|
13317
|
-
* Background tail of {@link initQuery}:
|
|
13318
|
-
* `config.instantHydrate`, and only when
|
|
13319
|
-
*
|
|
13320
|
-
*
|
|
13321
|
-
*
|
|
13322
|
-
*
|
|
13323
|
-
*
|
|
13324
|
-
*
|
|
13545
|
+
* Background tail of {@link initQuery}: enqueue the `register` down-event,
|
|
13546
|
+
* then instant-hydrate (opt-in via `config.instantHydrate`, and only when
|
|
13547
|
+
* the query is cold). Never rejects — both halves catch and log, so
|
|
13548
|
+
* `void`-ing the returned promise can't produce an unhandled rejection. By
|
|
13549
|
+
* default (hydrate off) the register lifecycle is the single freshness path;
|
|
13550
|
+
* the one-shot fetch is an optimization apps enable explicitly, and it runs
|
|
13551
|
+
* regardless of preload state — cache-first delivery never depends on WHY
|
|
13552
|
+
* rows are cached.
|
|
13553
|
+
*
|
|
13554
|
+
* Register goes first on purpose. The hydrate used to run strictly before
|
|
13555
|
+
* the enqueue, so a windowed list's one-shot read (a full-table scan when
|
|
13556
|
+
* the app has no matching index) held its own registration back by seconds
|
|
13557
|
+
* per window. Now whichever answer lands first paints; a hydrate that
|
|
13558
|
+
* arrives after the registration is dropped (`applyHydration`).
|
|
13325
13559
|
*/
|
|
13326
13560
|
async finishQueryInit(hash, q, params) {
|
|
13561
|
+
try {
|
|
13562
|
+
await this.sync.enqueueDownEvent({
|
|
13563
|
+
type: "register",
|
|
13564
|
+
payload: { hash }
|
|
13565
|
+
});
|
|
13566
|
+
} catch (err) {
|
|
13567
|
+
this.logger.error({
|
|
13568
|
+
err,
|
|
13569
|
+
hash,
|
|
13570
|
+
Category: "sp00ky-client::Sp00kyClient::initQuery"
|
|
13571
|
+
}, "Failed to enqueue register down-event");
|
|
13572
|
+
}
|
|
13327
13573
|
if (this.config.instantHydrate === true && this.dataModule.isCold(hash)) try {
|
|
13328
13574
|
const epoch = this.local.epoch;
|
|
13329
13575
|
const [rows] = await this.remote.query(q.selectQuery.query, params);
|
|
@@ -13339,18 +13585,6 @@ var Sp00kyClient = class {
|
|
|
13339
13585
|
Category: "sp00ky-client::Sp00kyClient::instantHydrate"
|
|
13340
13586
|
}, "Instant hydrate failed; proceeding with registration");
|
|
13341
13587
|
}
|
|
13342
|
-
try {
|
|
13343
|
-
await this.sync.enqueueDownEvent({
|
|
13344
|
-
type: "register",
|
|
13345
|
-
payload: { hash }
|
|
13346
|
-
});
|
|
13347
|
-
} catch (err) {
|
|
13348
|
-
this.logger.error({
|
|
13349
|
-
err,
|
|
13350
|
-
hash,
|
|
13351
|
-
Category: "sp00ky-client::Sp00kyClient::initQuery"
|
|
13352
|
-
}, "Failed to enqueue register down-event");
|
|
13353
|
-
}
|
|
13354
13588
|
}
|
|
13355
13589
|
/**
|
|
13356
13590
|
* Smart, awaitable preload/prewarm into the LOCAL cache — without registering a
|
|
@@ -13491,6 +13725,17 @@ var Sp00kyClient = class {
|
|
|
13491
13725
|
return fn(this.remote.getClient());
|
|
13492
13726
|
}
|
|
13493
13727
|
/**
|
|
13728
|
+
* Run one raw SurrealQL statement on the remote through the client's own
|
|
13729
|
+
* query path: the connect gate (it never runs on a socket whose namespace
|
|
13730
|
+
* and token are not applied yet), the per-statement timeout and the
|
|
13731
|
+
* concurrency limit. Prefer this over {@link useRemote} for one-shot reads
|
|
13732
|
+
* such as counts; `useRemote` hands out the bare SDK client and raced a
|
|
13733
|
+
* reconnect with "Specify a namespace to use".
|
|
13734
|
+
*/
|
|
13735
|
+
async remoteQuery(sql, vars) {
|
|
13736
|
+
return this.remote.query(sql, vars);
|
|
13737
|
+
}
|
|
13738
|
+
/**
|
|
13494
13739
|
* Mint the salt used for query-id hashing, so two sessions registering the
|
|
13495
13740
|
* same logical query get distinct `_00_query` rows.
|
|
13496
13741
|
*
|
package/dist/types.d.ts
CHANGED
|
@@ -956,6 +956,11 @@ interface QueryState {
|
|
|
956
956
|
/** Set once `applyHydration` has run for this query, so the cold instant-hydrate
|
|
957
957
|
* path fires at most once per query (see DataModule.isCold/applyHydration). */
|
|
958
958
|
hydrated?: boolean;
|
|
959
|
+
/** Set once a server-side membership set has been accepted this session
|
|
960
|
+
* (`updateQueryRemoteArray`). The `register` down-event is enqueued before
|
|
961
|
+
* the instant-hydrate, so a hydrate that finishes later must not prime
|
|
962
|
+
* membership over the authoritative set. Ephemeral. */
|
|
963
|
+
serverMembership?: boolean;
|
|
959
964
|
/** Set once `notifyQuerySynced` has emitted for this registration lifetime.
|
|
960
965
|
* Ephemeral (unlike the persisted `updateCount`), so a re-registered query
|
|
961
966
|
* always emits at least once even when its records are unchanged — otherwise
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.237",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
}
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {
|
|
63
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
64
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
63
|
+
"@spooky-sync/query-builder": "0.0.1-canary.237",
|
|
64
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.237",
|
|
65
65
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
66
66
|
"@surrealdb/wasm": "^3.0.3",
|
|
67
67
|
"blurhash": "^2.0.5",
|