@spooky-sync/core 0.0.1-canary.207 → 0.0.1-canary.209
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +69 -6
- package/dist/index.js +234 -61
- package/dist/sqlite-worker.js +22 -7
- package/dist/types.d.ts +10 -0
- package/package.json +3 -3
- package/src/index.ts +1 -0
- package/src/modules/data/data.pending-ids.test.ts +74 -0
- package/src/modules/data/data.rematerialize.test.ts +114 -0
- package/src/modules/data/index.ts +161 -62
- package/src/modules/devtools/index.ts +65 -18
- package/src/modules/devtools/notify-throttle.test.ts +6 -1
- package/src/modules/devtools/state-shape.test.ts +146 -0
- package/src/modules/sync/sync.live-removal.test.ts +1 -0
- package/src/modules/sync/sync.ts +10 -0
- package/src/services/database/database.ts +14 -5
- package/src/services/database/errors.ts +34 -0
- package/src/services/database/local.ts +6 -0
- package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
- package/src/services/database/sqlite-cache-engine.ts +27 -1
- package/src/services/database/sqlite-lock-verify.test.ts +33 -0
- package/src/services/database/sqlite-lock-verify.ts +45 -0
- package/src/services/database/sqlite-transport.ts +5 -2
- package/src/services/database/sqlite-worker.ts +11 -11
- package/src/services/stream-processor/index.ts +8 -0
- package/src/sp00ky.ts +3 -3
- package/src/types.ts +10 -0
- package/src/utils/index.ts +8 -2
- package/src/utils/parser.test.ts +49 -120
- package/src/utils/parser.ts +30 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as renderOrderSql, c as serializeRow, i as projectedDataSql, l as resolveRelations, o as renderWhereSql, r as project, s as reviveRow, t as PROMOTION_OPEN_OPTIONS, u as stableKey } from "./sqlite-open.js";
|
|
2
2
|
import { DateTime, Duration, RecordId, Surreal, Uuid, applyDiagnostics, createRemoteEngines } from "surrealdb";
|
|
3
|
-
import { QueryBuilder, RecordId as RecordId$1, cyrb53 } from "@spooky-sync/query-builder";
|
|
3
|
+
import { QueryBuilder, RecordId as RecordId$1, baseFieldOfParam, cyrb53 } from "@spooky-sync/query-builder";
|
|
4
4
|
import pino from "pino";
|
|
5
5
|
import { applyPatch } from "fast-json-patch";
|
|
6
6
|
import init, { Sp00kyProcessor } from "@spooky-sync/ssp-wasm";
|
|
@@ -84,6 +84,10 @@ function cleanRecord(tableSchema, record) {
|
|
|
84
84
|
for (const [key, value] of Object.entries(record)) if (key === "id" || key.startsWith("_00_") || key in tableSchema) cleaned[key] = value;
|
|
85
85
|
return cleaned;
|
|
86
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* Parse a RECORD's fields against the table schema. Anything the schema does not
|
|
89
|
+
* know is dropped, which is what keeps a stray field out of a write.
|
|
90
|
+
*/
|
|
87
91
|
function parseParams(tableSchema, params) {
|
|
88
92
|
const parsedParams = {};
|
|
89
93
|
for (const [key, value] of Object.entries(params)) {
|
|
@@ -92,6 +96,26 @@ function parseParams(tableSchema, params) {
|
|
|
92
96
|
}
|
|
93
97
|
return parsedParams;
|
|
94
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* Parse a QUERY's params. Unlike a record's fields, a param name is not always a
|
|
101
|
+
* column: an `_or` branch binds under a synthetic `white__or0` (see
|
|
102
|
+
* `orParamName` in @spooky-sync/query-builder), and the surql that references it
|
|
103
|
+
* is already written. So resolve the column through the synthetic name, and keep
|
|
104
|
+
* a param we cannot type rather than dropping it.
|
|
105
|
+
*
|
|
106
|
+
* Dropping was the bug: `parseParams` kept only column-named params, so every
|
|
107
|
+
* `_or` query registered with `$or0`/`$or1` unbound and matched no rows at all,
|
|
108
|
+
* silently (rowCount 0, errorCount 0).
|
|
109
|
+
*/
|
|
110
|
+
function parseQueryParams(tableSchema, params) {
|
|
111
|
+
const parsedParams = {};
|
|
112
|
+
for (const [key, value] of Object.entries(params)) {
|
|
113
|
+
if (value === void 0) continue;
|
|
114
|
+
const column = tableSchema[key] ?? tableSchema[baseFieldOfParam(key)];
|
|
115
|
+
parsedParams[key] = column ? parseValue(key, column, value) : value;
|
|
116
|
+
}
|
|
117
|
+
return parsedParams;
|
|
118
|
+
}
|
|
95
119
|
function parseValue(name, column, value) {
|
|
96
120
|
if (column.recordId) {
|
|
97
121
|
if (value instanceof RecordId$1) return value;
|
|
@@ -205,6 +229,7 @@ async function withRetry(logger, operation, retries = 3, delayMs = 100) {
|
|
|
205
229
|
return await operation();
|
|
206
230
|
} catch (err) {
|
|
207
231
|
lastError = err;
|
|
232
|
+
if (err?.retryable === false) throw err;
|
|
208
233
|
if (err?.message?.includes("Can not open transaction") || err?.message?.includes("transaction") || err?.message?.includes("Database is busy")) {
|
|
209
234
|
const msg = err instanceof Error ? err.message : String(err);
|
|
210
235
|
logger.warn({
|
|
@@ -240,7 +265,7 @@ async function withRetry(logger, operation, retries = 3, delayMs = 100) {
|
|
|
240
265
|
function withTimeout(promise, timeoutMs, message) {
|
|
241
266
|
if (!(timeoutMs > 0)) return promise;
|
|
242
267
|
return new Promise((resolve, reject) => {
|
|
243
|
-
const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
|
268
|
+
const timer = setTimeout(() => reject(typeof message === "function" ? message() : new Error(message)), timeoutMs);
|
|
244
269
|
promise.then((value) => {
|
|
245
270
|
clearTimeout(timer);
|
|
246
271
|
resolve(value);
|
|
@@ -258,12 +283,20 @@ var AbstractDatabaseService = class {
|
|
|
258
283
|
logger;
|
|
259
284
|
events;
|
|
260
285
|
/**
|
|
261
|
-
* Per-query deadline in ms; `0` disables.
|
|
262
|
-
* (see `RemoteDatabaseService`)
|
|
263
|
-
*
|
|
264
|
-
*
|
|
286
|
+
* Per-query deadline in ms; `0` disables. The remote service sets it from
|
|
287
|
+
* `queryTimeoutMs` (see `RemoteDatabaseService`), the local one from
|
|
288
|
+
* `localOpTimeoutMs` (see `LocalDatabaseService`): a local query can be
|
|
289
|
+
* legitimately slow, but it must never be endless - every query waits on the
|
|
290
|
+
* previous link of {@link query}'s chain, and one that never settled wedged
|
|
291
|
+
* every later local op behind it.
|
|
265
292
|
*/
|
|
266
293
|
queryTimeoutMs = 0;
|
|
294
|
+
/** The error a deadline expiry rejects with; the local service substitutes
|
|
295
|
+
* its typed `LocalOpTimeoutError`. "timed out" in the message is
|
|
296
|
+
* load-bearing either way: `classifySyncError` keys off it. */
|
|
297
|
+
timeoutError(_query) {
|
|
298
|
+
return /* @__PURE__ */ new Error(`Remote query timed out after ${this.queryTimeoutMs}ms`);
|
|
299
|
+
}
|
|
267
300
|
constructor(client, logger, events) {
|
|
268
301
|
this.client = client;
|
|
269
302
|
this.logger = logger.child({ service: "Database" });
|
|
@@ -297,7 +330,7 @@ var AbstractDatabaseService = class {
|
|
|
297
330
|
vars,
|
|
298
331
|
Category: "sp00ky-client::Database::query"
|
|
299
332
|
}, "Executing query");
|
|
300
|
-
const result = await withTimeout(this.client.query(query, vars), this.queryTimeoutMs,
|
|
333
|
+
const result = await withTimeout(this.client.query(query, vars), this.queryTimeoutMs, () => this.timeoutError(query));
|
|
301
334
|
const duration = performance.now() - startTime;
|
|
302
335
|
this.events.emit(this.eventType, {
|
|
303
336
|
query,
|
|
@@ -343,6 +376,41 @@ var AbstractDatabaseService = class {
|
|
|
343
376
|
}
|
|
344
377
|
};
|
|
345
378
|
|
|
379
|
+
//#endregion
|
|
380
|
+
//#region src/services/database/errors.ts
|
|
381
|
+
/**
|
|
382
|
+
* A local-store operation that did not answer within its deadline.
|
|
383
|
+
*
|
|
384
|
+
* The local write path (`db.create` / `db.update` / `db.delete`, every local
|
|
385
|
+
* query behind them) used to have no deadline anywhere: the SQLite worker
|
|
386
|
+
* transport parks a call until the worker replies, the surrealdb engine's
|
|
387
|
+
* query chain waits on the previous link, and `withRetry` retries without a
|
|
388
|
+
* clock. One op that never settled (a worker starved behind a long select, a
|
|
389
|
+
* lock verification awaiting `navigator.locks.query()` forever) left the
|
|
390
|
+
* caller's promise pending for the tab's lifetime - a chat composer that never
|
|
391
|
+
* re-enabled, a call that never got past "Connecting".
|
|
392
|
+
*
|
|
393
|
+
* The message says "timed out" on purpose: `classifySyncError` keys off it and
|
|
394
|
+
* treats the failure as transient (re-queue), never as an application error
|
|
395
|
+
* that rolls the mutation back. `retryable: false` keeps `withRetry` from
|
|
396
|
+
* spinning on it: the op is still running in the engine, retrying queues a
|
|
397
|
+
* second copy behind it.
|
|
398
|
+
*/
|
|
399
|
+
var LocalOpTimeoutError = class extends Error {
|
|
400
|
+
name = "LocalOpTimeoutError";
|
|
401
|
+
retryable = false;
|
|
402
|
+
op;
|
|
403
|
+
timeoutMs;
|
|
404
|
+
constructor(op, timeoutMs) {
|
|
405
|
+
super(`Local database operation timed out after ${timeoutMs}ms (${op})`);
|
|
406
|
+
this.op = op;
|
|
407
|
+
this.timeoutMs = timeoutMs;
|
|
408
|
+
}
|
|
409
|
+
};
|
|
410
|
+
/** Default deadline for one local-store operation. Generous: a cold 4k-row
|
|
411
|
+
* select on a throttled tab is seconds, not tens of seconds. */
|
|
412
|
+
const DEFAULT_LOCAL_OP_TIMEOUT_MS = 3e4;
|
|
413
|
+
|
|
346
414
|
//#endregion
|
|
347
415
|
//#region src/events/index.ts
|
|
348
416
|
/**
|
|
@@ -639,6 +707,10 @@ var LocalDatabaseService = class extends AbstractDatabaseService {
|
|
|
639
707
|
const events = createDatabaseEventSystem();
|
|
640
708
|
super(createBareSurrealClient(), logger, events);
|
|
641
709
|
this.config = config;
|
|
710
|
+
this.queryTimeoutMs = Math.max(0, config.localOpTimeoutMs ?? DEFAULT_LOCAL_OP_TIMEOUT_MS);
|
|
711
|
+
}
|
|
712
|
+
timeoutError(query) {
|
|
713
|
+
return new LocalOpTimeoutError(query.slice(0, 80), this.queryTimeoutMs);
|
|
642
714
|
}
|
|
643
715
|
getConfig() {
|
|
644
716
|
return this.config;
|
|
@@ -2052,7 +2124,7 @@ var WorkerSqliteTransport = class extends BaseTransport {
|
|
|
2052
2124
|
err: this.makeError(msg),
|
|
2053
2125
|
Category: "sp00ky-client::SqliteCacheEngine::worker"
|
|
2054
2126
|
}, "Worker error");
|
|
2055
|
-
this.
|
|
2127
|
+
this.close(msg);
|
|
2056
2128
|
};
|
|
2057
2129
|
this.worker.onerror = (e) => crash(e.message || "onerror");
|
|
2058
2130
|
this.worker.onmessageerror = () => crash("messageerror");
|
|
@@ -2205,6 +2277,8 @@ var SqliteCacheEngine = class {
|
|
|
2205
2277
|
workerSelectConfigured;
|
|
2206
2278
|
events = createDatabaseEventSystem();
|
|
2207
2279
|
bucketId = "anon";
|
|
2280
|
+
/** Deadline for one worker round trip; see `localOpTimeoutMs`. */
|
|
2281
|
+
localOpTimeoutMs;
|
|
2208
2282
|
/** Durability of the local store, set on every open. A plain Set of callbacks
|
|
2209
2283
|
* rather than a `DatabaseEventSystem` event: this changes at most once per
|
|
2210
2284
|
* open, and the typed event map is about query traffic. */
|
|
@@ -2226,6 +2300,7 @@ var SqliteCacheEngine = class {
|
|
|
2226
2300
|
constructor(config, logger, opts = {}) {
|
|
2227
2301
|
this.config = config;
|
|
2228
2302
|
this.logger = logger;
|
|
2303
|
+
this.localOpTimeoutMs = Math.max(0, config.localOpTimeoutMs ?? DEFAULT_LOCAL_OP_TIMEOUT_MS);
|
|
2229
2304
|
this.useOpfs = opts.useOpfs ?? true;
|
|
2230
2305
|
this.workerSelect = opts.workerSelect ?? config.workerSelect ?? true;
|
|
2231
2306
|
this.workerSelectConfigured = this.workerSelect;
|
|
@@ -2366,7 +2441,8 @@ var SqliteCacheEngine = class {
|
|
|
2366
2441
|
this.inFlightCalls.delete(tracked);
|
|
2367
2442
|
settle();
|
|
2368
2443
|
};
|
|
2369
|
-
|
|
2444
|
+
const timeoutMs = this.localOpTimeoutMs;
|
|
2445
|
+
return withTimeout(this.transport.call(type, payload), timeoutMs, () => new LocalOpTimeoutError(type, timeoutMs)).then((v) => {
|
|
2370
2446
|
finish();
|
|
2371
2447
|
s.inFlight--;
|
|
2372
2448
|
const wt = v?.wt;
|
|
@@ -2378,6 +2454,14 @@ var SqliteCacheEngine = class {
|
|
|
2378
2454
|
}, (e) => {
|
|
2379
2455
|
finish();
|
|
2380
2456
|
s.inFlight--;
|
|
2457
|
+
if (e instanceof LocalOpTimeoutError) {
|
|
2458
|
+
s.timeouts = (s.timeouts ?? 0) + 1;
|
|
2459
|
+
this.logger.error({
|
|
2460
|
+
type,
|
|
2461
|
+
timeoutMs,
|
|
2462
|
+
Category: "sp00ky-client::SqliteCacheEngine::rawCall"
|
|
2463
|
+
}, "Local store op did not answer within its deadline; rejecting the caller");
|
|
2464
|
+
}
|
|
2381
2465
|
throw e;
|
|
2382
2466
|
});
|
|
2383
2467
|
}
|
|
@@ -3541,6 +3625,11 @@ var DataModule = class DataModule {
|
|
|
3541
3625
|
await this.processStreamUpdate(update);
|
|
3542
3626
|
return;
|
|
3543
3627
|
}
|
|
3628
|
+
this.queueStreamUpdate(update);
|
|
3629
|
+
}
|
|
3630
|
+
/** Coalesce `update` onto the query's trailing timer (see onStreamUpdate). */
|
|
3631
|
+
queueStreamUpdate(update) {
|
|
3632
|
+
const { queryHash } = update;
|
|
3544
3633
|
if (this.debounceTimers.has(queryHash)) clearTimeout(this.debounceTimers.get(queryHash));
|
|
3545
3634
|
this.pendingStreamUpdates.set(queryHash, update);
|
|
3546
3635
|
const timer = setTimeout(async () => {
|
|
@@ -3551,6 +3640,30 @@ var DataModule = class DataModule {
|
|
|
3551
3640
|
this.debounceTimers.set(queryHash, timer);
|
|
3552
3641
|
}
|
|
3553
3642
|
/**
|
|
3643
|
+
* Re-materialize + notify a query whose MEMBERSHIP changed without any row
|
|
3644
|
+
* needing to be fetched, i.e. without the SSP stream update that normally
|
|
3645
|
+
* carries the notify. That is every row this client wrote itself: the local
|
|
3646
|
+
* CREATE memoized it at `_00_rv = 1`, the server publishes it at 1, so the
|
|
3647
|
+
* sync engine rightly fetches nothing - and then nobody told the subscribers
|
|
3648
|
+
* that `remoteArray` now holds the id. The row appeared on reload only.
|
|
3649
|
+
*
|
|
3650
|
+
* Routed through the same per-query debounce as a real stream update, so it
|
|
3651
|
+
* cannot race one: a pending real update already materializes against the
|
|
3652
|
+
* current `remoteArray` and wins. The synthetic update re-uses the circuit's
|
|
3653
|
+
* last `localArray` and skips the persist/metrics that describe an ingest.
|
|
3654
|
+
*/
|
|
3655
|
+
scheduleRematerialize(queryHash) {
|
|
3656
|
+
if (this.debounceTimers.has(queryHash)) return;
|
|
3657
|
+
const queryState = this.activeQueries.get(queryHash);
|
|
3658
|
+
if (!queryState) return;
|
|
3659
|
+
this.queueStreamUpdate({
|
|
3660
|
+
queryHash,
|
|
3661
|
+
localArray: queryState.config.localArray ?? [],
|
|
3662
|
+
op: "UPDATE",
|
|
3663
|
+
synthetic: true
|
|
3664
|
+
});
|
|
3665
|
+
}
|
|
3666
|
+
/**
|
|
3554
3667
|
* Process a query's pending (debounced) stream update NOW instead of on the
|
|
3555
3668
|
* trailing edge. Called by the sync engine before it flips a query back to
|
|
3556
3669
|
* `idle`, so the status change never races ahead of the rows it fetched.
|
|
@@ -3655,10 +3768,14 @@ var DataModule = class DataModule {
|
|
|
3655
3768
|
pendingIdsAt = 0;
|
|
3656
3769
|
pendingIdsInflight = null;
|
|
3657
3770
|
static PENDING_IDS_TTL_MS = 250;
|
|
3771
|
+
pendingIdsGen = 0;
|
|
3772
|
+
static PENDING_IDS_MAX_REREADS = 3;
|
|
3658
3773
|
/** Drop the cached outbox ids. Cheap; call it on anything that could change
|
|
3659
3774
|
* `_00_pending_mutations`. */
|
|
3660
3775
|
invalidatePendingIds() {
|
|
3661
3776
|
this.pendingIds = null;
|
|
3777
|
+
this.pendingIdsGen++;
|
|
3778
|
+
this.pendingIdsInflight = null;
|
|
3662
3779
|
}
|
|
3663
3780
|
/**
|
|
3664
3781
|
* Grace period for a settled write. Long enough to cover an SSP round trip
|
|
@@ -3744,7 +3861,7 @@ var DataModule = class DataModule {
|
|
|
3744
3861
|
try {
|
|
3745
3862
|
const newRecords = await this.materializeRecords(queryState, localArray);
|
|
3746
3863
|
if (epoch !== this.local.epoch) return;
|
|
3747
|
-
queryState.config.localArray = localArray;
|
|
3864
|
+
if (!update.synthetic) queryState.config.localArray = localArray;
|
|
3748
3865
|
const prevJson = JSON.stringify(queryState.records);
|
|
3749
3866
|
const newJson = JSON.stringify(newRecords);
|
|
3750
3867
|
queryState.records = newRecords;
|
|
@@ -3753,7 +3870,7 @@ var DataModule = class DataModule {
|
|
|
3753
3870
|
queryState.updateCount++;
|
|
3754
3871
|
queryState.lastUpdatedAt = Date.now();
|
|
3755
3872
|
}
|
|
3756
|
-
await this.local.query(surql.seal(surql.updateSet("id", [
|
|
3873
|
+
if (!update.synthetic) await this.local.query(surql.seal(surql.updateSet("id", [
|
|
3757
3874
|
"localArray",
|
|
3758
3875
|
"rowCount",
|
|
3759
3876
|
"updateCount",
|
|
@@ -4089,18 +4206,27 @@ var DataModule = class DataModule {
|
|
|
4089
4206
|
writes: new Set(cached.writes),
|
|
4090
4207
|
deletes: new Set(cached.deletes)
|
|
4091
4208
|
};
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4209
|
+
let fresh = null;
|
|
4210
|
+
for (let attempt = 0; attempt < DataModule.PENDING_IDS_MAX_REREADS; attempt++) {
|
|
4211
|
+
const gen = this.pendingIdsGen;
|
|
4212
|
+
if (!this.pendingIdsInflight) {
|
|
4213
|
+
const read = this.readPendingRecordIds(gen).finally(() => {
|
|
4214
|
+
if (this.pendingIdsInflight === read) this.pendingIdsInflight = null;
|
|
4215
|
+
});
|
|
4216
|
+
this.pendingIdsInflight = read;
|
|
4217
|
+
}
|
|
4218
|
+
fresh = await this.pendingIdsInflight;
|
|
4219
|
+
if (gen === this.pendingIdsGen) break;
|
|
4220
|
+
}
|
|
4096
4221
|
return {
|
|
4097
4222
|
writes: new Set(fresh.writes),
|
|
4098
4223
|
deletes: new Set(fresh.deletes)
|
|
4099
4224
|
};
|
|
4100
4225
|
}
|
|
4101
4226
|
/** The uncached read. Also the reload path after an invalidation, so the ids
|
|
4102
|
-
* still survive a reload exactly as before.
|
|
4103
|
-
|
|
4227
|
+
* still survive a reload exactly as before. `gen` is the generation the read
|
|
4228
|
+
* was issued under; the result is cached only if it is still current. */
|
|
4229
|
+
async readPendingRecordIds(gen) {
|
|
4104
4230
|
const writes = /* @__PURE__ */ new Set();
|
|
4105
4231
|
const deletes = /* @__PURE__ */ new Set();
|
|
4106
4232
|
try {
|
|
@@ -4121,11 +4247,13 @@ var DataModule = class DataModule {
|
|
|
4121
4247
|
deletes
|
|
4122
4248
|
};
|
|
4123
4249
|
}
|
|
4124
|
-
this.
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4250
|
+
if (gen === this.pendingIdsGen) {
|
|
4251
|
+
this.pendingIds = {
|
|
4252
|
+
writes,
|
|
4253
|
+
deletes
|
|
4254
|
+
};
|
|
4255
|
+
this.pendingIdsAt = Date.now();
|
|
4256
|
+
}
|
|
4129
4257
|
return {
|
|
4130
4258
|
writes,
|
|
4131
4259
|
deletes
|
|
@@ -4430,13 +4558,22 @@ var DataModule = class DataModule {
|
|
|
4430
4558
|
data: params,
|
|
4431
4559
|
...prefixedParams
|
|
4432
4560
|
}));
|
|
4561
|
+
this.invalidatePendingIds();
|
|
4433
4562
|
const parsedRecord = parseParams(tableSchema.columns, target);
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4563
|
+
try {
|
|
4564
|
+
await this.cache.save({
|
|
4565
|
+
table: tableName,
|
|
4566
|
+
op: "CREATE",
|
|
4567
|
+
record: parsedRecord,
|
|
4568
|
+
version: 1
|
|
4569
|
+
}, true);
|
|
4570
|
+
} catch (err) {
|
|
4571
|
+
this.logger.error({
|
|
4572
|
+
err,
|
|
4573
|
+
id,
|
|
4574
|
+
Category: "sp00ky-client::DataModule::create"
|
|
4575
|
+
}, "SSP create-ingest failed; the row is written and queued, but views only see it once membership arrives");
|
|
4576
|
+
}
|
|
4440
4577
|
const mutationEvent = {
|
|
4441
4578
|
type: "create",
|
|
4442
4579
|
mutation_id: mutationId,
|
|
@@ -4445,7 +4582,6 @@ var DataModule = class DataModule {
|
|
|
4445
4582
|
record: target,
|
|
4446
4583
|
tableName
|
|
4447
4584
|
};
|
|
4448
|
-
this.invalidatePendingIds();
|
|
4449
4585
|
for (const callback of this.mutationCallbacks) callback([mutationEvent]);
|
|
4450
4586
|
this.logger.debug({
|
|
4451
4587
|
id,
|
|
@@ -4482,13 +4618,22 @@ var DataModule = class DataModule {
|
|
|
4482
4618
|
for (const key of Object.keys(data)) if (key in target) updatedFields[key] = target[key];
|
|
4483
4619
|
if ("_00_rv" in target) updatedFields._00_rv = target._00_rv;
|
|
4484
4620
|
this.replaceRecordInQueries(updatedFields);
|
|
4621
|
+
this.invalidatePendingIds();
|
|
4485
4622
|
const parsedRecord = parseParams(tableSchema.columns, target);
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4623
|
+
try {
|
|
4624
|
+
await this.cache.save({
|
|
4625
|
+
table,
|
|
4626
|
+
op: "UPDATE",
|
|
4627
|
+
record: parsedRecord,
|
|
4628
|
+
version: target._00_rv
|
|
4629
|
+
}, true);
|
|
4630
|
+
} catch (err) {
|
|
4631
|
+
this.logger.error({
|
|
4632
|
+
err,
|
|
4633
|
+
id,
|
|
4634
|
+
Category: "sp00ky-client::DataModule::update"
|
|
4635
|
+
}, "SSP update-ingest failed; the row is written and queued");
|
|
4636
|
+
}
|
|
4492
4637
|
const pushEventOptions = parseUpdateOptions(id, data, options);
|
|
4493
4638
|
const mutationEvent = {
|
|
4494
4639
|
type: "update",
|
|
@@ -4499,7 +4644,6 @@ var DataModule = class DataModule {
|
|
|
4499
4644
|
beforeRecord: beforeRecord || void 0,
|
|
4500
4645
|
options: pushEventOptions
|
|
4501
4646
|
};
|
|
4502
|
-
this.invalidatePendingIds();
|
|
4503
4647
|
for (const callback of this.mutationCallbacks) callback([mutationEvent]);
|
|
4504
4648
|
this.logger.debug({
|
|
4505
4649
|
id,
|
|
@@ -4522,6 +4666,7 @@ var DataModule = class DataModule {
|
|
|
4522
4666
|
id: rid,
|
|
4523
4667
|
mid: mutationId
|
|
4524
4668
|
}));
|
|
4669
|
+
this.invalidatePendingIds();
|
|
4525
4670
|
try {
|
|
4526
4671
|
await this.cache.delete(table, id, true, beforeRecord);
|
|
4527
4672
|
} catch (err) {
|
|
@@ -4537,7 +4682,6 @@ var DataModule = class DataModule {
|
|
|
4537
4682
|
mutation_id: mutationId,
|
|
4538
4683
|
record_id: rid
|
|
4539
4684
|
};
|
|
4540
|
-
this.invalidatePendingIds();
|
|
4541
4685
|
for (const callback of this.mutationCallbacks) callback([mutationEvent]);
|
|
4542
4686
|
this.logger.debug({
|
|
4543
4687
|
id,
|
|
@@ -4735,7 +4879,7 @@ var DataModule = class DataModule {
|
|
|
4735
4879
|
...configRecord,
|
|
4736
4880
|
id: recordId,
|
|
4737
4881
|
plan,
|
|
4738
|
-
params:
|
|
4882
|
+
params: parseQueryParams(tableSchema.columns, params ?? configRecord.params),
|
|
4739
4883
|
membershipKey
|
|
4740
4884
|
};
|
|
4741
4885
|
if (membershipKey && !config.remoteArray?.length) {
|
|
@@ -6481,6 +6625,7 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6481
6625
|
Category: "sp00ky-client::Sp00kySync::refetchListRefForQuery"
|
|
6482
6626
|
}, "syncQuery failed during poll");
|
|
6483
6627
|
}
|
|
6628
|
+
if (changed) this.dataModule.scheduleRematerialize(queryHash);
|
|
6484
6629
|
await this.syncSubqueryChildren(queryHash).catch((err) => {
|
|
6485
6630
|
this.logger.info({
|
|
6486
6631
|
err: err?.message ?? err,
|
|
@@ -6677,6 +6822,7 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6677
6822
|
}
|
|
6678
6823
|
await this.runSyncForQuery(hash, diff);
|
|
6679
6824
|
if (diff.removed.length > 0 && diff.added.length === 0 && diff.updated.length === 0) await this.dataModule.notifyQuerySynced(hash);
|
|
6825
|
+
else if (diff.added.length === 0 && diff.updated.length === 0) this.dataModule.scheduleRematerialize(hash);
|
|
6680
6826
|
}
|
|
6681
6827
|
/**
|
|
6682
6828
|
* Handle a LIVE change to a SUBQUERY child edge (a `_00_list_ref` row with
|
|
@@ -7414,8 +7560,8 @@ function selfAllowlistedVariant(flag, userId) {
|
|
|
7414
7560
|
|
|
7415
7561
|
//#endregion
|
|
7416
7562
|
//#region src/modules/devtools/index.ts
|
|
7417
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
7418
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
7563
|
+
const CORE_VERSION = "0.0.1-canary.209";
|
|
7564
|
+
const WASM_VERSION = "0.0.1-canary.209";
|
|
7419
7565
|
const SURREAL_VERSION = "3.0.3";
|
|
7420
7566
|
var DevToolsService = class DevToolsService {
|
|
7421
7567
|
eventsHistory = [];
|
|
@@ -7426,6 +7572,10 @@ var DevToolsService = class DevToolsService {
|
|
|
7426
7572
|
static NOTIFY_MIN_INTERVAL_MS = 250;
|
|
7427
7573
|
notifyTimer = null;
|
|
7428
7574
|
lastNotifyAt = 0;
|
|
7575
|
+
/** How many ids a pushed state carries per view; the rest is on demand. */
|
|
7576
|
+
static STATE_IDS_CAP = 200;
|
|
7577
|
+
/** devtools numeric hash -> the query's `_00_query` id, for on-demand rows. */
|
|
7578
|
+
hashToQuery = /* @__PURE__ */ new Map();
|
|
7429
7579
|
/** Shared-tabs snapshot for the panel, wired by Sp00kyClient whenever the
|
|
7430
7580
|
* feature was REQUESTED (so an inactive/degraded tab still reports why). */
|
|
7431
7581
|
tabsInfoProvider = null;
|
|
@@ -7466,6 +7616,7 @@ var DevToolsService = class DevToolsService {
|
|
|
7466
7616
|
const type = e.data?.type;
|
|
7467
7617
|
if (type === "SP00KY_DEVTOOLS_CONNECT") {
|
|
7468
7618
|
this.enabled = true;
|
|
7619
|
+
this.refreshLocalTables();
|
|
7469
7620
|
this.notifyDevTools();
|
|
7470
7621
|
} else if (type === "SP00KY_DEVTOOLS_DISCONNECT") this.enabled = false;
|
|
7471
7622
|
});
|
|
@@ -7501,6 +7652,9 @@ var DevToolsService = class DevToolsService {
|
|
|
7501
7652
|
this.dataManager.getActiveQueries().forEach((q) => {
|
|
7502
7653
|
const queryHash = this.hashString(encodeRecordId(q.config.id));
|
|
7503
7654
|
const createdAt = q.config.lastActiveAt instanceof Date ? q.config.lastActiveAt.getTime() : new Date(q.config.lastActiveAt || Date.now()).getTime();
|
|
7655
|
+
this.hashToQuery.set(queryHash, q.config.id);
|
|
7656
|
+
const localArray = q.config.localArray ?? [];
|
|
7657
|
+
const remoteArray = q.config.remoteArray ?? [];
|
|
7504
7658
|
result.set(queryHash, {
|
|
7505
7659
|
queryHash,
|
|
7506
7660
|
status: "active",
|
|
@@ -7513,9 +7667,11 @@ var DevToolsService = class DevToolsService {
|
|
|
7513
7667
|
query: q.config.surql,
|
|
7514
7668
|
variables: q.config.params || {},
|
|
7515
7669
|
dataSize: q.records?.length || 0,
|
|
7516
|
-
|
|
7517
|
-
|
|
7518
|
-
|
|
7670
|
+
localCount: localArray.length,
|
|
7671
|
+
remoteCount: remoteArray.length,
|
|
7672
|
+
localIds: localArray.slice(0, DevToolsService.STATE_IDS_CAP).map(([id]) => id),
|
|
7673
|
+
remoteIds: remoteArray.slice(0, DevToolsService.STATE_IDS_CAP).map(([id]) => id),
|
|
7674
|
+
idsTruncated: localArray.length > DevToolsService.STATE_IDS_CAP || remoteArray.length > DevToolsService.STATE_IDS_CAP,
|
|
7519
7675
|
membershipKnown: q.config.membershipKnown === true,
|
|
7520
7676
|
remoteSeen: q.config.remoteSeen === true,
|
|
7521
7677
|
emptyReads: q.config.emptyReads ?? 0,
|
|
@@ -7545,23 +7701,32 @@ var DevToolsService = class DevToolsService {
|
|
|
7545
7701
|
const queryHash = this.hashString(payload.queryId.toString());
|
|
7546
7702
|
this.addEvent("QUERY_UPDATED", {
|
|
7547
7703
|
queryHash,
|
|
7548
|
-
|
|
7704
|
+
recordCount: Array.isArray(payload.records) ? payload.records.length : 0
|
|
7549
7705
|
});
|
|
7550
7706
|
this.notifyDevTools();
|
|
7551
7707
|
}
|
|
7552
7708
|
onStreamUpdate(update) {
|
|
7709
|
+
if (update.synthetic) return;
|
|
7553
7710
|
this.logger.debug({
|
|
7554
|
-
update,
|
|
7711
|
+
queryHash: update.queryHash,
|
|
7555
7712
|
Category: "sp00ky-client::DevToolsService::onStreamUpdate"
|
|
7556
7713
|
}, "StreamUpdate");
|
|
7557
|
-
this.addEvent("STREAM_UPDATE", {
|
|
7714
|
+
this.addEvent("STREAM_UPDATE", {
|
|
7715
|
+
queryHash: update.queryHash,
|
|
7716
|
+
op: update.op,
|
|
7717
|
+
localCount: update.localArray?.length ?? 0,
|
|
7718
|
+
materializationTimeMs: update.materializationTimeMs,
|
|
7719
|
+
storeApplyMs: update.storeApplyMs,
|
|
7720
|
+
circuitStepMs: update.circuitStepMs,
|
|
7721
|
+
transformMs: update.transformMs
|
|
7722
|
+
});
|
|
7558
7723
|
this.notifyDevTools();
|
|
7559
7724
|
}
|
|
7560
7725
|
onMutation(payload) {
|
|
7561
7726
|
payload.forEach((p) => {
|
|
7562
7727
|
this.addEvent("MUTATION_REQUEST_EXECUTION", { mutation: {
|
|
7563
|
-
type: "create",
|
|
7564
|
-
|
|
7728
|
+
type: p.type ?? "create",
|
|
7729
|
+
fields: "data" in p && p.data && typeof p.data === "object" ? Object.keys(p.data) : [],
|
|
7565
7730
|
selector: encodeRecordId(p.record_id)
|
|
7566
7731
|
} });
|
|
7567
7732
|
});
|
|
@@ -7606,7 +7771,7 @@ var DevToolsService = class DevToolsService {
|
|
|
7606
7771
|
*/
|
|
7607
7772
|
refreshLocalTables() {
|
|
7608
7773
|
if (this.localTablesFetching) return;
|
|
7609
|
-
if (Date.now() - this.localTablesAt <
|
|
7774
|
+
if (Date.now() - this.localTablesAt < 3e4) return;
|
|
7610
7775
|
this.localTablesFetching = true;
|
|
7611
7776
|
this.databaseService.query("INFO FOR DB").then((res) => {
|
|
7612
7777
|
const info = this.unwrapInfo(res);
|
|
@@ -7621,8 +7786,8 @@ var DevToolsService = class DevToolsService {
|
|
|
7621
7786
|
this.localTablesFetching = false;
|
|
7622
7787
|
});
|
|
7623
7788
|
}
|
|
7624
|
-
getState() {
|
|
7625
|
-
this.refreshLocalTables();
|
|
7789
|
+
getState(opts = {}) {
|
|
7790
|
+
if (opts.refreshTables) this.refreshLocalTables();
|
|
7626
7791
|
return this.serializeForDevTools({
|
|
7627
7792
|
eventsHistory: [...this.eventsHistory],
|
|
7628
7793
|
activeQueries: Object.fromEntries(this.getActiveQueries()),
|
|
@@ -7736,14 +7901,11 @@ var DevToolsService = class DevToolsService {
|
|
|
7736
7901
|
if (typeof window === "undefined") return;
|
|
7737
7902
|
if (this.notifyTimer !== null) return;
|
|
7738
7903
|
const waited = Date.now() - this.lastNotifyAt;
|
|
7739
|
-
|
|
7740
|
-
this.flushNotify();
|
|
7741
|
-
return;
|
|
7742
|
-
}
|
|
7904
|
+
const delay = Math.max(0, DevToolsService.NOTIFY_MIN_INTERVAL_MS - waited);
|
|
7743
7905
|
this.notifyTimer = setTimeout(() => {
|
|
7744
7906
|
this.notifyTimer = null;
|
|
7745
7907
|
if (this.enabled) this.flushNotify();
|
|
7746
|
-
},
|
|
7908
|
+
}, delay);
|
|
7747
7909
|
}
|
|
7748
7910
|
flushNotify() {
|
|
7749
7911
|
this.lastNotifyAt = Date.now();
|
|
@@ -7788,7 +7950,18 @@ var DevToolsService = class DevToolsService {
|
|
|
7788
7950
|
if (typeof window !== "undefined") {
|
|
7789
7951
|
window.__00__ = {
|
|
7790
7952
|
version: this.version,
|
|
7791
|
-
getState: () => this.getState(),
|
|
7953
|
+
getState: () => this.getState({ refreshTables: true }),
|
|
7954
|
+
getQueryRows: (queryHash) => {
|
|
7955
|
+
const id = this.hashToQuery.get(Number(queryHash));
|
|
7956
|
+
const q = id !== void 0 ? this.dataManager?.getQueryById(id) : void 0;
|
|
7957
|
+
if (!q) return null;
|
|
7958
|
+
return this.serializeForDevTools({
|
|
7959
|
+
queryHash: Number(queryHash),
|
|
7960
|
+
data: q.records,
|
|
7961
|
+
localArray: q.config.localArray,
|
|
7962
|
+
remoteArray: q.config.remoteArray
|
|
7963
|
+
});
|
|
7964
|
+
},
|
|
7792
7965
|
getFlags: () => this.flagsAdmin.getFlags(),
|
|
7793
7966
|
setFlagEnabled: (key, enabled) => this.flagsAdmin.setFlagEnabled(key, enabled),
|
|
7794
7967
|
setFlagUserVariant: (key, variant, remove, userId) => this.flagsAdmin.setFlagUserVariant(key, variant, remove, userId),
|
|
@@ -12399,7 +12572,7 @@ var Sp00kyClient = class {
|
|
|
12399
12572
|
return new TabsCoordinator({
|
|
12400
12573
|
tabId,
|
|
12401
12574
|
fingerprint: computeTabsFingerprint({
|
|
12402
|
-
coreVersion: "0.0.1-canary.
|
|
12575
|
+
coreVersion: "0.0.1-canary.209",
|
|
12403
12576
|
schemaHash: hash53(this.config.schemaSurql),
|
|
12404
12577
|
endpoint: this.config.database.endpoint ?? "",
|
|
12405
12578
|
namespace: this.config.database.namespace,
|
|
@@ -12836,7 +13009,7 @@ var Sp00kyClient = class {
|
|
|
12836
13009
|
async initQuery(table, q, ttl) {
|
|
12837
13010
|
const tableSchema = this.config.schema.tables.find((t) => t.name === table);
|
|
12838
13011
|
if (!tableSchema) throw new Error(`Table ${table} not found`);
|
|
12839
|
-
const params =
|
|
13012
|
+
const params = parseQueryParams(tableSchema.columns, q.selectQuery.vars ?? {});
|
|
12840
13013
|
const hash = await this.dataModule.query(table, q.selectQuery.query, params, ttl, q.selectQuery.plan);
|
|
12841
13014
|
if (!this.pendingQueryInits.has(hash)) {
|
|
12842
13015
|
const chain = this.finishQueryInit(hash, q, params).finally(() => {
|
|
@@ -12908,7 +13081,7 @@ var Sp00kyClient = class {
|
|
|
12908
13081
|
const tableName = q.tableName;
|
|
12909
13082
|
const tableSchema = this.config.schema.tables.find((t) => t.name === tableName);
|
|
12910
13083
|
if (!tableSchema) throw new Error(`Table ${tableName} not found`);
|
|
12911
|
-
const params =
|
|
13084
|
+
const params = parseQueryParams(tableSchema.columns, q.selectQuery.vars ?? {});
|
|
12912
13085
|
const hashKey = String(q.hash);
|
|
12913
13086
|
const marker = await this.dataModule.getPreloadMarker(hashKey);
|
|
12914
13087
|
if (!marker) {
|
|
@@ -13085,4 +13258,4 @@ var Sp00kyClient = class {
|
|
|
13085
13258
|
};
|
|
13086
13259
|
|
|
13087
13260
|
//#endregion
|
|
13088
|
-
export { AppReleaseHandle, AppReleaseModule, AuthEventTypes, AuthService, BLURHASH_IMAGE_EXTENSIONS, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, FeatureFlagHandle, FeatureFlagModule, MATERIALIZATION_SAMPLE_WINDOW, Sp00kyClient, blurhashSidecarPath, bucketContentToBlob, createAuthEventSystem, cursorColorFromName, decode as decodeBlurhash, encode as encodeBlurhash, encodeImageToBlurhash, fileToUint8Array, isBlurhashValid, isImagePath, semverGt, textToHtml };
|
|
13261
|
+
export { AppReleaseHandle, AppReleaseModule, AuthEventTypes, AuthService, BLURHASH_IMAGE_EXTENSIONS, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DEFAULT_LOCAL_OP_TIMEOUT_MS, FeatureFlagHandle, FeatureFlagModule, LocalOpTimeoutError, MATERIALIZATION_SAMPLE_WINDOW, Sp00kyClient, blurhashSidecarPath, bucketContentToBlob, createAuthEventSystem, cursorColorFromName, decode as decodeBlurhash, encode as encodeBlurhash, encodeImageToBlurhash, fileToUint8Array, isBlurhashValid, isImagePath, semverGt, textToHtml };
|
package/dist/sqlite-worker.js
CHANGED
|
@@ -1,6 +1,26 @@
|
|
|
1
1
|
import { a as renderOrderSql, i as projectedDataSql, l as resolveRelations, n as openDb, o as renderWhereSql, r as project, s as reviveRow, u as stableKey } from "./sqlite-open.js";
|
|
2
2
|
import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
|
|
3
3
|
|
|
4
|
+
//#region src/services/database/sqlite-lock-verify.ts
|
|
5
|
+
const LOCK_VERIFY_TIMEOUT_MS = 2e3;
|
|
6
|
+
async function verifyLockStillHeld$1(locks, name, fence, timeoutMs = LOCK_VERIFY_TIMEOUT_MS, warn = () => {}) {
|
|
7
|
+
if (!locks || !name || typeof locks.query !== "function") return;
|
|
8
|
+
let timer;
|
|
9
|
+
try {
|
|
10
|
+
const state = await Promise.race([locks.query(), new Promise((resolve) => {
|
|
11
|
+
timer = setTimeout(() => resolve(null), timeoutMs);
|
|
12
|
+
})]);
|
|
13
|
+
if (state === null) {
|
|
14
|
+
warn(`lock verification did not answer within ${timeoutMs}ms; trusting the steal callback`);
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
if (!(state.held ?? []).some((l) => l.name === name)) await fence("lock missing after suspected freeze");
|
|
18
|
+
} catch {} finally {
|
|
19
|
+
if (timer) clearTimeout(timer);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
//#endregion
|
|
4
24
|
//#region src/services/database/sqlite-select.ts
|
|
5
25
|
function ensureTable(db, table) {
|
|
6
26
|
if (db.knownTables.has(table)) return;
|
|
@@ -230,13 +250,8 @@ function noteTick() {
|
|
|
230
250
|
});
|
|
231
251
|
}
|
|
232
252
|
setInterval(noteTick, 1e3);
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
const name = currentLock?.name;
|
|
236
|
-
if (!locks || !name || typeof locks.query !== "function") return;
|
|
237
|
-
try {
|
|
238
|
-
if (!((await locks.query()).held ?? []).some((l) => l.name === name)) await fence("lock missing after suspected freeze");
|
|
239
|
-
} catch {}
|
|
253
|
+
function verifyLockStillHeld() {
|
|
254
|
+
return verifyLockStillHeld$1(workerLocks(), currentLock?.name, (reason) => fence(reason), void 0, (msg) => console.warn(`[sqlite-worker] ${msg}`));
|
|
240
255
|
}
|
|
241
256
|
async function open(dbName, useOpfs, systemTables = [], workerLockName, openOptions) {
|
|
242
257
|
if (workerLockName) {
|
package/dist/types.d.ts
CHANGED
|
@@ -504,6 +504,16 @@ interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
504
504
|
* failure the queue retries. `0` disables. Defaults to `60_000`.
|
|
505
505
|
*/
|
|
506
506
|
queryTimeoutMs?: number;
|
|
507
|
+
/**
|
|
508
|
+
* Deadline (ms) for every LOCAL store operation (a SQLite worker round trip,
|
|
509
|
+
* or a query on the in-process surrealdb engine). Local ops are serialized
|
|
510
|
+
* too, and one that never answered - a worker starved behind a long select,
|
|
511
|
+
* a lock verification with no clock - left every `db.create`/`db.update`
|
|
512
|
+
* promise pending for the tab's lifetime. On expiry the call rejects with
|
|
513
|
+
* `LocalOpTimeoutError` (the op itself keeps running in the engine and is
|
|
514
|
+
* not retried). `0` disables. Defaults to `30_000`.
|
|
515
|
+
*/
|
|
516
|
+
localOpTimeoutMs?: number;
|
|
507
517
|
};
|
|
508
518
|
/** The schema definition. */
|
|
509
519
|
schema: S;
|