@spooky-sync/core 0.0.1-canary.108 → 0.0.1-canary.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +10 -114
- package/dist/index.js +1070 -46
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +107 -0
- package/dist/types.d.ts +157 -2
- package/package.json +4 -3
- package/src/modules/cache/index.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +3 -3
- package/src/modules/crdt/crdt-hydration.test.ts +5 -5
- package/src/modules/crdt/index.ts +2 -2
- package/src/modules/data/index.ts +45 -17
- package/src/modules/data/window-query.ts +24 -0
- package/src/modules/devtools/index.ts +2 -2
- package/src/modules/sync/queue/queue-down.ts +2 -2
- package/src/modules/sync/queue/queue-up.ts +2 -2
- package/src/modules/sync/sync.ts +2 -2
- package/src/services/database/cache-engine.ts +143 -0
- package/src/services/database/engine-factory.ts +32 -0
- package/src/services/database/index.ts +6 -0
- package/src/services/database/local-migrator.ts +2 -2
- package/src/services/database/plan-render.test.ts +85 -0
- package/src/services/database/plan-render.ts +86 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/sqlite-cache-engine.ts +631 -0
- package/src/services/database/sqlite-worker.ts +116 -0
- package/src/services/database/surql-translate.ts +291 -0
- package/src/services/database/surreal-cache-engine.ts +122 -0
- package/src/services/persistence/surrealdb.ts +2 -2
- package/src/services/stream-processor/index.ts +2 -2
- package/src/sp00ky.ts +24 -7
- package/src/types.ts +17 -1
- package/tsdown.config.ts +10 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
|
|
2
|
+
|
|
3
|
+
//#region src/services/database/sqlite-worker.ts
|
|
4
|
+
/**
|
|
5
|
+
* Dedicated Web Worker that owns the SQLite-WASM handle. All DB access is
|
|
6
|
+
* funnelled through here (the main thread never touches the wasm module), which
|
|
7
|
+
* is also what the OPFS VFS requires — file access must happen off the main
|
|
8
|
+
* thread. Persistence uses the **OPFS SAHPool VFS**: durable, and (unlike the
|
|
9
|
+
* classic OPFS VFS) it does NOT require COOP/COEP cross-origin isolation
|
|
10
|
+
* headers, so host apps embedding the client need no server changes. Falls back
|
|
11
|
+
* to an in-memory DB when OPFS is unavailable.
|
|
12
|
+
*
|
|
13
|
+
* Message protocol (request/response keyed by `id`):
|
|
14
|
+
* { id, type: 'open', payload: { dbName, useOpfs } }
|
|
15
|
+
* { id, type: 'exec', payload: { sql, bind } } -> { id, ok, rows }
|
|
16
|
+
* { id, type: 'run', payload: { sql, bind } } -> { id, ok }
|
|
17
|
+
* { id, type: 'batch', payload: [{ sql, bind }] } (atomic BEGIN/COMMIT)
|
|
18
|
+
* { id, type: 'close' }
|
|
19
|
+
*/
|
|
20
|
+
let db = null;
|
|
21
|
+
async function open(dbName, useOpfs) {
|
|
22
|
+
const sqlite3 = await sqlite3InitModule();
|
|
23
|
+
let persisted = false;
|
|
24
|
+
if (useOpfs && sqlite3.installOpfsSAHPoolVfs) try {
|
|
25
|
+
db = new (await (sqlite3.installOpfsSAHPoolVfs({ name: `sp00ky-${dbName}` }))).OpfsSAHPoolDb(`/${dbName}.sqlite3`);
|
|
26
|
+
persisted = true;
|
|
27
|
+
} catch {}
|
|
28
|
+
if (!db) db = new sqlite3.oo1.DB(":memory:", "c");
|
|
29
|
+
try {
|
|
30
|
+
db.exec({ sql: "PRAGMA busy_timeout = 5000; PRAGMA cache_size = -32000;" });
|
|
31
|
+
} catch {}
|
|
32
|
+
return { persisted };
|
|
33
|
+
}
|
|
34
|
+
function exec(sql, bind) {
|
|
35
|
+
if (!db) throw new Error("sqlite: DB not open");
|
|
36
|
+
return db.exec({
|
|
37
|
+
sql,
|
|
38
|
+
bind,
|
|
39
|
+
rowMode: "object",
|
|
40
|
+
returnValue: "resultRows"
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
function run(sql, bind) {
|
|
44
|
+
if (!db) throw new Error("sqlite: DB not open");
|
|
45
|
+
db.exec({
|
|
46
|
+
sql,
|
|
47
|
+
bind
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
function batch(stmts) {
|
|
51
|
+
if (!db) throw new Error("sqlite: DB not open");
|
|
52
|
+
db.exec({ sql: "BEGIN" });
|
|
53
|
+
try {
|
|
54
|
+
for (const s of stmts) db.exec({
|
|
55
|
+
sql: s.sql,
|
|
56
|
+
bind: s.bind
|
|
57
|
+
});
|
|
58
|
+
db.exec({ sql: "COMMIT" });
|
|
59
|
+
} catch (e) {
|
|
60
|
+
try {
|
|
61
|
+
db.exec({ sql: "ROLLBACK" });
|
|
62
|
+
} catch {}
|
|
63
|
+
throw e;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
self.onmessage = async (ev) => {
|
|
67
|
+
const { id, type, payload } = ev.data ?? {};
|
|
68
|
+
try {
|
|
69
|
+
let result;
|
|
70
|
+
switch (type) {
|
|
71
|
+
case "open":
|
|
72
|
+
result = await open(payload.dbName, payload.useOpfs);
|
|
73
|
+
break;
|
|
74
|
+
case "exec":
|
|
75
|
+
result = { rows: exec(payload.sql, payload.bind) };
|
|
76
|
+
break;
|
|
77
|
+
case "run":
|
|
78
|
+
run(payload.sql, payload.bind);
|
|
79
|
+
result = {};
|
|
80
|
+
break;
|
|
81
|
+
case "batch":
|
|
82
|
+
batch(payload);
|
|
83
|
+
result = {};
|
|
84
|
+
break;
|
|
85
|
+
case "close":
|
|
86
|
+
db?.close();
|
|
87
|
+
db = null;
|
|
88
|
+
result = {};
|
|
89
|
+
break;
|
|
90
|
+
default: throw new Error(`sqlite worker: unknown message ${type}`);
|
|
91
|
+
}
|
|
92
|
+
self.postMessage({
|
|
93
|
+
id,
|
|
94
|
+
ok: true,
|
|
95
|
+
...result
|
|
96
|
+
});
|
|
97
|
+
} catch (err) {
|
|
98
|
+
self.postMessage({
|
|
99
|
+
id,
|
|
100
|
+
ok: false,
|
|
101
|
+
error: err instanceof Error ? err.message : String(err)
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
//#endregion
|
|
107
|
+
export { };
|
package/dist/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { RecordId } from "surrealdb";
|
|
2
|
-
import { RecordId as RecordId$1, SchemaStructure } from "@spooky-sync/query-builder";
|
|
2
|
+
import { QueryPlan, QueryPlan as QueryPlan$1, RecordId as RecordId$1, SchemaStructure, WhereNode } from "@spooky-sync/query-builder";
|
|
3
3
|
import { Level, Level as Level$1, Logger, LoggerOptions } from "pino";
|
|
4
4
|
|
|
5
5
|
//#region src/events/index.d.ts
|
|
@@ -154,6 +154,146 @@ type SyncEventSystem = EventSystem<SyncEventTypeMap>;
|
|
|
154
154
|
//#region src/services/logger/index.d.ts
|
|
155
155
|
type Logger$1 = Logger;
|
|
156
156
|
//#endregion
|
|
157
|
+
//#region src/services/database/events/index.d.ts
|
|
158
|
+
declare const DatabaseEventTypes: {
|
|
159
|
+
readonly LocalQuery: "DATABASE_LOCAL_QUERY";
|
|
160
|
+
readonly RemoteQuery: "DATABASE_REMOTE_QUERY";
|
|
161
|
+
};
|
|
162
|
+
interface DatabaseQueryEventPayload {
|
|
163
|
+
query: string;
|
|
164
|
+
vars?: Record<string, unknown>;
|
|
165
|
+
duration: number;
|
|
166
|
+
success: boolean;
|
|
167
|
+
error?: string;
|
|
168
|
+
timestamp: number;
|
|
169
|
+
}
|
|
170
|
+
type DatabaseEventTypeMap = {
|
|
171
|
+
[DatabaseEventTypes.LocalQuery]: EventDefinition<typeof DatabaseEventTypes.LocalQuery, DatabaseQueryEventPayload>;
|
|
172
|
+
[DatabaseEventTypes.RemoteQuery]: EventDefinition<typeof DatabaseEventTypes.RemoteQuery, DatabaseQueryEventPayload>;
|
|
173
|
+
};
|
|
174
|
+
type DatabaseEventSystem = EventSystem<DatabaseEventTypeMap>;
|
|
175
|
+
//#endregion
|
|
176
|
+
//#region src/utils/surql.d.ts
|
|
177
|
+
interface SealedQuery<T = void> {
|
|
178
|
+
readonly sql: string;
|
|
179
|
+
readonly extract: (results: unknown[]) => T;
|
|
180
|
+
}
|
|
181
|
+
//#endregion
|
|
182
|
+
//#region src/services/database/cache-engine.d.ts
|
|
183
|
+
/**
|
|
184
|
+
* A materialized row. Keys are field names; values are already decoded to the
|
|
185
|
+
* client's runtime shapes (RecordId stays a RecordId, bytes a Uint8Array, …) so
|
|
186
|
+
* every backend hands `DataModule` the same shape SurrealDB does today.
|
|
187
|
+
*/
|
|
188
|
+
type Row = Record<string, unknown>;
|
|
189
|
+
/** A record identifier — a `RecordId` or its stable string form (`table:id`). */
|
|
190
|
+
type Id = unknown;
|
|
191
|
+
/** How an order clause is expressed everywhere in the engine layer. */
|
|
192
|
+
type OrderBy = [field: string, direction: 'asc' | 'desc'][];
|
|
193
|
+
/**
|
|
194
|
+
* Batched relation fetch: "give me every row of `table` whose `matchField` is
|
|
195
|
+
* one of `keys`, filtered by `where`, ordered by `orderBy`". This is the single
|
|
196
|
+
* primitive relation decomposition (§3) leans on — implemented as
|
|
197
|
+
* `SELECT … WHERE <matchField> IN (…)` on SQLite, `SELECT … FROM $keys` /
|
|
198
|
+
* `WHERE <matchField> IN $keys` on SurrealDB, or an index scan elsewhere. Order
|
|
199
|
+
* here is a hint; the resolver re-applies order+limit PER PARENT after grouping.
|
|
200
|
+
*/
|
|
201
|
+
interface RelationFetch {
|
|
202
|
+
table: string;
|
|
203
|
+
matchField: string;
|
|
204
|
+
keys: Id[];
|
|
205
|
+
where?: WhereNode[];
|
|
206
|
+
orderBy?: OrderBy;
|
|
207
|
+
select?: string[];
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* The read side of an engine, minus relation resolution — the surface a
|
|
211
|
+
* {@link RelationResolver} needs. Kept separate so the resolver can be unit
|
|
212
|
+
* tested against an in-memory fake without a full engine.
|
|
213
|
+
*/
|
|
214
|
+
interface RowFetcher {
|
|
215
|
+
/** Batched fan-out fetch. See {@link RelationFetch}. */
|
|
216
|
+
fetchRelation(req: RelationFetch): Promise<Row[]>;
|
|
217
|
+
}
|
|
218
|
+
/** A transaction handle — the same verbs as the engine, but atomic. */
|
|
219
|
+
interface EngineTx {
|
|
220
|
+
upsert(table: string, id: Id, data: Row, mode: 'replace' | 'merge'): Promise<void>;
|
|
221
|
+
patch(table: string, id: Id, patches: unknown[]): Promise<void>;
|
|
222
|
+
delete(table: string, id: Id): Promise<void>;
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* A pluggable local cache backend. SurrealDB (the default) and SQLite both
|
|
226
|
+
* implement this; the rest of the client talks verbs, never SurrealQL.
|
|
227
|
+
*
|
|
228
|
+
* Reactivity is NOT part of this contract: the local cache is passive. The SSP
|
|
229
|
+
* (remote) drives change; `DataModule` writes rows here and re-reads them. The
|
|
230
|
+
* `epoch` field preserves the existing bucket-switch fencing (see
|
|
231
|
+
* `LocalDatabaseService.epoch`): an async chain captures it at start and its
|
|
232
|
+
* write is dropped if the epoch moved (a bucket switch) in between.
|
|
233
|
+
*/
|
|
234
|
+
interface LocalCacheEngine extends RowFetcher {
|
|
235
|
+
/** Monotonic store generation; bumped on every bucket switch. */
|
|
236
|
+
readonly epoch: number;
|
|
237
|
+
connect(bucketId: string): Promise<void>;
|
|
238
|
+
switchBucket(bucketId: string): Promise<void>;
|
|
239
|
+
close(): Promise<void>;
|
|
240
|
+
/** Run `fn` inside a single atomic transaction. */
|
|
241
|
+
transaction<T>(fn: (tx: EngineTx) => Promise<T>): Promise<T>;
|
|
242
|
+
/**
|
|
243
|
+
* Materialize a query, including its `.related()` tree (via §3
|
|
244
|
+
* decomposition). Params bind `where` `paramRef`s and any windowing id-set.
|
|
245
|
+
*/
|
|
246
|
+
select(plan: QueryPlan$1, params?: Record<string, unknown>): Promise<Row[]>;
|
|
247
|
+
/** Fetch rows by primary id, preserving `ids` order; missing ids are skipped. */
|
|
248
|
+
selectByIds(table: string, ids: Id[], opts?: {
|
|
249
|
+
select?: string[];
|
|
250
|
+
orderBy?: OrderBy;
|
|
251
|
+
}): Promise<Row[]>;
|
|
252
|
+
/** Single-record read by primary id, or `null`. */
|
|
253
|
+
getById(table: string, id: Id): Promise<Row | null>;
|
|
254
|
+
upsert(table: string, id: Id, data: Row, mode: 'replace' | 'merge'): Promise<void>;
|
|
255
|
+
patch(table: string, id: Id, patches: unknown[]): Promise<void>;
|
|
256
|
+
delete(table: string, id: Id): Promise<void>;
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* The full surface the client's `this.local` field depends on: the
|
|
260
|
+
* engine-neutral {@link LocalCacheEngine} verbs PLUS the legacy
|
|
261
|
+
* SurrealQL/lifecycle methods the not-yet-migrated call sites still use.
|
|
262
|
+
* `SurrealCacheEngine` (subclass of `LocalDatabaseService`) and
|
|
263
|
+
* `SqliteCacheEngine` (via a SurrealQL-vocabulary shim) both satisfy this, so
|
|
264
|
+
* either can back `this.local`.
|
|
265
|
+
*
|
|
266
|
+
* `getClient()` returns the underlying SurrealDB `Surreal` handle where one
|
|
267
|
+
* exists (SurrealDB backend); backends without one (SQLite) throw — it is only
|
|
268
|
+
* used by advanced/DevTools paths, never on the hot path.
|
|
269
|
+
*/
|
|
270
|
+
interface LocalStore extends LocalCacheEngine {
|
|
271
|
+
/**
|
|
272
|
+
* Whether this engine needs SurrealQL schema provisioning (`DEFINE TABLE`,
|
|
273
|
+
* `DEFINE FIELD`, …) run against it at init / bucket switch. SurrealDB → true;
|
|
274
|
+
* schemaless engines (SQLite creates tables lazily) → false, so the client
|
|
275
|
+
* skips the `LocalMigrator` entirely for them.
|
|
276
|
+
*/
|
|
277
|
+
readonly usesSurqlSchema: boolean;
|
|
278
|
+
query<T extends unknown[]>(query: string, vars?: Record<string, unknown>, opts?: {
|
|
279
|
+
epoch?: number;
|
|
280
|
+
}): Promise<T>;
|
|
281
|
+
execute<T>(query: SealedQuery<T>, vars?: Record<string, unknown>, opts?: {
|
|
282
|
+
epoch?: number;
|
|
283
|
+
}): Promise<T>;
|
|
284
|
+
queryUngated<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T>;
|
|
285
|
+
switchStore(bucketId: string): Promise<void>;
|
|
286
|
+
beginSwitch(): () => void;
|
|
287
|
+
getEvents(): DatabaseEventSystem;
|
|
288
|
+
getClient(): unknown;
|
|
289
|
+
getConfig(): Sp00kyConfig<any>['database'];
|
|
290
|
+
readonly currentBucketId: string;
|
|
291
|
+
}
|
|
292
|
+
/** Selected local cache backend. Mirrors the `persistenceClient` config pattern. */
|
|
293
|
+
type LocalEngineChoice = 'surrealdb' | 'sqlite' | LocalStore;
|
|
294
|
+
/** Thrown when relation decomposition nests past {@link MAX_RELATION_DEPTH} —
|
|
295
|
+
* a guard against a cyclic schema producing unbounded fan-out. */
|
|
296
|
+
//#endregion
|
|
157
297
|
//#region src/modules/sync/queue/queue-up.d.ts
|
|
158
298
|
type CreateEvent = {
|
|
159
299
|
type: 'create';
|
|
@@ -260,6 +400,14 @@ interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
260
400
|
* Can be a custom implementation, 'surrealdb' (default), or 'localstorage'.
|
|
261
401
|
*/
|
|
262
402
|
persistenceClient?: PersistenceClient | 'surrealdb' | 'localstorage';
|
|
403
|
+
/**
|
|
404
|
+
* Local cache engine backend. `'surrealdb'` (default) uses the in-browser
|
|
405
|
+
* SurrealDB-WASM store; `'sqlite'` uses official SQLite-WASM in a Worker with
|
|
406
|
+
* OPFS persistence; or pass a custom {@link LocalCacheEngine}. The local cache
|
|
407
|
+
* is a passive queryable store — reactivity is driven by the remote SSP, not
|
|
408
|
+
* this engine. See `services/database/cache-engine.ts`.
|
|
409
|
+
*/
|
|
410
|
+
localEngine?: LocalEngineChoice;
|
|
263
411
|
/** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
|
|
264
412
|
otelTransmit?: PinoTransmit;
|
|
265
413
|
/**
|
|
@@ -377,6 +525,13 @@ interface QueryConfig {
|
|
|
377
525
|
id: RecordId$1<string>;
|
|
378
526
|
/** The SURQL query string. */
|
|
379
527
|
surql: string;
|
|
528
|
+
/**
|
|
529
|
+
* Engine-neutral plan for `surql` (in-memory only; not persisted to
|
|
530
|
+
* `_00_query`). Present when the query came from the query-builder. Non-
|
|
531
|
+
* SurrealQL local engines (SQLite) materialize via `engine.select(plan)`
|
|
532
|
+
* instead of re-running `surql`, which they cannot parse.
|
|
533
|
+
*/
|
|
534
|
+
plan?: QueryPlan;
|
|
380
535
|
/** Parameters used in the query. */
|
|
381
536
|
params: Record<string, any>;
|
|
382
537
|
/** The version array representing the local state of results. */
|
|
@@ -563,4 +718,4 @@ interface DebounceOptions {
|
|
|
563
718
|
delay?: number;
|
|
564
719
|
}
|
|
565
720
|
//#endregion
|
|
566
|
-
export { SyncHealthStatus as A, RunOptions as C, StoreType as D, Sp00kyQueryResultPromise as E,
|
|
721
|
+
export { SyncHealthStatus as A, EventDefinition as B, RunOptions as C, StoreType as D, Sp00kyQueryResultPromise as E, SealedQuery as F, DatabaseEventSystem as I, DatabaseEventTypes as L, UpdateOptions as M, UpEvent as N, SyncHealth as O, LocalStore as P, Logger$1 as R, RegistrationTimings as S, Sp00kyQueryResult as T, EventSystem as V, QueryTimeToLive as _, MutationCallback as a, RecordVersionArray as b, PersistenceClient as c, QueryConfig as d, QueryConfigRecord as f, QueryStatusCallback as g, QueryStatus as h, MATERIALIZATION_SAMPLE_WINDOW as i, TimingPhase as j, SyncHealthConfig as k, PhaseStat as l, QueryState as m, EventSubscriptionOptions as n, MutationEvent as o, QueryHash as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, PinoTransmit as u, QueryTimings as v, Sp00kyConfig as w, RecordVersionDiff as x, QueryUpdateCallback as y, SyncEventSystem as z };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.109",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -59,8 +59,9 @@
|
|
|
59
59
|
}
|
|
60
60
|
},
|
|
61
61
|
"dependencies": {
|
|
62
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
63
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
62
|
+
"@spooky-sync/query-builder": "0.0.1-canary.109",
|
|
63
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.109",
|
|
64
|
+
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
64
65
|
"@surrealdb/wasm": "^3.0.3",
|
|
65
66
|
"fast-json-patch": "^3.1.1",
|
|
66
67
|
"loro-crdt": "^1.5.6",
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { LocalStore } from '../../services/database/index';
|
|
2
2
|
import { StaleEpochError } from '../../services/database/index';
|
|
3
3
|
import type {
|
|
4
4
|
StreamProcessorService,
|
|
@@ -24,7 +24,7 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
24
24
|
private versionLookups: Record<string, number> = {};
|
|
25
25
|
|
|
26
26
|
constructor(
|
|
27
|
-
private local:
|
|
27
|
+
private local: LocalStore,
|
|
28
28
|
private streamProcessor: StreamProcessorService,
|
|
29
29
|
streamUpdateCallback: (update: StreamUpdate) => void,
|
|
30
30
|
logger: Logger
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { LoroDoc } from 'loro-crdt';
|
|
2
|
-
import type {
|
|
2
|
+
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
3
3
|
import type { Logger } from '../../services/logger/index';
|
|
4
4
|
import { parseRecordIdString } from '../../utils/index';
|
|
5
5
|
|
|
@@ -23,7 +23,7 @@ export function cursorColorFromName(name: string): string {
|
|
|
23
23
|
export class CrdtField {
|
|
24
24
|
private doc: LoroDoc;
|
|
25
25
|
private pushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
26
|
-
private local:
|
|
26
|
+
private local: LocalStore | null = null;
|
|
27
27
|
private remote: RemoteDatabaseService | null = null;
|
|
28
28
|
private recordId: string | null = null;
|
|
29
29
|
private sessionId: string = '';
|
|
@@ -102,7 +102,7 @@ export class CrdtField {
|
|
|
102
102
|
}
|
|
103
103
|
|
|
104
104
|
startSync(
|
|
105
|
-
local:
|
|
105
|
+
local: LocalStore,
|
|
106
106
|
remote: RemoteDatabaseService,
|
|
107
107
|
recordId: string,
|
|
108
108
|
sessionId: string,
|
|
@@ -2,7 +2,7 @@ import { describe, it, expect, vi } from 'vitest';
|
|
|
2
2
|
import { LoroDoc } from 'loro-crdt';
|
|
3
3
|
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
4
4
|
import { CrdtManager } from './index';
|
|
5
|
-
import type {
|
|
5
|
+
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
6
6
|
import type { Logger } from '../../services/logger/index';
|
|
7
7
|
|
|
8
8
|
// Regression test for the offline-reload formatting-loss class of bug.
|
|
@@ -89,7 +89,7 @@ describe('CrdtManager.open hydration', () => {
|
|
|
89
89
|
if (sql.includes('SELECT VALUE body FROM ONLY')) return [null];
|
|
90
90
|
return [];
|
|
91
91
|
}),
|
|
92
|
-
} satisfies Partial<
|
|
92
|
+
} satisfies Partial<LocalStore> as unknown as LocalStore;
|
|
93
93
|
|
|
94
94
|
const remote = makeRemote({
|
|
95
95
|
selectRow: () => ({ id: 'thread:abc', body: remoteSnapshot }),
|
|
@@ -115,7 +115,7 @@ describe('CrdtManager.open hydration', () => {
|
|
|
115
115
|
|
|
116
116
|
const local = {
|
|
117
117
|
query: vi.fn().mockImplementation(async () => [null]),
|
|
118
|
-
} satisfies Partial<
|
|
118
|
+
} satisfies Partial<LocalStore> as unknown as LocalStore;
|
|
119
119
|
|
|
120
120
|
// Cursor-enabled field stores `{ state, cursors }`. The remote row
|
|
121
121
|
// returns this shape; the manager must drill into `.state` for the
|
|
@@ -150,7 +150,7 @@ describe('CrdtManager.open hydration', () => {
|
|
|
150
150
|
if (sql.includes('SELECT VALUE body FROM ONLY')) return [localSnapshot];
|
|
151
151
|
return [];
|
|
152
152
|
}),
|
|
153
|
-
} satisfies Partial<
|
|
153
|
+
} satisfies Partial<LocalStore> as unknown as LocalStore;
|
|
154
154
|
|
|
155
155
|
const remote = makeRemote({});
|
|
156
156
|
|
|
@@ -187,7 +187,7 @@ describe('CrdtManager.open hydration', () => {
|
|
|
187
187
|
}
|
|
188
188
|
return [];
|
|
189
189
|
}),
|
|
190
|
-
} satisfies Partial<
|
|
190
|
+
} satisfies Partial<LocalStore> as unknown as LocalStore;
|
|
191
191
|
|
|
192
192
|
const remote = makeRemote({});
|
|
193
193
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
-
import type {
|
|
2
|
+
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
3
3
|
import type { Logger } from '../../services/logger/index';
|
|
4
4
|
import type { Uuid } from 'surrealdb';
|
|
5
5
|
import { CrdtField } from './crdt-field';
|
|
@@ -40,7 +40,7 @@ export class CrdtManager {
|
|
|
40
40
|
|
|
41
41
|
constructor(
|
|
42
42
|
private schema: SchemaStructure,
|
|
43
|
-
private local:
|
|
43
|
+
private local: LocalStore,
|
|
44
44
|
private remote: RemoteDatabaseService,
|
|
45
45
|
logger: Logger,
|
|
46
46
|
private debounceMs: number = 500,
|
|
@@ -5,8 +5,9 @@ import type {
|
|
|
5
5
|
BackendNames,
|
|
6
6
|
BackendRoutes,
|
|
7
7
|
RoutePayload,
|
|
8
|
+
QueryPlan,
|
|
8
9
|
} from '@spooky-sync/query-builder';
|
|
9
|
-
import type {
|
|
10
|
+
import type { LocalStore } from '../../services/database/index';
|
|
10
11
|
import { StaleEpochError } from '../../services/database/index';
|
|
11
12
|
import type { CacheModule, RecordWithId, CacheRecord } from '../cache/index';
|
|
12
13
|
import type { Logger } from '../../services/logger/index';
|
|
@@ -42,7 +43,7 @@ import {
|
|
|
42
43
|
} from '../../utils/index';
|
|
43
44
|
import type { CreateEvent, DeleteEvent, UpdateEvent } from '../sync/index';
|
|
44
45
|
import type { PushEventOptions } from '../../events/index';
|
|
45
|
-
import { buildWindowMaterialization } from './window-query';
|
|
46
|
+
import { buildWindowMaterialization, buildWindowMaterializationPlan } from './window-query';
|
|
46
47
|
|
|
47
48
|
/** Push a timing sample (ms) into a rolling window, capped at the sample window. */
|
|
48
49
|
function pushSample(samples: number[], ms: number): void {
|
|
@@ -124,7 +125,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
124
125
|
|
|
125
126
|
constructor(
|
|
126
127
|
private cache: CacheModule,
|
|
127
|
-
private local:
|
|
128
|
+
private local: LocalStore,
|
|
128
129
|
private schema: S,
|
|
129
130
|
logger: Logger,
|
|
130
131
|
// Client-side SSP aggregation throttle: coalesces the in-browser
|
|
@@ -180,7 +181,8 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
180
181
|
tableName: T,
|
|
181
182
|
surqlString: string,
|
|
182
183
|
params: Record<string, any>,
|
|
183
|
-
ttl: QueryTimeToLive
|
|
184
|
+
ttl: QueryTimeToLive,
|
|
185
|
+
plan?: QueryPlan
|
|
184
186
|
): Promise<QueryHash> {
|
|
185
187
|
const hash = await this.calculateHash({ surql: surqlString, params });
|
|
186
188
|
this.logger.debug(
|
|
@@ -216,7 +218,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
216
218
|
);
|
|
217
219
|
|
|
218
220
|
// Create the query and track the pending promise
|
|
219
|
-
const promise = this.createAndRegisterQuery<T>(hash, recordId, surqlString, params, ttl, tableName);
|
|
221
|
+
const promise = this.createAndRegisterQuery<T>(hash, recordId, surqlString, params, ttl, tableName, plan);
|
|
220
222
|
this.pendingQueries.set(hash, promise);
|
|
221
223
|
try {
|
|
222
224
|
await promise;
|
|
@@ -433,6 +435,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
433
435
|
sspArray?: Array<[string, number]>
|
|
434
436
|
): Promise<Record<string, any>[]> {
|
|
435
437
|
const t0 = performance.now();
|
|
438
|
+
const plan = queryState.config.plan;
|
|
436
439
|
const windowMat = buildWindowMaterialization(queryState.config.surql);
|
|
437
440
|
let records: Record<string, any>[];
|
|
438
441
|
if (windowMat) {
|
|
@@ -442,11 +445,20 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
442
445
|
queryState.config.localArray ||
|
|
443
446
|
[];
|
|
444
447
|
const winIds = win.map(([id]) => parseRecordIdString(id));
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
448
|
+
if (plan) {
|
|
449
|
+
// Engine-neutral window materialization: select exactly the id-set,
|
|
450
|
+
// keeping ORDER BY + relations. Works on any local engine.
|
|
451
|
+
const winPlan = buildWindowMaterializationPlan(plan, winIds) ?? { ...plan, ids: winIds };
|
|
452
|
+
records = await this.local.select(winPlan, queryState.config.params);
|
|
453
|
+
} else {
|
|
454
|
+
const [rows] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
|
|
455
|
+
...queryState.config.params,
|
|
456
|
+
__win: winIds,
|
|
457
|
+
});
|
|
458
|
+
records = rows || [];
|
|
459
|
+
}
|
|
460
|
+
} else if (plan) {
|
|
461
|
+
records = await this.local.select(plan, queryState.config.params);
|
|
450
462
|
} else {
|
|
451
463
|
const [rows] = await this.local.query<[Record<string, any>[]]>(
|
|
452
464
|
queryState.config.surql,
|
|
@@ -1436,7 +1448,8 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1436
1448
|
surqlString: string,
|
|
1437
1449
|
params: Record<string, any>,
|
|
1438
1450
|
ttl: QueryTimeToLive,
|
|
1439
|
-
tableName: T
|
|
1451
|
+
tableName: T,
|
|
1452
|
+
plan?: QueryPlan
|
|
1440
1453
|
): Promise<QueryHash> {
|
|
1441
1454
|
const queryState = await this.createNewQuery<T>({
|
|
1442
1455
|
recordId,
|
|
@@ -1444,6 +1457,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1444
1457
|
params,
|
|
1445
1458
|
ttl,
|
|
1446
1459
|
tableName,
|
|
1460
|
+
plan,
|
|
1447
1461
|
});
|
|
1448
1462
|
|
|
1449
1463
|
const t0 = performance.now();
|
|
@@ -1487,11 +1501,16 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1487
1501
|
if (windowMat && localArray.length > 0) {
|
|
1488
1502
|
try {
|
|
1489
1503
|
const winIds = localArray.map(([id]) => parseRecordIdString(id));
|
|
1490
|
-
|
|
1491
|
-
...
|
|
1492
|
-
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1504
|
+
if (plan) {
|
|
1505
|
+
const winPlan = buildWindowMaterializationPlan(plan, winIds) ?? { ...plan, ids: winIds };
|
|
1506
|
+
queryState.records = await this.local.select(winPlan, params);
|
|
1507
|
+
} else {
|
|
1508
|
+
const [seeded] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
|
|
1509
|
+
...params,
|
|
1510
|
+
__win: winIds,
|
|
1511
|
+
});
|
|
1512
|
+
queryState.records = seeded || [];
|
|
1513
|
+
}
|
|
1495
1514
|
} catch (err) {
|
|
1496
1515
|
this.logger.warn(
|
|
1497
1516
|
{ err, hash, Category: 'sp00ky-client::DataModule::createAndRegisterQuery' },
|
|
@@ -1521,12 +1540,14 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1521
1540
|
params,
|
|
1522
1541
|
ttl,
|
|
1523
1542
|
tableName,
|
|
1543
|
+
plan,
|
|
1524
1544
|
}: {
|
|
1525
1545
|
recordId: RecordId;
|
|
1526
1546
|
surql: string;
|
|
1527
1547
|
params: Record<string, any>;
|
|
1528
1548
|
ttl: QueryTimeToLive;
|
|
1529
1549
|
tableName: T;
|
|
1550
|
+
plan?: QueryPlan;
|
|
1530
1551
|
}): Promise<QueryState> {
|
|
1531
1552
|
const tableSchema = this.schema.tables.find((t) => t.name === tableName);
|
|
1532
1553
|
if (!tableSchema) {
|
|
@@ -1564,6 +1585,9 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1564
1585
|
const config: QueryConfig = {
|
|
1565
1586
|
...configRecord,
|
|
1566
1587
|
id: recordId,
|
|
1588
|
+
// In-memory only — carries the engine-neutral plan so non-SurrealQL local
|
|
1589
|
+
// engines materialize via `select(plan)` instead of parsing `surql`.
|
|
1590
|
+
plan,
|
|
1567
1591
|
params: parseParams(tableSchema.columns, configRecord.params),
|
|
1568
1592
|
};
|
|
1569
1593
|
|
|
@@ -1575,7 +1599,11 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1575
1599
|
// seeded from the SSP `localArray` in `createAndRegisterQuery` instead.
|
|
1576
1600
|
if (buildWindowMaterialization(surqlString) === null) {
|
|
1577
1601
|
try {
|
|
1578
|
-
|
|
1602
|
+
// Prefer the engine-neutral plan (required for non-SurrealQL engines);
|
|
1603
|
+
// fall back to running the raw surql on the SurrealDB engine.
|
|
1604
|
+
const result = plan
|
|
1605
|
+
? await this.local.select(plan, params)
|
|
1606
|
+
: (await this.local.query<[Record<string, any>[]]>(surqlString, params))[0];
|
|
1579
1607
|
records = result || [];
|
|
1580
1608
|
} catch (err) {
|
|
1581
1609
|
this.logger.warn(
|
|
@@ -23,6 +23,30 @@
|
|
|
23
23
|
* shape — such windowed+subquery queries are rare and were already returning 0,
|
|
24
24
|
* so this is no regression.
|
|
25
25
|
*/
|
|
26
|
+
import type { QueryPlan } from '@spooky-sync/query-builder';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Plan-level window materialization: restrict a windowed query's base rows to
|
|
30
|
+
* exactly the SSP-computed id-set, dropping `where`/`limit`/`offset` but keeping
|
|
31
|
+
* `orderBy`, `select` and `relations`. The engine-neutral counterpart of
|
|
32
|
+
* {@link buildWindowMaterialization} (which does the same by string surgery for
|
|
33
|
+
* the raw-SurrealQL path). Returns `null` for non-offset queries so the caller
|
|
34
|
+
* keeps the normal re-query path.
|
|
35
|
+
*/
|
|
36
|
+
export function buildWindowMaterializationPlan(
|
|
37
|
+
plan: QueryPlan,
|
|
38
|
+
ids: unknown[]
|
|
39
|
+
): QueryPlan | null {
|
|
40
|
+
if (plan.offset === undefined || plan.offset <= 0) return null;
|
|
41
|
+
return {
|
|
42
|
+
...plan,
|
|
43
|
+
ids,
|
|
44
|
+
where: undefined,
|
|
45
|
+
limit: undefined,
|
|
46
|
+
offset: undefined,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
26
50
|
export function buildWindowMaterialization(
|
|
27
51
|
surql: string,
|
|
28
52
|
idsParam = '__win'
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
2
2
|
import type { Logger } from '../../services/logger/index';
|
|
3
3
|
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
4
4
|
import { RecordId } from 'surrealdb';
|
|
@@ -57,7 +57,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
57
57
|
private localTablesAt = 0;
|
|
58
58
|
|
|
59
59
|
constructor(
|
|
60
|
-
private databaseService:
|
|
60
|
+
private databaseService: LocalStore,
|
|
61
61
|
private remoteDatabaseService: RemoteDatabaseService,
|
|
62
62
|
private logger: Logger,
|
|
63
63
|
private schema: SchemaStructure,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { LocalStore } from '../../../services/database/index';
|
|
2
2
|
import type {
|
|
3
3
|
SyncQueueEventSystem} from '../events/index';
|
|
4
4
|
import {
|
|
@@ -47,7 +47,7 @@ export class DownQueue {
|
|
|
47
47
|
}
|
|
48
48
|
|
|
49
49
|
constructor(
|
|
50
|
-
private local:
|
|
50
|
+
private local: LocalStore,
|
|
51
51
|
logger: Logger
|
|
52
52
|
) {
|
|
53
53
|
this._events = createSyncQueueEventSystem();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RecordId } from 'surrealdb';
|
|
2
|
-
import type {
|
|
2
|
+
import type { LocalStore } from '../../../services/database/index';
|
|
3
3
|
import type {
|
|
4
4
|
SyncQueueEventSystem} from '../events/index';
|
|
5
5
|
import {
|
|
@@ -52,7 +52,7 @@ export class UpQueue {
|
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
constructor(
|
|
55
|
-
private local:
|
|
55
|
+
private local: LocalStore,
|
|
56
56
|
logger: Logger
|
|
57
57
|
) {
|
|
58
58
|
this._events = createSyncQueueEventSystem();
|