@spooky-sync/core 0.0.1-canary.108 → 0.0.1-canary.109

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/index.d.ts +10 -114
  2. package/dist/index.js +1070 -46
  3. package/dist/sqlite-worker.d.ts +1 -0
  4. package/dist/sqlite-worker.js +107 -0
  5. package/dist/types.d.ts +157 -2
  6. package/package.json +4 -3
  7. package/src/modules/cache/index.ts +2 -2
  8. package/src/modules/crdt/crdt-field.ts +3 -3
  9. package/src/modules/crdt/crdt-hydration.test.ts +5 -5
  10. package/src/modules/crdt/index.ts +2 -2
  11. package/src/modules/data/index.ts +45 -17
  12. package/src/modules/data/window-query.ts +24 -0
  13. package/src/modules/devtools/index.ts +2 -2
  14. package/src/modules/sync/queue/queue-down.ts +2 -2
  15. package/src/modules/sync/queue/queue-up.ts +2 -2
  16. package/src/modules/sync/sync.ts +2 -2
  17. package/src/services/database/cache-engine.ts +143 -0
  18. package/src/services/database/engine-factory.ts +32 -0
  19. package/src/services/database/index.ts +6 -0
  20. package/src/services/database/local-migrator.ts +2 -2
  21. package/src/services/database/plan-render.test.ts +85 -0
  22. package/src/services/database/plan-render.ts +86 -0
  23. package/src/services/database/relation-resolver.test.ts +413 -0
  24. package/src/services/database/relation-resolver.ts +0 -0
  25. package/src/services/database/sqlite-cache-engine.ts +631 -0
  26. package/src/services/database/sqlite-worker.ts +116 -0
  27. package/src/services/database/surql-translate.ts +291 -0
  28. package/src/services/database/surreal-cache-engine.ts +122 -0
  29. package/src/services/persistence/surrealdb.ts +2 -2
  30. package/src/services/stream-processor/index.ts +2 -2
  31. package/src/sp00ky.ts +24 -7
  32. package/src/types.ts +17 -1
  33. package/tsdown.config.ts +10 -1
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,107 @@
1
+ import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
2
+
3
+ //#region src/services/database/sqlite-worker.ts
4
+ /**
5
+ * Dedicated Web Worker that owns the SQLite-WASM handle. All DB access is
6
+ * funnelled through here (the main thread never touches the wasm module), which
7
+ * is also what the OPFS VFS requires — file access must happen off the main
8
+ * thread. Persistence uses the **OPFS SAHPool VFS**: durable, and (unlike the
9
+ * classic OPFS VFS) it does NOT require COOP/COEP cross-origin isolation
10
+ * headers, so host apps embedding the client need no server changes. Falls back
11
+ * to an in-memory DB when OPFS is unavailable.
12
+ *
13
+ * Message protocol (request/response keyed by `id`):
14
+ * { id, type: 'open', payload: { dbName, useOpfs } }
15
+ * { id, type: 'exec', payload: { sql, bind } } -> { id, ok, rows }
16
+ * { id, type: 'run', payload: { sql, bind } } -> { id, ok }
17
+ * { id, type: 'batch', payload: [{ sql, bind }] } (atomic BEGIN/COMMIT)
18
+ * { id, type: 'close' }
19
+ */
20
+ let db = null;
21
+ async function open(dbName, useOpfs) {
22
+ const sqlite3 = await sqlite3InitModule();
23
+ let persisted = false;
24
+ if (useOpfs && sqlite3.installOpfsSAHPoolVfs) try {
25
+ db = new (await (sqlite3.installOpfsSAHPoolVfs({ name: `sp00ky-${dbName}` }))).OpfsSAHPoolDb(`/${dbName}.sqlite3`);
26
+ persisted = true;
27
+ } catch {}
28
+ if (!db) db = new sqlite3.oo1.DB(":memory:", "c");
29
+ try {
30
+ db.exec({ sql: "PRAGMA busy_timeout = 5000; PRAGMA cache_size = -32000;" });
31
+ } catch {}
32
+ return { persisted };
33
+ }
34
+ function exec(sql, bind) {
35
+ if (!db) throw new Error("sqlite: DB not open");
36
+ return db.exec({
37
+ sql,
38
+ bind,
39
+ rowMode: "object",
40
+ returnValue: "resultRows"
41
+ });
42
+ }
43
+ function run(sql, bind) {
44
+ if (!db) throw new Error("sqlite: DB not open");
45
+ db.exec({
46
+ sql,
47
+ bind
48
+ });
49
+ }
50
+ function batch(stmts) {
51
+ if (!db) throw new Error("sqlite: DB not open");
52
+ db.exec({ sql: "BEGIN" });
53
+ try {
54
+ for (const s of stmts) db.exec({
55
+ sql: s.sql,
56
+ bind: s.bind
57
+ });
58
+ db.exec({ sql: "COMMIT" });
59
+ } catch (e) {
60
+ try {
61
+ db.exec({ sql: "ROLLBACK" });
62
+ } catch {}
63
+ throw e;
64
+ }
65
+ }
66
+ self.onmessage = async (ev) => {
67
+ const { id, type, payload } = ev.data ?? {};
68
+ try {
69
+ let result;
70
+ switch (type) {
71
+ case "open":
72
+ result = await open(payload.dbName, payload.useOpfs);
73
+ break;
74
+ case "exec":
75
+ result = { rows: exec(payload.sql, payload.bind) };
76
+ break;
77
+ case "run":
78
+ run(payload.sql, payload.bind);
79
+ result = {};
80
+ break;
81
+ case "batch":
82
+ batch(payload);
83
+ result = {};
84
+ break;
85
+ case "close":
86
+ db?.close();
87
+ db = null;
88
+ result = {};
89
+ break;
90
+ default: throw new Error(`sqlite worker: unknown message ${type}`);
91
+ }
92
+ self.postMessage({
93
+ id,
94
+ ok: true,
95
+ ...result
96
+ });
97
+ } catch (err) {
98
+ self.postMessage({
99
+ id,
100
+ ok: false,
101
+ error: err instanceof Error ? err.message : String(err)
102
+ });
103
+ }
104
+ };
105
+
106
+ //#endregion
107
+ export { };
package/dist/types.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { RecordId } from "surrealdb";
2
- import { RecordId as RecordId$1, SchemaStructure } from "@spooky-sync/query-builder";
2
+ import { QueryPlan, QueryPlan as QueryPlan$1, RecordId as RecordId$1, SchemaStructure, WhereNode } from "@spooky-sync/query-builder";
3
3
  import { Level, Level as Level$1, Logger, LoggerOptions } from "pino";
4
4
 
5
5
  //#region src/events/index.d.ts
@@ -154,6 +154,146 @@ type SyncEventSystem = EventSystem<SyncEventTypeMap>;
154
154
  //#region src/services/logger/index.d.ts
155
155
  type Logger$1 = Logger;
156
156
  //#endregion
157
+ //#region src/services/database/events/index.d.ts
158
+ declare const DatabaseEventTypes: {
159
+ readonly LocalQuery: "DATABASE_LOCAL_QUERY";
160
+ readonly RemoteQuery: "DATABASE_REMOTE_QUERY";
161
+ };
162
+ interface DatabaseQueryEventPayload {
163
+ query: string;
164
+ vars?: Record<string, unknown>;
165
+ duration: number;
166
+ success: boolean;
167
+ error?: string;
168
+ timestamp: number;
169
+ }
170
+ type DatabaseEventTypeMap = {
171
+ [DatabaseEventTypes.LocalQuery]: EventDefinition<typeof DatabaseEventTypes.LocalQuery, DatabaseQueryEventPayload>;
172
+ [DatabaseEventTypes.RemoteQuery]: EventDefinition<typeof DatabaseEventTypes.RemoteQuery, DatabaseQueryEventPayload>;
173
+ };
174
+ type DatabaseEventSystem = EventSystem<DatabaseEventTypeMap>;
175
+ //#endregion
176
+ //#region src/utils/surql.d.ts
177
+ interface SealedQuery<T = void> {
178
+ readonly sql: string;
179
+ readonly extract: (results: unknown[]) => T;
180
+ }
181
+ //#endregion
182
+ //#region src/services/database/cache-engine.d.ts
183
+ /**
184
+ * A materialized row. Keys are field names; values are already decoded to the
185
+ * client's runtime shapes (RecordId stays a RecordId, bytes a Uint8Array, …) so
186
+ * every backend hands `DataModule` the same shape SurrealDB does today.
187
+ */
188
+ type Row = Record<string, unknown>;
189
+ /** A record identifier — a `RecordId` or its stable string form (`table:id`). */
190
+ type Id = unknown;
191
+ /** How an order clause is expressed everywhere in the engine layer. */
192
+ type OrderBy = [field: string, direction: 'asc' | 'desc'][];
193
+ /**
194
+ * Batched relation fetch: "give me every row of `table` whose `matchField` is
195
+ * one of `keys`, filtered by `where`, ordered by `orderBy`". This is the single
196
+ * primitive relation decomposition (§3) leans on — implemented as
197
+ * `SELECT … WHERE <matchField> IN (…)` on SQLite, `SELECT … FROM $keys` /
198
+ * `WHERE <matchField> IN $keys` on SurrealDB, or an index scan elsewhere. Order
199
+ * here is a hint; the resolver re-applies order+limit PER PARENT after grouping.
200
+ */
201
+ interface RelationFetch {
202
+ table: string;
203
+ matchField: string;
204
+ keys: Id[];
205
+ where?: WhereNode[];
206
+ orderBy?: OrderBy;
207
+ select?: string[];
208
+ }
209
+ /**
210
+ * The read side of an engine, minus relation resolution — the surface a
211
+ * {@link RelationResolver} needs. Kept separate so the resolver can be unit
212
+ * tested against an in-memory fake without a full engine.
213
+ */
214
+ interface RowFetcher {
215
+ /** Batched fan-out fetch. See {@link RelationFetch}. */
216
+ fetchRelation(req: RelationFetch): Promise<Row[]>;
217
+ }
218
+ /** A transaction handle — the same verbs as the engine, but atomic. */
219
+ interface EngineTx {
220
+ upsert(table: string, id: Id, data: Row, mode: 'replace' | 'merge'): Promise<void>;
221
+ patch(table: string, id: Id, patches: unknown[]): Promise<void>;
222
+ delete(table: string, id: Id): Promise<void>;
223
+ }
224
+ /**
225
+ * A pluggable local cache backend. SurrealDB (the default) and SQLite both
226
+ * implement this; the rest of the client talks verbs, never SurrealQL.
227
+ *
228
+ * Reactivity is NOT part of this contract: the local cache is passive. The SSP
229
+ * (remote) drives change; `DataModule` writes rows here and re-reads them. The
230
+ * `epoch` field preserves the existing bucket-switch fencing (see
231
+ * `LocalDatabaseService.epoch`): an async chain captures it at start and its
232
+ * write is dropped if the epoch moved (a bucket switch) in between.
233
+ */
234
+ interface LocalCacheEngine extends RowFetcher {
235
+ /** Monotonic store generation; bumped on every bucket switch. */
236
+ readonly epoch: number;
237
+ connect(bucketId: string): Promise<void>;
238
+ switchBucket(bucketId: string): Promise<void>;
239
+ close(): Promise<void>;
240
+ /** Run `fn` inside a single atomic transaction. */
241
+ transaction<T>(fn: (tx: EngineTx) => Promise<T>): Promise<T>;
242
+ /**
243
+ * Materialize a query, including its `.related()` tree (via §3
244
+ * decomposition). Params bind `where` `paramRef`s and any windowing id-set.
245
+ */
246
+ select(plan: QueryPlan$1, params?: Record<string, unknown>): Promise<Row[]>;
247
+ /** Fetch rows by primary id, preserving `ids` order; missing ids are skipped. */
248
+ selectByIds(table: string, ids: Id[], opts?: {
249
+ select?: string[];
250
+ orderBy?: OrderBy;
251
+ }): Promise<Row[]>;
252
+ /** Single-record read by primary id, or `null`. */
253
+ getById(table: string, id: Id): Promise<Row | null>;
254
+ upsert(table: string, id: Id, data: Row, mode: 'replace' | 'merge'): Promise<void>;
255
+ patch(table: string, id: Id, patches: unknown[]): Promise<void>;
256
+ delete(table: string, id: Id): Promise<void>;
257
+ }
258
+ /**
259
+ * The full surface the client's `this.local` field depends on: the
260
+ * engine-neutral {@link LocalCacheEngine} verbs PLUS the legacy
261
+ * SurrealQL/lifecycle methods the not-yet-migrated call sites still use.
262
+ * `SurrealCacheEngine` (subclass of `LocalDatabaseService`) and
263
+ * `SqliteCacheEngine` (via a SurrealQL-vocabulary shim) both satisfy this, so
264
+ * either can back `this.local`.
265
+ *
266
+ * `getClient()` returns the underlying SurrealDB `Surreal` handle where one
267
+ * exists (SurrealDB backend); backends without one (SQLite) throw — it is only
268
+ * used by advanced/DevTools paths, never on the hot path.
269
+ */
270
+ interface LocalStore extends LocalCacheEngine {
271
+ /**
272
+ * Whether this engine needs SurrealQL schema provisioning (`DEFINE TABLE`,
273
+ * `DEFINE FIELD`, …) run against it at init / bucket switch. SurrealDB → true;
274
+ * schemaless engines (SQLite creates tables lazily) → false, so the client
275
+ * skips the `LocalMigrator` entirely for them.
276
+ */
277
+ readonly usesSurqlSchema: boolean;
278
+ query<T extends unknown[]>(query: string, vars?: Record<string, unknown>, opts?: {
279
+ epoch?: number;
280
+ }): Promise<T>;
281
+ execute<T>(query: SealedQuery<T>, vars?: Record<string, unknown>, opts?: {
282
+ epoch?: number;
283
+ }): Promise<T>;
284
+ queryUngated<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T>;
285
+ switchStore(bucketId: string): Promise<void>;
286
+ beginSwitch(): () => void;
287
+ getEvents(): DatabaseEventSystem;
288
+ getClient(): unknown;
289
+ getConfig(): Sp00kyConfig<any>['database'];
290
+ readonly currentBucketId: string;
291
+ }
292
+ /** Selected local cache backend. Mirrors the `persistenceClient` config pattern. */
293
+ type LocalEngineChoice = 'surrealdb' | 'sqlite' | LocalStore;
294
+ /** Thrown when relation decomposition nests past {@link MAX_RELATION_DEPTH} —
295
+ * a guard against a cyclic schema producing unbounded fan-out. */
296
+ //#endregion
157
297
  //#region src/modules/sync/queue/queue-up.d.ts
158
298
  type CreateEvent = {
159
299
  type: 'create';
@@ -260,6 +400,14 @@ interface Sp00kyConfig<S extends SchemaStructure> {
260
400
  * Can be a custom implementation, 'surrealdb' (default), or 'localstorage'.
261
401
  */
262
402
  persistenceClient?: PersistenceClient | 'surrealdb' | 'localstorage';
403
+ /**
404
+ * Local cache engine backend. `'surrealdb'` (default) uses the in-browser
405
+ * SurrealDB-WASM store; `'sqlite'` uses official SQLite-WASM in a Worker with
406
+ * OPFS persistence; or pass a custom {@link LocalCacheEngine}. The local cache
407
+ * is a passive queryable store — reactivity is driven by the remote SSP, not
408
+ * this engine. See `services/database/cache-engine.ts`.
409
+ */
410
+ localEngine?: LocalEngineChoice;
263
411
  /** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
264
412
  otelTransmit?: PinoTransmit;
265
413
  /**
@@ -377,6 +525,13 @@ interface QueryConfig {
377
525
  id: RecordId$1<string>;
378
526
  /** The SURQL query string. */
379
527
  surql: string;
528
+ /**
529
+ * Engine-neutral plan for `surql` (in-memory only; not persisted to
530
+ * `_00_query`). Present when the query came from the query-builder. Non-
531
+ * SurrealQL local engines (SQLite) materialize via `engine.select(plan)`
532
+ * instead of re-running `surql`, which they cannot parse.
533
+ */
534
+ plan?: QueryPlan;
380
535
  /** Parameters used in the query. */
381
536
  params: Record<string, any>;
382
537
  /** The version array representing the local state of results. */
@@ -563,4 +718,4 @@ interface DebounceOptions {
563
718
  delay?: number;
564
719
  }
565
720
  //#endregion
566
- export { SyncHealthStatus as A, RunOptions as C, StoreType as D, Sp00kyQueryResultPromise as E, SyncEventSystem as F, EventDefinition as I, EventSystem as L, UpdateOptions as M, UpEvent as N, SyncHealth as O, Logger$1 as P, RegistrationTimings as S, Sp00kyQueryResult as T, QueryTimeToLive as _, MutationCallback as a, RecordVersionArray as b, PersistenceClient as c, QueryConfig as d, QueryConfigRecord as f, QueryStatusCallback as g, QueryStatus as h, MATERIALIZATION_SAMPLE_WINDOW as i, TimingPhase as j, SyncHealthConfig as k, PhaseStat as l, QueryState as m, EventSubscriptionOptions as n, MutationEvent as o, QueryHash as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, PinoTransmit as u, QueryTimings as v, Sp00kyConfig as w, RecordVersionDiff as x, QueryUpdateCallback as y };
721
+ export { SyncHealthStatus as A, EventDefinition as B, RunOptions as C, StoreType as D, Sp00kyQueryResultPromise as E, SealedQuery as F, DatabaseEventSystem as I, DatabaseEventTypes as L, UpdateOptions as M, UpEvent as N, SyncHealth as O, LocalStore as P, Logger$1 as R, RegistrationTimings as S, Sp00kyQueryResult as T, EventSystem as V, QueryTimeToLive as _, MutationCallback as a, RecordVersionArray as b, PersistenceClient as c, QueryConfig as d, QueryConfigRecord as f, QueryStatusCallback as g, QueryStatus as h, MATERIALIZATION_SAMPLE_WINDOW as i, TimingPhase as j, SyncHealthConfig as k, PhaseStat as l, QueryState as m, EventSubscriptionOptions as n, MutationEvent as o, QueryHash as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, PinoTransmit as u, QueryTimings as v, Sp00kyConfig as w, RecordVersionDiff as x, QueryUpdateCallback as y, SyncEventSystem as z };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@spooky-sync/core",
3
- "version": "0.0.1-canary.108",
3
+ "version": "0.0.1-canary.109",
4
4
  "type": "module",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",
@@ -59,8 +59,9 @@
59
59
  }
60
60
  },
61
61
  "dependencies": {
62
- "@spooky-sync/query-builder": "0.0.1-canary.108",
63
- "@spooky-sync/ssp-wasm": "0.0.1-canary.108",
62
+ "@spooky-sync/query-builder": "0.0.1-canary.109",
63
+ "@spooky-sync/ssp-wasm": "0.0.1-canary.109",
64
+ "@sqlite.org/sqlite-wasm": "3.53.0-build1",
64
65
  "@surrealdb/wasm": "^3.0.3",
65
66
  "fast-json-patch": "^3.1.1",
66
67
  "loro-crdt": "^1.5.6",
@@ -1,4 +1,4 @@
1
- import type { LocalDatabaseService } from '../../services/database/index';
1
+ import type { LocalStore } from '../../services/database/index';
2
2
  import { StaleEpochError } from '../../services/database/index';
3
3
  import type {
4
4
  StreamProcessorService,
@@ -24,7 +24,7 @@ export class CacheModule implements StreamUpdateReceiver {
24
24
  private versionLookups: Record<string, number> = {};
25
25
 
26
26
  constructor(
27
- private local: LocalDatabaseService,
27
+ private local: LocalStore,
28
28
  private streamProcessor: StreamProcessorService,
29
29
  streamUpdateCallback: (update: StreamUpdate) => void,
30
30
  logger: Logger
@@ -1,5 +1,5 @@
1
1
  import { LoroDoc } from 'loro-crdt';
2
- import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
2
+ import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
3
3
  import type { Logger } from '../../services/logger/index';
4
4
  import { parseRecordIdString } from '../../utils/index';
5
5
 
@@ -23,7 +23,7 @@ export function cursorColorFromName(name: string): string {
23
23
  export class CrdtField {
24
24
  private doc: LoroDoc;
25
25
  private pushTimer: ReturnType<typeof setTimeout> | null = null;
26
- private local: LocalDatabaseService | null = null;
26
+ private local: LocalStore | null = null;
27
27
  private remote: RemoteDatabaseService | null = null;
28
28
  private recordId: string | null = null;
29
29
  private sessionId: string = '';
@@ -102,7 +102,7 @@ export class CrdtField {
102
102
  }
103
103
 
104
104
  startSync(
105
- local: LocalDatabaseService,
105
+ local: LocalStore,
106
106
  remote: RemoteDatabaseService,
107
107
  recordId: string,
108
108
  sessionId: string,
@@ -2,7 +2,7 @@ import { describe, it, expect, vi } from 'vitest';
2
2
  import { LoroDoc } from 'loro-crdt';
3
3
  import type { SchemaStructure } from '@spooky-sync/query-builder';
4
4
  import { CrdtManager } from './index';
5
- import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
5
+ import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
6
6
  import type { Logger } from '../../services/logger/index';
7
7
 
8
8
  // Regression test for the offline-reload formatting-loss class of bug.
@@ -89,7 +89,7 @@ describe('CrdtManager.open hydration', () => {
89
89
  if (sql.includes('SELECT VALUE body FROM ONLY')) return [null];
90
90
  return [];
91
91
  }),
92
- } satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
92
+ } satisfies Partial<LocalStore> as unknown as LocalStore;
93
93
 
94
94
  const remote = makeRemote({
95
95
  selectRow: () => ({ id: 'thread:abc', body: remoteSnapshot }),
@@ -115,7 +115,7 @@ describe('CrdtManager.open hydration', () => {
115
115
 
116
116
  const local = {
117
117
  query: vi.fn().mockImplementation(async () => [null]),
118
- } satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
118
+ } satisfies Partial<LocalStore> as unknown as LocalStore;
119
119
 
120
120
  // Cursor-enabled field stores `{ state, cursors }`. The remote row
121
121
  // returns this shape; the manager must drill into `.state` for the
@@ -150,7 +150,7 @@ describe('CrdtManager.open hydration', () => {
150
150
  if (sql.includes('SELECT VALUE body FROM ONLY')) return [localSnapshot];
151
151
  return [];
152
152
  }),
153
- } satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
153
+ } satisfies Partial<LocalStore> as unknown as LocalStore;
154
154
 
155
155
  const remote = makeRemote({});
156
156
 
@@ -187,7 +187,7 @@ describe('CrdtManager.open hydration', () => {
187
187
  }
188
188
  return [];
189
189
  }),
190
- } satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
190
+ } satisfies Partial<LocalStore> as unknown as LocalStore;
191
191
 
192
192
  const remote = makeRemote({});
193
193
 
@@ -1,5 +1,5 @@
1
1
  import type { SchemaStructure } from '@spooky-sync/query-builder';
2
- import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
2
+ import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
3
3
  import type { Logger } from '../../services/logger/index';
4
4
  import type { Uuid } from 'surrealdb';
5
5
  import { CrdtField } from './crdt-field';
@@ -40,7 +40,7 @@ export class CrdtManager {
40
40
 
41
41
  constructor(
42
42
  private schema: SchemaStructure,
43
- private local: LocalDatabaseService,
43
+ private local: LocalStore,
44
44
  private remote: RemoteDatabaseService,
45
45
  logger: Logger,
46
46
  private debounceMs: number = 500,
@@ -5,8 +5,9 @@ import type {
5
5
  BackendNames,
6
6
  BackendRoutes,
7
7
  RoutePayload,
8
+ QueryPlan,
8
9
  } from '@spooky-sync/query-builder';
9
- import type { LocalDatabaseService } from '../../services/database/index';
10
+ import type { LocalStore } from '../../services/database/index';
10
11
  import { StaleEpochError } from '../../services/database/index';
11
12
  import type { CacheModule, RecordWithId, CacheRecord } from '../cache/index';
12
13
  import type { Logger } from '../../services/logger/index';
@@ -42,7 +43,7 @@ import {
42
43
  } from '../../utils/index';
43
44
  import type { CreateEvent, DeleteEvent, UpdateEvent } from '../sync/index';
44
45
  import type { PushEventOptions } from '../../events/index';
45
- import { buildWindowMaterialization } from './window-query';
46
+ import { buildWindowMaterialization, buildWindowMaterializationPlan } from './window-query';
46
47
 
47
48
  /** Push a timing sample (ms) into a rolling window, capped at the sample window. */
48
49
  function pushSample(samples: number[], ms: number): void {
@@ -124,7 +125,7 @@ export class DataModule<S extends SchemaStructure> {
124
125
 
125
126
  constructor(
126
127
  private cache: CacheModule,
127
- private local: LocalDatabaseService,
128
+ private local: LocalStore,
128
129
  private schema: S,
129
130
  logger: Logger,
130
131
  // Client-side SSP aggregation throttle: coalesces the in-browser
@@ -180,7 +181,8 @@ export class DataModule<S extends SchemaStructure> {
180
181
  tableName: T,
181
182
  surqlString: string,
182
183
  params: Record<string, any>,
183
- ttl: QueryTimeToLive
184
+ ttl: QueryTimeToLive,
185
+ plan?: QueryPlan
184
186
  ): Promise<QueryHash> {
185
187
  const hash = await this.calculateHash({ surql: surqlString, params });
186
188
  this.logger.debug(
@@ -216,7 +218,7 @@ export class DataModule<S extends SchemaStructure> {
216
218
  );
217
219
 
218
220
  // Create the query and track the pending promise
219
- const promise = this.createAndRegisterQuery<T>(hash, recordId, surqlString, params, ttl, tableName);
221
+ const promise = this.createAndRegisterQuery<T>(hash, recordId, surqlString, params, ttl, tableName, plan);
220
222
  this.pendingQueries.set(hash, promise);
221
223
  try {
222
224
  await promise;
@@ -433,6 +435,7 @@ export class DataModule<S extends SchemaStructure> {
433
435
  sspArray?: Array<[string, number]>
434
436
  ): Promise<Record<string, any>[]> {
435
437
  const t0 = performance.now();
438
+ const plan = queryState.config.plan;
436
439
  const windowMat = buildWindowMaterialization(queryState.config.surql);
437
440
  let records: Record<string, any>[];
438
441
  if (windowMat) {
@@ -442,11 +445,20 @@ export class DataModule<S extends SchemaStructure> {
442
445
  queryState.config.localArray ||
443
446
  [];
444
447
  const winIds = win.map(([id]) => parseRecordIdString(id));
445
- const [rows] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
446
- ...queryState.config.params,
447
- __win: winIds,
448
- });
449
- records = rows || [];
448
+ if (plan) {
449
+ // Engine-neutral window materialization: select exactly the id-set,
450
+ // keeping ORDER BY + relations. Works on any local engine.
451
+ const winPlan = buildWindowMaterializationPlan(plan, winIds) ?? { ...plan, ids: winIds };
452
+ records = await this.local.select(winPlan, queryState.config.params);
453
+ } else {
454
+ const [rows] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
455
+ ...queryState.config.params,
456
+ __win: winIds,
457
+ });
458
+ records = rows || [];
459
+ }
460
+ } else if (plan) {
461
+ records = await this.local.select(plan, queryState.config.params);
450
462
  } else {
451
463
  const [rows] = await this.local.query<[Record<string, any>[]]>(
452
464
  queryState.config.surql,
@@ -1436,7 +1448,8 @@ export class DataModule<S extends SchemaStructure> {
1436
1448
  surqlString: string,
1437
1449
  params: Record<string, any>,
1438
1450
  ttl: QueryTimeToLive,
1439
- tableName: T
1451
+ tableName: T,
1452
+ plan?: QueryPlan
1440
1453
  ): Promise<QueryHash> {
1441
1454
  const queryState = await this.createNewQuery<T>({
1442
1455
  recordId,
@@ -1444,6 +1457,7 @@ export class DataModule<S extends SchemaStructure> {
1444
1457
  params,
1445
1458
  ttl,
1446
1459
  tableName,
1460
+ plan,
1447
1461
  });
1448
1462
 
1449
1463
  const t0 = performance.now();
@@ -1487,11 +1501,16 @@ export class DataModule<S extends SchemaStructure> {
1487
1501
  if (windowMat && localArray.length > 0) {
1488
1502
  try {
1489
1503
  const winIds = localArray.map(([id]) => parseRecordIdString(id));
1490
- const [seeded] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
1491
- ...params,
1492
- __win: winIds,
1493
- });
1494
- queryState.records = seeded || [];
1504
+ if (plan) {
1505
+ const winPlan = buildWindowMaterializationPlan(plan, winIds) ?? { ...plan, ids: winIds };
1506
+ queryState.records = await this.local.select(winPlan, params);
1507
+ } else {
1508
+ const [seeded] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
1509
+ ...params,
1510
+ __win: winIds,
1511
+ });
1512
+ queryState.records = seeded || [];
1513
+ }
1495
1514
  } catch (err) {
1496
1515
  this.logger.warn(
1497
1516
  { err, hash, Category: 'sp00ky-client::DataModule::createAndRegisterQuery' },
@@ -1521,12 +1540,14 @@ export class DataModule<S extends SchemaStructure> {
1521
1540
  params,
1522
1541
  ttl,
1523
1542
  tableName,
1543
+ plan,
1524
1544
  }: {
1525
1545
  recordId: RecordId;
1526
1546
  surql: string;
1527
1547
  params: Record<string, any>;
1528
1548
  ttl: QueryTimeToLive;
1529
1549
  tableName: T;
1550
+ plan?: QueryPlan;
1530
1551
  }): Promise<QueryState> {
1531
1552
  const tableSchema = this.schema.tables.find((t) => t.name === tableName);
1532
1553
  if (!tableSchema) {
@@ -1564,6 +1585,9 @@ export class DataModule<S extends SchemaStructure> {
1564
1585
  const config: QueryConfig = {
1565
1586
  ...configRecord,
1566
1587
  id: recordId,
1588
+ // In-memory only — carries the engine-neutral plan so non-SurrealQL local
1589
+ // engines materialize via `select(plan)` instead of parsing `surql`.
1590
+ plan,
1567
1591
  params: parseParams(tableSchema.columns, configRecord.params),
1568
1592
  };
1569
1593
 
@@ -1575,7 +1599,11 @@ export class DataModule<S extends SchemaStructure> {
1575
1599
  // seeded from the SSP `localArray` in `createAndRegisterQuery` instead.
1576
1600
  if (buildWindowMaterialization(surqlString) === null) {
1577
1601
  try {
1578
- const [result] = await this.local.query<[Record<string, any>[]]>(surqlString, params);
1602
+ // Prefer the engine-neutral plan (required for non-SurrealQL engines);
1603
+ // fall back to running the raw surql on the SurrealDB engine.
1604
+ const result = plan
1605
+ ? await this.local.select(plan, params)
1606
+ : (await this.local.query<[Record<string, any>[]]>(surqlString, params))[0];
1579
1607
  records = result || [];
1580
1608
  } catch (err) {
1581
1609
  this.logger.warn(
@@ -23,6 +23,30 @@
23
23
  * shape — such windowed+subquery queries are rare and were already returning 0,
24
24
  * so this is no regression.
25
25
  */
26
+ import type { QueryPlan } from '@spooky-sync/query-builder';
27
+
28
+ /**
29
+ * Plan-level window materialization: restrict a windowed query's base rows to
30
+ * exactly the SSP-computed id-set, dropping `where`/`limit`/`offset` but keeping
31
+ * `orderBy`, `select` and `relations`. The engine-neutral counterpart of
32
+ * {@link buildWindowMaterialization} (which does the same by string surgery for
33
+ * the raw-SurrealQL path). Returns `null` for non-offset queries so the caller
34
+ * keeps the normal re-query path.
35
+ */
36
+ export function buildWindowMaterializationPlan(
37
+ plan: QueryPlan,
38
+ ids: unknown[]
39
+ ): QueryPlan | null {
40
+ if (plan.offset === undefined || plan.offset <= 0) return null;
41
+ return {
42
+ ...plan,
43
+ ids,
44
+ where: undefined,
45
+ limit: undefined,
46
+ offset: undefined,
47
+ };
48
+ }
49
+
26
50
  export function buildWindowMaterialization(
27
51
  surql: string,
28
52
  idsParam = '__win'
@@ -1,4 +1,4 @@
1
- import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
1
+ import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
2
2
  import type { Logger } from '../../services/logger/index';
3
3
  import type { SchemaStructure } from '@spooky-sync/query-builder';
4
4
  import { RecordId } from 'surrealdb';
@@ -57,7 +57,7 @@ export class DevToolsService implements StreamUpdateReceiver {
57
57
  private localTablesAt = 0;
58
58
 
59
59
  constructor(
60
- private databaseService: LocalDatabaseService,
60
+ private databaseService: LocalStore,
61
61
  private remoteDatabaseService: RemoteDatabaseService,
62
62
  private logger: Logger,
63
63
  private schema: SchemaStructure,
@@ -1,4 +1,4 @@
1
- import type { LocalDatabaseService } from '../../../services/database/index';
1
+ import type { LocalStore } from '../../../services/database/index';
2
2
  import type {
3
3
  SyncQueueEventSystem} from '../events/index';
4
4
  import {
@@ -47,7 +47,7 @@ export class DownQueue {
47
47
  }
48
48
 
49
49
  constructor(
50
- private local: LocalDatabaseService,
50
+ private local: LocalStore,
51
51
  logger: Logger
52
52
  ) {
53
53
  this._events = createSyncQueueEventSystem();
@@ -1,5 +1,5 @@
1
1
  import type { RecordId } from 'surrealdb';
2
- import type { LocalDatabaseService } from '../../../services/database/index';
2
+ import type { LocalStore } from '../../../services/database/index';
3
3
  import type {
4
4
  SyncQueueEventSystem} from '../events/index';
5
5
  import {
@@ -52,7 +52,7 @@ export class UpQueue {
52
52
  }
53
53
 
54
54
  constructor(
55
- private local: LocalDatabaseService,
55
+ private local: LocalStore,
56
56
  logger: Logger
57
57
  ) {
58
58
  this._events = createSyncQueueEventSystem();