@spooky-sync/core 0.0.1-canary.136 → 0.0.1-canary.138
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +6 -1
- package/dist/index.js +69 -16
- package/dist/sqlite-worker.js +4 -2
- package/dist/types.d.ts +8 -0
- package/package.json +4 -3
- package/src/modules/crdt/crdt-field.ts +8 -2
- package/src/modules/crdt/index.ts +5 -1
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/services/database/local.ts +41 -16
- package/src/services/database/sqlite-cache-engine.test.ts +39 -0
- package/src/services/database/sqlite-cache-engine.ts +26 -0
- package/src/services/database/sqlite-worker.ts +15 -2
- package/src/sp00ky.ts +6 -0
- package/src/types.ts +8 -0
package/dist/index.d.ts
CHANGED
|
@@ -997,7 +997,12 @@ declare class CrdtField {
|
|
|
997
997
|
* Any cursor data that arrived before this callback was set will be replayed. */
|
|
998
998
|
set onCursorUpdate(cb: ((data: Uint8Array) => void) | null);
|
|
999
999
|
get onCursorUpdate(): ((data: Uint8Array) => void) | null;
|
|
1000
|
-
|
|
1000
|
+
/**
|
|
1001
|
+
* @param LoroDocClass the `LoroDoc` constructor, injected by the caller after
|
|
1002
|
+
* awaiting {@link loadLoro} — keeps `loro-crdt` out of this module's static
|
|
1003
|
+
* import graph so it only ships to apps that use CRDT fields.
|
|
1004
|
+
*/
|
|
1005
|
+
constructor(fieldName: string, cursorsEnabled: boolean, LoroDocClass: typeof LoroDoc, initialState?: Uint8Array, logger?: Logger$1 | null);
|
|
1001
1006
|
getDoc(): LoroDoc;
|
|
1002
1007
|
/** Whether the LoroDoc was loaded from saved CRDT state */
|
|
1003
1008
|
hasContent(): boolean;
|
package/dist/index.js
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import { a as serializeRow, i as reviveRow, n as renderOrderSql, o as resolveRelations, r as renderWhereSql, s as stableKey, t as project } from "./sqlite-plan-sql.js";
|
|
2
2
|
import { DateTime, Duration, RecordId, Surreal, Uuid, applyDiagnostics, createRemoteEngines } from "surrealdb";
|
|
3
|
-
import { createWasmWorkerEngines } from "@surrealdb/wasm";
|
|
4
3
|
import { QueryBuilder, RecordId as RecordId$1, cyrb53 } from "@spooky-sync/query-builder";
|
|
5
4
|
import pino from "pino";
|
|
6
5
|
import { applyPatch } from "fast-json-patch";
|
|
7
6
|
import init, { Sp00kyProcessor } from "@spooky-sync/ssp-wasm";
|
|
8
|
-
import { LoroDoc } from "loro-crdt";
|
|
9
7
|
|
|
10
8
|
//#region src/types.ts
|
|
11
9
|
/** Cap on the rolling materialization-sample window kept per query in memory. */
|
|
@@ -540,13 +538,31 @@ function bucketStoreUrl(bucketId) {
|
|
|
540
538
|
function bucketStoreName(bucketId) {
|
|
541
539
|
return `sp00ky-${bucketId}`;
|
|
542
540
|
}
|
|
543
|
-
|
|
541
|
+
/** Shared codec: mirrors SurrealDB RecordId/DateTime into our own encodings. */
|
|
542
|
+
const localCodecOptions = { valueDecodeVisitor(value) {
|
|
543
|
+
if (value instanceof RecordId) return encodeRecordId(value);
|
|
544
|
+
if (value instanceof DateTime) return value.toDate();
|
|
545
|
+
return value;
|
|
546
|
+
} };
|
|
547
|
+
/**
|
|
548
|
+
* Engine-less Surreal client used as the constructor placeholder. Building this
|
|
549
|
+
* pulls in NO `@surrealdb/wasm` — the ~6 MB wasm engine is deferred to
|
|
550
|
+
* {@link createLocalSurrealClient}, which runs lazily in `connect`/`switchStore`.
|
|
551
|
+
* `connect()` replaces `this.client` with an engine-backed client before any
|
|
552
|
+
* query runs, so this bare client is never actually opened against a store.
|
|
553
|
+
*/
|
|
554
|
+
function createBareSurrealClient() {
|
|
555
|
+
return new Surreal({ codecOptions: localCodecOptions });
|
|
556
|
+
}
|
|
557
|
+
/**
|
|
558
|
+
* Engine-backed client. Dynamically imports `@surrealdb/wasm` so the wasm engine
|
|
559
|
+
* only enters the graph as a separate chunk fetched on first connect (module-
|
|
560
|
+
* cached thereafter — `switchStore`'s await is instant).
|
|
561
|
+
*/
|
|
562
|
+
async function createLocalSurrealClient(logger) {
|
|
563
|
+
const { createWasmWorkerEngines } = await import("@surrealdb/wasm");
|
|
544
564
|
return new Surreal({
|
|
545
|
-
codecOptions:
|
|
546
|
-
if (value instanceof RecordId) return encodeRecordId(value);
|
|
547
|
-
if (value instanceof DateTime) return value.toDate();
|
|
548
|
-
return value;
|
|
549
|
-
} },
|
|
565
|
+
codecOptions: localCodecOptions,
|
|
550
566
|
engines: applyDiagnostics(createWasmWorkerEngines(), ({ key, type, phase, ...other }) => {
|
|
551
567
|
if (phase === "progress" || phase === "after") logger.trace({
|
|
552
568
|
...other,
|
|
@@ -578,7 +594,7 @@ var LocalDatabaseService = class extends AbstractDatabaseService {
|
|
|
578
594
|
pendingSwitchClient = null;
|
|
579
595
|
constructor(config, logger) {
|
|
580
596
|
const events = createDatabaseEventSystem();
|
|
581
|
-
super(
|
|
597
|
+
super(createBareSurrealClient(), logger, events);
|
|
582
598
|
this.config = config;
|
|
583
599
|
}
|
|
584
600
|
getConfig() {
|
|
@@ -630,6 +646,7 @@ var LocalDatabaseService = class extends AbstractDatabaseService {
|
|
|
630
646
|
Category: "sp00ky-client::LocalDatabaseService::connect"
|
|
631
647
|
}, "Connecting to local database");
|
|
632
648
|
this.registerUnloadClose();
|
|
649
|
+
this.client = await createLocalSurrealClient(this.logger);
|
|
633
650
|
await this.openWithRecovery(this.client, storeUrl, namespace, database, bucketId, store);
|
|
634
651
|
}
|
|
635
652
|
/**
|
|
@@ -659,7 +676,7 @@ var LocalDatabaseService = class extends AbstractDatabaseService {
|
|
|
659
676
|
}, "Reset in-memory local store for bucket switch");
|
|
660
677
|
return;
|
|
661
678
|
}
|
|
662
|
-
const next = createLocalSurrealClient(this.logger);
|
|
679
|
+
const next = await createLocalSurrealClient(this.logger);
|
|
663
680
|
this.pendingSwitchClient = next;
|
|
664
681
|
try {
|
|
665
682
|
await this.openWithRecovery(next, bucketStoreUrl(bucketId), namespace, database, bucketId, store);
|
|
@@ -1440,6 +1457,20 @@ function setPath(obj, path, value) {
|
|
|
1440
1457
|
* full merged row is only materialized for a LET-wrapped upsert (see the 'let'
|
|
1441
1458
|
* case). delete/deleteAll yield `[]`; noop yields `null`.
|
|
1442
1459
|
*/
|
|
1460
|
+
/**
|
|
1461
|
+
* The `_00_*` internal tables the client relies on. The LocalMigrator DEFINEs
|
|
1462
|
+
* them, but every DEFINE lowers to a noop on the SQLite engine, so they must be
|
|
1463
|
+
* created physically at open (see `openInternal`) or a read-before-first-write
|
|
1464
|
+
* on a fresh bucket throws "no such table". Keep in sync with the systemSchema
|
|
1465
|
+
* block in `local-migrator.ts`.
|
|
1466
|
+
*/
|
|
1467
|
+
const SYSTEM_TABLES = [
|
|
1468
|
+
"_00_stream_processor_state",
|
|
1469
|
+
"_00_query",
|
|
1470
|
+
"_00_preload",
|
|
1471
|
+
"_00_schema",
|
|
1472
|
+
"_00_pending_mutations"
|
|
1473
|
+
];
|
|
1443
1474
|
function pureWriteOpResult(op) {
|
|
1444
1475
|
switch (op.kind) {
|
|
1445
1476
|
case "upsert": return {
|
|
@@ -1594,9 +1625,11 @@ var SqliteCacheEngine = class {
|
|
|
1594
1625
|
this.worker = this.spawnWorker();
|
|
1595
1626
|
const { persisted } = await this.rawCall("open", {
|
|
1596
1627
|
dbName: bucketId,
|
|
1597
|
-
useOpfs: this.useOpfs
|
|
1628
|
+
useOpfs: this.useOpfs,
|
|
1629
|
+
systemTables: SYSTEM_TABLES
|
|
1598
1630
|
});
|
|
1599
1631
|
this.knownTables.clear();
|
|
1632
|
+
for (const t of SYSTEM_TABLES) this.knownTables.add(t);
|
|
1600
1633
|
this.bucketId = bucketId;
|
|
1601
1634
|
this.logger.info({
|
|
1602
1635
|
bucketId,
|
|
@@ -5073,8 +5106,8 @@ function parseBackendInfo(raw) {
|
|
|
5073
5106
|
|
|
5074
5107
|
//#endregion
|
|
5075
5108
|
//#region src/modules/devtools/index.ts
|
|
5076
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
5077
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
5109
|
+
const CORE_VERSION = "0.0.1-canary.138";
|
|
5110
|
+
const WASM_VERSION = "0.0.1-canary.138";
|
|
5078
5111
|
const SURREAL_VERSION = "3.0.3";
|
|
5079
5112
|
var DevToolsService = class {
|
|
5080
5113
|
eventsHistory = [];
|
|
@@ -6284,12 +6317,17 @@ var CrdtField = class {
|
|
|
6284
6317
|
get onCursorUpdate() {
|
|
6285
6318
|
return this._onCursorUpdate;
|
|
6286
6319
|
}
|
|
6287
|
-
|
|
6320
|
+
/**
|
|
6321
|
+
* @param LoroDocClass the `LoroDoc` constructor, injected by the caller after
|
|
6322
|
+
* awaiting {@link loadLoro} — keeps `loro-crdt` out of this module's static
|
|
6323
|
+
* import graph so it only ships to apps that use CRDT fields.
|
|
6324
|
+
*/
|
|
6325
|
+
constructor(fieldName, cursorsEnabled, LoroDocClass, initialState, logger) {
|
|
6288
6326
|
this.fieldName = fieldName;
|
|
6289
6327
|
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(fieldName)) throw new Error(`CrdtField: refusing unsafe field identifier '${fieldName}' — must match [a-zA-Z_][a-zA-Z0-9_]*`);
|
|
6290
6328
|
this.logger = logger ?? null;
|
|
6291
6329
|
this.cursorsEnabled = cursorsEnabled;
|
|
6292
|
-
this.doc = new
|
|
6330
|
+
this.doc = new LoroDocClass();
|
|
6293
6331
|
if (initialState && initialState.length > 0) try {
|
|
6294
6332
|
this.doc.import(initialState);
|
|
6295
6333
|
this.loadedFromCrdt = true;
|
|
@@ -6458,6 +6496,19 @@ function encodeBase64(bytes) {
|
|
|
6458
6496
|
return Buffer.from(bytes).toString("base64");
|
|
6459
6497
|
}
|
|
6460
6498
|
|
|
6499
|
+
//#endregion
|
|
6500
|
+
//#region src/modules/crdt/loro-loader.ts
|
|
6501
|
+
let loroPromise = null;
|
|
6502
|
+
/** Start (or return the in-flight/cached) `loro-crdt` import. Fire-and-forget. */
|
|
6503
|
+
function preloadLoro() {
|
|
6504
|
+
loroPromise ??= import("loro-crdt");
|
|
6505
|
+
return loroPromise;
|
|
6506
|
+
}
|
|
6507
|
+
/** Await the loro module, starting the import if `preloadLoro` hasn't run. */
|
|
6508
|
+
function loadLoro() {
|
|
6509
|
+
return preloadLoro();
|
|
6510
|
+
}
|
|
6511
|
+
|
|
6461
6512
|
//#endregion
|
|
6462
6513
|
//#region src/modules/crdt/index.ts
|
|
6463
6514
|
/**
|
|
@@ -6529,7 +6580,8 @@ var CrdtManager = class {
|
|
|
6529
6580
|
Category: "sp00ky-client::CrdtManager::open"
|
|
6530
6581
|
}, "No existing CRDT state found in local cache (continuing with empty doc)");
|
|
6531
6582
|
}
|
|
6532
|
-
|
|
6583
|
+
const { LoroDoc } = await loadLoro();
|
|
6584
|
+
crdtField = new CrdtField(field, cursorsEnabled, LoroDoc, initialCrdtState, this.logger);
|
|
6533
6585
|
crdtField.startSync(this.local, this.remote, recordId, this.sessionId, this.debounceMs);
|
|
6534
6586
|
this.fields.set(key, crdtField);
|
|
6535
6587
|
this.logger.info({
|
|
@@ -7079,6 +7131,7 @@ var Sp00kyClient = class {
|
|
|
7079
7131
|
},
|
|
7080
7132
|
Category: "sp00ky-client::Sp00kyClient::constructor"
|
|
7081
7133
|
}, "Sp00kyClient initialized");
|
|
7134
|
+
if (config.crdt) preloadLoro();
|
|
7082
7135
|
this.local = createLocalEngine(this.config.localEngine, this.config.database, logger);
|
|
7083
7136
|
this.remote = new RemoteDatabaseService(this.config.database, logger);
|
|
7084
7137
|
if (config.persistenceClient === "surrealdb") this.persistenceClient = new SurrealDBPersistenceClient(this.local, logger);
|
package/dist/sqlite-worker.js
CHANGED
|
@@ -96,7 +96,7 @@ async function executeSelect(plan, params, db) {
|
|
|
96
96
|
* hot path; the per-statement ops remain for the write/shim paths.
|
|
97
97
|
*/
|
|
98
98
|
let db = null;
|
|
99
|
-
async function open(dbName, useOpfs) {
|
|
99
|
+
async function open(dbName, useOpfs, systemTables = []) {
|
|
100
100
|
const sqlite3 = await sqlite3InitModule();
|
|
101
101
|
let persisted = false;
|
|
102
102
|
if (useOpfs && sqlite3.installOpfsSAHPoolVfs) try {
|
|
@@ -104,6 +104,7 @@ async function open(dbName, useOpfs) {
|
|
|
104
104
|
persisted = true;
|
|
105
105
|
} catch {}
|
|
106
106
|
if (!db) db = new sqlite3.oo1.DB(":memory:", "c");
|
|
107
|
+
for (const t of systemTables) db.exec({ sql: `CREATE TABLE IF NOT EXISTS "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)` });
|
|
107
108
|
try {
|
|
108
109
|
db.exec({ sql: "PRAGMA busy_timeout = 5000; PRAGMA cache_size = -32000;" });
|
|
109
110
|
} catch {}
|
|
@@ -157,7 +158,8 @@ self.onmessage = async (ev) => {
|
|
|
157
158
|
switch (type) {
|
|
158
159
|
case "open":
|
|
159
160
|
selectDb.knownTables.clear();
|
|
160
|
-
result = await open(payload.dbName, payload.useOpfs);
|
|
161
|
+
result = await open(payload.dbName, payload.useOpfs, payload.systemTables);
|
|
162
|
+
for (const t of payload.systemTables ?? []) selectDb.knownTables.add(t);
|
|
161
163
|
break;
|
|
162
164
|
case "select":
|
|
163
165
|
result = await executeSelect(payload.plan, payload.params ?? {}, selectDb);
|
package/dist/types.d.ts
CHANGED
|
@@ -449,6 +449,14 @@ interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
449
449
|
* collaborators. Defaults to 500ms.
|
|
450
450
|
*/
|
|
451
451
|
crdtDebounceMs?: number;
|
|
452
|
+
/**
|
|
453
|
+
* Enable collaborative CRDT fields. When `true`, the `loro-crdt` engine is
|
|
454
|
+
* preloaded at client startup (fetched as a separate chunk on page load) so
|
|
455
|
+
* the first `openCrdtField` is instant. When omitted/`false`, loro is never
|
|
456
|
+
* loaded unless a CRDT field is explicitly opened — keeping the loro chunk
|
|
457
|
+
* out of apps that don't use collaboration. Defaults to `false`.
|
|
458
|
+
*/
|
|
459
|
+
crdt?: boolean;
|
|
452
460
|
/**
|
|
453
461
|
* Cadence (ms) for the `_00_list_ref` poll that catches cross-session
|
|
454
462
|
* UPDATEs the SurrealDB v3 LIVE-permission gap drops. Lower = faster
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.138",
|
|
4
4
|
"type": "module",
|
|
5
|
+
"sideEffects": false,
|
|
5
6
|
"main": "./dist/index.js",
|
|
6
7
|
"types": "./dist/index.d.ts",
|
|
7
8
|
"exports": {
|
|
@@ -59,8 +60,8 @@
|
|
|
59
60
|
}
|
|
60
61
|
},
|
|
61
62
|
"dependencies": {
|
|
62
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
63
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
63
|
+
"@spooky-sync/query-builder": "0.0.1-canary.138",
|
|
64
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.138",
|
|
64
65
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
65
66
|
"@surrealdb/wasm": "^3.0.3",
|
|
66
67
|
"fast-json-patch": "^3.1.1",
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { LoroDoc } from 'loro-crdt';
|
|
1
|
+
import type { LoroDoc } from 'loro-crdt';
|
|
2
2
|
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
3
3
|
import type { Logger } from '../../services/logger/index';
|
|
4
4
|
import { parseRecordIdString } from '../../utils/index';
|
|
@@ -59,9 +59,15 @@ export class CrdtField {
|
|
|
59
59
|
|
|
60
60
|
get onCursorUpdate() { return this._onCursorUpdate; }
|
|
61
61
|
|
|
62
|
+
/**
|
|
63
|
+
* @param LoroDocClass the `LoroDoc` constructor, injected by the caller after
|
|
64
|
+
* awaiting {@link loadLoro} — keeps `loro-crdt` out of this module's static
|
|
65
|
+
* import graph so it only ships to apps that use CRDT fields.
|
|
66
|
+
*/
|
|
62
67
|
constructor(
|
|
63
68
|
private fieldName: string,
|
|
64
69
|
cursorsEnabled: boolean,
|
|
70
|
+
LoroDocClass: typeof LoroDoc,
|
|
65
71
|
initialState?: Uint8Array,
|
|
66
72
|
logger?: Logger | null,
|
|
67
73
|
) {
|
|
@@ -72,7 +78,7 @@ export class CrdtField {
|
|
|
72
78
|
}
|
|
73
79
|
this.logger = logger ?? null;
|
|
74
80
|
this.cursorsEnabled = cursorsEnabled;
|
|
75
|
-
this.doc = new
|
|
81
|
+
this.doc = new LoroDocClass();
|
|
76
82
|
if (initialState && initialState.length > 0) {
|
|
77
83
|
// Tolerance: catch bad-snapshot data (corrupt blob, stale legacy
|
|
78
84
|
// value left over from a pre-`bytes` migration) so the editor still
|
|
@@ -3,6 +3,7 @@ import type { LocalStore, RemoteDatabaseService } from '../../services/database/
|
|
|
3
3
|
import type { Logger } from '../../services/logger/index';
|
|
4
4
|
import type { Uuid } from 'surrealdb';
|
|
5
5
|
import { CrdtField } from './crdt-field';
|
|
6
|
+
import { loadLoro } from './loro-loader';
|
|
6
7
|
import { parseRecordIdString } from '../../utils/index';
|
|
7
8
|
|
|
8
9
|
export { CrdtField, cursorColorFromName, CURSOR_COLORS } from './crdt-field';
|
|
@@ -101,7 +102,10 @@ export class CrdtManager {
|
|
|
101
102
|
);
|
|
102
103
|
}
|
|
103
104
|
|
|
104
|
-
|
|
105
|
+
// Load loro lazily (usually already resolved — the client preloads it at
|
|
106
|
+
// startup when `config.crdt` is on).
|
|
107
|
+
const { LoroDoc } = await loadLoro();
|
|
108
|
+
crdtField = new CrdtField(field, cursorsEnabled, LoroDoc, initialCrdtState, this.logger);
|
|
105
109
|
crdtField.startSync(this.local, this.remote, recordId, this.sessionId, this.debounceMs);
|
|
106
110
|
this.fields.set(key, crdtField);
|
|
107
111
|
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lazy, cached loader for `loro-crdt`. Keeping loro behind a dynamic `import()`
|
|
3
|
+
* (instead of a static top-level import in `crdt-field.ts`, which is re-exported
|
|
4
|
+
* from the package entry) breaks the static graph edge, so the loro chunk only
|
|
5
|
+
* ships to apps that actually use CRDT fields.
|
|
6
|
+
*
|
|
7
|
+
* `preloadLoro()` is fired from the client constructor when `config.crdt` is on,
|
|
8
|
+
* so the chunk is fetched at page load and the first `openCrdtField` doesn't
|
|
9
|
+
* block on a network round-trip. `loadLoro()` kicks the same import if it hasn't
|
|
10
|
+
* started, so opening a field still works when the flag is off.
|
|
11
|
+
*/
|
|
12
|
+
type LoroModule = typeof import('loro-crdt');
|
|
13
|
+
|
|
14
|
+
let loroPromise: Promise<LoroModule> | null = null;
|
|
15
|
+
|
|
16
|
+
/** Start (or return the in-flight/cached) `loro-crdt` import. Fire-and-forget. */
|
|
17
|
+
export function preloadLoro(): Promise<LoroModule> {
|
|
18
|
+
loroPromise ??= import('loro-crdt');
|
|
19
|
+
return loroPromise;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Await the loro module, starting the import if `preloadLoro` hasn't run. */
|
|
23
|
+
export function loadLoro(): Promise<LoroModule> {
|
|
24
|
+
return preloadLoro();
|
|
25
|
+
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { Diagnostic} from 'surrealdb';
|
|
2
2
|
import { applyDiagnostics, DateTime, RecordId, Surreal } from 'surrealdb';
|
|
3
|
-
import { createWasmWorkerEngines } from '@surrealdb/wasm';
|
|
4
3
|
import type { Sp00kyConfig } from '../../types';
|
|
5
4
|
import type { Logger } from '../logger/index';
|
|
6
5
|
import { AbstractDatabaseService } from './database';
|
|
@@ -30,21 +29,41 @@ export function bucketStoreName(bucketId: string): string {
|
|
|
30
29
|
return `sp00ky-${bucketId}`;
|
|
31
30
|
}
|
|
32
31
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
}
|
|
32
|
+
/** Shared codec: mirrors SurrealDB RecordId/DateTime into our own encodings. */
|
|
33
|
+
const localCodecOptions = {
|
|
34
|
+
valueDecodeVisitor(value: unknown) {
|
|
35
|
+
if (value instanceof RecordId) {
|
|
36
|
+
return encodeRecordId(value);
|
|
37
|
+
}
|
|
40
38
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
39
|
+
if (value instanceof DateTime) {
|
|
40
|
+
return value.toDate();
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return value;
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Engine-less Surreal client used as the constructor placeholder. Building this
|
|
49
|
+
* pulls in NO `@surrealdb/wasm` — the ~6 MB wasm engine is deferred to
|
|
50
|
+
* {@link createLocalSurrealClient}, which runs lazily in `connect`/`switchStore`.
|
|
51
|
+
* `connect()` replaces `this.client` with an engine-backed client before any
|
|
52
|
+
* query runs, so this bare client is never actually opened against a store.
|
|
53
|
+
*/
|
|
54
|
+
function createBareSurrealClient(): Surreal {
|
|
55
|
+
return new Surreal({ codecOptions: localCodecOptions });
|
|
56
|
+
}
|
|
44
57
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Engine-backed client. Dynamically imports `@surrealdb/wasm` so the wasm engine
|
|
60
|
+
* only enters the graph as a separate chunk fetched on first connect (module-
|
|
61
|
+
* cached thereafter — `switchStore`'s await is instant).
|
|
62
|
+
*/
|
|
63
|
+
async function createLocalSurrealClient(logger: Logger): Promise<Surreal> {
|
|
64
|
+
const { createWasmWorkerEngines } = await import('@surrealdb/wasm');
|
|
65
|
+
return new Surreal({
|
|
66
|
+
codecOptions: localCodecOptions,
|
|
48
67
|
engines: applyDiagnostics(
|
|
49
68
|
createWasmWorkerEngines(),
|
|
50
69
|
({ key, type, phase, ...other }: Diagnostic) => {
|
|
@@ -87,7 +106,9 @@ export class LocalDatabaseService extends AbstractDatabaseService {
|
|
|
87
106
|
|
|
88
107
|
constructor(config: Sp00kyConfig<any>['database'], logger: Logger) {
|
|
89
108
|
const events = createDatabaseEventSystem();
|
|
90
|
-
|
|
109
|
+
// Placeholder client with no wasm engine; `connect()` swaps in the real
|
|
110
|
+
// engine-backed client (built lazily) before any query is issued.
|
|
111
|
+
super(createBareSurrealClient(), logger, events);
|
|
91
112
|
this.config = config;
|
|
92
113
|
}
|
|
93
114
|
|
|
@@ -159,6 +180,10 @@ export class LocalDatabaseService extends AbstractDatabaseService {
|
|
|
159
180
|
);
|
|
160
181
|
|
|
161
182
|
this.registerUnloadClose();
|
|
183
|
+
// Build the real engine-backed client lazily here (first `@surrealdb/wasm`
|
|
184
|
+
// load), replacing the constructor's engine-less placeholder before any
|
|
185
|
+
// query runs.
|
|
186
|
+
this.client = await createLocalSurrealClient(this.logger);
|
|
162
187
|
await this.openWithRecovery(this.client, storeUrl, namespace, database, bucketId, store);
|
|
163
188
|
}
|
|
164
189
|
|
|
@@ -195,7 +220,7 @@ export class LocalDatabaseService extends AbstractDatabaseService {
|
|
|
195
220
|
return;
|
|
196
221
|
}
|
|
197
222
|
|
|
198
|
-
const next = createLocalSurrealClient(this.logger);
|
|
223
|
+
const next = await createLocalSurrealClient(this.logger);
|
|
199
224
|
this.pendingSwitchClient = next;
|
|
200
225
|
try {
|
|
201
226
|
await this.openWithRecovery(
|
|
@@ -96,6 +96,45 @@ describe('SqliteCacheEngine.switchBucket serialization', () => {
|
|
|
96
96
|
});
|
|
97
97
|
});
|
|
98
98
|
|
|
99
|
+
// Regression: DEFINE is a noop on this engine, so the `_00_*` internal tables
|
|
100
|
+
// the migrator DEFINEs are never physically created — a fresh bucket that READS
|
|
101
|
+
// one before any write (the sync layer selects `_00_query` at startup) threw
|
|
102
|
+
// "no such table: _00_query" and wedged the client on "Loading database". The
|
|
103
|
+
// fix seeds them inside `open`; assert the open message carries them.
|
|
104
|
+
describe('SqliteCacheEngine system-table seeding', () => {
|
|
105
|
+
it('passes the _00_* system tables to the worker open (fresh-bucket safe)', async () => {
|
|
106
|
+
const msgs: any[] = [];
|
|
107
|
+
const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger());
|
|
108
|
+
(engine as any).spawnWorker = () => {
|
|
109
|
+
const w: any = { onmessage: null, onerror: null, onmessageerror: null, terminate() {} };
|
|
110
|
+
w.postMessage = (msg: any) => {
|
|
111
|
+
msgs.push(msg);
|
|
112
|
+
Promise.resolve().then(() => {
|
|
113
|
+
const rest = msg.type === 'open' ? { persisted: true } : {};
|
|
114
|
+
w.onmessage?.({ data: { id: msg.id, ok: true, ...rest } });
|
|
115
|
+
});
|
|
116
|
+
};
|
|
117
|
+
// Wire pending resolution like the real spawnWorker.
|
|
118
|
+
const inner = w.onmessage;
|
|
119
|
+
w.onmessage = (ev: any) => {
|
|
120
|
+
const { id, ok, error, ...rest } = ev.data ?? {};
|
|
121
|
+
const p = (engine as any).pending.get(id);
|
|
122
|
+
if (!p) return;
|
|
123
|
+
(engine as any).pending.delete(id);
|
|
124
|
+
if (ok) p.resolve(rest);
|
|
125
|
+
else p.reject(new Error(error));
|
|
126
|
+
void inner;
|
|
127
|
+
};
|
|
128
|
+
return w as unknown as Worker;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
await engine.connect('user:fresh');
|
|
132
|
+
const open = msgs.find((m) => m.type === 'open');
|
|
133
|
+
expect(open?.payload?.systemTables).toContain('_00_query');
|
|
134
|
+
expect(open?.payload?.systemTables).toContain('_00_pending_mutations');
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
|
|
99
138
|
// `pureWriteOpResult` is the single source of truth for what a pure-write op
|
|
100
139
|
// contributes to a query's per-statement results. The batched fast path in
|
|
101
140
|
// `query()` and the per-op `execOp` path BOTH route through it, so a caller that
|
|
@@ -34,6 +34,21 @@ import type { EngineTx, Id, LocalStore, OrderBy, RelationFetch, Row } from './ca
|
|
|
34
34
|
* full merged row is only materialized for a LET-wrapped upsert (see the 'let'
|
|
35
35
|
* case). delete/deleteAll yield `[]`; noop yields `null`.
|
|
36
36
|
*/
|
|
37
|
+
/**
|
|
38
|
+
* The `_00_*` internal tables the client relies on. The LocalMigrator DEFINEs
|
|
39
|
+
* them, but every DEFINE lowers to a noop on the SQLite engine, so they must be
|
|
40
|
+
* created physically at open (see `openInternal`) or a read-before-first-write
|
|
41
|
+
* on a fresh bucket throws "no such table". Keep in sync with the systemSchema
|
|
42
|
+
* block in `local-migrator.ts`.
|
|
43
|
+
*/
|
|
44
|
+
const SYSTEM_TABLES = [
|
|
45
|
+
'_00_stream_processor_state',
|
|
46
|
+
'_00_query',
|
|
47
|
+
'_00_preload',
|
|
48
|
+
'_00_schema',
|
|
49
|
+
'_00_pending_mutations',
|
|
50
|
+
] as const;
|
|
51
|
+
|
|
37
52
|
export function pureWriteOpResult(op: SqlOp): unknown {
|
|
38
53
|
switch (op.kind) {
|
|
39
54
|
case 'upsert':
|
|
@@ -217,11 +232,22 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
217
232
|
*/
|
|
218
233
|
private async openInternal(bucketId: string): Promise<void> {
|
|
219
234
|
this.worker = this.spawnWorker();
|
|
235
|
+
// Seed the `_00_*` system tables as part of `open` (worker-side, one round
|
|
236
|
+
// trip). The LocalMigrator DEFINEs them, but `translateSurql` lowers every
|
|
237
|
+
// DEFINE to a noop on this engine (SQLite has no DDL vocabulary), so
|
|
238
|
+
// provisioning never actually creates them — they were only made lazily on
|
|
239
|
+
// first WRITE. A fresh bucket (e.g. right after signup) that READS one first
|
|
240
|
+
// (the sync layer selects `_00_query` before any row lands) hit
|
|
241
|
+
// "no such table: _00_query" and the client wedged on "Loading database".
|
|
242
|
+
// Creating them inside `open` guarantees any access order is safe without
|
|
243
|
+
// adding ops to the engine's queue.
|
|
220
244
|
const { persisted } = await this.rawCall<{ persisted: boolean }>('open', {
|
|
221
245
|
dbName: bucketId,
|
|
222
246
|
useOpfs: this.useOpfs,
|
|
247
|
+
systemTables: SYSTEM_TABLES,
|
|
223
248
|
});
|
|
224
249
|
this.knownTables.clear();
|
|
250
|
+
for (const t of SYSTEM_TABLES) this.knownTables.add(t);
|
|
225
251
|
this.bucketId = bucketId;
|
|
226
252
|
this.logger.info(
|
|
227
253
|
{ bucketId, persisted, Category: 'sp00ky-client::SqliteCacheEngine::connect' },
|
|
@@ -34,7 +34,11 @@ let db: {
|
|
|
34
34
|
close: () => void;
|
|
35
35
|
} | null = null;
|
|
36
36
|
|
|
37
|
-
async function open(
|
|
37
|
+
async function open(
|
|
38
|
+
dbName: string,
|
|
39
|
+
useOpfs: boolean,
|
|
40
|
+
systemTables: readonly string[] = []
|
|
41
|
+
): Promise<{ persisted: boolean }> {
|
|
38
42
|
const sqlite3: any = await sqlite3InitModule();
|
|
39
43
|
let persisted = false;
|
|
40
44
|
if (useOpfs && sqlite3.installOpfsSAHPoolVfs) {
|
|
@@ -47,6 +51,12 @@ async function open(dbName: string, useOpfs: boolean): Promise<{ persisted: bool
|
|
|
47
51
|
}
|
|
48
52
|
}
|
|
49
53
|
if (!db) db = new sqlite3.oo1.DB(':memory:', 'c');
|
|
54
|
+
// Physically create the internal `_00_*` tables the client reads before any
|
|
55
|
+
// write (DEFINE is a noop on this engine, so the migrator can't). Prevents
|
|
56
|
+
// "no such table: _00_query" on a fresh bucket right after signup.
|
|
57
|
+
for (const t of systemTables) {
|
|
58
|
+
db!.exec({ sql: `CREATE TABLE IF NOT EXISTS "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)` });
|
|
59
|
+
}
|
|
50
60
|
// Retry (rather than instantly failing with SQLITE_BUSY=5) if a lock is held.
|
|
51
61
|
// Combined with the engine's single-flight op queue, overlap is avoided.
|
|
52
62
|
// `cache_size` is negated → KiB (here 32 MiB) so SQLite's page cache can't
|
|
@@ -106,7 +116,10 @@ self.onmessage = async (ev: MessageEvent) => {
|
|
|
106
116
|
switch (type) {
|
|
107
117
|
case 'open':
|
|
108
118
|
selectDb.knownTables.clear();
|
|
109
|
-
result = await open(payload.dbName, payload.useOpfs);
|
|
119
|
+
result = await open(payload.dbName, payload.useOpfs, payload.systemTables);
|
|
120
|
+
// The freshly seeded system tables exist — record them so the select
|
|
121
|
+
// path doesn't redundantly re-issue CREATE TABLE for each.
|
|
122
|
+
for (const t of (payload.systemTables ?? []) as string[]) selectDb.knownTables.add(t);
|
|
110
123
|
break;
|
|
111
124
|
case 'select':
|
|
112
125
|
result = await executeSelect(payload.plan, payload.params ?? {}, selectDb);
|
package/src/sp00ky.ts
CHANGED
|
@@ -43,6 +43,7 @@ import { EventSystem } from './events/index';
|
|
|
43
43
|
import { CacheModule } from './modules/cache/index';
|
|
44
44
|
import type { RecordWithId } from './modules/cache/index';
|
|
45
45
|
import { CrdtManager, CrdtField } from './modules/crdt/index';
|
|
46
|
+
import { preloadLoro } from './modules/crdt/loro-loader';
|
|
46
47
|
import { FeatureFlagModule, FeatureFlagHandle } from './modules/feature-flag/index';
|
|
47
48
|
import type { FeatureFlagOptions } from './modules/feature-flag/index';
|
|
48
49
|
import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
|
|
@@ -196,6 +197,11 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
196
197
|
'Sp00kyClient initialized'
|
|
197
198
|
);
|
|
198
199
|
|
|
200
|
+
// Preload the loro CRDT engine at startup (fetches the chunk on page load)
|
|
201
|
+
// so the first `openCrdtField` doesn't block on a network round-trip. Left
|
|
202
|
+
// off, loro is never loaded unless a CRDT field is explicitly opened.
|
|
203
|
+
if (config.crdt) void preloadLoro();
|
|
204
|
+
|
|
199
205
|
// The default ('surrealdb') engine is a SurrealCacheEngine — a drop-in
|
|
200
206
|
// subclass of LocalDatabaseService that adds the engine-neutral verb surface
|
|
201
207
|
// with zero behavior change. Alternate engines (e.g. 'sqlite') require the
|
package/src/types.ts
CHANGED
|
@@ -162,6 +162,14 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
162
162
|
* collaborators. Defaults to 500ms.
|
|
163
163
|
*/
|
|
164
164
|
crdtDebounceMs?: number;
|
|
165
|
+
/**
|
|
166
|
+
* Enable collaborative CRDT fields. When `true`, the `loro-crdt` engine is
|
|
167
|
+
* preloaded at client startup (fetched as a separate chunk on page load) so
|
|
168
|
+
* the first `openCrdtField` is instant. When omitted/`false`, loro is never
|
|
169
|
+
* loaded unless a CRDT field is explicitly opened — keeping the loro chunk
|
|
170
|
+
* out of apps that don't use collaboration. Defaults to `false`.
|
|
171
|
+
*/
|
|
172
|
+
crdt?: boolean;
|
|
165
173
|
/**
|
|
166
174
|
* Cadence (ms) for the `_00_list_ref` poll that catches cross-session
|
|
167
175
|
* UPDATEs the SurrealDB v3 LIVE-permission gap drops. Lower = faster
|