@spooky-sync/core 0.0.1-canary.13 → 0.0.1-canary.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +1171 -372
- package/dist/index.js +5716 -1278
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +107 -0
- package/dist/types.d.ts +746 -0
- package/package.json +39 -8
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +77 -32
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +288 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +357 -0
- package/src/modules/data/data.hydration.test.ts +150 -0
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.recurring.test.ts +137 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/index.ts +1228 -127
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +154 -0
- package/src/modules/devtools/index.ts +191 -30
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.test.ts +120 -0
- package/src/modules/feature-flag/index.ts +209 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.ts +12 -5
- package/src/modules/sync/queue/queue-up.ts +29 -14
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.ts +73 -7
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.ts +1017 -62
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/cache-engine.ts +143 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/engine-factory.ts +32 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +6 -0
- package/src/services/database/local-migrator.ts +28 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +452 -66
- package/src/services/database/plan-render.test.ts +133 -0
- package/src/services/database/plan-render.ts +100 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +13 -13
- package/src/services/database/sqlite-cache-engine.test.ts +85 -0
- package/src/services/database/sqlite-cache-engine.ts +699 -0
- package/src/services/database/sqlite-worker.ts +116 -0
- package/src/services/database/surql-translate.ts +291 -0
- package/src/services/database/surreal-cache-engine.ts +122 -0
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +295 -38
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +104 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +185 -0
- package/src/sp00ky.ts +1016 -0
- package/src/types.ts +258 -15
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +77 -1
- package/src/spooky.ts +0 -392
|
@@ -1,99 +1,485 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Diagnostic} from 'surrealdb';
|
|
2
|
+
import { applyDiagnostics, DateTime, RecordId, Surreal } from 'surrealdb';
|
|
2
3
|
import { createWasmWorkerEngines } from '@surrealdb/wasm';
|
|
3
|
-
import {
|
|
4
|
-
import { Logger } from '../logger/index';
|
|
4
|
+
import type { Sp00kyConfig } from '../../types';
|
|
5
|
+
import type { Logger } from '../logger/index';
|
|
5
6
|
import { AbstractDatabaseService } from './database';
|
|
6
7
|
import { createDatabaseEventSystem, DatabaseEventTypes } from './events/index';
|
|
7
|
-
import { encodeRecordId
|
|
8
|
+
import { encodeRecordId } from '../../utils/index';
|
|
9
|
+
import { ANON_USER_ID } from '../../modules/ref-tables';
|
|
10
|
+
import type { SealedQuery } from '../../utils/surql';
|
|
11
|
+
|
|
12
|
+
/** Thrown when a query carries an `epoch` from before a bucket switch. The
|
|
13
|
+
* caller's chain read from the previous user's store; its write must be
|
|
14
|
+
* dropped, not applied to the new bucket. */
|
|
15
|
+
export class StaleEpochError extends Error {
|
|
16
|
+
constructor() {
|
|
17
|
+
super('Local store epoch changed (bucket switch); stale write dropped');
|
|
18
|
+
this.name = 'StaleEpochError';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Store URL for a local bucket. One IndexedDB store per user (`anon` for
|
|
23
|
+
* signed-out) so cached rows never leak across accounts on a shared device. */
|
|
24
|
+
export function bucketStoreUrl(bucketId: string): string {
|
|
25
|
+
return `indxdb://${bucketStoreName(bucketId)}`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** The IndexedDB database name SurrealDB-WASM derives from the store URL. */
|
|
29
|
+
export function bucketStoreName(bucketId: string): string {
|
|
30
|
+
return `sp00ky-${bucketId}`;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function createLocalSurrealClient(logger: Logger): Surreal {
|
|
34
|
+
return new Surreal({
|
|
35
|
+
codecOptions: {
|
|
36
|
+
valueDecodeVisitor(value) {
|
|
37
|
+
if (value instanceof RecordId) {
|
|
38
|
+
return encodeRecordId(value);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
if (value instanceof DateTime) {
|
|
42
|
+
return value.toDate();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return value;
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
engines: applyDiagnostics(
|
|
49
|
+
createWasmWorkerEngines(),
|
|
50
|
+
({ key, type, phase, ...other }: Diagnostic) => {
|
|
51
|
+
if (phase === 'progress' || phase === 'after') {
|
|
52
|
+
logger.trace(
|
|
53
|
+
{
|
|
54
|
+
...other,
|
|
55
|
+
key,
|
|
56
|
+
type,
|
|
57
|
+
phase,
|
|
58
|
+
service: 'surrealdb:local',
|
|
59
|
+
Category: 'sp00ky-client::LocalDatabaseService::diagnostics',
|
|
60
|
+
},
|
|
61
|
+
`Local SurrealDB diagnostics captured ${type}:${phase}`
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
),
|
|
66
|
+
});
|
|
67
|
+
}
|
|
8
68
|
|
|
9
69
|
export class LocalDatabaseService extends AbstractDatabaseService {
|
|
10
|
-
private config:
|
|
70
|
+
private config: Sp00kyConfig<any>['database'];
|
|
11
71
|
protected eventType = DatabaseEventTypes.LocalQuery;
|
|
12
72
|
|
|
13
|
-
|
|
73
|
+
/** Bucket currently open. Set by `connect`/`switchStore`. */
|
|
74
|
+
private bucketId: string = ANON_USER_ID;
|
|
75
|
+
/**
|
|
76
|
+
* Monotonic store generation. Bumped on every `switchStore`. Async chains
|
|
77
|
+
* that read from the store, await something remote, and then write back
|
|
78
|
+
* (sync poll, SSP stream updates) capture this at chain start and drop
|
|
79
|
+
* their write when it no longer matches — a stale-epoch write would land
|
|
80
|
+
* another user's data in the new bucket.
|
|
81
|
+
*/
|
|
82
|
+
private storeEpoch = 0;
|
|
83
|
+
/** Gate that `query()`/`execute()` await; closed for the switch window. */
|
|
84
|
+
private gate: Promise<void> = Promise.resolve();
|
|
85
|
+
/** The incoming client while a switch is in flight (for unload cleanup). */
|
|
86
|
+
private pendingSwitchClient: Surreal | null = null;
|
|
87
|
+
|
|
88
|
+
constructor(config: Sp00kyConfig<any>['database'], logger: Logger) {
|
|
14
89
|
const events = createDatabaseEventSystem();
|
|
15
|
-
super(
|
|
16
|
-
new Surreal({
|
|
17
|
-
codecOptions: {
|
|
18
|
-
valueDecodeVisitor(value) {
|
|
19
|
-
if (value instanceof RecordId) {
|
|
20
|
-
return encodeRecordId(value);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
if (value instanceof DateTime) {
|
|
24
|
-
return value.toDate();
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
return value;
|
|
28
|
-
},
|
|
29
|
-
},
|
|
30
|
-
engines: applyDiagnostics(
|
|
31
|
-
createWasmWorkerEngines(),
|
|
32
|
-
({ key, type, phase, ...other }: Diagnostic) => {
|
|
33
|
-
if (phase === 'progress' || phase === 'after') {
|
|
34
|
-
logger.trace(
|
|
35
|
-
{
|
|
36
|
-
...other,
|
|
37
|
-
key,
|
|
38
|
-
type,
|
|
39
|
-
phase,
|
|
40
|
-
service: 'surrealdb:local',
|
|
41
|
-
Category: 'spooky-client::LocalDatabaseService::diagnostics',
|
|
42
|
-
},
|
|
43
|
-
`Local SurrealDB diagnostics captured ${type}:${phase}`
|
|
44
|
-
);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
),
|
|
48
|
-
}),
|
|
49
|
-
logger,
|
|
50
|
-
events
|
|
51
|
-
);
|
|
90
|
+
super(createLocalSurrealClient(logger), logger, events);
|
|
52
91
|
this.config = config;
|
|
53
92
|
}
|
|
54
93
|
|
|
55
|
-
getConfig():
|
|
94
|
+
getConfig(): Sp00kyConfig<any>['database'] {
|
|
56
95
|
return this.config;
|
|
57
96
|
}
|
|
58
97
|
|
|
59
|
-
|
|
98
|
+
get currentBucketId(): string {
|
|
99
|
+
return this.bucketId;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
get epoch(): number {
|
|
103
|
+
return this.storeEpoch;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Close the query gate for a bucket switch. Every `query()`/`execute()`
|
|
108
|
+
* issued after this waits until the returned release fn runs — so work
|
|
109
|
+
* triggered mid-switch (sibling auth subscribers registering queries)
|
|
110
|
+
* lands on the NEW bucket instead of racing the swap. The migrator uses
|
|
111
|
+
* `queryUngated()` to provision the new bucket while the gate is closed.
|
|
112
|
+
*/
|
|
113
|
+
beginSwitch(): () => void {
|
|
114
|
+
let release!: () => void;
|
|
115
|
+
this.gate = new Promise<void>((resolve) => {
|
|
116
|
+
release = resolve;
|
|
117
|
+
});
|
|
118
|
+
return release;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
override async query<T extends unknown[]>(
|
|
122
|
+
query: string,
|
|
123
|
+
vars?: Record<string, unknown>,
|
|
124
|
+
opts?: { epoch?: number }
|
|
125
|
+
): Promise<T> {
|
|
126
|
+
await this.gate;
|
|
127
|
+
// A write whose async chain started before a bucket switch must not land
|
|
128
|
+
// in the new bucket — that would be another user's data. Callers on such
|
|
129
|
+
// chains pass the epoch they captured at chain start; mismatches throw.
|
|
130
|
+
if (opts?.epoch !== undefined && opts.epoch !== this.storeEpoch) {
|
|
131
|
+
throw new StaleEpochError();
|
|
132
|
+
}
|
|
133
|
+
return super.query(query, vars);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
override async execute<T>(
|
|
137
|
+
query: SealedQuery<T>,
|
|
138
|
+
vars?: Record<string, unknown>,
|
|
139
|
+
opts?: { epoch?: number }
|
|
140
|
+
): Promise<T> {
|
|
141
|
+
const raw = await this.query<unknown[]>(query.sql, vars, opts);
|
|
142
|
+
return query.extract(raw);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Gate-bypassing query — ONLY for the switch path itself (schema
|
|
146
|
+
* provisioning must run while the gate is closed, or it deadlocks). */
|
|
147
|
+
queryUngated<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T> {
|
|
148
|
+
return super.query(query, vars);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async connect(bucketId: string = ANON_USER_ID): Promise<void> {
|
|
60
152
|
const { namespace, database } = this.getConfig();
|
|
153
|
+
const store = this.getConfig().store ?? 'memory';
|
|
154
|
+
this.bucketId = bucketId;
|
|
155
|
+
const storeUrl = store === 'memory' ? 'mem://' : bucketStoreUrl(bucketId);
|
|
61
156
|
this.logger.info(
|
|
62
|
-
{ namespace, database, Category: '
|
|
157
|
+
{ namespace, database, storeUrl, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
63
158
|
'Connecting to local database'
|
|
64
159
|
);
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
160
|
+
|
|
161
|
+
this.registerUnloadClose();
|
|
162
|
+
await this.openWithRecovery(this.client, storeUrl, namespace, database, bucketId, store);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Switch the local store to another user's bucket. Opens the NEW bucket on a
|
|
167
|
+
* second client first (with the same 3-tier recovery), then atomically swaps
|
|
168
|
+
* `this.client` and closes the old one — a failed open never leaves the
|
|
169
|
+
* service on a dead client. Bumps the store epoch so in-flight old-bucket
|
|
170
|
+
* async chains can detect they're stale.
|
|
171
|
+
*
|
|
172
|
+
* Callers own the drain/rebind choreography (close the gate, quiesce sync +
|
|
173
|
+
* timers BEFORE calling this; re-provision + rebind AFTER).
|
|
174
|
+
*/
|
|
175
|
+
async switchStore(bucketId: string): Promise<void> {
|
|
176
|
+
if (bucketId === this.bucketId) return;
|
|
177
|
+
const { namespace, database } = this.getConfig();
|
|
178
|
+
const store = this.getConfig().store ?? 'memory';
|
|
179
|
+
this.storeEpoch++;
|
|
180
|
+
|
|
181
|
+
if (store === 'memory') {
|
|
182
|
+
// mem:// has no per-user persistence; close + reopen the same client
|
|
183
|
+
// yields a fresh empty store, which is exactly the reset we want.
|
|
184
|
+
try {
|
|
185
|
+
await this.client.close();
|
|
186
|
+
} catch {
|
|
187
|
+
/* ignore */
|
|
188
|
+
}
|
|
189
|
+
await this.openStore(this.client, 'mem://', namespace, database);
|
|
190
|
+
this.bucketId = bucketId;
|
|
191
|
+
this.logger.info(
|
|
192
|
+
{ bucketId, Category: 'sp00ky-client::LocalDatabaseService::switchStore' },
|
|
193
|
+
'Reset in-memory local store for bucket switch'
|
|
76
194
|
);
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
77
197
|
|
|
78
|
-
|
|
198
|
+
const next = createLocalSurrealClient(this.logger);
|
|
199
|
+
this.pendingSwitchClient = next;
|
|
200
|
+
try {
|
|
201
|
+
await this.openWithRecovery(
|
|
202
|
+
next,
|
|
203
|
+
bucketStoreUrl(bucketId),
|
|
79
204
|
namespace,
|
|
80
205
|
database,
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
{ Category: 'spooky-client::LocalDatabaseService::connect' },
|
|
84
|
-
'[LocalDatabaseService] client.use returned'
|
|
206
|
+
bucketId,
|
|
207
|
+
store
|
|
85
208
|
);
|
|
209
|
+
} finally {
|
|
210
|
+
this.pendingSwitchClient = null;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const old = this.client;
|
|
214
|
+
this.client = next;
|
|
215
|
+
this.bucketId = bucketId;
|
|
216
|
+
try {
|
|
217
|
+
await old.close();
|
|
218
|
+
} catch {
|
|
219
|
+
/* best-effort — the old store's handle is released on unload regardless */
|
|
220
|
+
}
|
|
221
|
+
this.logger.info(
|
|
222
|
+
{ bucketId, Category: 'sp00ky-client::LocalDatabaseService::switchStore' },
|
|
223
|
+
'Switched local store bucket'
|
|
224
|
+
);
|
|
225
|
+
}
|
|
86
226
|
|
|
227
|
+
/**
|
|
228
|
+
* Open `storeUrl` on `client` with tiered recovery:
|
|
229
|
+
* tier 1 retries the same store (transient idb-handle races — preserves the
|
|
230
|
+
* cache), tier 2 drops THIS bucket's IndexedDB store and reconnects fresh,
|
|
231
|
+
* tier 3 falls back to `mem://` for the session. Only ever drops the bucket
|
|
232
|
+
* being opened — other users' buckets hold their own caches AND un-pushed
|
|
233
|
+
* mutation outboxes, which must survive another bucket's corruption.
|
|
234
|
+
*/
|
|
235
|
+
private async openWithRecovery(
|
|
236
|
+
client: Surreal,
|
|
237
|
+
storeUrl: string,
|
|
238
|
+
namespace: string,
|
|
239
|
+
database: string,
|
|
240
|
+
bucketId: string,
|
|
241
|
+
store: string
|
|
242
|
+
): Promise<void> {
|
|
243
|
+
try {
|
|
244
|
+
await this.openStore(client, storeUrl, namespace, database);
|
|
87
245
|
this.logger.info(
|
|
88
|
-
{ Category: '
|
|
246
|
+
{ Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
89
247
|
'Connected to local database'
|
|
90
248
|
);
|
|
249
|
+
return;
|
|
91
250
|
} catch (err) {
|
|
251
|
+
// A persistent (IndexedDB) local store can fail to open if it was left
|
|
252
|
+
// corrupt or version-incompatible by a prior session/crash/engine bump.
|
|
253
|
+
// The local store is only a cache (everything re-syncs from the server),
|
|
254
|
+
// so recover by dropping it and reconnecting rather than bricking startup.
|
|
255
|
+
if (store === 'memory' || !isLocalStoreOpenError(err)) {
|
|
256
|
+
this.logger.error(
|
|
257
|
+
{ err, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
258
|
+
'Failed to connect to local database'
|
|
259
|
+
);
|
|
260
|
+
throw err;
|
|
261
|
+
}
|
|
262
|
+
this.logger.warn(
|
|
263
|
+
{ err, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
264
|
+
'Local IndexedDB store failed to open; retrying before clearing'
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// Tier 1 — RETRY the SAME store WITHOUT dropping. The idb open/`use` failure
|
|
269
|
+
// is often transient (a not-yet-released handle from the previous page, or a
|
|
270
|
+
// first-open WAL-recovery race), not real corruption. Closing and reopening
|
|
271
|
+
// frequently succeeds — and crucially PRESERVES the cache, so a warm load
|
|
272
|
+
// stays warm. Dropping the store every time (the old behavior) silently wiped
|
|
273
|
+
// the cache on every reload, making warm loads as slow as cold ones.
|
|
274
|
+
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
275
|
+
try {
|
|
276
|
+
await client.close();
|
|
277
|
+
} catch {
|
|
278
|
+
/* ignore */
|
|
279
|
+
}
|
|
280
|
+
await delay(150 * attempt);
|
|
281
|
+
try {
|
|
282
|
+
await this.openStore(client, storeUrl, namespace, database);
|
|
283
|
+
this.logger.info(
|
|
284
|
+
{ attempt, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
285
|
+
'Connected to local database on retry (cache preserved)'
|
|
286
|
+
);
|
|
287
|
+
return;
|
|
288
|
+
} catch (retryErr) {
|
|
289
|
+
this.logger.warn(
|
|
290
|
+
{ err: retryErr, attempt, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
291
|
+
'Local store retry failed'
|
|
292
|
+
);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// Tier 2 — the store is genuinely unopenable; drop THIS bucket and reconnect
|
|
297
|
+
// fresh. This loses the bucket's cache (re-syncs from the server), so it's
|
|
298
|
+
// the last resort before in-memory.
|
|
299
|
+
try {
|
|
300
|
+
await client.close();
|
|
301
|
+
} catch {
|
|
302
|
+
/* ignore — closing a half-open connection is best-effort */
|
|
303
|
+
}
|
|
304
|
+
await dropLocalIndexedDbStores(this.logger, bucketStoreName(bucketId));
|
|
305
|
+
|
|
306
|
+
try {
|
|
307
|
+
await this.openStore(client, storeUrl, namespace, database);
|
|
308
|
+
this.logger.info(
|
|
309
|
+
{ Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
310
|
+
'Reconnected to local database after clearing the corrupt store'
|
|
311
|
+
);
|
|
312
|
+
} catch (retryErr) {
|
|
313
|
+
// Last resort: run in-memory so the app still loads. No local persistence
|
|
314
|
+
// this session; the freshly-dropped IndexedDB is recreated cleanly next
|
|
315
|
+
// load, and all data re-syncs from the server regardless.
|
|
92
316
|
this.logger.error(
|
|
93
|
-
{ err, Category: '
|
|
94
|
-
'
|
|
317
|
+
{ err: retryErr, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
318
|
+
'Local store still failing after clear; falling back to in-memory'
|
|
319
|
+
);
|
|
320
|
+
try {
|
|
321
|
+
await client.close();
|
|
322
|
+
} catch {
|
|
323
|
+
/* ignore */
|
|
324
|
+
}
|
|
325
|
+
await this.openStore(client, 'mem://', namespace, database);
|
|
326
|
+
this.logger.warn(
|
|
327
|
+
{ Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
328
|
+
'Connected to local database (in-memory fallback)'
|
|
95
329
|
);
|
|
96
|
-
throw err;
|
|
97
330
|
}
|
|
98
331
|
}
|
|
332
|
+
|
|
333
|
+
private unloadCloseRegistered = false;
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* Close the local DB on page unload so the SurrealDB-WASM worker releases its
|
|
337
|
+
* IndexedDB connection cleanly. Without this, the previous page's connection
|
|
338
|
+
* lingers; the next load's `client.connect` opens the store but the first
|
|
339
|
+
* write transaction in `client.use` hits an "IndexedDB error" — which then
|
|
340
|
+
* (mis)triggered the corrupt-store recovery and WIPED the cache on every
|
|
341
|
+
* reload, making warm loads as slow as cold ones. `pagehide` is the reliable
|
|
342
|
+
* unload signal (fires on bfcache + normal navigation); `close()` is async but
|
|
343
|
+
* the WASM worker initiates the IndexedDB connection teardown synchronously.
|
|
344
|
+
* Also closes a mid-switch incoming client so its fresh handle doesn't linger.
|
|
345
|
+
*/
|
|
346
|
+
private registerUnloadClose(): void {
|
|
347
|
+
if (this.unloadCloseRegistered || typeof window === 'undefined') return;
|
|
348
|
+
this.unloadCloseRegistered = true;
|
|
349
|
+
const close = () => {
|
|
350
|
+
try {
|
|
351
|
+
void this.client.close();
|
|
352
|
+
} catch {
|
|
353
|
+
/* best-effort */
|
|
354
|
+
}
|
|
355
|
+
try {
|
|
356
|
+
void this.pendingSwitchClient?.close();
|
|
357
|
+
} catch {
|
|
358
|
+
/* best-effort */
|
|
359
|
+
}
|
|
360
|
+
};
|
|
361
|
+
window.addEventListener('pagehide', close);
|
|
362
|
+
window.addEventListener('beforeunload', close);
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
private async openStore(
|
|
366
|
+
client: Surreal,
|
|
367
|
+
storeUrl: string,
|
|
368
|
+
namespace: string,
|
|
369
|
+
database: string
|
|
370
|
+
): Promise<void> {
|
|
371
|
+
this.logger.debug(
|
|
372
|
+
{ storeUrl, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
373
|
+
'[LocalDatabaseService] Calling client.connect'
|
|
374
|
+
);
|
|
375
|
+
await client.connect(storeUrl, {});
|
|
376
|
+
this.logger.debug(
|
|
377
|
+
{ namespace, database, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
378
|
+
'[LocalDatabaseService] client.connect returned. Calling client.use'
|
|
379
|
+
);
|
|
380
|
+
await client.use({ namespace, database });
|
|
381
|
+
this.logger.debug(
|
|
382
|
+
{ Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
383
|
+
'[LocalDatabaseService] client.use returned'
|
|
384
|
+
);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
function delay(ms: number): Promise<void> {
|
|
389
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** True for the SurrealDB-WASM error raised when its IndexedDB-backed key-value
|
|
393
|
+
* store can't be opened (corrupt / version-incompatible / blocked). Exported
|
|
394
|
+
* for unit testing the error-message match. */
|
|
395
|
+
export function isLocalStoreOpenError(err: unknown): boolean {
|
|
396
|
+
const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
397
|
+
return (
|
|
398
|
+
msg.includes('indexeddb') ||
|
|
399
|
+
msg.includes('idb error') ||
|
|
400
|
+
msg.includes('key-value store')
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/** Best-effort delete of ONE bucket's IndexedDB store(s). SurrealDB-WASM backs
|
|
405
|
+
* `indxdb://<name>` with one or more IndexedDB databases whose names include
|
|
406
|
+
* `<name>`. Scoped to the given store name — never wipe by the bare `sp00ky`
|
|
407
|
+
* substring, that would take every user's bucket (and their un-pushed mutation
|
|
408
|
+
* outboxes) down with one corrupt store. Resolves even on error/blocked so
|
|
409
|
+
* startup can proceed. No-op outside a browser. Exported for unit tests. */
|
|
410
|
+
export async function dropLocalIndexedDbStores(logger: Logger, storeName: string): Promise<void> {
|
|
411
|
+
if (typeof indexedDB === 'undefined') return;
|
|
412
|
+
try {
|
|
413
|
+
let names: string[] = [];
|
|
414
|
+
if (typeof indexedDB.databases === 'function') {
|
|
415
|
+
const dbs = await indexedDB.databases();
|
|
416
|
+
names = dbs
|
|
417
|
+
.map((d) => d.name)
|
|
418
|
+
.filter((n): n is string => !!n && matchesBucketStore(n, storeName));
|
|
419
|
+
}
|
|
420
|
+
// Fall back to the known store name if enumeration is unavailable/empty.
|
|
421
|
+
if (names.length === 0) names = [storeName];
|
|
422
|
+
await Promise.all(names.map(deleteIndexedDb));
|
|
423
|
+
logger.info(
|
|
424
|
+
{ names, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
425
|
+
'Cleared local IndexedDB store(s)'
|
|
426
|
+
);
|
|
427
|
+
} catch (e) {
|
|
428
|
+
logger.warn(
|
|
429
|
+
{ err: e, Category: 'sp00ky-client::LocalDatabaseService::connect' },
|
|
430
|
+
'Failed to enumerate/clear IndexedDB; proceeding anyway'
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/** True when idb database `name` belongs to the bucket store `storeName` —
|
|
436
|
+
* exact match or a derived name (`<storeName>`, `<storeName>-*`, `*<storeName>*`
|
|
437
|
+
* with a non-alphanumeric boundary so `sp00ky-abc` never matches
|
|
438
|
+
* `sp00ky-abcdef`'s store). Exported for unit tests. */
|
|
439
|
+
export function matchesBucketStore(name: string, storeName: string): boolean {
|
|
440
|
+
const lower = name.toLowerCase();
|
|
441
|
+
const target = storeName.toLowerCase();
|
|
442
|
+
const idx = lower.indexOf(target);
|
|
443
|
+
if (idx === -1) return false;
|
|
444
|
+
const after = lower[idx + target.length];
|
|
445
|
+
return after === undefined || !/[a-z0-9_]/.test(after);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/** Nuke EVERY sp00ky local bucket on this device (manual full reset only —
|
|
449
|
+
* the automated corruption recovery is scoped to one bucket). */
|
|
450
|
+
export async function dropAllSp00kyIndexedDbStores(logger: Logger): Promise<void> {
|
|
451
|
+
if (typeof indexedDB === 'undefined') return;
|
|
452
|
+
try {
|
|
453
|
+
let names: string[] = [];
|
|
454
|
+
if (typeof indexedDB.databases === 'function') {
|
|
455
|
+
const dbs = await indexedDB.databases();
|
|
456
|
+
names = dbs
|
|
457
|
+
.map((d) => d.name)
|
|
458
|
+
.filter((n): n is string => !!n && n.toLowerCase().includes('sp00ky'));
|
|
459
|
+
}
|
|
460
|
+
if (names.length === 0) names = ['sp00ky'];
|
|
461
|
+
await Promise.all(names.map(deleteIndexedDb));
|
|
462
|
+
logger.info(
|
|
463
|
+
{ names, Category: 'sp00ky-client::LocalDatabaseService::reset' },
|
|
464
|
+
'Cleared ALL sp00ky IndexedDB stores'
|
|
465
|
+
);
|
|
466
|
+
} catch (e) {
|
|
467
|
+
logger.warn(
|
|
468
|
+
{ err: e, Category: 'sp00ky-client::LocalDatabaseService::reset' },
|
|
469
|
+
'Failed to enumerate/clear IndexedDB; proceeding anyway'
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
function deleteIndexedDb(name: string): Promise<void> {
|
|
475
|
+
return new Promise((resolve) => {
|
|
476
|
+
try {
|
|
477
|
+
const req = indexedDB.deleteDatabase(name);
|
|
478
|
+
req.onsuccess = () => resolve();
|
|
479
|
+
req.onerror = () => resolve();
|
|
480
|
+
req.onblocked = () => resolve();
|
|
481
|
+
} catch {
|
|
482
|
+
resolve();
|
|
483
|
+
}
|
|
484
|
+
});
|
|
99
485
|
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import type { QueryPlan } from '@spooky-sync/query-builder';
|
|
3
|
+
import { renderBaseSelectSurql, renderRelationFetchSurql } from './plan-render';
|
|
4
|
+
import { buildWindowMaterializationPlan } from '../../modules/data/window-query';
|
|
5
|
+
|
|
6
|
+
describe('renderBaseSelectSurql', () => {
|
|
7
|
+
it('renders projection, where, order, limit, offset', () => {
|
|
8
|
+
const plan: QueryPlan = {
|
|
9
|
+
table: 'post',
|
|
10
|
+
select: ['title', 'body'],
|
|
11
|
+
where: [{ field: 'published', op: '=', value: true }],
|
|
12
|
+
orderBy: [['created', 'desc']],
|
|
13
|
+
limit: 10,
|
|
14
|
+
offset: 20,
|
|
15
|
+
};
|
|
16
|
+
const { sql, vars } = renderBaseSelectSurql(plan);
|
|
17
|
+
expect(sql).toBe('SELECT title, body FROM post WHERE published = $__p0 ORDER BY created desc LIMIT 10 START 20;');
|
|
18
|
+
expect(vars).toEqual({ __p0: true });
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('renders OR groups and comparison ops', () => {
|
|
22
|
+
const plan: QueryPlan = {
|
|
23
|
+
table: 'game',
|
|
24
|
+
where: [
|
|
25
|
+
{ or: [{ field: 'white', op: '=', value: 'u:1' }, { field: 'black', op: '=', value: 'u:1' }] },
|
|
26
|
+
{ field: 'moves', op: '>=', value: 5 },
|
|
27
|
+
],
|
|
28
|
+
};
|
|
29
|
+
const { sql, vars } = renderBaseSelectSurql(plan);
|
|
30
|
+
expect(sql).toBe('SELECT * FROM game WHERE (white = $__p0 OR black = $__p1) AND moves >= $__p2;');
|
|
31
|
+
expect(vars).toEqual({ __p0: 'u:1', __p1: 'u:1', __p2: 5 });
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('honors paramRef and swap without binding a new var', () => {
|
|
35
|
+
const plan: QueryPlan = {
|
|
36
|
+
table: 't',
|
|
37
|
+
where: [{ field: 'tags', op: 'CONTAINS', value: undefined, paramRef: 'tag', swap: true }],
|
|
38
|
+
};
|
|
39
|
+
const { sql, vars } = renderBaseSelectSurql(plan, { tag: 'x' });
|
|
40
|
+
expect(sql).toBe('SELECT * FROM t WHERE $tag CONTAINS tags;');
|
|
41
|
+
expect(vars).toEqual({ tag: 'x' });
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
// Regression guard for the "authors + comments don't load" class: any filter on
|
|
46
|
+
// a RecordId column (`id`) must coerce its string value with
|
|
47
|
+
// `type::record(<string> …)`. On SurrealDB `id = "thread:x"` (string) never
|
|
48
|
+
// matches a RecordId, so a base select filtered by id resolves EMPTY and its
|
|
49
|
+
// whole `.related()` subtree (author, comments) loads nothing. The string-based
|
|
50
|
+
// MemStore in relation-resolver.test.ts can't catch this (it compares keys as
|
|
51
|
+
// strings), so these assert the rendered SurrealQL directly — both the parent
|
|
52
|
+
// (base select) and child (relation fetch) sides.
|
|
53
|
+
describe('record-id coercion (authors/comments loading regression)', () => {
|
|
54
|
+
it('coerces a base-select `id = <value>` filter to a record id', () => {
|
|
55
|
+
const plan: QueryPlan = { table: 'thread', where: [{ field: 'id', op: '=', value: 'thread:abc' }] };
|
|
56
|
+
const { sql, vars } = renderBaseSelectSurql(plan);
|
|
57
|
+
expect(sql).toBe('SELECT * FROM thread WHERE id = type::record(<string> $__p0);');
|
|
58
|
+
expect(vars).toEqual({ __p0: 'thread:abc' });
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('coerces a base-select `id = $paramRef` filter (the ThreadDetail path)', () => {
|
|
62
|
+
const plan: QueryPlan = { table: 'thread', where: [{ field: 'id', op: '=', value: undefined, paramRef: 'id' }] };
|
|
63
|
+
const { sql } = renderBaseSelectSurql(plan, { id: 'thread:abc' });
|
|
64
|
+
expect(sql).toBe('SELECT * FROM thread WHERE id = type::record(<string> $id);');
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('does NOT coerce non-id fields (plain string/bool columns stay literal)', () => {
|
|
68
|
+
const plan: QueryPlan = {
|
|
69
|
+
table: 'thread',
|
|
70
|
+
where: [{ field: 'title', op: '=', value: 'hi' }, { field: 'published', op: '=', value: true }],
|
|
71
|
+
};
|
|
72
|
+
const { sql } = renderBaseSelectSurql(plan);
|
|
73
|
+
expect(sql).toBe('SELECT * FROM thread WHERE title = $__p0 AND published = $__p1;');
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('coerces `id` inside an OR group too', () => {
|
|
77
|
+
const plan: QueryPlan = {
|
|
78
|
+
table: 'thread',
|
|
79
|
+
where: [{ or: [{ field: 'id', op: '=', value: 'thread:a' }, { field: 'id', op: '=', value: 'thread:b' }] }],
|
|
80
|
+
};
|
|
81
|
+
const { sql } = renderBaseSelectSurql(plan);
|
|
82
|
+
expect(sql).toBe(
|
|
83
|
+
'SELECT * FROM thread WHERE (id = type::record(<string> $__p0) OR id = type::record(<string> $__p1));'
|
|
84
|
+
);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('relation fetch coerces its matchField keys (the .118 fix — kept locked)', () => {
|
|
88
|
+
const { sql } = renderRelationFetchSurql({ table: 'user', matchField: 'id', keys: ['user:1'] });
|
|
89
|
+
expect(sql).toContain('id IN $__keys.map(|$__k| type::record(<string> $__k))');
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
describe('renderRelationFetchSurql', () => {
|
|
94
|
+
it('builds a WHERE ... IN $__keys batch fetch, omitting LIMIT', () => {
|
|
95
|
+
const { sql, vars } = renderRelationFetchSurql({
|
|
96
|
+
table: 'comment',
|
|
97
|
+
matchField: 'post',
|
|
98
|
+
keys: ['post:1', 'post:2'],
|
|
99
|
+
where: [{ field: 'hidden', op: '=', value: false }],
|
|
100
|
+
orderBy: [['rank', 'asc']],
|
|
101
|
+
});
|
|
102
|
+
expect(sql).toBe('SELECT * FROM comment WHERE post IN $__keys.map(|$__k| type::record(<string> $__k)) AND hidden = $__p0 ORDER BY rank asc;');
|
|
103
|
+
expect(vars).toEqual({ __keys: ['post:1', 'post:2'], __p0: false });
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
describe('buildWindowMaterializationPlan', () => {
|
|
108
|
+
it('restricts to the id-set for offset queries, dropping where/limit/offset', () => {
|
|
109
|
+
const plan: QueryPlan = {
|
|
110
|
+
table: 'post',
|
|
111
|
+
where: [{ field: 'x', op: '=', value: 1 }],
|
|
112
|
+
orderBy: [['created', 'desc']],
|
|
113
|
+
limit: 10,
|
|
114
|
+
offset: 20,
|
|
115
|
+
relations: [{ alias: 'a', table: 'u', cardinality: 'one', foreignKeyField: 'a' }],
|
|
116
|
+
};
|
|
117
|
+
const win = buildWindowMaterializationPlan(plan, ['post:5', 'post:6']);
|
|
118
|
+
expect(win).toEqual({
|
|
119
|
+
table: 'post',
|
|
120
|
+
orderBy: [['created', 'desc']],
|
|
121
|
+
relations: [{ alias: 'a', table: 'u', cardinality: 'one', foreignKeyField: 'a' }],
|
|
122
|
+
ids: ['post:5', 'post:6'],
|
|
123
|
+
where: undefined,
|
|
124
|
+
limit: undefined,
|
|
125
|
+
offset: undefined,
|
|
126
|
+
});
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it('returns null for non-offset queries (keep normal path)', () => {
|
|
130
|
+
expect(buildWindowMaterializationPlan({ table: 'post', limit: 10 }, ['post:1'])).toBeNull();
|
|
131
|
+
expect(buildWindowMaterializationPlan({ table: 'post', offset: 0 }, ['post:1'])).toBeNull();
|
|
132
|
+
});
|
|
133
|
+
});
|