@spooky-sync/core 0.0.1-canary.13 → 0.0.1-canary.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +1171 -372
- package/dist/index.js +5716 -1278
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +107 -0
- package/dist/types.d.ts +746 -0
- package/package.json +39 -8
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +77 -32
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +288 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +357 -0
- package/src/modules/data/data.hydration.test.ts +150 -0
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.recurring.test.ts +137 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/index.ts +1228 -127
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +154 -0
- package/src/modules/devtools/index.ts +191 -30
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.test.ts +120 -0
- package/src/modules/feature-flag/index.ts +209 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.ts +12 -5
- package/src/modules/sync/queue/queue-up.ts +29 -14
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.ts +73 -7
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.ts +1017 -62
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/cache-engine.ts +143 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/engine-factory.ts +32 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +6 -0
- package/src/services/database/local-migrator.ts +28 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +452 -66
- package/src/services/database/plan-render.test.ts +133 -0
- package/src/services/database/plan-render.ts +100 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +13 -13
- package/src/services/database/sqlite-cache-engine.test.ts +85 -0
- package/src/services/database/sqlite-cache-engine.ts +699 -0
- package/src/services/database/sqlite-worker.ts +116 -0
- package/src/services/database/surql-translate.ts +291 -0
- package/src/services/database/surreal-cache-engine.ts +122 -0
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +295 -38
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +104 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +185 -0
- package/src/sp00ky.ts +1016 -0
- package/src/types.ts +258 -15
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +77 -1
- package/src/spooky.ts +0 -392
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
3
|
+
import type { Logger } from '../../services/logger/index';
|
|
4
|
+
import type { Uuid } from 'surrealdb';
|
|
5
|
+
import { CrdtField } from './crdt-field';
|
|
6
|
+
import { parseRecordIdString } from '../../utils/index';
|
|
7
|
+
|
|
8
|
+
export { CrdtField, cursorColorFromName, CURSOR_COLORS } from './crdt-field';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* CrdtManager manages active CrdtField instances and their sync channels.
|
|
12
|
+
*
|
|
13
|
+
* Collaborative state lives in two dedicated tables (defined in
|
|
14
|
+
* `apps/cli/src/meta_tables_remote.surql`):
|
|
15
|
+
* - `_00_crdt` { record_id, field, state } — one row per (record, field)
|
|
16
|
+
* - `_00_cursor` { record_id, session_id, field, state } — one row per
|
|
17
|
+
* (record, session, field)
|
|
18
|
+
*
|
|
19
|
+
* Splitting them off the parent row is what makes offline edits mergeable:
|
|
20
|
+
* each (record, field) gets its own row, so concurrent offline writes don't
|
|
21
|
+
* collide on the parent's last-write-wins semantics.
|
|
22
|
+
*
|
|
23
|
+
* Cross-browser delivery still rides the parent table's existing LIVE feed
|
|
24
|
+
* to avoid SurrealDB v3 LIVE bugs around dereference-based permission rules
|
|
25
|
+
* (issues 3602, 4026). On every meta UPSERT the writer also bumps the
|
|
26
|
+
* parent's `_00_rv` (a no-op assignment); that fires the parent's LIVE
|
|
27
|
+
* feed, and the receiver pulls the matching `_00_crdt` / `_00_cursor` rows
|
|
28
|
+
* via subquery. Permission inheritance happens server-side via
|
|
29
|
+
* `record_id.id != NONE` (SELECT) and `fn::can_update_record` (UPDATE).
|
|
30
|
+
*/
|
|
31
|
+
export class CrdtManager {
|
|
32
|
+
private fields = new Map<string, CrdtField>();
|
|
33
|
+
// One LIVE subscription per parent table (e.g. "thread" → uuid).
|
|
34
|
+
private liveByTable = new Map<string, Uuid>();
|
|
35
|
+
// Coalesces concurrent first-time subscribes for the same table.
|
|
36
|
+
private pendingLive = new Map<string, Promise<void>>();
|
|
37
|
+
private logger: Logger;
|
|
38
|
+
// SurrealDB session id, used as the per-session key inside `_00_cursor`.
|
|
39
|
+
private sessionId: string = '';
|
|
40
|
+
|
|
41
|
+
constructor(
|
|
42
|
+
private schema: SchemaStructure,
|
|
43
|
+
private local: LocalStore,
|
|
44
|
+
private remote: RemoteDatabaseService,
|
|
45
|
+
logger: Logger,
|
|
46
|
+
private debounceMs: number = 500,
|
|
47
|
+
) {
|
|
48
|
+
this.logger = logger.child({ service: 'CrdtManager' });
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Set the session id that scopes this client's cursor entries. Must be
|
|
52
|
+
* called before `open()` for cursors to be pushed under a stable key.
|
|
53
|
+
* Passed in from `sp00ky.ts` at boot (it already fetches `session::id()`
|
|
54
|
+
* for the data-module salt). */
|
|
55
|
+
setSessionId(sessionId: string): void {
|
|
56
|
+
this.sessionId = sessionId;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Open a CRDT field for collaborative editing.
|
|
61
|
+
*
|
|
62
|
+
* @param table - Table name
|
|
63
|
+
* @param recordId - Full record ID (e.g., "thread:abc")
|
|
64
|
+
* @param field - Field name (e.g., "title", "content")
|
|
65
|
+
* @param fallbackText - Current plain text from the record, used to seed the
|
|
66
|
+
* LoroDoc if no CRDT state exists yet (migration path)
|
|
67
|
+
*/
|
|
68
|
+
async open(
|
|
69
|
+
table: string,
|
|
70
|
+
recordId: string,
|
|
71
|
+
field: string,
|
|
72
|
+
fallbackText?: string,
|
|
73
|
+
): Promise<CrdtField> {
|
|
74
|
+
this.assertCrdtField(table, field);
|
|
75
|
+
const cursorsEnabled = this.fieldHasCursor(table, field);
|
|
76
|
+
const key = this.makeKey(table, recordId, field);
|
|
77
|
+
let crdtField = this.fields.get(key);
|
|
78
|
+
|
|
79
|
+
if (crdtField) {
|
|
80
|
+
return crdtField;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Read the snapshot directly off the parent row. `@crdt`-only fields
|
|
84
|
+
// hold the base64 snapshot inline; `@crdt @cursor` fields hold a
|
|
85
|
+
// `{ state, cursors }` object so we drill into `.state`. The query
|
|
86
|
+
// always selects the local row — sync-down already populated it
|
|
87
|
+
// (the snapshot is a column on the parent, not a sidecar table) so
|
|
88
|
+
// there is no separate fetch on the happy path.
|
|
89
|
+
let initialCrdtState: Uint8Array | undefined;
|
|
90
|
+
try {
|
|
91
|
+
const [result] = await this.local.query<[unknown]>(
|
|
92
|
+
`SELECT VALUE ${field} FROM ONLY $id`,
|
|
93
|
+
{ id: parseRecordIdString(recordId) },
|
|
94
|
+
);
|
|
95
|
+
const snapshot = this.extractSnapshot(result, cursorsEnabled);
|
|
96
|
+
if (snapshot) initialCrdtState = snapshot;
|
|
97
|
+
} catch (e) {
|
|
98
|
+
this.logger.info(
|
|
99
|
+
{ error: String(e), recordId, field, Category: 'sp00ky-client::CrdtManager::open' },
|
|
100
|
+
'No existing CRDT state found in local cache (continuing with empty doc)'
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
crdtField = new CrdtField(field, cursorsEnabled, initialCrdtState, this.logger);
|
|
105
|
+
crdtField.startSync(this.local, this.remote, recordId, this.sessionId, this.debounceMs);
|
|
106
|
+
this.fields.set(key, crdtField);
|
|
107
|
+
|
|
108
|
+
this.logger.info(
|
|
109
|
+
{ key, hasInitialState: !!initialCrdtState, hasFallback: !!fallbackText, Category: 'sp00ky-client::CrdtManager::open' },
|
|
110
|
+
'CrdtField opened'
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
// Fire-and-forget: the LIVE subscription receives *future* updates;
|
|
114
|
+
// the initial snapshot is already in hand. `ensureTableSubscription`
|
|
115
|
+
// coalesces concurrent calls via `pendingLive`, so this is safe.
|
|
116
|
+
void this.ensureTableSubscription(table);
|
|
117
|
+
|
|
118
|
+
// Local was empty — a fresh device, a memory-backed local DB after
|
|
119
|
+
// reload, or a record that hasn't been sync'd locally yet. Pull the
|
|
120
|
+
// parent row from remote and dispatch its CRDT field; otherwise the
|
|
121
|
+
// editor sits empty until the parent's LIVE feed happens to fire.
|
|
122
|
+
if (!initialCrdtState) {
|
|
123
|
+
void this.fetchAndDispatchRow(table, recordId);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return crdtField;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
close(table: string, recordId: string, field: string): void {
|
|
130
|
+
const key = this.makeKey(table, recordId, field);
|
|
131
|
+
const crdtField = this.fields.get(key);
|
|
132
|
+
if (crdtField) {
|
|
133
|
+
crdtField.stopSync();
|
|
134
|
+
this.fields.delete(key);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// If no fields on this table remain open, tear down the table-wide LIVE.
|
|
138
|
+
const tablePrefix = `${table}:`;
|
|
139
|
+
const stillOpen = Array.from(this.fields.keys()).some((k) => k.startsWith(tablePrefix));
|
|
140
|
+
if (!stillOpen) {
|
|
141
|
+
this.killTableSubscription(table);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
this.logger.debug(
|
|
145
|
+
{ key, Category: 'sp00ky-client::CrdtManager::close' },
|
|
146
|
+
'CrdtField closed'
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Close every open field + table LIVE. Fields flush a final remote push by
|
|
152
|
+
* default; pass `{ flush: false }` on a bucket switch, where that flush
|
|
153
|
+
* would push the previous user's snapshot under the next user's session.
|
|
154
|
+
*/
|
|
155
|
+
closeAll(options: { flush?: boolean } = {}): void {
|
|
156
|
+
for (const [_, field] of this.fields) {
|
|
157
|
+
field.stopSync(options);
|
|
158
|
+
}
|
|
159
|
+
this.fields.clear();
|
|
160
|
+
for (const table of Array.from(this.liveByTable.keys())) {
|
|
161
|
+
this.killTableSubscription(table);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Ensure a single `LIVE SELECT * FROM <table>` is running, shared across
|
|
166
|
+
* every open CrdtField on `table`. */
|
|
167
|
+
private async ensureTableSubscription(table: string): Promise<void> {
|
|
168
|
+
if (this.liveByTable.has(table)) return;
|
|
169
|
+
|
|
170
|
+
const pending = this.pendingLive.get(table);
|
|
171
|
+
if (pending) return pending;
|
|
172
|
+
|
|
173
|
+
const start = (async () => {
|
|
174
|
+
try {
|
|
175
|
+
const [uuid] = await this.remote.query<[Uuid]>(
|
|
176
|
+
`LIVE SELECT * FROM ${table}`,
|
|
177
|
+
);
|
|
178
|
+
|
|
179
|
+
const subscription = await this.remote.getClient().liveOf(uuid);
|
|
180
|
+
subscription.subscribe((message) => {
|
|
181
|
+
if (message.action === 'KILLED') return;
|
|
182
|
+
if (message.action !== 'CREATE' && message.action !== 'UPDATE') return;
|
|
183
|
+
this.dispatchRow(table, message.value as Record<string, unknown>);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
this.liveByTable.set(table, uuid);
|
|
187
|
+
this.logger.info(
|
|
188
|
+
{ table, Category: 'sp00ky-client::CrdtManager::ensureTableSubscription' },
|
|
189
|
+
'LIVE SELECT started'
|
|
190
|
+
);
|
|
191
|
+
} catch (e) {
|
|
192
|
+
this.logger.warn(
|
|
193
|
+
{ error: e, table, Category: 'sp00ky-client::CrdtManager::ensureTableSubscription' },
|
|
194
|
+
'Failed to start LIVE SELECT'
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
})();
|
|
198
|
+
|
|
199
|
+
this.pendingLive.set(table, start);
|
|
200
|
+
try {
|
|
201
|
+
await start;
|
|
202
|
+
} finally {
|
|
203
|
+
this.pendingLive.delete(table);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Apply a parent-row payload from a non-LIVE source (e.g. the
|
|
208
|
+
* list_ref-driven sync engine, when the cross-user LIVE on the
|
|
209
|
+
* parent table is filtered out by the SurrealDB cross-session
|
|
210
|
+
* permission gap). Same semantics as the internal `dispatchRow`. */
|
|
211
|
+
applyRow(table: string, row: Record<string, unknown>): void {
|
|
212
|
+
this.dispatchRow(table, row);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Dispatch a parent-row LIVE event to every open CrdtField on that
|
|
216
|
+
* record. Each open field reads its slice of the row directly — the
|
|
217
|
+
* CRDT snapshot is a column on the parent now, so there is no
|
|
218
|
+
* follow-up subquery. */
|
|
219
|
+
private dispatchRow(table: string, row: Record<string, unknown>): void {
|
|
220
|
+
const id = row.id != null ? String(row.id) : '';
|
|
221
|
+
if (!id) return;
|
|
222
|
+
|
|
223
|
+
const rowKeyPrefix = `${table}:${id}:`;
|
|
224
|
+
for (const [key, crdtField] of this.fields) {
|
|
225
|
+
if (!key.startsWith(rowKeyPrefix)) continue;
|
|
226
|
+
const fieldName = key.slice(rowKeyPrefix.length);
|
|
227
|
+
const cursorsEnabled = this.fieldHasCursor(table, fieldName);
|
|
228
|
+
const slice = row[fieldName];
|
|
229
|
+
const snapshot = this.extractSnapshot(slice, cursorsEnabled);
|
|
230
|
+
if (snapshot) crdtField.importRemote(snapshot);
|
|
231
|
+
|
|
232
|
+
if (cursorsEnabled && slice && typeof slice === 'object') {
|
|
233
|
+
const cursors = (slice as { cursors?: unknown }).cursors;
|
|
234
|
+
if (cursors && typeof cursors === 'object') {
|
|
235
|
+
for (const [sid, blob] of Object.entries(cursors as Record<string, unknown>)) {
|
|
236
|
+
if (sid === this.sessionId) continue;
|
|
237
|
+
if (typeof blob === 'string' && blob.length > 0) {
|
|
238
|
+
crdtField.importRemoteCursor(blob);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** One-shot remote fetch for a row whose CRDT field hasn't synced
|
|
247
|
+
* locally yet (fresh device, memory-backed local DB after reload, …).
|
|
248
|
+
* Used by `open()` when the local read came up empty. Subsequent
|
|
249
|
+
* cross-browser updates ride `dispatchRow` via the parent LIVE feed. */
|
|
250
|
+
private async fetchAndDispatchRow(table: string, id: string): Promise<void> {
|
|
251
|
+
try {
|
|
252
|
+
const recordId = parseRecordIdString(id);
|
|
253
|
+
const [row] = await this.remote.query<[Record<string, unknown> | null]>(
|
|
254
|
+
`SELECT * FROM ONLY $id`,
|
|
255
|
+
{ id: recordId },
|
|
256
|
+
);
|
|
257
|
+
if (!row || typeof row !== 'object') return;
|
|
258
|
+
this.dispatchRow(table, row as Record<string, unknown>);
|
|
259
|
+
} catch (e) {
|
|
260
|
+
this.logger.warn(
|
|
261
|
+
{ error: e, table, id, Category: 'sp00ky-client::CrdtManager::fetchAndDispatchRow' },
|
|
262
|
+
'Failed to fetch parent row for CRDT hydration'
|
|
263
|
+
);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Schema lookup: does `<table>.<field>` carry a `@cursor` annotation?
|
|
268
|
+
* Determines the on-disk shape (plain snapshot vs. `{ state, cursors }`). */
|
|
269
|
+
private fieldHasCursor(table: string, field: string): boolean {
|
|
270
|
+
const tableSchema = this.schema.tables.find((t) => t.name === table);
|
|
271
|
+
return !!tableSchema?.columns[field]?.cursor;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** Pull the LoroDoc snapshot bytes out of a row slice. For `@crdt`-only
|
|
275
|
+
* the slice IS the snapshot (Uint8Array); for `@crdt @cursor` it's
|
|
276
|
+
* `{ state, cursors }` where `state` carries the snapshot bytes. */
|
|
277
|
+
private extractSnapshot(value: unknown, cursorsEnabled: boolean): Uint8Array | undefined {
|
|
278
|
+
const asBytes = (v: unknown): Uint8Array | undefined => {
|
|
279
|
+
if (v instanceof Uint8Array) return v.length > 0 ? v : undefined;
|
|
280
|
+
// SurrealDB ferries bytes through several shapes depending on
|
|
281
|
+
// transport and on whether the field is a top-level `bytes` column
|
|
282
|
+
// or bytes nested inside a FLEXIBLE object. Round-tripping bytes
|
|
283
|
+
// through `option<object> FLEXIBLE` (the `@crdt @cursor` shape) in
|
|
284
|
+
// particular comes back as a plain `number[]` from the local WASM
|
|
285
|
+
// DB and as `Uint8Array` from the remote WS engine. Normalize all
|
|
286
|
+
// recognized variants here so the receiving CrdtField doesn't care.
|
|
287
|
+
if (v instanceof ArrayBuffer) return new Uint8Array(v);
|
|
288
|
+
if (ArrayBuffer.isView(v)) {
|
|
289
|
+
const view = v as ArrayBufferView;
|
|
290
|
+
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
|
|
291
|
+
}
|
|
292
|
+
if (Array.isArray(v) && v.length > 0 && v.every((n) => typeof n === 'number')) {
|
|
293
|
+
return Uint8Array.from(v as number[]);
|
|
294
|
+
}
|
|
295
|
+
return undefined;
|
|
296
|
+
};
|
|
297
|
+
|
|
298
|
+
if (cursorsEnabled) {
|
|
299
|
+
if (value && typeof value === 'object' && !(value instanceof Uint8Array)) {
|
|
300
|
+
return asBytes((value as { state?: unknown }).state);
|
|
301
|
+
}
|
|
302
|
+
return undefined;
|
|
303
|
+
}
|
|
304
|
+
return asBytes(value);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
private killTableSubscription(table: string): void {
|
|
308
|
+
const uuid = this.liveByTable.get(table);
|
|
309
|
+
if (uuid) {
|
|
310
|
+
// We're tearing down: KILL on an already-dead/closed LIVE throws, and the
|
|
311
|
+
// subscription may have ended on its own, so the failure is expected and
|
|
312
|
+
// safe to drop.
|
|
313
|
+
this.remote.query('KILL $uuid', { uuid }).catch((err) => {
|
|
314
|
+
this.logger.debug(
|
|
315
|
+
{ err, table, Category: 'sp00ky-client::CrdtManager::killTableSubscription' },
|
|
316
|
+
'KILL of table LIVE failed (already closed?)'
|
|
317
|
+
);
|
|
318
|
+
});
|
|
319
|
+
this.liveByTable.delete(table);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
private makeKey(table: string, recordId: string, field: string): string {
|
|
324
|
+
return `${table}:${recordId}:${field}`;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Throws if `<table>.<field>` is not annotated `@crdt` in the schema. Catches
|
|
329
|
+
* typos, removed annotations, and stale schema codegen at the call site instead
|
|
330
|
+
* of silently producing a non-CRDT writer.
|
|
331
|
+
*/
|
|
332
|
+
private assertCrdtField(table: string, field: string): void {
|
|
333
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(field)) {
|
|
334
|
+
throw new Error(
|
|
335
|
+
`openCrdtField: refusing unsafe field identifier '${field}' — must match [a-zA-Z_][a-zA-Z0-9_]*`
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
const tableSchema = this.schema.tables.find((t) => t.name === table);
|
|
339
|
+
if (!tableSchema) {
|
|
340
|
+
throw new Error(
|
|
341
|
+
`openCrdtField: unknown table '${table}'. Available: ${this.schema.tables.map((t) => t.name).join(', ')}`
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
const column = tableSchema.columns[field];
|
|
345
|
+
if (!column) {
|
|
346
|
+
throw new Error(
|
|
347
|
+
`openCrdtField: '${table}.${field}' is not in the schema. Available fields: ${Object.keys(tableSchema.columns).join(', ')}`
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
if (!column.crdt) {
|
|
351
|
+
throw new Error(
|
|
352
|
+
`openCrdtField: '${table}.${field}' is not annotated '@crdt' in the schema. ` +
|
|
353
|
+
`Add '-- @crdt text' above the field's DEFINE FIELD and regenerate the client schema.`
|
|
354
|
+
);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { DataModule } from './index';
|
|
4
|
+
import type { QueryState } from '../../types';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Tests for instant-hydrate's DataModule half: `applyHydration` (run-once,
|
|
8
|
+
* remoteArray priming, subscriber notify, and the epoch guard added when the
|
|
9
|
+
* hydrate fetch moved off the paint path into a background chain) and
|
|
10
|
+
* `isPreloadFresh` (the marker check that lets a freshly-preloaded query skip
|
|
11
|
+
* the duplicate one-shot fetch entirely).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
function makeLogger(): any {
|
|
15
|
+
const noop = () => {};
|
|
16
|
+
const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
17
|
+
logger.child = () => logger;
|
|
18
|
+
return logger;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function makeQueryState(hash: string): QueryState {
|
|
22
|
+
return {
|
|
23
|
+
config: {
|
|
24
|
+
id: new RecordId('_00_query', hash),
|
|
25
|
+
surql: 'SELECT * FROM user',
|
|
26
|
+
params: {},
|
|
27
|
+
localArray: [],
|
|
28
|
+
remoteArray: [],
|
|
29
|
+
ttl: '10m',
|
|
30
|
+
lastActiveAt: new Date(),
|
|
31
|
+
tableName: 'user',
|
|
32
|
+
},
|
|
33
|
+
records: [],
|
|
34
|
+
ttlTimer: null,
|
|
35
|
+
ttlDurationMs: 0,
|
|
36
|
+
updateCount: 0,
|
|
37
|
+
lastUpdatedAt: null,
|
|
38
|
+
materializationSamples: [],
|
|
39
|
+
lastIngestLatencyMs: null,
|
|
40
|
+
errorCount: 0,
|
|
41
|
+
status: 'fetching',
|
|
42
|
+
phaseSamples: {},
|
|
43
|
+
phaseLast: {},
|
|
44
|
+
registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const schema = { tables: [{ name: 'user', columns: {} }] } as any;
|
|
49
|
+
|
|
50
|
+
function makeRow(id: string, rv = 1) {
|
|
51
|
+
return { id: new RecordId('user', id), name: id, _00_rv: rv };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
describe('DataModule.applyHydration', () => {
|
|
55
|
+
const hash = 'h1';
|
|
56
|
+
|
|
57
|
+
function setup({ epochFlipsOnSave = false } = {}) {
|
|
58
|
+
let epoch = 1;
|
|
59
|
+
const saved: any[] = [];
|
|
60
|
+
const cache: any = {
|
|
61
|
+
saveBatch: async (batch: any[]) => {
|
|
62
|
+
saved.push(...batch);
|
|
63
|
+
if (epochFlipsOnSave) epoch = 2;
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
const local: any = {
|
|
67
|
+
get epoch() {
|
|
68
|
+
return epoch;
|
|
69
|
+
},
|
|
70
|
+
query: async () => [[makeRow('a')]],
|
|
71
|
+
};
|
|
72
|
+
const dm = new DataModule(cache, local, schema, makeLogger(), 100);
|
|
73
|
+
const state = makeQueryState(hash);
|
|
74
|
+
(dm as any).activeQueries.set(hash, state);
|
|
75
|
+
return { dm, state, saved };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
it('primes remoteArray, materializes and notifies subscribers', async () => {
|
|
79
|
+
const { dm, state, saved } = setup();
|
|
80
|
+
const emissions: any[] = [];
|
|
81
|
+
dm.subscribe(hash, (records) => emissions.push(records));
|
|
82
|
+
|
|
83
|
+
await dm.applyHydration(hash, [makeRow('a', 3)]);
|
|
84
|
+
|
|
85
|
+
expect(state.hydrated).toBe(true);
|
|
86
|
+
expect(state.config.remoteArray).toEqual([['user:a', 3]]);
|
|
87
|
+
expect(saved.length).toBe(1);
|
|
88
|
+
expect(emissions.length).toBe(1);
|
|
89
|
+
expect(dm.isCold(hash)).toBe(false);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it('runs once even when the remote returns nothing (hydrated flag)', async () => {
|
|
93
|
+
const { dm, state, saved } = setup();
|
|
94
|
+
await dm.applyHydration(hash, []);
|
|
95
|
+
expect(state.hydrated).toBe(true);
|
|
96
|
+
expect(saved).toEqual([]);
|
|
97
|
+
expect(dm.isCold(hash)).toBe(false); // hydrated → no longer cold
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('is a no-op for an unknown query', async () => {
|
|
101
|
+
const { dm, saved } = setup();
|
|
102
|
+
await dm.applyHydration('nope', [makeRow('a')]);
|
|
103
|
+
expect(saved).toEqual([]);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it('bails before mutating state when the bucket epoch moves mid-persist', async () => {
|
|
107
|
+
const { dm, state } = setup({ epochFlipsOnSave: true });
|
|
108
|
+
const emissions: any[] = [];
|
|
109
|
+
dm.subscribe(hash, (records) => emissions.push(records));
|
|
110
|
+
|
|
111
|
+
await dm.applyHydration(hash, [makeRow('a', 3)]);
|
|
112
|
+
|
|
113
|
+
// Rows fetched under the previous auth context must not prime the new
|
|
114
|
+
// bucket's query state — the rebind's re-registration refills it.
|
|
115
|
+
expect(state.config.remoteArray).toEqual([]);
|
|
116
|
+
expect(state.records).toEqual([]);
|
|
117
|
+
expect(emissions).toEqual([]);
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
describe('DataModule.isPreloadFresh', () => {
|
|
122
|
+
const maxAgeMs = 60_000;
|
|
123
|
+
|
|
124
|
+
function makeDm(getById: (table: string, id: string) => Promise<any>) {
|
|
125
|
+
const local: any = { getById };
|
|
126
|
+
return new DataModule({} as any, local, schema, makeLogger(), 100);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
it('true for a marker younger than maxAgeMs', async () => {
|
|
130
|
+
const dm = makeDm(async () => ({ fetchedAt: Date.now() - 1_000, rowCount: 5 }));
|
|
131
|
+
expect(await dm.isPreloadFresh('123', maxAgeMs)).toBe(true);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
it('false for a marker older than maxAgeMs', async () => {
|
|
135
|
+
const dm = makeDm(async () => ({ fetchedAt: Date.now() - maxAgeMs - 1, rowCount: 5 }));
|
|
136
|
+
expect(await dm.isPreloadFresh('123', maxAgeMs)).toBe(false);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('false when no marker exists', async () => {
|
|
140
|
+
const dm = makeDm(async () => null);
|
|
141
|
+
expect(await dm.isPreloadFresh('123', maxAgeMs)).toBe(false);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it('false when the marker read throws (treated as cold)', async () => {
|
|
145
|
+
const dm = makeDm(async () => {
|
|
146
|
+
throw new Error('boom');
|
|
147
|
+
});
|
|
148
|
+
expect(await dm.isPreloadFresh('123', maxAgeMs)).toBe(false);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { DataModule } from './index';
|
|
4
|
+
import type { QueryState } from '../../types';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Tests for the local-bucket-switch surface of DataModule:
|
|
8
|
+
* - `quiesce()` disarms every debounce + TTL timer;
|
|
9
|
+
* - `rebindAfterBucketSwitch()` keeps query hashes (subscriptions stay
|
|
10
|
+
* attached), resets sync state, notifies subscribers with the emptied
|
|
11
|
+
* records so the previous user's rows leave the UI, re-registers the SSP
|
|
12
|
+
* view, and returns the hashes for remote re-registration;
|
|
13
|
+
* - stale-epoch stream updates are dropped instead of applied.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
function makeLogger(): any {
|
|
17
|
+
const noop = () => {};
|
|
18
|
+
const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
19
|
+
logger.child = () => logger;
|
|
20
|
+
return logger;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function makeQueryState(hash: string, records: Record<string, any>[]): QueryState {
|
|
24
|
+
return {
|
|
25
|
+
config: {
|
|
26
|
+
id: new RecordId('_00_query', hash),
|
|
27
|
+
surql: 'SELECT * FROM user',
|
|
28
|
+
params: {},
|
|
29
|
+
localArray: [['user:a', 1]],
|
|
30
|
+
remoteArray: [['user:a', 1]],
|
|
31
|
+
ttl: '10m',
|
|
32
|
+
lastActiveAt: new Date(),
|
|
33
|
+
tableName: 'user',
|
|
34
|
+
},
|
|
35
|
+
records,
|
|
36
|
+
hydrated: true,
|
|
37
|
+
ttlTimer: null,
|
|
38
|
+
ttlDurationMs: 600_000,
|
|
39
|
+
updateCount: 3,
|
|
40
|
+
lastUpdatedAt: null,
|
|
41
|
+
materializationSamples: [],
|
|
42
|
+
lastIngestLatencyMs: null,
|
|
43
|
+
errorCount: 0,
|
|
44
|
+
status: 'idle',
|
|
45
|
+
phaseSamples: {},
|
|
46
|
+
phaseLast: {},
|
|
47
|
+
registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function makeHarness(epoch = 0) {
|
|
52
|
+
const localQueries: Array<{ query: unknown; vars: unknown; opts: unknown }> = [];
|
|
53
|
+
const local: any = {
|
|
54
|
+
epoch,
|
|
55
|
+
query: vi.fn(async (query: unknown, vars?: unknown, opts?: unknown) => {
|
|
56
|
+
localQueries.push({ query, vars, opts });
|
|
57
|
+
return [[]];
|
|
58
|
+
}),
|
|
59
|
+
};
|
|
60
|
+
const cache: any = {
|
|
61
|
+
registerQuery: vi.fn(() => ({ localArray: [] })),
|
|
62
|
+
};
|
|
63
|
+
const dm = new DataModule(cache, local, { tables: [] } as any, makeLogger(), 100);
|
|
64
|
+
return { dm, local, cache, localQueries };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
describe('DataModule.quiesce', () => {
|
|
68
|
+
it('clears debounce and TTL timers', () => {
|
|
69
|
+
vi.useFakeTimers();
|
|
70
|
+
try {
|
|
71
|
+
const { dm } = makeHarness();
|
|
72
|
+
const qs = makeQueryState('h1', []);
|
|
73
|
+
qs.ttlTimer = setTimeout(() => {}, 60_000);
|
|
74
|
+
(dm as any).activeQueries.set('h1', qs);
|
|
75
|
+
(dm as any).debounceTimers.set('h1', setTimeout(() => {}, 60_000));
|
|
76
|
+
|
|
77
|
+
dm.quiesce();
|
|
78
|
+
|
|
79
|
+
expect(qs.ttlTimer).toBeNull();
|
|
80
|
+
expect((dm as any).debounceTimers.size).toBe(0);
|
|
81
|
+
expect(vi.getTimerCount()).toBe(0);
|
|
82
|
+
} finally {
|
|
83
|
+
vi.useRealTimers();
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
describe('DataModule.rebindAfterBucketSwitch', () => {
|
|
89
|
+
let harness: ReturnType<typeof makeHarness>;
|
|
90
|
+
|
|
91
|
+
beforeEach(() => {
|
|
92
|
+
harness = makeHarness();
|
|
93
|
+
(harness.dm as any).activeQueries.set(
|
|
94
|
+
'h1',
|
|
95
|
+
makeQueryState('h1', [{ id: 'user:a', name: 'Previous User Row' }])
|
|
96
|
+
);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it('keeps the hash, resets sync state, and notifies subscribers with empty records', async () => {
|
|
100
|
+
const { dm, cache } = harness;
|
|
101
|
+
const emissions: unknown[][] = [];
|
|
102
|
+
dm.subscribe('h1', (records) => emissions.push(records as unknown[]));
|
|
103
|
+
|
|
104
|
+
const hashes = await dm.rebindAfterBucketSwitch();
|
|
105
|
+
|
|
106
|
+
expect(hashes).toEqual(['h1']);
|
|
107
|
+
const qs = (dm as any).activeQueries.get('h1') as QueryState;
|
|
108
|
+
expect(qs.records).toEqual([]);
|
|
109
|
+
expect(qs.config.localArray).toEqual([]);
|
|
110
|
+
expect(qs.config.remoteArray).toEqual([]);
|
|
111
|
+
expect(qs.hydrated).toBe(false);
|
|
112
|
+
expect(qs.status).toBe('fetching');
|
|
113
|
+
// Subscribers saw the previous user's rows drop out.
|
|
114
|
+
expect(emissions).toEqual([[]]);
|
|
115
|
+
// The SSP view was re-registered (on the fresh, post-reset processor).
|
|
116
|
+
expect(cache.registerQuery).toHaveBeenCalledWith(
|
|
117
|
+
expect.objectContaining({ queryHash: 'h1', surql: 'SELECT * FROM user' })
|
|
118
|
+
);
|
|
119
|
+
// The TTL heartbeat is re-armed.
|
|
120
|
+
expect(qs.ttlTimer).not.toBeNull();
|
|
121
|
+
if (qs.ttlTimer) clearTimeout(qs.ttlTimer);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
describe('stale-epoch stream updates', () => {
|
|
126
|
+
it('drops an update whose chain started before a bucket switch', async () => {
|
|
127
|
+
const { dm, local } = makeHarness();
|
|
128
|
+
const qs = makeQueryState('h1', [{ id: 'user:a' }]);
|
|
129
|
+
(dm as any).activeQueries.set('h1', qs);
|
|
130
|
+
|
|
131
|
+
// The materialize read happens, then the epoch moves (bucket switched).
|
|
132
|
+
local.query.mockImplementation(async () => {
|
|
133
|
+
local.epoch = 1;
|
|
134
|
+
return [[{ id: 'user:b', name: 'other user row' }]];
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
await (dm as any).processStreamUpdate({
|
|
138
|
+
queryHash: 'h1',
|
|
139
|
+
localArray: [['user:b', 1]],
|
|
140
|
+
op: 'CREATE',
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// Neither the records nor the persisted arrays moved.
|
|
144
|
+
expect(qs.records).toEqual([{ id: 'user:a' }]);
|
|
145
|
+
expect(qs.config.localArray).toEqual([['user:a', 1]]);
|
|
146
|
+
});
|
|
147
|
+
});
|