@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +57 -0
- package/dist/index.d.ts +2514 -58
- package/dist/index.js +12561 -2449
- package/dist/otel/index.d.ts +2 -2
- package/dist/otel/index.js +6 -6
- package/dist/sqlite-open.js +303 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +439 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +472 -0
- package/dist/types.d.ts +751 -11
- package/package.json +11 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
- package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
- package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
- package/src/bucket-blurhash.test.ts +148 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +36 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/auth.local-first.test.ts +101 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +127 -24
- package/src/modules/cache/cache.relay.test.ts +95 -0
- package/src/modules/cache/index.ts +163 -43
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +210 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +463 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.membership.test.ts +523 -0
- package/src/modules/data/data.notify-table.test.ts +41 -0
- package/src/modules/data/data.pending-ids.test.ts +199 -0
- package/src/modules/data/data.rebind.test.ts +170 -0
- package/src/modules/data/data.rematerialize.test.ts +114 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.settled-writes.test.ts +206 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +1815 -151
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +194 -0
- package/src/modules/devtools/flags.ts +349 -0
- package/src/modules/devtools/index.ts +450 -46
- package/src/modules/devtools/notify-throttle.test.ts +154 -0
- package/src/modules/devtools/state-shape.test.ts +146 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +168 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +110 -0
- package/src/modules/feature-flag/index.test.ts +251 -0
- package/src/modules/feature-flag/index.ts +308 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +164 -82
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.test.ts +180 -0
- package/src/modules/sync/queue/queue-down.ts +80 -13
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
- package/src/modules/sync/queue/queue-up.ts +241 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.retry.test.ts +237 -0
- package/src/modules/sync/scheduler.ts +215 -13
- package/src/modules/sync/sync.cleanup.test.ts +116 -0
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.heartbeat.test.ts +80 -0
- package/src/modules/sync/sync.live-removal.test.ts +175 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.tabs.test.ts +249 -0
- package/src/modules/sync/sync.ts +1726 -99
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +201 -17
- package/src/otel/index.ts +13 -10
- package/src/services/blobs/blob-cache.test.ts +359 -0
- package/src/services/blobs/blob-cache.ts +603 -0
- package/src/services/blobs/blob-manifest.ts +227 -0
- package/src/services/blobs/blob-store.test.ts +77 -0
- package/src/services/blobs/blob-store.ts +359 -0
- package/src/services/blobs/blob.fixture.ts +90 -0
- package/src/services/blobs/index.ts +70 -0
- package/src/services/database/cache-engine.ts +193 -0
- package/src/services/database/connection-supervisor.test.ts +289 -0
- package/src/services/database/connection-supervisor.ts +415 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +41 -12
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/errors.ts +34 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +7 -0
- package/src/services/database/local-migrator.ts +30 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +484 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +110 -14
- package/src/services/database/sqlite-cache-engine.test.ts +616 -0
- package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
- package/src/services/database/sqlite-cache-engine.ts +1358 -0
- package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
- package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
- package/src/services/database/sqlite-lock-verify.test.ts +33 -0
- package/src/services/database/sqlite-lock-verify.ts +45 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +138 -0
- package/src/services/database/sqlite-projection.test.ts +99 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +131 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +224 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +416 -0
- package/src/services/database/surreal-cache-engine.ts +161 -0
- package/src/services/logger/index.ts +3 -2
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +796 -84
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
- package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +59 -3
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +327 -0
- package/src/services/tabs/coordinator.test.ts +365 -0
- package/src/services/tabs/coordinator.ts +633 -0
- package/src/services/tabs/fake-ports.fixture.ts +112 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +258 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +640 -0
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.local-first.test.ts +60 -0
- package/src/sp00ky.ts +1693 -0
- package/src/types.ts +528 -13
- package/src/utils/blurhash.ts +90 -0
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +79 -13
- package/src/utils/parser.test.ts +49 -120
- package/src/utils/parser.ts +32 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
3
|
+
import type { Logger } from '../../services/logger/index';
|
|
4
|
+
import type { Uuid } from 'surrealdb';
|
|
5
|
+
import { CrdtField } from './crdt-field';
|
|
6
|
+
import { loadLoro } from './loro-loader';
|
|
7
|
+
import { parseRecordIdString } from '../../utils/index';
|
|
8
|
+
|
|
9
|
+
export { CrdtField, cursorColorFromName, CURSOR_COLORS } from './crdt-field';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* CrdtManager manages active CrdtField instances and their sync channels.
|
|
13
|
+
*
|
|
14
|
+
* Collaborative state lives in two dedicated tables (defined in
|
|
15
|
+
* `apps/cli/src/meta_tables_remote.surql`):
|
|
16
|
+
* - `_00_crdt` { record_id, field, state } — one row per (record, field)
|
|
17
|
+
* - `_00_cursor` { record_id, session_id, field, state } — one row per
|
|
18
|
+
* (record, session, field)
|
|
19
|
+
*
|
|
20
|
+
* Splitting them off the parent row is what makes offline edits mergeable:
|
|
21
|
+
* each (record, field) gets its own row, so concurrent offline writes don't
|
|
22
|
+
* collide on the parent's last-write-wins semantics.
|
|
23
|
+
*
|
|
24
|
+
* Cross-browser delivery still rides the parent table's existing LIVE feed
|
|
25
|
+
* to avoid SurrealDB v3 LIVE bugs around dereference-based permission rules
|
|
26
|
+
* (issues 3602, 4026). On every meta UPSERT the writer also bumps the
|
|
27
|
+
* parent's `_00_rv` (a no-op assignment); that fires the parent's LIVE
|
|
28
|
+
* feed, and the receiver pulls the matching `_00_crdt` / `_00_cursor` rows
|
|
29
|
+
* via subquery. Permission inheritance happens server-side via
|
|
30
|
+
* `record_id.id != NONE` (SELECT) and `fn::can_update_record` (UPDATE).
|
|
31
|
+
*/
|
|
32
|
+
export class CrdtManager {
|
|
33
|
+
private fields = new Map<string, CrdtField>();
|
|
34
|
+
// One LIVE subscription per parent table (e.g. "thread" → uuid).
|
|
35
|
+
private liveByTable = new Map<string, Uuid>();
|
|
36
|
+
// Coalesces concurrent first-time subscribes for the same table. Stamped with
|
|
37
|
+
// the connection generation it started under, so a restart after a drop can
|
|
38
|
+
// tell a reusable in-flight attempt from a doomed one.
|
|
39
|
+
private pendingLive = new Map<string, { promise: Promise<void>; generation: number }>();
|
|
40
|
+
// Tables whose LIVE died with a dropped socket, awaiting the next `connected`
|
|
41
|
+
// to be re-subscribed. Held separately because `liveByTable` is cleared on the
|
|
42
|
+
// drop (those uuids belong to a session that no longer exists).
|
|
43
|
+
private staleTables = new Set<string>();
|
|
44
|
+
// Bumped on every socket drop. A LIVE registration that started under an
|
|
45
|
+
// older generation is discarded rather than recorded, so a drop landing
|
|
46
|
+
// mid-registration can't leave a uuid from a dead session in `liveByTable`.
|
|
47
|
+
private connectionGeneration = 0;
|
|
48
|
+
private connectionUnsubscribes: Array<() => void> = [];
|
|
49
|
+
private logger: Logger;
|
|
50
|
+
// SurrealDB session id, used as the per-session key inside `_00_cursor`.
|
|
51
|
+
private sessionId: string = '';
|
|
52
|
+
|
|
53
|
+
constructor(
|
|
54
|
+
private schema: SchemaStructure,
|
|
55
|
+
private local: LocalStore,
|
|
56
|
+
private remote: RemoteDatabaseService,
|
|
57
|
+
logger: Logger,
|
|
58
|
+
private debounceMs: number = 500,
|
|
59
|
+
) {
|
|
60
|
+
this.logger = logger.child({ service: 'CrdtManager' });
|
|
61
|
+
this.subscribeToReconnect();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Re-establish table LIVEs after a socket drop.
|
|
66
|
+
*
|
|
67
|
+
* A LIVE subscription lives and dies with its WebSocket session, and
|
|
68
|
+
* `ensureTableSubscription` is memoized on `liveByTable` — so without this,
|
|
69
|
+
* the first reconnect leaves CRDT realtime permanently dead: the map still
|
|
70
|
+
* holds a uuid for a subscription the server has forgotten, so every later
|
|
71
|
+
* `open()` short-circuits and no LIVE is ever re-issued.
|
|
72
|
+
*
|
|
73
|
+
* Both drop events matter: the SDK publishes `reconnecting` (not
|
|
74
|
+
* `disconnected`) when it intends to recover on its own, and `disconnected`
|
|
75
|
+
* only once it has given up.
|
|
76
|
+
*/
|
|
77
|
+
private subscribeToReconnect(): void {
|
|
78
|
+
const onDrop = () => {
|
|
79
|
+
this.connectionGeneration++;
|
|
80
|
+
for (const table of this.liveByTable.keys()) this.staleTables.add(table);
|
|
81
|
+
// In-flight registrations count too: they'd otherwise complete against
|
|
82
|
+
// the dead session and be silently dropped by the generation guard.
|
|
83
|
+
for (const table of this.pendingLive.keys()) this.staleTables.add(table);
|
|
84
|
+
// No KILL: the server-side subscriptions are already gone, and these
|
|
85
|
+
// uuids would resolve against the new session.
|
|
86
|
+
this.liveByTable.clear();
|
|
87
|
+
};
|
|
88
|
+
this.connectionUnsubscribes.push(
|
|
89
|
+
this.remote.subscribeConnection('reconnecting', onDrop),
|
|
90
|
+
this.remote.subscribeConnection('disconnected', onDrop),
|
|
91
|
+
this.remote.subscribeConnection('connected', () => {
|
|
92
|
+
const tables = Array.from(this.staleTables);
|
|
93
|
+
this.staleTables.clear();
|
|
94
|
+
for (const table of tables) {
|
|
95
|
+
// Only resurrect tables that still have an open field — a drop can
|
|
96
|
+
// outlive the editor that needed the feed.
|
|
97
|
+
if (!this.hasOpenFieldFor(table)) continue;
|
|
98
|
+
void this.ensureTableSubscription(table);
|
|
99
|
+
}
|
|
100
|
+
})
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Stop observing transport events. Separate from {@link closeAll}, which also
|
|
105
|
+
* runs on a bucket switch where the manager keeps being used. */
|
|
106
|
+
dispose(): void {
|
|
107
|
+
for (const off of this.connectionUnsubscribes) {
|
|
108
|
+
try {
|
|
109
|
+
off();
|
|
110
|
+
} catch {
|
|
111
|
+
/* ignore */
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
this.connectionUnsubscribes = [];
|
|
115
|
+
this.staleTables.clear();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
private hasOpenFieldFor(table: string): boolean {
|
|
119
|
+
const prefix = `${table}:`;
|
|
120
|
+
return Array.from(this.fields.keys()).some((k) => k.startsWith(prefix));
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Set the session id that scopes this client's cursor entries. Must be
|
|
124
|
+
* called before `open()` for cursors to be pushed under a stable key.
|
|
125
|
+
* Passed in from `sp00ky.ts` at boot (it already fetches `session::id()`
|
|
126
|
+
* for the data-module salt). */
|
|
127
|
+
setSessionId(sessionId: string): void {
|
|
128
|
+
this.sessionId = sessionId;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Open a CRDT field for collaborative editing.
|
|
133
|
+
*
|
|
134
|
+
* @param table - Table name
|
|
135
|
+
* @param recordId - Full record ID (e.g., "thread:abc")
|
|
136
|
+
* @param field - Field name (e.g., "title", "content")
|
|
137
|
+
* @param fallbackText - Current plain text from the record, used to seed the
|
|
138
|
+
* LoroDoc if no CRDT state exists yet (migration path)
|
|
139
|
+
*/
|
|
140
|
+
async open(
|
|
141
|
+
table: string,
|
|
142
|
+
recordId: string,
|
|
143
|
+
field: string,
|
|
144
|
+
fallbackText?: string,
|
|
145
|
+
): Promise<CrdtField> {
|
|
146
|
+
this.assertCrdtField(table, field);
|
|
147
|
+
const cursorsEnabled = this.fieldHasCursor(table, field);
|
|
148
|
+
const key = this.makeKey(table, recordId, field);
|
|
149
|
+
let crdtField = this.fields.get(key);
|
|
150
|
+
|
|
151
|
+
if (crdtField) {
|
|
152
|
+
return crdtField;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Read the snapshot directly off the parent row. `@crdt`-only fields
|
|
156
|
+
// hold the base64 snapshot inline; `@crdt @cursor` fields hold a
|
|
157
|
+
// `{ state, cursors }` object so we drill into `.state`. The query
|
|
158
|
+
// always selects the local row — sync-down already populated it
|
|
159
|
+
// (the snapshot is a column on the parent, not a sidecar table) so
|
|
160
|
+
// there is no separate fetch on the happy path.
|
|
161
|
+
let initialCrdtState: Uint8Array | undefined;
|
|
162
|
+
try {
|
|
163
|
+
const [result] = await this.local.query<[unknown]>(
|
|
164
|
+
`SELECT VALUE ${field} FROM ONLY $id`,
|
|
165
|
+
{ id: parseRecordIdString(recordId) },
|
|
166
|
+
);
|
|
167
|
+
const snapshot = this.extractSnapshot(result, cursorsEnabled);
|
|
168
|
+
if (snapshot) initialCrdtState = snapshot;
|
|
169
|
+
} catch (e) {
|
|
170
|
+
this.logger.info(
|
|
171
|
+
{ error: String(e), recordId, field, Category: 'sp00ky-client::CrdtManager::open' },
|
|
172
|
+
'No existing CRDT state found in local cache (continuing with empty doc)'
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Load loro lazily (usually already resolved — the client preloads it at
|
|
177
|
+
// startup when `config.crdt` is on).
|
|
178
|
+
const { LoroDoc } = await loadLoro();
|
|
179
|
+
crdtField = new CrdtField(field, cursorsEnabled, LoroDoc, initialCrdtState, this.logger);
|
|
180
|
+
crdtField.startSync(this.local, this.remote, recordId, this.sessionId, this.debounceMs);
|
|
181
|
+
this.fields.set(key, crdtField);
|
|
182
|
+
|
|
183
|
+
this.logger.info(
|
|
184
|
+
{ key, hasInitialState: !!initialCrdtState, hasFallback: !!fallbackText, Category: 'sp00ky-client::CrdtManager::open' },
|
|
185
|
+
'CrdtField opened'
|
|
186
|
+
);
|
|
187
|
+
|
|
188
|
+
// Fire-and-forget: the LIVE subscription receives *future* updates;
|
|
189
|
+
// the initial snapshot is already in hand. `ensureTableSubscription`
|
|
190
|
+
// coalesces concurrent calls via `pendingLive`, so this is safe.
|
|
191
|
+
void this.ensureTableSubscription(table);
|
|
192
|
+
|
|
193
|
+
// Local was empty — a fresh device, a memory-backed local DB after
|
|
194
|
+
// reload, or a record that hasn't been sync'd locally yet. Pull the
|
|
195
|
+
// parent row from remote and dispatch its CRDT field; otherwise the
|
|
196
|
+
// editor sits empty until the parent's LIVE feed happens to fire.
|
|
197
|
+
if (!initialCrdtState) {
|
|
198
|
+
void this.fetchAndDispatchRow(table, recordId);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
return crdtField;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
close(table: string, recordId: string, field: string): void {
|
|
205
|
+
const key = this.makeKey(table, recordId, field);
|
|
206
|
+
const crdtField = this.fields.get(key);
|
|
207
|
+
if (crdtField) {
|
|
208
|
+
crdtField.stopSync();
|
|
209
|
+
this.fields.delete(key);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// If no fields on this table remain open, tear down the table-wide LIVE.
|
|
213
|
+
if (!this.hasOpenFieldFor(table)) {
|
|
214
|
+
this.killTableSubscription(table);
|
|
215
|
+
// Also drop any pending post-reconnect restart for it.
|
|
216
|
+
this.staleTables.delete(table);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
this.logger.debug(
|
|
220
|
+
{ key, Category: 'sp00ky-client::CrdtManager::close' },
|
|
221
|
+
'CrdtField closed'
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Close every open field + table LIVE. Fields flush a final remote push by
|
|
227
|
+
* default; pass `{ flush: false }` on a bucket switch, where that flush
|
|
228
|
+
* would push the previous user's snapshot under the next user's session.
|
|
229
|
+
*/
|
|
230
|
+
closeAll(options: { flush?: boolean } = {}): void {
|
|
231
|
+
for (const [_, field] of this.fields) {
|
|
232
|
+
field.stopSync(options);
|
|
233
|
+
}
|
|
234
|
+
this.fields.clear();
|
|
235
|
+
for (const table of Array.from(this.liveByTable.keys())) {
|
|
236
|
+
this.killTableSubscription(table);
|
|
237
|
+
}
|
|
238
|
+
// Nothing is open any more, so nothing should be resurrected on the next
|
|
239
|
+
// reconnect; `open()` will re-subscribe on demand.
|
|
240
|
+
this.staleTables.clear();
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Ensure a single `LIVE SELECT * FROM <table>` is running, shared across
|
|
244
|
+
* every open CrdtField on `table`. */
|
|
245
|
+
private async ensureTableSubscription(table: string): Promise<void> {
|
|
246
|
+
if (this.liveByTable.has(table)) return;
|
|
247
|
+
|
|
248
|
+
// Registering a LIVE takes two round-trips, so a socket drop can land
|
|
249
|
+
// mid-flight. The generation stamp is what keeps that honest: a registration
|
|
250
|
+
// that started before the drop must never record its uuid, because this
|
|
251
|
+
// method short-circuits on `liveByTable` and that stale entry (pointing at a
|
|
252
|
+
// session the server has forgotten) would block every future restart.
|
|
253
|
+
const generation = this.connectionGeneration;
|
|
254
|
+
|
|
255
|
+
const pending = this.pendingLive.get(table);
|
|
256
|
+
if (pending) {
|
|
257
|
+
if (pending.generation === generation) return pending.promise;
|
|
258
|
+
// A registration from before the drop is still settling. It will discard
|
|
259
|
+
// itself; wait it out, then register fresh on the new socket.
|
|
260
|
+
await pending.promise.catch(() => {});
|
|
261
|
+
if (this.liveByTable.has(table)) return;
|
|
262
|
+
if (generation !== this.connectionGeneration) return;
|
|
263
|
+
// Another caller may have won the race to re-register during that await;
|
|
264
|
+
// join it rather than opening a second LIVE on the same table.
|
|
265
|
+
const successor = this.pendingLive.get(table);
|
|
266
|
+
if (successor && successor.generation === generation) return successor.promise;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const start = (async () => {
|
|
270
|
+
try {
|
|
271
|
+
const [uuid] = await this.remote.query<[Uuid]>(
|
|
272
|
+
`LIVE SELECT * FROM ${table}`,
|
|
273
|
+
);
|
|
274
|
+
|
|
275
|
+
if (generation !== this.connectionGeneration) {
|
|
276
|
+
this.logger.debug(
|
|
277
|
+
{ table, Category: 'sp00ky-client::CrdtManager::ensureTableSubscription' },
|
|
278
|
+
'Socket dropped while registering LIVE; discarding it'
|
|
279
|
+
);
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const subscription = await this.remote.getClient().liveOf(uuid);
|
|
284
|
+
subscription.subscribe((message) => {
|
|
285
|
+
if (message.action === 'KILLED') return;
|
|
286
|
+
if (message.action !== 'CREATE' && message.action !== 'UPDATE') return;
|
|
287
|
+
this.dispatchRow(table, message.value as Record<string, unknown>);
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
if (generation !== this.connectionGeneration) return;
|
|
291
|
+
|
|
292
|
+
this.liveByTable.set(table, uuid);
|
|
293
|
+
this.logger.info(
|
|
294
|
+
{ table, Category: 'sp00ky-client::CrdtManager::ensureTableSubscription' },
|
|
295
|
+
'LIVE SELECT started'
|
|
296
|
+
);
|
|
297
|
+
} catch (e) {
|
|
298
|
+
this.logger.warn(
|
|
299
|
+
{ error: e, table, Category: 'sp00ky-client::CrdtManager::ensureTableSubscription' },
|
|
300
|
+
'Failed to start LIVE SELECT'
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
})();
|
|
304
|
+
|
|
305
|
+
this.pendingLive.set(table, { promise: start, generation });
|
|
306
|
+
try {
|
|
307
|
+
await start;
|
|
308
|
+
} finally {
|
|
309
|
+
this.pendingLive.delete(table);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** Apply a parent-row payload from a non-LIVE source (e.g. the
|
|
314
|
+
* list_ref-driven sync engine, when the cross-user LIVE on the
|
|
315
|
+
* parent table is filtered out by the SurrealDB cross-session
|
|
316
|
+
* permission gap). Same semantics as the internal `dispatchRow`. */
|
|
317
|
+
applyRow(table: string, row: Record<string, unknown>): void {
|
|
318
|
+
this.dispatchRow(table, row);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** Dispatch a parent-row LIVE event to every open CrdtField on that
|
|
322
|
+
* record. Each open field reads its slice of the row directly — the
|
|
323
|
+
* CRDT snapshot is a column on the parent now, so there is no
|
|
324
|
+
* follow-up subquery. */
|
|
325
|
+
private dispatchRow(table: string, row: Record<string, unknown>): void {
|
|
326
|
+
const id = row.id != null ? String(row.id) : '';
|
|
327
|
+
if (!id) return;
|
|
328
|
+
|
|
329
|
+
const rowKeyPrefix = `${table}:${id}:`;
|
|
330
|
+
for (const [key, crdtField] of this.fields) {
|
|
331
|
+
if (!key.startsWith(rowKeyPrefix)) continue;
|
|
332
|
+
const fieldName = key.slice(rowKeyPrefix.length);
|
|
333
|
+
const cursorsEnabled = this.fieldHasCursor(table, fieldName);
|
|
334
|
+
const slice = row[fieldName];
|
|
335
|
+
const snapshot = this.extractSnapshot(slice, cursorsEnabled);
|
|
336
|
+
if (snapshot) crdtField.importRemote(snapshot);
|
|
337
|
+
|
|
338
|
+
if (cursorsEnabled && slice && typeof slice === 'object') {
|
|
339
|
+
const cursors = (slice as { cursors?: unknown }).cursors;
|
|
340
|
+
if (cursors && typeof cursors === 'object') {
|
|
341
|
+
for (const [sid, blob] of Object.entries(cursors as Record<string, unknown>)) {
|
|
342
|
+
if (sid === this.sessionId) continue;
|
|
343
|
+
if (typeof blob === 'string' && blob.length > 0) {
|
|
344
|
+
crdtField.importRemoteCursor(blob);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/** One-shot remote fetch for a row whose CRDT field hasn't synced
|
|
353
|
+
* locally yet (fresh device, memory-backed local DB after reload, …).
|
|
354
|
+
* Used by `open()` when the local read came up empty. Subsequent
|
|
355
|
+
* cross-browser updates ride `dispatchRow` via the parent LIVE feed. */
|
|
356
|
+
private async fetchAndDispatchRow(table: string, id: string): Promise<void> {
|
|
357
|
+
try {
|
|
358
|
+
const recordId = parseRecordIdString(id);
|
|
359
|
+
const [row] = await this.remote.query<[Record<string, unknown> | null]>(
|
|
360
|
+
`SELECT * FROM ONLY $id`,
|
|
361
|
+
{ id: recordId },
|
|
362
|
+
);
|
|
363
|
+
if (!row || typeof row !== 'object') return;
|
|
364
|
+
this.dispatchRow(table, row as Record<string, unknown>);
|
|
365
|
+
} catch (e) {
|
|
366
|
+
this.logger.warn(
|
|
367
|
+
{ error: e, table, id, Category: 'sp00ky-client::CrdtManager::fetchAndDispatchRow' },
|
|
368
|
+
'Failed to fetch parent row for CRDT hydration'
|
|
369
|
+
);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** Schema lookup: does `<table>.<field>` carry a `@cursor` annotation?
|
|
374
|
+
* Determines the on-disk shape (plain snapshot vs. `{ state, cursors }`). */
|
|
375
|
+
private fieldHasCursor(table: string, field: string): boolean {
|
|
376
|
+
const tableSchema = this.schema.tables.find((t) => t.name === table);
|
|
377
|
+
return !!tableSchema?.columns[field]?.cursor;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** Pull the LoroDoc snapshot bytes out of a row slice. For `@crdt`-only
|
|
381
|
+
* the slice IS the snapshot (Uint8Array); for `@crdt @cursor` it's
|
|
382
|
+
* `{ state, cursors }` where `state` carries the snapshot bytes. */
|
|
383
|
+
private extractSnapshot(value: unknown, cursorsEnabled: boolean): Uint8Array | undefined {
|
|
384
|
+
const asBytes = (v: unknown): Uint8Array | undefined => {
|
|
385
|
+
if (v instanceof Uint8Array) return v.length > 0 ? v : undefined;
|
|
386
|
+
// SurrealDB ferries bytes through several shapes depending on
|
|
387
|
+
// transport and on whether the field is a top-level `bytes` column
|
|
388
|
+
// or bytes nested inside a FLEXIBLE object. Round-tripping bytes
|
|
389
|
+
// through `option<object> FLEXIBLE` (the `@crdt @cursor` shape) in
|
|
390
|
+
// particular comes back as a plain `number[]` from the local WASM
|
|
391
|
+
// DB and as `Uint8Array` from the remote WS engine. Normalize all
|
|
392
|
+
// recognized variants here so the receiving CrdtField doesn't care.
|
|
393
|
+
if (v instanceof ArrayBuffer) return new Uint8Array(v);
|
|
394
|
+
if (ArrayBuffer.isView(v)) {
|
|
395
|
+
const view = v as ArrayBufferView;
|
|
396
|
+
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
|
|
397
|
+
}
|
|
398
|
+
if (Array.isArray(v) && v.length > 0 && v.every((n) => typeof n === 'number')) {
|
|
399
|
+
return Uint8Array.from(v as number[]);
|
|
400
|
+
}
|
|
401
|
+
return undefined;
|
|
402
|
+
};
|
|
403
|
+
|
|
404
|
+
if (cursorsEnabled) {
|
|
405
|
+
if (value && typeof value === 'object' && !(value instanceof Uint8Array)) {
|
|
406
|
+
return asBytes((value as { state?: unknown }).state);
|
|
407
|
+
}
|
|
408
|
+
return undefined;
|
|
409
|
+
}
|
|
410
|
+
return asBytes(value);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
private killTableSubscription(table: string): void {
|
|
414
|
+
const uuid = this.liveByTable.get(table);
|
|
415
|
+
if (uuid) {
|
|
416
|
+
// We're tearing down: KILL on an already-dead/closed LIVE throws, and the
|
|
417
|
+
// subscription may have ended on its own, so the failure is expected and
|
|
418
|
+
// safe to drop.
|
|
419
|
+
this.remote.query('KILL $uuid', { uuid }).catch((err) => {
|
|
420
|
+
this.logger.debug(
|
|
421
|
+
{ err, table, Category: 'sp00ky-client::CrdtManager::killTableSubscription' },
|
|
422
|
+
'KILL of table LIVE failed (already closed?)'
|
|
423
|
+
);
|
|
424
|
+
});
|
|
425
|
+
this.liveByTable.delete(table);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
private makeKey(table: string, recordId: string, field: string): string {
|
|
430
|
+
return `${table}:${recordId}:${field}`;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Throws if `<table>.<field>` is not annotated `@crdt` in the schema. Catches
|
|
435
|
+
* typos, removed annotations, and stale schema codegen at the call site instead
|
|
436
|
+
* of silently producing a non-CRDT writer.
|
|
437
|
+
*/
|
|
438
|
+
private assertCrdtField(table: string, field: string): void {
|
|
439
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(field)) {
|
|
440
|
+
throw new Error(
|
|
441
|
+
`openCrdtField: refusing unsafe field identifier '${field}' — must match [a-zA-Z_][a-zA-Z0-9_]*`
|
|
442
|
+
);
|
|
443
|
+
}
|
|
444
|
+
const tableSchema = this.schema.tables.find((t) => t.name === table);
|
|
445
|
+
if (!tableSchema) {
|
|
446
|
+
throw new Error(
|
|
447
|
+
`openCrdtField: unknown table '${table}'. Available: ${this.schema.tables.map((t) => t.name).join(', ')}`
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
const column = tableSchema.columns[field];
|
|
451
|
+
if (!column) {
|
|
452
|
+
throw new Error(
|
|
453
|
+
`openCrdtField: '${table}.${field}' is not in the schema. Available fields: ${Object.keys(tableSchema.columns).join(', ')}`
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
if (!column.crdt) {
|
|
457
|
+
throw new Error(
|
|
458
|
+
`openCrdtField: '${table}.${field}' is not annotated '@crdt' in the schema. ` +
|
|
459
|
+
`Add '-- @crdt text' above the field's DEFINE FIELD and regenerate the client schema.`
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lazy, cached loader for `loro-crdt`. Keeping loro behind a dynamic `import()`
|
|
3
|
+
* (instead of a static top-level import in `crdt-field.ts`, which is re-exported
|
|
4
|
+
* from the package entry) breaks the static graph edge, so the loro chunk only
|
|
5
|
+
* ships to apps that actually use CRDT fields.
|
|
6
|
+
*
|
|
7
|
+
* `preloadLoro()` is fired from the client constructor when `config.crdt` is on,
|
|
8
|
+
* so the chunk is fetched at page load and the first `openCrdtField` doesn't
|
|
9
|
+
* block on a network round-trip. `loadLoro()` kicks the same import if it hasn't
|
|
10
|
+
* started, so opening a field still works when the flag is off.
|
|
11
|
+
*/
|
|
12
|
+
type LoroModule = typeof import('loro-crdt');
|
|
13
|
+
|
|
14
|
+
let loroPromise: Promise<LoroModule> | null = null;
|
|
15
|
+
|
|
16
|
+
/** Start (or return the in-flight/cached) `loro-crdt` import. Fire-and-forget. */
|
|
17
|
+
export function preloadLoro(): Promise<LoroModule> {
|
|
18
|
+
loroPromise ??= import('loro-crdt');
|
|
19
|
+
return loroPromise;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Await the loro module, starting the import if `preloadLoro` hasn't run. */
|
|
23
|
+
export function loadLoro(): Promise<LoroModule> {
|
|
24
|
+
return preloadLoro();
|
|
25
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { DataModule } from './index';
|
|
4
|
+
import type { QueryState } from '../../types';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Tests for instant-hydrate's DataModule half: `applyHydration` — run-once,
|
|
8
|
+
* remoteArray priming, subscriber notify, and the epoch guard added when the
|
|
9
|
+
* hydrate fetch moved off the paint path into a background chain.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
function makeLogger(): any {
|
|
13
|
+
const noop = () => {};
|
|
14
|
+
const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
15
|
+
logger.child = () => logger;
|
|
16
|
+
return logger;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function makeQueryState(hash: string): QueryState {
|
|
20
|
+
return {
|
|
21
|
+
config: {
|
|
22
|
+
id: new RecordId('_00_query', hash),
|
|
23
|
+
surql: 'SELECT * FROM user',
|
|
24
|
+
params: {},
|
|
25
|
+
localArray: [],
|
|
26
|
+
remoteArray: [],
|
|
27
|
+
ttl: '10m',
|
|
28
|
+
lastActiveAt: new Date(),
|
|
29
|
+
tableName: 'user',
|
|
30
|
+
},
|
|
31
|
+
records: [],
|
|
32
|
+
ttlTimer: null,
|
|
33
|
+
ttlDurationMs: 0,
|
|
34
|
+
updateCount: 0,
|
|
35
|
+
lastUpdatedAt: null,
|
|
36
|
+
materializationSamples: [],
|
|
37
|
+
lastIngestLatencyMs: null,
|
|
38
|
+
errorCount: 0,
|
|
39
|
+
status: 'fetching',
|
|
40
|
+
phaseSamples: {},
|
|
41
|
+
phaseLast: {},
|
|
42
|
+
registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const schema = { tables: [{ name: 'user', columns: {} }] } as any;
|
|
47
|
+
|
|
48
|
+
function makeRow(id: string, rv = 1) {
|
|
49
|
+
return { id: new RecordId('user', id), name: id, _00_rv: rv };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
describe('DataModule.applyHydration', () => {
|
|
53
|
+
const hash = 'h1';
|
|
54
|
+
|
|
55
|
+
function setup({ epochFlipsOnSave = false } = {}) {
|
|
56
|
+
let epoch = 1;
|
|
57
|
+
const saved: any[] = [];
|
|
58
|
+
const cache: any = {
|
|
59
|
+
saveBatch: async (batch: any[]) => {
|
|
60
|
+
saved.push(...batch);
|
|
61
|
+
if (epochFlipsOnSave) epoch = 2;
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
const local: any = {
|
|
65
|
+
get epoch() {
|
|
66
|
+
return epoch;
|
|
67
|
+
},
|
|
68
|
+
query: async () => [[makeRow('a')]],
|
|
69
|
+
};
|
|
70
|
+
const dm = new DataModule(cache, local, schema, makeLogger(), 100);
|
|
71
|
+
const state = makeQueryState(hash);
|
|
72
|
+
(dm as any).activeQueries.set(hash, state);
|
|
73
|
+
return { dm, state, saved };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
it('primes remoteArray, materializes and notifies subscribers', async () => {
|
|
77
|
+
const { dm, state, saved } = setup();
|
|
78
|
+
const emissions: any[] = [];
|
|
79
|
+
dm.subscribe(hash, (records) => emissions.push(records));
|
|
80
|
+
|
|
81
|
+
await dm.applyHydration(hash, [makeRow('a', 3)]);
|
|
82
|
+
|
|
83
|
+
expect(state.hydrated).toBe(true);
|
|
84
|
+
expect(state.config.remoteArray).toEqual([['user:a', 3]]);
|
|
85
|
+
expect(saved.length).toBe(1);
|
|
86
|
+
expect(emissions.length).toBe(1);
|
|
87
|
+
expect(dm.isCold(hash)).toBe(false);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('runs once even when the remote returns nothing (hydrated flag)', async () => {
|
|
91
|
+
const { dm, state, saved } = setup();
|
|
92
|
+
await dm.applyHydration(hash, []);
|
|
93
|
+
expect(state.hydrated).toBe(true);
|
|
94
|
+
expect(saved).toEqual([]);
|
|
95
|
+
expect(dm.isCold(hash)).toBe(false); // hydrated → no longer cold
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('is a no-op for an unknown query', async () => {
|
|
99
|
+
const { dm, saved } = setup();
|
|
100
|
+
await dm.applyHydration('nope', [makeRow('a')]);
|
|
101
|
+
expect(saved).toEqual([]);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it('bails before mutating state when the bucket epoch moves mid-persist', async () => {
|
|
105
|
+
const { dm, state } = setup({ epochFlipsOnSave: true });
|
|
106
|
+
const emissions: any[] = [];
|
|
107
|
+
dm.subscribe(hash, (records) => emissions.push(records));
|
|
108
|
+
|
|
109
|
+
await dm.applyHydration(hash, [makeRow('a', 3)]);
|
|
110
|
+
|
|
111
|
+
// Rows fetched under the previous auth context must not prime the new
|
|
112
|
+
// bucket's query state — the rebind's re-registration refills it.
|
|
113
|
+
expect(state.config.remoteArray).toEqual([]);
|
|
114
|
+
expect(state.records).toEqual([]);
|
|
115
|
+
expect(emissions).toEqual([]);
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
describe('DataModule.getPreloadMarker', () => {
|
|
120
|
+
function makeDm(getById: (table: string, id: unknown) => Promise<any>) {
|
|
121
|
+
const local: any = { getById };
|
|
122
|
+
return new DataModule({} as any, local, schema, makeLogger(), 100);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
it('returns the marker fields for a real row', async () => {
|
|
126
|
+
const dm = makeDm(async () => ({ fetchedAt: 123, rowCount: 5 }));
|
|
127
|
+
expect(await dm.getPreloadMarker('h')).toEqual({ fetchedAt: 123, rowCount: 5 });
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it('returns null on a miss and on a non-object echo (SurrealDB string quirk)', async () => {
|
|
131
|
+
expect(await makeDm(async () => null).getPreloadMarker('h')).toBeNull();
|
|
132
|
+
// `FROM ONLY <string>` used to echo the id string back — must not read as warm.
|
|
133
|
+
expect(await makeDm(async () => 'h' as any).getPreloadMarker('h')).toBeNull();
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it('returns null when the read throws', async () => {
|
|
137
|
+
const dm = makeDm(async () => {
|
|
138
|
+
throw new Error('boom');
|
|
139
|
+
});
|
|
140
|
+
expect(await dm.getPreloadMarker('h')).toBeNull();
|
|
141
|
+
});
|
|
142
|
+
});
|