@spooky-sync/core 0.0.1-canary.9 → 0.0.1-canary.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +907 -339
- package/dist/index.js +2837 -402
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +543 -0
- package/package.json +39 -9
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +41 -29
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +281 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +352 -0
- package/src/modules/data/data.status.test.ts +108 -0
- package/src/modules/data/index.ts +732 -109
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +130 -0
- package/src/modules/devtools/index.ts +115 -21
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.ts +166 -0
- package/src/modules/ref-tables.test.ts +66 -0
- package/src/modules/ref-tables.ts +69 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +40 -3
- package/src/modules/sync/sync.ts +893 -59
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.test.ts +33 -0
- package/src/services/database/local.ts +192 -32
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +248 -37
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +65 -0
- package/src/sp00ky.ts +648 -0
- package/src/types.ts +192 -15
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +55 -1
- package/src/spooky.ts +0 -392
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { LoroDoc } from 'loro-crdt';
|
|
3
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
4
|
+
import { CrdtManager } from './index';
|
|
5
|
+
import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
6
|
+
import type { Logger } from '../../services/logger/index';
|
|
7
|
+
|
|
8
|
+
// Regression test for the offline-reload formatting-loss class of bug.
|
|
9
|
+
//
|
|
10
|
+
// Repro pattern: app uses `store: 'memory'` for the local SurrealDB (or
|
|
11
|
+
// any scenario where the local row exists without a CRDT snapshot —
|
|
12
|
+
// fresh device, post-sync-down with stale meta, …). On reload, the
|
|
13
|
+
// editor mounts but stays empty until some unrelated edit fires the
|
|
14
|
+
// parent LIVE feed. From the user's POV, formatting is gone.
|
|
15
|
+
//
|
|
16
|
+
// Fix: when `CrdtManager.open` finds no local snapshot, it must fetch
|
|
17
|
+
// the parent row from remote (the snapshot lives inline on the row in
|
|
18
|
+
// the consolidated design) and dispatch its CRDT field. This test pins
|
|
19
|
+
// that behavior across both shapes — `@crdt`-only fields where the
|
|
20
|
+
// snapshot IS the field value, and `@crdt @cursor` fields where the
|
|
21
|
+
// snapshot lives at `<field>.state`.
|
|
22
|
+
|
|
23
|
+
function silentLogger(): Logger {
|
|
24
|
+
const noop = () => {};
|
|
25
|
+
const fake: any = {};
|
|
26
|
+
fake.info = noop;
|
|
27
|
+
fake.warn = noop;
|
|
28
|
+
fake.debug = noop;
|
|
29
|
+
fake.error = noop;
|
|
30
|
+
fake.trace = noop;
|
|
31
|
+
fake.child = () => fake;
|
|
32
|
+
return fake as Logger;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function singleTableSchema(opts: { cursor?: boolean } = {}): SchemaStructure {
|
|
36
|
+
return {
|
|
37
|
+
tables: [
|
|
38
|
+
{
|
|
39
|
+
name: 'thread',
|
|
40
|
+
columns: {
|
|
41
|
+
body: {
|
|
42
|
+
type: 'string',
|
|
43
|
+
optional: false,
|
|
44
|
+
crdt: 'text',
|
|
45
|
+
...(opts.cursor ? { cursor: true } : {}),
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
primaryKey: ['id'],
|
|
49
|
+
},
|
|
50
|
+
],
|
|
51
|
+
relationships: [],
|
|
52
|
+
backends: {},
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function buildSnapshot(text: string): Uint8Array {
|
|
57
|
+
const doc = new LoroDoc();
|
|
58
|
+
doc.getText('text').insert(0, text);
|
|
59
|
+
return doc.export({ mode: 'snapshot' });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function makeRemote(handlers: {
|
|
63
|
+
selectRow?: () => unknown;
|
|
64
|
+
}): RemoteDatabaseService {
|
|
65
|
+
const remote = {
|
|
66
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
67
|
+
if (sql.includes('LIVE SELECT')) return ['live-uuid'];
|
|
68
|
+
if (sql.includes('SELECT * FROM ONLY')) {
|
|
69
|
+
return [handlers.selectRow ? handlers.selectRow() : null];
|
|
70
|
+
}
|
|
71
|
+
return [];
|
|
72
|
+
}),
|
|
73
|
+
getClient: () =>
|
|
74
|
+
({
|
|
75
|
+
liveOf: async () => ({
|
|
76
|
+
subscribe: () => () => {},
|
|
77
|
+
}),
|
|
78
|
+
}) as any,
|
|
79
|
+
} satisfies Partial<RemoteDatabaseService> as unknown as RemoteDatabaseService;
|
|
80
|
+
return remote;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
describe('CrdtManager.open hydration', () => {
|
|
84
|
+
it('@crdt-only: fetches the parent row from remote when local is empty', async () => {
|
|
85
|
+
const remoteSnapshot = buildSnapshot('hello world');
|
|
86
|
+
|
|
87
|
+
const local = {
|
|
88
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
89
|
+
if (sql.includes('SELECT VALUE body FROM ONLY')) return [null];
|
|
90
|
+
return [];
|
|
91
|
+
}),
|
|
92
|
+
} satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
|
|
93
|
+
|
|
94
|
+
const remote = makeRemote({
|
|
95
|
+
selectRow: () => ({ id: 'thread:abc', body: remoteSnapshot }),
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
const manager = new CrdtManager(singleTableSchema(), local, remote, silentLogger());
|
|
99
|
+
manager.setSessionId('session-under-test');
|
|
100
|
+
|
|
101
|
+
const field = await manager.open('thread', 'thread:abc', 'body');
|
|
102
|
+
|
|
103
|
+
await vi.waitFor(() => {
|
|
104
|
+
expect(field.getDoc().getText('text').toString()).toBe('hello world');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
const sqls = (remote.query as ReturnType<typeof vi.fn>).mock.calls.map(
|
|
108
|
+
(c) => c[0] as string
|
|
109
|
+
);
|
|
110
|
+
expect(sqls.some((s) => s.includes('SELECT * FROM ONLY'))).toBe(true);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it('@crdt @cursor: extracts the snapshot from the object shape', async () => {
|
|
114
|
+
const remoteSnapshot = buildSnapshot('object shape');
|
|
115
|
+
|
|
116
|
+
const local = {
|
|
117
|
+
query: vi.fn().mockImplementation(async () => [null]),
|
|
118
|
+
} satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
|
|
119
|
+
|
|
120
|
+
// Cursor-enabled field stores `{ state, cursors }`. The remote row
|
|
121
|
+
// returns this shape; the manager must drill into `.state` for the
|
|
122
|
+
// snapshot bytes.
|
|
123
|
+
const remote = makeRemote({
|
|
124
|
+
selectRow: () => ({
|
|
125
|
+
id: 'thread:abc',
|
|
126
|
+
body: { state: remoteSnapshot, cursors: {} },
|
|
127
|
+
}),
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
const manager = new CrdtManager(
|
|
131
|
+
singleTableSchema({ cursor: true }),
|
|
132
|
+
local,
|
|
133
|
+
remote,
|
|
134
|
+
silentLogger()
|
|
135
|
+
);
|
|
136
|
+
manager.setSessionId('session-under-test');
|
|
137
|
+
|
|
138
|
+
const field = await manager.open('thread', 'thread:abc', 'body');
|
|
139
|
+
|
|
140
|
+
await vi.waitFor(() => {
|
|
141
|
+
expect(field.getDoc().getText('text').toString()).toBe('object shape');
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it('skips the remote fetch when a local snapshot already exists', async () => {
|
|
146
|
+
const localSnapshot = buildSnapshot('cached');
|
|
147
|
+
|
|
148
|
+
const local = {
|
|
149
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
150
|
+
if (sql.includes('SELECT VALUE body FROM ONLY')) return [localSnapshot];
|
|
151
|
+
return [];
|
|
152
|
+
}),
|
|
153
|
+
} satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
|
|
154
|
+
|
|
155
|
+
const remote = makeRemote({});
|
|
156
|
+
|
|
157
|
+
const manager = new CrdtManager(singleTableSchema(), local, remote, silentLogger());
|
|
158
|
+
manager.setSessionId('session-under-test');
|
|
159
|
+
|
|
160
|
+
const field = await manager.open('thread', 'thread:def', 'body');
|
|
161
|
+
|
|
162
|
+
expect(field.getDoc().getText('text').toString()).toBe('cached');
|
|
163
|
+
|
|
164
|
+
// Give any stray async work a tick to land before asserting absence.
|
|
165
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
166
|
+
|
|
167
|
+
const sqls = (remote.query as ReturnType<typeof vi.fn>).mock.calls.map(
|
|
168
|
+
(c) => c[0] as string
|
|
169
|
+
);
|
|
170
|
+
expect(sqls.some((s) => s.includes('SELECT * FROM ONLY'))).toBe(false);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
// Regression: SurrealDB returns bytes nested inside a FLEXIBLE object as a
|
|
174
|
+
// plain `number[]` from the local WASM engine (e.g. `state: [108, 111, 114,
|
|
175
|
+
// 111, ...]` — the literal LoroDoc magic header bytes). The cursor-enabled
|
|
176
|
+
// extractSnapshot path used to only accept Uint8Array and would silently
|
|
177
|
+
// return undefined for the array form, leaving every editor empty and
|
|
178
|
+
// killing realtime updates for `@crdt @cursor` fields.
|
|
179
|
+
it('@crdt @cursor: accepts number[] as the state shape (local WASM transport)', async () => {
|
|
180
|
+
const remoteSnapshot = buildSnapshot('from local wasm');
|
|
181
|
+
const remoteAsArray = Array.from(remoteSnapshot) as unknown as number[];
|
|
182
|
+
|
|
183
|
+
const local = {
|
|
184
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
185
|
+
if (sql.includes('SELECT VALUE body FROM ONLY')) {
|
|
186
|
+
return [{ state: remoteAsArray, cursors: {} }];
|
|
187
|
+
}
|
|
188
|
+
return [];
|
|
189
|
+
}),
|
|
190
|
+
} satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
|
|
191
|
+
|
|
192
|
+
const remote = makeRemote({});
|
|
193
|
+
|
|
194
|
+
const manager = new CrdtManager(
|
|
195
|
+
singleTableSchema({ cursor: true }),
|
|
196
|
+
local,
|
|
197
|
+
remote,
|
|
198
|
+
silentLogger()
|
|
199
|
+
);
|
|
200
|
+
manager.setSessionId('session-under-test');
|
|
201
|
+
|
|
202
|
+
const field = await manager.open('thread', 'thread:wasm', 'body');
|
|
203
|
+
|
|
204
|
+
expect(field.getDoc().getText('text').toString()).toBe('from local wasm');
|
|
205
|
+
});
|
|
206
|
+
});
|
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
3
|
+
import type { Logger } from '../../services/logger/index';
|
|
4
|
+
import type { Uuid } from 'surrealdb';
|
|
5
|
+
import { CrdtField } from './crdt-field';
|
|
6
|
+
import { parseRecordIdString } from '../../utils/index';
|
|
7
|
+
|
|
8
|
+
export { CrdtField, cursorColorFromName, CURSOR_COLORS } from './crdt-field';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* CrdtManager manages active CrdtField instances and their sync channels.
|
|
12
|
+
*
|
|
13
|
+
* Collaborative state lives in two dedicated tables (defined in
|
|
14
|
+
* `apps/cli/src/meta_tables_remote.surql`):
|
|
15
|
+
* - `_00_crdt` { record_id, field, state } — one row per (record, field)
|
|
16
|
+
* - `_00_cursor` { record_id, session_id, field, state } — one row per
|
|
17
|
+
* (record, session, field)
|
|
18
|
+
*
|
|
19
|
+
* Splitting them off the parent row is what makes offline edits mergeable:
|
|
20
|
+
* each (record, field) gets its own row, so concurrent offline writes don't
|
|
21
|
+
* collide on the parent's last-write-wins semantics.
|
|
22
|
+
*
|
|
23
|
+
* Cross-browser delivery still rides the parent table's existing LIVE feed
|
|
24
|
+
* to avoid SurrealDB v3 LIVE bugs around dereference-based permission rules
|
|
25
|
+
* (issues 3602, 4026). On every meta UPSERT the writer also bumps the
|
|
26
|
+
* parent's `_00_rv` (a no-op assignment); that fires the parent's LIVE
|
|
27
|
+
* feed, and the receiver pulls the matching `_00_crdt` / `_00_cursor` rows
|
|
28
|
+
* via subquery. Permission inheritance happens server-side via
|
|
29
|
+
* `record_id.id != NONE` (SELECT) and `fn::can_update_record` (UPDATE).
|
|
30
|
+
*/
|
|
31
|
+
export class CrdtManager {
|
|
32
|
+
private fields = new Map<string, CrdtField>();
|
|
33
|
+
// One LIVE subscription per parent table (e.g. "thread" → uuid).
|
|
34
|
+
private liveByTable = new Map<string, Uuid>();
|
|
35
|
+
// Coalesces concurrent first-time subscribes for the same table.
|
|
36
|
+
private pendingLive = new Map<string, Promise<void>>();
|
|
37
|
+
private logger: Logger;
|
|
38
|
+
// SurrealDB session id, used as the per-session key inside `_00_cursor`.
|
|
39
|
+
private sessionId: string = '';
|
|
40
|
+
|
|
41
|
+
constructor(
|
|
42
|
+
private schema: SchemaStructure,
|
|
43
|
+
private local: LocalDatabaseService,
|
|
44
|
+
private remote: RemoteDatabaseService,
|
|
45
|
+
logger: Logger,
|
|
46
|
+
private debounceMs: number = 500,
|
|
47
|
+
) {
|
|
48
|
+
this.logger = logger.child({ service: 'CrdtManager' });
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Set the session id that scopes this client's cursor entries. Must be
|
|
52
|
+
* called before `open()` for cursors to be pushed under a stable key.
|
|
53
|
+
* Passed in from `sp00ky.ts` at boot (it already fetches `session::id()`
|
|
54
|
+
* for the data-module salt). */
|
|
55
|
+
setSessionId(sessionId: string): void {
|
|
56
|
+
this.sessionId = sessionId;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Open a CRDT field for collaborative editing.
|
|
61
|
+
*
|
|
62
|
+
* @param table - Table name
|
|
63
|
+
* @param recordId - Full record ID (e.g., "thread:abc")
|
|
64
|
+
* @param field - Field name (e.g., "title", "content")
|
|
65
|
+
* @param fallbackText - Current plain text from the record, used to seed the
|
|
66
|
+
* LoroDoc if no CRDT state exists yet (migration path)
|
|
67
|
+
*/
|
|
68
|
+
async open(
|
|
69
|
+
table: string,
|
|
70
|
+
recordId: string,
|
|
71
|
+
field: string,
|
|
72
|
+
fallbackText?: string,
|
|
73
|
+
): Promise<CrdtField> {
|
|
74
|
+
this.assertCrdtField(table, field);
|
|
75
|
+
const cursorsEnabled = this.fieldHasCursor(table, field);
|
|
76
|
+
const key = this.makeKey(table, recordId, field);
|
|
77
|
+
let crdtField = this.fields.get(key);
|
|
78
|
+
|
|
79
|
+
if (crdtField) {
|
|
80
|
+
return crdtField;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Read the snapshot directly off the parent row. `@crdt`-only fields
|
|
84
|
+
// hold the base64 snapshot inline; `@crdt @cursor` fields hold a
|
|
85
|
+
// `{ state, cursors }` object so we drill into `.state`. The query
|
|
86
|
+
// always selects the local row — sync-down already populated it
|
|
87
|
+
// (the snapshot is a column on the parent, not a sidecar table) so
|
|
88
|
+
// there is no separate fetch on the happy path.
|
|
89
|
+
let initialCrdtState: Uint8Array | undefined;
|
|
90
|
+
try {
|
|
91
|
+
const [result] = await this.local.query<[unknown]>(
|
|
92
|
+
`SELECT VALUE ${field} FROM ONLY $id`,
|
|
93
|
+
{ id: parseRecordIdString(recordId) },
|
|
94
|
+
);
|
|
95
|
+
const snapshot = this.extractSnapshot(result, cursorsEnabled);
|
|
96
|
+
if (snapshot) initialCrdtState = snapshot;
|
|
97
|
+
} catch (e) {
|
|
98
|
+
this.logger.info(
|
|
99
|
+
{ error: String(e), recordId, field, Category: 'sp00ky-client::CrdtManager::open' },
|
|
100
|
+
'No existing CRDT state found in local cache (continuing with empty doc)'
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
crdtField = new CrdtField(field, cursorsEnabled, initialCrdtState, this.logger);
|
|
105
|
+
crdtField.startSync(this.local, this.remote, recordId, this.sessionId, this.debounceMs);
|
|
106
|
+
this.fields.set(key, crdtField);
|
|
107
|
+
|
|
108
|
+
this.logger.info(
|
|
109
|
+
{ key, hasInitialState: !!initialCrdtState, hasFallback: !!fallbackText, Category: 'sp00ky-client::CrdtManager::open' },
|
|
110
|
+
'CrdtField opened'
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
// Fire-and-forget: the LIVE subscription receives *future* updates;
|
|
114
|
+
// the initial snapshot is already in hand. `ensureTableSubscription`
|
|
115
|
+
// coalesces concurrent calls via `pendingLive`, so this is safe.
|
|
116
|
+
void this.ensureTableSubscription(table);
|
|
117
|
+
|
|
118
|
+
// Local was empty — a fresh device, a memory-backed local DB after
|
|
119
|
+
// reload, or a record that hasn't been sync'd locally yet. Pull the
|
|
120
|
+
// parent row from remote and dispatch its CRDT field; otherwise the
|
|
121
|
+
// editor sits empty until the parent's LIVE feed happens to fire.
|
|
122
|
+
if (!initialCrdtState) {
|
|
123
|
+
void this.fetchAndDispatchRow(table, recordId);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return crdtField;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
close(table: string, recordId: string, field: string): void {
|
|
130
|
+
const key = this.makeKey(table, recordId, field);
|
|
131
|
+
const crdtField = this.fields.get(key);
|
|
132
|
+
if (crdtField) {
|
|
133
|
+
crdtField.stopSync();
|
|
134
|
+
this.fields.delete(key);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// If no fields on this table remain open, tear down the table-wide LIVE.
|
|
138
|
+
const tablePrefix = `${table}:`;
|
|
139
|
+
const stillOpen = Array.from(this.fields.keys()).some((k) => k.startsWith(tablePrefix));
|
|
140
|
+
if (!stillOpen) {
|
|
141
|
+
this.killTableSubscription(table);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
this.logger.debug(
|
|
145
|
+
{ key, Category: 'sp00ky-client::CrdtManager::close' },
|
|
146
|
+
'CrdtField closed'
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
closeAll(): void {
|
|
151
|
+
for (const [_, field] of this.fields) {
|
|
152
|
+
field.stopSync();
|
|
153
|
+
}
|
|
154
|
+
this.fields.clear();
|
|
155
|
+
for (const table of Array.from(this.liveByTable.keys())) {
|
|
156
|
+
this.killTableSubscription(table);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Ensure a single `LIVE SELECT * FROM <table>` is running, shared across
|
|
161
|
+
* every open CrdtField on `table`. */
|
|
162
|
+
private async ensureTableSubscription(table: string): Promise<void> {
|
|
163
|
+
if (this.liveByTable.has(table)) return;
|
|
164
|
+
|
|
165
|
+
const pending = this.pendingLive.get(table);
|
|
166
|
+
if (pending) return pending;
|
|
167
|
+
|
|
168
|
+
const start = (async () => {
|
|
169
|
+
try {
|
|
170
|
+
const [uuid] = await this.remote.query<[Uuid]>(
|
|
171
|
+
`LIVE SELECT * FROM ${table}`,
|
|
172
|
+
);
|
|
173
|
+
|
|
174
|
+
const subscription = await this.remote.getClient().liveOf(uuid);
|
|
175
|
+
subscription.subscribe((message) => {
|
|
176
|
+
if (message.action === 'KILLED') return;
|
|
177
|
+
if (message.action !== 'CREATE' && message.action !== 'UPDATE') return;
|
|
178
|
+
this.dispatchRow(table, message.value as Record<string, unknown>);
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
this.liveByTable.set(table, uuid);
|
|
182
|
+
this.logger.info(
|
|
183
|
+
{ table, Category: 'sp00ky-client::CrdtManager::ensureTableSubscription' },
|
|
184
|
+
'LIVE SELECT started'
|
|
185
|
+
);
|
|
186
|
+
} catch (e) {
|
|
187
|
+
this.logger.warn(
|
|
188
|
+
{ error: e, table, Category: 'sp00ky-client::CrdtManager::ensureTableSubscription' },
|
|
189
|
+
'Failed to start LIVE SELECT'
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
})();
|
|
193
|
+
|
|
194
|
+
this.pendingLive.set(table, start);
|
|
195
|
+
try {
|
|
196
|
+
await start;
|
|
197
|
+
} finally {
|
|
198
|
+
this.pendingLive.delete(table);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Apply a parent-row payload from a non-LIVE source (e.g. the
|
|
203
|
+
* list_ref-driven sync engine, when the cross-user LIVE on the
|
|
204
|
+
* parent table is filtered out by the SurrealDB cross-session
|
|
205
|
+
* permission gap). Same semantics as the internal `dispatchRow`. */
|
|
206
|
+
applyRow(table: string, row: Record<string, unknown>): void {
|
|
207
|
+
this.dispatchRow(table, row);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Dispatch a parent-row LIVE event to every open CrdtField on that
|
|
211
|
+
* record. Each open field reads its slice of the row directly — the
|
|
212
|
+
* CRDT snapshot is a column on the parent now, so there is no
|
|
213
|
+
* follow-up subquery. */
|
|
214
|
+
private dispatchRow(table: string, row: Record<string, unknown>): void {
|
|
215
|
+
const id = row.id != null ? String(row.id) : '';
|
|
216
|
+
if (!id) return;
|
|
217
|
+
|
|
218
|
+
const rowKeyPrefix = `${table}:${id}:`;
|
|
219
|
+
for (const [key, crdtField] of this.fields) {
|
|
220
|
+
if (!key.startsWith(rowKeyPrefix)) continue;
|
|
221
|
+
const fieldName = key.slice(rowKeyPrefix.length);
|
|
222
|
+
const cursorsEnabled = this.fieldHasCursor(table, fieldName);
|
|
223
|
+
const slice = row[fieldName];
|
|
224
|
+
const snapshot = this.extractSnapshot(slice, cursorsEnabled);
|
|
225
|
+
if (snapshot) crdtField.importRemote(snapshot);
|
|
226
|
+
|
|
227
|
+
if (cursorsEnabled && slice && typeof slice === 'object') {
|
|
228
|
+
const cursors = (slice as { cursors?: unknown }).cursors;
|
|
229
|
+
if (cursors && typeof cursors === 'object') {
|
|
230
|
+
for (const [sid, blob] of Object.entries(cursors as Record<string, unknown>)) {
|
|
231
|
+
if (sid === this.sessionId) continue;
|
|
232
|
+
if (typeof blob === 'string' && blob.length > 0) {
|
|
233
|
+
crdtField.importRemoteCursor(blob);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** One-shot remote fetch for a row whose CRDT field hasn't synced
|
|
242
|
+
* locally yet (fresh device, memory-backed local DB after reload, …).
|
|
243
|
+
* Used by `open()` when the local read came up empty. Subsequent
|
|
244
|
+
* cross-browser updates ride `dispatchRow` via the parent LIVE feed. */
|
|
245
|
+
private async fetchAndDispatchRow(table: string, id: string): Promise<void> {
|
|
246
|
+
try {
|
|
247
|
+
const recordId = parseRecordIdString(id);
|
|
248
|
+
const [row] = await this.remote.query<[Record<string, unknown> | null]>(
|
|
249
|
+
`SELECT * FROM ONLY $id`,
|
|
250
|
+
{ id: recordId },
|
|
251
|
+
);
|
|
252
|
+
if (!row || typeof row !== 'object') return;
|
|
253
|
+
this.dispatchRow(table, row as Record<string, unknown>);
|
|
254
|
+
} catch (e) {
|
|
255
|
+
this.logger.warn(
|
|
256
|
+
{ error: e, table, id, Category: 'sp00ky-client::CrdtManager::fetchAndDispatchRow' },
|
|
257
|
+
'Failed to fetch parent row for CRDT hydration'
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** Schema lookup: does `<table>.<field>` carry a `@cursor` annotation?
|
|
263
|
+
* Determines the on-disk shape (plain snapshot vs. `{ state, cursors }`). */
|
|
264
|
+
private fieldHasCursor(table: string, field: string): boolean {
|
|
265
|
+
const tableSchema = this.schema.tables.find((t) => t.name === table);
|
|
266
|
+
return !!tableSchema?.columns[field]?.cursor;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** Pull the LoroDoc snapshot bytes out of a row slice. For `@crdt`-only
|
|
270
|
+
* the slice IS the snapshot (Uint8Array); for `@crdt @cursor` it's
|
|
271
|
+
* `{ state, cursors }` where `state` carries the snapshot bytes. */
|
|
272
|
+
private extractSnapshot(value: unknown, cursorsEnabled: boolean): Uint8Array | undefined {
|
|
273
|
+
const asBytes = (v: unknown): Uint8Array | undefined => {
|
|
274
|
+
if (v instanceof Uint8Array) return v.length > 0 ? v : undefined;
|
|
275
|
+
// SurrealDB ferries bytes through several shapes depending on
|
|
276
|
+
// transport and on whether the field is a top-level `bytes` column
|
|
277
|
+
// or bytes nested inside a FLEXIBLE object. Round-tripping bytes
|
|
278
|
+
// through `option<object> FLEXIBLE` (the `@crdt @cursor` shape) in
|
|
279
|
+
// particular comes back as a plain `number[]` from the local WASM
|
|
280
|
+
// DB and as `Uint8Array` from the remote WS engine. Normalize all
|
|
281
|
+
// recognized variants here so the receiving CrdtField doesn't care.
|
|
282
|
+
if (v instanceof ArrayBuffer) return new Uint8Array(v);
|
|
283
|
+
if (ArrayBuffer.isView(v)) {
|
|
284
|
+
const view = v as ArrayBufferView;
|
|
285
|
+
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
|
|
286
|
+
}
|
|
287
|
+
if (Array.isArray(v) && v.length > 0 && v.every((n) => typeof n === 'number')) {
|
|
288
|
+
return Uint8Array.from(v as number[]);
|
|
289
|
+
}
|
|
290
|
+
return undefined;
|
|
291
|
+
};
|
|
292
|
+
|
|
293
|
+
if (cursorsEnabled) {
|
|
294
|
+
if (value && typeof value === 'object' && !(value instanceof Uint8Array)) {
|
|
295
|
+
return asBytes((value as { state?: unknown }).state);
|
|
296
|
+
}
|
|
297
|
+
return undefined;
|
|
298
|
+
}
|
|
299
|
+
return asBytes(value);
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
private killTableSubscription(table: string): void {
|
|
303
|
+
const uuid = this.liveByTable.get(table);
|
|
304
|
+
if (uuid) {
|
|
305
|
+
// We're tearing down: KILL on an already-dead/closed LIVE throws, and the
|
|
306
|
+
// subscription may have ended on its own, so the failure is expected and
|
|
307
|
+
// safe to drop.
|
|
308
|
+
this.remote.query('KILL $uuid', { uuid }).catch((err) => {
|
|
309
|
+
this.logger.debug(
|
|
310
|
+
{ err, table, Category: 'sp00ky-client::CrdtManager::killTableSubscription' },
|
|
311
|
+
'KILL of table LIVE failed (already closed?)'
|
|
312
|
+
);
|
|
313
|
+
});
|
|
314
|
+
this.liveByTable.delete(table);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
private makeKey(table: string, recordId: string, field: string): string {
|
|
319
|
+
return `${table}:${recordId}:${field}`;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Throws if `<table>.<field>` is not annotated `@crdt` in the schema. Catches
|
|
324
|
+
* typos, removed annotations, and stale schema codegen at the call site instead
|
|
325
|
+
* of silently producing a non-CRDT writer.
|
|
326
|
+
*/
|
|
327
|
+
private assertCrdtField(table: string, field: string): void {
|
|
328
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(field)) {
|
|
329
|
+
throw new Error(
|
|
330
|
+
`openCrdtField: refusing unsafe field identifier '${field}' — must match [a-zA-Z_][a-zA-Z0-9_]*`
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
const tableSchema = this.schema.tables.find((t) => t.name === table);
|
|
334
|
+
if (!tableSchema) {
|
|
335
|
+
throw new Error(
|
|
336
|
+
`openCrdtField: unknown table '${table}'. Available: ${this.schema.tables.map((t) => t.name).join(', ')}`
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
const column = tableSchema.columns[field];
|
|
340
|
+
if (!column) {
|
|
341
|
+
throw new Error(
|
|
342
|
+
`openCrdtField: '${table}.${field}' is not in the schema. Available fields: ${Object.keys(tableSchema.columns).join(', ')}`
|
|
343
|
+
);
|
|
344
|
+
}
|
|
345
|
+
if (!column.crdt) {
|
|
346
|
+
throw new Error(
|
|
347
|
+
`openCrdtField: '${table}.${field}' is not annotated '@crdt' in the schema. ` +
|
|
348
|
+
`Add '-- @crdt text' above the field's DEFINE FIELD and regenerate the client schema.`
|
|
349
|
+
);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { DataModule } from './index';
|
|
4
|
+
import type { QueryState, QueryStatus } from '../../types';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Tests for the per-query fetch status (idle/fetching) added to DataModule:
|
|
8
|
+
* setQueryStatus updates the query object and notifies both the DevTools
|
|
9
|
+
* observer hook and subscribeStatus listeners.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
function makeLogger(): any {
|
|
13
|
+
const noop = () => {};
|
|
14
|
+
const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
15
|
+
logger.child = () => logger;
|
|
16
|
+
return logger;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function makeDataModule(): DataModule<any> {
|
|
20
|
+
return new DataModule(
|
|
21
|
+
{} as any, // cache — unused by status methods
|
|
22
|
+
{} as any, // local — unused by status methods
|
|
23
|
+
{ tables: [] } as any, // schema — unused by status methods
|
|
24
|
+
makeLogger(),
|
|
25
|
+
100
|
|
26
|
+
);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function makeQueryState(hash: string): QueryState {
|
|
30
|
+
return {
|
|
31
|
+
config: {
|
|
32
|
+
id: new RecordId('_00_query', hash),
|
|
33
|
+
surql: 'SELECT * FROM user',
|
|
34
|
+
params: {},
|
|
35
|
+
localArray: [],
|
|
36
|
+
remoteArray: [],
|
|
37
|
+
ttl: '10m',
|
|
38
|
+
lastActiveAt: new Date(),
|
|
39
|
+
tableName: 'user',
|
|
40
|
+
},
|
|
41
|
+
records: [],
|
|
42
|
+
ttlTimer: null,
|
|
43
|
+
ttlDurationMs: 0,
|
|
44
|
+
updateCount: 0,
|
|
45
|
+
materializationSamples: [],
|
|
46
|
+
lastIngestLatencyMs: null,
|
|
47
|
+
errorCount: 0,
|
|
48
|
+
status: 'idle',
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
describe('DataModule query fetch status', () => {
|
|
53
|
+
let dm: DataModule<any>;
|
|
54
|
+
const hash = 'h1';
|
|
55
|
+
|
|
56
|
+
beforeEach(() => {
|
|
57
|
+
dm = makeDataModule();
|
|
58
|
+
(dm as any).activeQueries.set(hash, makeQueryState(hash));
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('subscribeStatus with immediate reports the current status', () => {
|
|
62
|
+
const seen: QueryStatus[] = [];
|
|
63
|
+
dm.subscribeStatus(hash, (s) => seen.push(s), { immediate: true });
|
|
64
|
+
expect(seen).toEqual(['idle']);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('setQueryStatus updates the query object and notifies subscribers + observer', () => {
|
|
68
|
+
const observed: Array<[string, QueryStatus]> = [];
|
|
69
|
+
dm.onQueryStatusChange = (h, s) => observed.push([h, s]);
|
|
70
|
+
|
|
71
|
+
const seen: QueryStatus[] = [];
|
|
72
|
+
dm.subscribeStatus(hash, (s) => seen.push(s));
|
|
73
|
+
|
|
74
|
+
dm.setQueryStatus(hash, 'fetching');
|
|
75
|
+
expect((dm as any).activeQueries.get(hash).status).toBe('fetching');
|
|
76
|
+
expect(seen).toEqual(['fetching']);
|
|
77
|
+
expect(observed).toEqual([[hash, 'fetching']]);
|
|
78
|
+
|
|
79
|
+
dm.setQueryStatus(hash, 'idle');
|
|
80
|
+
expect(seen).toEqual(['fetching', 'idle']);
|
|
81
|
+
expect(observed).toEqual([[hash, 'fetching'], [hash, 'idle']]);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('setQueryStatus is a no-op when the status is unchanged', () => {
|
|
85
|
+
const seen: QueryStatus[] = [];
|
|
86
|
+
dm.subscribeStatus(hash, (s) => seen.push(s));
|
|
87
|
+
dm.setQueryStatus(hash, 'idle'); // already idle
|
|
88
|
+
expect(seen).toEqual([]);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('setQueryStatus is a no-op for an unknown query', () => {
|
|
92
|
+
let called = false;
|
|
93
|
+
dm.onQueryStatusChange = () => {
|
|
94
|
+
called = true;
|
|
95
|
+
};
|
|
96
|
+
dm.setQueryStatus('does-not-exist', 'fetching');
|
|
97
|
+
expect(called).toBe(false);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('unsubscribe stops further status notifications', () => {
|
|
101
|
+
const seen: QueryStatus[] = [];
|
|
102
|
+
const unsub = dm.subscribeStatus(hash, (s) => seen.push(s));
|
|
103
|
+
dm.setQueryStatus(hash, 'fetching');
|
|
104
|
+
unsub();
|
|
105
|
+
dm.setQueryStatus(hash, 'idle');
|
|
106
|
+
expect(seen).toEqual(['fetching']);
|
|
107
|
+
});
|
|
108
|
+
});
|