@spooky-sync/core 0.0.1-canary.8 → 0.0.1-canary.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +854 -339
- package/dist/index.js +2633 -396
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +494 -0
- package/package.json +39 -9
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +41 -29
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +281 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +352 -0
- package/src/modules/data/data.status.test.ts +108 -0
- package/src/modules/data/index.ts +726 -108
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +130 -0
- package/src/modules/devtools/index.ts +89 -21
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.ts +166 -0
- package/src/modules/ref-tables.test.ts +56 -0
- package/src/modules/ref-tables.ts +57 -0
- package/src/modules/sync/engine.ts +97 -37
- package/src/modules/sync/events/index.ts +3 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +2 -2
- package/src/modules/sync/sync.ts +676 -58
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.test.ts +33 -0
- package/src/services/database/local.ts +192 -32
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +248 -37
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +65 -0
- package/src/sp00ky.ts +625 -0
- package/src/types.ts +140 -13
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +55 -1
- package/src/spooky.ts +0 -392
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { LocalDatabaseService } from '../../services/database/index';
|
|
2
|
-
import {
|
|
1
|
+
import type { LocalDatabaseService } from '../../services/database/index';
|
|
2
|
+
import type {
|
|
3
3
|
StreamProcessorService,
|
|
4
4
|
StreamUpdate,
|
|
5
5
|
StreamUpdateReceiver,
|
|
6
6
|
} from '../../services/stream-processor/index';
|
|
7
|
-
import { Logger } from '../../services/logger/index';
|
|
7
|
+
import type { Logger } from '../../services/logger/index';
|
|
8
8
|
import { parseRecordIdString, encodeRecordId, surql } from '../../utils/index';
|
|
9
|
-
import { CacheRecord, QueryConfig } from './types';
|
|
10
|
-
import { RecordVersionArray } from '../../types';
|
|
9
|
+
import type { CacheRecord, QueryConfig } from './types';
|
|
10
|
+
import type { RecordVersionArray } from '../../types';
|
|
11
11
|
|
|
12
12
|
export * from './types';
|
|
13
13
|
|
|
@@ -43,7 +43,7 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
43
43
|
{
|
|
44
44
|
queryHash: update.queryHash,
|
|
45
45
|
arrayLength: update.localArray?.length,
|
|
46
|
-
Category: '
|
|
46
|
+
Category: 'sp00ky-client::CacheModule::onStreamUpdate',
|
|
47
47
|
},
|
|
48
48
|
'Stream update received'
|
|
49
49
|
);
|
|
@@ -73,7 +73,7 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
73
73
|
this.logger.debug(
|
|
74
74
|
{
|
|
75
75
|
count: records.length,
|
|
76
|
-
Category: '
|
|
76
|
+
Category: 'sp00ky-client::CacheModule::saveBatch',
|
|
77
77
|
},
|
|
78
78
|
'Saving record batch'
|
|
79
79
|
);
|
|
@@ -85,7 +85,7 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
85
85
|
...record,
|
|
86
86
|
record: {
|
|
87
87
|
...record.record,
|
|
88
|
-
|
|
88
|
+
_00_rv: record.version,
|
|
89
89
|
},
|
|
90
90
|
};
|
|
91
91
|
});
|
|
@@ -94,7 +94,12 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
94
94
|
const query = surql.seal<void>(
|
|
95
95
|
surql.tx(
|
|
96
96
|
populatedRecords.map((_, i) => {
|
|
97
|
-
|
|
97
|
+
// MERGE, not REPLACE: the remote payload omits local-only
|
|
98
|
+
// fields (`_00_crdt`, `_00_cursor`) injected by the CLI's
|
|
99
|
+
// local schema, so REPLACE would wipe the persisted CRDT
|
|
100
|
+
// snapshot on every sync-down round-trip and break offline
|
|
101
|
+
// reload of formatted text.
|
|
102
|
+
return surql.upsertMerge(`id${i}`, `content${i}`);
|
|
98
103
|
})
|
|
99
104
|
)
|
|
100
105
|
);
|
|
@@ -114,20 +119,24 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
114
119
|
await this.local.execute(query, params);
|
|
115
120
|
}
|
|
116
121
|
|
|
117
|
-
// 2.
|
|
118
|
-
|
|
122
|
+
// 2. Bulk ingest into DBSP (use populatedRecords which has _00_rv set).
|
|
123
|
+
// ingestMany coalesces the per-record stream updates into a single
|
|
124
|
+
// notification per affected query — the UI then updates once, after the
|
|
125
|
+
// whole batch is ingested, instead of row-by-row.
|
|
126
|
+
const bulk = populatedRecords.map((record) => {
|
|
119
127
|
const recordId = encodeRecordId(record.record.id);
|
|
120
128
|
this.versionLookups[recordId] = record.version;
|
|
121
|
-
|
|
122
|
-
}
|
|
129
|
+
return { table: record.table, op: record.op, id: recordId, record: record.record };
|
|
130
|
+
});
|
|
131
|
+
this.streamProcessor.ingestMany(bulk);
|
|
123
132
|
|
|
124
133
|
this.logger.debug(
|
|
125
|
-
{ count: records.length, Category: '
|
|
134
|
+
{ count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
|
|
126
135
|
'Batch saved successfully'
|
|
127
136
|
);
|
|
128
137
|
} catch (err) {
|
|
129
138
|
this.logger.error(
|
|
130
|
-
{ err, count: records.length, Category: '
|
|
139
|
+
{ err, count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
|
|
131
140
|
'Failed to save batch'
|
|
132
141
|
);
|
|
133
142
|
throw err;
|
|
@@ -137,9 +146,9 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
137
146
|
/**
|
|
138
147
|
* Delete a record from local DB and ingest deletion into DBSP
|
|
139
148
|
*/
|
|
140
|
-
async delete(table: string, id: string, skipDbDelete: boolean = false): Promise<void> {
|
|
149
|
+
async delete(table: string, id: string, skipDbDelete: boolean = false, recordData: Record<string, any> = {}): Promise<void> {
|
|
141
150
|
this.logger.debug(
|
|
142
|
-
{ table, id, Category: '
|
|
151
|
+
{ table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
143
152
|
'Deleting record'
|
|
144
153
|
);
|
|
145
154
|
|
|
@@ -149,17 +158,17 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
149
158
|
await this.local.query('DELETE $id', { id: parseRecordIdString(id) });
|
|
150
159
|
}
|
|
151
160
|
|
|
152
|
-
// 2. Ingest deletion into DBSP
|
|
161
|
+
// 2. Ingest deletion into DBSP (pass record data so predicates can be matched)
|
|
153
162
|
delete this.versionLookups[id];
|
|
154
|
-
|
|
163
|
+
this.streamProcessor.ingest(table, 'DELETE', id, recordData);
|
|
155
164
|
|
|
156
165
|
this.logger.debug(
|
|
157
|
-
{ table, id, Category: '
|
|
166
|
+
{ table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
158
167
|
'Record deleted successfully'
|
|
159
168
|
);
|
|
160
169
|
} catch (err) {
|
|
161
170
|
this.logger.error(
|
|
162
|
-
{ err, table, id, Category: '
|
|
171
|
+
{ err, table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
163
172
|
'Failed to delete record'
|
|
164
173
|
);
|
|
165
174
|
throw err;
|
|
@@ -170,12 +179,15 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
170
179
|
* Register a query with DBSP to create a materialized view
|
|
171
180
|
* Returns the initial result array
|
|
172
181
|
*/
|
|
173
|
-
registerQuery(config: QueryConfig): {
|
|
182
|
+
registerQuery(config: QueryConfig): {
|
|
183
|
+
localArray: RecordVersionArray;
|
|
184
|
+
registrationTimings?: { parseMs: number; planMs: number; snapshotMs: number };
|
|
185
|
+
} {
|
|
174
186
|
this.logger.debug(
|
|
175
187
|
{
|
|
176
188
|
queryHash: config.queryHash,
|
|
177
189
|
surql: config.surql,
|
|
178
|
-
Category: '
|
|
190
|
+
Category: 'sp00ky-client::CacheModule::registerQuery',
|
|
179
191
|
},
|
|
180
192
|
'Registering query'
|
|
181
193
|
);
|
|
@@ -202,15 +214,15 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
202
214
|
{
|
|
203
215
|
queryHash: config.queryHash,
|
|
204
216
|
arrayLength: update.localArray?.length,
|
|
205
|
-
Category: '
|
|
217
|
+
Category: 'sp00ky-client::CacheModule::registerQuery',
|
|
206
218
|
},
|
|
207
219
|
'Query registered successfully'
|
|
208
220
|
);
|
|
209
221
|
|
|
210
|
-
return { localArray: update.localArray };
|
|
222
|
+
return { localArray: update.localArray, registrationTimings: update.registration };
|
|
211
223
|
} catch (err) {
|
|
212
224
|
this.logger.error(
|
|
213
|
-
{ err, queryHash: config.queryHash, Category: '
|
|
225
|
+
{ err, queryHash: config.queryHash, Category: 'sp00ky-client::CacheModule::registerQuery' },
|
|
214
226
|
'Failed to register query'
|
|
215
227
|
);
|
|
216
228
|
throw err;
|
|
@@ -222,18 +234,18 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
222
234
|
*/
|
|
223
235
|
unregisterQuery(queryHash: string): void {
|
|
224
236
|
this.logger.debug(
|
|
225
|
-
{ queryHash, Category: '
|
|
237
|
+
{ queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
|
|
226
238
|
'Unregistering query'
|
|
227
239
|
);
|
|
228
240
|
try {
|
|
229
241
|
this.streamProcessor.unregisterQueryPlan(queryHash);
|
|
230
242
|
this.logger.debug(
|
|
231
|
-
{ queryHash, Category: '
|
|
243
|
+
{ queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
|
|
232
244
|
'Query unregistered successfully'
|
|
233
245
|
);
|
|
234
246
|
} catch (err) {
|
|
235
247
|
this.logger.error(
|
|
236
|
-
{ err, queryHash, Category: '
|
|
248
|
+
{ err, queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
|
|
237
249
|
'Failed to unregister query'
|
|
238
250
|
);
|
|
239
251
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { RecordId, Duration } from 'surrealdb';
|
|
2
|
-
import { QueryTimeToLive
|
|
1
|
+
import type { RecordId, Duration } from 'surrealdb';
|
|
2
|
+
import type { QueryTimeToLive } from '../../types';
|
|
3
3
|
|
|
4
4
|
export type RecordWithId = Record<string, any> & { id: RecordId<string> };
|
|
5
5
|
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
import { LoroDoc } from 'loro-crdt';
|
|
2
|
+
import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
3
|
+
import type { Logger } from '../../services/logger/index';
|
|
4
|
+
import { parseRecordIdString } from '../../utils/index';
|
|
5
|
+
|
|
6
|
+
// ==================== CURSOR UTILITIES ====================
|
|
7
|
+
|
|
8
|
+
export const CURSOR_COLORS = [
|
|
9
|
+
'#3b82f6', '#ef4444', '#22c55e', '#f59e0b',
|
|
10
|
+
'#8b5cf6', '#ec4899', '#14b8a6', '#f97316',
|
|
11
|
+
];
|
|
12
|
+
|
|
13
|
+
export function cursorColorFromName(name: string): string {
|
|
14
|
+
let hash = 0;
|
|
15
|
+
for (let i = 0; i < name.length; i++) {
|
|
16
|
+
hash = ((hash << 5) - hash + name.charCodeAt(i)) | 0;
|
|
17
|
+
}
|
|
18
|
+
return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// ==================== CRDT FIELD ====================
|
|
22
|
+
|
|
23
|
+
export class CrdtField {
|
|
24
|
+
private doc: LoroDoc;
|
|
25
|
+
private pushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
26
|
+
private local: LocalDatabaseService | null = null;
|
|
27
|
+
private remote: RemoteDatabaseService | null = null;
|
|
28
|
+
private recordId: string | null = null;
|
|
29
|
+
private sessionId: string = '';
|
|
30
|
+
private unsubscribe: (() => void) | null = null;
|
|
31
|
+
private lastPushTime = 0;
|
|
32
|
+
private lastCursorPushTime = 0;
|
|
33
|
+
private loadedFromCrdt = false;
|
|
34
|
+
private pushRetryCount = 0;
|
|
35
|
+
private logger: Logger | null;
|
|
36
|
+
private cursorsEnabled: boolean;
|
|
37
|
+
/** Remote-push debounce. Local writes happen immediately on every Loro
|
|
38
|
+
* update; the remote UPSERT is coalesced over this window. Configured
|
|
39
|
+
* via `Sp00kyConfig.crdtDebounceMs`, default 500. */
|
|
40
|
+
private remoteDebounceMs: number = 500;
|
|
41
|
+
|
|
42
|
+
private _onCursorUpdate: ((data: Uint8Array) => void) | null = null;
|
|
43
|
+
private pendingCursorUpdate: Uint8Array | null = null;
|
|
44
|
+
|
|
45
|
+
/** Callback set by the editor to receive remote cursor updates.
|
|
46
|
+
* Any cursor data that arrived before this callback was set will be replayed. */
|
|
47
|
+
set onCursorUpdate(cb: ((data: Uint8Array) => void) | null) {
|
|
48
|
+
this._onCursorUpdate = cb;
|
|
49
|
+
if (cb && this.pendingCursorUpdate) {
|
|
50
|
+
try { cb(this.pendingCursorUpdate); } catch (e) {
|
|
51
|
+
this.logger?.warn(
|
|
52
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::onCursorUpdate' },
|
|
53
|
+
'Failed to replay pending cursor update'
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
this.pendingCursorUpdate = null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
get onCursorUpdate() { return this._onCursorUpdate; }
|
|
61
|
+
|
|
62
|
+
constructor(
|
|
63
|
+
private fieldName: string,
|
|
64
|
+
cursorsEnabled: boolean,
|
|
65
|
+
initialState?: Uint8Array,
|
|
66
|
+
logger?: Logger | null,
|
|
67
|
+
) {
|
|
68
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(fieldName)) {
|
|
69
|
+
throw new Error(
|
|
70
|
+
`CrdtField: refusing unsafe field identifier '${fieldName}' — must match [a-zA-Z_][a-zA-Z0-9_]*`
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
this.logger = logger ?? null;
|
|
74
|
+
this.cursorsEnabled = cursorsEnabled;
|
|
75
|
+
this.doc = new LoroDoc();
|
|
76
|
+
if (initialState && initialState.length > 0) {
|
|
77
|
+
// Tolerance: catch bad-snapshot data (corrupt blob, stale legacy
|
|
78
|
+
// value left over from a pre-`bytes` migration) so the editor still
|
|
79
|
+
// mounts. Without this guard the rejection bubbles through
|
|
80
|
+
// `useCrdtField` → permanent fallback `<p>`, with no cursor.
|
|
81
|
+
try {
|
|
82
|
+
this.doc.import(initialState);
|
|
83
|
+
this.loadedFromCrdt = true;
|
|
84
|
+
} catch (e) {
|
|
85
|
+
this.logger?.warn(
|
|
86
|
+
{
|
|
87
|
+
error: e,
|
|
88
|
+
fieldName,
|
|
89
|
+
Category: 'sp00ky-client::CrdtField::constructor',
|
|
90
|
+
},
|
|
91
|
+
'Initial CRDT state is not a valid LoroDoc snapshot — starting empty and will seed from fallback text'
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
getDoc(): LoroDoc { return this.doc; }
|
|
98
|
+
|
|
99
|
+
/** Whether the LoroDoc was loaded from saved CRDT state */
|
|
100
|
+
hasContent(): boolean {
|
|
101
|
+
return this.loadedFromCrdt;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
startSync(
|
|
105
|
+
local: LocalDatabaseService,
|
|
106
|
+
remote: RemoteDatabaseService,
|
|
107
|
+
recordId: string,
|
|
108
|
+
sessionId: string,
|
|
109
|
+
debounceMs: number,
|
|
110
|
+
): void {
|
|
111
|
+
this.local = local;
|
|
112
|
+
this.remote = remote;
|
|
113
|
+
this.recordId = recordId;
|
|
114
|
+
this.sessionId = sessionId;
|
|
115
|
+
this.remoteDebounceMs = debounceMs;
|
|
116
|
+
// Every local Loro update writes the snapshot to the local cache
|
|
117
|
+
// *immediately* (so reload/offline see the latest text), then
|
|
118
|
+
// schedules a debounced push to remote. The local UPSERT is cheap —
|
|
119
|
+
// it's an in-memory SurrealKV write — but errors are swallowed so a
|
|
120
|
+
// bad write never blocks user input.
|
|
121
|
+
this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
|
|
122
|
+
void this.persistLocal();
|
|
123
|
+
this.scheduleRemotePush();
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
stopSync(): void {
|
|
128
|
+
if (this.unsubscribe) { this.unsubscribe(); this.unsubscribe = null; }
|
|
129
|
+
if (this.pushTimer) { clearTimeout(this.pushTimer); this.pushTimer = null; }
|
|
130
|
+
if (this.remote && this.recordId) { void this.pushToRemote(); }
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
importRemote(state: Uint8Array): void {
|
|
134
|
+
// Echo suppression: skip imports within `remoteDebounceMs + 200` of
|
|
135
|
+
// our own push. The +200 guards against round-trip jitter where our
|
|
136
|
+
// own write echoes back from the LIVE feed before the debounce
|
|
137
|
+
// window closes.
|
|
138
|
+
if (Date.now() - this.lastPushTime < this.remoteDebounceMs + 200) return;
|
|
139
|
+
try {
|
|
140
|
+
this.doc.import(state);
|
|
141
|
+
// Persist the merged snapshot locally so the next reload/offline
|
|
142
|
+
// open sees the freshest converged state without waiting for the
|
|
143
|
+
// LIVE feed.
|
|
144
|
+
void this.persistLocal();
|
|
145
|
+
} catch (e) {
|
|
146
|
+
this.logger?.warn(
|
|
147
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::importRemote' },
|
|
148
|
+
'Failed to import remote CRDT state'
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
exportSnapshot(): Uint8Array {
|
|
154
|
+
return this.doc.export({ mode: 'snapshot' });
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Push this session's cursor blob into the parent row at
|
|
158
|
+
* `<field>.cursors[$sid]`. No-op when cursors aren't enabled on this
|
|
159
|
+
* field — the editor still calls this method optimistically, but
|
|
160
|
+
* without `@cursor` on the schema there's nowhere to store the blob.
|
|
161
|
+
* The UPDATE itself fires the parent table's LIVE feed, so other
|
|
162
|
+
* browsers receive the cursor change without a separate `_00_rv` bump. */
|
|
163
|
+
async pushCursorState(encoded: Uint8Array): Promise<void> {
|
|
164
|
+
if (!this.remote || !this.recordId) return;
|
|
165
|
+
if (!this.cursorsEnabled) return;
|
|
166
|
+
this.lastCursorPushTime = Date.now();
|
|
167
|
+
try {
|
|
168
|
+
const state = encodeBase64(encoded);
|
|
169
|
+
await this.remote.query(
|
|
170
|
+
`UPDATE $id SET ${this.fieldName}.cursors[$sid] = $state RETURN NONE;`,
|
|
171
|
+
{
|
|
172
|
+
id: parseRecordIdString(this.recordId),
|
|
173
|
+
sid: this.sessionId,
|
|
174
|
+
state,
|
|
175
|
+
}
|
|
176
|
+
);
|
|
177
|
+
} catch (e) {
|
|
178
|
+
this.logger?.warn(
|
|
179
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::pushCursorState' },
|
|
180
|
+
'Failed to push cursor state'
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
|
|
186
|
+
importRemoteCursor(base64State: string): void {
|
|
187
|
+
if (Date.now() - this.lastCursorPushTime < 300) return; // echo suppression
|
|
188
|
+
try {
|
|
189
|
+
const data = decodeBase64(base64State);
|
|
190
|
+
if (this._onCursorUpdate) {
|
|
191
|
+
this._onCursorUpdate(data);
|
|
192
|
+
} else {
|
|
193
|
+
// Only keep the latest cursor state — older positions are useless
|
|
194
|
+
this.pendingCursorUpdate = data;
|
|
195
|
+
}
|
|
196
|
+
} catch (e) {
|
|
197
|
+
this.logger?.warn(
|
|
198
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::importRemoteCursor' },
|
|
199
|
+
'Failed to apply remote cursor data'
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
private scheduleRemotePush(): void {
|
|
205
|
+
if (this.pushTimer) clearTimeout(this.pushTimer);
|
|
206
|
+
this.pushTimer = setTimeout(() => void this.pushToRemote(), this.remoteDebounceMs);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** SET path inside a parent row for the current snapshot. `@crdt`-only
|
|
210
|
+
* fields hold the snapshot directly (`<field>`); `@crdt @cursor`
|
|
211
|
+
* fields hold a `{ state, cursors }` object so the snapshot lives at
|
|
212
|
+
* `<field>.state` next to per-session cursor blobs. */
|
|
213
|
+
private statePath(): string {
|
|
214
|
+
return this.cursorsEnabled ? `${this.fieldName}.state` : this.fieldName;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Mirror the LoroDoc snapshot into the parent row locally. Runs on
|
|
218
|
+
* every local update and every remote import so reloads (online or
|
|
219
|
+
* offline) see the freshest content immediately. Failures are
|
|
220
|
+
* swallowed — a stale local write must never block user input. */
|
|
221
|
+
private async persistLocal(): Promise<void> {
|
|
222
|
+
if (!this.local || !this.recordId) return;
|
|
223
|
+
try {
|
|
224
|
+
await this.local.query(
|
|
225
|
+
`UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
|
|
226
|
+
{ id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
|
|
227
|
+
);
|
|
228
|
+
} catch (e) {
|
|
229
|
+
this.logger?.debug(
|
|
230
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::persistLocal' },
|
|
231
|
+
'Local CRDT persist failed (best-effort)'
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
private async pushToRemote(): Promise<void> {
|
|
237
|
+
if (!this.remote || !this.recordId) return;
|
|
238
|
+
this.lastPushTime = Date.now();
|
|
239
|
+
try {
|
|
240
|
+
// The UPDATE on the parent fires the parent table's LIVE feed,
|
|
241
|
+
// so cross-browser receivers see this change directly in the LIVE
|
|
242
|
+
// payload — no separate `_00_rv` bump or sidecar UPSERT.
|
|
243
|
+
await this.remote.query(
|
|
244
|
+
`UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
|
|
245
|
+
{ id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
|
|
246
|
+
);
|
|
247
|
+
this.pushRetryCount = 0;
|
|
248
|
+
} catch (e) {
|
|
249
|
+
this.logger?.warn(
|
|
250
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::pushToRemote' },
|
|
251
|
+
'Failed to push CRDT state to remote'
|
|
252
|
+
);
|
|
253
|
+
// Bounded retry. Offline first-loads will exhaust this and stop
|
|
254
|
+
// hammering; the next user keystroke (or the next time a remote
|
|
255
|
+
// event lands once we're back online) will kick off another push.
|
|
256
|
+
if (this.pushRetryCount < 2) {
|
|
257
|
+
this.pushRetryCount++;
|
|
258
|
+
this.scheduleRemotePush();
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
export function decodeBase64(b64: string): Uint8Array {
|
|
265
|
+
if (typeof atob === 'function') {
|
|
266
|
+
const binary = atob(b64);
|
|
267
|
+
const bytes = new Uint8Array(binary.length);
|
|
268
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
269
|
+
return bytes;
|
|
270
|
+
}
|
|
271
|
+
return new Uint8Array(Buffer.from(b64, 'base64'));
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
export function encodeBase64(bytes: Uint8Array): string {
|
|
275
|
+
if (typeof btoa === 'function') {
|
|
276
|
+
let binary = '';
|
|
277
|
+
for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
|
|
278
|
+
return btoa(binary);
|
|
279
|
+
}
|
|
280
|
+
return Buffer.from(bytes).toString('base64');
|
|
281
|
+
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { LoroDoc } from 'loro-crdt';
|
|
3
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
4
|
+
import { CrdtManager } from './index';
|
|
5
|
+
import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
6
|
+
import type { Logger } from '../../services/logger/index';
|
|
7
|
+
|
|
8
|
+
// Regression test for the offline-reload formatting-loss class of bug.
|
|
9
|
+
//
|
|
10
|
+
// Repro pattern: app uses `store: 'memory'` for the local SurrealDB (or
|
|
11
|
+
// any scenario where the local row exists without a CRDT snapshot —
|
|
12
|
+
// fresh device, post-sync-down with stale meta, …). On reload, the
|
|
13
|
+
// editor mounts but stays empty until some unrelated edit fires the
|
|
14
|
+
// parent LIVE feed. From the user's POV, formatting is gone.
|
|
15
|
+
//
|
|
16
|
+
// Fix: when `CrdtManager.open` finds no local snapshot, it must fetch
|
|
17
|
+
// the parent row from remote (the snapshot lives inline on the row in
|
|
18
|
+
// the consolidated design) and dispatch its CRDT field. This test pins
|
|
19
|
+
// that behavior across both shapes — `@crdt`-only fields where the
|
|
20
|
+
// snapshot IS the field value, and `@crdt @cursor` fields where the
|
|
21
|
+
// snapshot lives at `<field>.state`.
|
|
22
|
+
|
|
23
|
+
function silentLogger(): Logger {
|
|
24
|
+
const noop = () => {};
|
|
25
|
+
const fake: any = {};
|
|
26
|
+
fake.info = noop;
|
|
27
|
+
fake.warn = noop;
|
|
28
|
+
fake.debug = noop;
|
|
29
|
+
fake.error = noop;
|
|
30
|
+
fake.trace = noop;
|
|
31
|
+
fake.child = () => fake;
|
|
32
|
+
return fake as Logger;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function singleTableSchema(opts: { cursor?: boolean } = {}): SchemaStructure {
|
|
36
|
+
return {
|
|
37
|
+
tables: [
|
|
38
|
+
{
|
|
39
|
+
name: 'thread',
|
|
40
|
+
columns: {
|
|
41
|
+
body: {
|
|
42
|
+
type: 'string',
|
|
43
|
+
optional: false,
|
|
44
|
+
crdt: 'text',
|
|
45
|
+
...(opts.cursor ? { cursor: true } : {}),
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
primaryKey: ['id'],
|
|
49
|
+
},
|
|
50
|
+
],
|
|
51
|
+
relationships: [],
|
|
52
|
+
backends: {},
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function buildSnapshot(text: string): Uint8Array {
|
|
57
|
+
const doc = new LoroDoc();
|
|
58
|
+
doc.getText('text').insert(0, text);
|
|
59
|
+
return doc.export({ mode: 'snapshot' });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function makeRemote(handlers: {
|
|
63
|
+
selectRow?: () => unknown;
|
|
64
|
+
}): RemoteDatabaseService {
|
|
65
|
+
const remote = {
|
|
66
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
67
|
+
if (sql.includes('LIVE SELECT')) return ['live-uuid'];
|
|
68
|
+
if (sql.includes('SELECT * FROM ONLY')) {
|
|
69
|
+
return [handlers.selectRow ? handlers.selectRow() : null];
|
|
70
|
+
}
|
|
71
|
+
return [];
|
|
72
|
+
}),
|
|
73
|
+
getClient: () =>
|
|
74
|
+
({
|
|
75
|
+
liveOf: async () => ({
|
|
76
|
+
subscribe: () => () => {},
|
|
77
|
+
}),
|
|
78
|
+
}) as any,
|
|
79
|
+
} satisfies Partial<RemoteDatabaseService> as unknown as RemoteDatabaseService;
|
|
80
|
+
return remote;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
describe('CrdtManager.open hydration', () => {
|
|
84
|
+
it('@crdt-only: fetches the parent row from remote when local is empty', async () => {
|
|
85
|
+
const remoteSnapshot = buildSnapshot('hello world');
|
|
86
|
+
|
|
87
|
+
const local = {
|
|
88
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
89
|
+
if (sql.includes('SELECT VALUE body FROM ONLY')) return [null];
|
|
90
|
+
return [];
|
|
91
|
+
}),
|
|
92
|
+
} satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
|
|
93
|
+
|
|
94
|
+
const remote = makeRemote({
|
|
95
|
+
selectRow: () => ({ id: 'thread:abc', body: remoteSnapshot }),
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
const manager = new CrdtManager(singleTableSchema(), local, remote, silentLogger());
|
|
99
|
+
manager.setSessionId('session-under-test');
|
|
100
|
+
|
|
101
|
+
const field = await manager.open('thread', 'thread:abc', 'body');
|
|
102
|
+
|
|
103
|
+
await vi.waitFor(() => {
|
|
104
|
+
expect(field.getDoc().getText('text').toString()).toBe('hello world');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
const sqls = (remote.query as ReturnType<typeof vi.fn>).mock.calls.map(
|
|
108
|
+
(c) => c[0] as string
|
|
109
|
+
);
|
|
110
|
+
expect(sqls.some((s) => s.includes('SELECT * FROM ONLY'))).toBe(true);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it('@crdt @cursor: extracts the snapshot from the object shape', async () => {
|
|
114
|
+
const remoteSnapshot = buildSnapshot('object shape');
|
|
115
|
+
|
|
116
|
+
const local = {
|
|
117
|
+
query: vi.fn().mockImplementation(async () => [null]),
|
|
118
|
+
} satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
|
|
119
|
+
|
|
120
|
+
// Cursor-enabled field stores `{ state, cursors }`. The remote row
|
|
121
|
+
// returns this shape; the manager must drill into `.state` for the
|
|
122
|
+
// snapshot bytes.
|
|
123
|
+
const remote = makeRemote({
|
|
124
|
+
selectRow: () => ({
|
|
125
|
+
id: 'thread:abc',
|
|
126
|
+
body: { state: remoteSnapshot, cursors: {} },
|
|
127
|
+
}),
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
const manager = new CrdtManager(
|
|
131
|
+
singleTableSchema({ cursor: true }),
|
|
132
|
+
local,
|
|
133
|
+
remote,
|
|
134
|
+
silentLogger()
|
|
135
|
+
);
|
|
136
|
+
manager.setSessionId('session-under-test');
|
|
137
|
+
|
|
138
|
+
const field = await manager.open('thread', 'thread:abc', 'body');
|
|
139
|
+
|
|
140
|
+
await vi.waitFor(() => {
|
|
141
|
+
expect(field.getDoc().getText('text').toString()).toBe('object shape');
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it('skips the remote fetch when a local snapshot already exists', async () => {
|
|
146
|
+
const localSnapshot = buildSnapshot('cached');
|
|
147
|
+
|
|
148
|
+
const local = {
|
|
149
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
150
|
+
if (sql.includes('SELECT VALUE body FROM ONLY')) return [localSnapshot];
|
|
151
|
+
return [];
|
|
152
|
+
}),
|
|
153
|
+
} satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
|
|
154
|
+
|
|
155
|
+
const remote = makeRemote({});
|
|
156
|
+
|
|
157
|
+
const manager = new CrdtManager(singleTableSchema(), local, remote, silentLogger());
|
|
158
|
+
manager.setSessionId('session-under-test');
|
|
159
|
+
|
|
160
|
+
const field = await manager.open('thread', 'thread:def', 'body');
|
|
161
|
+
|
|
162
|
+
expect(field.getDoc().getText('text').toString()).toBe('cached');
|
|
163
|
+
|
|
164
|
+
// Give any stray async work a tick to land before asserting absence.
|
|
165
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
166
|
+
|
|
167
|
+
const sqls = (remote.query as ReturnType<typeof vi.fn>).mock.calls.map(
|
|
168
|
+
(c) => c[0] as string
|
|
169
|
+
);
|
|
170
|
+
expect(sqls.some((s) => s.includes('SELECT * FROM ONLY'))).toBe(false);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
// Regression: SurrealDB returns bytes nested inside a FLEXIBLE object as a
|
|
174
|
+
// plain `number[]` from the local WASM engine (e.g. `state: [108, 111, 114,
|
|
175
|
+
// 111, ...]` — the literal LoroDoc magic header bytes). The cursor-enabled
|
|
176
|
+
// extractSnapshot path used to only accept Uint8Array and would silently
|
|
177
|
+
// return undefined for the array form, leaving every editor empty and
|
|
178
|
+
// killing realtime updates for `@crdt @cursor` fields.
|
|
179
|
+
it('@crdt @cursor: accepts number[] as the state shape (local WASM transport)', async () => {
|
|
180
|
+
const remoteSnapshot = buildSnapshot('from local wasm');
|
|
181
|
+
const remoteAsArray = Array.from(remoteSnapshot) as unknown as number[];
|
|
182
|
+
|
|
183
|
+
const local = {
|
|
184
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
185
|
+
if (sql.includes('SELECT VALUE body FROM ONLY')) {
|
|
186
|
+
return [{ state: remoteAsArray, cursors: {} }];
|
|
187
|
+
}
|
|
188
|
+
return [];
|
|
189
|
+
}),
|
|
190
|
+
} satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
|
|
191
|
+
|
|
192
|
+
const remote = makeRemote({});
|
|
193
|
+
|
|
194
|
+
const manager = new CrdtManager(
|
|
195
|
+
singleTableSchema({ cursor: true }),
|
|
196
|
+
local,
|
|
197
|
+
remote,
|
|
198
|
+
silentLogger()
|
|
199
|
+
);
|
|
200
|
+
manager.setSessionId('session-under-test');
|
|
201
|
+
|
|
202
|
+
const field = await manager.open('thread', 'thread:wasm', 'body');
|
|
203
|
+
|
|
204
|
+
expect(field.getDoc().getText('text').toString()).toBe('from local wasm');
|
|
205
|
+
});
|
|
206
|
+
});
|