@spooky-sync/core 0.0.1-canary.16 → 0.0.1-canary.160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +1406 -364
- package/dist/index.js +7754 -1390
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/sqlite-open.js +276 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +420 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +434 -0
- package/dist/types.d.ts +874 -0
- package/package.json +20 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +16 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +112 -32
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +361 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/index.ts +1151 -129
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +154 -0
- package/src/modules/devtools/index.ts +325 -37
- package/src/modules/devtools/notify-throttle.test.ts +149 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +143 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.test.ts +120 -0
- package/src/modules/feature-flag/index.ts +209 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.ts +12 -5
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +137 -0
- package/src/modules/sync/queue/queue-up.ts +198 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.ts +77 -9
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.ts +1333 -92
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/cache-engine.ts +160 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +6 -0
- package/src/services/database/local-migrator.ts +28 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +478 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +13 -13
- package/src/services/database/sqlite-cache-engine.test.ts +464 -0
- package/src/services/database/sqlite-cache-engine.ts +1154 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +106 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +113 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +219 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +291 -0
- package/src/services/database/surreal-cache-engine.ts +141 -0
- package/src/services/logger/index.ts +6 -109
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +444 -52
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +216 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +23 -2
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +278 -0
- package/src/services/tabs/coordinator.test.ts +192 -0
- package/src/services/tabs/coordinator.ts +567 -0
- package/src/services/tabs/fake-ports.fixture.ts +70 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +239 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +581 -0
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.ts +1225 -0
- package/src/types.ts +342 -15
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import type { LocalStore } from '../../services/database/index';
|
|
2
|
+
import { StaleEpochError } from '../../services/database/index';
|
|
3
|
+
import type {
|
|
3
4
|
StreamProcessorService,
|
|
4
5
|
StreamUpdate,
|
|
5
6
|
StreamUpdateReceiver,
|
|
6
7
|
} from '../../services/stream-processor/index';
|
|
7
|
-
import { Logger } from '../../services/logger/index';
|
|
8
|
+
import type { Logger } from '../../services/logger/index';
|
|
8
9
|
import { parseRecordIdString, encodeRecordId, surql } from '../../utils/index';
|
|
9
|
-
import { CacheRecord, QueryConfig } from './types';
|
|
10
|
-
import { RecordVersionArray } from '../../types';
|
|
10
|
+
import type { CacheRecord, QueryConfig } from './types';
|
|
11
|
+
import type { RecordVersionArray } from '../../types';
|
|
11
12
|
|
|
12
13
|
export * from './types';
|
|
13
14
|
|
|
@@ -17,13 +18,26 @@ export * from './types';
|
|
|
17
18
|
* Single responsibility: Handle all local storage operations and DBSP ingestion.
|
|
18
19
|
* This module acts as the bridge between data operations and persistence.
|
|
19
20
|
*/
|
|
21
|
+
/** One ingested change, in exactly the shape `ingestMany` consumes. Shared
|
|
22
|
+
* with the tabs protocol so a leader can relay its ingests to followers. */
|
|
23
|
+
export interface CacheIngestTuple {
|
|
24
|
+
table: string;
|
|
25
|
+
op: 'CREATE' | 'UPDATE' | 'DELETE';
|
|
26
|
+
id: string;
|
|
27
|
+
record: Record<string, unknown>;
|
|
28
|
+
}
|
|
29
|
+
|
|
20
30
|
export class CacheModule implements StreamUpdateReceiver {
|
|
21
31
|
private logger: Logger;
|
|
22
32
|
private streamUpdateCallback: (update: StreamUpdate) => void;
|
|
23
33
|
private versionLookups: Record<string, number> = {};
|
|
34
|
+
/** Shared-tabs leader: fan every committed ingest out to follower circuits.
|
|
35
|
+
* Fired AFTER the local tx (the rows are already in the shared store, so a
|
|
36
|
+
* follower only needs the circuit feed). Never set on followers. */
|
|
37
|
+
private ingestRelay: ((tuples: CacheIngestTuple[]) => void) | null = null;
|
|
24
38
|
|
|
25
39
|
constructor(
|
|
26
|
-
private local:
|
|
40
|
+
private local: LocalStore,
|
|
27
41
|
private streamProcessor: StreamProcessorService,
|
|
28
42
|
streamUpdateCallback: (update: StreamUpdate) => void,
|
|
29
43
|
logger: Logger
|
|
@@ -43,17 +57,43 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
43
57
|
{
|
|
44
58
|
queryHash: update.queryHash,
|
|
45
59
|
arrayLength: update.localArray?.length,
|
|
46
|
-
Category: '
|
|
60
|
+
Category: 'sp00ky-client::CacheModule::onStreamUpdate',
|
|
47
61
|
},
|
|
48
62
|
'Stream update received'
|
|
49
63
|
);
|
|
50
64
|
this.streamUpdateCallback(update);
|
|
51
65
|
}
|
|
52
66
|
|
|
67
|
+
setIngestRelay(cb: ((tuples: CacheIngestTuple[]) => void) | null): void {
|
|
68
|
+
this.ingestRelay = cb;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Shared-tabs follower: feed relayed tuples into THIS tab's circuit only.
|
|
73
|
+
* The rows are already in the shared store (the leader wrote them), so no
|
|
74
|
+
* local write happens here; the normal chain then runs: SSP -> stream update
|
|
75
|
+
* -> DataModule debounce -> materializeRecords (re-reads via the port
|
|
76
|
+
* transport) -> this tab's subscriptions fire with this tab's hashes.
|
|
77
|
+
*/
|
|
78
|
+
applyRelayedIngest(tuples: CacheIngestTuple[]): void {
|
|
79
|
+
for (const t of tuples) {
|
|
80
|
+
const rv = (t.record as { _00_rv?: number } | undefined)?._00_rv;
|
|
81
|
+
if (t.op === 'DELETE') delete this.versionLookups[t.id];
|
|
82
|
+
else if (typeof rv === 'number') this.versionLookups[t.id] = rv;
|
|
83
|
+
}
|
|
84
|
+
this.streamProcessor.ingestMany(tuples as Parameters<StreamProcessorService['ingestMany']>[0]);
|
|
85
|
+
}
|
|
86
|
+
|
|
53
87
|
public lookup(recordId: string): number {
|
|
54
88
|
return this.versionLookups[recordId] ?? 0;
|
|
55
89
|
}
|
|
56
90
|
|
|
91
|
+
/** Drop the version cache on a bucket switch — a stale version would make
|
|
92
|
+
* the sync diff skip fetching a body the new bucket legitimately needs. */
|
|
93
|
+
public clearVersionLookups(): void {
|
|
94
|
+
this.versionLookups = {};
|
|
95
|
+
}
|
|
96
|
+
|
|
57
97
|
/**
|
|
58
98
|
* Save a single record to local DB and ingest into DBSP
|
|
59
99
|
* Used by mutations (create/update)
|
|
@@ -70,10 +110,17 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
70
110
|
async saveBatch(records: CacheRecord[], skipDbInsert: boolean = false): Promise<void> {
|
|
71
111
|
if (records.length === 0) return;
|
|
72
112
|
|
|
113
|
+
// Fence against bucket switches: this batch's records were derived from
|
|
114
|
+
// reads against the CURRENT store/user. If a switch lands while we await
|
|
115
|
+
// the (gated) local write, the write throws StaleEpochError and the whole
|
|
116
|
+
// batch — including the SSP ingest — is dropped: the new bucket re-syncs
|
|
117
|
+
// its own data from the server.
|
|
118
|
+
const epoch = this.local.epoch;
|
|
119
|
+
|
|
73
120
|
this.logger.debug(
|
|
74
121
|
{
|
|
75
122
|
count: records.length,
|
|
76
|
-
Category: '
|
|
123
|
+
Category: 'sp00ky-client::CacheModule::saveBatch',
|
|
77
124
|
},
|
|
78
125
|
'Saving record batch'
|
|
79
126
|
);
|
|
@@ -85,7 +132,7 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
85
132
|
...record,
|
|
86
133
|
record: {
|
|
87
134
|
...record.record,
|
|
88
|
-
|
|
135
|
+
_00_rv: record.version,
|
|
89
136
|
},
|
|
90
137
|
};
|
|
91
138
|
});
|
|
@@ -94,7 +141,12 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
94
141
|
const query = surql.seal<void>(
|
|
95
142
|
surql.tx(
|
|
96
143
|
populatedRecords.map((_, i) => {
|
|
97
|
-
|
|
144
|
+
// MERGE, not REPLACE: the remote payload omits local-only
|
|
145
|
+
// fields (`_00_crdt`, `_00_cursor`) injected by the CLI's
|
|
146
|
+
// local schema, so REPLACE would wipe the persisted CRDT
|
|
147
|
+
// snapshot on every sync-down round-trip and break offline
|
|
148
|
+
// reload of formatted text.
|
|
149
|
+
return surql.upsertMerge(`id${i}`, `content${i}`);
|
|
98
150
|
})
|
|
99
151
|
)
|
|
100
152
|
);
|
|
@@ -111,23 +163,38 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
111
163
|
{} as Record<string, any>
|
|
112
164
|
);
|
|
113
165
|
|
|
114
|
-
await this.local.execute(query, params);
|
|
166
|
+
await this.local.execute(query, params, { epoch });
|
|
115
167
|
}
|
|
116
168
|
|
|
117
|
-
//
|
|
118
|
-
|
|
169
|
+
// Late fence for the skipDbInsert path (no gated write above to trip on).
|
|
170
|
+
if (this.local.epoch !== epoch) throw new StaleEpochError();
|
|
171
|
+
|
|
172
|
+
// 2. Bulk ingest into DBSP (use populatedRecords which has _00_rv set).
|
|
173
|
+
// ingestMany coalesces the per-record stream updates into a single
|
|
174
|
+
// notification per affected query — the UI then updates once, after the
|
|
175
|
+
// whole batch is ingested, instead of row-by-row.
|
|
176
|
+
const bulk = populatedRecords.map((record) => {
|
|
119
177
|
const recordId = encodeRecordId(record.record.id);
|
|
120
178
|
this.versionLookups[recordId] = record.version;
|
|
121
|
-
|
|
122
|
-
}
|
|
179
|
+
return { table: record.table, op: record.op, id: recordId, record: record.record };
|
|
180
|
+
});
|
|
181
|
+
this.streamProcessor.ingestMany(bulk);
|
|
182
|
+
this.ingestRelay?.(bulk as CacheIngestTuple[]);
|
|
123
183
|
|
|
124
184
|
this.logger.debug(
|
|
125
|
-
{ count: records.length, Category: '
|
|
185
|
+
{ count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
|
|
126
186
|
'Batch saved successfully'
|
|
127
187
|
);
|
|
128
188
|
} catch (err) {
|
|
189
|
+
if (err instanceof StaleEpochError) {
|
|
190
|
+
this.logger.debug(
|
|
191
|
+
{ count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
|
|
192
|
+
'Dropped batch from before a bucket switch'
|
|
193
|
+
);
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
129
196
|
this.logger.error(
|
|
130
|
-
{ err, count: records.length, Category: '
|
|
197
|
+
{ err, count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
|
|
131
198
|
'Failed to save batch'
|
|
132
199
|
);
|
|
133
200
|
throw err;
|
|
@@ -137,29 +204,39 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
137
204
|
/**
|
|
138
205
|
* Delete a record from local DB and ingest deletion into DBSP
|
|
139
206
|
*/
|
|
140
|
-
async delete(table: string, id: string, skipDbDelete: boolean = false): Promise<void> {
|
|
207
|
+
async delete(table: string, id: string, skipDbDelete: boolean = false, recordData: Record<string, any> = {}): Promise<void> {
|
|
141
208
|
this.logger.debug(
|
|
142
|
-
{ table, id, Category: '
|
|
209
|
+
{ table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
143
210
|
'Deleting record'
|
|
144
211
|
);
|
|
145
212
|
|
|
213
|
+
const epoch = this.local.epoch;
|
|
146
214
|
try {
|
|
147
215
|
// 1. Delete from local database
|
|
148
216
|
if (!skipDbDelete) {
|
|
149
|
-
await this.local.query('DELETE $id', { id: parseRecordIdString(id) });
|
|
217
|
+
await this.local.query('DELETE $id', { id: parseRecordIdString(id) }, { epoch });
|
|
150
218
|
}
|
|
219
|
+
if (this.local.epoch !== epoch) throw new StaleEpochError();
|
|
151
220
|
|
|
152
|
-
// 2. Ingest deletion into DBSP
|
|
221
|
+
// 2. Ingest deletion into DBSP (pass record data so predicates can be matched)
|
|
153
222
|
delete this.versionLookups[id];
|
|
154
|
-
|
|
223
|
+
this.streamProcessor.ingest(table, 'DELETE', id, recordData);
|
|
224
|
+
this.ingestRelay?.([{ table, op: 'DELETE', id, record: recordData }]);
|
|
155
225
|
|
|
156
226
|
this.logger.debug(
|
|
157
|
-
{ table, id, Category: '
|
|
227
|
+
{ table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
158
228
|
'Record deleted successfully'
|
|
159
229
|
);
|
|
160
230
|
} catch (err) {
|
|
231
|
+
if (err instanceof StaleEpochError) {
|
|
232
|
+
this.logger.debug(
|
|
233
|
+
{ table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
234
|
+
'Dropped delete from before a bucket switch'
|
|
235
|
+
);
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
161
238
|
this.logger.error(
|
|
162
|
-
{ err, table, id, Category: '
|
|
239
|
+
{ err, table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
163
240
|
'Failed to delete record'
|
|
164
241
|
);
|
|
165
242
|
throw err;
|
|
@@ -170,12 +247,15 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
170
247
|
* Register a query with DBSP to create a materialized view
|
|
171
248
|
* Returns the initial result array
|
|
172
249
|
*/
|
|
173
|
-
registerQuery(config: QueryConfig): {
|
|
250
|
+
registerQuery(config: QueryConfig): {
|
|
251
|
+
localArray: RecordVersionArray;
|
|
252
|
+
registrationTimings?: { parseMs: number; planMs: number; snapshotMs: number };
|
|
253
|
+
} {
|
|
174
254
|
this.logger.debug(
|
|
175
255
|
{
|
|
176
256
|
queryHash: config.queryHash,
|
|
177
257
|
surql: config.surql,
|
|
178
|
-
Category: '
|
|
258
|
+
Category: 'sp00ky-client::CacheModule::registerQuery',
|
|
179
259
|
},
|
|
180
260
|
'Registering query'
|
|
181
261
|
);
|
|
@@ -202,15 +282,15 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
202
282
|
{
|
|
203
283
|
queryHash: config.queryHash,
|
|
204
284
|
arrayLength: update.localArray?.length,
|
|
205
|
-
Category: '
|
|
285
|
+
Category: 'sp00ky-client::CacheModule::registerQuery',
|
|
206
286
|
},
|
|
207
287
|
'Query registered successfully'
|
|
208
288
|
);
|
|
209
289
|
|
|
210
|
-
return { localArray: update.localArray };
|
|
290
|
+
return { localArray: update.localArray, registrationTimings: update.registration };
|
|
211
291
|
} catch (err) {
|
|
212
292
|
this.logger.error(
|
|
213
|
-
{ err, queryHash: config.queryHash, Category: '
|
|
293
|
+
{ err, queryHash: config.queryHash, Category: 'sp00ky-client::CacheModule::registerQuery' },
|
|
214
294
|
'Failed to register query'
|
|
215
295
|
);
|
|
216
296
|
throw err;
|
|
@@ -222,18 +302,18 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
222
302
|
*/
|
|
223
303
|
unregisterQuery(queryHash: string): void {
|
|
224
304
|
this.logger.debug(
|
|
225
|
-
{ queryHash, Category: '
|
|
305
|
+
{ queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
|
|
226
306
|
'Unregistering query'
|
|
227
307
|
);
|
|
228
308
|
try {
|
|
229
309
|
this.streamProcessor.unregisterQueryPlan(queryHash);
|
|
230
310
|
this.logger.debug(
|
|
231
|
-
{ queryHash, Category: '
|
|
311
|
+
{ queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
|
|
232
312
|
'Query unregistered successfully'
|
|
233
313
|
);
|
|
234
314
|
} catch (err) {
|
|
235
315
|
this.logger.error(
|
|
236
|
-
{ err, queryHash, Category: '
|
|
316
|
+
{ err, queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
|
|
237
317
|
'Failed to unregister query'
|
|
238
318
|
);
|
|
239
319
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { RecordId, Duration } from 'surrealdb';
|
|
2
|
-
import { QueryTimeToLive
|
|
1
|
+
import type { RecordId, Duration } from 'surrealdb';
|
|
2
|
+
import type { QueryTimeToLive } from '../../types';
|
|
3
3
|
|
|
4
4
|
export type RecordWithId = Record<string, any> & { id: RecordId<string> };
|
|
5
5
|
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
import type { LoroDoc } from 'loro-crdt';
|
|
2
|
+
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
3
|
+
import type { Logger } from '../../services/logger/index';
|
|
4
|
+
import { parseRecordIdString } from '../../utils/index';
|
|
5
|
+
|
|
6
|
+
// ==================== CURSOR UTILITIES ====================
|
|
7
|
+
|
|
8
|
+
export const CURSOR_COLORS = [
|
|
9
|
+
'#3b82f6', '#ef4444', '#22c55e', '#f59e0b',
|
|
10
|
+
'#8b5cf6', '#ec4899', '#14b8a6', '#f97316',
|
|
11
|
+
];
|
|
12
|
+
|
|
13
|
+
export function cursorColorFromName(name: string): string {
|
|
14
|
+
let hash = 0;
|
|
15
|
+
for (let i = 0; i < name.length; i++) {
|
|
16
|
+
hash = ((hash << 5) - hash + name.charCodeAt(i)) | 0;
|
|
17
|
+
}
|
|
18
|
+
return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// ==================== CRDT FIELD ====================
|
|
22
|
+
|
|
23
|
+
export class CrdtField {
|
|
24
|
+
private doc: LoroDoc;
|
|
25
|
+
private pushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
26
|
+
private local: LocalStore | null = null;
|
|
27
|
+
private remote: RemoteDatabaseService | null = null;
|
|
28
|
+
private recordId: string | null = null;
|
|
29
|
+
private sessionId: string = '';
|
|
30
|
+
private unsubscribe: (() => void) | null = null;
|
|
31
|
+
private lastPushTime = 0;
|
|
32
|
+
private lastCursorPushTime = 0;
|
|
33
|
+
private loadedFromCrdt = false;
|
|
34
|
+
private pushRetryCount = 0;
|
|
35
|
+
private logger: Logger | null;
|
|
36
|
+
private cursorsEnabled: boolean;
|
|
37
|
+
/** Remote-push debounce. Local writes happen immediately on every Loro
|
|
38
|
+
* update; the remote UPSERT is coalesced over this window. Configured
|
|
39
|
+
* via `Sp00kyConfig.crdtDebounceMs`, default 500. */
|
|
40
|
+
private remoteDebounceMs: number = 500;
|
|
41
|
+
|
|
42
|
+
private _onCursorUpdate: ((data: Uint8Array) => void) | null = null;
|
|
43
|
+
private pendingCursorUpdate: Uint8Array | null = null;
|
|
44
|
+
|
|
45
|
+
/** Callback set by the editor to receive remote cursor updates.
|
|
46
|
+
* Any cursor data that arrived before this callback was set will be replayed. */
|
|
47
|
+
set onCursorUpdate(cb: ((data: Uint8Array) => void) | null) {
|
|
48
|
+
this._onCursorUpdate = cb;
|
|
49
|
+
if (cb && this.pendingCursorUpdate) {
|
|
50
|
+
try { cb(this.pendingCursorUpdate); } catch (e) {
|
|
51
|
+
this.logger?.warn(
|
|
52
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::onCursorUpdate' },
|
|
53
|
+
'Failed to replay pending cursor update'
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
this.pendingCursorUpdate = null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
get onCursorUpdate() { return this._onCursorUpdate; }
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* @param LoroDocClass the `LoroDoc` constructor, injected by the caller after
|
|
64
|
+
* awaiting {@link loadLoro} — keeps `loro-crdt` out of this module's static
|
|
65
|
+
* import graph so it only ships to apps that use CRDT fields.
|
|
66
|
+
*/
|
|
67
|
+
constructor(
|
|
68
|
+
private fieldName: string,
|
|
69
|
+
cursorsEnabled: boolean,
|
|
70
|
+
LoroDocClass: typeof LoroDoc,
|
|
71
|
+
initialState?: Uint8Array,
|
|
72
|
+
logger?: Logger | null,
|
|
73
|
+
) {
|
|
74
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(fieldName)) {
|
|
75
|
+
throw new Error(
|
|
76
|
+
`CrdtField: refusing unsafe field identifier '${fieldName}' — must match [a-zA-Z_][a-zA-Z0-9_]*`
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
this.logger = logger ?? null;
|
|
80
|
+
this.cursorsEnabled = cursorsEnabled;
|
|
81
|
+
this.doc = new LoroDocClass();
|
|
82
|
+
if (initialState && initialState.length > 0) {
|
|
83
|
+
// Tolerance: catch bad-snapshot data (corrupt blob, stale legacy
|
|
84
|
+
// value left over from a pre-`bytes` migration) so the editor still
|
|
85
|
+
// mounts. Without this guard the rejection bubbles through
|
|
86
|
+
// `useCrdtField` → permanent fallback `<p>`, with no cursor.
|
|
87
|
+
try {
|
|
88
|
+
this.doc.import(initialState);
|
|
89
|
+
this.loadedFromCrdt = true;
|
|
90
|
+
} catch (e) {
|
|
91
|
+
this.logger?.warn(
|
|
92
|
+
{
|
|
93
|
+
error: e,
|
|
94
|
+
fieldName,
|
|
95
|
+
Category: 'sp00ky-client::CrdtField::constructor',
|
|
96
|
+
},
|
|
97
|
+
'Initial CRDT state is not a valid LoroDoc snapshot — starting empty and will seed from fallback text'
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
getDoc(): LoroDoc { return this.doc; }
|
|
104
|
+
|
|
105
|
+
/** Whether the LoroDoc was loaded from saved CRDT state */
|
|
106
|
+
hasContent(): boolean {
|
|
107
|
+
return this.loadedFromCrdt;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
startSync(
|
|
111
|
+
local: LocalStore,
|
|
112
|
+
remote: RemoteDatabaseService,
|
|
113
|
+
recordId: string,
|
|
114
|
+
sessionId: string,
|
|
115
|
+
debounceMs: number,
|
|
116
|
+
): void {
|
|
117
|
+
this.local = local;
|
|
118
|
+
this.remote = remote;
|
|
119
|
+
this.recordId = recordId;
|
|
120
|
+
this.sessionId = sessionId;
|
|
121
|
+
this.remoteDebounceMs = debounceMs;
|
|
122
|
+
// Every local Loro update writes the snapshot to the local cache
|
|
123
|
+
// *immediately* (so reload/offline see the latest text), then
|
|
124
|
+
// schedules a debounced push to remote. The local UPSERT is cheap —
|
|
125
|
+
// it's an in-memory SurrealKV write — but errors are swallowed so a
|
|
126
|
+
// bad write never blocks user input.
|
|
127
|
+
this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
|
|
128
|
+
void this.persistLocal();
|
|
129
|
+
this.scheduleRemotePush();
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Stop syncing this field. Flushes one final remote push by default so the
|
|
135
|
+
* last keystrokes aren't lost. Pass `{ flush: false }` on a bucket switch —
|
|
136
|
+
* the remote session already belongs to the NEXT user, and pushing this
|
|
137
|
+
* (previous user's) snapshot under it would clobber the record remotely.
|
|
138
|
+
*/
|
|
139
|
+
stopSync(options: { flush?: boolean } = {}): void {
|
|
140
|
+
const flush = options.flush ?? true;
|
|
141
|
+
if (this.unsubscribe) { this.unsubscribe(); this.unsubscribe = null; }
|
|
142
|
+
if (this.pushTimer) { clearTimeout(this.pushTimer); this.pushTimer = null; }
|
|
143
|
+
if (flush && this.remote && this.recordId) { void this.pushToRemote(); }
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
importRemote(state: Uint8Array): void {
|
|
147
|
+
// Echo suppression: skip imports within `remoteDebounceMs + 200` of
|
|
148
|
+
// our own push. The +200 guards against round-trip jitter where our
|
|
149
|
+
// own write echoes back from the LIVE feed before the debounce
|
|
150
|
+
// window closes.
|
|
151
|
+
if (Date.now() - this.lastPushTime < this.remoteDebounceMs + 200) return;
|
|
152
|
+
try {
|
|
153
|
+
this.doc.import(state);
|
|
154
|
+
// Persist the merged snapshot locally so the next reload/offline
|
|
155
|
+
// open sees the freshest converged state without waiting for the
|
|
156
|
+
// LIVE feed.
|
|
157
|
+
void this.persistLocal();
|
|
158
|
+
} catch (e) {
|
|
159
|
+
this.logger?.warn(
|
|
160
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::importRemote' },
|
|
161
|
+
'Failed to import remote CRDT state'
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
exportSnapshot(): Uint8Array {
|
|
167
|
+
return this.doc.export({ mode: 'snapshot' });
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Push this session's cursor blob into the parent row at
|
|
171
|
+
* `<field>.cursors[$sid]`. No-op when cursors aren't enabled on this
|
|
172
|
+
* field — the editor still calls this method optimistically, but
|
|
173
|
+
* without `@cursor` on the schema there's nowhere to store the blob.
|
|
174
|
+
* The UPDATE itself fires the parent table's LIVE feed, so other
|
|
175
|
+
* browsers receive the cursor change without a separate `_00_rv` bump. */
|
|
176
|
+
async pushCursorState(encoded: Uint8Array): Promise<void> {
|
|
177
|
+
if (!this.remote || !this.recordId) return;
|
|
178
|
+
if (!this.cursorsEnabled) return;
|
|
179
|
+
this.lastCursorPushTime = Date.now();
|
|
180
|
+
try {
|
|
181
|
+
const state = encodeBase64(encoded);
|
|
182
|
+
await this.remote.query(
|
|
183
|
+
`UPDATE $id SET ${this.fieldName}.cursors[$sid] = $state RETURN NONE;`,
|
|
184
|
+
{
|
|
185
|
+
id: parseRecordIdString(this.recordId),
|
|
186
|
+
sid: this.sessionId,
|
|
187
|
+
state,
|
|
188
|
+
}
|
|
189
|
+
);
|
|
190
|
+
} catch (e) {
|
|
191
|
+
this.logger?.warn(
|
|
192
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::pushCursorState' },
|
|
193
|
+
'Failed to push cursor state'
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
|
|
199
|
+
importRemoteCursor(base64State: string): void {
|
|
200
|
+
if (Date.now() - this.lastCursorPushTime < 300) return; // echo suppression
|
|
201
|
+
try {
|
|
202
|
+
const data = decodeBase64(base64State);
|
|
203
|
+
if (this._onCursorUpdate) {
|
|
204
|
+
this._onCursorUpdate(data);
|
|
205
|
+
} else {
|
|
206
|
+
// Only keep the latest cursor state — older positions are useless
|
|
207
|
+
this.pendingCursorUpdate = data;
|
|
208
|
+
}
|
|
209
|
+
} catch (e) {
|
|
210
|
+
this.logger?.warn(
|
|
211
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::importRemoteCursor' },
|
|
212
|
+
'Failed to apply remote cursor data'
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
private scheduleRemotePush(): void {
|
|
218
|
+
if (this.pushTimer) clearTimeout(this.pushTimer);
|
|
219
|
+
this.pushTimer = setTimeout(() => void this.pushToRemote(), this.remoteDebounceMs);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** SET path inside a parent row for the current snapshot. `@crdt`-only
|
|
223
|
+
* fields hold the snapshot directly (`<field>`); `@crdt @cursor`
|
|
224
|
+
* fields hold a `{ state, cursors }` object so the snapshot lives at
|
|
225
|
+
* `<field>.state` next to per-session cursor blobs. */
|
|
226
|
+
private statePath(): string {
|
|
227
|
+
return this.cursorsEnabled ? `${this.fieldName}.state` : this.fieldName;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Mirror the LoroDoc snapshot into the parent row locally. Runs on
|
|
231
|
+
* every local update and every remote import so reloads (online or
|
|
232
|
+
* offline) see the freshest content immediately. Failures are
|
|
233
|
+
* swallowed — a stale local write must never block user input. */
|
|
234
|
+
private async persistLocal(): Promise<void> {
|
|
235
|
+
if (!this.local || !this.recordId) return;
|
|
236
|
+
try {
|
|
237
|
+
await this.local.query(
|
|
238
|
+
`UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
|
|
239
|
+
{ id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
|
|
240
|
+
);
|
|
241
|
+
} catch (e) {
|
|
242
|
+
this.logger?.debug(
|
|
243
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::persistLocal' },
|
|
244
|
+
'Local CRDT persist failed (best-effort)'
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
private async pushToRemote(): Promise<void> {
|
|
250
|
+
if (!this.remote || !this.recordId) return;
|
|
251
|
+
this.lastPushTime = Date.now();
|
|
252
|
+
try {
|
|
253
|
+
// The UPDATE on the parent fires the parent table's LIVE feed,
|
|
254
|
+
// so cross-browser receivers see this change directly in the LIVE
|
|
255
|
+
// payload — no separate `_00_rv` bump or sidecar UPSERT.
|
|
256
|
+
await this.remote.query(
|
|
257
|
+
`UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
|
|
258
|
+
{ id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
|
|
259
|
+
);
|
|
260
|
+
this.pushRetryCount = 0;
|
|
261
|
+
} catch (e) {
|
|
262
|
+
this.logger?.warn(
|
|
263
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::pushToRemote' },
|
|
264
|
+
'Failed to push CRDT state to remote'
|
|
265
|
+
);
|
|
266
|
+
// Bounded retry. Offline first-loads will exhaust this and stop
|
|
267
|
+
// hammering; the next user keystroke (or the next time a remote
|
|
268
|
+
// event lands once we're back online) will kick off another push.
|
|
269
|
+
if (this.pushRetryCount < 2) {
|
|
270
|
+
this.pushRetryCount++;
|
|
271
|
+
this.scheduleRemotePush();
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
export function decodeBase64(b64: string): Uint8Array {
|
|
278
|
+
if (typeof atob === 'function') {
|
|
279
|
+
const binary = atob(b64);
|
|
280
|
+
const bytes = new Uint8Array(binary.length);
|
|
281
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
282
|
+
return bytes;
|
|
283
|
+
}
|
|
284
|
+
return new Uint8Array(Buffer.from(b64, 'base64'));
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
export function encodeBase64(bytes: Uint8Array): string {
|
|
288
|
+
if (typeof btoa === 'function') {
|
|
289
|
+
let binary = '';
|
|
290
|
+
for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
|
|
291
|
+
return btoa(binary);
|
|
292
|
+
}
|
|
293
|
+
return Buffer.from(bytes).toString('base64');
|
|
294
|
+
}
|