@spooky-sync/core 0.0.1-canary.52 → 0.0.1-canary.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +86 -1
- package/dist/index.js +328 -3
- package/package.json +2 -1
- package/src/index.ts +2 -1
- package/src/modules/crdt/crdt-field.ts +189 -0
- package/src/modules/crdt/index.ts +178 -0
- package/src/modules/data/index.ts +3 -0
- package/src/modules/sync/sync.ts +2 -1
- package/src/sp00ky.ts +27 -0
- package/src/utils/index.ts +12 -0
- package/src/utils/parser.ts +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import * as surrealdb0 from "surrealdb";
|
|
|
3
3
|
import { Duration, RecordId, Surreal as Surreal$1, SurrealTransaction } from "surrealdb";
|
|
4
4
|
import { AccessDefinition, BackendNames, BackendRoutes, BucketNames, ColumnSchema, GetTable, QueryBuilder, QueryOptions, RoutePayload, SchemaStructure, TableModel, TableNames, TypeNameToTypeMap } from "@spooky-sync/query-builder";
|
|
5
5
|
import { Logger } from "pino";
|
|
6
|
+
import { LoroDoc } from "loro-crdt";
|
|
6
7
|
|
|
7
8
|
//#region src/services/database/events/index.d.ts
|
|
8
9
|
declare const DatabaseEventTypes: {
|
|
@@ -194,6 +195,74 @@ declare class AuthService<S extends SchemaStructure> {
|
|
|
194
195
|
signIn<Name extends keyof S['access'] & string>(accessName: Name, params: ExtractAccessParams<S, Name, 'signIn'>): Promise<void>;
|
|
195
196
|
}
|
|
196
197
|
//#endregion
|
|
198
|
+
//#region src/modules/crdt/crdt-field.d.ts
|
|
199
|
+
declare const CURSOR_COLORS: string[];
|
|
200
|
+
declare function cursorColorFromName(name: string): string;
|
|
201
|
+
declare class CrdtField {
|
|
202
|
+
private fieldName;
|
|
203
|
+
private doc;
|
|
204
|
+
private pushTimer;
|
|
205
|
+
private remote;
|
|
206
|
+
private recordId;
|
|
207
|
+
private unsubscribe;
|
|
208
|
+
private lastPushTime;
|
|
209
|
+
private lastCursorPushTime;
|
|
210
|
+
private loadedFromCrdt;
|
|
211
|
+
private pushRetryCount;
|
|
212
|
+
private logger;
|
|
213
|
+
private _onCursorUpdate;
|
|
214
|
+
private pendingCursorUpdate;
|
|
215
|
+
/** Callback set by the editor to receive remote cursor updates.
|
|
216
|
+
* Any cursor data that arrived before this callback was set will be replayed. */
|
|
217
|
+
set onCursorUpdate(cb: ((data: Uint8Array) => void) | null);
|
|
218
|
+
get onCursorUpdate(): ((data: Uint8Array) => void) | null;
|
|
219
|
+
constructor(fieldName: string, initialState?: string, logger?: Logger$1 | null);
|
|
220
|
+
getDoc(): LoroDoc;
|
|
221
|
+
/** Whether the LoroDoc was loaded from saved CRDT state */
|
|
222
|
+
hasContent(): boolean;
|
|
223
|
+
startSync(remote: RemoteDatabaseService, recordId: string): void;
|
|
224
|
+
stopSync(): void;
|
|
225
|
+
importRemote(base64State: string): void;
|
|
226
|
+
exportSnapshot(): string;
|
|
227
|
+
/** Push cursor ephemeral state to _00_crdt as a "_cursor_<fieldName>" entry */
|
|
228
|
+
pushCursorState(encoded: Uint8Array): Promise<void>;
|
|
229
|
+
/** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
|
|
230
|
+
importRemoteCursor(base64State: string): void;
|
|
231
|
+
private schedulePush;
|
|
232
|
+
private pushToRemote;
|
|
233
|
+
}
|
|
234
|
+
//#endregion
|
|
235
|
+
//#region src/modules/crdt/index.d.ts
|
|
236
|
+
/**
|
|
237
|
+
* CrdtManager manages active CrdtField instances and their sync channels.
|
|
238
|
+
*
|
|
239
|
+
* Each open record gets a LIVE SELECT on _00_crdt that delivers remote
|
|
240
|
+
* changes in real time.
|
|
241
|
+
*/
|
|
242
|
+
declare class CrdtManager {
|
|
243
|
+
private schema;
|
|
244
|
+
private remote;
|
|
245
|
+
private fields;
|
|
246
|
+
private liveQueries;
|
|
247
|
+
private logger;
|
|
248
|
+
constructor(schema: SchemaStructure, remote: RemoteDatabaseService, logger: Logger$1);
|
|
249
|
+
/**
|
|
250
|
+
* Open a CRDT field for collaborative editing.
|
|
251
|
+
*
|
|
252
|
+
* @param table - Table name
|
|
253
|
+
* @param recordId - Full record ID (e.g., "thread:abc")
|
|
254
|
+
* @param field - Field name (e.g., "title", "content")
|
|
255
|
+
* @param fallbackText - Current plain text from the record, used to seed the
|
|
256
|
+
* LoroDoc if no CRDT state exists yet (migration path)
|
|
257
|
+
*/
|
|
258
|
+
open(table: string, recordId: string, field: string, fallbackText?: string): Promise<CrdtField>;
|
|
259
|
+
close(table: string, recordId: string, field: string): void;
|
|
260
|
+
closeAll(): void;
|
|
261
|
+
private ensureLiveSelect;
|
|
262
|
+
private killLiveSelect;
|
|
263
|
+
private makeKey;
|
|
264
|
+
}
|
|
265
|
+
//#endregion
|
|
197
266
|
//#region src/sp00ky.d.ts
|
|
198
267
|
declare class BucketHandle {
|
|
199
268
|
private bucketName;
|
|
@@ -218,6 +287,7 @@ declare class Sp00kyClient<S extends SchemaStructure> {
|
|
|
218
287
|
private dataModule;
|
|
219
288
|
private sync;
|
|
220
289
|
private devTools;
|
|
290
|
+
private crdtManager;
|
|
221
291
|
private logger;
|
|
222
292
|
auth: AuthService<S>;
|
|
223
293
|
streamProcessor: StreamProcessorService;
|
|
@@ -233,6 +303,16 @@ declare class Sp00kyClient<S extends SchemaStructure> {
|
|
|
233
303
|
init(): Promise<void>;
|
|
234
304
|
close(): Promise<void>;
|
|
235
305
|
authenticate(token: string): Promise<surrealdb0.Tokens>;
|
|
306
|
+
/**
|
|
307
|
+
* Open a CRDT field for collaborative editing.
|
|
308
|
+
* Returns a CrdtField with a LoroDoc that can be bound to any editor.
|
|
309
|
+
* Also starts a LIVE SELECT on _00_crdt for real-time sync.
|
|
310
|
+
*/
|
|
311
|
+
openCrdtField(table: string, recordId: string, field: string, fallbackText?: string): Promise<CrdtField>;
|
|
312
|
+
/**
|
|
313
|
+
* Close a CRDT field when editing is done.
|
|
314
|
+
*/
|
|
315
|
+
closeCrdtField(table: string, recordId: string, field: string): void;
|
|
236
316
|
deauthenticate(): Promise<void>;
|
|
237
317
|
query<Table extends TableNames<S>>(table: Table, options: QueryOptions<TableModel<GetTable<S, Table>>, false>, ttl?: QueryTimeToLive): QueryBuilder<S, Table, Sp00kyQueryResultPromise>;
|
|
238
318
|
private initQuery;
|
|
@@ -254,9 +334,14 @@ declare class Sp00kyClient<S extends SchemaStructure> {
|
|
|
254
334
|
//#endregion
|
|
255
335
|
//#region src/utils/index.d.ts
|
|
256
336
|
declare function fileToUint8Array(file: File | Blob): Promise<Uint8Array>;
|
|
337
|
+
/**
|
|
338
|
+
* Convert plain text to simple HTML paragraphs.
|
|
339
|
+
* Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
|
|
340
|
+
*/
|
|
341
|
+
declare function textToHtml(text: string): string;
|
|
257
342
|
/**
|
|
258
343
|
* Helper for retrying DB operations with exponential backoff
|
|
259
344
|
*/
|
|
260
345
|
|
|
261
346
|
//#endregion
|
|
262
|
-
export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, DebounceOptions, EventSubscriptionOptions, Level, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryTimeToLive, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, UpdateOptions, createAuthEventSystem, fileToUint8Array };
|
|
347
|
+
export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, Level, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryTimeToLive, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
|
package/dist/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import { QueryBuilder, RecordId as RecordId$1 } from "@spooky-sync/query-builder
|
|
|
3
3
|
import { createWasmWorkerEngines } from "@surrealdb/wasm";
|
|
4
4
|
import pino from "pino";
|
|
5
5
|
import init, { Sp00kyProcessor } from "@spooky-sync/ssp-wasm";
|
|
6
|
+
import { LoroDoc } from "loro-crdt";
|
|
6
7
|
|
|
7
8
|
//#region src/utils/surql.ts
|
|
8
9
|
const surql = {
|
|
@@ -71,7 +72,7 @@ const surql = {
|
|
|
71
72
|
//#region src/utils/parser.ts
|
|
72
73
|
function cleanRecord(tableSchema, record) {
|
|
73
74
|
const cleaned = {};
|
|
74
|
-
for (const [key, value] of Object.entries(record)) if (key === "id" || key in tableSchema) cleaned[key] = value;
|
|
75
|
+
for (const [key, value] of Object.entries(record)) if (key === "id" || key.startsWith("_00_") || key in tableSchema) cleaned[key] = value;
|
|
75
76
|
return cleaned;
|
|
76
77
|
}
|
|
77
78
|
function parseParams(tableSchema, params) {
|
|
@@ -169,6 +170,13 @@ async function fileToUint8Array(file) {
|
|
|
169
170
|
return new Uint8Array(buffer);
|
|
170
171
|
}
|
|
171
172
|
/**
|
|
173
|
+
* Convert plain text to simple HTML paragraphs.
|
|
174
|
+
* Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
|
|
175
|
+
*/
|
|
176
|
+
function textToHtml(text) {
|
|
177
|
+
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").split("\n").map((l) => `<p>${l || "<br>"}</p>`).join("");
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
172
180
|
* Helper for retrying DB operations with exponential backoff
|
|
173
181
|
*/
|
|
174
182
|
async function withRetry(logger, operation, retries = 3, delayMs = 100) {
|
|
@@ -1867,7 +1875,7 @@ var Sp00kySync = class {
|
|
|
1867
1875
|
Category: "sp00ky-client::Sp00kySync::processUpEvent"
|
|
1868
1876
|
}, "Processing up event");
|
|
1869
1877
|
switch (event.type) {
|
|
1870
|
-
case "create":
|
|
1878
|
+
case "create": {
|
|
1871
1879
|
const dataKeys = Object.keys(event.data).map((key) => ({
|
|
1872
1880
|
key,
|
|
1873
1881
|
variable: `data_${key}`
|
|
@@ -1879,6 +1887,7 @@ var Sp00kySync = class {
|
|
|
1879
1887
|
...prefixedParams
|
|
1880
1888
|
});
|
|
1881
1889
|
break;
|
|
1890
|
+
}
|
|
1882
1891
|
case "update":
|
|
1883
1892
|
await this.remote.query(`UPDATE $id MERGE $data`, {
|
|
1884
1893
|
id: event.record_id,
|
|
@@ -2849,6 +2858,305 @@ var CacheModule = class {
|
|
|
2849
2858
|
}
|
|
2850
2859
|
};
|
|
2851
2860
|
|
|
2861
|
+
//#endregion
|
|
2862
|
+
//#region src/modules/crdt/crdt-field.ts
|
|
2863
|
+
const CURSOR_COLORS = [
|
|
2864
|
+
"#3b82f6",
|
|
2865
|
+
"#ef4444",
|
|
2866
|
+
"#22c55e",
|
|
2867
|
+
"#f59e0b",
|
|
2868
|
+
"#8b5cf6",
|
|
2869
|
+
"#ec4899",
|
|
2870
|
+
"#14b8a6",
|
|
2871
|
+
"#f97316"
|
|
2872
|
+
];
|
|
2873
|
+
function cursorColorFromName(name) {
|
|
2874
|
+
let hash = 0;
|
|
2875
|
+
for (let i = 0; i < name.length; i++) hash = (hash << 5) - hash + name.charCodeAt(i) | 0;
|
|
2876
|
+
return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
|
|
2877
|
+
}
|
|
2878
|
+
var CrdtField = class {
|
|
2879
|
+
doc;
|
|
2880
|
+
pushTimer = null;
|
|
2881
|
+
remote = null;
|
|
2882
|
+
recordId = null;
|
|
2883
|
+
unsubscribe = null;
|
|
2884
|
+
lastPushTime = 0;
|
|
2885
|
+
lastCursorPushTime = 0;
|
|
2886
|
+
loadedFromCrdt = false;
|
|
2887
|
+
pushRetryCount = 0;
|
|
2888
|
+
logger;
|
|
2889
|
+
_onCursorUpdate = null;
|
|
2890
|
+
pendingCursorUpdate = null;
|
|
2891
|
+
/** Callback set by the editor to receive remote cursor updates.
|
|
2892
|
+
* Any cursor data that arrived before this callback was set will be replayed. */
|
|
2893
|
+
set onCursorUpdate(cb) {
|
|
2894
|
+
this._onCursorUpdate = cb;
|
|
2895
|
+
if (cb && this.pendingCursorUpdate) {
|
|
2896
|
+
try {
|
|
2897
|
+
cb(this.pendingCursorUpdate);
|
|
2898
|
+
} catch (e) {
|
|
2899
|
+
this.logger?.warn({
|
|
2900
|
+
error: e,
|
|
2901
|
+
Category: "sp00ky-client::CrdtField::onCursorUpdate"
|
|
2902
|
+
}, "Failed to replay pending cursor update");
|
|
2903
|
+
}
|
|
2904
|
+
this.pendingCursorUpdate = null;
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
get onCursorUpdate() {
|
|
2908
|
+
return this._onCursorUpdate;
|
|
2909
|
+
}
|
|
2910
|
+
constructor(fieldName, initialState, logger) {
|
|
2911
|
+
this.fieldName = fieldName;
|
|
2912
|
+
this.logger = logger ?? null;
|
|
2913
|
+
this.doc = new LoroDoc();
|
|
2914
|
+
if (initialState) {
|
|
2915
|
+
this.doc.import(decodeBase64(initialState));
|
|
2916
|
+
this.loadedFromCrdt = true;
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2919
|
+
getDoc() {
|
|
2920
|
+
return this.doc;
|
|
2921
|
+
}
|
|
2922
|
+
/** Whether the LoroDoc was loaded from saved CRDT state */
|
|
2923
|
+
hasContent() {
|
|
2924
|
+
return this.loadedFromCrdt;
|
|
2925
|
+
}
|
|
2926
|
+
startSync(remote, recordId) {
|
|
2927
|
+
this.remote = remote;
|
|
2928
|
+
this.recordId = recordId;
|
|
2929
|
+
this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
|
|
2930
|
+
this.schedulePush();
|
|
2931
|
+
});
|
|
2932
|
+
}
|
|
2933
|
+
stopSync() {
|
|
2934
|
+
if (this.unsubscribe) {
|
|
2935
|
+
this.unsubscribe();
|
|
2936
|
+
this.unsubscribe = null;
|
|
2937
|
+
}
|
|
2938
|
+
if (this.pushTimer) {
|
|
2939
|
+
clearTimeout(this.pushTimer);
|
|
2940
|
+
this.pushTimer = null;
|
|
2941
|
+
}
|
|
2942
|
+
if (this.remote && this.recordId) this.pushToRemote();
|
|
2943
|
+
}
|
|
2944
|
+
importRemote(base64State) {
|
|
2945
|
+
if (Date.now() - this.lastPushTime < 500) return;
|
|
2946
|
+
try {
|
|
2947
|
+
this.doc.import(decodeBase64(base64State));
|
|
2948
|
+
} catch (e) {
|
|
2949
|
+
this.logger?.warn({
|
|
2950
|
+
error: e,
|
|
2951
|
+
Category: "sp00ky-client::CrdtField::importRemote"
|
|
2952
|
+
}, "Failed to import remote CRDT state");
|
|
2953
|
+
}
|
|
2954
|
+
}
|
|
2955
|
+
exportSnapshot() {
|
|
2956
|
+
return encodeBase64(this.doc.export({ mode: "snapshot" }));
|
|
2957
|
+
}
|
|
2958
|
+
/** Push cursor ephemeral state to _00_crdt as a "_cursor_<fieldName>" entry */
|
|
2959
|
+
async pushCursorState(encoded) {
|
|
2960
|
+
if (!this.remote || !this.recordId) return;
|
|
2961
|
+
this.lastCursorPushTime = Date.now();
|
|
2962
|
+
try {
|
|
2963
|
+
const state = encodeBase64(encoded);
|
|
2964
|
+
await this.remote.query(`INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
|
|
2965
|
+
ON DUPLICATE KEY UPDATE state = $state`, {
|
|
2966
|
+
rid: parseRecordIdString(this.recordId),
|
|
2967
|
+
field: `_cursor_${this.fieldName}`,
|
|
2968
|
+
state
|
|
2969
|
+
});
|
|
2970
|
+
} catch (e) {
|
|
2971
|
+
this.logger?.warn({
|
|
2972
|
+
error: e,
|
|
2973
|
+
Category: "sp00ky-client::CrdtField::pushCursorState"
|
|
2974
|
+
}, "Failed to push cursor state");
|
|
2975
|
+
}
|
|
2976
|
+
}
|
|
2977
|
+
/** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
|
|
2978
|
+
importRemoteCursor(base64State) {
|
|
2979
|
+
if (Date.now() - this.lastCursorPushTime < 300) return;
|
|
2980
|
+
try {
|
|
2981
|
+
const data = decodeBase64(base64State);
|
|
2982
|
+
if (this._onCursorUpdate) this._onCursorUpdate(data);
|
|
2983
|
+
else this.pendingCursorUpdate = data;
|
|
2984
|
+
} catch (e) {
|
|
2985
|
+
this.logger?.warn({
|
|
2986
|
+
error: e,
|
|
2987
|
+
Category: "sp00ky-client::CrdtField::importRemoteCursor"
|
|
2988
|
+
}, "Failed to apply remote cursor data");
|
|
2989
|
+
}
|
|
2990
|
+
}
|
|
2991
|
+
schedulePush() {
|
|
2992
|
+
if (this.pushTimer) clearTimeout(this.pushTimer);
|
|
2993
|
+
this.pushTimer = setTimeout(() => void this.pushToRemote(), 300);
|
|
2994
|
+
}
|
|
2995
|
+
async pushToRemote() {
|
|
2996
|
+
if (!this.remote || !this.recordId) return;
|
|
2997
|
+
this.lastPushTime = Date.now();
|
|
2998
|
+
try {
|
|
2999
|
+
await this.remote.query(`INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
|
|
3000
|
+
ON DUPLICATE KEY UPDATE state = $state`, {
|
|
3001
|
+
rid: parseRecordIdString(this.recordId),
|
|
3002
|
+
field: this.fieldName,
|
|
3003
|
+
state: this.exportSnapshot()
|
|
3004
|
+
});
|
|
3005
|
+
this.pushRetryCount = 0;
|
|
3006
|
+
} catch (e) {
|
|
3007
|
+
this.logger?.warn({
|
|
3008
|
+
error: e,
|
|
3009
|
+
Category: "sp00ky-client::CrdtField::pushToRemote"
|
|
3010
|
+
}, "Failed to push CRDT state to remote");
|
|
3011
|
+
if (this.pushRetryCount < 2) {
|
|
3012
|
+
this.pushRetryCount++;
|
|
3013
|
+
this.schedulePush();
|
|
3014
|
+
}
|
|
3015
|
+
}
|
|
3016
|
+
}
|
|
3017
|
+
};
|
|
3018
|
+
function decodeBase64(b64) {
|
|
3019
|
+
if (typeof atob === "function") {
|
|
3020
|
+
const binary = atob(b64);
|
|
3021
|
+
const bytes = new Uint8Array(binary.length);
|
|
3022
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
3023
|
+
return bytes;
|
|
3024
|
+
}
|
|
3025
|
+
return new Uint8Array(Buffer.from(b64, "base64"));
|
|
3026
|
+
}
|
|
3027
|
+
function encodeBase64(bytes) {
|
|
3028
|
+
if (typeof btoa === "function") {
|
|
3029
|
+
let binary = "";
|
|
3030
|
+
for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
|
|
3031
|
+
return btoa(binary);
|
|
3032
|
+
}
|
|
3033
|
+
return Buffer.from(bytes).toString("base64");
|
|
3034
|
+
}
|
|
3035
|
+
|
|
3036
|
+
//#endregion
|
|
3037
|
+
//#region src/modules/crdt/index.ts
|
|
3038
|
+
/**
|
|
3039
|
+
* CrdtManager manages active CrdtField instances and their sync channels.
|
|
3040
|
+
*
|
|
3041
|
+
* Each open record gets a LIVE SELECT on _00_crdt that delivers remote
|
|
3042
|
+
* changes in real time.
|
|
3043
|
+
*/
|
|
3044
|
+
var CrdtManager = class {
|
|
3045
|
+
fields = /* @__PURE__ */ new Map();
|
|
3046
|
+
liveQueries = /* @__PURE__ */ new Map();
|
|
3047
|
+
logger;
|
|
3048
|
+
constructor(schema, remote, logger) {
|
|
3049
|
+
this.schema = schema;
|
|
3050
|
+
this.remote = remote;
|
|
3051
|
+
this.logger = logger.child({ service: "CrdtManager" });
|
|
3052
|
+
}
|
|
3053
|
+
/**
|
|
3054
|
+
* Open a CRDT field for collaborative editing.
|
|
3055
|
+
*
|
|
3056
|
+
* @param table - Table name
|
|
3057
|
+
* @param recordId - Full record ID (e.g., "thread:abc")
|
|
3058
|
+
* @param field - Field name (e.g., "title", "content")
|
|
3059
|
+
* @param fallbackText - Current plain text from the record, used to seed the
|
|
3060
|
+
* LoroDoc if no CRDT state exists yet (migration path)
|
|
3061
|
+
*/
|
|
3062
|
+
async open(table, recordId, field, fallbackText) {
|
|
3063
|
+
const key = this.makeKey(table, recordId, field);
|
|
3064
|
+
let crdtField = this.fields.get(key);
|
|
3065
|
+
if (crdtField) return crdtField;
|
|
3066
|
+
let initialCrdtState;
|
|
3067
|
+
try {
|
|
3068
|
+
const [result] = await this.remote.query("SELECT VALUE state FROM _00_crdt WHERE record_id = $rid AND field = $field LIMIT 1", {
|
|
3069
|
+
rid: parseRecordIdString(recordId),
|
|
3070
|
+
field
|
|
3071
|
+
});
|
|
3072
|
+
if (result && result.length > 0 && result[0]) initialCrdtState = result[0];
|
|
3073
|
+
} catch (e) {
|
|
3074
|
+
this.logger.debug({
|
|
3075
|
+
error: e,
|
|
3076
|
+
Category: "sp00ky-client::CrdtManager::open"
|
|
3077
|
+
}, "No existing CRDT state found");
|
|
3078
|
+
}
|
|
3079
|
+
crdtField = new CrdtField(field, initialCrdtState, this.logger);
|
|
3080
|
+
crdtField.startSync(this.remote, recordId);
|
|
3081
|
+
this.fields.set(key, crdtField);
|
|
3082
|
+
this.logger.info({
|
|
3083
|
+
key,
|
|
3084
|
+
hasInitialState: !!initialCrdtState,
|
|
3085
|
+
hasFallback: !!fallbackText,
|
|
3086
|
+
Category: "sp00ky-client::CrdtManager::open"
|
|
3087
|
+
}, "CrdtField opened");
|
|
3088
|
+
await this.ensureLiveSelect(table, recordId);
|
|
3089
|
+
return crdtField;
|
|
3090
|
+
}
|
|
3091
|
+
close(table, recordId, field) {
|
|
3092
|
+
const key = this.makeKey(table, recordId, field);
|
|
3093
|
+
const crdtField = this.fields.get(key);
|
|
3094
|
+
if (crdtField) {
|
|
3095
|
+
crdtField.stopSync();
|
|
3096
|
+
this.fields.delete(key);
|
|
3097
|
+
}
|
|
3098
|
+
const prefix = `${table}:${recordId}:`;
|
|
3099
|
+
if (!Array.from(this.fields.keys()).some((k) => k !== key && k.startsWith(prefix))) this.killLiveSelect(recordId);
|
|
3100
|
+
this.logger.debug({
|
|
3101
|
+
key,
|
|
3102
|
+
Category: "sp00ky-client::CrdtManager::close"
|
|
3103
|
+
}, "CrdtField closed");
|
|
3104
|
+
}
|
|
3105
|
+
closeAll() {
|
|
3106
|
+
for (const [_, field] of this.fields) field.stopSync();
|
|
3107
|
+
this.fields.clear();
|
|
3108
|
+
for (const recordId of this.liveQueries.keys()) this.killLiveSelect(recordId);
|
|
3109
|
+
}
|
|
3110
|
+
async ensureLiveSelect(table, recordId) {
|
|
3111
|
+
if (this.liveQueries.has(recordId)) return;
|
|
3112
|
+
try {
|
|
3113
|
+
const [uuid] = await this.remote.query(`LIVE SELECT * FROM _00_crdt WHERE record_id = $rid`, { rid: parseRecordIdString(recordId) });
|
|
3114
|
+
this.liveQueries.set(recordId, {
|
|
3115
|
+
uuid,
|
|
3116
|
+
table
|
|
3117
|
+
});
|
|
3118
|
+
(await this.remote.getClient().liveOf(uuid)).subscribe((message) => {
|
|
3119
|
+
if (message.action === "KILLED") return;
|
|
3120
|
+
if (message.action === "CREATE" || message.action === "UPDATE") {
|
|
3121
|
+
const fieldName = message.value.field;
|
|
3122
|
+
const state = message.value.state;
|
|
3123
|
+
if (!fieldName || !state) return;
|
|
3124
|
+
if (fieldName.startsWith("_cursor_")) {
|
|
3125
|
+
const actualField = fieldName.slice(8);
|
|
3126
|
+
const key = this.makeKey(table, recordId, actualField);
|
|
3127
|
+
const crdtField = this.fields.get(key);
|
|
3128
|
+
if (crdtField) crdtField.importRemoteCursor(state);
|
|
3129
|
+
return;
|
|
3130
|
+
}
|
|
3131
|
+
const key = this.makeKey(table, recordId, fieldName);
|
|
3132
|
+
const crdtField = this.fields.get(key);
|
|
3133
|
+
if (crdtField) crdtField.importRemote(state);
|
|
3134
|
+
}
|
|
3135
|
+
});
|
|
3136
|
+
this.logger.info({
|
|
3137
|
+
recordId,
|
|
3138
|
+
Category: "sp00ky-client::CrdtManager::ensureLiveSelect"
|
|
3139
|
+
}, "LIVE SELECT on _00_crdt started");
|
|
3140
|
+
} catch (e) {
|
|
3141
|
+
this.logger.warn({
|
|
3142
|
+
error: e,
|
|
3143
|
+
recordId,
|
|
3144
|
+
Category: "sp00ky-client::CrdtManager::ensureLiveSelect"
|
|
3145
|
+
}, "Failed to start LIVE SELECT on _00_crdt");
|
|
3146
|
+
}
|
|
3147
|
+
}
|
|
3148
|
+
killLiveSelect(recordId) {
|
|
3149
|
+
const entry = this.liveQueries.get(recordId);
|
|
3150
|
+
if (entry) {
|
|
3151
|
+
this.remote.query("KILL $uuid", { uuid: entry.uuid }).catch(() => {});
|
|
3152
|
+
this.liveQueries.delete(recordId);
|
|
3153
|
+
}
|
|
3154
|
+
}
|
|
3155
|
+
makeKey(table, recordId, field) {
|
|
3156
|
+
return `${table}:${recordId}:${field}`;
|
|
3157
|
+
}
|
|
3158
|
+
};
|
|
3159
|
+
|
|
2852
3160
|
//#endregion
|
|
2853
3161
|
//#region src/services/persistence/localstorage.ts
|
|
2854
3162
|
var LocalStoragePersistenceClient = class {
|
|
@@ -2996,6 +3304,7 @@ var Sp00kyClient = class {
|
|
|
2996
3304
|
dataModule;
|
|
2997
3305
|
sync;
|
|
2998
3306
|
devTools;
|
|
3307
|
+
crdtManager;
|
|
2999
3308
|
logger;
|
|
3000
3309
|
auth;
|
|
3001
3310
|
streamProcessor;
|
|
@@ -3033,6 +3342,7 @@ var Sp00kyClient = class {
|
|
|
3033
3342
|
this.cache = new CacheModule(this.local, this.streamProcessor, (update) => {
|
|
3034
3343
|
this.dataModule.onStreamUpdate(update);
|
|
3035
3344
|
}, logger);
|
|
3345
|
+
this.crdtManager = new CrdtManager(this.config.schema, this.remote, logger);
|
|
3036
3346
|
this.dataModule = new DataModule(this.cache, this.local, this.config.schema, logger, this.config.streamDebounceTime);
|
|
3037
3347
|
this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
|
|
3038
3348
|
this.sync = new Sp00kySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger);
|
|
@@ -3091,12 +3401,27 @@ var Sp00kyClient = class {
|
|
|
3091
3401
|
}
|
|
3092
3402
|
}
|
|
3093
3403
|
async close() {
|
|
3404
|
+
this.crdtManager.closeAll();
|
|
3094
3405
|
await this.local.close();
|
|
3095
3406
|
await this.remote.close();
|
|
3096
3407
|
}
|
|
3097
3408
|
authenticate(token) {
|
|
3098
3409
|
return this.remote.getClient().authenticate(token);
|
|
3099
3410
|
}
|
|
3411
|
+
/**
|
|
3412
|
+
* Open a CRDT field for collaborative editing.
|
|
3413
|
+
* Returns a CrdtField with a LoroDoc that can be bound to any editor.
|
|
3414
|
+
* Also starts a LIVE SELECT on _00_crdt for real-time sync.
|
|
3415
|
+
*/
|
|
3416
|
+
async openCrdtField(table, recordId, field, fallbackText) {
|
|
3417
|
+
return this.crdtManager.open(table, recordId, field, fallbackText);
|
|
3418
|
+
}
|
|
3419
|
+
/**
|
|
3420
|
+
* Close a CRDT field when editing is done.
|
|
3421
|
+
*/
|
|
3422
|
+
closeCrdtField(table, recordId, field) {
|
|
3423
|
+
this.crdtManager.close(table, recordId, field);
|
|
3424
|
+
}
|
|
3100
3425
|
deauthenticate() {
|
|
3101
3426
|
return this.remote.getClient().invalidate();
|
|
3102
3427
|
}
|
|
@@ -3158,4 +3483,4 @@ var Sp00kyClient = class {
|
|
|
3158
3483
|
};
|
|
3159
3484
|
|
|
3160
3485
|
//#endregion
|
|
3161
|
-
export { AuthEventTypes, AuthService, BucketHandle, Sp00kyClient, createAuthEventSystem, fileToUint8Array };
|
|
3486
|
+
export { AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, Sp00kyClient, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.54",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -63,6 +63,7 @@
|
|
|
63
63
|
"@spooky-sync/ssp-wasm": "workspace:*",
|
|
64
64
|
"@surrealdb/wasm": "^3.0.0",
|
|
65
65
|
"fast-json-patch": "^3.1.1",
|
|
66
|
+
"loro-crdt": "^1.5.6",
|
|
66
67
|
"pino": "^10.1.0",
|
|
67
68
|
"surrealdb": "2.0.0"
|
|
68
69
|
},
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export * from './types';
|
|
2
2
|
export * from './sp00ky';
|
|
3
3
|
export * from './modules/auth/index';
|
|
4
|
-
export {
|
|
4
|
+
export { CrdtField, CrdtManager, cursorColorFromName, CURSOR_COLORS } from './modules/crdt/index';
|
|
5
|
+
export { fileToUint8Array, textToHtml } from './utils/index';
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { LoroDoc } from 'loro-crdt';
|
|
2
|
+
import type { RemoteDatabaseService } from '../../services/database/index';
|
|
3
|
+
import type { Logger } from '../../services/logger/index';
|
|
4
|
+
import { parseRecordIdString } from '../../utils/index';
|
|
5
|
+
|
|
6
|
+
// ==================== CURSOR UTILITIES ====================
|
|
7
|
+
|
|
8
|
+
export const CURSOR_COLORS = [
|
|
9
|
+
'#3b82f6', '#ef4444', '#22c55e', '#f59e0b',
|
|
10
|
+
'#8b5cf6', '#ec4899', '#14b8a6', '#f97316',
|
|
11
|
+
];
|
|
12
|
+
|
|
13
|
+
export function cursorColorFromName(name: string): string {
|
|
14
|
+
let hash = 0;
|
|
15
|
+
for (let i = 0; i < name.length; i++) {
|
|
16
|
+
hash = ((hash << 5) - hash + name.charCodeAt(i)) | 0;
|
|
17
|
+
}
|
|
18
|
+
return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// ==================== CRDT FIELD ====================
|
|
22
|
+
|
|
23
|
+
export class CrdtField {
|
|
24
|
+
private doc: LoroDoc;
|
|
25
|
+
private pushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
26
|
+
private remote: RemoteDatabaseService | null = null;
|
|
27
|
+
private recordId: string | null = null;
|
|
28
|
+
private unsubscribe: (() => void) | null = null;
|
|
29
|
+
private lastPushTime = 0;
|
|
30
|
+
private lastCursorPushTime = 0;
|
|
31
|
+
private loadedFromCrdt = false;
|
|
32
|
+
private pushRetryCount = 0;
|
|
33
|
+
private logger: Logger | null;
|
|
34
|
+
|
|
35
|
+
private _onCursorUpdate: ((data: Uint8Array) => void) | null = null;
|
|
36
|
+
private pendingCursorUpdate: Uint8Array | null = null;
|
|
37
|
+
|
|
38
|
+
/** Callback set by the editor to receive remote cursor updates.
|
|
39
|
+
* Any cursor data that arrived before this callback was set will be replayed. */
|
|
40
|
+
set onCursorUpdate(cb: ((data: Uint8Array) => void) | null) {
|
|
41
|
+
this._onCursorUpdate = cb;
|
|
42
|
+
if (cb && this.pendingCursorUpdate) {
|
|
43
|
+
try { cb(this.pendingCursorUpdate); } catch (e) {
|
|
44
|
+
this.logger?.warn(
|
|
45
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::onCursorUpdate' },
|
|
46
|
+
'Failed to replay pending cursor update'
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
this.pendingCursorUpdate = null;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
get onCursorUpdate() { return this._onCursorUpdate; }
|
|
54
|
+
|
|
55
|
+
constructor(
|
|
56
|
+
private fieldName: string,
|
|
57
|
+
initialState?: string,
|
|
58
|
+
logger?: Logger | null,
|
|
59
|
+
) {
|
|
60
|
+
this.logger = logger ?? null;
|
|
61
|
+
this.doc = new LoroDoc();
|
|
62
|
+
if (initialState) {
|
|
63
|
+
this.doc.import(decodeBase64(initialState));
|
|
64
|
+
this.loadedFromCrdt = true;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
getDoc(): LoroDoc { return this.doc; }
|
|
69
|
+
|
|
70
|
+
/** Whether the LoroDoc was loaded from saved CRDT state */
|
|
71
|
+
hasContent(): boolean {
|
|
72
|
+
return this.loadedFromCrdt;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
startSync(remote: RemoteDatabaseService, recordId: string): void {
|
|
76
|
+
this.remote = remote;
|
|
77
|
+
this.recordId = recordId;
|
|
78
|
+
this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
|
|
79
|
+
this.schedulePush();
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
stopSync(): void {
|
|
84
|
+
if (this.unsubscribe) { this.unsubscribe(); this.unsubscribe = null; }
|
|
85
|
+
if (this.pushTimer) { clearTimeout(this.pushTimer); this.pushTimer = null; }
|
|
86
|
+
if (this.remote && this.recordId) { void this.pushToRemote(); }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
importRemote(base64State: string): void {
|
|
90
|
+
// Suppress echo: skip imports within 500ms of our own push
|
|
91
|
+
if (Date.now() - this.lastPushTime < 500) return;
|
|
92
|
+
try {
|
|
93
|
+
this.doc.import(decodeBase64(base64State));
|
|
94
|
+
} catch (e) {
|
|
95
|
+
this.logger?.warn(
|
|
96
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::importRemote' },
|
|
97
|
+
'Failed to import remote CRDT state'
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
exportSnapshot(): string {
|
|
103
|
+
return encodeBase64(this.doc.export({ mode: 'snapshot' }));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Push cursor ephemeral state to _00_crdt as a "_cursor_<fieldName>" entry */
|
|
107
|
+
async pushCursorState(encoded: Uint8Array): Promise<void> {
|
|
108
|
+
if (!this.remote || !this.recordId) return;
|
|
109
|
+
this.lastCursorPushTime = Date.now();
|
|
110
|
+
try {
|
|
111
|
+
const state = encodeBase64(encoded);
|
|
112
|
+
await this.remote.query(
|
|
113
|
+
`INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
|
|
114
|
+
ON DUPLICATE KEY UPDATE state = $state`,
|
|
115
|
+
{ rid: parseRecordIdString(this.recordId), field: `_cursor_${this.fieldName}`, state }
|
|
116
|
+
);
|
|
117
|
+
} catch (e) {
|
|
118
|
+
this.logger?.warn(
|
|
119
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::pushCursorState' },
|
|
120
|
+
'Failed to push cursor state'
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
|
|
126
|
+
importRemoteCursor(base64State: string): void {
|
|
127
|
+
if (Date.now() - this.lastCursorPushTime < 300) return; // echo suppression
|
|
128
|
+
try {
|
|
129
|
+
const data = decodeBase64(base64State);
|
|
130
|
+
if (this._onCursorUpdate) {
|
|
131
|
+
this._onCursorUpdate(data);
|
|
132
|
+
} else {
|
|
133
|
+
// Only keep the latest cursor state — older positions are useless
|
|
134
|
+
this.pendingCursorUpdate = data;
|
|
135
|
+
}
|
|
136
|
+
} catch (e) {
|
|
137
|
+
this.logger?.warn(
|
|
138
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::importRemoteCursor' },
|
|
139
|
+
'Failed to apply remote cursor data'
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
private schedulePush(): void {
|
|
145
|
+
if (this.pushTimer) clearTimeout(this.pushTimer);
|
|
146
|
+
this.pushTimer = setTimeout(() => void this.pushToRemote(), 300);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
private async pushToRemote(): Promise<void> {
|
|
150
|
+
if (!this.remote || !this.recordId) return;
|
|
151
|
+
this.lastPushTime = Date.now();
|
|
152
|
+
try {
|
|
153
|
+
await this.remote.query(
|
|
154
|
+
`INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
|
|
155
|
+
ON DUPLICATE KEY UPDATE state = $state`,
|
|
156
|
+
{ rid: parseRecordIdString(this.recordId), field: this.fieldName, state: this.exportSnapshot() }
|
|
157
|
+
);
|
|
158
|
+
this.pushRetryCount = 0;
|
|
159
|
+
} catch (e) {
|
|
160
|
+
this.logger?.warn(
|
|
161
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::pushToRemote' },
|
|
162
|
+
'Failed to push CRDT state to remote'
|
|
163
|
+
);
|
|
164
|
+
if (this.pushRetryCount < 2) {
|
|
165
|
+
this.pushRetryCount++;
|
|
166
|
+
this.schedulePush();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function decodeBase64(b64: string): Uint8Array {
|
|
173
|
+
if (typeof atob === 'function') {
|
|
174
|
+
const binary = atob(b64);
|
|
175
|
+
const bytes = new Uint8Array(binary.length);
|
|
176
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
177
|
+
return bytes;
|
|
178
|
+
}
|
|
179
|
+
return new Uint8Array(Buffer.from(b64, 'base64'));
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export function encodeBase64(bytes: Uint8Array): string {
|
|
183
|
+
if (typeof btoa === 'function') {
|
|
184
|
+
let binary = '';
|
|
185
|
+
for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
|
|
186
|
+
return btoa(binary);
|
|
187
|
+
}
|
|
188
|
+
return Buffer.from(bytes).toString('base64');
|
|
189
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { RemoteDatabaseService } from '../../services/database/index';
|
|
3
|
+
import type { Logger } from '../../services/logger/index';
|
|
4
|
+
import type { Uuid } from 'surrealdb';
|
|
5
|
+
import { CrdtField } from './crdt-field';
|
|
6
|
+
import { parseRecordIdString } from '../../utils/index';
|
|
7
|
+
|
|
8
|
+
export { CrdtField, cursorColorFromName, CURSOR_COLORS } from './crdt-field';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* CrdtManager manages active CrdtField instances and their sync channels.
|
|
12
|
+
*
|
|
13
|
+
* Each open record gets a LIVE SELECT on _00_crdt that delivers remote
|
|
14
|
+
* changes in real time.
|
|
15
|
+
*/
|
|
16
|
+
export class CrdtManager {
|
|
17
|
+
private fields = new Map<string, CrdtField>();
|
|
18
|
+
private liveQueries = new Map<string, { uuid: Uuid; table: string }>();
|
|
19
|
+
private logger: Logger;
|
|
20
|
+
|
|
21
|
+
constructor(
|
|
22
|
+
private schema: SchemaStructure,
|
|
23
|
+
private remote: RemoteDatabaseService,
|
|
24
|
+
logger: Logger
|
|
25
|
+
) {
|
|
26
|
+
this.logger = logger.child({ service: 'CrdtManager' });
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Open a CRDT field for collaborative editing.
|
|
31
|
+
*
|
|
32
|
+
* @param table - Table name
|
|
33
|
+
* @param recordId - Full record ID (e.g., "thread:abc")
|
|
34
|
+
* @param field - Field name (e.g., "title", "content")
|
|
35
|
+
* @param fallbackText - Current plain text from the record, used to seed the
|
|
36
|
+
* LoroDoc if no CRDT state exists yet (migration path)
|
|
37
|
+
*/
|
|
38
|
+
async open(
|
|
39
|
+
table: string,
|
|
40
|
+
recordId: string,
|
|
41
|
+
field: string,
|
|
42
|
+
fallbackText?: string,
|
|
43
|
+
): Promise<CrdtField> {
|
|
44
|
+
const key = this.makeKey(table, recordId, field);
|
|
45
|
+
let crdtField = this.fields.get(key);
|
|
46
|
+
|
|
47
|
+
if (crdtField) {
|
|
48
|
+
return crdtField;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Load saved CRDT state from remote _00_crdt table
|
|
52
|
+
let initialCrdtState: string | undefined;
|
|
53
|
+
try {
|
|
54
|
+
const [result] = await this.remote.query<[string[]]>(
|
|
55
|
+
'SELECT VALUE state FROM _00_crdt WHERE record_id = $rid AND field = $field LIMIT 1',
|
|
56
|
+
{ rid: parseRecordIdString(recordId), field }
|
|
57
|
+
);
|
|
58
|
+
if (result && result.length > 0 && result[0]) {
|
|
59
|
+
initialCrdtState = result[0];
|
|
60
|
+
}
|
|
61
|
+
} catch (e) {
|
|
62
|
+
this.logger.debug(
|
|
63
|
+
{ error: e, Category: 'sp00ky-client::CrdtManager::open' },
|
|
64
|
+
'No existing CRDT state found'
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
crdtField = new CrdtField(field, initialCrdtState, this.logger);
|
|
69
|
+
crdtField.startSync(this.remote, recordId);
|
|
70
|
+
this.fields.set(key, crdtField);
|
|
71
|
+
|
|
72
|
+
this.logger.info(
|
|
73
|
+
{ key, hasInitialState: !!initialCrdtState, hasFallback: !!fallbackText, Category: 'sp00ky-client::CrdtManager::open' },
|
|
74
|
+
'CrdtField opened'
|
|
75
|
+
);
|
|
76
|
+
|
|
77
|
+
await this.ensureLiveSelect(table, recordId);
|
|
78
|
+
|
|
79
|
+
return crdtField;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
close(table: string, recordId: string, field: string): void {
|
|
83
|
+
const key = this.makeKey(table, recordId, field);
|
|
84
|
+
const crdtField = this.fields.get(key);
|
|
85
|
+
if (crdtField) {
|
|
86
|
+
crdtField.stopSync();
|
|
87
|
+
this.fields.delete(key);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const prefix = `${table}:${recordId}:`;
|
|
91
|
+
const hasOtherFields = Array.from(this.fields.keys()).some(
|
|
92
|
+
(k) => k !== key && k.startsWith(prefix)
|
|
93
|
+
);
|
|
94
|
+
if (!hasOtherFields) {
|
|
95
|
+
this.killLiveSelect(recordId);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
this.logger.debug(
|
|
99
|
+
{ key, Category: 'sp00ky-client::CrdtManager::close' },
|
|
100
|
+
'CrdtField closed'
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
closeAll(): void {
|
|
105
|
+
for (const [_, field] of this.fields) {
|
|
106
|
+
field.stopSync();
|
|
107
|
+
}
|
|
108
|
+
this.fields.clear();
|
|
109
|
+
for (const recordId of this.liveQueries.keys()) {
|
|
110
|
+
this.killLiveSelect(recordId);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
private async ensureLiveSelect(table: string, recordId: string): Promise<void> {
|
|
115
|
+
if (this.liveQueries.has(recordId)) return;
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const [uuid] = await this.remote.query<[Uuid]>(
|
|
119
|
+
`LIVE SELECT * FROM _00_crdt WHERE record_id = $rid`,
|
|
120
|
+
{ rid: parseRecordIdString(recordId) },
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
this.liveQueries.set(recordId, { uuid, table });
|
|
124
|
+
|
|
125
|
+
const subscription = await this.remote.getClient().liveOf(uuid);
|
|
126
|
+
subscription.subscribe((message) => {
|
|
127
|
+
if (message.action === 'KILLED') return;
|
|
128
|
+
|
|
129
|
+
if (message.action === 'CREATE' || message.action === 'UPDATE') {
|
|
130
|
+
const fieldName = message.value.field as string;
|
|
131
|
+
const state = message.value.state as string;
|
|
132
|
+
|
|
133
|
+
if (!fieldName || !state) return;
|
|
134
|
+
|
|
135
|
+
// Route cursor updates to the corresponding CrdtField
|
|
136
|
+
if (fieldName.startsWith('_cursor_')) {
|
|
137
|
+
const actualField = fieldName.slice('_cursor_'.length);
|
|
138
|
+
const key = this.makeKey(table, recordId, actualField);
|
|
139
|
+
const crdtField = this.fields.get(key);
|
|
140
|
+
if (crdtField) {
|
|
141
|
+
crdtField.importRemoteCursor(state);
|
|
142
|
+
}
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Route document updates
|
|
147
|
+
const key = this.makeKey(table, recordId, fieldName);
|
|
148
|
+
const crdtField = this.fields.get(key);
|
|
149
|
+
if (crdtField) {
|
|
150
|
+
crdtField.importRemote(state);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
this.logger.info(
|
|
156
|
+
{ recordId, Category: 'sp00ky-client::CrdtManager::ensureLiveSelect' },
|
|
157
|
+
'LIVE SELECT on _00_crdt started'
|
|
158
|
+
);
|
|
159
|
+
} catch (e) {
|
|
160
|
+
this.logger.warn(
|
|
161
|
+
{ error: e, recordId, Category: 'sp00ky-client::CrdtManager::ensureLiveSelect' },
|
|
162
|
+
'Failed to start LIVE SELECT on _00_crdt'
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
private killLiveSelect(recordId: string): void {
|
|
168
|
+
const entry = this.liveQueries.get(recordId);
|
|
169
|
+
if (entry) {
|
|
170
|
+
this.remote.query('KILL $uuid', { uuid: entry.uuid }).catch(() => {});
|
|
171
|
+
this.liveQueries.delete(recordId);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
private makeKey(table: string, recordId: string, field: string): string {
|
|
176
|
+
return `${table}:${recordId}:${field}`;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
@@ -468,6 +468,9 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
468
468
|
const params = parseParams(tableSchema.columns, data);
|
|
469
469
|
const mutationId = parseRecordIdString(`_00_pending_mutations:${Date.now()}`);
|
|
470
470
|
|
|
471
|
+
// Note: CRDT state is pushed directly to the _00_crdt table by CrdtField.pushToRemote(),
|
|
472
|
+
// NOT through the record update pipeline. This keeps the record data clean.
|
|
473
|
+
|
|
471
474
|
// Capture current record state before mutation for rollback support
|
|
472
475
|
const [beforeRecord] = await withRetry(this.logger, () =>
|
|
473
476
|
this.local.query<[Record<string, any>]>('SELECT * FROM ONLY $id', { id: rid })
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -183,7 +183,7 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
183
183
|
'Processing up event'
|
|
184
184
|
);
|
|
185
185
|
switch (event.type) {
|
|
186
|
-
case 'create':
|
|
186
|
+
case 'create': {
|
|
187
187
|
const dataKeys = Object.keys(event.data).map((key) => ({ key, variable: `data_${key}` }));
|
|
188
188
|
const prefixedParams = Object.fromEntries(
|
|
189
189
|
dataKeys.map(({ key, variable }) => [variable, event.data[key]])
|
|
@@ -194,6 +194,7 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
194
194
|
...prefixedParams,
|
|
195
195
|
});
|
|
196
196
|
break;
|
|
197
|
+
}
|
|
197
198
|
case 'update':
|
|
198
199
|
await this.remote.query(`UPDATE $id MERGE $data`, {
|
|
199
200
|
id: event.record_id,
|
package/src/sp00ky.ts
CHANGED
|
@@ -34,6 +34,7 @@ import { AuthService } from './modules/auth/index';
|
|
|
34
34
|
import { StreamProcessorService } from './services/stream-processor/index';
|
|
35
35
|
import { EventSystem } from './events/index';
|
|
36
36
|
import { CacheModule } from './modules/cache/index';
|
|
37
|
+
import { CrdtManager, CrdtField } from './modules/crdt/index';
|
|
37
38
|
import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
|
|
38
39
|
import { generateId, parseParams } from './utils/index';
|
|
39
40
|
import { SurrealDBPersistenceClient } from './services/persistence/surrealdb';
|
|
@@ -90,6 +91,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
90
91
|
private dataModule: DataModule<S>;
|
|
91
92
|
private sync: Sp00kySync<S>;
|
|
92
93
|
private devTools: DevToolsService;
|
|
94
|
+
private crdtManager: CrdtManager;
|
|
93
95
|
|
|
94
96
|
private logger: ReturnType<typeof createLogger>;
|
|
95
97
|
public auth: AuthService<S>;
|
|
@@ -154,6 +156,9 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
154
156
|
logger
|
|
155
157
|
);
|
|
156
158
|
|
|
159
|
+
// Initialize CRDT Manager (needs remote for _00_crdt LIVE SELECT)
|
|
160
|
+
this.crdtManager = new CrdtManager(this.config.schema, this.remote, logger);
|
|
161
|
+
|
|
157
162
|
this.dataModule = new DataModule(
|
|
158
163
|
this.cache,
|
|
159
164
|
this.local,
|
|
@@ -275,6 +280,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
275
280
|
}
|
|
276
281
|
|
|
277
282
|
async close() {
|
|
283
|
+
this.crdtManager.closeAll();
|
|
278
284
|
await this.local.close();
|
|
279
285
|
await this.remote.close();
|
|
280
286
|
}
|
|
@@ -283,6 +289,27 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
283
289
|
return this.remote.getClient().authenticate(token);
|
|
284
290
|
}
|
|
285
291
|
|
|
292
|
+
/**
|
|
293
|
+
* Open a CRDT field for collaborative editing.
|
|
294
|
+
* Returns a CrdtField with a LoroDoc that can be bound to any editor.
|
|
295
|
+
* Also starts a LIVE SELECT on _00_crdt for real-time sync.
|
|
296
|
+
*/
|
|
297
|
+
async openCrdtField(
|
|
298
|
+
table: string,
|
|
299
|
+
recordId: string,
|
|
300
|
+
field: string,
|
|
301
|
+
fallbackText?: string,
|
|
302
|
+
): Promise<CrdtField> {
|
|
303
|
+
return this.crdtManager.open(table, recordId, field, fallbackText);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Close a CRDT field when editing is done.
|
|
308
|
+
*/
|
|
309
|
+
closeCrdtField(table: string, recordId: string, field: string): void {
|
|
310
|
+
this.crdtManager.close(table, recordId, field);
|
|
311
|
+
}
|
|
312
|
+
|
|
286
313
|
deauthenticate() {
|
|
287
314
|
return this.remote.getClient().invalidate();
|
|
288
315
|
}
|
package/src/utils/index.ts
CHANGED
|
@@ -137,6 +137,18 @@ export async function fileToUint8Array(file: File | Blob): Promise<Uint8Array> {
|
|
|
137
137
|
return new Uint8Array(buffer);
|
|
138
138
|
}
|
|
139
139
|
|
|
140
|
+
// ==================== TEXT UTILITIES ====================
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Convert plain text to simple HTML paragraphs.
|
|
144
|
+
* Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
|
|
145
|
+
*/
|
|
146
|
+
export function textToHtml(text: string): string {
|
|
147
|
+
return text
|
|
148
|
+
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
149
|
+
.split('\n').map((l) => `<p>${l || '<br>'}</p>`).join('');
|
|
150
|
+
}
|
|
151
|
+
|
|
140
152
|
// ==================== DATABASE UTILITIES ====================
|
|
141
153
|
|
|
142
154
|
/**
|
package/src/utils/parser.ts
CHANGED
|
@@ -9,7 +9,7 @@ export function cleanRecord(
|
|
|
9
9
|
): Record<string, any> {
|
|
10
10
|
const cleaned: Record<string, any> = {};
|
|
11
11
|
for (const [key, value] of Object.entries(record)) {
|
|
12
|
-
if (key === 'id' || key in tableSchema) {
|
|
12
|
+
if (key === 'id' || key.startsWith('_00_') || key in tableSchema) {
|
|
13
13
|
cleaned[key] = value;
|
|
14
14
|
}
|
|
15
15
|
}
|