@spooky-sync/core 0.0.1-canary.53 → 0.0.1-canary.54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -3,6 +3,7 @@ import * as surrealdb0 from "surrealdb";
3
3
  import { Duration, RecordId, Surreal as Surreal$1, SurrealTransaction } from "surrealdb";
4
4
  import { AccessDefinition, BackendNames, BackendRoutes, BucketNames, ColumnSchema, GetTable, QueryBuilder, QueryOptions, RoutePayload, SchemaStructure, TableModel, TableNames, TypeNameToTypeMap } from "@spooky-sync/query-builder";
5
5
  import { Logger } from "pino";
6
+ import { LoroDoc } from "loro-crdt";
6
7
 
7
8
  //#region src/services/database/events/index.d.ts
8
9
  declare const DatabaseEventTypes: {
@@ -194,6 +195,74 @@ declare class AuthService<S extends SchemaStructure> {
194
195
  signIn<Name extends keyof S['access'] & string>(accessName: Name, params: ExtractAccessParams<S, Name, 'signIn'>): Promise<void>;
195
196
  }
196
197
  //#endregion
198
+ //#region src/modules/crdt/crdt-field.d.ts
199
+ declare const CURSOR_COLORS: string[];
200
+ declare function cursorColorFromName(name: string): string;
201
+ declare class CrdtField {
202
+ private fieldName;
203
+ private doc;
204
+ private pushTimer;
205
+ private remote;
206
+ private recordId;
207
+ private unsubscribe;
208
+ private lastPushTime;
209
+ private lastCursorPushTime;
210
+ private loadedFromCrdt;
211
+ private pushRetryCount;
212
+ private logger;
213
+ private _onCursorUpdate;
214
+ private pendingCursorUpdate;
215
+ /** Callback set by the editor to receive remote cursor updates.
216
+ * Any cursor data that arrived before this callback was set will be replayed. */
217
+ set onCursorUpdate(cb: ((data: Uint8Array) => void) | null);
218
+ get onCursorUpdate(): ((data: Uint8Array) => void) | null;
219
+ constructor(fieldName: string, initialState?: string, logger?: Logger$1 | null);
220
+ getDoc(): LoroDoc;
221
+ /** Whether the LoroDoc was loaded from saved CRDT state */
222
+ hasContent(): boolean;
223
+ startSync(remote: RemoteDatabaseService, recordId: string): void;
224
+ stopSync(): void;
225
+ importRemote(base64State: string): void;
226
+ exportSnapshot(): string;
227
+ /** Push cursor ephemeral state to _00_crdt as a "_cursor_<fieldName>" entry */
228
+ pushCursorState(encoded: Uint8Array): Promise<void>;
229
+ /** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
230
+ importRemoteCursor(base64State: string): void;
231
+ private schedulePush;
232
+ private pushToRemote;
233
+ }
234
+ //#endregion
235
+ //#region src/modules/crdt/index.d.ts
236
+ /**
237
+ * CrdtManager manages active CrdtField instances and their sync channels.
238
+ *
239
+ * Each open record gets a LIVE SELECT on _00_crdt that delivers remote
240
+ * changes in real time.
241
+ */
242
+ declare class CrdtManager {
243
+ private schema;
244
+ private remote;
245
+ private fields;
246
+ private liveQueries;
247
+ private logger;
248
+ constructor(schema: SchemaStructure, remote: RemoteDatabaseService, logger: Logger$1);
249
+ /**
250
+ * Open a CRDT field for collaborative editing.
251
+ *
252
+ * @param table - Table name
253
+ * @param recordId - Full record ID (e.g., "thread:abc")
254
+ * @param field - Field name (e.g., "title", "content")
255
+ * @param fallbackText - Current plain text from the record, used to seed the
256
+ * LoroDoc if no CRDT state exists yet (migration path)
257
+ */
258
+ open(table: string, recordId: string, field: string, fallbackText?: string): Promise<CrdtField>;
259
+ close(table: string, recordId: string, field: string): void;
260
+ closeAll(): void;
261
+ private ensureLiveSelect;
262
+ private killLiveSelect;
263
+ private makeKey;
264
+ }
265
+ //#endregion
197
266
  //#region src/sp00ky.d.ts
198
267
  declare class BucketHandle {
199
268
  private bucketName;
@@ -218,6 +287,7 @@ declare class Sp00kyClient<S extends SchemaStructure> {
218
287
  private dataModule;
219
288
  private sync;
220
289
  private devTools;
290
+ private crdtManager;
221
291
  private logger;
222
292
  auth: AuthService<S>;
223
293
  streamProcessor: StreamProcessorService;
@@ -233,6 +303,16 @@ declare class Sp00kyClient<S extends SchemaStructure> {
233
303
  init(): Promise<void>;
234
304
  close(): Promise<void>;
235
305
  authenticate(token: string): Promise<surrealdb0.Tokens>;
306
+ /**
307
+ * Open a CRDT field for collaborative editing.
308
+ * Returns a CrdtField with a LoroDoc that can be bound to any editor.
309
+ * Also starts a LIVE SELECT on _00_crdt for real-time sync.
310
+ */
311
+ openCrdtField(table: string, recordId: string, field: string, fallbackText?: string): Promise<CrdtField>;
312
+ /**
313
+ * Close a CRDT field when editing is done.
314
+ */
315
+ closeCrdtField(table: string, recordId: string, field: string): void;
236
316
  deauthenticate(): Promise<void>;
237
317
  query<Table extends TableNames<S>>(table: Table, options: QueryOptions<TableModel<GetTable<S, Table>>, false>, ttl?: QueryTimeToLive): QueryBuilder<S, Table, Sp00kyQueryResultPromise>;
238
318
  private initQuery;
@@ -254,9 +334,14 @@ declare class Sp00kyClient<S extends SchemaStructure> {
254
334
  //#endregion
255
335
  //#region src/utils/index.d.ts
256
336
  declare function fileToUint8Array(file: File | Blob): Promise<Uint8Array>;
337
+ /**
338
+ * Convert plain text to simple HTML paragraphs.
339
+ * Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
340
+ */
341
+ declare function textToHtml(text: string): string;
257
342
  /**
258
343
  * Helper for retrying DB operations with exponential backoff
259
344
  */
260
345
 
261
346
  //#endregion
262
- export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, DebounceOptions, EventSubscriptionOptions, Level, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryTimeToLive, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, UpdateOptions, createAuthEventSystem, fileToUint8Array };
347
+ export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, Level, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryTimeToLive, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
package/dist/index.js CHANGED
@@ -3,6 +3,7 @@ import { QueryBuilder, RecordId as RecordId$1 } from "@spooky-sync/query-builder
3
3
  import { createWasmWorkerEngines } from "@surrealdb/wasm";
4
4
  import pino from "pino";
5
5
  import init, { Sp00kyProcessor } from "@spooky-sync/ssp-wasm";
6
+ import { LoroDoc } from "loro-crdt";
6
7
 
7
8
  //#region src/utils/surql.ts
8
9
  const surql = {
@@ -71,7 +72,7 @@ const surql = {
71
72
  //#region src/utils/parser.ts
72
73
  function cleanRecord(tableSchema, record) {
73
74
  const cleaned = {};
74
- for (const [key, value] of Object.entries(record)) if (key === "id" || key in tableSchema) cleaned[key] = value;
75
+ for (const [key, value] of Object.entries(record)) if (key === "id" || key.startsWith("_00_") || key in tableSchema) cleaned[key] = value;
75
76
  return cleaned;
76
77
  }
77
78
  function parseParams(tableSchema, params) {
@@ -169,6 +170,13 @@ async function fileToUint8Array(file) {
169
170
  return new Uint8Array(buffer);
170
171
  }
171
172
  /**
173
+ * Convert plain text to simple HTML paragraphs.
174
+ * Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
175
+ */
176
+ function textToHtml(text) {
177
+ return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").split("\n").map((l) => `<p>${l || "<br>"}</p>`).join("");
178
+ }
179
+ /**
172
180
  * Helper for retrying DB operations with exponential backoff
173
181
  */
174
182
  async function withRetry(logger, operation, retries = 3, delayMs = 100) {
@@ -1867,7 +1875,7 @@ var Sp00kySync = class {
1867
1875
  Category: "sp00ky-client::Sp00kySync::processUpEvent"
1868
1876
  }, "Processing up event");
1869
1877
  switch (event.type) {
1870
- case "create":
1878
+ case "create": {
1871
1879
  const dataKeys = Object.keys(event.data).map((key) => ({
1872
1880
  key,
1873
1881
  variable: `data_${key}`
@@ -1879,6 +1887,7 @@ var Sp00kySync = class {
1879
1887
  ...prefixedParams
1880
1888
  });
1881
1889
  break;
1890
+ }
1882
1891
  case "update":
1883
1892
  await this.remote.query(`UPDATE $id MERGE $data`, {
1884
1893
  id: event.record_id,
@@ -2849,6 +2858,305 @@ var CacheModule = class {
2849
2858
  }
2850
2859
  };
2851
2860
 
2861
+ //#endregion
2862
+ //#region src/modules/crdt/crdt-field.ts
2863
+ const CURSOR_COLORS = [
2864
+ "#3b82f6",
2865
+ "#ef4444",
2866
+ "#22c55e",
2867
+ "#f59e0b",
2868
+ "#8b5cf6",
2869
+ "#ec4899",
2870
+ "#14b8a6",
2871
+ "#f97316"
2872
+ ];
2873
+ function cursorColorFromName(name) {
2874
+ let hash = 0;
2875
+ for (let i = 0; i < name.length; i++) hash = (hash << 5) - hash + name.charCodeAt(i) | 0;
2876
+ return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
2877
+ }
2878
+ var CrdtField = class {
2879
+ doc;
2880
+ pushTimer = null;
2881
+ remote = null;
2882
+ recordId = null;
2883
+ unsubscribe = null;
2884
+ lastPushTime = 0;
2885
+ lastCursorPushTime = 0;
2886
+ loadedFromCrdt = false;
2887
+ pushRetryCount = 0;
2888
+ logger;
2889
+ _onCursorUpdate = null;
2890
+ pendingCursorUpdate = null;
2891
+ /** Callback set by the editor to receive remote cursor updates.
2892
+ * Any cursor data that arrived before this callback was set will be replayed. */
2893
+ set onCursorUpdate(cb) {
2894
+ this._onCursorUpdate = cb;
2895
+ if (cb && this.pendingCursorUpdate) {
2896
+ try {
2897
+ cb(this.pendingCursorUpdate);
2898
+ } catch (e) {
2899
+ this.logger?.warn({
2900
+ error: e,
2901
+ Category: "sp00ky-client::CrdtField::onCursorUpdate"
2902
+ }, "Failed to replay pending cursor update");
2903
+ }
2904
+ this.pendingCursorUpdate = null;
2905
+ }
2906
+ }
2907
+ get onCursorUpdate() {
2908
+ return this._onCursorUpdate;
2909
+ }
2910
+ constructor(fieldName, initialState, logger) {
2911
+ this.fieldName = fieldName;
2912
+ this.logger = logger ?? null;
2913
+ this.doc = new LoroDoc();
2914
+ if (initialState) {
2915
+ this.doc.import(decodeBase64(initialState));
2916
+ this.loadedFromCrdt = true;
2917
+ }
2918
+ }
2919
+ getDoc() {
2920
+ return this.doc;
2921
+ }
2922
+ /** Whether the LoroDoc was loaded from saved CRDT state */
2923
+ hasContent() {
2924
+ return this.loadedFromCrdt;
2925
+ }
2926
+ startSync(remote, recordId) {
2927
+ this.remote = remote;
2928
+ this.recordId = recordId;
2929
+ this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
2930
+ this.schedulePush();
2931
+ });
2932
+ }
2933
+ stopSync() {
2934
+ if (this.unsubscribe) {
2935
+ this.unsubscribe();
2936
+ this.unsubscribe = null;
2937
+ }
2938
+ if (this.pushTimer) {
2939
+ clearTimeout(this.pushTimer);
2940
+ this.pushTimer = null;
2941
+ }
2942
+ if (this.remote && this.recordId) this.pushToRemote();
2943
+ }
2944
+ importRemote(base64State) {
2945
+ if (Date.now() - this.lastPushTime < 500) return;
2946
+ try {
2947
+ this.doc.import(decodeBase64(base64State));
2948
+ } catch (e) {
2949
+ this.logger?.warn({
2950
+ error: e,
2951
+ Category: "sp00ky-client::CrdtField::importRemote"
2952
+ }, "Failed to import remote CRDT state");
2953
+ }
2954
+ }
2955
+ exportSnapshot() {
2956
+ return encodeBase64(this.doc.export({ mode: "snapshot" }));
2957
+ }
2958
+ /** Push cursor ephemeral state to _00_crdt as a "_cursor_<fieldName>" entry */
2959
+ async pushCursorState(encoded) {
2960
+ if (!this.remote || !this.recordId) return;
2961
+ this.lastCursorPushTime = Date.now();
2962
+ try {
2963
+ const state = encodeBase64(encoded);
2964
+ await this.remote.query(`INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
2965
+ ON DUPLICATE KEY UPDATE state = $state`, {
2966
+ rid: parseRecordIdString(this.recordId),
2967
+ field: `_cursor_${this.fieldName}`,
2968
+ state
2969
+ });
2970
+ } catch (e) {
2971
+ this.logger?.warn({
2972
+ error: e,
2973
+ Category: "sp00ky-client::CrdtField::pushCursorState"
2974
+ }, "Failed to push cursor state");
2975
+ }
2976
+ }
2977
+ /** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
2978
+ importRemoteCursor(base64State) {
2979
+ if (Date.now() - this.lastCursorPushTime < 300) return;
2980
+ try {
2981
+ const data = decodeBase64(base64State);
2982
+ if (this._onCursorUpdate) this._onCursorUpdate(data);
2983
+ else this.pendingCursorUpdate = data;
2984
+ } catch (e) {
2985
+ this.logger?.warn({
2986
+ error: e,
2987
+ Category: "sp00ky-client::CrdtField::importRemoteCursor"
2988
+ }, "Failed to apply remote cursor data");
2989
+ }
2990
+ }
2991
+ schedulePush() {
2992
+ if (this.pushTimer) clearTimeout(this.pushTimer);
2993
+ this.pushTimer = setTimeout(() => void this.pushToRemote(), 300);
2994
+ }
2995
+ async pushToRemote() {
2996
+ if (!this.remote || !this.recordId) return;
2997
+ this.lastPushTime = Date.now();
2998
+ try {
2999
+ await this.remote.query(`INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
3000
+ ON DUPLICATE KEY UPDATE state = $state`, {
3001
+ rid: parseRecordIdString(this.recordId),
3002
+ field: this.fieldName,
3003
+ state: this.exportSnapshot()
3004
+ });
3005
+ this.pushRetryCount = 0;
3006
+ } catch (e) {
3007
+ this.logger?.warn({
3008
+ error: e,
3009
+ Category: "sp00ky-client::CrdtField::pushToRemote"
3010
+ }, "Failed to push CRDT state to remote");
3011
+ if (this.pushRetryCount < 2) {
3012
+ this.pushRetryCount++;
3013
+ this.schedulePush();
3014
+ }
3015
+ }
3016
+ }
3017
+ };
3018
+ function decodeBase64(b64) {
3019
+ if (typeof atob === "function") {
3020
+ const binary = atob(b64);
3021
+ const bytes = new Uint8Array(binary.length);
3022
+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
3023
+ return bytes;
3024
+ }
3025
+ return new Uint8Array(Buffer.from(b64, "base64"));
3026
+ }
3027
+ function encodeBase64(bytes) {
3028
+ if (typeof btoa === "function") {
3029
+ let binary = "";
3030
+ for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
3031
+ return btoa(binary);
3032
+ }
3033
+ return Buffer.from(bytes).toString("base64");
3034
+ }
3035
+
3036
+ //#endregion
3037
+ //#region src/modules/crdt/index.ts
3038
+ /**
3039
+ * CrdtManager manages active CrdtField instances and their sync channels.
3040
+ *
3041
+ * Each open record gets a LIVE SELECT on _00_crdt that delivers remote
3042
+ * changes in real time.
3043
+ */
3044
+ var CrdtManager = class {
3045
+ fields = /* @__PURE__ */ new Map();
3046
+ liveQueries = /* @__PURE__ */ new Map();
3047
+ logger;
3048
+ constructor(schema, remote, logger) {
3049
+ this.schema = schema;
3050
+ this.remote = remote;
3051
+ this.logger = logger.child({ service: "CrdtManager" });
3052
+ }
3053
+ /**
3054
+ * Open a CRDT field for collaborative editing.
3055
+ *
3056
+ * @param table - Table name
3057
+ * @param recordId - Full record ID (e.g., "thread:abc")
3058
+ * @param field - Field name (e.g., "title", "content")
3059
+ * @param fallbackText - Current plain text from the record, used to seed the
3060
+ * LoroDoc if no CRDT state exists yet (migration path)
3061
+ */
3062
+ async open(table, recordId, field, fallbackText) {
3063
+ const key = this.makeKey(table, recordId, field);
3064
+ let crdtField = this.fields.get(key);
3065
+ if (crdtField) return crdtField;
3066
+ let initialCrdtState;
3067
+ try {
3068
+ const [result] = await this.remote.query("SELECT VALUE state FROM _00_crdt WHERE record_id = $rid AND field = $field LIMIT 1", {
3069
+ rid: parseRecordIdString(recordId),
3070
+ field
3071
+ });
3072
+ if (result && result.length > 0 && result[0]) initialCrdtState = result[0];
3073
+ } catch (e) {
3074
+ this.logger.debug({
3075
+ error: e,
3076
+ Category: "sp00ky-client::CrdtManager::open"
3077
+ }, "No existing CRDT state found");
3078
+ }
3079
+ crdtField = new CrdtField(field, initialCrdtState, this.logger);
3080
+ crdtField.startSync(this.remote, recordId);
3081
+ this.fields.set(key, crdtField);
3082
+ this.logger.info({
3083
+ key,
3084
+ hasInitialState: !!initialCrdtState,
3085
+ hasFallback: !!fallbackText,
3086
+ Category: "sp00ky-client::CrdtManager::open"
3087
+ }, "CrdtField opened");
3088
+ await this.ensureLiveSelect(table, recordId);
3089
+ return crdtField;
3090
+ }
3091
+ close(table, recordId, field) {
3092
+ const key = this.makeKey(table, recordId, field);
3093
+ const crdtField = this.fields.get(key);
3094
+ if (crdtField) {
3095
+ crdtField.stopSync();
3096
+ this.fields.delete(key);
3097
+ }
3098
+ const prefix = `${table}:${recordId}:`;
3099
+ if (!Array.from(this.fields.keys()).some((k) => k !== key && k.startsWith(prefix))) this.killLiveSelect(recordId);
3100
+ this.logger.debug({
3101
+ key,
3102
+ Category: "sp00ky-client::CrdtManager::close"
3103
+ }, "CrdtField closed");
3104
+ }
3105
+ closeAll() {
3106
+ for (const [_, field] of this.fields) field.stopSync();
3107
+ this.fields.clear();
3108
+ for (const recordId of this.liveQueries.keys()) this.killLiveSelect(recordId);
3109
+ }
3110
+ async ensureLiveSelect(table, recordId) {
3111
+ if (this.liveQueries.has(recordId)) return;
3112
+ try {
3113
+ const [uuid] = await this.remote.query(`LIVE SELECT * FROM _00_crdt WHERE record_id = $rid`, { rid: parseRecordIdString(recordId) });
3114
+ this.liveQueries.set(recordId, {
3115
+ uuid,
3116
+ table
3117
+ });
3118
+ (await this.remote.getClient().liveOf(uuid)).subscribe((message) => {
3119
+ if (message.action === "KILLED") return;
3120
+ if (message.action === "CREATE" || message.action === "UPDATE") {
3121
+ const fieldName = message.value.field;
3122
+ const state = message.value.state;
3123
+ if (!fieldName || !state) return;
3124
+ if (fieldName.startsWith("_cursor_")) {
3125
+ const actualField = fieldName.slice(8);
3126
+ const key = this.makeKey(table, recordId, actualField);
3127
+ const crdtField = this.fields.get(key);
3128
+ if (crdtField) crdtField.importRemoteCursor(state);
3129
+ return;
3130
+ }
3131
+ const key = this.makeKey(table, recordId, fieldName);
3132
+ const crdtField = this.fields.get(key);
3133
+ if (crdtField) crdtField.importRemote(state);
3134
+ }
3135
+ });
3136
+ this.logger.info({
3137
+ recordId,
3138
+ Category: "sp00ky-client::CrdtManager::ensureLiveSelect"
3139
+ }, "LIVE SELECT on _00_crdt started");
3140
+ } catch (e) {
3141
+ this.logger.warn({
3142
+ error: e,
3143
+ recordId,
3144
+ Category: "sp00ky-client::CrdtManager::ensureLiveSelect"
3145
+ }, "Failed to start LIVE SELECT on _00_crdt");
3146
+ }
3147
+ }
3148
+ killLiveSelect(recordId) {
3149
+ const entry = this.liveQueries.get(recordId);
3150
+ if (entry) {
3151
+ this.remote.query("KILL $uuid", { uuid: entry.uuid }).catch(() => {});
3152
+ this.liveQueries.delete(recordId);
3153
+ }
3154
+ }
3155
+ makeKey(table, recordId, field) {
3156
+ return `${table}:${recordId}:${field}`;
3157
+ }
3158
+ };
3159
+
2852
3160
  //#endregion
2853
3161
  //#region src/services/persistence/localstorage.ts
2854
3162
  var LocalStoragePersistenceClient = class {
@@ -2996,6 +3304,7 @@ var Sp00kyClient = class {
2996
3304
  dataModule;
2997
3305
  sync;
2998
3306
  devTools;
3307
+ crdtManager;
2999
3308
  logger;
3000
3309
  auth;
3001
3310
  streamProcessor;
@@ -3033,6 +3342,7 @@ var Sp00kyClient = class {
3033
3342
  this.cache = new CacheModule(this.local, this.streamProcessor, (update) => {
3034
3343
  this.dataModule.onStreamUpdate(update);
3035
3344
  }, logger);
3345
+ this.crdtManager = new CrdtManager(this.config.schema, this.remote, logger);
3036
3346
  this.dataModule = new DataModule(this.cache, this.local, this.config.schema, logger, this.config.streamDebounceTime);
3037
3347
  this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
3038
3348
  this.sync = new Sp00kySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger);
@@ -3091,12 +3401,27 @@ var Sp00kyClient = class {
3091
3401
  }
3092
3402
  }
3093
3403
  async close() {
3404
+ this.crdtManager.closeAll();
3094
3405
  await this.local.close();
3095
3406
  await this.remote.close();
3096
3407
  }
3097
3408
  authenticate(token) {
3098
3409
  return this.remote.getClient().authenticate(token);
3099
3410
  }
3411
+ /**
3412
+ * Open a CRDT field for collaborative editing.
3413
+ * Returns a CrdtField with a LoroDoc that can be bound to any editor.
3414
+ * Also starts a LIVE SELECT on _00_crdt for real-time sync.
3415
+ */
3416
+ async openCrdtField(table, recordId, field, fallbackText) {
3417
+ return this.crdtManager.open(table, recordId, field, fallbackText);
3418
+ }
3419
+ /**
3420
+ * Close a CRDT field when editing is done.
3421
+ */
3422
+ closeCrdtField(table, recordId, field) {
3423
+ this.crdtManager.close(table, recordId, field);
3424
+ }
3100
3425
  deauthenticate() {
3101
3426
  return this.remote.getClient().invalidate();
3102
3427
  }
@@ -3158,4 +3483,4 @@ var Sp00kyClient = class {
3158
3483
  };
3159
3484
 
3160
3485
  //#endregion
3161
- export { AuthEventTypes, AuthService, BucketHandle, Sp00kyClient, createAuthEventSystem, fileToUint8Array };
3486
+ export { AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, Sp00kyClient, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@spooky-sync/core",
3
- "version": "0.0.1-canary.53",
3
+ "version": "0.0.1-canary.54",
4
4
  "type": "module",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",
@@ -63,6 +63,7 @@
63
63
  "@spooky-sync/ssp-wasm": "workspace:*",
64
64
  "@surrealdb/wasm": "^3.0.0",
65
65
  "fast-json-patch": "^3.1.1",
66
+ "loro-crdt": "^1.5.6",
66
67
  "pino": "^10.1.0",
67
68
  "surrealdb": "2.0.0"
68
69
  },
package/src/index.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  export * from './types';
2
2
  export * from './sp00ky';
3
3
  export * from './modules/auth/index';
4
- export { fileToUint8Array } from './utils/index';
4
+ export { CrdtField, CrdtManager, cursorColorFromName, CURSOR_COLORS } from './modules/crdt/index';
5
+ export { fileToUint8Array, textToHtml } from './utils/index';
@@ -0,0 +1,189 @@
1
+ import { LoroDoc } from 'loro-crdt';
2
+ import type { RemoteDatabaseService } from '../../services/database/index';
3
+ import type { Logger } from '../../services/logger/index';
4
+ import { parseRecordIdString } from '../../utils/index';
5
+
6
+ // ==================== CURSOR UTILITIES ====================
7
+
8
+ export const CURSOR_COLORS = [
9
+ '#3b82f6', '#ef4444', '#22c55e', '#f59e0b',
10
+ '#8b5cf6', '#ec4899', '#14b8a6', '#f97316',
11
+ ];
12
+
13
+ export function cursorColorFromName(name: string): string {
14
+ let hash = 0;
15
+ for (let i = 0; i < name.length; i++) {
16
+ hash = ((hash << 5) - hash + name.charCodeAt(i)) | 0;
17
+ }
18
+ return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
19
+ }
20
+
21
+ // ==================== CRDT FIELD ====================
22
+
23
+ export class CrdtField {
24
+ private doc: LoroDoc;
25
+ private pushTimer: ReturnType<typeof setTimeout> | null = null;
26
+ private remote: RemoteDatabaseService | null = null;
27
+ private recordId: string | null = null;
28
+ private unsubscribe: (() => void) | null = null;
29
+ private lastPushTime = 0;
30
+ private lastCursorPushTime = 0;
31
+ private loadedFromCrdt = false;
32
+ private pushRetryCount = 0;
33
+ private logger: Logger | null;
34
+
35
+ private _onCursorUpdate: ((data: Uint8Array) => void) | null = null;
36
+ private pendingCursorUpdate: Uint8Array | null = null;
37
+
38
+ /** Callback set by the editor to receive remote cursor updates.
39
+ * Any cursor data that arrived before this callback was set will be replayed. */
40
+ set onCursorUpdate(cb: ((data: Uint8Array) => void) | null) {
41
+ this._onCursorUpdate = cb;
42
+ if (cb && this.pendingCursorUpdate) {
43
+ try { cb(this.pendingCursorUpdate); } catch (e) {
44
+ this.logger?.warn(
45
+ { error: e, Category: 'sp00ky-client::CrdtField::onCursorUpdate' },
46
+ 'Failed to replay pending cursor update'
47
+ );
48
+ }
49
+ this.pendingCursorUpdate = null;
50
+ }
51
+ }
52
+
53
+ get onCursorUpdate() { return this._onCursorUpdate; }
54
+
55
+ constructor(
56
+ private fieldName: string,
57
+ initialState?: string,
58
+ logger?: Logger | null,
59
+ ) {
60
+ this.logger = logger ?? null;
61
+ this.doc = new LoroDoc();
62
+ if (initialState) {
63
+ this.doc.import(decodeBase64(initialState));
64
+ this.loadedFromCrdt = true;
65
+ }
66
+ }
67
+
68
+ getDoc(): LoroDoc { return this.doc; }
69
+
70
+ /** Whether the LoroDoc was loaded from saved CRDT state */
71
+ hasContent(): boolean {
72
+ return this.loadedFromCrdt;
73
+ }
74
+
75
+ startSync(remote: RemoteDatabaseService, recordId: string): void {
76
+ this.remote = remote;
77
+ this.recordId = recordId;
78
+ this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
79
+ this.schedulePush();
80
+ });
81
+ }
82
+
83
+ stopSync(): void {
84
+ if (this.unsubscribe) { this.unsubscribe(); this.unsubscribe = null; }
85
+ if (this.pushTimer) { clearTimeout(this.pushTimer); this.pushTimer = null; }
86
+ if (this.remote && this.recordId) { void this.pushToRemote(); }
87
+ }
88
+
89
+ importRemote(base64State: string): void {
90
+ // Suppress echo: skip imports within 500ms of our own push
91
+ if (Date.now() - this.lastPushTime < 500) return;
92
+ try {
93
+ this.doc.import(decodeBase64(base64State));
94
+ } catch (e) {
95
+ this.logger?.warn(
96
+ { error: e, Category: 'sp00ky-client::CrdtField::importRemote' },
97
+ 'Failed to import remote CRDT state'
98
+ );
99
+ }
100
+ }
101
+
102
+ exportSnapshot(): string {
103
+ return encodeBase64(this.doc.export({ mode: 'snapshot' }));
104
+ }
105
+
106
+ /** Push cursor ephemeral state to _00_crdt as a "_cursor_<fieldName>" entry */
107
+ async pushCursorState(encoded: Uint8Array): Promise<void> {
108
+ if (!this.remote || !this.recordId) return;
109
+ this.lastCursorPushTime = Date.now();
110
+ try {
111
+ const state = encodeBase64(encoded);
112
+ await this.remote.query(
113
+ `INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
114
+ ON DUPLICATE KEY UPDATE state = $state`,
115
+ { rid: parseRecordIdString(this.recordId), field: `_cursor_${this.fieldName}`, state }
116
+ );
117
+ } catch (e) {
118
+ this.logger?.warn(
119
+ { error: e, Category: 'sp00ky-client::CrdtField::pushCursorState' },
120
+ 'Failed to push cursor state'
121
+ );
122
+ }
123
+ }
124
+
125
+ /** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
126
+ importRemoteCursor(base64State: string): void {
127
+ if (Date.now() - this.lastCursorPushTime < 300) return; // echo suppression
128
+ try {
129
+ const data = decodeBase64(base64State);
130
+ if (this._onCursorUpdate) {
131
+ this._onCursorUpdate(data);
132
+ } else {
133
+ // Only keep the latest cursor state — older positions are useless
134
+ this.pendingCursorUpdate = data;
135
+ }
136
+ } catch (e) {
137
+ this.logger?.warn(
138
+ { error: e, Category: 'sp00ky-client::CrdtField::importRemoteCursor' },
139
+ 'Failed to apply remote cursor data'
140
+ );
141
+ }
142
+ }
143
+
144
+ private schedulePush(): void {
145
+ if (this.pushTimer) clearTimeout(this.pushTimer);
146
+ this.pushTimer = setTimeout(() => void this.pushToRemote(), 300);
147
+ }
148
+
149
+ private async pushToRemote(): Promise<void> {
150
+ if (!this.remote || !this.recordId) return;
151
+ this.lastPushTime = Date.now();
152
+ try {
153
+ await this.remote.query(
154
+ `INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
155
+ ON DUPLICATE KEY UPDATE state = $state`,
156
+ { rid: parseRecordIdString(this.recordId), field: this.fieldName, state: this.exportSnapshot() }
157
+ );
158
+ this.pushRetryCount = 0;
159
+ } catch (e) {
160
+ this.logger?.warn(
161
+ { error: e, Category: 'sp00ky-client::CrdtField::pushToRemote' },
162
+ 'Failed to push CRDT state to remote'
163
+ );
164
+ if (this.pushRetryCount < 2) {
165
+ this.pushRetryCount++;
166
+ this.schedulePush();
167
+ }
168
+ }
169
+ }
170
+ }
171
+
172
+ export function decodeBase64(b64: string): Uint8Array {
173
+ if (typeof atob === 'function') {
174
+ const binary = atob(b64);
175
+ const bytes = new Uint8Array(binary.length);
176
+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
177
+ return bytes;
178
+ }
179
+ return new Uint8Array(Buffer.from(b64, 'base64'));
180
+ }
181
+
182
+ export function encodeBase64(bytes: Uint8Array): string {
183
+ if (typeof btoa === 'function') {
184
+ let binary = '';
185
+ for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
186
+ return btoa(binary);
187
+ }
188
+ return Buffer.from(bytes).toString('base64');
189
+ }
@@ -0,0 +1,178 @@
1
+ import type { SchemaStructure } from '@spooky-sync/query-builder';
2
+ import type { RemoteDatabaseService } from '../../services/database/index';
3
+ import type { Logger } from '../../services/logger/index';
4
+ import type { Uuid } from 'surrealdb';
5
+ import { CrdtField } from './crdt-field';
6
+ import { parseRecordIdString } from '../../utils/index';
7
+
8
+ export { CrdtField, cursorColorFromName, CURSOR_COLORS } from './crdt-field';
9
+
10
+ /**
11
+ * CrdtManager manages active CrdtField instances and their sync channels.
12
+ *
13
+ * Each open record gets a LIVE SELECT on _00_crdt that delivers remote
14
+ * changes in real time.
15
+ */
16
+ export class CrdtManager {
17
+ private fields = new Map<string, CrdtField>();
18
+ private liveQueries = new Map<string, { uuid: Uuid; table: string }>();
19
+ private logger: Logger;
20
+
21
+ constructor(
22
+ private schema: SchemaStructure,
23
+ private remote: RemoteDatabaseService,
24
+ logger: Logger
25
+ ) {
26
+ this.logger = logger.child({ service: 'CrdtManager' });
27
+ }
28
+
29
+ /**
30
+ * Open a CRDT field for collaborative editing.
31
+ *
32
+ * @param table - Table name
33
+ * @param recordId - Full record ID (e.g., "thread:abc")
34
+ * @param field - Field name (e.g., "title", "content")
35
+ * @param fallbackText - Current plain text from the record, used to seed the
36
+ * LoroDoc if no CRDT state exists yet (migration path)
37
+ */
38
+ async open(
39
+ table: string,
40
+ recordId: string,
41
+ field: string,
42
+ fallbackText?: string,
43
+ ): Promise<CrdtField> {
44
+ const key = this.makeKey(table, recordId, field);
45
+ let crdtField = this.fields.get(key);
46
+
47
+ if (crdtField) {
48
+ return crdtField;
49
+ }
50
+
51
+ // Load saved CRDT state from remote _00_crdt table
52
+ let initialCrdtState: string | undefined;
53
+ try {
54
+ const [result] = await this.remote.query<[string[]]>(
55
+ 'SELECT VALUE state FROM _00_crdt WHERE record_id = $rid AND field = $field LIMIT 1',
56
+ { rid: parseRecordIdString(recordId), field }
57
+ );
58
+ if (result && result.length > 0 && result[0]) {
59
+ initialCrdtState = result[0];
60
+ }
61
+ } catch (e) {
62
+ this.logger.debug(
63
+ { error: e, Category: 'sp00ky-client::CrdtManager::open' },
64
+ 'No existing CRDT state found'
65
+ );
66
+ }
67
+
68
+ crdtField = new CrdtField(field, initialCrdtState, this.logger);
69
+ crdtField.startSync(this.remote, recordId);
70
+ this.fields.set(key, crdtField);
71
+
72
+ this.logger.info(
73
+ { key, hasInitialState: !!initialCrdtState, hasFallback: !!fallbackText, Category: 'sp00ky-client::CrdtManager::open' },
74
+ 'CrdtField opened'
75
+ );
76
+
77
+ await this.ensureLiveSelect(table, recordId);
78
+
79
+ return crdtField;
80
+ }
81
+
82
+ close(table: string, recordId: string, field: string): void {
83
+ const key = this.makeKey(table, recordId, field);
84
+ const crdtField = this.fields.get(key);
85
+ if (crdtField) {
86
+ crdtField.stopSync();
87
+ this.fields.delete(key);
88
+ }
89
+
90
+ const prefix = `${table}:${recordId}:`;
91
+ const hasOtherFields = Array.from(this.fields.keys()).some(
92
+ (k) => k !== key && k.startsWith(prefix)
93
+ );
94
+ if (!hasOtherFields) {
95
+ this.killLiveSelect(recordId);
96
+ }
97
+
98
+ this.logger.debug(
99
+ { key, Category: 'sp00ky-client::CrdtManager::close' },
100
+ 'CrdtField closed'
101
+ );
102
+ }
103
+
104
+ closeAll(): void {
105
+ for (const [_, field] of this.fields) {
106
+ field.stopSync();
107
+ }
108
+ this.fields.clear();
109
+ for (const recordId of this.liveQueries.keys()) {
110
+ this.killLiveSelect(recordId);
111
+ }
112
+ }
113
+
114
+ private async ensureLiveSelect(table: string, recordId: string): Promise<void> {
115
+ if (this.liveQueries.has(recordId)) return;
116
+
117
+ try {
118
+ const [uuid] = await this.remote.query<[Uuid]>(
119
+ `LIVE SELECT * FROM _00_crdt WHERE record_id = $rid`,
120
+ { rid: parseRecordIdString(recordId) },
121
+ );
122
+
123
+ this.liveQueries.set(recordId, { uuid, table });
124
+
125
+ const subscription = await this.remote.getClient().liveOf(uuid);
126
+ subscription.subscribe((message) => {
127
+ if (message.action === 'KILLED') return;
128
+
129
+ if (message.action === 'CREATE' || message.action === 'UPDATE') {
130
+ const fieldName = message.value.field as string;
131
+ const state = message.value.state as string;
132
+
133
+ if (!fieldName || !state) return;
134
+
135
+ // Route cursor updates to the corresponding CrdtField
136
+ if (fieldName.startsWith('_cursor_')) {
137
+ const actualField = fieldName.slice('_cursor_'.length);
138
+ const key = this.makeKey(table, recordId, actualField);
139
+ const crdtField = this.fields.get(key);
140
+ if (crdtField) {
141
+ crdtField.importRemoteCursor(state);
142
+ }
143
+ return;
144
+ }
145
+
146
+ // Route document updates
147
+ const key = this.makeKey(table, recordId, fieldName);
148
+ const crdtField = this.fields.get(key);
149
+ if (crdtField) {
150
+ crdtField.importRemote(state);
151
+ }
152
+ }
153
+ });
154
+
155
+ this.logger.info(
156
+ { recordId, Category: 'sp00ky-client::CrdtManager::ensureLiveSelect' },
157
+ 'LIVE SELECT on _00_crdt started'
158
+ );
159
+ } catch (e) {
160
+ this.logger.warn(
161
+ { error: e, recordId, Category: 'sp00ky-client::CrdtManager::ensureLiveSelect' },
162
+ 'Failed to start LIVE SELECT on _00_crdt'
163
+ );
164
+ }
165
+ }
166
+
167
+ private killLiveSelect(recordId: string): void {
168
+ const entry = this.liveQueries.get(recordId);
169
+ if (entry) {
170
+ this.remote.query('KILL $uuid', { uuid: entry.uuid }).catch(() => {});
171
+ this.liveQueries.delete(recordId);
172
+ }
173
+ }
174
+
175
+ private makeKey(table: string, recordId: string, field: string): string {
176
+ return `${table}:${recordId}:${field}`;
177
+ }
178
+ }
@@ -468,6 +468,9 @@ export class DataModule<S extends SchemaStructure> {
468
468
  const params = parseParams(tableSchema.columns, data);
469
469
  const mutationId = parseRecordIdString(`_00_pending_mutations:${Date.now()}`);
470
470
 
471
+ // Note: CRDT state is pushed directly to the _00_crdt table by CrdtField.pushToRemote(),
472
+ // NOT through the record update pipeline. This keeps the record data clean.
473
+
471
474
  // Capture current record state before mutation for rollback support
472
475
  const [beforeRecord] = await withRetry(this.logger, () =>
473
476
  this.local.query<[Record<string, any>]>('SELECT * FROM ONLY $id', { id: rid })
@@ -183,7 +183,7 @@ export class Sp00kySync<S extends SchemaStructure> {
183
183
  'Processing up event'
184
184
  );
185
185
  switch (event.type) {
186
- case 'create':
186
+ case 'create': {
187
187
  const dataKeys = Object.keys(event.data).map((key) => ({ key, variable: `data_${key}` }));
188
188
  const prefixedParams = Object.fromEntries(
189
189
  dataKeys.map(({ key, variable }) => [variable, event.data[key]])
@@ -194,6 +194,7 @@ export class Sp00kySync<S extends SchemaStructure> {
194
194
  ...prefixedParams,
195
195
  });
196
196
  break;
197
+ }
197
198
  case 'update':
198
199
  await this.remote.query(`UPDATE $id MERGE $data`, {
199
200
  id: event.record_id,
package/src/sp00ky.ts CHANGED
@@ -34,6 +34,7 @@ import { AuthService } from './modules/auth/index';
34
34
  import { StreamProcessorService } from './services/stream-processor/index';
35
35
  import { EventSystem } from './events/index';
36
36
  import { CacheModule } from './modules/cache/index';
37
+ import { CrdtManager, CrdtField } from './modules/crdt/index';
37
38
  import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
38
39
  import { generateId, parseParams } from './utils/index';
39
40
  import { SurrealDBPersistenceClient } from './services/persistence/surrealdb';
@@ -90,6 +91,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
90
91
  private dataModule: DataModule<S>;
91
92
  private sync: Sp00kySync<S>;
92
93
  private devTools: DevToolsService;
94
+ private crdtManager: CrdtManager;
93
95
 
94
96
  private logger: ReturnType<typeof createLogger>;
95
97
  public auth: AuthService<S>;
@@ -154,6 +156,9 @@ export class Sp00kyClient<S extends SchemaStructure> {
154
156
  logger
155
157
  );
156
158
 
159
+ // Initialize CRDT Manager (needs remote for _00_crdt LIVE SELECT)
160
+ this.crdtManager = new CrdtManager(this.config.schema, this.remote, logger);
161
+
157
162
  this.dataModule = new DataModule(
158
163
  this.cache,
159
164
  this.local,
@@ -275,6 +280,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
275
280
  }
276
281
 
277
282
  async close() {
283
+ this.crdtManager.closeAll();
278
284
  await this.local.close();
279
285
  await this.remote.close();
280
286
  }
@@ -283,6 +289,27 @@ export class Sp00kyClient<S extends SchemaStructure> {
283
289
  return this.remote.getClient().authenticate(token);
284
290
  }
285
291
 
292
+ /**
293
+ * Open a CRDT field for collaborative editing.
294
+ * Returns a CrdtField with a LoroDoc that can be bound to any editor.
295
+ * Also starts a LIVE SELECT on _00_crdt for real-time sync.
296
+ */
297
+ async openCrdtField(
298
+ table: string,
299
+ recordId: string,
300
+ field: string,
301
+ fallbackText?: string,
302
+ ): Promise<CrdtField> {
303
+ return this.crdtManager.open(table, recordId, field, fallbackText);
304
+ }
305
+
306
+ /**
307
+ * Close a CRDT field when editing is done.
308
+ */
309
+ closeCrdtField(table: string, recordId: string, field: string): void {
310
+ this.crdtManager.close(table, recordId, field);
311
+ }
312
+
286
313
  deauthenticate() {
287
314
  return this.remote.getClient().invalidate();
288
315
  }
@@ -137,6 +137,18 @@ export async function fileToUint8Array(file: File | Blob): Promise<Uint8Array> {
137
137
  return new Uint8Array(buffer);
138
138
  }
139
139
 
140
+ // ==================== TEXT UTILITIES ====================
141
+
142
+ /**
143
+ * Convert plain text to simple HTML paragraphs.
144
+ * Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
145
+ */
146
+ export function textToHtml(text: string): string {
147
+ return text
148
+ .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
149
+ .split('\n').map((l) => `<p>${l || '<br>'}</p>`).join('');
150
+ }
151
+
140
152
  // ==================== DATABASE UTILITIES ====================
141
153
 
142
154
  /**
@@ -9,7 +9,7 @@ export function cleanRecord(
9
9
  ): Record<string, any> {
10
10
  const cleaned: Record<string, any> = {};
11
11
  for (const [key, value] of Object.entries(record)) {
12
- if (key === 'id' || key in tableSchema) {
12
+ if (key === 'id' || key.startsWith('_00_') || key in tableSchema) {
13
13
  cleaned[key] = value;
14
14
  }
15
15
  }