@syncular/client-plugin-yjs 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -0
- package/package.json +57 -0
- package/src/index.test.ts +541 -0
- package/src/index.ts +784 -0
package/README.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# @syncular/client-plugin-crdt-yjs
|
|
2
|
+
|
|
3
|
+
Yjs-first CRDT plugin for Syncular client integration.
|
|
4
|
+
|
|
5
|
+
Implemented:
|
|
6
|
+
- Rule-based field mapping: `{ table, field, stateColumn }`.
|
|
7
|
+
- Kind-aware materialization: `text`, `xml-fragment`, `prosemirror`.
|
|
8
|
+
- Stable payload envelope key: `__yjs`.
|
|
9
|
+
- `beforePush` transformation:
|
|
10
|
+
- Applies Yjs update envelopes to local state.
|
|
11
|
+
- Materializes projection field values.
|
|
12
|
+
- Stores canonical snapshot state in `stateColumn`.
|
|
13
|
+
- Strips the envelope key by default to keep payload DB-safe.
|
|
14
|
+
- Can keep envelope for server-side CRDT merge via `stripEnvelopeBeforePush: false`.
|
|
15
|
+
- `afterPull` transformation for snapshot and incremental rows.
|
|
16
|
+
- `beforeApplyWsChanges` transformation for websocket inline rows.
|
|
17
|
+
- Helpers:
|
|
18
|
+
- `buildYjsTextUpdate`
|
|
19
|
+
- `applyYjsTextUpdates`
|
|
20
|
+
|
|
21
|
+
## Envelope Format
|
|
22
|
+
|
|
23
|
+
Outgoing payload example:
|
|
24
|
+
|
|
25
|
+
```ts
|
|
26
|
+
{
|
|
27
|
+
__yjs: {
|
|
28
|
+
content: {
|
|
29
|
+
updateId: "upd-123",
|
|
30
|
+
updateBase64: "<base64-encoded-yjs-update>"
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The plugin converts this to:
|
|
37
|
+
|
|
38
|
+
```ts
|
|
39
|
+
{
|
|
40
|
+
content: "materialized text",
|
|
41
|
+
content_yjs_state: "<base64-encoded-yjs-snapshot>"
|
|
42
|
+
}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Example
|
|
46
|
+
|
|
47
|
+
```ts
|
|
48
|
+
import { createYjsClientPlugin } from '@syncular/client-plugin-crdt-yjs';
|
|
49
|
+
|
|
50
|
+
const yjs = createYjsClientPlugin({
|
|
51
|
+
rules: [
|
|
52
|
+
{
|
|
53
|
+
table: 'tasks',
|
|
54
|
+
field: 'content',
|
|
55
|
+
stateColumn: 'content_yjs_state',
|
|
56
|
+
},
|
|
57
|
+
],
|
|
58
|
+
});
|
|
59
|
+
```
|
package/package.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@syncular/client-plugin-yjs",
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"description": "Yjs CRDT plugin primitives for Syncular client integration",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"author": "Benjamin Kniffler",
|
|
7
|
+
"homepage": "https://syncular.dev",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "https://github.com/syncular/syncular.git",
|
|
11
|
+
"directory": "plugins/crdt-yjs/client"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/syncular/syncular/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"sync",
|
|
18
|
+
"offline-first",
|
|
19
|
+
"realtime",
|
|
20
|
+
"database",
|
|
21
|
+
"typescript",
|
|
22
|
+
"crdt",
|
|
23
|
+
"yjs"
|
|
24
|
+
],
|
|
25
|
+
"private": false,
|
|
26
|
+
"publishConfig": {
|
|
27
|
+
"access": "public"
|
|
28
|
+
},
|
|
29
|
+
"type": "module",
|
|
30
|
+
"exports": {
|
|
31
|
+
".": {
|
|
32
|
+
"bun": "./src/index.ts",
|
|
33
|
+
"browser": "./src/index.ts",
|
|
34
|
+
"import": {
|
|
35
|
+
"types": "./dist/index.d.ts",
|
|
36
|
+
"default": "./dist/index.js"
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"scripts": {
|
|
41
|
+
"test": "bun test --pass-with-no-tests",
|
|
42
|
+
"tsgo": "tsgo --noEmit",
|
|
43
|
+
"build": "tsgo",
|
|
44
|
+
"release": "bunx syncular-publish"
|
|
45
|
+
},
|
|
46
|
+
"dependencies": {
|
|
47
|
+
"@syncular/client": "0.0.0",
|
|
48
|
+
"yjs": "^13.6.29"
|
|
49
|
+
},
|
|
50
|
+
"devDependencies": {
|
|
51
|
+
"@syncular/config": "0.0.0"
|
|
52
|
+
},
|
|
53
|
+
"files": [
|
|
54
|
+
"dist",
|
|
55
|
+
"src"
|
|
56
|
+
]
|
|
57
|
+
}
|
|
@@ -0,0 +1,541 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test';
|
|
2
|
+
import type {
|
|
3
|
+
SyncClientLocalMutationArgs,
|
|
4
|
+
SyncClientPluginContext,
|
|
5
|
+
SyncClientWsDeliveryArgs,
|
|
6
|
+
SyncPullResponse,
|
|
7
|
+
SyncPushRequest,
|
|
8
|
+
SyncSubscriptionRequest,
|
|
9
|
+
} from '@syncular/client';
|
|
10
|
+
import * as Y from 'yjs';
|
|
11
|
+
import {
|
|
12
|
+
applyYjsTextUpdates,
|
|
13
|
+
buildYjsTextUpdate,
|
|
14
|
+
createYjsClientPlugin,
|
|
15
|
+
YJS_PAYLOAD_KEY,
|
|
16
|
+
type YjsClientUpdateEnvelope,
|
|
17
|
+
} from './index';
|
|
18
|
+
|
|
19
|
+
const ctx: SyncClientPluginContext = {
|
|
20
|
+
actorId: 'actor-1',
|
|
21
|
+
clientId: 'client-1',
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
const emptyPullRequest = {
|
|
25
|
+
clientId: 'client-1',
|
|
26
|
+
limitCommits: 100,
|
|
27
|
+
subscriptions: [] as SyncSubscriptionRequest[],
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
async function callBeforePush(
|
|
31
|
+
plugin: ReturnType<typeof createYjsClientPlugin>,
|
|
32
|
+
request: SyncPushRequest
|
|
33
|
+
): Promise<SyncPushRequest> {
|
|
34
|
+
const hook = plugin.beforePush;
|
|
35
|
+
if (!hook) {
|
|
36
|
+
throw new Error('Expected beforePush hook');
|
|
37
|
+
}
|
|
38
|
+
return await hook(ctx, request);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function callAfterPull(
|
|
42
|
+
plugin: ReturnType<typeof createYjsClientPlugin>,
|
|
43
|
+
response: SyncPullResponse
|
|
44
|
+
): Promise<SyncPullResponse> {
|
|
45
|
+
const hook = plugin.afterPull;
|
|
46
|
+
if (!hook) {
|
|
47
|
+
throw new Error('Expected afterPull hook');
|
|
48
|
+
}
|
|
49
|
+
return await hook(ctx, {
|
|
50
|
+
request: emptyPullRequest,
|
|
51
|
+
response,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function callBeforeApplyWsChanges(
|
|
56
|
+
plugin: ReturnType<typeof createYjsClientPlugin>,
|
|
57
|
+
args: SyncClientWsDeliveryArgs
|
|
58
|
+
): Promise<SyncClientWsDeliveryArgs> {
|
|
59
|
+
const hook = plugin.beforeApplyWsChanges;
|
|
60
|
+
if (!hook) {
|
|
61
|
+
throw new Error('Expected beforeApplyWsChanges hook');
|
|
62
|
+
}
|
|
63
|
+
return await hook(ctx, args);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async function callBeforeApplyLocalMutations(
|
|
67
|
+
plugin: ReturnType<typeof createYjsClientPlugin>,
|
|
68
|
+
args: SyncClientLocalMutationArgs
|
|
69
|
+
): Promise<SyncClientLocalMutationArgs> {
|
|
70
|
+
const hook = plugin.beforeApplyLocalMutations;
|
|
71
|
+
if (!hook) {
|
|
72
|
+
throw new Error('Expected beforeApplyLocalMutations hook');
|
|
73
|
+
}
|
|
74
|
+
return await hook(ctx, args);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function createUpdate(
|
|
78
|
+
text: string,
|
|
79
|
+
previousStateBase64?: string
|
|
80
|
+
): { update: YjsClientUpdateEnvelope; state: string } {
|
|
81
|
+
const built = buildYjsTextUpdate({
|
|
82
|
+
previousStateBase64,
|
|
83
|
+
nextText: text,
|
|
84
|
+
containerKey: 'content',
|
|
85
|
+
});
|
|
86
|
+
return { update: built.update, state: built.nextStateBase64 };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function bytesToBase64(bytes: Uint8Array): string {
|
|
90
|
+
return Buffer.from(bytes).toString('base64');
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function createXmlState(text: string): string {
|
|
94
|
+
const doc = new Y.Doc();
|
|
95
|
+
const fragment = doc.getXmlFragment('content');
|
|
96
|
+
doc.transact(() => {
|
|
97
|
+
const paragraph = new Y.XmlElement('p');
|
|
98
|
+
const xmlText = new Y.XmlText();
|
|
99
|
+
xmlText.insert(0, text);
|
|
100
|
+
paragraph.insert(0, [xmlText]);
|
|
101
|
+
fragment.insert(0, [paragraph]);
|
|
102
|
+
});
|
|
103
|
+
const state = bytesToBase64(Y.encodeStateAsUpdate(doc));
|
|
104
|
+
doc.destroy();
|
|
105
|
+
return state;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
describe('@syncular/client-plugin-crdt-yjs', () => {
|
|
109
|
+
it('merges concurrent prepend and append updates without text duplication', () => {
|
|
110
|
+
const base = buildYjsTextUpdate({
|
|
111
|
+
nextText: '123',
|
|
112
|
+
containerKey: 'content',
|
|
113
|
+
});
|
|
114
|
+
const prepend = buildYjsTextUpdate({
|
|
115
|
+
previousStateBase64: base.nextStateBase64,
|
|
116
|
+
nextText: '0123',
|
|
117
|
+
containerKey: 'content',
|
|
118
|
+
});
|
|
119
|
+
const append = buildYjsTextUpdate({
|
|
120
|
+
previousStateBase64: base.nextStateBase64,
|
|
121
|
+
nextText: '1234',
|
|
122
|
+
containerKey: 'content',
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
const mergedForward = applyYjsTextUpdates({
|
|
126
|
+
previousStateBase64: base.nextStateBase64,
|
|
127
|
+
updates: [prepend.update, append.update],
|
|
128
|
+
containerKey: 'content',
|
|
129
|
+
});
|
|
130
|
+
const mergedReverse = applyYjsTextUpdates({
|
|
131
|
+
previousStateBase64: base.nextStateBase64,
|
|
132
|
+
updates: [append.update, prepend.update],
|
|
133
|
+
containerKey: 'content',
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
expect(mergedForward.text).toBe('01234');
|
|
137
|
+
expect(mergedReverse.text).toBe('01234');
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it('materializes outgoing payload from Yjs envelope updates and strips envelope key', async () => {
|
|
141
|
+
const plugin = createYjsClientPlugin({
|
|
142
|
+
rules: [
|
|
143
|
+
{
|
|
144
|
+
table: 'tasks',
|
|
145
|
+
field: 'content',
|
|
146
|
+
stateColumn: 'content_yjs_state',
|
|
147
|
+
},
|
|
148
|
+
],
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
const { update } = createUpdate('Hello Yjs');
|
|
152
|
+
const request: SyncPushRequest = {
|
|
153
|
+
clientId: 'client-1',
|
|
154
|
+
clientCommitId: 'commit-1',
|
|
155
|
+
schemaVersion: 1,
|
|
156
|
+
operations: [
|
|
157
|
+
{
|
|
158
|
+
table: 'tasks',
|
|
159
|
+
row_id: 'task-1',
|
|
160
|
+
op: 'upsert',
|
|
161
|
+
payload: {
|
|
162
|
+
[YJS_PAYLOAD_KEY]: {
|
|
163
|
+
content: update,
|
|
164
|
+
},
|
|
165
|
+
},
|
|
166
|
+
base_version: null,
|
|
167
|
+
},
|
|
168
|
+
],
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
const next = await callBeforePush(plugin, request);
|
|
172
|
+
const op = next.operations[0];
|
|
173
|
+
if (!op || op.op !== 'upsert' || !op.payload) {
|
|
174
|
+
throw new Error('Expected transformed upsert payload');
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
expect(op.payload.content).toBe('Hello Yjs');
|
|
178
|
+
expect(typeof op.payload.content_yjs_state).toBe('string');
|
|
179
|
+
expect(YJS_PAYLOAD_KEY in op.payload).toBe(false);
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it('can keep envelope key on push while still materializing payload', async () => {
|
|
183
|
+
const plugin = createYjsClientPlugin({
|
|
184
|
+
rules: [
|
|
185
|
+
{
|
|
186
|
+
table: 'tasks',
|
|
187
|
+
field: 'content',
|
|
188
|
+
stateColumn: 'content_yjs_state',
|
|
189
|
+
},
|
|
190
|
+
],
|
|
191
|
+
stripEnvelopeBeforePush: false,
|
|
192
|
+
stripEnvelopeBeforeApplyLocalMutations: true,
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
const { update } = createUpdate('Hello merge');
|
|
196
|
+
const request: SyncPushRequest = {
|
|
197
|
+
clientId: 'client-1',
|
|
198
|
+
clientCommitId: 'commit-keep-envelope',
|
|
199
|
+
schemaVersion: 1,
|
|
200
|
+
operations: [
|
|
201
|
+
{
|
|
202
|
+
table: 'tasks',
|
|
203
|
+
row_id: 'task-1',
|
|
204
|
+
op: 'upsert',
|
|
205
|
+
payload: {
|
|
206
|
+
[YJS_PAYLOAD_KEY]: {
|
|
207
|
+
content: update,
|
|
208
|
+
},
|
|
209
|
+
},
|
|
210
|
+
base_version: null,
|
|
211
|
+
},
|
|
212
|
+
],
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
const pushed = await callBeforePush(plugin, request);
|
|
216
|
+
const pushedPayload = pushed.operations[0]?.payload as
|
|
217
|
+
| Record<string, unknown>
|
|
218
|
+
| undefined;
|
|
219
|
+
if (!pushedPayload) {
|
|
220
|
+
throw new Error('Expected transformed push payload');
|
|
221
|
+
}
|
|
222
|
+
expect(pushedPayload.content).toBe('Hello merge');
|
|
223
|
+
expect(typeof pushedPayload.content_yjs_state).toBe('string');
|
|
224
|
+
expect(YJS_PAYLOAD_KEY in pushedPayload).toBe(true);
|
|
225
|
+
|
|
226
|
+
const local = await callBeforeApplyLocalMutations(plugin, {
|
|
227
|
+
operations: request.operations,
|
|
228
|
+
});
|
|
229
|
+
const localPayload = local.operations[0]?.payload as
|
|
230
|
+
| Record<string, unknown>
|
|
231
|
+
| undefined;
|
|
232
|
+
if (!localPayload) {
|
|
233
|
+
throw new Error('Expected transformed local payload');
|
|
234
|
+
}
|
|
235
|
+
expect(localPayload.content).toBe('Hello merge');
|
|
236
|
+
expect(typeof localPayload.content_yjs_state).toBe('string');
|
|
237
|
+
expect(YJS_PAYLOAD_KEY in localPayload).toBe(false);
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
it('uses cached row state from pull to apply future delta-only updates', async () => {
|
|
241
|
+
const plugin = createYjsClientPlugin({
|
|
242
|
+
rules: [
|
|
243
|
+
{
|
|
244
|
+
table: 'tasks',
|
|
245
|
+
field: 'content',
|
|
246
|
+
stateColumn: 'content_yjs_state',
|
|
247
|
+
},
|
|
248
|
+
],
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
const first = createUpdate('Hello');
|
|
252
|
+
const second = createUpdate('Hello world', first.state);
|
|
253
|
+
|
|
254
|
+
const pulled = await callAfterPull(plugin, {
|
|
255
|
+
ok: true,
|
|
256
|
+
subscriptions: [
|
|
257
|
+
{
|
|
258
|
+
id: 'tasks',
|
|
259
|
+
status: 'active',
|
|
260
|
+
scopes: {},
|
|
261
|
+
bootstrap: true,
|
|
262
|
+
nextCursor: 1,
|
|
263
|
+
bootstrapState: null,
|
|
264
|
+
commits: [],
|
|
265
|
+
snapshots: [
|
|
266
|
+
{
|
|
267
|
+
table: 'tasks',
|
|
268
|
+
rows: [
|
|
269
|
+
{
|
|
270
|
+
id: 'task-1',
|
|
271
|
+
content: 'stale',
|
|
272
|
+
content_yjs_state: first.state,
|
|
273
|
+
[YJS_PAYLOAD_KEY]: {
|
|
274
|
+
should_be_removed: true,
|
|
275
|
+
},
|
|
276
|
+
},
|
|
277
|
+
],
|
|
278
|
+
isFirstPage: true,
|
|
279
|
+
isLastPage: true,
|
|
280
|
+
},
|
|
281
|
+
],
|
|
282
|
+
},
|
|
283
|
+
],
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
const snapshotRow = pulled.subscriptions[0]?.snapshots?.[0]?.rows?.[0] as
|
|
287
|
+
| Record<string, unknown>
|
|
288
|
+
| undefined;
|
|
289
|
+
if (!snapshotRow) {
|
|
290
|
+
throw new Error('Expected transformed snapshot row');
|
|
291
|
+
}
|
|
292
|
+
expect(snapshotRow.content).toBe('Hello');
|
|
293
|
+
expect(YJS_PAYLOAD_KEY in snapshotRow).toBe(false);
|
|
294
|
+
|
|
295
|
+
const request: SyncPushRequest = {
|
|
296
|
+
clientId: 'client-1',
|
|
297
|
+
clientCommitId: 'commit-2',
|
|
298
|
+
schemaVersion: 1,
|
|
299
|
+
operations: [
|
|
300
|
+
{
|
|
301
|
+
table: 'tasks',
|
|
302
|
+
row_id: 'task-1',
|
|
303
|
+
op: 'upsert',
|
|
304
|
+
payload: {
|
|
305
|
+
[YJS_PAYLOAD_KEY]: {
|
|
306
|
+
content: second.update,
|
|
307
|
+
},
|
|
308
|
+
},
|
|
309
|
+
base_version: 1,
|
|
310
|
+
},
|
|
311
|
+
],
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
const next = await callBeforePush(plugin, request);
|
|
315
|
+
const payload = next.operations[0]?.payload as Record<string, unknown>;
|
|
316
|
+
expect(payload.content).toBe('Hello world');
|
|
317
|
+
expect(typeof payload.content_yjs_state).toBe('string');
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
it('materializes websocket inline changes through beforeApplyWsChanges', async () => {
|
|
321
|
+
const plugin = createYjsClientPlugin({
|
|
322
|
+
rules: [
|
|
323
|
+
{
|
|
324
|
+
table: 'tasks',
|
|
325
|
+
field: 'content',
|
|
326
|
+
stateColumn: 'content_yjs_state',
|
|
327
|
+
},
|
|
328
|
+
],
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
const first = createUpdate('Live update');
|
|
332
|
+
const next = await callBeforeApplyWsChanges(plugin, {
|
|
333
|
+
cursor: 10,
|
|
334
|
+
changes: [
|
|
335
|
+
{
|
|
336
|
+
table: 'tasks',
|
|
337
|
+
row_id: 'task-1',
|
|
338
|
+
op: 'upsert',
|
|
339
|
+
row_json: {
|
|
340
|
+
id: 'task-1',
|
|
341
|
+
content: 'stale',
|
|
342
|
+
content_yjs_state: first.state,
|
|
343
|
+
[YJS_PAYLOAD_KEY]: {
|
|
344
|
+
should_be_removed: true,
|
|
345
|
+
},
|
|
346
|
+
},
|
|
347
|
+
row_version: 2,
|
|
348
|
+
scopes: {},
|
|
349
|
+
},
|
|
350
|
+
],
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
const row = next.changes[0]?.row_json as
|
|
354
|
+
| Record<string, unknown>
|
|
355
|
+
| undefined;
|
|
356
|
+
if (!row) {
|
|
357
|
+
throw new Error('Expected transformed row_json');
|
|
358
|
+
}
|
|
359
|
+
expect(row.content).toBe('Live update');
|
|
360
|
+
expect(YJS_PAYLOAD_KEY in row).toBe(false);
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
it('materializes local mutation payloads through beforeApplyLocalMutations', async () => {
|
|
364
|
+
const plugin = createYjsClientPlugin({
|
|
365
|
+
rules: [
|
|
366
|
+
{
|
|
367
|
+
table: 'tasks',
|
|
368
|
+
field: 'content',
|
|
369
|
+
stateColumn: 'content_yjs_state',
|
|
370
|
+
},
|
|
371
|
+
],
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
const { update } = createUpdate('Local update');
|
|
375
|
+
const next = await callBeforeApplyLocalMutations(plugin, {
|
|
376
|
+
operations: [
|
|
377
|
+
{
|
|
378
|
+
table: 'tasks',
|
|
379
|
+
row_id: 'task-1',
|
|
380
|
+
op: 'upsert',
|
|
381
|
+
payload: {
|
|
382
|
+
[YJS_PAYLOAD_KEY]: {
|
|
383
|
+
content: update,
|
|
384
|
+
},
|
|
385
|
+
},
|
|
386
|
+
base_version: null,
|
|
387
|
+
},
|
|
388
|
+
],
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
const payload = next.operations[0]?.payload as
|
|
392
|
+
| Record<string, unknown>
|
|
393
|
+
| null
|
|
394
|
+
| undefined;
|
|
395
|
+
if (!payload) {
|
|
396
|
+
throw new Error('Expected transformed local payload');
|
|
397
|
+
}
|
|
398
|
+
expect(payload.content).toBe('Local update');
|
|
399
|
+
expect(typeof payload.content_yjs_state).toBe('string');
|
|
400
|
+
expect(YJS_PAYLOAD_KEY in payload).toBe(false);
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
it('materializes rows when state is delivered as UTF-8 bytes of base64 text', async () => {
|
|
404
|
+
const plugin = createYjsClientPlugin({
|
|
405
|
+
rules: [
|
|
406
|
+
{
|
|
407
|
+
table: 'tasks',
|
|
408
|
+
field: 'content',
|
|
409
|
+
stateColumn: 'content_yjs_state',
|
|
410
|
+
},
|
|
411
|
+
],
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
const first = createUpdate('Bytes snapshot');
|
|
415
|
+
const next = await callAfterPull(plugin, {
|
|
416
|
+
ok: true,
|
|
417
|
+
subscriptions: [
|
|
418
|
+
{
|
|
419
|
+
id: 'tasks',
|
|
420
|
+
status: 'active',
|
|
421
|
+
scopes: {},
|
|
422
|
+
bootstrap: true,
|
|
423
|
+
nextCursor: 1,
|
|
424
|
+
bootstrapState: null,
|
|
425
|
+
commits: [],
|
|
426
|
+
snapshots: [
|
|
427
|
+
{
|
|
428
|
+
table: 'tasks',
|
|
429
|
+
rows: [
|
|
430
|
+
{
|
|
431
|
+
id: 'task-1',
|
|
432
|
+
content: 'stale',
|
|
433
|
+
content_yjs_state: new TextEncoder().encode(first.state),
|
|
434
|
+
},
|
|
435
|
+
],
|
|
436
|
+
isFirstPage: true,
|
|
437
|
+
isLastPage: true,
|
|
438
|
+
},
|
|
439
|
+
],
|
|
440
|
+
},
|
|
441
|
+
],
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
const row = next.subscriptions[0]?.snapshots?.[0]?.rows?.[0] as
|
|
445
|
+
| Record<string, unknown>
|
|
446
|
+
| undefined;
|
|
447
|
+
if (!row) {
|
|
448
|
+
throw new Error('Expected transformed snapshot row');
|
|
449
|
+
}
|
|
450
|
+
expect(row.content).toBe('Bytes snapshot');
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
it('materializes xml-fragment kind rows during pull', async () => {
|
|
454
|
+
const plugin = createYjsClientPlugin({
|
|
455
|
+
rules: [
|
|
456
|
+
{
|
|
457
|
+
table: 'tasks',
|
|
458
|
+
field: 'content',
|
|
459
|
+
stateColumn: 'content_yjs_state',
|
|
460
|
+
kind: 'xml-fragment',
|
|
461
|
+
},
|
|
462
|
+
],
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
const xmlState = createXmlState('Hello XML');
|
|
466
|
+
const next = await callAfterPull(plugin, {
|
|
467
|
+
ok: true,
|
|
468
|
+
subscriptions: [
|
|
469
|
+
{
|
|
470
|
+
id: 'tasks',
|
|
471
|
+
status: 'active',
|
|
472
|
+
scopes: {},
|
|
473
|
+
bootstrap: true,
|
|
474
|
+
nextCursor: 1,
|
|
475
|
+
bootstrapState: null,
|
|
476
|
+
commits: [],
|
|
477
|
+
snapshots: [
|
|
478
|
+
{
|
|
479
|
+
table: 'tasks',
|
|
480
|
+
rows: [
|
|
481
|
+
{
|
|
482
|
+
id: 'task-1',
|
|
483
|
+
content: 'stale',
|
|
484
|
+
content_yjs_state: xmlState,
|
|
485
|
+
},
|
|
486
|
+
],
|
|
487
|
+
isFirstPage: true,
|
|
488
|
+
isLastPage: true,
|
|
489
|
+
},
|
|
490
|
+
],
|
|
491
|
+
},
|
|
492
|
+
],
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
const row = next.subscriptions[0]?.snapshots?.[0]?.rows?.[0] as
|
|
496
|
+
| Record<string, unknown>
|
|
497
|
+
| undefined;
|
|
498
|
+
if (!row) {
|
|
499
|
+
throw new Error('Expected transformed snapshot row');
|
|
500
|
+
}
|
|
501
|
+
expect(typeof row.content).toBe('string');
|
|
502
|
+
expect(String(row.content)).toContain('Hello XML');
|
|
503
|
+
});
|
|
504
|
+
|
|
505
|
+
it('throws when strict mode is enabled and envelope references unknown fields', async () => {
|
|
506
|
+
const plugin = createYjsClientPlugin({
|
|
507
|
+
rules: [
|
|
508
|
+
{
|
|
509
|
+
table: 'tasks',
|
|
510
|
+
field: 'content',
|
|
511
|
+
stateColumn: 'content_yjs_state',
|
|
512
|
+
},
|
|
513
|
+
],
|
|
514
|
+
strict: true,
|
|
515
|
+
});
|
|
516
|
+
|
|
517
|
+
const { update } = createUpdate('Hello');
|
|
518
|
+
const request: SyncPushRequest = {
|
|
519
|
+
clientId: 'client-1',
|
|
520
|
+
clientCommitId: 'commit-3',
|
|
521
|
+
schemaVersion: 1,
|
|
522
|
+
operations: [
|
|
523
|
+
{
|
|
524
|
+
table: 'tasks',
|
|
525
|
+
row_id: 'task-1',
|
|
526
|
+
op: 'upsert',
|
|
527
|
+
payload: {
|
|
528
|
+
[YJS_PAYLOAD_KEY]: {
|
|
529
|
+
unknown_field: update,
|
|
530
|
+
},
|
|
531
|
+
},
|
|
532
|
+
base_version: 1,
|
|
533
|
+
},
|
|
534
|
+
],
|
|
535
|
+
};
|
|
536
|
+
|
|
537
|
+
await expect(callBeforePush(plugin, request)).rejects.toThrow(
|
|
538
|
+
'No Yjs rule found for envelope field "unknown_field"'
|
|
539
|
+
);
|
|
540
|
+
});
|
|
541
|
+
});
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,784 @@
|
|
|
1
|
+
import {
|
|
2
|
+
PluginPriority,
|
|
3
|
+
type SyncChange,
|
|
4
|
+
type SyncClientLocalMutationArgs,
|
|
5
|
+
type SyncClientPlugin,
|
|
6
|
+
type SyncClientWsDeliveryArgs,
|
|
7
|
+
type SyncPullResponse,
|
|
8
|
+
type SyncPullSubscriptionResponse,
|
|
9
|
+
type SyncPushRequest,
|
|
10
|
+
} from '@syncular/client';
|
|
11
|
+
import * as Y from 'yjs';
|
|
12
|
+
|
|
13
|
+
export const YJS_PAYLOAD_KEY = '__yjs';
|
|
14
|
+
|
|
15
|
+
const BASE64_PATTERN =
|
|
16
|
+
/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
|
|
17
|
+
|
|
18
|
+
export type YjsFieldKind = 'text' | 'xml-fragment' | 'prosemirror';
|
|
19
|
+
|
|
20
|
+
export interface YjsClientFieldRule {
|
|
21
|
+
table: string;
|
|
22
|
+
field: string;
|
|
23
|
+
/**
|
|
24
|
+
* Column that stores canonical serialized Yjs state.
|
|
25
|
+
* Example: "content_yjs_state"
|
|
26
|
+
*/
|
|
27
|
+
stateColumn: string;
|
|
28
|
+
/**
|
|
29
|
+
* Container key inside the Yjs document. Defaults to `field`.
|
|
30
|
+
*/
|
|
31
|
+
containerKey?: string;
|
|
32
|
+
/**
|
|
33
|
+
* Snapshot row id column. Defaults to `id`.
|
|
34
|
+
*/
|
|
35
|
+
rowIdField?: string;
|
|
36
|
+
/**
|
|
37
|
+
* CRDT container type.
|
|
38
|
+
*/
|
|
39
|
+
kind?: YjsFieldKind;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
interface ResolvedYjsClientFieldRule extends YjsClientFieldRule {
|
|
43
|
+
containerKey: string;
|
|
44
|
+
rowIdField: string;
|
|
45
|
+
kind: YjsFieldKind;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface YjsClientUpdateEnvelope {
|
|
49
|
+
updateId: string;
|
|
50
|
+
updateBase64: string;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export type YjsClientUpdateInput =
|
|
54
|
+
| YjsClientUpdateEnvelope
|
|
55
|
+
| readonly YjsClientUpdateEnvelope[];
|
|
56
|
+
|
|
57
|
+
export interface YjsClientPayloadEnvelope {
|
|
58
|
+
[field: string]: YjsClientUpdateInput;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface BuildYjsTextUpdateArgs {
|
|
62
|
+
previousStateBase64?: string | Uint8Array | null;
|
|
63
|
+
nextText: string;
|
|
64
|
+
containerKey?: string;
|
|
65
|
+
updateId?: string;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface BuildYjsTextUpdateResult {
|
|
69
|
+
update: YjsClientUpdateEnvelope;
|
|
70
|
+
nextStateBase64: string;
|
|
71
|
+
nextText: string;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface ApplyYjsTextUpdatesArgs {
|
|
75
|
+
previousStateBase64?: string | Uint8Array | null;
|
|
76
|
+
updates: readonly YjsClientUpdateEnvelope[];
|
|
77
|
+
containerKey?: string;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface ApplyYjsTextUpdatesResult {
|
|
81
|
+
nextStateBase64: string;
|
|
82
|
+
text: string;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface CreateYjsClientPluginOptions {
|
|
86
|
+
name?: string;
|
|
87
|
+
rules: readonly YjsClientFieldRule[];
|
|
88
|
+
envelopeKey?: string;
|
|
89
|
+
priority?: number;
|
|
90
|
+
/**
|
|
91
|
+
* Throw when envelope payload references fields without matching rules.
|
|
92
|
+
* @default true
|
|
93
|
+
*/
|
|
94
|
+
strict?: boolean;
|
|
95
|
+
/**
|
|
96
|
+
* Remove the Yjs envelope key from outgoing/incoming records.
|
|
97
|
+
* @default true
|
|
98
|
+
*/
|
|
99
|
+
stripEnvelope?: boolean;
|
|
100
|
+
/**
|
|
101
|
+
* Remove the Yjs envelope key from push payloads.
|
|
102
|
+
* Default inherits from `stripEnvelope`.
|
|
103
|
+
*/
|
|
104
|
+
stripEnvelopeBeforePush?: boolean;
|
|
105
|
+
/**
|
|
106
|
+
* Remove the Yjs envelope key from local optimistic mutation payloads.
|
|
107
|
+
* Default inherits from `stripEnvelope`.
|
|
108
|
+
*/
|
|
109
|
+
stripEnvelopeBeforeApplyLocalMutations?: boolean;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
type RuleIndex = Map<string, Map<string, ResolvedYjsClientFieldRule>>;
|
|
113
|
+
|
|
114
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
115
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function readString(value: unknown): string | null {
|
|
119
|
+
return typeof value === 'string' && value.length > 0 ? value : null;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function bytesToBase64(bytes: Uint8Array): string {
|
|
123
|
+
if (typeof Buffer !== 'undefined') {
|
|
124
|
+
return Buffer.from(bytes).toString('base64');
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
let binary = '';
|
|
128
|
+
for (const byte of bytes) {
|
|
129
|
+
binary += String.fromCharCode(byte);
|
|
130
|
+
}
|
|
131
|
+
return btoa(binary);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function tryReadBase64TextFromBytes(bytes: Uint8Array): string | null {
|
|
135
|
+
if (bytes.length === 0) return null;
|
|
136
|
+
try {
|
|
137
|
+
const decoded = new TextDecoder().decode(bytes).trim();
|
|
138
|
+
if (!decoded || !BASE64_PATTERN.test(decoded)) return null;
|
|
139
|
+
return decoded;
|
|
140
|
+
} catch {
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function base64ToBytes(base64: string): Uint8Array {
|
|
146
|
+
if (!BASE64_PATTERN.test(base64)) {
|
|
147
|
+
throw new Error('Invalid base64 string');
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
if (typeof Buffer !== 'undefined') {
|
|
151
|
+
return new Uint8Array(Buffer.from(base64, 'base64'));
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const binary = atob(base64);
|
|
155
|
+
const out = new Uint8Array(binary.length);
|
|
156
|
+
for (let i = 0; i < binary.length; i++) {
|
|
157
|
+
out[i] = binary.charCodeAt(i);
|
|
158
|
+
}
|
|
159
|
+
return out;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function stateValueToBytes(value: unknown): Uint8Array | null {
|
|
163
|
+
if (value instanceof Uint8Array) {
|
|
164
|
+
const encoded = tryReadBase64TextFromBytes(value);
|
|
165
|
+
if (encoded) return base64ToBytes(encoded);
|
|
166
|
+
return value;
|
|
167
|
+
}
|
|
168
|
+
const str = readString(value);
|
|
169
|
+
if (!str) return null;
|
|
170
|
+
return base64ToBytes(str);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function stateValueToBase64(value: unknown): string | null {
|
|
174
|
+
const str = readString(value);
|
|
175
|
+
if (str) return str;
|
|
176
|
+
if (value instanceof Uint8Array) {
|
|
177
|
+
const encoded = tryReadBase64TextFromBytes(value);
|
|
178
|
+
if (encoded) return encoded;
|
|
179
|
+
return bytesToBase64(value);
|
|
180
|
+
}
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function createDocFromState(stateValue: unknown): Y.Doc {
|
|
185
|
+
const doc = new Y.Doc();
|
|
186
|
+
const bytes = stateValueToBytes(stateValue);
|
|
187
|
+
if (bytes && bytes.length > 0) {
|
|
188
|
+
Y.applyUpdate(doc, bytes);
|
|
189
|
+
}
|
|
190
|
+
return doc;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function exportSnapshotBase64(doc: Y.Doc): string {
|
|
194
|
+
return bytesToBase64(Y.encodeStateAsUpdate(doc));
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function replaceText(doc: Y.Doc, containerKey: string, nextText: string): void {
|
|
198
|
+
const text = doc.getText(containerKey);
|
|
199
|
+
const currentLength = text.length;
|
|
200
|
+
doc.transact(() => {
|
|
201
|
+
if (currentLength > 0) {
|
|
202
|
+
text.delete(0, currentLength);
|
|
203
|
+
}
|
|
204
|
+
if (nextText.length > 0) {
|
|
205
|
+
text.insert(0, nextText);
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function patchText(doc: Y.Doc, containerKey: string, nextText: string): void {
|
|
211
|
+
const text = doc.getText(containerKey);
|
|
212
|
+
const currentText = text.toString();
|
|
213
|
+
if (currentText === nextText) return;
|
|
214
|
+
|
|
215
|
+
const minLength = Math.min(currentText.length, nextText.length);
|
|
216
|
+
let prefixLength = 0;
|
|
217
|
+
while (
|
|
218
|
+
prefixLength < minLength &&
|
|
219
|
+
currentText.charCodeAt(prefixLength) === nextText.charCodeAt(prefixLength)
|
|
220
|
+
) {
|
|
221
|
+
prefixLength += 1;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
let currentSuffixStart = currentText.length;
|
|
225
|
+
let nextSuffixStart = nextText.length;
|
|
226
|
+
while (
|
|
227
|
+
currentSuffixStart > prefixLength &&
|
|
228
|
+
nextSuffixStart > prefixLength &&
|
|
229
|
+
currentText.charCodeAt(currentSuffixStart - 1) ===
|
|
230
|
+
nextText.charCodeAt(nextSuffixStart - 1)
|
|
231
|
+
) {
|
|
232
|
+
currentSuffixStart -= 1;
|
|
233
|
+
nextSuffixStart -= 1;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const deleteLength = currentSuffixStart - prefixLength;
|
|
237
|
+
const insertSegment = nextText.slice(prefixLength, nextSuffixStart);
|
|
238
|
+
|
|
239
|
+
doc.transact(() => {
|
|
240
|
+
if (deleteLength > 0) {
|
|
241
|
+
text.delete(prefixLength, deleteLength);
|
|
242
|
+
}
|
|
243
|
+
if (insertSegment.length > 0) {
|
|
244
|
+
text.insert(prefixLength, insertSegment);
|
|
245
|
+
}
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function ensureTextContainer(doc: Y.Doc, containerKey: string): string {
|
|
250
|
+
return doc.getText(containerKey).toString();
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function ensureXmlFragmentContainer(doc: Y.Doc, containerKey: string): string {
|
|
254
|
+
return doc.getXmlFragment(containerKey).toString();
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function materializeRuleValue(
|
|
258
|
+
doc: Y.Doc,
|
|
259
|
+
rule: ResolvedYjsClientFieldRule
|
|
260
|
+
): unknown {
|
|
261
|
+
if (rule.kind === 'text') {
|
|
262
|
+
return ensureTextContainer(doc, rule.containerKey);
|
|
263
|
+
}
|
|
264
|
+
return ensureXmlFragmentContainer(doc, rule.containerKey);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function seedRuleValueFromPayload(
|
|
268
|
+
doc: Y.Doc,
|
|
269
|
+
rule: ResolvedYjsClientFieldRule,
|
|
270
|
+
source: Record<string, unknown>
|
|
271
|
+
): void {
|
|
272
|
+
if (rule.kind !== 'text') return;
|
|
273
|
+
const initialText = readString(source[rule.field]);
|
|
274
|
+
if (initialText) {
|
|
275
|
+
replaceText(doc, rule.containerKey, initialText);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function buildRuleIndex(rules: readonly YjsClientFieldRule[]): {
|
|
280
|
+
index: RuleIndex;
|
|
281
|
+
normalizedRules: readonly ResolvedYjsClientFieldRule[];
|
|
282
|
+
} {
|
|
283
|
+
const index: RuleIndex = new Map();
|
|
284
|
+
const normalizedRules: ResolvedYjsClientFieldRule[] = [];
|
|
285
|
+
const seen = new Set<string>();
|
|
286
|
+
|
|
287
|
+
for (const rule of rules) {
|
|
288
|
+
if (!rule.table.trim()) {
|
|
289
|
+
throw new Error('YjsClientFieldRule.table cannot be empty');
|
|
290
|
+
}
|
|
291
|
+
if (!rule.field.trim()) {
|
|
292
|
+
throw new Error('YjsClientFieldRule.field cannot be empty');
|
|
293
|
+
}
|
|
294
|
+
if (!rule.stateColumn.trim()) {
|
|
295
|
+
throw new Error('YjsClientFieldRule.stateColumn cannot be empty');
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const key = `${rule.table}\u001f${rule.field}`;
|
|
299
|
+
if (seen.has(key)) {
|
|
300
|
+
throw new Error(
|
|
301
|
+
`Duplicate Yjs client rule for table "${rule.table}", field "${rule.field}"`
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
seen.add(key);
|
|
305
|
+
|
|
306
|
+
const resolved: ResolvedYjsClientFieldRule = {
|
|
307
|
+
...rule,
|
|
308
|
+
containerKey: rule.containerKey ?? rule.field,
|
|
309
|
+
rowIdField: rule.rowIdField ?? 'id',
|
|
310
|
+
kind: rule.kind ?? 'text',
|
|
311
|
+
};
|
|
312
|
+
normalizedRules.push(resolved);
|
|
313
|
+
|
|
314
|
+
const tableRules = index.get(resolved.table) ?? new Map();
|
|
315
|
+
tableRules.set(resolved.field, resolved);
|
|
316
|
+
index.set(resolved.table, tableRules);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
return { index, normalizedRules };
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function rowFieldCacheKey(table: string, rowId: string, field: string): string {
|
|
323
|
+
return `${table}\u001f${rowId}\u001f${field}`;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function resolveSnapshotRowId(
|
|
327
|
+
row: Record<string, unknown>,
|
|
328
|
+
rule: ResolvedYjsClientFieldRule
|
|
329
|
+
): string | null {
|
|
330
|
+
const candidate = row[rule.rowIdField];
|
|
331
|
+
return typeof candidate === 'string' && candidate.length > 0
|
|
332
|
+
? candidate
|
|
333
|
+
: null;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function normalizeUpdateEnvelope(
|
|
337
|
+
value: unknown,
|
|
338
|
+
context: string
|
|
339
|
+
): YjsClientUpdateEnvelope {
|
|
340
|
+
if (!isRecord(value)) {
|
|
341
|
+
throw new Error(`${context} must be an object`);
|
|
342
|
+
}
|
|
343
|
+
const updateId = readString(value.updateId);
|
|
344
|
+
const updateBase64 = readString(value.updateBase64);
|
|
345
|
+
if (!updateId) {
|
|
346
|
+
throw new Error(`${context}.updateId must be a non-empty string`);
|
|
347
|
+
}
|
|
348
|
+
if (!updateBase64) {
|
|
349
|
+
throw new Error(
|
|
350
|
+
`${context}.updateBase64 must be a non-empty base64 string`
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
return { updateId, updateBase64 };
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function normalizeUpdateEnvelopes(
|
|
357
|
+
value: unknown,
|
|
358
|
+
context: string
|
|
359
|
+
): YjsClientUpdateEnvelope[] {
|
|
360
|
+
if (Array.isArray(value)) {
|
|
361
|
+
return value.map((entry, i) =>
|
|
362
|
+
normalizeUpdateEnvelope(entry, `${context}[${i}]`)
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
return [normalizeUpdateEnvelope(value, context)];
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function createUpdateId(): string {
|
|
369
|
+
const randomUUID = globalThis.crypto?.randomUUID;
|
|
370
|
+
if (typeof randomUUID === 'function') {
|
|
371
|
+
return randomUUID.call(globalThis.crypto);
|
|
372
|
+
}
|
|
373
|
+
const ts = Date.now().toString(36);
|
|
374
|
+
const rnd = Math.random().toString(36).slice(2, 12);
|
|
375
|
+
return `yjs-${ts}-${rnd}`;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
export function applyYjsTextUpdates(
|
|
379
|
+
args: ApplyYjsTextUpdatesArgs
|
|
380
|
+
): ApplyYjsTextUpdatesResult {
|
|
381
|
+
const containerKey = args.containerKey ?? 'text';
|
|
382
|
+
const doc = createDocFromState(args.previousStateBase64);
|
|
383
|
+
try {
|
|
384
|
+
for (const update of args.updates) {
|
|
385
|
+
Y.applyUpdate(doc, base64ToBytes(update.updateBase64));
|
|
386
|
+
}
|
|
387
|
+
const text = ensureTextContainer(doc, containerKey);
|
|
388
|
+
const nextStateBase64 = exportSnapshotBase64(doc);
|
|
389
|
+
return { nextStateBase64, text };
|
|
390
|
+
} finally {
|
|
391
|
+
doc.destroy();
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
export function buildYjsTextUpdate(
|
|
396
|
+
args: BuildYjsTextUpdateArgs
|
|
397
|
+
): BuildYjsTextUpdateResult {
|
|
398
|
+
const containerKey = args.containerKey ?? 'text';
|
|
399
|
+
const doc = createDocFromState(args.previousStateBase64);
|
|
400
|
+
try {
|
|
401
|
+
const from = Y.encodeStateVector(doc);
|
|
402
|
+
patchText(doc, containerKey, args.nextText);
|
|
403
|
+
const update = bytesToBase64(Y.encodeStateAsUpdate(doc, from));
|
|
404
|
+
const nextText = ensureTextContainer(doc, containerKey);
|
|
405
|
+
const nextStateBase64 = exportSnapshotBase64(doc);
|
|
406
|
+
|
|
407
|
+
return {
|
|
408
|
+
update: {
|
|
409
|
+
updateId: args.updateId ?? createUpdateId(),
|
|
410
|
+
updateBase64: update,
|
|
411
|
+
},
|
|
412
|
+
nextStateBase64,
|
|
413
|
+
nextText,
|
|
414
|
+
};
|
|
415
|
+
} finally {
|
|
416
|
+
doc.destroy();
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
function materializeRowFromState(args: {
|
|
421
|
+
table: string;
|
|
422
|
+
rowId: string | null;
|
|
423
|
+
row: Record<string, unknown>;
|
|
424
|
+
index: RuleIndex;
|
|
425
|
+
stateByRowField: Map<string, string>;
|
|
426
|
+
envelopeKey: string;
|
|
427
|
+
stripEnvelope: boolean;
|
|
428
|
+
}): Record<string, unknown> {
|
|
429
|
+
const tableRules = args.index.get(args.table);
|
|
430
|
+
if (!tableRules) {
|
|
431
|
+
if (args.stripEnvelope && args.envelopeKey in args.row) {
|
|
432
|
+
const next = { ...args.row };
|
|
433
|
+
delete next[args.envelopeKey];
|
|
434
|
+
return next;
|
|
435
|
+
}
|
|
436
|
+
return args.row;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
let nextRow: Record<string, unknown> | null = null;
|
|
440
|
+
const ensureRow = (): Record<string, unknown> => {
|
|
441
|
+
if (nextRow) return nextRow;
|
|
442
|
+
nextRow = { ...args.row };
|
|
443
|
+
return nextRow;
|
|
444
|
+
};
|
|
445
|
+
|
|
446
|
+
for (const rule of tableRules.values()) {
|
|
447
|
+
const source = nextRow ?? args.row;
|
|
448
|
+
const stateBase64 = stateValueToBase64(source[rule.stateColumn]);
|
|
449
|
+
if (!stateBase64) continue;
|
|
450
|
+
|
|
451
|
+
const doc = createDocFromState(stateBase64);
|
|
452
|
+
try {
|
|
453
|
+
const nextValue = materializeRuleValue(doc, rule);
|
|
454
|
+
if (source[rule.field] !== nextValue) {
|
|
455
|
+
ensureRow()[rule.field] = nextValue;
|
|
456
|
+
}
|
|
457
|
+
if (args.rowId) {
|
|
458
|
+
args.stateByRowField.set(
|
|
459
|
+
rowFieldCacheKey(args.table, args.rowId, rule.field),
|
|
460
|
+
stateBase64
|
|
461
|
+
);
|
|
462
|
+
}
|
|
463
|
+
} finally {
|
|
464
|
+
doc.destroy();
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
if (args.stripEnvelope) {
|
|
469
|
+
const source = nextRow ?? args.row;
|
|
470
|
+
if (args.envelopeKey in source) {
|
|
471
|
+
const target = ensureRow();
|
|
472
|
+
delete target[args.envelopeKey];
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
return nextRow ?? args.row;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
function transformPushPayload(args: {
|
|
480
|
+
table: string;
|
|
481
|
+
rowId: string;
|
|
482
|
+
payload: Record<string, unknown>;
|
|
483
|
+
index: RuleIndex;
|
|
484
|
+
stateByRowField: Map<string, string>;
|
|
485
|
+
envelopeKey: string;
|
|
486
|
+
stripEnvelope: boolean;
|
|
487
|
+
strict: boolean;
|
|
488
|
+
}): Record<string, unknown> {
|
|
489
|
+
const tableRules = args.index.get(args.table);
|
|
490
|
+
const rawEnvelope = args.payload[args.envelopeKey];
|
|
491
|
+
|
|
492
|
+
if (!tableRules) {
|
|
493
|
+
if (rawEnvelope !== undefined && args.strict) {
|
|
494
|
+
throw new Error(
|
|
495
|
+
`Yjs envelope provided for table "${args.table}" without matching rules`
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
if (args.stripEnvelope && rawEnvelope !== undefined) {
|
|
499
|
+
const next = { ...args.payload };
|
|
500
|
+
delete next[args.envelopeKey];
|
|
501
|
+
return next;
|
|
502
|
+
}
|
|
503
|
+
return args.payload;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
let nextPayload: Record<string, unknown> | null = null;
|
|
507
|
+
const ensurePayload = (): Record<string, unknown> => {
|
|
508
|
+
if (nextPayload) return nextPayload;
|
|
509
|
+
nextPayload = { ...args.payload };
|
|
510
|
+
return nextPayload;
|
|
511
|
+
};
|
|
512
|
+
|
|
513
|
+
const sourceEnvelope = rawEnvelope;
|
|
514
|
+
if (sourceEnvelope !== undefined && !isRecord(sourceEnvelope)) {
|
|
515
|
+
throw new Error(
|
|
516
|
+
`Yjs payload key "${args.envelopeKey}" must be an object for table "${args.table}"`
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
for (const rule of tableRules.values()) {
|
|
521
|
+
const source = nextPayload ?? args.payload;
|
|
522
|
+
const stateBase64 = stateValueToBase64(source[rule.stateColumn]);
|
|
523
|
+
if (stateBase64) {
|
|
524
|
+
args.stateByRowField.set(
|
|
525
|
+
rowFieldCacheKey(args.table, args.rowId, rule.field),
|
|
526
|
+
stateBase64
|
|
527
|
+
);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
if (sourceEnvelope) {
|
|
532
|
+
for (const [field, rawUpdateInput] of Object.entries(sourceEnvelope)) {
|
|
533
|
+
const rule = tableRules.get(field);
|
|
534
|
+
if (!rule) {
|
|
535
|
+
if (args.strict) {
|
|
536
|
+
throw new Error(
|
|
537
|
+
`No Yjs rule found for envelope field "${field}" on table "${args.table}"`
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
continue;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
const updates = normalizeUpdateEnvelopes(
|
|
544
|
+
rawUpdateInput,
|
|
545
|
+
`yjs.${args.table}.${field}`
|
|
546
|
+
);
|
|
547
|
+
const cacheKey = rowFieldCacheKey(args.table, args.rowId, rule.field);
|
|
548
|
+
const source = nextPayload ?? args.payload;
|
|
549
|
+
const baseState =
|
|
550
|
+
stateValueToBase64(source[rule.stateColumn]) ??
|
|
551
|
+
args.stateByRowField.get(cacheKey) ??
|
|
552
|
+
null;
|
|
553
|
+
|
|
554
|
+
const doc = createDocFromState(baseState);
|
|
555
|
+
try {
|
|
556
|
+
if (!baseState) {
|
|
557
|
+
seedRuleValueFromPayload(doc, rule, source);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
for (const update of updates) {
|
|
561
|
+
Y.applyUpdate(doc, base64ToBytes(update.updateBase64));
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
const nextValue = materializeRuleValue(doc, rule);
|
|
565
|
+
const nextStateBase64 = exportSnapshotBase64(doc);
|
|
566
|
+
const target = ensurePayload();
|
|
567
|
+
target[rule.field] = nextValue;
|
|
568
|
+
target[rule.stateColumn] = nextStateBase64;
|
|
569
|
+
args.stateByRowField.set(cacheKey, nextStateBase64);
|
|
570
|
+
} finally {
|
|
571
|
+
doc.destroy();
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
if (args.stripEnvelope) {
|
|
577
|
+
const source = nextPayload ?? args.payload;
|
|
578
|
+
if (args.envelopeKey in source) {
|
|
579
|
+
const target = ensurePayload();
|
|
580
|
+
delete target[args.envelopeKey];
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
return nextPayload ?? args.payload;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
function transformPullSubscription(args: {
|
|
588
|
+
sub: SyncPullSubscriptionResponse;
|
|
589
|
+
index: RuleIndex;
|
|
590
|
+
stateByRowField: Map<string, string>;
|
|
591
|
+
envelopeKey: string;
|
|
592
|
+
stripEnvelope: boolean;
|
|
593
|
+
}): SyncPullSubscriptionResponse {
|
|
594
|
+
const nextSnapshots = (args.sub.snapshots ?? []).map((snapshot) => ({
|
|
595
|
+
...snapshot,
|
|
596
|
+
rows: (snapshot.rows ?? []).map((row) => {
|
|
597
|
+
if (!isRecord(row)) return row;
|
|
598
|
+
const tableRules = args.index.get(snapshot.table);
|
|
599
|
+
if (!tableRules) {
|
|
600
|
+
return materializeRowFromState({
|
|
601
|
+
table: snapshot.table,
|
|
602
|
+
rowId: null,
|
|
603
|
+
row,
|
|
604
|
+
index: args.index,
|
|
605
|
+
stateByRowField: args.stateByRowField,
|
|
606
|
+
envelopeKey: args.envelopeKey,
|
|
607
|
+
stripEnvelope: args.stripEnvelope,
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
let rowId: string | null = null;
|
|
612
|
+
for (const rule of tableRules.values()) {
|
|
613
|
+
rowId = resolveSnapshotRowId(row, rule);
|
|
614
|
+
if (rowId) break;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
return materializeRowFromState({
|
|
618
|
+
table: snapshot.table,
|
|
619
|
+
rowId,
|
|
620
|
+
row,
|
|
621
|
+
index: args.index,
|
|
622
|
+
stateByRowField: args.stateByRowField,
|
|
623
|
+
envelopeKey: args.envelopeKey,
|
|
624
|
+
stripEnvelope: args.stripEnvelope,
|
|
625
|
+
});
|
|
626
|
+
}),
|
|
627
|
+
}));
|
|
628
|
+
|
|
629
|
+
const nextCommits = (args.sub.commits ?? []).map((commit) => ({
|
|
630
|
+
...commit,
|
|
631
|
+
changes: (commit.changes ?? []).map((change) => {
|
|
632
|
+
if (change.op !== 'upsert' || !isRecord(change.row_json)) return change;
|
|
633
|
+
const nextRow = materializeRowFromState({
|
|
634
|
+
table: change.table,
|
|
635
|
+
rowId: change.row_id,
|
|
636
|
+
row: change.row_json,
|
|
637
|
+
index: args.index,
|
|
638
|
+
stateByRowField: args.stateByRowField,
|
|
639
|
+
envelopeKey: args.envelopeKey,
|
|
640
|
+
stripEnvelope: args.stripEnvelope,
|
|
641
|
+
});
|
|
642
|
+
if (nextRow === change.row_json) return change;
|
|
643
|
+
return { ...change, row_json: nextRow };
|
|
644
|
+
}),
|
|
645
|
+
}));
|
|
646
|
+
|
|
647
|
+
return {
|
|
648
|
+
...args.sub,
|
|
649
|
+
snapshots: nextSnapshots,
|
|
650
|
+
commits: nextCommits,
|
|
651
|
+
};
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
function transformWsChanges(args: {
|
|
655
|
+
changes: SyncChange[];
|
|
656
|
+
index: RuleIndex;
|
|
657
|
+
stateByRowField: Map<string, string>;
|
|
658
|
+
envelopeKey: string;
|
|
659
|
+
stripEnvelope: boolean;
|
|
660
|
+
}): SyncChange[] {
|
|
661
|
+
return args.changes.map((change) => {
|
|
662
|
+
if (change.op !== 'upsert' || !isRecord(change.row_json)) return change;
|
|
663
|
+
const nextRow = materializeRowFromState({
|
|
664
|
+
table: change.table,
|
|
665
|
+
rowId: change.row_id,
|
|
666
|
+
row: change.row_json,
|
|
667
|
+
index: args.index,
|
|
668
|
+
stateByRowField: args.stateByRowField,
|
|
669
|
+
envelopeKey: args.envelopeKey,
|
|
670
|
+
stripEnvelope: args.stripEnvelope,
|
|
671
|
+
});
|
|
672
|
+
if (nextRow === change.row_json) return change;
|
|
673
|
+
return { ...change, row_json: nextRow };
|
|
674
|
+
});
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
export function createYjsClientPlugin(
|
|
678
|
+
options: CreateYjsClientPluginOptions
|
|
679
|
+
): SyncClientPlugin {
|
|
680
|
+
if (options.rules.length === 0) {
|
|
681
|
+
throw new Error(
|
|
682
|
+
'createYjsClientPlugin requires at least one table/field rule'
|
|
683
|
+
);
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
const envelopeKey = options.envelopeKey ?? YJS_PAYLOAD_KEY;
|
|
687
|
+
const strict = options.strict ?? true;
|
|
688
|
+
const stripEnvelope = options.stripEnvelope ?? true;
|
|
689
|
+
const stripEnvelopeBeforePush =
|
|
690
|
+
options.stripEnvelopeBeforePush ?? stripEnvelope;
|
|
691
|
+
const stripEnvelopeBeforeApplyLocalMutations =
|
|
692
|
+
options.stripEnvelopeBeforeApplyLocalMutations ?? stripEnvelope;
|
|
693
|
+
const { index } = buildRuleIndex(options.rules);
|
|
694
|
+
const stateByRowField = new Map<string, string>();
|
|
695
|
+
|
|
696
|
+
return {
|
|
697
|
+
name: options.name ?? 'crdt-yjs-client',
|
|
698
|
+
priority: options.priority ?? PluginPriority.DEFAULT,
|
|
699
|
+
|
|
700
|
+
beforePush(_ctx, request): SyncPushRequest {
|
|
701
|
+
const nextOperations = request.operations.map((op) => {
|
|
702
|
+
if (op.op !== 'upsert') return op;
|
|
703
|
+
if (!isRecord(op.payload)) return op;
|
|
704
|
+
|
|
705
|
+
const nextPayload = transformPushPayload({
|
|
706
|
+
table: op.table,
|
|
707
|
+
rowId: op.row_id,
|
|
708
|
+
payload: op.payload,
|
|
709
|
+
index,
|
|
710
|
+
stateByRowField,
|
|
711
|
+
envelopeKey,
|
|
712
|
+
stripEnvelope: stripEnvelopeBeforePush,
|
|
713
|
+
strict,
|
|
714
|
+
});
|
|
715
|
+
|
|
716
|
+
if (nextPayload === op.payload) return op;
|
|
717
|
+
return { ...op, payload: nextPayload };
|
|
718
|
+
});
|
|
719
|
+
|
|
720
|
+
return { ...request, operations: nextOperations };
|
|
721
|
+
},
|
|
722
|
+
|
|
723
|
+
beforeApplyLocalMutations(
|
|
724
|
+
_ctx,
|
|
725
|
+
args: SyncClientLocalMutationArgs
|
|
726
|
+
): SyncClientLocalMutationArgs {
|
|
727
|
+
const nextOperations = args.operations.map((op) => {
|
|
728
|
+
if (op.op !== 'upsert') return op;
|
|
729
|
+
if (!isRecord(op.payload)) return op;
|
|
730
|
+
|
|
731
|
+
const nextPayload = transformPushPayload({
|
|
732
|
+
table: op.table,
|
|
733
|
+
rowId: op.row_id,
|
|
734
|
+
payload: op.payload,
|
|
735
|
+
index,
|
|
736
|
+
stateByRowField,
|
|
737
|
+
envelopeKey,
|
|
738
|
+
stripEnvelope: stripEnvelopeBeforeApplyLocalMutations,
|
|
739
|
+
strict,
|
|
740
|
+
});
|
|
741
|
+
|
|
742
|
+
if (nextPayload === op.payload) return op;
|
|
743
|
+
return { ...op, payload: nextPayload };
|
|
744
|
+
});
|
|
745
|
+
|
|
746
|
+
return { ...args, operations: nextOperations };
|
|
747
|
+
},
|
|
748
|
+
|
|
749
|
+
afterPull(_ctx, args: { response: SyncPullResponse }): SyncPullResponse {
|
|
750
|
+
const nextSubscriptions = args.response.subscriptions.map((sub) =>
|
|
751
|
+
transformPullSubscription({
|
|
752
|
+
sub,
|
|
753
|
+
index,
|
|
754
|
+
stateByRowField,
|
|
755
|
+
envelopeKey,
|
|
756
|
+
stripEnvelope,
|
|
757
|
+
})
|
|
758
|
+
);
|
|
759
|
+
|
|
760
|
+
return {
|
|
761
|
+
...args.response,
|
|
762
|
+
subscriptions: nextSubscriptions,
|
|
763
|
+
};
|
|
764
|
+
},
|
|
765
|
+
|
|
766
|
+
beforeApplyWsChanges(
|
|
767
|
+
_ctx,
|
|
768
|
+
args: SyncClientWsDeliveryArgs
|
|
769
|
+
): SyncClientWsDeliveryArgs {
|
|
770
|
+
const nextChanges = transformWsChanges({
|
|
771
|
+
changes: args.changes,
|
|
772
|
+
index,
|
|
773
|
+
stateByRowField,
|
|
774
|
+
envelopeKey,
|
|
775
|
+
stripEnvelope,
|
|
776
|
+
});
|
|
777
|
+
|
|
778
|
+
return {
|
|
779
|
+
...args,
|
|
780
|
+
changes: nextChanges,
|
|
781
|
+
};
|
|
782
|
+
},
|
|
783
|
+
};
|
|
784
|
+
}
|