braid-text 0.5.3 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server-demo.js +4 -3
- package/server.js +1004 -1050
package/server.js
CHANGED
|
@@ -7,6 +7,19 @@ let Y = null
|
|
|
7
7
|
try { Y = require('yjs') } catch(e) {}
|
|
8
8
|
|
|
9
9
|
|
|
10
|
+
var all_subscriptions = new Set()
|
|
11
|
+
|
|
12
|
+
// Converts this to a forced serialized function, that is only called one at a
|
|
13
|
+
// time, and in order. Even if the inner `fun` is async, it will guarantee
|
|
14
|
+
// that the prior one is always finished before the next one is called.
|
|
15
|
+
//
|
|
16
|
+
// We use this to guarantee that updates are all sent in order, with the prior
|
|
17
|
+
// one sent before the next one begins to send.
|
|
18
|
+
function one_at_a_time(fun) {
|
|
19
|
+
var queue = Promise.resolve()
|
|
20
|
+
return (message) => queue = queue.then(() => fun(message))
|
|
21
|
+
}
|
|
22
|
+
|
|
10
23
|
function create_braid_text() {
|
|
11
24
|
let braid_text = {
|
|
12
25
|
verbose: false,
|
|
@@ -14,6 +27,7 @@ function create_braid_text() {
|
|
|
14
27
|
length_cache_size: 10,
|
|
15
28
|
meta_file_save_period_ms: 1000,
|
|
16
29
|
debug_sync_checks: false,
|
|
30
|
+
simpletonSetTimeout: setTimeout, // Can be customized for fuzz testing
|
|
17
31
|
cache: {}
|
|
18
32
|
}
|
|
19
33
|
|
|
@@ -22,35 +36,39 @@ function create_braid_text() {
|
|
|
22
36
|
return Y
|
|
23
37
|
}
|
|
24
38
|
|
|
39
|
+
braid_text.end_all_subscriptions = function() {
|
|
40
|
+
var subs = [...all_subscriptions]
|
|
41
|
+
for (var res of subs) res.end()
|
|
42
|
+
}
|
|
43
|
+
|
|
25
44
|
let waiting_puts = 0
|
|
26
45
|
|
|
27
46
|
let max_encoded_key_size = 240
|
|
28
47
|
|
|
29
|
-
|
|
48
|
+
// Bidirectional sync between a local resource and a remote server.
|
|
49
|
+
// Keeps them in sync by forwarding DT updates in both directions.
|
|
50
|
+
// Reconnects automatically on disconnect.
|
|
51
|
+
braid_text.sync = async (local_key, remote_url, options = {}) => {
|
|
30
52
|
if (!options.merge_type) options.merge_type = 'dt'
|
|
31
53
|
|
|
32
|
-
//
|
|
33
|
-
// since it is unused, unoptimized,
|
|
34
|
-
// and not as well battle tested
|
|
35
|
-
if ((a instanceof URL) === (b instanceof URL))
|
|
36
|
-
throw new Error(`one parameter should be local string key, and the other a remote URL object`)
|
|
54
|
+
// ── Setup: identify local vs remote, prepare headers ──
|
|
37
55
|
|
|
38
|
-
|
|
39
|
-
|
|
56
|
+
if ((local_key instanceof URL) === (remote_url instanceof URL))
|
|
57
|
+
throw new Error(`one parameter should be local string key, and the other a remote URL object`)
|
|
40
58
|
|
|
41
|
-
//
|
|
42
|
-
|
|
59
|
+
// Normalize so local_key is the string, remote_url is the URL
|
|
60
|
+
if (local_key instanceof URL) { var swap = local_key; local_key = remote_url; remote_url = swap }
|
|
43
61
|
|
|
44
|
-
//
|
|
62
|
+
// Split caller's headers into GET headers (Accept) vs PUT headers (Content-Type)
|
|
45
63
|
var content_type
|
|
46
64
|
var get_headers = {}
|
|
47
65
|
var put_headers = {}
|
|
48
66
|
if (options.headers) {
|
|
49
67
|
for (var [k, v] of Object.entries(options.headers)) {
|
|
50
68
|
var lk = k.toLowerCase()
|
|
51
|
-
if (lk === 'accept' || lk === 'content-type')
|
|
69
|
+
if (lk === 'accept' || lk === 'content-type')
|
|
52
70
|
content_type = v
|
|
53
|
-
|
|
71
|
+
else {
|
|
54
72
|
get_headers[k] = v
|
|
55
73
|
put_headers[k] = v
|
|
56
74
|
}
|
|
@@ -61,7 +79,13 @@ function create_braid_text() {
|
|
|
61
79
|
put_headers['Content-Type'] = content_type
|
|
62
80
|
}
|
|
63
81
|
|
|
64
|
-
|
|
82
|
+
// ── Load the local resource and initialize the fork point ──
|
|
83
|
+
//
|
|
84
|
+
// The fork point is the most recent set of versions that both local
|
|
85
|
+
// and remote are known to share. It's persisted in resource meta
|
|
86
|
+
// so reconnections don't start from scratch.
|
|
87
|
+
|
|
88
|
+
var resource = (typeof local_key == 'string') ? await get_resource(local_key) : local_key
|
|
65
89
|
await ensure_dt_exists(resource)
|
|
66
90
|
|
|
67
91
|
if (!resource.meta.fork_point && options.fork_point_hint) {
|
|
@@ -69,19 +93,19 @@ function create_braid_text() {
|
|
|
69
93
|
resource.save_meta()
|
|
70
94
|
}
|
|
71
95
|
|
|
96
|
+
// Given a version frontier, incorporate a new update (version + parents)
|
|
97
|
+
// to compute the new frontier. Walks the DT version DAG if needed.
|
|
72
98
|
function extend_frontier(frontier, version, parents) {
|
|
73
|
-
// special case:
|
|
74
|
-
// if current frontier has all parents,
|
|
75
|
-
// then we can just remove those
|
|
76
|
-
// and add version
|
|
77
99
|
var frontier_set = new Set(frontier)
|
|
100
|
+
// Fast path: if the frontier contains all the update's parents,
|
|
101
|
+
// just swap them out for the new version
|
|
78
102
|
if (parents.length &&
|
|
79
103
|
parents.every(p => frontier_set.has(p))) {
|
|
80
104
|
parents.forEach(p => frontier_set.delete(p))
|
|
81
105
|
for (var event of version) frontier_set.add(event)
|
|
82
106
|
frontier = [...frontier_set.values()]
|
|
83
107
|
} else {
|
|
84
|
-
// full
|
|
108
|
+
// Slow path: walk the full DT history to compute the frontier
|
|
85
109
|
var looking_for = frontier_set
|
|
86
110
|
for (var event of version) looking_for.add(event)
|
|
87
111
|
|
|
@@ -104,70 +128,54 @@ function create_braid_text() {
|
|
|
104
128
|
return frontier.sort()
|
|
105
129
|
}
|
|
106
130
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
131
|
+
function extend_fork_point(update) {
|
|
132
|
+
resource.meta.fork_point =
|
|
133
|
+
extend_frontier(resource.meta.fork_point,
|
|
134
|
+
update.version, update.parents)
|
|
135
|
+
resource.save_meta()
|
|
136
|
+
}
|
|
111
137
|
|
|
112
|
-
|
|
113
|
-
|
|
138
|
+
// ── Reconnection wrapper ──
|
|
139
|
+
//
|
|
140
|
+
// Everything below runs inside reconnector(), which retries
|
|
141
|
+
// the entire connection on failure with backoff.
|
|
114
142
|
|
|
115
143
|
reconnector(options.signal, (_e, count) => {
|
|
116
|
-
// DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
|
|
117
|
-
options.do_investigating_disconnects_log_L04LPFHQ1M?.(a, 'braid-text handle_error: ' + _e)
|
|
118
|
-
|
|
119
144
|
var delay = Math.min(count, 3) * 1000
|
|
120
|
-
console.log(`disconnected from ${
|
|
145
|
+
console.log(`disconnected from ${remote_url}, retrying in ${delay}ms`)
|
|
121
146
|
return delay
|
|
122
147
|
}, async (signal, handle_error) => {
|
|
123
|
-
// DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
|
|
124
|
-
options.do_investigating_disconnects_log_L04LPFHQ1M?.(a, 'braid-text connect before on_pre_connect')
|
|
125
|
-
|
|
126
148
|
if (options.on_pre_connect) await options.on_pre_connect()
|
|
127
149
|
if (signal.aborted) return
|
|
128
150
|
|
|
129
|
-
// DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
|
|
130
|
-
options.do_investigating_disconnects_log_L04LPFHQ1M?.(a, 'braid-text connect before fork-point stuff')
|
|
131
|
-
|
|
132
151
|
try {
|
|
133
|
-
// fork
|
|
152
|
+
// ── Find the fork point ──
|
|
153
|
+
//
|
|
154
|
+
// The fork point tells us where to start syncing from.
|
|
155
|
+
// First check if the remote still has our saved fork point.
|
|
156
|
+
// If not, binary search through local history to find the
|
|
157
|
+
// latest version the remote recognizes.
|
|
158
|
+
|
|
134
159
|
async function check_version(version) {
|
|
135
|
-
var r = await braid_fetch(
|
|
136
|
-
signal,
|
|
137
|
-
method: "HEAD",
|
|
138
|
-
version,
|
|
139
|
-
headers: get_headers
|
|
160
|
+
var r = await braid_fetch(remote_url.href, {
|
|
161
|
+
signal, method: 'HEAD', version, headers: get_headers
|
|
140
162
|
})
|
|
141
163
|
if (signal.aborted) return
|
|
142
|
-
|
|
143
|
-
if (!r.ok && r.status !== 309 && r.status !== 500)
|
|
164
|
+
if (!r.ok && r.status !== 309 && r.status !== 404 && r.status !== 500)
|
|
144
165
|
throw new Error(`unexpected HEAD status: ${r.status}`)
|
|
145
166
|
return r.ok
|
|
146
167
|
}
|
|
147
168
|
|
|
148
|
-
function extend_fork_point(update) {
|
|
149
|
-
resource.meta.fork_point =
|
|
150
|
-
extend_frontier(resource.meta.fork_point,
|
|
151
|
-
update.version, update.parents)
|
|
152
|
-
resource.save_meta()
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
// see if remote has the fork point
|
|
156
169
|
if (resource.meta.fork_point &&
|
|
157
170
|
!(await check_version(resource.meta.fork_point))) {
|
|
158
171
|
if (signal.aborted) return
|
|
159
|
-
|
|
160
172
|
resource.meta.fork_point = null
|
|
161
173
|
resource.save_meta()
|
|
162
174
|
}
|
|
163
175
|
if (signal.aborted) return
|
|
164
176
|
|
|
165
|
-
// otherwise let's binary search for new fork point..
|
|
166
177
|
if (!resource.meta.fork_point) {
|
|
167
|
-
|
|
168
|
-
// DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
|
|
169
|
-
options.do_investigating_disconnects_log_L04LPFHQ1M?.(a, 'braid-text fork-point binary search')
|
|
170
|
-
|
|
178
|
+
// Binary search through local DT history
|
|
171
179
|
var bytes = resource.dt.doc.toBytes()
|
|
172
180
|
var [_, events, __] = braid_text.dt_parse([...bytes])
|
|
173
181
|
events = events.map(x => x.join('-'))
|
|
@@ -175,121 +183,87 @@ function create_braid_text() {
|
|
|
175
183
|
var min = -1
|
|
176
184
|
var max = events.length
|
|
177
185
|
while (min + 1 < max) {
|
|
178
|
-
var i = Math.floor((min + max)/2)
|
|
186
|
+
var i = Math.floor((min + max) / 2)
|
|
179
187
|
var version = [events[i]]
|
|
180
188
|
if (await check_version(version)) {
|
|
181
189
|
if (signal.aborted) return
|
|
182
|
-
|
|
183
190
|
min = i
|
|
184
191
|
resource.meta.fork_point = version
|
|
185
192
|
} else max = i
|
|
186
193
|
}
|
|
187
194
|
}
|
|
188
195
|
|
|
189
|
-
//
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
var temp_acs = new Set()
|
|
195
|
-
signal.addEventListener('abort', () => {
|
|
196
|
-
for (var t of temp_acs) t.abort()
|
|
197
|
-
})
|
|
196
|
+
// ── Local → Remote ──
|
|
197
|
+
//
|
|
198
|
+
// Subscribe to local changes (history since fork_point,
|
|
199
|
+
// then live updates) and forward each one to the remote
|
|
200
|
+
// server via PUT. Up to 10 PUTs in flight for throughput.
|
|
198
201
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
options.do_investigating_disconnects_log_L04LPFHQ1M?.(a, 'braid-text send_out')
|
|
202
|
+
var local_updates = []
|
|
203
|
+
var in_flight = 0
|
|
204
|
+
var max_in_flight = 10
|
|
203
205
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
var
|
|
206
|
+
// PUT a single update to the remote server.
|
|
207
|
+
// Extends the fork point on success so the next
|
|
208
|
+
// reconnection starts from where we left off.
|
|
209
|
+
async function send_to_remote(update) {
|
|
210
|
+
var {response} = await braid_text.put(remote_url, {
|
|
211
|
+
...update,
|
|
212
|
+
signal,
|
|
213
|
+
dont_retry: true,
|
|
214
|
+
peer: options.peer,
|
|
215
|
+
headers: put_headers,
|
|
216
|
+
})
|
|
209
217
|
if (signal.aborted) return
|
|
210
218
|
|
|
211
|
-
|
|
212
|
-
options.do_investigating_disconnects_log_L04LPFHQ1M?.(a, 'braid-text send_out result: ' + x.ok)
|
|
213
|
-
|
|
214
|
-
if (x.ok) {
|
|
215
|
-
local_first_put()
|
|
219
|
+
if (response.ok)
|
|
216
220
|
extend_fork_point(update)
|
|
217
|
-
|
|
221
|
+
else if (response.status === 401 || response.status === 403)
|
|
218
222
|
await options.on_unauthorized?.()
|
|
219
|
-
|
|
223
|
+
else
|
|
224
|
+
throw new Error('failed to PUT: ' + response.status)
|
|
220
225
|
}
|
|
221
226
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
frontier = extend_frontier(frontier, u.version, u.parents)
|
|
233
|
-
|
|
234
|
-
var temp_ac = new AbortController()
|
|
235
|
-
temp_acs.add(temp_ac)
|
|
236
|
-
var get_options = {
|
|
237
|
-
signal: temp_ac.signal,
|
|
238
|
-
parents: frontier,
|
|
239
|
-
merge_type: 'dt',
|
|
240
|
-
peer: options.peer,
|
|
241
|
-
subscribe: u => u.version?.length && q.push(u)
|
|
242
|
-
}
|
|
243
|
-
await braid_text.get(a, get_options)
|
|
244
|
-
await get_options.my_subscribe_chain
|
|
227
|
+
// Forward pending local updates to the remote,
|
|
228
|
+
// up to max_in_flight concurrent PUTs.
|
|
229
|
+
// When each PUT completes, check for more work.
|
|
230
|
+
function send_local_updates() {
|
|
231
|
+
if (signal.aborted) return
|
|
232
|
+
while (local_updates.length && in_flight < max_in_flight) {
|
|
233
|
+
var update = local_updates.shift()
|
|
234
|
+
if (!update.version?.length) continue
|
|
235
|
+
in_flight++
|
|
236
|
+
send_to_remote(update).then(() => {
|
|
245
237
|
if (signal.aborted) return
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
}
|
|
249
|
-
while (q.length && in_flight.size < max_in_flight) {
|
|
250
|
-
let u = q.shift()
|
|
251
|
-
if (!u.version?.length) continue
|
|
252
|
-
in_flight.set(u.version[0], u)
|
|
253
|
-
void (async () => {
|
|
254
|
-
try {
|
|
255
|
-
await send_out(u)
|
|
256
|
-
if (signal.aborted) return
|
|
257
|
-
in_flight.delete(u.version[0])
|
|
258
|
-
setTimeout(send_pump, 0)
|
|
259
|
-
} catch (e) { handle_error(e) }
|
|
260
|
-
})()
|
|
261
|
-
}
|
|
262
|
-
} finally {
|
|
263
|
-
if (send_pump_lock > 1) setTimeout(send_pump, 0)
|
|
264
|
-
send_pump_lock = 0
|
|
238
|
+
in_flight--
|
|
239
|
+
if (local_updates.length) send_local_updates()
|
|
240
|
+
}).catch(handle_error)
|
|
265
241
|
}
|
|
266
242
|
}
|
|
267
243
|
|
|
268
|
-
|
|
244
|
+
braid_text.get(local_key, {
|
|
269
245
|
signal,
|
|
270
246
|
merge_type: 'dt',
|
|
271
247
|
peer: options.peer,
|
|
248
|
+
...resource.meta.fork_point && {parents: resource.meta.fork_point},
|
|
272
249
|
subscribe: update => {
|
|
273
250
|
if (signal.aborted) return
|
|
274
251
|
if (update.version?.length) {
|
|
275
|
-
|
|
276
|
-
|
|
252
|
+
local_updates.push(update)
|
|
253
|
+
send_local_updates()
|
|
277
254
|
}
|
|
278
255
|
}
|
|
279
|
-
}
|
|
280
|
-
if (resource.meta.fork_point)
|
|
281
|
-
a_ops.parents = resource.meta.fork_point
|
|
282
|
-
braid_text.get(a, a_ops)
|
|
256
|
+
})
|
|
283
257
|
|
|
284
|
-
//
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
258
|
+
// ── Remote → Local ──
|
|
259
|
+
//
|
|
260
|
+
// Subscribe to remote changes and apply them locally via .put().
|
|
261
|
+
// Requests DT binary encoding for efficiency.
|
|
288
262
|
|
|
289
|
-
|
|
290
|
-
|
|
263
|
+
var remote_current_version = null
|
|
264
|
+
var remote_status = null
|
|
291
265
|
|
|
292
|
-
|
|
266
|
+
await braid_text.get(remote_url, {
|
|
293
267
|
signal,
|
|
294
268
|
dont_retry: true,
|
|
295
269
|
headers: { ...get_headers, 'Merge-Type': 'dt', 'accept-encoding': 'updates(dt)' },
|
|
@@ -297,33 +271,31 @@ function create_braid_text() {
|
|
|
297
271
|
peer: options.peer,
|
|
298
272
|
heartbeats: 120,
|
|
299
273
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
274
|
+
on_response: res => {
|
|
275
|
+
remote_status = res.status
|
|
276
|
+
remote_current_version = res.headers.get('current-version')
|
|
277
|
+
options.on_res?.(res)
|
|
303
278
|
},
|
|
304
279
|
|
|
305
280
|
subscribe: async update => {
|
|
306
|
-
|
|
307
|
-
// DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
|
|
308
|
-
options.do_investigating_disconnects_log_L04LPFHQ1M?.(a, 'braid-text got update')
|
|
309
|
-
|
|
310
|
-
// Wait for remote_res to be available
|
|
311
|
-
await remote_res_promise
|
|
312
281
|
if (signal.aborted) return
|
|
313
282
|
|
|
314
|
-
// Check if this is a dt-encoded update
|
|
315
283
|
if (update.extra_headers?.encoding === 'dt') {
|
|
316
|
-
|
|
317
|
-
await braid_text.put(
|
|
284
|
+
// DT binary: apply directly
|
|
285
|
+
await braid_text.put(local_key, {
|
|
318
286
|
body: update.body,
|
|
319
287
|
transfer_encoding: 'dt',
|
|
320
288
|
peer: options.peer
|
|
321
289
|
})
|
|
322
290
|
if (signal.aborted) return
|
|
323
|
-
if (
|
|
291
|
+
if (remote_current_version) extend_fork_point({
|
|
292
|
+
version: JSON.parse(`[${remote_current_version}]`),
|
|
293
|
+
parents: resource.meta.fork_point || []
|
|
294
|
+
})
|
|
324
295
|
} else {
|
|
296
|
+
// Text patches: forward as-is
|
|
325
297
|
if (options.peer) update.peer = options.peer
|
|
326
|
-
await braid_text.put(
|
|
298
|
+
await braid_text.put(local_key, update)
|
|
327
299
|
if (signal.aborted) return
|
|
328
300
|
if (update.version) extend_fork_point(update)
|
|
329
301
|
}
|
|
@@ -332,33 +304,27 @@ function create_braid_text() {
|
|
|
332
304
|
options.on_disconnect?.()
|
|
333
305
|
handle_error(e)
|
|
334
306
|
}
|
|
335
|
-
}
|
|
336
|
-
// Handle case where remote doesn't exist yet - wait for local to create it
|
|
337
|
-
remote_res = await braid_text.get(b, b_ops)
|
|
307
|
+
})
|
|
338
308
|
if (signal.aborted) return
|
|
339
309
|
|
|
340
|
-
//
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
if (remote_res === null) {
|
|
345
|
-
// Remote doesn't exist yet, wait for local to put something
|
|
346
|
-
await local_first_put_promise
|
|
347
|
-
return handle_error(new Error('try again'))
|
|
310
|
+
// If remote returned 404, reconnect with backoff
|
|
311
|
+
// (the resource might be created later by our local→remote PUTs)
|
|
312
|
+
if (remote_status === 404) {
|
|
313
|
+
return handle_error(new Error('remote returned 404'))
|
|
348
314
|
}
|
|
349
|
-
options.on_res?.(remote_res)
|
|
350
|
-
// on_error will call handle_error when connection drops
|
|
351
315
|
} catch (e) { handle_error(e) }
|
|
352
316
|
})
|
|
353
317
|
}
|
|
354
318
|
|
|
355
319
|
braid_text.serve = async (req, res, options = {}) => {
|
|
356
320
|
options = {
|
|
357
|
-
key: req.url.split('?')[0],
|
|
358
|
-
put_cb: (key, val, params) => { },
|
|
359
|
-
...options
|
|
321
|
+
key: req.url.split('?')[0],
|
|
322
|
+
put_cb: (key, val, params) => { },
|
|
323
|
+
...options
|
|
360
324
|
}
|
|
361
325
|
|
|
326
|
+
// ── Setup: prepare the response and load the resource ──
|
|
327
|
+
|
|
362
328
|
if (braid_text.cors !== false) braid_text.free_cors(res)
|
|
363
329
|
|
|
364
330
|
function my_end(statusCode, x, statusText, headers) {
|
|
@@ -366,60 +332,61 @@ function create_braid_text() {
|
|
|
366
332
|
res.end(x ?? '')
|
|
367
333
|
}
|
|
368
334
|
|
|
369
|
-
|
|
335
|
+
var resource = null
|
|
370
336
|
try {
|
|
371
337
|
resource = await get_resource(options.key)
|
|
372
338
|
|
|
339
|
+
// Add braid protocol support to the req/res objects
|
|
373
340
|
braidify(req, res)
|
|
374
341
|
if (res.is_multiplexer) return
|
|
375
342
|
|
|
376
|
-
// Sort version arrays from external sources
|
|
377
343
|
if (req.version) req.version.sort()
|
|
378
344
|
if (req.parents) req.parents.sort()
|
|
379
345
|
} catch (e) {
|
|
380
|
-
return my_end(500,
|
|
346
|
+
return my_end(500, 'The server failed to process this request. The error generated was: ' + e)
|
|
381
347
|
}
|
|
382
348
|
|
|
383
|
-
|
|
349
|
+
// ── Cursors get their own content-type and are handled independently ──
|
|
350
|
+
if (await handle_cursors(resource, req, res)) return
|
|
384
351
|
|
|
385
|
-
//
|
|
386
|
-
if (await handle_cursors(resource, req, res))
|
|
387
|
-
return
|
|
352
|
+
// ── Classify the request ──
|
|
388
353
|
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
if (merge_type !== 'simpleton' && merge_type !== 'dt'
|
|
354
|
+
var peer = req.headers['peer'],
|
|
355
|
+
merge_type = req.headers['merge-type'] || 'simpleton'
|
|
356
|
+
if (merge_type !== 'simpleton' && merge_type !== 'dt' && merge_type !== 'yjs')
|
|
357
|
+
return my_end(400, `Unknown merge type: ${merge_type}`)
|
|
392
358
|
|
|
393
|
-
|
|
394
|
-
|
|
359
|
+
var is_read = req.method === 'GET' || req.method === 'HEAD',
|
|
360
|
+
is_write = req.method === 'PUT' || req.method === 'POST' || req.method === 'PATCH',
|
|
361
|
+
is_head = req.method === 'HEAD'
|
|
395
362
|
|
|
396
|
-
//
|
|
397
|
-
// we want to set the charset to utf-8
|
|
398
|
-
const contentType = res.getHeader('Content-Type')
|
|
399
|
-
const parsedContentType = contentType.split(';').map(part => part.trim())
|
|
400
|
-
const charsetParam = parsedContentType.find(part => part.toLowerCase().startsWith('charset='))
|
|
401
|
-
if (!charsetParam)
|
|
402
|
-
res.setHeader('Content-Type', `${contentType}; charset=utf-8`)
|
|
403
|
-
else if (charsetParam.toLowerCase() !== 'charset=utf-8') {
|
|
404
|
-
// Replace the existing charset with utf-8
|
|
405
|
-
const updatedContentType = parsedContentType
|
|
406
|
-
.map(part => (part.toLowerCase().startsWith('charset=') ? 'charset=utf-8' : part))
|
|
407
|
-
.join('; ');
|
|
408
|
-
res.setHeader('Content-Type', updatedContentType);
|
|
409
|
-
}
|
|
363
|
+
// ── Ensure the response is labeled as utf-8 text ──
|
|
410
364
|
|
|
411
|
-
if (
|
|
412
|
-
|
|
413
|
-
|
|
365
|
+
if (!res.getHeader('content-type')) res.setHeader('Content-Type', 'text/plain')
|
|
366
|
+
var ct = res.getHeader('Content-Type'),
|
|
367
|
+
ct_parts = ct.split(';').map(p => p.trim())
|
|
368
|
+
var charset = ct_parts.find(p => p.toLowerCase().startsWith('charset='))
|
|
369
|
+
if (!charset)
|
|
370
|
+
res.setHeader('Content-Type', `${ct}; charset=utf-8`)
|
|
371
|
+
else if (charset.toLowerCase() !== 'charset=utf-8')
|
|
372
|
+
res.setHeader('Content-Type', ct_parts
|
|
373
|
+
.map(p => p.toLowerCase().startsWith('charset=') ? 'charset=utf-8' : p)
|
|
374
|
+
.join('; '))
|
|
375
|
+
|
|
376
|
+
// ── Handle simple methods that don't need further processing ──
|
|
377
|
+
|
|
378
|
+
if (req.method === 'OPTIONS') return my_end(200)
|
|
379
|
+
if (req.method === 'DELETE') {
|
|
414
380
|
await braid_text.delete(resource)
|
|
415
381
|
return my_end(200)
|
|
416
382
|
}
|
|
417
383
|
|
|
418
|
-
var
|
|
419
|
-
resource.version.map(x => JSON.stringify(x)).join(
|
|
384
|
+
var current_version = () => ascii_ify(
|
|
385
|
+
resource.version.map(x => JSON.stringify(x)).join(', '))
|
|
420
386
|
|
|
421
|
-
|
|
422
|
-
|
|
387
|
+
// ── Read state (with GET or HEAD) ──
|
|
388
|
+
if (is_read) {
|
|
389
|
+
// Validate requested versions exist
|
|
423
390
|
var unknowns = []
|
|
424
391
|
for (var event of (req.version || []).concat(req.parents || [])) {
|
|
425
392
|
var [actor, seq] = decode_version(event)
|
|
@@ -427,185 +394,212 @@ function create_braid_text() {
|
|
|
427
394
|
unknowns.push(event)
|
|
428
395
|
}
|
|
429
396
|
if (unknowns.length)
|
|
430
|
-
return my_end(309, '',
|
|
397
|
+
return my_end(309, '', 'Version Unknown Here', {
|
|
431
398
|
Version: ascii_ify(unknowns.map(e => JSON.stringify(e)).join(', '))
|
|
432
399
|
})
|
|
433
400
|
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
401
|
+
var has_parents = req.parents && req.parents.length > 0
|
|
402
|
+
var has_version = req.version && req.version.length > 0
|
|
403
|
+
|
|
404
|
+
if (req.subscribe && has_version)
|
|
405
|
+
return my_end(400, 'Version header is not allowed with Subscribe — use Parents instead')
|
|
406
|
+
|
|
407
|
+
var getting = {
|
|
408
|
+
subscribe: !!req.subscribe,
|
|
409
|
+
history: (has_parents && v_eq(req.parents, resource.version)) ? false
|
|
410
|
+
: has_parents ? 'since-parents'
|
|
411
|
+
: (req.subscribe || req.parents || req.headers['accept-transfer-encoding']) ? 'up-to-version'
|
|
412
|
+
: false,
|
|
413
|
+
transfer_encoding: req.headers['accept-transfer-encoding'],
|
|
414
|
+
}
|
|
415
|
+
getting.single_snapshot = !getting.subscribe && !getting.history
|
|
416
|
+
|
|
417
|
+
// Response headers
|
|
418
|
+
if (getting.subscribe && !res.hasHeader('editable'))
|
|
419
|
+
// BUG: This shouldn't be guarded behind "subscribe" because
|
|
420
|
+
// clients can also edit text without subscribing to it, just
|
|
421
|
+
// by doing PUTs and polling to see the updates to the state.
|
|
422
|
+
//
|
|
423
|
+
// But this should only be editable if the client can actually
|
|
424
|
+
// edit it, and so I am guessing that whoever wrote this might
|
|
425
|
+
// have been actually trying to guard something that happens
|
|
426
|
+
// to correlate with subscriptions, which is bogus, but needs
|
|
427
|
+
// to be thought through and fixed.
|
|
428
|
+
res.setHeader('Editable', 'true')
|
|
429
|
+
res.setHeader('Current-Version', current_version())
|
|
430
|
+
res.setHeader('Merge-Type', merge_type)
|
|
431
|
+
res.setHeader('Accept-Subscribe', 'true')
|
|
432
|
+
|
|
433
|
+
// HEAD: headers only, no body needed
|
|
434
|
+
if (is_head) {
|
|
435
|
+
// Always include the version of what would be returned
|
|
436
|
+
if (!getting.history)
|
|
437
|
+
res.setHeader('Version', current_version())
|
|
438
|
+
return my_end(200)
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
if (!getting.subscribe) {
|
|
442
|
+
// ── One-shot read ──
|
|
454
443
|
try {
|
|
455
|
-
|
|
444
|
+
var result = await braid_text.get(resource, {
|
|
456
445
|
version: req.version,
|
|
457
446
|
parents: req.parents,
|
|
458
|
-
transfer_encoding:
|
|
447
|
+
transfer_encoding: getting.transfer_encoding,
|
|
448
|
+
full_response: true,
|
|
459
449
|
})
|
|
460
450
|
} catch (e) {
|
|
461
|
-
return my_end(500,
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
if (
|
|
465
|
-
res.setHeader(
|
|
466
|
-
res.setHeader(
|
|
467
|
-
return my_end(209,
|
|
451
|
+
return my_end(500, 'The server at ' + resource + ' failed: ' + e)
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
if (getting.transfer_encoding === 'dt') {
|
|
455
|
+
res.setHeader('X-Transfer-Encoding', 'dt')
|
|
456
|
+
res.setHeader('Content-Length', result.body.length)
|
|
457
|
+
return my_end(209, result.body, 'Multiresponse')
|
|
458
|
+
} else if (Array.isArray(result)) {
|
|
459
|
+
// Range of history: send as 209 Multiresponse
|
|
460
|
+
res.startSubscription()
|
|
461
|
+
for (var u of result)
|
|
462
|
+
res.sendVersion({
|
|
463
|
+
version: [u.version],
|
|
464
|
+
parents: u.parents,
|
|
465
|
+
patches: [{ unit: u.unit, range: u.range, content: u.content }],
|
|
466
|
+
})
|
|
467
|
+
return res.end()
|
|
468
468
|
} else {
|
|
469
|
-
res.setHeader(
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
469
|
+
res.setHeader('Version', ascii_ify(result.version
|
|
470
|
+
.map(v => JSON.stringify(v))
|
|
471
|
+
.join(', ')))
|
|
472
|
+
var buffer = Buffer.from(result.body, 'utf8')
|
|
473
|
+
res.setHeader('Repr-Digest', get_digest(buffer))
|
|
474
|
+
res.setHeader('Content-Length', buffer.length)
|
|
475
|
+
return my_end(200, buffer)
|
|
474
476
|
}
|
|
475
477
|
} else {
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
if (req.method == "HEAD") return my_end(200)
|
|
480
|
-
|
|
481
|
-
let options = {
|
|
482
|
-
peer,
|
|
483
|
-
version: req.version,
|
|
484
|
-
parents: req.parents,
|
|
485
|
-
merge_type,
|
|
486
|
-
accept_encoding:
|
|
487
|
-
req.headers['x-accept-encoding'] ??
|
|
488
|
-
req.headers['accept-encoding'],
|
|
489
|
-
subscribe: update => {
|
|
490
|
-
|
|
491
|
-
// this is a sanity/rhobustness check..
|
|
492
|
-
// ..this digest is checked on the client..
|
|
493
|
-
if (update.version && v_eq(update.version, resource.version))
|
|
494
|
-
update["Repr-Digest"] = get_digest(resource.val)
|
|
495
|
-
|
|
496
|
-
if (update.patches && update.patches.length === 1) {
|
|
497
|
-
update.patch = update.patches[0]
|
|
498
|
-
delete update.patches
|
|
499
|
-
}
|
|
500
|
-
res.sendVersion(update)
|
|
501
|
-
},
|
|
502
|
-
write: (update) => res.write(update)
|
|
503
|
-
}
|
|
504
|
-
|
|
478
|
+
// ── Subscribe ──
|
|
479
|
+
all_subscriptions.add(res)
|
|
480
|
+
var aborter = new AbortController()
|
|
505
481
|
res.startSubscription({
|
|
506
482
|
onClose: () => {
|
|
507
|
-
|
|
508
|
-
|
|
483
|
+
all_subscriptions.delete(res)
|
|
484
|
+
aborter.abort()
|
|
509
485
|
}
|
|
510
486
|
})
|
|
511
487
|
|
|
512
488
|
try {
|
|
513
|
-
|
|
489
|
+
await braid_text.get(resource, {
|
|
490
|
+
peer,
|
|
491
|
+
version: req.version,
|
|
492
|
+
parents: req.parents,
|
|
493
|
+
merge_type,
|
|
494
|
+
signal: aborter.signal,
|
|
495
|
+
accept_encoding:
|
|
496
|
+
req.headers['x-accept-encoding'] ??
|
|
497
|
+
req.headers['accept-encoding'],
|
|
498
|
+
subscribe: update => {
|
|
499
|
+
// Add digest for integrity checking on the client
|
|
500
|
+
if (update.version && v_eq(update.version, resource.version))
|
|
501
|
+
update['Repr-Digest'] = get_digest(resource.val)
|
|
502
|
+
|
|
503
|
+
// Collapse single-element patches array for HTTP
|
|
504
|
+
if (update.patches && update.patches.length === 1) {
|
|
505
|
+
update.patch = update.patches[0]
|
|
506
|
+
delete update.patches
|
|
507
|
+
}
|
|
508
|
+
res.sendVersion(update)
|
|
509
|
+
},
|
|
510
|
+
})
|
|
511
|
+
// Ensure headers are sent even if .get() didn't send
|
|
512
|
+
// any initial data (e.g. subscribe when already current)
|
|
513
|
+
res.flushHeaders()
|
|
514
514
|
} catch (e) {
|
|
515
|
-
return my_end(500,
|
|
515
|
+
return my_end(500, 'The server failed to get something. The error generated was: ' + e)
|
|
516
516
|
}
|
|
517
|
+
return
|
|
517
518
|
}
|
|
518
519
|
}
|
|
519
520
|
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
return my_end(503,
|
|
524
|
-
}
|
|
521
|
+
// ── Write (PUT / POST / PATCH) ──
|
|
522
|
+
if (is_write) {
|
|
523
|
+
if (waiting_puts >= 100)
|
|
524
|
+
return my_end(503, 'The server is busy.')
|
|
525
525
|
|
|
526
526
|
waiting_puts++
|
|
527
|
-
|
|
528
|
-
let done_my_turn = (statusCode, x, statusText, headers) => {
|
|
527
|
+
var done_my_turn = (statusCode, x, statusText, headers) => {
|
|
529
528
|
waiting_puts--
|
|
530
|
-
if (braid_text.verbose) console.log(`waiting_puts(after--) = ${waiting_puts}`)
|
|
531
529
|
my_end(statusCode, x, statusText, headers)
|
|
532
530
|
}
|
|
533
531
|
|
|
534
532
|
try {
|
|
533
|
+
// Parse patches from request body
|
|
535
534
|
var patches = await req.patches()
|
|
536
|
-
for (
|
|
535
|
+
for (var p of patches) p.content = p.content_text
|
|
537
536
|
|
|
538
|
-
|
|
537
|
+
var body = null
|
|
539
538
|
if (patches[0]?.unit === 'everything') {
|
|
540
539
|
body = patches[0].content
|
|
541
540
|
patches = null
|
|
542
541
|
}
|
|
543
542
|
|
|
543
|
+
// Wait for parent versions to arrive (if needed)
|
|
544
544
|
if (req.parents) {
|
|
545
545
|
await ensure_dt_exists(resource)
|
|
546
546
|
await wait_for_events(
|
|
547
|
-
options.key,
|
|
548
|
-
req.parents,
|
|
547
|
+
options.key, req.parents,
|
|
549
548
|
resource.dt.known_versions,
|
|
550
|
-
// approximation of memory usage for this update
|
|
551
549
|
body != null ? body.length :
|
|
552
550
|
patches.reduce((a, b) => a + b.range.length + b.content.length, 0),
|
|
553
551
|
options.recv_buffer_max_time,
|
|
554
552
|
options.recv_buffer_max_space)
|
|
555
553
|
|
|
556
|
-
// make sure we have the necessary parents now
|
|
557
554
|
var unknowns = []
|
|
558
555
|
for (var event of req.parents) {
|
|
559
556
|
var [actor, seq] = decode_version(event)
|
|
560
|
-
if (!resource.dt.known_versions[actor]?.has(seq))
|
|
557
|
+
if (!resource.dt.known_versions[actor]?.has(seq))
|
|
558
|
+
unknowns.push(event)
|
|
561
559
|
}
|
|
562
560
|
if (unknowns.length)
|
|
563
|
-
return done_my_turn(309, '',
|
|
561
|
+
return done_my_turn(309, '', 'Version Unknown Here', {
|
|
564
562
|
Version: ascii_ify(unknowns.map(e => JSON.stringify(e)).join(', ')),
|
|
565
563
|
'Retry-After': '1'
|
|
566
564
|
})
|
|
567
565
|
}
|
|
568
566
|
|
|
567
|
+
// Apply the edit
|
|
569
568
|
var old_val = resource.val
|
|
570
569
|
var old_version = resource.version
|
|
571
|
-
var put_patches = patches?.map(p => ({
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
570
|
+
var put_patches = patches?.map(p => ({
|
|
571
|
+
unit: p.unit, range: p.range, content: p.content
|
|
572
|
+
})) || null
|
|
573
|
+
|
|
574
|
+
var {dt: {change_count}} = await braid_text.put(resource, {
|
|
575
|
+
peer, version: req.version, parents: req.parents,
|
|
576
|
+
patches, body, merge_type
|
|
577
|
+
})
|
|
578
578
|
|
|
579
|
-
//
|
|
580
|
-
// and the request version is also our new current version,
|
|
581
|
-
// then verify the digest..
|
|
579
|
+
// Verify Repr-Digest if present
|
|
582
580
|
if (req.headers['repr-digest'] &&
|
|
583
581
|
v_eq(req.version, resource.version) &&
|
|
584
|
-
req.headers['repr-digest'] !== get_digest(resource.val))
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
// we return a special 550 error code,
|
|
588
|
-
// which simpleton will pick up on to stop retrying
|
|
589
|
-
return done_my_turn(550, "repr-digest mismatch!")
|
|
590
|
-
}
|
|
582
|
+
req.headers['repr-digest'] !== get_digest(resource.val))
|
|
583
|
+
return done_my_turn(550, 'repr-digest mismatch!')
|
|
591
584
|
|
|
592
585
|
if (req.version?.length)
|
|
593
586
|
got_event(options.key, req.version[0], change_count)
|
|
594
|
-
|
|
595
|
-
res.setHeader("Version", get_current_version())
|
|
596
587
|
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
588
|
+
res.setHeader('Version', current_version())
|
|
589
|
+
|
|
590
|
+
options.put_cb(options.key, resource.val, {
|
|
591
|
+
old_val, patches: put_patches,
|
|
592
|
+
version: resource.version, parents: old_version
|
|
593
|
+
})
|
|
600
594
|
} catch (e) {
|
|
601
595
|
console.log(`${req.method} ERROR: ${e.stack}`)
|
|
602
|
-
return done_my_turn(500,
|
|
596
|
+
return done_my_turn(500, 'The server failed to apply this version. The error generated was: ' + e)
|
|
603
597
|
}
|
|
604
598
|
|
|
605
599
|
return done_my_turn(200)
|
|
606
600
|
}
|
|
607
601
|
|
|
608
|
-
throw new Error(
|
|
602
|
+
throw new Error('unknown method: ' + req.method)
|
|
609
603
|
}
|
|
610
604
|
|
|
611
605
|
braid_text.delete = async (key, options) => {
|
|
@@ -628,6 +622,42 @@ function create_braid_text() {
|
|
|
628
622
|
await resource.delete()
|
|
629
623
|
}
|
|
630
624
|
|
|
625
|
+
// Fetch from a remote braid-text server via HTTP
|
|
626
|
+
async function get_remote(url, options) {
|
|
627
|
+
if (!options) options = {}
|
|
628
|
+
|
|
629
|
+
var params = {
|
|
630
|
+
signal: options.signal,
|
|
631
|
+
subscribe: !!options.subscribe,
|
|
632
|
+
heartbeats: options.heartbeats ?? 120,
|
|
633
|
+
heartbeat_cb: options.heartbeat_cb
|
|
634
|
+
}
|
|
635
|
+
if (!options.dont_retry)
|
|
636
|
+
params.retry = (res) => res.status !== 404
|
|
637
|
+
|
|
638
|
+
for (var x of ['headers', 'parents', 'version', 'peer'])
|
|
639
|
+
if (options[x] != null) params[x] = options[x]
|
|
640
|
+
|
|
641
|
+
var res = await braid_fetch(url.href, params)
|
|
642
|
+
|
|
643
|
+
if (options.on_response) options.on_response(res)
|
|
644
|
+
|
|
645
|
+
if (res.status === 404) return ''
|
|
646
|
+
|
|
647
|
+
if (options.subscribe) {
|
|
648
|
+
res.subscribe(async update => {
|
|
649
|
+
// Convert binary to text, except for dt-encoded blobs
|
|
650
|
+
// which are passed through as-is for .sync() to handle
|
|
651
|
+
if (update.extra_headers?.encoding !== 'dt') {
|
|
652
|
+
update.body = update.body_text
|
|
653
|
+
if (update.patches)
|
|
654
|
+
for (var p of update.patches) p.content = p.content_text
|
|
655
|
+
}
|
|
656
|
+
await options.subscribe(update)
|
|
657
|
+
}, e => options.on_error?.(e))
|
|
658
|
+
} else return await res.text()
|
|
659
|
+
}
|
|
660
|
+
|
|
631
661
|
braid_text.get = async (key, options) => {
|
|
632
662
|
if (options && options.version) {
|
|
633
663
|
validate_version_array(options.version)
|
|
@@ -638,226 +668,198 @@ function create_braid_text() {
|
|
|
638
668
|
options.parents.sort()
|
|
639
669
|
}
|
|
640
670
|
|
|
641
|
-
if (key instanceof URL)
|
|
642
|
-
if (!options) options = {}
|
|
671
|
+
if (key instanceof URL) return await get_remote(key, options)
|
|
643
672
|
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
673
|
+
if (!options) options = {}
|
|
674
|
+
|
|
675
|
+
var resource = (typeof key == 'string') ? await get_resource(key) : key
|
|
676
|
+
var version = resource.version
|
|
677
|
+
var merge_type = options.range_unit === 'yjs-text' ? 'yjs'
|
|
678
|
+
: (options.merge_type || 'simpleton')
|
|
679
|
+
var has_parents = options.parents && options.parents.length > 0
|
|
680
|
+
var has_version = options.version && options.version.length > 0
|
|
681
|
+
|
|
682
|
+
if (options.subscribe && has_version)
|
|
683
|
+
throw new Error('version is not allowed with subscribe — use parents instead')
|
|
684
|
+
|
|
685
|
+
var getting = {
|
|
686
|
+
subscribe: !!options.subscribe,
|
|
687
|
+
// 'since-parents' = range of updates from parents to current
|
|
688
|
+
// 'up-to-version' = bring client up to current (from scratch)
|
|
689
|
+
// false = no history needed
|
|
690
|
+
history: (has_parents && v_eq(options.parents, version)) ? false
|
|
691
|
+
: has_parents ? 'since-parents'
|
|
692
|
+
: (options.subscribe || options.parents || options.transfer_encoding) ? 'up-to-version'
|
|
693
|
+
: false,
|
|
694
|
+
transfer_encoding: options.transfer_encoding,
|
|
695
|
+
}
|
|
696
|
+
getting.single_snapshot = !getting.subscribe && !getting.history
|
|
697
|
+
|
|
698
|
+
// Single snapshot: return the text (optionally at a specific version)
|
|
699
|
+
if (getting.single_snapshot) {
|
|
700
|
+
if (has_version) {
|
|
701
|
+
await ensure_dt_exists(resource)
|
|
702
|
+
return options.full_response
|
|
703
|
+
? { version: options.version, body: dt_get_string(resource.dt.doc, options.version) }
|
|
704
|
+
: dt_get_string(resource.dt.doc, options.version)
|
|
652
705
|
}
|
|
653
|
-
|
|
654
|
-
|
|
706
|
+
return options.full_response ? { version, body: resource.val } : resource.val
|
|
707
|
+
}
|
|
655
708
|
|
|
656
|
-
|
|
709
|
+
// DT binary encoding: a transport optimization usable by any merge type.
|
|
710
|
+
// Returns raw DT bytes instead of text.
|
|
711
|
+
if (getting.history && !getting.subscribe && getting.transfer_encoding === 'dt') {
|
|
712
|
+
// TODO: move this into the dt/simpleton merge_type cases below
|
|
713
|
+
await ensure_dt_exists(resource)
|
|
714
|
+
// If requesting the current version, skip the version lookup
|
|
715
|
+
// (faster than asking DT about a version we already have)
|
|
716
|
+
var req_version = options.version
|
|
717
|
+
if (req_version && v_eq(req_version, version)) req_version = null
|
|
718
|
+
|
|
719
|
+
var bytes = null
|
|
720
|
+
if (req_version || options.parents) {
|
|
721
|
+
if (req_version) {
|
|
722
|
+
var doc = dt_get(resource.dt.doc, req_version)
|
|
723
|
+
bytes = doc.toBytes()
|
|
724
|
+
} else {
|
|
725
|
+
bytes = resource.dt.doc.toBytes()
|
|
726
|
+
var doc = Doc.fromBytes(bytes)
|
|
727
|
+
}
|
|
728
|
+
if (options.parents)
|
|
729
|
+
bytes = doc.getPatchSince(
|
|
730
|
+
dt_get_local_version(bytes, options.parents))
|
|
731
|
+
doc.free()
|
|
732
|
+
} else bytes = resource.dt.doc.toBytes()
|
|
733
|
+
return { body: bytes }
|
|
734
|
+
}
|
|
657
735
|
|
|
658
|
-
|
|
736
|
+
switch (merge_type) {
|
|
659
737
|
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
// Don't convert to text for initial dt-encoded body (no status)
|
|
663
|
-
if (update.status) {
|
|
664
|
-
update.body = update.body_text
|
|
665
|
-
if (update.patches)
|
|
666
|
-
for (var p of update.patches) p.content = p.content_text
|
|
667
|
-
}
|
|
668
|
-
await options.subscribe(update)
|
|
669
|
-
}, e => options.on_error?.(e))
|
|
738
|
+
case 'yjs':
|
|
739
|
+
await ensure_yjs_exists(resource)
|
|
670
740
|
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
741
|
+
// Send history (for both one-shot and subscribe)
|
|
742
|
+
if (getting.history) {
|
|
743
|
+
if (getting.history === 'since-parents')
|
|
744
|
+
throw new Error('yjs-text from arbitrary parents not yet implemented')
|
|
674
745
|
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
if (!braid_text.cache[key]) return null
|
|
678
|
-
return (await get_resource(key)).val
|
|
679
|
-
}
|
|
746
|
+
var yjs_updates = braid_text.from_yjs_binary(
|
|
747
|
+
Y.encodeStateAsUpdate(resource.yjs.doc))
|
|
680
748
|
|
|
681
|
-
|
|
682
|
-
|
|
749
|
+
if (!getting.subscribe)
|
|
750
|
+
return yjs_updates
|
|
683
751
|
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
if (options.range_unit === 'yjs-text') {
|
|
687
|
-
await ensure_yjs_exists(resource)
|
|
688
|
-
if (options.parents && options.parents.length === 0) {
|
|
689
|
-
// Full history: root + initialization patch
|
|
690
|
-
var init_patches = [{
|
|
691
|
-
unit: 'yjs-text',
|
|
692
|
-
range: '(:)',
|
|
693
|
-
content: resource.val
|
|
694
|
-
}]
|
|
695
|
-
return {
|
|
696
|
-
version: ['999999999-' + (Math.max(0, [...resource.val].length - 1))],
|
|
697
|
-
parents: [],
|
|
698
|
-
patches: init_patches
|
|
699
|
-
}
|
|
700
|
-
}
|
|
701
|
-
return { version, body: resource.val }
|
|
752
|
+
for (var u of yjs_updates)
|
|
753
|
+
options.subscribe(u)
|
|
702
754
|
}
|
|
703
755
|
|
|
704
|
-
if (
|
|
705
|
-
//
|
|
706
|
-
//
|
|
707
|
-
//
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
bytes = resource.dt.doc.toBytes()
|
|
718
|
-
var doc = Doc.fromBytes(bytes)
|
|
719
|
-
}
|
|
720
|
-
if (options.parents) {
|
|
721
|
-
bytes = doc.getPatchSince(
|
|
722
|
-
dt_get_local_version(bytes, options.parents))
|
|
723
|
-
}
|
|
724
|
-
doc.free()
|
|
725
|
-
} else bytes = resource.dt.doc.toBytes()
|
|
726
|
-
return { body: bytes }
|
|
756
|
+
if (getting.subscribe) {
|
|
757
|
+
// Register for live updates
|
|
758
|
+
// NOTE: This stream mixes two version spaces:
|
|
759
|
+
// update.version: DT version space (frontier after this edit)
|
|
760
|
+
// update.patches[].version: Yjs version space (clientID-clock)
|
|
761
|
+
var client = {
|
|
762
|
+
merge_type: 'yjs',
|
|
763
|
+
peer: options.peer,
|
|
764
|
+
send_update: one_at_a_time(options.subscribe),
|
|
765
|
+
abort() { resource.yjs.clients.delete(client) },
|
|
766
|
+
}
|
|
767
|
+
resource.yjs.clients.add(client)
|
|
768
|
+
options.signal?.addEventListener('abort', () => client.abort())
|
|
727
769
|
}
|
|
770
|
+
break
|
|
728
771
|
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
return {
|
|
732
|
-
version: options.version || options.parents,
|
|
733
|
-
body: dt_get_string(resource.dt.doc, options.version || options.parents)
|
|
734
|
-
}
|
|
735
|
-
} else {
|
|
736
|
-
return { version, body: resource.val }
|
|
737
|
-
}
|
|
738
|
-
} else {
|
|
739
|
-
// yjs-text subscribe: send full history first, then live updates
|
|
740
|
-
if (options.range_unit === 'yjs-text') {
|
|
741
|
-
await ensure_yjs_exists(resource)
|
|
772
|
+
case 'simpleton':
|
|
773
|
+
await ensure_dt_exists(resource)
|
|
742
774
|
|
|
743
|
-
|
|
744
|
-
|
|
775
|
+
if (getting.history && !getting.subscribe)
|
|
776
|
+
return dt_get_patches(resource.dt.doc,
|
|
777
|
+
getting.history === 'since-parents' ? options.parents : undefined)
|
|
745
778
|
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
779
|
+
if (getting.subscribe) {
|
|
780
|
+
var client = {
|
|
781
|
+
merge_type: 'simpleton',
|
|
782
|
+
peer: options.peer,
|
|
783
|
+
send_update: one_at_a_time(options.subscribe),
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
// Send initial history
|
|
787
|
+
if (getting.history === 'up-to-version')
|
|
788
|
+
client.send_update({ version, parents: [], body: resource.val })
|
|
789
|
+
else if (getting.history === 'since-parents') {
|
|
790
|
+
var from = options.version || options.parents
|
|
791
|
+
var local_version = OpLog_remote_to_local(resource.dt.doc, from)
|
|
792
|
+
if (local_version)
|
|
793
|
+
client.send_update({
|
|
794
|
+
version, parents: from,
|
|
795
|
+
patches: get_xf_patches(resource.dt.doc, local_version)
|
|
796
|
+
})
|
|
753
797
|
}
|
|
754
798
|
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
options.signal?.addEventListener('abort', () =>
|
|
759
|
-
resource.yjs_clients.delete(options))
|
|
760
|
-
|
|
761
|
-
return
|
|
799
|
+
client.abort = () => resource.simpleton.clients.delete(client)
|
|
800
|
+
resource.simpleton.clients.add(client)
|
|
801
|
+
options.signal?.addEventListener('abort', () => client.abort())
|
|
762
802
|
}
|
|
803
|
+
break
|
|
763
804
|
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
options.my_subscribe_chain =
|
|
767
|
-
options.my_subscribe_chain.then(() =>
|
|
768
|
-
options.subscribe(x))
|
|
769
|
-
|
|
770
|
-
if (options.merge_type != "dt") {
|
|
771
|
-
// Simpleton clients require DT
|
|
772
|
-
await ensure_dt_exists(resource)
|
|
773
|
-
|
|
774
|
-
let x = { version }
|
|
805
|
+
case 'dt':
|
|
806
|
+
await ensure_dt_exists(resource)
|
|
775
807
|
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
options.my_subscribe(x)
|
|
780
|
-
} else {
|
|
781
|
-
x.parents = options.version ? options.version : options.parents
|
|
808
|
+
if (getting.history && !getting.subscribe)
|
|
809
|
+
return dt_get_patches(resource.dt.doc,
|
|
810
|
+
getting.history === 'since-parents' ? options.parents : undefined)
|
|
782
811
|
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
812
|
+
if (getting.subscribe) {
|
|
813
|
+
var client = {
|
|
814
|
+
merge_type: 'dt',
|
|
815
|
+
peer: options.peer,
|
|
816
|
+
send_update: one_at_a_time(options.subscribe),
|
|
817
|
+
accept_encoding_dt: !!options.accept_encoding?.match(/updates\s*\((.*)\)/)?.[1]?.split(',').map(x=>x.trim()).includes('dt'),
|
|
789
818
|
}
|
|
790
819
|
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
await ensure_dt_exists(resource)
|
|
797
|
-
|
|
798
|
-
if (options.accept_encoding?.match(/updates\s*\((.*)\)/)?.[1].split(',').map(x=>x.trim()).includes('dt')) {
|
|
799
|
-
// optimization: if client wants past current version,
|
|
800
|
-
// send empty dt
|
|
801
|
-
if (options.parents && v_eq(options.parents, version)) {
|
|
802
|
-
options.my_subscribe({ encoding: 'dt', body: new Doc().toBytes() })
|
|
803
|
-
} else {
|
|
820
|
+
// Send initial history
|
|
821
|
+
if (client.accept_encoding_dt) {
|
|
822
|
+
if (!getting.history)
|
|
823
|
+
client.send_update({ encoding: 'dt', body: new Doc().toBytes() })
|
|
824
|
+
else {
|
|
804
825
|
var bytes = resource.dt.doc.toBytes()
|
|
805
|
-
if (
|
|
826
|
+
if (getting.history === 'since-parents') {
|
|
806
827
|
var doc = Doc.fromBytes(bytes)
|
|
807
828
|
bytes = doc.getPatchSince(
|
|
808
829
|
dt_get_local_version(bytes, options.parents))
|
|
809
830
|
doc.free()
|
|
810
831
|
}
|
|
811
|
-
|
|
832
|
+
client.send_update({ encoding: 'dt', body: bytes })
|
|
812
833
|
}
|
|
813
834
|
} else {
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
835
|
+
if (getting.history === 'up-to-version') {
|
|
836
|
+
client.send_update({ version: [], parents: [], body: "" })
|
|
837
|
+
var updates = dt_get_patches(resource.dt.doc)
|
|
838
|
+
} else if (getting.history === 'since-parents')
|
|
839
|
+
var updates = dt_get_patches(resource.dt.doc, options.parents || options.version)
|
|
840
|
+
|
|
841
|
+
if (updates) {
|
|
842
|
+
for (var u of updates)
|
|
843
|
+
client.send_update({
|
|
844
|
+
version: [u.version], parents: u.parents,
|
|
845
|
+
patches: [{ unit: u.unit, range: u.range, content: u.content }],
|
|
846
|
+
})
|
|
821
847
|
|
|
822
|
-
updates = dt_get_patches(resource.dt.doc)
|
|
823
|
-
} else {
|
|
824
|
-
// Then start the subscription from the parents in options
|
|
825
|
-
updates = dt_get_patches(resource.dt.doc, options.parents || options.version)
|
|
826
848
|
}
|
|
827
|
-
|
|
828
|
-
for (let u of updates)
|
|
829
|
-
options.my_subscribe({
|
|
830
|
-
version: [u.version],
|
|
831
|
-
parents: u.parents,
|
|
832
|
-
patches: [{ unit: u.unit, range: u.range, content: u.content }],
|
|
833
|
-
})
|
|
834
|
-
|
|
835
|
-
// Output at least *some* data, or else chrome gets confused and
|
|
836
|
-
// thinks the connection failed. This isn't strictly necessary,
|
|
837
|
-
// but it makes fewer scary errors get printed out in the JS
|
|
838
|
-
// console.
|
|
839
|
-
if (updates.length === 0) options.write?.("\r\n")
|
|
840
849
|
}
|
|
841
850
|
|
|
842
|
-
resource.dt.clients.
|
|
843
|
-
|
|
844
|
-
|
|
851
|
+
client.abort = () => resource.dt.clients.delete(client)
|
|
852
|
+
resource.dt.clients.add(client)
|
|
853
|
+
options.signal?.addEventListener('abort', () => client.abort())
|
|
845
854
|
}
|
|
855
|
+
break
|
|
846
856
|
}
|
|
847
857
|
}
|
|
848
858
|
|
|
849
|
-
// Deprecated: Use
|
|
850
|
-
braid_text.forget = async (key,
|
|
851
|
-
console.warn('braid_text.forget() is deprecated. Use
|
|
852
|
-
if (
|
|
853
|
-
|
|
854
|
-
if (key instanceof URL) throw new Error('forget() does not support URLs. Use signal-based abort instead.')
|
|
855
|
-
|
|
856
|
-
let resource = (typeof key == 'string') ? await get_resource(key) : key
|
|
857
|
-
|
|
858
|
-
if (options.merge_type != "dt")
|
|
859
|
-
resource.dt.simpleton_clients.delete(options)
|
|
860
|
-
else resource.dt.clients.delete(options)
|
|
859
|
+
// Deprecated: Use client.abort() instead
|
|
860
|
+
braid_text.forget = async (key, client) => {
|
|
861
|
+
console.warn('braid_text.forget() is deprecated. Use client.abort() instead.')
|
|
862
|
+
if (client && client.abort) client.abort()
|
|
861
863
|
}
|
|
862
864
|
|
|
863
865
|
braid_text.put = async (key, options) => {
|
|
@@ -880,7 +882,7 @@ function create_braid_text() {
|
|
|
880
882
|
for (var x of ['headers', 'parents', 'version', 'peer', 'body', 'patches'])
|
|
881
883
|
if (options[x] != null) params[x] = options[x]
|
|
882
884
|
|
|
883
|
-
return await braid_fetch(key.href, params)
|
|
885
|
+
return { response: await braid_fetch(key.href, params) }
|
|
884
886
|
}
|
|
885
887
|
|
|
886
888
|
let resource = (typeof key == 'string') ? await get_resource(key) : key
|
|
@@ -898,98 +900,30 @@ function create_braid_text() {
|
|
|
898
900
|
|
|
899
901
|
let { version, parents, patches, body, peer } = options
|
|
900
902
|
|
|
901
|
-
//
|
|
903
|
+
// Yjs update: either raw binary (yjs_update) or yjs-text patches
|
|
904
|
+
var yjs_binary = null
|
|
905
|
+
var yjs_text_patches = null
|
|
902
906
|
if (options.yjs_update) {
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
}
|
|
912
|
-
}
|
|
913
|
-
|
|
914
|
-
var prev_text = resource.yjs.text.toString()
|
|
915
|
-
var delta = null
|
|
916
|
-
var observer = (e) => { delta = e.changes.delta }
|
|
917
|
-
resource.yjs.text.observe(observer)
|
|
918
|
-
try {
|
|
919
|
-
Y.applyUpdate(resource.yjs.doc,
|
|
920
|
-
options.yjs_update instanceof Uint8Array ? options.yjs_update : new Uint8Array(options.yjs_update))
|
|
921
|
-
} finally {
|
|
922
|
-
resource.yjs.text.unobserve(observer)
|
|
923
|
-
}
|
|
924
|
-
|
|
925
|
-
resource.val = resource.yjs.text.toString()
|
|
926
|
-
|
|
927
|
-
// Sync to DT if it exists
|
|
928
|
-
if (resource.dt && delta) {
|
|
929
|
-
var text_patches = yjs_delta_to_patches(delta, prev_text)
|
|
930
|
-
if (text_patches.length) {
|
|
931
|
-
var syn_actor = `yjs-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
|
932
|
-
var syn_seq = 0
|
|
933
|
-
var yjs_v_before = resource.dt.doc.getLocalVersion()
|
|
934
|
-
// Patches are sequential (each relative to state after previous)
|
|
935
|
-
// so no offset adjustment needed
|
|
936
|
-
var dt_bytes = []
|
|
937
|
-
var dt_ps = resource.version
|
|
938
|
-
for (var tp of text_patches) {
|
|
939
|
-
var tp_range = tp.range.match(/-?\d+/g).map(Number)
|
|
940
|
-
var tp_del = tp_range[1] - tp_range[0]
|
|
941
|
-
var syn_v = `${syn_actor}-${syn_seq}`
|
|
942
|
-
if (tp_del) {
|
|
943
|
-
dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], tp_del, null))
|
|
944
|
-
dt_ps = [`${syn_actor}-${syn_seq + tp_del - 1}`]
|
|
945
|
-
syn_seq += tp_del
|
|
946
|
-
syn_v = `${syn_actor}-${syn_seq}`
|
|
947
|
-
}
|
|
948
|
-
if (tp.content.length) {
|
|
949
|
-
dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], 0, tp.content))
|
|
950
|
-
var cp_len = [...tp.content].length
|
|
951
|
-
dt_ps = [`${syn_actor}-${syn_seq + cp_len - 1}`]
|
|
952
|
-
syn_seq += cp_len
|
|
953
|
-
}
|
|
954
|
-
}
|
|
955
|
-
for (var b of dt_bytes) resource.dt.doc.mergeBytes(b)
|
|
956
|
-
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
957
|
-
if (!resource.dt.known_versions[syn_actor]) resource.dt.known_versions[syn_actor] = new RangeSet()
|
|
958
|
-
resource.dt.known_versions[syn_actor].add_range(0, syn_seq - 1)
|
|
959
|
-
await resource.dt.save_delta(resource.dt.doc.getPatchSince(yjs_v_before))
|
|
960
|
-
}
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
// Persist Yjs
|
|
964
|
-
if (resource.yjs.save_delta) await resource.yjs.save_delta(
|
|
965
|
-
options.yjs_update instanceof Uint8Array ? options.yjs_update : new Uint8Array(options.yjs_update))
|
|
966
|
-
|
|
967
|
-
// Sanity check
|
|
968
|
-
if (braid_text.debug_sync_checks && resource.dt) {
|
|
969
|
-
var dt_text = resource.dt.doc.get()
|
|
970
|
-
var yjs_text = resource.yjs.text.toString()
|
|
971
|
-
if (dt_text !== yjs_text) {
|
|
972
|
-
console.error(`SYNC MISMATCH key=${resource.key}: DT text !== Y.Doc text`)
|
|
973
|
-
console.error(` DT: ${dt_text.slice(0, 100)}... (${dt_text.length})`)
|
|
974
|
-
console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
|
|
975
|
-
}
|
|
976
|
-
}
|
|
977
|
-
|
|
978
|
-
return { change_count: 1 }
|
|
907
|
+
yjs_binary = options.yjs_update instanceof Uint8Array
|
|
908
|
+
? options.yjs_update : new Uint8Array(options.yjs_update)
|
|
909
|
+
} else if (patches && patches.length && patches[0].unit === 'yjs-text') {
|
|
910
|
+
yjs_text_patches = patches
|
|
911
|
+
yjs_binary = braid_text.to_yjs_binary([{
|
|
912
|
+
version: options.version?.[0],
|
|
913
|
+
patches
|
|
914
|
+
}])
|
|
979
915
|
}
|
|
980
916
|
|
|
981
|
-
|
|
982
|
-
if (patches && patches.length && patches[0].unit === 'yjs-text') {
|
|
917
|
+
if (yjs_binary) {
|
|
983
918
|
await ensure_yjs_exists(resource)
|
|
984
919
|
|
|
985
|
-
//
|
|
986
|
-
var binary = braid_text.to_yjs_binary(patches)
|
|
920
|
+
// Apply binary update to Y.Doc, capturing the delta
|
|
987
921
|
var prev_text = resource.yjs.text.toString()
|
|
988
922
|
var delta = null
|
|
989
923
|
var observer = (e) => { delta = e.changes.delta }
|
|
990
924
|
resource.yjs.text.observe(observer)
|
|
991
925
|
try {
|
|
992
|
-
Y.applyUpdate(resource.yjs.doc,
|
|
926
|
+
Y.applyUpdate(resource.yjs.doc, yjs_binary)
|
|
993
927
|
} finally {
|
|
994
928
|
resource.yjs.text.unobserve(observer)
|
|
995
929
|
}
|
|
@@ -1000,12 +934,10 @@ function create_braid_text() {
|
|
|
1000
934
|
if (resource.dt && delta) {
|
|
1001
935
|
var text_patches = yjs_delta_to_patches(delta, prev_text)
|
|
1002
936
|
if (text_patches.length) {
|
|
1003
|
-
// Generate a synthetic version for DT
|
|
1004
937
|
var syn_actor = `yjs-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
|
1005
938
|
var syn_seq = 0
|
|
939
|
+
var version_before_yjs_sync = resource.version
|
|
1006
940
|
var yjs_v_before = resource.dt.doc.getLocalVersion()
|
|
1007
|
-
// Patches are sequential (each relative to state after previous)
|
|
1008
|
-
// so no offset adjustment needed
|
|
1009
941
|
var dt_bytes = []
|
|
1010
942
|
var dt_ps = resource.version
|
|
1011
943
|
for (var tp of text_patches) {
|
|
@@ -1027,32 +959,48 @@ function create_braid_text() {
|
|
|
1027
959
|
}
|
|
1028
960
|
for (var b of dt_bytes) resource.dt.doc.mergeBytes(b)
|
|
1029
961
|
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1030
|
-
|
|
1031
|
-
// Update known_versions for the synthetic versions
|
|
1032
962
|
if (!resource.dt.known_versions[syn_actor]) resource.dt.known_versions[syn_actor] = new RangeSet()
|
|
1033
963
|
resource.dt.known_versions[syn_actor].add_range(0, syn_seq - 1)
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
964
|
+
await resource.dt.log.save(resource.dt.doc.getPatchSince(yjs_v_before))
|
|
965
|
+
|
|
966
|
+
// Broadcast to simpleton and DT clients
|
|
967
|
+
var xf = get_xf_patches(resource.dt.doc, yjs_v_before)
|
|
968
|
+
for (let client of resource.simpleton.clients) {
|
|
969
|
+
if (!peer || client.peer !== peer)
|
|
970
|
+
await client.send_update({
|
|
971
|
+
version: resource.version,
|
|
972
|
+
parents: version_before_yjs_sync,
|
|
973
|
+
patches: xf
|
|
974
|
+
})
|
|
975
|
+
}
|
|
976
|
+
for (let client of resource.dt.clients) {
|
|
977
|
+
if (!peer || client.peer !== peer)
|
|
978
|
+
await client.send_update(
|
|
979
|
+
client.accept_encoding_dt
|
|
980
|
+
? { version: resource.version, parents: version_before_yjs_sync, body: resource.dt.doc.getPatchSince(yjs_v_before), encoding: 'dt' }
|
|
981
|
+
: { version: resource.version, parents: version_before_yjs_sync, patches: xf }
|
|
982
|
+
)
|
|
983
|
+
}
|
|
1039
984
|
}
|
|
1040
985
|
}
|
|
1041
986
|
|
|
1042
987
|
// Broadcast to yjs-text subscribers (skip sender)
|
|
1043
|
-
if (resource.
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
988
|
+
if (resource.yjs) {
|
|
989
|
+
// If we received yjs-text updates, reuse them; otherwise
|
|
990
|
+
// derive them from the binary update
|
|
991
|
+
var yjs_updates = yjs_text_patches
|
|
992
|
+
? [{version: options.version, patches: yjs_text_patches}]
|
|
993
|
+
: braid_text.from_yjs_binary(yjs_binary)
|
|
994
|
+
for (var yjs_update of yjs_updates) {
|
|
995
|
+
for (var client of resource.yjs.clients) {
|
|
996
|
+
if (!peer || client.peer !== peer)
|
|
997
|
+
await client.send_update(yjs_update)
|
|
1050
998
|
}
|
|
1051
999
|
}
|
|
1052
1000
|
}
|
|
1053
1001
|
|
|
1054
1002
|
// Persist Yjs delta
|
|
1055
|
-
if (resource.yjs.
|
|
1003
|
+
if (resource.yjs.log.save) await resource.yjs.log.save(yjs_binary)
|
|
1056
1004
|
|
|
1057
1005
|
// Sanity check
|
|
1058
1006
|
if (braid_text.debug_sync_checks && resource.dt) {
|
|
@@ -1065,7 +1013,7 @@ function create_braid_text() {
|
|
|
1065
1013
|
}
|
|
1066
1014
|
}
|
|
1067
1015
|
|
|
1068
|
-
return { change_count:
|
|
1016
|
+
return { dt: { change_count: yjs_text_patches?.length || 1 } }
|
|
1069
1017
|
}
|
|
1070
1018
|
|
|
1071
1019
|
if (options.transfer_encoding === 'dt') {
|
|
@@ -1083,15 +1031,15 @@ function create_braid_text() {
|
|
|
1083
1031
|
resource.val = resource.dt.doc.get()
|
|
1084
1032
|
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1085
1033
|
|
|
1086
|
-
await resource.dt.
|
|
1034
|
+
await resource.dt.log.save(body)
|
|
1087
1035
|
|
|
1088
1036
|
// Notify non-simpleton clients with the dt-encoded update
|
|
1089
1037
|
var dt_update = { body, encoding: 'dt' }
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1038
|
+
for (let client of resource.dt.clients)
|
|
1039
|
+
if (!peer || client.peer !== peer)
|
|
1040
|
+
await client.send_update(dt_update)
|
|
1093
1041
|
|
|
1094
|
-
return { change_count: end_i - start_i + 1 }
|
|
1042
|
+
return { dt: { change_count: end_i - start_i + 1 } }
|
|
1095
1043
|
}
|
|
1096
1044
|
|
|
1097
1045
|
// Text/DT patches require DT
|
|
@@ -1099,7 +1047,7 @@ function create_braid_text() {
|
|
|
1099
1047
|
|
|
1100
1048
|
if (version && !version.length) {
|
|
1101
1049
|
console.log(`warning: ignoring put with empty version`)
|
|
1102
|
-
return { change_count: 0 }
|
|
1050
|
+
return { dt: { change_count: 0 } }
|
|
1103
1051
|
}
|
|
1104
1052
|
if (version && version.length > 1)
|
|
1105
1053
|
throw new Error(`cannot put a version with multiple ids`)
|
|
@@ -1145,12 +1093,13 @@ function create_braid_text() {
|
|
|
1145
1093
|
|
|
1146
1094
|
// Nothing to do: e.g. PUT with empty body on an already-empty doc,
|
|
1147
1095
|
// or patches that delete and insert zero characters.
|
|
1148
|
-
if (change_count === 0) return { change_count }
|
|
1096
|
+
if (change_count === 0) return { dt: { change_count } }
|
|
1149
1097
|
|
|
1150
1098
|
version = version?.[0] || `${(is_valid_actor(peer) && peer) || Math.random().toString(36).slice(2, 7)}-${change_count - 1}`
|
|
1151
1099
|
|
|
1152
1100
|
let v = decode_version(version)
|
|
1153
1101
|
var low_seq = v[1] + 1 - change_count
|
|
1102
|
+
if (low_seq < 0) throw new Error(`version seq ${v[1]} is too low for ${change_count} changes — use seq >= ${change_count - 1}`)
|
|
1154
1103
|
|
|
1155
1104
|
// make sure we haven't seen this already
|
|
1156
1105
|
var intersects_range = resource.dt.known_versions[v[0]]?.has(low_seq, v[1])
|
|
@@ -1172,7 +1121,7 @@ function create_braid_text() {
|
|
|
1172
1121
|
if (options.validate_already_seen_versions)
|
|
1173
1122
|
validate_old_patches(resource, `${v[0]}-${low_seq}`, parents, patches)
|
|
1174
1123
|
|
|
1175
|
-
if (new_count <= 0) return { change_count }
|
|
1124
|
+
if (new_count <= 0) return { dt: { change_count } }
|
|
1176
1125
|
|
|
1177
1126
|
change_count = new_count
|
|
1178
1127
|
low_seq = v[1] + 1 - change_count
|
|
@@ -1242,7 +1191,7 @@ function create_braid_text() {
|
|
|
1242
1191
|
xf_patches_relative.push(
|
|
1243
1192
|
xf.kind == "Ins"
|
|
1244
1193
|
? { range: [xf.start, xf.start], content: xf.content }
|
|
1245
|
-
|
|
1194
|
+
: { range: [xf.start, xf.end], content: "" }
|
|
1246
1195
|
)
|
|
1247
1196
|
}
|
|
1248
1197
|
var xf_patches = relative_to_absolute_patches(
|
|
@@ -1277,20 +1226,20 @@ function create_braid_text() {
|
|
|
1277
1226
|
resource.yjs.doc.off('update', yjs_update_handler)
|
|
1278
1227
|
|
|
1279
1228
|
// Broadcast to yjs-text subscribers
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1229
|
+
// NOTE: There are two universes of version IDs here -- DT and Yjs --
|
|
1230
|
+
// and we are mixing them. The update-level .version is the DT frontier.
|
|
1231
|
+
// Each patch's .version is a Yjs item ID (clientID-clock).
|
|
1232
|
+
if (resource.yjs && captured_yjs_update) {
|
|
1233
|
+
var yjs_updates = braid_text.from_yjs_binary(captured_yjs_update)
|
|
1234
|
+
if (braid_text.verbose) console.log('DT→Yjs broadcast:', yjs_updates.length, 'updates to', resource.yjs.clients.size, 'yjs clients')
|
|
1235
|
+
for (var yjs_update of yjs_updates)
|
|
1236
|
+
for (var client of resource.yjs.clients)
|
|
1237
|
+
if (!peer || client.peer !== peer)
|
|
1238
|
+
await client.send_update(yjs_update)
|
|
1290
1239
|
}
|
|
1291
1240
|
|
|
1292
1241
|
// Persist Yjs delta
|
|
1293
|
-
if (captured_yjs_update) await resource.yjs.
|
|
1242
|
+
if (captured_yjs_update) await resource.yjs.log.save(captured_yjs_update)
|
|
1294
1243
|
|
|
1295
1244
|
// Sanity check
|
|
1296
1245
|
if (braid_text.debug_sync_checks) {
|
|
@@ -1312,37 +1261,30 @@ function create_braid_text() {
|
|
|
1312
1261
|
let patches = xf_patches
|
|
1313
1262
|
if (braid_text.verbose) console.log(JSON.stringify({ patches }))
|
|
1314
1263
|
|
|
1315
|
-
for (let client of resource.
|
|
1264
|
+
for (let client of resource.simpleton.clients) {
|
|
1316
1265
|
if (peer && client.peer === peer) {
|
|
1317
|
-
client.
|
|
1266
|
+
client.last_seen_version = [version]
|
|
1318
1267
|
}
|
|
1319
1268
|
|
|
1320
1269
|
function set_timeout(time_override) {
|
|
1321
|
-
if (client.
|
|
1322
|
-
client.
|
|
1270
|
+
if (client.timeout) clearTimeout(client.timeout)
|
|
1271
|
+
client.timeout = braid_text.simpletonSetTimeout(() => {
|
|
1323
1272
|
// if the doc has been freed, exit early
|
|
1324
1273
|
if (resource.dt.doc.__wbg_ptr === 0) return
|
|
1325
1274
|
|
|
1326
1275
|
let x = {
|
|
1327
1276
|
version: resource.version,
|
|
1328
|
-
parents: client.
|
|
1277
|
+
parents: client.last_seen_version
|
|
1329
1278
|
}
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
x.patches = get_xf_patches(resource.dt.doc, OpLog_remote_to_local(resource.dt.doc, client.my_last_seen_version))
|
|
1279
|
+
x.patches = get_xf_patches(resource.dt.doc, OpLog_remote_to_local(resource.dt.doc, client.last_seen_version))
|
|
1280
|
+
client.send_update(x)
|
|
1333
1281
|
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
delete client.my_timeout
|
|
1338
|
-
}, time_override ?? Math.min(3000, 23 * Math.pow(1.5, client.my_unused_version_count - 1)))
|
|
1282
|
+
delete client.timeout
|
|
1283
|
+
}, time_override ?? Math.min(3000, 23 * Math.pow(1.5, client.unused_version_count - 1)))
|
|
1339
1284
|
}
|
|
1340
1285
|
|
|
1341
|
-
if (client.
|
|
1286
|
+
if (client.timeout) {
|
|
1342
1287
|
if (peer && client.peer === peer) {
|
|
1343
|
-
// note: we don't add to client.my_unused_version_count,
|
|
1344
|
-
// because we're already in a timeout;
|
|
1345
|
-
// we'll just extend it here..
|
|
1346
1288
|
set_timeout()
|
|
1347
1289
|
}
|
|
1348
1290
|
continue
|
|
@@ -1351,37 +1293,30 @@ function create_braid_text() {
|
|
|
1351
1293
|
let x = { version: resource.version }
|
|
1352
1294
|
if (peer && client.peer === peer) {
|
|
1353
1295
|
if (!v_eq(resource.version, [version])) {
|
|
1354
|
-
|
|
1355
|
-
// PUT, so the client is behind — start a
|
|
1356
|
-
// timeout to batch and rebase
|
|
1357
|
-
client.my_unused_version_count = (client.my_unused_version_count ?? 0) + 1
|
|
1296
|
+
client.unused_version_count = (client.unused_version_count ?? 0) + 1
|
|
1358
1297
|
set_timeout()
|
|
1359
1298
|
continue
|
|
1360
1299
|
} else {
|
|
1361
|
-
delete client.
|
|
1300
|
+
delete client.unused_version_count
|
|
1362
1301
|
}
|
|
1363
1302
|
|
|
1364
|
-
// this client already has this version,
|
|
1365
|
-
// so let's pretend to send it back, but not
|
|
1366
|
-
if (braid_text.verbose) console.log(`not reflecting back to simpleton`)
|
|
1303
|
+
// this client already has this version, don't reflect back
|
|
1367
1304
|
continue
|
|
1368
1305
|
} else {
|
|
1369
1306
|
x.parents = version_before
|
|
1370
1307
|
x.patches = patches
|
|
1371
1308
|
}
|
|
1372
|
-
if (braid_text.verbose) console.log(`sending: ${JSON.stringify(x)}`)
|
|
1373
1309
|
post_commit_updates.push([client, x])
|
|
1374
1310
|
}
|
|
1375
1311
|
} else {
|
|
1376
|
-
if (resource.
|
|
1312
|
+
if (resource.simpleton.clients.size) {
|
|
1377
1313
|
let x = {
|
|
1378
1314
|
version: resource.version,
|
|
1379
1315
|
parents: version_before,
|
|
1380
1316
|
patches: xf_patches
|
|
1381
1317
|
}
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
if (client.my_timeout) continue
|
|
1318
|
+
for (let client of resource.simpleton.clients) {
|
|
1319
|
+
if (client.timeout) continue
|
|
1385
1320
|
post_commit_updates.push([client, x])
|
|
1386
1321
|
}
|
|
1387
1322
|
}
|
|
@@ -1401,11 +1336,11 @@ function create_braid_text() {
|
|
|
1401
1336
|
post_commit_updates.push([client, x])
|
|
1402
1337
|
}
|
|
1403
1338
|
|
|
1404
|
-
await resource.dt.
|
|
1339
|
+
await resource.dt.log.save(resource.dt.doc.getPatchSince(v_before))
|
|
1405
1340
|
|
|
1406
|
-
|
|
1341
|
+
for (let [client, x] of post_commit_updates) await client.send_update(x)
|
|
1407
1342
|
|
|
1408
|
-
return { change_count }
|
|
1343
|
+
return { dt: { change_count } }
|
|
1409
1344
|
})
|
|
1410
1345
|
}
|
|
1411
1346
|
|
|
@@ -1427,125 +1362,291 @@ function create_braid_text() {
|
|
|
1427
1362
|
res.setHeader("Access-Control-Expose-Headers", "*")
|
|
1428
1363
|
}
|
|
1429
1364
|
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
resource.version = []
|
|
1365
|
+
// ============================================================
|
|
1366
|
+
// Resource setup helpers
|
|
1367
|
+
// ============================================================
|
|
1368
|
+
|
|
1369
|
+
// Load resource meta from disk (JSON file at meta/[key])
|
|
1370
|
+
async function setup_meta(resource) {
|
|
1371
|
+
if (!braid_text.db_folder) {
|
|
1438
1372
|
resource.meta = {}
|
|
1439
1373
|
resource.save_meta = () => {}
|
|
1440
|
-
|
|
1374
|
+
return
|
|
1375
|
+
}
|
|
1376
|
+
await db_folder_init()
|
|
1377
|
+
var encoded = encode_filename(resource.key)
|
|
1378
|
+
var meta_path = `${braid_text.db_folder}/meta/${encoded}`
|
|
1379
|
+
try {
|
|
1380
|
+
resource.meta = JSON.parse(await fs.promises.readFile(meta_path))
|
|
1381
|
+
} catch (e) {
|
|
1382
|
+
resource.meta = {}
|
|
1383
|
+
}
|
|
1384
|
+
var dirty = false, saving = false
|
|
1385
|
+
resource.save_meta = async () => {
|
|
1386
|
+
dirty = true
|
|
1387
|
+
if (saving) return
|
|
1388
|
+
saving = true
|
|
1389
|
+
while (dirty) {
|
|
1390
|
+
dirty = false
|
|
1391
|
+
await atomic_write(meta_path, JSON.stringify(resource.meta),
|
|
1392
|
+
`${braid_text.db_folder}/temp`)
|
|
1393
|
+
await new Promise(done => setTimeout(done, braid_text.meta_file_save_period_ms))
|
|
1394
|
+
}
|
|
1395
|
+
saving = false
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
// Create a fresh DT backend object
|
|
1400
|
+
function make_dt_backend() {
|
|
1401
|
+
return {
|
|
1402
|
+
doc: new Doc("server"),
|
|
1403
|
+
known_versions: {},
|
|
1404
|
+
length_at_version: create_simple_cache(braid_text.length_cache_size),
|
|
1405
|
+
clients: new Set(),
|
|
1406
|
+
log: { save: () => {} },
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
// Create a fresh Yjs backend object
|
|
1411
|
+
function make_yjs_backend(channel) {
|
|
1412
|
+
require_yjs()
|
|
1413
|
+
var doc = new Y.Doc()
|
|
1414
|
+
return {
|
|
1415
|
+
doc,
|
|
1416
|
+
text: doc.getText(channel),
|
|
1417
|
+
channel,
|
|
1418
|
+
clients: new Set(),
|
|
1419
|
+
log: { save: () => {} },
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
// Rebuild DT version indexes from doc bytes
|
|
1424
|
+
function rebuild_dt_indexes(resource) {
|
|
1425
|
+
resource.dt.known_versions = {}
|
|
1426
|
+
dt_get_actor_seq_runs([...resource.dt.doc.toBytes()], (actor, base, len) => {
|
|
1427
|
+
if (!resource.dt.known_versions[actor])
|
|
1428
|
+
resource.dt.known_versions[actor] = new RangeSet()
|
|
1429
|
+
resource.dt.known_versions[actor].add_range(base, base + len - 1)
|
|
1430
|
+
})
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
// Set up a write-ahead-plus-compated-state log for a backend (DT or Yjs).
|
|
1434
|
+
// Loads existing data from disk via on_chunk callback.
|
|
1435
|
+
// Provides resource[type].log.save(bytes) for future writes.
|
|
1436
|
+
//
|
|
1437
|
+
// File format: [key].[type].[N]
|
|
1438
|
+
// Each file: [4-byte len][chunk] [4-byte len][chunk] ...
|
|
1439
|
+
// First chunk sets compaction threshold (10x its size)
|
|
1440
|
+
// When file exceeds threshold, get_compacted() is called to write a fresh file
|
|
1441
|
+
async function setup_compacting_log(resource, type, on_chunk, get_compacted) {
|
|
1442
|
+
if (!braid_text.db_folder) {
|
|
1443
|
+
resource[type].log = { save: () => {} }
|
|
1444
|
+
return
|
|
1445
|
+
}
|
|
1446
|
+
await db_folder_init()
|
|
1447
|
+
var encoded = encode_filename(resource.key)
|
|
1448
|
+
|
|
1449
|
+
var log = {
|
|
1450
|
+
file_number: 0,
|
|
1451
|
+
file_size: 0,
|
|
1452
|
+
threshold: 0,
|
|
1453
|
+
save: () => {}, // placeholder until fully initialized below
|
|
1454
|
+
}
|
|
1455
|
+
resource[type].log = log
|
|
1456
|
+
|
|
1457
|
+
// Load existing files
|
|
1458
|
+
var files = (await get_files_for_key(resource.key, type))
|
|
1459
|
+
.filter(x => x.match(/\.\d+$/))
|
|
1460
|
+
.sort((a, b) => parseInt(a.match(/\d+$/)[0]) - parseInt(b.match(/\d+$/)[0]))
|
|
1441
1461
|
|
|
1442
|
-
|
|
1443
|
-
|
|
1462
|
+
var loaded = false
|
|
1463
|
+
for (var i = files.length - 1; i >= 0; i--) {
|
|
1464
|
+
if (loaded) {
|
|
1465
|
+
await fs.promises.unlink(files[i])
|
|
1466
|
+
continue
|
|
1467
|
+
}
|
|
1468
|
+
try {
|
|
1469
|
+
var data = await fs.promises.readFile(files[i])
|
|
1470
|
+
var cursor = 0, first = true
|
|
1471
|
+
while (cursor < data.length) {
|
|
1472
|
+
var chunk_size = data.readUInt32LE(cursor)
|
|
1473
|
+
cursor += 4
|
|
1474
|
+
on_chunk(data.slice(cursor, cursor + chunk_size))
|
|
1475
|
+
cursor += chunk_size
|
|
1476
|
+
if (first) { log.threshold = chunk_size * 10; first = false }
|
|
1477
|
+
}
|
|
1478
|
+
log.file_size = data.length
|
|
1479
|
+
log.file_number = parseInt(files[i].match(/(\d+)$/)[1])
|
|
1480
|
+
loaded = true
|
|
1481
|
+
} catch (error) {
|
|
1482
|
+
console.error(`Error loading ${files[i]}: ${error.message}`)
|
|
1483
|
+
await fs.promises.unlink(files[i])
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
// Save function: append delta or compact with snapshot
|
|
1488
|
+
log.save = (bytes) => within_fiber('log:' + resource.key + ':' + type, async () => {
|
|
1489
|
+
log.file_size += bytes.length + 4
|
|
1490
|
+
var filename = `${braid_text.db_folder}/${encoded}.${type}.${log.file_number}`
|
|
1491
|
+
|
|
1492
|
+
if (log.file_size < log.threshold) {
|
|
1493
|
+
// Append with WAL-intent for crash safety
|
|
1494
|
+
var len_buf = Buffer.allocUnsafe(4)
|
|
1495
|
+
len_buf.writeUInt32LE(bytes.length, 0)
|
|
1496
|
+
var append_data = Buffer.concat([len_buf, bytes])
|
|
1497
|
+
|
|
1498
|
+
var basename = require('path').basename(filename)
|
|
1499
|
+
var intent_path = `${braid_text.db_folder}/wal-intent/${basename}`
|
|
1500
|
+
var stat = await fs.promises.stat(filename)
|
|
1501
|
+
var size_buf = Buffer.allocUnsafe(8)
|
|
1502
|
+
size_buf.writeBigUInt64LE(BigInt(stat.size), 0)
|
|
1503
|
+
|
|
1504
|
+
await atomic_write(intent_path, Buffer.concat([size_buf, append_data]),
|
|
1505
|
+
`${braid_text.db_folder}/temp`)
|
|
1506
|
+
await fs.promises.appendFile(filename, append_data)
|
|
1507
|
+
await fs.promises.unlink(intent_path)
|
|
1508
|
+
} else {
|
|
1509
|
+
// Compact: write full compaction to new file, delete old
|
|
1510
|
+
log.file_number++
|
|
1511
|
+
var snap = get_compacted()
|
|
1512
|
+
var buffer = Buffer.allocUnsafe(4)
|
|
1513
|
+
buffer.writeUInt32LE(snap.length, 0)
|
|
1514
|
+
|
|
1515
|
+
var new_filename = `${braid_text.db_folder}/${encoded}.${type}.${log.file_number}`
|
|
1516
|
+
await atomic_write(new_filename, Buffer.concat([buffer, snap]),
|
|
1517
|
+
`${braid_text.db_folder}/temp`)
|
|
1518
|
+
|
|
1519
|
+
log.file_size = 4 + snap.length
|
|
1520
|
+
log.threshold = log.file_size * 10
|
|
1521
|
+
try { await fs.promises.unlink(filename) } catch (e) {}
|
|
1522
|
+
}
|
|
1523
|
+
})
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
// ============================================================
|
|
1527
|
+
// get_resource
|
|
1528
|
+
// ============================================================
|
|
1529
|
+
|
|
1530
|
+
async function get_resource(key, options) {
|
|
1531
|
+
var cache = braid_text.cache
|
|
1532
|
+
if (!cache[key]) cache[key] = new Promise(async done => {
|
|
1533
|
+
var resource = {
|
|
1534
|
+
key,
|
|
1535
|
+
dt: null,
|
|
1536
|
+
yjs: null,
|
|
1537
|
+
simpleton: { clients: new Set() },
|
|
1538
|
+
val: '',
|
|
1539
|
+
version: [],
|
|
1540
|
+
cursors: null,
|
|
1541
|
+
// Returns all subscriber clients across all merge types
|
|
1542
|
+
clients() {
|
|
1543
|
+
var all = [...this.simpleton.clients]
|
|
1544
|
+
if (this.dt) all.push(...this.dt.clients)
|
|
1545
|
+
if (this.yjs) all.push(...this.yjs.clients)
|
|
1546
|
+
return all
|
|
1547
|
+
},
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
// Always load meta first
|
|
1551
|
+
await setup_meta(resource)
|
|
1552
|
+
|
|
1553
|
+
// Check what's on disk
|
|
1554
|
+
var has_dt = braid_text.db_folder
|
|
1444
1555
|
&& (await get_files_for_key(key, 'dt')).length > 0
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
simpleton_clients: new Set(),
|
|
1453
|
-
}
|
|
1454
|
-
let { change, change_meta } = await file_sync(key,
|
|
1455
|
-
(bytes) => resource.dt.doc.mergeBytes(bytes),
|
|
1456
|
-
() => resource.dt.doc.toBytes(),
|
|
1457
|
-
(meta) => resource.meta = meta,
|
|
1458
|
-
() => resource.meta,
|
|
1459
|
-
'dt')
|
|
1460
|
-
|
|
1461
|
-
resource.dt.save_delta = change
|
|
1462
|
-
resource.save_meta = change_meta
|
|
1463
|
-
|
|
1464
|
-
dt_get_actor_seq_runs([...resource.dt.doc.toBytes()], (actor, base, len) => {
|
|
1465
|
-
if (!resource.dt.known_versions[actor]) resource.dt.known_versions[actor] = new RangeSet()
|
|
1466
|
-
resource.dt.known_versions[actor].add_range(base, base + len - 1)
|
|
1467
|
-
})
|
|
1556
|
+
var has_yjs = braid_text.db_folder && Y
|
|
1557
|
+
&& (await get_files_for_key(key, 'yjs')).length > 0
|
|
1558
|
+
|
|
1559
|
+
// Get initializer spec if nothing on disk
|
|
1560
|
+
var init = (has_dt || has_yjs) ? null : (
|
|
1561
|
+
options?.initializer ? await options.initializer() : null
|
|
1562
|
+
)
|
|
1468
1563
|
|
|
1564
|
+
// --- Load from disk ---
|
|
1565
|
+
if (has_dt) {
|
|
1566
|
+
resource.dt = make_dt_backend()
|
|
1567
|
+
await setup_compacting_log(resource, 'dt',
|
|
1568
|
+
(bytes) => resource.dt.doc.mergeBytes(bytes),
|
|
1569
|
+
() => resource.dt.doc.toBytes())
|
|
1570
|
+
rebuild_dt_indexes(resource)
|
|
1469
1571
|
resource.val = resource.dt.doc.get()
|
|
1470
1572
|
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
'dt')
|
|
1479
|
-
resource.save_meta = change_meta
|
|
1480
|
-
}
|
|
1481
|
-
|
|
1482
|
-
// Load Yjs data from disk if it exists
|
|
1483
|
-
var has_yjs_files = braid_text.db_folder && Y
|
|
1484
|
-
&& (await get_files_for_key(key, 'yjs')).length > 0
|
|
1485
|
-
if (has_yjs_files) {
|
|
1486
|
-
resource.yjs = {
|
|
1487
|
-
doc: new Y.Doc(),
|
|
1488
|
-
text: null,
|
|
1489
|
-
save_delta: () => {},
|
|
1490
|
-
}
|
|
1491
|
-
resource.yjs.text = resource.yjs.doc.getText('text')
|
|
1492
|
-
let { change } = await file_sync(key,
|
|
1493
|
-
(bytes) => Y.applyUpdate(resource.yjs.doc, bytes),
|
|
1494
|
-
() => Y.encodeStateAsUpdate(resource.yjs.doc),
|
|
1495
|
-
() => {},
|
|
1496
|
-
() => ({}),
|
|
1497
|
-
'yjs')
|
|
1498
|
-
|
|
1499
|
-
resource.yjs.save_delta = change
|
|
1500
|
-
|
|
1573
|
+
}
|
|
1574
|
+
if (has_yjs) {
|
|
1575
|
+
var channel = resource.meta.yjs_channel || 'text'
|
|
1576
|
+
resource.yjs = make_yjs_backend(channel)
|
|
1577
|
+
await setup_compacting_log(resource, 'yjs',
|
|
1578
|
+
(bytes) => Y.applyUpdate(resource.yjs.doc, bytes),
|
|
1579
|
+
() => Y.encodeStateAsUpdate(resource.yjs.doc))
|
|
1501
1580
|
var yjs_text = resource.yjs.text.toString()
|
|
1502
1581
|
if (resource.dt) {
|
|
1503
|
-
// Both exist — sanity check they match
|
|
1504
1582
|
if (resource.val !== yjs_text) {
|
|
1505
|
-
console.error(`INIT MISMATCH key=${key}: DT
|
|
1506
|
-
console.error(` DT: ${resource.val.slice(0, 100)}... (${resource.val.length})`)
|
|
1507
|
-
console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
|
|
1583
|
+
console.error(`INIT MISMATCH key=${key}: DT="${resource.val.slice(0,50)}" Yjs="${yjs_text.slice(0,50)}"`)
|
|
1508
1584
|
}
|
|
1509
1585
|
} else {
|
|
1510
|
-
// Yjs only — use its text as resource.val
|
|
1511
1586
|
resource.val = yjs_text
|
|
1512
1587
|
}
|
|
1513
1588
|
}
|
|
1514
1589
|
|
|
1515
|
-
//
|
|
1590
|
+
// --- Initialize from external source (only when nothing on disk) ---
|
|
1591
|
+
if (init) {
|
|
1592
|
+
if (init.yjs) {
|
|
1593
|
+
var channel = (typeof init.yjs === 'object' && init.yjs.channel) || 'text'
|
|
1594
|
+
var history = typeof init.yjs === 'object' && init.yjs.history
|
|
1595
|
+
resource.yjs = make_yjs_backend(channel)
|
|
1596
|
+
resource.meta.yjs_channel = channel
|
|
1597
|
+
resource.save_meta()
|
|
1598
|
+
if (history) {
|
|
1599
|
+
Y.applyUpdate(resource.yjs.doc,
|
|
1600
|
+
history instanceof Uint8Array ? history : new Uint8Array(history))
|
|
1601
|
+
resource.val = resource.yjs.text.toString()
|
|
1602
|
+
}
|
|
1603
|
+
await setup_compacting_log(resource, 'yjs',
|
|
1604
|
+
(bytes) => Y.applyUpdate(resource.yjs.doc, bytes),
|
|
1605
|
+
() => Y.encodeStateAsUpdate(resource.yjs.doc))
|
|
1606
|
+
await resource.yjs.log.save(Y.encodeStateAsUpdate(resource.yjs.doc))
|
|
1607
|
+
}
|
|
1608
|
+
if (init.dt) {
|
|
1609
|
+
var dt_history = typeof init.dt === 'object' && init.dt.history
|
|
1610
|
+
resource.dt = make_dt_backend()
|
|
1611
|
+
if (dt_history) {
|
|
1612
|
+
resource.dt.doc.mergeBytes(
|
|
1613
|
+
dt_history instanceof Uint8Array ? dt_history : new Uint8Array(dt_history))
|
|
1614
|
+
} else if (resource.val) {
|
|
1615
|
+
resource.dt.doc.mergeBytes(
|
|
1616
|
+
dt_create_bytes('999999999-0', [], 0, 0, resource.val))
|
|
1617
|
+
}
|
|
1618
|
+
rebuild_dt_indexes(resource)
|
|
1619
|
+
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1620
|
+
if (!resource.val) resource.val = resource.dt.doc.get()
|
|
1621
|
+
await setup_compacting_log(resource, 'dt',
|
|
1622
|
+
(bytes) => resource.dt.doc.mergeBytes(bytes),
|
|
1623
|
+
() => resource.dt.doc.toBytes())
|
|
1624
|
+
await resource.dt.log.save(resource.dt.doc.toBytes())
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
// Sanity check
|
|
1629
|
+
if (resource.dt && resource.yjs) {
|
|
1630
|
+
var dt_text = resource.dt.doc.get()
|
|
1631
|
+
var yjs_text = resource.yjs.text.toString()
|
|
1632
|
+
if (dt_text !== yjs_text)
|
|
1633
|
+
console.error(`get_resource key=${key}: DT and Yjs disagree: DT="${dt_text.slice(0,50)}" Yjs="${yjs_text.slice(0,50)}"`)
|
|
1634
|
+
}
|
|
1516
1635
|
|
|
1517
|
-
//
|
|
1636
|
+
// Delete method
|
|
1518
1637
|
resource.delete = async () => {
|
|
1519
1638
|
if (resource.dt) resource.dt.doc.free()
|
|
1520
1639
|
if (resource.yjs) resource.yjs.doc.destroy()
|
|
1521
|
-
|
|
1522
|
-
// Remove from in-memory cache
|
|
1523
1640
|
delete braid_text.cache[key]
|
|
1524
|
-
|
|
1525
|
-
// Remove all files for this key from db_folder
|
|
1526
1641
|
if (braid_text.db_folder) {
|
|
1527
|
-
var
|
|
1528
|
-
|
|
1529
|
-
try {
|
|
1530
|
-
await fs.promises.unlink(file)
|
|
1531
|
-
} catch (e) {
|
|
1532
|
-
// File might not exist, that's ok
|
|
1533
|
-
}
|
|
1534
|
-
}
|
|
1535
|
-
|
|
1536
|
-
// Remove meta file if it exists
|
|
1642
|
+
for (var file of await get_files_for_key(key))
|
|
1643
|
+
try { await fs.promises.unlink(file) } catch (e) {}
|
|
1537
1644
|
try {
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
} catch (e) {
|
|
1541
|
-
// Meta file might not exist, that's ok
|
|
1542
|
-
}
|
|
1645
|
+
await fs.promises.unlink(`${braid_text.db_folder}/meta/${encode_filename(key)}`)
|
|
1646
|
+
} catch (e) {}
|
|
1543
1647
|
}
|
|
1544
|
-
|
|
1545
|
-
// Remove from filename mapping
|
|
1546
1648
|
if (key_to_filename.has(key)) {
|
|
1547
|
-
|
|
1548
|
-
ifilenames.delete(encoded.toLowerCase())
|
|
1649
|
+
ifilenames.delete(key_to_filename.get(key).toLowerCase())
|
|
1549
1650
|
key_to_filename.delete(key)
|
|
1550
1651
|
}
|
|
1551
1652
|
}
|
|
@@ -1555,64 +1656,34 @@ function create_braid_text() {
|
|
|
1555
1656
|
return await cache[key]
|
|
1556
1657
|
}
|
|
1557
1658
|
|
|
1659
|
+
// Internal: create DT backend on demand, synced from resource.val
|
|
1558
1660
|
async function ensure_dt_exists(resource) {
|
|
1559
1661
|
if (resource.dt) return
|
|
1560
|
-
resource.dt =
|
|
1561
|
-
doc: new Doc("server"),
|
|
1562
|
-
known_versions: {},
|
|
1563
|
-
save_delta: () => {},
|
|
1564
|
-
length_at_version: createSimpleCache(braid_text.length_cache_size),
|
|
1565
|
-
clients: new Set(),
|
|
1566
|
-
simpleton_clients: new Set(),
|
|
1567
|
-
}
|
|
1662
|
+
resource.dt = make_dt_backend()
|
|
1568
1663
|
if (resource.val) {
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
`999999999-0`, [], 0, 0, resource.val)
|
|
1572
|
-
resource.dt.doc.mergeBytes(bytes)
|
|
1664
|
+
resource.dt.doc.mergeBytes(
|
|
1665
|
+
dt_create_bytes('999999999-0', [], 0, 0, resource.val))
|
|
1573
1666
|
}
|
|
1667
|
+
rebuild_dt_indexes(resource)
|
|
1574
1668
|
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
resource.dt.known_versions[actor].add_range(base, base + len - 1)
|
|
1579
|
-
})
|
|
1580
|
-
|
|
1581
|
-
// Set up DT persistence if db_folder exists
|
|
1582
|
-
if (braid_text.db_folder) {
|
|
1583
|
-
let { change } = await file_sync(resource.key,
|
|
1584
|
-
(bytes) => resource.dt.doc.mergeBytes(bytes),
|
|
1585
|
-
() => resource.dt.doc.toBytes(),
|
|
1586
|
-
() => {},
|
|
1587
|
-
() => ({}),
|
|
1588
|
-
'dt')
|
|
1589
|
-
resource.dt.save_delta = change
|
|
1590
|
-
}
|
|
1669
|
+
await setup_compacting_log(resource, 'dt',
|
|
1670
|
+
(bytes) => resource.dt.doc.mergeBytes(bytes),
|
|
1671
|
+
() => resource.dt.doc.toBytes())
|
|
1591
1672
|
}
|
|
1592
1673
|
|
|
1593
|
-
|
|
1674
|
+
// Internal: create Yjs backend on demand, synced from resource.val
|
|
1675
|
+
async function ensure_yjs_exists(resource, options) {
|
|
1594
1676
|
if (resource.yjs) return
|
|
1595
|
-
|
|
1596
|
-
resource.yjs =
|
|
1597
|
-
doc: new Y.Doc(),
|
|
1598
|
-
text: null,
|
|
1599
|
-
save_delta: () => {},
|
|
1600
|
-
}
|
|
1601
|
-
resource.yjs.text = resource.yjs.doc.getText('text')
|
|
1677
|
+
var channel = options?.channel || resource.meta?.yjs_channel || 'text'
|
|
1678
|
+
resource.yjs = make_yjs_backend(channel)
|
|
1602
1679
|
if (resource.val) {
|
|
1603
1680
|
resource.yjs.text.insert(0, resource.val)
|
|
1604
1681
|
}
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
() => Y.encodeStateAsUpdate(resource.yjs.doc),
|
|
1611
|
-
() => {},
|
|
1612
|
-
() => ({}),
|
|
1613
|
-
'yjs')
|
|
1614
|
-
resource.yjs.save_delta = change
|
|
1615
|
-
}
|
|
1682
|
+
resource.meta.yjs_channel = channel
|
|
1683
|
+
resource.save_meta()
|
|
1684
|
+
await setup_compacting_log(resource, 'yjs',
|
|
1685
|
+
(bytes) => Y.applyUpdate(resource.yjs.doc, bytes),
|
|
1686
|
+
() => Y.encodeStateAsUpdate(resource.yjs.doc))
|
|
1616
1687
|
}
|
|
1617
1688
|
|
|
1618
1689
|
async function db_folder_init() {
|
|
@@ -1718,133 +1789,7 @@ function create_braid_text() {
|
|
|
1718
1789
|
} catch (e) { return [] }
|
|
1719
1790
|
}
|
|
1720
1791
|
|
|
1721
|
-
|
|
1722
|
-
await db_folder_init()
|
|
1723
|
-
let encoded = encode_filename(key)
|
|
1724
|
-
file_type = file_type || 'dt'
|
|
1725
|
-
|
|
1726
|
-
if (encoded.length > max_encoded_key_size) throw new Error(`invalid key: too long (max ${max_encoded_key_size})`)
|
|
1727
|
-
|
|
1728
|
-
let currentNumber = 0
|
|
1729
|
-
let currentSize = 0
|
|
1730
|
-
let threshold = 0
|
|
1731
|
-
|
|
1732
|
-
// Read existing files and sort by numbers.
|
|
1733
|
-
const files = (await get_files_for_key(key, file_type))
|
|
1734
|
-
.filter(x => x.match(/\.\d+$/))
|
|
1735
|
-
.sort((a, b) => parseInt(a.match(/\d+$/)[0]) - parseInt(b.match(/\d+$/)[0]))
|
|
1736
|
-
|
|
1737
|
-
// Try to process files starting from the highest number.
|
|
1738
|
-
let done = false
|
|
1739
|
-
for (let i = files.length - 1; i >= 0; i--) {
|
|
1740
|
-
if (done) {
|
|
1741
|
-
await fs.promises.unlink(files[i])
|
|
1742
|
-
continue
|
|
1743
|
-
}
|
|
1744
|
-
try {
|
|
1745
|
-
const filename = files[i]
|
|
1746
|
-
if (braid_text.verbose) console.log(`trying to process file: ${filename}`)
|
|
1747
|
-
const data = await fs.promises.readFile(filename)
|
|
1748
|
-
|
|
1749
|
-
let cursor = 0
|
|
1750
|
-
let isFirstChunk = true
|
|
1751
|
-
while (cursor < data.length) {
|
|
1752
|
-
const chunkSize = data.readUInt32LE(cursor)
|
|
1753
|
-
cursor += 4
|
|
1754
|
-
const chunk = data.slice(cursor, cursor + chunkSize)
|
|
1755
|
-
cursor += chunkSize
|
|
1756
|
-
|
|
1757
|
-
if (isFirstChunk) {
|
|
1758
|
-
isFirstChunk = false
|
|
1759
|
-
threshold = chunkSize * 10
|
|
1760
|
-
}
|
|
1761
|
-
process_delta(chunk)
|
|
1762
|
-
}
|
|
1763
|
-
|
|
1764
|
-
currentSize = data.length
|
|
1765
|
-
currentNumber = parseInt(filename.match(/(\d+)$/)[1])
|
|
1766
|
-
done = true
|
|
1767
|
-
} catch (error) {
|
|
1768
|
-
console.error(`Error processing file: ${files[i]}`)
|
|
1769
|
-
await fs.promises.unlink(files[i])
|
|
1770
|
-
}
|
|
1771
|
-
}
|
|
1772
|
-
|
|
1773
|
-
var meta_filename = `${braid_text.db_folder}/meta/${encoded}`
|
|
1774
|
-
var meta_dirty = null
|
|
1775
|
-
var meta_saving = null
|
|
1776
|
-
var meta_file_content = '{}'
|
|
1777
|
-
try {
|
|
1778
|
-
var meta_file_content = await fs.promises.readFile(meta_filename)
|
|
1779
|
-
} catch (e) {}
|
|
1780
|
-
set_meta(JSON.parse(meta_file_content))
|
|
1781
|
-
|
|
1782
|
-
return {
|
|
1783
|
-
change: (bytes) => within_fiber('file:' + key, async () => {
|
|
1784
|
-
if (!bytes) currentSize = threshold
|
|
1785
|
-
else currentSize += bytes.length + 4 // we account for the extra 4 bytes for uint32
|
|
1786
|
-
const filename = `${braid_text.db_folder}/${encoded}.${file_type}.${currentNumber}`
|
|
1787
|
-
if (currentSize < threshold) {
|
|
1788
|
-
if (braid_text.verbose) console.log(`appending to db..`)
|
|
1789
|
-
|
|
1790
|
-
let len_buf = Buffer.allocUnsafe(4)
|
|
1791
|
-
len_buf.writeUInt32LE(bytes.length, 0)
|
|
1792
|
-
let append_data = Buffer.concat([len_buf, bytes])
|
|
1793
|
-
|
|
1794
|
-
let basename = require('path').basename(filename)
|
|
1795
|
-
let intent_path = `${braid_text.db_folder}/wal-intent/${basename}`
|
|
1796
|
-
let stat = await fs.promises.stat(filename)
|
|
1797
|
-
let size_buf = Buffer.allocUnsafe(8)
|
|
1798
|
-
size_buf.writeBigUInt64LE(BigInt(stat.size), 0)
|
|
1799
|
-
|
|
1800
|
-
await atomic_write(intent_path, Buffer.concat([size_buf, append_data]),
|
|
1801
|
-
`${braid_text.db_folder}/temp`)
|
|
1802
|
-
await fs.promises.appendFile(filename, append_data)
|
|
1803
|
-
await fs.promises.unlink(intent_path)
|
|
1804
|
-
|
|
1805
|
-
if (braid_text.verbose) console.log("wrote to : " + filename)
|
|
1806
|
-
} else {
|
|
1807
|
-
try {
|
|
1808
|
-
if (braid_text.verbose) console.log(`starting new db..`)
|
|
1809
|
-
|
|
1810
|
-
currentNumber++
|
|
1811
|
-
const init = get_init()
|
|
1812
|
-
const buffer = Buffer.allocUnsafe(4)
|
|
1813
|
-
buffer.writeUInt32LE(init.length, 0)
|
|
1814
|
-
|
|
1815
|
-
const newFilename = `${braid_text.db_folder}/${encoded}.${file_type}.${currentNumber}`
|
|
1816
|
-
await atomic_write(newFilename, Buffer.concat([buffer, init]),
|
|
1817
|
-
`${braid_text.db_folder}/temp`)
|
|
1818
|
-
|
|
1819
|
-
if (braid_text.verbose) console.log("wrote to : " + newFilename)
|
|
1820
|
-
|
|
1821
|
-
currentSize = 4 + init.length
|
|
1822
|
-
threshold = currentSize * 10
|
|
1823
|
-
try {
|
|
1824
|
-
await fs.promises.unlink(filename)
|
|
1825
|
-
} catch (e) { }
|
|
1826
|
-
} catch (e) {
|
|
1827
|
-
if (braid_text.verbose) console.log(`e = ${e.stack}`)
|
|
1828
|
-
}
|
|
1829
|
-
}
|
|
1830
|
-
}),
|
|
1831
|
-
change_meta: async () => {
|
|
1832
|
-
meta_dirty = true
|
|
1833
|
-
if (meta_saving) return
|
|
1834
|
-
meta_saving = true
|
|
1835
|
-
|
|
1836
|
-
while (meta_dirty) {
|
|
1837
|
-
meta_dirty = false
|
|
1838
|
-
await atomic_write(meta_filename, JSON.stringify(get_meta()),
|
|
1839
|
-
`${braid_text.db_folder}/temp`)
|
|
1840
|
-
await new Promise(done => setTimeout(done,
|
|
1841
|
-
braid_text.meta_file_save_period_ms))
|
|
1842
|
-
}
|
|
1843
|
-
|
|
1844
|
-
meta_saving = false
|
|
1845
|
-
}
|
|
1846
|
-
}
|
|
1847
|
-
}
|
|
1792
|
+
// (file_sync removed — replaced by setup_meta and setup_compacting_log above)
|
|
1848
1793
|
|
|
1849
1794
|
async function wait_for_events(
|
|
1850
1795
|
key,
|
|
@@ -2085,11 +2030,11 @@ function create_braid_text() {
|
|
|
2085
2030
|
let I = base_i + j
|
|
2086
2031
|
if (
|
|
2087
2032
|
j == len ||
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2033
|
+
parentss[I].length != 1 ||
|
|
2034
|
+
parentss[I][0][0] != versions[I - 1][0] ||
|
|
2035
|
+
parentss[I][0][1] != versions[I - 1][1] ||
|
|
2036
|
+
versions[I][0] != versions[I - 1][0] ||
|
|
2037
|
+
versions[I][1] != versions[I - 1][1] + 1
|
|
2093
2038
|
) {
|
|
2094
2039
|
for (; i < I; i++) {
|
|
2095
2040
|
let version = versions[i].join("-")
|
|
@@ -2110,8 +2055,8 @@ function create_braid_text() {
|
|
|
2110
2055
|
parentss[og_i].map((x) => x.join("-")),
|
|
2111
2056
|
op_run.fwd ?
|
|
2112
2057
|
(op_run.content ?
|
|
2113
|
-
|
|
2114
|
-
|
|
2058
|
+
op_run.start + (og_i - base_i) :
|
|
2059
|
+
op_run.start) :
|
|
2115
2060
|
op_run.end - 1 - (i - base_i),
|
|
2116
2061
|
op_run.content ? 0 : i - og_i + 1,
|
|
2117
2062
|
content
|
|
@@ -2126,7 +2071,7 @@ function create_braid_text() {
|
|
|
2126
2071
|
|
|
2127
2072
|
function dt_get_patches(doc, version = null) {
|
|
2128
2073
|
if (version && v_eq(version,
|
|
2129
|
-
|
|
2074
|
+
doc.getRemoteVersion().map((x) => x.join("-")).sort())) {
|
|
2130
2075
|
// they want everything past the end, which is nothing
|
|
2131
2076
|
return []
|
|
2132
2077
|
}
|
|
@@ -2166,22 +2111,22 @@ function create_braid_text() {
|
|
|
2166
2111
|
let I = i + j
|
|
2167
2112
|
if (
|
|
2168
2113
|
(!op_run.content && op_run.fwd) ||
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2114
|
+
j == len ||
|
|
2115
|
+
parentss[I].length != 1 ||
|
|
2116
|
+
parentss[I][0][0] != versions[I - 1][0] ||
|
|
2117
|
+
parentss[I][0][1] != versions[I - 1][1] ||
|
|
2118
|
+
versions[I][0] != versions[I - 1][0] ||
|
|
2119
|
+
versions[I][1] != versions[I - 1][1] + 1
|
|
2175
2120
|
) {
|
|
2176
2121
|
let s = op_run.fwd ?
|
|
2177
2122
|
(op_run.content ?
|
|
2178
|
-
|
|
2179
|
-
|
|
2123
|
+
start :
|
|
2124
|
+
op_run.start) :
|
|
2180
2125
|
(op_run.start + (op_run.end - end))
|
|
2181
2126
|
let e = op_run.fwd ?
|
|
2182
2127
|
(op_run.content ?
|
|
2183
|
-
|
|
2184
|
-
|
|
2128
|
+
end :
|
|
2129
|
+
op_run.start + (end - start)) :
|
|
2185
2130
|
(op_run.end - (start - op_run.start))
|
|
2186
2131
|
patches.push({
|
|
2187
2132
|
version: `${version[0]}-${version[1] + e - s - 1}`,
|
|
@@ -2391,19 +2336,19 @@ function create_braid_text() {
|
|
|
2391
2336
|
return local_version
|
|
2392
2337
|
|
|
2393
2338
|
function splice_out_range(a, s, e) {
|
|
2394
|
-
if (!a?.length) return []
|
|
2395
|
-
|
|
2339
|
+
if (!a?.length) return []
|
|
2340
|
+
var l = 0, r = a.length
|
|
2396
2341
|
while (l < r) {
|
|
2397
|
-
|
|
2398
|
-
if (a[m] < s) l = m + 1; else r = m
|
|
2342
|
+
var m = Math.floor((l + r) / 2)
|
|
2343
|
+
if (a[m] < s) l = m + 1; else r = m
|
|
2399
2344
|
}
|
|
2400
|
-
|
|
2401
|
-
l = i; r = a.length
|
|
2345
|
+
var i = l
|
|
2346
|
+
l = i; r = a.length
|
|
2402
2347
|
while (l < r) {
|
|
2403
|
-
|
|
2404
|
-
if (a[m] <= e) l = m + 1; else r = m
|
|
2348
|
+
m = Math.floor((l + r) / 2)
|
|
2349
|
+
if (a[m] <= e) l = m + 1; else r = m
|
|
2405
2350
|
}
|
|
2406
|
-
return a.splice(i, l - i)
|
|
2351
|
+
return a.splice(i, l - i)
|
|
2407
2352
|
}
|
|
2408
2353
|
}
|
|
2409
2354
|
|
|
@@ -2623,11 +2568,11 @@ function create_braid_text() {
|
|
|
2623
2568
|
range: `[${xf.start}:${xf.start}]`,
|
|
2624
2569
|
content: xf.content,
|
|
2625
2570
|
}
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2571
|
+
: {
|
|
2572
|
+
unit: "text",
|
|
2573
|
+
range: `[${xf.start}:${xf.end}]`,
|
|
2574
|
+
content: "",
|
|
2575
|
+
}
|
|
2631
2576
|
)
|
|
2632
2577
|
}
|
|
2633
2578
|
var result = relative_to_absolute_patches(patches)
|
|
@@ -2686,7 +2631,7 @@ function create_braid_text() {
|
|
|
2686
2631
|
resize(x, count_code_points(x.content))
|
|
2687
2632
|
resize(node, node.size - (start + del))
|
|
2688
2633
|
} else {
|
|
2689
|
-
node.content = node.content.slice(0,
|
|
2634
|
+
node.content = node.content.slice(0, codepoints_to_index(node.content, start)) + p.content + node.content.slice(codepoints_to_index(node.content, start + del))
|
|
2690
2635
|
resize(node, count_code_points(node.content))
|
|
2691
2636
|
}
|
|
2692
2637
|
} else {
|
|
@@ -2719,7 +2664,7 @@ function create_braid_text() {
|
|
|
2719
2664
|
resize(next, next.size - remaining)
|
|
2720
2665
|
} else {
|
|
2721
2666
|
next.del += node.size - start + middle_del
|
|
2722
|
-
next.content = p.content + next.content.slice(
|
|
2667
|
+
next.content = p.content + next.content.slice(codepoints_to_index(next.content, remaining))
|
|
2723
2668
|
resize(node, start)
|
|
2724
2669
|
if (node.size == 0) avl.del(node)
|
|
2725
2670
|
resize(next, count_code_points(next.content))
|
|
@@ -2727,12 +2672,12 @@ function create_braid_text() {
|
|
|
2727
2672
|
} else {
|
|
2728
2673
|
if (next.content == null) {
|
|
2729
2674
|
node.del += middle_del + remaining
|
|
2730
|
-
node.content = node.content.slice(0,
|
|
2675
|
+
node.content = node.content.slice(0, codepoints_to_index(node.content, start)) + p.content
|
|
2731
2676
|
resize(node, count_code_points(node.content))
|
|
2732
2677
|
resize(next, next.size - remaining)
|
|
2733
2678
|
} else {
|
|
2734
2679
|
node.del += middle_del + next.del
|
|
2735
|
-
node.content = node.content.slice(0,
|
|
2680
|
+
node.content = node.content.slice(0, codepoints_to_index(node.content, start)) + p.content + next.content.slice(codepoints_to_index(next.content, remaining))
|
|
2736
2681
|
resize(node, count_code_points(node.content))
|
|
2737
2682
|
resize(next, 0)
|
|
2738
2683
|
avl.del(next)
|
|
@@ -2884,23 +2829,21 @@ function create_braid_text() {
|
|
|
2884
2829
|
return code_points;
|
|
2885
2830
|
}
|
|
2886
2831
|
|
|
2887
|
-
function
|
|
2888
|
-
|
|
2889
|
-
let c = 0
|
|
2832
|
+
function index_to_codepoints(str, index) {
|
|
2833
|
+
var i = 0, c = 0
|
|
2890
2834
|
while (i < index && i < str.length) {
|
|
2891
|
-
|
|
2892
|
-
i += (
|
|
2835
|
+
var code = str.charCodeAt(i)
|
|
2836
|
+
i += (code >= 0xd800 && code <= 0xdbff) ? 2 : 1
|
|
2893
2837
|
c++
|
|
2894
2838
|
}
|
|
2895
2839
|
return c
|
|
2896
2840
|
}
|
|
2897
2841
|
|
|
2898
|
-
function
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
i += (charCode >= 0xd800 && charCode <= 0xdbff) ? 2 : 1
|
|
2842
|
+
function codepoints_to_index(str, codepoints) {
|
|
2843
|
+
var i = 0, c = 0
|
|
2844
|
+
while (c < codepoints && i < str.length) {
|
|
2845
|
+
var code = str.charCodeAt(i)
|
|
2846
|
+
i += (code >= 0xd800 && code <= 0xdbff) ? 2 : 1
|
|
2904
2847
|
c++
|
|
2905
2848
|
}
|
|
2906
2849
|
return i
|
|
@@ -3016,12 +2959,12 @@ function create_braid_text() {
|
|
|
3016
2959
|
var yjs_id_pattern = '(\\d+-\\d+)'
|
|
3017
2960
|
var yjs_range_re = new RegExp(
|
|
3018
2961
|
'^\\s*' +
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
2962
|
+
'([\\(\\[])' + // open bracket: ( or [
|
|
2963
|
+
'\\s*' + yjs_id_pattern + '?' + // optional left ID
|
|
2964
|
+
'\\s*:\\s*' + // colon separator
|
|
2965
|
+
yjs_id_pattern + '?' + '\\s*' + // optional right ID
|
|
2966
|
+
'([\\)\\]])' + // close bracket: ) or ]
|
|
2967
|
+
'\\s*$'
|
|
3025
2968
|
)
|
|
3026
2969
|
|
|
3027
2970
|
function validate_yjs_patch(x) {
|
|
@@ -3123,48 +3066,60 @@ function create_braid_text() {
|
|
|
3123
3066
|
// Convert a Yjs binary update to yjs-text range patches.
|
|
3124
3067
|
// Decodes the binary without needing a Y.Doc.
|
|
3125
3068
|
// Returns array of {unit: 'yjs-text', range: '...', content: '...'}
|
|
3069
|
+
// Convert a Yjs binary update to an array of braid updates,
|
|
3070
|
+
// each with a version and patches in yjs-text format.
|
|
3126
3071
|
braid_text.from_yjs_binary = function(update) {
|
|
3127
3072
|
require_yjs()
|
|
3128
3073
|
var decoded = Y.decodeUpdate(
|
|
3129
3074
|
update instanceof Uint8Array ? update : new Uint8Array(update))
|
|
3130
|
-
var
|
|
3075
|
+
var updates = []
|
|
3131
3076
|
|
|
3132
|
-
//
|
|
3077
|
+
// Each inserted struct becomes one update with one insert patch.
|
|
3078
|
+
// GC'd structs (deleted items) have content.len but no content.str —
|
|
3079
|
+
// we emit placeholder text since the delete set will remove it anyway.
|
|
3133
3080
|
for (var struct of decoded.structs) {
|
|
3134
|
-
|
|
3081
|
+
var text = struct.content?.str
|
|
3082
|
+
if (!text && struct.content?.len) text = '_'.repeat(struct.content.len)
|
|
3083
|
+
if (!text) continue // skip non-text items (e.g. format, embed)
|
|
3135
3084
|
var id = struct.id
|
|
3136
3085
|
var origin = struct.origin
|
|
3137
3086
|
var rightOrigin = struct.rightOrigin
|
|
3138
3087
|
var left = origin ? `${origin.client}-${origin.clock}` : ''
|
|
3139
3088
|
var right = rightOrigin ? `${rightOrigin.client}-${rightOrigin.clock}` : ''
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3089
|
+
updates.push({
|
|
3090
|
+
version: [`${id.client}-${id.clock}`],
|
|
3091
|
+
patches: [{
|
|
3092
|
+
unit: 'yjs-text',
|
|
3093
|
+
range: `(${left}:${right})`,
|
|
3094
|
+
content: text,
|
|
3095
|
+
}]
|
|
3144
3096
|
})
|
|
3145
3097
|
}
|
|
3146
3098
|
|
|
3147
|
-
//
|
|
3099
|
+
// Each delete range becomes one update with one delete patch
|
|
3148
3100
|
for (var [clientID, deleteItems] of decoded.ds.clients) {
|
|
3149
3101
|
for (var item of deleteItems) {
|
|
3150
3102
|
var left = `${clientID}-${item.clock}`
|
|
3151
3103
|
var right = `${clientID}-${item.clock + item.len - 1}`
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3104
|
+
updates.push({
|
|
3105
|
+
version: [`${clientID}-${item.clock}`],
|
|
3106
|
+
patches: [{
|
|
3107
|
+
unit: 'yjs-text',
|
|
3108
|
+
range: `[${left}:${right}]`,
|
|
3109
|
+
content: ''
|
|
3110
|
+
}]
|
|
3156
3111
|
})
|
|
3157
3112
|
}
|
|
3158
3113
|
}
|
|
3159
3114
|
|
|
3160
|
-
return
|
|
3115
|
+
return updates
|
|
3161
3116
|
}
|
|
3162
3117
|
|
|
3163
3118
|
// Convert yjs-text range patches to a Yjs binary update.
|
|
3119
|
+
// Convert braid updates with yjs-text patches to a Yjs binary update.
|
|
3164
3120
|
// This is the inverse of from_yjs_binary.
|
|
3165
|
-
//
|
|
3166
|
-
|
|
3167
|
-
braid_text.to_yjs_binary = function(patches) {
|
|
3121
|
+
// Accepts an array of updates, each with {version, patches}.
|
|
3122
|
+
braid_text.to_yjs_binary = function(updates) {
|
|
3168
3123
|
require_yjs()
|
|
3169
3124
|
var lib0_encoding = require('lib0/encoding')
|
|
3170
3125
|
var encoder = new Y.UpdateEncoderV1()
|
|
@@ -3173,23 +3128,30 @@ function create_braid_text() {
|
|
|
3173
3128
|
var inserts_by_client = new Map()
|
|
3174
3129
|
var deletes_by_client = new Map()
|
|
3175
3130
|
|
|
3176
|
-
for (var
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
if (
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3131
|
+
for (var update of updates) {
|
|
3132
|
+
if (!update.patches) continue
|
|
3133
|
+
for (var p of update.patches) {
|
|
3134
|
+
var parsed = parse_yjs_range(p.range)
|
|
3135
|
+
if (!parsed) throw new Error(`invalid yjs-text range: ${p.range}`)
|
|
3136
|
+
|
|
3137
|
+
if (p.content.length > 0) {
|
|
3138
|
+
// Insert — version on the update is the item ID as ["client-clock"]
|
|
3139
|
+
var v_str = Array.isArray(update.version) ? update.version[0] : update.version
|
|
3140
|
+
if (!v_str) throw new Error('insert update requires .version = ["client-clock"]')
|
|
3141
|
+
var v_parts = v_str.match(/^(\d+)-(\d+)$/)
|
|
3142
|
+
if (!v_parts) throw new Error('invalid update version: ' + v_str)
|
|
3143
|
+
var item_id = { client: parseInt(v_parts[1]), clock: parseInt(v_parts[2]) }
|
|
3144
|
+
var list = inserts_by_client.get(item_id.client) || []
|
|
3145
|
+
list.push({ id: item_id, origin: parsed.left, rightOrigin: parsed.right, content: p.content })
|
|
3146
|
+
inserts_by_client.set(item_id.client, list)
|
|
3147
|
+
} else {
|
|
3148
|
+
// Delete
|
|
3149
|
+
if (!parsed.left) throw new Error('delete patch requires left ID')
|
|
3150
|
+
var client = parsed.left.client
|
|
3151
|
+
var list = deletes_by_client.get(client) || []
|
|
3152
|
+
list.push({ clock: parsed.left.clock, len: parsed.right ? parsed.right.clock - parsed.left.clock + 1 : 1 })
|
|
3153
|
+
deletes_by_client.set(client, list)
|
|
3154
|
+
}
|
|
3193
3155
|
}
|
|
3194
3156
|
}
|
|
3195
3157
|
|
|
@@ -3345,35 +3307,23 @@ function create_braid_text() {
|
|
|
3345
3307
|
return second_patches
|
|
3346
3308
|
}
|
|
3347
3309
|
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3310
|
+
// Create a LRU cache. It evicts stuff when it gets greater than `size` in LRU order.
|
|
3311
|
+
function create_simple_cache(size) {
|
|
3312
|
+
// This map will iterate over keys in the order they were inserted.
|
|
3313
|
+
// Eviction will remove from the front of the map.
|
|
3314
|
+
// So we want every delete() and set() operation to move a key/value to the end.
|
|
3315
|
+
var map = new Map()
|
|
3352
3316
|
return {
|
|
3353
3317
|
put(key, value) {
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
cache.set(key, value)
|
|
3358
|
-
} else {
|
|
3359
|
-
// If the cache is full, remove the oldest entry
|
|
3360
|
-
if (cache.size >= maxSize) {
|
|
3361
|
-
const oldestKey = cache.keys().next().value
|
|
3362
|
-
cache.delete(oldestKey)
|
|
3363
|
-
}
|
|
3364
|
-
// Add the new key-value pair
|
|
3365
|
-
cache.set(key, value)
|
|
3366
|
-
}
|
|
3318
|
+
map.delete(key) // remove first so existing keys don't trigger eviction
|
|
3319
|
+
if (map.size >= size) map.delete(map.keys().next().value)
|
|
3320
|
+
map.set(key, value)
|
|
3367
3321
|
},
|
|
3368
|
-
|
|
3369
3322
|
get(key) {
|
|
3370
|
-
if (!
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
const value = cache.get(key)
|
|
3375
|
-
cache.delete(key)
|
|
3376
|
-
cache.set(key, value)
|
|
3323
|
+
if (!map.has(key)) return null
|
|
3324
|
+
var value = map.get(key)
|
|
3325
|
+
map.delete(key)
|
|
3326
|
+
map.set(key, value)
|
|
3377
3327
|
return value
|
|
3378
3328
|
},
|
|
3379
3329
|
}
|
|
@@ -3441,7 +3391,7 @@ function create_braid_text() {
|
|
|
3441
3391
|
else { // - Array set
|
|
3442
3392
|
console.assert(slice_end >= 0, 'Index '+subpath+' is too small')
|
|
3443
3393
|
console.assert(slice_end <= curr_obj.length - 1,
|
|
3444
|
-
|
|
3394
|
+
'Index '+subpath+' is too big')
|
|
3445
3395
|
curr_obj[slice_end] = new_stuff
|
|
3446
3396
|
}
|
|
3447
3397
|
|
|
@@ -3465,16 +3415,15 @@ function create_braid_text() {
|
|
|
3465
3415
|
add_range(low_inclusive, high_inclusive) {
|
|
3466
3416
|
if (low_inclusive > high_inclusive) throw new Error('invalid range')
|
|
3467
3417
|
|
|
3468
|
-
|
|
3469
|
-
|
|
3418
|
+
var start_i = this._bs(mid => this.ranges[mid][1] >= low_inclusive - 1, this.ranges.length, true)
|
|
3419
|
+
var end_i = this._bs(mid => this.ranges[mid][0] <= high_inclusive + 1, -1, false)
|
|
3470
3420
|
|
|
3471
|
-
if (
|
|
3472
|
-
this.ranges.splice(
|
|
3473
|
-
|
|
3474
|
-
|
|
3475
|
-
|
|
3476
|
-
|
|
3477
|
-
this.ranges.splice(startIndex, removeCount, [mergedLow, mergedHigh])
|
|
3421
|
+
if (start_i > end_i)
|
|
3422
|
+
this.ranges.splice(start_i, 0, [low_inclusive, high_inclusive])
|
|
3423
|
+
else {
|
|
3424
|
+
var merged_low = Math.min(low_inclusive, this.ranges[start_i][0])
|
|
3425
|
+
var merged_high = Math.max(high_inclusive, this.ranges[end_i][1])
|
|
3426
|
+
this.ranges.splice(start_i, end_i - start_i + 1, [merged_low, merged_high])
|
|
3478
3427
|
}
|
|
3479
3428
|
}
|
|
3480
3429
|
|
|
@@ -3484,20 +3433,15 @@ function create_braid_text() {
|
|
|
3484
3433
|
return index !== -1 && x <= this.ranges[index][1] && this.ranges[index]
|
|
3485
3434
|
}
|
|
3486
3435
|
|
|
3487
|
-
_bs(condition,
|
|
3488
|
-
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
while (low <= high) {
|
|
3493
|
-
const mid = Math.floor((low + high) / 2)
|
|
3436
|
+
_bs(condition, default_r, move_left) {
|
|
3437
|
+
var lo = 0, hi = this.ranges.length - 1, result = default_r
|
|
3438
|
+
while (lo <= hi) {
|
|
3439
|
+
var mid = Math.floor((lo + hi) / 2)
|
|
3494
3440
|
if (condition(mid)) {
|
|
3495
3441
|
result = mid
|
|
3496
|
-
if (
|
|
3497
|
-
else low = mid + 1
|
|
3442
|
+
if (move_left) hi = mid - 1; else lo = mid + 1
|
|
3498
3443
|
} else {
|
|
3499
|
-
if (
|
|
3500
|
-
else high = mid - 1
|
|
3444
|
+
if (move_left) lo = mid + 1; else hi = mid - 1
|
|
3501
3445
|
}
|
|
3502
3446
|
}
|
|
3503
3447
|
return result
|
|
@@ -3653,8 +3597,13 @@ function create_braid_text() {
|
|
|
3653
3597
|
}
|
|
3654
3598
|
|
|
3655
3599
|
braid_text.get_resource = get_resource
|
|
3656
|
-
|
|
3657
|
-
|
|
3600
|
+
|
|
3601
|
+
// Returns the number of connected subscribers for a resource
|
|
3602
|
+
braid_text.subscriber_count = async function(key) {
|
|
3603
|
+
if (!braid_text.cache[key]) return 0
|
|
3604
|
+
var resource = await braid_text.cache[key]
|
|
3605
|
+
return resource.clients().length
|
|
3606
|
+
}
|
|
3658
3607
|
|
|
3659
3608
|
braid_text.db_folder_init = db_folder_init
|
|
3660
3609
|
braid_text.encode_filename = encode_filename
|
|
@@ -3813,8 +3762,13 @@ async function handle_cursors(resource, req, res) {
|
|
|
3813
3762
|
} else {
|
|
3814
3763
|
var subscriber = {peer, res}
|
|
3815
3764
|
cursors.subscribe(subscriber)
|
|
3765
|
+
all_subscriptions.add(res)
|
|
3816
3766
|
res.startSubscription({
|
|
3817
|
-
onClose: () =>
|
|
3767
|
+
onClose: () => {
|
|
3768
|
+
all_subscriptions.delete(res)
|
|
3769
|
+
cursors.subscribers.delete(subscriber)
|
|
3770
|
+
setTimeout(() => cursors.unsubscribe(subscriber), 0)
|
|
3771
|
+
}
|
|
3818
3772
|
})
|
|
3819
3773
|
res.sendUpdate({ body: JSON.stringify(cursors.snapshot()) })
|
|
3820
3774
|
}
|