braid-text 0.3.27 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/cursor-sync.js +24 -3
- package/client/editor.html +1 -1
- package/client/markdown-editor.html +1 -1
- package/client/simpleton-sync.js +22 -0
- package/client/textarea-highlights.js +21 -26
- package/package.json +9 -2
- package/server-demo.js +1 -1
- package/server.js +823 -120
- package/client/cursor-highlights.js +0 -246
package/server.js
CHANGED
|
@@ -3,6 +3,9 @@ let { Doc, OpLog, Branch } = require("@braid.org/diamond-types-node")
|
|
|
3
3
|
let {http_server: braidify, fetch: braid_fetch} = require("braid-http")
|
|
4
4
|
let fs = require("fs")
|
|
5
5
|
|
|
6
|
+
let Y = null
|
|
7
|
+
try { Y = require('yjs') } catch(e) {}
|
|
8
|
+
|
|
6
9
|
|
|
7
10
|
function create_braid_text() {
|
|
8
11
|
let braid_text = {
|
|
@@ -10,9 +13,15 @@ function create_braid_text() {
|
|
|
10
13
|
db_folder: './braid-text-db',
|
|
11
14
|
length_cache_size: 10,
|
|
12
15
|
meta_file_save_period_ms: 1000,
|
|
16
|
+
debug_sync_checks: false,
|
|
13
17
|
cache: {}
|
|
14
18
|
}
|
|
15
19
|
|
|
20
|
+
function require_yjs() {
|
|
21
|
+
if (!Y) throw new Error('yjs is not installed. Install it with: npm install yjs')
|
|
22
|
+
return Y
|
|
23
|
+
}
|
|
24
|
+
|
|
16
25
|
let waiting_puts = 0
|
|
17
26
|
|
|
18
27
|
let max_encoded_key_size = 240
|
|
@@ -53,10 +62,11 @@ function create_braid_text() {
|
|
|
53
62
|
}
|
|
54
63
|
|
|
55
64
|
var resource = (typeof a == 'string') ? await get_resource(a) : a
|
|
65
|
+
await ensure_dt_exists(resource)
|
|
56
66
|
|
|
57
67
|
if (!resource.meta.fork_point && options.fork_point_hint) {
|
|
58
68
|
resource.meta.fork_point = options.fork_point_hint
|
|
59
|
-
resource.
|
|
69
|
+
resource.save_meta()
|
|
60
70
|
}
|
|
61
71
|
|
|
62
72
|
function extend_frontier(frontier, version, parents) {
|
|
@@ -78,7 +88,7 @@ function create_braid_text() {
|
|
|
78
88
|
frontier = []
|
|
79
89
|
var shadow = new Set()
|
|
80
90
|
|
|
81
|
-
var bytes = resource.doc.toBytes()
|
|
91
|
+
var bytes = resource.dt.doc.toBytes()
|
|
82
92
|
var [_, events, parentss] = braid_text.dt_parse([...bytes])
|
|
83
93
|
for (var i = events.length - 1; i >= 0 && looking_for.size; i--) {
|
|
84
94
|
var e = events[i].join('-')
|
|
@@ -139,7 +149,7 @@ function create_braid_text() {
|
|
|
139
149
|
resource.meta.fork_point =
|
|
140
150
|
extend_frontier(resource.meta.fork_point,
|
|
141
151
|
update.version, update.parents)
|
|
142
|
-
resource.
|
|
152
|
+
resource.save_meta()
|
|
143
153
|
}
|
|
144
154
|
|
|
145
155
|
// see if remote has the fork point
|
|
@@ -148,7 +158,7 @@ function create_braid_text() {
|
|
|
148
158
|
if (signal.aborted) return
|
|
149
159
|
|
|
150
160
|
resource.meta.fork_point = null
|
|
151
|
-
resource.
|
|
161
|
+
resource.save_meta()
|
|
152
162
|
}
|
|
153
163
|
if (signal.aborted) return
|
|
154
164
|
|
|
@@ -158,7 +168,7 @@ function create_braid_text() {
|
|
|
158
168
|
// DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
|
|
159
169
|
options.do_investigating_disconnects_log_L04LPFHQ1M?.(a, 'braid-text fork-point binary search')
|
|
160
170
|
|
|
161
|
-
var bytes = resource.doc.toBytes()
|
|
171
|
+
var bytes = resource.dt.doc.toBytes()
|
|
162
172
|
var [_, events, __] = braid_text.dt_parse([...bytes])
|
|
163
173
|
events = events.map(x => x.join('-'))
|
|
164
174
|
|
|
@@ -413,7 +423,7 @@ function create_braid_text() {
|
|
|
413
423
|
var unknowns = []
|
|
414
424
|
for (var event of (req.version || []).concat(req.parents || [])) {
|
|
415
425
|
var [actor, seq] = decode_version(event)
|
|
416
|
-
if (!resource.
|
|
426
|
+
if (!resource.dt?.known_versions[actor]?.has(seq))
|
|
417
427
|
unknowns.push(event)
|
|
418
428
|
}
|
|
419
429
|
if (unknowns.length)
|
|
@@ -494,8 +504,8 @@ function create_braid_text() {
|
|
|
494
504
|
|
|
495
505
|
res.startSubscription({
|
|
496
506
|
onClose: () => {
|
|
497
|
-
if (merge_type === "dt") resource.clients.delete(options)
|
|
498
|
-
else resource.simpleton_clients.delete(options)
|
|
507
|
+
if (merge_type === "dt") resource.dt.clients.delete(options)
|
|
508
|
+
else resource.dt.simpleton_clients.delete(options)
|
|
499
509
|
}
|
|
500
510
|
})
|
|
501
511
|
|
|
@@ -532,10 +542,11 @@ function create_braid_text() {
|
|
|
532
542
|
}
|
|
533
543
|
|
|
534
544
|
if (req.parents) {
|
|
545
|
+
await ensure_dt_exists(resource)
|
|
535
546
|
await wait_for_events(
|
|
536
547
|
options.key,
|
|
537
548
|
req.parents,
|
|
538
|
-
resource.
|
|
549
|
+
resource.dt.known_versions,
|
|
539
550
|
// approximation of memory usage for this update
|
|
540
551
|
body != null ? body.length :
|
|
541
552
|
patches.reduce((a, b) => a + b.range.length + b.content.length, 0),
|
|
@@ -546,7 +557,7 @@ function create_braid_text() {
|
|
|
546
557
|
var unknowns = []
|
|
547
558
|
for (var event of req.parents) {
|
|
548
559
|
var [actor, seq] = decode_version(event)
|
|
549
|
-
if (!resource.
|
|
560
|
+
if (!resource.dt.known_versions[actor]?.has(seq)) unknowns.push(event)
|
|
550
561
|
}
|
|
551
562
|
if (unknowns.length)
|
|
552
563
|
return done_my_turn(309, '', "Version Unknown Here", {
|
|
@@ -557,8 +568,13 @@ function create_braid_text() {
|
|
|
557
568
|
|
|
558
569
|
var old_val = resource.val
|
|
559
570
|
var old_version = resource.version
|
|
560
|
-
var put_patches = patches
|
|
561
|
-
|
|
571
|
+
var put_patches = patches?.map(p => ({unit: p.unit,
|
|
572
|
+
range: p.range,
|
|
573
|
+
content: p.content})) || null
|
|
574
|
+
var {change_count} = await braid_text.put(
|
|
575
|
+
resource,
|
|
576
|
+
{ peer, version: req.version, parents: req.parents, patches, body, merge_type }
|
|
577
|
+
)
|
|
562
578
|
|
|
563
579
|
// if Repr-Digest is set,
|
|
564
580
|
// and the request version is also our new current version,
|
|
@@ -666,6 +682,25 @@ function create_braid_text() {
|
|
|
666
682
|
var version = resource.version
|
|
667
683
|
|
|
668
684
|
if (!options.subscribe) {
|
|
685
|
+
// yjs-text range unit: return current text or full history
|
|
686
|
+
if (options.range_unit === 'yjs-text') {
|
|
687
|
+
await ensure_yjs_exists(resource)
|
|
688
|
+
if (options.parents && options.parents.length === 0) {
|
|
689
|
+
// Full history: root + initialization patch
|
|
690
|
+
var init_patches = [{
|
|
691
|
+
unit: 'yjs-text',
|
|
692
|
+
range: '(:)',
|
|
693
|
+
content: resource.val
|
|
694
|
+
}]
|
|
695
|
+
return {
|
|
696
|
+
version: ['999999999-' + (Math.max(0, [...resource.val].length - 1))],
|
|
697
|
+
parents: [],
|
|
698
|
+
patches: init_patches
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
return { version, body: resource.val }
|
|
702
|
+
}
|
|
703
|
+
|
|
669
704
|
if (options.transfer_encoding === 'dt') {
|
|
670
705
|
// optimization: if requesting current version
|
|
671
706
|
// pretend as if they didn't set a version,
|
|
@@ -676,10 +711,10 @@ function create_braid_text() {
|
|
|
676
711
|
var bytes = null
|
|
677
712
|
if (op_v || options.parents) {
|
|
678
713
|
if (op_v) {
|
|
679
|
-
var doc = dt_get(resource.doc, op_v)
|
|
714
|
+
var doc = dt_get(resource.dt.doc, op_v)
|
|
680
715
|
bytes = doc.toBytes()
|
|
681
716
|
} else {
|
|
682
|
-
bytes = resource.doc.toBytes()
|
|
717
|
+
bytes = resource.dt.doc.toBytes()
|
|
683
718
|
var doc = Doc.fromBytes(bytes)
|
|
684
719
|
}
|
|
685
720
|
if (options.parents) {
|
|
@@ -687,18 +722,45 @@ function create_braid_text() {
|
|
|
687
722
|
dt_get_local_version(bytes, options.parents))
|
|
688
723
|
}
|
|
689
724
|
doc.free()
|
|
690
|
-
} else bytes = resource.doc.toBytes()
|
|
725
|
+
} else bytes = resource.dt.doc.toBytes()
|
|
691
726
|
return { body: bytes }
|
|
692
727
|
}
|
|
693
728
|
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
729
|
+
if (options.version || options.parents) {
|
|
730
|
+
await ensure_dt_exists(resource)
|
|
731
|
+
return {
|
|
732
|
+
version: options.version || options.parents,
|
|
733
|
+
body: dt_get_string(resource.dt.doc, options.version || options.parents)
|
|
734
|
+
}
|
|
735
|
+
} else {
|
|
736
|
+
return { version, body: resource.val }
|
|
700
737
|
}
|
|
701
738
|
} else {
|
|
739
|
+
// yjs-text subscribe: send full history first, then live updates
|
|
740
|
+
if (options.range_unit === 'yjs-text') {
|
|
741
|
+
await ensure_yjs_exists(resource)
|
|
742
|
+
|
|
743
|
+
// Send root
|
|
744
|
+
options.subscribe({version: [], parents: [], body: ""})
|
|
745
|
+
|
|
746
|
+
// Send initialization patch
|
|
747
|
+
if (resource.val) {
|
|
748
|
+
options.subscribe({
|
|
749
|
+
version: ['999999999-' + (Math.max(0, [...resource.val].length - 1))],
|
|
750
|
+
parents: [],
|
|
751
|
+
patches: [{unit: 'yjs-text', range: '(:)', content: resource.val}]
|
|
752
|
+
})
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
// Register for live updates
|
|
756
|
+
if (!resource.yjs_clients) resource.yjs_clients = new Set()
|
|
757
|
+
resource.yjs_clients.add(options)
|
|
758
|
+
options.signal?.addEventListener('abort', () =>
|
|
759
|
+
resource.yjs_clients.delete(options))
|
|
760
|
+
|
|
761
|
+
return
|
|
762
|
+
}
|
|
763
|
+
|
|
702
764
|
options.my_subscribe_chain = Promise.resolve()
|
|
703
765
|
options.my_subscribe = (x) =>
|
|
704
766
|
options.my_subscribe_chain =
|
|
@@ -706,27 +768,32 @@ function create_braid_text() {
|
|
|
706
768
|
options.subscribe(x))
|
|
707
769
|
|
|
708
770
|
if (options.merge_type != "dt") {
|
|
771
|
+
// Simpleton clients require DT
|
|
772
|
+
await ensure_dt_exists(resource)
|
|
773
|
+
|
|
709
774
|
let x = { version }
|
|
710
775
|
|
|
711
776
|
if (!options.parents && !options.version) {
|
|
712
777
|
x.parents = []
|
|
713
|
-
x.body = resource.
|
|
778
|
+
x.body = resource.val
|
|
714
779
|
options.my_subscribe(x)
|
|
715
780
|
} else {
|
|
716
781
|
x.parents = options.version ? options.version : options.parents
|
|
717
782
|
|
|
718
783
|
// only send them a version from these parents if we have these parents (otherwise we'll assume these parents are more recent, probably versions they created but haven't sent us yet, and we'll send them appropriate rebased updates when they send us these versions)
|
|
719
|
-
let local_version = OpLog_remote_to_local(resource.doc, x.parents)
|
|
784
|
+
let local_version = OpLog_remote_to_local(resource.dt.doc, x.parents)
|
|
720
785
|
if (local_version) {
|
|
721
|
-
x.patches = get_xf_patches(resource.doc, local_version)
|
|
786
|
+
x.patches = get_xf_patches(resource.dt.doc, local_version)
|
|
722
787
|
options.my_subscribe(x)
|
|
723
788
|
}
|
|
724
789
|
}
|
|
725
790
|
|
|
726
|
-
resource.simpleton_clients.add(options)
|
|
791
|
+
resource.dt.simpleton_clients.add(options)
|
|
727
792
|
options.signal?.addEventListener('abort', () =>
|
|
728
|
-
resource.simpleton_clients.delete(options))
|
|
793
|
+
resource.dt.simpleton_clients.delete(options))
|
|
729
794
|
} else {
|
|
795
|
+
// DT merge-type clients require DT
|
|
796
|
+
await ensure_dt_exists(resource)
|
|
730
797
|
|
|
731
798
|
if (options.accept_encoding?.match(/updates\s*\((.*)\)/)?.[1].split(',').map(x=>x.trim()).includes('dt')) {
|
|
732
799
|
// optimization: if client wants past current version,
|
|
@@ -734,7 +801,7 @@ function create_braid_text() {
|
|
|
734
801
|
if (options.parents && v_eq(options.parents, version)) {
|
|
735
802
|
options.my_subscribe({ encoding: 'dt', body: new Doc().toBytes() })
|
|
736
803
|
} else {
|
|
737
|
-
var bytes = resource.doc.toBytes()
|
|
804
|
+
var bytes = resource.dt.doc.toBytes()
|
|
738
805
|
if (options.parents) {
|
|
739
806
|
var doc = Doc.fromBytes(bytes)
|
|
740
807
|
bytes = doc.getPatchSince(
|
|
@@ -752,10 +819,10 @@ function create_braid_text() {
|
|
|
752
819
|
body: "",
|
|
753
820
|
})
|
|
754
821
|
|
|
755
|
-
updates = dt_get_patches(resource.doc)
|
|
822
|
+
updates = dt_get_patches(resource.dt.doc)
|
|
756
823
|
} else {
|
|
757
824
|
// Then start the subscription from the parents in options
|
|
758
|
-
updates = dt_get_patches(resource.doc, options.parents || options.version)
|
|
825
|
+
updates = dt_get_patches(resource.dt.doc, options.parents || options.version)
|
|
759
826
|
}
|
|
760
827
|
|
|
761
828
|
for (let u of updates)
|
|
@@ -772,9 +839,9 @@ function create_braid_text() {
|
|
|
772
839
|
if (updates.length === 0) options.write?.("\r\n")
|
|
773
840
|
}
|
|
774
841
|
|
|
775
|
-
resource.clients.add(options)
|
|
842
|
+
resource.dt.clients.add(options)
|
|
776
843
|
options.signal?.addEventListener('abort', () =>
|
|
777
|
-
resource.clients.delete(options))
|
|
844
|
+
resource.dt.clients.delete(options))
|
|
778
845
|
}
|
|
779
846
|
}
|
|
780
847
|
}
|
|
@@ -789,8 +856,8 @@ function create_braid_text() {
|
|
|
789
856
|
let resource = (typeof key == 'string') ? await get_resource(key) : key
|
|
790
857
|
|
|
791
858
|
if (options.merge_type != "dt")
|
|
792
|
-
resource.simpleton_clients.delete(options)
|
|
793
|
-
else resource.clients.delete(options)
|
|
859
|
+
resource.dt.simpleton_clients.delete(options)
|
|
860
|
+
else resource.dt.clients.delete(options)
|
|
794
861
|
}
|
|
795
862
|
|
|
796
863
|
braid_text.put = async (key, options) => {
|
|
@@ -823,7 +890,7 @@ function create_braid_text() {
|
|
|
823
890
|
// support for json patch puts..
|
|
824
891
|
if (options.patches && options.patches.length &&
|
|
825
892
|
options.patches.every(x => x.unit === 'json')) {
|
|
826
|
-
let x = JSON.parse(resource.
|
|
893
|
+
let x = JSON.parse(resource.val)
|
|
827
894
|
for (let p of options.patches)
|
|
828
895
|
apply_patch(x, p.range, p.content === '' ? undefined : JSON.parse(p.content))
|
|
829
896
|
options = { body: JSON.stringify(x, null, 4) }
|
|
@@ -831,31 +898,205 @@ function create_braid_text() {
|
|
|
831
898
|
|
|
832
899
|
let { version, parents, patches, body, peer } = options
|
|
833
900
|
|
|
901
|
+
// Raw Yjs binary update: apply directly to Y.Doc, sync to DT
|
|
902
|
+
if (options.yjs_update) {
|
|
903
|
+
await ensure_yjs_exists(resource)
|
|
904
|
+
|
|
905
|
+
// Pre-sync check: DT and Yjs should agree before we start
|
|
906
|
+
if (braid_text.debug_sync_checks && resource.dt) {
|
|
907
|
+
var pre_dt = resource.dt.doc.get()
|
|
908
|
+
var pre_yjs = resource.yjs.text.toString()
|
|
909
|
+
if (pre_dt !== pre_yjs) {
|
|
910
|
+
console.error(`PRE-SYNC MISMATCH key=${resource.key}: DT="${pre_dt.slice(0,50)}" Yjs="${pre_yjs.slice(0,50)}"`)
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
var prev_text = resource.yjs.text.toString()
|
|
915
|
+
var delta = null
|
|
916
|
+
var observer = (e) => { delta = e.changes.delta }
|
|
917
|
+
resource.yjs.text.observe(observer)
|
|
918
|
+
try {
|
|
919
|
+
Y.applyUpdate(resource.yjs.doc,
|
|
920
|
+
options.yjs_update instanceof Uint8Array ? options.yjs_update : new Uint8Array(options.yjs_update))
|
|
921
|
+
} finally {
|
|
922
|
+
resource.yjs.text.unobserve(observer)
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
resource.val = resource.yjs.text.toString()
|
|
926
|
+
|
|
927
|
+
// Sync to DT if it exists
|
|
928
|
+
if (resource.dt && delta) {
|
|
929
|
+
var text_patches = yjs_delta_to_patches(delta, prev_text)
|
|
930
|
+
if (text_patches.length) {
|
|
931
|
+
var syn_actor = `yjs-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
|
932
|
+
var syn_seq = 0
|
|
933
|
+
var yjs_v_before = resource.dt.doc.getLocalVersion()
|
|
934
|
+
// Patches are sequential (each relative to state after previous)
|
|
935
|
+
// so no offset adjustment needed
|
|
936
|
+
var dt_bytes = []
|
|
937
|
+
var dt_ps = resource.version
|
|
938
|
+
for (var tp of text_patches) {
|
|
939
|
+
var tp_range = tp.range.match(/-?\d+/g).map(Number)
|
|
940
|
+
var tp_del = tp_range[1] - tp_range[0]
|
|
941
|
+
var syn_v = `${syn_actor}-${syn_seq}`
|
|
942
|
+
if (tp_del) {
|
|
943
|
+
dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], tp_del, null))
|
|
944
|
+
dt_ps = [`${syn_actor}-${syn_seq + tp_del - 1}`]
|
|
945
|
+
syn_seq += tp_del
|
|
946
|
+
syn_v = `${syn_actor}-${syn_seq}`
|
|
947
|
+
}
|
|
948
|
+
if (tp.content.length) {
|
|
949
|
+
dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], 0, tp.content))
|
|
950
|
+
var cp_len = [...tp.content].length
|
|
951
|
+
dt_ps = [`${syn_actor}-${syn_seq + cp_len - 1}`]
|
|
952
|
+
syn_seq += cp_len
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
for (var b of dt_bytes) resource.dt.doc.mergeBytes(b)
|
|
956
|
+
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
957
|
+
if (!resource.dt.known_versions[syn_actor]) resource.dt.known_versions[syn_actor] = new RangeSet()
|
|
958
|
+
resource.dt.known_versions[syn_actor].add_range(0, syn_seq - 1)
|
|
959
|
+
await resource.dt.save_delta(resource.dt.doc.getPatchSince(yjs_v_before))
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
// Persist Yjs
|
|
964
|
+
if (resource.yjs.save_delta) await resource.yjs.save_delta(
|
|
965
|
+
options.yjs_update instanceof Uint8Array ? options.yjs_update : new Uint8Array(options.yjs_update))
|
|
966
|
+
|
|
967
|
+
// Sanity check
|
|
968
|
+
if (braid_text.debug_sync_checks && resource.dt) {
|
|
969
|
+
var dt_text = resource.dt.doc.get()
|
|
970
|
+
var yjs_text = resource.yjs.text.toString()
|
|
971
|
+
if (dt_text !== yjs_text) {
|
|
972
|
+
console.error(`SYNC MISMATCH key=${resource.key}: DT text !== Y.Doc text`)
|
|
973
|
+
console.error(` DT: ${dt_text.slice(0, 100)}... (${dt_text.length})`)
|
|
974
|
+
console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
return { change_count: 1 }
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
// yjs-text patches: apply to Y.Doc, sync to DT
|
|
982
|
+
if (patches && patches.length && patches[0].unit === 'yjs-text') {
|
|
983
|
+
await ensure_yjs_exists(resource)
|
|
984
|
+
|
|
985
|
+
// Convert yjs-text patches to binary and apply to Y.Doc
|
|
986
|
+
var binary = braid_text.to_yjs_binary(patches)
|
|
987
|
+
var prev_text = resource.yjs.text.toString()
|
|
988
|
+
var delta = null
|
|
989
|
+
var observer = (e) => { delta = e.changes.delta }
|
|
990
|
+
resource.yjs.text.observe(observer)
|
|
991
|
+
try {
|
|
992
|
+
Y.applyUpdate(resource.yjs.doc, binary)
|
|
993
|
+
} finally {
|
|
994
|
+
resource.yjs.text.unobserve(observer)
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
resource.val = resource.yjs.text.toString()
|
|
998
|
+
|
|
999
|
+
// Sync to DT if it exists
|
|
1000
|
+
if (resource.dt && delta) {
|
|
1001
|
+
var text_patches = yjs_delta_to_patches(delta, prev_text)
|
|
1002
|
+
if (text_patches.length) {
|
|
1003
|
+
// Generate a synthetic version for DT
|
|
1004
|
+
var syn_actor = `yjs-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
|
1005
|
+
var syn_seq = 0
|
|
1006
|
+
var yjs_v_before = resource.dt.doc.getLocalVersion()
|
|
1007
|
+
// Patches are sequential (each relative to state after previous)
|
|
1008
|
+
// so no offset adjustment needed
|
|
1009
|
+
var dt_bytes = []
|
|
1010
|
+
var dt_ps = resource.version
|
|
1011
|
+
for (var tp of text_patches) {
|
|
1012
|
+
var tp_range = tp.range.match(/-?\d+/g).map(Number)
|
|
1013
|
+
var tp_del = tp_range[1] - tp_range[0]
|
|
1014
|
+
var syn_v = `${syn_actor}-${syn_seq}`
|
|
1015
|
+
if (tp_del) {
|
|
1016
|
+
dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], tp_del, null))
|
|
1017
|
+
dt_ps = [`${syn_actor}-${syn_seq + tp_del - 1}`]
|
|
1018
|
+
syn_seq += tp_del
|
|
1019
|
+
syn_v = `${syn_actor}-${syn_seq}`
|
|
1020
|
+
}
|
|
1021
|
+
if (tp.content.length) {
|
|
1022
|
+
dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], 0, tp.content))
|
|
1023
|
+
var cp_len = [...tp.content].length
|
|
1024
|
+
dt_ps = [`${syn_actor}-${syn_seq + cp_len - 1}`]
|
|
1025
|
+
syn_seq += cp_len
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
for (var b of dt_bytes) resource.dt.doc.mergeBytes(b)
|
|
1029
|
+
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1030
|
+
|
|
1031
|
+
// Update known_versions for the synthetic versions
|
|
1032
|
+
if (!resource.dt.known_versions[syn_actor]) resource.dt.known_versions[syn_actor] = new RangeSet()
|
|
1033
|
+
resource.dt.known_versions[syn_actor].add_range(0, syn_seq - 1)
|
|
1034
|
+
|
|
1035
|
+
await resource.dt.save_delta(resource.dt.doc.getPatchSince(yjs_v_before))
|
|
1036
|
+
|
|
1037
|
+
// Broadcast to DT subscribers
|
|
1038
|
+
// TODO: broadcast to simpleton and dt clients
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
// Broadcast to yjs-text subscribers (skip sender)
|
|
1043
|
+
if (resource.yjs_clients) {
|
|
1044
|
+
for (var client of resource.yjs_clients) {
|
|
1045
|
+
if (!peer || client.peer !== peer) {
|
|
1046
|
+
client.subscribe({
|
|
1047
|
+
version: resource.version,
|
|
1048
|
+
patches: patches
|
|
1049
|
+
})
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
// Persist Yjs delta
|
|
1055
|
+
if (resource.yjs.save_delta) await resource.yjs.save_delta(binary)
|
|
1056
|
+
|
|
1057
|
+
// Sanity check
|
|
1058
|
+
if (braid_text.debug_sync_checks && resource.dt) {
|
|
1059
|
+
var dt_text = resource.dt.doc.get()
|
|
1060
|
+
var yjs_text = resource.yjs.text.toString()
|
|
1061
|
+
if (dt_text !== yjs_text) {
|
|
1062
|
+
console.error(`SYNC MISMATCH key=${resource.key}: DT text !== Y.Doc text`)
|
|
1063
|
+
console.error(` DT: ${dt_text.slice(0, 100)}... (${dt_text.length})`)
|
|
1064
|
+
console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
return { change_count: patches.length }
|
|
1069
|
+
}
|
|
1070
|
+
|
|
834
1071
|
if (options.transfer_encoding === 'dt') {
|
|
835
|
-
|
|
1072
|
+
await ensure_dt_exists(resource)
|
|
1073
|
+
var start_i = 1 + resource.dt.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
|
|
836
1074
|
|
|
837
|
-
resource.doc.mergeBytes(body)
|
|
1075
|
+
resource.dt.doc.mergeBytes(body)
|
|
838
1076
|
|
|
839
|
-
var end_i = resource.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
|
|
1077
|
+
var end_i = resource.dt.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
|
|
840
1078
|
for (var i = start_i; i <= end_i; i++) {
|
|
841
|
-
let v = resource.doc.localToRemoteVersion([i])[0]
|
|
842
|
-
if (!resource.
|
|
843
|
-
resource.
|
|
1079
|
+
let v = resource.dt.doc.localToRemoteVersion([i])[0]
|
|
1080
|
+
if (!resource.dt.known_versions[v[0]]) resource.dt.known_versions[v[0]] = new braid_text.RangeSet()
|
|
1081
|
+
resource.dt.known_versions[v[0]].add_range(v[1], v[1])
|
|
844
1082
|
}
|
|
845
|
-
resource.val = resource.doc.get()
|
|
846
|
-
resource.version = resource.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1083
|
+
resource.val = resource.dt.doc.get()
|
|
1084
|
+
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
847
1085
|
|
|
848
|
-
await resource.
|
|
1086
|
+
await resource.dt.save_delta(body)
|
|
849
1087
|
|
|
850
1088
|
// Notify non-simpleton clients with the dt-encoded update
|
|
851
1089
|
var dt_update = { body, encoding: 'dt' }
|
|
852
|
-
await Promise.all([...resource.clients].
|
|
1090
|
+
await Promise.all([...resource.dt.clients].
|
|
853
1091
|
filter(client => !peer || client.peer !== peer).
|
|
854
1092
|
map(client => client.my_subscribe(dt_update)))
|
|
855
1093
|
|
|
856
1094
|
return { change_count: end_i - start_i + 1 }
|
|
857
1095
|
}
|
|
858
1096
|
|
|
1097
|
+
// Text/DT patches require DT
|
|
1098
|
+
await ensure_dt_exists(resource)
|
|
1099
|
+
|
|
859
1100
|
if (version && !version.length) {
|
|
860
1101
|
console.log(`warning: ignoring put with empty version`)
|
|
861
1102
|
return { change_count: 0 }
|
|
@@ -871,15 +1112,15 @@ function create_braid_text() {
|
|
|
871
1112
|
// make sure we have all these parents
|
|
872
1113
|
for (let p of parents) {
|
|
873
1114
|
let P = decode_version(p)
|
|
874
|
-
if (!resource.
|
|
1115
|
+
if (!resource.dt.known_versions[P[0]]?.has(P[1]))
|
|
875
1116
|
throw new Error(`missing parent version: ${p}`)
|
|
876
1117
|
}
|
|
877
1118
|
}
|
|
878
1119
|
|
|
879
1120
|
if (!parents) parents = resource.version
|
|
880
1121
|
|
|
881
|
-
let max_pos = resource.
|
|
882
|
-
(v_eq(resource.version, parents) ? resource.doc.len() : dt_len(resource.doc, parents))
|
|
1122
|
+
let max_pos = resource.dt.length_at_version.get('' + parents) ??
|
|
1123
|
+
(v_eq(resource.version, parents) ? resource.dt.doc.len() : dt_len(resource.dt.doc, parents))
|
|
883
1124
|
|
|
884
1125
|
if (body != null) {
|
|
885
1126
|
patches = [{
|
|
@@ -912,7 +1153,7 @@ function create_braid_text() {
|
|
|
912
1153
|
var low_seq = v[1] + 1 - change_count
|
|
913
1154
|
|
|
914
1155
|
// make sure we haven't seen this already
|
|
915
|
-
var intersects_range = resource.
|
|
1156
|
+
var intersects_range = resource.dt.known_versions[v[0]]?.has(low_seq, v[1])
|
|
916
1157
|
if (intersects_range) {
|
|
917
1158
|
// if low_seq is below the range min,
|
|
918
1159
|
// then the intersection has gaps,
|
|
@@ -936,8 +1177,8 @@ function create_braid_text() {
|
|
|
936
1177
|
change_count = new_count
|
|
937
1178
|
low_seq = v[1] + 1 - change_count
|
|
938
1179
|
parents = [`${v[0]}-${low_seq - 1}`]
|
|
939
|
-
max_pos = resource.
|
|
940
|
-
(v_eq(resource.version, parents) ? resource.doc.len() : dt_len(resource.doc, parents))
|
|
1180
|
+
max_pos = resource.dt.length_at_version.get('' + parents) ??
|
|
1181
|
+
(v_eq(resource.version, parents) ? resource.dt.doc.len() : dt_len(resource.dt.doc, parents))
|
|
941
1182
|
patches = new_patches
|
|
942
1183
|
}
|
|
943
1184
|
|
|
@@ -951,12 +1192,12 @@ function create_braid_text() {
|
|
|
951
1192
|
must_be_at_least = p.range[1]
|
|
952
1193
|
}
|
|
953
1194
|
|
|
954
|
-
resource.
|
|
1195
|
+
resource.dt.length_at_version.put(`${v[0]}-${v[1]}`, patches.reduce((a, b) =>
|
|
955
1196
|
a + (b.content_codepoints?.length ?? 0) - (b.range[1] - b.range[0]),
|
|
956
1197
|
max_pos))
|
|
957
1198
|
|
|
958
|
-
if (!resource.
|
|
959
|
-
resource.
|
|
1199
|
+
if (!resource.dt.known_versions[v[0]]) resource.dt.known_versions[v[0]] = new RangeSet()
|
|
1200
|
+
resource.dt.known_versions[v[0]].add_range(low_seq, v[1])
|
|
960
1201
|
|
|
961
1202
|
// get the version of the first character-wise edit
|
|
962
1203
|
v = `${v[0]}-${low_seq}`
|
|
@@ -964,7 +1205,7 @@ function create_braid_text() {
|
|
|
964
1205
|
let ps = parents
|
|
965
1206
|
|
|
966
1207
|
let version_before = resource.version
|
|
967
|
-
let v_before = resource.doc.getLocalVersion()
|
|
1208
|
+
let v_before = resource.dt.doc.getLocalVersion()
|
|
968
1209
|
|
|
969
1210
|
let bytes = []
|
|
970
1211
|
|
|
@@ -989,20 +1230,89 @@ function create_braid_text() {
|
|
|
989
1230
|
}
|
|
990
1231
|
}
|
|
991
1232
|
|
|
992
|
-
for (let b of bytes) resource.doc.mergeBytes(b)
|
|
993
|
-
resource.val = resource.doc.get()
|
|
994
|
-
resource.version = resource.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1233
|
+
for (let b of bytes) resource.dt.doc.mergeBytes(b)
|
|
1234
|
+
resource.val = resource.dt.doc.get()
|
|
1235
|
+
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1236
|
+
|
|
1237
|
+
// Get transformed patches (resolved after DT merge)
|
|
1238
|
+
// xf_patches: absolute positions (for simpleton clients)
|
|
1239
|
+
// xf_patches_relative: sequential positions (for Yjs sync)
|
|
1240
|
+
var xf_patches_relative = []
|
|
1241
|
+
for (let xf of resource.dt.doc.xfSince(v_before)) {
|
|
1242
|
+
xf_patches_relative.push(
|
|
1243
|
+
xf.kind == "Ins"
|
|
1244
|
+
? { range: [xf.start, xf.start], content: xf.content }
|
|
1245
|
+
: { range: [xf.start, xf.end], content: "" }
|
|
1246
|
+
)
|
|
1247
|
+
}
|
|
1248
|
+
var xf_patches = relative_to_absolute_patches(
|
|
1249
|
+
xf_patches_relative.map(p => ({
|
|
1250
|
+
unit: 'text',
|
|
1251
|
+
range: `[${p.range[0]}:${p.range[1]}]`,
|
|
1252
|
+
content: p.content
|
|
1253
|
+
}))
|
|
1254
|
+
)
|
|
1255
|
+
|
|
1256
|
+
// Sync to Yjs if it exists, using relative (sequential) patches
|
|
1257
|
+
if (resource.yjs) {
|
|
1258
|
+
var captured_yjs_update = null
|
|
1259
|
+
var yjs_update_handler = (update, origin) => {
|
|
1260
|
+
if (origin === 'braid_text_dt_sync') captured_yjs_update = update
|
|
1261
|
+
}
|
|
1262
|
+
resource.yjs.doc.on('update', yjs_update_handler)
|
|
1263
|
+
resource.yjs.doc.transact(() => {
|
|
1264
|
+
// xf_patches_relative are sequential codepoint positions; Yjs uses UTF-16
|
|
1265
|
+
for (let p of xf_patches_relative) {
|
|
1266
|
+
var current_text = resource.yjs.text.toString()
|
|
1267
|
+
var cp_to_utf16 = codepoint_to_utf16_pos(current_text)
|
|
1268
|
+
var utf16_start = cp_to_utf16(p.range[0])
|
|
1269
|
+
var cp_del = p.range[1] - p.range[0]
|
|
1270
|
+
if (cp_del) {
|
|
1271
|
+
var utf16_end = cp_to_utf16(p.range[1])
|
|
1272
|
+
resource.yjs.text.delete(utf16_start, utf16_end - utf16_start)
|
|
1273
|
+
}
|
|
1274
|
+
if (p.content?.length) resource.yjs.text.insert(utf16_start, p.content)
|
|
1275
|
+
}
|
|
1276
|
+
}, 'braid_text_dt_sync')
|
|
1277
|
+
resource.yjs.doc.off('update', yjs_update_handler)
|
|
1278
|
+
|
|
1279
|
+
// Broadcast to yjs-text subscribers
|
|
1280
|
+
if (resource.yjs_clients && captured_yjs_update) {
|
|
1281
|
+
var yjs_patches = braid_text.from_yjs_binary(captured_yjs_update)
|
|
1282
|
+
for (var client of resource.yjs_clients) {
|
|
1283
|
+
if (!peer || client.peer !== peer) {
|
|
1284
|
+
client.subscribe({
|
|
1285
|
+
version: resource.version,
|
|
1286
|
+
patches: yjs_patches
|
|
1287
|
+
})
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
// Persist Yjs delta
|
|
1293
|
+
if (captured_yjs_update) await resource.yjs.save_delta(captured_yjs_update)
|
|
1294
|
+
|
|
1295
|
+
// Sanity check
|
|
1296
|
+
if (braid_text.debug_sync_checks) {
|
|
1297
|
+
var yjs_text = resource.yjs.text.toString()
|
|
1298
|
+
if (resource.val !== yjs_text) {
|
|
1299
|
+
console.error(`SYNC MISMATCH key=${resource.key}: DT→Yjs sync failed`)
|
|
1300
|
+
console.error(` val: ${resource.val.slice(0, 100)}... (${resource.val.length})`)
|
|
1301
|
+
console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
995
1305
|
|
|
996
1306
|
// Transform stored cursor positions through the applied patches
|
|
997
|
-
if (resource.
|
|
1307
|
+
if (resource.cursors) resource.cursors.transform(patches)
|
|
998
1308
|
|
|
999
1309
|
var post_commit_updates = []
|
|
1000
1310
|
|
|
1001
1311
|
if (options.merge_type != "dt") {
|
|
1002
|
-
let patches =
|
|
1312
|
+
let patches = xf_patches
|
|
1003
1313
|
if (braid_text.verbose) console.log(JSON.stringify({ patches }))
|
|
1004
1314
|
|
|
1005
|
-
for (let client of resource.simpleton_clients) {
|
|
1315
|
+
for (let client of resource.dt.simpleton_clients) {
|
|
1006
1316
|
if (peer && client.peer === peer) {
|
|
1007
1317
|
client.my_last_seen_version = [version]
|
|
1008
1318
|
}
|
|
@@ -1011,7 +1321,7 @@ function create_braid_text() {
|
|
|
1011
1321
|
if (client.my_timeout) clearTimeout(client.my_timeout)
|
|
1012
1322
|
client.my_timeout = setTimeout(() => {
|
|
1013
1323
|
// if the doc has been freed, exit early
|
|
1014
|
-
if (resource.doc.__wbg_ptr === 0) return
|
|
1324
|
+
if (resource.dt.doc.__wbg_ptr === 0) return
|
|
1015
1325
|
|
|
1016
1326
|
let x = {
|
|
1017
1327
|
version: resource.version,
|
|
@@ -1019,7 +1329,7 @@ function create_braid_text() {
|
|
|
1019
1329
|
}
|
|
1020
1330
|
if (braid_text.verbose) console.log("rebasing after timeout.. ")
|
|
1021
1331
|
if (braid_text.verbose) console.log(" client.my_unused_version_count = " + client.my_unused_version_count)
|
|
1022
|
-
x.patches = get_xf_patches(resource.doc, OpLog_remote_to_local(resource.doc, client.my_last_seen_version))
|
|
1332
|
+
x.patches = get_xf_patches(resource.dt.doc, OpLog_remote_to_local(resource.dt.doc, client.my_last_seen_version))
|
|
1023
1333
|
|
|
1024
1334
|
if (braid_text.verbose) console.log(`sending from rebase: ${JSON.stringify(x)}`)
|
|
1025
1335
|
client.my_subscribe(x)
|
|
@@ -1063,14 +1373,14 @@ function create_braid_text() {
|
|
|
1063
1373
|
post_commit_updates.push([client, x])
|
|
1064
1374
|
}
|
|
1065
1375
|
} else {
|
|
1066
|
-
if (resource.simpleton_clients.size) {
|
|
1376
|
+
if (resource.dt.simpleton_clients.size) {
|
|
1067
1377
|
let x = {
|
|
1068
1378
|
version: resource.version,
|
|
1069
1379
|
parents: version_before,
|
|
1070
|
-
patches:
|
|
1380
|
+
patches: xf_patches
|
|
1071
1381
|
}
|
|
1072
1382
|
if (braid_text.verbose) console.log(`sending: ${JSON.stringify(x)}`)
|
|
1073
|
-
for (let client of resource.simpleton_clients) {
|
|
1383
|
+
for (let client of resource.dt.simpleton_clients) {
|
|
1074
1384
|
if (client.my_timeout) continue
|
|
1075
1385
|
post_commit_updates.push([client, x])
|
|
1076
1386
|
}
|
|
@@ -1086,12 +1396,12 @@ function create_braid_text() {
|
|
|
1086
1396
|
content: p.content
|
|
1087
1397
|
})),
|
|
1088
1398
|
}
|
|
1089
|
-
for (let client of resource.clients) {
|
|
1399
|
+
for (let client of resource.dt.clients) {
|
|
1090
1400
|
if (!peer || client.peer !== peer)
|
|
1091
1401
|
post_commit_updates.push([client, x])
|
|
1092
1402
|
}
|
|
1093
1403
|
|
|
1094
|
-
await resource.
|
|
1404
|
+
await resource.dt.save_delta(resource.dt.doc.getPatchSince(v_before))
|
|
1095
1405
|
|
|
1096
1406
|
await Promise.all(post_commit_updates.map(([client, x]) => client.my_subscribe(x)))
|
|
1097
1407
|
|
|
@@ -1104,7 +1414,7 @@ function create_braid_text() {
|
|
|
1104
1414
|
if (braid_text.db_folder) {
|
|
1105
1415
|
await db_folder_init()
|
|
1106
1416
|
var pages = new Set()
|
|
1107
|
-
for (let x of await require('fs').promises.readdir(braid_text.db_folder)) if (
|
|
1417
|
+
for (let x of await require('fs').promises.readdir(braid_text.db_folder)) if (/\.(dt|yjs)\.\d+$/.test(x)) pages.add(decode_filename(x.replace(/\.(dt|yjs)\.\d+$/, '')))
|
|
1108
1418
|
return [...pages.keys()]
|
|
1109
1419
|
} else return Object.keys(braid_text.cache)
|
|
1110
1420
|
} catch (e) { return [] }
|
|
@@ -1121,39 +1431,93 @@ function create_braid_text() {
|
|
|
1121
1431
|
let cache = braid_text.cache
|
|
1122
1432
|
if (!cache[key]) cache[key] = new Promise(async done => {
|
|
1123
1433
|
let resource = {key}
|
|
1124
|
-
resource.
|
|
1125
|
-
resource.
|
|
1126
|
-
|
|
1127
|
-
resource.
|
|
1434
|
+
resource.dt = null
|
|
1435
|
+
resource.yjs = null
|
|
1436
|
+
resource.val = ""
|
|
1437
|
+
resource.version = []
|
|
1128
1438
|
resource.meta = {}
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1439
|
+
resource.save_meta = () => {}
|
|
1440
|
+
resource.cursors = null
|
|
1441
|
+
|
|
1442
|
+
// Load DT data from disk if it exists
|
|
1443
|
+
var has_dt_files = braid_text.db_folder
|
|
1444
|
+
&& (await get_files_for_key(key, 'dt')).length > 0
|
|
1445
|
+
if (has_dt_files) {
|
|
1446
|
+
resource.dt = {
|
|
1447
|
+
doc: new Doc("server"),
|
|
1448
|
+
known_versions: {},
|
|
1449
|
+
save_delta: () => {},
|
|
1450
|
+
length_at_version: createSimpleCache(braid_text.length_cache_size),
|
|
1451
|
+
clients: new Set(),
|
|
1452
|
+
simpleton_clients: new Set(),
|
|
1453
|
+
}
|
|
1454
|
+
let { change, change_meta } = await file_sync(key,
|
|
1455
|
+
(bytes) => resource.dt.doc.mergeBytes(bytes),
|
|
1456
|
+
() => resource.dt.doc.toBytes(),
|
|
1134
1457
|
(meta) => resource.meta = meta,
|
|
1135
|
-
() => resource.meta
|
|
1136
|
-
|
|
1458
|
+
() => resource.meta,
|
|
1459
|
+
'dt')
|
|
1137
1460
|
|
|
1138
|
-
|
|
1139
|
-
|
|
1461
|
+
resource.dt.save_delta = change
|
|
1462
|
+
resource.save_meta = change_meta
|
|
1140
1463
|
|
|
1141
|
-
|
|
1464
|
+
dt_get_actor_seq_runs([...resource.dt.doc.toBytes()], (actor, base, len) => {
|
|
1465
|
+
if (!resource.dt.known_versions[actor]) resource.dt.known_versions[actor] = new RangeSet()
|
|
1466
|
+
resource.dt.known_versions[actor].add_range(base, base + len - 1)
|
|
1467
|
+
})
|
|
1142
1468
|
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1469
|
+
resource.val = resource.dt.doc.get()
|
|
1470
|
+
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1471
|
+
} else if (braid_text.db_folder) {
|
|
1472
|
+
// No DT files — still load meta
|
|
1473
|
+
let { change, change_meta } = await file_sync(key,
|
|
1474
|
+
() => {},
|
|
1475
|
+
() => Buffer.alloc(0),
|
|
1476
|
+
(meta) => resource.meta = meta,
|
|
1477
|
+
() => resource.meta,
|
|
1478
|
+
'dt')
|
|
1479
|
+
resource.save_meta = change_meta
|
|
1480
|
+
}
|
|
1147
1481
|
|
|
1148
|
-
|
|
1149
|
-
|
|
1482
|
+
// Load Yjs data from disk if it exists
|
|
1483
|
+
var has_yjs_files = braid_text.db_folder && Y
|
|
1484
|
+
&& (await get_files_for_key(key, 'yjs')).length > 0
|
|
1485
|
+
if (has_yjs_files) {
|
|
1486
|
+
resource.yjs = {
|
|
1487
|
+
doc: new Y.Doc(),
|
|
1488
|
+
text: null,
|
|
1489
|
+
save_delta: () => {},
|
|
1490
|
+
}
|
|
1491
|
+
resource.yjs.text = resource.yjs.doc.getText('text')
|
|
1492
|
+
let { change } = await file_sync(key,
|
|
1493
|
+
(bytes) => Y.applyUpdate(resource.yjs.doc, bytes),
|
|
1494
|
+
() => Y.encodeStateAsUpdate(resource.yjs.doc),
|
|
1495
|
+
() => {},
|
|
1496
|
+
() => ({}),
|
|
1497
|
+
'yjs')
|
|
1498
|
+
|
|
1499
|
+
resource.yjs.save_delta = change
|
|
1500
|
+
|
|
1501
|
+
var yjs_text = resource.yjs.text.toString()
|
|
1502
|
+
if (resource.dt) {
|
|
1503
|
+
// Both exist — sanity check they match
|
|
1504
|
+
if (resource.val !== yjs_text) {
|
|
1505
|
+
console.error(`INIT MISMATCH key=${key}: DT text !== Yjs text`)
|
|
1506
|
+
console.error(` DT: ${resource.val.slice(0, 100)}... (${resource.val.length})`)
|
|
1507
|
+
console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
|
|
1508
|
+
}
|
|
1509
|
+
} else {
|
|
1510
|
+
// Yjs only — use its text as resource.val
|
|
1511
|
+
resource.val = yjs_text
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1150
1514
|
|
|
1151
|
-
resource.
|
|
1515
|
+
// length_at_version cache is created as part of resource.dt when DT is initialized
|
|
1152
1516
|
|
|
1153
1517
|
// Add delete method to resource
|
|
1154
1518
|
resource.delete = async () => {
|
|
1155
|
-
|
|
1156
|
-
if (resource.
|
|
1519
|
+
if (resource.dt) resource.dt.doc.free()
|
|
1520
|
+
if (resource.yjs) resource.yjs.doc.destroy()
|
|
1157
1521
|
|
|
1158
1522
|
// Remove from in-memory cache
|
|
1159
1523
|
delete braid_text.cache[key]
|
|
@@ -1172,7 +1536,7 @@ function create_braid_text() {
|
|
|
1172
1536
|
// Remove meta file if it exists
|
|
1173
1537
|
try {
|
|
1174
1538
|
var encoded = encode_filename(key)
|
|
1175
|
-
await fs.promises.unlink(`${braid_text.db_folder}
|
|
1539
|
+
await fs.promises.unlink(`${braid_text.db_folder}/meta/${encoded}`)
|
|
1176
1540
|
} catch (e) {
|
|
1177
1541
|
// Meta file might not exist, that's ok
|
|
1178
1542
|
}
|
|
@@ -1191,23 +1555,133 @@ function create_braid_text() {
|
|
|
1191
1555
|
return await cache[key]
|
|
1192
1556
|
}
|
|
1193
1557
|
|
|
1558
|
+
async function ensure_dt_exists(resource) {
|
|
1559
|
+
if (resource.dt) return
|
|
1560
|
+
resource.dt = {
|
|
1561
|
+
doc: new Doc("server"),
|
|
1562
|
+
known_versions: {},
|
|
1563
|
+
save_delta: () => {},
|
|
1564
|
+
length_at_version: createSimpleCache(braid_text.length_cache_size),
|
|
1565
|
+
clients: new Set(),
|
|
1566
|
+
simpleton_clients: new Set(),
|
|
1567
|
+
}
|
|
1568
|
+
if (resource.val) {
|
|
1569
|
+
// Seed DT with current text as a root insert
|
|
1570
|
+
var bytes = dt_create_bytes(
|
|
1571
|
+
`999999999-0`, [], 0, 0, resource.val)
|
|
1572
|
+
resource.dt.doc.mergeBytes(bytes)
|
|
1573
|
+
}
|
|
1574
|
+
resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
|
|
1575
|
+
|
|
1576
|
+
dt_get_actor_seq_runs([...resource.dt.doc.toBytes()], (actor, base, len) => {
|
|
1577
|
+
if (!resource.dt.known_versions[actor]) resource.dt.known_versions[actor] = new RangeSet()
|
|
1578
|
+
resource.dt.known_versions[actor].add_range(base, base + len - 1)
|
|
1579
|
+
})
|
|
1580
|
+
|
|
1581
|
+
// Set up DT persistence if db_folder exists
|
|
1582
|
+
if (braid_text.db_folder) {
|
|
1583
|
+
let { change } = await file_sync(resource.key,
|
|
1584
|
+
(bytes) => resource.dt.doc.mergeBytes(bytes),
|
|
1585
|
+
() => resource.dt.doc.toBytes(),
|
|
1586
|
+
() => {},
|
|
1587
|
+
() => ({}),
|
|
1588
|
+
'dt')
|
|
1589
|
+
resource.dt.save_delta = change
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
async function ensure_yjs_exists(resource) {
|
|
1594
|
+
if (resource.yjs) return
|
|
1595
|
+
require_yjs()
|
|
1596
|
+
resource.yjs = {
|
|
1597
|
+
doc: new Y.Doc(),
|
|
1598
|
+
text: null,
|
|
1599
|
+
save_delta: () => {},
|
|
1600
|
+
}
|
|
1601
|
+
resource.yjs.text = resource.yjs.doc.getText('text')
|
|
1602
|
+
if (resource.val) {
|
|
1603
|
+
resource.yjs.text.insert(0, resource.val)
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
// Set up Yjs persistence if db_folder exists
|
|
1607
|
+
if (braid_text.db_folder) {
|
|
1608
|
+
let { change } = await file_sync(resource.key,
|
|
1609
|
+
(bytes) => Y.applyUpdate(resource.yjs.doc, bytes),
|
|
1610
|
+
() => Y.encodeStateAsUpdate(resource.yjs.doc),
|
|
1611
|
+
() => {},
|
|
1612
|
+
() => ({}),
|
|
1613
|
+
'yjs')
|
|
1614
|
+
resource.yjs.save_delta = change
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1194
1618
|
async function db_folder_init() {
|
|
1195
1619
|
if (braid_text.verbose) console.log('__!')
|
|
1196
1620
|
if (!db_folder_init.p) db_folder_init.p = (async () => {
|
|
1197
1621
|
await fs.promises.mkdir(braid_text.db_folder, { recursive: true });
|
|
1198
|
-
await fs.promises.mkdir(`${braid_text.db_folder}/.meta`, { recursive: true })
|
|
1199
|
-
await fs.promises.mkdir(`${braid_text.db_folder}/.temp`, { recursive: true })
|
|
1200
|
-
await fs.promises.mkdir(`${braid_text.db_folder}/.wal-intent`, { recursive: true })
|
|
1201
1622
|
|
|
1202
|
-
//
|
|
1203
|
-
|
|
1623
|
+
// Migrate from old dot-prefixed directory names to new names
|
|
1624
|
+
// This is idempotent: safe to re-run if interrupted mid-migration
|
|
1625
|
+
async function migrate_dir(old_name, new_name) {
|
|
1626
|
+
var old_path = `${braid_text.db_folder}/${old_name}`
|
|
1627
|
+
var new_path = `${braid_text.db_folder}/${new_name}`
|
|
1628
|
+
try {
|
|
1629
|
+
await fs.promises.stat(old_path)
|
|
1630
|
+
// Old dir exists — ensure new dir exists and move contents
|
|
1631
|
+
await fs.promises.mkdir(new_path, { recursive: true })
|
|
1632
|
+
var entries = await fs.promises.readdir(old_path)
|
|
1633
|
+
for (var entry of entries) {
|
|
1634
|
+
var src = `${old_path}/${entry}`
|
|
1635
|
+
var dst = `${new_path}/${entry}`
|
|
1636
|
+
try { await fs.promises.stat(dst) } catch (e) {
|
|
1637
|
+
// dst doesn't exist, move src there
|
|
1638
|
+
await fs.promises.rename(src, dst)
|
|
1639
|
+
}
|
|
1640
|
+
// If dst already exists (crash recovery), just remove src
|
|
1641
|
+
try { await fs.promises.unlink(src) } catch (e) {}
|
|
1642
|
+
}
|
|
1643
|
+
// Remove old dir once empty
|
|
1644
|
+
try { await fs.promises.rmdir(old_path) } catch (e) {}
|
|
1645
|
+
} catch (e) {
|
|
1646
|
+
// Old dir doesn't exist — no migration needed
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
await migrate_dir('.meta', 'meta')
|
|
1650
|
+
await migrate_dir('.temp', 'temp')
|
|
1651
|
+
await migrate_dir('.wal-intent', 'wal-intent')
|
|
1652
|
+
|
|
1653
|
+
// Migrate data files from [key].N to [key].dt.N
|
|
1654
|
+
// Idempotent: only renames files that match old pattern but not new
|
|
1655
|
+
var all_files = await fs.promises.readdir(braid_text.db_folder)
|
|
1656
|
+
for (var f of all_files) {
|
|
1657
|
+
// Match old format: ends with .N (digits only) but not .dt.N or .yjs.N
|
|
1658
|
+
if (/\.\d+$/.test(f) && !/\.(dt|yjs)\.\d+$/.test(f)) {
|
|
1659
|
+
var new_name = f.replace(/\.(\d+)$/, '.dt.$1')
|
|
1660
|
+
var old_path = `${braid_text.db_folder}/${f}`
|
|
1661
|
+
var new_path = `${braid_text.db_folder}/${new_name}`
|
|
1662
|
+
try { await fs.promises.stat(new_path) } catch (e) {
|
|
1663
|
+
// New file doesn't exist, rename
|
|
1664
|
+
await fs.promises.rename(old_path, new_path)
|
|
1665
|
+
continue
|
|
1666
|
+
}
|
|
1667
|
+
// If new file already exists (crash recovery), remove old
|
|
1668
|
+
try { await fs.promises.unlink(old_path) } catch (e) {}
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
await fs.promises.mkdir(`${braid_text.db_folder}/meta`, { recursive: true })
|
|
1673
|
+
await fs.promises.mkdir(`${braid_text.db_folder}/temp`, { recursive: true })
|
|
1674
|
+
await fs.promises.mkdir(`${braid_text.db_folder}/wal-intent`, { recursive: true })
|
|
1675
|
+
|
|
1676
|
+
// Clean out temp directory on startup
|
|
1677
|
+
var temp_files = await fs.promises.readdir(`${braid_text.db_folder}/temp`)
|
|
1204
1678
|
for (var f of temp_files)
|
|
1205
|
-
await fs.promises.unlink(`${braid_text.db_folder}
|
|
1679
|
+
await fs.promises.unlink(`${braid_text.db_folder}/temp/${f}`)
|
|
1206
1680
|
|
|
1207
|
-
// Replay any pending
|
|
1208
|
-
var intent_files = await fs.promises.readdir(`${braid_text.db_folder}
|
|
1681
|
+
// Replay any pending wal-intent files
|
|
1682
|
+
var intent_files = await fs.promises.readdir(`${braid_text.db_folder}/wal-intent`)
|
|
1209
1683
|
for (var intent_name of intent_files) {
|
|
1210
|
-
var intent_path = `${braid_text.db_folder}
|
|
1684
|
+
var intent_path = `${braid_text.db_folder}/wal-intent/${intent_name}`
|
|
1211
1685
|
var target_path = `${braid_text.db_folder}/${intent_name}`
|
|
1212
1686
|
|
|
1213
1687
|
var intent_data = await fs.promises.readFile(intent_path)
|
|
@@ -1227,25 +1701,27 @@ function create_braid_text() {
|
|
|
1227
1701
|
|
|
1228
1702
|
// Populate key_to_filename mapping from existing files
|
|
1229
1703
|
var files = (await fs.promises.readdir(braid_text.db_folder))
|
|
1230
|
-
.filter(x =>
|
|
1704
|
+
.filter(x => /\.dt\.\d+$/.test(x))
|
|
1231
1705
|
init_filename_mapping(files)
|
|
1232
1706
|
})()
|
|
1233
1707
|
await db_folder_init.p
|
|
1234
1708
|
}
|
|
1235
1709
|
|
|
1236
|
-
async function get_files_for_key(key) {
|
|
1710
|
+
async function get_files_for_key(key, type) {
|
|
1237
1711
|
await db_folder_init()
|
|
1238
1712
|
try {
|
|
1239
|
-
|
|
1713
|
+
var suffix = type ? `\\.${type}\\.\\d+$` : `\\.(dt|yjs)\\.\\d+$`
|
|
1714
|
+
let re = new RegExp("^" + encode_filename(key).replace(/[^a-zA-Z0-9]/g, "\\$&") + suffix)
|
|
1240
1715
|
return (await fs.promises.readdir(braid_text.db_folder))
|
|
1241
1716
|
.filter((a) => re.test(a))
|
|
1242
1717
|
.map((a) => `${braid_text.db_folder}/${a}`)
|
|
1243
1718
|
} catch (e) { return [] }
|
|
1244
1719
|
}
|
|
1245
1720
|
|
|
1246
|
-
async function file_sync(key, process_delta, get_init, set_meta, get_meta) {
|
|
1721
|
+
async function file_sync(key, process_delta, get_init, set_meta, get_meta, file_type) {
|
|
1247
1722
|
await db_folder_init()
|
|
1248
1723
|
let encoded = encode_filename(key)
|
|
1724
|
+
file_type = file_type || 'dt'
|
|
1249
1725
|
|
|
1250
1726
|
if (encoded.length > max_encoded_key_size) throw new Error(`invalid key: too long (max ${max_encoded_key_size})`)
|
|
1251
1727
|
|
|
@@ -1254,7 +1730,7 @@ function create_braid_text() {
|
|
|
1254
1730
|
let threshold = 0
|
|
1255
1731
|
|
|
1256
1732
|
// Read existing files and sort by numbers.
|
|
1257
|
-
const files = (await get_files_for_key(key))
|
|
1733
|
+
const files = (await get_files_for_key(key, file_type))
|
|
1258
1734
|
.filter(x => x.match(/\.\d+$/))
|
|
1259
1735
|
.sort((a, b) => parseInt(a.match(/\d+$/)[0]) - parseInt(b.match(/\d+$/)[0]))
|
|
1260
1736
|
|
|
@@ -1286,7 +1762,7 @@ function create_braid_text() {
|
|
|
1286
1762
|
}
|
|
1287
1763
|
|
|
1288
1764
|
currentSize = data.length
|
|
1289
|
-
currentNumber = parseInt(filename.match(
|
|
1765
|
+
currentNumber = parseInt(filename.match(/(\d+)$/)[1])
|
|
1290
1766
|
done = true
|
|
1291
1767
|
} catch (error) {
|
|
1292
1768
|
console.error(`Error processing file: ${files[i]}`)
|
|
@@ -1294,7 +1770,7 @@ function create_braid_text() {
|
|
|
1294
1770
|
}
|
|
1295
1771
|
}
|
|
1296
1772
|
|
|
1297
|
-
var meta_filename = `${braid_text.db_folder}
|
|
1773
|
+
var meta_filename = `${braid_text.db_folder}/meta/${encoded}`
|
|
1298
1774
|
var meta_dirty = null
|
|
1299
1775
|
var meta_saving = null
|
|
1300
1776
|
var meta_file_content = '{}'
|
|
@@ -1307,7 +1783,7 @@ function create_braid_text() {
|
|
|
1307
1783
|
change: (bytes) => within_fiber('file:' + key, async () => {
|
|
1308
1784
|
if (!bytes) currentSize = threshold
|
|
1309
1785
|
else currentSize += bytes.length + 4 // we account for the extra 4 bytes for uint32
|
|
1310
|
-
const filename = `${braid_text.db_folder}/${encoded}.${currentNumber}`
|
|
1786
|
+
const filename = `${braid_text.db_folder}/${encoded}.${file_type}.${currentNumber}`
|
|
1311
1787
|
if (currentSize < threshold) {
|
|
1312
1788
|
if (braid_text.verbose) console.log(`appending to db..`)
|
|
1313
1789
|
|
|
@@ -1316,13 +1792,13 @@ function create_braid_text() {
|
|
|
1316
1792
|
let append_data = Buffer.concat([len_buf, bytes])
|
|
1317
1793
|
|
|
1318
1794
|
let basename = require('path').basename(filename)
|
|
1319
|
-
let intent_path = `${braid_text.db_folder}
|
|
1795
|
+
let intent_path = `${braid_text.db_folder}/wal-intent/${basename}`
|
|
1320
1796
|
let stat = await fs.promises.stat(filename)
|
|
1321
1797
|
let size_buf = Buffer.allocUnsafe(8)
|
|
1322
1798
|
size_buf.writeBigUInt64LE(BigInt(stat.size), 0)
|
|
1323
1799
|
|
|
1324
1800
|
await atomic_write(intent_path, Buffer.concat([size_buf, append_data]),
|
|
1325
|
-
`${braid_text.db_folder}
|
|
1801
|
+
`${braid_text.db_folder}/temp`)
|
|
1326
1802
|
await fs.promises.appendFile(filename, append_data)
|
|
1327
1803
|
await fs.promises.unlink(intent_path)
|
|
1328
1804
|
|
|
@@ -1336,9 +1812,9 @@ function create_braid_text() {
|
|
|
1336
1812
|
const buffer = Buffer.allocUnsafe(4)
|
|
1337
1813
|
buffer.writeUInt32LE(init.length, 0)
|
|
1338
1814
|
|
|
1339
|
-
const newFilename = `${braid_text.db_folder}/${encoded}.${currentNumber}`
|
|
1815
|
+
const newFilename = `${braid_text.db_folder}/${encoded}.${file_type}.${currentNumber}`
|
|
1340
1816
|
await atomic_write(newFilename, Buffer.concat([buffer, init]),
|
|
1341
|
-
`${braid_text.db_folder}
|
|
1817
|
+
`${braid_text.db_folder}/temp`)
|
|
1342
1818
|
|
|
1343
1819
|
if (braid_text.verbose) console.log("wrote to : " + newFilename)
|
|
1344
1820
|
|
|
@@ -1360,7 +1836,7 @@ function create_braid_text() {
|
|
|
1360
1836
|
while (meta_dirty) {
|
|
1361
1837
|
meta_dirty = false
|
|
1362
1838
|
await atomic_write(meta_filename, JSON.stringify(get_meta()),
|
|
1363
|
-
`${braid_text.db_folder}
|
|
1839
|
+
`${braid_text.db_folder}/temp`)
|
|
1364
1840
|
await new Promise(done => setTimeout(done,
|
|
1365
1841
|
braid_text.meta_file_save_period_ms))
|
|
1366
1842
|
}
|
|
@@ -1472,7 +1948,7 @@ function create_braid_text() {
|
|
|
1472
1948
|
|
|
1473
1949
|
function validate_old_patches(resource, base_v, parents, patches) {
|
|
1474
1950
|
// if we have seen it already, make sure it's the same as before
|
|
1475
|
-
let updates = dt_get_patches(resource.doc, parents)
|
|
1951
|
+
let updates = dt_get_patches(resource.dt.doc, parents)
|
|
1476
1952
|
|
|
1477
1953
|
let seen = {}
|
|
1478
1954
|
for (let u of updates) {
|
|
@@ -2470,8 +2946,8 @@ function create_braid_text() {
|
|
|
2470
2946
|
// Populate key_to_filename mapping from existing files on disk
|
|
2471
2947
|
function init_filename_mapping(files) {
|
|
2472
2948
|
for (var file of files) {
|
|
2473
|
-
// Extract the encoded key (strip extension like .0, .1, etc.)
|
|
2474
|
-
var encoded = file.replace(
|
|
2949
|
+
// Extract the encoded key (strip extension like .dt.0, .dt.1, etc.)
|
|
2950
|
+
var encoded = file.replace(/\.(dt|yjs)\.\d+$/, '')
|
|
2475
2951
|
var key = decode_filename(encoded)
|
|
2476
2952
|
|
|
2477
2953
|
if (!key_to_filename.has(key)) {
|
|
@@ -2524,12 +3000,237 @@ function create_braid_text() {
|
|
|
2524
3000
|
|
|
2525
3001
|
function validate_patch(x) {
|
|
2526
3002
|
if (typeof x != 'object') throw new Error(`invalid patch: not an object`)
|
|
2527
|
-
if (x.unit
|
|
3003
|
+
if (x.unit === 'yjs-text') return validate_yjs_patch(x)
|
|
3004
|
+
if (x.unit && x.unit !== 'text') throw new Error(`invalid patch unit '${x.unit}': only 'text' and 'yjs-text' supported`)
|
|
2528
3005
|
if (typeof x.range !== 'string') throw new Error(`invalid patch range: must be a string`)
|
|
2529
3006
|
if (!x.range.match(/^\s*\[\s*-?\d+\s*:\s*-?\d+\s*\]\s*$/)) throw new Error(`invalid patch range: ${x.range}`)
|
|
2530
3007
|
if (typeof x.content !== 'string') throw new Error(`invalid patch content: must be a string`)
|
|
2531
3008
|
}
|
|
2532
3009
|
|
|
3010
|
+
// yjs-text range format:
|
|
3011
|
+
// Inserts (exclusive): yjs-text (clientID-clock:clientID-clock)
|
|
3012
|
+
// Deletes (inclusive): yjs-text [clientID-clock:clientID-clock]
|
|
3013
|
+
// Null origins: (:), (:42-5), (42-5:)
|
|
3014
|
+
// Client IDs and clocks are always non-negative integers.
|
|
3015
|
+
|
|
3016
|
+
var yjs_id_pattern = '(\\d+-\\d+)'
|
|
3017
|
+
var yjs_range_re = new RegExp(
|
|
3018
|
+
'^\\s*' +
|
|
3019
|
+
'([\\(\\[])' + // open bracket: ( or [
|
|
3020
|
+
'\\s*' + yjs_id_pattern + '?' + // optional left ID
|
|
3021
|
+
'\\s*:\\s*' + // colon separator
|
|
3022
|
+
yjs_id_pattern + '?' + '\\s*' + // optional right ID
|
|
3023
|
+
'([\\)\\]])' + // close bracket: ) or ]
|
|
3024
|
+
'\\s*$'
|
|
3025
|
+
)
|
|
3026
|
+
|
|
3027
|
+
function validate_yjs_patch(x) {
|
|
3028
|
+
if (typeof x != 'object') throw new Error(`invalid yjs patch: not an object`)
|
|
3029
|
+
if (typeof x.range !== 'string') throw new Error(`invalid yjs patch range: must be a string`)
|
|
3030
|
+
if (typeof x.content !== 'string') throw new Error(`invalid yjs patch content: must be a string`)
|
|
3031
|
+
var parsed = parse_yjs_range(x.range)
|
|
3032
|
+
if (!parsed) throw new Error(`invalid yjs patch range: ${x.range}`)
|
|
3033
|
+
}
|
|
3034
|
+
|
|
3035
|
+
function parse_yjs_range(range_string) {
|
|
3036
|
+
var m = range_string.match(yjs_range_re)
|
|
3037
|
+
if (!m) return null
|
|
3038
|
+
|
|
3039
|
+
var open = m[1] // ( or [
|
|
3040
|
+
var left = m[2] // e.g. "42-5" or undefined
|
|
3041
|
+
var right = m[3] // e.g. "73-2" or undefined
|
|
3042
|
+
var close = m[4] // ) or ]
|
|
3043
|
+
|
|
3044
|
+
// Validate bracket pairing: () for exclusive, [] for inclusive
|
|
3045
|
+
var exclusive = (open === '(' && close === ')')
|
|
3046
|
+
var inclusive = (open === '[' && close === ']')
|
|
3047
|
+
if (!exclusive && !inclusive) return null
|
|
3048
|
+
|
|
3049
|
+
function parse_id(s) {
|
|
3050
|
+
if (!s) return null
|
|
3051
|
+
var dash = s.lastIndexOf('-')
|
|
3052
|
+
return { client: parseInt(s.slice(0, dash)), clock: parseInt(s.slice(dash + 1)) }
|
|
3053
|
+
}
|
|
3054
|
+
|
|
3055
|
+
var result = {
|
|
3056
|
+
inclusive: inclusive,
|
|
3057
|
+
left: parse_id(left),
|
|
3058
|
+
right: parse_id(right)
|
|
3059
|
+
}
|
|
3060
|
+
|
|
3061
|
+
// Inclusive ranges must have at least one ID (can't delete nothing)
|
|
3062
|
+
if (inclusive && !result.left && !result.right) return null
|
|
3063
|
+
|
|
3064
|
+
return result
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
// Convert a Yjs delta (array of {retain, insert, delete} ops)
|
|
3068
|
+
// to positional text patches [{unit, range, content}]
|
|
3069
|
+
// range is returned as a parsed array [start, end], not a string
|
|
3070
|
+
// Convert a Yjs delta to positional text patches with codepoint positions.
|
|
3071
|
+
// Yjs deltas use UTF-16 positions; we need to convert to codepoints for DT.
|
|
3072
|
+
// prev_text is the text BEFORE the delta was applied (in UTF-16, i.e. a JS string).
|
|
3073
|
+
function yjs_delta_to_patches(delta, prev_text) {
|
|
3074
|
+
var patches = []
|
|
3075
|
+
var utf16_pos = 0
|
|
3076
|
+
var cp_pos = 0
|
|
3077
|
+
for (var op of delta) {
|
|
3078
|
+
if (op.retain) {
|
|
3079
|
+
// Count codepoints in the retained region
|
|
3080
|
+
var retained = prev_text.slice(utf16_pos, utf16_pos + op.retain)
|
|
3081
|
+
cp_pos += [...retained].length
|
|
3082
|
+
utf16_pos += op.retain
|
|
3083
|
+
} else if (op.insert) {
|
|
3084
|
+
var cp_len = [...op.insert].length
|
|
3085
|
+
patches.push({
|
|
3086
|
+
unit: 'text',
|
|
3087
|
+
range: `[${cp_pos}:${cp_pos}]`,
|
|
3088
|
+
content: op.insert
|
|
3089
|
+
})
|
|
3090
|
+
cp_pos += cp_len
|
|
3091
|
+
} else if (op.delete) {
|
|
3092
|
+
var deleted = prev_text.slice(utf16_pos, utf16_pos + op.delete)
|
|
3093
|
+
var cp_del = [...deleted].length
|
|
3094
|
+
patches.push({
|
|
3095
|
+
unit: 'text',
|
|
3096
|
+
range: `[${cp_pos}:${cp_pos + cp_del}]`,
|
|
3097
|
+
content: ''
|
|
3098
|
+
})
|
|
3099
|
+
utf16_pos += op.delete
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
3102
|
+
return patches
|
|
3103
|
+
}
|
|
3104
|
+
|
|
3105
|
+
// Convert codepoint index to UTF-16 index in a string.
|
|
3106
|
+
// Returns a function that maps codepoint position -> UTF-16 position.
|
|
3107
|
+
function codepoint_to_utf16_pos(str) {
|
|
3108
|
+
return function(cp_pos) {
|
|
3109
|
+
var utf16 = 0
|
|
3110
|
+
var cp = 0
|
|
3111
|
+
while (cp < cp_pos && utf16 < str.length) {
|
|
3112
|
+
var code = str.charCodeAt(utf16)
|
|
3113
|
+
utf16 += (code >= 0xD800 && code <= 0xDBFF) ? 2 : 1
|
|
3114
|
+
cp++
|
|
3115
|
+
}
|
|
3116
|
+
return utf16
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
|
|
3120
|
+
braid_text.parse_yjs_range = parse_yjs_range
|
|
3121
|
+
braid_text.validate_patches = validate_patches
|
|
3122
|
+
|
|
3123
|
+
// Convert a Yjs binary update to yjs-text range patches.
|
|
3124
|
+
// Decodes the binary without needing a Y.Doc.
|
|
3125
|
+
// Returns array of {unit: 'yjs-text', range: '...', content: '...'}
|
|
3126
|
+
braid_text.from_yjs_binary = function(update) {
|
|
3127
|
+
require_yjs()
|
|
3128
|
+
var decoded = Y.decodeUpdate(
|
|
3129
|
+
update instanceof Uint8Array ? update : new Uint8Array(update))
|
|
3130
|
+
var patches = []
|
|
3131
|
+
|
|
3132
|
+
// Convert inserted structs to yjs-text patches
|
|
3133
|
+
for (var struct of decoded.structs) {
|
|
3134
|
+
if (!struct.content?.str) continue // skip non-text items
|
|
3135
|
+
var id = struct.id
|
|
3136
|
+
var origin = struct.origin
|
|
3137
|
+
var rightOrigin = struct.rightOrigin
|
|
3138
|
+
var left = origin ? `${origin.client}-${origin.clock}` : ''
|
|
3139
|
+
var right = rightOrigin ? `${rightOrigin.client}-${rightOrigin.clock}` : ''
|
|
3140
|
+
patches.push({
|
|
3141
|
+
unit: 'yjs-text',
|
|
3142
|
+
range: `(${left}:${right})`,
|
|
3143
|
+
content: struct.content.str
|
|
3144
|
+
})
|
|
3145
|
+
}
|
|
3146
|
+
|
|
3147
|
+
// Convert delete set entries to yjs-text patches
|
|
3148
|
+
for (var [clientID, deleteItems] of decoded.ds.clients) {
|
|
3149
|
+
for (var item of deleteItems) {
|
|
3150
|
+
var left = `${clientID}-${item.clock}`
|
|
3151
|
+
var right = `${clientID}-${item.clock + item.len - 1}`
|
|
3152
|
+
patches.push({
|
|
3153
|
+
unit: 'yjs-text',
|
|
3154
|
+
range: `[${left}:${right}]`,
|
|
3155
|
+
content: ''
|
|
3156
|
+
})
|
|
3157
|
+
}
|
|
3158
|
+
}
|
|
3159
|
+
|
|
3160
|
+
return patches
|
|
3161
|
+
}
|
|
3162
|
+
|
|
3163
|
+
// Convert yjs-text range patches to a Yjs binary update.
|
|
3164
|
+
// This is the inverse of from_yjs_binary.
|
|
3165
|
+
// Each insert patch needs a clientID and clock for the new item.
|
|
3166
|
+
// These are passed as patch.id = {client, clock}.
|
|
3167
|
+
braid_text.to_yjs_binary = function(patches) {
|
|
3168
|
+
require_yjs()
|
|
3169
|
+
var lib0_encoding = require('lib0/encoding')
|
|
3170
|
+
var encoder = new Y.UpdateEncoderV1()
|
|
3171
|
+
|
|
3172
|
+
// Group inserts by client
|
|
3173
|
+
var inserts_by_client = new Map()
|
|
3174
|
+
var deletes_by_client = new Map()
|
|
3175
|
+
|
|
3176
|
+
for (var p of patches) {
|
|
3177
|
+
var parsed = parse_yjs_range(p.range)
|
|
3178
|
+
if (!parsed) throw new Error(`invalid yjs-text range: ${p.range}`)
|
|
3179
|
+
|
|
3180
|
+
if (p.content.length > 0) {
|
|
3181
|
+
// Insert
|
|
3182
|
+
if (!p.id) throw new Error('insert patch requires .id = {client, clock}')
|
|
3183
|
+
var list = inserts_by_client.get(p.id.client) || []
|
|
3184
|
+
list.push({ id: p.id, origin: parsed.left, rightOrigin: parsed.right, content: p.content })
|
|
3185
|
+
inserts_by_client.set(p.id.client, list)
|
|
3186
|
+
} else {
|
|
3187
|
+
// Delete
|
|
3188
|
+
if (!parsed.left) throw new Error('delete patch requires left ID')
|
|
3189
|
+
var client = parsed.left.client
|
|
3190
|
+
var list = deletes_by_client.get(client) || []
|
|
3191
|
+
list.push({ clock: parsed.left.clock, len: parsed.right ? parsed.right.clock - parsed.left.clock + 1 : 1 })
|
|
3192
|
+
deletes_by_client.set(client, list)
|
|
3193
|
+
}
|
|
3194
|
+
}
|
|
3195
|
+
|
|
3196
|
+
// Write structs
|
|
3197
|
+
lib0_encoding.writeVarUint(encoder.restEncoder, inserts_by_client.size)
|
|
3198
|
+
for (var [client, items] of inserts_by_client) {
|
|
3199
|
+
items.sort((a, b) => a.id.clock - b.id.clock)
|
|
3200
|
+
lib0_encoding.writeVarUint(encoder.restEncoder, items.length)
|
|
3201
|
+
encoder.writeClient(client)
|
|
3202
|
+
lib0_encoding.writeVarUint(encoder.restEncoder, items[0].id.clock)
|
|
3203
|
+
for (var item of items) {
|
|
3204
|
+
var has_origin = item.origin !== null
|
|
3205
|
+
var has_right_origin = item.rightOrigin !== null
|
|
3206
|
+
// info byte: content ref (4 = string) | origin flags
|
|
3207
|
+
var info = 4 | (has_origin ? 0x80 : 0) | (has_right_origin ? 0x40 : 0)
|
|
3208
|
+
encoder.writeInfo(info)
|
|
3209
|
+
if (has_origin) encoder.writeLeftID(Y.createID(item.origin.client, item.origin.clock))
|
|
3210
|
+
if (has_right_origin) encoder.writeRightID(Y.createID(item.rightOrigin.client, item.rightOrigin.clock))
|
|
3211
|
+
if (!has_origin && !has_right_origin) {
|
|
3212
|
+
// Root insert — write parent type key
|
|
3213
|
+
encoder.writeParentInfo(true)
|
|
3214
|
+
encoder.writeString('text')
|
|
3215
|
+
}
|
|
3216
|
+
encoder.writeString(item.content)
|
|
3217
|
+
}
|
|
3218
|
+
}
|
|
3219
|
+
|
|
3220
|
+
// Write delete set
|
|
3221
|
+
lib0_encoding.writeVarUint(encoder.restEncoder, deletes_by_client.size)
|
|
3222
|
+
for (var [client, deletes] of deletes_by_client) {
|
|
3223
|
+
lib0_encoding.writeVarUint(encoder.restEncoder, client)
|
|
3224
|
+
lib0_encoding.writeVarUint(encoder.restEncoder, deletes.length)
|
|
3225
|
+
for (var d of deletes) {
|
|
3226
|
+
encoder.writeDsClock(d.clock)
|
|
3227
|
+
encoder.writeDsLen(d.len)
|
|
3228
|
+
}
|
|
3229
|
+
}
|
|
3230
|
+
|
|
3231
|
+
return encoder.toUint8Array()
|
|
3232
|
+
}
|
|
3233
|
+
|
|
2533
3234
|
// Splits an array of patches at a given character position within the
|
|
2534
3235
|
// combined delete+insert sequence.
|
|
2535
3236
|
//
|
|
@@ -2952,6 +3653,8 @@ function create_braid_text() {
|
|
|
2952
3653
|
}
|
|
2953
3654
|
|
|
2954
3655
|
braid_text.get_resource = get_resource
|
|
3656
|
+
braid_text.ensure_dt_exists = ensure_dt_exists
|
|
3657
|
+
braid_text.ensure_yjs_exists = ensure_yjs_exists
|
|
2955
3658
|
|
|
2956
3659
|
braid_text.db_folder_init = db_folder_init
|
|
2957
3660
|
braid_text.encode_filename = encode_filename
|
|
@@ -3096,8 +3799,8 @@ async function handle_cursors(resource, req, res) {
|
|
|
3096
3799
|
|
|
3097
3800
|
res.setHeader('Content-Type', 'application/text-cursors+json')
|
|
3098
3801
|
|
|
3099
|
-
if (!resource.
|
|
3100
|
-
var cursors = resource.
|
|
3802
|
+
if (!resource.cursors) resource.cursors = new cursor_state()
|
|
3803
|
+
var cursors = resource.cursors
|
|
3101
3804
|
var peer = req.headers['peer']
|
|
3102
3805
|
|
|
3103
3806
|
if (req.method === 'GET' || req.method === 'HEAD') {
|