braid-text 0.3.27 → 0.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/server.js CHANGED
@@ -3,6 +3,9 @@ let { Doc, OpLog, Branch } = require("@braid.org/diamond-types-node")
3
3
  let {http_server: braidify, fetch: braid_fetch} = require("braid-http")
4
4
  let fs = require("fs")
5
5
 
6
+ let Y = null
7
+ try { Y = require('yjs') } catch(e) {}
8
+
6
9
 
7
10
  function create_braid_text() {
8
11
  let braid_text = {
@@ -10,9 +13,15 @@ function create_braid_text() {
10
13
  db_folder: './braid-text-db',
11
14
  length_cache_size: 10,
12
15
  meta_file_save_period_ms: 1000,
16
+ debug_sync_checks: false,
13
17
  cache: {}
14
18
  }
15
19
 
20
+ function require_yjs() {
21
+ if (!Y) throw new Error('yjs is not installed. Install it with: npm install yjs')
22
+ return Y
23
+ }
24
+
16
25
  let waiting_puts = 0
17
26
 
18
27
  let max_encoded_key_size = 240
@@ -53,10 +62,11 @@ function create_braid_text() {
53
62
  }
54
63
 
55
64
  var resource = (typeof a == 'string') ? await get_resource(a) : a
65
+ await ensure_dt_exists(resource)
56
66
 
57
67
  if (!resource.meta.fork_point && options.fork_point_hint) {
58
68
  resource.meta.fork_point = options.fork_point_hint
59
- resource.change_meta()
69
+ resource.save_meta()
60
70
  }
61
71
 
62
72
  function extend_frontier(frontier, version, parents) {
@@ -78,7 +88,7 @@ function create_braid_text() {
78
88
  frontier = []
79
89
  var shadow = new Set()
80
90
 
81
- var bytes = resource.doc.toBytes()
91
+ var bytes = resource.dt.doc.toBytes()
82
92
  var [_, events, parentss] = braid_text.dt_parse([...bytes])
83
93
  for (var i = events.length - 1; i >= 0 && looking_for.size; i--) {
84
94
  var e = events[i].join('-')
@@ -139,7 +149,7 @@ function create_braid_text() {
139
149
  resource.meta.fork_point =
140
150
  extend_frontier(resource.meta.fork_point,
141
151
  update.version, update.parents)
142
- resource.change_meta()
152
+ resource.save_meta()
143
153
  }
144
154
 
145
155
  // see if remote has the fork point
@@ -148,7 +158,7 @@ function create_braid_text() {
148
158
  if (signal.aborted) return
149
159
 
150
160
  resource.meta.fork_point = null
151
- resource.change_meta()
161
+ resource.save_meta()
152
162
  }
153
163
  if (signal.aborted) return
154
164
 
@@ -158,7 +168,7 @@ function create_braid_text() {
158
168
  // DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
159
169
  options.do_investigating_disconnects_log_L04LPFHQ1M?.(a, 'braid-text fork-point binary search')
160
170
 
161
- var bytes = resource.doc.toBytes()
171
+ var bytes = resource.dt.doc.toBytes()
162
172
  var [_, events, __] = braid_text.dt_parse([...bytes])
163
173
  events = events.map(x => x.join('-'))
164
174
 
@@ -413,7 +423,7 @@ function create_braid_text() {
413
423
  var unknowns = []
414
424
  for (var event of (req.version || []).concat(req.parents || [])) {
415
425
  var [actor, seq] = decode_version(event)
416
- if (!resource.actor_seqs[actor]?.has(seq))
426
+ if (!resource.dt?.known_versions[actor]?.has(seq))
417
427
  unknowns.push(event)
418
428
  }
419
429
  if (unknowns.length)
@@ -494,8 +504,8 @@ function create_braid_text() {
494
504
 
495
505
  res.startSubscription({
496
506
  onClose: () => {
497
- if (merge_type === "dt") resource.clients.delete(options)
498
- else resource.simpleton_clients.delete(options)
507
+ if (merge_type === "dt") resource.dt.clients.delete(options)
508
+ else resource.dt.simpleton_clients.delete(options)
499
509
  }
500
510
  })
501
511
 
@@ -532,10 +542,11 @@ function create_braid_text() {
532
542
  }
533
543
 
534
544
  if (req.parents) {
545
+ await ensure_dt_exists(resource)
535
546
  await wait_for_events(
536
547
  options.key,
537
548
  req.parents,
538
- resource.actor_seqs,
549
+ resource.dt.known_versions,
539
550
  // approximation of memory usage for this update
540
551
  body != null ? body.length :
541
552
  patches.reduce((a, b) => a + b.range.length + b.content.length, 0),
@@ -546,7 +557,7 @@ function create_braid_text() {
546
557
  var unknowns = []
547
558
  for (var event of req.parents) {
548
559
  var [actor, seq] = decode_version(event)
549
- if (!resource.actor_seqs[actor]?.has(seq)) unknowns.push(event)
560
+ if (!resource.dt.known_versions[actor]?.has(seq)) unknowns.push(event)
550
561
  }
551
562
  if (unknowns.length)
552
563
  return done_my_turn(309, '', "Version Unknown Here", {
@@ -557,8 +568,13 @@ function create_braid_text() {
557
568
 
558
569
  var old_val = resource.val
559
570
  var old_version = resource.version
560
- var put_patches = patches ? patches.map(p => ({unit: p.unit, range: p.range, content: p.content})) : null
561
- var {change_count} = await braid_text.put(resource, { peer, version: req.version, parents: req.parents, patches, body, merge_type })
571
+ var put_patches = patches?.map(p => ({unit: p.unit,
572
+ range: p.range,
573
+ content: p.content})) || null
574
+ var {change_count} = await braid_text.put(
575
+ resource,
576
+ { peer, version: req.version, parents: req.parents, patches, body, merge_type }
577
+ )
562
578
 
563
579
  // if Repr-Digest is set,
564
580
  // and the request version is also our new current version,
@@ -666,6 +682,25 @@ function create_braid_text() {
666
682
  var version = resource.version
667
683
 
668
684
  if (!options.subscribe) {
685
+ // yjs-text range unit: return current text or full history
686
+ if (options.range_unit === 'yjs-text') {
687
+ await ensure_yjs_exists(resource)
688
+ if (options.parents && options.parents.length === 0) {
689
+ // Full history: root + initialization patch
690
+ var init_patches = [{
691
+ unit: 'yjs-text',
692
+ range: '(:)',
693
+ content: resource.val
694
+ }]
695
+ return {
696
+ version: ['999999999-' + (Math.max(0, [...resource.val].length - 1))],
697
+ parents: [],
698
+ patches: init_patches
699
+ }
700
+ }
701
+ return { version, body: resource.val }
702
+ }
703
+
669
704
  if (options.transfer_encoding === 'dt') {
670
705
  // optimization: if requesting current version
671
706
  // pretend as if they didn't set a version,
@@ -676,10 +711,10 @@ function create_braid_text() {
676
711
  var bytes = null
677
712
  if (op_v || options.parents) {
678
713
  if (op_v) {
679
- var doc = dt_get(resource.doc, op_v)
714
+ var doc = dt_get(resource.dt.doc, op_v)
680
715
  bytes = doc.toBytes()
681
716
  } else {
682
- bytes = resource.doc.toBytes()
717
+ bytes = resource.dt.doc.toBytes()
683
718
  var doc = Doc.fromBytes(bytes)
684
719
  }
685
720
  if (options.parents) {
@@ -687,18 +722,45 @@ function create_braid_text() {
687
722
  dt_get_local_version(bytes, options.parents))
688
723
  }
689
724
  doc.free()
690
- } else bytes = resource.doc.toBytes()
725
+ } else bytes = resource.dt.doc.toBytes()
691
726
  return { body: bytes }
692
727
  }
693
728
 
694
- return options.version || options.parents ? {
695
- version: options.version || options.parents,
696
- body: dt_get_string(resource.doc, options.version || options.parents)
697
- } : {
698
- version,
699
- body: resource.doc.get()
729
+ if (options.version || options.parents) {
730
+ await ensure_dt_exists(resource)
731
+ return {
732
+ version: options.version || options.parents,
733
+ body: dt_get_string(resource.dt.doc, options.version || options.parents)
734
+ }
735
+ } else {
736
+ return { version, body: resource.val }
700
737
  }
701
738
  } else {
739
+ // yjs-text subscribe: send full history first, then live updates
740
+ if (options.range_unit === 'yjs-text') {
741
+ await ensure_yjs_exists(resource)
742
+
743
+ // Send root
744
+ options.subscribe({version: [], parents: [], body: ""})
745
+
746
+ // Send initialization patch
747
+ if (resource.val) {
748
+ options.subscribe({
749
+ version: ['999999999-' + (Math.max(0, [...resource.val].length - 1))],
750
+ parents: [],
751
+ patches: [{unit: 'yjs-text', range: '(:)', content: resource.val}]
752
+ })
753
+ }
754
+
755
+ // Register for live updates
756
+ if (!resource.yjs_clients) resource.yjs_clients = new Set()
757
+ resource.yjs_clients.add(options)
758
+ options.signal?.addEventListener('abort', () =>
759
+ resource.yjs_clients.delete(options))
760
+
761
+ return
762
+ }
763
+
702
764
  options.my_subscribe_chain = Promise.resolve()
703
765
  options.my_subscribe = (x) =>
704
766
  options.my_subscribe_chain =
@@ -706,27 +768,32 @@ function create_braid_text() {
706
768
  options.subscribe(x))
707
769
 
708
770
  if (options.merge_type != "dt") {
771
+ // Simpleton clients require DT
772
+ await ensure_dt_exists(resource)
773
+
709
774
  let x = { version }
710
775
 
711
776
  if (!options.parents && !options.version) {
712
777
  x.parents = []
713
- x.body = resource.doc.get()
778
+ x.body = resource.val
714
779
  options.my_subscribe(x)
715
780
  } else {
716
781
  x.parents = options.version ? options.version : options.parents
717
782
 
718
783
  // only send them a version from these parents if we have these parents (otherwise we'll assume these parents are more recent, probably versions they created but haven't sent us yet, and we'll send them appropriate rebased updates when they send us these versions)
719
- let local_version = OpLog_remote_to_local(resource.doc, x.parents)
784
+ let local_version = OpLog_remote_to_local(resource.dt.doc, x.parents)
720
785
  if (local_version) {
721
- x.patches = get_xf_patches(resource.doc, local_version)
786
+ x.patches = get_xf_patches(resource.dt.doc, local_version)
722
787
  options.my_subscribe(x)
723
788
  }
724
789
  }
725
790
 
726
- resource.simpleton_clients.add(options)
791
+ resource.dt.simpleton_clients.add(options)
727
792
  options.signal?.addEventListener('abort', () =>
728
- resource.simpleton_clients.delete(options))
793
+ resource.dt.simpleton_clients.delete(options))
729
794
  } else {
795
+ // DT merge-type clients require DT
796
+ await ensure_dt_exists(resource)
730
797
 
731
798
  if (options.accept_encoding?.match(/updates\s*\((.*)\)/)?.[1].split(',').map(x=>x.trim()).includes('dt')) {
732
799
  // optimization: if client wants past current version,
@@ -734,7 +801,7 @@ function create_braid_text() {
734
801
  if (options.parents && v_eq(options.parents, version)) {
735
802
  options.my_subscribe({ encoding: 'dt', body: new Doc().toBytes() })
736
803
  } else {
737
- var bytes = resource.doc.toBytes()
804
+ var bytes = resource.dt.doc.toBytes()
738
805
  if (options.parents) {
739
806
  var doc = Doc.fromBytes(bytes)
740
807
  bytes = doc.getPatchSince(
@@ -752,10 +819,10 @@ function create_braid_text() {
752
819
  body: "",
753
820
  })
754
821
 
755
- updates = dt_get_patches(resource.doc)
822
+ updates = dt_get_patches(resource.dt.doc)
756
823
  } else {
757
824
  // Then start the subscription from the parents in options
758
- updates = dt_get_patches(resource.doc, options.parents || options.version)
825
+ updates = dt_get_patches(resource.dt.doc, options.parents || options.version)
759
826
  }
760
827
 
761
828
  for (let u of updates)
@@ -772,9 +839,9 @@ function create_braid_text() {
772
839
  if (updates.length === 0) options.write?.("\r\n")
773
840
  }
774
841
 
775
- resource.clients.add(options)
842
+ resource.dt.clients.add(options)
776
843
  options.signal?.addEventListener('abort', () =>
777
- resource.clients.delete(options))
844
+ resource.dt.clients.delete(options))
778
845
  }
779
846
  }
780
847
  }
@@ -789,8 +856,8 @@ function create_braid_text() {
789
856
  let resource = (typeof key == 'string') ? await get_resource(key) : key
790
857
 
791
858
  if (options.merge_type != "dt")
792
- resource.simpleton_clients.delete(options)
793
- else resource.clients.delete(options)
859
+ resource.dt.simpleton_clients.delete(options)
860
+ else resource.dt.clients.delete(options)
794
861
  }
795
862
 
796
863
  braid_text.put = async (key, options) => {
@@ -823,7 +890,7 @@ function create_braid_text() {
823
890
  // support for json patch puts..
824
891
  if (options.patches && options.patches.length &&
825
892
  options.patches.every(x => x.unit === 'json')) {
826
- let x = JSON.parse(resource.doc.get())
893
+ let x = JSON.parse(resource.val)
827
894
  for (let p of options.patches)
828
895
  apply_patch(x, p.range, p.content === '' ? undefined : JSON.parse(p.content))
829
896
  options = { body: JSON.stringify(x, null, 4) }
@@ -831,31 +898,205 @@ function create_braid_text() {
831
898
 
832
899
  let { version, parents, patches, body, peer } = options
833
900
 
901
+ // Raw Yjs binary update: apply directly to Y.Doc, sync to DT
902
+ if (options.yjs_update) {
903
+ await ensure_yjs_exists(resource)
904
+
905
+ // Pre-sync check: DT and Yjs should agree before we start
906
+ if (braid_text.debug_sync_checks && resource.dt) {
907
+ var pre_dt = resource.dt.doc.get()
908
+ var pre_yjs = resource.yjs.text.toString()
909
+ if (pre_dt !== pre_yjs) {
910
+ console.error(`PRE-SYNC MISMATCH key=${resource.key}: DT="${pre_dt.slice(0,50)}" Yjs="${pre_yjs.slice(0,50)}"`)
911
+ }
912
+ }
913
+
914
+ var prev_text = resource.yjs.text.toString()
915
+ var delta = null
916
+ var observer = (e) => { delta = e.changes.delta }
917
+ resource.yjs.text.observe(observer)
918
+ try {
919
+ Y.applyUpdate(resource.yjs.doc,
920
+ options.yjs_update instanceof Uint8Array ? options.yjs_update : new Uint8Array(options.yjs_update))
921
+ } finally {
922
+ resource.yjs.text.unobserve(observer)
923
+ }
924
+
925
+ resource.val = resource.yjs.text.toString()
926
+
927
+ // Sync to DT if it exists
928
+ if (resource.dt && delta) {
929
+ var text_patches = yjs_delta_to_patches(delta, prev_text)
930
+ if (text_patches.length) {
931
+ var syn_actor = `yjs-${Date.now()}-${Math.random().toString(36).slice(2)}`
932
+ var syn_seq = 0
933
+ var yjs_v_before = resource.dt.doc.getLocalVersion()
934
+ // Patches are sequential (each relative to state after previous)
935
+ // so no offset adjustment needed
936
+ var dt_bytes = []
937
+ var dt_ps = resource.version
938
+ for (var tp of text_patches) {
939
+ var tp_range = tp.range.match(/-?\d+/g).map(Number)
940
+ var tp_del = tp_range[1] - tp_range[0]
941
+ var syn_v = `${syn_actor}-${syn_seq}`
942
+ if (tp_del) {
943
+ dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], tp_del, null))
944
+ dt_ps = [`${syn_actor}-${syn_seq + tp_del - 1}`]
945
+ syn_seq += tp_del
946
+ syn_v = `${syn_actor}-${syn_seq}`
947
+ }
948
+ if (tp.content.length) {
949
+ dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], 0, tp.content))
950
+ var cp_len = [...tp.content].length
951
+ dt_ps = [`${syn_actor}-${syn_seq + cp_len - 1}`]
952
+ syn_seq += cp_len
953
+ }
954
+ }
955
+ for (var b of dt_bytes) resource.dt.doc.mergeBytes(b)
956
+ resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
957
+ if (!resource.dt.known_versions[syn_actor]) resource.dt.known_versions[syn_actor] = new RangeSet()
958
+ resource.dt.known_versions[syn_actor].add_range(0, syn_seq - 1)
959
+ await resource.dt.save_delta(resource.dt.doc.getPatchSince(yjs_v_before))
960
+ }
961
+ }
962
+
963
+ // Persist Yjs
964
+ if (resource.yjs.save_delta) await resource.yjs.save_delta(
965
+ options.yjs_update instanceof Uint8Array ? options.yjs_update : new Uint8Array(options.yjs_update))
966
+
967
+ // Sanity check
968
+ if (braid_text.debug_sync_checks && resource.dt) {
969
+ var dt_text = resource.dt.doc.get()
970
+ var yjs_text = resource.yjs.text.toString()
971
+ if (dt_text !== yjs_text) {
972
+ console.error(`SYNC MISMATCH key=${resource.key}: DT text !== Y.Doc text`)
973
+ console.error(` DT: ${dt_text.slice(0, 100)}... (${dt_text.length})`)
974
+ console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
975
+ }
976
+ }
977
+
978
+ return { change_count: 1 }
979
+ }
980
+
981
+ // yjs-text patches: apply to Y.Doc, sync to DT
982
+ if (patches && patches.length && patches[0].unit === 'yjs-text') {
983
+ await ensure_yjs_exists(resource)
984
+
985
+ // Convert yjs-text patches to binary and apply to Y.Doc
986
+ var binary = braid_text.to_yjs_binary(patches)
987
+ var prev_text = resource.yjs.text.toString()
988
+ var delta = null
989
+ var observer = (e) => { delta = e.changes.delta }
990
+ resource.yjs.text.observe(observer)
991
+ try {
992
+ Y.applyUpdate(resource.yjs.doc, binary)
993
+ } finally {
994
+ resource.yjs.text.unobserve(observer)
995
+ }
996
+
997
+ resource.val = resource.yjs.text.toString()
998
+
999
+ // Sync to DT if it exists
1000
+ if (resource.dt && delta) {
1001
+ var text_patches = yjs_delta_to_patches(delta, prev_text)
1002
+ if (text_patches.length) {
1003
+ // Generate a synthetic version for DT
1004
+ var syn_actor = `yjs-${Date.now()}-${Math.random().toString(36).slice(2)}`
1005
+ var syn_seq = 0
1006
+ var yjs_v_before = resource.dt.doc.getLocalVersion()
1007
+ // Patches are sequential (each relative to state after previous)
1008
+ // so no offset adjustment needed
1009
+ var dt_bytes = []
1010
+ var dt_ps = resource.version
1011
+ for (var tp of text_patches) {
1012
+ var tp_range = tp.range.match(/-?\d+/g).map(Number)
1013
+ var tp_del = tp_range[1] - tp_range[0]
1014
+ var syn_v = `${syn_actor}-${syn_seq}`
1015
+ if (tp_del) {
1016
+ dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], tp_del, null))
1017
+ dt_ps = [`${syn_actor}-${syn_seq + tp_del - 1}`]
1018
+ syn_seq += tp_del
1019
+ syn_v = `${syn_actor}-${syn_seq}`
1020
+ }
1021
+ if (tp.content.length) {
1022
+ dt_bytes.push(dt_create_bytes(syn_v, dt_ps, tp_range[0], 0, tp.content))
1023
+ var cp_len = [...tp.content].length
1024
+ dt_ps = [`${syn_actor}-${syn_seq + cp_len - 1}`]
1025
+ syn_seq += cp_len
1026
+ }
1027
+ }
1028
+ for (var b of dt_bytes) resource.dt.doc.mergeBytes(b)
1029
+ resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
1030
+
1031
+ // Update known_versions for the synthetic versions
1032
+ if (!resource.dt.known_versions[syn_actor]) resource.dt.known_versions[syn_actor] = new RangeSet()
1033
+ resource.dt.known_versions[syn_actor].add_range(0, syn_seq - 1)
1034
+
1035
+ await resource.dt.save_delta(resource.dt.doc.getPatchSince(yjs_v_before))
1036
+
1037
+ // Broadcast to DT subscribers
1038
+ // TODO: broadcast to simpleton and dt clients
1039
+ }
1040
+ }
1041
+
1042
+ // Broadcast to yjs-text subscribers (skip sender)
1043
+ if (resource.yjs_clients) {
1044
+ for (var client of resource.yjs_clients) {
1045
+ if (!peer || client.peer !== peer) {
1046
+ client.subscribe({
1047
+ version: resource.version,
1048
+ patches: patches
1049
+ })
1050
+ }
1051
+ }
1052
+ }
1053
+
1054
+ // Persist Yjs delta
1055
+ if (resource.yjs.save_delta) await resource.yjs.save_delta(binary)
1056
+
1057
+ // Sanity check
1058
+ if (braid_text.debug_sync_checks && resource.dt) {
1059
+ var dt_text = resource.dt.doc.get()
1060
+ var yjs_text = resource.yjs.text.toString()
1061
+ if (dt_text !== yjs_text) {
1062
+ console.error(`SYNC MISMATCH key=${resource.key}: DT text !== Y.Doc text`)
1063
+ console.error(` DT: ${dt_text.slice(0, 100)}... (${dt_text.length})`)
1064
+ console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
1065
+ }
1066
+ }
1067
+
1068
+ return { change_count: patches.length }
1069
+ }
1070
+
834
1071
  if (options.transfer_encoding === 'dt') {
835
- var start_i = 1 + resource.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
1072
+ await ensure_dt_exists(resource)
1073
+ var start_i = 1 + resource.dt.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
836
1074
 
837
- resource.doc.mergeBytes(body)
1075
+ resource.dt.doc.mergeBytes(body)
838
1076
 
839
- var end_i = resource.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
1077
+ var end_i = resource.dt.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
840
1078
  for (var i = start_i; i <= end_i; i++) {
841
- let v = resource.doc.localToRemoteVersion([i])[0]
842
- if (!resource.actor_seqs[v[0]]) resource.actor_seqs[v[0]] = new braid_text.RangeSet()
843
- resource.actor_seqs[v[0]].add_range(v[1], v[1])
1079
+ let v = resource.dt.doc.localToRemoteVersion([i])[0]
1080
+ if (!resource.dt.known_versions[v[0]]) resource.dt.known_versions[v[0]] = new braid_text.RangeSet()
1081
+ resource.dt.known_versions[v[0]].add_range(v[1], v[1])
844
1082
  }
845
- resource.val = resource.doc.get()
846
- resource.version = resource.doc.getRemoteVersion().map(x => x.join("-")).sort()
1083
+ resource.val = resource.dt.doc.get()
1084
+ resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
847
1085
 
848
- await resource.db_delta(body)
1086
+ await resource.dt.save_delta(body)
849
1087
 
850
1088
  // Notify non-simpleton clients with the dt-encoded update
851
1089
  var dt_update = { body, encoding: 'dt' }
852
- await Promise.all([...resource.clients].
1090
+ await Promise.all([...resource.dt.clients].
853
1091
  filter(client => !peer || client.peer !== peer).
854
1092
  map(client => client.my_subscribe(dt_update)))
855
1093
 
856
1094
  return { change_count: end_i - start_i + 1 }
857
1095
  }
858
1096
 
1097
+ // Text/DT patches require DT
1098
+ await ensure_dt_exists(resource)
1099
+
859
1100
  if (version && !version.length) {
860
1101
  console.log(`warning: ignoring put with empty version`)
861
1102
  return { change_count: 0 }
@@ -871,15 +1112,15 @@ function create_braid_text() {
871
1112
  // make sure we have all these parents
872
1113
  for (let p of parents) {
873
1114
  let P = decode_version(p)
874
- if (!resource.actor_seqs[P[0]]?.has(P[1]))
1115
+ if (!resource.dt.known_versions[P[0]]?.has(P[1]))
875
1116
  throw new Error(`missing parent version: ${p}`)
876
1117
  }
877
1118
  }
878
1119
 
879
1120
  if (!parents) parents = resource.version
880
1121
 
881
- let max_pos = resource.length_cache.get('' + parents) ??
882
- (v_eq(resource.version, parents) ? resource.doc.len() : dt_len(resource.doc, parents))
1122
+ let max_pos = resource.dt.length_at_version.get('' + parents) ??
1123
+ (v_eq(resource.version, parents) ? resource.dt.doc.len() : dt_len(resource.dt.doc, parents))
883
1124
 
884
1125
  if (body != null) {
885
1126
  patches = [{
@@ -912,7 +1153,7 @@ function create_braid_text() {
912
1153
  var low_seq = v[1] + 1 - change_count
913
1154
 
914
1155
  // make sure we haven't seen this already
915
- var intersects_range = resource.actor_seqs[v[0]]?.has(low_seq, v[1])
1156
+ var intersects_range = resource.dt.known_versions[v[0]]?.has(low_seq, v[1])
916
1157
  if (intersects_range) {
917
1158
  // if low_seq is below the range min,
918
1159
  // then the intersection has gaps,
@@ -936,8 +1177,8 @@ function create_braid_text() {
936
1177
  change_count = new_count
937
1178
  low_seq = v[1] + 1 - change_count
938
1179
  parents = [`${v[0]}-${low_seq - 1}`]
939
- max_pos = resource.length_cache.get('' + parents) ??
940
- (v_eq(resource.version, parents) ? resource.doc.len() : dt_len(resource.doc, parents))
1180
+ max_pos = resource.dt.length_at_version.get('' + parents) ??
1181
+ (v_eq(resource.version, parents) ? resource.dt.doc.len() : dt_len(resource.dt.doc, parents))
941
1182
  patches = new_patches
942
1183
  }
943
1184
 
@@ -951,12 +1192,12 @@ function create_braid_text() {
951
1192
  must_be_at_least = p.range[1]
952
1193
  }
953
1194
 
954
- resource.length_cache.put(`${v[0]}-${v[1]}`, patches.reduce((a, b) =>
1195
+ resource.dt.length_at_version.put(`${v[0]}-${v[1]}`, patches.reduce((a, b) =>
955
1196
  a + (b.content_codepoints?.length ?? 0) - (b.range[1] - b.range[0]),
956
1197
  max_pos))
957
1198
 
958
- if (!resource.actor_seqs[v[0]]) resource.actor_seqs[v[0]] = new RangeSet()
959
- resource.actor_seqs[v[0]].add_range(low_seq, v[1])
1199
+ if (!resource.dt.known_versions[v[0]]) resource.dt.known_versions[v[0]] = new RangeSet()
1200
+ resource.dt.known_versions[v[0]].add_range(low_seq, v[1])
960
1201
 
961
1202
  // get the version of the first character-wise edit
962
1203
  v = `${v[0]}-${low_seq}`
@@ -964,7 +1205,7 @@ function create_braid_text() {
964
1205
  let ps = parents
965
1206
 
966
1207
  let version_before = resource.version
967
- let v_before = resource.doc.getLocalVersion()
1208
+ let v_before = resource.dt.doc.getLocalVersion()
968
1209
 
969
1210
  let bytes = []
970
1211
 
@@ -989,20 +1230,89 @@ function create_braid_text() {
989
1230
  }
990
1231
  }
991
1232
 
992
- for (let b of bytes) resource.doc.mergeBytes(b)
993
- resource.val = resource.doc.get()
994
- resource.version = resource.doc.getRemoteVersion().map(x => x.join("-")).sort()
1233
+ for (let b of bytes) resource.dt.doc.mergeBytes(b)
1234
+ resource.val = resource.dt.doc.get()
1235
+ resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
1236
+
1237
+ // Get transformed patches (resolved after DT merge)
1238
+ // xf_patches: absolute positions (for simpleton clients)
1239
+ // xf_patches_relative: sequential positions (for Yjs sync)
1240
+ var xf_patches_relative = []
1241
+ for (let xf of resource.dt.doc.xfSince(v_before)) {
1242
+ xf_patches_relative.push(
1243
+ xf.kind == "Ins"
1244
+ ? { range: [xf.start, xf.start], content: xf.content }
1245
+ : { range: [xf.start, xf.end], content: "" }
1246
+ )
1247
+ }
1248
+ var xf_patches = relative_to_absolute_patches(
1249
+ xf_patches_relative.map(p => ({
1250
+ unit: 'text',
1251
+ range: `[${p.range[0]}:${p.range[1]}]`,
1252
+ content: p.content
1253
+ }))
1254
+ )
1255
+
1256
+ // Sync to Yjs if it exists, using relative (sequential) patches
1257
+ if (resource.yjs) {
1258
+ var captured_yjs_update = null
1259
+ var yjs_update_handler = (update, origin) => {
1260
+ if (origin === 'braid_text_dt_sync') captured_yjs_update = update
1261
+ }
1262
+ resource.yjs.doc.on('update', yjs_update_handler)
1263
+ resource.yjs.doc.transact(() => {
1264
+ // xf_patches_relative are sequential codepoint positions; Yjs uses UTF-16
1265
+ for (let p of xf_patches_relative) {
1266
+ var current_text = resource.yjs.text.toString()
1267
+ var cp_to_utf16 = codepoint_to_utf16_pos(current_text)
1268
+ var utf16_start = cp_to_utf16(p.range[0])
1269
+ var cp_del = p.range[1] - p.range[0]
1270
+ if (cp_del) {
1271
+ var utf16_end = cp_to_utf16(p.range[1])
1272
+ resource.yjs.text.delete(utf16_start, utf16_end - utf16_start)
1273
+ }
1274
+ if (p.content?.length) resource.yjs.text.insert(utf16_start, p.content)
1275
+ }
1276
+ }, 'braid_text_dt_sync')
1277
+ resource.yjs.doc.off('update', yjs_update_handler)
1278
+
1279
+ // Broadcast to yjs-text subscribers
1280
+ if (resource.yjs_clients && captured_yjs_update) {
1281
+ var yjs_patches = braid_text.from_yjs_binary(captured_yjs_update)
1282
+ for (var client of resource.yjs_clients) {
1283
+ if (!peer || client.peer !== peer) {
1284
+ client.subscribe({
1285
+ version: resource.version,
1286
+ patches: yjs_patches
1287
+ })
1288
+ }
1289
+ }
1290
+ }
1291
+
1292
+ // Persist Yjs delta
1293
+ if (captured_yjs_update) await resource.yjs.save_delta(captured_yjs_update)
1294
+
1295
+ // Sanity check
1296
+ if (braid_text.debug_sync_checks) {
1297
+ var yjs_text = resource.yjs.text.toString()
1298
+ if (resource.val !== yjs_text) {
1299
+ console.error(`SYNC MISMATCH key=${resource.key}: DT→Yjs sync failed`)
1300
+ console.error(` val: ${resource.val.slice(0, 100)}... (${resource.val.length})`)
1301
+ console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
1302
+ }
1303
+ }
1304
+ }
995
1305
 
996
1306
  // Transform stored cursor positions through the applied patches
997
- if (resource.cursor_state) resource.cursor_state.transform(patches)
1307
+ if (resource.cursors) resource.cursors.transform(patches)
998
1308
 
999
1309
  var post_commit_updates = []
1000
1310
 
1001
1311
  if (options.merge_type != "dt") {
1002
- let patches = get_xf_patches(resource.doc, v_before)
1312
+ let patches = xf_patches
1003
1313
  if (braid_text.verbose) console.log(JSON.stringify({ patches }))
1004
1314
 
1005
- for (let client of resource.simpleton_clients) {
1315
+ for (let client of resource.dt.simpleton_clients) {
1006
1316
  if (peer && client.peer === peer) {
1007
1317
  client.my_last_seen_version = [version]
1008
1318
  }
@@ -1011,7 +1321,7 @@ function create_braid_text() {
1011
1321
  if (client.my_timeout) clearTimeout(client.my_timeout)
1012
1322
  client.my_timeout = setTimeout(() => {
1013
1323
  // if the doc has been freed, exit early
1014
- if (resource.doc.__wbg_ptr === 0) return
1324
+ if (resource.dt.doc.__wbg_ptr === 0) return
1015
1325
 
1016
1326
  let x = {
1017
1327
  version: resource.version,
@@ -1019,7 +1329,7 @@ function create_braid_text() {
1019
1329
  }
1020
1330
  if (braid_text.verbose) console.log("rebasing after timeout.. ")
1021
1331
  if (braid_text.verbose) console.log(" client.my_unused_version_count = " + client.my_unused_version_count)
1022
- x.patches = get_xf_patches(resource.doc, OpLog_remote_to_local(resource.doc, client.my_last_seen_version))
1332
+ x.patches = get_xf_patches(resource.dt.doc, OpLog_remote_to_local(resource.dt.doc, client.my_last_seen_version))
1023
1333
 
1024
1334
  if (braid_text.verbose) console.log(`sending from rebase: ${JSON.stringify(x)}`)
1025
1335
  client.my_subscribe(x)
@@ -1063,14 +1373,14 @@ function create_braid_text() {
1063
1373
  post_commit_updates.push([client, x])
1064
1374
  }
1065
1375
  } else {
1066
- if (resource.simpleton_clients.size) {
1376
+ if (resource.dt.simpleton_clients.size) {
1067
1377
  let x = {
1068
1378
  version: resource.version,
1069
1379
  parents: version_before,
1070
- patches: get_xf_patches(resource.doc, v_before)
1380
+ patches: xf_patches
1071
1381
  }
1072
1382
  if (braid_text.verbose) console.log(`sending: ${JSON.stringify(x)}`)
1073
- for (let client of resource.simpleton_clients) {
1383
+ for (let client of resource.dt.simpleton_clients) {
1074
1384
  if (client.my_timeout) continue
1075
1385
  post_commit_updates.push([client, x])
1076
1386
  }
@@ -1086,12 +1396,12 @@ function create_braid_text() {
1086
1396
  content: p.content
1087
1397
  })),
1088
1398
  }
1089
- for (let client of resource.clients) {
1399
+ for (let client of resource.dt.clients) {
1090
1400
  if (!peer || client.peer !== peer)
1091
1401
  post_commit_updates.push([client, x])
1092
1402
  }
1093
1403
 
1094
- await resource.db_delta(resource.doc.getPatchSince(v_before))
1404
+ await resource.dt.save_delta(resource.dt.doc.getPatchSince(v_before))
1095
1405
 
1096
1406
  await Promise.all(post_commit_updates.map(([client, x]) => client.my_subscribe(x)))
1097
1407
 
@@ -1104,7 +1414,7 @@ function create_braid_text() {
1104
1414
  if (braid_text.db_folder) {
1105
1415
  await db_folder_init()
1106
1416
  var pages = new Set()
1107
- for (let x of await require('fs').promises.readdir(braid_text.db_folder)) if (/\.\d+$/.test(x)) pages.add(decode_filename(x.replace(/\.\d+$/, '')))
1417
+ for (let x of await require('fs').promises.readdir(braid_text.db_folder)) if (/\.(dt|yjs)\.\d+$/.test(x)) pages.add(decode_filename(x.replace(/\.(dt|yjs)\.\d+$/, '')))
1108
1418
  return [...pages.keys()]
1109
1419
  } else return Object.keys(braid_text.cache)
1110
1420
  } catch (e) { return [] }
@@ -1121,39 +1431,93 @@ function create_braid_text() {
1121
1431
  let cache = braid_text.cache
1122
1432
  if (!cache[key]) cache[key] = new Promise(async done => {
1123
1433
  let resource = {key}
1124
- resource.clients = new Set()
1125
- resource.simpleton_clients = new Set()
1126
-
1127
- resource.doc = new Doc("server")
1434
+ resource.dt = null
1435
+ resource.yjs = null
1436
+ resource.val = ""
1437
+ resource.version = []
1128
1438
  resource.meta = {}
1129
-
1130
- let { change, change_meta } = braid_text.db_folder
1131
- ? await file_sync(key,
1132
- (bytes) => resource.doc.mergeBytes(bytes),
1133
- () => resource.doc.toBytes(),
1439
+ resource.save_meta = () => {}
1440
+ resource.cursors = null
1441
+
1442
+ // Load DT data from disk if it exists
1443
+ var has_dt_files = braid_text.db_folder
1444
+ && (await get_files_for_key(key, 'dt')).length > 0
1445
+ if (has_dt_files) {
1446
+ resource.dt = {
1447
+ doc: new Doc("server"),
1448
+ known_versions: {},
1449
+ save_delta: () => {},
1450
+ length_at_version: createSimpleCache(braid_text.length_cache_size),
1451
+ clients: new Set(),
1452
+ simpleton_clients: new Set(),
1453
+ }
1454
+ let { change, change_meta } = await file_sync(key,
1455
+ (bytes) => resource.dt.doc.mergeBytes(bytes),
1456
+ () => resource.dt.doc.toBytes(),
1134
1457
  (meta) => resource.meta = meta,
1135
- () => resource.meta)
1136
- : { change: () => { }, change_meta: () => {} }
1458
+ () => resource.meta,
1459
+ 'dt')
1137
1460
 
1138
- resource.db_delta = change
1139
- resource.change_meta = change_meta
1461
+ resource.dt.save_delta = change
1462
+ resource.save_meta = change_meta
1140
1463
 
1141
- resource.actor_seqs = {}
1464
+ dt_get_actor_seq_runs([...resource.dt.doc.toBytes()], (actor, base, len) => {
1465
+ if (!resource.dt.known_versions[actor]) resource.dt.known_versions[actor] = new RangeSet()
1466
+ resource.dt.known_versions[actor].add_range(base, base + len - 1)
1467
+ })
1142
1468
 
1143
- dt_get_actor_seq_runs([...resource.doc.toBytes()], (actor, base, len) => {
1144
- if (!resource.actor_seqs[actor]) resource.actor_seqs[actor] = new RangeSet()
1145
- resource.actor_seqs[actor].add_range(base, base + len - 1)
1146
- })
1469
+ resource.val = resource.dt.doc.get()
1470
+ resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
1471
+ } else if (braid_text.db_folder) {
1472
+ // No DT files — still load meta
1473
+ let { change, change_meta } = await file_sync(key,
1474
+ () => {},
1475
+ () => Buffer.alloc(0),
1476
+ (meta) => resource.meta = meta,
1477
+ () => resource.meta,
1478
+ 'dt')
1479
+ resource.save_meta = change_meta
1480
+ }
1147
1481
 
1148
- resource.val = resource.doc.get()
1149
- resource.version = resource.doc.getRemoteVersion().map(x => x.join("-")).sort()
1482
+ // Load Yjs data from disk if it exists
1483
+ var has_yjs_files = braid_text.db_folder && Y
1484
+ && (await get_files_for_key(key, 'yjs')).length > 0
1485
+ if (has_yjs_files) {
1486
+ resource.yjs = {
1487
+ doc: new Y.Doc(),
1488
+ text: null,
1489
+ save_delta: () => {},
1490
+ }
1491
+ resource.yjs.text = resource.yjs.doc.getText('text')
1492
+ let { change } = await file_sync(key,
1493
+ (bytes) => Y.applyUpdate(resource.yjs.doc, bytes),
1494
+ () => Y.encodeStateAsUpdate(resource.yjs.doc),
1495
+ () => {},
1496
+ () => ({}),
1497
+ 'yjs')
1498
+
1499
+ resource.yjs.save_delta = change
1500
+
1501
+ var yjs_text = resource.yjs.text.toString()
1502
+ if (resource.dt) {
1503
+ // Both exist — sanity check they match
1504
+ if (resource.val !== yjs_text) {
1505
+ console.error(`INIT MISMATCH key=${key}: DT text !== Yjs text`)
1506
+ console.error(` DT: ${resource.val.slice(0, 100)}... (${resource.val.length})`)
1507
+ console.error(` Yjs: ${yjs_text.slice(0, 100)}... (${yjs_text.length})`)
1508
+ }
1509
+ } else {
1510
+ // Yjs only — use its text as resource.val
1511
+ resource.val = yjs_text
1512
+ }
1513
+ }
1150
1514
 
1151
- resource.length_cache = createSimpleCache(braid_text.length_cache_size)
1515
+ // length_at_version cache is created as part of resource.dt when DT is initialized
1152
1516
 
1153
1517
  // Add delete method to resource
1154
1518
  resource.delete = async () => {
1155
- // Free the diamond-types document
1156
- if (resource.doc) resource.doc.free()
1519
+ if (resource.dt) resource.dt.doc.free()
1520
+ if (resource.yjs) resource.yjs.doc.destroy()
1157
1521
 
1158
1522
  // Remove from in-memory cache
1159
1523
  delete braid_text.cache[key]
@@ -1172,7 +1536,7 @@ function create_braid_text() {
1172
1536
  // Remove meta file if it exists
1173
1537
  try {
1174
1538
  var encoded = encode_filename(key)
1175
- await fs.promises.unlink(`${braid_text.db_folder}/.meta/${encoded}`)
1539
+ await fs.promises.unlink(`${braid_text.db_folder}/meta/${encoded}`)
1176
1540
  } catch (e) {
1177
1541
  // Meta file might not exist, that's ok
1178
1542
  }
@@ -1191,23 +1555,133 @@ function create_braid_text() {
1191
1555
  return await cache[key]
1192
1556
  }
1193
1557
 
1558
+ async function ensure_dt_exists(resource) {
1559
+ if (resource.dt) return
1560
+ resource.dt = {
1561
+ doc: new Doc("server"),
1562
+ known_versions: {},
1563
+ save_delta: () => {},
1564
+ length_at_version: createSimpleCache(braid_text.length_cache_size),
1565
+ clients: new Set(),
1566
+ simpleton_clients: new Set(),
1567
+ }
1568
+ if (resource.val) {
1569
+ // Seed DT with current text as a root insert
1570
+ var bytes = dt_create_bytes(
1571
+ `999999999-0`, [], 0, 0, resource.val)
1572
+ resource.dt.doc.mergeBytes(bytes)
1573
+ }
1574
+ resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
1575
+
1576
+ dt_get_actor_seq_runs([...resource.dt.doc.toBytes()], (actor, base, len) => {
1577
+ if (!resource.dt.known_versions[actor]) resource.dt.known_versions[actor] = new RangeSet()
1578
+ resource.dt.known_versions[actor].add_range(base, base + len - 1)
1579
+ })
1580
+
1581
+ // Set up DT persistence if db_folder exists
1582
+ if (braid_text.db_folder) {
1583
+ let { change } = await file_sync(resource.key,
1584
+ (bytes) => resource.dt.doc.mergeBytes(bytes),
1585
+ () => resource.dt.doc.toBytes(),
1586
+ () => {},
1587
+ () => ({}),
1588
+ 'dt')
1589
+ resource.dt.save_delta = change
1590
+ }
1591
+ }
1592
+
1593
+ async function ensure_yjs_exists(resource) {
1594
+ if (resource.yjs) return
1595
+ require_yjs()
1596
+ resource.yjs = {
1597
+ doc: new Y.Doc(),
1598
+ text: null,
1599
+ save_delta: () => {},
1600
+ }
1601
+ resource.yjs.text = resource.yjs.doc.getText('text')
1602
+ if (resource.val) {
1603
+ resource.yjs.text.insert(0, resource.val)
1604
+ }
1605
+
1606
+ // Set up Yjs persistence if db_folder exists
1607
+ if (braid_text.db_folder) {
1608
+ let { change } = await file_sync(resource.key,
1609
+ (bytes) => Y.applyUpdate(resource.yjs.doc, bytes),
1610
+ () => Y.encodeStateAsUpdate(resource.yjs.doc),
1611
+ () => {},
1612
+ () => ({}),
1613
+ 'yjs')
1614
+ resource.yjs.save_delta = change
1615
+ }
1616
+ }
1617
+
1194
1618
  async function db_folder_init() {
1195
1619
  if (braid_text.verbose) console.log('__!')
1196
1620
  if (!db_folder_init.p) db_folder_init.p = (async () => {
1197
1621
  await fs.promises.mkdir(braid_text.db_folder, { recursive: true });
1198
- await fs.promises.mkdir(`${braid_text.db_folder}/.meta`, { recursive: true })
1199
- await fs.promises.mkdir(`${braid_text.db_folder}/.temp`, { recursive: true })
1200
- await fs.promises.mkdir(`${braid_text.db_folder}/.wal-intent`, { recursive: true })
1201
1622
 
1202
- // Clean out .temp directory on startup
1203
- var temp_files = await fs.promises.readdir(`${braid_text.db_folder}/.temp`)
1623
+ // Migrate from old dot-prefixed directory names to new names
1624
+ // This is idempotent: safe to re-run if interrupted mid-migration
1625
+ async function migrate_dir(old_name, new_name) {
1626
+ var old_path = `${braid_text.db_folder}/${old_name}`
1627
+ var new_path = `${braid_text.db_folder}/${new_name}`
1628
+ try {
1629
+ await fs.promises.stat(old_path)
1630
+ // Old dir exists — ensure new dir exists and move contents
1631
+ await fs.promises.mkdir(new_path, { recursive: true })
1632
+ var entries = await fs.promises.readdir(old_path)
1633
+ for (var entry of entries) {
1634
+ var src = `${old_path}/${entry}`
1635
+ var dst = `${new_path}/${entry}`
1636
+ try { await fs.promises.stat(dst) } catch (e) {
1637
+ // dst doesn't exist, move src there
1638
+ await fs.promises.rename(src, dst)
1639
+ }
1640
+ // If dst already exists (crash recovery), just remove src
1641
+ try { await fs.promises.unlink(src) } catch (e) {}
1642
+ }
1643
+ // Remove old dir once empty
1644
+ try { await fs.promises.rmdir(old_path) } catch (e) {}
1645
+ } catch (e) {
1646
+ // Old dir doesn't exist — no migration needed
1647
+ }
1648
+ }
1649
+ await migrate_dir('.meta', 'meta')
1650
+ await migrate_dir('.temp', 'temp')
1651
+ await migrate_dir('.wal-intent', 'wal-intent')
1652
+
1653
+ // Migrate data files from [key].N to [key].dt.N
1654
+ // Idempotent: only renames files that match old pattern but not new
1655
+ var all_files = await fs.promises.readdir(braid_text.db_folder)
1656
+ for (var f of all_files) {
1657
+ // Match old format: ends with .N (digits only) but not .dt.N or .yjs.N
1658
+ if (/\.\d+$/.test(f) && !/\.(dt|yjs)\.\d+$/.test(f)) {
1659
+ var new_name = f.replace(/\.(\d+)$/, '.dt.$1')
1660
+ var old_path = `${braid_text.db_folder}/${f}`
1661
+ var new_path = `${braid_text.db_folder}/${new_name}`
1662
+ try { await fs.promises.stat(new_path) } catch (e) {
1663
+ // New file doesn't exist, rename
1664
+ await fs.promises.rename(old_path, new_path)
1665
+ continue
1666
+ }
1667
+ // If new file already exists (crash recovery), remove old
1668
+ try { await fs.promises.unlink(old_path) } catch (e) {}
1669
+ }
1670
+ }
1671
+
1672
+ await fs.promises.mkdir(`${braid_text.db_folder}/meta`, { recursive: true })
1673
+ await fs.promises.mkdir(`${braid_text.db_folder}/temp`, { recursive: true })
1674
+ await fs.promises.mkdir(`${braid_text.db_folder}/wal-intent`, { recursive: true })
1675
+
1676
+ // Clean out temp directory on startup
1677
+ var temp_files = await fs.promises.readdir(`${braid_text.db_folder}/temp`)
1204
1678
  for (var f of temp_files)
1205
- await fs.promises.unlink(`${braid_text.db_folder}/.temp/${f}`)
1679
+ await fs.promises.unlink(`${braid_text.db_folder}/temp/${f}`)
1206
1680
 
1207
- // Replay any pending .wal-intent files
1208
- var intent_files = await fs.promises.readdir(`${braid_text.db_folder}/.wal-intent`)
1681
+ // Replay any pending wal-intent files
1682
+ var intent_files = await fs.promises.readdir(`${braid_text.db_folder}/wal-intent`)
1209
1683
  for (var intent_name of intent_files) {
1210
- var intent_path = `${braid_text.db_folder}/.wal-intent/${intent_name}`
1684
+ var intent_path = `${braid_text.db_folder}/wal-intent/${intent_name}`
1211
1685
  var target_path = `${braid_text.db_folder}/${intent_name}`
1212
1686
 
1213
1687
  var intent_data = await fs.promises.readFile(intent_path)
@@ -1227,25 +1701,27 @@ function create_braid_text() {
1227
1701
 
1228
1702
  // Populate key_to_filename mapping from existing files
1229
1703
  var files = (await fs.promises.readdir(braid_text.db_folder))
1230
- .filter(x => /\.\d+$/.test(x))
1704
+ .filter(x => /\.dt\.\d+$/.test(x))
1231
1705
  init_filename_mapping(files)
1232
1706
  })()
1233
1707
  await db_folder_init.p
1234
1708
  }
1235
1709
 
1236
- async function get_files_for_key(key) {
1710
+ async function get_files_for_key(key, type) {
1237
1711
  await db_folder_init()
1238
1712
  try {
1239
- let re = new RegExp("^" + encode_filename(key).replace(/[^a-zA-Z0-9]/g, "\\$&") + "\\.\\w+$")
1713
+ var suffix = type ? `\\.${type}\\.\\d+$` : `\\.(dt|yjs)\\.\\d+$`
1714
+ let re = new RegExp("^" + encode_filename(key).replace(/[^a-zA-Z0-9]/g, "\\$&") + suffix)
1240
1715
  return (await fs.promises.readdir(braid_text.db_folder))
1241
1716
  .filter((a) => re.test(a))
1242
1717
  .map((a) => `${braid_text.db_folder}/${a}`)
1243
1718
  } catch (e) { return [] }
1244
1719
  }
1245
1720
 
1246
- async function file_sync(key, process_delta, get_init, set_meta, get_meta) {
1721
+ async function file_sync(key, process_delta, get_init, set_meta, get_meta, file_type) {
1247
1722
  await db_folder_init()
1248
1723
  let encoded = encode_filename(key)
1724
+ file_type = file_type || 'dt'
1249
1725
 
1250
1726
  if (encoded.length > max_encoded_key_size) throw new Error(`invalid key: too long (max ${max_encoded_key_size})`)
1251
1727
 
@@ -1254,7 +1730,7 @@ function create_braid_text() {
1254
1730
  let threshold = 0
1255
1731
 
1256
1732
  // Read existing files and sort by numbers.
1257
- const files = (await get_files_for_key(key))
1733
+ const files = (await get_files_for_key(key, file_type))
1258
1734
  .filter(x => x.match(/\.\d+$/))
1259
1735
  .sort((a, b) => parseInt(a.match(/\d+$/)[0]) - parseInt(b.match(/\d+$/)[0]))
1260
1736
 
@@ -1286,7 +1762,7 @@ function create_braid_text() {
1286
1762
  }
1287
1763
 
1288
1764
  currentSize = data.length
1289
- currentNumber = parseInt(filename.match(/\d+$/)[0])
1765
+ currentNumber = parseInt(filename.match(/(\d+)$/)[1])
1290
1766
  done = true
1291
1767
  } catch (error) {
1292
1768
  console.error(`Error processing file: ${files[i]}`)
@@ -1294,7 +1770,7 @@ function create_braid_text() {
1294
1770
  }
1295
1771
  }
1296
1772
 
1297
- var meta_filename = `${braid_text.db_folder}/.meta/${encoded}`
1773
+ var meta_filename = `${braid_text.db_folder}/meta/${encoded}`
1298
1774
  var meta_dirty = null
1299
1775
  var meta_saving = null
1300
1776
  var meta_file_content = '{}'
@@ -1307,7 +1783,7 @@ function create_braid_text() {
1307
1783
  change: (bytes) => within_fiber('file:' + key, async () => {
1308
1784
  if (!bytes) currentSize = threshold
1309
1785
  else currentSize += bytes.length + 4 // we account for the extra 4 bytes for uint32
1310
- const filename = `${braid_text.db_folder}/${encoded}.${currentNumber}`
1786
+ const filename = `${braid_text.db_folder}/${encoded}.${file_type}.${currentNumber}`
1311
1787
  if (currentSize < threshold) {
1312
1788
  if (braid_text.verbose) console.log(`appending to db..`)
1313
1789
 
@@ -1316,13 +1792,13 @@ function create_braid_text() {
1316
1792
  let append_data = Buffer.concat([len_buf, bytes])
1317
1793
 
1318
1794
  let basename = require('path').basename(filename)
1319
- let intent_path = `${braid_text.db_folder}/.wal-intent/${basename}`
1795
+ let intent_path = `${braid_text.db_folder}/wal-intent/${basename}`
1320
1796
  let stat = await fs.promises.stat(filename)
1321
1797
  let size_buf = Buffer.allocUnsafe(8)
1322
1798
  size_buf.writeBigUInt64LE(BigInt(stat.size), 0)
1323
1799
 
1324
1800
  await atomic_write(intent_path, Buffer.concat([size_buf, append_data]),
1325
- `${braid_text.db_folder}/.temp`)
1801
+ `${braid_text.db_folder}/temp`)
1326
1802
  await fs.promises.appendFile(filename, append_data)
1327
1803
  await fs.promises.unlink(intent_path)
1328
1804
 
@@ -1336,9 +1812,9 @@ function create_braid_text() {
1336
1812
  const buffer = Buffer.allocUnsafe(4)
1337
1813
  buffer.writeUInt32LE(init.length, 0)
1338
1814
 
1339
- const newFilename = `${braid_text.db_folder}/${encoded}.${currentNumber}`
1815
+ const newFilename = `${braid_text.db_folder}/${encoded}.${file_type}.${currentNumber}`
1340
1816
  await atomic_write(newFilename, Buffer.concat([buffer, init]),
1341
- `${braid_text.db_folder}/.temp`)
1817
+ `${braid_text.db_folder}/temp`)
1342
1818
 
1343
1819
  if (braid_text.verbose) console.log("wrote to : " + newFilename)
1344
1820
 
@@ -1360,7 +1836,7 @@ function create_braid_text() {
1360
1836
  while (meta_dirty) {
1361
1837
  meta_dirty = false
1362
1838
  await atomic_write(meta_filename, JSON.stringify(get_meta()),
1363
- `${braid_text.db_folder}/.temp`)
1839
+ `${braid_text.db_folder}/temp`)
1364
1840
  await new Promise(done => setTimeout(done,
1365
1841
  braid_text.meta_file_save_period_ms))
1366
1842
  }
@@ -1472,7 +1948,7 @@ function create_braid_text() {
1472
1948
 
1473
1949
  function validate_old_patches(resource, base_v, parents, patches) {
1474
1950
  // if we have seen it already, make sure it's the same as before
1475
- let updates = dt_get_patches(resource.doc, parents)
1951
+ let updates = dt_get_patches(resource.dt.doc, parents)
1476
1952
 
1477
1953
  let seen = {}
1478
1954
  for (let u of updates) {
@@ -2470,8 +2946,8 @@ function create_braid_text() {
2470
2946
  // Populate key_to_filename mapping from existing files on disk
2471
2947
  function init_filename_mapping(files) {
2472
2948
  for (var file of files) {
2473
- // Extract the encoded key (strip extension like .0, .1, etc.)
2474
- var encoded = file.replace(/\.\d+$/, '')
2949
+ // Extract the encoded key (strip extension like .dt.0, .dt.1, etc.)
2950
+ var encoded = file.replace(/\.(dt|yjs)\.\d+$/, '')
2475
2951
  var key = decode_filename(encoded)
2476
2952
 
2477
2953
  if (!key_to_filename.has(key)) {
@@ -2524,12 +3000,237 @@ function create_braid_text() {
2524
3000
 
2525
3001
  function validate_patch(x) {
2526
3002
  if (typeof x != 'object') throw new Error(`invalid patch: not an object`)
2527
- if (x.unit && x.unit !== 'text') throw new Error(`invalid patch unit '${x.unit}': only 'text' supported`)
3003
+ if (x.unit === 'yjs-text') return validate_yjs_patch(x)
3004
+ if (x.unit && x.unit !== 'text') throw new Error(`invalid patch unit '${x.unit}': only 'text' and 'yjs-text' supported`)
2528
3005
  if (typeof x.range !== 'string') throw new Error(`invalid patch range: must be a string`)
2529
3006
  if (!x.range.match(/^\s*\[\s*-?\d+\s*:\s*-?\d+\s*\]\s*$/)) throw new Error(`invalid patch range: ${x.range}`)
2530
3007
  if (typeof x.content !== 'string') throw new Error(`invalid patch content: must be a string`)
2531
3008
  }
2532
3009
 
3010
+ // yjs-text range format:
3011
+ // Inserts (exclusive): yjs-text (clientID-clock:clientID-clock)
3012
+ // Deletes (inclusive): yjs-text [clientID-clock:clientID-clock]
3013
+ // Null origins: (:), (:42-5), (42-5:)
3014
+ // Client IDs and clocks are always non-negative integers.
3015
+
3016
+ var yjs_id_pattern = '(\\d+-\\d+)'
3017
+ var yjs_range_re = new RegExp(
3018
+ '^\\s*' +
3019
+ '([\\(\\[])' + // open bracket: ( or [
3020
+ '\\s*' + yjs_id_pattern + '?' + // optional left ID
3021
+ '\\s*:\\s*' + // colon separator
3022
+ yjs_id_pattern + '?' + '\\s*' + // optional right ID
3023
+ '([\\)\\]])' + // close bracket: ) or ]
3024
+ '\\s*$'
3025
+ )
3026
+
3027
+ function validate_yjs_patch(x) {
3028
+ if (typeof x != 'object') throw new Error(`invalid yjs patch: not an object`)
3029
+ if (typeof x.range !== 'string') throw new Error(`invalid yjs patch range: must be a string`)
3030
+ if (typeof x.content !== 'string') throw new Error(`invalid yjs patch content: must be a string`)
3031
+ var parsed = parse_yjs_range(x.range)
3032
+ if (!parsed) throw new Error(`invalid yjs patch range: ${x.range}`)
3033
+ }
3034
+
3035
+ function parse_yjs_range(range_string) {
3036
+ var m = range_string.match(yjs_range_re)
3037
+ if (!m) return null
3038
+
3039
+ var open = m[1] // ( or [
3040
+ var left = m[2] // e.g. "42-5" or undefined
3041
+ var right = m[3] // e.g. "73-2" or undefined
3042
+ var close = m[4] // ) or ]
3043
+
3044
+ // Validate bracket pairing: () for exclusive, [] for inclusive
3045
+ var exclusive = (open === '(' && close === ')')
3046
+ var inclusive = (open === '[' && close === ']')
3047
+ if (!exclusive && !inclusive) return null
3048
+
3049
+ function parse_id(s) {
3050
+ if (!s) return null
3051
+ var dash = s.lastIndexOf('-')
3052
+ return { client: parseInt(s.slice(0, dash)), clock: parseInt(s.slice(dash + 1)) }
3053
+ }
3054
+
3055
+ var result = {
3056
+ inclusive: inclusive,
3057
+ left: parse_id(left),
3058
+ right: parse_id(right)
3059
+ }
3060
+
3061
+ // Inclusive ranges must have at least one ID (can't delete nothing)
3062
+ if (inclusive && !result.left && !result.right) return null
3063
+
3064
+ return result
3065
+ }
3066
+
3067
+ // Convert a Yjs delta (array of {retain, insert, delete} ops)
3068
+ // to positional text patches [{unit, range, content}]
3069
+ // range is returned as a parsed array [start, end], not a string
3070
+ // Convert a Yjs delta to positional text patches with codepoint positions.
3071
+ // Yjs deltas use UTF-16 positions; we need to convert to codepoints for DT.
3072
+ // prev_text is the text BEFORE the delta was applied (in UTF-16, i.e. a JS string).
3073
+ function yjs_delta_to_patches(delta, prev_text) {
3074
+ var patches = []
3075
+ var utf16_pos = 0
3076
+ var cp_pos = 0
3077
+ for (var op of delta) {
3078
+ if (op.retain) {
3079
+ // Count codepoints in the retained region
3080
+ var retained = prev_text.slice(utf16_pos, utf16_pos + op.retain)
3081
+ cp_pos += [...retained].length
3082
+ utf16_pos += op.retain
3083
+ } else if (op.insert) {
3084
+ var cp_len = [...op.insert].length
3085
+ patches.push({
3086
+ unit: 'text',
3087
+ range: `[${cp_pos}:${cp_pos}]`,
3088
+ content: op.insert
3089
+ })
3090
+ cp_pos += cp_len
3091
+ } else if (op.delete) {
3092
+ var deleted = prev_text.slice(utf16_pos, utf16_pos + op.delete)
3093
+ var cp_del = [...deleted].length
3094
+ patches.push({
3095
+ unit: 'text',
3096
+ range: `[${cp_pos}:${cp_pos + cp_del}]`,
3097
+ content: ''
3098
+ })
3099
+ utf16_pos += op.delete
3100
+ }
3101
+ }
3102
+ return patches
3103
+ }
3104
+
3105
+ // Convert codepoint index to UTF-16 index in a string.
3106
+ // Returns a function that maps codepoint position -> UTF-16 position.
3107
+ function codepoint_to_utf16_pos(str) {
3108
+ return function(cp_pos) {
3109
+ var utf16 = 0
3110
+ var cp = 0
3111
+ while (cp < cp_pos && utf16 < str.length) {
3112
+ var code = str.charCodeAt(utf16)
3113
+ utf16 += (code >= 0xD800 && code <= 0xDBFF) ? 2 : 1
3114
+ cp++
3115
+ }
3116
+ return utf16
3117
+ }
3118
+ }
3119
+
3120
+ braid_text.parse_yjs_range = parse_yjs_range
3121
+ braid_text.validate_patches = validate_patches
3122
+
3123
+ // Convert a Yjs binary update to yjs-text range patches.
3124
+ // Decodes the binary without needing a Y.Doc.
3125
+ // Returns array of {unit: 'yjs-text', range: '...', content: '...'}
3126
+ braid_text.from_yjs_binary = function(update) {
3127
+ require_yjs()
3128
+ var decoded = Y.decodeUpdate(
3129
+ update instanceof Uint8Array ? update : new Uint8Array(update))
3130
+ var patches = []
3131
+
3132
+ // Convert inserted structs to yjs-text patches
3133
+ for (var struct of decoded.structs) {
3134
+ if (!struct.content?.str) continue // skip non-text items
3135
+ var id = struct.id
3136
+ var origin = struct.origin
3137
+ var rightOrigin = struct.rightOrigin
3138
+ var left = origin ? `${origin.client}-${origin.clock}` : ''
3139
+ var right = rightOrigin ? `${rightOrigin.client}-${rightOrigin.clock}` : ''
3140
+ patches.push({
3141
+ unit: 'yjs-text',
3142
+ range: `(${left}:${right})`,
3143
+ content: struct.content.str
3144
+ })
3145
+ }
3146
+
3147
+ // Convert delete set entries to yjs-text patches
3148
+ for (var [clientID, deleteItems] of decoded.ds.clients) {
3149
+ for (var item of deleteItems) {
3150
+ var left = `${clientID}-${item.clock}`
3151
+ var right = `${clientID}-${item.clock + item.len - 1}`
3152
+ patches.push({
3153
+ unit: 'yjs-text',
3154
+ range: `[${left}:${right}]`,
3155
+ content: ''
3156
+ })
3157
+ }
3158
+ }
3159
+
3160
+ return patches
3161
+ }
3162
+
3163
+ // Convert yjs-text range patches to a Yjs binary update.
3164
+ // This is the inverse of from_yjs_binary.
3165
+ // Each insert patch needs a clientID and clock for the new item.
3166
+ // These are passed as patch.id = {client, clock}.
3167
+ braid_text.to_yjs_binary = function(patches) {
3168
+ require_yjs()
3169
+ var lib0_encoding = require('lib0/encoding')
3170
+ var encoder = new Y.UpdateEncoderV1()
3171
+
3172
+ // Group inserts by client
3173
+ var inserts_by_client = new Map()
3174
+ var deletes_by_client = new Map()
3175
+
3176
+ for (var p of patches) {
3177
+ var parsed = parse_yjs_range(p.range)
3178
+ if (!parsed) throw new Error(`invalid yjs-text range: ${p.range}`)
3179
+
3180
+ if (p.content.length > 0) {
3181
+ // Insert
3182
+ if (!p.id) throw new Error('insert patch requires .id = {client, clock}')
3183
+ var list = inserts_by_client.get(p.id.client) || []
3184
+ list.push({ id: p.id, origin: parsed.left, rightOrigin: parsed.right, content: p.content })
3185
+ inserts_by_client.set(p.id.client, list)
3186
+ } else {
3187
+ // Delete
3188
+ if (!parsed.left) throw new Error('delete patch requires left ID')
3189
+ var client = parsed.left.client
3190
+ var list = deletes_by_client.get(client) || []
3191
+ list.push({ clock: parsed.left.clock, len: parsed.right ? parsed.right.clock - parsed.left.clock + 1 : 1 })
3192
+ deletes_by_client.set(client, list)
3193
+ }
3194
+ }
3195
+
3196
+ // Write structs
3197
+ lib0_encoding.writeVarUint(encoder.restEncoder, inserts_by_client.size)
3198
+ for (var [client, items] of inserts_by_client) {
3199
+ items.sort((a, b) => a.id.clock - b.id.clock)
3200
+ lib0_encoding.writeVarUint(encoder.restEncoder, items.length)
3201
+ encoder.writeClient(client)
3202
+ lib0_encoding.writeVarUint(encoder.restEncoder, items[0].id.clock)
3203
+ for (var item of items) {
3204
+ var has_origin = item.origin !== null
3205
+ var has_right_origin = item.rightOrigin !== null
3206
+ // info byte: content ref (4 = string) | origin flags
3207
+ var info = 4 | (has_origin ? 0x80 : 0) | (has_right_origin ? 0x40 : 0)
3208
+ encoder.writeInfo(info)
3209
+ if (has_origin) encoder.writeLeftID(Y.createID(item.origin.client, item.origin.clock))
3210
+ if (has_right_origin) encoder.writeRightID(Y.createID(item.rightOrigin.client, item.rightOrigin.clock))
3211
+ if (!has_origin && !has_right_origin) {
3212
+ // Root insert — write parent type key
3213
+ encoder.writeParentInfo(true)
3214
+ encoder.writeString('text')
3215
+ }
3216
+ encoder.writeString(item.content)
3217
+ }
3218
+ }
3219
+
3220
+ // Write delete set
3221
+ lib0_encoding.writeVarUint(encoder.restEncoder, deletes_by_client.size)
3222
+ for (var [client, deletes] of deletes_by_client) {
3223
+ lib0_encoding.writeVarUint(encoder.restEncoder, client)
3224
+ lib0_encoding.writeVarUint(encoder.restEncoder, deletes.length)
3225
+ for (var d of deletes) {
3226
+ encoder.writeDsClock(d.clock)
3227
+ encoder.writeDsLen(d.len)
3228
+ }
3229
+ }
3230
+
3231
+ return encoder.toUint8Array()
3232
+ }
3233
+
2533
3234
  // Splits an array of patches at a given character position within the
2534
3235
  // combined delete+insert sequence.
2535
3236
  //
@@ -2952,6 +3653,8 @@ function create_braid_text() {
2952
3653
  }
2953
3654
 
2954
3655
  braid_text.get_resource = get_resource
3656
+ braid_text.ensure_dt_exists = ensure_dt_exists
3657
+ braid_text.ensure_yjs_exists = ensure_yjs_exists
2955
3658
 
2956
3659
  braid_text.db_folder_init = db_folder_init
2957
3660
  braid_text.encode_filename = encode_filename
@@ -3096,8 +3799,8 @@ async function handle_cursors(resource, req, res) {
3096
3799
 
3097
3800
  res.setHeader('Content-Type', 'application/text-cursors+json')
3098
3801
 
3099
- if (!resource.cursor_state) resource.cursor_state = new cursor_state()
3100
- var cursors = resource.cursor_state
3802
+ if (!resource.cursors) resource.cursors = new cursor_state()
3803
+ var cursors = resource.cursors
3101
3804
  var peer = req.headers['peer']
3102
3805
 
3103
3806
  if (req.method === 'GET' || req.method === 'HEAD') {