braid-text 0.2.22 → 0.2.24

package/index.js

@@ -334,7 +334,8 @@ braid_text.put = async (key, options) => {
         // make sure we have all these parents
         for (let p of options_parents) {
             let P = decode_version(p)
-            if (P[1] > (resource.actor_seqs[P[0]] ?? -1)) throw new Error(`${MISSING_PARENT_VERSION}: ${p}`)
+            if (!resource.actor_seqs[P[0]]?.has(P[1]))
+                throw new Error(`${MISSING_PARENT_VERSION}: ${p}`)
         }
     }
 
@@ -378,7 +379,7 @@ braid_text.put = async (key, options) => {
         max_pos))
 
     // validate version: make sure we haven't seen it already
-    if (v[1] <= (resource.actor_seqs[v[0]] ?? -1)) {
+    if (resource.actor_seqs[v[0]]?.has(v[1])) {
 
         if (!options.validate_already_seen_versions) return
 
@@ -439,7 +440,8 @@ braid_text.put = async (key, options) => {
         // we already have this version, so nothing left to do
         return
     }
-    resource.actor_seqs[v[0]] = v[1]
+    if (!resource.actor_seqs[v[0]]) resource.actor_seqs[v[0]] = new RangeSet()
+    resource.actor_seqs[v[0]].add_range(v[1] + 1 - change_count, v[1])
 
     // reduce the version sequence by the number of char-edits
     v = `${v[0]}-${v[1] + 1 - change_count}`
@@ -580,28 +582,6 @@ braid_text.put = async (key, options) => {
     await resource.db_delta(resource.doc.getPatchSince(v_before))
 }
 
-braid_text.revert = async (key, version) => {
-    var resource = (typeof key == 'string') ? await get_resource(key) : key
-
-    // revert dt
-    var old_doc = resource.doc
-    resource.doc = dt_get(resource.doc, null, null, version)
-    old_doc.free()
-
-    for (let v of version) {
-        var [actor, seq] = decode_version(v)
-        if ((resource.actor_seqs[actor] ?? -1) > seq - 1) {
-            if (seq > 0) resource.actor_seqs[actor] = seq - 1
-            else delete resource.actor_seqs[actor]
-        }
-    }
-
-    resource.val = resource.doc.get()
-
-    // save it
-    await resource.db_delta()
-}
-
 braid_text.list = async () => {
     try {
         if (braid_text.db_folder) {
@@ -645,7 +625,8 @@ async function get_resource(key) {
         let max_version = resource.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
         for (let i = 0; i <= max_version; i++) {
             let v = resource.doc.localToRemoteVersion([i])[0]
-            resource.actor_seqs[v[0]] = Math.max(v[1], resource.actor_seqs[v[0]] ?? -1)
+            if (!resource.actor_seqs[v[0]]) resource.actor_seqs[v[0]] = new RangeSet()
+            resource.actor_seqs[v[0]].add_range(v[1], v[1])
         }
 
         resource.val = resource.doc.get()
@@ -819,20 +800,8 @@ function dt_len(doc, version) {
 function dt_get_string(doc, version) {
     var bytes = doc.toBytes()
     var oplog = OpLog.fromBytes(bytes)
-    var [_agents, versions, _parentss] = dt_parse([...bytes])
-
-    var frontier = new Set(version)
 
-    var local_version = []
-    for (var i = 0; i < versions.length; i++) {
-        var v = versions[i].join("-")
-        if (frontier.has(v)) {
-            local_version.push(i)
-            frontier.delete(v)
-        }
-    }
-
-    if (frontier.size) throw new Error(`version not found: ${version}`)
+    var local_version = dt_get_local_version(bytes, version)
 
     var b = new Branch()
     b.merge(oplog, new Uint32Array(local_version))
@@ -1027,25 +996,25 @@ function dt_parse(byte_array) {
 
     while (byte_array.length) {
         let id = byte_array.shift()
-        let len = read_varint(byte_array)
+        let len = dt_read_varint(byte_array)
         if (id == 1) {
         } else if (id == 3) {
             let goal = byte_array.length - len
             while (byte_array.length > goal) {
-                agents.push(read_string(byte_array))
+                agents.push(dt_read_string(byte_array))
             }
         } else if (id == 20) {
         } else if (id == 21) {
             let seqs = {}
             let goal = byte_array.length - len
             while (byte_array.length > goal) {
-                let part0 = read_varint(byte_array)
+                let part0 = dt_read_varint(byte_array)
                 let has_jump = part0 & 1
                 let agent_i = (part0 >> 1) - 1
-                let run_length = read_varint(byte_array)
+                let run_length = dt_read_varint(byte_array)
                 let jump = 0
                 if (has_jump) {
-                    let part2 = read_varint(byte_array)
+                    let part2 = dt_read_varint(byte_array)
                     jump = part2 >> 1
                     if (part2 & 1) jump *= -1
                 }
@@ -1060,12 +1029,12 @@ function dt_parse(byte_array) {
             let count = 0
             let goal = byte_array.length - len
             while (byte_array.length > goal) {
-                let run_len = read_varint(byte_array)
+                let run_len = dt_read_varint(byte_array)
 
                 let parents = []
                 let has_more = 1
                 while (has_more) {
-                    let x = read_varint(byte_array)
+                    let x = dt_read_varint(byte_array)
                     let is_foreign = 0x1 & x
                     has_more = 0x2 & x
                     let num = x >> 2
@@ -1075,7 +1044,7 @@ function dt_parse(byte_array) {
                     } else if (!is_foreign) {
                         parents.push(versions[count - num])
                     } else {
-                        parents.push([agents[num - 1], read_varint(byte_array)])
+                        parents.push([agents[num - 1], dt_read_varint(byte_array)])
                     }
                 }
                 parentss.push(parents)
@@ -1091,24 +1060,107 @@ function dt_parse(byte_array) {
         }
     }
 
-    function read_string(byte_array) {
-        return new TextDecoder().decode(new Uint8Array(byte_array.splice(0, read_varint(byte_array))))
+    return [agents, versions, parentss]
+}
+
+function dt_get_local_version(bytes, version) {
+    var looking_for = new Map()
+    for (var event of version) {
+        var [agent, seq] = decode_version(event)
+        if (!looking_for.has(agent)) looking_for.set(agent, [])
+        looking_for.get(agent).push(seq)
     }
+    for (var seqs of looking_for.values())
+        seqs.sort((a, b) => a - b)
 
-    function read_varint(byte_array) {
-        let result = 0
-        let shift = 0
-        while (true) {
-            if (byte_array.length === 0) throw new Error("byte array does not contain varint")
+    var byte_array = [...bytes]
+    var local_version = []
+    var local_version_base = 0
+
+    if (new TextDecoder().decode(new Uint8Array(byte_array.splice(0, 8))) !== "DMNDTYPS") throw new Error("dt parse error, expected DMNDTYPS")
+
+    if (byte_array.shift() != 0) throw new Error("dt parse error, expected version 0")
+
+    let agents = []
+
+    while (byte_array.length && looking_for.size) {
+        let id = byte_array.shift()
+        let len = dt_read_varint(byte_array)
+        if (id == 1) {
+        } else if (id == 3) {
+            let goal = byte_array.length - len
+            while (byte_array.length > goal) {
+                agents.push(dt_read_string(byte_array))
+            }
+        } else if (id == 20) {
+        } else if (id == 21) {
+            let seqs = {}
+            let goal = byte_array.length - len
+            while (byte_array.length > goal && looking_for.size) {
+                let part0 = dt_read_varint(byte_array)
+                let has_jump = part0 & 1
+                let agent_i = (part0 >> 1) - 1
+                let run_length = dt_read_varint(byte_array)
+                let jump = 0
+                if (has_jump) {
+                    let part2 = dt_read_varint(byte_array)
+                    jump = part2 >> 1
+                    if (part2 & 1) jump *= -1
+                }
+                let base = (seqs[agent_i] || 0) + jump
+
+                var agent = agents[agent_i]
+                looking_for_seqs = looking_for.get(agent)
+                if (looking_for_seqs) {
+                    for (var seq of splice_out_range(
+                        looking_for_seqs, base, base + run_length - 1))
+                        local_version.push(local_version_base + (seq - base))
+                    if (!looking_for_seqs.length) looking_for.delete(agent)
+                }
+                local_version_base += run_length
+
+                seqs[agent_i] = base + run_length
+            }
+        } else {
+            byte_array.splice(0, len)
+        }
+    }
+
+    if (looking_for.size) throw new Error(`version not found: ${version}`)
+    return local_version
 
-            let byte_val = byte_array.shift()
-            result |= (byte_val & 0x7f) << shift
-            if ((byte_val & 0x80) == 0) return result
-            shift += 7
+    function splice_out_range(a, s, e) {
+        if (!a?.length) return [];
+        let l = 0, r = a.length;
+        while (l < r) {
+            const m = Math.floor((l + r) / 2);
+            if (a[m] < s) l = m + 1; else r = m;
         }
+        const i = l;
+        l = i; r = a.length;
+        while (l < r) {
+            const m = Math.floor((l + r) / 2);
+            if (a[m] <= e) l = m + 1; else r = m;
+        }
+        return a.splice(i, l - i);
     }
+}
 
-    return [agents, versions, parentss]
+function dt_read_string(byte_array) {
+    return new TextDecoder().decode(new Uint8Array(byte_array.splice(0, dt_read_varint(byte_array))))
+}
+
+function dt_read_varint(byte_array) {
+    let result = 0
+    let shift = 0
+    while (true) {
+        if (byte_array.length === 0) throw new Error("byte array does not contain varint")
+
+        let byte_val = byte_array.shift()
+        result |= (byte_val & 0x7f) << shift
+        if ((byte_val & 0x80) == 0) return result
+        shift += 7
+    }
 }
 
 function dt_create_bytes(version, parents, pos, del, ins) {
@@ -1772,6 +1824,52 @@ function apply_patch(obj, range, content) {
     }
 }
 
+class RangeSet {
+    constructor() {
+        this.ranges = []
+    }
+
+    add_range(low_inclusive, high_inclusive) {
+        if (low_inclusive > high_inclusive) return
+
+        const startIndex = this._bs(mid => this.ranges[mid][1] >= low_inclusive - 1, this.ranges.length, true)
+        const endIndex = this._bs(mid => this.ranges[mid][0] <= high_inclusive + 1, -1, false)
+
+        if (startIndex > endIndex) {
+            this.ranges.splice(startIndex, 0, [low_inclusive, high_inclusive])
+        } else {
+            const mergedLow = Math.min(low_inclusive, this.ranges[startIndex][0])
+            const mergedHigh = Math.max(high_inclusive, this.ranges[endIndex][1])
+            const removeCount = endIndex - startIndex + 1
+            this.ranges.splice(startIndex, removeCount, [mergedLow, mergedHigh])
+        }
+    }
+
+    has(x) {
+        var index = this._bs(mid => this.ranges[mid][0] <= x, -1, false)
+        return index !== -1 && x <= this.ranges[index][1]
+    }
+
+    _bs(condition, defaultR, moveLeft) {
+        let low = 0
+        let high = this.ranges.length - 1
+        let result = defaultR
+        
+        while (low <= high) {
+            const mid = Math.floor((low + high) / 2)
+            if (condition(mid)) {
+                result = mid
+                if (moveLeft) high = mid - 1
+                else low = mid + 1
+            } else {
+                if (moveLeft) low = mid + 1
+                else high = mid - 1
+            }
+        }
+        return result
+    }
+}
+
 braid_text.get_resource = get_resource
 
 braid_text.encode_filename = encode_filename
@@ -1781,8 +1879,10 @@ braid_text.get_files_for_key = get_files_for_key
 braid_text.dt_get = dt_get
 braid_text.dt_get_patches = dt_get_patches
 braid_text.dt_parse = dt_parse
+braid_text.dt_get_local_version = dt_get_local_version
 braid_text.dt_create_bytes = dt_create_bytes
 
 braid_text.decode_version = decode_version
+braid_text.RangeSet = RangeSet
 
 module.exports = braid_text

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-text",
-  "version": "0.2.22",
+  "version": "0.2.24",
   "description": "Library for collaborative text over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-text",

package/test/test.js

@@ -1,7 +1,7 @@
 
 let { Doc } = require("diamond-types-node")
 let braid_text = require('../index.js')
-let {dt_get, dt_get_patches, dt_parse, dt_create_bytes} = braid_text
+let {dt_get, dt_get_patches, dt_parse, dt_create_bytes, dt_get_local_version, RangeSet} = braid_text
 
 process.on("unhandledRejection", (x) =>
     console.log(`unhandledRejection: ${x.stack ?? x}`)
@@ -12,43 +12,138 @@ process.on("uncaughtException", (x) =>
 
 braid_text.db_folder = null
 
-async function test_revert() {
+async function test_db() {
     braid_text.db_folder = './braid-text-db'
     var key = Math.random().toString(36).slice(2)
     await braid_text.put(key, {version: ['a-0'], body: 'A'})
-    await braid_text.put(key, {version: ['a-1'], parents: ['a-0'], patches: [{range: '[1:1]', content: 'B'}]})
-    await braid_text.revert(key, ['a-1'])
+    await braid_text.put(key, {version: ['a-2'], parents: ['a-0'], patches: [{range: '[1:1]', content: 'B'}]})
     delete braid_text.cache[key]
     var {version, body} = await braid_text.get(key, {})
-    if (version[0] != 'a-0') throw new Error('revert error: wrong version')
-    if (body != 'A') throw new Error('revert error: wrong text')
 
-    await braid_text.put(key, {version: ['b-0'], parents: ['a-0'], patches: [{range: '[1:1]', content: 'C'}]})
-    if ('AC' !== await braid_text.get(key)) throw new Error('revert error: wrong text')
-    await braid_text.revert(key, ['b-0'])
-    if ('A' !== await braid_text.get(key)) throw new Error('revert error: wrong text')
-    delete braid_text.cache[key]
-    var {version, body} = await braid_text.get(key, {})
-    if (version[0] != 'a-0') throw new Error('revert error: wrong version')
-    if (body != 'A') throw new Error('revert error: wrong text')
+    if (body != 'AB') throw new Error('db error')
 
     braid_text.db_folder = null
 }
 
+async function test_dt_get_local_version() {
+    for (var tt = 0; tt < 100; tt++) {
+        var st = Date.now()
+
+        var doc = new Doc('x')
+        var n = 100
+        var v_array = []
+        var p_array = []
+        var agents = ['hi']
+        var agent_rangesets = {}
+
+        for (var i = 0; i < n; i++) {
+            // console.log(`i = ${i}`)
+
+            // Math.randomSeed('seed::::' + i)
+
+            var agent = agents[0]
+
+            var seq = null
+            if (!agent_rangesets[agent]) agent_rangesets[agent] = new RangeSet()
+            var ranges = agent_rangesets[agent].ranges
+            if (!ranges.length) {
+                seq = Math.floor(Math.random() * 10)
+            } else {
+                var ii
+                if (ranges[0]?.[0] === 0)
+                    ii = Math.floor(Math.random() * ranges.length)
+                else
+                    ii = Math.floor(Math.random() * (ranges.length + 1)) - 1
+                var low = ii < 0 ? 0 : ranges[ii][1] + 1
+                var high = ii + 1 < ranges.length ? ranges[ii + 1][0] - 1 : ranges[ii][1] + 10
+                seq = Math.random() < 0.8 ? low : low + Math.floor(Math.random() * (high - low + 1))
+            }
+            agent_rangesets[agent].add_range(seq, seq)
+            v_array.push(`${agent}-${seq}`)
+
+            var parents = []
+            var shadow = new Set()
+            if (p_array.length) {
+                var starting_point = Math.floor(Math.random() * p_array.length)
+                parents.push(starting_point)
+                for (var p of p_array[starting_point]) shadow.add(p)
+                for (var ii = starting_point - 1; ii >= 0 && Math.random() < 0.9; ii--) {
+                    if (!shadow.has(ii) && Math.random() < 0.5) {
+                        parents.push(ii)
+                        shadow.add(ii)
+                    }
+                    if (shadow.has(ii))
+                        for (var p of p_array[ii])
+                            shadow.add(p)
+                }
+            }
+            p_array.push(parents)
+            parents = parents.map(p => v_array[p])
+        
+            let args = [`${agent}-${seq}`, parents, 0, 0, 'x']
+            // console.log(args)
+
+            doc.mergeBytes(dt_create_bytes(...args))
+        }
+
+        var bytes = doc.toBytes()
+
+        for (var t = 0; t < 100; t++) {
+            // if (t % 10 === 0) console.log(`  t = ${t}`)
+
+            var version = []
+            var vv_array = [...v_array]
+            while (!version.length || (vv_array.length && Math.random() < 0.9)) {
+                version.push(vv_array.splice(Math.floor(Math.random() * vv_array.length), 1)[0])
+            }
+
+            var local_new = dt_get_local_version(bytes, version)
+            var local_old = old_dt_get_local_version(bytes, version)
+
+            if (local_new.some((x, i) => x != local_old[i])) {
+                throw new Error('bad!')
+            }
+        }
+
+        console.log(`[tt=${tt}] total T = ${Date.now() - st}`)
+    }
+}
+
+function old_dt_get_local_version(bytes, version) {
+    var [_agents, versions, _parentss] = dt_parse([...bytes])
+
+    var frontier = new Set(version)
+
+    var local_version = []
+    for (var i = 0; i < versions.length; i++) {
+        var v = versions[i].join("-")
+        if (frontier.has(v)) {
+            local_version.push(i)
+            frontier.delete(v)
+        }
+    }
+
+    if (frontier.size) throw new Error(`version not found: ${version}`)
+
+    return local_version
+}
+
 async function main() {
     let best_seed = NaN
     let best_n = Infinity
     let base = Math.floor(Math.random() * 10000000)
     let st = Date.now()
 
-    await test_revert()
+    await test_dt_get_local_version()
+
+    await test_db()
 
     let og_log = console.log
     console.log = () => {}
     for (let t = 0; t < 10000; t++) {
         let seed = base + t
     // for (let t = 0; t < 1; t++) {
-        // let seed = 2746153
+    //     let seed = 403279
 
         og_log(`t = ${t}, seed = ${seed}, best_n = ${best_n} @ ${best_seed}`)
         Math.randomSeed(seed)
@@ -96,14 +191,6 @@ async function main() {
                     await braid_text.put(key, y)
                     y.validate_already_seen_versions = true
                     await braid_text.put(key, y)
-
-                    // test revert
-                    var range = x.range.match(/\d+/g).map((x) => parseInt(x))
-                    var change_count = [...x.content].length + range[1] - range[0]
-                    await braid_text.revert(key, [`${actor}-${seq + 1 - change_count}`])
-                    var new_text = await braid_text.get(key)
-                    if (old_text !== new_text) throw new Error('revert failed!')
-                    await braid_text.put(key, y)
                 }
             }
             await dt_to_braid(doc, 'doc')
@@ -134,16 +221,7 @@ async function main() {
                 await braid_text.get('middle_doc', {peer: 'sim', subscribe: async update => {
                     if (first_time) {
                         first_time = false
-                        if (update.body !== await braid_text.get('middle_doc')) fail(new Error('test 2'))
-
-                        var x = await braid_text.get('middle_doc', {})
-                        await braid_text.put('middle_doc', {peer: 'sim', version: ['sim-0'], parents: x.version, patches: [{content: 'A', range: '[0:0]'}]})
-                        await braid_text.put('middle_doc', {peer: 'other', merge_type: 'dt', version: ['other-0'], parents: [], patches: [{content: 'B', range: '[0:0]'}]})
-                        await braid_text.put('middle_doc', {peer: 'sim', version: ['sim-1'], parents: ['sim-0'], patches: [{content: 'b', range: '[0:0]'}]})
-                        await braid_text.put('middle_doc', {peer: 'sim', version: ['sim-2'], parents: ['sim-1'], patches: [{content: 'c', range: '[0:0]'}]})
-                        await braid_text.revert('middle_doc', ['sim-0'])
-                        await braid_text.revert('middle_doc', ['other-0'])
-
+                        if (update.body !== await braid_text.get('middle_doc')) fail(new Error('simpleton fail'))
                         done()
                     }
                 }})