braid-text 0.2.23 → 0.2.25

package/index.js

@@ -76,6 +76,16 @@ braid_text.serve = async (req, res, options = {}) => {
         if (!req.subscribe) {
             res.setHeader("Accept-Subscribe", "true")
 
+            // special case for HEAD asking for version/parents,
+            // to be faster by not reconstructing body
+            if (req.method === "HEAD" && (req.version || req.parents)) {
+                if ((req.version || req.parents).every(event => {
+                    var [actor, seq] = decode_version(event)
+                    return resource.actor_seqs[actor]?.has(seq)
+                })) return my_end(200)
+                else return my_end(500, "Unknown Version")
+            }
+
             let x = null
             try {
                 x = await braid_text.get(resource, { version: req.version, parents: req.parents })
@@ -800,20 +810,8 @@ function dt_len(doc, version) {
 function dt_get_string(doc, version) {
     var bytes = doc.toBytes()
     var oplog = OpLog.fromBytes(bytes)
-    var [_agents, versions, _parentss] = dt_parse([...bytes])
-
-    var frontier = new Set(version)
-
-    var local_version = []
-    for (var i = 0; i < versions.length; i++) {
-        var v = versions[i].join("-")
-        if (frontier.has(v)) {
-            local_version.push(i)
-            frontier.delete(v)
-        }
-    }
 
-    if (frontier.size) throw new Error(`version not found: ${version}`)
+    var local_version = dt_get_local_version(bytes, version)
 
     var b = new Branch()
     b.merge(oplog, new Uint32Array(local_version))
@@ -1008,25 +1006,25 @@ function dt_parse(byte_array) {
 
     while (byte_array.length) {
         let id = byte_array.shift()
-        let len = read_varint(byte_array)
+        let len = dt_read_varint(byte_array)
         if (id == 1) {
         } else if (id == 3) {
             let goal = byte_array.length - len
             while (byte_array.length > goal) {
-                agents.push(read_string(byte_array))
+                agents.push(dt_read_string(byte_array))
             }
         } else if (id == 20) {
         } else if (id == 21) {
             let seqs = {}
             let goal = byte_array.length - len
             while (byte_array.length > goal) {
-                let part0 = read_varint(byte_array)
+                let part0 = dt_read_varint(byte_array)
                 let has_jump = part0 & 1
                 let agent_i = (part0 >> 1) - 1
-                let run_length = read_varint(byte_array)
+                let run_length = dt_read_varint(byte_array)
                 let jump = 0
                 if (has_jump) {
-                    let part2 = read_varint(byte_array)
+                    let part2 = dt_read_varint(byte_array)
                     jump = part2 >> 1
                     if (part2 & 1) jump *= -1
                 }
@@ -1041,12 +1039,12 @@ function dt_parse(byte_array) {
             let count = 0
             let goal = byte_array.length - len
             while (byte_array.length > goal) {
-                let run_len = read_varint(byte_array)
+                let run_len = dt_read_varint(byte_array)
 
                 let parents = []
                 let has_more = 1
                 while (has_more) {
-                    let x = read_varint(byte_array)
+                    let x = dt_read_varint(byte_array)
                     let is_foreign = 0x1 & x
                     has_more = 0x2 & x
                     let num = x >> 2
@@ -1056,7 +1054,7 @@ function dt_parse(byte_array) {
                     } else if (!is_foreign) {
                         parents.push(versions[count - num])
                     } else {
-                        parents.push([agents[num - 1], read_varint(byte_array)])
+                        parents.push([agents[num - 1], dt_read_varint(byte_array)])
                     }
                 }
                 parentss.push(parents)
@@ -1072,24 +1070,107 @@ function dt_parse(byte_array) {
         }
     }
 
-    function read_string(byte_array) {
-        return new TextDecoder().decode(new Uint8Array(byte_array.splice(0, read_varint(byte_array))))
+    return [agents, versions, parentss]
+}
+
+function dt_get_local_version(bytes, version) {
+    var looking_for = new Map()
+    for (var event of version) {
+        var [agent, seq] = decode_version(event)
+        if (!looking_for.has(agent)) looking_for.set(agent, [])
+        looking_for.get(agent).push(seq)
     }
+    for (var seqs of looking_for.values())
+        seqs.sort((a, b) => a - b)
 
-    function read_varint(byte_array) {
-        let result = 0
-        let shift = 0
-        while (true) {
-            if (byte_array.length === 0) throw new Error("byte array does not contain varint")
+    var byte_array = [...bytes]
+    var local_version = []
+    var local_version_base = 0
+
+    if (new TextDecoder().decode(new Uint8Array(byte_array.splice(0, 8))) !== "DMNDTYPS") throw new Error("dt parse error, expected DMNDTYPS")
+
+    if (byte_array.shift() != 0) throw new Error("dt parse error, expected version 0")
+
+    let agents = []
+
+    while (byte_array.length && looking_for.size) {
+        let id = byte_array.shift()
+        let len = dt_read_varint(byte_array)
+        if (id == 1) {
+        } else if (id == 3) {
+            let goal = byte_array.length - len
+            while (byte_array.length > goal) {
+                agents.push(dt_read_string(byte_array))
+            }
+        } else if (id == 20) {
+        } else if (id == 21) {
+            let seqs = {}
+            let goal = byte_array.length - len
+            while (byte_array.length > goal && looking_for.size) {
+                let part0 = dt_read_varint(byte_array)
+                let has_jump = part0 & 1
+                let agent_i = (part0 >> 1) - 1
+                let run_length = dt_read_varint(byte_array)
+                let jump = 0
+                if (has_jump) {
+                    let part2 = dt_read_varint(byte_array)
+                    jump = part2 >> 1
+                    if (part2 & 1) jump *= -1
+                }
+                let base = (seqs[agent_i] || 0) + jump
+
+                var agent = agents[agent_i]
+                looking_for_seqs = looking_for.get(agent)
+                if (looking_for_seqs) {
+                    for (var seq of splice_out_range(
+                        looking_for_seqs, base, base + run_length - 1))
+                        local_version.push(local_version_base + (seq - base))
+                    if (!looking_for_seqs.length) looking_for.delete(agent)
+                }
+                local_version_base += run_length
+
+                seqs[agent_i] = base + run_length
+            }
+        } else {
+            byte_array.splice(0, len)
+        }
+    }
+
+    if (looking_for.size) throw new Error(`version not found: ${version}`)
+    return local_version
 
-            let byte_val = byte_array.shift()
-            result |= (byte_val & 0x7f) << shift
-            if ((byte_val & 0x80) == 0) return result
-            shift += 7
+    function splice_out_range(a, s, e) {
+        if (!a?.length) return [];
+        let l = 0, r = a.length;
+        while (l < r) {
+            const m = Math.floor((l + r) / 2);
+            if (a[m] < s) l = m + 1; else r = m;
+        }
+        const i = l;
+        l = i; r = a.length;
+        while (l < r) {
+            const m = Math.floor((l + r) / 2);
+            if (a[m] <= e) l = m + 1; else r = m;
         }
+        return a.splice(i, l - i);
     }
+}
 
-    return [agents, versions, parentss]
+function dt_read_string(byte_array) {
+    return new TextDecoder().decode(new Uint8Array(byte_array.splice(0, dt_read_varint(byte_array))))
+}
+
+function dt_read_varint(byte_array) {
+    let result = 0
+    let shift = 0
+    while (true) {
+        if (byte_array.length === 0) throw new Error("byte array does not contain varint")
+
+        let byte_val = byte_array.shift()
+        result |= (byte_val & 0x7f) << shift
+        if ((byte_val & 0x80) == 0) return result
+        shift += 7
+    }
 }
 
 function dt_create_bytes(version, parents, pos, del, ins) {
@@ -1808,8 +1889,10 @@ braid_text.get_files_for_key = get_files_for_key
 braid_text.dt_get = dt_get
 braid_text.dt_get_patches = dt_get_patches
 braid_text.dt_parse = dt_parse
+braid_text.dt_get_local_version = dt_get_local_version
 braid_text.dt_create_bytes = dt_create_bytes
 
 braid_text.decode_version = decode_version
+braid_text.RangeSet = RangeSet
 
 module.exports = braid_text

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-text",
-  "version": "0.2.23",
+  "version": "0.2.25",
   "description": "Library for collaborative text over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-text",

package/test/test.html

@@ -266,4 +266,34 @@ runTest(
     'retried!'
 )
 
+runTest(
+    "test asking for a version that should and shouldn't be there",
+    async () => {
+        var key = 'test-' + Math.random().toString(36).slice(2)
+
+        var r = await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            version: ['hi-10'],
+            parents: [],
+            body: 'x'
+        })
+        if (!r.ok) throw 'got: ' + r.statusCode
+
+        var r = await braid_fetch(`/${key}`, {
+            method: 'HEAD',
+            version: ['hi-5']
+        })
+        if (r.ok) throw 'found version we should not have found'
+
+        var r = await braid_fetch(`/${key}`, {
+            method: 'HEAD',
+            version: ['hi-10']
+        })
+        if (!r.ok) throw 'could not find version we should have found'
+
+        return 'worked out!'
+    },
+    'worked out!'
+)
+
 </script>

package/test/test.js

@@ -1,7 +1,7 @@
 
 let { Doc } = require("diamond-types-node")
 let braid_text = require('../index.js')
-let {dt_get, dt_get_patches, dt_parse, dt_create_bytes} = braid_text
+let {dt_get, dt_get_patches, dt_parse, dt_create_bytes, dt_get_local_version, RangeSet} = braid_text
 
 process.on("unhandledRejection", (x) =>
     console.log(`unhandledRejection: ${x.stack ?? x}`)
@@ -25,12 +25,117 @@ async function test_db() {
     braid_text.db_folder = null
 }
 
+async function test_dt_get_local_version() {
+    for (var tt = 0; tt < 100; tt++) {
+        var st = Date.now()
+
+        var doc = new Doc('x')
+        var n = 100
+        var v_array = []
+        var p_array = []
+        var agents = ['hi']
+        var agent_rangesets = {}
+
+        for (var i = 0; i < n; i++) {
+            // console.log(`i = ${i}`)
+
+            // Math.randomSeed('seed::::' + i)
+
+            var agent = agents[0]
+
+            var seq = null
+            if (!agent_rangesets[agent]) agent_rangesets[agent] = new RangeSet()
+            var ranges = agent_rangesets[agent].ranges
+            if (!ranges.length) {
+                seq = Math.floor(Math.random() * 10)
+            } else {
+                var ii
+                if (ranges[0]?.[0] === 0)
+                    ii = Math.floor(Math.random() * ranges.length)
+                else
+                    ii = Math.floor(Math.random() * (ranges.length + 1)) - 1
+                var low = ii < 0 ? 0 : ranges[ii][1] + 1
+                var high = ii + 1 < ranges.length ? ranges[ii + 1][0] - 1 : ranges[ii][1] + 10
+                seq = Math.random() < 0.8 ? low : low + Math.floor(Math.random() * (high - low + 1))
+            }
+            agent_rangesets[agent].add_range(seq, seq)
+            v_array.push(`${agent}-${seq}`)
+
+            var parents = []
+            var shadow = new Set()
+            if (p_array.length) {
+                var starting_point = Math.floor(Math.random() * p_array.length)
+                parents.push(starting_point)
+                for (var p of p_array[starting_point]) shadow.add(p)
+                for (var ii = starting_point - 1; ii >= 0 && Math.random() < 0.9; ii--) {
+                    if (!shadow.has(ii) && Math.random() < 0.5) {
+                        parents.push(ii)
+                        shadow.add(ii)
+                    }
+                    if (shadow.has(ii))
+                        for (var p of p_array[ii])
+                            shadow.add(p)
+                }
+            }
+            p_array.push(parents)
+            parents = parents.map(p => v_array[p])
+        
+            let args = [`${agent}-${seq}`, parents, 0, 0, 'x']
+            // console.log(args)
+
+            doc.mergeBytes(dt_create_bytes(...args))
+        }
+
+        var bytes = doc.toBytes()
+
+        for (var t = 0; t < 100; t++) {
+            // if (t % 10 === 0) console.log(`  t = ${t}`)
+
+            var version = []
+            var vv_array = [...v_array]
+            while (!version.length || (vv_array.length && Math.random() < 0.9)) {
+                version.push(vv_array.splice(Math.floor(Math.random() * vv_array.length), 1)[0])
+            }
+
+            var local_new = dt_get_local_version(bytes, version)
+            var local_old = old_dt_get_local_version(bytes, version)
+
+            if (local_new.some((x, i) => x != local_old[i])) {
+                throw new Error('bad!')
+            }
+        }
+
+        console.log(`[tt=${tt}] total T = ${Date.now() - st}`)
+    }
+}
+
+function old_dt_get_local_version(bytes, version) {
+    var [_agents, versions, _parentss] = dt_parse([...bytes])
+
+    var frontier = new Set(version)
+
+    var local_version = []
+    for (var i = 0; i < versions.length; i++) {
+        var v = versions[i].join("-")
+        if (frontier.has(v)) {
+            local_version.push(i)
+            frontier.delete(v)
+        }
+    }
+
+    if (frontier.size) throw new Error(`version not found: ${version}`)
+
+    return local_version
+}
+
 async function main() {
     let best_seed = NaN
     let best_n = Infinity
     let base = Math.floor(Math.random() * 10000000)
     let st = Date.now()
 
+    await test_dt_get_local_version()
+
     await test_db()
 
     let og_log = console.log
@@ -38,7 +143,7 @@ async function main() {
     for (let t = 0; t < 10000; t++) {
         let seed = base + t
     // for (let t = 0; t < 1; t++) {
-        // let seed = 2746153
+    //     let seed = 403279
 
         og_log(`t = ${t}, seed = ${seed}, best_n = ${best_n} @ ${best_seed}`)
         Math.randomSeed(seed)