braid-text 0.1.2 → 0.1.3

package/index.js

@@ -220,10 +220,14 @@ braid_text.get = async (key, options) => {
         let doc = resource.doc
         if (options.version || options.parents) doc = dt_get(doc, options.version || options.parents)
 
-        return {
+        let ret = {
             version: doc.getRemoteVersion().map((x) => x.join("-")).sort(),
             body: doc.get()
         }
+
+        if (options.version || options.parents) doc.free()
+
+        return ret
     } else {
         if (options.merge_type != "dt") {
             let version = resource.doc.getRemoteVersion().map((x) => x.join("-")).sort()
@@ -269,17 +273,12 @@ braid_text.get = async (key, options) => {
                 updates = dt_get_patches(resource.doc, options.parents || options.version)
             }
 
-            for (let u of updates) {
-                u.version = decode_version(u.version)
-                u.version[1] += u.end - u.start - 1
-                u.version = u.version.join("-")
-
+            for (let u of updates)
                 options.subscribe({
                     version: [u.version],
                     parents: u.parents,
                     patches: [{ unit: u.unit, range: u.range, content: u.content }],
                 })
-            }
 
             // Output at least *some* data, or else chrome gets confused and
             // thinks the connection failed.  This isn't strictly necessary,
@@ -292,6 +291,16 @@ braid_text.get = async (key, options) => {
     }
 }
 
+braid_text.forget = async (key, options) => {
+    if (!options) throw new Error('options is required')
+
+    let resource = (typeof key == 'string') ? await get_resource(key) : key
+
+    if (options.merge_type != "dt")
+        resource.simpleton_clients.delete(options)
+    else resource.clients.delete(options)
+}
+
 braid_text.put = async (key, options) => {
     let { version, patches, body, peer } = options
 
@@ -320,9 +329,15 @@ braid_text.put = async (key, options) => {
     let parents = resource.doc.getRemoteVersion().map((x) => x.join("-")).sort()
     let og_parents = options_parents || parents
 
+    function get_len() {
+        let d = dt_get(resource.doc, og_parents)
+        let len = d.len()
+        d.free()
+        return len
+    }
+
     let max_pos = resource.length_cache.get('' + og_parents) ??
-        (v_eq(parents, og_parents) ? resource.doc.len() :
-            dt_get(resource.doc, og_parents).len())
+        (v_eq(parents, og_parents) ? resource.doc.len() : get_len())
     
     if (body != null) {
         patches = [{
@@ -336,7 +351,7 @@ braid_text.put = async (key, options) => {
     patches = patches.map((p) => ({
         ...p,
         range: p.range.match(/\d+/g).map((x) => parseInt(x)),
-        content: [...p.content],
+        content_codepoints: [...p.content],
     })).sort((a, b) => a.range[0] - b.range[0])
 
     // validate patch positions
@@ -347,7 +362,7 @@ braid_text.put = async (key, options) => {
         must_be_at_least = p.range[1]
     }
 
-    let change_count = patches.reduce((a, b) => a + b.content.length + (b.range[1] - b.range[0]), 0)
+    let change_count = patches.reduce((a, b) => a + b.content_codepoints.length + (b.range[1] - b.range[0]), 0)
 
     let og_v = version?.[0] || `${(is_valid_actor(peer) && peer) || Math.random().toString(36).slice(2, 7)}-${change_count - 1}`
 
@@ -365,7 +380,6 @@ braid_text.put = async (key, options) => {
         let seen = {}
         for (let u of updates) {
             u.version = decode_version(u.version)
-            u.version[1] += u.end - u.start - 1
 
             if (!u.content) {
                 // delete
@@ -401,9 +415,9 @@ braid_text.put = async (key, options) => {
                 v = `${v[0]}-${v[1] + 1}`
             }
             // insert
-            for (let i = 0; i < p.content?.length ?? 0; i++) {
+            for (let i = 0; i < p.content_codepoints?.length ?? 0; i++) {
                 let vv = decode_version(v)
-                let c = p.content[i]
+                let c = p.content_codepoints[i]
 
                 if (!seen[JSON.stringify([vv[0], vv[1], ps, p.range[1] + offset, c])]) throw new Error('invalid update: different from previous update with same version')
 
@@ -420,7 +434,7 @@ braid_text.put = async (key, options) => {
     resource.actor_seqs[v[0]] = v[1]
 
     resource.length_cache.put(`${v[0]}-${v[1]}`, patches.reduce((a, b) =>
-        a + (b.content.length ? b.content.length : -(b.range[1] - b.range[0])),
+        a + (b.content_codepoints.length ? b.content_codepoints.length : -(b.range[1] - b.range[0])),
         max_pos))
 
     v = `${v[0]}-${v[1] + 1 - change_count}`
@@ -434,21 +448,21 @@ braid_text.put = async (key, options) => {
     let offset = 0
     for (let p of patches) {
         // delete
-        for (let i = p.range[0]; i < p.range[1]; i++) {
-            bytes.push(dt_create_bytes(v, ps, p.range[1] - 1 + offset, null))
-            offset--
-            ps = [v]
+        let del = p.range[1] - p.range[0]
+        if (del) {
+            bytes.push(dt_create_bytes(v, ps, p.range[0] + offset, del, null))
+            offset -= del
             v = decode_version(v)
-            v = `${v[0]}-${v[1] + 1}`
+            ps = [`${v[0]}-${v[1] + (del - 1)}`]
+            v = `${v[0]}-${v[1] + del}`
         }
         // insert
-        for (let i = 0; i < p.content?.length ?? 0; i++) {
-            let c = p.content[i]
-            bytes.push(dt_create_bytes(v, ps, p.range[1] + offset, c))
-            offset++
-            ps = [v]
+        if (p.content?.length) {
+            bytes.push(dt_create_bytes(v, ps, p.range[1] + offset, 0, p.content))
+            offset += p.content_codepoints.length
             v = decode_version(v)
-            v = `${v[0]}-${v[1] + 1}`
+            ps = [`${v[0]}-${v[1] + (p.content_codepoints.length - 1)}`]
+            v = `${v[0]}-${v[1] + p.content_codepoints.length}`
         }
     }
 
@@ -787,6 +801,8 @@ async function file_sync(key, process_delta, get_init) {
 //////////////////////////////////////////////////////////////////
 
 function dt_get(doc, version, agent = null) {
+    if (dt_get.last_doc) dt_get.last_doc.free()
+
     let bytes = doc.toBytes()
     dt_get.last_doc = doc = Doc.fromBytes(bytes, agent)
 
@@ -834,6 +850,7 @@ function dt_get(doc, version, agent = null) {
                                     op_run.start + (i - base_i) :
                                     op_run.start) :
                                 op_run.end - 1 - (i - base_i),
+                            op_run.content?.[i - base_i] != null ? 0 : 1,
                             op_run.content?.[i - base_i]
                         )
                     )
@@ -866,12 +883,16 @@ function dt_get_patches(doc, version = null) {
 
         before_doc.mergeBytes(after_bytes)
         op_runs = before_doc.getOpsSince(before_doc_frontier)
+
+        before_doc.free()
     } else op_runs = doc.getOpsSince([])
 
+    doc.free()
+
     let i = 0
     let patches = []
     op_runs.forEach((op_run) => {
-        let version = versions[i].join("-")
+        let version = versions[i]
         let parents = parentss[i].map((x) => x.join("-")).sort()
         let start = op_run.start
         let end = start + 1
@@ -899,7 +920,7 @@ function dt_get_patches(doc, version = null) {
                         op_run.start + (end - start)) :
                     (op_run.end - (start - op_run.start))
                 patches.push({
-                    version,
+                    version: `${version[0]}-${version[1] + e - s - 1}`,
                     parents,
                     unit: "text",
                     range: op_run.content ? `[${s}:${s}]` : `[${s}:${e}]`,
@@ -908,7 +929,7 @@ function dt_get_patches(doc, version = null) {
                     end: e,
                 })
                 if (j == len) break
-                version = versions[I].join("-")
+                version = versions[I]
                 parents = parentss[I].map((x) => x.join("-")).sort()
                 start = op_run.start + j
             }
@@ -1014,7 +1035,8 @@ function dt_parse(byte_array) {
     return [agents, versions, parentss]
 }
 
-function dt_create_bytes(version, parents, pos, ins) {
+function dt_create_bytes(version, parents, pos, del, ins) {
+    if (del) pos += del - 1
 
     function write_varint(bytes, value) {
         while (value >= 0x80) {
@@ -1085,6 +1107,8 @@ function dt_create_bytes(version, parents, pos, ins) {
 
     let patches = []
 
+    let unicode_chars = ins ? [...ins] : []
+
     if (ins) {
         let inserted_content_bytes = []
 
@@ -1095,18 +1119,21 @@ function dt_create_bytes(version, parents, pos, ins) {
         let encoder = new TextEncoder()
         let utf8Bytes = encoder.encode(ins)
 
-        inserted_content_bytes.push(1 + utf8Bytes.length) // length of content chunk
+        write_varint(inserted_content_bytes, 1 + utf8Bytes.length)
+        // inserted_content_bytes.push(1 + utf8Bytes.length) // length of content chunk
         inserted_content_bytes.push(4) // "plain text" enum
 
         for (let b of utf8Bytes) inserted_content_bytes.push(b) // actual text
 
         inserted_content_bytes.push(25) // "known" enum
-        inserted_content_bytes.push(1) // length of "known" chunk
-        inserted_content_bytes.push(3) // content of length 1, and we "know" it
+        let known_chunk = []
+        write_varint(known_chunk, unicode_chars.length * 2 + 1)
+        write_varint(inserted_content_bytes, known_chunk.length)
+        inserted_content_bytes.push(...known_chunk)
 
         patches.push(24)
         write_varint(patches, inserted_content_bytes.length)
-        patches.push(...inserted_content_bytes)
+        for (let b of inserted_content_bytes) patches.push(b)
     }
 
     // write in the version
@@ -1117,26 +1144,43 @@ function dt_create_bytes(version, parents, pos, ins) {
     let jump = seq
 
     write_varint(version_bytes, ((agent_i + 1) << 1) | (jump != 0 ? 1 : 0))
-    write_varint(version_bytes, 1)
+    write_varint(version_bytes, ins ? unicode_chars.length : del)
     if (jump) write_varint(version_bytes, jump << 1)
 
     patches.push(21)
     write_varint(patches, version_bytes.length)
-    patches.push(...version_bytes)
+    for (let b of version_bytes) patches.push(b)
 
     // write in "op" bytes (some encoding of position)
     let op_bytes = []
 
-    write_varint(op_bytes, (pos << 4) | (pos ? 2 : 0) | (ins ? 0 : 4))
+    if (del) {
+        if (pos == 0) {
+            write_varint(op_bytes, 4)
+        } else if (del == 1) {
+            write_varint(op_bytes, pos * 16 + 6)
+        } else {
+            write_varint(op_bytes, del * 16 + 7)
+            write_varint(op_bytes, pos * 2 + 2)
+        }
+    } else if (unicode_chars.length == 1) {
+        if (pos == 0) write_varint(op_bytes, 0)
+        else write_varint(op_bytes, pos * 16 + 2)
+    } else if (pos == 0) {
+        write_varint(op_bytes, unicode_chars.length * 8 + 1)
+    } else {
+        write_varint(op_bytes, unicode_chars.length * 8 + 3)
+        write_varint(op_bytes, pos * 2)
+    }
 
     patches.push(22)
     write_varint(patches, op_bytes.length)
-    patches.push(...op_bytes)
+    for (let b of op_bytes) patches.push(b)
 
     // write in parents
     let parents_bytes = []
 
-    write_varint(parents_bytes, 1)
+    write_varint(parents_bytes, ins ? unicode_chars.length : del)
 
     if (parents.length) {
         for (let [i, [agent, seq]] of parents.entries()) {
@@ -1154,14 +1198,16 @@ function dt_create_bytes(version, parents, pos, ins) {
     // write in patches
     bytes.push(20)
     write_varint(bytes, patches.length)
-    bytes.push(...patches)
+    for (let b of patches) bytes.push(b)
 
     //   console.log(bytes);
     return bytes
 }
 
 function defrag_dt(doc) {
-    return Doc.fromBytes(doc.toBytes(), 'server')
+    let bytes = doc.toBytes()
+    doc.free()
+    return Doc.fromBytes(bytes, 'server')
 }
 
 function OpLog_remote_to_local(doc, frontier) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-text",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Library for collaborative text over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braidjs",

package/test.js

@@ -1,9 +1,19 @@
 
 let { Doc } = require("diamond-types-node")
-let {dt_get, dt_get_patches, dt_parse, dt_create_bytes} = require('./index.js')
+let braid_text = require('./index.js')
+let {dt_get, dt_get_patches, dt_parse, dt_create_bytes} = braid_text
+
+process.on("unhandledRejection", (x) =>
+    console.log(`unhandledRejection: ${x.stack}`)
+)
+process.on("uncaughtException", (x) =>
+    console.log(`uncaughtException: ${x.stack}`)
+)
+
+braid_text.db_folder = null
 
 async function main() {
-    let best_seed = 0
+    let best_seed = NaN
     let best_n = Infinity
     let base = Math.floor(Math.random() * 10000000)
 
@@ -11,89 +21,89 @@ async function main() {
     console.log = () => {}
     for (let t = 0; t < 10000; t++) {
         let seed = base + t
-    // for (let t = 0; t < 10; t++) {
-    //     let seed = 1188661 + t
+    // for (let t = 0; t < 1; t++) {
+    //     let seed = 7572861
 
         og_log(`t = ${t}, seed = ${seed}, best_n = ${best_n} @ ${best_seed}`)
         Math.randomSeed(seed)
 
-        let n = Math.floor(Math.random() * 15)
+        let n = Math.floor(Math.random() * 15) + 5
         console.log(`n = ${n}`)
 
         try {
-            // 1. create a bunch of edits to a dt
+            // create a bunch of edits called doc,
+            // and remember a point along the way of adding all these edits,
+            // called middle_doc
             let doc = new Doc('server')
-
             let middle_doc = null
 
-            if (!middle_doc && (Math.random() < 1/n || n == 0)) {
+            if (!middle_doc && (Math.random() < 1/n || n == 0))
                 middle_doc = Doc.fromBytes(doc.toBytes())
-            }
             for (let i = 0; i < n; i++) {
-                make_random_edit(doc)
+                console.log(`edit ${i}`)
 
-                if (!middle_doc && (Math.random() < 1/n || i == n - 1)) {
+                make_random_edit(doc)
+                if (!middle_doc && (Math.random() < 1/n || i == n - 1))
                     middle_doc = Doc.fromBytes(doc.toBytes())
-                }
-            }
-            if (!middle_doc) throw 'bad'
-
-            // 2. let x = the resulting string
-            let x = doc.get()
-            console.log('x = ' + x)
-
-            // // 3. use the code for sending these edits over the wire to create a new dt
-            let updates = dt_get_patches(doc)
-            console.log(updates)
-
-            let new_doc = new Doc('server')
-            apply_updates(new_doc, updates)
-            let y = new_doc.get()
-            console.log('y = ' + y)
-
-            // 4. is the resulting string == x?
-            console.log(x == y)
-            if (x != y && n < best_n) {
-                best_n = n
-                best_seed = seed
-            }
-
-            // 5. test dt_get
-            let middle_v = middle_doc.getRemoteVersion().map(x => x.join('-'))
-            let new_middle_doc = dt_get(doc, middle_v)
-            console.log('new_middle_doc = ' + new_middle_doc.get())
-            if (middle_doc.get() != new_middle_doc.get() && n < best_n) {
-                best_n = n
-                best_seed = seed
             }
-
-            // 6. test dt_get_patches(doc, version)
-            if (true) {
-                let updates = dt_get_patches(doc, middle_v)
-                console.log(`updates:`, updates)
-
-                apply_updates(middle_doc, updates)
-                console.log(`middle_doc2:${middle_doc.get()}`)
-                if (middle_doc.get() != doc.get() && n < best_n) {
-                    best_n = n
-                    best_seed = seed
+            if (!middle_doc) throw new Error('bad')
+
+            // put them into braid-text
+            let dt_to_braid = async (doc, key) => {
+                await braid_text.get(key, {})
+                for (let x of dt_get_patches(doc)) {
+                    console.log(`x = `, x)
+                    let y = {
+                        merge_type: 'dt',
+                        version: [x.version],
+                        parents: x.parents,
+                        patches: [{
+                            unit: x.unit,
+                            range: x.range,
+                            content: x.content
+                        }]
+                    }
+                    await braid_text.put(key, y)
+                    y.validate_already_seen_versions = true
+                    await braid_text.put(key, y)
                 }
             }
-
-            // 7. try applying a patch that's out of range..
-            // if (true) {
-            //     let agent = Math.random().toString(36).slice(2)
-            //     let parents = doc.getRemoteVersion().map(x => x.join('-'))
-            //     let len = doc.len()
-            //     let args = [`${agent}-0`, parents, len + 1, 'c']
-            //     console.log('ARGS:', args)
-            //     try {
-            //         doc.mergeBytes(dt_create_bytes(...args))
-            //     } catch (e) {
-            //         console.log(`EEEE = ${e}`)
-            //     }
-            //     console.log('did that..')
-            // }
+            await dt_to_braid(doc, 'doc')
+            await dt_to_braid(middle_doc, 'middle_doc')
+            console.log(`doc dt = ${doc.get()}`)
+            console.log(`middle_doc dt = ${middle_doc.get()}`)
+            console.log(`doc = ${await braid_text.get('doc')}`)
+            console.log(`middle_doc = ${await braid_text.get('middle_doc')}`)
+
+            // ensure they look right
+            if (doc.get() != await braid_text.get('doc')) throw new Error('bad')
+            if (middle_doc.get() != await braid_text.get('middle_doc')) throw new Error('bad')
+
+            // test getting old version
+            let middle_v = middle_doc.getRemoteVersion().map(x => x.join('-'))
+            console.log(`middle_doc = ${await braid_text.get('middle_doc')}`)
+            console.log(`middle_v = `, middle_v)
+            
+            let doc_v = doc.getRemoteVersion().map(x => x.join('-'))
+            console.log(`doc_v = `, doc_v)
+
+            console.log(`doc = `, await braid_text.get('doc', {version: middle_v}))
+            if (await braid_text.get('middle_doc') != (await braid_text.get('doc', {version: middle_v})).body) throw new Error('bad')
+
+            // try getting updates from middle_doc to doc
+            let o = {merge_type: 'dt', parents: middle_v, subscribe: update => {
+                braid_text.put('middle_doc', update)
+            }}
+            await braid_text.get('doc', o)
+            await braid_text.forget('doc', o)
+
+            if (await braid_text.get('middle_doc') != await braid_text.get('doc')) throw new Error('bad')
+
+            doc.free()
+            middle_doc.free()
+            for (let p of Object.values(braid_text.cache))
+                (await p).doc.free()
+            braid_text.cache = {}
         } catch (e) {
             if (console.log == og_log) throw e
             if (n < best_n) {
@@ -132,74 +142,30 @@ function make_random_edit(doc) {
     let len = parent_doc.len()
     console.log(`len = ${len}`)
 
+    parent_doc.free()
+
     if (len && Math.random() > 0.5) {
         // delete
         let start = Math.floor(Math.random() * len)
         let del_len = Math.floor(Math.random() * (len - start - 1)) + 1
 
-        for (let i = 0; i < del_len; i++) {
-            let v = `${agent}-${base_seq++}`
-            let args = [v, parents, start + del_len - 1 - i, null]
-            console.log(args)
-            doc.mergeBytes(dt_create_bytes(...args))
-            parents = [v]
-        }
+        let args = [`${agent}-${base_seq}`, parents, start, del_len, null]
+        console.log(args)
+        doc.mergeBytes(dt_create_bytes(...args))
     } else {
         // insert
         let start = Math.floor(Math.random() * (len + 1))
-        let ins_len = Math.floor(Math.random() * 10) + 1
-
-        for (let i = 0; i < ins_len; i++) {
-            let v = `${agent}-${base_seq++}`
-            let args = [v, parents, start++, getRandomCharacter()]
-            console.log(args)
-            doc.mergeBytes(dt_create_bytes(...args))
-            parents = [v]
-        }
+        let ins = Array(Math.floor(Math.random() * 10) + 1).fill(0).map(() => getRandomCharacter()).join('')
+       
+        let args = [`${agent}-${base_seq}`, parents, start, 0, ins]
+        console.log(args)
+        doc.mergeBytes(dt_create_bytes(...args))
     }
 
     // work here
     console.log(`doc => ${doc.get()}`)
 }
 
-function apply_updates(doc, updates) {
-    for (let u of updates) {
-        u.range = u.range.match(/\d+/g).map((x) => parseInt(x))
-        u.content = [...u.content]
-
-        let v = u.version
-        let ps = u.parents
-
-        console.log('UPDATE:', u)
-
-        // delete
-        for (let i = u.range[1] - 1; i >= u.range[0]; i--) {
-
-            // work here
-            let args = [v, ps, i, null]
-            console.log(`args`, args)
-
-            doc.mergeBytes(dt_create_bytes(...args))
-            ps = [v]
-            v = decode_version(v)
-            v = `${v[0]}-${v[1] + 1}`
-        }
-        // insert
-        for (let i = 0; i < u.content?.length ?? 0; i++) {
-            let c = u.content[i]
-
-            // work here
-            let args = [v, ps, u.range[0] + i, c]
-            console.log(`args`, args)
-
-            doc.mergeBytes(dt_create_bytes(...args))
-            ps = [v]
-            v = decode_version(v)
-            v = `${v[0]}-${v[1] + 1}`
-        }
-    }
-}
-
 //////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////