braid-text 0.0.28 → 0.0.30

package/index.js

@@ -4,7 +4,9 @@ let braidify = require("braid-http").http_server
 let fs = require("fs")
 
 let braid_text = {
+    verbose: false,
     db_folder: './braid-text-db',
+    length_cache_size: 10,
     cache: {}
 }
 
@@ -126,7 +128,7 @@ braid_text.serve = async (req, res, options = {}) => {
         }
 
         waiting_puts++
-        console.log(`waiting_puts(after++) = ${waiting_puts}`)
+        if (braid_text.verbose) console.log(`waiting_puts(after++) = ${waiting_puts}`)
 
         let my_prev_put_p = prev_put_p
         let done_my_turn = null
@@ -134,7 +136,7 @@ braid_text.serve = async (req, res, options = {}) => {
             (done) =>
             (done_my_turn = (statusCode, x) => {
                 waiting_puts--
-                console.log(`waiting_puts(after--) = ${waiting_puts}`)
+                if (braid_text.verbose) console.log(`waiting_puts(after--) = ${waiting_puts}`)
                 my_end(statusCode, x)
                 done()
             })
@@ -240,7 +242,7 @@ braid_text.get = async (key, options) => {
             let updates = null
 
             if (resource.need_defrag) {
-                console.log(`doing defrag..`)
+                if (braid_text.verbose) console.log(`doing defrag..`)
                 resource.need_defrag = false
                 resource.doc = defrag_dt(resource.doc)
             }
@@ -301,10 +303,10 @@ braid_text.put = async (key, options) => {
     let parents = resource.doc.getRemoteVersion().map((x) => x.join("-")).sort()
     let og_parents = options_parents || parents
 
-    let max_pos = count_code_points(v_eq(parents, og_parents) ?
-        resource.doc.get() :
-        dt_get(resource.doc, og_parents).get())
-
+    let max_pos = resource.length_cache.get('' + og_parents) ??
+        (v_eq(parents, og_parents) ? resource.doc.len() :
+            dt_get(resource.doc, og_parents).len())
+    
     if (body != null) {
         patches = [{
             unit: 'text',
@@ -400,6 +402,10 @@ braid_text.put = async (key, options) => {
     }
     resource.actor_seqs[v[0]] = v[1]
 
+    resource.length_cache.put(`${v[0]}-${v[1]}`, patches.reduce((a, b) =>
+        a + (b.content.length ? b.content.length : -(b.range[1] - b.range[0])),
+        max_pos))
+
     v = `${v[0]}-${v[1] + 1 - change_count}`
 
     let ps = og_parents
@@ -435,7 +441,7 @@ braid_text.put = async (key, options) => {
 
     if (options.merge_type != "dt") {
         patches = get_xf_patches(resource.doc, v_before)
-        console.log(JSON.stringify({ patches }))
+        if (braid_text.verbose) console.log(JSON.stringify({ patches }))
 
         let version = resource.doc.getRemoteVersion().map((x) => x.join("-")).sort()
 
@@ -451,11 +457,11 @@ braid_text.put = async (key, options) => {
                     let x = { version }
                     x.parents = client.my_last_seen_version
 
-                    console.log("rebasing after timeout.. ")
-                    console.log("    client.my_unused_version_count = " + client.my_unused_version_count)
+                    if (braid_text.verbose) console.log("rebasing after timeout.. ")
+                    if (braid_text.verbose) console.log("    client.my_unused_version_count = " + client.my_unused_version_count)
                     x.patches = get_xf_patches(resource.doc, OpLog_remote_to_local(resource.doc, client.my_last_seen_version))
 
-                    console.log(`sending from rebase: ${JSON.stringify(x)}`)
+                    if (braid_text.verbose) console.log(`sending from rebase: ${JSON.stringify(x)}`)
                     client.subscribe(x)
                     client.my_last_sent_version = x.version
 
@@ -491,12 +497,12 @@ braid_text.put = async (key, options) => {
 
                 x.parents = options.version
                 if (!v_eq(version, options.version)) {
-                    console.log("rebasing..")
+                    if (braid_text.verbose) console.log("rebasing..")
                     x.patches = get_xf_patches(resource.doc, OpLog_remote_to_local(resource.doc, [og_v]))
                 } else {
                     // this client already has this version,
                     // so let's pretend to send it back, but not
-                    console.log(`not reflecting back to simpleton`)
+                    if (braid_text.verbose) console.log(`not reflecting back to simpleton`)
                     client.my_last_sent_version = x.version
                     continue
                 }
@@ -504,7 +510,7 @@ braid_text.put = async (key, options) => {
                 x.parents = parents
                 x.patches = patches
             }
-            console.log(`sending: ${JSON.stringify(x)}`)
+            if (braid_text.verbose) console.log(`sending: ${JSON.stringify(x)}`)
             client.subscribe(x)
             client.my_last_sent_version = x.version
         }
@@ -513,7 +519,7 @@ braid_text.put = async (key, options) => {
             let version = resource.doc.getRemoteVersion().map((x) => x.join("-")).sort()
             patches = get_xf_patches(resource.doc, v_before)
             let x = { version, parents, patches }
-            console.log(`sending: ${JSON.stringify(x)}`)
+            if (braid_text.verbose) console.log(`sending: ${JSON.stringify(x)}`)
             for (let client of resource.simpleton_clients) {
                 if (client.my_timeout) continue
                 client.subscribe(x)
@@ -581,13 +587,15 @@ async function get_resource(key) {
 
         resource.val = resource.doc.get()
 
+        resource.length_cache = createSimpleCache(braid_text.length_cache_size)
+
         done(resource)
     })
     return await cache[key]
 }
 
 async function db_folder_init() {
-    console.log('__!')
+    if (braid_text.verbose) console.log('__!')
     if (!db_folder_init.p) db_folder_init.p = new Promise(async done => {
         await fs.promises.mkdir(braid_text.db_folder, { recursive: true });
 
@@ -614,7 +622,7 @@ async function db_folder_init() {
                     console.log(`trying to convert file to new format, but the key is too big: ${braid_text.db_folder}/${x}`)
                     process.exit()
                 }
-                console.log(`deleting: ${braid_text.db_folder}/${x}`)
+                if (braid_text.verbose) console.log(`deleting: ${braid_text.db_folder}/${x}`)
                 await fs.promises.unlink(`${braid_text.db_folder}/${x}`)
             }
         }
@@ -622,7 +630,7 @@ async function db_folder_init() {
             for (let x of await fs.promises.readdir(braid_text.db_folder)) {
                 let [_, k, num] = x.match(/^(.*)\.(\d+)$/s)
                 if (!convert_us[k]) continue
-                console.log(`renaming: ${braid_text.db_folder}/${x} -> ${braid_text.db_folder}/${convert_us[k]}.${num}`)
+                if (braid_text.verbose) console.log(`renaming: ${braid_text.db_folder}/${x} -> ${braid_text.db_folder}/${convert_us[k]}.${num}`)
                 if (convert_us[k]) await fs.promises.rename(`${braid_text.db_folder}/${x}`, `${braid_text.db_folder}/${convert_us[k]}.${num}`)
             }
         }
@@ -665,7 +673,7 @@ async function file_sync(key, process_delta, get_init) {
         }
         try {
             const filename = files[i]
-            console.log(`trying to process file: ${filename}`)
+            if (braid_text.verbose) console.log(`trying to process file: ${filename}`)
             const data = await fs.promises.readFile(filename)
 
             let cursor = 0
@@ -699,17 +707,17 @@ async function file_sync(key, process_delta, get_init) {
                 currentSize += bytes.length + 4 // we account for the extra 4 bytes for uint32
                 const filename = `${braid_text.db_folder}/${encoded}.${currentNumber}`
                 if (currentSize < threshold) {
-                    console.log(`appending to db..`)
+                    if (braid_text.verbose) console.log(`appending to db..`)
 
                     let buffer = Buffer.allocUnsafe(4)
                     buffer.writeUInt32LE(bytes.length, 0)
                     await fs.promises.appendFile(filename, buffer)
                     await fs.promises.appendFile(filename, bytes)
 
-                    console.log("wrote to : " + filename)
+                    if (braid_text.verbose) console.log("wrote to : " + filename)
                 } else {
                     try {
-                        console.log(`starting new db..`)
+                        if (braid_text.verbose) console.log(`starting new db..`)
 
                         currentNumber++
                         const init = get_init()
@@ -720,7 +728,7 @@ async function file_sync(key, process_delta, get_init) {
                         await fs.promises.writeFile(newFilename, buffer)
                         await fs.promises.appendFile(newFilename, init)
 
-                        console.log("wrote to : " + newFilename)
+                        if (braid_text.verbose) console.log("wrote to : " + newFilename)
 
                         currentSize = 4 + init.length
                         threshold = currentSize * 10
@@ -728,7 +736,7 @@ async function file_sync(key, process_delta, get_init) {
                             await fs.promises.unlink(filename)
                         } catch (e) { }
                     } catch (e) {
-                        console.log(`e = ${e.stack}`)
+                        if (braid_text.verbose) console.log(`e = ${e.stack}`)
                     }
                 }
             }))
@@ -745,7 +753,7 @@ async function file_sync(key, process_delta, get_init) {
                                     console.error(`Error deleting file: ${file}`)
                                     reject(err)
                                 } else {
-                                    console.log(`Deleted file: ${file}`)
+                                    if (braid_text.verbose) console.log(`Deleted file: ${file}`)
                                     resolve()
                                 }
                             })
@@ -1513,6 +1521,40 @@ function validate_patch(x) {
     if (typeof x.content !== 'string') throw new Error(`invalid patch content: must be a string`)
 }
 
+function createSimpleCache(size) {
+    const maxSize = size
+    const cache = new Map()
+
+    return {
+        put(key, value) {
+            if (cache.has(key)) {
+                // If the key already exists, update its value and move it to the end
+                cache.delete(key)
+                cache.set(key, value)
+            } else {
+                // If the cache is full, remove the oldest entry
+                if (cache.size >= maxSize) {
+                    const oldestKey = cache.keys().next().value
+                    cache.delete(oldestKey)
+                }
+                // Add the new key-value pair
+                cache.set(key, value)
+            }
+        },
+
+        get(key) {
+            if (!cache.has(key)) {
+                return null
+            }
+            // Move the accessed item to the end (most recently used)
+            const value = cache.get(key)
+            cache.delete(key)
+            cache.set(key, value)
+            return value
+        },
+    }
+}
+
 braid_text.encode_filename = encode_filename
 braid_text.decode_filename = decode_filename
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-text",
-  "version": "0.0.28",
+  "version": "0.0.30",
   "description": "Library for collaborative text over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braidjs",

package/test.js

@@ -9,10 +9,10 @@ async function main() {
 
     let og_log = console.log
     console.log = () => {}
-    for (let t = 0; t < 50000; t++) {
+    for (let t = 0; t < 10000; t++) {
         let seed = base + t
-    // for (let t = 0; t < 1; t++) {
-    //     let seed = 7271846
+    // for (let t = 0; t < 10; t++) {
+    //     let seed = 1188661 + t
 
         og_log(`t = ${t}, seed = ${seed}, best_n = ${best_n} @ ${best_seed}`)
         Math.randomSeed(seed)
@@ -24,9 +24,19 @@ async function main() {
             // 1. create a bunch of edits to a dt
             let doc = new Doc('server')
 
+            let middle_doc = null
+
+            if (!middle_doc && (Math.random() < 1/n || n == 0)) {
+                middle_doc = Doc.fromBytes(doc.toBytes())
+            }
             for (let i = 0; i < n; i++) {
                 make_random_edit(doc)
+
+                if (!middle_doc && (Math.random() < 1/n || i == n - 1)) {
+                    middle_doc = Doc.fromBytes(doc.toBytes())
+                }
             }
+            if (!middle_doc) throw 'bad'
 
             // 2. let x = the resulting string
             let x = doc.get()
@@ -49,37 +59,43 @@ async function main() {
             }
 
             // 5. test dt_get
-            let z = dt_get(doc, doc.getRemoteVersion().map(x => x.join('-'))).get()
-            console.log('z = ' + z)
-            console.log(x == z)
-            if (x != z && n < best_n) {
+            let middle_v = middle_doc.getRemoteVersion().map(x => x.join('-'))
+            let new_middle_doc = dt_get(doc, middle_v)
+            console.log('new_middle_doc = ' + new_middle_doc.get())
+            if (middle_doc.get() != new_middle_doc.get() && n < best_n) {
                 best_n = n
                 best_seed = seed
             }
 
             // 6. test dt_get_patches(doc, version)
             if (true) {
-                let [_agents, versions, _parentss] = dt_parse([...doc.toBytes()])
-                let v = []
-                for (let i = 0; i < versions.length; i++)
-                    if (Math.random() > 0.5) v.push(versions[i].join('-'))
-
-                console.log(`v:${v}`)
-
-                let temp_doc = dt_get(doc, v)
-                console.log(`temp_doc1:${temp_doc.get()}`)
-
-                let updates = dt_get_patches(doc, v)
+                let updates = dt_get_patches(doc, middle_v)
                 console.log(`updates:`, updates)
 
-                apply_updates(temp_doc, updates)
-                console.log(`temp_doc2:${temp_doc.get()}`)
-                if (temp_doc.get() != doc.get() && n < best_n) {
+                apply_updates(middle_doc, updates)
+                console.log(`middle_doc2:${middle_doc.get()}`)
+                if (middle_doc.get() != doc.get() && n < best_n) {
                     best_n = n
                     best_seed = seed
                 }
             }
+
+            // 7. try applying a patch that's out of range..
+            // if (true) {
+            //     let agent = Math.random().toString(36).slice(2)
+            //     let parents = doc.getRemoteVersion().map(x => x.join('-'))
+            //     let len = doc.len()
+            //     let args = [`${agent}-0`, parents, len + 1, 'c']
+            //     console.log('ARGS:', args)
+            //     try {
+            //         doc.mergeBytes(dt_create_bytes(...args))
+            //     } catch (e) {
+            //         console.log(`EEEE = ${e}`)
+            //     }
+            //     console.log('did that..')
+            // }
         } catch (e) {
+            if (console.log == og_log) throw e
             if (n < best_n) {
                 best_n = n
                 best_seed = seed