braidfs 0.0.149 → 0.0.152

package/README.md

@@ -80,7 +80,7 @@ diamond-types CRDT over the Braid HTTP protocol, guaranteeing conflict-free
 editing with multiple editors, whether online or offline.
 
 A novel trick using [Time Machines](https://braid.org/time-machines) lets us
-making regular text editors conflict-free, as well, without speaking CRDT!
+make regular text editors conflict-free, as well, without speaking CRDT!
 This means that you can edit a file in Emacs, even while other people edit the
 same file, without write conflicts, and without adding CRDT code to Emacs.
 (Still under development.)
@@ -171,12 +171,20 @@ applied to the running braidfs daemon.  The only exception is the `port`
 setting, which requires restarting the daemon after a change.
 
 
+## Testing
+
+Run the test suite with:
+
+```
+npm test
+```
+
+This starts a local braid-text and braid-blob server, spawns the braidfs daemon,
+and runs through a series of sync, edit, readonly, and restart scenarios to
+verify everything works end-to-end.
+
 ## Limitations & Future Work
 
-- Doesn't sync binary yet.  Just text text mime-types:
-  `text/*`, `application/html`, and `application/json`
-  - Binary blob support would be pretty easy and nice to add.
-  - Contact us if you'd like to add it!
 - Doesn't update your editor's text with remote updates until you save
   - It's not hard to make it live-update, though, so that you can see your edits integrated with others' before you save.
   - Contact us if you'd like to help!  It would be a fun project!

package/index.js

@@ -7,7 +7,13 @@ var { diff_main } = require(`${__dirname}/diff.js`),
 
 // Helper function to check if a file is binary based on its extension
 function is_binary(filename) {
-    const binaryExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.mp4', '.mp3', '.zip', '.tar', '.rar', '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.exe', '.dll', '.so', '.dylib', '.bin', '.iso', '.img', '.bmp', '.tiff', '.svg', '.webp', '.avi', '.mov', '.wmv', '.flv', '.mkv', '.wav', '.flac', '.aac', '.ogg', '.wma', '.7z', '.gz', '.bz2', '.xz'];
+    const binaryExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.mp4', '.mp3',
+                              '.zip', '.tar', '.rar', '.pdf', '.doc', '.docx',
+                              '.xls', '.xlsx', '.ppt', '.pptx', '.exe', '.dll',
+                              '.so', '.dylib', '.bin', '.iso', '.img', '.bmp',
+                              '.tiff', '.svg', '.webp', '.avi', '.mov', '.wmv',
+                              '.flv', '.mkv', '.wav', '.flac', '.aac', '.ogg',
+                              '.wma', '.7z', '.gz', '.bz2', '.xz'];
     return binaryExtensions.includes(require('path').extname(filename).toLowerCase());
 }
 
@@ -102,7 +108,8 @@ console.log(`braidfs version: ${require(`${__dirname}/package.json`).version}`)
 // process command line args (argv was already processed above for --sync-base)
 if (argv.length === 1 && argv[0].match(/^(run|serve)$/)) {
     return main()
-} else if (argv.length && argv.length % 2 == 0 && argv.every((x, i) => i % 2 != 0 || x.match(/^(sync|unsync)$/))) {
+} else if (argv.length && argv.length % 2 == 0
+           && argv.every((x, i) => i % 2 != 0 || x.match(/^(sync|unsync)$/))) {
     return (async () => {
         for (let i = 0; i < argv.length; i += 2) {
             var sync = argv[i] === 'sync',
@@ -124,7 +131,8 @@ if (argv.length === 1 && argv[0].match(/^(run|serve)$/)) {
                     }]
                 })
                 if (res.ok) {
-                    console.log(`Now ${sync ? '' : 'un'}subscribed ${sync ? 'to' : 'from'} ${url} in ~/http/.braidfs/config`)
+                    console.log(`Now ${sync ? '' : 'un'}subscribed ${sync ? 'to' : 'from'}`
+                                + ` ${url} in ~/http/.braidfs/config`)
                 } else {
                     console.log(`failed to ${operation} ${url}`)
                     console.log(`server responded with ${res.status}: ${await res.text()}`)
@@ -183,7 +191,8 @@ async function main() {
                 var sync = await sync_url.cache[normalize_url(path)]
 
                 var parents = JSON.parse(decodeURIComponent(m[2]))
-                var parent_text = sync?.version_to_text_cache.get(JSON.stringify(parents)) ?? (await braid_text.get(sync.url, { parents })).body
+                var parent_text = sync?.version_to_text_cache.get(JSON.stringify(parents))
+                    ?? (await braid_text.get(sync.url, { version: parents, full_response: true })).body
 
                 var text = await new Promise(done => {
                     const chunks = []
@@ -201,9 +210,14 @@ async function main() {
                     await braid_text.put(sync.url, { version, parents, patches, merge_type: 'dt' })
 
                     // DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
-                    require('fs').appendFileSync(investigating_disconnects_log, `${Date.now()}:${sync.url} -- plugin edited (${sync.investigating_disconnects_thinks_connected})\n`)
-
-                    // may be able to do this more efficiently.. we want to make sure we're capturing a file write that is after our version was written.. there may be a way we can avoid calling file_needs_writing here
+                    require('fs').appendFileSync(investigating_disconnects_log,
+                                                 `${Date.now()}:${sync.url} -- plugin edited `
+                                                 + `(${sync.investigating_disconnects_thinks_connected})\n`)
+
+                    // may be able to do this more efficiently.. we want to
+                    // make sure we're capturing a file write that is after
+                    // our version was written.. there may be a way we can
+                    // avoid calling file_needs_writing here
                     await new Promise(done => {
                         sync.file_written_cbs.push(done)
                         sync.signal_file_needs_writing()
@@ -214,7 +228,9 @@ async function main() {
 
             if (url !== '.braidfs/config' && url !== '.braidfs/errors') {
                 res.writeHead(404, { 'Content-Type': 'text/html' })
-                return res.end('Nothing to see here. You can go to <a href=".braidfs/config">.braidfs/config</a> or <a href=".braidfs/errors">.braidfs/errors</a>')
+                return res.end('Nothing to see here. You can go to '
+                               + '<a href=".braidfs/config">.braidfs/config</a>'
+                               + 'or <a href=".braidfs/errors">.braidfs/errors</a>')
             }
 
             braid_text.serve(req, res, { key: normalize_url(url) })
@@ -232,18 +248,25 @@ async function main() {
             subscribe: async update => {
                 let prev = config
 
-                let x = await braid_text.get('.braidfs/config')
+                let config_text = await braid_text.get('.braidfs/config')
                 try {
-                    config = JSON.parse(x)
+                    config = JSON.parse(config_text)
 
                     // did anything get deleted?
-                    var old_syncs = Object.entries(prev.sync).filter(x => x[1]).map(x => normalize_url(x[0]).replace(/^https?:\/\//, ''))
-                    var new_syncs = new Set(Object.entries(config.sync).filter(x => x[1]).map(x => normalize_url(x[0]).replace(/^https?:\/\//, '')))
-                    for (let url of old_syncs.filter(x => !new_syncs.has(x)))
+                    var old_urls = Object.entries(prev.sync)
+                        .filter(sync_entry => sync_entry[1])
+                        .map(sync_entry => normalize_url(sync_entry[0])
+                                    .replace(/^https?:\/\//, ''))
+                    var new_urls = new Set(Object.entries(config.sync)
+                                            .filter(sync_entry => sync_entry[1])
+                                            .map(sync_entry => normalize_url(sync_entry[0])
+                                                        .replace(/^https?:\/\//, '')))
+                    for (let url of old_urls.filter(url => !new_urls.has(url)))
                         unsync_url(url)
 
                     // sync all the new stuff
-                    for (let x of Object.entries(config.sync)) if (x[1]) sync_url(x[0])
+                    for (let sync_entry of Object.entries(config.sync))
+                        if (sync_entry[1]) sync_url(sync_entry[0])
 
                     // if any auth stuff has changed,
                     // have the appropriate connections reconnect
@@ -257,11 +280,11 @@ async function main() {
                             || JSON.stringify(prev.cookies[domain]) !== JSON.stringify(v))
                             changed.add(domain)
                     // ok, have every domain which has changed reconnect
-                    for (let [path, x] of Object.entries(sync_url.cache))
+                    for (let [path, sync] of Object.entries(sync_url.cache))
                         if (changed.has(path.split(/\//)[0].split(/:/)[0]))
-                            (await x).reconnect?.()
+                            (await sync).reconnect?.()
                 } catch (e) {
-                    if (x !== '') console.log(`warning: config file is currently invalid.`)
+                    if (config !== '') console.log(`warning: config file is currently invalid.`)
                     return
                 }
             }
@@ -269,7 +292,9 @@ async function main() {
         sync_url('.braidfs/errors')
 
         // console.log({ sync: config.sync })
-        for (let x of Object.entries(config.sync)) if (x[1]) sync_url(x[0])
+        for (let sync_entry of Object.entries(config.sync))
+            if (sync_entry[1])
+                sync_url(sync_entry[0])
 
         watch_files()
         setTimeout(scan_files, 1200)
@@ -524,6 +549,7 @@ function sync_url(url) {
                         headers: {
                             // in case it supports dt, so it doesn't give us "simpleton"
                             'Merge-Type': 'dt',
+                            Cookie: config.cookies?.[new URL(url).hostname] || undefined
                         }
                     })
                     if (self.ac.signal.aborted) return
@@ -634,7 +660,8 @@ function sync_url(url) {
                         await save_meta()
                         if (self.ac.signal.aborted) return
 
-                        if (self.file_read_only !== null && await is_read_only(fullpath) !== self.file_read_only) {
+                        if (self.file_read_only !== null
+                            && await is_read_only(fullpath) !== self.file_read_only) {
                             if (self.ac.signal.aborted) return
                             await set_read_only(fullpath, self.file_read_only)
                         }
@@ -825,7 +852,7 @@ function sync_url(url) {
                 if (!self.local_edit_counter) self.local_edit_counter = 0
 
                 try {
-                    self.file_last_text = (await wait_on(braid_text.get(url, { version: file_last_version }))).body
+                    self.file_last_text = (await wait_on(braid_text.get(url, { version: file_last_version, full_response: true }))).body
                 } catch (e) {
                     if (self.ac.signal.aborted) return
                     // the version from the meta file doesn't exist..
@@ -834,8 +861,9 @@ function sync_url(url) {
                         // we want to load the current file contents,
                         // which we can acheive by setting file_last_version
                         // to the latest
-                        console.log(`WARNING: there was an issue with the config file, and it is reverting to the contents at: ${braidfs_config_file}`)
-                        var x = await wait_on(braid_text.get(url, {}))
+                        console.log(`WARNING: there was an issue with the config file, `
+                                    + `and it is reverting to the contents at: ${braidfs_config_file}`)
+                        var x = await wait_on(braid_text.get(url, {full_response: true}))
                         if (self.ac.signal.aborted) return
                         file_last_version = x.version
                         self.file_last_text = x.body
@@ -844,11 +872,15 @@ function sync_url(url) {
                 }
                 if (self.ac.signal.aborted) return
 
-                file_needs_writing = !v_eq(file_last_version, (await wait_on(braid_text.get(url, {}))).version)
+                file_needs_writing = !v_eq(
+                    file_last_version,
+                    (await wait_on(braid_text.get(url, {full_response: true}))).version
+                )
                 if (self.ac.signal.aborted) return
 
                 // sanity check
-                if (file_last_digest && sha256(self.file_last_text) != file_last_digest) throw new Error('file_last_text does not match file_last_digest')
+                if (file_last_digest && sha256(self.file_last_text) != file_last_digest)
+                    throw new Error('file_last_text does not match file_last_digest')
             } else if (await wait_on(require('fs').promises.access(fullpath).then(() => 1, () => 0))) {
                 if (self.ac.signal.aborted) return
                 // file exists, but not meta file
@@ -861,6 +893,8 @@ function sync_url(url) {
         // DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
         do_investigating_disconnects_log(url, 'after within_fiber')
 
+        var file_peer = Math.random().toString(36).slice(2)
+
         await file_loop_pump()
         async function file_loop_pump() {
             if (self.ac.signal.aborted) return
@@ -902,7 +936,9 @@ function sync_url(url) {
                         }
                         if (self.ac.signal.aborted) return
 
-                        if (self.file_read_only === null) try { self.file_read_only = await wait_on(is_read_only(fullpath)) } catch (e) { }
+                        if (self.file_read_only === null)
+                            try { self.file_read_only = await wait_on(is_read_only(fullpath)) }
+                            catch (e) { }
                         if (self.ac.signal.aborted) return
 
                         let text = await wait_on(require('fs').promises.readFile(
@@ -927,11 +963,16 @@ function sync_url(url) {
 
                             add_to_version_cache(text, version)
 
-                            await wait_on(braid_text.put(url, { version, parents, patches, merge_type: 'dt' }))
+                            await wait_on(braid_text.put(url, { version, parents, patches,
+                                                                merge_type: 'dt', peer: file_peer }))
                             if (self.ac.signal.aborted) return
 
                             // DEBUGGING HACK ID: L04LPFHQ1M -- INVESTIGATING DISCONNECTS
-                            require('fs').appendFileSync(investigating_disconnects_log, `${Date.now()}:${url} -- file edited (${self.investigating_disconnects_thinks_connected})\n`)
+                            require('fs').appendFileSync(
+                                investigating_disconnects_log,
+                                `${Date.now()}:${url} -- file edited `
+                                + `(${self.investigating_disconnects_thinks_connected})\n`
+                            )
 
                             await write_meta_file()
                             if (self.ac.signal.aborted) return
@@ -954,12 +995,14 @@ function sync_url(url) {
                         if (self.ac.signal.aborted) return
                     } else if (file_needs_writing) {
                         file_needs_writing = false
-                        let { version, body } = await wait_on(braid_text.get(url, {}))
+                        let { version, body } = await wait_on(braid_text.get(url, {full_response: true}))
                         if (self.ac.signal.aborted) return
                         if (!v_eq(version, file_last_version)) {
                             // let's do a final check to see if anything has changed
                             // before writing out a new version of the file
-                            let text = await wait_on(require('fs').promises.readFile(fullpath, { encoding: 'utf8' }))
+                            let text = await wait_on(
+                                require('fs').promises.readFile(fullpath, { encoding: 'utf8' })
+                            )
                             if (self.ac.signal.aborted) return
                             if (self.file_last_text != text) {
                                 // if the text is different, let's read it first..
@@ -972,7 +1015,10 @@ function sync_url(url) {
 
                             add_to_version_cache(body, version)
 
-                            try { if (await wait_on(is_read_only(fullpath))) await wait_on(set_read_only(fullpath, false)) } catch (e) { }
+                            try {
+                                if (await wait_on(is_read_only(fullpath)))
+                                    await wait_on(set_read_only(fullpath, false))
+                            } catch (e) { }
                             if (self.ac.signal.aborted) return
 
                             file_last_version = version
@@ -992,7 +1038,9 @@ function sync_url(url) {
                         }
                         if (self.ac.signal.aborted) return
 
-                        self.file_mtimeNs_str = '' + (await wait_on(require('fs').promises.stat(fullpath, { bigint: true }))).mtimeNs
+                        self.file_mtimeNs_str = '' + (await wait_on(
+                            require('fs').promises.stat(fullpath, { bigint: true })
+                        )).mtimeNs
                         if (self.ac.signal.aborted) return
 
                         for (var cb of self.file_written_cbs) cb()
@@ -1026,11 +1074,16 @@ function sync_url(url) {
                 aborts.clear()
             }
 
-            // Subscribe to local changes to trigger file writes
+            // Subscribe to local changes to trigger file writes.
+            // head: true = header-only updates: we only need to know *that*
+            // something changed. (Without it, braid-text materializes the
+            // doc's entire edit history as patch objects just for this
+            // subscription, which caused huge transient memory at startup.)
+            // The single initial head update triggers the initial write.
             braid_text.get(url, {
                 signal: ac.signal,
-                peer: self.peer,
-                merge_type: 'dt',
+                peer: file_peer,
+                head: true,
                 subscribe: () => {
                     if (self.ac.signal.aborted) return
                     self.signal_file_needs_writing()
@@ -1045,6 +1098,7 @@ function sync_url(url) {
 
                 fork_point_hint: old_meta_fork_point,
                 signal: ac.signal,
+                peer: self.peer,
                 headers: {
                     'Content-Type': 'text/plain',
                     ...(x => x && { Cookie: x })(config.cookies?.[new URL(url).hostname])
@@ -1338,7 +1392,10 @@ async function set_read_only(fullpath, read_only) {
 
     if (require('os').platform() === "win32") {
         await new Promise((resolve, reject) => {
-            require("child_process").exec(`fsutil file setattr readonly "${fullpath}" ${!!read_only}`, (error) => error ? reject(error) : resolve())
+            require("child_process").exec(
+                `fsutil file setattr readonly "${fullpath}" ${!!read_only}`
+                , (error) => error ? reject(error) : resolve()
+            )
         })
     } else {
         let mode = (await require('fs').promises.stat(fullpath)).mode

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "braidfs",
-  "version": "0.0.149",
+  "version": "0.0.152",
   "description": "braid technology synchronizing files and webpages",
   "author": "Braid Working Group",
   "repository": "braid-org/braidfs",
   "homepage": "https://braid.org",
   "dependencies": {
     "braid-http": "~1.3.89",
-    "braid-text": "~0.2.108",
+    "braid-text": "~0.5.16",
     "braid-blob": "~0.0.72",
     "chokidar": "^4.0.3",
     "undici": "^7.18.2"
@@ -15,6 +15,9 @@
   "bin": {
     "braidfs": "./index.sh"
   },
+  "scripts": {
+    "test": "node test/test.js"
+  },
   "files": [
     "index.sh",
     "index.js",