braidfs 0.0.119 → 0.0.121

package/index.js

@@ -4,10 +4,26 @@ var { diff_main } = require(`${__dirname}/diff.js`),
     braid_text = require("braid-text"),
     braid_fetch = require('braid-http').fetch
 
+// Helper function to check if a file is binary based on its extension
+function is_binary(filename) {
+    const binaryExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.mp4', '.mp3', '.zip', '.tar', '.rar', '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.exe', '.dll', '.so', '.dylib', '.bin', '.iso', '.img', '.bmp', '.tiff', '.svg', '.webp', '.avi', '.mov', '.wmv', '.flv', '.mkv', '.wav', '.flac', '.aac', '.ogg', '.wma', '.7z', '.gz', '.bz2', '.xz'];
+    return binaryExtensions.includes(require('path').extname(filename).toLowerCase());
+}
+
 braid_fetch.set_fetch(fetch_http2)
 
-var sync_base = `${require('os').homedir()}/http`,
-    braidfs_config_dir = `${sync_base}/.braidfs`,
+var sync_base = `${require('os').homedir()}/http`
+// Check for --sync-base argument (hidden for testing)
+var argv = process.argv.slice(2)
+var sync_base_index = argv.indexOf('--sync-base')
+if (sync_base_index !== -1 && sync_base_index < argv.length - 1) {
+    sync_base = argv[sync_base_index + 1]
+    // Remove the --sync-base and its value from argv
+    argv.splice(sync_base_index, 2)
+    console.log(`[Testing mode] Using sync_base: ${sync_base}`)
+}
+
+var braidfs_config_dir = `${sync_base}/.braidfs`,
     braidfs_config_file = `${braidfs_config_dir}/config`,
     sync_base_meta = `${braidfs_config_dir}/proxy_base_meta`
 braid_text.db_folder = `${braidfs_config_dir}/braid-text-db`
@@ -62,8 +78,7 @@ to_run_in_background = ''
 
 console.log(`braidfs version: ${require(`${__dirname}/package.json`).version}`)
 
-// process command line args
-var argv = process.argv.slice(2)
+// process command line args (argv was already processed above for --sync-base)
 if (argv.length === 1 && argv[0].match(/^(run|serve)$/)) {
     return main()
 } else if (argv.length && argv.length % 2 == 0 && argv.every((x, i) => i % 2 != 0 || x.match(/^(sync|unsync)$/))) {
@@ -166,13 +181,11 @@ async function main() {
                     await braid_text.put(sync.url, { version, parents, patches, merge_type: 'dt' })
 
                     // may be able to do this more efficiently.. we want to make sure we're capturing a file write that is after our version was written.. there may be a way we can avoid calling file_needs_writing here
-                    var stat = await new Promise(done => {
+                    await new Promise(done => {
                         sync.file_written_cbs.push(done)
                         sync.signal_file_needs_writing()
                     })
-
-                    res.writeHead(200, { 'Content-Type': 'application/json' })
-                    return res.end(stat.mtimeMs.toString())
+                    return res.end('')
                 } else return res.end('null')
             }
 
@@ -239,7 +252,7 @@ async function main() {
         watch_files()
         setTimeout(scan_files, 1200)
     }).on('error', e => {
-        if (e.code === 'EADDRINUSE') return console.log(`port ${config.port} is in use`)
+        if (e.code === 'EADDRINUSE') return console.log(`ERROR: port ${config.port} is in use`)
         throw e
     })
 }
@@ -365,7 +378,7 @@ async function scan_files() {
             if (!sync) return await trash_file(fullpath, path)
 
             stat = await require('fs').promises.stat(fullpath, { bigint: true })
-            if (!stat_eq(stat, sync.file_last_stat)) {
+            if ('' + stat.mtimeNs !== sync.file_mtimeNs_str) {
                 console.log(`scan thinks ${path} has changed`)
                 sync.signal_file_needs_reading()
                 return true
@@ -440,14 +453,175 @@ async function sync_url(url) {
         }
         sync_url.cache[path] = sync_url.chain = sync_url.chain.then(init)
     }
-    async function init() {
+    
+    async function init_binary_sync() {
         if (freed) return
 
-        // console.log(`sync_url: ${url}`)
+        console.log(`init_binary_sync: ${url}`)
 
-        var resource = await braid_text.get_resource(url)
+        async function save_meta() {
+            await require('fs').promises.writeFile(meta_path, JSON.stringify({
+                peer: self.peer,
+                version: self.version,
+                file_mtimeNs_str: self.file_mtimeNs_str
+            }))
+        }
+
+        await within_fiber(url, async () => {
+            try {
+                Object.assign(self, JSON.parse(
+                    await require('fs').promises.readFile(meta_path, 'utf8')))
+            } catch (e) {}
+            if (freed) return
+
+            if (!self.peer) self.peer = Math.random().toString(36).slice(2)
+
+            await save_meta()
+        })
+        if (freed) return
+
+        var waitTime = 1
+        var last_connect_timer = null
+
+        self.signal_file_needs_reading = () => {}
+
+        connect()
+        async function connect() {
+            if (freed) return
+            if (last_connect_timer) return
+
+            var closed = false
+            var prev_disconnect = self.disconnect
+            self.disconnect = async () => {
+                if (closed) return
+                closed = true
+                reconnect_rate_limiter.on_diss(url)
+                for (var a of aborts) a.abort()
+                aborts.clear()
+            }
+            self.reconnect = connect
+
+            await prev_disconnect?.()
+            if (freed || closed) return
+            
+            await reconnect_rate_limiter.get_turn(url)
+            if (freed || closed) return
+
+            function retry(e) {
+                if (freed || closed) return
+                var p = self.disconnect()
+
+                console.log(`reconnecting in ${waitTime}s: ${url} after error: ${e}`)
+                last_connect_timer = setTimeout(async () => {
+                    await p
+                    last_connect_timer = null
+                    connect()
+                }, waitTime * 1000)
+                waitTime = Math.min(waitTime + 1, 3)
+            }
+
+            try {
+                var a = new AbortController()
+                aborts.add(a)
+                var res = await braid_fetch(url, {
+                    signal: a.signal,
+                    headers: {
+                        ...(x => x && {Cookie: x})(config.cookies?.[new URL(url).hostname]),
+                    },
+                    subscribe: true,
+                    heartbeats: 120,
+                    peer: self.peer,
+                    ...(self.version != null ? {parents: ['' + self.version]} : {})
+                })
+                if (freed || closed) return
+
+                if (res.status < 200 || res.status >= 300) return retry(new Error(`unexpected status: ${res.status}`))
+
+                if (res.status !== 209)
+                    return log_error(`Can't sync ${url} -- got bad response ${res.status} from server (expected 209)`)
+
+                console.log(`connected to ${url}${res.headers.get('editable') !== 'true' ? ' (readonly)' : ''}`)
+
+                reconnect_rate_limiter.on_conn(url)
+                
+                res.subscribe(async update => {
+                    if (freed || closed) return
+                    if (update.version.length === 0) return
+                    if (update.version.length !== 1) throw 'unexpected'
+                    var version = 1*update.version[0]
+                    if (!update.body) return
+
+                    if (self.version != null &&
+                        version <= self.version) return
+                    self.version = version
+
+                    await within_fiber(url, async () => {
+                        var fullpath = await get_fullpath()
+                        if (freed || closed) return
+
+                        await wait_on(require('fs').promises.writeFile(fullpath, update.body))
+                        if (freed || closed) return
+
+                        var stat = await require('fs').promises.stat(fullpath, { bigint: true })
+                        if (freed || closed) return
+                        self.file_mtimeNs_str = '' + stat.mtimeNs
+                        self.last_touch = Date.now()
+
+                        await save_meta()
+                    })
+                }, retry)
+
+                self.signal_file_needs_reading = () => {
+                    within_fiber(url, async () => {
+                        if (freed || closed) return
+
+                        var fullpath = await get_fullpath()
+                        if (freed || closed) return
+
+                        try {
+                            var stat = await require('fs').promises.stat(fullpath, { bigint: true })
+                        } catch (e) { return }
+                        if (freed || closed) return
+
+                        if ('' + stat.mtimeNs !== self.file_mtimeNs_str) {
+                            self.version = Math.max((self.version || 0) + 1,
+                                Math.round(Number(stat.mtimeNs) / 1000000))
+
+                            self.file_mtimeNs_str = '' + stat.mtimeNs
+                            self.last_touch = Date.now()
+
+                            await save_meta()
+
+                            var body = await require('fs').promises.readFile(fullpath)
+                            if (freed || closed) return
+
+                            var a = new AbortController()
+                            aborts.add(a)
+
+                            await braid_fetch(url, {
+                                method: 'PUT',
+                                signal: a.signal,
+                                version: ['' + self.version],
+                                body,
+                                headers: {
+                                    ...(x => x && {Cookie: x})(config.cookies?.[new URL(url).hostname])
+                                },
+                            })
+                        }
+                    })
+                }
+                self.signal_file_needs_reading()
+            } catch (e) { return retry(e) }            
+        }
+
+        return self
+    }
+    
+    async function init() {
         if (freed) return
 
+        // console.log(`sync_url: ${url}`)
+
         // if we're accessing /blah/index, it will be normalized to /blah,
         // but we still want to create a directory out of blah in this case
         if (wasnt_normal && !(await is_dir(fullpath))) {
@@ -459,13 +633,25 @@ async function sync_url(url) {
         await ensure_path(require("path").dirname(fullpath))
         if (freed) return
 
+        // Check if this is a binary file
+        // for now, this will be stuff in the "blobs" directory..
+        if (is_external_link && path.split('/')[1] === 'blobs') {
+            // Opts into the code for FS watcher (file_needs_reading, file_needs_writing) & unsyncing (disconnect)
+            // It notably does NOT handle `.braidfs/set_version/` and `.braidfs/get_version/` correctly!
+            // Search ` sync_url.cache[` to see how it's all handled.
+            return await init_binary_sync() 
+        }
+
+        var resource = await braid_text.get_resource(url)
+        if (freed) return
+
         self.peer = Math.random().toString(36).slice(2)
         self.local_edit_counter = 0
         self.fork_point = null
         var file_last_version = null,
             file_last_digest = null
         self.file_last_text = null
-        self.file_last_stat = null
+        self.file_mtimeNs_str = null
         self.file_read_only = null
         var file_needs_reading = true,
             file_needs_writing = null,
@@ -647,14 +833,14 @@ async function sync_url(url) {
                             add_to_version_cache(text, file_last_version)
 
                             // console.log(`no changes found in: ${fullpath}`)
-                            if (stat_eq(stat, self.file_last_stat)) {
-                                if (Date.now() > (self.file_ignore_until ?? 0))
+                            if ('' + stat.mtimeNs === self.file_mtimeNs_str) {
+                                if (Date.now() > (self.last_touch ?? 0) + 1000)
                                     on_watcher_miss(`expected change to: ${fullpath}`)
                                 // else console.log(`no changes expected`)
                             } // else console.log('found change in file stat')
                         }
-                        self.file_last_stat = stat
-                        self.file_ignore_until = Date.now() + 1000
+                        self.file_mtimeNs_str = '' + stat.mtimeNs
+                        self.last_touch = Date.now()
                     }
                     if (file_needs_writing === 'just_meta_file') {
                         file_needs_writing = false
@@ -685,7 +871,7 @@ async function sync_url(url) {
 
                             file_last_version = version
                             self.file_last_text = body
-                            self.file_ignore_until = Date.now() + 1000
+                            self.last_touch = Date.now()
                             await wait_on(require('fs').promises.writeFile(fullpath, self.file_last_text))
                             if (freed) return
                         }
@@ -695,15 +881,15 @@ async function sync_url(url) {
 
                         if (await wait_on(is_read_only(fullpath)) !== self.file_read_only) {
                             if (freed) return
-                            self.file_ignore_until = Date.now() + 1000
+                            self.last_touch = Date.now()
                             await wait_on(set_read_only(fullpath, self.file_read_only))
                         }
                         if (freed) return
 
-                        self.file_last_stat = await wait_on(require('fs').promises.stat(fullpath, { bigint: true }))
+                        self.file_mtimeNs_str = '' + (await wait_on(require('fs').promises.stat(fullpath, { bigint: true }))).mtimeNs
                         if (freed) return
 
-                        for (var cb of self.file_written_cbs) cb(self.file_last_stat)
+                        for (var cb of self.file_written_cbs) cb()
                         self.file_written_cbs = []
                     }
                 }
@@ -935,24 +1121,10 @@ async function sync_url(url) {
                     if (!update.status) {
                         // console.log(`got initial update about ${url}`)
 
-                        // manually apply the dt bytes..
-                        // ..code bits taken from braid-text put..
-                        var bytes = update.body
-
-                        var start_i = 1 + resource.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
-                        resource.doc.mergeBytes(bytes)
-
-                        // update resource.actor_seqs
-                        var end_i = resource.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
-                        for (var i = start_i; i <= end_i; i++) {
-                            var v = resource.doc.localToRemoteVersion([i])[0]
-                            if (!resource.actor_seqs[v[0]]) resource.actor_seqs[v[0]] = new braid_text.RangeSet()
-                            resource.actor_seqs[v[0]].add_range(v[1], v[1])
-                        }
-
-                        resource.val = resource.doc.get()
-                        resource.need_defrag = true
-                        await resource.db_delta(bytes)
+                        await braid_text.put(resource, {
+                            body: update.body,
+                            transfer_encoding: 'dt'
+                        })
                         if (freed || closed) return
 
                         self.update_fork_point(JSON.parse(`[${res.headers.get('current-version')}]`), self.fork_point)
@@ -1340,14 +1512,6 @@ function v_eq(v1, v2) {
     return v1.length === v2?.length && v1.every((x, i) => x == v2[i])
 }
 
-function stat_eq(a, b) {
-    return (!a && !b) || (a && b &&
-        a.mode === b.mode &&
-        a.size === b.size &&
-        a.mtimeNs === b.mtimeNs &&
-        a.ctimeNs === b.ctimeNs)
-}
-
 async function is_read_only(fullpath) {
     const stat = await require('fs').promises.stat(fullpath)
     return require('os').platform() === "win32" ?

package/index.sh

@@ -35,8 +35,11 @@ urlencode() {
     echo "$encoded"
 }
 
+# Get the base directory (can be overridden by setting BRAIDFS_BASE_DIR)
+BASE_DIR="${BRAIDFS_BASE_DIR:-${HOME}/http}"
+
 # Get the configuration file to read the port
-CONFIG_FILE="${HOME}/http/.braidfs/config"
+CONFIG_FILE="${BASE_DIR}/.braidfs/config"
 if [ -f "$CONFIG_FILE" ]; then
     PORT=$(grep -o '"port":[^,}]*' "$CONFIG_FILE" | sed 's/"port"://; s/ //g')
     if [ -z "$PORT" ]; then

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "braidfs",
-  "version": "0.0.119",
+  "version": "0.0.121",
   "description": "braid technology synchronizing files and webpages",
   "author": "Braid Working Group",
   "repository": "braid-org/braidfs",
   "homepage": "https://braid.org",
   "dependencies": {
     "braid-http": "~1.3.79",
-    "braid-text": "~0.2.45",
+    "braid-text": "~0.2.64",
+    "braid-blob": "~0.0.8",
     "chokidar": "^4.0.3"
   },
   "bin": {