braidfs 0.0.39 → 0.0.40

package/index.js

@@ -15,6 +15,7 @@ var proxy_base = `${require('os').homedir()}/http`,
     proxy_base_meta = `${braidfs_config_dir}/proxy_base_meta`
 braid_text.db_folder = `${braidfs_config_dir}/braid-text-db`
 var trash = `${braidfs_config_dir}/trash`
+var temp_folder = `${braidfs_config_dir}/temp`
 
 var config = null,
     path_to_func = {},
@@ -30,7 +31,8 @@ if (require('fs').existsSync(proxy_base)) {
     config = {
         port: 10000,
         sync: {},
-        domains: { 'example.com': { auth_headers: { Cookie: "secret_pass" } } }
+        domains: { 'example.com': { auth_headers: { Cookie: "secret_pass" } } },
+        scan_interval_ms: 1000 * 20,
     }
     require('fs').mkdirSync(braidfs_config_dir, { recursive: true })
     require('fs').writeFileSync(braidfs_config_file, JSON.stringify(config, null, 4))
@@ -38,6 +40,7 @@ if (require('fs').existsSync(proxy_base)) {
 
 require('fs').mkdirSync(proxy_base_meta, { recursive: true })
 require('fs').mkdirSync(trash, { recursive: true })
+require('fs').mkdirSync(temp_folder, { recursive: true })
 
 // process command line args
 let to_run_in_background = process.platform === 'darwin' ? `
@@ -140,9 +143,12 @@ async function main() {
                         config = JSON.parse(x)
 
                         // did anything get deleted?
-                        for (let url of Object.keys(prev.sync))
-                            if (!config.sync[url]) unproxy_url(url)
+                        var old_syncs = new Set(Object.keys(prev.sync).map(url => normalize_url(url).replace(/^https?:\/\//, '')))
+                        var new_syncs = new Set(Object.keys(config.sync).map(url => normalize_url(url).replace(/^https?:\/\//, '')))
+                        for (let url of old_syncs.difference(new_syncs))
+                            unproxy_url(url)
 
+                        // proxy all the new stuff
                         for (let url of Object.keys(config.sync)) proxy_url(url)
 
                         // if any auth stuff has changed,
@@ -157,10 +163,9 @@ async function main() {
                                 || JSON.stringify(prev.domains[domain]) !== JSON.stringify(v))
                                 changed.add(domain)
                         // ok, have every domain which has changed reconnect
-                        for (let [url, x] of Object.entries(proxy_url.cache))
-                            if (url.match(/^https?:\/\//)
-                                && changed.has(new URL(url).hostname))
-                                (await x).reconnect()
+                        for (let [path, x] of Object.entries(proxy_url.cache))
+                            if (changed.has(path.split(/\//)[0].split(/:/)[0]))
+                                (await x).reconnect?.()
                     } catch (e) {
                         if (x !== '') console.log(`warning: config file is currently invalid.`)
                         return
@@ -174,62 +179,131 @@ async function main() {
         console.log({ sync: config.sync })
         for (let url of Object.keys(config.sync)) proxy_url(url)
 
-        require('chokidar').watch(proxy_base).on('change', x => chokidar_handler(x, false)).on('add', x => chokidar_handler(x, true))
-        async function chokidar_handler(fullpath, added) {
-
-            // Make sure the path is within proxy_base..
-            if (!fullpath.startsWith(proxy_base))
-                return on_watcher_misses(`path ${fullpath} outside ${proxy_base}`)
-
-            // Make sure the path is to a file, and not a directory
-            if ((await require('fs').promises.stat(fullpath)).isDirectory())
-                return on_watcher_misses(`expected file, got: ${fullpath}`)
-
-            var path = require('path').relative(proxy_base, fullpath)
-
-            // Files to skip
-            if (// Paths with a # in the name can't map to real URLs
-                path.includes('#')
-                // .DS_store
-                || path.endsWith('.DS_store')
-                // Skip stuff in .braidfs/ except for config and errors
-                || (path.startsWith('.braidfs')
-                    && !path.match(/^\.braidfs\/(config|errors)$/)))
-                return
-
-            console.log(`file event: ${path}, added: ${added}`)
-
-            var update_func = await path_to_func[normalize_url(path)]
-
-            console.log(`has update_func: ${!!update_func}`)
-
-            if (update_func && !added) update_func()
-            if (!update_func) {
-                // throw this unrecognized file into the trash,
-                let dest = `${trash}/${braid_text.encode_filename(path)}_${Math.random().toString(36).slice(2)}`
-                console.log(`moving untracked file ${fullpath} to ${dest}`)
-                await require('fs').promises.rename(fullpath, dest)
-
-                // and log an error
-                var x = await braid_text.get('.braidfs/errors', {}),
-                    len = [...x.body].length
-                await braid_text.put('.braidfs/errors', {
-                    parents: x.version,
-                    patches: [{
-                        unit: 'text',
-                        range: `[${len}:${len}]`,
-                        content: `error: unsynced file ${fullpath}; moved to ${dest}\n`
-                    }]
-                })
-            }
-        }
+        watch_files()
+        setTimeout(scan_files, 1200)
     }).on('error', e => {
         if (e.code === 'EADDRINUSE') return console.log(`server already running on port ${config.port}`)
         throw e
     })
 }
 
+function on_watcher_miss(message, scan = true) {
+    console.log(`watcher miss: ${message}`)
+    console.log(`\x1b[33;40m[${++watcher_misses}] watcher misfires\x1b[0m`)
+    watch_files()
+    if (scan) setTimeout(scan_files, 1200)
+}
+
+function skip_file(path) {
+    // Files to skip
+    return (
+        // Paths with a # in the name can't map to real URLs
+        path.includes('#')
+        // .DS_store
+        || path.endsWith('.DS_store')
+        // Skip stuff in .braidfs/ except for config and errors
+        || (path.startsWith('.braidfs')
+            && !path.match(/^\.braidfs\/(config|errors)$/))
+    )
+}
+
+async function trash_file(fullpath, path) {
+    // throw this unrecognized file into the trash,
+    let dest = `${trash}/${braid_text.encode_filename(path)}_${Math.random().toString(36).slice(2)}`
+    console.log(`moving untracked file ${fullpath} to ${dest}`)
+    await require('fs').promises.rename(fullpath, dest)
+
+    // and log an error
+    var x = await braid_text.get('.braidfs/errors', {}),
+        len = [...x.body].length
+    await braid_text.put('.braidfs/errors', {
+        parents: x.version,
+        patches: [{
+            unit: 'text',
+            range: `[${len}:${len}]`,
+            content: `error: unsynced file ${fullpath}; moved to ${dest}\n`
+        }]
+    })
+}
+
+async function watch_files() {
+    if (watch_files.watcher === 42) return
+    let w = watch_files.watcher
+    watch_files.watcher = 42
+    await w?.close()
+
+    console.log('watch files..')
+    watch_files.watcher = require('chokidar').watch(proxy_base).
+        on('change', x => chokidar_handler(x, false)).
+        on('add', x => chokidar_handler(x, true))
+
+    async function chokidar_handler(fullpath, added) {
+        // Make sure the path is within proxy_base..
+        if (!fullpath.startsWith(proxy_base))
+            return on_watcher_miss(`path ${fullpath} outside ${proxy_base}`)
+
+        // Make sure the path is to a file, and not a directory
+        if ((await require('fs').promises.stat(fullpath)).isDirectory())
+            return on_watcher_miss(`expected file, got: ${fullpath}`)
+
+        var path = require('path').relative(proxy_base, fullpath)
+        if (skip_file(path)) return
+        console.log(`file event: ${path}, added: ${added}`)
+
+        var proxy = await proxy_url.cache[normalize_url(path)]
+
+        if (proxy && !added) proxy.signal_file_needs_reading()
+        if (!proxy) await trash_file(fullpath, path)
+    }
+}
+
+async function scan_files() {
+    scan_files.do_again = true
+    if (scan_files.running) return
+    if (scan_files.timeout) clearTimeout(scan_files.timeout)
+
+    scan_files.running = true
+    while (scan_files.do_again) {
+        scan_files.do_again = false
+        console.log(`scan files..`)
+        if (await f(proxy_base))
+            on_watcher_miss(`scanner picked up a change that the watcher should have gotten`, false)
+    }
+    scan_files.running = false
+
+    scan_files.timeout = setTimeout(scan_files, config.scan_interval_ms ?? (20 * 1000))
+
+    async function f(fullpath) {
+        let stat = await require('fs').promises.stat(fullpath, { bigint: true })
+        if (stat.isDirectory()) {
+            let found
+            for (let file of await require('fs').promises.readdir(fullpath))
+                found ||= await f(`${fullpath}/${file}`)
+            return found
+        } else {
+            path = require('path').relative(proxy_base, fullpath)
+            if (skip_file(path)) return
+
+            var proxy = await proxy_url.cache[normalize_url(path)]
+            if (!proxy) return await trash_file(fullpath, path)
+
+            if (!stat_eq(stat, proxy.file_last_stat)) {
+                console.log(`scan thinks ${path} has changed`)
+
+
+                console.log(stat)
+                console.log(proxy.file_last_stat)
+
+
+                proxy.signal_file_needs_reading()
+                return true
+            }
+        }
+    }
+}
+
 function unproxy_url(url) {
+    url = normalize_url(url).replace(/^https?:\/\//, '')
     if (!proxy_url.cache?.[url]) return
 
     console.log(`unproxy_url: ${url}`)
@@ -249,18 +323,15 @@ async function proxy_url(url) {
         fullpath = `${proxy_base}/${path}`,
         meta_path = `${proxy_base_meta}/${braid_text.encode_filename(url)}`
 
-    let set_path_to_func
-    if (!path_to_func[path]) path_to_func[path] = new Promise(done => set_path_to_func = done)
-
     if (!proxy_url.cache) proxy_url.cache = {}
     if (!proxy_url.chain) proxy_url.chain = Promise.resolve()
-    if (!proxy_url.cache[url]) proxy_url.cache[url] = proxy_url.chain = proxy_url.chain.then(async () => {
+    if (!proxy_url.cache[path]) proxy_url.cache[path] = proxy_url.chain = proxy_url.chain.then(async () => {
         var freed = false,
             aborts = new Set(),
             braid_text_get_options = null,
             wait_count = 0
-         var wait_promise, wait_promise_done
-         var start_something = () => {
+        var wait_promise, wait_promise_done
+        var start_something = () => {
             if (freed) return
             if (!wait_count) wait_promise = new Promise(done => wait_promise_done = done)
             return ++wait_count
@@ -270,10 +341,9 @@ async function proxy_url(url) {
             if (!wait_count) wait_promise_done()
         }
         if (!unproxy_url.cache) unproxy_url.cache = {}
-        var old_unproxy = unproxy_url.cache[url]
-        unproxy_url.cache[url] = async () => {
+        var old_unproxy = unproxy_url.cache[path]
+        unproxy_url.cache[path] = async () => {
             freed = true
-            delete path_to_func[path]
             for (let a of aborts) a.abort()
             await wait_promise
             if (braid_text_get_options) await braid_text.forget(url, braid_text_get_options)
@@ -306,15 +376,15 @@ async function proxy_url(url) {
         var peer = Math.random().toString(36).slice(2),
             char_counter = -1,
             file_last_version = null,
-            file_last_digest = null,
-            file_last_text = null,
-            file_last_stat = null
+            file_last_digest = null
+        self.file_last_text = null
+        self.file_last_stat = null
         self.file_read_only = null
         var file_needs_reading = true,
             file_needs_writing = null,
             file_loop_pump_lock = 0
 
-        function signal_file_needs_reading() {
+        self.signal_file_needs_reading = () => {
             if (freed) return
             file_needs_reading = true
             file_loop_pump()
@@ -353,8 +423,6 @@ async function proxy_url(url) {
             finish_something()
         }
 
-        set_path_to_func(signal_file_needs_reading)
-
         file_loop_pump()
         async function file_loop_pump() {
             if (file_loop_pump_lock) return
@@ -362,98 +430,109 @@ async function proxy_url(url) {
 
             if (!start_something()) return
 
-            if (file_last_version === null) {
-                if (await require('fs').promises.access(meta_path).then(() => 1, () => 0)) {
-                    // meta file exists
-                    let meta = JSON.parse(await require('fs').promises.readFile(meta_path, { encoding: 'utf8' }))
-                    let _ = ({ version: file_last_version, digest: file_last_digest } = Array.isArray(meta) ? { version: meta } : meta)
-
-                    file_last_text = (await braid_text.get(url, { version: file_last_version })).body
-                    file_needs_writing = !v_eq(file_last_version, (await braid_text.get(url, {})).version)
-
-                    // sanity check
-                    if (file_last_digest && require('crypto').createHash('sha256').update(file_last_text).digest('base64') != file_last_digest) throw new Error('file_last_text does not match file_last_digest')
-                } else if (await require('fs').promises.access(await get_fullpath()).then(() => 1, () => 0)) {
-                    // file exists, but not meta file
-                    file_last_version = []
-                    file_last_text = ''
-                } else {
-                    // file doesn't exist, nor does meta file
-                    file_needs_writing = true
-                    file_last_version = []
-                    file_last_text = ''
-                    await require('fs').promises.writeFile(await get_fullpath(), file_last_text)
+            await within_file_lock(fullpath, async () => {
+                var fullpath = await get_fullpath()
+
+                if (file_last_version === null) {
+                    if (await require('fs').promises.access(meta_path).then(
+                        () => 1, () => 0)) {
+                        // meta file exists
+                        let meta = JSON.parse(await require('fs').promises.readFile(meta_path, { encoding: 'utf8' }))
+                        let _ = ({ version: file_last_version, digest: file_last_digest } = Array.isArray(meta) ? { version: meta } : meta)
+
+                        self.file_last_text = (await braid_text.get(url, { version: file_last_version })).body
+                        file_needs_writing = !v_eq(file_last_version, (await braid_text.get(url, {})).version)
+
+                        // sanity check
+                        if (file_last_digest && require('crypto').createHash('sha256').update(self.file_last_text).digest('base64') != file_last_digest) throw new Error('file_last_text does not match file_last_digest')
+                    } else if (await require('fs').promises.access(fullpath).then(() => 1, () => 0)) {
+                        // file exists, but not meta file
+                        file_last_version = []
+                        self.file_last_text = ''
+                    } else {
+                        // file doesn't exist, nor does meta file
+                        file_needs_writing = true
+                        file_last_version = []
+                        self.file_last_text = ''
+
+                        await require('fs').promises.writeFile(fullpath, self.file_last_text)
+                    }
                 }
-            }
 
-            while (file_needs_reading || file_needs_writing) {
-                if (file_needs_reading) {
-                    console.log(`reading file: ${await get_fullpath()}`)
+                while (file_needs_reading || file_needs_writing) {
+                    if (file_needs_reading) {
+                        console.log(`reading file: ${fullpath}`)
 
-                    file_needs_reading = false
+                        file_needs_reading = false
 
-                    if (self.file_read_only === null) try { self.file_read_only = await is_read_only(await get_fullpath()) } catch (e) { }
+                        if (self.file_read_only === null) try { self.file_read_only = await is_read_only(fullpath) } catch (e) { }
 
-                    let text = await require('fs').promises.readFile(
-                        await get_fullpath(), { encoding: 'utf8' })
+                        let text = await require('fs').promises.readFile(
+                            fullpath, { encoding: 'utf8' })
 
-                    var stat = await require('fs').promises.stat(await get_fullpath(), {bigint: true})
-                    var stat_ms = Number(stat.mtimeMs)
-                    stat = JSON.stringify(stat, (k, v) => typeof v === 'bigint' ? v.toString() : v)
+                        var stat = await require('fs').promises.stat(fullpath, { bigint: true })
 
-                    var patches = diff(file_last_text, text)
-                    if (patches.length) {
-                        console.log(`found changes in: ${await get_fullpath()}`)
+                        var patches = diff(self.file_last_text, text)
+                        if (patches.length) {
+                            console.log(`found changes in: ${fullpath}`)
 
-                        // convert from js-indicies to code-points
-                        char_counter += patches_to_code_points(patches, file_last_text)
+                            // convert from js-indicies to code-points
+                            char_counter += patches_to_code_points(patches, self.file_last_text)
 
-                        file_last_text = text
+                            self.file_last_text = text
 
-                        var version = [peer + "-" + char_counter]
-                        var parents = file_last_version
-                        file_last_version = version
+                            var version = [peer + "-" + char_counter]
+                            var parents = file_last_version
+                            file_last_version = version
 
-                        send_out({ version, parents, patches, peer })
+                            send_out({ version, parents, patches, peer })
 
-                        await braid_text.put(url, { version, parents, patches, peer })
+                            await braid_text.put(url, { version, parents, patches, peer })
 
-                        await require('fs').promises.writeFile(meta_path, JSON.stringify({ version: file_last_version, digest: require('crypto').createHash('sha256').update(file_last_text).digest('base64') }))
-                    } else {
-                        console.log(`no changes found in: ${await get_fullpath()}`)
-                        if (stat === file_last_stat && Date.now() > stat_ms + 300)
-                            on_watcher_miss(`expected change to: ${await get_fullpath()}`)
-                    }
-                    file_last_stat = stat
-                }
-                if (file_needs_writing) {
-                    file_needs_writing = false
-                    let { version, body } = await braid_text.get(url, {})
-                    if (!v_eq(version, file_last_version)) {
-                        console.log(`writing file ${await get_fullpath()}`)
-
-                        // make sure the file has what it had before
-                        let text = await require('fs').promises.readFile(await get_fullpath(), { encoding: 'utf8' })
-                        if (file_last_text != text) crash(new Error('File changed without us noticing.'))
-                        
-                        try { if (await is_read_only(await get_fullpath())) await set_read_only(await get_fullpath(), false) } catch (e) { }
-
-                        file_last_version = version
-                        file_last_text = body
-                        await require('fs').promises.writeFile(await get_fullpath(), file_last_text)
-
-                        file_last_stat = JSON.stringify(await require('fs').promises.stat(await get_fullpath(), {bigint: true}), (k, v) => typeof v === 'bigint' ? v.toString() : v)
-
-                        await require('fs').promises.writeFile(meta_path, JSON.stringify({
-                            version: file_last_version,
-                            digest: require('crypto').createHash('sha256')
-                                .update(file_last_text).digest('base64')
-                        }))
+                            await require('fs').promises.writeFile(meta_path, JSON.stringify({ version: file_last_version, digest: require('crypto').createHash('sha256').update(self.file_last_text).digest('base64') }))
+                        } else {
+                            console.log(`no changes found in: ${fullpath}`)
+                            if (stat_eq(stat, self.file_last_stat)
+                                && !self.file_change_expected)
+                                on_watcher_miss(`expected change to: ${fullpath}`)
+                        }
+                        self.file_last_stat = stat
+                        self.file_change_expected = null
                     }
+                    if (file_needs_writing) {
+                        file_needs_writing = false
+                        let { version, body } = await braid_text.get(url, {})
+                        if (!v_eq(version, file_last_version)) {
+                            console.log(`writing file ${fullpath}`)
+
+                            // make sure the file has what it had before
+                            let text = await require('fs').promises.readFile(fullpath, { encoding: 'utf8' })
+                            if (self.file_last_text != text) crash(new Error('File changed without us noticing.'))
+
+                            try { if (await is_read_only(fullpath)) await set_read_only(fullpath, false) } catch (e) { }
+
+                            file_last_version = version
+                            self.file_last_text = body
+                            self.file_change_expected = true
+                            await require('fs').promises.writeFile(fullpath, self.file_last_text)
+
+
+                            await require('fs').promises.writeFile(meta_path, JSON.stringify({
+                                version: file_last_version,
+                                digest: require('crypto').createHash('sha256')
+                                    .update(self.file_last_text).digest('base64')
+                            }))
+                        }
 
-                    if (await is_read_only(await get_fullpath()) !== self.file_read_only) await set_read_only(await get_fullpath(), self.file_read_only)
+                        if (await is_read_only(fullpath) !== self.file_read_only) {
+                            self.file_change_expected = true
+                            await set_read_only(fullpath, self.file_read_only)
+                        }
+
+                        self.file_last_stat = await require('fs').promises.stat(fullpath, { bigint: true })
+                    }
                 }
-            }
+            })
 
             finish_something()
 
@@ -497,7 +576,7 @@ async function proxy_url(url) {
 
             await new Promise(done => {
                 if (!Object.keys(waiting_for_versions).length) {
-                    console.log('got everything we were waiting for..')
+                    console.log('got everything we were waiting for: ' + url)
                     done()
                     done = null
                 }
@@ -618,21 +697,17 @@ async function ensure_path(path) {
     try {
         await require("fs").promises.mkdir(path, { recursive: true })
     } catch (e) {
-        let parts = path.split(require("path").sep).slice(1)
-        for (let i = 1; i <= parts.length; i++) {
-            let partial = require("path").sep + require("path").join(...parts.slice(0, i))
+        var parts = path.split('/').slice(1)
+        for (var i = 1; i <= parts.length; i++) {
+            var partial = '/' + parts.slice(0, i).join('/')
 
             if (!(await is_dir(partial))) {
-                let save = await require("fs").promises.readFile(partial)
-
-                await require("fs").promises.unlink(partial)
-                await require("fs").promises.mkdir(path, { recursive: true })
-
-                while (await is_dir(partial))
-                    partial = require("path").join(partial, 'index')
-
-                await require("fs").promises.writeFile(partial, save)
-                break
+                await within_file_lock(normalize_url(partial), async () => {
+                    let temp = `${temp_folder}/${Math.random().toString(36).slice(2)}`
+                    await require('fs').promises.rename(partial, temp)
+                    await require("fs").promises.mkdir(partial)
+                    await require('fs').promises.rename(temp, partial + '/index')
+                })
             }
         }
     }
@@ -729,11 +804,19 @@ function v_eq(v1, v2) {
     return v1.length === v2?.length && v1.every((x, i) => x == v2[i])
 }
 
+function stat_eq(a, b) {
+    return (!a && !b) || (a && b &&
+        a.mode === b.mode &&
+        a.size === b.size &&
+        a.mtimeNs === b.mtimeNs &&
+        a.ctimeNs === b.ctimeNs)
+}
+
 async function is_read_only(fullpath) {
-    const stats = await require('fs').promises.stat(fullpath)
+    const stat = await require('fs').promises.stat(fullpath)
     return require('os').platform() === "win32" ?
-        !!(stats.mode & 0x1) :
-        !(stats.mode & 0o200)
+        !!(stat.mode & 0x1) :
+        !(stat.mode & 0o200)
 }
 
 async function set_read_only(fullpath, read_only) {
@@ -756,7 +839,19 @@ function crash(e) {
     process.exit(1)
 }
 
-function on_watcher_miss(message) {
-    console.log(`watcher miss: ${message}`)
-    console.log(`\x1b[33;40m[${++watcher_misses}] watcher misfires\x1b[0m`);
+async function get_file_lock(fullpath) {
+    if (!get_file_lock.locks) get_file_lock.locks = {}
+    if (!get_file_lock.locks[fullpath]) get_file_lock.locks[fullpath] = Promise.resolve()
+    return new Promise(done =>
+        get_file_lock.locks[fullpath] = get_file_lock.locks[fullpath].then(() =>
+            new Promise(done2 => done(done2))))
+}
+
+async function within_file_lock(fullpath, func) {
+    var lock = await get_file_lock(fullpath)
+    try {
+        return await func()
+    } finally {
+        lock()
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braidfs",
-  "version": "0.0.39",
+  "version": "0.0.40",
   "description": "braid technology synchronizing files and webpages",
   "author": "Braid Working Group",
   "repository": "braid-org/braidfs",