braidfs 0.0.98 → 0.0.101

package/index.js

@@ -4,6 +4,8 @@ var { diff_main } = require(`${__dirname}/diff.js`),
     braid_text = require("braid-text"),
     braid_fetch = require('braid-http').fetch
 
+braid_fetch.set_fetch(fetch_http2)
+
 var sync_base = `${require('os').homedir()}/http`,
     braidfs_config_dir = `${sync_base}/.braidfs`,
     braidfs_config_file = `${braidfs_config_dir}/config`,
@@ -324,7 +326,10 @@ async function scan_files() {
     scan_files.running = true
     while (scan_files.do_again) {
         scan_files.do_again = false
-        console.log(`scan files..`)
+        var timestamp = new Date().toLocaleTimeString(
+            'en-US', {minute: '2-digit', second: '2-digit', hour: '2-digit'}
+        )
+        console.log(`scan files.. `, timestamp)
         if (await f(sync_base))
             on_watcher_miss(`scanner picked up a change that the watcher should have gotten`, false)
     }
@@ -688,26 +693,26 @@ async function sync_url(url) {
             file_loop_pump_lock--
         }
 
-        self.update_fork_point = (event, parents) => {
-            self.fork_point = extend_frontier(self.fork_point, event, parents)
+        self.update_fork_point = (version, parents) => {
+            self.fork_point = extend_frontier(self.fork_point, version, parents)
             self.signal_file_needs_writing(true)
         }
 
-        function extend_frontier(frontier, event, parents) {
+        function extend_frontier(frontier, version, parents) {
             // special case:
             // if current frontier has all parents,
             //    then we can just remove those
-            //    and add event
+            //    and add version
             var frontier_set = new Set(frontier)
             if (parents.length &&
                 parents.every(p => frontier_set.has(p))) {
                 parents.forEach(p => frontier_set.delete(p))
-                frontier_set.add(event)
+                for (var event of version) frontier_set.add(event)
                 frontier = [...frontier_set.values()]
             } else {
                 // full-proof approach..
                 var looking_for = frontier_set
-                looking_for.add(event)
+                for (var event of version) looking_for.add(event)
 
                 frontier = []
                 var shadow = new Set()
@@ -777,13 +782,14 @@ async function sync_url(url) {
                         var a = new AbortController()
                         aborts.add(a)
                         return await braid_fetch(url, {
+                            ...params,
                             signal: a.signal,
                             headers: {
+                                ...params.headers,
                                 "Merge-Type": "dt",
                                 "Content-Type": 'text/plain',
                                 ...(x => x && {Cookie: x})(config.cookies?.[new URL(url).hostname])
                             },
-                            ...params
                         })
                     } catch (e) {
                         if (freed || closed) return
@@ -803,7 +809,7 @@ async function sync_url(url) {
 
                     // the server has acknowledged this version,
                     // so add it to the fork point
-                    if (r.ok) self.update_fork_point(stuff.version[0], stuff.parents)
+                    if (r.ok) self.update_fork_point(stuff.version, stuff.parents)
 
                     // if we're not authorized,
                     if (r.status == 401 || r.status == 403) {
@@ -825,7 +831,7 @@ async function sync_url(url) {
                     if (self.fork_point) {
                         var r = await my_fetch({ method: "HEAD", version: self.fork_point })
                         if (freed || closed) return
-                        if (r.ok) return console.log(`[find_fork_point] it has our latest fork point, hooray!`)
+                        if (r.ok) return console.log(`[find_fork_point] "${url.split('/').pop()}" has our latest fork point, hooray!`)
                     }
 
                     // otherwise let's binary search for new fork point..
@@ -862,7 +868,52 @@ async function sync_url(url) {
                 await send_new_stuff()
                 if (freed || closed) return
 
-                let a = new AbortController()
+                // attempt to download the initial stuff in one go,
+                // using transfer-encoding dt
+                //
+                // first check for support..
+                //
+                var res = await my_fetch({
+                    method: 'HEAD',
+                    headers: { 'accept-transfer-encoding': 'dt' },
+                })
+                if (freed || closed) return
+
+                if (res.ok && res.headers.get('x-transfer-encoding') === 'dt') {
+                    var res = await my_fetch({
+                        headers: { 'accept-transfer-encoding': 'dt' },
+                        parents: self.fork_point,
+                    })
+                    if (freed || closed) return
+                    console.log(`got external updates about ${url}`)
+
+                    // manually apply the dt bytes..
+                    // ..code bits taken from braid-text put..
+                    var bytes = new Uint8Array(await res.arrayBuffer())
+                    if (freed || closed) return
+
+                    var start_i = 1 + resource.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
+                    resource.doc.mergeBytes(bytes)
+
+                    // update resource.actor_seqs
+                    var end_i = resource.doc.getLocalVersion().reduce((a, b) => Math.max(a, b), -1)
+                    for (var i = start_i; i <= end_i; i++) {
+                        var v = resource.doc.localToRemoteVersion([i])[0]
+                        if (!resource.actor_seqs[v[0]]) resource.actor_seqs[v[0]] = new braid_text.RangeSet()
+                        resource.actor_seqs[v[0]].add_range(v[1], v[1])
+                    }
+
+                    resource.val = resource.doc.get()
+                    resource.need_defrag = true
+                    await resource.db_delta(bytes)
+                    if (freed || closed) return
+
+                    // ..do the things we do when getting subscribe updates..
+                    self.update_fork_point(JSON.parse(`[${res.headers.get('current-version')}]`), self.fork_point)
+                    self.signal_file_needs_writing()
+                }
+
+                var a = new AbortController()
                 aborts.add(a)
                 var res = await braid_fetch(url, {
                     signal: a.signal,
@@ -883,8 +934,8 @@ async function sync_url(url) {
                 if (res.status !== 209)
                     return log_error(`Can't sync ${url} -- got bad response ${res.status} from server (expected 209)`)
 
-                console.log(`connected to ${url}`)
-                console.log(`  editable = ${res.headers.get('editable')}`)
+                console.log(`connected to ${url}`.padEnd(70, ' ')
+                            + `(editable: ${res.headers.get('editable')})`)
 
                 reconnect_rate_limiter.on_conn(url)
 
@@ -909,7 +960,7 @@ async function sync_url(url) {
                     // the server is giving us this version,
                     // so it must have it,
                     // so let's add it to our fork point
-                    self.update_fork_point(update.version[0], update.parents)
+                    self.update_fork_point(update.version, update.parents)
 
                     self.signal_file_needs_writing()
                 }, retry)
@@ -931,7 +982,7 @@ async function sync_url(url) {
                             if (!q.length) {
                                 var frontier = self.fork_point
                                 for (var u of in_flight.values())
-                                    frontier = extend_frontier(frontier, u.version[0], u.parents)
+                                    frontier = extend_frontier(frontier, u.version, u.parents)
 
                                 var options = {
                                     parents: frontier,
@@ -1092,6 +1143,98 @@ function ReconnectRateLimiter(get_wait_time) {
     return self
 }
 
+async function fetch_http2(url, options = {}) {
+    if (!fetch_http2.sessions) {
+        fetch_http2.sessions = new Map()
+        process.on("exit", () => fetch_http2.sessions.forEach(s => s.close()))
+    }
+
+    var u = new URL(url)
+    if (u.protocol !== "https:") return fetch(url, options)
+
+    try {
+        var session = fetch_http2.sessions.get(u.origin)
+        if (!session || session.closed) {
+            session = require("http2").connect(u.origin, {
+                rejectUnauthorized: options.rejectUnauthorized !== false,
+            })
+            session.on("error", () => fetch_http2.sessions.delete(u.origin))
+            session.on("close", () => fetch_http2.sessions.delete(u.origin))
+            fetch_http2.sessions.set(u.origin, session)
+        }
+
+        return await new Promise((resolve, reject) => {
+            var stream = session.request({
+                ":method": options.method || "GET",
+                ":path": u.pathname + u.search,
+                ":scheme": "https",
+                ":authority": u.host,
+                ...Object.fromEntries(options.headers || []),
+            })
+
+            options.signal?.addEventListener("abort",
+                () => stream.destroy(new Error("Request aborted")),
+                { once: true })
+
+            stream.on("response", headers => {
+                var status = +headers[":status"]
+                resolve({
+                    ok: status >= 200 && status < 300,
+                    status,
+                    statusText: "",
+                    headers: new Headers(Object.fromEntries(
+                        Object.entries(headers).filter(([k]) =>
+                            typeof k === "string" && !k.startsWith(":")))),
+                    body: new ReadableStream({
+                        start(ctrl) {
+                            stream.on("data", x => ctrl.enqueue(new Uint8Array(x)))
+                            stream.on("end", () => ctrl.close())
+                            stream.on("error", err => ctrl.error(err))
+                        },
+                        cancel() { stream.destroy() },
+                    }),
+                    bodyUsed: false,
+                    async _consumeBody() {
+                        this.bodyUsed = true
+                        var chunks = []
+                        var reader = this.body.getReader()
+
+                        while (true) {
+                            var { done, value } = await reader.read()
+                            if (done) break
+                            chunks.push(value)
+                        }
+                        return Buffer.concat(chunks.map((c) => (Buffer.isBuffer(c) ? c : Buffer.from(c))))
+                    },
+                    async text() { return (await this._consumeBody()).toString() },
+                    async json() { return JSON.parse(await this.text()) },
+                    async arrayBuffer() {
+                        var b = await this._consumeBody()
+                        return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength)
+                    },
+                })
+            })
+
+            stream.on("error", reject)
+
+            var body = options.body
+            if (!body) return stream.end()
+
+            if (body instanceof Uint8Array || Buffer.isBuffer(body)) stream.end(body)
+            else if (body instanceof Blob) body.arrayBuffer()
+                .then((b) => stream.end(Buffer.from(b)))
+                .catch(reject)
+            else stream.end(typeof body === "string" ? body : JSON.stringify(body))
+        })
+    } catch (err) {
+        if (err.code?.includes("HTTP2") || err.message?.includes("HTTP/2")) {
+            console.log("HTTP/2 failed, falling back to HTTP/1.1:", err.message)
+            return fetch(url, options)
+        }
+        throw err
+    }
+}
+
 ////////////////////////////////
 
 function normalize_url(url) {

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "braidfs",
-  "version": "0.0.98",
+  "version": "0.0.101",
   "description": "braid technology synchronizing files and webpages",
   "author": "Braid Working Group",
   "repository": "braid-org/braidfs",
   "homepage": "https://braid.org",
   "dependencies": {
-    "braid-http": "^1.3.75",
-    "braid-text": "^0.2.30",
+    "braid-http": "^1.3.76",
+    "braid-text": "^0.2.35",
     "chokidar": "^3.6.0"
   },
   "bin": {