braid-blob 0.0.18 → 0.0.20

package/.claude/settings.local.json

@@ -1,7 +1,9 @@
 {
   "permissions": {
     "allow": [
-      "Bash(node test/test.js:*)"
+      "Bash(node test/test.js:*)",
+      "Bash(lsof:*)",
+      "Bash(xargs kill -9)"
     ],
     "deny": [],
     "ask": []

package/index.js

@@ -1,4 +1,4 @@
-var {http_server: braidify} = require('braid-http'),
+var {http_server: braidify, fetch: braid_fetch} = require('braid-http'),
     {url_file_db} = require('url-file-db'),
     fs = require('fs'),
     path = require('path')
@@ -44,6 +44,29 @@ function create_braid_blob() {
     }
 
     braid_blob.put = async (key, body, options = {}) => {
+        // Handle URL case - make a remote PUT request
+        if (key instanceof URL) {
+            options.my_abort = new AbortController()
+            if (options.signal) {
+                options.signal.addEventListener('abort', () =>
+                    options.my_abort.abort())
+            }
+
+            var params = {
+                method: 'PUT',
+                signal: options.my_abort.signal,
+                retry: () => true,
+                body: body
+            }
+            for (var x of ['headers', 'version', 'peer'])
+                if (options[x] != null) params[x] = options[x]
+            if (options.content_type) {
+                params.headers = { ...params.headers, 'Content-Type': options.content_type }
+            }
+
+            return await braid_fetch(key.href, params)
+        }
+
         await braid_blob.init()
 
         // Read the meta data from meta_db
@@ -92,6 +115,42 @@ function create_braid_blob() {
     }
 
     braid_blob.get = async (key, options = {}) => {
+        // Handle URL case - make a remote GET request
+        if (key instanceof URL) {
+            options.my_abort = new AbortController()
+
+            var params = {
+                signal: options.my_abort.signal,
+                subscribe: !!options.subscribe,
+                heartbeats: 120,
+            }
+            if (!options.dont_retry) {
+                params.retry = () => true
+            }
+            for (var x of ['headers', 'parents', 'version', 'peer'])
+                if (options[x] != null) params[x] = options[x]
+
+            var res = await braid_fetch(key.href, params)
+
+            if (options.subscribe) {
+                if (options.dont_retry) {
+                    var error_happened
+                    var error_promise = new Promise((_, fail) => error_happened = fail)
+                }
+
+                res.subscribe(async update => {
+                    await options.subscribe(update)
+                }, e => options.dont_retry && error_happened(e))
+
+                if (options.dont_retry) {
+                    return await error_promise
+                }
+                return res
+            } else {
+                return await res.arrayBuffer()
+            }
+        }
+
         await braid_blob.init()
 
         // Read the meta data from meta_db
@@ -106,6 +165,11 @@ function create_braid_blob() {
             content_type: meta.content_type
         }
         if (options.header_cb) await options.header_cb(result)
+        // Check if requested version/parents is newer than what we have - if so, we don't have it
+        if (options.version && options.version.length && compare_events(options.version[0], meta.event) > 0)
+            throw new Error('unkown version: ' + options.version)
+        if (options.parents && options.parents.length && compare_events(options.parents[0], meta.event) > 0)
+            throw new Error('unkown version: ' + options.parents)
         if (options.head) return
 
         if (options.subscribe) {
@@ -177,32 +241,42 @@ function create_braid_blob() {
             if (meta_content)
                 meta = JSON.parse(meta_content.toString('utf8'))
 
-            if (req.method === 'GET') {
+            if (req.method === 'GET' || req.method === 'HEAD') {
                 if (!res.hasHeader("editable")) res.setHeader("Editable", "true")
                 if (!req.subscribe) res.setHeader("Accept-Subscribe", "true")
                 res.setHeader("Merge-Type", "lww")
 
-                var result = await braid_blob.get(options.key, {
-                    peer: req.peer,
-                    head: req.method == "HEAD",
-                    parents: req.parents || null,
-                    header_cb: (result) => {
-                        res.setHeader((req.subscribe ? "Current-" : "") +
-                            "Version", ascii_ify(result.version.map((x) =>
-                                JSON.stringify(x)).join(", ")))
-                        if (result.content_type)
-                            res.setHeader('Content-Type', result.content_type)
-                    },
-                    before_send_cb: (result) =>
-                        res.startSubscription({ onClose: result.unsubscribe }),
-                    subscribe: req.subscribe ? (update) => {
-                        res.sendUpdate({
-                            version: update.version,
-                            'Merge-Type': 'lww',
-                            body: update.body
-                        })
-                    } : null
-                })
+                try {
+                    var result = await braid_blob.get(options.key, {
+                        peer: req.peer,
+                        head: req.method == "HEAD",
+                        version: req.version || null,
+                        parents: req.parents || null,
+                        header_cb: (result) => {
+                            res.setHeader((req.subscribe ? "Current-" : "") +
+                                "Version", ascii_ify(result.version.map((x) =>
+                                    JSON.stringify(x)).join(", ")))
+                            if (result.content_type)
+                                res.setHeader('Content-Type', result.content_type)
+                        },
+                        before_send_cb: (result) =>
+                            res.startSubscription({ onClose: result.unsubscribe }),
+                        subscribe: req.subscribe ? (update) => {
+                            res.sendUpdate({
+                                version: update.version,
+                                'Merge-Type': 'lww',
+                                body: update.body
+                            })
+                        } : null
+                    })
+                } catch (e) {
+                    if (e.message && e.message.startsWith('unkown version')) {
+                        // Server doesn't have this version
+                        res.statusCode = 309
+                        res.statusMessage = 'Version Unknown Here'
+                        return res.end('')
+                    } else throw e
+                }
 
                 if (!result) {
                     res.statusCode = 404
@@ -240,6 +314,109 @@ function create_braid_blob() {
         })
     }
 
+    braid_blob.sync = async (a, b, options = {}) => {
+        var unsync_cbs = []
+        options.my_unsync = () => unsync_cbs.forEach(cb => cb())
+
+        if ((a instanceof URL) === (b instanceof URL)) {
+            // Both are URLs or both are local keys
+            var a_ops = {
+                subscribe: update => braid_blob.put(b, update.body, {
+                    version: update.version,
+                    content_type: update.headers?.['content-type']
+                })
+            }
+            braid_blob.get(a, a_ops)
+
+            var b_ops = {
+                subscribe: update => braid_blob.put(a, update.body, {
+                    version: update.version,
+                    content_type: update.headers?.['content-type']
+                })
+            }
+            braid_blob.get(b, b_ops)
+        } else {
+            // One is local, one is remote - make a=local and b=remote (swap if not)
+            if (a instanceof URL) {
+                let swap = a; a = b; b = swap
+            }
+
+            var closed = false
+            options.my_unsync = () => { closed = true; disconnect() }
+
+            var disconnect = () => { }
+            async function connect() {
+                var ac = new AbortController()
+                var disconnect_cbs = [() => ac.abort()]
+                disconnect = () => disconnect_cbs.forEach(cb => cb())
+
+                try {
+                    // Check if remote has our current version (simple fork-point check)
+                    var local_result = await braid_blob.get(a)
+                    var local_version = local_result ? local_result.version : null
+                    var server_has_our_version = false
+
+                    if (local_version) {
+                        // Check if server has our version
+                        var r = await braid_fetch(b.href, {
+                            signal: ac.signal,
+                            method: "HEAD",
+                            version: local_version
+                        })
+                        server_has_our_version = r.ok
+                    }
+
+                    // Local -> remote: subscribe to future local changes
+                    var a_ops = {
+                        subscribe: update => {
+                            update.signal = ac.signal
+                            braid_blob.put(b, update.body, {
+                                version: update.version,
+                                content_type: update.content_type
+                            }).catch(e => {
+                                if (e.name === 'AbortError') {
+                                    // ignore
+                                } else throw e
+                            })
+                        }
+                    }
+                    // Only set parents if server already has our version
+                    // If server doesn't have it, omit parents so subscription sends everything
+                    if (server_has_our_version) {
+                        a_ops.parents = local_version
+                    }
+                    braid_blob.get(a, a_ops)
+
+                    // Remote -> local: subscribe to remote updates
+                    var b_ops = {
+                        dont_retry: true,
+                        subscribe: async update => {
+                            await braid_blob.put(a, update.body, {
+                                version: update.version,
+                                content_type: update.headers?.['content-type']
+                            })
+                        },
+                    }
+                    // Use fork-point (parents) to avoid receiving data we already have
+                    if (local_version) {
+                        b_ops.parents = local_version
+                    }
+                    // NOTE: this should not return, but it might throw
+                    await braid_blob.get(b, b_ops)
+                } catch (e) {
+                    if (closed) {
+                        return
+                    }
+
+                    disconnect()
+                    console.log(`disconnected, retrying in 1 second`)
+                    setTimeout(connect, 1000)
+                }
+            }
+            connect()
+        }
+    }
+
     function compare_events(a, b) {
         var a_num = get_event_seq(a)
         var b_num = get_event_seq(b)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-blob",
-  "version": "0.0.18",
+  "version": "0.0.20",
   "description": "Library for collaborative blobs over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-blob",

package/test/tests.js

@@ -725,6 +725,423 @@ runTest(
     '"100"|"200"'
 )
 
+runTest(
+    "test put with URL (no content_type)",
+    async () => {
+        var key = 'test-url-put-' + Math.random().toString(36).slice(2)
+
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var braid_blob = require(\`\${__dirname}/../index.js\`)
+                var url = new URL('http://localhost:' + req.socket.localPort + '/${key}')
+                await braid_blob.put(url, Buffer.from('url put test'), { version: ['100'] })
+                res.end('done')
+            })()`
+        })
+        await r1.text()
+
+        var r = await braid_fetch(`/${key}`)
+        return await r.text()
+    },
+    'url put test'
+)
+
+runTest(
+    "test put with URL (with content_type)",
+    async () => {
+        var key = 'test-url-put-ct-' + Math.random().toString(36).slice(2)
+
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var braid_blob = require(\`\${__dirname}/../index.js\`)
+                var url = new URL('http://localhost:' + req.socket.localPort + '/${key}')
+                await braid_blob.put(url, Buffer.from('url put with ct'), {
+                    version: ['200'],
+                    content_type: 'text/plain'
+                })
+                res.end('done')
+            })()`
+        })
+        await r1.text()
+
+        var r = await braid_fetch(`/${key}`)
+        return r.headers.get('content-type') + '|' + await r.text()
+    },
+    'text/plain|url put with ct'
+)
+
+runTest(
+    "test get with URL (no subscribe)",
+    async () => {
+        var key = 'test-url-get-' + Math.random().toString(36).slice(2)
+
+        await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            version: ['300'],
+            body: 'url get test'
+        })
+
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var braid_blob = require(\`\${__dirname}/../index.js\`)
+                var url = new URL('http://localhost:' + req.socket.localPort + '/${key}')
+                var result = await braid_blob.get(url)
+                res.end(Buffer.from(result).toString('utf8'))
+            })()`
+        })
+
+        return await r1.text()
+    },
+    'url get test'
+)
+
+runTest(
+    "test get with URL (with subscribe)",
+    async () => {
+        var key = 'test-url-get-sub-' + Math.random().toString(36).slice(2)
+
+        await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            version: ['400'],
+            body: 'initial'
+        })
+
+        // Use a promise to wait for the eval to complete
+        var evalPromise = braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var braid_blob = require(\`\${__dirname}/../index.js\`)
+                var url = new URL('http://localhost:' + req.socket.localPort + '/${key}')
+
+                var updates = []
+                var a = new AbortController()
+
+                // Don't await - braid_blob.get returns immediately when subscribe is used
+                braid_blob.get(url, {
+                    subscribe: update => {
+                        updates.push(Buffer.from(update.body).toString('utf8'))
+                        if (updates.length === 2) {
+                            a.abort()
+                            res.end(updates.join('|'))
+                        }
+                    },
+                    signal: a.signal
+                })
+            })()`
+        })
+
+        // Wait a bit for subscription to be established
+        await new Promise(done => setTimeout(done, 100))
+
+        // Send update
+        await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            version: ['500'],
+            body: 'updated'
+        })
+
+        // Wait for the eval to complete
+        var r1 = await evalPromise
+        return await r1.text()
+    },
+    'initial|updated'
+)
+
+runTest(
+    "test sync local to remote",
+    async () => {
+        var local_key = 'test-sync-local-' + Math.random().toString(36).slice(2)
+        var remote_key = 'test-sync-remote-' + Math.random().toString(36).slice(2)
+
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                try {
+                    var braid_blob = require(\`\${__dirname}/../index.js\`)
+
+                    // Put something locally first
+                    await braid_blob.put('${local_key}', Buffer.from('local content'), { version: ['600'] })
+
+                    var remote_url = new URL('http://localhost:' + req.socket.localPort + '/${remote_key}')
+
+                    // Start sync
+                    braid_blob.sync('${local_key}', remote_url)
+
+                    res.end('syncing')
+                } catch (e) {
+                    res.end('error: ' + e.message + ' ' + e.stack)
+                }
+            })()`
+        })
+        var result = await r1.text()
+        if (result.startsWith('error:')) return result
+
+        // Wait a bit for sync to happen
+        await new Promise(done => setTimeout(done, 100))
+
+        // Check remote has the content
+        var r = await braid_fetch(`/${remote_key}`)
+        return await r.text()
+    },
+    'local content'
+)
+
+runTest(
+    "test sync two local keys",
+    async () => {
+        var key1 = 'test-sync-local1-' + Math.random().toString(36).slice(2)
+        var key2 = 'test-sync-local2-' + Math.random().toString(36).slice(2)
+
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                try {
+                    var braid_blob = require(\`\${__dirname}/../index.js\`)
+
+                    // Put something to first key
+                    await braid_blob.put('${key1}', Buffer.from('sync local content'), { version: ['700'] })
+
+                    // Start sync between two local keys
+                    braid_blob.sync('${key1}', '${key2}')
+
+                    res.end('syncing')
+                } catch (e) {
+                    res.end('error: ' + e.message + ' ' + e.stack)
+                }
+            })()`
+        })
+        var result = await r1.text()
+        if (result.startsWith('error:')) return result
+
+        // Wait a bit for sync to happen
+        await new Promise(done => setTimeout(done, 100))
+
+        // Check second key has the content
+        var r = await braid_fetch(`/${key2}`)
+        return await r.text()
+    },
+    'sync local content'
+)
+
+runTest(
+    "test sync remote to local (swap)",
+    async () => {
+        var local_key = 'test-sync-swap-local-' + Math.random().toString(36).slice(2)
+        var remote_key = 'test-sync-swap-remote-' + Math.random().toString(36).slice(2)
+
+        // Put something on the server first
+        await braid_fetch(`/${remote_key}`, {
+            method: 'PUT',
+            version: ['800'],
+            body: 'remote content'
+        })
+
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                try {
+                    var braid_blob = require(\`\${__dirname}/../index.js\`)
+                    var remote_url = new URL('http://localhost:' + req.socket.localPort + '/${remote_key}')
+
+                    // Start sync with URL as first argument (should swap internally)
+                    braid_blob.sync(remote_url, '${local_key}')
+
+                    res.end('syncing')
+                } catch (e) {
+                    res.end('error: ' + e.message + ' ' + e.stack)
+                }
+            })()`
+        })
+        var result = await r1.text()
+        if (result.startsWith('error:')) return result
+
+        // Wait a bit for sync to happen
+        await new Promise(done => setTimeout(done, 100))
+
+        // Check local key has the remote content
+        var r = await braid_fetch(`/${local_key}`)
+        return await r.text()
+    },
+    'remote content'
+)
+
+runTest(
+    "test sync when server already has our version",
+    async () => {
+        var local_key = 'test-sync-has-version-local-' + Math.random().toString(36).slice(2)
+        var remote_key = 'test-sync-has-version-remote-' + Math.random().toString(36).slice(2)
+
+        // Put the same content on both local and remote with the same version
+        var version = ['900']
+        var content = 'shared content'
+
+        // Put on remote first
+        await braid_fetch(`/${remote_key}`, {
+            method: 'PUT',
+            version: version,
+            body: content
+        })
+
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                try {
+                    var braid_blob = require(\`\${__dirname}/../index.js\`)
+
+                    // Put the same content locally with the same version
+                    await braid_blob.put('${local_key}', Buffer.from('${content}'), { version: ${JSON.stringify(version)} })
+
+                    var remote_url = new URL('http://localhost:' + req.socket.localPort + '/${remote_key}')
+
+                    // Start sync - this should trigger the "server already has our version" path
+                    braid_blob.sync('${local_key}', remote_url)
+
+                    res.end('syncing')
+                } catch (e) {
+                    res.end('error: ' + e.message + ' ' + e.stack)
+                }
+            })()`
+        })
+        var result = await r1.text()
+        if (result.startsWith('error:')) return result
+
+        // Wait a bit for sync to initialize (the console.log should happen quickly)
+        await new Promise(done => setTimeout(done, 100))
+
+        // Verify that both still have the same content
+        var r = await braid_fetch(`/${remote_key}`)
+        return await r.text()
+    },
+    'shared content'
+)
+
+runTest(
+    "test sync closed during error",
+    async () => {
+        var local_key = 'test-sync-closed-local-' + Math.random().toString(36).slice(2)
+        var remote_key = 'test-sync-closed-remote-' + Math.random().toString(36).slice(2)
+
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                try {
+                    var braid_blob = require(\`\${__dirname}/../index.js\`)
+
+                    // Use an invalid/unreachable URL to trigger an error
+                    var remote_url = new URL('http://localhost:9999/${remote_key}')
+
+                    // Start sync
+                    var sync_options = {}
+                    braid_blob.sync('${local_key}', remote_url, sync_options)
+
+                    // Close the sync immediately to trigger the closed path when error occurs
+                    sync_options.my_unsync()
+
+                    res.end('sync started and closed')
+                } catch (e) {
+                    res.end('error: ' + e.message + ' ' + e.stack)
+                }
+            })()`
+        })
+        var result = await r1.text()
+        if (result.startsWith('error:')) return result
+
+        // Wait for the connection error and closed message
+        await new Promise(done => setTimeout(done, 200))
+
+        return result
+    },
+    'sync started and closed'
+)
+
+runTest(
+    "test sync error with retry",
+    async () => {
+        var local_key = 'test-sync-retry-local-' + Math.random().toString(36).slice(2)
+        var remote_key = 'test-sync-retry-remote-' + Math.random().toString(36).slice(2)
+
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                try {
+                    var braid_blob = require(\`\${__dirname}/../index.js\`)
+
+                    // Use an invalid/unreachable URL to trigger an error
+                    var remote_url = new URL('http://localhost:9999/${remote_key}')
+
+                    // Start sync without closing it - should trigger retry
+                    var sync_options = {}
+                    braid_blob.sync('${local_key}', remote_url, sync_options)
+
+                    // Wait a bit for the error to occur and retry message to print
+                    await new Promise(done => setTimeout(done, 200))
+
+                    // Now close it to stop retrying
+                    sync_options.my_unsync()
+
+                    res.end('sync error occurred')
+                } catch (e) {
+                    res.end('error: ' + e.message + ' ' + e.stack)
+                }
+            })()`
+        })
+        var result = await r1.text()
+
+        return result
+    },
+    'sync error occurred'
+)
+
+runTest(
+    "test requesting with version/parents server doesn't have",
+    async () => {
+        var key = 'test-parents-unknown-' + Math.random().toString(36).slice(2)
+
+        // Put with version 100
+        await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            version: ['100'],
+            body: 'content v100'
+        })
+
+        // Try to subscribe with parents 200 (newer than what server has)
+        // This triggers the "unkown version" error which gets caught and returns 309
+        var r = await braid_fetch(`/${key}`, {
+            subscribe: true,
+            parents: ['200']
+        })
+
+        return r.status
+    },
+    '309'
+)
+
+runTest(
+    "test requesting specific version server doesn't have",
+    async () => {
+        var key = 'test-version-unknown-' + Math.random().toString(36).slice(2)
+
+        // Put with version 100
+        await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            version: ['100'],
+            body: 'content v100'
+        })
+
+        // Try to GET with version 200 (newer than what server has)
+        // This should trigger line 269 when req.version is checked
+        var r = await braid_fetch(`/${key}`, {
+            version: ['200']
+        })
+
+        return r.status
+    },
+    '309'
+)
+
 }
 
 // Export for Node.js (CommonJS)