braid-blob 0.0.16 → 0.0.18

package/.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(node test/test.js:*)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

package/README.md

@@ -61,9 +61,14 @@ curl -X DELETE http://localhost:8888/text
 ```javascript
 var braid_blob = require('braid-blob')
 
-// Set custom storage location (default: './braid-blob-db')
+// Set custom blob storage location (default: './braid-blob-db')
+// This uses url-file-db for efficient URL-to-file mapping
 braid_blob.db_folder = './custom_files_folder'
 
+// Set custom metadata storage location (default: './braid-blob-meta')
+// Stores version metadata and peer information
+braid_blob.meta_folder = './custom_meta_folder'
+
 // Set custom peer ID (default: auto-generated and persisted)
 braid_blob.peer = 'my-server-id'
 ```

package/index.js

@@ -1,35 +1,165 @@
-var {http_server: braidify, free_cors} = require('braid-http'),
+var {http_server: braidify} = require('braid-http'),
+    {url_file_db} = require('url-file-db'),
     fs = require('fs'),
     path = require('path')
 
 function create_braid_blob() {
     var braid_blob = {
         db_folder: './braid-blob-db',
+        meta_folder: './braid-blob-meta',
         cache: {},
         key_to_subs: {},
-        peer: null // we'll try to load this from a file, if not set by the user
+        peer: null, // we'll try to load this from a file, if not set by the user
+        db: null, // url-file-db instance for blob storage
+        meta_db: null // url-file-db instance for meta storage
     }
 
     braid_blob.init = async () => {
-        braid_blob.init = () => {}
+        // We only want to initialize once
+        var init_p = real_init()
+        braid_blob.init = () => init_p
+        await braid_blob.init()
 
-        await fs.promises.mkdir(`${braid_blob.db_folder}/blob`, { recursive: true })
-        await fs.promises.mkdir(`${braid_blob.db_folder}/meta`, { recursive: true })
+        async function real_init() {
+            // Create url-file-db instance for blob storage
+            braid_blob.db = await url_file_db.create(braid_blob.db_folder, async (key) => {
+                // File changed externally, notify subscriptions
+                var body = await braid_blob.db.read(key)
+                await braid_blob.put(key, body, { skip_write: true })
+            })
 
-        // establish a peer id
-        if (!braid_blob.peer)
-            try {
-                braid_blob.peer = await fs.promises.readFile(`${braid_blob.db_folder}/peer.txt`, 'utf8')
-            } catch (e) {}
-        if (!braid_blob.peer)
-            braid_blob.peer = Math.random().toString(36).slice(2)
-        await fs.promises.writeFile(`${braid_blob.db_folder}/peer.txt`, braid_blob.peer)
+            // Create url-file-db instance for meta storage (in a subfolder)
+            // This will create both meta_folder and the db subfolder with recursive: true
+            braid_blob.meta_db = await url_file_db.create(`${braid_blob.meta_folder}/db`)
+
+            // establish a peer id (stored at root of meta_folder, sibling to db subfolder)
+            if (!braid_blob.peer)
+                try {
+                    braid_blob.peer = await fs.promises.readFile(`${braid_blob.meta_folder}/peer.txt`, 'utf8')
+                } catch (e) {}
+            if (!braid_blob.peer)
+                braid_blob.peer = Math.random().toString(36).slice(2)
+            await fs.promises.writeFile(`${braid_blob.meta_folder}/peer.txt`, braid_blob.peer)
+        }
+    }
+
+    braid_blob.put = async (key, body, options = {}) => {
+        await braid_blob.init()
+
+        // Read the meta data from meta_db
+        var meta = {}
+        var meta_content = await braid_blob.meta_db.read(key)
+        if (meta_content)
+            meta = JSON.parse(meta_content.toString('utf8'))
+
+        var their_e =
+            !options.version ?
+                // we'll give them a event id in this case
+                `${braid_blob.peer}-${Math.max(Date.now(),
+                    meta.event ? 1*get_event_seq(meta.event) + 1 : -Infinity)}` :
+            !options.version.length ?
+                null :
+            options.version[0]
+
+        if (their_e != null &&
+            (meta.event == null ||
+                compare_events(their_e, meta.event) > 0)) {
+            meta.event = their_e
+
+            // Write the file using url-file-db (unless skip_write is set)
+            if (!options.skip_write)
+                await braid_blob.db.write(key, body)
+
+            // Write the meta data
+            if (options.content_type)
+                meta.content_type = options.content_type
+
+            await braid_blob.meta_db.write(key, JSON.stringify(meta))
+
+            // Notify all subscriptions of the update
+            // (except the peer which made the PUT request itself)
+            if (braid_blob.key_to_subs[key])
+                for (var [peer, sub] of braid_blob.key_to_subs[key].entries())
+                    if (peer !== options.peer)
+                        sub.sendUpdate({
+                            version: [meta.event],
+                            'Merge-Type': 'lww',
+                            body
+                        })
+        }
+
+        return meta.event
+    }
+
+    braid_blob.get = async (key, options = {}) => {
+        await braid_blob.init()
+
+        // Read the meta data from meta_db
+        var meta = {}
+        var meta_content = await braid_blob.meta_db.read(key)
+        if (meta_content)
+            meta = JSON.parse(meta_content.toString('utf8'))
+        if (meta.event == null) return null
+
+        var result = {
+            version: [meta.event],
+            content_type: meta.content_type
+        }
+        if (options.header_cb) await options.header_cb(result)
+        if (options.head) return
+
+        if (options.subscribe) {
+            var subscribe_chain = Promise.resolve()
+            options.my_subscribe = (x) => subscribe_chain =
+                subscribe_chain.then(() => options.subscribe(x))
+
+            // Start a subscription for future updates
+            if (!braid_blob.key_to_subs[key])
+                braid_blob.key_to_subs[key] = new Map()
+
+            var peer = options.peer || Math.random().toString(36).slice(2)
+            braid_blob.key_to_subs[key].set(peer, {
+                sendUpdate: (update) => {
+                    options.my_subscribe({
+                        body: update.body,
+                        version: update.version,
+                        content_type: meta.content_type
+                    })
+                }
+            })
+
+            // Store unsubscribe function
+            result.unsubscribe = () => {
+                braid_blob.key_to_subs[key].delete(peer)
+                if (!braid_blob.key_to_subs[key].size)
+                    delete braid_blob.key_to_subs[key]
+            }
+
+            if (options.before_send_cb) await options.before_send_cb(result)
+
+            // Send an immediate update if needed
+            if (!options.parents ||
+                !options.parents.length ||
+                compare_events(result.version[0], options.parents[0]) > 0) {
+                result.sent = true
+                options.my_subscribe({
+                    body: await braid_blob.db.read(key),
+                    version: result.version,
+                    content_type: result.content_type
+                })
+            }
+        } else {
+            // If not subscribe, send the body now
+            result.body = await braid_blob.db.read(key)
+        }
+
+        return result
     }
 
     braid_blob.serve = async (req, res, options = {}) => {
         await braid_blob.init()
 
-        if (!options.key) options.key = decodeURIComponent(req.url.split('?')[0])
+        if (!options.key) options.key = url_file_db.get_key(req.url)
 
         braidify(req, res)
         if (res.is_multiplexer) return
@@ -41,116 +171,69 @@ function create_braid_blob() {
         var body = req.method === 'PUT' && await slurp(req)
 
         await within_fiber(options.key, async () => {
-            const filename = `${braid_blob.db_folder}/blob/${encode_filename(options.key)}`
-            const metaname = `${braid_blob.db_folder}/meta/${encode_filename(options.key)}`
-
-            // Read the meta file
+            // Read the meta data from meta_db
             var meta = {}
-            try {
-                meta = JSON.parse(await fs.promises.readFile(metaname, 'utf8'))
-            } catch (e) {}
+            var meta_content = await braid_blob.meta_db.read(options.key)
+            if (meta_content)
+                meta = JSON.parse(meta_content.toString('utf8'))
 
             if (req.method === 'GET') {
-                // Handle GET request for binary files
-
-                if (meta.event == null) {
+                if (!res.hasHeader("editable")) res.setHeader("Editable", "true")
+                if (!req.subscribe) res.setHeader("Accept-Subscribe", "true")
+                res.setHeader("Merge-Type", "lww")
+
+                var result = await braid_blob.get(options.key, {
+                    peer: req.peer,
+                    head: req.method == "HEAD",
+                    parents: req.parents || null,
+                    header_cb: (result) => {
+                        res.setHeader((req.subscribe ? "Current-" : "") +
+                            "Version", ascii_ify(result.version.map((x) =>
+                                JSON.stringify(x)).join(", ")))
+                        if (result.content_type)
+                            res.setHeader('Content-Type', result.content_type)
+                    },
+                    before_send_cb: (result) =>
+                        res.startSubscription({ onClose: result.unsubscribe }),
+                    subscribe: req.subscribe ? (update) => {
+                        res.sendUpdate({
+                            version: update.version,
+                            'Merge-Type': 'lww',
+                            body: update.body
+                        })
+                    } : null
+                })
+
+                if (!result) {
                     res.statusCode = 404
-                    res.setHeader('Content-Type', 'text/plain')
                     return res.end('File Not Found')
                 }
 
-                if (meta.content_type && req.headers.accept &&
-                    !isAcceptable(meta.content_type, req.headers.accept)) {
+                if (result.content_type && req.headers.accept &&
+                    !isAcceptable(result.content_type, req.headers.accept)) {
                     res.statusCode = 406
-                    res.setHeader('Content-Type', 'text/plain')
-                    return res.end(`Content-Type of ${meta.content_type} not in Accept: ${req.headers.accept}`)
+                    return res.end(`Content-Type of ${result.content_type} not in Accept: ${req.headers.accept}`)
                 }
 
-                // Set Version header;
-                //   but if this is a subscription,
-                //     then we set Current-Version instead
-                res.setHeader((req.subscribe ? 'Current-' : '') + 'Version',
-                    JSON.stringify(meta.event))
-
-                // Set Content-Type
-                if (meta.content_type)
-                    res.setHeader('Content-Type', meta.content_type)
-
-                if (!req.subscribe)
-                    return res.end(await fs.promises.readFile(filename))
-
-                if (!res.hasHeader("editable"))
-                    res.setHeader("Editable", "true")
-
-                // Start a subscription for future updates.
-                if (!braid_blob.key_to_subs[options.key])
-                    braid_blob.key_to_subs[options.key] = new Map()
-                var peer = req.peer || Math.random().toString(36).slice(2)
-                braid_blob.key_to_subs[options.key].set(peer, res)
-
-                res.startSubscription({ onClose: () => {
-                    braid_blob.key_to_subs[options.key].delete(peer)
-                    if (!braid_blob.key_to_subs[options.key].size)
-                        delete braid_blob.key_to_subs[options.key]
-                }})
-
-                // Send an immediate update when:
-                if (!req.parents ||            // 1) They want everything,
-                    !req.parents.length ||     // 2) Or everything past the empty set,
-                    compare_events(meta.event, req.parents[0]) > 0
-                                            // 3) Or what we have is newer
-                )
-                    return res.sendUpdate({
-                        version: [meta.event],
-                        'Merge-Type': 'lww',
-                        body: await fs.promises.readFile(filename)
-                    })
-                else res.write('\n\n') // get the node http code to send headers
+                if (req.method == "HEAD") return res.end('')
+                else if (!req.subscribe) return res.end(result.body)
+                else {
+                    // If no immediate update was sent,
+                    // get the node http code to send headers
+                    if (!result.sent) res.write('\n\n') 
+                }
             } else if (req.method === 'PUT') {
                 // Handle PUT request to update binary files
-
-                var their_e =
-                    !req.version ?
-                        // we'll give them a event id in this case
-                        `${braid_blob.peer}-${Math.max(Date.now(),
-                            meta.event ? 1*get_event_seq(meta.event) + 1 : -Infinity)}` :
-                    !req.version.length ?
-                        null :
-                    req.version[0]
-
-                if (their_e != null &&
-                    (meta.event == null ||
-                        compare_events(their_e, meta.event) > 0)) {
-                    meta.event = their_e
-
-                    // Write the file
-                    await fs.promises.writeFile(filename, body)
-
-                    // Write the meta file
-                    if (req.headers['content-type'])
-                        meta.content_type = req.headers['content-type']
-                    await fs.promises.writeFile(metaname, JSON.stringify(meta))
-
-                    // Notify all subscriptions of the update
-                    // (except the peer which made the PUT request itself)
-                    if (braid_blob.key_to_subs[options.key])
-                        for (var [peer, sub] of braid_blob.key_to_subs[options.key].entries())
-                            if (peer !== req.peer)
-                                sub.sendUpdate({
-                                    version: [meta.event],
-                                    'Merge-Type': 'lww',
-                                    body
-                                })
-                }
-                res.setHeader("Version", meta.event != null ? JSON.stringify(meta.event) : '')
+                meta.event = await braid_blob.put(options.key, body, {
+                    version: req.version,
+                    content_type: req.headers['content-type'],
+                    peer: req.peer
+                })
+                res.setHeader("Version", version_to_header(meta.event != null ? [meta.event] : []))
                 res.end('')
             } else if (req.method === 'DELETE') {
-                try {
-                    await fs.promises.unlink(filename)
-                } catch (e) {}
-                try {
-                    await fs.promises.unlink(metaname)
-                } catch (e) {}
+                await braid_blob.db.delete(options.key)
+                await braid_blob.meta_db.delete(options.key)
                 res.statusCode = 204 // No Content
                 res.end('')
             }
@@ -176,14 +259,14 @@ function create_braid_blob() {
         return e
     }
 
-    function encode_filename(filename) {
-        // Swap all "!" and "/" characters
-        let swapped = filename.replace(/[!/]/g, (match) => (match === "!" ? "/" : "!"))
-
-        // Encode the filename using encodeURIComponent()
-        let encoded = encodeURIComponent(swapped)
+    function ascii_ify(s) {
+        return s.replace(/[^\x20-\x7E]/g, c => '\\u' + c.charCodeAt(0).toString(16).padStart(4, '0'))
+    }
 
-        return encoded
+    function version_to_header(version) {
+        // Convert version array to header format: JSON without outer brackets
+        if (!version || !version.length) return ''
+        return ascii_ify(version.map(v => JSON.stringify(v)).join(', '))
     }
 
     function within_fiber(id, func) {

package/package.json

@@ -1,11 +1,16 @@
 {
   "name": "braid-blob",
-  "version": "0.0.16",
+  "version": "0.0.18",
   "description": "Library for collaborative blobs over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-blob",
   "homepage": "https://braid.org",
+  "scripts": {
+    "test": "node test/test.js",
+    "test:browser": "node test/test.js --browser"
+  },
   "dependencies": {
-    "braid-http": "~1.3.82"
+    "braid-http": "~1.3.82",
+    "url-file-db": "~0.0.8"
   }
 }

package/server-demo.js

@@ -5,7 +5,10 @@ var braid_blob = require(`${__dirname}/index.js`)
 // TODO: set a custom storage base
 // (the default is ./braid-blob-files)
 //
-// braid_blob.storage_base = './custom_files_folder'
+// braid_blob.db_folder = './custom_files_folder'
+// braid_blob.meta_folder = './custom_meta_folder'
+
+braid_blob.init()
 
 var server = require("http").createServer(async (req, res) => {
     console.log(`${req.method} ${req.url}`)
@@ -18,5 +21,5 @@ server.listen(port, () => {
     console.log(`files stored in: ${braid_blob.db_folder}`)
 })
 
-// curl -X PUT --data-binary @image.png http://localhost:8888/image.png
-// curl http://localhost:8888/image.png --output downloaded_image.png
+// curl -X PUT -H "Content-Type: image/png" --data-binary @blob.png http://localhost:8888/blob.png
+// curl http://localhost:8888/blob.png --output new-blob.png