braid-blob 0.0.14 → 0.0.16

package/LICENSE.txt

@@ -0,0 +1,11 @@
+Invisible Property License
+
+   1. You have the right to use this IP for any purpose.
+   2. If you make profit, you agree to give back a fair share of the profit to
+      the creators of this IP.
+   3. The creators will tell you how much they think is a fair share, if your
+      usage matters to them, and promise not to take you to court to enforce
+      the agreement.
+
+(In other words, this license thus runs on the honor system.  You are invited
+to participate in our community with honor.)

package/README.md

@@ -1,5 +1,87 @@
 # braid-blob
-Blob Synchronization libraries over Braid-HTTP
+
+A simple, self-contained library for synchronizing binary blobs (files, images, etc.) over HTTP using [Braid-HTTP](https://braid.org). It provides real-time synchronization with last-write-wins (LWW) conflict resolution and persistent storage.
+
+## Quick Start
+
+### Installation
+
+```bash
+npm install braid-blob
+```
+
+### Basic Server
+
+```javascript
+var braid_blob = require('braid-blob')
+
+require('http').createServer((req, res) => {
+    braid_blob.serve(req, res)
+}).listen(8888)
+```
+
+That's it! You now have a blob synchronization server.
+
+### Usage Examples
+
+First let's upload a file:
+```bash
+curl -X PUT -H "Content-Type: image/png" -T blob.png http://localhost:8888/image.png
+```
+
+View image in browser at http://localhost:8888/image.png
+
+To see updates, let's do a textual example for easy viewing:
+
+```
+curl -X PUT -H "Content-Type: text/plain" -d "hello" http://localhost:8888/text
+```
+
+Next, subscribe for updates:
+```
+curl -H "Subscribe: true" http://localhost:8888/text
+```
+
+Now, in another terminal, write over the file:
+```bash
+curl -X PUT -H "Content-Type: text/plain" -d "world" http://localhost:8888/text
+```
+
+Should see activity in the first terminal showing the update.
+
+```
+# Delete a file
+curl -X DELETE http://localhost:8888/text
+```
+
+## API
+
+### Configuration
+
+```javascript
+var braid_blob = require('braid-blob')
+
+// Set custom storage location (default: './braid-blob-db')
+braid_blob.db_folder = './custom_files_folder'
+
+// Set custom peer ID (default: auto-generated and persisted)
+braid_blob.peer = 'my-server-id'
+```
+
+### `braid_blob.serve(req, res, options)`
+
+Handles HTTP requests for blob storage and synchronization.
+
+**Parameters:**
+- `req` - HTTP request object
+- `res` - HTTP response object
+- `options` - Optional configuration object
+  - `key` - Override the resource key (default: URL path)
+
+**Supported HTTP Methods:**
+- `GET` - Retrieve a blob (with optional `Subscribe: true` header)
+- `PUT` - Store/update a blob
+- `DELETE` - Remove a blob
 
 ## Testing
 

package/index.js

@@ -1,208 +1,244 @@
-
 var {http_server: braidify, free_cors} = require('braid-http'),
     fs = require('fs'),
     path = require('path')
 
-var braid_blob = {
-    db_folder: './braid-blob-db',
-    meta_folder: './braid-blob-meta',
-    cache: {}
-}
+function create_braid_blob() {
+    var braid_blob = {
+        db_folder: './braid-blob-db',
+        cache: {},
+        key_to_subs: {},
+        peer: null // we'll try to load this from a file, if not set by the user
+    }
 
-var key_to_subs = {}
+    braid_blob.init = async () => {
+        braid_blob.init = () => {}
 
-braid_blob.serve = async (req, res, options = {}) => {
-    if (!options.key) options.key = decodeURIComponent(req.url.split('?')[0])
+        await fs.promises.mkdir(`${braid_blob.db_folder}/blob`, { recursive: true })
+        await fs.promises.mkdir(`${braid_blob.db_folder}/meta`, { recursive: true })
 
-    braidify(req, res)
-    if (res.is_multiplexer) return
+        // establish a peer id
+        if (!braid_blob.peer)
+            try {
+                braid_blob.peer = await fs.promises.readFile(`${braid_blob.db_folder}/peer.txt`, 'utf8')
+            } catch (e) {}
+        if (!braid_blob.peer)
+            braid_blob.peer = Math.random().toString(36).slice(2)
+        await fs.promises.writeFile(`${braid_blob.db_folder}/peer.txt`, braid_blob.peer)
+    }
 
-    // Handle OPTIONS request
-    if (req.method === 'OPTIONS') return res.end();
+    braid_blob.serve = async (req, res, options = {}) => {
+        await braid_blob.init()
 
-    // consume PUT body
-    var body = req.method === 'PUT' && await slurp(req)
+        if (!options.key) options.key = decodeURIComponent(req.url.split('?')[0])
 
-    await within_fiber(options.key, async () => {
-        const filename = `${braid_blob.db_folder}/${encode_filename(options.key)}`
-        const metaname = `${braid_blob.meta_folder}/${encode_filename(options.key)}`
-        
-        // Read the meta file
-        var meta = {}
-        try {
-            meta = JSON.parse(await fs.promises.readFile(metaname, 'utf8'))
-        } catch (e) {}
-        var our_v = meta.version
-
-        if (req.method === 'GET') {
-            // Handle GET request for binary files
-
-            if (our_v == null) {
-                res.statusCode = 404
-                res.setHeader('Content-Type', 'text/plain')
-                return res.end('File Not Found')
-            }
+        braidify(req, res)
+        if (res.is_multiplexer) return
 
-            if (meta.content_type && req.headers.accept &&
-                !isAcceptable(meta.content_type, req.headers.accept)) {
-                res.statusCode = 406
-                res.setHeader('Content-Type', 'text/plain')
-                return res.end(`Content-Type of ${meta.content_type} not in Accept: ${req.headers.accept}`)
-            }
+        // Handle OPTIONS request
+        if (req.method === 'OPTIONS') return res.end();
 
-            // Set Version header;
-            //   but if this is a subscription,
-            //     then we set Current-Version instead
-            res.setHeader((req.subscribe ? 'Current-' : '') + 'Version', `"${our_v}"`)
-            
-            // Set Content-Type
-            if (meta.content_type)
-                res.setHeader('Content-Type', meta.content_type)
-                
-            if (!req.subscribe)
-                return res.end(await fs.promises.readFile(filename))
-
-            if (!res.hasHeader("editable"))
-                res.setHeader("Editable", "true")
-
-            // Start a subscription for future updates.
-            if (!key_to_subs[options.key]) key_to_subs[options.key] = new Map()
-            var peer = req.peer || Math.random().toString(36).slice(2)
-            key_to_subs[options.key].set(peer, res)
-
-            res.startSubscription({ onClose: () => {
-                key_to_subs[options.key].delete(peer)
-                if (!key_to_subs[options.key].size)
-                    delete key_to_subs[options.key]
-            }})
-
-
-            // Send an immediate update when:
-            if (!req.parents ||          // 1) They have no version history
-                                         //    (need full sync)
-                !req.parents.length ||   // 2) Or their version is the empty set
-                our_v > 1*req.parents[0] // 3) Or our version is newer
-                )
-                return res.sendUpdate({
-                    version: ['' + our_v],
-                    'Merge-Type': 'lww',
-                    body: await fs.promises.readFile(filename)
-                })
-            else res.write('\n\n') // get the node http code to send headers
-        } else if (req.method === 'PUT') {
-            // Handle PUT request to update binary files
-
-            // Ensure directory exists
-            await fs.promises.mkdir(path.dirname(filename), { recursive: true })
-            await fs.promises.mkdir(path.dirname(metaname), { recursive: true })
-
-            var their_v =
-                !req.version ?
-                    // we'll give them a version in this case
-                    Math.max(our_v != null ? our_v + 1 : 0, Date.now()) :
-                !req.version.length ?
-                    null :
-                1*req.version[0]
-
-            if (their_v != null &&
-                (our_v == null || their_v > our_v)) {
-
-                // Write the file
-                await fs.promises.writeFile(filename, body)
-
-                // Write the meta file
-                meta.version = their_v
-                if (req.headers['content-type'])
-                    meta.content_type = req.headers['content-type']
-                await fs.promises.writeFile(metaname, JSON.stringify(meta))
-
-                // Notify all subscriptions of the update
-                // (except the peer which made the PUT request itself)
-                if (key_to_subs[options.key])
-                    for (var [peer, sub] of key_to_subs[options.key].entries())
-                        if (peer !== req.peer)
-                            sub.sendUpdate({
-                                version: ['' + their_v],
-                                'Merge-Type': 'lww',
-                                body
-                            })
-
-                res.setHeader("Version", `"${their_v}"`)
-            } else {
-                res.setHeader("Version", our_v != null ? `"${our_v}"` : '')
-            }
-            res.end('')
-        } else if (req.method === 'DELETE') {
-            try {
-                await fs.promises.unlink(filename)
-            } catch (e) {}
+        // consume PUT body
+        var body = req.method === 'PUT' && await slurp(req)
+
+        await within_fiber(options.key, async () => {
+            const filename = `${braid_blob.db_folder}/blob/${encode_filename(options.key)}`
+            const metaname = `${braid_blob.db_folder}/meta/${encode_filename(options.key)}`
+
+            // Read the meta file
+            var meta = {}
             try {
-                await fs.promises.unlink(metaname)
+                meta = JSON.parse(await fs.promises.readFile(metaname, 'utf8'))
             } catch (e) {}
-            res.statusCode = 204 // No Content
-            res.end('')
-        }
-    })
-}
 
-function encode_filename(filename) {
-    // Swap all "!" and "/" characters
-    let swapped = filename.replace(/[!/]/g, (match) => (match === "!" ? "/" : "!"))
+            if (req.method === 'GET') {
+                // Handle GET request for binary files
+
+                if (meta.event == null) {
+                    res.statusCode = 404
+                    res.setHeader('Content-Type', 'text/plain')
+                    return res.end('File Not Found')
+                }
+
+                if (meta.content_type && req.headers.accept &&
+                    !isAcceptable(meta.content_type, req.headers.accept)) {
+                    res.statusCode = 406
+                    res.setHeader('Content-Type', 'text/plain')
+                    return res.end(`Content-Type of ${meta.content_type} not in Accept: ${req.headers.accept}`)
+                }
+
+                // Set Version header;
+                //   but if this is a subscription,
+                //     then we set Current-Version instead
+                res.setHeader((req.subscribe ? 'Current-' : '') + 'Version',
+                    JSON.stringify(meta.event))
+
+                // Set Content-Type
+                if (meta.content_type)
+                    res.setHeader('Content-Type', meta.content_type)
+
+                if (!req.subscribe)
+                    return res.end(await fs.promises.readFile(filename))
+
+                if (!res.hasHeader("editable"))
+                    res.setHeader("Editable", "true")
+
+                // Start a subscription for future updates.
+                if (!braid_blob.key_to_subs[options.key])
+                    braid_blob.key_to_subs[options.key] = new Map()
+                var peer = req.peer || Math.random().toString(36).slice(2)
+                braid_blob.key_to_subs[options.key].set(peer, res)
+
+                res.startSubscription({ onClose: () => {
+                    braid_blob.key_to_subs[options.key].delete(peer)
+                    if (!braid_blob.key_to_subs[options.key].size)
+                        delete braid_blob.key_to_subs[options.key]
+                }})
+
+                // Send an immediate update when:
+                if (!req.parents ||            // 1) They want everything,
+                    !req.parents.length ||     // 2) Or everything past the empty set,
+                    compare_events(meta.event, req.parents[0]) > 0
+                                            // 3) Or what we have is newer
+                )
+                    return res.sendUpdate({
+                        version: [meta.event],
+                        'Merge-Type': 'lww',
+                        body: await fs.promises.readFile(filename)
+                    })
+                else res.write('\n\n') // get the node http code to send headers
+            } else if (req.method === 'PUT') {
+                // Handle PUT request to update binary files
+
+                var their_e =
+                    !req.version ?
+                        // we'll give them a event id in this case
+                        `${braid_blob.peer}-${Math.max(Date.now(),
+                            meta.event ? 1*get_event_seq(meta.event) + 1 : -Infinity)}` :
+                    !req.version.length ?
+                        null :
+                    req.version[0]
+
+                if (their_e != null &&
+                    (meta.event == null ||
+                        compare_events(their_e, meta.event) > 0)) {
+                    meta.event = their_e
+
+                    // Write the file
+                    await fs.promises.writeFile(filename, body)
+
+                    // Write the meta file
+                    if (req.headers['content-type'])
+                        meta.content_type = req.headers['content-type']
+                    await fs.promises.writeFile(metaname, JSON.stringify(meta))
+
+                    // Notify all subscriptions of the update
+                    // (except the peer which made the PUT request itself)
+                    if (braid_blob.key_to_subs[options.key])
+                        for (var [peer, sub] of braid_blob.key_to_subs[options.key].entries())
+                            if (peer !== req.peer)
+                                sub.sendUpdate({
+                                    version: [meta.event],
+                                    'Merge-Type': 'lww',
+                                    body
+                                })
+                }
+                res.setHeader("Version", meta.event != null ? JSON.stringify(meta.event) : '')
+                res.end('')
+            } else if (req.method === 'DELETE') {
+                try {
+                    await fs.promises.unlink(filename)
+                } catch (e) {}
+                try {
+                    await fs.promises.unlink(metaname)
+                } catch (e) {}
+                res.statusCode = 204 // No Content
+                res.end('')
+            }
+        })
+    }
 
-    // Encode the filename using encodeURIComponent()
-    let encoded = encodeURIComponent(swapped)
+    function compare_events(a, b) {
+        var a_num = get_event_seq(a)
+        var b_num = get_event_seq(b)
 
-    return encoded
-}
+        var c = a_num.length - b_num.length
+        if (c) return c
 
-function within_fiber(id, func) {
-    if (!within_fiber.chains) within_fiber.chains = {}
-    var prev = within_fiber.chains[id] || Promise.resolve()
-    var curr = prev.then(async () => {
-        try {
-            return await func()
-        } finally {
-            if (within_fiber.chains[id] === curr)
-                delete within_fiber.chains[id]
-        }
-    })
-    return within_fiber.chains[id] = curr
-}
+        var c = a_num.localeCompare(b_num)
+        if (c) return c
 
-async function slurp(req) {
-    return await new Promise(done => {
-        var chunks = []
-        req.on('data', chunk => chunks.push(chunk))
-        req.on('end', () => done(Buffer.concat(chunks)))
-    })
-}
+        return a.localeCompare(b)
+    }
+
+    function get_event_seq(e) {
+        for (let i = e.length - 1; i >= 0; i--)
+            if (e[i] === '-') return e.slice(i + 1)
+        return e
+    }
+
+    function encode_filename(filename) {
+        // Swap all "!" and "/" characters
+        let swapped = filename.replace(/[!/]/g, (match) => (match === "!" ? "/" : "!"))
+
+        // Encode the filename using encodeURIComponent()
+        let encoded = encodeURIComponent(swapped)
+
+        return encoded
+    }
+
+    function within_fiber(id, func) {
+        if (!within_fiber.chains) within_fiber.chains = {}
+        var prev = within_fiber.chains[id] || Promise.resolve()
+        var curr = prev.then(async () => {
+            try {
+                return await func()
+            } finally {
+                if (within_fiber.chains[id] === curr)
+                    delete within_fiber.chains[id]
+            }
+        })
+        return within_fiber.chains[id] = curr
+    }
 
-function isAcceptable(contentType, acceptHeader) {
-    // If no Accept header or Accept is */*, accept everything
-    if (!acceptHeader || acceptHeader === '*/*') return true;
-    
-    // Parse the Accept header into individual media types
-    const acceptTypes = acceptHeader.split(',').map(type => type.trim());
-    
-    for (const acceptType of acceptTypes) {
-        // Remove quality values (e.g., "text/html;q=0.9" -> "text/html")
-        const cleanAcceptType = acceptType.split(';')[0].trim();
+    async function slurp(req) {
+        return await new Promise(done => {
+            var chunks = []
+            req.on('data', chunk => chunks.push(chunk))
+            req.on('end', () => done(Buffer.concat(chunks)))
+        })
+    }
+
+    function isAcceptable(contentType, acceptHeader) {
+        // If no Accept header or Accept is */*, accept everything
+        if (!acceptHeader || acceptHeader === '*/*') return true;
         
-        // Exact match
-        if (cleanAcceptType === contentType) return true;
+        // Parse the Accept header into individual media types
+        const acceptTypes = acceptHeader.split(',').map(type => type.trim());
         
-        // Wildcard subtype match (e.g., "image/*" matches "image/png")
-        if (cleanAcceptType.endsWith('/*')) {
-            const acceptMain = cleanAcceptType.slice(0, -2);
-            const contentMain = contentType.split('/')[0];
-            if (acceptMain === contentMain) return true;
+        for (const acceptType of acceptTypes) {
+            // Remove quality values (e.g., "text/html;q=0.9" -> "text/html")
+            const cleanAcceptType = acceptType.split(';')[0].trim();
+            
+            // Exact match
+            if (cleanAcceptType === contentType) return true;
+            
+            // Wildcard subtype match (e.g., "image/*" matches "image/png")
+            if (cleanAcceptType.endsWith('/*')) {
+                const acceptMain = cleanAcceptType.slice(0, -2);
+                const contentMain = contentType.split('/')[0];
+                if (acceptMain === contentMain) return true;
+            }
+            
+            // Full wildcard
+            if (cleanAcceptType === '*/*') return true;
         }
         
-        // Full wildcard
-        if (cleanAcceptType === '*/*') return true;
+        return false;
     }
-    
-    return false;
+
+    braid_blob.create_braid_blob = create_braid_blob
+
+    return braid_blob
 }
 
-module.exports = braid_blob
+module.exports = create_braid_blob()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-blob",
-  "version": "0.0.14",
+  "version": "0.0.16",
   "description": "Library for collaborative blobs over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-blob",

package/test/server.js

@@ -4,7 +4,6 @@ var port = process.argv[2] || 8889
 var braid_blob = require(`${__dirname}/../index.js`)
 var {free_cors} = require("braid-http")
 braid_blob.db_folder = `${__dirname}/test_db_folder`
-braid_blob.meta_folder = `${__dirname}/test_meta_folder`
 
 var server = require("http").createServer(async (req, res) => {
     console.log(`${req.method} ${req.url}`)

package/test/test.html

@@ -89,6 +89,152 @@ async function runTest(testName, testFunction, expectedResult) {
     }
 }
 
+runTest(
+    "test that peer.txt gets initialized on a fresh run",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var db_name = 'test-db-' + Math.random().toString(36).slice(2)
+
+                var new_bb = braid_blob.create_braid_blob()
+                new_bb.db_folder = __dirname + '/' + db_name
+
+                try {
+                    await new_bb.serve({}, {})
+                } catch (e) {}
+
+                await require('fs').promises.rm(new_bb.db_folder,
+                    { recursive: true, force: true })
+
+                res.end(new_bb.peer)
+
+            })()`
+        })
+        return '' + ((await r1.text()).length > 5)
+    },
+    'true'
+)
+
+runTest(
+    "test that peer is same the second time we run from same db folder",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var db = __dirname + '/test-db-' + Math.random().toString(36).slice(2)
+
+                var bb1 = braid_blob.create_braid_blob()
+                bb1.db_folder = db
+
+                try {
+                    await bb1.serve({}, {})
+                } catch (e) {}
+
+                var bb2 = braid_blob.create_braid_blob()
+                bb2.db_folder = db
+
+                try {
+                    await bb2.serve({}, {})
+                } catch (e) {}
+
+                await require('fs').promises.rm(db, { recursive: true, force: true })
+
+                res.end('' + (bb1.peer === bb2.peer))
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test that we can set the peer of a braid_blob object",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var db = __dirname + '/test-db-' + Math.random().toString(36).slice(2)
+
+                var bb1 = braid_blob.create_braid_blob()
+                bb1.db_folder = db
+                bb1.peer = 'test_peer'
+
+                try {
+                    await bb1.serve({}, {})
+                } catch (e) {}
+
+                await require('fs').promises.rm(db, { recursive: true, force: true })
+
+                res.end(bb1.peer)
+            })()`
+        })
+        return await r1.text()
+    },
+    'test_peer'
+)
+
+runTest(
+    "test that PUTing with shorter event id doesn't do anything.",
+    async () => {
+        var key = 'test-' + Math.random().toString(36).slice(2)
+
+        var r = await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            version: ['11'],
+            body: 'xyz'
+        })
+        if (!r.ok) throw 'got: ' + r.statusCode
+
+        var r = await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            version: ['9'],
+            body: 'abc'
+        })
+        if (!r.ok) throw 'got: ' + r.statusCode
+
+        var r = await braid_fetch(`/${key}`)
+        return await r.text()
+    },
+    'xyz'
+)
+
+runTest(
+    "test that we ignore stuff after the ? in a url",
+    async () => {
+        var key = 'test-' + Math.random().toString(36).slice(2)
+
+        var r = await braid_fetch(`/${key}?blah`, {
+            method: 'PUT',
+            version: ['11'],
+            body: 'yo!'
+        })
+        if (!r.ok) throw 'got: ' + r.statusCode
+
+        var r = await braid_fetch(`/${key}`)
+        return await r.text()
+    },
+    'yo!'
+)
+
+runTest(
+    "test that we ignore stuff after the # in a url",
+    async () => {
+        var key = 'test-' + Math.random().toString(36).slice(2)
+
+        var r = await braid_fetch(`/${key}#blah?bloop`, {
+            method: 'PUT',
+            version: ['11'],
+            body: 'hi!'
+        })
+        if (!r.ok) throw 'got: ' + r.statusCode
+
+        var r = await braid_fetch(`/${key}`)
+        return await r.text()
+    },
+    'hi!'
+)
+
 runTest(
     "test send an update to another peer",
     async () => {
@@ -130,6 +276,80 @@ runTest(
     'abc'
 )
 
+runTest(
+    "test having multiple subs",
+    async () => {
+        var key = 'test-' + Math.random().toString(36).slice(2)
+        var key2 = 'test2-' + Math.random().toString(36).slice(2)
+
+        var r = await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            headers: {'Content-Type': 'text/plain'},
+            version: ['1'],
+            body: 'xyz'
+        })
+        if (!r.ok) throw 'got: ' + r.statusCode
+
+        var a = new AbortController()
+        var r = await braid_fetch(`/${key}`, {
+            signal: a.signal,
+            subscribe: true,
+            peer: key
+        })
+
+        var p = new Promise(done => {
+            r.subscribe(update => {
+                if (update.version?.[0] !== '2') return
+                done(update.body_text)
+            })
+        })
+
+        var r = await braid_fetch(`/${key2}`, {
+            method: 'PUT',
+            headers: {'Content-Type': 'text/plain'},
+            version: ['1'],
+            body: 'xyz2'
+        })
+        if (!r.ok) throw 'got: ' + r.statusCode
+
+        var a = new AbortController()
+        var r = await braid_fetch(`/${key2}`, {
+            signal: a.signal,
+            subscribe: true,
+            peer: key2
+        })
+
+        var p2 = new Promise(done => {
+            r.subscribe(update => {
+                if (update.version?.[0] !== '2') return
+                done(update.body_text)
+            })
+        })
+
+        var r = await braid_fetch(`/${key}`, {
+            method: 'PUT',
+            headers: {'Content-Type': 'text/plain'},
+            version: ['2'],
+            body: 'abc'
+        })
+        if (!r.ok) throw 'got: ' + r.statusCode
+
+        var r = await braid_fetch(`/${key2}`, {
+            method: 'PUT',
+            headers: {'Content-Type': 'text/plain'},
+            version: ['2'],
+            body: 'abc2'
+        })
+        if (!r.ok) throw 'got: ' + r.statusCode
+
+        var ret = await Promise.all([p, p2])
+        a.abort()
+        return 'got: ' + ret
+
+    },
+    'got: abc,abc2'
+)
+
 runTest(
     "test getting a 406",
     async () => {