braid-blob 0.0.31 → 0.0.33

package/AI-README.md

@@ -103,7 +103,7 @@ COMPARISON_RULES:
   1. Compare numeric length of timestamp
   2. If equal, lexicographic compare timestamp
   3. If equal, lexicographic compare full version string
-MERGE_TYPE: last-write-wins (lww)
+MERGE_TYPE: arbitrary-writer-wins (aww)
 ```
 
 ## METADATA_SCHEMA
@@ -145,7 +145,7 @@ SIDE_EFFECTS:
 VERSION_LOGIC:
   - If options.version provided: use options.version[0]
   - Else: generate "{peer}-{max(Date.now(), last_version_seq+1)}"
-  - Only write if new version > existing version (lww)
+  - Only write if new version > existing version (aww)
 ```
 
 ### get(key, options)
@@ -320,7 +320,7 @@ HTTP_HEADERS:
     Current-Version: "v1"            # for subscribed GET
     Editable: true
     Accept-Subscribe: true
-    Merge-Type: lww
+    Merge-Type: aww
     Content-Type: mime/type
 
 STATUS_CODES:
@@ -332,7 +332,7 @@ STATUS_CODES:
 
 BRAID_UPDATE_FORMAT:
   Version: "v1"\r\n
-  Merge-Type: lww\r\n
+  Merge-Type: aww\r\n
   Content-Length: N\r\n
   \r\n
   {body}

package/README.md

@@ -1,6 +1,6 @@
 # braid-blob
 
-A simple, self-contained library for synchronizing binary blobs (files, images, etc.) over HTTP using [Braid-HTTP](https://braid.org). It provides real-time synchronization with last-write-wins (LWW) conflict resolution and persistent storage.
+A simple, self-contained library for synchronizing binary blobs (files, images, etc.) over HTTP using [Braid-HTTP](https://braid.org). It provides real-time synchronization with arbitrary-writer-wins (AWW) conflict resolution and persistent storage.
 
 ## Quick Start
 

package/index.js

@@ -1,5 +1,6 @@
 var {http_server: braidify, fetch: braid_fetch} = require('braid-http'),
     {url_file_db} = require('url-file-db'),
+    fs = require('fs'),
     path = require('path')
 
 function create_braid_blob() {
@@ -7,9 +8,10 @@ function create_braid_blob() {
         db_folder: './braid-blob-db',
         meta_folder: './braid-blob-meta',
         cache: {},
+        meta_cache: {},
         key_to_subs: {},
         peer: null, // will be auto-generated if not set by the user
-        db: null // url-file-db instance with integrated meta storage
+        db: null // url-file-db instance
     }
 
     braid_blob.init = async () => {
@@ -19,10 +21,17 @@ function create_braid_blob() {
         await braid_blob.init()
 
         async function real_init() {
-            // Create url-file-db instance with integrated meta storage
+            // Ensure our meta folder exists
+            await fs.promises.mkdir(braid_blob.meta_folder, { recursive: true })
+
+            // Create a fake meta folder for url-file-db (we manage our own meta)
+            var fake_meta_folder = braid_blob.meta_folder + '-fake'
+            await fs.promises.mkdir(fake_meta_folder, { recursive: true })
+
+            // Create url-file-db instance (with fake meta folder - we manage our own)
             braid_blob.db = await url_file_db.create(
                 braid_blob.db_folder,
-                braid_blob.meta_folder,
+                fake_meta_folder,
                 async (db, key) => {
                     // File changed externally, notify subscriptions
                     // Use db parameter instead of braid_blob.db to avoid race condition
@@ -37,6 +46,37 @@ function create_braid_blob() {
         }
     }
 
+    function get_meta(key) {
+        if (braid_blob.meta_cache[key]) return braid_blob.meta_cache[key]
+        var meta_path = path.join(braid_blob.meta_folder, encode_filename(key))
+        try {
+            var data = fs.readFileSync(meta_path, 'utf8')
+            braid_blob.meta_cache[key] = JSON.parse(data)
+            return braid_blob.meta_cache[key]
+        } catch (e) {
+            if (e.code === 'ENOENT') return null
+            throw e
+        }
+    }
+
+    async function update_meta(key, updates) {
+        var meta = get_meta(key) || {}
+        Object.assign(meta, updates)
+        braid_blob.meta_cache[key] = meta
+        var meta_path = path.join(braid_blob.meta_folder, encode_filename(key))
+        await fs.promises.writeFile(meta_path, JSON.stringify(meta))
+    }
+
+    async function delete_meta(key) {
+        delete braid_blob.meta_cache[key]
+        var meta_path = path.join(braid_blob.meta_folder, encode_filename(key))
+        try {
+            await fs.promises.unlink(meta_path)
+        } catch (e) {
+            if (e.code !== 'ENOENT') throw e
+        }
+    }
+
     braid_blob.put = async (key, body, options = {}) => {
         // Handle URL case - make a remote PUT request
         if (key instanceof URL) {
@@ -59,8 +99,7 @@ function create_braid_blob() {
         await braid_blob.init()
         if (options.signal?.aborted) return
 
-        // Read the meta data using new meta API
-        var meta = braid_blob.db.get_meta(key) || {}
+        var meta = get_meta(key) || {}
 
         var their_e =
             !options.version ?
@@ -86,7 +125,7 @@ function create_braid_blob() {
             if (options.content_type)
                 meta_updates.content_type = options.content_type
 
-            await braid_blob.db.update_meta(key, meta_updates)
+            await update_meta(key, meta_updates)
             if (options.signal?.aborted) return
 
             // Notify all subscriptions of the update
@@ -134,8 +173,7 @@ function create_braid_blob() {
 
         await braid_blob.init()
 
-        // Read the meta data using new meta API
-        var meta = braid_blob.db.get_meta(key) || {}
+        var meta = get_meta(key) || {}
         if (meta.event == null) return null
 
         var result = {
@@ -217,8 +255,9 @@ function create_braid_blob() {
         await braid_blob.init()
         if (options.signal?.aborted) return
 
-        // Delete the file from the database
+        // Delete the file and its metadata
         await braid_blob.db.delete(key)
+        await delete_meta(key)
 
         // TODO: notify subscribers of deletion once we have a protocol for that
         // For now, just clean up the subscriptions
@@ -416,17 +455,6 @@ function create_braid_blob() {
                     }
 
                     // Remote -> local: subscribe to remote updates
-                    // We need both: remote_res (for Editable header) and local file exists
-                    var got_remote_res, got_local_file
-                    var remote_res_promise = new Promise(done => got_remote_res = done)
-                    var local_file_promise = new Promise(done => got_local_file = done)
-
-                    // Apply read-only once we have both remote response and local file
-                    Promise.all([remote_res_promise, local_file_promise]).then(async () => {
-                        var read_only = remote_res.headers?.get('editable') === 'false'
-                        await braid_blob.db.set_read_only(a, read_only)
-                    })
-
                     var b_ops = {
                         signal: ac.signal,
                         dont_retry: true,
@@ -435,7 +463,6 @@ function create_braid_blob() {
                                 version: update.version,
                                 content_type: update.headers?.['content-type']
                             })
-                            got_local_file()
                             remote_first_put()
                         },
                         on_error: handle_error
@@ -446,15 +473,11 @@ function create_braid_blob() {
                     }
 
                     // Set up both subscriptions, handling cases where one doesn't exist yet
-                    braid_blob.get(a, a_ops).then(x => {
-                        if (x) got_local_file()
-                        else remote_first_put_promise.then(() =>
-                            braid_blob.get(a, a_ops))
-                    })
+                    braid_blob.get(a, a_ops).then(x =>
+                        x || remote_first_put_promise.then(() =>
+                            braid_blob.get(a, a_ops)))
 
-                    // Get the response to check Editable header
                     var remote_res = await braid_blob.get(b, b_ops)
-                    if (remote_res) got_remote_res()
 
                     // If remote doesn't exist yet, wait for it to be created then reconnect
                     if (!remote_res) {
@@ -550,6 +573,35 @@ function create_braid_blob() {
         return false;
     }
 
+    function encode_filename(s) {
+        // Deal with case insensitivity
+        var bits = s.match(/\p{L}/ug).
+            map(c => +(c === c.toUpperCase())).join('')
+        var postfix = BigInt('0b0' + bits).toString(16)
+
+        // Swap ! and /
+        s = s.replace(/[\/!]/g, x => x === '/' ? '!' : '/')
+
+        // Encode characters that are unsafe on various filesystems:
+        //   < > : " / \ | ? *  - Windows restrictions
+        //   %                  - Reserved for encoding
+        //   \x00-\x1f, \x7f    - Control characters
+        s = s.replace(/[<>:"/|\\?*%\x00-\x1f\x7f]/g, encode_char)
+
+        // Deal with windows reserved words
+        if (s.match(/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$/i))
+            s = s.slice(0, 2) + encode_char(s[2]) + s.slice(3)
+
+        // Deal with case insensitivity
+        s += '.' + postfix
+
+        return s
+
+        function encode_char(char) {
+            return '%' + char.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0')
+        }
+    }
+
     braid_blob.create_braid_blob = create_braid_blob
 
     return braid_blob

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-blob",
-  "version": "0.0.31",
+  "version": "0.0.33",
   "description": "Library for collaborative blobs over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-blob",

package/test/tests.js

@@ -1021,8 +1021,8 @@ runTest(
 runTest(
     "test sync two local keys",
     async () => {
-        var key1 = 'test-sync-local1-' + Math.random().toString(36).slice(2)
-        var key2 = 'test-sync-local2-' + Math.random().toString(36).slice(2)
+        var key1 = '/test-sync-local1-' + Math.random().toString(36).slice(2)
+        var key2 = '/test-sync-local2-' + Math.random().toString(36).slice(2)
 
         var r1 = await braid_fetch(`/eval`, {
             method: 'POST',
@@ -1049,7 +1049,7 @@ runTest(
         await new Promise(done => setTimeout(done, 100))
 
         // Check second key has the content
-        var r = await braid_fetch(`/${key2}`)
+        var r = await braid_fetch(`${key2}`)
         return await r.text()
     },
     'sync local content'
@@ -1058,11 +1058,11 @@ runTest(
 runTest(
     "test sync remote to local (swap)",
     async () => {
-        var local_key = 'test-sync-swap-local-' + Math.random().toString(36).slice(2)
-        var remote_key = 'test-sync-swap-remote-' + Math.random().toString(36).slice(2)
+        var local_key = '/test-sync-swap-local-' + Math.random().toString(36).slice(2)
+        var remote_key = '/test-sync-swap-remote-' + Math.random().toString(36).slice(2)
 
         // Put something on the server first
-        await braid_fetch(`/${remote_key}`, {
+        await braid_fetch(`${remote_key}`, {
             method: 'PUT',
             version: ['800'],
             body: 'remote content'
@@ -1073,7 +1073,7 @@ runTest(
             body: `void (async () => {
                 try {
                     var braid_blob = require(\`\${__dirname}/../index.js\`)
-                    var remote_url = new URL('http://localhost:' + req.socket.localPort + '/${remote_key}')
+                    var remote_url = new URL('http://localhost:' + req.socket.localPort + '${remote_key}')
 
                     // Start sync with URL as first argument (should swap internally)
                     braid_blob.sync(remote_url, '${local_key}')
@@ -1091,7 +1091,7 @@ runTest(
         await new Promise(done => setTimeout(done, 100))
 
         // Check local key has the remote content
-        var r = await braid_fetch(`/${local_key}`)
+        var r = await braid_fetch(`${local_key}`)
         return await r.text()
     },
     'remote content'
@@ -1147,101 +1147,6 @@ runTest(
     'shared content'
 )
 
-runTest(
-    "test sync readonly remote to nonexistent local",
-    async () => {
-        var local_key = 'test-sync-readonly-local-' + Math.random().toString(36).slice(2)
-        var remote_key = 'test-sync-readonly-remote-' + Math.random().toString(36).slice(2)
-
-        // Put something on the server first with Editable: false
-        var r1 = await braid_fetch(`/eval`, {
-            method: 'POST',
-            body: `void (async () => {
-                try {
-                    // Create a custom handler that sets Editable: false
-                    req.method = 'PUT'
-                    req.version = ['100']
-                    req.body = Buffer.from('readonly content')
-                    await braid_blob.put('${remote_key}', req.body, { version: req.version })
-                    res.end('created')
-                } catch (e) {
-                    res.end('error: ' + e.message + ' ' + e.stack)
-                }
-            })()`
-        })
-        var result = await r1.text()
-        if (result.startsWith('error:')) return result
-
-        // Now sync to a local key that doesn't exist, with a readonly remote
-        var r2 = await braid_fetch(`/eval`, {
-            method: 'POST',
-            body: `void (async () => {
-                try {
-                    var fs = require('fs').promises
-                    var test_id = 'test-sync-readonly-' + Math.random().toString(36).slice(2)
-                    var db_folder = __dirname + '/' + test_id + '-db'
-                    var meta_folder = __dirname + '/' + test_id + '-meta'
-
-                    var bb = braid_blob.create_braid_blob()
-                    bb.db_folder = db_folder
-                    bb.meta_folder = meta_folder
-
-                    // Create a mock server URL that returns Editable: false
-                    var http = require('http')
-                    var {http_server: braidify} = require('braid-http')
-
-                    var mock_server = http.createServer((req2, res2) => {
-                        braidify(req2, res2)
-                        res2.setHeader('Editable', 'false')
-                        braid_blob.serve(req2, res2, { key: '${remote_key}' })
-                    })
-
-                    await new Promise(resolve => mock_server.listen(0, resolve))
-                    var port = mock_server.address().port
-
-                    var remote_url = new URL('http://localhost:' + port + '/${remote_key}')
-                    var ac = new AbortController()
-
-                    // Start sync - local key doesn't exist yet
-                    bb.sync('${local_key}', remote_url, { signal: ac.signal })
-
-                    // Wait for sync to happen
-                    await new Promise(done => setTimeout(done, 500))
-
-                    // Stop sync
-                    ac.abort()
-                    mock_server.close()
-
-                    // Check if local file exists and has the content
-                    var result = await bb.get('${local_key}')
-
-                    var response
-                    if (!result) {
-                        response = 'file not created'
-                    } else if (result.body.toString() !== 'readonly content') {
-                        response = 'wrong content: ' + result.body.toString()
-                    } else {
-                        // Check if the file is actually read-only
-                        await bb.init()
-                        var is_readonly = await bb.db.is_read_only('${local_key}')
-                        response = is_readonly ? 'synced and readonly' : 'synced but NOT readonly'
-                    }
-
-                    // Clean up
-                    await fs.rm(db_folder, { recursive: true, force: true })
-                    await fs.rm(meta_folder, { recursive: true, force: true })
-
-                    res.end(response)
-                } catch (e) {
-                    res.end('error: ' + e.message + ' ' + e.stack)
-                }
-            })()`
-        })
-        return await r2.text()
-    },
-    'synced and readonly'
-)
-
 runTest(
     "test sync does not disconnect unnecessarily",
     async () => {