braid-blob 0.0.40 → 0.0.42

package/AI-README.md

@@ -54,11 +54,6 @@ DEPENDENCY_UPDATES:
   - Format: "updates to {package}@{version}"
   - Example: "0.0.18 - ... updates to url-file-db 0.0.8"
 
-RESOLVED_ISSUES:
-  - url-file-db < 0.0.13: Bug where reading non-existent files returned "index" file contents
-    - Fixed in url-file-db 0.0.13+ (properly returns null for non-existent files)
-    - Caused "test sync local to remote" to fail with unexpected "shared content"
-    - url-file-db 0.0.15+ also relaxed path requirements (no longer requires leading "/")
 ```
 
 ## MODULE_STRUCTURE
@@ -67,7 +62,7 @@ RESOLVED_ISSUES:
 EXPORT: create_braid_blob() -> braid_blob_instance
 MODULE_TYPE: CommonJS
 MAIN_ENTRY: index.js
-DEPENDENCIES: [braid-http, url-file-db, fs, path]
+DEPENDENCIES: [braid-http, fs]
 ```
 
 ## DATA_MODEL
@@ -81,9 +76,9 @@ braid_blob_instance = {
 
   // Runtime state
   cache: object                                   // internal cache
+  meta_cache: object                              // metadata cache
   key_to_subs: Map<key, Map<peer, subscription>> // subscription tracking
-  db: url_file_db_instance                        // blob storage backend
-  meta_db: url_file_db_instance                   // metadata storage backend
+  db: {read, write, delete}                       // blob storage backend (auto-created or custom)
 
   // Methods
   init: async () -> void
@@ -130,6 +125,7 @@ INPUT:
     content_type?: string         // MIME type
     peer?: string                 // peer identifier
     skip_write?: boolean          // skip disk write (for external changes)
+    db?: {read, write, delete}    // custom db backend (overrides braid_blob.db)
     signal?: AbortSignal          // for URL mode
     headers?: object              // for URL mode
   }
@@ -137,8 +133,8 @@ INPUT:
 OUTPUT: version_string
 
 SIDE_EFFECTS:
-  - Writes blob to db_folder via url-file-db
-  - Writes metadata to meta_folder/db
+  - Writes blob to db (options.db or braid_blob.db)
+  - Writes metadata to meta_folder
   - Notifies active subscriptions (except originating peer)
   - If key instanceof URL: makes remote HTTP PUT via braid_fetch
 
@@ -163,6 +159,7 @@ INPUT:
     parents?: [version]           // fork-point for subscriptions
     version?: [version]           // request specific version
     peer?: string                 // peer identifier
+    db?: {read, write, delete}    // custom db backend (overrides braid_blob.db)
     signal?: AbortSignal          // for URL mode
     dont_retry?: boolean          // for URL mode subscriptions
   }
@@ -294,14 +291,11 @@ isAcceptable(contentType, acceptHeader) -> boolean
 
 ```
 db_folder/
-  {url_file_db structure}
-  - Blob data stored via url-file-db
-  - Key mapping: URL-safe encoding of keys
+  {encoded_key}       # Blob data files
+  - Key encoding: encode_filename() escapes special chars
 
 meta_folder/
-  peer.txt          # Peer ID (auto-generated if missing)
-  db/               # url-file-db for metadata
-    {encoded_key}.txt  # JSON: {event: version, content_type: mime}
+  {encoded_key}       # JSON metadata files: {event: version, content_type: mime}
 ```
 
 ## PROTOCOL_DETAILS
@@ -344,8 +338,8 @@ BRAID_UPDATE_FORMAT:
 INITIALIZATION:
   - init() called lazily by put/get/serve
   - init() runs once (subsequent calls return same promise)
-  - Creates db and meta_db url-file-db instances
-  - Loads or generates peer ID
+  - Creates db object with read/write/delete methods (or uses provided db_folder object)
+  - Generates peer ID if not set
 
 SUBSCRIPTION_MANAGEMENT:
   - key_to_subs: Map<string, Map<string, {sendUpdate}>>
@@ -354,11 +348,6 @@ SUBSCRIPTION_MANAGEMENT:
   - Prevents echo: put() doesn't notify originating peer
   - Serialized updates: subscribe_chain ensures sequential callback execution
 
-FILE_WATCHING:
-  - url-file-db monitors db_folder for external changes
-  - External changes trigger put() with skip_write: true
-  - Subscriptions notified of external changes
-
 CONCURRENCY_CONTROL:
   - within_fiber(key, fn) serializes operations per key
   - Uses promise chain stored in within_fiber.chains[key]
@@ -413,14 +402,6 @@ braid-http:
   - http_server (braidify): Adds Braid protocol support to Node.js HTTP
   - fetch (braid_fetch): Braid-aware fetch implementation
   - Handles: Subscribe headers, Version headers, streaming updates
-
-url-file-db (^0.0.15):
-  - Bidirectional URL ↔ filesystem mapping
-  - Collision-resistant encoding (case-insensitive filesystem safe)
-  - File watching for external changes
-  - Separate instances for blobs (db) and metadata (meta_db)
-  - API change in 0.0.15: use get_canonical_path() instead of url_path_to_canonical_path()
-  - Fixed in 0.0.13+: properly returns null for non-existent files (not "index" content)
 ```
 
 ## ERROR_CONDITIONS
@@ -455,7 +436,7 @@ TEST_RUNNER: test/test.js
   - Browser mode: Opens puppeteer, loads test.html
 
 TEST_SUITE: test/tests.js
-  - 40+ test cases covering:
+  - 50+ test cases covering:
     - Basic put/get operations
     - Subscriptions and updates
     - Version conflict resolution

package/index.js

@@ -11,6 +11,8 @@ function create_braid_blob() {
         db: null // object with read/write/delete methods
     }
 
+    var temp_folder = null // will be set in init
+
     braid_blob.init = async () => {
         // We only want to initialize once
         var init_p = real_init()
@@ -21,6 +23,15 @@ function create_braid_blob() {
             // Ensure our meta folder exists
             await require('fs').promises.mkdir(braid_blob.meta_folder, { recursive: true })
 
+            // Create a temp folder inside the meta folder for writing temp files,
+            // for atomic writing.
+            // The temp folder is called "temp",
+            // And this is guaranteed not to conflict with any other files,
+            // because other files are the result of encode_filename,
+            // which always ends with a ".XX" (for handling insensitive filesystems)
+            temp_folder = `${braid_blob.meta_folder}/temp`
+            await require('fs').promises.mkdir(temp_folder, { recursive: true })
+
             // Set up db - either use provided object or create file-based storage
             if (typeof braid_blob.db_folder === 'string') {
                 await require('fs').promises.mkdir(braid_blob.db_folder, { recursive: true })
@@ -36,7 +47,7 @@ function create_braid_blob() {
                     },
                     write: async (key, data) => {
                         var file_path = `${braid_blob.db_folder}/${encode_filename(key)}`
-                        await require('fs').promises.writeFile(file_path, data)
+                        await atomic_write(file_path, data, temp_folder)
                     },
                     delete: async (key) => {
                         var file_path = `${braid_blob.db_folder}/${encode_filename(key)}`
@@ -74,9 +85,8 @@ function create_braid_blob() {
     }
 
     async function save_meta(key) {
-        await require('fs').promises.writeFile(
-            `${braid_blob.meta_folder}/${encode_filename(key)}`,
-            JSON.stringify(braid_blob.meta_cache[key]))
+        await atomic_write(`${braid_blob.meta_folder}/${encode_filename(key)}`,
+            JSON.stringify(braid_blob.meta_cache[key]), temp_folder)
     }
 
     async function delete_meta(key) {
@@ -126,7 +136,7 @@ function create_braid_blob() {
                 meta.event = their_e
 
                 if (!options.skip_write)
-                    await braid_blob.db.write(key, body)
+                    await (options.db || braid_blob.db).write(key, body)
                 if (options.signal?.aborted) return
 
                 if (options.content_type)
@@ -254,12 +264,12 @@ function create_braid_blob() {
                 // Send an immediate update if needed
                 if (compare_events(result.version?.[0], options.parents?.[0]) > 0) {
                     result.sent = true
-                    result.body = await braid_blob.db.read(key)
+                    result.body = await (options.db || braid_blob.db).read(key)
                     options.my_subscribe(result)
                 }
             } else {
                 // If not subscribe, send the body now
-                result.body = await braid_blob.db.read(key)
+                result.body = await (options.db || braid_blob.db).read(key)
             }
 
             return result
@@ -294,7 +304,7 @@ function create_braid_blob() {
             var meta = await get_meta(key)
             if (options.signal?.aborted) return
 
-            await braid_blob.db.delete(key)
+            await (options.db || braid_blob.db).delete(key)
             await delete_meta(key)
 
             // Notify all subscriptions of the delete
@@ -733,6 +743,12 @@ function create_braid_blob() {
         return normalized
     }
 
+    async function atomic_write(final_destination, data, temp_folder) {
+        var temp = `${temp_folder}/${Math.random().toString(36).slice(2)}`
+        await require('fs').promises.writeFile(temp, data)
+        await require('fs').promises.rename(temp, final_destination)
+    }
+
     braid_blob.create_braid_blob = create_braid_blob
 
     return braid_blob

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-blob",
-  "version": "0.0.40",
+  "version": "0.0.42",
   "description": "Library for collaborative blobs over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-blob",

package/test/tests.js

@@ -1635,6 +1635,376 @@ runTest(
     'stopped'
 )
 
+runTest(
+    "test options.db in put writes to custom db",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var test_key = '/test-custom-db-put-' + Math.random().toString(36).slice(2)
+
+                // Create a simple in-memory db
+                var custom_storage = {}
+                var custom_db = {
+                    read: async (key) => custom_storage[key] || null,
+                    write: async (key, data) => { custom_storage[key] = data },
+                    delete: async (key) => { delete custom_storage[key] }
+                }
+
+                // Put using the custom db
+                await braid_blob.put(test_key, Buffer.from('custom db content'), {
+                    version: ['100'],
+                    db: custom_db
+                })
+
+                // Verify content is in custom db
+                var custom_content = await custom_db.read(test_key)
+                var custom_ok = custom_content && custom_content.toString() === 'custom db content'
+
+                // Verify content is NOT in the default db
+                var default_content = await braid_blob.db.read(test_key)
+                var default_empty = default_content === null
+
+                res.end(custom_ok && default_empty ? 'true' :
+                    'custom_ok=' + custom_ok + ', default_empty=' + default_empty)
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test options.db in get reads from custom db",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var test_key = '/test-custom-db-get-' + Math.random().toString(36).slice(2)
+
+                // Create a simple in-memory db with some content
+                var custom_storage = {}
+                custom_storage[test_key] = Buffer.from('from custom db')
+                var custom_db = {
+                    read: async (key) => custom_storage[key] || null,
+                    write: async (key, data) => { custom_storage[key] = data },
+                    delete: async (key) => { delete custom_storage[key] }
+                }
+
+                // Put with skip_write to just create meta
+                await braid_blob.put(test_key, Buffer.from('ignored'), {
+                    version: ['200'],
+                    skip_write: true
+                })
+
+                // Get using the custom db - should read from custom db
+                var result = await braid_blob.get(test_key, { db: custom_db })
+
+                res.end(result && result.body.toString() === 'from custom db' ? 'true' :
+                    'got: ' + (result ? result.body.toString() : 'null'))
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test options.db in delete deletes from custom db",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var test_key = '/test-custom-db-delete-' + Math.random().toString(36).slice(2)
+
+                // Create a simple in-memory db
+                var custom_storage = {}
+                custom_storage[test_key] = Buffer.from('custom content')
+                var custom_db = {
+                    read: async (key) => custom_storage[key] || null,
+                    write: async (key, data) => { custom_storage[key] = data },
+                    delete: async (key) => { delete custom_storage[key] }
+                }
+
+                // Also put to default db
+                await braid_blob.put(test_key, Buffer.from('default content'), {
+                    version: ['300']
+                })
+
+                // Delete using custom db - should only delete from custom db
+                await braid_blob.delete(test_key, { db: custom_db })
+
+                // Verify custom db content is gone
+                var custom_content = await custom_db.read(test_key)
+                var custom_deleted = custom_content === null
+
+                // Verify default db content still exists
+                var default_content = await braid_blob.db.read(test_key)
+                var default_exists = default_content && default_content.toString() === 'default content'
+
+                res.end(custom_deleted && default_exists ? 'true' :
+                    'custom_deleted=' + custom_deleted + ', default_exists=' + default_exists)
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test options.db in get subscribe uses custom db for initial update",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var test_key = '/test-custom-db-sub-' + Math.random().toString(36).slice(2)
+
+                // Create a simple in-memory db with content
+                var custom_storage = {}
+                custom_storage[test_key] = Buffer.from('subscribe custom content')
+                var custom_db = {
+                    read: async (key) => custom_storage[key] || null,
+                    write: async (key, data) => { custom_storage[key] = data },
+                    delete: async (key) => { delete custom_storage[key] }
+                }
+
+                // Create meta with version using skip_write
+                await braid_blob.put(test_key, Buffer.from('ignored'), {
+                    version: ['400'],
+                    skip_write: true
+                })
+
+                // Subscribe using custom db - initial update should come from custom db
+                var ac = new AbortController()
+                var received_content = null
+
+                await braid_blob.get(test_key, {
+                    db: custom_db,
+                    signal: ac.signal,
+                    subscribe: (update) => {
+                        received_content = update.body.toString()
+                    }
+                })
+
+                // Wait for update
+                await new Promise(done => setTimeout(done, 50))
+                ac.abort()
+
+                res.end(received_content === 'subscribe custom content' ? 'true' :
+                    'got: ' + received_content)
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test atomic write creates temp folder on init",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-init-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Initialize
+                    await bb.init()
+
+                    // Check that temp folder exists inside meta folder
+                    var temp_folder = meta_folder + '/temp'
+                    var stat = await fs.stat(temp_folder)
+                    var is_dir = stat.isDirectory()
+
+                    res.end(is_dir ? 'true' : 'not a directory')
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test atomic write leaves no temp files after successful write",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-cleanup-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Do a write
+                    await bb.put('/test-file', Buffer.from('hello'), { version: ['1'] })
+
+                    // Check that temp folder is empty (no leftover temp files)
+                    var temp_folder = meta_folder + '/temp'
+                    var files = await fs.readdir(temp_folder)
+
+                    res.end(files.length === 0 ? 'true' : 'leftover files: ' + files.join(', '))
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test atomic write data file integrity",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-integrity-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Write initial content
+                    await bb.put('/test-file', Buffer.from('initial content'), { version: ['1'] })
+
+                    // Verify we can read it back correctly
+                    var result = await bb.get('/test-file')
+                    var content = result.body.toString()
+
+                    res.end(content === 'initial content' ? 'true' : 'wrong content: ' + content)
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test atomic write - multiple rapid writes preserve last value",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-rapid-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Do multiple rapid writes
+                    await bb.put('/test-file', Buffer.from('write1'), { version: ['1'] })
+                    await bb.put('/test-file', Buffer.from('write2'), { version: ['2'] })
+                    await bb.put('/test-file', Buffer.from('write3'), { version: ['3'] })
+
+                    // Verify last write won
+                    var result = await bb.get('/test-file')
+                    var content = result.body.toString()
+                    var version = result.version[0]
+
+                    // Also verify temp folder is clean
+                    var temp_folder = meta_folder + '/temp'
+                    var files = await fs.readdir(temp_folder)
+
+                    res.end(content === 'write3' && version === '3' && files.length === 0 ? 'true' :
+                        'content=' + content + ', version=' + version + ', temp_files=' + files.length)
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test atomic write - meta file is also written atomically",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-meta-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Write with content_type to test meta file
+                    await bb.put('/test-file', Buffer.from('content'), {
+                        version: ['test-version'],
+                        content_type: 'text/plain'
+                    })
+
+                    // Create new instance to read from disk (not cache)
+                    var bb2 = braid_blob.create_braid_blob()
+                    bb2.db_folder = db_folder
+                    bb2.meta_folder = meta_folder
+
+                    var result = await bb2.get('/test-file')
+
+                    // Verify both version and content_type are correctly persisted
+                    var version_ok = result.version[0] === 'test-version'
+                    var ct_ok = result.content_type === 'text/plain'
+
+                    res.end(version_ok && ct_ok ? 'true' :
+                        'version_ok=' + version_ok + ', ct_ok=' + ct_ok +
+                        ', version=' + result.version[0] + ', ct=' + result.content_type)
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
 }
 
 // Export for Node.js (CommonJS)