braid-blob 0.0.41 → 0.0.42

package/index.js

@@ -11,6 +11,8 @@ function create_braid_blob() {
         db: null // object with read/write/delete methods
     }
 
+    var temp_folder = null // will be set in init
+
     braid_blob.init = async () => {
         // We only want to initialize once
         var init_p = real_init()
@@ -21,6 +23,15 @@ function create_braid_blob() {
             // Ensure our meta folder exists
             await require('fs').promises.mkdir(braid_blob.meta_folder, { recursive: true })
 
+            // Create a temp folder inside the meta folder for writing temp files,
+            // for atomic writing.
+            // The temp folder is called "temp",
+            // And this is guaranteed not to conflict with any other files,
+            // because other files are the result of encode_filename,
+            // which always ends with a ".XX" (for handling insensitive filesystems)
+            temp_folder = `${braid_blob.meta_folder}/temp`
+            await require('fs').promises.mkdir(temp_folder, { recursive: true })
+
             // Set up db - either use provided object or create file-based storage
             if (typeof braid_blob.db_folder === 'string') {
                 await require('fs').promises.mkdir(braid_blob.db_folder, { recursive: true })
@@ -36,7 +47,7 @@ function create_braid_blob() {
                     },
                     write: async (key, data) => {
                         var file_path = `${braid_blob.db_folder}/${encode_filename(key)}`
-                        await require('fs').promises.writeFile(file_path, data)
+                        await atomic_write(file_path, data, temp_folder)
                     },
                     delete: async (key) => {
                         var file_path = `${braid_blob.db_folder}/${encode_filename(key)}`
@@ -74,9 +85,8 @@ function create_braid_blob() {
     }
 
     async function save_meta(key) {
-        await require('fs').promises.writeFile(
-            `${braid_blob.meta_folder}/${encode_filename(key)}`,
-            JSON.stringify(braid_blob.meta_cache[key]))
+        await atomic_write(`${braid_blob.meta_folder}/${encode_filename(key)}`,
+            JSON.stringify(braid_blob.meta_cache[key]), temp_folder)
     }
 
     async function delete_meta(key) {
@@ -733,6 +743,12 @@ function create_braid_blob() {
         return normalized
     }
 
+    async function atomic_write(final_destination, data, temp_folder) {
+        var temp = `${temp_folder}/${Math.random().toString(36).slice(2)}`
+        await require('fs').promises.writeFile(temp, data)
+        await require('fs').promises.rename(temp, final_destination)
+    }
+
     braid_blob.create_braid_blob = create_braid_blob
 
     return braid_blob

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-blob",
-  "version": "0.0.41",
+  "version": "0.0.42",
   "description": "Library for collaborative blobs over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-blob",

package/test/tests.js

@@ -1799,6 +1799,212 @@ runTest(
     'true'
 )
 
+runTest(
+    "test atomic write creates temp folder on init",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-init-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Initialize
+                    await bb.init()
+
+                    // Check that temp folder exists inside meta folder
+                    var temp_folder = meta_folder + '/temp'
+                    var stat = await fs.stat(temp_folder)
+                    var is_dir = stat.isDirectory()
+
+                    res.end(is_dir ? 'true' : 'not a directory')
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test atomic write leaves no temp files after successful write",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-cleanup-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Do a write
+                    await bb.put('/test-file', Buffer.from('hello'), { version: ['1'] })
+
+                    // Check that temp folder is empty (no leftover temp files)
+                    var temp_folder = meta_folder + '/temp'
+                    var files = await fs.readdir(temp_folder)
+
+                    res.end(files.length === 0 ? 'true' : 'leftover files: ' + files.join(', '))
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test atomic write data file integrity",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-integrity-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Write initial content
+                    await bb.put('/test-file', Buffer.from('initial content'), { version: ['1'] })
+
+                    // Verify we can read it back correctly
+                    var result = await bb.get('/test-file')
+                    var content = result.body.toString()
+
+                    res.end(content === 'initial content' ? 'true' : 'wrong content: ' + content)
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test atomic write - multiple rapid writes preserve last value",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-rapid-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Do multiple rapid writes
+                    await bb.put('/test-file', Buffer.from('write1'), { version: ['1'] })
+                    await bb.put('/test-file', Buffer.from('write2'), { version: ['2'] })
+                    await bb.put('/test-file', Buffer.from('write3'), { version: ['3'] })
+
+                    // Verify last write won
+                    var result = await bb.get('/test-file')
+                    var content = result.body.toString()
+                    var version = result.version[0]
+
+                    // Also verify temp folder is clean
+                    var temp_folder = meta_folder + '/temp'
+                    var files = await fs.readdir(temp_folder)
+
+                    res.end(content === 'write3' && version === '3' && files.length === 0 ? 'true' :
+                        'content=' + content + ', version=' + version + ', temp_files=' + files.length)
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
+runTest(
+    "test atomic write - meta file is also written atomically",
+    async () => {
+        var r1 = await braid_fetch(`/eval`, {
+            method: 'POST',
+            body: `void (async () => {
+                var fs = require('fs').promises
+                var test_id = 'test-atomic-meta-' + Math.random().toString(36).slice(2)
+                var db_folder = __dirname + '/' + test_id + '-db'
+                var meta_folder = __dirname + '/' + test_id + '-meta'
+
+                try {
+                    var bb = braid_blob.create_braid_blob()
+                    bb.db_folder = db_folder
+                    bb.meta_folder = meta_folder
+
+                    // Write with content_type to test meta file
+                    await bb.put('/test-file', Buffer.from('content'), {
+                        version: ['test-version'],
+                        content_type: 'text/plain'
+                    })
+
+                    // Create new instance to read from disk (not cache)
+                    var bb2 = braid_blob.create_braid_blob()
+                    bb2.db_folder = db_folder
+                    bb2.meta_folder = meta_folder
+
+                    var result = await bb2.get('/test-file')
+
+                    // Verify both version and content_type are correctly persisted
+                    var version_ok = result.version[0] === 'test-version'
+                    var ct_ok = result.content_type === 'text/plain'
+
+                    res.end(version_ok && ct_ok ? 'true' :
+                        'version_ok=' + version_ok + ', ct_ok=' + ct_ok +
+                        ', version=' + result.version[0] + ', ct=' + result.content_type)
+                } catch (e) {
+                    res.end('error: ' + e.message)
+                } finally {
+                    await fs.rm(db_folder, { recursive: true, force: true })
+                    await fs.rm(meta_folder, { recursive: true, force: true })
+                }
+            })()`
+        })
+        return await r1.text()
+    },
+    'true'
+)
+
 }
 
 // Export for Node.js (CommonJS)