braid-text 0.2.99 → 0.2.101

package/index.js

@@ -1293,17 +1293,43 @@ function create_braid_text() {
 
     async function db_folder_init() {
         if (braid_text.verbose) console.log('__!')
-        if (!db_folder_init.p) db_folder_init.p = new Promise(async done => {
+        if (!db_folder_init.p) db_folder_init.p = (async () => {
             await fs.promises.mkdir(braid_text.db_folder, { recursive: true });
             await fs.promises.mkdir(`${braid_text.db_folder}/.meta`, { recursive: true })
+            await fs.promises.mkdir(`${braid_text.db_folder}/.temp`, { recursive: true })
+            await fs.promises.mkdir(`${braid_text.db_folder}/.wal-intent`, { recursive: true })
+
+            // Clean out .temp directory on startup
+            var temp_files = await fs.promises.readdir(`${braid_text.db_folder}/.temp`)
+            for (var f of temp_files)
+                await fs.promises.unlink(`${braid_text.db_folder}/.temp/${f}`)
+
+            // Replay any pending .wal-intent files
+            var intent_files = await fs.promises.readdir(`${braid_text.db_folder}/.wal-intent`)
+            for (var intent_name of intent_files) {
+                var intent_path = `${braid_text.db_folder}/.wal-intent/${intent_name}`
+                var target_path = `${braid_text.db_folder}/${intent_name}`
+
+                var intent_data = await fs.promises.readFile(intent_path)
+                var expected_size = Number(intent_data.readBigUInt64LE(0))
+                var append_data = intent_data.subarray(8)
+
+                var stat = await fs.promises.stat(target_path)
+                if (stat.size < expected_size || stat.size > expected_size + append_data.length)
+                    throw new Error(`wal-intent replay failed: ${target_path} size ${stat.size}, expected ${expected_size} to ${expected_size + append_data.length}`)
+
+                // Append whatever portion hasn't been written yet
+                var already_written = stat.size - expected_size
+                if (already_written < append_data.length)
+                    await fs.promises.appendFile(target_path, append_data.subarray(already_written))
+                await fs.promises.unlink(intent_path)
+            }
 
             // Populate key_to_filename mapping from existing files
             var files = (await fs.promises.readdir(braid_text.db_folder))
                 .filter(x => /\.\d+$/.test(x))
             init_filename_mapping(files)
-
-            done()
-        })
+        })()
         await db_folder_init.p
     }
 
@@ -1385,10 +1411,20 @@ function create_braid_text() {
                 if (currentSize < threshold) {
                     if (braid_text.verbose) console.log(`appending to db..`)
 
-                    let buffer = Buffer.allocUnsafe(4)
-                    buffer.writeUInt32LE(bytes.length, 0)
-                    await fs.promises.appendFile(filename, buffer)
-                    await fs.promises.appendFile(filename, bytes)
+                    let len_buf = Buffer.allocUnsafe(4)
+                    len_buf.writeUInt32LE(bytes.length, 0)
+                    let append_data = Buffer.concat([len_buf, bytes])
+
+                    let basename = require('path').basename(filename)
+                    let intent_path = `${braid_text.db_folder}/.wal-intent/${basename}`
+                    let stat = await fs.promises.stat(filename)
+                    let size_buf = Buffer.allocUnsafe(8)
+                    size_buf.writeBigUInt64LE(BigInt(stat.size), 0)
+
+                    await atomic_write(intent_path, Buffer.concat([size_buf, append_data]),
+                        `${braid_text.db_folder}/.temp`)
+                    await fs.promises.appendFile(filename, append_data)
+                    await fs.promises.unlink(intent_path)
 
                     if (braid_text.verbose) console.log("wrote to : " + filename)
                 } else {
@@ -1401,8 +1437,8 @@ function create_braid_text() {
                         buffer.writeUInt32LE(init.length, 0)
 
                         const newFilename = `${braid_text.db_folder}/${encoded}.${currentNumber}`
-                        await fs.promises.writeFile(newFilename, buffer)
-                        await fs.promises.appendFile(newFilename, init)
+                        await atomic_write(newFilename, Buffer.concat([buffer, init]),
+                            `${braid_text.db_folder}/.temp`)
 
                         if (braid_text.verbose) console.log("wrote to : " + newFilename)
 
@@ -1423,8 +1459,8 @@ function create_braid_text() {
 
                 while (meta_dirty) {
                     meta_dirty = false
-                    await fs.promises.writeFile(meta_filename,
-                        JSON.stringify(get_meta()))
+                    await atomic_write(meta_filename, JSON.stringify(get_meta()),
+                        `${braid_text.db_folder}/.temp`)
                     await new Promise(done => setTimeout(done,
                         braid_text.meta_file_save_period_ms))
                 }
@@ -2727,6 +2763,12 @@ function create_braid_text() {
         return `sha-256=:${require('crypto').createHash('sha256').update(s).digest('base64')}:`
     }
 
+    async function atomic_write(final_destination, data, temp_folder) {
+        var temp = `${temp_folder}/${Math.random().toString(36).slice(2)}`
+        await fs.promises.writeFile(temp, data)
+        await fs.promises.rename(temp, final_destination)
+    }
+
     function within_fiber(id, func) {
         if (!within_fiber.chains) within_fiber.chains = {}
         var prev = within_fiber.chains[id] || Promise.resolve()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-text",
-  "version": "0.2.99",
+  "version": "0.2.101",
   "description": "Library for collaborative text over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-text",

package/test/tests.js

@@ -2665,6 +2665,197 @@ runTest(
     'filename conflict detected'
 )
 
+runTest(
+    "test wal-intent recovery after simulated crash during append",
+    async () => {
+        var r = await braid_fetch(`/eval`, {
+            method: 'PUT',
+            body: `void (async () => {
+                var fs = require('fs')
+                var test_db = __dirname + '/test_wal_recovery_' + Math.random().toString(36).slice(2)
+                try {
+                    // Create a fresh braid_text instance with its own db_folder
+                    var bt = braid_text.create_braid_text()
+                    bt.db_folder = test_db
+
+                    // Do initial PUT (5 chars = 'hello', version ends at 4)
+                    await bt.put('/test', {
+                        version: ['a-4'],
+                        parents: [],
+                        body: 'hello'
+                    })
+
+                    // Do second PUT to trigger an append (6 more chars = ' world', version ends at 10)
+                    await bt.put('/test', {
+                        version: ['a-10'],
+                        parents: ['a-4'],
+                        body: 'hello world'
+                    })
+
+                    var encoded = bt.encode_filename('/test')
+                    var db_file = test_db + '/' + encoded + '.1'
+                    var intent_file = test_db + '/.wal-intent/' + encoded + '.1'
+
+                    // Read the db file
+                    var data = await fs.promises.readFile(db_file)
+
+                    // Parse chunks to find the last one
+                    var cursor = 0
+                    var chunks = []
+                    while (cursor < data.length) {
+                        var chunk_start = cursor
+                        var chunk_size = data.readUInt32LE(cursor)
+                        cursor += 4 + chunk_size
+                        chunks.push({ start: chunk_start, size: chunk_size, end: cursor })
+                    }
+
+                    if (chunks.length < 2) {
+                        res.end('expected at least 2 chunks, got ' + chunks.length)
+                        return
+                    }
+
+                    var last_chunk = chunks[chunks.length - 1]
+                    var prev_end = chunks[chunks.length - 2].end
+
+                    // Create the wal-intent file: 8-byte size + the last chunk data
+                    var size_buf = Buffer.allocUnsafe(8)
+                    size_buf.writeBigUInt64LE(BigInt(prev_end), 0)
+                    var last_chunk_data = data.subarray(last_chunk.start, last_chunk.end)
+                    var intent_data = Buffer.concat([size_buf, last_chunk_data])
+                    await fs.promises.writeFile(intent_file, intent_data)
+
+                    // Truncate the db file partway through the last chunk (keep ~half)
+                    var truncate_point = last_chunk.start + Math.floor((last_chunk.end - last_chunk.start) / 2)
+                    await fs.promises.truncate(db_file, truncate_point)
+
+                    // Create a new braid_text instance to simulate restart
+                    var bt2 = braid_text.create_braid_text()
+                    bt2.db_folder = test_db
+
+                    // Get the resource - this should trigger wal-intent replay
+                    var resource = await bt2.get_resource('/test')
+                    var text = resource.doc.get()
+
+                    // Verify intent file was cleaned up
+                    var intent_exists = true
+                    try {
+                        await fs.promises.access(intent_file)
+                    } catch (e) {
+                        intent_exists = false
+                    }
+
+                    await fs.promises.rm(test_db, { recursive: true, force: true })
+
+                    if (intent_exists) {
+                        res.end('intent file still exists')
+                        return
+                    }
+
+                    res.end(text)
+                } catch (e) {
+                    await fs.promises.rm(test_db, { recursive: true, force: true }).catch(() => {})
+                    res.end('error: ' + e.message + ' ' + e.stack)
+                }
+            })()`
+        })
+
+        return await r.text()
+    },
+    'hello world'
+)
+
+runTest(
+    "test wal-intent throws error when db file is too large",
+    async () => {
+        var r = await braid_fetch(`/eval`, {
+            method: 'PUT',
+            body: `void (async () => {
+                var fs = require('fs')
+                var test_db = __dirname + '/test_wal_error_' + Math.random().toString(36).slice(2)
+                try {
+                    // Create a fresh braid_text instance with its own db_folder
+                    var bt = braid_text.create_braid_text()
+                    bt.db_folder = test_db
+
+                    // Do initial PUT (5 chars = 'hello', version ends at 4)
+                    await bt.put('/test', {
+                        version: ['a-4'],
+                        parents: [],
+                        body: 'hello'
+                    })
+
+                    // Do second PUT to trigger an append (6 more chars = ' world', version ends at 10)
+                    await bt.put('/test', {
+                        version: ['a-10'],
+                        parents: ['a-4'],
+                        body: 'hello world'
+                    })
+
+                    var encoded = bt.encode_filename('/test')
+                    var db_file = test_db + '/' + encoded + '.1'
+                    var intent_file = test_db + '/.wal-intent/' + encoded + '.1'
+
+                    // Read the db file
+                    var data = await fs.promises.readFile(db_file)
+
+                    // Parse chunks to find the last one
+                    var cursor = 0
+                    var chunks = []
+                    while (cursor < data.length) {
+                        var chunk_start = cursor
+                        var chunk_size = data.readUInt32LE(cursor)
+                        cursor += 4 + chunk_size
+                        chunks.push({ start: chunk_start, size: chunk_size, end: cursor })
+                    }
+
+                    if (chunks.length < 2) {
+                        res.end('expected at least 2 chunks, got ' + chunks.length)
+                        return
+                    }
+
+                    var last_chunk = chunks[chunks.length - 1]
+                    var prev_end = chunks[chunks.length - 2].end
+
+                    // Create the wal-intent file: 8-byte size + the last chunk data
+                    var size_buf = Buffer.allocUnsafe(8)
+                    size_buf.writeBigUInt64LE(BigInt(prev_end), 0)
+                    var last_chunk_data = data.subarray(last_chunk.start, last_chunk.end)
+                    var intent_data = Buffer.concat([size_buf, last_chunk_data])
+                    await fs.promises.writeFile(intent_file, intent_data)
+
+                    // Append extra garbage to the db file (making it too large)
+                    await fs.promises.appendFile(db_file, Buffer.from('extra garbage data'))
+
+                    // Create a new braid_text instance to simulate restart
+                    var bt2 = braid_text.create_braid_text()
+                    bt2.db_folder = test_db
+
+                    // Now try to init - this should throw an error
+                    var result
+                    try {
+                        await bt2.db_folder_init()
+                        result = 'should have thrown an error'
+                    } catch (e) {
+                        if (e.message.includes('wal-intent replay failed')) {
+                            result = 'correctly threw error'
+                        } else {
+                            result = 'wrong error: ' + e.message
+                        }
+                    }
+                    await fs.promises.rm(test_db, { recursive: true, force: true })
+                    res.end(result)
+                } catch (e) {
+                    await fs.promises.rm(test_db, { recursive: true, force: true }).catch(() => {})
+                    res.end('error: ' + e.message)
+                }
+            })()`
+        })
+
+        return await r.text()
+    },
+    'correctly threw error'
+)
+
 }
 
 // Export for both Node.js and browser environments