@mblaney/holster 1.0.2 → 1.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/radisk.js

@@ -135,11 +135,12 @@ const Radisk = opt => {
           save.end = file
           // ! is used as the first file name as it's the first printable
           // character, so always matches as lexically less than any node.
-          save.mix(save.file || "!", save.start, save.end)
-          return true
+          // Also save.start can be set to undefined by a previous call to
+          // save.mix, so don't continue in this case.
+          if (save.start) save.mix(save.file || "!", save.start, save.end)
+        } else {
+          save.file = file
         }
-
-        save.file = file
       },
       mix: (file, start, end) => {
         save.start = save.end = save.file = u
@@ -151,7 +152,7 @@ const Radisk = opt => {
 
             if (end && end < key) {
               save.start = key
-              return save.start
+              return
             }
 
             disk(key, value)
@@ -185,8 +186,13 @@ const Radisk = opt => {
           Radisk.encode(k) +
           (typeof value === "undefined" ? "" : "=" + Radisk.encode(value)) +
           "\n"
-        // Cannot split the file if only have one entry to write.
-        if (write.count > 1 && write.text.length + enc.length > opt.size) {
+        // Cannot split the file if only have one entry to write. Also don't
+        // start a split if in the middle of writing a node (pre > 0).
+        if (
+          pre.length === 0 &&
+          write.count > 1 &&
+          write.text.length + enc.length > opt.size
+        ) {
           write.text = ""
           // Otherwise split the entries in half.
           write.limit = Math.ceil(write.count / 2)
@@ -214,10 +220,6 @@ const Radisk = opt => {
           } else {
             write.file = key.substring(0, end)
           }
-          // write.limit can be reached after already writing properties of
-          // the current node, so remove it from write.sub before writing to
-          // disk so that it's not duplicated across files.
-          write.sub(write.file, null)
           write.count = 0
           radisk.write(name, write.sub, write.next)
           return true

package/src/store.js

@@ -35,11 +35,11 @@ const fileSystem = opt => {
         })
       },
       put: (file, data, cb) => {
-        var random = Math.random().toString(36).slice(-9)
         // Don't put tmp files under dir so that they're not listed.
-        var tmp = file + "." + random + ".tmp"
+        var tmp = file + "." + utils.text.random(9) + ".tmp"
         fs.writeFile(tmp, data, err => {
           if (err) {
+            console.log("fs.writeFile error:", err)
             cb(err)
             return
           }
@@ -90,7 +90,7 @@ const fileSystem = opt => {
 
     return {
       get: (file, cb) => {
-        if (db) {
+        const _get = (file, cb) => {
           const tx = db.transaction([dir], "readonly")
           const req = tx.objectStore(dir).get(file)
           req.onerror = () => {
@@ -99,12 +99,28 @@ const fileSystem = opt => {
           req.onsuccess = () => {
             cb(null, req.result)
           }
-        } else {
-          cb("error indexedDB not available")
         }
+        if (db) {
+          _get(file, cb)
+          return
+        }
+
+        let retry = 0
+        const interval = setInterval(() => {
+          if (db) {
+            clearInterval(interval)
+            _get(file, cb)
+            return
+          }
+
+          if (retry++ > 5) {
+            clearInterval(interval)
+            cb("error indexedDB not available")
+          }
+        }, 1000)
       },
       put: (file, data, cb) => {
-        if (db) {
+        const _put = (file, data, cb) => {
           const tx = db.transaction([dir], "readwrite")
           const req = tx.objectStore(dir).put(data, file)
           req.onerror = () => {
@@ -113,12 +129,28 @@ const fileSystem = opt => {
           req.onsuccess = () => {
             cb(null)
           }
-        } else {
-          cb("error indexedDB not available")
         }
+        if (db) {
+          _put(file, data, cb)
+          return
+        }
+
+        let retry = 0
+        const interval = setInterval(() => {
+          if (db) {
+            clearInterval(interval)
+            _put(file, data, cb)
+            return
+          }
+
+          if (retry++ > 5) {
+            clearInterval(interval)
+            cb("error indexedDB not available")
+          }
+        }, 1000)
       },
       list: cb => {
-        if (db) {
+        const _list = cb => {
           const tx = db.transaction([dir], "readonly")
           const req = tx.objectStore(dir).getAllKeys()
           req.onerror = () => console.log("error getting keys for", dir)
@@ -126,10 +158,26 @@ const fileSystem = opt => {
             req.result.forEach(cb)
             cb()
           }
-        } else {
-          console.log("error indexedDB not available")
-          cb()
         }
+        if (db) {
+          _list(cb)
+          return
+        }
+
+        let retry = 0
+        const interval = setInterval(() => {
+          if (db) {
+            clearInterval(interval)
+            _list(cb)
+            return
+          }
+
+          if (retry++ > 5) {
+            clearInterval(interval)
+            console.log("error indexedDB not available")
+            cb()
+          }
+        }, 1000)
       },
     }
   }

package/src/wire.js

@@ -220,8 +220,22 @@ const Wire = opt => {
 
     const send = (data, isBinary) => {
       clients().forEach(client => {
-        if (client.readyState === WebSocket.OPEN) {
+        if (client && client.readyState === WebSocket.OPEN) {
           client.send(data, {binary: isBinary})
+        } else {
+          let retry = 0
+          const interval = setInterval(() => {
+            if (client && client.readyState === WebSocket.OPEN) {
+              clearInterval(interval)
+              client.send(data, {binary: isBinary})
+              return
+            }
+
+            if (retry++ > 5) {
+              clearInterval(interval)
+              console.log("websocket not available")
+            }
+          }, 1000)
         }
       })
     }
@@ -257,7 +271,19 @@ const Wire = opt => {
       if (peer && peer.readyState === WebSocket.OPEN) {
         peer.send(data)
       } else {
-        console.log("websocket not available")
+        let retry = 0
+        const interval = setInterval(() => {
+          if (peer && peer.readyState === WebSocket.OPEN) {
+            clearInterval(interval)
+            peer.send(data)
+            return
+          }
+
+          if (retry++ > 5) {
+            clearInterval(interval)
+            console.log("websocket not available")
+          }
+        }, 1000)
       }
     })
   }
@@ -265,10 +291,9 @@ const Wire = opt => {
     opt.peers = [`ws://localhost:${opt.port || 8765}`]
   }
   opt.peers.forEach(peer => {
-    let ws = new WebSocket(peer)
-    peers.push(ws)
     const start = () => {
-      if (!ws) ws = new WebSocket(peer)
+      let ws = new WebSocket(peer)
+      peers.push(ws)
       ws.onclose = c => {
         if (peers.indexOf(ws) !== -1) {
           peers.splice(peers.indexOf(ws), 1)

package/test/radisk.test.js

@@ -25,6 +25,8 @@ describe("radisk", () => {
     },
   }
   const radisk = Radisk(opt)
+  const big =
+    "file size is only 100 bytes so writing this value requires calling slice"
 
   test("write and read from memory", (t, done) => {
     radisk("key", "value", () => {
@@ -164,8 +166,6 @@ describe("radisk", () => {
   })
 
   test("write and read value bigger than file size", (t, done) => {
-    const big =
-      "file size is only 100 bytes so writing this value requires calling slice"
     radisk("newFile", big)
     radisk("newFile", (err, value) => {
       assert.equal(value, big)
@@ -175,11 +175,8 @@ describe("radisk", () => {
         "!":
           '\x1F+0\x1F#\x1F"key\x1F=\x1F"value\ncontinued\x0312345\x1F\n' +
           '\x1F+1\x1F#\x1F"A\x1F=\x1F"valueA\x1F\n' +
-          '\x1F+1\x1F#\x1F"B\x1F=\x1F"valueB\x1F\n' +
-          '\x1F+1\x1F#\x1F"C\x1F=\x1F \x1F\n',
-        keyC:
-          '\x1F+0\x1F#\x1F"keyC\x1F=\x1F"valueC\x1F\n' +
-          '\x1F+0\x1F#\x1F"newFile\x1F=\x1F \x1F\n',
+          '\x1F+1\x1F#\x1F"B\x1F=\x1F"valueB\x1F\n',
+        keyC: '\x1F+0\x1F#\x1F"keyC\x1F=\x1F"valueC\x1F\n',
         newFile: '\x1F+0\x1F#\x1F"newFile\x1F=\x1F"' + big + "\x1F\n",
       })
       radisk("newFile", (err, value) => {
@@ -188,4 +185,150 @@ describe("radisk", () => {
       })
     }, 10)
   })
+
+  // This tests trying to add to a file that is already too long, so needs to
+  // split again. Previously wasn't able to do this and would loop.
+  test("write and read a small value after split", (t, done) => {
+    radisk("small", "small value")
+    radisk("small", (err, value) => {
+      assert.equal(value, "small value")
+    })
+    setTimeout(() => {
+      assert.deepEqual(puts, {
+        "!":
+          '\x1F+0\x1F#\x1F"key\x1F=\x1F"value\ncontinued\x0312345\x1F\n' +
+          '\x1F+1\x1F#\x1F"A\x1F=\x1F"valueA\x1F\n' +
+          '\x1F+1\x1F#\x1F"B\x1F=\x1F"valueB\x1F\n',
+        keyC: '\x1F+0\x1F#\x1F"keyC\x1F=\x1F"valueC\x1F\n',
+        newFile: '\x1F+0\x1F#\x1F"newFile\x1F=\x1F"' + big + "\x1F\n",
+        small: '\x1F+0\x1F#\x1F"small\x1F=\x1F"small value\x1F\n',
+      })
+      done()
+    }, 10)
+  })
+
+  // This tests how existing files are handled when they are made smaller than
+  // the maximum file size. (They should be kept as they are created so that
+  // we don't have to deal with removing files.)
+  test("write small value to newFile", (t, done) => {
+    radisk("newFile", "removed...")
+    radisk("newFile", (err, value) => {
+      assert.equal(value, "removed...")
+    })
+    setTimeout(() => {
+      assert.deepEqual(puts, {
+        "!":
+          '\x1F+0\x1F#\x1F"key\x1F=\x1F"value\ncontinued\x0312345\x1F\n' +
+          '\x1F+1\x1F#\x1F"A\x1F=\x1F"valueA\x1F\n' +
+          '\x1F+1\x1F#\x1F"B\x1F=\x1F"valueB\x1F\n',
+        keyC: '\x1F+0\x1F#\x1F"keyC\x1F=\x1F"valueC\x1F\n',
+        newFile: '\x1F+0\x1F#\x1F"newFile\x1F=\x1F"removed...\x1F\n',
+        small: '\x1F+0\x1F#\x1F"small\x1F=\x1F"small value\x1F\n',
+      })
+      done()
+    }, 10)
+  })
+
+  // This tests ignoring the maximum file size when writing sub-keys. Since
+  // radisk.write has already been called for a key it would previously get
+  // duplicated when trying to split due to reaching the max length.
+  test("write big value to first file", (t, done) => {
+    radisk("keyA", big)
+    radisk("keyA", (err, value) => {
+      assert.equal(value, big)
+    })
+    setTimeout(() => {
+      assert.deepEqual(puts, {
+        "!":
+          '\x1F+0\x1F#\x1F"key\x1F=\x1F"value\ncontinued\x0312345\x1F\n' +
+          '\x1F+1\x1F#\x1F"A\x1F=\x1F"' +
+          big +
+          "\x1F\n" +
+          '\x1F+1\x1F#\x1F"B\x1F=\x1F"valueB\x1F\n',
+        keyC: '\x1F+0\x1F#\x1F"keyC\x1F=\x1F"valueC\x1F\n',
+        newFile: '\x1F+0\x1F#\x1F"newFile\x1F=\x1F"removed...\x1F\n',
+        small: '\x1F+0\x1F#\x1F"small\x1F=\x1F"small value\x1F\n',
+      })
+      done()
+    }, 10)
+  })
+
+  test("add to last file", (t, done) => {
+    radisk("smallContinued", "continued value")
+    radisk("smallContinued", (err, value) => {
+      assert.equal(value, "continued value")
+    })
+    setTimeout(() => {
+      assert.deepEqual(puts, {
+        "!":
+          '\x1F+0\x1F#\x1F"key\x1F=\x1F"value\ncontinued\x0312345\x1F\n' +
+          '\x1F+1\x1F#\x1F"A\x1F=\x1F"' +
+          big +
+          "\x1F\n" +
+          '\x1F+1\x1F#\x1F"B\x1F=\x1F"valueB\x1F\n',
+        keyC: '\x1F+0\x1F#\x1F"keyC\x1F=\x1F"valueC\x1F\n',
+        newFile: '\x1F+0\x1F#\x1F"newFile\x1F=\x1F"removed...\x1F\n',
+        small:
+          '\x1F+0\x1F#\x1F"small\x1F=\x1F"small value\x1F\n' +
+          '\x1F+1\x1F#\x1F"Continued\x1F=\x1F"continued value\x1F\n',
+      })
+      done()
+    }, 10)
+  })
+
+  test("set a key to null and add a value to that file", (t, done) => {
+    radisk("keyC", null)
+    radisk("keyC", (err, value) => {
+      assert.equal(value, null)
+    })
+    radisk("keyD", "valueD")
+    radisk("keyD", (err, value) => {
+      assert.equal(value, "valueD")
+    })
+    setTimeout(() => {
+      assert.deepEqual(puts, {
+        "!":
+          '\x1F+0\x1F#\x1F"key\x1F=\x1F"value\ncontinued\x0312345\x1F\n' +
+          '\x1F+1\x1F#\x1F"A\x1F=\x1F"' +
+          big +
+          "\x1F\n" +
+          '\x1F+1\x1F#\x1F"B\x1F=\x1F"valueB\x1F\n',
+        keyC:
+          '\x1F+0\x1F#\x1F"key\x1F\n\x1F+1\x1F#\x1F"C\x1F=\x1F \x1F\n' +
+          '\x1F+1\x1F#\x1F"D\x1F=\x1F"valueD\x1F\n',
+        newFile: '\x1F+0\x1F#\x1F"newFile\x1F=\x1F"removed...\x1F\n',
+        small:
+          '\x1F+0\x1F#\x1F"small\x1F=\x1F"small value\x1F\n' +
+          '\x1F+1\x1F#\x1F"Continued\x1F=\x1F"continued value\x1F\n',
+      })
+      done()
+    }, 10)
+  })
+
+  test("set first keys to null", (t, done) => {
+    radisk("key", null)
+    radisk("key", (err, value) => {
+      assert.equal(value, null)
+    })
+    radisk("keyA", null)
+    radisk("keyA", (err, value) => {
+      assert.equal(value, null)
+    })
+    setTimeout(() => {
+      assert.deepEqual(puts, {
+        "!":
+          '\x1F+0\x1F#\x1F"key\x1F=\x1F \x1F\n' +
+          '\x1F+1\x1F#\x1F"A\x1F=\x1F \x1F\n' +
+          '\x1F+1\x1F#\x1F"B\x1F=\x1F"valueB\x1F\n',
+        keyC:
+          '\x1F+0\x1F#\x1F"key\x1F\n\x1F+1\x1F#\x1F"C\x1F=\x1F \x1F\n' +
+          '\x1F+1\x1F#\x1F"D\x1F=\x1F"valueD\x1F\n',
+        newFile: '\x1F+0\x1F#\x1F"newFile\x1F=\x1F"removed...\x1F\n',
+        small:
+          '\x1F+0\x1F#\x1F"small\x1F=\x1F"small value\x1F\n' +
+          '\x1F+1\x1F#\x1F"Continued\x1F=\x1F"continued value\x1F\n',
+      })
+      done()
+    }, 10)
+  })
 })

package/test/store.test.js

@@ -75,7 +75,7 @@ describe("store", () => {
       },
       err => {
         assert.equal(err, null)
-        fs.access("test/multiple/keyC", err => {
+        fs.access("test/multiple/!", err => {
           assert.equal(err, null)
           fs.access("test/multiple/newFile", err => {
             assert.equal(err, null)
@@ -103,7 +103,7 @@ describe("store", () => {
     })
   })
 
-  test("get keys from second file", (t, done) => {
+  test("get another key from first file", (t, done) => {
     multiple.get({"#": "keyC"}, (err, value) => {
       assert.deepEqual(value, {
         keyC: {
@@ -135,7 +135,7 @@ describe("store", () => {
     })
   })
 
-  test("get key from third file", (t, done) => {
+  test("get key from second file", (t, done) => {
     multiple.get({"#": "newFile"}, (err, value) => {
       assert.deepEqual(value, {
         newFile: {