@mblaney/holster 1.0.16 → 1.0.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.16",
+  "version": "1.0.18",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/holster.js

@@ -67,12 +67,22 @@ const Holster = opt => {
             for (const key of Object.keys(msg.put[soul])) {
               const id = utils.rel.is(msg.put[soul][key])
               if (id) {
-                const data = await new Promise(res => {
-                  const _ctxid = utils.text.random()
-                  allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
-                  api(_ctxid).next(null, res, _opt)
-                })
-                msg.put[soul][key] = data
+                // Due to message queuing, the related node might not be available yet
+                // Retry with delays if we get null
+                const attemptRead = async (retries = 0) => {
+                  const data = await new Promise(res => {
+                    const _ctxid = utils.text.random()
+                    allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
+                    api(_ctxid).next(null, res, _opt)
+                  })
+                  if (data !== null || retries >= 5) {
+                    return data
+                  }
+                  // Data not ready, retry after delay
+                  await new Promise(resolve => setTimeout(resolve, 50))
+                  return attemptRead(retries + 1)
+                }
+                msg.put[soul][key] = await attemptRead()
               }
             }
             ack(msg.put[soul])
@@ -547,8 +557,51 @@ const Holster = opt => {
         allctx.set(ctxid, {chain: [{item: item, soul: soul}], on: true})
         // Map the user's callback because it can also be passed to off,
         // so need a reference to it to compare them.
-        map.set(cb, () => api(ctxid).next(null, cb, _opt))
-        // Check if item is a rel and add event listener for the node.
+        // Create a new context for each listener invocation to avoid mutation
+        map.set(cb, () => {
+          // When listener fires, re-check if the item is now a rel.
+          wire.get(
+            {"#": soul, ".": item},
+            msg => {
+              const current = msg.put && msg.put[soul] && msg.put[soul][item]
+              const id = utils.rel.is(current)
+              if (id) {
+                // It's a rel, read the related node. It might not be in the graph
+                // yet when the listener fires, so we retry with delays.
+                const attemptRead = async (retries = 0) => {
+                  const data = await new Promise(res => {
+                    const _ctxid = utils.text.random()
+                    allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
+                    api(_ctxid).next(null, res, _opt)
+                  })
+                  if (data !== null || retries >= 5) {
+                    cb(data)
+                  } else {
+                    // Data not ready, retry after delay
+                    await new Promise(resolve => setTimeout(resolve, 50))
+                    attemptRead(retries + 1)
+                  }
+                }
+                attemptRead()
+              } else {
+                // It's a direct property or null
+                cb(current !== undefined ? current : null)
+              }
+            },
+            _opt,
+          )
+        })
+
+        // Register listener immediately to avoid missing updates due to
+        // queueing. Initially register the soul without _get, then update if
+        // it's a rel.
+        let initialLex
+        if (lex) initialLex = utils.obj.put(lex, "#", soul)
+        else initialLex = {"#": soul, ".": item}
+        wire.on(initialLex, map.get(cb), false, _opt)
+
+        // Check if item is a rel and update listener if needed.
+        // This happens async but the listener is already registered above.
         wire.get(
           {"#": soul, ".": item},
           msg => {
@@ -560,13 +613,18 @@ const Holster = opt => {
             const current = msg.put && msg.put[soul] && msg.put[soul][item]
             const id = utils.rel.is(current)
             if (id) {
-              if (lex) lex = utils.obj.put(lex, "#", id)
-              else lex = {"#": id, ".": null}
-            } else {
-              if (lex) lex = utils.obj.put(lex, "#", soul)
-              else lex = {"#": soul, ".": item}
+              // It's a rel, need to switch listener to the related node
+              // First remove the initial listener
+              wire.off(initialLex, map.get(cb))
+              // Then add listener on the related node
+              let relLex
+              if (lex) relLex = utils.obj.put(lex, "#", id)
+              else relLex = {"#": id, ".": null}
+              wire.on(relLex, map.get(cb), _get, _opt)
+            } else if (_get) {
+              // Not a rel, but _get was requested, so trigger callback.
+              map.get(cb)()
             }
-            wire.on(lex, map.get(cb), _get, _opt)
           },
           _opt,
         )

package/src/radisk.js

@@ -58,7 +58,7 @@ const Radisk = opt => {
   // Performance logging
   const perfLog = (operation, startTime, key, size) => {
     const duration = Date.now() - startTime
-    if (duration > 1000) {
+    if (duration > 2000) {
       console.log(
         `[RADISK-SLOW] ${operation}: ${duration}ms for ${key}${size ? ` (${size} bytes)` : ""}`,
       )

package/src/wire.js

@@ -178,6 +178,9 @@ const Wire = opt => {
   // Track references we want but don't have yet
   const pendingReferences = new Set()
 
+  // Track pending timeouts that should start when message is sent
+  const pendingTimeouts = new Map()
+
   // Helper to check if we have a soul in memory or storage
   const hasSoul = async soul => {
     if (graph[soul]) return true
@@ -318,24 +321,20 @@ const Wire = opt => {
       if (err) console.log(err)
 
       queue[track] = cb
+
+      // Store timeout config to start after message is sent from queue
+      pendingTimeouts.set(track, {
+        lex: lex,
+        wait: _opt.wait || 100,
+      })
+
       const sendResult = send(request)
       if (sendResult && sendResult.err) {
         cb({err: sendResult.err})
         delete queue[track]
+        pendingTimeouts.delete(track)
         return
       }
-
-      // Respond to callback with null if no response.
-      setTimeout(() => {
-        const cb = queue[track]
-        if (cb) {
-          const id = lex["#"]
-          const ack = {[id]: null}
-          if (typeof lex["."] === "string") ack[id] = {[lex["."]]: null}
-          cb({put: ack})
-          delete queue[track]
-        }
-      }, _opt.wait || 100)
     })
   }
 
@@ -354,37 +353,37 @@ const Wire = opt => {
         // used whereas wire spec needs to handle clock skew for updates
         // across the network.
         const update = await Ham.mix(data, graph, opt.secure, listen)
-        const none = Object.keys(update.now).length === 0
-        if (none) {
-          // Nothing to do.
-          cb(null)
-          return
-        }
 
-        if (!(await check(update.now, send, cb))) return
-
-        // Seed pendingReferences with any new references from API calls
-        for (const [soul, node] of Object.entries(update.now)) {
-          if (node && typeof node === "object") {
-            for (const [key, value] of Object.entries(node)) {
-              const soulId = utils.rel.is(value)
-              if (soulId) {
-                // Add referenced soul to pending list
-                pendingReferences.add(soulId)
+        if (Object.keys(update.now).length !== 0) {
+          if (!(await check(update.now, send, cb))) return
+
+          // Seed pendingReferences with any new references from API calls
+          for (const [soul, node] of Object.entries(update.now)) {
+            if (node && typeof node === "object") {
+              for (const [key, value] of Object.entries(node)) {
+                const soulId = utils.rel.is(value)
+                if (soulId) {
+                  // Add referenced soul to pending list
+                  pendingReferences.add(soulId)
+                }
               }
             }
           }
+
+          store.put(update.now, cb)
+        } else {
+          // No local update needed, but still callback
+          if (cb) cb(null)
         }
 
-        store.put(update.now, cb)
-        // Also put data on the wire spec.
+        // Always put data on the wire spec even if no local update needed
         const sendResult = send(
           JSON.stringify({
             "#": dup.track(utils.text.random(9)),
             put: data,
           }),
         )
-        // Handle queue overflow error
+        // Handle queue overflow error.
         if (sendResult && sendResult.err) {
           if (cb) cb(sendResult.err)
           return
@@ -432,6 +431,28 @@ const Wire = opt => {
     }
 
     const send = (data, isBinary) => {
+      // Start timeout for this message now that it's being sent
+      const msg = JSON.parse(data)
+      const trackId = msg["#"]
+      if (trackId && pendingTimeouts.has(trackId)) {
+        const timeoutConfig = pendingTimeouts.get(trackId)
+        pendingTimeouts.delete(trackId)
+
+        // Respond to callback with null if no response.
+        timeoutConfig.timeoutId = setTimeout(() => {
+          const cb = queue[trackId]
+          if (cb) {
+            const id = timeoutConfig.lex["#"]
+            const ack = {[id]: null}
+            if (typeof timeoutConfig.lex["."] === "string") {
+              ack[id] = {[timeoutConfig.lex["."]]: null}
+            }
+            cb({put: ack})
+            delete queue[trackId]
+          }
+        }, timeoutConfig.wait)
+      }
+
       clients().forEach(client => {
         if (client && client.readyState === WebSocket.OPEN) {
           client.send(data, {binary: isBinary})
@@ -587,8 +608,35 @@ const Wire = opt => {
     }
 
     // Process next message in queue
-    const data = messageQueue.shift()
-    sendToPeers(data)
+    const data = messageQueue[0]
+    const sent = sendToPeers(data)
+
+    if (sent) {
+      // Only remove from queue if successfully sent
+      messageQueue.shift()
+
+      // Start timeout for this message now that it's been sent
+      const msg = JSON.parse(data)
+      const trackId = msg["#"]
+      if (trackId && pendingTimeouts.has(trackId)) {
+        const timeoutConfig = pendingTimeouts.get(trackId)
+        pendingTimeouts.delete(trackId)
+
+        // Respond to callback with null if no response.
+        timeoutConfig.timeoutId = setTimeout(() => {
+          const cb = queue[trackId]
+          if (cb) {
+            const id = timeoutConfig.lex["#"]
+            const ack = {[id]: null}
+            if (typeof timeoutConfig.lex["."] === "string") {
+              ack[id] = {[timeoutConfig.lex["."]]: null}
+            }
+            cb({put: ack})
+            delete queue[trackId]
+          }
+        }, timeoutConfig.wait)
+      }
+    }
 
     // Schedule processing of next message if queue not empty
     if (messageQueue.length > 0) {
@@ -600,9 +648,11 @@ const Wire = opt => {
   }
 
   const sendToPeers = data => {
+    let sentToAtLeastOne = false
     peers.forEach(peer => {
       if (peer && peer.readyState === WebSocket.OPEN) {
         peer.send(data)
+        sentToAtLeastOne = true
       } else {
         const retryHandler = peer._retryHandler || createRetryHandler()
         peer._retryHandler = retryHandler
@@ -620,6 +670,7 @@ const Wire = opt => {
         }
       }
     })
+    return sentToAtLeastOne
   }
 
   const send = data => {

package/test/holster.user.on.test.js

@@ -35,11 +35,12 @@ describe("holster.user.on", () => {
     user.get("key").put("value", err => {
       assert.equal(err, null)
 
-      user.get("key").on(data => {
+      const callback = data => {
         assert.equal(data, "update")
-        user.get("key").off()
+        user.get("key").off(callback)
         done()
-      })
+      }
+      user.get("key").on(callback)
 
       user.get("key").put("update", err => {
         assert.equal(err, null)
@@ -64,15 +65,16 @@ describe("holster.user.on", () => {
       assert.equal(err, null)
 
       let first = true
-      user.get("key3").on(data => {
+      const callback = data => {
         if (first) {
           assert.equal(data, "update1")
         } else {
           assert.equal(data, "update2")
-          user.get("key3").off()
+          user.get("key3").off(callback)
           done()
         }
-      })
+      }
+      user.get("key3").on(callback)
 
       user.get("key3").put("update1", err => {
         assert.equal(err, null)
@@ -103,11 +105,13 @@ describe("holster.user.on", () => {
     setTimeout(async () => {
       let count = 0
       for (let i = 0; i < 5; i++) {
-        user.get("for" + i).on(data => {
-          assert.equal(data, "update" + i)
-          user.get("for" + i).off()
+        const idx = i // Capture value
+        const callback = data => {
+          assert.equal(data, "update" + idx)
+          user.get("for" + idx).off(callback)
           count++
-        })
+        }
+        user.get("for" + idx).on(callback)
       }
 
       setTimeout(async () => {
@@ -130,11 +134,12 @@ describe("holster.user.on", () => {
       assert.equal(err, null)
 
       setTimeout(() => {
-        user.get("key4").on(data => {
+        const callback = data => {
           assert.equal(data, "value4")
-          user.get("key4").off()
+          user.get("key4").off(callback)
           done()
-        }, true)
+        }
+        user.get("key4").on(callback, true)
       }, 100)
     })
   })
@@ -147,14 +152,12 @@ describe("holster.user.on", () => {
       .put("ok", err => {
         assert.equal(err, null)
 
-        user
-          .get("hello")
-          .next("world!")
-          .on(data => {
-            assert.equal(data, "update")
-            user.get("hello").next("world!").off()
-            done()
-          })
+        const callback = data => {
+          assert.equal(data, "update")
+          user.get("hello").next("world!").off(callback)
+          done()
+        }
+        user.get("hello").next("world!").on(callback)
 
         setTimeout(() => {
           user
@@ -181,10 +184,12 @@ describe("holster.user.on", () => {
         number: 42,
       }
 
-      user.get("plain").on(data => {
+      const callback = data => {
         assert.deepEqual(data, update)
+        user.get("plain").off(callback)
         done()
-      })
+      }
+      user.get("plain").on(callback)
 
       setTimeout(() => {
         user.get("plain").put(update, err => {
@@ -212,17 +217,18 @@ describe("holster.user.on", () => {
         },
       }
 
-      user.get("nested").on(data => {
+      const parentCallback = data => {
         assert.deepEqual(data, update)
-      })
+      }
+      user.get("nested").on(parentCallback)
 
-      user
-        .get("nested")
-        .next("child")
-        .on(data => {
-          assert.deepEqual(data, update.child)
-          done()
-        })
+      const childCallback = data => {
+        assert.deepEqual(data, update.child)
+        user.get("nested").off(parentCallback)
+        user.get("nested").next("child").off(childCallback)
+        done()
+      }
+      user.get("nested").next("child").on(childCallback)
 
       setTimeout(() => {
         user.get("nested").put(update, err => {

package/test/system/basic.test.js

@@ -0,0 +1,29 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - basic setup", () => {
+  const wss = new Server("ws://localhost:9001")
+  const holster = Holster({file: "test/system/basic", wss: wss})
+
+  test("put and get data", (t, done) => {
+    holster.get("test").put({message: "hello"}, err => {
+      assert.equal(err, null)
+
+      holster.get("test", data => {
+        assert.notEqual(data, null)
+        assert.equal(data.message, "hello")
+        done()
+      })
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/system/basic", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})

package/test/system/chained-on.test.js

@@ -0,0 +1,41 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - chained get with on", () => {
+  const wss = new Server("ws://localhost:9002")
+  const holster = Holster({file: "test/system/chained-on", wss: wss})
+
+  test("on listener fires with null when no data exists", (t, done) => {
+    holster.get("key").on(data => {
+      // This is the actual behavior - on fires with null for non-existent keys
+      assert.equal(data, null)
+      done()
+    }, true)
+  })
+
+  test("put then on listener receives data", (t, done) => {
+    // Put data first
+    holster.get("key2").put({value: "test"}, err => {
+      assert.equal(err, null)
+
+      // Then set up listener - should get the existing data
+      setTimeout(() => {
+        holster.get("key2").on(data => {
+          assert.notEqual(data, null)
+          assert.equal(data.value, "test")
+          done()
+        }, true)
+      }, 50)
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/system/chained-on", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})

package/test/system/concurrent-listeners.test.js

@@ -0,0 +1,124 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - concurrent listeners", () => {
+  const wss = new Server("ws://localhost:9005")
+  const holster = Holster({file: "test/system/concurrent-listeners", wss: wss})
+
+  test("multiple listeners on same path receive data", (t, done) => {
+    let listener1Called = false
+    let listener2Called = false
+
+    holster.get("key").put({value: "test"}, err => {
+      assert.equal(err, null)
+
+      // Set up two listeners on the same path
+      holster.get("key").on(data => {
+        listener1Called = true
+        assert.notEqual(data, null)
+        assert.equal(data.value, "test")
+        checkComplete()
+      }, true)
+
+      holster.get("key").on(data => {
+        listener2Called = true
+        assert.notEqual(data, null)
+        assert.equal(data.value, "test")
+        checkComplete()
+      }, true)
+
+      function checkComplete() {
+        if (listener1Called && listener2Called) {
+          done()
+        }
+      }
+    })
+  })
+
+  test("nested listeners on parent and child paths", (t, done) => {
+    let parentCalled = false
+    let childCalled = false
+
+    holster.get("parent").put({value: "parent-value"}, err => {
+      assert.equal(err, null)
+
+      holster
+        .get("parent")
+        .next("child")
+        .put({value: "child-value"}, err => {
+          assert.equal(err, null)
+
+          // Listen on parent
+          holster.get("parent").on(data => {
+            parentCalled = true
+            assert.notEqual(data, null)
+            assert.equal(data.value, "parent-value")
+            checkComplete()
+          }, true)
+
+          // Listen on child
+          holster
+            .get("parent")
+            .next("child")
+            .on(data => {
+              childCalled = true
+              assert.notEqual(data, null)
+              assert.equal(data.value, "child-value")
+              checkComplete()
+            }, true)
+
+          function checkComplete() {
+            if (parentCalled && childCalled) {
+              done()
+            }
+          }
+        })
+    })
+  })
+
+  test("listener race condition - data before and after listener setup", (t, done) => {
+    const results = []
+
+    // Put data first
+    holster.get("race").put({value: "first"}, err => {
+      assert.equal(err, null)
+
+      // Set up listener with once=true to read existing data
+      holster.get("race").on(data => {
+        results.push(data?.value)
+      }, true)
+
+      // Immediately put new data
+      setTimeout(() => {
+        holster.get("race").put({value: "second"}, err => {
+          assert.equal(err, null)
+
+          setTimeout(() => {
+            // Should have received both values
+            assert.equal(
+              results.length >= 1,
+              true,
+              "Should receive at least initial value",
+            )
+            assert.equal(results[0], "first", "First value should be 'first'")
+            done()
+          }, 100)
+        })
+      }, 50)
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm(
+      "test/system/concurrent-listeners",
+      {recursive: true, force: true},
+      err => {
+        assert.equal(err, null)
+        done()
+      },
+    )
+  })
+})

package/test/system/immediate-read.test.js

@@ -0,0 +1,30 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - immediate read after write", () => {
+  const wss = new Server("ws://localhost:9008")
+  const holster = Holster({file: "test/system/immediate-read", wss: wss})
+
+  test("get immediately after put returns data despite queue delays", (t, done) => {
+    holster.get("testkey").put({value: "test"}, err => {
+      assert.equal(err, null)
+
+      // Immediate get after put - validates retry logic handles queued writes
+      holster.get("testkey", data => {
+        assert.notEqual(data, null)
+        assert.equal(data.value, "test")
+        done()
+      })
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/system/immediate-read", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})