@mblaney/holster 1.0.16 → 1.0.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.16",
+  "version": "1.0.17",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/holster.js

@@ -67,12 +67,22 @@ const Holster = opt => {
             for (const key of Object.keys(msg.put[soul])) {
               const id = utils.rel.is(msg.put[soul][key])
               if (id) {
-                const data = await new Promise(res => {
-                  const _ctxid = utils.text.random()
-                  allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
-                  api(_ctxid).next(null, res, _opt)
-                })
-                msg.put[soul][key] = data
+                // Due to message queuing, the related node might not be available yet
+                // Retry with delays if we get null
+                const attemptRead = async (retries = 0) => {
+                  const data = await new Promise(res => {
+                    const _ctxid = utils.text.random()
+                    allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
+                    api(_ctxid).next(null, res, _opt)
+                  })
+                  if (data !== null || retries >= 5) {
+                    return data
+                  }
+                  // Data not ready, retry after delay
+                  await new Promise(resolve => setTimeout(resolve, 50))
+                  return attemptRead(retries + 1)
+                }
+                msg.put[soul][key] = await attemptRead()
               }
             }
             ack(msg.put[soul])
@@ -547,8 +557,57 @@ const Holster = opt => {
         allctx.set(ctxid, {chain: [{item: item, soul: soul}], on: true})
         // Map the user's callback because it can also be passed to off,
         // so need a reference to it to compare them.
-        map.set(cb, () => api(ctxid).next(null, cb, _opt))
-        // Check if item is a rel and add event listener for the node.
+        // Create a new context for each listener invocation to avoid mutation
+        map.set(cb, () => {
+          // When listener fires, re-check if the item is now a rel.
+          wire.get(
+            {"#": soul, ".": item},
+            msg => {
+              const current = msg.put && msg.put[soul] && msg.put[soul][item]
+              const id = utils.rel.is(current)
+              if (id) {
+                // It's a rel, read the related node. It might not be in the graph
+                // yet when the listener fires, so we retry with delays.
+                const attemptRead = (retries = 0) => {
+                  wire.get(
+                    {"#": id},
+                    relMsg => {
+                      if (relMsg.put && relMsg.put[id]) {
+                        delete relMsg.put[id]._
+                        delete relMsg.put[id][utils.userPublicKey]
+                        delete relMsg.put[id][utils.userSignature]
+                        cb(relMsg.put[id])
+                      } else if (retries < 5) {
+                        // Data not ready yet, retry after a delay
+                        setTimeout(() => attemptRead(retries + 1), 50)
+                      } else {
+                        // Give up after retries
+                        cb(null)
+                      }
+                    },
+                    _opt,
+                  )
+                }
+                attemptRead()
+              } else {
+                // It's a direct property or null
+                cb(current !== undefined ? current : null)
+              }
+            },
+            _opt,
+          )
+        })
+
+        // Register listener immediately to avoid missing updates due to
+        // queueing. Initially register the soul without _get, then update if
+        // it's a rel.
+        let initialLex
+        if (lex) initialLex = utils.obj.put(lex, "#", soul)
+        else initialLex = {"#": soul, ".": item}
+        wire.on(initialLex, map.get(cb), false, _opt)
+
+        // Check if item is a rel and update listener if needed.
+        // This happens async but the listener is already registered above.
         wire.get(
           {"#": soul, ".": item},
           msg => {
@@ -560,13 +619,18 @@ const Holster = opt => {
             const current = msg.put && msg.put[soul] && msg.put[soul][item]
             const id = utils.rel.is(current)
             if (id) {
-              if (lex) lex = utils.obj.put(lex, "#", id)
-              else lex = {"#": id, ".": null}
-            } else {
-              if (lex) lex = utils.obj.put(lex, "#", soul)
-              else lex = {"#": soul, ".": item}
+              // It's a rel, need to switch listener to the related node
+              // First remove the initial listener
+              wire.off(initialLex, map.get(cb))
+              // Then add listener on the related node
+              let relLex
+              if (lex) relLex = utils.obj.put(lex, "#", id)
+              else relLex = {"#": id, ".": null}
+              wire.on(relLex, map.get(cb), _get, _opt)
+            } else if (_get) {
+              // Not a rel, but _get was requested, so trigger callback.
+              map.get(cb)()
             }
-            wire.on(lex, map.get(cb), _get, _opt)
           },
           _opt,
         )

package/src/radisk.js

@@ -58,7 +58,7 @@ const Radisk = opt => {
   // Performance logging
   const perfLog = (operation, startTime, key, size) => {
     const duration = Date.now() - startTime
-    if (duration > 1000) {
+    if (duration > 2000) {
       console.log(
         `[RADISK-SLOW] ${operation}: ${duration}ms for ${key}${size ? ` (${size} bytes)` : ""}`,
       )

package/src/wire.js

@@ -178,6 +178,9 @@ const Wire = opt => {
   // Track references we want but don't have yet
   const pendingReferences = new Set()
 
+  // Track pending timeouts that should start when message is sent
+  const pendingTimeouts = new Map()
+
   // Helper to check if we have a soul in memory or storage
   const hasSoul = async soul => {
     if (graph[soul]) return true
@@ -318,24 +321,20 @@ const Wire = opt => {
       if (err) console.log(err)
 
       queue[track] = cb
+
+      // Store timeout config to start after message is sent from queue
+      pendingTimeouts.set(track, {
+        lex: lex,
+        wait: _opt.wait || 100,
+      })
+
       const sendResult = send(request)
       if (sendResult && sendResult.err) {
         cb({err: sendResult.err})
         delete queue[track]
+        pendingTimeouts.delete(track)
         return
       }
-
-      // Respond to callback with null if no response.
-      setTimeout(() => {
-        const cb = queue[track]
-        if (cb) {
-          const id = lex["#"]
-          const ack = {[id]: null}
-          if (typeof lex["."] === "string") ack[id] = {[lex["."]]: null}
-          cb({put: ack})
-          delete queue[track]
-        }
-      }, _opt.wait || 100)
     })
   }
 
@@ -384,7 +383,7 @@ const Wire = opt => {
             put: data,
           }),
         )
-        // Handle queue overflow error
+        // Handle queue overflow error.
         if (sendResult && sendResult.err) {
           if (cb) cb(sendResult.err)
           return
@@ -432,6 +431,28 @@ const Wire = opt => {
     }
 
     const send = (data, isBinary) => {
+      // Start timeout for this message now that it's being sent
+      const msg = JSON.parse(data)
+      const trackId = msg["#"]
+      if (trackId && pendingTimeouts.has(trackId)) {
+        const timeoutConfig = pendingTimeouts.get(trackId)
+        pendingTimeouts.delete(trackId)
+
+        // Respond to callback with null if no response.
+        timeoutConfig.timeoutId = setTimeout(() => {
+          const cb = queue[trackId]
+          if (cb) {
+            const id = timeoutConfig.lex["#"]
+            const ack = {[id]: null}
+            if (typeof timeoutConfig.lex["."] === "string") {
+              ack[id] = {[timeoutConfig.lex["."]]: null}
+            }
+            cb({put: ack})
+            delete queue[trackId]
+          }
+        }, timeoutConfig.wait)
+      }
+
       clients().forEach(client => {
         if (client && client.readyState === WebSocket.OPEN) {
           client.send(data, {binary: isBinary})
@@ -590,6 +611,28 @@ const Wire = opt => {
     const data = messageQueue.shift()
     sendToPeers(data)
 
+    // Start timeout for this message now that it's been sent
+    const msg = JSON.parse(data)
+    const trackId = msg["#"]
+    if (trackId && pendingTimeouts.has(trackId)) {
+      const timeoutConfig = pendingTimeouts.get(trackId)
+      pendingTimeouts.delete(trackId)
+
+      // Respond to callback with null if no response.
+      timeoutConfig.timeoutId = setTimeout(() => {
+        const cb = queue[trackId]
+        if (cb) {
+          const id = timeoutConfig.lex["#"]
+          const ack = {[id]: null}
+          if (typeof timeoutConfig.lex["."] === "string") {
+            ack[id] = {[timeoutConfig.lex["."]]: null}
+          }
+          cb({put: ack})
+          delete queue[trackId]
+        }
+      }, timeoutConfig.wait)
+    }
+
     // Schedule processing of next message if queue not empty
     if (messageQueue.length > 0) {
       // Add delay between messages to respect rate limits (300 req/min = 200ms)

package/test/holster.user.on.test.js

@@ -35,11 +35,12 @@ describe("holster.user.on", () => {
     user.get("key").put("value", err => {
       assert.equal(err, null)
 
-      user.get("key").on(data => {
+      const callback = data => {
         assert.equal(data, "update")
-        user.get("key").off()
+        user.get("key").off(callback)
         done()
-      })
+      }
+      user.get("key").on(callback)
 
       user.get("key").put("update", err => {
         assert.equal(err, null)
@@ -64,15 +65,16 @@ describe("holster.user.on", () => {
       assert.equal(err, null)
 
       let first = true
-      user.get("key3").on(data => {
+      const callback = data => {
         if (first) {
           assert.equal(data, "update1")
         } else {
           assert.equal(data, "update2")
-          user.get("key3").off()
+          user.get("key3").off(callback)
           done()
         }
-      })
+      }
+      user.get("key3").on(callback)
 
       user.get("key3").put("update1", err => {
         assert.equal(err, null)
@@ -103,11 +105,13 @@ describe("holster.user.on", () => {
     setTimeout(async () => {
       let count = 0
       for (let i = 0; i < 5; i++) {
-        user.get("for" + i).on(data => {
-          assert.equal(data, "update" + i)
-          user.get("for" + i).off()
+        const idx = i // Capture value
+        const callback = data => {
+          assert.equal(data, "update" + idx)
+          user.get("for" + idx).off(callback)
           count++
-        })
+        }
+        user.get("for" + idx).on(callback)
       }
 
       setTimeout(async () => {
@@ -130,11 +134,12 @@ describe("holster.user.on", () => {
       assert.equal(err, null)
 
       setTimeout(() => {
-        user.get("key4").on(data => {
+        const callback = data => {
           assert.equal(data, "value4")
-          user.get("key4").off()
+          user.get("key4").off(callback)
           done()
-        }, true)
+        }
+        user.get("key4").on(callback, true)
       }, 100)
     })
   })
@@ -147,14 +152,12 @@ describe("holster.user.on", () => {
       .put("ok", err => {
         assert.equal(err, null)
 
-        user
-          .get("hello")
-          .next("world!")
-          .on(data => {
-            assert.equal(data, "update")
-            user.get("hello").next("world!").off()
-            done()
-          })
+        const callback = data => {
+          assert.equal(data, "update")
+          user.get("hello").next("world!").off(callback)
+          done()
+        }
+        user.get("hello").next("world!").on(callback)
 
         setTimeout(() => {
           user
@@ -181,10 +184,12 @@ describe("holster.user.on", () => {
         number: 42,
       }
 
-      user.get("plain").on(data => {
+      const callback = data => {
         assert.deepEqual(data, update)
+        user.get("plain").off(callback)
         done()
-      })
+      }
+      user.get("plain").on(callback)
 
       setTimeout(() => {
         user.get("plain").put(update, err => {
@@ -212,17 +217,18 @@ describe("holster.user.on", () => {
         },
       }
 
-      user.get("nested").on(data => {
+      const parentCallback = data => {
         assert.deepEqual(data, update)
-      })
+      }
+      user.get("nested").on(parentCallback)
 
-      user
-        .get("nested")
-        .next("child")
-        .on(data => {
-          assert.deepEqual(data, update.child)
-          done()
-        })
+      const childCallback = data => {
+        assert.deepEqual(data, update.child)
+        user.get("nested").off(parentCallback)
+        user.get("nested").next("child").off(childCallback)
+        done()
+      }
+      user.get("nested").next("child").on(childCallback)
 
       setTimeout(() => {
         user.get("nested").put(update, err => {

package/test/system/basic.test.js

@@ -0,0 +1,29 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - basic setup", () => {
+  const wss = new Server("ws://localhost:9001")
+  const holster = Holster({file: "test/system/basic", wss: wss})
+
+  test("put and get data", (t, done) => {
+    holster.get("test").put({message: "hello"}, err => {
+      assert.equal(err, null)
+
+      holster.get("test", data => {
+        assert.notEqual(data, null)
+        assert.equal(data.message, "hello")
+        done()
+      })
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/system/basic", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})

package/test/system/chained-on.test.js

@@ -0,0 +1,41 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - chained get with on", () => {
+  const wss = new Server("ws://localhost:9002")
+  const holster = Holster({file: "test/system/chained-on", wss: wss})
+
+  test("on listener fires with null when no data exists", (t, done) => {
+    holster.get("key").on(data => {
+      // This is the actual behavior - on fires with null for non-existent keys
+      assert.equal(data, null)
+      done()
+    }, true)
+  })
+
+  test("put then on listener receives data", (t, done) => {
+    // Put data first
+    holster.get("key2").put({value: "test"}, err => {
+      assert.equal(err, null)
+
+      // Then set up listener - should get the existing data
+      setTimeout(() => {
+        holster.get("key2").on(data => {
+          assert.notEqual(data, null)
+          assert.equal(data.value, "test")
+          done()
+        }, true)
+      }, 50)
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/system/chained-on", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})

package/test/system/concurrent-listeners.test.js

@@ -0,0 +1,124 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - concurrent listeners", () => {
+  const wss = new Server("ws://localhost:9005")
+  const holster = Holster({file: "test/system/concurrent-listeners", wss: wss})
+
+  test("multiple listeners on same path receive data", (t, done) => {
+    let listener1Called = false
+    let listener2Called = false
+
+    holster.get("key").put({value: "test"}, err => {
+      assert.equal(err, null)
+
+      // Set up two listeners on the same path
+      holster.get("key").on(data => {
+        listener1Called = true
+        assert.notEqual(data, null)
+        assert.equal(data.value, "test")
+        checkComplete()
+      }, true)
+
+      holster.get("key").on(data => {
+        listener2Called = true
+        assert.notEqual(data, null)
+        assert.equal(data.value, "test")
+        checkComplete()
+      }, true)
+
+      function checkComplete() {
+        if (listener1Called && listener2Called) {
+          done()
+        }
+      }
+    })
+  })
+
+  test("nested listeners on parent and child paths", (t, done) => {
+    let parentCalled = false
+    let childCalled = false
+
+    holster.get("parent").put({value: "parent-value"}, err => {
+      assert.equal(err, null)
+
+      holster
+        .get("parent")
+        .next("child")
+        .put({value: "child-value"}, err => {
+          assert.equal(err, null)
+
+          // Listen on parent
+          holster.get("parent").on(data => {
+            parentCalled = true
+            assert.notEqual(data, null)
+            assert.equal(data.value, "parent-value")
+            checkComplete()
+          }, true)
+
+          // Listen on child
+          holster
+            .get("parent")
+            .next("child")
+            .on(data => {
+              childCalled = true
+              assert.notEqual(data, null)
+              assert.equal(data.value, "child-value")
+              checkComplete()
+            }, true)
+
+          function checkComplete() {
+            if (parentCalled && childCalled) {
+              done()
+            }
+          }
+        })
+    })
+  })
+
+  test("listener race condition - data before and after listener setup", (t, done) => {
+    const results = []
+
+    // Put data first
+    holster.get("race").put({value: "first"}, err => {
+      assert.equal(err, null)
+
+      // Set up listener with once=true to read existing data
+      holster.get("race").on(data => {
+        results.push(data?.value)
+      }, true)
+
+      // Immediately put new data
+      setTimeout(() => {
+        holster.get("race").put({value: "second"}, err => {
+          assert.equal(err, null)
+
+          setTimeout(() => {
+            // Should have received both values
+            assert.equal(
+              results.length >= 1,
+              true,
+              "Should receive at least initial value",
+            )
+            assert.equal(results[0], "first", "First value should be 'first'")
+            done()
+          }, 100)
+        })
+      }, 50)
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm(
+      "test/system/concurrent-listeners",
+      {recursive: true, force: true},
+      err => {
+        assert.equal(err, null)
+        done()
+      },
+    )
+  })
+})

package/test/system/immediate-read.test.js

@@ -0,0 +1,30 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - immediate read after write", () => {
+  const wss = new Server("ws://localhost:9008")
+  const holster = Holster({file: "test/system/immediate-read", wss: wss})
+
+  test("get immediately after put returns data despite queue delays", (t, done) => {
+    holster.get("testkey").put({value: "test"}, err => {
+      assert.equal(err, null)
+
+      // Immediate get after put - validates retry logic handles queued writes
+      holster.get("testkey", data => {
+        assert.notEqual(data, null)
+        assert.equal(data.value, "test")
+        done()
+      })
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/system/immediate-read", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})

package/test/system/listener-before-data.test.js

@@ -0,0 +1,50 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - listener before data", () => {
+  const wss = new Server("ws://localhost:9007")
+  const holster = Holster({file: "test/system/listener-before-data", wss: wss})
+
+  test("listener set up before data exists receives data when populated", (t, done) => {
+    const results = []
+
+    // Set up listener first - this is the browser Display.js pattern
+    holster.get("testkey").on(data => {
+      results.push(data)
+    })
+
+    // Give listener time to set up
+    setTimeout(() => {
+      holster.get("testkey").put({value: "test"}, err => {
+        assert.equal(err, null)
+
+        // Give listener time to fire
+        setTimeout(() => {
+          // Should have received null (initial) and the test value
+          const nonNull = results.filter(r => r !== null)
+          assert.equal(
+            nonNull.length >= 1,
+            true,
+            "Should have at least one non-null result",
+          )
+          assert.equal(nonNull[0].value, "test")
+          done()
+        }, 200)
+      })
+    }, 100)
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm(
+      "test/system/listener-before-data",
+      {recursive: true, force: true},
+      err => {
+        assert.equal(err, null)
+        done()
+      },
+    )
+  })
+})