@mblaney/holster 1.0.15 → 1.0.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.15",
+  "version": "1.0.17",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/holster.js

@@ -67,12 +67,22 @@ const Holster = opt => {
             for (const key of Object.keys(msg.put[soul])) {
               const id = utils.rel.is(msg.put[soul][key])
               if (id) {
-                const data = await new Promise(res => {
-                  const _ctxid = utils.text.random()
-                  allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
-                  api(_ctxid).next(null, res, _opt)
-                })
-                msg.put[soul][key] = data
+                // Due to message queuing, the related node might not be available yet
+                // Retry with delays if we get null
+                const attemptRead = async (retries = 0) => {
+                  const data = await new Promise(res => {
+                    const _ctxid = utils.text.random()
+                    allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
+                    api(_ctxid).next(null, res, _opt)
+                  })
+                  if (data !== null || retries >= 5) {
+                    return data
+                  }
+                  // Data not ready, retry after delay
+                  await new Promise(resolve => setTimeout(resolve, 50))
+                  return attemptRead(retries + 1)
+                }
+                msg.put[soul][key] = await attemptRead()
               }
             }
             ack(msg.put[soul])
@@ -547,8 +557,57 @@ const Holster = opt => {
         allctx.set(ctxid, {chain: [{item: item, soul: soul}], on: true})
         // Map the user's callback because it can also be passed to off,
         // so need a reference to it to compare them.
-        map.set(cb, () => api(ctxid).next(null, cb, _opt))
-        // Check if item is a rel and add event listener for the node.
+        // Create a new context for each listener invocation to avoid mutation
+        map.set(cb, () => {
+          // When listener fires, re-check if the item is now a rel.
+          wire.get(
+            {"#": soul, ".": item},
+            msg => {
+              const current = msg.put && msg.put[soul] && msg.put[soul][item]
+              const id = utils.rel.is(current)
+              if (id) {
+                // It's a rel, read the related node. It might not be in the graph
+                // yet when the listener fires, so we retry with delays.
+                const attemptRead = (retries = 0) => {
+                  wire.get(
+                    {"#": id},
+                    relMsg => {
+                      if (relMsg.put && relMsg.put[id]) {
+                        delete relMsg.put[id]._
+                        delete relMsg.put[id][utils.userPublicKey]
+                        delete relMsg.put[id][utils.userSignature]
+                        cb(relMsg.put[id])
+                      } else if (retries < 5) {
+                        // Data not ready yet, retry after a delay
+                        setTimeout(() => attemptRead(retries + 1), 50)
+                      } else {
+                        // Give up after retries
+                        cb(null)
+                      }
+                    },
+                    _opt,
+                  )
+                }
+                attemptRead()
+              } else {
+                // It's a direct property or null
+                cb(current !== undefined ? current : null)
+              }
+            },
+            _opt,
+          )
+        })
+
+        // Register listener immediately to avoid missing updates due to
+        // queueing. Initially register the soul without _get, then update if
+        // it's a rel.
+        let initialLex
+        if (lex) initialLex = utils.obj.put(lex, "#", soul)
+        else initialLex = {"#": soul, ".": item}
+        wire.on(initialLex, map.get(cb), false, _opt)
+
+        // Check if item is a rel and update listener if needed.
+        // This happens async but the listener is already registered above.
         wire.get(
           {"#": soul, ".": item},
           msg => {
@@ -560,13 +619,18 @@ const Holster = opt => {
             const current = msg.put && msg.put[soul] && msg.put[soul][item]
             const id = utils.rel.is(current)
             if (id) {
-              if (lex) lex = utils.obj.put(lex, "#", id)
-              else lex = {"#": id, ".": null}
-            } else {
-              if (lex) lex = utils.obj.put(lex, "#", soul)
-              else lex = {"#": soul, ".": item}
+              // It's a rel, need to switch listener to the related node
+              // First remove the initial listener
+              wire.off(initialLex, map.get(cb))
+              // Then add listener on the related node
+              let relLex
+              if (lex) relLex = utils.obj.put(lex, "#", id)
+              else relLex = {"#": id, ".": null}
+              wire.on(relLex, map.get(cb), _get, _opt)
+            } else if (_get) {
+              // Not a rel, but _get was requested, so trigger callback.
+              map.get(cb)()
             }
-            wire.on(lex, map.get(cb), _get, _opt)
           },
           _opt,
         )

package/src/radisk.js

@@ -58,7 +58,7 @@ const Radisk = opt => {
   // Performance logging
   const perfLog = (operation, startTime, key, size) => {
     const duration = Date.now() - startTime
-    if (duration > 1000) {
+    if (duration > 2000) {
       console.log(
         `[RADISK-SLOW] ${operation}: ${duration}ms for ${key}${size ? ` (${size} bytes)` : ""}`,
       )

package/src/wire.js

@@ -13,13 +13,11 @@ if (typeof globalThis.WebSocket === "undefined") {
 }
 
 // Rate limiting with throttling
-const createRateLimiter = opt => {
+const createRateLimiter = isTestEnv => {
   const clients = new Map()
-  const maxRequests = opt.maxRequestsPerMinute || 100
-  const windowMs = opt.rateLimitWindow || 60000
-  const disconnectThreshold = opt.disconnectThreshold || 10
-  // Check if test environment (mock-socket usage)
-  const isTestEnv = opt.wss && opt.wss.constructor.name === "Server"
+  const maxRequests = 300 // 5 requests per second
+  const windowMs = 60000 // 1 minute window
+  const disconnectThreshold = 10 // Disconnect after 10 violations
   let cleanupInterval = null
 
   const cleanup = () => {
@@ -180,6 +178,9 @@ const Wire = opt => {
   // Track references we want but don't have yet
   const pendingReferences = new Set()
 
+  // Track pending timeouts that should start when message is sent
+  const pendingTimeouts = new Map()
+
   // Helper to check if we have a soul in memory or storage
   const hasSoul = async soul => {
     if (graph[soul]) return true
@@ -191,7 +192,9 @@ const Wire = opt => {
   }
 
   // Initialize rate limiting and connection management
-  const rateLimiter = createRateLimiter(opt)
+  // Check if test environment (mock-socket usage)
+  const isTestEnv = opt.wss && opt.wss.constructor.name === "Server"
+  const rateLimiter = createRateLimiter(isTestEnv)
   const connectionManager = createConnectionManager(opt.maxConnections || 1000)
 
   // The check function is required because user data must provide a public key
@@ -318,24 +321,20 @@ const Wire = opt => {
       if (err) console.log(err)
 
       queue[track] = cb
+
+      // Store timeout config to start after message is sent from queue
+      pendingTimeouts.set(track, {
+        lex: lex,
+        wait: _opt.wait || 100,
+      })
+
       const sendResult = send(request)
       if (sendResult && sendResult.err) {
         cb({err: sendResult.err})
         delete queue[track]
+        pendingTimeouts.delete(track)
         return
       }
-
-      // Respond to callback with null if no response.
-      setTimeout(() => {
-        const cb = queue[track]
-        if (cb) {
-          const id = lex["#"]
-          const ack = {[id]: null}
-          if (typeof lex["."] === "string") ack[id] = {[lex["."]]: null}
-          cb({put: ack})
-          delete queue[track]
-        }
-      }, _opt.wait || 100)
     })
   }
 
@@ -384,7 +383,7 @@ const Wire = opt => {
             put: data,
           }),
         )
-        // Handle queue overflow error
+        // Handle queue overflow error.
         if (sendResult && sendResult.err) {
           if (cb) cb(sendResult.err)
           return
@@ -432,6 +431,28 @@ const Wire = opt => {
     }
 
     const send = (data, isBinary) => {
+      // Start timeout for this message now that it's being sent
+      const msg = JSON.parse(data)
+      const trackId = msg["#"]
+      if (trackId && pendingTimeouts.has(trackId)) {
+        const timeoutConfig = pendingTimeouts.get(trackId)
+        pendingTimeouts.delete(trackId)
+
+        // Respond to callback with null if no response.
+        timeoutConfig.timeoutId = setTimeout(() => {
+          const cb = queue[trackId]
+          if (cb) {
+            const id = timeoutConfig.lex["#"]
+            const ack = {[id]: null}
+            if (typeof timeoutConfig.lex["."] === "string") {
+              ack[id] = {[timeoutConfig.lex["."]]: null}
+            }
+            cb({put: ack})
+            delete queue[trackId]
+          }
+        }, timeoutConfig.wait)
+      }
+
       clients().forEach(client => {
         if (client && client.readyState === WebSocket.OPEN) {
           client.send(data, {binary: isBinary})
@@ -463,9 +484,7 @@ const Wire = opt => {
 
       // Generate unique client ID for rate limiting
       const clientId = utils.text.random(9)
-      console.log(
-        `[HOLSTER-CONNECT] New WebSocket client connected: ${clientId}`,
-      )
+      console.log(`New WebSocket client connected: ${clientId}`)
 
       ws.on("error", error => {
         console.log("WebSocket error:", error)
@@ -473,9 +492,7 @@ const Wire = opt => {
       })
 
       ws.on("close", () => {
-        console.log(
-          `[HOLSTER-DISCONNECT] WebSocket client disconnected: ${clientId}`,
-        )
+        console.log(`WebSocket client disconnected: ${clientId}`)
         connectionManager.remove(ws)
       })
 
@@ -505,14 +522,14 @@ const Wire = opt => {
         if (delay > 0) {
           const throttleCount = rateLimiter.getThrottleCount(clientId)
           console.log(
-            `[HOLSTER-THROTTLE] Client ${clientId}: rate limit exceeded, delay would be ${delay}ms, dropping message (throttle count: ${throttleCount})`,
+            `Client ${clientId}: rate limit exceeded, delay would be ${delay}ms, dropping message (throttle count: ${throttleCount})`,
           )
           // Check if we should disconnect the bad actor
           if (rateLimiter.shouldDisconnect(clientId)) {
             console.log(
-              `[HOLSTER-DISCONNECT] Client ${clientId}: Disconnecting bad actor after ${throttleCount} throttle violations`,
+              `Client ${clientId}: Disconnecting after ${throttleCount} throttle violations`,
             )
-            ws.close(1008, "Rate limit violations - bad actor")
+            ws.close(1008, "Rate limit violations")
             return
           }
 
@@ -565,7 +582,7 @@ const Wire = opt => {
   let throttleUntil = 0
   let messageQueue = []
   let queueProcessor = null
-  const maxQueueLength = opt.maxQueueLength || 1000
+  const maxQueueLength = opt.maxQueueLength || 10000
 
   const processQueue = () => {
     if (messageQueue.length === 0) {
@@ -585,9 +602,7 @@ const Wire = opt => {
 
     // Throttle period ended, resume processing
     if (clientThrottled && now >= throttleUntil) {
-      console.log(
-        `[CLIENT-THROTTLED] Throttle period ended, resuming queue processing`,
-      )
+      console.log(`Throttle period ended, resuming queue processing`)
       clientThrottled = false
       throttleUntil = 0
     }
@@ -596,10 +611,32 @@ const Wire = opt => {
     const data = messageQueue.shift()
     sendToPeers(data)
 
+    // Start timeout for this message now that it's been sent
+    const msg = JSON.parse(data)
+    const trackId = msg["#"]
+    if (trackId && pendingTimeouts.has(trackId)) {
+      const timeoutConfig = pendingTimeouts.get(trackId)
+      pendingTimeouts.delete(trackId)
+
+      // Respond to callback with null if no response.
+      timeoutConfig.timeoutId = setTimeout(() => {
+        const cb = queue[trackId]
+        if (cb) {
+          const id = timeoutConfig.lex["#"]
+          const ack = {[id]: null}
+          if (typeof timeoutConfig.lex["."] === "string") {
+            ack[id] = {[timeoutConfig.lex["."]]: null}
+          }
+          cb({put: ack})
+          delete queue[trackId]
+        }
+      }, timeoutConfig.wait)
+    }
+
     // Schedule processing of next message if queue not empty
     if (messageQueue.length > 0) {
-      // Add small delay between messages to avoid immediate re-throttling
-      queueProcessor = setTimeout(processQueue, 100)
+      // Add delay between messages to respect rate limits (300 req/min = 200ms)
+      queueProcessor = setTimeout(processQueue, 250)
     } else {
       queueProcessor = null
     }
@@ -629,30 +666,19 @@ const Wire = opt => {
   }
 
   const send = data => {
-    // Check if client is throttled or queue is active
-    if (clientThrottled || messageQueue.length > 0) {
-      if (messageQueue.length >= maxQueueLength) {
-        return {
-          err: `Message queue exceeded maximum length (${maxQueueLength}). Update query logic to request less data.`,
-          queueLength: messageQueue.length,
-          maxQueueLength: maxQueueLength,
-        }
+    if (messageQueue.length >= maxQueueLength) {
+      return {
+        err: `Message queue exceeded maximum length (${maxQueueLength}). Update query logic to request less data.`,
+        queueLength: messageQueue.length,
+        maxQueueLength: maxQueueLength,
       }
-
-      // Add message to queue instead of sending immediately
-      messageQueue.push(data)
-      console.log(
-        `Queuing message (queue length: ${messageQueue.length}/${maxQueueLength})`,
-      )
-      // Start queue processor if not already running
-      if (!queueProcessor) {
-        queueProcessor = setTimeout(processQueue, 100)
-      }
-      return
     }
 
-    // Normal case - send immediately
-    sendToPeers(data)
+    messageQueue.push(data)
+    // Start queue processor if not already running
+    if (!queueProcessor) {
+      queueProcessor = setTimeout(processQueue, 250)
+    }
   }
   if (!(opt.peers instanceof Array)) {
     opt.peers = [`ws://localhost:${opt.port || 8765}`]
@@ -689,7 +715,7 @@ const Wire = opt => {
 
         // Handle throttle messages from server
         if (msg.throttle && msg.err) {
-          console.log(`[CLIENT-THROTTLED] Server throttling: ${msg.err}`)
+          console.log(`Server throttling: ${msg.err}`)
           clientThrottled = true
           throttleUntil = Date.now() + msg.throttle
           return

package/test/holster.user.on.test.js

@@ -35,11 +35,12 @@ describe("holster.user.on", () => {
     user.get("key").put("value", err => {
       assert.equal(err, null)
 
-      user.get("key").on(data => {
+      const callback = data => {
         assert.equal(data, "update")
-        user.get("key").off()
+        user.get("key").off(callback)
         done()
-      })
+      }
+      user.get("key").on(callback)
 
       user.get("key").put("update", err => {
         assert.equal(err, null)
@@ -64,15 +65,16 @@ describe("holster.user.on", () => {
       assert.equal(err, null)
 
       let first = true
-      user.get("key3").on(data => {
+      const callback = data => {
         if (first) {
           assert.equal(data, "update1")
         } else {
           assert.equal(data, "update2")
-          user.get("key3").off()
+          user.get("key3").off(callback)
           done()
         }
-      })
+      }
+      user.get("key3").on(callback)
 
       user.get("key3").put("update1", err => {
         assert.equal(err, null)
@@ -103,11 +105,13 @@ describe("holster.user.on", () => {
     setTimeout(async () => {
       let count = 0
       for (let i = 0; i < 5; i++) {
-        user.get("for" + i).on(data => {
-          assert.equal(data, "update" + i)
-          user.get("for" + i).off()
+        const idx = i // Capture value
+        const callback = data => {
+          assert.equal(data, "update" + idx)
+          user.get("for" + idx).off(callback)
           count++
-        })
+        }
+        user.get("for" + idx).on(callback)
       }
 
       setTimeout(async () => {
@@ -130,11 +134,12 @@ describe("holster.user.on", () => {
       assert.equal(err, null)
 
       setTimeout(() => {
-        user.get("key4").on(data => {
+        const callback = data => {
           assert.equal(data, "value4")
-          user.get("key4").off()
+          user.get("key4").off(callback)
           done()
-        }, true)
+        }
+        user.get("key4").on(callback, true)
       }, 100)
     })
   })
@@ -147,14 +152,12 @@ describe("holster.user.on", () => {
       .put("ok", err => {
         assert.equal(err, null)
 
-        user
-          .get("hello")
-          .next("world!")
-          .on(data => {
-            assert.equal(data, "update")
-            user.get("hello").next("world!").off()
-            done()
-          })
+        const callback = data => {
+          assert.equal(data, "update")
+          user.get("hello").next("world!").off(callback)
+          done()
+        }
+        user.get("hello").next("world!").on(callback)
 
         setTimeout(() => {
           user
@@ -181,10 +184,12 @@ describe("holster.user.on", () => {
         number: 42,
       }
 
-      user.get("plain").on(data => {
+      const callback = data => {
         assert.deepEqual(data, update)
+        user.get("plain").off(callback)
         done()
-      })
+      }
+      user.get("plain").on(callback)
 
       setTimeout(() => {
         user.get("plain").put(update, err => {
@@ -212,17 +217,18 @@ describe("holster.user.on", () => {
         },
       }
 
-      user.get("nested").on(data => {
+      const parentCallback = data => {
         assert.deepEqual(data, update)
-      })
+      }
+      user.get("nested").on(parentCallback)
 
-      user
-        .get("nested")
-        .next("child")
-        .on(data => {
-          assert.deepEqual(data, update.child)
-          done()
-        })
+      const childCallback = data => {
+        assert.deepEqual(data, update.child)
+        user.get("nested").off(parentCallback)
+        user.get("nested").next("child").off(childCallback)
+        done()
+      }
+      user.get("nested").next("child").on(childCallback)
 
       setTimeout(() => {
         user.get("nested").put(update, err => {

package/test/system/basic.test.js

@@ -0,0 +1,29 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - basic setup", () => {
+  const wss = new Server("ws://localhost:9001")
+  const holster = Holster({file: "test/system/basic", wss: wss})
+
+  test("put and get data", (t, done) => {
+    holster.get("test").put({message: "hello"}, err => {
+      assert.equal(err, null)
+
+      holster.get("test", data => {
+        assert.notEqual(data, null)
+        assert.equal(data.message, "hello")
+        done()
+      })
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/system/basic", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})

package/test/system/chained-on.test.js

@@ -0,0 +1,41 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../../src/holster.js"
+
+describe("system - chained get with on", () => {
+  const wss = new Server("ws://localhost:9002")
+  const holster = Holster({file: "test/system/chained-on", wss: wss})
+
+  test("on listener fires with null when no data exists", (t, done) => {
+    holster.get("key").on(data => {
+      // This is the actual behavior - on fires with null for non-existent keys
+      assert.equal(data, null)
+      done()
+    }, true)
+  })
+
+  test("put then on listener receives data", (t, done) => {
+    // Put data first
+    holster.get("key2").put({value: "test"}, err => {
+      assert.equal(err, null)
+
+      // Then set up listener - should get the existing data
+      setTimeout(() => {
+        holster.get("key2").on(data => {
+          assert.notEqual(data, null)
+          assert.equal(data.value, "test")
+          done()
+        }, true)
+      }, 50)
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/system/chained-on", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})