@mblaney/holster 1.0.11 → 1.0.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.11",
+  "version": "1.0.13",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/radisk.js

@@ -15,6 +15,8 @@ const Radisk = opt => {
 
   // Multi-file cache for all parsed radix trees
   const cache = new Map()
+  // Pending reads queue to avoid duplicate reads for the same key
+  const pendingReads = new Map()
 
   if (!opt) opt = {}
   if (!opt.log) opt.log = console.log
@@ -43,8 +45,8 @@ const Radisk = opt => {
   // Performance logging
   const perfLog = (operation, startTime, key, size) => {
     const duration = Date.now() - startTime
-    if (duration > 500) {
-      // Log operations taking more than 500ms
+    if (duration > 1000) {
+      // Log operations taking more than 1000ms
       console.log(
         `[RADISK-SLOW] ${operation}: ${duration}ms for ${key}${size ? ` (${size} bytes)` : ""}`,
       )
@@ -76,10 +78,23 @@ const Radisk = opt => {
         }
       }
 
+      // Check if there's already a pending read for this key
+      if (pendingReads.has(key)) {
+        // Add callback to existing pending read
+        pendingReads.get(key).push(cb)
+        return
+      }
+
+      // Start new read and create callback queue
+      pendingReads.set(key, [cb])
+
       const readStart = Date.now()
       return radisk.read(key, (err, result) => {
         perfLog("read", readStart, key)
-        cb(err, result)
+        // Execute all pending callbacks for this key
+        const callbacks = pendingReads.get(key) || []
+        pendingReads.delete(key)
+        callbacks.forEach(callback => callback(err, result))
       })
     }
 
@@ -123,7 +138,7 @@ const Radisk = opt => {
       // file needs to be split.
       if (++i > 1) return
 
-      perfLog("thrash", thrashStart, `batch-${radisk.batch.ed}`)
+      perfLog("thrash", thrashStart, `batch-${batch.ed}`)
       if (err) opt.log(err)
       batch.acks.forEach(cb => cb(err))
       radisk.thrash.at = null
@@ -319,6 +334,7 @@ const Radisk = opt => {
         if (!data) return cb(u, parse.disk)
 
         let pre = []
+        let preString = ""
         // Work though data by splitting into 3 values. The first value says
         // if the second value is one of: the radix level for a key, the key
         // iteself, or a value. The third is the rest of the data to work with.
@@ -330,15 +346,23 @@ const Radisk = opt => {
           tmp = parse.split(tmp[2]) || ""
           if (tmp[0] === "#") {
             key = tmp[1]
-            pre = pre.slice(0, i)
-            if (i <= pre.length) pre.push(key)
+            // Optimize prefix building - avoid repeated joins
+            if (i < pre.length) {
+              pre.length = i
+            }
+            if (i <= pre.length) {
+              pre[i] = key
+              pre.length = i + 1
+            }
+            // Only rebuild preString when prefix changes
+            preString = pre.join("")
           }
           tmp = parse.split(tmp[2]) || ""
           if (tmp[0] === "\n") continue
 
           if (tmp[0] === "=") value = tmp[1]
           if (typeof key !== "undefined" && typeof value !== "undefined") {
-            parse.disk(pre.join(""), value)
+            parse.disk(preString, value)
           }
           tmp = parse.split(tmp[2])
         }

package/src/wire.js

@@ -17,6 +17,7 @@ const createRateLimiter = opt => {
   const clients = new Map()
   const maxRequests = opt.maxRequestsPerMinute || 100
   const windowMs = opt.rateLimitWindow || 60000
+  const disconnectThreshold = opt.disconnectThreshold || 10
   // Check if test environment (mock-socket usage)
   const isTestEnv = opt.wss && opt.wss.constructor.name === "Server"
   let cleanupInterval = null
@@ -29,6 +30,10 @@ const createRateLimiter = opt => {
         data.lastCleanup = now
       }
       data.requests = data.requests.filter(time => now - time < windowMs)
+      // Reset throttle counts periodically
+      if (now - data.lastCleanup > windowMs * 10) {
+        data.throttleCount = 0
+      }
     }
   }
 
@@ -42,6 +47,7 @@ const createRateLimiter = opt => {
       const client = clients.get(clientId) || {
         requests: [],
         lastCleanup: now,
+        throttleCount: 0,
       }
 
       // Filter old requests
@@ -51,6 +57,9 @@ const createRateLimiter = opt => {
         // Calculate delay based on oldest request that will expire
         const oldestRequest = Math.min(...client.requests)
         const delay = windowMs - (now - oldestRequest)
+        // Increment throttle count and update client data
+        client.throttleCount = (client.throttleCount || 0) + 1
+        clients.set(clientId, client)
         return Math.max(0, delay)
       }
 
@@ -71,6 +80,17 @@ const createRateLimiter = opt => {
       return Math.max(0, maxRequests - validRequests.length)
     },
 
+    getThrottleCount: clientId => {
+      const client = clients.get(clientId)
+      return client ? client.throttleCount || 0 : 0
+    },
+
+    shouldDisconnect: clientId => {
+      const client = clients.get(clientId)
+      if (!client) return false
+      return client.throttleCount >= disconnectThreshold
+    },
+
     destroy: () => {
       if (cleanupInterval) {
         clearInterval(cleanupInterval)
@@ -274,7 +294,11 @@ const Wire = opt => {
 
     if (ack) {
       // Also send request on the wire to check for updates.
-      send(request)
+      const sendResult = send(request)
+      if (sendResult && sendResult.err) {
+        cb({err: sendResult.err})
+        return
+      }
       cb({put: ack})
       return
     }
@@ -282,7 +306,11 @@ const Wire = opt => {
     store.get(lex, (err, ack) => {
       if (ack) {
         // Also send request on the wire to check for updates.
-        send(request)
+        const sendResult = send(request)
+        if (sendResult && sendResult.err) {
+          cb({err: sendResult.err})
+          return
+        }
         cb({put: ack, err: err})
         return
       }
@@ -290,7 +318,13 @@ const Wire = opt => {
       if (err) console.log(err)
 
       queue[track] = cb
-      send(request)
+      const sendResult = send(request)
+      if (sendResult && sendResult.err) {
+        cb({err: sendResult.err})
+        delete queue[track]
+        return
+      }
+
       // Respond to callback with null if no response.
       setTimeout(() => {
         const cb = queue[track]
@@ -344,12 +378,17 @@ const Wire = opt => {
 
         store.put(update.now, cb)
         // Also put data on the wire spec.
-        send(
+        const sendResult = send(
           JSON.stringify({
             "#": dup.track(utils.text.random(9)),
             put: data,
           }),
         )
+        // Handle queue overflow error
+        if (sendResult && sendResult.err) {
+          if (cb) cb(sendResult.err)
+          return
+        }
       },
       on: (lex, cb, _get, _opt) => {
         const soul = lex && lex["#"]
@@ -464,9 +503,19 @@ const Wire = opt => {
         const delay = rateLimiter.getDelay(clientId)
         // Log rate limiting activity and enforce stricter limits
         if (delay > 0) {
+          const throttleCount = rateLimiter.getThrottleCount(clientId)
           console.log(
-            `[HOLSTER-THROTTLE] Client ${clientId}: rate limit exceeded, delay would be ${delay}ms, dropping message`,
+            `[HOLSTER-THROTTLE] Client ${clientId}: rate limit exceeded, delay would be ${delay}ms, dropping message (throttle count: ${throttleCount})`,
           )
+          // Check if we should disconnect the bad actor
+          if (rateLimiter.shouldDisconnect(clientId)) {
+            console.error(
+              `[HOLSTER-DISCONNECT] Client ${clientId}: Disconnecting bad actor after ${throttleCount} throttle violations`,
+            )
+            ws.close(1008, "Rate limit violations - bad actor")
+            return
+          }
+
           // Send throttle warning back to client instead of processing
           ws.send(
             JSON.stringify({
@@ -514,6 +563,47 @@ const Wire = opt => {
   // Client-side throttling state
   let clientThrottled = false
   let throttleUntil = 0
+  let messageQueue = []
+  let queueProcessor = null
+  const maxQueueLength = opt.maxQueueLength || 1000
+
+  const processQueue = () => {
+    if (messageQueue.length === 0) {
+      queueProcessor = null
+      return
+    }
+
+    const now = Date.now()
+    if (clientThrottled && now < throttleUntil) {
+      // Still throttled, schedule next queue check
+      queueProcessor = setTimeout(
+        processQueue,
+        Math.min(1000, throttleUntil - now),
+      )
+      return
+    }
+
+    // Throttle period ended, resume processing
+    if (clientThrottled && now >= throttleUntil) {
+      console.log(
+        `[CLIENT-THROTTLED] Throttle period ended, resuming queue processing`,
+      )
+      clientThrottled = false
+      throttleUntil = 0
+    }
+
+    // Process next message in queue
+    const data = messageQueue.shift()
+    sendToPeers(data)
+
+    // Schedule processing of next message if queue not empty
+    if (messageQueue.length > 0) {
+      // Add small delay between messages to avoid immediate re-throttling
+      queueProcessor = setTimeout(processQueue, 100)
+    } else {
+      queueProcessor = null
+    }
+  }
 
   const sendToPeers = data => {
     peers.forEach(peer => {
@@ -539,22 +629,29 @@ const Wire = opt => {
   }
 
   const send = data => {
-    // Check if client is throttled before sending
-    const now = Date.now()
-    if (clientThrottled && now < throttleUntil) {
-      const delay = throttleUntil - now
+    // Check if client is throttled or queue is active
+    if (clientThrottled || messageQueue.length > 0) {
+      if (messageQueue.length >= maxQueueLength) {
+        return {
+          err: `Message queue exceeded maximum length (${maxQueueLength}). Update query logic to request less data.`,
+          queueLength: messageQueue.length,
+          maxQueueLength: maxQueueLength,
+        }
+      }
+
+      // Add message to queue instead of sending immediately
+      messageQueue.push(data)
       console.log(
-        `[CLIENT-THROTTLED] Delaying message for ${delay}ms, throttled until ${new Date(throttleUntil).toLocaleTimeString()}`,
+        `Queuing message (queue length: ${messageQueue.length}/${maxQueueLength})`,
       )
-      setTimeout(() => sendToPeers(data), delay)
+      // Start queue processor if not already running
+      if (!queueProcessor) {
+        queueProcessor = setTimeout(processQueue, 100)
+      }
       return
-    } else if (clientThrottled && now >= throttleUntil) {
-      console.log(
-        `[CLIENT-THROTTLED] Throttle period ended, resuming normal operation`,
-      )
-      clientThrottled = false
-      throttleUntil = 0
     }
+
+    // Normal case - send immediately
     sendToPeers(data)
   }
   if (!(opt.peers instanceof Array)) {