@mblaney/holster 1.0.10 → 1.0.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.10",
+  "version": "1.0.12",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/radisk.js

@@ -12,11 +12,13 @@ const unit = String.fromCharCode(31)
 // opt.store interface.
 const Radisk = opt => {
   var u
-  var cache = null
+
+  // Multi-file cache for all parsed radix trees
+  const cache = new Map()
 
   if (!opt) opt = {}
   if (!opt.log) opt.log = console.log
-  if (!opt.batch) opt.batch = 10 * 1000
+  if (!opt.batch) opt.batch = 100 // Balanced batch size for performance
   if (!opt.write) opt.write = 1 // Wait time before write in milliseconds.
   if (!opt.size) opt.size = 1024 * 1024 // File size on disk, default 1MB.
   if (!opt.store) {
@@ -38,6 +40,17 @@ const Radisk = opt => {
     return
   }
 
+  // Performance logging
+  const perfLog = (operation, startTime, key, size) => {
+    const duration = Date.now() - startTime
+    if (duration > 500) {
+      // Log operations taking more than 500ms
+      console.log(
+        `[RADISK-SLOW] ${operation}: ${duration}ms for ${key}${size ? ` (${size} bytes)` : ""}`,
+      )
+    }
+  }
+
   // Any and all storage adapters should:
   // 1. Because writing to disk takes time, we should batch data to disk.
   //    This improves performance, and reduces potential disk corruption.
@@ -63,7 +76,11 @@ const Radisk = opt => {
         }
       }
 
-      return radisk.read(key, cb)
+      const readStart = Date.now()
+      return radisk.read(key, (err, result) => {
+        perfLog("read", readStart, key)
+        cb(err, result)
+      })
     }
 
     // Otherwise store the value provided.
@@ -90,6 +107,7 @@ const Radisk = opt => {
       return (radisk.thrash.more = true)
     }
 
+    const thrashStart = Date.now()
     clearTimeout(radisk.batch.timeout)
     radisk.thrash.more = false
     radisk.thrash.ing = true
@@ -105,6 +123,7 @@ const Radisk = opt => {
       // file needs to be split.
       if (++i > 1) return
 
+      perfLog("thrash", thrashStart, `batch-${radisk.batch.ed}`)
       if (err) opt.log(err)
       batch.acks.forEach(cb => cb(err))
       radisk.thrash.at = null
@@ -143,9 +162,9 @@ const Radisk = opt => {
       },
       mix: (file, start, end) => {
         save.start = save.end = save.file = u
-        radisk.parse(file, (err, disk) => {
-          if (err) return cb(err)
-
+        // Use cache if available, otherwise parse from disk
+        if (cache.has(file)) {
+          const disk = cache.get(file)
           Radix.map(rad, (value, key) => {
             if (key < start) return
 
@@ -157,7 +176,23 @@ const Radisk = opt => {
             disk(key, value)
           })
           radisk.write(file, disk, save.next)
-        })
+        } else {
+          radisk.parse(file, (err, disk) => {
+            if (err) return cb(err)
+
+            Radix.map(rad, (value, key) => {
+              if (key < start) return
+
+              if (end && end < key) {
+                save.start = key
+                return
+              }
+
+              disk(key, value)
+            })
+            radisk.write(file, disk, save.next)
+          })
+        }
       },
       next: err => {
         if (err) return cb(err)
@@ -171,8 +206,8 @@ const Radisk = opt => {
   }
 
   radisk.write = (file, rad, cb) => {
-    // Invalidate cache on write.
-    cache = null
+    // Invalidate cache entry for this file on write
+    cache.delete(file)
     const write = {
       text: "",
       limit: "",
@@ -218,14 +253,14 @@ const Radisk = opt => {
     Radix.map(rad, write.each, true)
     // There is always accumulated write.text to store once write.each has
     // finished.
-    opt.store.put(file, write.text, cb)
+    const writeStart = Date.now()
+    opt.store.put(file, write.text, err => {
+      perfLog("file-write", writeStart, file, write.text.length)
+      cb(err)
+    })
   }
 
   radisk.read = (key, cb) => {
-    if (cache) {
-      let value = cache(key)
-      if (typeof value !== "undefined") return cb(u, value)
-    }
     // Only the soul of the key is compared to filenames (see radisk.write).
     const end = key.indexOf(enq)
     const soul = end === -1 ? key : key.substring(0, end)
@@ -240,6 +275,14 @@ const Radisk = opt => {
             return
           }
 
+          // Check multi-file cache first
+          if (cache.has(read.file)) {
+            const cachedRadix = cache.get(read.file)
+            read.value = cachedRadix(key)
+            if (typeof read.value !== "undefined") {
+              return cb(u, read.value)
+            }
+          }
           radisk.parse(read.file, read.it)
           return
         }
@@ -252,7 +295,8 @@ const Radisk = opt => {
       it: (err, disk) => {
         if (err) opt.log(err)
         if (disk) {
-          cache = disk
+          // Store in multi-file cache
+          cache.set(read.file, disk)
           read.value = disk(key)
         }
         cb(err, read.value)
@@ -303,18 +347,12 @@ const Radisk = opt => {
       split: data => {
         if (!data) return
 
-        let i = -1
-        let a = ""
-        let c = null
-        while ((c = data[++i])) {
-          if (c === unit) break
+        const i = data.indexOf(unit)
+        if (i === -1) return
 
-          a += c
-        }
+        const a = data.slice(0, i)
         let o = {}
-        if (c) {
-          return [a, Radisk.decode(data.slice(i), o), data.slice(i + o.i)]
-        }
+        return [a, Radisk.decode(data.slice(i), o), data.slice(i + o.i)]
       },
     }
     opt.store.get(file, parse.read)
@@ -355,21 +393,25 @@ Radisk.encode = data => {
 }
 
 Radisk.decode = (data, obj) => {
-  var text = ""
   var i = -1
   var n = 0
   var current = null
   var previous = null
+  var textStart = -1
+  var textEnd = -1
   if (data[0] !== unit) return
 
   // Find a control character previous to the text we want, skipping
   // consecutive unit separator characters at the beginning of the data.
   while ((current = data[++i])) {
     if (previous) {
+      if (textStart === -1) textStart = i
       if (current === unit) {
-        if (--n <= 0) break
+        if (--n <= 0) {
+          textEnd = i
+          break
+        }
       }
-      text += current
     } else if (current === unit) {
       n++
     } else {
@@ -377,6 +419,9 @@ Radisk.decode = (data, obj) => {
     }
   }
 
+  const text =
+    textStart !== -1 ? data.slice(textStart, textEnd !== -1 ? textEnd : i) : ""
+
   if (obj) obj.i = i + 1
 
   let [value, state] = text.split(etx)

package/src/wire.js

@@ -424,6 +424,9 @@ const Wire = opt => {
 
       // Generate unique client ID for rate limiting
       const clientId = utils.text.random(9)
+      console.log(
+        `[HOLSTER-CONNECT] New WebSocket client connected: ${clientId}`,
+      )
 
       ws.on("error", error => {
         console.error("WebSocket error:", error)
@@ -431,6 +434,9 @@ const Wire = opt => {
       })
 
       ws.on("close", () => {
+        console.log(
+          `[HOLSTER-DISCONNECT] WebSocket client disconnected: ${clientId}`,
+        )
         connectionManager.remove(ws)
       })
 
@@ -456,6 +462,22 @@ const Wire = opt => {
 
         // Check rate limit and get delay
         const delay = rateLimiter.getDelay(clientId)
+        // Log rate limiting activity and enforce stricter limits
+        if (delay > 0) {
+          console.log(
+            `[HOLSTER-THROTTLE] Client ${clientId}: rate limit exceeded, delay would be ${delay}ms, dropping message`,
+          )
+          // Send throttle warning back to client instead of processing
+          ws.send(
+            JSON.stringify({
+              "#": dup.track(utils.text.random(9)),
+              "@": msg["#"],
+              err: `Rate limit exceeded. Slow down requests. Wait ${Math.ceil(delay / 1000)}s`,
+              throttle: delay,
+            }),
+          )
+          return
+        }
 
         const processMessage = () => {
           dup.track(msg["#"])
@@ -489,7 +511,51 @@ const Wire = opt => {
 
   // Browser logic.
   const peers = []
-  const send = data => {
+  // Client-side throttling state
+  let clientThrottled = false
+  let throttleUntil = 0
+  let messageQueue = []
+  let queueProcessor = null
+
+  const processQueue = () => {
+    if (messageQueue.length === 0) {
+      queueProcessor = null
+      return
+    }
+
+    const now = Date.now()
+    if (clientThrottled && now < throttleUntil) {
+      // Still throttled, schedule next queue check
+      queueProcessor = setTimeout(
+        processQueue,
+        Math.min(1000, throttleUntil - now),
+      )
+      return
+    }
+
+    // Throttle period ended, resume processing
+    if (clientThrottled && now >= throttleUntil) {
+      console.log(
+        `[CLIENT-THROTTLED] Throttle period ended, resuming queue processing`,
+      )
+      clientThrottled = false
+      throttleUntil = 0
+    }
+
+    // Process next message in queue
+    const data = messageQueue.shift()
+    sendToPeers(data)
+
+    // Schedule processing of next message if queue not empty
+    if (messageQueue.length > 0) {
+      // Add small delay between messages to avoid immediate re-throttling
+      queueProcessor = setTimeout(processQueue, 100)
+    } else {
+      queueProcessor = null
+    }
+  }
+
+  const sendToPeers = data => {
     peers.forEach(peer => {
       if (peer && peer.readyState === WebSocket.OPEN) {
         peer.send(data)
@@ -511,6 +577,27 @@ const Wire = opt => {
       }
     })
   }
+
+  const send = data => {
+    // Check if client is throttled or queue is active
+    const now = Date.now()
+    if (clientThrottled || messageQueue.length > 0) {
+      // Add message to queue instead of sending immediately
+      messageQueue.push(data)
+      console.log(
+        `[CLIENT-THROTTLED] Queuing message (queue length: ${messageQueue.length})`,
+      )
+
+      // Start queue processor if not already running
+      if (!queueProcessor) {
+        queueProcessor = setTimeout(processQueue, 100)
+      }
+      return
+    }
+
+    // Normal case - send immediately
+    sendToPeers(data)
+  }
   if (!(opt.peers instanceof Array)) {
     opt.peers = [`ws://localhost:${opt.port || 8765}`]
   }
@@ -544,6 +631,14 @@ const Wire = opt => {
         const msg = JSON.parse(m.data)
         if (dup.check(msg["#"])) return
 
+        // Handle throttle messages from server
+        if (msg.throttle && msg.err) {
+          console.log(`[CLIENT-THROTTLED] Server throttling: ${msg.err}`)
+          clientThrottled = true
+          throttleUntil = Date.now() + msg.throttle
+          return
+        }
+
         dup.track(msg["#"])
         if (msg.get) get(msg, send)
         if (msg.put) {