@mblaney/holster 2.0.1 → 2.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/ham.js

@@ -35,7 +35,14 @@ const Ham = (state, currentState, value, currentValue, signed = false) => {
   return {incoming: true}
 }
 
-Ham.mix = async (change, graph, hasNode, secure, listen) => {
+Ham.mix = async (
+  change,
+  graph,
+  hasNode,
+  secure,
+  listen,
+  maxGraphSize = MAX_GRAPH_SIZE,
+) => {
   if (!change || typeof change !== "object") {
     throw new TypeError("change must be an object")
   }
@@ -216,7 +223,7 @@ Ham.mix = async (change, graph, hasNode, secure, listen) => {
   }
 
   const souls = Object.keys(graph)
-  if (souls.length > MAX_GRAPH_SIZE) {
+  if (souls.length > maxGraphSize) {
     // Sort by oldest state timestamp and remove oldest entries
     const soulsByAge = souls
       .map(soul => {
@@ -225,7 +232,7 @@ Ham.mix = async (change, graph, hasNode, secure, listen) => {
         return {soul, maxState}
       })
       .sort((a, b) => a.maxState - b.maxState)
-    const remove = soulsByAge.slice(0, souls.length - MAX_GRAPH_SIZE)
+    const remove = soulsByAge.slice(0, souls.length - maxGraphSize)
     remove.forEach(({soul}) => {
       delete graph[soul]
       if (hasNode) hasNode.delete(soul)

package/src/holster.js

@@ -64,28 +64,30 @@ const Holster = opt => {
             delete msg.put[soul][utils.userPublicKey]
             delete msg.put[soul][utils.userSignature]
             // Resolve any rels on the node before returning to the user.
-            await Promise.all(Object.keys(msg.put[soul]).map(async key => {
-              const id = utils.rel.is(msg.put[soul][key])
-              if (!id) return
-              const attemptRead = async (retries = 0) => {
-                const data = await new Promise(res => {
-                  const _ctxid = utils.text.random()
-                  const ctx = allctx.get(ctxid)
-                  allctx.set(_ctxid, {
-                    chain: [{item: null, soul: id}],
-                    user: ctx ? ctx.user : null,
+            await Promise.all(
+              Object.keys(msg.put[soul]).map(async key => {
+                const id = utils.rel.is(msg.put[soul][key])
+                if (!id) return
+                const attemptRead = async (retries = 0) => {
+                  const data = await new Promise(res => {
+                    const _ctxid = utils.text.random()
+                    const ctx = allctx.get(ctxid)
+                    allctx.set(_ctxid, {
+                      chain: [{item: null, soul: id}],
+                      user: ctx ? ctx.user : null,
+                    })
+                    api(_ctxid).next(null, res, _opt)
                   })
-                  api(_ctxid).next(null, res, _opt)
-                })
-                if (data !== null || retries >= 5) {
-                  return data
+                  if (data !== null || retries >= 5) {
+                    return data
+                  }
+                  // Data not ready, retry after delay
+                  await new Promise(resolve => setTimeout(resolve, 50))
+                  return attemptRead(retries + 1)
                 }
-                // Data not ready, retry after delay
-                await new Promise(resolve => setTimeout(resolve, 50))
-                return attemptRead(retries + 1)
-              }
-              msg.put[soul][key] = await attemptRead()
-            }))
+                msg.put[soul][key] = await attemptRead()
+              }),
+            )
             ack(msg.put[soul])
           } else {
             // No data callback.
@@ -281,7 +283,11 @@ const Holster = opt => {
                 if (request._get) cb(null)
               } else if (get && node) {
                 const sv = node._ && node._[">"]
-                if (sv && Object.keys(sv).length > 0 && typeof sv[item] === "undefined") {
+                if (
+                  sv &&
+                  Object.keys(sv).length > 0 &&
+                  typeof sv[item] === "undefined"
+                ) {
                   // State vector has other properties but not this one — never written.
                   if (cb) cb(null)
                 } else {
@@ -666,8 +672,11 @@ const Holster = opt => {
                       chain: [{item: null, soul: id}],
                       user: ctx ? ctx.user : null,
                     })
-                    api(_ctxid).next(null, res, retries === 0 ?
-                                     utils.obj.put(_opt, "fast", true) : _opt)
+                    api(_ctxid).next(
+                      null,
+                      res,
+                      retries === 0 ? utils.obj.put(_opt, "fast", true) : _opt,
+                    )
                   })
                   if (data !== null || retries >= 5) {
                     retryCount = 0 // Reset retry counter on success
@@ -693,8 +702,10 @@ const Holster = opt => {
                   // miss and never fire again if the data doesn't change while
                   // the listener is attached, so we must poll for initial load.
                   const retry = () => {
-                    const delay =
-                          Math.min(retryDelay * Math.pow(2, retryCount), 30000)
+                    const delay = Math.min(
+                      retryDelay * Math.pow(2, retryCount),
+                      30000,
+                    )
                     retryCount++
                     retryTimer = setTimeout(() => {
                       wire.get(

package/src/radisk.js

@@ -34,7 +34,7 @@ const Radisk = opt => {
   if (!opt.write) opt.write = 1 // Wait time before write in milliseconds
   if (!opt.size) opt.size = 1024 * 1024 // File size on disk, default 1MB
   if (!opt.memoryLimit) opt.memoryLimit = 500 // Memory limit in MB
-  if (typeof opt.cache === "undefined") opt.cache = true
+  if (typeof opt.cache === "undefined") opt.cache = true // Used in tests
 
   if (!opt.store) {
     opt.log(
@@ -55,16 +55,6 @@ const Radisk = opt => {
     return
   }
 
-  // Performance logging
-  const perfLog = (operation, startTime, key, size) => {
-    const duration = Date.now() - startTime
-    if (duration > 3000) {
-      console.log(
-        `[RADISK-SLOW] ${operation}: ${duration}ms for ${key}${size ? ` (${size} bytes)` : ""}`,
-      )
-    }
-  }
-
   // Memory monitoring and cache cleanup
   const checkMemoryUsage = () => {
     if (typeof process === "undefined" || !process.memoryUsage) return
@@ -129,7 +119,6 @@ const Radisk = opt => {
       }
 
       pendingReads.set(key, [cb])
-      const readStart = Date.now()
       // If radisk.read genuinely hangs (fd leak, stuck I/O), recover
       // callbacks as a cache miss so callers can fall through to the wire path.
       // 30s is long enough to distinguish a hang from a slow-but-healthy read.
@@ -142,7 +131,6 @@ const Radisk = opt => {
       }, 30000)
       return radisk.read(key, (err, result) => {
         clearTimeout(timeout)
-        perfLog("read", readStart, key)
         const callbacks = pendingReads.get(key) || []
         pendingReads.delete(key)
         callbacks.forEach(callback => callback(err, result))
@@ -175,7 +163,6 @@ const Radisk = opt => {
       return (radisk.thrash.more = true)
     }
 
-    const thrashStart = Date.now()
     clearTimeout(radisk.batch.timeout)
     radisk.thrash.more = false
     radisk.thrash.ing = true
@@ -191,7 +178,6 @@ const Radisk = opt => {
       // file needs to be split.
       if (++i > 1) return
 
-      perfLog("thrash", thrashStart, `batch-${batch.ed}`)
       if (err) opt.log(err)
       batch.acks.forEach(cb => cb(err))
       radisk.thrash.at = null
@@ -279,7 +265,7 @@ const Radisk = opt => {
       limit: "",
       done: false,
       count: 0,
-      halfKey: "",   // soul ID at the ~opt.size/2 mark for a balanced split
+      halfKey: "", // soul ID at the ~opt.size/2 mark for a balanced split
       halfOffset: 0, // write.text.length before that entry
       each: (value, key, k, pre) => {
         // each is called for all keys, but stop adding to write.text when
@@ -309,8 +295,7 @@ const Radisk = opt => {
         ) {
           const fullKey = pre.join("") + k
           const end = fullKey.indexOf(enq)
-          write.halfKey =
-            end === -1 ? fullKey : fullKey.substring(0, end)
+          write.halfKey = end === -1 ? fullKey : fullKey.substring(0, end)
           write.halfOffset = write.text.length
         }
 
@@ -322,8 +307,7 @@ const Radisk = opt => {
           const fullKey = pre.join("") + k
           const end = fullKey.indexOf(enq)
           write.limit =
-            write.halfKey ||
-            (end === -1 ? fullKey : fullKey.substring(0, end))
+            write.halfKey || (end === -1 ? fullKey : fullKey.substring(0, end))
           // Cannot split if the key is the same as the current file name.
           if (write.limit !== file) {
             // Truncate to the balanced split point — text after halfOffset is
@@ -350,7 +334,6 @@ const Radisk = opt => {
     Radix.map(rad, write.each, true)
     // There is always accumulated write.text to store once write.each has
     // finished.
-    const writeStart = Date.now()
     opt.store.put(file, write.text, err => {
       // If a split occurred the cached radix tree still contains entries that
       // were moved to the sub-file. Invalidate so the next save.mix re-reads
@@ -359,7 +342,6 @@ const Radisk = opt => {
         cache.delete(file)
         fileListCache = null
       }
-      perfLog("file-write", writeStart, file, write.text.length)
       cb(err)
     })
   }
@@ -371,17 +353,26 @@ const Radisk = opt => {
 
     // Binary search for largest filename <= soul in a sorted array.
     const findFile = files => {
-      let lo = 0, hi = files.length - 1, result = null
+      let lo = 0,
+        hi = files.length - 1,
+        result = null
       while (lo <= hi) {
         const mid = (lo + hi) >> 1
-        if (files[mid] <= soul) { result = files[mid]; lo = mid + 1 }
-        else { hi = mid - 1 }
+        if (files[mid] <= soul) {
+          result = files[mid]
+          lo = mid + 1
+        } else {
+          hi = mid - 1
+        }
       }
       return result
     }
 
     const read = file => {
-      if (!file) { cb("no file found", u); return }
+      if (!file) {
+        cb("no file found", u)
+        return
+      }
       if (opt.cache && cache.has(file)) {
         return cb(u, cache.get(file)(key))
       }
@@ -396,12 +387,19 @@ const Radisk = opt => {
     }
 
     const now = Date.now()
-    if (opt.cache && fileListCache && now - fileListCacheTime < FILE_LIST_CACHE_TTL) {
+    if (
+      opt.cache &&
+      fileListCache &&
+      now - fileListCacheTime < FILE_LIST_CACHE_TTL
+    ) {
       read(findFile(fileListCache, soul))
     } else {
       const files = []
       opt.store.list(file => {
-        if (file) { files.push(file); return }
+        if (file) {
+          files.push(file)
+          return
+        }
         files.sort()
         if (opt.cache) {
           fileListCache = files

package/src/store.js

@@ -197,7 +197,7 @@ const Store = opt => {
   const radisk = Radisk(opt)
 
   return {
-    get: (lex, cb, _opt) => {
+    get: (lex, cb) => {
       if (!lex || !utils.obj.is(lex)) {
         cb("lex required")
         return
@@ -207,41 +207,54 @@ const Store = opt => {
         return
       }
 
-      if (!_opt) _opt = {}
-
       var soul = lex["#"]
-      var key = !_opt.secure && typeof lex["."] === "string" ? lex["."] : ""
+      var key = typeof lex["."] === "string" ? lex["."] : ""
       var node
       var signatures = {}
-      const each = (value, key) => {
-        // Always include userPublicKey for verification, regardless of filter
-        if (key !== utils.userPublicKey && !utils.match(lex["."], key)) return
+      const each = (value, k) => {
+        if (k !== utils.userPublicKey && !utils.match(lex["."], k)) return
 
         if (!node) node = {_: {"#": soul, ">": {}}}
-        node[key] = value[0]
-        node._[">"][key] = value[1]
+        node[k] = value[0]
+        node._[">"][k] = value[1]
         // If signature is present, store it in _["s"]
         if (value.length === 3 && value[2]) {
           signatures[value[1]] = value[2]
         }
       }
 
+      // When doing a targeted property lookup, also fetch userPublicKey in
+      // parallel so ham.js can verify signatures regardless of secure mode.
+      // Full-soul scans (key="") already surface it via Radix.map.
+      const needsPubKey = key !== "" && key !== utils.userPublicKey
+      let pending = needsPubKey ? 2 : 1
+      let cbErr = null
+      const finish = err => {
+        if (err && !cbErr) cbErr = err
+        if (--pending > 0) return
+        const graph = {[soul]: node}
+        if (node && Object.keys(signatures).length > 0) {
+          node._["s"] = signatures
+        }
+        cb(cbErr, graph)
+      }
+
       radisk(soul + enq + key, (err, value) => {
-        let graph
         if (utils.obj.is(value)) {
           Radix.map(value, each)
           if (!node) each(value, key)
-          graph = {[soul]: node}
         } else if (value) {
           each(value, key)
-          graph = {[soul]: node}
         }
-        // Only add _["s"] if we found any signatures
-        if (graph && graph[soul] && Object.keys(signatures).length > 0) {
-          graph[soul]._["s"] = signatures
-        }
-        cb(err, graph)
+        finish(err)
       })
+
+      if (needsPubKey) {
+        radisk(soul + enq + utils.userPublicKey, (err, value) => {
+          if (value && !utils.obj.is(value)) each(value, utils.userPublicKey)
+          finish(err)
+        })
+      }
     },
     put: (graph, cb) => {
       if (!graph) {

package/src/user.js

@@ -28,54 +28,51 @@ const User = (opt, wire) => {
       }
 
       const pub = pubs.shift()
-      wire.get(
-        {"#": pub, ".": ["auth", "pub", "epub"]},
-        async msg => {
-          if (msg.err) {
-            done(`error getting ${pub}: ${msg.err}`)
-            return
-          }
+      wire.get({"#": pub, ".": ["auth", "pub", "epub"]}, async msg => {
+        if (msg.err) {
+          done(`error getting ${pub}: ${msg.err}`)
+          return
+        }
 
-          const data = msg.put && msg.put[pub]
-          if (!data || !data.auth) return next()
+        const data = msg.put && msg.put[pub]
+        if (!data || !data.auth) return next()
 
-          const auth = JSON.parse(data.auth)
-          const work = await SEA.work(password, auth.salt)
-          const dec = await SEA.decrypt(auth.enc, work)
-          if (!dec) return next()
+        const auth = JSON.parse(data.auth)
+        const work = await SEA.work(password, auth.salt)
+        const dec = await SEA.decrypt(auth.enc, work)
+        if (!dec) return next()
 
-          user.is = {
-            username: username,
-            pub: data.pub,
-            epub: data.epub,
-            priv: dec.priv,
-            epriv: dec.epriv,
+        user.is = {
+          username: username,
+          pub: data.pub,
+          epub: data.epub,
+          priv: dec.priv,
+          epriv: dec.epriv,
+        }
+        if (newPassword !== "") {
+          // Encrypt private key using new password and a new salt.
+          const salt = utils.text.random(64)
+          const work = await SEA.work(newPassword, salt)
+          const enc = await SEA.encrypt(dec, work)
+          // Only update the auth property with the new password encryption.
+          const update = {
+            auth: JSON.stringify({enc: enc, salt: salt}),
           }
-          if (newPassword !== "") {
-            // Encrypt private key using new password and a new salt.
-            const salt = utils.text.random(64)
-            const work = await SEA.work(newPassword, salt)
-            const enc = await SEA.encrypt(dec, work)
-            // Only update the auth property with the new password encryption.
-            const update = {
-              auth: JSON.stringify({enc: enc, salt: salt}),
+          const timestamp = Date.now()
+          const sig = await SEA.signTimestamp(timestamp, user.is)
+          const graph = utils.graph(pub, update, sig, data.pub, timestamp)
+          wire.put(graph, err => {
+            if (err) {
+              done(`error putting ${update} on ${pub}: ${err}`)
+            } else {
+              done(null)
             }
-            const timestamp = Date.now()
-            const sig = await SEA.signTimestamp(timestamp, user.is)
-            const graph = utils.graph(pub, update, sig, data.pub, timestamp)
-            wire.put(graph, err => {
-              if (err) {
-                done(`error putting ${update} on ${pub}: ${err}`)
-              } else {
-                done(null)
-              }
-            })
-            return
-          }
+          })
+          return
+        }
 
-          done(null)
-        },
-      )
+        done(null)
+      })
     }
 
     if (retries > 9) {
@@ -84,24 +81,21 @@ const User = (opt, wire) => {
     }
 
     const soul = "~@" + username
-    wire.get(
-      {"#": soul},
-      async msg => {
-        if (msg.err) {
-          done(`error getting ${soul}: ${msg.err}`)
-          return
-        }
+    wire.get({"#": soul}, async msg => {
+      if (msg.err) {
+        done(`error getting ${soul}: ${msg.err}`)
+        return
+      }
 
-        const data = msg.put && msg.put[soul]
-        if (!data) return retry(username)
+      const data = msg.put && msg.put[soul]
+      if (!data) return retry(username)
 
-        delete msg.put[soul]._
-        // Usernames aren't guaranteed to be unique, so store the list and then
-        // try each of them until one is successful with the given password.
-        pubs = Object.keys(data)
-        next()
-      },
-    )
+      delete msg.put[soul]._
+      // Usernames aren't guaranteed to be unique, so store the list and then
+      // try each of them until one is successful with the given password.
+      pubs = Object.keys(data)
+      next()
+    })
   }
 
   const user = {
@@ -129,58 +123,55 @@ const User = (opt, wire) => {
       creating = true
 
       const soul = "~@" + username
-      wire.get(
-        {"#": soul},
-        async msg => {
-          if (msg.err) {
-            creating = false
-            ack(`error getting ${soul}: ${msg.err}`)
-            return
-          }
+      wire.get({"#": soul}, async msg => {
+        if (msg.err) {
+          creating = false
+          ack(`error getting ${soul}: ${msg.err}`)
+          return
+        }
 
-          if (msg.put && msg.put[soul]) {
-            creating = false
-            ack("Username already exists")
-            return
-          }
+        if (msg.put && msg.put[soul]) {
+          creating = false
+          ack("Username already exists")
+          return
+        }
 
-          const salt = utils.text.random(64)
-          const work = await SEA.work(password, salt)
-          const pair = await SEA.pair()
-          const priv = {priv: pair.priv, epriv: pair.epriv}
-          const enc = await SEA.encrypt(priv, work)
-          const data = {
-            username: username,
-            pub: pair.pub,
-            epub: pair.epub,
-            auth: JSON.stringify({enc: enc, salt: salt}),
+        const salt = utils.text.random(64)
+        const work = await SEA.work(password, salt)
+        const pair = await SEA.pair()
+        const priv = {priv: pair.priv, epriv: pair.epriv}
+        const enc = await SEA.encrypt(priv, work)
+        const data = {
+          username: username,
+          pub: pair.pub,
+          epub: pair.epub,
+          auth: JSON.stringify({enc: enc, salt: salt}),
+        }
+
+        const pub = "~" + pair.pub
+        // Sign the timestamp for new account
+        const timestamp = Date.now()
+        const sig = await SEA.signTimestamp(timestamp, pair)
+        const graph = utils.graph(pub, data, sig, pair.pub, timestamp)
+        wire.put(graph, err => {
+          creating = false
+          if (err) {
+            ack(`error putting ${data} on ${pub}: ${err}`)
+            return
           }
 
-          const pub = "~" + pair.pub
-          // Sign the timestamp for new account
-          const timestamp = Date.now()
-          const sig = await SEA.signTimestamp(timestamp, pair)
-          const graph = utils.graph(pub, data, sig, pair.pub, timestamp)
-          wire.put(graph, err => {
-            creating = false
+          const rel = {[pub]: {"#": pub}}
+          wire.put(utils.graph(soul, rel), err => {
             if (err) {
-              ack(`error putting ${data} on ${pub}: ${err}`)
+              ack(`error putting ${rel} on ${soul}: ${err}`)
               return
             }
 
-            const rel = {[pub]: {"#": pub}}
-            wire.put(utils.graph(soul, rel), err => {
-              if (err) {
-                ack(`error putting ${rel} on ${soul}: ${err}`)
-                return
-              }
-
-              // Return null on success.
-              if (cb) cb(null)
-            })
+            // Return null on success.
+            if (cb) cb(null)
           })
-        },
-      )
+        })
+      })
     },
     auth: (username, password, cb) => {
       const ack = err => {