@mblaney/holster 1.0.26 → 1.0.28

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.26",
+  "version": "1.0.28",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/get.js

@@ -1,4 +1,4 @@
-import {match} from "./utils.js"
+import {match, userPublicKey} from "./utils.js"
 
 const Get = (lex, graph) => {
   if (!lex || typeof lex !== "object") {
@@ -34,7 +34,8 @@ const Get = (lex, graph) => {
     }
   } else {
     for (const key of Object.keys(graph[soul])) {
-      if (match(lex["."], key)) {
+      // Always include userPublicKey for verification, regardless of filter
+      if (key === userPublicKey || match(lex["."], key)) {
         node[key] = graph[soul][key]
         node._[">"][key] = graph[soul]._[">"][key]
         const state = graph[soul]._[">"][key]

package/src/holster.js

@@ -67,12 +67,15 @@ const Holster = opt => {
             for (const key of Object.keys(msg.put[soul])) {
               const id = utils.rel.is(msg.put[soul][key])
               if (id) {
-                // Due to message queuing, the related node might not be available yet
                 // Retry with delays if we get null
                 const attemptRead = async (retries = 0) => {
                   const data = await new Promise(res => {
                     const _ctxid = utils.text.random()
-                    allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
+                    const ctx = allctx.get(ctxid)
+                    allctx.set(_ctxid, {
+                      chain: [{item: null, soul: id}],
+                      user: ctx ? ctx.user : null,
+                    })
                     api(_ctxid).next(null, res, _opt)
                   })
                   if (data !== null || retries >= 5) {
@@ -157,11 +160,7 @@ const Holster = opt => {
           {"#": soul, ".": item},
           async msg => {
             if (msg.err) {
-              if (ctx.user || opt.secure) {
-                console.log(`error getting ${soul}: ${msg.err}`)
-              } else {
-                console.log(`error getting ${item} on ${soul}: ${msg.err}`)
-              }
+              console.log(`error getting ${item} on ${soul}: ${msg.err}`)
               if (cb) cb(null)
               return
             }
@@ -230,7 +229,7 @@ const Holster = opt => {
               if (cb) cb(null)
             }
           },
-          request._opt,
+          {...request._opt, secure: ctx.user || opt.secure},
         )
         // Callback has been passed to next soul lookup or called above, so
         // return false as the calling code should not continue.
@@ -382,66 +381,70 @@ const Holster = opt => {
           // When result is true data is a property to put on the current soul.
           // Need to check if item is a rel and also set the node to null. (This
           // applies for any update from a rel to a property, not just null.)
-          wire.get({"#": soul, ".": item}, async msg => {
-            if (msg.err) {
-              _ack(`error getting ${soul}: ${msg.err}`)
-              return
-            }
-
-            let node = msg.put && msg.put[soul]
-            const current = node && node[item]
-            const id = utils.rel.is(current)
-            if (!id) {
-              // Not a rel, can just put the data.
-              if (!node) node = {}
-              node[item] = data
-              const g = await graph(soul, node, ctx.user, _ack)
-              if (g === null) return
-
-              wire.put(g, _ack)
-              return
-            }
-
-            wire.get({"#": id}, async msg => {
+          wire.get(
+            {"#": soul, ".": item},
+            async msg => {
               if (msg.err) {
-                _ack(`error getting ${id}: ${msg.err}`)
+                _ack(`error getting ${soul}: ${msg.err}`)
                 return
               }
 
-              if (!msg.put || !msg.put[id]) {
-                _ack(`error ${id} not found`)
+              let node = msg.put && msg.put[soul]
+              const current = node && node[item]
+              const id = utils.rel.is(current)
+              if (!id) {
+                // Not a rel, can just put the data.
+                if (!node) node = {}
+                node[item] = data
+                const g = await graph(soul, node, ctx.user, _ack)
+                if (g === null) return
+
+                wire.put(g, _ack)
                 return
               }
 
-              // null each of the properties on the node before putting data.
-              for (const key of Object.keys(msg.put[id])) {
-                if (
-                  key === "_" ||
-                  key === utils.userPublicKey ||
-                  key === utils.userSignature
-                ) {
-                  continue
+              wire.get({"#": id}, async msg => {
+                if (msg.err) {
+                  _ack(`error getting ${id}: ${msg.err}`)
+                  return
                 }
 
-                const err = await new Promise(res => {
-                  const _ctxid = utils.text.random()
-                  const chain = [{item: key, soul: id}]
-                  allctx.set(_ctxid, {chain: chain, user: ctx.user})
-                  api(_ctxid).put(null, res)
-                })
-                if (err) {
-                  _ack(err)
+                if (!msg.put || !msg.put[id]) {
+                  _ack(`error ${id} not found`)
                   return
                 }
-              }
-              if (!node) node = {}
-              node[item] = data
-              const g = await graph(soul, node, ctx.user, _ack)
-              if (g === null) return
 
-              wire.put(g, _ack)
-            })
-          })
+                // null each of the properties on the node before putting data.
+                for (const key of Object.keys(msg.put[id])) {
+                  if (
+                    key === "_" ||
+                    key === utils.userPublicKey ||
+                    key === utils.userSignature
+                  ) {
+                    continue
+                  }
+
+                  const err = await new Promise(res => {
+                    const _ctxid = utils.text.random()
+                    const chain = [{item: key, soul: id}]
+                    allctx.set(_ctxid, {chain: chain, user: ctx.user})
+                    api(_ctxid).put(null, res)
+                  })
+                  if (err) {
+                    _ack(err)
+                    return
+                  }
+                }
+                if (!node) node = {}
+                node[item] = data
+                const g = await graph(soul, node, ctx.user, _ack)
+                if (g === null) return
+
+                wire.put(g, _ack)
+              })
+            },
+            {secure: ctx.user || opt.secure},
+          )
           return
         }
 
@@ -449,88 +452,97 @@ const Holster = opt => {
         // Need to check if a rel has already been added on the current node.
         // Use attemptRead to retry if data hasn't propagated yet.
         const attemptRead = (retries = 0) => {
-          wire.get({"#": soul, ".": item}, async msg => {
-            if (msg.err) {
-              _ack(`error getting ${soul}.${item}: ${msg.err}`)
-              return
-            }
+          wire.get(
+            {"#": soul, ".": item},
+            async msg => {
+              if (msg.err) {
+                _ack(`error getting ${soul}.${item}: ${msg.err}`)
+                return
+              }
 
-            let node = msg.put && msg.put[soul]
-            const current = node && node[item]
+              let node = msg.put && msg.put[soul]
+              const current = node && node[item]
 
-            // If we didn't find the item but still have retries, retry
-            if (!current && retries < 5) {
-              await new Promise(r => setTimeout(r, 50))
-              return attemptRead(retries + 1)
-            }
+              // If we didn't find the item but still have retries, retry
+              if (!current && retries < 5) {
+                await new Promise(r => setTimeout(r, 50))
+                return attemptRead(retries + 1)
+              }
 
-            const id = utils.rel.is(current)
-            if (!id) {
-              // The current rel doesn't exist, so add it first.
-              if (!node) node = {}
-              node[item] = utils.rel.ify(utils.text.random())
-              const g = await graph(soul, node, ctx.user, _ack)
-              if (g === null) return
+              const id = utils.rel.is(current)
+              if (!id) {
+                // The current rel doesn't exist, so add it first.
+                if (!node) node = {}
+                node[item] = utils.rel.ify(utils.text.random())
+                const g = await graph(soul, node, ctx.user, _ack)
+                if (g === null) return
 
-              wire.put(g, err => {
+                wire.put(g, err => {
+                  if (err) {
+                    _ack(`error putting ${item} on ${soul}: ${err}`)
+                  } else {
+                    const _ctxid = utils.text.random()
+                    const chain = [{item: item, soul: soul}]
+                    // Pass on the previous context's callback and user flag here.
+                    allctx.set(_ctxid, {
+                      chain: chain,
+                      user: ctx.user,
+                      cb: ctx.cb,
+                    })
+                    api(_ctxid).put(data)
+                  }
+                })
+                return
+              }
+
+              const update = []
+              for (const key of result) {
+                const err = await new Promise(res => {
+                  if (utils.obj.is(data[key]) && !utils.rel.is(data[key])) {
+                    // Use the current rel as the context for nested objects.
+                    const _ctxid = utils.text.random()
+                    const chain = [{item: key, soul: id}]
+                    allctx.set(_ctxid, {chain: chain, user: ctx.user})
+                    api(_ctxid).put(data[key], res)
+                  } else {
+                    // Group the rest of the updates for put below.
+                    update.push(key)
+                    res(null)
+                  }
+                })
                 if (err) {
-                  _ack(`error putting ${item} on ${soul}: ${err}`)
-                } else {
-                  const _ctxid = utils.text.random()
-                  const chain = [{item: item, soul: soul}]
-                  // Pass on the previous context's callback and user flag here.
-                  allctx.set(_ctxid, {chain: chain, user: ctx.user, cb: ctx.cb})
-                  api(_ctxid).put(data)
+                  _ack(err, ctxid)
+                  return
                 }
-              })
-              return
-            }
+              }
 
-            const update = []
-            for (const key of result) {
-              const err = await new Promise(res => {
-                if (utils.obj.is(data[key]) && !utils.rel.is(data[key])) {
-                  // Use the current rel as the context for nested objects.
-                  const _ctxid = utils.text.random()
-                  const chain = [{item: key, soul: id}]
-                  allctx.set(_ctxid, {chain: chain, user: ctx.user})
-                  api(_ctxid).put(data[key], res)
-                } else {
-                  // Group the rest of the updates for put below.
-                  update.push(key)
-                  res(null)
-                }
-              })
-              if (err) {
-                _ack(err, ctxid)
+              if (update.length === 0) {
+                _ack(null)
                 return
               }
-            }
 
-            if (update.length === 0) {
-              _ack(null)
-              return
-            }
+              // The nested objects created above will also have rels on the
+              // parent object, so fetch the node so that the rest of the updates
+              // can be added.
+              wire.get({"#": id}, async msg => {
+                if (msg.err) {
+                  _ack(`error getting ${id}: ${msg.err}`)
+                  return
+                }
 
-            // The nested objects created above will also have rels on the parent
-            // object, so fetch the node so the rest of the updates can be added.
-            wire.get({"#": id}, async msg => {
-              if (msg.err) {
-                _ack(`error getting ${id}: ${msg.err}`)
-                return
-              }
+                let node = msg.put && msg.put[id]
+                if (!node) node = {}
+                update.forEach(key => {
+                  node[key] = data[key]
+                })
+                const g = await graph(id, node, ctx.user, _ack)
+                if (g === null) return
 
-              let node = msg.put && msg.put[id]
-              if (!node) node = {}
-              update.forEach(key => {
-                node[key] = data[key]
+                wire.put(g, _ack)
               })
-              const g = await graph(id, node, ctx.user, _ack)
-              if (g === null) return
-
-              wire.put(g, _ack)
-            })
-          })
+            },
+            {secure: ctx.user || opt.secure},
+          )
         }
         attemptRead()
       },
@@ -556,8 +568,15 @@ const Holster = opt => {
         const {item, soul} = resolve({on: lex, _get: _get, _opt: _opt}, cb)
         if (!soul) return
 
+        // Get the context to check if it has user info
+        const ctx = allctx.get(ctxid)
+
         // Flag that this context is set from on and shouldn't be removed.
-        allctx.set(ctxid, {chain: [{item: item, soul: soul}], on: true})
+        allctx.set(ctxid, {
+          chain: [{item: item, soul: soul}],
+          on: true,
+          user: ctx ? ctx.user : null,
+        })
         // Map the user's callback because it can also be passed to off,
         // so need a reference to it to compare them.
         // Create a new context for each listener invocation to avoid mutation
@@ -574,7 +593,10 @@ const Holster = opt => {
                 const attemptRead = async (retries = 0) => {
                   const data = await new Promise(res => {
                     const _ctxid = utils.text.random()
-                    allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
+                    allctx.set(_ctxid, {
+                      chain: [{item: null, soul: id}],
+                      user: ctx ? ctx.user : null,
+                    })
                     api(_ctxid).next(null, res, _opt)
                   })
                   if (data !== null || retries >= 5) {
@@ -591,7 +613,7 @@ const Holster = opt => {
                 cb(current !== undefined ? current : null)
               }
             },
-            _opt,
+            {..._opt, secure: ctx.user || opt.secure},
           )
         })
 
@@ -637,7 +659,7 @@ const Holster = opt => {
               map.get(cb)()
             }
           },
-          _opt,
+          {..._opt, secure: ctx.user || opt.secure},
         )
       },
       off: cb => {

package/src/radisk.js

@@ -34,6 +34,7 @@ const Radisk = opt => {
   if (!opt.write) opt.write = 1 // Wait time before write in milliseconds
   if (!opt.size) opt.size = 1024 * 1024 // File size on disk, default 1MB
   if (!opt.memoryLimit) opt.memoryLimit = 500 // Memory limit in MB
+  if (!opt.readTimeout) opt.readTimeout = 1000 // Read timeout in milliseconds
   if (typeof opt.cache === "undefined") opt.cache = true
 
   if (!opt.store) {
@@ -126,20 +127,34 @@ const Radisk = opt => {
       // Check if there's already a pending read for this key
       if (pendingReads.has(key)) {
         // Add callback to existing pending read
-        pendingReads.get(key).push(cb)
+        pendingReads.get(key).callbacks.push(cb)
         return
       }
 
-      // Start new read and create callback queue
-      pendingReads.set(key, [cb])
+      // Start new read and create callback queue with fired flag
+      const pending = {callbacks: [cb], fired: false, timeoutId: null}
+      pendingReads.set(key, pending)
+
+      // Set a timeout to fire callbacks if read takes too long
+      pending.timeoutId = setTimeout(() => {
+        if (!pending.fired) {
+          pending.fired = true
+          pendingReads.delete(key)
+          const err = new Error("radisk read timeout")
+          pending.callbacks.forEach(callback => callback(err, undefined))
+        }
+      }, opt.readTimeout)
 
       const readStart = Date.now()
       return radisk.read(key, (err, result) => {
+        clearTimeout(pending.timeoutId)
         perfLog("read", readStart, key)
-        // Execute all pending callbacks for this key
-        const callbacks = pendingReads.get(key) || []
-        pendingReads.delete(key)
-        callbacks.forEach(callback => callback(err, result))
+        // Execute all pending callbacks for this key if timeout hasn't fired
+        if (!pending.fired) {
+          pending.fired = true
+          pendingReads.delete(key)
+          pending.callbacks.forEach(callback => callback(err, result))
+        }
       })
     }
 

package/src/sea.js

@@ -264,7 +264,6 @@ const SEA = {
 
       const propSig = propertySignatures[k]
       if (!propSig) {
-        console.log(`warning: property '${k}' missing signature`)
         continue
       }
 

package/src/store.js

@@ -211,7 +211,7 @@ const Store = opt => {
   const radisk = Radisk(opt)
 
   return {
-    get: (lex, cb) => {
+    get: (lex, cb, _opt) => {
       if (!lex || !utils.obj.is(lex)) {
         cb("lex required")
         return
@@ -221,12 +221,15 @@ const Store = opt => {
         return
       }
 
+      if (!_opt) _opt = {}
+
       var soul = lex["#"]
-      var key = typeof lex["."] === "string" ? lex["."] : ""
+      var key = !_opt.secure && typeof lex["."] === "string" ? lex["."] : ""
       var node
       var signatures = {}
       const each = (value, key) => {
-        if (!utils.match(lex["."], key)) return
+        // Always include userPublicKey for verification, regardless of filter
+        if (key !== utils.userPublicKey && !utils.match(lex["."], key)) return
 
         if (!node) node = {_: {"#": soul, ">": {}}}
         node[key] = value[0]

package/src/wire.js

@@ -318,36 +318,40 @@ const Wire = opt => {
       return
     }
 
-    store.get(lex, (err, ack) => {
-      if (ack) {
-        // Also send request on the wire to check for updates.
-        const sendResult = send(request)
-        if (sendResult && sendResult.err) {
-          cb({err: sendResult.err})
+    store.get(
+      lex,
+      (err, ack) => {
+        if (ack) {
+          // Also send request on the wire to check for updates.
+          const sendResult = send(request)
+          if (sendResult && sendResult.err) {
+            cb({err: sendResult.err})
+            return
+          }
+          cb({put: ack, err: err})
           return
         }
-        cb({put: ack, err: err})
-        return
-      }
 
-      if (err) console.log(err)
+        if (err) console.log(err)
 
-      queue[track] = cb
+        queue[track] = cb
 
-      // Store timeout config to start after message is sent from queue
-      pendingTimeouts.set(track, {
-        lex: lex,
-        wait: _opt.wait || 100,
-      })
+        // Store timeout config to start after message is sent from queue
+        pendingTimeouts.set(track, {
+          lex: lex,
+          wait: _opt.wait || 100,
+        })
 
-      const sendResult = send(request)
-      if (sendResult && sendResult.err) {
-        cb({err: sendResult.err})
-        delete queue[track]
-        pendingTimeouts.delete(track)
-        return
-      }
-    })
+        const sendResult = send(request)
+        if (sendResult && sendResult.err) {
+          cb({err: sendResult.err})
+          delete queue[track]
+          pendingTimeouts.delete(track)
+          return
+        }
+      },
+      _opt,
+    )
   }
 
   const api = send => {