@mblaney/holster 1.0.27 → 1.0.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.27",
+  "version": "1.0.29",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/get.js

@@ -32,6 +32,20 @@ const Get = (lex, graph) => {
         signatures[key] = graph[soul]._["s"][key]
       }
     }
+
+    // Always include userPublicKey for verification, regardless of filter
+    if (userPublicKey && typeof graph[soul][userPublicKey] !== "undefined") {
+      node[userPublicKey] = graph[soul][userPublicKey]
+      node._[">"][userPublicKey] = graph[soul]._[">"][userPublicKey]
+      const pkState = graph[soul]._[">"][userPublicKey]
+      if (graph[soul]._["s"]) {
+        if (graph[soul]._["s"][pkState]) {
+          signatures[pkState] = graph[soul]._["s"][pkState]
+        } else if (graph[soul]._["s"][userPublicKey]) {
+          signatures[userPublicKey] = graph[soul]._["s"][userPublicKey]
+        }
+      }
+    }
   } else {
     for (const key of Object.keys(graph[soul])) {
       // Always include userPublicKey for verification, regardless of filter

package/src/ham.js

@@ -130,9 +130,6 @@ Ham.mix = async (change, graph, secure, listen) => {
     for (const key of Object.keys(node)) {
       if (key === "_") continue
 
-      // Skip metadata fields (including userSignature for old data)
-      if (key === utils.userSignature || key === utils.userPublicKey) continue
-
       // Properties without signatures in this update are left unchanged
       if (validProperties.has(soul) && !validProperties.get(soul).has(key)) {
         continue

package/src/holster.js

@@ -67,12 +67,15 @@ const Holster = opt => {
             for (const key of Object.keys(msg.put[soul])) {
               const id = utils.rel.is(msg.put[soul][key])
               if (id) {
-                // Due to message queuing, the related node might not be available yet
                 // Retry with delays if we get null
                 const attemptRead = async (retries = 0) => {
                   const data = await new Promise(res => {
                     const _ctxid = utils.text.random()
-                    allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
+                    const ctx = allctx.get(ctxid)
+                    allctx.set(_ctxid, {
+                      chain: [{item: null, soul: id}],
+                      user: ctx ? ctx.user : null,
+                    })
                     api(_ctxid).next(null, res, _opt)
                   })
                   if (data !== null || retries >= 5) {
@@ -157,11 +160,7 @@ const Holster = opt => {
           {"#": soul, ".": item},
           async msg => {
             if (msg.err) {
-              if (ctx.user || opt.secure) {
-                console.log(`error getting ${soul}: ${msg.err}`)
-              } else {
-                console.log(`error getting ${item} on ${soul}: ${msg.err}`)
-              }
+              console.log(`error getting ${item} on ${soul}: ${msg.err}`)
               if (cb) cb(null)
               return
             }
@@ -230,7 +229,7 @@ const Holster = opt => {
               if (cb) cb(null)
             }
           },
-          request._opt,
+          {...request._opt, secure: ctx.user || opt.secure},
         )
         // Callback has been passed to next soul lookup or called above, so
         // return false as the calling code should not continue.
@@ -382,66 +381,70 @@ const Holster = opt => {
           // When result is true data is a property to put on the current soul.
           // Need to check if item is a rel and also set the node to null. (This
           // applies for any update from a rel to a property, not just null.)
-          wire.get({"#": soul, ".": item}, async msg => {
-            if (msg.err) {
-              _ack(`error getting ${soul}: ${msg.err}`)
-              return
-            }
-
-            let node = msg.put && msg.put[soul]
-            const current = node && node[item]
-            const id = utils.rel.is(current)
-            if (!id) {
-              // Not a rel, can just put the data.
-              if (!node) node = {}
-              node[item] = data
-              const g = await graph(soul, node, ctx.user, _ack)
-              if (g === null) return
-
-              wire.put(g, _ack)
-              return
-            }
-
-            wire.get({"#": id}, async msg => {
+          wire.get(
+            {"#": soul, ".": item},
+            async msg => {
               if (msg.err) {
-                _ack(`error getting ${id}: ${msg.err}`)
+                _ack(`error getting ${soul}: ${msg.err}`)
                 return
               }
 
-              if (!msg.put || !msg.put[id]) {
-                _ack(`error ${id} not found`)
+              let node = msg.put && msg.put[soul]
+              const current = node && node[item]
+              const id = utils.rel.is(current)
+              if (!id) {
+                // Not a rel, can just put the data.
+                if (!node) node = {}
+                node[item] = data
+                const g = await graph(soul, node, ctx.user, _ack)
+                if (g === null) return
+
+                wire.put(g, _ack)
                 return
               }
 
-              // null each of the properties on the node before putting data.
-              for (const key of Object.keys(msg.put[id])) {
-                if (
-                  key === "_" ||
-                  key === utils.userPublicKey ||
-                  key === utils.userSignature
-                ) {
-                  continue
+              wire.get({"#": id}, async msg => {
+                if (msg.err) {
+                  _ack(`error getting ${id}: ${msg.err}`)
+                  return
                 }
 
-                const err = await new Promise(res => {
-                  const _ctxid = utils.text.random()
-                  const chain = [{item: key, soul: id}]
-                  allctx.set(_ctxid, {chain: chain, user: ctx.user})
-                  api(_ctxid).put(null, res)
-                })
-                if (err) {
-                  _ack(err)
+                if (!msg.put || !msg.put[id]) {
+                  _ack(`error ${id} not found`)
                   return
                 }
-              }
-              if (!node) node = {}
-              node[item] = data
-              const g = await graph(soul, node, ctx.user, _ack)
-              if (g === null) return
 
-              wire.put(g, _ack)
-            })
-          })
+                // null each of the properties on the node before putting data.
+                for (const key of Object.keys(msg.put[id])) {
+                  if (
+                    key === "_" ||
+                    key === utils.userPublicKey ||
+                    key === utils.userSignature
+                  ) {
+                    continue
+                  }
+
+                  const err = await new Promise(res => {
+                    const _ctxid = utils.text.random()
+                    const chain = [{item: key, soul: id}]
+                    allctx.set(_ctxid, {chain: chain, user: ctx.user})
+                    api(_ctxid).put(null, res)
+                  })
+                  if (err) {
+                    _ack(err)
+                    return
+                  }
+                }
+                if (!node) node = {}
+                node[item] = data
+                const g = await graph(soul, node, ctx.user, _ack)
+                if (g === null) return
+
+                wire.put(g, _ack)
+              })
+            },
+            {secure: ctx.user || opt.secure},
+          )
           return
         }
 
@@ -449,88 +452,97 @@ const Holster = opt => {
         // Need to check if a rel has already been added on the current node.
         // Use attemptRead to retry if data hasn't propagated yet.
         const attemptRead = (retries = 0) => {
-          wire.get({"#": soul, ".": item}, async msg => {
-            if (msg.err) {
-              _ack(`error getting ${soul}.${item}: ${msg.err}`)
-              return
-            }
+          wire.get(
+            {"#": soul, ".": item},
+            async msg => {
+              if (msg.err) {
+                _ack(`error getting ${soul}.${item}: ${msg.err}`)
+                return
+              }
 
-            let node = msg.put && msg.put[soul]
-            const current = node && node[item]
+              let node = msg.put && msg.put[soul]
+              const current = node && node[item]
 
-            // If we didn't find the item but still have retries, retry
-            if (!current && retries < 5) {
-              await new Promise(r => setTimeout(r, 50))
-              return attemptRead(retries + 1)
-            }
+              // If we didn't find the item but still have retries, retry
+              if (!current && retries < 5) {
+                await new Promise(r => setTimeout(r, 50))
+                return attemptRead(retries + 1)
+              }
 
-            const id = utils.rel.is(current)
-            if (!id) {
-              // The current rel doesn't exist, so add it first.
-              if (!node) node = {}
-              node[item] = utils.rel.ify(utils.text.random())
-              const g = await graph(soul, node, ctx.user, _ack)
-              if (g === null) return
+              const id = utils.rel.is(current)
+              if (!id) {
+                // The current rel doesn't exist, so add it first.
+                if (!node) node = {}
+                node[item] = utils.rel.ify(utils.text.random())
+                const g = await graph(soul, node, ctx.user, _ack)
+                if (g === null) return
 
-              wire.put(g, err => {
+                wire.put(g, err => {
+                  if (err) {
+                    _ack(`error putting ${item} on ${soul}: ${err}`)
+                  } else {
+                    const _ctxid = utils.text.random()
+                    const chain = [{item: item, soul: soul}]
+                    // Pass on the previous context's callback and user flag here.
+                    allctx.set(_ctxid, {
+                      chain: chain,
+                      user: ctx.user,
+                      cb: ctx.cb,
+                    })
+                    api(_ctxid).put(data)
+                  }
+                })
+                return
+              }
+
+              const update = []
+              for (const key of result) {
+                const err = await new Promise(res => {
+                  if (utils.obj.is(data[key]) && !utils.rel.is(data[key])) {
+                    // Use the current rel as the context for nested objects.
+                    const _ctxid = utils.text.random()
+                    const chain = [{item: key, soul: id}]
+                    allctx.set(_ctxid, {chain: chain, user: ctx.user})
+                    api(_ctxid).put(data[key], res)
+                  } else {
+                    // Group the rest of the updates for put below.
+                    update.push(key)
+                    res(null)
+                  }
+                })
                 if (err) {
-                  _ack(`error putting ${item} on ${soul}: ${err}`)
-                } else {
-                  const _ctxid = utils.text.random()
-                  const chain = [{item: item, soul: soul}]
-                  // Pass on the previous context's callback and user flag here.
-                  allctx.set(_ctxid, {chain: chain, user: ctx.user, cb: ctx.cb})
-                  api(_ctxid).put(data)
+                  _ack(err, ctxid)
+                  return
                 }
-              })
-              return
-            }
+              }
 
-            const update = []
-            for (const key of result) {
-              const err = await new Promise(res => {
-                if (utils.obj.is(data[key]) && !utils.rel.is(data[key])) {
-                  // Use the current rel as the context for nested objects.
-                  const _ctxid = utils.text.random()
-                  const chain = [{item: key, soul: id}]
-                  allctx.set(_ctxid, {chain: chain, user: ctx.user})
-                  api(_ctxid).put(data[key], res)
-                } else {
-                  // Group the rest of the updates for put below.
-                  update.push(key)
-                  res(null)
-                }
-              })
-              if (err) {
-                _ack(err, ctxid)
+              if (update.length === 0) {
+                _ack(null)
                 return
               }
-            }
 
-            if (update.length === 0) {
-              _ack(null)
-              return
-            }
+              // The nested objects created above will also have rels on the
+              // parent object, so fetch the node so that the rest of the
+              // updates can be added.
+              wire.get({"#": id}, async msg => {
+                if (msg.err) {
+                  _ack(`error getting ${id}: ${msg.err}`)
+                  return
+                }
 
-            // The nested objects created above will also have rels on the parent
-            // object, so fetch the node so the rest of the updates can be added.
-            wire.get({"#": id}, async msg => {
-              if (msg.err) {
-                _ack(`error getting ${id}: ${msg.err}`)
-                return
-              }
+                let node = msg.put && msg.put[id]
+                if (!node) node = {}
+                update.forEach(key => {
+                  node[key] = data[key]
+                })
+                const g = await graph(id, node, ctx.user, _ack)
+                if (g === null) return
 
-              let node = msg.put && msg.put[id]
-              if (!node) node = {}
-              update.forEach(key => {
-                node[key] = data[key]
+                wire.put(g, _ack)
               })
-              const g = await graph(id, node, ctx.user, _ack)
-              if (g === null) return
-
-              wire.put(g, _ack)
-            })
-          })
+            },
+            {secure: ctx.user || opt.secure},
+          )
         }
         attemptRead()
       },
@@ -556,8 +568,15 @@ const Holster = opt => {
         const {item, soul} = resolve({on: lex, _get: _get, _opt: _opt}, cb)
         if (!soul) return
 
+        // Get the context to check if it has user info
+        const ctx = allctx.get(ctxid)
+
         // Flag that this context is set from on and shouldn't be removed.
-        allctx.set(ctxid, {chain: [{item: item, soul: soul}], on: true})
+        allctx.set(ctxid, {
+          chain: [{item: item, soul: soul}],
+          on: true,
+          user: ctx ? ctx.user : null,
+        })
         // Map the user's callback because it can also be passed to off,
         // so need a reference to it to compare them.
         // Create a new context for each listener invocation to avoid mutation
@@ -574,7 +593,10 @@ const Holster = opt => {
                 const attemptRead = async (retries = 0) => {
                   const data = await new Promise(res => {
                     const _ctxid = utils.text.random()
-                    allctx.set(_ctxid, {chain: [{item: null, soul: id}]})
+                    allctx.set(_ctxid, {
+                      chain: [{item: null, soul: id}],
+                      user: ctx ? ctx.user : null,
+                    })
                     api(_ctxid).next(null, res, _opt)
                   })
                   if (data !== null || retries >= 5) {
@@ -591,7 +613,7 @@ const Holster = opt => {
                 cb(current !== undefined ? current : null)
               }
             },
-            _opt,
+            {..._opt, secure: ctx.user || opt.secure},
           )
         })
 
@@ -637,7 +659,7 @@ const Holster = opt => {
               map.get(cb)()
             }
           },
-          _opt,
+          {..._opt, secure: ctx.user || opt.secure},
         )
       },
       off: cb => {

package/src/sea.js

@@ -264,7 +264,6 @@ const SEA = {
 
       const propSig = propertySignatures[k]
       if (!propSig) {
-        console.log(`warning: property '${k}' missing signature`)
         continue
       }
 

package/src/store.js

@@ -211,7 +211,7 @@ const Store = opt => {
   const radisk = Radisk(opt)
 
   return {
-    get: (lex, cb) => {
+    get: (lex, cb, _opt) => {
       if (!lex || !utils.obj.is(lex)) {
         cb("lex required")
         return
@@ -221,8 +221,10 @@ const Store = opt => {
         return
       }
 
+      if (!_opt) _opt = {}
+
       var soul = lex["#"]
-      var key = typeof lex["."] === "string" ? lex["."] : ""
+      var key = !_opt.secure && typeof lex["."] === "string" ? lex["."] : ""
       var node
       var signatures = {}
       const each = (value, key) => {

package/src/wire.js

@@ -197,46 +197,6 @@ const Wire = opt => {
   const rateLimiter = createRateLimiter(isTestEnv)
   const connectionManager = createConnectionManager(opt.maxConnections || 1000)
 
-  // The check function is required because user data must provide a public key
-  // so that it can be verified. The public key might verify the provided
-  // signature but not actually match the user under which the data is being
-  // stored. To avoid this, the current data on a soul needs to be checked to
-  // make sure the stored public key matches the one provided with the update.
-  const check = async (data, send, cb) => {
-    for (const soul of Object.keys(data)) {
-      const msg = await new Promise(res => {
-        getWithCallback({"#": soul}, res, send)
-      })
-      if (msg.err) {
-        if (cb) cb(msg.err)
-        return false
-      }
-
-      const node = data[soul]
-      const key = utils.userPublicKey
-      // If there is no current node then the data is ok to write without
-      // matching public keys, as the provided soul also needs a rel on the
-      // parent node which then also requires checking. Otherwise publc keys
-      // need to match for existing data.
-      if (!msg.put || !msg.put[soul] || msg.put[soul][key] === node[key]) {
-        continue
-      }
-
-      // If a soul exists but does not have a public key, then one should not be
-      // added because the node is not user data. The above check fails in this
-      // case if a public key is provided. Note that this is only an error case
-      // if called via the API, which is when a callback is provided here.
-      // (The wire spec can fetch and put data on the wire without a signature
-      // or public key and this can be ignored.)
-      if (cb) {
-        cb(`error in wire check public key does not match for soul: ${soul}`)
-      }
-      return false
-    }
-
-    return true
-  }
-
   const get = (msg, send) => {
     const ack = Get(msg.get, graph)
     if (ack) {
@@ -248,16 +208,22 @@ const Wire = opt => {
         }),
       )
     } else {
-      store.get(msg.get, (err, ack) => {
-        send(
-          JSON.stringify({
-            "#": dup.track(utils.text.random(9)),
-            "@": msg["#"],
-            put: ack,
-            err: err,
-          }),
-        )
-      })
+      // Always pass secure to ensure public key is fetched. User data may be
+      // requested that requires the public key for verification.
+      store.get(
+        msg.get,
+        (err, ack) => {
+          send(
+            JSON.stringify({
+              "#": dup.track(utils.text.random(9)),
+              "@": msg["#"],
+              put: ack,
+              err: err,
+            }),
+          )
+        },
+        {secure: true},
+      )
     }
   }
 
@@ -275,8 +241,6 @@ const Wire = opt => {
       return
     }
 
-    if (!(await check(update.now, send))) return
-
     store.put(update.now, err => {
       send(
         JSON.stringify({
@@ -318,36 +282,40 @@ const Wire = opt => {
       return
     }
 
-    store.get(lex, (err, ack) => {
-      if (ack) {
-        // Also send request on the wire to check for updates.
-        const sendResult = send(request)
-        if (sendResult && sendResult.err) {
-          cb({err: sendResult.err})
+    store.get(
+      lex,
+      (err, ack) => {
+        if (ack) {
+          // Also send request on the wire to check for updates.
+          const sendResult = send(request)
+          if (sendResult && sendResult.err) {
+            cb({err: sendResult.err})
+            return
+          }
+          cb({put: ack, err: err})
           return
         }
-        cb({put: ack, err: err})
-        return
-      }
 
-      if (err) console.log(err)
+        if (err) console.log(err)
 
-      queue[track] = cb
+        queue[track] = cb
 
-      // Store timeout config to start after message is sent from queue
-      pendingTimeouts.set(track, {
-        lex: lex,
-        wait: _opt.wait || 100,
-      })
+        // Store timeout config to start after message is sent from queue
+        pendingTimeouts.set(track, {
+          lex: lex,
+          wait: _opt.wait || 100,
+        })
 
-      const sendResult = send(request)
-      if (sendResult && sendResult.err) {
-        cb({err: sendResult.err})
-        delete queue[track]
-        pendingTimeouts.delete(track)
-        return
-      }
-    })
+        const sendResult = send(request)
+        if (sendResult && sendResult.err) {
+          cb({err: sendResult.err})
+          delete queue[track]
+          pendingTimeouts.delete(track)
+          return
+        }
+      },
+      _opt,
+    )
   }
 
   const api = send => {
@@ -374,8 +342,6 @@ const Wire = opt => {
           return
         }
 
-        if (!(await check(update.now, send, cb))) return
-
         // Seed pendingReferences with any new references from API calls
         for (const [soul, node] of Object.entries(update.now)) {
           if (node && typeof node === "object") {