@mblaney/holster 1.0.21 → 1.0.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.21",
+  "version": "1.0.23",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/get.js

@@ -16,6 +16,7 @@ const Get = (lex, graph) => {
   if (!graph[soul]) return
 
   const node = {_: {"#": soul, ">": {}}}
+  let signatures = {}
 
   if (typeof lex["."] === "string") {
     const key = lex["."]
@@ -23,14 +24,23 @@ const Get = (lex, graph) => {
 
     node[key] = graph[soul][key]
     node._[">"][key] = graph[soul]._[">"][key]
+    if (graph[soul]._["s"] && graph[soul]._["s"][key]) {
+      signatures[key] = graph[soul]._["s"][key]
+    }
   } else {
     for (const key of Object.keys(graph[soul])) {
       if (match(lex["."], key)) {
         node[key] = graph[soul][key]
         node._[">"][key] = graph[soul]._[">"][key]
+        if (graph[soul]._["s"] && graph[soul]._["s"][key]) {
+          signatures[key] = graph[soul]._["s"][key]
+        }
       }
     }
   }
+  if (Object.keys(signatures).length > 0) {
+    node._["s"] = signatures
+  }
   return {[soul]: node}
 }
 

package/src/ham.js

@@ -43,6 +43,7 @@ Ham.mix = async (change, graph, secure, listen) => {
   const machine = Date.now()
   const now = {}
   const defer = {}
+  const validProperties = new Map() // Track valid properties per soul
   let wait = 0
 
   for (const soul of Object.keys(change)) {
@@ -54,10 +55,9 @@ Ham.mix = async (change, graph, secure, listen) => {
 
     if (!node || !node._) continue
 
-    const sig = node[utils.userSignature]
     const pub = node[utils.userPublicKey]
-    // If a signature and public key are provided then always verify.
-    if (sig && pub) verify = true
+    // If per-property signatures and public key are provided then always verify.
+    if (node._["s"] && pub) verify = true
 
     // Special case if soul starts with "~". Node must be system data ie,
     // ~@alias or ~publickey. For aliases, key and value must be a self
@@ -77,17 +77,35 @@ Ham.mix = async (change, graph, secure, listen) => {
       }
     }
     if (verify) {
-      // Partial nodes can be read from disk, which are ignored. This could be
-      // a missing signature or public key, or missing properties on the node
-      // which means verify won't work either.
-      if (!sig || !pub || !(await SEA.verify({m: node, s: sig}, {pub: pub}))) {
+      // Per-property signatures stored in node._["s"]
+      if (!pub || !node._) {
         continue
       }
+
+      // Verify properties individually, returns array of valid property names
+      const validProps = await SEA.verifyProperties(node, pub)
+
+      // If no properties verified, skip this update
+      if (validProps.length === 0) {
+        continue
+      }
+
+      // Track valid properties for this soul
+      validProperties.set(soul, new Set(validProps))
     }
 
     for (const key of Object.keys(node)) {
       if (key === "_") continue
 
+      // Skip metadata fields (including userSignature for old data)
+      if (key === utils.userSignature || key === utils.userPublicKey) continue
+
+      // If this soul had verification, only process properties that have valid signatures
+      // Properties without signatures in this update are left unchanged
+      if (validProperties.has(soul) && !validProperties.get(soul).has(key)) {
+        continue
+      }
+
       const value = node[key]
       const state = node._ && node._[">"] ? node._[">"][key] : 0
       const currentValue = (graph[soul] || {})[key]
@@ -107,21 +125,27 @@ Ham.mix = async (change, graph, secure, listen) => {
         // Wait the shortest difference before trying the updates again.
         if (wait === 0 || skew < wait) wait = nodeWait = skew
         if (!defer[soul]) {
-          defer[soul] = {_: {"#": soul, ">": {}}}
+          defer[soul] = {_: {"#": soul, ">": {}, s: {}}}
         }
         defer[soul][key] = value
         defer[soul]._[">"][key] = state
+        if (node._["s"] && node._["s"][key]) {
+          defer[soul]._["s"][key] = node._["s"][key]
+        }
       } else {
         const result = Ham(state, currentState, value, currentValue)
         if (result.incoming) {
           if (!now[soul]) {
-            now[soul] = {_: {"#": soul, ">": {}}}
+            now[soul] = {_: {"#": soul, ">": {}, s: {}}}
           }
           if (!graph[soul]) {
-            graph[soul] = {_: {"#": soul, ">": {}}}
+            graph[soul] = {_: {"#": soul, ">": {}, s: {}}}
           }
           graph[soul][key] = now[soul][key] = value
           graph[soul]._[">"][key] = now[soul]._[">"][key] = state
+          if (node._["s"] && node._["s"][key]) {
+            graph[soul]._["s"][key] = now[soul]._["s"][key] = node._["s"][key]
+          }
           // Call event listeners for update on key, mix is called before
           // put has finished so wait for what could be multiple nested
           // updates on a node.

package/src/holster.js

@@ -97,8 +97,9 @@ const Holster = opt => {
 
     const graph = async (soul, data, userctx, cb) => {
       if (userctx) {
-        const signed = await SEA.sign(data, userctx)
-        return utils.graph(soul, signed.m, signed.s, userctx.pub)
+        // Sign each property individually
+        const propertySignatures = await SEA.signProperties(data, userctx)
+        return utils.graph(soul, data, propertySignatures, userctx.pub)
       }
 
       if (opt.secure) {
@@ -151,12 +152,8 @@ const Holster = opt => {
         // Found a soul that needs resolving, need the previous context
         // (ie the parent node) to find a soul relation for it.
         const {item, soul} = ctx.chain[i - 1]
-        // If this is a user context or using secure mode then need to get the
-        // whole node for verification.
-        const lex =
-          ctx.user || opt.secure ? {"#": soul} : {"#": soul, ".": item}
         wire.get(
-          lex,
+          {"#": soul, ".": item},
           async msg => {
             if (msg.err) {
               if (ctx.user || opt.secure) {
@@ -384,10 +381,7 @@ const Holster = opt => {
           // When result is true data is a property to put on the current soul.
           // Need to check if item is a rel and also set the node to null. (This
           // applies for any update from a rel to a property, not just null.)
-          // If using secure mode need to get the whole node for verification.
-          const lex =
-            ctx.user || opt.secure ? {"#": soul} : {"#": soul, ".": item}
-          wire.get(lex, async msg => {
+          wire.get({"#": soul, ".": item}, async msg => {
             if (msg.err) {
               _ack(`error getting ${soul}: ${msg.err}`)
               return
@@ -452,10 +446,7 @@ const Holster = opt => {
 
         // Otherwise put the data using the keys returned in result.
         // Need to check if a rel has already been added on the current node.
-        // If using secure mode need to get the whole node for verification.
-        const lex =
-          ctx.user || opt.secure ? {"#": soul} : {"#": soul, ".": item}
-        wire.get(lex, async msg => {
+        wire.get({"#": soul, ".": item}, async msg => {
           if (msg.err) {
             _ack(`error getting ${soul}.${item}: ${msg.err}`)
             return

package/src/radisk.js

@@ -461,10 +461,14 @@ const Radisk = opt => {
 
 Radisk.encode = data => {
   // A key should be passed in as a string to encode, a value can optionally be
-  // an array of 2 items to include the value's state, as is done by store.js.
+  // an array of 2-3 items: [value, state] or [value, state, signature].
   let state = ""
-  if (data instanceof Array && data.length === 2) {
+  let sig = ""
+  if (data instanceof Array && (data.length === 2 || data.length === 3)) {
     state = etx + data[1]
+    if (data.length === 3 && data[2]) {
+      sig = etx + data[2]
+    }
     data = data[0]
   }
 
@@ -475,19 +479,19 @@ Radisk.encode = data => {
     while ((current = data[i++])) {
       if (current === unit) text += unit
     }
-    return text + '"' + data + state + unit
+    return text + '"' + data + state + sig + unit
   }
 
   const rel = utils.rel.is(data)
-  if (rel) return unit + "#" + rel + state + unit
+  if (rel) return unit + "#" + rel + state + sig + unit
 
-  if (utils.num.is(data)) return unit + "+" + (data || 0) + state + unit
+  if (utils.num.is(data)) return unit + "+" + (data || 0) + state + sig + unit
 
-  if (data === true) return unit + "+" + state + unit
+  if (data === true) return unit + "+" + state + sig + unit
 
-  if (data === false) return unit + "-" + state + unit
+  if (data === false) return unit + "-" + state + sig + unit
 
-  if (data === null) return unit + " " + state + unit
+  if (data === null) return unit + " " + state + sig + unit
 }
 
 Radisk.decode = (data, obj) => {
@@ -522,7 +526,11 @@ Radisk.decode = (data, obj) => {
 
   if (obj) obj.i = i + 1
 
-  let [value, state] = text.split(etx)
+  let parts = text.split(etx)
+  let value = parts[0]
+  let state = parts[1]
+  let sig = parts[2]
+
   if (!state) {
     if (previous === '"') return text
 
@@ -539,20 +547,36 @@ Radisk.decode = (data, obj) => {
     if (previous === " ") return null
   } else {
     state = parseFloat(state)
-    // If state was found then return an array.
-    if (previous === '"') return [value, state]
+    // If state was found then return an array with [value, state] or [value, state, sig]
+    if (sig) {
+      if (previous === '"') return [value, state, sig]
 
-    if (previous === "#") return [utils.rel.ify(value), state]
+      if (previous === "#") return [utils.rel.ify(value), state, sig]
 
-    if (previous === "+") {
-      if (value.length === 0) return [true, state]
+      if (previous === "+") {
+        if (value.length === 0) return [true, state, sig]
 
-      return [parseFloat(value), state]
-    }
+        return [parseFloat(value), state, sig]
+      }
+
+      if (previous === "-") return [false, state, sig]
 
-    if (previous === "-") return [false, state]
+      if (previous === " ") return [null, state, sig]
+    } else {
+      if (previous === '"') return [value, state]
 
-    if (previous === " ") return [null, state]
+      if (previous === "#") return [utils.rel.ify(value), state]
+
+      if (previous === "+") {
+        if (value.length === 0) return [true, state]
+
+        return [parseFloat(value), state]
+      }
+
+      if (previous === "-") return [false, state]
+
+      if (previous === " ") return [null, state]
+    }
   }
 }
 

package/src/sea.js

@@ -118,7 +118,7 @@ const SEA = {
       // Allow data to be passed in with graph meta data,
       // (which should not be part of signature, so not verified).
       for (const k of Object.keys(signed.m).sort()) {
-        if (k !== "_" && k != userPublicKey && k != userSignature) {
+        if (k !== "_" && k !== userPublicKey && k !== userSignature) {
           msg[k] = signed.m[k]
         }
       }
@@ -135,6 +135,59 @@ const SEA = {
     if (cb) cb(null)
     return null
   },
+  // Verify individual property signatures stored in node._["s"]
+  // Returns array of valid property names
+  verifyProperties: async (node, pub, cb) => {
+    if (!pub || !node) {
+      if (cb) cb([])
+      return []
+    }
+
+    const key = await utils.subtle.importKey(
+      "jwk",
+      utils.jwk(pub),
+      {name: "ECDSA", namedCurve: "P-256"},
+      false,
+      ["verify"],
+    )
+
+    const alg = {name: "ECDSA", hash: {name: "SHA-256"}}
+    const valid = []
+    const propertySignatures = (node._ && node._["s"]) || {}
+
+    // Verify each property individually
+    for (const k of Object.keys(node).sort()) {
+      if (k === "_" || k == userPublicKey) continue
+
+      const propSig = propertySignatures[k]
+      if (!propSig) {
+        console.log(`warning: property '${k}' missing signature`)
+        continue
+      }
+
+      try {
+        const hash = await utils.sha256(node[k])
+        const sig = new Uint8Array(SafeBuffer.from(propSig, "base64"))
+
+        const isValid = await utils.subtle.verify(
+          alg,
+          key,
+          sig,
+          new Uint8Array(hash),
+        )
+        if (isValid) {
+          valid.push(k)
+        } else {
+          console.log(`warning: property '${k}' signature verification failed`)
+        }
+      } catch (err) {
+        console.log(`warning: property '${k}' error verifying: ${err.message}`)
+      }
+    }
+
+    if (cb) cb(valid)
+    return valid
+  },
   sign: async (data, pair, cb) => {
     if (!pair || !pair.pub || !pair.priv) {
       if (cb) cb(null)
@@ -147,7 +200,7 @@ const SEA = {
     } else {
       const check = utils.parse(data)
       for (const k of Object.keys(check).sort()) {
-        if (k !== "_" && k != userPublicKey && k != userSignature) {
+        if (k !== "_" && k !== userPublicKey && k !== userSignature) {
           msg[k] = check[k]
         }
       }
@@ -172,6 +225,37 @@ const SEA = {
     if (cb) cb(signed)
     return signed
   },
+  // Sign individual properties for per-property verification
+  signProperties: async (data, pair, cb) => {
+    if (!pair || !pair.pub || !pair.priv) {
+      if (cb) cb(null)
+      return null
+    }
+
+    const check = utils.parse(data)
+    const propertySignatures = {}
+    const jwk = utils.jwk(pair.pub, pair.priv)
+    const alg = {name: "ECDSA", namedCurve: "P-256"}
+    const key = await utils.subtle.importKey("jwk", jwk, alg, false, ["sign"])
+
+    // Sign each property individually
+    for (const k of Object.keys(check).sort()) {
+      if (k !== "_" && k != userPublicKey && k != userSignature) {
+        const hash = await utils.sha256(check[k])
+        const sig = await utils.subtle.sign(
+          {name: "ECDSA", hash: {name: "SHA-256"}},
+          key,
+          new Uint8Array(hash),
+        )
+        propertySignatures[k] = SafeBuffer.from(sig, "binary").toString(
+          "base64",
+        )
+      }
+    }
+
+    if (cb) cb(propertySignatures)
+    return propertySignatures
+  },
   work: async (data, salt, cb) => {
     if (typeof salt === "function") {
       cb = salt

package/src/store.js

@@ -224,12 +224,17 @@ const Store = opt => {
       var soul = lex["#"]
       var key = typeof lex["."] === "string" ? lex["."] : ""
       var node
+      var signatures = {}
       const each = (value, key) => {
         if (!utils.match(lex["."], key)) return
 
         if (!node) node = {_: {"#": soul, ">": {}}}
         node[key] = value[0]
         node._[">"][key] = value[1]
+        // If signature is present, store it in _["s"]
+        if (value.length === 3 && value[2]) {
+          signatures[key] = value[2]
+        }
       }
 
       radisk(soul + enq + key, (err, value) => {
@@ -242,6 +247,10 @@ const Store = opt => {
           each(value, key)
           graph = {[soul]: node}
         }
+        // Only add _["s"] if we found any signatures
+        if (graph && graph[soul] && Object.keys(signatures).length > 0) {
+          graph[soul]._["s"] = signatures
+        }
         cb(err, graph)
       })
     },
@@ -277,7 +286,10 @@ const Store = opt => {
           count++
           let value = node[key]
           let state = node._[">"][key]
-          radisk(soul + enq + key, [value, state], ack)
+          let sig = node._["s"] && node._["s"][key]
+          // Only pass signature if it exists, otherwise just [value, state]
+          let data = sig ? [value, state, sig] : [value, state]
+          radisk(soul + enq + key, data, ack)
         })
       })
     },

package/src/user.js

@@ -1,6 +1,7 @@
 import * as utils from "./utils.js"
 import Wire from "./wire.js"
 import SEA from "./sea.js"
+import {userPublicKey, userSignature} from "./utils.js"
 
 const User = (opt, wire) => {
   if (!wire) wire = Wire(opt)
@@ -55,15 +56,12 @@ const User = (opt, wire) => {
             const salt = utils.text.random(64)
             const work = await SEA.work(newPassword, salt)
             const enc = await SEA.encrypt(dec, work)
-            // Put all fields so that verification applies to the whole node.
+            // Only update the auth property with the new password encryption.
             const update = {
-              username: username,
-              pub: data.pub,
-              epub: data.epub,
               auth: JSON.stringify({enc: enc, salt: salt}),
             }
-            const signed = await SEA.sign(update, user.is)
-            const graph = utils.graph(pub, signed.m, signed.s, data.pub)
+            const propertySignatures = await SEA.signProperties(update, user.is)
+            const graph = utils.graph(pub, update, propertySignatures, data.pub)
             wire.put(graph, err => {
               if (err) {
                 done(`error putting ${update} on ${pub}: ${err}`)
@@ -74,9 +72,51 @@ const User = (opt, wire) => {
             return
           }
 
+          // Migrate old per-node signatures to per-property signatures
+          // Check if data has per-property signatures
+          if (
+            !data._ ||
+            !data._["s"] ||
+            Object.keys(data._["s"]).length === 0
+          ) {
+            // Data has old per-node signatures, migrate to per-property
+            // Copy all properties from existing data, excluding metadata
+            const migrationData = {}
+            for (const key of Object.keys(data)) {
+              if (
+                key !== "_" &&
+                key !== userPublicKey &&
+                key !== userSignature
+              ) {
+                migrationData[key] = data[key]
+              }
+            }
+            // Sign each property individually
+            const propertySignatures = await SEA.signProperties(
+              migrationData,
+              user.is,
+            )
+            const graph = utils.graph(
+              pub,
+              migrationData,
+              propertySignatures,
+              data.pub,
+            )
+            wire.put(graph, err => {
+              if (err) {
+                // Log migration error but don't fail authentication
+                console.log(
+                  `warning: failed to migrate signatures on login: ${err}`,
+                )
+              }
+              done(null)
+            })
+            return
+          }
+
           return done(null)
         },
-        {wait: 1000},
+        {wait: 5000},
       )
     }
 
@@ -103,7 +143,7 @@ const User = (opt, wire) => {
         pubs = Object.keys(data)
         next()
       },
-      {wait: 1000},
+      {wait: 5000},
     )
   }
 
@@ -160,8 +200,9 @@ const User = (opt, wire) => {
           }
 
           const pub = "~" + pair.pub
-          const signed = await SEA.sign(data, pair)
-          const graph = utils.graph(pub, signed.m, signed.s, pair.pub)
+          // Sign each property individually for new account
+          const propertySignatures = await SEA.signProperties(data, pair)
+          const graph = utils.graph(pub, data, propertySignatures, pair.pub)
           wire.put(graph, err => {
             creating = false
             if (err) {
@@ -181,7 +222,7 @@ const User = (opt, wire) => {
             })
           })
         },
-        {wait: 1000},
+        {wait: 5000},
       )
     },
     auth: (username, password, cb) => {
@@ -318,9 +359,10 @@ const User = (opt, wire) => {
         }
 
         const data = {username: null, pub: null, epub: null, auth: null}
-        const signed = await SEA.sign(data, user.is)
+        // Sign each property individually for account deletion
+        const propertySignatures = await SEA.signProperties(data, user.is)
         const pub = "~" + user.is.pub
-        const graph = utils.graph(pub, signed.m, signed.s, user.is.pub)
+        const graph = utils.graph(pub, data, propertySignatures, user.is.pub)
         wire.put(graph, err => {
           if (err) {
             ack(`error putting null on ${pub}: ${err}`)

package/src/utils.js

@@ -90,13 +90,13 @@ export const graph = (soul, data, sig, pub) => {
       g[soul]._[">"][key] = timestamp
     }
   }
-  // If a signature and public key are provided they also need to be stored on
-  // the node to ensure that future updates are only possible with the same
-  // public key. The signature is requried because later get requests will
-  // broadcast the data as a put, which other devices will need to verify.
+  // Store per-property signatures in node._["s"]
+  // sig is an object with per-property signatures
   if (sig && pub) {
-    g[soul][userSignature] = sig
-    g[soul]._[">"][userSignature] = timestamp
+    g[soul]._["s"] = {}
+    for (const [key, signature] of Object.entries(sig)) {
+      g[soul]._["s"][key] = signature
+    }
     g[soul][userPublicKey] = pub
     g[soul]._[">"][userPublicKey] = timestamp
   }

package/src/wire.js

@@ -15,7 +15,7 @@ if (typeof globalThis.WebSocket === "undefined") {
 // Rate limiting with throttling
 const createRateLimiter = isTestEnv => {
   const clients = new Map()
-  const maxRequests = 1000 // requests per minute
+  const maxRequests = 1500 // requests per minute
   const windowMs = 60000 // 1 minute window
   const disconnectThreshold = 10 // Disconnect after 10 violations
   let cleanupInterval = null
@@ -654,7 +654,7 @@ const Wire = opt => {
     // Schedule processing of next message if queue not empty
     if (messageQueue.length > 0) {
       // Add delay between messages to respect rate limit
-      queueProcessor = setTimeout(processQueue, 100)
+      queueProcessor = setTimeout(processQueue, 50)
     } else {
       queueProcessor = null
     }
@@ -698,7 +698,7 @@ const Wire = opt => {
     messageQueue.push(data)
     // Start queue processor if not already running
     if (!queueProcessor) {
-      queueProcessor = setTimeout(processQueue, 100)
+      queueProcessor = setTimeout(processQueue, 50)
     }
   }
   if (!(opt.peers instanceof Array)) {

package/test/holster.on.test.js

@@ -16,16 +16,47 @@ describe("holster.on", () => {
   })
 
   test("on for property on root then update - event", (t, done) => {
-    // The property needs to exist before it can be listend to for updates.
+    // Listener doesn't get called for initial put.
+    const callback = data => {
+      assert.equal(data, "update")
+      holster.get("key").off(callback)
+      done()
+    }
+    holster.get("key").on(callback)
+
     holster.get("key").put("value", err => {
       assert.equal(err, null)
 
-      holster.get("key").on(data => {
-        assert.equal(data, "update")
-        done()
+      holster.get("key").put("update", err => {
+        assert.equal(err, null)
       })
+    })
+  })
 
-      holster.get("key").put("update", err => {
+  test("on for property then update - two listeners", (t, done) => {
+    let done1 = false
+    let done2 = false
+
+    const callback1 = data => {
+      assert.equal(data, "update")
+      holster.get("two").off(callback1)
+      if (done2) done()
+      else done1 = true
+    }
+    holster.get("two").on(callback1)
+
+    const callback2 = data => {
+      assert.equal(data, "update")
+      holster.get("two").off(callback2)
+      if (done1) done()
+      else done2 = true
+    }
+    holster.get("two").on(callback2)
+
+    holster.get("two").put("value", err => {
+      assert.equal(err, null)
+
+      holster.get("two").put("update", err => {
         assert.equal(err, null)
       })
     })