@mblaney/holster 1.0.29 → 1.0.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.29",
+  "version": "1.0.30",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/ham.js

@@ -1,7 +1,6 @@
 import * as utils from "./utils.js"
 import SEA from "./sea.js"
 
-const LISTENER_DELAY = 100
 // Maximum number of souls to keep in memory
 const MAX_GRAPH_SIZE = 10000
 
@@ -51,6 +50,7 @@ Ham.mix = async (change, graph, secure, listen) => {
   const machine = Date.now()
   const now = {}
   const defer = {}
+  const listeners = [] // Collect listeners to fire after store.put
   const validProperties = new Map() // Track valid properties per soul
   const validTimestamps = new Map() // Track valid timestamps per soul
   let wait = 0
@@ -187,30 +187,23 @@ Ham.mix = async (change, graph, secure, listen) => {
               graph[soul]._["s"][key] = now[soul]._["s"][key] = node._["s"][key]
             }
           }
-          // Call event listeners for update on key, mix is called before
-          // put has finished so wait for what could be multiple nested
-          // updates on a node.
+          // Collect event listeners to fire after store.put completes.
+          // This ensures listeners always fire with data that's been stored.
           if (listen[soul]) {
-            setTimeout(() => {
-              if (listen[soul]) {
-                listen[soul]
-                  .filter(l => utils.match(l["."], key))
-                  .forEach(l => l.cb())
-              }
-            }, LISTENER_DELAY)
+            listen[soul]
+              .filter(l => utils.match(l["."], key))
+              .forEach(l => listeners.push(l.cb))
           }
           updated = true
         }
       }
     }
 
-    // Call event listeners for update on soul.
+    // Collect event listeners for update on soul.
     if (updated && listen[soul]) {
-      setTimeout(() => {
-        if (listen[soul]) {
-          listen[soul].filter(l => utils.match(l["."])).forEach(l => l.cb())
-        }
-      }, LISTENER_DELAY)
+      listen[soul]
+        .filter(l => utils.match(l["."]))
+        .forEach(l => listeners.push(l.cb))
     }
   }
 
@@ -228,7 +221,7 @@ Ham.mix = async (change, graph, secure, listen) => {
     remove.forEach(({soul}) => delete graph[soul])
   }
 
-  return {now: now, defer: defer, wait: wait}
+  return {now: now, defer: defer, wait: wait, listeners: listeners}
 }
 
 export default Ham

package/src/holster.js

@@ -673,17 +673,23 @@ const Holster = opt => {
         const {item, soul} = resolve({off: true}, cb)
         if (!soul) return
 
-        // Check if item is a rel and remove event listener for the node.
+        // Remove listener immediately from the original soul to avoid race condition.
+        // The listener may be on a related soul if item is a rel, but we remove
+        // from the original soul first since that's where it's initially registered.
+        wire.off({"#": soul}, map.get(cb))
+
+        // Check if item is a rel and remove from the related node as well.
         wire.get({"#": soul, ".": item}, msg => {
           if (msg.err) {
-            console.log(`error getting ${soul}.${item}: ${msg.err}`)
+            // Even if get fails, we've already removed from original soul
+            map.delete(cb)
+            allctx.delete(ctxid)
             return
           }
 
           const current = msg.put && msg.put[soul] && msg.put[soul][item]
           const id = utils.rel.is(current)
           if (id) wire.off({"#": id}, map.get(cb))
-          else wire.off({"#": soul}, map.get(cb))
           map.delete(cb)
           allctx.delete(ctxid)
         })

package/src/user.js

@@ -29,7 +29,7 @@ const User = (opt, wire) => {
 
       const pub = pubs.shift()
       wire.get(
-        {"#": pub},
+        {"#": pub, ".": ["auth", "pub", "epub"]},
         async msg => {
           if (msg.err) {
             done(`error getting ${pub}: ${msg.err}`)

package/src/utils.js

@@ -118,6 +118,9 @@ export const match = (lex, key) => {
 
   if (typeof lex === "string") return lex === key
 
+  // Array support: check if key is in the array
+  if (Array.isArray(lex)) return lex.includes(key)
+
   if (!obj.is(lex) || !key) return false
 
   const prefix = lex["*"]

package/src/wire.js

@@ -197,6 +197,54 @@ const Wire = opt => {
   const rateLimiter = createRateLimiter(isTestEnv)
   const connectionManager = createConnectionManager(opt.maxConnections || 1000)
 
+  // The check function is required because user data must provide a public key
+  // so that it can be verified. The public key might verify the provided
+  // signature but not actually match the user under which the data is being
+  // stored. To avoid this, the current data on a soul needs to be checked to
+  // make sure the stored public key matches the one provided with the update.
+  const check = async (data, send, cb) => {
+    const key = utils.userPublicKey
+
+    for (const soul of Object.keys(data)) {
+      const msg = await new Promise(res => {
+        getWithCallback({"#": soul, ".": key}, res, send)
+      })
+      if (msg.err) {
+        if (cb) cb(msg.err)
+        return false
+      }
+
+      const node = data[soul]
+      // If there is no current node then the data is ok to write without
+      // matching public keys, as the provided soul also needs a rel on the
+      // parent node which then also requires checking. Otherwise public keys
+      // need to match for existing data. If the incoming update doesn't
+      // include a userPublicKey property at all, allow it - the update is
+      // not trying to change the public key.
+      if (
+        !msg.put ||
+        !msg.put[soul] ||
+        node[key] === undefined ||
+        msg.put[soul][key] === node[key]
+      ) {
+        continue
+      }
+
+      // If a soul exists but does not have a public key, then one should not be
+      // added because the node is not user data. The above check fails in this
+      // case if a public key is provided. Note that this is only an error case
+      // if called via the API, which is when a callback is provided here.
+      // (The wire spec can fetch and put data on the wire without a signature
+      // or public key and this can be ignored.)
+      if (cb) {
+        cb(`error in wire check public key does not match for soul: ${soul}`)
+      }
+      return false
+    }
+
+    return true
+  }
+
   const get = (msg, send) => {
     const ack = Get(msg.get, graph)
     if (ack) {
@@ -241,6 +289,8 @@ const Wire = opt => {
       return
     }
 
+    if (!(await check(update.now, send))) return
+
     store.put(update.now, err => {
       send(
         JSON.stringify({
@@ -249,6 +299,8 @@ const Wire = opt => {
           err: err,
         }),
       )
+      // Fire listeners after data is stored
+      update.listeners.forEach(cb => cb())
     })
 
     if (Object.keys(update.defer).length !== 0) {
@@ -342,6 +394,10 @@ const Wire = opt => {
           return
         }
 
+        if (!(await check(update.now, send, cb))) {
+          return
+        }
+
         // Seed pendingReferences with any new references from API calls
         for (const [soul, node] of Object.entries(update.now)) {
           if (node && typeof node === "object") {
@@ -355,7 +411,11 @@ const Wire = opt => {
           }
         }
 
-        store.put(update.now, cb)
+        store.put(update.now, err => {
+          if (cb) cb(err)
+          // Fire listeners after data is stored
+          update.listeners.forEach(l => l())
+        })
 
         // Always put data on the wire spec even if no local update needed
         const sendResult = send(

package/test/holster.on.test.js

@@ -16,13 +16,17 @@ describe("holster.on", () => {
   })
 
   test("on for property on root then update - event", (t, done) => {
-    // Listener doesn't get called for initial put.
-    const callback = data => {
-      assert.equal(data, "update")
-      holster.get("key").off(callback)
-      done()
-    }
-    holster.get("key").on(callback)
+    // Listener is now called for initial put and updates
+    let callCount = 0
+    holster.get("key").on(data => {
+      callCount++
+      if (callCount === 1) {
+        assert.equal(data, "value")
+      } else if (callCount === 2) {
+        assert.equal(data, "update")
+        done()
+      }
+    })
 
     holster.get("key").put("value", err => {
       assert.equal(err, null)
@@ -37,19 +41,27 @@ describe("holster.on", () => {
     let done1 = false
     let done2 = false
 
+    let callback1Count = 0
     const callback1 = data => {
-      assert.equal(data, "update")
-      holster.get("two").off(callback1)
-      if (done2) done()
-      else done1 = true
+      callback1Count++
+      if (callback1Count === 2) {
+        assert.equal(data, "update")
+        holster.get("two").off(callback1)
+        if (done2) done()
+        else done1 = true
+      }
     }
     holster.get("two").on(callback1)
 
+    let callback2Count = 0
     const callback2 = data => {
-      assert.equal(data, "update")
-      holster.get("two").off(callback2)
-      if (done1) done()
-      else done2 = true
+      callback2Count++
+      if (callback2Count === 2) {
+        assert.equal(data, "update")
+        holster.get("two").off(callback2)
+        if (done1) done()
+        else done2 = true
+      }
     }
     holster.get("two").on(callback2)
 
@@ -80,11 +92,12 @@ describe("holster.on", () => {
     holster.get("key3").put("value3", err => {
       assert.equal(err, null)
 
-      let first = true
+      let callCount = 0
       holster.get("key3").on(data => {
-        if (first) {
+        callCount++
+        if (callCount === 1) {
           assert.equal(data, "update1")
-        } else {
+        } else if (callCount === 2) {
           assert.equal(data, "update2")
           done()
         }
@@ -95,7 +108,6 @@ describe("holster.on", () => {
       })
 
       setTimeout(() => {
-        first = false
         holster.get("key3").put("update2", err => {
           assert.equal(err, null)
         })
@@ -112,10 +124,14 @@ describe("holster.on", () => {
 
     // Need to wait until after all initial puts are done to set on listeners.
     setTimeout(() => {
+      let completed = 0
       for (let i = 0; i < 5; i++) {
-        holster.get("for" + i).on(data => {
-          assert.equal(data, "update" + i)
-          if (i === 4) done()
+        const idx = i // Capture value
+        holster.get("for" + idx).on(data => {
+          // Listener fires for updates (initial puts already completed before listener setup)
+          assert.equal(data, "update" + idx)
+          completed++
+          if (completed === 5) done()
         })
       }
 
@@ -154,6 +170,7 @@ describe("holster.on", () => {
           .get("hello")
           .next("world!")
           .on(data => {
+            // Listener fires for updates (initial put already completed before listener setup)
             assert.equal(data, "update")
             holster.get("hello").next("world!").off()
             done()
@@ -183,6 +200,7 @@ describe("holster.on", () => {
         number: 42,
       }
       holster.get("plain").on(data => {
+        // Listener fires for updates (initial put already completed before listener setup)
         assert.deepEqual(data, update)
         done()
       })
@@ -211,14 +229,20 @@ describe("holster.on", () => {
           has: "child update",
         },
       }
+      let parentFired = false
       holster.get("nested").on(data => {
-        assert.deepEqual(data, {key: nested.key, ...update})
+        // Listener fires for updates (initial put already completed before listener setup)
+        if (!parentFired) {
+          parentFired = true
+          assert.deepEqual(data, {key: nested.key, ...update})
+        }
       })
 
       holster
         .get("nested")
         .next("child")
         .on(data => {
+          // Listener fires for updates (initial put already completed before listener setup)
           assert.deepEqual(data, update.child)
           done()
         })

package/test/holster.user.on.test.js

@@ -31,11 +31,17 @@ describe("holster.user.on", () => {
   })
 
   test("on for property then update - event", (t, done) => {
-    // Listener doesn't get called for initial put.
+    // Listener is now called for initial put and updates
+    let callCount = 0
     const callback = data => {
-      assert.equal(data, "update")
-      user.get("key").off(callback)
-      done()
+      callCount++
+      if (callCount === 1) {
+        assert.equal(data, "value")
+      } else if (callCount === 2) {
+        assert.equal(data, "update")
+        user.get("key").off(callback)
+        done()
+      }
     }
     user.get("key").on(callback)
 
@@ -52,19 +58,27 @@ describe("holster.user.on", () => {
     let done1 = false
     let done2 = false
 
+    let callback1Count = 0
     const callback1 = data => {
-      assert.equal(data, "update")
-      user.get("two").off(callback1)
-      if (done2) done()
-      else done1 = true
+      callback1Count++
+      if (callback1Count === 2) {
+        assert.equal(data, "update")
+        user.get("two").off(callback1)
+        if (done2) done()
+        else done1 = true
+      }
     }
     user.get("two").on(callback1)
 
+    let callback2Count = 0
     const callback2 = data => {
-      assert.equal(data, "update")
-      user.get("two").off(callback2)
-      if (done1) done()
-      else done2 = true
+      callback2Count++
+      if (callback2Count === 2) {
+        assert.equal(data, "update")
+        user.get("two").off(callback2)
+        if (done1) done()
+        else done2 = true
+      }
     }
     user.get("two").on(callback2)
 
@@ -92,11 +106,12 @@ describe("holster.user.on", () => {
     user.get("key3").put("value3", err => {
       assert.equal(err, null)
 
-      let first = true
+      let callCount = 0
       const callback = data => {
-        if (first) {
+        callCount++
+        if (callCount === 1) {
           assert.equal(data, "update1")
-        } else {
+        } else if (callCount === 2) {
           assert.equal(data, "update2")
           user.get("key3").off(callback)
           done()
@@ -109,7 +124,6 @@ describe("holster.user.on", () => {
       })
 
       setTimeout(() => {
-        first = false
         user.get("key3").put("update2", err => {
           assert.equal(err, null)
         })
@@ -132,6 +146,7 @@ describe("holster.user.on", () => {
       for (let i = 0; i < 5; i++) {
         const idx = i // Capture value
         const callback = data => {
+          // Listener fires for updates (initial puts already completed before listener setup)
           assert.equal(data, "update" + idx)
           user.get("for" + idx).off(callback)
           count++
@@ -176,6 +191,7 @@ describe("holster.user.on", () => {
         assert.equal(err, null)
 
         const callback = data => {
+          // Listener fires for updates (initial put already completed before listener setup)
           assert.equal(data, "update")
           user.get("hello").next("world!").off(callback)
           done()
@@ -208,6 +224,7 @@ describe("holster.user.on", () => {
       }
 
       const callback = data => {
+        // Listener fires for updates (initial put already completed before listener setup)
         assert.deepEqual(data, update)
         user.get("plain").off(callback)
         done()
@@ -240,12 +257,18 @@ describe("holster.user.on", () => {
         },
       }
 
+      let parentFired = false
       const parentCallback = data => {
-        assert.deepEqual(data, update)
+        // Listener fires for updates (initial put already completed before listener setup)
+        if (!parentFired) {
+          parentFired = true
+          assert.deepEqual(data, update)
+        }
       }
       user.get("nested").on(parentCallback)
 
       const childCallback = data => {
+        // Listener fires for updates (initial put already completed before listener setup)
         assert.deepEqual(data, update.child)
         user.get("nested").off(parentCallback)
         user.get("nested").next("child").off(childCallback)

package/test/system/concurrent-listeners.test.js

@@ -86,22 +86,23 @@ describe("system - concurrent listeners", () => {
     holster.get("race").put({value: "first"}, err => {
       assert.equal(err, null)
 
-      // Set up listener with once=true to read existing data
+      // Set up listener with _get=true to read existing data
+      // The listener will be called with the existing data via the get request
       holster.get("race").on(data => {
         results.push(data?.value)
       }, true)
 
-      // Immediately put new data
+      // Immediately put new data - listener will fire for this put as well
       setTimeout(() => {
         holster.get("race").put({value: "second"}, err => {
           assert.equal(err, null)
 
           setTimeout(() => {
-            // Should have received both values
+            // Should have received both values - one from get, one from put
             assert.equal(
               results.length >= 1,
               true,
-              "Should receive at least initial value",
+              "Should receive at least initial value from get",
             )
             assert.equal(results[0], "first", "First value should be 'first'")
             done()

package/test/system/listener-before-data.test.js

@@ -23,7 +23,8 @@ describe("system - listener before data", () => {
 
         // Give listener time to fire
         setTimeout(() => {
-          // Should have received null (initial) and the test value
+          // With the new behavior, listener fires after data is stored
+          // Should have received the test value from the put
           const nonNull = results.filter(r => r !== null)
           assert.equal(
             nonNull.length >= 1,

package/test/system/user-concurrent-listeners.test.js

@@ -103,21 +103,22 @@ describe("system - user concurrent listeners", () => {
       assert.equal(err, null)
 
       // Set up listener with _get=true to read existing data
+      // The listener will be called with the existing data via the get request
       user.get("race").on(data => {
         results.push(data?.value)
       }, true)
 
-      // Immediately put new data
+      // Immediately put new data - listener will fire for this put as well
       setTimeout(() => {
         user.get("race").put({value: "second"}, err => {
           assert.equal(err, null)
 
           setTimeout(() => {
-            // Should have received both values
+            // Should have received both values - one from get, one from put
             assert.equal(
               results.length >= 1,
               true,
-              "Should receive at least initial value",
+              "Should receive at least initial value from get",
             )
             assert.equal(results[0], "first", "First value should be 'first'")
             done()

package/test/system/user-listener-before-data.test.js

@@ -37,7 +37,8 @@ describe("system - user listener before data", () => {
 
         // Give listener time to fire
         setTimeout(() => {
-          // Should have received the test value
+          // With the new behavior, listener fires after data is stored
+          // Should have received the test value from the put
           const nonNull = results.filter(r => r !== null)
           assert.equal(
             nonNull.length >= 1,