@mblaney/holster 1.0.35 → 2.0.0

package/package.json

@@ -1,25 +1,42 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.35",
+  "version": "2.0.0",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",
   "scripts": {
     "test": "node --test",
-    "build": "esbuild src/holster.js --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=build"
-  },
+    "test:ts": "node --test --import tsx/esm ts/test/*.test.ts ts/test/system/*.test.ts",
+    "build": "esbuild src/holster.js --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=build",
+    "build:ts": "esbuild ts/src/holster.ts --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=dist",
+    "build:all": "npm run build && npm run build:ts",
+    "prepublishOnly": "npm run build:all",
+    "type-check": "tsc -p tsconfig.json",
+    "type-check:tests": "tsc -p tsconfig.test.json"
+  },
+  "files": [
+    "src",
+    "ts/src",
+    "build",
+    "dist"
+  ],
   "keywords": [],
   "author": "Malcolm Blaney",
   "license": "Zlib, MIT, Apache-2.0",
   "dependencies": {
     "express": "^5.1.0",
     "fs": "^0.0.1-security",
-    "ws": "^8.18.0"
+    "ws": "^8.18.0",
+    "zod": "^4.1.13"
   },
   "devDependencies": {
+    "@types/node": "^24.10.1",
+    "@types/ws": "^8.18.1",
     "esbuild": "0.25.3",
     "mock-socket": "^9.3.1",
-    "prettier": "^3.4.2"
+    "prettier": "^3.4.2",
+    "tsx": "^4.0.0",
+    "typescript": "^5.9.3"
   },
   "prettier": {
     "semi": false,

package/src/dup.js

@@ -1,6 +1,5 @@
-const Dup = maxAge => {
-  // Allow maxAge to be passed in as tests wait on the setTimeout.
-  if (!maxAge) maxAge = 9000
+const Dup = () => {
+  const maxAge = 9000
   const dup = {store: {}}
   dup.check = id => (dup.store[id] ? dup.track(id) : false)
   dup.track = id => {
@@ -8,14 +7,14 @@ const Dup = maxAge => {
     dup.store[id] = Date.now()
     if (!dup.expiry) {
       dup.expiry = setTimeout(() => {
-        if (dup.expiry) return
-
         const now = Date.now()
         Object.keys(dup.store).forEach(id => {
           if (now - dup.store[id] > maxAge) delete dup.store[id]
         })
         dup.expiry = null
       }, maxAge)
+      // Don't block process exit — cleanup is best-effort.
+      if (dup.expiry.unref) dup.expiry.unref()
     }
     return id
   }

package/src/get.js

@@ -25,12 +25,8 @@ const Get = (lex, graph, fast) => {
     node[key] = graph[soul][key]
     node._[">"][key] = graph[soul]._[">"][key]
     const state = graph[soul]._[">"][key]
-    if (graph[soul]._["s"]) {
-      if (graph[soul]._["s"][state]) {
-        signatures[state] = graph[soul]._["s"][state]
-      } else if (graph[soul]._["s"][key]) {
-        signatures[key] = graph[soul]._["s"][key]
-      }
+    if (graph[soul]._["s"] && graph[soul]._["s"][state]) {
+      signatures[state] = graph[soul]._["s"][state]
     }
 
     // Always include userPublicKey for verification, regardless of filter
@@ -38,12 +34,8 @@ const Get = (lex, graph, fast) => {
       node[userPublicKey] = graph[soul][userPublicKey]
       node._[">"][userPublicKey] = graph[soul]._[">"][userPublicKey]
       const pkState = graph[soul]._[">"][userPublicKey]
-      if (graph[soul]._["s"]) {
-        if (graph[soul]._["s"][pkState]) {
-          signatures[pkState] = graph[soul]._["s"][pkState]
-        } else if (graph[soul]._["s"][userPublicKey]) {
-          signatures[userPublicKey] = graph[soul]._["s"][userPublicKey]
-        }
+      if (graph[soul]._["s"] && graph[soul]._["s"][pkState]) {
+        signatures[pkState] = graph[soul]._["s"][pkState]
       }
     }
   } else {
@@ -51,21 +43,20 @@ const Get = (lex, graph, fast) => {
     // return undefined to ensure we get all matching properties from store.
     if (!fast) return
 
+    let found = false
     for (const key of Object.keys(graph[soul])) {
       // Always include userPublicKey for verification, regardless of filter
-      if (key === userPublicKey || match(lex["."], key)) {
+      if (key !== "_" && (key === userPublicKey || match(lex["."], key))) {
         node[key] = graph[soul][key]
         node._[">"][key] = graph[soul]._[">"][key]
         const state = graph[soul]._[">"][key]
-        if (graph[soul]._["s"]) {
-          if (graph[soul]._["s"][state]) {
-            signatures[state] = graph[soul]._["s"][state]
-          } else if (graph[soul]._["s"][key]) {
-            signatures[key] = graph[soul]._["s"][key]
-          }
+        if (graph[soul]._["s"] && graph[soul]._["s"][state]) {
+          signatures[state] = graph[soul]._["s"][state]
         }
+        found = true
       }
     }
+    if (!found) return
   }
   if (Object.keys(signatures).length > 0) {
     node._["s"] = signatures

package/src/ham.js

@@ -35,7 +35,7 @@ const Ham = (state, currentState, value, currentValue, signed = false) => {
   return {incoming: true}
 }
 
-Ham.mix = async (change, graph, secure, listen) => {
+Ham.mix = async (change, graph, hasNode, secure, listen) => {
   if (!change || typeof change !== "object") {
     throw new TypeError("change must be an object")
   }
@@ -102,21 +102,21 @@ Ham.mix = async (change, graph, secure, listen) => {
       for (const [key, sig] of Object.entries(node._["s"] || {})) {
         const asNumber = Number(key)
         if (!isNaN(asNumber) && asNumber > 0) {
-          // Check if this timestamp is for properties that already have newer
-          // versions in graph.
-          const hasNewerInGraph = incomingTimestamps.some(([prop, ts]) => {
-            if (Number(ts) === asNumber) {
+          // Skip verification only if ALL properties for this timestamp are
+          // already newer in the graph. Using some() would incorrectly skip
+          // valid updates when only one property in a batch is outdated.
+          const propsForTimestamp = incomingTimestamps.filter(
+            ([, ts]) => Number(ts) === asNumber,
+          )
+          const allNewerInGraph =
+            propsForTimestamp.length > 0 &&
+            propsForTimestamp.every(([prop]) => {
               const graphState =
                 graph[soul] && graph[soul]._[">"] ? graph[soul]._[">"][prop] : 0
-              // Skip verification if graph has a newer timestamp for property.
-              if (graphState > asNumber) {
-                return true
-              }
-            }
-            return false
-          })
+              return graphState > asNumber
+            })
 
-          if (hasNewerInGraph) {
+          if (allNewerInGraph) {
             continue
           }
 
@@ -133,28 +133,6 @@ Ham.mix = async (change, graph, secure, listen) => {
         }
       }
 
-      // TODO: Remove per-property signature verification after old data has
-      // been cleared. Only check per-property signatures if there are
-      // properties without timestamp signatures.
-      const propertiesWithoutTimestampSigs = Object.keys(node).filter(k => {
-        if (k === "_" || k === utils.userPublicKey) return false
-        return !incomingTimestamps.some(
-          ([prop, ts]) => prop === k && signedTimestamps.has(Number(ts)),
-        )
-      })
-
-      if (propertiesWithoutTimestampSigs.length > 0) {
-        const nodeToVerify = {_: node._}
-        for (const prop of propertiesWithoutTimestampSigs) {
-          nodeToVerify[prop] = node[prop]
-        }
-        const perPropertySignedProps = await SEA.verifyProperties(
-          nodeToVerify,
-          pub,
-        )
-        perPropertySignedProps.forEach(prop => signedProperties.add(prop))
-      }
-
       // If no properties verified, skip this update
       if (signedProperties.size === 0) {
         continue
@@ -196,12 +174,8 @@ Ham.mix = async (change, graph, secure, listen) => {
         }
         defer[soul][key] = value
         defer[soul]._[">"][key] = state
-        if (node._["s"]) {
-          if (node._["s"][state]) {
-            defer[soul]._["s"][state] = node._["s"][state]
-          } else if (node._["s"][key]) {
-            defer[soul]._["s"][key] = node._["s"][key]
-          }
+        if (node._["s"] && node._["s"][state]) {
+          defer[soul]._["s"][state] = node._["s"][state]
         }
       } else {
         // Check if this property has a signed timestamp
@@ -217,13 +191,9 @@ Ham.mix = async (change, graph, secure, listen) => {
           }
           graph[soul][key] = now[soul][key] = value
           graph[soul]._[">"][key] = now[soul]._[">"][key] = state
-          if (node._["s"]) {
-            if (node._["s"][state]) {
-              graph[soul]._["s"][state] = now[soul]._["s"][state] =
-                node._["s"][state]
-            } else if (node._["s"][key]) {
-              graph[soul]._["s"][key] = now[soul]._["s"][key] = node._["s"][key]
-            }
+          if (node._["s"] && node._["s"][state]) {
+            graph[soul]._["s"][state] = now[soul]._["s"][state] =
+              node._["s"][state]
           }
           // Collect event listeners to fire after store.put completes.
           // This ensures listeners always fire with data that's been stored.
@@ -256,7 +226,10 @@ Ham.mix = async (change, graph, secure, listen) => {
       })
       .sort((a, b) => a.maxState - b.maxState)
     const remove = soulsByAge.slice(0, souls.length - MAX_GRAPH_SIZE)
-    remove.forEach(({soul}) => delete graph[soul])
+    remove.forEach(({soul}) => {
+      delete graph[soul]
+      if (hasNode) hasNode.delete(soul)
+    })
   }
 
   return {now: now, defer: defer, wait: wait, listeners: listeners}

package/src/holster.js

@@ -64,30 +64,28 @@ const Holster = opt => {
             delete msg.put[soul][utils.userPublicKey]
             delete msg.put[soul][utils.userSignature]
             // Resolve any rels on the node before returning to the user.
-            for (const key of Object.keys(msg.put[soul])) {
+            await Promise.all(Object.keys(msg.put[soul]).map(async key => {
               const id = utils.rel.is(msg.put[soul][key])
-              if (id) {
-                // Retry with delays if we get null
-                const attemptRead = async (retries = 0) => {
-                  const data = await new Promise(res => {
-                    const _ctxid = utils.text.random()
-                    const ctx = allctx.get(ctxid)
-                    allctx.set(_ctxid, {
-                      chain: [{item: null, soul: id}],
-                      user: ctx ? ctx.user : null,
-                    })
-                    api(_ctxid).next(null, res, _opt)
+              if (!id) return
+              const attemptRead = async (retries = 0) => {
+                const data = await new Promise(res => {
+                  const _ctxid = utils.text.random()
+                  const ctx = allctx.get(ctxid)
+                  allctx.set(_ctxid, {
+                    chain: [{item: null, soul: id}],
+                    user: ctx ? ctx.user : null,
                   })
-                  if (data !== null || retries >= 5) {
-                    return data
-                  }
-                  // Data not ready, retry after delay
-                  await new Promise(resolve => setTimeout(resolve, 50))
-                  return attemptRead(retries + 1)
+                  api(_ctxid).next(null, res, _opt)
+                })
+                if (data !== null || retries >= 5) {
+                  return data
                 }
-                msg.put[soul][key] = await attemptRead()
+                // Data not ready, retry after delay
+                await new Promise(resolve => setTimeout(resolve, 50))
+                return attemptRead(retries + 1)
               }
-            }
+              msg.put[soul][key] = await attemptRead()
+            }))
             ack(msg.put[soul])
           } else {
             // No data callback.
@@ -132,6 +130,55 @@ const Holster = opt => {
       }
     }
 
+    const watchForRel = (soul, item, ctx, i, request, cb, on = true) => {
+      let timer = null
+      const handler = () => {
+        // Bail out if the context was removed by off().
+        if (!allctx.has(ctxid)) {
+          wire.off({"#": soul, ".": item}, handler)
+          if (timer) clearTimeout(timer)
+          return
+        }
+        wire.get(
+          {"#": soul, ".": item},
+          msg => {
+            const node = msg.put && msg.put[soul]
+            const id = utils.rel.is(node && node[item])
+            if (!id) {
+              // If it's a plain value (not a rel) and this is a get, deliver it.
+              // node._ guards against timeout null-acks which lack metadata.
+              if (!on && node && node._ && typeof node[item] !== "undefined") {
+                if (timer) clearTimeout(timer)
+                wire.off({"#": soul, ".": item}, handler)
+                cb(node[item])
+              }
+              return
+            }
+            if (timer) clearTimeout(timer)
+            wire.off({"#": soul, ".": item}, handler)
+            ctx.chain[i].soul = id
+            allctx.set(ctxid, {...ctx})
+            if (on) {
+              api(ctxid).on(request.on, cb, request._get, request._opt)
+            } else {
+              api(ctxid).next(null, request.get, cb, request._opt)
+            }
+          },
+          {...request._opt, secure: !!(ctx.user || opt.secure)},
+        )
+      }
+      wire.on({"#": soul, ".": item}, handler, false, request._opt)
+      // Time out after the same total duration as the on() retry loop
+      // (1+2+4+8+16 = 31s) so callers are not blocked forever if the node
+      // genuinely doesn't exist.
+      if (!on) {
+        timer = setTimeout(() => {
+          wire.off({"#": soul, ".": item}, handler)
+          if (cb) cb(null)
+        }, 31000)
+      }
+    }
+
     const resolve = (request, cb) => {
       if (!request) {
         console.log("error resolve request parameter required")
@@ -145,6 +192,7 @@ const Holster = opt => {
 
       let found = false
       const ctx = allctx.get(ctxid)
+      if (!ctx) return {item: null, soul: null}
       for (var i = 1; i < ctx.chain.length; i++) {
         if (ctx.chain[i].soul !== null) continue
 
@@ -201,8 +249,10 @@ const Holster = opt => {
                   api(ctxid).put(request.put, cb)
                 })
               } else if (on) {
-                // Allow listening to a node that doesn't exist.
-                cb(null)
+                // Item is not a rel yet — watch the parent soul for when it
+                // becomes one, then retry chain resolution.
+                watchForRel(soul, item, ctx, i, request, cb)
+                if (request._get) cb(null)
               } else if (off) {
                 // Allow stop listening to a node that doesn't exist.
                 if (cb) cb(null)
@@ -225,11 +275,26 @@ const Holster = opt => {
                 api(ctxid).put(request.put, cb)
               })
             } else {
-              // Allow querying a node that doesn't exist.
-              if (cb) cb(null)
+              if (on) {
+                // Node doesn't exist yet — watch for it to appear.
+                watchForRel(soul, item, ctx, i, request, cb)
+                if (request._get) cb(null)
+              } else if (get && node) {
+                const sv = node._ && node._[">"]
+                if (sv && Object.keys(sv).length > 0 && typeof sv[item] === "undefined") {
+                  // State vector has other properties but not this one — never written.
+                  if (cb) cb(null)
+                } else {
+                  // Empty state vector or item is tracked — may arrive via push.
+                  watchForRel(soul, item, ctx, i, request, cb, false)
+                }
+              } else {
+                // Soul doesn't exist or no get — return null.
+                if (cb) cb(null)
+              }
             }
           },
-          {...request._opt, secure: ctx.user || opt.secure},
+          {...request._opt, secure: !!(ctx.user || opt.secure)},
         )
         // Callback has been passed to next soul lookup or called above, so
         // return false as the calling code should not continue.
@@ -374,7 +439,7 @@ const Holster = opt => {
 
         // Resolve the current context before putting data. (Note that set is
         // not passed to resolve because it's already been applied above.)
-        const {item, soul} = resolve({put: data}, _ack)
+        const {item, soul} = resolve({put: data, _opt: {put: true}}, _ack)
         if (!soul) return
 
         if (result === true) {
@@ -443,7 +508,7 @@ const Holster = opt => {
                 wire.put(g, _ack)
               })
             },
-            {secure: ctx.user || opt.secure},
+            {secure: !!(ctx.user || opt.secure), put: true},
           )
           return
         }
@@ -483,7 +548,7 @@ const Holster = opt => {
                   } else {
                     const _ctxid = utils.text.random()
                     const chain = [{item: item, soul: soul}]
-                    // Pass on the previous context's callback and user flag here.
+                    // Pass on the previous ctx's callback and user flag here.
                     allctx.set(_ctxid, {
                       chain: chain,
                       user: ctx.user,
@@ -521,27 +586,18 @@ const Holster = opt => {
                 return
               }
 
-              // The nested objects created above will also have rels on the
-              // parent object, so fetch the node so that the rest of the
-              // updates can be added.
-              wire.get({"#": id}, async msg => {
-                if (msg.err) {
-                  _ack(`error getting ${id}: ${msg.err}`)
-                  return
-                }
-
-                let node = msg.put && msg.put[id]
-                if (!node) node = {}
-                update.forEach(key => {
-                  node[key] = data[key]
-                })
-                const g = await graph(id, node, ctx.user, _ack)
-                if (g === null) return
-
-                wire.put(g, _ack)
+              // Only write the primitive update keys. Rels created by the
+              // recursive puts above are already in the in-memory graph.
+              const updateNode = {}
+              update.forEach(key => {
+                updateNode[key] = data[key]
               })
+              const g = await graph(id, updateNode, ctx.user, _ack)
+              if (g === null) return
+
+              wire.put(g, _ack)
             },
-            {secure: ctx.user || opt.secure},
+            {secure: !!(ctx.user || opt.secure), put: true},
           )
         }
         attemptRead()
@@ -579,13 +635,28 @@ const Holster = opt => {
         })
         // Map the user's callback because it can also be passed to off,
         // so need a reference to it to compare them.
-        // Create a new context for each listener invocation to avoid mutation
+        // Shared retry state for this on() subscription.
+        let retryCount = 0
+        const maxRetries = 5
+        const retryDelay = 1000 // Start with 1 second
+        let retryTimer = null
+
         map.set(cb, () => {
+          // Bail out if off() has already cleaned up this context — the mapped
+          // callback can fire twice (once from wire.on and once from the _get
+          // immediate-read path) and the second call after async context
+          // cleanup would crash in resolve(). Matches the guard in watchForRel.
+          if (!allctx.has(ctxid)) return
+          // Cancel any pending retry — the persistent listener firing means we
+          // have an update and should not call cb twice.
+          clearTimeout(retryTimer)
+          retryTimer = null
           // When listener fires, re-check if the item is now a rel.
           wire.get(
             {"#": soul, ".": item},
             msg => {
-              const current = msg.put && msg.put[soul] && msg.put[soul][item]
+              const node = msg.put && msg.put[soul]
+              const current = node && node[item]
               const id = utils.rel.is(current)
               if (id) {
                 // It's a rel, read the related node. It might not be in the
@@ -597,9 +668,11 @@ const Holster = opt => {
                       chain: [{item: null, soul: id}],
                       user: ctx ? ctx.user : null,
                     })
-                    api(_ctxid).next(null, res, _opt)
+                    api(_ctxid).next(null, res, retries === 0 ?
+                                     utils.obj.put(_opt, "fast", true) : _opt)
                   })
                   if (data !== null || retries >= 5) {
+                    retryCount = 0 // Reset retry counter on success
                     cb(data)
                   } else {
                     // Data not ready, retry after delay
@@ -609,11 +682,50 @@ const Holster = opt => {
                 }
                 attemptRead()
               } else {
-                // It's a direct property or null
-                cb(current !== undefined ? current : null)
+                // It's a direct property or null.
+                const nodeExists = !!node
+
+                if (nodeExists) {
+                  // Node exists — return the value and reset retry state.
+                  retryCount = 0
+                  cb(current !== undefined ? current : null)
+                } else if (retryCount < maxRetries) {
+                  // No data yet — keep polling until we get a response or
+                  // exhaust retries. on() with _get can return null on a cache
+                  // miss and never fire again if the data doesn't change while
+                  // the listener is attached, so we must poll for initial load.
+                  const retry = () => {
+                    const delay =
+                          Math.min(retryDelay * Math.pow(2, retryCount), 30000)
+                    retryCount++
+                    retryTimer = setTimeout(() => {
+                      wire.get(
+                        {"#": soul, ".": item},
+                        msg => {
+                          const retryNode = msg.put && msg.put[soul]
+                          if (retryNode) {
+                            retryCount = 0
+                            retryTimer = null
+                            const retryValue = retryNode[item]
+                            cb(retryValue !== undefined ? retryValue : null)
+                          } else if (retryCount < maxRetries) {
+                            retry()
+                          } else {
+                            retryTimer = null
+                            cb(null)
+                          }
+                        },
+                        {..._opt, secure: !!(ctx.user || opt.secure)},
+                      )
+                    }, delay)
+                  }
+                  retry()
+                } else {
+                  cb(null)
+                }
               }
             },
-            {..._opt, secure: ctx.user || opt.secure},
+            {..._opt, secure: !!(ctx.user || opt.secure)},
           )
         })
 
@@ -653,13 +765,14 @@ const Holster = opt => {
               let relLex
               if (lex) relLex = utils.obj.put(initialLex, "#", id)
               else relLex = {"#": id, ".": null}
-              wire.on(relLex, map.get(cb), _get, _opt)
+              wire.on(relLex, map.get(cb), false, _opt)
+              if (_get) map.get(cb)()
             } else if (_get) {
               // Not a rel, but _get was requested, so trigger callback.
               map.get(cb)()
             }
           },
-          {..._opt, secure: ctx.user || opt.secure},
+          {..._opt, secure: !!(ctx.user || opt.secure)},
         )
       },
       off: cb => {
@@ -673,9 +786,10 @@ const Holster = opt => {
         const {item, soul} = resolve({off: true}, cb)
         if (!soul) return
 
-        // Remove listener immediately from the original soul to avoid race condition.
-        // The listener may be on a related soul if item is a rel, but we remove
-        // from the original soul first since that's where it's initially registered.
+        // Remove listener immediately from the original soul to avoid a race
+        // condition. The listener may be on a related soul if item is a rel,
+        // but we remove from the original soul first since that's where it's
+        //initially registered.
         wire.off({"#": soul}, map.get(cb))
 
         // Check if item is a rel and remove from the related node as well.

package/src/index.js

@@ -3,7 +3,7 @@ import path from "path"
 import {fileURLToPath} from "url"
 import Holster from "./holster.js"
 
-const holster = Holster()
+const holster = Holster({port: 8765})
 
 // Note that express is only used here to serve the examples folder. It is not
 // required to run Holster itself.