@mblaney/holster 1.0.34 → 1.0.36

package/package.json

@@ -1,25 +1,43 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.34",
+  "version": "1.0.36",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",
   "scripts": {
     "test": "node --test",
-    "build": "esbuild src/holster.js --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=build"
-  },
+    "test:ts": "node --test --import tsx/esm ts/test/**/*.test.ts",
+    "test:ts:system": "node --test --import tsx/esm ts/test/system/**/*.test.ts",
+    "build": "esbuild src/holster.js --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=build",
+    "build:ts": "esbuild ts/src/holster.ts --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=dist",
+    "build:all": "npm run build && npm run build:ts",
+    "prepublishOnly": "npm run build:all",
+    "type-check": "tsc --noEmit",
+    "type-check:tests": "tsc --noEmit --project tsconfig.test.json"
+  },
+  "files": [
+    "src",
+    "ts/src",
+    "build",
+    "dist"
+  ],
   "keywords": [],
   "author": "Malcolm Blaney",
   "license": "Zlib, MIT, Apache-2.0",
   "dependencies": {
     "express": "^5.1.0",
     "fs": "^0.0.1-security",
-    "ws": "^8.18.0"
+    "ws": "^8.18.0",
+    "zod": "^4.1.13"
   },
   "devDependencies": {
+    "@types/node": "^24.10.1",
+    "@types/ws": "^8.18.1",
     "esbuild": "0.25.3",
     "mock-socket": "^9.3.1",
-    "prettier": "^3.4.2"
+    "prettier": "^3.4.2",
+    "tsx": "^4.0.0",
+    "typescript": "^5.9.3"
   },
   "prettier": {
     "semi": false,

package/src/get.js

@@ -1,6 +1,6 @@
 import {match, userPublicKey} from "./utils.js"
 
-const Get = (lex, graph) => {
+const Get = (lex, graph, fast) => {
   if (!lex || typeof lex !== "object") {
     throw new TypeError("lex must be an object")
   }
@@ -25,12 +25,8 @@ const Get = (lex, graph) => {
     node[key] = graph[soul][key]
     node._[">"][key] = graph[soul]._[">"][key]
     const state = graph[soul]._[">"][key]
-    if (graph[soul]._["s"]) {
-      if (graph[soul]._["s"][state]) {
-        signatures[state] = graph[soul]._["s"][state]
-      } else if (graph[soul]._["s"][key]) {
-        signatures[key] = graph[soul]._["s"][key]
-      }
+    if (graph[soul]._["s"] && graph[soul]._["s"][state]) {
+      signatures[state] = graph[soul]._["s"][state]
     }
 
     // Always include userPublicKey for verification, regardless of filter
@@ -38,27 +34,23 @@ const Get = (lex, graph) => {
       node[userPublicKey] = graph[soul][userPublicKey]
       node._[">"][userPublicKey] = graph[soul]._[">"][userPublicKey]
       const pkState = graph[soul]._[">"][userPublicKey]
-      if (graph[soul]._["s"]) {
-        if (graph[soul]._["s"][pkState]) {
-          signatures[pkState] = graph[soul]._["s"][pkState]
-        } else if (graph[soul]._["s"][userPublicKey]) {
-          signatures[userPublicKey] = graph[soul]._["s"][userPublicKey]
-        }
+      if (graph[soul]._["s"] && graph[soul]._["s"][pkState]) {
+        signatures[pkState] = graph[soul]._["s"][pkState]
       }
     }
   } else {
+    // For fast requests, return properties currently in the graph. Otherwise
+    // return undefined to ensure we get all matching properties from store.
+    if (!fast) return
+
     for (const key of Object.keys(graph[soul])) {
       // Always include userPublicKey for verification, regardless of filter
       if (key === userPublicKey || match(lex["."], key)) {
         node[key] = graph[soul][key]
         node._[">"][key] = graph[soul]._[">"][key]
         const state = graph[soul]._[">"][key]
-        if (graph[soul]._["s"]) {
-          if (graph[soul]._["s"][state]) {
-            signatures[state] = graph[soul]._["s"][state]
-          } else if (graph[soul]._["s"][key]) {
-            signatures[key] = graph[soul]._["s"][key]
-          }
+        if (graph[soul]._["s"] && graph[soul]._["s"][state]) {
+          signatures[state] = graph[soul]._["s"][state]
         }
       }
     }

package/src/ham.js

@@ -102,21 +102,21 @@ Ham.mix = async (change, graph, secure, listen) => {
       for (const [key, sig] of Object.entries(node._["s"] || {})) {
         const asNumber = Number(key)
         if (!isNaN(asNumber) && asNumber > 0) {
-          // Check if this timestamp is for properties that already have newer
-          // versions in graph.
-          const hasNewerInGraph = incomingTimestamps.some(([prop, ts]) => {
-            if (Number(ts) === asNumber) {
+          // Skip verification only if ALL properties for this timestamp are
+          // already newer in the graph. Using some() would incorrectly skip
+          // valid updates when only one property in a batch is outdated.
+          const propsForTimestamp = incomingTimestamps.filter(
+            ([, ts]) => Number(ts) === asNumber,
+          )
+          const allNewerInGraph =
+            propsForTimestamp.length > 0 &&
+            propsForTimestamp.every(([prop]) => {
               const graphState =
                 graph[soul] && graph[soul]._[">"] ? graph[soul]._[">"][prop] : 0
-              // Skip verification if graph has a newer timestamp for property.
-              if (graphState > asNumber) {
-                return true
-              }
-            }
-            return false
-          })
+              return graphState > asNumber
+            })
 
-          if (hasNewerInGraph) {
+          if (allNewerInGraph) {
             continue
           }
 
@@ -133,28 +133,6 @@ Ham.mix = async (change, graph, secure, listen) => {
         }
       }
 
-      // TODO: Remove per-property signature verification after old data has
-      // been cleared. Only check per-property signatures if there are
-      // properties without timestamp signatures.
-      const propertiesWithoutTimestampSigs = Object.keys(node).filter(k => {
-        if (k === "_" || k === utils.userPublicKey) return false
-        return !incomingTimestamps.some(
-          ([prop, ts]) => prop === k && signedTimestamps.has(Number(ts)),
-        )
-      })
-
-      if (propertiesWithoutTimestampSigs.length > 0) {
-        const nodeToVerify = {_: node._}
-        for (const prop of propertiesWithoutTimestampSigs) {
-          nodeToVerify[prop] = node[prop]
-        }
-        const perPropertySignedProps = await SEA.verifyProperties(
-          nodeToVerify,
-          pub,
-        )
-        perPropertySignedProps.forEach(prop => signedProperties.add(prop))
-      }
-
       // If no properties verified, skip this update
       if (signedProperties.size === 0) {
         continue
@@ -196,12 +174,8 @@ Ham.mix = async (change, graph, secure, listen) => {
         }
         defer[soul][key] = value
         defer[soul]._[">"][key] = state
-        if (node._["s"]) {
-          if (node._["s"][state]) {
-            defer[soul]._["s"][state] = node._["s"][state]
-          } else if (node._["s"][key]) {
-            defer[soul]._["s"][key] = node._["s"][key]
-          }
+        if (node._["s"] && node._["s"][state]) {
+          defer[soul]._["s"][state] = node._["s"][state]
         }
       } else {
         // Check if this property has a signed timestamp
@@ -217,13 +191,9 @@ Ham.mix = async (change, graph, secure, listen) => {
           }
           graph[soul][key] = now[soul][key] = value
           graph[soul]._[">"][key] = now[soul]._[">"][key] = state
-          if (node._["s"]) {
-            if (node._["s"][state]) {
-              graph[soul]._["s"][state] = now[soul]._["s"][state] =
-                node._["s"][state]
-            } else if (node._["s"][key]) {
-              graph[soul]._["s"][key] = now[soul]._["s"][key] = node._["s"][key]
-            }
+          if (node._["s"] && node._["s"][state]) {
+            graph[soul]._["s"][state] = now[soul]._["s"][state] =
+              node._["s"][state]
           }
           // Collect event listeners to fire after store.put completes.
           // This ensures listeners always fire with data that's been stored.

package/src/holster.js

@@ -132,6 +132,30 @@ const Holster = opt => {
       }
     }
 
+    const watchForRel = (soul, item, ctx, i, request, cb) => {
+      const handler = () => {
+        // Bail out if the context was removed by off().
+        if (!allctx.has(ctxid)) {
+          wire.off({"#": soul, ".": item}, handler)
+          return
+        }
+        wire.get(
+          {"#": soul, ".": item},
+          msg => {
+            const node = msg.put && msg.put[soul]
+            const id = utils.rel.is(node && node[item])
+            if (!id) return
+            wire.off({"#": soul, ".": item}, handler)
+            ctx.chain[i].soul = id
+            allctx.set(ctxid, {...ctx})
+            api(ctxid).on(request.on, cb, request._get, request._opt)
+          },
+          {...request._opt, secure: ctx.user || opt.secure},
+        )
+      }
+      wire.on({"#": soul, ".": item}, handler, false, request._opt)
+    }
+
     const resolve = (request, cb) => {
       if (!request) {
         console.log("error resolve request parameter required")
@@ -201,8 +225,10 @@ const Holster = opt => {
                   api(ctxid).put(request.put, cb)
                 })
               } else if (on) {
-                // Allow listening to a node that doesn't exist.
-                cb(null)
+                // Item is not a rel yet — watch the parent soul for when it
+                // becomes one, then retry chain resolution.
+                watchForRel(soul, item, ctx, i, request, cb)
+                if (request._get) cb(null)
               } else if (off) {
                 // Allow stop listening to a node that doesn't exist.
                 if (cb) cb(null)
@@ -225,8 +251,14 @@ const Holster = opt => {
                 api(ctxid).put(request.put, cb)
               })
             } else {
-              // Allow querying a node that doesn't exist.
-              if (cb) cb(null)
+              if (on) {
+                // Node doesn't exist yet — watch for it to appear.
+                watchForRel(soul, item, ctx, i, request, cb)
+                if (request._get) cb(null)
+              } else {
+                // Allow querying a node that doesn't exist.
+                if (cb) cb(null)
+              }
             }
           },
           {...request._opt, secure: ctx.user || opt.secure},
@@ -521,25 +553,16 @@ const Holster = opt => {
                 return
               }
 
-              // The nested objects created above will also have rels on the
-              // parent object, so fetch the node so that the rest of the
-              // updates can be added.
-              wire.get({"#": id}, async msg => {
-                if (msg.err) {
-                  _ack(`error getting ${id}: ${msg.err}`)
-                  return
-                }
-
-                let node = msg.put && msg.put[id]
-                if (!node) node = {}
-                update.forEach(key => {
-                  node[key] = data[key]
-                })
-                const g = await graph(id, node, ctx.user, _ack)
-                if (g === null) return
-
-                wire.put(g, _ack)
+              // Only write the primitive update keys. Rels created by the
+              // recursive puts above are already in the in-memory graph.
+              const updateNode = {}
+              update.forEach(key => {
+                updateNode[key] = data[key]
               })
+              const g = await graph(id, updateNode, ctx.user, _ack)
+              if (g === null) return
+
+              wire.put(g, _ack)
             },
             {secure: ctx.user || opt.secure},
           )
@@ -579,13 +602,28 @@ const Holster = opt => {
         })
         // Map the user's callback because it can also be passed to off,
         // so need a reference to it to compare them.
-        // Create a new context for each listener invocation to avoid mutation
+        // Shared retry state for this on() subscription.
+        let retryCount = 0
+        const maxRetries = 5
+        const retryDelay = 1000 // Start with 1 second
+        let retryTimer = null
+
         map.set(cb, () => {
+          // Bail out if off() has already cleaned up this context — the mapped
+          // callback can fire twice (once from wire.on and once from the _get
+          // immediate-read path) and the second call after async context cleanup
+          // would crash in resolve(). Matches the guard in watchForRel.
+          if (!allctx.has(ctxid)) return
+          // Cancel any pending retry — the persistent listener firing means we
+          // have an update and should not call cb twice.
+          clearTimeout(retryTimer)
+          retryTimer = null
           // When listener fires, re-check if the item is now a rel.
           wire.get(
             {"#": soul, ".": item},
             msg => {
-              const current = msg.put && msg.put[soul] && msg.put[soul][item]
+              const node = msg.put && msg.put[soul]
+              const current = node && node[item]
               const id = utils.rel.is(current)
               if (id) {
                 // It's a rel, read the related node. It might not be in the
@@ -597,9 +635,10 @@ const Holster = opt => {
                       chain: [{item: null, soul: id}],
                       user: ctx ? ctx.user : null,
                     })
-                    api(_ctxid).next(null, res, _opt)
+                    api(_ctxid).next(null, res, retries === 0 ? utils.obj.put(_opt, "fast", true) : _opt)
                   })
                   if (data !== null || retries >= 5) {
+                    retryCount = 0 // Reset retry counter on success
                     cb(data)
                   } else {
                     // Data not ready, retry after delay
@@ -609,8 +648,46 @@ const Holster = opt => {
                 }
                 attemptRead()
               } else {
-                // It's a direct property or null
-                cb(current !== undefined ? current : null)
+                // It's a direct property or null.
+                const nodeExists = !!node
+
+                if (nodeExists) {
+                  // Node exists — return the value and reset retry state.
+                  retryCount = 0
+                  cb(current !== undefined ? current : null)
+                } else if (retryCount < maxRetries) {
+                  // No data yet — keep polling until we get a response or
+                  // exhaust retries. on() with _get can return null on a cache
+                  // miss and never fire again if the data doesn't change while
+                  // the listener is attached, so we must poll for initial load.
+                  const retry = () => {
+                    const delay = Math.min(retryDelay * Math.pow(2, retryCount), 30000)
+                    retryCount++
+                    retryTimer = setTimeout(() => {
+                      wire.get(
+                        {"#": soul, ".": item},
+                        msg => {
+                          const retryNode = msg.put && msg.put[soul]
+                          if (retryNode) {
+                            retryCount = 0
+                            retryTimer = null
+                            const retryValue = retryNode[item]
+                            cb(retryValue !== undefined ? retryValue : null)
+                          } else if (retryCount < maxRetries) {
+                            retry()
+                          } else {
+                            retryTimer = null
+                            cb(null)
+                          }
+                        },
+                        {..._opt, secure: ctx.user || opt.secure},
+                      )
+                    }, delay)
+                  }
+                  retry()
+                } else {
+                  cb(null)
+                }
               }
             },
             {..._opt, secure: ctx.user || opt.secure},
@@ -653,7 +730,8 @@ const Holster = opt => {
               let relLex
               if (lex) relLex = utils.obj.put(initialLex, "#", id)
               else relLex = {"#": id, ".": null}
-              wire.on(relLex, map.get(cb), _get, _opt)
+              wire.on(relLex, map.get(cb), false, _opt)
+              if (_get) map.get(cb)()
             } else if (_get) {
               // Not a rel, but _get was requested, so trigger callback.
               map.get(cb)()

package/src/sea.js

@@ -238,58 +238,6 @@ const SEA = {
       return false
     }
   },
-  // Verify individual property signatures stored in node._["s"]
-  // Returns array of valid property names
-  verifyProperties: async (node, pub, cb) => {
-    if (!pub || !node) {
-      if (cb) cb([])
-      return []
-    }
-
-    const key = await utils.subtle.importKey(
-      "jwk",
-      utils.jwk(pub),
-      {name: "ECDSA", namedCurve: "P-256"},
-      false,
-      ["verify"],
-    )
-
-    const alg = {name: "ECDSA", hash: {name: "SHA-256"}}
-    const valid = []
-    const propertySignatures = (node._ && node._["s"]) || {}
-
-    // Verify each property individually
-    for (const k of Object.keys(node).sort()) {
-      if (k === "_" || k == userPublicKey) continue
-
-      const propSig = propertySignatures[k]
-      if (!propSig) {
-        continue
-      }
-
-      try {
-        const hash = await utils.sha256(node[k])
-        const sig = new Uint8Array(SafeBuffer.from(propSig, "base64"))
-
-        const isValid = await utils.subtle.verify(
-          alg,
-          key,
-          sig,
-          new Uint8Array(hash),
-        )
-        if (isValid) {
-          valid.push(k)
-        } else {
-          console.log(`warning: property '${k}' signature verification failed`)
-        }
-      } catch (err) {
-        console.log(`warning: property '${k}' error verifying: ${err.message}`)
-      }
-    }
-
-    if (cb) cb(valid)
-    return valid
-  },
   work: async (data, salt, cb) => {
     if (typeof salt === "function") {
       cb = salt

package/src/store.js

@@ -112,6 +112,7 @@ const fileSystem = opt => {
           const req = tx.objectStore(dir).get(file)
           req.onerror = () => {
             console.log(`error getting ${dir}/${file}`)
+            cb(req.error)
           }
           req.onsuccess = () => {
             cb(null, req.result)
@@ -132,6 +133,7 @@ const fileSystem = opt => {
           const req = tx.objectStore(dir).put(data, file)
           req.onerror = () => {
             console.log(`error putting data on ${dir}/${file}`)
+            cb(req.error)
           }
           req.onsuccess = () => {
             cb(null)
@@ -150,7 +152,10 @@ const fileSystem = opt => {
         const _list = cb => {
           const tx = db.transaction([dir], "readonly")
           const req = tx.objectStore(dir).getAllKeys()
-          req.onerror = () => console.log("error getting keys for", dir)
+          req.onerror = () => {
+            console.log("error getting keys for", dir)
+            cb()
+          }
           req.onsuccess = () => {
             req.result.forEach(cb)
             cb()
@@ -217,12 +222,7 @@ const Store = opt => {
         node._[">"][key] = value[1]
         // If signature is present, store it in _["s"]
         if (value.length === 3 && value[2]) {
-          const state = value[1]
-          // Store signature under both state and key for compatibility
-          signatures[state] = value[2]
-          // TODO: Can probably remove the following when old data is no
-          // longer a problem.
-          signatures[key] = value[2]
+          signatures[value[1]] = value[2]
         }
       }
 

package/src/wire.js

@@ -316,7 +316,7 @@ const Wire = opt => {
 
     if (!utils.obj.is(_opt)) _opt = {}
 
-    const ack = Get(lex, graph)
+    const ack = Get(lex, graph, _opt.fast)
     const track = utils.text.random(9)
     const request = JSON.stringify({
       "#": dup.track(track),
@@ -338,13 +338,19 @@ const Wire = opt => {
       lex,
       (err, ack) => {
         if (ack) {
+          // Populate in-memory graph from store data before sending the server
+          // request, so stale server responses with older state are correctly
+          // rejected by ham.mix.
+          for (const [soul, node] of Object.entries(ack)) {
+            if (!graph[soul] && node) graph[soul] = {...node}
+          }
           // Also send request on the wire to check for updates.
           const sendResult = send(request)
           if (sendResult && sendResult.err) {
             cb({err: sendResult.err})
             return
           }
-          cb({put: ack, err: err})
+          cb(err ? {put: ack, err: err} : {put: ack})
           return
         }
 

package/ts/src/array.ts

@@ -0,0 +1,78 @@
+/**
+ * SeaArray - Extended Array with custom toString for encoding support
+ * Supports utf8, hex, and base64 encoding
+ */
+
+if (typeof btoa === "undefined") {
+  globalThis.btoa = (data: string): string =>
+    Buffer.from(data, "binary").toString("base64")
+  globalThis.atob = (data: string): string =>
+    Buffer.from(data, "base64").toString("binary")
+}
+
+type Encoding = "utf8" | "hex" | "base64"
+
+/**
+ * SeaArray extends Array to provide encoding capabilities
+ */
+class SeaArray extends Array<number> {
+  /**
+   * Convert array to string with specified encoding
+   * @param enc - Encoding type (utf8, hex, or base64)
+   * @param start - Start index (default: 0)
+   * @param end - End index (default: array length)
+   */
+  override toString(enc?: Encoding, start?: number, end?: number): string {
+    const encoding = enc || "utf8"
+    const startIdx = start || 0
+    const endIdx = end ? Math.min(Math.max(end, startIdx), this.length) : this.length
+
+    if (encoding === "hex") {
+      const buf = new Uint8Array(this.slice(startIdx, endIdx))
+      return Array.from(buf, byte => byte.toString(16).padStart(2, "0")).join("")
+    }
+
+    if (encoding === "utf8") {
+      return Array.from({ length: endIdx - startIdx }, (_, i) => {
+        const charCode = this[i + startIdx]
+        // Use unicode replacement character for invalid character codes
+        if (
+          charCode === undefined ||
+          charCode < 0 ||
+          charCode > 0x10ffff
+        ) {
+          return String.fromCharCode(0xfffd)
+        }
+        return String.fromCharCode(charCode)
+      }).join("")
+    }
+
+    if (encoding === "base64") {
+      const utf8String = Array.from({ length: endIdx - startIdx }, (_, i) => {
+        const charCode = this[i + startIdx]
+        if (charCode === undefined || charCode < 0 || charCode > 255) {
+          return String.fromCharCode(0xfffd)
+        }
+        return String.fromCharCode(charCode)
+      }).join("")
+      return btoa(utf8String)
+    }
+
+    // Default to utf8
+    return this.toString("utf8", startIdx, endIdx)
+  }
+
+  /**
+   * Create SeaArray from iterable or array-like
+   */
+  static override from<T>(
+    arrayLike: ArrayLike<T> | Iterable<T>
+  ): SeaArray {
+    const arr = Array.from(arrayLike) as number[]
+    const seaArray = new SeaArray(...arr)
+    return seaArray
+  }
+}
+
+export default SeaArray
+