@mblaney/holster 2.1.4 → 2.2.1

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@mblaney/holster",
-  "version": "2.1.4",
+  "version": "2.2.1",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",
   "scripts": {
-    "test": "node --test",
+    "test": "node --test test/*.test.js test/system/*.test.js",
     "test:ts": "node --test --import tsx/esm ts/test/*.test.ts ts/test/system/*.test.ts",
     "build": "esbuild src/holster.js --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=build",
     "build:ts": "esbuild ts/src/holster.ts --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=dist",

package/src/holster.js

@@ -2,6 +2,7 @@ import * as utils from "./utils.js"
 import Wire from "./wire.js"
 import User from "./user.js"
 import SEA from "./sea.js"
+import {attachNested} from "./nested.js"
 
 const Holster = opt => {
   if (typeof opt === "string") opt = {peers: [opt]}
@@ -14,6 +15,12 @@ const Holster = opt => {
   const map = new Map()
   // Allow concurrent calls to the api by storing each context.
   const allctx = new Map()
+  // Tracks on(..., {nested: true}) subscriptions, keyed by the caller's
+  // own callback (the same reference off() will be called with) - see
+  // attachNested in nested.js. The actual top-level/child listeners it
+  // creates use their own, separate contexts, so off() just needs to
+  // call the detach function this stores to tear the whole thing down.
+  const nested = new Map()
   // Serializes concurrent creation of a missing rel for the same
   // soul+item, so only the first caller creates it and the rest reuse
   // its result rather than each minting a competing soul. Map<soul,
@@ -58,6 +65,25 @@ const Holster = opt => {
     return `error ${error} cannot be converted to a graph`
   }
 
+  // Reads the current data at a rel's target soul, retrying briefly since a
+  // freshly created rel's target can still be mid-flight when first read.
+  // Shared by a one-off get() resolving rels found on the returned node,
+  // and by on()'s live retry when a watched property turns out to be a rel.
+  const readRelTarget = async (id, ctxUser, _opt, retries = 0) => {
+    const data = await new Promise(res => {
+      const _ctxid = utils.text.random()
+      allctx.set(_ctxid, {chain: [{item: null, soul: id}], user: ctxUser})
+      api(_ctxid).next(
+        null,
+        res,
+        retries === 0 ? utils.obj.put(_opt, "fast", true) : _opt,
+      )
+    })
+    if (data !== null || retries >= 5) return data
+    await new Promise(resolve => setTimeout(resolve, 50))
+    return readRelTarget(id, ctxUser, _opt, retries + 1)
+  }
+
   const api = ctxid => {
     const get = (lex, soul, ack, _opt) => {
       wire.get(
@@ -73,24 +99,13 @@ const Holster = opt => {
               Object.keys(msg.put[soul]).map(async key => {
                 const id = utils.rel.is(msg.put[soul][key])
                 if (!id) return
-                const attemptRead = async (retries = 0) => {
-                  const data = await new Promise(res => {
-                    const _ctxid = utils.text.random()
-                    const ctx = allctx.get(ctxid)
-                    allctx.set(_ctxid, {
-                      chain: [{item: null, soul: id}],
-                      user: ctx ? ctx.user : null,
-                    })
-                    api(_ctxid).next(null, res, _opt)
-                  })
-                  if (data !== null || retries >= 5) {
-                    return data
-                  }
-                  // Data not ready, retry after delay
-                  await new Promise(resolve => setTimeout(resolve, 50))
-                  return attemptRead(retries + 1)
-                }
-                msg.put[soul][key] = await attemptRead()
+
+                const ctx = allctx.get(ctxid)
+                msg.put[soul][key] = await readRelTarget(
+                  id,
+                  ctx ? ctx.user : null,
+                  _opt,
+                )
               }),
             )
             ack(msg.put[soul])
@@ -680,6 +695,23 @@ const Holster = opt => {
         // Get the context to check if it has user info
         const ctx = allctx.get(ctxid)
 
+        if (_opt && _opt.nested) {
+          // attachNested calls this once for the top level and again for
+          // every discovered child, each needing its own independent
+          // context - a single shared _ctxid would accumulate .next()
+          // calls onto the same path instead of giving sibling paths.
+          const factory = () => {
+            const _ctxid = utils.text.random()
+            allctx.set(_ctxid, {
+              chain: [{item: item, soul: soul}],
+              user: ctx ? ctx.user : null,
+            })
+            return api(_ctxid)
+          }
+          nested.set(cb, attachNested(factory, cb, _get, _opt))
+          return
+        }
+
         // Flag that this context is set from on and shouldn't be removed.
         allctx.set(ctxid, {
           chain: [{item: item, soul: soul}],
@@ -689,10 +721,19 @@ const Holster = opt => {
         // Map the user's callback because it can also be passed to off,
         // so need a reference to it to compare them.
         // Shared retry state for this on() subscription.
-        let retryCount = 0
         const maxRetries = 5
         const retryDelay = 1000 // Start with 1 second
         let retryTimer = null
+        // Which soul the persistent wire listener is currently attached to
+        // - starts on the parent (soul/item), and moves to a rel's target
+        // soul once item turns out to be one, since further updates land
+        // on the target's own soul, not the parent's. Tracked (rather than
+        // switching unconditionally) so the two places that can discover a
+        // rel - the initial check below and a later resolveValue retry -
+        // don't both re-register the same listener. Scoped to this one
+        // on() call's closure, so concurrent subscriptions (even on the
+        // same key) each track their own independently.
+        let listenedSoul = soul
 
         map.set(cb, () => {
           // Bail out if off() has already cleaned up this context — the mapped
@@ -704,84 +745,75 @@ const Holster = opt => {
           // have an update and should not call cb twice.
           clearTimeout(retryTimer)
           retryTimer = null
-          // When listener fires, re-check if the item is now a rel.
+
+          // Resolves {soul, item} to a genuine value, following a rel to its
+          // target if the property turns out to be one - checked fresh on
+          // every attempt, not just the first, since a property can still
+          // be a plain undefined/null on an early check and only become a
+          // rel once a later retry catches up (e.g. a nested rel-creating
+          // put both adds "item" as a rel on this node AND populates its
+          // target, and either half can still be mid-flight when the
+          // listener first fires). wire.get() can't reliably tell
+          // "genuinely null" apart from "hasn't landed yet" either - its
+          // own internal timeout fallback explicitly answers null when
+          // nothing comes back in time - so a null/missing value is
+          // retried a few times before being trusted as final, the same
+          // way the rel branch below already retries on a null target
+          // read.
+          const resolveValue = (retries, node, value) => {
+            const id = utils.rel.is(value)
+            if (id) {
+              if (listenedSoul !== id) {
+                wire.off({"#": listenedSoul}, map.get(cb))
+                wire.on({"#": id, ".": null}, map.get(cb), false, _opt)
+                listenedSoul = id
+              }
+              readRelTarget(id, ctx ? ctx.user : null, _opt).then(data => {
+                retryTimer = null
+                cb(data)
+              })
+              return
+            }
+
+            if (value !== undefined && value !== null) {
+              retryTimer = null
+              cb(value)
+              return
+            }
+            if (retries >= maxRetries) {
+              retryTimer = null
+              cb(value !== undefined ? value : null)
+              return
+            }
+            // Fast once the parent node already exists (a value is likely
+            // imminent); slow/exponential when it doesn't, since there's
+            // no signal anything is coming soon and fast polling would
+            // just waste requests.
+            const delay = node
+              ? 50
+              : Math.min(retryDelay * Math.pow(2, retries), 30000)
+            retryTimer = setTimeout(() => {
+              wire.get(
+                {"#": soul, ".": item},
+                msg => {
+                  const retryNode = msg.put && msg.put[soul]
+                  resolveValue(
+                    retries + 1,
+                    retryNode,
+                    retryNode && retryNode[item],
+                  )
+                },
+                {..._opt, secure: !!(ctx.user || opt.secure)},
+              )
+            }, delay)
+          }
+
+          // When listener fires, re-check the item's current state.
           wire.get(
             {"#": soul, ".": item},
             msg => {
               const node = msg.put && msg.put[soul]
-              const current = node && node[item]
-              const id = utils.rel.is(current)
-              if (id) {
-                // It's a rel, read the related node. It might not be in the
-                // graph yet when the listener fires, so we retry with delays.
-                const attemptRead = async (retries = 0) => {
-                  const data = await new Promise(res => {
-                    const _ctxid = utils.text.random()
-                    allctx.set(_ctxid, {
-                      chain: [{item: null, soul: id}],
-                      user: ctx ? ctx.user : null,
-                    })
-                    api(_ctxid).next(
-                      null,
-                      res,
-                      retries === 0 ? utils.obj.put(_opt, "fast", true) : _opt,
-                    )
-                  })
-                  if (data !== null || retries >= 5) {
-                    retryCount = 0 // Reset retry counter on success
-                    cb(data)
-                  } else {
-                    // Data not ready, retry after delay
-                    await new Promise(resolve => setTimeout(resolve, 50))
-                    attemptRead(retries + 1)
-                  }
-                }
-                attemptRead()
-              } else {
-                // It's a direct property or null.
-                const nodeExists = !!node
-
-                if (nodeExists) {
-                  // Node exists — return the value and reset retry state.
-                  retryCount = 0
-                  cb(current !== undefined ? current : null)
-                } else if (retryCount < maxRetries) {
-                  // No data yet — keep polling until we get a response or
-                  // exhaust retries. on() with _get can return null on a cache
-                  // miss and never fire again if the data doesn't change while
-                  // the listener is attached, so we must poll for initial load.
-                  const retry = () => {
-                    const delay = Math.min(
-                      retryDelay * Math.pow(2, retryCount),
-                      30000,
-                    )
-                    retryCount++
-                    retryTimer = setTimeout(() => {
-                      wire.get(
-                        {"#": soul, ".": item},
-                        msg => {
-                          const retryNode = msg.put && msg.put[soul]
-                          if (retryNode) {
-                            retryCount = 0
-                            retryTimer = null
-                            const retryValue = retryNode[item]
-                            cb(retryValue !== undefined ? retryValue : null)
-                          } else if (retryCount < maxRetries) {
-                            retry()
-                          } else {
-                            retryTimer = null
-                            cb(null)
-                          }
-                        },
-                        {..._opt, secure: !!(ctx.user || opt.secure)},
-                      )
-                    }, delay)
-                  }
-                  retry()
-                } else {
-                  cb(null)
-                }
-              }
+              resolveValue(0, node, node && node[item])
             },
             {..._opt, secure: !!(ctx.user || opt.secure)},
           )
@@ -816,11 +848,14 @@ const Holster = opt => {
             const current = msg.put && msg.put[soul] && msg.put[soul][item]
             const id = utils.rel.is(current)
             if (id) {
-              // It's a rel, need to switch listener to the related node
-              // First remove the initial listener
-              wire.off(initialLex, map.get(cb))
-              // Then add listener on the related node
-              wire.on({"#": id, ".": null}, map.get(cb), false, _opt)
+              // It's a rel, need to switch listener to the related node -
+              // unless resolveValue (triggered by the initial listener
+              // firing) already beat this check to it.
+              if (listenedSoul !== id) {
+                wire.off({"#": listenedSoul}, map.get(cb))
+                wire.on({"#": id, ".": null}, map.get(cb), false, _opt)
+                listenedSoul = id
+              }
               if (_get) map.get(cb)()
             } else if (_get) {
               // Not a rel, but _get was requested, so trigger callback.
@@ -837,6 +872,12 @@ const Holster = opt => {
           return
         }
 
+        if (nested.has(cb)) {
+          nested.get(cb)()
+          nested.delete(cb)
+          return
+        }
+
         // Resolve the current context before removing event listener.
         const {item, soul} = resolve({off: true}, cb)
         if (!soul) return

package/src/nested.js

@@ -0,0 +1,89 @@
+// Backs on(cb, _get, {nested: true}) - see holster.js. A plain on() only
+// resolves the node it's given; it never re-checks a child whose own
+// value is an object (which always has its own soul, since only scalars
+// live directly on a node) once that child's content changes in place -
+// nothing on the parent's own soul changes when that happens, so a
+// listener on the parent alone never sees it. get()'s own read-resolution
+// (holster.js) already recursively resolves every such child into plain
+// data for a one-time read - this doesn't repeat that; it adds a live
+// subscription for each child that resolution already exposed, so a
+// later change to it reaches the caller too.
+//
+// Pure and holster-internals-free: chain is a factory - () =>
+// user.get([host, "shared"]).next("domains").next(code) - not a
+// pre-built chained object, since a fresh, independent instance is
+// needed per discovered child (reusing one chained object across
+// multiple .next() calls would push them onto the same path instead of
+// giving sibling paths).
+export function attachNested(chain, cb, _get, _opt) {
+  // Stripped before being used on any .on() call this function makes
+  // itself (top-level or child) - otherwise a caller-provided {nested:
+  // true} would re-trigger holster.js's own nested check on re-entry,
+  // before the normal rel-resolution logic that actually finds the right
+  // soul ever runs.
+  const opt = {..._opt}
+  delete opt.nested
+
+  const childListeners = new Map() // key -> {api, handler}
+  let latest = null
+
+  const notify = () => cb(latest === null ? null : {...latest})
+
+  const topLevelHandler = data => {
+    if (data === null || typeof data !== "object") {
+      latest = data
+      notify()
+      return
+    }
+
+    if (!latest || typeof latest !== "object") latest = {}
+    for (const [key, value] of Object.entries(data)) {
+      if (key === "_") continue
+      latest[key] = value
+
+      if (
+        value !== null &&
+        typeof value === "object" &&
+        !childListeners.has(key)
+      ) {
+        const childApi = chain().next(key)
+        const handler = childData => {
+          if (childData === null) {
+            delete latest[key]
+            childListeners.delete(key)
+          } else {
+            latest[key] = childData
+          }
+          notify()
+        }
+        childListeners.set(key, {api: childApi, handler})
+        // false, not true - the current value is already in data above,
+        // so this only needs to watch for future changes, not trigger a
+        // redundant immediate re-read.
+        childApi.on(handler, false, opt)
+      }
+    }
+
+    for (const key of [...childListeners.keys()]) {
+      if (!(key in data)) {
+        const {api: childApi, handler} = childListeners.get(key)
+        childApi.off(handler)
+        childListeners.delete(key)
+        delete latest[key]
+      }
+    }
+
+    notify()
+  }
+
+  const topLevelApi = chain()
+  topLevelApi.on(topLevelHandler, _get, opt)
+
+  return () => {
+    topLevelApi.off(topLevelHandler)
+    for (const {api: childApi, handler} of childListeners.values()) {
+      childApi.off(handler)
+    }
+    childListeners.clear()
+  }
+}

package/src/wire.js

@@ -452,14 +452,18 @@ const Wire = opt => {
           continue
         }
         const limit = userLimits.get(pub) ?? defaultLimit
-        if ((userStorage.get(pub) ?? 0) >= limit) continue
-        toStore[soul] = node
+        const current = userStorage.get(pub) ?? 0
         const oldBytes = oldGraphBytes[soul] ?? 0
         const newBytes = graph[soul] ? JSON.stringify(graph[soul]).length : 0
-        userStorage.set(
-          pub,
-          Math.max(0, (userStorage.get(pub) ?? 0) - oldBytes + newBytes),
-        )
+        const updated = Math.max(0, current - oldBytes + newBytes)
+        // A write that doesn't grow this user's total usage (e.g. a
+        // delete) is always let through, even past the limit - otherwise
+        // a user already over quota could never delete anything to get
+        // back under it. Only a write that both grows usage and pushes
+        // the total past the limit is rejected.
+        if (updated > current && updated > limit) continue
+        toStore[soul] = node
+        userStorage.set(pub, updated)
         saveUserStorage()
       }
       if (Object.keys(toStore).length > 0) {
@@ -683,16 +687,22 @@ const Wire = opt => {
             if (!pub) continue
             const limit = userLimits.get(pub) ?? defaultLimit
             const current = userStorage.get(pub) ?? 0
-            if (current >= limit) {
-              if (cb) cb("storage limit exceeded")
-              return
-            }
-
             const oldBytes = oldGraphBytes[soul] ?? 0
             const newBytes = graph[soul]
               ? JSON.stringify(graph[soul]).length
               : 0
-            userStorage.set(pub, Math.max(0, current - oldBytes + newBytes))
+            const updated = Math.max(0, current - oldBytes + newBytes)
+            // A write that doesn't grow this user's total usage (e.g. a
+            // delete) is always let through, even past the limit - otherwise
+            // a user already over quota could never delete anything to get
+            // back under it. Only a write that both grows usage and pushes
+            // the total past the limit is rejected.
+            if (updated > current && updated > limit) {
+              if (cb) cb("storage limit exceeded")
+              return
+            }
+
+            userStorage.set(pub, updated)
             saveUserStorage()
           }
         }
@@ -1048,6 +1058,19 @@ const Wire = opt => {
   }
 
   const send = data => {
+    // Reject client-side, before ever attempting to send, rather than
+    // silently succeeding locally while the receiving peer's own
+    // validateMessage (wire.js's server-side inbound check) drops it and
+    // replies with an error nothing here is listening for - so a save
+    // that's too big to ever sync would otherwise look like it worked.
+    const maxMessageSize = opt.maxMessageSize || 1024 * 1024
+    if (data.length > maxMessageSize) {
+      return {
+        err: `Message too large (${data.length} > ${maxMessageSize} bytes). Reduce the size of the data being saved.`,
+        size: data.length,
+        maxMessageSize,
+      }
+    }
     if (messageQueue.length >= maxQueueLength) {
       return {
         err: `Message queue exceeded maximum length (${maxQueueLength}). Update query logic to request less data.`,