@mblaney/holster 2.1.4 → 2.2.1

package/ts/src/holster.ts

@@ -7,6 +7,7 @@ import * as utils from "./utils.ts"
 import Wire, {type WireAPI} from "./wire.ts"
 import User, {type UserInterface} from "./user.ts"
 import SEA from "./sea.ts"
+import {attachNested} from "./nested.ts"
 import type {
   HolsterOptions,
   ChainItem,
@@ -55,12 +56,15 @@ export interface HolsterAPI {
     set?: boolean | ((data?: string | null) => void),
     cb?: (data?: string | null) => void,
   ) => HolsterAPI | void
-  on: (
-    lex: LexWithDot | ((data: unknown) => void),
-    cb?: ((data: unknown) => void) | boolean,
-    _get?: boolean,
-    _opt?: WireOptions,
-  ) => void
+  on: {
+    (
+      lex: LexWithDot,
+      cb: (data: unknown) => void,
+      _get?: boolean,
+      _opt?: WireOptions,
+    ): void
+    (cb: (data: unknown) => void, _get?: boolean, _opt?: WireOptions): void
+  }
   off: (cb?: (data: unknown) => void) => HolsterAPI | void
   user: () => UserInterface & HolsterAPI
   wire: WireAPI
@@ -78,6 +82,12 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
   const user = User(options, wire as never)
   const map = new Map<(data: unknown) => void, () => void>()
   const allctx = new Map<string, ApiContext>()
+  // Tracks on(..., {nested: true}) subscriptions, keyed by the caller's
+  // own callback (the same reference off() will be called with) - see
+  // attachNested in nested.ts. The actual top-level/child listeners it
+  // creates use their own, separate contexts, so off() just needs to
+  // call the detach function this stores to tear the whole thing down.
+  const nested = new Map<(data: unknown) => void, () => void>()
   // Serializes concurrent creation of a missing rel for the same
   // soul+item, so only the first caller creates it and the rest reuse
   // its result rather than each minting a competing soul. Map<soul,
@@ -122,6 +132,30 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
     return `error ${error} cannot be converted to a graph`
   }
 
+  // Reads the current data at a rel's target soul, retrying briefly since a
+  // freshly created rel's target can still be mid-flight when first read.
+  // Shared by a one-off get() resolving rels found on the returned node,
+  // and by on()'s live retry when a watched property turns out to be a rel.
+  const readRelTarget = async (
+    id: string,
+    ctxUser: UserIdentity | null | undefined,
+    opt: WireOptions | undefined,
+    retries = 0,
+  ): Promise<unknown> => {
+    const data = await new Promise<unknown>(res => {
+      const _ctxid = utils.text.random()
+      allctx.set(_ctxid, {chain: [{item: null, soul: id}], user: ctxUser})
+      api(_ctxid).next(
+        null as never,
+        res as never,
+        (retries === 0 ? utils.obj.put(opt, "fast", true) : opt) as never,
+      )
+    })
+    if (data !== null || retries >= 5) return data
+    await new Promise(resolve => setTimeout(resolve, 50))
+    return readRelTarget(id, ctxUser, opt, retries + 1)
+  }
+
   const api = (initCtxid?: string): HolsterAPI => {
     let ctxid = initCtxid
     const get = (
@@ -147,23 +181,12 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
               Object.keys(node).map(async key => {
                 const id = utils.rel.is(node[key] as GraphValue)
                 if (!id) return
-                const attemptRead = async (retries = 0): Promise<unknown> => {
-                  const data = await new Promise<unknown>(res => {
-                    const _ctxid = utils.text.random()
-                    const ctx = allctx.get(ctxid!)
-                    allctx.set(_ctxid, {
-                      chain: [{item: null, soul: id}],
-                      user: ctx ? ctx.user : null,
-                    })
-                    api(_ctxid).next(null as never, res as never, _opt)
-                  })
-                  if (data !== null || retries >= 5) {
-                    return data
-                  }
-                  await new Promise(resolve => setTimeout(resolve, 50))
-                  return attemptRead(retries + 1)
-                }
-                node[key] = (await attemptRead()) as GraphValue
+                const ctx = allctx.get(ctxid!)
+                node[key] = (await readRelTarget(
+                  id,
+                  ctx ? ctx.user : null,
+                  _opt,
+                )) as GraphValue
               }),
             )
             ack(msg.put[soul])
@@ -884,6 +907,23 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
 
         const ctx = allctx.get(ctxid)
 
+        if (opts && opts.nested) {
+          // attachNested calls this once for the top level and again for
+          // every discovered child, each needing its own independent
+          // context - a single shared _ctxid would accumulate .next()
+          // calls onto the same path instead of giving sibling paths.
+          const factory = (): HolsterAPI => {
+            const _ctxid = utils.text.random()
+            allctx.set(_ctxid, {
+              chain: [{item: item, soul: soul}],
+              user: ctx ? ctx.user : null,
+            })
+            return api(_ctxid)
+          }
+          nested.set(callback, attachNested(factory, callback, _get, opts))
+          return
+        }
+
         allctx.set(ctxid, {
           chain: [{item: item, soul: soul}],
           on: true,
@@ -891,10 +931,19 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
         })
 
         // Shared retry state for this on() subscription.
-        let retryCount = 0
         const maxRetries = 5
         const retryDelay = 1000
         let retryTimer: ReturnType<typeof setTimeout> | null = null
+        // Which soul the persistent wire listener is currently attached to
+        // - starts on the parent (soul/item), and moves to a rel's target
+        // soul once item turns out to be one, since further updates land
+        // on the target's own soul, not the parent's. Tracked (rather than
+        // switching unconditionally) so the two places that can discover a
+        // rel - the initial check below and a later resolveValue retry -
+        // don't both re-register the same listener. Scoped to this one
+        // on() call's closure, so concurrent subscriptions (even on the
+        // same key) each track their own independently.
+        let listenedSoul = soul
 
         map.set(callback, () => {
           // Bail out if off() has already cleaned up this context — the mapped
@@ -906,89 +955,89 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
           // have an update and should not call callback twice.
           clearTimeout(retryTimer!)
           retryTimer = null
+
+          // Resolves {soul, item} to a genuine value, following a rel to its
+          // target if the property turns out to be one - checked fresh on
+          // every attempt, not just the first, since a property can still
+          // be a plain undefined/null on an early check and only become a
+          // rel once a later retry catches up (e.g. a nested rel-creating
+          // put both adds "item" as a rel on this node AND populates its
+          // target, and either half can still be mid-flight when the
+          // listener first fires). wire.get() can't reliably tell
+          // "genuinely null" apart from "hasn't landed yet" either - its
+          // own internal timeout fallback explicitly answers null when
+          // nothing comes back in time - so a null/missing value is
+          // retried a few times before being trusted as final, the same
+          // way the rel branch below already retries on a null target
+          // read.
+          const resolveValue = (
+            retries: number,
+            node: Record<string, GraphValue> | null | undefined,
+            value: unknown,
+          ): void => {
+            const id = utils.rel.is(value as GraphValue)
+            if (id) {
+              if (listenedSoul !== id) {
+                wire.off({"#": listenedSoul} as never, map.get(callback)!)
+                wire.on(
+                  {"#": id, ".": null} as never,
+                  map.get(callback)!,
+                  false,
+                  opts,
+                )
+                listenedSoul = id
+              }
+              readRelTarget(id, ctx ? ctx.user : null, opts).then(data => {
+                retryTimer = null
+                callback(data)
+              })
+              return
+            }
+
+            if (value !== undefined && value !== null) {
+              retryTimer = null
+              callback(value)
+              return
+            }
+            if (retries >= maxRetries) {
+              retryTimer = null
+              callback(value !== undefined ? value : null)
+              return
+            }
+            // Fast once the parent node already exists (a value is likely
+            // imminent); slow/exponential when it doesn't, since there's
+            // no signal anything is coming soon and fast polling would
+            // just waste requests.
+            const delay = node
+              ? 50
+              : Math.min(retryDelay * Math.pow(2, retries), 30000)
+            retryTimer = setTimeout(() => {
+              wire.get(
+                {"#": soul, ".": item!},
+                retryMsg => {
+                  const retryNode = retryMsg.put && retryMsg.put[soul]
+                  resolveValue(
+                    retries + 1,
+                    retryNode,
+                    retryNode && retryNode[item!],
+                  )
+                },
+                {
+                  ...opts,
+                  secure: ctx
+                    ? (typeof ctx.user === "boolean" ? ctx.user : !!ctx.user) ||
+                      options.secure
+                    : options.secure,
+                },
+              )
+            }, delay)
+          }
+
           wire.get(
             {"#": soul, ".": item!},
             msg => {
               const node = msg.put && msg.put[soul]
-              const current = node && node[item!]
-              const id = utils.rel.is(current as GraphValue)
-              if (id) {
-                const attemptRead = async (retries = 0): Promise<void> => {
-                  const data = await new Promise<unknown>(res => {
-                    const _ctxid = utils.text.random()
-                    allctx.set(_ctxid, {
-                      chain: [{item: null, soul: id}],
-                      user: ctx ? ctx.user : null,
-                    })
-                    api(_ctxid).next(
-                      null as never,
-                      res as never,
-                      (retries === 0
-                        ? utils.obj.put(opts, "fast", true)
-                        : opts) as never,
-                    )
-                  })
-                  if (data !== null || retries >= 5) {
-                    retryCount = 0
-                    callback(data)
-                  } else {
-                    await new Promise(resolve => setTimeout(resolve, 50))
-                    attemptRead(retries + 1)
-                  }
-                }
-                attemptRead()
-              } else {
-                const nodeExists = !!node
-                if (nodeExists) {
-                  retryCount = 0
-                  callback(current !== undefined ? current : null)
-                } else if (retryCount < maxRetries) {
-                  // No data yet — keep polling until we get a response or
-                  // exhaust retries. on() with _get can return null on a cache
-                  // miss and never fire again if the data doesn't change while
-                  // the listener is attached, so we must poll for initial load.
-                  const secureOpt = {
-                    ...opts,
-                    secure: ctx
-                      ? (typeof ctx.user === "boolean"
-                          ? ctx.user
-                          : !!ctx.user) || options.secure
-                      : options.secure,
-                  }
-                  const retry = (): void => {
-                    const delay = Math.min(
-                      retryDelay * Math.pow(2, retryCount),
-                      30000,
-                    )
-                    retryCount++
-                    retryTimer = setTimeout(() => {
-                      wire.get(
-                        {"#": soul, ".": item!},
-                        retryMsg => {
-                          const retryNode = retryMsg.put && retryMsg.put[soul]
-                          if (retryNode) {
-                            retryCount = 0
-                            retryTimer = null
-                            const retryValue = retryNode[item!]
-                            callback(
-                              retryValue !== undefined ? retryValue : null,
-                            )
-                          } else if (retryCount < maxRetries) {
-                            retry()
-                          } else {
-                            retryTimer = null
-                            callback(null)
-                          }
-                        },
-                        secureOpt,
-                      )
-                    }, delay)
-                  }
-                  retry()
-                } else {
-                  callback(null)
-                }
-              }
+              resolveValue(0, node, node && node[item!])
             },
             {
               ...opts,
@@ -1022,13 +1071,19 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
             const current = msg.put && msg.put[soul] && msg.put[soul]![item!]
             const id = utils.rel.is(current as GraphValue)
             if (id) {
-              wire.off(initialLex as never, map.get(callback)!)
-              wire.on(
-                {"#": id, ".": null} as never,
-                map.get(callback)!,
-                false,
-                opts,
-              )
+              // It's a rel, need to switch listener to the related node -
+              // unless resolveValue (triggered by the initial listener
+              // firing) already beat this check to it.
+              if (listenedSoul !== id) {
+                wire.off({"#": listenedSoul} as never, map.get(callback)!)
+                wire.on(
+                  {"#": id, ".": null} as never,
+                  map.get(callback)!,
+                  false,
+                  opts,
+                )
+                listenedSoul = id
+              }
               if (_get) map.get(callback)!()
             } else if (_get) {
               // Not a rel, but _get was requested, so trigger callback.
@@ -1043,7 +1098,7 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
               : options.secure,
           },
         )
-      },
+      } as HolsterAPI["on"],
 
       off: function (cb?: (data: unknown) => void): HolsterAPI | void {
         if (!ctxid) {
@@ -1052,6 +1107,12 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
           return
         }
 
+        if (cb && nested.has(cb)) {
+          nested.get(cb)!()
+          nested.delete(cb)
+          return
+        }
+
         const resolved = resolve({off: true}, cb)
         if (!resolved) return
 

package/ts/src/nested.ts

@@ -0,0 +1,101 @@
+import type {HolsterAPI} from "./holster.ts"
+import type {WireOptions} from "./schemas.ts"
+
+// Backs on(cb, _get, {nested: true}) - see holster.ts. A plain on() only
+// resolves the node it's given; it never re-checks a child whose own
+// value is an object (which always has its own soul, since only scalars
+// live directly on a node) once that child's content changes in place -
+// nothing on the parent's own soul changes when that happens, so a
+// listener on the parent alone never sees it. get()'s own read-resolution
+// (holster.ts) already recursively resolves every such child into plain
+// data for a one-time read - this doesn't repeat that; it adds a live
+// subscription for each child that resolution already exposed, so a
+// later change to it reaches the caller too.
+//
+// Pure and holster-internals-free: chain is a factory - () =>
+// user.get([host, "shared"]).next("domains").next(code) - not a
+// pre-built chained object, since a fresh, independent instance is
+// needed per discovered child (reusing one chained object across
+// multiple .next() calls would push them onto the same path instead of
+// giving sibling paths).
+export function attachNested(
+  chain: () => HolsterAPI,
+  cb: (data: unknown) => void,
+  _get?: boolean,
+  _opt?: WireOptions,
+): () => void {
+  // Stripped before being used on any .on() call this function makes
+  // itself (top-level or child) - otherwise a caller-provided {nested:
+  // true} would re-trigger holster.ts's own nested check on re-entry,
+  // before the normal rel-resolution logic that actually finds the right
+  // soul ever runs.
+  const opt: WireOptions = {..._opt}
+  delete opt.nested
+
+  const childListeners = new Map<
+    string,
+    {api: HolsterAPI; handler: (data: unknown) => void}
+  >()
+  let latest: Record<string, unknown> | null = null
+
+  const notify = () => cb(latest === null ? null : {...latest})
+
+  const topLevelHandler = (data: unknown) => {
+    if (data === null || typeof data !== "object") {
+      latest = data as Record<string, unknown> | null
+      notify()
+      return
+    }
+
+    if (!latest || typeof latest !== "object") latest = {}
+    const node = data as Record<string, unknown>
+    for (const [key, value] of Object.entries(node)) {
+      if (key === "_") continue
+      latest[key] = value
+
+      if (
+        value !== null &&
+        typeof value === "object" &&
+        !childListeners.has(key)
+      ) {
+        const childApi = chain().next(key)
+        const handler = (childData: unknown) => {
+          if (childData === null) {
+            delete latest![key]
+            childListeners.delete(key)
+          } else {
+            latest![key] = childData
+          }
+          notify()
+        }
+        childListeners.set(key, {api: childApi, handler})
+        // false, not true - the current value is already in data above,
+        // so this only needs to watch for future changes, not trigger a
+        // redundant immediate re-read.
+        childApi.on(handler, false, opt)
+      }
+    }
+
+    for (const key of [...childListeners.keys()]) {
+      if (!(key in node)) {
+        const entry = childListeners.get(key)!
+        entry.api.off(entry.handler)
+        childListeners.delete(key)
+        delete latest[key]
+      }
+    }
+
+    notify()
+  }
+
+  const topLevelApi = chain()
+  topLevelApi.on(topLevelHandler, _get, opt)
+
+  return () => {
+    topLevelApi.off(topLevelHandler)
+    for (const {api: childApi, handler} of childListeners.values()) {
+      childApi.off(handler)
+    }
+    childListeners.clear()
+  }
+}

package/ts/src/schemas.ts

@@ -185,6 +185,7 @@ export const WireOptionsSchema = z
     secure: z.boolean().optional(),
     fast: z.boolean().optional(),
     put: z.boolean().optional(),
+    nested: z.boolean().optional(),
   })
   .optional()
 export type WireOptions = z.infer<typeof WireOptionsSchema>

package/ts/src/wire.ts

@@ -509,14 +509,18 @@ const Wire = (opt: HolsterOptions): WireAPI => {
           continue
         }
         const limit = userLimits.get(pub) ?? defaultLimit
-        if ((userStorage.get(pub) ?? 0) >= limit) continue
-        toStore[soul] = node
+        const current = userStorage.get(pub) ?? 0
         const oldBytes = oldGraphBytes[soul] ?? 0
         const newBytes = graph[soul] ? JSON.stringify(graph[soul]).length : 0
-        userStorage.set(
-          pub,
-          Math.max(0, (userStorage.get(pub) ?? 0) - oldBytes + newBytes),
-        )
+        const updated = Math.max(0, current - oldBytes + newBytes)
+        // A write that doesn't grow this user's total usage (e.g. a
+        // delete) is always let through, even past the limit - otherwise
+        // a user already over quota could never delete anything to get
+        // back under it. Only a write that both grows usage and pushes
+        // the total past the limit is rejected.
+        if (updated > current && updated > limit) continue
+        toStore[soul] = node
+        userStorage.set(pub, updated)
         saveUserStorage()
       }
       if (Object.keys(toStore).length > 0) {
@@ -730,15 +734,21 @@ const Wire = (opt: HolsterOptions): WireAPI => {
             if (!pub) continue
             const limit = userLimits.get(pub) ?? defaultLimit
             const current = userStorage.get(pub) ?? 0
-            if (current >= limit) {
-              if (cb) cb("storage limit exceeded")
-              return
-            }
             const oldBytes = oldGraphBytes[soul] ?? 0
             const newBytes = graph[soul]
               ? JSON.stringify(graph[soul]).length
               : 0
-            userStorage.set(pub, Math.max(0, current - oldBytes + newBytes))
+            const updated = Math.max(0, current - oldBytes + newBytes)
+            // A write that doesn't grow this user's total usage (e.g. a
+            // delete) is always let through, even past the limit -
+            // otherwise a user already over quota could never delete
+            // anything to get back under it. Only a write that both grows
+            // usage and pushes the total past the limit is rejected.
+            if (updated > current && updated > limit) {
+              if (cb) cb("storage limit exceeded")
+              return
+            }
+            userStorage.set(pub, updated)
             saveUserStorage()
           }
         }
@@ -1082,6 +1092,17 @@ const Wire = (opt: HolsterOptions): WireAPI => {
   }
 
   const send = (data: string): {err?: string} | void => {
+    // Reject client-side, before ever attempting to send, rather than
+    // silently succeeding locally while the receiving peer's own
+    // validateMessage (wire.ts's server-side inbound check) drops it and
+    // replies with an error nothing here is listening for - so a save
+    // that's too big to ever sync would otherwise look like it worked.
+    const maxMessageSize = options.maxMessageSize || 1024 * 1024
+    if (data.length > maxMessageSize) {
+      return {
+        err: `Message too large (${data.length} > ${maxMessageSize} bytes). Reduce the size of the data being saved.`,
+      }
+    }
     if (messageQueue.length >= maxQueueLength) {
       return {
         err: `Message queue exceeded maximum length (${maxQueueLength}). Update query logic to request less data.`,

package/utils/user-storage.js

@@ -39,8 +39,6 @@ const formatBytes = bytes => {
   return `${(bytes / 1073741824).toFixed(1)} GB`
 }
 
-const DEFAULT_STORAGE_LIMIT = 1 // MB
-
 const userStorage = readJSON(".user_storage.json")
 const userLimit = readJSON(".user_limit.json")
 
@@ -77,10 +75,13 @@ if (page.length > 0) {
 
 const data = userLimit.data || {}
 for (const {pub, total} of page) {
-  const pubLimit =
-    typeof data[pub] === "number" ? data[pub] : DEFAULT_STORAGE_LIMIT
+  const pubLimit = data[pub]
   const limitStr =
-    pubLimit === 0 ? "0 B (blocked)" : formatBytes(pubLimit * 1048576)
+    typeof pubLimit !== "number"
+      ? "no override (see server logs for default)"
+      : pubLimit === 0
+        ? "0 B (blocked)"
+        : formatBytes(pubLimit * 1048576)
   console.log(`  ${pub}`)
   console.log(`    used: ${formatBytes(total)} / limit: ${limitStr}`)
 }