@mblaney/holster 2.1.3 → 2.2.0

package/src/wire.js

@@ -319,6 +319,9 @@ const Wire = opt => {
   // signature but not actually match the user under which the data is being
   // stored. To avoid this, the current data on a soul needs to be checked to
   // make sure the stored public key matches the one provided with the update.
+  // This must run before Ham.mix, which updates graph in place — checking
+  // afterwards would compare the incoming update's public key against itself
+  // instead of the previously stored one.
   const check = async (data, send, cb) => {
     const key = utils.userPublicKey
 
@@ -328,8 +331,12 @@ const Wire = opt => {
       // allow it - the update is not trying to change the public key.
       if (node[key] === undefined) continue
 
+      // A full-node lookup is required (rather than a targeted property
+      // lookup) so that a soul which exists but has never had a public key
+      // can be told apart from a soul that doesn't exist yet — both would
+      // otherwise resolve to the same "not found" shape for this one key.
       const msg = await new Promise(res => {
-        getWithCallback({"#": soul, ".": key}, res, send, {put: true})
+        getWithCallback({"#": soul}, res, send, {put: true})
       })
       if (msg.err) {
         if (cb) cb(msg.err)
@@ -340,7 +347,12 @@ const Wire = opt => {
       // matching public keys, as the provided soul also needs a rel on the
       // parent node which then also requires checking. Otherwise public keys
       // need to match for existing data.
-      if (!msg.put || !msg.put[soul] || msg.put[soul][key] === node[key]) {
+      const existing = msg.put && msg.put[soul]
+      if (
+        !existing ||
+        existing[key] === undefined ||
+        existing[key] === node[key]
+      ) {
         continue
       }
 
@@ -394,6 +406,8 @@ const Wire = opt => {
       }
     }
 
+    if (!(await check(msg.put, send))) return
+
     // Store updates returned from Ham.mix and defer updates if required.
     const update = await Ham.mix(
       msg.put,
@@ -414,8 +428,6 @@ const Wire = opt => {
       return
     }
 
-    if (!(await check(update.now, send))) return
-
     const finish = err => {
       if (err) console.warn("store.put", err)
       send(
@@ -440,14 +452,18 @@ const Wire = opt => {
           continue
         }
         const limit = userLimits.get(pub) ?? defaultLimit
-        if ((userStorage.get(pub) ?? 0) >= limit) continue
-        toStore[soul] = node
+        const current = userStorage.get(pub) ?? 0
         const oldBytes = oldGraphBytes[soul] ?? 0
         const newBytes = graph[soul] ? JSON.stringify(graph[soul]).length : 0
-        userStorage.set(
-          pub,
-          Math.max(0, (userStorage.get(pub) ?? 0) - oldBytes + newBytes),
-        )
+        const updated = Math.max(0, current - oldBytes + newBytes)
+        // A write that doesn't grow this user's total usage (e.g. a
+        // delete) is always let through, even past the limit - otherwise
+        // a user already over quota could never delete anything to get
+        // back under it. Only a write that both grows usage and pushes
+        // the total past the limit is rejected.
+        if (updated > current && updated > limit) continue
+        toStore[soul] = node
+        userStorage.set(pub, updated)
         saveUserStorage()
       }
       if (Object.keys(toStore).length > 0) {
@@ -636,6 +652,10 @@ const Wire = opt => {
             }
           }
         }
+        if (!(await check(data, send, cb))) {
+          return
+        }
+
         const update = await Ham.mix(
           data,
           graph,
@@ -650,8 +670,15 @@ const Wire = opt => {
           return
         }
 
-        if (!(await check(update.now, send, cb))) {
-          return
+        // Ham.mix has already merged this locally-authored update into
+        // graph[soul], synchronously, before store.put below even starts
+        // persisting it - mark it as fully known now rather than waiting
+        // for a future disk fetch to do so, so a read landing in that
+        // window doesn't get forced through a disk round-trip that could
+        // itself race ahead of this same write and find nothing, even
+        // though the correct answer was already sitting in memory.
+        for (const soul of Object.keys(update.now)) {
+          hasNode.add(soul)
         }
 
         if (opt.userLimit) {
@@ -660,16 +687,22 @@ const Wire = opt => {
             if (!pub) continue
             const limit = userLimits.get(pub) ?? defaultLimit
             const current = userStorage.get(pub) ?? 0
-            if (current >= limit) {
-              if (cb) cb("storage limit exceeded")
-              return
-            }
-
             const oldBytes = oldGraphBytes[soul] ?? 0
             const newBytes = graph[soul]
               ? JSON.stringify(graph[soul]).length
               : 0
-            userStorage.set(pub, Math.max(0, current - oldBytes + newBytes))
+            const updated = Math.max(0, current - oldBytes + newBytes)
+            // A write that doesn't grow this user's total usage (e.g. a
+            // delete) is always let through, even past the limit - otherwise
+            // a user already over quota could never delete anything to get
+            // back under it. Only a write that both grows usage and pushes
+            // the total past the limit is rejected.
+            if (updated > current && updated > limit) {
+              if (cb) cb("storage limit exceeded")
+              return
+            }
+
+            userStorage.set(pub, updated)
             saveUserStorage()
           }
         }

package/ts/src/holster.ts

@@ -7,6 +7,7 @@ import * as utils from "./utils.ts"
 import Wire, {type WireAPI} from "./wire.ts"
 import User, {type UserInterface} from "./user.ts"
 import SEA from "./sea.ts"
+import {attachNested} from "./nested.ts"
 import type {
   HolsterOptions,
   ChainItem,
@@ -55,12 +56,15 @@ export interface HolsterAPI {
     set?: boolean | ((data?: string | null) => void),
     cb?: (data?: string | null) => void,
   ) => HolsterAPI | void
-  on: (
-    lex: LexWithDot | ((data: unknown) => void),
-    cb?: ((data: unknown) => void) | boolean,
-    _get?: boolean,
-    _opt?: WireOptions,
-  ) => void
+  on: {
+    (
+      lex: LexWithDot,
+      cb: (data: unknown) => void,
+      _get?: boolean,
+      _opt?: WireOptions,
+    ): void
+    (cb: (data: unknown) => void, _get?: boolean, _opt?: WireOptions): void
+  }
   off: (cb?: (data: unknown) => void) => HolsterAPI | void
   user: () => UserInterface & HolsterAPI
   wire: WireAPI
@@ -78,6 +82,17 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
   const user = User(options, wire as never)
   const map = new Map<(data: unknown) => void, () => void>()
   const allctx = new Map<string, ApiContext>()
+  // Tracks on(..., {nested: true}) subscriptions, keyed by the caller's
+  // own callback (the same reference off() will be called with) - see
+  // attachNested in nested.ts. The actual top-level/child listeners it
+  // creates use their own, separate contexts, so off() just needs to
+  // call the detach function this stores to tear the whole thing down.
+  const nested = new Map<(data: unknown) => void, () => void>()
+  // Serializes concurrent creation of a missing rel for the same
+  // soul+item, so only the first caller creates it and the rest reuse
+  // its result rather than each minting a competing soul. Map<soul,
+  // Map<item, promise resolving to the soul id>>.
+  const pendingRel = new Map<string, Map<string, Promise<string>>>()
 
   const ok = (data: GraphValue): boolean => {
     return (
@@ -275,6 +290,72 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
       }
     }
 
+    // Creates a rel for item on soul, or reuses one already being created
+    // by a concurrent caller for the same soul+item (see pendingRel above).
+    // Calls cb(err, id) with a ready-to-display err, already prefixed where
+    // needed - callers should pass it straight through, not wrap it again.
+    const createRel = (
+      soul: string,
+      item: string,
+      node: Record<string, GraphValue>,
+      user: UserIdentity | null | undefined,
+      cb: (err?: string | null, id?: string) => void,
+    ): void => {
+      let bySoul = pendingRel.get(soul)
+      const pending = bySoul && bySoul.get(item)
+      if (pending) {
+        pending.then(
+          id => cb(undefined, id),
+          err => cb(err as string),
+        )
+        return
+      }
+
+      const id = utils.text.random()
+      node[item] = utils.rel.ify(id)
+      let settle!: (id: string) => void
+      let fail!: (err: unknown) => void
+      const promise = new Promise<string>((res, rej) => {
+        settle = res
+        fail = rej
+      })
+      promise.catch(() => {})
+      if (!bySoul) {
+        bySoul = new Map()
+        pendingRel.set(soul, bySoul)
+      }
+      bySoul.set(item, promise)
+      const clear = () => {
+        bySoul!.delete(item)
+        if (bySoul!.size === 0) pendingRel.delete(soul)
+      }
+
+      void (async () => {
+        // graph() itself calls cb with an already-formatted message when it
+        // fails (secure mode, no user) - pass cb straight through rather
+        // than wrapping, so that message isn't prefixed a second time.
+        const g = await graph(soul, node, user, cb)
+        if (g === null) {
+          clear()
+          fail(new Error("secure mode"))
+          return
+        }
+
+        wire.put(g, err => {
+          clear()
+          if (err) {
+            const message = `error putting ${item} on ${soul}: ${err}`
+            fail(message)
+            cb(message)
+            return
+          }
+
+          settle(id)
+          cb(undefined, id)
+        })
+      })()
+    }
+
     const resolve = (
       request: {
         get?: LexFilter
@@ -341,28 +422,20 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
               } else if (get) {
                 cb!(node[item!])
               } else if (put) {
-                id = utils.text.random()
-                node[item!] = utils.rel.ify(id)
-                const g = await graph(
+                createRel(
                   soul!,
+                  item!,
                   node as never,
                   ctx.user,
-                  cb as never,
+                  (err, relId) => {
+                    if (err) {
+                      ;(cb as (err: string) => void)(err)
+                      return
+                    }
+                    ctx.chain[i]!.soul = relId!
+                    api(ctxid).put(request.put!, cb as never)
+                  },
                 )
-                if (g === null) return
-
-                wire.put(g, err => {
-                  if (err) {
-                    ;(cb as (err: string) => void)(
-                      `error putting ${item} on ${soul}: ${err}`,
-                    )
-                    return
-                  }
-                  if (id) {
-                    ctx.chain[i]!.soul = id
-                  }
-                  api(ctxid).put(request.put!, cb as never)
-                })
               } else if (on) {
                 // Item is not a rel yet — watch the parent soul for when it
                 // becomes one, then retry chain resolution.
@@ -372,20 +445,13 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
                 if (cb) cb(null)
               }
             } else if (put) {
-              const id = utils.text.random()
               if (!node) node = {} as never
-              node[item!] = utils.rel.ify(id)
-              const g = await graph(soul!, node as never, ctx.user, cb as never)
-              if (g === null) return
-
-              wire.put(g, err => {
+              createRel(soul!, item!, node as never, ctx.user, (err, relId) => {
                 if (err) {
-                  ;(cb as (err: string) => void)(
-                    `error putting ${item} on ${soul}: ${err}`,
-                  )
+                  ;(cb as (err: string) => void)(err)
                   return
                 }
-                ctx.chain[i]!.soul = id
+                ctx.chain[i]!.soul = relId!
                 api(ctxid).put(request.put!, cb as never)
               })
             } else {
@@ -708,28 +774,20 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
               const id = utils.rel.is(current as GraphValue)
               if (!id) {
                 if (!node) node = {} as never
-                node[item!] = utils.rel.ify(utils.text.random())
-                const g = await graph(
-                  soul,
-                  node as never,
-                  ctx.user,
-                  _ack as never,
-                )
-                if (g === null) return
-
-                wire.put(g, err => {
+                createRel(soul, item!, node as never, ctx.user, err => {
                   if (err) {
-                    _ack(`error putting ${item} on ${soul}: ${err}`)
-                  } else {
-                    const _ctxid = utils.text.random()
-                    const chain: ChainItem[] = [{item: item!, soul: soul}]
-                    allctx.set(_ctxid, {
-                      chain: chain,
-                      user: ctx.user,
-                      cb: ctx.cb,
-                    })
-                    api(_ctxid).put(data)
+                    _ack(err)
+                    return
                   }
+
+                  const _ctxid = utils.text.random()
+                  const chain: ChainItem[] = [{item: item!, soul: soul}]
+                  allctx.set(_ctxid, {
+                    chain: chain,
+                    user: ctx.user,
+                    cb: ctx.cb,
+                  })
+                  api(_ctxid).put(data)
                 })
                 return
               }
@@ -836,6 +894,23 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
 
         const ctx = allctx.get(ctxid)
 
+        if (opts && opts.nested) {
+          // attachNested calls this once for the top level and again for
+          // every discovered child, each needing its own independent
+          // context - a single shared _ctxid would accumulate .next()
+          // calls onto the same path instead of giving sibling paths.
+          const factory = (): HolsterAPI => {
+            const _ctxid = utils.text.random()
+            allctx.set(_ctxid, {
+              chain: [{item: item, soul: soul}],
+              user: ctx ? ctx.user : null,
+            })
+            return api(_ctxid)
+          }
+          nested.set(callback, attachNested(factory, callback, _get, opts))
+          return
+        }
+
         allctx.set(ctxid, {
           chain: [{item: item, soul: soul}],
           on: true,
@@ -995,7 +1070,7 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
               : options.secure,
           },
         )
-      },
+      } as HolsterAPI["on"],
 
       off: function (cb?: (data: unknown) => void): HolsterAPI | void {
         if (!ctxid) {
@@ -1004,6 +1079,12 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
           return
         }
 
+        if (cb && nested.has(cb)) {
+          nested.get(cb)!()
+          nested.delete(cb)
+          return
+        }
+
         const resolved = resolve({off: true}, cb)
         if (!resolved) return
 

package/ts/src/nested.ts

@@ -0,0 +1,101 @@
+import type {HolsterAPI} from "./holster.ts"
+import type {WireOptions} from "./schemas.ts"
+
+// Backs on(cb, _get, {nested: true}) - see holster.ts. A plain on() only
+// resolves the node it's given; it never re-checks a child whose own
+// value is an object (which always has its own soul, since only scalars
+// live directly on a node) once that child's content changes in place -
+// nothing on the parent's own soul changes when that happens, so a
+// listener on the parent alone never sees it. get()'s own read-resolution
+// (holster.ts) already recursively resolves every such child into plain
+// data for a one-time read - this doesn't repeat that; it adds a live
+// subscription for each child that resolution already exposed, so a
+// later change to it reaches the caller too.
+//
+// Pure and holster-internals-free: chain is a factory - () =>
+// user.get([host, "shared"]).next("domains").next(code) - not a
+// pre-built chained object, since a fresh, independent instance is
+// needed per discovered child (reusing one chained object across
+// multiple .next() calls would push them onto the same path instead of
+// giving sibling paths).
+export function attachNested(
+  chain: () => HolsterAPI,
+  cb: (data: unknown) => void,
+  _get?: boolean,
+  _opt?: WireOptions,
+): () => void {
+  // Stripped before being used on any .on() call this function makes
+  // itself (top-level or child) - otherwise a caller-provided {nested:
+  // true} would re-trigger holster.ts's own nested check on re-entry,
+  // before the normal rel-resolution logic that actually finds the right
+  // soul ever runs.
+  const opt: WireOptions = {..._opt}
+  delete opt.nested
+
+  const childListeners = new Map<
+    string,
+    {api: HolsterAPI; handler: (data: unknown) => void}
+  >()
+  let latest: Record<string, unknown> | null = null
+
+  const notify = () => cb(latest === null ? null : {...latest})
+
+  const topLevelHandler = (data: unknown) => {
+    if (data === null || typeof data !== "object") {
+      latest = data as Record<string, unknown> | null
+      notify()
+      return
+    }
+
+    if (!latest || typeof latest !== "object") latest = {}
+    const node = data as Record<string, unknown>
+    for (const [key, value] of Object.entries(node)) {
+      if (key === "_") continue
+      latest[key] = value
+
+      if (
+        value !== null &&
+        typeof value === "object" &&
+        !childListeners.has(key)
+      ) {
+        const childApi = chain().next(key)
+        const handler = (childData: unknown) => {
+          if (childData === null) {
+            delete latest![key]
+            childListeners.delete(key)
+          } else {
+            latest![key] = childData
+          }
+          notify()
+        }
+        childListeners.set(key, {api: childApi, handler})
+        // false, not true - the current value is already in data above,
+        // so this only needs to watch for future changes, not trigger a
+        // redundant immediate re-read.
+        childApi.on(handler, false, opt)
+      }
+    }
+
+    for (const key of [...childListeners.keys()]) {
+      if (!(key in node)) {
+        const entry = childListeners.get(key)!
+        entry.api.off(entry.handler)
+        childListeners.delete(key)
+        delete latest[key]
+      }
+    }
+
+    notify()
+  }
+
+  const topLevelApi = chain()
+  topLevelApi.on(topLevelHandler, _get, opt)
+
+  return () => {
+    topLevelApi.off(topLevelHandler)
+    for (const {api: childApi, handler} of childListeners.values()) {
+      childApi.off(handler)
+    }
+    childListeners.clear()
+  }
+}

package/ts/src/radisk.ts

@@ -229,10 +229,7 @@ const Radisk = (opt: RadiskOptions) => {
     radisk.batch = Radix() as RadiskInterface["batch"]
     radisk.batch.acks = []
     radisk.batch.ed = 0
-    let i = 0
     radisk.save(batch, err => {
-      if (++i > 1) return
-
       if (err) options.log(err)
       batch.acks.forEach(cb => cb(err))
       radisk.thrash.at = undefined
@@ -246,26 +243,54 @@ const Radisk = (opt: RadiskOptions) => {
       find: (_tree: unknown, key: string): boolean | undefined => {
         if (key < (save.start || "")) return undefined
         save.start = key
-        store.list(save.lex)
+        // store.list's own streamed order isn't guaranteed to be sorted -
+        // fs.readdir in particular makes no such promise, unlike
+        // radisk.read's own file lookup, which already sorts before
+        // searching. Collect the full list and sort it first, rather
+        // than scanning the raw stream as it arrives, since save.lex's
+        // own boundary search below only works correctly over genuinely
+        // sorted input - fed an unsorted stream, it can match the wrong
+        // file, or stop tracking new candidates the moment the stream
+        // stops looking monotonic, silently losing track of files that
+        // arrive afterward.
+        const files: string[] = []
+        store.list(file => {
+          if (file) {
+            files.push(file)
+            return
+          }
+          files.sort()
+          save.lex(files)
+        })
         return true
       },
-      lex: (file?: string): void => {
-        if (!file || file > (save.start || "")) {
-          save.end = file
-          if (save.start) save.mix(save.file || "!", save.start, save.end)
-        } else {
-          save.file = file
+      lex: (files: string[]): void => {
+        // save.start can be set to undefined by a previous call to
+        // save.mix, so don't continue in this case.
+        if (!save.start) return
+
+        let candidate: string | null = null
+        for (const file of files) {
+          if (file > save.start) {
+            // ! is used as the first file name as it's the first
+            // printable character, so always matches as lexically less
+            // than any node.
+            save.mix(candidate || "!", save.start, file)
+            return
+          }
+          candidate = file
         }
+        save.mix(candidate || "!", save.start, undefined)
       },
       mix: (file: string, start: string, end?: string): void => {
-        save.start = save.end = save.file = undefined
+        save.start = undefined
         if (cache.has(file)) {
           const disk = cache.get(file)!
           Radix.map(rad, (value, key) => {
             if (key < start) return undefined
             if (end && end < key) {
               save.start = key
-              return undefined
+              return true
             }
             disk(key, value)
             return undefined
@@ -278,11 +303,21 @@ const Radisk = (opt: RadiskOptions) => {
               if (key < start) return undefined
               if (end && end < key) {
                 save.start = key
-                return undefined
+                return true
               }
               disk!(key, value)
               return undefined
             })
+            // Populate the cache with this merge synchronously, before the
+            // async store.put below even starts - a write's own merged
+            // result is always authoritative (it's this batch's actual
+            // content), unlike a plain read's parse, which can't tell
+            // whether it raced ahead of a write. Closes the window where a
+            // concurrent read's own parse-from-disk (started before this
+            // write, resolving after it) would otherwise be free to
+            // populate the cache with stale, pre-write data - see
+            // radisk.read's own matching change below.
+            if (options.cache) cache.set(file, disk!)
             radisk.write(file, disk!, save.next)
           })
         }
@@ -296,13 +331,26 @@ const Radisk = (opt: RadiskOptions) => {
         cb(err)
       },
       start: undefined as string | undefined,
-      end: undefined as string | undefined,
-      file: undefined as string | undefined,
     }
     Radix.map(rad, save.find)
   }
 
   radisk.write = (file, rad, cb) => {
+    // A split produces two independent store.put calls - this file's own
+    // (truncated) remainder below, and the recursive call for the
+    // split-off sub-file. pending starts at 1 for this file's own write,
+    // and is bumped to 2 the moment a split is decided, so cb only fires
+    // once both halves are actually durable - calling it as soon as
+    // whichever one finishes first (the previous behavior) let a caller
+    // continue - including resolving the original put()'s own callback -
+    // before the other half's data had actually been written.
+    let pending = 1
+    let firstErr: string | null = null
+    const settle = (err?: string | null): void => {
+      if (err && !firstErr) firstErr = err
+      if (--pending > 0) return
+      cb(firstErr)
+    }
     const write = {
       text: "",
       limit: "",
@@ -360,7 +408,8 @@ const Radisk = (opt: RadiskOptions) => {
             write.done = true
             write.sub = Radix()
             Radix.map(rad, write.slice)
-            radisk.write(write.limit, write.sub, cb)
+            pending++
+            radisk.write(write.limit, write.sub, settle)
             return undefined
           }
         }
@@ -380,7 +429,7 @@ const Radisk = (opt: RadiskOptions) => {
         cache.delete(file)
         fileListCache = null
       }
-      cb(err)
+      settle(err)
     })
   }
 
@@ -421,7 +470,13 @@ const Radisk = (opt: RadiskOptions) => {
       }
       radisk.parse(file, (err, disk) => {
         if (err) options.log(err)
-        if (disk && options.cache) {
+        // Never overwrite an existing cache entry from a plain read - it
+        // can't tell whether it raced ahead of a concurrent write (started
+        // before that write, but resolving after the write already
+        // installed its own authoritative, freshly-merged entry - see
+        // save.mix's own matching change). Only populate the cache when
+        // nothing beat this read to it.
+        if (disk && options.cache && !cache.has(file)) {
           cache.set(file, disk)
           checkMemoryUsage()
         }