@mblaney/holster 2.1.3 → 2.2.0

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@mblaney/holster",
-  "version": "2.1.3",
+  "version": "2.2.0",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",
   "scripts": {
-    "test": "node --test",
+    "test": "node --test test/*.test.js test/system/*.test.js",
     "test:ts": "node --test --import tsx/esm ts/test/*.test.ts ts/test/system/*.test.ts",
     "build": "esbuild src/holster.js --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=build",
     "build:ts": "esbuild ts/src/holster.ts --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=dist",

package/src/holster.js

@@ -2,6 +2,7 @@ import * as utils from "./utils.js"
 import Wire from "./wire.js"
 import User from "./user.js"
 import SEA from "./sea.js"
+import {attachNested} from "./nested.js"
 
 const Holster = opt => {
   if (typeof opt === "string") opt = {peers: [opt]}
@@ -14,6 +15,17 @@ const Holster = opt => {
   const map = new Map()
   // Allow concurrent calls to the api by storing each context.
   const allctx = new Map()
+  // Tracks on(..., {nested: true}) subscriptions, keyed by the caller's
+  // own callback (the same reference off() will be called with) - see
+  // attachNested in nested.js. The actual top-level/child listeners it
+  // creates use their own, separate contexts, so off() just needs to
+  // call the detach function this stores to tear the whole thing down.
+  const nested = new Map()
+  // Serializes concurrent creation of a missing rel for the same
+  // soul+item, so only the first caller creates it and the rest reuse
+  // its result rather than each minting a competing soul. Map<soul,
+  // Map<item, promise resolving to the soul id>>.
+  const pendingRel = new Map()
 
   const ok = data => {
     return (
@@ -181,6 +193,65 @@ const Holster = opt => {
       }
     }
 
+    // Creates a rel for item on soul, or reuses one already being created
+    // by a concurrent caller for the same soul+item (see pendingRel above).
+    // Calls cb(err, id) with a ready-to-display err, already prefixed where
+    // needed - callers should pass it straight through, not wrap it again.
+    const createRel = (soul, item, node, user, cb) => {
+      let bySoul = pendingRel.get(soul)
+      const pending = bySoul && bySoul.get(item)
+      if (pending) {
+        pending.then(
+          id => cb(null, id),
+          err => cb(err),
+        )
+        return
+      }
+
+      const id = utils.text.random()
+      node[item] = utils.rel.ify(id)
+      let settle, fail
+      const promise = new Promise((res, rej) => {
+        settle = res
+        fail = rej
+      })
+      promise.catch(() => {})
+      if (!bySoul) {
+        bySoul = new Map()
+        pendingRel.set(soul, bySoul)
+      }
+      bySoul.set(item, promise)
+      const clear = () => {
+        bySoul.delete(item)
+        if (bySoul.size === 0) pendingRel.delete(soul)
+      }
+
+      ;(async () => {
+        // graph() itself calls cb with an already-formatted message when it
+        // fails (secure mode, no user) - pass cb straight through rather
+        // than wrapping, so that message isn't prefixed a second time.
+        const g = await graph(soul, node, user, cb)
+        if (g === null) {
+          clear()
+          fail(new Error("secure mode"))
+          return
+        }
+
+        wire.put(g, err => {
+          clear()
+          if (err) {
+            const message = `error putting ${item} on ${soul}: ${err}`
+            fail(message)
+            cb(message)
+            return
+          }
+
+          settle(id)
+          cb(null, id)
+        })
+      })()
+    }
+
     const resolve = (request, cb) => {
       if (!request) {
         console.log("error resolve request parameter required")
@@ -236,18 +307,13 @@ const Holster = opt => {
                 cb(node[item])
               } else if (put) {
                 // Request was chained before put, so rel doesn't exist yet.
-                id = utils.text.random()
-                node[item] = utils.rel.ify(id)
-                const g = await graph(soul, node, ctx.user, cb)
-                if (g === null) return
-
-                wire.put(g, err => {
+                createRel(soul, item, node, ctx.user, (err, relId) => {
                   if (err) {
-                    cb(`error putting ${item} on ${soul}: ${err}`)
+                    cb(err)
                     return
                   }
 
-                  ctx.chain[i].soul = id
+                  ctx.chain[i].soul = relId
                   api(ctxid).put(request.put, cb)
                 })
               } else if (on) {
@@ -261,19 +327,13 @@ const Holster = opt => {
               }
             } else if (put) {
               // Request was chained before put, so rel doesn't exist yet.
-              const id = utils.text.random()
-              if (!node) node = {}
-              node[item] = utils.rel.ify(id)
-              const g = await graph(soul, node, ctx.user, cb)
-              if (g === null) return
-
-              wire.put(g, err => {
+              createRel(soul, item, node || {}, ctx.user, (err, relId) => {
                 if (err) {
-                  cb(`error putting ${item} on ${soul}: ${err}`)
+                  cb(err)
                   return
                 }
 
-                ctx.chain[i].soul = id
+                ctx.chain[i].soul = relId
                 api(ctxid).put(request.put, cb)
               })
             } else {
@@ -541,25 +601,21 @@ const Holster = opt => {
               const id = utils.rel.is(current)
               if (!id) {
                 // The current rel doesn't exist, so add it first.
-                if (!node) node = {}
-                node[item] = utils.rel.ify(utils.text.random())
-                const g = await graph(soul, node, ctx.user, _ack)
-                if (g === null) return
-
-                wire.put(g, err => {
+                createRel(soul, item, node || {}, ctx.user, err => {
                   if (err) {
-                    _ack(`error putting ${item} on ${soul}: ${err}`)
-                  } else {
-                    const _ctxid = utils.text.random()
-                    const chain = [{item: item, soul: soul}]
-                    // Pass on the previous ctx's callback and user flag here.
-                    allctx.set(_ctxid, {
-                      chain: chain,
-                      user: ctx.user,
-                      cb: ctx.cb,
-                    })
-                    api(_ctxid).put(data)
+                    _ack(err)
+                    return
                   }
+
+                  const _ctxid = utils.text.random()
+                  const chain = [{item: item, soul: soul}]
+                  // Pass on the previous ctx's callback and user flag here.
+                  allctx.set(_ctxid, {
+                    chain: chain,
+                    user: ctx.user,
+                    cb: ctx.cb,
+                  })
+                  api(_ctxid).put(data)
                 })
                 return
               }
@@ -631,6 +687,23 @@ const Holster = opt => {
         // Get the context to check if it has user info
         const ctx = allctx.get(ctxid)
 
+        if (_opt && _opt.nested) {
+          // attachNested calls this once for the top level and again for
+          // every discovered child, each needing its own independent
+          // context - a single shared _ctxid would accumulate .next()
+          // calls onto the same path instead of giving sibling paths.
+          const factory = () => {
+            const _ctxid = utils.text.random()
+            allctx.set(_ctxid, {
+              chain: [{item: item, soul: soul}],
+              user: ctx ? ctx.user : null,
+            })
+            return api(_ctxid)
+          }
+          nested.set(cb, attachNested(factory, cb, _get, _opt))
+          return
+        }
+
         // Flag that this context is set from on and shouldn't be removed.
         allctx.set(ctxid, {
           chain: [{item: item, soul: soul}],
@@ -788,6 +861,12 @@ const Holster = opt => {
           return
         }
 
+        if (nested.has(cb)) {
+          nested.get(cb)()
+          nested.delete(cb)
+          return
+        }
+
         // Resolve the current context before removing event listener.
         const {item, soul} = resolve({off: true}, cb)
         if (!soul) return

package/src/nested.js

@@ -0,0 +1,89 @@
+// Backs on(cb, _get, {nested: true}) - see holster.js. A plain on() only
+// resolves the node it's given; it never re-checks a child whose own
+// value is an object (which always has its own soul, since only scalars
+// live directly on a node) once that child's content changes in place -
+// nothing on the parent's own soul changes when that happens, so a
+// listener on the parent alone never sees it. get()'s own read-resolution
+// (holster.js) already recursively resolves every such child into plain
+// data for a one-time read - this doesn't repeat that; it adds a live
+// subscription for each child that resolution already exposed, so a
+// later change to it reaches the caller too.
+//
+// Pure and holster-internals-free: chain is a factory - () =>
+// user.get([host, "shared"]).next("domains").next(code) - not a
+// pre-built chained object, since a fresh, independent instance is
+// needed per discovered child (reusing one chained object across
+// multiple .next() calls would push them onto the same path instead of
+// giving sibling paths).
+export function attachNested(chain, cb, _get, _opt) {
+  // Stripped before being used on any .on() call this function makes
+  // itself (top-level or child) - otherwise a caller-provided {nested:
+  // true} would re-trigger holster.js's own nested check on re-entry,
+  // before the normal rel-resolution logic that actually finds the right
+  // soul ever runs.
+  const opt = {..._opt}
+  delete opt.nested
+
+  const childListeners = new Map() // key -> {api, handler}
+  let latest = null
+
+  const notify = () => cb(latest === null ? null : {...latest})
+
+  const topLevelHandler = data => {
+    if (data === null || typeof data !== "object") {
+      latest = data
+      notify()
+      return
+    }
+
+    if (!latest || typeof latest !== "object") latest = {}
+    for (const [key, value] of Object.entries(data)) {
+      if (key === "_") continue
+      latest[key] = value
+
+      if (
+        value !== null &&
+        typeof value === "object" &&
+        !childListeners.has(key)
+      ) {
+        const childApi = chain().next(key)
+        const handler = childData => {
+          if (childData === null) {
+            delete latest[key]
+            childListeners.delete(key)
+          } else {
+            latest[key] = childData
+          }
+          notify()
+        }
+        childListeners.set(key, {api: childApi, handler})
+        // false, not true - the current value is already in data above,
+        // so this only needs to watch for future changes, not trigger a
+        // redundant immediate re-read.
+        childApi.on(handler, false, opt)
+      }
+    }
+
+    for (const key of [...childListeners.keys()]) {
+      if (!(key in data)) {
+        const {api: childApi, handler} = childListeners.get(key)
+        childApi.off(handler)
+        childListeners.delete(key)
+        delete latest[key]
+      }
+    }
+
+    notify()
+  }
+
+  const topLevelApi = chain()
+  topLevelApi.on(topLevelHandler, _get, opt)
+
+  return () => {
+    topLevelApi.off(topLevelHandler)
+    for (const {api: childApi, handler} of childListeners.values()) {
+      childApi.off(handler)
+    }
+    childListeners.clear()
+  }
+}

package/src/radisk.js

@@ -171,13 +171,7 @@ const Radisk = opt => {
     radisk.batch = Radix()
     radisk.batch.acks = []
     radisk.batch.ed = 0
-    let i = 0
     radisk.save(batch, err => {
-      // This is to ignore multiple callbacks from radisk.save calling
-      // radisk.write? It looks like multiple callbacks will be made if a
-      // file needs to be split.
-      if (++i > 1) return
-
       if (err) opt.log(err)
       batch.acks.forEach(cb => cb(err))
       radisk.thrash.at = null
@@ -199,23 +193,47 @@ const Radisk = opt => {
         if (key < save.start) return
 
         save.start = key
-        opt.store.list(save.lex)
+        // store.list's own streamed order isn't guaranteed to be sorted -
+        // fs.readdir in particular makes no such promise, unlike
+        // radisk.read's own file lookup, which already sorts before
+        // searching. Collect the full list and sort it first, rather
+        // than scanning the raw stream as it arrives, since save.lex's
+        // own boundary search below only works correctly over genuinely
+        // sorted input - fed an unsorted stream, it can match the wrong
+        // file, or stop tracking new candidates the moment the stream
+        // stops looking monotonic, silently losing track of files that
+        // arrive afterward.
+        const files = []
+        opt.store.list(file => {
+          if (file) {
+            files.push(file)
+            return
+          }
+          files.sort()
+          save.lex(files)
+        })
         return true
       },
-      lex: file => {
-        if (!file || file > save.start) {
-          save.end = file
-          // ! is used as the first file name as it's the first printable
-          // character, so always matches as lexically less than any node.
-          // Also save.start can be set to undefined by a previous call to
-          // save.mix, so don't continue in this case.
-          if (save.start) save.mix(save.file || "!", save.start, save.end)
-        } else {
-          save.file = file
+      lex: files => {
+        // save.start can be set to undefined by a previous call to
+        // save.mix, so don't continue in this case.
+        if (!save.start) return
+
+        let candidate = null
+        for (const file of files) {
+          if (file > save.start) {
+            // ! is used as the first file name as it's the first
+            // printable character, so always matches as lexically less
+            // than any node.
+            save.mix(candidate || "!", save.start, file)
+            return
+          }
+          candidate = file
         }
+        save.mix(candidate || "!", save.start, u)
       },
       mix: (file, start, end) => {
-        save.start = save.end = save.file = u
+        save.start = u
         // Use cache if available, otherwise parse from disk
         if (cache.has(file)) {
           const disk = cache.get(file)
@@ -223,8 +241,17 @@ const Radisk = opt => {
             if (key < start) return
 
             if (end && end < key) {
+              // Found where the next file segment starts - stop here (the
+              // non-undefined return is what actually stops Radix.map's
+              // own iteration). Without it, the scan kept going through
+              // every remaining entry in the whole batch, each one
+              // overwriting save.start in turn, so save.next's own resume
+              // point ended up as the LAST key in the batch rather than
+              // the very next one - silently skipping every entry in
+              // between as "already processed" when none of them ever
+              // were.
               save.start = key
-              return
+              return true
             }
 
             disk(key, value)
@@ -239,11 +266,21 @@ const Radisk = opt => {
 
               if (end && end < key) {
                 save.start = key
-                return
+                return true
               }
 
               disk(key, value)
             })
+            // Populate the cache with this merge synchronously, before the
+            // async store.put below even starts - a write's own merged
+            // result is always authoritative (it's this batch's actual
+            // content), unlike a plain read's parse, which can't tell
+            // whether it raced ahead of a write. Closes the window where a
+            // concurrent read's own parse-from-disk (started before this
+            // write, resolving after it) would otherwise be free to
+            // populate the cache with stale, pre-write data - see
+            // radisk.read's own matching change below.
+            if (opt.cache) cache.set(file, disk)
             radisk.write(file, disk, save.next)
           })
         }
@@ -260,6 +297,21 @@ const Radisk = opt => {
   }
 
   radisk.write = (file, rad, cb) => {
+    // A split produces two independent opt.store.put calls - this file's
+    // own (truncated) remainder below, and the recursive call for the
+    // split-off sub-file. pending starts at 1 for this file's own write,
+    // and is bumped to 2 the moment a split is decided, so cb only fires
+    // once both halves are actually durable - calling it as soon as
+    // whichever one finishes first (the previous behavior) let a caller
+    // continue - including resolving the original put()'s own callback -
+    // before the other half's data had actually been written.
+    let pending = 1
+    let firstErr = null
+    const settle = err => {
+      if (err && !firstErr) firstErr = err
+      if (--pending > 0) return
+      cb(firstErr)
+    }
     const write = {
       text: "",
       limit: "",
@@ -318,7 +370,8 @@ const Radisk = opt => {
             write.done = true
             write.sub = Radix()
             Radix.map(rad, write.slice)
-            radisk.write(write.limit, write.sub, cb)
+            pending++
+            radisk.write(write.limit, write.sub, settle)
             return
           }
         }
@@ -342,7 +395,7 @@ const Radisk = opt => {
         cache.delete(file)
         fileListCache = null
       }
-      cb(err)
+      settle(err)
     })
   }
 
@@ -378,7 +431,13 @@ const Radisk = opt => {
       }
       radisk.parse(file, (err, disk) => {
         if (err) opt.log(err)
-        if (disk && opt.cache) {
+        // Never overwrite an existing cache entry from a plain read - it
+        // can't tell whether it raced ahead of a concurrent write (started
+        // before that write, but resolving after the write already
+        // installed its own authoritative, freshly-merged entry - see
+        // save.mix's own matching change). Only populate the cache when
+        // nothing beat this read to it.
+        if (disk && opt.cache && !cache.has(file)) {
           cache.set(file, disk)
           checkMemoryUsage()
         }

package/src/radix.js

@@ -26,38 +26,68 @@ const Radix = () => {
     }
 
     if (!found) {
-      // If not found from the provided keys try matching with an existing key.
-      const result = utils.obj.map(tree, (hasValue, hasKey) => {
-        let j = 0
-        let matchingKey = ""
-        while (hasKey[j] === keys[j]) {
-          matchingKey += hasKey[j++]
-        }
-        if (matchingKey) {
-          if (noValue) {
-            // matchingKey has to be as long as the original keys when reading.
+      if (noValue) {
+        // If not found from the provided keys try matching with an existing
+        // key.
+        const result = utils.obj.map(tree, (hasValue, hasKey) => {
+          let j = 0
+          let matchingKey = ""
+          while (hasKey[j] === keys[j]) {
+            matchingKey += hasKey[j++]
+          }
+          if (matchingKey) {
+            // matchingKey has to be as long as the original keys when
+            // reading.
             if (j <= max) return
 
             tmp[hasKey.slice(j)] = hasValue
             return hasValue
           }
+        })
+        if (!result) return
 
-          let replace = {
-            [hasKey.slice(j)]: hasValue,
-            [keys.slice(j)]: {[record]: value},
-          }
-          tree[matchingKey] = {[group]: replace}
-          delete tree[hasKey]
-          return true
+        return tmp
+      }
+
+      // Writing: find the EXISTING key with the longest shared prefix,
+      // not just the first one with any overlap at all (obj.map stops at
+      // the first non-undefined result, so a plain find here would settle
+      // for whichever candidate happens to come first in Object.keys'
+      // own iteration order). A radix tree's own invariant - no two
+      // sibling keys share a prefix - only holds if every insertion
+      // merges with its one true match; with many keys sharing this tree
+      // at once (many concurrent writers batched together), a
+      // coincidentally shorter, unrelated key can otherwise "win" over
+      // the real, much longer match - silently misplacing the new key
+      // under the wrong group, while its actual sibling is never touched
+      // at all and stays wherever it already was, invisible to any read
+      // that expects it inside this new group.
+      let bestKey = null
+      let bestMatch = ""
+      for (const hasKey of Object.keys(tree)) {
+        let j = 0
+        let matchingKey = ""
+        while (hasKey[j] === keys[j]) {
+          matchingKey += hasKey[j++]
         }
-      })
-      if (!result) {
-        if (noValue) return
+        if (matchingKey.length > bestMatch.length) {
+          bestKey = hasKey
+          bestMatch = matchingKey
+        }
+      }
 
+      if (bestKey) {
+        const hasValue = tree[bestKey]
+        const j = bestMatch.length
+        let replace = {
+          [bestKey.slice(j)]: hasValue,
+          [keys.slice(j)]: {[record]: value},
+        }
+        tree[bestMatch] = {[group]: replace}
+        delete tree[bestKey]
+      } else {
         if (!tree[key]) tree[key] = {}
         tree[key][record] = value
-      } else if (noValue) {
-        return tmp
       }
     } else if (i === max) {
       // If no value use the key provided to return a whole group or record.

package/src/store.js

@@ -297,4 +297,5 @@ const Store = opt => {
   }
 }
 
+export {fileSystem}
 export default Store