@mblaney/holster 2.1.3 → 2.1.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "2.1.3",
+  "version": "2.1.4",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/holster.js

@@ -14,6 +14,11 @@ const Holster = opt => {
   const map = new Map()
   // Allow concurrent calls to the api by storing each context.
   const allctx = new Map()
+  // Serializes concurrent creation of a missing rel for the same
+  // soul+item, so only the first caller creates it and the rest reuse
+  // its result rather than each minting a competing soul. Map<soul,
+  // Map<item, promise resolving to the soul id>>.
+  const pendingRel = new Map()
 
   const ok = data => {
     return (
@@ -181,6 +186,65 @@ const Holster = opt => {
       }
     }
 
+    // Creates a rel for item on soul, or reuses one already being created
+    // by a concurrent caller for the same soul+item (see pendingRel above).
+    // Calls cb(err, id) with a ready-to-display err, already prefixed where
+    // needed - callers should pass it straight through, not wrap it again.
+    const createRel = (soul, item, node, user, cb) => {
+      let bySoul = pendingRel.get(soul)
+      const pending = bySoul && bySoul.get(item)
+      if (pending) {
+        pending.then(
+          id => cb(null, id),
+          err => cb(err),
+        )
+        return
+      }
+
+      const id = utils.text.random()
+      node[item] = utils.rel.ify(id)
+      let settle, fail
+      const promise = new Promise((res, rej) => {
+        settle = res
+        fail = rej
+      })
+      promise.catch(() => {})
+      if (!bySoul) {
+        bySoul = new Map()
+        pendingRel.set(soul, bySoul)
+      }
+      bySoul.set(item, promise)
+      const clear = () => {
+        bySoul.delete(item)
+        if (bySoul.size === 0) pendingRel.delete(soul)
+      }
+
+      ;(async () => {
+        // graph() itself calls cb with an already-formatted message when it
+        // fails (secure mode, no user) - pass cb straight through rather
+        // than wrapping, so that message isn't prefixed a second time.
+        const g = await graph(soul, node, user, cb)
+        if (g === null) {
+          clear()
+          fail(new Error("secure mode"))
+          return
+        }
+
+        wire.put(g, err => {
+          clear()
+          if (err) {
+            const message = `error putting ${item} on ${soul}: ${err}`
+            fail(message)
+            cb(message)
+            return
+          }
+
+          settle(id)
+          cb(null, id)
+        })
+      })()
+    }
+
     const resolve = (request, cb) => {
       if (!request) {
         console.log("error resolve request parameter required")
@@ -236,18 +300,13 @@ const Holster = opt => {
                 cb(node[item])
               } else if (put) {
                 // Request was chained before put, so rel doesn't exist yet.
-                id = utils.text.random()
-                node[item] = utils.rel.ify(id)
-                const g = await graph(soul, node, ctx.user, cb)
-                if (g === null) return
-
-                wire.put(g, err => {
+                createRel(soul, item, node, ctx.user, (err, relId) => {
                   if (err) {
-                    cb(`error putting ${item} on ${soul}: ${err}`)
+                    cb(err)
                     return
                   }
 
-                  ctx.chain[i].soul = id
+                  ctx.chain[i].soul = relId
                   api(ctxid).put(request.put, cb)
                 })
               } else if (on) {
@@ -261,19 +320,13 @@ const Holster = opt => {
               }
             } else if (put) {
               // Request was chained before put, so rel doesn't exist yet.
-              const id = utils.text.random()
-              if (!node) node = {}
-              node[item] = utils.rel.ify(id)
-              const g = await graph(soul, node, ctx.user, cb)
-              if (g === null) return
-
-              wire.put(g, err => {
+              createRel(soul, item, node || {}, ctx.user, (err, relId) => {
                 if (err) {
-                  cb(`error putting ${item} on ${soul}: ${err}`)
+                  cb(err)
                   return
                 }
 
-                ctx.chain[i].soul = id
+                ctx.chain[i].soul = relId
                 api(ctxid).put(request.put, cb)
               })
             } else {
@@ -541,25 +594,21 @@ const Holster = opt => {
               const id = utils.rel.is(current)
               if (!id) {
                 // The current rel doesn't exist, so add it first.
-                if (!node) node = {}
-                node[item] = utils.rel.ify(utils.text.random())
-                const g = await graph(soul, node, ctx.user, _ack)
-                if (g === null) return
-
-                wire.put(g, err => {
+                createRel(soul, item, node || {}, ctx.user, err => {
                   if (err) {
-                    _ack(`error putting ${item} on ${soul}: ${err}`)
-                  } else {
-                    const _ctxid = utils.text.random()
-                    const chain = [{item: item, soul: soul}]
-                    // Pass on the previous ctx's callback and user flag here.
-                    allctx.set(_ctxid, {
-                      chain: chain,
-                      user: ctx.user,
-                      cb: ctx.cb,
-                    })
-                    api(_ctxid).put(data)
+                    _ack(err)
+                    return
                   }
+
+                  const _ctxid = utils.text.random()
+                  const chain = [{item: item, soul: soul}]
+                  // Pass on the previous ctx's callback and user flag here.
+                  allctx.set(_ctxid, {
+                    chain: chain,
+                    user: ctx.user,
+                    cb: ctx.cb,
+                  })
+                  api(_ctxid).put(data)
                 })
                 return
               }

package/src/radisk.js

@@ -171,13 +171,7 @@ const Radisk = opt => {
     radisk.batch = Radix()
     radisk.batch.acks = []
     radisk.batch.ed = 0
-    let i = 0
     radisk.save(batch, err => {
-      // This is to ignore multiple callbacks from radisk.save calling
-      // radisk.write? It looks like multiple callbacks will be made if a
-      // file needs to be split.
-      if (++i > 1) return
-
       if (err) opt.log(err)
       batch.acks.forEach(cb => cb(err))
       radisk.thrash.at = null
@@ -199,23 +193,47 @@ const Radisk = opt => {
         if (key < save.start) return
 
         save.start = key
-        opt.store.list(save.lex)
+        // store.list's own streamed order isn't guaranteed to be sorted -
+        // fs.readdir in particular makes no such promise, unlike
+        // radisk.read's own file lookup, which already sorts before
+        // searching. Collect the full list and sort it first, rather
+        // than scanning the raw stream as it arrives, since save.lex's
+        // own boundary search below only works correctly over genuinely
+        // sorted input - fed an unsorted stream, it can match the wrong
+        // file, or stop tracking new candidates the moment the stream
+        // stops looking monotonic, silently losing track of files that
+        // arrive afterward.
+        const files = []
+        opt.store.list(file => {
+          if (file) {
+            files.push(file)
+            return
+          }
+          files.sort()
+          save.lex(files)
+        })
         return true
       },
-      lex: file => {
-        if (!file || file > save.start) {
-          save.end = file
-          // ! is used as the first file name as it's the first printable
-          // character, so always matches as lexically less than any node.
-          // Also save.start can be set to undefined by a previous call to
-          // save.mix, so don't continue in this case.
-          if (save.start) save.mix(save.file || "!", save.start, save.end)
-        } else {
-          save.file = file
+      lex: files => {
+        // save.start can be set to undefined by a previous call to
+        // save.mix, so don't continue in this case.
+        if (!save.start) return
+
+        let candidate = null
+        for (const file of files) {
+          if (file > save.start) {
+            // ! is used as the first file name as it's the first
+            // printable character, so always matches as lexically less
+            // than any node.
+            save.mix(candidate || "!", save.start, file)
+            return
+          }
+          candidate = file
         }
+        save.mix(candidate || "!", save.start, u)
       },
       mix: (file, start, end) => {
-        save.start = save.end = save.file = u
+        save.start = u
         // Use cache if available, otherwise parse from disk
         if (cache.has(file)) {
           const disk = cache.get(file)
@@ -223,8 +241,17 @@ const Radisk = opt => {
             if (key < start) return
 
             if (end && end < key) {
+              // Found where the next file segment starts - stop here (the
+              // non-undefined return is what actually stops Radix.map's
+              // own iteration). Without it, the scan kept going through
+              // every remaining entry in the whole batch, each one
+              // overwriting save.start in turn, so save.next's own resume
+              // point ended up as the LAST key in the batch rather than
+              // the very next one - silently skipping every entry in
+              // between as "already processed" when none of them ever
+              // were.
               save.start = key
-              return
+              return true
             }
 
             disk(key, value)
@@ -239,11 +266,21 @@ const Radisk = opt => {
 
               if (end && end < key) {
                 save.start = key
-                return
+                return true
               }
 
               disk(key, value)
             })
+            // Populate the cache with this merge synchronously, before the
+            // async store.put below even starts - a write's own merged
+            // result is always authoritative (it's this batch's actual
+            // content), unlike a plain read's parse, which can't tell
+            // whether it raced ahead of a write. Closes the window where a
+            // concurrent read's own parse-from-disk (started before this
+            // write, resolving after it) would otherwise be free to
+            // populate the cache with stale, pre-write data - see
+            // radisk.read's own matching change below.
+            if (opt.cache) cache.set(file, disk)
             radisk.write(file, disk, save.next)
           })
         }
@@ -260,6 +297,21 @@ const Radisk = opt => {
   }
 
   radisk.write = (file, rad, cb) => {
+    // A split produces two independent opt.store.put calls - this file's
+    // own (truncated) remainder below, and the recursive call for the
+    // split-off sub-file. pending starts at 1 for this file's own write,
+    // and is bumped to 2 the moment a split is decided, so cb only fires
+    // once both halves are actually durable - calling it as soon as
+    // whichever one finishes first (the previous behavior) let a caller
+    // continue - including resolving the original put()'s own callback -
+    // before the other half's data had actually been written.
+    let pending = 1
+    let firstErr = null
+    const settle = err => {
+      if (err && !firstErr) firstErr = err
+      if (--pending > 0) return
+      cb(firstErr)
+    }
     const write = {
       text: "",
       limit: "",
@@ -318,7 +370,8 @@ const Radisk = opt => {
             write.done = true
             write.sub = Radix()
             Radix.map(rad, write.slice)
-            radisk.write(write.limit, write.sub, cb)
+            pending++
+            radisk.write(write.limit, write.sub, settle)
             return
           }
         }
@@ -342,7 +395,7 @@ const Radisk = opt => {
         cache.delete(file)
         fileListCache = null
       }
-      cb(err)
+      settle(err)
     })
   }
 
@@ -378,7 +431,13 @@ const Radisk = opt => {
       }
       radisk.parse(file, (err, disk) => {
         if (err) opt.log(err)
-        if (disk && opt.cache) {
+        // Never overwrite an existing cache entry from a plain read - it
+        // can't tell whether it raced ahead of a concurrent write (started
+        // before that write, but resolving after the write already
+        // installed its own authoritative, freshly-merged entry - see
+        // save.mix's own matching change). Only populate the cache when
+        // nothing beat this read to it.
+        if (disk && opt.cache && !cache.has(file)) {
           cache.set(file, disk)
           checkMemoryUsage()
         }

package/src/radix.js

@@ -26,38 +26,68 @@ const Radix = () => {
     }
 
     if (!found) {
-      // If not found from the provided keys try matching with an existing key.
-      const result = utils.obj.map(tree, (hasValue, hasKey) => {
-        let j = 0
-        let matchingKey = ""
-        while (hasKey[j] === keys[j]) {
-          matchingKey += hasKey[j++]
-        }
-        if (matchingKey) {
-          if (noValue) {
-            // matchingKey has to be as long as the original keys when reading.
+      if (noValue) {
+        // If not found from the provided keys try matching with an existing
+        // key.
+        const result = utils.obj.map(tree, (hasValue, hasKey) => {
+          let j = 0
+          let matchingKey = ""
+          while (hasKey[j] === keys[j]) {
+            matchingKey += hasKey[j++]
+          }
+          if (matchingKey) {
+            // matchingKey has to be as long as the original keys when
+            // reading.
             if (j <= max) return
 
             tmp[hasKey.slice(j)] = hasValue
             return hasValue
           }
+        })
+        if (!result) return
 
-          let replace = {
-            [hasKey.slice(j)]: hasValue,
-            [keys.slice(j)]: {[record]: value},
-          }
-          tree[matchingKey] = {[group]: replace}
-          delete tree[hasKey]
-          return true
+        return tmp
+      }
+
+      // Writing: find the EXISTING key with the longest shared prefix,
+      // not just the first one with any overlap at all (obj.map stops at
+      // the first non-undefined result, so a plain find here would settle
+      // for whichever candidate happens to come first in Object.keys'
+      // own iteration order). A radix tree's own invariant - no two
+      // sibling keys share a prefix - only holds if every insertion
+      // merges with its one true match; with many keys sharing this tree
+      // at once (many concurrent writers batched together), a
+      // coincidentally shorter, unrelated key can otherwise "win" over
+      // the real, much longer match - silently misplacing the new key
+      // under the wrong group, while its actual sibling is never touched
+      // at all and stays wherever it already was, invisible to any read
+      // that expects it inside this new group.
+      let bestKey = null
+      let bestMatch = ""
+      for (const hasKey of Object.keys(tree)) {
+        let j = 0
+        let matchingKey = ""
+        while (hasKey[j] === keys[j]) {
+          matchingKey += hasKey[j++]
         }
-      })
-      if (!result) {
-        if (noValue) return
+        if (matchingKey.length > bestMatch.length) {
+          bestKey = hasKey
+          bestMatch = matchingKey
+        }
+      }
 
+      if (bestKey) {
+        const hasValue = tree[bestKey]
+        const j = bestMatch.length
+        let replace = {
+          [bestKey.slice(j)]: hasValue,
+          [keys.slice(j)]: {[record]: value},
+        }
+        tree[bestMatch] = {[group]: replace}
+        delete tree[bestKey]
+      } else {
         if (!tree[key]) tree[key] = {}
         tree[key][record] = value
-      } else if (noValue) {
-        return tmp
       }
     } else if (i === max) {
       // If no value use the key provided to return a whole group or record.

package/src/store.js

@@ -297,4 +297,5 @@ const Store = opt => {
   }
 }
 
+export {fileSystem}
 export default Store

package/src/wire.js

@@ -319,6 +319,9 @@ const Wire = opt => {
   // signature but not actually match the user under which the data is being
   // stored. To avoid this, the current data on a soul needs to be checked to
   // make sure the stored public key matches the one provided with the update.
+  // This must run before Ham.mix, which updates graph in place — checking
+  // afterwards would compare the incoming update's public key against itself
+  // instead of the previously stored one.
   const check = async (data, send, cb) => {
     const key = utils.userPublicKey
 
@@ -328,8 +331,12 @@ const Wire = opt => {
       // allow it - the update is not trying to change the public key.
       if (node[key] === undefined) continue
 
+      // A full-node lookup is required (rather than a targeted property
+      // lookup) so that a soul which exists but has never had a public key
+      // can be told apart from a soul that doesn't exist yet — both would
+      // otherwise resolve to the same "not found" shape for this one key.
       const msg = await new Promise(res => {
-        getWithCallback({"#": soul, ".": key}, res, send, {put: true})
+        getWithCallback({"#": soul}, res, send, {put: true})
       })
       if (msg.err) {
         if (cb) cb(msg.err)
@@ -340,7 +347,12 @@ const Wire = opt => {
       // matching public keys, as the provided soul also needs a rel on the
       // parent node which then also requires checking. Otherwise public keys
       // need to match for existing data.
-      if (!msg.put || !msg.put[soul] || msg.put[soul][key] === node[key]) {
+      const existing = msg.put && msg.put[soul]
+      if (
+        !existing ||
+        existing[key] === undefined ||
+        existing[key] === node[key]
+      ) {
         continue
       }
 
@@ -394,6 +406,8 @@ const Wire = opt => {
       }
     }
 
+    if (!(await check(msg.put, send))) return
+
     // Store updates returned from Ham.mix and defer updates if required.
     const update = await Ham.mix(
       msg.put,
@@ -414,8 +428,6 @@ const Wire = opt => {
       return
     }
 
-    if (!(await check(update.now, send))) return
-
     const finish = err => {
       if (err) console.warn("store.put", err)
       send(
@@ -636,6 +648,10 @@ const Wire = opt => {
             }
           }
         }
+        if (!(await check(data, send, cb))) {
+          return
+        }
+
         const update = await Ham.mix(
           data,
           graph,
@@ -650,8 +666,15 @@ const Wire = opt => {
           return
         }
 
-        if (!(await check(update.now, send, cb))) {
-          return
+        // Ham.mix has already merged this locally-authored update into
+        // graph[soul], synchronously, before store.put below even starts
+        // persisting it - mark it as fully known now rather than waiting
+        // for a future disk fetch to do so, so a read landing in that
+        // window doesn't get forced through a disk round-trip that could
+        // itself race ahead of this same write and find nothing, even
+        // though the correct answer was already sitting in memory.
+        for (const soul of Object.keys(update.now)) {
+          hasNode.add(soul)
         }
 
         if (opt.userLimit) {