@mblaney/holster 2.1.2 → 2.1.4

package/ts/src/holster.ts

@@ -78,6 +78,11 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
   const user = User(options, wire as never)
   const map = new Map<(data: unknown) => void, () => void>()
   const allctx = new Map<string, ApiContext>()
+  // Serializes concurrent creation of a missing rel for the same
+  // soul+item, so only the first caller creates it and the rest reuse
+  // its result rather than each minting a competing soul. Map<soul,
+  // Map<item, promise resolving to the soul id>>.
+  const pendingRel = new Map<string, Map<string, Promise<string>>>()
 
   const ok = (data: GraphValue): boolean => {
     return (
@@ -275,6 +280,72 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
       }
     }
 
+    // Creates a rel for item on soul, or reuses one already being created
+    // by a concurrent caller for the same soul+item (see pendingRel above).
+    // Calls cb(err, id) with a ready-to-display err, already prefixed where
+    // needed - callers should pass it straight through, not wrap it again.
+    const createRel = (
+      soul: string,
+      item: string,
+      node: Record<string, GraphValue>,
+      user: UserIdentity | null | undefined,
+      cb: (err?: string | null, id?: string) => void,
+    ): void => {
+      let bySoul = pendingRel.get(soul)
+      const pending = bySoul && bySoul.get(item)
+      if (pending) {
+        pending.then(
+          id => cb(undefined, id),
+          err => cb(err as string),
+        )
+        return
+      }
+
+      const id = utils.text.random()
+      node[item] = utils.rel.ify(id)
+      let settle!: (id: string) => void
+      let fail!: (err: unknown) => void
+      const promise = new Promise<string>((res, rej) => {
+        settle = res
+        fail = rej
+      })
+      promise.catch(() => {})
+      if (!bySoul) {
+        bySoul = new Map()
+        pendingRel.set(soul, bySoul)
+      }
+      bySoul.set(item, promise)
+      const clear = () => {
+        bySoul!.delete(item)
+        if (bySoul!.size === 0) pendingRel.delete(soul)
+      }
+
+      void (async () => {
+        // graph() itself calls cb with an already-formatted message when it
+        // fails (secure mode, no user) - pass cb straight through rather
+        // than wrapping, so that message isn't prefixed a second time.
+        const g = await graph(soul, node, user, cb)
+        if (g === null) {
+          clear()
+          fail(new Error("secure mode"))
+          return
+        }
+
+        wire.put(g, err => {
+          clear()
+          if (err) {
+            const message = `error putting ${item} on ${soul}: ${err}`
+            fail(message)
+            cb(message)
+            return
+          }
+
+          settle(id)
+          cb(undefined, id)
+        })
+      })()
+    }
+
     const resolve = (
       request: {
         get?: LexFilter
@@ -341,28 +412,20 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
               } else if (get) {
                 cb!(node[item!])
               } else if (put) {
-                id = utils.text.random()
-                node[item!] = utils.rel.ify(id)
-                const g = await graph(
+                createRel(
                   soul!,
+                  item!,
                   node as never,
                   ctx.user,
-                  cb as never,
+                  (err, relId) => {
+                    if (err) {
+                      ;(cb as (err: string) => void)(err)
+                      return
+                    }
+                    ctx.chain[i]!.soul = relId!
+                    api(ctxid).put(request.put!, cb as never)
+                  },
                 )
-                if (g === null) return
-
-                wire.put(g, err => {
-                  if (err) {
-                    ;(cb as (err: string) => void)(
-                      `error putting ${item} on ${soul}: ${err}`,
-                    )
-                    return
-                  }
-                  if (id) {
-                    ctx.chain[i]!.soul = id
-                  }
-                  api(ctxid).put(request.put!, cb as never)
-                })
               } else if (on) {
                 // Item is not a rel yet — watch the parent soul for when it
                 // becomes one, then retry chain resolution.
@@ -372,20 +435,13 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
                 if (cb) cb(null)
               }
             } else if (put) {
-              const id = utils.text.random()
               if (!node) node = {} as never
-              node[item!] = utils.rel.ify(id)
-              const g = await graph(soul!, node as never, ctx.user, cb as never)
-              if (g === null) return
-
-              wire.put(g, err => {
+              createRel(soul!, item!, node as never, ctx.user, (err, relId) => {
                 if (err) {
-                  ;(cb as (err: string) => void)(
-                    `error putting ${item} on ${soul}: ${err}`,
-                  )
+                  ;(cb as (err: string) => void)(err)
                   return
                 }
-                ctx.chain[i]!.soul = id
+                ctx.chain[i]!.soul = relId!
                 api(ctxid).put(request.put!, cb as never)
               })
             } else {
@@ -708,28 +764,20 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
               const id = utils.rel.is(current as GraphValue)
               if (!id) {
                 if (!node) node = {} as never
-                node[item!] = utils.rel.ify(utils.text.random())
-                const g = await graph(
-                  soul,
-                  node as never,
-                  ctx.user,
-                  _ack as never,
-                )
-                if (g === null) return
-
-                wire.put(g, err => {
+                createRel(soul, item!, node as never, ctx.user, err => {
                   if (err) {
-                    _ack(`error putting ${item} on ${soul}: ${err}`)
-                  } else {
-                    const _ctxid = utils.text.random()
-                    const chain: ChainItem[] = [{item: item!, soul: soul}]
-                    allctx.set(_ctxid, {
-                      chain: chain,
-                      user: ctx.user,
-                      cb: ctx.cb,
-                    })
-                    api(_ctxid).put(data)
+                    _ack(err)
+                    return
                   }
+
+                  const _ctxid = utils.text.random()
+                  const chain: ChainItem[] = [{item: item!, soul: soul}]
+                  allctx.set(_ctxid, {
+                    chain: chain,
+                    user: ctx.user,
+                    cb: ctx.cb,
+                  })
+                  api(_ctxid).put(data)
                 })
                 return
               }

package/ts/src/radisk.ts

@@ -229,10 +229,7 @@ const Radisk = (opt: RadiskOptions) => {
     radisk.batch = Radix() as RadiskInterface["batch"]
     radisk.batch.acks = []
     radisk.batch.ed = 0
-    let i = 0
     radisk.save(batch, err => {
-      if (++i > 1) return
-
       if (err) options.log(err)
       batch.acks.forEach(cb => cb(err))
       radisk.thrash.at = undefined
@@ -246,26 +243,54 @@ const Radisk = (opt: RadiskOptions) => {
       find: (_tree: unknown, key: string): boolean | undefined => {
         if (key < (save.start || "")) return undefined
         save.start = key
-        store.list(save.lex)
+        // store.list's own streamed order isn't guaranteed to be sorted -
+        // fs.readdir in particular makes no such promise, unlike
+        // radisk.read's own file lookup, which already sorts before
+        // searching. Collect the full list and sort it first, rather
+        // than scanning the raw stream as it arrives, since save.lex's
+        // own boundary search below only works correctly over genuinely
+        // sorted input - fed an unsorted stream, it can match the wrong
+        // file, or stop tracking new candidates the moment the stream
+        // stops looking monotonic, silently losing track of files that
+        // arrive afterward.
+        const files: string[] = []
+        store.list(file => {
+          if (file) {
+            files.push(file)
+            return
+          }
+          files.sort()
+          save.lex(files)
+        })
         return true
       },
-      lex: (file?: string): void => {
-        if (!file || file > (save.start || "")) {
-          save.end = file
-          if (save.start) save.mix(save.file || "!", save.start, save.end)
-        } else {
-          save.file = file
+      lex: (files: string[]): void => {
+        // save.start can be set to undefined by a previous call to
+        // save.mix, so don't continue in this case.
+        if (!save.start) return
+
+        let candidate: string | null = null
+        for (const file of files) {
+          if (file > save.start) {
+            // ! is used as the first file name as it's the first
+            // printable character, so always matches as lexically less
+            // than any node.
+            save.mix(candidate || "!", save.start, file)
+            return
+          }
+          candidate = file
         }
+        save.mix(candidate || "!", save.start, undefined)
       },
       mix: (file: string, start: string, end?: string): void => {
-        save.start = save.end = save.file = undefined
+        save.start = undefined
         if (cache.has(file)) {
           const disk = cache.get(file)!
           Radix.map(rad, (value, key) => {
             if (key < start) return undefined
             if (end && end < key) {
               save.start = key
-              return undefined
+              return true
             }
             disk(key, value)
             return undefined
@@ -278,11 +303,21 @@ const Radisk = (opt: RadiskOptions) => {
               if (key < start) return undefined
               if (end && end < key) {
                 save.start = key
-                return undefined
+                return true
               }
               disk!(key, value)
               return undefined
             })
+            // Populate the cache with this merge synchronously, before the
+            // async store.put below even starts - a write's own merged
+            // result is always authoritative (it's this batch's actual
+            // content), unlike a plain read's parse, which can't tell
+            // whether it raced ahead of a write. Closes the window where a
+            // concurrent read's own parse-from-disk (started before this
+            // write, resolving after it) would otherwise be free to
+            // populate the cache with stale, pre-write data - see
+            // radisk.read's own matching change below.
+            if (options.cache) cache.set(file, disk!)
             radisk.write(file, disk!, save.next)
           })
         }
@@ -296,13 +331,26 @@ const Radisk = (opt: RadiskOptions) => {
         cb(err)
       },
       start: undefined as string | undefined,
-      end: undefined as string | undefined,
-      file: undefined as string | undefined,
     }
     Radix.map(rad, save.find)
   }
 
   radisk.write = (file, rad, cb) => {
+    // A split produces two independent store.put calls - this file's own
+    // (truncated) remainder below, and the recursive call for the
+    // split-off sub-file. pending starts at 1 for this file's own write,
+    // and is bumped to 2 the moment a split is decided, so cb only fires
+    // once both halves are actually durable - calling it as soon as
+    // whichever one finishes first (the previous behavior) let a caller
+    // continue - including resolving the original put()'s own callback -
+    // before the other half's data had actually been written.
+    let pending = 1
+    let firstErr: string | null = null
+    const settle = (err?: string | null): void => {
+      if (err && !firstErr) firstErr = err
+      if (--pending > 0) return
+      cb(firstErr)
+    }
     const write = {
       text: "",
       limit: "",
@@ -360,7 +408,8 @@ const Radisk = (opt: RadiskOptions) => {
             write.done = true
             write.sub = Radix()
             Radix.map(rad, write.slice)
-            radisk.write(write.limit, write.sub, cb)
+            pending++
+            radisk.write(write.limit, write.sub, settle)
             return undefined
           }
         }
@@ -380,7 +429,7 @@ const Radisk = (opt: RadiskOptions) => {
         cache.delete(file)
         fileListCache = null
       }
-      cb(err)
+      settle(err)
     })
   }
 
@@ -421,7 +470,13 @@ const Radisk = (opt: RadiskOptions) => {
       }
       radisk.parse(file, (err, disk) => {
         if (err) options.log(err)
-        if (disk && options.cache) {
+        // Never overwrite an existing cache entry from a plain read - it
+        // can't tell whether it raced ahead of a concurrent write (started
+        // before that write, but resolving after the write already
+        // installed its own authoritative, freshly-merged entry - see
+        // save.mix's own matching change). Only populate the cache when
+        // nothing beat this read to it.
+        if (disk && options.cache && !cache.has(file)) {
           cache.set(file, disk)
           checkMemoryUsage()
         }

package/ts/src/radix.ts

@@ -57,44 +57,74 @@ const Radix = (): RadixFunction => {
     }
 
     if (!found) {
-      // If not found from the provided keys try matching with an existing key
-      const result = utils.obj.map(
-        tree!,
-        (hasValue: RadixValue, hasKey: string) => {
-          let j = 0
-          let matchingKey = ""
-          while (hasKey[j] === keys[j]) {
-            matchingKey += hasKey[j]
-            j++
-          }
-          if (matchingKey) {
-            if (noValue) {
-              // matchingKey has to be as long as the original keys when reading
+      if (noValue) {
+        // If not found from the provided keys try matching with an
+        // existing key.
+        const result = utils.obj.map(
+          tree!,
+          (hasValue: RadixValue, hasKey: string) => {
+            let j = 0
+            let matchingKey = ""
+            while (hasKey[j] === keys[j]) {
+              matchingKey += hasKey[j]
+              j++
+            }
+            if (matchingKey) {
+              // matchingKey has to be as long as the original keys when
+              // reading.
               if (j <= max) return undefined
 
               tmp[hasKey.slice(j)] = hasValue
               return hasValue
             }
+            return undefined
+          },
+        )
+        if (!result) return undefined
 
-            const replace: RadixTree = {
-              [hasKey.slice(j)]: hasValue,
-              [keys.slice(j)]: {[record]: value!},
-            }
-            tree![matchingKey] = {[group]: replace}
-            delete tree![hasKey]
-            return true
-          }
-          return undefined
-        },
-      )
+        return tmp
+      }
 
-      if (!result) {
-        if (noValue) return undefined
+      // Writing: find the EXISTING key with the longest shared prefix,
+      // not just the first one with any overlap at all (obj.map stops at
+      // the first non-undefined result, so a plain find here would settle
+      // for whichever candidate happens to come first in Object.keys'
+      // own iteration order). A radix tree's own invariant - no two
+      // sibling keys share a prefix - only holds if every insertion
+      // merges with its one true match; with many keys sharing this tree
+      // at once (many concurrent writers batched together), a
+      // coincidentally shorter, unrelated key can otherwise "win" over
+      // the real, much longer match - silently misplacing the new key
+      // under the wrong group, while its actual sibling is never touched
+      // at all and stays wherever it already was, invisible to any read
+      // that expects it inside this new group.
+      let bestKey: string | null = null
+      let bestMatch = ""
+      for (const hasKey of Object.keys(tree!)) {
+        let j = 0
+        let matchingKey = ""
+        while (hasKey[j] === keys[j]) {
+          matchingKey += hasKey[j]
+          j++
+        }
+        if (matchingKey.length > bestMatch.length) {
+          bestKey = hasKey
+          bestMatch = matchingKey
+        }
+      }
 
+      if (bestKey) {
+        const hasValue = tree![bestKey]!
+        const j = bestMatch.length
+        const replace: RadixTree = {
+          [bestKey.slice(j)]: hasValue,
+          [keys.slice(j)]: {[record]: value!},
+        }
+        tree![bestMatch] = {[group]: replace}
+        delete tree![bestKey]
+      } else {
         if (!tree![key]) tree![key] = {}
         tree![key]![record] = value!
-      } else if (noValue) {
-        return tmp
       }
     } else if (i === max) {
       // If no value use the key provided to return a whole group or record

package/ts/src/store.ts

@@ -336,4 +336,5 @@ const Store = (opt?: StoreOptions): StoreInterface => {
   }
 }
 
+export {fileSystem}
 export default Store

package/ts/src/wire.ts

@@ -367,11 +367,17 @@ const Wire = (opt: HolsterOptions): WireAPI => {
       console.log("  Top 5 by storage used:")
       sorted.slice(0, 5).forEach(([pub]) => logUser(pub))
     }
-    console.log(
-      "  Run `npx holster-user-storage` to view full keys and all users",
-    )
+    console.log("Run npx holster-user-storage to view full keys and all users")
   }
 
+  // The check function is required because user data must provide a public key
+  // so that it can be verified. The public key might verify the provided
+  // signature but not actually match the user under which the data is being
+  // stored. To avoid this, the current data on a soul needs to be checked to
+  // make sure the stored public key matches the one provided with the update.
+  // This must run before Ham.mix, which updates graph in place — checking
+  // afterwards would compare the incoming update's public key against itself
+  // instead of the previously stored one.
   const check = async (
     data: Graph,
     send: (msg: string) => {err?: string} | void,
@@ -385,8 +391,12 @@ const Wire = (opt: HolsterOptions): WireAPI => {
       // allow it - the update is not trying to change the public key.
       if (node[key] === undefined) continue
 
+      // A full-node lookup is required (rather than a targeted property
+      // lookup) so that a soul which exists but has never had a public key
+      // can be told apart from a soul that doesn't exist yet — both would
+      // otherwise resolve to the same "not found" shape for this one key.
       const msg = await new Promise<WireMessage>(res => {
-        getWithCallback({"#": soul, ".": key}, res, send, {put: true})
+        getWithCallback({"#": soul}, res, send, {put: true})
       })
       if (msg.err) {
         if (cb) cb(msg.err)
@@ -397,7 +407,12 @@ const Wire = (opt: HolsterOptions): WireAPI => {
       // matching public keys, as the provided soul also needs a rel on the
       // parent node which then also requires checking. Otherwise public keys
       // need to match for existing data.
-      if (!msg.put || !msg.put[soul] || msg.put[soul]![key] === node[key]) {
+      const existing = msg.put && msg.put[soul]
+      if (
+        !existing ||
+        existing[key] === undefined ||
+        existing[key] === node[key]
+      ) {
         continue
       }
 
@@ -451,6 +466,8 @@ const Wire = (opt: HolsterOptions): WireAPI => {
       }
     }
 
+    if (!(await check(msg.put, send as never))) return
+
     const update = await Ham.mix(
       msg.put,
       graph,
@@ -469,8 +486,6 @@ const Wire = (opt: HolsterOptions): WireAPI => {
       return
     }
 
-    if (!(await check(update.now, send as never))) return
-
     const finish = (err?: string | null): void => {
       if (err) console.warn("store.put", err)
       send(
@@ -681,6 +696,10 @@ const Wire = (opt: HolsterOptions): WireAPI => {
             }
           }
         }
+        if (!(await check(data, send as never, cb))) {
+          return
+        }
+
         const update = await Ham.mix(
           data,
           graph,
@@ -694,8 +713,15 @@ const Wire = (opt: HolsterOptions): WireAPI => {
           return
         }
 
-        if (!(await check(update.now, send as never, cb))) {
-          return
+        // Ham.mix has already merged this locally-authored update into
+        // graph[soul], synchronously, before store.put below even starts
+        // persisting it - mark it as fully known now rather than waiting
+        // for a future disk fetch to do so, so a read landing in that
+        // window doesn't get forced through a disk round-trip that could
+        // itself race ahead of this same write and find nothing, even
+        // though the correct answer was already sitting in memory.
+        for (const soul of Object.keys(update.now)) {
+          hasNode.add(soul)
         }
 
         if (options.userLimit) {