@mblaney/holster 2.1.3 → 2.2.0

package/ts/src/radix.ts

@@ -57,44 +57,74 @@ const Radix = (): RadixFunction => {
     }
 
     if (!found) {
-      // If not found from the provided keys try matching with an existing key
-      const result = utils.obj.map(
-        tree!,
-        (hasValue: RadixValue, hasKey: string) => {
-          let j = 0
-          let matchingKey = ""
-          while (hasKey[j] === keys[j]) {
-            matchingKey += hasKey[j]
-            j++
-          }
-          if (matchingKey) {
-            if (noValue) {
-              // matchingKey has to be as long as the original keys when reading
+      if (noValue) {
+        // If not found from the provided keys try matching with an
+        // existing key.
+        const result = utils.obj.map(
+          tree!,
+          (hasValue: RadixValue, hasKey: string) => {
+            let j = 0
+            let matchingKey = ""
+            while (hasKey[j] === keys[j]) {
+              matchingKey += hasKey[j]
+              j++
+            }
+            if (matchingKey) {
+              // matchingKey has to be as long as the original keys when
+              // reading.
               if (j <= max) return undefined
 
               tmp[hasKey.slice(j)] = hasValue
               return hasValue
             }
+            return undefined
+          },
+        )
+        if (!result) return undefined
 
-            const replace: RadixTree = {
-              [hasKey.slice(j)]: hasValue,
-              [keys.slice(j)]: {[record]: value!},
-            }
-            tree![matchingKey] = {[group]: replace}
-            delete tree![hasKey]
-            return true
-          }
-          return undefined
-        },
-      )
+        return tmp
+      }
 
-      if (!result) {
-        if (noValue) return undefined
+      // Writing: find the EXISTING key with the longest shared prefix,
+      // not just the first one with any overlap at all (obj.map stops at
+      // the first non-undefined result, so a plain find here would settle
+      // for whichever candidate happens to come first in Object.keys'
+      // own iteration order). A radix tree's own invariant - no two
+      // sibling keys share a prefix - only holds if every insertion
+      // merges with its one true match; with many keys sharing this tree
+      // at once (many concurrent writers batched together), a
+      // coincidentally shorter, unrelated key can otherwise "win" over
+      // the real, much longer match - silently misplacing the new key
+      // under the wrong group, while its actual sibling is never touched
+      // at all and stays wherever it already was, invisible to any read
+      // that expects it inside this new group.
+      let bestKey: string | null = null
+      let bestMatch = ""
+      for (const hasKey of Object.keys(tree!)) {
+        let j = 0
+        let matchingKey = ""
+        while (hasKey[j] === keys[j]) {
+          matchingKey += hasKey[j]
+          j++
+        }
+        if (matchingKey.length > bestMatch.length) {
+          bestKey = hasKey
+          bestMatch = matchingKey
+        }
+      }
 
+      if (bestKey) {
+        const hasValue = tree![bestKey]!
+        const j = bestMatch.length
+        const replace: RadixTree = {
+          [bestKey.slice(j)]: hasValue,
+          [keys.slice(j)]: {[record]: value!},
+        }
+        tree![bestMatch] = {[group]: replace}
+        delete tree![bestKey]
+      } else {
         if (!tree![key]) tree![key] = {}
         tree![key]![record] = value!
-      } else if (noValue) {
-        return tmp
       }
     } else if (i === max) {
       // If no value use the key provided to return a whole group or record

package/ts/src/schemas.ts

@@ -185,6 +185,7 @@ export const WireOptionsSchema = z
     secure: z.boolean().optional(),
     fast: z.boolean().optional(),
     put: z.boolean().optional(),
+    nested: z.boolean().optional(),
   })
   .optional()
 export type WireOptions = z.infer<typeof WireOptionsSchema>

package/ts/src/store.ts

@@ -336,4 +336,5 @@ const Store = (opt?: StoreOptions): StoreInterface => {
   }
 }
 
+export {fileSystem}
 export default Store

package/ts/src/wire.ts

@@ -370,6 +370,14 @@ const Wire = (opt: HolsterOptions): WireAPI => {
     console.log("Run npx holster-user-storage to view full keys and all users")
   }
 
+  // The check function is required because user data must provide a public key
+  // so that it can be verified. The public key might verify the provided
+  // signature but not actually match the user under which the data is being
+  // stored. To avoid this, the current data on a soul needs to be checked to
+  // make sure the stored public key matches the one provided with the update.
+  // This must run before Ham.mix, which updates graph in place — checking
+  // afterwards would compare the incoming update's public key against itself
+  // instead of the previously stored one.
   const check = async (
     data: Graph,
     send: (msg: string) => {err?: string} | void,
@@ -383,8 +391,12 @@ const Wire = (opt: HolsterOptions): WireAPI => {
       // allow it - the update is not trying to change the public key.
       if (node[key] === undefined) continue
 
+      // A full-node lookup is required (rather than a targeted property
+      // lookup) so that a soul which exists but has never had a public key
+      // can be told apart from a soul that doesn't exist yet — both would
+      // otherwise resolve to the same "not found" shape for this one key.
       const msg = await new Promise<WireMessage>(res => {
-        getWithCallback({"#": soul, ".": key}, res, send, {put: true})
+        getWithCallback({"#": soul}, res, send, {put: true})
       })
       if (msg.err) {
         if (cb) cb(msg.err)
@@ -395,7 +407,12 @@ const Wire = (opt: HolsterOptions): WireAPI => {
       // matching public keys, as the provided soul also needs a rel on the
       // parent node which then also requires checking. Otherwise public keys
       // need to match for existing data.
-      if (!msg.put || !msg.put[soul] || msg.put[soul]![key] === node[key]) {
+      const existing = msg.put && msg.put[soul]
+      if (
+        !existing ||
+        existing[key] === undefined ||
+        existing[key] === node[key]
+      ) {
         continue
       }
 
@@ -449,6 +466,8 @@ const Wire = (opt: HolsterOptions): WireAPI => {
       }
     }
 
+    if (!(await check(msg.put, send as never))) return
+
     const update = await Ham.mix(
       msg.put,
       graph,
@@ -467,8 +486,6 @@ const Wire = (opt: HolsterOptions): WireAPI => {
       return
     }
 
-    if (!(await check(update.now, send as never))) return
-
     const finish = (err?: string | null): void => {
       if (err) console.warn("store.put", err)
       send(
@@ -492,14 +509,18 @@ const Wire = (opt: HolsterOptions): WireAPI => {
           continue
         }
         const limit = userLimits.get(pub) ?? defaultLimit
-        if ((userStorage.get(pub) ?? 0) >= limit) continue
-        toStore[soul] = node
+        const current = userStorage.get(pub) ?? 0
         const oldBytes = oldGraphBytes[soul] ?? 0
         const newBytes = graph[soul] ? JSON.stringify(graph[soul]).length : 0
-        userStorage.set(
-          pub,
-          Math.max(0, (userStorage.get(pub) ?? 0) - oldBytes + newBytes),
-        )
+        const updated = Math.max(0, current - oldBytes + newBytes)
+        // A write that doesn't grow this user's total usage (e.g. a
+        // delete) is always let through, even past the limit - otherwise
+        // a user already over quota could never delete anything to get
+        // back under it. Only a write that both grows usage and pushes
+        // the total past the limit is rejected.
+        if (updated > current && updated > limit) continue
+        toStore[soul] = node
+        userStorage.set(pub, updated)
         saveUserStorage()
       }
       if (Object.keys(toStore).length > 0) {
@@ -679,6 +700,10 @@ const Wire = (opt: HolsterOptions): WireAPI => {
             }
           }
         }
+        if (!(await check(data, send as never, cb))) {
+          return
+        }
+
         const update = await Ham.mix(
           data,
           graph,
@@ -692,8 +717,15 @@ const Wire = (opt: HolsterOptions): WireAPI => {
           return
         }
 
-        if (!(await check(update.now, send as never, cb))) {
-          return
+        // Ham.mix has already merged this locally-authored update into
+        // graph[soul], synchronously, before store.put below even starts
+        // persisting it - mark it as fully known now rather than waiting
+        // for a future disk fetch to do so, so a read landing in that
+        // window doesn't get forced through a disk round-trip that could
+        // itself race ahead of this same write and find nothing, even
+        // though the correct answer was already sitting in memory.
+        for (const soul of Object.keys(update.now)) {
+          hasNode.add(soul)
         }
 
         if (options.userLimit) {
@@ -702,15 +734,21 @@ const Wire = (opt: HolsterOptions): WireAPI => {
             if (!pub) continue
             const limit = userLimits.get(pub) ?? defaultLimit
             const current = userStorage.get(pub) ?? 0
-            if (current >= limit) {
-              if (cb) cb("storage limit exceeded")
-              return
-            }
             const oldBytes = oldGraphBytes[soul] ?? 0
             const newBytes = graph[soul]
               ? JSON.stringify(graph[soul]).length
               : 0
-            userStorage.set(pub, Math.max(0, current - oldBytes + newBytes))
+            const updated = Math.max(0, current - oldBytes + newBytes)
+            // A write that doesn't grow this user's total usage (e.g. a
+            // delete) is always let through, even past the limit -
+            // otherwise a user already over quota could never delete
+            // anything to get back under it. Only a write that both grows
+            // usage and pushes the total past the limit is rejected.
+            if (updated > current && updated > limit) {
+              if (cb) cb("storage limit exceeded")
+              return
+            }
+            userStorage.set(pub, updated)
             saveUserStorage()
           }
         }