@mblaney/holster 1.0.36 → 2.0.1

package/ts/src/holster.ts

@@ -123,28 +123,28 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
             delete (msg.put[soul] as { _?: unknown })._
             delete (msg.put[soul] as Record<string, unknown>)[utils.userPublicKey]
             delete (msg.put[soul] as Record<string, unknown>)[utils.userSignature]
-            for (const key of Object.keys(msg.put[soul]!)) {
-              const id = utils.rel.is(msg.put[soul]![key] as GraphValue)
-              if (id) {
-                const attemptRead = async (retries = 0): Promise<unknown> => {
-                  const data = await new Promise<unknown>(res => {
-                    const _ctxid = utils.text.random()
-                    const ctx = allctx.get(ctxid!)
-                    allctx.set(_ctxid, {
-                      chain: [{ item: null, soul: id }],
-                      user: ctx ? ctx.user : null,
-                    })
-                    api(_ctxid).next(null as never, res as never, _opt)
+            const node = msg.put[soul]!
+            await Promise.all(Object.keys(node).map(async key => {
+              const id = utils.rel.is(node[key] as GraphValue)
+              if (!id) return
+              const attemptRead = async (retries = 0): Promise<unknown> => {
+                const data = await new Promise<unknown>(res => {
+                  const _ctxid = utils.text.random()
+                  const ctx = allctx.get(ctxid!)
+                  allctx.set(_ctxid, {
+                    chain: [{ item: null, soul: id }],
+                    user: ctx ? ctx.user : null,
                   })
-                  if (data !== null || retries >= 5) {
-                    return data
-                  }
-                  await new Promise(resolve => setTimeout(resolve, 50))
-                  return attemptRead(retries + 1)
+                  api(_ctxid).next(null as never, res as never, _opt)
+                })
+                if (data !== null || retries >= 5) {
+                  return data
                 }
-                msg.put[soul]![key] = (await attemptRead()) as GraphValue
+                await new Promise(resolve => setTimeout(resolve, 50))
+                return attemptRead(retries + 1)
               }
-            }
+              node[key] = (await attemptRead()) as GraphValue
+            }))
             ack(msg.put[soul])
           } else {
             ack(null)
@@ -192,13 +192,16 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
       item: string,
       ctx: ApiContext,
       i: number,
-      request: { on?: LexFilter; _get?: boolean; _opt?: WireOptions },
-      cb: (data: unknown) => void
+      request: { on?: LexFilter; get?: LexFilter; _get?: boolean; _opt?: WireOptions },
+      cb: (data: unknown) => void,
+      on = true
     ): void => {
+      let timer: ReturnType<typeof setTimeout> | null = null
       const handler = (): void => {
         // Bail out if the context was removed by off().
         if (!allctx.has(ctxid!)) {
           wire.off({ "#": soul, ".": item }, handler)
+          if (timer) clearTimeout(timer)
           return
         }
         wire.get(
@@ -206,16 +209,39 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
           msg => {
             const node = msg.put && msg.put[soul]
             const id = utils.rel.is(node && node[item] as GraphValue)
-            if (!id) return
+            if (!id) {
+              // If it's a plain value (not a rel) and this is a get, deliver it.
+              // node._ guards against timeout null-acks which lack metadata.
+              if (!on && node && node._ && typeof node[item] !== "undefined") {
+                if (timer) clearTimeout(timer)
+                wire.off({ "#": soul, ".": item }, handler)
+                cb(node[item])
+              }
+              return
+            }
+            if (timer) clearTimeout(timer)
             wire.off({ "#": soul, ".": item }, handler)
             ctx.chain[i]!.soul = id
             allctx.set(ctxid!, { ...ctx })
-            api(ctxid).on(request.on!, cb, request._get, request._opt)
+            if (on) {
+              api(ctxid).on(request.on!, cb, request._get, request._opt)
+            } else {
+              api(ctxid).next(null as never, request.get as never, cb as never, request._opt)
+            }
           },
           { ...request._opt, secure: (typeof ctx.user === "boolean" ? ctx.user : !!ctx.user) || options.secure }
         )
       }
       wire.on({ "#": soul, ".": item }, handler, false, request._opt)
+      // Time out after the same total duration as the on() retry loop
+      // (1+2+4+8+16 = 31s) so callers are not blocked forever if the node
+      // genuinely doesn't exist.
+      if (!on) {
+        timer = setTimeout(() => {
+          wire.off({ "#": soul, ".": item }, handler)
+          if (cb) cb(null)
+        }, 31000)
+      }
     }
 
     const resolve = (
@@ -240,7 +266,8 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
       const off = typeof request.off !== "undefined"
 
       let found = false
-      const ctx = allctx.get(ctxid!)!
+      const ctx = allctx.get(ctxid!)
+      if (!ctx) return {item: null, soul: null}
       for (let i = 1; i < ctx.chain.length; i++) {
         if (ctx.chain[i]!.soul !== null) continue
         found = true
@@ -321,7 +348,17 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
                 // Node doesn't exist yet — watch for it to appear.
                 watchForRel(soul!, item!, ctx, i, request, cb!)
                 if (request._get) cb!(null)
+              } else if (get && node) {
+                const sv = (node as Record<string, unknown> & {_?: {">": Record<string, unknown>}})._?.[">"]
+                if (sv && Object.keys(sv).length > 0 && typeof sv[item!] === "undefined") {
+                  // State vector has other properties but not this one — never written.
+                  if (cb) cb(null)
+                } else {
+                  // Empty state vector or item is tracked — may arrive via push.
+                  watchForRel(soul!, item!, ctx, i, request, cb!, false)
+                }
               } else {
+                // Soul doesn't exist or no get — return null.
                 if (cb) cb(null)
               }
             }
@@ -484,8 +521,7 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
         const ctx = allctx.get(ctxid!)
         if (!ctx) return
 
-        if (!ctx.cb) {
-          if (!callback) return
+        if (!ctx.cb && callback) {
           ctx.cb = callback as never
           callback = undefined
         }
@@ -499,7 +535,7 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
           return
         }
 
-        const resolved = resolve({ put: data }, _ack as never)
+        const resolved = resolve({ put: data, _opt: {put: true} }, _ack as never)
         if (!resolved) return
 
         const { item, soul } = resolved
@@ -566,7 +602,7 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
                 wire.put(g, _ack as never)
               })
             },
-            { secure: (typeof ctx.user === "boolean" ? ctx.user : !!ctx.user) || options.secure }
+            { secure: (typeof ctx.user === "boolean" ? ctx.user : !!ctx.user) || options.secure, put: true }
           )
           return
         }
@@ -650,7 +686,7 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
 
               wire.put(g, _ack as never)
             },
-            { secure: (typeof ctx.user === "boolean" ? ctx.user : !!ctx.user) || options.secure }
+            { secure: (typeof ctx.user === "boolean" ? ctx.user : !!ctx.user) || options.secure, put: true }
           )
         }
         attemptRead()
@@ -808,10 +844,7 @@ const Holster = (opt?: HolsterOptions | string | string[]): HolsterAPI => {
             const id = utils.rel.is(current as GraphValue)
             if (id) {
               wire.off(initialLex as never, map.get(callback)!)
-              const relLex = (lexFilter
-                ? utils.obj.put(initialLex as never, "#", id)
-                : { "#": id, ".": null }) as Lex
-              wire.on(relLex, map.get(callback)!, false, opts)
+              wire.on({ "#": id, ".": null } as never, map.get(callback)!, false, opts)
               if (_get) map.get(callback)!()
             } else if (_get) {
               // Not a rel, but _get was requested, so trigger callback.

package/ts/src/radisk.ts

@@ -48,11 +48,7 @@ const Radisk = (opt: RadiskOptions) => {
 
   const pendingReads = new Map<
     string,
-    {
-      callbacks: Array<(err?: string, result?: EncodedValue | Record<string, EncodedValue>) => void>
-      fired: boolean
-      timeoutId: NodeJS.Timeout | null
-    }
+    Array<(err?: string | null, result?: EncodedValue | Record<string, EncodedValue>) => void>
   >()
 
   let lastMemoryCheck = 0
@@ -66,7 +62,6 @@ const Radisk = (opt: RadiskOptions) => {
     write: opt.write || 1,
     size: opt.size || 1024 * 1024,
     memoryLimit: opt.memoryLimit || 500,
-    readTimeout: opt.readTimeout || 1000,
     cache: opt.cache !== undefined ? opt.cache : true,
     store: opt.store,
   }
@@ -89,7 +84,7 @@ const Radisk = (opt: RadiskOptions) => {
 
   const perfLog = (operation: string, startTime: number, key: string, size?: number): void => {
     const duration = Date.now() - startTime
-    if (duration > 2000) {
+    if (duration > 3000) {
       console.log(
         `[RADISK-SLOW] ${operation}: ${duration}ms for ${key}${size ? ` (${size} bytes)` : ""}`
       )
@@ -143,35 +138,25 @@ const Radisk = (opt: RadiskOptions) => {
       }
 
       if (pendingReads.has(key)) {
-        pendingReads.get(key)!.callbacks.push(callback)
+        pendingReads.get(key)!.push(callback)
         return
       }
 
-      const pending: {
-        callbacks: Array<(err?: string | null, value?: EncodedValue | Record<string, EncodedValue>) => void>
-        fired: boolean
-        timeoutId: NodeJS.Timeout | null
-      } = { callbacks: [callback], fired: false, timeoutId: null }
-      pendingReads.set(key, pending)
-
-      pending.timeoutId = setTimeout(() => {
-        if (!pending.fired) {
-          pending.fired = true
-          pendingReads.delete(key)
-          const err = "radisk read timeout"
-          pending.callbacks.forEach(cb => cb(err, undefined))
-        }
-      }, options.readTimeout)
-
+      pendingReads.set(key, [callback])
       const readStart = Date.now()
+      const timeout = setTimeout(() => {
+        const callbacks = pendingReads.get(key)
+        if (!callbacks) return
+        options.log(`radisk read hang detected for key: ${key}`)
+        pendingReads.delete(key)
+        callbacks.forEach(cb => cb(null, null as unknown as undefined))
+      }, 30000)
       return radisk.read(key, (err, result) => {
-        if (pending.timeoutId) clearTimeout(pending.timeoutId)
+        clearTimeout(timeout)
         perfLog("read", readStart, key)
-        if (!pending.fired) {
-          pending.fired = true
-          pendingReads.delete(key)
-          pending.callbacks.forEach(callback => callback(err, result))
-        }
+        const callbacks = pendingReads.get(key) || []
+        pendingReads.delete(key)
+        callbacks.forEach(callback => callback(err, result))
       })
     }
 
@@ -288,22 +273,55 @@ const Radisk = (opt: RadiskOptions) => {
       done: false,
       count: 0,
       sub: undefined as RadixFunction | undefined,
-      each: (value: EncodedValue, _key: string, k: string, pre: string[]): void | boolean => {
+      halfKey: "",   // soul ID at the ~options.size/2 mark for a balanced split
+      halfOffset: 0, // write.text.length before that entry
+      each: (
+        value: EncodedValue,
+        _key: string,
+        k: string,
+        pre: string[],
+      ): void | boolean => {
         if (write.done) return
         write.count++
         const valueStr =
           typeof value === "undefined" ? "" : "=" + Radisk.encode(value)
         const enc =
           Radisk.encode(pre.length) + "#" + Radisk.encode(k) + valueStr + "\n"
+        const validBoundary =
+          pre.length === 0 || (pre.length === 1 && k.includes(enq))
+
+        // Record the first valid split boundary past the halfway mark so a
+        // full overflow can split here instead of at the tail, keeping both
+        // halves near options.size/2 rather than leaving the file always full.
+        if (
+          !write.halfKey &&
+          write.count > 1 &&
+          write.text.length > options.size / 2 &&
+          validBoundary
+        ) {
+          const fullKey = pre.join("") + k
+          const endIdx = fullKey.indexOf(enq)
+          write.halfKey =
+            endIdx === -1 ? fullKey : fullKey.substring(0, endIdx)
+          write.halfOffset = write.text.length
+        }
 
         if (
           write.count > 1 &&
-          pre.length === 0 &&
-          write.text.length + enc.length > options.size
+          write.text.length + enc.length > options.size &&
+          validBoundary
         ) {
-          const endIdx = k.indexOf(enq)
-          write.limit = endIdx === -1 ? k : k.substring(0, endIdx)
+          const fullKey = pre.join("") + k
+          const endIdx = fullKey.indexOf(enq)
+          write.limit =
+            write.halfKey ||
+            (endIdx === -1 ? fullKey : fullKey.substring(0, endIdx))
           if (write.limit !== file) {
+            // Truncate to the balanced split point — text after halfOffset is
+            // handled by the write.slice call below, which reads from rad.
+            if (write.halfKey) {
+              write.text = write.text.substring(0, write.halfOffset)
+            }
             write.done = true
             write.sub = Radix()
             Radix.map(rad, write.slice)
@@ -321,6 +339,13 @@ const Radisk = (opt: RadiskOptions) => {
     Radix.map(rad, write.each, true)
     const writeStart = Date.now()
     store.put(file, write.text, err => {
+      // If a split occurred the cached radix tree still contains entries that
+      // were moved to the sub-file. Invalidate so the next save.mix re-reads
+      // from disk rather than re-writing those entries back to this file.
+      if (write.done) {
+        cache.delete(file)
+        fileListCache = null
+      }
       perfLog("file-write", writeStart, file, write.text.length)
       cb(err)
     })
@@ -330,64 +355,45 @@ const Radisk = (opt: RadiskOptions) => {
     const endIdx = key.indexOf(enq)
     const soul = endIdx === -1 ? key : key.substring(0, endIdx)
 
-    const read = {
-      lex: (file?: string): void => {
-        if (!file) {
-          if (!read.file) {
-            cb("no file found", undefined)
-            return
-          }
-
-          if (options.cache && cache.has(read.file)) {
-            const cachedRadix = cache.get(read.file)!
-            read.value = cachedRadix(key) as EncodedValue | Record<string, EncodedValue> | undefined
-            return cb(undefined, read.value)
-          }
-
-          radisk.parse(read.file, read.it)
-          return
-        }
+    const findFile = (files: string[]): string | null => {
+      let lo = 0, hi = files.length - 1, result: string | null = null
+      while (lo <= hi) {
+        const mid = (lo + hi) >> 1
+        const f = files[mid]!
+        if (f <= soul) { result = f; lo = mid + 1 }
+        else { hi = mid - 1 }
+      }
+      return result
+    }
 
-        if (file > soul || file < (read.file || "")) return
-        read.file = file
-      },
-      it: (err?: string | null, disk?: RadixFunction): void => {
+    const read = (file: string | null): void => {
+      if (!file) { cb("no file found", undefined); return }
+      if (options.cache && cache.has(file)) {
+        return cb(undefined, cache.get(file)!(key) as EncodedValue | Record<string, EncodedValue> | undefined)
+      }
+      radisk.parse(file, (err, disk) => {
         if (err) options.log(err)
-        if (disk) {
-          if (options.cache) {
-            cache.set(read.file!, disk)
-            checkMemoryUsage()
-          }
-          read.value = disk(key) as EncodedValue | Record<string, EncodedValue> | undefined
+        if (disk && options.cache) {
+          cache.set(file, disk)
+          checkMemoryUsage()
         }
-        cb(err, read.value)
-      },
-      file: undefined as string | undefined,
-      value: undefined as EncodedValue | Record<string, EncodedValue> | undefined,
+        cb(err, disk ? disk(key) as EncodedValue | Record<string, EncodedValue> | undefined : undefined)
+      })
     }
 
     const now = Date.now()
-    if (
-      options.cache &&
-      fileListCache &&
-      now - fileListCacheTime < FILE_LIST_CACHE_TTL
-    ) {
-      fileListCache.forEach(file => read.lex(file))
-      read.lex()
+    if (options.cache && fileListCache && now - fileListCacheTime < FILE_LIST_CACHE_TTL) {
+      read(findFile(fileListCache))
     } else {
       const files: string[] = []
-      const originalLex = read.lex
-      read.lex = (file?: string): void => {
-        if (file) files.push(file)
-        return originalLex(file)
-      }
-
       store.list((file?: string) => {
-        read.lex(file)
-        if (!file && options.cache) {
+        if (file) { files.push(file); return }
+        files.sort()
+        if (options.cache) {
           fileListCache = files
           fileListCacheTime = now
         }
+        read(findFile(files))
       })
     }
   }

package/ts/src/schemas.ts

@@ -184,7 +184,7 @@ export const WireOptionsSchema = z
   .object({
     secure: z.boolean().optional(),
     fast: z.boolean().optional(),
-    wait: z.number().optional(),
+    put: z.boolean().optional(),
   })
   .optional()
 export type WireOptions = z.infer<typeof WireOptionsSchema>
@@ -319,7 +319,6 @@ export const RadiskOptionsSchema = z.object({
   write: z.number().optional(),
   size: z.number().optional(),
   memoryLimit: z.number().optional(),
-  readTimeout: z.number().optional(),
   cache: z.boolean().optional(),
   store: FileSystemInterfaceSchema.optional(),
 })
@@ -330,7 +329,6 @@ export interface RadiskOptions {
   write?: number
   size?: number
   memoryLimit?: number
-  readTimeout?: number
   cache?: boolean
   store?: FileSystemInterface
 }
@@ -404,11 +402,9 @@ export const HolsterOptionsSchema = z
     port: z.number().optional(),
     server: z.any().optional(),
     wss: z.any().optional(),
-    maxAge: z.number().optional(),
     maxConnections: z.number().optional(),
     maxMessageSize: z.number().optional(),
     maxQueueLength: z.number().optional(),
-    wait: z.number().optional(),
   })
   .passthrough()
 export type HolsterOptions = z.infer<typeof HolsterOptionsSchema>

package/ts/src/sea-utils.ts

@@ -6,7 +6,7 @@
 import SafeBuffer from "./buffer.ts"
 import type { JWK } from "./schemas.ts"
 
-const isNode = typeof document === "undefined"
+const isNode = typeof process !== "undefined" && process.versions?.node != null
 
 // Get crypto based on environment - dynamic import for Node.js compatibility
 const crypto: Crypto = isNode

package/ts/src/store.ts

@@ -15,7 +15,7 @@ import type {
   EncodedValue,
 } from "./schemas.ts"
 
-const isNode = typeof document === "undefined"
+const isNode = typeof process !== "undefined" && process.versions?.node != null
 
 // Dynamic import for Node.js fs module - won't execute in browser/service worker
 const fs = isNode

package/ts/src/user.ts

@@ -38,7 +38,6 @@ export interface UserInterface {
  */
 const User = (opt: HolsterOptions | undefined, wire?: WireInterface): UserInterface => {
   if (!wire) wire = Wire(opt as HolsterOptions)
-  const wait = opt?.wait ?? 5000
   let pubs: string[] = []
   let creating = false
   let authing = false
@@ -117,7 +116,6 @@ const User = (opt: HolsterOptions | undefined, wire?: WireInterface): UserInterf
 
           done(null)
         },
-        { wait: wait }
       )
     }
 
@@ -142,7 +140,6 @@ const User = (opt: HolsterOptions | undefined, wire?: WireInterface): UserInterf
         pubs = Object.keys(data)
         next()
       },
-      { wait: wait }
     )
   }
 
@@ -221,7 +218,6 @@ const User = (opt: HolsterOptions | undefined, wire?: WireInterface): UserInterf
             })
           })
         },
-        { wait: wait }
       )
     },
     auth: (username, password, cb) => {