@mblaney/holster 0.1.0

package/src/radisk.js

@@ -0,0 +1,438 @@
+import Radix from "./radix.js"
+import * as utils from "./utils.js"
+
+// ASCII character for end of text.
+const etx = String.fromCharCode(3)
+// ASCII character for enquiry.
+const enq = String.fromCharCode(5)
+// ASCII character for unit separator.
+const unit = String.fromCharCode(31)
+
+// Radisk provides access to a radix tree that is stored in the provided
+// opt.store interface.
+const Radisk = opt => {
+  var u
+  var cache = null
+
+  if (!opt) opt = {}
+  if (!opt.log) opt.log = console.log
+  if (!opt.batch) opt.batch = 10 * 1000
+  if (!opt.write) opt.write = 1 // Wait time before write in milliseconds.
+  if (!opt.size) opt.size = 1024 * 1024 // File size on disk, default 1MB.
+  if (!opt.store) {
+    opt.log(
+      "Radisk needs `store` interface with `{get: fn, put: fn, list: fn}`",
+    )
+    return
+  }
+  if (!opt.store.get) {
+    opt.log("Radisk needs `store.get` interface with `(file, cb)`")
+    return
+  }
+  if (!opt.store.put) {
+    opt.log("Radisk needs `store.put` interface with `(file, data, cb)`")
+    return
+  }
+  if (!opt.store.list) {
+    opt.log("Radisk needs a streaming `store.list` interface with `(cb)`")
+    return
+  }
+
+  // Any and all storage adapters should:
+  // 1. Because writing to disk takes time, we should batch data to disk.
+  //    This improves performance, and reduces potential disk corruption.
+  // 2. If a batch exceeds a certain number of writes, we should immediately
+  //    write to disk when physically possible. This caps total performance,
+  //    but reduces potential loss.
+  const radisk = (key, value, cb) => {
+    key = "" + key
+
+    // If no value is provided then the second parameter is the callback
+    // function. Read value from memory or disk and call callback with it.
+    if (typeof value === "function") {
+      cb = value
+      value = radisk.batch(key)
+      if (typeof value !== "undefined") {
+        return cb(u, value)
+      }
+
+      if (radisk.thrash.at) {
+        value = radisk.thrash.at(key)
+        if (typeof value !== "undefined") {
+          return cb(u, value)
+        }
+      }
+
+      return radisk.read(key, cb)
+    }
+
+    // Otherwise store the value provided.
+    radisk.batch(key, value)
+    if (cb) {
+      radisk.batch.acks.push(cb)
+    }
+    // Don't wait if we have batched too many.
+    if (++radisk.batch.ed >= opt.batch) {
+      return radisk.thrash()
+    }
+
+    // Otherwise wait for more updates before writing.
+    clearTimeout(radisk.batch.timeout)
+    radisk.batch.timeout = setTimeout(radisk.thrash, opt.write)
+  }
+
+  radisk.batch = Radix()
+  radisk.batch.acks = []
+  radisk.batch.ed = 0
+
+  radisk.thrash = () => {
+    if (radisk.thrash.ing) {
+      return (radisk.thrash.more = true)
+    }
+
+    clearTimeout(radisk.batch.timeout)
+    radisk.thrash.more = false
+    radisk.thrash.ing = true
+    var batch = (radisk.thrash.at = radisk.batch)
+    radisk.batch = null
+    radisk.batch = Radix()
+    radisk.batch.acks = []
+    radisk.batch.ed = 0
+    let i = 0
+    radisk.save(batch, err => {
+      // This is to ignore multiple callbacks from radisk.save calling
+      // radisk.write? It looks like multiple callbacks will be made if a
+      // file needs to be split.
+      if (++i > 1) return
+
+      if (err) opt.log(err)
+      batch.acks.forEach(cb => cb(err))
+      radisk.thrash.at = null
+      radisk.thrash.ing = false
+      if (radisk.thrash.more) radisk.thrash()
+    })
+  }
+
+  // 1. Find the first radix item in memory
+  // 2. Use that as the starting index in the directory of files
+  // 3. Find the first file that is lexically larger than it
+  // 4. Read the previous file into memory
+  // 5. Scan through in memory radix for all values lexically less than limit
+  // 6. Merge and write all of those to the in-memory file and back to disk
+  // 7. If file is to large then split. More details needed here
+  radisk.save = (rad, cb) => {
+    const save = {
+      find: (tree, key) => {
+        // This is false for any key until save.start is set to an initial key.
+        if (key < save.start) return
+
+        save.start = key
+        opt.store.list(save.lex)
+        return true
+      },
+      lex: file => {
+        if (!file || file > save.start) {
+          save.end = file
+          // ! is used as the first file name as it's the first printable
+          // character, so always matches as lexically less than any node.
+          save.mix(save.file || "!", save.start, save.end)
+          return true
+        }
+
+        save.file = file
+      },
+      mix: (file, start, end) => {
+        save.start = save.end = save.file = u
+        radisk.parse(file, (err, disk) => {
+          if (err) return cb(err)
+
+          Radix.map(rad, (value, key) => {
+            if (key < start) return
+
+            if (end && end < key) {
+              save.start = key
+              return save.start
+            }
+
+            disk(key, value)
+          })
+          radisk.write(file, disk, save.next)
+        })
+      },
+      next: err => {
+        if (err) return cb(err)
+
+        if (save.start) return Radix.map(rad, save.find)
+
+        cb(err)
+      },
+    }
+    Radix.map(rad, save.find)
+  }
+
+  radisk.write = (file, rad, cb) => {
+    // Invalidate cache on write.
+    cache = null
+    const write = {
+      text: "",
+      count: 0,
+      file: file,
+      each: (value, key, k, pre) => {
+        write.count++
+        var enc =
+          Radisk.encode(pre.length) +
+          "#" +
+          Radisk.encode(k) +
+          (typeof value === "undefined" ? "" : "=" + Radisk.encode(value)) +
+          "\n"
+        // Cannot split the file if only have one entry to write.
+        if (write.count > 1 && write.text.length + enc.length > opt.size) {
+          write.text = ""
+          // Otherwise split the entries in half.
+          write.limit = Math.ceil(write.count / 2)
+          write.count = 0
+          write.sub = Radix()
+          Radix.map(rad, write.slice)
+          return true
+        }
+
+        write.text += enc
+      },
+      put: () => {
+        opt.store.put(file, write.text, cb)
+      },
+      slice: (value, key) => {
+        if (key < write.file) return
+
+        if (++write.count > write.limit) {
+          var name = write.file
+          // Use only the soul of the key as the filename so that all
+          // properties of a soul are written to the same file.
+          let end = key.indexOf(enq)
+          if (end === -1) {
+            write.file = key
+          } else {
+            write.file = key.substring(0, end)
+          }
+          // write.limit can be reached after already writing properties of
+          // the current node, so remove it from write.sub before writing to
+          // disk so that it's not duplicated across files.
+          write.sub(write.file, null)
+          write.count = 0
+          radisk.write(name, write.sub, write.next)
+          return true
+        }
+
+        write.sub(key, value)
+      },
+      next: err => {
+        if (err) return cb(err)
+
+        write.sub = Radix()
+        if (!Radix.map(rad, write.slice)) {
+          radisk.write(write.file, write.sub, cb)
+        }
+      },
+    }
+    // If Radix.map doesn't return true when called with write.each as a
+    // callback then didn't need to split the data. The accumulated write.text
+    // can then be stored with write.put().
+    if (!Radix.map(rad, write.each, true)) write.put()
+  }
+
+  radisk.read = (key, cb) => {
+    if (cache) {
+      let value = cache(key)
+      if (typeof value !== "undefined") return cb(u, value)
+    }
+    // Only the soul of the key is compared to filenames (see radisk.write).
+    let soul = key
+    let end = key.indexOf(enq)
+    if (end !== -1) {
+      soul = key.substring(0, end)
+    }
+
+    const read = {
+      lex: file => {
+        // store.list should call lex without a file last, which means all file
+        // names were compared to soul, so the current read.file is ok to use.
+        if (!file) {
+          if (!read.file) {
+            cb("no file found", u)
+            return
+          }
+
+          radisk.parse(read.file, read.it)
+          return
+        }
+
+        // Want the filename closest to soul.
+        if (file > soul || file < read.file) return
+
+        read.file = file
+      },
+      it: (err, disk) => {
+        if (err) opt.log(err)
+        if (disk) {
+          cache = disk
+          read.value = disk(key)
+        }
+        cb(err, read.value)
+      },
+    }
+    opt.store.list(read.lex)
+  }
+
+  // Let us start by assuming we are the only process that is
+  // changing the directory or bucket. Not because we do not want
+  // to be multi-process/machine, but because we want to experiment
+  // with how much performance and scale we can get out of only one.
+  // Then we can work on the harder problem of being multi-process.
+  radisk.parse = (file, cb) => {
+    const parse = {
+      disk: Radix(),
+      read: (err, data) => {
+        if (err) return cb(err)
+
+        if (!data) return cb(u, parse.disk)
+
+        let pre = []
+        // Work though data by splitting into 3 values. The first value says
+        // if the second value is one of: the radix level for a key, the key
+        // iteself, or a value. The third is the rest of the data to work with.
+        let tmp = parse.split(data)
+        while (tmp) {
+          let key
+          let value
+          let i = tmp[1]
+          tmp = parse.split(tmp[2]) || ""
+          if (tmp[0] === "#") {
+            key = tmp[1]
+            pre = pre.slice(0, i)
+            if (i <= pre.length) pre.push(key)
+          }
+          tmp = parse.split(tmp[2]) || ""
+          if (tmp[0] === "\n") continue
+
+          if (tmp[0] === "=") value = tmp[1]
+          if (typeof key !== "undefined" && typeof value !== "undefined") {
+            parse.disk(pre.join(""), value)
+          }
+          tmp = parse.split(tmp[2])
+        }
+        cb(u, parse.disk)
+      },
+      split: data => {
+        if (!data) return
+
+        let i = -1
+        let a = ""
+        let c = null
+        while ((c = data[++i])) {
+          if (c === unit) break
+
+          a += c
+        }
+        let o = {}
+        if (c) {
+          return [a, Radisk.decode(data.slice(i), o), data.slice(i + o.i)]
+        }
+      },
+    }
+    opt.store.get(file, parse.read)
+  }
+
+  return radisk
+}
+
+Radisk.encode = data => {
+  // A key should be passed in as a string to encode, a value can optionally be
+  // an array of 2 items to include the value's state, as is done by store.js.
+  let state = ""
+  if (data instanceof Array && data.length === 2) {
+    state = etx + data[1]
+    data = data[0]
+  }
+
+  if (typeof data === "string") {
+    let i = 0
+    let current = null
+    let text = unit
+    while ((current = data[i++])) {
+      if (current === unit) text += unit
+    }
+    return text + '"' + data + state + unit
+  }
+
+  const rel = utils.rel.is(data)
+  if (rel) return unit + "#" + rel + state + unit
+
+  if (utils.num.is(data)) return unit + "+" + (data || 0) + state + unit
+
+  if (data === true) return unit + "+" + state + unit
+
+  if (data === false) return unit + "-" + state + unit
+
+  if (data === null) return unit + " " + state + unit
+}
+
+Radisk.decode = (data, obj) => {
+  var text = ""
+  var i = -1
+  var n = 0
+  var current = null
+  var previous = null
+  if (data[0] !== unit) return
+
+  // Find a control character previous to the text we want, skipping
+  // consecutive unit separator characters at the beginning of the data.
+  while ((current = data[++i])) {
+    if (previous) {
+      if (current === unit) {
+        if (--n <= 0) break
+      }
+      text += current
+    } else if (current === unit) {
+      n++
+    } else {
+      previous = current || true
+    }
+  }
+
+  if (obj) obj.i = i + 1
+
+  let [value, state] = text.split(etx)
+  if (!state) {
+    if (previous === '"') return text
+
+    if (previous === "#") return utils.rel.ify(text)
+
+    if (previous === "+") {
+      if (text.length === 0) return true
+
+      return parseFloat(text)
+    }
+
+    if (previous === "-") return false
+
+    if (previous === " ") return null
+  } else {
+    state = parseFloat(state)
+    // If state was found then return an array.
+    if (previous === '"') return [value, state]
+
+    if (previous === "#") return [utils.rel.ify(value), state]
+
+    if (previous === "+") {
+      if (value.length === 0) return [true, state]
+
+      return [parseFloat(value), state]
+    }
+
+    if (previous === "-") return [false, state]
+
+    if (previous === " ") return [null, state]
+  }
+}
+
+export default Radisk

package/src/radix.js

@@ -0,0 +1,106 @@
+import * as utils from "./utils.js"
+
+// ASCII character for group separator.
+const group = String.fromCharCode(29)
+// ASCII character for record separator.
+const record = String.fromCharCode(30)
+
+const Radix = () => {
+  const radix = (keys, value, tree) => {
+    if (!tree) {
+      if (!radix[group]) radix[group] = {}
+      tree = radix[group]
+    }
+    if (!keys) return tree
+
+    let i = 0
+    let tmp = {}
+    let key = keys[i]
+    const max = keys.length - 1
+    const noValue = typeof value === "undefined"
+    // Find a matching value using the shortest string from keys.
+    let found = tree[key]
+    while (!found && i < max) {
+      key += keys[++i]
+      found = tree[key]
+    }
+
+    if (!found) {
+      // If not found from the provided keys try matching with an existing key.
+      const result = utils.obj.map(tree, (hasValue, hasKey) => {
+        let j = 0
+        let matchingKey = ""
+        while (hasKey[j] === keys[j]) {
+          matchingKey += hasKey[j++]
+        }
+        if (matchingKey) {
+          if (noValue) {
+            // matchingKey has to be as long as the original keys when reading.
+            if (j <= max) return
+
+            tmp[hasKey.slice(j)] = hasValue
+            return hasValue
+          }
+
+          let replace = {
+            [hasKey.slice(j)]: hasValue,
+            [keys.slice(j)]: {[record]: value},
+          }
+          tree[matchingKey] = {[group]: replace}
+          delete tree[hasKey]
+          return true
+        }
+      })
+      if (!result) {
+        if (noValue) return
+
+        if (!tree[key]) tree[key] = {}
+        tree[key][record] = value
+      } else if (noValue) {
+        return tmp
+      }
+    } else if (i === max) {
+      // If no value use the key provided to return a whole group or record.
+      if (noValue) {
+        // If an individual record isn't found then return the whole group.
+        return typeof found[record] === "undefined"
+          ? found[group]
+          : found[record]
+      }
+      // Otherwise create a new record at the provided key for value.
+      found[record] = value
+    } else {
+      // Found at a shorter key, try again.
+      if (!found[group] && !noValue) found[group] = {}
+      return radix(keys.slice(++i), value, found[group])
+    }
+  }
+  return radix
+}
+
+Radix.map = function map(radix, cb, opt, pre) {
+  if (!pre) pre = []
+  var tree = radix[group] || radix
+  var keys = Object.keys(tree).sort()
+  var u
+
+  for (let i = 0; i < keys.length; i++) {
+    let key = keys[i]
+    let found = tree[key]
+    let tmp = found[record]
+    if (typeof tmp !== "undefined") {
+      tmp = cb(tmp, pre.join("") + key, key, pre)
+      if (typeof tmp !== "undefined") return tmp
+    } else if (opt) {
+      cb(u, pre.join(""), key, pre)
+    }
+    if (found[group]) {
+      pre.push(key)
+      tmp = map(found[group], cb, opt, pre)
+      if (typeof tmp !== "undefined") return tmp
+      pre.pop()
+    }
+  }
+}
+
+export default Radix

package/src/sea-utils.js

@@ -0,0 +1,65 @@
+import SafeBuffer from "./buffer.js"
+
+const isNode = typeof document === "undefined"
+const crypto = isNode
+  ? (await import("node:crypto")).webcrypto
+  : globalThis.crypto
+export const subtle = crypto.subtle
+
+export const stringify = data => {
+  return typeof data === "string" ? data : JSON.stringify(data)
+}
+
+export const parse = text => {
+  try {
+    return JSON.parse(text)
+  } catch {
+    return text
+  }
+}
+
+export const random = length => {
+  const array = new Uint8Array(SafeBuffer.alloc(length))
+  return SafeBuffer.from(crypto.getRandomValues(array))
+}
+
+export const jwk = (pub, priv) => {
+  const [x, y] = pub.split(".")
+  const ops = priv ? ["sign"] : ["verify"]
+  return {
+    kty: "EC",
+    crv: "P-256",
+    x: x,
+    y: y,
+    d: priv,
+    ext: true,
+    key_ops: ops,
+  }
+}
+
+export const sha256 = async data => {
+  const hash = await subtle.digest(
+    {name: "SHA-256"},
+    new TextEncoder().encode(stringify(data)),
+  )
+  return SafeBuffer.from(hash)
+}
+
+export const aeskey = async (key, salt) => {
+  const combined = key + salt.toString("utf8")
+  const hash = SafeBuffer.from(await sha256(combined), "binary")
+  const jwk = keyToJwk(hash)
+  return await subtle.importKey("jwk", jwk, {name: "AES-GCM"}, false, [
+    "encrypt",
+    "decrypt",
+  ])
+}
+
+const keyToJwk = key => {
+  const k = key
+    .toString("base64")
+    .replace(/\+/g, "-")
+    .replace(/\//g, "_")
+    .replace(/\=/g, "")
+  return {kty: "oct", k: k, ext: false, alg: "A256GCM"}
+}