@fireproof/vendor 1.1.1 → 1.2.0

package/src/cborg/bench/json.js

@@ -0,0 +1,124 @@
+// can be run in a browser with `polendina --runner=bare-sync --timeout 6000 --cleanup bench.js`
+// with additional dependencies for cborg installed here
+
+import assert from 'assert'
+import { garbage } from 'ipld-garbage'
+import { decode, encode } from '../lib/json/json.js'
+
+let writebuf = ''
+const write = process.stdout
+  ? process.stdout.write.bind(process.stdout)
+  : (str) => {
+      writebuf += str
+      if (str.endsWith('\n')) {
+        console.log(writebuf.replace(/\n$/, ''))
+        writebuf = ''
+      }
+    }
+
+const textEncoder = new TextEncoder()
+const textDecoder = new TextDecoder()
+
+function jencode (obj) {
+  return textEncoder.encode(JSON.stringify(obj))
+}
+
+function jdecode (buf) {
+  return JSON.parse(textDecoder.decode(buf))
+}
+
+function runWith (description, count, targetTime, size, options) {
+  const fixtures = []
+
+  console.log(`${description} @ ${count.toLocaleString()}`)
+  for (let i = 0; i < count; i++) {
+    const obj = garbage(size, options)
+    const cbyts = encode(obj)
+    /*
+    const jbyts = jencode(obj)
+    if (Buffer.compare(Buffer.from(cbyts), jbyts) !== 0) {
+      console.log(`mismatch for obj: ${JSON.stringify(obj)}`)
+      console.log('\tc> ', Buffer.from(cbyts).toString('utf8'))
+      console.log('\tj> ', Buffer.from(jbyts).toString('utf8'))
+    }
+    */
+    if (cbyts.length <= size * 2) {
+      fixtures.push([obj, cbyts])
+    }
+  }
+  const avgSize = Math.round(fixtures.reduce((p, c) => p + c[1].length, 0) / fixtures.length)
+
+  const enc = (encoder) => {
+    for (const [obj, byts] of fixtures) {
+      const ebyts = encoder(obj)
+      if (byts.length !== ebyts.length) {
+        throw new Error('bork')
+      }
+    }
+    return fixtures.length
+  }
+
+  const bench = (bfn) => {
+    const start = Date.now()
+    let opcount = 0
+    do {
+      opcount += bfn()
+    } while (Date.now() - start < targetTime)
+    const ops = Math.round(opcount / ((Date.now() - start) / 1000))
+    return ops
+  }
+
+  const dec = (decoder) => {
+    for (const [obj, byts] of fixtures) {
+      let cobj
+      try {
+        cobj = decoder(byts)
+      } catch (e) {
+        console.log('Failed to decode:', Buffer.from(byts).toString('utf8'))
+        throw e
+      }
+      if (obj != null && typeof obj === 'object') {
+        assert.deepStrictEqual(Object.keys(cobj).length, Object.keys(obj).length)
+      } else {
+        assert.deepStrictEqual(obj, cobj)
+      }
+    }
+    return fixtures.length
+  }
+
+  const cmp = (desc, cbfn, bofn) => {
+    write(`\t${desc} (avg ${avgSize.toLocaleString()} b):`)
+    const cborgOps = bench(cbfn)
+    write(` cborg @ ${cborgOps.toLocaleString()} op/s`)
+    const jOps = bench(bofn)
+    write(` / JSON @ ${jOps.toLocaleString()} op/s`)
+    const percent = Math.round((cborgOps / jOps) * 1000) / 10
+    write(` = ${(percent).toLocaleString()} %\n`)
+    return percent
+  }
+
+  return [
+    cmp('encode', () => enc(encode), () => enc(jencode)),
+    cmp('decode', () => dec(decode), () => dec(jdecode))
+  ]
+}
+
+const targetTime = 1000
+const accum = []
+accum.push(runWith('rnd-100', 1000, targetTime, 100, { weights: { CID: 0, bytes: 0 } }))
+accum.push(runWith('rnd-300', 1000, targetTime, 300, { weights: { CID: 0, bytes: 0 } }))
+accum.push(runWith('rnd-nomap-300', 1000, targetTime, 300, { weights: { CID: 0, bytes: 0, map: 0 } }))
+accum.push(runWith('rnd-nolist-300', 1000, targetTime, 300, { weights: { CID: 0, bytes: 0, list: 0 } }))
+accum.push(runWith('rnd-nofloat-300', 1000, targetTime, 300, { weights: { CID: 0, bytes: 0, float: 0 } }))
+accum.push(runWith('rnd-nomaj7-300', 1000, targetTime, 300, { weights: { CID: 0, bytes: 0, float: 0, null: 0, boolean: 0 } }))
+accum.push(runWith('rnd-nostr-300', 1000, targetTime, 300, { weights: { CID: 0, bytes: 0, string: 0 } }))
+accum.push(runWith('rnd-nostrbyts-300', 1000, targetTime, 300, { weights: { CID: 0, bytes: 0, string: 0 } }))
+accum.push(runWith('rnd-1000', 1000, targetTime, 1000, { weights: { CID: 0, bytes: 0 } }))
+accum.push(runWith('rnd-2000', 1000, targetTime, 2000, { weights: { CID: 0, bytes: 0 } }))
+accum.push(runWith('rnd-fil-100', 1000, targetTime, 100, { weights: { float: 0, map: 0, CID: 0, bytes: 0 } }))
+accum.push(runWith('rnd-fil-300', 1000, targetTime, 300, { weights: { float: 0, map: 0, CID: 0, bytes: 0 } }))
+accum.push(runWith('rnd-fil-500', 1000, targetTime, 500, { weights: { float: 0, map: 0, CID: 0, bytes: 0 } }))
+accum.push(runWith('rnd-fil-1000', 1000, targetTime, 1000, { weights: { float: 0, map: 0, CID: 0, bytes: 0 } }))
+accum.push(runWith('rnd-fil-2000', 1000, targetTime, 2000, { weights: { float: 0, map: 0, CID: 0, bytes: 0 } }))
+console.log(`Avg encode: ${Math.round(accum.reduce((p, c) => p + c[0], 0) / accum.length).toLocaleString()} %`)
+console.log(`Avg decode: ${Math.round(accum.reduce((p, c) => p + c[1], 0) / accum.length).toLocaleString()} %`)

package/src/cborg/bench/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "cborg-bench",
+  "type": "module",
+  "dependencies": {
+    "borc": "^2.1.2",
+    "ipld-garbage": "^3.0.3"
+  }
+}

package/src/cborg/cborg/bin.js

@@ -0,0 +1,189 @@
+#!/usr/bin/env node
+
+import process from 'process'
+import { decode, encode } from "@fireproof/vendor/cborg"
+import { tokensToDiagnostic, fromDiag } from './diagnostic.js'
+import { fromHex as _fromHex, toHex } from "@fireproof/vendor/cborg/utils"
+
+/**
+ * @param {number} code
+ */
+function usage (code) {
+  console.error('Usage: cborg <command> <args>')
+  console.error('Valid commands:')
+  console.error('\tbin2diag [--width <width>] [binary input]')
+  console.error('\tbin2hex [binary input]')
+  console.error('\tbin2json [--pretty] [binary input]')
+  console.error('\tdiag2bin [diagnostic input]')
+  console.error('\tdiag2hex [diagnostic input]')
+  console.error('\tdiag2json [--pretty] [diagnostic input]')
+  console.error('\thex2bin [hex input]')
+  console.error('\thex2diag [--width <width>] [hex input]')
+  console.error('\thex2json [--pretty] [hex input]')
+  console.error('\tjson2bin \'[json input]\'')
+  console.error('\tjson2diag [--width <width>] \'[json input]\'')
+  console.error('\tjson2hex \'[json input]\'')
+  console.error('Input may either be supplied as an argument or piped via stdin')
+  process.exit(code || 0)
+}
+
+async function fromStdin () {
+  const chunks = []
+  for await (const chunk of process.stdin) {
+    chunks.push(chunk)
+  }
+  return Buffer.concat(chunks)
+}
+
+/**
+ * @param {string} str
+ * @returns {Uint8Array}
+ */
+function fromHex (str) {
+  str = str.replace(/\r?\n/g, '') // let's be charitable
+  /* c8 ignore next 3 */
+  if (!(/^([0-9a-f]{2})*$/i).test(str)) {
+    throw new Error('Input string is not hexadecimal format')
+  }
+  return _fromHex(str)
+}
+
+function argvPretty () {
+  const argv = process.argv.filter((s) => s !== '--pretty')
+  const pretty = argv.length !== process.argv.length
+  return { argv, pretty }
+}
+
+function argvWidth () {
+  const widthIndex = process.argv.findIndex((s) => s === '--width')
+  if (widthIndex <= 0 || widthIndex + 1 >= process.argv.length) {
+    return { argv: process.argv.filter((s) => s !== '--width'), width: undefined }
+  }
+  const width = parseInt(process.argv[widthIndex + 1], 10)
+  if (!width || width < 20) {
+    return { argv: process.argv.filter((s) => s !== '--width'), width: undefined }
+  }
+  const argv = process.argv
+  argv.splice(widthIndex, 2)
+  return { argv, width }
+}
+
+async function run () {
+  const cmd = process.argv[2]
+
+  switch (cmd) {
+    case 'help': {
+      return usage(0)
+    }
+
+    case 'bin2diag': {
+      /* c8 ignore next 1 */
+      const { argv, width } = argvWidth()
+      const bin = argv.length < 4 ? (await fromStdin()) : new TextEncoder().encode(argv[3])
+      for (const line of tokensToDiagnostic(bin, width)) {
+        console.log(line)
+      }
+      return
+    }
+
+    case 'bin2hex': {
+      // this is really nothing to do with cbor.. just handy
+      /* c8 ignore next 1 */
+      const bin = process.argv.length < 4 ? (await fromStdin()) : new TextEncoder().encode(process.argv[3])
+      return console.log(toHex(bin))
+    }
+
+    case 'bin2json': {
+      const { argv, pretty } = argvPretty()
+      /* c8 ignore next 1 */
+      const bin = argv.length < 4 ? (await fromStdin()) : new TextEncoder().encode(argv[3])
+      return console.log(JSON.stringify(decode(bin), undefined, pretty ? 2 : undefined))
+    }
+
+    case 'diag2bin': {
+      // no coverage on windows for non-stdin input
+      /* c8 ignore next 1 */
+      const bin = fromDiag(process.argv.length < 4 ? (await fromStdin()).toString() : process.argv[3])
+      return process.stdout.write(bin)
+    }
+
+    case 'diag2hex': {
+      // no coverage on windows for non-stdin input
+      /* c8 ignore next 1 */
+      const bin = fromDiag(process.argv.length < 4 ? (await fromStdin()).toString() : process.argv[3])
+      return console.log(toHex(bin))
+    }
+
+    case 'diag2json': {
+      const { argv, pretty } = argvPretty()
+      // no coverage on windows for non-stdin input
+      /* c8 ignore next 1 */
+      const bin = fromDiag(argv.length < 4 ? (await fromStdin()).toString() : argv[3])
+      return console.log(JSON.stringify(decode(bin), undefined, pretty ? 2 : undefined))
+    }
+
+    case 'hex2bin': {
+      // this is really nothing to do with cbor.. just handy
+      const bin = fromHex(process.argv.length < 4 ? (await fromStdin()).toString() : process.argv[3])
+      return process.stdout.write(bin)
+    }
+
+    case 'hex2diag': {
+      const { argv, width } = argvWidth()
+      const bin = fromHex(argv.length < 4 ? (await fromStdin()).toString() : argv[3])
+      for (const line of tokensToDiagnostic(bin, width)) {
+        console.log(line)
+      }
+      return
+    }
+
+    case 'hex2json': {
+      const { argv, pretty } = argvPretty()
+      const bin = fromHex(argv.length < 4 ? (await fromStdin()).toString() : argv[3])
+      return console.log(JSON.stringify(decode(bin), undefined, pretty ? 2 : undefined))
+    }
+
+    case 'json2bin': {
+      const inp = process.argv.length < 4 ? (await fromStdin()).toString() : process.argv[3]
+      const obj = JSON.parse(inp)
+      return process.stdout.write(encode(obj))
+    }
+
+    case 'json2diag': {
+      const { argv, width } = argvWidth()
+      const inp = argv.length < 4 ? (await fromStdin()).toString() : argv[3]
+      const obj = JSON.parse(inp)
+      for (const line of tokensToDiagnostic(encode(obj), width)) {
+        console.log(line)
+      }
+      return
+    }
+
+    case 'json2hex': {
+      const inp = process.argv.length < 4 ? (await fromStdin()).toString() : process.argv[3]
+      const obj = JSON.parse(inp)
+      return console.log(toHex(encode(obj)))
+    }
+
+    default: { // no, or unknown cmd
+      // this is a dirty hack to allow import of this package by the tests
+      // for inclusion in ipjs bundling, but to silently ignore it so we don't
+      // print usage and exit(1).
+      if (process.argv.findIndex((a) => a.endsWith('mocha')) === -1) {
+        if (cmd) {
+          console.error(`Unknown command: '${cmd}'`)
+        }
+        usage(1)
+      }
+    }
+  }
+}
+
+run().catch((err) => {
+  /* c8 ignore next 2 */
+  console.error(err)
+  process.exit(1)
+})
+
+// for ipjs, to get it to compile
+export default true

package/src/cborg/cborg/common.js

@@ -0,0 +1,28 @@
+// const decodeErrPrefix = 'CBOR decode error:'
+// const encodeErrPrefix = 'CBOR encode error:'
+
+const uintMinorPrefixBytes = []
+uintMinorPrefixBytes[23] = 1
+uintMinorPrefixBytes[24] = 2
+uintMinorPrefixBytes[25] = 3
+uintMinorPrefixBytes[26] = 5
+uintMinorPrefixBytes[27] = 9
+
+// /**
+//  * @param {Uint8Array} data
+//  * @param {number} pos
+//  * @param {number} need
+//  * @param {string} decodeErrPrefix
+//  */
+// function assertEnoughData (data, pos, need, decodeErrPrefix) {
+//   if (data.length - pos < need) {
+//     throw new Error(`${decodeErrPrefix} not enough data for type`)
+//   }
+// }
+
+export {
+  // decodeErrPrefix,
+  // encodeErrPrefix,
+  uintMinorPrefixBytes
+  // assertEnoughData
+}

package/src/cborg/cborg/decode.js

@@ -0,0 +1,211 @@
+import { jump, quick, Type } from "@fireproof/vendor/cborg/utils"
+
+/**
+ * @typedef {import('cborg/utils').Token} Token
+ * @typedef {import('../interface').DecodeOptions} DecodeOptions
+ * @typedef {import('../interface').DecodeTokenizer} DecodeTokenizer
+ * @typedef {import('../interface').DecodeFunction} DecodeFunction
+ */
+
+const /** @type {DecodeOptions} */defaultDecodeOptions = {
+  strict: false,
+  allowIndefinite: true,
+  allowUndefined: true,
+  allowBigInt: true,
+  decodeErrPrefix: 'CBOR decode error:'
+}
+
+/**
+ * @implements {DecodeTokenizer}
+ */
+class Tokeniser {
+  /**
+   * @param {Uint8Array} data
+   * @param {DecodeOptions} options
+   */
+  constructor (data, options) {
+    this._pos = 0
+    this.data = data
+    this.options = options
+    this.jump = jump(options.decodeErrPrefix)
+  }
+
+  pos () {
+    return this._pos
+  }
+
+  done () {
+    return this._pos >= this.data.length
+  }
+
+  next () {
+    const byt = this.data[this._pos]
+    let token = quick[byt]
+    if (token === undefined) {
+      const decoder = this.jump[byt]
+      /* c8 ignore next 4 */
+      // if we're here then there's something wrong with our jump or quick lists!
+      if (!decoder) {
+        throw new Error(`${this.options.decodeErrPrefix} no decoder for major type ${byt >>> 5} (byte 0x${byt.toString(16).padStart(2, '0')})`)
+      }
+      const minor = byt & 31
+      token = decoder(this.data, this._pos, minor, this.options)
+    }
+    // @ts-ignore we get to assume encodedLength is set (crossing fingers slightly)
+    this._pos += token.encodedLength
+    return token
+  }
+}
+
+const DONE = Symbol.for('DONE')
+const BREAK = Symbol.for('BREAK')
+
+/**
+ * @param {Token} token
+ * @param {DecodeTokenizer} tokeniser
+ * @param {DecodeOptions} options
+ * @returns {any|BREAK|DONE}
+ */
+function tokenToArray (token, tokeniser, options) {
+  const arr = []
+  for (let i = 0; i < token.value; i++) {
+    const value = tokensToObject(tokeniser, options)
+    if (value === BREAK) {
+      if (token.value === Infinity) {
+        // normal end to indefinite length array
+        break
+      }
+      throw new Error(`${options.decodeErrPrefix} got unexpected break to lengthed array`)
+    }
+    if (value === DONE) {
+      throw new Error(`${options.decodeErrPrefix} found array but not enough entries (got ${i}, expected ${token.value})`)
+    }
+    arr[i] = value
+  }
+  return arr
+}
+
+/**
+ * @param {Token} token
+ * @param {DecodeTokenizer} tokeniser
+ * @param {DecodeOptions} options
+ * @returns {any|BREAK|DONE}
+ */
+function tokenToMap (token, tokeniser, options) {
+  const useMaps = options.useMaps === true
+  const obj = useMaps ? undefined : {}
+  const m = useMaps ? new Map() : undefined
+  for (let i = 0; i < token.value; i++) {
+    const key = tokensToObject(tokeniser, options)
+    if (key === BREAK) {
+      if (token.value === Infinity) {
+        // normal end to indefinite length map
+        break
+      }
+      throw new Error(`${options.decodeErrPrefix} got unexpected break to lengthed map`)
+    }
+    if (key === DONE) {
+      throw new Error(`${options.decodeErrPrefix} found map but not enough entries (got ${i} [no key], expected ${token.value})`)
+    }
+    if (useMaps !== true && typeof key !== 'string') {
+      throw new Error(`${options.decodeErrPrefix} non-string keys not supported (got ${typeof key})`)
+    }
+    if (options.rejectDuplicateMapKeys === true) {
+      // @ts-ignore
+      if ((useMaps && m.has(key)) || (!useMaps && (key in obj))) {
+        throw new Error(`${options.decodeErrPrefix} found repeat map key "${key}"`)
+      }
+    }
+    const value = tokensToObject(tokeniser, options)
+    if (value === DONE) {
+      throw new Error(`${options.decodeErrPrefix} found map but not enough entries (got ${i} [no value], expected ${token.value})`)
+    }
+    if (useMaps) {
+      // @ts-ignore TODO reconsider this .. maybe needs to be strict about key types
+      m.set(key, value)
+    } else {
+      // @ts-ignore TODO reconsider this .. maybe needs to be strict about key types
+      obj[key] = value
+    }
+  }
+  // @ts-ignore c'mon man
+  return useMaps ? m : obj
+}
+
+/**
+ * @param {DecodeTokenizer} tokeniser
+ * @param {DecodeOptions} options
+ * @returns {any|BREAK|DONE}
+ */
+function tokensToObject (tokeniser, options) {
+  // should we support array as an argument?
+  // check for tokenIter[Symbol.iterator] and replace tokenIter with what that returns?
+  if (tokeniser.done()) {
+    return DONE
+  }
+
+  const token = tokeniser.next()
+
+  if (token.type.equals(Type.break)) {
+    return BREAK
+  }
+
+  if (token.type.terminal) {
+    return token.value
+  }
+
+  if (token.type.equals(Type.array)) {
+    return tokenToArray(token, tokeniser, options)
+  }
+
+  if (token.type.equals(Type.map)) {
+    return tokenToMap(token, tokeniser, options)
+  }
+
+  if (token.type.equals(Type.tag)) {
+    if (options.tags && typeof options.tags[token.value] === 'function') {
+      const tagged = tokensToObject(tokeniser, options)
+      return options.tags[token.value](tagged)
+    }
+    throw new Error(`${options.decodeErrPrefix} tag not supported (${token.value})`)
+  }
+  /* c8 ignore next */
+  throw new Error('unsupported')
+}
+
+/**
+ * @param {Uint8Array} data
+ * @param {DecodeOptions} [options]
+ * @returns {[any, Uint8Array]}
+ */
+function decodeFirst (data, options) {
+  options = Object.assign({}, defaultDecodeOptions, options)
+  if (!(data instanceof Uint8Array)) {
+    throw new Error(`${options.decodeErrPrefix} data to decode must be a Uint8Array`)
+  }
+  const tokeniser = options.tokenizer || new Tokeniser(data, options)
+  const decoded = tokensToObject(tokeniser, options)
+  if (decoded === DONE) {
+    throw new Error(`${options.decodeErrPrefix} did not find any content to decode`)
+  }
+  if (decoded === BREAK) {
+    throw new Error(`${options.decodeErrPrefix} got unexpected break`)
+  }
+  return [decoded, data.subarray(tokeniser.pos())]
+}
+
+/**
+ * @param {Uint8Array} data
+ * @param {DecodeOptions} [options]
+ * @returns {any}
+ */
+function decode (data, options) {
+  options = Object.assign({}, defaultDecodeOptions, options)
+  const [decoded, remainder] = decodeFirst(data, options)
+  if (remainder.length > 0) {
+    throw new Error(`${options.decodeErrPrefix} too many terminals, data makes no sense:${remainder.length}`)
+  }
+  return decoded
+}
+
+export { Tokeniser, tokensToObject, decode, decodeFirst }