@exodus/bytes 1.0.0-rc.1 → 1.0.0-rc.11

package/README.md

@@ -1,3 +1,173 @@
-# bytes
+# `@exodus/bytes`
 
-Data structures handling
+`Uint8Array` conversion to and from `base64`, `base32`, `base58`, `hex`, `utf8`, `utf16`, `bech32` and `wif`
+
+And a [`TextEncoder` / `TextDecoder` polyfill](#textencoder--textdecoder-polyfill)
+
+## Strict
+
+Performs proper input validation, ensures no garbage-in-garbage-out
+
+Tested on Node.js, Deno, Bun, browsers (including Servo), Hermes, QuickJS and barebone engines in CI [(how?)](https://github.com/ExodusMovement/test#exodustest)
+
+## Fast
+
+* `10-20x` faster than `Buffer` polyfill
+* `2-10x` faster than `iconv-lite`
+
+The above was for the js fallback
+
+It's up to `100x` when native impl is available \
+e.g. in `utf8fromString` on Hermes / React Native or `fromHex` in Chrome
+
+Also:
+* `3-8x` faster than `bs58`
+* `10-30x` faster than `@scure/base` (or `>100x` on Node.js <25)
+* Faster in `utf8toString` / `utf8fromString` than `Buffer` or `TextDecoder` / `TextEncoder` on Node.js
+
+See [Performance](./Performance.md) for more info
+
+## TextEncoder / TextDecoder polyfill
+
+```js
+import { TextDecoder, TextEncoder } from '@exodus/bytes/encoding.js'
+```
+
+Less than half the bundle size of [text-encoding](https://npmjs.com/text-encoding), [whatwg-encoding](https://npmjs.com/whatwg-encoding) or [iconv-lite](https://npmjs.com/iconv-lite) (gzipped or not), and [is much faster](#fast).
+See also [lite version](#lite-version).
+
+Spec compliant, passing WPT and covered with extra tests.
+
+Moreover, tests for this library uncovered [bugs in all major implementations](https://docs.google.com/spreadsheets/d/1pdEefRG6r9fZy61WHGz0TKSt8cO4ISWqlpBN5KntIvQ/edit).
+
+[Faster than Node.js native implementation on Node.js](https://github.com/nodejs/node/issues/61041#issuecomment-3649242024).
+
+### Caveat: `TextDecoder` / `TextEncoder` APIs are lossy by default per spec
+
+_These are only provided as a compatibility layer, prefer hardened APIs instead in new code._
+
+ * `TextDecoder` can (and should) be used with `{ fatal: true }` option for all purposes demanding correctness / lossless transforms
+
+ * `TextEncoder` does not support a fatal mode per spec, it always performs replacement.
+
+   That is not suitable for hashing, cryptography or consensus applications.\
+   Otherwise there would be non-equal strings with equal signatures and hashes — the collision is caused by the lossy transform of a JS string to bytes.
+   Those also survive e.g. `JSON.stringify`/`JSON.parse` or being sent over network.
+
+   Use strict APIs in new applications, see `utf8fromString` / `utf16fromString` below.\
+   Those throw on non-well-formed strings by default.
+
+### Lite version
+
+If you don't need support for legacy multi-byte encodings, you can use the lite import:
+```js
+import { TextDecoder, TextEncoder } from '@exodus/bytes/encoding-lite.js'
+```
+
+This reduces the bundle size 9x:\
+from 91 KiB gzipped for `@exodus/bytes/encoding.js` to 10 KiB gzipped for `@exodus/bytes/encoding-lite.js`.\
+(For comparison, `text-encoding` module is 190 KiB gzipped, and `iconv-lite` is 194 KiB gzipped).
+
+It still supports `utf-8`, `utf-16le`, `utf-16be` and all single-byte encodings specified by the spec,
+the only difference is support for legacy multi-byte encodings.
+
+See [the list of encodings](https://encoding.spec.whatwg.org/#names-and-labels).
+
+## API
+
+### `@exodus/bytes/utf8.js`
+
+##### `utf8fromString(str, format = 'uint8')`
+##### `utf8fromStringLoose(str, format = 'uint8')`
+##### `utf8toString(arr)`
+##### `utf8toStringLoose(arr)`
+
+### `@exodus/bytes/utf16.js`
+
+##### `utf16fromString(str, format = 'uint16')`
+##### `utf16fromStringLoose(str, format = 'uint16')`
+##### `utf16toString(arr, 'uint16')`
+##### `utf16toStringLoose(arr, 'uint16')`
+
+### `@exodus/bytes/single-byte.js`
+
+##### `createSinglebyteDecoder(encoding, loose = false)`
+
+Create a decoder for a supported one-byte `encoding`.
+
+Returns a function `decode(arr)` that decodes bytes to a string.
+
+### `@exodus/bytes/multi-byte.js`
+
+##### `createMultibyteDecoder(encoding, loose = false)`
+
+Create a decoder for a supported legacy multi-byte `encoding`.
+
+Returns a function `decode(arr, stream = false)` that decodes bytes to a string.
+
+That function will have state while `stream = true` is used.
+
+##### `windows1252toString(arr)`
+
+Decode `windows-1252` bytes to a string.
+
+Also supports `ascii` and `latin-1` as those are strict subsets of `windows-1252`.
+
+There is no loose variant for this encoding, all bytes can be decoded.
+
+Same as `windows1252toString = createSinglebyteDecoder('windows-1252')`.
+
+### `@exodus/bytes/hex.js`
+
+##### `toHex(arr)`
+##### `fromHex(string)`
+
+### `@exodus/bytes/base64.js`
+
+##### `toBase64(arr, { padding = true })`
+##### `toBase64url(arr, { padding = false })`
+##### `fromBase64(str, { format = 'uint8', padding = 'both' })`
+##### `fromBase64url(str, { format = 'uint8', padding = false })`
+##### `fromBase64any(str, { format = 'uint8', padding = 'both' })`
+
+### `@exodus/bytes/base32.js`
+
+##### `toBase32(arr, { padding = false })`
+##### `toBase32hex(arr, { padding = false })`
+##### `fromBase32(str, { format = 'uint8', padding = 'both' })`
+##### `fromBase32hex(str, { format = 'uint8', padding = 'both' })`
+
+### `@exodus/bytes/bech32.js`
+
+##### `getPrefix(str, limit = 90)`
+##### `toBech32(prefix, bytes, limit = 90)`
+##### `fromBech32(str, limit = 90)`
+##### `toBech32m(prefix, bytes, limit = 90)`
+##### `fromBech32m(str, limit = 90)`
+
+### `@exodus/bytes/base58.js`
+
+##### `toBase58(arr)`
+##### `fromBase58(str, format = 'uint8')`
+
+##### `toBase58xrp(arr)`
+##### `fromBase58xrp(str, format = 'uint8')`
+
+### `@exodus/bytes/base58check.js`
+
+##### `async toBase58check(arr)`
+##### `toBase58checkSync(arr)`
+##### `async fromBase58check(str, format = 'uint8')`
+##### `fromBase58checkSync(str, format = 'uint8')`
+##### `makeBase58check(hashAlgo, hashAlgoSync)`
+
+### `@exodus/bytes/wif.js`
+
+##### `async fromWifString(string, version)`
+##### `fromWifStringSync(string, version)`
+##### `async toWifString({ version, privateKey, compressed })`
+##### `toWifStringSync({ version, privateKey, compressed })`
+
+## License
+
+[MIT](./LICENSE)

package/array.js

@@ -2,7 +2,7 @@ import { assertTypedArray } from './assert.js'
 
 const { Buffer } = globalThis // Buffer is optional
 
-export function fromTypedArray(arr, format) {
+export function typedView(arr, format) {
   assertTypedArray(arr)
   switch (format) {
     case 'uint8':

package/assert.js

@@ -1,7 +1,3 @@
-export function assert(x, msg) {
-  if (!x) throw new Error(msg || 'Assertion failed')
-}
-
 export function assertEmptyRest(rest) {
   if (Object.keys(rest).length > 0) throw new TypeError('Unexpected extra options')
 }
@@ -12,10 +8,18 @@ const makeMessage = (name, extra) => `Expected${name ? ` ${name} to be` : ''} an
 const TypedArray = Object.getPrototypeOf(Uint8Array)
 
 export function assertTypedArray(arr) {
-  assert(arr instanceof TypedArray, 'Expected a TypedArray instance')
+  if (arr instanceof TypedArray) return
+  throw new TypeError('Expected a TypedArray instance')
 }
 
-export function assertUint8(arr, { name, length, ...rest } = {}) {
+export function assertUint8(arr, options) {
+  if (!options) {
+    // fast path
+    if (arr instanceof Uint8Array) return
+    throw new TypeError('Expected an Uint8Array')
+  }
+
+  const { name, length, ...rest } = options
   assertEmptyRest(rest)
   if (arr instanceof Uint8Array && (length === undefined || arr.length === length)) return
   throw new TypeError(makeMessage(name, length === undefined ? '' : ` of size ${Number(length)}`))

package/base32.js

@@ -0,0 +1,40 @@
+import { assertEmptyRest } from './assert.js'
+import { typedView } from './array.js'
+import * as js from './fallback/base32.js'
+
+// See https://datatracker.ietf.org/doc/html/rfc4648
+
+// 8 chars per 5 bytes
+
+const { E_PADDING } = js
+
+export const toBase32 = (arr, { padding = false } = {}) => js.toBase32(arr, false, padding)
+export const toBase32hex = (arr, { padding = false } = {}) => js.toBase32(arr, true, padding)
+
+// By default, valid padding is accepted but not required
+export function fromBase32(str, options) {
+  if (!options) return fromBase32common(str, false, 'both', 'uint8', null)
+  const { format = 'uint8', padding = 'both', ...rest } = options
+  return fromBase32common(str, false, padding, format, rest)
+}
+
+export function fromBase32hex(str, options) {
+  if (!options) return fromBase32common(str, true, 'both', 'uint8', null)
+  const { format = 'uint8', padding = 'both', ...rest } = options
+  return fromBase32common(str, true, padding, format, rest)
+}
+
+function fromBase32common(str, isBase32Hex, padding, format, rest) {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  if (rest !== null) assertEmptyRest(rest)
+
+  if (padding === true) {
+    if (str.length % 8 !== 0) throw new SyntaxError(E_PADDING)
+  } else if (padding === false) {
+    if (str.endsWith('=')) throw new SyntaxError('Did not expect padding in base32 input')
+  } else if (padding !== 'both') {
+    throw new TypeError('Invalid padding option')
+  }
+
+  return typedView(js.fromBase32(str, isBase32Hex), format)
+}

package/base58.js

@@ -0,0 +1,220 @@
+import { typedView } from './array.js'
+import { assertUint8 } from './assert.js'
+import { nativeDecoder, nativeEncoder, isHermes } from './fallback/_utils.js'
+import { encodeAscii, decodeAscii } from './fallback/latin1.js'
+
+const alphabet58 = [...'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz']
+const alphabetXRP = [...'rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz']
+const codes58 = new Uint8Array(alphabet58.map((x) => x.charCodeAt(0)))
+const codesXRP = new Uint8Array(alphabetXRP.map((x) => x.charCodeAt(0)))
+
+const _0n = BigInt(0)
+const _1n = BigInt(1)
+const _8n = BigInt(8)
+const _32n = BigInt(32)
+const _58n = BigInt(58)
+const _0xffffffffn = BigInt(0xff_ff_ff_ff)
+
+let table // 15 * 82, diagonal, <1kb
+const fromMaps = new Map()
+
+const E_CHAR = 'Invalid character in base58 input'
+
+const shouldUseBigIntFrom = isHermes // faster only on Hermes, numbers path beats it on normal engines
+
+function toBase58core(arr, alphabet, codes) {
+  assertUint8(arr)
+  const length = arr.length
+  if (length === 0) return ''
+
+  const ZERO = alphabet[0]
+  let zeros = 0
+  while (zeros < length && arr[zeros] === 0) zeros++
+
+  if (length > 60) {
+    // Slow path. Can be optimized ~10%, but the main factor is /58n division anyway, so doesn't matter much
+    let x = _0n
+    for (let i = 0; i < arr.length; i++) x = (x << _8n) | BigInt(arr[i])
+
+    let out = ''
+    while (x) {
+      const d = x / _58n
+      out = alphabet[Number(x - _58n * d)] + out
+      x = d
+    }
+
+    return ZERO.repeat(zeros) + out
+  }
+
+  // We run fast mode operations only on short (<=60 bytes) inputs, via precomputation table
+  if (!table) {
+    table = []
+    let x = _1n
+    for (let i = 0; i < 15; i++) {
+      // Convert x to base 58 digits
+      const in58 = []
+      let y = x
+      while (y) {
+        const d = y / _58n
+        in58.push(Number(y - _58n * d))
+        y = d
+      }
+
+      table.push(new Uint8Array(in58))
+      x <<= _32n
+    }
+  }
+
+  const res = []
+  {
+    let j = 0
+    // We group each 4 bytes into 32-bit chunks
+    // Not using u32arr to not deal with remainder + BE/LE differences
+    for (let i = length - 1; i >= 0; i -= 4) {
+      let c
+      if (i > 2) {
+        c = (arr[i] | (arr[i - 1] << 8) | (arr[i - 2] << 16) | (arr[i - 3] << 24)) >>> 0
+      } else if (i > 1) {
+        c = arr[i] | (arr[i - 1] << 8) | (arr[i - 2] << 16)
+      } else {
+        c = i === 1 ? arr[i] | (arr[i - 1] << 8) : arr[i]
+      }
+
+      const row = table[j++]
+      if (c === 0) continue
+      const olen = res.length
+      const nlen = row.length
+      let k = 0
+      for (; k < olen; k++) res[k] += c * row[k]
+      while (k < nlen) res.push(c * row[k++])
+    }
+  }
+
+  // We can now do a single scan over regular numbers under MAX_SAFE_INTEGER
+  // Note: can't use int32 operations on them, as they are outside of 2**32 range
+  // This is faster though
+  {
+    let carry = 0
+    let i = 0
+    while (i < res.length) {
+      const c = res[i] + carry
+      carry = Math.floor(c / 58)
+      res[i++] = c - carry * 58
+    }
+
+    while (carry) {
+      const c = carry
+      carry = Math.floor(c / 58)
+      res.push(c - carry * 58)
+    }
+  }
+
+  if (nativeDecoder) {
+    const oa = new Uint8Array(res.length)
+    let j = 0
+    for (let i = res.length - 1; i >= 0; i--) oa[j++] = codes[res[i]]
+    return ZERO.repeat(zeros) + decodeAscii(oa)
+  }
+
+  let out = ''
+  for (let i = res.length - 1; i >= 0; i--) out += alphabet[res[i]]
+  return ZERO.repeat(zeros) + out
+}
+
+function fromBase58core(str, alphabet, codes, format = 'uint8') {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  const length = str.length
+  if (length === 0) return typedView(new Uint8Array(), format)
+
+  const zeroC = codes[0]
+  let zeros = 0
+  while (zeros < length && str.charCodeAt(zeros) === zeroC) zeros++
+
+  let fromMap = fromMaps.get(alphabet)
+  if (!fromMap) {
+    fromMap = new Int8Array(256).fill(-1)
+    for (let i = 0; i < 58; i++) fromMap[alphabet[i].charCodeAt(0)] = i
+    fromMaps.set(alphabet, fromMap)
+  }
+
+  const size = zeros + (((length - zeros + 1) * 3) >> 2) // 3/4 rounded up, larger than ~0.73 coef to fit everything
+  const res = new Uint8Array(size)
+  let at = size // where is the first significant byte written
+
+  if (shouldUseBigIntFrom) {
+    let x = _0n
+
+    // nativeEncoder gives a benefit here
+    if (nativeEncoder) {
+      const codes = encodeAscii(str, E_CHAR)
+      for (let i = zeros; i < length; i++) {
+        const c = fromMap[codes[i]]
+        if (c < 0) throw new SyntaxError(E_CHAR)
+        x = x * _58n + BigInt(c)
+      }
+    } else {
+      for (let i = zeros; i < length; i++) {
+        const charCode = str.charCodeAt(i)
+        const c = fromMap[charCode]
+        if (charCode > 255 || c < 0) throw new SyntaxError(E_CHAR)
+        x = x * _58n + BigInt(c)
+      }
+    }
+
+    while (x) {
+      let y = Number(x & _0xffffffffn)
+      x >>= 32n
+      res[--at] = y & 0xff
+      y >>>= 8
+      if (!x && !y) break
+      res[--at] = y & 0xff
+      y >>>= 8
+      if (!x && !y) break
+      res[--at] = y & 0xff
+      y >>>= 8
+      if (!x && !y) break
+      res[--at] = y & 0xff
+    }
+  } else {
+    for (let i = zeros; i < length; i++) {
+      const charCode = str.charCodeAt(i)
+      let c = fromMap[charCode]
+      if (charCode > 255 || c < 0) throw new SyntaxError(E_CHAR)
+
+      let k = size - 1
+      for (;;) {
+        if (c === 0 && k < at) break
+        c += 58 * res[k]
+        res[k] = c & 0xff
+        c >>>= 8
+        k--
+        // unroll a bit
+        if (c === 0 && k < at) break
+        c += 58 * res[k]
+        res[k] = c & 0xff
+        c >>>= 8
+        k--
+        if (c === 0 && k < at) break
+        c += 58 * res[k]
+        res[k] = c & 0xff
+        c >>>= 8
+        k--
+        if (c === 0 && k < at) break
+        c += 58 * res[k]
+        res[k] = c & 0xff
+        c >>>= 8
+        k--
+      }
+
+      at = k + 1
+      if (c !== 0 || at < zeros) throw new Error('Unexpected') // unreachable
+    }
+  }
+
+  return typedView(res.slice(at - zeros), format) // slice is faster for small sizes than subarray
+}
+
+export const toBase58 = (arr) => toBase58core(arr, alphabet58, codes58)
+export const fromBase58 = (str, format) => fromBase58core(str, alphabet58, codes58, format)
+export const toBase58xrp = (arr) => toBase58core(arr, alphabetXRP, codesXRP)
+export const fromBase58xrp = (str, format) => fromBase58core(str, alphabetXRP, codesXRP, format)

package/base58check.js

@@ -0,0 +1,69 @@
+import { typedView } from './array.js'
+import { assertUint8 } from './assert.js'
+import { toBase58, fromBase58 } from './base58.js'
+import { hashSync } from '@exodus/crypto/hash' // eslint-disable-line @exodus/import/no-deprecated
+
+// Note: while API is async, we use hashSync for now until we improve webcrypto perf for hash256
+// Inputs to base58 are typically very small, and that makes a difference
+
+const E_CHECKSUM = 'Invalid checksum'
+
+// checksum length is 4, i.e. only the first 4 bytes of the hash are used
+
+function encodeWithChecksum(arr, checksum) {
+  // arr type in already validated in input
+  const res = new Uint8Array(arr.length + 4)
+  res.set(arr, 0)
+  res.set(checksum.subarray(0, 4), arr.length)
+  return toBase58(res)
+}
+
+function decodeWithChecksum(str) {
+  const arr = fromBase58(str) // checks input
+  const payloadSize = arr.length - 4
+  if (payloadSize < 0) throw new Error(E_CHECKSUM)
+  return [arr.subarray(0, payloadSize), arr.subarray(payloadSize)]
+}
+
+function assertChecksum(c, r) {
+  if ((c[0] ^ r[0]) | (c[1] ^ r[1]) | (c[2] ^ r[2]) | (c[3] ^ r[3])) throw new Error(E_CHECKSUM)
+}
+
+export const makeBase58check = (hashAlgo, hashAlgoSync) => {
+  const apis = {
+    async encode(arr) {
+      assertUint8(arr)
+      return encodeWithChecksum(arr, await hashAlgo(arr))
+    },
+    async decode(str, format = 'uint8') {
+      const [payload, checksum] = decodeWithChecksum(str)
+      assertChecksum(checksum, await hashAlgo(payload))
+      return typedView(payload, format)
+    },
+  }
+  if (!hashAlgoSync) return apis
+  return {
+    ...apis,
+    encodeSync(arr) {
+      assertUint8(arr)
+      return encodeWithChecksum(arr, hashAlgoSync(arr))
+    },
+    decodeSync(str, format = 'uint8') {
+      const [payload, checksum] = decodeWithChecksum(str)
+      assertChecksum(checksum, hashAlgoSync(payload))
+      return typedView(payload, format)
+    },
+  }
+}
+
+// eslint-disable-next-line @exodus/import/no-deprecated
+const hash256sync = (x) => hashSync('sha256', hashSync('sha256', x, 'uint8'), 'uint8')
+const hash256 = hash256sync // See note at the top
+const {
+  encode: toBase58check,
+  decode: fromBase58check,
+  encodeSync: toBase58checkSync,
+  decodeSync: fromBase58checkSync,
+} = makeBase58check(hash256, hash256sync)
+
+export { toBase58check, fromBase58check, toBase58checkSync, fromBase58checkSync }