@exodus/bytes 1.0.0-rc.3 → 1.0.0-rc.5

package/README.md

@@ -1,3 +1,11 @@
-# bytes
+# `@exodus/bytes`
 
-Data structures handling
+`Uint8Array` conversion to and from `base64`, `base32`, `hex` and `utf8`
+
+[Fast](./Performance.md)
+
+Performs proper input validation
+
+## License
+
+[MIT](./LICENSE)

package/assert.js

@@ -1,7 +1,3 @@
-export function assert(x, msg) {
-  if (!x) throw new Error(msg || 'Assertion failed')
-}
-
 export function assertEmptyRest(rest) {
   if (Object.keys(rest).length > 0) throw new TypeError('Unexpected extra options')
 }
@@ -16,7 +12,14 @@ export function assertTypedArray(arr) {
   throw new TypeError('Expected a TypedArray instance')
 }
 
-export function assertUint8(arr, { name, length, ...rest } = {}) {
+export function assertUint8(arr, options) {
+  if (!options) {
+    // fast path
+    if (arr instanceof Uint8Array) return
+    throw new TypeError('Expected an Uint8Array')
+  }
+
+  const { name, length, ...rest } = options
   assertEmptyRest(rest)
   if (arr instanceof Uint8Array && (length === undefined || arr.length === length)) return
   throw new TypeError(makeMessage(name, length === undefined ? '' : ` of size ${Number(length)}`))

package/base32.js

@@ -0,0 +1,33 @@
+import { assertEmptyRest } from './assert.js'
+import { typedView } from './array.js'
+import * as js from './fallback/base32.js'
+
+// See https://datatracker.ietf.org/doc/html/rfc4648
+
+// 8 chars per 5 bytes
+
+const { E_PADDING } = js
+
+export const toBase32 = (arr, { padding = false } = {}) => js.toBase32(arr, false, padding)
+export const toBase32hex = (arr, { padding = false } = {}) => js.toBase32(arr, true, padding)
+
+// By default, valid padding is accepted but not required
+export const fromBase32 = (str, { format = 'uint8', padding = 'both', ...rest } = {}) =>
+  fromBase32common(str, false, padding, format, rest)
+export const fromBase32hex = (str, { format = 'uint8', padding = 'both', ...rest } = {}) =>
+  fromBase32common(str, true, padding, format, rest)
+
+function fromBase32common(str, isBase32Hex, padding, format, rest) {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  assertEmptyRest(rest)
+
+  if (padding === true) {
+    if (str.length % 8 !== 0) throw new SyntaxError(E_PADDING)
+  } else if (padding === false) {
+    if (str.endsWith('=')) throw new SyntaxError('Did not expect padding in base32 input')
+  } else if (padding !== 'both') {
+    throw new TypeError('Invalid padding option')
+  }
+
+  return typedView(js.fromBase32(str, isBase32Hex), format)
+}

package/base58.js

@@ -0,0 +1,212 @@
+import { typedView } from './array.js'
+import { assertUint8 } from './assert.js'
+import { nativeDecoder, nativeEncoder } from './fallback/_utils.js'
+
+const alphabet = [...'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz']
+const codes = new Uint8Array(alphabet.map((x) => x.charCodeAt(0)))
+const ZERO = alphabet[0]
+const zeroC = codes[0]
+
+const _0n = BigInt(0)
+const _1n = BigInt(1)
+const _8n = BigInt(8)
+const _32n = BigInt(32)
+const _58n = BigInt(58)
+const _0xffffffffn = BigInt(0xff_ff_ff_ff)
+
+let table // 15 * 82, diagonal, <1kb
+let fromMap
+
+const E_CHAR = 'Invalid character in base58 input'
+
+const shouldUseBigIntFrom = Boolean(globalThis.HermesInternal) // faster only on Hermes, numbers path beats it on normal engines
+
+export function toBase58(arr) {
+  assertUint8(arr)
+  const length = arr.length
+  if (length === 0) return ''
+
+  let zeros = 0
+  while (zeros < length && arr[zeros] === 0) zeros++
+
+  if (length > 60) {
+    // Slow path. Can be optimized ~10%, but the main factor is /58n division anyway, so doesn't matter much
+    let x = _0n
+    for (let i = 0; i < arr.length; i++) x = (x << _8n) | BigInt(arr[i])
+
+    let out = ''
+    while (x) {
+      const d = x / _58n
+      out = alphabet[Number(x - _58n * d)] + out
+      x = d
+    }
+
+    return ZERO.repeat(zeros) + out
+  }
+
+  // We run fast mode operations only on short (<=60 bytes) inputs, via precomputation table
+  if (!table) {
+    table = []
+    let x = _1n
+    for (let i = 0; i < 15; i++) {
+      // Convert x to base 58 digits
+      const in58 = []
+      let y = x
+      while (y) {
+        const d = y / _58n
+        in58.push(Number(y - _58n * d))
+        y = d
+      }
+
+      table.push(new Uint8Array(in58))
+      x <<= _32n
+    }
+  }
+
+  const res = []
+  {
+    let j = 0
+    // We group each 4 bytes into 32-bit chunks
+    // Not using u32arr to not deal with remainder + BE/LE differences
+    for (let i = length - 1; i >= 0; i -= 4) {
+      let c
+      if (i > 2) {
+        c = (arr[i] | (arr[i - 1] << 8) | (arr[i - 2] << 16) | (arr[i - 3] << 24)) >>> 0
+      } else if (i > 1) {
+        c = arr[i] | (arr[i - 1] << 8) | (arr[i - 2] << 16)
+      } else {
+        c = i === 1 ? arr[i] | (arr[i - 1] << 8) : arr[i]
+      }
+
+      const row = table[j++]
+      if (c === 0) continue
+      const olen = res.length
+      const nlen = row.length
+      let k = 0
+      for (; k < olen; k++) res[k] += c * row[k]
+      while (k < nlen) res.push(c * row[k++])
+    }
+  }
+
+  // We can now do a single scan over regular numbers under MAX_SAFE_INTEGER
+  // Note: can't use int32 operations on them, as they are outside of 2**32 range
+  // This is faster though
+  {
+    let carry = 0
+    let i = 0
+    while (i < res.length) {
+      const c = res[i] + carry
+      carry = Math.floor(c / 58)
+      res[i++] = c - carry * 58
+    }
+
+    while (carry) {
+      const c = carry
+      carry = Math.floor(c / 58)
+      res[i++] = c - carry * 58
+    }
+  }
+
+  if (nativeDecoder) {
+    const oa = new Uint8Array(res.length)
+    let j = 0
+    for (let i = res.length - 1; i >= 0; i--) oa[j++] = codes[res[i]]
+    return ZERO.repeat(zeros) + nativeDecoder.decode(oa)
+  }
+
+  let out = ''
+  for (let i = res.length - 1; i >= 0; i--) out += alphabet[res[i]]
+  return ZERO.repeat(zeros) + out
+}
+
+// TODO: test on 'z'.repeat(from 1 to smth)
+export function fromBase58(str, format = 'uint8') {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  const length = str.length
+  if (length === 0) return typedView(new Uint8Array(), format)
+
+  let zeros = 0
+  while (zeros < length && str.charCodeAt(zeros) === zeroC) zeros++
+
+  if (!fromMap) {
+    fromMap = new Int8Array(256).fill(-1)
+    for (let i = 0; i < 58; i++) fromMap[alphabet[i].charCodeAt(0)] = i
+  }
+
+  const size = zeros + (((length - zeros) * 3 + 1) >> 2) // 3/4 rounded up, larger than ~0.73 coef to fit everything
+  const res = new Uint8Array(size)
+  let at = size // where is the first significant byte written
+
+  if (shouldUseBigIntFrom) {
+    let x = _0n
+
+    // nativeEncoder gives a benefit here
+    if (nativeEncoder) {
+      const codes = nativeEncoder.encode(str)
+      if (codes.length !== str.length) throw new SyntaxError(E_CHAR) // non-ascii
+      for (let i = zeros; i < length; i++) {
+        const c = fromMap[codes[i]]
+        if (c < 0) throw new SyntaxError(E_CHAR)
+        x = x * _58n + BigInt(c)
+      }
+    } else {
+      for (let i = zeros; i < length; i++) {
+        const charCode = str.charCodeAt(i)
+        const c = fromMap[charCode]
+        if (charCode > 255 || c < 0) throw new SyntaxError(E_CHAR)
+        x = x * _58n + BigInt(c)
+      }
+    }
+
+    while (x) {
+      let y = Number(x & _0xffffffffn)
+      x >>= 32n
+      res[--at] = y & 0xff
+      y >>>= 8
+      if (!x && !y) break
+      res[--at] = y & 0xff
+      y >>>= 8
+      if (!x && !y) break
+      res[--at] = y & 0xff
+      y >>>= 8
+      if (!x && !y) break
+      res[--at] = y & 0xff
+    }
+  } else {
+    for (let i = zeros; i < length; i++) {
+      const charCode = str.charCodeAt(i)
+      let c = fromMap[charCode]
+      if (charCode > 255 || c < 0) throw new SyntaxError(E_CHAR)
+
+      let k = size - 1
+      for (;;) {
+        if (c === 0 && k < at) break
+        c += 58 * res[k]
+        res[k] = c & 0xff
+        c >>>= 8
+        k--
+        // unroll a bit
+        if (c === 0 && k < at) break
+        c += 58 * res[k]
+        res[k] = c & 0xff
+        c >>>= 8
+        k--
+        if (c === 0 && k < at) break
+        c += 58 * res[k]
+        res[k] = c & 0xff
+        c >>>= 8
+        k--
+        if (c === 0 && k < at) break
+        c += 58 * res[k]
+        res[k] = c & 0xff
+        c >>>= 8
+        k--
+      }
+
+      at = k + 1
+      if (c !== 0 || at < zeros) throw new Error('Unexpected') // unreachable
+    }
+  }
+
+  return typedView(res.slice(at - zeros), format) // slice is faster for small sizes than subarray
+}

package/base58check.js

@@ -0,0 +1,30 @@
+import { typedView } from './array.js'
+import { assertUint8 } from './assert.js'
+import { toBase58, fromBase58 } from './base58.js'
+import { hashSync } from '@exodus/crypto/hash'
+
+// Note: while API is async, we use hashSync for now until we improve webcrypto perf for hash256
+// Inputs to base58 are typically very small, and that makes a difference
+
+const hash256 = (x) => hashSync('sha256', hashSync('sha256', x, 'uint8'), 'uint8')
+
+const E_CHECKSUM = 'Invalid checksum'
+
+export async function toBase58check(arr) {
+  assertUint8(arr)
+  const checksum = hash256(arr)
+  const res = new Uint8Array(arr.length + 4)
+  res.set(arr, 0)
+  res.set(checksum.subarray(0, 4), arr.length)
+  return toBase58(res)
+}
+
+export async function fromBase58check(str, format = 'uint8') {
+  const arr = fromBase58(str) // checks input
+  const len4 = arr.length - 4
+  const payload = arr.subarray(0, len4)
+  const c = arr.subarray(len4)
+  const r = hash256(payload)
+  if ((c[0] ^ r[0]) | (c[1] ^ r[1]) | (c[2] ^ r[2]) | (c[3] ^ r[3])) throw new Error(E_CHECKSUM)
+  return typedView(payload, format)
+}

package/base64.js

@@ -1,94 +1,134 @@
-import { assert, assertUint8 } from './assert.js'
+import { assertUint8, assertEmptyRest } from './assert.js'
 import { typedView } from './array.js'
 import * as js from './fallback/base64.js'
 
 // See https://datatracker.ietf.org/doc/html/rfc4648
 
-// base64:    A-Za-z0-9+/ and =
-// base64url: A-Za-z0-9_-
+// base64:    A-Za-z0-9+/ and = if padding not disabled
+// base64url: A-Za-z0-9_- and = if padding enabled
 
 const { Buffer, atob } = globalThis // Buffer is optional, only used when native
 const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
 const { toBase64: web64 } = Uint8Array.prototype // Modern engines have this
 
-export function toBase64(x) {
+const { E_CHAR, E_PADDING, E_LENGTH, E_LAST } = js
+
+const shouldUseAtob = atob && Boolean(globalThis.HermesInternal) // faster only on Hermes (and a little in old Chrome), js path beats it on normal engines
+
+// For native Buffer codepaths only
+const isBuffer = (x) => x.constructor === Buffer && Buffer.isBuffer(x)
+const toBuffer = (x) => (isBuffer(x) ? x : Buffer.from(x.buffer, x.byteOffset, x.byteLength))
+
+export function toBase64(x, { padding = true } = {}) {
   assertUint8(x)
-  if (web64 && x.toBase64 === web64) return x.toBase64() // Modern
-  if (!haveNativeBuffer) return js.toBase64(x, false, true) // Fallback
-  if (x.constructor === Buffer && Buffer.isBuffer(x)) return x.toString('base64') // Older Node.js
-  return Buffer.from(x.buffer, x.byteOffset, x.byteLength).toString('base64') // Older Node.js
+  if (web64 && x.toBase64 === web64) {
+    return padding ? x.toBase64() : x.toBase64({ omitPadding: !padding }) // Modern, optionless is slightly faster
+  }
+
+  if (!haveNativeBuffer) return js.toBase64(x, false, padding) // Fallback
+  const res = toBuffer(x).toString('base64') // Older Node.js
+  if (padding) return res
+  const at = res.indexOf('=', res.length - 3)
+  return at === -1 ? res : res.slice(0, at)
 }
 
-// NOTE: base64url omits padding
-export function toBase64url(x) {
+// NOTE: base64url omits padding by default
+export function toBase64url(x, { padding = false } = {}) {
   assertUint8(x)
-  if (web64 && x.toBase64 === web64) return x.toBase64({ alphabet: 'base64url', omitPadding: true }) // Modern
-  if (!haveNativeBuffer) return js.toBase64(x, true, false) // Fallback
+  if (web64 && x.toBase64 === web64) {
+    return x.toBase64({ alphabet: 'base64url', omitPadding: !padding }) // Modern
+  }
+
+  if (!haveNativeBuffer) return js.toBase64(x, true, padding) // Fallback
   if (x.constructor === Buffer && Buffer.isBuffer(x)) return x.toString('base64url') // Older Node.js
-  return Buffer.from(x.buffer, x.byteOffset, x.byteLength).toString('base64url') // Older Node.js
+  const res = toBuffer(x).toString('base64url') // Older Node.js
+  return padding && res.length % 4 !== 0 ? res + '='.repeat(4 - (res.length % 4)) : res
 }
 
 // Unlike Buffer.from(), throws on invalid input (non-base64 symbols and incomplete chunks)
 // Unlike Buffer.from() and Uint8Array.fromBase64(), does not allow spaces
 // NOTE: Always operates in strict mode for last chunk
 
-// Accepts both padded and non-padded variants, only strict base64
-export function fromBase64(str, format = 'uint8') {
-  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+// By default accepts both padded and non-padded variants, only strict base64
+export function fromBase64(str, options = {}) {
+  if (typeof options === 'string') options = { format: options } // Compat due to usage, TODO: remove
+  const { format = 'uint8', padding = 'both', ...rest } = options
+  return fromBase64common(str, false, padding, format, rest)
+}
 
-  // These checks should be needed only for Buffer path, not Uint8Array.fromBase64 path, but JSC lacks proper checks
-  assert(str.length % 4 !== 1, 'Invalid base64 length') // JSC misses this in fromBase64
-  if (str.endsWith('=')) {
-    assert(str.length % 4 === 0, 'Invalid padded length') // JSC misses this too
-    assert(str[str.length - 3] !== '=', 'Excessive padding') // no more than two = at the end
-  }
+// By default accepts only non-padded strict base64url
+export function fromBase64url(str, { format = 'uint8', padding = false, ...rest } = {}) {
+  return fromBase64common(str, true, padding, format, rest)
+}
 
-  return typedView(fromBase64common(str, false), format)
+// By default accepts both padded and non-padded variants, base64 or base64url
+export function fromBase64any(str, { format = 'uint8', padding = 'both', ...rest } = {}) {
+  const isBase64url = !str.includes('+') && !str.includes('/') // likely to fail fast, as most input is non-url, also double scan is faster than regex
+  return fromBase64common(str, isBase64url, padding, format, rest)
 }
 
-// Accepts both only non-padded strict base64url
-export function fromBase64url(str, format = 'uint8') {
+function fromBase64common(str, isBase64url, padding, format, rest) {
   if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  assertEmptyRest(rest)
+  const auto = padding === 'both' ? str.endsWith('=') : undefined
+  // Older JSC supporting Uint8Array.fromBase64 lacks proper checks
+  if (padding === true || auto === true) {
+    if (str.length % 4 !== 0) throw new SyntaxError(E_PADDING) // JSC misses this
+    if (str[str.length - 3] === '=') throw new SyntaxError(E_PADDING) // no more than two = at the end
+  } else if (padding === false || auto === false) {
+    if (str.length % 4 === 1) throw new SyntaxError(E_LENGTH) // JSC misses this in fromBase64
+    if (padding === false && str.endsWith('=')) {
+      throw new SyntaxError('Did not expect padding in base64 input') // inclusion is checked separately
+    }
+  } else {
+    throw new TypeError('Invalid padding option')
+  }
 
-  // These checks should be needed only for Buffer path, not Uint8Array.fromBase64 path, but JSC lacks proper checks
-  assert(str.length % 4 !== 1, 'Invalid base64 length') // JSC misses this in fromBase64
-  assert(!str.endsWith('='), 'Did not expect padding in base64url input') // inclusion is checked separately
-
-  return typedView(fromBase64common(str, true), format)
+  return typedView(fromBase64impl(str, isBase64url), format)
 }
 
-let fromBase64common
+// ASCII whitespace is U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, or U+0020 SPACE
+const ASCII_WHITESPACE = /[\t\n\f\r ]/ // non-u for JSC perf
+
+let fromBase64impl
 if (Uint8Array.fromBase64) {
   // NOTICE: this is actually slower than our JS impl in older JavaScriptCore and (slightly) in SpiderMonkey, but faster on V8 and new JavaScriptCore
-  fromBase64common = (str, isBase64url) => {
+  fromBase64impl = (str, isBase64url) => {
     const alphabet = isBase64url ? 'base64url' : 'base64'
-    assert(!/\s/u.test(str), `Invalid character in ${alphabet} input`) // all other chars are checked natively
+    if (ASCII_WHITESPACE.test(str)) throw new SyntaxError(E_CHAR) // all other chars are checked natively
     const padded = str.length % 4 > 0 ? `${str}${'='.repeat(4 - (str.length % 4))}` : str
     return Uint8Array.fromBase64(padded, { alphabet, lastChunkHandling: 'strict' })
   }
 } else {
-  fromBase64common = (str, isBase64url) => {
-    if (isBase64url) {
-      assert(!/[^0-9a-z_-]/iu.test(str), 'Invalid character in base64url input')
-    } else {
-      assert(!/[^0-9a-z=+/]/iu.test(str), 'Invalid character in base64 input')
-    }
-
+  fromBase64impl = (str, isBase64url) => {
     let arr
-    if (!haveNativeBuffer && atob) {
+    if (haveNativeBuffer) {
+      const invalidRegex = isBase64url ? /[^0-9a-z=_-]/iu : /[^0-9a-z=+/]/iu
+      if (invalidRegex.test(str)) throw new SyntaxError(E_CHAR)
+      const at = str.indexOf('=')
+      if (at >= 0 && /[^=]/iu.test(str.slice(at))) throw new SyntaxError(E_PADDING)
+      arr = Buffer.from(str, 'base64')
+    } else if (shouldUseAtob) {
       // atob is faster than manual parsing on Hermes
-      const raw = atob(isBase64url ? str.replaceAll('-', '+').replaceAll('_', '/') : str)
+      if (isBase64url) {
+        if (/[\t\n\f\r +/]/.test(str)) throw new SyntaxError(E_CHAR) // atob verifies other invalid input
+        str = str.replaceAll('-', '+').replaceAll('_', '/')
+      } else {
+        if (ASCII_WHITESPACE.test(str)) throw new SyntaxError(E_CHAR) // all other chars are checked natively
+      }
+
+      let raw
+      try {
+        raw = atob(str)
+      } catch {
+        throw new SyntaxError(E_CHAR) // convert atob errors
+      }
+
       const length = raw.length
       arr = new Uint8Array(length)
       for (let i = 0; i < length; i++) arr[i] = raw.charCodeAt(i)
     } else {
-      // base64url is already checked to have no padding via a regex above
-      if (!isBase64url) {
-        const at = str.indexOf('=')
-        if (at >= 0) assert(!/[^=]/iu.test(str.slice(at)), 'Invalid padding')
-      }
-
-      arr = haveNativeBuffer ? Buffer.from(str, 'base64') : js.fromBase64(str)
+      return js.fromBase64(str, isBase64url) // early return to skip last chunk verification, it's already validated in js
     }
 
     if (arr.length % 3 !== 0) {
@@ -96,7 +136,7 @@ if (Uint8Array.fromBase64) {
       const expected = toBase64(arr.subarray(-(arr.length % 3)))
       const end = str.length % 4 === 0 ? str.slice(-4) : str.slice(-(str.length % 4)).padEnd(4, '=')
       const actual = isBase64url ? end.replaceAll('-', '+').replaceAll('_', '/') : end
-      if (expected !== actual) throw new Error('Invalid last chunk')
+      if (expected !== actual) throw new SyntaxError(E_LAST)
     }
 
     return arr

package/fallback/_utils.js

@@ -0,0 +1,6 @@
+const { Buffer, TextEncoder, TextDecoder } = globalThis
+const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
+const isNative = (x) => x && (haveNativeBuffer || `${x}`.includes('[native code]')) // we consider Node.js TextDecoder/TextEncoder native
+const nativeEncoder = isNative(TextEncoder) ? new TextEncoder() : null
+const nativeDecoder = isNative(TextDecoder) ? new TextDecoder('utf8', { ignoreBOM: true }) : null
+export { nativeEncoder, nativeDecoder }