@exodus/bytes 1.0.0-rc.3 → 1.0.0-rc.4

package/README.md

@@ -1,3 +1,11 @@
-# bytes
+# `@exodus/bytes`
 
-Data structures handling
+`Uint8Array` conversion to and from `base64`, `base32`, `hex` and `utf8`
+
+[Fast](./Performance.md)
+
+Performs proper input validation
+
+## License
+
+[MIT](./LICENSE)

package/assert.js

@@ -16,7 +16,14 @@ export function assertTypedArray(arr) {
   throw new TypeError('Expected a TypedArray instance')
 }
 
-export function assertUint8(arr, { name, length, ...rest } = {}) {
+export function assertUint8(arr, options) {
+  if (!options) {
+    // fast path
+    if (arr instanceof Uint8Array) return
+    throw new TypeError('Expected an Uint8Array')
+  }
+
+  const { name, length, ...rest } = options
   assertEmptyRest(rest)
   if (arr instanceof Uint8Array && (length === undefined || arr.length === length)) return
   throw new TypeError(makeMessage(name, length === undefined ? '' : ` of size ${Number(length)}`))

package/base32.js

@@ -0,0 +1,33 @@
+import { assertEmptyRest } from './assert.js'
+import { typedView } from './array.js'
+import * as js from './fallback/base32.js'
+
+// See https://datatracker.ietf.org/doc/html/rfc4648
+
+// 8 chars per 5 bytes
+
+const { E_PADDING } = js
+
+export const toBase32 = (arr, { padding = false } = {}) => js.toBase32(arr, false, padding)
+export const toBase32hex = (arr, { padding = false } = {}) => js.toBase32(arr, true, padding)
+
+// By default, valid padding is accepted but not required
+export const fromBase32 = (str, { format = 'uint8', padding = 'both', ...rest } = {}) =>
+  fromBase32common(str, false, padding, format, rest)
+export const fromBase32hex = (str, { format = 'uint8', padding = 'both', ...rest } = {}) =>
+  fromBase32common(str, true, padding, format, rest)
+
+function fromBase32common(str, isBase32Hex, padding, format, rest) {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  assertEmptyRest(rest)
+
+  if (padding === true) {
+    if (str.length % 8 !== 0) throw new SyntaxError(E_PADDING)
+  } else if (padding === false) {
+    if (str.endsWith('=')) throw new SyntaxError('Did not expect padding in base32 input')
+  } else if (padding !== 'both') {
+    throw new TypeError('Invalid padding option')
+  }
+
+  return typedView(js.fromBase32(str, isBase32Hex), format)
+}

package/base64.js

@@ -1,94 +1,134 @@
-import { assert, assertUint8 } from './assert.js'
+import { assertUint8, assertEmptyRest } from './assert.js'
 import { typedView } from './array.js'
 import * as js from './fallback/base64.js'
 
 // See https://datatracker.ietf.org/doc/html/rfc4648
 
-// base64:    A-Za-z0-9+/ and =
-// base64url: A-Za-z0-9_-
+// base64:    A-Za-z0-9+/ and = if padding not disabled
+// base64url: A-Za-z0-9_- and = if padding enabled
 
 const { Buffer, atob } = globalThis // Buffer is optional, only used when native
 const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
 const { toBase64: web64 } = Uint8Array.prototype // Modern engines have this
 
-export function toBase64(x) {
+const { E_CHAR, E_PADDING, E_LENGTH, E_LAST } = js
+
+const shouldUseAtob = atob && Boolean(globalThis.HermesInternal) // faster only on Hermes (and a little in old Chrome), js path beats it on normal engines
+
+// For native Buffer codepaths only
+const isBuffer = (x) => x.constructor === Buffer && Buffer.isBuffer(x)
+const toBuffer = (x) => (isBuffer(x) ? x : Buffer.from(x.buffer, x.byteOffset, x.byteLength))
+
+export function toBase64(x, { padding = true } = {}) {
   assertUint8(x)
-  if (web64 && x.toBase64 === web64) return x.toBase64() // Modern
-  if (!haveNativeBuffer) return js.toBase64(x, false, true) // Fallback
-  if (x.constructor === Buffer && Buffer.isBuffer(x)) return x.toString('base64') // Older Node.js
-  return Buffer.from(x.buffer, x.byteOffset, x.byteLength).toString('base64') // Older Node.js
+  if (web64 && x.toBase64 === web64) {
+    return padding ? x.toBase64() : x.toBase64({ omitPadding: !padding }) // Modern, optionless is slightly faster
+  }
+
+  if (!haveNativeBuffer) return js.toBase64(x, false, padding) // Fallback
+  const res = toBuffer(x).toString('base64') // Older Node.js
+  if (padding) return res
+  const at = res.indexOf('=', res.length - 3)
+  return at === -1 ? res : res.slice(0, at)
 }
 
-// NOTE: base64url omits padding
-export function toBase64url(x) {
+// NOTE: base64url omits padding by default
+export function toBase64url(x, { padding = false } = {}) {
   assertUint8(x)
-  if (web64 && x.toBase64 === web64) return x.toBase64({ alphabet: 'base64url', omitPadding: true }) // Modern
-  if (!haveNativeBuffer) return js.toBase64(x, true, false) // Fallback
+  if (web64 && x.toBase64 === web64) {
+    return x.toBase64({ alphabet: 'base64url', omitPadding: !padding }) // Modern
+  }
+
+  if (!haveNativeBuffer) return js.toBase64(x, true, padding) // Fallback
   if (x.constructor === Buffer && Buffer.isBuffer(x)) return x.toString('base64url') // Older Node.js
-  return Buffer.from(x.buffer, x.byteOffset, x.byteLength).toString('base64url') // Older Node.js
+  const res = toBuffer(x).toString('base64url') // Older Node.js
+  return padding && res.length % 4 !== 0 ? res + '='.repeat(4 - (res.length % 4)) : res
 }
 
 // Unlike Buffer.from(), throws on invalid input (non-base64 symbols and incomplete chunks)
 // Unlike Buffer.from() and Uint8Array.fromBase64(), does not allow spaces
 // NOTE: Always operates in strict mode for last chunk
 
-// Accepts both padded and non-padded variants, only strict base64
-export function fromBase64(str, format = 'uint8') {
-  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+// By default accepts both padded and non-padded variants, only strict base64
+export function fromBase64(str, options = {}) {
+  if (typeof options === 'string') options = { format: options } // Compat due to usage, TODO: remove
+  const { format = 'uint8', padding = 'both', ...rest } = options
+  return fromBase64common(str, false, padding, format, rest)
+}
 
-  // These checks should be needed only for Buffer path, not Uint8Array.fromBase64 path, but JSC lacks proper checks
-  assert(str.length % 4 !== 1, 'Invalid base64 length') // JSC misses this in fromBase64
-  if (str.endsWith('=')) {
-    assert(str.length % 4 === 0, 'Invalid padded length') // JSC misses this too
-    assert(str[str.length - 3] !== '=', 'Excessive padding') // no more than two = at the end
-  }
+// By default accepts only non-padded strict base64url
+export function fromBase64url(str, { format = 'uint8', padding = false, ...rest } = {}) {
+  return fromBase64common(str, true, padding, format, rest)
+}
 
-  return typedView(fromBase64common(str, false), format)
+// By default accepts both padded and non-padded variants, base64 or base64url
+export function fromBase64any(str, { format = 'uint8', padding = 'both', ...rest } = {}) {
+  const isBase64url = !str.includes('+') && !str.includes('/') // likely to fail fast, as most input is non-url, also double scan is faster than regex
+  return fromBase64common(str, isBase64url, padding, format, rest)
 }
 
-// Accepts both only non-padded strict base64url
-export function fromBase64url(str, format = 'uint8') {
+function fromBase64common(str, isBase64url, padding, format, rest) {
   if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  assertEmptyRest(rest)
+  const auto = padding === 'both' ? str.endsWith('=') : undefined
+  // Older JSC supporting Uint8Array.fromBase64 lacks proper checks
+  if (padding === true || auto === true) {
+    if (str.length % 4 !== 0) throw new SyntaxError(E_PADDING) // JSC misses this
+    if (str[str.length - 3] === '=') throw new SyntaxError(E_PADDING) // no more than two = at the end
+  } else if (padding === false || auto === false) {
+    if (str.length % 4 === 1) throw new SyntaxError(E_LENGTH) // JSC misses this in fromBase64
+    if (padding === false && str.endsWith('=')) {
+      throw new SyntaxError('Did not expect padding in base64 input') // inclusion is checked separately
+    }
+  } else {
+    throw new TypeError('Invalid padding option')
+  }
 
-  // These checks should be needed only for Buffer path, not Uint8Array.fromBase64 path, but JSC lacks proper checks
-  assert(str.length % 4 !== 1, 'Invalid base64 length') // JSC misses this in fromBase64
-  assert(!str.endsWith('='), 'Did not expect padding in base64url input') // inclusion is checked separately
-
-  return typedView(fromBase64common(str, true), format)
+  return typedView(fromBase64impl(str, isBase64url), format)
 }
 
-let fromBase64common
+// ASCII whitespace is U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, or U+0020 SPACE
+const ASCII_WHITESPACE = /[\t\n\f\r ]/ // non-u for JSC perf
+
+let fromBase64impl
 if (Uint8Array.fromBase64) {
   // NOTICE: this is actually slower than our JS impl in older JavaScriptCore and (slightly) in SpiderMonkey, but faster on V8 and new JavaScriptCore
-  fromBase64common = (str, isBase64url) => {
+  fromBase64impl = (str, isBase64url) => {
     const alphabet = isBase64url ? 'base64url' : 'base64'
-    assert(!/\s/u.test(str), `Invalid character in ${alphabet} input`) // all other chars are checked natively
+    if (ASCII_WHITESPACE.test(str)) throw new SyntaxError(E_CHAR) // all other chars are checked natively
     const padded = str.length % 4 > 0 ? `${str}${'='.repeat(4 - (str.length % 4))}` : str
     return Uint8Array.fromBase64(padded, { alphabet, lastChunkHandling: 'strict' })
   }
 } else {
-  fromBase64common = (str, isBase64url) => {
-    if (isBase64url) {
-      assert(!/[^0-9a-z_-]/iu.test(str), 'Invalid character in base64url input')
-    } else {
-      assert(!/[^0-9a-z=+/]/iu.test(str), 'Invalid character in base64 input')
-    }
-
+  fromBase64impl = (str, isBase64url) => {
     let arr
-    if (!haveNativeBuffer && atob) {
+    if (haveNativeBuffer) {
+      const invalidRegex = isBase64url ? /[^0-9a-z=_-]/iu : /[^0-9a-z=+/]/iu
+      if (invalidRegex.test(str)) throw new SyntaxError(E_CHAR)
+      const at = str.indexOf('=')
+      if (at >= 0 && /[^=]/iu.test(str.slice(at))) throw new SyntaxError(E_PADDING)
+      arr = Buffer.from(str, 'base64')
+    } else if (shouldUseAtob) {
       // atob is faster than manual parsing on Hermes
-      const raw = atob(isBase64url ? str.replaceAll('-', '+').replaceAll('_', '/') : str)
+      if (isBase64url) {
+        if (/[\t\n\f\r +/]/.test(str)) throw new SyntaxError(E_CHAR) // atob verifies other invalid input
+        str = str.replaceAll('-', '+').replaceAll('_', '/')
+      } else {
+        if (ASCII_WHITESPACE.test(str)) throw new SyntaxError(E_CHAR) // all other chars are checked natively
+      }
+
+      let raw
+      try {
+        raw = atob(str)
+      } catch {
+        throw new SyntaxError(E_CHAR) // convert atob errors
+      }
+
       const length = raw.length
       arr = new Uint8Array(length)
       for (let i = 0; i < length; i++) arr[i] = raw.charCodeAt(i)
     } else {
-      // base64url is already checked to have no padding via a regex above
-      if (!isBase64url) {
-        const at = str.indexOf('=')
-        if (at >= 0) assert(!/[^=]/iu.test(str.slice(at)), 'Invalid padding')
-      }
-
-      arr = haveNativeBuffer ? Buffer.from(str, 'base64') : js.fromBase64(str)
+      return js.fromBase64(str, isBase64url) // early return to skip last chunk verification, it's already validated in js
     }
 
     if (arr.length % 3 !== 0) {
@@ -96,7 +136,7 @@ if (Uint8Array.fromBase64) {
       const expected = toBase64(arr.subarray(-(arr.length % 3)))
       const end = str.length % 4 === 0 ? str.slice(-4) : str.slice(-(str.length % 4)).padEnd(4, '=')
       const actual = isBase64url ? end.replaceAll('-', '+').replaceAll('_', '/') : end
-      if (expected !== actual) throw new Error('Invalid last chunk')
+      if (expected !== actual) throw new SyntaxError(E_LAST)
     }
 
     return arr

package/fallback/_utils.js

@@ -0,0 +1,6 @@
+const { Buffer, TextEncoder, TextDecoder } = globalThis
+const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
+const isNative = (x) => x && (haveNativeBuffer || `${x}`.includes('[native code]')) // we consider Node.js TextDecoder/TextEncoder native
+const nativeEncoder = isNative(TextEncoder) ? new TextEncoder() : null
+const nativeDecoder = isNative(TextDecoder) ? new TextDecoder('utf8', { ignoreBOM: true }) : null
+export { nativeEncoder, nativeDecoder }

package/fallback/base32.js

@@ -0,0 +1,198 @@
+import { assertUint8 } from '../assert.js'
+import { nativeEncoder, nativeDecoder } from './_utils.js'
+
+// See https://datatracker.ietf.org/doc/html/rfc4648
+
+const BASE32 = [...'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'] // RFC 4648, #6
+const BASE32HEX = [...'0123456789ABCDEFGHIJKLMNOPQRSTUV'] // RFC 4648, #7
+const BASE32_HELPERS = {}
+const BASE32HEX_HELPERS = {}
+
+export const E_CHAR = 'Invalid character in base32 input'
+export const E_PADDING = 'Invalid base32 padding'
+export const E_LENGTH = 'Invalid base32 length'
+export const E_LAST = 'Invalid last chunk'
+
+// We construct output by concatenating chars, this seems to be fine enough on modern JS engines
+export function toBase32(arr, isBase32Hex, padding) {
+  assertUint8(arr)
+  const fullChunks = Math.floor(arr.length / 5)
+  const fullChunksBytes = fullChunks * 5
+  let o = ''
+  let i = 0
+
+  const alphabet = isBase32Hex ? BASE32HEX : BASE32
+  const helpers = isBase32Hex ? BASE32HEX_HELPERS : BASE32_HELPERS
+  if (!helpers.pairs) {
+    helpers.pairs = []
+    if (nativeDecoder) {
+      // Lazy to save memory in case if this is not needed
+      helpers.codepairs = new Uint16Array(32 * 32)
+      const u16 = helpers.codepairs
+      const u8 = new Uint8Array(u16.buffer, u16.byteOffset, u16.byteLength) // write as 1-byte to ignore BE/LE difference
+      for (let i = 0; i < 32; i++) {
+        const ic = alphabet[i].charCodeAt(0)
+        for (let j = 0; j < 32; j++) u8[(i << 6) | (j << 1)] = u8[(j << 6) | ((i << 1) + 1)] = ic
+      }
+    } else {
+      const p = helpers.pairs
+      for (let i = 0; i < 32; i++) {
+        for (let j = 0; j < 32; j++) p.push(`${alphabet[i]}${alphabet[j]}`)
+      }
+    }
+  }
+
+  const { pairs, codepairs } = helpers
+
+  // Fast path for complete blocks
+  // This whole loop can be commented out, the algorithm won't change, it's just an optimization of the next loop
+  if (nativeDecoder) {
+    const oa = new Uint16Array(fullChunks * 4)
+    for (let j = 0; i < fullChunksBytes; i += 5) {
+      const a = arr[i]
+      const b = arr[i + 1]
+      const c = arr[i + 2]
+      const d = arr[i + 3]
+      const e = arr[i + 4]
+      oa[j++] = codepairs[(a << 2) | (b >> 6)] // 8 + 8 - 5 - 5 = 6 left
+      oa[j++] = codepairs[((b & 0x3f) << 4) | (c >> 4)] // 6 + 8 - 5 - 5 = 4 left
+      oa[j++] = codepairs[((c & 0xf) << 6) | (d >> 2)] // 4 + 8 - 5 - 5 = 2 left
+      oa[j++] = codepairs[((d & 0x3) << 8) | e] // 2 + 8 - 5 - 5 = 0 left
+    }
+
+    o = nativeDecoder.decode(oa)
+  } else {
+    for (; i < fullChunksBytes; i += 5) {
+      const a = arr[i]
+      const b = arr[i + 1]
+      const c = arr[i + 2]
+      const d = arr[i + 3]
+      const e = arr[i + 4]
+      o += pairs[(a << 2) | (b >> 6)] // 8 + 8 - 5 - 5 = 6 left
+      o += pairs[((b & 0x3f) << 4) | (c >> 4)] // 6 + 8 - 5 - 5 = 4 left
+      o += pairs[((c & 0xf) << 6) | (d >> 2)] // 4 + 8 - 5 - 5 = 2 left
+      o += pairs[((d & 0x3) << 8) | e] // 2 + 8 - 5 - 5 = 0 left
+    }
+  }
+
+  // If we have something left, process it with a full algo
+  let carry = 0
+  let shift = 3 // First byte needs to be shifted by 3 to get 5 bits
+  for (; i < arr.length; i++) {
+    const x = arr[i]
+    o += alphabet[carry | (x >> shift)] // shift >= 3, so this fits
+    if (shift >= 5) {
+      shift -= 5
+      o += alphabet[(x >> shift) & 0x1f]
+    }
+
+    carry = (x << (5 - shift)) & 0x1f
+    shift += 3 // Each byte prints 5 bits and leaves 3 bits
+  }
+
+  if (shift !== 3) o += alphabet[carry] // shift 3 means we have no carry left
+  if (padding) o += ['', '======', '====', '===', '='][arr.length - fullChunksBytes]
+
+  return o
+}
+
+// TODO: can this be optimized? This only affects non-Hermes barebone engines though
+const mapSize = nativeEncoder ? 256 : 65_536 // we have to store 64 KiB map or recheck everything if we can't decode to byte array
+
+export function fromBase32(str, isBase32Hex) {
+  let inputLength = str.length
+  while (str[inputLength - 1] === '=') inputLength--
+  const paddingLength = str.length - inputLength
+  const tailLength = inputLength % 8
+  const mainLength = inputLength - tailLength // multiples of 8
+  if (![0, 2, 4, 5, 7].includes(tailLength)) throw new SyntaxError(E_LENGTH) // fast verification
+  if (paddingLength > 7 || (paddingLength !== 0 && str.length % 8 !== 0)) {
+    throw new SyntaxError(E_PADDING)
+  }
+
+  const alphabet = isBase32Hex ? BASE32HEX : BASE32
+  const helpers = isBase32Hex ? BASE32HEX_HELPERS : BASE32_HELPERS
+
+  if (!helpers.fromMap) {
+    helpers.fromMap = new Int8Array(mapSize).fill(-1) // no regex input validation here, so we map all other bytes to -1 and recheck sign
+    alphabet.forEach((c, i) => {
+      helpers.fromMap[c.charCodeAt(0)] = helpers.fromMap[c.toLowerCase().charCodeAt(0)] = i
+    })
+  }
+
+  const m = helpers.fromMap
+
+  const arr = new Uint8Array(Math.floor((inputLength * 5) / 8))
+  let at = 0
+  let i = 0
+
+  if (nativeEncoder) {
+    const codes = nativeEncoder.encode(str)
+    if (codes.length !== str.length) throw new SyntaxError(E_CHAR) // non-ascii
+    while (i < mainLength) {
+      // each 5 bits, grouped 5 * 4 = 20
+      const a = (m[codes[i++]] << 15) | (m[codes[i++]] << 10) | (m[codes[i++]] << 5) | m[codes[i++]]
+      const b = (m[codes[i++]] << 15) | (m[codes[i++]] << 10) | (m[codes[i++]] << 5) | m[codes[i++]]
+      if (a < 0 || b < 0) throw new SyntaxError(E_CHAR)
+      arr[at++] = a >> 12
+      arr[at++] = (a >> 4) & 0xff
+      arr[at++] = ((a << 4) & 0xff) | (b >> 16)
+      arr[at++] = (b >> 8) & 0xff
+      arr[at++] = b & 0xff
+    }
+  } else {
+    while (i < mainLength) {
+      // each 5 bits, grouped 5 * 4 = 20
+      const a =
+        (m[str.charCodeAt(i++)] << 15) |
+        (m[str.charCodeAt(i++)] << 10) |
+        (m[str.charCodeAt(i++)] << 5) |
+        m[str.charCodeAt(i++)]
+      const b =
+        (m[str.charCodeAt(i++)] << 15) |
+        (m[str.charCodeAt(i++)] << 10) |
+        (m[str.charCodeAt(i++)] << 5) |
+        m[str.charCodeAt(i++)]
+      if (a < 0 || b < 0) throw new SyntaxError(E_CHAR)
+      arr[at++] = a >> 12
+      arr[at++] = (a >> 4) & 0xff
+      arr[at++] = ((a << 4) & 0xff) | (b >> 16)
+      arr[at++] = (b >> 8) & 0xff
+      arr[at++] = b & 0xff
+    }
+  }
+
+  // Last block, valid tailLength: 0 2 4 5 7, checked already
+  // We check last chunk to be strict
+  if (tailLength < 2) return arr
+  const ab = (m[str.charCodeAt(i++)] << 5) | m[str.charCodeAt(i++)]
+  if (ab < 0) throw new SyntaxError(E_CHAR)
+  arr[at++] = ab >> 2
+  if (tailLength < 4) {
+    if (ab & 0x3) throw new SyntaxError(E_LAST)
+    return arr
+  }
+
+  const cd = (m[str.charCodeAt(i++)] << 5) | m[str.charCodeAt(i++)]
+  if (cd < 0) throw new SyntaxError(E_CHAR)
+  arr[at++] = ((ab << 6) & 0xff) | (cd >> 4)
+  if (tailLength < 5) {
+    if (cd & 0xf) throw new SyntaxError(E_LAST)
+    return arr
+  }
+
+  const e = m[str.charCodeAt(i++)]
+  if (e < 0) throw new SyntaxError(E_CHAR)
+  arr[at++] = ((cd << 4) & 0xff) | (e >> 1) // 4 + 4
+  if (tailLength < 7) {
+    if (e & 0x1) throw new SyntaxError(E_LAST)
+    return arr
+  }
+
+  const fg = (m[str.charCodeAt(i++)] << 5) | m[str.charCodeAt(i++)]
+  if (fg < 0) throw new SyntaxError(E_CHAR)
+  arr[at++] = ((e << 7) & 0xff) | (fg >> 3) // 1 + 5 + 2
+  // Can't be 8, so no h
+  if (fg & 0x7) throw new SyntaxError(E_LAST)
+  return arr
+}

package/fallback/base64.js

@@ -1,15 +1,17 @@
 import { assertUint8 } from '../assert.js'
+import { nativeEncoder, nativeDecoder } from './_utils.js'
 
 // See https://datatracker.ietf.org/doc/html/rfc4648
 
-const { TextDecoder } = globalThis
-const nativeDecoder = TextDecoder?.toString().includes('[native code]') ? new TextDecoder() : null
 const BASE64 = [...'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/']
 const BASE64URL = [...'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_']
-const BASE64_PAIRS = []
-const BASE64URL_PAIRS = []
-const BASE64_CODES = nativeDecoder ? new Uint8Array(64) : null
-const BASE64URL_CODES = nativeDecoder ? new Uint8Array(64) : null
+const BASE64_HELPERS = {}
+const BASE64URL_HELPERS = {}
+
+export const E_CHAR = 'Invalid character in base64 input'
+export const E_PADDING = 'Invalid base64 padding'
+export const E_LENGTH = 'Invalid base64 length'
+export const E_LAST = 'Invalid last chunk'
 
 // Alternatively, we could have mapped 0-255 bytes to charcodes and just used btoa(ascii),
 // but that approach is _slower_ than our toBase64js function, even on Hermes
@@ -23,27 +25,38 @@ export function toBase64(arr, isURL, padding) {
   let i = 0
 
   const alphabet = isURL ? BASE64URL : BASE64
-  const pairs = isURL ? BASE64URL_PAIRS : BASE64_PAIRS
-  const map = isURL ? BASE64_CODES : BASE64URL_CODES
-  if (pairs.length === 0) {
-    for (let i = 0; i < 64; i++) {
-      for (let j = 0; j < 64; j++) pairs.push(`${alphabet[i]}${alphabet[j]}`)
-      if (map) map[i] = alphabet[i].charCodeAt(0)
+  const helpers = isURL ? BASE64URL_HELPERS : BASE64_HELPERS
+  if (!helpers.pairs) {
+    helpers.pairs = []
+    if (nativeDecoder) {
+      // Lazy to save memory in case if this is not needed
+      helpers.codepairs = new Uint16Array(64 * 64)
+      const u16 = helpers.codepairs
+      const u8 = new Uint8Array(u16.buffer, u16.byteOffset, u16.byteLength) // write as 1-byte to ignore BE/LE difference
+      for (let i = 0; i < 64; i++) {
+        const ic = alphabet[i].charCodeAt(0)
+        for (let j = 0; j < 64; j++) u8[(i << 7) | (j << 1)] = u8[(j << 7) | ((i << 1) + 1)] = ic
+      }
+    } else {
+      const p = helpers.pairs
+      for (let i = 0; i < 64; i++) {
+        for (let j = 0; j < 64; j++) p.push(`${alphabet[i]}${alphabet[j]}`)
+      }
     }
   }
 
+  const { pairs, codepairs } = helpers
+
   // Fast path for complete blocks
   // This whole loop can be commented out, the algorithm won't change, it's just an optimization of the next loop
   if (nativeDecoder) {
-    const oa = new Uint8Array(fullChunks * 4)
+    const oa = new Uint16Array(fullChunks * 2)
     for (let j = 0; i < fullChunksBytes; i += 3) {
       const a = arr[i]
       const b = arr[i + 1]
       const c = arr[i + 2]
-      oa[j++] = map[a >> 2]
-      oa[j++] = map[((a & 0x3) << 4) | (b >> 4)]
-      oa[j++] = map[((b & 0xf) << 2) | (c >> 6)]
-      oa[j++] = map[c & 0x3f]
+      oa[j++] = codepairs[(a << 4) | (b >> 4)]
+      oa[j++] = codepairs[((b & 0x0f) << 8) | c]
     }
 
     o = nativeDecoder.decode(oa)
@@ -78,50 +91,72 @@ export function toBase64(arr, isURL, padding) {
   return o
 }
 
-let fromBase64jsMap
-
-// Assumes valid input and no chars after =, checked at API
-// Last chunk is rechecked at API too
-export function fromBase64(str) {
-  const map = fromBase64jsMap || new Array(256)
-  if (!fromBase64jsMap) {
-    fromBase64jsMap = map
-    BASE64.forEach((c, i) => (map[c.charCodeAt(0)] = i))
-    map['-'.charCodeAt(0)] = map['+'.charCodeAt(0)] // for base64url
-    map['_'.charCodeAt(0)] = map['/'.charCodeAt(0)] // for base64url
-  }
+// TODO: can this be optimized? This only affects non-Hermes barebone engines though
+const mapSize = nativeEncoder ? 256 : 65_536 // we have to store 64 KiB map or recheck everything if we can't decode to byte array
 
+// Last chunk is rechecked at API
+export function fromBase64(str, isURL) {
   let inputLength = str.length
   while (str[inputLength - 1] === '=') inputLength--
-
-  const arr = new Uint8Array(Math.floor((inputLength * 3) / 4))
+  const paddingLength = str.length - inputLength
   const tailLength = inputLength % 4
   const mainLength = inputLength - tailLength // multiples of 4
+  if (tailLength === 1) throw new SyntaxError(E_LENGTH)
+  if (paddingLength > 3 || (paddingLength !== 0 && str.length % 4 !== 0)) {
+    throw new SyntaxError(E_PADDING)
+  }
+
+  const alphabet = isURL ? BASE64URL : BASE64
+  const helpers = isURL ? BASE64URL_HELPERS : BASE64_HELPERS
 
+  if (!helpers.fromMap) {
+    helpers.fromMap = new Int8Array(mapSize).fill(-1) // no regex input validation here, so we map all other bytes to -1 and recheck sign
+    alphabet.forEach((c, i) => (helpers.fromMap[c.charCodeAt(0)] = i))
+  }
+
+  const m = helpers.fromMap
+
+  const arr = new Uint8Array(Math.floor((inputLength * 3) / 4))
   let at = 0
   let i = 0
-  let tmp
-
-  while (i < mainLength) {
-    // a [ b c ] d, each 6 bits
-    const bc = (map[str.charCodeAt(i + 1)] << 6) | map[str.charCodeAt(i + 2)]
-    arr[at++] = (map[str.charCodeAt(i)] << 2) | (bc >> 10)
-    arr[at++] = (bc >> 2) & 0xff
-    arr[at++] = ((bc << 6) & 0xff) | map[str.charCodeAt(i + 3)]
-    i += 4
+
+  if (nativeEncoder) {
+    const codes = nativeEncoder.encode(str)
+    if (codes.length !== str.length) throw new SyntaxError(E_CHAR) // non-ascii
+    while (i < mainLength) {
+      const a = (m[codes[i++]] << 18) | (m[codes[i++]] << 12) | (m[codes[i++]] << 6) | m[codes[i++]]
+      if (a < 0) throw new SyntaxError(E_CHAR)
+      arr[at++] = a >> 16
+      arr[at++] = (a >> 8) & 0xff
+      arr[at++] = a & 0xff
+    }
+  } else {
+    while (i < mainLength) {
+      const a =
+        (m[str.charCodeAt(i++)] << 18) |
+        (m[str.charCodeAt(i++)] << 12) |
+        (m[str.charCodeAt(i++)] << 6) |
+        m[str.charCodeAt(i++)]
+      if (a < 0) throw new SyntaxError(E_CHAR)
+      arr[at++] = a >> 16
+      arr[at++] = (a >> 8) & 0xff
+      arr[at++] = a & 0xff
+    }
   }
 
-  if (tailLength === 3) {
-    tmp =
-      (map[str.charCodeAt(i)] << 10) |
-      (map[str.charCodeAt(i + 1)] << 4) |
-      (map[str.charCodeAt(i + 2)] >> 2)
-    arr[at++] = (tmp >> 8) & 0xff
-    arr[at++] = tmp & 0xff
-  } else if (tailLength === 2) {
-    tmp = (map[str.charCodeAt(i)] << 2) | (map[str.charCodeAt(i + 1)] >> 4)
-    arr[at++] = tmp & 0xff
+  // Can be 0, 2 or 3, verified by padding checks already
+  if (tailLength < 2) return arr // 0
+  const ab = (m[str.charCodeAt(i++)] << 6) | m[str.charCodeAt(i++)]
+  if (ab < 0) throw new SyntaxError(E_CHAR)
+  arr[at++] = ab >> 4
+  if (tailLength < 3) {
+    if (ab & 0xf) throw new SyntaxError(E_LAST)
+    return arr // 2
   }
 
-  return arr
+  const c = m[str.charCodeAt(i++)]
+  if (c < 0) throw new SyntaxError(E_CHAR)
+  arr[at++] = ((ab << 4) & 0xff) | (c >> 2)
+  if (c & 0x3) throw new SyntaxError(E_LAST)
+  return arr // 3
 }

package/fallback/hex.js

@@ -1,12 +1,11 @@
-import { assert, assertUint8 } from '../assert.js'
-
-// We use TextEncoder here to parse strings to charcodes, this is faster than individual charCodeAt calls
-const { TextEncoder } = globalThis // Buffer is optional, only used when native
-const nativeEncoder = TextEncoder?.toString().includes('[native code]') ? new TextEncoder() : null
+import { assertUint8 } from '../assert.js'
+import { nativeEncoder } from './_utils.js'
 
 let hexArray
 let dehexArray
 
+export const E_HEX = 'Input is not a hex string'
+
 function toHexPart(arr, start, end) {
   let o = ''
   let i = start
@@ -52,15 +51,18 @@ export function toHex(arr) {
   return toHexPart(arr, 0, length)
 }
 
+// TODO: can this be optimized? This only affects non-Hermes barebone engines though
+const mapSize = nativeEncoder ? 256 : 65_536 // we have to store 64 KiB map or recheck everything if we can't decode to byte array
+
 export function fromHex(str) {
   if (typeof str !== 'string') throw new TypeError('Input is not a string')
-  assert(str.length % 2 === 0, 'Input is not a hex string')
+  if (str.length % 2 !== 0) throw new SyntaxError(E_HEX)
 
   // We don't use native Buffer impl, as rechecking input make it slower than pure js
   // This path is used only on older engines though
 
   if (!dehexArray) {
-    dehexArray = new Array(103) // f is 102
+    dehexArray = new Int8Array(mapSize).fill(-1) // no regex input validation here, so we map all other bytes to -1 and recheck sign
     for (let i = 0; i < 16; i++) {
       const s = i.toString(16)
       dehexArray[s.charCodeAt(0)] = dehexArray[s.toUpperCase().charCodeAt(0)] = i
@@ -73,16 +75,31 @@ export function fromHex(str) {
   if (nativeEncoder) {
     // Native encoder path is beneficial even for small arrays in Hermes
     const codes = nativeEncoder.encode(str)
-    for (let i = 0; i < length; i++) {
-      const a = dehexArray[codes[j++]] * 16 + dehexArray[codes[j++]]
-      if (!a && a !== 0) throw new Error('Input is not a hex string')
-      arr[i] = a
+    if (codes.length !== str.length) throw new SyntaxError(E_HEX) // non-ascii
+    const last3 = length - 3 // Unroll nativeEncoder path as this is what modern Hermes takes and a small perf improvement is nice there
+    let i = 0
+    while (i < last3) {
+      const a = (dehexArray[codes[j++]] << 4) | dehexArray[codes[j++]]
+      const b = (dehexArray[codes[j++]] << 4) | dehexArray[codes[j++]]
+      const c = (dehexArray[codes[j++]] << 4) | dehexArray[codes[j++]]
+      const d = (dehexArray[codes[j++]] << 4) | dehexArray[codes[j++]]
+      if (a < 0 || b < 0 || c < 0 || d < 0) throw new SyntaxError(E_HEX)
+      arr[i++] = a
+      arr[i++] = b
+      arr[i++] = c
+      arr[i++] = d
+    }
+
+    while (i < length) {
+      const res = (dehexArray[codes[j++]] << 4) | dehexArray[codes[j++]]
+      if (res < 0) throw new SyntaxError(E_HEX)
+      arr[i++] = res
     }
   } else {
     for (let i = 0; i < length; i++) {
-      const a = dehexArray[str.charCodeAt(j++)] * 16 + dehexArray[str.charCodeAt(j++)]
-      if (!a && a !== 0) throw new Error('Input is not a hex string')
-      arr[i] = a
+      const res = (dehexArray[str.charCodeAt(j++)] << 4) | dehexArray[str.charCodeAt(j++)]
+      if (res < 0) throw new SyntaxError(E_HEX)
+      arr[i] = res
     }
   }
 

package/fallback/utf8.js

@@ -0,0 +1,280 @@
+export const E_STRICT = 'Input is not well-formed utf8'
+export const E_STRICT_UNICODE = 'Input is not well-formed Unicode'
+
+const replacementPoint = 0xff_fd
+
+// https://encoding.spec.whatwg.org/#utf-8-decoder
+// We are most likely in loose mode, for non-loose escape & decodeURIComponent solved everything
+export function decode(arr, loose) {
+  const start = 0
+  const end = arr.length
+  let out = ''
+  const tmp = []
+
+  for (let i = start; i < end; i++) {
+    if (tmp.length > 0x2_00) {
+      // far below MAX_ARGUMENTS_LENGTH in npmjs.com/buffer, we use smaller chunks
+      // length can be off by a few as large code points produce two utf-16 char codes, also we overshoot in unrolled loop
+      out += String.fromCharCode.apply(String, tmp)
+      tmp.length = 0
+    }
+
+    const byte = arr[i]
+    if (byte < 0x80) {
+      // Fast path ascii
+      tmp.push(byte)
+      // Unroll the loop a bit for faster ops, overshoot by 20 chars
+      for (let j = 0; j < 5; j++) {
+        if (i + 1 >= end) break
+        const byte1 = arr[i + 1]
+        if (byte1 >= 0x80) break
+        tmp.push(byte1)
+        i++
+        if (i + 1 >= end) break
+        const byte2 = arr[i + 1]
+        if (byte2 >= 0x80) break
+        tmp.push(byte2)
+        i++
+        if (i + 1 >= end) break
+        const byte3 = arr[i + 1]
+        if (byte3 >= 0x80) break
+        tmp.push(byte3)
+        i++
+        if (i + 1 >= end) break
+        const byte4 = arr[i + 1]
+        if (byte4 >= 0x80) break
+        tmp.push(byte4)
+        i++
+      }
+    } else if (byte < 0xc2) {
+      if (!loose) throw new TypeError(E_STRICT)
+      tmp.push(replacementPoint)
+    } else if (byte < 0xe0) {
+      // need 1 more
+      if (i + 1 >= end) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        break
+      }
+
+      const byte1 = arr[i + 1]
+      if (byte1 < 0x80 || byte1 > 0xbf) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        continue
+      }
+
+      i++
+      tmp.push(((byte & 0x1f) << 6) | (byte1 & 0x3f))
+    } else if (byte < 0xf0) {
+      // need 2 more
+      if (i + 1 >= end) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        break
+      }
+
+      const lower = byte === 0xe0 ? 0xa0 : 0x80
+      const upper = byte === 0xed ? 0x9f : 0xbf
+      const byte1 = arr[i + 1]
+      if (byte1 < lower || byte1 > upper) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        continue
+      }
+
+      i++
+      if (i + 1 >= end) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        break
+      }
+
+      const byte2 = arr[i + 1]
+      if (byte2 < 0x80 || byte2 > 0xbf) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        continue
+      }
+
+      i++
+      tmp.push(((byte & 0xf) << 12) | ((byte1 & 0x3f) << 6) | (byte2 & 0x3f))
+    } else if (byte <= 0xf4) {
+      // need 3 more
+      if (i + 1 >= end) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        break
+      }
+
+      const lower = byte === 0xf0 ? 0x90 : 0x80
+      const upper = byte === 0xf4 ? 0x8f : 0xbf
+      const byte1 = arr[i + 1]
+      if (byte1 < lower || byte1 > upper) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        continue
+      }
+
+      i++
+      if (i + 1 >= end) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        break
+      }
+
+      const byte2 = arr[i + 1]
+      if (byte2 < 0x80 || byte2 > 0xbf) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        continue
+      }
+
+      i++
+      if (i + 1 >= end) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        break
+      }
+
+      const byte3 = arr[i + 1]
+      if (byte3 < 0x80 || byte3 > 0xbf) {
+        if (!loose) throw new TypeError(E_STRICT)
+        tmp.push(replacementPoint)
+        continue
+      }
+
+      i++
+      const codePoint =
+        ((byte & 0xf) << 18) | ((byte1 & 0x3f) << 12) | ((byte2 & 0x3f) << 6) | (byte3 & 0x3f)
+      if (codePoint > 0xff_ff) {
+        // split into char codes as String.fromCharCode is faster than String.fromCodePoint
+        const u = codePoint - 0x1_00_00
+        tmp.push(0xd8_00 + ((u >> 10) & 0x3_ff), 0xdc_00 + (u & 0x3_ff))
+      } else {
+        tmp.push(codePoint)
+      }
+      // eslint-disable-next-line sonarjs/no-duplicated-branches
+    } else {
+      if (!loose) throw new TypeError(E_STRICT)
+      tmp.push(replacementPoint)
+    }
+  }
+
+  if (tmp.length > 0) out += String.fromCharCode.apply(String, tmp)
+  return out
+}
+
+export function encode(string, loose) {
+  const length = string.length
+  let lead = null
+  let small = true
+  let bytes = new Uint8Array(length) // assume ascii
+  let p = 0
+
+  for (let i = 0; i < length; i++) {
+    const code = string.charCodeAt(i)
+    if (code < 0x80) {
+      // Fast path for ascii
+      if (lead) {
+        if (!loose) throw new TypeError(E_STRICT_UNICODE)
+        bytes[p++] = 0xef
+        bytes[p++] = 0xbf
+        bytes[p++] = 0xbd
+        lead = null
+      }
+
+      bytes[p++] = code
+      // Unroll the loop a bit for faster ops
+      for (let j = 0; j < 5; j++) {
+        if (i + 1 >= length) break
+        const c1 = string.charCodeAt(i + 1)
+        if (c1 >= 0x80) break
+        bytes[p++] = c1
+        i++
+        if (i + 1 >= length) break
+        const c2 = string.charCodeAt(i + 1)
+        if (c2 >= 0x80) break
+        bytes[p++] = c2
+        i++
+        if (i + 1 >= length) break
+        const c3 = string.charCodeAt(i + 1)
+        if (c3 >= 0x80) break
+        bytes[p++] = c3
+        i++
+        if (i + 1 >= length) break
+        const c4 = string.charCodeAt(i + 1)
+        if (c4 >= 0x80) break
+        bytes[p++] = c4
+        i++
+      }
+
+      continue
+    }
+
+    if (small) {
+      // TODO: use resizable array buffers? will have to return a non-resizeable one
+      const bytesNew = new Uint8Array(length * 3) // maximium can be 3x of the string length in charcodes
+      bytesNew.set(bytes)
+      bytes = bytesNew
+      small = false
+    }
+
+    // surrogate, charcodes = [d800 + a & 3ff, dc00 + b & 3ff]; codePoint = 0x1_00_00 | (a << 10) | b
+    // lead: d800 - dbff
+    // trail: dc00 - dfff
+    if (code >= 0xd8_00 && code < 0xe0_00) {
+      if (lead && code < 0xdc_00) {
+        // a second lead, meaning the previous one was unpaired
+        if (!loose) throw new TypeError(E_STRICT_UNICODE)
+        bytes[p++] = 0xef
+        bytes[p++] = 0xbf
+        bytes[p++] = 0xbd
+        lead = null
+        // code is still processed as a new lead
+      }
+
+      if (!lead) {
+        if (code > 0xdb_ff || i + 1 >= length) {
+          // lead out of range || unpaired
+          if (!loose) throw new TypeError(E_STRICT_UNICODE)
+          bytes[p++] = 0xef
+          bytes[p++] = 0xbf
+          bytes[p++] = 0xbd
+          continue
+        }
+
+        lead = code
+        continue
+      }
+
+      // here, codePoint is always between 0x1_00_00 and 0x11_00_00, we encode as 4 bytes
+      const codePoint = (((lead - 0xd8_00) << 10) | (code - 0xdc_00)) + 0x1_00_00
+      bytes[p++] = (codePoint >> 18) | 0xf0
+      bytes[p++] = ((codePoint >> 12) & 0x3f) | 0x80
+      bytes[p++] = ((codePoint >> 6) & 0x3f) | 0x80
+      bytes[p++] = (codePoint & 0x3f) | 0x80
+      lead = null
+      continue
+    } else if (lead) {
+      if (!loose) throw new TypeError(E_STRICT_UNICODE)
+      bytes[p++] = 0xef
+      bytes[p++] = 0xbf
+      bytes[p++] = 0xbd
+      lead = null
+      // code is still processed
+    }
+
+    // We are left with a non-pair char code above ascii, it gets encoded to 2 or 3 bytes
+    if (code < 0x8_00) {
+      bytes[p++] = (code >> 6) | 0xc0
+      bytes[p++] = (code & 0x3f) | 0x80
+    } else {
+      bytes[p++] = (code >> 12) | 0xe0
+      bytes[p++] = ((code >> 6) & 0x3f) | 0x80
+      bytes[p++] = (code & 0x3f) | 0x80
+    }
+  }
+
+  return bytes.length === p ? bytes : bytes.slice(0, p)
+}

package/hex.js

@@ -1,4 +1,4 @@
-import { assertTypedArray } from './assert.js'
+import { assertUint8 } from './assert.js'
 import { typedView } from './array.js'
 import * as js from './fallback/hex.js'
 
@@ -7,8 +7,7 @@ const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
 const { toHex: webHex } = Uint8Array.prototype // Modern engines have this
 
 export function toHex(arr) {
-  assertTypedArray(arr)
-  if (!(arr instanceof Uint8Array)) arr = new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength)
+  assertUint8(arr)
   if (arr.length === 0) return ''
   if (webHex && arr.toHex === webHex) return arr.toHex()
   if (!haveNativeBuffer) return js.toHex(arr)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@exodus/bytes",
-  "version": "1.0.0-rc.3",
+  "version": "1.0.0-rc.4",
   "description": "Various operations on Uint8Array data",
   "scripts": {
     "lint": "eslint .",
@@ -39,19 +39,25 @@
   },
   "type": "module",
   "files": [
+    "/fallback/_utils.js",
+    "/fallback/base32.js",
     "/fallback/base64.js",
     "/fallback/hex.js",
+    "/fallback/utf8.js",
     "/array.js",
     "/assert.js",
+    "/base32.js",
     "/base64.js",
-    "/hex.js"
+    "/hex.js",
+    "/utf8.js"
   ],
   "exports": {
     "./array.js": "./array.js",
+    "./base32.js": "./base32.js",
     "./base64.js": "./base64.js",
-    "./hex.js": "./hex.js"
+    "./hex.js": "./hex.js",
+    "./utf8.js": "./utf8.js"
   },
-  "dependencies": {},
   "devDependencies": {
     "@exodus/eslint-config": "^5.24.0",
     "@exodus/prettier": "^1.0.0",
@@ -64,7 +70,10 @@
     "buffer": "^6.0.3",
     "electron": "36.5.0",
     "eslint": "^8.44.0",
-    "jsvu": "^3.0.0"
+    "fast-base64-decode": "^2.0.0",
+    "hi-base32": "^0.5.1",
+    "jsvu": "^3.0.0",
+    "text-encoding": "^0.7.0"
   },
   "prettier": "@exodus/prettier",
   "packageManager": "pnpm@10.12.1+sha256.889bac470ec93ccc3764488a19d6ba8f9c648ad5e50a9a6e4be3768a5de387a3"

package/utf8.js

@@ -0,0 +1,110 @@
+import { assert, assertUint8 } from './assert.js'
+import { typedView } from './array.js'
+import * as js from './fallback/utf8.js'
+
+const { Buffer, TextEncoder, TextDecoder, decodeURIComponent, escape } = globalThis // Buffer is optional
+const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
+const isNative = (x) => x && (haveNativeBuffer || `${x}`.includes('[native code]')) // we consider Node.js TextDecoder/TextEncoder native
+const haveDecoder = isNative(TextDecoder)
+const nativeEncoder = isNative(TextEncoder) ? new TextEncoder() : null
+// ignoreBOM: true means that BOM will be left as-is, i.e. will be present in the output
+// We don't want to strip anything unexpectedly
+const decoderFatal = haveDecoder ? new TextDecoder('utf8', { ignoreBOM: true, fatal: true }) : null
+const decoderLoose = haveDecoder ? new TextDecoder('utf8', { ignoreBOM: true }) : null
+
+const { E_STRICT, E_STRICT_UNICODE } = js
+
+const shouldUseEscapePath = Boolean(globalThis.HermesInternal) // faster only on Hermes, js path beats it on normal engines
+
+function deLoose(str, loose, res) {
+  if (loose) return res
+  // Recheck if the string was encoded correctly
+  let start = 0
+  const last = res.length - 2
+  // Search for EFBFBD
+  while (start < last) {
+    const pos = res.indexOf(0xef, start)
+    if (pos === -1) break
+    start = pos + 1
+    if (res[pos + 1] === 0xbf && res[pos + 2] === 0xbd) {
+      // Found a replacement char in output, need to recheck if we encoded the input correctly
+      assert(str === decode(res), E_STRICT_UNICODE)
+      return res
+    }
+  }
+
+  return res
+}
+
+function encode(str, loose = false) {
+  assert(typeof str === 'string')
+  if (haveNativeBuffer) return deLoose(str, loose, Buffer.from(str)) // faster on ascii on Node.js
+  if (nativeEncoder) return deLoose(str, loose, nativeEncoder.encode(str)) // Node.js, browsers, and Hermes
+  // No reason to use unescape + encodeURIComponent: it's slower than JS on normal engines, and modern Hermes already has TextEncoder
+  return js.encode(str, loose)
+}
+
+let escapes
+
+function toEscapesPart(arr, start, end) {
+  let o = ''
+  let i = start
+  const last3 = end - 3
+  // Unrolled loop is faster
+  while (i < last3) {
+    const a = arr[i++]
+    const b = arr[i++]
+    const c = arr[i++]
+    const d = arr[i++]
+    o += escapes[a]
+    o += escapes[b]
+    o += escapes[c]
+    o += escapes[d]
+  }
+
+  while (i < end) o += escapes[arr[i++]]
+  return o
+}
+
+function decode(arr, loose = false) {
+  assertUint8(arr)
+  if (haveDecoder) return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr) // Node.js and browsers
+  // No reason to use native Buffer: it's not faster than TextDecoder, needs rechecks in non-loose mode, and Node.js has TextDecoder
+
+  // This codepath gives a ~2x perf boost on Hermes
+  if (shouldUseEscapePath && escape && decodeURIComponent) {
+    if (!escapes) escapes = Array.from({ length: 256 }, (_, i) => escape(String.fromCharCode(i)))
+    const length = arr.length
+    let o
+    if (length > 30_000) {
+      // Limit concatenation to avoid excessive GC
+      // TODO: recheck thresholds on Hermes (taken from hex)
+      const concat = []
+      for (let i = 0; i < length; ) {
+        const step = i + 500
+        const end = step > length ? length : step
+        concat.push(toEscapesPart(arr, i, end))
+        i = end
+      }
+
+      o = concat.join('')
+      concat.length = 0
+    } else {
+      o = toEscapesPart(arr, 0, length)
+    }
+
+    try {
+      return decodeURIComponent(o) // asci to utf8, escape() is precalucated
+    } catch {
+      if (!loose) throw new TypeError(E_STRICT)
+      // Ok, we have to use manual implementation for loose decoder
+    }
+  }
+
+  return js.decode(arr, loose)
+}
+
+export const utf8fromString = (str, format = 'uint8') => typedView(encode(str, false), format)
+export const utf8fromStringLoose = (str, format = 'uint8') => typedView(encode(str, true), format)
+export const utf8toString = (arr) => decode(arr, false)
+export const utf8toStringLoose = (arr) => decode(arr, true)