@exodus/bytes 1.0.0-rc.6 → 1.0.0-rc.7

package/base58.js

@@ -1,7 +1,7 @@
 import { typedView } from './array.js'
 import { assertUint8 } from './assert.js'
 import { nativeDecoder, nativeEncoder } from './fallback/_utils.js'
-import { encodeAscii } from './fallback/latin1.js'
+import { encodeAscii, decodeAscii } from './fallback/latin1.js'
 
 const alphabet = [...'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz']
 const codes = new Uint8Array(alphabet.map((x) => x.charCodeAt(0)))
@@ -104,7 +104,7 @@ export function toBase58(arr) {
     while (carry) {
       const c = carry
       carry = Math.floor(c / 58)
-      res[i++] = c - carry * 58
+      res.push(c - carry * 58)
     }
   }
 
@@ -112,7 +112,7 @@ export function toBase58(arr) {
     const oa = new Uint8Array(res.length)
     let j = 0
     for (let i = res.length - 1; i >= 0; i--) oa[j++] = codes[res[i]]
-    return ZERO.repeat(zeros) + nativeDecoder.decode(oa)
+    return ZERO.repeat(zeros) + decodeAscii(oa)
   }
 
   let out = ''

package/base64.js

@@ -39,7 +39,7 @@ const haveWeb = (x) => web64 && x.toBase64 === web64
 export function toBase64(x, { padding = true } = {}) {
   assertUint8(x)
   if (haveWeb(x)) return padding ? x.toBase64() : x.toBase64({ omitPadding: !padding }) // Modern, optionless is slightly faster
-  if (haveNativeBuffer) return maybeUnpad(toBuffer(x).toString('base64'), padding) // Older Node.js
+  if (haveNativeBuffer) return maybeUnpad(toBuffer(x).base64Slice(0, x.byteLength), padding) // Older Node.js
   if (shouldUseBtoa) return maybeUnpad(btoa(decodeLatin1(x)), padding)
   return js.toBase64(x, false, padding) // Fallback
 }
@@ -48,7 +48,7 @@ export function toBase64(x, { padding = true } = {}) {
 export function toBase64url(x, { padding = false } = {}) {
   assertUint8(x)
   if (haveWeb(x)) return x.toBase64({ alphabet: 'base64url', omitPadding: !padding }) // Modern
-  if (haveNativeBuffer) return maybePad(toBuffer(x).toString('base64url'), padding) // Older Node.js
+  if (haveNativeBuffer) return maybePad(toBuffer(x).base64urlSlice(0, x.byteLength), padding) // Older Node.js
   if (shouldUseBtoa) return maybeUnpad(toUrl(btoa(decodeLatin1(x))), padding)
   return js.toBase64(x, true, padding) // Fallback
 }

package/fallback/_utils.js

@@ -3,4 +3,13 @@ const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
 const isNative = (x) => x && (haveNativeBuffer || `${x}`.includes('[native code]')) // we consider Node.js TextDecoder/TextEncoder native
 const nativeEncoder = isNative(TextEncoder) ? new TextEncoder() : null
 const nativeDecoder = isNative(TextDecoder) ? new TextDecoder('utf8', { ignoreBOM: true }) : null
-export { nativeEncoder, nativeDecoder }
+const nativeBuffer = haveNativeBuffer ? Buffer : null
+
+// Actually windows-1252, compatible with ascii and latin1 decoding
+// Beware that on non-latin1, i.e. on windows-1252, this is broken in ~all Node.js versions released
+// in 2025 due to a regression, so we call it Latin1 as it's usable only for that
+const nativeDecoderLatin1 = isNative(TextDecoder)
+  ? new TextDecoder('latin1', { ignoreBOM: true })
+  : null
+
+export { nativeEncoder, nativeDecoder, nativeDecoderLatin1, nativeBuffer }

package/fallback/base32.js

@@ -1,6 +1,6 @@
 import { assertUint8 } from '../assert.js'
 import { nativeEncoder, nativeDecoder } from './_utils.js'
-import { encodeAscii } from './latin1.js'
+import { encodeAscii, decodeAscii } from './latin1.js'
 
 // See https://datatracker.ietf.org/doc/html/rfc4648
 
@@ -68,7 +68,7 @@ export function toBase32(arr, isBase32Hex, padding) {
       j += 4
     }
 
-    o = nativeDecoder.decode(oa)
+    o = decodeAscii(oa)
   } else if (useTemplates) {
     // Templates are faster only on Hermes and JSC. Browsers have TextDecoder anyway
     for (; i < fullChunksBytes; i += 5) {

package/fallback/base64.js

@@ -1,6 +1,6 @@
 import { assertUint8 } from '../assert.js'
 import { nativeEncoder, nativeDecoder } from './_utils.js'
-import { encodeAscii } from './latin1.js'
+import { encodeAscii, decodeAscii } from './latin1.js'
 
 // See https://datatracker.ietf.org/doc/html/rfc4648
 
@@ -82,7 +82,7 @@ export function toBase64(arr, isURL, padding) {
       oa[j + 1] = codepairs[((b & 0x0f) << 8) | c]
     }
 
-    o = nativeDecoder.decode(oa)
+    o = decodeAscii(oa)
   } else {
     // This can be optimized by ~25% with templates on Hermes, but this codepath is not called on Hermes, it uses btoa
     // Check git history for templates version

package/fallback/hex.js

@@ -1,6 +1,6 @@
 import { assertUint8 } from '../assert.js'
 import { nativeDecoder, nativeEncoder } from './_utils.js'
-import { encodeAscii } from './latin1.js'
+import { encodeAscii, decodeAscii } from './latin1.js'
 
 let hexArray // array of 256 bytes converted to two-char hex strings
 let hexCodes // hexArray converted to u16 code pairs
@@ -97,7 +97,7 @@ export function toHex(arr) {
     }
 
     for (; i < length; i++) oa[i] = hexCodes[arr[i]]
-    return nativeDecoder.decode(oa)
+    return decodeAscii(oa)
   }
 
   if (length > 30_000) {

package/fallback/latin1.js

@@ -1,4 +1,4 @@
-import { nativeEncoder } from './_utils.js'
+import { nativeEncoder, nativeDecoder, nativeDecoderLatin1, nativeBuffer } from './_utils.js'
 
 // See http://stackoverflow.com/a/22747272/680742, which says that lowest limit is in Chrome, with 0xffff args
 // On Hermes, actual max is 0x20_000 minus current stack depth, 1/16 of that should be safe
@@ -32,6 +32,7 @@ export function asciiPrefix(arr) {
   return length
 }
 
+// Capable of decoding Uint16Array to UTF-16 as well as Uint8Array to Latin-1
 export function decodeLatin1(arr, start = 0, stop = arr.length) {
   start |= 0
   stop |= 0
@@ -52,10 +53,23 @@ export function decodeLatin1(arr, start = 0, stop = arr.length) {
   return String.fromCharCode.apply(String, sliced)
 }
 
-export const encodeLatin1 = globalThis.HermesInternal
-  ? (str) => {
+// Does not check input, uses best available method
+// Building an array for this is only faster than proper string concatenation when TextDecoder or native Buffer are available
+export const decodeAscii = nativeBuffer
+  ? (a) =>
+      // Buffer is faster on Node.js (but only for long enough data), if we know that output is ascii
+      a.byteLength >= 0x3_00
+        ? nativeBuffer.from(a.buffer, a.byteOffset, a.byteLength).latin1Slice(0, a.byteLength) // .latin1Slice is faster than .asciiSlice
+        : nativeDecoder.decode(a) // On Node.js, utf8 decoder is faster than latin1
+  : nativeDecoderLatin1
+    ? (a) => nativeDecoderLatin1.decode(a) // On browsers (specifically WebKit), latin1 decoder is faster than utf8
+    : (a) => decodeLatin1(new Uint8Array(a.buffer, a.byteOffset, a.byteLength)) // Fallback. We shouldn't get here, constructing with strings directly is faster
+
+/* eslint-disable @exodus/mutable/no-param-reassign-prop-only */
+
+export const encodeCharcodes = globalThis.HermesInternal
+  ? (str, arr) => {
       const length = str.length
-      const arr = new Uint8Array(length)
       if (length > 64) {
         const at = str.charCodeAt.bind(str) // faster on strings from ~64 chars on Hermes, but can be 10x slower on e.g. JSC
         for (let i = 0; i < length; i++) arr[i] = at(i)
@@ -65,14 +79,17 @@ export const encodeLatin1 = globalThis.HermesInternal
 
       return arr
     }
-  : (str) => {
+  : (str, arr) => {
       const length = str.length
-      const arr = new Uint8Array(length)
       // Can be optimized with unrolling, but this is not used on non-Hermes atm
       for (let i = 0; i < length; i++) arr[i] = str.charCodeAt(i)
       return arr
     }
 
+/* eslint-enable @exodus/mutable/no-param-reassign-prop-only */
+
+export const encodeLatin1 = (str) => encodeCharcodes(str, new Uint8Array(str.length))
+
 // Expects nativeEncoder to be present
 export const encodeAscii = globalThis.HermesInternal
   ? (str, ERR) => {
@@ -82,8 +99,15 @@ export const encodeAscii = globalThis.HermesInternal
       if (info.read !== str.length || info.written !== str.length) throw new SyntaxError(ERR) // non-ascii
       return codes.subarray(0, str.length)
     }
-  : (str, ERR) => {
-      const codes = nativeEncoder.encode(str)
-      if (codes.length !== str.length) throw new SyntaxError(ERR) // non-ascii
-      return codes
-    }
+  : nativeBuffer
+    ? (str, ERR) => {
+        // TextEncoder is slow on Node.js 24 / 25 (was ok on 22)
+        const codes = nativeBuffer.from(str, 'utf8') // ascii/latin1 coerces, we need to check
+        if (codes.length !== str.length) throw new SyntaxError(ERR) // non-ascii
+        return new Uint8Array(codes.buffer, codes.byteOffset, codes.byteLength)
+      }
+    : (str, ERR) => {
+        const codes = nativeEncoder.encode(str)
+        if (codes.length !== str.length) throw new SyntaxError(ERR) // non-ascii
+        return codes
+      }

package/hex.js

@@ -2,30 +2,16 @@ import { assertUint8 } from './assert.js'
 import { typedView } from './array.js'
 import * as js from './fallback/hex.js'
 
-const { Buffer } = globalThis // Buffer is optional, only used when native
-const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
 const { toHex: webHex } = Uint8Array.prototype // Modern engines have this
 
-const { E_HEX } = js
-
 export function toHex(arr) {
   assertUint8(arr)
   if (arr.length === 0) return ''
   if (webHex && arr.toHex === webHex) return arr.toHex()
-  if (!haveNativeBuffer) return js.toHex(arr)
-  if (arr.constructor === Buffer && Buffer.isBuffer(arr)) return arr.toString('hex')
-  return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).toString('hex')
+  return js.toHex(arr)
 }
 
 // Unlike Buffer.from(), throws on invalid input
 export const fromHex = Uint8Array.fromHex
   ? (str, format = 'uint8') => typedView(Uint8Array.fromHex(str), format)
-  : haveNativeBuffer
-    ? (str, format = 'uint8') => {
-        if (typeof str !== 'string') throw new TypeError('Input is not a string')
-        if (str.length % 2 !== 0) throw new SyntaxError(E_HEX)
-        const buf = Buffer.from(str, 'hex') // will stop on first non-hex character, so we can just validate length
-        if (buf.length * 2 !== str.length) throw new SyntaxError(E_HEX)
-        return typedView(buf, format)
-      }
-    : (str, format = 'uint8') => typedView(js.fromHex(str), format)
+  : (str, format = 'uint8') => typedView(js.fromHex(str), format)

package/hex.node.js

@@ -0,0 +1,26 @@
+import { assertUint8 } from './assert.js'
+import { typedView } from './array.js'
+import { E_HEX } from './fallback/hex.js'
+
+if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
+
+const { toHex: webHex } = Uint8Array.prototype // Modern engines have this
+
+export function toHex(arr) {
+  assertUint8(arr)
+  if (arr.length === 0) return ''
+  if (webHex && arr.toHex === webHex) return arr.toHex()
+  if (arr.constructor === Buffer && Buffer.isBuffer(arr)) return arr.hexSlice(0, arr.byteLength)
+  return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).hexSlice(0, arr.byteLength)
+}
+
+// Unlike Buffer.from(), throws on invalid input
+export const fromHex = Uint8Array.fromHex
+  ? (str, format = 'uint8') => typedView(Uint8Array.fromHex(str), format)
+  : (str, format = 'uint8') => {
+      if (typeof str !== 'string') throw new TypeError('Input is not a string')
+      if (str.length % 2 !== 0) throw new SyntaxError(E_HEX)
+      const buf = Buffer.from(str, 'hex') // will stop on first non-hex character, so we can just validate length
+      if (buf.length * 2 !== str.length) throw new SyntaxError(E_HEX)
+      return typedView(buf, format)
+    }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@exodus/bytes",
-  "version": "1.0.0-rc.6",
+  "version": "1.0.0-rc.7",
   "description": "Various operations on Uint8Array data",
   "scripts": {
     "lint": "eslint .",
@@ -50,7 +50,9 @@
     "/base58.js",
     "/base58check.js",
     "/base64.js",
+    "/hex.node.js",
     "/hex.js",
+    "/utf8.node.js",
     "/utf8.js"
   ],
   "exports": {
@@ -59,8 +61,18 @@
     "./base58.js": "./base58.js",
     "./base58check.js": "./base58check.js",
     "./base64.js": "./base64.js",
-    "./hex.js": "./hex.js",
-    "./utf8.js": "./utf8.js"
+    "./hex.js": {
+      "node": "./hex.node.js",
+      "default": "./hex.js"
+    },
+    "./utf16.js": {
+      "node": "./utf16.node.js",
+      "default": "./utf16.js"
+    },
+    "./utf8.js": {
+      "node": "./utf8.node.js",
+      "default": "./utf8.js"
+    }
   },
   "peerDependencies": {
     "@exodus/crypto": "^1.0.0-rc.4"
@@ -94,6 +106,7 @@
     "fast-base64-encode": "^1.0.0",
     "hextreme": "^1.0.7",
     "hi-base32": "^0.5.1",
+    "iconv-lite": "^0.7.0",
     "jsvu": "^3.0.0",
     "text-encoding": "^0.7.0",
     "typescript": "^5.9.3"

package/utf8.node.js

@@ -0,0 +1,33 @@
+import { assertUint8 } from './assert.js'
+import { typedView } from './array.js'
+import { E_STRICT_UNICODE } from './fallback/utf8.js'
+import { isAscii } from 'node:buffer'
+
+if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
+
+const decoderFatal = new TextDecoder('utf8', { ignoreBOM: true, fatal: true })
+const decoderLoose = new TextDecoder('utf8', { ignoreBOM: true })
+const { isWellFormed } = String.prototype
+
+function encode(str, loose = false) {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  const res = Buffer.from(str)
+  if (loose || str.length === res.length || isWellFormed.call(str)) return res // length is equal only for ascii, which is automatically fine
+  throw new TypeError(E_STRICT_UNICODE)
+}
+
+function decode(arr, loose = false) {
+  assertUint8(arr)
+  if (isAscii(arr)) {
+    // On non-ascii strings, this loses ~10% * [relative position of the first non-ascii byte] (up to 10% total)
+    // On ascii strings, this wins 1.5x on loose = false and 1.3x on loose = true
+    return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).latin1Slice(0, arr.byteLength) // .latin1Slice is faster than .asciiSlice
+  }
+
+  return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr)
+}
+
+export const utf8fromString = (str, format = 'uint8') => typedView(encode(str, false), format)
+export const utf8fromStringLoose = (str, format = 'uint8') => typedView(encode(str, true), format)
+export const utf8toString = (arr) => decode(arr, false)
+export const utf8toStringLoose = (arr) => decode(arr, true)