@exodus/bytes 1.8.0 → 1.9.0

package/fallback/single-byte.js

@@ -13,7 +13,7 @@ export const assertEncoding = (encoding) => {
 
 const r = 0xff_fd
 
-function getEncoding(encoding) {
+export function getEncoding(encoding) {
   assertEncoding(encoding)
   if (encoding === xUserDefined) return Array.from({ length: 128 }, (_, i) => 0xf7_80 + i)
   if (encoding === iso8i) encoding = 'iso-8859-8'

package/fallback/utf16.js

@@ -12,45 +12,48 @@ const to16 = (a) => new Uint16Array(a.buffer, a.byteOffset, a.byteLength / 2) //
 export function to16input(u8, le) {
   // Assume even number of bytes
   if (le === isLE) return to16(u8.byteOffset % 2 === 0 ? u8 : Uint8Array.from(u8))
+  return to16(swap16(Uint8Array.from(u8)))
+}
+
+export const decode = (u16, loose = false, checked = false) => {
+  if (checked || isWellFormed(u16)) return decodeUCS2(u16)
+  if (!loose) throw new TypeError(E_STRICT)
+  return decodeUCS2(toWellFormed(Uint16Array.from(u16))) // cloned for replacement
+}
+
+export function encode(str, loose = false, checked = false, swapped = false) {
+  const arr = new Uint16Array(str.length)
+  if (checked) return swapped ? encodeCheckedSwapped(str, arr) : encodeChecked(str, arr)
+  return swapped ? encodeUncheckedSwapped(str, arr, loose) : encodeUnchecked(str, arr, loose)
+}
 
-  const res = new Uint8Array(u8.length)
+/* eslint-disable @exodus/mutable/no-param-reassign-prop-only */
 
+// Assumes checked length % 2 === 0, otherwise does not swap tail
+function swap16(u8) {
   let i = 0
   for (const last3 = u8.length - 3; i < last3; i += 4) {
     const x0 = u8[i]
     const x1 = u8[i + 1]
     const x2 = u8[i + 2]
     const x3 = u8[i + 3]
-    res[i] = x1
-    res[i + 1] = x0
-    res[i + 2] = x3
-    res[i + 3] = x2
+    u8[i] = x1
+    u8[i + 1] = x0
+    u8[i + 2] = x3
+    u8[i + 3] = x2
   }
 
   for (const last = u8.length - 1; i < last; i += 2) {
     const x0 = u8[i]
     const x1 = u8[i + 1]
-    res[i] = x1
-    res[i + 1] = x0
+    u8[i] = x1
+    u8[i + 1] = x0
   }
 
-  return to16(res)
-}
-
-export const decode = (u16, loose = false, checked = false) => {
-  if (checked || isWellFormed(u16)) return decodeUCS2(u16)
-  if (!loose) throw new TypeError(E_STRICT)
-  return decodeUCS2(toWellFormed(Uint16Array.from(u16))) // cloned for replacement
-}
-
-export function encode(str, loose = false, checked = false, swapped = false) {
-  const arr = new Uint16Array(str.length)
-  if (checked) return swapped ? encodeCheckedSwapped(str, arr) : encodeChecked(str, arr)
-  return swapped ? encodeUncheckedSwapped(str, arr, loose) : encodeUnchecked(str, arr, loose)
+  return u8
 }
 
 // Splitting paths into small functions helps (at least on SpiderMonkey)
-/* eslint-disable @exodus/mutable/no-param-reassign-prop-only */
 
 const encodeChecked = (str, arr) => encodeCharcodes(str, arr) // Same as encodeLatin1, but with Uint16Array
 
@@ -121,6 +124,7 @@ function encodeUncheckedSwapped(str, arr, loose = false) {
   return arr
 }
 
+// Only needed on Hermes, everything else has native impl
 export function toWellFormed(u16) {
   const length = u16.length
   for (let i = 0; i < length; i++) {
@@ -143,10 +147,15 @@ export function toWellFormed(u16) {
   return u16
 }
 
+// Only needed on Hermes, everything else has native impl
 export function isWellFormed(u16) {
   const length = u16.length
   let i = 0
 
+  const m = 0x80_00_80_00
+  const l = 0xd8_00
+  const h = 0xe0_00
+
   // Speedup with u32, by skipping to the first surrogate
   // Only implemented for aligned input for now, but almost all input is aligned (pooled Buffer or 0 offset)
   if (length > 32 && u16.byteOffset % 4 === 0) {
@@ -158,21 +167,31 @@ export function isWellFormed(u16) {
       const b = u32[i + 1]
       const c = u32[i + 2]
       const d = u32[i + 3]
-      if (a & 0x80_00_80_00 || b & 0x80_00_80_00 || c & 0x80_00_80_00 || d & 0x80_00_80_00) break
+      if (a & m || b & m || c & m || d & m) break // bitwise OR does not make this faster on Hermes
     }
 
-    for (; i < u32length; i++) if (u32[i] & 0x80_00_80_00) break
+    for (; i < u32length; i++) if (u32[i] & m) break
     i *= 2
   }
 
+  // An extra loop gives ~30-40% speedup e.g. on English text without surrogates but with other symbols above 0x80_00
+  for (const last3 = length - 3; ; i += 4) {
+    if (i >= last3) break
+    const a = u16[i]
+    const b = u16[i + 1]
+    const c = u16[i + 2]
+    const d = u16[i + 3]
+    if ((a >= l && a < h) || (b >= l && b < h) || (c >= l && c < h) || (d >= l && d < h)) break
+  }
+
   for (; i < length; i++) {
     const code = u16[i]
-    if (code >= 0xd8_00 && code < 0xe0_00) {
+    if (code >= l && code < h) {
       // An unexpected trail or a lead at the very end of input
-      if (code > 0xdb_ff || i + 1 >= length) return false
+      if (code >= 0xdc_00 || i + 1 >= length) return false
       i++ // consume next
       const next = u16[i] // Process valid pairs immediately
-      if (next < 0xdc_00 || next >= 0xe0_00) return false
+      if (next < 0xdc_00 || next >= h) return false
     }
   }
 

package/fallback/utf8.js

@@ -191,7 +191,7 @@ export function encode(string, loose) {
 
     if (small) {
       // TODO: use resizable array buffers? will have to return a non-resizeable one
-      if (p !== i) throw new Error('Unreachable') // Here, p === i (only when small is still true)
+      if (p !== i) /* c8 ignore next */ throw new Error('Unreachable') // Here, p === i (only when small is still true)
       const bytesNew = new Uint8Array(p + (length - i) * 3) // maximium can be 3x of the string length in charcodes
       bytesNew.set(bytes)
       bytes = bytesNew

package/hex.d.ts

@@ -19,4 +19,3 @@ export function toHex(arr: Uint8ArrayBuffer): string;
 export function fromHex(str: string, format?: 'uint8'): Uint8ArrayBuffer;
 export function fromHex(str: string, format: 'buffer'): Buffer;
 export function fromHex(str: string, format?: OutputFormat): Uint8ArrayBuffer | Buffer;
-

package/index.d.ts

@@ -0,0 +1,43 @@
+/**
+ * ### The `@exodus/bytes` package consists of submodules, there is no single export.
+ * Import specific submodules instead.
+ *
+ * See [README](https://github.com/ExodusOSS/bytes/blob/main/README.md).
+ *
+ * Example:
+ * ```js
+ * import { fromHex, toHex } from '@exodus/bytes/hex.js'
+ * import { fromBase64, toBase64, fromBase64url, toBase64url, fromBase64any } from '@exodus/bytes/base64.js'
+ * import { fromBase32, toBase32, fromBase32hex, toBase32hex } from '@exodus/bytes/base32.js'
+ * import { fromBase58, toBase58, fromBase58xrp, toBase58xrp } from '@exodus/bytes/base58.js'
+ * import { fromBech32, toBech32, fromBech32m, toBech32m, getPrefix } from '@exodus/bytes/bech32.js'
+ * import { fromBigInt, toBigInt } from '@exodus/bytes/bigint.js'
+ *
+ * import { utf8fromString, utf8toString, utf8fromStringLoose, utf8toStringLoose } from '@exodus/bytes/utf8.js'
+ * import { utf16fromString, utf16toString, utf16fromStringLoose, utf16toStringLoose } from '@exodus/bytes/utf16.js'
+ * import {
+ *   createSinglebyteDecoder, createSinglebyteEncoder,
+ *   windows1252toString, windows1252fromString,
+ *   latin1toString, latin1fromString } from '@exodus/bytes/single-byte.js'
+ * import { createMultibyteDecoder, createMultibyteEncoder } from '@exodus/bytes/multi-byte.js'
+ *
+ * import {
+ *   fromBase58check, toBase58check,
+ *   fromBase58checkSync, toBase58checkSync,
+ *   makeBase58check } from '@exodus/bytes/base58check.js'
+ * import { fromWifString, toWifString, fromWifStringSync, toWifStringSync } from '@exodus/bytes/wif.js'
+ *
+ * // All encodings from the WHATWG Encoding spec
+ * import { TextDecoder, TextEncoder, TextDecoderStream, TextEncoderStream } from '@exodus/bytes/encoding.js'
+ * import { getBOMEncoding, legacyHookDecode, labelToName, normalizeEncoding } from '@exodus/bytes/encoding.js'
+ *
+ * // Omits legacy multi-byte decoders to save bundle size
+ * import { TextDecoder, TextEncoder, TextDecoderStream, TextEncoderStream } from '@exodus/bytes/encoding-lite.js'
+ * import { getBOMEncoding, legacyHookDecode, labelToName, normalizeEncoding } from '@exodus/bytes/encoding-lite.js'
+ *
+ * // In browser bundles, uses built-in TextDecoder / TextEncoder to save bundle size
+ * import { TextDecoder, TextEncoder, TextDecoderStream, TextEncoderStream } from '@exodus/bytes/encoding-browser.js'
+ * import { getBOMEncoding, legacyHookDecode, labelToName, normalizeEncoding } from '@exodus/bytes/encoding-browser.js'
+ * ```
+ */
+declare module "@exodus/bytes" {}

package/index.js

@@ -0,0 +1,5 @@
+throw new Error(
+  `This package consists of submodules, there is no single export. Import specific submodules instead.
+See README: https://github.com/ExodusOSS/bytes/blob/main/README.md
+`
+)

package/multi-byte.js

@@ -1,5 +1,5 @@
 import { assertUint8 } from './assert.js'
-import { multibyteDecoder } from './fallback/multi-byte.js'
+import { multibyteDecoder, multibyteEncoder } from './fallback/multi-byte.js'
 
 export function createMultibyteDecoder(encoding, loose = false) {
   const jsDecoder = multibyteDecoder(encoding, loose) // asserts
@@ -11,3 +11,9 @@ export function createMultibyteDecoder(encoding, loose = false) {
     return jsDecoder(arr, stream)
   }
 }
+
+export function createMultibyteEncoder(encoding, { mode = 'fatal' } = {}) {
+  // TODO: replacement, truncate (replacement will need varying length)
+  if (mode !== 'fatal') throw new Error('Unsupported mode')
+  return multibyteEncoder(encoding) // asserts
+}

package/multi-byte.node.js

@@ -1,6 +1,6 @@
 import { assertUint8 } from './assert.js'
 import { isDeno, toBuf } from './fallback/_utils.js'
-import { isAsciiSuperset, multibyteDecoder } from './fallback/multi-byte.js'
+import { isAsciiSuperset, multibyteDecoder, multibyteEncoder } from './fallback/multi-byte.js'
 import { isAscii } from 'node:buffer'
 
 export function createMultibyteDecoder(encoding, loose = false) {
@@ -21,3 +21,9 @@ export function createMultibyteDecoder(encoding, loose = false) {
     return jsDecoder(arr, stream)
   }
 }
+
+export function createMultibyteEncoder(encoding, { mode = 'fatal' } = {}) {
+  // TODO: replacement, truncate (replacement will need varying length)
+  if (mode !== 'fatal') throw new Error('Unsupported mode')
+  return multibyteEncoder(encoding) // asserts
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@exodus/bytes",
-  "version": "1.8.0",
+  "version": "1.9.0",
   "description": "Various operations on Uint8Array data",
   "scripts": {
     "lint": "eslint .",
@@ -23,6 +23,7 @@
     "test:firefox:playwright": "exodus-test --engine=firefox:playwright",
     "test:servo:bundle": "exodus-test --engine=servo:bundle",
     "test": "exodus-test",
+    "size": "esbuild --minify --bundle",
     "jsvu": "jsvu",
     "playwright": "exodus-test --playwright",
     "benchmark": "exodus-test --concurrency=1 benchmarks/*.bench.js",
@@ -48,6 +49,7 @@
     "/fallback/base58check.js",
     "/fallback/base64.js",
     "/fallback/encoding.js",
+    "/fallback/encoding.api.js",
     "/fallback/encoding.labels.js",
     "/fallback/encoding.util.js",
     "/fallback/hex.js",
@@ -71,6 +73,10 @@
     "/base64.d.ts",
     "/bech32.js",
     "/bigint.js",
+    "/encoding-browser.js",
+    "/encoding-browser.browser.js",
+    "/encoding-browser.native.js",
+    "/encoding-browser.d.ts",
     "/encoding.js",
     "/encoding.d.ts",
     "/encoding-lite.js",
@@ -78,6 +84,8 @@
     "/hex.js",
     "/hex.d.ts",
     "/hex.node.js",
+    "/index.js",
+    "/index.d.ts",
     "/multi-byte.js",
     "/multi-byte.node.js",
     "/single-byte.js",
@@ -89,7 +97,14 @@
     "/utf8.node.js",
     "/wif.js"
   ],
+  "main": "index.js",
+  "module": "index.js",
+  "types": "index.d.ts",
   "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./index.js"
+    },
     "./array.js": {
       "types": "./array.d.ts",
       "default": "./array.js"
@@ -127,6 +142,13 @@
       "types": "./encoding-lite.d.ts",
       "default": "./encoding-lite.js"
     },
+    "./encoding-browser.js": {
+      "types": "./encoding-browser.d.ts",
+      "node": "./encoding-browser.js",
+      "react-native": "./encoding-browser.native.js",
+      "browser": "./encoding-browser.browser.js",
+      "default": "./encoding-browser.js"
+    },
     "./utf16.js": {
       "node": "./utf16.node.js",
       "default": "./utf16.js"
@@ -138,11 +160,14 @@
     },
     "./wif.js": "./wif.js"
   },
+  "react-native": {
+    "./encoding-browser.js": "./encoding-browser.native.js"
+  },
   "peerDependencies": {
-    "@exodus/crypto": "^1.0.0-rc.4"
+    "@noble/hashes": "^1.8.0 || ^2.0.0"
   },
   "peerDependenciesMeta": {
-    "@exodus/crypto": {
+    "@noble/hashes": {
       "optional": true
     }
   },
@@ -153,6 +178,7 @@
     "@exodus/prettier": "^1.0.0",
     "@exodus/test": "^1.0.0-rc.109",
     "@noble/hashes": "^2.0.1",
+    "@oslojs/encoding": "^1.1.0",
     "@petamoriken/float16": "^3.9.3",
     "@scure/base": "^1.2.6",
     "@stablelib/base64": "^2.0.1",
@@ -167,9 +193,11 @@
     "bs58check": "^4.0.0",
     "bstring": "^0.3.9",
     "buffer": "^6.0.3",
+    "c8": "^10.1.3",
     "decode-utf8": "^1.0.1",
     "electron": "36.5.0",
     "encode-utf8": "^2.0.0",
+    "esbuild": "^0.27.2",
     "eslint": "^8.44.0",
     "fast-base64-decode": "^2.0.0",
     "fast-base64-encode": "^1.0.0",
@@ -177,6 +205,7 @@
     "hi-base32": "^0.5.1",
     "iconv-lite": "^0.7.0",
     "jsvu": "^3.0.3",
+    "punycode": "^2.3.1",
     "text-encoding": "^0.7.0",
     "typescript": "^5.9.3",
     "uint8array-tools": "^0.0.9",

package/single-byte.js

@@ -1,5 +1,5 @@
 import { assertUint8 } from './assert.js'
-import { canDecoders, nativeEncoder, isHermes, skipWeb, E_STRING } from './fallback/_utils.js'
+import { canDecoders, nativeEncoder, skipWeb, E_STRING } from './fallback/_utils.js'
 import { encodeAscii, encodeAsciiPrefix, encodeLatin1 } from './fallback/latin1.js'
 import { assertEncoding, encodingDecoder, encodeMap, E_STRICT } from './fallback/single-byte.js'
 
@@ -66,17 +66,15 @@ function encode(s, m) {
   const x = new Uint8Array(len)
   let i = nativeEncoder ? 0 : encodeAsciiPrefix(x, s)
 
-  if (!isHermes) {
-    for (const len3 = len - 3; i < len3; i += 4) {
-      const x0 = s.charCodeAt(i), x1 = s.charCodeAt(i + 1), x2 = s.charCodeAt(i + 2), x3 = s.charCodeAt(i + 3) // prettier-ignore
-      const c0 = m[x0], c1 = m[x1], c2 = m[x2], c3 = m[x3] // prettier-ignore
-      if ((!c0 && x0) || (!c1 && x1) || (!c2 && x2) || (!c3 && x3)) throw new TypeError(E_STRICT)
+  for (const len3 = len - 3; i < len3; i += 4) {
+    const x0 = s.charCodeAt(i), x1 = s.charCodeAt(i + 1), x2 = s.charCodeAt(i + 2), x3 = s.charCodeAt(i + 3) // prettier-ignore
+    const c0 = m[x0], c1 = m[x1], c2 = m[x2], c3 = m[x3] // prettier-ignore
+    if ((!c0 && x0) || (!c1 && x1) || (!c2 && x2) || (!c3 && x3)) return null
 
-      x[i] = c0
-      x[i + 1] = c1
-      x[i + 2] = c2
-      x[i + 3] = c3
-    }
+    x[i] = c0
+    x[i + 1] = c1
+    x[i + 2] = c2
+    x[i + 3] = c3
   }
 
   for (; i < len; i++) {

package/single-byte.node.js

@@ -61,6 +61,32 @@ export function createSinglebyteDecoder(encoding, loose = false) {
 
 const NON_LATIN = /[^\x00-\xFF]/ // eslint-disable-line no-control-regex
 
+function encode(s, m) {
+  const len = s.length
+  let i = 0
+  const b = Buffer.from(s, 'utf-16le') // aligned
+  if (!isLE) b.swap16()
+  const x = new Uint16Array(b.buffer, b.byteOffset, b.byteLength / 2)
+  for (const len3 = len - 3; i < len3; i += 4) {
+    const x0 = x[i], x1 = x[i + 1], x2 = x[i + 2], x3 = x[i + 3] // prettier-ignore
+    const c0 = m[x0], c1 = m[x1], c2 = m[x2], c3 = m[x3] // prettier-ignore
+    if (!(c0 && c1 && c2 && c3) && ((!c0 && x0) || (!c1 && x1) || (!c2 && x2) || (!c3 && x3))) return null // prettier-ignore
+    x[i] = c0
+    x[i + 1] = c1
+    x[i + 2] = c2
+    x[i + 3] = c3
+  }
+
+  for (; i < len; i++) {
+    const x0 = x[i]
+    const c0 = m[x0]
+    if (!c0 && x0) return null
+    x[i] = c0
+  }
+
+  return new Uint8Array(x)
+}
+
 export function createSinglebyteEncoder(encoding, { mode = 'fatal' } = {}) {
   // TODO: replacement, truncate (replacement will need varying length)
   if (mode !== 'fatal') throw new Error('Unsupported mode')
@@ -82,32 +108,9 @@ export function createSinglebyteEncoder(encoding, { mode = 'fatal' } = {}) {
       if (b.length === s.length) return new Uint8Array(b.buffer, b.byteOffset, b.byteLength)
     }
 
-    const len = s.length
-    let i = 0
-    const b = Buffer.from(s, 'utf-16le') // aligned
-    if (!isLE) b.swap16()
-    const x = new Uint16Array(b.buffer, b.byteOffset, b.byteLength / 2)
-    for (const len3 = len - 3; i < len3; i += 4) {
-      const x0 = x[i], x1 = x[i + 1], x2 = x[i + 2], x3 = x[i + 3] // prettier-ignore
-      const c0 = m[x0], c1 = m[x1], c2 = m[x2], c3 = m[x3] // prettier-ignore
-      if (!(c0 && c1 && c2 && c3) && ((!c0 && x0) || (!c1 && x1) || (!c2 && x2) || (!c3 && x3))) {
-        throw new TypeError(E_STRICT)
-      }
-
-      x[i] = c0
-      x[i + 1] = c1
-      x[i + 2] = c2
-      x[i + 3] = c3
-    }
-
-    for (; i < len; i++) {
-      const x0 = x[i]
-      const c0 = m[x0]
-      if (!c0 && x0) throw new TypeError(E_STRICT)
-      x[i] = c0
-    }
-
-    return new Uint8Array(x)
+    const res = encode(s, m)
+    if (!res) throw new TypeError(E_STRICT)
+    return res
   }
 }
 

package/utf16.js

@@ -34,6 +34,7 @@ function encode(str, loose = false, format = 'uint16') {
 
   if (format === 'uint8-le' || format === 'uint8-be') return to8(u16) // Already swapped
   if (format === 'uint16') return u16
+  /* c8 ignore next */
   throw new Error('Unreachable')
 }
 

package/utf16.node.js

@@ -30,11 +30,15 @@ function encode(str, loose = false, format = 'uint16') {
     return new Uint16Array(b.buffer, b.byteOffset, b.byteLength / 2)
   }
 
+  /* c8 ignore next */
   throw new Error('Unreachable')
 }
 
-const swapped = (x, swap) =>
-  swap ? Buffer.from(x).swap16() : Buffer.from(x.buffer, x.byteOffset, x.byteLength)
+// Convert to Buffer view or a swapped Buffer copy
+const swapped = (x, swap) => {
+  const b = Buffer.from(x.buffer, x.byteOffset, x.byteLength)
+  return swap ? Buffer.from(b).swap16() : b
+}
 
 // We skip TextDecoder on Node.js, as it's is somewhy significantly slower than Buffer for utf16
 // Also, it incorrectly misses replacements with Node.js is built without ICU, we fix that