@exodus/bytes 1.1.0 → 1.3.0

package/README.md

@@ -41,6 +41,7 @@ Spec compliant, passing WPT and covered with extra tests.
 Moreover, tests for this library uncovered [bugs in all major implementations](https://docs.google.com/spreadsheets/d/1pdEefRG6r9fZy61WHGz0TKSt8cO4ISWqlpBN5KntIvQ/edit).
 
 [Faster than Node.js native implementation on Node.js](https://github.com/nodejs/node/issues/61041#issuecomment-3649242024).
+Runs (and passes WPT) on Node.js built without ICU.
 
 ### Caveat: `TextDecoder` / `TextEncoder` APIs are lossy by default per spec
 
@@ -160,6 +161,8 @@ Same as `windows1252toString = createSinglebyteDecoder('windows-1252')`.
 
 ### `@exodus/bytes/base58check.js`
 
+On non-Node.js, requires peer dependency [@exodus/crypto](https://www.npmjs.com/package/@exodus/crypto) to be installed.
+
 ##### `async toBase58check(arr)`
 ##### `toBase58checkSync(arr)`
 ##### `async fromBase58check(str, format = 'uint8')`
@@ -184,7 +187,7 @@ some [hooks](https://encoding.spec.whatwg.org/#specification-hooks) (see below).
 import { TextDecoder, TextDecoder } from '@exodus/bytes/encoding.js'
 
 // Hooks for standards
-import { getBOMEncoding, legacyHookDecode, normalizeEncoding } from '@exodus/bytes/encoding.js'
+import { getBOMEncoding, legacyHookDecode, labelToName, normalizeEncoding } from '@exodus/bytes/encoding.js'
 ```
 
 #### `new TextDecoder(label = 'utf-8', { fatal = false, ignoreBOM = false })`
@@ -195,10 +198,19 @@ import { getBOMEncoding, legacyHookDecode, normalizeEncoding } from '@exodus/byt
 
 [TextEncoder](https://encoding.spec.whatwg.org/#interface-textdecoder) implementation/polyfill.
 
-#### `normalizeEncoding(label)`
+#### `labelToName(label)`
 
 Implements [get an encoding from a string `label`](https://encoding.spec.whatwg.org/#concept-encoding-get).
 
+Converts an encoding [label](https://encoding.spec.whatwg.org/#names-and-labels) to its name,
+as a case-sensitive string.
+
+If an encoding with that label does not exist, returns `null`.
+
+All encoding names are also valid labels for corresponding encodings.
+
+#### `normalizeEncoding(label)`
+
 Converts an encoding [label](https://encoding.spec.whatwg.org/#names-and-labels) to its name,
 as an ASCII-lowercased string.
 
@@ -210,6 +222,11 @@ except that it:
     [labels](https://encoding.spec.whatwg.org/#ref-for-replacement%E2%91%A1)
  2. Does not throw for invalid labels and instead returns `null`
 
+It is identical to:
+```js
+labelToName(label)?.toLowerCase() ?? null
+```
+
 All encoding names are also valid labels for corresponding encodings.
 
 #### `getBOMEncoding(input)`
@@ -251,7 +268,7 @@ new TextDecoder(getBOMEncoding(input) ?? fallbackEncoding).decode(input)
 import { TextDecoder, TextDecoder } from '@exodus/bytes/encoding-lite.js'
 
 // Hooks for standards
-import { getBOMEncoding, legacyHookDecode, normalizeEncoding } from '@exodus/bytes/encoding-lite.js'
+import { getBOMEncoding, legacyHookDecode, labelToName, normalizeEncoding } from '@exodus/bytes/encoding-lite.js'
 ```
 
 The exact same exports as `@exodus/bytes/encoding.js` are also exported as
@@ -263,7 +280,7 @@ and their [labels](https://encoding.spec.whatwg.org/#names-and-labels) when used
 
 Legacy single-byte encodingds are loaded by default in both cases.
 
-`TextEncoder` and hooks for standards (including `normalizeEncoding`) do not have any behavior
+`TextEncoder` and hooks for standards (including `labelToName` / `normalizeEncoding`) do not have any behavior
 differences in the lite version and support full range if inputs.
 
 To avoid inconsistencies, the exported classes and methods are exactly the same objects.
@@ -274,6 +291,7 @@ To avoid inconsistencies, the exported classes and methods are exactly the same
   TextDecoder: [class TextDecoder],
   TextEncoder: [class TextEncoder],
   getBOMEncoding: [Function: getBOMEncoding],
+  labelToName: [Function: labelToName],
   legacyHookDecode: [Function: legacyHookDecode],
   normalizeEncoding: [Function: normalizeEncoding]
 }
@@ -286,6 +304,7 @@ Error: Legacy multi-byte encodings are disabled in /encoding-lite.js, use /encod
   TextDecoder: [class TextDecoder],
   TextEncoder: [class TextEncoder],
   getBOMEncoding: [Function: getBOMEncoding],
+  labelToName: [Function: labelToName],
   legacyHookDecode: [Function: legacyHookDecode],
   normalizeEncoding: [Function: normalizeEncoding]
 }

package/base58check.js

@@ -1,63 +1,12 @@
-import { typedView } from './array.js'
-import { assertUint8 } from './assert.js'
-import { toBase58, fromBase58 } from './base58.js'
 import { hashSync } from '@exodus/crypto/hash' // eslint-disable-line @exodus/import/no-deprecated
+import { makeBase58check } from './fallback/base58check.js'
 
 // Note: while API is async, we use hashSync for now until we improve webcrypto perf for hash256
 // Inputs to base58 are typically very small, and that makes a difference
 
-const E_CHECKSUM = 'Invalid checksum'
-
-// checksum length is 4, i.e. only the first 4 bytes of the hash are used
-
-function encodeWithChecksum(arr, checksum) {
-  // arr type in already validated in input
-  const res = new Uint8Array(arr.length + 4)
-  res.set(arr, 0)
-  res.set(checksum.subarray(0, 4), arr.length)
-  return toBase58(res)
-}
-
-function decodeWithChecksum(str) {
-  const arr = fromBase58(str) // checks input
-  const payloadSize = arr.length - 4
-  if (payloadSize < 0) throw new Error(E_CHECKSUM)
-  return [arr.subarray(0, payloadSize), arr.subarray(payloadSize)]
-}
-
-function assertChecksum(c, r) {
-  if ((c[0] ^ r[0]) | (c[1] ^ r[1]) | (c[2] ^ r[2]) | (c[3] ^ r[3])) throw new Error(E_CHECKSUM)
-}
-
-export const makeBase58check = (hashAlgo, hashAlgoSync) => {
-  const apis = {
-    async encode(arr) {
-      assertUint8(arr)
-      return encodeWithChecksum(arr, await hashAlgo(arr))
-    },
-    async decode(str, format = 'uint8') {
-      const [payload, checksum] = decodeWithChecksum(str)
-      assertChecksum(checksum, await hashAlgo(payload))
-      return typedView(payload, format)
-    },
-  }
-  if (!hashAlgoSync) return apis
-  return {
-    ...apis,
-    encodeSync(arr) {
-      assertUint8(arr)
-      return encodeWithChecksum(arr, hashAlgoSync(arr))
-    },
-    decodeSync(str, format = 'uint8') {
-      const [payload, checksum] = decodeWithChecksum(str)
-      assertChecksum(checksum, hashAlgoSync(payload))
-      return typedView(payload, format)
-    },
-  }
-}
-
 // eslint-disable-next-line @exodus/import/no-deprecated
-const hash256sync = (x) => hashSync('sha256', hashSync('sha256', x, 'uint8'), 'uint8')
+const sha256 = (x) => hashSync('sha256', x, 'uint8')
+const hash256sync = (x) => sha256(sha256(x))
 const hash256 = hash256sync // See note at the top
 const {
   encode: toBase58check,
@@ -66,4 +15,5 @@ const {
   decodeSync: fromBase58checkSync,
 } = makeBase58check(hash256, hash256sync)
 
+export { makeBase58check } from './fallback/base58check.js'
 export { toBase58check, fromBase58check, toBase58checkSync, fromBase58checkSync }

package/base58check.node.js

@@ -0,0 +1,14 @@
+import { hash } from 'node:crypto'
+import { makeBase58check } from './fallback/base58check.js'
+
+const sha256 = (x) => hash('sha256', x, 'buffer')
+const hash256 = (x) => sha256(sha256(x))
+const {
+  encode: toBase58check,
+  decode: fromBase58check,
+  encodeSync: toBase58checkSync,
+  decodeSync: fromBase58checkSync,
+} = makeBase58check(hash256, hash256)
+
+export { makeBase58check } from './fallback/base58check.js'
+export { toBase58check, fromBase58check, toBase58checkSync, fromBase58checkSync }

package/encoding-lite.js

@@ -3,5 +3,6 @@ export {
   TextEncoder,
   normalizeEncoding,
   getBOMEncoding,
+  labelToName,
   legacyHookDecode,
 } from './fallback/encoding.js'

package/encoding.js

@@ -8,5 +8,6 @@ export {
   TextEncoder,
   normalizeEncoding,
   getBOMEncoding,
+  labelToName,
   legacyHookDecode,
 } from './fallback/encoding.js'

package/fallback/base58check.js

@@ -0,0 +1,53 @@
+import { typedView } from '@exodus/bytes/array.js'
+import { toBase58, fromBase58 } from '@exodus/bytes/base58.js'
+import { assertUint8 } from '../assert.js'
+
+const E_CHECKSUM = 'Invalid checksum'
+
+// checksum length is 4, i.e. only the first 4 bytes of the hash are used
+
+function encodeWithChecksum(arr, checksum) {
+  // arr type in already validated in input
+  const res = new Uint8Array(arr.length + 4)
+  res.set(arr, 0)
+  res.set(checksum.subarray(0, 4), arr.length)
+  return toBase58(res)
+}
+
+function decodeWithChecksum(str) {
+  const arr = fromBase58(str) // checks input
+  const payloadSize = arr.length - 4
+  if (payloadSize < 0) throw new Error(E_CHECKSUM)
+  return [arr.subarray(0, payloadSize), arr.subarray(payloadSize)]
+}
+
+function assertChecksum(c, r) {
+  if ((c[0] ^ r[0]) | (c[1] ^ r[1]) | (c[2] ^ r[2]) | (c[3] ^ r[3])) throw new Error(E_CHECKSUM)
+}
+
+export const makeBase58check = (hashAlgo, hashAlgoSync) => {
+  const apis = {
+    async encode(arr) {
+      assertUint8(arr)
+      return encodeWithChecksum(arr, await hashAlgo(arr))
+    },
+    async decode(str, format = 'uint8') {
+      const [payload, checksum] = decodeWithChecksum(str)
+      assertChecksum(checksum, await hashAlgo(payload))
+      return typedView(payload, format)
+    },
+  }
+  if (!hashAlgoSync) return apis
+  return {
+    ...apis,
+    encodeSync(arr) {
+      assertUint8(arr)
+      return encodeWithChecksum(arr, hashAlgoSync(arr))
+    },
+    decodeSync(str, format = 'uint8') {
+      const [payload, checksum] = decodeWithChecksum(str)
+      assertChecksum(checksum, hashAlgoSync(payload))
+      return typedView(payload, format)
+    },
+  }
+}

package/fallback/encoding.js

@@ -288,3 +288,16 @@ export function legacyHookDecode(input, fallbackEncoding = 'utf-8') {
 
   return createSinglebyteDecoder(enc, true)(u8)
 }
+
+const uppercasePrefixes = new Set(['utf', 'iso', 'koi', 'euc', 'ibm', 'gbk'])
+
+// Unlike normalizeEncoding, case-sensitive
+// https://encoding.spec.whatwg.org/#names-and-labels
+export function labelToName(label) {
+  const enc = normalizeEncoding(label)
+  if (!enc) return enc
+  if (uppercasePrefixes.has(enc.slice(0, 3))) return enc.toUpperCase()
+  if (enc === 'big5') return 'Big5'
+  if (enc === 'shift_jis') return 'Shift_JIS'
+  return enc
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@exodus/bytes",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "description": "Various operations on Uint8Array data",
   "scripts": {
     "lint": "eslint .",
@@ -45,6 +45,7 @@
   "files": [
     "/fallback/_utils.js",
     "/fallback/base32.js",
+    "/fallback/base58check.js",
     "/fallback/base64.js",
     "/fallback/encoding.js",
     "/fallback/encoding.labels.js",
@@ -65,6 +66,7 @@
     "/base32.js",
     "/base58.js",
     "/base58check.js",
+    "/base58check.node.js",
     "/base64.js",
     "/base64.d.ts",
     "/bech32.js",
@@ -92,7 +94,10 @@
     },
     "./base32.js": "./base32.js",
     "./base58.js": "./base58.js",
-    "./base58check.js": "./base58check.js",
+    "./base58check.js": {
+      "node": "./base58check.node.js",
+      "default": "./base58check.js"
+    },
     "./base64.js": {
       "types": "./base64.d.ts",
       "default": "./base64.js"

package/utf16.js

@@ -9,7 +9,7 @@ const decoderFatalBE = canDecoders ? new TextDecoder('utf-16be', { ignoreBOM, fa
 const decoderLooseBE = canDecoders ? new TextDecoder('utf-16be', { ignoreBOM }) : null
 const decoderFatal16 = isLE ? decoderFatalLE : decoderFatalBE
 const decoderLoose16 = isLE ? decoderLooseLE : decoderFatalBE
-const { isWellFormed } = String.prototype
+const { isWellFormed, toWellFormed } = String.prototype
 
 const { E_STRICT, E_STRICT_UNICODE } = js
 
@@ -61,8 +61,9 @@ function decode(input, loose = false, format = 'uint16') {
       throw new TypeError('Unknown format')
   }
 
-  const str = js.decode(u16, loose, !loose && isWellFormed)
+  const str = js.decode(u16, loose, (!loose && isWellFormed) || (loose && toWellFormed))
   if (!loose && isWellFormed && !isWellFormed.call(str)) throw new TypeError(E_STRICT)
+  if (loose && toWellFormed) return toWellFormed.call(str)
 
   return str
 }

package/utf16.node.js

@@ -1,9 +1,9 @@
-import { nativeDecoder, isDeno, isLE } from './fallback/_utils.js'
+import { isDeno, isLE } from './fallback/_utils.js'
 import { E_STRICT, E_STRICT_UNICODE } from './fallback/utf16.js'
 
 if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
 
-const { isWellFormed } = String.prototype
+const { isWellFormed, toWellFormed } = String.prototype
 const to8 = (a) => new Uint8Array(a.buffer, a.byteOffset, a.byteLength)
 
 // Unlike utf8, operates on Uint16Arrays by default
@@ -14,9 +14,10 @@ function encode(str, loose = false, format = 'uint16') {
     throw new TypeError('Unknown format')
   }
 
-  if (!isWellFormed.call(str)) {
-    if (!loose) throw new TypeError(E_STRICT_UNICODE)
-    str = nativeDecoder.decode(Buffer.from(str)) // well, let's fix up (Buffer doesn't do this with utf16 encoding)
+  if (loose) {
+    str = toWellFormed.call(str) // Buffer doesn't do this with utf16 encoding
+  } else if (!isWellFormed.call(str)) {
+    throw new TypeError(E_STRICT_UNICODE)
   }
 
   const ble = Buffer.from(str, 'utf-16le')
@@ -36,6 +37,7 @@ const swapped = (x, swap) =>
   swap ? Buffer.from(x).swap16() : Buffer.from(x.buffer, x.byteOffset, x.byteLength)
 
 // We skip TextDecoder on Node.js, as it's is somewhy significantly slower than Buffer for utf16
+// Also, it incorrectly misses replacements with Node.js is built without ICU, we fix that
 function decodeNode(input, loose = false, format = 'uint16') {
   let ble
   if (format === 'uint16') {
@@ -50,9 +52,9 @@ function decodeNode(input, loose = false, format = 'uint16') {
   }
 
   const str = ble.ucs2Slice(0, ble.byteLength)
+  if (loose) return toWellFormed.call(str)
   if (isWellFormed.call(str)) return str
-  if (!loose) throw new TypeError(E_STRICT)
-  return nativeDecoder.decode(Buffer.from(str)) // fixup (see above)
+  throw new TypeError(E_STRICT)
 }
 
 function decodeDecoder(input, loose = false, format = 'uint16') {

package/utf8.node.js

@@ -1,15 +1,21 @@
 import { assertUint8 } from './assert.js'
 import { typedView } from './array.js'
-import { E_STRICT_UNICODE } from './fallback/utf8.js'
+import { E_STRICT, E_STRICT_UNICODE } from './fallback/utf8.js'
 import { isAscii } from 'node:buffer'
 
 if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
 
-const decoderFatal = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
+let decoderFatal
 const decoderLoose = new TextDecoder('utf-8', { ignoreBOM: true })
 const { isWellFormed } = String.prototype
 const isDeno = Boolean(globalThis.Deno)
 
+try {
+  decoderFatal = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
+} catch {
+  // Without ICU, Node.js doesn't support fatal option for utf-8
+}
+
 function encode(str, loose = false) {
   if (typeof str !== 'string') throw new TypeError('Input is not a string')
   const strLength = str.length
@@ -45,7 +51,14 @@ function decode(arr, loose = false) {
     return buf.latin1Slice(0, arr.byteLength) // .latin1Slice is faster than .asciiSlice
   }
 
-  return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr)
+  if (loose) return decoderLoose.decode(arr)
+  if (decoderFatal) return decoderFatal.decode(arr)
+
+  // We are in an env without native fatal decoder support (non-fixed Node.js without ICU)
+  // Well, just recheck against encode if it contains replacement then, this is still faster than js impl
+  const str = decoderLoose.decode(arr)
+  if (str.includes('\uFFFD') && !Buffer.from(str).equals(arr)) throw new TypeError(E_STRICT)
+  return str
 }
 
 export const utf8fromString = (str, format = 'uint8') => typedView(encode(str, false), format)