@exodus/bytes 1.8.0 → 1.9.0

package/README.md

@@ -1,7 +1,8 @@
 # `@exodus/bytes`
 
 [![](https://flat.badgen.net/npm/v/@exodus/bytes)](https://npmjs.org/package/@exodus/bytes)
-![](https://flat.badgen.net/npm/dm/@exodus/bytes)
+[![](https://flat.badgen.net/github/release/ExodusOSS/bytes?icon=github)](https://github.com/ExodusOSS/bytes/releases)
+[![](https://flat.badgen.net/npm/dm/@exodus/bytes)](https://www.npmcharts.com/compare/@exodus/bytes?minimal=true)
 [![](https://flat.badgen.net/npm/license/@exodus/bytes)](https://github.com/ExodusOSS/bytes/blob/HEAD/LICENSE)
 
 `Uint8Array` conversion to and from `base64`, `base32`, `base58`, `hex`, `utf8`, `utf16`, `bech32` and `wif`
@@ -129,6 +130,7 @@ import { utf16fromStringLoose, utf16toStringLoose } from '@exodus/bytes/utf16.js
 ```js
 import { createSinglebyteDecoder, createSinglebyteEncoder } from '@exodus/bytes/single-byte.js'
 import { windows1252toString, windows1252fromString } from '@exodus/bytes/single-byte.js'
+import { latin1toString, latin1fromString } from '@exodus/bytes/single-byte.js'
 ```
 
 Decode / encode the legacy single-byte encodings according to the
@@ -167,6 +169,11 @@ Also supports `iso-8859-1`, `iso-8859-9`, `iso-8859-11` as defined at
 '\x80\x81Ğ' // this is iso-8859-9 as defined at https://unicode.org/Public/MAPPINGS/ISO8859/8859-9.txt
 ```
 
+All WHATWG Encoding spec [`windows-*` encodings](https://encoding.spec.whatwg.org/#windows-874) are supersets of
+corresponding [unicode.org encodings](https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/), meaning that
+they encode/decode all the old valid (non-replacement) strings / byte sequences identically, but can also support
+a wider range of inputs.
+
 ##### `createSinglebyteDecoder(encoding, loose = false)`
 
 Create a decoder for a supported one-byte `encoding`, given its lowercased name `encoding`.
@@ -308,8 +315,8 @@ Implements [BIP-0173](https://github.com/bitcoin/bips/blob/master/bip-0173.media
 
 ```js
 import { fromBech32, toBech32 } from '@exodus/bytes/bech32.js'
-import { fromBech32m, toBech32m } from '@exodus/bytes/base32.js'
-import { getPrefix } from '@exodus/bytes/base32.js'
+import { fromBech32m, toBech32m } from '@exodus/bytes/bech32.js'
+import { getPrefix } from '@exodus/bytes/bech32.js'
 ```
 
 ##### `getPrefix(str, limit = 90)`
@@ -341,7 +348,7 @@ import { fromBase58checkSync, toBase58checkSync } from '@exodus/bytes/base58chec
 import { makeBase58check } from '@exodus/bytes/base58check.js'
 ```
 
-On non-Node.js, requires peer dependency [@exodus/crypto](https://www.npmjs.com/package/@exodus/crypto) to be installed.
+On non-Node.js, requires peer dependency [@noble/hashes](https://www.npmjs.com/package/@noble/hashes) to be installed.
 
 ##### `async fromBase58check(str, format = 'uint8')`
 ##### `async toBase58check(arr)`
@@ -356,7 +363,7 @@ import { fromWifString, toWifString } from '@exodus/bytes/wif.js'
 import { fromWifStringSync, toWifStringSync } from '@exodus/bytes/wif.js'
 ```
 
-On non-Node.js, requires peer dependency [@exodus/crypto](https://www.npmjs.com/package/@exodus/crypto) to be installed.
+On non-Node.js, requires peer dependency [@noble/hashes](https://www.npmjs.com/package/@noble/hashes) to be installed.
 
 ##### `async fromWifString(string, version)`
 ##### `fromWifStringSync(string, version)`
@@ -525,6 +532,27 @@ true
 '%'
 ```
 
+### `@exodus/bytes/encoding-browser.js`
+
+```js
+import { TextDecoder, TextEncoder } from '@exodus/bytes/encoding-browser.js'
+import { TextDecoderStream, TextEncoderStream } from '@exodus/bytes/encoding-browser.js' // Requires Streams
+
+// Hooks for standards
+import { getBOMEncoding, legacyHookDecode, labelToName, normalizeEncoding } from '@exodus/bytes/encoding-browser.js'
+```
+
+Same as `@exodus/bytes/encoding.js`, but in browsers instead of polyfilling just uses whatever the
+browser provides, drastically reducing the bundle size (to less than 2 KiB gzipped).
+
+Under non-browser engines (Node.js, React Native, etc.) a full polyfill is used as those platforms
+do not provide sufficiently complete / non-buggy `TextDecoder` APIs.
+
+> [!NOTE]
+> Implementations in browsers [have bugs](https://docs.google.com/spreadsheets/d/1pdEefRG6r9fZy61WHGz0TKSt8cO4ISWqlpBN5KntIvQ/edit),
+> but they are fixing them and the expected update window is short.\
+> If you want to circumvent browser bugs, use full `@exodus/bytes/encoding.js` import.
+
 ## License
 
 [MIT](./LICENSE)

package/array.d.ts

@@ -21,4 +21,3 @@ export type OutputFormat = 'uint8' | 'buffer';
 export function typedView(arr: ArrayBufferView, format: 'uint8'): Uint8Array;
 export function typedView(arr: ArrayBufferView, format: 'buffer'): Buffer;
 export function typedView(arr: ArrayBufferView, format: OutputFormat): Uint8Array | Buffer;
-

package/base58.js

@@ -207,7 +207,7 @@ function fromBase58core(str, alphabet, codes, format = 'uint8') {
       }
 
       at = k + 1
-      if (c !== 0 || at < zeros) throw new Error('Unexpected') // unreachable
+      if (c !== 0 || at < zeros) /* c8 ignore next */ throw new Error('Unexpected') // unreachable
     }
   }
 

package/base58check.js

@@ -1,11 +1,10 @@
-import { hashSync } from '@exodus/crypto/hash' // eslint-disable-line @exodus/import/no-deprecated
+import { sha256 } from '@noble/hashes/sha2.js'
 import { makeBase58check } from './fallback/base58check.js'
 
 // Note: while API is async, we use hashSync for now until we improve webcrypto perf for hash256
 // Inputs to base58 are typically very small, and that makes a difference
 
 // eslint-disable-next-line @exodus/import/no-deprecated
-const sha256 = (x) => hashSync('sha256', x, 'uint8')
 const hash256sync = (x) => sha256(sha256(x))
 const hash256 = hash256sync // See note at the top
 const {

package/base64.d.ts

@@ -73,4 +73,3 @@ export function fromBase64url(str: string, options: FromBase64Options & { format
  */
 export function fromBase64any(str: string, options?: FromBase64Options): Uint8ArrayBuffer;
 export function fromBase64any(str: string, options: FromBase64Options & { format: 'buffer' }): Buffer;
-

package/encoding-browser.browser.js

@@ -0,0 +1,29 @@
+import {
+  fromSource,
+  getBOMEncoding,
+  normalizeEncoding,
+  E_ENCODING,
+} from './fallback/encoding.api.js'
+import labels from './fallback/encoding.labels.js'
+
+// Lite-weight version which re-exports existing implementations on browsers,
+// while still being aliased to the full impl in RN and Node.js
+
+// WARNING: Note that browsers have bugs (which hopefully will get fixed soon)
+
+const { TextDecoder, TextEncoder, TextDecoderStream, TextEncoderStream } = globalThis
+
+export { normalizeEncoding, getBOMEncoding, labelToName } from './fallback/encoding.api.js'
+export { TextDecoder, TextEncoder, TextDecoderStream, TextEncoderStream }
+
+// https://encoding.spec.whatwg.org/#decode
+export function legacyHookDecode(input, fallbackEncoding = 'utf-8') {
+  let u8 = fromSource(input)
+  const bomEncoding = getBOMEncoding(u8)
+  if (bomEncoding) u8 = u8.subarray(bomEncoding === 'utf-8' ? 3 : 2)
+  const enc = bomEncoding ?? normalizeEncoding(fallbackEncoding) // "the byte order mark is more authoritative than anything else"
+  if (enc === 'utf-8') return new TextDecoder('utf-8', { ignoreBOM: true }).decode(u8) // fast path
+  if (enc === 'replacement') return u8.byteLength > 0 ? '\uFFFD' : ''
+  if (!Object.hasOwn(labels, enc)) throw new RangeError(E_ENCODING)
+  return new TextDecoder(enc, { ignoreBOM: true }).decode(u8)
+}

package/encoding-browser.d.ts

@@ -0,0 +1 @@
+export * from './encoding.js'

package/encoding-browser.js

@@ -0,0 +1 @@
+export * from './encoding.js'

package/encoding-browser.native.js

@@ -0,0 +1 @@
+export * from './encoding.js'

package/fallback/_utils.js

@@ -47,6 +47,7 @@ function shouldSkipBuiltins() {
     return /firefox/i.test(g.navigator.userAgent || '') // as simple as we can
   }
 
+  /* c8 ignore next */
   return false // eslint-disable-line no-unreachable
 }
 

package/fallback/encoding.api.js

@@ -0,0 +1,81 @@
+import labels from './encoding.labels.js'
+
+let labelsMap
+
+export const E_ENCODING = 'Unknown encoding'
+
+// Warning: unlike whatwg-encoding, returns lowercased labels
+// Those are case-insensitive and that's how TextDecoder encoding getter normalizes them
+// https://encoding.spec.whatwg.org/#names-and-labels
+export function normalizeEncoding(label) {
+  // fast path
+  if (label === 'utf-8' || label === 'utf8' || label === 'UTF-8' || label === 'UTF8') return 'utf-8'
+  if (label === 'windows-1252' || label === 'ascii' || label === 'latin1') return 'windows-1252'
+  // full map
+  if (/[^\w\t\n\f\r .:-]/i.test(label)) return null // must be ASCII (with ASCII whitespace)
+  const low = `${label}`.trim().toLowerCase()
+  if (Object.hasOwn(labels, low)) return low
+  if (!labelsMap) {
+    labelsMap = new Map()
+    for (const [label, aliases] of Object.entries(labels)) {
+      for (const alias of aliases) labelsMap.set(alias, label)
+    }
+  }
+
+  const mapped = labelsMap.get(low)
+  if (mapped) return mapped
+  return null
+}
+
+// TODO: make this more strict against Symbol.toStringTag
+// Is not very significant though, anything faking Symbol.toStringTag could as well override
+// prototypes, which is not something we protect against
+
+function isAnyArrayBuffer(x) {
+  if (x instanceof ArrayBuffer) return true
+  if (globalThis.SharedArrayBuffer && x instanceof SharedArrayBuffer) return true
+  if (!x || typeof x.byteLength !== 'number') return false
+  const s = Object.prototype.toString.call(x)
+  return s === '[object ArrayBuffer]' || s === '[object SharedArrayBuffer]'
+}
+
+export function fromSource(x) {
+  if (x instanceof Uint8Array) return x
+  if (ArrayBuffer.isView(x)) return new Uint8Array(x.buffer, x.byteOffset, x.byteLength)
+  if (isAnyArrayBuffer(x)) {
+    if ('detached' in x) return x.detached === true ? new Uint8Array() : new Uint8Array(x)
+    // Old engines without .detached, try-catch
+    try {
+      return new Uint8Array(x)
+    } catch {
+      return new Uint8Array()
+    }
+  }
+
+  throw new TypeError('Argument must be a SharedArrayBuffer, ArrayBuffer or ArrayBufferView')
+}
+
+// Warning: unlike whatwg-encoding, returns lowercased labels
+// Those are case-insensitive and that's how TextDecoder encoding getter normalizes them
+export function getBOMEncoding(input) {
+  const u8 = fromSource(input) // asserts
+  if (u8.length >= 3 && u8[0] === 0xef && u8[1] === 0xbb && u8[2] === 0xbf) return 'utf-8'
+  if (u8.length < 2) return null
+  if (u8[0] === 0xff && u8[1] === 0xfe) return 'utf-16le'
+  if (u8[0] === 0xfe && u8[1] === 0xff) return 'utf-16be'
+  return null
+}
+
+const uppercasePrefixes = new Set(['utf', 'iso', 'koi', 'euc', 'ibm', 'gbk'])
+
+// Unlike normalizeEncoding, case-sensitive
+// https://encoding.spec.whatwg.org/#names-and-labels
+export function labelToName(label) {
+  const enc = normalizeEncoding(label)
+  if (enc === 'utf-8') return 'UTF-8' // fast path
+  if (!enc) return enc
+  if (uppercasePrefixes.has(enc.slice(0, 3))) return enc.toUpperCase()
+  if (enc === 'big5') return 'Big5'
+  if (enc === 'shift_jis') return 'Shift_JIS'
+  return enc
+}

package/fallback/encoding.js

@@ -5,14 +5,15 @@ import { utf16toString, utf16toStringLoose } from '@exodus/bytes/utf16.js'
 import { utf8fromStringLoose, utf8toString, utf8toStringLoose } from '@exodus/bytes/utf8.js'
 import { createSinglebyteDecoder } from '@exodus/bytes/single-byte.js'
 import labels from './encoding.labels.js'
+import { fromSource, getBOMEncoding, normalizeEncoding, E_ENCODING } from './encoding.api.js'
 import { unfinishedBytes } from './encoding.util.js'
 
-const E_OPTIONS = 'The "options" argument must be of type object'
-const E_ENCODING = 'Unknown encoding'
-const replacementChar = '\uFFFD'
+export { labelToName, getBOMEncoding, normalizeEncoding } from './encoding.api.js'
 
+const E_OPTIONS = 'The "options" argument must be of type object'
 const E_MULTI =
   'Legacy multi-byte encodings are disabled in /encoding-lite.js, use /encoding.js for full encodings range support'
+const replacementChar = '\uFFFD'
 const multibyteSet = new Set(['big5', 'euc-kr', 'euc-jp', 'iso-2022-jp', 'shift_jis', 'gbk', 'gb18030']) // prettier-ignore
 let createMultibyteDecoder
 
@@ -20,67 +21,14 @@ export function setMultibyteDecoder(createDecoder) {
   createMultibyteDecoder = createDecoder
 }
 
-let labelsMap
-
-// Warning: unlike whatwg-encoding, returns lowercased labels
-// Those are case-insensitive and that's how TextDecoder encoding getter normalizes them
-// https://encoding.spec.whatwg.org/#names-and-labels
-export function normalizeEncoding(label) {
-  // fast path
-  if (label === 'utf-8' || label === 'utf8' || label === 'UTF-8' || label === 'UTF8') return 'utf-8'
-  if (label === 'windows-1252' || label === 'ascii' || label === 'latin1') return 'windows-1252'
-  // full map
-  if (/[^\w\t\n\f\r .:-]/i.test(label)) return null // must be ASCII (with ASCII whitespace)
-  const low = `${label}`.trim().toLowerCase()
-  if (Object.hasOwn(labels, low)) return low
-  if (!labelsMap) {
-    labelsMap = new Map()
-    for (const [label, aliases] of Object.entries(labels)) {
-      for (const alias of aliases) labelsMap.set(alias, label)
-    }
-  }
-
-  const mapped = labelsMap.get(low)
-  if (mapped) return mapped
-  return null
-}
-
 const define = (obj, key, value) => Object.defineProperty(obj, key, { value, writable: false })
 
-// TODO: make this more strict against Symbol.toStringTag
-// Is not very significant though, anything faking Symbol.toStringTag could as well override
-// prototypes, which is not something we protect against
-
-function isAnyArrayBuffer(x) {
-  if (x instanceof ArrayBuffer) return true
-  if (globalThis.SharedArrayBuffer && x instanceof SharedArrayBuffer) return true
-  if (!x || typeof x.byteLength !== 'number') return false
-  const s = Object.prototype.toString.call(x)
-  return s === '[object ArrayBuffer]' || s === '[object SharedArrayBuffer]'
-}
-
 function isAnyUint8Array(x) {
   if (x instanceof Uint8Array) return true
   if (!x || !ArrayBuffer.isView(x) || x.BYTES_PER_ELEMENT !== 1) return false
   return Object.prototype.toString.call(x) === '[object Uint8Array]'
 }
 
-const fromSource = (x) => {
-  if (x instanceof Uint8Array) return x
-  if (ArrayBuffer.isView(x)) return new Uint8Array(x.buffer, x.byteOffset, x.byteLength)
-  if (isAnyArrayBuffer(x)) {
-    if ('detached' in x) return x.detached === true ? new Uint8Array() : new Uint8Array(x)
-    // Old engines without .detached, try-catch
-    try {
-      return new Uint8Array(x)
-    } catch {
-      return new Uint8Array()
-    }
-  }
-
-  throw new TypeError('Argument must be a SharedArrayBuffer, ArrayBuffer or ArrayBufferView')
-}
-
 function unicodeDecoder(encoding, loose) {
   if (encoding === 'utf-8') return loose ? utf8toStringLoose : utf8toString // likely
   const form = encoding === 'utf-16le' ? 'uint8-le' : 'uint8-be'
@@ -215,6 +163,7 @@ export class TextDecoder {
         return u.byteLength >= 2 && u[0] === 0xfe && u[1] === 0xff ? 2 : 0
     }
 
+    /* c8 ignore next */
     throw new Error('Unreachable')
   }
 }
@@ -341,17 +290,6 @@ export class TextEncoderStream {
   }
 }
 
-// Warning: unlike whatwg-encoding, returns lowercased labels
-// Those are case-insensitive and that's how TextDecoder encoding getter normalizes them
-export function getBOMEncoding(input) {
-  const u8 = fromSource(input) // asserts
-  if (u8.length >= 3 && u8[0] === 0xef && u8[1] === 0xbb && u8[2] === 0xbf) return 'utf-8'
-  if (u8.length < 2) return null
-  if (u8[0] === 0xff && u8[1] === 0xfe) return 'utf-16le'
-  if (u8[0] === 0xfe && u8[1] === 0xff) return 'utf-16be'
-  return null
-}
-
 // https://encoding.spec.whatwg.org/#decode
 // Warning: encoding sniffed from BOM takes preference over the supplied one
 // Warning: lossy, performs replacement, no option of throwing
@@ -368,7 +306,7 @@ export function legacyHookDecode(input, fallbackEncoding = 'utf-8') {
     let suffix = ''
     if (u8.byteLength % 2 !== 0) {
       suffix = replacementChar
-      u8 = u8.subarray(0, -1)
+      u8 = u8.subarray(0, -unfinishedBytes(u8, u8.byteLength, enc))
     }
 
     return utf16toStringLoose(u8, enc === 'utf-16le' ? 'uint8-le' : 'uint8-be') + suffix
@@ -387,17 +325,3 @@ export function legacyHookDecode(input, fallbackEncoding = 'utf-8') {
 
   return createSinglebyteDecoder(enc, true)(u8)
 }
-
-const uppercasePrefixes = new Set(['utf', 'iso', 'koi', 'euc', 'ibm', 'gbk'])
-
-// Unlike normalizeEncoding, case-sensitive
-// https://encoding.spec.whatwg.org/#names-and-labels
-export function labelToName(label) {
-  const enc = normalizeEncoding(label)
-  if (enc === 'utf-8') return 'UTF-8' // fast path
-  if (!enc) return enc
-  if (uppercasePrefixes.has(enc.slice(0, 3))) return enc.toUpperCase()
-  if (enc === 'big5') return 'Big5'
-  if (enc === 'shift_jis') return 'Shift_JIS'
-  return enc
-}

package/fallback/latin1.js

@@ -37,6 +37,7 @@ export function asciiPrefix(arr) {
       const b = u32[i + 1]
       const c = u32[i + 2]
       const d = u32[i + 3]
+      // "(a | b | c | d) & mask" is slower on Hermes though faster on v8
       if (a & 0x80_80_80_80 || b & 0x80_80_80_80 || c & 0x80_80_80_80 || d & 0x80_80_80_80) break
     }