@exodus/bytes 1.0.0-rc.8 → 1.0.0-rc.9

package/fallback/utf16.js

@@ -0,0 +1,180 @@
+import { decodeLatin1, encodeCharcodes } from './latin1.js'
+import { isLE } from './_utils.js'
+
+export const E_STRICT = 'Input is not well-formed utf16'
+export const E_STRICT_UNICODE = 'Input is not well-formed Unicode'
+
+const replacementCodepoint = 0xff_fd
+const replacementCodepointSwapped = 0xfd_ff
+
+const to16 = (a) => new Uint16Array(a.buffer, a.byteOffset, a.byteLength / 2) // Requires checked length and alignment!
+
+export function to16input(u8, le) {
+  // Assume even number of bytes
+  if (le === isLE) return to16(u8.byteOffset % 2 === 0 ? u8 : Uint8Array.from(u8))
+
+  const res = new Uint8Array(u8.length)
+
+  let i = 0
+  for (const last3 = u8.length - 3; i < last3; i += 4) {
+    const x0 = u8[i]
+    const x1 = u8[i + 1]
+    const x2 = u8[i + 2]
+    const x3 = u8[i + 3]
+    res[i] = x1
+    res[i + 1] = x0
+    res[i + 2] = x3
+    res[i + 3] = x2
+  }
+
+  for (const last = u8.length - 1; i < last; i += 2) {
+    const x0 = u8[i]
+    const x1 = u8[i + 1]
+    res[i] = x1
+    res[i + 1] = x0
+  }
+
+  return to16(res)
+}
+
+export const decode = (u16, loose = false, checked = false) => {
+  if (checked || isWellFormed(u16)) return decodeLatin1(u16, 0, u16.length) // it's capable of decoding Uint16Array to UTF-16 as well
+  if (!loose) throw new TypeError(E_STRICT)
+  return decodeLatin1(toWellFormed(Uint16Array.from(u16)), 0, u16.length) // cloned for replacement
+}
+
+export function encode(str, loose = false, checked = false, swapped = false) {
+  const arr = new Uint16Array(str.length)
+  if (checked) return swapped ? encodeCheckedSwapped(str, arr) : encodeChecked(str, arr)
+  return swapped ? encodeUncheckedSwapped(str, arr, loose) : encodeUnchecked(str, arr, loose)
+}
+
+// Splitting paths into small functions helps (at least on SpiderMonkey)
+/* eslint-disable @exodus/mutable/no-param-reassign-prop-only */
+
+const encodeChecked = (str, arr) => encodeCharcodes(str, arr) // Same as encodeLatin1, but with Uint16Array
+
+function encodeCheckedSwapped(str, arr) {
+  // TODO: faster path for Hermes? See encodeCharcodes
+  const length = str.length
+  for (let i = 0; i < length; i++) {
+    const x = str.charCodeAt(i)
+    arr[i] = ((x & 0xff) << 8) | (x >> 8)
+  }
+
+  return arr
+}
+
+// lead: d800 - dbff, trail: dc00 - dfff
+
+function encodeUnchecked(str, arr, loose = false) {
+  // TODO: faster path for Hermes? See encodeCharcodes
+  const length = str.length
+  for (let i = 0; i < length; i++) {
+    const code = str.charCodeAt(i)
+    arr[i] = code
+    if (code >= 0xd8_00 && code < 0xe0_00) {
+      // An unexpected trail or a lead at the very end of input
+      if (code > 0xdb_ff || i + 1 >= length) {
+        if (!loose) throw new TypeError(E_STRICT_UNICODE)
+        arr[i] = replacementCodepoint
+      } else {
+        const next = str.charCodeAt(i + 1) // Process valid pairs immediately
+        if (next < 0xdc_00 || next >= 0xe0_00) {
+          if (!loose) throw new TypeError(E_STRICT_UNICODE)
+          arr[i] = replacementCodepoint
+        } else {
+          i++ // consume next
+          arr[i] = next
+        }
+      }
+    }
+  }
+
+  return arr
+}
+
+function encodeUncheckedSwapped(str, arr, loose = false) {
+  // TODO: faster path for Hermes? See encodeCharcodes
+  const length = str.length
+  for (let i = 0; i < length; i++) {
+    const code = str.charCodeAt(i)
+    arr[i] = ((code & 0xff) << 8) | (code >> 8)
+    if (code >= 0xd8_00 && code < 0xe0_00) {
+      // An unexpected trail or a lead at the very end of input
+      if (code > 0xdb_ff || i + 1 >= length) {
+        if (!loose) throw new TypeError(E_STRICT_UNICODE)
+        arr[i] = replacementCodepointSwapped
+      } else {
+        const next = str.charCodeAt(i + 1) // Process valid pairs immediately
+        if (next < 0xdc_00 || next >= 0xe0_00) {
+          if (!loose) throw new TypeError(E_STRICT_UNICODE)
+          arr[i] = replacementCodepointSwapped
+        } else {
+          i++ // consume next
+          arr[i] = ((next & 0xff) << 8) | (next >> 8)
+        }
+      }
+    }
+  }
+
+  return arr
+}
+
+export function toWellFormed(u16) {
+  const length = u16.length
+  for (let i = 0; i < length; i++) {
+    const code = u16[i]
+    if (code >= 0xd8_00 && code < 0xe0_00) {
+      // An unexpected trail or a lead at the very end of input
+      if (code > 0xdb_ff || i + 1 >= length) {
+        u16[i] = replacementCodepoint
+      } else {
+        const next = u16[i + 1] // Process valid pairs immediately
+        if (next < 0xdc_00 || next >= 0xe0_00) {
+          u16[i] = replacementCodepoint
+        } else {
+          i++ // consume next
+        }
+      }
+    }
+  }
+
+  return u16
+}
+
+export function isWellFormed(u16) {
+  const length = u16.length
+  let i = 0
+
+  // Speedup with u32, by skipping to the first surrogate
+  // Only implemented for aligned input for now, but almost all input is aligned (pooled Buffer or 0 offset)
+  if (length > 32 && u16.byteOffset % 4 === 0) {
+    const u32length = (u16.byteLength / 4) | 0
+    const u32 = new Uint32Array(u16.buffer, u16.byteOffset, u32length)
+    for (const last3 = u32length - 3; ; i += 4) {
+      if (i >= last3) break // loop is fast enough for moving this here to be _very_ useful, likely due to array access checks
+      const a = u32[i]
+      const b = u32[i + 1]
+      const c = u32[i + 2]
+      const d = u32[i + 3]
+      if (a & 0x80_00_80_00 || b & 0x80_00_80_00 || c & 0x80_00_80_00 || d & 0x80_00_80_00) break
+    }
+
+    for (; i < u32length; i++) if (u32[i] & 0x80_00_80_00) break
+    i *= 2
+  }
+
+  for (; i < length; i++) {
+    const code = u16[i]
+    if (code >= 0xd8_00 && code < 0xe0_00) {
+      // An unexpected trail or a lead at the very end of input
+      if (code > 0xdb_ff || i + 1 >= length) return false
+      i++ // consume next
+      const next = u16[i] // Process valid pairs immediately
+      if (next < 0xdc_00 || next >= 0xe0_00) return false
+    }
+  }
+
+  return true
+}

package/hex.node.js

@@ -5,6 +5,7 @@ import { E_HEX } from './fallback/hex.js'
 if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
 
 const { toHex: webHex } = Uint8Array.prototype // Modern engines have this
+const denoBug = Buffer.from('ag', 'hex').length > 0
 
 export function toHex(arr) {
   assertUint8(arr)
@@ -20,6 +21,7 @@ export const fromHex = Uint8Array.fromHex
   : (str, format = 'uint8') => {
       if (typeof str !== 'string') throw new TypeError('Input is not a string')
       if (str.length % 2 !== 0) throw new SyntaxError(E_HEX)
+      if (denoBug && /[^\dA-Fa-f]/.test(str)) throw new SyntaxError(E_HEX)
       const buf = Buffer.from(str, 'hex') // will stop on first non-hex character, so we can just validate length
       if (buf.length * 2 !== str.length) throw new SyntaxError(E_HEX)
       return typedView(buf, format)

package/multi-byte.js

@@ -0,0 +1,13 @@
+import { assertUint8 } from './assert.js'
+import { multibyteDecoder } from './fallback/multi-byte.js'
+
+export function createMultibyteDecoder(encoding, loose = false) {
+  const jsDecoder = multibyteDecoder(encoding, loose) // asserts
+  let streaming = false
+  return (arr, stream = false) => {
+    assertUint8(arr)
+    if (!streaming && arr.byteLength === 0) return ''
+    streaming = stream
+    return jsDecoder(arr, stream)
+  }
+}

package/multi-byte.node.js

@@ -0,0 +1,25 @@
+import { assertUint8 } from './assert.js'
+import { isDeno } from './fallback/_utils.js'
+import { isAsciiSuperset, multibyteDecoder } from './fallback/multi-byte.js'
+import { isAscii } from 'node:buffer'
+
+const toBuf = (x) => Buffer.from(x.buffer, x.byteOffset, x.byteLength)
+
+export function createMultibyteDecoder(encoding, loose = false) {
+  const jsDecoder = multibyteDecoder(encoding, loose) // asserts
+  let streaming = false
+  const asciiSuperset = isAsciiSuperset(encoding)
+  return (arr, stream = false) => {
+    assertUint8(arr)
+    if (!streaming) {
+      if (arr.byteLength === 0) return ''
+      if (asciiSuperset && isAscii(arr)) {
+        if (isDeno) return toBuf(arr).toString()
+        return toBuf(arr).latin1Slice(0, arr.byteLength) // .latin1Slice is faster than .asciiSlice
+      }
+    }
+
+    streaming = stream
+    return jsDecoder(arr, stream)
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@exodus/bytes",
-  "version": "1.0.0-rc.8",
+  "version": "1.0.0-rc.9",
   "description": "Various operations on Uint8Array data",
   "scripts": {
     "lint": "eslint .",
@@ -10,6 +10,10 @@
     "test:spidermonkey": "exodus-test --engine=spidermonkey:bundle",
     "test:hermes": "exodus-test --engine=hermes:bundle",
     "test:quickjs": "exodus-test --engine=quickjs:bundle",
+    "test:xs": "exodus-test --engine=xs:bundle",
+    "test:engine262": "exodus-test --engine=engine262:bundle",
+    "test:deno": "exodus-test --engine=deno:pure",
+    "test:bun": "exodus-test --engine=bun:pure",
     "test:electron:bundle": "exodus-test --engine=electron:bundle",
     "test:electron:as-node": "exodus-test --engine=electron-as-node:test",
     "test:chrome:puppeteer": "exodus-test --engine=chrome:puppeteer",
@@ -17,6 +21,7 @@
     "test:webkit:playwright": "exodus-test --engine=webkit:playwright",
     "test:firefox:puppeteer": "exodus-test --engine=firefox:puppeteer",
     "test:firefox:playwright": "exodus-test --engine=firefox:playwright",
+    "test:servo:bundle": "exodus-test --engine=servo:bundle",
     "test": "exodus-test",
     "jsvu": "jsvu",
     "playwright": "exodus-test --playwright",
@@ -25,24 +30,33 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/ExodusMovement/bytes.git"
+    "url": "git+https://github.com/ExodusOSS/bytes.git"
   },
   "author": "Exodus Movement, Inc.",
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/ExodusMovement/bytes/issues"
+    "url": "https://github.com/ExodusOSS/bytes/issues"
   },
-  "homepage": "https://github.com/ExodusMovement/bytes#readme",
+  "homepage": "https://github.com/ExodusOSS/bytes#readme",
   "engines": {
     "node": "^20.19.0 || >=22.13.0"
   },
   "type": "module",
   "files": [
     "/fallback/_utils.js",
-    "/fallback/latin1.js",
     "/fallback/base32.js",
     "/fallback/base64.js",
+    "/fallback/encoding.labels.js",
+    "/fallback/encoding.util.js",
     "/fallback/hex.js",
+    "/fallback/latin1.js",
+    "/fallback/multi-byte.encodings.cjs",
+    "/fallback/multi-byte.encodings.json",
+    "/fallback/multi-byte.js",
+    "/fallback/multi-byte.table.js",
+    "/fallback/single-byte.encodings.js",
+    "/fallback/single-byte.js",
+    "/fallback/utf16.js",
     "/fallback/utf8.js",
     "/array.js",
     "/assert.js",
@@ -51,10 +65,17 @@
     "/base58check.js",
     "/base64.js",
     "/bech32.js",
-    "/hex.node.js",
+    "/encoding.js",
     "/hex.js",
-    "/utf8.node.js",
+    "/hex.node.js",
+    "/multi-byte.js",
+    "/multi-byte.node.js",
+    "/single-byte.js",
+    "/single-byte.node.js",
+    "/utf16.js",
+    "/utf16.node.js",
     "/utf8.js",
+    "/utf8.node.js",
     "/wif.js"
   ],
   "exports": {
@@ -68,6 +89,15 @@
       "node": "./hex.node.js",
       "default": "./hex.js"
     },
+    "./multi-byte.js": {
+      "node": "./multi-byte.node.js",
+      "default": "./multi-byte.js"
+    },
+    "./single-byte.js": {
+      "node": "./single-byte.node.js",
+      "default": "./single-byte.js"
+    },
+    "./encoding.js": "./encoding.js",
     "./utf16.js": {
       "node": "./utf16.node.js",
       "default": "./utf16.js"
@@ -91,7 +121,7 @@
     "@exodus/crypto": "^1.0.0-rc.30",
     "@exodus/eslint-config": "^5.24.0",
     "@exodus/prettier": "^1.0.0",
-    "@exodus/test": "^1.0.0-rc.108",
+    "@exodus/test": "^1.0.0-rc.109",
     "@noble/hashes": "^2.0.1",
     "@scure/base": "^1.2.6",
     "@stablelib/base64": "^2.0.1",
@@ -119,6 +149,7 @@
     "typescript": "^5.9.3",
     "uint8array-tools": "^0.0.9",
     "utf8": "^3.0.0",
+    "whatwg-encoding": "^3.1.1",
     "wif": "^5.0.0"
   },
   "prettier": "@exodus/prettier",

package/single-byte.js

@@ -0,0 +1,55 @@
+import { assertUint8 } from './assert.js'
+import { canDecoders } from './fallback/_utils.js'
+import { assertEncoding, encodingDecoder } from './fallback/single-byte.js'
+
+const { TextDecoder } = globalThis
+
+let windows1252works
+
+function shouldUseNative(enc) {
+  // https://issues.chromium.org/issues/468458388
+  // Also might be incorrectly imlemented on platforms as Latin1 (e.g. in Node.js) or regress
+  // This is the most significant single-byte encoding, 'ascii' and 'latin1' alias to this
+  // Even after Chrome bug is fixed, this should serve as a quick correctness check that it's actually windows-1252
+  if (enc === 'windows-1252') {
+    if (windows1252works === undefined) {
+      windows1252works = false
+      try {
+        const u = new Uint8Array(9) // using 9 bytes is significant to catch the bug
+        u[8] = 128
+        windows1252works = new TextDecoder(enc).decode(u).codePointAt(8) === 0x20_ac
+      } catch {}
+    }
+
+    return windows1252works
+  }
+
+  // iso-8859-16 is somehow broken in WebKit, at least on CI
+  return enc !== 'iso-8859-16'
+}
+
+export function createSinglebyteDecoder(encoding, loose = false) {
+  if (encoding === 'iso-8859-8-i') encoding = 'iso-8859-8'
+  assertEncoding(encoding)
+
+  if (canDecoders && shouldUseNative(encoding)) {
+    // In try, as not all encodings might be implemented in all engines which have native TextDecoder
+    try {
+      const decoder = new TextDecoder(encoding, { fatal: !loose })
+      return (arr) => {
+        assertUint8(arr)
+        if (arr.byteLength === 0) return ''
+        return decoder.decode(arr)
+      }
+    } catch {}
+  }
+
+  const jsDecoder = encodingDecoder(encoding)
+  return (arr) => {
+    assertUint8(arr)
+    if (arr.byteLength === 0) return ''
+    return jsDecoder(arr, loose)
+  }
+}
+
+export const windows1252toString = createSinglebyteDecoder('windows-1252')

package/single-byte.node.js

@@ -0,0 +1,62 @@
+import { assertUint8 } from './assert.js'
+import { isAscii } from 'node:buffer'
+import { isDeno, isLE } from './fallback/_utils.js'
+import { asciiPrefix, decodeLatin1 } from './fallback/latin1.js'
+import { encodingMapper, encodingDecoder, E_STRICT } from './fallback/single-byte.js'
+
+const toBuf = (x) => Buffer.from(x.buffer, x.byteOffset, x.byteLength)
+
+function latin1Prefix(arr, start) {
+  let p = start | 0
+  const length = arr.length
+  for (const len3 = length - 3; p < len3; p += 4) {
+    if ((arr[p] & 0xe0) === 0x80) return p
+    if ((arr[p + 1] & 0xe0) === 0x80) return p + 1
+    if ((arr[p + 2] & 0xe0) === 0x80) return p + 2
+    if ((arr[p + 3] & 0xe0) === 0x80) return p + 3
+  }
+
+  for (; p < length; p++) {
+    if ((arr[p] & 0xe0) === 0x80) return p
+  }
+
+  return length
+}
+
+export function createSinglebyteDecoder(encoding, loose = false) {
+  if (encoding === 'iso-8859-8-i') encoding = 'iso-8859-8'
+  const latin1path = encoding === 'windows-1252' // TODO: are there more?
+  if (isDeno) {
+    const jsDecoder = encodingDecoder(encoding) // asserts
+    return (arr) => {
+      assertUint8(arr)
+      if (arr.byteLength === 0) return ''
+      if (isAscii(arr)) return toBuf(arr).toString()
+      return jsDecoder(arr, loose) // somewhy faster on Deno anyway, TODO: optimize?
+    }
+  }
+
+  const { incomplete, mapper } = encodingMapper(encoding) // asserts
+  return (arr) => {
+    assertUint8(arr)
+    if (arr.byteLength === 0) return ''
+    if (isAscii(arr)) return toBuf(arr).latin1Slice(0, arr.byteLength) // .latin1Slice is faster than .asciiSlice
+
+    // Node.js TextDecoder is broken, so we can't use it. It's also slow anyway
+
+    let prefixBytes = asciiPrefix(arr)
+    let prefix = ''
+    if (latin1path) prefixBytes = latin1Prefix(arr, prefixBytes)
+    if (prefixBytes > 64 || prefixBytes === arr.length) {
+      prefix = toBuf(arr).latin1Slice(0, prefixBytes) // .latin1Slice is faster than .asciiSlice
+      if (prefixBytes === arr.length) return prefix
+    }
+
+    const b = mapper(arr, prefix.length) // prefix.length can mismatch prefixBytes
+    const suffix = isLE ? toBuf(b).ucs2Slice(0, b.byteLength) : decodeLatin1(b, 0, b.length) // decodeLatin1 is actually capable of decoding 16-bit codepoints
+    if (!loose && incomplete && suffix.includes('\uFFFD')) throw new TypeError(E_STRICT)
+    return prefix + suffix
+  }
+}
+
+export const windows1252toString = createSinglebyteDecoder('windows-1252')

package/utf16.js

@@ -0,0 +1,73 @@
+import * as js from './fallback/utf16.js'
+import { canDecoders, isLE } from './fallback/_utils.js'
+
+const { TextDecoder } = globalThis // Buffer is optional
+const ignoreBOM = true
+const decoderFatalLE = canDecoders ? new TextDecoder('utf-16le', { ignoreBOM, fatal: true }) : null
+const decoderLooseLE = canDecoders ? new TextDecoder('utf-16le', { ignoreBOM }) : null
+const decoderFatalBE = canDecoders ? new TextDecoder('utf-16be', { ignoreBOM, fatal: true }) : null
+const decoderLooseBE = canDecoders ? new TextDecoder('utf-16be', { ignoreBOM }) : null
+const decoderFatal16 = isLE ? decoderFatalLE : decoderFatalBE
+const decoderLoose16 = isLE ? decoderLooseLE : decoderFatalBE
+const { isWellFormed } = String.prototype
+
+const { E_STRICT, E_STRICT_UNICODE } = js
+
+// Unlike utf8, operates on Uint16Arrays by default
+
+const to8 = (a) => new Uint8Array(a.buffer, a.byteOffset, a.byteLength)
+
+function encode(str, loose = false, format = 'uint16') {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  if (format !== 'uint16' && format !== 'uint8-le' && format !== 'uint8-be') {
+    throw new TypeError('Unknown format')
+  }
+
+  const shouldSwap = (isLE && format === 'uint8-be') || (!isLE && format === 'uint8-le')
+
+  // On v8 and SpiderMonkey, check via isWellFormed is faster than js
+  // On JSC, check during loop is faster than isWellFormed
+  // If isWellFormed is available, we skip check during decoding and recheck after
+  // If isWellFormed is unavailable, we check in js during decoding
+  if (!loose && isWellFormed && !isWellFormed.call(str)) throw new TypeError(E_STRICT_UNICODE)
+  const u16 = js.encode(str, loose, !loose && isWellFormed, shouldSwap)
+
+  if (format === 'uint8-le' || format === 'uint8-be') return to8(u16) // Already swapped
+  if (format === 'uint16') return u16
+  throw new Error('Unreachable')
+}
+
+function decode(input, loose = false, format = 'uint16') {
+  let u16
+  switch (format) {
+    case 'uint16':
+      if (!(input instanceof Uint16Array)) throw new TypeError('Expected an Uint16Array')
+      if (canDecoders) return loose ? decoderLoose16.decode(input) : decoderFatal16.decode(input)
+      u16 = input
+      break
+    case 'uint8-le':
+      if (!(input instanceof Uint8Array)) throw new TypeError('Expected an Uint8Array')
+      if (input.byteLength % 2 !== 0) throw new TypeError('Expected even number of bytes')
+      if (canDecoders) return loose ? decoderLooseLE.decode(input) : decoderFatalLE.decode(input)
+      u16 = js.to16input(input, true)
+      break
+    case 'uint8-be':
+      if (!(input instanceof Uint8Array)) throw new TypeError('Expected an Uint8Array')
+      if (input.byteLength % 2 !== 0) throw new TypeError('Expected even number of bytes')
+      if (canDecoders) return loose ? decoderLooseBE.decode(input) : decoderFatalBE.decode(input)
+      u16 = js.to16input(input, false)
+      break
+    default:
+      throw new TypeError('Unknown format')
+  }
+
+  const str = js.decode(u16, loose, !loose && isWellFormed)
+  if (!loose && isWellFormed && !isWellFormed.call(str)) throw new TypeError(E_STRICT)
+
+  return str
+}
+
+export const utf16fromString = (str, format = 'uint16') => encode(str, false, format)
+export const utf16fromStringLoose = (str, format = 'uint16') => encode(str, true, format)
+export const utf16toString = (arr, format = 'uint16') => decode(arr, false, format)
+export const utf16toStringLoose = (arr, format = 'uint16') => decode(arr, true, format)

package/utf16.node.js

@@ -0,0 +1,79 @@
+import { nativeDecoder, isDeno, isLE } from './fallback/_utils.js'
+import { E_STRICT, E_STRICT_UNICODE } from './fallback/utf16.js'
+
+if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
+
+const { isWellFormed } = String.prototype
+const to8 = (a) => new Uint8Array(a.buffer, a.byteOffset, a.byteLength)
+
+// Unlike utf8, operates on Uint16Arrays by default
+
+function encode(str, loose = false, format = 'uint16') {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  if (format !== 'uint16' && format !== 'uint8-le' && format !== 'uint8-be') {
+    throw new TypeError('Unknown format')
+  }
+
+  if (!isWellFormed.call(str)) {
+    if (!loose) throw new TypeError(E_STRICT_UNICODE)
+    str = nativeDecoder.decode(Buffer.from(str)) // well, let's fix up (Buffer doesn't do this with utf16 encoding)
+  }
+
+  const ble = Buffer.from(str, 'utf-16le')
+
+  if (format === 'uint8-le') return to8(ble)
+  if (format === 'uint8-be') return to8(ble.swap16())
+  if (format === 'uint16') {
+    const b = ble.byteOffset % 2 === 0 ? ble : Buffer.from(ble) // it should be already aligned, but just in case
+    if (!isLE) b.swap16()
+    return new Uint16Array(b.buffer, b.byteOffset, b.byteLength / 2)
+  }
+
+  throw new Error('Unreachable')
+}
+
+const swapped = (x, swap) =>
+  swap ? Buffer.from(x).swap16() : Buffer.from(x.buffer, x.byteOffset, x.byteLength)
+
+// We skip TextDecoder on Node.js, as it's is somewhy significantly slower than Buffer for utf16
+function decodeNode(input, loose = false, format = 'uint16') {
+  let ble
+  if (format === 'uint16') {
+    if (!(input instanceof Uint16Array)) throw new TypeError('Expected an Uint16Array')
+    ble = swapped(input, !isLE)
+  } else if (format === 'uint8-le' || format === 'uint8-be') {
+    if (!(input instanceof Uint8Array)) throw new TypeError('Expected an Uint8Array')
+    if (input.byteLength % 2 !== 0) throw new TypeError('Expected even number of bytes')
+    ble = swapped(input, format === 'uint8-be')
+  } else {
+    throw new TypeError('Unknown format')
+  }
+
+  const str = ble.ucs2Slice(0, ble.byteLength)
+  if (isWellFormed.call(str)) return str
+  if (!loose) throw new TypeError(E_STRICT)
+  return nativeDecoder.decode(Buffer.from(str)) // fixup (see above)
+}
+
+function decodeDecoder(input, loose = false, format = 'uint16') {
+  let encoding
+  if (format === 'uint16') {
+    if (!(input instanceof Uint16Array)) throw new TypeError('Expected an Uint16Array')
+    encoding = isLE ? 'utf-16le' : 'utf-16be'
+  } else if (format === 'uint8-le' || format === 'uint8-be') {
+    if (!(input instanceof Uint8Array)) throw new TypeError('Expected an Uint8Array')
+    if (input.byteLength % 2 !== 0) throw new TypeError('Expected even number of bytes')
+    encoding = format === 'uint8-le' ? 'utf-16le' : 'utf-16be'
+  } else {
+    throw new TypeError('Unknown format')
+  }
+
+  return new TextDecoder(encoding, { ignoreBOM: true, fatal: !loose }).decode(input) // TODO: cache decoder?
+}
+
+const decode = isDeno ? decodeDecoder : decodeNode
+
+export const utf16fromString = (str, format = 'uint16') => encode(str, false, format)
+export const utf16fromStringLoose = (str, format = 'uint16') => encode(str, true, format)
+export const utf16toString = (arr, format = 'uint16') => decode(arr, false, format)
+export const utf16toStringLoose = (arr, format = 'uint16') => decode(arr, true, format)

package/utf8.js

@@ -1,18 +1,16 @@
 import { assertUint8 } from './assert.js'
 import { typedView } from './array.js'
-import { isHermes } from './fallback/_utils.js'
+import { isHermes, nativeDecoder, nativeEncoder } from './fallback/_utils.js'
 import { asciiPrefix, decodeLatin1 } from './fallback/latin1.js'
 import * as js from './fallback/utf8.js'
 
-const { Buffer, TextEncoder, TextDecoder, decodeURIComponent, escape } = globalThis // Buffer is optional
-const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
-const isNative = (x) => x && (haveNativeBuffer || `${x}`.includes('[native code]')) // we consider Node.js TextDecoder/TextEncoder native
-const haveDecoder = isNative(TextDecoder)
-const nativeEncoder = isNative(TextEncoder) ? new TextEncoder() : null
+const { TextDecoder, decodeURIComponent, escape } = globalThis // Buffer is optional
 // ignoreBOM: true means that BOM will be left as-is, i.e. will be present in the output
 // We don't want to strip anything unexpectedly
-const decoderFatal = haveDecoder ? new TextDecoder('utf8', { ignoreBOM: true, fatal: true }) : null
-const decoderLoose = haveDecoder ? new TextDecoder('utf8', { ignoreBOM: true }) : null
+const decoderLoose = nativeDecoder
+const decoderFatal = nativeDecoder
+  ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
+  : null
 const { isWellFormed } = String.prototype
 
 const { E_STRICT, E_STRICT_UNICODE } = js
@@ -56,7 +54,7 @@ function encode(str, loose = false) {
 function decode(arr, loose = false) {
   assertUint8(arr)
   if (arr.byteLength === 0) return ''
-  if (haveDecoder) return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr) // Node.js and browsers
+  if (nativeDecoder) return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr) // Node.js and browsers
 
   // Fast path for ASCII prefix, this is faster than all alternatives below
   const prefix = decodeLatin1(arr, 0, asciiPrefix(arr))