@exodus/bytes 1.1.0 → 1.2.0

package/README.md

@@ -41,6 +41,7 @@ Spec compliant, passing WPT and covered with extra tests.
 Moreover, tests for this library uncovered [bugs in all major implementations](https://docs.google.com/spreadsheets/d/1pdEefRG6r9fZy61WHGz0TKSt8cO4ISWqlpBN5KntIvQ/edit).
 
 [Faster than Node.js native implementation on Node.js](https://github.com/nodejs/node/issues/61041#issuecomment-3649242024).
+Runs (and passes WPT) on Node.js built without ICU.
 
 ### Caveat: `TextDecoder` / `TextEncoder` APIs are lossy by default per spec
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@exodus/bytes",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Various operations on Uint8Array data",
   "scripts": {
     "lint": "eslint .",

package/utf16.node.js

@@ -36,6 +36,7 @@ const swapped = (x, swap) =>
   swap ? Buffer.from(x).swap16() : Buffer.from(x.buffer, x.byteOffset, x.byteLength)
 
 // We skip TextDecoder on Node.js, as it's is somewhy significantly slower than Buffer for utf16
+// Also, it incorrectly misses replacements with Node.js is built without ICU, we fix that
 function decodeNode(input, loose = false, format = 'uint16') {
   let ble
   if (format === 'uint16') {

package/utf8.node.js

@@ -1,15 +1,21 @@
 import { assertUint8 } from './assert.js'
 import { typedView } from './array.js'
-import { E_STRICT_UNICODE } from './fallback/utf8.js'
+import { E_STRICT, E_STRICT_UNICODE } from './fallback/utf8.js'
 import { isAscii } from 'node:buffer'
 
 if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
 
-const decoderFatal = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
+let decoderFatal
 const decoderLoose = new TextDecoder('utf-8', { ignoreBOM: true })
 const { isWellFormed } = String.prototype
 const isDeno = Boolean(globalThis.Deno)
 
+try {
+  decoderFatal = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
+} catch {
+  // Without ICU, Node.js doesn't support fatal option for utf-8
+}
+
 function encode(str, loose = false) {
   if (typeof str !== 'string') throw new TypeError('Input is not a string')
   const strLength = str.length
@@ -45,7 +51,14 @@ function decode(arr, loose = false) {
     return buf.latin1Slice(0, arr.byteLength) // .latin1Slice is faster than .asciiSlice
   }
 
-  return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr)
+  if (loose) return decoderLoose.decode(arr)
+  if (decoderFatal) return decoderFatal.decode(arr)
+
+  // We are in an env without native fatal decoder support (non-fixed Node.js without ICU)
+  // Well, just recheck against encode if it contains replacement then, this is still faster than js impl
+  const str = decoderLoose.decode(arr)
+  if (str.includes('\uFFFD') && !Buffer.from(str).equals(arr)) throw new TypeError(E_STRICT)
+  return str
 }
 
 export const utf8fromString = (str, format = 'uint8') => typedView(encode(str, false), format)