@exodus/bytes 1.0.0-rc.8 → 1.0.0

package/utf16.js

@@ -0,0 +1,73 @@
+import * as js from './fallback/utf16.js'
+import { canDecoders, isLE } from './fallback/_utils.js'
+
+const { TextDecoder } = globalThis // Buffer is optional
+const ignoreBOM = true
+const decoderFatalLE = canDecoders ? new TextDecoder('utf-16le', { ignoreBOM, fatal: true }) : null
+const decoderLooseLE = canDecoders ? new TextDecoder('utf-16le', { ignoreBOM }) : null
+const decoderFatalBE = canDecoders ? new TextDecoder('utf-16be', { ignoreBOM, fatal: true }) : null
+const decoderLooseBE = canDecoders ? new TextDecoder('utf-16be', { ignoreBOM }) : null
+const decoderFatal16 = isLE ? decoderFatalLE : decoderFatalBE
+const decoderLoose16 = isLE ? decoderLooseLE : decoderFatalBE
+const { isWellFormed } = String.prototype
+
+const { E_STRICT, E_STRICT_UNICODE } = js
+
+// Unlike utf8, operates on Uint16Arrays by default
+
+const to8 = (a) => new Uint8Array(a.buffer, a.byteOffset, a.byteLength)
+
+function encode(str, loose = false, format = 'uint16') {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  if (format !== 'uint16' && format !== 'uint8-le' && format !== 'uint8-be') {
+    throw new TypeError('Unknown format')
+  }
+
+  const shouldSwap = (isLE && format === 'uint8-be') || (!isLE && format === 'uint8-le')
+
+  // On v8 and SpiderMonkey, check via isWellFormed is faster than js
+  // On JSC, check during loop is faster than isWellFormed
+  // If isWellFormed is available, we skip check during decoding and recheck after
+  // If isWellFormed is unavailable, we check in js during decoding
+  if (!loose && isWellFormed && !isWellFormed.call(str)) throw new TypeError(E_STRICT_UNICODE)
+  const u16 = js.encode(str, loose, !loose && isWellFormed, shouldSwap)
+
+  if (format === 'uint8-le' || format === 'uint8-be') return to8(u16) // Already swapped
+  if (format === 'uint16') return u16
+  throw new Error('Unreachable')
+}
+
+function decode(input, loose = false, format = 'uint16') {
+  let u16
+  switch (format) {
+    case 'uint16':
+      if (!(input instanceof Uint16Array)) throw new TypeError('Expected an Uint16Array')
+      if (canDecoders) return loose ? decoderLoose16.decode(input) : decoderFatal16.decode(input)
+      u16 = input
+      break
+    case 'uint8-le':
+      if (!(input instanceof Uint8Array)) throw new TypeError('Expected an Uint8Array')
+      if (input.byteLength % 2 !== 0) throw new TypeError('Expected even number of bytes')
+      if (canDecoders) return loose ? decoderLooseLE.decode(input) : decoderFatalLE.decode(input)
+      u16 = js.to16input(input, true)
+      break
+    case 'uint8-be':
+      if (!(input instanceof Uint8Array)) throw new TypeError('Expected an Uint8Array')
+      if (input.byteLength % 2 !== 0) throw new TypeError('Expected even number of bytes')
+      if (canDecoders) return loose ? decoderLooseBE.decode(input) : decoderFatalBE.decode(input)
+      u16 = js.to16input(input, false)
+      break
+    default:
+      throw new TypeError('Unknown format')
+  }
+
+  const str = js.decode(u16, loose, !loose && isWellFormed)
+  if (!loose && isWellFormed && !isWellFormed.call(str)) throw new TypeError(E_STRICT)
+
+  return str
+}
+
+export const utf16fromString = (str, format = 'uint16') => encode(str, false, format)
+export const utf16fromStringLoose = (str, format = 'uint16') => encode(str, true, format)
+export const utf16toString = (arr, format = 'uint16') => decode(arr, false, format)
+export const utf16toStringLoose = (arr, format = 'uint16') => decode(arr, true, format)

package/utf16.node.js

@@ -0,0 +1,79 @@
+import { nativeDecoder, isDeno, isLE } from './fallback/_utils.js'
+import { E_STRICT, E_STRICT_UNICODE } from './fallback/utf16.js'
+
+if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
+
+const { isWellFormed } = String.prototype
+const to8 = (a) => new Uint8Array(a.buffer, a.byteOffset, a.byteLength)
+
+// Unlike utf8, operates on Uint16Arrays by default
+
+function encode(str, loose = false, format = 'uint16') {
+  if (typeof str !== 'string') throw new TypeError('Input is not a string')
+  if (format !== 'uint16' && format !== 'uint8-le' && format !== 'uint8-be') {
+    throw new TypeError('Unknown format')
+  }
+
+  if (!isWellFormed.call(str)) {
+    if (!loose) throw new TypeError(E_STRICT_UNICODE)
+    str = nativeDecoder.decode(Buffer.from(str)) // well, let's fix up (Buffer doesn't do this with utf16 encoding)
+  }
+
+  const ble = Buffer.from(str, 'utf-16le')
+
+  if (format === 'uint8-le') return to8(ble)
+  if (format === 'uint8-be') return to8(ble.swap16())
+  if (format === 'uint16') {
+    const b = ble.byteOffset % 2 === 0 ? ble : Buffer.from(ble) // it should be already aligned, but just in case
+    if (!isLE) b.swap16()
+    return new Uint16Array(b.buffer, b.byteOffset, b.byteLength / 2)
+  }
+
+  throw new Error('Unreachable')
+}
+
+const swapped = (x, swap) =>
+  swap ? Buffer.from(x).swap16() : Buffer.from(x.buffer, x.byteOffset, x.byteLength)
+
+// We skip TextDecoder on Node.js, as it's is somewhy significantly slower than Buffer for utf16
+function decodeNode(input, loose = false, format = 'uint16') {
+  let ble
+  if (format === 'uint16') {
+    if (!(input instanceof Uint16Array)) throw new TypeError('Expected an Uint16Array')
+    ble = swapped(input, !isLE)
+  } else if (format === 'uint8-le' || format === 'uint8-be') {
+    if (!(input instanceof Uint8Array)) throw new TypeError('Expected an Uint8Array')
+    if (input.byteLength % 2 !== 0) throw new TypeError('Expected even number of bytes')
+    ble = swapped(input, format === 'uint8-be')
+  } else {
+    throw new TypeError('Unknown format')
+  }
+
+  const str = ble.ucs2Slice(0, ble.byteLength)
+  if (isWellFormed.call(str)) return str
+  if (!loose) throw new TypeError(E_STRICT)
+  return nativeDecoder.decode(Buffer.from(str)) // fixup (see above)
+}
+
+function decodeDecoder(input, loose = false, format = 'uint16') {
+  let encoding
+  if (format === 'uint16') {
+    if (!(input instanceof Uint16Array)) throw new TypeError('Expected an Uint16Array')
+    encoding = isLE ? 'utf-16le' : 'utf-16be'
+  } else if (format === 'uint8-le' || format === 'uint8-be') {
+    if (!(input instanceof Uint8Array)) throw new TypeError('Expected an Uint8Array')
+    if (input.byteLength % 2 !== 0) throw new TypeError('Expected even number of bytes')
+    encoding = format === 'uint8-le' ? 'utf-16le' : 'utf-16be'
+  } else {
+    throw new TypeError('Unknown format')
+  }
+
+  return new TextDecoder(encoding, { ignoreBOM: true, fatal: !loose }).decode(input) // TODO: cache decoder?
+}
+
+const decode = isDeno ? decodeDecoder : decodeNode
+
+export const utf16fromString = (str, format = 'uint16') => encode(str, false, format)
+export const utf16fromStringLoose = (str, format = 'uint16') => encode(str, true, format)
+export const utf16toString = (arr, format = 'uint16') => decode(arr, false, format)
+export const utf16toStringLoose = (arr, format = 'uint16') => decode(arr, true, format)

package/utf8.d.ts

@@ -0,0 +1,42 @@
+/// <reference types="node" />
+
+import type { OutputFormat, Uint8ArrayBuffer } from './array.js';
+
+/**
+ * Encodes a string to UTF-8 bytes (strict mode)
+ * Throws on invalid Unicode (unpaired surrogates)
+ * @param str - The string to encode
+ * @param format - Output format (default: 'uint8')
+ * @returns The encoded bytes
+ */
+export function utf8fromString(str: string, format?: 'uint8'): Uint8ArrayBuffer;
+export function utf8fromString(str: string, format: 'buffer'): Buffer;
+export function utf8fromString(str: string, format?: OutputFormat): Uint8ArrayBuffer | Buffer;
+
+/**
+ * Encodes a string to UTF-8 bytes (loose mode)
+ * Replaces invalid Unicode with replacement character
+ * @param str - The string to encode
+ * @param format - Output format (default: 'uint8')
+ * @returns The encoded bytes
+ */
+export function utf8fromStringLoose(str: string, format?: 'uint8'): Uint8ArrayBuffer;
+export function utf8fromStringLoose(str: string, format: 'buffer'): Buffer;
+export function utf8fromStringLoose(str: string, format?: OutputFormat): Uint8ArrayBuffer | Buffer;
+
+/**
+ * Decodes UTF-8 bytes to a string (strict mode)
+ * Throws on invalid UTF-8 sequences
+ * @param arr - The bytes to decode
+ * @returns The decoded string
+ */
+export function utf8toString(arr: Uint8ArrayBuffer): string;
+
+/**
+ * Decodes UTF-8 bytes to a string (loose mode)
+ * Replaces invalid sequences with replacement character
+ * @param arr - The bytes to decode
+ * @returns The decoded string
+ */
+export function utf8toStringLoose(arr: Uint8ArrayBuffer): string;
+

package/utf8.js

@@ -1,18 +1,16 @@
 import { assertUint8 } from './assert.js'
 import { typedView } from './array.js'
-import { isHermes } from './fallback/_utils.js'
+import { isHermes, nativeDecoder, nativeEncoder } from './fallback/_utils.js'
 import { asciiPrefix, decodeLatin1 } from './fallback/latin1.js'
 import * as js from './fallback/utf8.js'
 
-const { Buffer, TextEncoder, TextDecoder, decodeURIComponent, escape } = globalThis // Buffer is optional
-const haveNativeBuffer = Buffer && !Buffer.TYPED_ARRAY_SUPPORT
-const isNative = (x) => x && (haveNativeBuffer || `${x}`.includes('[native code]')) // we consider Node.js TextDecoder/TextEncoder native
-const haveDecoder = isNative(TextDecoder)
-const nativeEncoder = isNative(TextEncoder) ? new TextEncoder() : null
+const { TextDecoder, decodeURIComponent, escape } = globalThis // Buffer is optional
 // ignoreBOM: true means that BOM will be left as-is, i.e. will be present in the output
 // We don't want to strip anything unexpectedly
-const decoderFatal = haveDecoder ? new TextDecoder('utf8', { ignoreBOM: true, fatal: true }) : null
-const decoderLoose = haveDecoder ? new TextDecoder('utf8', { ignoreBOM: true }) : null
+const decoderLoose = nativeDecoder
+const decoderFatal = nativeDecoder
+  ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
+  : null
 const { isWellFormed } = String.prototype
 
 const { E_STRICT, E_STRICT_UNICODE } = js
@@ -56,7 +54,7 @@ function encode(str, loose = false) {
 function decode(arr, loose = false) {
   assertUint8(arr)
   if (arr.byteLength === 0) return ''
-  if (haveDecoder) return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr) // Node.js and browsers
+  if (nativeDecoder) return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr) // Node.js and browsers
 
   // Fast path for ASCII prefix, this is faster than all alternatives below
   const prefix = decodeLatin1(arr, 0, asciiPrefix(arr))

package/utf8.node.js

@@ -5,16 +5,17 @@ import { isAscii } from 'node:buffer'
 
 if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
 
-const decoderFatal = new TextDecoder('utf8', { ignoreBOM: true, fatal: true })
-const decoderLoose = new TextDecoder('utf8', { ignoreBOM: true })
+const decoderFatal = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
+const decoderLoose = new TextDecoder('utf-8', { ignoreBOM: true })
 const { isWellFormed } = String.prototype
+const isDeno = Boolean(globalThis.Deno)
 
 function encode(str, loose = false) {
   if (typeof str !== 'string') throw new TypeError('Input is not a string')
   const strLength = str.length
   if (strLength === 0) return new Uint8Array() // faster than Uint8Array.of
   let res
-  if (strLength > 0x4_00) {
+  if (strLength > 0x4_00 && !isDeno) {
     // Faster for large strings
     const byteLength = Buffer.byteLength(str)
     res = Buffer.allocUnsafe(byteLength)
@@ -35,11 +36,13 @@ function decode(arr, loose = false) {
   assertUint8(arr)
   const byteLength = arr.byteLength
   if (byteLength === 0) return ''
-  if (byteLength > 0x6_00 && isAscii(arr)) {
+  if (byteLength > 0x6_00 && !(isDeno && loose) && isAscii(arr)) {
     // On non-ascii strings, this loses ~10% * [relative position of the first non-ascii byte] (up to 10% total)
     // On ascii strings, this wins 1.5x on loose = false and 1.3x on loose = true
     // Only makes sense for large enough strings
-    return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).latin1Slice(0, arr.byteLength) // .latin1Slice is faster than .asciiSlice
+    const buf = Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength)
+    if (isDeno) return buf.toString() // Deno suffers from .latin1Slice
+    return buf.latin1Slice(0, arr.byteLength) // .latin1Slice is faster than .asciiSlice
   }
 
   return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr)