@exodus/bytes 1.0.0-rc.8 → 1.0.0

package/fallback/encoding.js

@@ -0,0 +1,290 @@
+// We can't return native TextDecoder if it's present, as Node.js one is broken on windows-1252 and we fix that
+// We are also faster than Node.js built-in on both TextEncoder and TextDecoder
+
+import { utf16toString, utf16toStringLoose } from '@exodus/bytes/utf16.js'
+import { utf8fromStringLoose, utf8toString, utf8toStringLoose } from '@exodus/bytes/utf8.js'
+import { createSinglebyteDecoder } from '@exodus/bytes/single-byte.js'
+import labels from './encoding.labels.js'
+import { unfinishedBytes } from './encoding.util.js'
+
+const E_OPTIONS = 'The "options" argument must be of type object'
+const E_ENCODING = 'Unknown encoding'
+const replacementChar = '\uFFFD'
+
+const E_MULTI =
+  'Legacy multi-byte encodings are disabled in /encoding-lite.js, use /encoding.js for full encodings range support'
+const multibyteSet = new Set(['big5', 'euc-kr', 'euc-jp', 'iso-2022-jp', 'shift_jis', 'gbk', 'gb18030']) // prettier-ignore
+let createMultibyteDecoder
+
+export function setMultibyteDecoder(createDecoder) {
+  createMultibyteDecoder = createDecoder
+}
+
+let labelsMap
+
+// Warning: unlike whatwg-encoding, returns lowercased labels
+// Those are case-insensitive and that's how TextDecoder encoding getter normalizes them
+// https://encoding.spec.whatwg.org/#names-and-labels
+export function normalizeEncoding(label) {
+  // fast path
+  if (label === 'utf-8' || label === 'utf8' || label === 'UTF-8' || label === 'UTF8') return 'utf-8'
+  if (label === 'windows-1252' || label === 'ascii' || label === 'latin1') return 'windows-1252'
+  // full map
+  if (/[^\w\t\n\f\r .:-]/i.test(label)) return null // must be ASCII (with ASCII whitespace)
+  const low = `${label}`.trim().toLowerCase()
+  if (Object.hasOwn(labels, low)) return low
+  if (!labelsMap) {
+    labelsMap = new Map()
+    for (const [label, aliases] of Object.entries(labels)) {
+      for (const alias of aliases) labelsMap.set(alias, label)
+    }
+  }
+
+  const mapped = labelsMap.get(low)
+  if (mapped) return mapped
+  return null
+}
+
+const define = (obj, key, value) => Object.defineProperty(obj, key, { value, writable: false })
+
+const fromSource = (x) => {
+  if (x instanceof Uint8Array) return x
+  if (x instanceof ArrayBuffer) return new Uint8Array(x)
+  if (ArrayBuffer.isView(x)) return new Uint8Array(x.buffer, x.byteOffset, x.byteLength)
+  if (globalThis.SharedArrayBuffer && x instanceof SharedArrayBuffer) return new Uint8Array(x)
+  throw new TypeError('Argument must be a SharedArrayBuffer, ArrayBuffer or ArrayBufferView')
+}
+
+function unicodeDecoder(encoding, loose) {
+  if (encoding === 'utf-8') return loose ? utf8toStringLoose : utf8toString // likely
+  const form = encoding === 'utf-16le' ? 'uint8-le' : 'uint8-be'
+  return loose ? (u) => utf16toStringLoose(u, form) : (u) => utf16toString(u, form)
+}
+
+export class TextDecoder {
+  #decode
+  #unicode
+  #multibyte
+  #chunk
+  #canBOM
+
+  constructor(encoding = 'utf-8', options = {}) {
+    if (typeof options !== 'object') throw new TypeError(E_OPTIONS)
+    const enc = normalizeEncoding(encoding)
+    if (!enc || enc === 'replacement') throw new RangeError(E_ENCODING)
+    define(this, 'encoding', enc)
+    define(this, 'fatal', Boolean(options.fatal))
+    define(this, 'ignoreBOM', Boolean(options.ignoreBOM))
+    this.#unicode = enc === 'utf-8' || enc === 'utf-16le' || enc === 'utf-16be'
+    this.#multibyte = !this.#unicode && multibyteSet.has(enc)
+    this.#canBOM = this.#unicode && !this.ignoreBOM
+  }
+
+  get [Symbol.toStringTag]() {
+    return 'TextDecoder'
+  }
+
+  decode(input, options = {}) {
+    if (typeof options !== 'object') throw new TypeError(E_OPTIONS)
+    const stream = Boolean(options.stream)
+    let u = input === undefined ? new Uint8Array() : fromSource(input)
+
+    if (this.#unicode) {
+      let prefix
+      if (this.#chunk) {
+        if (u.length === 0) {
+          if (stream) return '' // no change
+          u = this.#chunk // process as final chunk to handle errors and state changes
+        } else if (u.length < 3) {
+          // No reason to bruteforce offsets, also it's possible this doesn't yet end the sequence
+          const a = new Uint8Array(u.length + this.#chunk.length)
+          a.set(this.#chunk)
+          a.set(u, this.#chunk.length)
+          u = a
+        } else {
+          // Slice off a small portion of u into prefix chunk so we can decode them separately without extending array size
+          const t = new Uint8Array(this.#chunk.length + 3) // We have 1-3 bytes and need 1-3 more bytes
+          t.set(this.#chunk)
+          t.set(u.subarray(0, 3), this.#chunk.length)
+
+          // Stop at the first offset where unfinished bytes reaches 0 or fits into u
+          // If that doesn't happen (u too short), just concat chunk and u completely
+          for (let i = 1; i <= 3; i++) {
+            const unfinished = unfinishedBytes(t, this.#chunk.length + i, this.encoding) // 0-3
+            if (unfinished <= i) {
+              // Always reachable at 3, but we still need 'unfinished' value for it
+              const add = i - unfinished // 0-3
+              prefix = add > 0 ? t.subarray(0, this.#chunk.length + add) : this.#chunk
+              if (add > 0) u = u.subarray(add)
+              break
+            }
+          }
+        }
+
+        this.#chunk = null
+      } else if (u.byteLength === 0) {
+        if (!stream) this.#canBOM = !this.ignoreBOM
+        return ''
+      }
+
+      // For non-stream utf-8 we don't have to do this as it matches utf8toStringLoose already
+      // For non-stream loose utf-16 we still have to do this as this API supports uneven byteLength unlike utf16toStringLoose
+      let suffix = ''
+      if (stream || (!this.fatal && this.encoding !== 'utf-8')) {
+        const trail = unfinishedBytes(u, u.byteLength, this.encoding)
+        if (trail > 0) {
+          if (stream) {
+            this.#chunk = Uint8Array.from(u.subarray(-trail)) // copy
+          } else {
+            // non-fatal mode as already checked
+            suffix = replacementChar
+          }
+
+          u = u.subarray(0, -trail)
+        }
+      }
+
+      if (this.#canBOM) {
+        const bom = this.#findBom(prefix ?? u)
+        if (bom) {
+          if (stream) this.#canBOM = false
+          if (prefix) {
+            prefix = prefix.subarray(bom)
+          } else {
+            u = u.subarray(bom)
+          }
+        }
+      }
+
+      if (!this.#decode) this.#decode = unicodeDecoder(this.encoding, !this.fatal)
+      try {
+        const res = (prefix ? this.#decode(prefix) : '') + this.#decode(u) + suffix
+        if (res.length > 0 && stream) this.#canBOM = false
+
+        if (!stream) this.#canBOM = !this.ignoreBOM
+        return res
+      } catch (err) {
+        this.#chunk = null // reset unfinished chunk on errors
+        throw err
+      }
+
+      // eslint-disable-next-line no-else-return
+    } else if (this.#multibyte) {
+      if (!createMultibyteDecoder) throw new Error(E_MULTI)
+      if (!this.#decode) this.#decode = createMultibyteDecoder(this.encoding, !this.fatal) // can contain state!
+      return this.#decode(u, stream)
+    } else {
+      if (!this.#decode) this.#decode = createSinglebyteDecoder(this.encoding, !this.fatal)
+      return this.#decode(u)
+    }
+  }
+
+  #findBom(u) {
+    switch (this.encoding) {
+      case 'utf-8':
+        return u.byteLength >= 3 && u[0] === 0xef && u[1] === 0xbb && u[2] === 0xbf ? 3 : 0
+      case 'utf-16le':
+        return u.byteLength >= 2 && u[0] === 0xff && u[1] === 0xfe ? 2 : 0
+      case 'utf-16be':
+        return u.byteLength >= 2 && u[0] === 0xfe && u[1] === 0xff ? 2 : 0
+    }
+
+    throw new Error('Unreachable')
+  }
+}
+
+export class TextEncoder {
+  constructor() {
+    define(this, 'encoding', 'utf-8')
+  }
+
+  get [Symbol.toStringTag]() {
+    return 'TextEncoder'
+  }
+
+  encode(str = '') {
+    if (typeof str !== 'string') str = `${str}`
+    const res = utf8fromStringLoose(str)
+    return res.byteOffset === 0 ? res : res.slice(0) // Ensure 0-offset, to match new Uint8Array (per spec), which is non-pooled
+  }
+
+  encodeInto(str, target) {
+    if (typeof str !== 'string') str = `${str}`
+    if (!(target instanceof Uint8Array)) throw new TypeError('Target must be an Uint8Array')
+    if (target.buffer.detached) return { read: 0, written: 0 } // Until https://github.com/whatwg/encoding/issues/324 is resolved
+
+    const tlen = target.length
+    if (tlen < str.length) str = str.slice(0, tlen)
+    let u8 = utf8fromStringLoose(str)
+    let read
+    if (tlen >= u8.length) {
+      read = str.length
+    } else if (u8.length === str.length) {
+      if (u8.length > tlen) u8 = u8.subarray(0, tlen) // ascii can be truncated
+      read = u8.length
+    } else {
+      u8 = u8.subarray(0, tlen)
+      const unfinished = unfinishedBytes(u8, u8.length, 'utf-8')
+      if (unfinished > 0) u8 = u8.subarray(0, u8.length - unfinished)
+
+      // We can do this because loose str -> u8 -> str preserves length, unlike loose u8 -> str -> u8
+      // Each unpaired surrogate (1 charcode) is replaced with a single charcode
+      read = utf8toStringLoose(u8).length // FIXME: Converting back is very inefficient
+    }
+
+    try {
+      target.set(u8)
+    } catch {
+      return { read: 0, written: 0 } // see above, likely detached but no .detached property support
+    }
+
+    return { read, written: u8.length }
+  }
+}
+
+// Warning: unlike whatwg-encoding, returns lowercased labels
+// Those are case-insensitive and that's how TextDecoder encoding getter normalizes them
+export function getBOMEncoding(input) {
+  const u8 = fromSource(input) // asserts
+  if (u8.length >= 3 && u8[0] === 0xef && u8[1] === 0xbb && u8[2] === 0xbf) return 'utf-8'
+  if (u8.length < 2) return null
+  if (u8[0] === 0xff && u8[1] === 0xfe) return 'utf-16le'
+  if (u8[0] === 0xfe && u8[1] === 0xff) return 'utf-16be'
+  return null
+}
+
+// https://encoding.spec.whatwg.org/#decode
+// Warning: encoding sniffed from BOM takes preference over the supplied one
+// Warning: lossy, performs replacement, no option of throwing
+// Expects normalized (lower-case) encoding as input. Completely ignores it and even skips validation when BOM is found
+// Unlike TextDecoder public API, additionally supports 'replacement' encoding
+export function legacyHookDecode(input, fallbackEncoding) {
+  let u8 = fromSource(input)
+  const bomEncoding = getBOMEncoding(u8)
+  if (bomEncoding) u8 = u8.subarray(bomEncoding === 'utf-8' ? 3 : 2)
+  const enc = bomEncoding ?? fallbackEncoding ?? 'utf-8' // "the byte order mark is more authoritative than anything else"
+
+  if (enc === 'utf-8') return utf8toStringLoose(u8)
+  if (enc === 'utf-16le' || enc === 'utf-16be') {
+    let suffix = ''
+    if (u8.byteLength % 2 !== 0) {
+      suffix = replacementChar
+      u8 = u8.subarray(0, -1)
+    }
+
+    return utf16toStringLoose(u8, enc === 'utf-16le' ? 'uint8-le' : 'uint8-be') + suffix
+  }
+
+  if (!Object.hasOwn(labels, enc)) throw new RangeError(E_ENCODING)
+
+  if (multibyteSet.has(enc)) {
+    if (!createMultibyteDecoder) throw new Error(E_MULTI)
+    return createMultibyteDecoder(enc, true)(u8)
+  }
+
+  // https://encoding.spec.whatwg.org/#replacement-decoder
+  // On non-streaming non-fatal case, it just replaces any non-empty input with a single replacement char
+  if (enc === 'replacement') return input.byteLength > 0 ? replacementChar : ''
+
+  return createSinglebyteDecoder(enc, true)(u8)
+}

package/fallback/encoding.labels.js

@@ -0,0 +1,46 @@
+// See https://encoding.spec.whatwg.org/#names-and-labels
+
+/* eslint-disable @exodus/export-default/named */
+// prettier-ignore
+export default {
+  'utf-8': ['unicode-1-1-utf-8', 'unicode11utf8', 'unicode20utf8', 'utf8', 'x-unicode20utf8'],
+  ibm866: ['866', 'cp866', 'csibm866'],
+  'iso-8859-2': ['csisolatin2', 'iso-ir-101', 'iso8859-2', 'iso88592', 'iso_8859-2', 'iso_8859-2:1987', 'l2', 'latin2'],
+  'iso-8859-3': ['csisolatin3', 'iso-ir-109', 'iso8859-3', 'iso88593', 'iso_8859-3', 'iso_8859-3:1988', 'l3', 'latin3'],
+  'iso-8859-4': ['csisolatin4', 'iso-ir-110', 'iso8859-4', 'iso88594', 'iso_8859-4', 'iso_8859-4:1988', 'l4', 'latin4'],
+  'iso-8859-5': ['csisolatincyrillic', 'cyrillic', 'iso-ir-144', 'iso8859-5', 'iso88595', 'iso_8859-5', 'iso_8859-5:1988'],
+  'iso-8859-6': ['arabic', 'asmo-708', 'csiso88596e', 'csiso88596i', 'csisolatinarabic', 'ecma-114', 'iso-8859-6-e', 'iso-8859-6-i', 'iso-ir-127', 'iso8859-6', 'iso88596', 'iso_8859-6', 'iso_8859-6:1987'],
+  'iso-8859-7': ['csisolatingreek', 'ecma-118', 'elot_928', 'greek', 'greek8', 'iso-ir-126', 'iso8859-7', 'iso88597', 'iso_8859-7', 'iso_8859-7:1987', 'sun_eu_greek'],
+  'iso-8859-8': ['csiso88598e', 'csisolatinhebrew', 'hebrew', 'iso-8859-8-e', 'iso-ir-138', 'iso8859-8', 'iso88598', 'iso_8859-8', 'iso_8859-8:1988', 'visual'],
+  'iso-8859-8-i': ['csiso88598i', 'logical'],
+  'iso-8859-10': ['csisolatin6', 'iso-ir-157', 'iso8859-10', 'iso885910', 'l6', 'latin6'],
+  'iso-8859-13': ['iso8859-13', 'iso885913'],
+  'iso-8859-14': ['iso8859-14', 'iso885914'],
+  'iso-8859-15': ['csisolatin9', 'iso8859-15', 'iso885915', 'iso_8859-15', 'l9'],
+  'iso-8859-16': [],
+  'koi8-r': ['cskoi8r', 'koi', 'koi8', 'koi8_r'],
+  'koi8-u': ['koi8-ru'],
+  macintosh: ['csmacintosh', 'mac', 'x-mac-roman'],
+  'windows-874': ['dos-874', 'iso-8859-11', 'iso8859-11', 'iso885911', 'tis-620'],
+  'windows-1250': ['cp1250', 'x-cp1250'],
+  'windows-1251': ['cp1251', 'x-cp1251'],
+  'windows-1252': ['ansi_x3.4-1968', 'ascii', 'cp1252', 'cp819', 'csisolatin1', 'ibm819', 'iso-8859-1', 'iso-ir-100', 'iso8859-1', 'iso88591', 'iso_8859-1', 'iso_8859-1:1987', 'l1', 'latin1', 'us-ascii', 'x-cp1252'],
+  'windows-1253': ['cp1253', 'x-cp1253'],
+  'windows-1254': ['cp1254', 'csisolatin5', 'iso-8859-9', 'iso-ir-148', 'iso8859-9', 'iso88599', 'iso_8859-9', 'iso_8859-9:1989', 'l5', 'latin5', 'x-cp1254'],
+  'windows-1255': ['cp1255', 'x-cp1255'],
+  'windows-1256': ['cp1256', 'x-cp1256'],
+  'windows-1257': ['cp1257', 'x-cp1257'],
+  'windows-1258': ['cp1258', 'x-cp1258'],
+  'x-mac-cyrillic': ['x-mac-ukrainian'],
+  gbk: ['chinese', 'csgb2312', 'csiso58gb231280', 'gb2312', 'gb_2312', 'gb_2312-80', 'iso-ir-58', 'x-gbk'],
+  gb18030: [],
+  big5: ['big5-hkscs', 'cn-big5', 'csbig5', 'x-x-big5'],
+  'euc-jp': ['cseucpkdfmtjapanese', 'x-euc-jp'],
+  'iso-2022-jp': ['csiso2022jp'],
+  shift_jis: ['csshiftjis', 'ms932', 'ms_kanji', 'shift-jis', 'sjis', 'windows-31j', 'x-sjis'],
+  'euc-kr': ['cseuckr', 'csksc56011987', 'iso-ir-149', 'korean', 'ks_c_5601-1987', 'ks_c_5601-1989', 'ksc5601', 'ksc_5601', 'windows-949'],
+  replacement: ['csiso2022kr', 'hz-gb-2312', 'iso-2022-cn', 'iso-2022-cn-ext', 'iso-2022-kr'],
+  'utf-16be': ['unicodefffe'],
+  'utf-16le': ['csunicode', 'iso-10646-ucs-2', 'ucs-2', 'unicode', 'unicodefeff', 'utf-16'],
+  'x-user-defined': [],
+}

package/fallback/encoding.util.js

@@ -0,0 +1,34 @@
+export function unfinishedBytes(u, len, enc) {
+  switch (enc) {
+    case 'utf-8': {
+      // 0-3
+      let p = 0
+      while (p < 2 && p < len && (u[len - p - 1] & 0xc0) === 0x80) p++ // go back 0-2 trailing bytes
+      if (p === len) return 0 // no space for lead
+      const l = u[len - p - 1]
+      if (l < 0xc2 || l > 0xf4) return 0 // not a lead
+      if (p === 0) return 1 // nothing to recheck, we have only lead, return it. 2-byte must return here
+      if (l < 0xe0 || (l < 0xf0 && p >= 2)) return 0 // 2-byte, or 3-byte or less and we already have 2 trailing
+      const lower = l === 0xf0 ? 0x90 : l === 0xe0 ? 0xa0 : 0x80
+      const upper = l === 0xf4 ? 0x8f : l === 0xed ? 0x9f : 0xbf
+      const n = u[len - p]
+      return n >= lower && n <= upper ? p + 1 : 0
+    }
+
+    case 'utf-16le':
+    case 'utf-16be': {
+      // 0-3
+      let p = 0
+      if (len % 2 !== 0) p++ // uneven bytes
+      const l = len - p - 1
+      if (len - p >= 2) {
+        const last = enc === 'utf-16le' ? (u[l] << 8) ^ u[l - 1] : (u[l - 1] << 8) ^ u[l]
+        if (last >= 0xd8_00 && last < 0xdc_00) p += 2 // lone lead
+      }
+
+      return p
+    }
+  }
+
+  throw new Error('Unsupported encoding')
+}

package/fallback/hex.js

@@ -1,5 +1,5 @@
 import { assertUint8 } from '../assert.js'
-import { nativeDecoder, nativeEncoder, isHermes } from './_utils.js'
+import { nativeDecoder, nativeEncoder, decode2string } from './_utils.js'
 import { encodeAscii, decodeAscii } from './latin1.js'
 
 let hexArray // array of 256 bytes converted to two-char hex strings
@@ -11,58 +11,6 @@ const allowed = '0123456789ABCDEFabcdef'
 
 export const E_HEX = 'Input is not a hex string'
 
-function toHexPartAddition(a, start, end) {
-  let o = ''
-  let i = start
-  const h = hexArray
-  for (const last3 = end - 3; i < last3; i += 4) {
-    const x0 = a[i]
-    const x1 = a[i + 1]
-    const x2 = a[i + 2]
-    const x3 = a[i + 3]
-    o += h[x0]
-    o += h[x1]
-    o += h[x2]
-    o += h[x3]
-  }
-
-  while (i < end) o += h[a[i++]]
-  return o
-}
-
-// Optimiziation for Hermes which is the main user of fallback
-function toHexPartTemplates(a, start, end) {
-  let o = ''
-  let i = start
-  const h = hexArray
-  for (const last15 = end - 15; i < last15; i += 16) {
-    const x0 = a[i]
-    const x1 = a[i + 1]
-    const x2 = a[i + 2]
-    const x3 = a[i + 3]
-    const x4 = a[i + 4]
-    const x5 = a[i + 5]
-    const x6 = a[i + 6]
-    const x7 = a[i + 7]
-    const x8 = a[i + 8]
-    const x9 = a[i + 9]
-    const x10 = a[i + 10]
-    const x11 = a[i + 11]
-    const x12 = a[i + 12]
-    const x13 = a[i + 13]
-    const x14 = a[i + 14]
-    const x15 = a[i + 15]
-    o += `${h[x0]}${h[x1]}${h[x2]}${h[x3]}${h[x4]}${h[x5]}${h[x6]}${h[x7]}${h[x8]}${h[x9]}${h[x10]}${h[x11]}${h[x12]}${h[x13]}${h[x14]}${h[x15]}`
-  }
-
-  while (i < end) o += h[a[i++]]
-  return o
-}
-
-// Using templates is significantly faster in Hermes and JSC
-// It's harder to detect JSC and not important anyway as it has native impl, so we detect only Hermes
-const toHexPart = isHermes ? toHexPartTemplates : toHexPartAddition
-
 export function toHex(arr) {
   assertUint8(arr)
 
@@ -100,23 +48,7 @@ export function toHex(arr) {
     return decodeAscii(oa)
   }
 
-  if (length > 30_000) {
-    // Limit concatenation to avoid excessive GC
-    // Thresholds checked on Hermes
-    const concat = []
-    for (let i = 0; i < length; ) {
-      const step = i + 500
-      const end = step > length ? length : step
-      concat.push(toHexPart(arr, i, end))
-      i = end
-    }
-
-    const res = concat.join('')
-    concat.length = 0
-    return res
-  }
-
-  return toHexPart(arr, 0, length)
+  return decode2string(arr, 0, length, hexArray)
 }
 
 export function fromHex(str) {

package/fallback/latin1.js

@@ -4,6 +4,7 @@ import {
   nativeDecoderLatin1,
   nativeBuffer,
   isHermes,
+  isDeno,
 } from './_utils.js'
 
 // See http://stackoverflow.com/a/22747272/680742, which says that lowest limit is in Chrome, with 0xffff args
@@ -64,7 +65,7 @@ export function decodeLatin1(arr, start = 0, stop = arr.length) {
 export const decodeAscii = nativeBuffer
   ? (a) =>
       // Buffer is faster on Node.js (but only for long enough data), if we know that output is ascii
-      a.byteLength >= 0x3_00
+      a.byteLength >= 0x3_00 && !isDeno
         ? nativeBuffer.from(a.buffer, a.byteOffset, a.byteLength).latin1Slice(0, a.byteLength) // .latin1Slice is faster than .asciiSlice
         : nativeDecoder.decode(a) // On Node.js, utf8 decoder is faster than latin1
   : nativeDecoderLatin1

package/fallback/multi-byte.encodings.cjs

@@ -0,0 +1 @@
+module.exports = () => require('./multi-byte.encodings.json') // lazy-load