@exodus/bytes 1.8.0 → 1.10.0

package/fallback/multi-byte.js

@@ -1,76 +1,89 @@
-import { asciiPrefix, decodeAscii, decodeLatin1, decodeUCS2 } from './latin1.js'
+import { E_STRING } from './_utils.js'
+import { asciiPrefix, decodeAscii, decodeLatin1, decodeUCS2, encodeAscii } from './latin1.js'
 import { getTable } from './multi-byte.table.js'
 
 export const E_STRICT = 'Input is not well-formed for this encoding'
 
-// TODO: optimize
+/* Decoders */
 
 // If the decoder is not cleared properly, state can be preserved between non-streaming calls!
 // See comment about fatal stream
 
-// Common between euc-kr and big5
-function bigDecoder(err, pair) {
-  let lead = 0
-  let oi = 0
-  let o16
-
-  const decodeLead = (b) => {
-    const p = pair(lead, b)
-    lead = 0
-    if (typeof p === 'number') {
-      o16[oi++] = p
-    } else if (p) {
-      // This is still faster than string concatenation. Can we optimize strings though?
-      for (let i = 0; i < p.length; i++) o16[oi++] = p.charCodeAt(i)
-    } else {
-      o16[oi++] = err()
-      if (b < 128) o16[oi++] = b
-    }
-  }
+// All except iso-2022-jp are ASCII supersets
+// When adding something that is not an ASCII superset, ajust the ASCII fast path
+const mappers = {
+  // https://encoding.spec.whatwg.org/#euc-kr-decoder
+  'euc-kr': (err) => {
+    const euc = getTable('euc-kr')
+    let lead = 0
+    let oi = 0
+    let o16
 
-  const decode = (arr, start, end, stream) => {
-    let i = start
-    o16 = new Uint16Array(end - start + (lead ? 1 : 0)) // there are pairs but they consume more than one byte
-    oi = 0
-
-    if (lead && i < end) decodeLead(arr[i++])
-    while (i < end) {
-      const b = arr[i++]
-      if (b < 128) {
-        o16[oi++] = b
-      } else if (b === 0x80 || b === 0xff) {
+    const decodeLead = (b) => {
+      if (b < 0x41 || b > 0xfe) {
+        lead = 0
         o16[oi++] = err()
+        if (b < 128) o16[oi++] = b
       } else {
-        lead = b
-        if (i < end) decodeLead(arr[i++])
+        const p = euc[(lead - 0x81) * 190 + b - 0x41]
+        lead = 0
+        if (p) {
+          o16[oi++] = p
+        } else {
+          o16[oi++] = err()
+          if (b < 128) o16[oi++] = b
+        }
       }
     }
 
-    if (lead && !stream) {
-      lead = 0
-      o16[oi++] = err()
-    }
+    const decode = (arr, start, end, stream) => {
+      let i = start
+      o16 = new Uint16Array(end - start + (lead ? 1 : 0)) // there are pairs but they consume more than one byte
+      oi = 0
 
-    const res = decodeUCS2(o16, oi)
-    o16 = null
-    return res
-  }
+      // Fast path
+      if (!lead) {
+        for (const last1 = end - 1; i < last1; ) {
+          const l = arr[i]
+          if (l < 128) {
+            o16[oi++] = l
+            i++
+          } else {
+            if (l === 0x80 || l === 0xff) break
+            const b = arr[i + 1]
+            if (b < 0x41 || b === 0xff) break
+            const p = euc[(l - 0x81) * 190 + b - 0x41]
+            if (!p) break
+            o16[oi++] = p
+            i += 2
+          }
+        }
+      }
 
-  return { decode, isAscii: () => lead === 0 }
-}
+      if (lead && i < end) decodeLead(arr[i++])
+      while (i < end) {
+        const b = arr[i++]
+        if (b < 128) {
+          o16[oi++] = b
+        } else if (b === 0x80 || b === 0xff) {
+          o16[oi++] = err()
+        } else {
+          lead = b
+          if (i < end) decodeLead(arr[i++])
+        }
+      }
 
-// All except iso-2022-jp are ASCII supersets
-// When adding something that is not an ASCII superset, ajust the ASCII fast path
-const REP = 0xff_fd
-const mappers = {
-  // https://encoding.spec.whatwg.org/#euc-kr-decoder
-  'euc-kr': (err) => {
-    const euc = getTable('euc-kr')
-    return bigDecoder(err, (l, b) => {
-      if (b < 0x41 || b > 0xfe) return
-      const cp = euc[(l - 0x81) * 190 + b - 0x41]
-      return cp !== undefined && cp !== REP ? cp : undefined
-    })
+      if (lead && !stream) {
+        lead = 0
+        o16[oi++] = err()
+      }
+
+      const res = decodeUCS2(o16, oi)
+      o16 = null
+      return res
+    }
+
+    return { decode, isAscii: () => lead === 0 }
   },
   // https://encoding.spec.whatwg.org/#euc-jp-decoder
   'euc-jp': (err) => {
@@ -96,7 +109,7 @@ const mappers = {
 
         lead = 0
         j12 = false
-        if (cp !== undefined && cp !== REP) {
+        if (cp) {
           o16[oi++] = cp
         } else {
           o16[oi++] = err()
@@ -110,6 +123,30 @@ const mappers = {
       o16 = new Uint16Array(end - start + (lead ? 1 : 0))
       oi = 0
 
+      // Fast path, non-j12
+      // lead = 0 means j12 = 0
+      if (!lead) {
+        for (const last1 = end - 1; i < last1; ) {
+          const l = arr[i]
+          if (l < 128) {
+            o16[oi++] = l
+            i++
+          } else {
+            const b = arr[i + 1]
+            if (l === 0x8e && b >= 0xa1 && b <= 0xdf) {
+              o16[oi++] = 0xfe_c0 + b
+              i += 2
+            } else {
+              if (l < 0xa1 || l === 0xff || b < 0xa1 || b === 0xff) break
+              const cp = jis0208[(l - 0xa1) * 94 + b - 0xa1]
+              if (!cp) break
+              o16[oi++] = cp
+              i += 2
+            }
+          }
+        }
+      }
+
       if (lead && i < end) decodeLead(arr[i++])
       if (lead && i < end) decodeLead(arr[i++]) // could be two leads, but no more
       while (i < end) {
@@ -187,7 +224,7 @@ const mappers = {
           dState = 4
           if (b >= 0x21 && b <= 0x7e) {
             const cp = jis0208[(lead - 0x21) * 94 + b - 0x21]
-            if (cp !== undefined && cp !== REP) return cp
+            if (cp) return cp
           }
 
           return err()
@@ -314,7 +351,7 @@ const mappers = {
         }
 
         const cp = jis0208[p]
-        if (cp !== undefined && cp !== REP) {
+        if (cp) {
           o16[oi++] = cp
           return
         }
@@ -329,6 +366,34 @@ const mappers = {
       oi = 0
       let i = start
 
+      // Fast path
+      if (!lead) {
+        for (const last1 = end - 1; i < last1; ) {
+          const l = arr[i]
+          if (l <= 0x80) {
+            o16[oi++] = l
+            i++
+          } else if (l >= 0xa1 && l <= 0xdf) {
+            o16[oi++] = 0xfe_c0 + l
+            i++
+          } else {
+            if (l === 0xa0 || l > 0xfc) break
+            const b = arr[i + 1]
+            if (b < 0x40 || b > 0xfc || b === 0x7f) break
+            const p = (l - (l < 0xa0 ? 0x81 : 0xc1)) * 188 + b - (b < 0x7f ? 0x40 : 0x41)
+            if (p >= 8836 && p <= 10_715) {
+              o16[oi++] = 0xe0_00 - 8836 + p
+              i += 2
+            } else {
+              const cp = jis0208[p]
+              if (!cp) break
+              o16[oi++] = cp
+              i += 2
+            }
+          }
+        }
+      }
+
       if (lead && i < end) decodeLead(arr[i++])
       while (i < end) {
         const b = arr[i++]
@@ -386,6 +451,36 @@ const mappers = {
       let i = start
       const pushback = [] // local and auto-cleared
 
+      // Fast path for 2-byte only
+      // pushback is always empty ad start, and g1 = 0 means g2 = g3 = 0
+      if (g1 === 0) {
+        for (const last1 = end - 1; i < last1; ) {
+          const b = arr[i]
+          if (b < 128) {
+            o16[oi++] = b
+            i++
+          } else if (b === 0x80) {
+            o16[oi++] = 0x20_ac
+            i++
+          } else {
+            if (b === 0xff) break
+            const n = arr[i + 1]
+            let cp
+            if (n < 0x7f) {
+              if (n < 0x40) break
+              cp = gb18030[(b - 0x81) * 190 + n - 0x40]
+            } else {
+              if (n === 0xff || n === 0x7f) break
+              cp = gb18030[(b - 0x81) * 190 + n - 0x41]
+            }
+
+            if (!cp) break
+            o16[oi++] = cp // 16-bit
+            i += 2
+          }
+        }
+      }
+
       // First, dump everything until EOF
       // Same as the full loop, but without EOF handling
       while (i < end || pushback.length > 0) {
@@ -395,11 +490,7 @@ const mappers = {
           // hence, 3 checks for g3 is faster than 3 checks for g1
           if (g2) {
             if (g3) {
-              if (b < 0x30 || b > 0x39) {
-                pushback.push(b, g3, g2)
-                g1 = g2 = g3 = 0
-                o16[oi++] = err()
-              } else {
+              if (b <= 0x39 && b >= 0x30) {
                 const p = index(
                   (g1 - 0x81) * 12_600 + (g2 - 0x30) * 1260 + (g3 - 0x81) * 10 + b - 0x30
                 )
@@ -413,6 +504,10 @@ const mappers = {
                   o16[oi++] = 0xd8_00 | (d >> 10)
                   o16[oi++] = 0xdc_00 | (d & 0x3_ff)
                 }
+              } else {
+                pushback.push(b, g3, g2)
+                g1 = g2 = g3 = 0
+                o16[oi++] = err()
               }
             } else if (b >= 0x81 && b <= 0xfe) {
               g3 = b
@@ -421,7 +516,7 @@ const mappers = {
               g1 = g2 = 0
               o16[oi++] = err()
             }
-          } else if (b >= 0x30 && b <= 0x39) {
+          } else if (b <= 0x39 && b >= 0x30) {
             g2 = b
           } else {
             let cp
@@ -430,7 +525,7 @@ const mappers = {
             }
 
             g1 = 0
-            if (cp !== undefined && cp !== REP) {
+            if (cp) {
               o16[oi++] = cp // 16-bit
             } else {
               o16[oi++] = err()
@@ -464,10 +559,85 @@ const mappers = {
     // The only decoder which returns multiple codepoints per byte, also has non-charcode codepoints
     // We store that as strings
     const big5 = getTable('big5')
-    return bigDecoder(err, (l, b) => {
-      if (b < 0x40 || (b > 0x7e && b < 0xa1) || b === 0xff) return
-      return big5[(l - 0x81) * 157 + b - (b < 0x7f ? 0x40 : 0x62)] // strings
-    })
+    let lead = 0
+    let oi = 0
+    let o16
+
+    const decodeLead = (b) => {
+      if (b < 0x40 || (b > 0x7e && b < 0xa1) || b === 0xff) {
+        lead = 0
+        o16[oi++] = err()
+        if (b < 128) o16[oi++] = b
+      } else {
+        const p = big5[(lead - 0x81) * 157 + b - (b < 0x7f ? 0x40 : 0x62)]
+        lead = 0
+        if (p > 0x1_00_00) {
+          o16[oi++] = p >> 16
+          o16[oi++] = p & 0xff_ff
+        } else if (p) {
+          o16[oi++] = p
+        } else {
+          o16[oi++] = err()
+          if (b < 128) o16[oi++] = b
+        }
+      }
+    }
+
+    // eslint-disable-next-line sonarjs/no-identical-functions
+    const decode = (arr, start, end, stream) => {
+      let i = start
+      o16 = new Uint16Array(end - start + (lead ? 1 : 0)) // there are pairs but they consume more than one byte
+      oi = 0
+
+      // Fast path
+      if (!lead) {
+        for (const last1 = end - 1; i < last1; ) {
+          const l = arr[i]
+          if (l < 128) {
+            o16[oi++] = l
+            i++
+          } else {
+            if (l === 0x80 || l === 0xff) break
+            const b = arr[i + 1]
+            if (b < 0x40 || (b > 0x7e && b < 0xa1) || b === 0xff) break
+            const p = big5[(l - 0x81) * 157 + b - (b < 0x7f ? 0x40 : 0x62)]
+            if (p > 0x1_00_00) {
+              o16[oi++] = p >> 16
+              o16[oi++] = p & 0xff_ff
+            } else {
+              if (!p) break
+              o16[oi++] = p
+            }
+
+            i += 2
+          }
+        }
+      }
+
+      if (lead && i < end) decodeLead(arr[i++])
+      while (i < end) {
+        const b = arr[i++]
+        if (b < 128) {
+          o16[oi++] = b
+        } else if (b === 0x80 || b === 0xff) {
+          o16[oi++] = err()
+        } else {
+          lead = b
+          if (i < end) decodeLead(arr[i++])
+        }
+      }
+
+      if (lead && !stream) {
+        lead = 0
+        o16[oi++] = err()
+      }
+
+      const res = decodeUCS2(o16, oi)
+      o16 = null
+      return res
+    }
+
+    return { decode, isAscii: () => lead === 0 }
   },
 }
 
@@ -482,7 +652,7 @@ export function multibyteDecoder(enc, loose = false) {
   const asciiSuperset = isAsciiSuperset(enc)
   let streaming // because onErr is cached in mapper
   const onErr = loose
-    ? () => REP
+    ? () => 0xff_fd
     : () => {
         // The correct way per spec seems to be not destoying the decoder state in stream mode, even when fatal
         // Decoders big5, euc-jp, euc-kr, shift_jis, gb18030 / gbk - all clear state before throwing unless EOF, so not affected
@@ -504,3 +674,289 @@ export function multibyteDecoder(enc, loose = false) {
     return res + mapper.decode(arr, res.length, arr.length, stream)
   }
 }
+
+/* Encoders */
+
+const maps = new Map()
+const e7 = [[148, 236], [149, 237], [150, 243]] // prettier-ignore
+const e8 = [[30, 89], [38, 97], [43, 102], [44, 103], [50, 109], [67, 126], [84, 144], [100, 160]] // prettier-ignore
+const preencoders = {
+  __proto__: null,
+  big5: (p) => ((((p / 157) | 0) + 0x81) << 8) | ((p % 157 < 0x3f ? 0x40 : 0x62) + (p % 157)),
+  shift_jis: (p) => {
+    const l = (p / 188) | 0
+    const t = p % 188
+    return ((l + (l < 0x1f ? 0x81 : 0xc1)) << 8) | ((t < 0x3f ? 0x40 : 0x41) + t)
+  },
+  'iso-2022-jp': (p) => ((((p / 94) | 0) + 0x21) << 8) | ((p % 94) + 0x21),
+  'euc-jp': (p) => ((((p / 94) | 0) + 0xa1) << 8) | ((p % 94) + 0xa1),
+  'euc-kr': (p) => ((((p / 190) | 0) + 0x81) << 8) | ((p % 190) + 0x41),
+  gb18030: (p) => ((((p / 190) | 0) + 0x81) << 8) | ((p % 190 < 0x3f ? 0x40 : 0x41) + (p % 190)),
+}
+
+preencoders.gbk = preencoders.gb18030
+
+// We accept that encoders use non-trivial amount of mem, for perf
+// most are are 128 KiB mem, big5 is 380 KiB, lazy-loaded at first use
+function getMap(id, size, ascii) {
+  const cached = maps.get(id)
+  if (cached) return cached
+  let tname = id
+  const sjis = id === 'shift_jis'
+  const iso2022jp = id === 'iso-2022-jp'
+  if (iso2022jp) tname = 'jis0208'
+  if (id === 'gbk') tname = 'gb18030'
+  if (id === 'euc-jp' || sjis) tname = 'jis0208'
+  const table = getTable(tname)
+  const map = new Uint16Array(size)
+  const enc = preencoders[id] || ((p) => p + 1)
+  for (let i = 0; i < table.length; i++) {
+    const c = table[i]
+    if (!c) continue
+    if (id === 'big5') {
+      if (i < 5024) continue // this also skips multi-codepoint strings
+      // In big5, all return first entries except for these
+      if (
+        map[c] &&
+        c !== 0x25_50 &&
+        c !== 0x25_5e &&
+        c !== 0x25_61 &&
+        c !== 0x25_6a &&
+        c !== 0x53_41 &&
+        c !== 0x53_45
+      ) {
+        continue
+      }
+    } else {
+      if (sjis && i >= 8272 && i <= 8835) continue
+      if (map[c]) continue
+    }
+
+    if (c > 0xff_ff) {
+      // always a single codepoint here
+      const s = String.fromCharCode(c >> 16, c & 0xff_ff)
+      map[s.codePointAt(0)] = enc(i)
+    } else {
+      map[c] = enc(i)
+    }
+  }
+
+  if (ascii) for (let i = 0; i < 0x80; i++) map[i] = i
+  if (sjis || id === 'euc-jp') {
+    if (sjis) map[0x80] = 0x80
+    const d = sjis ? 0xfe_c0 : 0x70_c0
+    for (let i = 0xff_61; i <= 0xff_9f; i++) map[i] = i - d
+    map[0x22_12] = map[0xff_0d]
+    map[0xa5] = 0x5c
+    map[0x20_3e] = 0x7e
+  } else if (tname === 'gb18030') {
+    if (id === 'gbk') map[0x20_ac] = 0x80
+    for (let i = 0xe7_8d; i <= 0xe7_93; i++) map[i] = i - 0x40_b4
+    for (const [a, b] of e7) map[0xe7_00 | a] = 0xa6_00 | b
+    for (const [a, b] of e8) map[0xe8_00 | a] = 0xfe_00 | b
+  }
+
+  maps.set(id, map)
+  return map
+}
+
+const NON_LATIN = /[^\x00-\xFF]/ // eslint-disable-line no-control-regex
+let gb18030r, katakana
+
+export function multibyteEncoder(enc, onError) {
+  if (!Object.hasOwn(mappers, enc)) throw new RangeError('Unsupported encoding')
+  const size = enc === 'big5' ? 0x2_f8_a7 : 0x1_00_00 // for big5, max codepoint in table + 1
+  const iso2022jp = enc === 'iso-2022-jp'
+  const gb18030 = enc === 'gb18030'
+  const ascii = isAsciiSuperset(enc)
+  const width = iso2022jp ? 5 : gb18030 ? 4 : 2
+  const tailsize = iso2022jp ? 3 : 0
+  const map = getMap(enc, size, ascii)
+  if (gb18030 && !gb18030r) gb18030r = getTable('gb18030-ranges')
+  if (iso2022jp && !katakana) katakana = getTable('iso-2022-jp-katakana')
+  return (str) => {
+    if (typeof str !== 'string') throw new TypeError(E_STRING)
+    if (ascii && !NON_LATIN.test(str)) {
+      try {
+        return encodeAscii(str, E_STRICT)
+      } catch {}
+    }
+
+    const length = str.length
+    const u8 = new Uint8Array(length * width + tailsize)
+    let i = 0
+
+    if (ascii) {
+      while (i < length) {
+        const x = str.charCodeAt(i)
+        if (x >= 128) break
+        u8[i++] = x
+      }
+    }
+
+    // eslint-disable-next-line unicorn/consistent-function-scoping
+    const err = (code) => {
+      if (onError) return onError(code, u8, i)
+      throw new TypeError(E_STRICT)
+    }
+
+    if (!map || map.length < size) /* c8 ignore next */ throw new Error('Unreachable') // Important for perf
+
+    if (iso2022jp) {
+      let state = 0 // 0 = ASCII, 1 = Roman, 2 = jis0208
+      const restore = () => {
+        state = 0
+        u8[i++] = 0x1b
+        u8[i++] = 0x28
+        u8[i++] = 0x42
+      }
+
+      for (let j = 0; j < length; j++) {
+        let x = str.charCodeAt(j)
+        if (x >= 0xd8_00 && x < 0xe0_00) {
+          if (state === 2) restore()
+          if (x >= 0xdc_00 || j + 1 === length) {
+            i += err(x) // lone
+          } else {
+            const x1 = str.charCodeAt(j + 1)
+            if (x1 < 0xdc_00 || x1 >= 0xe0_00) {
+              i += err(x) // lone
+            } else {
+              j++ // consume x1
+              i += err(0x1_00_00 + ((x1 & 0x3_ff) | ((x & 0x3_ff) << 10)))
+            }
+          }
+        } else if (x < 0x80) {
+          if (state === 2 || (state === 1 && (x === 0x5c || x === 0x7e))) restore()
+          if (x === 0xe || x === 0xf || x === 0x1b) {
+            i += err(0xff_fd) // 12.2.2. step 3: This returns U+FFFD rather than codePoint to prevent attacks
+          } else {
+            u8[i++] = x
+          }
+        } else if (x === 0xa5 || x === 0x20_3e) {
+          if (state !== 1) {
+            state = 1
+            u8[i++] = 0x1b
+            u8[i++] = 0x28
+            u8[i++] = 0x4a
+          }
+
+          u8[i++] = x === 0xa5 ? 0x5c : 0x7e
+        } else {
+          if (x === 0x22_12) x = 0xff_0d
+          if (x >= 0xff_61 && x <= 0xff_9f) x = katakana[x - 0xff_61]
+          const e = map[x]
+          if (e) {
+            if (state !== 2) {
+              state = 2
+              u8[i++] = 0x1b
+              u8[i++] = 0x24
+              u8[i++] = 0x42
+            }
+
+            u8[i++] = e >> 8
+            u8[i++] = e & 0xff
+          } else {
+            if (state === 2) restore()
+            i += err(x)
+          }
+        }
+      }
+
+      if (state) restore()
+    } else if (gb18030) {
+      // Deduping this branch hurts other encoders perf
+      const encode = (cp) => {
+        let a = 0, b = 0 // prettier-ignore
+        for (const [c, d] of gb18030r) {
+          if (d > cp) break
+          a = c
+          b = d
+        }
+
+        let rp = cp === 0xe7_c7 ? 7457 : a + cp - b
+        u8[i++] = 0x81 + ((rp / 12_600) | 0)
+        rp %= 12_600
+        u8[i++] = 0x30 + ((rp / 1260) | 0)
+        rp %= 1260
+        u8[i++] = 0x81 + ((rp / 10) | 0)
+        u8[i++] = 0x30 + (rp % 10)
+      }
+
+      for (let j = i; j < length; j++) {
+        const x = str.charCodeAt(j)
+        if (x >= 0xd8_00 && x < 0xe0_00) {
+          if (x >= 0xdc_00 || j + 1 === length) {
+            i += err(x) // lone
+          } else {
+            const x1 = str.charCodeAt(j + 1)
+            if (x1 < 0xdc_00 || x1 >= 0xe0_00) {
+              i += err(x) // lone
+            } else {
+              j++ // consume x1
+              encode(0x1_00_00 + ((x1 & 0x3_ff) | ((x & 0x3_ff) << 10)))
+            }
+          }
+        } else {
+          const e = map[x]
+          if (e & 0xff_00) {
+            u8[i++] = e >> 8
+            u8[i++] = e & 0xff
+          } else if (e || x === 0) {
+            u8[i++] = e
+          } else if (x === 0xe5_e5) {
+            i += err(x)
+          } else {
+            encode(x)
+          }
+        }
+      }
+    } else {
+      const long =
+        enc === 'big5'
+          ? (x) => {
+              const e = map[x]
+              if (e & 0xff_00) {
+                u8[i++] = e >> 8
+                u8[i++] = e & 0xff
+              } else if (e || x === 0) {
+                u8[i++] = e
+              } else {
+                i += err(x)
+              }
+            }
+          : (x) => {
+              i += err(x)
+            }
+
+      for (let j = i; j < length; j++) {
+        const x = str.charCodeAt(j)
+        if (x >= 0xd8_00 && x < 0xe0_00) {
+          if (x >= 0xdc_00 || j + 1 === length) {
+            i += err(x) // lone
+          } else {
+            const x1 = str.charCodeAt(j + 1)
+            if (x1 < 0xdc_00 || x1 >= 0xe0_00) {
+              i += err(x) // lone
+            } else {
+              j++ // consume x1
+              long(0x1_00_00 + ((x1 & 0x3_ff) | ((x & 0x3_ff) << 10)))
+            }
+          }
+        } else {
+          const e = map[x]
+          if (e & 0xff_00) {
+            u8[i++] = e >> 8
+            u8[i++] = e & 0xff
+          } else if (e || x === 0) {
+            u8[i++] = e
+          } else {
+            i += err(x)
+          }
+        }
+      }
+    }
+
+    return i === u8.length ? u8 : u8.subarray(0, i)
+  }
+}