@xterm/xterm 5.4.0-beta.1

package/src/common/input/TextDecoder.ts

@@ -0,0 +1,346 @@
+/**
+ * Copyright (c) 2019 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+/**
+ * Polyfill - Convert UTF32 codepoint into JS string.
+ * Note: The built-in String.fromCodePoint happens to be much slower
+ *       due to additional sanity checks. We can avoid them since
+ *       we always operate on legal UTF32 (granted by the input decoders)
+ *       and use this faster version instead.
+ */
+export function stringFromCodePoint(codePoint: number): string {
+  if (codePoint > 0xFFFF) {
+    codePoint -= 0x10000;
+    return String.fromCharCode((codePoint >> 10) + 0xD800) + String.fromCharCode((codePoint % 0x400) + 0xDC00);
+  }
+  return String.fromCharCode(codePoint);
+}
+
+/**
+ * Convert UTF32 char codes into JS string.
+ * Basically the same as `stringFromCodePoint` but for multiple codepoints
+ * in a loop (which is a lot faster).
+ */
+export function utf32ToString(data: Uint32Array, start: number = 0, end: number = data.length): string {
+  let result = '';
+  for (let i = start; i < end; ++i) {
+    let codepoint = data[i];
+    if (codepoint > 0xFFFF) {
+      // JS strings are encoded as UTF16, thus a non BMP codepoint gets converted into a surrogate
+      // pair conversion rules:
+      //  - subtract 0x10000 from code point, leaving a 20 bit number
+      //  - add high 10 bits to 0xD800  --> first surrogate
+      //  - add low 10 bits to 0xDC00   --> second surrogate
+      codepoint -= 0x10000;
+      result += String.fromCharCode((codepoint >> 10) + 0xD800) + String.fromCharCode((codepoint % 0x400) + 0xDC00);
+    } else {
+      result += String.fromCharCode(codepoint);
+    }
+  }
+  return result;
+}
+
+/**
+ * StringToUtf32 - decodes UTF16 sequences into UTF32 codepoints.
+ * To keep the decoder in line with JS strings it handles single surrogates as UCS2.
+ */
+export class StringToUtf32 {
+  private _interim: number = 0;
+
+  /**
+   * Clears interim and resets decoder to clean state.
+   */
+  public clear(): void {
+    this._interim = 0;
+  }
+
+  /**
+   * Decode JS string to UTF32 codepoints.
+   * The methods assumes stream input and will store partly transmitted
+   * surrogate pairs and decode them with the next data chunk.
+   * Note: The method does no bound checks for target, therefore make sure
+   * the provided input data does not exceed the size of `target`.
+   * Returns the number of written codepoints in `target`.
+   */
+  public decode(input: string, target: Uint32Array): number {
+    const length = input.length;
+
+    if (!length) {
+      return 0;
+    }
+
+    let size = 0;
+    let startPos = 0;
+
+    // handle leftover surrogate high
+    if (this._interim) {
+      const second = input.charCodeAt(startPos++);
+      if (0xDC00 <= second && second <= 0xDFFF) {
+        target[size++] = (this._interim - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
+      } else {
+        // illegal codepoint (USC2 handling)
+        target[size++] = this._interim;
+        target[size++] = second;
+      }
+      this._interim = 0;
+    }
+
+    for (let i = startPos; i < length; ++i) {
+      const code = input.charCodeAt(i);
+      // surrogate pair first
+      if (0xD800 <= code && code <= 0xDBFF) {
+        if (++i >= length) {
+          this._interim = code;
+          return size;
+        }
+        const second = input.charCodeAt(i);
+        if (0xDC00 <= second && second <= 0xDFFF) {
+          target[size++] = (code - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
+        } else {
+          // illegal codepoint (USC2 handling)
+          target[size++] = code;
+          target[size++] = second;
+        }
+        continue;
+      }
+      if (code === 0xFEFF) {
+        // BOM
+        continue;
+      }
+      target[size++] = code;
+    }
+    return size;
+  }
+}
+
+/**
+ * Utf8Decoder - decodes UTF8 byte sequences into UTF32 codepoints.
+ */
+export class Utf8ToUtf32 {
+  public interim: Uint8Array = new Uint8Array(3);
+
+  /**
+   * Clears interim bytes and resets decoder to clean state.
+   */
+  public clear(): void {
+    this.interim.fill(0);
+  }
+
+  /**
+   * Decodes UTF8 byte sequences in `input` to UTF32 codepoints in `target`.
+   * The methods assumes stream input and will store partly transmitted bytes
+   * and decode them with the next data chunk.
+   * Note: The method does no bound checks for target, therefore make sure
+   * the provided data chunk does not exceed the size of `target`.
+   * Returns the number of written codepoints in `target`.
+   */
+  public decode(input: Uint8Array, target: Uint32Array): number {
+    const length = input.length;
+
+    if (!length) {
+      return 0;
+    }
+
+    let size = 0;
+    let byte1: number;
+    let byte2: number;
+    let byte3: number;
+    let byte4: number;
+    let codepoint = 0;
+    let startPos = 0;
+
+    // handle leftover bytes
+    if (this.interim[0]) {
+      let discardInterim = false;
+      let cp = this.interim[0];
+      cp &= ((((cp & 0xE0) === 0xC0)) ? 0x1F : (((cp & 0xF0) === 0xE0)) ? 0x0F : 0x07);
+      let pos = 0;
+      let tmp: number;
+      while ((tmp = this.interim[++pos] & 0x3F) && pos < 4) {
+        cp <<= 6;
+        cp |= tmp;
+      }
+      // missing bytes - read ahead from input
+      const type = (((this.interim[0] & 0xE0) === 0xC0)) ? 2 : (((this.interim[0] & 0xF0) === 0xE0)) ? 3 : 4;
+      const missing = type - pos;
+      while (startPos < missing) {
+        if (startPos >= length) {
+          return 0;
+        }
+        tmp = input[startPos++];
+        if ((tmp & 0xC0) !== 0x80) {
+          // wrong continuation, discard interim bytes completely
+          startPos--;
+          discardInterim = true;
+          break;
+        } else {
+          // need to save so we can continue short inputs in next call
+          this.interim[pos++] = tmp;
+          cp <<= 6;
+          cp |= tmp & 0x3F;
+        }
+      }
+      if (!discardInterim) {
+        // final test is type dependent
+        if (type === 2) {
+          if (cp < 0x80) {
+            // wrong starter byte
+            startPos--;
+          } else {
+            target[size++] = cp;
+          }
+        } else if (type === 3) {
+          if (cp < 0x0800 || (cp >= 0xD800 && cp <= 0xDFFF) || cp === 0xFEFF) {
+            // illegal codepoint or BOM
+          } else {
+            target[size++] = cp;
+          }
+        } else {
+          if (cp < 0x010000 || cp > 0x10FFFF) {
+            // illegal codepoint
+          } else {
+            target[size++] = cp;
+          }
+        }
+      }
+      this.interim.fill(0);
+    }
+
+    // loop through input
+    const fourStop = length - 4;
+    let i = startPos;
+    while (i < length) {
+      /**
+       * ASCII shortcut with loop unrolled to 4 consecutive ASCII chars.
+       * This is a compromise between speed gain for ASCII
+       * and penalty for non ASCII:
+       * For best ASCII performance the char should be stored directly into target,
+       * but even a single attempt to write to target and compare afterwards
+       * penalizes non ASCII really bad (-50%), thus we load the char into byteX first,
+       * which reduces ASCII performance by ~15%.
+       * This trial for ASCII reduces non ASCII performance by ~10% which seems acceptible
+       * compared to the gains.
+       * Note that this optimization only takes place for 4 consecutive ASCII chars,
+       * for any shorter it bails out. Worst case - all 4 bytes being read but
+       * thrown away due to the last being a non ASCII char (-10% performance).
+       */
+      while (i < fourStop
+        && !((byte1 = input[i]) & 0x80)
+        && !((byte2 = input[i + 1]) & 0x80)
+        && !((byte3 = input[i + 2]) & 0x80)
+        && !((byte4 = input[i + 3]) & 0x80))
+      {
+        target[size++] = byte1;
+        target[size++] = byte2;
+        target[size++] = byte3;
+        target[size++] = byte4;
+        i += 4;
+      }
+
+      // reread byte1
+      byte1 = input[i++];
+
+      // 1 byte
+      if (byte1 < 0x80) {
+        target[size++] = byte1;
+
+        // 2 bytes
+      } else if ((byte1 & 0xE0) === 0xC0) {
+        if (i >= length) {
+          this.interim[0] = byte1;
+          return size;
+        }
+        byte2 = input[i++];
+        if ((byte2 & 0xC0) !== 0x80) {
+          // wrong continuation
+          i--;
+          continue;
+        }
+        codepoint = (byte1 & 0x1F) << 6 | (byte2 & 0x3F);
+        if (codepoint < 0x80) {
+          // wrong starter byte
+          i--;
+          continue;
+        }
+        target[size++] = codepoint;
+
+        // 3 bytes
+      } else if ((byte1 & 0xF0) === 0xE0) {
+        if (i >= length) {
+          this.interim[0] = byte1;
+          return size;
+        }
+        byte2 = input[i++];
+        if ((byte2 & 0xC0) !== 0x80) {
+          // wrong continuation
+          i--;
+          continue;
+        }
+        if (i >= length) {
+          this.interim[0] = byte1;
+          this.interim[1] = byte2;
+          return size;
+        }
+        byte3 = input[i++];
+        if ((byte3 & 0xC0) !== 0x80) {
+          // wrong continuation
+          i--;
+          continue;
+        }
+        codepoint = (byte1 & 0x0F) << 12 | (byte2 & 0x3F) << 6 | (byte3 & 0x3F);
+        if (codepoint < 0x0800 || (codepoint >= 0xD800 && codepoint <= 0xDFFF) || codepoint === 0xFEFF) {
+          // illegal codepoint or BOM, no i-- here
+          continue;
+        }
+        target[size++] = codepoint;
+
+        // 4 bytes
+      } else if ((byte1 & 0xF8) === 0xF0) {
+        if (i >= length) {
+          this.interim[0] = byte1;
+          return size;
+        }
+        byte2 = input[i++];
+        if ((byte2 & 0xC0) !== 0x80) {
+          // wrong continuation
+          i--;
+          continue;
+        }
+        if (i >= length) {
+          this.interim[0] = byte1;
+          this.interim[1] = byte2;
+          return size;
+        }
+        byte3 = input[i++];
+        if ((byte3 & 0xC0) !== 0x80) {
+          // wrong continuation
+          i--;
+          continue;
+        }
+        if (i >= length) {
+          this.interim[0] = byte1;
+          this.interim[1] = byte2;
+          this.interim[2] = byte3;
+          return size;
+        }
+        byte4 = input[i++];
+        if ((byte4 & 0xC0) !== 0x80) {
+          // wrong continuation
+          i--;
+          continue;
+        }
+        codepoint = (byte1 & 0x07) << 18 | (byte2 & 0x3F) << 12 | (byte3 & 0x3F) << 6 | (byte4 & 0x3F);
+        if (codepoint < 0x010000 || codepoint > 0x10FFFF) {
+          // illegal codepoint, no i-- here
+          continue;
+        }
+        target[size++] = codepoint;
+      } else {
+        // illegal byte, just skip
+      }
+    }
+    return size;
+  }
+}

package/src/common/input/UnicodeV6.ts

@@ -0,0 +1,145 @@
+/**
+ * Copyright (c) 2019 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+import { IUnicodeVersionProvider, UnicodeCharProperties, UnicodeCharWidth } from 'common/services/Services';
+import { UnicodeService } from 'common/services/UnicodeService';
+
+const BMP_COMBINING = [
+  [0x0300, 0x036F], [0x0483, 0x0486], [0x0488, 0x0489],
+  [0x0591, 0x05BD], [0x05BF, 0x05BF], [0x05C1, 0x05C2],
+  [0x05C4, 0x05C5], [0x05C7, 0x05C7], [0x0600, 0x0603],
+  [0x0610, 0x0615], [0x064B, 0x065E], [0x0670, 0x0670],
+  [0x06D6, 0x06E4], [0x06E7, 0x06E8], [0x06EA, 0x06ED],
+  [0x070F, 0x070F], [0x0711, 0x0711], [0x0730, 0x074A],
+  [0x07A6, 0x07B0], [0x07EB, 0x07F3], [0x0901, 0x0902],
+  [0x093C, 0x093C], [0x0941, 0x0948], [0x094D, 0x094D],
+  [0x0951, 0x0954], [0x0962, 0x0963], [0x0981, 0x0981],
+  [0x09BC, 0x09BC], [0x09C1, 0x09C4], [0x09CD, 0x09CD],
+  [0x09E2, 0x09E3], [0x0A01, 0x0A02], [0x0A3C, 0x0A3C],
+  [0x0A41, 0x0A42], [0x0A47, 0x0A48], [0x0A4B, 0x0A4D],
+  [0x0A70, 0x0A71], [0x0A81, 0x0A82], [0x0ABC, 0x0ABC],
+  [0x0AC1, 0x0AC5], [0x0AC7, 0x0AC8], [0x0ACD, 0x0ACD],
+  [0x0AE2, 0x0AE3], [0x0B01, 0x0B01], [0x0B3C, 0x0B3C],
+  [0x0B3F, 0x0B3F], [0x0B41, 0x0B43], [0x0B4D, 0x0B4D],
+  [0x0B56, 0x0B56], [0x0B82, 0x0B82], [0x0BC0, 0x0BC0],
+  [0x0BCD, 0x0BCD], [0x0C3E, 0x0C40], [0x0C46, 0x0C48],
+  [0x0C4A, 0x0C4D], [0x0C55, 0x0C56], [0x0CBC, 0x0CBC],
+  [0x0CBF, 0x0CBF], [0x0CC6, 0x0CC6], [0x0CCC, 0x0CCD],
+  [0x0CE2, 0x0CE3], [0x0D41, 0x0D43], [0x0D4D, 0x0D4D],
+  [0x0DCA, 0x0DCA], [0x0DD2, 0x0DD4], [0x0DD6, 0x0DD6],
+  [0x0E31, 0x0E31], [0x0E34, 0x0E3A], [0x0E47, 0x0E4E],
+  [0x0EB1, 0x0EB1], [0x0EB4, 0x0EB9], [0x0EBB, 0x0EBC],
+  [0x0EC8, 0x0ECD], [0x0F18, 0x0F19], [0x0F35, 0x0F35],
+  [0x0F37, 0x0F37], [0x0F39, 0x0F39], [0x0F71, 0x0F7E],
+  [0x0F80, 0x0F84], [0x0F86, 0x0F87], [0x0F90, 0x0F97],
+  [0x0F99, 0x0FBC], [0x0FC6, 0x0FC6], [0x102D, 0x1030],
+  [0x1032, 0x1032], [0x1036, 0x1037], [0x1039, 0x1039],
+  [0x1058, 0x1059], [0x1160, 0x11FF], [0x135F, 0x135F],
+  [0x1712, 0x1714], [0x1732, 0x1734], [0x1752, 0x1753],
+  [0x1772, 0x1773], [0x17B4, 0x17B5], [0x17B7, 0x17BD],
+  [0x17C6, 0x17C6], [0x17C9, 0x17D3], [0x17DD, 0x17DD],
+  [0x180B, 0x180D], [0x18A9, 0x18A9], [0x1920, 0x1922],
+  [0x1927, 0x1928], [0x1932, 0x1932], [0x1939, 0x193B],
+  [0x1A17, 0x1A18], [0x1B00, 0x1B03], [0x1B34, 0x1B34],
+  [0x1B36, 0x1B3A], [0x1B3C, 0x1B3C], [0x1B42, 0x1B42],
+  [0x1B6B, 0x1B73], [0x1DC0, 0x1DCA], [0x1DFE, 0x1DFF],
+  [0x200B, 0x200F], [0x202A, 0x202E], [0x2060, 0x2063],
+  [0x206A, 0x206F], [0x20D0, 0x20EF], [0x302A, 0x302F],
+  [0x3099, 0x309A], [0xA806, 0xA806], [0xA80B, 0xA80B],
+  [0xA825, 0xA826], [0xFB1E, 0xFB1E], [0xFE00, 0xFE0F],
+  [0xFE20, 0xFE23], [0xFEFF, 0xFEFF], [0xFFF9, 0xFFFB]
+];
+const HIGH_COMBINING = [
+  [0x10A01, 0x10A03], [0x10A05, 0x10A06], [0x10A0C, 0x10A0F],
+  [0x10A38, 0x10A3A], [0x10A3F, 0x10A3F], [0x1D167, 0x1D169],
+  [0x1D173, 0x1D182], [0x1D185, 0x1D18B], [0x1D1AA, 0x1D1AD],
+  [0x1D242, 0x1D244], [0xE0001, 0xE0001], [0xE0020, 0xE007F],
+  [0xE0100, 0xE01EF]
+];
+
+// BMP lookup table, lazy initialized during first addon loading
+let table: Uint8Array;
+
+function bisearch(ucs: number, data: number[][]): boolean {
+  let min = 0;
+  let max = data.length - 1;
+  let mid;
+  if (ucs < data[0][0] || ucs > data[max][1]) {
+    return false;
+  }
+  while (max >= min) {
+    mid = (min + max) >> 1;
+    if (ucs > data[mid][1]) {
+      min = mid + 1;
+    } else if (ucs < data[mid][0]) {
+      max = mid - 1;
+    } else {
+      return true;
+    }
+  }
+  return false;
+}
+
+export class UnicodeV6 implements IUnicodeVersionProvider {
+  public readonly version = '6';
+
+  constructor() {
+    // init lookup table once
+    if (!table) {
+      table = new Uint8Array(65536);
+      table.fill(1);
+      table[0] = 0;
+      // control chars
+      table.fill(0, 1, 32);
+      table.fill(0, 0x7f, 0xa0);
+
+      // apply wide char rules first
+      // wide chars
+      table.fill(2, 0x1100, 0x1160);
+      table[0x2329] = 2;
+      table[0x232a] = 2;
+      table.fill(2, 0x2e80, 0xa4d0);
+      table[0x303f] = 1;  // wrongly in last line
+
+      table.fill(2, 0xac00, 0xd7a4);
+      table.fill(2, 0xf900, 0xfb00);
+      table.fill(2, 0xfe10, 0xfe1a);
+      table.fill(2, 0xfe30, 0xfe70);
+      table.fill(2, 0xff00, 0xff61);
+      table.fill(2, 0xffe0, 0xffe7);
+
+      // apply combining last to ensure we overwrite
+      // wrongly wide set chars:
+      //    the original algo evals combining first and falls
+      //    through to wide check so we simply do here the opposite
+      // combining 0
+      for (let r = 0; r < BMP_COMBINING.length; ++r) {
+        table.fill(0, BMP_COMBINING[r][0], BMP_COMBINING[r][1] + 1);
+      }
+    }
+  }
+
+  public wcwidth(num: number): UnicodeCharWidth {
+    if (num < 32) return 0;
+    if (num < 127) return 1;
+    if (num < 65536) return table[num] as UnicodeCharWidth;
+    if (bisearch(num, HIGH_COMBINING)) return 0;
+    if ((num >= 0x20000 && num <= 0x2fffd) || (num >= 0x30000 && num <= 0x3fffd)) return 2;
+    return 1;
+  }
+
+  public charProperties(codepoint: number, preceding: UnicodeCharProperties): UnicodeCharProperties {
+    let width = this.wcwidth(codepoint);
+    let shouldJoin = width === 0 && preceding !== 0;
+    if (shouldJoin) {
+      const oldWidth = UnicodeService.extractWidth(preceding);
+      if (oldWidth === 0) {
+        shouldJoin = false;
+      } else if (oldWidth > width) {
+        width = oldWidth;
+      }
+    }
+    return UnicodeService.createPropertyValue(0, width, shouldJoin);
+  }
+}