@ansi-tools/parser 0.0.0 → 0.0.1

package/src/tokenize.escaped.ts

@@ -1,191 +0,0 @@
-import { BACKSLASH, CSI, CSI_OPEN, ESC, OSC, OSC_OPEN, STRING_OPENERS, TOKEN_TYPES } from "./constants.ts";
-import type { TOKEN, CODE } from "./types.ts";
-import { parser } from "./parse.ts";
-
-type State = "GROUND" | "SEQUENCE";
-
-const debug = false;
-
-const CSI_ESCAPED = "\\u009b";
-
-const INTRODUCERS = [
-  ["\\u001b", 6],
-  [CSI_ESCAPED, 6],
-  ["\\x1b", 4],
-  ["\\033", 4],
-  ["\\e", 2],
-] as const;
-
-const INTRODUCER_LOOKUP = new Map<string, [string, number][]>();
-for (const [sequence, len] of INTRODUCERS) {
-  const secondChar = sequence[1];
-  if (!INTRODUCER_LOOKUP.has(secondChar)) INTRODUCER_LOOKUP.set(secondChar, []);
-  INTRODUCER_LOOKUP.get(secondChar)?.push([sequence, len]);
-}
-
-const STRING_TERMINATORS = new Map([
-  ["\\x9c", 4],
-  ["\\e\\\\", 4],
-  ["\\x1b\\\\", 8],
-]);
-
-const OSC_ONLY_TERMINATORS = new Map([
-  ["\\a", 2],
-  ["\\x07", 4],
-  ["\\u0007", 6],
-]);
-
-const ST_MAX_LENGTH = Math.max(...STRING_TERMINATORS.values());
-const OSC_TERM_MAX_LENGTH = Math.max(...OSC_ONLY_TERMINATORS.values());
-const INTRODUCER_PEEK_AHEAD = new Set(INTRODUCERS.map(entry => entry[0][1]));
-
-function emit(token: TOKEN) {
-  if (debug) console.log("token", token);
-  return token;
-}
-
-export function* tokenizer(input: string): Generator<TOKEN> {
-  let i = 0;
-  let state: State = "GROUND";
-  let currentCode: string | undefined;
-
-  function setState(next: State, code?: string) {
-    if (debug) console.log(`state ${state} → ${next}`);
-    state = next;
-    currentCode = code;
-  }
-
-  while (i < input.length) {
-    if (state === "GROUND") {
-      const textStart = i;
-      while (i < input.length) {
-        const backslashIndex = input.indexOf(BACKSLASH, i);
-
-        if (backslashIndex === -1) {
-          i = input.length;
-          break;
-        }
-
-        const nextChar = input[backslashIndex + 1];
-        if (nextChar && INTRODUCER_PEEK_AHEAD.has(nextChar)) {
-          i = backslashIndex;
-          break;
-        } else {
-          i = backslashIndex + 1;
-        }
-      }
-
-      if (i > textStart) {
-        yield emit({ type: TOKEN_TYPES.TEXT, pos: textStart, raw: input.substring(textStart, i) });
-      }
-
-      if (i < input.length) {
-        const candidates = INTRODUCER_LOOKUP.get(input[i + 1]);
-        if (candidates) {
-          for (const [sequence, len] of candidates) {
-            if (i + len <= input.length && input.substring(i, i + len) === sequence) {
-              if (sequence === CSI_ESCAPED) {
-                yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: sequence, code: CSI });
-                i += len;
-                setState("SEQUENCE", CSI);
-              } else {
-                const nextChar = input[i + len];
-                if (nextChar === CSI_OPEN) {
-                  yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: sequence + nextChar, code: CSI });
-                  i += len + 1;
-                  setState("SEQUENCE", CSI);
-                } else if (nextChar === OSC_OPEN) {
-                  yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: sequence + nextChar, code: OSC });
-                  i += len + 1;
-                  setState("SEQUENCE", OSC);
-                } else if (STRING_OPENERS.has(nextChar)) {
-                  yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: sequence + nextChar, code: nextChar });
-                  i += len + 1;
-                  setState("SEQUENCE", nextChar);
-                } else if (nextChar && nextChar.charCodeAt(0) >= 0x20 && nextChar.charCodeAt(0) <= 0x2f) {
-                  yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: sequence + nextChar, code: CSI });
-                  i += len + 1;
-                  setState("SEQUENCE", CSI);
-                } else if (nextChar) {
-                  yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: sequence, code: ESC });
-                  i += len;
-                  yield emit({ type: TOKEN_TYPES.FINAL, pos: i, raw: nextChar });
-                  i++;
-                } else {
-                  yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: sequence, code: ESC });
-                  i += len;
-                }
-              }
-              break;
-            }
-          }
-        }
-      }
-    } else {
-      let terminator = "";
-      let terminatorPos = -1;
-      const pos = i;
-      const code = currentCode;
-
-      while (!terminator && i < input.length) {
-        const char = input[i];
-        if (code === CSI) {
-          const charCode = input.charCodeAt(i);
-          if (charCode >= 0x40 && charCode < 0x7e) {
-            terminator = char;
-            terminatorPos = i;
-            i++;
-          }
-        } else if (code) {
-          if (char === BACKSLASH) {
-            if (code === OSC) {
-              for (let len = OSC_TERM_MAX_LENGTH; len >= 2; len -= 2) {
-                if (i + len <= input.length) {
-                  const sequence = input.substring(i, i + len);
-                  if (OSC_ONLY_TERMINATORS.has(sequence)) {
-                    terminator = sequence;
-                    terminatorPos = i;
-                    i += len;
-                    break;
-                  }
-                }
-              }
-            }
-            if (!terminator) {
-              for (let len = ST_MAX_LENGTH; len >= 2; len -= 2) {
-                if (i + len <= input.length) {
-                  const sequence = input.substring(i, i + len);
-                  if (STRING_TERMINATORS.has(sequence)) {
-                    terminator = sequence;
-                    terminatorPos = i;
-                    i += len;
-                    break;
-                  }
-                }
-              }
-            }
-          }
-        }
-
-        if (!terminator) {
-          i++;
-        }
-      }
-
-      if (terminatorPos > pos) {
-        const data = input.substring(pos, terminatorPos);
-        yield emit({ type: TOKEN_TYPES.DATA, pos, raw: data });
-      }
-
-      if (terminator) {
-        yield emit({ type: TOKEN_TYPES.FINAL, pos: terminatorPos, raw: terminator });
-      }
-
-      setState("GROUND");
-    }
-  }
-}
-
-export function tokenize(input: string): TOKEN[] {
-  return Array.from(tokenizer(input));
-}

package/src/tokenize.test.ts

@@ -1,118 +0,0 @@
-import assert from "node:assert/strict";
-import { test } from "node:test";
-import { tokenize } from "./tokenize.ts";
-
-test("unescaped (ESC)", () => {
-  const input = "\x1b[2q";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\x1b[", code: "\x9b" },
-    { type: "DATA", pos: 2, raw: "2" },
-    { type: "FINAL", pos: 3, raw: "q" },
-  ]);
-});
-
-test("unescaped (ESC/2)", () => {
-  const input = "\u001b[2q";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\u001b[", code: "\x9b" },
-    { type: "DATA", pos: 2, raw: "2" },
-    { type: "FINAL", pos: 3, raw: "q" },
-  ]);
-});
-
-test("unescaped (CSI)", () => {
-  const input = "\u009b32mGreen text\u009b0m.";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\u009b", code: "\x9b" },
-    { type: "DATA", pos: 1, raw: "32" },
-    { type: "FINAL", pos: 3, raw: "m" },
-    { type: "TEXT", pos: 4, raw: "Green text" },
-    { type: "INTRODUCER", pos: 14, raw: "\u009b", code: "\x9b" },
-    { type: "DATA", pos: 15, raw: "0" },
-    { type: "FINAL", pos: 16, raw: "m" },
-    { type: "TEXT", pos: 17, raw: "." },
-  ]);
-});
-
-test("unescaped (BEL)", () => {
-  const input = "\x1b]0;title\x07";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\x1b]", code: "\x9d" },
-    { type: "DATA", pos: 2, raw: "0;title" },
-    { type: "FINAL", pos: 9, raw: "\x07" },
-  ]);
-});
-
-test("unescaped (ST - String Terminator)", () => {
-  const input = "\x1b]0;title\x9c";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\x1b]", code: "\x9d" },
-    { type: "DATA", pos: 2, raw: "0;title" },
-    { type: "FINAL", pos: 9, raw: "\x9c" },
-  ]);
-});
-
-test("unescaped (OSC - Operating System Command)", () => {
-  const input = "\x9d0;title\x07";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\x9d", code: "\x9d" },
-    { type: "DATA", pos: 1, raw: "0;title" },
-    { type: "FINAL", pos: 8, raw: "\x07" },
-  ]);
-});
-
-test("unescaped (DCS - Device Control String)", () => {
-  const input = "\x900;1|data\x9c";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\x90", code: "\x90" },
-    { type: "DATA", pos: 1, raw: "0;1|data" },
-    { type: "FINAL", pos: 9, raw: "\x9c" },
-  ]);
-});
-
-test("unescaped (APC - Application Program Command)", () => {
-  const input = "\x9fapp data\x9c";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\x9f", code: "\x9f" },
-    { type: "DATA", pos: 1, raw: "app data" },
-    { type: "FINAL", pos: 9, raw: "\x9c" },
-  ]);
-});
-
-test("unescaped (PM - Privacy Message)", () => {
-  const input = "\x9eprivacy data\x9c";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\x9e", code: "\x9e" },
-    { type: "DATA", pos: 1, raw: "privacy data" },
-    { type: "FINAL", pos: 13, raw: "\x9c" },
-  ]);
-});
-
-test("unescaped (SOS - Start of String)", () => {
-  const input = "\x98string data\x9c";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\x98", code: "\x98" },
-    { type: "DATA", pos: 1, raw: "string data" },
-    { type: "FINAL", pos: 12, raw: "\x9c" },
-  ]);
-});
-
-test("unescaped (ESC with backslash terminator)", () => {
-  const input = "\x1b_payload\x1b\\";
-  const tokens = tokenize(input);
-  assert.deepEqual(tokens, [
-    { type: "INTRODUCER", pos: 0, raw: "\x1b_", code: "_" },
-    { type: "DATA", pos: 2, raw: "payload" },
-    { type: "FINAL", pos: 9, raw: "\x1b\\" },
-  ]);
-});

package/src/tokenize.ts

@@ -1,140 +0,0 @@
-import {
-  APC,
-  BACKSLASH,
-  BELL,
-  CSI,
-  CSI_OPEN,
-  DCS,
-  ESC,
-  OSC,
-  OSC_OPEN,
-  PM,
-  SOS,
-  ST,
-  STRING_OPENERS,
-  TOKEN_TYPES,
-} from "./constants.ts";
-import type { TOKEN } from "./types.ts";
-
-type State = "GROUND" | "SEQUENCE";
-
-const debug = false;
-
-const INTRODUCERS = new Set([ESC, CSI, OSC, DCS, APC, PM, SOS]);
-
-function emit(token: TOKEN) {
-  if (debug) console.log("token", token);
-  return token;
-}
-
-export function* tokenizer(input: string): Generator<TOKEN> {
-  let i = 0;
-  let state: State = "GROUND";
-  let currentCode: string | undefined;
-
-  function setState(next: State, code?: string) {
-    if (debug) console.log(`state ${state} → ${next}`);
-    state = next;
-    currentCode = code;
-  }
-
-  while (i < input.length) {
-    if (state === "GROUND") {
-      const textStart = i;
-      while (i < input.length) {
-        const char = input[i];
-        if (INTRODUCERS.has(char)) {
-          break;
-        }
-        i++;
-      }
-
-      if (i > textStart) {
-        yield emit({ type: TOKEN_TYPES.TEXT, pos: textStart, raw: input.substring(textStart, i) });
-      }
-
-      if (i < input.length) {
-        const char = input[i];
-        if (char === CSI || char === OSC || char === DCS || char === APC || char === PM || char === SOS) {
-          yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char, code: char });
-          i++;
-          setState("SEQUENCE", char);
-        } else if (char === ESC) {
-          const next = input[i + 1];
-          if (next === CSI_OPEN) {
-            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char + next, code: CSI });
-            i += 2;
-            setState("SEQUENCE", CSI);
-          } else if (next === OSC_OPEN) {
-            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char + next, code: OSC });
-            i += 2;
-            setState("SEQUENCE", OSC);
-          } else if (STRING_OPENERS.has(next)) {
-            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char + next, code: next });
-            i += 2;
-            setState("SEQUENCE", next);
-          } else if (next && next.charCodeAt(0) >= 0x20 && next.charCodeAt(0) <= 0x2f) {
-            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char, code: ESC });
-            i += 1;
-            yield emit({ type: TOKEN_TYPES.FINAL, pos: i, raw: next });
-            i++;
-          } else if (next) {
-            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char, code: ESC });
-            yield emit({ type: TOKEN_TYPES.FINAL, pos: i + 1, raw: next });
-            i += 2;
-          } else {
-            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char, code: ESC });
-            i++;
-          }
-        }
-      }
-    } else {
-      const pos = i;
-      const code = currentCode;
-      let data = "";
-
-      if (code === CSI) {
-        while (i < input.length) {
-          const char = input[i];
-          const charCode = char.charCodeAt(0);
-          if (charCode >= 0x40 && charCode < 0x7e) {
-            if (data) yield emit({ type: TOKEN_TYPES.DATA, pos, raw: data });
-            yield emit({ type: TOKEN_TYPES.FINAL, pos: i, raw: char });
-            i++;
-            break;
-          }
-          data += char;
-          i++;
-        }
-      } else if (code) {
-        while (i < input.length) {
-          const char = input[i];
-          let terminator: string | undefined;
-
-          if (char === ST) {
-            terminator = ST;
-          } else if (char === BELL && code === OSC) {
-            terminator = BELL;
-          } else if (char === ESC && input[i + 1] === BACKSLASH) {
-            terminator = ESC + BACKSLASH;
-          }
-
-          if (terminator) {
-            if (data) yield emit({ type: TOKEN_TYPES.DATA, pos, raw: data });
-            yield emit({ type: TOKEN_TYPES.FINAL, pos: i, raw: terminator });
-            i += terminator.length;
-            break;
-          }
-
-          data += char;
-          i++;
-        }
-      }
-      setState("GROUND");
-    }
-  }
-}
-
-export function tokenize(input: string): TOKEN[] {
-  return Array.from(tokenizer(input));
-}

package/src/types.ts

@@ -1,24 +0,0 @@
-import type { TOKEN_TYPES } from "./constants.ts";
-
-export type TOKEN = {
-  type: keyof typeof TOKEN_TYPES;
-  pos: number;
-  raw: string;
-  code?: string;
-};
-
-export type CONTROL_CODE = {
-  type: "CSI" | "DCS" | "DEC" | "ESC" | "OSC" | "SGR" | "STRING" | "PRIVATE";
-  command: string;
-  raw: string;
-  params: string[];
-  pos: number;
-};
-
-export type CONTROL_CODE_TEXT = {
-  type: "TEXT";
-  raw: string;
-  pos: number;
-};
-
-export type CODE = CONTROL_CODE | CONTROL_CODE_TEXT;

package/tsconfig.json

@@ -1,16 +0,0 @@
-{
-  "compilerOptions": {
-    "allowImportingTsExtensions": true,
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "module": "nodenext",
-    "moduleResolution": "nodenext",
-    "noEmit": true,
-    "outDir": "./dist",
-    "skipLibCheck": true,
-    "strict": true,
-    "target": "esnext"
-  },
-  "include": ["src"],
-  "exclude": ["node_modules", "dist"]
-}