@ansi-tools/parser 0.0.0

package/src/tokenize.test.ts

@@ -0,0 +1,118 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { tokenize } from "./tokenize.ts";
+
+test("unescaped (ESC)", () => {
+  const input = "\x1b[2q";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\x1b[", code: "\x9b" },
+    { type: "DATA", pos: 2, raw: "2" },
+    { type: "FINAL", pos: 3, raw: "q" },
+  ]);
+});
+
+test("unescaped (ESC/2)", () => {
+  const input = "\u001b[2q";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\u001b[", code: "\x9b" },
+    { type: "DATA", pos: 2, raw: "2" },
+    { type: "FINAL", pos: 3, raw: "q" },
+  ]);
+});
+
+test("unescaped (CSI)", () => {
+  const input = "\u009b32mGreen text\u009b0m.";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\u009b", code: "\x9b" },
+    { type: "DATA", pos: 1, raw: "32" },
+    { type: "FINAL", pos: 3, raw: "m" },
+    { type: "TEXT", pos: 4, raw: "Green text" },
+    { type: "INTRODUCER", pos: 14, raw: "\u009b", code: "\x9b" },
+    { type: "DATA", pos: 15, raw: "0" },
+    { type: "FINAL", pos: 16, raw: "m" },
+    { type: "TEXT", pos: 17, raw: "." },
+  ]);
+});
+
+test("unescaped (BEL)", () => {
+  const input = "\x1b]0;title\x07";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\x1b]", code: "\x9d" },
+    { type: "DATA", pos: 2, raw: "0;title" },
+    { type: "FINAL", pos: 9, raw: "\x07" },
+  ]);
+});
+
+test("unescaped (ST - String Terminator)", () => {
+  const input = "\x1b]0;title\x9c";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\x1b]", code: "\x9d" },
+    { type: "DATA", pos: 2, raw: "0;title" },
+    { type: "FINAL", pos: 9, raw: "\x9c" },
+  ]);
+});
+
+test("unescaped (OSC - Operating System Command)", () => {
+  const input = "\x9d0;title\x07";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\x9d", code: "\x9d" },
+    { type: "DATA", pos: 1, raw: "0;title" },
+    { type: "FINAL", pos: 8, raw: "\x07" },
+  ]);
+});
+
+test("unescaped (DCS - Device Control String)", () => {
+  const input = "\x900;1|data\x9c";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\x90", code: "\x90" },
+    { type: "DATA", pos: 1, raw: "0;1|data" },
+    { type: "FINAL", pos: 9, raw: "\x9c" },
+  ]);
+});
+
+test("unescaped (APC - Application Program Command)", () => {
+  const input = "\x9fapp data\x9c";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\x9f", code: "\x9f" },
+    { type: "DATA", pos: 1, raw: "app data" },
+    { type: "FINAL", pos: 9, raw: "\x9c" },
+  ]);
+});
+
+test("unescaped (PM - Privacy Message)", () => {
+  const input = "\x9eprivacy data\x9c";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\x9e", code: "\x9e" },
+    { type: "DATA", pos: 1, raw: "privacy data" },
+    { type: "FINAL", pos: 13, raw: "\x9c" },
+  ]);
+});
+
+test("unescaped (SOS - Start of String)", () => {
+  const input = "\x98string data\x9c";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\x98", code: "\x98" },
+    { type: "DATA", pos: 1, raw: "string data" },
+    { type: "FINAL", pos: 12, raw: "\x9c" },
+  ]);
+});
+
+test("unescaped (ESC with backslash terminator)", () => {
+  const input = "\x1b_payload\x1b\\";
+  const tokens = tokenize(input);
+  assert.deepEqual(tokens, [
+    { type: "INTRODUCER", pos: 0, raw: "\x1b_", code: "_" },
+    { type: "DATA", pos: 2, raw: "payload" },
+    { type: "FINAL", pos: 9, raw: "\x1b\\" },
+  ]);
+});

package/src/tokenize.ts

@@ -0,0 +1,140 @@
+import {
+  APC,
+  BACKSLASH,
+  BELL,
+  CSI,
+  CSI_OPEN,
+  DCS,
+  ESC,
+  OSC,
+  OSC_OPEN,
+  PM,
+  SOS,
+  ST,
+  STRING_OPENERS,
+  TOKEN_TYPES,
+} from "./constants.ts";
+import type { TOKEN } from "./types.ts";
+
+type State = "GROUND" | "SEQUENCE";
+
+const debug = false;
+
+const INTRODUCERS = new Set([ESC, CSI, OSC, DCS, APC, PM, SOS]);
+
+function emit(token: TOKEN) {
+  if (debug) console.log("token", token);
+  return token;
+}
+
+export function* tokenizer(input: string): Generator<TOKEN> {
+  let i = 0;
+  let state: State = "GROUND";
+  let currentCode: string | undefined;
+
+  function setState(next: State, code?: string) {
+    if (debug) console.log(`state ${state} → ${next}`);
+    state = next;
+    currentCode = code;
+  }
+
+  while (i < input.length) {
+    if (state === "GROUND") {
+      const textStart = i;
+      while (i < input.length) {
+        const char = input[i];
+        if (INTRODUCERS.has(char)) {
+          break;
+        }
+        i++;
+      }
+
+      if (i > textStart) {
+        yield emit({ type: TOKEN_TYPES.TEXT, pos: textStart, raw: input.substring(textStart, i) });
+      }
+
+      if (i < input.length) {
+        const char = input[i];
+        if (char === CSI || char === OSC || char === DCS || char === APC || char === PM || char === SOS) {
+          yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char, code: char });
+          i++;
+          setState("SEQUENCE", char);
+        } else if (char === ESC) {
+          const next = input[i + 1];
+          if (next === CSI_OPEN) {
+            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char + next, code: CSI });
+            i += 2;
+            setState("SEQUENCE", CSI);
+          } else if (next === OSC_OPEN) {
+            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char + next, code: OSC });
+            i += 2;
+            setState("SEQUENCE", OSC);
+          } else if (STRING_OPENERS.has(next)) {
+            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char + next, code: next });
+            i += 2;
+            setState("SEQUENCE", next);
+          } else if (next && next.charCodeAt(0) >= 0x20 && next.charCodeAt(0) <= 0x2f) {
+            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char, code: ESC });
+            i += 1;
+            yield emit({ type: TOKEN_TYPES.FINAL, pos: i, raw: next });
+            i++;
+          } else if (next) {
+            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char, code: ESC });
+            yield emit({ type: TOKEN_TYPES.FINAL, pos: i + 1, raw: next });
+            i += 2;
+          } else {
+            yield emit({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: char, code: ESC });
+            i++;
+          }
+        }
+      }
+    } else {
+      const pos = i;
+      const code = currentCode;
+      let data = "";
+
+      if (code === CSI) {
+        while (i < input.length) {
+          const char = input[i];
+          const charCode = char.charCodeAt(0);
+          if (charCode >= 0x40 && charCode < 0x7e) {
+            if (data) yield emit({ type: TOKEN_TYPES.DATA, pos, raw: data });
+            yield emit({ type: TOKEN_TYPES.FINAL, pos: i, raw: char });
+            i++;
+            break;
+          }
+          data += char;
+          i++;
+        }
+      } else if (code) {
+        while (i < input.length) {
+          const char = input[i];
+          let terminator: string | undefined;
+
+          if (char === ST) {
+            terminator = ST;
+          } else if (char === BELL && code === OSC) {
+            terminator = BELL;
+          } else if (char === ESC && input[i + 1] === BACKSLASH) {
+            terminator = ESC + BACKSLASH;
+          }
+
+          if (terminator) {
+            if (data) yield emit({ type: TOKEN_TYPES.DATA, pos, raw: data });
+            yield emit({ type: TOKEN_TYPES.FINAL, pos: i, raw: terminator });
+            i += terminator.length;
+            break;
+          }
+
+          data += char;
+          i++;
+        }
+      }
+      setState("GROUND");
+    }
+  }
+}
+
+export function tokenize(input: string): TOKEN[] {
+  return Array.from(tokenizer(input));
+}

package/src/types.ts

@@ -0,0 +1,24 @@
+import type { TOKEN_TYPES } from "./constants.ts";
+
+export type TOKEN = {
+  type: keyof typeof TOKEN_TYPES;
+  pos: number;
+  raw: string;
+  code?: string;
+};
+
+export type CONTROL_CODE = {
+  type: "CSI" | "DCS" | "DEC" | "ESC" | "OSC" | "SGR" | "STRING" | "PRIVATE";
+  command: string;
+  raw: string;
+  params: string[];
+  pos: number;
+};
+
+export type CONTROL_CODE_TEXT = {
+  type: "TEXT";
+  raw: string;
+  pos: number;
+};
+
+export type CODE = CONTROL_CODE | CONTROL_CODE_TEXT;

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "allowImportingTsExtensions": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "module": "nodenext",
+    "moduleResolution": "nodenext",
+    "noEmit": true,
+    "outDir": "./dist",
+    "skipLibCheck": true,
+    "strict": true,
+    "target": "esnext"
+  },
+  "include": ["src"],
+  "exclude": ["node_modules", "dist"]
+}