@ansi-tools/parser 1.0.12 → 1.0.13

package/dist/tokenize.js

@@ -0,0 +1,314 @@
+import { APC, BACKSLASH, BACKSLASH_CODE, BELL, BELL_CODE, CAN, CSI, CSI_CODE, CSI_OPEN, DCS, ESC, ESC_CODE, OSC, OSC_CODE, OSC_OPEN, PM, SOS, ST, ST_CODE, STRING_OPENERS, SUB, TOKEN_TYPES, } from "./constants.js";
+function isInterrupter(c) {
+    return (c === CAN || c === SUB || c === ESC || c === CSI || c === OSC || c === DCS || c === APC || c === PM || c === SOS);
+}
+function isC0Interrupter(c) {
+    return c === CAN || c === SUB;
+}
+function isSequenceStart(c) {
+    return c === ESC || c === CSI || c === OSC || c === DCS || c === APC || c === PM || c === SOS;
+}
+function is8BitIntroducer(c) {
+    return c === CSI || c === OSC || c === DCS || c === APC || c === PM || c === SOS;
+}
+export function* tokenizer(input) {
+    let i = 0;
+    let state = 0; // 0 = GROUND, 1 = SEQUENCE
+    let currentCode = 0;
+    const len = input.length;
+    while (i < len) {
+        if (state === 0) {
+            const textStart = i;
+            let charCode = input.charCodeAt(i);
+            while (i < len && !isSequenceStart(charCode)) {
+                charCode = input.charCodeAt(++i);
+            }
+            if (i > textStart) {
+                yield { type: TOKEN_TYPES.TEXT, pos: textStart, raw: input.substring(textStart, i) };
+            }
+            if (i >= len)
+                break;
+            if (is8BitIntroducer(charCode)) {
+                yield { type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input[i], code: input[i] };
+                i++;
+                state = 1;
+                currentCode = charCode;
+            }
+            else {
+                // ESC
+                const nextCode = input.charCodeAt(i + 1);
+                if (nextCode === CSI_OPEN) {
+                    yield { type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input.substring(i, i + 2), code: CSI_CODE };
+                    i += 2;
+                    state = 1;
+                    currentCode = CSI;
+                }
+                else if (nextCode === OSC_OPEN) {
+                    yield { type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input.substring(i, i + 2), code: OSC_CODE };
+                    i += 2;
+                    state = 1;
+                    currentCode = OSC;
+                }
+                else if (i + 1 < len && STRING_OPENERS.has(input[i + 1])) {
+                    yield { type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input.substring(i, i + 2), code: input[i + 1] };
+                    i += 2;
+                    state = 1;
+                    currentCode = nextCode;
+                }
+                else if (i + 1 < len) {
+                    let j = i + 1;
+                    while (j < len && input.charCodeAt(j) >= 0x20 && input.charCodeAt(j) <= 0x2f)
+                        j++;
+                    if (j < len) {
+                        if (j > i + 1) {
+                            const intermediate = input.substring(i + 1, j);
+                            yield { type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input.substring(i, j), code: ESC_CODE, intermediate };
+                        }
+                        else {
+                            yield { type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input[i], code: ESC_CODE };
+                        }
+                        i = j;
+                        state = 1;
+                        currentCode = ESC;
+                    }
+                    else {
+                        i = j;
+                    }
+                }
+                else {
+                    i++;
+                }
+            }
+        }
+        else {
+            const pos = i;
+            if (currentCode === CSI) {
+                const dataStart = i;
+                while (i < len) {
+                    const charCode = input.charCodeAt(i);
+                    if (isInterrupter(charCode)) {
+                        if (i > dataStart)
+                            yield { type: TOKEN_TYPES.DATA, pos, raw: input.substring(dataStart, i) };
+                        state = 0;
+                        if (isC0Interrupter(charCode))
+                            i++;
+                        break;
+                    }
+                    if (charCode >= 0x40 && charCode <= 0x7e) {
+                        if (i > dataStart)
+                            yield { type: TOKEN_TYPES.DATA, pos, raw: input.substring(dataStart, i) };
+                        yield { type: TOKEN_TYPES.FINAL, pos: i, raw: input[i] };
+                        i++;
+                        state = 0;
+                        break;
+                    }
+                    i++;
+                }
+            }
+            else if (currentCode === ESC) {
+                if (i < len) {
+                    const charCode = input.charCodeAt(i);
+                    if (isInterrupter(charCode)) {
+                        state = 0;
+                        if (isC0Interrupter(charCode))
+                            i++;
+                    }
+                    else {
+                        yield { type: TOKEN_TYPES.FINAL, pos: i, raw: input[i] };
+                        i++;
+                        state = 0;
+                    }
+                }
+            }
+            else {
+                const dataStart = i;
+                while (i < len) {
+                    const charCode = input.charCodeAt(i);
+                    let terminator;
+                    if (charCode === ESC && input.charCodeAt(i + 1) === BACKSLASH) {
+                        terminator = ESC_CODE + BACKSLASH_CODE;
+                    }
+                    else if (charCode === ST) {
+                        terminator = ST_CODE;
+                    }
+                    else if (charCode === BELL && currentCode === OSC) {
+                        terminator = BELL_CODE;
+                    }
+                    if (terminator) {
+                        if (i > dataStart)
+                            yield { type: TOKEN_TYPES.DATA, pos, raw: input.substring(dataStart, i) };
+                        yield { type: TOKEN_TYPES.FINAL, pos: i, raw: terminator };
+                        i += terminator.length;
+                        state = 0;
+                        break;
+                    }
+                    if (isInterrupter(charCode)) {
+                        if (i > dataStart)
+                            yield { type: TOKEN_TYPES.DATA, pos, raw: input.substring(dataStart, i) };
+                        state = 0;
+                        if (isC0Interrupter(charCode))
+                            i++;
+                        break;
+                    }
+                    i++;
+                }
+            }
+            if (state === 1)
+                state = 0;
+        }
+    }
+}
+export function tokenize(input) {
+    const result = [];
+    let i = 0;
+    let state = 0;
+    let currentCode = 0;
+    const len = input.length;
+    while (i < len) {
+        if (state === 0) {
+            const textStart = i;
+            let charCode = input.charCodeAt(i);
+            while (i < len && !isSequenceStart(charCode)) {
+                charCode = input.charCodeAt(++i);
+            }
+            if (i > textStart) {
+                result.push({ type: TOKEN_TYPES.TEXT, pos: textStart, raw: input.substring(textStart, i) });
+            }
+            if (i >= len)
+                break;
+            if (is8BitIntroducer(charCode)) {
+                result.push({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input[i], code: input[i] });
+                i++;
+                state = 1;
+                currentCode = charCode;
+            }
+            else {
+                const nextCode = input.charCodeAt(i + 1);
+                if (nextCode === CSI_OPEN) {
+                    result.push({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input.substring(i, i + 2), code: CSI_CODE });
+                    i += 2;
+                    state = 1;
+                    currentCode = CSI;
+                }
+                else if (nextCode === OSC_OPEN) {
+                    result.push({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input.substring(i, i + 2), code: OSC_CODE });
+                    i += 2;
+                    state = 1;
+                    currentCode = OSC;
+                }
+                else if (i + 1 < len && STRING_OPENERS.has(input[i + 1])) {
+                    result.push({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input.substring(i, i + 2), code: input[i + 1] });
+                    i += 2;
+                    state = 1;
+                    currentCode = nextCode;
+                }
+                else if (i + 1 < len) {
+                    let j = i + 1;
+                    while (j < len && input.charCodeAt(j) >= 0x20 && input.charCodeAt(j) <= 0x2f)
+                        j++;
+                    if (j < len) {
+                        if (j > i + 1) {
+                            const intermediate = input.substring(i + 1, j);
+                            result.push({
+                                type: TOKEN_TYPES.INTRODUCER,
+                                pos: i,
+                                raw: input.substring(i, j),
+                                code: ESC_CODE,
+                                intermediate,
+                            });
+                        }
+                        else {
+                            result.push({ type: TOKEN_TYPES.INTRODUCER, pos: i, raw: input[i], code: ESC_CODE });
+                        }
+                        i = j;
+                        state = 1;
+                        currentCode = ESC;
+                    }
+                    else {
+                        i = j;
+                    }
+                }
+                else {
+                    i++;
+                }
+            }
+        }
+        else {
+            const pos = i;
+            if (currentCode === CSI) {
+                const dataStart = i;
+                while (i < len) {
+                    const charCode = input.charCodeAt(i);
+                    if (isInterrupter(charCode)) {
+                        if (i > dataStart)
+                            result.push({ type: TOKEN_TYPES.DATA, pos, raw: input.substring(dataStart, i) });
+                        state = 0;
+                        if (isC0Interrupter(charCode))
+                            i++;
+                        break;
+                    }
+                    if (charCode >= 0x40 && charCode <= 0x7e) {
+                        if (i > dataStart)
+                            result.push({ type: TOKEN_TYPES.DATA, pos, raw: input.substring(dataStart, i) });
+                        result.push({ type: TOKEN_TYPES.FINAL, pos: i, raw: input[i] });
+                        i++;
+                        state = 0;
+                        break;
+                    }
+                    i++;
+                }
+            }
+            else if (currentCode === ESC) {
+                if (i < len) {
+                    const charCode = input.charCodeAt(i);
+                    if (isInterrupter(charCode)) {
+                        state = 0;
+                        if (isC0Interrupter(charCode))
+                            i++;
+                    }
+                    else {
+                        result.push({ type: TOKEN_TYPES.FINAL, pos: i, raw: input[i] });
+                        i++;
+                        state = 0;
+                    }
+                }
+            }
+            else {
+                const dataStart = i;
+                while (i < len) {
+                    const charCode = input.charCodeAt(i);
+                    let terminator;
+                    if (charCode === ESC && input.charCodeAt(i + 1) === BACKSLASH) {
+                        terminator = ESC_CODE + BACKSLASH_CODE;
+                    }
+                    else if (charCode === ST) {
+                        terminator = ST_CODE;
+                    }
+                    else if (charCode === BELL && currentCode === OSC) {
+                        terminator = BELL_CODE;
+                    }
+                    if (terminator) {
+                        if (i > dataStart)
+                            result.push({ type: TOKEN_TYPES.DATA, pos, raw: input.substring(dataStart, i) });
+                        result.push({ type: TOKEN_TYPES.FINAL, pos: i, raw: terminator });
+                        i += terminator.length;
+                        state = 0;
+                        break;
+                    }
+                    if (isInterrupter(charCode)) {
+                        if (i > dataStart)
+                            result.push({ type: TOKEN_TYPES.DATA, pos, raw: input.substring(dataStart, i) });
+                        state = 0;
+                        if (isC0Interrupter(charCode))
+                            i++;
+                        break;
+                    }
+                    i++;
+                }
+            }
+            if (state === 1)
+                state = 0;
+        }
+    }
+    return result;
+}

package/dist/types.d.ts

@@ -0,0 +1,23 @@
+import type { CODE_TYPES, TOKEN_TYPES } from "./constants.ts";
+export type TOKEN = {
+    type: keyof typeof TOKEN_TYPES;
+    pos: number;
+    raw: string;
+    code?: string;
+    intermediate?: string;
+};
+export type CONTROL_CODE_TYPE = Exclude<keyof typeof CODE_TYPES, TEXT["type"]>;
+export type CONTROL_CODE = {
+    type: CONTROL_CODE_TYPE;
+    command: string;
+    raw: string;
+    params: string[];
+    pos: number;
+};
+export type TEXT = {
+    type: "TEXT";
+    raw: string;
+    pos: number;
+};
+export type CODE = CONTROL_CODE | TEXT;
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE9D,MAAM,MAAM,KAAK,GAAG;IAClB,IAAI,EAAE,MAAM,OAAO,WAAW,CAAC;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC,MAAM,OAAO,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AAE/E,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,iBAAiB,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,IAAI,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,IAAI,GAAG,YAAY,GAAG,IAAI,CAAC"}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,9 +1,26 @@
 {
   "name": "@ansi-tools/parser",
-  "version": "1.0.12",
+  "version": "1.0.13",
   "description": "Tokenize and parse strings containing ANSI escape sequences and control codes",
-  "main": "./dist/index.js",
+  "keywords": [
+    "ansi",
+    "escape codes",
+    "web application"
+  ],
+  "homepage": "https://github.com/webpro/ANSI.tools/tree/main/packages/parser",
+  "bugs": "https://github.com/webpro/ANSI.tools/issues",
+  "license": "ISC",
+  "author": "Lars Kappert <lars@webpro.nl>",
+  "repository": {
+    "type": "git",
+    "url": "github:webpro/ANSI.tools",
+    "directory": "packages/parser"
+  },
+  "files": [
+    "dist"
+  ],
   "type": "module",
+  "main": "./dist/index.js",
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
@@ -16,37 +33,19 @@
       "default": "./dist/escaped.js"
     }
   },
-  "files": [
-    "dist"
-  ],
-  "scripts": {
-    "prebuild": "pnpm type-check && pnpm test",
-    "build": "tsdown --dts src/index.ts src/escaped.ts",
-    "dev": "tsdown --dts src/index.ts src/escaped.ts --watch",
-    "test": "node --import ./test/assertions.ts --test",
-    "type-check": "tsc",
-    "prepack": "pnpm build"
-  },
-  "keywords": [
-    "ansi",
-    "escape codes",
-    "web application"
-  ],
-  "author": "Lars Kappert <lars@webpro.nl>",
-  "license": "ISC",
   "publishConfig": {
     "access": "public"
   },
-  "homepage": "https://github.com/webpro/ANSI.tools/tree/main/packages/parser",
-  "bugs": "https://github.com/webpro/ANSI.tools/issues",
-  "repository": {
-    "type": "git",
-    "url": "github:webpro/ANSI.tools",
-    "directory": "packages/parser"
+  "scripts": {
+    "prebuild": "pnpm test",
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "test": "node --import ./test/assertions.ts --test",
+    "prepack": "pnpm build"
   },
   "devDependencies": {
-    "@types/node": "^24.7.2",
-    "tsdown": "^0.15.7",
+    "@types/node": "^24.12.0",
+    "mitata": "^1.0.34",
     "typescript": "^5.9.3"
   }
 }