@il4mb/css-tokenizer 1.0.0

package/index.test.ts

@@ -0,0 +1,24 @@
+import { Registry } from "@/registry";
+import { Tokenizer } from "@/tokenizer";
+import { readWhile } from "@/tools";
+
+const registry = new Registry();
+registry.add({
+    type: "var",
+    kind: "class",
+    regex: /^--[a-z][a-z0-9-_]+/i,
+    priority: 10,
+    reader({ index, content }) {
+        let nextIndex = readWhile(content, index + 2, /[a-z0-9-_]/);
+        return [index, nextIndex];
+    },
+});
+
+const tokenizer = new Tokenizer(registry);
+
+
+const content = `--primary`;
+const result = tokenizer.tokenize(content);
+
+console.log(result.toArray()); 
+console.log(JSON.stringify(result.toTokenTree(content), null, 2));

package/package.json

@@ -0,0 +1,33 @@
+{
+    "name": "@il4mb/css-tokenizer",
+    "version": "1.0.0",
+    "description": "Modular, extensible CSS tokenizer for parsing CSS-like strings into tuple lists and token trees.",
+    "keywords": [
+        "css",
+        "tokenizer",
+        "modular"
+    ],
+    "homepage": "https://github.com/il4mb/css-tokenizer#readme",
+    "bugs": {
+        "url": "https://github.com/il4mb/css-tokenizer/issues"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/il4mb/css-tokenizer.git"
+    },
+    "license": "MIT",
+    "author": "il4mb",
+    "type": "module",
+    "exports": {
+        ".": "./src/index.ts"
+    },
+    "main": "index.test.ts",
+    "scripts": {
+        "test": "vitest index.test.ts"
+    },
+    "devDependencies": {
+        "@types/node": "^26.4.1",
+        "typescript": "7.0.2",
+        "vitest": "^5.0.0"
+    }
+}

package/readme.md

@@ -0,0 +1,87 @@
+# css-tokenizer
+
+Modular, extensible CSS tokenizer for parsing CSS-like strings into tuple lists and token trees.
+
+## Get started
+
+Install:
+
+```bash
+npm install @il4mb/css-tokenizer
+```
+
+## Quick usage
+
+```ts
+import { tokenize } from "@il4mb/css-tokenizer";
+
+const content = "width: 10px";
+const result = tokenize(content);
+
+// Tuple list entries are [type, start, end]
+console.log(result.toArray());
+
+// Convert tuples to a structured token tree (useful for walking semantic tokens)
+console.log(JSON.stringify(result.toTokenTree(content), null, 2));
+```
+
+Example token tree (for the value `10px`):
+
+```json
+[
+    {
+        "type": "dimension",
+        "start": 7,
+        "end": 11,
+        "value": "10px",
+        "children": [
+            { "type": "number", "start": 7, "end": 9, "value": "10" },
+            { "type": "word", "start": 9, "end": 11, "value": "px" }
+        ]
+    }
+]
+```
+
+## Custom token types
+
+You can register custom token readers with a `Registry` and use a `Tokenizer` built from that registry.
+
+```ts
+import { Registry, Tokenizer, readWhile } from "@il4mb/css-tokenizer";
+
+const registry = new Registry();
+
+registry.add({
+    type: "var",           // token name
+    kind: "class",        // category hint (class|char|keyword)
+    regex: /^--[a-z][a-z0-9-_]+/i,
+    priority: 10,
+    reader({ index, content }) {
+        // advance past the leading `--`
+        const nextIndex = readWhile(content, index + 2, /[a-z0-9-_]/i);
+        return [index, nextIndex];
+    },
+});
+
+const tokenizer = new Tokenizer(registry);
+const content = `--primary`;
+const result = tokenizer.tokenize(content);
+
+console.log(result.toArray());
+console.log(JSON.stringify(result.toTokenTree(content), null, 2));
+```
+
+## API (high level)
+
+- `tokenize(content: string)` — tokenize using the default registry
+- `Registry` — create and customize token definitions
+- `Tokenizer` — tokenizer instance that uses a `Registry`
+- helpers: `readWhile`, `findFunctionEnd` and others from the `tools` export
+
+## Contributing
+
+PRs and issues welcome. Please add tests for new token readers.
+
+## License
+
+MIT

package/src/global.d.ts

@@ -0,0 +1,20 @@
+export declare global {
+
+    export type Exp = string | RegExp;
+
+    export interface TokenPlain {
+        id: string;
+        type: string;
+        value: string;
+        start: number;
+        end: number;
+        number?: number;
+        unit?: string;
+        children?: TokenPlain[]
+    }
+
+    export type Tuple = [type: number, start: number, end: number];
+    export type TRange = [start: number, end: number];
+}
+
+export { };

package/src/index.ts

@@ -0,0 +1,15 @@
+/// <reference path="./global.d.ts" />
+
+import { Registry } from "./registry";
+import { Tokenizer } from "./tokenizer";
+
+export * from "./type";
+
+export { Registry } from "./registry";
+export { Tokenizer } from "./tokenizer";
+export * from "./tools";
+export * from "./tupleList";
+
+
+const tokenizer = new Tokenizer(new Registry());
+export const tokenize = (content: string) => tokenizer.tokenize(content);

package/src/registry.ts

@@ -0,0 +1,174 @@
+import { findFunctionEnd, readWhile } from "./tools";
+import { ModelDefinition } from "./type";
+
+const CSS_NAMED_COLORS = [
+    "aliceblue", "antiquewhite", "aqua", "aquamarine", "azure",
+    "beige", "bisque", "black", "blanchedalmond", "blue", "blueviolet",
+    "brown", "burlywood", "cadetblue", "chartreuse", "chocolate",
+    "coral", "cornflowerblue", "cornsilk", "crimson", "cyan",
+    "darkblue", "darkcyan", "darkgoldenrod", "darkgray", "darkgrey",
+    "darkgreen", "darkkhaki", "darkmagenta", "darkolivegreen",
+    "darkorange", "darkorchid", "darkred", "darksalmon", "darkseagreen",
+    "darkslateblue", "darkslategray", "darkslategrey", "darkturquoise",
+    "darkviolet", "deeppink", "deepskyblue", "dimgray", "dimgrey",
+    "dodgerblue", "firebrick", "floralwhite", "forestgreen", "fuchsia",
+    "gainsboro", "ghostwhite", "gold", "goldenrod", "gray", "grey",
+    "green", "greenyellow", "honeydew", "hotpink", "indianred",
+    "indigo", "ivory", "khaki", "lavender", "lavenderblush",
+    "lawngreen", "lemonchiffon", "lightblue", "lightcoral", "lightcyan",
+    "lightgoldenrodyellow", "lightgray", "lightgrey", "lightgreen",
+    "lightpink", "lightsalmon", "lightseagreen", "lightskyblue",
+    "lightslategray", "lightslategrey", "lightsteelblue", "lightyellow",
+    "lime", "limegreen", "linen", "magenta", "maroon",
+    "mediumaquamarine", "mediumblue", "mediumorchid", "mediumpurple",
+    "mediumseagreen", "mediumslateblue", "mediumspringgreen",
+    "mediumturquoise", "mediumvioletred", "midnightblue", "mintcream",
+    "mistyrose", "moccasin", "navajowhite", "navy", "oldlace",
+    "olive", "olivedrab", "orange", "orangered", "orchid",
+    "palegoldenrod", "palegreen", "paleturquoise", "palevioletred",
+    "papayawhip", "peachpuff", "peru", "pink", "plum", "powderblue",
+    "purple", "rebeccapurple", "red", "rosybrown", "royalblue",
+    "saddlebrown", "salmon", "sandybrown", "seagreen", "seashell",
+    "sienna", "silver", "skyblue", "slateblue", "slategray",
+    "slategrey", "snow", "springgreen", "steelblue", "tan", "teal",
+    "thistle", "tomato", "turquoise", "violet", "wheat", "white",
+    "whitesmoke", "yellow", "yellowgreen", "transparent", "currentcolor",
+];
+
+const COLOR_FUNCTIONS = [
+    /^rgba?\(/i, /^hsla?\(/i, /^hwb\(/i, /^lab\(/i, /^lch\(/i,
+    /^oklab\(/i, /^oklch\(/i, /^color\(/i, /^color-mix\(/i
+];
+
+const HEX_COLOR = /^#(?:[0-9a-f]{3}|[0-9a-f]{4}|[0-9a-f]{6}|[0-9a-f]{8})\b/i;
+
+
+
+export class Registry implements Iterable<ModelDefinition> {
+
+    private items: ModelDefinition[] = [
+        {
+            type: "symbol",
+            kind: "char",
+            regex: /[\s\S]/,
+            priority: 0
+        },
+        {
+            type: "dimension",
+            kind: "keyword",
+            exp: [/^[+-]?[0-9]+[a-z]+/],
+            priority: 110,
+            reader({ index, content, deepReader }) {
+                let end = index;
+                if (content[end] === "-" || content[end] === "+") end++;
+                while (end < content.length && /[0-9]/.test(content[end])) end++;
+
+                if (content[end] === "." && /[0-9]/.test(content[end + 1] ?? "")) {
+                    end++;
+                    while (end < content.length && /[0-9]/.test(content[end])) end++;
+                }
+
+                while (end < content.length && /[a-zA-Z%]/.test(content[end])) end++;
+                return deepReader([index, end]);
+            }
+        },
+        {
+            type: "number",
+            kind: "class",
+            regex: /^[0-9]/,
+            priority: 5,
+            reader({ index, content }) {
+                let nextIndex = index;
+                while (/^[0-9]$/.test(content[nextIndex])) {
+                    nextIndex++;
+                }
+                return [index, nextIndex];
+            }
+        },
+        {
+            type: "word",
+            kind: "class",
+            regex: /^[a-z]/i, // Constrained to single char match
+            priority: 5,
+            reader({ index, content }) {
+                let nextIndex = readWhile(content, index, /[a-z0-9_-]+/);
+                return [index, nextIndex];
+            },
+        },
+        {
+            type: "hash",
+            kind: "class",
+            regex: /^#[a-z0-9]+/i,
+            priority: 50,
+            reader({ index, content }) {
+                let nextIndex = index + 1;
+                while (nextIndex < content.length && /^[a-z0-9]+/i.test(content[nextIndex])) {
+                    nextIndex++
+                }
+                return [index, nextIndex];
+            },
+        },
+        {
+            type: "color",
+            kind: "keyword",
+            priority: 120,
+            exp: [...CSS_NAMED_COLORS, HEX_COLOR, ...COLOR_FUNCTIONS],
+            reader({ index, content, matched, deepReader }) {
+                if (matched instanceof RegExp) {
+                    const match = matched.exec(content.slice(index));
+                    if (!match) return [index, index + 1];
+                    const value = match[0];
+                    if (value.endsWith('(')) {
+                        const end = findFunctionEnd(content, index);
+                        const fnName = value.slice(0, -1);
+                        const innerStart = index + value.length;
+                        const innerEnd = Math.max(innerStart, end);
+                        return deepReader([index, end]);
+                    }
+                }
+                const nextIndex = readWhile(content, index + 1, /[a-zA-Z0-9_-]/);
+                return [index, nextIndex];
+            },
+        }
+    ];
+
+    constructor(models: ModelDefinition[] = []) {
+        this.items.push(...models);
+        this.sort();
+    }
+
+    add(def: ModelDefinition) {
+        this.items.push({ priority: 0, ...def });
+        this.sort();
+    }
+
+    get(index: number) {
+        return this.items[index];
+    }
+
+    get length() {
+        return this.items.length;
+    }
+
+    [Symbol.iterator]() {
+        return this.items[Symbol.iterator]();
+    }
+
+    indexOf(item: ModelDefinition) {
+        return this.items.indexOf(item);
+    }
+
+    sort() {
+        this.items.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
+    }
+
+    sortExps(exps: Exp[]): Exp[] {
+        return [...exps].sort((a, b) => {
+            const aIsRegex = a instanceof RegExp;
+            const bIsRegex = b instanceof RegExp;
+            if (aIsRegex !== bIsRegex) return aIsRegex ? -1 : 1;
+            if (aIsRegex && bIsRegex) return 0;
+            return (b as string).length - (a as string).length;
+        });
+    }
+}

package/src/tokenizer.ts

@@ -0,0 +1,144 @@
+import { Registry } from "./registry";
+import { TupleList } from "./tupleList";
+import { ModelDefinition } from "./type";
+
+export class Tokenizer {
+
+    constructor(public registry: Registry) { }
+
+    getReader(def: ModelDefinition, match?: Exp) {
+        const type = this.registry.indexOf(def);
+        return ((content: string, index: number): Tuple[] => {
+            if (def.kind === "char") {
+                // char only single character so just + 1 for nextIndex
+                return [[type, index, index + 1]];
+            }
+
+            const deepReader = (range: TRange): Tuple[] => {
+                const tuples: Tuple[] = [[type, range[0], range[1]]];
+                const rangeContent = content.slice(range[0], range[1]);
+                const others = this.tokenize(rangeContent, { ignoreTypes: [type] })
+                    .toArray()
+                    .map(([type, start, end]) => [type, start + range[0], end + range[0]] as Tuple);
+
+                tuples.push(...others);
+                return tuples;
+            }
+
+            if (def.kind === "class") {
+                if ('reader' in def && def.reader) {
+                    let ranges = def.reader({ content, index, deepReader });
+                    if (ranges.length > 0 && Array.isArray(ranges[0])) {
+                        return ranges.map((range: any) => range.length === 2 ? [type, ...range] : range) as Tuple[];
+                    }
+                    return [[type, ...(ranges as TRange)]];
+                }
+
+                const start = index;
+                let nextIndex = start + 1;
+
+                // Added length check and optional chaining to prevent infinite loops and crashes
+                while (nextIndex < content.length) {
+                    const nextMatch = this.findMatch(content, nextIndex);
+                    if (nextMatch && type === nextMatch[0]) {
+                        nextIndex++;
+                    } else {
+                        break;
+                    }
+                }
+
+                return [[type, start, nextIndex]];
+            }
+
+            // Keyword fallback check (in case reader is undefined)
+            if (!def.reader) {
+                const length = typeof match === "string" ? match.length : 1;
+                return [[type, index, index + length]];
+            }
+
+            const ranges = def.reader({ content, index, matched: match, deepReader });
+            if (ranges.length > 0 && Array.isArray(ranges[0])) {
+                return ranges.map((range: any) => range.length === 2 ? [type, ...range] : range) as Tuple[];
+            }
+            return [[type, ...(ranges as TRange)]];
+        });
+    }
+
+    findMatch(content: string, index: number, ignoreTypes?: number[]): [number, (c: string, i: number) => Tuple[]] | undefined {
+        for (let i = 0; i < this.registry.length; i++) {
+            // Bypass ignored rules so fallback tokens get a chance
+            if (ignoreTypes && ignoreTypes.includes(i)) {
+                continue;
+            }
+
+            const match = this.tryMatch(this.registry.get(i), content, index);
+            if (match) return [i, match];
+        }
+        return undefined;
+    }
+
+    tryMatch(def: ModelDefinition, content: string, index: number) {
+        // Guard against out-of-bounds indexing
+        if (index >= content.length) return false;
+
+        if (def.kind === "keyword") {
+            const right = content.slice(index);
+            const matched = this.registry.sortExps(def.exp).find(exp =>
+                exp instanceof RegExp ? exp.test(right) : right.startsWith(exp as string)
+            );
+
+            return matched ? this.getReader(def, matched) : false;
+        }
+
+        if (def.kind === "class") {
+            if (!def.regex.test(content.slice(index))) return false;
+            return this.getReader(def);
+        }
+
+        if (def.kind === "char") {
+            if ('regex' in def) {
+                if (!def.regex.test(content.slice(index))) return false;
+                return this.getReader(def);
+            }
+            return def.char === content[index] ? this.getReader(def) : false;
+        }
+        return false;
+    }
+
+    tokenize(content: string, options?: { ignoreTypes?: number[] }) {
+        const tupleList = new TupleList(this.registry);
+        let i = 0;
+
+        while (i < content.length) {
+            let hit = this.findMatch(content, i, options?.ignoreTypes);
+            if (hit) {
+                const results = hit[1](content, i);
+                if (results && results.length > 0) {
+                    const nextIndex = Math.max(i, ...results.map(tuple => tuple[2]));
+                    if (nextIndex <= i) {
+                        tupleList.push([-1, i, i + 1]);
+                        i++;
+                        continue;
+                    }
+
+                    tupleList.push(...results);
+                    i = nextIndex;
+                    continue;
+                }
+            }
+
+            tupleList.push([-1, i, i + 1]);
+            i++;
+        }
+
+        return tupleList;
+    }
+
+    isRange(data: any): data is TRange {
+        return Array.isArray(data) && data.length === 2 && data.every(t => typeof t === "number");
+    }
+
+    isTuple(data: any): data is Tuple {
+        return Array.isArray(data) && data.length === 3 && data.every(t => typeof t === "number");
+    }
+}

package/src/tools.ts

@@ -0,0 +1,37 @@
+export function findFunctionEnd(source: string, index: number): number {
+    let depth = 0;
+    let cursor = index;
+    let inString = false;
+    let stringChar = '';
+
+    while (cursor < source.length) {
+        const char = source[cursor];
+
+        // Handle strings
+        if (!inString && (char === '"' || char === "'")) {
+            inString = true;
+            stringChar = char;
+        } else if (inString && char === stringChar) {
+            inString = false;
+        }
+
+        // Only count parentheses outside of strings
+        if (!inString) {
+            if (char === '(') depth++;
+            else if (char === ')') {
+                depth--;
+                if (depth === 0) return cursor + 1;
+            }
+        }
+
+        cursor++;
+    }
+    return source.length;
+}
+
+
+export const readWhile = (content: string, index: number, test: RegExp) => {
+    let nextIndex = index;
+    while (nextIndex < content.length && test.test(content[nextIndex])) nextIndex++;
+    return nextIndex;
+}

package/src/tupleList.ts

@@ -0,0 +1,73 @@
+import { Registry } from "./registry";
+
+export type TokenTree = {
+    type: string;
+    start: number;
+    end: number
+    value: string;
+    children?: TokenTree[]
+}
+
+export class TupleList {
+    protected items: Tuple[] = [];
+
+    constructor(protected registry: Registry) { }
+
+    push(...items: Tuple[]) {
+        items.forEach(item => {
+            this.items.push(item);
+        });
+    }
+
+    toArray() {
+        return [...this.items];
+    }
+
+    toTokenList(content: string): TokenTree[] {
+        return this.items.map(([type, start, end]) => {
+            const model = this.registry.get(type);
+            return {
+                type: model?.type ?? "unknown",
+                start,
+                end,
+                value: content.slice(start, end),
+            };
+        });
+    }
+
+    toTokenTree(content: string): TokenTree[] {
+        const tokens = this.toTokenList(content);
+        
+        tokens.sort((a, b) => {
+            if (a.start !== b.start) {
+                return a.start - b.start;
+            }
+            return b.end - a.end;
+        });
+
+        const roots: TokenTree[] = [];
+        const stack: TokenTree[] = [];
+
+        for (const token of tokens) {
+            /*
+             * FIXED: Robust parent exit check. 
+             * Pop parents whose physical range strictly ends before or exactly where the new token starts.
+             * (e.g. `parent.end <= token.start`)
+             */
+            while (stack.length > 0 && stack[stack.length - 1].end <= token.start) {
+                stack.pop();
+            }
+            const parent = stack[stack.length - 1];
+
+            if (parent) {
+                if (!parent.children) parent.children = [];
+                parent.children.push(token);
+            } else {
+                roots.push(token);
+            }
+            stack.push(token);
+        }
+
+        return roots;
+    }
+}

package/src/type.d.ts

@@ -0,0 +1,35 @@
+export type ReaderContext = {
+    content: string;
+    index: number;
+    deepReader: (ranges: TRange) => Tuple[]
+}
+
+
+
+export interface CharModel {
+    kind: "char";
+    char: string;
+}
+export interface CharRegexModel {
+    kind: "char";
+    regex: RegExp;
+}
+
+export interface KeywordModel {
+    kind: "keyword";
+    exp: Exp[];
+    reader: (ctx: ReaderContext & { matched: Exp }) => TRange[] | TRange | Tuple[];
+}
+
+export interface ClassModel {
+    kind: "class";
+    regex: RegExp;
+    reader?: (ctx: ReaderContext) => TRange[] | TRange | Tuple[];
+}
+
+export type Model = CharModel | CharRegexModel | KeywordModel | ClassModel;
+
+export type ModelDefinition<T extends Model = Model> = {
+    type: string;
+    priority?: number;
+} & T;

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+    "compilerOptions": {
+        "target": "ES2017",
+        "lib": ["dom", "dom.iterable", "esnext"],
+        "allowJs": true,
+        "skipLibCheck": true,
+        "strict": false,
+        "incremental": true,
+        "module": "esnext",
+        "esModuleInterop": true,
+        "moduleResolution": "bundler",
+        "resolveJsonModule": true,
+        "isolatedModules": true,
+        "jsx": "react-jsx",
+        "paths": {
+            "@/*": ["./src/*"]
+        }
+    },
+    "include": ["**/*.ts", "src/**/*.ts", "src/**/*.tsx", "src/global.d.ts"],
+    "exclude": ["node_modules"]
+}