@il4mb/css-tokenizer 1.0.0 → 1.0.1

package/dist/index.js

@@ -0,0 +1 @@
+var y=["aliceblue","antiquewhite","aqua","aquamarine","azure","beige","bisque","black","blanchedalmond","blue","blueviolet","brown","burlywood","cadetblue","chartreuse","chocolate","coral","cornflowerblue","cornsilk","crimson","cyan","darkblue","darkcyan","darkgoldenrod","darkgray","darkgrey","darkgreen","darkkhaki","darkmagenta","darkolivegreen","darkorange","darkorchid","darkred","darksalmon","darkseagreen","darkslateblue","darkslategray","darkslategrey","darkturquoise","darkviolet","deeppink","deepskyblue","dimgray","dimgrey","dodgerblue","firebrick","floralwhite","forestgreen","fuchsia","gainsboro","ghostwhite","gold","goldenrod","gray","grey","green","greenyellow","honeydew","hotpink","indianred","indigo","ivory","khaki","lavender","lavenderblush","lawngreen","lemonchiffon","lightblue","lightcoral","lightcyan","lightgoldenrodyellow","lightgray","lightgrey","lightgreen","lightpink","lightsalmon","lightseagreen","lightskyblue","lightslategray","lightslategrey","lightsteelblue","lightyellow","lime","limegreen","linen","magenta","maroon","mediumaquamarine","mediumblue","mediumorchid","mediumpurple","mediumseagreen","mediumslateblue","mediumspringgreen","mediumturquoise","mediumvioletred","midnightblue","mintcream","mistyrose","moccasin","navajowhite","navy","oldlace","olive","olivedrab","orange","orangered","orchid","palegoldenrod","palegreen","paleturquoise","palevioletred","papayawhip","peachpuff","peru","pink","plum","powderblue","purple","rebeccapurple","red","rosybrown","royalblue","saddlebrown","salmon","sandybrown","seagreen","seashell","sienna","silver","skyblue","slateblue","slategray","slategrey","snow","springgreen","steelblue","tan","teal","thistle","tomato","turquoise","violet","wheat","white","whitesmoke","yellow","yellowgreen","transparent","currentcolor"],f=[/^rgba?\(/i,/^hsla?\(/i,/^hwb\(/i,/^lab\(/i,/^lch\(/i,/^oklab\(/i,/^oklch\(/i,/^color\(/i,/^color-mix\(/i],k=/^#(?:[0-9a-f]{3}|[0-9a-f]{4}|[0-9a-f]{6}|[0-9a-f]{8})\b/i,b=["px","cm","mm","q","in","pt","pc","em","ex","ch","rem","lh","rlh","cap","ic","vw","vh","vi","vb","vmin","vmax","svw","svh","svi","svb","svmin","svmax","lvw","lvh","lvi","lvb","lvmin","lvmax","dvw","dvh","dvi","dvb","dvmin","dvmax","cqw","cqh","cqi","cqb","cqmin","cqmax","%","deg","grad","rad","turn","s","ms","hz","khz","dpi","dpcm","dppx","x","fr"],p=(e,i)=>{let r=0,t=i,s=!1,n="";while(t<e.length){let a=e[t];if(!s&&(a==='"'||a==="'"))s=!0,n=a;else if(s&&a===n)s=!1;if(!s){if(a==="(")r++;else if(a===")"){if(r--,r===0)return t+1}}t++}return e.length},g=(e,i,r)=>{let t=i;while(t<e.length&&r.test(e[t]))t++;return t};class c{items=[{type:"symbol",kind:"char",regex:/[\s\S]/,priority:0},{type:"escape",kind:"class",regex:/^\\[0-9a-f]{1,6}\s?/i,priority:110,reader({index:e,content:i}){let r=e+1,t=0;while(r<i.length&&/[0-9a-f]/i.test(i[r])&&t<6)r++,t++;if(r<i.length&&/\s/.test(i[r]))r++;return[e,r]}},{type:"unicode-range",kind:"keyword",exp:[/^u\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/i],priority:110,reader({index:e,content:i,matched:r}){let t=r instanceof RegExp?r.exec(i.slice(e)):null;if(!t)return[e,e+1];return[e,e+t[0].length]}},{type:"punctuation",kind:"class",regex:/^[{}()\[\],;:.]/,priority:80,reader({index:e,content:i}){return[e,e+1]}},{type:"operator",kind:"class",regex:/^[+\-*/=<>!~^|&]+/,priority:80,reader({index:e,content:i}){let r=e;while(r<i.length&&/[+\-*/=<>!~^|&]/.test(i[r]))r++;return[e,r]}},{type:"variable",kind:"keyword",exp:[/^--[a-zA-Z0-9-_]+/],priority:110,reader({index:e,content:i,matched:r}){let t=r instanceof RegExp?r.exec(i.slice(e)):null;if(!t)return[e,e+1];return[e,e+t[0].length]}},{type:"comment",kind:"class",regex:/^\/\*/,priority:90,reader({index:e,content:i}){let r=e+2;while(r<i.length&&!(i[r]==="*"&&i[r+1]==="/"))r++;if(r<i.length)r+=2;return[e,r]}},{type:"whitespace",kind:"class",regex:/^\s+/,priority:90,reader({index:e,content:i}){let r=e;while(r<i.length&&/\s/.test(i[r]))r++;return[e,r]}},{type:"url",kind:"class",regex:/^\"https?:\/\/[^\"]+/i,priority:50,reader({index:e,content:i}){let r=e+1;while(r<i.length&&i[r]!=='"')r++;if(r<i.length&&i[r]==='"')r++;return[e,r]}},{type:"string",kind:"class",regex:/^\"[^\"]*\"|^\'[^\']*\'/,priority:20,reader({index:e,content:i}){let r=i[e],t=e+1;while(t<i.length&&i[t]!==r)t++;if(t<i.length&&i[t]===r)t++;return[e,t]}},{type:"atrule",kind:"keyword",exp:[/^@[a-z]+/i],priority:100,reader({index:e,content:i,deepReader:r}){let t=/^@[\w-]+/.exec(i.slice(e));if(!t)return[e,e+1];let s=t[0],n=e+s.length;return r([e,n])}},{type:"function",kind:"keyword",exp:[/^[a-z][a-z0-9-]*\(/i],priority:150,reader({index:e,content:i,deepReader:r}){if(!/^[a-z][a-z0-9-]*\(/i.exec(i.slice(e)))return[e,e+1];let s=p(i,e);return r([e,s])}},{type:"dimension",kind:"keyword",exp:[/^[+-]?[0-9]+[a-z]+/],priority:110,reader({index:e,content:i,deepReader:r}){let t=e;if(i[t]==="-"||i[t]==="+")t++;while(t<i.length&&/[0-9]/.test(i[t]))t++;if(i[t]==="."&&/[0-9]/.test(i[t+1]??"")){t++;while(t<i.length&&/[0-9]/.test(i[t]))t++}while(t<i.length&&/[a-zA-Z%]/.test(i[t]))t++;return r([e,t])}},{type:"number",kind:"class",regex:/^[0-9]/,priority:5,reader({index:e,content:i}){let r=e;while(/^[0-9]$/.test(i[r]))r++;return[e,r]}},{type:"unit",kind:"keyword",exp:b.map((e)=>new RegExp(`^${e}\\b`,"i")),priority:10,reader({index:e,content:i,matched:r}){let t=r instanceof RegExp?r.exec(i.slice(e)):null;if(!t)return[e,e+1];return[e,e+t[0].length]}},{type:"word",kind:"class",regex:/^[a-z]/i,priority:5,reader({index:e,content:i}){let r=g(i,e,/[a-z0-9_-]+/);return[e,r]}},{type:"hash",kind:"class",regex:/^#[a-z0-9]+/i,priority:50,reader({index:e,content:i}){let r=e+1;while(r<i.length&&/^[a-z0-9]+/i.test(i[r]))r++;return[e,r]}},{type:"color",kind:"keyword",priority:120,exp:[...y,k,...f],reader({index:e,content:i,matched:r,deepReader:t}){if(r instanceof RegExp){let n=r.exec(i.slice(e));if(!n)return[e,e+1];let a=n[0];if(a.endsWith("(")){let l=p(i,e);return t([e,l],e+a.length-1)}}let s=g(i,e+1,/[a-zA-Z0-9_-]/);return[e,s]}}];constructor(e=[]){this.items.push(...e),this.sort()}all(){return[...this.items]}add(e){this.items.push({priority:0,...e}),this.sort()}get(e){return this.items[e]}get length(){return this.items.length}[Symbol.iterator](){return this.items[Symbol.iterator]()}indexOf(e){return this.items.indexOf(e)}sort(){this.items.sort((e,i)=>(i.priority??0)-(e.priority??0))}sortExps(e){return[...e].sort((i,r)=>{let t=i instanceof RegExp,s=r instanceof RegExp;if(t!==s)return t?-1:1;if(t&&s)return 0;return r.length-i.length})}}class m{registry;items=[];constructor(e){this.registry=e}push(...e){e.forEach((i)=>{this.items.push(i)})}toArray(){return[...this.items]}toTokenList(e){return this.items.map(([i,r,t])=>({type:this.registry.get(i)?.type??"unknown",start:r,end:t,value:e.slice(r,t)}))}toTokenTree(e){let i=this.toTokenList(e);i.sort((s,n)=>{if(s.start!==n.start)return s.start-n.start;return n.end-s.end});let r=[],t=[];for(let s of i){while(t.length>0&&t[t.length-1].end<=s.start)t.pop();let n=t[t.length-1];if(n){if(!n.children)n.children=[];n.children.push(s)}else r.push(s);t.push(s)}return r}}class d{registry;constructor(e){this.registry=e}getReader(e,i){let r=this.registry.indexOf(e);return(t,s)=>{if(e.kind==="char")return[[r,s,s+1]];let n=(l,u)=>{let o=[[r,l[0],l[1]]],h=t.slice(u??l[0],l[1]),w=this.tokenize(h,{ignoreTypes:[r]}).toArray().map(([x,v,T])=>[x,v+(u??l[0]),T+(u??l[0])]);return o.push(...w),o};if(e.kind==="class"){if("reader"in e&&e.reader){let o=e.reader({content:t,index:s,deepReader:n});if(o.length>0&&Array.isArray(o[0]))return o.map((h)=>h.length===2?[r,...h]:h);return[[r,...o]]}let l=s,u=l+1;while(u<t.length){let o=this.findMatch(t,u);if(o&&r===o[0])u++;else break}return[[r,l,u]]}if(!e.reader){let l=typeof i==="string"?i.length:1;return[[r,s,s+l]]}let a=e.reader({content:t,index:s,matched:i,deepReader:n});if(a.length>0&&Array.isArray(a[0]))return a.map((l)=>l.length===2?[r,...l]:l);return[[r,...a]]}}findMatch(e,i,r){for(let t=0;t<this.registry.length;t++){if(r&&r.includes(t))continue;let s=this.tryMatch(this.registry.get(t),e,i);if(s)return[t,s]}return}tryMatch(e,i,r){if(r>=i.length)return!1;if(e.kind==="keyword"){let t=i.slice(r),s=this.registry.sortExps(e.exp).find((n)=>n instanceof RegExp?n.test(t):t.startsWith(n));return s?this.getReader(e,s):!1}if(e.kind==="class"){if(!e.regex.test(i.slice(r)))return!1;return this.getReader(e)}if(e.kind==="char"){if("regex"in e){if(!e.regex.test(i.slice(r)))return!1;return this.getReader(e)}return e.char===i[r]?this.getReader(e):!1}return!1}tokenize(e,i){let r=new m(this.registry),t=0;while(t<e.length){let s=this.findMatch(e,t,i?.ignoreTypes);if(s){let n=s[1](e,t);if(n&&n.length>0){let a=Math.max(t,...n.map((l)=>l[2]));if(a<=t){r.push([-1,t,t+1]),t++;continue}r.push(...n),t=a;continue}}r.push([-1,t,t+1]),t++}return r}isRange(e){return Array.isArray(e)&&e.length===2&&e.every((i)=>typeof i==="number")}isTuple(e){return Array.isArray(e)&&e.length===3&&e.every((i)=>typeof i==="number")}}var R=new d(new c),A=(e)=>R.tokenize(e);export{f as COLOR_FUNCTIONS,y as CSS_NAMED_COLORS,b as CSS_UNITS,k as HEX_COLOR,c as Registry,d as Tokenizer,m as TupleList,p as findFunctionEnd,g as readWhile,A as tokenizeImpl};

package/dist/types/index.d.ts

@@ -0,0 +1,5 @@
+export { Registry } from "./registry";
+export { Tokenizer } from "./tokenizer";
+export * from "./tools";
+export * from "./tupleList";
+export declare const tokenize: (content: string) => import("./tupleList").TupleList;

package/dist/types/registry.d.ts

@@ -0,0 +1,12 @@
+import { ModelDefinition } from "./type";
+export declare class Registry implements Iterable<ModelDefinition> {
+    private items;
+    constructor(models?: ModelDefinition[]);
+    add(def: ModelDefinition): void;
+    get(index: number): ModelDefinition;
+    get length(): number;
+    [Symbol.iterator](): ArrayIterator<ModelDefinition>;
+    indexOf(item: ModelDefinition): number;
+    sort(): void;
+    sortExps(exps: Exp[]): Exp[];
+}

package/dist/types/tokenizer.d.ts

@@ -0,0 +1,15 @@
+import { Registry } from "./registry";
+import { TupleList } from "./tupleList";
+import { ModelDefinition } from "./type";
+export declare class Tokenizer {
+    registry: Registry;
+    constructor(registry: Registry);
+    getReader(def: ModelDefinition, match?: Exp): (content: string, index: number) => Tuple[];
+    findMatch(content: string, index: number, ignoreTypes?: number[]): [number, (c: string, i: number) => Tuple[]] | undefined;
+    tryMatch(def: ModelDefinition, content: string, index: number): false | ((content: string, index: number) => Tuple[]);
+    tokenize(content: string, options?: {
+        ignoreTypes?: number[];
+    }): TupleList;
+    isRange(data: any): data is TRange;
+    isTuple(data: any): data is Tuple;
+}

package/dist/types/tools.d.ts

@@ -0,0 +1,2 @@
+export declare function findFunctionEnd(source: string, index: number): number;
+export declare const readWhile: (content: string, index: number, test: RegExp) => number;

package/dist/types/tupleList.d.ts

@@ -0,0 +1,17 @@
+import { Registry } from "./registry";
+export type TokenTree = {
+    type: string;
+    start: number;
+    end: number;
+    value: string;
+    children?: TokenTree[];
+};
+export declare class TupleList {
+    protected registry: Registry;
+    protected items: Tuple[];
+    constructor(registry: Registry);
+    push(...items: Tuple[]): void;
+    toArray(): Tuple[];
+    toTokenList(content: string): TokenTree[];
+    toTokenTree(content: string): TokenTree[];
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@il4mb/css-tokenizer",
-    "version": "1.0.0",
+    "version": "1.0.1",
     "description": "Modular, extensible CSS tokenizer for parsing CSS-like strings into tuple lists and token trees.",
     "keywords": [
         "css",
@@ -19,15 +19,21 @@
     "author": "il4mb",
     "type": "module",
     "exports": {
-        ".": "./src/index.ts"
+        "import": "./dist/index.js",
+        "default": "./dist/index.js",
+        "types": "./dist/types/index.d.ts"
     },
-    "main": "index.test.ts",
+    "main": "./dist/index.js",
+    "types": "./dist/types/index.d.ts",
     "scripts": {
-        "test": "vitest index.test.ts"
+        "test": "vitest index.test.ts",
+        "build:types": "npx tsc --build tsconfig.build.json",
+        "build": "bun build ./src/index.ts --outdir ./dist --minify --format esm",
+        "prepublishOnly": "npm run build"
     },
     "devDependencies": {
         "@types/node": "^26.4.1",
-        "typescript": "7.0.2",
+        "typescript": "^7.0.2",
         "vitest": "^5.0.0"
     }
 }

package/readme.md

@@ -13,10 +13,10 @@ npm install @il4mb/css-tokenizer
 ## Quick usage
 
 ```ts
-import { tokenize } from "@il4mb/css-tokenizer";
+import { tokenizeImpl } from "@il4mb/css-tokenizer";
 
 const content = "width: 10px";
-const result = tokenize(content);
+const result = tokenizeImpl(content);
 
 // Tuple list entries are [type, start, end]
 console.log(result.toArray());

package/tsconfig.build.json

@@ -0,0 +1,15 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "declaration": true,
+    "emitDeclarationOnly": true,
+    "declarationMap": false,
+    "declarationDir": "dist/types",
+    "rootDir": "src",
+    "outDir": "dist",
+    "skipLibCheck": true,
+    "isolatedModules": false
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/index.test.ts

@@ -1,24 +0,0 @@
-import { Registry } from "@/registry";
-import { Tokenizer } from "@/tokenizer";
-import { readWhile } from "@/tools";
-
-const registry = new Registry();
-registry.add({
-    type: "var",
-    kind: "class",
-    regex: /^--[a-z][a-z0-9-_]+/i,
-    priority: 10,
-    reader({ index, content }) {
-        let nextIndex = readWhile(content, index + 2, /[a-z0-9-_]/);
-        return [index, nextIndex];
-    },
-});
-
-const tokenizer = new Tokenizer(registry);
-
-
-const content = `--primary`;
-const result = tokenizer.tokenize(content);
-
-console.log(result.toArray()); 
-console.log(JSON.stringify(result.toTokenTree(content), null, 2));

package/src/global.d.ts

@@ -1,20 +0,0 @@
-export declare global {
-
-    export type Exp = string | RegExp;
-
-    export interface TokenPlain {
-        id: string;
-        type: string;
-        value: string;
-        start: number;
-        end: number;
-        number?: number;
-        unit?: string;
-        children?: TokenPlain[]
-    }
-
-    export type Tuple = [type: number, start: number, end: number];
-    export type TRange = [start: number, end: number];
-}
-
-export { };

package/src/index.ts

@@ -1,15 +0,0 @@
-/// <reference path="./global.d.ts" />
-
-import { Registry } from "./registry";
-import { Tokenizer } from "./tokenizer";
-
-export * from "./type";
-
-export { Registry } from "./registry";
-export { Tokenizer } from "./tokenizer";
-export * from "./tools";
-export * from "./tupleList";
-
-
-const tokenizer = new Tokenizer(new Registry());
-export const tokenize = (content: string) => tokenizer.tokenize(content);

package/src/registry.ts

@@ -1,174 +0,0 @@
-import { findFunctionEnd, readWhile } from "./tools";
-import { ModelDefinition } from "./type";
-
-const CSS_NAMED_COLORS = [
-    "aliceblue", "antiquewhite", "aqua", "aquamarine", "azure",
-    "beige", "bisque", "black", "blanchedalmond", "blue", "blueviolet",
-    "brown", "burlywood", "cadetblue", "chartreuse", "chocolate",
-    "coral", "cornflowerblue", "cornsilk", "crimson", "cyan",
-    "darkblue", "darkcyan", "darkgoldenrod", "darkgray", "darkgrey",
-    "darkgreen", "darkkhaki", "darkmagenta", "darkolivegreen",
-    "darkorange", "darkorchid", "darkred", "darksalmon", "darkseagreen",
-    "darkslateblue", "darkslategray", "darkslategrey", "darkturquoise",
-    "darkviolet", "deeppink", "deepskyblue", "dimgray", "dimgrey",
-    "dodgerblue", "firebrick", "floralwhite", "forestgreen", "fuchsia",
-    "gainsboro", "ghostwhite", "gold", "goldenrod", "gray", "grey",
-    "green", "greenyellow", "honeydew", "hotpink", "indianred",
-    "indigo", "ivory", "khaki", "lavender", "lavenderblush",
-    "lawngreen", "lemonchiffon", "lightblue", "lightcoral", "lightcyan",
-    "lightgoldenrodyellow", "lightgray", "lightgrey", "lightgreen",
-    "lightpink", "lightsalmon", "lightseagreen", "lightskyblue",
-    "lightslategray", "lightslategrey", "lightsteelblue", "lightyellow",
-    "lime", "limegreen", "linen", "magenta", "maroon",
-    "mediumaquamarine", "mediumblue", "mediumorchid", "mediumpurple",
-    "mediumseagreen", "mediumslateblue", "mediumspringgreen",
-    "mediumturquoise", "mediumvioletred", "midnightblue", "mintcream",
-    "mistyrose", "moccasin", "navajowhite", "navy", "oldlace",
-    "olive", "olivedrab", "orange", "orangered", "orchid",
-    "palegoldenrod", "palegreen", "paleturquoise", "palevioletred",
-    "papayawhip", "peachpuff", "peru", "pink", "plum", "powderblue",
-    "purple", "rebeccapurple", "red", "rosybrown", "royalblue",
-    "saddlebrown", "salmon", "sandybrown", "seagreen", "seashell",
-    "sienna", "silver", "skyblue", "slateblue", "slategray",
-    "slategrey", "snow", "springgreen", "steelblue", "tan", "teal",
-    "thistle", "tomato", "turquoise", "violet", "wheat", "white",
-    "whitesmoke", "yellow", "yellowgreen", "transparent", "currentcolor",
-];
-
-const COLOR_FUNCTIONS = [
-    /^rgba?\(/i, /^hsla?\(/i, /^hwb\(/i, /^lab\(/i, /^lch\(/i,
-    /^oklab\(/i, /^oklch\(/i, /^color\(/i, /^color-mix\(/i
-];
-
-const HEX_COLOR = /^#(?:[0-9a-f]{3}|[0-9a-f]{4}|[0-9a-f]{6}|[0-9a-f]{8})\b/i;
-
-
-
-export class Registry implements Iterable<ModelDefinition> {
-
-    private items: ModelDefinition[] = [
-        {
-            type: "symbol",
-            kind: "char",
-            regex: /[\s\S]/,
-            priority: 0
-        },
-        {
-            type: "dimension",
-            kind: "keyword",
-            exp: [/^[+-]?[0-9]+[a-z]+/],
-            priority: 110,
-            reader({ index, content, deepReader }) {
-                let end = index;
-                if (content[end] === "-" || content[end] === "+") end++;
-                while (end < content.length && /[0-9]/.test(content[end])) end++;
-
-                if (content[end] === "." && /[0-9]/.test(content[end + 1] ?? "")) {
-                    end++;
-                    while (end < content.length && /[0-9]/.test(content[end])) end++;
-                }
-
-                while (end < content.length && /[a-zA-Z%]/.test(content[end])) end++;
-                return deepReader([index, end]);
-            }
-        },
-        {
-            type: "number",
-            kind: "class",
-            regex: /^[0-9]/,
-            priority: 5,
-            reader({ index, content }) {
-                let nextIndex = index;
-                while (/^[0-9]$/.test(content[nextIndex])) {
-                    nextIndex++;
-                }
-                return [index, nextIndex];
-            }
-        },
-        {
-            type: "word",
-            kind: "class",
-            regex: /^[a-z]/i, // Constrained to single char match
-            priority: 5,
-            reader({ index, content }) {
-                let nextIndex = readWhile(content, index, /[a-z0-9_-]+/);
-                return [index, nextIndex];
-            },
-        },
-        {
-            type: "hash",
-            kind: "class",
-            regex: /^#[a-z0-9]+/i,
-            priority: 50,
-            reader({ index, content }) {
-                let nextIndex = index + 1;
-                while (nextIndex < content.length && /^[a-z0-9]+/i.test(content[nextIndex])) {
-                    nextIndex++
-                }
-                return [index, nextIndex];
-            },
-        },
-        {
-            type: "color",
-            kind: "keyword",
-            priority: 120,
-            exp: [...CSS_NAMED_COLORS, HEX_COLOR, ...COLOR_FUNCTIONS],
-            reader({ index, content, matched, deepReader }) {
-                if (matched instanceof RegExp) {
-                    const match = matched.exec(content.slice(index));
-                    if (!match) return [index, index + 1];
-                    const value = match[0];
-                    if (value.endsWith('(')) {
-                        const end = findFunctionEnd(content, index);
-                        const fnName = value.slice(0, -1);
-                        const innerStart = index + value.length;
-                        const innerEnd = Math.max(innerStart, end);
-                        return deepReader([index, end]);
-                    }
-                }
-                const nextIndex = readWhile(content, index + 1, /[a-zA-Z0-9_-]/);
-                return [index, nextIndex];
-            },
-        }
-    ];
-
-    constructor(models: ModelDefinition[] = []) {
-        this.items.push(...models);
-        this.sort();
-    }
-
-    add(def: ModelDefinition) {
-        this.items.push({ priority: 0, ...def });
-        this.sort();
-    }
-
-    get(index: number) {
-        return this.items[index];
-    }
-
-    get length() {
-        return this.items.length;
-    }
-
-    [Symbol.iterator]() {
-        return this.items[Symbol.iterator]();
-    }
-
-    indexOf(item: ModelDefinition) {
-        return this.items.indexOf(item);
-    }
-
-    sort() {
-        this.items.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
-    }
-
-    sortExps(exps: Exp[]): Exp[] {
-        return [...exps].sort((a, b) => {
-            const aIsRegex = a instanceof RegExp;
-            const bIsRegex = b instanceof RegExp;
-            if (aIsRegex !== bIsRegex) return aIsRegex ? -1 : 1;
-            if (aIsRegex && bIsRegex) return 0;
-            return (b as string).length - (a as string).length;
-        });
-    }
-}

package/src/tokenizer.ts

@@ -1,144 +0,0 @@
-import { Registry } from "./registry";
-import { TupleList } from "./tupleList";
-import { ModelDefinition } from "./type";
-
-export class Tokenizer {
-
-    constructor(public registry: Registry) { }
-
-    getReader(def: ModelDefinition, match?: Exp) {
-        const type = this.registry.indexOf(def);
-        return ((content: string, index: number): Tuple[] => {
-            if (def.kind === "char") {
-                // char only single character so just + 1 for nextIndex
-                return [[type, index, index + 1]];
-            }
-
-            const deepReader = (range: TRange): Tuple[] => {
-                const tuples: Tuple[] = [[type, range[0], range[1]]];
-                const rangeContent = content.slice(range[0], range[1]);
-                const others = this.tokenize(rangeContent, { ignoreTypes: [type] })
-                    .toArray()
-                    .map(([type, start, end]) => [type, start + range[0], end + range[0]] as Tuple);
-
-                tuples.push(...others);
-                return tuples;
-            }
-
-            if (def.kind === "class") {
-                if ('reader' in def && def.reader) {
-                    let ranges = def.reader({ content, index, deepReader });
-                    if (ranges.length > 0 && Array.isArray(ranges[0])) {
-                        return ranges.map((range: any) => range.length === 2 ? [type, ...range] : range) as Tuple[];
-                    }
-                    return [[type, ...(ranges as TRange)]];
-                }
-
-                const start = index;
-                let nextIndex = start + 1;
-
-                // Added length check and optional chaining to prevent infinite loops and crashes
-                while (nextIndex < content.length) {
-                    const nextMatch = this.findMatch(content, nextIndex);
-                    if (nextMatch && type === nextMatch[0]) {
-                        nextIndex++;
-                    } else {
-                        break;
-                    }
-                }
-
-                return [[type, start, nextIndex]];
-            }
-
-            // Keyword fallback check (in case reader is undefined)
-            if (!def.reader) {
-                const length = typeof match === "string" ? match.length : 1;
-                return [[type, index, index + length]];
-            }
-
-            const ranges = def.reader({ content, index, matched: match, deepReader });
-            if (ranges.length > 0 && Array.isArray(ranges[0])) {
-                return ranges.map((range: any) => range.length === 2 ? [type, ...range] : range) as Tuple[];
-            }
-            return [[type, ...(ranges as TRange)]];
-        });
-    }
-
-    findMatch(content: string, index: number, ignoreTypes?: number[]): [number, (c: string, i: number) => Tuple[]] | undefined {
-        for (let i = 0; i < this.registry.length; i++) {
-            // Bypass ignored rules so fallback tokens get a chance
-            if (ignoreTypes && ignoreTypes.includes(i)) {
-                continue;
-            }
-
-            const match = this.tryMatch(this.registry.get(i), content, index);
-            if (match) return [i, match];
-        }
-        return undefined;
-    }
-
-    tryMatch(def: ModelDefinition, content: string, index: number) {
-        // Guard against out-of-bounds indexing
-        if (index >= content.length) return false;
-
-        if (def.kind === "keyword") {
-            const right = content.slice(index);
-            const matched = this.registry.sortExps(def.exp).find(exp =>
-                exp instanceof RegExp ? exp.test(right) : right.startsWith(exp as string)
-            );
-
-            return matched ? this.getReader(def, matched) : false;
-        }
-
-        if (def.kind === "class") {
-            if (!def.regex.test(content.slice(index))) return false;
-            return this.getReader(def);
-        }
-
-        if (def.kind === "char") {
-            if ('regex' in def) {
-                if (!def.regex.test(content.slice(index))) return false;
-                return this.getReader(def);
-            }
-            return def.char === content[index] ? this.getReader(def) : false;
-        }
-        return false;
-    }
-
-    tokenize(content: string, options?: { ignoreTypes?: number[] }) {
-        const tupleList = new TupleList(this.registry);
-        let i = 0;
-
-        while (i < content.length) {
-            let hit = this.findMatch(content, i, options?.ignoreTypes);
-            if (hit) {
-                const results = hit[1](content, i);
-                if (results && results.length > 0) {
-                    const nextIndex = Math.max(i, ...results.map(tuple => tuple[2]));
-                    if (nextIndex <= i) {
-                        tupleList.push([-1, i, i + 1]);
-                        i++;
-                        continue;
-                    }
-
-                    tupleList.push(...results);
-                    i = nextIndex;
-                    continue;
-                }
-            }
-
-            tupleList.push([-1, i, i + 1]);
-            i++;
-        }
-
-        return tupleList;
-    }
-
-    isRange(data: any): data is TRange {
-        return Array.isArray(data) && data.length === 2 && data.every(t => typeof t === "number");
-    }
-
-    isTuple(data: any): data is Tuple {
-        return Array.isArray(data) && data.length === 3 && data.every(t => typeof t === "number");
-    }
-}

package/src/tools.ts

@@ -1,37 +0,0 @@
-export function findFunctionEnd(source: string, index: number): number {
-    let depth = 0;
-    let cursor = index;
-    let inString = false;
-    let stringChar = '';
-
-    while (cursor < source.length) {
-        const char = source[cursor];
-
-        // Handle strings
-        if (!inString && (char === '"' || char === "'")) {
-            inString = true;
-            stringChar = char;
-        } else if (inString && char === stringChar) {
-            inString = false;
-        }
-
-        // Only count parentheses outside of strings
-        if (!inString) {
-            if (char === '(') depth++;
-            else if (char === ')') {
-                depth--;
-                if (depth === 0) return cursor + 1;
-            }
-        }
-
-        cursor++;
-    }
-    return source.length;
-}
-
-
-export const readWhile = (content: string, index: number, test: RegExp) => {
-    let nextIndex = index;
-    while (nextIndex < content.length && test.test(content[nextIndex])) nextIndex++;
-    return nextIndex;
-}

package/src/tupleList.ts

@@ -1,73 +0,0 @@
-import { Registry } from "./registry";
-
-export type TokenTree = {
-    type: string;
-    start: number;
-    end: number
-    value: string;
-    children?: TokenTree[]
-}
-
-export class TupleList {
-    protected items: Tuple[] = [];
-
-    constructor(protected registry: Registry) { }
-
-    push(...items: Tuple[]) {
-        items.forEach(item => {
-            this.items.push(item);
-        });
-    }
-
-    toArray() {
-        return [...this.items];
-    }
-
-    toTokenList(content: string): TokenTree[] {
-        return this.items.map(([type, start, end]) => {
-            const model = this.registry.get(type);
-            return {
-                type: model?.type ?? "unknown",
-                start,
-                end,
-                value: content.slice(start, end),
-            };
-        });
-    }
-
-    toTokenTree(content: string): TokenTree[] {
-        const tokens = this.toTokenList(content);
-        
-        tokens.sort((a, b) => {
-            if (a.start !== b.start) {
-                return a.start - b.start;
-            }
-            return b.end - a.end;
-        });
-
-        const roots: TokenTree[] = [];
-        const stack: TokenTree[] = [];
-
-        for (const token of tokens) {
-            /*
-             * FIXED: Robust parent exit check. 
-             * Pop parents whose physical range strictly ends before or exactly where the new token starts.
-             * (e.g. `parent.end <= token.start`)
-             */
-            while (stack.length > 0 && stack[stack.length - 1].end <= token.start) {
-                stack.pop();
-            }
-            const parent = stack[stack.length - 1];
-
-            if (parent) {
-                if (!parent.children) parent.children = [];
-                parent.children.push(token);
-            } else {
-                roots.push(token);
-            }
-            stack.push(token);
-        }
-
-        return roots;
-    }
-}

package/src/type.d.ts

@@ -1,35 +0,0 @@
-export type ReaderContext = {
-    content: string;
-    index: number;
-    deepReader: (ranges: TRange) => Tuple[]
-}
-
-
-
-export interface CharModel {
-    kind: "char";
-    char: string;
-}
-export interface CharRegexModel {
-    kind: "char";
-    regex: RegExp;
-}
-
-export interface KeywordModel {
-    kind: "keyword";
-    exp: Exp[];
-    reader: (ctx: ReaderContext & { matched: Exp }) => TRange[] | TRange | Tuple[];
-}
-
-export interface ClassModel {
-    kind: "class";
-    regex: RegExp;
-    reader?: (ctx: ReaderContext) => TRange[] | TRange | Tuple[];
-}
-
-export type Model = CharModel | CharRegexModel | KeywordModel | ClassModel;
-
-export type ModelDefinition<T extends Model = Model> = {
-    type: string;
-    priority?: number;
-} & T;