grammar-well 1.3.2 → 2.0.0

package/build/utility/text-format.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-format.js","sourceRoot":"","sources":["../../src/utility/text-format.ts"],"names":[],"mappings":";;;AAGA,MAAa,aAAa;IAEtB,MAAM,CAAC,eAAe,CAAC,KAAkB,EAAE,QAA8C;QACrF,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,MAAM,YAAY,GAAG,aAAa,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,oBAAoB,GAAG,YAAY,GAAG,YAAY,GAAG,KAAK,CAAC,IAAI,GAAG,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QACzG,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;YACvB,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;SAC5C;aAAM;YACH,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxB,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE;gBACvB,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;gBACxC,MAAM,aAAa,GAAG,aAAa,CAAC,iBAAiB,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBAC/E,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC,CAAC;gBACzC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;aAC9D;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC3B;IACL,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,KAAiB;QACpC,IAAI,KAAK,CAAC,IAAI;YACV,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5D,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,KAAkB;QACrC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,IAAI,KAAK,CAAC;QACV,IAAI,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACjC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;QAGvC,OAAO,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE;YAC5B,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI,EAAE;gBACrB,KAAK,EAAE,CAAC;gBACR,MAAM,GAAG,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;aAC9C;YACD,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;YAC9B,IAAI,KAAK,IAAI,CAAC;gBACV,MAAM;SACb;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;QAE/C,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChC,MAAM,IAAI,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,OAAO,MAAM,IAAI,WAAW,IAAI,OAAO,IAAI,IAAI,WAAW;YAC1D,OAAO,wBAAwB,IAAI,GAAG,CAAC,QAAQ,MAAM,GAAG,CAAC,QAAQ,MAAM,IAAI,CAAC;QAChF,OAAO,yBAAyB,MAAM,QAAQ,MAAM,IAAI,CAAC;IAC7D,CAAC;IAED,MAAM,CAAC,iBAAiB,CAAC,MAAyB,EAAE,KAAe,EAAE,KAAe;QAChF,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;YAC5B,OAAO,MAAM,CAAC;SACjB;aAAM,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE;YACrC,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,kBAAkB,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;SACnF;aAAM;YACH,IAAI,SAAS,IAAI,MAAM,EAAE;gBACrB,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aACzC;iBAAM,IAAI,MAAM,YAAY,MAAM,EAAE;gBACjC,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,sBAAsB,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;aAChF;iBAAM,IAAI,OAAO,IAAI,MAAM,EAAE;gBAC1B,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,QAAQ,CAAC;aAC/D;iBAAM,IAAI,KAAK,EAAE;gBACd,OAAO,uBAAuB,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;aAC3D;SACJ;IACL,CAAC;IAED,MAAM,CAAC,WAAW,CAAC,IAAiB,EAAE,YAAqB;QACvD,IAAI,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5H,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;YACrC,cAAc,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACjI;QACD,OAAO,IAAI,CAAC,IAAI,GAAG,KAAK,GAAG,cAAc,CAAC;IAC9C,CAAC;CACJ;AAhFD,sCAgFC"}
+{"version":3,"file":"text-format.js","sourceRoot":"","sources":["../../src/utility/text-format.ts"],"names":[],"mappings":"AAGA,MAAM,OAAO,aAAa;IAEtB,MAAM,CAAC,eAAe,CAAC,KAAkB,EAAE,QAA+D;QACtG,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,MAAM,YAAY,GAAG,aAAa,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,oBAAoB,GAAG,YAAY,GAAG,YAAY,GAAG,KAAK,CAAC,IAAI,GAAG,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QACzG,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC7C,CAAC;aAAM,CAAC;YACJ,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxB,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;gBACxB,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;gBACxC,MAAM,aAAa,GAAG,aAAa,CAAC,iBAAiB,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBAC/E,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC,CAAC;gBACzC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/D,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,KAAwB;QAC3C,IAAI,KAAK,CAAC,IAAI;YACV,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5D,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,KAAkB;QACrC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,IAAI,KAAK,CAAC;QACV,IAAI,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACjC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;QAGvC,OAAO,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACtB,KAAK,EAAE,CAAC;gBACR,MAAM,GAAG,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YAC/C,CAAC;YACD,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;YAC9B,IAAI,KAAK,IAAI,CAAC;gBACV,MAAM;QACd,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;QAE/C,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChC,MAAM,IAAI,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,OAAO,MAAM,IAAI,WAAW,IAAI,OAAO,IAAI,IAAI,WAAW;YAC1D,OAAO,wBAAwB,IAAI,GAAG,CAAC,QAAQ,MAAM,GAAG,CAAC,QAAQ,MAAM,IAAI,CAAC;QAChF,OAAO,yBAAyB,MAAM,QAAQ,MAAM,IAAI,CAAC;IAC7D,CAAC;IAED,MAAM,CAAC,iBAAiB,CAAC,MAAgC,EAAE,KAAe,EAAE,KAAe;QACvF,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,MAAM,CAAC;QAClB,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,kBAAkB,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QACpF,CAAC;aAAM,CAAC;YACJ,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;gBACtB,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;iBAAM,IAAI,MAAM,YAAY,MAAM,EAAE,CAAC;gBAClC,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,sBAAsB,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;YACjF,CAAC;iBAAM,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;gBAC3B,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,QAAQ,CAAC;YACjE,CAAC;iBAAM,IAAI,KAAK,EAAE,CAAC;gBACf,OAAO,uBAAuB,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC5D,CAAC;QACL,CAAC;IACL,CAAC;IAED,MAAM,CAAC,WAAW,CAAC,IAAkC,EAAE,YAAqB;QACxE,IAAI,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5H,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE,CAAC;YACtC,cAAc,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClI,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,GAAG,KAAK,GAAG,cAAc,CAAC;IAC9C,CAAC;CACJ"}

package/package.json

@@ -1,22 +1,28 @@
 {
   "name": "grammar-well",
-  "version": "1.3.2",
+  "type": "module",
+  "version": "2.0.0",
   "description": "Cross-platform Parser written in TypeScript",
   "scripts": {
-    "prebuild": "tsc --build --clean",
+    "prebuild": "tsc --build --clean && tsx scripts/prebuild.ts",
+    "grammar": "tsc --build --clean && tsx scripts/grammar.ts",
     "build": "tsc --build",
-    "bootstrap": "tsx bootstrap.ts",
-    "testing": "tsx testing.ts",
+    "builde": "tsc --build",
     "benchmark": "node tests/performance/benchmark.js",
-    "test": "nyc mocha tests/**/*.spec.ts",
+    "test": "node --test --import tsx --test-reporter ./tests/reporter.mjs  tests/**/*.spec.ts",
     "profile": "node tests/performance/profile.js"
   },
   "exports": {
     ".": "./build/index.js",
+    "./parse": "./build/parser/parse.js",
+    "./algorithms/*": "./build/parser/algorithms/*.js",
     "./parser": "./build/parser/parser.js",
-    "./compiler": "./build/compiler/index.js",
+    "./generator": "./build/generator/index.js",
+    "./import-resolvers/*": "./build/generator/import-resolvers/*.js",
+    "./grammars": "./build/generator/grammars/index.js",
     "./lexer": "./build/lexers/stateful-lexer.js",
-    "./utility": "./build/utility/index.js"
+    "./utility": "./build/utility/index.js",
+    "./typings": "./build/typings/index.js"
   },
   "author": "0x6563",
   "license": "LGPL-3.0",
@@ -29,23 +35,10 @@
   },
   "homepage": "https://github.com/0x6563/grammar-well#readme",
   "devDependencies": {
-    "@types/chai": "^4.3.4",
-    "@types/mocha": "^10.0.1",
-    "@types/node": "^18.15.0",
-    "@typescript-eslint/eslint-plugin": "^5.54.1",
-    "@typescript-eslint/parser": "^5.54.1",
-    "babel-cli": "^6.26.0",
-    "babel-preset-env": "^1.7.0",
-    "benny": "^3.7.1",
-    "chai": "^4.3.7",
-    "eslint": "^8.36.0",
-    "expect": "^29.5.0",
-    "mocha": "^10.2.0",
-    "nyc": "^15.1.0",
-    "ts-node": "^10.9.1",
-    "tsx": "^3.12.3",
-    "typescript": "^4.9.5",
-    "yaml": "^2.2.1"
+    "@types/node": "^22.10.1",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.2",
+    "yaml": "^2.6.1"
   },
   "keywords": [
     "parser",

package/src/generator/artifacts/basic.ts

@@ -0,0 +1,26 @@
+import { CommonGenerator } from "../stringify/common.js";
+import { JavaScriptGenerator } from "../stringify/javascript.js";
+
+export class BasicGrammarTable {
+
+    constructor(private generator: JavaScriptGenerator) { }
+
+    stringify(depth: number = 0) {
+        if (!this.generator.state.grammar) {
+            return null;
+        }
+        return CommonGenerator.JSON({
+            start: JSON.stringify(this.generator.state.grammar.start),
+            rules: this.stringifyGrammarRules(depth + 1)
+        }, depth);
+    }
+
+    private stringifyGrammarRules(depth: number = 0) {
+        const map = {};
+        for (const rule in this.generator.state.grammar.rules) {
+            map[rule] = this.generator.state.grammar.rules[rule].map(v => this.generator.grammarRule(v))
+        }
+        return CommonGenerator.JSON(map, depth);
+    }
+
+}

package/src/generator/artifacts/lexer.ts

@@ -0,0 +1,228 @@
+import { ASTGrammarSymbolRegex, ASTLexerStateImportRule, ASTLexerStateMatchRule, ASTLexerStateNonMatchRule, Dictionary, GeneratorLexerConfig, GeneratorLexerState } from "../../typings/index.js";
+import { CommonGenerator } from "../stringify/common.js";
+
+export class LexerArtifact {
+    private resolved = new Set<string>();
+    private resolving = new Set<string>();
+
+    constructor(private lexer: GeneratorLexerConfig) { }
+
+    output(depth: number = 0) {
+        if (!this.lexer)
+            return null;
+
+        if (typeof this.lexer === 'string')
+            return this.lexer;
+
+        return CommonGenerator.JSON({
+            start: JSON.stringify(this.lexer.start),
+            states: this.lexerConfigStates(this.lexer.states, depth + 1)
+        }, depth);
+    }
+
+    private lexerConfigStates(states: Dictionary<GeneratorLexerState>, depth: number) {
+        this.resolveStates(this.lexer.start);
+        const map = {};
+        for (const key in states) {
+            const state = states[key];
+            map[key] = CommonGenerator.JSON({
+                unmatched: state.unmatched ? this.lexerConfigStateRule(state.unmatched) : null,
+                rules: this.lexerConfigStateRules(state.rules, depth + 2),
+                regex: CompileRegExp(key, state as unknown as any)
+            }, depth + 1);
+        }
+
+        return CommonGenerator.JSON(map, depth);
+    }
+
+    private lexerConfigStateRules(rules: (ASTLexerStateImportRule | ASTLexerStateMatchRule)[], depth: number) {
+        const ary = rules.map(rule => {
+            if ('import' in rule)
+                return CommonGenerator.JSON({ import: JSON.stringify(rule.import) }, -1)
+            return this.lexerConfigStateRule(rule as ASTLexerStateMatchRule)
+        });
+        return CommonGenerator.JSON(ary, depth);
+    }
+
+    private lexerConfigStateRule(rule: ASTLexerStateMatchRule | ASTLexerStateNonMatchRule) {
+        return CommonGenerator.JSON({
+            when: 'when' in rule ? CommonGenerator.SerializeSymbol(rule.when as any) : null,
+            before: JSON.stringify(rule.before),
+            skip: JSON.stringify(rule.skip),
+
+            type: JSON.stringify(rule.type),
+            tag: JSON.stringify(rule.tag),
+            open: JSON.stringify(rule.open),
+            close: JSON.stringify(rule.close),
+            highlight: JSON.stringify(rule.highlight),
+
+            pop: JSON.stringify(rule.pop),
+            set: JSON.stringify(rule.set),
+            inset: JSON.stringify(rule.inset),
+            goto: JSON.stringify(rule.goto),
+        }, -1);
+    }
+
+    private resolveStates(start: string) {
+        this.resolved = new Set<string>();
+        this.resolving = new Set<string>();
+        this.resolveRuleImports(start, new Set());
+        for (const key in this.lexer.states) {
+            if (!this.resolved.has(key)) {
+                delete this.lexer.states[key];
+            }
+        }
+    }
+
+    private resolveRuleImports(name: string, chain: Set<string>) {
+        if (chain.has(name))
+            throw new Error(`Can not resolve circular import of ${name}`);
+        if (!this.lexer.states[name])
+            throw new Error(`Can not import unknown state ${name}`);
+        if (this.resolved.has(name) || this.resolving.has(name))
+            return;
+        const state = this.lexer.states[name];
+        const rules = new UniqueRules();
+        chain.add(name);
+        this.resolving.add(name);
+        for (let i = 0; i < state.rules.length; i++) {
+            const rule = state.rules[i];
+            if ("import" in rule) {
+                for (const ref of rule.import) {
+                    this.resolveRuleImports(ref, chain);
+                    rules.push(...this.lexer.states[ref].rules as ASTLexerStateMatchRule[]);
+                }
+            } else {
+                rules.push(rule);
+                if (rule.set && !this.resolving.has(rule.set)) {
+                    this.resolveRuleImports(rule.set, new Set());
+                }
+                if (rule.goto && !this.resolving.has(rule.goto)) {
+                    this.resolveRuleImports(rule.goto, new Set());
+                }
+            }
+        }
+        state.rules = rules.rules;
+        chain.delete(name);
+        this.resolved.add(name);
+    }
+}
+
+class UniqueRules {
+    private regexps = new Set<string>();
+    private strings = new Set<string>();
+    rules: ASTLexerStateMatchRule[] = [];
+
+    push(...rules: (ASTLexerStateMatchRule)[]) {
+        for (const rule of rules) {
+            if (typeof rule.when === 'object') {
+                if (!this.regexps.has(rule.when.regex)) {
+                    this.regexps.add(rule.when.regex);
+                    this.rules.push(rule);
+                }
+            } else {
+                if (!this.strings.has(rule.when as string)) {
+                    this.strings.add(rule.when);
+                    this.rules.push(rule);
+                }
+            }
+        }
+    }
+}
+
+
+function CompileRegExp(name: string, state: { unmatched?: ASTLexerStateNonMatchRule, rules: ASTLexerStateMatchRule[] }): string {
+    const rules = [];
+    const subexpressions = [];
+
+    let isUnicode = null;
+    let isCI = null;
+    for (const options of state.rules) {
+        if (RegexLib.IsRegex(options.when)) {
+            const { flags } = options.when as ASTGrammarSymbolRegex;
+            const u = flags.includes('u');
+            const i = flags.includes('i');
+
+            if (isUnicode === null) {
+                isUnicode = u;
+            } else if (isUnicode !== u && !state.unmatched) {
+                throw new Error(`Inconsistent Regex Flag /u in state: ${name}`);
+            }
+
+            if (isCI === null) {
+                isCI = i;
+            } else if (isCI !== i) {
+                throw new Error(`Inconsistent Regex Flag /i in state: ${name}`);
+            }
+
+        } else {
+            if (!isCI) {
+                isCI = false;
+            } else if (isCI) {
+                throw new Error(`Inconsistent Regex Flag /i in state: ${name}`);
+            }
+        }
+
+        rules.push(options);
+        const pat = RegexLib.Source(options.when);
+        const regexp = new RegExp(pat)
+        if (regexp.test("")) {
+            throw new Error("RegExp matches empty string: " + regexp);
+        }
+
+        if (RegexLib.HasGroups(pat)) {
+            throw new Error("RegExp has capture groups: " + regexp + "\nUse (?: … ) instead");
+        }
+
+        subexpressions.push(RegexLib.Capture(pat))
+    }
+
+    let flags = !state.unmatched ? 'ym' : 'gm';
+    if (isUnicode === true)
+        flags += "u"
+    if (isCI === true)
+        flags += "i"
+    return `/${new RegExp(RegexLib.Join(subexpressions)).source}/${flags}`;
+}
+
+
+class RegexLib {
+
+    static IsRegex(o: any) {
+        return typeof o == 'object' && 'regex' in o;
+    }
+
+    static Escape(s: string) {
+        return s.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+    }
+
+    static HasGroups(s: string) {
+        return (new RegExp('|' + s)).exec('').length > 1;
+    }
+
+    static Capture(source: string) {
+        return '(' + source + ')';
+    }
+
+    static Join(regexps: string[]) {
+        if (!regexps.length)
+            return '(?!)';
+        const source = regexps.map((s) => `(?:${s})`).join('|');
+        return `(?:${source})`;
+    }
+
+    static Source(search: string | ASTGrammarSymbolRegex) {
+        if (typeof search === 'string') {
+            return `(?:${RegexLib.Escape(search)})`;
+        }
+        try {
+            if (RegexLib.IsRegex(search)) {
+                return search.regex;
+            }
+            throw new Error('Not a pattern: ' + search)
+        } catch (error) {
+            throw error;
+        }
+    }
+
+}

package/src/{compiler/generator → generator}/artifacts/lr.ts

@@ -1,14 +1,15 @@
-import { Generator } from "../generator";
-import { Dictionary, GeneratorGrammarRule, GeneratorGrammarSymbol, GrammarTypeRule } from "../../../typings";
-import { Collection, GeneratorSymbolCollection } from "../../../utility/general";
+import { ASTGrammarSymbolNonTerminal, Dictionary, GeneratorGrammarProductionRule, GeneratorGrammarSymbol } from "../../typings/index.js";
+import { Collection, GeneratorSymbolCollection } from "../../utility/general.js";
+import { CommonGenerator } from "../stringify/common.js";
+import { JavaScriptGenerator } from "../stringify/javascript.js";
 
 
 export class LRParseTableBuilder {
-    rules: Collection<GeneratorGrammarRule> = new Collection();
+    rules: Collection<GeneratorGrammarProductionRule> = new Collection();
     table: Dictionary<StateBuilder> = Object.create(null)
     symbols: GeneratorSymbolCollection = new GeneratorSymbolCollection();
 
-    constructor(public generator: Generator) {
+    constructor(public generator: JavaScriptGenerator) {
         const augmented = { name: Symbol() as unknown as string, symbols: [{ rule: generator.state.grammar.start }] }
         generator.state.grammar.rules[augmented.name] = [augmented];
         this.addState([{ rule: augmented, dot: 0 }]);
@@ -27,7 +28,7 @@ export class LRParseTableBuilder {
         state.queue = {};
     }
 
-    encodeRule(rule: GeneratorGrammarRule, dot: number) {
+    encodeRule(rule: GeneratorGrammarProductionRule, dot: number) {
         return this.rules.encode(rule) + '.' + dot;
     }
 
@@ -35,26 +36,26 @@ export class LRParseTableBuilder {
         return Array.from(new Set(seed)).map(v => this.encodeRule(v.rule, v.dot)).sort().join()
     }
 
-    serialize(depth: number = 0) {
+    stringify(depth: number = 0) {
         const map = {};
         for (const key in this.table) {
-            map[key] = this.serializeState(this.table[key].serialize(), depth + 1);
+            map[key] = this.stringifyState(this.table[key].export(), depth + 1);
         }
 
-        return Generator.Pretty(map, depth)
+        return CommonGenerator.JSON(map, depth)
     }
 
-    serializeState(state: State, depth: number = 0) {
-        return Generator.Pretty({
-            actions: state.actions.map(v => this.serializeNext(v, depth + 1)),
-            goto: Generator.Pretty(state.goto, depth + 1),
-            reduce: state.reduce ? this.generator.serializeGrammarRule(state.reduce) : null,
+    stringifyState(state: State, depth: number = 0) {
+        return CommonGenerator.JSON({
+            actions: state.actions.map(v => this.stringifyNext(v, depth + 1)),
+            goto: CommonGenerator.JSON(state.goto, depth + 1),
+            reduce: state.reduce ? this.generator.grammarRule(state.reduce) : null,
             isFinal: state.isFinal ? 'true' : 'false'
         }, depth);
     }
-    serializeNext(next: Next, depth: number) {
-        return Generator.Pretty({
-            symbol: Generator.SerializeSymbol(next.symbol),
+    stringifyNext(next: Next, depth: number) {
+        return CommonGenerator.JSON({
+            symbol: CommonGenerator.SerializeSymbol(next.symbol),
             next: JSON.stringify(next.next)
         }, -1);
     }
@@ -71,7 +72,7 @@ class StateBuilder {
     queue: { [key: string]: StateItem[] } = {};
     actions: Map<GeneratorGrammarSymbol, string> = new Map();
     goto: Map<GeneratorGrammarSymbol, string> = new Map();
-    reduce?: GeneratorGrammarRule;
+    reduce?: GeneratorGrammarProductionRule;
 
     constructor(private collection: LRParseTableBuilder, items: StateItem[]) {
         const visited = new Set<GeneratorGrammarSymbol>();
@@ -98,7 +99,7 @@ class StateBuilder {
         }
     }
 
-    private closure(rule: GeneratorGrammarRule, dot: number, visited: Set<GeneratorGrammarSymbol>) {
+    private closure(rule: GeneratorGrammarProductionRule, dot: number, visited: Set<GeneratorGrammarSymbol>) {
         const isFinal = rule.symbols.length == dot;
         this.isFinal = isFinal || this.isFinal;
         const symbol = rule.symbols[dot];
@@ -109,13 +110,13 @@ class StateBuilder {
         visited.add(symbol);
         const stateItem = { rule, dot: dot + 1 };
 
-        if (Generator.SymbolIsTerminal(symbol)) {
+        if (CommonGenerator.SymbolIsTerminal(symbol)) {
             const id = this.collection.symbols.encode(symbol);
             this.outputs.action[id] = this.outputs.action[id] || [];
             this.outputs.action[id].push(stateItem);
         } else {
             const id = this.collection.symbols.encode(symbol);
-            const name = typeof symbol === 'string' ? symbol : (symbol as GrammarTypeRule).rule;
+            const name = typeof symbol === 'string' ? symbol : (symbol as ASTGrammarSymbolNonTerminal).rule;
             this.outputs.goto[id] = this.outputs.goto[id] || [];
             this.outputs.goto[id].push(stateItem);
             for (const rule of this.collection.generator.state.grammar.rules[name]) {
@@ -124,7 +125,7 @@ class StateBuilder {
         }
     }
 
-    serialize(): State {
+    export(): State {
         const actions: Next[] = [];
         const goto: { [key: string]: string } = {}
         for (const [symbol, next] of this.actions) {
@@ -141,13 +142,13 @@ class StateBuilder {
 interface State {
     actions: Next[];
     goto: { [key: string]: string };
-    reduce: GeneratorGrammarRule;
+    reduce: GeneratorGrammarProductionRule;
     isFinal: boolean;
 }
 
 type Next = { symbol: GeneratorGrammarSymbol, next: string };
 
-type StateItem = { rule: GeneratorGrammarRule, dot: number };
+type StateItem = { rule: GeneratorGrammarProductionRule, dot: number };
 
 interface StateOut {
     action: Dictionary<StateItem[]>;

package/src/generator/builtin/character.well

@@ -0,0 +1,7 @@
+lexer {
+	start: "character"
+
+	[character]
+		- when r:{.}
+
+}

package/src/generator/builtin/json.well

@@ -0,0 +1,85 @@
+import * from whitespace;
+
+lexer {
+	start: "json"
+
+	[json]
+		- import whitespace
+		- when r:{-?(?:[0-9]|[1-9][0-9]+)(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+)?\b} tag "number"
+		- when r:{"(?:\\["bfnrt/\\]|\\u[a-fA-F0-9]{4}|[^"\\])*"} tag "string"
+		- when "{" tag "{"
+		- when "}" tag "}"
+		- when "[" tag "["
+		- when "]" tag "]"
+		- when "," tag ","
+		- when ":" tag ":"
+		- when "true" tag "true"
+		- when "false" tag "false"
+		- when "null" tag "null"
+
+}
+
+grammar {
+	start: "json"
+
+	[json]
+		| _ (object | array) _ => ( $1[0] )
+
+	[object]
+		| "{" _ "}" => ( {} )
+		| "{" _ pair (_ "," _ pair)* _ "}" => ${ extractObject }
+
+	[array]
+		| "[" _ "]" => ( [] )
+		| "[" _ value (_ "," _ value)* _ "]" => ${ extractArray }
+
+	[value]
+		| object => ( $0 )
+		| array => ( $0 )
+		| number => ( $0 )
+		| string => ( $0 )
+		| "true" => ( true )
+		| "false" => ( false )
+		| "null" => ( null )
+
+	[number]
+		| <number> => ( parseFloat($0.value) )
+
+	[string]
+		| <string> => ( JSON.parse($0.value) )
+
+	[pair]
+		| key@k _ ":" _ value@v => ( [$k, $v] )
+
+	[key]
+		| string => ( $0 )
+
+}
+
+on:import {
+function extractPair(kv, output) {
+        if(kv[0]) { output[kv[0]] = kv[1]; }
+    }
+
+    function extractObject({data}) {
+        let output = {};
+
+        extractPair(data[2], output);
+
+        for (let i in data[3]) {
+            extractPair(data[3][i][3], output);
+        }
+
+        return output;
+    }
+
+    function extractArray({data}) {
+        let output = [data[2]];
+
+        for (let i in data[3]) {
+            output.push(data[3][i][3]);
+        }
+
+        return output;
+    }
+}

package/src/generator/builtin/number.well

@@ -0,0 +1,21 @@
+grammar {
+
+	[unsigned_int]
+		| r:{[0-9]}+ => ( parseInt($0.join("")) )
+
+	[int]
+		| ("-" | "+")? r:{[0-9]}+ => ( $0 ? parseInt($0[0]+$1.join("")) : parseInt($1.join("")) )
+
+	[unsigned_decimal]
+		| r:{[0-9]}+ ("." r:{[0-9]}+)? => ( parseFloat($0.join("") + ($1 ? "."+$1[1].join("") : "")) )
+
+	[decimal]
+		| "-"? r:{[0-9]}+ ("." r:{[0-9]}+)? => ( parseFloat( ($0 || "") + $1.join("") +($2 ? "."+$2[1].join("") : "")) )
+
+	[percentage]
+		| decimal "%" => ( $0/100 )
+
+	[jsonfloat]
+		| "-"? r:{[0-9]}+ ("." r:{[0-9]}+)? (r:{[eE]} r:{[+-]}? r:{[0-9]}+)? => ( parseFloat( ($0 || "") + $1.join("") + ($2 ? "."+$2[1].join("") : "") + ($3 ? "e" + ($3[1] || "+") + $3[2].join("") : "")) )
+
+}

package/src/generator/builtin/registry.json

@@ -0,0 +1 @@
+{"character":"lexer {\n\tstart: \"character\"\n\n\t[character]\n\t\t- when r:{.}\n\n}\n","json":"import * from whitespace;\n\nlexer {\n\tstart: \"json\"\n\n\t[json]\n\t\t- import whitespace\n\t\t- when r:{-?(?:[0-9]|[1-9][0-9]+)(?:\\.[0-9]+)?(?:[eE][-+]?[0-9]+)?\\b} tag \"number\"\n\t\t- when r:{\"(?:\\\\[\"bfnrt/\\\\]|\\\\u[a-fA-F0-9]{4}|[^\"\\\\])*\"} tag \"string\"\n\t\t- when \"{\" tag \"{\"\n\t\t- when \"}\" tag \"}\"\n\t\t- when \"[\" tag \"[\"\n\t\t- when \"]\" tag \"]\"\n\t\t- when \",\" tag \",\"\n\t\t- when \":\" tag \":\"\n\t\t- when \"true\" tag \"true\"\n\t\t- when \"false\" tag \"false\"\n\t\t- when \"null\" tag \"null\"\n\n}\n\ngrammar {\n\tstart: \"json\"\n\n\t[json]\n\t\t| _ (object | array) _ => ( $1[0] )\n\n\t[object]\n\t\t| \"{\" _ \"}\" => ( {} )\n\t\t| \"{\" _ pair (_ \",\" _ pair)* _ \"}\" => ${ extractObject }\n\n\t[array]\n\t\t| \"[\" _ \"]\" => ( [] )\n\t\t| \"[\" _ value (_ \",\" _ value)* _ \"]\" => ${ extractArray }\n\n\t[value]\n\t\t| object => ( $0 )\n\t\t| array => ( $0 )\n\t\t| number => ( $0 )\n\t\t| string => ( $0 )\n\t\t| \"true\" => ( true )\n\t\t| \"false\" => ( false )\n\t\t| \"null\" => ( null )\n\n\t[number]\n\t\t| <number> => ( parseFloat($0.value) )\n\n\t[string]\n\t\t| <string> => ( JSON.parse($0.value) )\n\n\t[pair]\n\t\t| key@k _ \":\" _ value@v => ( [$k, $v] )\n\n\t[key]\n\t\t| string => ( $0 )\n\n}\n\non:import {\nfunction extractPair(kv, output) {\n        if(kv[0]) { output[kv[0]] = kv[1]; }\n    }\n\n    function extractObject({data}) {\n        let output = {};\n\n        extractPair(data[2], output);\n\n        for (let i in data[3]) {\n            extractPair(data[3][i][3], output);\n        }\n\n        return output;\n    }\n\n    function extractArray({data}) {\n        let output = [data[2]];\n\n        for (let i in data[3]) {\n            output.push(data[3][i][3]);\n        }\n\n        return output;\n    }\n}\n","number":"grammar {\n\n\t[unsigned_int]\n\t\t| r:{[0-9]}+ => ( parseInt($0.join(\"\")) )\n\n\t[int]\n\t\t| (\"-\" | \"+\")? r:{[0-9]}+ => ( $0 ? parseInt($0[0]+$1.join(\"\")) : parseInt($1.join(\"\")) )\n\n\t[unsigned_decimal]\n\t\t| r:{[0-9]}+ (\".\" r:{[0-9]}+)? => ( parseFloat($0.join(\"\") + ($1 ? \".\"+$1[1].join(\"\") : \"\")) )\n\n\t[decimal]\n\t\t| \"-\"? r:{[0-9]}+ (\".\" r:{[0-9]}+)? => ( parseFloat( ($0 || \"\") + $1.join(\"\") +($2 ? \".\"+$2[1].join(\"\") : \"\")) )\n\n\t[percentage]\n\t\t| decimal \"%\" => ( $0/100 )\n\n\t[jsonfloat]\n\t\t| \"-\"? r:{[0-9]}+ (\".\" r:{[0-9]}+)? (r:{[eE]} r:{[+-]}? r:{[0-9]}+)? => ( parseFloat( ($0 || \"\") + $1.join(\"\") + ($2 ? \".\"+$2[1].join(\"\") : \"\") + ($3 ? \"e\" + ($3[1] || \"+\") + $3[2].join(\"\") : \"\")) )\n\n}\n","string":"lexer {\n\n\t[string]\n\t\t- import singleQuoteString, doubleQuoteString\n\n\t[doubleQuoteString]\n\t\t- when \"\\\"\" tag \"dquote\" highlight \"string\" goto doubleQuoteStringEnd\n\n\t[singleQuoteString]\n\t\t- when \"'\" tag \"squote\" highlight \"string\" goto singleQuoteStringEnd\n\n\t[doubleQuoteStringEnd]\n\t\t- when r:{\\\\[\\\\/bnrft]} tag \"escaped\" highlight \"constant\"\n\t\t- when r:{\\\\\"} tag \"quoteEscape\"\n\t\t- when r:{\\\\u[A-Fa-f\\d]{4}} tag \"escaped\" highlight \"constant\"\n\t\t- when r:{\\\\.} tag \"badEscape\"\n\t\t- when r:{[^\"\\\\]+} tag \"string\" highlight \"string\"\n\t\t- when \"\\\"\" tag \"dquote\" highlight \"string\" pop\n\n\t[singleQuoteStringEnd]\n\t\t- when r:{\\\\[\\\\/bnrft]} tag \"escaped\"\n\t\t- when r:{\\\\'} tag \"quoteEscape\"\n\t\t- when r:{\\\\u[A-Fa-f\\d]{4}} tag \"escaped\"\n\t\t- when r:{\\\\.} tag \"badEscape\"\n\t\t- when r:{[^'\\\\]+} tag \"string\" highlight \"string\"\n\t\t- when \"'\" tag \"squote\" highlight \"string\" pop\n\n}\n\ngrammar {\n\n\t[string] => ( $0 )\n\t\t| singleQuoteString\n\t\t| doubleQuoteString\n\n\t[doubleQuoteString]\n\t\t| <dquote> stringInner <dquote> => ( $1 )\n\t\t| <dquote> <dquote> => ( '' )\n\n\t[singleQuoteString]\n\t\t| <squote> stringInner <squote> => ( $1 )\n\t\t| <squote> <squote> => ( '' )\n\n\t[stringInner]\n\t\t| stringEscape => ( $0 )\n\t\t| <string> => ( $0.value )\n\t\t| stringInner stringEscape => ( $0 + $1 )\n\t\t| stringInner <string> => ( $0 + $1.value )\n\n\t[stringEscape]\n\t\t| <escaped> => ( JSON.parse('\"' + $0.value + '\"') )\n\t\t| <quoteEscape> => ( $0.value[1] )\n\n}\n","whitespace":"lexer {\n\n\t[whitespace]\n\t\t- when r:{\\s+} tag \"whitespace\"\n\n}\n\ngrammar {\n\n\t[_]\n\t\t| <whitespace>? => ( null )\n\n\t[__]\n\t\t| <whitespace>+ => ( null )\n\n}\n"}

package/src/generator/builtin/string.well

@@ -0,0 +1,54 @@
+lexer {
+
+	[string]
+		- import singleQuoteString, doubleQuoteString
+
+	[doubleQuoteString]
+		- when "\"" tag "dquote" highlight "string" goto doubleQuoteStringEnd
+
+	[singleQuoteString]
+		- when "'" tag "squote" highlight "string" goto singleQuoteStringEnd
+
+	[doubleQuoteStringEnd]
+		- when r:{\\[\\/bnrft]} tag "escaped" highlight "constant"
+		- when r:{\\"} tag "quoteEscape"
+		- when r:{\\u[A-Fa-f\d]{4}} tag "escaped" highlight "constant"
+		- when r:{\\.} tag "badEscape"
+		- when r:{[^"\\]+} tag "string" highlight "string"
+		- when "\"" tag "dquote" highlight "string" pop
+
+	[singleQuoteStringEnd]
+		- when r:{\\[\\/bnrft]} tag "escaped"
+		- when r:{\\'} tag "quoteEscape"
+		- when r:{\\u[A-Fa-f\d]{4}} tag "escaped"
+		- when r:{\\.} tag "badEscape"
+		- when r:{[^'\\]+} tag "string" highlight "string"
+		- when "'" tag "squote" highlight "string" pop
+
+}
+
+grammar {
+
+	[string] => ( $0 )
+		| singleQuoteString
+		| doubleQuoteString
+
+	[doubleQuoteString]
+		| <dquote> stringInner <dquote> => ( $1 )
+		| <dquote> <dquote> => ( '' )
+
+	[singleQuoteString]
+		| <squote> stringInner <squote> => ( $1 )
+		| <squote> <squote> => ( '' )
+
+	[stringInner]
+		| stringEscape => ( $0 )
+		| <string> => ( $0.value )
+		| stringInner stringEscape => ( $0 + $1 )
+		| stringInner <string> => ( $0 + $1.value )
+
+	[stringEscape]
+		| <escaped> => ( JSON.parse('"' + $0.value + '"') )
+		| <quoteEscape> => ( $0.value[1] )
+
+}

package/src/generator/builtin/whitespace.well

@@ -0,0 +1,16 @@
+lexer {
+
+	[whitespace]
+		- when r:{\s+} tag "whitespace"
+
+}
+
+grammar {
+
+	[_]
+		| <whitespace>? => ( null )
+
+	[__]
+		| <whitespace>+ => ( null )
+
+}