grammar-well 1.2.1 → 1.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grammar-well",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Cross-platform Parser written in TypeScript",
   "main": "build/index.js",
   "scripts": {

package/src/compiler/builtin/json.gwell

@@ -0,0 +1,74 @@
+import * from whitespace;
+
+lexer: {{
+    start: "json"
+
+    json ->
+        - import: whitespace
+        - when: /-?(?:[0-9]|[1-9][0-9]+)(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+)?\b/ tag: "number"
+        - when: /"(?:\\["bfnrt\/\\]|\\u[a-fA-F0-9]{4}|[^"\\])*"/ tag: "string"
+        - when: "{" tag: "{"
+        - when: "}" tag: "}"
+        - when: "[" tag: "["
+        - when: "]" tag: "]"
+        - when: "," tag: ","
+        - when: ":" tag: ":"
+        - when: "true" tag: "true"
+        - when: "false" tag: "false"
+        - when: "null" tag: "null"
+}}
+
+grammar: {{
+    json -> _ (object | array) _ : {{ $1[0] }}
+
+    object -> "{" _ "}" : {{ {} }}
+        | "{" _ pair (_ "," _ pair)* _ "}" : ${ extractObject }
+
+    array -> "[" _ "]" : {{ [] }}
+        | "[" _ value (_ "," _ value)* _ "]" : ${ extractArray }
+
+    value : {{ $0 }} ->
+        object
+        | array
+        | number
+        | string
+        | "true" : {{ true }}
+        | "false" : {{ false }}
+        | "null" : {{ null }}
+
+    number -> $number : {{ parseFloat($0.value) }}
+
+    string -> $string : {{ JSON.parse($0.value) }}
+
+    pair -> key:k _ ":" _ value:v : {{ [$k, $v] }}
+
+    key -> string : {{ $0 }}
+}}
+
+head: ${
+    function extractPair(kv, output) {
+        if(kv[0]) { output[kv[0]] = kv[1]; }
+    }
+
+    function extractObject({data}) {
+        let output = {};
+
+        extractPair(data[2], output);
+
+        for (let i in data[3]) {
+            extractPair(data[3][i][3], output);
+        }
+
+        return output;
+    }
+
+    function extractArray({data}) {
+        let output = [data[2]];
+
+        for (let i in data[3]) {
+            output.push(data[3][i][3]);
+        }
+
+        return output;
+    }
+}

package/src/compiler/builtin/number.gwell

@@ -0,0 +1,20 @@
+grammar: {{
+
+    unsigned_int -> 
+        /[0-9]/+ : {{ parseInt($0.join("")) }}
+
+    int -> 
+        ("-"|"+")? /[0-9]/+ : {{ $0 ? parseInt($0[0]+$1.join("")) : parseInt($1.join("")) }}
+
+    unsigned_decimal -> 
+        /[0-9]/+ ("." /[0-9]/+)? : {{ parseFloat($0.join("") + ($1 ? "."+$1[1].join("") : "")) }}
+
+    decimal -> 
+        "-"? /[0-9]/+ ("." /[0-9]/+)? : {{ parseFloat( ($0 || "") + $1.join("") +($2 ? "."+$2[1].join("") : "")) }}
+
+    percentage -> 
+        decimal "%" : {{ $0/100 }}
+        
+    jsonfloat -> 
+        "-"? /[0-9]/+ ("." /[0-9]/+)? (/[eE]/ /[+-]/? /[0-9]/+)? : {{ parseFloat( ($0 || "") + $1.join("") + ($2 ? "."+$2[1].join("") : "") + ($3 ? "e" + ($3[1] || "+") + $3[2].join("") : "")) }}
+}}

package/src/compiler/builtin/string.gwell

@@ -0,0 +1,48 @@
+lexer: {{
+    string ->
+        - import: singleQuoteString, doubleQuoteString
+
+    doubleQuoteString ->
+        - when: /"/ tag: "dquote" highlight:"string" goto: doubleQuoteStringEnd 
+
+    singleQuoteString ->
+        - when: /'/ tag: "squote" highlight:"string" goto: singleQuoteStringEnd 
+
+    doubleQuoteStringEnd ->
+        - when: /\\[\\\/bnrft]/ tag: "escaped" highlight:"constant"
+        - when: /\\"/ tag: "quoteEscape"
+        - when: /\\u[A-Fa-f\d]{4}/ tag: "escaped" highlight:"constant"
+        - when: /\\./ tag:"badEscape"
+        - when: /[^"\\]+/ tag: "string" highlight:"string"
+        - when: "\"" tag: "dquote" highlight:"string" pop
+
+    singleQuoteStringEnd ->
+        - when: /\\[\\\/bnrft]/ tag: "escaped"
+        - when: /\\'/ tag: "quoteEscape"
+        - when: /\\u[A-Fa-f\d]{4}/ tag: "escaped" 
+        - when: /\\./ tag:"badEscape"
+        - when: /[^'\\]+/ tag: "string" highlight:"string"
+        - when: "'" tag: "squote" highlight:"string" pop
+}}
+
+grammar: {{
+    string : {{ $0 }} 
+        -> singleQuoteString 
+        | doubleQuoteString 
+
+    doubleQuoteString 
+        -> $dquote stringInner $dquote : {{ $1 }}
+        | $dquote $dquote : {{ '' }}
+    
+    singleQuoteString 
+        -> $squote stringInner $squote : {{ $1 }}
+        | $squote $squote : {{ '' }}
+
+    stringInner -> stringEscape : {{ $0 }}
+        | $string : {{  $0.value }}
+        | stringInner stringEscape : {{ $0 + $1 }}
+        | stringInner $string : {{ $0 + $1.value }}
+    
+    stringEscape -> $escaped : {{  JSON.parse('"' +$0.value + '"') }}
+        | $quoteEscape : {{ $0.value[1] }}
+}}

package/src/compiler/builtin/whitespace.gwell

@@ -0,0 +1,10 @@
+lexer: {{
+    start: "whitespace"
+    whitespace ->
+        - when: /\s+/ tag: "whitespace" 
+}}
+
+grammar: {{
+    _  -> $whitespace* : {{ null }}
+    __ -> $whitespace+ : {{ null }}
+}}

package/src/compiler/builtin.json

@@ -0,0 +1 @@
+{"json":"import * from whitespace;\r\n\r\nlexer: {{\r\n    start: \"json\"\r\n\r\n    json ->\r\n        - import: whitespace\r\n        - when: /-?(?:[0-9]|[1-9][0-9]+)(?:\\.[0-9]+)?(?:[eE][-+]?[0-9]+)?\\b/ tag: \"number\"\r\n        - when: /\"(?:\\\\[\"bfnrt\\/\\\\]|\\\\u[a-fA-F0-9]{4}|[^\"\\\\])*\"/ tag: \"string\"\r\n        - when: \"{\" tag: \"{\"\r\n        - when: \"}\" tag: \"}\"\r\n        - when: \"[\" tag: \"[\"\r\n        - when: \"]\" tag: \"]\"\r\n        - when: \",\" tag: \",\"\r\n        - when: \":\" tag: \":\"\r\n        - when: \"true\" tag: \"true\"\r\n        - when: \"false\" tag: \"false\"\r\n        - when: \"null\" tag: \"null\"\r\n}}\r\n\r\ngrammar: {{\r\n    json -> _ (object | array) _ : {{ $1[0] }}\r\n\r\n    object -> \"{\" _ \"}\" : {{ {} }}\r\n        | \"{\" _ pair (_ \",\" _ pair)* _ \"}\" : ${ extractObject }\r\n\r\n    array -> \"[\" _ \"]\" : {{ [] }}\r\n        | \"[\" _ value (_ \",\" _ value)* _ \"]\" : ${ extractArray }\r\n\r\n    value : {{ $0 }} ->\r\n        object\r\n        | array\r\n        | number\r\n        | string\r\n        | \"true\" : {{ true }}\r\n        | \"false\" : {{ false }}\r\n        | \"null\" : {{ null }}\r\n\r\n    number -> $number : {{ parseFloat($0.value) }}\r\n\r\n    string -> $string : {{ JSON.parse($0.value) }}\r\n\r\n    pair -> key:k _ \":\" _ value:v : {{ [$k, $v] }}\r\n\r\n    key -> string : {{ $0 }}\r\n}}\r\n\r\nhead: ${\r\n    function extractPair(kv, output) {\r\n        if(kv[0]) { output[kv[0]] = kv[1]; }\r\n    }\r\n\r\n    function extractObject({data}) {\r\n        let output = {};\r\n\r\n        extractPair(data[2], output);\r\n\r\n        for (let i in data[3]) {\r\n            extractPair(data[3][i][3], output);\r\n        }\r\n\r\n        return output;\r\n    }\r\n\r\n    function extractArray({data}) {\r\n        let output = [data[2]];\r\n\r\n        for (let i in data[3]) {\r\n            output.push(data[3][i][3]);\r\n        }\r\n\r\n        return output;\r\n    }\r\n}\r\n","number":"grammar: {{\r\n\r\n    unsigned_int -> \r\n        /[0-9]/+ : {{ parseInt($0.join(\"\")) }}\r\n\r\n    int -> \r\n        (\"-\"|\"+\")? /[0-9]/+ : {{ $0 ? parseInt($0[0]+$1.join(\"\")) : parseInt($1.join(\"\")) }}\r\n\r\n    unsigned_decimal -> \r\n        /[0-9]/+ (\".\" /[0-9]/+)? : {{ parseFloat($0.join(\"\") + ($1 ? \".\"+$1[1].join(\"\") : \"\")) }}\r\n\r\n    decimal -> \r\n        \"-\"? /[0-9]/+ (\".\" /[0-9]/+)? : {{ parseFloat( ($0 || \"\") + $1.join(\"\") +($2 ? \".\"+$2[1].join(\"\") : \"\")) }}\r\n\r\n    percentage -> \r\n        decimal \"%\" : {{ $0/100 }}\r\n        \r\n    jsonfloat -> \r\n        \"-\"? /[0-9]/+ (\".\" /[0-9]/+)? (/[eE]/ /[+-]/? /[0-9]/+)? : {{ parseFloat( ($0 || \"\") + $1.join(\"\") + ($2 ? \".\"+$2[1].join(\"\") : \"\") + ($3 ? \"e\" + ($3[1] || \"+\") + $3[2].join(\"\") : \"\")) }}\r\n}}","string":"lexer: {{\r\n    string ->\r\n        - import: singleQuoteString, doubleQuoteString\r\n\r\n    doubleQuoteString ->\r\n        - when: /\"/ tag: \"dquote\" highlight:\"string\" goto: doubleQuoteStringEnd \r\n\r\n    singleQuoteString ->\r\n        - when: /'/ tag: \"squote\" highlight:\"string\" goto: singleQuoteStringEnd \r\n\r\n    doubleQuoteStringEnd ->\r\n        - when: /\\\\[\\\\\\/bnrft]/ tag: \"escaped\" highlight:\"constant\"\r\n        - when: /\\\\\"/ tag: \"quoteEscape\"\r\n        - when: /\\\\u[A-Fa-f\\d]{4}/ tag: \"escaped\" highlight:\"constant\"\r\n        - when: /\\\\./ tag:\"badEscape\"\r\n        - when: /[^\"\\\\]+/ tag: \"string\" highlight:\"string\"\r\n        - when: \"\\\"\" tag: \"dquote\" highlight:\"string\" pop\r\n\r\n    singleQuoteStringEnd ->\r\n        - when: /\\\\[\\\\\\/bnrft]/ tag: \"escaped\"\r\n        - when: /\\\\'/ tag: \"quoteEscape\"\r\n        - when: /\\\\u[A-Fa-f\\d]{4}/ tag: \"escaped\" \r\n        - when: /\\\\./ tag:\"badEscape\"\r\n        - when: /[^'\\\\]+/ tag: \"string\" highlight:\"string\"\r\n        - when: \"'\" tag: \"squote\" highlight:\"string\" pop\r\n}}\r\n\r\ngrammar: {{\r\n    string : {{ $0 }} \r\n        -> singleQuoteString \r\n        | doubleQuoteString \r\n\r\n    doubleQuoteString \r\n        -> $dquote stringInner $dquote : {{ $1 }}\r\n        | $dquote $dquote : {{ '' }}\r\n    \r\n    singleQuoteString \r\n        -> $squote stringInner $squote : {{ $1 }}\r\n        | $squote $squote : {{ '' }}\r\n\r\n    stringInner -> stringEscape : {{ $0 }}\r\n        | $string : {{  $0.value }}\r\n        | stringInner stringEscape : {{ $0 + $1 }}\r\n        | stringInner $string : {{ $0 + $1.value }}\r\n    \r\n    stringEscape -> $escaped : {{  JSON.parse('\"' +$0.value + '\"') }}\r\n        | $quoteEscape : {{ $0.value[1] }}\r\n}}","whitespace":"lexer: {{\r\n    start: \"whitespace\"\r\n    whitespace ->\r\n        - when: /\\s+/ tag: \"whitespace\" \r\n}}\r\n\r\ngrammar: {{\r\n    _  -> $whitespace* : {{ null }}\r\n    __ -> $whitespace+ : {{ null }}\r\n}}"}

package/src/compiler/compiler.ts

@@ -2,22 +2,15 @@ import { CompileOptions, GrammarBuilderContext, TemplateFormat, LanguageDirectiv
 
 import { Parser } from "../parser/parser";
 import { FileSystemResolver } from "./import-resolver";
-import Language from '../grammars/gwell';
+import Language from './gwell';
 
 import { ESMOutput, JavascriptOutput } from "./outputs/javascript";
 import { TypescriptFormat } from "./outputs/typescript";
 import { JSONFormatter } from "./outputs/json";
 
-import * as number from '../grammars/number.json';
-import * as string from '../grammars/string.json';
-import * as whitespace from '../grammars/whitespace.json';
 import { Generator } from "./generator/generator";
+import * as BuiltInRegistry from "./builtin.json"
 
-const BuiltInRegistry = {
-    number,
-    string,
-    whitespace,
-}
 const TemplateFormats = {
     _default: JavascriptOutput,
     object: (grammar, exportName) => ({ grammar, exportName }),
@@ -44,7 +37,7 @@ export class GrammarBuilder {
 
     generator = new Generator();
 
-    constructor(private config: CompileOptions = {}, context?: GrammarBuilderContext) {
+    constructor(private config: CompileOptions = {}, context?: GrammarBuilderContext, private alias: string = '') {
         this.context = context || {
             alreadyCompiled: new Set(),
             resolver: config.resolverInstance ? config.resolverInstance : config.resolver ? new config.resolver(config.basedir) : new FileSystemResolver(config.basedir),
@@ -90,7 +83,7 @@ export class GrammarBuilder {
 
     private async processImportDirective(directive: ImportDirective) {
         if (directive.path) {
-            await this.importGrammar(directive.import);
+            await this.importGrammar(directive.import, directive.alias);
         } else {
             this.importBuiltIn(directive.import);
         }
@@ -102,15 +95,22 @@ export class GrammarBuilder {
 
     private processGrammarDirective(directive: GrammarDirective) {
         if (directive.grammar.config) {
-            this.generator.state.grammar.start = directive.grammar.config.start || this.generator.state.grammar.start;
+            if (directive.grammar.config.start) {
+                this.generator.state.grammar.start = this.alias + directive.grammar.config.start;
+            }
+
             Object.assign(this.generator.state.grammar.config, directive.grammar.config);
             // this.generator.state.grammar.postprocessDefault =  directive.grammar.config.postprocessDefault || this.generator.state.grammar.postprocessDefault;
             // this.generator.state.grammar.postprocessOverride =  directive.grammar.config.postprocessOverride || this.generator.state.grammar.postprocessOverride;
         }
 
+        if (!this.generator.state.grammar.start && directive.grammar.rules.length) {
+            this.generator.state.grammar.start = this.alias + directive.grammar.rules[0].name;
+        }
+
         for (const rule of directive.grammar.rules) {
+            rule.name = this.alias + rule.name;
             this.buildRules(rule.name, rule.expressions, rule);
-            this.generator.state.grammar.start = this.generator.state.grammar.start || rule.name;
         }
     }
 
@@ -121,33 +121,54 @@ export class GrammarBuilder {
                 states: {}
             };
         }
-        this.generator.state.lexer.start = directive.lexer.start || this.generator.state.lexer.start || (directive.lexer.states.length ? directive.lexer.states[0].name : '');
+        if (directive.lexer.start) {
+            this.generator.state.lexer.start = this.alias + directive.lexer.start;
+        }
+
+        if (!this.generator.state.lexer.start && directive.lexer.states.length) {
+            this.generator.state.lexer.start = this.alias + directive.lexer.states[0].name
+        }
+
         for (const state of directive.lexer.states) {
+            state.name = this.alias + state.name;
+            if (this.alias) {
+                state.rules.forEach(v => {
+                    if ('import' in v) {
+                        v.import = v.import.map(v2 => this.alias + v2);
+                    }
+                    if ('set' in v) {
+                        v.set = this.alias + v.set;
+                    }
+                    if ('goto' in v) {
+                        v.goto = this.alias + v.goto;
+                    }
+                })
+            }
             this.generator.addLexerState(state);
         }
     }
 
-    private importBuiltIn(name: string) {
+    private async importBuiltIn(name: string, alias?: string) {
         name = name.toLowerCase();
         if (!this.context.alreadyCompiled.has(name)) {
             this.context.alreadyCompiled.add(name);
             if (!BuiltInRegistry[name])
                 return;
-            this.generator.merge(BuiltInRegistry[name].state);
+            await this.mergeLanguageDefinitionString(BuiltInRegistry[name], alias);
         }
     }
 
-    private async importGrammar(name) {
+    private async importGrammar(path: string, alias?: string) {
         const resolver = this.context.resolver;
-        const path = resolver.path(name);
-        if (!this.context.alreadyCompiled.has(path)) {
-            this.context.alreadyCompiled.add(path);
-            await this.mergeLanguageDefinitionString(await resolver.body(path))
+        const fullPath = resolver.path(path);
+        if (!this.context.alreadyCompiled.has(fullPath)) {
+            this.context.alreadyCompiled.add(fullPath);
+            await this.mergeLanguageDefinitionString(await resolver.body(fullPath), alias);
         }
     }
 
-    private async mergeLanguageDefinitionString(body: string) {
-        const builder = new GrammarBuilder(this.config, this.context);
+    private async mergeLanguageDefinitionString(body: string, alias: string = '') {
+        const builder = new GrammarBuilder(this.config, this.context, alias);
         await builder.import(this.parser.run(body).results[0]);
         this.generator.merge(builder.generator.state);
         return;
@@ -174,7 +195,7 @@ export class GrammarBuilder {
             return this.buildRepeatRules(name, symbol);
         }
         if ('rule' in symbol) {
-            return symbol;
+            return { ...symbol, rule: this.alias + symbol.rule };
         }
         if ('regex' in symbol) {
             return symbol;

package/src/compiler/gwell.gwell

@@ -0,0 +1,283 @@
+lexer: {{
+    start: "start"
+
+    start ->
+        - import: string, js, ws, comment, l_scolon, l_star
+        - when: /lexer(?![a-zA-Z\d_])/ tag: "T_WORD" goto: lexer highlight: "type"
+        - when: /grammar(?![a-zA-Z\d_])/ tag: "T_WORD" goto: grammar highlight: "type"
+        - when: /config(?![a-zA-Z\d_])/ tag: "T_WORD" goto: config highlight: "type"
+        - import: kv
+    config ->
+        - import: ws, l_colon
+        - when: "{{" tag: "L_TEMPLATEL" set: config_inner
+    config_inner ->
+        - import: comment, kv
+        - when: "}}" tag: "L_TEMPLATER" pop: 1
+    grammar ->
+        - import: ws, l_colon
+        - when: "{{" tag: "L_TEMPLATEL" set: grammar_inner
+    grammar_inner ->
+        - import: comment, js, js_template, ws, regex, l_qmark, l_plus, l_star, kv, l_colon, l_comma, l_pipe, l_parenl, l_parenr, l_arrow, l_dsign, l_dash
+        - when: "}}" tag: "L_TEMPLATER" pop: 1
+    lexer ->
+        - import: ws, l_colon
+        - when: "{{" tag: "L_TEMPLATEL" set: lexer_inner
+    lexer_inner ->
+        - import: ws, comment, regex, l_comma, l_arrow, l_dash, kv, js
+        - when: "}}" tag: "L_TEMPLATER" pop: 1
+    js ->
+        - when: "${" tag: "L_JSL" goto: js_wrap
+    js_wrap ->
+        default: "T_JSBODY"
+        unmatched: "T_JSBODY"
+        - import: jsignore
+        - when: "{" tag: "T_JSBODY" goto: js_literal
+        - when: "}" tag: "L_JSR" pop: 1
+    js_literal ->
+        default: "T_JSBODY"
+        unmatched: "T_JSBODY"
+        - import: jsignore
+        - when: "{" tag: "T_JSBODY" goto: js_literal
+        - when: "}" tag: "T_JSBODY" pop: 1
+    js_template ->
+        - when: "{{" tag: "L_TEMPLATEL" goto: js_template_inner
+    js_template_inner ->
+        default: "T_JSBODY"
+        unmatched: "T_JSBODY"
+        - import: jsignore
+        - when: "{" tag: "T_JSBODY" goto: js_literal
+        - when: "}}" tag: "L_TEMPLATER" pop: 1
+    kv ->
+        - import: string, ws, word, l_colon, integer
+    jsignore ->
+        - when: /"(?:[^"\\\r\n]|\\.)*"/ tag: "T_JSBODY"
+        - when: /'(?:[^'\\\r\n]|\\.)*'/ tag: "T_JSBODY"
+        - when: /`(?:[^`\\]|\\.)*`/ tag: "T_JSBODY"
+        - when: /\/(?:[^\/\\\r\n]|\\.)+\/[gmiyu]*/ tag: "T_JSBODY"
+        - when: /\/\/[^\n]*/ tag: "T_JSBODY"
+        - when: /\/\*.*\*\// tag: "T_JSBODY"
+    string ->
+        - when: /"(?:[^"\\\r\n]|\\.)*"/ tag: "T_STRING" highlight: "string"
+    string2 ->
+        - when: /'(?:[^'\\\r\n]|\\.)*'/ tag: "T_STRING" highlight: "string"
+    string3 ->
+        - when: /`(?:[^`\\]|\\.)*`/ tag: "T_STRING" highlight: "string"
+    regex ->
+        - when: /\/(?:[^\/\\\r\n]|\\.)+\// tag: "T_REGEX" highlight: "regexp"
+    integer ->
+        - when: /\d+/ tag: "T_INTEGER" highlight: "number"
+    word ->
+        - when: /[a-zA-Z_][a-zA-Z_\d]*/ tag: "T_WORD"
+    ws ->
+        - when: /\s+/ tag: "T_WS"
+    l_colon ->
+        - when: ":" tag: "L_COLON" highlight: "keyword"
+    l_scolon ->
+        - when: ";" tag: "L_SCOLON"
+    l_qmark ->
+        - when: "?" tag: "L_QMARK"
+    l_plus ->
+        - when: "+" tag: "L_PLUS"
+    l_star ->
+        - when: "*" tag: "L_STAR"
+    l_comma ->
+        - when: "," tag: "L_COMMA"
+    l_pipe ->
+        - when: "|" tag: "L_PIPE" highlight: "keyword"
+    l_parenl ->
+        - when: "(" tag: "L_PARENL"
+    l_parenr ->
+        - when: ")" tag: "L_PARENR"
+    l_templatel ->
+        - when: "{{" tag: "L_TEMPLATEL"
+    l_templater ->
+        - when: "}}" tag: "L_TEMPLATER"
+    l_arrow ->
+        - when: "->" tag: "L_ARROW" highlight: "keyword"
+    l_dsign ->
+        - when: "$" tag: "L_DSIGN"
+    l_dash ->
+        - when: "-" tag: "L_DASH"
+    comment ->
+        - when: /\/\/[^\n]*/ tag: "T_COMMENT" highlight: "comment"
+    commentmulti ->
+        - when: /\/\*.*\*\// tag: "T_COMMENT" highlight: "comment"
+
+}}
+
+grammar: {{
+
+    main ->
+        _ section_list _ : {{ $1 }}
+
+    section_list ->
+        section : {{ [$0] }}
+        | section T_WS section_list : {{ [$0].concat($2) }}
+
+    section ->
+        K_CONFIG _ L_COLON _ L_TEMPLATEL _ kv_list:list _ L_TEMPLATER : {{ { config: Object.assign(...$list) } }}
+        | K_IMPORT _ L_STAR _ K_FROM __ T_WORD:import _ L_SCOLON : {{ { import: $import } }}
+        | K_IMPORT _ L_STAR _ K_FROM __ T_STRING:import _ L_SCOLON : {{ { import: $import, path: true } }}
+        | K_IMPORT _ L_STAR _ "as" _ T_WORD:alias _ K_FROM __ T_WORD:import _ L_SCOLON : {{ { import: $import, alias: $alias} }}
+        | K_IMPORT _ L_STAR _ "as" _ T_WORD:alias _ K_FROM __ T_STRING:import _ L_SCOLON : {{ { import: $import, path: true, alias: $alias} }}
+        | K_LEXER _ L_COLON _ L_TEMPLATEL _ lexer:lexer _ L_TEMPLATER : {{ { lexer: Object.assign(...$lexer) } }}
+        | K_GRAMMAR _ L_COLON _ L_TEMPLATEL _ grammar:grammar _ L_TEMPLATER : {{ { grammar: $grammar } }}
+        | K_BODY _ L_COLON _ T_JS:js : {{ { body: $js } }}
+        | K_BODY _ L_COLON _ T_STRING:js : {{ { body: $js, path: true } }}
+        | K_HEAD _ L_COLON _ T_JS:js : {{ { head: $js } }}
+        | K_HEAD _ L_COLON _ T_STRING:js : {{ { head: $js, path: true } }}
+
+    lexer ->
+        kv_list _ state_list : {{ $0.concat({ states: $2 }) }}
+        | state_list : {{ [{ states: $0 }] }}
+
+    state_list ->
+        state : {{ data }}
+        | state _ state_list : {{ [$0].concat($2) }}
+
+    state ->
+        state_declare _ state_definition : {{ Object.assign({ name: $0 }, $2) }}
+
+    state_declare ->
+        T_WORD _ L_ARROW : {{ $0 }}
+
+    state_definition ->
+        kv_list _ token_list : {{ Object.assign(...$0, { rules: $2 }) }}
+        | token_list : {{ { rules: $0 } }}
+
+    token_list ->
+        token : {{ data }}
+        | token _ token_list : {{ [$0].concat($2) }}
+
+    token ->
+        L_DASH _ K_IMPORT _ L_COLON _ word_list : {{ { import: $6 } }}
+        | L_DASH _ token_definition_list : {{ Object.assign(...$2) }}
+
+    token_definition_list ->
+        token_definition : {{ data }}
+        | token_definition _ token_definition_list : {{ [$0].concat($2) }}
+
+    token_definition ->
+        K_TAG _ L_COLON _ string_list : {{ { tag: $4 } }}
+        | K_WHEN _ L_COLON _ T_STRING : {{ { when: $4 } }}
+        | K_WHEN _ L_COLON _ T_REGEX : {{ { when: $4 } }}
+        | K_POP : {{ { pop: 1 } }}
+        | K_POP _ L_COLON _ T_INTEGER : {{ { pop: parseInt($4) } }}
+        | K_POP _ L_COLON _ K_ALL : {{ { pop: "all" } }}
+        | K_HIGHLIGHT _ L_COLON _ T_STRING : {{ { highlight: $4 } }}
+        | K_INSET : {{ { inset: 1 } }}
+        | K_INSET _ L_COLON _ T_INTEGER : {{ { inset: parseInt($4) } }}
+        | K_SET _ L_COLON _ T_WORD : {{ { set: $4 } }}
+        | K_GOTO _ L_COLON _ T_WORD : {{ { goto: $4 } }}
+        | K_TYPE _ L_COLON _ T_STRING : {{ { type: $4 } }}
+
+    grammar ->
+        kv_list _ grammar_rule_list : {{ { config: Object.assign(...$0), rules: $2 } }}
+        | grammar_rule_list : {{ { rules: $0 } }}
+
+    grammar_rule_list ->
+        grammar_rule : {{ [$0] }}
+        | grammar_rule _ grammar_rule_list : {{ [$0].concat($2) }}
+
+    grammar_rule ->
+        T_WORD _ L_ARROW _ expression_list : {{ { name: $0, expressions: $4 } }}
+        | T_WORD  __ L_COLON _ T_JS:js _ L_ARROW _ expression_list:expressions : {{ { name: $0, expressions: $expressions, postprocess: $js } }}
+        | T_WORD  __ L_COLON _ T_GRAMMAR_TEMPLATE:template _ L_ARROW _ expression_list:expressions : {{ { name: $0, expressions: $expressions, postprocess: $template } }}
+
+    expression_list ->
+        expression
+        | expression_list _ L_PIPE _ expression : {{ $0.concat([$4]) }}
+
+    expression ->
+        expression_symbol_list : {{ { symbols: $0 } }}
+        | expression_symbol_list __ L_COLON _ T_JS:js : {{ { symbols: $0, postprocess: $js } }}
+        | expression_symbol_list __ L_COLON _ T_GRAMMAR_TEMPLATE:template : {{ { symbols: $0, postprocess: $template } }}
+
+    expression_symbol_list ->
+        expression_symbol
+        | expression_symbol_list T_WS expression_symbol : {{ $0.concat([$2]) }}
+
+    
+    expression_symbol ->
+        expression_symbol_match : {{ $0 }}
+        | expression_symbol_match L_COLON T_WORD : {{ { ...$0,  alias: $2 } }}
+        | expression_symbol_match expression_repeater : {{ { expression: $0, repeat: $1 } }}
+        | expression_symbol_match expression_repeater L_COLON T_WORD : {{ { expression: $0, repeat: $1, alias: $4 } }}
+
+
+    expression_symbol_match ->
+        T_WORD : {{ { rule: $0 } }}
+        | T_STRING "i"? : {{ { literal: $0, insensitive: !!$1 } }}
+        | L_DSIGN T_WORD : {{ { token: $1} }}
+        | L_DSIGN T_STRING : {{ { token: $1} }}
+        | T_REGEX : {{ $0 }}
+        | L_PARENL _ expression_list _ L_PARENR : {{ { subexpression: $2 } }}
+        | T_JS : {{ $0 }}
+
+    expression_repeater : {{ $0[0].value }} ->
+        L_QMARK
+        | L_PLUS
+        | L_STAR
+
+    kv_list ->
+        kv : {{ data }}
+        | kv _ kv_list : {{ [$0].concat($2) }}
+
+    kv ->
+        T_WORD _ L_COLON _ ( T_WORD| T_STRING| T_INTEGER | T_JS | T_GRAMMAR_TEMPLATE) : {{ { [$0]: $4[0] } }}
+
+    string_list ->
+        T_STRING : {{ [$0] }}
+        | T_STRING _ L_COMMA _ string_list : {{ [$0].concat($4) }}
+
+    word_list ->
+        T_WORD : {{ [$0] }}
+        | T_WORD _ L_COMMA _ word_list : {{ [$0].concat($4) }}
+
+    _ ->
+        ( T_WS | T_COMMENT )* : {{ null }} 
+
+    __ ->
+        ( T_WS | T_COMMENT )+ : {{ null }}
+
+    L_COLON -> $L_COLON
+    L_SCOLON -> $L_SCOLON
+    L_QMARK -> $L_QMARK
+    L_PLUS -> $L_PLUS
+    L_STAR -> $L_STAR
+    L_COMMA -> $L_COMMA
+    L_PIPE -> $L_PIPE
+    L_PARENL -> $L_PARENL
+    L_PARENR -> $L_PARENR
+    L_TEMPLATEL -> $L_TEMPLATEL
+    L_TEMPLATER -> $L_TEMPLATER
+    L_ARROW -> $L_ARROW
+    L_DSIGN -> $L_DSIGN
+    L_DASH -> $L_DASH
+
+    K_ALL -> "all"
+    K_TAG -> "tag"
+    K_FROM -> "from"
+    K_TYPE -> "type"
+    K_WHEN -> "when"
+    K_POP -> "pop"
+    K_HIGHLIGHT -> "highlight"
+    K_INSET -> "inset"
+    K_SET -> "set"
+    K_GOTO -> "goto"
+    K_CONFIG -> "config"
+    K_LEXER -> "lexer"
+    K_GRAMMAR -> "grammar"
+    K_IMPORT -> "import"
+    K_BODY -> "body" 
+    K_HEAD -> "head"
+
+    T_JS -> $L_JSL $T_JSBODY* $L_JSR : {{ { js: $1.map(v=>v.value).join('') } }}
+    T_GRAMMAR_TEMPLATE -> $L_TEMPLATEL _ $T_JSBODY* _ $L_TEMPLATER : {{ { template: $2.map(v=>v.value).join('').trim() } }}
+    T_STRING -> $T_STRING : {{ JSON.parse($0.value) }}
+    T_WORD -> $T_WORD : {{ $0.value }}
+    T_REGEX -> $T_REGEX /[gmiuy]/* : {{ { regex: $0.value.replace(/\\\\\//g,'/').slice(1,-1), flags: $1.map(v=>v.value).join('').trim() } }}
+    T_COMMENT -> $T_COMMENT
+    T_INTEGER -> $T_INTEGER : {{ $0.value }}
+    T_WS -> $T_WS : {{ null }}
+}}