grammar-well 1.1.0

package/.eslintrc.cjs

@@ -0,0 +1,14 @@
+module.exports = {
+  extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended'],
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint'],
+  root: true,
+  rules: {
+    "@typescript-eslint/no-inferrable-types": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-var-requires": "off",
+    "@typescript-eslint/no-unused-vars": ["error", {argsIgnorePattern: "^_", destructuredArrayIgnorePattern: "^_", varsIgnorePattern: "^_"}],
+    "no-unused-vars": "off"
+  },
+  ignorePatterns: ["**/grammars/*.js"],
+};

package/README.md

@@ -0,0 +1,288 @@
+# Grammar Well
+A cross-platform grammar compiler and interpreter. That aims to facilitate a simple way to create and evaluate custom grammars on the front-end and back-end. Formerly a TypeScript port of [Nearley](https://github.com/kach/nearley).
+
+# Warning
+A lot has changed and documentation needs to be written. For now here's the Grammar Well grammar file that parses Garmmar Well's syntax.
+
+
+```
+lexer: {{
+    start: "start"
+
+    start ->
+        - import: string, js, ws, comment, l_scolon, l_star
+        - when: /lexer(?![a-zA-Z\d_])/ tag: "T_WORD" goto: lexer
+        - when: /grammar(?![a-zA-Z\d_])/ tag: "T_WORD" goto: grammar
+        - when: /config(?![a-zA-Z\d_])/ tag: "T_WORD" goto: config
+        - import: kv
+    config ->
+        - import: ws, l_colon
+        - when: "{{" tag: "L_TEMPLATEL" set: config_inner
+    config_inner ->
+        - import: comment, kv
+        - when: "}}" tag: "L_TEMPLATER" pop: 1
+    grammar ->
+        - import: ws, l_colon
+        - when: "{{" tag: "L_TEMPLATEL" set: grammar_inner
+    grammar_inner ->
+        - import: comment, js, js_template, ws, regex, l_qmark, l_plus, l_star, kv, l_colon, l_comma, l_pipe, l_parenl, l_parenr, l_arrow, l_dsign, l_dash
+        - when: "}}" tag: "L_TEMPLATER" pop: 1
+    lexer ->
+        - import: ws, l_colon
+        - when: "{{" tag: "L_TEMPLATEL" set: lexer_inner
+    lexer_inner ->
+        - import: ws, comment, regex, l_comma, l_arrow, l_dash, kv, js
+        - when: "}}" tag: "L_TEMPLATER" pop: 1
+    js ->
+        - when: "${" tag: "L_JSL" goto: js_wrap
+    js_wrap ->
+        default: "T_JSBODY"
+        unmatched: "T_JSBODY"
+        - import: jsignore
+        - when: "{" tag: "T_JSBODY" goto: js_literal
+        - when: "}" tag: "L_JSR" pop: 1
+    js_literal ->
+        default: "T_JSBODY"
+        unmatched: "T_JSBODY"
+        - import: jsignore
+        - when: "{" tag: "T_JSBODY" goto: js_literal
+        - when: "}" tag: "T_JSBODY" pop: 1
+    js_template ->
+        - when: "{{" tag: "L_TEMPLATEL" goto: js_template_inner
+    js_template_inner ->
+        default: "T_JSBODY"
+        unmatched: "T_JSBODY"
+        - import: jsignore
+        - when: "{" tag: "T_JSBODY" goto: js_literal
+        - when: "}}" tag: "L_TEMPLATER" pop: 1
+    kv ->
+        - import: string, ws, word, l_colon, integer
+    jsignore ->
+        - when: /"(?:[^"\\\r\n]|\\.)*"/ tag: "T_JSBODY"
+        - when: /'(?:[^'\\\r\n]|\\.)*'/ tag: "T_JSBODY"
+        - when: /`(?:[^`\\]|\\.)*`/ tag: "T_JSBODY"
+        - when: /\/(?:[^\/\\\r\n]|\\.)+\/[gmiyu]*/ tag: "T_JSBODY"
+        - when: /\/\/[^\n]*/ tag: "T_JSBODY"
+        - when: /\/\*.*\*\// tag: "T_JSBODY"
+    string ->
+        - when: /"(?:[^"\\\r\n]|\\.)*"/ tag: "T_STRING"
+    string2 ->
+        - when: /'(?:[^'\\\r\n]|\\.)*'/ tag: "T_STRING"
+    string3 ->
+        - when: /`(?:[^`\\]|\\.)*`/ tag: "T_STRING"
+    regex ->
+        - when: /\/(?:[^\/\\\r\n]|\\.)+\// tag: "T_REGEX"
+    integer ->
+        - when: /\d+/ tag: "T_INTEGER"
+    word ->
+        - when: /[a-zA-Z_][a-zA-Z_\d]*/ tag: "T_WORD"
+    ws ->
+        - when: /\s+/ tag: "T_WS"
+    l_colon ->
+        - when: ":" tag: "L_COLON"
+    l_scolon ->
+        - when: ";" tag: "L_SCOLON"
+    l_qmark ->
+        - when: "?" tag: "L_QMARK"
+    l_plus ->
+        - when: "+" tag: "L_PLUS"
+    l_star ->
+        - when: "*" tag: "L_STAR"
+    l_comma ->
+        - when: "," tag: "L_COMMA"
+    l_pipe ->
+        - when: "|" tag: "L_PIPE"
+    l_parenl ->
+        - when: "(" tag: "L_PARENL"
+    l_parenr ->
+        - when: ")" tag: "L_PARENR"
+    l_templatel ->
+        - when: "{{" tag: "L_TEMPLATEL"
+    l_templater ->
+        - when: "}}" tag: "L_TEMPLATER"
+    l_arrow ->
+        - when: "->" tag: "L_ARROW"
+    l_dsign ->
+        - when: "$" tag: "L_DSIGN"
+    l_dash ->
+        - when: "-" tag: "L_DASH"
+    comment ->
+        - when: /\/\/[^\n]*/ tag: "T_COMMENT"
+    commentmulti ->
+        - when: /\/\*.*\*\// tag: "T_COMMENT"
+
+}}
+
+grammar: {{
+
+    main ->
+        _ section_list _ : {{ $1 }}
+
+    section_list ->
+        section : {{ [$0] }}
+        | section T_WS section_list : {{ [$0].concat($2) }}
+
+    section ->
+        K_CONFIG _ L_COLON _ L_TEMPLATEL _ kv_list _ L_TEMPLATER : {{ { config: Object.assign(...$4) } }}
+        | K_IMPORT _ L_STAR _ K_FROM __ T_WORD:import _ L_SCOLON : {{ { import: $import } }}
+        | K_IMPORT _ L_STAR _ K_FROM __ T_STRING:import _ L_SCOLON : {{ { import: $import, path: true } }}
+        | K_LEXER _ L_COLON _ L_TEMPLATEL _ lexer:lexer _ L_TEMPLATER : {{ { lexer: Object.assign(...$lexer) } }}
+        | K_GRAMMAR _ L_COLON _ L_TEMPLATEL _ grammar:grammar _ L_TEMPLATER : {{ { grammar: $grammar } }}
+        | K_BODY _ L_COLON _ T_JS:js : {{ { body: $js } }}
+        | K_BODY _ L_COLON _ T_STRING:js : {{ { body: $js, path: true } }}
+        | K_HEAD _ L_COLON _ T_JS:js : {{ { head: $js } }}
+        | K_HEAD _ L_COLON _ T_STRING:js : {{ { head: $js, path: true } }}
+
+    lexer ->
+        kv_list _ state_list : {{ $0.concat({ states: $2 }) }}
+        | state_list : {{ [{ states: $0 }] }}
+
+    state_list ->
+        state : {{ data }}
+        | state _ state_list : {{ [$0].concat($2) }}
+
+    state ->
+        state_declare _ state_definition : {{ Object.assign({ name: $0 }, $2) }}
+
+    state_declare ->
+        T_WORD _ L_ARROW : {{ $0 }}
+
+    state_definition ->
+        kv_list _ token_list : {{ Object.assign(...$0, { rules: $2 }) }}
+        | token_list : {{ { rules: $0 } }}
+
+    token_list ->
+        token : {{ data }}
+        | token _ token_list : {{ [$0].concat($2) }}
+
+    token ->
+        L_DASH _ K_IMPORT _ L_COLON _ word_list : {{ { import: $6 } }}
+        | L_DASH _ token_definition_list : {{ Object.assign(...$2) }}
+
+    token_definition_list ->
+        token_definition : {{ data }}
+        | token_definition _ token_definition_list : {{ [$0].concat($2) }}
+
+    token_definition ->
+        K_TAG _ L_COLON _ string_list : {{ { tag: $4 } }}
+        | K_WHEN _ L_COLON _ T_STRING : {{ { when: $4 } }}
+        | K_WHEN _ L_COLON _ T_REGEX : {{ { when: $4 } }}
+        | K_POP : {{ { pop: 1 } }}
+        | K_POP _ L_COLON _ T_INTEGER : {{ { pop: parseInt($4) } }}
+        | K_POP _ L_COLON _ K_ALL : {{ { pop: "all" } }}
+        | K_INSET : {{ { inset: 1 } }}
+        | K_INSET _ L_COLON _ T_INTEGER : {{ { inset: parseInt($4) } }}
+        | K_SET _ L_COLON _ T_WORD : {{ { set: $4 } }}
+        | K_GOTO _ L_COLON _ T_WORD : {{ { goto: $4 } }}
+        | K_TYPE _ L_COLON _ T_STRING : {{ { type: $4 } }}
+
+    grammar ->
+        kv_list _ grammar_rule_list : {{ { config: Object.assign(...$0), rules: $2 } }}
+        | grammar_rule_list : {{ { rules: $0 } }}
+
+    grammar_rule_list ->
+        grammar_rule : {{ [$0] }}
+        | grammar_rule _ grammar_rule_list : {{ [$0].concat($2) }}
+
+    grammar_rule ->
+        T_WORD _ L_ARROW _ expression_list : {{ { name: $0, expressions: $4 } }}
+        | T_WORD  __ L_COLON _ T_JS:js _ L_ARROW _ expression_list:expressions : {{ { name: $0, expressions: $expressions, postprocess: $js } }}
+        | T_WORD  __ L_COLON _ T_GRAMMAR_TEMPLATE:template _ L_ARROW _ expression_list:expressions : {{ { name: $0, expressions: $expressions, postprocess: $template } }}
+
+    expression_list ->
+        expression
+        | expression_list _ L_PIPE _ expression : {{ $0.concat([$4]) }}
+
+    expression ->
+        expression_symbol_list : {{ { symbols: $0 } }}
+        | expression_symbol_list __ L_COLON _ T_JS:js : {{ { symbols: $0, postprocess: $js } }}
+        | expression_symbol_list __ L_COLON _ T_GRAMMAR_TEMPLATE:template : {{ { symbols: $0, postprocess: $template } }}
+
+    expression_symbol_list ->
+        expression_symbol
+        | expression_symbol_list T_WS expression_symbol : {{ $0.concat([$2]) }}
+
+    
+    expression_symbol ->
+        expression_symbol_match : {{ $0 }}
+        | expression_symbol_match L_COLON T_WORD : {{ { ...$0,  alias: $2 } }}
+        | expression_symbol_match expression_repeater : {{ { expression: $0, repeat: $1 } }}
+        | expression_symbol_match expression_repeater L_COLON T_WORD : {{ { expression: $0, repeat: $1, alias: $4 } }}
+
+
+    expression_symbol_match ->
+        T_WORD : {{ { rule: $0 } }}
+        | T_STRING "i"? : {{ { literal: $0, insensitive: !!$1 } }}
+        | L_DSIGN T_WORD : {{ { token: $1} }}
+        | L_DSIGN T_STRING : {{ { token: $1} }}
+        | T_REGEX : {{ $0 }}
+        | L_PARENL _ expression_list _ L_PARENR : {{ { subexpression: $2 } }}
+        | T_JS : {{ $0 }}
+
+    expression_repeater : {{ $0[0].value }} ->
+        L_QMARK
+        | L_PLUS
+        | L_STAR
+
+    kv_list ->
+        kv : {{ data }}
+        | kv _ kv_list : {{ [$0].concat($2) }}
+
+    kv ->
+        T_WORD _ L_COLON _ ( T_WORD| T_STRING| T_INTEGER | T_JS | T_GRAMMAR_TEMPLATE) : {{ { [$0]: $4[0] } }}
+
+    string_list ->
+        T_STRING : {{ [$0] }}
+        | T_STRING _ L_COMMA _ string_list : {{ [$0].concat($4) }}
+
+    word_list ->
+        T_WORD : {{ [$0] }}
+        | T_WORD _ L_COMMA _ word_list : {{ [$0].concat($4) }}
+
+    _ ->
+        T_WS? : {{ null }}
+    __ ->
+        T_WS+ : {{ null }}
+
+    L_COLON -> $L_COLON
+    L_SCOLON -> $L_SCOLON
+    L_QMARK -> $L_QMARK
+    L_PLUS -> $L_PLUS
+    L_STAR -> $L_STAR
+    L_COMMA -> $L_COMMA
+    L_PIPE -> $L_PIPE
+    L_PARENL -> $L_PARENL
+    L_PARENR -> $L_PARENR
+    L_TEMPLATEL -> $L_TEMPLATEL
+    L_TEMPLATER -> $L_TEMPLATER
+    L_ARROW -> $L_ARROW
+    L_DSIGN -> $L_DSIGN
+    L_DASH -> $L_DASH
+
+    K_ALL -> "all"
+    K_TAG -> "tag"
+    K_FROM -> "from"
+    K_TYPE -> "type"
+    K_WHEN -> "when"
+    K_POP -> "pop"
+    K_INSET -> "inset"
+    K_SET -> "set"
+    K_GOTO -> "goto"
+    K_CONFIG -> "config"
+    K_LEXER -> "lexer"
+    K_GRAMMAR -> "grammar"
+    K_IMPORT -> "import"
+    K_BODY -> "body"
+    K_HEAD -> "head"
+
+    T_JS -> $L_JSL $T_JSBODY* $L_JSR : {{ { js: $1.map(v=>v.value).join('') } }}
+    T_GRAMMAR_TEMPLATE -> $L_TEMPLATEL _ $T_JSBODY* _ $L_TEMPLATER : {{ { template: $2.map(v=>v.value).join('').trim() } }}
+    T_STRING -> $T_STRING : {{ JSON.parse($0.value) }}
+    T_WORD -> $T_WORD : {{ $0.value }}
+    T_REGEX -> $T_REGEX /[gmiuy]/* : {{ { regex: $0.value.replace(/\\\\\//g,'/').slice(1,-1), flags: $1.join('') } }}
+    T_COMMENT -> $T_COMMENT
+    T_INTEGER -> $T_INTEGER : {{ $0.value }}
+    T_WS -> $T_WS : {{ null }}
+}}
+
+```

package/bootstrap.ts

@@ -0,0 +1,35 @@
+import { readdirSync, readFileSync, writeFileSync } from "fs";
+import { resolve } from "path";
+import { Compile } from "./src";
+const BaseDir = './src/grammars';
+
+(async () => {
+    const files = readdirSync(BaseDir);
+    for (const file of files) {
+        try {
+            console.log(fullpath(file))
+            if (/\.gwell$/.test(file)) {
+                const json = await Compile(read(file), { format: 'json' });
+                const js = await Compile(read(file), { exportName: 'grammar', format: 'esmodule' });
+                write(file.replace(/.gwell$/, '.json'), json);
+                write(file.replace(/.gwell$/, '.js'), js);
+            }
+        } catch (error) {
+            console.log(file);
+            console.log(error);
+            throw error;
+        }
+    }
+})();
+
+function read(filename) {
+    return readFileSync(fullpath(filename), 'utf-8')
+}
+
+function write(filename, body) {
+    return writeFileSync(fullpath(filename), body, 'utf8');
+}
+
+function fullpath(file: string) {
+    return resolve(BaseDir, file)
+}

package/build/compiler/compiler.d.ts

@@ -0,0 +1,48 @@
+import { CompileOptions, GrammarBuilderContext, OutputFormat, LanguageDirective } from "../typings";
+import { ESMOutput, JavascriptOutput } from "./outputs/javascript";
+import { TypescriptFormat } from "./outputs/typescript";
+import { JSONFormatter } from "./outputs/json";
+import { Generator } from "./generator";
+declare const OutputFormats: {
+    _default: typeof JavascriptOutput;
+    object: (grammar: any, exportName: any) => {
+        grammar: any;
+        exportName: any;
+    };
+    json: typeof JSONFormatter;
+    js: typeof JavascriptOutput;
+    javascript: typeof JavascriptOutput;
+    module: typeof ESMOutput;
+    esmodule: typeof ESMOutput;
+    ts: typeof TypescriptFormat;
+    typescript: typeof TypescriptFormat;
+};
+export declare function Compile(rules: string | LanguageDirective | (LanguageDirective[]), config?: CompileOptions): Promise<string | {
+    grammar: any;
+    exportName: any;
+}>;
+export declare class GrammarBuilder {
+    private config;
+    private parser;
+    private context;
+    generator: Generator;
+    constructor(config?: CompileOptions, context?: GrammarBuilderContext);
+    export<T extends OutputFormat = '_default'>(format: T, name?: string): ReturnType<typeof OutputFormats[T]>;
+    import(source: string): Promise<void>;
+    import(directive: LanguageDirective): Promise<void>;
+    import(directives: LanguageDirective[]): Promise<void>;
+    private processImportDirective;
+    private processConfigDirective;
+    private processGrammarDirective;
+    private processLexerDirective;
+    private importBuiltIn;
+    private importGrammar;
+    private mergeLanguageDefinitionString;
+    private buildRules;
+    private buildRule;
+    private buildSymbol;
+    private buildCharacterRules;
+    private buildSubExpressionRules;
+    private buildRepeatRules;
+}
+export {};

package/build/compiler/compiler.js

@@ -0,0 +1,227 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GrammarBuilder = exports.Compile = void 0;
+const parser_1 = require("../parser/parser");
+const import_resolver_1 = require("./import-resolver");
+const gwell_1 = require("../grammars/gwell");
+const javascript_1 = require("./outputs/javascript");
+const typescript_1 = require("./outputs/typescript");
+const json_1 = require("./outputs/json");
+const number = require("../grammars/number.json");
+const string = require("../grammars/string.json");
+const whitespace = require("../grammars/whitespace.json");
+const generator_1 = require("./generator");
+const BuiltInRegistry = {
+    number,
+    string,
+    whitespace,
+};
+const OutputFormats = {
+    _default: javascript_1.JavascriptOutput,
+    object: (grammar, exportName) => ({ grammar, exportName }),
+    json: json_1.JSONFormatter,
+    js: javascript_1.JavascriptOutput,
+    javascript: javascript_1.JavascriptOutput,
+    module: javascript_1.ESMOutput,
+    esmodule: javascript_1.ESMOutput,
+    ts: typescript_1.TypescriptFormat,
+    typescript: typescript_1.TypescriptFormat
+};
+async function Compile(rules, config = {}) {
+    const builder = new GrammarBuilder(config);
+    await builder.import(rules);
+    return builder.export(config.format);
+}
+exports.Compile = Compile;
+class GrammarBuilder {
+    config;
+    parser = new parser_1.Parser((0, gwell_1.default)());
+    context;
+    generator = new generator_1.Generator();
+    constructor(config = {}, context) {
+        this.config = config;
+        this.context = context || {
+            alreadyCompiled: new Set(),
+            resolver: config.resolverInstance ? config.resolverInstance : config.resolver ? new config.resolver(config.basedir) : new import_resolver_1.FileSystemResolver(config.basedir),
+            uuids: {}
+        };
+        this.generator.state.grammar.uuids = this.context.uuids;
+    }
+    export(format, name = 'GWLanguage') {
+        const grammar = this.generator.state;
+        const output = format || grammar.config.preprocessor || '_default';
+        if (OutputFormats[output]) {
+            return OutputFormats[output](this.generator, name);
+        }
+        throw new Error("No such preprocessor: " + output);
+    }
+    async import(directives) {
+        if (typeof directives == 'string') {
+            await this.mergeLanguageDefinitionString(directives);
+            return;
+        }
+        directives = Array.isArray(directives) ? directives : [directives];
+        for (const directive of directives) {
+            if ("head" in directive) {
+                this.generator.state.head.push(directive.head.js);
+            }
+            else if ("body" in directive) {
+                this.generator.state.body.push(directive.body.js);
+            }
+            else if ("import" in directive) {
+                await this.processImportDirective(directive);
+            }
+            else if ("config" in directive) {
+                this.processConfigDirective(directive);
+            }
+            else if ("grammar" in directive) {
+                this.processGrammarDirective(directive);
+            }
+            else if ("lexer" in directive) {
+                this.processLexerDirective(directive);
+            }
+        }
+    }
+    async processImportDirective(directive) {
+        if (directive.path) {
+            await this.importGrammar(directive.import);
+        }
+        else {
+            this.importBuiltIn(directive.import);
+        }
+    }
+    processConfigDirective(directive) {
+        Object.assign(this.generator.state.config, directive.config);
+    }
+    processGrammarDirective(directive) {
+        if (directive.grammar.config) {
+            this.generator.state.grammar.start = directive.grammar.config.start || this.generator.state.grammar.start;
+        }
+        for (const rule of directive.grammar.rules) {
+            this.buildRules(rule.name, rule.expressions, rule);
+            this.generator.state.grammar.start = this.generator.state.grammar.start || rule.name;
+        }
+    }
+    processLexerDirective(directive) {
+        if (!this.generator.state.lexer) {
+            this.generator.state.lexer = {
+                start: '',
+                states: {}
+            };
+        }
+        this.generator.state.lexer.start = directive.lexer.start || this.generator.state.lexer.start || (directive.lexer.states.length ? directive.lexer.states[0].name : '');
+        for (const state of directive.lexer.states) {
+            this.generator.addLexerState(state);
+        }
+    }
+    importBuiltIn(name) {
+        name = name.toLowerCase();
+        if (!this.context.alreadyCompiled.has(name)) {
+            this.context.alreadyCompiled.add(name);
+            if (!BuiltInRegistry[name])
+                return;
+            this.generator.merge(BuiltInRegistry[name].state);
+        }
+    }
+    async importGrammar(name) {
+        const resolver = this.context.resolver;
+        const path = resolver.path(name);
+        if (!this.context.alreadyCompiled.has(path)) {
+            this.context.alreadyCompiled.add(path);
+            await this.mergeLanguageDefinitionString(await resolver.body(path));
+        }
+    }
+    async mergeLanguageDefinitionString(body) {
+        const builder = new GrammarBuilder(this.config, this.context);
+        await builder.import(this.parser.run(body).results[0]);
+        this.generator.merge(builder.generator.state);
+        return;
+    }
+    buildRules(name, expressions, rule) {
+        for (const expression of expressions) {
+            this.generator.addGrammarRule(this.buildRule(name, expression, rule));
+        }
+    }
+    buildRule(name, expression, rule) {
+        const symbols = [];
+        for (let i = 0; i < expression.symbols.length; i++) {
+            const symbol = this.buildSymbol(name, expression.symbols[i]);
+            if (symbol)
+                symbols.push(symbol);
+        }
+        return { name, symbols, postprocess: expression.postprocess || rule?.postprocess };
+    }
+    buildSymbol(name, symbol) {
+        if ('repeat' in symbol) {
+            return this.buildRepeatRules(name, symbol);
+        }
+        if ('rule' in symbol) {
+            return symbol;
+        }
+        if ('regex' in symbol) {
+            return symbol;
+        }
+        if ('token' in symbol) {
+            return symbol;
+        }
+        if ('literal' in symbol) {
+            if (!symbol.literal.length) {
+                return null;
+            }
+            if (symbol.literal.length === 1 || this.generator.state.lexer) {
+                return symbol;
+            }
+            return this.buildCharacterRules(name, symbol);
+        }
+        if ('subexpression' in symbol) {
+            return this.buildSubExpressionRules(name, symbol);
+        }
+    }
+    buildCharacterRules(name, symbol) {
+        const id = this.generator.grammarUUID(name + "$STR");
+        this.buildRules(id, [
+            {
+                symbols: symbol.literal
+                    .split("")
+                    .map((literal) => {
+                    if (symbol.insensitive && literal.toLowerCase() != literal.toUpperCase())
+                        return { regex: literal, flags: 'i' };
+                    return { literal };
+                }),
+                postprocess: { builtin: "join" }
+            }
+        ]);
+        return { rule: id };
+    }
+    buildSubExpressionRules(name, symbol) {
+        const id = this.generator.grammarUUID(name + "$SUB");
+        this.buildRules(id, symbol.subexpression);
+        return { rule: id };
+    }
+    buildRepeatRules(name, symbol) {
+        let id;
+        const expr1 = { symbols: [] };
+        const expr2 = { symbols: [] };
+        if (symbol.repeat == '+') {
+            id = this.generator.grammarUUID(name + "$RPT1N");
+            expr1.symbols = [symbol.expression];
+            expr2.symbols = [{ rule: id }, symbol.expression];
+            expr2.postprocess = { builtin: "concat" };
+        }
+        else if (symbol.repeat == '*') {
+            id = this.generator.grammarUUID(name + "$RPT0N");
+            expr2.symbols = [{ rule: id }, symbol.expression];
+            expr2.postprocess = { builtin: "concat" };
+        }
+        else if (symbol.repeat == '?') {
+            id = this.generator.grammarUUID(name + "$RPT01");
+            expr1.symbols = [symbol.expression];
+            expr1.postprocess = { builtin: "first" };
+            expr2.postprocess = { builtin: "null" };
+        }
+        this.buildRules(id, [expr1, expr2]);
+        return { rule: id };
+    }
+}
+exports.GrammarBuilder = GrammarBuilder;
+//# sourceMappingURL=compiler.js.map

package/build/compiler/compiler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compiler.js","sourceRoot":"","sources":["../../src/compiler/compiler.ts"],"names":[],"mappings":";;;AAEA,6CAA0C;AAC1C,uDAAuD;AACvD,6CAAyC;AAEzC,qDAAmE;AACnE,qDAAwD;AACxD,yCAA+C;AAE/C,kDAAkD;AAClD,kDAAkD;AAClD,0DAA0D;AAC1D,2CAAwC;AAExC,MAAM,eAAe,GAAG;IACpB,MAAM;IACN,MAAM;IACN,UAAU;CACb,CAAA;AACD,MAAM,aAAa,GAAG;IAClB,QAAQ,EAAE,6BAAgB;IAC1B,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;IAC1D,IAAI,EAAE,oBAAa;IACnB,EAAE,EAAE,6BAAgB;IACpB,UAAU,EAAE,6BAAgB;IAC5B,MAAM,EAAE,sBAAS;IACjB,QAAQ,EAAE,sBAAS;IACnB,EAAE,EAAE,6BAAgB;IACpB,UAAU,EAAE,6BAAgB;CAC/B,CAAA;AAEM,KAAK,UAAU,OAAO,CAAC,KAAyD,EAAE,SAAyB,EAAE;IAChH,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,OAAO,CAAC,MAAM,CAAC,KAAY,CAAC,CAAC;IACnC,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACzC,CAAC;AAJD,0BAIC;AAED,MAAa,cAAc;IAMH;IALZ,MAAM,GAAG,IAAI,eAAM,CAAC,IAAA,eAAQ,GAAE,CAAC,CAAC;IAChC,OAAO,CAAwB;IAEvC,SAAS,GAAG,IAAI,qBAAS,EAAE,CAAC;IAE5B,YAAoB,SAAyB,EAAE,EAAE,OAA+B;QAA5D,WAAM,GAAN,MAAM,CAAqB;QAC3C,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI;YACtB,eAAe,EAAE,IAAI,GAAG,EAAE;YAC1B,QAAQ,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,oCAAkB,CAAC,MAAM,CAAC,OAAO,CAAC;YAC5J,KAAK,EAAE,EAAE;SACZ,CAAA;QACD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;IAC5D,CAAC;IAED,MAAM,CAAsC,MAAS,EAAE,OAAe,YAAY;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,YAAY,IAAI,UAAU,CAAC;QACnE,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE;YACvB,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;SACtD;QACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,MAAM,CAAC,CAAA;IACtD,CAAC;IAKD,KAAK,CAAC,MAAM,CAAC,UAA8D;QACvE,IAAI,OAAO,UAAU,IAAI,QAAQ,EAAE;YAC/B,MAAM,IAAI,CAAC,6BAA6B,CAAC,UAAU,CAAC,CAAC;YACrD,OAAO;SACV;QACD,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACnE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE;YAChC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACrD;iBAAM,IAAI,MAAM,IAAI,SAAS,EAAE;gBAC5B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACrD;iBAAM,IAAI,QAAQ,IAAI,SAAS,EAAE;gBAC9B,MAAM,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;aAChD;iBAAM,IAAI,QAAQ,IAAI,SAAS,EAAE;gBAC9B,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;aAC1C;iBAAM,IAAI,SAAS,IAAI,SAAS,EAAE;gBAC/B,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC;aAC3C;iBAAM,IAAI,OAAO,IAAI,SAAS,EAAE;gBAC7B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;aACzC;SACJ;IACL,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA0B;QAC3D,IAAI,SAAS,CAAC,IAAI,EAAE;YAChB,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;SAC9C;aAAM;YACH,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;SACxC;IACL,CAAC;IAEO,sBAAsB,CAAC,SAA0B;QACrD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IACjE,CAAC;IAEO,uBAAuB,CAAC,SAA2B;QACvD,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE;YAC1B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;SAC7G;QAED,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE;YACxC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YACnD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC;SACxF;IACL,CAAC;IAEO,qBAAqB,CAAC,SAAyB;QACnD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE;YAC7B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,GAAG;gBACzB,KAAK,EAAE,EAAE;gBACT,MAAM,EAAE,EAAE;aACb,CAAC;SACL;QACD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtK,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE;YACxC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;SACvC;IACL,CAAC;IAEO,aAAa,CAAC,IAAY;QAC9B,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YACzC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;gBACtB,OAAO;YACX,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;SACrD;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,IAAI;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QACvC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YACzC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;SACtE;IACL,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,IAAY;QACpD,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC9C,OAAO;IACX,CAAC;IAEO,UAAU,CAAC,IAAY,EAAE,WAAuC,EAAE,IAAyB;QAC/F,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;YAClC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;SACzE;IACL,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,UAAoC,EAAE,IAAyB;QAC3F,MAAM,OAAO,GAA6B,EAAE,CAAC;QAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAChD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7D,IAAI,MAAM;gBACN,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SAC5B;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,IAAI,EAAE,WAAW,EAAE,CAAC;IACvF,CAAC;IAEO,WAAW,CAAC,IAAY,EAAE,MAA4B;QAC1D,IAAI,QAAQ,IAAI,MAAM,EAAE;YACpB,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;SAC9C;QACD,IAAI,MAAM,IAAI,MAAM,EAAE;YAClB,OAAO,MAAM,CAAC;SACjB;QACD,IAAI,OAAO,IAAI,MAAM,EAAE;YACnB,OAAO,MAAM,CAAC;SACjB;QACD,IAAI,OAAO,IAAI,MAAM,EAAE;YACnB,OAAO,MAAM,CAAC;SACjB;QACD,IAAI,SAAS,IAAI,MAAM,EAAE;YACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE;gBACxB,OAAO,IAAI,CAAC;aACf;YACD,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE;gBAC3D,OAAO,MAAM,CAAC;aACjB;YACD,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;SACjD;QACD,IAAI,eAAe,IAAI,MAAM,EAAE;YAC3B,OAAO,IAAI,CAAC,uBAAuB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;SACrD;IACL,CAAC;IAEO,mBAAmB,CAAC,IAAY,EAAE,MAA0B;QAChE,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE;YAChB;gBACI,OAAO,EAAE,MAAM,CAAC,OAAO;qBAClB,KAAK,CAAC,EAAE,CAAC;qBACT,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;oBACb,IAAI,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,EAAE,IAAI,OAAO,CAAC,WAAW,EAAE;wBACpE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAA;oBACzC,OAAO,EAAE,OAAO,EAAE,CAAA;gBACtB,CAAC,CAAC;gBACN,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;aACnC;SACJ,CAAC,CAAC;QACH,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACxB,CAAC;IAEO,uBAAuB,CAAC,IAAY,EAAE,MAAyC;QACnF,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QAC1C,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACxB,CAAC;IAEO,gBAAgB,CAAC,IAAY,EAAE,MAAkC;QACrE,IAAI,EAAU,CAAC;QACf,MAAM,KAAK,GAA6B,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACxD,MAAM,KAAK,GAA6B,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACxD,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE;YACtB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC;YACjD,KAAK,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACpC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;YAClD,KAAK,CAAC,WAAW,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;SAC7C;aAAM,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE;YAC7B,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC;YACjD,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;YAClD,KAAK,CAAC,WAAW,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;SAC7C;aAAM,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE;YAC7B,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC;YACjD,KAAK,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACpC,KAAK,CAAC,WAAW,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;YACzC,KAAK,CAAC,WAAW,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACxB,CAAC;CACJ;AAxMD,wCAwMC"}

package/build/compiler/generator.d.ts

@@ -0,0 +1,23 @@
+import { GeneratorState, GeneratorGrammarRule, LexerStateDefinition } from "../typings";
+export declare class Generator {
+    state: GeneratorState;
+    serializeHead(): string;
+    serializeBody(): string;
+    serializeLanguage(depth?: number): string;
+    merge(state: GeneratorState): void;
+    grammarUUID(name: string): string;
+    addGrammarRule(rule: GeneratorGrammarRule): void;
+    addLexerState(state: LexerStateDefinition): void;
+    private serializeGrammar;
+    private serializeGrammarRules;
+    private serializeSymbol;
+    private serializeGrammarRule;
+    private serializePostProcess;
+    private templatePostProcess;
+    private serializeLexerConfig;
+    private serializeLexerConfigStates;
+    private serializeLexerConfigStateRules;
+    private newLine;
+    private pretty;
+    private isVal;
+}