grammar-well 1.3.2 → 2.0.0

package/build/typings.d.ts

@@ -1,226 +0,0 @@
-import { TokenBuffer } from "./lexers/token-buffer";
-import { ParserUtility } from "./parser/parser";
-export interface Dictionary<T> {
-    [key: string]: T;
-}
-export interface CompileOptions {
-    version?: string;
-    noscript?: boolean;
-    basedir?: string;
-    resolver?: ImportResolverConstructor;
-    resolverInstance?: ImportResolver;
-    exportName?: string;
-    template?: TemplateFormat;
-    overrides?: Dictionary<string>;
-}
-export type TemplateFormat = '_default' | 'object' | 'json' | 'js' | 'javascript' | 'module' | 'esmodule' | 'esm' | 'ts' | 'typescript';
-export interface GrammarBuilderContext {
-    alreadyCompiled: Set<string>;
-    resolver: ImportResolver;
-    uuids: Dictionary<number>;
-}
-export interface ImportResolver {
-    path(path: string): string;
-    body(path: string): Promise<string>;
-}
-export interface ImportResolverConstructor {
-    new (basePath: string): ImportResolver;
-}
-export type PostProcessor = (payload: PostProcessorPayload) => any;
-interface PostProcessorPayload {
-    data: any[];
-    rule: GrammarRule;
-    meta: any;
-}
-export type JavascriptDirective = {
-    body: GrammarTypeJS;
-} | {
-    head: GrammarTypeJS;
-};
-export interface ImportDirective {
-    import: string;
-    path?: boolean;
-    alias?: string;
-}
-export interface ConfigDirective {
-    config: Dictionary<any>;
-}
-export interface GrammarDirective {
-    grammar: {
-        config?: Dictionary<any>;
-        rules: GrammarBuilderRule[];
-    };
-}
-export interface LexerDirective {
-    lexer: {
-        start?: string;
-        states: LexerStateDefinition[];
-    };
-}
-export interface GrammarBuilderRule {
-    name: string;
-    expressions: GrammarBuilderExpression[];
-    postprocess?: GrammarTypeJS | GrammarTypeBuiltIn | GrammarTypeTemplate;
-}
-export interface GrammarBuilderExpression {
-    symbols: GrammarBuilderSymbol[];
-    postprocess?: GrammarTypeJS | GrammarTypeBuiltIn | GrammarTypeTemplate;
-}
-export type GrammarBuilderSymbol = GrammarTypeRule | GrammarTypeRegex | GrammarTypeToken | GrammarTypeLiteral | GrammarBuilderSymbolRepeat | GrammarBuilderSymbolSubexpression;
-export interface GrammarBuilderSymbolSubexpression {
-    subexpression: GrammarBuilderExpression[];
-}
-export interface GrammarBuilderSymbolRepeat {
-    expression: GrammarBuilderSymbol;
-    repeat: "+" | "*" | "?";
-}
-export interface GrammarTypeRule {
-    rule: string;
-}
-export interface GrammarTypeRegex {
-    regex: string;
-    flags?: string;
-}
-export interface GrammarTypeToken {
-    token: string;
-}
-export interface GrammarTypeLiteral {
-    literal: string;
-    insensitive?: boolean;
-}
-export type GrammarTypeBuiltIn = {
-    builtin: string;
-};
-export type GrammarTypeTemplate = {
-    template: string;
-};
-export type GrammarTypeJS = {
-    js: string;
-};
-export type ParserAlgorithm = ((language: LanguageDefinition & {
-    tokens: TokenBuffer;
-    utility: ParserUtility;
-}, options?: any) => {
-    results: any[];
-    info?: any;
-});
-export type LanguageDirective = (JavascriptDirective | ImportDirective | ConfigDirective | GrammarDirective | LexerDirective);
-type GrammarRuleSymbolFunction = (data: LexerToken) => boolean;
-export interface GrammarRule {
-    name: string;
-    symbols: GrammarRuleSymbol[];
-    postprocess?: PostProcessor;
-}
-export type GrammarRuleSymbol = string | RegExp | GrammarTypeLiteral | GrammarTypeToken | GrammarRuleSymbolFunction;
-export interface GeneratorGrammarRule {
-    name: string;
-    symbols: GeneratorGrammarSymbol[];
-    postprocess?: GrammarTypeTemplate | GrammarTypeBuiltIn | GrammarTypeJS;
-}
-export type GeneratorGrammarSymbol = {
-    alias?: string;
-} & (GrammarTypeRule | GrammarTypeRegex | GrammarTypeLiteral | GrammarTypeToken);
-export interface LanguageDefinition {
-    lexer?: Lexer | LexerConfig;
-    grammar: {
-        start: string;
-        rules: Dictionary<GrammarRule[]>;
-    };
-    lr?: {
-        k: number;
-        table: Dictionary<LRState>;
-    };
-}
-export interface LRState {
-    actions: Next[];
-    goto: {
-        [key: string]: string;
-    };
-    reduce?: GrammarRule;
-    isFinal: boolean;
-}
-type Next = {
-    symbol: GrammarRuleSymbol;
-    next: string;
-};
-export interface TQRestorePoint {
-    historyIndex: number;
-    offset: number;
-}
-export interface Lexer {
-    next(): LexerToken | undefined;
-    feed(chunk?: string, state?: ReturnType<Lexer['state']>): void;
-    state(): any;
-    flush?(): void;
-}
-export interface LexerToken {
-    type?: string | undefined;
-    tag?: Set<string>;
-    value: string;
-    offset: number;
-    line: number;
-    column: number;
-}
-export interface LexerStatus {
-    index: number;
-    line: number;
-    column: number;
-    state: string;
-}
-export interface LexerStateDefinition {
-    name: string;
-    unmatched?: string;
-    default?: string;
-    rules: (LexerStateImportRule | LexerStateMatchRule)[];
-}
-export interface LexerStateImportRule {
-    import: string[];
-}
-export interface LexerStateMatchRule {
-    when: string | RegExp;
-    type?: string;
-    tag?: string[];
-    pop?: number | 'all';
-    before?: boolean;
-    highlight?: string;
-    open?: string;
-    close?: string;
-    embed?: string;
-    unembed?: boolean;
-    inset?: number;
-    goto?: string;
-    set?: string;
-}
-export interface ResolvedStateDefinition {
-    name: string;
-    unmatched?: string;
-    rules: LexerStateMatchRule[];
-}
-export interface CompiledStateDefinition {
-    regexp: RegExp;
-    unmatched?: LexerStateMatchRule;
-    rules: LexerStateMatchRule[];
-}
-export interface LexerConfig {
-    start?: string;
-    states: Dictionary<LexerStateDefinition>;
-}
-export interface GeneratorState {
-    version: string;
-    config: Dictionary<string>;
-    head: string[];
-    body: string[];
-    lexer?: LexerConfig;
-    grammar: {
-        start: string;
-        config: {
-            postprocessorDefault?: GrammarTypeJS | GrammarTypeTemplate;
-            postprocessorOverride?: GrammarTypeJS | GrammarTypeTemplate;
-        };
-        rules: Dictionary<GeneratorGrammarRule[]>;
-        uuids: {
-            [key: string]: number;
-        };
-    };
-}
-export {};

package/build/typings.js

@@ -1,3 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=typings.js.map

package/build/typings.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"typings.js","sourceRoot":"","sources":["../src/typings.ts"],"names":[],"mappings":""}

package/src/compiler/builtin/json.gwell

@@ -1,74 +0,0 @@
-import * from whitespace;
-
-lexer: {{
-    start: "json"
-
-    json ->
-        - import: whitespace
-        - when: /-?(?:[0-9]|[1-9][0-9]+)(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+)?\b/ tag: "number"
-        - when: /"(?:\\["bfnrt\/\\]|\\u[a-fA-F0-9]{4}|[^"\\])*"/ tag: "string"
-        - when: "{" tag: "{"
-        - when: "}" tag: "}"
-        - when: "[" tag: "["
-        - when: "]" tag: "]"
-        - when: "," tag: ","
-        - when: ":" tag: ":"
-        - when: "true" tag: "true"
-        - when: "false" tag: "false"
-        - when: "null" tag: "null"
-}}
-
-grammar: {{
-    json -> _ (object | array) _ : {{ $1[0] }}
-
-    object -> "{" _ "}" : {{ {} }}
-        | "{" _ pair (_ "," _ pair)* _ "}" : ${ extractObject }
-
-    array -> "[" _ "]" : {{ [] }}
-        | "[" _ value (_ "," _ value)* _ "]" : ${ extractArray }
-
-    value : {{ $0 }} ->
-        object
-        | array
-        | number
-        | string
-        | "true" : {{ true }}
-        | "false" : {{ false }}
-        | "null" : {{ null }}
-
-    number -> $number : {{ parseFloat($0.value) }}
-
-    string -> $string : {{ JSON.parse($0.value) }}
-
-    pair -> key:k _ ":" _ value:v : {{ [$k, $v] }}
-
-    key -> string : {{ $0 }}
-}}
-
-head: ${
-    function extractPair(kv, output) {
-        if(kv[0]) { output[kv[0]] = kv[1]; }
-    }
-
-    function extractObject({data}) {
-        let output = {};
-
-        extractPair(data[2], output);
-
-        for (let i in data[3]) {
-            extractPair(data[3][i][3], output);
-        }
-
-        return output;
-    }
-
-    function extractArray({data}) {
-        let output = [data[2]];
-
-        for (let i in data[3]) {
-            output.push(data[3][i][3]);
-        }
-
-        return output;
-    }
-}

package/src/compiler/builtin/number.gwell

@@ -1,20 +0,0 @@
-grammar: {{
-
-    unsigned_int -> 
-        /[0-9]/+ : {{ parseInt($0.join("")) }}
-
-    int -> 
-        ("-"|"+")? /[0-9]/+ : {{ $0 ? parseInt($0[0]+$1.join("")) : parseInt($1.join("")) }}
-
-    unsigned_decimal -> 
-        /[0-9]/+ ("." /[0-9]/+)? : {{ parseFloat($0.join("") + ($1 ? "."+$1[1].join("") : "")) }}
-
-    decimal -> 
-        "-"? /[0-9]/+ ("." /[0-9]/+)? : {{ parseFloat( ($0 || "") + $1.join("") +($2 ? "."+$2[1].join("") : "")) }}
-
-    percentage -> 
-        decimal "%" : {{ $0/100 }}
-        
-    jsonfloat -> 
-        "-"? /[0-9]/+ ("." /[0-9]/+)? (/[eE]/ /[+-]/? /[0-9]/+)? : {{ parseFloat( ($0 || "") + $1.join("") + ($2 ? "."+$2[1].join("") : "") + ($3 ? "e" + ($3[1] || "+") + $3[2].join("") : "")) }}
-}}

package/src/compiler/builtin/string.gwell

@@ -1,48 +0,0 @@
-lexer: {{
-    string ->
-        - import: singleQuoteString, doubleQuoteString
-
-    doubleQuoteString ->
-        - when: /"/ tag: "dquote" highlight:"string" goto: doubleQuoteStringEnd 
-
-    singleQuoteString ->
-        - when: /'/ tag: "squote" highlight:"string" goto: singleQuoteStringEnd 
-
-    doubleQuoteStringEnd ->
-        - when: /\\[\\\/bnrft]/ tag: "escaped" highlight:"constant"
-        - when: /\\"/ tag: "quoteEscape"
-        - when: /\\u[A-Fa-f\d]{4}/ tag: "escaped" highlight:"constant"
-        - when: /\\./ tag:"badEscape"
-        - when: /[^"\\]+/ tag: "string" highlight:"string"
-        - when: "\"" tag: "dquote" highlight:"string" pop
-
-    singleQuoteStringEnd ->
-        - when: /\\[\\\/bnrft]/ tag: "escaped"
-        - when: /\\'/ tag: "quoteEscape"
-        - when: /\\u[A-Fa-f\d]{4}/ tag: "escaped" 
-        - when: /\\./ tag:"badEscape"
-        - when: /[^'\\]+/ tag: "string" highlight:"string"
-        - when: "'" tag: "squote" highlight:"string" pop
-}}
-
-grammar: {{
-    string : {{ $0 }} 
-        -> singleQuoteString 
-        | doubleQuoteString 
-
-    doubleQuoteString 
-        -> $dquote stringInner $dquote : {{ $1 }}
-        | $dquote $dquote : {{ '' }}
-    
-    singleQuoteString 
-        -> $squote stringInner $squote : {{ $1 }}
-        | $squote $squote : {{ '' }}
-
-    stringInner -> stringEscape : {{ $0 }}
-        | $string : {{  $0.value }}
-        | stringInner stringEscape : {{ $0 + $1 }}
-        | stringInner $string : {{ $0 + $1.value }}
-    
-    stringEscape -> $escaped : {{  JSON.parse('"' +$0.value + '"') }}
-        | $quoteEscape : {{ $0.value[1] }}
-}}

package/src/compiler/builtin/whitespace.gwell

@@ -1,10 +0,0 @@
-lexer: {{
-    start: "whitespace"
-    whitespace ->
-        - when: /\s+/ tag: "whitespace" 
-}}
-
-grammar: {{
-    _  -> $whitespace* : {{ null }}
-    __ -> $whitespace+ : {{ null }}
-}}

package/src/compiler/builtin.json

@@ -1 +0,0 @@
-{"json":"import * from whitespace;\r\n\r\nlexer: {{\r\n    start: \"json\"\r\n\r\n    json ->\r\n        - import: whitespace\r\n        - when: /-?(?:[0-9]|[1-9][0-9]+)(?:\\.[0-9]+)?(?:[eE][-+]?[0-9]+)?\\b/ tag: \"number\"\r\n        - when: /\"(?:\\\\[\"bfnrt\\/\\\\]|\\\\u[a-fA-F0-9]{4}|[^\"\\\\])*\"/ tag: \"string\"\r\n        - when: \"{\" tag: \"{\"\r\n        - when: \"}\" tag: \"}\"\r\n        - when: \"[\" tag: \"[\"\r\n        - when: \"]\" tag: \"]\"\r\n        - when: \",\" tag: \",\"\r\n        - when: \":\" tag: \":\"\r\n        - when: \"true\" tag: \"true\"\r\n        - when: \"false\" tag: \"false\"\r\n        - when: \"null\" tag: \"null\"\r\n}}\r\n\r\ngrammar: {{\r\n    json -> _ (object | array) _ : {{ $1[0] }}\r\n\r\n    object -> \"{\" _ \"}\" : {{ {} }}\r\n        | \"{\" _ pair (_ \",\" _ pair)* _ \"}\" : ${ extractObject }\r\n\r\n    array -> \"[\" _ \"]\" : {{ [] }}\r\n        | \"[\" _ value (_ \",\" _ value)* _ \"]\" : ${ extractArray }\r\n\r\n    value : {{ $0 }} ->\r\n        object\r\n        | array\r\n        | number\r\n        | string\r\n        | \"true\" : {{ true }}\r\n        | \"false\" : {{ false }}\r\n        | \"null\" : {{ null }}\r\n\r\n    number -> $number : {{ parseFloat($0.value) }}\r\n\r\n    string -> $string : {{ JSON.parse($0.value) }}\r\n\r\n    pair -> key:k _ \":\" _ value:v : {{ [$k, $v] }}\r\n\r\n    key -> string : {{ $0 }}\r\n}}\r\n\r\nhead: ${\r\n    function extractPair(kv, output) {\r\n        if(kv[0]) { output[kv[0]] = kv[1]; }\r\n    }\r\n\r\n    function extractObject({data}) {\r\n        let output = {};\r\n\r\n        extractPair(data[2], output);\r\n\r\n        for (let i in data[3]) {\r\n            extractPair(data[3][i][3], output);\r\n        }\r\n\r\n        return output;\r\n    }\r\n\r\n    function extractArray({data}) {\r\n        let output = [data[2]];\r\n\r\n        for (let i in data[3]) {\r\n            output.push(data[3][i][3]);\r\n        }\r\n\r\n        return output;\r\n    }\r\n}\r\n","number":"grammar: {{\r\n\r\n    unsigned_int -> \r\n        /[0-9]/+ : {{ parseInt($0.join(\"\")) }}\r\n\r\n    int -> \r\n        (\"-\"|\"+\")? /[0-9]/+ : {{ $0 ? parseInt($0[0]+$1.join(\"\")) : parseInt($1.join(\"\")) }}\r\n\r\n    unsigned_decimal -> \r\n        /[0-9]/+ (\".\" /[0-9]/+)? : {{ parseFloat($0.join(\"\") + ($1 ? \".\"+$1[1].join(\"\") : \"\")) }}\r\n\r\n    decimal -> \r\n        \"-\"? /[0-9]/+ (\".\" /[0-9]/+)? : {{ parseFloat( ($0 || \"\") + $1.join(\"\") +($2 ? \".\"+$2[1].join(\"\") : \"\")) }}\r\n\r\n    percentage -> \r\n        decimal \"%\" : {{ $0/100 }}\r\n        \r\n    jsonfloat -> \r\n        \"-\"? /[0-9]/+ (\".\" /[0-9]/+)? (/[eE]/ /[+-]/? /[0-9]/+)? : {{ parseFloat( ($0 || \"\") + $1.join(\"\") + ($2 ? \".\"+$2[1].join(\"\") : \"\") + ($3 ? \"e\" + ($3[1] || \"+\") + $3[2].join(\"\") : \"\")) }}\r\n}}","string":"lexer: {{\r\n    string ->\r\n        - import: singleQuoteString, doubleQuoteString\r\n\r\n    doubleQuoteString ->\r\n        - when: /\"/ tag: \"dquote\" highlight:\"string\" goto: doubleQuoteStringEnd \r\n\r\n    singleQuoteString ->\r\n        - when: /'/ tag: \"squote\" highlight:\"string\" goto: singleQuoteStringEnd \r\n\r\n    doubleQuoteStringEnd ->\r\n        - when: /\\\\[\\\\\\/bnrft]/ tag: \"escaped\" highlight:\"constant\"\r\n        - when: /\\\\\"/ tag: \"quoteEscape\"\r\n        - when: /\\\\u[A-Fa-f\\d]{4}/ tag: \"escaped\" highlight:\"constant\"\r\n        - when: /\\\\./ tag:\"badEscape\"\r\n        - when: /[^\"\\\\]+/ tag: \"string\" highlight:\"string\"\r\n        - when: \"\\\"\" tag: \"dquote\" highlight:\"string\" pop\r\n\r\n    singleQuoteStringEnd ->\r\n        - when: /\\\\[\\\\\\/bnrft]/ tag: \"escaped\"\r\n        - when: /\\\\'/ tag: \"quoteEscape\"\r\n        - when: /\\\\u[A-Fa-f\\d]{4}/ tag: \"escaped\" \r\n        - when: /\\\\./ tag:\"badEscape\"\r\n        - when: /[^'\\\\]+/ tag: \"string\" highlight:\"string\"\r\n        - when: \"'\" tag: \"squote\" highlight:\"string\" pop\r\n}}\r\n\r\ngrammar: {{\r\n    string : {{ $0 }} \r\n        -> singleQuoteString \r\n        | doubleQuoteString \r\n\r\n    doubleQuoteString \r\n        -> $dquote stringInner $dquote : {{ $1 }}\r\n        | $dquote $dquote : {{ '' }}\r\n    \r\n    singleQuoteString \r\n        -> $squote stringInner $squote : {{ $1 }}\r\n        | $squote $squote : {{ '' }}\r\n\r\n    stringInner -> stringEscape : {{ $0 }}\r\n        | $string : {{  $0.value }}\r\n        | stringInner stringEscape : {{ $0 + $1 }}\r\n        | stringInner $string : {{ $0 + $1.value }}\r\n    \r\n    stringEscape -> $escaped : {{  JSON.parse('\"' +$0.value + '\"') }}\r\n        | $quoteEscape : {{ $0.value[1] }}\r\n}}","whitespace":"lexer: {{\r\n    start: \"whitespace\"\r\n    whitespace ->\r\n        - when: /\\s+/ tag: \"whitespace\" \r\n}}\r\n\r\ngrammar: {{\r\n    _  -> $whitespace* : {{ null }}\r\n    __ -> $whitespace+ : {{ null }}\r\n}}"}

package/src/compiler/compiler.ts

@@ -1,265 +0,0 @@
-import { CompileOptions, GrammarBuilderContext, TemplateFormat, LanguageDirective, ConfigDirective, GrammarBuilderSymbolRepeat, GrammarBuilderExpression, GeneratorGrammarRule, GrammarDirective, ImportDirective, LexerDirective, GrammarBuilderSymbolSubexpression, GrammarTypeLiteral, GeneratorGrammarSymbol, GrammarBuilderSymbol, GrammarBuilderRule } from "../typings";
-
-import { Parser } from "../parser/parser";
-import { FileSystemResolver } from "./import-resolver";
-import Language from './gwell';
-
-import { ESMOutput, JavascriptOutput } from "./outputs/javascript";
-import { TypescriptFormat } from "./outputs/typescript";
-import { JSONFormatter } from "./outputs/json";
-
-import { Generator } from "./generator/generator";
-import * as BuiltInRegistry from "./builtin.json"
-
-const TemplateFormats = {
-    _default: JavascriptOutput,
-    object: (grammar, exportName) => ({ grammar, exportName }),
-    json: JSONFormatter,
-    js: JavascriptOutput,
-    javascript: JavascriptOutput,
-    module: ESMOutput,
-    esmodule: ESMOutput,
-    esm: ESMOutput,
-    ts: TypescriptFormat,
-    typescript: TypescriptFormat
-}
-
-export async function Compile(rules: string | LanguageDirective | (LanguageDirective[]), config: CompileOptions = {}) {
-    const builder = new GrammarBuilder(config);
-    await builder.import(rules as any);
-    Object.assign(builder.generator.state.config, config.overrides);
-    return builder.export(config.template);
-}
-
-export class GrammarBuilder {
-    private parser = new Parser(Language() as any);
-    private context: GrammarBuilderContext;
-
-    generator = new Generator();
-
-    constructor(private config: CompileOptions = {}, context?: GrammarBuilderContext, private alias: string = '') {
-        this.context = context || {
-            alreadyCompiled: new Set(),
-            resolver: config.resolverInstance ? config.resolverInstance : config.resolver ? new config.resolver(config.basedir) : new FileSystemResolver(config.basedir),
-            uuids: {}
-        }
-        this.generator.state.grammar.uuids = this.context.uuids;
-    }
-
-    export<T extends TemplateFormat = '_default'>(format: T, name: string = 'GWLanguage'): ReturnType<typeof TemplateFormats[T]> {
-        const grammar = this.generator.state;
-        const output = format || grammar.config.preprocessor || '_default';
-        if (TemplateFormats[output]) {
-            return TemplateFormats[output](this.generator, name);
-        }
-        throw new Error("No such preprocessor: " + output)
-    }
-
-    async import(source: string): Promise<void>
-    async import(directive: LanguageDirective): Promise<void>
-    async import(directives: LanguageDirective[]): Promise<void>
-    async import(directives: string | LanguageDirective | (LanguageDirective[])): Promise<void> {
-        if (typeof directives == 'string') {
-            await this.mergeLanguageDefinitionString(directives);
-            return;
-        }
-        directives = Array.isArray(directives) ? directives : [directives];
-        for (const directive of directives) {
-            if ("head" in directive) {
-                this.generator.state.head.push(directive.head.js);
-            } else if ("body" in directive) {
-                this.generator.state.body.push(directive.body.js);
-            } else if ("import" in directive) {
-                await this.processImportDirective(directive);
-            } else if ("config" in directive) {
-                this.processConfigDirective(directive);
-            } else if ("grammar" in directive) {
-                this.processGrammarDirective(directive);
-            } else if ("lexer" in directive) {
-                this.processLexerDirective(directive);
-            }
-        }
-    }
-
-    private async processImportDirective(directive: ImportDirective) {
-        if (directive.path) {
-            await this.importGrammar(directive.import, this.alias + (directive.alias || ''));
-        } else {
-            await this.importBuiltIn(directive.import, this.alias + (directive.alias || ''));
-        }
-    }
-
-    private processConfigDirective(directive: ConfigDirective) {
-        Object.assign(this.generator.state.config, directive.config);
-    }
-
-    private processGrammarDirective(directive: GrammarDirective) {
-        if (directive.grammar.config) {
-            if (directive.grammar.config.start) {
-                this.generator.state.grammar.start = this.alias + directive.grammar.config.start;
-            }
-
-            Object.assign(this.generator.state.grammar.config, directive.grammar.config);
-            // this.generator.state.grammar.postprocessDefault =  directive.grammar.config.postprocessDefault || this.generator.state.grammar.postprocessDefault;
-            // this.generator.state.grammar.postprocessOverride =  directive.grammar.config.postprocessOverride || this.generator.state.grammar.postprocessOverride;
-        }
-
-        if (!this.generator.state.grammar.start && directive.grammar.rules.length) {
-            this.generator.state.grammar.start = this.alias + directive.grammar.rules[0].name;
-        }
-
-        for (const rule of directive.grammar.rules) {
-            rule.name = this.alias + rule.name;
-            this.buildRules(rule.name, rule.expressions, rule);
-        }
-    }
-
-    private processLexerDirective(directive: LexerDirective) {
-        if (!this.generator.state.lexer) {
-            this.generator.state.lexer = {
-                start: '',
-                states: {}
-            };
-        }
-        if (directive.lexer.start) {
-            this.generator.state.lexer.start = this.alias + directive.lexer.start;
-        }
-
-        if (!this.generator.state.lexer.start && directive.lexer.states.length) {
-            this.generator.state.lexer.start = this.alias + directive.lexer.states[0].name
-        }
-
-        for (const state of directive.lexer.states) {
-            state.name = this.alias + state.name;
-            if (this.alias) {
-                state.rules.forEach(v => {
-                    if ('import' in v) {
-                        v.import = v.import.map(v2 => this.alias + v2);
-                    }
-                    if ('set' in v) {
-                        v.set = this.alias + v.set;
-                    }
-                    if ('goto' in v) {
-                        v.goto = this.alias + v.goto;
-                    }
-                })
-            }
-            this.generator.addLexerState(state);
-        }
-    }
-
-    private async importBuiltIn(name: string, alias?: string) {
-        name = name.toLowerCase();
-        if (!this.context.alreadyCompiled.has(name)) {
-            this.context.alreadyCompiled.add(name);
-            if (!BuiltInRegistry[name])
-                return;
-            await this.mergeLanguageDefinitionString(BuiltInRegistry[name], alias);
-        }
-    }
-
-    private async importGrammar(path: string, alias?: string) {
-        const resolver = this.context.resolver;
-        const fullPath = resolver.path(path);
-        if (!this.context.alreadyCompiled.has(fullPath)) {
-            this.context.alreadyCompiled.add(fullPath);
-            await this.mergeLanguageDefinitionString(await resolver.body(fullPath), alias);
-        }
-    }
-
-    private async mergeLanguageDefinitionString(body: string, alias: string = '') {
-        const builder = new GrammarBuilder(this.config, this.context, alias);
-        await builder.import(this.parser.run(body).results[0]);
-        this.generator.merge(builder.generator.state);
-        return;
-    }
-
-    private buildRules(name: string, expressions: GrammarBuilderExpression[], rule?: GrammarBuilderRule) {
-        for (const expression of expressions) {
-            this.generator.addGrammarRule(this.buildRule(name, expression, rule));
-        }
-    }
-
-    private buildRule(name: string, expression: GrammarBuilderExpression, rule?: GrammarBuilderRule): GeneratorGrammarRule {
-        const symbols: GeneratorGrammarSymbol[] = [];
-        for (let i = 0; i < expression.symbols.length; i++) {
-            const symbol = this.buildSymbol(name, expression.symbols[i]);
-            if (symbol)
-                symbols.push(symbol);
-        }
-        return { name, symbols, postprocess: expression.postprocess || rule?.postprocess };
-    }
-
-    private buildSymbol(name: string, symbol: GrammarBuilderSymbol): GeneratorGrammarSymbol {
-        if ('repeat' in symbol) {
-            return this.buildRepeatRules(name, symbol);
-        }
-        if ('rule' in symbol) {
-            return { ...symbol, rule: this.alias + symbol.rule };
-        }
-        if ('regex' in symbol) {
-            return symbol;
-        }
-        if ('token' in symbol) {
-            return symbol;
-        }
-        if ('literal' in symbol) {
-            if (!symbol.literal.length) {
-                return null;
-            }
-            if (symbol.literal.length === 1 || this.generator.state.lexer) {
-                return symbol;
-            }
-            return this.buildCharacterRules(name, symbol);
-        }
-        if ('subexpression' in symbol) {
-            return this.buildSubExpressionRules(name, symbol);
-        }
-    }
-
-    private buildCharacterRules(name: string, symbol: GrammarTypeLiteral) {
-        const id = this.generator.grammarUUID(name + "$STR");
-        this.buildRules(id, [
-            {
-                symbols: symbol.literal
-                    .split("")
-                    .map((literal) => {
-                        if (symbol.insensitive && literal.toLowerCase() != literal.toUpperCase())
-                            return { regex: literal, flags: 'i' }
-                        return { literal }
-                    }),
-                postprocess: { builtin: "join" }
-            }
-        ]);
-        return { rule: id };
-    }
-
-    private buildSubExpressionRules(name: string, symbol: GrammarBuilderSymbolSubexpression) {
-        const id = this.generator.grammarUUID(name + "$SUB");
-        this.buildRules(id, symbol.subexpression);
-        return { rule: id };
-    }
-
-    private buildRepeatRules(name: string, symbol: GrammarBuilderSymbolRepeat) {
-        let id: string;
-        const expr1: GrammarBuilderExpression = { symbols: [] };
-        const expr2: GrammarBuilderExpression = { symbols: [] };
-        if (symbol.repeat == '+') {
-            id = this.generator.grammarUUID(name + "$RPT1N");
-            expr1.symbols = [symbol.expression];
-            expr2.symbols = [{ rule: id }, symbol.expression];
-            expr2.postprocess = { builtin: "concat" };
-        } else if (symbol.repeat == '*') {
-            id = this.generator.grammarUUID(name + "$RPT0N");
-            expr2.symbols = [{ rule: id }, symbol.expression];
-            expr2.postprocess = { builtin: "concat" };
-        } else if (symbol.repeat == '?') {
-            id = this.generator.grammarUUID(name + "$RPT01");
-            expr1.symbols = [symbol.expression];
-            expr1.postprocess = { builtin: "first" };
-            expr2.postprocess = { builtin: "null" };
-        }
-        this.buildRules(id, [expr1, expr2]);
-        return { rule: id };
-    }
-}