grammar-well 1.3.3 → 2.0.1

package/src/compiler/compiler.ts

@@ -1,265 +0,0 @@
-import { CompileOptions, GrammarBuilderContext, TemplateFormat, LanguageDirective, ConfigDirective, GrammarBuilderSymbolRepeat, GrammarBuilderExpression, GeneratorGrammarRule, GrammarDirective, ImportDirective, LexerDirective, GrammarBuilderSymbolSubexpression, GrammarTypeLiteral, GeneratorGrammarSymbol, GrammarBuilderSymbol, GrammarBuilderRule } from "../typings";
-
-import { Parser } from "../parser/parser";
-import { FileSystemResolver } from "./import-resolver";
-import Language from './gwell';
-
-import { ESMOutput, JavascriptOutput } from "./outputs/javascript";
-import { TypescriptFormat } from "./outputs/typescript";
-import { JSONFormatter } from "./outputs/json";
-
-import { Generator } from "./generator/generator";
-import * as BuiltInRegistry from "./builtin.json"
-
-const TemplateFormats = {
-    _default: JavascriptOutput,
-    object: (grammar, exportName) => ({ grammar, exportName }),
-    json: JSONFormatter,
-    js: JavascriptOutput,
-    javascript: JavascriptOutput,
-    module: ESMOutput,
-    esmodule: ESMOutput,
-    esm: ESMOutput,
-    ts: TypescriptFormat,
-    typescript: TypescriptFormat
-}
-
-export async function Compile(rules: string | LanguageDirective | (LanguageDirective[]), config: CompileOptions = {}) {
-    const builder = new GrammarBuilder(config);
-    await builder.import(rules as any);
-    Object.assign(builder.generator.state.config, config.overrides);
-    return builder.export(config.template);
-}
-
-export class GrammarBuilder {
-    private parser = new Parser(Language() as any);
-    private context: GrammarBuilderContext;
-
-    generator = new Generator();
-
-    constructor(private config: CompileOptions = {}, context?: GrammarBuilderContext, private alias: string = '') {
-        this.context = context || {
-            alreadyCompiled: new Set(),
-            resolver: config.resolverInstance ? config.resolverInstance : config.resolver ? new config.resolver(config.basedir) : new FileSystemResolver(config.basedir),
-            uuids: {}
-        }
-        this.generator.state.grammar.uuids = this.context.uuids;
-    }
-
-    export<T extends TemplateFormat = '_default'>(format: T, name: string = 'GWLanguage'): ReturnType<typeof TemplateFormats[T]> {
-        const grammar = this.generator.state;
-        const output = format || grammar.config.preprocessor || '_default';
-        if (TemplateFormats[output]) {
-            return TemplateFormats[output](this.generator, name);
-        }
-        throw new Error("No such preprocessor: " + output)
-    }
-
-    async import(source: string): Promise<void>
-    async import(directive: LanguageDirective): Promise<void>
-    async import(directives: LanguageDirective[]): Promise<void>
-    async import(directives: string | LanguageDirective | (LanguageDirective[])): Promise<void> {
-        if (typeof directives == 'string') {
-            await this.mergeLanguageDefinitionString(directives);
-            return;
-        }
-        directives = Array.isArray(directives) ? directives : [directives];
-        for (const directive of directives) {
-            if ("head" in directive) {
-                this.generator.state.head.push(directive.head.js);
-            } else if ("body" in directive) {
-                this.generator.state.body.push(directive.body.js);
-            } else if ("import" in directive) {
-                await this.processImportDirective(directive);
-            } else if ("config" in directive) {
-                this.processConfigDirective(directive);
-            } else if ("grammar" in directive) {
-                this.processGrammarDirective(directive);
-            } else if ("lexer" in directive) {
-                this.processLexerDirective(directive);
-            }
-        }
-    }
-
-    private async processImportDirective(directive: ImportDirective) {
-        if (directive.path) {
-            await this.importGrammar(directive.import, this.alias + (directive.alias || ''));
-        } else {
-            await this.importBuiltIn(directive.import, this.alias + (directive.alias || ''));
-        }
-    }
-
-    private processConfigDirective(directive: ConfigDirective) {
-        Object.assign(this.generator.state.config, directive.config);
-    }
-
-    private processGrammarDirective(directive: GrammarDirective) {
-        if (directive.grammar.config) {
-            if (directive.grammar.config.start) {
-                this.generator.state.grammar.start = this.alias + directive.grammar.config.start;
-            }
-
-            Object.assign(this.generator.state.grammar.config, directive.grammar.config);
-            // this.generator.state.grammar.postprocessDefault =  directive.grammar.config.postprocessDefault || this.generator.state.grammar.postprocessDefault;
-            // this.generator.state.grammar.postprocessOverride =  directive.grammar.config.postprocessOverride || this.generator.state.grammar.postprocessOverride;
-        }
-
-        if (!this.generator.state.grammar.start && directive.grammar.rules.length) {
-            this.generator.state.grammar.start = this.alias + directive.grammar.rules[0].name;
-        }
-
-        for (const rule of directive.grammar.rules) {
-            rule.name = this.alias + rule.name;
-            this.buildRules(rule.name, rule.expressions, rule);
-        }
-    }
-
-    private processLexerDirective(directive: LexerDirective) {
-        if (!this.generator.state.lexer) {
-            this.generator.state.lexer = {
-                start: '',
-                states: {}
-            };
-        }
-        if (directive.lexer.start) {
-            this.generator.state.lexer.start = this.alias + directive.lexer.start;
-        }
-
-        if (!this.generator.state.lexer.start && directive.lexer.states.length) {
-            this.generator.state.lexer.start = this.alias + directive.lexer.states[0].name
-        }
-
-        for (const state of directive.lexer.states) {
-            state.name = this.alias + state.name;
-            if (this.alias) {
-                state.rules.forEach(v => {
-                    if ('import' in v) {
-                        v.import = v.import.map(v2 => this.alias + v2);
-                    }
-                    if ('set' in v) {
-                        v.set = this.alias + v.set;
-                    }
-                    if ('goto' in v) {
-                        v.goto = this.alias + v.goto;
-                    }
-                })
-            }
-            this.generator.addLexerState(state);
-        }
-    }
-
-    private async importBuiltIn(name: string, alias?: string) {
-        name = name.toLowerCase();
-        if (!this.context.alreadyCompiled.has(name)) {
-            this.context.alreadyCompiled.add(name);
-            if (!BuiltInRegistry[name])
-                return;
-            await this.mergeLanguageDefinitionString(BuiltInRegistry[name], alias);
-        }
-    }
-
-    private async importGrammar(path: string, alias?: string) {
-        const resolver = this.context.resolver;
-        const fullPath = resolver.path(path);
-        if (!this.context.alreadyCompiled.has(fullPath)) {
-            this.context.alreadyCompiled.add(fullPath);
-            await this.mergeLanguageDefinitionString(await resolver.body(fullPath), alias);
-        }
-    }
-
-    private async mergeLanguageDefinitionString(body: string, alias: string = '') {
-        const builder = new GrammarBuilder(this.config, this.context, alias);
-        await builder.import(this.parser.run(body).results[0]);
-        this.generator.merge(builder.generator.state);
-        return;
-    }
-
-    private buildRules(name: string, expressions: GrammarBuilderExpression[], rule?: GrammarBuilderRule) {
-        for (const expression of expressions) {
-            this.generator.addGrammarRule(this.buildRule(name, expression, rule));
-        }
-    }
-
-    private buildRule(name: string, expression: GrammarBuilderExpression, rule?: GrammarBuilderRule): GeneratorGrammarRule {
-        const symbols: GeneratorGrammarSymbol[] = [];
-        for (let i = 0; i < expression.symbols.length; i++) {
-            const symbol = this.buildSymbol(name, expression.symbols[i]);
-            if (symbol)
-                symbols.push(symbol);
-        }
-        return { name, symbols, postprocess: expression.postprocess || rule?.postprocess };
-    }
-
-    private buildSymbol(name: string, symbol: GrammarBuilderSymbol): GeneratorGrammarSymbol {
-        if ('repeat' in symbol) {
-            return this.buildRepeatRules(name, symbol);
-        }
-        if ('rule' in symbol) {
-            return { ...symbol, rule: this.alias + symbol.rule };
-        }
-        if ('regex' in symbol) {
-            return symbol;
-        }
-        if ('token' in symbol) {
-            return symbol;
-        }
-        if ('literal' in symbol) {
-            if (!symbol.literal.length) {
-                return null;
-            }
-            if (symbol.literal.length === 1 || this.generator.state.lexer) {
-                return symbol;
-            }
-            return this.buildCharacterRules(name, symbol);
-        }
-        if ('subexpression' in symbol) {
-            return this.buildSubExpressionRules(name, symbol);
-        }
-    }
-
-    private buildCharacterRules(name: string, symbol: GrammarTypeLiteral) {
-        const id = this.generator.grammarUUID(name + "$STR");
-        this.buildRules(id, [
-            {
-                symbols: symbol.literal
-                    .split("")
-                    .map((literal) => {
-                        if (symbol.insensitive && literal.toLowerCase() != literal.toUpperCase())
-                            return { regex: literal, flags: 'i' }
-                        return { literal }
-                    }),
-                postprocess: { builtin: "join" }
-            }
-        ]);
-        return { rule: id };
-    }
-
-    private buildSubExpressionRules(name: string, symbol: GrammarBuilderSymbolSubexpression) {
-        const id = this.generator.grammarUUID(name + "$SUB");
-        this.buildRules(id, symbol.subexpression);
-        return { rule: id };
-    }
-
-    private buildRepeatRules(name: string, symbol: GrammarBuilderSymbolRepeat) {
-        let id: string;
-        const expr1: GrammarBuilderExpression = { symbols: [] };
-        const expr2: GrammarBuilderExpression = { symbols: [] };
-        if (symbol.repeat == '+') {
-            id = this.generator.grammarUUID(name + "$RPT1N");
-            expr1.symbols = [symbol.expression];
-            expr2.symbols = [{ rule: id }, symbol.expression];
-            expr2.postprocess = { builtin: "concat" };
-        } else if (symbol.repeat == '*') {
-            id = this.generator.grammarUUID(name + "$RPT0N");
-            expr2.symbols = [{ rule: id }, symbol.expression];
-            expr2.postprocess = { builtin: "concat" };
-        } else if (symbol.repeat == '?') {
-            id = this.generator.grammarUUID(name + "$RPT01");
-            expr1.symbols = [symbol.expression];
-            expr1.postprocess = { builtin: "first" };
-            expr2.postprocess = { builtin: "null" };
-        }
-        this.buildRules(id, [expr1, expr2]);
-        return { rule: id };
-    }
-}

package/src/compiler/generator/artifacts/standard.ts

@@ -1,26 +0,0 @@
-import { GeneratorGrammarRule, } from "../../../typings";
-import { Generator } from "../generator";
-
-export class StandardGrammar {
-
-    constructor(private generator: Generator) { }
-
-    serialize(depth: number = 0) {
-        if (!this.generator.state.grammar) {
-            return null;
-        }
-        return Generator.Pretty({
-            start: JSON.stringify(this.generator.state.grammar.start),
-            rules: this.serializeGrammarRules(depth + 1)
-        }, depth);
-    }
-
-    private serializeGrammarRules(depth: number = 0) {
-        const map = {};
-        for (const rule in this.generator.state.grammar.rules) {
-            map[rule] = this.generator.state.grammar.rules[rule].map(v => this.generator.serializeGrammarRule(v))
-        }
-        return Generator.Pretty(map, depth);
-    }
-
-}

package/src/compiler/generator/generator.ts

@@ -1,237 +0,0 @@
-import { Dictionary, GeneratorState, GeneratorGrammarRule, GeneratorGrammarSymbol, LexerConfig, LexerStateDefinition } from "../../typings";
-import { LRParseTableBuilder } from "./artifacts/lr";
-import { StandardGrammar } from "./artifacts/standard";
-
-const PostProcessors = {
-    "join": "({data}) => data.join('')",
-    "concat": "({data}) => data[0].concat([data[1]])",
-    "null": "() => null",
-    "first": "({data}) => data[0]"
-};
-
-export class Generator {
-
-    state: GeneratorState = {
-        grammar: {
-            start: '',
-            config: {},
-            rules: {},
-            uuids: {},
-        },
-        lexer: null,
-        head: [],
-        body: [],
-        config: {},
-        version: 'unknown',
-    }
-
-    serializeHead() {
-        if (this.state.config.noscript)
-            return '';
-        return this.state.head.join('\n');
-    }
-
-    serializeBody() {
-        if (this.state.config.noscript)
-            return '';
-        return this.state.body.join('\n');
-    }
-
-    serializeLanguage(depth: number = 0) {
-        const serialier = new StandardGrammar(this);
-        let lr = null;
-
-        if ('lr' in this.state.config) {
-            const table = new LRParseTableBuilder(this);
-            lr = Generator.Pretty({
-                k: "0",
-                table: table.serialize(depth + 2)
-            }, depth + 1);
-        }
-
-        return Generator.Pretty({
-            grammar: serialier.serialize(depth + 1),
-            lexer: this.serializeLexerConfig(depth + 1),
-            lr
-        }, depth);
-    }
-
-    merge(state: GeneratorState) {
-        // TODO: Resolve Conflicting Rules and UUIDS
-        Object.assign(this.state.grammar.rules, state.grammar.rules);
-        this.state.grammar.start = state.grammar.start || this.state.grammar.start;
-        this.state.grammar.config.postprocessorDefault = state.grammar.config.postprocessorDefault || this.state.grammar.config.postprocessorDefault;
-        this.state.grammar.config.postprocessorOverride = state.grammar.config.postprocessorOverride || this.state.grammar.config.postprocessorOverride;
-
-        if (state.lexer) {
-            if (this.state.lexer) {
-                Object.assign(this.state.lexer.states, state.lexer.states);
-            } else {
-                this.state.lexer = state.lexer;
-            }
-            this.state.lexer.start = state.lexer.start || this.state.lexer.start;
-        }
-        this.state.head.push(...state.head);
-        this.state.body.push(...state.body);
-        Object.assign(this.state.config, state.config);
-    }
-
-    grammarUUID(name: string) {
-        this.state.grammar.uuids[name] = (this.state.grammar.uuids[name] || 0) + 1;
-        return name + 'x' + this.state.grammar.uuids[name];
-    }
-
-    addGrammarRule(rule: GeneratorGrammarRule) {
-        this.state.grammar.rules[rule.name] = this.state.grammar.rules[rule.name] || [];
-        this.state.grammar.rules[rule.name].push(rule);
-    }
-
-    addLexerState(state: LexerStateDefinition) {
-        this.state.lexer.states[state.name] = this.state.lexer.states[state.name] || { name: state.name, rules: [] }
-        const target = this.state.lexer.states[state.name];
-        target.default = typeof state.default == 'string' ? state.default : target.default;
-        target.unmatched = typeof state.unmatched == 'string' ? state.unmatched : target.unmatched;
-        target.rules.push(...state.rules);
-    }
-
-    serializePostProcess(postprocess: GeneratorGrammarRule['postprocess'], alias: Dictionary<number>) {
-        postprocess = this.state.grammar.config.postprocessorOverride || postprocess || this.state.grammar.config.postprocessorDefault;
-        if (!postprocess)
-            return null;
-        if ('builtin' in postprocess)
-            return PostProcessors[postprocess.builtin];
-        if (this.state.config.noscript)
-            return;
-        if (typeof postprocess == 'string')
-            return postprocess;
-        if ('js' in postprocess)
-            return postprocess.js;
-        if ('template' in postprocess)
-            return this.templatePostProcess(postprocess.template, alias);
-    }
-
-    private templatePostProcess(templateBody: string, alias: { [key: string]: number }) {
-        for (const key in alias) {
-            templateBody = templateBody.replace(new RegExp('(?:\\$)' + key + '(?![a-zA-Z\\d\\$_])'), `data[${alias[key]}]`);
-        }
-        return "({data}) => { return " + templateBody.replace(/\$(\d+)/g, "data[$1]") + "; }";
-    }
-
-    private serializeLexerConfig(depth: number = 0) {
-        if (!this.state.lexer)
-            return null;
-
-        if (typeof this.state.lexer === 'string')
-            return this.state.lexer;
-
-        return Generator.Pretty({
-            start: JSON.stringify(this.state.lexer.start),
-            states: this.serializeLexerConfigStates(depth + 1)
-        }, depth);
-    }
-
-    private serializeLexerConfigStates(depth: number) {
-        const map = {};
-        for (const key in this.state.lexer.states) {
-            const state = this.state.lexer.states[key];
-            map[state.name] = Generator.Pretty({
-                name: JSON.stringify(state.name),
-                default: state.default ? JSON.stringify(state.default) : null,
-                unmatched: state.unmatched ? JSON.stringify(state.unmatched) : null,
-                rules: this.serializeLexerConfigStateRules(state.rules, depth + 2)
-            }, depth + 1);
-        }
-        return Generator.Pretty(map, depth);
-    }
-
-    private serializeLexerConfigStateRules(rules: LexerConfig['states'][0]['rules'], depth: number) {
-        const ary = rules.map(rule => {
-            if ('import' in rule)
-                return Generator.Pretty({ import: JSON.stringify(rule.import) }, -1)
-            return Generator.Pretty({
-                when: Generator.SerializeSymbol(rule.when as any),
-                type: JSON.stringify(rule.type),
-                tag: JSON.stringify(rule.tag),
-                pop: JSON.stringify(rule.pop),
-                before: JSON.stringify(rule.before),
-                open: JSON.stringify(rule.open),
-                close: JSON.stringify(rule.close),
-                highlight: JSON.stringify(rule.highlight),
-                set: JSON.stringify(rule.set),
-                inset: JSON.stringify(rule.inset),
-                goto: JSON.stringify(rule.goto),
-            }, -1);
-        });
-        return Generator.Pretty(ary, depth);
-    }
-
-    serializeGrammarRule(rule: GeneratorGrammarRule) {
-        const symbols = [];
-        const alias = {};
-        for (let i = 0; i < rule.symbols.length; i++) {
-            symbols.push(Generator.SerializeSymbol(rule.symbols[i]));
-            if (rule.symbols[i].alias) {
-                alias[rule.symbols[i].alias] = i;
-            }
-        }
-
-        return Generator.Pretty({
-            name: JSON.stringify(rule.name),
-            symbols: Generator.Pretty(symbols, -1),
-            postprocess: this.serializePostProcess(rule.postprocess, alias)
-        }, -1);
-    }
-
-    static NewLine(depth: number) {
-        return '\n' + ' '.repeat(depth * 4);
-    }
-
-    static Pretty(obj: string[] | { [key: string]: string | (string[]) }, depth = 0) {
-        if (Array.isArray(obj)) {
-            let r = `[`;
-            for (let i = 0; i < obj.length; i++) {
-                const value = obj[i];
-                r += `${depth >= 0 ? Generator.NewLine(depth + 1) : ' '}${value}${(Generator.IsVal(obj[i + 1]) ? ',' : '')}`;
-            }
-            r += `${depth >= 0 ? Generator.NewLine(depth) : ' '}]`;
-            return r;
-        }
-
-        let r = `{`;
-        const keys = Object.keys(obj).filter(v => Generator.IsVal(obj[v]));
-        const prefix = depth >= 0 ? Generator.NewLine(depth + 1) : ' ';
-        for (let i = 0; i < keys.length; i++) {
-            const key = /[a-z_][a-z\d_$]*/i.test(keys[i]) ? keys[i] : JSON.stringify(keys[i]);
-            const value = Array.isArray(obj[keys[i]]) ? Generator.Pretty(obj[keys[i]] as string[], depth >= 0 ? depth + 1 : -1) : obj[keys[i]];
-            const suffix = (Generator.IsVal(obj[keys[i + 1]]) ? ',' : '');
-            r += `${prefix}${key}: ${value}${suffix}`;
-        }
-        r += `${depth >= 0 ? Generator.NewLine(depth) : ' '}}`;
-        return r;
-    }
-
-    static IsVal(value) {
-        return typeof value !== 'undefined' && value !== null;
-    }
-
-    static SerializeSymbol(s: GeneratorGrammarSymbol) {
-        if (typeof s === 'string') {
-            return JSON.stringify(s);
-        } else if ('rule' in s) {
-            return JSON.stringify(s.rule);
-        } else if ('regex' in s) {
-            return `/${s.regex}/${s.flags || ''}`;
-        } else if ('token' in s) {
-            return `{ token: ${JSON.stringify(s.token)} }`;
-        } else if ('literal' in s) {
-            return `{ literal: ${JSON.stringify(s.literal)} }`;
-        } else {
-            return JSON.stringify(s);
-        }
-    }
-
-    static SymbolIsTerminal(s: GeneratorGrammarSymbol) {
-        return !(typeof s === 'string' || 'rule' in s);
-    }
-
-}