grammar-well 1.1.0

package/licenses/nearley.license

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014, 2015, 2016, 2017, 2018, 2019 Kartik Chandra, Tim Radvan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "grammar-well",
+  "version": "1.1.0",
+  "description": "Cross-platform Parser written in TypeScript",
+  "main": "build/index.js",
+  "scripts": {
+    "prebuild": "tsc --build --clean",
+    "build": "tsc --build",
+    "bootstrap": "ts-node bootstrap.ts",
+    "testing": "ts-node testing.ts",
+    "benchmark": "node tests/performance/benchmark.js",
+    "test": "nyc mocha tests/**/*.spec.ts",
+    "profile": "node tests/performance/profile.js"
+  },
+  "author": "0x6563",
+  "license": "LGPL-3.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/0x6563/grammar-well.git"
+  },
+  "bugs": {
+    "url": "https://github.com/0x6563/grammar-well/issues"
+  },
+  "homepage": "https://github.com/0x6563/grammar-well#readme",
+  "devDependencies": {
+    "@types/chai": "^4.3.4",
+    "@types/mocha": "^10.0.1",
+    "@types/node": "^18.11.18",
+    "@typescript-eslint/eslint-plugin": "^5.49.0",
+    "@typescript-eslint/parser": "^5.49.0",
+    "babel-cli": "^6.26.0",
+    "babel-preset-env": "^1.7.0",
+    "benny": "^3.7.1",
+    "chai": "^4.3.7",
+    "eslint": "^8.33.0",
+    "expect": "^29.4.1",
+    "mocha": "^10.2.0",
+    "nyc": "^15.1.0",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.4",
+    "yaml": "^2.2.1"
+  },
+  "keywords": [
+    "parser",
+    "parse",
+    "generator",
+    "compiler",
+    "compile",
+    "grammar",
+    "language"
+  ]
+}

package/src/compiler/compiler.ts

@@ -0,0 +1,239 @@
+import { CompileOptions, GrammarBuilderContext, OutputFormat, LanguageDirective, ConfigDirective, GrammarBuilderSymbolRepeat, GrammarBuilderExpression, GeneratorGrammarRule, GrammarDirective, ImportDirective, LexerDirective, GrammarBuilderSymbolSubexpression, GrammarTypeLiteral, GeneratorGrammarSymbol, GrammarBuilderSymbol, GrammarBuilderRule } from "../typings";
+
+import { Parser } from "../parser/parser";
+import { FileSystemResolver } from "./import-resolver";
+import Language from '../grammars/gwell';
+
+import { ESMOutput, JavascriptOutput } from "./outputs/javascript";
+import { TypescriptFormat } from "./outputs/typescript";
+import { JSONFormatter } from "./outputs/json";
+
+import * as number from '../grammars/number.json';
+import * as string from '../grammars/string.json';
+import * as whitespace from '../grammars/whitespace.json';
+import { Generator } from "./generator";
+
+const BuiltInRegistry = {
+    number,
+    string,
+    whitespace,
+}
+const OutputFormats = {
+    _default: JavascriptOutput,
+    object: (grammar, exportName) => ({ grammar, exportName }),
+    json: JSONFormatter,
+    js: JavascriptOutput,
+    javascript: JavascriptOutput,
+    module: ESMOutput,
+    esmodule: ESMOutput,
+    ts: TypescriptFormat,
+    typescript: TypescriptFormat
+}
+
+export async function Compile(rules: string | LanguageDirective | (LanguageDirective[]), config: CompileOptions = {}) {
+    const builder = new GrammarBuilder(config);
+    await builder.import(rules as any);
+    return builder.export(config.format);
+}
+
+export class GrammarBuilder {
+    private parser = new Parser(Language());
+    private context: GrammarBuilderContext;
+
+    generator = new Generator();
+
+    constructor(private config: CompileOptions = {}, context?: GrammarBuilderContext) {
+        this.context = context || {
+            alreadyCompiled: new Set(),
+            resolver: config.resolverInstance ? config.resolverInstance : config.resolver ? new config.resolver(config.basedir) : new FileSystemResolver(config.basedir),
+            uuids: {}
+        }
+        this.generator.state.grammar.uuids = this.context.uuids;
+    }
+
+    export<T extends OutputFormat = '_default'>(format: T, name: string = 'GWLanguage'): ReturnType<typeof OutputFormats[T]> {
+        const grammar = this.generator.state;
+        const output = format || grammar.config.preprocessor || '_default';
+        if (OutputFormats[output]) {
+            return OutputFormats[output](this.generator, name);
+        }
+        throw new Error("No such preprocessor: " + output)
+    }
+
+    async import(source: string): Promise<void>
+    async import(directive: LanguageDirective): Promise<void>
+    async import(directives: LanguageDirective[]): Promise<void>
+    async import(directives: string | LanguageDirective | (LanguageDirective[])): Promise<void> {
+        if (typeof directives == 'string') {
+            await this.mergeLanguageDefinitionString(directives);
+            return;
+        }
+        directives = Array.isArray(directives) ? directives : [directives];
+        for (const directive of directives) {
+            if ("head" in directive) {
+                this.generator.state.head.push(directive.head.js);
+            } else if ("body" in directive) {
+                this.generator.state.body.push(directive.body.js);
+            } else if ("import" in directive) {
+                await this.processImportDirective(directive);
+            } else if ("config" in directive) {
+                this.processConfigDirective(directive);
+            } else if ("grammar" in directive) {
+                this.processGrammarDirective(directive);
+            } else if ("lexer" in directive) {
+                this.processLexerDirective(directive);
+            }
+        }
+    }
+
+    private async processImportDirective(directive: ImportDirective) {
+        if (directive.path) {
+            await this.importGrammar(directive.import);
+        } else {
+            this.importBuiltIn(directive.import);
+        }
+    }
+
+    private processConfigDirective(directive: ConfigDirective) {
+        Object.assign(this.generator.state.config, directive.config);
+    }
+
+    private processGrammarDirective(directive: GrammarDirective) {
+        if (directive.grammar.config) {
+            this.generator.state.grammar.start = directive.grammar.config.start || this.generator.state.grammar.start;
+        }
+
+        for (const rule of directive.grammar.rules) {
+            this.buildRules(rule.name, rule.expressions, rule);
+            this.generator.state.grammar.start = this.generator.state.grammar.start || rule.name;
+        }
+    }
+
+    private processLexerDirective(directive: LexerDirective) {
+        if (!this.generator.state.lexer) {
+            this.generator.state.lexer = {
+                start: '',
+                states: {}
+            };
+        }
+        this.generator.state.lexer.start = directive.lexer.start || this.generator.state.lexer.start || (directive.lexer.states.length ? directive.lexer.states[0].name : '');
+        for (const state of directive.lexer.states) {
+            this.generator.addLexerState(state);
+        }
+    }
+
+    private importBuiltIn(name: string) {
+        name = name.toLowerCase();
+        if (!this.context.alreadyCompiled.has(name)) {
+            this.context.alreadyCompiled.add(name);
+            if (!BuiltInRegistry[name])
+                return;
+            this.generator.merge(BuiltInRegistry[name].state);
+        }
+    }
+
+    private async importGrammar(name) {
+        const resolver = this.context.resolver;
+        const path = resolver.path(name);
+        if (!this.context.alreadyCompiled.has(path)) {
+            this.context.alreadyCompiled.add(path);
+            await this.mergeLanguageDefinitionString(await resolver.body(path))
+        }
+    }
+
+    private async mergeLanguageDefinitionString(body: string) {
+        const builder = new GrammarBuilder(this.config, this.context);
+        await builder.import(this.parser.run(body).results[0]);
+        this.generator.merge(builder.generator.state);
+        return;
+    }
+
+    private buildRules(name: string, expressions: GrammarBuilderExpression[], rule?: GrammarBuilderRule) {
+        for (const expression of expressions) {
+            this.generator.addGrammarRule(this.buildRule(name, expression, rule));
+        }
+    }
+
+    private buildRule(name: string, expression: GrammarBuilderExpression, rule?: GrammarBuilderRule): GeneratorGrammarRule {
+        const symbols: GeneratorGrammarSymbol[] = [];
+        for (let i = 0; i < expression.symbols.length; i++) {
+            const symbol = this.buildSymbol(name, expression.symbols[i]);
+            if (symbol)
+                symbols.push(symbol);
+        }
+        return { name, symbols, postprocess: expression.postprocess || rule?.postprocess };
+    }
+
+    private buildSymbol(name: string, symbol: GrammarBuilderSymbol): GeneratorGrammarSymbol {
+        if ('repeat' in symbol) {
+            return this.buildRepeatRules(name, symbol);
+        }
+        if ('rule' in symbol) {
+            return symbol;
+        }
+        if ('regex' in symbol) {
+            return symbol;
+        }
+        if ('token' in symbol) {
+            return symbol;
+        }
+        if ('literal' in symbol) {
+            if (!symbol.literal.length) {
+                return null;
+            }
+            if (symbol.literal.length === 1 || this.generator.state.lexer) {
+                return symbol;
+            }
+            return this.buildCharacterRules(name, symbol);
+        }
+        if ('subexpression' in symbol) {
+            return this.buildSubExpressionRules(name, symbol);
+        }
+    }
+
+    private buildCharacterRules(name: string, symbol: GrammarTypeLiteral) {
+        const id = this.generator.grammarUUID(name + "$STR");
+        this.buildRules(id, [
+            {
+                symbols: symbol.literal
+                    .split("")
+                    .map((literal) => {
+                        if (symbol.insensitive && literal.toLowerCase() != literal.toUpperCase())
+                            return { regex: literal, flags: 'i' }
+                        return { literal }
+                    }),
+                postprocess: { builtin: "join" }
+            }
+        ]);
+        return { rule: id };
+    }
+
+    private buildSubExpressionRules(name: string, symbol: GrammarBuilderSymbolSubexpression) {
+        const id = this.generator.grammarUUID(name + "$SUB");
+        this.buildRules(id, symbol.subexpression);
+        return { rule: id };
+    }
+
+    private buildRepeatRules(name: string, symbol: GrammarBuilderSymbolRepeat) {
+        let id: string;
+        const expr1: GrammarBuilderExpression = { symbols: [] };
+        const expr2: GrammarBuilderExpression = { symbols: [] };
+        if (symbol.repeat == '+') {
+            id = this.generator.grammarUUID(name + "$RPT1N");
+            expr1.symbols = [symbol.expression];
+            expr2.symbols = [{ rule: id }, symbol.expression];
+            expr2.postprocess = { builtin: "concat" };
+        } else if (symbol.repeat == '*') {
+            id = this.generator.grammarUUID(name + "$RPT0N");
+            expr2.symbols = [{ rule: id }, symbol.expression];
+            expr2.postprocess = { builtin: "concat" };
+        } else if (symbol.repeat == '?') {
+            id = this.generator.grammarUUID(name + "$RPT01");
+            expr1.symbols = [symbol.expression];
+            expr1.postprocess = { builtin: "first" };
+            expr2.postprocess = { builtin: "null" };
+        }
+        this.buildRules(id, [expr1, expr2]);
+        return { rule: id };
+    }
+}

package/src/compiler/generator.ts

@@ -0,0 +1,229 @@
+import { Dictionary, GeneratorState, GeneratorGrammarRule, GeneratorGrammarSymbol, LexerConfig, LexerStateDefinition } from "../typings";
+
+const PostProcessors = {
+    "join": "({data}) => data.join('')",
+    "concat": "({data}) => data[0].concat([data[1]])",
+    "null": "() => null",
+    "first": "({data}) => data[0]"
+};
+
+export class Generator {
+
+    state: GeneratorState = {
+        grammar: {
+            start: '',
+            rules: {},
+            uuids: {}
+        },
+        lexer: null,
+        head: [],
+        body: [],
+        config: {},
+        version: 'unknown',
+    }
+
+    serializeHead() {
+        if (this.state.config.noscript)
+            return '';
+        return this.state.head.join('\n');
+    }
+
+    serializeBody() {
+        if (this.state.config.noscript)
+            return '';
+        return this.state.body.join('\n');
+    }
+
+    serializeLanguage(depth: number = 0) {
+        return this.pretty({
+            grammar: this.serializeGrammar(depth + 1),
+            lexer: this.serializeLexerConfig(depth + 1)
+        }, depth);
+    }
+
+    merge(state: GeneratorState) {
+        // TODO: Resolve Conflicting Rules and UUIDS
+        Object.assign(this.state.grammar.rules, state.grammar.rules);
+        this.state.grammar.start = state.grammar.start || this.state.grammar.start;
+
+        if (state.lexer) {
+            if (this.state.lexer) {
+                Object.assign(this.state.lexer.states, state.lexer.states);
+            } else {
+                this.state.lexer = state.lexer;
+            }
+            this.state.lexer.start = state.lexer.start || this.state.lexer.start;
+        }
+        this.state.head.push(...state.head);
+        this.state.body.push(...state.body);
+        Object.assign(this.state.config, state.config);
+    }
+
+    grammarUUID(name: string) {
+        this.state.grammar.uuids[name] = (this.state.grammar.uuids[name] || 0) + 1;
+        return name + 'x' + this.state.grammar.uuids[name];
+    }
+
+    addGrammarRule(rule: GeneratorGrammarRule) {
+        this.state.grammar.rules[rule.name] = this.state.grammar.rules[rule.name] || [];
+        this.state.grammar.rules[rule.name].push(rule);
+    }
+
+    addLexerState(state: LexerStateDefinition) {
+        this.state.lexer.states[state.name] = this.state.lexer.states[state.name] || { name: state.name, rules: [] }
+        const target = this.state.lexer.states[state.name];
+        target.default = typeof state.default == 'string' ? state.default : target.default;
+        target.unmatched = typeof state.unmatched == 'string' ? state.unmatched : target.unmatched;
+        target.rules.push(...state.rules);
+    }
+
+    private serializeGrammar(depth: number = 0) {
+        if (!this.state.grammar) {
+            return null;
+        }
+        return this.pretty({
+            start: JSON.stringify(this.state.grammar.start),
+            rules: this.serializeGrammarRules(depth + 1)
+        }, depth);
+    }
+
+    private serializeGrammarRules(depth: number = 0) {
+        const map = {};
+        for (const rule in this.state.grammar.rules) {
+            map[rule] = this.state.grammar.rules[rule].map(v => this.serializeGrammarRule(v))
+        }
+        return this.pretty(map, depth);
+    }
+
+    private serializeSymbol(s: GeneratorGrammarSymbol) {
+        if (typeof s === 'string') {
+            return JSON.stringify(s);
+        } else if ('rule' in s) {
+            return JSON.stringify(s.rule);
+        } else if ('regex' in s) {
+            return `/${s.regex}/${s.flags || ''}`;
+        } else if ('token' in s) {
+            return `{ token: ${JSON.stringify(s.token)} }`;
+        } else if ('literal' in s) {
+            return `{ literal: ${JSON.stringify(s.literal)} }`;
+        } else if ('js' in s) {
+            return s.js;
+        } else {
+            return JSON.stringify(s);
+        }
+    }
+
+    private serializeGrammarRule(rule: GeneratorGrammarRule) {
+        const symbols = [];
+        const alias = {};
+        for (let i = 0; i < rule.symbols.length; i++) {
+            symbols.push(this.serializeSymbol(rule.symbols[i]));
+            if (rule.symbols[i].alias) {
+                alias[rule.symbols[i].alias] = i;
+            }
+        }
+        return this.pretty({
+            name: JSON.stringify(rule.name),
+            symbols: this.pretty(symbols, -1),
+            postprocess: this.serializePostProcess(rule.postprocess, alias)
+        }, -1);
+    }
+
+    private serializePostProcess(postprocess: GeneratorGrammarRule['postprocess'], alias: Dictionary<number>) {
+        if (!postprocess)
+            return null;
+        if ('builtin' in postprocess)
+            return PostProcessors[postprocess.builtin];
+        if (this.state.config.noscript)
+            return;
+        if (typeof postprocess == 'string')
+            return postprocess;
+        if ('js' in postprocess)
+            return postprocess.js;
+        if ('template' in postprocess)
+            return this.templatePostProcess(postprocess.template, alias);
+    }
+
+    private templatePostProcess(templateBody: string, alias: { [key: string]: number }) {
+        for (const key in alias) {
+            templateBody = templateBody.replace(new RegExp('(?:\\$)' + key + '(?![a-zA-Z\\d\\$_])'), `data[${alias[key]}]`);
+        }
+        return "({data}) => { return " + templateBody.replace(/\$(\d+)/g, "data[$1]") + "; }";
+    }
+
+    private serializeLexerConfig(depth: number = 0) {
+        if (!this.state.lexer)
+            return null;
+
+        if (typeof this.state.lexer === 'string')
+            return this.state.lexer;
+
+        return this.pretty({
+            start: JSON.stringify(this.state.lexer.start),
+            states: this.serializeLexerConfigStates(depth + 1)
+        }, depth);
+    }
+
+    private serializeLexerConfigStates(depth: number) {
+        const map = {};
+        for (const key in this.state.lexer.states) {
+            const state = this.state.lexer.states[key];
+            map[state.name] = this.pretty({
+                name: JSON.stringify(state.name),
+                default: state.default ? JSON.stringify(state.default) : null,
+                unmatched: state.unmatched ? JSON.stringify(state.unmatched) : null,
+                rules: this.serializeLexerConfigStateRules(state.rules, depth + 2)
+            }, depth + 1);
+        }
+        return this.pretty(map, depth);
+    }
+
+    private serializeLexerConfigStateRules(rules: LexerConfig['states'][0]['rules'], depth: number) {
+        const ary = rules.map(rule => {
+            if ('import' in rule)
+                return this.pretty({ import: JSON.stringify(rule.import) }, -1)
+            return this.pretty({
+                when: this.serializeSymbol(rule.when as any),
+                type: JSON.stringify(rule.type),
+                tag: JSON.stringify(rule.tag),
+                pop: JSON.stringify(rule.pop),
+                set: JSON.stringify(rule.set),
+                inset: JSON.stringify(rule.inset),
+                goto: JSON.stringify(rule.goto),
+            }, -1);
+        });
+        return this.pretty(ary, depth);
+    }
+
+    private newLine(depth: number) {
+        return '\n' + ' '.repeat(depth * 4);
+    }
+
+    private pretty(obj: string[] | { [key: string]: string | (string[]) }, depth = 0) {
+        if (Array.isArray(obj)) {
+            let r = `[`;
+            for (let i = 0; i < obj.length; i++) {
+                const value = obj[i];
+                r += `${depth >= 0 ? this.newLine(depth + 1) : ' '}${value}${(this.isVal(obj[i + 1]) ? ',' : '')}`;
+            }
+            r += `${depth >= 0 ? this.newLine(depth) : ' '}]`;
+            return r;
+        }
+
+        let r = `{`;
+        const keys = Object.keys(obj).filter(v => this.isVal(obj[v]));
+        const prefix = depth >= 0 ? this.newLine(depth + 1) : ' ';
+        for (let i = 0; i < keys.length; i++) {
+            const key = /[a-z_][a-z\d_$]*/i.test(keys[i]) ? keys[i] : keys[i];
+            const value = Array.isArray(obj[keys[i]]) ? this.pretty(obj[keys[i]] as string[], depth >= 0 ? depth + 1 : -1) : obj[keys[i]];
+            const suffix = (this.isVal(obj[keys[i + 1]]) ? ',' : '');
+            r += `${prefix}${key}: ${value}${suffix}`;
+        }
+        r += `${depth >= 0 ? this.newLine(depth) : ' '}}`;
+        return r;
+    }
+
+    private isVal(value) {
+        return typeof value !== 'undefined' && value !== null;
+    }
+}

package/src/compiler/import-resolver.ts

@@ -0,0 +1,36 @@
+/// <reference types="typescript/lib/lib.webworker" />
+import { ImportResolver } from "../typings";
+export class FileSystemResolver implements ImportResolver {
+    private baseDir: string;
+    private readFile: (path: string, type: string) => Promise<string>;
+    private resolve: (path: string, ...paths: string[]) => string;
+
+    constructor(baseDir: string) {
+        const { readFile } = require('fs');
+        const { resolve, dirname } = require('path');
+        const { promisify } = require('util');
+        this.readFile = promisify(readFile);
+        this.resolve = resolve;
+        this.baseDir = baseDir ? dirname(baseDir) : process?.cwd();
+    }
+
+    path(path: string) {
+        return this.resolve(this.baseDir, path);
+    }
+
+    body(path: string) {
+        return this.readFile(path, 'utf-8');
+    }
+}
+
+export class BrowserImportResolver implements ImportResolver {
+    constructor(private baseURL: string) { }
+
+    path(path: string) {
+        return (new URL(path, this.baseURL)).href;
+    }
+
+    async body(path: string) {
+        return (await fetch(path)).text();
+    }
+}

package/src/compiler/outputs/javascript.ts

@@ -0,0 +1,27 @@
+import { Generator } from "../generator";
+
+export function JavascriptOutput(generator: Generator, exportName: string) {
+    return `${Generate(generator, exportName)}
+
+if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') {
+    module.exports = ${exportName};
+} else {
+    window.${exportName} = ${exportName};
+}`;
+}
+
+export function ESMOutput(generator: Generator, exportName: string) {
+    return `${Generate(generator, exportName)}
+
+export default ${exportName};`;
+}
+
+function Generate(generator: Generator, exportName: string) {
+    return `// Generated automatically by Grammar-Well, version ${generator.state.version} 
+// https://github.com/0x6563/grammar-well
+${generator.serializeHead()}
+function ${exportName}(){
+    ${generator.serializeBody()}
+    return ${generator.serializeLanguage(1)}
+}`;
+}

package/src/compiler/outputs/json.ts

@@ -0,0 +1,5 @@
+import { Generator } from "../generator";
+
+export function JSONFormatter(generator: Generator, exportName) {
+    return JSON.stringify({ state: generator.state, exportName });
+}