grammar-well 1.3.3 → 2.0.0

package/src/generator/stringify/exports/typescript.ts

@@ -0,0 +1,17 @@
+import { JavaScriptGenerator } from "../javascript.js";
+
+export function TypescriptFormat(generator: JavaScriptGenerator) {
+    const exportName = generator.name();
+    return `// Generated automatically by Grammar-Well, version ${generator.state.version} 
+// https://github.com/0x6563/grammar-well
+// @ts-nocheck
+
+${generator.lifecycle('import')}
+
+class ${exportName} {
+    artifacts = ${generator.artifacts(1)}
+    constructor(){${generator.lifecycle('new')}}
+}
+
+export default ${exportName};`;
+}

package/src/generator/stringify/grammar/v2.ts

@@ -0,0 +1,223 @@
+import { ASTConfig, ASTDirectives, ASTGrammar, ASTGrammarProduction, ASTGrammarSymbol, ASTImport, ASTLexer, ASTLexerState, ASTLexerStateImportRule, ASTLexerStateMatchRule, ASTLexerStateNonMatchRule, GeneratorGrammarSymbol } from "../../../typings/index.js";
+
+export class V2GrammarString {
+    source: string = '';
+    append(directives: ASTDirectives | (ASTDirectives[])) {
+        directives = Array.isArray(directives) ? directives : [directives];
+        for (const directive of directives) {
+            if ("lifecycle" in directive) {
+                this.appendSection("on:" + directive.lifecycle, directive.js.js.trim());
+            } else if ("import" in directive) {
+                this.appendImportDirective(directive);
+            } else if ("config" in directive) {
+                this.appendConfigDirective(directive);
+            } else if ("grammar" in directive) {
+                this.appendGrammarDirective(directive);
+            } else if ("lexer" in directive) {
+                this.appendLexerDirective(directive);
+            }
+        }
+    }
+
+    appendImportDirective(directive: ASTImport) {
+        this.source += `import * from ${directive.path ? JSON.stringify(directive.import) : directive.import};\n`;
+    }
+
+    appendConfigDirective(directive: ASTConfig) {
+        this.appendSection('config', this.formatKV(directive.config, 1));
+    }
+
+    appendGrammarDirective(directive: ASTGrammar) {
+        let body = '';
+        if (directive.grammar.config) {
+            body += this.formatKV(directive.grammar.config, 1);
+        }
+        if (directive.grammar.rules) {
+            for (const rule of directive.grammar.rules) {
+                body += this.formatGrammarRule(rule);
+            }
+        }
+        this.appendSection('grammar', body);
+    }
+
+    formatGrammarRule(rule: ASTGrammarProduction) {
+        let body = '\n' + this.indent(1, `[${rule.name}]`);
+        if (rule.postprocess) {
+            body += ` ${this.formatPostProcess(rule.postprocess)}`;
+        }
+        for (const exp of rule.expressions) {
+            body += '\n' + this.indent(2, '| ') + this.formatSymbols(exp.symbols as any);
+            if (exp.postprocess) {
+                body += ` ${this.formatPostProcess(exp.postprocess)}`;
+            }
+        }
+        body += '\n';
+        return body;
+    }
+
+    formatSymbols(exp: GeneratorGrammarSymbol[]) {
+        return exp.map(v => this.formatSymbol(v) + (v.alias ? "@" + v.alias : '')).join(' ');
+    }
+
+    formatSymbol(exp: ASTGrammarSymbol | GeneratorGrammarSymbol | string) {
+        if (typeof exp == 'string') {
+            return JSON.stringify(exp);
+        }
+        if ('subexpression' in exp) {
+            return `(${exp.subexpression.map(v => this.formatSymbols(v.symbols as any)).join(' | ')})`;
+        }
+        if ('rule' in exp) {
+            return exp.rule;
+        }
+        if ('literal' in exp) {
+            return (exp.insensitive ? '\\i ' : '') + JSON.stringify(exp.literal);
+        }
+        if ('regex' in exp) {
+            let { regex, flags } = exp;
+            return `r${flags || ''}:{${regex}}`;
+        }
+        if ('token' in exp) {
+            return `<${exp.token}>`;
+        }
+        if ('expression' in exp) {
+            return this.formatSymbol(exp.expression) + (exp.repeat || '');
+        }
+    }
+
+    formatPostProcess(postProcess: ASTGrammarProduction['postprocess']) {
+
+        // if ('builtin' in postProcess) {
+        //     return postProcess;
+        // }
+        if ('js' in postProcess) {
+            return `=> \${ ${postProcess.js} }`;
+        }
+        if ('template' in postProcess) {
+            const prefix = postProcess.template.slice(0, 1);
+            const suffix = postProcess.template.slice(-1);
+            return `=> ${prefix} ${postProcess.template.slice(1, -1).trim()} ${suffix}`;
+        }
+    }
+
+    appendLexerDirective(directive: ASTLexer) {
+        let body = '';
+        if (directive.lexer.start) {
+            body += this.formatKV({ start: directive.lexer.start }, 1);
+        }
+        if (directive.lexer.states) {
+            for (const { state, name } of directive.lexer.states) {
+                if ('span' in state) {
+                    body += '\n' + this.indent(1, `[${name}] span {\n`);
+                    const stateOpen = state.span.find(v => v.name == 'start');
+                    const stateBody = state.span.find(v => v.name == 'span');
+                    const stateClose = state.span.find(v => v.name == 'stop');
+                    if (state.config) {
+                        body += this.formatKV(state.config, 2);
+                    }
+                    if (stateOpen) {
+                        body += this.formatLexerState('start', stateOpen.state, 2);
+                    }
+                    if (stateBody) {
+                        body += this.formatLexerState('span', stateBody.state, 2);
+                    }
+                    if (stateClose) {
+                        body += this.formatLexerState('stop', stateClose.state, 2);
+                    }
+                    body += '\n' + this.indent(1, `}\n`);
+                } else {
+                    body += this.formatLexerState(name, state, 1)
+                }
+            }
+        }
+
+        this.appendSection('lexer', body);
+    }
+
+    formatLexerState(name: string, state: ASTLexerState, depth: number = 0) {
+        let body = '\n' + this.indent(depth, `[${name}]\n`);
+        if (state.default) {
+            body += this.indent(depth + 1, 'default: ' + this.formatLexerStateRule(state.default) + ';\n');
+        }
+        if (state.unmatched) {
+            body += this.indent(depth + 1, 'unmatched: ' + this.formatLexerStateRule(state.unmatched) + ';\n');
+        }
+        for (const rule of state.rules) {
+            if ('span' in rule)
+                continue;
+            body += this.indent(depth + 1, '- ' + this.formatLexerStateRule(rule) + '\n');
+        }
+        return body;
+    }
+
+    formatLexerStateRule(rule: ASTLexerStateMatchRule | ASTLexerStateNonMatchRule | ASTLexerStateImportRule) {
+        let body = '';
+        if ('import' in rule) {
+            body += 'import ' + rule.import.join(', ');
+        } else {
+            if (rule.before) {
+                body += 'before ' + this.formatSymbol((rule as ASTLexerStateMatchRule).when);
+            } else if (rule.skip) {
+                body += 'skip ' + this.formatSymbol((rule as ASTLexerStateMatchRule).when);
+            } else if ('when' in rule) {
+                body += 'when ' + this.formatSymbol(rule.when as any);
+            }
+
+            if (typeof rule.type != 'undefined') {
+                body += ` type ${JSON.stringify(rule.type)}`;
+            }
+            if (typeof rule.tag != 'undefined') {
+                body += ` tag ${rule.tag.map(v => JSON.stringify(v)).join(', ')}`;
+            }
+            if (typeof rule.open != 'undefined') {
+                body += ` open ${JSON.stringify(rule.open)}`;
+            }
+            if (typeof rule.close != 'undefined') {
+                body += ` close ${JSON.stringify(rule.close)}`;
+            }
+            if (typeof rule.highlight != 'undefined') {
+                body += ` highlight ${JSON.stringify(rule.highlight)}`;
+            }
+            if (typeof rule.unembed != 'undefined') {
+                body += ` unembed`;
+            }
+            if (typeof rule.embed != 'undefined') {
+                body += ` embed`;
+            }
+            if (typeof rule.pop != 'undefined') {
+                body += ` pop` + (rule.pop == 'all' || rule.pop > 1 ? ' ' + rule.pop : '');
+            }
+            if (typeof rule.inset != 'undefined') {
+                body += ` inset` + (rule.inset > 1 ? ' ' + rule.inset : '');
+            }
+            if (typeof rule.goto != 'undefined') {
+                body += ` goto ${rule.goto}`;
+            }
+            if (typeof rule.set != 'undefined') {
+                body += ` set ${rule.set}`;
+            }
+            if ("stay" in rule && rule.stay) {
+                body += ` stay`;
+            }
+        }
+        return body.trim();
+    }
+
+    formatKV(obj: { [key: string]: any }, depth: number = 0) {
+        let body = '';
+        for (const key in obj) {
+            body += this.indent(depth, `${key}: ${JSON.stringify(obj[key])}\n`);
+        }
+        return body;
+    }
+
+    appendSection(label: string, body: string) {
+        if (this.source) {
+            this.source += '\n';
+        }
+        this.source += `${label} {\n${body}\n}\n`;
+    }
+
+    indent(depth: number = 0, content: string) {
+        return `\t`.repeat(depth) + content;
+    }
+}

package/src/generator/stringify/javascript.ts

@@ -0,0 +1,94 @@
+import { Dictionary, GeneratorOutputOptions, GeneratorGrammarProductionRule } from "../../typings/index.js";
+import { BasicGrammarTable } from "../artifacts/basic.js";
+import { LexerArtifact } from "../artifacts/lexer.js";
+import { LRParseTableBuilder } from "../artifacts/lr.js";
+import { GeneratorState } from "../state.js";
+import { CommonGenerator } from "./common.js";
+
+const PostProcessors = {
+    "join": "({data}) => data.join('')",
+    "concat": "({data}) => data[0].concat([data[1]])",
+    "null": "() => null",
+    "first": "({data}) => data[0]"
+};
+
+export class JavaScriptGenerator {
+
+    constructor(public state: GeneratorState, public options: GeneratorOutputOptions) { }
+
+    name() {
+        return this.options.name || 'GWLanguage';
+    }
+
+    lifecycle(lifecycle: string) {
+        if (this.options.noscript)
+            return '';
+        return this.state.lifecycle[lifecycle] || '';
+    }
+
+    artifacts(depth: number = 0) {
+        let output: { [key: string]: string } = {};
+        const artifacts = this.options.artifacts || { grammar: true, lexer: true };
+
+        if (artifacts && artifacts.lr) {
+            const table = new LRParseTableBuilder(this);
+            output.lr = CommonGenerator.JSON({
+                k: "0",
+                table: table.stringify(depth + 2)
+            }, depth + 1);
+        }
+
+        if ('lexer' in this.state && artifacts.lexer) {
+            const l = new LexerArtifact(this.state.lexer);
+            output.lexer = l.output(depth + 1);
+        }
+
+        if (artifacts.grammar) {
+            const basic = new BasicGrammarTable(this);
+            output.grammar = basic.stringify(depth + 1);
+        }
+
+        return CommonGenerator.JSON(output, depth);
+    }
+
+    postProcess(postprocess: GeneratorGrammarProductionRule['postprocess'], alias: Dictionary<number>) {
+        postprocess = this.state.grammar.config.postprocessorOverride || postprocess || this.state.grammar.config.postprocessorDefault;
+        if (!postprocess)
+            return null;
+        if ('builtin' in postprocess)
+            return PostProcessors[postprocess.builtin];
+        if (this.options.noscript)
+            return;
+        if (typeof postprocess == 'string')
+            return postprocess;
+        if ('js' in postprocess)
+            return postprocess.js;
+        if ('template' in postprocess)
+            return this.templatePostProcess(postprocess.template, alias);
+    }
+
+    grammarRule(rule: GeneratorGrammarProductionRule) {
+        const symbols = [];
+        const alias = {};
+        for (let i = 0; i < rule.symbols.length; i++) {
+            symbols.push(CommonGenerator.SerializeSymbol(rule.symbols[i]));
+            if (rule.symbols[i].alias) {
+                alias[rule.symbols[i].alias] = i;
+            }
+        }
+
+        return CommonGenerator.JSON({
+            name: JSON.stringify(rule.name),
+            symbols: CommonGenerator.JSON(symbols, -1),
+            postprocess: this.postProcess(rule.postprocess, alias)
+        }, -1);
+    }
+
+    private templatePostProcess(templateBody: string, alias: { [key: string]: number }) {
+        for (const key in alias) {
+            templateBody = templateBody.replace(new RegExp('(?:\\$)' + key + '(?![a-zA-Z\\d\\$_])'), `data[${alias[key]}]`);
+        }
+        return "({data}) => { return " + templateBody.replace(/\$(\d+)/g, "data[$1]") + "; }";
+    }
+
+}

package/src/index.ts

@@ -1,6 +1,7 @@
-export * from './compiler/compiler';
-export * from './compiler/import-resolver';
-export * from './parser/parser';
-export * from './typings'
-export { StatefulLexer } from './lexers/stateful-lexer';
-export * from './utility/monarch';
+export * from './generator/generator.js';
+export * from './generator/import-resolvers/auto.js';
+export { StatefulLexer } from './lexers/stateful-lexer.js';
+export * from './parser/parse.js';
+export * from './utility/parsing.js';
+export * from './typings/index.js';
+export * from './utility/index.js';

package/src/lexers/character-lexer.ts

@@ -1,6 +1,6 @@
-import { Lexer } from "../typings";
+import { RuntimeLexer } from "../typings/index.js";
 
-export class CharacterLexer implements Lexer {
+export class CharacterLexer implements RuntimeLexer {
     private buffer: string | any[] = '';
 
     private $indexOffset = 0;

package/src/lexers/stateful-lexer.ts

@@ -1,8 +1,8 @@
-import { CompiledStateDefinition, LexerStateMatchRule, ResolvedStateDefinition, LexerStateDefinition, LexerConfig, Lexer } from "../typings";
+import { RuntimeLexerStateMatchRule, ASTLexerStateNonMatchRule, RuntimeLexer, RuntimeLexerConfig, StatefulLexerStateDefinition } from "../typings/index.js";
 
-export class StatefulLexer implements Lexer {
+export class StatefulLexer implements RuntimeLexer {
     private start: string;
-    private states: { [key: string]: CompiledStateDefinition } = Object.create(null);
+    private states: { [key: string]: StatefulLexerStateDefinition } = Object.create(null);
     private buffer: string;
     private stack: string[];
     private index: number;
@@ -10,20 +10,13 @@ export class StatefulLexer implements Lexer {
     private column: number;
     private prefetched?: RegExpExecArray;
     private current: string;
-    private unmatched: LexerStateMatchRule;
-    private rules: LexerStateMatchRule[];
+    private unmatched: ASTLexerStateNonMatchRule;
+    private rules: RuntimeLexerStateMatchRule[];
     private regexp: RegExp;
     private tags = new Map<string[], Set<string>>();
 
-    constructor({ states, start }: LexerConfig) {
-        ResolveStates(states, start);
-        for (const key in states) {
-            this.states[key] = {
-                regexp: CompileRegExp(states[key] as ResolvedStateDefinition),
-                rules: states[key].rules as LexerStateMatchRule[],
-                unmatched: states[key].unmatched ? { type: states[key].unmatched } as LexerStateMatchRule : null
-            };
-        }
+    constructor({ states, start }: RuntimeLexerConfig) {
+        this.states = states;
         this.start = start;
         this.buffer = '';
         this.stack = [];
@@ -50,16 +43,31 @@ export class StatefulLexer implements Lexer {
         }
     }
 
-    next() {
-        const next = this.matchNext();
+    next(skipped: boolean = false) {
+        const next = this.matchNext(skipped);
         if (!next) {
             return
         }
         const { rule, text, index } = next;
         if (!rule) {
-            throw new Error(`No matching rule for ${text}`);
+            throw new Error(`No matching rule for ${text.split(/\n|\r\n/).slice(0, 3).join('\n')}`);
+        }
+        const token = {
+            type: rule.type,
+            highlight: rule.highlight,
+            open: rule.open,
+            close: rule.close,
+            tag: this.getTags(rule.tag),
+            value: text,
+            text: text,
+            offset: index,
+            line: this.line,
+            lines: 0,
+            column: this.column,
+            state: this.current
         }
-        const token = this.createToken(rule, text, index)
+        this.adjustPosition(text);
+        token.lines = this.line - token.line;
         this.adjustStack(rule);
         return token;
     }
@@ -71,7 +79,7 @@ export class StatefulLexer implements Lexer {
         this.current = current;
         this.rules = info.rules;
         this.unmatched = info.unmatched;
-        this.regexp = info.regexp;
+        this.regexp = info.regex;
     }
 
     private pop() {
@@ -83,14 +91,14 @@ export class StatefulLexer implements Lexer {
         this.set(state)
     }
 
-    private matchNext() {
+    private matchNext(skipped: boolean = false) {
         if (this.index === this.buffer.length) {
             return;
         }
 
         const { index, buffer } = this;
         let text;
-        let rule: LexerStateMatchRule;
+        let rule: RuntimeLexerStateMatchRule | ASTLexerStateNonMatchRule;
         let match;
 
         this.regexp.lastIndex = index;
@@ -111,6 +119,10 @@ export class StatefulLexer implements Lexer {
             rule = this.getGroup(match);
             text = match[0];
             if (rule.before) {
+                this.adjustStack(rule);
+                return this.matchNext(skipped);
+            } else if (rule.skip && !skipped) {
+                this.adjustPosition(text);
                 this.adjustStack(rule);
                 return this.matchNext();
             }
@@ -119,33 +131,6 @@ export class StatefulLexer implements Lexer {
         return { index, rule, text }
     }
 
-    private createToken(rule: LexerStateMatchRule, text: string, offset: number) {
-        const token = {
-            type: rule.type,
-            highlight: rule.highlight,
-            open: rule.open,
-            close: rule.close,
-            tag: this.getTags(rule.tag),
-            value: text,
-            text: text,
-            offset: offset,
-            line: this.line,
-            lines: 0,
-            column: this.column,
-            state: this.current
-        }
-        for (let i = 0; i < text.length; i++) {
-            this.column++;
-            if (text[i] == '\n') {
-                token.lines++;
-                this.column = 1;
-            }
-        }
-        this.index += text.length;
-        this.line += token.lines;
-        return token;
-    }
-
     private getTags(tags?: string[]) {
         if (!tags)
             return undefined;
@@ -154,7 +139,18 @@ export class StatefulLexer implements Lexer {
         return this.tags.get(tags);
     }
 
-    private adjustStack(rule: LexerStateMatchRule) {
+    private adjustPosition(text: string) {
+        this.index += text.length;
+        for (let i = 0; i < text.length; i++) {
+            this.column++;
+            if (text[i] == '\n') {
+                this.line++;
+                this.column = 1;
+            }
+        }
+    }
+
+    private adjustStack(rule: RuntimeLexerStateMatchRule) {
         if (rule.pop) {
             let i = rule.pop === 'all' ? this.stack.length : rule.pop;
             while (i-- > 0) {
@@ -177,7 +173,7 @@ export class StatefulLexer implements Lexer {
         }
     }
 
-    private getGroup(match): LexerStateMatchRule {
+    private getGroup(match): RuntimeLexerStateMatchRule {
         for (let i = 0; i < this.rules.length; i++) {
             if (match[i + 1] !== undefined) {
                 return this.rules[i];
@@ -185,157 +181,4 @@ export class StatefulLexer implements Lexer {
         }
         throw new Error('Cannot find token type for matched text')
     }
-}
-
-class RegexLib {
-
-    static IsRegex(o: any) {
-        return o instanceof RegExp;
-    }
-
-    static Escape(s: string) {
-        return s.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
-    }
-
-    static HasGroups(s: string) {
-        return (new RegExp('|' + s)).exec('').length > 1;
-    }
-
-    static Capture(source: string) {
-        return '(' + source + ')';
-    }
-
-    static Join(regexps: string[]) {
-        if (!regexps.length)
-            return '(?!)';
-        const source = regexps.map((s) => `(?:${s})`).join('|');
-        return `(?:${source})`;
-    }
-
-    static Source(search: string | RegExp) {
-        if (typeof search === 'string') {
-            return `(?:${RegexLib.Escape(search)})`;
-        }
-        if (RegexLib.IsRegex(search)) {
-            return search.source;
-        }
-        throw new Error('Not a pattern: ' + search)
-    }
-
-}
-
-function CompileRegExp(state: ResolvedStateDefinition): RegExp {
-    const rules = [];
-    const subexpressions = [];
-
-    let isUnicode = null;
-    let isCI = null;
-    for (const options of state.rules) {
-        if (RegexLib.IsRegex(options.when)) {
-            const when = options.when as RegExp;
-            if (isUnicode === null) {
-                isUnicode = when.unicode
-            } else if (isUnicode !== when.unicode && !state.unmatched) {
-                throw new Error(`Inconsistent Regex Flag /u in state: ${state.name}`);
-            }
-            if (isCI === null) {
-                isCI = when.ignoreCase
-            } else if (isCI !== when.ignoreCase) {
-                throw new Error(`Inconsistent Regex Flag /i in state: ${state.name}`);
-            }
-        } else {
-            if (isCI == null) {
-                isCI = false;
-            } else if (isCI != false) {
-                throw new Error(`Inconsistent Regex Flag /i in state: ${state.name}`);
-            }
-        }
-
-        rules.push(options);
-        const pat = RegexLib.Source(options.when);
-        const regexp = new RegExp(pat)
-        if (regexp.test("")) {
-            throw new Error("RegExp matches empty string: " + regexp)
-        }
-
-        if (RegexLib.HasGroups(pat)) {
-            throw new Error("RegExp has capture groups: " + regexp + "\nUse (?: … ) instead")
-        }
-
-        subexpressions.push(RegexLib.Capture(pat))
-    }
-
-    let flags = !state.unmatched ? 'ym' : 'gm';
-    if (isUnicode === true)
-        flags += "u"
-    if (isCI === true)
-        flags += "i"
-    return new RegExp(RegexLib.Join(subexpressions), flags);
-}
-
-export function ResolveStates(states: { [key: string]: LexerStateDefinition }, start: string) {
-    const resolved = new Set<string>();
-    const resolving = new Set<string>();
-    const chain = new Set<string>();
-
-    ResolveRuleImports(start, states, resolved, resolving, chain);
-    for (const key in states) {
-        if (!resolved.has(key)) {
-            delete states[key];
-        }
-    }
-    return states;
-}
-
-function ResolveRuleImports(name: string, states: { [key: string]: LexerStateDefinition }, resolved: Set<string>, resolving: Set<string>, chain: Set<string>) {
-    if (chain.has(name))
-        throw new Error(`Can not resolve circular import of ${name}`);
-    if (!states[name])
-        throw new Error(`Can not import unknown state ${name}`);
-    if (resolved.has(name) || resolving.has(name))
-        return;
-    const state = states[name];
-    const rules = new UniqueRules();
-    chain.add(name);
-    resolving.add(name);
-    for (let i = 0; i < state.rules.length; i++) {
-        const rule = state.rules[i];
-        if ("import" in rule) {
-            for (const ref of rule.import) {
-                ResolveRuleImports(ref, states, resolved, resolving, chain);
-                rules.push(...states[ref].rules as LexerStateMatchRule[]);
-            }
-        } else {
-            rules.push(rule);
-            if ("set" in rule && !resolving.has(rule.set)) {
-                ResolveRuleImports(rule.set, states, resolved, resolving, new Set());
-            }
-            if ("goto" in rule && !resolving.has(rule.goto)) {
-                ResolveRuleImports(rule.goto, states, resolved, resolving, new Set());
-            }
-        }
-    }
-    state.rules = rules.rules;
-    chain.delete(name);
-    resolved.add(name);
-}
-
-class UniqueRules {
-    private regexps = new Set<string>();
-    private strings = new Set<string>();
-    rules: LexerStateMatchRule[] = [];
-
-    push(...rules: LexerStateMatchRule[]) {
-        for (const rule of rules) {
-            if (RegexLib.IsRegex(rule.when)) {
-                if (!this.regexps.has((rule.when as RegExp).source)) {
-                    this.rules.push(rule);
-                }
-            } else {
-                if (!this.strings.has(rule.when as string)) {
-                    this.rules.push(rule);
-                }
-            }
-        }
-    }
 }