grammar-well 1.2.0 → 1.2.2

package/src/parser/algorithms/lrk/bimap.ts

@@ -0,0 +1,17 @@
+export class BiMap<T>{
+    private map: Map<T, number> = new Map();
+    private items: T[] = [];
+
+    id(ref: T) {
+        if (!this.map.has(ref)) {
+            this.map.set(ref, this.items.length);
+            this.items.push(ref);
+        }
+
+        return this.map.get(ref);
+    }
+
+    fetch(index: number) {
+        return this.items[index];
+    }
+}

package/src/parser/algorithms/lrk/canonical-collection.ts

@@ -0,0 +1,79 @@
+import { GrammarRule, GrammarRuleSymbol, LanguageDefinition } from "../../../typings";
+import { ParserUtility } from "../../parser";
+import { BiMap } from "./bimap";
+import { ClosureBuilder } from "./closure";
+import { State } from "./state";
+
+export class CanonicalCollection {
+    states: Map<string, State> = new Map();
+    rules: BiMap<GrammarRule> = new BiMap();
+    terminals: BiMap<GrammarRuleSymbol> = new BiMap();
+
+    private closure: ClosureBuilder;
+    constructor(
+        public grammar: LanguageDefinition['grammar']
+    ) {
+        const augmented = {
+            name: Symbol() as unknown as string,
+            symbols: [grammar.start]
+        }
+        grammar['rules'][augmented.name] = [augmented];
+        this.closure = new ClosureBuilder(grammar);
+        this.rules.id(augmented);
+        this.addState(grammar['rules'][augmented.name][0], 0);
+        this.linkStates('0.0');
+    }
+
+    private addState(rule: GrammarRule, dot: number) {
+        const id = this.getStateId(rule, dot);
+        if (this.states.has(id))
+            return;
+
+        const state: State = {
+            items: [],
+            isFinal: false,
+            actions: new Map(),
+            goto: new Map(),
+            reduce: null,
+            rule: rule
+        }
+
+        state.items.push({ rule, dot });
+        if (rule.symbols.length == dot)
+            state.isFinal = true;
+
+        this.states.set(id, state);
+
+        state.items.push(...this.closure.get(rule.symbols[dot] as string))
+
+        if (!state.isFinal)
+            for (const { rule, dot } of state.items) {
+                this.addState(rule, dot + 1);
+            }
+    }
+
+    private linkStates(id: string, completed: Set<string> = new Set()) {
+        completed.add(id);
+        const state = this.states.get(id);
+        if (!state.isFinal) {
+            for (const { rule, dot } of state.items) {
+                const symbol = rule.symbols[dot];
+                const itemStateId = this.getStateId(rule, dot + 1);
+                if (ParserUtility.SymbolIsTerminal(symbol) && typeof symbol != 'symbol') {
+                    state.actions.set(symbol, itemStateId);
+                } else {
+                    state.goto.set(symbol, itemStateId);
+                }
+
+                if (!completed.has(itemStateId))
+                    this.linkStates(itemStateId, completed);
+            }
+        } else {
+            state.reduce = this.rules.id(state.rule);
+        }
+    }
+
+    private getStateId(rule: GrammarRule, dot: number) {
+        return this.rules.id(rule) + '.' + dot;
+    }
+}

package/src/parser/algorithms/lrk/closure.ts

@@ -0,0 +1,37 @@
+import { GrammarRule, GrammarRuleSymbol, LanguageDefinition } from "../../../typings"
+import { ParserUtility } from "../../parser"
+
+export class ClosureBuilder {
+    constructor(
+        private grammar: LanguageDefinition['grammar'],
+    ) { }
+
+    get(rule: string) {
+        const closure: RuleClosure = { items: [], visited: new Set() };
+        this.addClosure(closure, rule);
+        return closure.items;
+    }
+
+    private addClosure(closure: RuleClosure, symbol: GrammarRuleSymbol) {
+        if (!ParserUtility.SymbolIsTerminal(symbol)) {
+            const key = symbol as string;
+            if (!(closure.visited.has(key))) {
+                closure.visited.add(key);
+
+                const rules = this.grammar.rules[key];
+                for (const rule of rules) {
+                    closure.items.push({ rule, dot: 0 })
+                    this.addClosure(closure, rule.symbols[0]);
+                }
+            }
+        }
+    }
+}
+
+interface RuleClosure {
+    items: {
+        rule: GrammarRule,
+        dot: number,
+    }[]
+    visited: Set<GrammarRuleSymbol>;
+}

package/src/parser/algorithms/lrk/stack.ts

@@ -0,0 +1,53 @@
+import { GrammarRule, GrammarRuleSymbol } from "../../../typings";
+import { State } from "./state";
+
+export class LRStack {
+
+    stack: LRStackItem[] = [];
+
+    get current() {
+        return this.stack[this.stack.length - 1];
+    }
+
+    get previous() {
+        return this.stack[this.stack.length - 2];
+    }
+
+
+    shift(state: State) {
+        this.current.state = state;
+    }
+
+    reduce(rule: GrammarRule) {
+        const n = LRStack.NewItem();
+        const l = rule.symbols.length;
+        n.children = this.stack.splice(l * -1, l);
+        n.children.forEach(v => delete v.state);
+        n.rule = rule;
+        n.symbol = rule.name;
+        this.stack.push(n);
+    }
+
+    append(symbol: GrammarRuleSymbol) {
+        this.stack.push(LRStack.NewItem())
+        this.current.symbol = symbol;
+    }
+
+    static NewItem(): LRStackItem {
+        return {
+            children: [],
+            state: null,
+            symbol: null,
+            rule: null,
+            value: null
+        }
+    }
+}
+
+interface LRStackItem {
+    children: LRStackItem[];
+    state: State;
+    symbol: GrammarRuleSymbol;
+    rule: GrammarRule;
+    value: any;
+}

package/src/parser/algorithms/lrk/state.ts

@@ -0,0 +1,10 @@
+import { GrammarRule, GrammarRuleSymbol } from "../../../typings";
+
+export interface State {
+    items: { rule: GrammarRule, dot: number }[];
+    isFinal: boolean;
+    actions: Map<GrammarRuleSymbol, string>;
+    goto: Map<GrammarRuleSymbol, string>;
+    reduce: number;
+    rule: GrammarRule
+}

package/src/parser/parser.ts

@@ -4,12 +4,12 @@ import { TokenBuffer } from "../lexers/token-buffer";
 import { GrammarRule, GrammarRuleSymbol, LanguageDefinition, LexerToken, ParserAlgorithm } from "../typings";
 import { CYK } from "./algorithms/cyk";
 import { Earley } from "./algorithms/earley";
-import { LR } from "./algorithms/lr";
+import { LRK } from "./algorithms/lrk/algorithm";
 
 const ParserRegistry: { [key: string]: ParserAlgorithm } = {
     earley: Earley,
     cyk: CYK,
-    lr: LR
+    lr0: LRK
 }
 
 export function Parse(language: LanguageDefinition, input: string, options?: ParserOptions) {
@@ -42,9 +42,9 @@ export class Parser {
 }
 
 
-export class ParserUtility {
+export abstract class ParserUtility {
 
-    static TokenMatchesSymbol(token: LexerToken, symbol: GrammarRuleSymbol) {
+    static SymbolMatchesToken(symbol: GrammarRuleSymbol, token: LexerToken) {
         if (typeof symbol === 'string')
             throw 'Attempted to match token against non-terminal';
         if (typeof symbol == 'function')
@@ -59,7 +59,7 @@ export class ParserUtility {
             return symbol.literal === token.value;
     }
 
-    static SymbolIsTerminal<T extends GrammarRuleSymbol>(symbol: T) {
+    static SymbolIsTerminal(symbol: GrammarRuleSymbol) {
         return typeof symbol != 'string';
     }
 

package/src/typings.ts

@@ -47,6 +47,7 @@ export type JavascriptDirective = { body: GrammarTypeJS; } | { head: GrammarType
 export interface ImportDirective {
     import: string;
     path?: boolean;
+    alias?: string;
 }
 
 export interface ConfigDirective {

package/src/utility/monarch.ts

@@ -0,0 +1,36 @@
+import { LexerConfig } from "../typings";
+
+export function CreateMonarchTokenizer(lexer: LexerConfig) {
+    const tokenizer: any = {}; // languages.IMonarchLanguage['tokenizer']
+    const { start, states } = lexer;
+    for (const key in states) {
+        const { name, rules } = states[key];
+        tokenizer[name] = [];
+        for (const rule of rules) {
+            if ('import' in rule) {
+                for (const i of rule.import) {
+                    tokenizer[name].push({ include: i })
+                }
+            } else if ('pop' in rule) {
+                tokenizer[name].push([TransformWhen(rule.when), { token: rule.highlight || 'source', next: '@pop' }])
+            } else if ('goto' in rule) {
+                tokenizer[name].push([TransformWhen(rule.when), { token: rule.highlight || 'source', next: '@' + rule.goto }])
+            } else if ('set' in rule) {
+                tokenizer[name].push([TransformWhen(rule.when), { token: rule.highlight || 'source', switchTo: '@' + rule.set }])
+            } else if ('inset' in rule) {
+                tokenizer[name].push([TransformWhen(rule.when), { token: rule.highlight || 'source', next: '@push' }])
+            } else if ('when' in rule) {
+                tokenizer[name].push([TransformWhen(rule.when), { token: rule.highlight || 'source' }])
+            }
+        }
+    }
+    return { start, tokenizer };
+}
+
+function TransformWhen(obj) {
+    return typeof obj == 'string' ? new RegExp(RegexEscape(obj)) : new RegExp(obj.regex, obj.flags);
+}
+
+function RegexEscape(string) {
+    return string.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&')
+}

package/src/parser/algorithms/lr.ts

@@ -1,74 +0,0 @@
-import type { TokenBuffer } from "../../lexers/token-buffer";
-import type { LanguageDefinition } from "../../typings";
-import type { GrammarRule, GrammarRuleSymbol, LRState } from "../../typings";
-import { ParserUtility } from "../parser";
-
-export function LR(language: LanguageDefinition & { tokens: TokenBuffer }, _options = {}) {
-    const { lr, tokens } = language;
-    const { table } = lr;
-    const stack = new LRStack();
-    stack.push({ state: table['0.0'] });
-
-    let token;
-
-    // eslint-disable-next-line no-cond-assign
-    while (token = tokens.next()) {
-        for (const { symbol, next } of stack.current.state.actions) {
-            if (ParserUtility.TokenMatchesSymbol(token, symbol)) {
-                stack.push({ symbol, state: table[next], value: token });
-                break;
-            }
-        }
-
-        while (stack.current.state?.isFinal) {
-            const rule = stack.current.state.reduce;
-            stack.reduce(rule);
-            stack.current.value = ParserUtility.PostProcess(rule, stack.current.children.map(v => v.value));
-            const s = stack.previous?.state.goto[rule.name];
-            stack.shift(table[s]);
-        }
-    }
-
-    return { results: [stack.current.value] }
-}
-
-
-class LRStack {
-
-    stack: LRStackItem[] = [];
-
-    get current() {
-        return this.stack[this.stack.length - 1];
-    }
-
-    get previous() {
-        return this.stack[this.stack.length - 2];
-    }
-
-    shift(state: LRState) {
-        this.current.state = state;
-    }
-
-    reduce(rule: GrammarRule) {
-        const n = new LRStackItem();
-        const l = rule.symbols.length;
-        n.children = this.stack.splice(l * -1, l);
-        n.children.forEach(v => delete v.state);
-        n.rule = rule;
-        n.symbol = rule.name;
-        this.stack.push(n);
-    }
-
-    push(item: Partial<LRStackItem>) {
-        this.stack.push(new LRStackItem());
-        Object.assign(this.current, item);
-    }
-}
-
-class LRStackItem {
-    children: LRStackItem[] = [];
-    state: LRState;
-    symbol: GrammarRuleSymbol;
-    rule: GrammarRule;
-    value: any;
-}