grammar-well 1.1.0

package/src/lexers/stateful-lexer.ts

@@ -0,0 +1,335 @@
+import { CompiledStateDefinition, LexerStateMatchRule, ResolvedStateDefinition, LexerStateDefinition, LexerConfig } from "../typings";
+
+export class StatefulLexer {
+    private start: string;
+    private states: { [key: string]: CompiledStateDefinition } = Object.create(null);
+    private buffer: string;
+    private stack: string[];
+    private index: number;
+    private line: number;
+    private column: number;
+    private prefetched?: RegExpExecArray;
+    private current: string;
+    private unmatched: LexerStateMatchRule;
+    private rules: LexerStateMatchRule[];
+    private regexp: RegExp;
+    private tags = new Map<string[], Set<string>>();
+
+    constructor({ states, start }: LexerConfig) {
+        ResolveStates(states, start);
+        for (const key in states) {
+            this.states[key] = {
+                regexp: CompileRegExp(states[key] as ResolvedStateDefinition),
+                rules: states[key].rules as LexerStateMatchRule[],
+                unmatched: states[key].unmatched ? { type: states[key].unmatched } as LexerStateMatchRule : null
+            };
+        }
+        this.start = start;
+        this.buffer = '';
+        this.stack = [];
+        this.feed();
+    }
+
+    feed(data?: string, state?: ReturnType<StatefulLexer['state']>) {
+        this.buffer = data || '';
+        this.index = 0;
+        this.line = state ? state.line : 1;
+        this.column = state ? state.column : 1;
+        this.prefetched = state?.prefetched;
+        this.set(state ? state.state : this.start);
+        this.stack = state && state.stack ? state.stack.slice() : [];
+    }
+
+    state() {
+        return {
+            line: this.line,
+            column: this.column,
+            state: this.current,
+            stack: this.stack.slice(),
+            prefetched: this.prefetched,
+        }
+    }
+
+    next() {
+        const next = this.matchNext();
+        if (!next) {
+            return
+        }
+        const { rule, text, index } = next;
+        if (!rule) {
+            throw new Error(`No matching rule for ${text}`);
+        }
+        const token = this.createToken(rule, text, index)
+        this.processRule(rule);
+        return token;
+    }
+
+    private set(current: string) {
+        if (!current || this.current === current)
+            return
+        const info = this.states[current];
+        this.current = current;
+        this.rules = info.rules;
+        this.unmatched = info.unmatched;
+        this.regexp = info.regexp;
+    }
+
+    private pop() {
+        this.set(this.stack.pop());
+    }
+
+    private goto(state: string) {
+        this.stack.push(this.current)
+        this.set(state)
+    }
+
+    private matchNext() {
+        if (this.index === this.buffer.length) {
+            return;
+        }
+
+        const { index, buffer } = this;
+        let text;
+        let rule;
+        let match;
+
+        this.regexp.lastIndex = index;
+        if (this.prefetched) {
+            match = this.prefetched;
+            this.prefetched = null;
+        } else {
+            match = this.regexp.exec(buffer)
+        }
+        if (match == null) {
+            rule = this.unmatched;
+            text = buffer.slice(index, buffer.length);
+        } else if (match.index !== index) {
+            rule = this.unmatched;
+            text = buffer.slice(index, match.index)
+            this.prefetched = match;
+        } else {
+            rule = this.getGroup(match)
+            text = match[0]
+        }
+
+        return { index, rule, text }
+    }
+
+    private createToken(rule: LexerStateMatchRule, text: string, offset: number) {
+        const token = {
+            type: rule.type,
+            tag: this.getTags(rule.tag),
+            value: text,
+            text: text,
+            offset: offset,
+            line: this.line,
+            column: this.column,
+            state: this.current
+        }
+
+        for (let i = 0; i < text.length; i++) {
+            this.index++;
+            this.column++;
+            if (text[i] == '\n') {
+                this.line++;
+                this.column = 1;
+            }
+        }
+        return token;
+    }
+
+    private getTags(tags?: string[]) {
+        if (!tags)
+            return undefined;
+        if (!this.tags.has(tags))
+            this.tags.set(tags, new Set(tags));
+        return this.tags.get(tags);
+    }
+
+    private processRule(rule: LexerStateMatchRule) {
+        if (rule.pop) {
+            let i = rule.pop === 'all' ? this.stack.length : rule.pop;
+            while (i-- > 0) {
+                this.pop();
+            }
+        }
+
+        if (rule.set) {
+            this.set(rule.set);
+        }
+        if (rule.goto) {
+            this.goto(rule.goto);
+        }
+
+        if (rule.inset) {
+            let i = rule.inset;
+            while (--i >= 0) {
+                this.goto(this.current);
+            }
+        }
+    }
+
+    private getGroup(match): LexerStateMatchRule {
+        for (let i = 0; i < this.rules.length; i++) {
+            if (match[i + 1] !== undefined) {
+                return this.rules[i];
+            }
+        }
+        throw new Error('Cannot find token type for matched text')
+    }
+}
+
+class RegexLib {
+
+    static IsRegex(o: any) {
+        return o instanceof RegExp
+    }
+    static Escape(s: string) {
+        return s.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&')
+    }
+
+    static HasGroups(s: string) {
+        return (new RegExp('|' + s)).exec('').length > 1
+    }
+
+    static Capture(source: string) {
+        return '(' + source + ')'
+    }
+
+    static Join(regexps: string[]) {
+        if (!regexps.length)
+            return '(?!)';
+        const source = regexps.map((s) => `(?:${s})`).join('|');
+        return `(?:${source})`;
+    }
+
+    static Source(search: string | RegExp) {
+        if (typeof search === 'string') {
+            return `(?:${RegexLib.Escape(search)})`;
+        }
+        if (RegexLib.IsRegex(search)) {
+            return search.source;
+        }
+        throw new Error('Not a pattern: ' + search)
+    }
+
+}
+
+function CompileRegExp(state: ResolvedStateDefinition): RegExp {
+    const rules = [];
+    const subexpressions = [];
+
+    let isUnicode = null;
+    let isCI = null;
+    for (const options of state.rules) {
+        if (RegexLib.IsRegex(options.when)) {
+            const when = options.when as RegExp;
+            if (isUnicode === null) {
+                isUnicode = when.unicode
+            } else if (isUnicode !== when.unicode && !state.unmatched) {
+                throw new Error(`Inconsistent Regex Flag /u in state: ${state.name}`);
+            }
+            if (isCI === null) {
+                isCI = when.ignoreCase
+            } else if (isCI !== when.ignoreCase) {
+                throw new Error(`Inconsistent Regex Flag /i in state: ${state.name}`);
+            }
+        } else {
+            if (isCI == null) {
+                isCI = false;
+            } else if (isCI != false) {
+                throw new Error(`Inconsistent Regex Flag /i in state: ${state.name}`);
+            }
+        }
+
+        rules.push(options);
+        const pat = RegexLib.Source(options.when);
+        const regexp = new RegExp(pat)
+        if (regexp.test("")) {
+            throw new Error("RegExp matches empty string: " + regexp)
+        }
+
+        if (RegexLib.HasGroups(pat)) {
+            throw new Error("RegExp has capture groups: " + regexp + "\nUse (?: … ) instead")
+        }
+
+        subexpressions.push(RegexLib.Capture(pat))
+    }
+
+    let flags = !state.unmatched ? 'ym' : 'gm';
+    if (isUnicode === true)
+        flags += "u"
+    if (isCI === true)
+        flags += "i"
+    return new RegExp(RegexLib.Join(subexpressions), flags);
+}
+
+export function ResolveStates(states: { [key: string]: LexerStateDefinition }, start: string) {
+
+    const resolved = new Set<string>();
+    const resolving = new Set<string>();
+    const chain = new Set<string>();
+
+
+    ResolveRuleImports(start, states, resolved, resolving, chain);
+    for (const key in states) {
+        if (!resolved.has(key)) {
+            delete states[key];
+        }
+    }
+    return states;
+}
+
+function ResolveRuleImports(name: string, states: { [key: string]: LexerStateDefinition }, resolved: Set<string>, resolving: Set<string>, chain: Set<string>) {
+    if (chain.has(name))
+        throw new Error(`Can not resolve circular import of ${name}`);
+    if (!states[name])
+        throw new Error(`Can not import unknown state ${name}`);
+    if (resolved.has(name) || resolving.has(name))
+        return;
+    const state = states[name];
+    const rules = new UniqueRules();
+    chain.add(name);
+    resolving.add(name);
+    for (let i = 0; i < state.rules.length; i++) {
+        const rule = state.rules[i];
+        if ("import" in rule) {
+            for (const ref of rule.import) {
+                ResolveRuleImports(ref, states, resolved, resolving, chain);
+                rules.push(...states[ref].rules as LexerStateMatchRule[]);
+            }
+        } else {
+            rules.push(rule);
+            if ("set" in rule && !resolving.has(rule.set)) {
+                ResolveRuleImports(rule.set, states, resolved, resolving, new Set());
+            }
+            if ("goto" in rule && !resolving.has(rule.goto)) {
+                ResolveRuleImports(rule.goto, states, resolved, resolving, new Set());
+            }
+        }
+    }
+    state.rules = rules.rules;
+    chain.delete(name);
+    resolved.add(name);
+}
+
+class UniqueRules {
+    private regexps = new Set<string>();
+    private strings = new Set<string>();
+    rules: LexerStateMatchRule[] = [];
+
+    push(...rules: LexerStateMatchRule[]) {
+        for (const rule of rules) {
+            if (RegexLib.IsRegex(rule.when)) {
+                if (!this.regexps.has((rule.when as RegExp).source)) {
+                    this.rules.push(rule);
+                }
+            } else {
+                if (!this.strings.has(rule.when as string)) {
+                    this.rules.push(rule);
+                }
+            }
+        }
+    }
+
+}

package/src/lexers/token-buffer.ts

@@ -0,0 +1,102 @@
+import { Lexer, TQRestorePoint, LexerToken } from '../typings';
+
+export class TokenBuffer {
+    private history: LexerToken[] = [];
+    private queued: string = '';
+
+    private $historyIndex = -1;
+
+    get offset() { return this.active?.offset || 0 }
+    get line() { return this.active?.line || 0 }
+    get column() { return this.active?.column || 0; }
+    get active() { return this.history[this.$historyIndex]; }
+
+    get state(): TQRestorePoint {
+        return { historyIndex: this.$historyIndex, offset: this.offset };
+    }
+
+    constructor(private lexer: Lexer) { }
+
+    reset(buffer: string) {
+        this.lexer.feed(buffer);
+        this.history = [];
+        this.$historyIndex = -1;
+    }
+
+    restore(state: TQRestorePoint) {
+        if (this.history[state.historyIndex].offset != state.offset) {
+            return;
+        }
+        this.$historyIndex = state.historyIndex;
+    }
+
+    feed(buffer: string, flush?: boolean) {
+        this.queued += buffer;
+        if (flush) {
+            this.flush();
+        }
+    }
+
+    flush() {
+        this.history = this.history.slice(this.$historyIndex);
+        this.$historyIndex = 0;
+        if (this.lexer.flush) {
+            this.lexer.flush()
+        }
+    }
+
+    previous() {
+        if (this.$historyIndex > 0) {
+            return this.history[--this.$historyIndex];
+        }
+    }
+
+    next() {
+        if (this.$historyIndex + 1 >= this.history.length) {
+            this.lexerNext();
+        }
+        if (this.$historyIndex + 1 < this.history.length) {
+            return this.history[++this.$historyIndex];
+        }
+    }
+
+    peek(offset: number) {
+        offset += this.$historyIndex;
+        while ((offset >= this.history.length) && this.lexerNext()) {
+            // Seeking
+        }
+        if (offset >= 0 && offset < this.history.length)
+            return this.history[offset];
+    }
+
+    private lexerNext() {
+        let token = this.lexer.next();
+
+        if (typeof token === 'undefined' && this.queued) {
+            this.lexer.feed(this.queued, this.$historyIndex >= 0 ? this.lexer.state() : undefined);
+            this.queued = '';
+            token = this.lexer.next();
+        }
+        if (token)
+            this.history.push(token);
+        return token;
+    }
+
+    [Symbol.iterator]() {
+        return new TokenIterator(this)
+    }
+
+}
+
+class TokenIterator {
+    constructor(private buffer: TokenBuffer) { }
+
+    next() {
+        const token = this.buffer.next()
+        return { value: token, done: !token }
+    }
+
+    [Symbol.iterator]() {
+        return this
+    }
+}

package/src/parser/algorithms/cyk.ts

@@ -0,0 +1,74 @@
+import { TokenBuffer } from "../../lexers/token-buffer";
+import { GrammarRule, GrammarRuleSymbol, LanguageDefinition, LexerToken } from "../../typings";
+import { Matrix } from "../../utility/general";
+import { ParserUtility } from "../parser";
+
+export function CYK(language: LanguageDefinition & { tokens: TokenBuffer }, _options = {}) {
+    const { grammar, tokens } = language;
+
+    const terminals: GrammarRule[] = [];
+    const nonTerminals: GrammarRule[] = [];
+
+    for (const name in grammar.rules) {
+        for (const rule of grammar.rules[name]) {
+            const { symbols } = rule;
+            if (ParserUtility.SymbolIsTerminal(symbols[0])) {
+                terminals.push(rule);
+            } else {
+                nonTerminals.push(rule);
+            }
+        }
+    }
+
+    let currentTokenIndex = -1;
+    const chart = new Matrix(0, 0, () => new Map<GrammarRuleSymbol, Terminal | NonTerminal>());
+    for (const token of tokens) {
+        currentTokenIndex++;
+        chart.resize(currentTokenIndex + 2, currentTokenIndex + 2);
+        for (const rule of terminals) {
+            if (ParserUtility.TokenMatchesSymbol(token, rule.symbols[0])) {
+                chart.get(currentTokenIndex, currentTokenIndex).set(rule.name, { rule, token })
+            }
+        }
+
+
+        for (let floor = currentTokenIndex; floor >= 0; floor--) {
+            for (let inner = floor; inner <= currentTokenIndex; inner++) {
+                const leftCell = chart.get(floor, inner);
+                const rightCell = chart.get(inner + 1, currentTokenIndex);
+
+                for (const rule of nonTerminals) {
+                    const { symbols: [leftSymbol, rightSymbol] } = rule;
+                    const left: Terminal | NonTerminal = leftCell.get(leftSymbol);
+                    const right: Terminal | NonTerminal = rightCell.get(rightSymbol);
+                    if (left && right) {
+                        chart.get(floor, currentTokenIndex).set(rule.name, { rule, left, right });
+                    }
+                }
+            }
+        }
+    }
+
+    const results = Array.from(chart.get(0, currentTokenIndex).values()).map(v => GetValue(v));
+    return { results };
+}
+
+function GetValue(ref: Terminal | NonTerminal) {
+    if (!ref)
+        return;
+    if ('token' in ref) {
+        return ParserUtility.PostProcess(ref.rule, [ref.token]);
+    }
+    return ParserUtility.PostProcess(ref.rule, [GetValue(ref.left), GetValue(ref.right)])
+}
+
+export interface NonTerminal {
+    rule: GrammarRule;
+    left: NonTerminal | Terminal;
+    right: NonTerminal | Terminal;
+}
+
+export interface Terminal {
+    rule: GrammarRule;
+    token: LexerToken;
+}

package/src/parser/algorithms/earley.ts

@@ -0,0 +1,193 @@
+import { Dictionary, GrammarRule, LanguageDefinition } from "../../typings";
+import { TokenBuffer } from "../../lexers/token-buffer";
+import { TextFormatter } from "../../utility/text-format";
+import { ParserUtility } from "../parser";
+
+export interface EarleyParserOptions {
+    keepHistory?: boolean;
+}
+
+export function Earley(language: LanguageDefinition & { tokens: TokenBuffer }, options: EarleyParserOptions = {}) {
+    const { tokens } = language;
+    const { rules, start } = language.grammar;
+    const column = new Column(rules, 0);
+    const table: Column[] = [column];
+    column.wants[start] = [];
+    column.predict(start);
+    column.process();
+
+    let current: number = 0;
+
+    for (const token of tokens) {
+        const previousColumn: Column = table[current];
+
+        if (!(options.keepHistory)) {
+            delete table[current - 1];
+        }
+
+        current++;
+
+        const nextColumn = new Column(rules, current);
+        table.push(nextColumn);
+
+        const literal = token.value;
+        const data = token;
+        nextColumn.data = literal;
+        const { scannable } = previousColumn;
+        for (let w = scannable.length; w--;) {
+            const state = scannable[w];
+            const symbol = state.rule.symbols[state.dot];
+            if (ParserUtility.TokenMatchesSymbol(token, symbol)) {
+                const next = state.nextState({ data, token, isToken: true, reference: current - 1 });
+                nextColumn.states.push(next);
+            }
+        }
+
+        nextColumn.process();
+
+        if (nextColumn.states.length === 0) {
+            throw TextFormatter.UnexpectedToken(tokens, previousColumn.expects());
+        }
+    }
+
+    const results = [];
+    const { states } = table[table.length - 1];
+    for (const { rule: { name, symbols }, dot, reference, data } of states) {
+        if (name === start && dot === symbols.length && reference == 0) {
+            results.push(data);
+        }
+    }
+    return { results, info: { table } };
+}
+
+
+class Column {
+    data: any;
+    states: State[] = [];
+    wants: Dictionary<State[]> = Object.create(null);// states indexed by the non-terminal they expect
+    scannable: State[] = [];// list of states that expect a token
+    completed: Dictionary<State[]> = Object.create(null);  // states that are nullable
+
+    constructor(
+        private rules: Dictionary<GrammarRule[]>,
+        public index: number
+    ) { }
+
+
+    process() {
+        let w = 0;
+        let state: State;
+
+        // eslint-disable-next-line no-cond-assign
+        while (state = this.states[w++]) { // nb. we push() during iteration
+            if (state.isComplete) {
+                state.finish();
+                const { wantedBy } = state;
+                for (let i = wantedBy.length; i--;) { // this line is hot
+                    this.complete(wantedBy[i], state);
+                }
+
+                // special-case nullables
+                if (state.reference === this.index) {
+                    const { name } = state.rule;
+                    this.completed[name] = this.completed[name] || [];
+                    this.completed[name].push(state);
+                }
+            } else {
+                const exp = state.rule.symbols[state.dot];
+                if (typeof exp !== 'string') {
+                    this.scannable.push(state);
+                    continue;
+                }
+
+                // predict
+                if (this.wants[exp]) {
+                    this.wants[exp].push(state);
+
+                    if (this.completed[exp]) {
+                        for (const right of this.completed[exp]) {
+                            this.complete(state, right);
+                        }
+                    }
+                } else {
+                    this.wants[exp] = [state];
+                    this.predict(exp);
+                }
+            }
+        }
+    }
+
+    predict(exp: string) {
+        if (!this.rules[exp])
+            return;
+
+        for (const rule of this.rules[exp]) {
+            this.states.push(new State(rule, 0, this.index, this.wants[exp]));
+        }
+    }
+
+    expects(): GrammarRule[] {
+        return this.states
+            .filter((state) => {
+                const nextSymbol = state.rule.symbols[state.dot];
+                return nextSymbol && typeof nextSymbol !== "string";
+            })
+            .map(v => ({ ...v.rule, index: v.dot }));
+    }
+
+    private complete(left: State, right: State) {
+        const copy = left.nextState(right);
+        this.states.push(copy);
+    }
+
+
+}
+
+class State {
+    isComplete: boolean;
+    data: any = [];
+    left: State;
+    right: State | StateToken;
+    constructor(
+        public rule: GrammarRule,
+        public dot: number,
+        public reference: number,
+        public wantedBy: State[]
+    ) {
+        this.isComplete = this.dot === rule.symbols.length;
+    }
+
+    nextState(child: State | StateToken) {
+        const state = new State(this.rule, this.dot + 1, this.reference, this.wantedBy);
+        state.left = this;
+        state.right = child;
+        if (state.isComplete) {
+            state.data = state.build();
+            state.right = undefined;
+        }
+        return state;
+    }
+
+
+    finish() {
+        this.data = ParserUtility.PostProcess(this.rule, this.data, { reference: this.reference, dot: this.dot });
+    }
+
+    protected build() {
+        const children = [];
+        // eslint-disable-next-line @typescript-eslint/no-this-alias
+        let node: State = this;
+        do {
+            children[node.dot - 1] = node.right.data;
+            node = node.left;
+        } while (node.left);
+        return children;
+    }
+}
+
+interface StateToken {
+    data: any,
+    token: any,
+    isToken: boolean,
+    reference: number
+}