grammar-well 1.1.0

package/src/parser/algorithms/lr.ts

@@ -0,0 +1,37 @@
+import { TokenBuffer } from "../../lexers/token-buffer";
+import { LanguageDefinition } from "../../typings";
+import { CanonicalCollection, LRStack } from "../../utility/lr";
+import { ParserUtility } from "../parser";
+
+export function LR(language: LanguageDefinition & { tokens: TokenBuffer, canonical?: CanonicalCollection }, _options = {}) {
+    const { grammar, tokens, canonical } = language;
+    const collection = canonical || new CanonicalCollection(grammar);
+    const stack = new LRStack();
+    const s = collection.states['0.0'];
+    stack.add(null);
+    stack.shift(s);
+    let token;
+
+    // eslint-disable-next-line no-cond-assign
+    while (token = tokens.next()) {
+        for (const [symbol, state] of stack.current.state.actions) {
+            if (ParserUtility.TokenMatchesSymbol(token, symbol)) {
+                stack.add(symbol);
+                stack.shift(collection.states[state]);
+                stack.current.value = token;
+                break;
+            }
+        }
+
+        while (stack.current.state?.isFinal) {
+            const rule = stack.current.state.reduce;
+            stack.reduce(rule);
+            stack.current.value = ParserUtility.PostProcess(rule, stack.current.children.map(v => v.value));
+            const s = stack.previous.state.goto.get(rule.name);
+            stack.shift(collection.states[s]);
+
+        }
+    }
+
+    return { results: [stack.current.value], canonical: collection }
+}

package/src/parser/parser.ts

@@ -0,0 +1,77 @@
+import { CharacterLexer } from "../lexers/character-lexer";
+import { StatefulLexer } from "../lexers/stateful-lexer";
+import { TokenBuffer } from "../lexers/token-buffer";
+import { GrammarRule, GrammarRuleSymbol, LanguageDefinition, LexerToken, ParserAlgorithm } from "../typings";
+import { CYK } from "./algorithms/cyk";
+import { Earley } from "./algorithms/earley";
+import { LR } from "./algorithms/lr";
+
+const ParserRegistry: { [key: string]: ParserAlgorithm } = {
+    earley: Earley,
+    cyk: CYK,
+    lr: LR
+}
+
+export function Parse(language: LanguageDefinition, input: string, options?: ParserOptions) {
+    const i = new Parser(language, options);
+    return i.run(input);
+}
+
+export class Parser {
+
+    constructor(private language: LanguageDefinition, private options: ParserOptions = { algorithm: 'earley', parserOptions: {} }) { }
+
+    run(input: string): { results: any[] } {
+        const tokenQueue = this.getTokenQueue();
+        tokenQueue.feed(input);
+        if (typeof this.options.algorithm == 'function')
+            return this.options.algorithm({ ...this.language, tokens: tokenQueue, utility: ParserUtility }, this.options.parserOptions);
+        return ParserRegistry[this.options.algorithm]({ ...this.language, tokens: tokenQueue, utility: ParserUtility }, this.options.parserOptions);
+    }
+
+    private getTokenQueue() {
+        const { lexer } = this.language;
+        if (!lexer) {
+            return new TokenBuffer(new CharacterLexer());
+        } else if ("feed" in lexer && typeof lexer.feed == 'function') {
+            return new TokenBuffer(lexer);
+        } else if ('states' in lexer) {
+            return new TokenBuffer(new StatefulLexer(lexer));
+        }
+    }
+}
+
+
+export class ParserUtility {
+
+    static TokenMatchesSymbol(token: LexerToken, symbol: GrammarRuleSymbol) {
+        if (typeof symbol === 'string')
+            throw 'Attempted to match token against non-terminal';
+        if (typeof symbol == 'function')
+            return symbol(token);
+        if (!symbol)
+            return
+        if ("test" in symbol)
+            return symbol.test(token.value);
+        if ("token" in symbol)
+            return symbol.token === token.type || token.tag?.has(symbol.token);
+        if ("literal" in symbol)
+            return symbol.literal === token.value;
+    }
+
+    static SymbolIsTerminal<T extends GrammarRuleSymbol>(symbol: T) {
+        return typeof symbol != 'string';
+    }
+
+    static PostProcess(rule: GrammarRule, data: any, meta?: any) {
+        if (rule.postprocess) {
+            return rule.postprocess({ rule, data, meta });
+        }
+        return data;
+    }
+}
+
+interface ParserOptions {
+    algorithm: (keyof typeof ParserRegistry) | ParserAlgorithm,
+    parserOptions?: any;
+}

package/src/typings.ts

@@ -0,0 +1,221 @@
+import { TokenBuffer } from "./lexers/token-buffer";
+import { ParserUtility } from "./parser/parser";
+
+export interface Dictionary<T> {
+    [key: string]: T;
+}
+
+export interface CompileOptions {
+    version?: string;
+    noscript?: boolean;
+    basedir?: string;
+    resolver?: ImportResolverConstructor;
+    resolverInstance?: ImportResolver;
+    exportName?: string;
+    format?: OutputFormat;
+}
+
+export type OutputFormat = '_default' | 'object' | 'json' | 'js' | 'javascript' | 'module' | 'esmodule' | 'ts' | 'typescript'
+
+export interface GrammarBuilderContext {
+    alreadyCompiled: Set<string>;
+    resolver: ImportResolver;
+    uuids: Dictionary<number>;
+}
+
+
+export interface ImportResolver {
+    path(path: string): string;
+    body(path: string): Promise<string>;
+}
+
+export interface ImportResolverConstructor {
+    new(basePath: string): ImportResolver;
+}
+
+export type PostProcessor = (payload: PostProcessorPayload) => any;
+
+interface PostProcessorPayload {
+    data: any[];
+    rule: GrammarRule;
+    meta: any;
+}
+
+export type JavascriptDirective = { body: GrammarTypeJS; } | { head: GrammarTypeJS }
+
+export interface ImportDirective {
+    import: string;
+    path?: boolean;
+}
+
+export interface ConfigDirective {
+    config: Dictionary<any>;
+}
+
+export interface GrammarDirective {
+    grammar: {
+        config?: Dictionary<any>;
+        rules: GrammarBuilderRule[];
+    }
+}
+
+export interface LexerDirective {
+    lexer: {
+        start?: string,
+        states: LexerStateDefinition[];
+    };
+}
+
+export interface GrammarBuilderRule {
+    name: string;
+    expressions: GrammarBuilderExpression[];
+    postprocess?: GrammarTypeJS | GrammarTypeBuiltIn | GrammarTypeTemplate;
+}
+
+export interface GrammarBuilderExpression {
+    symbols: GrammarBuilderSymbol[];
+    postprocess?: GrammarTypeJS | GrammarTypeBuiltIn | GrammarTypeTemplate;
+}
+
+export type GrammarBuilderSymbol = GrammarTypeRule | GrammarTypeRegex | GrammarTypeToken | GrammarTypeLiteral | GrammarBuilderSymbolRepeat | GrammarBuilderSymbolSubexpression;
+
+export interface GrammarBuilderSymbolSubexpression {
+    subexpression: GrammarBuilderExpression[];
+}
+
+export interface GrammarBuilderSymbolRepeat {
+    expression: GrammarBuilderSymbol;
+    repeat: "+" | "*" | "?";
+}
+
+export interface GrammarTypeRule {
+    rule: string;
+}
+
+export interface GrammarTypeRegex {
+    regex: string;
+    flags?: string
+}
+
+export interface GrammarTypeToken {
+    token: string;
+}
+
+export interface GrammarTypeLiteral {
+    literal: string;
+    insensitive?: boolean;
+}
+
+export type GrammarTypeBuiltIn = { builtin: string };
+export type GrammarTypeTemplate = { template: string };
+export type GrammarTypeJS = { js: string };
+
+
+export type ParserAlgorithm = ((language: LanguageDefinition & { tokens: TokenBuffer; utility: ParserUtility }, options?: any) => { results: any[], info?: any });
+
+export type LanguageDirective = (JavascriptDirective | ImportDirective | ConfigDirective | GrammarDirective | LexerDirective);
+
+type GrammarRuleSymbolFunction = (data: LexerToken) => boolean;
+
+export interface GrammarRule {
+    name: string;
+    symbols: GrammarRuleSymbol[];
+    postprocess?: PostProcessor;
+}
+
+export type GrammarRuleSymbol = string | RegExp | GrammarTypeLiteral | GrammarTypeToken | GrammarRuleSymbolFunction;
+
+export interface GeneratorGrammarRule {
+    name: string;
+    symbols: GeneratorGrammarSymbol[];
+    postprocess?: GrammarTypeTemplate | GrammarTypeBuiltIn | GrammarTypeJS;
+}
+
+export type GeneratorGrammarSymbol = { alias?: string } & (GrammarTypeRule | GrammarTypeRegex | GrammarTypeLiteral | GrammarTypeToken | GrammarTypeJS);
+
+export interface LanguageDefinition {
+    lexer?: Lexer | LexerConfig;
+    grammar: {
+        start: string;
+        rules: Dictionary<GrammarRule[]>;
+    }
+}
+
+export interface TQRestorePoint {
+    historyIndex: number;
+    offset: number;
+}
+
+export interface Lexer {
+    next(): LexerToken | undefined;
+    feed(chunk?: string, state?: ReturnType<Lexer['state']>): void;
+    state(): any;
+    flush?(): void;
+}
+
+export interface LexerToken {
+    type?: string | undefined;
+    tag?: Set<string>;
+    value: string;
+    offset: number;
+    line: number;
+    column: number;
+}
+
+export interface LexerStatus {
+    index: number;
+    line: number;
+    column: number;
+    state: string;
+}
+
+export interface LexerStateDefinition {
+    name: string;
+    unmatched?: string;
+    default?: string;
+    rules: (LexerStateImportRule | LexerStateMatchRule)[];
+}
+
+export interface LexerStateImportRule {
+    import: string[]
+}
+
+export interface LexerStateMatchRule {
+    when: string | RegExp
+    type?: string;
+    tag?: string[];
+    pop?: number | 'all';
+    inset?: number;
+    goto?: string;
+    set?: string;
+}
+
+export interface ResolvedStateDefinition {
+    name: string;
+    unmatched?: string;
+    rules: LexerStateMatchRule[];
+}
+
+export interface CompiledStateDefinition {
+    regexp: RegExp;
+    unmatched?: LexerStateMatchRule;
+    rules: LexerStateMatchRule[];
+}
+
+export interface LexerConfig {
+    start?: string
+    states: Dictionary<LexerStateDefinition>;
+}
+
+export interface GeneratorState {
+    version: string;
+    config: Dictionary<string>;
+    head: string[];
+    body: string[];
+    lexer?: LexerConfig;
+    grammar: {
+        start: string;
+        rules: Dictionary<GeneratorGrammarRule[]>,
+        uuids: { [key: string]: number }
+    }
+}

package/src/utility/general.ts

@@ -0,0 +1,120 @@
+import { Dictionary, GrammarRuleSymbol } from "../typings";
+
+
+export class Collection<T> {
+    categorized: Dictionary<Dictionary<number>> = {};
+    private uncategorized = new Map<T, number>();
+    private items: T[] = [];
+
+    constructor(ref: T[] = []) {
+        for (const s of ref) {
+            this.encode(s);
+        }
+    }
+
+    encode(ref: T): number {
+        const c = this.resolve(ref);
+        if (c)
+            return this.addCategorized(c.category, c.key, ref);
+        return this.addUncategorized(ref);
+    }
+
+    decode(id: number | string): T {
+        return this.items[typeof id == 'string' ? parseInt(id) : id];
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-function
+    resolve(_: T): { category: keyof Collection<T>['categorized'], key: string } | void { }
+
+    private addCategorized(category: keyof Collection<T>['categorized'], key: string, ref: T): number {
+        if (!(key in this.categorized[category])) {
+            this.categorized[category][key] = this.items.length;
+            this.items.push(ref);
+        }
+        return this.categorized[category][key];
+    }
+
+    private addUncategorized(ref: T): number {
+        if (!this.uncategorized.has(ref)) {
+            this.uncategorized.set(ref, this.items.length);
+            this.items.push(ref);
+        }
+        return this.uncategorized.get(ref);
+    }
+
+}
+
+export class SymbolCollection extends Collection<GrammarRuleSymbol>{
+    categorized = {
+        nonTerminal: {},
+        literalI: {},
+        literalS: {},
+        token: {},
+        regex: {},
+        function: {},
+    }
+
+    resolve(symbol: GrammarRuleSymbol) {
+        if (typeof symbol == 'string') {
+            return { category: 'nonTerminal', key: symbol };
+        } else if ('literal' in symbol) {
+            if (symbol.insensitive)
+                return { category: 'literalI', key: symbol.literal }
+            return { category: 'literalS', key: symbol.literal }
+        } else if ('token' in symbol) {
+            return { category: 'token', key: symbol.token }
+        } else if ('test' in symbol) {
+            return { category: 'regex', key: symbol.toString() }
+        } else if (typeof symbol == 'function') {
+            return { category: 'function', key: symbol.toString() }
+        }
+    }
+}
+
+export class Matrix<T> {
+    private $x = 0;
+    private $y = 0;
+    get x() { return this.$x }
+    set x(x: number) { x != this.$x && this.resize(x, this.y); }
+    get y() { return this.$y }
+    set y(y: number) { y != this.$y && this.resize(this.x, y); }
+
+    matrix: GetCallbackOrValue<T>[][] = [];
+
+    constructor(x: number, y: number, private initial?: T | ((...args: any) => T)) {
+        this.resize(x, y);
+    }
+
+    get(x: number, y: number): T {
+        return this.matrix[x][y];
+    }
+
+    set(x: number, y: number, value: any) {
+        return this.matrix[x][y] = value;
+    }
+
+    resize(x: number, y: number) {
+        if (x < this.x) {
+            this.matrix.splice(x);
+            this.$x = x;
+        }
+        if (y > this.y) {
+            this.matrix.forEach(a => a.push(...Matrix.Array(y - a.length, this.initial)));
+            this.$y = y;
+        } else if (y < this.y) {
+            this.matrix.forEach(a => a.splice(y + 1));
+            this.$y = y;
+        }
+        if (x > this.x) {
+            const ext = Matrix.Array(x - this.x, () => Matrix.Array(this.y, this.initial))
+            this.matrix.push(...ext);
+            this.$x = x;
+        }
+    }
+
+    static Array<T>(length, initial?: T | ((...args: any) => T)): GetCallbackOrValue<T>[] {
+        return Array.from({ length }, (typeof initial == 'function' ? initial : () => initial) as any);
+    }
+}
+
+type GetCallbackOrValue<T> = T extends (...args: any) => any ? ReturnType<T> : T;

package/src/utility/lint.ts

@@ -0,0 +1,26 @@
+import { LanguageDefinition, GrammarRuleSymbol, Dictionary, GrammarRule } from "../typings";
+
+
+export function LintGrammarSymbols(language: LanguageDefinition): GrammarRuleSymbol[] {
+    const unused = new Set<string>();
+    const { rules, start } = language.grammar;
+    for (const rule in rules) {
+        unused.add(rule);
+    }
+    TraverseRule(start, rules, unused);
+    return Array.from(unused);
+}
+function TraverseRule(name: string, rules: Dictionary<GrammarRule[]>, unvisited: Set<string>) {
+    if (!unvisited.has(name)) {
+        return;
+    }
+    unvisited.add(name);
+    const n = rules[name];
+    for (const { symbols } of n) {
+        for (const symbol of symbols) {
+            if (typeof symbol == 'string') {
+                TraverseRule(symbol, rules, unvisited);
+            }
+        }
+    }
+}

package/src/utility/lr.ts

@@ -0,0 +1,153 @@
+import { ParserUtility } from "../parser/parser";
+import { Dictionary, GrammarRule, GrammarRuleSymbol, LanguageDefinition } from "../typings";
+import { Collection, SymbolCollection } from "./general";
+
+export class CanonicalCollection {
+    rules: Collection<GrammarRule> = new Collection();
+    states: { [key: string]: State } = Object.create(null)
+    symbols: SymbolCollection;
+    
+    constructor(public grammar: LanguageGrammar & { symbols?: SymbolCollection }) {
+        this.symbols = grammar.symbols || new SymbolCollection();
+        const augmented = { name: Symbol() as unknown as string, symbols: [grammar.start] }
+        grammar.rules[augmented.name] = [augmented];
+        this.addState([{ rule: augmented, dot: 0 }]);
+    }
+
+    addState(seed: StateItem[]) {
+        const id = this.encodeStateItems(seed);
+        if (id in this.states)
+            return this.states[id];
+
+        const state = new State(this, seed);
+        this.states[id] = state;
+        for (const q in state.queue) {
+            this.addState(state.queue[q])
+        }
+        state.queue = {};
+    }
+
+    encodeRule(rule: GrammarRule, dot: number) {
+        return this.rules.encode(rule) + '.' + dot;
+    }
+
+    encodeStateItems(seed: StateItem[]) {
+        return Array.from(new Set(seed)).map(v => this.encodeRule(v.rule, v.dot)).sort().join()
+    }
+}
+
+class State {
+    isFinal = false;
+
+    outputs: StateOut = {
+        goto: {},
+        action: {}
+    };
+
+    queue: { [key: string]: StateItem[] } = {};
+    actions: Map<GrammarRuleSymbol, string> = new Map();
+    goto: Map<GrammarRuleSymbol, string> = new Map();
+    reduce?: GrammarRule;
+
+    constructor(private collection: CanonicalCollection, items: StateItem[]) {
+        const visited = new Set<GrammarRuleSymbol>();
+        for (const item of items) {
+            this.closure(item.rule, item.dot, visited);
+        }
+        if (this.isFinal) {
+            if (items.length == 1 && visited.size < 1) {
+                this.reduce = items[0].rule;
+            } else {
+                throw 'Conflict Detected';
+            }
+        }
+
+        for (const k in this.outputs.goto) {
+            const seed = this.outputs.goto[k];
+            const stateId = this.collection.encodeStateItems(seed);
+            this.queue[stateId] = seed;
+            this.goto.set(this.collection.symbols.decode(k), stateId);
+        }
+
+        for (const k in this.outputs.action) {
+            const seed = this.outputs.action[k];
+            const stateId = this.collection.encodeStateItems(seed);
+            this.queue[stateId] = seed;
+            this.actions.set(this.collection.symbols.decode(k), stateId);
+        }
+    }
+
+    private closure(rule: GrammarRule, dot: number, visited: Set<GrammarRuleSymbol>) {
+        const isFinal = rule.symbols.length == dot;
+        this.isFinal = isFinal || this.isFinal;
+        const { [dot]: symbol } = rule.symbols;
+
+        if (isFinal || visited.has(symbol))
+            return;
+
+        visited.add(symbol);
+        const stateItem = { rule, dot: dot + 1 };
+
+        if (ParserUtility.SymbolIsTerminal(symbol)) {
+            const id = this.collection.symbols.encode(symbol);
+            this.outputs.action[id] = this.outputs.action[id] || [];
+            this.outputs.action[id].push(stateItem);
+        } else {
+            const id = this.collection.symbols.encode(symbol);
+            this.outputs.goto[id] = this.outputs.goto[id] || [];
+            this.outputs.goto[id].push(stateItem);
+            for (const rule of this.collection.grammar.rules[symbol as string]) {
+                this.closure(rule, 0, visited)
+            }
+        }
+    }
+}
+
+export class LRStack {
+
+    stack: LRStackItem[] = [];
+
+    get current() {
+        return this.stack[this.stack.length - 1];
+    }
+
+    get previous() {
+        return this.stack[this.stack.length - 2];
+    }
+
+    shift(state: State) {
+        this.current.state = state;
+    }
+
+    reduce(rule: GrammarRule) {
+        const n = new LRStackItem();
+        const l = rule.symbols.length;
+        n.children = this.stack.splice(l * -1, l);
+        n.children.forEach(v => delete v.state);
+        n.rule = rule;
+        n.symbol = rule.name;
+        this.stack.push(n);
+    }
+
+    add(symbol: GrammarRuleSymbol) {
+        this.stack.push(new LRStackItem())
+        this.current.symbol = symbol;
+    }
+}
+
+
+class LRStackItem {
+    children: LRStackItem[] = [];
+    state: State;
+    symbol: GrammarRuleSymbol;
+    rule: GrammarRule;
+    value: any;
+}
+
+type LanguageGrammar = LanguageDefinition['grammar'];
+type StateItem = { rule: GrammarRule, dot: number };
+
+interface StateOut {
+    action: Dictionary<StateItem[]>;
+    goto: Dictionary<StateItem[]>;
+}