clarity-pattern-parser 6.0.2 → 7.0.1

package/src/grammar/Grammar.test.ts

@@ -0,0 +1,306 @@
+import { And } from "../patterns/And";
+import { Literal } from "../patterns/Literal";
+import { Not } from "../patterns/Not";
+import { Or } from "../patterns/Or";
+import { Pattern } from "../patterns/Pattern";
+import { Regex } from "../patterns/Regex";
+import { Repeat } from "../patterns/Repeat";
+import { Grammar } from "./Grammar";
+
+describe("Grammar", () => {
+    test("Literal", () => {
+        const expression = `
+            name = "John"
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const namePattern = patterns.get("name");
+        const expected = new Literal("name", "John");
+
+        expect(namePattern).toEqual(expected);
+    });
+
+    test("Regex", () => {
+        const expression = `
+            name = /\\w/
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("name");
+        const name = new Regex("name", "\\w");
+
+        expect(pattern).toEqual(name);
+    });
+
+    test("Or", () => {
+        const expression = `
+            john = "John"
+            jane = "Jane"
+            names = john | jane
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("names");
+        const john = new Literal("john", "John");
+        const jane = new Literal("jane", "Jane");
+        const names = new Or("names", [john, jane]);
+
+        expect(pattern).toEqual(names);
+    });
+
+    test("And", () => {
+        const expression = `
+            space = " "
+            first-name = /\\w/
+            last-name = /\\w/
+            full-name = first-name & space & last-name
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("full-name");
+        const space = new Literal("space", " ");
+        const firstName = new Regex("first-name", "\\w");
+        const lastName = new Regex("last-name", "\\w");
+        const fullName = new And("full-name", [firstName, space, lastName]);
+
+        expect(pattern).toEqual(fullName);
+    });
+
+    test("And With Optional Pattern", () => {
+        const expression = `
+            space = " "
+            first-name = /\\w/
+            last-name = /\\w/
+            middle-name = /\\w/
+            middle-name-with-space = middle-name & space
+            full-name = first-name & space & middle-name-with-space? & last-name
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("full-name");
+        const space = new Literal("space", " ");
+        const firstName = new Regex("first-name", "\\w");
+        const lastName = new Regex("last-name", "\\w");
+        const middleName = new Regex("middle-name", "\\w");
+        const middleNameWithSpace = new And("middle-name-with-space", [middleName, space], true);
+        const fullName = new And("full-name", [firstName, space, middleNameWithSpace, lastName]);
+
+        expect(pattern).toEqual(fullName);
+    });
+
+    test("And With Not Pattern", () => {
+        const expression = `
+            space = " "
+            first-name = /\\w/
+            last-name = /\\w/
+            middle-name = /\\w/
+            jack = "Jack"
+            middle-name-with-space = middle-name & space
+            full-name = !jack & first-name & space & middle-name-with-space? & last-name
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("full-name");
+        const space = new Literal("space", " ");
+        const firstName = new Regex("first-name", "\\w");
+        const lastName = new Regex("last-name", "\\w");
+        const middleName = new Regex("middle-name", "\\w");
+        const jack = new Literal("jack", "Jack");
+        const notJack = new Not("not-jack", jack);
+        const middleNameWithSpace = new And("middle-name-with-space", [middleName, space], true);
+        const fullName = new And("full-name", [notJack, firstName, space, middleNameWithSpace, lastName]);
+
+        expect(pattern).toEqual(fullName);
+    });
+
+    test("Repeat", () => {
+        const expression = `
+            digit = /\\d/
+            digits = (digit)+
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("digits");
+        const digit = new Regex("digit", "\\d");
+        const digits = new Repeat("digits", digit);
+
+        expect(pattern).toEqual(digits);
+    });
+
+    test("Repeat Zero Or More", () => {
+        const expression = `
+            digit = /\\d/
+            digits = (digit)*
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("digits");
+        const digit = new Regex("digit", "\\d");
+        const digits = new Repeat("digits", digit, { min: 0 });
+
+        expect(pattern).toEqual(digits);
+    });
+
+    test("Repeat Lower Limit", () => {
+        const expression = `
+            digit = /\\d+/
+            digits = (digit){1,}
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("digits");
+        const digit = new Regex("digit", "\\d+");
+        const digits = new Repeat("digits", digit, { min: 1 });
+
+        expect(pattern).toEqual(digits);
+    });
+
+    test("Repeat Bounded", () => {
+        const expression = `
+            digit = /\\d+/
+            digits = (digit){1,3}
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("digits");
+        const digit = new Regex("digit", "\\d+");
+        const digits = new Repeat("digits", digit, { min: 1, max: 3 });
+
+        expect(pattern).toEqual(digits);
+    });
+
+    test("Repeat Upper Limit", () => {
+        const expression = `
+            digit = /\\d+/
+            digits = (digit){,3}
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("digits");
+        const digit = new Regex("digit", "\\d+");
+        const digits = new Repeat("digits", digit, { min: 0, max: 3 });
+
+        expect(pattern).toEqual(digits);
+    });
+
+    test("Repeat Exact", () => {
+        const expression = `
+            digit = /\\d+/
+            digits = (digit){3}
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("digits");
+        const digit = new Regex("digit", "\\d+");
+        const digits = new Repeat("digits", digit, { min: 3, max: 3 });
+
+        expect(pattern).toEqual(digits);
+    });
+
+    test("Repeat Divider", () => {
+        const expression = `
+            digit = /\\d+/
+            comma = ","
+            digits = (digit, comma){3}
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("digits");
+        const digit = new Regex("digit", "\\d+");
+        const divider = new Literal("comma", ",");
+        const digits = new Repeat("digits", digit, { divider, min: 3, max: 3 });
+
+        expect(pattern).toEqual(digits);
+    });
+
+    test("Repeat Divider With Trim Divider", () => {
+        const expression = `
+            digit = /\\d+/
+            comma = ","
+            digits = (digit, comma){3} -t
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("digits");
+        const digit = new Regex("digit", "\\d+");
+        const divider = new Literal("comma", ",");
+        const digits = new Repeat("digits", digit, { divider, min: 3, max: 3, trimDivider: true });
+
+        expect(pattern).toEqual(digits);
+    });
+
+    test("Repeat Divider With Optional Pattern", () => {
+        const expression = `
+            digit = /\\d+/
+            comma = ","
+            digits = (digit?, comma)+
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("digits") as Pattern;
+        const digit = new Regex("digit", "\\d+", true);
+        const comma = new Literal("comma", ",");
+        const digits = new Repeat("digits", digit, { divider: comma });
+
+        expect(pattern).toEqual(digits)
+    });
+
+    test("Reference", () => {
+        const expression = `
+            digit = /\\d+/
+            divider = /\\s*,\\s*/
+            open-bracket = "["
+            close-bracket = "]"
+            spaces = /\\s+/
+            items = digit | array
+            array-items = (items, divider)* -t
+            array = open-bracket & spaces? & array-items? & spaces? & close-bracket
+        `;
+
+        const patterns = Grammar.parse(expression);
+        const pattern = patterns.get("array") as Pattern;
+
+        let text = "[1, []]";
+        let result = pattern.exec(text);
+
+        expect(result.ast?.value).toEqual("[1, []]");
+    });
+
+    test("Alias", () => {
+        const expression = `
+            name = /regex/
+            alias = name
+        `;
+
+        const patterns = Grammar.parse(expression);
+        
+        const name = patterns.get("name");
+        const expectedName = new Regex("name", "regex");
+
+        const alias = patterns.get("alias");
+        const expectedAlias = new Regex("alias", "regex");
+
+        expect(name).toEqual(expectedName);
+        expect(alias).toEqual(expectedAlias);
+    });
+
+    test("Bad Grammar At Beginning", () => {
+
+        expect(() => {
+            const expression = `Just Junk`;
+            Grammar.parse(expression);
+        }).toThrowError("[Parse Error] Found: 'Just Junk', expected: ' ='.");
+
+    });
+
+    test("Bad Grammar Further In", () => {
+
+        expect(() => {
+            const expression = `name = /\\w/
+                age = /()
+            `;
+            Grammar.parse(expression);
+        }).toThrowError("[Parse Error] Found: '    age = /()\n      ', expected: '    age = !' or '    age = (' or '    age = [Pattern Name]' or '    age = [Regular Expression]' or '    age = [String]'.")
+
+    });
+});

package/src/grammar/Grammar.ts

@@ -0,0 +1,249 @@
+import { Node } from "../ast/Node";
+import { Literal } from "../patterns/Literal";
+import { Pattern } from "../patterns/Pattern";
+import { Regex } from "../patterns/Regex";
+import { Reference } from "../patterns/Reference";
+import { grammar } from "./patterns/grammar";
+import { Or } from "../patterns/Or";
+import { Not } from "../patterns/Not";
+import { And } from "../patterns/And";
+import { Repeat, RepeatOptions } from "../patterns/Repeat";
+import { AutoComplete } from "../intellisense/AutoComplete";
+
+class ParseContext {
+    patternsByName = new Map<string, Pattern>();
+}
+
+export class Grammar {
+    private _parseContext: ParseContext;
+    private _autoComplete: AutoComplete;
+
+    constructor() {
+        this._parseContext = new ParseContext();
+        this._autoComplete = new AutoComplete(grammar, {
+            greedyPatternNames: ["spaces", "optional-spaces", "whitespace", "new-line"],
+            customTokens: {
+                "regex-literal": ["[Regular Expression]"],
+                "literal": ["[String]"],
+                "name": ["[Pattern Name]"],
+                "pattern-name": ["[Pattern Name]"]
+            }
+        });
+    }
+
+    parse(expression: string) {
+        this._parseContext = new ParseContext();
+        this._tryToParse(expression);
+
+        return this._parseContext.patternsByName;
+    }
+
+    private _tryToParse(expression: string) {
+        const { ast, cursor, options, isComplete } = this._autoComplete.suggestFor(expression);
+
+        if (!isComplete) {
+            const text = cursor?.text || "";
+            const index = options.reduce((num, o) => Math.max(o.startIndex, num), 0);
+            const foundText = text.slice(Math.max(index - 10, 0), index + 10);
+            const expectedTexts = "'" + options.map(o => {
+                const startText = text.slice(Math.max(o.startIndex - 10), o.startIndex);
+                return startText + o.text;
+            }).join("' or '") + "'";
+            const message = `[Parse Error] Found: '${foundText}', expected: ${expectedTexts}.`
+            throw new Error(message);
+        }
+
+        // If it is complete it will always have a node. So we have to cast it.
+        this._cleanAst(ast as Node);
+        this._buildPatterns(ast as Node);
+
+    }
+
+    private _cleanAst(ast: Node) {
+        ast.findAll(
+            n => n.name === "spaces" ||
+                n.name === "optional-spaces" ||
+                n.name === "new-line" ||
+                n.name.includes("whitespace") ||
+                n.name.includes("comment")
+        ).forEach(n => n.remove());
+    }
+
+    private _buildPatterns(ast: Node) {
+        ast.children.forEach((n) => {
+            const typeNode = n.find(n => n.name.includes("literal"));
+            const type = typeNode?.name || "unknown";
+
+            switch (type) {
+                case "literal": {
+                    this._buildLiteral(n)
+                    break;
+                }
+                case "regex-literal": {
+                    this._buildRegex(n);
+                    break;
+                }
+                case "or-literal": {
+                    this._buildOr(n);
+                    break;
+                }
+                case "and-literal": {
+                    this._buildAnd(n)
+                    break;
+                }
+                case "repeat-literal": {
+                    this._buildRepeat(n)
+                    break;
+                }
+                case "alias-literal": {
+                    this._buildAlias(n)
+                    break;
+                }
+            }
+        });
+    }
+
+    private _buildLiteral(statementNode: Node) {
+        const nameNode = statementNode.find(n => n.name === "name") as Node;
+        const literalNode = statementNode.find(n => n.name === "literal") as Node;
+        const value = literalNode.value.slice(1, literalNode.value.length - 1);
+        const name = nameNode.value;
+        const literal = new Literal(name, value);
+
+        this._parseContext.patternsByName.set(name, literal)
+    }
+
+    private _buildRegex(statementNode: Node) {
+        const nameNode = statementNode.find(n => n.name === "name") as Node;
+        const regexNode = statementNode.find(n => n.name === "regex-literal") as Node;
+        const value = regexNode.value.slice(1, regexNode.value.length - 1);
+        const name = nameNode.value;
+
+        const regex = new Regex(name, value);
+
+        this._parseContext.patternsByName.set(name, regex);
+    }
+
+    private _buildOr(statementNode: Node) {
+        const nameNode = statementNode.find(n => n.name === "name") as Node;
+        const orNode = statementNode.find(n => n.name === "or-literal") as Node;
+        const patternNodes = orNode.children.filter(n => n.name == "pattern-name");
+
+        const name = nameNode.value;
+        const patterns = patternNodes.map(n => this._getPattern(n.value));
+        const or = new Or(name, patterns);
+
+        this._parseContext.patternsByName.set(name, or);
+    }
+
+    private _getPattern(name: string) {
+        const pattern = this._parseContext.patternsByName.get(name);
+
+        if (pattern == null) {
+            return new Reference(name);
+        }
+
+        return pattern;
+    }
+
+    private _buildAnd(statementNode: Node) {
+        const nameNode = statementNode.find(n => n.name === "name") as Node;
+        const andNode = statementNode.find(n => n.name === "and-literal") as Node;
+        const patternNodes = andNode.children.filter(n => n.name == "pattern");
+
+        const name = nameNode.value;
+        const patterns = patternNodes.map(n => {
+            const nameNode = n.find(n => n.name === "pattern-name") as Node;
+            const isNot = n.find(n => n.name === "not") != null;
+            const isOptional = n.find(n => n.name === "is-optional") != null;
+            const name = nameNode.value;
+            const pattern = this._getPattern(name);
+
+            if (isNot) {
+                return new Not(`not-${name}`, pattern.clone(name, isOptional));
+            }
+
+            return pattern.clone(name, isOptional);
+        });
+
+        const and = new And(name, patterns);
+
+        this._parseContext.patternsByName.set(name, and);
+    }
+
+    private _buildRepeat(statementNode: Node) {
+        const nameNode = statementNode.find(n => n.name === "name") as Node;
+        const repeatNode = statementNode.find(n => n.name === "repeat-literal") as Node;
+        const patternNode = repeatNode.find(n => n.name == "pattern") as Node;
+        const patternNameNode = patternNode.find(n => n.name === "pattern-name") as Node;
+        const dividerNode = repeatNode.find(n => n.name === "divider-pattern");
+        const bounds = repeatNode.find(n => n.name === "bounds");
+        const exactCount = repeatNode.find(n => n.name === "exact-count");
+        const quantifier = repeatNode.find(n => n.name === "quantifier-shorthand");
+        const isPatternOptional = repeatNode.find(n => n.name === "is-optional") != null;
+        const trimDivider = repeatNode.find(n => n.name === "trim-divider") != null;
+
+        const name = nameNode.value;
+        const pattern = this._getPattern(patternNameNode.value);
+
+        const options: RepeatOptions = {
+            min: 1,
+            max: Infinity
+        }
+
+        if (trimDivider) {
+            options.trimDivider = trimDivider;
+        }
+
+        if (dividerNode != null) {
+            options.divider = this._getPattern(dividerNode.value);
+        }
+
+        if (bounds != null) {
+            const minNode = bounds.find(p => p.name === "min");
+            const maxNode = bounds.find(p => p.name === "max");
+
+            const min = minNode == null ? 0 : Number(minNode.value);
+            const max = maxNode == null ? Infinity : Number(maxNode.value);
+
+            options.min = min;
+            options.max = max;
+        } else if (exactCount != null) {
+            const integerNode = exactCount.find(p => p.name === "integer") as Node;
+            const integer = Number(integerNode.value);
+
+            options.min = integer;
+            options.max = integer;
+        } else if (quantifier != null) {
+            const type = quantifier.value;
+            if (type === "+") {
+                options.min = 1;
+                options.max = Infinity;
+            } else {
+                options.min = 0;
+                options.max = Infinity;
+            }
+        }
+
+        const repeat = new Repeat(name, pattern.clone(pattern.name, isPatternOptional), options);
+
+        this._parseContext.patternsByName.set(name, repeat);
+    }
+
+    private _buildAlias(statementNode: Node) {
+        const nameNode = statementNode.find(n => n.name === "name") as Node;
+        const aliasNode = statementNode.find(n => n.name === "alias-literal") as Node;
+        const aliasName = aliasNode.value;
+        const name = nameNode.value;
+        const pattern = this._getPattern(aliasName);
+        const alias = pattern.clone(name);
+
+        this._parseContext.patternsByName.set(name, alias)
+    }
+
+    static parse(expression: string) {
+        const grammar = new Grammar();
+        return grammar.parse(expression);
+    }
+
+}

package/src/grammar/patterns/andLiteral.ts

@@ -0,0 +1,8 @@
+import { Repeat } from "../../patterns/Repeat"
+import { Regex } from "../../patterns/Regex";
+import { pattern } from "./pattern";
+
+const divider = new Regex("and-divider", "\\s*[&]\\s*");
+divider.setTokens([" & "]);
+
+export const andLiteral = new Repeat("and-literal", pattern, { divider, min: 2 });

package/src/grammar/patterns/comment.ts

@@ -0,0 +1,3 @@
+import { Regex } from "../../patterns/Regex"
+
+export const comment = new Regex("comment", "#[^\r\n]+");

package/src/grammar/patterns/grammar.ts

@@ -0,0 +1,19 @@
+import { Or } from "../../patterns/Or";
+import { Regex } from "../../patterns/Regex";
+import { Repeat } from "../../patterns/Repeat";
+import { comment } from "./comment";
+import { statement } from "./statement";
+
+const whitespace = new Regex("whitespace", "[ \\t]+");
+const newLine = new Regex("new-line", "(\\r?\\n)+");
+
+whitespace.setTokens([" "]);
+newLine.setTokens(["\n"])
+
+const line = new Or("line", [
+    comment,
+    statement,
+    whitespace
+], true);
+
+export const grammar = new Repeat("grammar", line, { divider: newLine });

package/src/grammar/patterns/literal.ts

@@ -0,0 +1,5 @@
+import { Regex } from "../../patterns/Regex";
+
+export const literal = new Regex("literal", "\"(?:\\\\[\"\\\\]|[^\n\"\\\\])*\"");
+
+

package/src/grammar/patterns/name.ts

@@ -0,0 +1,3 @@
+import { Regex } from "../../patterns/Regex";
+
+export const name = new Regex("name", "[a-zA-Z_-]+[a-zA-Z0-9_-]*");

package/src/grammar/patterns/orLiteral.ts

@@ -0,0 +1,8 @@
+import { Repeat } from "../../patterns/Repeat"
+import { name } from "./name"
+import { Regex } from "../../patterns/Regex";
+
+const divider = new Regex("or-divider", "\\s*[|]\\s*");
+divider.setTokens([" | "]);
+
+export const orLiteral = new Repeat("or-literal", name.clone("pattern-name"), { divider, min: 2 });

package/src/grammar/patterns/pattern.ts

@@ -0,0 +1,13 @@
+import { And } from "../../patterns/And";
+import { Literal } from "../../patterns/Literal";
+import { name } from "./name";
+
+const optionalNot = new Literal("not", "!", true);
+const optionalIsOptional = new Literal("is-optional", "?", true);
+const patternName = name.clone("pattern-name");
+
+export const pattern = new And("pattern", [
+    optionalNot,
+    patternName,
+    optionalIsOptional,
+]);

package/src/grammar/patterns/regexLiteral.ts

@@ -0,0 +1,4 @@
+import { Regex } from "../../patterns/Regex";
+
+export const regexLiteral = new Regex("regex-literal", "/(\\\\/|[^/\\n\\r])*/");
+