clarity-pattern-parser 8.2.1 → 8.3.0

package/src/grammar/Grammar.test.ts

@@ -13,7 +13,7 @@ describe("Grammar", () => {
             name = "John"
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const namePattern = patterns.get("name");
         const expected = new Literal("name", "John");
 
@@ -25,7 +25,7 @@ describe("Grammar", () => {
             name = /\\w/
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("name");
         const name = new Regex("name", "\\w");
 
@@ -39,7 +39,7 @@ describe("Grammar", () => {
             names = john | jane
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("names");
         const john = new Literal("john", "John");
         const jane = new Literal("jane", "Jane");
@@ -56,7 +56,7 @@ describe("Grammar", () => {
             full-name = first-name & space & last-name
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("full-name");
         const space = new Literal("space", " ");
         const firstName = new Regex("first-name", "\\w");
@@ -76,7 +76,7 @@ describe("Grammar", () => {
             full-name = first-name & space & middle-name-with-space? & last-name
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("full-name");
         const space = new Literal("space", " ");
         const firstName = new Regex("first-name", "\\w");
@@ -99,7 +99,7 @@ describe("Grammar", () => {
             full-name = !jack & first-name & space & middle-name-with-space? & last-name
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("full-name");
         const space = new Literal("space", " ");
         const firstName = new Regex("first-name", "\\w");
@@ -119,7 +119,7 @@ describe("Grammar", () => {
             digits = (digit)+
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("digits");
         const digit = new Regex("digit", "\\d");
         const digits = new Repeat("digits", digit);
@@ -133,7 +133,7 @@ describe("Grammar", () => {
             digits = (digit)*
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("digits");
         const digit = new Regex("digit", "\\d");
         const digits = new Repeat("digits", digit, { min: 0 });
@@ -147,7 +147,7 @@ describe("Grammar", () => {
             digits = (digit){1,}
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("digits");
         const digit = new Regex("digit", "\\d+");
         const digits = new Repeat("digits", digit, { min: 1 });
@@ -161,7 +161,7 @@ describe("Grammar", () => {
             digits = (digit){1,3}
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("digits");
         const digit = new Regex("digit", "\\d+");
         const digits = new Repeat("digits", digit, { min: 1, max: 3 });
@@ -175,7 +175,7 @@ describe("Grammar", () => {
             digits = (digit){,3}
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("digits");
         const digit = new Regex("digit", "\\d+");
         const digits = new Repeat("digits", digit, { min: 0, max: 3 });
@@ -189,7 +189,7 @@ describe("Grammar", () => {
             digits = (digit){3}
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("digits");
         const digit = new Regex("digit", "\\d+");
         const digits = new Repeat("digits", digit, { min: 3, max: 3 });
@@ -204,7 +204,7 @@ describe("Grammar", () => {
             digits = (digit, comma){3}
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("digits");
         const digit = new Regex("digit", "\\d+");
         const divider = new Literal("comma", ",");
@@ -220,7 +220,7 @@ describe("Grammar", () => {
             digits = (digit, comma){3} -t
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("digits");
         const digit = new Regex("digit", "\\d+");
         const divider = new Literal("comma", ",");
@@ -236,7 +236,7 @@ describe("Grammar", () => {
             digits = (digit?, comma)+
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("digits") as Pattern;
         const digit = new Regex("digit", "\\d+", true);
         const comma = new Literal("comma", ",");
@@ -257,7 +257,7 @@ describe("Grammar", () => {
             array = open-bracket & spaces? & array-items? & spaces? & close-bracket
         `;
 
-        const patterns = Grammar.parse(expression);
+        const patterns = Grammar.parseString(expression);
         const pattern = patterns.get("array") as Pattern;
 
         let text = "[1, []]";
@@ -272,8 +272,8 @@ describe("Grammar", () => {
             alias = name
         `;
 
-        const patterns = Grammar.parse(expression);
-        
+        const patterns = Grammar.parseString(expression);
+
         const name = patterns.get("name");
         const expectedName = new Regex("name", "regex");
 
@@ -288,7 +288,7 @@ describe("Grammar", () => {
 
         expect(() => {
             const expression = `Just Junk`;
-            Grammar.parse(expression);
+            Grammar.parseString(expression);
         }).toThrowError("[Parse Error] Found: 'Just Junk', expected: ' ='.");
 
     });
@@ -299,8 +299,54 @@ describe("Grammar", () => {
             const expression = `name = /\\w/
                 age = /()
             `;
-            Grammar.parse(expression);
+            Grammar.parseString(expression);
         }).toThrowError("[Parse Error] Found: '    age = /()\n      ', expected: '    age = !' or '    age = (' or '    age = [Pattern Name]' or '    age = [Regular Expression]' or '    age = [String]'.")
 
     });
+
+    test("Import", async () => {
+        const importExpression = `first-name = "John"`;
+        const expression = `
+        import { first-name } from "some/path/to/file.cpat"
+        last-name = "Doe"
+        space = " "
+        full-name = first-name & space & last-name
+        `
+        function resolveImport(path: string) {
+            expect(path).toBe("some/path/to/file.cpat");
+            return Promise.resolve({ expression: importExpression, path });
+        }
+
+        const patterns = await Grammar.parse(expression, { resolveImport });
+        const fullname = patterns.get("full-name") as Pattern;
+        const result = fullname.exec("John Doe");
+
+        expect(result?.ast?.value).toBe("John Doe");
+    });
+
+    test("Imports", async () => {
+        const importExpression = `first-name = "John"`;
+        const spaceExpression = `space = " "`
+
+        const pathMap: Record<string, string> = {
+            "space.cpat": spaceExpression,
+            "first-name.cpat": importExpression
+        };
+
+        const expression = `
+        import { first-name } from "first-name.cpat"
+        import { space } from "space.cpat"
+        last-name = "Doe"
+        full-name = first-name & space & last-name
+        `
+        function resolveImport(path: string) {
+            return Promise.resolve({ expression: pathMap[path], path });
+        }
+
+        const patterns = await Grammar.parse(expression, { resolveImport });
+        const fullname = patterns.get("full-name") as Pattern;
+        const result = fullname.exec("John Doe");
+        expect(result?.ast?.value).toBe("John Doe");
+    });
+
 });

package/src/grammar/Grammar.ts

@@ -12,13 +12,36 @@ import { AutoComplete } from "../intellisense/AutoComplete";
 
 class ParseContext {
     patternsByName = new Map<string, Pattern>();
+    importedPatternsByName = new Map<string, Pattern>();
+}
+
+function defaultImportResolver(_path: string, _basePath: string | null): Promise<GrammarFile> {
+    throw new Error("No import resolver supplied.");
+}
+
+export interface GrammarMeta {
+    originPath?: string;
+}
+
+export interface GrammarFile {
+    path: string;
+    expression: string;
+}
+
+export interface GrammarOptions {
+    resolveImport?: (path: string, originPath: string | null) => Promise<GrammarFile>;
+    meta?: GrammarMeta | null;
 }
 
 export class Grammar {
+    private _meta: GrammarMeta | null;
+    private _resolveImport: (path: string, basePath: string | null) => Promise<GrammarFile>;
     private _parseContext: ParseContext;
     private _autoComplete: AutoComplete;
 
-    constructor() {
+    constructor(options: GrammarOptions = {}) {
+        this._meta = options.meta == null ? null : options.meta;
+        this._resolveImport = options.resolveImport == null ? defaultImportResolver : options.resolveImport;
         this._parseContext = new ParseContext();
         this._autoComplete = new AutoComplete(grammar, {
             greedyPatternNames: ["spaces", "optional-spaces", "whitespace", "new-line"],
@@ -31,14 +54,39 @@ export class Grammar {
         });
     }
 
-    parse(expression: string) {
+    async import(path: string) {
+        const grammarFile = await this._resolveImport(path, null);
+        const grammar = new Grammar({ resolveImport: this._resolveImport, meta: { originPath: grammarFile.path } });
+
+        return grammar.parse(grammarFile.expression);
+    }
+
+    async parse(expression: string) {
         this._parseContext = new ParseContext();
-        this._tryToParse(expression);
+        const ast = this._tryToParse(expression);
+
+        await this._resolveImports(ast);
+        this._buildPatterns(ast);
+        this._cleanAst(ast);
 
         return this._parseContext.patternsByName;
     }
 
-    private _tryToParse(expression: string) {
+    parseString(expression: string) {
+        this._parseContext = new ParseContext();
+        const ast = this._tryToParse(expression);
+
+        if (this._hasImports(ast)) {
+            throw new Error("Cannot use imports on parseString, use parse instead.");
+        }
+
+        this._buildPatterns(ast);
+        this._cleanAst(ast);
+
+        return this._parseContext.patternsByName;
+    }
+
+    private _tryToParse(expression: string): Node {
         const { ast, cursor, options, isComplete } = this._autoComplete.suggestFor(expression);
 
         if (!isComplete) {
@@ -54,9 +102,16 @@ export class Grammar {
         }
 
         // If it is complete it will always have a node. So we have to cast it.
-        this._cleanAst(ast as Node);
-        this._buildPatterns(ast as Node);
+        return ast as Node;
+    }
+
+    private _hasImports(ast: Node) {
+        const importBlock = ast.find(n => n.name === "import-block");
+        if (importBlock == null) {
+            return false;
+        }
 
+        return importBlock && importBlock.children.length > 0;
     }
 
     private _cleanAst(ast: Node) {
@@ -99,10 +154,48 @@ export class Grammar {
                     this._buildAlias(n)
                     break;
                 }
+                default: {
+                    break;
+                }
             }
         });
     }
 
+    private async _resolveImports(ast: Node) {
+        const parseContext = this._parseContext;
+        const importStatements = ast.findAll(n => n.name === "import-statement");
+
+        for (const importStatement of importStatements) {
+            const urlNode = importStatement.find(n => n.name === "url") as Node;
+
+            const url = urlNode.value.slice(1, -1);
+            const grammarFile = await this._resolveImport(url, this._meta?.originPath || null);
+            const grammar = new Grammar({ resolveImport: this._resolveImport, meta: { originPath: grammarFile.path } });
+
+            try {
+                const patterns = await grammar.parse(grammarFile.expression);
+                const importNames = importStatement.findAll(n => n.name === "import-name").map(n => n.value);
+
+                importNames.forEach((importName) => {
+                    if (parseContext.importedPatternsByName.has(importName)) {
+                        throw new Error(`'${importName}' was already used within another import.`);
+                    }
+
+                    const pattern = patterns.get(importName);
+                    if (pattern == null) {
+                        throw new Error(`Couldn't find pattern with name: ${importName}, from import: ${url}.`);
+                    }
+
+                    parseContext.importedPatternsByName.set(importName, pattern);
+                })
+
+            } catch (e: any) {
+                throw new Error(`Failed loading expression from: "${url}". Error details: "${e.message}"`);
+            }
+
+        }
+    }
+
     private _buildLiteral(statementNode: Node) {
         const nameNode = statementNode.find(n => n.name === "name") as Node;
         const literalNode = statementNode.find(n => n.name === "literal") as Node;
@@ -137,7 +230,11 @@ export class Grammar {
     }
 
     private _getPattern(name: string) {
-        const pattern = this._parseContext.patternsByName.get(name);
+        let pattern = this._parseContext.patternsByName.get(name);
+
+        if (pattern == null) {
+            pattern = this._parseContext.importedPatternsByName.get(name);
+        }
 
         if (pattern == null) {
             return new Reference(name);
@@ -241,9 +338,19 @@ export class Grammar {
         this._parseContext.patternsByName.set(name, alias)
     }
 
-    static parse(expression: string) {
-        const grammar = new Grammar();
+    static parse(expression: string, options?: GrammarOptions) {
+        const grammar = new Grammar(options);
         return grammar.parse(expression);
     }
 
+    static import(path: string, options?: GrammarOptions) {
+        const grammar = new Grammar(options);
+        return grammar.import(path);
+    }
+
+    static parseString(expression: string) {
+        const grammar = new Grammar();
+        return grammar.parseString(expression);
+    }
+
 }

package/src/grammar/patterns/grammar.ts

@@ -3,17 +3,20 @@ import { Regex } from "../../patterns/Regex";
 import { Repeat } from "../../patterns/Repeat";
 import { comment } from "./comment";
 import { statement } from "./statement";
+import { importStatement } from './import';
 
-const whitespace = new Regex("whitespace", "[ \\t]+");
+const whitespace = new Regex("whitespace", "[ \\t]+((\\r?\\n)+)?");
 const newLine = new Regex("new-line", "(\\r?\\n)+");
 
 whitespace.setTokens([" "]);
-newLine.setTokens(["\n"])
+newLine.setTokens(["\n"]);
 
 const line = new Or("line", [
+    newLine,
+    whitespace,
     comment,
-    statement,
-    whitespace
-], true);
+    importStatement,
+    statement
+]);
 
-export const grammar = new Repeat("grammar", line, { divider: newLine });
+export const grammar = new Repeat("grammer", line);

package/src/grammar/patterns/import.ts

@@ -0,0 +1,29 @@
+import { And } from "../../patterns/And";
+import { Repeat } from "../../patterns/Repeat";
+import { Literal } from "../../patterns/Literal";
+import { Regex } from "../../patterns/Regex";
+import { literal } from "./literal";
+
+const spaces = new Regex("spaces", "\\s+", true);
+const importNameDivider = new Regex("import-name-divider", "(\\s+)?,(\\s+)?");
+const importKeyword = new Literal("import", "import");
+const fromKeyword = new Literal("from", "from");
+const openBracket = new Literal("open-bracket", "{");
+const closeBracket = new Literal("close-bracket", "}");
+const name = new Regex("import-name", "[^}\\s,]+");
+const importedNames = new Repeat("imported-names", name, { divider: importNameDivider });
+const optionalSpaces = spaces.clone("optional-spaces", true);
+
+export const importStatement = new And("import-statement", [
+    importKeyword,
+    optionalSpaces,
+    openBracket,
+    optionalSpaces,
+    importedNames,
+    optionalSpaces,
+    closeBracket,
+    optionalSpaces,
+    fromKeyword,
+    spaces,
+    literal.clone("url"),
+]);

package/src/grammar/spec.md

@@ -69,7 +69,6 @@ digits = (digit)*
 ```
 
 ### Zero Or More Pattern
-
 ```
 digit = /\d/
 comma = ","
@@ -138,5 +137,30 @@ digits = (digit, comma){1,3}
 ```
 digit = /\d/
 comma = ","
-digits = (digit, comma){1,3}
+digits = (digit, comma){1,}
+```
+
+### Import
+```
+import { spaces } from "https://some.cdn.com/some/spaces.cpat"
+
+first-name = "John"
+last-name = "Doe"
+full-name = first-name & spaces & last-name
+```
+
+### Muliple Named Imports
+```
+import { spaces, first-name } from "https://some.cdn.com/some/spaces.cpat"
+
+last-name = "Doe"
+full-name = first-name & spaces & last-name
+```
+
+### Muliple Imports
+```
+import { spaces, first-name } from "https://some.cdn.com/some/patterns.cpat"
+import { last-name } from "https://some.cdn.com/some/last-name.cpat"
+
+full-name = first-name & spaces & last-name
 ```

package/src/patterns/And.ts

@@ -190,7 +190,6 @@ export class And implements Pattern {
     const children = filterOutNull(this._nodes);
 
     const lastIndex = children[children.length - 1].lastIndex;
-    const value = cursor.getChars(this._firstIndex, lastIndex);
 
     cursor.moveTo(lastIndex)
 
@@ -250,15 +249,12 @@ export class And implements Pattern {
 
   getPatternsAfter(childReference: Pattern): Pattern[] {
     const patterns: Pattern[] = [];
-    let nextSibling: Pattern | null = null;
     let nextSiblingIndex = -1;
     let index = -1;
 
     for (let i = 0; i < this._children.length; i++) {
       if (this._children[i] === childReference) {
-        if (i + 1 < this._children.length) {
-          nextSibling = this._children[i + 1];
-        }
+
         nextSiblingIndex = i + 1;
         index = i;
         break;

package/src/patterns/FiniteRepeat.ts

@@ -111,8 +111,9 @@ export class FiniteRepeat implements Pattern {
         }
 
         if (matchCount < this._min) {
+            const lastIndex = cursor.index;
             cursor.moveTo(startIndex);
-            cursor.recordErrorAt(startIndex, startIndex, this);
+            cursor.recordErrorAt(startIndex, lastIndex, this);
             return null;
         } else if (nodes.length === 0) {
             cursor.resolveError();

package/src/patterns/InfiniteRepeat.test.ts

@@ -109,29 +109,6 @@ describe("InfiniteRepeat", () => {
         expect(cursor.hasError).toBeFalsy()
     });
 
-    test("Optional Repeating Pattern", () => {
-        const digit = new Regex("digit", "\\d+", true);
-        const divider = new Regex("divider", "\\s");
-        const integer = new InfiniteRepeat("number", digit, { divider });
-        const cursor = new Cursor(
-            "\n" +
-            "3\n" +
-            "\n" +
-            "\n"
-        );
-        const result = integer.parse(cursor);
-        const expected = new Node("infinite-repeat", "number", 0, 4, [
-            new Node("regex", "divider", 0, 0, [], "\n"),
-            new Node("regex", "digit", 1, 1, [], "3"),
-            new Node("regex", "divider", 2, 2, [], "\n"),
-            new Node("regex", "divider", 3, 3, [], "\n"),
-            new Node("regex", "divider", 4, 4, [], "\n"),
-        ]);
-
-        expect(result).toEqual(expected)
-        expect(cursor.hasError).toBeFalsy()
-    });
-
     test("Failed (Optional)", () => {
         const digit = new Regex("digit", "\\d");
         const integer = new InfiniteRepeat("number", digit, { min: 0 });
@@ -293,4 +270,13 @@ describe("InfiniteRepeat", () => {
 
         expect(clone).toEqual(expected);
     });
+
+    test("No Results, min is 0", () => {
+        const numbers = new InfiniteRepeat("numbers", new Regex("number", "\\d"), { divider: new Literal(",", ","), min: 0 });
+        const result = numbers.exec("j");
+        expect(result.ast).toBeNull();
+        expect(result.cursor.index).toBe(0);
+    });
+
+
 });

package/src/patterns/InfiniteRepeat.ts

@@ -56,9 +56,9 @@ export class InfiniteRepeat implements Pattern {
     let children: Pattern[];
 
     if (divider != null) {
-      children = [pattern.clone(), divider.clone(divider.name, false)]
+      children = [pattern.clone(pattern.name, false), divider.clone(divider.name, false)]
     } else {
-      children = [pattern.clone()]
+      children = [pattern.clone(pattern.name, false)]
     }
 
     this._assignChildrenToParent(children);
@@ -208,6 +208,11 @@ export class InfiniteRepeat implements Pattern {
       cursor.moveTo(dividerNode.firstIndex);
     }
 
+    // if (this._nodes.length === 0) {
+    //   cursor.moveTo(this._firstIndex);
+    //   return null;
+    // }
+
     const lastIndex = this._nodes[this._nodes.length - 1].lastIndex;
     cursor.moveTo(lastIndex);