clarity-pattern-parser 11.3.13 → 11.4.1

package/dist/patterns/Cursor.d.ts

@@ -3,7 +3,7 @@ import { Match } from "./CursorHistory";
 import { ParseError } from "./ParseError";
 import { Pattern } from "./Pattern";
 export declare class Cursor {
-    private _text;
+    private _chars;
     private _index;
     private _length;
     private _history;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-pattern-parser",
-  "version": "11.3.13",
+  "version": "11.4.1",
   "description": "Parsing Library for Typescript and Javascript.",
   "main": "./dist/index.js",
   "module": "./dist/index.esm.js",

package/src/grammar/Grammar.test.ts

@@ -779,4 +779,42 @@ describe("Grammar", () => {
         expect(result?.value).toBe("function(){}");
     });
 
+    test("Import Sync", () => {
+        function resolveImportSync(path: string, importer: string | null) {
+            return {
+                expression: pathMap[path],
+                resource: path,
+            }
+        }
+        
+        const rootExpression = `
+        import { name } from "first-name.cpat"
+        full-name = name
+        `;
+
+        const firstNameExpression = `
+        import { last-name } from "last-name.cpat"
+        first-name = "John"
+        name = first-name + " " +  last-name
+        `;
+
+        const lastNameExpression = `
+        last-name = "Doe"
+        `;
+
+        const pathMap: Record<string, string> = {
+            "first-name.cpat": firstNameExpression,
+            "last-name.cpat": lastNameExpression,
+        };
+
+        const patterns = Grammar.parseString(rootExpression, {
+            resolveImportSync: resolveImportSync,
+            originResource: "/root.cpat",
+        });
+
+        expect(patterns["full-name"]).not.toBeNull();
+        const result = patterns["full-name"].exec("John Doe");
+        expect(result?.ast?.value).toBe("John Doe");
+    });
+
 });

package/src/grammar/Grammar.ts

@@ -50,6 +50,10 @@ function defaultImportResolver(_path: string, _basePath: string | null): Promise
     throw new Error("No import resolver supplied.");
 }
 
+function defaultImportResolverSync(_path: string, _basePath: string | null): GrammarFile {
+    throw new Error("No import resolver supplied.");
+}
+
 export interface GrammarFile {
     resource: string;
     expression: string;
@@ -57,6 +61,7 @@ export interface GrammarFile {
 
 export interface GrammarOptions {
     resolveImport?: (resource: string, originResource: string | null) => Promise<GrammarFile>;
+    resolveImportSync?: (resource: string, originResource: string | null) => GrammarFile;
     originResource?: string | null;
     params?: Pattern[];
     decorators?: Record<string, Decorator>;
@@ -66,12 +71,14 @@ export class Grammar {
     private _params: Pattern[];
     private _originResource?: string | null;
     private _resolveImport: (resource: string, originResource: string | null) => Promise<GrammarFile>;
+    private _resolveImportSync: (resource: string, originResource: string | null) => GrammarFile;
     private _parseContext: ParseContext;
 
     constructor(options: GrammarOptions = {}) {
         this._params = options?.params == null ? [] : options.params;
         this._originResource = options?.originResource == null ? null : options.originResource;
         this._resolveImport = options.resolveImport == null ? defaultImportResolver : options.resolveImport;
+        this._resolveImportSync = options.resolveImportSync == null ? defaultImportResolverSync : options.resolveImportSync;
         this._parseContext = new ParseContext(this._params, options.decorators || {});
     }
 
@@ -101,10 +108,7 @@ export class Grammar {
         this._parseContext = new ParseContext(this._params, this._parseContext.decorators);
         const ast = this._tryToParse(expression);
 
-        if (this._hasImports(ast)) {
-            throw new Error("Cannot use imports on parseString, use parse instead.");
-        }
-
+        this._resolveImportsSync(ast);
         this._buildPatterns(ast);
 
         return this._buildPatternRecord();
@@ -476,14 +480,89 @@ export class Grammar {
 
         for (const statement of importStatements) {
             if (statement.name === "import-from") {
-                await this.processImport(statement);
+                await this._processImport(statement);
             } else {
-                this.processUseParams(statement);
+                this._processUseParams(statement);
             }
         }
     }
 
-    private async processImport(importStatement: Node) {
+    private _resolveImportsSync(ast: Node) {
+        const importStatements = ast.findAll(n => {
+            return n.name === "import-from" || n.name === "param-name-with-default-value";
+        });
+
+        for (const statement of importStatements) {
+            if (statement.name === "import-from") {
+                this._processImportSync(statement);
+            } else {
+                this._processUseParams(statement);
+            }
+        }
+    }
+
+    private _processImportSync(importStatement: Node) {
+        const parseContext = this._parseContext;
+        const resourceNode = importStatement.find(n => n.name === "resource") as Node;
+        const params = this._getParams(importStatement);
+        const resource = resourceNode.value.slice(1, -1);
+        const grammarFile = this._resolveImportSync(resource, this._originResource || null);
+        const grammar = new Grammar({
+            resolveImport: this._resolveImport,
+            resolveImportSync: this._resolveImportSync,
+            originResource: grammarFile.resource,
+            params,
+            decorators: this._parseContext.decorators
+        });
+
+        try {
+            const patterns = grammar.parseString(grammarFile.expression);
+            const importStatements = importStatement.findAll(n => n.name === "import-name" || n.name === "import-alias");
+
+            importStatements.forEach((node) => {
+                if (node.name === "import-name" && node.parent?.name === "import-alias") {
+                    return;
+                }
+
+                if (node.name === "import-name" && node.parent?.name !== "import-alias") {
+                    const importName = node.value;
+
+                    if (parseContext.importedPatternsByName.has(importName)) {
+                        throw new Error(`'${importName}' was already used within another import.`);
+                    }
+
+                    const pattern = patterns[importName];
+                    if (pattern == null) {
+                        throw new Error(`Couldn't find pattern with name: ${importName}, from import: ${resource}.`);
+                    }
+
+                    parseContext.importedPatternsByName.set(importName, pattern);
+                } else {
+                    const importNameNode = node.find(n => n.name === "import-name") as Node;
+                    const importName = importNameNode.value;
+                    const aliasNode = node.find(n => n.name === "import-name-alias") as Node;
+                    const alias = aliasNode.value;
+
+                    if (parseContext.importedPatternsByName.has(alias)) {
+                        throw new Error(`'${alias}' was already used within another import.`);
+                    }
+
+                    const pattern = patterns[importName];
+                    if (pattern == null) {
+                        throw new Error(`Couldn't find pattern with name: ${importName}, from import: ${resource}.`);
+                    }
+
+                    parseContext.importedPatternsByName.set(alias, pattern.clone(alias));
+                }
+            });
+
+        } catch (e: any) {
+            throw new Error(`Failed loading expression from: "${resource}". Error details: "${e.message}"`);
+        }
+
+    }
+
+    private async _processImport(importStatement: Node) {
         const parseContext = this._parseContext;
         const resourceNode = importStatement.find(n => n.name === "resource") as Node;
         const params = this._getParams(importStatement);
@@ -543,7 +622,7 @@ export class Grammar {
 
     }
 
-    private processUseParams(paramName: Node) {
+    private _processUseParams(paramName: Node) {
         const defaultValueNode = paramName.find(n => n.name === "param-default");
         if (defaultValueNode === null) {
             return;

package/src/grammar_v2/patterns/Grammar.ts

@@ -0,0 +1,170 @@
+import { Node } from "../../ast/Node";
+import { tokens } from "../../grammar/decorators/tokens";
+import { GrammarFile } from "../../grammar/Grammar";
+import { generateErrorMessage } from "../../patterns/generate_error_message";
+import { Pattern } from "../../patterns/Pattern";
+import { grammar } from "./patterns/grammar"
+
+let anonymousIndexId = 0;
+
+function defaultImportResolver(_path: string, _basePath: string | null): Promise<GrammarFile> {
+    throw new Error("No import resolver supplied.");
+}
+
+function defaultImportResolverSync(_path: string, _basePath: string | null): GrammarFile {
+    throw new Error("No import resolver supplied.");
+}
+
+export type Decorator = (pattern: Pattern, arg?: string | boolean | number | null | Record<string, any> | any[]) => void;
+
+const defaultDecorators = {
+    tokens: tokens
+};
+
+export interface GrammarOptions {
+    resolveImport?: (resource: string, originResource: string | null) => Promise<GrammarFile>;
+    resolveImportSync?: (resource: string, originResource: string | null) => GrammarFile;
+    originResource?: string | null;
+    params?: Pattern[];
+    decorators?: Record<string, Decorator>;
+}
+
+class ParseContext {
+    patternsByName = new Map<string, Pattern>();
+    importedPatternsByName = new Map<string, Pattern>();
+    paramsByName = new Map<string, Pattern>();
+    decorators: Record<string, Decorator>;
+
+    constructor(params: Pattern[], decorators: Record<string, Decorator> = {}) {
+        params.forEach(p => this.paramsByName.set(p.name, p));
+        this.decorators = { ...decorators, ...defaultDecorators };
+    }
+
+    getImportedPatterns() {
+        return Array.from(this.importedPatternsByName.values());
+    }
+
+    getParams() {
+        return Array.from(this.paramsByName.values());
+    }
+
+}
+
+export class Grammar {
+    private _options: GrammarOptions;
+    private _parseContext: ParseContext;
+    private _resolveImportSync: (resource: string, originResource: string | null) => GrammarFile;
+
+    constructor(options: GrammarOptions) {
+        this._options = options;
+        this._parseContext = new ParseContext(options.params || [], options.decorators || {});
+        this._resolveImportSync = options.resolveImportSync == null ? defaultImportResolverSync : options.resolveImportSync;
+    }
+
+    // parse(cpat: string): Promise<Pattern> { }
+    // import(path: string): Promise<Pattern> { }
+
+    // parseString(cpat: string): Pattern {
+    //     const ast = this._tryToParse(cpat);
+
+    //     this._flattenExpressions(ast);
+
+    // }
+
+    private _resolveImportsSync(ast: Node) {
+        const importStatements = ast.findAll(n => {
+            return n.name === "importStatement" || n.name === "useParamsStatement";
+        });
+
+        for (const importStatement of importStatements) {
+            if (importStatement.name === "importStatement") {
+                this._processImportSync(importStatement);
+            } else {
+                this._processUseParams(importStatement);
+            }
+        }
+    }
+
+    private _processImportSync(importStatement: Node) {
+        const parseContext = this._parseContext;
+        const resourceNode = importStatement.find(n => n.name === "resource") as Node;
+        const params = this._getWithParams(importStatement);
+        const resource = resourceNode.value.slice(1, -1);
+        const grammarFile = this._resolveImportSync(resource, this._options.originResource || null);
+
+
+        if (resourceNode == null) {
+            throw new Error("Invalid import statement: resource is required");
+        }
+
+
+    }
+
+    private _getWithParams(importStatement: Node) {
+        const withParamsStatements = importStatement.find(n => n.name === "withParamsStatements");
+
+        if (withParamsStatements == null) {
+            return [];
+        }
+
+        const expression = withParamsStatements.toString();
+        const importedValues = this._parseContext.getImportedPatterns();
+
+        const grammar = new Grammar({
+            params: [...importedValues, ...this._parseContext.paramsByName.values()],
+            originResource: this._options.originResource,
+            resolveImport: this._options.resolveImport,
+            decorators: this._parseContext.decorators
+        });
+
+        // const patterns = grammar.parseString(expression);
+        // return Array.from(Object.values(patterns));
+        return[]
+    }
+
+
+    private _processUseParams(useParamsStatement: Node) {
+    }
+
+    private _tryToParse(cpat: string): Node {
+        const { ast, cursor } = grammar.exec(cpat, true);
+
+        if (ast == null) {
+            const message = generateErrorMessage(grammar, cursor);
+            throw new Error(`[Invalid Grammar] ${message}`);
+        }
+
+        return ast;
+    }
+
+
+    private _flattenExpressions(node: Node) {
+        switch (node.name) {
+            case "sequenceExpr":
+                return this._unwrapNode("sequenceExpr", node);
+            case "optionsExpr":
+                return this._unwrapNode("optionsExpr", node);
+            case "greedyOptionsExpr":
+                return this._unwrapNode("greedyOptionsExpr", node);
+            default:
+                return node;
+        }
+    }
+
+    private _unwrapNode(type: string, node: Node) {
+        if (node.name !== type) {
+            return node;
+        }
+
+        for (let x = 0; x < node.children.length; x++) {
+            const child = node.children[x];
+
+            if (child.name === type) {
+                node.spliceChildren(x, 1, ...child.children);
+                x--;
+            }
+        }
+
+        return node;
+    }
+}

package/src/grammar_v2/patterns/patterns/cpat.cpat

@@ -0,0 +1,91 @@
+# syntax
+syntax = "syntax"
+import = "import"
+useParams = "use params"
+withParams = "with params"
+from = "from"
+right = "right"
+ws = /\s+/
+ls = /[ \t]+/
+assign = "="
+bar = "|"
+concat = "+"
+colon = ":"
+openParen = "("
+closeParen = ")"
+openSquareBracket = "["
+closeSquareBracket = "]"
+openBracket = "{"
+closeBracket = "}"
+comma = /\s*[,]\s*/
+trim = "trim"
+not = "!"
+optional = "?"
+newLine = /\s*(\r\n|\r|\n)\s*/
+syntaxVersion = /\S*/
+at = "@"
+
+literal = /"(?:\\.|[^"\\])*"/
+regexLiteral = /\/(\\/|[^\n\r])*\//
+
+integer = /[1-9][0-9]*|[0]/ 
+name = /[a-zA-Z_-]+[a-zA-Z0-9_-]*/
+patternName = name
+patternIdentifier = name
+resource = literal
+
+comment = /#[^\n\r]*/
+
+# JSON
+jsonString = literal
+jsonNumber = /-?\d+(\.\d+)?/
+jsonBoolean = "true" | "false"
+jsonNull = "null"
+jsonValue = jsonString | jsonNumber | jsonBoolean | jsonNull | jsonArray | jsonObject
+jsonArrayItems = (jsonValue, comma trim)+
+jsonArray = openSquareBracket +ws? + jsonArrayItems? + ws? + closeSquareBracket
+jsonObjectPropertyName = literal
+jsonObjectProperty = jsonObjectPropertyName + ws? + colon + ws? + jsonValue
+jsonObjectProperties = (jsonObjectProperty, comma trim)+
+jsonObject = openBracket + ws? + jsonObjectProperties? + ws? + closeBracket
+
+syntaxStatement = syntax + ws? + syntaxVersion
+
+decorationName = name
+decorationStatement = at + ws? + decorationName + ws? + openParen + ws? + jsonValue? + ws? + closeParen
+
+useParamPatterns = (patternName, comma trim)+
+useParamsStatement = useParams + ls? + openBracket + ws? + useParamPatterns + ws? + closeBracket
+
+withParamStatements = (patternAssignment, newLine trim)+
+withParamsExpr = withParams + ls? + openBracket + ws? + withParamStatements + ws? + closeBracket
+
+patternNames = (patternName, comma trim)+
+importedPatterns = openBracket + ws? + patternNames + ws? + closeBracket 
+importStatement = import + ls? + importedPatterns + ls? + from + ls? + resource + ls? + withParamsExpr?
+
+notExpr = not + ls? + unaryPatternExpr
+optionalExpr = unaryPatternExpr + ls? + optional
+rightAssociationExpr = unaryPatternExpr + ls? + right
+
+unaryPatternExpr = notExpr | optionalExpr | rightAssociationExpr | patternIdentifier
+
+repeatBounds = openBracket + integer? + comma? + integer? + closeBracket
+oneOrMore = "+"
+zeroOrMore = "*"
+repeatOptions = oneOrMore | zeroOrMore | repeatBounds
+delimiter = comma + patternExpr + ws? + trim?
+repeatExpr = openParen + ws? + patternExpr + ws? + delimiter? + ws? + closeParen + repeatOptions
+
+sequenceExpr = patternExpr + ws? + "+" + ws? + patternExpr 
+optionsExpr = patternExpr + ws? + "|" + ws? + patternExpr
+greedyOptionsExpr = patternExpr + ws? + "<|>" + ws? + patternExpr
+patternGroupExpr = openParen + ws? + patternExpr + ws? + closeParen
+
+exportPattern = patternName
+patternExpr =  sequenceExpr | optionsExpr | greedyOptionsExpr |  repeatExpr | patternGroupExpr | literal | regexLiteral | unaryPatternExpr 
+patternAssignment = patternName + ws? + assign + ws? + patternExpr
+statement = useParamsStatement | importStatement  | patternAssignment | decorationStatement | exportPattern | comment 
+statements = (statement, newLine)+
+cpat = ws? + syntaxStatement? + ws? + statements?
+