clarity-pattern-parser 11.0.30 → 11.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-pattern-parser",
-  "version": "11.0.30",
+  "version": "11.1.0",
   "description": "Parsing Library for Typescript and Javascript.",
   "main": "./dist/index.js",
   "module": "./dist/index.esm.js",

package/src/grammar/Grammar.test.ts

@@ -9,7 +9,7 @@ import { Repeat } from "../patterns/Repeat";
 import { Grammar } from "./Grammar";
 import { Optional } from "../patterns/Optional";
 import { Context } from "../patterns/Context";
-import { patterns } from "./patterns";
+import { createPatternsTemplate, patterns } from "./patterns";
 import { Expression } from "../patterns/Expression";
 
 describe("Grammar", () => {
@@ -176,7 +176,7 @@ describe("Grammar", () => {
         const digits = new Optional("optional-digits", new Repeat("digits", digit, { min: 0 }));
 
         const expected = new Context("optional-digits", digits, [digit]);
-debugger;
+        debugger;
         expect(pattern.isEqual(expected)).toBeTruthy();
     });
 
@@ -596,4 +596,90 @@ debugger;
         expect(result).toBe(result);
     });
 
+    test("Decorators", () => {
+        const { spaces } = patterns`
+            @tokens([" "])
+            spaces = /\\s+/
+        `;
+
+        expect(spaces.getTokens()).toEqual([" "]);
+    });
+
+    test("Decorators No Args", () => {
+        const { spaces } = patterns`
+            @tokens()
+            spaces = /\\s+/
+        `;
+
+        expect(spaces.getTokens()).toEqual([]);
+    });
+
+    test("Decorators Bad Args", () => {
+        const { spaces } = patterns`
+            @tokens("Bad")
+            spaces = /\\s+/
+        `;
+
+        expect(spaces.getTokens()).toEqual([]);
+    });
+
+    test("Decorators Bad Decorator", () => {
+        const { spaces } = patterns`
+            @bad-decorator()
+            spaces = /\\s+/
+        `;
+
+        expect(spaces.getTokens()).toEqual([]);
+    });
+
+    test("Decorators On Multiple Patterns", () => {
+        const { spaces, digits } = patterns`
+            @tokens([" "])
+            spaces = /\\s+/
+
+            @tokens(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"])
+            digits = /\\d+/
+        `;
+
+        expect(spaces.getTokens()).toEqual([" "]);
+        expect(digits.getTokens()).toEqual(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]);
+    });
+
+    test("Decorators On Multiple Patterns And Comments", () => {
+        const { spaces, digits } = patterns`
+            #Comment
+            @tokens([" "])
+            spaces = /\\s+/
+
+            #Comment
+            @tokens(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"])
+            #Comment
+            digits = /\\d+/
+        `;
+
+        expect(spaces.getTokens()).toEqual([" "]);
+        expect(digits.getTokens()).toEqual(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]);
+    });
+
+    test("Custom Named Decorator", () => {
+        const allRecordedPatterns: string[] = [];
+        const patterns = createPatternsTemplate({
+            decorators: {
+                record: (pattern) => {
+                    allRecordedPatterns.push(pattern.name);
+                }
+            }
+        });
+
+        patterns`
+            @record
+            spaces = /\\s+/
+
+            @record
+            digits = /\\d+/
+        `;
+
+        expect(allRecordedPatterns).toEqual(["spaces", "digits"]);
+    });
+
 });

package/src/grammar/Grammar.ts

@@ -13,9 +13,16 @@ import { Context } from "../patterns/Context";
 import { Expression } from "../patterns/Expression";
 import { RightAssociated } from "../patterns/RightAssociated";
 import { generateErrorMessage } from "../patterns/generate_error_message";
+import { tokens } from "./decorators/tokens";
 
 let anonymousIndexId = 0;
 
+export type Decorator = (pattern: Pattern, arg?: string | boolean | number | null | Record<string, any> | any[]) => void;
+
+const defaultDecorators = {
+    tokens: tokens
+};
+
 const patternNodes: Record<string, boolean> = {
     "literal": true,
     "regex-literal": true,
@@ -27,12 +34,14 @@ const patternNodes: Record<string, boolean> = {
 };
 
 class ParseContext {
-    constructor(params: Pattern[]) {
-        params.forEach(p => this.paramsByName.set(p.name, p));
-    }
     patternsByName = new Map<string, Pattern>();
     importedPatternsByName = new Map<string, Pattern>();
     paramsByName = new Map<string, Pattern>();
+    decorators: Record<string, Decorator>;
+    constructor(params: Pattern[], decorators: Record<string, Decorator> = {}) {
+        params.forEach(p => this.paramsByName.set(p.name, p));
+        this.decorators = { ...decorators, ...defaultDecorators };
+    }
 }
 
 function defaultImportResolver(_path: string, _basePath: string | null): Promise<GrammarFile> {
@@ -48,6 +57,7 @@ export interface GrammarOptions {
     resolveImport?: (resource: string, originResource: string | null) => Promise<GrammarFile>;
     originResource?: string | null;
     params?: Pattern[];
+    decorators?: Record<string, Decorator>;
 }
 
 export class Grammar {
@@ -60,7 +70,7 @@ export class Grammar {
         this._params = options?.params == null ? [] : options.params;
         this._originResource = options?.originResource == null ? null : options.originResource;
         this._resolveImport = options.resolveImport == null ? defaultImportResolver : options.resolveImport;
-        this._parseContext = new ParseContext(this._params);
+        this._parseContext = new ParseContext(this._params, options.decorators || {});
     }
 
     async import(path: string) {
@@ -68,14 +78,15 @@ export class Grammar {
         const grammar = new Grammar({
             resolveImport: this._resolveImport,
             originResource: grammarFile.resource,
-            params: this._params
+            params: this._params,
+            decorators: this._parseContext.decorators
         });
 
         return grammar.parse(grammarFile.expression);
     }
 
     async parse(expression: string) {
-        this._parseContext = new ParseContext(this._params);
+        this._parseContext = new ParseContext(this._params, this._parseContext.decorators);
         const ast = this._tryToParse(expression);
 
         await this._resolveImports(ast);
@@ -85,7 +96,7 @@ export class Grammar {
     }
 
     parseString(expression: string) {
-        this._parseContext = new ParseContext(this._params);
+        this._parseContext = new ParseContext(this._params, this._parseContext.decorators);
         const ast = this._tryToParse(expression);
 
         if (this._hasImports(ast)) {
@@ -191,6 +202,7 @@ export class Grammar {
         const name = nameNode.value;
         const literal = this._buildLiteral(name, literalNode);
 
+        this._applyDecorators(statementNode, literal);
         this._parseContext.patternsByName.set(name, literal);
     }
 
@@ -217,6 +229,7 @@ export class Grammar {
         const name = nameNode.value;
         const regex = this._buildRegex(name, regexNode);
 
+        this._applyDecorators(statementNode, regex);
         this._parseContext.patternsByName.set(name, regex);
     }
 
@@ -231,6 +244,7 @@ export class Grammar {
         const optionsNode = statementNode.find(n => n.name === "options-literal") as Node;
         const options = this._buildOptions(name, optionsNode);
 
+        this._applyDecorators(statementNode, options);
         this._parseContext.patternsByName.set(name, options);
     }
 
@@ -322,6 +336,7 @@ export class Grammar {
         const sequenceNode = statementNode.find(n => n.name === "sequence-literal") as Node;
         const sequence = this._buildSequence(name, sequenceNode);
 
+        this._applyDecorators(statementNode, sequence);
         this._parseContext.patternsByName.set(name, sequence);
     }
 
@@ -351,6 +366,7 @@ export class Grammar {
         const repeatNode = statementNode.find(n => n.name === "repeat-literal") as Node;
         const repeat = this._buildRepeat(name, repeatNode);
 
+        this._applyDecorators(statementNode, repeat);
         this._parseContext.patternsByName.set(name, repeat);
     }
 
@@ -413,6 +429,8 @@ export class Grammar {
         const isOptional = node.children[1] != null;
 
         const anonymous = isOptional ? new Optional(name, this._buildPattern(anonymousNode.children[0])) : this._buildPattern(anonymousNode.children[0]);
+
+        this._applyDecorators(node, anonymous);
         this._parseContext.patternsByName.set(name, anonymous);
     }
 
@@ -433,7 +451,8 @@ export class Grammar {
             const grammar = new Grammar({
                 resolveImport: this._resolveImport,
                 originResource: grammarFile.resource,
-                params
+                params,
+                decorators: this._parseContext.decorators
             });
 
             try {
@@ -486,6 +505,59 @@ export class Grammar {
         }
     }
 
+    private _applyDecorators(statementNode: Node, pattern: Pattern) {
+        const decorators = this._parseContext.decorators;
+        const bodyLine = statementNode.parent;
+
+        if (bodyLine == null) {
+            return;
+        }
+
+        let prevSibling = bodyLine.previousSibling();
+        let decoratorNodes: Node[] = [];
+
+        while (prevSibling != null) {
+            if (prevSibling.find(n => n.name === "assign-statement")) {
+                break;
+            }
+            decoratorNodes.push(prevSibling);
+            prevSibling = prevSibling.previousSibling();
+        }
+
+        decoratorNodes = decoratorNodes.filter(n => n.find(n => n.name.includes("decorator")) != null);
+
+        decoratorNodes.forEach((d) => {
+            const nameNode = d.find(n => n.name === "decorator-name");
+
+            if (nameNode == null || decorators[nameNode.value] == null) {
+                return;
+            }
+
+            const nameDocorator = d.find(n => n.name === "name-decorator");
+
+            if (nameDocorator != null) {
+                decorators[nameNode.value](pattern);
+                return;
+            }
+
+            const methodDecorator = d.find(n => n.name === "method-decorator");
+
+            if (methodDecorator == null) {
+                return;
+            }
+
+            methodDecorator.findAll(n => n.name.includes("space")).forEach(n => n.remove());
+            const argsNode = methodDecorator.children[3];
+
+            if (argsNode == null || argsNode.name === "close-paren") {
+                decorators[nameNode.value](pattern);
+            } else {
+                decorators[nameNode.value](pattern, JSON.parse(argsNode.value));
+            }
+        });
+
+    }
+
     private _getParams(importStatement: Node) {
         let params: Pattern[] = [];
         const paramsStatement = importStatement.find(n => n.name === "with-params-statement");
@@ -504,7 +576,8 @@ export class Grammar {
                 const grammar = new Grammar({
                     params: [...importedValues, ...this._parseContext.paramsByName.values()],
                     originResource: this._originResource,
-                    resolveImport: this._resolveImport
+                    resolveImport: this._resolveImport,
+                    decorators: this._parseContext.decorators
                 });
 
                 const patterns = grammar.parseString(expression);
@@ -539,13 +612,15 @@ export class Grammar {
         const aliasName = aliasNode.value;
         const name = nameNode.value;
         const aliasPattern = this._getPattern(aliasName);
-        
+
         // This solves the problem for an alias pointing to a reference.
-        if (aliasPattern.type === "reference"){
+        if (aliasPattern.type === "reference") {
             const reference = new Reference(name, aliasName);
+            this._applyDecorators(statementNode, reference);
             this._parseContext.patternsByName.set(name, reference);
         } else {
             const alias = aliasPattern.clone(name);
+            this._applyDecorators(statementNode, alias);
             this._parseContext.patternsByName.set(name, alias);
         }
 

package/src/grammar/decorators/tokens.ts

@@ -0,0 +1,18 @@
+import { Pattern } from "../../patterns/Pattern";
+import { Regex } from "../../patterns/Regex";
+import { Decorator } from "../Grammar";
+
+export const tokens: Decorator = (pattern: Pattern, arg: any) => {
+    if (pattern.type === "regex" && Array.isArray(arg)) {
+        const regex = pattern as Regex;
+        const tokens: string[] = [];
+
+        arg.forEach(token => {
+            if (typeof token === "string") {
+                tokens.push(token);
+            }
+        });
+
+        regex.setTokens(tokens);
+    }
+};

package/src/grammar/patterns/decoratorStatement.ts

@@ -0,0 +1,85 @@
+import { Literal } from "../../patterns/Literal";
+import { Optional } from "../../patterns/Optional";
+import { Options } from "../../patterns/Options";
+import { Reference } from "../../patterns/Reference";
+import { Regex } from "../../patterns/Regex";
+import { Repeat } from "../../patterns/Repeat";
+import { Sequence } from "../../patterns/Sequence";
+import { name } from "./name";
+import { allSpaces } from "./spaces";
+
+const colon = new Literal("colon", ":");
+const comma = new Regex("comma", "\\s*,\\s*");
+const openBracket = new Literal("open-bracket", "{");
+const closeBracket = new Literal("close-bracket", "}");
+const openSquareBracket = new Literal("open-square-bracket", "[");
+const closeSquareBracket = new Literal("close-square-bracket", "]");
+const optionalAllSpaces = new Optional("optional-all-spaces", allSpaces);
+
+const stringLiteral = new Regex("string-literal", '"(?:\\\\.|[^"\\\\])*"');
+const numberLiteral = new Regex("number-literal", '[+-]?\\d+(\\\\.\\d+)?([eE][+-]?\\d+)?');
+const nullLiteral = new Literal("null-literal", "null");
+const trueLiteral = new Literal("true-literal", "true");
+const falseLiteral = new Literal("false-literal", "false");
+const booleanLiteral = new Options("", [trueLiteral, falseLiteral]);
+
+const objectKey = name.clone("object-key");
+const objectProperty = new Sequence("object-property", [
+    objectKey,
+    optionalAllSpaces,
+    colon,
+    optionalAllSpaces,
+    new Reference("literal"),
+]);
+const objectProperies = new Repeat("object-properties", objectProperty, { divider: comma });
+const objectLiteral = new Sequence("object-literal", [
+    openBracket,
+    optionalAllSpaces,
+    new Optional("optional-object-properties", objectProperies),
+    optionalAllSpaces,
+    closeBracket
+]);
+
+const arrayItems = new Repeat("array-items", new Reference("literal"), { divider: comma });
+const arrayLiteral = new Sequence("array-literal", [
+    openSquareBracket,
+    optionalAllSpaces,
+    arrayItems,
+    optionalAllSpaces,
+    closeSquareBracket,
+]);
+
+const literal = new Options("literal", [
+    objectLiteral,
+    arrayLiteral,
+    stringLiteral,
+    booleanLiteral,
+    nullLiteral,
+    numberLiteral,
+]);
+
+const decoratorPrefix = new Literal("decorator-prefix", "@");
+const decoratorName = name.clone("decorator-name");
+const openParen = new Literal("open-paren", "(");
+const closeParen = new Literal("close-paren", ")");
+
+const methodDecoration = new Sequence("method-decorator", [
+    decoratorPrefix,
+    decoratorName,
+    optionalAllSpaces,
+    openParen,
+    optionalAllSpaces,
+    new Optional("optional-args", literal),
+    optionalAllSpaces,
+    closeParen
+]);
+
+const nameDecoration = new Sequence("name-decorator", [
+    decoratorPrefix,
+    decoratorName,
+]);
+
+export const decoratorStatement = new Options("decorator-statement", [
+    methodDecoration,
+    nameDecoration,
+]);

package/src/grammar/patterns/statement.ts

@@ -5,6 +5,7 @@ import { name } from "./name";
 import { spaces } from "./spaces";
 import { pattern } from "./pattern";
 import { Optional } from "../../patterns/Optional";
+import {decoratorStatement} from './decoratorStatement';
 
 const optionalSpaces = new Optional("optional-spaces", spaces);
 const assignOperator = new Literal("assign-operator", "=");
@@ -18,4 +19,4 @@ const assignStatement = new Sequence("assign-statement", [
     pattern,
 ]);
 
-export const statement = new Options("statement", [assignStatement, name.clone("export-name")]);
+export const statement = new Options("statement", [decoratorStatement, assignStatement, name.clone("export-name")]);

package/src/grammar/patterns.ts

@@ -1,5 +1,5 @@
 import { Pattern } from "../patterns/Pattern";
-import { Grammar } from "./Grammar";
+import { Grammar, GrammarOptions } from "./Grammar";
 
 const kebabRegex = /-([a-z])/g; // Define the regex once
 
@@ -21,4 +21,22 @@ export function patterns(strings: TemplateStringsArray, ...values: any) {
     });
 
     return result;
+}
+
+export function createPatternsTemplate(options: GrammarOptions){
+    return function patterns(strings: TemplateStringsArray, ...values: any) {
+        const combinedString = strings.reduce(
+            (result, str, i) => result + str + (values[i] || ''),
+            ''
+        );
+    
+        const result: Record<string, Pattern> = {};
+        const patterns = Grammar.parseString(combinedString, options);
+    
+        Object.keys(patterns).forEach(k => {
+            result[kebabToCamelCase(k)] = patterns[k];
+        });
+    
+        return result;
+    }
 }