clarity-pattern-parser 11.3.13 → 11.4.1

package/src/grammar_v2/patterns/patterns/grammar.test.ts

@@ -0,0 +1,218 @@
+import fs from "fs";
+import { grammar } from "./grammar";
+import path from "path";
+import { Node } from "../../../ast/Node";
+
+const dir = path.join(__dirname, "./cpat.cpat");
+
+describe("grammar", () => {
+    test("syntax version", () => {
+        const { ast } = grammar.exec("syntax 1.0");
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "syntax")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "syntax")?.value).toBe("syntax");
+        expect(ast?.find((p: Node)=>p.name === "syntaxVersion")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "syntaxVersion")?.value).toBe("1.0");
+    });
+
+    test("import", () => {
+        const { ast } = grammar.exec('import {pattern} from "resource"');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "importStatement")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "import")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "import")?.value).toBe("import");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("pattern");
+        expect(ast?.find((p: Node)=>p.name === "from")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "from")?.value).toBe("from");
+        expect(ast?.find((p: Node)=>p.name === "resource")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "resource")?.value).toBe('"resource"');
+    });
+
+    test("import with params", () => {
+        const { ast } = grammar.exec('import {pattern} from "resource" with params {param = "value"}');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "importStatement")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "import")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "import")?.value).toBe("import");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("pattern");
+        expect(ast?.find((p: Node)=>p.name === "from")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "from")?.value).toBe("from");
+        expect(ast?.find((p: Node)=>p.name === "resource")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "resource")?.value).toBe('"resource"');
+        expect(ast?.find((p: Node)=>p.name === "withParamsExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "withParamsExpr")?.value).toBe("with params {param = \"value\"}");
+        expect(ast?.find((p: Node)=>p.name === "statements")?.find((p: Node)=>p.name === "param = \"value\"")).toBeDefined();
+    });
+
+    test("use params", () => {
+        const { ast } = grammar.exec('use params { param }');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "useParamsStatement")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "useParamsStatement")?.value).toBe("use params { param }");
+        expect(ast?.find((p: Node)=>p.name === "useParams")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "useParams")?.value).toBe("use params");
+        expect(ast?.find((p: Node)=>p.name === "useParamPatterns")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "useParamPatterns")?.value).toBe("param");
+    });
+
+    test("literal", () => {
+        const { ast  } = grammar.exec('literal = "Hello, world!"');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("literal = \"Hello, world!\"");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("literal");
+        expect(ast?.find((p: Node)=>p.name === "literal")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "literal")?.value).toBe("\"Hello, world!\"");
+    });
+
+
+    test("regex", () => {
+        const { ast } = grammar.exec('regex = /Hello, world!/');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("regex = /Hello, world!/");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("regex");
+        expect(ast?.find((p: Node)=>p.name === "regex")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "regex")?.value).toBe("/Hello, world!/");
+    });
+
+    test("comment", () => {
+        const { ast } = grammar.exec('# Comment');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "comment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "comment")?.value).toBe("# Comment");
+    });
+
+    test("sequence", () => {
+        const { ast } = grammar.exec('sequence = "Hello" + "world"');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("sequence = \"Hello\" + \"world\"");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("sequence");
+        expect(ast?.find((p: Node)=>p.name === "sequenceExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "sequenceExpr")?.value).toBe("\"Hello\" + \"world\"");
+    });
+
+    test("options", () => {
+        const { ast } = grammar.exec('options = "Hello" | "world"');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("options = \"Hello\" | \"world\"");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("options");
+        expect(ast?.find((p: Node)=>p.name === "optionsExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "optionsExpr")?.value).toBe("\"Hello\" | \"world\"");
+    });
+
+    test("greedy options", () => {
+        const { ast } = grammar.exec('greedyOptions = "Hello" <|> "world"');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("greedyOptions = \"Hello\" <|> \"world\"");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("greedyOptions");
+        expect(ast?.find((p: Node)=>p.name === "greedyOptionsExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "greedyOptionsExpr")?.value).toBe("\"Hello\" <|> \"world\"");
+    });
+
+    test("repeat", () => {
+        const { ast } = grammar.exec('repeat = (Hello)+');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("repeat = (Hello)+");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("repeat");
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")?.value).toBe("(Hello)+");
+    });
+
+    test("repeat zero or more", () => {
+        const { ast } = grammar.exec('repeat = (Hello)*');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("repeat = (Hello)*");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("repeat");
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")?.value).toBe("(Hello)*");
+    });
+
+    test("repeat with bounds", () => {
+        const { ast } = grammar.exec('repeat = (Hello){2, 4}');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("repeat = (Hello){2, 4}");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("repeat");
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")?.value).toBe("(Hello){2, 4}");
+    });
+
+    test("repeat with upper limit", () => {
+        const { ast } = grammar.exec('repeat = (Hello){, 4}');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("repeat = (Hello){, 4}");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("repeat");
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")?.value).toBe("(Hello){, 4}");
+    });
+
+    test("repeat with lower limit", () => {
+        const { ast } = grammar.exec('repeat = (Hello){1,}');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("repeat = (Hello){1,}");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("repeat");
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")?.value).toBe("(Hello){1,}");
+    });
+    
+    test("repeat with delimiter", () => {
+        const { ast } = grammar.exec('repeat = (Hello, "world")+');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("repeat = (Hello, \"world\")+");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("repeat");
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")?.value).toBe("(Hello, \"world\")+");
+    });
+
+    test("repeat with trim delimiter", () => {
+        const { ast } = grammar.exec('repeat = (Hello, "world" trim)+');
+
+        expect(ast).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternAssignment")?.value).toBe("repeat = (Hello, \"world\" trim)+");
+        expect(ast?.find((p: Node)=>p.name === "patternName")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "patternName")?.value).toBe("repeat");
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")).toBeDefined();
+        expect(ast?.find((p: Node)=>p.name === "repeatExpr")?.value).toBe("(Hello, \"world\" trim)+");
+    });
+
+
+});

package/src/grammar_v2/patterns/patterns/grammar.ts

@@ -0,0 +1,103 @@
+import { Expression } from "../../../patterns/Expression";
+import { Literal } from "../../../patterns/Literal";
+import { Optional } from "../../../patterns/Optional";
+import { Options } from "../../../patterns/Options";
+import { Reference } from "../../../patterns/Reference";
+import { Regex } from "../../../patterns/Regex";
+import { Repeat } from "../../../patterns/Repeat";
+import { Sequence } from "../../../patterns/Sequence";
+
+export const syntax = new Literal("syntax", "syntax");
+export const imprt = new Literal("import", "import");
+export const useParams = new Literal("useParams", "use params");
+export const withParams = new Literal("withParams", "with params");
+export const from = new Literal("from", "from");
+export const right = new Literal("right", "right");
+export const ws = new Regex("ws", "\\s+");
+export const ls = new Regex("ls", "[ \\t]+");
+export const assign = new Literal("assign", "=");
+export const bar = new Literal("|", "|");
+export const greedyBar = new Literal("<|>", "<|>");
+export const concat = new Literal("+", "+");
+export const colon = new Literal(":", ":");
+export const openParen = new Literal("(", "(");
+export const closeParen = new Literal(")", ")");
+export const openSquareBracket = new Literal("[", "[");
+export const closeSquareBracket = new Literal("]", "]");
+export const openBracket = new Literal("{", "{");
+export const closeBracket = new Literal("}", "}");
+export const comma = new Regex(",", "\\s*[,]\\s*");
+export const trim = new Literal("trim", "trim");
+export const not = new Literal("not", "!");
+export const optional = new Literal("?", "?");
+export const newLine = new Regex("newLine", "\\s*(\\r\\n|\\r|\\n)\\s*");
+export const syntaxVersion = new Regex("syntaxVersion", "\\S*");
+export const at = new Literal("@", "@");
+export const optionalWS = new Optional("optionalWS", ws);
+export const optionalLS = new Optional("optionalLS", ls);
+
+export const literal = new Regex("literal", '"(?:\\\\.|[^"\\\\])*"');
+export const regex = new Regex("regex", "\\/(\\\\|[^\\n\\r])*\\/");
+
+export const integer = new Regex("integer", "[1-9][0-9]*|[0]");
+export const name = new Regex("name", "[a-zA-Z_-]+[a-zA-Z0-9_-]*");
+export const patternName = name.clone("patternName");
+export const patternIdentifier = name.clone("patternIdentifier");
+export const resource = literal.clone("resource");
+
+export const comment = new Regex("comment", "#[^\\n\\r]*");
+
+export const jsonString = literal.clone("jsonString");
+export const jsonNumber = new Regex("jsonNumber", "-?\\d+(\\.\\d+)?");
+export const trueLiteral = new Literal("trueLiteral", "true");
+export const falseLiteral = new Literal("falseLiteral", "false");
+export const jsonBoolean = new Options("jsonBoolean", [trueLiteral, falseLiteral]);
+export const jsonNull = new Literal("jsonNull", "null");
+export const jsonArrayItems = new Repeat("jsonArrayItems", new Reference("jsonValue"), { divider: comma, trimDivider: true });
+export const jsonArray = new Sequence("jsonArray", [openSquareBracket, optionalWS, jsonArrayItems, optionalWS, closeSquareBracket]);
+export const jsonObjectPropertyName = literal.clone("jsonObjectPropertyName");
+export const jsonObjectProperty = new Sequence("jsonObjectProperty", [jsonObjectPropertyName, optionalWS, colon, optionalWS, new Reference("jsonValue")]);
+export const jsonObjectProperties = new Repeat("jsonObjectProperties", jsonObjectProperty, { divider: comma, trimDivider: true });
+export const jsonObject = new Sequence("jsonObject", [openBracket, optionalWS, jsonObjectProperties, optionalWS, closeBracket]);
+export const jsonValue = new Options("jsonValue", [jsonString, jsonNumber, jsonBoolean, jsonNull, jsonArray, jsonObject]);
+
+export const syntaxStatement = new Sequence("syntaxStatement", [syntax, optionalWS, syntaxVersion]);
+
+export const decorationName = name.clone("decorationName");
+export const decorationStatement = new Sequence("decorationStatement", [at, optionalWS, decorationName, optionalWS, openParen, optionalWS, jsonValue, optionalWS, closeParen]);
+
+export const useParamPatterns = new Repeat("useParamPatterns", patternName, { divider: comma, trimDivider: true });
+export const useParamsStatement = new Sequence("useParamsStatement", [useParams, optionalLS, openBracket, optionalWS, useParamPatterns, optionalWS, closeBracket]);
+
+export const withParamStatements = new Repeat("withParamStatements", new Reference("patternAssignment"), { divider: newLine, trimDivider: true });
+export const withParamsExpr = new Sequence("withParamsExpr", [withParams, optionalLS, openBracket, optionalWS, withParamStatements, optionalWS, closeBracket]);
+
+export const patternNames = new Repeat("patternNames", patternName, { divider: comma, trimDivider: true });
+export const importedPatterns = new Sequence("importedPatterns", [openBracket, optionalWS, patternNames, optionalWS, closeBracket]);
+export const importStatement = new Sequence("importStatement", [imprt, optionalLS, importedPatterns, optionalLS, from, optionalLS, resource, optionalLS, new Optional("optionalWithParamsExpr", withParamsExpr)]);
+
+export const notExpr = new Sequence("notExpr", [not, optionalLS, new Reference("unaryPatternExpr")]);
+export const optionalExpr = new Sequence("optionalExpr", [new Reference("unaryPatternExpr"), optionalLS, optional]);
+export const rightAssociationExpr = new Sequence("rightAssociationExpr", [new Reference("unaryPatternExpr"), optionalLS, right]);
+export const unaryPatternExpr = new Expression("unaryPatternExpr", [notExpr, optionalExpr, rightAssociationExpr, patternIdentifier]);
+
+export const repeatBounds = new Sequence("repeatBounds", [openBracket, optionalWS, new Optional("optionalInteger", integer), optionalWS, new Optional("optionalComma", comma), optionalWS, new Optional("optionalInteger", integer), optionalWS, closeBracket]);
+export const oneOrMore = new Literal("oneOrMore", "+");
+export const zeroOrMore = new Literal("zeroOrMore", "*");
+export const repeatOptions = new Options("repeatOptions", [oneOrMore, zeroOrMore, repeatBounds]);
+export const delimiter = new Sequence("delimiter", [comma, new Reference("patternExpr"), optionalWS, new Optional("optionalTrim", trim)]);
+export const repeatExpr = new Sequence("repeatExpr", [openParen, optionalWS, new Reference("patternExpr"), optionalWS, new Optional("optionalDelimiter", delimiter), optionalWS, closeParen, repeatOptions]);
+
+export const sequenceExpr = new Sequence("sequenceExpr", [new Reference("patternExpr"), optionalWS, concat, optionalWS, new Reference("patternExpr")]);
+export const optionsExpr = new Sequence("optionsExpr", [new Reference("patternExpr"), optionalWS, bar, optionalWS, new Reference("patternExpr")]);
+export const greedyOptionsExpr = new Sequence("greedyOptionsExpr", [new Reference("patternExpr"), optionalWS, greedyBar, optionalWS, new Reference("patternExpr")]);
+export const patternGroupExpr = new Sequence("patternGroupExpr", [openParen, optionalWS, new Reference("patternExpr"), optionalWS, closeParen]);
+
+export const exportPattern = patternName.clone("exportPattern");
+export const patternExpr = new Expression("patternExpr", [sequenceExpr, optionsExpr, greedyOptionsExpr, repeatExpr, patternGroupExpr, literal, regex, unaryPatternExpr]);
+export const patternAssignment = new Sequence("patternAssignment", [patternName, optionalWS, assign, optionalWS, patternExpr]);
+export const statement = new Options("statement", [useParamsStatement, importStatement, patternAssignment, decorationStatement, exportPattern, comment]);
+export const statements = new Repeat("statements", statement, { divider: newLine});
+export const cpat = new Sequence("cpat", [optionalWS, new Optional("optionalSyntaxStatement", syntaxStatement), optionalWS, new Optional("optionalStatements", statements)]);
+
+export const grammar = cpat;

package/src/patterns/Cursor.ts

@@ -4,13 +4,13 @@ import { ParseError } from "./ParseError";
 import { Pattern } from "./Pattern";
 
 export class Cursor {
-  private _text: string;
+  private _chars: string[];
   private _index: number;
   private _length: number;
   private _history: CursorHistory;
 
   get text(): string {
-    return this._text;
+    return this._chars.join("");
   }
 
   get isOnFirst(): boolean {
@@ -74,13 +74,13 @@ export class Cursor {
   }
 
   get currentChar(): string {
-    return this._text[this._index];
+    return this._chars[this._index];
   }
 
   constructor(text: string) {
-    this._text = text;
+    this._chars = [...text];
     this._index = 0;
-    this._length = [...text].length;
+    this._length = this._chars.length;
     this._history = new CursorHistory();
   }
 
@@ -123,7 +123,7 @@ export class Cursor {
   }
 
   getChars(first: number, last: number): string {
-    return this._text.slice(first, last + 1);
+    return this._chars.slice(first, last + 1).join("");
   }
 
   recordMatch(pattern: Pattern, node: Node): void {

package/src/patterns/Regex.ts

@@ -107,7 +107,7 @@ export class Regex implements Pattern {
 
   private tryToParse(cursor: Cursor) {
     const result = this._regex.exec(this._substring);
-
+    
     if (result != null && result[0].length > 0 && result.index === 0) {
       this.processResult(cursor, result);
     } else {
@@ -117,7 +117,9 @@ export class Regex implements Pattern {
 
   private processResult(cursor: Cursor, result: RegExpExecArray) {
     const currentIndex = cursor.index;
-    const newIndex = currentIndex + result[0].length - 1;
+    const match = result[0];
+    const matchLength = [...match].length;
+    const newIndex = currentIndex + matchLength - 1;
 
     this._node = new Node(
       "regex",

package/src/patterns/Sequence.test.ts

@@ -4,6 +4,7 @@ import { Literal } from "./Literal";
 import { Node } from "../ast/Node";
 import { Optional } from "./Optional";
 import { Pattern } from "./Pattern";
+import { Regex } from "./Regex";
 
 describe("Sequence", () => {
     test("No Patterns", () => {
@@ -334,4 +335,14 @@ describe("Sequence", () => {
         expect(result.cursor.hasError).toBeFalsy();
     });
 
+    test("Unicode Characters", ()=>{
+        const quote = new Literal("'", "'");
+        const content = new Regex("content", "[^']*");
+        const sequence = new Sequence("sequence", [quote, content, quote]);
+        const result = sequence.exec("'🔴'");
+
+        expect(result.ast).not.toBe(null);
+        expect(result.cursor.hasError).toBeFalsy();
+    })
+
 });