clarity-pattern-parser 11.3.4 → 11.3.6

package/grammar.md

@@ -0,0 +1,254 @@
+# Clarity Pattern Parser Grammar
+
+This document describes the grammar features supported by the Clarity Pattern Parser.
+
+## Basic Patterns
+
+### Literal Strings
+Define literal string patterns using double quotes:
+```
+name = "John"
+```
+
+Escaped characters are supported in literals:
+- `\n` - newline
+- `\r` - carriage return
+- `\t` - tab
+- `\b` - backspace
+- `\f` - form feed
+- `\v` - vertical tab
+- `\0` - null character
+- `\x00` - hex character
+- `\u0000` - unicode character
+- `\"` - escaped quote
+- `\\` - escaped backslash
+
+### Regular Expressions
+Define regex patterns using forward slashes:
+```
+name = /\w/
+```
+
+## Pattern Operators
+
+### Options (|)
+Match one of multiple patterns using the `|` operator. This is used for simple alternatives where order doesn't matter:
+```
+names = john | jane
+```
+
+### Expression (|)
+Expression patterns also use the `|` operator but are used for defining operator precedence in expressions. The order of alternatives determines precedence, with earlier alternatives having higher precedence. By default, operators are left-associative.
+
+Example of an arithmetic expression grammar:
+```
+prefix-operators = "+" | "-"
+prefix-expression = prefix-operators + expression
+postfix-operators = "++" | "--"
+postfix-expression = expression + postfix-operators
+add-sub-operators = "+" | "-"
+add-sub-expression = expression + add-sub-operators + expression
+mul-div-operators = "*" | "/"
+mul-div-expression = expression + mul-div-operators + expression
+expression = prefix-expression | mul-div-expression | add-sub-expression | postfix-expression
+```
+
+In this example:
+- `prefix-expression` has highest precedence
+- `mul-div-expression` has next highest precedence
+- `add-sub-expression` has next highest precedence
+- `postfix-expression` has lowest precedence
+
+To make an operator right-associative, add the `right` keyword:
+```
+expression = prefix-expression | mul-div-expression | add-sub-expression right | postfix-expression
+```
+
+### Sequence (+)
+Concatenate patterns in sequence using the `+` operator:
+```
+full-name = first-name + space + last-name
+```
+
+### Optional (?)
+Make a pattern optional using the `?` operator:
+```
+full-name = first-name + space + middle-name? + last-name
+```
+
+### Not (!)
+Negative lookahead using the `!` operator:
+```
+pattern = !excluded-pattern + actual-pattern
+```
+
+### Take Until (?->|)
+Match all characters until a specific pattern is found:
+```
+script-text = ?->| "</script"
+```
+
+## Repetition
+
+### Basic Repeat
+Repeat a pattern one or more times using `+`:
+```
+digits = (digit)+
+```
+
+### Zero or More
+Repeat a pattern zero or more times using `*`:
+```
+digits = (digit)*
+```
+
+### Bounded Repetition
+Specify exact repetition counts using curly braces:
+- `{n}` - Exactly n times: `(pattern){3}`
+- `{n,}` - At least n times: `(pattern){1,}`
+- `{,n}` - At most n times: `(pattern){,3}`
+- `{n,m}` - Between n and m times: `(pattern){1,3}`
+
+### Repetition with Divider
+Repeat patterns with a divider between occurrences:
+```
+digits = (digit, comma){3}
+```
+
+Add `trim` keyword to trim the divider from the end:
+```
+digits = (digit, comma trim)+
+```
+
+## Imports and Parameters
+
+### Basic Import
+Import patterns from other files:
+```
+import { pattern-name } from "path/to/file.cpat"
+```
+
+### Import with Parameters
+Import with custom parameters:
+```
+import { pattern } from "file.cpat" with params {
+  custom-param = "value"
+}
+```
+
+### Parameter Declaration
+Declare parameters that can be passed to the grammar:
+```
+use params {
+  param-name
+}
+```
+
+### Default Parameters
+Specify default values for parameters:
+```
+use params {
+  param = default-value
+}
+```
+
+## Decorators
+
+Decorators can be applied to patterns using the `@` syntax:
+
+### Token Decorator
+Specify tokens for a pattern:
+```
+@tokens([" "])
+spaces = /\s+/
+```
+
+### Custom Decorators
+Support for custom decorators with various argument types:
+```
+@decorator()  // No arguments
+@decorator(["value"])  // Array argument
+@decorator({"prop": value})  // Object argument
+```
+
+## Comments
+Add comments using the `#` symbol:
+```
+# This is a comment
+pattern = "value"
+```
+
+## Expression Patterns
+Support for recursive expression patterns with optional right association:
+```
+expression = ternary | variables
+ternary = expression + " ? " + expression + " : " + expression
+```
+
+Add `right` keyword for right association:
+```
+expression = ternary right | variables
+```
+
+## Pattern References
+Reference other patterns by name:
+```
+pattern1 = "value"
+pattern2 = pattern1
+```
+
+## Pattern Aliasing
+Import patterns with aliases:
+```
+import { original as alias } from "file.cpat"
+```
+
+## String Template Patterns
+
+Patterns can be defined inline using string templates. This allows for quick pattern definition and testing without creating separate files.
+
+### Basic Example
+```typescript
+const { fullName } = patterns`
+    first-name = "John"
+    last-name = "Doe"
+    space = /\s+/
+    full-name = first-name + space + last-name
+`;
+
+const result = fullName.exec("John Doe");
+// result.ast.value will be "John Doe"
+```
+
+### Complex Example (HTML-like Markup)
+```typescript
+const { body } = patterns`
+    tag-name = /[a-zA-Z_-]+[a-zA-Z0-9_-]*/
+    space = /\s+/
+    opening-tag = "<" + tag-name + space? + ">"
+    closing-tag = "</" + tag-name + space? + ">"
+    child = space? + element + space?
+    children = (child)*
+    element = opening-tag + children + closing-tag
+    body = space? + element + space?
+`;
+
+const result = body.exec(`
+    <div>
+        <div></div>
+        <div></div>    
+    </div>
+`, true);
+
+// Clean up spaces from the AST
+result?.ast?.findAll(n => n.name.includes("space")).forEach(n => n.remove());
+// result.ast.value will be "<div><div></div><div></div></div>"
+```
+
+### Key Features
+1. Patterns are defined using backticks (`)
+2. Each pattern definition is on a new line
+3. The `patterns` function returns an object with all defined patterns
+4. Patterns can be used immediately after definition
+5. The AST can be manipulated after parsing (e.g., removing spaces)
+6. The `exec` method can take an optional second parameter to enable debug mode 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-pattern-parser",
-  "version": "11.3.4",
+  "version": "11.3.6",
   "description": "Parsing Library for Typescript and Javascript.",
   "main": "./dist/index.js",
   "module": "./dist/index.esm.js",
@@ -10,7 +10,28 @@
     "build": "rollup -c -m",
     "test": "jest --config=jest.coverage.config.js --coverage"
   },
-  "keywords": [],
+  "keywords": [
+    "parser",
+    "ast",
+    "pattern-matching",
+    "grammar",
+    "typescript",
+    "javascript",
+    "parsing",
+    "tree",
+    "pattern",
+    "syntax",
+    "language",
+    "compiler",
+    "interpreter",
+    "lexer",
+    "tokenizer",
+    "regex",
+    "regular-expressions",
+    "text-processing",
+    "validation",
+    "parser-generator"
+  ],
   "devDependencies": {
     "@types/jest": "^26.0.23",
     "jest": "^26.6.3",

package/src/generator/generator.ts

@@ -1,9 +1,4 @@
-import { Context } from "../patterns/Context";
-import { Expression } from "../patterns/Expression";
-import { Literal } from "../patterns/Literal";
 import { Pattern } from "../patterns/Pattern";
-import { Regex } from "../patterns/Regex";
-import { Repeat } from "../patterns/Repeat";
 import { IVisitor } from "./ivisitor";
 
 export class Generator {

package/src/intellisense/AutoComplete.test.ts

@@ -81,7 +81,7 @@ describe("AutoComplete", () => {
         expect(result.isComplete).toBeFalsy();
     });
 
-    test("Full Pattern Match", () => {
+    test("Full Pattern Match Simple", () => {
         const john = new Literal("john", "John");
         const space = new Literal("space", " ");
         const doe = new Literal("doe", "Doe");
@@ -123,6 +123,63 @@ describe("AutoComplete", () => {
         expect(result.cursor).not.toBeNull();
     });
 
+    test("Root Regex Pattern suggests customTokens", () => {
+        const freeTextPattern = new Regex(
+            `free-text`,
+            '[(\\w)\\s]+'
+          );
+
+          const customTokensMap:Record<string, string[]> = {
+            'free-text': ['luke',"leia skywalker",'luke skywalker']
+        }
+        const autoComplete = new AutoComplete(freeTextPattern,{
+            customTokens:customTokensMap
+        });
+        const result = autoComplete.suggestFor("luke");
+
+        const expected = [
+            { text: " skywalker", startIndex: 4 },
+        ];       
+
+        expect(result.ast?.value).toBe("luke");
+        expect(result.options).toEqual(expected);
+        expect(result.errorAtIndex).toBeNull()
+        expect(result.isComplete).toBeTruthy();
+        expect(result.cursor).not.toBeNull();
+
+    });
+
+
+    test("Sequence Regex Pattern suggests customTokens", () => {
+        const jediLiteral = new Literal("jedi", "jedi ");
+        const freeTextPattern = new Regex(
+            `free-text`,
+            '[(\\w)\\s]+'
+          );
+        const sequence = new Sequence('sequence', [jediLiteral,freeTextPattern])
+
+
+        const customTokensMap:Record<string, string[]> = {
+            'free-text': ['luke',"leia skywalker",'luke skywalker']
+        }
+        const autoComplete = new AutoComplete(sequence,{
+            customTokens:customTokensMap
+        });
+        const result = autoComplete.suggestFor("jedi luke sky");
+
+        const expected = [
+            { text: "walker", startIndex: 13 },
+        ];       
+
+        expect(result.ast?.value).toBe("jedi luke sky");
+        expect(result.options).toEqual(expected);
+        expect(result.errorAtIndex).toBeNull()
+        expect(result.isComplete).toBeTruthy();
+        expect(result.cursor).not.toBeNull();
+    });
+
+
+
     test("Full Pattern Match With Root Repeat", () => {
         const john = new Literal("john", "John");
         const space = new Literal("space", " ");
@@ -148,7 +205,7 @@ describe("AutoComplete", () => {
         expect(result.cursor).not.toBeNull();
     });
 
-    test("Partial", () => {
+    test("Partial Simple", () => {
         const name = new Literal("name", "Name");
         const autoComplete = new AutoComplete(name);
         // Use deprecated suggest for code coverage.
@@ -236,6 +293,37 @@ describe("AutoComplete", () => {
 
     });
 
+
+    test("Options AutoComplete on Root Pattern", () => {
+
+        const jack = new Literal("first-name", "Jack");
+        const john = new Literal("first-name", "John");
+
+        const names = new Options('names', [jack,john]);
+        const divider = new Literal('divider', ', ');
+        const repeat = new Repeat('name-list', names, { divider, trimDivider: true });
+
+        const text = ''
+
+        const autoCompleteOptions: AutoCompleteOptions = {
+            customTokens: {
+                'first-name': ["James"]
+            }
+        };
+        const autoComplete = new AutoComplete(repeat,autoCompleteOptions);
+
+        const suggestion = autoComplete.suggestFor(text)
+
+        const expectedOptions = [
+            { text: "Jack", startIndex: 0 },
+            { text: "John", startIndex: 0 },
+            { text: "James", startIndex: 0 },
+        ];
+
+        expect(suggestion.options).toEqual(expectedOptions)
+    
+    })
+
     test("Options AutoComplete On Leaf Pattern", () => {
         const autoCompleteOptions: AutoCompleteOptions = {
             greedyPatternNames: ["space"],
@@ -648,4 +736,33 @@ describe("AutoComplete", () => {
         }
     ]);
     });
-});
+
+
+    test("Calling AutoComplete -> _getAllOptions Does not mutate customTokensMap", () => {
+
+        const jediLuke = new Literal(`jedi`, 'luke');
+        const names = new Options('names', [jediLuke]);
+        const divider = new Literal('divider', ', ');
+        const pattern = new Repeat('name-list', names, { divider, trimDivider: true });
+        
+        const customTokensMap:Record<string, string[]> = {
+            'jedi': ["leia"]
+        }
+        
+        const copiedCustomTokensMap:Record<string, string[]>  = JSON.parse(JSON.stringify(customTokensMap));
+        
+        const autoCompleteOptions: AutoCompleteOptions = {
+            greedyPatternNames:['jedi'],
+            customTokens: customTokensMap
+        };
+        const autoComplete = new AutoComplete(pattern,autoCompleteOptions);
+        
+        // provide a non-empty input to trigger the logic flow that hits _getAllOptions
+        autoComplete.suggestFor('l')
+        
+        expect(customTokensMap).toEqual(copiedCustomTokensMap)
+    })
+
+});
+
+  

package/src/intellisense/AutoComplete.ts

@@ -159,7 +159,7 @@ export class AutoComplete {
 
   private _createSuggestionsFromRoot(): SuggestionOption[] {
     const suggestions: SuggestionOption[] = [];
-    const tokens = this._pattern.getTokens();
+    const tokens = [...this._pattern.getTokens(),... this._getTokensForPattern(this._pattern)];
 
     for (const token of tokens) {
       if (suggestions.findIndex(s => s.text === token) === -1) {
@@ -175,18 +175,31 @@ export class AutoComplete {
       return this._createSuggestions(-1, this._getTokensForPattern(this._pattern));
     }
 
-    const leafPattern = match.pattern;
-    const parent = match.pattern.parent;
-
-    if (parent !== null && match.node != null) {
-      const patterns = leafPattern.getNextPatterns();
-
-      const tokens = patterns.reduce((acc: string[], pattern) => {
-        acc.push(...this._getTokensForPattern(pattern));
-        return acc;
-      }, []);
-
-      return this._createSuggestions(match.node.lastIndex, tokens);
+    if ( match.node != null) {
+    const textStartingMatch = this._text.slice(match.node.startIndex,match.node.endIndex)
+    const currentPatternsTokens = this._getTokensForPattern(match.pattern);
+    /**
+     * Compares tokens to current text and extracts remainder tokens
+     * - IE. **currentText:** *abc*, **baseToken:** *abcdef*, **trailingToken:** *def*
+     */
+    const trailingTokens = currentPatternsTokens.reduce<string[]>((acc, token) => {
+      if (token.startsWith(textStartingMatch)) {
+        const sliced = token.slice(textStartingMatch.length);
+        if (sliced !== '') {
+          acc.push(sliced);
+        }
+      }
+      return acc;
+    }, []);
+    
+    const leafPatterns = match.pattern.getNextPatterns();
+    const leafTokens = leafPatterns.reduce((acc: string[], leafPattern) => {
+      acc.push(...this._getTokensForPattern(leafPattern));
+      return acc;
+    }, []);
+
+    const allTokens = [...trailingTokens,...leafTokens]
+      return this._createSuggestions(match.node.lastIndex, allTokens);
     } else {
       return [];
     }
@@ -197,21 +210,21 @@ export class AutoComplete {
 
     if (this._options.greedyPatternNames != null && this._options.greedyPatternNames.includes(pattern.name)) {
       const nextPatterns = pattern.getNextPatterns();
-      const tokens: string[] = [];
-
       const nextPatternTokens = nextPatterns.reduce((acc: string[], pattern) => {
         acc.push(...this._getTokensForPattern(pattern));
         return acc;
       }, []);
-
-      for (let token of augmentedTokens) {
-        for (let nextPatternToken of nextPatternTokens) {
-          tokens.push(token + nextPatternToken);
+      // using set to prevent duplicates
+      const tokens = new Set<string>();
+      
+      for (const token of augmentedTokens) {
+        for (const nextPatternToken of nextPatternTokens) {    
+          tokens.add(token + nextPatternToken);
         }
       }
 
-      return tokens;
-    } else {
+      return [...tokens]
+        } else {
       return augmentedTokens;
     }
   }
@@ -219,30 +232,39 @@ export class AutoComplete {
   private _getAugmentedTokens(pattern: Pattern) {
     const customTokensMap: any = this._options.customTokens || {};
     const leafPatterns = pattern.getPatterns();
-    const tokens: string[] = customTokensMap[pattern.name] || [];
 
-    leafPatterns.forEach(p => {
-      const augmentedTokens = customTokensMap[p.name] || [];
-      tokens.push(...p.getTokens(), ...augmentedTokens);
-    });
+  /** Using Set to
+   * - prevent duplicates
+   * - prevent mutation of original customTokensMap
+   */
+  const customTokensForPattern = new Set<string>(customTokensMap[pattern.name] ?? []);
+
+  for (const lp of leafPatterns) {
+    const augmentedTokens = customTokensMap[lp.name] ?? [];
+    const lpsCombinedTokens = [...lp.getTokens(), ...augmentedTokens];
 
-    return tokens;
+  for (const token of lpsCombinedTokens) {    
+    customTokensForPattern.add(token);
+  }
+  }
+    return [...customTokensForPattern];
   }
 
   private _createSuggestions(lastIndex: number, tokens: string[]): SuggestionOption[] {
-    let substring = lastIndex === -1 ? "" : this._cursor.getChars(0, lastIndex);
+    let textToIndex = lastIndex === -1 ? "" : this._cursor.getChars(0, lastIndex);
     const suggestionStrings: string[] = [];
     const options: SuggestionOption[] = [];
 
     for (const token of tokens) {
-      const suggestion = substring + token;
-      const startsWith = suggestion.startsWith(substring);
+      // concatenated for start index identification inside createSuggestion
+      const suggestion = textToIndex + token;
       const alreadyExist = suggestionStrings.includes(suggestion);
       const isSameAsText = suggestion === this._text;
 
-      if (startsWith && !alreadyExist && !isSameAsText) {
+      if ( !alreadyExist && !isSameAsText) {
         suggestionStrings.push(suggestion);
-        options.push(this._createSuggestion(this._cursor.text, suggestion));
+        const suggestionOption = this._createSuggestion(this._cursor.text, suggestion)
+        options.push(suggestionOption);
       }
     }
 
@@ -256,12 +278,14 @@ export class AutoComplete {
     const furthestMatch = findMatchIndex(suggestion, fullText);
     const text = suggestion.slice(furthestMatch);
 
-    return {
+    const option:SuggestionOption = {
       text: text,
       startIndex: furthestMatch,
     };
+    return option
   }
 
+
   static suggestFor(text: string, pattern: Pattern, options?: AutoCompleteOptions) {
     return new AutoComplete(pattern, options).suggestFor(text);
   }