clarity-pattern-parser 11.3.5 → 11.3.7

package/grammar.md

@@ -0,0 +1,254 @@
+# Clarity Pattern Parser Grammar
+
+This document describes the grammar features supported by the Clarity Pattern Parser.
+
+## Basic Patterns
+
+### Literal Strings
+Define literal string patterns using double quotes:
+```
+name = "John"
+```
+
+Escaped characters are supported in literals:
+- `\n` - newline
+- `\r` - carriage return
+- `\t` - tab
+- `\b` - backspace
+- `\f` - form feed
+- `\v` - vertical tab
+- `\0` - null character
+- `\x00` - hex character
+- `\u0000` - unicode character
+- `\"` - escaped quote
+- `\\` - escaped backslash
+
+### Regular Expressions
+Define regex patterns using forward slashes:
+```
+name = /\w/
+```
+
+## Pattern Operators
+
+### Options (|)
+Match one of multiple patterns using the `|` operator. This is used for simple alternatives where order doesn't matter:
+```
+names = john | jane
+```
+
+### Expression (|)
+Expression patterns also use the `|` operator but are used for defining operator precedence in expressions. The order of alternatives determines precedence, with earlier alternatives having higher precedence. By default, operators are left-associative.
+
+Example of an arithmetic expression grammar:
+```
+prefix-operators = "+" | "-"
+prefix-expression = prefix-operators + expression
+postfix-operators = "++" | "--"
+postfix-expression = expression + postfix-operators
+add-sub-operators = "+" | "-"
+add-sub-expression = expression + add-sub-operators + expression
+mul-div-operators = "*" | "/"
+mul-div-expression = expression + mul-div-operators + expression
+expression = prefix-expression | mul-div-expression | add-sub-expression | postfix-expression
+```
+
+In this example:
+- `prefix-expression` has highest precedence
+- `mul-div-expression` has next highest precedence
+- `add-sub-expression` has next highest precedence
+- `postfix-expression` has lowest precedence
+
+To make an operator right-associative, add the `right` keyword:
+```
+expression = prefix-expression | mul-div-expression | add-sub-expression right | postfix-expression
+```
+
+### Sequence (+)
+Concatenate patterns in sequence using the `+` operator:
+```
+full-name = first-name + space + last-name
+```
+
+### Optional (?)
+Make a pattern optional using the `?` operator:
+```
+full-name = first-name + space + middle-name? + last-name
+```
+
+### Not (!)
+Negative lookahead using the `!` operator:
+```
+pattern = !excluded-pattern + actual-pattern
+```
+
+### Take Until (?->|)
+Match all characters until a specific pattern is found:
+```
+script-text = ?->| "</script"
+```
+
+## Repetition
+
+### Basic Repeat
+Repeat a pattern one or more times using `+`:
+```
+digits = (digit)+
+```
+
+### Zero or More
+Repeat a pattern zero or more times using `*`:
+```
+digits = (digit)*
+```
+
+### Bounded Repetition
+Specify exact repetition counts using curly braces:
+- `{n}` - Exactly n times: `(pattern){3}`
+- `{n,}` - At least n times: `(pattern){1,}`
+- `{,n}` - At most n times: `(pattern){,3}`
+- `{n,m}` - Between n and m times: `(pattern){1,3}`
+
+### Repetition with Divider
+Repeat patterns with a divider between occurrences:
+```
+digits = (digit, comma){3}
+```
+
+Add `trim` keyword to trim the divider from the end:
+```
+digits = (digit, comma trim)+
+```
+
+## Imports and Parameters
+
+### Basic Import
+Import patterns from other files:
+```
+import { pattern-name } from "path/to/file.cpat"
+```
+
+### Import with Parameters
+Import with custom parameters:
+```
+import { pattern } from "file.cpat" with params {
+  custom-param = "value"
+}
+```
+
+### Parameter Declaration
+Declare parameters that can be passed to the grammar:
+```
+use params {
+  param-name
+}
+```
+
+### Default Parameters
+Specify default values for parameters:
+```
+use params {
+  param = default-value
+}
+```
+
+## Decorators
+
+Decorators can be applied to patterns using the `@` syntax:
+
+### Token Decorator
+Specify tokens for a pattern:
+```
+@tokens([" "])
+spaces = /\s+/
+```
+
+### Custom Decorators
+Support for custom decorators with various argument types:
+```
+@decorator()  // No arguments
+@decorator(["value"])  // Array argument
+@decorator({"prop": value})  // Object argument
+```
+
+## Comments
+Add comments using the `#` symbol:
+```
+# This is a comment
+pattern = "value"
+```
+
+## Expression Patterns
+Support for recursive expression patterns with optional right association:
+```
+expression = ternary | variables
+ternary = expression + " ? " + expression + " : " + expression
+```
+
+Add `right` keyword for right association:
+```
+expression = ternary right | variables
+```
+
+## Pattern References
+Reference other patterns by name:
+```
+pattern1 = "value"
+pattern2 = pattern1
+```
+
+## Pattern Aliasing
+Import patterns with aliases:
+```
+import { original as alias } from "file.cpat"
+```
+
+## String Template Patterns
+
+Patterns can be defined inline using string templates. This allows for quick pattern definition and testing without creating separate files.
+
+### Basic Example
+```typescript
+const { fullName } = patterns`
+    first-name = "John"
+    last-name = "Doe"
+    space = /\s+/
+    full-name = first-name + space + last-name
+`;
+
+const result = fullName.exec("John Doe");
+// result.ast.value will be "John Doe"
+```
+
+### Complex Example (HTML-like Markup)
+```typescript
+const { body } = patterns`
+    tag-name = /[a-zA-Z_-]+[a-zA-Z0-9_-]*/
+    space = /\s+/
+    opening-tag = "<" + tag-name + space? + ">"
+    closing-tag = "</" + tag-name + space? + ">"
+    child = space? + element + space?
+    children = (child)*
+    element = opening-tag + children + closing-tag
+    body = space? + element + space?
+`;
+
+const result = body.exec(`
+    <div>
+        <div></div>
+        <div></div>    
+    </div>
+`, true);
+
+// Clean up spaces from the AST
+result?.ast?.findAll(n => n.name.includes("space")).forEach(n => n.remove());
+// result.ast.value will be "<div><div></div><div></div></div>"
+```
+
+### Key Features
+1. Patterns are defined using backticks (`)
+2. Each pattern definition is on a new line
+3. The `patterns` function returns an object with all defined patterns
+4. Patterns can be used immediately after definition
+5. The AST can be manipulated after parsing (e.g., removing spaces)
+6. The `exec` method can take an optional second parameter to enable debug mode 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-pattern-parser",
-  "version": "11.3.5",
+  "version": "11.3.7",
   "description": "Parsing Library for Typescript and Javascript.",
   "main": "./dist/index.js",
   "module": "./dist/index.esm.js",
@@ -10,7 +10,28 @@
     "build": "rollup -c -m",
     "test": "jest --config=jest.coverage.config.js --coverage"
   },
-  "keywords": [],
+  "keywords": [
+    "parser",
+    "ast",
+    "pattern-matching",
+    "grammar",
+    "typescript",
+    "javascript",
+    "parsing",
+    "tree",
+    "pattern",
+    "syntax",
+    "language",
+    "compiler",
+    "interpreter",
+    "lexer",
+    "tokenizer",
+    "regex",
+    "regular-expressions",
+    "text-processing",
+    "validation",
+    "parser-generator"
+  ],
   "devDependencies": {
     "@types/jest": "^26.0.23",
     "jest": "^26.6.3",

package/src/intellisense/AutoComplete.test.ts

@@ -81,7 +81,7 @@ describe("AutoComplete", () => {
         expect(result.isComplete).toBeFalsy();
     });
 
-    test("Full Pattern Match", () => {
+    test("Full Pattern Match Simple", () => {
         const john = new Literal("john", "John");
         const space = new Literal("space", " ");
         const doe = new Literal("doe", "Doe");
@@ -123,6 +123,63 @@ describe("AutoComplete", () => {
         expect(result.cursor).not.toBeNull();
     });
 
+    test("Root Regex Pattern suggests customTokens", () => {
+        const freeTextPattern = new Regex(
+            `free-text`,
+            '[(\\w)\\s]+'
+          );
+
+          const customTokensMap:Record<string, string[]> = {
+            'free-text': ['luke',"leia skywalker",'luke skywalker']
+        }
+        const autoComplete = new AutoComplete(freeTextPattern,{
+            customTokens:customTokensMap
+        });
+        const result = autoComplete.suggestFor("luke");
+
+        const expected = [
+            { text: " skywalker", startIndex: 4 },
+        ];       
+
+        expect(result.ast?.value).toBe("luke");
+        expect(result.options).toEqual(expected);
+        expect(result.errorAtIndex).toBeNull()
+        expect(result.isComplete).toBeTruthy();
+        expect(result.cursor).not.toBeNull();
+
+    });
+
+
+    test("Sequence Regex Pattern suggests customTokens", () => {
+        const jediLiteral = new Literal("jedi", "jedi ");
+        const freeTextPattern = new Regex(
+            `free-text`,
+            '[(\\w)\\s]+'
+          );
+        const sequence = new Sequence('sequence', [jediLiteral,freeTextPattern])
+
+
+        const customTokensMap:Record<string, string[]> = {
+            'free-text': ['luke',"leia skywalker",'luke skywalker']
+        }
+        const autoComplete = new AutoComplete(sequence,{
+            customTokens:customTokensMap
+        });
+        const result = autoComplete.suggestFor("jedi luke sky");
+
+        const expected = [
+            { text: "walker", startIndex: 13 },
+        ];       
+
+        expect(result.ast?.value).toBe("jedi luke sky");
+        expect(result.options).toEqual(expected);
+        expect(result.errorAtIndex).toBeNull()
+        expect(result.isComplete).toBeTruthy();
+        expect(result.cursor).not.toBeNull();
+    });
+
+
+
     test("Full Pattern Match With Root Repeat", () => {
         const john = new Literal("john", "John");
         const space = new Literal("space", " ");
@@ -148,7 +205,7 @@ describe("AutoComplete", () => {
         expect(result.cursor).not.toBeNull();
     });
 
-    test("Partial", () => {
+    test("Partial Simple", () => {
         const name = new Literal("name", "Name");
         const autoComplete = new AutoComplete(name);
         // Use deprecated suggest for code coverage.
@@ -257,8 +314,6 @@ describe("AutoComplete", () => {
 
         const suggestion = autoComplete.suggestFor(text)
 
-        console.log('suggestion: ',suggestion)
-
         const expectedOptions = [
             { text: "Jack", startIndex: 0 },
             { text: "John", startIndex: 0 },

package/src/intellisense/AutoComplete.ts

@@ -175,18 +175,31 @@ export class AutoComplete {
       return this._createSuggestions(-1, this._getTokensForPattern(this._pattern));
     }
 
-    const leafPattern = match.pattern;
-    const parent = match.pattern.parent;
-
-    if (parent !== null && match.node != null) {
-      const patterns = leafPattern.getNextPatterns();
-
-      const tokens = patterns.reduce((acc: string[], pattern) => {
-        acc.push(...this._getTokensForPattern(pattern));
-        return acc;
-      }, []);
-
-      return this._createSuggestions(match.node.lastIndex, tokens);
+    if ( match.node != null) {
+    const textStartingMatch = this._text.slice(match.node.startIndex,match.node.endIndex)
+    const currentPatternsTokens = this._getTokensForPattern(match.pattern);
+    /**
+     * Compares tokens to current text and extracts remainder tokens
+     * - IE. **currentText:** *abc*, **baseToken:** *abcdef*, **trailingToken:** *def*
+     */
+    const trailingTokens = currentPatternsTokens.reduce<string[]>((acc, token) => {
+      if (token.startsWith(textStartingMatch)) {
+        const sliced = token.slice(textStartingMatch.length);
+        if (sliced !== '') {
+          acc.push(sliced);
+        }
+      }
+      return acc;
+    }, []);
+    
+    const leafPatterns = match.pattern.getNextPatterns();
+    const leafTokens = leafPatterns.reduce((acc: string[], leafPattern) => {
+      acc.push(...this._getTokensForPattern(leafPattern));
+      return acc;
+    }, []);
+
+    const allTokens = [...trailingTokens,...leafTokens]
+      return this._createSuggestions(match.node.lastIndex, allTokens);
     } else {
       return [];
     }
@@ -238,19 +251,20 @@ export class AutoComplete {
   }
 
   private _createSuggestions(lastIndex: number, tokens: string[]): SuggestionOption[] {
-    let substring = lastIndex === -1 ? "" : this._cursor.getChars(0, lastIndex);
+    let textToIndex = lastIndex === -1 ? "" : this._cursor.getChars(0, lastIndex);
     const suggestionStrings: string[] = [];
     const options: SuggestionOption[] = [];
 
     for (const token of tokens) {
-      const suggestion = substring + token;
-      const startsWith = suggestion.startsWith(substring);
+      // concatenated for start index identification inside createSuggestion
+      const suggestion = textToIndex + token;
       const alreadyExist = suggestionStrings.includes(suggestion);
       const isSameAsText = suggestion === this._text;
 
-      if (startsWith && !alreadyExist && !isSameAsText) {
+      if ( !alreadyExist && !isSameAsText) {
         suggestionStrings.push(suggestion);
-        options.push(this._createSuggestion(this._cursor.text, suggestion));
+        const suggestionOption = this._createSuggestion(this._cursor.text, suggestion)
+        options.push(suggestionOption);
       }
     }
 
@@ -264,12 +278,14 @@ export class AutoComplete {
     const furthestMatch = findMatchIndex(suggestion, fullText);
     const text = suggestion.slice(furthestMatch);
 
-    return {
+    const option:SuggestionOption = {
       text: text,
       startIndex: furthestMatch,
     };
+    return option
   }
 
+
   static suggestFor(text: string, pattern: Pattern, options?: AutoCompleteOptions) {
     return new AutoComplete(pattern, options).suggestFor(text);
   }