clarity-pattern-parser 6.0.2 → 7.0.0

package/src/grammar/patterns/statement.ts

@@ -0,0 +1,35 @@
+import { And } from "../../patterns/And";
+import { Literal } from "../../patterns/Literal";
+import { Or } from "../../patterns/Or";
+import { andLiteral } from "./andLiteral";
+import { name } from "./name";
+import { orLiteral } from "./orLiteral";
+import { regexLiteral } from "./regexLiteral";
+import { repeatLiteral } from "./repeatLiteral";
+import { spaces } from "./spaces";
+import { literal } from "./literal";
+import { comment } from "./comment";
+
+const optionalSpaces = spaces.clone("optional-spaces", true);
+const assignOperator = new Literal("assign-operator", "=");
+const optionalComment = comment.clone("inline-comment", true);
+
+const statements = new Or("statements", [
+    literal,
+    regexLiteral,
+    orLiteral,
+    andLiteral,
+    repeatLiteral,
+]);
+
+export const statement = new And("statement", [
+    optionalSpaces,
+    name,
+    optionalSpaces,
+    assignOperator,
+    optionalSpaces,
+    statements,
+    optionalSpaces,
+    optionalComment,
+    optionalSpaces,
+]);

package/src/grammar/spec.md

@@ -0,0 +1,142 @@
+# Grammar
+
+## Literal
+
+```
+name = "Literal"
+```
+
+## Regex
+
+```
+digits = /\d+/
+```
+
+### Regex Caveats
+
+Do not use "^" at the beginning or "$" at the end of your regular expression. If
+you are creating a regular expression that is concerned about the beginning and
+end of the text you should probably just use a regular expression.
+
+## Or
+This succeeds if it matches any of the options.
+```
+multiply = "*"
+divide = "/"
+operators = mulitply | divide
+```
+
+## And
+This succeeds if it matches all of the patterns in the order they are placed.
+```
+space = " "
+first-name = "John"
+last-name = "Doe"
+full-name = first-name & space & last-name
+```
+
+### Optional pattern
+Patterns within the and sequence can be optional.
+```
+space = " "
+first-name = /\w+/
+last-name = /\w+/
+middle-name = /\w+/
+middle-name-with-space = middle-name & space
+
+full-name = first-name & space & middle-name-with-space? & last-name
+```
+
+### Negative Look Ahead
+This will ensure that the first name isn't `Jack` before continuing to match for
+a name.
+
+```
+space = " "
+first-name = /\w+/
+last-name = /\w+/
+middle-name = /\w+/
+middle-name-with-space = m-middle-name & space
+jack = "Jack"
+
+full-name = !jack & first-name & space & middle-name-with-space? & last-name
+```
+
+## Repeat
+```
+digit = /\d/
+digits = (digit)*
+```
+
+### Zero Or More Pattern
+
+```
+digit = /\d/
+comma = ","
+digits = (digit, comma)*
+```
+
+### Zero Or More & Trim Trailing Divider
+```
+digit = /\d/
+comma = ","
+digits = (digit, comma)* -t
+```
+
+This is a useful feature if you don't want the divider to be the last pattern found. Let's look at the example below to understand.
+
+```ts
+const expression = `
+    digit = /\d/
+    comma = ","
+    digits = (digit, comma)* -t
+`;
+
+const { digits } = Gammar.parse(expression);
+
+let result = digits.exec("1,2,3");
+expect(result.ast?.value).toBe("1,2,3");
+
+result = digits.exec("1,2,");
+expect(result.ast).toBeNull();
+```
+
+### Zero Or More With Optional Repeated Pattern
+
+```
+digit = /\d/
+comma = ","
+digits = (digit?, comma)*
+```
+
+### One Or More With Optional Repeated Pattern
+
+```
+digit = /\d/
+comma = ","
+digits = (digit?, comma)+
+```
+
+### Upper Limit
+
+```
+digit = /\d/
+comma = ","
+digits = (digit, comma){,3}
+```
+
+### Bounded
+
+```
+digit = /\d/
+comma = ","
+digits = (digit, comma){1,3}
+```
+
+### Lower Limit
+
+```
+digit = /\d/
+comma = ","
+digits = (digit, comma){1,3}
+```

package/src/index.ts

@@ -1,4 +1,8 @@
 import { Node } from "./ast/Node";
+import { Grammar } from "./grammar/Grammar";
+import { Suggestion } from "./intellisense/Suggestion";
+import { SuggestionOption } from "./intellisense/SuggestionOption";
+import { AutoComplete, AutoCompleteOptions } from './intellisense/AutoComplete';
 import { Cursor } from "./patterns/Cursor";
 import { Regex } from "./patterns/Regex";
 import { And } from "./patterns/And";
@@ -9,15 +13,14 @@ import { Repeat } from "./patterns/Repeat";
 import { ParseError } from "./patterns/ParseError";
 import { Pattern } from "./patterns/Pattern";
 import { Reference } from "./patterns/Reference";
-import { AutoComplete } from './intellisense/AutoComplete';
 import { CursorHistory, Match } from "./patterns/CursorHistory";
 import { ParseResult } from "./patterns/ParseResult";
-import { Suggestion } from "./intellisense/Suggestion";
-import { SuggestionOption } from "./intellisense/SuggestionOption";
 
 export {
   Node,
+  Grammar,
   AutoComplete,
+  AutoCompleteOptions,
   Suggestion,
   SuggestionOption,
   And,

package/src/intellisense/AutoComplete.test.ts

@@ -1,5 +1,4 @@
 import { And } from "../patterns/And";
-import { findPattern } from "../patterns/findPattern";
 import { Literal } from "../patterns/Literal";
 import { Or } from "../patterns/Or";
 import { Regex } from "../patterns/Regex";
@@ -71,7 +70,7 @@ describe("AutoComplete", () => {
         divider.setTokens([", "])
 
         const text = "John Doe";
-        const autoComplete = new AutoComplete(new Repeat("last-names", name, divider));
+        const autoComplete = new AutoComplete(new Repeat("last-names", name, { divider }));
         const result = autoComplete.suggestFor(text);
         const expectedOptions = [{
             text: ", ",
@@ -88,6 +87,7 @@ describe("AutoComplete", () => {
     test("Partial", () => {
         const name = new Literal("name", "Name");
         const autoComplete = new AutoComplete(name);
+        // Use deprecated suggest for code coverage.
         const result = autoComplete.suggestFor("Na");
         const expectedOptions = [{
             text: "me",
@@ -214,4 +214,23 @@ describe("AutoComplete", () => {
 
     });
 
+    test("Match On Different Pattern Roots", () => {
+        const start = new Literal("start", "John went to");
+        const a = new Literal("a", "a bank.");
+        const the = new Literal("the", "the store.");
+
+        const first = new And("first", [start, a]);
+        const second = new And("second", [start, the]);
+
+        const both = new Or("both", [first, second]);
+
+        const autoComplete = new AutoComplete(both);
+        const result = autoComplete.suggestFor("John went to a gas station.");
+        const expected = [
+            { text: "the store.", startIndex: 12 },
+            { text: "a bank.", startIndex: 12 }
+        ];
+        expect(result.options).toEqual(expected);
+    });
+
 });

package/src/intellisense/AutoComplete.ts

@@ -1,4 +1,5 @@
 import { Cursor } from "../patterns/Cursor";
+import { Match } from "../patterns/CursorHistory";
 import { Pattern } from "../patterns/Pattern";
 import { Suggestion } from "./Suggestion";
 import { SuggestionOption } from "./SuggestionOption";
@@ -31,14 +32,6 @@ export class AutoComplete {
     this._text = "";
   }
 
-  /**
-   * @deprecated Use suggestFor instead.
-   * @param text The text to suggest for.
-   */
-  suggest(text: string): Suggestion {
-    return this.suggestFor(text);
-  }
-
   suggestFor(text: string): Suggestion {
     if (text.length === 0) {
       return {
@@ -52,18 +45,12 @@ export class AutoComplete {
 
     this._text = text;
     this._cursor = new Cursor(text);
-    const ast = this._pattern.parse(this._cursor);
-
-    const leafPattern = this._cursor.leafMatch.pattern;
-    const isComplete = ast?.value === text;
-    const options = this._createSuggestionsFromTokens();
 
-    let nextPatterns = [this._pattern];
     let errorAtIndex = null;
 
-    if (leafPattern != null) {
-      nextPatterns = leafPattern.getNextPatterns();
-    }
+    const ast = this._pattern.parse(this._cursor);
+    const isComplete = ast?.value === text;
+    const options = this._getAllOptions();
 
     if (this._cursor.hasError && this._cursor.furthestError != null) {
       errorAtIndex = this._cursor.furthestError.index;
@@ -82,6 +69,10 @@ export class AutoComplete {
     }
   }
 
+  private _getAllOptions() {
+    return this._cursor.leafMatches.map((m) => this._createSuggestionsFromMatch(m)).flat();
+  }
+
   private _createSuggestionsFromRoot(): SuggestionOption[] {
     const suggestions: SuggestionOption[] = [];
     const tokens = this._pattern.getTokens();
@@ -93,17 +84,15 @@ export class AutoComplete {
     return suggestions;
   }
 
-  private _createSuggestionsFromTokens(): SuggestionOption[] {
-    const leafMatch = this._cursor.leafMatch;
-
-    if (!leafMatch.pattern) {
+  private _createSuggestionsFromMatch(match: Match): SuggestionOption[] {
+    if (!match.pattern) {
       return this._createSuggestions(-1, this._getTokensForPattern(this._pattern));
     }
 
-    const leafPattern = leafMatch.pattern;
-    const parent = leafMatch.pattern.parent;
+    const leafPattern = match.pattern;
+    const parent = match.pattern.parent;
 
-    if (parent !== null && leafMatch.node != null) {
+    if (parent !== null && match.node != null) {
       const patterns = leafPattern.getNextPatterns();
 
       const tokens = patterns.reduce((acc: string[], pattern) => {
@@ -111,7 +100,7 @@ export class AutoComplete {
         return acc;
       }, []);
 
-      return this._createSuggestions(leafMatch.node.lastIndex, tokens);
+      return this._createSuggestions(match.node.lastIndex, tokens);
     } else {
       return [];
     }

package/src/intellisense/Suggestion.ts

@@ -1,6 +1,5 @@
 import { Node } from "../ast/Node";
 import { Cursor } from "../patterns/Cursor";
-import { Pattern } from "../patterns/Pattern";
 import { SuggestionOption } from "./SuggestionOption";
 
 export interface Suggestion {

package/src/intellisense/css/cssValue.ts

@@ -2,6 +2,6 @@ import { Repeat } from "../../patterns/Repeat";
 import divider from "./divider";
 import values from "./values";
 
-const cssValue = new Repeat("css-value", values, divider);
+const cssValue = new Repeat("css-value", values, { divider });
 
 export default cssValue;

package/src/intellisense/css/method.ts

@@ -9,7 +9,7 @@ import divider from "./divider";
 const openParen = new Literal("open-paren", "(");
 const closeParen = new Literal("close-paren", ")");
 const values = new Reference("values");
-const args = new Repeat("arguments", values, divider, true);
+const args = new Repeat("arguments", values, { divider, min: 0 });
 const methodName = name.clone("method-name");
 methodName.setTokens(["rgba", "radial-gradient", "linear-gradient"]);
 

package/src/intellisense/css/values.ts

@@ -2,6 +2,6 @@ import { Repeat } from "../../patterns/Repeat";
 import value from "./value";
 import spaces from "./spaces";
 
-const values = new Repeat("values", value, spaces);
+const values = new Repeat("values", value, { divider: spaces });
 
 export default values;

package/src/intellisense/javascript/Javascript.test.ts

@@ -8,7 +8,6 @@ import { integer } from "./integer";
 import { name } from "./name";
 import { parameters } from "./parameters";
 import { prefixOperator } from "./prefixOperator";
-import { objectAccess } from "./objectAccess";
 
 describe("Ecmascript 3", () => {
     test("Escaped Character", () => {

package/src/intellisense/javascript/arrayLiteral.ts

@@ -7,7 +7,7 @@ import { Repeat } from "../../patterns/Repeat";
 import { optionalSpaces } from "./optionalSpaces";
 
 const divider = new Regex("array-divider", "\\s*,\\s*");
-const arrayItems = new Repeat("array-items", new Reference("expression"), divider, true);
+const arrayItems = new Repeat("array-items", new Reference("expression"), { divider, min: 0 });
 
 export const arrayLiteral = new Or("array-literal",
     [new And("empty-array-literal", [

package/src/intellisense/javascript/expression.ts

@@ -1,8 +1,8 @@
-import { Regex } from "../..";
 import { And } from "../../patterns/And";
 import { Literal } from "../../patterns/Literal";
 import { Or } from "../../patterns/Or";
 import { Reference } from "../../patterns/Reference";
+import { Regex } from "../../patterns/Regex";
 import { Repeat } from "../../patterns/Repeat";
 import { infixOperator } from "./infixOperator";
 import { invocation } from "./invocation";

package/src/intellisense/javascript/invocation.ts

@@ -11,7 +11,7 @@ const divider = new Regex("invocation-divider", "\\s*,\\s*");
 const invocationWithArguments = new And("invocation-with-arguments", [
     new Literal("open-paren", "("),
     optionalSpaces,
-    new Repeat("expressions", new Reference("expression"), divider, true),
+    new Repeat("expressions", new Reference("expression"), { divider, min: 0 }),
     optionalSpaces,
     new Literal("close-paren", ")"),
 ]);

package/src/intellisense/javascript/objectLiteral.ts

@@ -17,7 +17,7 @@ const property = new And("property", [
     new Reference("expression"),
 ]);
 const divider = new Regex("property-divider", "\\s*,\\s*");
-const optionalProperties = new Repeat("properties", property, divider, true);
+const optionalProperties = new Repeat("properties", property, { divider, min: 0 });
 
 const objectLiteral = new And("object-literal", [
     new Literal("open-curly-bracket", "{"),

package/src/intellisense/javascript/parameters.ts

@@ -12,7 +12,7 @@ const optionalSpace = new Regex("optional-space", "\\s", true)
 const parameters = new And("parameters", [
     new Literal("open-paren", "("),
     optionalSpace,
-    new Repeat("arguments", name, divider),
+    new Repeat("arguments", name, { divider, trimDivider: true }),
     optionalSpace,
     new Literal("close-paren", ")"),
 ]);

package/src/intellisense/javascript/stringLiteral.ts

@@ -12,8 +12,7 @@ const doubleQuoteStringLiteral = new And("double-string-literal", [
             new Regex("normal-characters", "[^\\\"]+"),
             escapedCharacter
         ]),
-        undefined,
-        true
+        { min: 0 }
     ),
     new Literal("double-quote", "\""),
 ]);
@@ -25,8 +24,7 @@ const singleQuoteStringLiteral = new And("single-string-literal", [
             new Regex("normal-characters", "[^\\']+"),
             escapedCharacter
         ]),
-        undefined,
-        true
+        { min: 0 }
     ),
     new Literal("single-quote", "'"),
 ]);

package/src/patterns/And.test.ts

@@ -276,7 +276,7 @@ describe("And", () => {
         const sequence = new And("sequence", [new Literal("a", "A")]);
         const parent = new And("parent", [sequence, new Literal("b", "B")]);
 
-        const sequenceClone = parent.findPattern(p => p.name === "sequence");
+        const sequenceClone = parent.find(p => p.name === "sequence");
         const tokens = sequenceClone?.getNextTokens() || [];
 
         expect(tokens[0]).toBe("B");
@@ -295,8 +295,8 @@ describe("And", () => {
             new Literal("b", "B"),
         ], true);
         const tokens = sequence.getPatterns();
-        const a = sequence.findPattern(p=>p.name === "a");
-        const b = sequence.findPattern(p=>p.name === "b");
+        const a = sequence.find(p=>p.name === "a");
+        const b = sequence.find(p=>p.name === "b");
         const expected = [a, b]
 
         expect(tokens).toEqual(expected);
@@ -306,9 +306,9 @@ describe("And", () => {
         const sequence = new And("sequence", [new Literal("a", "A")]);
         const parent = new And("parent", [sequence, new Literal("b", "B")]);
 
-        const sequenceClone = parent.findPattern(p => p.name === "sequence");
+        const sequenceClone = parent.find(p => p.name === "sequence");
         const nextPatterns = sequenceClone?.getNextPatterns() || [];
-        const b = parent.findPattern(p => p.name === "b")
+        const b = parent.find(p => p.name === "b")
 
         expect(nextPatterns[0]).toBe(b);
     });

package/src/patterns/And.ts

@@ -249,10 +249,10 @@ export class And implements Pattern {
   }
 
   getPatternsAfter(childReference: Pattern): Pattern[] {
+    const patterns: Pattern[] = [];
     let nextSibling: Pattern | null = null;
     let nextSiblingIndex = -1;
     let index = -1;
-    const patterns: Pattern[] = [];
 
     for (let i = 0; i < this._children.length; i++) {
       if (this._children[i] === childReference) {
@@ -275,24 +275,18 @@ export class And implements Pattern {
       return this._parent.getPatternsAfter(this);
     }
 
-    // Next pattern isn't optional so send it back as the next patterns.
-    if (nextSibling !== null && !nextSibling.isOptional) {
-      return [nextSibling];
-    }
-
     // Send back as many optional patterns as possible.
-    if (nextSibling !== null && nextSibling.isOptional) {
-      for (let i = nextSiblingIndex; i < this._children.length; i++) {
-        const child = this._children[i];
-        patterns.push(child);
+    for (let i = nextSiblingIndex; i < this._children.length; i++) {
+      const child = this._children[i];
+      patterns.push(child);
 
-        if (!child.isOptional) {
-          break;
-        }
+      if (!child.isOptional) {
+        break;
+      }
 
-        if (i === this._children.length - 1 && this._parent !== null) {
-          patterns.push(...this._parent.getPatternsAfter(this));
-        }
+      // If we are on the last child and its options then ask for the next pattern from the parent.
+      if (i === this._children.length - 1 && this._parent !== null) {
+        patterns.push(...this._parent.getPatternsAfter(this));
       }
     }
 
@@ -307,7 +301,7 @@ export class And implements Pattern {
     return this.parent.getPatternsAfter(this)
   }
 
-  findPattern(predicate: (p: Pattern) => boolean): Pattern | null {
+  find(predicate: (p: Pattern) => boolean): Pattern | null {
     return findPattern(this, predicate);
   }
 

package/src/patterns/Cursor.ts

@@ -33,6 +33,10 @@ export class Cursor {
     return this._history.leafMatch;
   }
 
+  get leafMatches(): Match[] {
+    return this._history.leafMatches;
+  }
+
   get furthestError(): ParseError | null {
     return this._history.furthestError;
   }

package/src/patterns/CursorHistory.ts

@@ -9,7 +9,7 @@ export interface Match {
 
 export class CursorHistory {
   private _isRecording: boolean = false;
-  private _leafMatch: Match = { pattern: null, node: null };
+  private _leafMatches: Match[] = [{ pattern: null, node: null }];
   private _furthestError: ParseError | null = null;
   private _currentError: ParseError | null = null;
   private _rootMatch: Match = { pattern: null, node: null };
@@ -26,7 +26,11 @@ export class CursorHistory {
   }
 
   get leafMatch(): Match {
-    return this._leafMatch;
+    return this._leafMatches[this._leafMatches.length - 1];
+  }
+
+  get leafMatches() {
+    return this._leafMatches;
   }
 
   get furthestError(): ParseError | null {
@@ -57,13 +61,38 @@ export class CursorHistory {
 
     this._rootMatch.pattern = pattern;
     this._rootMatch.node = node;
+    const leafMatch = this._leafMatches[this._leafMatches.length - 1];
 
     const isFurthestMatch =
-      this._leafMatch.node === null || node.lastIndex > this._leafMatch.node.lastIndex;
+      leafMatch.node === null || node.lastIndex > leafMatch.node.lastIndex;
+
+    const isSameIndexMatch =
+      leafMatch.node === null || node.lastIndex === leafMatch.node.lastIndex
 
     if (isFurthestMatch) {
-      this._leafMatch.pattern = pattern;
-      this._leafMatch.node = node;
+      // This is to save on GC churn.
+      const match = this._leafMatches.pop() as Match;
+      match.pattern = pattern;
+      match.node = node;
+
+      this._leafMatches.length = 0;
+      this._leafMatches.push(match);
+    } else if (isSameIndexMatch) {
+      const isAncestor = this._leafMatches.some((m) => {
+        let parent = m.pattern?.parent;
+
+        while (parent != null) {
+          if (parent == pattern.parent) {
+            return true;
+          }
+          parent = parent.parent
+        }
+        return false;
+      })
+
+      if (!isAncestor) {
+        this._leafMatches.unshift({ pattern, node });
+      }
     }
   }