clarity-pattern-parser 11.4.1 → 11.4.2

package/dist/intellisense/AutoComplete.d.ts

@@ -42,7 +42,7 @@ export declare class AutoComplete {
    * ie. sequence pattern segments ≈ [{look}, {an example}, {phrase}]
    * fullText = "look an"
    * remove {look} segment as its already been completed by the existing text.
- */
+  */
     private _filterCompletedSubSegments;
     private _getCompositeSuggestionsForPattern;
     private _getCustomTokens;

package/dist/patterns/Cursor.d.ts

@@ -3,7 +3,9 @@ import { Match } from "./CursorHistory";
 import { ParseError } from "./ParseError";
 import { Pattern } from "./Pattern";
 export declare class Cursor {
-    private _chars;
+    private _text;
+    private _charSize;
+    private _charMap;
     private _index;
     private _length;
     private _history;
@@ -33,10 +35,13 @@ export declare class Cursor {
     moveToFirstChar(): void;
     moveToLastChar(): void;
     getLastIndex(): number;
-    getChars(first: number, last: number): string;
+    substring(first: number, last: number): string;
     recordMatch(pattern: Pattern, node: Node): void;
     recordErrorAt(startIndex: number, lastIndex: number, onPattern: Pattern): void;
     resolveError(): void;
     startRecording(): void;
     stopRecording(): void;
+    getCharStartIndex(index: number): number;
+    getCharEndIndex(index: number): number;
+    getCharLastIndex(index: number): number;
 }

package/dist/patterns/Literal.d.ts

@@ -8,10 +8,8 @@ export declare class Literal implements Pattern {
     private _name;
     private _parent;
     private _token;
-    private _runes;
     private _firstIndex;
     private _lastIndex;
-    private _endIndex;
     get id(): string;
     get type(): string;
     get name(): string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-pattern-parser",
-  "version": "11.4.1",
+  "version": "11.4.2",
   "description": "Parsing Library for Typescript and Javascript.",
   "main": "./dist/index.js",
   "module": "./dist/index.esm.js",

package/src/intellisense/AutoComplete.test.ts

@@ -194,7 +194,7 @@ describe("AutoComplete", () => {
 
         expect(result.ast).toBeNull();
         optionsMatchExpected(result.options, expectedOptions);
-        expect(result.errorAtIndex).toBe(text.length);
+        expect(result.errorAtIndex).toBe(text.length - 1);
         expect(result.isComplete).toBeFalsy();
         expect(result.cursor).not.toBeNull();
     });
@@ -313,7 +313,7 @@ describe("AutoComplete", () => {
 
         expect(result.ast).toBeNull();
         optionsMatchExpected(result.options, expectedOptions);
-        expect(result.errorAtIndex).toBe(2);
+        expect(result.errorAtIndex).toBe(1);
         expect(result.isComplete).toBeFalsy();
         expect(result.cursor).not.toBeNull();
     });
@@ -944,7 +944,7 @@ describe("AutoComplete", () => {
 
         optionsMatchExpected(suggestion.options, expected);
 
-        expect(suggestion.error?.lastIndex).toBe(2);
+        expect(suggestion.error?.lastIndex).toBe(1);
     });
 
     test("Recursion With And", () => {

package/src/intellisense/AutoComplete.ts

@@ -107,7 +107,7 @@ export class AutoComplete {
     const furthestMatch = cursor.allMatchedNodes[cursor.allMatchedNodes.length - 1];
 
     if (furthestError && furthestMatch) {
-      if (furthestMatch.endIndex  > furthestError.lastIndex ) {
+      if (furthestMatch.endIndex > furthestError.lastIndex) {
         return furthestMatch.endIndex;
       } else {
         return furthestError.lastIndex;
@@ -131,7 +131,7 @@ export class AutoComplete {
 
     const errorMatchOptions = this._createSuggestionOptionsFromErrors();
     const leafMatchOptions = this._cursor.leafMatches.map((m) => this._createSuggestionOptionsFromMatch(m)).flat();
-    
+
     const finalResults: SuggestionOption[] = [];
     [...leafMatchOptions, ...errorMatchOptions].forEach(m => {
       const index = finalResults.findIndex(f => m.text === f.text);
@@ -145,15 +145,15 @@ export class AutoComplete {
 
   private _createSuggestionOptionsFromErrors() {
     // These errored because the length of the string.
-    const errors = this._cursor.errors.filter(e => e.lastIndex === this._cursor.length);
-    
+    const errors = this._cursor.errors.filter(e => e.lastIndex === this._cursor.length - 1);
+
     const errorSuggestionOptions = errors.map(parseError => {
 
-      const currentText = this._cursor.getChars(parseError.startIndex, parseError.lastIndex);
-    
+      const currentText = this._cursor.substring(parseError.startIndex, parseError.lastIndex);
+
       const compositeSuggestions = this._getCompositeSuggestionsForPattern(parseError.pattern);
       const trimmedErrorCompositeSuggestions = this._trimSuggestionsByExistingText(currentText, compositeSuggestions);
-  
+
       return this._createSuggestions(parseError.lastIndex, trimmedErrorCompositeSuggestions);
     }).flat();
 
@@ -170,25 +170,25 @@ export class AutoComplete {
       return this._createSuggestions(-1, compositeSuggestions);
     }
 
-    if ( match?.node != null) {
-      const currentText = this._text.slice(match.node.startIndex,match.node.endIndex)
-      
-      
+    if (match?.node != null) {
+      const currentText = this._text.slice(match.node.startIndex, match.node.endIndex)
+
+
       /**Captures suggestions for a "completed" match pattern that still has existing possible suggestions.
        * particularly relevant when working with set/custom tokens.
       */
       const matchCompositeSuggestions = this._getCompositeSuggestionsForPattern(match.pattern)
-      const trimmedMatchCompositeSuggestions =  this._trimSuggestionsByExistingText(currentText, matchCompositeSuggestions) 
+      const trimmedMatchCompositeSuggestions = this._trimSuggestionsByExistingText(currentText, matchCompositeSuggestions)
+
 
-      
       const leafPatterns = match.pattern.getNextPatterns();
-      const leafCompositeSuggestions = leafPatterns.flatMap(leafPattern => 
-      this._getCompositeSuggestionsForPattern(leafPattern)
-    );
+      const leafCompositeSuggestions = leafPatterns.flatMap(leafPattern =>
+        this._getCompositeSuggestionsForPattern(leafPattern)
+      );
 
-     const allCompositeSuggestions = [...leafCompositeSuggestions,...trimmedMatchCompositeSuggestions,]
+      const allCompositeSuggestions = [...leafCompositeSuggestions, ...trimmedMatchCompositeSuggestions,]
 
-     const dedupedCompositeSuggestions = this._deDupeCompositeSuggestions(allCompositeSuggestions);
+      const dedupedCompositeSuggestions = this._deDupeCompositeSuggestions(allCompositeSuggestions);
 
       return this._createSuggestions(match.node.lastIndex, dedupedCompositeSuggestions);
     } else {
@@ -202,58 +202,58 @@ export class AutoComplete {
    *   - refines to {d}{ef}
    */
   private _trimSuggestionsByExistingText(currentText: string, compositeSuggestions: CompositeSuggestion[]): CompositeSuggestion[] {
-    
-        const trimmedSuggestions = compositeSuggestions.reduce<CompositeSuggestion[]>((acc, compositeSuggestion) => {
-          if (compositeSuggestion.text.startsWith(currentText)) {
-    
-            const filteredSegments = this._filterCompletedSubSegments(currentText, compositeSuggestion);
-            const slicedSuggestionText = compositeSuggestion.text.slice(currentText.length);
-    
-            if (slicedSuggestionText !== '') {
-              const refinedCompositeSuggestion: CompositeSuggestion = {
-                text: slicedSuggestionText,
-                suggestionSequence: filteredSegments,
-              }
-    
-              acc.push(refinedCompositeSuggestion);
-            }
+
+    const trimmedSuggestions = compositeSuggestions.reduce<CompositeSuggestion[]>((acc, compositeSuggestion) => {
+      if (compositeSuggestion.text.startsWith(currentText)) {
+
+        const filteredSegments = this._filterCompletedSubSegments(currentText, compositeSuggestion);
+        const slicedSuggestionText = compositeSuggestion.text.slice(currentText.length);
+
+        if (slicedSuggestionText !== '') {
+          const refinedCompositeSuggestion: CompositeSuggestion = {
+            text: slicedSuggestionText,
+            suggestionSequence: filteredSegments,
           }
-          return acc;
-        }, []);
 
-        return trimmedSuggestions
+          acc.push(refinedCompositeSuggestion);
+        }
+      }
+      return acc;
+    }, []);
+
+    return trimmedSuggestions
   }
 
-    /** Removed segments already accounted for in the existing text.
-   * ie. sequence pattern segments ≈ [{look}, {an example}, {phrase}]
-   * fullText = "look an"
-   * remove {look} segment as its already been completed by the existing text.
- */
-    private _filterCompletedSubSegments(currentText: string, compositeSuggestion: CompositeSuggestion){
-
-      let elementsToRemove: SuggestionSegment [] = [];
-      let workingText = currentText;
-  
-      compositeSuggestion.suggestionSequence.forEach(segment => {
-        /**sub segment has been completed, remove it from the sequence */
-        if(workingText.startsWith(segment.text)){
-          workingText = workingText.slice(segment.text.length);
-          elementsToRemove.push (segment);
-        }
-      })
-  
-      const filteredSegments = compositeSuggestion.suggestionSequence.filter(segment => !elementsToRemove.includes (segment) );
-  
-      return filteredSegments
-    }
+  /** Removed segments already accounted for in the existing text.
+ * ie. sequence pattern segments ≈ [{look}, {an example}, {phrase}]
+ * fullText = "look an"
+ * remove {look} segment as its already been completed by the existing text.
+*/
+  private _filterCompletedSubSegments(currentText: string, compositeSuggestion: CompositeSuggestion) {
+
+    let elementsToRemove: SuggestionSegment[] = [];
+    let workingText = currentText;
+
+    compositeSuggestion.suggestionSequence.forEach(segment => {
+      /**sub segment has been completed, remove it from the sequence */
+      if (workingText.startsWith(segment.text)) {
+        workingText = workingText.slice(segment.text.length);
+        elementsToRemove.push(segment);
+      }
+    })
+
+    const filteredSegments = compositeSuggestion.suggestionSequence.filter(segment => !elementsToRemove.includes(segment));
+
+    return filteredSegments
+  }
+
+  private _getCompositeSuggestionsForPattern(pattern: Pattern): CompositeSuggestion[] {
 
-  private _getCompositeSuggestionsForPattern(pattern: Pattern):CompositeSuggestion[] {
+    const suggestionsToReturn: CompositeSuggestion[] = [];
 
-    const suggestionsToReturn:CompositeSuggestion[] = [];
-    
     const leafPatterns = pattern.getPatterns();
     // for when pattern has no leafPatterns and only returns itself
-    if(leafPatterns.length === 1 && leafPatterns[0].id === pattern.id) { 
+    if (leafPatterns.length === 1 && leafPatterns[0].id === pattern.id) {
 
       const currentCustomTokens = this._getCustomTokens(pattern);
       const currentTokens = pattern.getTokens();
@@ -261,12 +261,12 @@ export class AutoComplete {
 
       const leafCompositeSuggestions: CompositeSuggestion[] = allTokens.map(token => {
 
-        const segment:SuggestionSegment = {
+        const segment: SuggestionSegment = {
           text: token,
           pattern: pattern,
         }
-  
-        const compositeSuggestion:CompositeSuggestion = { 
+
+        const compositeSuggestion: CompositeSuggestion = {
           text: token,
           suggestionSequence: [segment],
         }
@@ -274,17 +274,17 @@ export class AutoComplete {
       })
       suggestionsToReturn.push(...leafCompositeSuggestions);
 
-    }else{
+    } else {
 
       const currentCustomTokens = this._getCustomTokens(pattern);
 
       const patternsSuggestionList = currentCustomTokens.map(token => {
-        const segment:SuggestionSegment = {
+        const segment: SuggestionSegment = {
           text: token,
           pattern: pattern,
         }
 
-        const patternSuggestion:CompositeSuggestion = {
+        const patternSuggestion: CompositeSuggestion = {
           text: token,
           suggestionSequence: [segment],
         }
@@ -306,15 +306,15 @@ export class AutoComplete {
 
         return acc;
       }, []);
-      
-      const compositeSuggestionList:CompositeSuggestion[] = [];
-      
+
+      const compositeSuggestionList: CompositeSuggestion[] = [];
+
       for (const currentSuggestion of suggestionsToReturn) {
-        for (const nextSuggestionWithSubElements of nextPatternedTokensList) { 
+        for (const nextSuggestionWithSubElements of nextPatternedTokensList) {
 
-          const augmentedTokenWithPattern:CompositeSuggestion = {
+          const augmentedTokenWithPattern: CompositeSuggestion = {
             text: currentSuggestion.text + nextSuggestionWithSubElements.text,
-            suggestionSequence: [...currentSuggestion.suggestionSequence,...nextSuggestionWithSubElements.suggestionSequence ],
+            suggestionSequence: [...currentSuggestion.suggestionSequence, ...nextSuggestionWithSubElements.suggestionSequence],
           }
 
           compositeSuggestionList.push(augmentedTokenWithPattern);
@@ -323,7 +323,7 @@ export class AutoComplete {
 
       return compositeSuggestionList;
 
-    } else {   
+    } else {
 
       const dedupedSuggestions = this._deDupeCompositeSuggestions(suggestionsToReturn);
       return dedupedSuggestions;
@@ -332,16 +332,16 @@ export class AutoComplete {
 
   private _getCustomTokens(pattern: Pattern) {
 
-   const customTokensMap: Record<string, string[]> = this._options.customTokens || {};
-   const customTokens = customTokensMap[pattern.name] ?? [];
+    const customTokensMap: Record<string, string[]> = this._options.customTokens || {};
+    const customTokens = customTokensMap[pattern.name] ?? [];
 
-   const allTokens = [...customTokens];
+    const allTokens = [...customTokens];
 
-   return allTokens;
+    return allTokens;
   }
 
 
-  
+
   private _deDupeCompositeSuggestions<T extends CompositeSuggestion>(suggestions: T[]): T[] {
 
     if (this._options.disableDedupe) {
@@ -370,9 +370,9 @@ export class AutoComplete {
     return unique;
   }
 
-  private _createSuggestions(lastIndex: number, compositeSuggestionList:CompositeSuggestion[]): SuggestionOption[] {
+  private _createSuggestions(lastIndex: number, compositeSuggestionList: CompositeSuggestion[]): SuggestionOption[] {
 
-    let textToIndex = lastIndex === -1 ? "" : this._cursor.getChars(0, lastIndex);
+    let textToIndex = lastIndex === -1 ? "" : this._cursor.substring(0, lastIndex);
     const suggestionStrings: string[] = [];
     const options: SuggestionOption[] = [];
 
@@ -384,9 +384,9 @@ export class AutoComplete {
       const isSameAsText = existingTextWithSuggestion === this._text;
 
       // if ( !alreadyExist && !isSameAsText) {
-        suggestionStrings.push(existingTextWithSuggestion);
-        const suggestionOption = this._createSuggestionOption(this._cursor.text, existingTextWithSuggestion, compositeSuggestion.suggestionSequence);
-        options.push(suggestionOption);
+      suggestionStrings.push(existingTextWithSuggestion);
+      const suggestionOption = this._createSuggestionOption(this._cursor.text, existingTextWithSuggestion, compositeSuggestion.suggestionSequence);
+      options.push(suggestionOption);
       // }
     }
 
@@ -400,7 +400,7 @@ export class AutoComplete {
     const furthestMatch = findMatchIndex(suggestion, fullText);
     const text = suggestion.slice(furthestMatch);
 
-    const option:SuggestionOption = {
+    const option: SuggestionOption = {
       text: text,
       startIndex: furthestMatch,
       suggestionSequence: segments,

package/src/intellisense/javascript/stringLiteral.ts

@@ -10,7 +10,7 @@ const doubleQuoteStringLiteral = new Sequence("double-string-literal", [
     new Literal("double-quote", "\""),
     new Optional("optional-characters", new Repeat("characters",
         new Options("characters", [
-            new Regex("normal-characters", "[^\\\"]+"),
+            new Regex("normal-characters", '[^"]+'),
             escapedCharacter
         ])
     )),
@@ -21,7 +21,7 @@ const singleQuoteStringLiteral = new Sequence("single-string-literal", [
     new Literal("single-quote", "'"),
     new Optional("optional-characters", new Repeat("characters",
         new Options("characters", [
-            new Regex("normal-characters", "[^\\']+"),
+            new Regex("normal-characters", "[^']+"),
             escapedCharacter
         ]),
     )),

package/src/patterns/Cursor.test.ts

@@ -27,7 +27,7 @@ describe("Cursor", () => {
         cursor.previous();
         expect(cursor.index).toBe(0);
 
-        const text = cursor.getChars(0, 1);
+        const text = cursor.substring(0, 1);
         expect(text).toBe("");
     });
 
@@ -107,7 +107,7 @@ describe("Cursor", () => {
 
     test("Text Information", () => {
         const cursor = new Cursor("Hello World!");
-        const hello = cursor.getChars(0, 4);
+        const hello = cursor.substring(0, 4);
 
         expect(hello).toBe("Hello");
         expect(cursor.length).toBe(12);
@@ -153,4 +153,12 @@ describe("Cursor", () => {
         expect(records[2].pattern.name).toBe("first-names");
     });
 
+    test("Text with Emojis", () => {
+        const cursor = new Cursor("🔴 World!");
+        expect(cursor.currentChar).toBe("🔴");
+        cursor.next();
+        expect(cursor.currentChar).toBe(" ");
+        expect(cursor.length).toBe(9);
+    });
+
 });

package/src/patterns/Cursor.ts

@@ -3,14 +3,18 @@ import { CursorHistory, Match } from "./CursorHistory";
 import { ParseError } from "./ParseError";
 import { Pattern } from "./Pattern";
 
+const segmenter = new Intl.Segmenter("und", { granularity: "grapheme" });
+
 export class Cursor {
-  private _chars: string[];
+  private _text: string;
+  private _charSize: number[];
+  private _charMap: number[];
   private _index: number;
   private _length: number;
   private _history: CursorHistory;
 
   get text(): string {
-    return this._chars.join("");
+    return this._text;
   }
 
   get isOnFirst(): boolean {
@@ -74,39 +78,62 @@ export class Cursor {
   }
 
   get currentChar(): string {
-    return this._chars[this._index];
+    const index = this.getCharStartIndex(this._index);
+    return this.text.slice(index, index + this._charSize[index]);
   }
 
   constructor(text: string) {
-    this._chars = [...text];
+    this._text = text;
+    this._length = text.length;
+    this._charSize = [];
+    this._charMap = [];
+
     this._index = 0;
-    this._length = this._chars.length;
     this._history = new CursorHistory();
+
+    let index = 0;
+    for (const segment of segmenter.segment(text)) {
+      const size = segment.segment.length;
+      for (let i = 0; i < size; i++) {
+        this._charMap.push(index);
+        this._charSize.push(size);
+      }
+      index += size;
+    }
   }
 
   hasNext(): boolean {
-    return this._index + 1 < this._length;
+    const index = this._charMap[this._index];
+    const charSize = this._charSize[index];
+    return index + charSize < this._length;
   }
 
   next(): void {
     if (this.hasNext()) {
-      this._index++;
+      const index = this._charMap[this._index];
+      const size = this._charSize[index];
+      this.moveTo(index + size);
     }
   }
 
   hasPrevious(): boolean {
-    return this._index - 1 >= 0;
+    const index = this._charMap[this._index];
+    const previousIndex = this._charMap[index - 1] ?? -1;
+
+    return previousIndex >= 0;
   }
 
   previous(): void {
     if (this.hasPrevious()) {
-      this._index--;
+      const index = this._charMap[this._index];
+      const previousIndex = this._charMap[index - 1] ?? -1;;
+      this.moveTo(previousIndex);
     }
   }
 
   moveTo(position: number): void {
     if (position >= 0 && position < this._length) {
-      this._index = position;
+      this._index = this._charMap[position];
     }
   }
 
@@ -122,8 +149,8 @@ export class Cursor {
     return this._length - 1;
   }
 
-  getChars(first: number, last: number): string {
-    return this._chars.slice(first, last + 1).join("");
+  substring(first: number, last: number): string {
+    return this._text.slice(first, last + 1);
   }
 
   recordMatch(pattern: Pattern, node: Node): void {
@@ -146,4 +173,17 @@ export class Cursor {
     this._history.stopRecording();
   }
 
+  getCharStartIndex(index: number): number {
+    return this._charMap[index];
+  }
+
+  getCharEndIndex(index: number): number {
+    let startIndex = this.getCharStartIndex(index);
+    return startIndex + this._charSize[startIndex] ?? 1;
+  }
+
+  getCharLastIndex(index: number): number {
+    return this.getCharEndIndex(index) - 1 ?? 0;
+  }
+
 }

package/src/patterns/Literal.test.ts

@@ -48,7 +48,7 @@ describe("Literal", () => {
         expect(result).toEqual(null);
         expect(cursor.index).toBe(10);
         expect(cursor.error?.startIndex).toBe(0);
-        expect(cursor.error?.lastIndex).toBe(11);
+        expect(cursor.error?.lastIndex).toBe(10);
         expect(cursor.error?.pattern).toBe(literal);
     });
 
@@ -163,4 +163,44 @@ describe("Literal", () => {
 
         expect(pattern).toBeNull();
     });
+
+    test("Unicode", () => {
+        const literal = new Literal("a", "🔴");
+        const cursor = new Cursor("🔴");
+        const result = literal.parse(cursor);
+        expect(result).not.toBeNull();
+        expect(cursor.index).toBe(0);
+        expect(result?.toString()).toBe("🔴");
+        expect(cursor.error).toBeNull();
+    });
+
+    test("Unicode In Middle", () => {
+        const literal = new Literal("a", "|🔴|");
+        const cursor = new Cursor("|🔴|");
+        const result = literal.parse(cursor);
+        expect(result).not.toBeNull();
+        expect(cursor.index).toBe(3);
+        expect(result?.toString()).toBe("|🔴|");
+        expect(cursor.error).toBeNull();
+    });
+
+    test("Unicode At the End", () => {
+        const literal = new Literal("a", "|🔴");
+        const cursor = new Cursor("|🔴");
+        const result = literal.parse(cursor);
+        expect(result).not.toBeNull();
+        expect(cursor.index).toBe(1);
+        expect(result?.toString()).toBe("|🔴");
+        expect(cursor.error).toBeNull();
+    });
+
+    test("Unicode At the Beginning", () => {
+        const literal = new Literal("a", "🔴|");
+        const cursor = new Cursor("🔴|");
+        const result = literal.parse(cursor);
+        expect(result).not.toBeNull();
+        expect(cursor.index).toBe(2);
+        expect(result?.toString()).toBe("🔴|");
+        expect(cursor.error).toBeNull();
+    });
 });