@graffiticode/parser 1.0.0 → 1.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graffiticode/parser",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "type": "module",
   "publishConfig": {
     "access": "public"

package/src/parse.js

@@ -174,6 +174,49 @@ export const parse = (function () {
   const TK_STRSUFFIX = 0xB4;
   const TK_DOTDOT = 0xB5;
 
+  // Process escape sequences in a string lexeme
+  function processEscapeSequences(str) {
+    // The string still has backslash escape sequences
+    // Process them to get the actual string value
+    let result = "";
+    let i = 0;
+    while (i < str.length) {
+      if (str[i] === '\\' && i + 1 < str.length) {
+        // Handle escape sequence
+        const nextChar = str[i + 1];
+        switch (nextChar) {
+          case '\\':
+          case '"':
+          case "'":
+          case '`':
+            result += nextChar;
+            break;
+          case 'n':
+            result += '\n';
+            break;
+          case 't':
+            result += '\t';
+            break;
+          case 'r':
+            result += '\r';
+            break;
+          case '$':
+            result += '$';
+            break;
+          default:
+            // Unknown escape, keep the backslash and character
+            result += '\\' + nextChar;
+            break;
+        }
+        i += 2;
+      } else {
+        result += str[i];
+        i++;
+      }
+    }
+    return result;
+  }
+
   function tokenToLexeme(tk) {
     switch (tk) {
       case TK_EQUAL: return "a '=' symbol";
@@ -300,20 +343,24 @@ export const parse = (function () {
   function str(ctx, cc) {
     if (match(ctx, TK_STR)) {
       eat(ctx, TK_STR);
-      Ast.string(ctx, lexeme, getCoord(ctx)); // strip quotes;
+      // Process escape sequences in the lexeme
+      const processedStr = processEscapeSequences(lexeme);
+      Ast.string(ctx, processedStr, getCoord(ctx)); // strip quotes;
       cc.cls = "string";
       return cc;
     } else if (match(ctx, TK_STRPREFIX)) {
       ctx.state.inStr++;
       eat(ctx, TK_STRPREFIX);
       startCounter(ctx);
-      Ast.string(ctx, lexeme, getCoord(ctx)); // strip quotes;
+      const processedPrefix = processEscapeSequences(lexeme);
+      Ast.string(ctx, processedPrefix, getCoord(ctx)); // strip quotes;
       countCounter(ctx);
       const ret = function (ctx) {
         return strSuffix(ctx, function (ctx) {
           ctx.state.inStr--;
           eat(ctx, TK_STRSUFFIX);
-          Ast.string(ctx, lexeme, getCoord(ctx)); // strip quotes;
+          const processedSuffix = processEscapeSequences(lexeme);
+          Ast.string(ctx, processedSuffix, getCoord(ctx)); // strip quotes;
           countCounter(ctx);
           Ast.list(ctx, ctx.state.exprc);
           stopCounter(ctx);
@@ -337,7 +384,8 @@ export const parse = (function () {
       if (match(ctx, TK_STRMIDDLE)) {
         // Not done yet.
         eat(ctx, TK_STRMIDDLE);
-        Ast.string(ctx, lexeme, getCoord(ctx)); // strip quotes;
+        const processedMiddle = processEscapeSequences(lexeme);
+        Ast.string(ctx, processedMiddle, getCoord(ctx)); // strip quotes;
         countCounter(ctx);
         ret = function (ctx) {
           return strSuffix(ctx, resume);
@@ -1199,24 +1247,44 @@ export const parse = (function () {
       lexeme += String.fromCharCode(c);
       c = nextCC();
       const inTemplateLiteral = quoteChar === CC_BACKTICK;
+      let escaped = false;
+
       if (inTemplateLiteral) {
         while (
-          c !== quoteChar &&
+          (c !== quoteChar || escaped) &&
           c !== 0 &&
-          !(c === CC_DOLLAR && peekCC() === CC_LEFTBRACE)) {
-          lexeme += String.fromCharCode(c);
+          !(c === CC_DOLLAR && peekCC() === CC_LEFTBRACE && !escaped)) {
+          if (escaped) {
+            // Handle escaped characters
+            lexeme += String.fromCharCode(c);
+            escaped = false;
+          } else if (c === 92) { // backslash
+            lexeme += String.fromCharCode(c);
+            escaped = true;
+          } else {
+            lexeme += String.fromCharCode(c);
+          }
           c = nextCC();
         }
       } else {
-        while (c !== quoteChar && c !== 0) {
-          lexeme += String.fromCharCode(c);
+        while ((c !== quoteChar || escaped) && c !== 0) {
+          if (escaped) {
+            // Handle escaped characters
+            lexeme += String.fromCharCode(c);
+            escaped = false;
+          } else if (c === 92) { // backslash
+            lexeme += String.fromCharCode(c);
+            escaped = true;
+          } else {
+            lexeme += String.fromCharCode(c);
+          }
           c = nextCC();
         }
       }
       const coord = { from: getPos(ctx) - lexeme.length, to: getPos(ctx) };
       assertErr(ctx, c !== 0, `Unterminated string: ${lexeme}`, coord);
       if (quoteChar === CC_BACKTICK && c === CC_DOLLAR &&
-          peekCC() === CC_LEFTBRACE) {
+          peekCC() === CC_LEFTBRACE && !escaped) {
         nextCC(); // Eat CC_LEFTBRACE
         lexeme = lexeme.substring(1); // Strip off punct.
         return TK_STRPREFIX;
@@ -1234,21 +1302,41 @@ export const parse = (function () {
       const quoteChar = quoteCharStack[quoteCharStack.length - 1];
       c = nextCC();
       const inTemplateLiteral = quoteChar === CC_BACKTICK;
+      let escaped = false;
+
       if (inTemplateLiteral) {
-        while (c !== quoteChar && c !== 0 &&
+        while ((c !== quoteChar || escaped) && c !== 0 &&
              !(c === CC_DOLLAR &&
-               peekCC() === CC_LEFTBRACE)) {
-          lexeme += String.fromCharCode(c);
+               peekCC() === CC_LEFTBRACE && !escaped)) {
+          if (escaped) {
+            // Handle escaped characters
+            lexeme += String.fromCharCode(c);
+            escaped = false;
+          } else if (c === 92) { // backslash
+            lexeme += String.fromCharCode(c);
+            escaped = true;
+          } else {
+            lexeme += String.fromCharCode(c);
+          }
           c = nextCC();
         }
       } else {
-        while (c !== quoteChar && c !== 0) {
-          lexeme += String.fromCharCode(c);
+        while ((c !== quoteChar || escaped) && c !== 0) {
+          if (escaped) {
+            // Handle escaped characters
+            lexeme += String.fromCharCode(c);
+            escaped = false;
+          } else if (c === 92) { // backslash
+            lexeme += String.fromCharCode(c);
+            escaped = true;
+          } else {
+            lexeme += String.fromCharCode(c);
+          }
           c = nextCC();
         }
       }
       if (quoteChar === CC_BACKTICK && c === CC_DOLLAR &&
-          peekCC() === CC_LEFTBRACE) {
+          peekCC() === CC_LEFTBRACE && !escaped) {
         nextCC(); // Eat brace.
         lexeme = lexeme.substring(1); // Strip off leading brace and trailing brace.
         return TK_STRMIDDLE;

package/src/parser.spec.js

@@ -425,4 +425,149 @@ describe("parser integration tests", () => {
     expect(found123).toBe(false);
     expect(found456).toBe(false);
   });
+
+  // Tests for escaped quotes
+  it("should parse strings with escaped double quotes", async () => {
+    // Arrange & Act
+    const result = await parser.parse(0, '"He said \\"Hello\\""..', basisLexicon);
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Find the STR node
+    let strNode = null;
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "STR" && node.elts[0] === 'He said "Hello"') {
+          strNode = node;
+          break;
+        }
+      }
+    }
+
+    expect(strNode).not.toBeNull();
+    expect(strNode.tag).toBe("STR");
+    expect(strNode.elts[0]).toBe('He said "Hello"');
+  });
+
+  it("should parse strings with escaped single quotes", async () => {
+    // Arrange & Act
+    const result = await parser.parse(0, "'It\\'s working!'..", basisLexicon);
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Find the STR node
+    let strNode = null;
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "STR" && node.elts[0] === "It's working!") {
+          strNode = node;
+          break;
+        }
+      }
+    }
+
+    expect(strNode).not.toBeNull();
+    expect(strNode.tag).toBe("STR");
+    expect(strNode.elts[0]).toBe("It's working!");
+  });
+
+  it("should parse strings with escaped backticks", async () => {
+    // Arrange & Act
+    const result = await parser.parse(0, "`This has a \\` backtick`..", basisLexicon);
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Find the STR node
+    let strNode = null;
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "STR" && node.elts[0] === "This has a ` backtick") {
+          strNode = node;
+          break;
+        }
+      }
+    }
+
+    expect(strNode).not.toBeNull();
+    expect(strNode.tag).toBe("STR");
+    expect(strNode.elts[0]).toBe("This has a ` backtick");
+  });
+
+  it("should parse strings with escaped backslashes", async () => {
+    // Arrange & Act
+    const result = await parser.parse(0, '"Path: C:\\\\Users\\\\Test"..', basisLexicon);
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Find the STR node
+    let strNode = null;
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "STR" && node.elts[0] === "Path: C:\\Users\\Test") {
+          strNode = node;
+          break;
+        }
+      }
+    }
+
+    expect(strNode).not.toBeNull();
+    expect(strNode.tag).toBe("STR");
+    expect(strNode.elts[0]).toBe("Path: C:\\Users\\Test");
+  });
+
+  it("should parse template literals with escaped interpolation", async () => {
+    // Arrange & Act
+    const result = await parser.parse(0, "`Price: \\${amount}`..", basisLexicon);
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Find the STR node
+    let strNode = null;
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "STR" && node.elts[0] === "Price: ${amount}") {
+          strNode = node;
+          break;
+        }
+      }
+    }
+
+    expect(strNode).not.toBeNull();
+    expect(strNode.tag).toBe("STR");
+    expect(strNode.elts[0]).toBe("Price: ${amount}");
+  });
+
+  it("should parse strings with mixed escape sequences", async () => {
+    // Arrange & Act
+    const result = await parser.parse(0, '"Line 1\\nTab\\t\\"Quote\\""..', basisLexicon);
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Find the STR node
+    let strNode = null;
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "STR" && node.elts[0] === 'Line 1\nTab\t"Quote"') {
+          strNode = node;
+          break;
+        }
+      }
+    }
+
+    expect(strNode).not.toBeNull();
+    expect(strNode.tag).toBe("STR");
+    expect(strNode.elts[0]).toBe('Line 1\nTab\t"Quote"');
+  });
 });

package/src/unparse-l0166.spec.js

@@ -117,6 +117,13 @@ describe("unparse with L0166 lexicon", () => {
       "length": 2,
       "arity": 2,
     },
+    "row": {
+      "tk": 1,
+      "name": "ROW",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
     "column": {
       "tk": 1,
       "name": "COLUMN",