@graffiticode/parser 0.1.5 → 0.3.0

package/CLAUDE.md

@@ -0,0 +1,72 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Development Commands
+
+### Testing
+```bash
+# Run all tests with experimental VM modules
+npm test
+
+# Run specific test files
+NODE_OPTIONS=--experimental-vm-modules jest src/parser.spec.js
+```
+
+### Linting
+```bash
+# Lint code
+npm run lint
+
+# Lint and automatically fix issues
+npm run lint:fix
+```
+
+## Architecture Overview
+
+This is the Graffiticode parser package - a core component that parses Graffiticode language syntax into ASTs (Abstract Syntax Trees).
+
+### Package Structure
+
+The parser is a workspace package within the Graffiticode monorepo. It's an ES module package (`"type": "module"`) that exports parsing functionality used by the API and language compilers.
+
+### Core Components
+
+1. **Parser Entry Point** (`src/parser.js`):
+   - `buildParser()` - Factory function that creates a parser instance with dependencies
+   - Integrates with language lexicons loaded from the API
+   - Uses Node.js VM module for sandboxed execution
+
+2. **Core Parser** (`src/parse.js`):
+   - Implements the main parsing logic with a state machine approach
+   - Handles tokenization and AST construction
+   - Includes error tracking and position coordinates
+   - Supports keywords, operators, and language-specific lexicons
+
+3. **AST Module** (`src/ast.js`):
+   - Manages AST node creation and manipulation
+   - Node pooling for memory efficiency
+   - Error node generation
+
+4. **Environment** (`src/env.js`):
+   - Manages parsing environment and scopes
+   - Handles lexicon lookups
+
+5. **Folder** (`src/folder.js`):
+   - AST transformation and folding operations
+
+## Testing Strategy
+
+- Uses Jest with experimental VM modules support
+- Test files follow `*.spec.js` pattern
+- Main test file: `src/parser.spec.js` contains comprehensive parsing tests
+
+## Monorepo Context
+
+This parser package is part of the Graffiticode monorepo:
+- Parent monorepo runs Firebase emulators for integration testing
+- API package (`../api`) depends on this parser
+- Auth packages (`../auth`, `../auth-client`) handle authentication
+- Common package (`../common`) contains shared utilities
+
+When working with the parser, be aware that it integrates tightly with the API's language loading mechanism (`../../api/src/lang/index.js`).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graffiticode/parser",
-  "version": "0.1.5",
+  "version": "0.3.0",
   "type": "module",
   "publishConfig": {
     "access": "public"
@@ -20,5 +20,8 @@
   "keywords": [],
   "author": "",
   "license": "MIT",
-  "description": ""
+  "description": "",
+  "dependencies": {
+    "@graffiticode/basis": "^1.6.2"
+  }
 }

package/src/index.js

@@ -1 +1,2 @@
 export { parser } from "./parser.js";
+export { unparse } from "./unparse.js";

package/src/parse.js

@@ -469,14 +469,27 @@ export const parse = (function () {
     });
   }
   function binding(ctx, cc) {
+    // Save the current lexeme before bindingName consumes it
+    const savedLexeme = lexeme;
+    const savedCoord = getCoord(ctx);
     return bindingName(ctx, function (ctx) {
-      eat(ctx, TK_COLON);
-      const ret = function (ctx) {
+      // Check if we have a colon for full syntax, otherwise use shorthand
+      if (match(ctx, TK_COLON)) {
+        eat(ctx, TK_COLON);
+        const ret = function (ctx) {
+          countCounter(ctx);
+          return expr(ctx, cc);
+        };
+        ret.cls = "punc";
+        return ret;
+      } else {
+        // Shorthand syntax - create a name reference for the value
+        // The binding name was already pushed as a string by bindingName
+        // Now we need to push a name reference (identifier) as the value
+        Ast.name(ctx, savedLexeme, savedCoord);
         countCounter(ctx);
-        return expr(ctx, cc);
-      };
-      ret.cls = "punc";
-      return ret;
+        return cc;
+      }
     });
   }
   function lambda(ctx, cc) {

package/src/parser.js

@@ -1,6 +1,7 @@
 import vm from "vm";
 import { getLangAsset } from "../../api/src/lang/index.js";
 import { parse } from "./parse.js";
+import { unparse } from "./unparse.js";
 
 // commonjs export
 const main = {
@@ -91,3 +92,6 @@ export const parser = buildParser({
   main,
   vm
 });
+
+// Add unparse as a property of parser
+parser.unparse = unparse;

package/src/unparse.js

@@ -0,0 +1,330 @@
+// Pretty printer that converts an AST back to source code
+import { lexicon as basisLexicon } from "@graffiticode/basis";
+
+/**
+ * Unparse an AST node to source code
+ * @param {object} node - The AST node to unparse
+ * @param {object} lexicon - The lexicon containing operator and keyword definitions
+ * @returns {string} The unparsed source code
+ */
+function unparseNode(node, lexicon) {
+  if (!node) {
+    return "";
+  }
+
+  // Handle primitive values
+  if (typeof node === "string" || typeof node === "number" || typeof node === "boolean") {
+    return String(node);
+  }
+
+  // Handle AST nodes
+  switch (node.tag) {
+    case "PROG":
+      // Program is a list of expressions ending with ".."
+      if (node.elts && node.elts.length > 0) {
+        const exprs = unparseNode(node.elts[0], lexicon);
+        return exprs + "..";
+      }
+      return "..";
+
+    case "EXPRS":
+      // Multiple expressions separated by periods
+      if (!node.elts || node.elts.length === 0) {
+        return "";
+      }
+      // Check if this looks like a function application that wasn't folded
+      // e.g., sub followed by arguments as separate expressions
+      if (node.elts.length >= 3) {
+        const first = node.elts[0];
+        // Check if first element is an identifier that could be a function
+        if (first && first.tag && first.elts && first.elts.length === 0) {
+          // This might be a function name followed by arguments
+          const funcName = first.tag;
+          // Check if this matches a lexicon function
+          if (lexicon && lexicon[funcName]) {
+            const arity = lexicon[funcName].arity || 0;
+            if (arity > 0 && node.elts.length === arity + 1) {
+              // Treat this as a function application
+              const args = node.elts.slice(1).map(elt => unparseNode(elt, lexicon)).join(" ");
+              return `${funcName} ${args}`;
+            }
+          }
+        }
+      }
+      return node.elts.map(elt => unparseNode(elt, lexicon)).join(".");
+
+    case "NUM":
+      return node.elts[0];
+
+    case "STR": {
+      // Escape quotes and backslashes in the string
+      const str = node.elts[0];
+      const escaped = str.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+      return `'${escaped}'`;
+    }
+
+    case "BOOL":
+      return node.elts[0] ? "true" : "false";
+
+    case "NULL":
+      return "null";
+
+    case "IDENT":
+      return node.elts[0];
+
+    case "LIST": {
+      // Array literal [a, b, c]
+      if (!node.elts || node.elts.length === 0) {
+        return "[]";
+      }
+      const items = node.elts.map(elt => unparseNode(elt, lexicon));
+      return "[" + items.join(", ") + "]";
+    }
+
+    case "RECORD": {
+      // Object literal {a: 1, b: 2}
+      if (!node.elts || node.elts.length === 0) {
+        return "{}";
+      }
+      const bindings = node.elts.map(elt => unparseNode(elt, lexicon));
+      return "{" + bindings.join(", ") + "}";
+    }
+
+    case "BINDING": {
+      // Key-value pair in a record
+      if (node.elts && node.elts.length >= 2) {
+        // If the key is a string node, unparse it without quotes for object keys
+        let key;
+        if (node.elts[0] && node.elts[0].tag === "STR") {
+          key = node.elts[0].elts[0]; // Get the raw string without quotes
+        } else {
+          key = unparseNode(node.elts[0], lexicon);
+        }
+        const value = unparseNode(node.elts[1], lexicon);
+        return `${key}: ${value}`;
+      }
+      return "";
+    }
+
+    case "PAREN":
+      // Parenthesized expression
+      if (node.elts && node.elts.length > 0) {
+        return "(" + unparseNode(node.elts[0], lexicon) + ")";
+      }
+      return "()";
+
+    case "APPLY":
+      // Function application
+      if (node.elts && node.elts.length >= 2) {
+        const func = unparseNode(node.elts[0], lexicon);
+        const args = unparseNode(node.elts[1], lexicon);
+        return func + " " + args;
+      }
+      return "";
+
+    case "LAMBDA":
+      // Lambda function
+      if (node.elts && node.elts.length >= 3) {
+        const params = node.elts[1];
+        const body = node.elts[2];
+
+        // Extract parameter names
+        let paramStr = "";
+        if (params && params.elts) {
+          paramStr = params.elts.map(p => unparseNode(p, lexicon)).join(" ");
+        }
+
+        // Unparse body
+        const bodyStr = unparseNode(body, lexicon);
+
+        if (paramStr) {
+          return `\\${paramStr} . ${bodyStr}`;
+        } else {
+          return `\\. ${bodyStr}`;
+        }
+      }
+      return "";
+
+    case "LET":
+      // Let binding
+      if (node.elts && node.elts.length >= 2) {
+        const bindings = node.elts[0];
+        const body = node.elts[1];
+
+        let bindingStr = "";
+        if (bindings && bindings.elts) {
+          bindingStr = bindings.elts.map(b => {
+            if (b.elts && b.elts.length >= 2) {
+              const name = unparseNode(b.elts[0], lexicon);
+              const value = unparseNode(b.elts[1], lexicon);
+              return `${name} = ${value}`;
+            }
+            return "";
+          }).filter(s => s).join(", ");
+        }
+
+        const bodyStr = unparseNode(body, lexicon);
+        return `let ${bindingStr} in ${bodyStr}`;
+      }
+      return "";
+
+    case "IF":
+      // If-then-else
+      if (node.elts && node.elts.length >= 2) {
+        const cond = unparseNode(node.elts[0], lexicon);
+        const thenExpr = unparseNode(node.elts[1], lexicon);
+
+        if (node.elts.length >= 3) {
+          const elseExpr = unparseNode(node.elts[2], lexicon);
+          return `if ${cond} then ${thenExpr} else ${elseExpr}`;
+        } else {
+          return `if ${cond} then ${thenExpr}`;
+        }
+      }
+      return "";
+
+    case "CASE":
+      // Case expression
+      if (node.elts && node.elts.length > 0) {
+        const expr = unparseNode(node.elts[0], lexicon);
+        const cases = node.elts.slice(1).map(c => unparseNode(c, lexicon));
+        return `case ${expr} of ${cases.join(" | ")}`;
+      }
+      return "";
+
+    case "OF":
+      // Case branch
+      if (node.elts && node.elts.length >= 2) {
+        const pattern = unparseNode(node.elts[0], lexicon);
+        const expr = unparseNode(node.elts[1], lexicon);
+        return `${pattern} => ${expr}`;
+      }
+      return "";
+
+    // Unary operator - negative
+    case "NEG":
+      if (node.elts && node.elts.length >= 1) {
+        const expr = unparseNode(node.elts[0], lexicon);
+        return `-${expr}`;
+      }
+      return "";
+
+    case "ERROR":
+      // Error nodes - include as comments
+      if (node.elts && node.elts.length > 0) {
+        return `/* ERROR: ${node.elts[0]} */`;
+      }
+      return "/* ERROR */";
+
+    default: {
+      // Check if this is a lexicon-defined function
+      // First, find the source name for this tag in the lexicon
+      let sourceName = null;
+      if (lexicon) {
+        for (const [key, value] of Object.entries(lexicon)) {
+          if (value && value.name === node.tag) {
+            sourceName = key;
+            break;
+          }
+        }
+      }
+
+      if (sourceName) {
+        // This is a known lexicon function - unparse in prefix notation
+        if (node.elts && node.elts.length > 0) {
+          const args = node.elts.map(elt => unparseNode(elt, lexicon)).join(" ");
+          return `${sourceName} ${args}`;
+        }
+        return sourceName;
+      }
+
+      // Handle identifiers that aren't in the lexicon (like lowercase "sub")
+      if (node.elts && node.elts.length === 0) {
+        // This is likely an identifier
+        return node.tag;
+      }
+
+      // Fallback for unknown nodes
+      console.warn(`Unknown node tag: ${node.tag}`);
+      return `/* ${node.tag} */`;
+    }
+  }
+}
+
+/**
+ * Unparse an AST pool (as returned by the parser) to source code
+ * @param {object} ast - The AST pool with a root property
+ * @param {object} dialectLexicon - The dialect-specific lexicon (optional)
+ * @returns {string} The unparsed source code
+ */
+export function unparse(ast, dialectLexicon = {}) {
+  if (!ast || !ast.root) {
+    return "";
+  }
+
+  // Merge basis lexicon with dialect lexicon (dialect takes precedence)
+  const mergedLexicon = { ...basisLexicon, ...dialectLexicon };
+
+  // The AST is in pool format - reconstruct the tree from the root
+  const rootId = ast.root;
+  const rootNode = reconstructNode(ast, rootId);
+
+  return unparseNode(rootNode, mergedLexicon);
+}
+
+/**
+ * Reconstruct a node from the AST pool format
+ * @param {object} pool - The AST pool
+ * @param {string|number} nodeId - The node ID to reconstruct
+ * @returns {object} The reconstructed node
+ */
+function reconstructNode(pool, nodeId) {
+  if (!nodeId || nodeId === "0" || nodeId === 0) {
+    return null;
+  }
+
+  const node = pool[nodeId];
+  if (!node) {
+    return null;
+  }
+
+  // Create a new node with the same structure
+  const result = {
+    tag: node.tag,
+    elts: []
+  };
+
+  // Handle different node types
+  switch (node.tag) {
+    case "NUM":
+    case "STR":
+    case "IDENT":
+    case "BOOL":
+      // These nodes have primitive values in elts[0]
+      result.elts = [node.elts[0]];
+      break;
+
+    case "NULL":
+      // NULL nodes have no elements
+      result.elts = [];
+      break;
+
+    default:
+      // For all other nodes, recursively reconstruct child nodes
+      if (node.elts && Array.isArray(node.elts)) {
+        result.elts = node.elts.map(eltId => {
+          // Check if this is a node ID (number or string number)
+          if (typeof eltId === "number" || (typeof eltId === "string" && /^\d+$/.test(eltId))) {
+            // This is a reference to another node in the pool
+            return reconstructNode(pool, eltId);
+          } else {
+            // This is a primitive value
+            return eltId;
+          }
+        });
+      }
+      break;
+  }
+
+  return result;
+}

package/src/unparse.spec.js

@@ -0,0 +1,291 @@
+import { parser } from "./parser.js";
+import { unparse } from "./unparse.js";
+
+describe("unparse", () => {
+  // Helper function to test round-trip parsing
+  async function testRoundTrip(source, lexicon = {}) {
+    const ast = await parser.parse(0, source);
+    const unparsed = unparse(ast, lexicon);
+    return unparsed;
+  }
+
+  describe("literals", () => {
+    it("should unparse string literals", async () => {
+      const source = "'hello, world'..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("'hello, world'..");
+    });
+
+    it("should unparse string literals with escaped quotes", async () => {
+      const source = "'it\\'s working'..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("'it\\'s working'..");
+    });
+
+    it("should unparse numeric literals", async () => {
+      const source = "42..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("42..");
+    });
+
+    it("should unparse negative numbers", async () => {
+      const source = "-42..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("-42..");
+    });
+
+    it("should unparse decimal numbers", async () => {
+      const source = "3.14159..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("3.14159..");
+    });
+
+    it("should unparse boolean true", async () => {
+      const source = "true..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("true..");
+    });
+
+    it("should unparse boolean false", async () => {
+      const source = "false..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("false..");
+    });
+
+    it("should unparse null", async () => {
+      const source = "null..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("null..");
+    });
+  });
+
+  describe("data structures", () => {
+    it("should unparse empty list", async () => {
+      const source = "[]..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("[]..");
+    });
+
+    it("should unparse list with single element", async () => {
+      const source = "[42]..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("[42]..");
+    });
+
+    it("should unparse list with multiple elements", async () => {
+      const source = "[1, 2, 3]..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("[1, 2, 3]..");
+    });
+
+    it("should unparse nested lists", async () => {
+      const source = "[[1, 2], [3, 4]]..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("[[1, 2], [3, 4]]..");
+    });
+
+    it("should unparse empty record", async () => {
+      const source = "{}..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("{}..");
+    });
+
+    it("should unparse record with single field", async () => {
+      const source = "{x: 10}..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("{x: 10}..");
+    });
+
+    it("should unparse record with multiple fields", async () => {
+      const source = "{x: 10, y: 20}..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("{x: 10, y: 20}..");
+    });
+
+    it("should unparse nested records", async () => {
+      const source = "{a: {b: 1}, c: 2}..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("{a: {b: 1}, c: 2}..");
+    });
+  });
+
+  describe("expressions", () => {
+    it("should unparse parenthesized expression", async () => {
+      const source = "(42)..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("(42)..");
+    });
+
+    it("should unparse addition", async () => {
+      const source = "1 + 2..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("1 + 2..");
+    });
+
+    it("should unparse subtraction", async () => {
+      const source = "10 - 5..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("10 - 5..");
+    });
+
+    it("should unparse multiplication", async () => {
+      const source = "3 * 4..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("3 * 4..");
+    });
+
+    it("should unparse division", async () => {
+      const source = "10 / 2..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("10 / 2..");
+    });
+
+    it("should unparse modulo", async () => {
+      const source = "10 % 3..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("10 % 3..");
+    });
+
+    it("should unparse power", async () => {
+      const source = "2 ^ 3..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("2 ^ 3..");
+    });
+
+    it("should unparse string concatenation", async () => {
+      const source = "'hello' ++ ' world'..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("'hello' ++ ' world'..");
+    });
+
+    it("should unparse complex arithmetic expression", async () => {
+      const source = "(1 + 2) * 3..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("(1 + 2) * 3..");
+    });
+  });
+
+  describe("multiple expressions", () => {
+    it("should unparse multiple expressions separated by periods", async () => {
+      const source = "1.2.3..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("1.2.3..");
+    });
+
+    it("should unparse mixed expressions", async () => {
+      const source = "'hello'.[1, 2].{x: 10}..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("'hello'.[1, 2].{x: 10}..");
+    });
+  });
+
+  describe("identifiers and function calls", () => {
+    it("should unparse identifier", async () => {
+      const source = "foo..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("foo..");
+    });
+
+    it("should unparse function application", async () => {
+      const source = "foo 42..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("foo 42..");
+    });
+
+    it("should unparse function with multiple arguments", async () => {
+      const source = "foo [1, 2, 3]..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("foo [1, 2, 3]..");
+    });
+
+    it("should unparse nested function applications", async () => {
+      const source = "foo (bar 42)..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("foo (bar 42)..");
+    });
+  });
+
+  describe("control flow", () => {
+    it("should unparse if-then expression", async () => {
+      const source = "if true then 1..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("if true then 1..");
+    });
+
+    it("should unparse if-then-else expression", async () => {
+      const source = "if true then 1 else 2..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("if true then 1 else 2..");
+    });
+
+    it("should unparse nested if expressions", async () => {
+      const source = "if true then (if false then 1 else 2) else 3..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("if true then (if false then 1 else 2) else 3..");
+    });
+  });
+
+  describe("lambda expressions", () => {
+    it("should unparse lambda with no parameters", async () => {
+      const source = "\\. 42..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("\\. 42..");
+    });
+
+    it("should unparse lambda with one parameter", async () => {
+      const source = "\\x . x + 1..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("\\x . x + 1..");
+    });
+
+    it("should unparse lambda with multiple parameters", async () => {
+      const source = "\\x y . x + y..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("\\x y . x + y..");
+    });
+
+    it("should unparse lambda application", async () => {
+      const source = "(\\x . x + 1) 5..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("(\\x . x + 1) 5..");
+    });
+  });
+
+  describe("let bindings", () => {
+    it("should unparse let with single binding", async () => {
+      const source = "let x = 10 in x + 1..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("let x = 10 in x + 1..");
+    });
+
+    it("should unparse let with multiple bindings", async () => {
+      const source = "let x = 10, y = 20 in x + y..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("let x = 10, y = 20 in x + y..");
+    });
+
+    it("should unparse nested let bindings", async () => {
+      const source = "let x = 10 in (let y = 20 in x + y)..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("let x = 10 in (let y = 20 in x + y)..");
+    });
+  });
+
+  describe("edge cases", () => {
+    it("should handle empty program", async () => {
+      const source = "..";
+      const unparsed = await testRoundTrip(source);
+      expect(unparsed).toBe("..");
+    });
+
+    it("should handle null AST", () => {
+      const unparsed = unparse(null);
+      expect(unparsed).toBe("");
+    });
+
+    it("should handle AST without root", () => {
+      const unparsed = unparse({});
+      expect(unparsed).toBe("");
+    });
+  });
+});