@graffiticode/parser 0.2.0 → 0.4.0

package/CLAUDE.md

@@ -0,0 +1,72 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Development Commands
+
+### Testing
+```bash
+# Run all tests with experimental VM modules
+npm test
+
+# Run specific test files
+NODE_OPTIONS=--experimental-vm-modules jest src/parser.spec.js
+```
+
+### Linting
+```bash
+# Lint code
+npm run lint
+
+# Lint and automatically fix issues
+npm run lint:fix
+```
+
+## Architecture Overview
+
+This is the Graffiticode parser package - a core component that parses Graffiticode language syntax into ASTs (Abstract Syntax Trees).
+
+### Package Structure
+
+The parser is a workspace package within the Graffiticode monorepo. It's an ES module package (`"type": "module"`) that exports parsing functionality used by the API and language compilers.
+
+### Core Components
+
+1. **Parser Entry Point** (`src/parser.js`):
+   - `buildParser()` - Factory function that creates a parser instance with dependencies
+   - Integrates with language lexicons loaded from the API
+   - Uses Node.js VM module for sandboxed execution
+
+2. **Core Parser** (`src/parse.js`):
+   - Implements the main parsing logic with a state machine approach
+   - Handles tokenization and AST construction
+   - Includes error tracking and position coordinates
+   - Supports keywords, operators, and language-specific lexicons
+
+3. **AST Module** (`src/ast.js`):
+   - Manages AST node creation and manipulation
+   - Node pooling for memory efficiency
+   - Error node generation
+
+4. **Environment** (`src/env.js`):
+   - Manages parsing environment and scopes
+   - Handles lexicon lookups
+
+5. **Folder** (`src/folder.js`):
+   - AST transformation and folding operations
+
+## Testing Strategy
+
+- Uses Jest with experimental VM modules support
+- Test files follow `*.spec.js` pattern
+- Main test file: `src/parser.spec.js` contains comprehensive parsing tests
+
+## Monorepo Context
+
+This parser package is part of the Graffiticode monorepo:
+- Parent monorepo runs Firebase emulators for integration testing
+- API package (`../api`) depends on this parser
+- Auth packages (`../auth`, `../auth-client`) handle authentication
+- Common package (`../common`) contains shared utilities
+
+When working with the parser, be aware that it integrates tightly with the API's language loading mechanism (`../../api/src/lang/index.js`).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graffiticode/parser",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "type": "module",
   "publishConfig": {
     "access": "public"
@@ -20,5 +20,8 @@
   "keywords": [],
   "author": "",
   "license": "MIT",
-  "description": ""
+  "description": "",
+  "dependencies": {
+    "@graffiticode/basis": "^1.6.2"
+  }
 }

package/src/parser.js

@@ -1,6 +1,7 @@
 import vm from "vm";
 import { getLangAsset } from "../../api/src/lang/index.js";
 import { parse } from "./parse.js";
+import { unparse } from "./unparse.js";
 
 // commonjs export
 const main = {
@@ -47,7 +48,13 @@ export const buildParser = ({
   vm
 }) => {
   return {
-    async parse(lang, src) {
+    async parse(lang, src, lexicon = null) {
+      // If lexicon is provided, use it directly
+      if (lexicon) {
+        return await main.parse(src, lexicon);
+      }
+
+      // Otherwise, load from cache or remote
       if (!cache.has(lang)) {
         let data = await getLangAsset(lang, "/lexicon.js");
         // TODO Make lexicon JSON.
@@ -59,9 +66,9 @@ export const buildParser = ({
           throw new Error("unable to use lexicon");
         }
         const lstr = data.substring(data.indexOf("{"));
-        let lexicon;
+        let loadedLexicon;
         try {
-          lexicon = JSON.parse(lstr);
+          loadedLexicon = JSON.parse(lstr);
         } catch (err) {
           if (err instanceof SyntaxError) {
             log(`failed to parse ${lang} lexicon: ${err.message}`);
@@ -69,17 +76,17 @@ export const buildParser = ({
             vm.createContext(context);
             vm.runInContext(data, context);
             if (typeof (context.window.gcexports.globalLexicon) === "object") {
-              lexicon = context.window.gcexports.globalLexicon;
+              loadedLexicon = context.window.gcexports.globalLexicon;
             }
           }
-          if (!lexicon) {
+          if (!loadedLexicon) {
             throw new Error("Malformed lexicon");
           }
         }
-        cache.set(lang, lexicon);
+        cache.set(lang, loadedLexicon);
       };
-      const lexicon = cache.get(lang);
-      return await main.parse(src, lexicon);
+      const cachedLexicon = cache.get(lang);
+      return await main.parse(src, cachedLexicon);
     }
   };
 };
@@ -91,3 +98,12 @@ export const parser = buildParser({
   main,
   vm
 });
+
+// Add unparse as a property of parser
+parser.unparse = unparse;
+
+// Add reformat function that parses and unparses code
+parser.reformat = async function(lang, src, lexicon, options = {}) {
+  const ast = await this.parse(lang, src, lexicon);
+  return unparse(ast, lexicon, options);
+};

package/src/parser.spec.js

@@ -5,6 +5,27 @@ import vm from "vm";
 
 describe("lang/parser", () => {
   const log = jest.fn();
+  it("should use provided lexicon directly", async () => {
+    // Arrange
+    const cache = new Map();
+    const getLangAsset = jest.fn(); // Should not be called
+    const main = {
+      parse: mockPromiseValue({ root: "0" })
+    };
+    const parser = buildParser({ log, cache, getLangAsset, main });
+    const lang = "0";
+    const src = "'foo'..";
+    const providedLexicon = { test: "lexicon" };
+
+    // Act
+    await expect(parser.parse(lang, src, providedLexicon)).resolves.toStrictEqual({ root: "0" });
+
+    // Assert
+    expect(getLangAsset).not.toHaveBeenCalled(); // Should not fetch when lexicon is provided
+    expect(main.parse).toHaveBeenCalledWith(src, providedLexicon);
+    expect(cache.has(lang)).toBe(false); // Should not cache when lexicon is provided
+  });
+
   it("should call main parser language lexicon", async () => {
     // Arrange
     const cache = new Map();

package/src/unparse-l0166.spec.js

@@ -0,0 +1,361 @@
+import { parser } from "./parser.js";
+import { unparse } from "./unparse.js";
+
+describe("unparse with L0166 lexicon", () => {
+  // L0166 lexicon for spreadsheet operations (from l0166/packages/api/src/lexicon.js)
+  const l0166Lexicon = {
+    "title": {
+      "tk": 1,
+      "name": "TITLE",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "instructions": {
+      "tk": 1,
+      "name": "INSTRUCTIONS",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "params": {
+      "tk": 1,
+      "name": "PARAMS",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "cell": {
+      "tk": 1,
+      "name": "CELL",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "text": {
+      "tk": 1,
+      "name": "TEXT",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "assess": {
+      "tk": 1,
+      "name": "ASSESS",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "method": {
+      "tk": 1,
+      "name": "METHOD",
+      "cls": "function",
+      "length": 1,
+      "arity": 1,
+    },
+    "expected": {
+      "tk": 1,
+      "name": "EXPECTED",
+      "cls": "function",
+      "length": 1,
+      "arity": 1,
+    },
+    "width": {
+      "tk": 1,
+      "name": "WIDTH",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "align": {
+      "tk": 1,
+      "name": "ALIGN",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "background-color": {
+      "tk": 1,
+      "name": "BACKGROUND_COLOR",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "font-weight": {
+      "tk": 1,
+      "name": "FONT_WEIGHT",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "format": {
+      "tk": 1,
+      "name": "FORMAT",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "protected": {
+      "tk": 1,
+      "name": "PROTECTED",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "cells": {
+      "tk": 1,
+      "name": "CELLS",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "rows": {
+      "tk": 1,
+      "name": "ROWS",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "column": {
+      "tk": 1,
+      "name": "COLUMN",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    },
+    "columns": {
+      "tk": 1,
+      "name": "COLUMNS",
+      "cls": "function",
+      "length": 2,
+      "arity": 2,
+    }
+  };
+
+  it("should unparse L0166 spreadsheet code", async () => {
+    const source = `columns [
+  column A width 100 align "center" protected true {}
+]
+rows [
+  row 1 background-color "#eee" protected true {}
+]
+cells [
+  cell A1 text "A1" protected true {}
+]
+{
+  v: "0.0.1"
+}..`;
+
+    // Note: The parser may transform this code, so we test that unparse
+    // produces valid code that can be parsed again
+    // Pass the lexicon directly to avoid fetching
+
+    // For complex L0166 code, we'll just parse with language 0
+    // since the specific L0166 syntax may require special handling
+    const ast = await parser.parse(0, source);
+
+    // Log the AST pool
+    console.log("AST Pool:", JSON.stringify(ast, null, 2));
+
+    const unparsed = unparse(ast, l0166Lexicon);
+
+    // The unparsed code should be valid and parseable
+    expect(unparsed).toBeDefined();
+    expect(unparsed.endsWith("..")).toBe(true);
+
+    // Check that key elements appear in the output
+    // (the exact format may differ due to how the parser handles the syntax)
+    console.log("Original source:", source);
+    console.log("Unparsed:", unparsed);
+  });
+
+  it("should handle individual L0166 constructs", async () => {
+    const tests = [
+      {
+        source: '{v: "0.0.1"}..',
+        description: "version record"
+      },
+      {
+        source: '[]..',
+        description: "empty list"
+      },
+      {
+        source: '{}..',
+        description: "empty record"
+      },
+      {
+        source: '"A1"..',
+        description: "string literal"
+      },
+      {
+        source: '100..',
+        description: "number literal"
+      },
+      {
+        source: 'true..',
+        description: "boolean literal"
+      }
+    ];
+
+    for (const { source, description } of tests) {
+      const ast = await parser.parse(166, source, l0166Lexicon);
+      const unparsed = unparse(ast, l0166Lexicon);
+
+      // Check that unparse produces output
+      expect(unparsed).toBeDefined();
+      expect(unparsed).not.toBe("");
+
+      // The output should end with ..
+      if (!unparsed.endsWith("..")) {
+        console.log(`${description}: "${source}" -> "${unparsed}"`);
+      }
+      expect(unparsed.endsWith("..")).toBe(true);
+    }
+  });
+
+  it("should preserve simple L0166 expressions", async () => {
+    // Test simpler L0166 expressions that should parse correctly
+    const tests = [
+      'column A {}..',
+      'row 1 {}..',
+      'cell A1 {}..',
+    ];
+
+    for (const source of tests) {
+      const ast = await parser.parse(0, source);
+      const unparsed = unparse(ast, l0166Lexicon);
+
+      // Should produce valid output
+      expect(unparsed).toBeDefined();
+      expect(unparsed.endsWith("..")).toBe(true);
+
+      console.log(`Simple L0166: "${source}" -> "${unparsed}"`);
+    }
+  });
+
+  it("should handle complex L0166 budget assessment code", async () => {
+    const source = `title "Home Budget Assessment"
+instructions \`
+- Calculate your monthly budget based on income percentages
+- Fill in the empty cells with the correct formulas
+- Ensure all expenses and savings are properly allocated
+\`
+columns [
+  column A width 150 align "left" {}
+  column B width 100 format "($#,##0)" {}
+  column C width 250 align "left" {}
+]
+cells [
+  cell A1 text "CATEGORY" font-weight "bold" {}
+  cell B1 text "AMOUNT" font-weight "bold" {}
+  cell C1 text "DETAILS" font-weight "bold" {}
+
+  cell A2 text "Income" {}
+  cell B2 text "4000" {}
+  cell C2 text "Total monthly income" {}
+
+  cell A3 text "Rent" {}
+  cell B3
+    text "",
+    assess [
+      method "value"
+      expected "1400"
+    ] {}
+  cell C3 text "35% of your total income" {}
+
+  cell A4 text "Utilities" {}
+  cell B4 text "200" {}
+  cell C4 text "Fixed expense" {}
+
+  cell A5 text "Food" {}
+  cell B5
+    text "",
+    assess [
+      method "value"
+      expected "600"
+    ] {}
+  cell C5 text "15% of your total income" {}
+
+  cell A6 text "Transportation" {}
+  cell B6
+    text "",
+    assess [
+      method "value"
+      expected "400"
+    ] {}
+  cell C6 text "10% of your total income" {}
+
+  cell A7 text "Entertainment" {}
+  cell B7 text "150" {}
+  cell C7 text "Fixed expense" {}
+
+  cell A8 text "Savings" {}
+  cell B8
+    text "",
+    assess [
+      method "value"
+      expected "800"
+    ] {}
+  cell C8 text "20% of your total income" {}
+
+  cell A9 text "Miscellaneous" {}
+  cell B9
+    text "",
+    assess [
+      method "value"
+      expected "450"
+    ] {}
+  cell C9 text "Remaining income after all other expenses" {}
+]
+{
+  v: "0.0.1"
+}..`;
+
+    // Parse with L0166 lexicon
+    const ast = await parser.parse("0166", source, l0166Lexicon);
+
+    console.log("Complex L0166 AST nodes:", Object.keys(ast).length);
+
+    const unparsed = unparse(ast, l0166Lexicon);
+
+    // The unparsed code should be valid and parseable
+    expect(unparsed).toBeDefined();
+    expect(unparsed.endsWith("..")).toBe(true);
+
+    // Check that key elements appear in the output
+    expect(unparsed).toContain("title");
+    expect(unparsed).toContain("columns");
+    expect(unparsed).toContain("cells");
+    expect(unparsed).toContain("column A");
+    expect(unparsed).toContain("column B");
+    expect(unparsed).toContain("column C");
+
+    // Log a portion of the output to see the pretty printing
+    const lines = unparsed.split("\n");
+    console.log("First 20 lines of unparsed output:");
+    console.log(lines.slice(0, 20).join("\n"));
+    console.log("...");
+    console.log("Last 10 lines of unparsed output:");
+    console.log(lines.slice(-10).join("\n"));
+    console.log(unparsed);
+  });
+
+  it("should reformat L0166 code using parser.reformat", async () => {
+    const source = `columns [column A width 100 {}] rows [row 1 {}] cells [cell A1 text "Hello" {}] {v: "0.0.1"}..`;
+
+    // Reformat with L0166 lexicon
+    const reformatted = await parser.reformat("0166", source, l0166Lexicon);
+
+    // Check that it produces valid output
+    expect(reformatted).toBeDefined();
+    expect(reformatted.endsWith("..")).toBe(true);
+
+    // Check for pretty printing
+    expect(reformatted).toContain("columns [\n");
+    expect(reformatted).toContain("rows [\n");
+    expect(reformatted).toContain("cells [\n");
+
+    console.log("Reformatted L0166 code:");
+    console.log(reformatted);
+  });
+});