@graffiticode/parser 0.4.0 → 1.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graffiticode/parser",
-  "version": "0.4.0",
+  "version": "1.0.0",
   "type": "module",
   "publishConfig": {
     "access": "public"

package/src/parser.js

@@ -1,5 +1,3 @@
-import vm from "vm";
-import { getLangAsset } from "../../api/src/lang/index.js";
 import { parse } from "./parse.js";
 import { unparse } from "./unparse.js";
 
@@ -40,63 +38,20 @@ const main = {
   }
 };
 
-export const buildParser = ({
-  log,
-  cache,
-  getLangAsset,
-  main,
-  vm
-}) => {
+export const buildParser = ({ main }) => {
   return {
-    async parse(lang, src, lexicon = null) {
-      // If lexicon is provided, use it directly
-      if (lexicon) {
-        return await main.parse(src, lexicon);
+    async parse(lang, src, lexicon) {
+      // Lexicon is now required
+      if (!lexicon) {
+        throw new Error("Lexicon is required for parsing");
       }
-
-      // Otherwise, load from cache or remote
-      if (!cache.has(lang)) {
-        let data = await getLangAsset(lang, "/lexicon.js");
-        // TODO Make lexicon JSON.
-        if (data instanceof Buffer) {
-          data = data.toString();
-        }
-        if (typeof (data) !== "string") {
-          log(`Failed to get usable lexicon for ${lang}`, typeof (data), data);
-          throw new Error("unable to use lexicon");
-        }
-        const lstr = data.substring(data.indexOf("{"));
-        let loadedLexicon;
-        try {
-          loadedLexicon = JSON.parse(lstr);
-        } catch (err) {
-          if (err instanceof SyntaxError) {
-            log(`failed to parse ${lang} lexicon: ${err.message}`);
-            const context = { window: { gcexports: {} } };
-            vm.createContext(context);
-            vm.runInContext(data, context);
-            if (typeof (context.window.gcexports.globalLexicon) === "object") {
-              loadedLexicon = context.window.gcexports.globalLexicon;
-            }
-          }
-          if (!loadedLexicon) {
-            throw new Error("Malformed lexicon");
-          }
-        }
-        cache.set(lang, loadedLexicon);
-      };
-      const cachedLexicon = cache.get(lang);
-      return await main.parse(src, cachedLexicon);
+      return await main.parse(src, lexicon);
     }
   };
 };
 
 export const parser = buildParser({
-  log: console.log,
-  cache: new Map(),
-  getLangAsset,
-  main,
-  vm
+  main
 });
 
 // Add unparse as a property of parser
@@ -104,6 +59,9 @@ parser.unparse = unparse;
 
 // Add reformat function that parses and unparses code
 parser.reformat = async function(lang, src, lexicon, options = {}) {
+  if (!lexicon) {
+    throw new Error("Lexicon is required for reformatting");
+  }
   const ast = await this.parse(lang, src, lexicon);
   return unparse(ast, lexicon, options);
 };

package/src/parser.spec.js

@@ -1,18 +1,16 @@
 import { jest } from "@jest/globals";
 import { buildParser, parser } from "./parser.js";
 import { mockPromiseValue, mockPromiseError } from "./testing/index.js";
-import vm from "vm";
+import { lexicon as basisLexicon } from "@graffiticode/basis";
 
 describe("lang/parser", () => {
   const log = jest.fn();
-  it("should use provided lexicon directly", async () => {
+  it("should use provided lexicon", async () => {
     // Arrange
-    const cache = new Map();
-    const getLangAsset = jest.fn(); // Should not be called
     const main = {
       parse: mockPromiseValue({ root: "0" })
     };
-    const parser = buildParser({ log, cache, getLangAsset, main });
+    const parser = buildParser({ main });
     const lang = "0";
     const src = "'foo'..";
     const providedLexicon = { test: "lexicon" };
@@ -21,154 +19,78 @@ describe("lang/parser", () => {
     await expect(parser.parse(lang, src, providedLexicon)).resolves.toStrictEqual({ root: "0" });
 
     // Assert
-    expect(getLangAsset).not.toHaveBeenCalled(); // Should not fetch when lexicon is provided
     expect(main.parse).toHaveBeenCalledWith(src, providedLexicon);
-    expect(cache.has(lang)).toBe(false); // Should not cache when lexicon is provided
   });
 
-  it("should call main parser language lexicon", async () => {
+  it("should throw error when lexicon is missing", async () => {
     // Arrange
-    const cache = new Map();
-    const getLangAsset = mockPromiseValue("{}");
     const main = {
       parse: mockPromiseValue({ root: "0" })
     };
-    const parser = buildParser({ log, cache, getLangAsset, main });
+    const parser = buildParser({ main });
     const lang = "0";
     const src = "'foo'..";
 
-    // Act
-    await expect(parser.parse(lang, src)).resolves.toStrictEqual({ root: "0" });
-
-    // Assert
-    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
-    expect(main.parse).toHaveBeenCalledWith(src, {});
-    expect(cache.has(lang)).toBe(true);
-    expect(cache.get(lang)).toStrictEqual({});
+    // Act & Assert
+    await expect(parser.parse(lang, src)).rejects.toThrow("Lexicon is required for parsing");
   });
-  it("should call main parser cached lexicon", async () => {
+  it("should pass lexicon to main parser", async () => {
     // Arrange
-    const cache = new Map();
     const main = {
       parse: mockPromiseValue({ root: "0" })
     };
-    const parser = buildParser({
-      cache,
-      main
-    });
+    const parser = buildParser({ main });
     const lang = "0";
     const src = "'foo'..";
-    cache.set(lang, {});
+    const lexicon = { someFunc: { name: "SOMEFUNC" } };
 
     // Act
-    await expect(parser.parse(lang, src)).resolves.toStrictEqual({ root: "0" });
+    await expect(parser.parse(lang, src, lexicon)).resolves.toStrictEqual({ root: "0" });
 
     // Assert
-    expect(main.parse).toHaveBeenCalledWith(src, {});
+    expect(main.parse).toHaveBeenCalledWith(src, lexicon);
   });
-  it("should return error if get language asset fails", async () => {
+  it("should return error if main parser fails with lexicon", async () => {
     // Arrange
-    const cache = new Map();
-    const err = new Error("failed to get lexicon");
-    const getLangAsset = mockPromiseError(err);
-    const parser = buildParser({
-      cache,
-      getLangAsset
-    });
+    const err = new Error("parser failed");
+    const main = { parse: mockPromiseError(err) };
+    const parser = buildParser({ main });
     const lang = "00";
     const src = "'foo'..";
+    const lexicon = {};
 
     // Act
-    await expect(parser.parse(lang, src)).rejects.toBe(err);
+    await expect(parser.parse(lang, src, lexicon)).rejects.toBe(err);
 
     // Assert
-    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+    expect(main.parse).toHaveBeenCalledWith(src, lexicon);
   });
   it("should return error if main parser fails", async () => {
     // Arrange
-    const log = jest.fn();
-    const cache = new Map();
-    const getLangAsset = mockPromiseValue("{}");
     const err = new Error("main parser failed");
     const main = { parse: mockPromiseError(err) };
-    const parser = buildParser({ log, cache, getLangAsset, main });
-    const lang = "0";
-    const src = "'foo'..";
-
-    // Act
-    await expect(parser.parse(lang, src)).rejects.toBe(err);
-
-    // Assert
-    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
-    expect(main.parse).toHaveBeenCalledWith(src, {});
-    expect(cache.has(lang)).toBe(true);
-    expect(cache.get(lang)).toStrictEqual({});
-  });
-  it("should return succeed if lexicon is a buffer", async () => {
-    // Arrange
-    const log = jest.fn();
-    const cache = new Map();
-    const getLangAsset = mockPromiseValue(Buffer.from("{}"));
-    const ast = { root: "0" };
-    const main = { parse: mockPromiseValue(ast) };
-    const parser = buildParser({ log, cache, getLangAsset, main });
-    const lang = "0";
-    const src = "'foo'..";
-
-    // Act
-    await expect(parser.parse(lang, src)).resolves.toStrictEqual(ast);
-
-    // Assert
-    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
-    expect(main.parse).toHaveBeenCalledWith(src, {});
-    expect(cache.has(lang)).toBe(true);
-    expect(cache.get(lang)).toStrictEqual({});
-  });
-  it("should try vm if lexicon cannot parse JSON", async () => {
-    // Arrange
-    const log = jest.fn();
-    const cache = new Map();
-    const rawLexicon = `
-    (() => {
-      window.gcexports.globalLexicon = {};
-    })();
-    `;
-    const getLangAsset = mockPromiseValue(rawLexicon);
-    const ast = { root: "0" };
-    const main = { parse: mockPromiseValue(ast) };
-    const vm = {
-      createContext: jest.fn(),
-      runInContext: jest.fn().mockImplementation((data, context) => {
-        context.window.gcexports.globalLexicon = {};
-      })
-    };
-    const parser = buildParser({ log, cache, getLangAsset, main, vm });
+    const parser = buildParser({ main });
     const lang = "0";
     const src = "'foo'..";
+    const lexicon = {};
 
     // Act
-    await expect(parser.parse(lang, src)).resolves.toStrictEqual(ast);
+    await expect(parser.parse(lang, src, lexicon)).rejects.toBe(err);
 
     // Assert
-    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
-    expect(main.parse).toHaveBeenCalledWith(src, {});
-    expect(cache.has(lang)).toBe(true);
-    expect(cache.get(lang)).toStrictEqual({});
-    expect(vm.createContext).toHaveBeenCalled();
-    expect(vm.runInContext).toHaveBeenCalledWith(rawLexicon, expect.anything());
+    expect(main.parse).toHaveBeenCalledWith(src, lexicon);
   });
   it("should parse error", async () => {
     // Arrange
-    const cache = new Map();
-    const getLangAsset = mockPromiseValue("{}");
     const err = new Error("End of program reached.");
     const main = { parse: mockPromiseError(err) };
-    const parser = buildParser({ log, cache, getLangAsset, main });
+    const parser = buildParser({ main });
     const lang = "0";
     const src = "'hello, world'";
+    const lexicon = {};
 
     // Act & Assert
-    await expect(parser.parse(lang, src)).rejects.toBe(err);
+    await expect(parser.parse(lang, src, lexicon)).rejects.toBe(err);
   });
 });
 
@@ -176,7 +98,7 @@ describe("parser integration tests", () => {
   // Tests using the actual parser
   it("should parse string literals", async () => {
     // Arrange & Act
-    const result = await parser.parse(0, "'hello, world'..");
+    const result = await parser.parse(0, "'hello, world'..", basisLexicon);
 
     // Assert
     expect(result).toHaveProperty("root");
@@ -205,7 +127,7 @@ describe("parser integration tests", () => {
 
   it("should parse numeric literals", async () => {
     // Arrange & Act
-    const result = await parser.parse(0, "42..");
+    const result = await parser.parse(0, "42..", basisLexicon);
 
     // Assert
     expect(result).toHaveProperty("root");
@@ -229,7 +151,7 @@ describe("parser integration tests", () => {
 
   it("should have a PROG node at the root", async () => {
     // Let's test the most basic structure that should always work
-    const result = await parser.parse(0, "123..");
+    const result = await parser.parse(0, "123..", basisLexicon);
 
     // Assert
     expect(result).toHaveProperty("root");
@@ -257,9 +179,8 @@ describe("parser integration tests", () => {
   });
 
   it("should parse complex program: apply (<a b: add a b>) [10 20]..", async () => {
-    // Create parser with custom lexicon
-    const customLexiconCache = new Map();
-    customLexiconCache.set(0, {
+    // Create custom lexicon
+    const customLexicon = {
       add: {
         tk: 2,
         name: "add",
@@ -272,23 +193,10 @@ describe("parser integration tests", () => {
         cls: "function",
         length: 2
       }
-    });
-
-    // Use the parser with our custom cache
-    const customParser = buildParser({
-      log: console.log,
-      cache: customLexiconCache,
-      getLangAsset: async () => ({}),
-      main: {
-        parse: (src, lexicon) => {
-          return Promise.resolve(parser.parse(0, src));
-        }
-      },
-      vm
-    });
+    };
 
     // Act
-    const result = await customParser.parse(0, "apply (<a b: add a b>) [10 20]..");
+    const result = await parser.parse(0, "apply (<a b: add a b>) [10 20]..", customLexicon);
 
     // Assert
     expect(result).toHaveProperty("root");
@@ -348,7 +256,7 @@ describe("parser integration tests", () => {
 
     try {
       // Unclosed string - missing closing quote
-      result = await parser.parse(0, "'unclosed string..");
+      result = await parser.parse(0, "'unclosed string..", basisLexicon);
     } catch (e) {
       // Check for expected error (we should now have a robust parser that doesn't throw)
       console.error("Unexpected error:", e);
@@ -388,7 +296,7 @@ describe("parser integration tests", () => {
 
     try {
       // Missing closing bracket
-      result = await parser.parse(0, "[1, 2, 3..");
+      result = await parser.parse(0, "[1, 2, 3..", basisLexicon);
     } catch (e) {
       console.error("Unexpected error:", e);
       throw e;
@@ -427,7 +335,7 @@ describe("parser integration tests", () => {
 
     try {
       // Invalid sequence of tokens
-      result = await parser.parse(0, "if then else..");
+      result = await parser.parse(0, "if then else..", basisLexicon);
     } catch (e) {
       console.error("Unexpected error:", e);
       throw e;
@@ -462,32 +370,9 @@ describe("parser integration tests", () => {
   });
 
   it("should perform parse-time evaluation for adding two numbers", async () => {
-    // Create parser with custom lexicon that defines 'add' function
-    const customLexiconCache = new Map();
-    customLexiconCache.set(0, {
-      add: {
-        tk: 2,
-        name: "add",
-        cls: "function",
-        length: 2
-      }
-    });
-
-    // Use the parser with our custom cache
-    const customParser = buildParser({
-      log: console.log,
-      cache: customLexiconCache,
-      getLangAsset: async () => ({}),
-      main: {
-        parse: (src, lexicon) => {
-          return Promise.resolve(parser.parse(0, src));
-        }
-      },
-      vm
-    });
-
+    // Use basis lexicon which includes add function
     // Act - parse a simple addition expression
-    const result = await customParser.parse(0, "add 123 456..");
+    const result = await parser.parse(0, "add 123 456..", basisLexicon);
     console.log(
       "TEST",
       "result=" + JSON.stringify(result, null, 2),

package/src/unparse-l0166.spec.js

@@ -1,5 +1,6 @@
 import { parser } from "./parser.js";
 import { unparse } from "./unparse.js";
+import { lexicon as basisLexicon } from "@graffiticode/basis";
 
 describe("unparse with L0166 lexicon", () => {
   // L0166 lexicon for spreadsheet operations (from l0166/packages/api/src/lexicon.js)
@@ -132,6 +133,9 @@ describe("unparse with L0166 lexicon", () => {
     }
   };
 
+  // Merge basis and L0166 lexicons
+  const mergedLexicon = { ...basisLexicon, ...l0166Lexicon };
+
   it("should unparse L0166 spreadsheet code", async () => {
     const source = `columns [
   column A width 100 align "center" protected true {}
@@ -150,9 +154,8 @@ cells [
     // produces valid code that can be parsed again
     // Pass the lexicon directly to avoid fetching
 
-    // For complex L0166 code, we'll just parse with language 0
-    // since the specific L0166 syntax may require special handling
-    const ast = await parser.parse(0, source);
+    // Parse with merged lexicon
+    const ast = await parser.parse(0, source, mergedLexicon);
 
     // Log the AST pool
     console.log("AST Pool:", JSON.stringify(ast, null, 2));
@@ -198,7 +201,7 @@ cells [
     ];
 
     for (const { source, description } of tests) {
-      const ast = await parser.parse(166, source, l0166Lexicon);
+      const ast = await parser.parse(166, source, mergedLexicon);
       const unparsed = unparse(ast, l0166Lexicon);
 
       // Check that unparse produces output
@@ -222,7 +225,7 @@ cells [
     ];
 
     for (const source of tests) {
-      const ast = await parser.parse(0, source);
+      const ast = await parser.parse(0, source, mergedLexicon);
       const unparsed = unparse(ast, l0166Lexicon);
 
       // Should produce valid output
@@ -311,8 +314,8 @@ cells [
   v: "0.0.1"
 }..`;
 
-    // Parse with L0166 lexicon
-    const ast = await parser.parse("0166", source, l0166Lexicon);
+    // Parse with merged lexicon
+    const ast = await parser.parse("0166", source, mergedLexicon);
 
     console.log("Complex L0166 AST nodes:", Object.keys(ast).length);
 
@@ -343,8 +346,8 @@ cells [
   it("should reformat L0166 code using parser.reformat", async () => {
     const source = `columns [column A width 100 {}] rows [row 1 {}] cells [cell A1 text "Hello" {}] {v: "0.0.1"}..`;
 
-    // Reformat with L0166 lexicon
-    const reformatted = await parser.reformat("0166", source, l0166Lexicon);
+    // Reformat with merged lexicon
+    const reformatted = await parser.reformat("0166", source, mergedLexicon);
 
     // Check that it produces valid output
     expect(reformatted).toBeDefined();

package/src/unparse.spec.js

@@ -1,11 +1,14 @@
 import { parser } from "./parser.js";
 import { unparse } from "./unparse.js";
+import { lexicon as basisLexicon } from "@graffiticode/basis";
 
 describe("unparse", () => {
   // Helper function to test round-trip parsing
-  async function testRoundTrip(source, lexicon = {}, options = { compact: true }) {
-    const ast = await parser.parse(0, source);
-    const unparsed = unparse(ast, lexicon, options);
+  async function testRoundTrip(source, dialectLexicon = {}, options = { compact: true }) {
+    // Merge basis lexicon with dialect lexicon for parsing
+    const lexicon = { ...basisLexicon, ...dialectLexicon };
+    const ast = await parser.parse(0, source, lexicon);
+    const unparsed = unparse(ast, dialectLexicon, options);
     return unparsed;
   }
 
@@ -293,13 +296,13 @@ describe("unparse", () => {
   describe("parser.reformat", () => {
     it("should reformat simple expressions", async () => {
       const source = "42..";
-      const reformatted = await parser.reformat(0, source, {});
+      const reformatted = await parser.reformat(0, source, basisLexicon);
       expect(reformatted).toBe("42..");
     });
 
     it("should reformat and pretty print lists", async () => {
       const source = "[1,2,3]..";
-      const reformatted = await parser.reformat(0, source, {});
+      const reformatted = await parser.reformat(0, source, basisLexicon);
       expect(reformatted).toContain("[\n");
       expect(reformatted).toContain("  1");
       expect(reformatted).toContain("  2");
@@ -324,7 +327,7 @@ describe("unparse", () => {
 
     it("should reformat multiple expressions", async () => {
       const source = "'hello'.[1, 2].{x: 10}..";
-      const reformatted = await parser.reformat(0, source, {});
+      const reformatted = await parser.reformat(0, source, basisLexicon);
       expect(reformatted).toContain("'hello'");
       expect(reformatted).toContain("[\n  1");
       expect(reformatted).toContain("{\n  x: 10");
@@ -333,13 +336,13 @@ describe("unparse", () => {
 
     it("should support compact option", async () => {
       const source = "[1, 2, 3]..";
-      const reformatted = await parser.reformat(0, source, {}, { compact: true });
+      const reformatted = await parser.reformat(0, source, basisLexicon, { compact: true });
       expect(reformatted).toBe("[1, 2, 3]..");
     });
 
     it("should support custom indent size", async () => {
       const source = "[1, 2]..";
-      const reformatted = await parser.reformat(0, source, {}, { indentSize: 4 });
+      const reformatted = await parser.reformat(0, source, basisLexicon, { indentSize: 4 });
       expect(reformatted).toContain("    1"); // 4 spaces
       expect(reformatted).toContain("    2"); // 4 spaces
     });