@graffiticode/parser 0.1.1 → 0.1.3

package/jest.config.js

@@ -0,0 +1,5 @@
+export default {
+  testEnvironment: "node",
+  forceExit: true,
+  verbose: true,
+};

package/jest.config.js~

@@ -0,0 +1,6 @@
+export default {
+  testEnvironment: "node",
+  forceExit: true,
+  // Set to true to see all test details
+  verbose: false
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graffiticode/parser",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "type": "module",
   "publishConfig": {
     "access": "public"
@@ -8,18 +8,17 @@
   "main": "./src/index.js",
   "devDependencies": {
     "eslint": "^8.38.0",
-    "jest": "^29.5.0",
+    "jest": "^29.7.0",
     "supertest": "^6.3.3"
   },
   "scripts": {
     "lint": "eslint src/",
     "lint:fix": "eslint --fix src/",
-    "test": "NODE_OPTIONS=--experimental-vm-modules jest"
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest",
+    "console-test": "node console-output-test.js"
   },
   "keywords": [],
   "author": "",
-  "license": "ISC",
-  "description": "",
-  "dependencies": {
-  }
+  "license": "MIT",
+  "description": ""
 }

package/src/ast.js

@@ -20,23 +20,23 @@ function nodeToJSON(n) {
   let obj;
   if (typeof n === "object") {
     switch (n.tag) {
-    case "num":
-      obj = n.elts[0];
-      break;
-    case "str":
-      obj = n.elts[0];
-      break;
-    default:
-      obj = {};
-      obj.tag = n.tag;
-      obj.elts = [];
-      if (n.coord) {
-        obj.coord = n.coord;
-      }
-      for (let i = 0; i < n.elts.length; i++) {
-        obj.elts[i] = nodeToJSON(n.elts[i]);
-      }
-      break;
+      case "num":
+        obj = n.elts[0];
+        break;
+      case "str":
+        obj = n.elts[0];
+        break;
+      default:
+        obj = {};
+        obj.tag = n.tag;
+        obj.elts = [];
+        if (n.coord) {
+          obj.coord = n.coord;
+        }
+        for (let i = 0; i < n.elts.length; i++) {
+          obj.elts[i] = nodeToJSON(n.elts[i]);
+        }
+        break;
     }
   } else if (typeof n === "string") {
     obj = n;
@@ -148,13 +148,27 @@ export class Ast {
 
   static poolToJSON(ctx) {
     const nodePool = ctx.state.nodePool;
-    const obj = {};
-    for (let i = 1; i < nodePool.length; i++) {
-      const n = nodePool[i];
-      obj[i] = nodeToJSON(n);
-    }
-    obj.root = (nodePool.length - 1);
-    return obj;
+    const rootId = nodePool.length - 1;
+    // Find all nodes transitively referenced from root
+    const referencedNodes = new Set();
+    const collectReferences = (nodeId) => {
+      if (!nodeId || referencedNodes.has(nodeId)) return;
+      referencedNodes.add(nodeId);
+
+      const node = nodePool[nodeId];
+      if (node && node.elts) {
+        node.elts.forEach(elt => {
+          if (typeof elt === "number") collectReferences(elt);
+        });
+      }
+    };
+    collectReferences(rootId);
+    return {
+      ...Object.fromEntries(
+        Array.from(referencedNodes).map(id => [id, nodeToJSON(nodePool[id])])
+      ),
+      root: rootId
+    };
   }
 
   static dump(n) {
@@ -666,7 +680,7 @@ export class Ast {
       });
       nids.push(word.nid || 0);
     }
-    const pattern = env.pattern;
+    // const pattern = env.pattern;
     Ast.push(ctx, {
       tag: "LAMBDA",
       elts: [{
@@ -674,7 +688,9 @@ export class Ast {
         elts: names
       }, nid, {
         tag: "LIST",
-        elts: pattern
+        elts: [
+          // pattern   // FIXME
+        ],
       }, {
         tag: "LIST",
         elts: nids

package/src/env.js

@@ -1,5 +1,9 @@
 // env
 
+function topEnv(ctx) {
+  return ctx.state.env[ctx.state.env.length - 1];
+}
+
 export class Env {
   static findWord(ctx, lexeme) {
     const env = ctx.state.env;
@@ -13,12 +17,12 @@ export class Env {
   }
 
   static addWord(ctx, lexeme, entry) {
-    window.gcexports.topEnv(ctx).lexicon[lexeme] = entry;
+    topEnv(ctx).lexicon[lexeme] = entry;
     return null;
   }
 
   static addPattern(ctx, pattern) {
-    window.gcexports.topEnv(ctx).pattern.push(pattern);
+    topEnv(ctx).pattern.push(pattern);
   }
 
   static enterEnv(ctx, name) {
@@ -31,7 +35,7 @@ export class Env {
       // return;  // just stop recursing
       throw new Error("runaway recursion");
     }
-    window.gcexports.topEnv(ctx).paramc = ctx.state.paramc;
+    topEnv(ctx).paramc = ctx.state.paramc;
     ctx.state.env.push({
       name,
       lexicon: {},
@@ -41,6 +45,6 @@ export class Env {
 
   static exitEnv(ctx) {
     ctx.state.env.pop();
-    ctx.state.paramc = window.gcexports.topEnv(ctx).paramc;
+    ctx.state.paramc = topEnv(ctx).paramc;
   }
 }

package/src/folder.js

@@ -203,11 +203,16 @@ export class Folder {
         const elts = [];
         for (let i = 0; i < word.length; i++) {
           const elt = Ast.pop(ctx);
-          assertErr(ctx, elt, `Too few arguments for ${word.name}. Expected ${word.length}.`, node.coord);
+          assertErr(
+            ctx,
+            elt,
+            `Too few arguments for ${word.name}. Expected ${word.length}.`,
+            node.coord
+          );
           elts.push(elt);
         }
         if (word.nid) {
-          Ast.fold(ctx, word, elts);
+          Ast.foldApply(ctx, word, elts);
         } else {
           Ast.push(ctx, {
             tag: word.name,
@@ -220,8 +225,11 @@ export class Folder {
         assert(false);
       }
     } else {
-      // assert(false, "unresolved ident "+name);
-      Ast.push(ctx, node);
+      // Tag value.
+      Ast.push(ctx, {
+        tag: name,
+        elts: [],
+      });
     }
   }
 

package/src/folder.spec.js

@@ -0,0 +1,42 @@
+import { Folder } from "./folder.js";
+import { Ast } from "./ast.js";
+
+describe("folder", () => {
+  it("should fold 'add 2 3' to 5", () => {
+    // Arrange
+    const ctx = {
+      state: {
+        nodePool: ["unused"],
+        nodeStack: [],
+        nodeStackStack: [],
+        nodeMap: {},
+        env: [{ name: "global", lexicon: {} }]
+      }
+    };
+
+    // Create nodes for "add 2 3"
+    const n1Id = Ast.intern(ctx, {
+      tag: "NUM",
+      elts: ["2"]
+    });
+
+    const n2Id = Ast.intern(ctx, {
+      tag: "NUM",
+      elts: ["3"]
+    });
+
+    const addNodeId = Ast.intern(ctx, {
+      tag: "ADD",
+      elts: [n1Id, n2Id]
+    });
+
+    // Act
+    Folder.fold(ctx, addNodeId);
+    const resultId = ctx.state.nodeStack.pop();
+    const resultNode = ctx.state.nodePool[resultId];
+
+    // Assert
+    expect(resultNode.tag).toBe("NUM");
+    expect(resultNode.elts[0]).toBe("5");
+  });
+});

package/src/parse.js

@@ -239,7 +239,7 @@ export const parse = (function () {
       scanTime += (t1 - t0);
       ctx.state.nextToken = tk;
       const to = getPos(ctx);
-      ctx.state.nextTokenCoord = {from, to};
+      ctx.state.nextTokenCoord = { from, to };
     } else {
       tk = nextToken;
     }
@@ -266,7 +266,7 @@ export const parse = (function () {
   function number(ctx, cc) {
     eat(ctx, TK_NUM);
     cc.cls = "number";
-    Ast.number(ctx, lexeme, getCoord(ctx));
+    Ast.number(ctx, lexeme);
     return cc;
   }
 
@@ -281,28 +281,24 @@ export const parse = (function () {
   */
 
   function str(ctx, cc) {
-    let coord;
     if (match(ctx, TK_STR)) {
       eat(ctx, TK_STR);
-      coord = getCoord(ctx);
-      Ast.string(ctx, lexeme, coord); // strip quotes;
+      Ast.string(ctx, lexeme); // strip quotes;
       cc.cls = "string";
       return cc;
     } else if (match(ctx, TK_STRPREFIX)) {
       ctx.state.inStr++;
       eat(ctx, TK_STRPREFIX);
       startCounter(ctx);
-      coord = getCoord(ctx);
-      Ast.string(ctx, lexeme, coord); // strip quotes;
+      Ast.string(ctx, lexeme); // strip quotes;
       countCounter(ctx);
       const ret = function (ctx) {
         return strSuffix(ctx, function (ctx) {
           ctx.state.inStr--;
           eat(ctx, TK_STRSUFFIX);
-          const coord = getCoord(ctx);
-          Ast.string(ctx, lexeme, coord); // strip quotes;
+          Ast.string(ctx, lexeme); // strip quotes;
           countCounter(ctx);
-          Ast.list(ctx, ctx.state.exprc, getCoord(ctx));
+          Ast.list(ctx, ctx.state.exprc);
           stopCounter(ctx);
           Ast.concat(ctx);
           cc.cls = "string";
@@ -324,8 +320,7 @@ export const parse = (function () {
       if (match(ctx, TK_STRMIDDLE)) {
         // Not done yet.
         eat(ctx, TK_STRMIDDLE);
-        const coord = getCoord(ctx);
-        Ast.string(ctx, lexeme, coord); // strip quotes;
+        Ast.string(ctx, lexeme); // strip quotes;
         countCounter(ctx);
         ret = function (ctx) {
           return strSuffix(ctx, resume);
@@ -355,7 +350,7 @@ export const parse = (function () {
   function bindingName(ctx, cc) {
     if (match(ctx, TK_IDENT)) {
       eat(ctx, TK_IDENT);
-      Ast.string(ctx, lexeme, getCoord(ctx));
+      Ast.string(ctx, lexeme);
       cc.cls = "variable";
       return cc;
     }
@@ -389,7 +384,7 @@ export const parse = (function () {
         offset: ctx.state.paramc,
         nid: 0
       });
-      Ast.name(ctx, lexeme, getCoord(ctx));
+      Ast.name(ctx, lexeme);
       cc.cls = "val";
       return cc;
     }
@@ -414,8 +409,10 @@ export const parse = (function () {
         }
       }
     } else {
-      cc.cls = "error";
-      Ast.error(ctx, "Name '" + lexeme + "' not found.", coord);
+      // cc.cls = "error";
+      // Ast.error(ctx, "Name '" + lexeme + "' not found.", coord);
+      // Create a tag value.
+      Ast.name(ctx, lexeme, coord);
     }
     // assert(cc, "name");
     return cc;
@@ -505,14 +502,12 @@ export const parse = (function () {
     return ret;
   }
   function list(ctx, cc) {
-    const coord = getCoord(ctx);
     eat(ctx, TK_LEFTBRACKET);
     startCounter(ctx);
     const ret = function (ctx) {
       return elements(ctx, function (ctx) {
         eat(ctx, TK_RIGHTBRACKET);
-        coord.to = getCoord(ctx).to;
-        Ast.list(ctx, ctx.state.exprc, coord);
+        Ast.list(ctx, ctx.state.exprc);
         stopCounter(ctx);
         cc.cls = "punc";
         return cc;
@@ -984,7 +979,7 @@ export const parse = (function () {
         stream.next();
       }
       if (cc && !stream.peek()) {
-        assertErr(ctx, false, "End of progam reached.", getCoord(ctx));
+        assertErr(ctx, false, "Missing program terminator.", getCoord(ctx));
       }
     } catch (x) {
       // console.log("catch() x=" + x);
@@ -1166,7 +1161,7 @@ export const parse = (function () {
           c = nextCC();
         }
       }
-      const coord = {from: getPos(ctx) - lexeme.length, to: getPos(ctx)};
+      const coord = { from: getPos(ctx) - lexeme.length, to: getPos(ctx) };
       assertErr(ctx, c !== 0, `Unterminated string: ${lexeme}`, coord);
       if (quoteChar === CC_BACKTICK && c === CC_DOLLAR &&
           peekCC() === CC_LEFTBRACE) {

package/src/parse.spec.js~

@@ -0,0 +1,119 @@
+import { jest } from "@jest/globals";
+import { parser, buildParser } from "./parser.js";
+import { mockPromiseValue, mockPromiseError } from "./testing/index.js";
+
+describe("parse operations", () => {
+  const log = jest.fn();
+  
+  it("should parse 'apply (<a b: add a b>) [10 20]'", async () => {
+    // Arrange
+    const cache = new Map();
+    const getLangAsset = mockPromiseValue("{}");
+    const main = {
+      parse: mockPromiseValue({
+        1: {
+          elts: ["a"],
+          tag: "IDENT"
+        },
+        2: {
+          elts: ["b"],
+          tag: "IDENT"
+        },
+        3: {
+          elts: [1, 2],
+          tag: "ADD"
+        },
+        4: {
+          elts: [3],
+          tag: "EXPRS"
+        },
+        5: {
+          elts: [1, 2],
+          tag: "LIST"
+        },
+        6: {
+          elts: [5, 4],
+          tag: "LAMBDA"
+        },
+        7: {
+          elts: ["10"],
+          tag: "NUM"
+        },
+        8: {
+          elts: ["20"],
+          tag: "NUM"
+        },
+        9: {
+          elts: [7, 8],
+          tag: "LIST"
+        },
+        10: {
+          elts: [6, 9],
+          tag: "APPLY"
+        },
+        11: {
+          elts: [10],
+          tag: "EXPRS"
+        },
+        12: {
+          elts: [11],
+          tag: "PROG"
+        },
+        root: 12
+      })
+    };
+    const parser = buildParser({ log, cache, getLangAsset, main });
+    const lang = "0";
+    const src = "apply (<a b: add a b>) [10 20]..";
+
+    // Act
+    const result = await parser.parse(lang, src);
+
+    // Assert
+    expect(result).toHaveProperty("root", 12);
+    expect(result[10].tag).toBe("APPLY");
+    expect(result[10].elts).toEqual([6, 9]);
+    expect(result[6].tag).toBe("LAMBDA");
+    expect(result[9].tag).toBe("LIST");
+  });
+
+  it("should parse 'hello, world!'", async () => {
+    // Arrange
+    const cache = new Map();
+    const getLangAsset = mockPromiseValue("{}");
+    const main = {
+      parse: mockPromiseValue({
+        1: {
+          elts: [
+            "hello, world"
+          ],
+          tag: "STR"
+        },
+        2: {
+          elts: [
+            1
+          ],
+          tag: "EXPRS"
+        },
+        3: {
+          elts: [
+            2
+          ],
+          tag: "PROG"
+        },
+        root: 3
+      })
+    };
+    const parser = buildParser({ log, cache, getLangAsset, main });
+    const lang = "0";
+    const src = "'hello, world'..";
+
+    // Act
+    const result = await parser.parse(lang, src);
+
+    // Assert
+    expect(result).toHaveProperty("root", 3);
+    expect(result[1].tag).toBe("STR");
+    expect(result[1].elts).toEqual(["hello, world"]);
+  });
+});

package/src/parser.js

@@ -35,10 +35,6 @@ const main = {
     if (state.cc) {
       throw new Error("End of program reached.");
     }
-    console.log(
-      "parse()",
-      "ast=" + JSON.stringify(ast, null, 2),
-    );
     return ast;
   }
 };

package/src/parser.spec.js

@@ -1,6 +1,7 @@
 import { jest } from "@jest/globals";
-import { parser, buildParser } from "./parser.js";
+import { buildParser, parser } from "./parser.js";
 import { mockPromiseValue, mockPromiseError } from "./testing/index.js";
+import vm from "vm";
 
 describe("lang/parser", () => {
   const log = jest.fn();
@@ -135,41 +136,267 @@ describe("lang/parser", () => {
     expect(vm.createContext).toHaveBeenCalled();
     expect(vm.runInContext).toHaveBeenCalledWith(rawLexicon, expect.anything());
   });
-  it.skip("should parse 'hello, world!'", async () => {
+  it("should parse error", async () => {
+    // Arrange
+    const cache = new Map();
+    const getLangAsset = mockPromiseValue("{}");
+    const err = new Error("End of program reached.");
+    const main = { parse: mockPromiseError(err) };
+    const parser = buildParser({ log, cache, getLangAsset, main });
     const lang = "0";
-    const src = "'hello, world'..";
-    const ast = {
-      1: {
-        elts: [
-          "hello, world"
-        ],
-        tag: "STR"
-      },
-      2: {
-        elts: [
-          1
-        ],
-        tag: "EXPRS"
+    const src = "'hello, world'";
+
+    // Act & Assert
+    await expect(parser.parse(lang, src)).rejects.toBe(err);
+  });
+});
+
+describe("parser integration tests", () => {
+  // Tests using the actual parser
+  it("should parse string literals", async () => {
+    // Arrange & Act
+    const result = await parser.parse(0, "'hello, world'..");
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Find the STR node
+    let strNode = null;
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "STR" && node.elts[0] === "hello, world") {
+          strNode = node;
+          break;
+        }
+      }
+    }
+
+    expect(strNode).not.toBeNull();
+    expect(strNode.tag).toBe("STR");
+    expect(strNode.elts).toEqual(["hello, world"]);
+
+    // Program structure verification
+    const rootId = result.root;
+    const rootNode = result[rootId];
+    expect(rootNode.tag).toBe("PROG");
+  });
+
+  it("should parse numeric literals", async () => {
+    // Arrange & Act
+    const result = await parser.parse(0, "42..");
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Find the NUM node
+    let numNode = null;
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "NUM" && node.elts[0] === "42") {
+          numNode = node;
+          break;
+        }
+      }
+    }
+
+    expect(numNode).not.toBeNull();
+    expect(numNode.tag).toBe("NUM");
+    expect(numNode.elts).toEqual(["42"]);
+  });
+
+  it("should have a PROG node at the root", async () => {
+    // Let's test the most basic structure that should always work
+    const result = await parser.parse(0, "123..");
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Verify the structure: we need to have a PROG node at the root
+    const rootId = result.root;
+    const rootNode = result[rootId];
+    expect(rootNode.tag).toBe("PROG");
+
+    // Find the NUM node for 123
+    let numNode = null;
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "NUM" && node.elts[0] === "123") {
+          numNode = node;
+          break;
+        }
+      }
+    }
+
+    expect(numNode).not.toBeNull();
+    expect(numNode.tag).toBe("NUM");
+    expect(numNode.elts[0]).toBe("123");
+  });
+
+  it("should parse complex program: apply (<a b: add a b>) [10 20]..", async () => {
+    // Create parser with custom lexicon
+    const customLexiconCache = new Map();
+    customLexiconCache.set(0, {
+      add: {
+        tk: 2,
+        name: "add",
+        cls: "function",
+        length: 2
       },
-      3: {
-        elts: [
-          2
-        ],
-        tag: "PROG"
+      apply: {
+        tk: 40,
+        name: "apply",
+        cls: "function",
+        length: 2
+      }
+    });
+
+    // Use the parser with our custom cache
+    const customParser = buildParser({
+      log: console.log,
+      cache: customLexiconCache,
+      getLangAsset: async () => ({}),
+      main: {
+        parse: (src, lexicon) => {
+          return Promise.resolve(parser.parse(0, src));
+        }
       },
-      root: 3
-    };
-    await expect(parser.parse(lang, src)).resolves.toStrictEqual(ast);
+      vm
+    });
+
+    // Act
+    const result = await customParser.parse(0, "apply (<a b: add a b>) [10 20]..");
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Verify basic structure
+    const rootId = result.root;
+    const rootNode = result[rootId];
+    expect(rootNode.tag).toBe("PROG");
+
+    // Find NUM nodes with values 10 and 20
+    let num10Node = null;
+    let num20Node = null;
+
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "NUM") {
+          if (node.elts[0] === "10") {
+            num10Node = node;
+          } else if (node.elts[0] === "20") {
+            num20Node = node;
+          }
+        }
+      }
+    }
+
+    // At minimum, we should be able to find the number values
+    expect(num10Node).not.toBeNull();
+    expect(num20Node).not.toBeNull();
+
+    // Find IDENT nodes with names 'a' and 'b'
+    let identNodeA = null;
+    let identNodeB = null;
+
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "IDENT") {
+          if (node.elts[0] === "a") {
+            identNodeA = node;
+          } else if (node.elts[0] === "b") {
+            identNodeB = node;
+          }
+        }
+      }
+    }
+
+    // Check that we found the identifiers
+    expect(identNodeA).not.toBeNull();
+    expect(identNodeB).not.toBeNull();
   });
-  it.skip("should parse error", async () => {
-    const lang = "0";
-    const src = "'hello, world'";
-    await expect(parser.parse(lang, src)).rejects.toThrow("End of program reached");
+
+  it("should perform parse-time evaluation for adding two numbers", async () => {
+    // Create parser with custom lexicon that defines 'add' function
+    const customLexiconCache = new Map();
+    customLexiconCache.set(0, {
+      add: {
+        tk: 2,
+        name: "add",
+        cls: "function",
+        length: 2
+      }
+    });
+
+    // Use the parser with our custom cache
+    const customParser = buildParser({
+      log: console.log,
+      cache: customLexiconCache,
+      getLangAsset: async () => ({}),
+      main: {
+        parse: (src, lexicon) => {
+          return Promise.resolve(parser.parse(0, src));
+        }
+      },
+      vm
+    });
+
+    // Act - parse a simple addition expression
+    const result = await customParser.parse(0, "add 123 456..");
+    console.log(
+      "TEST",
+      "result=" + JSON.stringify(result, null, 2),
+    );
+
+    // Assert
+    expect(result).toHaveProperty("root");
+
+    // Verify basic structure
+    const rootId = result.root;
+    const rootNode = result[rootId];
+    expect(rootNode.tag).toBe("PROG");
+
+    // Find the result node - expecting a single NUM node with the sum
+    let resultNode = null;
+
+    // Check all nodes for the result of evaluation (123 + 456 = 579)
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "NUM" && node.elts[0] === "579") {
+          resultNode = node;
+          break;
+        }
+      }
+    }
+
+    // We should find a node with the computed value (579)
+    expect(resultNode).not.toBeNull();
+    expect(resultNode.tag).toBe("NUM");
+    expect(resultNode.elts[0]).toBe("579");
+
+    // The original numbers should not be in the final AST
+    // if parse-time evaluation is working correctly
+    let found123 = false;
+    let found456 = false;
+
+    for (const key in result) {
+      if (key !== "root") {
+        const node = result[key];
+        if (node.tag === "NUM") {
+          if (node.elts[0] === "123") found123 = true;
+          if (node.elts[0] === "456") found456 = true;
+        }
+      }
+    }
+
+    // The original operands should not be in the final AST
+    // if they were properly evaluated at parse time
+    expect(found123).toBe(false);
+    expect(found456).toBe(false);
   });
-  // it("should parse error", async () => {
-  //   const lang = "0";
-  //   const src = "'hello, world..";
-  //   const ast = {};
-  //   await expect(parser.parse(lang, src)).rejects.toThrow("End of program reached");
-  // });
 });