@graffiticode/parser 0.1.0

package/src/parser.js

@@ -0,0 +1,97 @@
+import vm from "vm";
+import { getLangAsset } from "../../api/src/lang/index.js";
+import { parse } from "./parse.js";
+
+// commonjs export
+const main = {
+  parse(src, lexicon) {
+    const stream = new parse.StringStream(src);
+    const state = {
+      cc: parse.program, // top level parsing function
+      argc: 0,
+      argcStack: [0],
+      paramc: 0,
+      paramcStack: [0],
+      env: [{ name: "global", lexicon }],
+      exprc: 0,
+      exprcStack: [0],
+      nodeStack: [],
+      nodeStackStack: [],
+      nodePool: ["unused"],
+      nodeMap: {},
+      nextToken: -1,
+      errors: [],
+      coords: [],
+      inStr: 0,
+      quoteCharStack: []
+    };
+    const next = function () {
+      return parse.parse(stream, state);
+    };
+    let ast;
+    while (state.cc !== null && stream.peek()) {
+      ast = next();
+    }
+    if (state.cc) {
+      throw new Error("End of program reached.");
+    }
+    console.log(
+      "parse()",
+      "ast=" + JSON.stringify(ast, null, 2),
+    );
+    return ast;
+  }
+};
+
+export const buildParser = ({
+  log,
+  cache,
+  getLangAsset,
+  main,
+  vm
+}) => {
+  return {
+    async parse(lang, src) {
+      if (!cache.has(lang)) {
+        let data = await getLangAsset(lang, "/lexicon.js");
+        // TODO Make lexicon JSON.
+        if (data instanceof Buffer) {
+          data = data.toString();
+        }
+        if (typeof (data) !== "string") {
+          log(`Failed to get usable lexicon for ${lang}`, typeof (data), data);
+          throw new Error("unable to use lexicon");
+        }
+        const lstr = data.substring(data.indexOf("{"));
+        let lexicon;
+        try {
+          lexicon = JSON.parse(lstr);
+        } catch (err) {
+          if (err instanceof SyntaxError) {
+            log(`failed to parse ${lang} lexicon: ${err.message}`);
+            const context = { window: { gcexports: {} } };
+            vm.createContext(context);
+            vm.runInContext(data, context);
+            if (typeof (context.window.gcexports.globalLexicon) === "object") {
+              lexicon = context.window.gcexports.globalLexicon;
+            }
+          }
+          if (!lexicon) {
+            throw new Error("Malformed lexicon");
+          }
+        }
+        cache.set(lang, lexicon);
+      };
+      const lexicon = cache.get(lang);
+      return await main.parse(src, lexicon);
+    }
+  };
+};
+
+export const parser = buildParser({
+  log: console.log,
+  cache: new Map(),
+  getLangAsset,
+  main,
+  vm
+});

package/src/parser.js~

@@ -0,0 +1,97 @@
+import vm from "vm";
+import { getLangAsset } from "../../../api/src/lang/index.js";
+import { parse } from "./parse.js";
+
+// commonjs export
+const main = {
+  parse(src, lexicon) {
+    const stream = new parse.StringStream(src);
+    const state = {
+      cc: parse.program, // top level parsing function
+      argc: 0,
+      argcStack: [0],
+      paramc: 0,
+      paramcStack: [0],
+      env: [{ name: "global", lexicon }],
+      exprc: 0,
+      exprcStack: [0],
+      nodeStack: [],
+      nodeStackStack: [],
+      nodePool: ["unused"],
+      nodeMap: {},
+      nextToken: -1,
+      errors: [],
+      coords: [],
+      inStr: 0,
+      quoteCharStack: []
+    };
+    const next = function () {
+      return parse.parse(stream, state);
+    };
+    let ast;
+    while (state.cc !== null && stream.peek()) {
+      ast = next();
+    }
+    if (state.cc) {
+      throw new Error("End of program reached.");
+    }
+    console.log(
+      "parse()",
+      "ast=" + JSON.stringify(ast, null, 2),
+    );
+    return ast;
+  }
+};
+
+export const buildParser = ({
+  log,
+  cache,
+  getLangAsset,
+  main,
+  vm
+}) => {
+  return {
+    async parse(lang, src) {
+      if (!cache.has(lang)) {
+        let data = await getLangAsset(lang, "/lexicon.js");
+        // TODO Make lexicon JSON.
+        if (data instanceof Buffer) {
+          data = data.toString();
+        }
+        if (typeof (data) !== "string") {
+          log(`Failed to get usable lexicon for ${lang}`, typeof (data), data);
+          throw new Error("unable to use lexicon");
+        }
+        const lstr = data.substring(data.indexOf("{"));
+        let lexicon;
+        try {
+          lexicon = JSON.parse(lstr);
+        } catch (err) {
+          if (err instanceof SyntaxError) {
+            log(`failed to parse ${lang} lexicon: ${err.message}`);
+            const context = { window: { gcexports: {} } };
+            vm.createContext(context);
+            vm.runInContext(data, context);
+            if (typeof (context.window.gcexports.globalLexicon) === "object") {
+              lexicon = context.window.gcexports.globalLexicon;
+            }
+          }
+          if (!lexicon) {
+            throw new Error("Malformed lexicon");
+          }
+        }
+        cache.set(lang, lexicon);
+      };
+      const lexicon = cache.get(lang);
+      return await main.parse(src, lexicon);
+    }
+  };
+};
+
+export const parser = buildParser({
+  log: console.log,
+  cache: new Map(),
+  getLangAsset,
+  main,
+  vm
+});

package/src/parser.spec.js

@@ -0,0 +1,175 @@
+import { jest } from "@jest/globals";
+import { parser, buildParser } from "./parser.js";
+import { mockPromiseValue, mockPromiseError } from "./testing/index.js";
+
+describe("lang/parser", () => {
+  const log = jest.fn();
+  it("should call main parser language lexicon", async () => {
+    // Arrange
+    const cache = new Map();
+    const getLangAsset = mockPromiseValue("{}");
+    const main = {
+      parse: mockPromiseValue({ root: "0" })
+    };
+    const parser = buildParser({ log, cache, getLangAsset, main });
+    const lang = "0";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual({ root: "0" });
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+    expect(cache.has(lang)).toBe(true);
+    expect(cache.get(lang)).toStrictEqual({});
+  });
+  it("should call main parser cached lexicon", async () => {
+    // Arrange
+    const cache = new Map();
+    const main = {
+      parse: mockPromiseValue({ root: "0" })
+    };
+    const parser = buildParser({
+      cache,
+      main
+    });
+    const lang = "0";
+    const src = "'foo'..";
+    cache.set(lang, {});
+
+    // Act
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual({ root: "0" });
+
+    // Assert
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+  });
+  it("should return error if get language asset fails", async () => {
+    // Arrange
+    const cache = new Map();
+    const err = new Error("failed to get lexicon");
+    const getLangAsset = mockPromiseError(err);
+    const parser = buildParser({
+      cache,
+      getLangAsset
+    });
+    const lang = "00";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).rejects.toBe(err);
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+  });
+  it("should return error if main parser fails", async () => {
+    // Arrange
+    const log = jest.fn();
+    const cache = new Map();
+    const getLangAsset = mockPromiseValue("{}");
+    const err = new Error("main parser failed");
+    const main = { parse: mockPromiseError(err) };
+    const parser = buildParser({ log, cache, getLangAsset, main });
+    const lang = "0";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).rejects.toBe(err);
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+    expect(cache.has(lang)).toBe(true);
+    expect(cache.get(lang)).toStrictEqual({});
+  });
+  it("should return succeed if lexicon is a buffer", async () => {
+    // Arrange
+    const log = jest.fn();
+    const cache = new Map();
+    const getLangAsset = mockPromiseValue(Buffer.from("{}"));
+    const ast = { root: "0" };
+    const main = { parse: mockPromiseValue(ast) };
+    const parser = buildParser({ log, cache, getLangAsset, main });
+    const lang = "0";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual(ast);
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+    expect(cache.has(lang)).toBe(true);
+    expect(cache.get(lang)).toStrictEqual({});
+  });
+  it("should try vm if lexicon cannot parse JSON", async () => {
+    // Arrange
+    const log = jest.fn();
+    const cache = new Map();
+    const rawLexicon = `
+    (() => {
+      window.gcexports.globalLexicon = {};
+    })();
+    `;
+    const getLangAsset = mockPromiseValue(rawLexicon);
+    const ast = { root: "0" };
+    const main = { parse: mockPromiseValue(ast) };
+    const vm = {
+      createContext: jest.fn(),
+      runInContext: jest.fn().mockImplementation((data, context) => {
+        context.window.gcexports.globalLexicon = {};
+      })
+    };
+    const parser = buildParser({ log, cache, getLangAsset, main, vm });
+    const lang = "0";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual(ast);
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+    expect(cache.has(lang)).toBe(true);
+    expect(cache.get(lang)).toStrictEqual({});
+    expect(vm.createContext).toHaveBeenCalled();
+    expect(vm.runInContext).toHaveBeenCalledWith(rawLexicon, expect.anything());
+  });
+  it.skip("should parse 'hello, world!'", async () => {
+    const lang = "0";
+    const src = "'hello, world'..";
+    const ast = {
+      1: {
+        elts: [
+          "hello, world"
+        ],
+        tag: "STR"
+      },
+      2: {
+        elts: [
+          1
+        ],
+        tag: "EXPRS"
+      },
+      3: {
+        elts: [
+          2
+        ],
+        tag: "PROG"
+      },
+      root: 3
+    };
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual(ast);
+  });
+  it.skip("should parse error", async () => {
+    const lang = "0";
+    const src = "'hello, world'";
+    await expect(parser.parse(lang, src)).rejects.toThrow("End of program reached");
+  });
+  // it("should parse error", async () => {
+  //   const lang = "0";
+  //   const src = "'hello, world..";
+  //   const ast = {};
+  //   await expect(parser.parse(lang, src)).rejects.toThrow("End of program reached");
+  // });
+});

package/src/parser.spec.js~

@@ -0,0 +1,175 @@
+import { jest } from "@jest/globals";
+import { parser, buildParser } from "./parser.js";
+import { mockPromiseValue, mockPromiseError } from "../testing/index.js";
+
+describe("lang/parser", () => {
+  const log = jest.fn();
+  it("should call main parser language lexicon", async () => {
+    // Arrange
+    const cache = new Map();
+    const getLangAsset = mockPromiseValue("{}");
+    const main = {
+      parse: mockPromiseValue({ root: "0" })
+    };
+    const parser = buildParser({ log, cache, getLangAsset, main });
+    const lang = "0";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual({ root: "0" });
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+    expect(cache.has(lang)).toBe(true);
+    expect(cache.get(lang)).toStrictEqual({});
+  });
+  it("should call main parser cached lexicon", async () => {
+    // Arrange
+    const cache = new Map();
+    const main = {
+      parse: mockPromiseValue({ root: "0" })
+    };
+    const parser = buildParser({
+      cache,
+      main
+    });
+    const lang = "0";
+    const src = "'foo'..";
+    cache.set(lang, {});
+
+    // Act
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual({ root: "0" });
+
+    // Assert
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+  });
+  it("should return error if get language asset fails", async () => {
+    // Arrange
+    const cache = new Map();
+    const err = new Error("failed to get lexicon");
+    const getLangAsset = mockPromiseError(err);
+    const parser = buildParser({
+      cache,
+      getLangAsset
+    });
+    const lang = "00";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).rejects.toBe(err);
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+  });
+  it("should return error if main parser fails", async () => {
+    // Arrange
+    const log = jest.fn();
+    const cache = new Map();
+    const getLangAsset = mockPromiseValue("{}");
+    const err = new Error("main parser failed");
+    const main = { parse: mockPromiseError(err) };
+    const parser = buildParser({ log, cache, getLangAsset, main });
+    const lang = "0";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).rejects.toBe(err);
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+    expect(cache.has(lang)).toBe(true);
+    expect(cache.get(lang)).toStrictEqual({});
+  });
+  it("should return succeed if lexicon is a buffer", async () => {
+    // Arrange
+    const log = jest.fn();
+    const cache = new Map();
+    const getLangAsset = mockPromiseValue(Buffer.from("{}"));
+    const ast = { root: "0" };
+    const main = { parse: mockPromiseValue(ast) };
+    const parser = buildParser({ log, cache, getLangAsset, main });
+    const lang = "0";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual(ast);
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+    expect(cache.has(lang)).toBe(true);
+    expect(cache.get(lang)).toStrictEqual({});
+  });
+  it("should try vm if lexicon cannot parse JSON", async () => {
+    // Arrange
+    const log = jest.fn();
+    const cache = new Map();
+    const rawLexicon = `
+    (() => {
+      window.gcexports.globalLexicon = {};
+    })();
+    `;
+    const getLangAsset = mockPromiseValue(rawLexicon);
+    const ast = { root: "0" };
+    const main = { parse: mockPromiseValue(ast) };
+    const vm = {
+      createContext: jest.fn(),
+      runInContext: jest.fn().mockImplementation((data, context) => {
+        context.window.gcexports.globalLexicon = {};
+      })
+    };
+    const parser = buildParser({ log, cache, getLangAsset, main, vm });
+    const lang = "0";
+    const src = "'foo'..";
+
+    // Act
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual(ast);
+
+    // Assert
+    expect(getLangAsset).toHaveBeenCalledWith(lang, "/lexicon.js");
+    expect(main.parse).toHaveBeenCalledWith(src, {});
+    expect(cache.has(lang)).toBe(true);
+    expect(cache.get(lang)).toStrictEqual({});
+    expect(vm.createContext).toHaveBeenCalled();
+    expect(vm.runInContext).toHaveBeenCalledWith(rawLexicon, expect.anything());
+  });
+  it.skip("should parse 'hello, world!'", async () => {
+    const lang = "0";
+    const src = "'hello, world'..";
+    const ast = {
+      1: {
+        elts: [
+          "hello, world"
+        ],
+        tag: "STR"
+      },
+      2: {
+        elts: [
+          1
+        ],
+        tag: "EXPRS"
+      },
+      3: {
+        elts: [
+          2
+        ],
+        tag: "PROG"
+      },
+      root: 3
+    };
+    await expect(parser.parse(lang, src)).resolves.toStrictEqual(ast);
+  });
+  it.skip("should parse error", async () => {
+    const lang = "0";
+    const src = "'hello, world'";
+    await expect(parser.parse(lang, src)).rejects.toThrow("End of program reached");
+  });
+  // it("should parse error", async () => {
+  //   const lang = "0";
+  //   const src = "'hello, world..";
+  //   const ast = {};
+  //   await expect(parser.parse(lang, src)).rejects.toThrow("End of program reached");
+  // });
+});

package/src/testing/index.js

@@ -0,0 +1,16 @@
+import { jest } from "@jest/globals";
+export const mockPromiseError = (err) => {
+  return jest.fn().mockImplementation((...params) => {
+    if (params.length > 0) {
+      throw err;
+    } else {
+      throw new Error("no callback paramter given");
+    }
+  });
+};
+
+export const mockPromiseValue = (value) => {
+  return jest.fn().mockImplementation((...params) => {
+    return value;
+  });
+};