@weborigami/language 0.6.17 → 0.7.0-beta.2

package/src/runtime/ops.js

@@ -6,13 +6,15 @@
  * @typedef {import("@weborigami/async-tree").SyncOrAsyncMap} SyncOrAsyncMap
  */
 
-import { getParent, isUnpackable, Tree } from "@weborigami/async-tree";
+import { getParent, isUnpackable, SyncMap, Tree } from "@weborigami/async-tree";
 import os from "node:os";
 import execute from "./execute.js";
 import expressionObject from "./expressionObject.js";
 import mergeTrees from "./mergeTrees.js";
 import OrigamiFileMap from "./OrigamiFileMap.js";
+import ScopeMap from "./ScopeMap.js";
 import { codeSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
 
 function addOpLabel(op, label) {
   Object.defineProperty(op, "toString", {
@@ -81,31 +83,18 @@ addOpLabel(bitwiseXor, "«ops.bitwiseXor»");
 /**
  * Cache the value of the code for an external reference
  *
- * @param {any} cache
- * @param {string} path
+ * @param {string} cachePath
  * @param {AnnotatedCode} code
+ * @param {RuntimeState} state
  */
-export async function cache(cache, path, code) {
-  if (path in cache) {
-    // Cache hit
-    return cache[path];
-  }
-
-  // Don't await: might get another request for this before promise resolves
-  const promise = execute(code);
-
-  // Save promise so another request will get the same promise
-  cache[path] = promise;
-
-  // Now wait for the value
-  const value = await promise;
-
-  // Update the cache with the actual value
-  cache[path] = value;
-
-  return value;
+export function cache(cachePath, code, state) {
+  const result = systemCache.getOrInsertComputedAsync(cachePath, () =>
+    execute(code, state),
+  );
+  return result;
 }
 addOpLabel(cache, "«ops.cache»");
+cache.needsState = true;
 cache.unevaluatedArgs = true;
 
 /**
@@ -114,13 +103,17 @@ cache.unevaluatedArgs = true;
  * @param  {...AnnotatedCode} args
  */
 export async function comma(...args) {
+  /** @type {RuntimeState} */
+  // @ts-ignore
+  const state = args.pop(); // The runtime state is passed as the last argument
   let result;
   for (const arg of args) {
-    result = await execute(arg);
+    result = await execute(arg, state);
   }
   return result;
 }
 addOpLabel(comma, "«ops.comma»");
+comma.needsState = true;
 comma.unevaluatedArgs = true;
 
 export async function conditional(condition, truthy, falsy) {
@@ -143,7 +136,7 @@ export async function construct(constructor, ...args) {
 export async function deepText(...args) {
   return Tree.deepText(args);
 }
-addOpLabel(deepText, "«ops.deepText");
+addOpLabel(deepText, "«ops.deepText»");
 
 /**
  * Default value for a parameter: if the value is defined, return that;
@@ -180,8 +173,7 @@ addOpLabel(exponentiation, "«ops.exponentiation»");
 export async function flat(...args) {
   // Unpack packed arguments so they can be flattened
   const unpacked = args.map((arg) => (isUnpackable(arg) ? arg.unpack() : arg));
-  // Flatten to depth 2: 1 for args array, 1 for flattening
-  return Tree.flat(unpacked, 2);
+  return Tree.flat(unpacked);
 }
 addOpLabel(flat, "«ops.flat»");
 
@@ -206,6 +198,7 @@ addOpLabel(greaterThanOrEqual, "«ops.greaterThanOrEqual»");
  */
 export async function homeDirectory(...keys) {
   const tree = new OrigamiFileMap(os.homedir());
+  await tree.initializeGlobals();
   return keys.length > 0 ? Tree.traverse(tree, ...keys) : tree;
 }
 addOpLabel(homeDirectory, "«ops.homeDirectory»");
@@ -488,7 +481,8 @@ export async function property(object, key) {
 
   if (isUnpackable(object)) {
     object = await object.unpack();
-  } else if (typeof object === "string") {
+  }
+  if (typeof object === "string") {
     object = new String(object);
   } else if (typeof object === "number") {
     object = new Number(object);
@@ -529,17 +523,18 @@ addOpLabel(rootDirectory, "«ops.rootDirectory»");
 /**
  * Return the scope of the current tree
  *
- * @param {SyncOrAsyncMap} parent
+ * @param {RuntimeState} state
  */
-export async function scope(parent) {
+export async function scope(state = {}) {
+  const { parent } = state;
   if (!parent) {
-    throw new ReferenceError(
-      "Tried to find a value in scope, but no container was provided as the parent.",
-    );
+    return new SyncMap(); // empty scope if there's no parent
   }
-  return Tree.scope(parent);
+  const scopeMap = new ScopeMap(parent);
+  return scopeMap;
 }
 addOpLabel(scope, "«ops.scope»");
+scope.needsState = true;
 
 export function shiftLeft(a, b) {
   return a << b;

package/src/runtime/symbols.js

@@ -1,4 +1,7 @@
+export const cachePathSymbol = Symbol("cachePath");
 export const codeSymbol = Symbol("code");
 export const configSymbol = Symbol("config");
+export const noCacheSymbol = Symbol("noCache");
 export const scopeSymbol = Symbol("scope");
 export const sourceSymbol = Symbol("source");
+export const volatileSymbol = Symbol("volatile");

package/src/runtime/systemCache.js

@@ -0,0 +1,3 @@
+import SystemCacheMap from "./SystemCacheMap.js";
+
+export default new SystemCacheMap();

package/src/runtime/volatile.js

@@ -0,0 +1,14 @@
+import { box } from "@weborigami/async-tree";
+import { volatileSymbol } from "./symbols.js";
+
+/**
+ * Mark the indicated value as volatile so it won't be cached.
+ */
+export default function volatile(value) {
+  const boxed = box(value);
+  Object.defineProperty(boxed, volatileSymbol, {
+    value: true,
+    enumerable: false,
+  });
+  return boxed;
+}

package/test/compiler/codeHelpers.js

@@ -23,11 +23,12 @@ export function assertCodeLocations(code) {
  */
 export function createCode(array) {
   const code = array.map((item) =>
-    item instanceof Array ? createCode(item) : item
+    item instanceof Array ? createCode(item) : item,
   );
   /** @type {any} */ (code).location = {
     end: 0,
     source: {
+      relativePath: "test.ori",
       text: "",
     },
     start: 0,

package/test/compiler/optimize.test.js

@@ -46,15 +46,7 @@ describe("optimize", () => {
   });
 
   test("resolve deeper context references", () => {
-    // Compilation of `{ a: 1, more: { a } }`
-    const code = createCode([
-      ops.object,
-      ["a", [ops.literal, 1]],
-      [
-        "more",
-        [ops.object, ["a", [markers.traverse, [markers.reference, "a"]]]],
-      ],
-    ]);
+    const expression = `{ a: 1, more: { a } }`;
     const expected = [
       ops.object,
       ["a", 1],
@@ -72,7 +64,7 @@ describe("optimize", () => {
         ],
       ],
     ];
-    assertCodeEqual(optimize(code), expected);
+    assertCompile(expression, expected);
   });
 
   test("when defining a property, avoid recursive references", () => {
@@ -103,7 +95,7 @@ describe("optimize", () => {
     assertCompile(expression, expected, "shell");
   });
 
-  describe("resolve reference", () => {
+  describe.only("resolve reference", () => {
     test("external reference", () => {
       // Compilation of `folder` where folder isn't a variable
       const code = createCode([
@@ -113,35 +105,36 @@ describe("optimize", () => {
       const parent = {};
       const expected = [
         ops.cache,
-        {},
-        "folder",
-        [
-          [ops.scope, parent],
-          [ops.literal, "folder"],
-        ],
+        "test.ori/_refs/folder",
+        [[ops.scope], [ops.literal, "folder"]],
       ];
       const globals = {};
-      assertCodeEqual(optimize(code, { globals, parent }), expected);
+      assertCodeEqual(
+        optimize(code, { cachePath: "test.ori", globals, parent }),
+        expected,
+      );
     });
 
-    test("external reference", () => {
-      // Compilation of `index.html` where `index` isn't a variable
+    test("external reference with implied unpack", () => {
+      // Compilation of `greet.ori("world")`
       const code = createCode([
-        markers.traverse,
-        [markers.reference, "index.html"],
+        [markers.traverse, [markers.reference, "greet.ori"]],
+        [ops.literal, "world"],
       ]);
       const parent = {};
       const expected = [
-        ops.cache,
-        {},
-        "index.html",
         [
-          [ops.scope, parent],
-          [ops.literal, "index.html"],
+          ops.cache,
+          "test.ori/_refs/greet.ori/",
+          [ops.unpack, [[ops.scope], [ops.literal, "greet.ori"]]],
         ],
+        "world",
       ];
       const globals = {};
-      assertCodeEqual(optimize(code, { globals, parent }), expected);
+      assertCodeEqual(
+        optimize(code, { cachePath: "test.ori", globals, parent }),
+        expected,
+      );
     });
 
     test("external reference inside object with matching key", () => {
@@ -162,18 +155,17 @@ describe("optimize", () => {
             ops.getter,
             [
               ops.cache,
-              {},
-              "posts.txt",
-              [
-                [ops.scope, parent],
-                [ops.literal, "posts.txt"],
-              ],
+              "test.ori/_refs/posts.txt",
+              [[ops.scope], [ops.literal, "posts.txt"]],
             ],
           ],
         ],
       ];
       const globals = {};
-      assertCodeEqual(optimize(code, { globals, parent }), expected);
+      assertCodeEqual(
+        optimize(code, { cachePath: "test.ori", globals, parent }),
+        expected,
+      );
     });
 
     test("global reference", () => {
@@ -233,17 +225,26 @@ describe("optimize", () => {
     });
 
     test("root directory", () => {
-      // Compilation of `</>`
-      const code = createCode([markers.traverse, [markers.external, "/"]]);
-      const expected = [ops.cache, {}, "/", [ops.rootDirectory]];
-      assertCodeEqual(optimize(code), expected);
+      // Compilation of `</etc/passwd>`
+      const code = createCode([
+        markers.traverse,
+        [markers.external, "/"],
+        [ops.literal, "etc/"],
+        [ops.literal, "passwd"],
+      ]);
+      const expected = [
+        ops.cache,
+        "test.ori/_refs//etc/passwd",
+        [[ops.rootDirectory], [ops.literal, "etc/"], [ops.literal, "passwd"]],
+      ];
+      assertCodeEqual(optimize(code, { cachePath: "test.ori" }), expected);
     });
 
     test("home directory", () => {
       // Compilation of `<~>`
       const code = createCode([markers.traverse, [ops.homeDirectory]]);
-      const expected = [ops.cache, {}, "~", [ops.homeDirectory]];
-      assertCodeEqual(optimize(code), expected);
+      const expected = [ops.cache, "test.ori/_refs/~", [ops.homeDirectory]];
+      assertCodeEqual(optimize(code, { cachePath: "test.ori" }), expected);
     });
   });
 
@@ -259,16 +260,18 @@ describe("optimize", () => {
       const parent = {};
       const expected = [
         ops.cache,
-        {},
-        "path/to/file",
+        "test.ori/_refs/path/to/file",
         [
-          [ops.scope, parent],
+          [ops.scope],
           [ops.literal, "path/"],
           [ops.literal, "to/"],
           [ops.literal, "file"],
         ],
       ];
-      assertCodeEqual(optimize(code, { parent }), expected);
+      assertCodeEqual(
+        optimize(code, { cachePath: "test.ori", parent }),
+        expected,
+      );
     });
 
     test("implicit external path", () => {
@@ -282,15 +285,13 @@ describe("optimize", () => {
       const parent = {};
       const expected = [
         ops.cache,
-        {},
-        "package.json/name",
-        [
-          [ops.scope, parent],
-          [ops.literal, "package.json/"],
-          [ops.literal, "name"],
-        ],
+        "test.ori/_refs/package.json/name",
+        [[ops.scope], [ops.literal, "package.json/"], [ops.literal, "name"]],
       ];
-      assertCodeEqual(optimize(code, { globals, parent }), expected);
+      assertCodeEqual(
+        optimize(code, { cachePath: "test.ori", globals, parent }),
+        expected,
+      );
     });
 
     test("local path", () => {
@@ -314,9 +315,16 @@ describe("optimize", () => {
 });
 
 function assertCompile(expression, expected, mode = "shell") {
+  const source =
+    typeof expression !== "string"
+      ? expression
+      : {
+          text: expression,
+          relativePath: "test.ori",
+        };
   const parent = new SyncMap();
   const globals = {};
-  const fn = compile.expression(expression, { globals, mode, parent });
+  const fn = compile.expression(source, { globals, mode, parent });
   const actual = fn.code;
   assertCodeLocations(actual);
   assertCodeEqual(actual, expected);

package/test/handlers/epub_handler.test.js

@@ -0,0 +1,27 @@
+import Zip from "adm-zip";
+import assert from "node:assert";
+import { describe, test } from "node:test";
+import epub_handler from "../../src/handlers/epub_handler.js";
+
+describe("EPUB handler", () => {
+  test("ensures mimetype file comes first", async () => {
+    const tree = {
+      EPUB: {
+        "index.xhtml": "This is where the book content goes",
+      },
+      "META-INF": {
+        "container.xml": "This is where the metadata goes",
+      },
+      mimetype: "application/epub+zip",
+    };
+    const buffer = await epub_handler.pack(tree);
+    const unzipped = new Zip(buffer);
+    const entries = unzipped.getEntries();
+    const entryNames = entries.map((entry) => entry.entryName);
+    assert.deepEqual(entryNames, [
+      "mimetype",
+      "EPUB/index.xhtml",
+      "META-INF/container.xml",
+    ]);
+  });
+});

package/test/handlers/ori_handler.test.js

@@ -3,9 +3,11 @@ import assert from "node:assert";
 import { describe, test } from "node:test";
 import ori_handler from "../../src/handlers/ori_handler.js";
 import OrigamiFileMap from "../../src/runtime/OrigamiFileMap.js";
+import { cachePathSymbol } from "../../src/runtime/symbols.js";
 
 const fixturesUrl = new URL("fixtures", import.meta.url);
 const fixtures = new OrigamiFileMap(fixturesUrl);
+await fixtures.initializeGlobals();
 
 describe(".ori handler", async () => {
   test("loads a string expression", async () => {
@@ -38,16 +40,18 @@ describe(".ori handler", async () => {
     const tree = await ori_handler.unpack(source);
     assert.deepEqual(
       await Tree.traverse(tree, "public", "message"),
-      "Hello, world!"
+      "Hello, world!",
     );
   });
 
   test("loads an object containing an object shorthand", async () => {
-    const assets = new ObjectMap({});
+    const assets = new ObjectMap({ a: 1, b: 2 });
     const parent = new ObjectMap({ assets });
     const source = `{ assets }`;
     const object = await ori_handler.unpack(source, { parent });
-    assert.equal(object.assets, assets);
+    const expectedValues = Array.from(assets.values());
+    const actualValues = Array.from(object.assets.values());
+    assert.deepEqual(actualValues, expectedValues);
   });
 
   test("loads a template literal", async () => {
@@ -84,4 +88,19 @@ describe(".ori handler", async () => {
     const indexHtml = await tree["index.html"];
     assert.equal(indexHtml, "Hello, world!");
   });
+
+  test("enables caching for top-level object", async () => {
+    const parent = new ObjectMap({});
+    const source = `{
+      message = "Hello"
+    }`;
+    const object = await ori_handler.unpack(source, {
+      key: "test.ori",
+      parent,
+    });
+    assert.equal(object[cachePathSymbol], "test.ori/");
+    assert.deepEqual(await Tree.plain(object), {
+      message: "Hello",
+    });
+  });
 });

package/test/handlers/oridocument_handler.test.js

@@ -22,7 +22,7 @@ describe("Origami document handler", () => {
 
   test("Argument to template document available as underscore", async () => {
     const text = "<h1>${ _ }</h1>";
-    const fn = await oridocument_handler.unpack(text);
+    const fn = await oridocument_handler.unpack(text, { key: "test.ori.html" });
     const result = await fn("Home");
     assert.equal(result, "<h1>Home</h1>");
   });

package/test/handlers/zip_handler.test.js

@@ -0,0 +1,45 @@
+import { Tree } from "@weborigami/async-tree";
+import Zip from "adm-zip";
+import assert from "node:assert";
+import fs from "node:fs/promises";
+import { describe, test } from "node:test";
+import zip_handler from "../../src/handlers/zip_handler.js";
+
+describe("ZIP handler", () => {
+  test("creates a ZIP file as Buffer", async () => {
+    const tree = {
+      "ReadMe.md": "This is a ReadMe file.",
+      sub: {
+        "file.txt": "This is a text file in a subfolder.",
+      },
+    };
+    const buffer = await zip_handler.pack(tree);
+    const unzipped = new Zip(buffer);
+    const entries = unzipped.getEntries();
+    assert.equal(entries.length, 2);
+    assert.equal(entries[0].entryName, "ReadMe.md");
+    assert.equal(
+      entries[0].getData().toString("utf8"),
+      "This is a ReadMe file.",
+    );
+    assert.equal(entries[1].entryName, "sub/file.txt");
+    assert.equal(
+      entries[1].getData().toString("utf8"),
+      "This is a text file in a subfolder.",
+    );
+  });
+
+  test("reads a ZIP file", async () => {
+    const fixturePath = new URL("fixtures/test.zip", import.meta.url);
+    const buffer = await fs.readFile(fixturePath);
+    const tree = await zip_handler.unpack(buffer);
+    const plain = await Tree.plain(tree);
+    assert.deepEqual(Object.keys(plain), ["ReadMe.md", "sub"]);
+    assert.deepEqual(Object.keys(plain.sub), ["file.txt"]);
+    assert.equal(String(plain["ReadMe.md"]), "This is a ReadMe file.\n");
+    assert.equal(
+      String(plain.sub["file.txt"]),
+      "This is a text file in a subfolder.\n",
+    );
+  });
+});

package/test/protocols/https.test.js

@@ -0,0 +1,19 @@
+import assert from "node:assert";
+import { beforeEach, describe, test } from "node:test";
+import https from "../../src/protocols/https.js";
+import systemCache from "../../src/runtime/systemCache.js";
+
+describe("https", () => {
+  beforeEach(() => {
+    systemCache.clear();
+  });
+
+  test("caches fetched resources", async () => {
+    const result1 = await https("example.com", {});
+    assert(systemCache.has("https://example.com"));
+    const text = new TextDecoder().decode(result1);
+    assert(text.includes("Example Domain"));
+    const result2 = await https("example.com", {});
+    assert.strictEqual(result1, result2);
+  });
+});

package/test/protocols/package.test.js

@@ -1,16 +1,21 @@
 import assert from "node:assert";
 import process from "node:process";
 import { describe, test } from "node:test";
+import coreGlobals from "../../src/project/coreGlobals.js";
 import projectRootFromPath from "../../src/project/projectRootFromPath.js";
 import packageProtocol from "../../src/protocols/package.js";
 
 describe("package: protocol", () => {
   test("returns a package's main export(s)", async () => {
     // Reproduce the type of evaluation context object the runtime would create
-    const projectRoot = await projectRootFromPath(process.cwd());
-    const state = { parent: projectRoot };
+    const parent = await projectRootFromPath(process.cwd());
+    const globals = await coreGlobals();
+    /** @type {any} */ (parent).globals = globals;
+    const state = { globals, parent };
 
     const result = await packageProtocol("@weborigami", "async-tree", state);
+
+    // Try a method from the package
     const { toString } = result;
     assert.equal(toString(123), "123");
   });

package/test/runtime/AsyncCacheTransform.test.js

@@ -0,0 +1,91 @@
+import assert from "node:assert";
+import { beforeEach, describe, test } from "node:test";
+import systemCache from "../../src/runtime/systemCache.js";
+
+// We test AsyncCacheTransform via asyncCalcs since it's a fairly small
+// application of the transform.
+import { Tree } from "@weborigami/async-tree";
+import asyncCalcs from "./asyncCalcs.js";
+
+describe("AsyncCacheTransform", () => {
+  beforeEach(() => {
+    systemCache.clear();
+  });
+
+  test("caches values and records dependencies", async () => {
+    // { a = 2 * b, b = c + 1, c = 3 }
+    let log = [];
+    const { calcs, data } = asyncCalcs([
+      [
+        "a",
+        async () => {
+          log.push("a");
+          const b = await calcs.get("b");
+          return 2 * b;
+        },
+      ],
+      [
+        "b",
+        async () => {
+          log.push("b");
+          const c = await calcs.get("c");
+          return c + 1;
+        },
+      ],
+      [
+        "c",
+        async () => {
+          log.push("c");
+          return 3;
+        },
+      ],
+    ]);
+
+    assert.deepEqual(await Tree.entries(calcs), [
+      ["a", 8],
+      ["b", 4],
+      ["c", 3],
+    ]);
+    assert.deepEqual(log, ["a", "b", "c"]);
+
+    log = [];
+    const a1 = await calcs.get("a");
+    assert.strictEqual(a1, 8);
+    assert.deepEqual(log, []); // a is cached, no new calcs
+
+    // Replace formula for a
+    // { a = 3 * b, b = c + 1, c = 3 }
+    data.set("a", async () => {
+      log.push("a");
+      const b = await calcs.get("b");
+      return 3 * b;
+    });
+    log = [];
+    const a2 = await calcs.get("a");
+    assert.strictEqual(a2, 12);
+    assert.deepEqual(log, ["a"]); // recalc only a
+
+    // Replace formula for b
+    // { a = 3 * b, b = c + 10, c = 3 }
+    data.set("b", async () => {
+      log.push("b");
+      const c = await calcs.get("c");
+      return c + 10;
+    });
+    log = [];
+    const a3 = await calcs.get("a");
+    assert.strictEqual(a3, 39);
+    assert.deepEqual(log, ["a", "b"]); // recalc a and b
+
+    // Replace value of c
+    // { a = 3 * b, b = c + 10, c = 100 }
+    data.set("c", async () => {
+      log.push("c");
+      return 100;
+    });
+    log = [];
+    const a4 = await calcs.get("a");
+    assert.strictEqual(a4, 330);
+    assert.deepEqual(log, ["a", "b", "c"]); // recalc all
+  });
+});