@weborigami/language 0.6.17 → 0.7.0-beta.2

package/src/handlers/txt_handler.js

@@ -6,7 +6,8 @@ import {
 } from "@weborigami/async-tree";
 import * as YAMLModule from "yaml";
 import * as compile from "../compiler/compile.js";
-import projectGlobals from "../project/projectGlobals.js";
+import coreGlobals from "../project/coreGlobals.js";
+import getGlobalsForTree from "../project/getGlobalsForTree.js";
 import parseFrontMatter from "./parseFrontMatter.js";
 
 // The "yaml" package doesn't seem to provide a default export that the browser can
@@ -80,7 +81,8 @@ export default {
       const { body, frontText, isOrigami } = parsed;
       let frontData;
       if (isOrigami) {
-        const globals = await projectGlobals(parent);
+        const globals =
+          options.globals ?? getGlobalsForTree(parent) ?? (await coreGlobals());
         const compiled = compile.expression(frontText.trim(), {
           globals,
           parent,

package/src/handlers/xhtml_handler.js

@@ -1,2 +1,2 @@
 // .xhtml is a synonynm for .html
-export { html_handler as default } from "./handlers.js";
+export { default } from "./html_handler.js";

package/src/handlers/yaml_handler.js

@@ -8,7 +8,8 @@ import {
 } from "@weborigami/async-tree";
 import * as YAMLModule from "yaml";
 import * as compile from "../compiler/compile.js";
-import projectGlobals from "../project/projectGlobals.js";
+import coreGlobals from "../project/coreGlobals.js";
+import getGlobalsForTree from "../project/getGlobalsForTree.js";
 import getSource from "./getSource.js";
 
 // The "yaml" package doesn't seem to provide a default export that the browser can
@@ -104,7 +105,8 @@ export default {
 };
 
 async function oriCallTagForParent(parent, options, yaml) {
-  const globals = await projectGlobals(parent);
+  const globals =
+    options.globals ?? getGlobalsForTree(parent) ?? (await coreGlobals());
   return {
     collection: "seq",
 
@@ -168,7 +170,8 @@ async function oriCallTagForParent(parent, options, yaml) {
 // Define the !ori tag for YAML parsing. This will run in the context of the
 // supplied parent.
 async function oriTagForParent(parent, options, yaml) {
-  const globals = await projectGlobals(parent);
+  const globals =
+    options.globals ?? getGlobalsForTree(parent) ?? (await coreGlobals());
   return {
     resolve(text) {
       hasOriTags = true;

package/src/handlers/zip_handler.js

@@ -0,0 +1,112 @@
+import { isUnpackable, SyncMap, Tree } from "@weborigami/async-tree";
+import {
+  getGlobalsForTree,
+  HandleExtensionsTransform,
+} from "@weborigami/language";
+import Zip from "adm-zip";
+import addExtensionKeyFn from "./addExtensionKeyFn.js";
+
+/**
+ * Handler for ZIP files
+ */
+const zip_handler = {
+  mediaType: "application/zip",
+
+  /**
+   * Pack a tree of files as a ZIP file in Buffer form.
+   *
+   * @param {import("@weborigami/async-tree").Maplike} maplike
+   */
+  async pack(maplike) {
+    // The ZIP file should leave the files in tree order.
+    const zip = new Zip({ noSort: true });
+
+    if (isUnpackable(maplike)) {
+      maplike = await maplike.unpack();
+    }
+    const tree = Tree.from(maplike, { deep: true });
+    const deflated = await Tree.deflatePaths(tree);
+    for (let [path, value] of deflated) {
+      if (typeof value === "function") {
+        value = value();
+      }
+      if (value instanceof Promise) {
+        value = await value;
+      } else if (value instanceof String) {
+        value = value.toString(); // adm-zip wants simple strings
+      }
+      zip.addFile(path, value);
+
+      // Special case for EPUB files, where `mimetype` must be uncompressed.
+      if (path === "mimetype") {
+        const entry = zip.getEntry(path);
+        entry.header.method = 0; // STORE (not DEFLATE)
+      }
+    }
+    const buffer = zip.toBuffer();
+    return buffer;
+  },
+
+  /**
+   * Unpack a ZIP file
+   */
+  async unpack(buffer, options) {
+    // Origami generally prefers keeping things as an Uint8Array or ArrayBuffer,
+    // but adm-zip only accepts a Buffer.
+    if (buffer instanceof Uint8Array || buffer instanceof ArrayBuffer) {
+      // @ts-ignore
+      buffer = Buffer.from(buffer);
+    }
+
+    const zip = new Zip(buffer);
+
+    const entries = zip.getEntries();
+    const filtered = entries.filter(
+      (entry) =>
+        !entry.entryName.startsWith("__MACOSX/") &&
+        !entry.entryName.endsWith("/"),
+    );
+    const deflated = Object.fromEntries(
+      filtered.map((entry) => [entry.entryName, () => entry.getData()]),
+    );
+
+    // The final tree will include extension handlers and have functions invoked
+    // to retrieve data from the ZIP file. While the base map is a SyncMap, the
+    // final tree will be async.
+    const classFn = HandleExtensionsTransform(
+      InvokeFunctionsTransform(SyncMap),
+    );
+    const result = await Tree.inflatePaths(deflated, { classFn });
+
+    const parent = options?.parent;
+    const globals = parent ? getGlobalsForTree(parent) : null;
+    if (globals) {
+      result.globals = globals;
+    }
+
+    return result;
+  },
+};
+
+/** @type {any} */ (zip_handler.pack).key = addExtensionKeyFn(".zip");
+
+export default zip_handler;
+
+function InvokeFunctionsTransform(Base) {
+  return class InvokeFunctions extends Base {
+    delete(key) {
+      return super.delete(key);
+    }
+
+    get(key) {
+      const value = super.get(key);
+      return typeof value === "function" ? value() : value;
+    }
+
+    set(key, value) {
+      return super.set(key, value);
+    }
+
+    trailingSlashKeys = true;
+  };
+}

package/src/project/activeProjectRoot.js

@@ -0,0 +1,9 @@
+let activeProjectRoot = null;
+
+export function get() {
+  return activeProjectRoot;
+}
+
+export function set(projectRoot) {
+  activeProjectRoot = projectRoot;
+}

package/src/project/getGlobalsForTree.js

@@ -0,0 +1,5 @@
+import { Tree } from "@weborigami/async-tree";
+
+export default function getGlobalsForTree(map) {
+  return map ? Tree.root(map).globals : null;
+}

package/src/project/{projectGlobals.js → initializeGlobalsForTree.js}

@@ -1,10 +1,9 @@
-import { Tree } from "@weborigami/async-tree";
-import assignPropertyDescriptors from "../runtime/assignPropertyDescriptors.js";
+import { Tree, assignPropertyDescriptors } from "@weborigami/async-tree";
 import coreGlobals from "./coreGlobals.js";
 import projectConfig from "./projectConfig.js";
 
 /**
- * Return the complete set of globals available to code running in the given
+ * Make the complete set of globals available to code running in the given
  * container. This will be the core globals plus any configuration specified in
  * the project's config.ori file.
  *
@@ -12,17 +11,13 @@ import projectConfig from "./projectConfig.js";
  * container.
  *
  * @typedef {import("@weborigami/async-tree").SyncOrAsyncMap} SyncOrAsyncMap
- * @param {SyncOrAsyncMap|null} parent
+ * @param {SyncOrAsyncMap} parent
  */
-export default async function projectGlobals(parent) {
-  if (!parent) {
-    return coreGlobals();
-  }
-
-  const projectRoot = await Tree.root(parent);
+export default async function initializeGlobalsForTree(parent) {
+  const projectRoot = Tree.root(parent);
   if (!projectRoot.globals) {
-    // Start with core globals
-    const globals = await coreGlobals();
+    // Start with a copy of the core globals
+    const globals = { ...(await coreGlobals()) };
 
     if (parent) {
       // Get config for the given container and add it to the globals. During
@@ -35,7 +30,7 @@ export default async function projectGlobals(parent) {
       assignPropertyDescriptors(globals, config);
     }
 
-    // Cache globals on project root
+    // Store globals on project root
     projectRoot.globals = globals;
   }
 

package/src/project/jsGlobals.js

@@ -92,6 +92,7 @@ const globals = {
   SuppressedError: globalThis.SuppressedError,
   Symbol,
   SyntaxError,
+  Temporal: globalThis.Temporal,
   TextDecoder,
   TextDecoderStream,
   TextEncoder,

package/src/project/projectConfig.js

@@ -10,8 +10,8 @@ import coreGlobals from "./coreGlobals.js";
  * @typedef {import("@weborigami/async-tree").SyncOrAsyncMap} SyncOrAsyncMap
  * @param {SyncOrAsyncMap} parent
  */
-export default async function config(parent) {
-  const projectRoot = await Tree.root(parent);
+export default async function projectConfig(parent) {
+  const projectRoot = Tree.root(parent);
 
   let configObject = {};
   const configBuffer = await projectRoot.get("config.ori");

package/src/project/projectRootFromPath.js

@@ -55,5 +55,7 @@ export default async function projectRootFromPath(dirname) {
     root = new OrigamiFileMap(dirname);
   }
 
+  await root.initializeGlobals();
+
   return root;
 }

package/src/protocols/constructHref.js

@@ -8,11 +8,11 @@ import { pathFromKeys } from "@weborigami/async-tree";
  * @param  {string[]} keys
  */
 export default function constructHref(protocol, host, ...keys) {
+  let href = host.endsWith("/") ? host.slice(0, -1) : host;
   const path = pathFromKeys(keys);
-  if (host.endsWith("/")) {
-    host = host.slice(0, -1);
+  if (path) {
+    href += "/" + path;
   }
-  let href = [host, path].join("/");
   if (!href.startsWith(protocol)) {
     if (!href.startsWith("//")) {
       href = `//${href}`;

package/src/protocols/constructSiteTree.js

@@ -1,5 +1,6 @@
 import { trailingSlash } from "@weborigami/async-tree";
 import HandleExtensionsTransform from "../runtime/HandleExtensionsTransform.js";
+import systemCache from "../runtime/systemCache.js";
 import constructHref from "./constructHref.js";
 
 /**
@@ -12,7 +13,12 @@ import constructHref from "./constructHref.js";
  * @param {string} host
  * @param  {string[]} keys
  */
-export default function constructSiteTree(protocol, mapClass, host, ...keys) {
+export default async function constructSiteTree(
+  protocol,
+  mapClass,
+  host,
+  ...keys
+) {
   // If the last key doesn't end in a slash, remove it for now.
   let lastKey;
   if (keys.length > 0 && keys.at(-1) && !trailingSlash.has(keys.at(-1))) {
@@ -20,7 +26,10 @@ export default function constructSiteTree(protocol, mapClass, host, ...keys) {
   }
 
   const href = constructHref(protocol, host, ...keys);
-  let result = new (HandleExtensionsTransform(mapClass))(href);
+  const result = await systemCache.getOrInsertComputedAsync(
+    href,
+    () => new (HandleExtensionsTransform(mapClass))(href),
+  );
 
   return lastKey ? result.get(lastKey) : result;
 }

package/src/protocols/explore.js

@@ -8,6 +8,6 @@ import constructSiteTree from "./constructSiteTree.js";
  * @param {string} host
  * @param  {...string} keys
  */
-export default function explore(host, ...keys) {
+export default async function explore(host, ...keys) {
   return constructSiteTree("https:", ExplorableSiteMap, host, ...keys);
 }

package/src/protocols/explorehttp.js

@@ -8,6 +8,6 @@ import constructSiteTree from "./constructSiteTree.js";
  * @param {string} host
  * @param  {...string} keys
  */
-export default function explorehttp(host, ...keys) {
+export default async function explorehttp(host, ...keys) {
   return constructSiteTree("http:", ExplorableSiteMap, host, ...keys);
 }

package/src/protocols/fetchAndHandleExtension.js

@@ -1,18 +1,26 @@
-import handleExtension from "../runtime/handleExtension.js";
+import { args, Tree } from "@weborigami/async-tree";
+import handleExtension from "../../src/runtime/handleExtension.js";
 
 /**
- * Fetch the resource at the given href.
- *
- * @typedef {import("@weborigami/async-tree").SyncOrAsyncMap} SyncOrAsyncMap
+ * Extend the JavaScript `fetch` function to implicity return an ArrayBuffer
+ * with an unpack() method if the resource has a known file extension or MIME
+ * type.
  *
  * @param {string} href
- * @param {SyncOrAsyncMap} parent
  */
-export default async function fetchAndHandleExtension(href, parent) {
-  const response = await fetch(href);
+export default async function fetchAndHandleExtension(href, options, state) {
+  if (options && state === undefined) {
+    // Options weren't provided
+    state = options;
+    options = undefined;
+  }
+
+  href = args.string(href, "Origami.fetch");
+  const response = await fetch(href, options);
   if (!response.ok) {
     return undefined;
   }
+
   let buffer = await response.arrayBuffer();
 
   const mediaType = response.headers.get("Content-Type");
@@ -20,12 +28,16 @@ export default async function fetchAndHandleExtension(href, parent) {
     /** @type {any} */ (buffer).mediaType = mediaType;
   }
 
-  // Attach any loader defined for the file type.
+  // Attach any handler defined for the file type or MIME type.
+  const parent = state?.parent;
   const url = new URL(href);
-  const filename = url.pathname.split("/").pop();
-  if (filename) {
-    buffer = await handleExtension(buffer, filename, parent);
+  if (parent) {
+    const root = await Tree.root(parent);
+    const globals = root.globals;
+    const filename = url.pathname.split("/").pop();
+    buffer = await handleExtension(buffer, filename, globals, parent);
   }
 
   return buffer;
 }
+fetchAndHandleExtension.needsState = true;

package/src/protocols/files.js

@@ -22,6 +22,7 @@ export default async function files(...args) {
   const resolved = path.resolve(basePath, relativePath);
 
   const result = new OrigamiFileMap(resolved);
+  await result.initializeGlobals();
   return result;
 }
 files.needsState = true;

package/src/protocols/http.js

@@ -1,3 +1,4 @@
+import systemCache from "../runtime/systemCache.js";
 import constructHref from "./constructHref.js";
 import fetchAndHandleExtension from "./fetchAndHandleExtension.js";
 
@@ -11,6 +12,8 @@ import fetchAndHandleExtension from "./fetchAndHandleExtension.js";
 export default async function http(host, ...keys) {
   const state = keys.pop();
   const href = constructHref("http:", host, ...keys);
-  return fetchAndHandleExtension(href, state.parent);
+  return systemCache.getOrInsertComputedAsync(href, () =>
+    fetchAndHandleExtension(href, null, state),
+  );
 }
 http.needsState = true;

package/src/protocols/https.js

@@ -1,3 +1,4 @@
+import systemCache from "../runtime/systemCache.js";
 import constructHref from "./constructHref.js";
 import fetchAndHandleExtension from "./fetchAndHandleExtension.js";
 
@@ -11,6 +12,8 @@ import fetchAndHandleExtension from "./fetchAndHandleExtension.js";
 export default async function https(host, ...keys) {
   const state = keys.pop();
   const href = constructHref("https:", host, ...keys);
-  return fetchAndHandleExtension(href, state.parent);
+  return systemCache.getOrInsertComputedAsync(href, () =>
+    fetchAndHandleExtension(href, null, state),
+  );
 }
 https.needsState = true;

package/src/protocols/httpstree.js

@@ -8,6 +8,6 @@ import constructSiteTree from "./constructSiteTree.js";
  * @param {string} host
  * @param  {...string} keys
  */
-export default function httpstree(host, ...keys) {
+export default async function httpstree(host, ...keys) {
   return constructSiteTree("https:", SiteMap, host, ...keys);
 }

package/src/protocols/httptree.js

@@ -8,6 +8,6 @@ import constructSiteTree from "./constructSiteTree.js";
  * @param {string} host
  * @param  {...string} keys
  */
-export default function httptree(host, ...keys) {
+export default async function httptree(host, ...keys) {
   return constructSiteTree("http:", SiteMap, host, ...keys);
 }

package/src/protocols/package.js

@@ -1,5 +1,6 @@
 import { Tree, keysFromPath, pathFromKeys } from "@weborigami/async-tree";
 import projectRoot from "../project/projectRoot.js";
+import systemCache from "../runtime/systemCache.js";
 
 /**
  * The package: protocol handler
@@ -9,15 +10,27 @@ import projectRoot from "../project/projectRoot.js";
 export default async function packageProtocol(...args) {
   const state = args.pop(); // Remaining args are the path
   const root = await projectRoot(state);
+  const path = pathFromKeys(args);
+  if (!path) {
+    throw new Error("package: protocol requires a package name");
+  }
+  const href = `package:${path}`;
+  const result = await systemCache.getOrInsertComputedAsync(href, () =>
+    loadPackage(root, args),
+  );
+  return result;
+}
+packageProtocol.needsState = true;
 
+async function loadPackage(root, args) {
   // Identify the path to the package root
-  const packageRootKeys = ["node_modules"];
+  const packageRootKeys = ["node_modules/"];
   let name = args.shift();
   packageRootKeys.push(name);
   if (name.startsWith("@")) {
     // First key is an npm organization, add next key as name
     const nameArg = args.shift();
-    name += nameArg;
+    name = pathFromKeys([name, nameArg]);
     packageRootKeys.push(nameArg);
   }
   const packageRootPath = pathFromKeys(packageRootKeys);
@@ -60,4 +73,3 @@ export default async function packageProtocol(...args) {
 
   return result;
 }
-packageProtocol.needsState = true;

package/src/runtime/AsyncCacheTransform.d.ts

@@ -0,0 +1,5 @@
+import { Mixin } from "../../index.ts";
+
+declare const AsyncCacheTransform: Mixin<{}>
+
+export default AsyncCacheTransform;

package/src/runtime/AsyncCacheTransform.js

@@ -0,0 +1,134 @@
+import enableValueCaching from "./enableValueCaching.js";
+import { cachePathSymbol, noCacheSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
+import SystemCacheMap from "./SystemCacheMap.js";
+
+/**
+ * General-purpose mixin for Origami maps with dependency tracking, used for:
+ * files, site resources, and scope references in Origami files
+ *
+ * This wraps a map's get() and keys() methods to add caching and dependency tracking.
+ * It tracks which cached values are downstream of other cached values so that if
+ * an upstream value changes, all dependent downstream cached values can be
+ * invalidated efficiently.
+ *
+ * Cache entries look like:
+ *
+ *      key -> {
+ *        downstreams: Set(path),
+ *        value
+ *      }
+ *
+ * This allows for efficiently evicting all a value and all its downstream
+ * dependent cached values.
+ *
+ * Example project:
+ *
+ *     site.ori loads a.ori and b.ori
+ *     a.ori loads c.ori
+ *     b.ori loads c.ori
+ *     c.ori doesn't load anything
+ *
+ * Resulting cache:
+ *
+ *      site.ori -> { value: ... }
+ *      a.ori -> { downstreams: Set(site.ori), value: ... }
+ *      b.ori -> { downstreams: Set(site.ori), value: ... }
+ *      c.ori -> { downstreams: Set(a.ori, b.ori), value: ... }
+ */
+export default function AsyncCacheTransform(Base) {
+  return class AsyncCache extends Base {
+    constructor(...args) {
+      super(...args);
+
+      // Expose cache for debugging
+      this.cache = systemCache;
+    }
+
+    get cachePath() {
+      // @ts-ignore
+      return this[cachePathSymbol];
+    }
+
+    cachePathForKey(key) {
+      return key === "."
+        ? this.cachePath
+        : SystemCacheMap.joinPath(this.cachePath, key);
+    }
+
+    async delete(key) {
+      const deleted = await super.delete(key);
+      if (typeof key === "string") {
+        systemCache.delete(this.cachePathForKey(key));
+      }
+      return deleted;
+    }
+
+    async get(key) {
+      if (typeof key !== "string" || key.length === 0) {
+        // Non-string keys and non-empty strings can't be cached
+        return super.get(key);
+      }
+      const cachePath = this.cachePathForKey(key);
+      const value = await systemCache.getOrInsertComputedAsync(
+        cachePath,
+        async () => {
+          let result = await super.get(key);
+          if (result !== undefined) {
+            // @ts-ignore
+            if (this[noCacheSymbol]) {
+              result[noCacheSymbol] = true;
+            } else {
+              result = enableValueCaching(result, cachePath);
+            }
+          }
+          return result;
+        },
+      );
+      return value;
+    }
+
+    invalidateKeys() {
+      const keysPath = this.cachePathForKey("_keys");
+      systemCache.delete(keysPath);
+    }
+
+    async *keys() {
+      const keysPath = this.cachePathForKey("_keys");
+      const keys = await systemCache.getOrInsertComputedAsync(
+        keysPath,
+        async () => {
+          // We can't cache an iterator; convert to array
+          const result = [];
+          for await (const key of super.keys()) {
+            result.push(key);
+          }
+          return result;
+        },
+      );
+      yield* keys;
+    }
+
+    onKeysChange(key) {
+      super.onKeysChange?.(key);
+      this.invalidateKeys();
+    }
+
+    onValueChange(key) {
+      super.onValueChange?.(key);
+      systemCache.delete(this.cachePathForKey(key));
+    }
+
+    async set(key, value) {
+      if (typeof key !== "string") {
+        return super.set(key, value);
+      }
+      systemCache.delete(this.cachePathForKey(key));
+      if (!this.has(key)) {
+        // Adding a new key, need to invalidate cached keys
+        this.invalidateKeys();
+      }
+      super.set(key, value);
+    }
+  };
+}

package/src/runtime/HandleExtensionsTransform.d.ts

@@ -1,5 +1,7 @@
 import { Mixin } from "../../index.ts";
 
-declare const HandleExtensionsTransform: Mixin<{}>;
+declare const HandleExtensionsTransform: Mixin<{
+  initializeGlobals(): Promise<void>;
+}>;
 
 export default HandleExtensionsTransform;

package/src/runtime/HandleExtensionsTransform.js

@@ -1,3 +1,5 @@
+import getGlobalsForTree from "../project/getGlobalsForTree.js";
+import initializeGlobalsForTree from "../project/initializeGlobalsForTree.js";
 import handleExtension from "./handleExtension.js";
 
 /**
@@ -13,11 +15,25 @@ export default function HandleExtensionsTransform(Base) {
       return super.delete(key);
     }
 
+    /**
+     * Initialize the globals on the project root. This makes the file handlers
+     * available for use from any folder inside the tree.
+     *
+     * This is an async operation because it can load JavaScript files, so it
+     * can't be done in the constructor.
+     */
+    async initializeGlobals() {
+      await initializeGlobalsForTree(this);
+    }
+
     get(key) {
+      const globals = getGlobalsForTree(this);
       const value = super.get(key);
       return value instanceof Promise
-        ? value.then((resolved) => handleExtension(resolved, key, this))
-        : handleExtension(value, key, this);
+        ? value.then((resolved) =>
+            handleExtension(resolved, key, globals, this),
+          )
+        : handleExtension(value, key, globals, this);
     }
 
     // See delete()