@weborigami/language 0.6.16 → 0.7.0-beta.1

package/src/project/{projectGlobals.js → initializeGlobalsForTree.js}

@@ -1,10 +1,9 @@
-import { Tree } from "@weborigami/async-tree";
-import assignPropertyDescriptors from "../runtime/assignPropertyDescriptors.js";
+import { Tree, assignPropertyDescriptors } from "@weborigami/async-tree";
 import coreGlobals from "./coreGlobals.js";
 import projectConfig from "./projectConfig.js";
 
 /**
- * Return the complete set of globals available to code running in the given
+ * Make the complete set of globals available to code running in the given
  * container. This will be the core globals plus any configuration specified in
  * the project's config.ori file.
  *
@@ -12,17 +11,13 @@ import projectConfig from "./projectConfig.js";
  * container.
  *
  * @typedef {import("@weborigami/async-tree").SyncOrAsyncMap} SyncOrAsyncMap
- * @param {SyncOrAsyncMap|null} parent
+ * @param {SyncOrAsyncMap} parent
  */
-export default async function projectGlobals(parent) {
-  if (!parent) {
-    return coreGlobals();
-  }
-
-  const projectRoot = await Tree.root(parent);
+export default async function initializeGlobalsForTree(parent) {
+  const projectRoot = Tree.root(parent);
   if (!projectRoot.globals) {
-    // Start with core globals
-    const globals = await coreGlobals();
+    // Start with a copy of the core globals
+    const globals = { ...(await coreGlobals()) };
 
     if (parent) {
       // Get config for the given container and add it to the globals. During
@@ -35,7 +30,7 @@ export default async function projectGlobals(parent) {
       assignPropertyDescriptors(globals, config);
     }
 
-    // Cache globals on project root
+    // Store globals on project root
     projectRoot.globals = globals;
   }
 

package/src/project/jsGlobals.js

@@ -92,6 +92,7 @@ const globals = {
   SuppressedError: globalThis.SuppressedError,
   Symbol,
   SyntaxError,
+  Temporal: globalThis.Temporal,
   TextDecoder,
   TextDecoderStream,
   TextEncoder,

package/src/project/projectConfig.js

@@ -10,8 +10,8 @@ import coreGlobals from "./coreGlobals.js";
  * @typedef {import("@weborigami/async-tree").SyncOrAsyncMap} SyncOrAsyncMap
  * @param {SyncOrAsyncMap} parent
  */
-export default async function config(parent) {
-  const projectRoot = await Tree.root(parent);
+export default async function projectConfig(parent) {
+  const projectRoot = Tree.root(parent);
 
   let configObject = {};
   const configBuffer = await projectRoot.get("config.ori");

package/src/project/projectRootFromPath.js

@@ -55,5 +55,7 @@ export default async function projectRootFromPath(dirname) {
     root = new OrigamiFileMap(dirname);
   }
 
+  await root.initializeGlobals();
+
   return root;
 }

package/src/protocols/constructHref.js

@@ -8,11 +8,11 @@ import { pathFromKeys } from "@weborigami/async-tree";
  * @param  {string[]} keys
  */
 export default function constructHref(protocol, host, ...keys) {
+  let href = host.endsWith("/") ? host.slice(0, -1) : host;
   const path = pathFromKeys(keys);
-  if (host.endsWith("/")) {
-    host = host.slice(0, -1);
+  if (path) {
+    href += "/" + path;
   }
-  let href = [host, path].join("/");
   if (!href.startsWith(protocol)) {
     if (!href.startsWith("//")) {
       href = `//${href}`;

package/src/protocols/constructSiteTree.js

@@ -1,5 +1,6 @@
 import { trailingSlash } from "@weborigami/async-tree";
 import HandleExtensionsTransform from "../runtime/HandleExtensionsTransform.js";
+import systemCache from "../runtime/systemCache.js";
 import constructHref from "./constructHref.js";
 
 /**
@@ -12,7 +13,12 @@ import constructHref from "./constructHref.js";
  * @param {string} host
  * @param  {string[]} keys
  */
-export default function constructSiteTree(protocol, mapClass, host, ...keys) {
+export default async function constructSiteTree(
+  protocol,
+  mapClass,
+  host,
+  ...keys
+) {
   // If the last key doesn't end in a slash, remove it for now.
   let lastKey;
   if (keys.length > 0 && keys.at(-1) && !trailingSlash.has(keys.at(-1))) {
@@ -20,7 +26,10 @@ export default function constructSiteTree(protocol, mapClass, host, ...keys) {
   }
 
   const href = constructHref(protocol, host, ...keys);
-  let result = new (HandleExtensionsTransform(mapClass))(href);
+  const result = await systemCache.getOrInsertComputedAsync(
+    href,
+    () => new (HandleExtensionsTransform(mapClass))(href),
+  );
 
   return lastKey ? result.get(lastKey) : result;
 }

package/src/protocols/explore.js

@@ -8,6 +8,6 @@ import constructSiteTree from "./constructSiteTree.js";
  * @param {string} host
  * @param  {...string} keys
  */
-export default function explore(host, ...keys) {
+export default async function explore(host, ...keys) {
   return constructSiteTree("https:", ExplorableSiteMap, host, ...keys);
 }

package/src/protocols/explorehttp.js

@@ -8,6 +8,6 @@ import constructSiteTree from "./constructSiteTree.js";
  * @param {string} host
  * @param  {...string} keys
  */
-export default function explorehttp(host, ...keys) {
+export default async function explorehttp(host, ...keys) {
   return constructSiteTree("http:", ExplorableSiteMap, host, ...keys);
 }

package/src/protocols/fetchAndHandleExtension.js

@@ -1,18 +1,26 @@
-import handleExtension from "../runtime/handleExtension.js";
+import { args, Tree } from "@weborigami/async-tree";
+import handleExtension from "../../src/runtime/handleExtension.js";
 
 /**
- * Fetch the resource at the given href.
- *
- * @typedef {import("@weborigami/async-tree").SyncOrAsyncMap} SyncOrAsyncMap
+ * Extend the JavaScript `fetch` function to implicity return an ArrayBuffer
+ * with an unpack() method if the resource has a known file extension or MIME
+ * type.
  *
  * @param {string} href
- * @param {SyncOrAsyncMap} parent
  */
-export default async function fetchAndHandleExtension(href, parent) {
-  const response = await fetch(href);
+export default async function fetchAndHandleExtension(href, options, state) {
+  if (options && state === undefined) {
+    // Options weren't provided
+    state = options;
+    options = undefined;
+  }
+
+  href = args.string(href, "Origami.fetch");
+  const response = await fetch(href, options);
   if (!response.ok) {
     return undefined;
   }
+
   let buffer = await response.arrayBuffer();
 
   const mediaType = response.headers.get("Content-Type");
@@ -20,12 +28,16 @@ export default async function fetchAndHandleExtension(href, parent) {
     /** @type {any} */ (buffer).mediaType = mediaType;
   }
 
-  // Attach any loader defined for the file type.
+  // Attach any handler defined for the file type or MIME type.
+  const parent = state?.parent;
   const url = new URL(href);
-  const filename = url.pathname.split("/").pop();
-  if (filename) {
-    buffer = await handleExtension(buffer, filename, parent);
+  if (parent) {
+    const root = await Tree.root(parent);
+    const globals = root.globals;
+    const filename = url.pathname.split("/").pop();
+    buffer = await handleExtension(buffer, filename, globals, parent);
   }
 
   return buffer;
 }
+fetchAndHandleExtension.needsState = true;

package/src/protocols/files.js

@@ -22,6 +22,7 @@ export default async function files(...args) {
   const resolved = path.resolve(basePath, relativePath);
 
   const result = new OrigamiFileMap(resolved);
+  await result.initializeGlobals();
   return result;
 }
 files.needsState = true;

package/src/protocols/http.js

@@ -1,3 +1,4 @@
+import systemCache from "../runtime/systemCache.js";
 import constructHref from "./constructHref.js";
 import fetchAndHandleExtension from "./fetchAndHandleExtension.js";
 
@@ -11,6 +12,8 @@ import fetchAndHandleExtension from "./fetchAndHandleExtension.js";
 export default async function http(host, ...keys) {
   const state = keys.pop();
   const href = constructHref("http:", host, ...keys);
-  return fetchAndHandleExtension(href, state.parent);
+  return systemCache.getOrInsertComputedAsync(href, () =>
+    fetchAndHandleExtension(href, null, state),
+  );
 }
 http.needsState = true;

package/src/protocols/https.js

@@ -1,3 +1,4 @@
+import systemCache from "../runtime/systemCache.js";
 import constructHref from "./constructHref.js";
 import fetchAndHandleExtension from "./fetchAndHandleExtension.js";
 
@@ -11,6 +12,8 @@ import fetchAndHandleExtension from "./fetchAndHandleExtension.js";
 export default async function https(host, ...keys) {
   const state = keys.pop();
   const href = constructHref("https:", host, ...keys);
-  return fetchAndHandleExtension(href, state.parent);
+  return systemCache.getOrInsertComputedAsync(href, () =>
+    fetchAndHandleExtension(href, null, state),
+  );
 }
 https.needsState = true;

package/src/protocols/httpstree.js

@@ -8,6 +8,6 @@ import constructSiteTree from "./constructSiteTree.js";
  * @param {string} host
  * @param  {...string} keys
  */
-export default function httpstree(host, ...keys) {
+export default async function httpstree(host, ...keys) {
   return constructSiteTree("https:", SiteMap, host, ...keys);
 }

package/src/protocols/httptree.js

@@ -8,6 +8,6 @@ import constructSiteTree from "./constructSiteTree.js";
  * @param {string} host
  * @param  {...string} keys
  */
-export default function httptree(host, ...keys) {
+export default async function httptree(host, ...keys) {
   return constructSiteTree("http:", SiteMap, host, ...keys);
 }

package/src/protocols/package.js

@@ -1,5 +1,6 @@
 import { Tree, keysFromPath, pathFromKeys } from "@weborigami/async-tree";
 import projectRoot from "../project/projectRoot.js";
+import systemCache from "../runtime/systemCache.js";
 
 /**
  * The package: protocol handler
@@ -9,15 +10,27 @@ import projectRoot from "../project/projectRoot.js";
 export default async function packageProtocol(...args) {
   const state = args.pop(); // Remaining args are the path
   const root = await projectRoot(state);
+  const path = pathFromKeys(args);
+  if (!path) {
+    throw new Error("package: protocol requires a package name");
+  }
+  const href = `package:${path}`;
+  const result = await systemCache.getOrInsertComputedAsync(href, () =>
+    loadPackage(root, args),
+  );
+  return result;
+}
+packageProtocol.needsState = true;
 
+async function loadPackage(root, args) {
   // Identify the path to the package root
-  const packageRootKeys = ["node_modules"];
+  const packageRootKeys = ["node_modules/"];
   let name = args.shift();
   packageRootKeys.push(name);
   if (name.startsWith("@")) {
     // First key is an npm organization, add next key as name
     const nameArg = args.shift();
-    name += nameArg;
+    name = pathFromKeys([name, nameArg]);
     packageRootKeys.push(nameArg);
   }
   const packageRootPath = pathFromKeys(packageRootKeys);
@@ -60,4 +73,3 @@ export default async function packageProtocol(...args) {
 
   return result;
 }
-packageProtocol.needsState = true;

package/src/runtime/AsyncCacheTransform.d.ts

@@ -0,0 +1,5 @@
+import { Mixin } from "../../index.ts";
+
+declare const AsyncCacheTransform: Mixin<{}>
+
+export default AsyncCacheTransform;

package/src/runtime/AsyncCacheTransform.js

@@ -0,0 +1,134 @@
+import enableValueCaching from "./enableValueCaching.js";
+import { cachePathSymbol, noCacheSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
+import SystemCacheMap from "./SystemCacheMap.js";
+
+/**
+ * General-purpose mixin for Origami maps with dependency tracking, used for:
+ * files, site resources, and scope references in Origami files
+ *
+ * This wraps a map's get() and keys() methods to add caching and dependency tracking.
+ * It tracks which cached values are downstream of other cached values so that if
+ * an upstream value changes, all dependent downstream cached values can be
+ * invalidated efficiently.
+ *
+ * Cache entries look like:
+ *
+ *      key -> {
+ *        downstreams: Set(path),
+ *        value
+ *      }
+ *
+ * This allows for efficiently evicting all a value and all its downstream
+ * dependent cached values.
+ *
+ * Example project:
+ *
+ *     site.ori loads a.ori and b.ori
+ *     a.ori loads c.ori
+ *     b.ori loads c.ori
+ *     c.ori doesn't load anything
+ *
+ * Resulting cache:
+ *
+ *      site.ori -> { value: ... }
+ *      a.ori -> { downstreams: Set(site.ori), value: ... }
+ *      b.ori -> { downstreams: Set(site.ori), value: ... }
+ *      c.ori -> { downstreams: Set(a.ori, b.ori), value: ... }
+ */
+export default function AsyncCacheTransform(Base) {
+  return class AsyncCache extends Base {
+    constructor(...args) {
+      super(...args);
+
+      // Expose cache for debugging
+      this.cache = systemCache;
+    }
+
+    get cachePath() {
+      // @ts-ignore
+      return this[cachePathSymbol];
+    }
+
+    cachePathForKey(key) {
+      return key === "."
+        ? this.cachePath
+        : SystemCacheMap.joinPath(this.cachePath, key);
+    }
+
+    async delete(key) {
+      const deleted = await super.delete(key);
+      if (typeof key === "string") {
+        systemCache.delete(this.cachePathForKey(key));
+      }
+      return deleted;
+    }
+
+    async get(key) {
+      if (typeof key !== "string" || key.length === 0) {
+        // Non-string keys and non-empty strings can't be cached
+        return super.get(key);
+      }
+      const cachePath = this.cachePathForKey(key);
+      const value = await systemCache.getOrInsertComputedAsync(
+        cachePath,
+        async () => {
+          let result = await super.get(key);
+          if (result !== undefined) {
+            // @ts-ignore
+            if (this[noCacheSymbol]) {
+              result[noCacheSymbol] = true;
+            } else {
+              result = enableValueCaching(result, cachePath);
+            }
+          }
+          return result;
+        },
+      );
+      return value;
+    }
+
+    invalidateKeys() {
+      const keysPath = this.cachePathForKey("_keys");
+      systemCache.delete(keysPath);
+    }
+
+    async *keys() {
+      const keysPath = this.cachePathForKey("_keys");
+      const keys = await systemCache.getOrInsertComputedAsync(
+        keysPath,
+        async () => {
+          // We can't cache an iterator; convert to array
+          const result = [];
+          for await (const key of super.keys()) {
+            result.push(key);
+          }
+          return result;
+        },
+      );
+      yield* keys;
+    }
+
+    onKeysChange(key) {
+      super.onKeysChange?.(key);
+      this.invalidateKeys();
+    }
+
+    onValueChange(key) {
+      super.onValueChange?.(key);
+      systemCache.delete(this.cachePathForKey(key));
+    }
+
+    async set(key, value) {
+      if (typeof key !== "string") {
+        return super.set(key, value);
+      }
+      systemCache.delete(this.cachePathForKey(key));
+      if (!this.has(key)) {
+        // Adding a new key, need to invalidate cached keys
+        this.invalidateKeys();
+      }
+      super.set(key, value);
+    }
+  };
+}

package/src/runtime/HandleExtensionsTransform.d.ts

@@ -1,5 +1,7 @@
 import { Mixin } from "../../index.ts";
 
-declare const HandleExtensionsTransform: Mixin<{}>;
+declare const HandleExtensionsTransform: Mixin<{
+  initializeGlobals(): Promise<void>;
+}>;
 
 export default HandleExtensionsTransform;

package/src/runtime/HandleExtensionsTransform.js

@@ -1,3 +1,5 @@
+import getGlobalsForTree from "../project/getGlobalsForTree.js";
+import initializeGlobalsForTree from "../project/initializeGlobalsForTree.js";
 import handleExtension from "./handleExtension.js";
 
 /**
@@ -13,11 +15,25 @@ export default function HandleExtensionsTransform(Base) {
       return super.delete(key);
     }
 
+    /**
+     * Initialize the globals on the project root. This makes the file handlers
+     * available for use from any folder inside the tree.
+     *
+     * This is an async operation because it can load JavaScript files, so it
+     * can't be done in the constructor.
+     */
+    async initializeGlobals() {
+      await initializeGlobalsForTree(this);
+    }
+
     get(key) {
+      const globals = getGlobalsForTree(this);
       const value = super.get(key);
       return value instanceof Promise
-        ? value.then((resolved) => handleExtension(resolved, key, this))
-        : handleExtension(value, key, this);
+        ? value.then((resolved) =>
+            handleExtension(resolved, key, globals, this),
+          )
+        : handleExtension(value, key, globals, this);
     }
 
     // See delete()

package/src/runtime/OrigamiFileMap.d.ts

@@ -2,8 +2,11 @@ import { FileMap } from "@weborigami/async-tree";
 import EventTargetMixin from "./EventTargetMixin.js";
 import HandleExtensionsTransform from "./HandleExtensionsTransform.js";
 import ImportModulesMixin from "./ImportModulesMixin.js";
+import SyncCacheTransform from "./SyncCacheTransform.js";
 import WatchFilesMixin from "./WatchFilesMixin.js";
 
-export default class OrigamiFileMap extends HandleExtensionsTransform(
+export default class OrigamiFileMap extends SyncCacheTransform(HandleExtensionsTransform(
   ImportModulesMixin(WatchFilesMixin(EventTargetMixin(FileMap)))
-) {}
+)) {
+  globals: any;
+}

package/src/runtime/OrigamiFileMap.js

@@ -1,9 +1,32 @@
-import { FileMap } from "@weborigami/async-tree";
+import { FileMap, trailingSlash } from "@weborigami/async-tree";
 import EventTargetMixin from "./EventTargetMixin.js";
 import HandleExtensionsTransform from "./HandleExtensionsTransform.js";
 import ImportModulesMixin from "./ImportModulesMixin.js";
+import SyncCacheTransform from "./SyncCacheTransform.js";
 import WatchFilesMixin from "./WatchFilesMixin.js";
+import { noCacheSymbol } from "./symbols.js";
 
-export default class OrigamiFileMap extends HandleExtensionsTransform(
-  ImportModulesMixin(WatchFilesMixin(EventTargetMixin(FileMap)))
-) {}
+export default class OrigamiFileMap extends SyncCacheTransform(
+  HandleExtensionsTransform(
+    ImportModulesMixin(WatchFilesMixin(EventTargetMixin(FileMap))),
+  ),
+) {
+  get cachePath() {
+    return super.cachePath ?? this.path;
+  }
+
+  // Workaround to register file paths in the system cache without trailing
+  // slahes. This is so that if someone calls `get("site.ori/")`, the cache path
+  // will be "site.ori". It's not clear whether this is the best solution, but
+  // hopefully suffices for now.
+  cachePathForKey(key) {
+    const normalized = trailingSlash.remove(key);
+    return super.cachePathForKey(normalized);
+  }
+
+  globals = null;
+
+  // Don't cache files, just record their dependencies. The OS already caches
+  // files, so caching them just consumes memory and may slow things down.
+  [noCacheSymbol] = true;
+}

package/src/runtime/ScopeMap.js

@@ -0,0 +1,72 @@
+import { SyncMap } from "@weborigami/async-tree";
+import path from "node:path";
+import { cachePathSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
+
+/**
+ * Return a sync map of all values in scope for the given sync source map.
+ *
+ * @param {SyncMap} source
+ */
+export default class ScopeMap extends SyncMap {
+  constructor(source) {
+    super();
+    this.source = source;
+    this.trailingSlashKeys = source.trailingSlashKeys;
+  }
+
+  get(key) {
+    let value;
+
+    // Starting with this map, search up its parent hierarchy
+    let current = this.source;
+    while (current) {
+      // If the keys of this folder change, the scope request for this key could
+      // return a different value, so a change in keys needs to invalidate the
+      // value. Whether or not the get() request below succeeds, track the keys
+      // of this folder as an upstream dependency of the value being requested.
+      if (current[cachePathSymbol]) {
+        const folderKeysPath = path.join(current[cachePathSymbol], "_keys");
+        systemCache.trackCurrentDependency(folderKeysPath);
+      }
+
+      value = current.get(key);
+      if (value !== undefined) {
+        break;
+      }
+      current = current.parent;
+    }
+
+    return value;
+  }
+
+  // Collect all keys for this tree and all parents
+  keys() {
+    const scopeKeys = new Set();
+    let current = this.source;
+    while (current) {
+      for (const key of current.keys()) {
+        scopeKeys.add(key);
+      }
+      current = current.parent;
+    }
+    return scopeKeys[Symbol.iterator]();
+  }
+
+  // Collect all keys for this tree and all parents.
+  //
+  // This method exists for debugging purposes, as it's helpful to be able to
+  // quickly flatten and view the entire scope chain.
+  get trees() {
+    const result = [];
+
+    /** @type {SyncMap|null} */
+    let current = this.source;
+    while (current) {
+      result.push(current);
+      current = "parent" in current ? current.parent : null;
+    }
+
+    return result;
+  }
+}

package/src/runtime/SyncCacheTransform.d.ts

@@ -0,0 +1,8 @@
+import { Mixin } from "../../index.ts";
+
+declare const SyncCacheTransform: Mixin<{
+  get cachePath(): string;
+  cachePathForKey(key: string): string;
+}>
+
+export default SyncCacheTransform;