@weborigami/language 0.6.16 → 0.7.0-beta.1

package/src/runtime/execute.js

@@ -1,5 +1,5 @@
 import { isUnpackable, Tree } from "@weborigami/async-tree";
-import asyncStorage from "./asyncStorage.js";
+import executionContext from "./executionContext.js";
 import "./interop.js";
 
 /**
@@ -68,7 +68,7 @@ export default async function execute(code, state = {}) {
   // Execute the function or traverse the map.
   let result;
   try {
-    result = await asyncStorage.run(
+    result = await executionContext.run(
       context,
       async () =>
         fn instanceof Function

package/src/runtime/executionContext.js

@@ -0,0 +1,7 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+
+/**
+ * The execute() function's context made available to async functions called
+ * during evaluation.
+ */
+export default new AsyncLocalStorage();

package/src/runtime/explainReferenceError.js

@@ -39,8 +39,13 @@ export default async function explainReferenceError(code, state) {
   let key;
   if (code[0] === ops.cache) {
     // External scope reference
-    const scopeCall = code[3].slice(1); // drop the ops.scope
-    const keys = scopeCall.map((part) => part[1]);
+    let refCall = code[2];
+    if (refCall[0] === ops.unpack) {
+      // Unwrap implied unpack
+      refCall = refCall[1];
+    }
+    const scopeArgs = refCall.slice(1); // get the scope reference arguments
+    const keys = scopeArgs.map((part) => part[1]);
     const path = pathFromKeys(keys);
 
     if (keys.length > 1) {

package/src/runtime/expressionObject.js

@@ -6,9 +6,13 @@ import {
   trailingSlash,
   Tree,
 } from "@weborigami/async-tree";
+import enableValueCaching from "./enableValueCaching.js";
 import execute from "./execute.js";
 import handleExtension from "./handleExtension.js";
 import { ops } from "./internal.js";
+import { cachePathSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
+import SystemCacheMap from "./SystemCacheMap.js";
 
 export const KEY_TYPE = {
   STRING: 0, // Simple string key: `a: 1`
@@ -36,6 +40,7 @@ export default async function expressionObject(entries, state = {}) {
   if (parent !== null && !Tree.isMap(parent)) {
     throw new TypeError(`Parent must be a map or null`);
   }
+
   setParent(object, parent);
 
   // The object in Map form for use on the stack
@@ -51,17 +56,7 @@ export default async function expressionObject(entries, state = {}) {
     }
   }
 
-  // Second pass: redefine eager string-keyed properties with actual values.
-  for (const info of infos) {
-    if (
-      info.keyType === KEY_TYPE.STRING &&
-      info.valueType === VALUE_TYPE.EAGER
-    ) {
-      await redefineProperty(object, info);
-    }
-  }
-
-  // Third pass: define all computed properties. These may refer to the
+  // Second pass: define all computed properties. These may refer to the
   // properties we just defined.
   for (const info of infos) {
     if (info.keyType === KEY_TYPE.COMPUTED) {
@@ -73,15 +68,11 @@ export default async function expressionObject(entries, state = {}) {
     }
   }
 
-  // Fourth pass: redefine eager computed-keyed properties with actual values.
-  for (const info of infos) {
-    if (
-      info.keyType === KEY_TYPE.COMPUTED &&
-      info.valueType === VALUE_TYPE.EAGER
-    ) {
-      await redefineProperty(object, info);
-    }
-  }
+  // Third pass: retrieve eager properties, memoizing them on the object
+  const eagerKeys = infos
+    .filter((info) => info.valueType === VALUE_TYPE.EAGER)
+    .map((info) => info.key);
+  await Promise.all(eagerKeys.map((key) => object[key]));
 
   // Attach a keys method, where keys for primitive/eager properties with
   // maplike values get a trailing slash.
@@ -95,16 +86,6 @@ export default async function expressionObject(entries, state = {}) {
     writable: true,
   });
 
-  // TODO: If there are any getters, mark the object as async. Note: this code
-  // was added so that Tree.from() could know whether to return an ObjectMap or
-  // a hypothetical AsyncObjectMap, which in turn would let a map operation know
-  // whether to expect async property values. const hasGetters =
-  // infos.some((info) => info.valueType === VALUE_TYPE.GETTER); if (hasGetters)
-  // { Object.defineProperty(object, symbols.async, { configurable: true,
-  // enumerable: false, value: true, writable: true,
-  //   });
-  // }
-
   return object;
 }
 
@@ -112,6 +93,7 @@ export default async function expressionObject(entries, state = {}) {
  * Define a single property on the object
  */
 function defineProperty(object, propertyInfo, state, map) {
+  const { globals } = state;
   let { enumerable, hasExtension, key, value, valueType } = propertyInfo;
   if (valueType == VALUE_TYPE.PRIMITIVE) {
     // Define simple property
@@ -127,14 +109,54 @@ function defineProperty(object, propertyInfo, state, map) {
       configurable: true,
       enumerable,
       get: async () => {
+        // Execute the code to get the value of the property
+        const propertyCachePath = getPropertyCachePath(object, key);
+
         const newState = Object.assign({}, state, { object: map });
-        const result = await execute(value, newState);
-        return hasExtension ? handleExtension(result, key, map) : result;
+        let result = propertyCachePath
+          ? await systemCache.getOrInsertComputedAsync(propertyCachePath, () =>
+              execute(value, newState),
+            )
+          : await execute(value, newState);
+
+        if (hasExtension) {
+          // Handle extension
+          result = handleExtension(result, key, globals, map);
+        }
+
+        if (valueType === VALUE_TYPE.EAGER) {
+          // Memoize result on the object itself
+          Object.defineProperty(object, key, {
+            configurable: true,
+            enumerable,
+            value: result,
+            writable: true,
+          });
+        } else if (propertyCachePath) {
+          result = enableValueCaching(result, propertyCachePath);
+        }
+
+        return result;
       },
     });
   }
 }
 
+function getPropertyCachePath(object, key) {
+  // Follow parent chain looking for a parent that has caching enabled
+  let current = object;
+  while (current[cachePathSymbol] === undefined) {
+    current = current[symbols.parent];
+    if (!current) {
+      // Caching isn't enabled on this object tree
+      return null;
+    }
+  }
+
+  const cachePath = SystemCacheMap.joinPath(current[cachePathSymbol], key);
+  return cachePath;
+}
+
 /**
  * Return a normalized version of the property key for use in the keys() method.
  * Among other things, this adds trailing slashes to keys that correspond to
@@ -227,17 +249,3 @@ export function propertyInfo(key, value) {
 
   return { enumerable, hasExtension, key, keyType, value, valueType };
 }
-
-/**
- * Get the value of the indicated eager property and overwrite the property
- * definition with the actual value.
- */
-async function redefineProperty(object, info) {
-  const value = await object[info.key];
-  Object.defineProperty(object, info.key, {
-    configurable: true,
-    enumerable: info.enumerable,
-    value,
-    writable: true,
-  });
-}

package/src/runtime/handleExtension.js

@@ -8,7 +8,10 @@ import {
   trailingSlash,
 } from "@weborigami/async-tree";
 import getPackedPath from "../handlers/getPackedPath.js";
-import projectGlobals from "../project/projectGlobals.js";
+import mediaTypeExtensions from "../handlers/mediaTypeExtensions.json" with { type: "json" };
+import { cachePathSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
+import SystemCacheMap from "./SystemCacheMap.js";
 
 /**
  * If the given value is packed (e.g., buffer) and the key is a string-like path
@@ -17,28 +20,33 @@ import projectGlobals from "../project/projectGlobals.js";
  *
  * @param {any} value
  * @param {any} key
+ * @param {any} handlers
  * @param {import("@weborigami/async-tree").SyncOrAsyncMap|null} [parent]
  */
-export default async function handleExtension(value, key, parent = null) {
-  if (isPacked(value) && isStringlike(key) && value.unpack === undefined) {
-    const hasSlash = trailingSlash.has(key);
-    if (hasSlash) {
-      key = trailingSlash.remove(key);
-    }
+export default function handleExtension(value, key, handlers, parent = null) {
+  if (
+    isPacked(value) &&
+    isStringlike(key) &&
+    value.unpack === undefined &&
+    handlers
+  ) {
+    const normalized = trailingSlash.remove(key);
 
     // Special cases: `.ori.<ext>` extensions are Origami documents
-    const extname = key.match(/\.ori\.\S+$/)
+    let extname = normalized.match(/\.ori\.\S+$/)
       ? ".oridocument"
-      : extension.extname(key);
+      : extension.extname(normalized);
+
+    if (!extname && /** @type {any} */ (value)?.mediaType) {
+      extname = extensionFromMediaType(/** @type {any} */ (value).mediaType);
+    }
+
     if (extname) {
       const handlerName = `${extname.slice(1)}_handler`;
-      const handlers = await projectGlobals(parent);
-      let handler = await handlers[handlerName];
-      if (handler) {
-        if (isUnpackable(handler)) {
-          // The extension handler itself needs to be unpacked
-          handler = await handler.unpack();
-        }
+      // Use `in` to look for handle so that, if the handler is a promise, we
+      // can still find it without awaiting it here.
+      if (handlerName in handlers) {
+        let handler = handlers[handlerName];
 
         // If the value is a primitive, box it so we can attach data to it.
         value = box(value);
@@ -51,9 +59,45 @@ export default async function handleExtension(value, key, parent = null) {
           setParent(value, parent);
         }
 
-        if (handler.unpack) {
-          value.unpack = wrapUnpack(handler.unpack, value, key, parent);
+        // Wrap the unpack function so it caches the unpacked value, and so we
+        // can add the file path to any errors the unpack function throws.
+        const filePath = getPackedPath(value, { key: normalized, parent });
+        let fileCachePath;
+        if (parent?.[cachePathSymbol]) {
+          fileCachePath = SystemCacheMap.joinPath(
+            parent[cachePathSymbol],
+            normalized,
+          );
+        } else {
+          fileCachePath = filePath;
         }
+        const unpackCachePath = trailingSlash.add(fileCachePath);
+        value.unpack = async () =>
+          systemCache.getOrInsertComputedAsync(unpackCachePath, async () => {
+            if (handler instanceof Promise) {
+              handler = await handler;
+            }
+            if (isUnpackable(handler)) {
+              // The extension handler itself needs to be unpacked
+              handler = await handler.unpack();
+            }
+
+            const unpacked = await handler.unpack(value, {
+              key: normalized,
+              parent,
+            });
+
+            // Now that we know the file was unpacked, we cache the file value
+            // itself so that subsequent requests for the file can be fulfilled
+            // from the cache. Doing this sort of manipulation outside of the
+            // cache doesn't feel great, but works.
+            const fileCacheEntry = systemCache.get(fileCachePath);
+            if (fileCacheEntry) {
+              fileCacheEntry.value = value;
+            }
+
+            return unpacked;
+          });
       }
     }
   }
@@ -61,20 +105,7 @@ export default async function handleExtension(value, key, parent = null) {
   return value;
 }
 
-// Wrap the unpack function so it's only called once per value, and so we can
-// add the file path to any errors it throws.
-function wrapUnpack(unpack, value, key, parent) {
-  let result;
-  return async () => {
-    if (!result) {
-      try {
-        result = await unpack(value, { key, parent });
-      } catch (/** @type {any} */ error) {
-        const filePath = getPackedPath(value, { key, parent });
-        const message = `Can't unpack ${filePath}\n${error.message}`;
-        throw new error.constructor(message, { cause: error });
-      }
-    }
-    return result;
-  };
+function extensionFromMediaType(mediaType) {
+  const essence = mediaType.split(";")[0].trim();
+  return mediaTypeExtensions[essence];
 }

package/src/runtime/interop.js

@@ -1,13 +1,13 @@
 import { interop } from "@weborigami/async-tree";
-import asyncStorage from "./asyncStorage.js";
 import { lineInfo } from "./errors.js";
+import executionContext from "./executionContext.js";
 
 /**
  * Inject our warning function into async-tree calls
  */
 interop.warn = function warn(...args) {
   console.warn(...args);
-  const context = asyncStorage.getStore();
+  const context = executionContext.getStore();
   const location = context?.code?.location;
   if (location) {
     console.warn(lineInfo(location));

package/src/runtime/mergeTrees.js

@@ -1,11 +1,11 @@
 import {
+  assignPropertyDescriptors,
   isPlainObject,
   isUnpackable,
   symbols,
   trailingSlash,
   Tree,
 } from "@weborigami/async-tree";
-import assignPropertyDescriptors from "./assignPropertyDescriptors.js";
 
 /**
  * Create a tree that's the result of merging the given trees.

package/src/runtime/ops.js

@@ -6,13 +6,15 @@
  * @typedef {import("@weborigami/async-tree").SyncOrAsyncMap} SyncOrAsyncMap
  */
 
-import { getParent, isUnpackable, Tree } from "@weborigami/async-tree";
+import { getParent, isUnpackable, SyncMap, Tree } from "@weborigami/async-tree";
 import os from "node:os";
 import execute from "./execute.js";
 import expressionObject from "./expressionObject.js";
 import mergeTrees from "./mergeTrees.js";
 import OrigamiFileMap from "./OrigamiFileMap.js";
+import ScopeMap from "./ScopeMap.js";
 import { codeSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
 
 function addOpLabel(op, label) {
   Object.defineProperty(op, "toString", {
@@ -81,31 +83,18 @@ addOpLabel(bitwiseXor, "«ops.bitwiseXor»");
 /**
  * Cache the value of the code for an external reference
  *
- * @param {any} cache
- * @param {string} path
+ * @param {string} cachePath
  * @param {AnnotatedCode} code
+ * @param {RuntimeState} state
  */
-export async function cache(cache, path, code) {
-  if (path in cache) {
-    // Cache hit
-    return cache[path];
-  }
-
-  // Don't await: might get another request for this before promise resolves
-  const promise = execute(code);
-
-  // Save promise so another request will get the same promise
-  cache[path] = promise;
-
-  // Now wait for the value
-  const value = await promise;
-
-  // Update the cache with the actual value
-  cache[path] = value;
-
-  return value;
+export function cache(cachePath, code, state) {
+  const result = systemCache.getOrInsertComputedAsync(cachePath, () =>
+    execute(code, state),
+  );
+  return result;
 }
 addOpLabel(cache, "«ops.cache»");
+cache.needsState = true;
 cache.unevaluatedArgs = true;
 
 /**
@@ -114,13 +103,17 @@ cache.unevaluatedArgs = true;
  * @param  {...AnnotatedCode} args
  */
 export async function comma(...args) {
+  /** @type {RuntimeState} */
+  // @ts-ignore
+  const state = args.pop(); // The runtime state is passed as the last argument
   let result;
   for (const arg of args) {
-    result = await execute(arg);
+    result = await execute(arg, state);
   }
   return result;
 }
 addOpLabel(comma, "«ops.comma»");
+comma.needsState = true;
 comma.unevaluatedArgs = true;
 
 export async function conditional(condition, truthy, falsy) {
@@ -143,7 +136,7 @@ export async function construct(constructor, ...args) {
 export async function deepText(...args) {
   return Tree.deepText(args);
 }
-addOpLabel(deepText, "«ops.deepText");
+addOpLabel(deepText, "«ops.deepText»");
 
 /**
  * Default value for a parameter: if the value is defined, return that;
@@ -180,8 +173,7 @@ addOpLabel(exponentiation, "«ops.exponentiation»");
 export async function flat(...args) {
   // Unpack packed arguments so they can be flattened
   const unpacked = args.map((arg) => (isUnpackable(arg) ? arg.unpack() : arg));
-  // Flatten to depth 2: 1 for args array, 1 for flattening
-  return Tree.flat(unpacked, 2);
+  return Tree.flat(unpacked);
 }
 addOpLabel(flat, "«ops.flat»");
 
@@ -206,6 +198,7 @@ addOpLabel(greaterThanOrEqual, "«ops.greaterThanOrEqual»");
  */
 export async function homeDirectory(...keys) {
   const tree = new OrigamiFileMap(os.homedir());
+  await tree.initializeGlobals();
   return keys.length > 0 ? Tree.traverse(tree, ...keys) : tree;
 }
 addOpLabel(homeDirectory, "«ops.homeDirectory»");
@@ -488,7 +481,8 @@ export async function property(object, key) {
 
   if (isUnpackable(object)) {
     object = await object.unpack();
-  } else if (typeof object === "string") {
+  }
+  if (typeof object === "string") {
     object = new String(object);
   } else if (typeof object === "number") {
     object = new Number(object);
@@ -529,17 +523,18 @@ addOpLabel(rootDirectory, "«ops.rootDirectory»");
 /**
  * Return the scope of the current tree
  *
- * @param {SyncOrAsyncMap} parent
+ * @param {RuntimeState} state
  */
-export async function scope(parent) {
+export async function scope(state = {}) {
+  const { parent } = state;
   if (!parent) {
-    throw new ReferenceError(
-      "Tried to find a value in scope, but no container was provided as the parent.",
-    );
+    return new SyncMap(); // empty scope if there's no parent
   }
-  return Tree.scope(parent);
+  const scopeMap = new ScopeMap(parent);
+  return scopeMap;
 }
 addOpLabel(scope, "«ops.scope»");
+scope.needsState = true;
 
 export function shiftLeft(a, b) {
   return a << b;

package/src/runtime/symbols.js

@@ -1,4 +1,7 @@
+export const cachePathSymbol = Symbol("cachePath");
 export const codeSymbol = Symbol("code");
 export const configSymbol = Symbol("config");
+export const noCacheSymbol = Symbol("noCache");
 export const scopeSymbol = Symbol("scope");
 export const sourceSymbol = Symbol("source");
+export const volatileSymbol = Symbol("volatile");

package/src/runtime/systemCache.js

@@ -0,0 +1,3 @@
+import SystemCacheMap from "./SystemCacheMap.js";
+
+export default new SystemCacheMap();

package/src/runtime/volatile.js

@@ -0,0 +1,14 @@
+import { box } from "@weborigami/async-tree";
+import { volatileSymbol } from "./symbols.js";
+
+/**
+ * Mark the indicated value as volatile so it won't be cached.
+ */
+export default function volatile(value) {
+  const boxed = box(value);
+  Object.defineProperty(boxed, volatileSymbol, {
+    value: true,
+    enumerable: false,
+  });
+  return boxed;
+}

package/test/compiler/codeHelpers.js

@@ -23,11 +23,12 @@ export function assertCodeLocations(code) {
  */
 export function createCode(array) {
   const code = array.map((item) =>
-    item instanceof Array ? createCode(item) : item
+    item instanceof Array ? createCode(item) : item,
   );
   /** @type {any} */ (code).location = {
     end: 0,
     source: {
+      relativePath: "test.ori",
       text: "",
     },
     start: 0,