@weborigami/language 0.7.0-beta.1 → 0.7.0-beta.3

package/src/runtime/SyncCacheTransform.js

@@ -45,28 +45,25 @@ export default function SyncCacheTransform(Base) {
       this.cache = systemCache;
     }
 
-    get cachePath() {
-      // @ts-ignore
-      return this[cachePathSymbol];
-    }
-
     cachePathForKey(key) {
       return key === "."
-        ? this.cachePath
-        : SystemCacheMap.joinPath(this.cachePath, key);
+        ? // @ts-ignore
+          this[cachePathSymbol]
+        : // @ts-ignore
+          SystemCacheMap.joinPath(this[cachePathSymbol], key);
     }
 
-    delete(key) {
-      const deleted = super.delete(key);
-      if (typeof key === "string") {
-        systemCache.delete(this.cachePathForKey(key));
-        if (deleted) {
-          // Deleted an existing key, need to invalidate cached keys
-          this.invalidateKeys();
-        }
-      }
-      return deleted;
-    }
+    // delete(key) {
+    //   const deleted = super.delete(key);
+    //   if (typeof key === "string") {
+    //     systemCache.delete(this.cachePathForKey(key));
+    //     if (deleted) {
+    //       // Deleted an existing key, need to invalidate cached keys
+    //       this.invalidateKeys();
+    //     }
+    //   }
+    //   return deleted;
+    // }
 
     get(key) {
       if (typeof key !== "string" || key.length === 0) {
@@ -113,21 +110,21 @@ export default function SyncCacheTransform(Base) {
       systemCache.delete(this.cachePathForKey(key));
     }
 
-    set(key, value) {
-      if (!this._self) {
-        // Initializing in constructor
-        super.set(key, value);
-        return;
-      }
-      if (typeof key !== "string") {
-        return super.set(key, value);
-      }
-      systemCache.delete(this.cachePathForKey(key));
-      if (!this.has(key)) {
-        // Adding a new key, need to invalidate cached keys
-        this.invalidateKeys();
-      }
-      super.set(key, value);
-    }
+    // set(key, value) {
+    //   if (!this._self) {
+    //     // Initializing in constructor
+    //     super.set(key, value);
+    //     return;
+    //   }
+    //   if (typeof key !== "string") {
+    //     return super.set(key, value);
+    //   }
+    //   systemCache.delete(this.cachePathForKey(key));
+    //   if (!this.has(key)) {
+    //     // Adding a new key, need to invalidate cached keys
+    //     this.invalidateKeys();
+    //   }
+    //   super.set(key, value);
+    // }
   };
 }

package/src/runtime/SystemCacheMap.js

@@ -28,15 +28,16 @@ let nextPathId = 0;
 export default class SystemCacheMap extends SyncMap {
   delete(path) {
     // Find all entries that depend on this path directly or indirectly
-    const toDelete = this.dependentEntries(path);
+    const dependentEntries = new Map();
+    this.visitDependentEntries(path, dependentEntries);
 
     // Delete those dependent entries
-    for (const deletePath of toDelete.keys()) {
+    for (const deletePath of dependentEntries.keys()) {
       super.delete(deletePath);
     }
 
     // Remove deleted entries as being downstream from still-existing entries
-    for (const [deletePath, deleteEntry] of toDelete.entries()) {
+    for (const [deletePath, deleteEntry] of dependentEntries.entries()) {
       for (const upstreamPath of deleteEntry.upstreams ?? []) {
         const upstreamEntry = this.get(upstreamPath);
         if (upstreamEntry?.downstreams) {
@@ -44,6 +45,14 @@ export default class SystemCacheMap extends SyncMap {
           if (upstreamEntry.downstreams.size === 0) {
             // No more downstream dependencies, clean up entry
             delete upstreamEntry.downstreams;
+            if (upstreamEntry.upstreams?.size === 0) {
+              // No more upstream dependencies either, clean up entry
+              delete upstreamEntry.upstreams;
+              if (!("value" in upstreamEntry)) {
+                // No more data in this entry, delete it
+                super.delete(upstreamPath);
+              }
+            }
           }
         }
       }
@@ -53,36 +62,38 @@ export default class SystemCacheMap extends SyncMap {
   }
 
   // Return all entries that directly or indirectly depend on the given path
-  dependentEntries(path) {
-    const result = new Map();
+  visitDependentEntries(path, dependentEntries) {
+    // The entries that will be added for the given path
+    const pathAdditions = new Map();
 
     const entry = this.get(path);
     if (entry) {
       // Path itself has an entry
-      result.set(path, entry);
+      pathAdditions.set(path, entry);
     }
 
     // Add all entries with child paths that implicitly depend on this entry
+    // REVIEW: This is expensive: it iterates through the entire cache, and is
+    // done for each node visit.
     for (const [otherPath, otherEntry] of this.entries()) {
       if (this.isChildPath(path, otherPath)) {
-        result.set(otherPath, otherEntry);
+        pathAdditions.set(otherPath, otherEntry);
       }
     }
 
-    // For each entry, add all entries downstream of it
-    for (const entry of result.values()) {
-      for (const downstreamPath of entry.downstreams ?? []) {
-        if (!result.has(downstreamPath)) {
-          for (const [key, value] of this.dependentEntries(downstreamPath)) {
-            if (!result.has(key)) {
-              result.set(key, value);
-            }
+    // Add all the direct dependencies
+    for (const [dependentPath, dependentEntry] of pathAdditions.entries()) {
+      if (!dependentEntries.has(dependentPath)) {
+        // Didn't already have this path in the result set
+        dependentEntries.set(dependentPath, dependentEntry);
+        // Visit all entries downstream of this one
+        for (const downstreamPath of dependentEntry.downstreams ?? []) {
+          if (!dependentEntries.has(downstreamPath)) {
+            this.visitDependentEntries(downstreamPath, dependentEntries);
           }
         }
       }
     }
-
-    return result;
   }
 
   // REVIEW: This doesn't have the correct signature for getOrInsertComputed,

package/src/runtime/counters.js

@@ -0,0 +1,3 @@
+export default {
+  executions: 0,
+};

package/src/runtime/enableValueCaching.js

@@ -19,14 +19,15 @@ const AsyncFunction = async function () {}.constructor;
  * @param {string} cachePath
  */
 export default function enableValueCaching(value, cachePath) {
-  if (!(typeof value === "object" || typeof value === "function")) {
-    // Don't need to apply caching to primitive value
-    return value;
-  }
+  // Value must be a defined object or function, and not have already have a
+  // cache path, and be maplike.
+  const cachable =
+    value &&
+    (typeof value === "object" || typeof value === "function") &&
+    value[cachePathSymbol] === undefined &&
+    Tree.isMaplike(value);
 
-  const cacheable =
-    value[cachePathSymbol] === undefined && Tree.isMaplike(value);
-  if (cacheable) {
+  if (cachable) {
     if (isPlainObject(value)) {
       // Expression objects do their own caching
       // TODO: What if it's some other kind of plain object?

package/src/runtime/errors.js

@@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url";
 import codeFragment from "./codeFragment.js";
 import explainReferenceError from "./explainReferenceError.js";
 import explainTraverseError from "./explainTraverseError.js";
+import { ops } from "./internal.js";
 
 // Text we look for in an error stack to guess whether a given line represents a
 // function in the Origami source code.
@@ -61,7 +62,8 @@ export async function formatError(error) {
         message += "\n" + explanation;
       }
     } else if (error instanceof TraverseError) {
-      const explanation = await explainTraverseError(error);
+      // TraverseError explainer wants the full code
+      const explanation = await explainTraverseError(context.code, error);
       if (explanation) {
         message += "\n" + explanation;
       }
@@ -158,3 +160,33 @@ export function lineInfo(location) {
 export function maybeOrigamiSourceCode(text) {
   return origamiSourceSignals.some((signal) => text.includes(signal));
 }
+
+// Return the keys traversed by the given code, or null if undetermined.
+export function reconstructTraversalKeys(code) {
+  const [op, ...args] = code;
+
+  switch (op) {
+    case ops.cache:
+      return reconstructTraversalKeys(code[2]);
+
+    case ops.unpack:
+      return reconstructTraversalKeys(code[1]);
+  }
+
+  if (op instanceof Array) {
+    if (op[0] === ops.scope) {
+      // Simple scope reference
+      const key = args[0][1];
+      return [key];
+    } else if (op[0] === ops.cache) {
+      // Traversal starting with external scope reference
+      const head = op[2][1][1];
+      const tail = args.map((code) => code[1]);
+      const keys = [head, ...tail];
+      return keys;
+    }
+  }
+
+  // Can't determine keys
+  return null;
+}

package/src/runtime/execute.js

@@ -1,4 +1,5 @@
 import { isUnpackable, Tree } from "@weborigami/async-tree";
+import counters from "../runtime/counters.js";
 import executionContext from "./executionContext.js";
 import "./interop.js";
 
@@ -14,6 +15,8 @@ import "./interop.js";
  * @param {RuntimeState} [state]
  */
 export default async function execute(code, state = {}) {
+  counters.executions++;
+
   if (!(code instanceof Array)) {
     // Simple scalar; return as is.
     return code;

package/src/runtime/explainReferenceError.js

@@ -4,6 +4,7 @@ import {
   trailingSlash,
   Tree,
 } from "@weborigami/async-tree";
+import { reconstructTraversalKeys } from "./errors.js";
 import { ops } from "./internal.js";
 import { typos } from "./typos.js";
 
@@ -35,39 +36,25 @@ export default async function explainReferenceError(code, state) {
     return explanation;
   }
 
-  // See if the code looks like an external scope reference that failed
-  let key;
-  if (code[0] === ops.cache) {
-    // External scope reference
-    let refCall = code[2];
-    if (refCall[0] === ops.unpack) {
-      // Unwrap implied unpack
-      refCall = refCall[1];
-    }
-    const scopeArgs = refCall.slice(1); // get the scope reference arguments
-    const keys = scopeArgs.map((part) => part[1]);
-    const path = pathFromKeys(keys);
-
-    if (keys.length > 1) {
-      return `This path returned undefined: ${path}`;
-    }
-    key = keys[0];
-  } else if (code[0]?.[0] === ops.scope) {
-    // Simple scope reference
-    key = code[1][1];
-  } else {
-    // Generic reference error, can't offer help
+  const keys = reconstructTraversalKeys(code);
+  if (!keys) {
     return null;
+  } else if (keys.length > 1) {
+    const path = pathFromKeys(keys);
+    return `This path returned undefined: ${path}`;
   }
 
-  key = trailingSlash.remove(key);
+  // A single key caused the problem, try to explain it
+  let key = keys[0];
   if (typeof key !== "string") {
     return null;
   }
+  key = trailingSlash.remove(key);
 
-  // Common case of a single key
+  // Baseline message
   let message = `"${key}" is not in scope.`;
 
+  // Various explainers can add suggestions
   const explainers = [
     mathExplainer,
     qualifiedReferenceExplainer,

package/src/runtime/explainTraverseError.js

@@ -6,6 +6,7 @@ import {
   TraverseError,
   Tree,
 } from "@weborigami/async-tree";
+import { reconstructTraversalKeys } from "./errors.js";
 import { typos } from "./typos.js";
 
 /**
@@ -14,7 +15,7 @@ import { typos } from "./typos.js";
  *
  * @param {TraverseError} error
  */
-export default async function explainTraverseError(error) {
+export default async function explainTraverseError(code, error) {
   const { lastValue, keys, position } = error;
   if (lastValue === undefined || keys === undefined || position === undefined) {
     // Don't have sufficient information; shouldn't happen
@@ -27,9 +28,15 @@ export default async function explainTraverseError(error) {
   }
 
   // The key that caused the error is the one before the current position
-  const path = pathFromKeys(keys);
-  const problemKey = keys[position - 1];
-  let message = `Tried to traverse path:  ${path}\nStopped unexpectedly at: ${problemKey}`;
+  const problemKey = error.keys[position - 1];
+  const reconstructedKeys = reconstructTraversalKeys(code);
+  let message;
+  if (reconstructedKeys === null) {
+    message = `Path traversal stopped unexpectedly at: ${problemKey}`;
+  } else {
+    const path = pathFromKeys(reconstructedKeys);
+    message = `Tried to traverse path:  ${path}\nStopped unexpectedly at: ${problemKey}`;
+  }
 
   const key = trailingSlash.remove(keys[position - 1]);
 
@@ -50,10 +57,12 @@ export default async function explainTraverseError(error) {
     const normalizedKeys = lastValueKeys.map(trailingSlash.remove);
 
     const keyAsNumber = Number(key);
-    if (!isNaN(keyAsNumber)) {
+    if (reconstructedKeys && !isNaN(keyAsNumber)) {
       // See if the string version of the key is present
       if (lastValueKeys.includes(keyAsNumber)) {
-        const suggestedPath = `${pathFromKeys(keys.slice(0, position - 1))}(${keyAsNumber})`;
+        const suggestedKeys = reconstructedKeys?.slice(0);
+        suggestedKeys[position] = `(${keyAsNumber})`;
+        const suggestedPath = pathFromKeys(suggestedKeys);
         message += `\nSlash-separated keys are searched as strings. Here there's no string "${key}" key, but there is a number ${keyAsNumber} key.
 To get the value for that number key, use parentheses: ${suggestedPath}`;
       }

package/src/runtime/expressionObject.js

@@ -25,24 +25,34 @@ const VALUE_TYPE = {
   GETTER: 2, // Calculated on demand: `a = fn()`
 };
 
+const mapUnattachedObjectToIndex = new Map();
+
 /**
  * Given an array of entries with string keys and Origami code values (arrays of
  * ops and operands), return an object with the same keys defining properties
  * whose getters evaluate the code.
-
+ *
+ * @param {string|null} cachePath
  * @param {*} entries
  * @param {import("../../index.ts").RuntimeState} [state]
  */
-export default async function expressionObject(entries, state = {}) {
+export default async function expressionObject(cachePath, entries, state = {}) {
   // Create the object and set its parent
   const object = {};
   const parent = state?.object ?? null;
   if (parent !== null && !Tree.isMap(parent)) {
     throw new TypeError(`Parent must be a map or null`);
   }
-
   setParent(object, parent);
 
+  if (cachePath?.endsWith("/_objects/")) {
+    // Unattached object, add a unique index to the cache path
+    let index = mapUnattachedObjectToIndex.get(cachePath) ?? 0;
+    mapUnattachedObjectToIndex.set(cachePath, index + 1);
+    cachePath += index + "/";
+  }
+  object[cachePathSymbol] = cachePath;
+
   // The object in Map form for use on the stack
   const map = new ObjectMap(object);
 
@@ -93,8 +103,7 @@ export default async function expressionObject(entries, state = {}) {
  * Define a single property on the object
  */
 function defineProperty(object, propertyInfo, state, map) {
-  const { globals } = state;
-  let { enumerable, hasExtension, key, value, valueType } = propertyInfo;
+  let { enumerable, key, value, valueType } = propertyInfo;
   if (valueType == VALUE_TYPE.PRIMITIVE) {
     // Define simple property
     Object.defineProperty(object, key, {
@@ -109,43 +118,53 @@ function defineProperty(object, propertyInfo, state, map) {
       configurable: true,
       enumerable,
       get: async () => {
-        // Execute the code to get the value of the property
-        const propertyCachePath = getPropertyCachePath(object, key);
-
-        const newState = Object.assign({}, state, { object: map });
-        let result = propertyCachePath
-          ? await systemCache.getOrInsertComputedAsync(propertyCachePath, () =>
-              execute(value, newState),
-            )
-          : await execute(value, newState);
-
-        if (hasExtension) {
-          // Handle extension
-          result = handleExtension(result, key, globals, map);
-        }
-
-        if (valueType === VALUE_TYPE.EAGER) {
-          // Memoize result on the object itself
-          Object.defineProperty(object, key, {
-            configurable: true,
-            enumerable,
-            value: result,
-            writable: true,
-          });
-        } else if (propertyCachePath) {
-          result = enableValueCaching(result, propertyCachePath);
-        }
-
-        return result;
+        return executeProperty(object, propertyInfo, state, map);
       },
     });
   }
 }
 
+// Execute the property code to get the value of the property
+async function executeProperty(object, propertyInfo, state, map) {
+  const { globals } = state;
+  let { enumerable, hasExtension, key, value, valueType } = propertyInfo;
+  const propertyCachePath = getPropertyCachePath(object, key);
+
+  const newState = Object.assign({}, state, { object: map });
+  const cacheProperty = valueType === VALUE_TYPE.GETTER && propertyCachePath;
+  let result = cacheProperty
+    ? await systemCache.getOrInsertComputedAsync(propertyCachePath, () =>
+        execute(value, newState),
+      )
+    : await execute(value, newState);
+
+  if (hasExtension) {
+    // Handle extension
+    result = handleExtension(result, key, globals, map);
+  }
+
+  if (propertyCachePath) {
+    // Enable caching on value
+    result = enableValueCaching(result, propertyCachePath);
+  }
+
+  if (valueType === VALUE_TYPE.EAGER) {
+    // Memoize result on the object itself
+    Object.defineProperty(object, key, {
+      configurable: true,
+      enumerable,
+      value: result,
+      writable: true,
+    });
+  }
+
+  return result;
+}
+
 function getPropertyCachePath(object, key) {
   // Follow parent chain looking for a parent that has caching enabled
   let current = object;
-  while (current[cachePathSymbol] === undefined) {
+  while (current[cachePathSymbol] == null) {
     current = current[symbols.parent];
     if (!current) {
       // Caching isn't enabled on this object tree

package/src/runtime/handleExtension.js

@@ -13,6 +13,9 @@ import { cachePathSymbol } from "./symbols.js";
 import systemCache from "./systemCache.js";
 import SystemCacheMap from "./SystemCacheMap.js";
 
+// Base class for async functions
+const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
+
 /**
  * If the given value is packed (e.g., buffer) and the key is a string-like path
  * that ends in an extension, search for a handler for that extension and, if
@@ -46,19 +49,27 @@ export default function handleExtension(value, key, handlers, parent = null) {
       // Use `in` to look for handle so that, if the handler is a promise, we
       // can still find it without awaiting it here.
       if (handlerName in handlers) {
-        let handler = handlers[handlerName];
-
         // If the value is a primitive, box it so we can attach data to it.
         value = box(value);
 
-        if (handler.mediaType) {
-          value.mediaType = handler.mediaType;
-        }
-
         if (parent) {
           setParent(value, parent);
         }
 
+        // Avoid triggering the loading of the handler if it's async because
+        // we may be doing this work inside a sync function.
+        const descriptor = Object.getOwnPropertyDescriptor(
+          handlers,
+          handlerName,
+        );
+        const lazyHandler = descriptor?.get instanceof AsyncFunction;
+        if (!lazyHandler) {
+          const mediaType = handlers[handlerName]?.mediaType;
+          if (mediaType) {
+            value.mediaType = mediaType;
+          }
+        }
+
         // Wrap the unpack function so it caches the unpacked value, and so we
         // can add the file path to any errors the unpack function throws.
         const filePath = getPackedPath(value, { key: normalized, parent });
@@ -74,9 +85,7 @@ export default function handleExtension(value, key, handlers, parent = null) {
         const unpackCachePath = trailingSlash.add(fileCachePath);
         value.unpack = async () =>
           systemCache.getOrInsertComputedAsync(unpackCachePath, async () => {
-            if (handler instanceof Promise) {
-              handler = await handler;
-            }
+            let handler = await handlers[handlerName];
             if (isUnpackable(handler)) {
               // The extension handler itself needs to be unpacked
               handler = await handler.unpack();

package/src/runtime/ops.js

@@ -255,7 +255,7 @@ export function lambda(length, parameters, code, state = {}) {
       const interimStack = stack.slice();
       interimStack.push(args);
       const paramState = { ...state, stack: interimStack };
-      const frame = await expressionObject(parameters, paramState);
+      const frame = await expressionObject(null, parameters, paramState);
       // Record which code this stack frame is associated with
       Object.defineProperty(frame, codeSymbol, {
         value: code,
@@ -414,11 +414,12 @@ addOpLabel(nullishCoalescing, "«ops.nullishCoalescing»");
  * parameter's, and the values will be the results of evaluating the
  * corresponding code values in `obj`.
  *
+ * @param {string} cachePath
  * @param {any[]} entries
  */
-export async function object(...entries) {
+export async function object(cachePath, ...entries) {
   const state = entries.pop();
-  return expressionObject(entries, state);
+  return expressionObject(cachePath, entries, state);
 }
 addOpLabel(object, "«ops.object»");
 object.unevaluatedArgs = true;

package/test/compiler/compile.test.js

@@ -16,7 +16,6 @@ const globals = {
  * features, which compare the output of evaluate2() against an expected result.
  */
 describe("compile", () => {
-
   test("async object", async () => {
     const fn = compile.expression("{ a: { b = name }}", { globals });
     const object = await fn();