@weborigami/language 0.3.3-jse.3 → 0.3.3

package/src/compiler/parserHelpers.js

@@ -15,12 +15,8 @@ const YAML = YAMLModule.default ?? YAMLModule.YAML;
 /** @typedef {import("../../index.ts").CodeLocation} CodeLocation */
 /** @typedef {import("../../index.ts").Code} Code */
 
-// Markers in compiled output, will get optimized away
-export const markers = {
-  global: Symbol("global"), // Global reference
-  traverse: Symbol("traverse"), // Continuation of path traversal
-  reference: Symbol("reference"), // Reference to local, scope, or global
-};
+// Marker for a reference that may be a builtin or a scope reference
+export const undetermined = Symbol("undetermined");
 
 const builtinRegex = /^[A-Za-z][A-Za-z0-9]*$/;
 
@@ -41,7 +37,7 @@ export function annotate(code, location) {
 }
 
 /**
- * In the given code, replace all function calls to the given name with the
+ * In the given code, replace all scope refernces to the given name with the
  * given macro code.
  *
  * @param {AnnotatedCode} code
@@ -53,16 +49,8 @@ export function applyMacro(code, name, macro) {
     return code;
   }
 
-  // We're looking for a function call with the given name.
-  // For `foo`, the call would be: [[reference, [ops.literal, "foo"]], undefined]
-  if (
-    code[0] &&
-    code[0][0] === markers.reference &&
-    code[0][1] instanceof Array &&
-    code[0][1][0] === ops.literal &&
-    code[0][1][1] === name &&
-    code[1] === undefined
-  ) {
+  const [fn, ...args] = code;
+  if (fn === ops.scope && args[0] === name) {
     return macro;
   }
 
@@ -72,7 +60,7 @@ export function applyMacro(code, name, macro) {
 
 /**
  * The indicated code is being used to define a property named by the given key.
- * Rewrite any [[ops.scope], key] calls to be [ops.inherited, key] to avoid
+ * Rewrite any [ops.scope, key] calls to be [ops.inherited, key] to avoid
  * infinite recursion.
  *
  * @param {AnnotatedCode} code
@@ -85,12 +73,11 @@ function avoidRecursivePropertyCalls(code, key) {
   /** @type {Code} */
   let modified;
   if (
-    code[0] instanceof Array &&
-    code[0][0] === ops.scope &&
-    trailingSlash.remove(code[1][1]) === trailingSlash.remove(key)
+    code[0] === ops.scope &&
+    trailingSlash.remove(code[1]) === trailingSlash.remove(key)
   ) {
     // Rewrite to avoid recursion
-    modified = [ops.inherited, code[1][1]];
+    modified = [ops.inherited, code[1]];
   } else if (
     code[0] === ops.lambda &&
     code[1].some((param) => param[1] === key)
@@ -105,13 +92,13 @@ function avoidRecursivePropertyCalls(code, key) {
 }
 
 /**
- * Downgrade a potential global reference to a reference.
+ * Downgrade a potential builtin reference to a scope reference.
  *
  * @param {AnnotatedCode} code
  */
 export function downgradeReference(code) {
-  if (code && code.length === 2 && code[0] === markers.reference) {
-    return annotate([markers.reference, code[1]], code.location);
+  if (code && code.length === 2 && code[0] === undetermined) {
+    return annotate([ops.scope, code[1]], code.location);
   } else {
     return code;
   }
@@ -152,7 +139,7 @@ export function makeArray(entries, location) {
 
   let result;
   if (spreads.length > 1) {
-    result = [ops.flat, ...spreads];
+    result = [ops.merge, ...spreads];
   } else if (spreads.length === 1) {
     result = spreads[0];
   } else {
@@ -209,7 +196,7 @@ export function makeBinaryOperation(left, [operatorToken, right]) {
  * @param {AnnotatedCode} target
  * @param {any[]} args
  */
-export function makeCall(target, args, mode) {
+export function makeCall(target, args) {
   if (!(target instanceof Array)) {
     const error = new SyntaxError(`Can't call this like a function: ${target}`);
     /** @type {any} */ (error).location = /** @type {any} */ (target).location;
@@ -217,56 +204,38 @@ export function makeCall(target, args, mode) {
   }
 
   let fnCall;
-  const op = args[0];
-  if (op === markers.traverse || op === ops.optionalTraverse) {
+  if (args[0] === ops.traverse) {
     let tree = target;
 
-    if (tree[0] === markers.reference && !trailingSlash.has(tree[1][1])) {
-      // Target didn't parse with a trailing slash; add one
-      tree[1][1] = trailingSlash.add(tree[1][1]);
-    }
-
-    // Is the target an existing traversal that can be extended? It should be a
-    // reference or global where all the args are literals.
-    const extend =
-      (tree[0] === markers.reference ||
-        (tree[0] instanceof Array && tree[0][0] === markers.global)) &&
-      !tree
-        .slice(1)
-        .some((arg) => !(arg instanceof Array && arg[0] === ops.literal));
-    if (extend) {
-      fnCall = tree;
-      // If last key doesn't end with slash, add one
-      const last = tree.at(-1);
-      if (last instanceof Array && last[0] === ops.literal) {
-        last[1] = trailingSlash.add(last[1]);
+    if (tree[0] === undetermined) {
+      // In a traversal, downgrade ops.builtin references to ops.scope
+      tree = downgradeReference(tree);
+      if (tree[0] === ops.scope && !trailingSlash.has(tree[1])) {
+        // Target didn't parse with a trailing slash; add one
+        tree[1] = trailingSlash.add(tree[1]);
       }
-    } else {
-      fnCall = [tree];
     }
 
     if (args.length > 1) {
       // Regular traverse
       const keys = args.slice(1);
-      fnCall.push(...keys);
-    } else if (tree[0] !== ops.rootDirectory) {
+      fnCall = [ops.traverse, tree, ...keys];
+    } else {
       // Traverse without arguments equates to unpack
       fnCall = [ops.unpack, tree];
-    } else {
-      fnCall = tree;
     }
-  } else if (op === ops.templateStandard || op === ops.templateTree) {
+  } else if (args[0] === ops.template) {
     // Tagged template
     const strings = args[1];
     const values = args.slice(2);
     fnCall = makeTaggedTemplateCall(
-      upgradeReference(target, mode),
+      upgradeReference(target),
       strings,
       ...values
     );
   } else {
     // Function call with explicit or implicit parentheses
-    fnCall = [upgradeReference(target, mode), ...args];
+    fnCall = [upgradeReference(target), ...args];
   }
 
   // Create a location spanning the newly-constructed function call.
@@ -306,94 +275,6 @@ export function makeDeferredArguments(args) {
   });
 }
 
-export function makeDocument(mode, front, body, location) {
-  // In order for template expressions to see the front matter properties,
-  // we translate the top-level front properties to object entries.
-  const entries = Object.entries(front).map(([key, value]) =>
-    annotate([key, annotate([ops.literal, value], location)], location)
-  );
-
-  // Add an entry for the body
-  const bodyKey = mode === "jse" ? "_body" : "@text";
-  entries.push(annotate([bodyKey, body], location));
-
-  // Return the code for the document object
-  return annotate([ops.object, ...entries], location);
-}
-
-export function makeJsPropertyAccess(expression, property) {
-  const location = {
-    source: expression.location.source,
-    start: expression.location.start,
-    end: property.location.end,
-  };
-  return annotate([expression, property], location);
-}
-
-/**
- * From the given spreads within an object spread, return the merge.
- *
- * Example:
- *
- *      {
- *        x = { a: 1 }
- *        …x
- *        y = x
- *      }
- *
- *  will be treated as:
- *
- *      {
- *        x = { a: 1 }
- *        y = x
- *        _result: {
- *          x
- *          …x
- *          y
- *        }
- *      }.result
- *
- * @param {*} spreads
- * @param {CodeLocation} location
- */
-function makeMerge(spreads, location) {
-  const topEntries = [];
-  const resultEntries = [];
-  for (const spread of spreads) {
-    if (spread[0] === ops.object) {
-      topEntries.push(...spread.slice(1));
-      // Also add an object to the result with indirect references
-      const indirectEntries = spread.slice(1).map((entry) => {
-        const [key] = entry;
-        const context = annotate([ops.context, 1], entry.location);
-        const reference = annotate([context, key], entry.location);
-        const getter = annotate([ops.getter, reference], entry.location);
-        return annotate([key, getter], entry.location);
-      });
-      const indirectObject = annotate(
-        [ops.object, ...indirectEntries],
-        location
-      );
-      resultEntries.push(indirectObject);
-    } else {
-      resultEntries.push(spread);
-    }
-  }
-
-  // Merge to create result
-  const result = annotate([ops.merge, ...resultEntries], location);
-
-  // Add the result to the top-level object as _result
-  topEntries.push(annotate(["_result", result], location));
-
-  // Construct the top-level object
-  const topObject = annotate([ops.object, ...topEntries], location);
-
-  // Get the _result property
-  const code = annotate([topObject, "_result"], location);
-  return code;
-}
-
 /**
  * Make an object.
  *
@@ -450,7 +331,7 @@ export function makeObject(entries, location) {
   let code;
   if (spreads.length > 1) {
     // Merge multiple spreads
-    code = makeMerge(spreads, location);
+    code = [ops.merge, ...spreads];
   } else if (spreads.length === 1) {
     // A single spread can just be the object
     code = spreads[0];
@@ -467,11 +348,10 @@ export function makeObject(entries, location) {
  *
  * @param {AnnotatedCode} arg
  * @param {AnnotatedCode} fn
- * @param {string} mode
  */
-export function makePipeline(arg, fn, mode) {
-  const upgraded = upgradeReference(fn, mode);
-  const result = makeCall(upgraded, [arg], mode);
+export function makePipeline(arg, fn) {
+  const upgraded = upgradeReference(fn);
+  const result = makeCall(upgraded, [arg]);
   const source = fn.location.source;
   let start = arg.location.start;
   let end = fn.location.end;
@@ -484,6 +364,30 @@ export function makeProperty(key, value) {
   return [key, modified];
 }
 
+export function makeJsPropertyAccess(expression, property) {
+  const location = {
+    source: expression.location.source,
+    start: expression.location.start,
+    end: property.location.end,
+  };
+  return annotate([expression, property], location);
+}
+
+export function makeReference(identifier) {
+  // We can't know for sure that an identifier is a builtin reference until we
+  // see whether it's being called as a function.
+  let op;
+  if (builtinRegex.test(identifier)) {
+    op = identifier.endsWith(":")
+      ? // Namespace is always a builtin reference
+        ops.builtin
+      : undetermined;
+  } else {
+    op = ops.scope;
+  }
+  return [op, identifier];
+}
+
 /**
  * Make a tagged template call
  *
@@ -591,7 +495,7 @@ export function makeYamlObject(text, location) {
     throw error;
   }
 
-  return parsed;
+  return annotate([ops.literal, parsed], location);
 }
 
 /**
@@ -599,14 +503,9 @@ export function makeYamlObject(text, location) {
  *
  * @param {AnnotatedCode} code
  */
-export function upgradeReference(code, mode) {
-  if (
-    mode === "shell" &&
-    code.length === 2 &&
-    code[0] === markers.reference &&
-    builtinRegex.exec(code[1][1])
-  ) {
-    const result = [markers.global, code[1][1]];
+export function upgradeReference(code) {
+  if (code.length === 2 && code[0] === undetermined) {
+    const result = [ops.builtin, code[1]];
     return annotate(result, code.location);
   } else {
     return code;

package/src/runtime/HandleExtensionsTransform.js

@@ -1,4 +1,3 @@
-import getHandlers from "./getHandlers.js";
 import { handleExtension } from "./handlers.js";
 
 /**
@@ -10,17 +9,9 @@ import { handleExtension } from "./handlers.js";
  */
 export default function HandleExtensionsTransform(Base) {
   return class FileLoaders extends Base {
-    constructor(...args) {
-      super(...args);
-
-      // Callers should set this to the set of supported extension handlers
-      this.handlers = null;
-    }
-
     async get(key) {
       const value = await super.get(key);
-      const handlers = getHandlers(this);
-      return handleExtension(this, value, key, handlers);
+      return handleExtension(this, value, key);
     }
   };
 }

package/src/runtime/evaluate.js

@@ -1,5 +1,6 @@
-import { Tree, isUnpackable } from "@weborigami/async-tree";
+import { Tree, isUnpackable, scope } from "@weborigami/async-tree";
 import codeFragment from "./codeFragment.js";
+import { codeSymbol, scopeSymbol, sourceSymbol } from "./symbols.js";
 
 /**
  * Evaluate the given code and return the result.
@@ -51,7 +52,7 @@ export default async function evaluate(code) {
     result =
       fn instanceof Function
         ? await fn.call(tree, ...args) // Invoke the function
-        : await Tree.traverseOrThrow.call(tree, fn, ...args); // Traverse the tree.
+        : await Tree.traverseOrThrow(fn, ...args); // Traverse the tree.
   } catch (/** @type {any} */ error) {
     if (!error.location) {
       // Attach the location of the code we tried to evaluate.
@@ -66,33 +67,39 @@ export default async function evaluate(code) {
     throw error;
   }
 
+  // If the result is a tree, then the default parent of the tree is the current
+  // tree.
+  if (Tree.isAsyncTree(result) && !result.parent) {
+    result.parent = tree;
+  }
+
   // To aid debugging, add the code to the result.
-  // if (Object.isExtensible(result)) {
-  //   try {
-  //     if (code.location && !result[sourceSymbol]) {
-  //       Object.defineProperty(result, sourceSymbol, {
-  //         value: codeFragment(code.location),
-  //         enumerable: false,
-  //       });
-  //     }
-  //     if (!result[codeSymbol]) {
-  //       Object.defineProperty(result, codeSymbol, {
-  //         value: code,
-  //         enumerable: false,
-  //       });
-  //     }
-  //     if (!result[scopeSymbol]) {
-  //       Object.defineProperty(result, scopeSymbol, {
-  //         get() {
-  //           return scope(this).trees;
-  //         },
-  //         enumerable: false,
-  //       });
-  //     }
-  //   } catch (/** @type {any} */ error) {
-  //     // Ignore errors.
-  //   }
-  // }
+  if (Object.isExtensible(result)) {
+    try {
+      if (code.location && !result[sourceSymbol]) {
+        Object.defineProperty(result, sourceSymbol, {
+          value: codeFragment(code.location),
+          enumerable: false,
+        });
+      }
+      if (!result[codeSymbol]) {
+        Object.defineProperty(result, codeSymbol, {
+          value: code,
+          enumerable: false,
+        });
+      }
+      if (!result[scopeSymbol]) {
+        Object.defineProperty(result, scopeSymbol, {
+          get() {
+            return scope(this).trees;
+          },
+          enumerable: false,
+        });
+      }
+    } catch (/** @type {any} */ error) {
+      // Ignore errors.
+    }
+  }
 
   return result;
 }

package/src/runtime/expressionObject.js

@@ -6,7 +6,6 @@ import {
   trailingSlash,
   Tree,
 } from "@weborigami/async-tree";
-import getHandlers from "./getHandlers.js";
 import { handleExtension } from "./handlers.js";
 import { evaluate, ops } from "./internal.js";
 
@@ -33,7 +32,6 @@ export default async function expressionObject(entries, parent) {
   if (parent !== null && !Tree.isAsyncTree(parent)) {
     throw new TypeError(`Parent must be an AsyncTree or null`);
   }
-  setParent(object, parent);
 
   let tree;
   const eagerProperties = [];
@@ -92,8 +90,7 @@ export default async function expressionObject(entries, parent) {
         get = async () => {
           tree ??= new ObjectTree(object);
           const result = await evaluate.call(tree, code);
-          const handlers = getHandlers(tree);
-          return handleExtension(tree, result, key, handlers);
+          return handleExtension(tree, result, key);
         };
       } else {
         // No extension, so getter just invokes code.
@@ -119,6 +116,9 @@ export default async function expressionObject(entries, parent) {
     writable: true,
   });
 
+  // Attach the parent
+  setParent(object, parent);
+
   // Evaluate any properties that were declared as immediate: get their value
   // and overwrite the property getter with the actual value.
   for (const key of eagerProperties) {

package/src/runtime/handlers.js

@@ -4,10 +4,60 @@ import {
   isPacked,
   isStringLike,
   isUnpackable,
+  scope,
   setParent,
   trailingSlash,
 } from "@weborigami/async-tree";
 
+/** @typedef {import("../../index.ts").ExtensionHandler} ExtensionHandler */
+
+// Track extensions handlers for a given containing tree.
+const handlersForContainer = new Map();
+
+/**
+ * Find an extension handler for a file in the given container.
+ *
+ * @typedef {import("@weborigami/types").AsyncTree} AsyncTree
+ *
+ * @param {AsyncTree} parent
+ * @param {string} extension
+ */
+export async function getExtensionHandler(parent, extension) {
+  let handlers = handlersForContainer.get(parent);
+  if (handlers) {
+    if (handlers[extension]) {
+      return handlers[extension];
+    }
+  } else {
+    handlers = {};
+    handlersForContainer.set(parent, handlers);
+  }
+
+  const handlerName = `${extension.slice(1)}.handler`;
+  const parentScope = scope(parent);
+
+  /** @type {Promise<ExtensionHandler>} */
+  let handlerPromise = parentScope
+    ?.get(handlerName)
+    .then(async (extensionHandler) => {
+      if (isUnpackable(extensionHandler)) {
+        // The extension handler itself needs to be unpacked. E.g., if it's a
+        // buffer containing JavaScript file, we need to unpack it to get its
+        // default export.
+        // @ts-ignore
+        extensionHandler = await extensionHandler.unpack();
+      }
+      // Update cache with actual handler
+      handlers[extension] = extensionHandler;
+      return extensionHandler;
+    });
+
+  // Cache handler even if it's undefined so we don't look it up again
+  handlers[extension] = handlerPromise;
+
+  return handlerPromise;
+}
+
 /**
  * If the given value is packed (e.g., buffer) and the key is a string-like path
  * that ends in an extension, search for a handler for that extension and, if
@@ -17,34 +67,20 @@ import {
  * @param {any} value
  * @param {any} key
  */
-export async function handleExtension(parent, value, key, handlers) {
-  if (
-    handlers &&
-    isPacked(value) &&
-    isStringLike(key) &&
-    value.unpack === undefined
-  ) {
+export async function handleExtension(parent, value, key) {
+  if (isPacked(value) && isStringLike(key) && value.unpack === undefined) {
     const hasSlash = trailingSlash.has(key);
     if (hasSlash) {
       key = trailingSlash.remove(key);
     }
 
-    // Special cases: `.ori.<ext>` extensions are Origami documents,
-    // `.jse.<ext>` are JSE documents.
+    // Special case: `.ori.<ext>` extensions are Origami documents.
     const extname = key.match(/\.ori\.\S+$/)
       ? ".oridocument"
-      : key.match(/\.jse\.\S+$/)
-      ? ".jsedocument"
       : extension.extname(key);
     if (extname) {
-      const handlerName = `${extname.slice(1)}.handler`;
-      let handler = await handlers.get(handlerName);
+      const handler = await getExtensionHandler(parent, extname);
       if (handler) {
-        if (isUnpackable(handler)) {
-          // The extension handler itself needs to be unpacked
-          handler = await handler.unpack();
-        }
-
         if (hasSlash && handler.unpack) {
           // Key like `data.json/` ends in slash -- unpack immediately
           return handler.unpack(value, { key, parent });

package/src/runtime/mergeTrees.js

@@ -53,6 +53,11 @@ export default async function mergeTrees(...trees) {
     return result;
   }
 
+  // If all trees are arrays, return an array.
+  if (unpacked.every((tree) => Array.isArray(tree))) {
+    return unpacked.flat();
+  }
+
   // Merge the trees.
   const result = merge(...unpacked);
   setParent(result, this);