@weborigami/language 0.3.3-jse.1 → 0.3.3-jse.3

package/src/compiler/parserHelpers.js

@@ -15,8 +15,12 @@ const YAML = YAMLModule.default ?? YAMLModule.YAML;
 /** @typedef {import("../../index.ts").CodeLocation} CodeLocation */
 /** @typedef {import("../../index.ts").Code} Code */
 
-// Marker for a reference that may be a builtin or a scope reference
-export const undetermined = Symbol("undetermined");
+// Markers in compiled output, will get optimized away
+export const markers = {
+  global: Symbol("global"), // Global reference
+  traverse: Symbol("traverse"), // Continuation of path traversal
+  reference: Symbol("reference"), // Reference to local, scope, or global
+};
 
 const builtinRegex = /^[A-Za-z][A-Za-z0-9]*$/;
 
@@ -37,7 +41,7 @@ export function annotate(code, location) {
 }
 
 /**
- * In the given code, replace all scope refernces to the given name with the
+ * In the given code, replace all function calls to the given name with the
  * given macro code.
  *
  * @param {AnnotatedCode} code
@@ -49,8 +53,16 @@ export function applyMacro(code, name, macro) {
     return code;
   }
 
-  const [fn, ...args] = code;
-  if (fn === ops.scope && args[0] === name) {
+  // We're looking for a function call with the given name.
+  // For `foo`, the call would be: [[reference, [ops.literal, "foo"]], undefined]
+  if (
+    code[0] &&
+    code[0][0] === markers.reference &&
+    code[0][1] instanceof Array &&
+    code[0][1][0] === ops.literal &&
+    code[0][1][1] === name &&
+    code[1] === undefined
+  ) {
     return macro;
   }
 
@@ -60,7 +72,7 @@ export function applyMacro(code, name, macro) {
 
 /**
  * The indicated code is being used to define a property named by the given key.
- * Rewrite any [ops.scope, key] calls to be [ops.inherited, key] to avoid
+ * Rewrite any [[ops.scope], key] calls to be [ops.inherited, key] to avoid
  * infinite recursion.
  *
  * @param {AnnotatedCode} code
@@ -73,11 +85,12 @@ function avoidRecursivePropertyCalls(code, key) {
   /** @type {Code} */
   let modified;
   if (
-    code[0] === ops.scope &&
-    trailingSlash.remove(code[1]) === trailingSlash.remove(key)
+    code[0] instanceof Array &&
+    code[0][0] === ops.scope &&
+    trailingSlash.remove(code[1][1]) === trailingSlash.remove(key)
   ) {
     // Rewrite to avoid recursion
-    modified = [ops.inherited, code[1]];
+    modified = [ops.inherited, code[1][1]];
   } else if (
     code[0] === ops.lambda &&
     code[1].some((param) => param[1] === key)
@@ -92,13 +105,13 @@ function avoidRecursivePropertyCalls(code, key) {
 }
 
 /**
- * Downgrade a potential builtin reference to a scope reference.
+ * Downgrade a potential global reference to a reference.
  *
  * @param {AnnotatedCode} code
  */
 export function downgradeReference(code) {
-  if (code && code.length === 2 && code[0] === undetermined) {
-    return annotate([ops.scope, code[1]], code.location);
+  if (code && code.length === 2 && code[0] === markers.reference) {
+    return annotate([markers.reference, code[1]], code.location);
   } else {
     return code;
   }
@@ -139,7 +152,7 @@ export function makeArray(entries, location) {
 
   let result;
   if (spreads.length > 1) {
-    result = [ops.merge, ...spreads];
+    result = [ops.flat, ...spreads];
   } else if (spreads.length === 1) {
     result = spreads[0];
   } else {
@@ -196,7 +209,7 @@ export function makeBinaryOperation(left, [operatorToken, right]) {
  * @param {AnnotatedCode} target
  * @param {any[]} args
  */
-export function makeCall(target, args) {
+export function makeCall(target, args, mode) {
   if (!(target instanceof Array)) {
     const error = new SyntaxError(`Can't call this like a function: ${target}`);
     /** @type {any} */ (error).location = /** @type {any} */ (target).location;
@@ -205,38 +218,55 @@ export function makeCall(target, args) {
 
   let fnCall;
   const op = args[0];
-  if (op === ops.traverse || op === ops.optionalTraverse) {
+  if (op === markers.traverse || op === ops.optionalTraverse) {
     let tree = target;
 
-    if (tree[0] === undetermined) {
-      // In a traversal, downgrade ops.builtin references to ops.scope
-      tree = downgradeReference(tree);
-      if (tree[0] === ops.scope && !trailingSlash.has(tree[1])) {
-        // Target didn't parse with a trailing slash; add one
-        tree[1] = trailingSlash.add(tree[1]);
+    if (tree[0] === markers.reference && !trailingSlash.has(tree[1][1])) {
+      // Target didn't parse with a trailing slash; add one
+      tree[1][1] = trailingSlash.add(tree[1][1]);
+    }
+
+    // Is the target an existing traversal that can be extended? It should be a
+    // reference or global where all the args are literals.
+    const extend =
+      (tree[0] === markers.reference ||
+        (tree[0] instanceof Array && tree[0][0] === markers.global)) &&
+      !tree
+        .slice(1)
+        .some((arg) => !(arg instanceof Array && arg[0] === ops.literal));
+    if (extend) {
+      fnCall = tree;
+      // If last key doesn't end with slash, add one
+      const last = tree.at(-1);
+      if (last instanceof Array && last[0] === ops.literal) {
+        last[1] = trailingSlash.add(last[1]);
       }
+    } else {
+      fnCall = [tree];
     }
 
     if (args.length > 1) {
       // Regular traverse
       const keys = args.slice(1);
-      fnCall = [op, tree, ...keys];
-    } else {
+      fnCall.push(...keys);
+    } else if (tree[0] !== ops.rootDirectory) {
       // Traverse without arguments equates to unpack
       fnCall = [ops.unpack, tree];
+    } else {
+      fnCall = tree;
     }
   } else if (op === ops.templateStandard || op === ops.templateTree) {
     // Tagged template
     const strings = args[1];
     const values = args.slice(2);
     fnCall = makeTaggedTemplateCall(
-      upgradeReference(target),
+      upgradeReference(target, mode),
       strings,
       ...values
     );
   } else {
     // Function call with explicit or implicit parentheses
-    fnCall = [upgradeReference(target), ...args];
+    fnCall = [upgradeReference(target, mode), ...args];
   }
 
   // Create a location spanning the newly-constructed function call.
@@ -276,6 +306,21 @@ export function makeDeferredArguments(args) {
   });
 }
 
+export function makeDocument(mode, front, body, location) {
+  // In order for template expressions to see the front matter properties,
+  // we translate the top-level front properties to object entries.
+  const entries = Object.entries(front).map(([key, value]) =>
+    annotate([key, annotate([ops.literal, value], location)], location)
+  );
+
+  // Add an entry for the body
+  const bodyKey = mode === "jse" ? "_body" : "@text";
+  entries.push(annotate([bodyKey, body], location));
+
+  // Return the code for the document object
+  return annotate([ops.object, ...entries], location);
+}
+
 export function makeJsPropertyAccess(expression, property) {
   const location = {
     source: expression.location.source,
@@ -285,6 +330,70 @@ export function makeJsPropertyAccess(expression, property) {
   return annotate([expression, property], location);
 }
 
+/**
+ * From the given spreads within an object spread, return the merge.
+ *
+ * Example:
+ *
+ *      {
+ *        x = { a: 1 }
+ *        …x
+ *        y = x
+ *      }
+ *
+ *  will be treated as:
+ *
+ *      {
+ *        x = { a: 1 }
+ *        y = x
+ *        _result: {
+ *          x
+ *          …x
+ *          y
+ *        }
+ *      }.result
+ *
+ * @param {*} spreads
+ * @param {CodeLocation} location
+ */
+function makeMerge(spreads, location) {
+  const topEntries = [];
+  const resultEntries = [];
+  for (const spread of spreads) {
+    if (spread[0] === ops.object) {
+      topEntries.push(...spread.slice(1));
+      // Also add an object to the result with indirect references
+      const indirectEntries = spread.slice(1).map((entry) => {
+        const [key] = entry;
+        const context = annotate([ops.context, 1], entry.location);
+        const reference = annotate([context, key], entry.location);
+        const getter = annotate([ops.getter, reference], entry.location);
+        return annotate([key, getter], entry.location);
+      });
+      const indirectObject = annotate(
+        [ops.object, ...indirectEntries],
+        location
+      );
+      resultEntries.push(indirectObject);
+    } else {
+      resultEntries.push(spread);
+    }
+  }
+
+  // Merge to create result
+  const result = annotate([ops.merge, ...resultEntries], location);
+
+  // Add the result to the top-level object as _result
+  topEntries.push(annotate(["_result", result], location));
+
+  // Construct the top-level object
+  const topObject = annotate([ops.object, ...topEntries], location);
+
+  // Get the _result property
+  const code = annotate([topObject, "_result"], location);
+  return code;
+}
+
 /**
  * Make an object.
  *
@@ -341,7 +450,7 @@ export function makeObject(entries, location) {
   let code;
   if (spreads.length > 1) {
     // Merge multiple spreads
-    code = [ops.merge, ...spreads];
+    code = makeMerge(spreads, location);
   } else if (spreads.length === 1) {
     // A single spread can just be the object
     code = spreads[0];
@@ -358,10 +467,11 @@ export function makeObject(entries, location) {
  *
  * @param {AnnotatedCode} arg
  * @param {AnnotatedCode} fn
+ * @param {string} mode
  */
-export function makePipeline(arg, fn) {
-  const upgraded = upgradeReference(fn);
-  const result = makeCall(upgraded, [arg]);
+export function makePipeline(arg, fn, mode) {
+  const upgraded = upgradeReference(fn, mode);
+  const result = makeCall(upgraded, [arg], mode);
   const source = fn.location.source;
   let start = arg.location.start;
   let end = fn.location.end;
@@ -374,21 +484,6 @@ export function makeProperty(key, value) {
   return [key, modified];
 }
 
-export function makeReference(identifier) {
-  // We can't know for sure that an identifier is a builtin reference until we
-  // see whether it's being called as a function.
-  let op;
-  if (builtinRegex.test(identifier)) {
-    op = identifier.endsWith(":")
-      ? // Namespace is always a builtin reference
-        ops.builtin
-      : undetermined;
-  } else {
-    op = ops.scope;
-  }
-  return [op, identifier];
-}
-
 /**
  * Make a tagged template call
  *
@@ -496,7 +591,7 @@ export function makeYamlObject(text, location) {
     throw error;
   }
 
-  return annotate([ops.literal, parsed], location);
+  return parsed;
 }
 
 /**
@@ -504,9 +599,14 @@ export function makeYamlObject(text, location) {
  *
  * @param {AnnotatedCode} code
  */
-export function upgradeReference(code) {
-  if (code.length === 2 && code[0] === undetermined) {
-    const result = [ops.builtin, code[1]];
+export function upgradeReference(code, mode) {
+  if (
+    mode === "shell" &&
+    code.length === 2 &&
+    code[0] === markers.reference &&
+    builtinRegex.exec(code[1][1])
+  ) {
+    const result = [markers.global, code[1][1]];
     return annotate(result, code.location);
   } else {
     return code;

package/src/runtime/HandleExtensionsTransform.js

@@ -1,3 +1,4 @@
+import getHandlers from "./getHandlers.js";
 import { handleExtension } from "./handlers.js";
 
 /**
@@ -9,9 +10,17 @@ import { handleExtension } from "./handlers.js";
  */
 export default function HandleExtensionsTransform(Base) {
   return class FileLoaders extends Base {
+    constructor(...args) {
+      super(...args);
+
+      // Callers should set this to the set of supported extension handlers
+      this.handlers = null;
+    }
+
     async get(key) {
       const value = await super.get(key);
-      return handleExtension(this, value, key);
+      const handlers = getHandlers(this);
+      return handleExtension(this, value, key, handlers);
     }
   };
 }

package/src/runtime/evaluate.js

@@ -1,6 +1,5 @@
-import { Tree, isUnpackable, scope } from "@weborigami/async-tree";
+import { Tree, isUnpackable } from "@weborigami/async-tree";
 import codeFragment from "./codeFragment.js";
-import { codeSymbol, scopeSymbol, sourceSymbol } from "./symbols.js";
 
 /**
  * Evaluate the given code and return the result.
@@ -52,7 +51,7 @@ export default async function evaluate(code) {
     result =
       fn instanceof Function
         ? await fn.call(tree, ...args) // Invoke the function
-        : await Tree.traverseOrThrow(fn, ...args); // Traverse the tree.
+        : await Tree.traverseOrThrow.call(tree, fn, ...args); // Traverse the tree.
   } catch (/** @type {any} */ error) {
     if (!error.location) {
       // Attach the location of the code we tried to evaluate.
@@ -67,39 +66,33 @@ export default async function evaluate(code) {
     throw error;
   }
 
-  // If the result is a tree, then the default parent of the tree is the current
-  // tree.
-  if (Tree.isAsyncTree(result) && !result.parent) {
-    result.parent = tree;
-  }
-
   // To aid debugging, add the code to the result.
-  if (Object.isExtensible(result)) {
-    try {
-      if (code.location && !result[sourceSymbol]) {
-        Object.defineProperty(result, sourceSymbol, {
-          value: codeFragment(code.location),
-          enumerable: false,
-        });
-      }
-      if (!result[codeSymbol]) {
-        Object.defineProperty(result, codeSymbol, {
-          value: code,
-          enumerable: false,
-        });
-      }
-      if (!result[scopeSymbol]) {
-        Object.defineProperty(result, scopeSymbol, {
-          get() {
-            return scope(this).trees;
-          },
-          enumerable: false,
-        });
-      }
-    } catch (/** @type {any} */ error) {
-      // Ignore errors.
-    }
-  }
+  // if (Object.isExtensible(result)) {
+  //   try {
+  //     if (code.location && !result[sourceSymbol]) {
+  //       Object.defineProperty(result, sourceSymbol, {
+  //         value: codeFragment(code.location),
+  //         enumerable: false,
+  //       });
+  //     }
+  //     if (!result[codeSymbol]) {
+  //       Object.defineProperty(result, codeSymbol, {
+  //         value: code,
+  //         enumerable: false,
+  //       });
+  //     }
+  //     if (!result[scopeSymbol]) {
+  //       Object.defineProperty(result, scopeSymbol, {
+  //         get() {
+  //           return scope(this).trees;
+  //         },
+  //         enumerable: false,
+  //       });
+  //     }
+  //   } catch (/** @type {any} */ error) {
+  //     // Ignore errors.
+  //   }
+  // }
 
   return result;
 }

package/src/runtime/expressionObject.js

@@ -6,6 +6,7 @@ import {
   trailingSlash,
   Tree,
 } from "@weborigami/async-tree";
+import getHandlers from "./getHandlers.js";
 import { handleExtension } from "./handlers.js";
 import { evaluate, ops } from "./internal.js";
 
@@ -32,6 +33,7 @@ export default async function expressionObject(entries, parent) {
   if (parent !== null && !Tree.isAsyncTree(parent)) {
     throw new TypeError(`Parent must be an AsyncTree or null`);
   }
+  setParent(object, parent);
 
   let tree;
   const eagerProperties = [];
@@ -90,7 +92,8 @@ export default async function expressionObject(entries, parent) {
         get = async () => {
           tree ??= new ObjectTree(object);
           const result = await evaluate.call(tree, code);
-          return handleExtension(tree, result, key);
+          const handlers = getHandlers(tree);
+          return handleExtension(tree, result, key, handlers);
         };
       } else {
         // No extension, so getter just invokes code.
@@ -116,9 +119,6 @@ export default async function expressionObject(entries, parent) {
     writable: true,
   });
 
-  // Attach the parent
-  setParent(object, parent);
-
   // Evaluate any properties that were declared as immediate: get their value
   // and overwrite the property getter with the actual value.
   for (const key of eagerProperties) {

package/src/runtime/getHandlers.js

@@ -0,0 +1,10 @@
+import { Tree } from "@weborigami/async-tree";
+
+// Return the extension handlers for the given tree
+export default function getHandlers(tree) {
+  if (!tree) {
+    return null;
+  }
+  const root = Tree.root(tree);
+  return root.handlers;
+}

package/src/runtime/handlers.js

@@ -4,60 +4,10 @@ import {
   isPacked,
   isStringLike,
   isUnpackable,
-  scope,
   setParent,
   trailingSlash,
 } from "@weborigami/async-tree";
 
-/** @typedef {import("../../index.ts").ExtensionHandler} ExtensionHandler */
-
-// Track extensions handlers for a given containing tree.
-const handlersForContainer = new Map();
-
-/**
- * Find an extension handler for a file in the given container.
- *
- * @typedef {import("@weborigami/types").AsyncTree} AsyncTree
- *
- * @param {AsyncTree} parent
- * @param {string} extension
- */
-export async function getExtensionHandler(parent, extension) {
-  let handlers = handlersForContainer.get(parent);
-  if (handlers) {
-    if (handlers[extension]) {
-      return handlers[extension];
-    }
-  } else {
-    handlers = {};
-    handlersForContainer.set(parent, handlers);
-  }
-
-  const handlerName = `${extension.slice(1)}.handler`;
-  const parentScope = scope(parent);
-
-  /** @type {Promise<ExtensionHandler>} */
-  let handlerPromise = parentScope
-    ?.get(handlerName)
-    .then(async (extensionHandler) => {
-      if (isUnpackable(extensionHandler)) {
-        // The extension handler itself needs to be unpacked. E.g., if it's a
-        // buffer containing JavaScript file, we need to unpack it to get its
-        // default export.
-        // @ts-ignore
-        extensionHandler = await extensionHandler.unpack();
-      }
-      // Update cache with actual handler
-      handlers[extension] = extensionHandler;
-      return extensionHandler;
-    });
-
-  // Cache handler even if it's undefined so we don't look it up again
-  handlers[extension] = handlerPromise;
-
-  return handlerPromise;
-}
-
 /**
  * If the given value is packed (e.g., buffer) and the key is a string-like path
  * that ends in an extension, search for a handler for that extension and, if
@@ -67,8 +17,13 @@ export async function getExtensionHandler(parent, extension) {
  * @param {any} value
  * @param {any} key
  */
-export async function handleExtension(parent, value, key) {
-  if (isPacked(value) && isStringLike(key) && value.unpack === undefined) {
+export async function handleExtension(parent, value, key, handlers) {
+  if (
+    handlers &&
+    isPacked(value) &&
+    isStringLike(key) &&
+    value.unpack === undefined
+  ) {
     const hasSlash = trailingSlash.has(key);
     if (hasSlash) {
       key = trailingSlash.remove(key);
@@ -82,8 +37,14 @@ export async function handleExtension(parent, value, key) {
       ? ".jsedocument"
       : extension.extname(key);
     if (extname) {
-      const handler = await getExtensionHandler(parent, extname);
+      const handlerName = `${extname.slice(1)}.handler`;
+      let handler = await handlers.get(handlerName);
       if (handler) {
+        if (isUnpackable(handler)) {
+          // The extension handler itself needs to be unpacked
+          handler = await handler.unpack();
+        }
+
         if (hasSlash && handler.unpack) {
           // Key like `data.json/` ends in slash -- unpack immediately
           return handler.unpack(value, { key, parent });

package/src/runtime/jsGlobals.js

@@ -0,0 +1,99 @@
+import path from "node:path";
+
+/**
+ * The complete set of support JavaScript globals and global-like values.
+ *
+ * See
+ * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects.
+ * That page lists some things like `TypedArrays` which are not globals so are
+ * omitted here.
+ */
+export default {
+  AggregateError,
+  Array,
+  ArrayBuffer,
+  Atomics,
+  BigInt,
+  BigInt64Array,
+  BigUint64Array,
+  Boolean,
+  DataView,
+  Date,
+  Error,
+  EvalError,
+  FinalizationRegistry,
+  Float32Array,
+  Float64Array,
+  Function,
+  Infinity,
+  Int16Array,
+  Int32Array,
+  Int8Array,
+  Intl,
+  // @ts-ignore Iterator does exist despite what TypeScript thinks
+  Iterator,
+  JSON,
+  Map,
+  Math,
+  NaN,
+  Number,
+  Object,
+  Promise,
+  Proxy,
+  RangeError,
+  ReferenceError,
+  Reflect,
+  RegExp,
+  Set,
+  SharedArrayBuffer,
+  String,
+  Symbol,
+  SyntaxError,
+  TypeError,
+  URIError,
+  Uint16Array,
+  Uint32Array,
+  Uint8Array,
+  Uint8ClampedArray,
+  WeakMap,
+  WeakRef,
+  WeakSet,
+  decodeURI,
+  decodeURIComponent,
+  encodeURI,
+  encodeURIComponent,
+  eval,
+  false: false, // treat like a global
+  fetch: fetchWrapper, // special case
+  globalThis,
+  import: importWrapper, // not a function in JS but acts like one
+  isFinite,
+  isNaN,
+  null: null, // treat like a global
+  parseFloat,
+  parseInt,
+  true: true, // treat like a global
+  undefined,
+};
+
+async function fetchWrapper(resource, options) {
+  const response = await fetch(resource, options);
+  return response.ok ? await response.arrayBuffer() : undefined;
+}
+
+/** @this {import("@weborigami/types").AsyncTree|null|undefined} */
+async function importWrapper(modulePath) {
+  // Walk up parent tree looking for a FileTree or other object with a `path`
+  /** @type {any} */
+  let current = this;
+  while (current && !("path" in current)) {
+    current = current.parent;
+  }
+  if (!current) {
+    throw new TypeError(
+      "Modules can only be imported from a folder or other object with a path property."
+    );
+  }
+  const filePath = path.resolve(current.path, modulePath);
+  return import(filePath);
+}

package/src/runtime/mergeTrees.js

@@ -53,11 +53,6 @@ export default async function mergeTrees(...trees) {
     return result;
   }
 
-  // If all trees are arrays, return an array.
-  if (unpacked.every((tree) => Array.isArray(tree))) {
-    return unpacked.flat();
-  }
-
   // Merge the trees.
   const result = merge(...unpacked);
   setParent(result, this);