@weborigami/language 0.3.3 → 0.3.4-jse.5

package/src/compiler/parserHelpers.js

@@ -15,10 +15,14 @@ const YAML = YAMLModule.default ?? YAMLModule.YAML;
 /** @typedef {import("../../index.ts").CodeLocation} CodeLocation */
 /** @typedef {import("../../index.ts").Code} Code */
 
-// Marker for a reference that may be a builtin or a scope reference
-export const undetermined = Symbol("undetermined");
-
-const builtinRegex = /^[A-Za-z][A-Za-z0-9]*$/;
+// Markers in compiled output, will get optimized away
+export const markers = {
+  global: Symbol("global"), // Global reference
+  external: Symbol("external"), // External reference
+  property: Symbol("property"), // Property access
+  reference: Symbol("reference"), // Reference to local, scope, or global
+  traverse: Symbol("traverse"), // Path traversal
+};
 
 /**
  * If a parse result is an object that will be evaluated at runtime, attach the
@@ -37,7 +41,7 @@ export function annotate(code, location) {
 }
 
 /**
- * In the given code, replace all scope refernces to the given name with the
+ * In the given code, replace all function calls to the given name with the
  * given macro code.
  *
  * @param {AnnotatedCode} code
@@ -49,59 +53,20 @@ export function applyMacro(code, name, macro) {
     return code;
   }
 
-  const [fn, ...args] = code;
-  if (fn === ops.scope && args[0] === name) {
-    return macro;
-  }
-
-  const applied = code.map((child) => applyMacro(child, name, macro));
-  return annotate(applied, code.location);
-}
-
-/**
- * The indicated code is being used to define a property named by the given key.
- * Rewrite any [ops.scope, key] calls to be [ops.inherited, key] to avoid
- * infinite recursion.
- *
- * @param {AnnotatedCode} code
- * @param {string} key
- */
-function avoidRecursivePropertyCalls(code, key) {
-  if (!(code instanceof Array)) {
-    return code;
-  }
-  /** @type {Code} */
-  let modified;
+  // We're looking for a function call with the given name.
+  // For `foo`, the call would be: [[markers.traverse, [markers.reference,  "foo"]], undefined]
   if (
-    code[0] === ops.scope &&
-    trailingSlash.remove(code[1]) === trailingSlash.remove(key)
+    code[0] instanceof Array &&
+    code[0][0] === markers.traverse &&
+    code[0][1][0] === markers.reference &&
+    code[0][1][1] === name
   ) {
-    // Rewrite to avoid recursion
-    modified = [ops.inherited, code[1]];
-  } else if (
-    code[0] === ops.lambda &&
-    code[1].some((param) => param[1] === key)
-  ) {
-    // Lambda that defines the key; don't rewrite
-    return code;
-  } else {
-    // Process any nested code
-    modified = code.map((value) => avoidRecursivePropertyCalls(value, key));
+    // Replace the call with the macro
+    return annotate(macro, code.location);
   }
-  return annotate(modified, code.location);
-}
 
-/**
- * Downgrade a potential builtin reference to a scope reference.
- *
- * @param {AnnotatedCode} code
- */
-export function downgradeReference(code) {
-  if (code && code.length === 2 && code[0] === undetermined) {
-    return annotate([ops.scope, code[1]], code.location);
-  } else {
-    return code;
-  }
+  const applied = code.map((child) => applyMacro(child, name, macro));
+  return annotate(applied, code.location);
 }
 
 /**
@@ -139,7 +104,7 @@ export function makeArray(entries, location) {
 
   let result;
   if (spreads.length > 1) {
-    result = [ops.merge, ...spreads];
+    result = [ops.flat, ...spreads];
   } else if (spreads.length === 1) {
     result = spreads[0];
   } else {
@@ -196,7 +161,7 @@ export function makeBinaryOperation(left, [operatorToken, right]) {
  * @param {AnnotatedCode} target
  * @param {any[]} args
  */
-export function makeCall(target, args) {
+export function makeCall(target, args, location) {
   if (!(target instanceof Array)) {
     const error = new SyntaxError(`Can't call this like a function: ${target}`);
     /** @type {any} */ (error).location = /** @type {any} */ (target).location;
@@ -204,53 +169,23 @@ export function makeCall(target, args) {
   }
 
   let fnCall;
-  if (args[0] === ops.traverse) {
-    let tree = target;
-
-    if (tree[0] === undetermined) {
-      // In a traversal, downgrade ops.builtin references to ops.scope
-      tree = downgradeReference(tree);
-      if (tree[0] === ops.scope && !trailingSlash.has(tree[1])) {
-        // Target didn't parse with a trailing slash; add one
-        tree[1] = trailingSlash.add(tree[1]);
-      }
-    }
-
-    if (args.length > 1) {
-      // Regular traverse
-      const keys = args.slice(1);
-      fnCall = [ops.traverse, tree, ...keys];
-    } else {
-      // Traverse without arguments equates to unpack
-      fnCall = [ops.unpack, tree];
-    }
-  } else if (args[0] === ops.template) {
+  const op = args[0];
+  if (op === markers.traverse || op === ops.optionalTraverse) {
+    // Traverse
+    const keys = args.slice(1);
+    fnCall = [target, ...keys];
+  } else if (op === markers.property) {
+    // Property access
+    const property = args[1];
+    fnCall = [target, property];
+  } else if (op === ops.templateTree) {
     // Tagged template
     const strings = args[1];
     const values = args.slice(2);
-    fnCall = makeTaggedTemplateCall(
-      upgradeReference(target),
-      strings,
-      ...values
-    );
+    fnCall = makeTaggedTemplateCall(target, strings, ...values);
   } else {
     // Function call with explicit or implicit parentheses
-    fnCall = [upgradeReference(target), ...args];
-  }
-
-  // Create a location spanning the newly-constructed function call.
-  const location = { ...target.location };
-  if (args instanceof Array) {
-    let end;
-    if ("location" in args) {
-      end = /** @type {any} */ (args).location.end;
-    } else if ("location" in args.at(-1)) {
-      end = args.at(-1).location.end;
-    }
-    if (end === undefined) {
-      throw "Internal parser error: no location for function call argument";
-    }
-    location.end = end;
+    fnCall = [target, ...args];
   }
 
   return annotate(fnCall, location);
@@ -275,6 +210,86 @@ export function makeDeferredArguments(args) {
   });
 }
 
+export function makeDocument(front, body, location) {
+  // In order for template expressions to see the front matter properties,
+  // we translate the top-level front properties to object entries.
+  const entries = Object.entries(front).map(([key, value]) =>
+    annotate([key, annotate([ops.literal, value], location)], location)
+  );
+
+  // Add an entry for the body
+  // TODO: Deprecate @text
+  entries.push(annotate(["(@text)", body], location));
+  entries.push(annotate(["_body", body], location));
+
+  // Return the code for the document object
+  return annotate([ops.object, ...entries], location);
+}
+
+/**
+ * From the given spreads within an object spread, return the merge.
+ *
+ * Example:
+ *
+ *      {
+ *        x = { a: 1 }
+ *        …x
+ *        y = x
+ *      }
+ *
+ *  will be treated as:
+ *
+ *      {
+ *        x = { a: 1 }
+ *        y = x
+ *        _result: {
+ *          x
+ *          …x
+ *          y
+ *        }
+ *      }.result
+ *
+ * @param {*} spreads
+ * @param {CodeLocation} location
+ */
+function makeMerge(spreads, location) {
+  const topEntries = [];
+  const resultEntries = [];
+  for (const spread of spreads) {
+    if (spread[0] === ops.object) {
+      topEntries.push(...spread.slice(1));
+      // Also add an object to the result with indirect references
+      const indirectEntries = spread.slice(1).map((entry) => {
+        const [key] = entry;
+        const context = annotate([ops.context, 1], entry.location);
+        const reference = annotate([context, key], entry.location);
+        const getter = annotate([ops.getter, reference], entry.location);
+        return annotate([key, getter], entry.location);
+      });
+      const indirectObject = annotate(
+        [ops.object, ...indirectEntries],
+        location
+      );
+      resultEntries.push(indirectObject);
+    } else {
+      resultEntries.push(spread);
+    }
+  }
+
+  // Merge to create result
+  const result = annotate([ops.merge, ...resultEntries], location);
+
+  // Add the result to the top-level object as _result
+  topEntries.push(annotate(["_result", result], location));
+
+  // Construct the top-level object
+  const topObject = annotate([ops.object, ...topEntries], location);
+
+  // Get the _result property
+  const code = annotate([topObject, "_result"], location);
+  return code;
+}
+
 /**
  * Make an object.
  *
@@ -331,7 +346,7 @@ export function makeObject(entries, location) {
   let code;
   if (spreads.length > 1) {
     // Merge multiple spreads
-    code = [ops.merge, ...spreads];
+    code = makeMerge(spreads, location);
   } else if (spreads.length === 1) {
     // A single spread can just be the object
     code = spreads[0];
@@ -343,51 +358,47 @@ export function makeObject(entries, location) {
   return annotate(code, location);
 }
 
+/**
+ * Handle a path with one or more segments separated by slashes.
+ *
+ * @param {AnnotatedCode} keys
+ */
+export function makePath(keys) {
+  // Remove empty segments
+  const args = keys.filter(
+    (key, index) => index === 0 || (key[1] !== "" && key[1] !== "/")
+  );
+
+  // Upgrade head to a reference
+  const [head, ...tail] = args;
+  const headKey = head[1];
+  const reference = annotate([markers.reference, headKey], head.location);
+
+  let code = [markers.traverse, reference, ...tail];
+  code.location = spanLocations(code);
+
+  // Last key has trailing slash implies unpack operation
+  if (trailingSlash.has(args.at(-1)[1])) {
+    code = annotate([ops.unpack, code], code.location);
+  }
+
+  return code;
+}
+
 /**
  * Make a pipline: similar to a function call, but the order is reversed.
  *
  * @param {AnnotatedCode} arg
  * @param {AnnotatedCode} fn
  */
-export function makePipeline(arg, fn) {
-  const upgraded = upgradeReference(fn);
-  const result = makeCall(upgraded, [arg]);
+export function makePipeline(arg, fn, location) {
+  const result = makeCall(fn, [arg], location);
   const source = fn.location.source;
   let start = arg.location.start;
   let end = fn.location.end;
   return annotate(result, { start, source, end });
 }
 
-// Define a property on an object.
-export function makeProperty(key, value) {
-  const modified = avoidRecursivePropertyCalls(value, key);
-  return [key, modified];
-}
-
-export function makeJsPropertyAccess(expression, property) {
-  const location = {
-    source: expression.location.source,
-    start: expression.location.start,
-    end: property.location.end,
-  };
-  return annotate([expression, property], location);
-}
-
-export function makeReference(identifier) {
-  // We can't know for sure that an identifier is a builtin reference until we
-  // see whether it's being called as a function.
-  let op;
-  if (builtinRegex.test(identifier)) {
-    op = identifier.endsWith(":")
-      ? // Namespace is always a builtin reference
-        ops.builtin
-      : undetermined;
-  } else {
-    op = ops.scope;
-  }
-  return [op, identifier];
-}
-
 /**
  * Make a tagged template call
  *
@@ -495,19 +506,17 @@ export function makeYamlObject(text, location) {
     throw error;
   }
 
-  return annotate([ops.literal, parsed], location);
+  return parsed;
 }
 
-/**
- * Upgrade a potential builtin reference to an actual builtin reference.
- *
- * @param {AnnotatedCode} code
- */
-export function upgradeReference(code) {
-  if (code.length === 2 && code[0] === undetermined) {
-    const result = [ops.builtin, code[1]];
-    return annotate(result, code.location);
-  } else {
-    return code;
-  }
+// Create a locations that spans those in the array. This assumes the locations
+// are in order and non-overlapping.
+function spanLocations(code) {
+  const first = code.find((item) => item.location).location;
+  const last = code[code.findLastIndex((item) => item.location)].location;
+  return {
+    source: first.source,
+    start: first.start,
+    end: last.end,
+  };
 }

package/src/runtime/HandleExtensionsTransform.js

@@ -1,3 +1,4 @@
+import getHandlers from "./getHandlers.js";
 import { handleExtension } from "./handlers.js";
 
 /**
@@ -9,9 +10,17 @@ import { handleExtension } from "./handlers.js";
  */
 export default function HandleExtensionsTransform(Base) {
   return class FileLoaders extends Base {
+    constructor(...args) {
+      super(...args);
+
+      // Callers should set this to the set of supported extension handlers
+      this.handlers = null;
+    }
+
     async get(key) {
       const value = await super.get(key);
-      return handleExtension(this, value, key);
+      const handlers = getHandlers(this);
+      return handleExtension(this, value, key, handlers);
     }
   };
 }

package/src/runtime/evaluate.js

@@ -1,6 +1,5 @@
-import { Tree, isUnpackable, scope } from "@weborigami/async-tree";
+import { Tree, isUnpackable } from "@weborigami/async-tree";
 import codeFragment from "./codeFragment.js";
-import { codeSymbol, scopeSymbol, sourceSymbol } from "./symbols.js";
 
 /**
  * Evaluate the given code and return the result.
@@ -52,7 +51,7 @@ export default async function evaluate(code) {
     result =
       fn instanceof Function
         ? await fn.call(tree, ...args) // Invoke the function
-        : await Tree.traverseOrThrow(fn, ...args); // Traverse the tree.
+        : await Tree.traverseOrThrow.call(tree, fn, ...args); // Traverse the tree.
   } catch (/** @type {any} */ error) {
     if (!error.location) {
       // Attach the location of the code we tried to evaluate.
@@ -67,39 +66,33 @@ export default async function evaluate(code) {
     throw error;
   }
 
-  // If the result is a tree, then the default parent of the tree is the current
-  // tree.
-  if (Tree.isAsyncTree(result) && !result.parent) {
-    result.parent = tree;
-  }
-
   // To aid debugging, add the code to the result.
-  if (Object.isExtensible(result)) {
-    try {
-      if (code.location && !result[sourceSymbol]) {
-        Object.defineProperty(result, sourceSymbol, {
-          value: codeFragment(code.location),
-          enumerable: false,
-        });
-      }
-      if (!result[codeSymbol]) {
-        Object.defineProperty(result, codeSymbol, {
-          value: code,
-          enumerable: false,
-        });
-      }
-      if (!result[scopeSymbol]) {
-        Object.defineProperty(result, scopeSymbol, {
-          get() {
-            return scope(this).trees;
-          },
-          enumerable: false,
-        });
-      }
-    } catch (/** @type {any} */ error) {
-      // Ignore errors.
-    }
-  }
+  // if (Object.isExtensible(result)) {
+  //   try {
+  //     if (code.location && !result[sourceSymbol]) {
+  //       Object.defineProperty(result, sourceSymbol, {
+  //         value: codeFragment(code.location),
+  //         enumerable: false,
+  //       });
+  //     }
+  //     if (!result[codeSymbol]) {
+  //       Object.defineProperty(result, codeSymbol, {
+  //         value: code,
+  //         enumerable: false,
+  //       });
+  //     }
+  //     if (!result[scopeSymbol]) {
+  //       Object.defineProperty(result, scopeSymbol, {
+  //         get() {
+  //           return scope(this).trees;
+  //         },
+  //         enumerable: false,
+  //       });
+  //     }
+  //   } catch (/** @type {any} */ error) {
+  //     // Ignore errors.
+  //   }
+  // }
 
   return result;
 }

package/src/runtime/expressionObject.js

@@ -6,6 +6,7 @@ import {
   trailingSlash,
   Tree,
 } from "@weborigami/async-tree";
+import getHandlers from "./getHandlers.js";
 import { handleExtension } from "./handlers.js";
 import { evaluate, ops } from "./internal.js";
 
@@ -32,6 +33,7 @@ export default async function expressionObject(entries, parent) {
   if (parent !== null && !Tree.isAsyncTree(parent)) {
     throw new TypeError(`Parent must be an AsyncTree or null`);
   }
+  setParent(object, parent);
 
   let tree;
   const eagerProperties = [];
@@ -90,7 +92,8 @@ export default async function expressionObject(entries, parent) {
         get = async () => {
           tree ??= new ObjectTree(object);
           const result = await evaluate.call(tree, code);
-          return handleExtension(tree, result, key);
+          const handlers = getHandlers(tree);
+          return handleExtension(tree, result, key, handlers);
         };
       } else {
         // No extension, so getter just invokes code.
@@ -116,9 +119,6 @@ export default async function expressionObject(entries, parent) {
     writable: true,
   });
 
-  // Attach the parent
-  setParent(object, parent);
-
   // Evaluate any properties that were declared as immediate: get their value
   // and overwrite the property getter with the actual value.
   for (const key of eagerProperties) {
@@ -136,17 +136,23 @@ export default async function expressionObject(entries, parent) {
   return object;
 }
 
-function entryKey(object, eagerProperties, entry) {
-  const [key, value] = entry;
+export function entryKey(entry, object = null, eagerProperties = []) {
+  let [key, value] = entry;
+
+  if (key[0] === "(" && key[key.length - 1] === ")") {
+    // Non-enumerable property, remove parentheses. This doesn't come up in the
+    // constructor, but can happen in situations encountered by the compiler's
+    // optimizer.
+    key = key.slice(1, -1);
+  }
 
-  const hasExplicitSlash = trailingSlash.has(key);
-  if (hasExplicitSlash) {
-    // Return key as is
+  if (trailingSlash.has(key)) {
+    // Explicit trailing slash, return as is
     return key;
   }
 
   // If eager property value is treelike, add slash to the key
-  if (eagerProperties.includes(key) && Tree.isTreelike(object[key])) {
+  if (eagerProperties.includes(key) && Tree.isTreelike(object?.[key])) {
     return trailingSlash.add(key);
   }
 
@@ -163,5 +169,5 @@ function entryKey(object, eagerProperties, entry) {
 function keys(object, eagerProperties, propertyIsEnumerable, entries) {
   return entries
     .filter(([key]) => propertyIsEnumerable[key])
-    .map((entry) => entryKey(object, eagerProperties, entry));
+    .map((entry) => entryKey(entry, object, eagerProperties));
 }

package/src/runtime/getHandlers.js

@@ -0,0 +1,10 @@
+import { Tree } from "@weborigami/async-tree";
+
+// Return the extension handlers for the given tree
+export default function getHandlers(tree) {
+  if (!tree) {
+    return null;
+  }
+  const root = Tree.root(tree);
+  return root.handlers;
+}

package/src/runtime/handlers.js

@@ -4,60 +4,10 @@ import {
   isPacked,
   isStringLike,
   isUnpackable,
-  scope,
   setParent,
   trailingSlash,
 } from "@weborigami/async-tree";
 
-/** @typedef {import("../../index.ts").ExtensionHandler} ExtensionHandler */
-
-// Track extensions handlers for a given containing tree.
-const handlersForContainer = new Map();
-
-/**
- * Find an extension handler for a file in the given container.
- *
- * @typedef {import("@weborigami/types").AsyncTree} AsyncTree
- *
- * @param {AsyncTree} parent
- * @param {string} extension
- */
-export async function getExtensionHandler(parent, extension) {
-  let handlers = handlersForContainer.get(parent);
-  if (handlers) {
-    if (handlers[extension]) {
-      return handlers[extension];
-    }
-  } else {
-    handlers = {};
-    handlersForContainer.set(parent, handlers);
-  }
-
-  const handlerName = `${extension.slice(1)}.handler`;
-  const parentScope = scope(parent);
-
-  /** @type {Promise<ExtensionHandler>} */
-  let handlerPromise = parentScope
-    ?.get(handlerName)
-    .then(async (extensionHandler) => {
-      if (isUnpackable(extensionHandler)) {
-        // The extension handler itself needs to be unpacked. E.g., if it's a
-        // buffer containing JavaScript file, we need to unpack it to get its
-        // default export.
-        // @ts-ignore
-        extensionHandler = await extensionHandler.unpack();
-      }
-      // Update cache with actual handler
-      handlers[extension] = extensionHandler;
-      return extensionHandler;
-    });
-
-  // Cache handler even if it's undefined so we don't look it up again
-  handlers[extension] = handlerPromise;
-
-  return handlerPromise;
-}
-
 /**
  * If the given value is packed (e.g., buffer) and the key is a string-like path
  * that ends in an extension, search for a handler for that extension and, if
@@ -67,20 +17,34 @@ export async function getExtensionHandler(parent, extension) {
  * @param {any} value
  * @param {any} key
  */
-export async function handleExtension(parent, value, key) {
-  if (isPacked(value) && isStringLike(key) && value.unpack === undefined) {
+export async function handleExtension(parent, value, key, handlers) {
+  if (
+    handlers &&
+    isPacked(value) &&
+    isStringLike(key) &&
+    value.unpack === undefined
+  ) {
     const hasSlash = trailingSlash.has(key);
     if (hasSlash) {
       key = trailingSlash.remove(key);
     }
 
-    // Special case: `.ori.<ext>` extensions are Origami documents.
+    // Special cases: `.ori.<ext>` extensions are Origami documents,
+    // `.jse.<ext>` are JSE documents.
     const extname = key.match(/\.ori\.\S+$/)
       ? ".oridocument"
+      : key.match(/\.jse\.\S+$/)
+      ? ".jsedocument"
       : extension.extname(key);
     if (extname) {
-      const handler = await getExtensionHandler(parent, extname);
+      const handlerName = `${extname.slice(1)}.handler`;
+      let handler = await handlers[handlerName];
       if (handler) {
+        if (isUnpackable(handler)) {
+          // The extension handler itself needs to be unpacked
+          handler = await handler.unpack();
+        }
+
         if (hasSlash && handler.unpack) {
           // Key like `data.json/` ends in slash -- unpack immediately
           return handler.unpack(value, { key, parent });