@weborigami/language 0.3.3-jse.2 → 0.3.3-jse.3

package/main.js

@@ -7,10 +7,12 @@ export { default as evaluate } from "./src/runtime/evaluate.js";
 export { default as EventTargetMixin } from "./src/runtime/EventTargetMixin.js";
 export * as expressionFunction from "./src/runtime/expressionFunction.js";
 export { default as functionResultsMap } from "./src/runtime/functionResultsMap.js";
+export { default as getHandlers } from "./src/runtime/getHandlers.js";
 export { default as HandleExtensionsTransform } from "./src/runtime/HandleExtensionsTransform.js";
 export * from "./src/runtime/handlers.js";
 export { default as ImportModulesMixin } from "./src/runtime/ImportModulesMixin.js";
 export { default as InvokeFunctionsTransform } from "./src/runtime/InvokeFunctionsTransform.js";
+export { default as jsGlobals } from "./src/runtime/jsGlobals.js";
 export * as moduleCache from "./src/runtime/moduleCache.js";
 export { default as OrigamiFiles } from "./src/runtime/OrigamiFiles.js";
 export * as symbols from "./src/runtime/symbols.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@weborigami/language",
-  "version": "0.3.3-jse.2",
+  "version": "0.3.3-jse.3",
   "description": "Web Origami expression language compiler and runtime",
   "type": "module",
   "main": "./main.js",
@@ -11,8 +11,8 @@
     "typescript": "5.8.2"
   },
   "dependencies": {
-    "@weborigami/async-tree": "0.3.3-jse.2",
-    "@weborigami/types": "0.3.3-jse.2",
+    "@weborigami/async-tree": "0.3.3-jse.3",
+    "@weborigami/types": "0.3.3-jse.3",
     "watcher": "2.3.1",
     "yaml": "2.7.0"
   },

package/src/compiler/compile.js

@@ -1,10 +1,12 @@
 import { createExpressionFunction } from "../runtime/expressionFunction.js";
+import jsGlobals from "../runtime/jsGlobals.js";
 import optimize from "./optimize.js";
 import { parse } from "./parse.js";
 
 function compile(source, options) {
-  const { macros, startRule } = options;
+  const { startRule } = options;
   const mode = options.mode ?? "shell";
+  const globals = options.globals ?? jsGlobals;
   const enableCaching = options.scopeCaching ?? true;
   if (typeof source === "string") {
     source = { text: source };
@@ -14,7 +16,11 @@ function compile(source, options) {
     mode,
     startRule,
   });
-  const optimized = optimize(code, enableCaching, macros);
+  const optimized = optimize(code, {
+    enableCaching,
+    globals,
+    mode,
+  });
   const fn = createExpressionFunction(optimized);
   return fn;
 }

package/src/compiler/optimize.js

@@ -1,40 +1,60 @@
 import { pathFromKeys, trailingSlash } from "@weborigami/async-tree";
 import { ops } from "../runtime/internal.js";
-import { annotate, undetermined } from "./parserHelpers.js";
+import jsGlobals from "../runtime/jsGlobals.js";
+import { annotate, markers } from "./parserHelpers.js";
 
 /**
  * Optimize an Origami code instruction:
  *
- * - Transform any remaining undetermined references to scope references.
+ * - Transform any remaining reference references to scope references.
  * - Transform those or explicit ops.scope calls to ops.external calls unless
  *   they refer to local variables (variables defined by object literals or
  *   lambda parameters).
- * - Apply any macros to the code.
  *
  * @typedef {import("./parserHelpers.js").AnnotatedCode} AnnotatedCode
  * @typedef {import("./parserHelpers.js").Code} Code
  *
  * @param {AnnotatedCode} code
- * @param {boolean} enableCaching
- * @param {Record<string, AnnotatedCode>} macros
- * @param {Record<string, AnnotatedCode>} cache
- * @param {Record<string, boolean>} locals
+ * @param {any} options
  * @returns {AnnotatedCode}
  */
-export default function optimize(
-  code,
-  enableCaching = true,
-  macros = {},
-  cache = {},
-  locals = {}
-) {
+export default function optimize(code, options = {}) {
+  const enableCaching = options.enableCaching ?? true;
+  const globals = options.globals ?? jsGlobals;
+  const mode = options.mode ?? "shell";
+  const cache = options.cache ?? {};
+
+  // The locals is an array, one item for each function or object context that
+  // has been entered. The array grows to the right. The array items are
+  // subarrays containing the names of local variables defined in that context.
+  const locals = options.locals ? options.locals.slice() : [];
+
+  const externalScope =
+    mode === "shell"
+      ? // External scope is parent scope + globals
+        annotate(
+          [ops.merge, globals, annotate([ops.scope], code.location)],
+          code.location
+        )
+      : annotate([ops.scope], code.location);
+
   // See if we can optimize this level of the code
   const [fn, ...args] = code;
-  let additionalLocalNames;
+  let optimized = code;
+  let externalReference = fn instanceof Array && fn[0] === ops.scope;
+  let depth;
   switch (fn) {
+    case markers.global:
+      // Replace global op with the globals
+      optimized = annotate([globals, args[0]], code.location);
+      break;
+
     case ops.lambda:
       const parameters = args[0];
-      additionalLocalNames = parameters.map((param) => param[1]);
+      if (parameters.length > 0) {
+        const names = parameters.map((param) => param[1]);
+        locals.push(names);
+      }
       break;
 
     case ops.literal:
@@ -46,97 +66,131 @@ export default function optimize(
 
     case ops.object:
       const entries = args;
-      additionalLocalNames = entries.map(([key]) => trailingSlash.remove(key));
+      const keys = entries.map(([key]) => propertyName(key));
+      locals.push(keys);
       break;
 
-    // Both of these are handled the same way
-    case undetermined:
-    case ops.scope:
-      const key = args[0];
-      const normalizedKey = trailingSlash.remove(key);
-      if (macros?.[normalizedKey]) {
-        // Apply macro
-        const macro = macros?.[normalizedKey];
-        return applyMacro(macro, code, enableCaching, macros, cache, locals);
-      } else if (enableCaching && !locals[normalizedKey]) {
-        // Upgrade to cached external scope reference
-        return annotate(
-          [ops.external, key, annotate([ops.scope, key], code.location), cache],
-          code.location
-        );
-      } else if (fn === undetermined) {
-        // Transform undetermined reference to regular scope call
-        return annotate([ops.scope, key], code.location);
-      } else {
-        // Internal ops.scope call; leave as is
-        return code;
+    case markers.reference:
+      // Determine whether reference is local and, if so, transform to
+      // ops.local call. Otherwise transform to ops.scope call.
+      let key = args[0];
+      if (key instanceof Array && key[0] === ops.literal) {
+        key = key[1];
       }
-
-    case ops.traverse:
-      // Is the first argument a nonscope/undetermined reference?
-      const isScopeRef =
-        args[0]?.[0] === ops.scope || args[0]?.[0] === undetermined;
-      if (enableCaching && isScopeRef) {
-        // Is the first argument a nonlocal reference?
-        const normalizedKey = trailingSlash.remove(args[0][1]);
-        if (!locals[normalizedKey]) {
-          // Are the remaining arguments all literals?
-          const allLiterals = args
-            .slice(1)
-            .every((arg) => arg[0] === ops.literal);
-          if (allLiterals) {
-            // Convert to ops.external
-            const keys = args.map((arg) => arg[1]);
-            const path = pathFromKeys(keys);
-            /** @type {Code} */
-            const optimized = [ops.external, path, code, cache];
-            return annotate(optimized, code.location);
-          }
+      const normalizedKey = trailingSlash.remove(key);
+      let target;
+      depth = getLocalReferenceDepth(locals, normalizedKey);
+      if (depth >= 0) {
+        // Transform local reference
+        const contextCode = [ops.context];
+        if (depth > 0) {
+          contextCode.push(depth);
         }
+        target = annotate(contextCode, code.location);
+      } else if (mode === "shell") {
+        // Transform non-local reference
+        target = externalScope;
+        externalReference = true;
+      } else if (mode === "jse") {
+        target = globals;
       }
+      optimized = annotate([target, ...args], code.location);
       break;
-  }
 
-  // Add any locals introduced by this code to the list that will be consulted
-  // when we descend into child nodes.
-  let updatedLocals;
-  if (additionalLocalNames) {
-    updatedLocals = { ...locals };
-    for (const key of additionalLocalNames) {
-      updatedLocals[key] = true;
-    }
-  } else {
-    updatedLocals = locals;
+    case ops.scope:
+      depth = locals.length;
+      if (depth === 0) {
+        // Use scope call as is
+        optimized = code;
+      } else {
+        // Add context for appropriate depth to scope call
+        const contextCode = annotate([ops.context, depth], code.location);
+        optimized = annotate([ops.scope, contextCode], code.location);
+      }
+      break;
   }
 
   // Optimize children
-  const optimized = code.map((child, index) => {
-    // Don't optimize lambda parameter names
-    if (fn === ops.lambda && index === 1) {
-      return child;
-    } else if (Array.isArray(child) && "location" in child) {
-      // Review: This currently descends into arrays that are not instructions,
-      // such as the entries of an ops.object. This should be harmless, but it'd
-      // be preferable to only descend into instructions. This would require
-      // surrounding ops.object entries with ops.array.
-      return optimize(
-        /** @type {AnnotatedCode} */ (child),
-        enableCaching,
-        macros,
-        cache,
-        updatedLocals
+  optimized = annotate(
+    optimized.map((child, index) => {
+      // Don't optimize lambda parameter names
+      if (fn === ops.lambda && index === 1) {
+        return child;
+      } else if (fn === ops.object && index > 0) {
+        // Code that defines a property `x` that contains references to `x`
+        // shouldn't find this context but look further up.
+        const [key, value] = child;
+        const normalizedKey = trailingSlash.remove(key);
+        let adjustedLocals;
+        if (locals.at(-1)?.includes(normalizedKey)) {
+          adjustedLocals = locals.slice();
+          // Remove the key from the current context's locals
+          adjustedLocals[adjustedLocals.length - 1] = locals
+            .at(-1)
+            .filter((name) => name !== normalizedKey);
+        } else {
+          adjustedLocals = locals;
+        }
+        return [
+          key,
+          optimize(/** @type {AnnotatedCode} */ (value), {
+            ...options,
+            locals: adjustedLocals,
+          }),
+        ];
+      } else if (Array.isArray(child) && "location" in child) {
+        // Review: Aside from ops.object (above), what non-instruction arrays
+        // does this descend into?
+        return optimize(/** @type {AnnotatedCode} */ (child), {
+          ...options,
+          locals,
+        });
+      } else {
+        return child;
+      }
+    }),
+    optimized.location
+  );
+
+  // Cache external scope or merged globals + scope references
+  if (enableCaching && externalReference) {
+    // Get all the keys so we can construct a path as a cache key
+    const keys = optimized
+      .slice(1)
+      .map((arg) =>
+        typeof arg === "string" || typeof arg === "number"
+          ? arg
+          : arg instanceof Array && arg[0] === ops.literal
+          ? arg[1]
+          : null
       );
-    } else {
-      return child;
+    if (keys.some((key) => key === null)) {
+      throw new Error("Internal error: scope reference with non-literal key");
     }
-  });
+    const path = pathFromKeys(keys);
+    optimized = annotate(
+      [ops.cache, cache, path, optimized],
+      optimized.location
+    );
+  }
 
   return annotate(optimized, code.location);
 }
 
-function applyMacro(macro, code, enableCaching, macros, cache, locals) {
-  const optimized = optimize(macro, enableCaching, macros, cache, locals);
-  return optimized instanceof Array
-    ? annotate(optimized, code.location)
-    : optimized;
+// Determine how many contexts up we need to go for a local
+function getLocalReferenceDepth(locals, key) {
+  const contextIndex = locals.findLastIndex((names) => names.includes(key));
+  if (contextIndex < 0) {
+    return -1; // Not a local reference
+  }
+  const depth = locals.length - contextIndex - 1;
+  return depth;
+}
+
+function propertyName(key) {
+  if (key[0] === "(" && key[key.length - 1] === ")") {
+    // Non-enumerable property, remove parentheses
+    key = key.slice(1, -1);
+  }
+  return trailingSlash.remove(key);
 }

package/src/compiler/origami.pegjs

@@ -20,14 +20,15 @@ import {
   makeBinaryOperation,
   makeCall,
   makeDeferredArguments,
+  makeDocument,
   makeJsPropertyAccess,
   makeObject,
   makePipeline,
   makeProperty,
-  makeReference,
   makeTemplate,
   makeUnaryOperation,
-  makeYamlObject
+  makeYamlObject,
+  markers,
 } from "./parserHelpers.js";
 import isOrigamiFrontMatter from "./isOrigamiFrontMatter.js";
 
@@ -49,31 +50,32 @@ additiveOperator
   / "-"
 
 angleBracketLiteral
-  = "<" __ protocol:angleBracketProtocol "//" path:angleBracketPath __ ">" {
+  = "<" __ protocol:angleBracketProtocol "//"? path:angleBracketPath __ ">" {
     return annotate([protocol, ...path], location());
     }
-  / "<" protocol:angleBracketProtocol path:angleBracketPath ">" {
-      const [head, ...tail] = path;
-      const root = annotate([protocol, head], location());
-      return annotate([ops.traverse, root, ...tail], location());
+  / "<" __ "/" path:angleBracketPath __ ">" {
+      const root = annotate([ops.rootDirectory], location());
+      return path.length > 0 ? annotate([root, ...path], location()) : root;
+    }
+  / "<" __ "~" "/"? path:angleBracketPath __ ">" {
+      const home = annotate([ops.homeDirectory], location());
+      return path.length > 0 ? annotate([home, ...path], location()) : home;
     }
   / "<" __ path:angleBracketPath __ ">" {
-    const [head, ...tail] = path;
-    const root = annotate([ops.scope, head[1]], location());
-    return tail.length === 0
-      ? root
-      : annotate([ops.traverse, root, ...tail], location())
-  }
+      // Angle bracket paths always reference scope
+      const scope = annotate([ops.scope], location());
+      return annotate([scope, ...path], location());
+    }
 
 angleBracketPath
   = @angleBracketPathKey|0.., "/"| "/"?
 
 angleBracketPathKey
   = chars:angleBracketPathChar+ slashFollows:slashFollows? {
-    // Append a trailing slash if one follows (but don't consume it)
-    const key = chars.join("") + (slashFollows ? "/" : "");
-    return annotate([ops.literal, key], location());
-  }
+      // Append a trailing slash if one follows (but don't consume it)
+      const key = chars.join("") + (slashFollows ? "/" : "");
+      return annotate([ops.literal, key], location());
+    }
 
 // A single character in a slash-separated path segment
 angleBracketPathChar
@@ -82,7 +84,7 @@ angleBracketPathChar
 
 angleBracketProtocol
   = protocol:jsIdentifier ":" {
-      return annotate([ops.builtin, `${protocol}:`], location());
+      return annotate([markers.global, `${protocol[1]}:`], location());
     }
 
 arguments "function arguments"
@@ -150,7 +152,7 @@ bitwiseXorOperator
 // `fn(arg1)(arg2)(arg3)`.
 callExpression "function call"
   = head:protocolExpression tail:arguments* {
-      return tail.reduce(makeCall, head);
+      return tail.reduce((target, args) => makeCall(target, args, options.mode), head);
     }
 
 // A comma-separated list of expressions: `x, y, z`
@@ -169,7 +171,7 @@ comment "comment"
 
 computedPropertyAccess
   = __ "[" expression:expression expectClosingBracket {
-      return annotate([ops.traverse, expression], location());
+      return annotate([markers.traverse, expression], location());
     }
 
 conditionalExpression
@@ -362,7 +364,7 @@ identifierChar
 
 implicitParenthesesCallExpression "function call with implicit parentheses"
   = head:arrowFunction args:(inlineSpace+ @implicitParensthesesArguments)? {
-      return args ? makeCall(head, args) : head;
+      return args ? makeCall(head, args, options.mode) : head;
     }
     
 // A separated list of values for an implicit parens call. This differs from
@@ -384,7 +386,9 @@ jseMode
   = &{ return options.mode === "jse" }
 
 jsIdentifier
-  = $( jsIdentifierStart jsIdentifierPart* )
+  = id:$( jsIdentifierStart jsIdentifierPart* ) {
+    return annotate([ops.literal, id], location());
+  }
 
 // Continuation of a JavaScript identifier
 // https://tc39.es/ecma262/multipage/ecmascript-language-lexical-grammar.html#prod-IdentifierPart
@@ -398,13 +402,12 @@ jsIdentifierStart "JavaScript identifier start"
 
 jsPropertyAccess
   = __ "." __ property:jsIdentifier {
-    const literal = annotate([ops.literal, property], location());
-    return annotate([ops.traverse, literal], location());
+    return annotate([markers.traverse, property], location());
   }
 
 jsReference "identifier reference"
   = id:jsIdentifier {
-      return annotate([ops.scope, id], location());
+      return annotate([markers.reference, id], location());
     }
 
 // A separated list of values
@@ -449,7 +452,7 @@ multiplicativeOperator
 // A namespace reference is a string of letters only, followed by a colon.
 namespace
   = chars:[A-Za-z]+ ":" {
-    return annotate([ops.builtin, chars.join("") + ":"], location());
+    return annotate([markers.global, chars.join("") + ":"], location());
   }
 
 // A new expression: `new Foo()`
@@ -458,6 +461,10 @@ newExpression
       const args = tail?.[0] !== undefined ? tail : [];
       return annotate([ops.construct, head, ...args], location());
     }
+  / "new:" head:jsReference tail:parenthesesArguments {
+      const args = tail?.[0] !== undefined ? tail : [];
+      return annotate([ops.construct, head, ...args], location());
+    }
 
 newLine
   = "\n"
@@ -522,9 +529,16 @@ objectProperty "object property"
 // A shorthand reference inside an object literal: `foo`
 objectShorthandProperty "object identifier"
   = key:objectPublicKey {
-      const inherited = annotate([ops.inherited, key], location());
-      return annotate([key, inherited], location());
+      const reference = annotate([markers.reference, key], location());
+      return annotate([key, reference], location());
     }
+  / jseMode path:angleBracketLiteral {
+    let lastKey = path.at(-1);
+    if (lastKey instanceof Array) {
+      lastKey = lastKey[1]; // get scope identifier or literal
+    }
+    return annotate([lastKey, path], location());
+  }
 
 objectPublicKey
   = identifier:identifier slash:"/"? {
@@ -537,11 +551,12 @@ objectPublicKey
 
 optionalChaining
   = __ "?." __ property:jsIdentifier {
-    const literal = annotate([ops.literal, property], location());
-    return annotate([ops.optionalTraverse, literal], location());
+    return annotate([ops.optionalTraverse, property], location());
   }
+  
 parameter
-  = identifier:identifier {
+  = jseMode @jsIdentifier
+  / shellMode identifier:identifier {
       return annotate([ops.literal, identifier], location());
     }
 
@@ -577,7 +592,7 @@ path "slash-separated path"
 // A slash-separated path of keys that follows a call target
 pathArguments
   = path:path {
-      return annotate([ops.traverse, ...path], location());
+      return annotate([markers.traverse, ...path], location());
     }
 
 // A single key in a slash-separated path: `/a`
@@ -603,7 +618,7 @@ pathSegmentChar
 pipelineExpression
   = head:shorthandFunction tail:(__ singleArrow __ @shorthandFunction)* {
       return annotate(
-        tail.reduce(makePipeline, downgradeReference(head)),
+        tail.reduce((arg, fn) => makePipeline(arg, fn, options.mode), downgradeReference(head)),
         location()
       );
     }
@@ -639,7 +654,7 @@ program "Origami program"
 protocolExpression
   = fn:namespace "//" host:(host / slash) path:path? {
       const keys = annotate([host, ...(path ?? [])], location());
-      return makeCall(fn, keys);
+      return makeCall(fn, keys, options.mode);
     }
   / newExpression
   / primary
@@ -647,8 +662,7 @@ protocolExpression
 // A namespace followed by a key: `foo:x`
 qualifiedReference
   = fn:namespace reference:scopeReference {
-      const literal = annotate([ops.literal, reference[1]], reference.location);
-      return makeCall(fn, [literal]);
+      return makeCall(fn, [reference[1]], options.mode);
     }
 
 regexFlags
@@ -667,7 +681,11 @@ regexLiteralChar
   / escapedChar
 
 relationalExpression
-  = head:shiftExpression tail:(__ @relationalOperator __ @shiftExpression)* {
+  // We disallow a newline before the relational operator to support a newline
+  // as a separator in an object literal that has an object shorthand property
+  // with an angle bracket path. Otherwise the opening angle bracket would be
+  // interpreted as a relational operator.
+  = head:shiftExpression tail:(inlineSpace @relationalOperator __ @shiftExpression)* {
       return tail.reduce(makeBinaryOperation, head);
     }
 
@@ -677,25 +695,22 @@ relationalOperator
   / ">="
   / ">"
 
-// A top-level folder below the root: `/foo`
-// or the root folder itself: `/`
+// The root folder: `/`
 rootDirectory
-  = "/" key:pathKey {
-      return annotate([ops.rootDirectory, key], location());
-    }
-  / "/" !"/" {
+  = &("/" !"/") {
       return annotate([ops.rootDirectory], location());
     }
 
 scopeReference "scope reference"
   = identifier:identifier slashFollows:slashFollows? {
       const id = identifier + (slashFollows ? "/" : "");
-      return annotate(makeReference(id), location());
+      const idCode = annotate([ops.literal, identifier], location());
+      return annotate([markers.reference, idCode], location());
     }
 
 separator
   = __ "," __
-  / shellMode @whitespaceWithNewLine
+  / @whitespaceWithNewLine
 
 shebang
   = "#!" [^\n\r]* { return null; }
@@ -754,7 +769,7 @@ slashFollows
     }
 
 spreadElement
-  = ellipsis __ value:pipelineExpression {
+  = ellipsis __ value:expectPipelineExpression {
       return annotate([ops.spread, value], location());
     }
 
@@ -767,14 +782,7 @@ stringLiteral "string"
 // contain backticks at the top level.
 templateBody "template"
   = head:templateBodyText tail:(templateSubstitution templateBodyText)* {
-      const lambdaParameters = annotate(
-        [annotate([ops.literal, "_"], location())],
-        location()
-      );
-      return annotate(
-        [ops.lambda, lambdaParameters, makeTemplate(ops.templateIndent, head, tail, location())],
-        location()
-      );
+      return makeTemplate(ops.templateIndent, head, tail, location());
     }
 
 // Template document bodies can contain backticks at the top level
@@ -791,10 +799,15 @@ templateDocument "template document"
       const macroName = options.mode === "jse" ? "_template" : "@template";
       return annotate(applyMacro(front, macroName, body), location());
     }
-  / front:frontMatterYaml? body:templateBody {
-      return front
-        ? annotate([ops.document, front, body], location())
-        : annotate(body, location());
+  / front:frontMatterYaml body:templateBody {
+      return makeDocument(options.mode, front, body, location());
+    }
+  / body:templateBody {
+      const lambdaParameters = annotate(
+        [annotate([ops.literal, "_"], location())],
+        location()
+      );
+      return annotate([ops.lambda, lambdaParameters, body], location());
     }
 
 // A backtick-quoted template literal
@@ -836,14 +849,15 @@ unaryOperator
   // Don't match a front matter delimiter. For some reason, the negative
   // lookahead !"--\n" doesn't work.
   / @"-" !"-\n"
-  / "~"
+  // Don't match a path that starts with a tilde: ~/foo
+  / @"~" !"/"
 
 whitespace
   = inlineSpace
   / newLine
   / comment
 
-// Whitespace requires in shell mode, optional in JSE mode
+// Whitespace required in shell mode, optional in JSE mode
 whitespaceShell
   = shellMode whitespace
   / jseMode __