@weborigami/language 0.2.10 → 0.2.12

package/src/compiler/parserHelpers.js

@@ -1,7 +1,13 @@
 import { trailingSlash } from "@weborigami/async-tree";
+import * as YAMLModule from "yaml";
 import codeFragment from "../runtime/codeFragment.js";
 import * as ops from "../runtime/ops.js";
 
+// The "yaml" package doesn't seem to provide a default export that the browser can
+// recognize, so we have to handle two ways to accommodate Node and the browser.
+// @ts-ignore
+const YAML = YAMLModule.default ?? YAMLModule.YAML;
+
 // Parser helpers
 
 /** @typedef {import("../../index.ts").AnnotatedCode} AnnotatedCode */
@@ -30,6 +36,28 @@ export function annotate(code, location) {
   return annotated;
 }
 
+/**
+ * In the given code, replace all scope refernces to the given name with the
+ * given macro code.
+ *
+ * @param {AnnotatedCode} code
+ * @param {string} name
+ * @param {AnnotatedCode} macro
+ */
+export function applyMacro(code, name, macro) {
+  if (!(code instanceof Array)) {
+    return code;
+  }
+
+  const [fn, ...args] = code;
+  if (fn === ops.scope && args[0] === name) {
+    return macro;
+  }
+
+  const applied = code.map((child) => applyMacro(child, name, macro));
+  return annotate(applied, code.location);
+}
+
 /**
  * The indicated code is being used to define a property named by the given key.
  * Rewrite any [ops.scope, key] calls to be [ops.inherited, key] to avoid
@@ -198,7 +226,13 @@ export function makeCall(target, args) {
     }
   } else if (args[0] === ops.template) {
     // Tagged template
-    fnCall = [upgradeReference(target), ...args.slice(1)];
+    const strings = args[1];
+    const values = args.slice(2);
+    fnCall = makeTaggedTemplateCall(
+      upgradeReference(target),
+      strings,
+      ...values
+    );
   } else {
     // Function call with explicit or implicit parentheses
     fnCall = [upgradeReference(target), ...args];
@@ -345,6 +379,24 @@ export function makeReference(identifier) {
   return [op, identifier];
 }
 
+/**
+ * Make a tagged template call
+ *
+ * Because the tagged template function may not be an Origami function, we wrap
+ * each argument in a ops.concat call to convert it to a string.
+ *
+ * @param {AnnotatedCode} fn
+ * @param {AnnotatedCode} strings
+ * @param {AnnotatedCode[]} values
+ */
+function makeTaggedTemplateCall(fn, strings, ...values) {
+  const args = values.map((value) =>
+    // @ts-ignore
+    annotate([ops.concat, value], value.location)
+  );
+  return annotate([fn, strings, ...args], strings.location);
+}
+
 /**
  * Make a template
  *
@@ -357,8 +409,7 @@ export function makeTemplate(op, head, tail, location) {
   const strings = [head[1]];
   const values = [];
   for (const [value, literal] of tail) {
-    const concat = annotate([ops.concat, value], value.location);
-    values.push(concat);
+    values.push(value);
     strings.push(literal[1]);
   }
   const stringsCode = annotate(strings, location);
@@ -385,6 +436,59 @@ export function makeUnaryOperation(operator, value, location) {
   return annotate([operators[operator], value], location);
 }
 
+/**
+ * Make an object from YAML front matter
+ *
+ * @param {string} text
+ * @param {CodeLocation} location
+ */
+export function makeYamlObject(text, location) {
+  // Account for the "---" delimiter at the beginning of the YAML front matter
+  const yamlLineDelta = 1;
+  const yamlOffsetDelta = 4; // 3 dashes + 1 newline
+
+  let parsed;
+  try {
+    parsed = YAML.parse(text);
+  } catch (/** @type {any} */ yamlError) {
+    // Convert YAML error to a SyntaxError
+
+    let { message } = yamlError;
+    // Remove the line number and column if present
+    const lineNumberRegex = /( at line )(\d+)(,)/;
+    const lineNumberMatch = message.match(lineNumberRegex);
+    if (lineNumberMatch) {
+      message = message.slice(0, lineNumberMatch.index);
+    }
+
+    /** @type {any} */
+    const error = new SyntaxError(message);
+    error.location = {
+      end: {
+        column: yamlError.linePos[1].col,
+        line: yamlError.linePos[1].line + yamlLineDelta,
+        offset: yamlError.pos[1] + yamlOffsetDelta,
+      },
+      source: location.source,
+      start: {
+        column: yamlError.linePos[0].col,
+        line: yamlError.linePos[0].line + yamlLineDelta,
+        offset: yamlError.pos[0] + yamlOffsetDelta,
+      },
+    };
+    throw error;
+  }
+
+  if (!(parsed instanceof Object)) {
+    /** @type {any} */
+    const error = new SyntaxError("YAML front matter must be an object.");
+    error.location = location;
+    throw error;
+  }
+
+  return annotate([ops.literal, parsed], location);
+}
+
 /**
  * Upgrade a potential builtin reference to an actual builtin reference.
  *

package/src/runtime/errors.js

@@ -5,6 +5,8 @@ import {
   trailingSlash,
   TraverseError,
 } from "@weborigami/async-tree";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
 import codeFragment from "./codeFragment.js";
 import { typos } from "./typos.js";
 
@@ -85,9 +87,24 @@ export function formatError(error) {
     if (!fragmentInMessage) {
       message += `\nevaluating: ${fragment}`;
     }
+
     if (typeof source === "object" && source.url) {
-      message += `\n    at ${source.url.href}:${line}:${start.column}`;
+      const { url } = source;
+      let fileRef;
+      // If URL is a file: URL, change to a relative path
+      if (url.protocol === "file:") {
+        fileRef = fileURLToPath(url);
+        const relativePath = path.relative(process.cwd(), fileRef);
+        if (!relativePath.startsWith("..")) {
+          fileRef = relativePath;
+        }
+      } else {
+        // Not a file: URL, use as is
+        fileRef = url.href;
+      }
+      message += `\n    at ${fileRef}:${line}:${start.column}`;
     } else if (source.text.includes("\n")) {
+      // Don't know the URL, but has multiple lines so add line number
       message += `\n    at line ${line}, column ${start.column}`;
     }
   }
@@ -113,7 +130,7 @@ export function maybeOrigamiSourceCode(text) {
 
 export async function scopeReferenceError(scope, key) {
   const messages = [
-    `"${key}" is not in scope.`,
+    `"${key}" is not in scope or is undefined.`,
     await formatScopeTypos(scope, key),
   ];
   const message = messages.join(" ");

package/src/runtime/evaluate.js

@@ -1,6 +1,5 @@
 import { Tree, isUnpackable, scope } from "@weborigami/async-tree";
 import codeFragment from "./codeFragment.js";
-import { ops } from "./internal.js";
 import { codeSymbol, scopeSymbol, sourceSymbol } from "./symbols.js";
 
 /**
@@ -20,14 +19,7 @@ export default async function evaluate(code) {
   }
 
   let evaluated;
-  const unevaluatedFns = [
-    ops.external,
-    ops.lambda,
-    ops.merge,
-    ops.object,
-    ops.literal,
-  ];
-  if (unevaluatedFns.includes(code[0])) {
+  if (code[0]?.unevaluatedArgs) {
     // Don't evaluate instructions, use as is.
     evaluated = code;
   } else {

package/src/runtime/ops.js

@@ -8,8 +8,10 @@
 import {
   ObjectTree,
   Tree,
+  concatTrees,
   isUnpackable,
   scope as scopeFn,
+  symbols,
   concat as treeConcat,
 } from "@weborigami/async-tree";
 import os from "node:os";
@@ -20,7 +22,6 @@ import { evaluate } from "./internal.js";
 import mergeTrees from "./mergeTrees.js";
 import OrigamiFiles from "./OrigamiFiles.js";
 import { codeSymbol } from "./symbols.js";
-import taggedTemplate from "./taggedTemplate.js";
 
 function addOpLabel(op, label) {
   Object.defineProperty(op, "toString", {
@@ -113,6 +114,29 @@ export async function conditional(condition, truthy, falsy) {
   return value instanceof Function ? await value() : value;
 }
 
+/**
+ * Construct a document object by invoking the body code (a lambda) and adding
+ * the resulting text to the front data.
+ *
+ * @this {AsyncTree|null}
+ * @param {any} frontData
+ * @param {AnnotatedCode} bodyCode
+ */
+export async function document(frontData, bodyCode) {
+  const context = new ObjectTree(frontData);
+  context.parent = this;
+  const bodyFn = await evaluate.call(context, bodyCode);
+  const body = await bodyFn();
+  const object = {
+    ...frontData,
+    "@text": body,
+  };
+  object[symbols.parent] = this;
+  return object;
+}
+addOpLabel(document, "«ops.document");
+document.unevaluatedArgs = true;
+
 export function division(a, b) {
   return a / b;
 }
@@ -158,6 +182,7 @@ export async function external(path, code, cache) {
   return value;
 }
 addOpLabel(external, "«ops.external»");
+external.unevaluatedArgs = true;
 
 /**
  * This op is only used during parsing. It signals to ops.object that the
@@ -263,6 +288,7 @@ export function lambda(parameters, code) {
   return invoke;
 }
 addOpLabel(lambda, "«ops.lambda");
+lambda.unevaluatedArgs = true;
 
 export function lessThan(a, b) {
   return a < b;
@@ -284,6 +310,7 @@ export async function literal(value) {
   return value;
 }
 addOpLabel(literal, "«ops.literal»");
+literal.unevaluatedArgs = true;
 
 /**
  * Logical AND operator
@@ -377,6 +404,7 @@ export async function merge(...codes) {
   return mergeTrees.call(this, ...trees);
 }
 addOpLabel(merge, "«ops.merge»");
+merge.unevaluatedArgs = true;
 
 export function multiplication(a, b) {
   return a * b;
@@ -425,6 +453,7 @@ export async function object(...entries) {
   return expressionObject(entries, this);
 }
 addOpLabel(object, "«ops.object»");
+object.unevaluatedArgs = true;
 
 export function remainder(a, b) {
   return a % b;
@@ -503,15 +532,15 @@ addOpLabel(subtraction, "«ops.subtraction»");
 /**
  * Apply the default tagged template function.
  */
-export function template(strings, ...values) {
-  return taggedTemplate(strings, ...values);
+export async function template(strings, ...values) {
+  return concatTrees(strings, ...values);
 }
 addOpLabel(template, "«ops.template»");
 
 /**
  * Apply the tagged template indent function.
  */
-export function templateIndent(strings, ...values) {
+export async function templateIndent(strings, ...values) {
   return taggedTemplateIndent(strings, ...values);
 }
 addOpLabel(templateIndent, "«ops.templateIndent");

package/src/runtime/taggedTemplateIndent.js

@@ -1,22 +1,27 @@
+import { concat, toString, Tree } from "@weborigami/async-tree";
+
 const lastLineWhitespaceRegex = /\n(?<indent>[ \t]*)$/;
 
 const mapStringsToModifications = new Map();
 
 /**
- * Normalize indentation in a tagged template string.
+ * Normalize indentation in a tagged template string
  *
  * @param {TemplateStringsArray} strings
  * @param  {...any} values
- * @returns {string}
+ * @returns {Promise<string>}
  */
-export default function indent(strings, ...values) {
+export default async function indent(strings, ...values) {
   let modified = mapStringsToModifications.get(strings);
   if (!modified) {
     modified = modifyStrings(strings);
     mapStringsToModifications.set(strings, modified);
   }
   const { blockIndentations, strings: modifiedStrings } = modified;
-  return joinBlocks(modifiedStrings, values, blockIndentations);
+  const valueTexts = await Promise.all(
+    values.map((value) => (Tree.isTreelike(value) ? concat(value) : value))
+  );
+  return joinBlocks(modifiedStrings, valueTexts, blockIndentations);
 }
 
 // Join strings and values, applying the given block indentation to the lines of
@@ -24,7 +29,7 @@ export default function indent(strings, ...values) {
 function joinBlocks(strings, values, blockIndentations) {
   let result = strings[0];
   for (let i = 0; i < values.length; i++) {
-    let text = values[i];
+    let text = toString(values[i]);
     if (text) {
       const blockIndentation = blockIndentations[i];
       if (blockIndentation) {

package/test/compiler/codeHelpers.js

@@ -4,7 +4,7 @@ import assert from "node:assert";
 export function assertCodeEqual(actual, expected) {
   const actualStripped = stripCodeLocations(actual);
   const expectedStripped = stripCodeLocations(expected);
-  assert.deepEqual(actualStripped, expectedStripped);
+  assert.deepStrictEqual(actualStripped, expectedStripped);
 }
 
 /**

package/test/compiler/parse.test.js

@@ -251,20 +251,20 @@ describe("Origami parser", () => {
     assertParse("conditionalExpression", "true ? 1 : 0", [
       ops.conditional,
       [ops.scope, "true"],
-      [ops.literal, "1"],
-      [ops.literal, "0"],
+      [ops.literal, 1],
+      [ops.literal, 0],
     ]);
     assertParse("conditionalExpression", "false ? () => 1 : 0", [
       ops.conditional,
       [ops.scope, "false"],
-      [ops.lambda, [], [ops.lambda, [], [ops.literal, "1"]]],
-      [ops.literal, "0"],
+      [ops.lambda, [], [ops.lambda, [], [ops.literal, 1]]],
+      [ops.literal, 0],
     ]);
     assertParse("conditionalExpression", "false ? =1 : 0", [
       ops.conditional,
       [ops.scope, "false"],
-      [ops.lambda, [], [ops.lambda, [[ops.literal, "_"]], [ops.literal, "1"]]],
-      [ops.literal, "0"],
+      [ops.lambda, [], [ops.lambda, [[ops.literal, "_"]], [ops.literal, 1]]],
+      [ops.literal, 0],
     ]);
   });
 
@@ -286,7 +286,7 @@ describe("Origami parser", () => {
     ]);
   });
 
-  test("errors for missing pieces", () => {
+  test("error thrown for missing token", () => {
     assertThrows("arrowFunction", "(a) => ", "Expected an expression");
     assertThrows("arrowFunction", "a ⇒ ", "Expected an expression");
     assertThrows("callExpression", "fn(a", "Expected right parenthesis");
@@ -298,6 +298,31 @@ describe("Origami parser", () => {
     assertThrows("templateLiteral", "`foo", "Expected closing backtick");
   });
 
+  test("error thrown for invalid Origami front matter expression", () => {
+    assertThrows(
+      "templateDocument",
+      `---
+(name) => foo)
+---
+Body`,
+      'Expected "---"',
+      { line: 2, column: 14 }
+    );
+  });
+
+  test("error thrown for invalid YAML front matter", () => {
+    assertThrows(
+      "templateDocument",
+      `---
+a : 1
+}
+---
+Body`,
+      "Unexpected flow-map-end token",
+      { line: 3, column: 1 }
+    );
+  });
+
   test("exponentiationExpression", () => {
     assertParse("exponentiationExpression", "2 ** 2 ** 3", [
       ops.exponentiation,
@@ -309,12 +334,10 @@ describe("Origami parser", () => {
   test("expression", () => {
     assertParse(
       "expression",
-      `
-        {
-          index.html = index.ori(teamData.yaml)
-          thumbnails = map(images, { value: thumbnail.js })
-        }
-      `,
+      `{
+        index.html = index.ori(teamData.yaml)
+        thumbnails = map(images, { value: thumbnail.js })
+      }`,
       [
         ops.object,
         [
@@ -423,11 +446,7 @@ describe("Origami parser", () => {
       [
         ops.lambda,
         [[ops.literal, "_"]],
-        [
-          ops.template,
-          [ops.literal, ["<li>", "</li>"]],
-          [ops.concat, [ops.scope, "_"]],
-        ],
+        [ops.template, [ops.literal, ["<li>", "</li>"]], [ops.scope, "_"]],
       ],
     ]);
     assertParse("expression", `https://example.com/about/`, [
@@ -968,11 +987,7 @@ describe("Origami parser", () => {
     assertParse("shorthandFunction", "=`Hello, ${name}.`", [
       ops.lambda,
       [[ops.literal, "_"]],
-      [
-        ops.template,
-        [ops.literal, ["Hello, ", "."]],
-        [ops.concat, [ops.scope, "name"]],
-      ],
+      [ops.template, [ops.literal, ["Hello, ", "."]], [ops.scope, "name"]],
     ]);
     assertParse("shorthandFunction", "=indent`hello`", [
       ops.lambda,
@@ -1007,17 +1022,17 @@ describe("Origami parser", () => {
     ]);
   });
 
-  test("templateDocument", () => {
-    assertParse("templateDocument", "hello${foo}world", [
+  test("templateBody", () => {
+    assertParse("templateBody", "hello${foo}world", [
       ops.lambda,
       [[ops.literal, "_"]],
       [
         ops.templateIndent,
         [ops.literal, ["hello", "world"]],
-        [ops.concat, [ops.scope, "foo"]],
+        [ops.scope, "foo"],
       ],
     ]);
-    assertParse("templateDocument", "Documents can contain ` backticks", [
+    assertParse("templateBody", "Documents can contain ` backticks", [
       ops.lambda,
       [[ops.literal, "_"]],
       [
@@ -1027,6 +1042,66 @@ describe("Origami parser", () => {
     ]);
   });
 
+  test("templateDocument with no front matter", () => {
+    assertParse("templateDocument", "Hello, world!", [
+      ops.lambda,
+      [[ops.literal, "_"]],
+      [ops.templateIndent, [ops.literal, ["Hello, world!"]]],
+    ]);
+  });
+
+  test("templateDocument with YAML front matter", () => {
+    assertParse(
+      "templateDocument",
+      `---
+title: Title goes here
+---
+Body text`,
+      [
+        ops.document,
+        [ops.literal, { title: "Title goes here" }],
+        [
+          ops.lambda,
+          [[ops.literal, "_"]],
+          [ops.templateIndent, [ops.literal, ["Body text"]]],
+        ],
+      ]
+    );
+  });
+
+  test("templateDocument with Origami front matter", () => {
+    assertParse(
+      "templateDocument",
+      `---
+{
+  title: "Title"
+  @text: @template()
+}
+---
+<h1>\${ title }</h1>
+`,
+      [
+        ops.object,
+        ["title", [ops.literal, "Title"]],
+        [
+          "@text",
+          [
+            [
+              ops.lambda,
+              [[ops.literal, "_"]],
+              [
+                ops.templateIndent,
+                [ops.literal, ["<h1>", "</h1>\n"]],
+                [ops.scope, "title"],
+              ],
+            ],
+            undefined,
+          ],
+        ],
+      ]
+    );
+  });
+
   test("templateLiteral", () => {
     assertParse("templateLiteral", "`Hello, world.`", [
       ops.template,
@@ -1035,30 +1110,23 @@ describe("Origami parser", () => {
     assertParse("templateLiteral", "`foo ${x} bar`", [
       ops.template,
       [ops.literal, ["foo ", " bar"]],
-      [ops.concat, [ops.scope, "x"]],
+      [ops.scope, "x"],
     ]);
     assertParse("templateLiteral", "`${`nested`}`", [
       ops.template,
       [ops.literal, ["", ""]],
-      [ops.concat, [ops.template, [ops.literal, ["nested"]]]],
+      [ops.template, [ops.literal, ["nested"]]],
     ]);
     assertParse("templateLiteral", "`${ map:(people, =`${name}`) }`", [
       ops.template,
       [ops.literal, ["", ""]],
       [
-        ops.concat,
+        [ops.builtin, "map:"],
+        [ops.scope, "people"],
         [
-          [ops.builtin, "map:"],
-          [ops.scope, "people"],
-          [
-            ops.lambda,
-            [[ops.literal, "_"]],
-            [
-              ops.template,
-              [ops.literal, ["", ""]],
-              [ops.concat, [ops.scope, "name"]],
-            ],
-          ],
+          ops.lambda,
+          [[ops.literal, "_"]],
+          [ops.template, [ops.literal, ["", ""]], [ops.scope, "name"]],
         ],
       ],
     ]);
@@ -1106,7 +1174,7 @@ function assertParse(startRule, source, expected, checkLocation = true) {
       code.location.start.offset,
       code.location.end.offset
     );
-    assert.equal(resultSource, source.trim());
+    assert.equal(resultSource, source);
   }
 
   assertCodeEqual(code, expected);
@@ -1121,7 +1189,7 @@ function assertCodeLocations(code) {
   }
 }
 
-function assertThrows(startRule, source, message) {
+function assertThrows(startRule, source, message, position) {
   try {
     parse(source, {
       grammarSource: { text: source },
@@ -1132,6 +1200,10 @@ function assertThrows(startRule, source, message) {
       error.message.includes(message),
       `Error message incorrect:\n  expected: "${message}"\n  actual: "${error.message}"`
     );
+    if (position) {
+      assert.equal(error.location.start.line, position.line);
+      assert.equal(error.location.start.column, position.column);
+    }
     return;
   }
   assert.fail(`Expected error: ${message}`);

package/test/runtime/expressionObject.test.js

@@ -5,7 +5,7 @@ import { describe, test } from "node:test";
 import expressionObject from "../../src/runtime/expressionObject.js";
 import { ops } from "../../src/runtime/internal.js";
 
-describe.only("expressionObject", () => {
+describe("expressionObject", () => {
   test("can instantiate an object", async () => {
     const scope = new ObjectTree({
       upper: (s) => s.toUpperCase(),
@@ -74,7 +74,7 @@ describe.only("expressionObject", () => {
     assert.equal(object["hidden"], "shh");
   });
 
-  test.only("provides a symbols.keys method", async () => {
+  test("provides a symbols.keys method", async () => {
     const entries = [
       // Will return a tree, should have a slash
       ["getter", [ops.getter, [ops.object, ["b", [ops.literal, 2]]]]],

package/test/runtime/ops.test.js

@@ -80,6 +80,29 @@ describe("ops", () => {
     assert.strictEqual(await ops.conditional(false, errorFn, trueFn), true);
   });
 
+  test("ops.documentFunction", async () => {
+    const code = createCode([
+      ops.document,
+      {
+        a: 1,
+      },
+      [
+        ops.lambda,
+        [["_"]],
+        [
+          ops.template,
+          [ops.literal, ["a = ", ""]],
+          [ops.concat, [ops.scope, "a"]],
+        ],
+      ],
+    ]);
+    const result = await evaluate.call(null, code);
+    assert.deepEqual(result, {
+      a: 1,
+      "@text": "a = 1",
+    });
+  });
+
   test("ops.division divides two numbers", async () => {
     assert.strictEqual(ops.division(12, 2), 6);
     assert.strictEqual(ops.division(3, 2), 1.5);