@weborigami/language 0.6.17 → 0.7.0-beta.2

package/index.ts

@@ -90,6 +90,7 @@ export type RuntimeState = {
  * Source code representation used by the parser.
  */
 export type Source = {
+  cachePath?: string;
   name?: string;
   text: string;
   url?: URL;

package/main.js

@@ -5,12 +5,15 @@ export { default as isOrigamiFrontMatter } from "./src/compiler/isOrigamiFrontMa
 export { parse } from "./src/compiler/parse.js";
 export * from "./src/compiler/parserHelpers.js";
 export * as Handlers from "./src/handlers/handlers.js";
+export * as activeProjectRoot from "./src/project/activeProjectRoot.js";
 export { default as coreGlobals } from "./src/project/coreGlobals.js";
+export { default as getGlobalsForTree } from "./src/project/getGlobalsForTree.js";
+export { default as initializeGlobalsForTree } from "./src/project/initializeGlobalsForTree.js";
 export { default as jsGlobals } from "./src/project/jsGlobals.js";
 export { default as projectConfig } from "./src/project/projectConfig.js";
-export { default as projectGlobals } from "./src/project/projectGlobals.js";
 export { default as projectRoot } from "./src/project/projectRoot.js";
 export { default as projectRootFromPath } from "./src/project/projectRootFromPath.js";
+export { default as fetchAndHandleExtension } from "./src/protocols/fetchAndHandleExtension.js";
 export * as Protocols from "./src/protocols/protocols.js";
 export { formatError, highlightError, lineInfo } from "./src/runtime/errors.js";
 export { default as evaluate } from "./src/runtime/evaluate.js";
@@ -25,5 +28,8 @@ export { default as ImportModulesMixin } from "./src/runtime/ImportModulesMixin.
 export * as moduleCache from "./src/runtime/moduleCache.js";
 export { default as OrigamiFileMap } from "./src/runtime/OrigamiFileMap.js";
 export * as symbols from "./src/runtime/symbols.js";
+export { default as systemCache } from "./src/runtime/systemCache.js";
+export { default as SystemCacheMap } from "./src/runtime/SystemCacheMap.js";
 export { default as TreeEvent } from "./src/runtime/TreeEvent.js";
+export { default as volatile } from "./src/runtime/volatile.js";
 export { default as WatchFilesMixin } from "./src/runtime/WatchFilesMixin.js";

package/package.json

@@ -1,20 +1,21 @@
 {
   "name": "@weborigami/language",
-  "version": "0.6.17",
+  "version": "0.7.0-beta.2",
   "description": "Web Origami expression language compiler and runtime",
   "type": "module",
   "main": "./main.js",
   "types": "./index.ts",
   "devDependencies": {
-    "@types/node": "25.3.2",
-    "peggy": "5.0.6",
-    "typescript": "5.9.3"
+    "@types/node": "25.9.1",
+    "peggy": "5.1.0",
+    "typescript": "6.0.3"
   },
   "dependencies": {
-    "@weborigami/async-tree": "0.6.17",
+    "@weborigami/async-tree": "0.7.0-beta.2",
+    "adm-zip": "0.5.17",
     "exif-parser": "0.1.12",
     "watcher": "2.3.1",
-    "yaml": "2.8.2"
+    "yaml": "2.9.0"
   },
   "scripts": {
     "build": "peggy --allowed-start-rules=\"*\" --format es src/compiler/origami.pegjs --output src/compiler/parse.js",

package/src/compiler/compile.js

@@ -1,7 +1,11 @@
+import { trailingSlash } from "@weborigami/async-tree";
 import { createExpressionFunction } from "../runtime/expressionFunction.js";
 import optimize from "./optimize.js";
 import { parse } from "./parse.js";
 
+// For caching while evaluating expressions with no identified source file
+let count = 0;
+
 /**
  * Compile the given Origami source code into a JavaScript function.
  *
@@ -27,12 +31,15 @@ function compile(source, options) {
     startRule,
   });
 
+  // Select a path the code will use for caching
+  const cachePath = source.cachePath ?? `_compile${count++}`;
+  const objectCachePath = cachePath ? trailingSlash.add(cachePath) : null;
+
   // Optimize the code
-  const cache = mode === "program" ? {} : null;
   const optimized = optimize(code, {
-    cache,
+    cachePath,
     globals,
-    parent,
+    objectCachePath,
   });
 
   // Create a function that executes the optimized code.

package/src/compiler/optimize.js

@@ -6,6 +6,7 @@ import {
   propertyInfo,
 } from "../runtime/expressionObject.js";
 import { ops } from "../runtime/internal.js";
+import SystemCacheMap from "../runtime/SystemCacheMap.js";
 import { annotate, markers, spanLocations } from "./parserHelpers.js";
 
 export const REFERENCE_PARAM = 1;
@@ -30,9 +31,9 @@ export const REFERENCE_EXTERNAL = 4;
  * @returns {AnnotatedCode}
  */
 export default function optimize(code, options = {}) {
+  // Cache path for this source file
+  const cachePath = options.cachePath ?? null;
   const globals = options.globals ?? jsGlobals;
-  const cache = options.cache === undefined ? {} : options.cache;
-  const parent = options.parent ?? null;
 
   // The locals is an array, one item for each function or object context that
   // has been entered. The array grows to the right. Array items are objects
@@ -48,7 +49,7 @@ export default function optimize(code, options = {}) {
       return globals[args[0]];
 
     case markers.traverse:
-      return resolvePath(code, globals, parent, locals, cache);
+      return resolvePath(code, globals, locals, cachePath);
 
     case ops.lambda:
       const parameters = args[1];
@@ -65,8 +66,7 @@ export default function optimize(code, options = {}) {
       return inlineLiteral(code);
 
     case ops.object:
-      const entries = args;
-      const propertyNames = getPropertyNames(entries);
+      const propertyNames = getPropertyNames(args);
       locals.push({
         type: REFERENCE_INHERITED,
         names: propertyNames,
@@ -75,37 +75,48 @@ export default function optimize(code, options = {}) {
   }
 
   // Optimize children
-  const optimized = annotate(
+  let optimized = annotate(
     code.map((child, index) => {
-      if (op === ops.object && index > 0) {
-        const [key, value] = child;
-        const adjustedLocals = avoidLocalRecursion(locals, key);
-        const optimizedKey =
-          typeof key === "string"
+      if (op === ops.object) {
+        if (index === 0) {
+          return child; // return op as is
+        } else {
+          // Object entry
+          const [key, value] = child;
+          const adjustedLocals = avoidLocalRecursion(locals, key);
+          const isStringKey = typeof key === "string";
+          const childOptions = {
+            ...options,
+            locals: adjustedLocals,
+          };
+          const optimizedKey = isStringKey
             ? key
-            : optimize(/** @type {AnnotatedCode} */ (key), {
-                ...options,
-                locals: adjustedLocals,
-              });
-        const optimizedValue = optimize(/** @type {AnnotatedCode} */ (value), {
-          ...options,
-          locals: adjustedLocals,
-        });
-        return annotate([optimizedKey, optimizedValue], child.location);
+            : optimize(/** @type {AnnotatedCode} */ (key), childOptions);
+          const optimizedValue = optimize(
+            /** @type {AnnotatedCode} */ (value),
+            childOptions,
+          );
+          return annotate([optimizedKey, optimizedValue], child.location);
+        }
       } else if (Array.isArray(child) && "location" in child) {
         // Review: Aside from ops.object (above), what non-instruction arrays
         // does this descend into?
-        return optimize(/** @type {AnnotatedCode} */ (child), {
-          ...options,
-          locals,
-        });
+        const childOptions = { ...options, locals };
+        // Objects below here are detached from top-level object; not cached
+        delete childOptions.objectCachePath;
+        return optimize(/** @type {AnnotatedCode} */ (child), childOptions);
       } else {
         return child;
       }
     }),
-    code.location
+    code.location,
   );
 
+  if (optimized[0] instanceof Array && optimized[0][0] === ops.cache) {
+    // An external reference in function position: add implied unpack
+    optimized = impliedUnpack(optimized);
+  }
+
   return annotate(optimized, code.location);
 }
 
@@ -129,7 +140,7 @@ function avoidLocalRecursion(locals, key) {
   const matchingKeyIndex = locals[currentFrameIndex].names.findIndex(
     (name) =>
       // Ignore trailing slashes when comparing keys
-      trailingSlash.remove(name) === trailingSlash.remove(key)
+      trailingSlash.remove(name) === trailingSlash.remove(key),
   );
 
   if (matchingKeyIndex >= 0) {
@@ -147,10 +158,11 @@ function avoidLocalRecursion(locals, key) {
   }
 }
 
-function cachePath(code, cache) {
+function cacheExternalPath(code, cachePath) {
   const keys = code.map(keyFromCode).filter((key) => key !== null);
-  const path = pathFromKeys(keys);
-  return annotate([ops.cache, cache, path, code], code.location);
+  const keysPath = pathFromKeys(keys);
+  const refCachePath = SystemCacheMap.joinPath(cachePath, "_refs", keysPath);
+  return annotate([ops.cache, refCachePath, code], code.location);
 }
 
 // A reference with periods like x.y.z
@@ -180,8 +192,8 @@ function compoundReference(key, globals, locals, location) {
   return { type: headReference.type, result };
 }
 
-function externalReference(key, parent, location) {
-  const scope = annotate([ops.scope, parent], location);
+function externalReference(key, location) {
+  const scope = annotate([ops.scope], location);
   const literal = annotate([ops.literal, key], location);
   return annotate([scope, literal], location);
 }
@@ -193,7 +205,7 @@ function findLocalDetails(key, locals) {
   for (let i = locals.length - 1; i >= 0; i--) {
     const { type, names } = locals[i];
     const local = names.find(
-      (name) => trailingSlash.remove(name) === normalized
+      (name) => trailingSlash.remove(name) === normalized,
     );
     if (local) {
       const depth = type === REFERENCE_PARAM ? paramDepth : inheritedDepth;
@@ -221,6 +233,25 @@ function globalReference(key, globals) {
   return globals[normalized];
 }
 
+// The code starts with a cached external reference in function position. If the
+// cache path doesn't end in a trailing slash, wrap the external reference in an
+// unpack operation.
+function impliedUnpack(code) {
+  const [opCache, cachePath, refCode] = code[0];
+  if (cachePath.endsWith("/")) {
+    // Will already be unpacked
+    return code;
+  }
+  const modifiedPath = cachePath + "/";
+  const modifiedRefCode = annotate([ops.unpack, refCode], code[0].location);
+  const unpacked = annotate(
+    [opCache, modifiedPath, modifiedRefCode],
+    code[0].location,
+  );
+  const modified = annotate([unpacked, ...code.slice(1)], code.location);
+  return modified;
+}
+
 function inheritedReference(key, depth, location) {
   const literal = annotate([ops.literal, key], location);
   const inherited = annotate([ops.inherited, depth], location);
@@ -283,7 +314,7 @@ function paramReference(key, depth, location) {
   return annotate([params, literal], location);
 }
 
-function reference(code, globals, parent, locals) {
+function reference(code, globals, locals) {
   const key = keyFromCode(code);
   const normalized = trailingSlash.remove(key);
   const location = code.location;
@@ -306,7 +337,7 @@ function reference(code, globals, parent, locals) {
     // Explicit external reference
     return {
       type: REFERENCE_EXTERNAL,
-      result: externalReference(key, parent, location),
+      result: externalReference(key, location),
     };
   }
 
@@ -325,15 +356,15 @@ function reference(code, globals, parent, locals) {
   // Must be external
   return {
     type: REFERENCE_EXTERNAL,
-    result: externalReference(key, parent, location),
+    result: externalReference(key, location),
   };
 }
 
-function resolvePath(code, globals, parent, locals, cache) {
+function resolvePath(code, globals, locals, cachePath) {
   const args = code.slice(1);
   const [head, ...tail] = args;
 
-  let { type, result } = reference(head, globals, parent, locals);
+  let { type, result } = reference(head, globals, locals);
 
   if (tail.length > 0) {
     // If the result is a traversal, we can safely extend it
@@ -352,9 +383,9 @@ function resolvePath(code, globals, parent, locals, cache) {
     }
   }
 
-  if (type === REFERENCE_EXTERNAL && cache !== null) {
-    // Cache external path
-    return cachePath(result, cache);
+  if (type === REFERENCE_EXTERNAL && cachePath) {
+    // External references should be cached
+    return cacheExternalPath(result, cachePath);
   }
 
   return result;

package/src/compiler/parse.js

@@ -1,4 +1,4 @@
-// @generated by Peggy 5.0.6.
+// @generated by Peggy 5.1.0.
 //
 // https://peggyjs.org/
 

package/src/compiler/parserHelpers.js

@@ -341,9 +341,10 @@ function makeMerge(spreads, location) {
 
   for (const spread of spreads) {
     if (spread[0] === ops.object) {
-      topEntries.push(...spread.slice(1));
+      const spreadEntries = spread.slice(1);
+      topEntries.push(...spreadEntries);
       // Also add an object to the result with indirect references
-      const indirectEntries = spread.slice(1).map((entry) => {
+      const indirectEntries = spreadEntries.map((entry) => {
         const [key] = entry;
         const parent = annotate([ops.inherited, 1], entry.location);
         const reference = annotate([parent, key], entry.location);
@@ -389,7 +390,8 @@ export function makeObject(entries, location) {
     if (key === markers.spread) {
       if (value[0] === ops.object) {
         // Spread of an object; fold into current object
-        currentEntries.push(...value.slice(1));
+        const spreadEntries = value.slice(1);
+        currentEntries.push(...spreadEntries);
       } else {
         // Spread of a tree; accumulate
         if (currentEntries.length > 0) {

package/src/handlers/addExtensionKeyFn.js

@@ -0,0 +1,18 @@
+import { extension, trailingSlash } from "@weborigami/async-tree";
+
+// Return a function that adds the given extension
+export default function addExtensionKeyFn(resultExtension) {
+  const keyFn = (sourceValue, sourceKey) => {
+    if (sourceKey === undefined) {
+      return undefined;
+    }
+    const normalizedKey = trailingSlash.remove(sourceKey);
+    const sourceExtension = extension.extname(normalizedKey);
+    const resultKey = sourceExtension
+      ? extension.replace(normalizedKey, sourceExtension, resultExtension)
+      : normalizedKey + resultExtension;
+    return resultKey;
+  };
+  keyFn.needsSourceValue = false;
+  return keyFn;
+}

package/src/handlers/epub_handler.js

@@ -0,0 +1,54 @@
+import { AsyncMap, isUnpackable, Tree } from "@weborigami/async-tree";
+import addExtensionKeyFn from "./addExtensionKeyFn.js";
+import zip_handler from "./zip_handler.js";
+
+/**
+ * Handler for EPUB files
+ */
+const epub_handler = {
+  mediaType: "application/epub+zip",
+
+  /**
+   * Package a tree of files as an EPUB file in Buffer form.
+   *
+   * This calls the pack() method for ZIP files, but ensures the `mimetype` file
+   * is the first file in the package -- a requirement for EPUB files.
+   *
+   * @param {import("@weborigami/async-tree").Maplike} maplike
+   */
+  async pack(maplike) {
+    if (isUnpackable(maplike)) {
+      maplike = await maplike.unpack();
+    }
+    const tree = Tree.from(maplike, { deep: true });
+    return zip_handler.pack(mimetypeFirst(tree));
+  },
+
+  async unpack(buffer, options) {
+    return zip_handler.unpack(buffer, options);
+  },
+};
+
+/** @type {any} */ (epub_handler.pack).key = addExtensionKeyFn(".epub");
+
+export default epub_handler;
+
+// A tree with its `mimetype` file first
+function mimetypeFirst(tree) {
+  return Object.assign(new AsyncMap(), {
+    async get(key) {
+      return tree.get(key);
+    },
+
+    async *keys() {
+      const keys = await Tree.keys(tree);
+      // Move `mimetype` (if present) to the front of the list.
+      const index = keys.indexOf("mimetype");
+      if (index >= 0) {
+        keys.splice(index, 1);
+        keys.unshift("mimetype");
+      }
+      yield* keys;
+    },
+  });
+}

package/src/handlers/getSource.js

@@ -1,4 +1,5 @@
 import { getParent, toString } from "@weborigami/async-tree";
+import SystemCacheMap from "../runtime/SystemCacheMap.js";
 
 /**
  * Given packed source text and a handler's options, return a source
@@ -9,6 +10,7 @@ export default function getSource(packed, options = {}) {
 
   // Try to determine a URL for error messages
   const sourceName = options.key;
+  let cachePath;
   let url;
   if (sourceName) {
     if (/** @type {any} */ (parent)?.url) {
@@ -24,6 +26,11 @@ export default function getSource(packed, options = {}) {
         parentHref += "/";
       }
       url = new URL(sourceName, parentHref);
+
+      const parentPath = /** @type {any} */ (parent).path;
+      cachePath = SystemCacheMap.joinPath(parentPath, sourceName);
+    } else {
+      cachePath = sourceName;
     }
   }
 
@@ -32,5 +39,9 @@ export default function getSource(packed, options = {}) {
     name: options.key,
     url,
   };
+  if (cachePath) {
+    source.cachePath = cachePath;
+  }
+
   return source;
 }

package/src/handlers/handlers.js

@@ -20,6 +20,7 @@ export { default as txt_handler } from "./txt_handler.js";
 
 export { default as css_handler } from "./css_handler.js";
 export { default as csv_handler } from "./csv_handler.js";
+export { default as epub_handler } from "./epub_handler.js";
 export { default as htm_handler } from "./htm_handler.js";
 export { default as html_handler } from "./html_handler.js";
 export { default as jpeg_handler } from "./jpeg_handler.js";
@@ -33,3 +34,4 @@ export { default as wasm_handler } from "./wasm_handler.js";
 export { default as xhtml_handler } from "./xhtml_handler.js";
 export { default as yaml_handler } from "./yaml_handler.js";
 export { default as yml_handler } from "./yml_handler.js";
+export { default as zip_handler } from "./zip_handler.js";

package/src/handlers/htm_handler.js

@@ -1,2 +1,2 @@
 // .htm is a synonynm for .html
-export { html_handler as default } from "./handlers.js";
+export { default } from "./html_handler.js";

package/src/handlers/js_handler.js

@@ -10,23 +10,32 @@ export default {
   /** @type {import("@weborigami/async-tree").UnpackFunction} */
   async unpack(packed, options = {}) {
     const { key, parent } = options;
-    if (!(parent && "import" in parent)) {
+    let importTarget = parent;
+    while (importTarget.source && !importTarget.import) {
+      importTarget = importTarget.source;
+    }
+
+    if (!importTarget) {
       throw new TypeError(
         "The parent tree must support importing modules to unpack JavaScript files.",
       );
     }
 
-    const object = await /** @type {any} */ (parent).import?.(key);
+    const object = await importTarget.import?.(key);
 
     let bound;
+    let bindTarget = parent;
+    while (bindTarget.result) {
+      bindTarget = bindTarget.result;
+    }
     if ("default" in object) {
       // Module with a default export; return that.
-      bound = bindToParent(object.default, parent);
+      bound = bindToParent(object.default, bindTarget);
     } else {
       // Module with multiple named exports.
       bound = {};
       for (const [name, value] of Object.entries(object)) {
-        bound[name] = bindToParent(value, parent);
+        bound[name] = bindToParent(value, bindTarget);
       }
     }
 

package/src/handlers/mediaTypeExtensions.json

@@ -0,0 +1,15 @@
+{
+  "application/json": ".json",
+  "application/wasm": ".wasm",
+  "application/xhtml+xml": ".xhtml",
+  "application/xml": ".xml",
+  "application/yaml": ".yaml",
+  "image/jpeg": ".jpg",
+  "image/jpg": ".jpg",
+  "text/css": ".css",
+  "text/csv": ".csv",
+  "text/html": ".html",
+  "text/plain": ".txt",
+  "text/tsv": ".tsv",
+  "text/xml": ".xml"
+}

package/src/handlers/ori_handler.js

@@ -1,7 +1,9 @@
-import { getParent, setParent } from "@weborigami/async-tree";
+import { getParent } from "@weborigami/async-tree";
 import * as compile from "../compiler/compile.js";
-import projectGlobals from "../project/projectGlobals.js";
+import coreGlobals from "../project/coreGlobals.js";
+import getGlobalsForTree from "../project/getGlobalsForTree.js";
 import getSource from "./getSource.js";
+import processOriExport from "./processOriExport.js";
 
 /**
  * An Origami expression file
@@ -18,7 +20,8 @@ export default {
 
     // Compile the source code as an Origami program
     const compiler = options.compiler ?? compile.program;
-    const globals = options.globals ?? (await projectGlobals(parent));
+    const globals =
+      options.globals ?? getGlobalsForTree(parent) ?? (await coreGlobals());
     const fn = compiler(source, {
       globals,
       mode: "program",
@@ -26,11 +29,9 @@ export default {
     });
 
     // Evaluate the program
-    const result = await fn();
+    let result = await fn();
 
-    if (parent) {
-      setParent(result, parent);
-    }
+    result = processOriExport(result, source, parent);
 
     return result;
   },

package/src/handlers/oridocument_handler.js

@@ -1,7 +1,10 @@
-import { extension, getParent, trailingSlash } from "@weborigami/async-tree";
+import { extension, getParent } from "@weborigami/async-tree";
 import * as compile from "../compiler/compile.js";
-import projectGlobals from "../project/projectGlobals.js";
+import coreGlobals from "../project/coreGlobals.js";
+import getGlobalsForTree from "../project/getGlobalsForTree.js";
+import addExtensionKeyFn from "./addExtensionKeyFn.js";
 import getSource from "./getSource.js";
+import processOriExport from "./processOriExport.js";
 
 /**
  * An Origami template document: a plain text file that contains Origami
@@ -16,41 +19,29 @@ export default {
     const source = getSource(packed, options);
 
     // Compile the source code as an Origami template document
-    const globals = options.globals ?? (await projectGlobals(parent));
-    const defineFn = compile.templateDocument(source, {
+    const globals =
+      options.globals ?? getGlobalsForTree(parent) ?? (await coreGlobals());
+    const fn = compile.templateDocument(source, {
       front: options.front,
       globals,
       mode: "program",
       parent,
     });
 
-    // Invoke the definition to get back the template function
-    const result = await defineFn();
+    // Invoke the definition to get back the template function or object
+    let result = await fn();
 
-    const key = options.key;
-    const resultExtension = key ? extension.extname(key) : null;
-    if (resultExtension && Object.isExtensible(result)) {
-      // Add sidecar function so this template can be used in a map.
-      result.key = addExtension(resultExtension);
+    result = processOriExport(result, source, parent);
+
+    if (result instanceof Function) {
+      const key = options.key;
+      const resultExtension = key ? extension.extname(key) : null;
+      if (resultExtension && Object.isExtensible(result)) {
+        // Add sidecar function so this template can be used in a map.
+        result.key = addExtensionKeyFn(resultExtension);
+      }
     }
 
     return result;
   },
 };
-
-// Return a function that adds the given extension
-function addExtension(resultExtension) {
-  const keyFn = (sourceValue, sourceKey) => {
-    if (sourceKey === undefined) {
-      return undefined;
-    }
-    const normalizedKey = trailingSlash.remove(sourceKey);
-    const sourceExtension = extension.extname(normalizedKey);
-    const resultKey = sourceExtension
-      ? extension.replace(normalizedKey, sourceExtension, resultExtension)
-      : normalizedKey + resultExtension;
-    return resultKey;
-  };
-  keyFn.needsSourceValue = false;
-  return keyFn;
-}

package/src/handlers/processOriExport.js

@@ -0,0 +1,17 @@
+import { setParent, trailingSlash } from "@weborigami/async-tree";
+import enableValueCaching from "../runtime/enableValueCaching.js";
+
+/**
+ * Given an object that's the top-level result of an Origami file, perform any
+ * necessary processing.
+ */
+export default function processOriExport(value, source, parent) {
+  setParent(value, parent);
+
+  if (source.cachePath) {
+    const cachePath = trailingSlash.add(source.cachePath);
+    value = enableValueCaching(value, cachePath);
+  }
+
+  return value;
+}

package/src/handlers/tsv_handler.js

@@ -1,7 +1,7 @@
 import { symbols, toString } from "@weborigami/async-tree";
 
 export default {
-  mediaType: "text/csv",
+  mediaType: "text/tsv",
 
   unpack(packed, options = {}) {
     const parent = options.parent ?? null;