@weborigami/language 0.7.0-beta.1 → 0.7.0-beta.3

package/main.js

@@ -15,6 +15,7 @@ export { default as projectRoot } from "./src/project/projectRoot.js";
 export { default as projectRootFromPath } from "./src/project/projectRootFromPath.js";
 export { default as fetchAndHandleExtension } from "./src/protocols/fetchAndHandleExtension.js";
 export * as Protocols from "./src/protocols/protocols.js";
+export { default as counters } from "./src/runtime/counters.js";
 export { formatError, highlightError, lineInfo } from "./src/runtime/errors.js";
 export { default as evaluate } from "./src/runtime/evaluate.js";
 export { default as EventTargetMixin } from "./src/runtime/EventTargetMixin.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@weborigami/language",
-  "version": "0.7.0-beta.1",
+  "version": "0.7.0-beta.3",
   "description": "Web Origami expression language compiler and runtime",
   "type": "module",
   "main": "./main.js",
@@ -11,7 +11,8 @@
     "typescript": "6.0.3"
   },
   "dependencies": {
-    "@weborigami/async-tree": "0.7.0-beta.1",
+    "@weborigami/async-tree": "0.7.0-beta.3",
+    "adm-zip": "0.5.17",
     "exif-parser": "0.1.12",
     "watcher": "2.3.1",
     "yaml": "2.9.0"

package/src/compiler/compile.js

@@ -1,4 +1,3 @@
-import { trailingSlash } from "@weborigami/async-tree";
 import { createExpressionFunction } from "../runtime/expressionFunction.js";
 import optimize from "./optimize.js";
 import { parse } from "./parse.js";
@@ -33,13 +32,11 @@ function compile(source, options) {
 
   // Select a path the code will use for caching
   const cachePath = source.cachePath ?? `_compile${count++}`;
-  const objectCachePath = cachePath ? trailingSlash.add(cachePath) : null;
 
   // Optimize the code
   const optimized = optimize(code, {
     cachePath,
     globals,
-    objectCachePath,
   });
 
   // Create a function that executes the optimized code.

package/src/compiler/optimize.js

@@ -1,4 +1,4 @@
-import { pathFromKeys, trailingSlash } from "@weborigami/async-tree";
+import { trailingSlash } from "@weborigami/async-tree";
 import jsGlobals from "../project/jsGlobals.js";
 import {
   KEY_TYPE,
@@ -31,9 +31,13 @@ export const REFERENCE_EXTERNAL = 4;
  * @returns {AnnotatedCode}
  */
 export default function optimize(code, options = {}) {
-  // Cache path for this source file
-  const cachePath = options.cachePath ?? null;
   const globals = options.globals ?? jsGlobals;
+  const cachePath = options.cachePath; // top-level cache path for this code
+  // Cache path for attached objects
+  const objectCachePath =
+    options.objectCachePath ??
+    (cachePath ? trailingSlash.add(cachePath) : null);
+  const attached = options.attached ?? true;
 
   // The locals is an array, one item for each function or object context that
   // has been entered. The array grows to the right. Array items are objects
@@ -66,7 +70,8 @@ export default function optimize(code, options = {}) {
       return inlineLiteral(code);
 
     case ops.object:
-      const propertyNames = getPropertyNames(args);
+      const entries = args.slice(1);
+      const propertyNames = getPropertyNames(entries);
       locals.push({
         type: REFERENCE_INHERITED,
         names: propertyNames,
@@ -80,14 +85,24 @@ export default function optimize(code, options = {}) {
       if (op === ops.object) {
         if (index === 0) {
           return child; // return op as is
+        } else if (index === 1) {
+          // Replace null cache path with actual value
+          return objectCachePath;
         } else {
           // Object entry
           const [key, value] = child;
           const adjustedLocals = avoidLocalRecursion(locals, key);
           const isStringKey = typeof key === "string";
+          const cachePathKey = isStringKey
+            ? baseKeyName(key)
+            : `_arg${index - 2}`;
+          const childObjectCachePath = attached
+            ? SystemCacheMap.joinPath(objectCachePath, cachePathKey + "/")
+            : objectCachePath;
           const childOptions = {
             ...options,
             locals: adjustedLocals,
+            objectCachePath: childObjectCachePath,
           };
           const optimizedKey = isStringKey
             ? key
@@ -98,12 +113,30 @@ export default function optimize(code, options = {}) {
           );
           return annotate([optimizedKey, optimizedValue], child.location);
         }
+      } else if (op === ops.getter) {
+        if (index === 0) {
+          return child; // return op as is
+        } else {
+          // Already set up to optimize the (single) child
+          return optimize(/** @type {AnnotatedCode} */ (child), options);
+        }
       } else if (Array.isArray(child) && "location" in child) {
         // Review: Aside from ops.object (above), what non-instruction arrays
         // does this descend into?
-        const childOptions = { ...options, locals };
-        // Objects below here are detached from top-level object; not cached
-        delete childOptions.objectCachePath;
+        let childObjectCachePath = objectCachePath;
+        if (attached) {
+          // Objects below this point are detached, use `_objects` cache path
+          childObjectCachePath = SystemCacheMap.joinPath(
+            objectCachePath,
+            `_objects/`,
+          );
+        }
+        const childOptions = {
+          ...options,
+          attached: false,
+          locals,
+          objectCachePath: childObjectCachePath,
+        };
         return optimize(/** @type {AnnotatedCode} */ (child), childOptions);
       } else {
         return child;
@@ -124,23 +157,20 @@ export default function optimize(code, options = {}) {
 // remove any local variable with that name from the stack of locals to avoid a
 // recursive reference.
 function avoidLocalRecursion(locals, key) {
-  if (key[0] === "(" && key[key.length - 1] === ")") {
-    // Non-enumerable property, remove parentheses
-    key = key.slice(1, -1);
-  }
-
   const currentFrameIndex = locals.length - 1;
   if (locals[currentFrameIndex]?.type !== REFERENCE_INHERITED) {
     // Not an inherited context, nothing to do
     return locals;
   }
 
+  const baseKey = baseKeyName(key);
+
   // See if the key matches any of the local variable names in the current
   // context (ignoring trailing slashes)
   const matchingKeyIndex = locals[currentFrameIndex].names.findIndex(
     (name) =>
       // Ignore trailing slashes when comparing keys
-      trailingSlash.remove(name) === trailingSlash.remove(key),
+      trailingSlash.remove(name) === trailingSlash.remove(baseKey),
   );
 
   if (matchingKeyIndex >= 0) {
@@ -158,11 +188,12 @@ function avoidLocalRecursion(locals, key) {
   }
 }
 
-function cacheExternalPath(code, cachePath) {
-  const keys = code.map(keyFromCode).filter((key) => key !== null);
-  const keysPath = pathFromKeys(keys);
-  const refCachePath = SystemCacheMap.joinPath(cachePath, "_refs", keysPath);
-  return annotate([ops.cache, refCachePath, code], code.location);
+function baseKeyName(key) {
+  if (key[0] === "(" && key[key.length - 1] === ")") {
+    // Non-enumerable property, remove parentheses
+    key = key.slice(1, -1);
+  }
+  return key;
 }
 
 // A reference with periods like x.y.z
@@ -366,6 +397,13 @@ function resolvePath(code, globals, locals, cachePath) {
 
   let { type, result } = reference(head, globals, locals);
 
+  if (type === REFERENCE_EXTERNAL && cachePath) {
+    // External references should be cached
+    const headKey = keyFromCode(head);
+    const refCachePath = SystemCacheMap.joinPath(cachePath, "_refs", headKey);
+    result = annotate([ops.cache, refCachePath, result], head.location);
+  }
+
   if (tail.length > 0) {
     // If the result is a traversal, we can safely extend it
     const extendResult =
@@ -383,10 +421,5 @@ function resolvePath(code, globals, locals, cachePath) {
     }
   }
 
-  if (type === REFERENCE_EXTERNAL && cachePath) {
-    // External references should be cached
-    return cacheExternalPath(result, cachePath);
-  }
-
   return result;
 }

package/src/compiler/parserHelpers.js

@@ -272,7 +272,7 @@ export function makeDocument(front, body, location) {
   entries.push(annotate(["_body", body], location));
 
   // Return the code for the document object
-  return annotate([ops.object, ...entries], location);
+  return annotate([ops.object, null, ...entries], location);
 }
 
 /**
@@ -341,7 +341,7 @@ function makeMerge(spreads, location) {
 
   for (const spread of spreads) {
     if (spread[0] === ops.object) {
-      const spreadEntries = spread.slice(1);
+      const spreadEntries = spread.slice(2);
       topEntries.push(...spreadEntries);
       // Also add an object to the result with indirect references
       const indirectEntries = spreadEntries.map((entry) => {
@@ -352,7 +352,7 @@ function makeMerge(spreads, location) {
         return annotate([key, getter], entry.location);
       });
       const indirectObject = annotate(
-        [ops.object, ...indirectEntries],
+        [ops.object, null, ...indirectEntries],
         location,
       );
       resultEntries.push(indirectObject);
@@ -368,7 +368,7 @@ function makeMerge(spreads, location) {
   topEntries.push(annotate(["_result", result], location));
 
   // Construct the top-level object
-  const topObject = annotate([ops.object, ...topEntries], location);
+  const topObject = annotate([ops.object, null, ...topEntries], location);
 
   // Get the _result property
   const code = annotate([topObject, "_result"], location);
@@ -390,14 +390,17 @@ export function makeObject(entries, location) {
     if (key === markers.spread) {
       if (value[0] === ops.object) {
         // Spread of an object; fold into current object
-        const spreadEntries = value.slice(1);
+        const spreadEntries = value.slice(2);
         currentEntries.push(...spreadEntries);
       } else {
         // Spread of a tree; accumulate
         if (currentEntries.length > 0) {
           const location = { ...currentEntries[0].location };
           location.end = currentEntries[currentEntries.length - 1].location.end;
-          const spread = annotate([ops.object, ...currentEntries], location);
+          const spread = annotate(
+            [ops.object, null, ...currentEntries],
+            location,
+          );
           spreads.push(spread);
           currentEntries = [];
         }
@@ -424,7 +427,7 @@ export function makeObject(entries, location) {
   if (currentEntries.length > 0) {
     const location = { ...currentEntries[0].location };
     location.end = currentEntries[currentEntries.length - 1].location.end;
-    const spread = annotate([ops.object, ...currentEntries], location);
+    const spread = annotate([ops.object, null, ...currentEntries], location);
     spreads.push(spread);
     currentEntries = [];
   }
@@ -442,7 +445,7 @@ export function makeObject(entries, location) {
     }
   } else {
     // Empty object
-    code = [ops.object];
+    code = [ops.object, null];
   }
 
   return annotate(code, location);

package/src/handlers/addExtensionKeyFn.js

@@ -0,0 +1,18 @@
+import { extension, trailingSlash } from "@weborigami/async-tree";
+
+// Return a function that adds the given extension
+export default function addExtensionKeyFn(resultExtension) {
+  const keyFn = (sourceValue, sourceKey) => {
+    if (sourceKey === undefined) {
+      return undefined;
+    }
+    const normalizedKey = trailingSlash.remove(sourceKey);
+    const sourceExtension = extension.extname(normalizedKey);
+    const resultKey = sourceExtension
+      ? extension.replace(normalizedKey, sourceExtension, resultExtension)
+      : normalizedKey + resultExtension;
+    return resultKey;
+  };
+  keyFn.needsSourceValue = false;
+  return keyFn;
+}

package/src/handlers/epub_handler.js

@@ -0,0 +1,54 @@
+import { AsyncMap, isUnpackable, Tree } from "@weborigami/async-tree";
+import addExtensionKeyFn from "./addExtensionKeyFn.js";
+import zip_handler from "./zip_handler.js";
+
+/**
+ * Handler for EPUB files
+ */
+const epub_handler = {
+  mediaType: "application/epub+zip",
+
+  /**
+   * Package a tree of files as an EPUB file in Buffer form.
+   *
+   * This calls the pack() method for ZIP files, but ensures the `mimetype` file
+   * is the first file in the package -- a requirement for EPUB files.
+   *
+   * @param {import("@weborigami/async-tree").Maplike} maplike
+   */
+  async pack(maplike) {
+    if (isUnpackable(maplike)) {
+      maplike = await maplike.unpack();
+    }
+    const tree = Tree.from(maplike, { deep: true });
+    return zip_handler.pack(mimetypeFirst(tree));
+  },
+
+  async unpack(buffer, options) {
+    return zip_handler.unpack(buffer, options);
+  },
+};
+
+/** @type {any} */ (epub_handler.pack).key = addExtensionKeyFn(".epub");
+
+export default epub_handler;
+
+// A tree with its `mimetype` file first
+function mimetypeFirst(tree) {
+  return Object.assign(new AsyncMap(), {
+    async get(key) {
+      return tree.get(key);
+    },
+
+    async *keys() {
+      const keys = await Tree.keys(tree);
+      // Move `mimetype` (if present) to the front of the list.
+      const index = keys.indexOf("mimetype");
+      if (index >= 0) {
+        keys.splice(index, 1);
+        keys.unshift("mimetype");
+      }
+      yield* keys;
+    },
+  });
+}

package/src/handlers/getSource.js

@@ -30,6 +30,7 @@ export default function getSource(packed, options = {}) {
       const parentPath = /** @type {any} */ (parent).path;
       cachePath = SystemCacheMap.joinPath(parentPath, sourceName);
     } else {
+      // TODO: Avoid cache path collisions in this case
       cachePath = sourceName;
     }
   }

package/src/handlers/handlers.js

@@ -20,6 +20,7 @@ export { default as txt_handler } from "./txt_handler.js";
 
 export { default as css_handler } from "./css_handler.js";
 export { default as csv_handler } from "./csv_handler.js";
+export { default as epub_handler } from "./epub_handler.js";
 export { default as htm_handler } from "./htm_handler.js";
 export { default as html_handler } from "./html_handler.js";
 export { default as jpeg_handler } from "./jpeg_handler.js";
@@ -33,3 +34,4 @@ export { default as wasm_handler } from "./wasm_handler.js";
 export { default as xhtml_handler } from "./xhtml_handler.js";
 export { default as yaml_handler } from "./yaml_handler.js";
 export { default as yml_handler } from "./yml_handler.js";
+export { default as zip_handler } from "./zip_handler.js";

package/src/handlers/ori_handler.js

@@ -3,7 +3,7 @@ import * as compile from "../compiler/compile.js";
 import coreGlobals from "../project/coreGlobals.js";
 import getGlobalsForTree from "../project/getGlobalsForTree.js";
 import getSource from "./getSource.js";
-import processOriExport from "./processOriExport.js";
+import processOrigamiExport from "./processOrigamiExport.js";
 
 /**
  * An Origami expression file
@@ -31,7 +31,7 @@ export default {
     // Evaluate the program
     let result = await fn();
 
-    result = processOriExport(result, source, parent);
+    result = processOrigamiExport(result, source, parent);
 
     return result;
   },

package/src/handlers/oridocument_handler.js

@@ -1,9 +1,10 @@
-import { extension, getParent, trailingSlash } from "@weborigami/async-tree";
+import { extension, getParent } from "@weborigami/async-tree";
 import * as compile from "../compiler/compile.js";
 import coreGlobals from "../project/coreGlobals.js";
 import getGlobalsForTree from "../project/getGlobalsForTree.js";
+import addExtensionKeyFn from "./addExtensionKeyFn.js";
 import getSource from "./getSource.js";
-import processOriExport from "./processOriExport.js";
+import processOrigamiExport from "./processOrigamiExport.js";
 
 /**
  * An Origami template document: a plain text file that contains Origami
@@ -30,34 +31,17 @@ export default {
     // Invoke the definition to get back the template function or object
     let result = await fn();
 
-    result = processOriExport(result, source, parent);
+    result = processOrigamiExport(result, source, parent);
 
     if (result instanceof Function) {
       const key = options.key;
       const resultExtension = key ? extension.extname(key) : null;
       if (resultExtension && Object.isExtensible(result)) {
         // Add sidecar function so this template can be used in a map.
-        result.key = addExtension(resultExtension);
+        result.key = addExtensionKeyFn(resultExtension);
       }
     }
 
     return result;
   },
 };
-
-// Return a function that adds the given extension
-function addExtension(resultExtension) {
-  const keyFn = (sourceValue, sourceKey) => {
-    if (sourceKey === undefined) {
-      return undefined;
-    }
-    const normalizedKey = trailingSlash.remove(sourceKey);
-    const sourceExtension = extension.extname(normalizedKey);
-    const resultKey = sourceExtension
-      ? extension.replace(normalizedKey, sourceExtension, resultExtension)
-      : normalizedKey + resultExtension;
-    return resultKey;
-  };
-  keyFn.needsSourceValue = false;
-  return keyFn;
-}

package/src/handlers/{processOriExport.js → processOrigamiExport.js}

@@ -5,7 +5,7 @@ import enableValueCaching from "../runtime/enableValueCaching.js";
  * Given an object that's the top-level result of an Origami file, perform any
  * necessary processing.
  */
-export default function processOriExport(value, source, parent) {
+export default function processOrigamiExport(value, source, parent) {
   setParent(value, parent);
 
   if (source.cachePath) {

package/src/handlers/zip_handler.js

@@ -0,0 +1,112 @@
+import { isUnpackable, SyncMap, Tree } from "@weborigami/async-tree";
+import {
+  getGlobalsForTree,
+  HandleExtensionsTransform,
+} from "@weborigami/language";
+import Zip from "adm-zip";
+import addExtensionKeyFn from "./addExtensionKeyFn.js";
+
+/**
+ * Handler for ZIP files
+ */
+const zip_handler = {
+  mediaType: "application/zip",
+
+  /**
+   * Pack a tree of files as a ZIP file in Buffer form.
+   *
+   * @param {import("@weborigami/async-tree").Maplike} maplike
+   */
+  async pack(maplike) {
+    // The ZIP file should leave the files in tree order.
+    const zip = new Zip({ noSort: true });
+
+    if (isUnpackable(maplike)) {
+      maplike = await maplike.unpack();
+    }
+    const tree = Tree.from(maplike, { deep: true });
+    const deflated = await Tree.deflatePaths(tree);
+    for (let [path, value] of deflated) {
+      if (typeof value === "function") {
+        value = value();
+      }
+      if (value instanceof Promise) {
+        value = await value;
+      } else if (value instanceof String) {
+        value = value.toString(); // adm-zip wants simple strings
+      }
+      zip.addFile(path, value);
+
+      // Special case for EPUB files, where `mimetype` must be uncompressed.
+      if (path === "mimetype") {
+        const entry = zip.getEntry(path);
+        entry.header.method = 0; // STORE (not DEFLATE)
+      }
+    }
+    const buffer = zip.toBuffer();
+    return buffer;
+  },
+
+  /**
+   * Unpack a ZIP file
+   */
+  async unpack(buffer, options) {
+    // Origami generally prefers keeping things as an Uint8Array or ArrayBuffer,
+    // but adm-zip only accepts a Buffer.
+    if (buffer instanceof Uint8Array || buffer instanceof ArrayBuffer) {
+      // @ts-ignore
+      buffer = Buffer.from(buffer);
+    }
+
+    const zip = new Zip(buffer);
+
+    const entries = zip.getEntries();
+    const filtered = entries.filter(
+      (entry) =>
+        !entry.entryName.startsWith("__MACOSX/") &&
+        !entry.entryName.endsWith("/"),
+    );
+    const deflated = Object.fromEntries(
+      filtered.map((entry) => [entry.entryName, () => entry.getData()]),
+    );
+
+    // The final tree will include extension handlers and have functions invoked
+    // to retrieve data from the ZIP file. While the base map is a SyncMap, the
+    // final tree will be async.
+    const classFn = HandleExtensionsTransform(
+      InvokeFunctionsTransform(SyncMap),
+    );
+    const result = await Tree.inflatePaths(deflated, { classFn });
+
+    const parent = options?.parent;
+    const globals = parent ? getGlobalsForTree(parent) : null;
+    if (globals) {
+      result.globals = globals;
+    }
+
+    return result;
+  },
+};
+
+/** @type {any} */ (zip_handler.pack).key = addExtensionKeyFn(".zip");
+
+export default zip_handler;
+
+function InvokeFunctionsTransform(Base) {
+  return class InvokeFunctions extends Base {
+    delete(key) {
+      return super.delete(key);
+    }
+
+    get(key) {
+      const value = super.get(key);
+      return typeof value === "function" ? value() : value;
+    }
+
+    set(key, value) {
+      return super.set(key, value);
+    }
+
+    trailingSlashKeys = true;
+  };
+}

package/src/runtime/AsyncCacheTransform.d.ts

@@ -1,5 +1,9 @@
 import { Mixin } from "../../index.ts";
 
-declare const AsyncCacheTransform: Mixin<{}>
+declare const AsyncCacheTransform: Mixin<{
+  cachePathForKey(key: string): string;
+  onKeysChange(key: string): void;
+  onValueChange(key: string): void;
+}>
 
 export default AsyncCacheTransform;

package/src/runtime/AsyncCacheTransform.js

@@ -45,24 +45,21 @@ export default function AsyncCacheTransform(Base) {
       this.cache = systemCache;
     }
 
-    get cachePath() {
-      // @ts-ignore
-      return this[cachePathSymbol];
-    }
-
     cachePathForKey(key) {
       return key === "."
-        ? this.cachePath
-        : SystemCacheMap.joinPath(this.cachePath, key);
+        ? // @ts-ignore
+          this[cachePathSymbol]
+        : // @ts-ignore
+          SystemCacheMap.joinPath(this[cachePathSymbol], key);
     }
 
-    async delete(key) {
-      const deleted = await super.delete(key);
-      if (typeof key === "string") {
-        systemCache.delete(this.cachePathForKey(key));
-      }
-      return deleted;
-    }
+    // async delete(key) {
+    //   const deleted = await super.delete(key);
+    //   if (typeof key === "string") {
+    //     systemCache.delete(this.cachePathForKey(key));
+    //   }
+    //   return deleted;
+    // }
 
     async get(key) {
       if (typeof key !== "string" || key.length === 0) {
@@ -119,16 +116,16 @@ export default function AsyncCacheTransform(Base) {
       systemCache.delete(this.cachePathForKey(key));
     }
 
-    async set(key, value) {
-      if (typeof key !== "string") {
-        return super.set(key, value);
-      }
-      systemCache.delete(this.cachePathForKey(key));
-      if (!this.has(key)) {
-        // Adding a new key, need to invalidate cached keys
-        this.invalidateKeys();
-      }
-      super.set(key, value);
-    }
+    // async set(key, value) {
+    //   if (typeof key !== "string") {
+    //     return super.set(key, value);
+    //   }
+    //   systemCache.delete(this.cachePathForKey(key));
+    //   if (!this.has(key)) {
+    //     // Adding a new key, need to invalidate cached keys
+    //     this.invalidateKeys();
+    //   }
+    //   super.set(key, value);
+    // }
   };
 }

package/src/runtime/OrigamiFileMap.js

@@ -4,15 +4,15 @@ import HandleExtensionsTransform from "./HandleExtensionsTransform.js";
 import ImportModulesMixin from "./ImportModulesMixin.js";
 import SyncCacheTransform from "./SyncCacheTransform.js";
 import WatchFilesMixin from "./WatchFilesMixin.js";
-import { noCacheSymbol } from "./symbols.js";
+import { cachePathSymbol, noCacheSymbol } from "./symbols.js";
 
 export default class OrigamiFileMap extends SyncCacheTransform(
   HandleExtensionsTransform(
     ImportModulesMixin(WatchFilesMixin(EventTargetMixin(FileMap))),
   ),
 ) {
-  get cachePath() {
-    return super.cachePath ?? this.path;
+  get [cachePathSymbol]() {
+    return this.path;
   }
 
   // Workaround to register file paths in the system cache without trailing

package/src/runtime/SyncCacheTransform.d.ts

@@ -1,8 +1,9 @@
 import { Mixin } from "../../index.ts";
 
 declare const SyncCacheTransform: Mixin<{
-  get cachePath(): string;
   cachePathForKey(key: string): string;
+  onKeysChange(key: string): void;
+  onValueChange(key: string): void;
 }>
 
 export default SyncCacheTransform;