@weborigami/language 0.6.17 → 0.7.0-beta.1

package/src/runtime/SyncCacheTransform.js

@@ -0,0 +1,133 @@
+import enableValueCaching from "./enableValueCaching.js";
+import { cachePathSymbol, noCacheSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
+import SystemCacheMap from "./SystemCacheMap.js";
+
+/**
+ * General-purpose mixin for Origami maps with dependency tracking, used for:
+ * files, site resources, and scope references in Origami files
+ *
+ * This wraps a map's get() and keys() methods to add caching and dependency tracking.
+ * It tracks which cached values are downstream of other cached values so that if
+ * an upstream value changes, all dependent downstream cached values can be
+ * invalidated efficiently.
+ *
+ * Cache entries look like:
+ *
+ *      key -> {
+ *        downstreams: Set(path),
+ *        value
+ *      }
+ *
+ * This allows for efficiently evicting all a value and all its downstream
+ * dependent cached values.
+ *
+ * Example project:
+ *
+ *     site.ori loads a.ori and b.ori
+ *     a.ori loads c.ori
+ *     b.ori loads c.ori
+ *     c.ori doesn't load anything
+ *
+ * Resulting cache:
+ *
+ *      site.ori -> { value: ... }
+ *      a.ori -> { downstreams: Set(site.ori), value: ... }
+ *      b.ori -> { downstreams: Set(site.ori), value: ... }
+ *      c.ori -> { downstreams: Set(a.ori, b.ori), value: ... }
+ */
+export default function SyncCacheTransform(Base) {
+  return class SyncCache extends Base {
+    constructor(...args) {
+      super(...args);
+
+      // Expose cache for debugging
+      this.cache = systemCache;
+    }
+
+    get cachePath() {
+      // @ts-ignore
+      return this[cachePathSymbol];
+    }
+
+    cachePathForKey(key) {
+      return key === "."
+        ? this.cachePath
+        : SystemCacheMap.joinPath(this.cachePath, key);
+    }
+
+    delete(key) {
+      const deleted = super.delete(key);
+      if (typeof key === "string") {
+        systemCache.delete(this.cachePathForKey(key));
+        if (deleted) {
+          // Deleted an existing key, need to invalidate cached keys
+          this.invalidateKeys();
+        }
+      }
+      return deleted;
+    }
+
+    get(key) {
+      if (typeof key !== "string" || key.length === 0) {
+        // Non-string keys and non-empty strings can't be cached
+        return super.get(key);
+      }
+      const cachePath = this.cachePathForKey(key);
+      const value = systemCache.getOrInsertComputed(cachePath, () => {
+        let result = super.get(key);
+        if (result !== undefined) {
+          // @ts-ignore
+          if (this[noCacheSymbol]) {
+            result[noCacheSymbol] = true;
+          } else {
+            result = enableValueCaching(result, cachePath);
+          }
+        }
+        return result;
+      });
+      return value;
+    }
+
+    invalidateKeys() {
+      const keysPath = this.cachePathForKey("_keys");
+      systemCache.delete(keysPath);
+    }
+
+    *keys() {
+      const keysPath = this.cachePathForKey("_keys");
+      const keys = systemCache.getOrInsertComputed(keysPath, () =>
+        // We can't cache an iterator; convert to array
+        Array.from(super.keys()),
+      );
+      yield* keys;
+    }
+
+    onKeysChange(key) {
+      super.onKeysChange?.(key);
+      this.invalidateKeys();
+    }
+
+    onValueChange(key) {
+      super.onValueChange?.(key);
+      systemCache.delete(this.cachePathForKey(key));
+    }
+
+    set(key, value) {
+      if (!this._self) {
+        // Initializing in constructor
+        super.set(key, value);
+        return;
+      }
+      if (typeof key !== "string") {
+        return super.set(key, value);
+      }
+      systemCache.delete(this.cachePathForKey(key));
+      if (!this.has(key)) {
+        // Adding a new key, need to invalidate cached keys
+        this.invalidateKeys();
+      }
+      super.set(key, value);
+    }
+  };
+}

package/src/runtime/SystemCacheMap.js

@@ -0,0 +1,259 @@
+import { SyncMap, trailingSlash } from "@weborigami/async-tree";
+import { AsyncLocalStorage } from "node:async_hooks";
+import { noCacheSymbol, volatileSymbol } from "./symbols.js";
+
+// Async storage for tracking dependencies encountered during function evaluation
+const asyncStorage = new AsyncLocalStorage();
+
+// Sync analogue to AsyncLocalStorage for tracking dependencies in sync functions
+const syncStorage = {
+  getStore() {
+    return this.stack.at(-1);
+  },
+
+  run(context, fn) {
+    this.stack.push(context);
+    const value = fn();
+    this.stack.pop();
+    return value;
+  },
+
+  /** @type {any[]} */
+  stack: [],
+};
+
+// For choosing a quasi-unique path for maps without a `cachePath` property
+let nextPathId = 0;
+
+export default class SystemCacheMap extends SyncMap {
+  delete(path) {
+    // Find all entries that depend on this path directly or indirectly
+    const toDelete = this.dependentEntries(path);
+
+    // Delete those dependent entries
+    for (const deletePath of toDelete.keys()) {
+      super.delete(deletePath);
+    }
+
+    // Remove deleted entries as being downstream from still-existing entries
+    for (const [deletePath, deleteEntry] of toDelete.entries()) {
+      for (const upstreamPath of deleteEntry.upstreams ?? []) {
+        const upstreamEntry = this.get(upstreamPath);
+        if (upstreamEntry?.downstreams) {
+          upstreamEntry.downstreams.delete(deletePath);
+          if (upstreamEntry.downstreams.size === 0) {
+            // No more downstream dependencies, clean up entry
+            delete upstreamEntry.downstreams;
+          }
+        }
+      }
+    }
+
+    return true;
+  }
+
+  // Return all entries that directly or indirectly depend on the given path
+  dependentEntries(path) {
+    const result = new Map();
+
+    const entry = this.get(path);
+    if (entry) {
+      // Path itself has an entry
+      result.set(path, entry);
+    }
+
+    // Add all entries with child paths that implicitly depend on this entry
+    for (const [otherPath, otherEntry] of this.entries()) {
+      if (this.isChildPath(path, otherPath)) {
+        result.set(otherPath, otherEntry);
+      }
+    }
+
+    // For each entry, add all entries downstream of it
+    for (const entry of result.values()) {
+      for (const downstreamPath of entry.downstreams ?? []) {
+        if (!result.has(downstreamPath)) {
+          for (const [key, value] of this.dependentEntries(downstreamPath)) {
+            if (!result.has(key)) {
+              result.set(key, value);
+            }
+          }
+        }
+      }
+    }
+
+    return result;
+  }
+
+  // REVIEW: This doesn't have the correct signature for getOrInsertComputed,
+  // because it returns the entry `value` property, not the actual entry.
+  getOrInsertComputed(path, computeFn) {
+    let entry = this.get(path);
+
+    if (entry && "value" in entry) {
+      // Cache hit, value already computed
+      this.trackCurrentDependency(path, entry);
+      return entry.value;
+    }
+
+    // Cache miss, or entry has no value yet
+    let value;
+
+    if (!entry) {
+      // Create empty entry for this path
+      entry = {};
+      this.set(path, entry);
+    }
+
+    // Create new sync context to track entries downstream of this value
+    const context = { downstream: path };
+
+    // Get value in sync context
+    value = syncStorage.run(context, computeFn);
+    if (value?.[noCacheSymbol] || value?.[volatileSymbol]) {
+      // Don't cache value
+      delete entry.value;
+    } else {
+      // Add resolved value to cache
+      entry.value = value;
+    }
+
+    this.trackCurrentDependency(path, entry);
+
+    return value;
+  }
+
+  async getOrInsertComputedAsync(path, computeFn) {
+    let entry = this.get(path);
+
+    if (entry && "value" in entry) {
+      // Cache hit, value already computed
+      this.trackCurrentDependency(path, entry);
+      return entry.value;
+    }
+
+    // Cache miss, or entry has no value yet
+    if (syncStorage.getStore()) {
+      // A function that was supposed to be sync called an async function
+      throw new Error("Cannot track async dependencies in a sync context");
+    }
+
+    if (!entry) {
+      // Create empty entry for this path
+      entry = {};
+      this.set(path, entry);
+    }
+
+    // Create new async context to track entries downstream of this value
+    const context = { downstream: path };
+
+    // Get value in async context, don't await the result yet. Add promise to
+    // cache so concurrent requests get the same promise.
+    entry.value = asyncStorage.run(context, async () => {
+      const value = await computeFn();
+      if (value?.[noCacheSymbol] || value?.[volatileSymbol]) {
+        // Don't cache value
+        delete entry.value;
+      } else {
+        // Add resolved value to cache
+        entry.value = value;
+      }
+      return value;
+    });
+
+    this.trackCurrentDependency(path, entry);
+
+    return entry.value;
+  }
+
+  /**
+   * Like standard path.join(), but without special handling for absolute or
+   * relative paths: adding "/", ".", or ".." adds those strings to the path.
+   * This also avoids the behavior in path.join() where consecutive separators
+   * are collapsed. We want the cache path `a//b` to be distinct from `a/b`.
+   *
+   * @param {string[]} segments
+   */
+  static joinPath(...segments) {
+    let result = segments.shift() ?? "";
+    while (segments.length > 0) {
+      if (!result.endsWith("/")) {
+        result += "/";
+      }
+      let segment = segments.shift() ?? "";
+      result += segment;
+    }
+    return result;
+  }
+
+  // A path is considered a child path if the parent path (including a trailing
+  // slash) is a prefix of the child path.
+  isChildPath(parentPath, childPath) {
+    const normalized = trailingSlash.add(parentPath);
+    return childPath.startsWith(normalized);
+  }
+
+  static nextDefaultCachePath() {
+    const cachePath = `_object${nextPathId}`;
+    nextPathId++;
+    return cachePath;
+  }
+
+  runInContext(cachePath, fn) {
+    const context = { downstream: cachePath };
+    return syncStorage.run(context, fn);
+  }
+
+  runInContextAsync(cachePath, fn) {
+    const context = { downstream: cachePath };
+    return asyncStorage.run(context, fn);
+  }
+
+  /**
+   * Given a path for an upstream dependency, and optionally the entry for that
+   * path if it has already been retrieved, track the dependency between the
+   * upstream entry and the currently running downstream path.
+   *
+   * @param {string} upstreamPath
+   * @param {any} [upstreamEntry]
+   */
+  trackCurrentDependency(upstreamPath, upstreamEntry = null) {
+    if (!upstreamEntry) {
+      upstreamEntry = this.get(upstreamPath);
+      if (!upstreamEntry) {
+        // Create empty entry for this path, so that dependencies can be tracked
+        // for values that aren't cached.
+        upstreamEntry = {};
+        this.set(upstreamPath, upstreamEntry);
+      }
+    }
+
+    // Is this call happening downstream of another cached value?
+    const { downstream } =
+      syncStorage.getStore() ?? asyncStorage.getStore() ?? {};
+    if (downstream) {
+      if (this.isChildPath(upstreamPath, downstream)) {
+        // Downstream path is a child of the upstream path, no need to record
+        // explicit dependency
+        return;
+      }
+
+      let downstreamEntry = this.get(downstream);
+      if (!downstreamEntry) {
+        // The downstream entry has been deleted from the cache. It seems that
+        // Node can resurrect an asyncStorage for a run that has already
+        // finished. To cope, we reconstruct an entry.
+        downstreamEntry = {};
+        this.set(downstream, downstreamEntry);
+      }
+
+      // Add the downstream entry to the upstream entry's downstreams
+      upstreamEntry.downstreams ??= new Set();
+      upstreamEntry.downstreams.add(downstream);
+
+      // Add the upstream entry to the downstream entry's upstreams
+      downstreamEntry.upstreams ??= new Set();
+      downstreamEntry.upstreams.add(upstreamPath);
+    }
+  }
+}

package/src/runtime/WatchFilesMixin.js

@@ -1,24 +1,16 @@
+import { keysFromPath, Tree } from "@weborigami/async-tree";
 import * as fs from "node:fs";
 import path from "node:path";
 import Watcher from "watcher";
 import TreeEvent from "./TreeEvent.js";
 
-// Map of paths to trees used by watcher
-const pathTreeMap = new Map();
-
 export default function WatchFilesMixin(Base) {
   return class WatchFiles extends Base {
     addEventListener(type, listener) {
       super.addEventListener(type, listener);
-      if (type === "change") {
-        this.watch();
-      }
     }
 
     onChange(filePath) {
-      // Reset cached values.
-      this.subfoldersMap = new Map();
-
       // Special case: ignore events in .git folder
       if (filePath.includes(`${path.sep}.git${path.sep}`)) {
         return;
@@ -27,38 +19,79 @@ export default function WatchFilesMixin(Base) {
       this.dispatchEvent(new TreeEvent("change", { filePath }));
     }
 
+    onKeysChange(key) {
+      super.onKeysChange?.(key);
+      // this.dispatchEvent(new TreeEvent("keyschange", { action, key }));
+    }
+
+    onValueChange(key) {
+      super.onValueChange?.(key);
+      // this.dispatchEvent(new TreeEvent("valuechange", { key }));
+    }
+
     unwatch() {
       if (!this.watching) {
         return;
       }
 
       this.watcher?.close();
-      this.watching = false;
+      this.watching = null;
     }
 
-    // Turn on watching for the directory.
+    // Turn on watching for the directory; resolves when the watcher is ready.
     watch() {
       if (this.watching) {
-        return;
+        return this.watching;
       }
-      this.watching = true;
 
       // Ensure the directory exists.
       fs.mkdirSync(this.dirname, { recursive: true });
 
       this.watcher = new Watcher(this.dirname, {
         ignoreInitial: true,
-        persistent: false,
         recursive: true,
       });
-      this.watcher.on("all", (event, filePath) => {
+      this.watching = new Promise((resolve) => {
+        this.watcher?.on("ready", resolve);
+      });
+      this.watcher.on("all", async (event, filePath) => {
         this.onChange(filePath);
+
+        const relativePath = path.relative(this.dirname, filePath);
+        if (relativePath.startsWith(".git")) {
+          return; // Ignore noisy events in .git folder
+        }
+        if (relativePath.startsWith("..")) {
+          // Event outside the watched directory, shouldn't happen but ignore
+          return;
+        }
+
+        const keys = keysFromPath(relativePath);
+        const key = keys.pop();
+
+        const target = await Tree.traverse(this, ...keys);
+        if (target) {
+          switch (event) {
+            case "add":
+            case "addDir":
+              target.onKeysChange(key);
+              break;
+
+            case "change":
+              target.onValueChange(key);
+              break;
+
+            case "unlink":
+            case "unlinkDir":
+              // Removing file/folder invalidates both its value and the keys
+              target.onValueChange(key);
+              target.onKeysChange(key);
+              break;
+          }
+        }
       });
 
-      // Add to the list of FileTree instances watching this directory.
-      const treeRefs = pathTreeMap.get(this.dirname) ?? [];
-      treeRefs.push(new WeakRef(this));
-      pathTreeMap.set(this.dirname, treeRefs);
+      return this.watching;
     }
   };
 }

package/src/runtime/enableValueCaching.js

@@ -0,0 +1,192 @@
+import { AsyncMap, isPlainObject, SyncMap, Tree } from "@weborigami/async-tree";
+import AsyncCacheTransform from "./AsyncCacheTransform.js";
+import { cachePathSymbol } from "./symbols.js";
+import SyncCacheTransform from "./SyncCacheTransform.js";
+import systemCache from "./systemCache.js";
+import SystemCacheMap from "./SystemCacheMap.js";
+
+// For detecting async functions
+const AsyncFunction = async function () {}.constructor;
+
+/**
+ * Given a maplike object whose values can be cached, enable caching. This may
+ * apply a caching transform, and sets a cache path on the object so that it can
+ * use as the prefix for cache paths.
+ *
+ * Note: this typically destructively modifies the given value.
+ *
+ * @param {any} value
+ * @param {string} cachePath
+ */
+export default function enableValueCaching(value, cachePath) {
+  if (!(typeof value === "object" || typeof value === "function")) {
+    // Don't need to apply caching to primitive value
+    return value;
+  }
+
+  const cacheable =
+    value[cachePathSymbol] === undefined && Tree.isMaplike(value);
+  if (cacheable) {
+    if (isPlainObject(value)) {
+      // Expression objects do their own caching
+      // TODO: What if it's some other kind of plain object?
+      markCacheable(value, cachePath);
+    } else if (Array.isArray(value)) {
+      // Cache arrays
+      markCacheable(value, cachePath);
+    } else if (value instanceof Function) {
+      // Cache a function
+      value = cacheFunction(value, cachePath);
+    } else if (
+      isTransformApplied(SyncCacheTransform, value) ||
+      isTransformApplied(AsyncCacheTransform, value)
+    ) {
+      // Already has caching transform applied; just mark cacheable
+      markCacheable(value, cachePath);
+    } else {
+      // Other maplike; convert to a Map/AsyncMap
+      value = Tree.from(value);
+      if (value instanceof Map) {
+        if (!(value instanceof SyncMap)) {
+          // Convert regular Map to SyncMap so we can extend it
+          value = new (SyncCacheTransform(SyncMap))(value);
+        } else {
+          // Cache a SyncMap
+          value = transformObject(SyncCacheTransform, value);
+        }
+      } else if (value instanceof AsyncMap) {
+        // Cache an AsyncMap
+        value = transformObject(AsyncCacheTransform, value);
+      }
+      markCacheable(value, cachePath);
+    }
+  }
+
+  return value;
+}
+
+/**
+ * Cache a function with arity 1 or greater that takes string arguments
+ *
+ * @param {Function} fn
+ * @param {string} cachePath
+ */
+export function cacheFunction(fn, cachePath) {
+  if (fn.length === 0) {
+    // Return as is
+    return fn;
+  }
+  let result;
+  if (fn instanceof AsyncFunction) {
+    // Return an async function that caches results for a unary argument
+    result = async (...args) => {
+      if (!allStringArguments(args)) {
+        // Run function in context of this cache path, but don't cache result
+        return systemCache.runInContextAsync(cachePath, () => fn(...args));
+      }
+      const keyCachePath = SystemCacheMap.joinPath(cachePath, args.join("/"));
+      let result = systemCache.getOrInsertComputedAsync(
+        keyCachePath,
+        async () => fn(...args),
+      );
+      result = enableValueCaching(result, keyCachePath);
+      return result;
+    };
+  } else {
+    // Return a sync function that caches results for a unary argument
+    result = (...args) => {
+      if (!allStringArguments(args)) {
+        // Run function in context of this cache path, but don't cache result
+        return systemCache.runInContext(cachePath, () => fn(...args));
+      }
+      const keyCachePath = SystemCacheMap.joinPath(cachePath, args.join("/"));
+      let result = systemCache.getOrInsertComputed(keyCachePath, () =>
+        fn(...args),
+      );
+      result = enableValueCaching(result, keyCachePath);
+      return result;
+    };
+  }
+  Object.defineProperty(result, "length", {
+    value: fn.length,
+    configurable: true,
+  });
+  markCacheable(result, cachePath);
+  return result;
+}
+
+// Non-string keys and non-empty strings can't be cached
+function allStringArguments(args) {
+  return (
+    args.length > 0 &&
+    args.every((arg) => typeof arg === "string" && arg.length > 0)
+  );
+}
+
+export function isTransformApplied(Transform, obj) {
+  let transformName = Transform.name;
+  if (!transformName) {
+    throw `isTransformApplied was called on an unnamed transform function, but a name is required.`;
+  }
+  if (transformName.endsWith("Transform")) {
+    transformName = transformName.slice(0, -9);
+  }
+  // Walk up prototype chain looking for a constructor with the same name as the
+  // transform. This is not a great test.
+  for (let proto = obj; proto; proto = Object.getPrototypeOf(proto)) {
+    if (proto.constructor.name === transformName) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function markCacheable(object, cachePath) {
+  Object.defineProperty(object, cachePathSymbol, {
+    configurable: true,
+    enumerable: false,
+    value: cachePath,
+  });
+}
+/**
+ * Apply a functional class mixin to an individual object instance.
+ *
+ * This works by create an intermediate class, creating an instance of that, and
+ * then setting the intermediate class's prototype to the given individual
+ * object. The resulting, extended object is then returned.
+ *
+ * This manipulation of the prototype chain is generally sound in JavaScript,
+ * with some caveats. In particular, the original object class cannot make
+ * direct use of private members; JavaScript will complain if the extended
+ * object does anything that requires access to those private members.
+ *
+ * @param {Function} Transform
+ * @param {any} obj
+ */
+export function transformObject(Transform, obj) {
+  // Apply the mixin to Object and instantiate that. The Object base class here
+  // is going to be cut out of the prototype chain in a moment; we just use
+  // Object as a convenience because its constructor takes no arguments.
+  const mixed = new (Transform(Object))();
+
+  // Find the highest prototype in the chain that was added by the class mixin.
+  // The mixin may have added multiple prototypes to the chain. Walk up the
+  // prototype chain until we hit Object.
+  let mixinProto = Object.getPrototypeOf(mixed);
+  while (Object.getPrototypeOf(mixinProto) !== Object.prototype) {
+    mixinProto = Object.getPrototypeOf(mixinProto);
+  }
+
+  // Redirect the prototype chain above the mixin to point to the original
+  // object. The mixed object now extends the original object with the mixin.
+  Object.setPrototypeOf(mixinProto, obj);
+
+  // Create a new constructor for this mixed object that reflects its prototype
+  // chain. Because we've already got the instance we want, we won't use this
+  // constructor now, but this can be used later to instantiate other objects
+  // that look like the mixed one.
+  mixed.constructor = Transform(obj.constructor);
+
+  // Return the mixed object.
+  return mixed;
+}