@weborigami/language 0.6.17 → 0.7.0-beta.2

package/src/runtime/OrigamiFileMap.d.ts

@@ -2,8 +2,11 @@ import { FileMap } from "@weborigami/async-tree";
 import EventTargetMixin from "./EventTargetMixin.js";
 import HandleExtensionsTransform from "./HandleExtensionsTransform.js";
 import ImportModulesMixin from "./ImportModulesMixin.js";
+import SyncCacheTransform from "./SyncCacheTransform.js";
 import WatchFilesMixin from "./WatchFilesMixin.js";
 
-export default class OrigamiFileMap extends HandleExtensionsTransform(
+export default class OrigamiFileMap extends SyncCacheTransform(HandleExtensionsTransform(
   ImportModulesMixin(WatchFilesMixin(EventTargetMixin(FileMap)))
-) {}
+)) {
+  globals: any;
+}

package/src/runtime/OrigamiFileMap.js

@@ -1,9 +1,32 @@
-import { FileMap } from "@weborigami/async-tree";
+import { FileMap, trailingSlash } from "@weborigami/async-tree";
 import EventTargetMixin from "./EventTargetMixin.js";
 import HandleExtensionsTransform from "./HandleExtensionsTransform.js";
 import ImportModulesMixin from "./ImportModulesMixin.js";
+import SyncCacheTransform from "./SyncCacheTransform.js";
 import WatchFilesMixin from "./WatchFilesMixin.js";
+import { noCacheSymbol } from "./symbols.js";
 
-export default class OrigamiFileMap extends HandleExtensionsTransform(
-  ImportModulesMixin(WatchFilesMixin(EventTargetMixin(FileMap)))
-) {}
+export default class OrigamiFileMap extends SyncCacheTransform(
+  HandleExtensionsTransform(
+    ImportModulesMixin(WatchFilesMixin(EventTargetMixin(FileMap))),
+  ),
+) {
+  get cachePath() {
+    return super.cachePath ?? this.path;
+  }
+
+  // Workaround to register file paths in the system cache without trailing
+  // slahes. This is so that if someone calls `get("site.ori/")`, the cache path
+  // will be "site.ori". It's not clear whether this is the best solution, but
+  // hopefully suffices for now.
+  cachePathForKey(key) {
+    const normalized = trailingSlash.remove(key);
+    return super.cachePathForKey(normalized);
+  }
+
+  globals = null;
+
+  // Don't cache files, just record their dependencies. The OS already caches
+  // files, so caching them just consumes memory and may slow things down.
+  [noCacheSymbol] = true;
+}

package/src/runtime/ScopeMap.js

@@ -0,0 +1,72 @@
+import { SyncMap } from "@weborigami/async-tree";
+import path from "node:path";
+import { cachePathSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
+
+/**
+ * Return a sync map of all values in scope for the given sync source map.
+ *
+ * @param {SyncMap} source
+ */
+export default class ScopeMap extends SyncMap {
+  constructor(source) {
+    super();
+    this.source = source;
+    this.trailingSlashKeys = source.trailingSlashKeys;
+  }
+
+  get(key) {
+    let value;
+
+    // Starting with this map, search up its parent hierarchy
+    let current = this.source;
+    while (current) {
+      // If the keys of this folder change, the scope request for this key could
+      // return a different value, so a change in keys needs to invalidate the
+      // value. Whether or not the get() request below succeeds, track the keys
+      // of this folder as an upstream dependency of the value being requested.
+      if (current[cachePathSymbol]) {
+        const folderKeysPath = path.join(current[cachePathSymbol], "_keys");
+        systemCache.trackCurrentDependency(folderKeysPath);
+      }
+
+      value = current.get(key);
+      if (value !== undefined) {
+        break;
+      }
+      current = current.parent;
+    }
+
+    return value;
+  }
+
+  // Collect all keys for this tree and all parents
+  keys() {
+    const scopeKeys = new Set();
+    let current = this.source;
+    while (current) {
+      for (const key of current.keys()) {
+        scopeKeys.add(key);
+      }
+      current = current.parent;
+    }
+    return scopeKeys[Symbol.iterator]();
+  }
+
+  // Collect all keys for this tree and all parents.
+  //
+  // This method exists for debugging purposes, as it's helpful to be able to
+  // quickly flatten and view the entire scope chain.
+  get trees() {
+    const result = [];
+
+    /** @type {SyncMap|null} */
+    let current = this.source;
+    while (current) {
+      result.push(current);
+      current = "parent" in current ? current.parent : null;
+    }
+
+    return result;
+  }
+}

package/src/runtime/SyncCacheTransform.d.ts

@@ -0,0 +1,8 @@
+import { Mixin } from "../../index.ts";
+
+declare const SyncCacheTransform: Mixin<{
+  get cachePath(): string;
+  cachePathForKey(key: string): string;
+}>
+
+export default SyncCacheTransform;

package/src/runtime/SyncCacheTransform.js

@@ -0,0 +1,133 @@
+import enableValueCaching from "./enableValueCaching.js";
+import { cachePathSymbol, noCacheSymbol } from "./symbols.js";
+import systemCache from "./systemCache.js";
+import SystemCacheMap from "./SystemCacheMap.js";
+
+/**
+ * General-purpose mixin for Origami maps with dependency tracking, used for:
+ * files, site resources, and scope references in Origami files
+ *
+ * This wraps a map's get() and keys() methods to add caching and dependency tracking.
+ * It tracks which cached values are downstream of other cached values so that if
+ * an upstream value changes, all dependent downstream cached values can be
+ * invalidated efficiently.
+ *
+ * Cache entries look like:
+ *
+ *      key -> {
+ *        downstreams: Set(path),
+ *        value
+ *      }
+ *
+ * This allows for efficiently evicting all a value and all its downstream
+ * dependent cached values.
+ *
+ * Example project:
+ *
+ *     site.ori loads a.ori and b.ori
+ *     a.ori loads c.ori
+ *     b.ori loads c.ori
+ *     c.ori doesn't load anything
+ *
+ * Resulting cache:
+ *
+ *      site.ori -> { value: ... }
+ *      a.ori -> { downstreams: Set(site.ori), value: ... }
+ *      b.ori -> { downstreams: Set(site.ori), value: ... }
+ *      c.ori -> { downstreams: Set(a.ori, b.ori), value: ... }
+ */
+export default function SyncCacheTransform(Base) {
+  return class SyncCache extends Base {
+    constructor(...args) {
+      super(...args);
+
+      // Expose cache for debugging
+      this.cache = systemCache;
+    }
+
+    get cachePath() {
+      // @ts-ignore
+      return this[cachePathSymbol];
+    }
+
+    cachePathForKey(key) {
+      return key === "."
+        ? this.cachePath
+        : SystemCacheMap.joinPath(this.cachePath, key);
+    }
+
+    delete(key) {
+      const deleted = super.delete(key);
+      if (typeof key === "string") {
+        systemCache.delete(this.cachePathForKey(key));
+        if (deleted) {
+          // Deleted an existing key, need to invalidate cached keys
+          this.invalidateKeys();
+        }
+      }
+      return deleted;
+    }
+
+    get(key) {
+      if (typeof key !== "string" || key.length === 0) {
+        // Non-string keys and non-empty strings can't be cached
+        return super.get(key);
+      }
+      const cachePath = this.cachePathForKey(key);
+      const value = systemCache.getOrInsertComputed(cachePath, () => {
+        let result = super.get(key);
+        if (result !== undefined) {
+          // @ts-ignore
+          if (this[noCacheSymbol]) {
+            result[noCacheSymbol] = true;
+          } else {
+            result = enableValueCaching(result, cachePath);
+          }
+        }
+        return result;
+      });
+      return value;
+    }
+
+    invalidateKeys() {
+      const keysPath = this.cachePathForKey("_keys");
+      systemCache.delete(keysPath);
+    }
+
+    *keys() {
+      const keysPath = this.cachePathForKey("_keys");
+      const keys = systemCache.getOrInsertComputed(keysPath, () =>
+        // We can't cache an iterator; convert to array
+        Array.from(super.keys()),
+      );
+      yield* keys;
+    }
+
+    onKeysChange(key) {
+      super.onKeysChange?.(key);
+      this.invalidateKeys();
+    }
+
+    onValueChange(key) {
+      super.onValueChange?.(key);
+      systemCache.delete(this.cachePathForKey(key));
+    }
+
+    set(key, value) {
+      if (!this._self) {
+        // Initializing in constructor
+        super.set(key, value);
+        return;
+      }
+      if (typeof key !== "string") {
+        return super.set(key, value);
+      }
+      systemCache.delete(this.cachePathForKey(key));
+      if (!this.has(key)) {
+        // Adding a new key, need to invalidate cached keys
+        this.invalidateKeys();
+      }
+      super.set(key, value);
+    }
+  };
+}

package/src/runtime/SystemCacheMap.js

@@ -0,0 +1,259 @@
+import { SyncMap, trailingSlash } from "@weborigami/async-tree";
+import { AsyncLocalStorage } from "node:async_hooks";
+import { noCacheSymbol, volatileSymbol } from "./symbols.js";
+
+// Async storage for tracking dependencies encountered during function evaluation
+const asyncStorage = new AsyncLocalStorage();
+
+// Sync analogue to AsyncLocalStorage for tracking dependencies in sync functions
+const syncStorage = {
+  getStore() {
+    return this.stack.at(-1);
+  },
+
+  run(context, fn) {
+    this.stack.push(context);
+    const value = fn();
+    this.stack.pop();
+    return value;
+  },
+
+  /** @type {any[]} */
+  stack: [],
+};
+
+// For choosing a quasi-unique path for maps without a `cachePath` property
+let nextPathId = 0;
+
+export default class SystemCacheMap extends SyncMap {
+  delete(path) {
+    // Find all entries that depend on this path directly or indirectly
+    const toDelete = this.dependentEntries(path);
+
+    // Delete those dependent entries
+    for (const deletePath of toDelete.keys()) {
+      super.delete(deletePath);
+    }
+
+    // Remove deleted entries as being downstream from still-existing entries
+    for (const [deletePath, deleteEntry] of toDelete.entries()) {
+      for (const upstreamPath of deleteEntry.upstreams ?? []) {
+        const upstreamEntry = this.get(upstreamPath);
+        if (upstreamEntry?.downstreams) {
+          upstreamEntry.downstreams.delete(deletePath);
+          if (upstreamEntry.downstreams.size === 0) {
+            // No more downstream dependencies, clean up entry
+            delete upstreamEntry.downstreams;
+          }
+        }
+      }
+    }
+
+    return true;
+  }
+
+  // Return all entries that directly or indirectly depend on the given path
+  dependentEntries(path) {
+    const result = new Map();
+
+    const entry = this.get(path);
+    if (entry) {
+      // Path itself has an entry
+      result.set(path, entry);
+    }
+
+    // Add all entries with child paths that implicitly depend on this entry
+    for (const [otherPath, otherEntry] of this.entries()) {
+      if (this.isChildPath(path, otherPath)) {
+        result.set(otherPath, otherEntry);
+      }
+    }
+
+    // For each entry, add all entries downstream of it
+    for (const entry of result.values()) {
+      for (const downstreamPath of entry.downstreams ?? []) {
+        if (!result.has(downstreamPath)) {
+          for (const [key, value] of this.dependentEntries(downstreamPath)) {
+            if (!result.has(key)) {
+              result.set(key, value);
+            }
+          }
+        }
+      }
+    }
+
+    return result;
+  }
+
+  // REVIEW: This doesn't have the correct signature for getOrInsertComputed,
+  // because it returns the entry `value` property, not the actual entry.
+  getOrInsertComputed(path, computeFn) {
+    let entry = this.get(path);
+
+    if (entry && "value" in entry) {
+      // Cache hit, value already computed
+      this.trackCurrentDependency(path, entry);
+      return entry.value;
+    }
+
+    // Cache miss, or entry has no value yet
+    let value;
+
+    if (!entry) {
+      // Create empty entry for this path
+      entry = {};
+      this.set(path, entry);
+    }
+
+    // Create new sync context to track entries downstream of this value
+    const context = { downstream: path };
+
+    // Get value in sync context
+    value = syncStorage.run(context, computeFn);
+    if (value?.[noCacheSymbol] || value?.[volatileSymbol]) {
+      // Don't cache value
+      delete entry.value;
+    } else {
+      // Add resolved value to cache
+      entry.value = value;
+    }
+
+    this.trackCurrentDependency(path, entry);
+
+    return value;
+  }
+
+  async getOrInsertComputedAsync(path, computeFn) {
+    let entry = this.get(path);
+
+    if (entry && "value" in entry) {
+      // Cache hit, value already computed
+      this.trackCurrentDependency(path, entry);
+      return entry.value;
+    }
+
+    // Cache miss, or entry has no value yet
+    if (syncStorage.getStore()) {
+      // A function that was supposed to be sync called an async function
+      throw new Error("Cannot track async dependencies in a sync context");
+    }
+
+    if (!entry) {
+      // Create empty entry for this path
+      entry = {};
+      this.set(path, entry);
+    }
+
+    // Create new async context to track entries downstream of this value
+    const context = { downstream: path };
+
+    // Get value in async context, don't await the result yet. Add promise to
+    // cache so concurrent requests get the same promise.
+    entry.value = asyncStorage.run(context, async () => {
+      const value = await computeFn();
+      if (value?.[noCacheSymbol] || value?.[volatileSymbol]) {
+        // Don't cache value
+        delete entry.value;
+      } else {
+        // Add resolved value to cache
+        entry.value = value;
+      }
+      return value;
+    });
+
+    this.trackCurrentDependency(path, entry);
+
+    return entry.value;
+  }
+
+  /**
+   * Like standard path.join(), but without special handling for absolute or
+   * relative paths: adding "/", ".", or ".." adds those strings to the path.
+   * This also avoids the behavior in path.join() where consecutive separators
+   * are collapsed. We want the cache path `a//b` to be distinct from `a/b`.
+   *
+   * @param {string[]} segments
+   */
+  static joinPath(...segments) {
+    let result = segments.shift() ?? "";
+    while (segments.length > 0) {
+      if (!result.endsWith("/")) {
+        result += "/";
+      }
+      let segment = segments.shift() ?? "";
+      result += segment;
+    }
+    return result;
+  }
+
+  // A path is considered a child path if the parent path (including a trailing
+  // slash) is a prefix of the child path.
+  isChildPath(parentPath, childPath) {
+    const normalized = trailingSlash.add(parentPath);
+    return childPath.startsWith(normalized);
+  }
+
+  static nextDefaultCachePath() {
+    const cachePath = `_object${nextPathId}`;
+    nextPathId++;
+    return cachePath;
+  }
+
+  runInContext(cachePath, fn) {
+    const context = { downstream: cachePath };
+    return syncStorage.run(context, fn);
+  }
+
+  runInContextAsync(cachePath, fn) {
+    const context = { downstream: cachePath };
+    return asyncStorage.run(context, fn);
+  }
+
+  /**
+   * Given a path for an upstream dependency, and optionally the entry for that
+   * path if it has already been retrieved, track the dependency between the
+   * upstream entry and the currently running downstream path.
+   *
+   * @param {string} upstreamPath
+   * @param {any} [upstreamEntry]
+   */
+  trackCurrentDependency(upstreamPath, upstreamEntry = null) {
+    if (!upstreamEntry) {
+      upstreamEntry = this.get(upstreamPath);
+      if (!upstreamEntry) {
+        // Create empty entry for this path, so that dependencies can be tracked
+        // for values that aren't cached.
+        upstreamEntry = {};
+        this.set(upstreamPath, upstreamEntry);
+      }
+    }
+
+    // Is this call happening downstream of another cached value?
+    const { downstream } =
+      syncStorage.getStore() ?? asyncStorage.getStore() ?? {};
+    if (downstream) {
+      if (this.isChildPath(upstreamPath, downstream)) {
+        // Downstream path is a child of the upstream path, no need to record
+        // explicit dependency
+        return;
+      }
+
+      let downstreamEntry = this.get(downstream);
+      if (!downstreamEntry) {
+        // The downstream entry has been deleted from the cache. It seems that
+        // Node can resurrect an asyncStorage for a run that has already
+        // finished. To cope, we reconstruct an entry.
+        downstreamEntry = {};
+        this.set(downstream, downstreamEntry);
+      }
+
+      // Add the downstream entry to the upstream entry's downstreams
+      upstreamEntry.downstreams ??= new Set();
+      upstreamEntry.downstreams.add(downstream);
+
+      // Add the upstream entry to the downstream entry's upstreams
+      downstreamEntry.upstreams ??= new Set();
+      downstreamEntry.upstreams.add(upstreamPath);
+    }
+  }
+}

package/src/runtime/WatchFilesMixin.js

@@ -1,24 +1,16 @@
+import { keysFromPath, Tree } from "@weborigami/async-tree";
 import * as fs from "node:fs";
 import path from "node:path";
 import Watcher from "watcher";
 import TreeEvent from "./TreeEvent.js";
 
-// Map of paths to trees used by watcher
-const pathTreeMap = new Map();
-
 export default function WatchFilesMixin(Base) {
   return class WatchFiles extends Base {
     addEventListener(type, listener) {
       super.addEventListener(type, listener);
-      if (type === "change") {
-        this.watch();
-      }
     }
 
     onChange(filePath) {
-      // Reset cached values.
-      this.subfoldersMap = new Map();
-
       // Special case: ignore events in .git folder
       if (filePath.includes(`${path.sep}.git${path.sep}`)) {
         return;
@@ -27,38 +19,79 @@ export default function WatchFilesMixin(Base) {
       this.dispatchEvent(new TreeEvent("change", { filePath }));
     }
 
+    onKeysChange(key) {
+      super.onKeysChange?.(key);
+      // this.dispatchEvent(new TreeEvent("keyschange", { action, key }));
+    }
+
+    onValueChange(key) {
+      super.onValueChange?.(key);
+      // this.dispatchEvent(new TreeEvent("valuechange", { key }));
+    }
+
     unwatch() {
       if (!this.watching) {
         return;
       }
 
       this.watcher?.close();
-      this.watching = false;
+      this.watching = null;
     }
 
-    // Turn on watching for the directory.
+    // Turn on watching for the directory; resolves when the watcher is ready.
     watch() {
       if (this.watching) {
-        return;
+        return this.watching;
       }
-      this.watching = true;
 
       // Ensure the directory exists.
       fs.mkdirSync(this.dirname, { recursive: true });
 
       this.watcher = new Watcher(this.dirname, {
         ignoreInitial: true,
-        persistent: false,
         recursive: true,
       });
-      this.watcher.on("all", (event, filePath) => {
+      this.watching = new Promise((resolve) => {
+        this.watcher?.on("ready", resolve);
+      });
+      this.watcher.on("all", async (event, filePath) => {
         this.onChange(filePath);
+
+        const relativePath = path.relative(this.dirname, filePath);
+        if (relativePath.startsWith(".git")) {
+          return; // Ignore noisy events in .git folder
+        }
+        if (relativePath.startsWith("..")) {
+          // Event outside the watched directory, shouldn't happen but ignore
+          return;
+        }
+
+        const keys = keysFromPath(relativePath);
+        const key = keys.pop();
+
+        const target = await Tree.traverse(this, ...keys);
+        if (target) {
+          switch (event) {
+            case "add":
+            case "addDir":
+              target.onKeysChange(key);
+              break;
+
+            case "change":
+              target.onValueChange(key);
+              break;
+
+            case "unlink":
+            case "unlinkDir":
+              // Removing file/folder invalidates both its value and the keys
+              target.onValueChange(key);
+              target.onKeysChange(key);
+              break;
+          }
+        }
       });
 
-      // Add to the list of FileTree instances watching this directory.
-      const treeRefs = pathTreeMap.get(this.dirname) ?? [];
-      treeRefs.push(new WeakRef(this));
-      pathTreeMap.set(this.dirname, treeRefs);
+      return this.watching;
     }
   };
 }