@weborigami/origami 0.6.14 → 0.6.16

package/src/dev/debug2/debugParent.js

@@ -0,0 +1,369 @@
+import { fork } from "node:child_process";
+import { EventEmitter } from "node:events";
+import http from "node:http";
+import { findOpenPort } from "../../common/findOpenPort.js";
+
+const PUBLIC_HOST = "127.0.0.1";
+
+// Module that loads the server in the child process
+const childModuleUrl = new URL("./debugChild.js", import.meta.url);
+
+// The public-facing server that proxies to the child process
+let publicServer;
+let publicOrigin;
+
+// The active child process and port
+/** @typedef {import("node:child_process").ChildProcess} ChildProcess */
+/** @typedef {{ process: ChildProcess, port: number | null }} ChildInfo */
+/** @type {ChildInfo | null} */
+let activeChild = null;
+
+// The most recently started child (may not be ready yet)
+/** @type {ChildInfo | null} */
+let pendingChild = null;
+
+// Used to communicate errors to caller
+let emitter = null;
+
+/**
+ * Start a new debug parent server for the given Origami expression and runtime
+ * state.
+ *
+ * This function will start a child server that evaluates the given expression
+ * with the given parent path. This arrangement ensures the expression is
+ * evaluated in a clean Node context (not polluted by previous evaluations). The
+ * parent server proxies requests to the child server.
+ *
+ * The debug parent monitors the parent tree for changes, and restarts the child
+ * whenever files in the parent tree change.
+ *
+ * Supported `options`:
+ * - `expression` (required): the Origami expression to evaluate in the child
+ *   process
+ * - `parentPath` (required): the path to the parent tree used for evaluation
+ *
+ * The returned `emitter` is an EventEmitter that emits "error" events when the
+ * child server encounters an Origami error while handling a request.
+ *
+ * @param {Object} options
+ * @param {string} options.expression
+ * @param {string} options.parentPath
+ * @param {number} [options.port]
+ * @param {boolean} [options.quiet]
+ */
+export default async function debugParent(options) {
+  const { parentPath } = options;
+  if (parentPath === undefined) {
+    throw new Error("Debugger couldn't work out the parent path.");
+  }
+
+  const port = options.port ?? (await findOpenPort());
+  publicOrigin = `http://${PUBLIC_HOST}:${port}`;
+
+  publicServer = http.createServer(proxyRequest);
+  await new Promise((resolve) =>
+    publicServer.listen(port, PUBLIC_HOST, resolve),
+  );
+  await startChild(options);
+
+  emitter = Object.assign(new EventEmitter(), {
+    close,
+    origin: publicOrigin,
+    reevaluate,
+    restart: () => startChild(options),
+  });
+  return emitter;
+}
+
+/**
+ * Gracefully stop the parent server and any active child server, giving the
+ * child a chance to finish any in-flight requests before exiting.
+ */
+async function close() {
+  // Stop accepting new connections and force-close any keep-alive connections
+  // so the close callback fires promptly.
+  const closed = new Promise((resolve) => publicServer.close(resolve));
+  publicServer.closeAllConnections();
+  await closed;
+  publicServer = null;
+
+  // Drain and stop any children concurrently
+  const children = [pendingChild?.process, activeChild?.process].filter(
+    /** @returns {child is ChildProcess} */
+    (child) => child !== undefined,
+  );
+  pendingChild = null;
+  activeChild = null;
+  await Promise.all(children.map(drainAndStopChild));
+
+  emitter.emit("close");
+  emitter.removeAllListeners();
+  emitter = null;
+}
+
+/**
+ * Give a child process a chance to finish any in-flight requests before we kill
+ * it.
+ *
+ * @param {ChildProcess} childProcess
+ */
+async function drainAndStopChild(childProcess) {
+  if (childProcess.killed) {
+    return;
+  }
+
+  // Ask it to drain first.
+  try {
+    childProcess.send({ type: "DRAIN" });
+  } catch {
+    // ignore
+  }
+
+  const drained = new Promise((resolve) => {
+    const onMessage = (msg) => {
+      if (msg && typeof msg === "object" && msg.type === "DRAINED") {
+        cleanup(resolve);
+      }
+    };
+    const onExit = () => cleanup(resolve);
+
+    function cleanup(done) {
+      childProcess.off("message", onMessage);
+      childProcess.off("exit", onExit);
+      done();
+    }
+
+    childProcess.on("message", onMessage);
+    childProcess.on("exit", onExit);
+  });
+
+  // Give it a short grace window to finish in-flight work.
+  const GRACE_MS = 1500;
+  await Promise.race([
+    drained,
+    new Promise((r) => setTimeout(r, GRACE_MS).unref()),
+  ]);
+
+  if (!childProcess.killed) {
+    childProcess.kill("SIGTERM");
+  }
+
+  // Final escalation.
+  setTimeout(() => {
+    // Child should have exited by now, but if not kill it
+    if (!childProcess.killed) {
+      childProcess.kill("SIGKILL");
+    }
+  }, GRACE_MS).unref();
+}
+
+/**
+ * Proxy incoming requests to the active child server, or return a 503 if not
+ * ready.
+ *
+ * @param {import("node:http").IncomingMessage} request
+ * @param {import("node:http").ServerResponse} response
+ */
+function proxyRequest(request, response) {
+  if (!activeChild) {
+    response.statusCode = 503;
+    response.setHeader("content-type", "text/plain; charset=utf-8");
+    response.end("Dev server is starting…\n");
+    return;
+  }
+
+  const { port } = activeChild;
+
+  // Minimal hop-by-hop header stripping
+  const headers = { ...request.headers };
+  delete headers.connection;
+  delete headers["proxy-connection"];
+  delete headers["keep-alive"];
+  delete headers.te;
+  delete headers.trailer;
+  delete headers["transfer-encoding"];
+  delete headers.upgrade;
+
+  const upstreamRequest = http.request(
+    {
+      host: PUBLIC_HOST,
+      port,
+      method: request.method,
+      path: request.url,
+      headers,
+    },
+    (upstreamResponse) => {
+      const { statusCode } = upstreamResponse;
+      response.writeHead(
+        statusCode ?? 502,
+        upstreamResponse.statusMessage,
+        upstreamResponse.headers,
+      );
+      upstreamResponse.pipe(response);
+
+      // Let caller know about the Origami error messages
+      if (statusCode !== undefined && statusCode >= 500 && emitter) {
+        const rawHeader = upstreamResponse.headers["x-error-details"];
+        const raw = Array.isArray(rawHeader) ? rawHeader[0] : rawHeader;
+        const message = raw ? decodeURIComponent(raw) : undefined;
+        if (message) {
+          emitter.emit("origami-error", message);
+        }
+      }
+    },
+  );
+
+  upstreamRequest.on("error", (err) => {
+    // Stop piping the request body
+    request.unpipe(upstreamRequest);
+    upstreamRequest.destroy();
+
+    // Only send error response if headers haven't been sent yet
+    if (!response.headersSent) {
+      response.statusCode = 502;
+      response.setHeader("content-type", "text/plain; charset=utf-8");
+      response.end(`Upstream error: ${err.message}\n`);
+    } else {
+      // Headers already sent, can't send error message - just close
+      response.destroy();
+    }
+  });
+
+  // Also handle errors on the incoming request
+  request.on("error", () => {
+    upstreamRequest.destroy();
+  });
+
+  request.pipe(upstreamRequest);
+}
+
+async function reevaluate() {
+  if (!activeChild) {
+    return;
+  }
+
+  const child = activeChild;
+
+  // Wait for the next EVALUATED message from the child
+  const evaluated = /** @type {Promise<void>} */ (
+    new Promise((resolve) => {
+      const onMessage = (/** @type {any} */ msg) => {
+        if (msg && typeof msg === "object" && msg.type === "EVALUATED") {
+          child.process.off("message", onMessage);
+          resolve();
+        }
+      };
+      child.process.on("message", onMessage);
+    })
+  );
+
+  child.process.send({ type: "REEVALUATE" });
+  await evaluated;
+}
+
+/**
+ * Start a new child process.
+ *
+ * This will be a pending process until it sends a READY message, at which point
+ * it becomes active and any previous active child is drained and stopped.
+ */
+function startChild(options) {
+  const { expression, parentPath } = options;
+  const quiet = options.quiet ?? false;
+
+  // Start the child process, passing parent path via an environment variable.
+  /** @type {ChildProcess} */
+  let childProcess;
+  try {
+    childProcess = fork(childModuleUrl, [], {
+      stdio: ["inherit", "inherit", "inherit", "ipc"],
+      env: {
+        ...process.env,
+        ORIGAMI_EXPRESSION: expression,
+        ORIGAMI_PARENT_PATH: parentPath,
+        ORIGAMI_QUIET: quiet ? "1" : "0",
+      },
+    });
+  } catch (error) {
+    throw new Error("Dev.debug2: failed to start child server:", {
+      cause: error,
+    });
+  }
+
+  // This becomes the pending child immediately
+  pendingChild = { process: childProcess, port: null };
+
+  // Returns a Promise that resolves when the child signals READY, or rejects
+  // on FATAL or unexpected exit before ready.
+  return /** @type {Promise<void>} */ (
+    new Promise((resolve, reject) => {
+      // Listen for messages from the child about its status
+      childProcess.on("message", (/** @type {any} */ message) => {
+        if (!message || typeof message !== "object") {
+          return;
+        } else if (
+          message.type === "READY" &&
+          typeof message.port === "number"
+        ) {
+          // Only promote to active if this is still the pending child
+          if (pendingChild?.process === childProcess) {
+            const previousChild = activeChild;
+
+            activeChild = pendingChild;
+            pendingChild.port = message.port;
+            pendingChild = null;
+
+            // Drain previous child in background (don't wait)
+            if (
+              previousChild?.process &&
+              previousChild.process !== childProcess
+            ) {
+              drainAndStopChild(previousChild.process).catch((err) =>
+                console.error("[drain]", err),
+              );
+            }
+
+            if (emitter) {
+              emitter.emit("ready", { origin: publicOrigin });
+            }
+            resolve();
+          } else {
+            // This child was superseded by a newer one, kill it
+            // console.log("Child process superseded by newer one, killing it...");
+            childProcess.kill("SIGTERM");
+          }
+        } else if (message.type === "EVALUATED") {
+          // Let caller know child has reevaluated the expression (after a file change)
+          if (emitter) {
+            emitter.emit("evaluated");
+          }
+        } else if (message.type === "FATAL") {
+          // Child couldn't start (import error, etc.)
+          // Keep previous active child if any; otherwise we'll serve 500/503.
+          console.error("[child fatal]", message.error ?? message);
+          if (pendingChild?.process === childProcess) {
+            pendingChild = null;
+          }
+          reject(new Error(message.error ?? "Child server failed to start"));
+        }
+      });
+
+      childProcess.on("exit", (code, signal) => {
+        if (activeChild?.process === childProcess) {
+          // Active child died unexpectedly.
+          activeChild = null;
+        }
+        if (pendingChild?.process === childProcess) {
+          pendingChild = null;
+          // Child died before it was ready. If child was ready and resolve()
+          // was called, when the child exits, then reject() is a no-op.
+          reject(
+            new Error(
+              `Child exited before ready (code=${code}, signal=${signal})`,
+            ),
+          );
+        }
+      });
+    })
+  );
+}

package/src/dev/debug2/debugTransform.js

@@ -9,9 +9,9 @@ import {
   scope,
   trailingSlash,
 } from "@weborigami/async-tree";
-import { projectGlobals } from "@weborigami/language";
 import indexPage from "../../origami/indexPage.js";
 import yaml from "../../origami/yaml.js";
+import * as debugCommands from "./debugCommands.js";
 
 /**
  * Transform the given map-based tree to add debugging resources:
@@ -22,10 +22,30 @@ import yaml from "../../origami/yaml.js";
  *
  * Also transform a simple object result to YAML for viewing.
  *
- * @param {import("@weborigami/async-tree").Maplike} maplike
+ * @typedef {import("@weborigami/async-tree").Maplike} Maplike
+ * @typedef {import("@weborigami/async-tree").Packed} Packed
+ *
+ * @param {Maplike|Packed} input
  */
-export default function debugTransform(maplike) {
-  const source = Tree.from(maplike, { deep: true });
+export default function debugTransform(input) {
+  if (isUnpackable(input)) {
+    // If the value isn't a tree, but has a tree attached via an `unpack`
+    // method, destructively wrap the unpack method to add this transform.
+    const original = input.unpack.bind(input);
+    input.unpack = async () => {
+      const content = await original();
+      if (!Tree.isTraversable(content) || typeof content === "function") {
+        return content;
+      }
+      /** @type {any} */
+      let tree = Tree.from(content);
+      return debugTransform(tree);
+    };
+    return input;
+  }
+
+  const source = Tree.from(input, { deep: true });
+
   return Object.assign(new AsyncMap(), {
     description: "debug resources",
 
@@ -34,7 +54,7 @@ export default function debugTransform(maplike) {
       let value = await source.get(key);
 
       if (value === undefined) {
-        // The tree doesn't have the key; try the defaults.
+        // Try the defaults and commands
         if (key === "index.html") {
           // Generate an index page for this site
           value = await indexPage(source);
@@ -54,28 +74,21 @@ export default function debugTransform(maplike) {
           Tree.merge(object, {
             "index.html": yamlText,
           });
-      } else if (Tree.isMaplike(value) && !Tree.isMap(value)) {
+      } else if (
+        Tree.isMaplike(value) &&
+        !Tree.isMap(value) &&
+        typeof value !== "function"
+      ) {
         // Make it a map so we can debug it
         value = Tree.from(value);
       }
 
-      if (Tree.isMap(value)) {
-        // Ensure this transform is applied to any map result
+      // Ensure this transform is applied to any map result, or any object with
+      // an unpack method that returns a map.
+      if (Tree.isMap(value) || value?.unpack) {
         value = debugTransform(value);
-      } else if (value?.unpack) {
-        // If the value isn't a tree, but has a tree attached via an `unpack`
-        // method, wrap the unpack method to add this transform.
-        const original = value.unpack.bind(value);
-        value.unpack = async () => {
-          const content = await original();
-          if (!Tree.isTraversable(content) || typeof content === "function") {
-            return content;
-          }
-          /** @type {any} */
-          let tree = Tree.from(content);
-          return debugTransform(tree);
-        };
       }
+
       return value;
     },
 
@@ -97,14 +110,13 @@ export default function debugTransform(maplike) {
 
 async function invokeOrigamiCommand(tree, key) {
   // Key is an Origami command; invoke it.
-  const globals = await projectGlobals(tree);
   const commandName = trailingSlash.remove(key.slice(1).trim());
 
-  // Look for command as a global or Dev command
-  const command = globals[commandName] ?? globals.Dev?.[commandName];
+  // Look for the indicated command
+  const command = debugCommands[commandName];
   let value;
   if (command) {
-    value = await command(tree);
+    value = command instanceof Function ? await command(tree) : command;
   } else {
     // Look for command in scope
     const parentScope = await scope(tree);

package/src/dev/debug2/expressionTree.js

@@ -14,10 +14,13 @@ let version = 0;
  * Evaluate the given expression using the indicated parent path to produce a
  * resource tree, then transform that tree with debug resources and return it.
  *
- * @param {string} expression
- * @param {string} parentPath
+ * @param {Object} options
+ * @param {string} options.expression
+ * @param {string} options.parentPath
  */
-export default async function expressionTree(expression, parentPath) {
+export default async function expressionTree(options) {
+  const { expression, parentPath } = options;
+
   const parent = new OrigamiFileMap(parentPath);
   const globals = await projectGlobals(parent);
 

package/src/dev/debug2/oriEval.js

@@ -0,0 +1,49 @@
+import { trailingSlash, Tree } from "@weborigami/async-tree";
+import { evaluate, projectGlobals } from "@weborigami/language";
+import debugTransform from "./debugTransform.js";
+
+const mapParentToResult = new WeakMap();
+
+/**
+ * Return a route that evaluates a key containing an expression.
+ *
+ * Because this route may be used multiple times to return the resources for a
+ * page, we cache the result for each parent and key.
+ *
+ * To allow multiple evaluations of the same expression, we expect the key to be
+ * of the form `<counter>,<expression>`, where the counter is a number that will
+ * be used for caching but ignored for evaluation. The debugger increments the
+ * counter to force reevaluation of the same expression.
+ */
+export default async function oriEval(parent) {
+  const globals = await projectGlobals(parent);
+  return async (key) => {
+    const normalizedKey = trailingSlash.remove(key);
+    let result = mapParentToResult.get(parent);
+    if (result?.key === normalizedKey) {
+      return result.value;
+    }
+
+    const regex = /^\d+,(?<expression>.*)$/;
+    const match = normalizedKey.match(regex);
+    let value;
+    if (match) {
+      const expression = decodeURIComponent(match.groups.expression);
+      value = await evaluate(expression, {
+        globals,
+        mode: "shell",
+        parent,
+      });
+      if (
+        (Tree.isMaplike(value) && typeof value !== "function") ||
+        value?.unpack
+      ) {
+        value = debugTransform(value);
+      }
+    }
+
+    mapParentToResult.set(parent, { key: normalizedKey, value });
+
+    return value;
+  };
+}

package/src/dev/help.yaml

@@ -79,6 +79,9 @@ Origami:
     inline:
       args: (text)
       description: Inline Origami expressions found in the text
+    htmlParse:
+      args: (html)
+      description: Parse HTML into a plain JavaScript object
     json:
       args: (obj)
       description: Render the object in JSON format
@@ -139,6 +142,9 @@ Origami:
     unpack:
       args: (buffer)
       description: Unpack the buffer into a usable form
+    xmlParse:
+      args: (xml)
+      description: Parse XML into a plain JavaScript object
     yaml:
       args: (obj)
       description: Render the object in YAML format
@@ -211,6 +217,9 @@ Tree:
     deepValues:
       args: (tree)
       description: The in-order leaf values of the tree
+    deflatePaths:
+      args: (tree)
+      description: Convert a tree to a mapping of paths to values
     delete:
       args: (map, key)
       description: Delete the value for the key from map
@@ -244,6 +253,9 @@ Tree:
     indent:
       args: "`…`"
       description: Tagged template literal for normalizing indentation
+    inflatePaths:
+      args: (map)
+      description: Convert mapping of paths to values into a tree
     inners:
       args: (tree)
       description: The tree's interior nodes

package/src/handlers/origamiHandlers.js

@@ -0,0 +1 @@
+export { default as xml_handler } from "./xml_handler.js";

package/src/handlers/xml_handler.js

@@ -0,0 +1,23 @@
+import { symbols, toString } from "@weborigami/async-tree";
+import xmlParse from "../origami/xmlParse.js";
+
+export default {
+  mediaType: "application/xml",
+
+  async unpack(packed, options = {}) {
+    const parent = options.parent ?? null;
+    const text = toString(packed);
+    if (text === null) {
+      throw new TypeError("XML handler can only unpack text.");
+    }
+    const data = await xmlParse(text);
+    // Define `parent` as non-enumerable property
+    Object.defineProperty(data, symbols.parent, {
+      configurable: true,
+      enumerable: false,
+      value: parent,
+      writable: true,
+    });
+    return data;
+  },
+};

package/src/origami/domNodeToObject.js

@@ -0,0 +1,93 @@
+const ELEMENT_NODE = 1;
+const TEXT_NODE = 3;
+const CDATA_SECTION_NODE = 4;
+const DOCUMENT_NODE = 9;
+const DOCUMENT_FRAGMENT_NODE = 11;
+
+export default function domNodeToObject(node) {
+  switch (node.nodeType) {
+    case DOCUMENT_NODE:
+      return {
+        name: "#document",
+        children: [...node.childNodes]
+          .filter((child) => !isWhitespaceOnly(child))
+          .map(domNodeToObject),
+      };
+
+    case DOCUMENT_FRAGMENT_NODE:
+      return {
+        name: "#document-fragment",
+        children: [...node.childNodes]
+          .filter((child) => !isWhitespaceOnly(child))
+          .map(domNodeToObject),
+      };
+
+    case ELEMENT_NODE: {
+      const attributes = Object.fromEntries(
+        [...node.attributes].map((attr) => [attr.name, attr.value]),
+      );
+
+      const relevantChildren = [...node.childNodes].filter(
+        (child) =>
+          (child.nodeType === ELEMENT_NODE ||
+            child.nodeType === TEXT_NODE ||
+            child.nodeType === CDATA_SECTION_NODE) &&
+          !isWhitespaceOnly(child),
+      );
+
+      const onlyText = relevantChildren.every(
+        (child) =>
+          child.nodeType === TEXT_NODE || child.nodeType === CDATA_SECTION_NODE,
+      );
+
+      const result = {
+        name: node.localName,
+      };
+      if (Object.keys(attributes).length > 0) {
+        result.attributes = attributes;
+      }
+      if (onlyText) {
+        const text = relevantChildren
+          .map((child) => collapseWhitespace(child.nodeValue ?? ""))
+          .join("")
+          .trim();
+        if (text.length > 0) {
+          result.text = text;
+        }
+      } else if (relevantChildren.length > 0) {
+        result.children = relevantChildren.map(domNodeToObject);
+      }
+
+      return result;
+    }
+
+    case TEXT_NODE:
+      return {
+        name: "#text",
+        text: collapseWhitespace(node.nodeValue ?? ""),
+      };
+
+    case CDATA_SECTION_NODE:
+      return {
+        name: "#cdata-section",
+        text: collapseWhitespace(node.nodeValue ?? ""),
+      };
+
+    default:
+      return {
+        name: `#node-${node.nodeType}`,
+      };
+  }
+}
+
+// Collapse leading or trailing whitespace characters to a single space
+function collapseWhitespace(str) {
+  return str.replace(/^\s+/, " ").replace(/\s+$/, " ");
+}
+
+function isWhitespaceOnly(node) {
+  return (
+    (node.nodeType === TEXT_NODE || node.nodeType === CDATA_SECTION_NODE) &&
+    node.nodeValue.trim() === ""
+  );
+}