@weborigami/origami 0.3.1 → 0.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@weborigami/origami",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "Web Origami language, CLI, framework, and server",
   "type": "module",
   "repository": {
@@ -17,13 +17,15 @@
     "typescript": "5.8.2"
   },
   "dependencies": {
-    "@weborigami/async-tree": "0.3.1",
-    "@weborigami/language": "0.3.1",
+    "@weborigami/async-tree": "0.3.2",
     "@weborigami/json-feed-to-rss": "1.0.0",
-    "@weborigami/types": "0.3.1",
+    "@weborigami/language": "0.3.2",
+    "@weborigami/types": "0.3.2",
+    "css-tree": "3.1.0",
     "exif-parser": "0.1.12",
     "graphviz-wasm": "3.0.2",
     "highlight.js": "11.11.1",
+    "jsdom": "26.1.0",
     "marked": "15.0.7",
     "marked-gfm-heading-id": "4.1.1",
     "marked-highlight": "2.2.1",

package/src/dev/crawler/audit.js

@@ -0,0 +1,85 @@
+import { pathFromKeys, symbols, Tree } from "@weborigami/async-tree";
+import getTreeArgument from "../../common/getTreeArgument.js";
+import crawlResources from "./crawlResources.js";
+import { getBaseUrl } from "./utilities.js";
+
+/**
+ * Crawl the indicated tree and return an audit of any broken links to internal
+ * pages or other resources.
+ *
+ * @typedef  {import("@weborigami/types").AsyncTree} AsyncTree
+ * @typedef {import("@weborigami/async-tree").Treelike} Treelike
+ *
+ * @this {AsyncTree|null}
+ * @param {Treelike} treelike
+ * @param {string} [baseHref]
+ */
+export default async function audit(treelike, baseHref) {
+  const tree = await getTreeArgument(this, arguments, treelike, "site:audit");
+  const baseUrl = getBaseUrl(baseHref, treelike);
+
+  let errors = {};
+  let report;
+  const resourceReferences = {};
+  const resourcePromises = {};
+
+  // Iterate through all the resources to crawl the whole tree.
+  for await (const result of crawlResources(tree, baseUrl)) {
+    const { normalizedKeys, resourcePaths, value: resource } = result;
+    const normalizedPath = pathFromKeys(normalizedKeys);
+    if (normalizedPath === "crawl-errors.json") {
+      // Final error report; add missing pages to the errors
+      report = JSON.parse(resource);
+      for (const [path, pagePaths] of Object.entries(report)) {
+        if (!errors[path]) {
+          errors[path] = [];
+        }
+        errors[path].push(...pagePaths);
+      }
+    } else {
+      // Record which resources this path references
+      resourceReferences[normalizedPath] = resourcePaths;
+
+      // Add all resources to the set that should be verified
+      for (const resourcePath of resourcePaths) {
+        // Start request, don't wait for it to complete yet
+        resourcePromises[resourcePath] ??= Tree.traversePath(
+          tree,
+          resourcePath
+        ).then(
+          // Just return true or false to indicate if value is defined
+          (value) => value !== undefined
+        );
+      }
+    }
+  }
+
+  // Add any references to missing resources to the errors
+  for (const [refererPath, resourcePaths] of Object.entries(
+    resourceReferences
+  )) {
+    for (const resourcePath of resourcePaths) {
+      const found = await resourcePromises[resourcePath];
+      if (!found) {
+        if (!errors[refererPath]) {
+          errors[refererPath] = [];
+        }
+        errors[refererPath].push(resourcePath);
+      }
+    }
+  }
+
+  if (Object.keys(errors).length === 0) {
+    return undefined;
+  }
+
+  Object.defineProperty(errors, symbols.parent, {
+    enumerable: false,
+    value: this,
+  });
+  Object.defineProperty(errors, symbols.deep, {
+    enumerable: false,
+    value: true,
+  });
+  return errors;
+}

package/src/{site → dev}/crawler/crawl.js

@@ -2,13 +2,12 @@ import {
   DeepObjectTree,
   Tree,
   deepMerge,
-  isPlainObject,
   keysFromPath,
-  trailingSlash,
 } from "@weborigami/async-tree";
 import { InvokeFunctionsTransform } from "@weborigami/language";
 import getTreeArgument from "../../common/getTreeArgument.js";
 import crawlResources from "./crawlResources.js";
+import { addValueToObject, getBaseUrl } from "./utilities.js";
 
 /**
  * Crawl a tree, starting its root index.html page, and following links to
@@ -20,6 +19,7 @@ import crawlResources from "./crawlResources.js";
  *
  * @typedef  {import("@weborigami/types").AsyncTree} AsyncTree
  * @typedef {import("@weborigami/async-tree").Treelike} Treelike
+ *
  * @this {AsyncTree|null}
  * @param {Treelike} treelike
  * @param {string} [baseHref]
@@ -27,39 +27,10 @@ import crawlResources from "./crawlResources.js";
  */
 export default async function crawlBuiltin(treelike, baseHref) {
   const tree = await getTreeArgument(this, arguments, treelike, "site:crawl");
-
-  if (baseHref === undefined) {
-    // Ask tree or original treelike if it has an `href` property we can use as
-    // the base href to determine whether a link is local within the tree or
-    // not. If not, use a fake `local:/` base href.
-    baseHref =
-      /** @type {any} */ (tree).href ??
-      /** @type {any} */ (treelike).href ??
-      "local:/";
-    if (!baseHref?.endsWith("/")) {
-      baseHref += "/";
-    }
-  } else {
-    // Is the href already valid?
-    let isHrefValid = false;
-    try {
-      new URL(baseHref);
-      isHrefValid = true;
-    } catch (e) {
-      // Ignore
-    }
-    if (!isHrefValid) {
-      // Use a fake base href.
-      baseHref = `local:/${baseHref}`;
-    }
-  }
-
-  // @ts-ignore
-  const baseUrl = new URL(baseHref);
+  const baseUrl = getBaseUrl(baseHref, treelike);
 
   const cache = {};
   const resources = {};
-  const errors = [];
 
   // We iterate until there are no more promises to wait for.
   for await (const result of crawlResources(tree, baseUrl)) {
@@ -81,14 +52,6 @@ export default async function crawlBuiltin(treelike, baseHref) {
     }
   }
 
-  if (errors.length) {
-    addValueToObject(
-      cache,
-      ["crawl-errors.json"],
-      JSON.stringify(errors, null, 2)
-    );
-  }
-
   // Merge the cache on top of the resources tree. If we have an actual value
   // for something already, that's better than a function that will get that
   // value.
@@ -98,29 +61,3 @@ export default async function crawlBuiltin(treelike, baseHref) {
   );
   return result;
 }
-
-function addValueToObject(object, keys, value) {
-  for (let i = 0, current = object; i < keys.length; i++) {
-    const key = trailingSlash.remove(keys[i]);
-    if (i === keys.length - 1) {
-      // Write out value
-      if (isPlainObject(current[key])) {
-        // Route with existing values; treat the new value as an index.html
-        current[key]["index.html"] = value;
-      } else {
-        current[key] = value;
-      }
-    } else {
-      // Traverse further
-      if (!current[key]) {
-        current[key] = {};
-      } else if (!isPlainObject(current[key])) {
-        // Already have a value at this point. The site has a page at a route
-        // like /foo, and the site also has resources within that at routes like
-        // /foo/bar.jpg. We move the current value to "index.html".
-        current[key] = { "index.html": current[key] };
-      }
-      current = current[key];
-    }
-  }
-}

package/src/{site → dev}/crawler/crawlResources.js

@@ -129,28 +129,54 @@ async function processPath(tree, path, baseUrl) {
   keys = keys.map(decodeURIComponent);
 
   // Traverse tree to get value.
-  let value = await Tree.traverse(tree, ...keys);
-  const normalizedKeys = keys.slice();
-  let normalizedPath = path;
-  if (Tree.isTreelike(value)) {
-    // Path is actually a directory. See if we can get the empty string or
-    // "index.html".
-    value =
-      (await Tree.traverse(value, "")) ??
-      (await Tree.traverse(value, "index.html"));
-    if (value !== undefined) {
-      if (path.length > 0) {
-        // Mark the path as ending in a slash
-        normalizedPath = trailingSlash.add(path);
-        const key = normalizedKeys.pop();
-        normalizedKeys.push(trailingSlash.add(key));
+  let value;
+  let normalizedKeys;
+  let normalizedPath;
+  try {
+    value = await Tree.traverse(tree, ...keys);
+    normalizedKeys = keys.slice();
+    normalizedPath = path;
+    if (Tree.isTreelike(value)) {
+      // Path is actually a directory. See if we can get the empty string or
+      // "index.html".
+      value =
+        (await Tree.traverse(value, "")) ??
+        (await Tree.traverse(value, "index.html"));
+      if (value !== undefined) {
+        if (path.length > 0) {
+          // Mark the path as ending in a slash
+          normalizedPath = trailingSlash.add(path);
+          const key = normalizedKeys.pop();
+          normalizedKeys.push(trailingSlash.add(key));
+        }
+
+        // Add index.html to keys if it's not already there
+        if (normalizedKeys.at(-1) !== "index.html") {
+          normalizedKeys.push("index.html");
+        }
       }
+    }
 
-      // Add index.html to keys if it's not already there
-      if (normalizedKeys.at(-1) !== "index.html") {
-        normalizedKeys.push("index.html");
+    if (value === undefined && path.length > 0) {
+      // The path may be a URL like `foo` or `foo/` that points to `foo.html`, so
+      // we'll try looking adding `.html` to the end. We don't want to check every
+      // path twice, so we only do this if the last key does *not* include an
+      // extension.
+      const lastKey = keys.at(-1);
+      if (lastKey !== "" && !lastKey?.includes(".")) {
+        const adjustedLastKey = `${trailingSlash.remove(lastKey)}.html`;
+        const adjustedKeys = [...keys.slice(0, -1), adjustedLastKey];
+        value = await Tree.traverse(tree, ...adjustedKeys);
+        if (value !== undefined) {
+          // Page exists at foo.html
+          normalizedPath = pathFromKeys(adjustedKeys);
+          normalizedKeys = adjustedKeys;
+        }
       }
     }
+  } catch (error) {
+    // Ignore errors, return empty paths below
+    value = undefined;
   }
 
   if (value === undefined) {

package/src/dev/crawler/findPaths.js

@@ -0,0 +1,90 @@
+import { extension, toString } from "@weborigami/async-tree";
+import pathsInCss from "./pathsInCss.js";
+import pathsInHtml from "./pathsInHtml.js";
+import pathsInImageMap from "./pathsInImageMap.js";
+import pathsInJs from "./pathsInJs.js";
+import pathsInRobotsTxt from "./pathsInRobotsTxt.js";
+import pathsInSitemap from "./pathsInSitemap.js";
+
+// Filter the paths to those that are local to the site.
+function filterPaths(paths, baseUrl, localPath) {
+  // Convert paths to absolute URLs.
+  const localUrl = new URL(localPath, baseUrl);
+  const basePathname = baseUrl.pathname;
+  // @ts-ignore
+  const absoluteUrls = paths.map((path) => new URL(path, localUrl));
+
+  // Convert the absolute URLs to paths relative to the baseHref. If the URL
+  // points outside the tree rooted at the baseHref, the relative path will be
+  // null. We ignore the protocol in this test, because in practice sites often
+  // fumble the use of http and https, treating them interchangeably.
+  const relativePaths = absoluteUrls.map((url) => {
+    if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
+      const path = url.pathname.slice(basePathname.length);
+      // The process of creating the URLs will have escaped characters. We
+      // remove them. This has the side-effect of removing them if they existed
+      // in the original path; it would be better if we avoided that.
+      return decodeURIComponent(path);
+    } else {
+      return null;
+    }
+  });
+
+  // Filter out the null paths.
+  /** @type {string[]} */
+  // @ts-ignore
+  const filteredPaths = relativePaths.filter((path) => path);
+  return filteredPaths;
+}
+
+/**
+ * Given a value retrieved from a site using a given key (name), determine what
+ * kind of file it is and, based on that, find the paths it references.
+ */
+export default function findPaths(value, key, baseUrl, localPath) {
+  const text = toString(value);
+
+  // We guess the value is HTML is if its key has an .html extension or
+  // doesn't have an extension, or the value starts with `<`.
+  const ext = key ? extension.extname(key).toLowerCase() : "";
+  let foundPaths;
+  if (ext === ".html" || ext === ".htm" || ext === ".xhtml") {
+    foundPaths = pathsInHtml(text);
+  } else if (ext === ".css") {
+    foundPaths = pathsInCss(text);
+  } else if (ext === ".js") {
+    foundPaths = pathsInJs(text);
+  } else if (ext === ".map") {
+    foundPaths = pathsInImageMap(text);
+  } else if (key === "robots.txt") {
+    foundPaths = pathsInRobotsTxt(text);
+  } else if (key === "sitemap.xml") {
+    foundPaths = pathsInSitemap(text);
+  } else if (ext === "" && text?.trim().startsWith("<")) {
+    // Probably HTML
+    foundPaths = pathsInHtml(text);
+  } else {
+    // Doesn't have an extension we want to process
+    return {
+      crawlablePaths: [],
+      resourcePaths: [],
+    };
+  }
+
+  const crawlablePaths = filterPaths(
+    foundPaths.crawlablePaths,
+    baseUrl,
+    localPath
+  );
+
+  const resourcePaths = filterPaths(
+    foundPaths.resourcePaths,
+    baseUrl,
+    localPath
+  );
+
+  return {
+    crawlablePaths,
+    resourcePaths,
+  };
+}

package/src/dev/crawler/pathsInCss.js

@@ -0,0 +1,51 @@
+import { parse, walk } from "css-tree";
+import { addHref } from "./utilities.js";
+
+const imageFunctions = ["cross-fade", "image", "image-set"];
+
+export default function pathsInCss(css, context = "stylesheet") {
+  const paths = {
+    crawlablePaths: [],
+    resourcePaths: [],
+  };
+
+  let ast;
+  try {
+    ast = parse(css, { context });
+  } catch (e) {
+    // If the CSS is invalid, we can't parse it, so we can't extract paths. For
+    // now we just return no paths.
+    return paths;
+  }
+
+  if (!ast) {
+    // Unclear why parser sometimes returns an undefined AST
+    return paths;
+  }
+
+  walk(
+    ast,
+    /** @this {any} */
+    function (node) {
+      const { type, value } = node;
+      if (
+        this.atrule?.name === "import" &&
+        (type === "String" || type === "Url")
+      ) {
+        // A plain string or url() in an @import
+        addHref(paths, value, true);
+      } else if (
+        type === "String" &&
+        imageFunctions.includes(this.function?.name)
+      ) {
+        // A plain string in an cross-fade(), image(), or image-set()
+        addHref(paths, value, false);
+      } else if (type === "Url") {
+        // A url() anywhere else
+        addHref(paths, value, false);
+      }
+    }
+  );
+
+  return paths;
+}

package/src/dev/crawler/pathsInHtml.js

@@ -0,0 +1,161 @@
+import { JSDOM, VirtualConsole } from "jsdom";
+import pathsInCss from "./pathsInCss.js";
+import pathsInJs from "./pathsInJs.js";
+import { addHref } from "./utilities.js";
+
+export default function pathsInHtml(html) {
+  const paths = {
+    crawlablePaths: [],
+    resourcePaths: [],
+  };
+
+  // Create a virtual console to avoid logging errors to the console
+  const virtualConsole = new VirtualConsole();
+  const document = new JSDOM(html, { virtualConsole }).window.document;
+
+  // Find `href` attributes in anchor, area, link, SVG tags.
+  //
+  // NOTE: As of April 2024, jsdom querySelectorAll does not appear to find
+  // elements with mixed-case tag names.
+  const hrefTags = document.querySelectorAll(
+    "a[href], area[href], image[href], feImage[href], filter[href], linearGradient[href], link[href], mpath[href], pattern[href], radialGradient[href], textPath[href], use[href]"
+  );
+  for (const hrefTag of hrefTags) {
+    const crawlable = ["A", "AREA"].includes(hrefTag.tagName)
+      ? true
+      : undefined;
+    addHref(paths, hrefTag.getAttribute("href"), crawlable);
+  }
+
+  // Find `src` attributes in input, frame, media, and script tags.
+  const srcTags = document.querySelectorAll(
+    "audio[src], embed[src], frame[src], iframe[src], img[src], input[src], script[src], source[src], track[src], video[src]"
+  );
+  for (const srcTag of srcTags) {
+    const crawlable = ["FRAME", "IFRAME"].includes(srcTag.tagName)
+      ? true
+      : srcTag.tagName === "SCRIPT"
+      ? srcTag.type === "module" // Only crawl modules
+      : undefined;
+    addHref(paths, srcTag.getAttribute("src"), crawlable);
+  }
+
+  // Find `srcset` attributes in image and source tags.
+  const srcsetTags = document.querySelectorAll("img[srcset], source[srcset]");
+  for (const srcsetTag of srcsetTags) {
+    const srcset = srcsetTag.getAttribute("srcset");
+    const srcRegex = /(?<url>[^\s,]+)(?=\s+\d+(?:\.\d+)?[wxh])/g;
+    let match;
+    while ((match = srcRegex.exec(srcset))) {
+      if (match.groups?.url) {
+        addHref(paths, match.groups.url, false);
+      }
+    }
+  }
+
+  // Find `poster` attributes in <video> tags.
+  const posterTags = document.querySelectorAll("video[poster]");
+  for (const posterTag of posterTags) {
+    addHref(paths, posterTag.getAttribute("poster"), false);
+  }
+
+  // Find `data` attributes in <object> tags.
+  const objectTags = document.querySelectorAll("object[data]");
+  for (const objectTag of objectTags) {
+    addHref(paths, objectTag.getAttribute("data"), false);
+  }
+
+  // Find deprecated `background` attribute on body and table tags.
+  const backgroundTags = document.querySelectorAll(
+    "body[background], table[background], td[background], th[background]"
+  );
+  for (const backgroundTag of backgroundTags) {
+    addHref(paths, backgroundTag.getAttribute("background"), false);
+  }
+
+  // Find deprecated `longdesc` attributes on <img> tags.
+  const longdescTags = document.querySelectorAll("img[longdesc]");
+  for (const longdescTag of longdescTags) {
+    addHref(paths, longdescTag.getAttribute("longdesc"), false);
+  }
+
+  // Find paths in <meta> image tags.
+  const imageMetaTags = document.querySelectorAll('meta[property$=":image"]');
+  for (const imageMetaTag of imageMetaTags) {
+    const content = imageMetaTag.getAttribute("content");
+    if (content) {
+      addHref(paths, content, false);
+    }
+  }
+
+  // Find paths in CSS in <style> tags.
+  const styleTags = document.querySelectorAll("style");
+  for (const styleAttribute of styleTags) {
+    const cssPaths = pathsInCss(styleAttribute.textContent);
+    paths.crawlablePaths.push(...cssPaths.crawlablePaths);
+    paths.resourcePaths.push(...cssPaths.resourcePaths);
+  }
+
+  // Find URLs in CSS in `style` attributes.
+  const styleAttributeTags = document.querySelectorAll("[style]");
+  for (const tag of styleAttributeTags) {
+    const style = tag.getAttribute("style");
+    const stylePaths = pathsInCss(style, "declarationList");
+    stylePaths.resourcePaths.forEach((href) => {
+      addHref(paths, href, false);
+    });
+  }
+
+  // Find URLs in SVG attributes.
+  const svgAttributeNames = [
+    "clip-path",
+    "fill",
+    "filter",
+    "marker-end",
+    "marker-start",
+    "mask",
+    "stroke",
+  ];
+  const svgTags = document.querySelectorAll(
+    svgAttributeNames.map((name) => `[${name}]`).join(", ")
+  );
+  for (const svgTag of svgTags) {
+    for (const name of svgAttributeNames) {
+      const attributeValue = svgTag.getAttribute(name);
+      if (!attributeValue) {
+        continue;
+      }
+      const urlRegex = /url\((['"]?)(?<href>.*?)\1\)/g;
+      const attributeValueMatch = urlRegex.exec(attributeValue);
+      if (attributeValueMatch) {
+        const href = attributeValueMatch.groups?.href;
+        if (href) {
+          addHref(paths, href, false);
+        }
+      }
+    }
+  }
+
+  // Also look for JS `import` statements that might be in <script type="module"> tags.
+  const scriptTags = document.querySelectorAll("script[type='module']");
+  for (const scriptTag of scriptTags) {
+    const jsPaths = pathsInJs(scriptTag.textContent);
+    paths.crawlablePaths.push(...jsPaths.crawlablePaths);
+  }
+
+  // Special handling for <noframes> in framesets. We need to use a regex for
+  // this because the jsdom parser supports frames, so it will treat a
+  // <noframes> tag as a text node.
+  const noframesRegex = /<noframes>(?<html>[\s\S]*?)<\/noframes>/g;
+  let match;
+  while ((match = noframesRegex.exec(html))) {
+    const noframesHtml = match.groups?.html;
+    if (noframesHtml) {
+      const noframesPaths = pathsInHtml(noframesHtml);
+      paths.crawlablePaths.push(...noframesPaths.crawlablePaths);
+      paths.resourcePaths.push(...noframesPaths.resourcePaths);
+    }
+  }
+
+  return paths;
+}

package/src/dev/crawler/pathsInImageMap.js

@@ -0,0 +1,25 @@
+import { normalizeHref } from "./utilities.js";
+
+// These are ancient server-side image maps. They're so old that it's hard to
+// find documentation on them, but they're used on the reference Space Jam
+// website we use for testing the crawler.
+//
+// Example: https://www.spacejam.com/1996/bin/bball.map
+export default function pathsInImageMap(imageMap) {
+  const resourcePaths = [];
+  let match;
+
+  // Find hrefs as the second column in each line.
+  const hrefRegex = /^\w+ (?<href>\S+)(\s*$| [\d, ]+$)/gm;
+  while ((match = hrefRegex.exec(imageMap))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths: [],
+    resourcePaths,
+  };
+}

package/src/dev/crawler/pathsInJs.js

@@ -0,0 +1,140 @@
+/**
+ * Find static module references in JavaScript code.
+ *
+ * Matches:
+ *
+ * * `import … from "x"`
+ * * `import "x"`
+ * * `export … from "x"`
+ * * `export { … } from "x"`
+ *
+ * This does simple lexical analysis to avoid matching paths inside comments or
+ * string literals.
+ *
+ * @param {string} js
+ */
+export default function pathsInJs(js) {
+  return {
+    crawlablePaths: modulePaths(js),
+    resourcePaths: [],
+  };
+}
+
+function modulePaths(src) {
+  const tokens = Array.from(tokenize(src));
+  const paths = new Set();
+
+  for (let i = 0; i < tokens.length; i++) {
+    const t = tokens[i];
+
+    // static import
+    if (t.type === "Identifier" && t.value === "import") {
+      // look ahead for either:
+      //   import "mod"
+      //   import … from "mod"
+      let j = i + 1;
+      // skip any punctuation or identifiers until we hit 'from' or a StringLiteral
+      while (
+        j < tokens.length &&
+        tokens[j].type !== "StringLiteral" &&
+        !(tokens[j].type === "Identifier" && tokens[j].value === "from")
+      ) {
+        j++;
+      }
+      // import "mod"
+      if (tokens[j]?.type === "StringLiteral") {
+        paths.add(tokens[j].value);
+      } else if (
+        // import … from "mod"
+        tokens[j]?.value === "from" &&
+        tokens[j + 1]?.type === "StringLiteral"
+      ) {
+        paths.add(tokens[j + 1].value);
+      }
+    } else if (t.type === "Identifier" && t.value === "export") {
+      // re-export or export‐from
+
+      // find a 'from' token on the same statement
+      let j = i + 1;
+      while (
+        j < tokens.length &&
+        !(tokens[j].type === "Identifier" && tokens[j].value === "from")
+      ) {
+        // stop at semicolon so we don't run past the statement
+        if (tokens[j].type === "Punctuator" && tokens[j].value === ";") {
+          break;
+        }
+        j++;
+      }
+
+      if (
+        tokens[j]?.value === "from" &&
+        tokens[j + 1]?.type === "StringLiteral"
+      ) {
+        paths.add(tokens[j + 1].value);
+      }
+    }
+  }
+
+  return [...paths];
+}
+
+// Lexer emits Identifiers, StringLiterals, and Punctuators
+function* tokenize(src) {
+  let i = 0;
+  while (i < src.length) {
+    const c = src[i];
+
+    // Skip single‐line comments
+    if (c === "/" && src[i + 1] === "/") {
+      i += 2;
+      while (i < src.length && src[i] !== "\n") {
+        i++;
+      }
+    } else if (c === "/" && src[i + 1] === "*") {
+      // Skip multi‐line comments
+      i += 2;
+      while (i < src.length && !(src[i] === "*" && src[i + 1] === "/")) {
+        i++;
+      }
+      i += 2;
+      continue;
+    } else if (c === '"' || c === "'" || c === "`") {
+      // Skip string literals (but capture them)
+      const quote = c;
+      let start = i + 1;
+      i++;
+      while (i < src.length) {
+        if (src[i] === "\\") {
+          i += 2;
+          continue;
+        }
+        if (src[i] === quote) {
+          break;
+        }
+        i++;
+      }
+      const str = src.slice(start, i);
+      i++;
+      yield { type: "StringLiteral", value: str };
+      continue;
+    } else if (/[A-Za-z_$]/.test(c)) {
+      // Identifier
+      let start = i;
+      i++;
+      while (i < src.length && /[\w$]/.test(src[i])) {
+        i++;
+      }
+      yield { type: "Identifier", value: src.slice(start, i) };
+      continue;
+    } else if (/[{}();,]/.test(c)) {
+      // Punctuator (we still keep braces/semis for possible future use)
+      yield { type: "Punctuator", value: c };
+      i++;
+      continue;
+    } else {
+      // Skip everything else (whitespace, operators, etc.)
+      i++;
+    }
+  }
+}

package/src/dev/crawler/pathsInRobotsTxt.js

@@ -0,0 +1,20 @@
+import { normalizeHref } from "./utilities.js";
+
+export default function pathsInRobotsTxt(txt) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `Sitemap` directives.
+  const sitemapRegex = /Sitemap:\s*(?<href>[^\s]*)/g;
+  while ((match = sitemapRegex.exec(txt))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}

package/src/dev/crawler/pathsInSitemap.js

@@ -0,0 +1,20 @@
+import { normalizeHref } from "./utilities.js";
+
+export default function pathsInSitemap(xml) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `loc` elements.
+  const locRegex = /<loc>(?<href>[^<]*)<\/loc>/g;
+  while ((match = locRegex.exec(xml))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}

package/src/dev/crawler/utilities.js

@@ -0,0 +1,125 @@
+import {
+  extension,
+  isPlainObject,
+  trailingSlash,
+} from "@weborigami/async-tree";
+
+// A fake base URL used to handle cases where an href is relative and must be
+// treated relative to some base URL.
+const fakeBaseUrl = new URL("fake:/");
+
+/**
+ * Destructively add a path to the paths object
+ */
+export function addHref(paths, href, isCrawlable) {
+  href = normalizeHref(href);
+  if (href === null) {
+    // Normalized href is null, was just an anchor or search; skip
+    return;
+  }
+  isCrawlable ??= isCrawlableHref(href);
+  if (isCrawlable) {
+    paths.crawlablePaths.push(href);
+  } else {
+    paths.resourcePaths.push(href);
+  }
+}
+
+/**
+ * Add the value to the object at the path given by the keys
+ *
+ * @param {any} object
+ * @param {string[]} keys
+ * @param {any} value
+ */
+export function addValueToObject(object, keys, value) {
+  for (let i = 0, current = object; i < keys.length; i++) {
+    const key = trailingSlash.remove(keys[i]);
+    if (i === keys.length - 1) {
+      // Write out value
+      if (isPlainObject(current[key])) {
+        // Route with existing values; treat the new value as an index.html
+        current[key]["index.html"] = value;
+      } else {
+        current[key] = value;
+      }
+    } else {
+      // Traverse further
+      if (!current[key]) {
+        current[key] = {};
+      } else if (!isPlainObject(current[key])) {
+        // Already have a value at this point. The site has a page at a route
+        // like /foo, and the site also has resources within that at routes like
+        // /foo/bar.jpg. We move the current value to "index.html".
+        current[key] = { "index.html": current[key] };
+      }
+      current = current[key];
+    }
+  }
+}
+
+/**
+ * Determine a URL we can use to determine whether a link is local within the
+ * tree or not.
+ *
+ * If a baseHref is supplied, convert that to a URL. If it's a relative path,
+ * use a fake base URL. If no baseHref is supplied, see if the `object`
+ * parameter defines an `href` property and use that to construct a URL.
+ *
+ * @param {string|undefined} baseHref
+ * @param {any} object
+ */
+export function getBaseUrl(baseHref, object) {
+  let url;
+  if (baseHref !== undefined) {
+    // See if the href is valid
+    try {
+      url = new URL(baseHref);
+    } catch (e) {
+      // Invalid, probably a path; use a fake protocol
+      url = new URL(baseHref, fakeBaseUrl);
+    }
+  } else if (object.href) {
+    // Use href property on object
+    let href = object.href;
+    if (!href?.endsWith("/")) {
+      href += "/";
+    }
+    url = new URL(href);
+  } else {
+    url = fakeBaseUrl;
+  }
+  return url;
+}
+
+export function isCrawlableHref(href) {
+  // Use a fake base URL to cover the case where the href is relative.
+  const url = new URL(href, fakeBaseUrl);
+  const pathname = url.pathname;
+  const lastKey = pathname.split("/").pop() ?? "";
+  if (lastKey === "robots.txt" || lastKey === "sitemap.xml") {
+    return true;
+  }
+  const ext = extension.extname(lastKey);
+  // We assume an empty extension is HTML.
+  const crawlableExtensions = [".html", ".css", ".js", ".map", ".xhtml", ""];
+  return crawlableExtensions.includes(ext);
+}
+
+// Remove any search parameters or hash from the href. Preserve absolute or
+// relative nature of URL. If the URL only has a search or hash, return null.
+export function normalizeHref(href) {
+  // Remove everything after a `#` or `?` character.
+  const normalized = href.split(/[?#]/)[0];
+  return normalized === "" ? null : normalized;
+}
+
+// For indexing and storage purposes, treat a path that ends in a trailing slash
+// as if it ends in index.html.
+export function normalizeKeys(keys) {
+  const normalized = keys.slice();
+  if (normalized.length === 0 || trailingSlash.has(normalized.at(-1))) {
+    normalized.push("index.html");
+  }
+  return normalized;
+}

package/src/dev/dev.js

@@ -1,6 +1,8 @@
 export { default as breakpoint } from "./breakpoint.js";
 export { default as changes } from "./changes.js";
 export { default as code } from "./code.js";
+export { default as audit } from "./crawler/audit.js";
+export { default as crawl } from "./crawler/crawl.js";
 export { default as debug } from "./debug.js";
 export { default as explore } from "./explore.js";
 export { default as log } from "./log.js";

package/src/handlers/handlers.js

@@ -14,6 +14,7 @@ import jpgHandler from "./jpg.handler.js";
 import jsonHandler from "./json.handler.js";
 import mdHandler from "./md.handler.js";
 import mjsHandler from "./mjs.handler.js";
+import tsHandler from "./ts.handler.js";
 import txtHandler from "./txt.handler.js";
 import xhtmlHandler from "./xhtml.handler.js";
 import ymlHandler from "./yml.handler.js";
@@ -31,6 +32,7 @@ export default {
   "mjs.handler": mjsHandler,
   "ori.handler": oriHandler,
   "oridocument.handler": oridocumentHandler,
+  "ts.handler": tsHandler,
   "txt.handler": txtHandler,
   "wasm.handler": wasmHandler,
   "xhtml.handler": xhtmlHandler,

package/src/handlers/ts.handler.js

@@ -0,0 +1 @@
+export { default as default } from "./js.handler.js";

package/src/help/help.yaml

@@ -1,12 +1,18 @@
 dev:
   description: Develop and debug Origami projects
   commands:
+    audit:
+      args: (tree)
+      description: Identify broken internal links and references
     breakpoint:
       args: (a)
       description: Break into the JavaScript debugger, then return a
     changes:
       args: (old, new)
       description: Return a tree of changes
+    crawl:
+      args: (tree, base)
+      description: A tree of a site's discoverable resources
     debug:
       args: (tree)
       description: Add debug features to the tree
@@ -213,12 +219,6 @@ scope:
 site:
   description: Add common website features
   commands:
-    audit:
-      args: (tree)
-      description: Identify broken internal links and references
-    crawl:
-      args: (tree, base)
-      description: A tree of a site's discoverable resources
     index:
       args: (tree)
       description: A default index.html page for the tree

package/src/site/site.js

@@ -1,5 +1,3 @@
-export { default as audit } from "./audit.js";
-export { default as crawl } from "./crawler/crawl.js";
 export { default as index } from "./index.js";
 export { default as jsonKeys } from "./jsonKeys.js";
 export { default as redirect } from "./redirect.js";

package/src/text/htmlDom.js

@@ -0,0 +1,6 @@
+import { JSDOM } from "jsdom";
+
+export default function htmlDom(html) {
+  const dom = JSDOM.fragment(html);
+  return dom;
+}

package/src/text/text.js

@@ -1,4 +1,5 @@
 export { taggedTemplateIndent as indent } from "@weborigami/language";
 export { default as document } from "./document.js";
+export { default as htmlDom } from "./htmlDom.js";
 export { default as inline } from "./inline.js";
 export { default as mdHtml } from "./mdHtml.js";

package/src/site/audit.js

@@ -1,19 +0,0 @@
-import { Tree } from "@weborigami/async-tree";
-import getTreeArgument from "../common/getTreeArgument.js";
-import crawl from "./crawler/crawl.js";
-
-/**
- * @this {import("@weborigami/types").AsyncTree|null}
- * @param {import("@weborigami/async-tree").Treelike} treelike
- */
-export default async function audit(treelike) {
-  const tree = await getTreeArgument(this, arguments, treelike, "site:audit");
-  const crawled = await crawl.call(this, tree);
-  let crawlErrorsJson = await crawled.get("crawl-errors.json");
-  if (!crawlErrorsJson) {
-    return undefined;
-  }
-  const errors = Tree.from(JSON.parse(crawlErrorsJson), { deep: true });
-  errors.parent = this;
-  return errors;
-}

package/src/site/crawler/findPaths.js

@@ -1,266 +0,0 @@
-import { extension, toString } from "@weborigami/async-tree";
-import { isCrawlableHref, normalizeHref } from "./utilities.js";
-
-// Filter the paths to those that are local to the site.
-function filterPaths(paths, baseUrl, localPath) {
-  // Convert paths to absolute URLs.
-  const localUrl = new URL(localPath, baseUrl);
-  const basePathname = baseUrl.pathname;
-  // @ts-ignore
-  const absoluteUrls = paths.map((path) => new URL(path, localUrl));
-
-  // Convert the absolute URLs to paths relative to the baseHref. If the URL
-  // points outside the tree rooted at the baseHref, the relative path will be
-  // null. We ignore the protocol in this test, because in practice sites often
-  // fumble the use of http and https, treating them interchangeably.
-  const relativePaths = absoluteUrls.map((url) => {
-    if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
-      const path = url.pathname.slice(basePathname.length);
-      // The process of creating the URLs will have escaped characters. We
-      // remove them. This has the side-effect of removing them if they existed
-      // in the original path; it would be better if we avoided that.
-      return decodeURIComponent(path);
-    } else {
-      return null;
-    }
-  });
-
-  // Filter out the null paths.
-  /** @type {string[]} */
-  // @ts-ignore
-  const filteredPaths = relativePaths.filter((path) => path);
-  return filteredPaths;
-}
-
-/**
- * Given a value retrieved from a site using a given key (name), determine what
- * kind of file it is and, based on that, find the paths it references.
- */
-export default function findPaths(value, key, baseUrl, localPath) {
-  const text = toString(value);
-
-  // We guess the value is HTML is if its key has an .html extension or
-  // doesn't have an extension, or the value starts with `<`.
-  const ext = key ? extension.extname(key).toLowerCase() : "";
-  let foundPaths;
-  if (ext === ".html" || ext === ".htm" || ext === ".xhtml") {
-    foundPaths = findPathsInHtml(text);
-  } else if (ext === ".css") {
-    foundPaths = findPathsInCss(text);
-  } else if (ext === ".js") {
-    foundPaths = findPathsInJs(text);
-  } else if (ext === ".map") {
-    foundPaths = findPathsInImageMap(text);
-  } else if (key === "robots.txt") {
-    foundPaths = findPathsInRobotsTxt(text);
-  } else if (key === "sitemap.xml") {
-    foundPaths = findPathsInSitemapXml(text);
-  } else if (ext === "" && text?.trim().startsWith("<")) {
-    // Probably HTML
-    foundPaths = findPathsInHtml(text);
-  } else {
-    // Doesn't have an extension we want to process
-    return {
-      crawlablePaths: [],
-      resourcePaths: [],
-    };
-  }
-
-  const crawlablePaths = filterPaths(
-    foundPaths.crawlablePaths,
-    baseUrl,
-    localPath
-  );
-
-  const resourcePaths = filterPaths(
-    foundPaths.resourcePaths,
-    baseUrl,
-    localPath
-  );
-
-  return {
-    crawlablePaths,
-    resourcePaths,
-  };
-}
-
-function findPathsInCss(css) {
-  const resourcePaths = [];
-  let match;
-
-  // Find `url()` functions.
-  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
-  while ((match = urlRegex.exec(css))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths: [],
-    resourcePaths,
-  };
-}
-
-// These are ancient server-side image maps. They're so old that it's hard to
-// find documentation on them, but they're used on the reference Space Jam
-// website we use for testing the crawler. Example:
-// https://www.spacejam.com/1996/bin/bball.map
-function findPathsInImageMap(imageMap) {
-  const resourcePaths = [];
-  let match;
-
-  // Find hrefs as the second column in each line.
-  const hrefRegex = /^\w+ (?<href>\S+)(\s*$| [\d, ]+$)/gm;
-  while ((match = hrefRegex.exec(imageMap))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths: [],
-    resourcePaths,
-  };
-}
-
-function findPathsInJs(js) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `import` statements.
-  const importRegex = /import [\s\S]+?from\s+["'](?<import>[^"']*)["'];/g;
-  while ((match = importRegex.exec(js))) {
-    const href = normalizeHref(match.groups?.import);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function findPathsInHtml(html) {
-  const crawlablePaths = [];
-  const resourcePaths = [];
-  let match;
-
-  // Find `href` attributes in anchor and link tags.
-  const linkRegex =
-    /<(?:a|A|link|LINK)[\s][^>]*?(?:href|HREF)=["'](?<link>[^>]*?)["'][^>]*>/g;
-  while ((match = linkRegex.exec(html))) {
-    // Links can point to be other crawlable paths and resource paths.
-    // We guess the type based on the extension.
-    const href = normalizeHref(match.groups?.link);
-    if (href) {
-      if (isCrawlableHref(href)) {
-        crawlablePaths.push(href);
-      } else {
-        resourcePaths.push(href);
-      }
-    }
-  }
-
-  // Find `src` attributes in img and script tags.
-  const srcRegex =
-    /<(?<tag>img|IMG|script|SCRIPT)[\s][^>]*?(?:src|SRC)=["'](?<src>[^>]*?)["'][^>]*>/g;
-  while ((match = srcRegex.exec(html))) {
-    const tag = match.groups?.tag;
-    const src = normalizeHref(match.groups?.src);
-    if (src) {
-      if (tag === "script" || tag === "SCRIPT") {
-        crawlablePaths.push(src);
-      } else {
-        resourcePaths.push(src);
-      }
-    }
-  }
-
-  // Find `url()` functions in CSS.
-  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
-  while ((match = urlRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  // Find `src` attribute on frame tags.
-  const frameRegex =
-    /<(?:frame|FRAME)[\s][^>]*?(?:src|SRC)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = frameRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  // Find ancient `background` attribute on body tag.
-  const backgroundRegex =
-    /<(?:body|BODY)[\s][^>]*?(?:background|BACKGROUND)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = backgroundRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  // Find `href` attribute on area tags.
-  const areaRegex =
-    /<(?:area|AREA)[\s][^>]*?(?:href|HREF)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = areaRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  // Also look for JS `import` statements that might be in <script type="module"> tags.
-  const jsResults = findPathsInJs(html);
-  crawlablePaths.push(...jsResults.crawlablePaths);
-
-  return { crawlablePaths, resourcePaths };
-}
-
-function findPathsInRobotsTxt(txt) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `Sitemap` directives.
-  const sitemapRegex = /Sitemap:\s*(?<href>[^\s]*)/g;
-  while ((match = sitemapRegex.exec(txt))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function findPathsInSitemapXml(xml) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `loc` elements.
-  const locRegex = /<loc>(?<href>[^<]*)<\/loc>/g;
-  while ((match = locRegex.exec(xml))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}

package/src/site/crawler/utilities.js

@@ -1,37 +0,0 @@
-import { extension, trailingSlash } from "@weborigami/async-tree";
-
-// A fake base URL used to handle cases where an href is relative and must be
-// treated relative to some base URL.
-const fakeBaseUrl = new URL("https://fake");
-
-export function isCrawlableHref(href) {
-  // Use a fake base URL to cover the case where the href is relative.
-  const url = new URL(href, fakeBaseUrl);
-  const pathname = url.pathname;
-  const lastKey = pathname.split("/").pop() ?? "";
-  if (lastKey === "robots.txt" || lastKey === "sitemap.xml") {
-    return true;
-  }
-  const ext = extension.extname(lastKey);
-  // We assume an empty extension is HTML.
-  const crawlableExtensions = [".html", ".css", ".js", ".map", ".xhtml", ""];
-  return crawlableExtensions.includes(ext);
-}
-
-// Remove any search parameters or hash from the href. Preserve absolute or
-// relative nature of URL. If the URL only has a search or hash, return null.
-export function normalizeHref(href) {
-  // Remove everything after a `#` or `?` character.
-  const normalized = href.split(/[?#]/)[0];
-  return normalized === "" ? null : normalized;
-}
-
-// For indexing and storage purposes, treat a path that ends in a trailing slash
-// as if it ends in index.html.
-export function normalizeKeys(keys) {
-  const normalized = keys.slice();
-  if (normalized.length === 0 || trailingSlash.has(normalized.at(-1))) {
-    normalized.push("index.html");
-  }
-  return normalized;
-}