@weborigami/origami 0.0.67-beta.2 → 0.0.69

package/exports/exports.js

@@ -86,6 +86,7 @@ export { default as serve } from "../src/builtins/@serve.js";
 export { default as setDeep } from "../src/builtins/@setDeep.js";
 export { default as shell } from "../src/builtins/@shell.js";
 export { default as shuffle } from "../src/builtins/@shuffle.js";
+export { default as siteAudit } from "../src/builtins/@siteAudit.js";
 export { default as sitemap } from "../src/builtins/@sitemap.js";
 export * from "../src/builtins/@slash.js";
 export { default as slug } from "../src/builtins/@slug.js";
@@ -136,6 +137,9 @@ export { default as processUnpackedContent } from "../src/common/processUnpacked
 export * from "../src/common/serialize.js";
 export { default as ShuffleTransform } from "../src/common/ShuffleTransform.js";
 export * from "../src/common/utilities.js";
+export { default as crawlResources } from "../src/crawler/crawlResources.js";
+export { default as findPaths } from "../src/crawler/findPaths.js";
+export * from "../src/crawler/utilities.js";
 export { default as assertTreeIsDefined } from "../src/misc/assertTreeIsDefined.js";
 export { default as getTreeArgument } from "../src/misc/getTreeArgument.js";
 export { default as OriCommandTransform } from "../src/misc/OriCommandTransform.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@weborigami/origami",
-  "version": "0.0.67-beta.2",
+  "version": "0.0.69",
   "description": "Web Origami language, CLI, framework, and server",
   "type": "module",
   "repository": {
@@ -17,9 +17,9 @@
     "typescript": "5.6.2"
   },
   "dependencies": {
-    "@weborigami/async-tree": "0.0.67-beta.2",
-    "@weborigami/language": "0.0.67-beta.2",
-    "@weborigami/types": "0.0.67-beta.2",
+    "@weborigami/async-tree": "0.0.69",
+    "@weborigami/language": "0.0.69",
+    "@weborigami/types": "0.0.69",
     "exif-parser": "0.1.12",
     "graphviz-wasm": "3.0.2",
     "highlight.js": "11.10.0",

package/src/builtins/@changes.js

@@ -1,4 +1,4 @@
-import { Tree } from "@weborigami/async-tree";
+import { trailingSlash, Tree } from "@weborigami/async-tree";
 
 /**
  * Given an old tree and a new tree, return a tree of changes indicated
@@ -17,36 +17,46 @@ export default async function changes(oldTreelike, newTreelike) {
   const oldKeys = Array.from(await oldTree.keys());
   const newKeys = Array.from(await newTree.keys());
 
-  const result = {};
+  const oldKeysNormalized = oldKeys.map(trailingSlash.remove);
+  const newKeysNormalized = newKeys.map(trailingSlash.remove);
 
-  for (const key of oldKeys) {
-    if (!newKeys.includes(key)) {
-      result[key] = "deleted";
+  let result;
+
+  for (const oldKey of oldKeys) {
+    const oldNormalized = trailingSlash.remove(oldKey);
+    if (!newKeysNormalized.includes(oldNormalized)) {
+      result ??= {};
+      result[oldKey] = "deleted";
       continue;
     }
 
-    const oldValue = await oldTree.get(key);
-    const newValue = await newTree.get(key);
+    const oldValue = await oldTree.get(oldKey);
+    const newValue = await newTree.get(oldKey);
 
     if (Tree.isAsyncTree(oldValue) && Tree.isAsyncTree(newValue)) {
       const treeChanges = await changes.call(this, oldValue, newValue);
-      if (Object.keys(treeChanges).length > 0) {
-        result[key] = treeChanges;
+      if (treeChanges && Object.keys(treeChanges).length > 0) {
+        result ??= {};
+        result[oldKey] = treeChanges;
       }
     } else if (oldValue?.toString && newValue?.toString) {
       const oldText = oldValue.toString();
       const newText = newValue.toString();
       if (oldText !== newText) {
-        result[key] = "changed";
+        result ??= {};
+        result[oldKey] = "changed";
       }
     } else {
-      result[key] = "changed";
+      result ??= {};
+      result[oldKey] = "changed";
     }
   }
 
-  for (const key of newKeys) {
-    if (!oldKeys.includes(key)) {
-      result[key] = "added";
+  for (const newKey of newKeys) {
+    const newNormalized = trailingSlash.remove(newKey);
+    if (!oldKeysNormalized.includes(newNormalized)) {
+      result ??= {};
+      result[newKey] = "added";
     }
   }
 

package/src/builtins/@crawl.js

@@ -6,14 +6,10 @@ import {
   keysFromPath,
   trailingSlash,
 } from "@weborigami/async-tree";
-import { InvokeFunctionsTransform, extname } from "@weborigami/language";
-import * as utilities from "../common/utilities.js";
-import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
-import treeHttps from "./@treeHttps.js";
-
-// A fake base URL used to handle cases where an href is relative and must be
-// treated relative to some base URL.
-const fakeBaseUrl = new URL("https://fake");
+import { InvokeFunctionsTransform } from "@weborigami/language";
+import crawlResources from "../crawler/crawlResources.js";
+import { normalizeKeys } from "../crawler/utilities.js";
+import getTreeArgument from "../misc/getTreeArgument.js";
 
 /**
  * Crawl a tree, starting its root index.html page, and following links to
@@ -24,23 +20,19 @@ const fakeBaseUrl = new URL("https://fake");
  * that obtain the requested value from the original site.
  *
  * @typedef  {import("@weborigami/types").AsyncTree} AsyncTree
- * @typedef {import("@weborigami/async-tree").Treelike|string} Treelike
+ * @typedef {import("@weborigami/async-tree").Treelike} Treelike
  * @this {AsyncTree|null}
  * @param {Treelike} treelike
  * @param {string} [baseHref]
  * @returns {Promise<AsyncTree>}
  */
-export default async function crawl(treelike, baseHref) {
-  assertTreeIsDefined(this, "crawl");
-  const tree =
-    typeof treelike === "string"
-      ? treeHttps.call(this, treelike)
-      : Tree.from(treelike, { parent: this });
+export default async function crawlBuiltin(treelike, baseHref) {
+  const tree = await getTreeArgument(this, arguments, treelike, "@crawl");
 
   if (baseHref === undefined) {
     // Ask tree or original treelike if it has an `href` property we can use as
     // the base href to determine whether a link is local within the tree or
-    // not. If not, use a fake `local:/` href.
+    // not. If not, use a fake `local:/` base href.
     baseHref =
       /** @type {any} */ (tree).href ??
       /** @type {any} */ (treelike).href ??
@@ -48,7 +40,21 @@ export default async function crawl(treelike, baseHref) {
     if (!baseHref?.endsWith("/")) {
       baseHref += "/";
     }
+  } else {
+    // Is the href already valid?
+    let isHrefValid = false;
+    try {
+      new URL(baseHref);
+      isHrefValid = true;
+    } catch (e) {
+      // Ignore
+    }
+    if (!isHrefValid) {
+      // Use a fake base href.
+      baseHref = `local:/${baseHref}`;
+    }
   }
+
   // @ts-ignore
   const baseUrl = new URL(baseHref);
 
@@ -57,24 +63,18 @@ export default async function crawl(treelike, baseHref) {
   const errors = [];
 
   // We iterate until there are no more promises to wait for.
-  for await (const result of crawlPaths(tree, baseUrl)) {
-    const { keys, resourcePaths, value } = result;
+  for await (const result of crawlResources(tree, baseUrl)) {
+    const { normalizedKeys, resourcePaths, value } = result;
 
     // Cache the value
     if (value) {
-      addValueToObject(cache, keys, value);
-    } else if (keys) {
-      // A missing robots.txt isn't an error; anything else missing is.
-      const path = keys.join("/");
-      if (path !== "robots.txt") {
-        errors.push(path);
-      }
+      addValueToObject(cache, normalizedKeys, value);
     }
 
     // Add indirect resource functions to the resource tree. When requested,
     // these functions will obtain the resource from the original site.
     for (const resourcePath of resourcePaths) {
-      const resourceKeys = adjustKeys(keysFromPath(resourcePath));
+      const resourceKeys = normalizeKeys(keysFromPath(resourcePath));
       const fn = () => {
         return Tree.traverse(tree, ...resourceKeys);
       };
@@ -100,17 +100,6 @@ export default async function crawl(treelike, baseHref) {
   return result;
 }
 
-// For indexing and storage purposes, treat a path that ends in a trailing slash
-// as if it ends in index.html.
-function adjustKeys(keys) {
-  if (keys.length > 0 && !trailingSlash.has(keys.at(-1))) {
-    return keys;
-  }
-  const adjustedKeys = keys.slice();
-  adjustedKeys.push("index.html");
-  return adjustedKeys;
-}
-
 function addValueToObject(object, keys, value) {
   for (let i = 0, current = object; i < keys.length; i++) {
     const key = trailingSlash.remove(keys[i]);
@@ -127,10 +116,9 @@ function addValueToObject(object, keys, value) {
       if (!current[key]) {
         current[key] = {};
       } else if (!isPlainObject(current[key])) {
-        // Already have a value at this point. The site has a page
-        // at a route like /foo, and the site also has resources
-        // within that at routes like /foo/bar.jpg. We move the
-        // current value to "index.html".
+        // Already have a value at this point. The site has a page at a route
+        // like /foo, and the site also has resources within that at routes like
+        // /foo/bar.jpg. We move the current value to "index.html".
         current[key] = { "index.html": current[key] };
       }
       current = current[key];
@@ -138,376 +126,5 @@ function addValueToObject(object, keys, value) {
   }
 }
 
-// Crawl the paths for the given tree, starting at the given base URL, and
-// yield the results. The results will include the HTML/script/stylesheet value
-// retrieved at a path, along with the paths to other resources found in that
-// text.
-async function* crawlPaths(tree, baseUrl) {
-  // We want to kick off requests for new paths as quickly as we find them, then
-  // yield whichever result finishes first. Unfortunately, Promise.any() only
-  // tells us the result of the first promise to resolve, not which promise that
-  // was. So we keep track of a dictionary mapping paths to a promise for the
-  // value at that path. When a promise resolves, we mark it as resolved by
-  // setting its entry in the dictionary to null.
-  const promisesForPaths = {};
-
-  // Seed the promise dictionary with robots.txt and the root path.
-  const initialPaths = ["/robots.txt", "/"];
-  initialPaths.forEach((path) => {
-    promisesForPaths[path] = processPath(tree, path, baseUrl);
-  });
-
-  while (true) {
-    // Get the latest array of promises that haven't been resolved yet.
-    const promises = Object.values(promisesForPaths).filter(
-      (promise) => promise !== null
-    );
-
-    if (promises.length === 0) {
-      // No unresolved promises; we're done.
-      break;
-    }
-
-    // Wait for the first promise to resolve.
-    const result = await Promise.any(promises);
-
-    // Mark the promise for that result as resolved.
-    promisesForPaths[result.path] = null;
-
-    // Add promises for crawlable paths in the result.
-    result.crawlablePaths.forEach((path) => {
-      // Only add a promise for this path if we don't already have one.
-      if (promisesForPaths[path] === undefined) {
-        promisesForPaths[path] = processPath(tree, path, baseUrl);
-      }
-    });
-
-    yield result;
-  }
-}
-
-// Filter the paths to those that are local to the site.
-function filterPaths(paths, baseUrl, localPath) {
-  // Convert paths to absolute URLs.
-  const localUrl = new URL(localPath, baseUrl);
-  const basePathname = baseUrl.pathname;
-  // @ts-ignore
-  const absoluteUrls = paths.map((path) => new URL(path, localUrl));
-
-  // Convert the absolute URLs to paths relative to the baseHref. If the URL
-  // points outside the tree rooted at the baseHref, the relative path will be
-  // null. We ignore the protocol in this test, because in practice sites often
-  // fumble the use of http and https, treating them interchangeably.
-  const relativePaths = absoluteUrls.map((url) => {
-    if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
-      return url.pathname.slice(basePathname.length);
-    } else {
-      return null;
-    }
-  });
-
-  // Filter out the null paths.
-  /** @type {string[]} */
-  // @ts-ignore
-  const filteredPaths = relativePaths.filter((path) => path);
-  return filteredPaths;
-}
-
-function findPaths(value, key, baseUrl, localPath) {
-  const text = utilities.toString(value);
-
-  // We guess the value is HTML is if its key has an .html extension or
-  // doesn't have an extension, or the value starts with `<`.
-  const ext = key ? extname(key).toLowerCase() : "";
-  const maybeHtml = ext === "" || text?.trim().startsWith("<");
-  let foundPaths;
-  if (ext === ".html" || ext === ".htm") {
-    foundPaths = findPathsInHtml(text);
-  } else if (ext === ".css") {
-    foundPaths = findPathsInCss(text);
-  } else if (ext === ".js") {
-    foundPaths = findPathsInJs(text);
-  } else if (ext === ".map") {
-    foundPaths = findPathsInImageMap(text);
-  } else if (key === "robots.txt") {
-    foundPaths = findPathsInRobotsTxt(text);
-  } else if (key === "sitemap.xml") {
-    foundPaths = findPathsInSitemapXml(text);
-  } else if (maybeHtml) {
-    foundPaths = findPathsInHtml(text);
-  } else {
-    // Doesn't have an extension we want to process
-    return {
-      crawlablePaths: [],
-      resourcePaths: [],
-    };
-  }
-
-  const crawlablePaths = filterPaths(
-    foundPaths.crawlablePaths,
-    baseUrl,
-    localPath
-  );
-  const resourcePaths = filterPaths(
-    foundPaths.resourcePaths,
-    baseUrl,
-    localPath
-  );
-  return {
-    crawlablePaths,
-    resourcePaths,
-  };
-}
-
-function findPathsInCss(css) {
-  const resourcePaths = [];
-  let match;
-
-  // Find `url()` functions.
-  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
-  while ((match = urlRegex.exec(css))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push();
-    }
-  }
-
-  return {
-    crawlablePaths: [],
-    resourcePaths,
-  };
-}
-
-// These are ancient server-side image maps. They're so old that it's hard to
-// find documentation on them, but they're used on the reference Space Jam
-// website we use for testing the crawler. Example:
-// https://www.spacejam.com/1996/bin/bball.map
-function findPathsInImageMap(imageMap) {
-  const resourcePaths = [];
-  let match;
-
-  // Find hrefs as the second column in each line.
-  const hrefRegex = /^\w+ (?<href>\S+)(\s*$| [\d, ]+$)/gm;
-  while ((match = hrefRegex.exec(imageMap))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths: [],
-    resourcePaths,
-  };
-}
-
-function findPathsInJs(js) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `import` statements.
-  const importRegex = /import [\s\S]+?from\s+["'](?<import>[^"']*)["'];/g;
-  while ((match = importRegex.exec(js))) {
-    const href = normalizeHref(match.groups?.import);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function findPathsInHtml(html) {
-  const crawlablePaths = [];
-  const resourcePaths = [];
-  let match;
-
-  // Find `href` attributes in anchor and link tags.
-  const linkRegex =
-    /<(?:a|A|link|LINK) [^>]*?(?:href|HREF)=["'](?<link>[^>]*?)["'][^>]*>/g;
-  while ((match = linkRegex.exec(html))) {
-    // Links can point to be other crawlable paths and resource paths.
-    // We guess the type based on the extension.
-    const href = normalizeHref(match.groups?.link);
-    if (href) {
-      if (isCrawlableHref(href)) {
-        crawlablePaths.push(href);
-      } else {
-        resourcePaths.push(href);
-      }
-    }
-  }
-
-  // Find `src` attributes in img and script tags.
-  const srcRegex =
-    /<(?<tag>img|IMG|script|SCRIPT) [^>]*?(?:src|SRC)=["'](?<src>[^>]*?)["'][^>]*>/g;
-  while ((match = srcRegex.exec(html))) {
-    const tag = match.groups?.tag;
-    const src = normalizeHref(match.groups?.src);
-    if (src) {
-      if (tag === "script" || tag === "SCRIPT") {
-        crawlablePaths.push(src);
-      } else {
-        resourcePaths.push(src);
-      }
-    }
-  }
-
-  // Find `url()` functions in CSS.
-  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
-  while ((match = urlRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  // Find `src` attribute on frame tags.
-  const frameRegex =
-    /<(?:frame|FRAME) [^>]*?(?:src|SRC)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = frameRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  // Find ancient `background` attribute on body tag.
-  const backgroundRegex =
-    /<(?:body|BODY) [^>]*?(?:background|BACKGROUND)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = backgroundRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  // Find `href` attribute on area tags.
-  const areaRegex =
-    /<(?:area|AREA) [^>]*?(?:href|HREF)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = areaRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return { crawlablePaths, resourcePaths };
-}
-
-function findPathsInRobotsTxt(txt) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `Sitemap` directives.
-  const sitemapRegex = /Sitemap:\s*(?<href>[^\s]*)/g;
-  while ((match = sitemapRegex.exec(txt))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function findPathsInSitemapXml(xml) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `loc` elements.
-  const locRegex = /<loc>(?<href>[^<]*)<\/loc>/g;
-  while ((match = locRegex.exec(xml))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function isCrawlableHref(href) {
-  // Use a fake base URL to cover the case where the href is relative.
-  const url = new URL(href, fakeBaseUrl);
-  const pathname = url.pathname;
-  const lastKey = pathname.split("/").pop() ?? "";
-  if (lastKey === "robots.txt" || lastKey === "sitemap.xml") {
-    return true;
-  }
-  const ext = extname(lastKey);
-  // We assume an empty extension is HTML.
-  const crawlableExtensions = [".html", ".css", ".js", ".map", ""];
-  return crawlableExtensions.includes(ext);
-}
-
-// Remove any search parameters or hash from the href. Preserve absolute or
-// relative nature of URL. If the URL only has a search or hash, return null.
-function normalizeHref(href) {
-  // Remove everything after a `#` or `?` character.
-  const normalized = href.split(/[?#]/)[0];
-  return normalized === "" ? null : normalized;
-}
-
-async function processPath(tree, path, baseUrl) {
-  if (path === undefined) {
-    return {
-      crawlablePaths: [],
-      keys: null,
-      path,
-      resourcePaths: [],
-      value: null,
-    };
-  }
-
-  // Convert path to keys
-  const keys = keysFromPath(path);
-
-  // Traverse tree to get value.
-  let value = await Tree.traverse(tree, ...keys);
-  if (Tree.isAsyncTree(value)) {
-    // Path is actually a directory; see if it has an index.html
-    value = await Tree.traverse(value, "index.html");
-  }
-
-  const adjustedKeys = adjustKeys(keys);
-
-  if (value === undefined) {
-    return {
-      crawlablePaths: [],
-      keys: adjustedKeys,
-      path,
-      resourcePaths: [],
-      value: null,
-    };
-  }
-
-  // Find paths in the value
-  const key = adjustedKeys.at(-1);
-  const { crawlablePaths, resourcePaths } = await findPaths(
-    value,
-    key,
-    baseUrl,
-    path
-  );
-
-  return {
-    crawlablePaths,
-    keys: adjustedKeys,
-    path,
-    resourcePaths,
-    value,
-  };
-}
-
-crawl.usage = `@crawl <tree>\tCrawl a tree`;
-crawl.documentation = "https://weborigami.org/language/@crawl.html";
+crawlBuiltin.usage = `@crawl <tree>\tCrawl a tree`;
+crawlBuiltin.documentation = "https://weborigami.org/language/@crawl.html";

package/src/builtins/@keysTree.js

@@ -8,7 +8,7 @@ import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
  *
  * @this {AsyncTree|null}
  * @param {string} host
- * @param  {...string|Symbol} keys
+ * @param  {...string} keys
  */
 export default function keysTree(host, ...keys) {
   assertTreeIsDefined(this, "keysTree");

package/src/builtins/@shuffle.js

@@ -1,4 +1,7 @@
-import ShuffleTransform from "../common/ShuffleTransform.js";
+import {
+  default as ShuffleTransform,
+  shuffle,
+} from "../common/ShuffleTransform.js";
 import { transformObject } from "../common/utilities.js";
 import getTreeArgument from "../misc/getTreeArgument.js";
 
@@ -11,10 +14,21 @@ import getTreeArgument from "../misc/getTreeArgument.js";
  * @this {AsyncTree|null}
  * @param {Treelike} [treelike]
  */
-export default async function shuffle(treelike) {
+export default async function shuffleTree(treelike) {
+  // Special case: If the treelike is an array, shuffle it directly. Otherwise
+  // we'll end up shuffling the array's indexes, and if this is directly
+  // displayed by the ori CLI, this will end up creating a plain object. Even
+  // though this object will be created with the keys in the correct shuffled
+  // order, a JS object will always return numeric keys in numeric order --
+  // undoing the shuffle.
+  if (Array.isArray(treelike)) {
+    const array = treelike.slice();
+    shuffle(array);
+    return array;
+  }
   const tree = await getTreeArgument(this, arguments, treelike, "@shuffle");
   return transformObject(ShuffleTransform, tree);
 }
 
-shuffle.usage = `@shuffle <tree>\tReturn a new tree with the original's keys shuffled`;
-shuffle.documentation = "https://weborigami.org/cli/builtins.html#shuffle";
+shuffleTree.usage = `@shuffle <tree>\tReturn a new tree with the original's keys shuffled`;
+shuffleTree.documentation = "https://weborigami.org/cli/builtins.html#shuffle";

package/src/builtins/@siteAudit.js

@@ -0,0 +1,19 @@
+import { Tree } from "@weborigami/async-tree";
+import getTreeArgument from "../misc/getTreeArgument.js";
+import crawl from "./@crawl.js";
+
+/**
+ * @this {import("@weborigami/types").AsyncTree|null}
+ * @param {import("@weborigami/async-tree").Treelike} treelike
+ */
+export default async function siteAudit(treelike) {
+  const tree = await getTreeArgument(this, arguments, treelike, "@siteAudit");
+  const crawled = await crawl.call(this, tree);
+  let crawlErrorsJson = await crawled.get("crawl-errors.json");
+  if (!crawlErrorsJson) {
+    return undefined;
+  }
+  const errors = Tree.from(JSON.parse(crawlErrorsJson), { deep: true });
+  errors.parent = this;
+  return errors;
+}

package/src/builtins/@slug.js

@@ -11,5 +11,8 @@ export default function slug(filename) {
   slug = slug.replace(/^-+/, "");
   slug = slug.replace(/-+$/, "");
 
+  // Collapse consecutive dashes to a single dash.
+  slug = slug.replace(/-+/g, "-");
+
   return slug;
 }

package/src/builtins/@treeHttp.js

@@ -8,7 +8,7 @@ import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
  *
  * @this {AsyncTree|null}
  * @param {string} host
- * @param  {...string|Symbol} keys
+ * @param  {...string} keys
  */
 export default function treeHttp(host, ...keys) {
   assertTreeIsDefined(this, "treeHttp");

package/src/builtins/@treeHttps.js

@@ -8,7 +8,7 @@ import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
  *
  * @this {AsyncTree|null}
  * @param {string} host
- * @param  {...string|Symbol} keys
+ * @param  {...string} keys
  */
 export default function treeHttps(host, ...keys) {
   assertTreeIsDefined(this, "treeHttps");

package/src/common/ShuffleTransform.js

@@ -18,7 +18,7 @@ export default function ShuffleTransform(Base) {
  *
  * Performs a Fisher-Yates shuffle. From http://sedition.com/perl/javascript-fy.html
  */
-function shuffle(array) {
+export function shuffle(array) {
   let i = array.length;
   while (--i >= 0) {
     const j = Math.floor(Math.random() * (i + 1));

package/src/crawler/crawlResources.js

@@ -0,0 +1,179 @@
+import {
+  keysFromPath,
+  pathFromKeys,
+  trailingSlash,
+  Tree,
+} from "@weborigami/async-tree";
+import findPaths from "./findPaths.js";
+import { normalizeKeys } from "./utilities.js";
+
+/**
+ * Crawl the paths for the given tree, starting at the given base URL, and yield
+ * the crawled resources.
+ *
+ * Each will include the HTML/script/stylesheet value retrieved at a given path.
+ */
+export default async function* crawlResources(tree, baseUrl) {
+  // We want to kick off requests for new paths as quickly as we find them, then
+  // yield whichever result finishes first. Unfortunately, Promise.any() only
+  // tells us the result of the first promise to resolve, not which promise that
+  // was. So we keep track of a dictionary mapping paths to a promise for the
+  // value at that path. When a promise resolves, we mark it as resolved by
+  // setting its entry in the dictionary to null.
+  const promisesForPaths = {};
+
+  // Keep track of which resources make which outbound links.
+  const resourceOutboundReferences = {};
+
+  let errorPaths = [];
+
+  // Seed the promise dictionary with robots.txt at the root and an empty path
+  // indicating the current directory (relative to the baseUrl).
+  const initialPaths = ["/robots.txt", ""];
+  initialPaths.forEach((path) => {
+    promisesForPaths[path] = processPath(tree, path, baseUrl);
+  });
+
+  while (true) {
+    // Get the latest array of promises that haven't been resolved yet.
+    const promises = Object.values(promisesForPaths).filter(
+      (promise) => promise !== null
+    );
+
+    if (promises.length === 0) {
+      // No unresolved promises; we're done.
+      break;
+    }
+
+    // Wait for the first promise to resolve.
+    const result = await Promise.any(promises);
+
+    // Mark the promise for that result as resolved.
+    promisesForPaths[result.path] = null;
+
+    if (result.value === null) {
+      // Expected resource doesn't exist; add this to the errors. Exception: a
+      // path in the set of initialPaths that doesn't exist is not an error.
+      if (!initialPaths.includes(result.path)) {
+        errorPaths.push(result.path);
+      }
+      continue;
+    }
+
+    // Add the crawlable paths to the map. Use the normalized keys (will include
+    // "index.html" if the path ends in a trailing slash).
+    const normalizedPath = pathFromKeys(result.normalizedKeys);
+    resourceOutboundReferences[normalizedPath] = result.crawlablePaths;
+
+    // Add promises for crawlable paths in the result.
+    result.crawlablePaths.forEach((path) => {
+      // Only add a promise for this path if we don't already have one.
+      if (promisesForPaths[path] === undefined) {
+        promisesForPaths[path] = processPath(tree, path, baseUrl);
+      }
+    });
+
+    yield result;
+  }
+
+  if (errorPaths.length > 0) {
+    // Create a map of the resources that refer to each missing resource.
+    const errorsMap = {};
+    for (const sourcePath in resourceOutboundReferences) {
+      // Does this resource refer to any of the error paths?
+      const targetPaths = resourceOutboundReferences[sourcePath];
+      for (const targetPath of targetPaths) {
+        if (errorPaths.includes(targetPath)) {
+          errorsMap[sourcePath] ??= [];
+          errorsMap[sourcePath].push(targetPath);
+        }
+      }
+    }
+
+    // Review the errors map to find any paths that could not be traced back to
+    // a referring resource. These are internal crawler errors. We log them so
+    // that the use can report them and we can investigate them.
+    for (const errorPath of errorPaths) {
+      if (!Object.values(errorsMap).flat().includes(errorPath)) {
+        errorsMap["(unknown)"] ??= [];
+        errorsMap["(unknown)"].push(errorPath);
+      }
+    }
+
+    const errorsJson = JSON.stringify(errorsMap, null, 2);
+    yield {
+      normalizedKeys: ["crawl-errors.json"],
+      path: "crawl-errors.json",
+      resourcePaths: [],
+      value: errorsJson,
+    };
+  }
+}
+
+async function processPath(tree, path, baseUrl) {
+  // Don't process any path outside the baseUrl.
+  const url = new URL(path, baseUrl);
+  if (!url.pathname.startsWith(baseUrl.pathname)) {
+    return {
+      path,
+      value: null,
+    };
+  }
+
+  // Convert path to keys
+  let keys = keysFromPath(path);
+
+  // Paths (including those created by the filterPaths function above) will have
+  // spaces, etc., escaped. In general, these need to be unescaped so we can
+  // find them in the tree.
+  keys = keys.map(decodeURIComponent);
+
+  // Traverse tree to get value.
+  let value = await Tree.traverse(tree, ...keys);
+  const normalizedKeys = normalizeKeys(keys);
+  let normalizedPath = path;
+  if (Tree.isTreelike(value)) {
+    // Path is actually a directory; see if it has an index.html
+    value = await Tree.traverse(value, "index.html");
+    if (value !== undefined) {
+      if (path.length > 0) {
+        // Mark the path as ending in a slash
+        normalizedPath = trailingSlash.add(path);
+      }
+
+      // Add index.html to keys if it's not already there
+      if (normalizedKeys.at(-1) !== "index.html") {
+        normalizedKeys.push("index.html");
+      }
+    }
+  }
+
+  if (value === undefined) {
+    return {
+      crawlablePaths: [],
+      keys,
+      normalizedKeys,
+      path,
+      resourcePaths: [],
+      value: null,
+    };
+  }
+
+  // Find paths in the value
+  const key = normalizedKeys.at(-1);
+  const { crawlablePaths, resourcePaths } = await findPaths(
+    value,
+    key,
+    baseUrl,
+    normalizedPath
+  );
+
+  return {
+    crawlablePaths,
+    keys,
+    normalizedKeys,
+    path,
+    resourcePaths,
+    value,
+  };
+}

package/src/crawler/findPaths.js

@@ -0,0 +1,259 @@
+import { toString } from "@weborigami/async-tree";
+import { extname } from "@weborigami/language";
+import { isCrawlableHref, normalizeHref } from "./utilities.js";
+
+// Filter the paths to those that are local to the site.
+function filterPaths(paths, baseUrl, localPath) {
+  // Convert paths to absolute URLs.
+  const localUrl = new URL(localPath, baseUrl);
+  const basePathname = baseUrl.pathname;
+  // @ts-ignore
+  const absoluteUrls = paths.map((path) => new URL(path, localUrl));
+
+  // Convert the absolute URLs to paths relative to the baseHref. If the URL
+  // points outside the tree rooted at the baseHref, the relative path will be
+  // null. We ignore the protocol in this test, because in practice sites often
+  // fumble the use of http and https, treating them interchangeably.
+  const relativePaths = absoluteUrls.map((url) => {
+    if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
+      return url.pathname.slice(basePathname.length);
+    } else {
+      return null;
+    }
+  });
+
+  // Filter out the null paths.
+  /** @type {string[]} */
+  // @ts-ignore
+  const filteredPaths = relativePaths.filter((path) => path);
+  return filteredPaths;
+}
+
+/**
+ * Given a value retrieved from a site using a given key (name), determine what
+ * kind of file it is and, based on that, find the paths it references.
+ */
+export default function findPaths(value, key, baseUrl, localPath) {
+  const text = toString(value);
+
+  // We guess the value is HTML is if its key has an .html extension or
+  // doesn't have an extension, or the value starts with `<`.
+  const ext = key ? extname(key).toLowerCase() : "";
+  const maybeHtml = ext === "" || text?.trim().startsWith("<");
+  let foundPaths;
+  if (ext === ".html" || ext === ".htm" || ext === ".xhtml") {
+    foundPaths = findPathsInHtml(text);
+  } else if (ext === ".css") {
+    foundPaths = findPathsInCss(text);
+  } else if (ext === ".js") {
+    foundPaths = findPathsInJs(text);
+  } else if (ext === ".map") {
+    foundPaths = findPathsInImageMap(text);
+  } else if (key === "robots.txt") {
+    foundPaths = findPathsInRobotsTxt(text);
+  } else if (key === "sitemap.xml") {
+    foundPaths = findPathsInSitemapXml(text);
+  } else if (maybeHtml) {
+    foundPaths = findPathsInHtml(text);
+  } else {
+    // Doesn't have an extension we want to process
+    return {
+      crawlablePaths: [],
+      resourcePaths: [],
+    };
+  }
+
+  const crawlablePaths = filterPaths(
+    foundPaths.crawlablePaths,
+    baseUrl,
+    localPath
+  );
+
+  const resourcePaths = filterPaths(
+    foundPaths.resourcePaths,
+    baseUrl,
+    localPath
+  );
+
+  return {
+    crawlablePaths,
+    resourcePaths,
+  };
+}
+
+function findPathsInCss(css) {
+  const resourcePaths = [];
+  let match;
+
+  // Find `url()` functions.
+  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
+  while ((match = urlRegex.exec(css))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push();
+    }
+  }
+
+  return {
+    crawlablePaths: [],
+    resourcePaths,
+  };
+}
+
+// These are ancient server-side image maps. They're so old that it's hard to
+// find documentation on them, but they're used on the reference Space Jam
+// website we use for testing the crawler. Example:
+// https://www.spacejam.com/1996/bin/bball.map
+function findPathsInImageMap(imageMap) {
+  const resourcePaths = [];
+  let match;
+
+  // Find hrefs as the second column in each line.
+  const hrefRegex = /^\w+ (?<href>\S+)(\s*$| [\d, ]+$)/gm;
+  while ((match = hrefRegex.exec(imageMap))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths: [],
+    resourcePaths,
+  };
+}
+
+function findPathsInJs(js) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `import` statements.
+  const importRegex = /import [\s\S]+?from\s+["'](?<import>[^"']*)["'];/g;
+  while ((match = importRegex.exec(js))) {
+    const href = normalizeHref(match.groups?.import);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}
+
+function findPathsInHtml(html) {
+  const crawlablePaths = [];
+  const resourcePaths = [];
+  let match;
+
+  // Find `href` attributes in anchor and link tags.
+  const linkRegex =
+    /<(?:a|A|link|LINK)[\s][^>]*?(?:href|HREF)=["'](?<link>[^>]*?)["'][^>]*>/g;
+  while ((match = linkRegex.exec(html))) {
+    // Links can point to be other crawlable paths and resource paths.
+    // We guess the type based on the extension.
+    const href = normalizeHref(match.groups?.link);
+    if (href) {
+      if (isCrawlableHref(href)) {
+        crawlablePaths.push(href);
+      } else {
+        resourcePaths.push(href);
+      }
+    }
+  }
+
+  // Find `src` attributes in img and script tags.
+  const srcRegex =
+    /<(?<tag>img|IMG|script|SCRIPT)[\s][^>]*?(?:src|SRC)=["'](?<src>[^>]*?)["'][^>]*>/g;
+  while ((match = srcRegex.exec(html))) {
+    const tag = match.groups?.tag;
+    const src = normalizeHref(match.groups?.src);
+    if (src) {
+      if (tag === "script" || tag === "SCRIPT") {
+        crawlablePaths.push(src);
+      } else {
+        resourcePaths.push(src);
+      }
+    }
+  }
+
+  // Find `url()` functions in CSS.
+  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
+  while ((match = urlRegex.exec(html))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push(href);
+    }
+  }
+
+  // Find `src` attribute on frame tags.
+  const frameRegex =
+    /<(?:frame|FRAME)[\s][^>]*?(?:src|SRC)=["'](?<href>[^>]*?)["'][^>]*>/g;
+  while ((match = frameRegex.exec(html))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  // Find ancient `background` attribute on body tag.
+  const backgroundRegex =
+    /<(?:body|BODY)[\s][^>]*?(?:background|BACKGROUND)=["'](?<href>[^>]*?)["'][^>]*>/g;
+  while ((match = backgroundRegex.exec(html))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push(href);
+    }
+  }
+
+  // Find `href` attribute on area tags.
+  const areaRegex =
+    /<(?:area|AREA)[\s][^>]*?(?:href|HREF)=["'](?<href>[^>]*?)["'][^>]*>/g;
+  while ((match = areaRegex.exec(html))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return { crawlablePaths, resourcePaths };
+}
+
+function findPathsInRobotsTxt(txt) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `Sitemap` directives.
+  const sitemapRegex = /Sitemap:\s*(?<href>[^\s]*)/g;
+  while ((match = sitemapRegex.exec(txt))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}
+
+function findPathsInSitemapXml(xml) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `loc` elements.
+  const locRegex = /<loc>(?<href>[^<]*)<\/loc>/g;
+  while ((match = locRegex.exec(xml))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}

package/src/crawler/utilities.js

@@ -0,0 +1,38 @@
+import { trailingSlash } from "@weborigami/async-tree";
+import { extname } from "@weborigami/language";
+
+// A fake base URL used to handle cases where an href is relative and must be
+// treated relative to some base URL.
+const fakeBaseUrl = new URL("https://fake");
+
+export function isCrawlableHref(href) {
+  // Use a fake base URL to cover the case where the href is relative.
+  const url = new URL(href, fakeBaseUrl);
+  const pathname = url.pathname;
+  const lastKey = pathname.split("/").pop() ?? "";
+  if (lastKey === "robots.txt" || lastKey === "sitemap.xml") {
+    return true;
+  }
+  const ext = extname(lastKey);
+  // We assume an empty extension is HTML.
+  const crawlableExtensions = [".html", ".css", ".js", ".map", ".xhtml", ""];
+  return crawlableExtensions.includes(ext);
+}
+
+// Remove any search parameters or hash from the href. Preserve absolute or
+// relative nature of URL. If the URL only has a search or hash, return null.
+export function normalizeHref(href) {
+  // Remove everything after a `#` or `?` character.
+  const normalized = href.split(/[?#]/)[0];
+  return normalized === "" ? null : normalized;
+}
+
+// For indexing and storage purposes, treat a path that ends in a trailing slash
+// as if it ends in index.html.
+export function normalizeKeys(keys) {
+  const normalized = keys.slice();
+  if (normalized.length === 0 || trailingSlash.has(normalized.at(-1))) {
+    normalized.push("index.html");
+  }
+  return normalized;
+}