@weborigami/origami 0.0.68 → 0.0.70

package/src/builtins/@treeHttp.js

@@ -8,7 +8,7 @@ import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
  *
  * @this {AsyncTree|null}
  * @param {string} host
- * @param  {...string|Symbol} keys
+ * @param  {...string} keys
  */
 export default function treeHttp(host, ...keys) {
   assertTreeIsDefined(this, "treeHttp");

package/src/builtins/@treeHttps.js

@@ -8,7 +8,7 @@ import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
  *
  * @this {AsyncTree|null}
  * @param {string} host
- * @param  {...string|Symbol} keys
+ * @param  {...string} keys
  */
 export default function treeHttps(host, ...keys) {
   assertTreeIsDefined(this, "treeHttps");

package/src/common/processUnpackedContent.js

@@ -24,7 +24,11 @@ export default function processUnpackedContent(content, parent, attachedData) {
     } else {
       target = base;
     }
-    return content.bind(target);
+    const result = content.bind(target);
+    if (content.code) {
+      result.code = content.code;
+    }
+    return result;
   } else if (Tree.isAsyncTree(content) && !content.parent) {
     const result = Object.create(content);
     result.parent = parent;

package/src/common/utilities.js

@@ -64,13 +64,16 @@ export function toFunction(obj) {
     return obj;
   } else if (isUnpackable(obj)) {
     // Extract the contents of the object and convert that to a function.
-    let fn;
+    let fnPromise;
     /** @this {any} */
     return async function (...args) {
-      if (!fn) {
-        const content = await obj.unpack();
-        fn = toFunction(content);
+      if (!fnPromise) {
+        // unpack() may return a function or a promise for a function; normalize
+        // to a promise for a function
+        const unpackPromise = Promise.resolve(obj.unpack());
+        fnPromise = unpackPromise.then((content) => toFunction(content));
       }
+      const fn = await fnPromise;
       return fn.call(this, ...args);
     };
   } else if (Tree.isTreelike(obj)) {

package/src/crawler/crawlResources.js

@@ -0,0 +1,180 @@
+import {
+  keysFromPath,
+  pathFromKeys,
+  trailingSlash,
+  Tree,
+} from "@weborigami/async-tree";
+import findPaths from "./findPaths.js";
+import { normalizeKeys } from "./utilities.js";
+
+/**
+ * Crawl the paths for the given tree, starting at the given base URL, and yield
+ * the crawled resources.
+ *
+ * Each will include the HTML/script/stylesheet value retrieved at a given path.
+ */
+export default async function* crawlResources(tree, baseUrl) {
+  // We want to kick off requests for new paths as quickly as we find them, then
+  // yield whichever result finishes first. Unfortunately, Promise.any() only
+  // tells us the result of the first promise to resolve, not which promise that
+  // was. So we keep track of a dictionary mapping paths to a promise for the
+  // value at that path. When a promise resolves, we mark it as resolved by
+  // setting its entry in the dictionary to null.
+  const promisesForPaths = {};
+
+  // Keep track of which resources make which outbound links.
+  const resourceOutboundReferences = {};
+
+  let errorPaths = [];
+
+  // Seed the promise dictionary with robots.txt at the root, a sitemap.xml at
+  // the root, and an empty path indicating the current directory (relative to
+  // the baseUrl).
+  const initialPaths = ["/robots.txt", "/sitemap.xml", ""];
+  initialPaths.forEach((path) => {
+    promisesForPaths[path] = processPath(tree, path, baseUrl);
+  });
+
+  while (true) {
+    // Get the latest array of promises that haven't been resolved yet.
+    const promises = Object.values(promisesForPaths).filter(
+      (promise) => promise !== null
+    );
+
+    if (promises.length === 0) {
+      // No unresolved promises; we're done.
+      break;
+    }
+
+    // Wait for the first promise to resolve.
+    const result = await Promise.any(promises);
+
+    // Mark the promise for that result as resolved.
+    promisesForPaths[result.path] = null;
+
+    if (result.value === null) {
+      // Expected resource doesn't exist; add this to the errors. Exception: a
+      // path in the set of initialPaths that doesn't exist is not an error.
+      if (!initialPaths.includes(result.path)) {
+        errorPaths.push(result.path);
+      }
+      continue;
+    }
+
+    // Add the crawlable paths to the map. Use the normalized keys (will include
+    // "index.html" if the path ends in a trailing slash).
+    const normalizedPath = pathFromKeys(result.normalizedKeys);
+    resourceOutboundReferences[normalizedPath] = result.crawlablePaths;
+
+    // Add promises for crawlable paths in the result.
+    result.crawlablePaths.forEach((path) => {
+      // Only add a promise for this path if we don't already have one.
+      if (promisesForPaths[path] === undefined) {
+        promisesForPaths[path] = processPath(tree, path, baseUrl);
+      }
+    });
+
+    yield result;
+  }
+
+  if (errorPaths.length > 0) {
+    // Create a map of the resources that refer to each missing resource.
+    const errorsMap = {};
+    for (const sourcePath in resourceOutboundReferences) {
+      // Does this resource refer to any of the error paths?
+      const targetPaths = resourceOutboundReferences[sourcePath];
+      for (const targetPath of targetPaths) {
+        if (errorPaths.includes(targetPath)) {
+          errorsMap[sourcePath] ??= [];
+          errorsMap[sourcePath].push(targetPath);
+        }
+      }
+    }
+
+    // Review the errors map to find any paths that could not be traced back to
+    // a referring resource. These are internal crawler errors. We log them so
+    // that the use can report them and we can investigate them.
+    for (const errorPath of errorPaths) {
+      if (!Object.values(errorsMap).flat().includes(errorPath)) {
+        errorsMap["(unknown)"] ??= [];
+        errorsMap["(unknown)"].push(errorPath);
+      }
+    }
+
+    const errorsJson = JSON.stringify(errorsMap, null, 2);
+    yield {
+      normalizedKeys: ["crawl-errors.json"],
+      path: "crawl-errors.json",
+      resourcePaths: [],
+      value: errorsJson,
+    };
+  }
+}
+
+async function processPath(tree, path, baseUrl) {
+  // Don't process any path outside the baseUrl.
+  const url = new URL(path, baseUrl);
+  if (!url.pathname.startsWith(baseUrl.pathname)) {
+    return {
+      path,
+      value: null,
+    };
+  }
+
+  // Convert path to keys
+  let keys = keysFromPath(path);
+
+  // Paths (including those created by the filterPaths function above) will have
+  // spaces, etc., escaped. In general, these need to be unescaped so we can
+  // find them in the tree.
+  keys = keys.map(decodeURIComponent);
+
+  // Traverse tree to get value.
+  let value = await Tree.traverse(tree, ...keys);
+  const normalizedKeys = normalizeKeys(keys);
+  let normalizedPath = path;
+  if (Tree.isTreelike(value)) {
+    // Path is actually a directory; see if it has an index.html
+    value = await Tree.traverse(value, "index.html");
+    if (value !== undefined) {
+      if (path.length > 0) {
+        // Mark the path as ending in a slash
+        normalizedPath = trailingSlash.add(path);
+      }
+
+      // Add index.html to keys if it's not already there
+      if (normalizedKeys.at(-1) !== "index.html") {
+        normalizedKeys.push("index.html");
+      }
+    }
+  }
+
+  if (value === undefined) {
+    return {
+      crawlablePaths: [],
+      keys,
+      normalizedKeys,
+      path,
+      resourcePaths: [],
+      value: null,
+    };
+  }
+
+  // Find paths in the value
+  const key = normalizedKeys.at(-1);
+  const { crawlablePaths, resourcePaths } = await findPaths(
+    value,
+    key,
+    baseUrl,
+    normalizedPath
+  );
+
+  return {
+    crawlablePaths,
+    keys,
+    normalizedKeys,
+    path,
+    resourcePaths,
+    value,
+  };
+}

package/src/crawler/findPaths.js

@@ -0,0 +1,259 @@
+import { toString } from "@weborigami/async-tree";
+import { extname } from "@weborigami/language";
+import { isCrawlableHref, normalizeHref } from "./utilities.js";
+
+// Filter the paths to those that are local to the site.
+function filterPaths(paths, baseUrl, localPath) {
+  // Convert paths to absolute URLs.
+  const localUrl = new URL(localPath, baseUrl);
+  const basePathname = baseUrl.pathname;
+  // @ts-ignore
+  const absoluteUrls = paths.map((path) => new URL(path, localUrl));
+
+  // Convert the absolute URLs to paths relative to the baseHref. If the URL
+  // points outside the tree rooted at the baseHref, the relative path will be
+  // null. We ignore the protocol in this test, because in practice sites often
+  // fumble the use of http and https, treating them interchangeably.
+  const relativePaths = absoluteUrls.map((url) => {
+    if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
+      return url.pathname.slice(basePathname.length);
+    } else {
+      return null;
+    }
+  });
+
+  // Filter out the null paths.
+  /** @type {string[]} */
+  // @ts-ignore
+  const filteredPaths = relativePaths.filter((path) => path);
+  return filteredPaths;
+}
+
+/**
+ * Given a value retrieved from a site using a given key (name), determine what
+ * kind of file it is and, based on that, find the paths it references.
+ */
+export default function findPaths(value, key, baseUrl, localPath) {
+  const text = toString(value);
+
+  // We guess the value is HTML is if its key has an .html extension or
+  // doesn't have an extension, or the value starts with `<`.
+  const ext = key ? extname(key).toLowerCase() : "";
+  const maybeHtml = ext === "" || text?.trim().startsWith("<");
+  let foundPaths;
+  if (ext === ".html" || ext === ".htm" || ext === ".xhtml") {
+    foundPaths = findPathsInHtml(text);
+  } else if (ext === ".css") {
+    foundPaths = findPathsInCss(text);
+  } else if (ext === ".js") {
+    foundPaths = findPathsInJs(text);
+  } else if (ext === ".map") {
+    foundPaths = findPathsInImageMap(text);
+  } else if (key === "robots.txt") {
+    foundPaths = findPathsInRobotsTxt(text);
+  } else if (key === "sitemap.xml") {
+    foundPaths = findPathsInSitemapXml(text);
+  } else if (maybeHtml) {
+    foundPaths = findPathsInHtml(text);
+  } else {
+    // Doesn't have an extension we want to process
+    return {
+      crawlablePaths: [],
+      resourcePaths: [],
+    };
+  }
+
+  const crawlablePaths = filterPaths(
+    foundPaths.crawlablePaths,
+    baseUrl,
+    localPath
+  );
+
+  const resourcePaths = filterPaths(
+    foundPaths.resourcePaths,
+    baseUrl,
+    localPath
+  );
+
+  return {
+    crawlablePaths,
+    resourcePaths,
+  };
+}
+
+function findPathsInCss(css) {
+  const resourcePaths = [];
+  let match;
+
+  // Find `url()` functions.
+  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
+  while ((match = urlRegex.exec(css))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push();
+    }
+  }
+
+  return {
+    crawlablePaths: [],
+    resourcePaths,
+  };
+}
+
+// These are ancient server-side image maps. They're so old that it's hard to
+// find documentation on them, but they're used on the reference Space Jam
+// website we use for testing the crawler. Example:
+// https://www.spacejam.com/1996/bin/bball.map
+function findPathsInImageMap(imageMap) {
+  const resourcePaths = [];
+  let match;
+
+  // Find hrefs as the second column in each line.
+  const hrefRegex = /^\w+ (?<href>\S+)(\s*$| [\d, ]+$)/gm;
+  while ((match = hrefRegex.exec(imageMap))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths: [],
+    resourcePaths,
+  };
+}
+
+function findPathsInJs(js) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `import` statements.
+  const importRegex = /import [\s\S]+?from\s+["'](?<import>[^"']*)["'];/g;
+  while ((match = importRegex.exec(js))) {
+    const href = normalizeHref(match.groups?.import);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}
+
+function findPathsInHtml(html) {
+  const crawlablePaths = [];
+  const resourcePaths = [];
+  let match;
+
+  // Find `href` attributes in anchor and link tags.
+  const linkRegex =
+    /<(?:a|A|link|LINK)[\s][^>]*?(?:href|HREF)=["'](?<link>[^>]*?)["'][^>]*>/g;
+  while ((match = linkRegex.exec(html))) {
+    // Links can point to be other crawlable paths and resource paths.
+    // We guess the type based on the extension.
+    const href = normalizeHref(match.groups?.link);
+    if (href) {
+      if (isCrawlableHref(href)) {
+        crawlablePaths.push(href);
+      } else {
+        resourcePaths.push(href);
+      }
+    }
+  }
+
+  // Find `src` attributes in img and script tags.
+  const srcRegex =
+    /<(?<tag>img|IMG|script|SCRIPT)[\s][^>]*?(?:src|SRC)=["'](?<src>[^>]*?)["'][^>]*>/g;
+  while ((match = srcRegex.exec(html))) {
+    const tag = match.groups?.tag;
+    const src = normalizeHref(match.groups?.src);
+    if (src) {
+      if (tag === "script" || tag === "SCRIPT") {
+        crawlablePaths.push(src);
+      } else {
+        resourcePaths.push(src);
+      }
+    }
+  }
+
+  // Find `url()` functions in CSS.
+  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
+  while ((match = urlRegex.exec(html))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push(href);
+    }
+  }
+
+  // Find `src` attribute on frame tags.
+  const frameRegex =
+    /<(?:frame|FRAME)[\s][^>]*?(?:src|SRC)=["'](?<href>[^>]*?)["'][^>]*>/g;
+  while ((match = frameRegex.exec(html))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  // Find ancient `background` attribute on body tag.
+  const backgroundRegex =
+    /<(?:body|BODY)[\s][^>]*?(?:background|BACKGROUND)=["'](?<href>[^>]*?)["'][^>]*>/g;
+  while ((match = backgroundRegex.exec(html))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push(href);
+    }
+  }
+
+  // Find `href` attribute on area tags.
+  const areaRegex =
+    /<(?:area|AREA)[\s][^>]*?(?:href|HREF)=["'](?<href>[^>]*?)["'][^>]*>/g;
+  while ((match = areaRegex.exec(html))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return { crawlablePaths, resourcePaths };
+}
+
+function findPathsInRobotsTxt(txt) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `Sitemap` directives.
+  const sitemapRegex = /Sitemap:\s*(?<href>[^\s]*)/g;
+  while ((match = sitemapRegex.exec(txt))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}
+
+function findPathsInSitemapXml(xml) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `loc` elements.
+  const locRegex = /<loc>(?<href>[^<]*)<\/loc>/g;
+  while ((match = locRegex.exec(xml))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}

package/src/crawler/utilities.js

@@ -0,0 +1,38 @@
+import { trailingSlash } from "@weborigami/async-tree";
+import { extname } from "@weborigami/language";
+
+// A fake base URL used to handle cases where an href is relative and must be
+// treated relative to some base URL.
+const fakeBaseUrl = new URL("https://fake");
+
+export function isCrawlableHref(href) {
+  // Use a fake base URL to cover the case where the href is relative.
+  const url = new URL(href, fakeBaseUrl);
+  const pathname = url.pathname;
+  const lastKey = pathname.split("/").pop() ?? "";
+  if (lastKey === "robots.txt" || lastKey === "sitemap.xml") {
+    return true;
+  }
+  const ext = extname(lastKey);
+  // We assume an empty extension is HTML.
+  const crawlableExtensions = [".html", ".css", ".js", ".map", ".xhtml", ""];
+  return crawlableExtensions.includes(ext);
+}
+
+// Remove any search parameters or hash from the href. Preserve absolute or
+// relative nature of URL. If the URL only has a search or hash, return null.
+export function normalizeHref(href) {
+  // Remove everything after a `#` or `?` character.
+  const normalized = href.split(/[?#]/)[0];
+  return normalized === "" ? null : normalized;
+}
+
+// For indexing and storage purposes, treat a path that ends in a trailing slash
+// as if it ends in index.html.
+export function normalizeKeys(keys) {
+  const normalized = keys.slice();
+  if (normalized.length === 0 || trailingSlash.has(normalized.at(-1))) {
+    normalized.push("index.html");
+  }
+  return normalized;
+}