@weborigami/origami 0.0.68 → 0.0.70

package/src/builtins/@crawl.js

@@ -6,14 +6,10 @@ import {
   keysFromPath,
   trailingSlash,
 } from "@weborigami/async-tree";
-import { InvokeFunctionsTransform, extname } from "@weborigami/language";
-import * as utilities from "../common/utilities.js";
-import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
-import treeHttps from "./@treeHttps.js";
-
-// A fake base URL used to handle cases where an href is relative and must be
-// treated relative to some base URL.
-const fakeBaseUrl = new URL("https://fake");
+import { InvokeFunctionsTransform } from "@weborigami/language";
+import crawlResources from "../crawler/crawlResources.js";
+import { normalizeKeys } from "../crawler/utilities.js";
+import getTreeArgument from "../misc/getTreeArgument.js";
 
 /**
  * Crawl a tree, starting its root index.html page, and following links to
@@ -24,23 +20,19 @@ const fakeBaseUrl = new URL("https://fake");
  * that obtain the requested value from the original site.
  *
  * @typedef  {import("@weborigami/types").AsyncTree} AsyncTree
- * @typedef {import("@weborigami/async-tree").Treelike|string} Treelike
+ * @typedef {import("@weborigami/async-tree").Treelike} Treelike
  * @this {AsyncTree|null}
  * @param {Treelike} treelike
  * @param {string} [baseHref]
  * @returns {Promise<AsyncTree>}
  */
-export default async function crawl(treelike, baseHref) {
-  assertTreeIsDefined(this, "crawl");
-  const tree =
-    typeof treelike === "string"
-      ? treeHttps.call(this, treelike)
-      : Tree.from(treelike, { parent: this });
+export default async function crawlBuiltin(treelike, baseHref) {
+  const tree = await getTreeArgument(this, arguments, treelike, "@crawl");
 
   if (baseHref === undefined) {
     // Ask tree or original treelike if it has an `href` property we can use as
     // the base href to determine whether a link is local within the tree or
-    // not. If not, use a fake `local:/` href.
+    // not. If not, use a fake `local:/` base href.
     baseHref =
       /** @type {any} */ (tree).href ??
       /** @type {any} */ (treelike).href ??
@@ -48,7 +40,21 @@ export default async function crawl(treelike, baseHref) {
     if (!baseHref?.endsWith("/")) {
       baseHref += "/";
     }
+  } else {
+    // Is the href already valid?
+    let isHrefValid = false;
+    try {
+      new URL(baseHref);
+      isHrefValid = true;
+    } catch (e) {
+      // Ignore
+    }
+    if (!isHrefValid) {
+      // Use a fake base href.
+      baseHref = `local:/${baseHref}`;
+    }
   }
+
   // @ts-ignore
   const baseUrl = new URL(baseHref);
 
@@ -57,25 +63,18 @@ export default async function crawl(treelike, baseHref) {
   const errors = [];
 
   // We iterate until there are no more promises to wait for.
-  for await (const result of crawlPaths(tree, baseUrl)) {
-    const { keys, resourcePaths, value } = result;
+  for await (const result of crawlResources(tree, baseUrl)) {
+    const { normalizedKeys, resourcePaths, value } = result;
 
     // Cache the value
     if (value) {
-      addValueToObject(cache, keys, value);
+      addValueToObject(cache, normalizedKeys, value);
     }
-    // else if (keys) {
-    //   // A missing robots.txt isn't an error; anything else missing is.
-    //   const path = keys.join("/");
-    //   if (path !== "robots.txt") {
-    //     errors.push(path);
-    //   }
-    // }
 
     // Add indirect resource functions to the resource tree. When requested,
     // these functions will obtain the resource from the original site.
     for (const resourcePath of resourcePaths) {
-      const resourceKeys = adjustKeys(keysFromPath(resourcePath));
+      const resourceKeys = normalizeKeys(keysFromPath(resourcePath));
       const fn = () => {
         return Tree.traverse(tree, ...resourceKeys);
       };
@@ -101,17 +100,6 @@ export default async function crawl(treelike, baseHref) {
   return result;
 }
 
-// For indexing and storage purposes, treat a path that ends in a trailing slash
-// as if it ends in index.html.
-function adjustKeys(keys) {
-  if (keys.length > 0 && !trailingSlash.has(keys.at(-1))) {
-    return keys;
-  }
-  const adjustedKeys = keys.slice();
-  adjustedKeys.push("index.html");
-  return adjustedKeys;
-}
-
 function addValueToObject(object, keys, value) {
   for (let i = 0, current = object; i < keys.length; i++) {
     const key = trailingSlash.remove(keys[i]);
@@ -128,10 +116,9 @@ function addValueToObject(object, keys, value) {
       if (!current[key]) {
         current[key] = {};
       } else if (!isPlainObject(current[key])) {
-        // Already have a value at this point. The site has a page
-        // at a route like /foo, and the site also has resources
-        // within that at routes like /foo/bar.jpg. We move the
-        // current value to "index.html".
+        // Already have a value at this point. The site has a page at a route
+        // like /foo, and the site also has resources within that at routes like
+        // /foo/bar.jpg. We move the current value to "index.html".
         current[key] = { "index.html": current[key] };
       }
       current = current[key];
@@ -139,416 +126,5 @@ function addValueToObject(object, keys, value) {
   }
 }
 
-// Crawl the paths for the given tree, starting at the given base URL, and
-// yield the results. The results will include the HTML/script/stylesheet value
-// retrieved at a path, along with the paths to other resources found in that
-// text.
-async function* crawlPaths(tree, baseUrl) {
-  // We want to kick off requests for new paths as quickly as we find them, then
-  // yield whichever result finishes first. Unfortunately, Promise.any() only
-  // tells us the result of the first promise to resolve, not which promise that
-  // was. So we keep track of a dictionary mapping paths to a promise for the
-  // value at that path. When a promise resolves, we mark it as resolved by
-  // setting its entry in the dictionary to null.
-  const promisesForPaths = {};
-
-  // Keep track of which resources refer to which paths.
-  const mapResourceToPaths = {};
-
-  let errorPaths = [];
-
-  // Seed the promise dictionary with robots.txt and the root path.
-  const initialPaths = ["/robots.txt", "/"];
-  initialPaths.forEach((path) => {
-    promisesForPaths[path] = processPath(tree, path, baseUrl);
-  });
-
-  while (true) {
-    // Get the latest array of promises that haven't been resolved yet.
-    const promises = Object.values(promisesForPaths).filter(
-      (promise) => promise !== null
-    );
-
-    if (promises.length === 0) {
-      // No unresolved promises; we're done.
-      break;
-    }
-
-    // Wait for the first promise to resolve.
-    const result = await Promise.any(promises);
-
-    // Mark the promise for that result as resolved.
-    promisesForPaths[result.path] = null;
-
-    // Add the crawlable paths to the map.
-    mapResourceToPaths[result.path] = result.crawlablePaths;
-
-    // Add promises for crawlable paths in the result.
-    result.crawlablePaths.forEach((path) => {
-      // Only add a promise for this path if we don't already have one.
-      if (promisesForPaths[path] === undefined) {
-        promisesForPaths[path] = processPath(tree, path, baseUrl);
-      }
-    });
-
-    // If there was no value, add this to the errors.
-    // A missing robots.txt isn't an error; anything else missing is.
-    if (result.value === null && result.path !== "/robots.txt") {
-      errorPaths.push(result.path);
-    }
-
-    yield result;
-  }
-
-  if (errorPaths.length > 0) {
-    // Create a map of the resources that refer to each error.
-    const errorsMap = {};
-    for (const resource in mapResourceToPaths) {
-      const paths = mapResourceToPaths[resource];
-      for (const path of paths) {
-        if (errorPaths.includes(path)) {
-          errorsMap[resource] ??= [];
-          errorsMap[resource].push(path);
-        }
-      }
-    }
-    const errorsJson = JSON.stringify(errorsMap, null, 2);
-    yield {
-      keys: ["crawl-errors.json"],
-      path: "crawl-errors.json",
-      resourcePaths: [],
-      value: errorsJson,
-    };
-  }
-}
-
-// Filter the paths to those that are local to the site.
-function filterPaths(paths, baseUrl, localPath) {
-  // Convert paths to absolute URLs.
-  const localUrl = new URL(localPath, baseUrl);
-  const basePathname = baseUrl.pathname;
-  // @ts-ignore
-  const absoluteUrls = paths.map((path) => new URL(path, localUrl));
-
-  // Convert the absolute URLs to paths relative to the baseHref. If the URL
-  // points outside the tree rooted at the baseHref, the relative path will be
-  // null. We ignore the protocol in this test, because in practice sites often
-  // fumble the use of http and https, treating them interchangeably.
-  const relativePaths = absoluteUrls.map((url) => {
-    if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
-      return url.pathname.slice(basePathname.length);
-    } else {
-      return null;
-    }
-  });
-
-  // Filter out the null paths.
-  /** @type {string[]} */
-  // @ts-ignore
-  const filteredPaths = relativePaths.filter((path) => path);
-  return filteredPaths;
-}
-
-function findPaths(value, key, baseUrl, localPath) {
-  const text = utilities.toString(value);
-
-  // We guess the value is HTML is if its key has an .html extension or
-  // doesn't have an extension, or the value starts with `<`.
-  const ext = key ? extname(key).toLowerCase() : "";
-  const maybeHtml = ext === "" || text?.trim().startsWith("<");
-  let foundPaths;
-  if (ext === ".html" || ext === ".htm") {
-    foundPaths = findPathsInHtml(text);
-  } else if (ext === ".css") {
-    foundPaths = findPathsInCss(text);
-  } else if (ext === ".js") {
-    foundPaths = findPathsInJs(text);
-  } else if (ext === ".map") {
-    foundPaths = findPathsInImageMap(text);
-  } else if (key === "robots.txt") {
-    foundPaths = findPathsInRobotsTxt(text);
-  } else if (key === "sitemap.xml") {
-    foundPaths = findPathsInSitemapXml(text);
-  } else if (maybeHtml) {
-    foundPaths = findPathsInHtml(text);
-  } else {
-    // Doesn't have an extension we want to process
-    return {
-      crawlablePaths: [],
-      resourcePaths: [],
-    };
-  }
-
-  const crawlablePaths = filterPaths(
-    foundPaths.crawlablePaths,
-    baseUrl,
-    localPath
-  );
-  const resourcePaths = filterPaths(
-    foundPaths.resourcePaths,
-    baseUrl,
-    localPath
-  );
-  return {
-    crawlablePaths,
-    resourcePaths,
-  };
-}
-
-function findPathsInCss(css) {
-  const resourcePaths = [];
-  let match;
-
-  // Find `url()` functions.
-  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
-  while ((match = urlRegex.exec(css))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push();
-    }
-  }
-
-  return {
-    crawlablePaths: [],
-    resourcePaths,
-  };
-}
-
-// These are ancient server-side image maps. They're so old that it's hard to
-// find documentation on them, but they're used on the reference Space Jam
-// website we use for testing the crawler. Example:
-// https://www.spacejam.com/1996/bin/bball.map
-function findPathsInImageMap(imageMap) {
-  const resourcePaths = [];
-  let match;
-
-  // Find hrefs as the second column in each line.
-  const hrefRegex = /^\w+ (?<href>\S+)(\s*$| [\d, ]+$)/gm;
-  while ((match = hrefRegex.exec(imageMap))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths: [],
-    resourcePaths,
-  };
-}
-
-function findPathsInJs(js) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `import` statements.
-  const importRegex = /import [\s\S]+?from\s+["'](?<import>[^"']*)["'];/g;
-  while ((match = importRegex.exec(js))) {
-    const href = normalizeHref(match.groups?.import);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function findPathsInHtml(html) {
-  const crawlablePaths = [];
-  const resourcePaths = [];
-  let match;
-
-  // Find `href` attributes in anchor and link tags.
-  const linkRegex =
-    /<(?:a|A|link|LINK) [^>]*?(?:href|HREF)=["'](?<link>[^>]*?)["'][^>]*>/g;
-  while ((match = linkRegex.exec(html))) {
-    // Links can point to be other crawlable paths and resource paths.
-    // We guess the type based on the extension.
-    const href = normalizeHref(match.groups?.link);
-    if (href) {
-      if (isCrawlableHref(href)) {
-        crawlablePaths.push(href);
-      } else {
-        resourcePaths.push(href);
-      }
-    }
-  }
-
-  // Find `src` attributes in img and script tags.
-  const srcRegex =
-    /<(?<tag>img|IMG|script|SCRIPT) [^>]*?(?:src|SRC)=["'](?<src>[^>]*?)["'][^>]*>/g;
-  while ((match = srcRegex.exec(html))) {
-    const tag = match.groups?.tag;
-    const src = normalizeHref(match.groups?.src);
-    if (src) {
-      if (tag === "script" || tag === "SCRIPT") {
-        crawlablePaths.push(src);
-      } else {
-        resourcePaths.push(src);
-      }
-    }
-  }
-
-  // Find `url()` functions in CSS.
-  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
-  while ((match = urlRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  // Find `src` attribute on frame tags.
-  const frameRegex =
-    /<(?:frame|FRAME) [^>]*?(?:src|SRC)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = frameRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  // Find ancient `background` attribute on body tag.
-  const backgroundRegex =
-    /<(?:body|BODY) [^>]*?(?:background|BACKGROUND)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = backgroundRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  // Find `href` attribute on area tags.
-  const areaRegex =
-    /<(?:area|AREA) [^>]*?(?:href|HREF)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = areaRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return { crawlablePaths, resourcePaths };
-}
-
-function findPathsInRobotsTxt(txt) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `Sitemap` directives.
-  const sitemapRegex = /Sitemap:\s*(?<href>[^\s]*)/g;
-  while ((match = sitemapRegex.exec(txt))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function findPathsInSitemapXml(xml) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `loc` elements.
-  const locRegex = /<loc>(?<href>[^<]*)<\/loc>/g;
-  while ((match = locRegex.exec(xml))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function isCrawlableHref(href) {
-  // Use a fake base URL to cover the case where the href is relative.
-  const url = new URL(href, fakeBaseUrl);
-  const pathname = url.pathname;
-  const lastKey = pathname.split("/").pop() ?? "";
-  if (lastKey === "robots.txt" || lastKey === "sitemap.xml") {
-    return true;
-  }
-  const ext = extname(lastKey);
-  // We assume an empty extension is HTML.
-  const crawlableExtensions = [".html", ".css", ".js", ".map", ""];
-  return crawlableExtensions.includes(ext);
-}
-
-// Remove any search parameters or hash from the href. Preserve absolute or
-// relative nature of URL. If the URL only has a search or hash, return null.
-function normalizeHref(href) {
-  // Remove everything after a `#` or `?` character.
-  const normalized = href.split(/[?#]/)[0];
-  return normalized === "" ? null : normalized;
-}
-
-async function processPath(tree, path, baseUrl) {
-  if (path === undefined) {
-    return {
-      crawlablePaths: [],
-      keys: null,
-      path,
-      resourcePaths: [],
-      value: null,
-    };
-  }
-
-  // Convert path to keys
-  let keys = keysFromPath(path);
-
-  // Paths (including those created by the filterPaths function above) will have
-  // spaces, etc., escaped. In general, these need to be unescaped so we can
-  // find them in the tree.
-  keys = keys.map(decodeURIComponent);
-
-  // Traverse tree to get value.
-  let value = await Tree.traverse(tree, ...keys);
-  if (Tree.isAsyncTree(value)) {
-    // Path is actually a directory; see if it has an index.html
-    value = await Tree.traverse(value, "index.html");
-  }
-
-  const adjustedKeys = adjustKeys(keys);
-
-  if (value === undefined) {
-    return {
-      crawlablePaths: [],
-      keys: adjustedKeys,
-      path,
-      resourcePaths: [],
-      value: null,
-    };
-  }
-
-  // Find paths in the value
-  const key = adjustedKeys.at(-1);
-  const { crawlablePaths, resourcePaths } = await findPaths(
-    value,
-    key,
-    baseUrl,
-    path
-  );
-
-  return {
-    crawlablePaths,
-    keys: adjustedKeys,
-    path,
-    resourcePaths,
-    value,
-  };
-}
-
-crawl.usage = `@crawl <tree>\tCrawl a tree`;
-crawl.documentation = "https://weborigami.org/language/@crawl.html";
+crawlBuiltin.usage = `@crawl <tree>\tCrawl a tree`;
+crawlBuiltin.documentation = "https://weborigami.org/language/@crawl.html";

package/src/builtins/@indent.js

@@ -0,0 +1,115 @@
+const lastLineWhitespaceRegex = /\n(?<indent>[ \t]*)$/;
+
+const mapStringsToModifications = new Map();
+
+/**
+ * Normalize indentation in a tagged template string.
+ *
+ * @param {TemplateStringsArray} strings
+ * @param  {...any} values
+ * @returns {string}
+ */
+export default function indent(strings, ...values) {
+  let modified = mapStringsToModifications.get(strings);
+  if (!modified) {
+    modified = modifyStrings(strings);
+    mapStringsToModifications.set(strings, modified);
+  }
+  const { blockIndentations, strings: modifiedStrings } = modified;
+  return joinBlocks(modifiedStrings, values, blockIndentations);
+}
+
+// Join strings and values, applying the given block indentation to the lines of
+// values for block placholders.
+function joinBlocks(strings, values, blockIndentations) {
+  let result = strings[0];
+  for (let i = 0; i < values.length; i++) {
+    let text = values[i];
+    if (text) {
+      const blockIndentation = blockIndentations[i];
+      if (blockIndentation) {
+        const lines = text.split("\n");
+        text = "";
+        if (lines.at(-1) === "") {
+          // Drop empty last line
+          lines.pop();
+        }
+        for (let line of lines) {
+          text += blockIndentation + line + "\n";
+        }
+      }
+      result += text;
+    }
+    result += strings[i + 1];
+  }
+  return result;
+}
+
+// Given an array of template boilerplate strings, return an object { modified,
+// blockIndentations } where `strings` is the array of strings with indentation
+// removed, and `blockIndentations` is an array of indentation strings for each
+// block placeholder.
+function modifyStrings(strings) {
+  // Phase one: Identify the indentation based on the first real line of the
+  // first string (skipping the initial newline), and remove this indentation
+  // from all lines of all strings.
+  let indent;
+  if (strings.length > 0 && strings[0].startsWith("\n")) {
+    // Look for indenttation
+    const firstLineWhitespaceRegex = /^\n(?<indent>[ \t]*)/;
+    const match = strings[0].match(firstLineWhitespaceRegex);
+    indent = match?.groups.indent;
+  }
+
+  // Determine the modified strings. If this invoked as a JS tagged template
+  // literal, the `strings` argument will be an odd array-ish object that we'll
+  // want to convert to a real array.
+  let modified;
+  if (indent) {
+    // De-indent the strings.
+    const indentationRegex = new RegExp(`\n${indent}`, "g");
+    // The `replaceAll` also converts strings to a real array.
+    modified = strings.map((string) =>
+      string.replaceAll(indentationRegex, "\n")
+    );
+    // Remove indentation from last line of last string
+    modified[modified.length - 1] = modified
+      .at(-1)
+      .replace(lastLineWhitespaceRegex, "\n");
+  } else {
+    // No indentation; just copy the strings so we have a real array
+    modified = strings.slice();
+  }
+
+  // Phase two: Identify any block placholders, identify and remove their
+  // preceding indentation, and remove the following newline. Work backward from
+  // the end towards the start because we're modifying the strings in place and
+  // our pattern matching won't work going forward from start to end.
+  let blockIndentations = [];
+  for (let i = modified.length - 2; i >= 0; i--) {
+    // Get the modified before and after substitution with index `i`
+    const beforeString = modified[i];
+    const afterString = modified[i + 1];
+    const match = beforeString.match(lastLineWhitespaceRegex);
+    if (match && afterString.startsWith("\n")) {
+      // The substitution between these strings is a block substitution
+      let blockIndentation = match.groups.indent;
+      blockIndentations[i] = blockIndentation;
+      // Trim the before and after strings
+      if (blockIndentation) {
+        modified[i] = beforeString.slice(0, -blockIndentation.length);
+      }
+      modified[i + 1] = afterString.slice(1);
+    }
+  }
+
+  // Remove newline from start of first string *after* removing indentation.
+  if (modified[0].startsWith("\n")) {
+    modified[0] = modified[0].slice(1);
+  }
+
+  return {
+    blockIndentations,
+    strings: modified,
+  };
+}

package/src/builtins/@keysTree.js

@@ -8,7 +8,7 @@ import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
  *
  * @this {AsyncTree|null}
  * @param {string} host
- * @param  {...string|Symbol} keys
+ * @param  {...string} keys
  */
 export default function keysTree(host, ...keys) {
   assertTreeIsDefined(this, "keysTree");

package/src/builtins/@siteAudit.js

@@ -0,0 +1,19 @@
+import { Tree } from "@weborigami/async-tree";
+import getTreeArgument from "../misc/getTreeArgument.js";
+import crawl from "./@crawl.js";
+
+/**
+ * @this {import("@weborigami/types").AsyncTree|null}
+ * @param {import("@weborigami/async-tree").Treelike} treelike
+ */
+export default async function siteAudit(treelike) {
+  const tree = await getTreeArgument(this, arguments, treelike, "@siteAudit");
+  const crawled = await crawl.call(this, tree);
+  let crawlErrorsJson = await crawled.get("crawl-errors.json");
+  if (!crawlErrorsJson) {
+    return undefined;
+  }
+  const errors = Tree.from(JSON.parse(crawlErrorsJson), { deep: true });
+  errors.parent = this;
+  return errors;
+}

package/src/builtins/@slug.js

@@ -11,5 +11,8 @@ export default function slug(filename) {
   slug = slug.replace(/^-+/, "");
   slug = slug.replace(/-+$/, "");
 
+  // Collapse consecutive dashes to a single dash.
+  slug = slug.replace(/-+/g, "-");
+
   return slug;
 }