@weborigami/origami 0.3.0 → 0.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@weborigami/origami",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Web Origami language, CLI, framework, and server",
   "type": "module",
   "repository": {
@@ -17,13 +17,15 @@
     "typescript": "5.8.2"
   },
   "dependencies": {
-    "@weborigami/async-tree": "0.3.0",
-    "@weborigami/language": "0.3.0",
+    "@weborigami/async-tree": "0.3.2",
     "@weborigami/json-feed-to-rss": "1.0.0",
-    "@weborigami/types": "0.3.0",
+    "@weborigami/language": "0.3.2",
+    "@weborigami/types": "0.3.2",
+    "css-tree": "3.1.0",
     "exif-parser": "0.1.12",
     "graphviz-wasm": "3.0.2",
     "highlight.js": "11.11.1",
+    "jsdom": "26.1.0",
     "marked": "15.0.7",
     "marked-gfm-heading-id": "4.1.1",
     "marked-highlight": "2.2.1",

package/src/dev/crawler/audit.js

@@ -0,0 +1,85 @@
+import { pathFromKeys, symbols, Tree } from "@weborigami/async-tree";
+import getTreeArgument from "../../common/getTreeArgument.js";
+import crawlResources from "./crawlResources.js";
+import { getBaseUrl } from "./utilities.js";
+
+/**
+ * Crawl the indicated tree and return an audit of any broken links to internal
+ * pages or other resources.
+ *
+ * @typedef  {import("@weborigami/types").AsyncTree} AsyncTree
+ * @typedef {import("@weborigami/async-tree").Treelike} Treelike
+ *
+ * @this {AsyncTree|null}
+ * @param {Treelike} treelike
+ * @param {string} [baseHref]
+ */
+export default async function audit(treelike, baseHref) {
+  const tree = await getTreeArgument(this, arguments, treelike, "site:audit");
+  const baseUrl = getBaseUrl(baseHref, treelike);
+
+  let errors = {};
+  let report;
+  const resourceReferences = {};
+  const resourcePromises = {};
+
+  // Iterate through all the resources to crawl the whole tree.
+  for await (const result of crawlResources(tree, baseUrl)) {
+    const { normalizedKeys, resourcePaths, value: resource } = result;
+    const normalizedPath = pathFromKeys(normalizedKeys);
+    if (normalizedPath === "crawl-errors.json") {
+      // Final error report; add missing pages to the errors
+      report = JSON.parse(resource);
+      for (const [path, pagePaths] of Object.entries(report)) {
+        if (!errors[path]) {
+          errors[path] = [];
+        }
+        errors[path].push(...pagePaths);
+      }
+    } else {
+      // Record which resources this path references
+      resourceReferences[normalizedPath] = resourcePaths;
+
+      // Add all resources to the set that should be verified
+      for (const resourcePath of resourcePaths) {
+        // Start request, don't wait for it to complete yet
+        resourcePromises[resourcePath] ??= Tree.traversePath(
+          tree,
+          resourcePath
+        ).then(
+          // Just return true or false to indicate if value is defined
+          (value) => value !== undefined
+        );
+      }
+    }
+  }
+
+  // Add any references to missing resources to the errors
+  for (const [refererPath, resourcePaths] of Object.entries(
+    resourceReferences
+  )) {
+    for (const resourcePath of resourcePaths) {
+      const found = await resourcePromises[resourcePath];
+      if (!found) {
+        if (!errors[refererPath]) {
+          errors[refererPath] = [];
+        }
+        errors[refererPath].push(resourcePath);
+      }
+    }
+  }
+
+  if (Object.keys(errors).length === 0) {
+    return undefined;
+  }
+
+  Object.defineProperty(errors, symbols.parent, {
+    enumerable: false,
+    value: this,
+  });
+  Object.defineProperty(errors, symbols.deep, {
+    enumerable: false,
+    value: true,
+  });
+  return errors;
+}

package/src/{site → dev}/crawler/crawl.js

@@ -2,13 +2,12 @@ import {
   DeepObjectTree,
   Tree,
   deepMerge,
-  isPlainObject,
   keysFromPath,
-  trailingSlash,
 } from "@weborigami/async-tree";
 import { InvokeFunctionsTransform } from "@weborigami/language";
 import getTreeArgument from "../../common/getTreeArgument.js";
 import crawlResources from "./crawlResources.js";
+import { addValueToObject, getBaseUrl } from "./utilities.js";
 
 /**
  * Crawl a tree, starting its root index.html page, and following links to
@@ -20,6 +19,7 @@ import crawlResources from "./crawlResources.js";
  *
  * @typedef  {import("@weborigami/types").AsyncTree} AsyncTree
  * @typedef {import("@weborigami/async-tree").Treelike} Treelike
+ *
  * @this {AsyncTree|null}
  * @param {Treelike} treelike
  * @param {string} [baseHref]
@@ -27,39 +27,10 @@ import crawlResources from "./crawlResources.js";
  */
 export default async function crawlBuiltin(treelike, baseHref) {
   const tree = await getTreeArgument(this, arguments, treelike, "site:crawl");
-
-  if (baseHref === undefined) {
-    // Ask tree or original treelike if it has an `href` property we can use as
-    // the base href to determine whether a link is local within the tree or
-    // not. If not, use a fake `local:/` base href.
-    baseHref =
-      /** @type {any} */ (tree).href ??
-      /** @type {any} */ (treelike).href ??
-      "local:/";
-    if (!baseHref?.endsWith("/")) {
-      baseHref += "/";
-    }
-  } else {
-    // Is the href already valid?
-    let isHrefValid = false;
-    try {
-      new URL(baseHref);
-      isHrefValid = true;
-    } catch (e) {
-      // Ignore
-    }
-    if (!isHrefValid) {
-      // Use a fake base href.
-      baseHref = `local:/${baseHref}`;
-    }
-  }
-
-  // @ts-ignore
-  const baseUrl = new URL(baseHref);
+  const baseUrl = getBaseUrl(baseHref, treelike);
 
   const cache = {};
   const resources = {};
-  const errors = [];
 
   // We iterate until there are no more promises to wait for.
   for await (const result of crawlResources(tree, baseUrl)) {
@@ -81,14 +52,6 @@ export default async function crawlBuiltin(treelike, baseHref) {
     }
   }
 
-  if (errors.length) {
-    addValueToObject(
-      cache,
-      ["crawl-errors.json"],
-      JSON.stringify(errors, null, 2)
-    );
-  }
-
   // Merge the cache on top of the resources tree. If we have an actual value
   // for something already, that's better than a function that will get that
   // value.
@@ -98,29 +61,3 @@ export default async function crawlBuiltin(treelike, baseHref) {
   );
   return result;
 }
-
-function addValueToObject(object, keys, value) {
-  for (let i = 0, current = object; i < keys.length; i++) {
-    const key = trailingSlash.remove(keys[i]);
-    if (i === keys.length - 1) {
-      // Write out value
-      if (isPlainObject(current[key])) {
-        // Route with existing values; treat the new value as an index.html
-        current[key]["index.html"] = value;
-      } else {
-        current[key] = value;
-      }
-    } else {
-      // Traverse further
-      if (!current[key]) {
-        current[key] = {};
-      } else if (!isPlainObject(current[key])) {
-        // Already have a value at this point. The site has a page at a route
-        // like /foo, and the site also has resources within that at routes like
-        // /foo/bar.jpg. We move the current value to "index.html".
-        current[key] = { "index.html": current[key] };
-      }
-      current = current[key];
-    }
-  }
-}

package/src/{site → dev}/crawler/crawlResources.js

@@ -129,28 +129,54 @@ async function processPath(tree, path, baseUrl) {
   keys = keys.map(decodeURIComponent);
 
   // Traverse tree to get value.
-  let value = await Tree.traverse(tree, ...keys);
-  const normalizedKeys = keys.slice();
-  let normalizedPath = path;
-  if (Tree.isTreelike(value)) {
-    // Path is actually a directory. See if we can get the empty string or
-    // "index.html".
-    value =
-      (await Tree.traverse(value, "")) ??
-      (await Tree.traverse(value, "index.html"));
-    if (value !== undefined) {
-      if (path.length > 0) {
-        // Mark the path as ending in a slash
-        normalizedPath = trailingSlash.add(path);
-        const key = normalizedKeys.pop();
-        normalizedKeys.push(trailingSlash.add(key));
+  let value;
+  let normalizedKeys;
+  let normalizedPath;
+  try {
+    value = await Tree.traverse(tree, ...keys);
+    normalizedKeys = keys.slice();
+    normalizedPath = path;
+    if (Tree.isTreelike(value)) {
+      // Path is actually a directory. See if we can get the empty string or
+      // "index.html".
+      value =
+        (await Tree.traverse(value, "")) ??
+        (await Tree.traverse(value, "index.html"));
+      if (value !== undefined) {
+        if (path.length > 0) {
+          // Mark the path as ending in a slash
+          normalizedPath = trailingSlash.add(path);
+          const key = normalizedKeys.pop();
+          normalizedKeys.push(trailingSlash.add(key));
+        }
+
+        // Add index.html to keys if it's not already there
+        if (normalizedKeys.at(-1) !== "index.html") {
+          normalizedKeys.push("index.html");
+        }
       }
+    }
 
-      // Add index.html to keys if it's not already there
-      if (normalizedKeys.at(-1) !== "index.html") {
-        normalizedKeys.push("index.html");
+    if (value === undefined && path.length > 0) {
+      // The path may be a URL like `foo` or `foo/` that points to `foo.html`, so
+      // we'll try looking adding `.html` to the end. We don't want to check every
+      // path twice, so we only do this if the last key does *not* include an
+      // extension.
+      const lastKey = keys.at(-1);
+      if (lastKey !== "" && !lastKey?.includes(".")) {
+        const adjustedLastKey = `${trailingSlash.remove(lastKey)}.html`;
+        const adjustedKeys = [...keys.slice(0, -1), adjustedLastKey];
+        value = await Tree.traverse(tree, ...adjustedKeys);
+        if (value !== undefined) {
+          // Page exists at foo.html
+          normalizedPath = pathFromKeys(adjustedKeys);
+          normalizedKeys = adjustedKeys;
+        }
       }
     }
+  } catch (error) {
+    // Ignore errors, return empty paths below
+    value = undefined;
   }
 
   if (value === undefined) {

package/src/dev/crawler/findPaths.js

@@ -0,0 +1,90 @@
+import { extension, toString } from "@weborigami/async-tree";
+import pathsInCss from "./pathsInCss.js";
+import pathsInHtml from "./pathsInHtml.js";
+import pathsInImageMap from "./pathsInImageMap.js";
+import pathsInJs from "./pathsInJs.js";
+import pathsInRobotsTxt from "./pathsInRobotsTxt.js";
+import pathsInSitemap from "./pathsInSitemap.js";
+
+// Filter the paths to those that are local to the site.
+function filterPaths(paths, baseUrl, localPath) {
+  // Convert paths to absolute URLs.
+  const localUrl = new URL(localPath, baseUrl);
+  const basePathname = baseUrl.pathname;
+  // @ts-ignore
+  const absoluteUrls = paths.map((path) => new URL(path, localUrl));
+
+  // Convert the absolute URLs to paths relative to the baseHref. If the URL
+  // points outside the tree rooted at the baseHref, the relative path will be
+  // null. We ignore the protocol in this test, because in practice sites often
+  // fumble the use of http and https, treating them interchangeably.
+  const relativePaths = absoluteUrls.map((url) => {
+    if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
+      const path = url.pathname.slice(basePathname.length);
+      // The process of creating the URLs will have escaped characters. We
+      // remove them. This has the side-effect of removing them if they existed
+      // in the original path; it would be better if we avoided that.
+      return decodeURIComponent(path);
+    } else {
+      return null;
+    }
+  });
+
+  // Filter out the null paths.
+  /** @type {string[]} */
+  // @ts-ignore
+  const filteredPaths = relativePaths.filter((path) => path);
+  return filteredPaths;
+}
+
+/**
+ * Given a value retrieved from a site using a given key (name), determine what
+ * kind of file it is and, based on that, find the paths it references.
+ */
+export default function findPaths(value, key, baseUrl, localPath) {
+  const text = toString(value);
+
+  // We guess the value is HTML is if its key has an .html extension or
+  // doesn't have an extension, or the value starts with `<`.
+  const ext = key ? extension.extname(key).toLowerCase() : "";
+  let foundPaths;
+  if (ext === ".html" || ext === ".htm" || ext === ".xhtml") {
+    foundPaths = pathsInHtml(text);
+  } else if (ext === ".css") {
+    foundPaths = pathsInCss(text);
+  } else if (ext === ".js") {
+    foundPaths = pathsInJs(text);
+  } else if (ext === ".map") {
+    foundPaths = pathsInImageMap(text);
+  } else if (key === "robots.txt") {
+    foundPaths = pathsInRobotsTxt(text);
+  } else if (key === "sitemap.xml") {
+    foundPaths = pathsInSitemap(text);
+  } else if (ext === "" && text?.trim().startsWith("<")) {
+    // Probably HTML
+    foundPaths = pathsInHtml(text);
+  } else {
+    // Doesn't have an extension we want to process
+    return {
+      crawlablePaths: [],
+      resourcePaths: [],
+    };
+  }
+
+  const crawlablePaths = filterPaths(
+    foundPaths.crawlablePaths,
+    baseUrl,
+    localPath
+  );
+
+  const resourcePaths = filterPaths(
+    foundPaths.resourcePaths,
+    baseUrl,
+    localPath
+  );
+
+  return {
+    crawlablePaths,
+    resourcePaths,
+  };
+}

package/src/dev/crawler/pathsInCss.js

@@ -0,0 +1,51 @@
+import { parse, walk } from "css-tree";
+import { addHref } from "./utilities.js";
+
+const imageFunctions = ["cross-fade", "image", "image-set"];
+
+export default function pathsInCss(css, context = "stylesheet") {
+  const paths = {
+    crawlablePaths: [],
+    resourcePaths: [],
+  };
+
+  let ast;
+  try {
+    ast = parse(css, { context });
+  } catch (e) {
+    // If the CSS is invalid, we can't parse it, so we can't extract paths. For
+    // now we just return no paths.
+    return paths;
+  }
+
+  if (!ast) {
+    // Unclear why parser sometimes returns an undefined AST
+    return paths;
+  }
+
+  walk(
+    ast,
+    /** @this {any} */
+    function (node) {
+      const { type, value } = node;
+      if (
+        this.atrule?.name === "import" &&
+        (type === "String" || type === "Url")
+      ) {
+        // A plain string or url() in an @import
+        addHref(paths, value, true);
+      } else if (
+        type === "String" &&
+        imageFunctions.includes(this.function?.name)
+      ) {
+        // A plain string in an cross-fade(), image(), or image-set()
+        addHref(paths, value, false);
+      } else if (type === "Url") {
+        // A url() anywhere else
+        addHref(paths, value, false);
+      }
+    }
+  );
+
+  return paths;
+}

package/src/dev/crawler/pathsInHtml.js

@@ -0,0 +1,161 @@
+import { JSDOM, VirtualConsole } from "jsdom";
+import pathsInCss from "./pathsInCss.js";
+import pathsInJs from "./pathsInJs.js";
+import { addHref } from "./utilities.js";
+
+export default function pathsInHtml(html) {
+  const paths = {
+    crawlablePaths: [],
+    resourcePaths: [],
+  };
+
+  // Create a virtual console to avoid logging errors to the console
+  const virtualConsole = new VirtualConsole();
+  const document = new JSDOM(html, { virtualConsole }).window.document;
+
+  // Find `href` attributes in anchor, area, link, SVG tags.
+  //
+  // NOTE: As of April 2024, jsdom querySelectorAll does not appear to find
+  // elements with mixed-case tag names.
+  const hrefTags = document.querySelectorAll(
+    "a[href], area[href], image[href], feImage[href], filter[href], linearGradient[href], link[href], mpath[href], pattern[href], radialGradient[href], textPath[href], use[href]"
+  );
+  for (const hrefTag of hrefTags) {
+    const crawlable = ["A", "AREA"].includes(hrefTag.tagName)
+      ? true
+      : undefined;
+    addHref(paths, hrefTag.getAttribute("href"), crawlable);
+  }
+
+  // Find `src` attributes in input, frame, media, and script tags.
+  const srcTags = document.querySelectorAll(
+    "audio[src], embed[src], frame[src], iframe[src], img[src], input[src], script[src], source[src], track[src], video[src]"
+  );
+  for (const srcTag of srcTags) {
+    const crawlable = ["FRAME", "IFRAME"].includes(srcTag.tagName)
+      ? true
+      : srcTag.tagName === "SCRIPT"
+      ? srcTag.type === "module" // Only crawl modules
+      : undefined;
+    addHref(paths, srcTag.getAttribute("src"), crawlable);
+  }
+
+  // Find `srcset` attributes in image and source tags.
+  const srcsetTags = document.querySelectorAll("img[srcset], source[srcset]");
+  for (const srcsetTag of srcsetTags) {
+    const srcset = srcsetTag.getAttribute("srcset");
+    const srcRegex = /(?<url>[^\s,]+)(?=\s+\d+(?:\.\d+)?[wxh])/g;
+    let match;
+    while ((match = srcRegex.exec(srcset))) {
+      if (match.groups?.url) {
+        addHref(paths, match.groups.url, false);
+      }
+    }
+  }
+
+  // Find `poster` attributes in <video> tags.
+  const posterTags = document.querySelectorAll("video[poster]");
+  for (const posterTag of posterTags) {
+    addHref(paths, posterTag.getAttribute("poster"), false);
+  }
+
+  // Find `data` attributes in <object> tags.
+  const objectTags = document.querySelectorAll("object[data]");
+  for (const objectTag of objectTags) {
+    addHref(paths, objectTag.getAttribute("data"), false);
+  }
+
+  // Find deprecated `background` attribute on body and table tags.
+  const backgroundTags = document.querySelectorAll(
+    "body[background], table[background], td[background], th[background]"
+  );
+  for (const backgroundTag of backgroundTags) {
+    addHref(paths, backgroundTag.getAttribute("background"), false);
+  }
+
+  // Find deprecated `longdesc` attributes on <img> tags.
+  const longdescTags = document.querySelectorAll("img[longdesc]");
+  for (const longdescTag of longdescTags) {
+    addHref(paths, longdescTag.getAttribute("longdesc"), false);
+  }
+
+  // Find paths in <meta> image tags.
+  const imageMetaTags = document.querySelectorAll('meta[property$=":image"]');
+  for (const imageMetaTag of imageMetaTags) {
+    const content = imageMetaTag.getAttribute("content");
+    if (content) {
+      addHref(paths, content, false);
+    }
+  }
+
+  // Find paths in CSS in <style> tags.
+  const styleTags = document.querySelectorAll("style");
+  for (const styleAttribute of styleTags) {
+    const cssPaths = pathsInCss(styleAttribute.textContent);
+    paths.crawlablePaths.push(...cssPaths.crawlablePaths);
+    paths.resourcePaths.push(...cssPaths.resourcePaths);
+  }
+
+  // Find URLs in CSS in `style` attributes.
+  const styleAttributeTags = document.querySelectorAll("[style]");
+  for (const tag of styleAttributeTags) {
+    const style = tag.getAttribute("style");
+    const stylePaths = pathsInCss(style, "declarationList");
+    stylePaths.resourcePaths.forEach((href) => {
+      addHref(paths, href, false);
+    });
+  }
+
+  // Find URLs in SVG attributes.
+  const svgAttributeNames = [
+    "clip-path",
+    "fill",
+    "filter",
+    "marker-end",
+    "marker-start",
+    "mask",
+    "stroke",
+  ];
+  const svgTags = document.querySelectorAll(
+    svgAttributeNames.map((name) => `[${name}]`).join(", ")
+  );
+  for (const svgTag of svgTags) {
+    for (const name of svgAttributeNames) {
+      const attributeValue = svgTag.getAttribute(name);
+      if (!attributeValue) {
+        continue;
+      }
+      const urlRegex = /url\((['"]?)(?<href>.*?)\1\)/g;
+      const attributeValueMatch = urlRegex.exec(attributeValue);
+      if (attributeValueMatch) {
+        const href = attributeValueMatch.groups?.href;
+        if (href) {
+          addHref(paths, href, false);
+        }
+      }
+    }
+  }
+
+  // Also look for JS `import` statements that might be in <script type="module"> tags.
+  const scriptTags = document.querySelectorAll("script[type='module']");
+  for (const scriptTag of scriptTags) {
+    const jsPaths = pathsInJs(scriptTag.textContent);
+    paths.crawlablePaths.push(...jsPaths.crawlablePaths);
+  }
+
+  // Special handling for <noframes> in framesets. We need to use a regex for
+  // this because the jsdom parser supports frames, so it will treat a
+  // <noframes> tag as a text node.
+  const noframesRegex = /<noframes>(?<html>[\s\S]*?)<\/noframes>/g;
+  let match;
+  while ((match = noframesRegex.exec(html))) {
+    const noframesHtml = match.groups?.html;
+    if (noframesHtml) {
+      const noframesPaths = pathsInHtml(noframesHtml);
+      paths.crawlablePaths.push(...noframesPaths.crawlablePaths);
+      paths.resourcePaths.push(...noframesPaths.resourcePaths);
+    }
+  }
+
+  return paths;
+}

package/src/dev/crawler/pathsInImageMap.js

@@ -0,0 +1,25 @@
+import { normalizeHref } from "./utilities.js";
+
+// These are ancient server-side image maps. They're so old that it's hard to
+// find documentation on them, but they're used on the reference Space Jam
+// website we use for testing the crawler.
+//
+// Example: https://www.spacejam.com/1996/bin/bball.map
+export default function pathsInImageMap(imageMap) {
+  const resourcePaths = [];
+  let match;
+
+  // Find hrefs as the second column in each line.
+  const hrefRegex = /^\w+ (?<href>\S+)(\s*$| [\d, ]+$)/gm;
+  while ((match = hrefRegex.exec(imageMap))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths: [],
+    resourcePaths,
+  };
+}