@weborigami/origami 0.3.1 → 0.3.3-jse.1

package/src/dev/crawler/pathsInHtml.js

@@ -0,0 +1,161 @@
+import { JSDOM, VirtualConsole } from "jsdom";
+import pathsInCss from "./pathsInCss.js";
+import pathsInJs from "./pathsInJs.js";
+import { addHref } from "./utilities.js";
+
+export default function pathsInHtml(html) {
+  const paths = {
+    crawlablePaths: [],
+    resourcePaths: [],
+  };
+
+  // Create a virtual console to avoid logging errors to the console
+  const virtualConsole = new VirtualConsole();
+  const document = new JSDOM(html, { virtualConsole }).window.document;
+
+  // Find `href` attributes in anchor, area, link, SVG tags.
+  //
+  // NOTE: As of April 2024, jsdom querySelectorAll does not appear to find
+  // elements with mixed-case tag names.
+  const hrefTags = document.querySelectorAll(
+    "a[href], area[href], image[href], feImage[href], filter[href], linearGradient[href], link[href], mpath[href], pattern[href], radialGradient[href], textPath[href], use[href]"
+  );
+  for (const hrefTag of hrefTags) {
+    const crawlable = ["A", "AREA"].includes(hrefTag.tagName)
+      ? true
+      : undefined;
+    addHref(paths, hrefTag.getAttribute("href"), crawlable);
+  }
+
+  // Find `src` attributes in input, frame, media, and script tags.
+  const srcTags = document.querySelectorAll(
+    "audio[src], embed[src], frame[src], iframe[src], img[src], input[src], script[src], source[src], track[src], video[src]"
+  );
+  for (const srcTag of srcTags) {
+    const crawlable = ["FRAME", "IFRAME"].includes(srcTag.tagName)
+      ? true
+      : srcTag.tagName === "SCRIPT"
+      ? srcTag.type === "module" // Only crawl modules
+      : undefined;
+    addHref(paths, srcTag.getAttribute("src"), crawlable);
+  }
+
+  // Find `srcset` attributes in image and source tags.
+  const srcsetTags = document.querySelectorAll("img[srcset], source[srcset]");
+  for (const srcsetTag of srcsetTags) {
+    const srcset = srcsetTag.getAttribute("srcset");
+    const srcRegex = /(?<url>[^\s,]+)(?=\s+\d+(?:\.\d+)?[wxh])/g;
+    let match;
+    while ((match = srcRegex.exec(srcset))) {
+      if (match.groups?.url) {
+        addHref(paths, match.groups.url, false);
+      }
+    }
+  }
+
+  // Find `poster` attributes in <video> tags.
+  const posterTags = document.querySelectorAll("video[poster]");
+  for (const posterTag of posterTags) {
+    addHref(paths, posterTag.getAttribute("poster"), false);
+  }
+
+  // Find `data` attributes in <object> tags.
+  const objectTags = document.querySelectorAll("object[data]");
+  for (const objectTag of objectTags) {
+    addHref(paths, objectTag.getAttribute("data"), false);
+  }
+
+  // Find deprecated `background` attribute on body and table tags.
+  const backgroundTags = document.querySelectorAll(
+    "body[background], table[background], td[background], th[background]"
+  );
+  for (const backgroundTag of backgroundTags) {
+    addHref(paths, backgroundTag.getAttribute("background"), false);
+  }
+
+  // Find deprecated `longdesc` attributes on <img> tags.
+  const longdescTags = document.querySelectorAll("img[longdesc]");
+  for (const longdescTag of longdescTags) {
+    addHref(paths, longdescTag.getAttribute("longdesc"), false);
+  }
+
+  // Find paths in <meta> image tags.
+  const imageMetaTags = document.querySelectorAll('meta[property$=":image"]');
+  for (const imageMetaTag of imageMetaTags) {
+    const content = imageMetaTag.getAttribute("content");
+    if (content) {
+      addHref(paths, content, false);
+    }
+  }
+
+  // Find paths in CSS in <style> tags.
+  const styleTags = document.querySelectorAll("style");
+  for (const styleAttribute of styleTags) {
+    const cssPaths = pathsInCss(styleAttribute.textContent);
+    paths.crawlablePaths.push(...cssPaths.crawlablePaths);
+    paths.resourcePaths.push(...cssPaths.resourcePaths);
+  }
+
+  // Find URLs in CSS in `style` attributes.
+  const styleAttributeTags = document.querySelectorAll("[style]");
+  for (const tag of styleAttributeTags) {
+    const style = tag.getAttribute("style");
+    const stylePaths = pathsInCss(style, "declarationList");
+    stylePaths.resourcePaths.forEach((href) => {
+      addHref(paths, href, false);
+    });
+  }
+
+  // Find URLs in SVG attributes.
+  const svgAttributeNames = [
+    "clip-path",
+    "fill",
+    "filter",
+    "marker-end",
+    "marker-start",
+    "mask",
+    "stroke",
+  ];
+  const svgTags = document.querySelectorAll(
+    svgAttributeNames.map((name) => `[${name}]`).join(", ")
+  );
+  for (const svgTag of svgTags) {
+    for (const name of svgAttributeNames) {
+      const attributeValue = svgTag.getAttribute(name);
+      if (!attributeValue) {
+        continue;
+      }
+      const urlRegex = /url\((['"]?)(?<href>.*?)\1\)/g;
+      const attributeValueMatch = urlRegex.exec(attributeValue);
+      if (attributeValueMatch) {
+        const href = attributeValueMatch.groups?.href;
+        if (href) {
+          addHref(paths, href, false);
+        }
+      }
+    }
+  }
+
+  // Also look for JS `import` statements that might be in <script type="module"> tags.
+  const scriptTags = document.querySelectorAll("script[type='module']");
+  for (const scriptTag of scriptTags) {
+    const jsPaths = pathsInJs(scriptTag.textContent);
+    paths.crawlablePaths.push(...jsPaths.crawlablePaths);
+  }
+
+  // Special handling for <noframes> in framesets. We need to use a regex for
+  // this because the jsdom parser supports frames, so it will treat a
+  // <noframes> tag as a text node.
+  const noframesRegex = /<noframes>(?<html>[\s\S]*?)<\/noframes>/g;
+  let match;
+  while ((match = noframesRegex.exec(html))) {
+    const noframesHtml = match.groups?.html;
+    if (noframesHtml) {
+      const noframesPaths = pathsInHtml(noframesHtml);
+      paths.crawlablePaths.push(...noframesPaths.crawlablePaths);
+      paths.resourcePaths.push(...noframesPaths.resourcePaths);
+    }
+  }
+
+  return paths;
+}

package/src/dev/crawler/pathsInImageMap.js

@@ -0,0 +1,25 @@
+import { normalizeHref } from "./utilities.js";
+
+// These are ancient server-side image maps. They're so old that it's hard to
+// find documentation on them, but they're used on the reference Space Jam
+// website we use for testing the crawler.
+//
+// Example: https://www.spacejam.com/1996/bin/bball.map
+export default function pathsInImageMap(imageMap) {
+  const resourcePaths = [];
+  let match;
+
+  // Find hrefs as the second column in each line.
+  const hrefRegex = /^\w+ (?<href>\S+)(\s*$| [\d, ]+$)/gm;
+  while ((match = hrefRegex.exec(imageMap))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      resourcePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths: [],
+    resourcePaths,
+  };
+}

package/src/dev/crawler/pathsInJs.js

@@ -0,0 +1,140 @@
+/**
+ * Find static module references in JavaScript code.
+ *
+ * Matches:
+ *
+ * * `import … from "x"`
+ * * `import "x"`
+ * * `export … from "x"`
+ * * `export { … } from "x"`
+ *
+ * This does simple lexical analysis to avoid matching paths inside comments or
+ * string literals.
+ *
+ * @param {string} js
+ */
+export default function pathsInJs(js) {
+  return {
+    crawlablePaths: modulePaths(js),
+    resourcePaths: [],
+  };
+}
+
+function modulePaths(src) {
+  const tokens = Array.from(tokenize(src));
+  const paths = new Set();
+
+  for (let i = 0; i < tokens.length; i++) {
+    const t = tokens[i];
+
+    // static import
+    if (t.type === "Identifier" && t.value === "import") {
+      // look ahead for either:
+      //   import "mod"
+      //   import … from "mod"
+      let j = i + 1;
+      // skip any punctuation or identifiers until we hit 'from' or a StringLiteral
+      while (
+        j < tokens.length &&
+        tokens[j].type !== "StringLiteral" &&
+        !(tokens[j].type === "Identifier" && tokens[j].value === "from")
+      ) {
+        j++;
+      }
+      // import "mod"
+      if (tokens[j]?.type === "StringLiteral") {
+        paths.add(tokens[j].value);
+      } else if (
+        // import … from "mod"
+        tokens[j]?.value === "from" &&
+        tokens[j + 1]?.type === "StringLiteral"
+      ) {
+        paths.add(tokens[j + 1].value);
+      }
+    } else if (t.type === "Identifier" && t.value === "export") {
+      // re-export or export‐from
+
+      // find a 'from' token on the same statement
+      let j = i + 1;
+      while (
+        j < tokens.length &&
+        !(tokens[j].type === "Identifier" && tokens[j].value === "from")
+      ) {
+        // stop at semicolon so we don't run past the statement
+        if (tokens[j].type === "Punctuator" && tokens[j].value === ";") {
+          break;
+        }
+        j++;
+      }
+
+      if (
+        tokens[j]?.value === "from" &&
+        tokens[j + 1]?.type === "StringLiteral"
+      ) {
+        paths.add(tokens[j + 1].value);
+      }
+    }
+  }
+
+  return [...paths];
+}
+
+// Lexer emits Identifiers, StringLiterals, and Punctuators
+function* tokenize(src) {
+  let i = 0;
+  while (i < src.length) {
+    const c = src[i];
+
+    // Skip single‐line comments
+    if (c === "/" && src[i + 1] === "/") {
+      i += 2;
+      while (i < src.length && src[i] !== "\n") {
+        i++;
+      }
+    } else if (c === "/" && src[i + 1] === "*") {
+      // Skip multi‐line comments
+      i += 2;
+      while (i < src.length && !(src[i] === "*" && src[i + 1] === "/")) {
+        i++;
+      }
+      i += 2;
+      continue;
+    } else if (c === '"' || c === "'" || c === "`") {
+      // Skip string literals (but capture them)
+      const quote = c;
+      let start = i + 1;
+      i++;
+      while (i < src.length) {
+        if (src[i] === "\\") {
+          i += 2;
+          continue;
+        }
+        if (src[i] === quote) {
+          break;
+        }
+        i++;
+      }
+      const str = src.slice(start, i);
+      i++;
+      yield { type: "StringLiteral", value: str };
+      continue;
+    } else if (/[A-Za-z_$]/.test(c)) {
+      // Identifier
+      let start = i;
+      i++;
+      while (i < src.length && /[\w$]/.test(src[i])) {
+        i++;
+      }
+      yield { type: "Identifier", value: src.slice(start, i) };
+      continue;
+    } else if (/[{}();,]/.test(c)) {
+      // Punctuator (we still keep braces/semis for possible future use)
+      yield { type: "Punctuator", value: c };
+      i++;
+      continue;
+    } else {
+      // Skip everything else (whitespace, operators, etc.)
+      i++;
+    }
+  }
+}

package/src/dev/crawler/pathsInRobotsTxt.js

@@ -0,0 +1,20 @@
+import { normalizeHref } from "./utilities.js";
+
+export default function pathsInRobotsTxt(txt) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `Sitemap` directives.
+  const sitemapRegex = /Sitemap:\s*(?<href>[^\s]*)/g;
+  while ((match = sitemapRegex.exec(txt))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}

package/src/dev/crawler/pathsInSitemap.js

@@ -0,0 +1,20 @@
+import { normalizeHref } from "./utilities.js";
+
+export default function pathsInSitemap(xml) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `loc` elements.
+  const locRegex = /<loc>(?<href>[^<]*)<\/loc>/g;
+  while ((match = locRegex.exec(xml))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}

package/src/dev/crawler/utilities.js

@@ -0,0 +1,125 @@
+import {
+  extension,
+  isPlainObject,
+  trailingSlash,
+} from "@weborigami/async-tree";
+
+// A fake base URL used to handle cases where an href is relative and must be
+// treated relative to some base URL.
+const fakeBaseUrl = new URL("fake:/");
+
+/**
+ * Destructively add a path to the paths object
+ */
+export function addHref(paths, href, isCrawlable) {
+  href = normalizeHref(href);
+  if (href === null) {
+    // Normalized href is null, was just an anchor or search; skip
+    return;
+  }
+  isCrawlable ??= isCrawlableHref(href);
+  if (isCrawlable) {
+    paths.crawlablePaths.push(href);
+  } else {
+    paths.resourcePaths.push(href);
+  }
+}
+
+/**
+ * Add the value to the object at the path given by the keys
+ *
+ * @param {any} object
+ * @param {string[]} keys
+ * @param {any} value
+ */
+export function addValueToObject(object, keys, value) {
+  for (let i = 0, current = object; i < keys.length; i++) {
+    const key = trailingSlash.remove(keys[i]);
+    if (i === keys.length - 1) {
+      // Write out value
+      if (isPlainObject(current[key])) {
+        // Route with existing values; treat the new value as an index.html
+        current[key]["index.html"] = value;
+      } else {
+        current[key] = value;
+      }
+    } else {
+      // Traverse further
+      if (!current[key]) {
+        current[key] = {};
+      } else if (!isPlainObject(current[key])) {
+        // Already have a value at this point. The site has a page at a route
+        // like /foo, and the site also has resources within that at routes like
+        // /foo/bar.jpg. We move the current value to "index.html".
+        current[key] = { "index.html": current[key] };
+      }
+      current = current[key];
+    }
+  }
+}
+
+/**
+ * Determine a URL we can use to determine whether a link is local within the
+ * tree or not.
+ *
+ * If a baseHref is supplied, convert that to a URL. If it's a relative path,
+ * use a fake base URL. If no baseHref is supplied, see if the `object`
+ * parameter defines an `href` property and use that to construct a URL.
+ *
+ * @param {string|undefined} baseHref
+ * @param {any} object
+ */
+export function getBaseUrl(baseHref, object) {
+  let url;
+  if (baseHref !== undefined) {
+    // See if the href is valid
+    try {
+      url = new URL(baseHref);
+    } catch (e) {
+      // Invalid, probably a path; use a fake protocol
+      url = new URL(baseHref, fakeBaseUrl);
+    }
+  } else if (object.href) {
+    // Use href property on object
+    let href = object.href;
+    if (!href?.endsWith("/")) {
+      href += "/";
+    }
+    url = new URL(href);
+  } else {
+    url = fakeBaseUrl;
+  }
+  return url;
+}
+
+export function isCrawlableHref(href) {
+  // Use a fake base URL to cover the case where the href is relative.
+  const url = new URL(href, fakeBaseUrl);
+  const pathname = url.pathname;
+  const lastKey = pathname.split("/").pop() ?? "";
+  if (lastKey === "robots.txt" || lastKey === "sitemap.xml") {
+    return true;
+  }
+  const ext = extension.extname(lastKey);
+  // We assume an empty extension is HTML.
+  const crawlableExtensions = [".html", ".css", ".js", ".map", ".xhtml", ""];
+  return crawlableExtensions.includes(ext);
+}
+
+// Remove any search parameters or hash from the href. Preserve absolute or
+// relative nature of URL. If the URL only has a search or hash, return null.
+export function normalizeHref(href) {
+  // Remove everything after a `#` or `?` character.
+  const normalized = href.split(/[?#]/)[0];
+  return normalized === "" ? null : normalized;
+}
+
+// For indexing and storage purposes, treat a path that ends in a trailing slash
+// as if it ends in index.html.
+export function normalizeKeys(keys) {
+  const normalized = keys.slice();
+  if (normalized.length === 0 || trailingSlash.has(normalized.at(-1))) {
+    normalized.push("index.html");
+  }
+  return normalized;
+}

package/src/dev/dev.js

@@ -1,6 +1,8 @@
 export { default as breakpoint } from "./breakpoint.js";
 export { default as changes } from "./changes.js";
 export { default as code } from "./code.js";
+export { default as audit } from "./crawler/audit.js";
+export { default as crawl } from "./crawler/crawl.js";
 export { default as debug } from "./debug.js";
 export { default as explore } from "./explore.js";
 export { default as log } from "./log.js";

package/src/handlers/handlers.js

@@ -11,9 +11,12 @@ import htmHandler from "./htm.handler.js";
 import htmlHandler from "./html.handler.js";
 import jpegHandler from "./jpeg.handler.js";
 import jpgHandler from "./jpg.handler.js";
+import jseHandler from "./jse.handler.js";
+import jseDocumentHandler from "./jsedocument.handler.js";
 import jsonHandler from "./json.handler.js";
 import mdHandler from "./md.handler.js";
 import mjsHandler from "./mjs.handler.js";
+import tsHandler from "./ts.handler.js";
 import txtHandler from "./txt.handler.js";
 import xhtmlHandler from "./xhtml.handler.js";
 import ymlHandler from "./yml.handler.js";
@@ -26,11 +29,15 @@ export default {
   "jpeg.handler": jpegHandler,
   "jpg.handler": jpgHandler,
   "js.handler": jsHandler,
+  "jse.handler": jseHandler,
+  "jsep.handler": jseHandler,
+  "jsedocument.handler": jseDocumentHandler,
   "json.handler": jsonHandler,
   "md.handler": mdHandler,
   "mjs.handler": mjsHandler,
   "ori.handler": oriHandler,
   "oridocument.handler": oridocumentHandler,
+  "ts.handler": tsHandler,
   "txt.handler": txtHandler,
   "wasm.handler": wasmHandler,
   "xhtml.handler": xhtmlHandler,

package/src/handlers/jse.handler.js

@@ -0,0 +1,16 @@
+import { oriHandler } from "../internal.js";
+import getParent from "./getParent.js";
+import jseModeParent from "./jseModeParent.js";
+
+export default {
+  ...oriHandler,
+
+  async unpack(packed, options = {}) {
+    const parent = getParent(packed, options);
+    return oriHandler.unpack(packed, {
+      ...options,
+      mode: "jse",
+      parent: await jseModeParent(parent),
+    });
+  },
+};

package/src/handlers/jseModeParent.js

@@ -0,0 +1,30 @@
+import { FileTree, ObjectTree } from "@weborigami/async-tree";
+
+let builtinsNew;
+
+// Adapt the existing parent chain to use the new builtins
+export default async function jseModeParent(parent) {
+  builtinsNew ??= (await import("../builtinsNew.js")).default;
+  return cloneParent(parent);
+}
+
+function cloneParent(parent) {
+  let clone;
+  // We expect the parent to be a FileTree (or a subclass), ObjectTree (or a
+  // subclass), or builtins.
+  if (!parent) {
+    return null;
+  } else if (parent instanceof FileTree) {
+    clone = Reflect.construct(parent.constructor, [parent.path]);
+  } else if (parent instanceof ObjectTree) {
+    clone = Reflect.construct(parent.constructor, [parent.object]);
+  } else if (!parent.parent) {
+    // Builtins
+    clone = builtinsNew;
+  } else {
+    // Maybe a map? Skip it and hope for the best.
+    return cloneParent(parent.parent);
+  }
+  clone.parent = cloneParent(parent.parent);
+  return clone;
+}

package/src/handlers/jsedocument.handler.js

@@ -0,0 +1,16 @@
+import { oridocumentHandler } from "../internal.js";
+import getParent from "./getParent.js";
+import jseModeParent from "./jseModeParent.js";
+
+export default {
+  ...oridocumentHandler,
+
+  async unpack(packed, options = {}) {
+    const parent = getParent(packed, options);
+    return oridocumentHandler.unpack(packed, {
+      ...options,
+      mode: "jse",
+      parent: await jseModeParent(parent),
+    });
+  },
+};

package/src/handlers/ori.handler.js

@@ -34,7 +34,8 @@ export default {
 
     // Compile the source code as an Origami program and evaluate it.
     const compiler = options.compiler ?? compile.program;
-    const fn = compiler(source);
+    const mode = options.mode ?? "shell";
+    const fn = compiler(source, { mode });
     const target = parent ?? builtinsTree;
     let content = await fn.call(target);
 

package/src/handlers/oridocument.handler.js

@@ -35,7 +35,8 @@ export default {
       text,
       url,
     };
-    const defineFn = compile.templateDocument(source);
+    const mode = options.mode ?? "shell";
+    const defineFn = compile.templateDocument(source, { mode });
 
     // Invoke the definition to get back the template function
     const result = await defineFn.call(parent);

package/src/handlers/ts.handler.js

@@ -0,0 +1 @@
+export { default as default } from "./js.handler.js";

package/src/handlers/txt.handler.js

@@ -39,7 +39,8 @@ export default {
       throw new TypeError("The input to pack must be a JavaScript object.");
     }
 
-    const text = object["@text"] ?? "";
+    // TODO: Deprecate @text
+    const text = object._body ?? object["@text"] ?? "";
 
     /** @type {any} */
     const dataWithoutText = Object.assign({}, object);
@@ -72,7 +73,14 @@ export default {
       } else {
         frontData = parseYaml(frontText);
       }
+      // TODO: Deprecate @text
       unpacked = Object.assign({}, frontData, { "@text": body });
+      Object.defineProperty(unpacked, "_body", {
+        configurable: true,
+        value: text,
+        enumerable: false, // TODO: Make enumerable
+        writable: true,
+      });
     } else {
       // Plain text
       unpacked = new String(text);

package/src/help/help.yaml

@@ -1,12 +1,18 @@
 dev:
   description: Develop and debug Origami projects
   commands:
+    audit:
+      args: (tree)
+      description: Identify broken internal links and references
     breakpoint:
       args: (a)
       description: Break into the JavaScript debugger, then return a
     changes:
       args: (old, new)
       description: Return a tree of changes
+    crawl:
+      args: (tree, base)
+      description: A tree of a site's discoverable resources
     debug:
       args: (tree)
       description: Add debug features to the tree
@@ -213,12 +219,6 @@ scope:
 site:
   description: Add common website features
   commands:
-    audit:
-      args: (tree)
-      description: Identify broken internal links and references
-    crawl:
-      args: (tree, base)
-      description: A tree of a site's discoverable resources
     index:
       args: (tree)
       description: A default index.html page for the tree