@weborigami/origami 0.3.1 → 0.3.3-jse.1

package/src/internal.js

@@ -15,6 +15,10 @@ export { default as oriHandler } from "./handlers/ori.handler.js";
 
 export { default as oridocumentHandler } from "./handlers/oridocument.handler.js";
 
+export { default as jseHandler } from "./handlers/jse.handler.js";
+
+export { default as jsedocumentHandler } from "./handlers/jsedocument.handler.js";
+
 export { default as processUnpackedContent } from "./common/processUnpackedContent.js";
 
 export { default as wasmHandler } from "./handlers/wasm.handler.js";

package/src/js.js

@@ -1,38 +1,79 @@
-async function fetchWrapper(resource, options) {
-  const response = await fetch(resource, options);
-  return response.ok ? await response.arrayBuffer() : undefined;
-}
-
+/**
+ * The complete set of support JavaScript globals and global-like values.
+ *
+ * See
+ * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects.
+ * That page lists some things like `TypedArrays` which are not globals so are
+ * omitted here.
+ */
 export default {
+  AggregateError,
   Array,
+  ArrayBuffer,
+  Atomics,
   BigInt,
+  BigInt64Array,
+  BigUint64Array,
   Boolean,
+  DataView,
   Date,
   Error,
+  EvalError,
+  FinalizationRegistry,
+  Float32Array,
+  Float64Array,
+  Function,
   Infinity,
+  Int16Array,
+  Int32Array,
+  Int8Array,
   Intl,
+  // @ts-ignore Iterator does exist despite what TypeScript thinks
+  Iterator,
   JSON,
   Map,
   Math,
   NaN,
   Number,
   Object,
+  Promise,
+  Proxy,
+  RangeError,
+  ReferenceError,
+  Reflect,
   RegExp,
-  Response,
   Set,
+  SharedArrayBuffer,
   String,
   Symbol,
+  SyntaxError,
+  TypeError,
+  URIError,
+  Uint16Array,
+  Uint32Array,
+  Uint8Array,
+  Uint8ClampedArray,
+  WeakMap,
+  WeakRef,
+  WeakSet,
   decodeURI,
   decodeURIComponent,
   encodeURI,
   encodeURIComponent,
-  false: false,
-  fetch: fetchWrapper,
+  eval,
+  false: false, // treat like a global
+  fetch: fetchWrapper, // special case
+  globalThis,
   isFinite,
   isNaN,
-  null: null,
+  null: null, // treat like a global
   parseFloat,
   parseInt,
-  true: true,
-  undefined: undefined,
+  true: true, // treat like a global
+  undefined,
 };
+
+async function fetchWrapper(resource, options) {
+  const response = await fetch(resource, options);
+  return response.ok ? await response.arrayBuffer() : undefined;
+}

package/src/site/site.js

@@ -1,5 +1,3 @@
-export { default as audit } from "./audit.js";
-export { default as crawl } from "./crawler/crawl.js";
 export { default as index } from "./index.js";
 export { default as jsonKeys } from "./jsonKeys.js";
 export { default as redirect } from "./redirect.js";

package/src/text/htmlDom.js

@@ -0,0 +1,6 @@
+import { JSDOM } from "jsdom";
+
+export default function htmlDom(html) {
+  const dom = JSDOM.fragment(html);
+  return dom;
+}

package/src/text/text.js

@@ -1,4 +1,5 @@
 export { taggedTemplateIndent as indent } from "@weborigami/language";
 export { default as document } from "./document.js";
+export { default as htmlDom } from "./htmlDom.js";
 export { default as inline } from "./inline.js";
 export { default as mdHtml } from "./mdHtml.js";

package/src/calc/calc.js

@@ -1,81 +0,0 @@
-import { Tree } from "@weborigami/async-tree";
-import assertTreeIsDefined from "../common/assertTreeIsDefined.js";
-
-export function add(...args) {
-  console.warn(`Warning: "add" is deprecated. Use the "+" operator instead.`);
-  const numbers = args.map((arg) => Number(arg));
-  return numbers.reduce((acc, val) => acc + val, 0);
-}
-
-export function and(...args) {
-  console.warn(`Warning: "and" is deprecated. Use the "&&" operator instead.`);
-  return args.every((arg) => arg);
-}
-
-export function divide(a, b) {
-  console.warn(
-    `Warning: "divide" is deprecated. Use the "/" operator instead.`
-  );
-  return Number(a) / Number(b);
-}
-
-export function equals(a, b) {
-  console.warn(
-    `Warning: "equals" is deprecated. Use the "===" operator instead.`
-  );
-  return a === b;
-}
-
-/**
- * @typedef {import("@weborigami/types").AsyncTree} AsyncTree
- *
- * @this {AsyncTree|null}
- * @param {any} value
- * @param {any} trueResult
- * @param {any} [falseResult]
- */
-export async function ifBuiltin(value, trueResult, falseResult) {
-  console.warn(
-    `Warning: "if" is deprecated. Use the conditional "a ? b : c" operator instead.`
-  );
-
-  assertTreeIsDefined(this, "calc:if");
-  let condition = await value;
-  if (Tree.isAsyncTree(condition)) {
-    const keys = Array.from(await condition.keys());
-    condition = keys.length > 0;
-  }
-
-  // 0 is true, null/undefined/false is false
-  let result = condition || condition === 0 ? trueResult : falseResult;
-  if (typeof result === "function") {
-    result = await result.call(this);
-  }
-  return result;
-}
-ifBuiltin.key = "if";
-
-export function multiply(...args) {
-  console.warn(
-    `Warning: "multiply" is deprecated. Use the "*" operator instead.`
-  );
-  const numbers = args.map((arg) => Number(arg));
-  return numbers.reduce((acc, val) => acc * val, 1);
-}
-
-export function not(value) {
-  console.warn(`Warning: "not" is deprecated. Use the "!" operator instead.`);
-  return !value;
-}
-
-export function or(...args) {
-  console.warn(`Warning: "or" is deprecated. Use the "||" operator instead.`);
-  return args.find((arg) => arg);
-}
-
-export function subtract(a, b) {
-  console.warn(
-    `Warning: "subtract" is deprecated. Use the "-" operator instead.`
-  );
-  return Number(a) - Number(b);
-}

package/src/site/audit.js

@@ -1,19 +0,0 @@
-import { Tree } from "@weborigami/async-tree";
-import getTreeArgument from "../common/getTreeArgument.js";
-import crawl from "./crawler/crawl.js";
-
-/**
- * @this {import("@weborigami/types").AsyncTree|null}
- * @param {import("@weborigami/async-tree").Treelike} treelike
- */
-export default async function audit(treelike) {
-  const tree = await getTreeArgument(this, arguments, treelike, "site:audit");
-  const crawled = await crawl.call(this, tree);
-  let crawlErrorsJson = await crawled.get("crawl-errors.json");
-  if (!crawlErrorsJson) {
-    return undefined;
-  }
-  const errors = Tree.from(JSON.parse(crawlErrorsJson), { deep: true });
-  errors.parent = this;
-  return errors;
-}

package/src/site/crawler/findPaths.js

@@ -1,266 +0,0 @@
-import { extension, toString } from "@weborigami/async-tree";
-import { isCrawlableHref, normalizeHref } from "./utilities.js";
-
-// Filter the paths to those that are local to the site.
-function filterPaths(paths, baseUrl, localPath) {
-  // Convert paths to absolute URLs.
-  const localUrl = new URL(localPath, baseUrl);
-  const basePathname = baseUrl.pathname;
-  // @ts-ignore
-  const absoluteUrls = paths.map((path) => new URL(path, localUrl));
-
-  // Convert the absolute URLs to paths relative to the baseHref. If the URL
-  // points outside the tree rooted at the baseHref, the relative path will be
-  // null. We ignore the protocol in this test, because in practice sites often
-  // fumble the use of http and https, treating them interchangeably.
-  const relativePaths = absoluteUrls.map((url) => {
-    if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
-      const path = url.pathname.slice(basePathname.length);
-      // The process of creating the URLs will have escaped characters. We
-      // remove them. This has the side-effect of removing them if they existed
-      // in the original path; it would be better if we avoided that.
-      return decodeURIComponent(path);
-    } else {
-      return null;
-    }
-  });
-
-  // Filter out the null paths.
-  /** @type {string[]} */
-  // @ts-ignore
-  const filteredPaths = relativePaths.filter((path) => path);
-  return filteredPaths;
-}
-
-/**
- * Given a value retrieved from a site using a given key (name), determine what
- * kind of file it is and, based on that, find the paths it references.
- */
-export default function findPaths(value, key, baseUrl, localPath) {
-  const text = toString(value);
-
-  // We guess the value is HTML is if its key has an .html extension or
-  // doesn't have an extension, or the value starts with `<`.
-  const ext = key ? extension.extname(key).toLowerCase() : "";
-  let foundPaths;
-  if (ext === ".html" || ext === ".htm" || ext === ".xhtml") {
-    foundPaths = findPathsInHtml(text);
-  } else if (ext === ".css") {
-    foundPaths = findPathsInCss(text);
-  } else if (ext === ".js") {
-    foundPaths = findPathsInJs(text);
-  } else if (ext === ".map") {
-    foundPaths = findPathsInImageMap(text);
-  } else if (key === "robots.txt") {
-    foundPaths = findPathsInRobotsTxt(text);
-  } else if (key === "sitemap.xml") {
-    foundPaths = findPathsInSitemapXml(text);
-  } else if (ext === "" && text?.trim().startsWith("<")) {
-    // Probably HTML
-    foundPaths = findPathsInHtml(text);
-  } else {
-    // Doesn't have an extension we want to process
-    return {
-      crawlablePaths: [],
-      resourcePaths: [],
-    };
-  }
-
-  const crawlablePaths = filterPaths(
-    foundPaths.crawlablePaths,
-    baseUrl,
-    localPath
-  );
-
-  const resourcePaths = filterPaths(
-    foundPaths.resourcePaths,
-    baseUrl,
-    localPath
-  );
-
-  return {
-    crawlablePaths,
-    resourcePaths,
-  };
-}
-
-function findPathsInCss(css) {
-  const resourcePaths = [];
-  let match;
-
-  // Find `url()` functions.
-  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
-  while ((match = urlRegex.exec(css))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths: [],
-    resourcePaths,
-  };
-}
-
-// These are ancient server-side image maps. They're so old that it's hard to
-// find documentation on them, but they're used on the reference Space Jam
-// website we use for testing the crawler. Example:
-// https://www.spacejam.com/1996/bin/bball.map
-function findPathsInImageMap(imageMap) {
-  const resourcePaths = [];
-  let match;
-
-  // Find hrefs as the second column in each line.
-  const hrefRegex = /^\w+ (?<href>\S+)(\s*$| [\d, ]+$)/gm;
-  while ((match = hrefRegex.exec(imageMap))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths: [],
-    resourcePaths,
-  };
-}
-
-function findPathsInJs(js) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `import` statements.
-  const importRegex = /import [\s\S]+?from\s+["'](?<import>[^"']*)["'];/g;
-  while ((match = importRegex.exec(js))) {
-    const href = normalizeHref(match.groups?.import);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function findPathsInHtml(html) {
-  const crawlablePaths = [];
-  const resourcePaths = [];
-  let match;
-
-  // Find `href` attributes in anchor and link tags.
-  const linkRegex =
-    /<(?:a|A|link|LINK)[\s][^>]*?(?:href|HREF)=["'](?<link>[^>]*?)["'][^>]*>/g;
-  while ((match = linkRegex.exec(html))) {
-    // Links can point to be other crawlable paths and resource paths.
-    // We guess the type based on the extension.
-    const href = normalizeHref(match.groups?.link);
-    if (href) {
-      if (isCrawlableHref(href)) {
-        crawlablePaths.push(href);
-      } else {
-        resourcePaths.push(href);
-      }
-    }
-  }
-
-  // Find `src` attributes in img and script tags.
-  const srcRegex =
-    /<(?<tag>img|IMG|script|SCRIPT)[\s][^>]*?(?:src|SRC)=["'](?<src>[^>]*?)["'][^>]*>/g;
-  while ((match = srcRegex.exec(html))) {
-    const tag = match.groups?.tag;
-    const src = normalizeHref(match.groups?.src);
-    if (src) {
-      if (tag === "script" || tag === "SCRIPT") {
-        crawlablePaths.push(src);
-      } else {
-        resourcePaths.push(src);
-      }
-    }
-  }
-
-  // Find `url()` functions in CSS.
-  const urlRegex = /url\(["']?(?<href>[^"')]*?)["']?\)/g;
-  while ((match = urlRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  // Find `src` attribute on frame tags.
-  const frameRegex =
-    /<(?:frame|FRAME)[\s][^>]*?(?:src|SRC)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = frameRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  // Find ancient `background` attribute on body tag.
-  const backgroundRegex =
-    /<(?:body|BODY)[\s][^>]*?(?:background|BACKGROUND)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = backgroundRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      resourcePaths.push(href);
-    }
-  }
-
-  // Find `href` attribute on area tags.
-  const areaRegex =
-    /<(?:area|AREA)[\s][^>]*?(?:href|HREF)=["'](?<href>[^>]*?)["'][^>]*>/g;
-  while ((match = areaRegex.exec(html))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  // Also look for JS `import` statements that might be in <script type="module"> tags.
-  const jsResults = findPathsInJs(html);
-  crawlablePaths.push(...jsResults.crawlablePaths);
-
-  return { crawlablePaths, resourcePaths };
-}
-
-function findPathsInRobotsTxt(txt) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `Sitemap` directives.
-  const sitemapRegex = /Sitemap:\s*(?<href>[^\s]*)/g;
-  while ((match = sitemapRegex.exec(txt))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}
-
-function findPathsInSitemapXml(xml) {
-  const crawlablePaths = [];
-  let match;
-
-  // Find `loc` elements.
-  const locRegex = /<loc>(?<href>[^<]*)<\/loc>/g;
-  while ((match = locRegex.exec(xml))) {
-    const href = normalizeHref(match.groups?.href);
-    if (href) {
-      crawlablePaths.push(href);
-    }
-  }
-
-  return {
-    crawlablePaths,
-    resourcePaths: [],
-  };
-}

package/src/site/crawler/utilities.js

@@ -1,37 +0,0 @@
-import { extension, trailingSlash } from "@weborigami/async-tree";
-
-// A fake base URL used to handle cases where an href is relative and must be
-// treated relative to some base URL.
-const fakeBaseUrl = new URL("https://fake");
-
-export function isCrawlableHref(href) {
-  // Use a fake base URL to cover the case where the href is relative.
-  const url = new URL(href, fakeBaseUrl);
-  const pathname = url.pathname;
-  const lastKey = pathname.split("/").pop() ?? "";
-  if (lastKey === "robots.txt" || lastKey === "sitemap.xml") {
-    return true;
-  }
-  const ext = extension.extname(lastKey);
-  // We assume an empty extension is HTML.
-  const crawlableExtensions = [".html", ".css", ".js", ".map", ".xhtml", ""];
-  return crawlableExtensions.includes(ext);
-}
-
-// Remove any search parameters or hash from the href. Preserve absolute or
-// relative nature of URL. If the URL only has a search or hash, return null.
-export function normalizeHref(href) {
-  // Remove everything after a `#` or `?` character.
-  const normalized = href.split(/[?#]/)[0];
-  return normalized === "" ? null : normalized;
-}
-
-// For indexing and storage purposes, treat a path that ends in a trailing slash
-// as if it ends in index.html.
-export function normalizeKeys(keys) {
-  const normalized = keys.slice();
-  if (normalized.length === 0 || trailingSlash.has(normalized.at(-1))) {
-    normalized.push("index.html");
-  }
-  return normalized;
-}