@weborigami/origami 0.3.0 → 0.3.2

package/src/dev/crawler/pathsInJs.js

@@ -0,0 +1,140 @@
+/**
+ * Find static module references in JavaScript code.
+ *
+ * Matches:
+ *
+ * * `import … from "x"`
+ * * `import "x"`
+ * * `export … from "x"`
+ * * `export { … } from "x"`
+ *
+ * This does simple lexical analysis to avoid matching paths inside comments or
+ * string literals.
+ *
+ * @param {string} js
+ */
+export default function pathsInJs(js) {
+  return {
+    crawlablePaths: modulePaths(js),
+    resourcePaths: [],
+  };
+}
+
+function modulePaths(src) {
+  const tokens = Array.from(tokenize(src));
+  const paths = new Set();
+
+  for (let i = 0; i < tokens.length; i++) {
+    const t = tokens[i];
+
+    // static import
+    if (t.type === "Identifier" && t.value === "import") {
+      // look ahead for either:
+      //   import "mod"
+      //   import … from "mod"
+      let j = i + 1;
+      // skip any punctuation or identifiers until we hit 'from' or a StringLiteral
+      while (
+        j < tokens.length &&
+        tokens[j].type !== "StringLiteral" &&
+        !(tokens[j].type === "Identifier" && tokens[j].value === "from")
+      ) {
+        j++;
+      }
+      // import "mod"
+      if (tokens[j]?.type === "StringLiteral") {
+        paths.add(tokens[j].value);
+      } else if (
+        // import … from "mod"
+        tokens[j]?.value === "from" &&
+        tokens[j + 1]?.type === "StringLiteral"
+      ) {
+        paths.add(tokens[j + 1].value);
+      }
+    } else if (t.type === "Identifier" && t.value === "export") {
+      // re-export or export‐from
+
+      // find a 'from' token on the same statement
+      let j = i + 1;
+      while (
+        j < tokens.length &&
+        !(tokens[j].type === "Identifier" && tokens[j].value === "from")
+      ) {
+        // stop at semicolon so we don't run past the statement
+        if (tokens[j].type === "Punctuator" && tokens[j].value === ";") {
+          break;
+        }
+        j++;
+      }
+
+      if (
+        tokens[j]?.value === "from" &&
+        tokens[j + 1]?.type === "StringLiteral"
+      ) {
+        paths.add(tokens[j + 1].value);
+      }
+    }
+  }
+
+  return [...paths];
+}
+
+// Lexer emits Identifiers, StringLiterals, and Punctuators
+function* tokenize(src) {
+  let i = 0;
+  while (i < src.length) {
+    const c = src[i];
+
+    // Skip single‐line comments
+    if (c === "/" && src[i + 1] === "/") {
+      i += 2;
+      while (i < src.length && src[i] !== "\n") {
+        i++;
+      }
+    } else if (c === "/" && src[i + 1] === "*") {
+      // Skip multi‐line comments
+      i += 2;
+      while (i < src.length && !(src[i] === "*" && src[i + 1] === "/")) {
+        i++;
+      }
+      i += 2;
+      continue;
+    } else if (c === '"' || c === "'" || c === "`") {
+      // Skip string literals (but capture them)
+      const quote = c;
+      let start = i + 1;
+      i++;
+      while (i < src.length) {
+        if (src[i] === "\\") {
+          i += 2;
+          continue;
+        }
+        if (src[i] === quote) {
+          break;
+        }
+        i++;
+      }
+      const str = src.slice(start, i);
+      i++;
+      yield { type: "StringLiteral", value: str };
+      continue;
+    } else if (/[A-Za-z_$]/.test(c)) {
+      // Identifier
+      let start = i;
+      i++;
+      while (i < src.length && /[\w$]/.test(src[i])) {
+        i++;
+      }
+      yield { type: "Identifier", value: src.slice(start, i) };
+      continue;
+    } else if (/[{}();,]/.test(c)) {
+      // Punctuator (we still keep braces/semis for possible future use)
+      yield { type: "Punctuator", value: c };
+      i++;
+      continue;
+    } else {
+      // Skip everything else (whitespace, operators, etc.)
+      i++;
+    }
+  }
+}

package/src/dev/crawler/pathsInRobotsTxt.js

@@ -0,0 +1,20 @@
+import { normalizeHref } from "./utilities.js";
+
+export default function pathsInRobotsTxt(txt) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `Sitemap` directives.
+  const sitemapRegex = /Sitemap:\s*(?<href>[^\s]*)/g;
+  while ((match = sitemapRegex.exec(txt))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}

package/src/dev/crawler/pathsInSitemap.js

@@ -0,0 +1,20 @@
+import { normalizeHref } from "./utilities.js";
+
+export default function pathsInSitemap(xml) {
+  const crawlablePaths = [];
+  let match;
+
+  // Find `loc` elements.
+  const locRegex = /<loc>(?<href>[^<]*)<\/loc>/g;
+  while ((match = locRegex.exec(xml))) {
+    const href = normalizeHref(match.groups?.href);
+    if (href) {
+      crawlablePaths.push(href);
+    }
+  }
+
+  return {
+    crawlablePaths,
+    resourcePaths: [],
+  };
+}

package/src/dev/crawler/utilities.js

@@ -0,0 +1,125 @@
+import {
+  extension,
+  isPlainObject,
+  trailingSlash,
+} from "@weborigami/async-tree";
+
+// A fake base URL used to handle cases where an href is relative and must be
+// treated relative to some base URL.
+const fakeBaseUrl = new URL("fake:/");
+
+/**
+ * Destructively add a path to the paths object
+ */
+export function addHref(paths, href, isCrawlable) {
+  href = normalizeHref(href);
+  if (href === null) {
+    // Normalized href is null, was just an anchor or search; skip
+    return;
+  }
+  isCrawlable ??= isCrawlableHref(href);
+  if (isCrawlable) {
+    paths.crawlablePaths.push(href);
+  } else {
+    paths.resourcePaths.push(href);
+  }
+}
+
+/**
+ * Add the value to the object at the path given by the keys
+ *
+ * @param {any} object
+ * @param {string[]} keys
+ * @param {any} value
+ */
+export function addValueToObject(object, keys, value) {
+  for (let i = 0, current = object; i < keys.length; i++) {
+    const key = trailingSlash.remove(keys[i]);
+    if (i === keys.length - 1) {
+      // Write out value
+      if (isPlainObject(current[key])) {
+        // Route with existing values; treat the new value as an index.html
+        current[key]["index.html"] = value;
+      } else {
+        current[key] = value;
+      }
+    } else {
+      // Traverse further
+      if (!current[key]) {
+        current[key] = {};
+      } else if (!isPlainObject(current[key])) {
+        // Already have a value at this point. The site has a page at a route
+        // like /foo, and the site also has resources within that at routes like
+        // /foo/bar.jpg. We move the current value to "index.html".
+        current[key] = { "index.html": current[key] };
+      }
+      current = current[key];
+    }
+  }
+}
+
+/**
+ * Determine a URL we can use to determine whether a link is local within the
+ * tree or not.
+ *
+ * If a baseHref is supplied, convert that to a URL. If it's a relative path,
+ * use a fake base URL. If no baseHref is supplied, see if the `object`
+ * parameter defines an `href` property and use that to construct a URL.
+ *
+ * @param {string|undefined} baseHref
+ * @param {any} object
+ */
+export function getBaseUrl(baseHref, object) {
+  let url;
+  if (baseHref !== undefined) {
+    // See if the href is valid
+    try {
+      url = new URL(baseHref);
+    } catch (e) {
+      // Invalid, probably a path; use a fake protocol
+      url = new URL(baseHref, fakeBaseUrl);
+    }
+  } else if (object.href) {
+    // Use href property on object
+    let href = object.href;
+    if (!href?.endsWith("/")) {
+      href += "/";
+    }
+    url = new URL(href);
+  } else {
+    url = fakeBaseUrl;
+  }
+  return url;
+}
+
+export function isCrawlableHref(href) {
+  // Use a fake base URL to cover the case where the href is relative.
+  const url = new URL(href, fakeBaseUrl);
+  const pathname = url.pathname;
+  const lastKey = pathname.split("/").pop() ?? "";
+  if (lastKey === "robots.txt" || lastKey === "sitemap.xml") {
+    return true;
+  }
+  const ext = extension.extname(lastKey);
+  // We assume an empty extension is HTML.
+  const crawlableExtensions = [".html", ".css", ".js", ".map", ".xhtml", ""];
+  return crawlableExtensions.includes(ext);
+}
+
+// Remove any search parameters or hash from the href. Preserve absolute or
+// relative nature of URL. If the URL only has a search or hash, return null.
+export function normalizeHref(href) {
+  // Remove everything after a `#` or `?` character.
+  const normalized = href.split(/[?#]/)[0];
+  return normalized === "" ? null : normalized;
+}
+
+// For indexing and storage purposes, treat a path that ends in a trailing slash
+// as if it ends in index.html.
+export function normalizeKeys(keys) {
+  const normalized = keys.slice();
+  if (normalized.length === 0 || trailingSlash.has(normalized.at(-1))) {
+    normalized.push("index.html");
+  }
+  return normalized;
+}

package/src/dev/dev.js

@@ -1,6 +1,8 @@
 export { default as breakpoint } from "./breakpoint.js";
 export { default as changes } from "./changes.js";
 export { default as code } from "./code.js";
+export { default as audit } from "./crawler/audit.js";
+export { default as crawl } from "./crawler/crawl.js";
 export { default as debug } from "./debug.js";
 export { default as explore } from "./explore.js";
 export { default as log } from "./log.js";

package/src/handlers/handlers.js

@@ -14,6 +14,7 @@ import jpgHandler from "./jpg.handler.js";
 import jsonHandler from "./json.handler.js";
 import mdHandler from "./md.handler.js";
 import mjsHandler from "./mjs.handler.js";
+import tsHandler from "./ts.handler.js";
 import txtHandler from "./txt.handler.js";
 import xhtmlHandler from "./xhtml.handler.js";
 import ymlHandler from "./yml.handler.js";
@@ -31,6 +32,7 @@ export default {
   "mjs.handler": mjsHandler,
   "ori.handler": oriHandler,
   "oridocument.handler": oridocumentHandler,
+  "ts.handler": tsHandler,
   "txt.handler": txtHandler,
   "wasm.handler": wasmHandler,
   "xhtml.handler": xhtmlHandler,

package/src/handlers/ts.handler.js

@@ -0,0 +1 @@
+export { default as default } from "./js.handler.js";

package/src/help/help.yaml

@@ -1,12 +1,18 @@
 dev:
   description: Develop and debug Origami projects
   commands:
+    audit:
+      args: (tree)
+      description: Identify broken internal links and references
     breakpoint:
       args: (a)
       description: Break into the JavaScript debugger, then return a
     changes:
       args: (old, new)
       description: Return a tree of changes
+    crawl:
+      args: (tree, base)
+      description: A tree of a site's discoverable resources
     debug:
       args: (tree)
       description: Add debug features to the tree
@@ -213,12 +219,6 @@ scope:
 site:
   description: Add common website features
   commands:
-    audit:
-      args: (tree)
-      description: Identify broken internal links and references
-    crawl:
-      args: (tree, base)
-      description: A tree of a site's discoverable resources
     index:
       args: (tree)
       description: A default index.html page for the tree

package/src/origami/csv.js

@@ -12,11 +12,16 @@ import assertTreeIsDefined from "../common/assertTreeIsDefined.js";
  */
 export default async function csv(object) {
   assertTreeIsDefined(this, "origami:csv");
+  object = object ?? this;
+  if (object === undefined) {
+    return undefined;
+  }
   if (isUnpackable(object)) {
     object = await object.unpack();
   }
   const value = await toPlainValue(object);
-  const text = formatCsv(value);
+  const array = Array.isArray(value) ? value : Object.values(value);
+  const text = formatCsv(array);
   return text;
 }
 

package/src/site/site.js

@@ -1,5 +1,3 @@
-export { default as audit } from "./audit.js";
-export { default as crawl } from "./crawler/crawl.js";
 export { default as index } from "./index.js";
 export { default as jsonKeys } from "./jsonKeys.js";
 export { default as redirect } from "./redirect.js";

package/src/text/htmlDom.js

@@ -0,0 +1,6 @@
+import { JSDOM } from "jsdom";
+
+export default function htmlDom(html) {
+  const dom = JSDOM.fragment(html);
+  return dom;
+}

package/src/text/text.js

@@ -1,4 +1,5 @@
 export { taggedTemplateIndent as indent } from "@weborigami/language";
 export { default as document } from "./document.js";
+export { default as htmlDom } from "./htmlDom.js";
 export { default as inline } from "./inline.js";
 export { default as mdHtml } from "./mdHtml.js";

package/src/tree/map.js

@@ -1,13 +1,10 @@
 import {
-  cachedKeyFunctions,
-  extensionKeyFunctions,
   isPlainObject,
   isUnpackable,
   map as mapTransform,
 } from "@weborigami/async-tree";
 import getTreeArgument from "../common/getTreeArgument.js";
 import { toFunction } from "../common/utilities.js";
-import parseExtensions from "./parseExtensions.js";
 
 /**
  * Map a hierarchical tree of keys and values to a new tree of keys and values.
@@ -81,12 +78,6 @@ function extendedOptions(context, operation) {
   let keyFn = options.key;
   let inverseKeyFn = options.inverseKey;
 
-  if (extension && (keyFn || inverseKeyFn)) {
-    throw new TypeError(
-      `map: You can't specify extensions and also a key or inverseKey function`
-    );
-  }
-
   if (valueFn) {
     // @ts-ignore
     valueFn = toFunction(valueFn);
@@ -94,36 +85,23 @@ function extendedOptions(context, operation) {
     // Origami builtins can be used as value functions.
     // @ts-ignore
     const bound = valueFn.bind(context);
+    // Transfer sidecar functions
     // @ts-ignore
     Object.assign(bound, valueFn);
     valueFn = bound;
   }
 
-  if (extension) {
-    // Generate key/inverseKey functions from the extension
-    let { resultExtension, sourceExtension } = parseExtensions(extension);
-    const keyFns = extensionKeyFunctions(sourceExtension, resultExtension);
-    keyFn = keyFns.key;
-    inverseKeyFn = keyFns.inverseKey;
-  } else if (keyFn) {
-    // Extend the key function to include a value parameter
-    keyFn = extendKeyFn(keyFn);
-  } else {
-    // Use sidecar key/inverseKey functions if the valueFn defines them
-    keyFn = /** @type {any} */ (valueFn)?.key;
-    inverseKeyFn = /** @type {any} */ (valueFn)?.inverseKey;
-  }
-
-  if (keyFn && !inverseKeyFn) {
-    // Only keyFn was provided, so we need to generate the inverseKeyFn
-    const keyFns = cachedKeyFunctions(keyFn, deep);
-    keyFn = keyFns.key;
-    inverseKeyFn = keyFns.inverseKey;
+  if (!extension) {
+    if (keyFn) {
+      // Extend the key function to include a value parameter
+      keyFn = extendKeyFn(keyFn);
+    }
   }
 
   return {
     deep,
     description,
+    extension,
     inverseKey: inverseKeyFn,
     key: keyFn,
     needsSourceValue,

package/src/site/audit.js

@@ -1,19 +0,0 @@
-import { Tree } from "@weborigami/async-tree";
-import getTreeArgument from "../common/getTreeArgument.js";
-import crawl from "./crawler/crawl.js";
-
-/**
- * @this {import("@weborigami/types").AsyncTree|null}
- * @param {import("@weborigami/async-tree").Treelike} treelike
- */
-export default async function audit(treelike) {
-  const tree = await getTreeArgument(this, arguments, treelike, "site:audit");
-  const crawled = await crawl.call(this, tree);
-  let crawlErrorsJson = await crawled.get("crawl-errors.json");
-  if (!crawlErrorsJson) {
-    return undefined;
-  }
-  const errors = Tree.from(JSON.parse(crawlErrorsJson), { deep: true });
-  errors.parent = this;
-  return errors;
-}