@weborigami/origami 0.3.1 → 0.3.3-jse.1

package/main.js

@@ -1,4 +1,3 @@
-export * from "./src/calc/calc.js";
 export { default as documentObject } from "./src/common/documentObject.js";
 export { toString } from "./src/common/utilities.js";
 export * from "./src/dev/dev.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@weborigami/origami",
-  "version": "0.3.1",
+  "version": "0.3.3-jse.1",
   "description": "Web Origami language, CLI, framework, and server",
   "type": "module",
   "repository": {
@@ -17,13 +17,15 @@
     "typescript": "5.8.2"
   },
   "dependencies": {
-    "@weborigami/async-tree": "0.3.1",
-    "@weborigami/language": "0.3.1",
+    "@weborigami/async-tree": "0.3.3-jse.1",
     "@weborigami/json-feed-to-rss": "1.0.0",
-    "@weborigami/types": "0.3.1",
+    "@weborigami/language": "0.3.3-jse.1",
+    "@weborigami/types": "0.3.3-jse.1",
+    "css-tree": "3.1.0",
     "exif-parser": "0.1.12",
     "graphviz-wasm": "3.0.2",
     "highlight.js": "11.11.1",
+    "jsdom": "26.1.0",
     "marked": "15.0.7",
     "marked-gfm-heading-id": "4.1.1",
     "marked-highlight": "2.2.1",

package/src/builtins.js

@@ -1,4 +1,3 @@
-import * as calc from "./calc/calc.js";
 import * as dev from "./dev/dev.js";
 import * as handlers from "./handlers/handlers.js";
 import help from "./help/help.js";
@@ -22,7 +21,6 @@ import * as tree from "./tree/tree.js";
 
 /** @type {any} */
 export default {
-  "calc:": adjustReservedWords(calc),
   "dev:": dev,
   "explore:": explore,
   "files:": files,

package/src/builtinsNew.js

@@ -0,0 +1,84 @@
+import {
+  ObjectTree,
+  trailingSlash,
+  text as treeText,
+} from "@weborigami/async-tree";
+import * as dev from "./dev/dev.js";
+import * as handlers from "./handlers/handlers.js";
+import help from "./help/help.js";
+import * as image from "./image/image.js";
+import js from "./js.js";
+import node from "./node.js";
+import * as origami from "./origami/origami.js";
+import explore from "./protocols/explore.js";
+import files from "./protocols/files.js";
+import http from "./protocols/http.js";
+import https from "./protocols/https.js";
+import httpstree from "./protocols/httpstree.js";
+import httptree from "./protocols/httptree.js";
+import inherited from "./protocols/inherited.js";
+import instantiate from "./protocols/new.js";
+import packageNamespace from "./protocols/package.js";
+import scope from "./protocols/scope.js";
+import * as site from "./site/site.js";
+import * as text from "./text/text.js";
+import * as tree from "./tree/tree.js";
+
+// See notes in builtinsTree.js
+class BuiltinsTree {
+  constructor(object) {
+    this.object = object;
+  }
+
+  async get(key) {
+    const normalizedKey = trailingSlash.remove(key);
+    return this.object[normalizedKey];
+  }
+
+  async keys() {
+    return Object.keys(this.object);
+  }
+}
+
+const Tree = new BuiltinsTree({
+  ...tree,
+  indent: text.indent,
+  json: origami.json,
+  text: treeText,
+});
+
+const Origami = new BuiltinsTree({
+  ...dev,
+  ...origami,
+  ...site,
+  ...text,
+});
+
+const Image = new BuiltinsTree({
+  ...image,
+});
+
+/** @type {any} */
+export default new ObjectTree({
+  "explore:": explore,
+  "files:": files,
+  "help:": help,
+  "http:": http,
+  "https:": https,
+  "httpstree:": httpstree,
+  "httptree:": httptree,
+  "inherited:": inherited,
+  "new:": instantiate,
+  "node:": node,
+  "package:": packageNamespace,
+  "scope:": scope,
+
+  ...js,
+
+  Tree,
+  Origami,
+  Image,
+
+  // Some builtins need to be exposed at top level
+  ...handlers.default,
+});

package/src/common/documentObject.js

@@ -37,6 +37,13 @@ export default async function documentObject(input, data) {
   // };
   // const result = Object.create(base);
   const result = {};
+  // TODO: Deprecate @text
   Object.assign(result, inputData, data, { "@text": text });
+  Object.defineProperty(result, "_body", {
+    configurable: true,
+    value: text,
+    enumerable: false, // TODO: Make enumerable
+    writable: true,
+  });
   return result;
 }

package/src/dev/crawler/audit.js

@@ -0,0 +1,85 @@
+import { pathFromKeys, symbols, Tree } from "@weborigami/async-tree";
+import getTreeArgument from "../../common/getTreeArgument.js";
+import crawlResources from "./crawlResources.js";
+import { getBaseUrl } from "./utilities.js";
+
+/**
+ * Crawl the indicated tree and return an audit of any broken links to internal
+ * pages or other resources.
+ *
+ * @typedef  {import("@weborigami/types").AsyncTree} AsyncTree
+ * @typedef {import("@weborigami/async-tree").Treelike} Treelike
+ *
+ * @this {AsyncTree|null}
+ * @param {Treelike} treelike
+ * @param {string} [baseHref]
+ */
+export default async function audit(treelike, baseHref) {
+  const tree = await getTreeArgument(this, arguments, treelike, "site:audit");
+  const baseUrl = getBaseUrl(baseHref, treelike);
+
+  let errors = {};
+  let report;
+  const resourceReferences = {};
+  const resourcePromises = {};
+
+  // Iterate through all the resources to crawl the whole tree.
+  for await (const result of crawlResources(tree, baseUrl)) {
+    const { normalizedKeys, resourcePaths, value: resource } = result;
+    const normalizedPath = pathFromKeys(normalizedKeys);
+    if (normalizedPath === "crawl-errors.json") {
+      // Final error report; add missing pages to the errors
+      report = JSON.parse(resource);
+      for (const [path, pagePaths] of Object.entries(report)) {
+        if (!errors[path]) {
+          errors[path] = [];
+        }
+        errors[path].push(...pagePaths);
+      }
+    } else {
+      // Record which resources this path references
+      resourceReferences[normalizedPath] = resourcePaths;
+
+      // Add all resources to the set that should be verified
+      for (const resourcePath of resourcePaths) {
+        // Start request, don't wait for it to complete yet
+        resourcePromises[resourcePath] ??= Tree.traversePath(
+          tree,
+          resourcePath
+        ).then(
+          // Just return true or false to indicate if value is defined
+          (value) => value !== undefined
+        );
+      }
+    }
+  }
+
+  // Add any references to missing resources to the errors
+  for (const [refererPath, resourcePaths] of Object.entries(
+    resourceReferences
+  )) {
+    for (const resourcePath of resourcePaths) {
+      const found = await resourcePromises[resourcePath];
+      if (!found) {
+        if (!errors[refererPath]) {
+          errors[refererPath] = [];
+        }
+        errors[refererPath].push(resourcePath);
+      }
+    }
+  }
+
+  if (Object.keys(errors).length === 0) {
+    return undefined;
+  }
+
+  Object.defineProperty(errors, symbols.parent, {
+    enumerable: false,
+    value: this,
+  });
+  Object.defineProperty(errors, symbols.deep, {
+    enumerable: false,
+    value: true,
+  });
+  return errors;
+}

package/src/{site → dev}/crawler/crawl.js

@@ -2,13 +2,12 @@ import {
   DeepObjectTree,
   Tree,
   deepMerge,
-  isPlainObject,
   keysFromPath,
-  trailingSlash,
 } from "@weborigami/async-tree";
 import { InvokeFunctionsTransform } from "@weborigami/language";
 import getTreeArgument from "../../common/getTreeArgument.js";
 import crawlResources from "./crawlResources.js";
+import { addValueToObject, getBaseUrl } from "./utilities.js";
 
 /**
  * Crawl a tree, starting its root index.html page, and following links to
@@ -20,6 +19,7 @@ import crawlResources from "./crawlResources.js";
  *
  * @typedef  {import("@weborigami/types").AsyncTree} AsyncTree
  * @typedef {import("@weborigami/async-tree").Treelike} Treelike
+ *
  * @this {AsyncTree|null}
  * @param {Treelike} treelike
  * @param {string} [baseHref]
@@ -27,39 +27,10 @@ import crawlResources from "./crawlResources.js";
  */
 export default async function crawlBuiltin(treelike, baseHref) {
   const tree = await getTreeArgument(this, arguments, treelike, "site:crawl");
-
-  if (baseHref === undefined) {
-    // Ask tree or original treelike if it has an `href` property we can use as
-    // the base href to determine whether a link is local within the tree or
-    // not. If not, use a fake `local:/` base href.
-    baseHref =
-      /** @type {any} */ (tree).href ??
-      /** @type {any} */ (treelike).href ??
-      "local:/";
-    if (!baseHref?.endsWith("/")) {
-      baseHref += "/";
-    }
-  } else {
-    // Is the href already valid?
-    let isHrefValid = false;
-    try {
-      new URL(baseHref);
-      isHrefValid = true;
-    } catch (e) {
-      // Ignore
-    }
-    if (!isHrefValid) {
-      // Use a fake base href.
-      baseHref = `local:/${baseHref}`;
-    }
-  }
-
-  // @ts-ignore
-  const baseUrl = new URL(baseHref);
+  const baseUrl = getBaseUrl(baseHref, treelike);
 
   const cache = {};
   const resources = {};
-  const errors = [];
 
   // We iterate until there are no more promises to wait for.
   for await (const result of crawlResources(tree, baseUrl)) {
@@ -81,14 +52,6 @@ export default async function crawlBuiltin(treelike, baseHref) {
     }
   }
 
-  if (errors.length) {
-    addValueToObject(
-      cache,
-      ["crawl-errors.json"],
-      JSON.stringify(errors, null, 2)
-    );
-  }
-
   // Merge the cache on top of the resources tree. If we have an actual value
   // for something already, that's better than a function that will get that
   // value.
@@ -98,29 +61,3 @@ export default async function crawlBuiltin(treelike, baseHref) {
   );
   return result;
 }
-
-function addValueToObject(object, keys, value) {
-  for (let i = 0, current = object; i < keys.length; i++) {
-    const key = trailingSlash.remove(keys[i]);
-    if (i === keys.length - 1) {
-      // Write out value
-      if (isPlainObject(current[key])) {
-        // Route with existing values; treat the new value as an index.html
-        current[key]["index.html"] = value;
-      } else {
-        current[key] = value;
-      }
-    } else {
-      // Traverse further
-      if (!current[key]) {
-        current[key] = {};
-      } else if (!isPlainObject(current[key])) {
-        // Already have a value at this point. The site has a page at a route
-        // like /foo, and the site also has resources within that at routes like
-        // /foo/bar.jpg. We move the current value to "index.html".
-        current[key] = { "index.html": current[key] };
-      }
-      current = current[key];
-    }
-  }
-}

package/src/{site → dev}/crawler/crawlResources.js

@@ -129,28 +129,54 @@ async function processPath(tree, path, baseUrl) {
   keys = keys.map(decodeURIComponent);
 
   // Traverse tree to get value.
-  let value = await Tree.traverse(tree, ...keys);
-  const normalizedKeys = keys.slice();
-  let normalizedPath = path;
-  if (Tree.isTreelike(value)) {
-    // Path is actually a directory. See if we can get the empty string or
-    // "index.html".
-    value =
-      (await Tree.traverse(value, "")) ??
-      (await Tree.traverse(value, "index.html"));
-    if (value !== undefined) {
-      if (path.length > 0) {
-        // Mark the path as ending in a slash
-        normalizedPath = trailingSlash.add(path);
-        const key = normalizedKeys.pop();
-        normalizedKeys.push(trailingSlash.add(key));
+  let value;
+  let normalizedKeys;
+  let normalizedPath;
+  try {
+    value = await Tree.traverse(tree, ...keys);
+    normalizedKeys = keys.slice();
+    normalizedPath = path;
+    if (Tree.isTreelike(value)) {
+      // Path is actually a directory. See if we can get the empty string or
+      // "index.html".
+      value =
+        (await Tree.traverse(value, "")) ??
+        (await Tree.traverse(value, "index.html"));
+      if (value !== undefined) {
+        if (path.length > 0) {
+          // Mark the path as ending in a slash
+          normalizedPath = trailingSlash.add(path);
+          const key = normalizedKeys.pop();
+          normalizedKeys.push(trailingSlash.add(key));
+        }
+
+        // Add index.html to keys if it's not already there
+        if (normalizedKeys.at(-1) !== "index.html") {
+          normalizedKeys.push("index.html");
+        }
       }
+    }
 
-      // Add index.html to keys if it's not already there
-      if (normalizedKeys.at(-1) !== "index.html") {
-        normalizedKeys.push("index.html");
+    if (value === undefined && path.length > 0) {
+      // The path may be a URL like `foo` or `foo/` that points to `foo.html`, so
+      // we'll try looking adding `.html` to the end. We don't want to check every
+      // path twice, so we only do this if the last key does *not* include an
+      // extension.
+      const lastKey = keys.at(-1);
+      if (lastKey !== "" && !lastKey?.includes(".")) {
+        const adjustedLastKey = `${trailingSlash.remove(lastKey)}.html`;
+        const adjustedKeys = [...keys.slice(0, -1), adjustedLastKey];
+        value = await Tree.traverse(tree, ...adjustedKeys);
+        if (value !== undefined) {
+          // Page exists at foo.html
+          normalizedPath = pathFromKeys(adjustedKeys);
+          normalizedKeys = adjustedKeys;
+        }
       }
     }
+  } catch (error) {
+    // Ignore errors, return empty paths below
+    value = undefined;
   }
 
   if (value === undefined) {

package/src/dev/crawler/findPaths.js

@@ -0,0 +1,90 @@
+import { extension, toString } from "@weborigami/async-tree";
+import pathsInCss from "./pathsInCss.js";
+import pathsInHtml from "./pathsInHtml.js";
+import pathsInImageMap from "./pathsInImageMap.js";
+import pathsInJs from "./pathsInJs.js";
+import pathsInRobotsTxt from "./pathsInRobotsTxt.js";
+import pathsInSitemap from "./pathsInSitemap.js";
+
+// Filter the paths to those that are local to the site.
+function filterPaths(paths, baseUrl, localPath) {
+  // Convert paths to absolute URLs.
+  const localUrl = new URL(localPath, baseUrl);
+  const basePathname = baseUrl.pathname;
+  // @ts-ignore
+  const absoluteUrls = paths.map((path) => new URL(path, localUrl));
+
+  // Convert the absolute URLs to paths relative to the baseHref. If the URL
+  // points outside the tree rooted at the baseHref, the relative path will be
+  // null. We ignore the protocol in this test, because in practice sites often
+  // fumble the use of http and https, treating them interchangeably.
+  const relativePaths = absoluteUrls.map((url) => {
+    if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
+      const path = url.pathname.slice(basePathname.length);
+      // The process of creating the URLs will have escaped characters. We
+      // remove them. This has the side-effect of removing them if they existed
+      // in the original path; it would be better if we avoided that.
+      return decodeURIComponent(path);
+    } else {
+      return null;
+    }
+  });
+
+  // Filter out the null paths.
+  /** @type {string[]} */
+  // @ts-ignore
+  const filteredPaths = relativePaths.filter((path) => path);
+  return filteredPaths;
+}
+
+/**
+ * Given a value retrieved from a site using a given key (name), determine what
+ * kind of file it is and, based on that, find the paths it references.
+ */
+export default function findPaths(value, key, baseUrl, localPath) {
+  const text = toString(value);
+
+  // We guess the value is HTML is if its key has an .html extension or
+  // doesn't have an extension, or the value starts with `<`.
+  const ext = key ? extension.extname(key).toLowerCase() : "";
+  let foundPaths;
+  if (ext === ".html" || ext === ".htm" || ext === ".xhtml") {
+    foundPaths = pathsInHtml(text);
+  } else if (ext === ".css") {
+    foundPaths = pathsInCss(text);
+  } else if (ext === ".js") {
+    foundPaths = pathsInJs(text);
+  } else if (ext === ".map") {
+    foundPaths = pathsInImageMap(text);
+  } else if (key === "robots.txt") {
+    foundPaths = pathsInRobotsTxt(text);
+  } else if (key === "sitemap.xml") {
+    foundPaths = pathsInSitemap(text);
+  } else if (ext === "" && text?.trim().startsWith("<")) {
+    // Probably HTML
+    foundPaths = pathsInHtml(text);
+  } else {
+    // Doesn't have an extension we want to process
+    return {
+      crawlablePaths: [],
+      resourcePaths: [],
+    };
+  }
+
+  const crawlablePaths = filterPaths(
+    foundPaths.crawlablePaths,
+    baseUrl,
+    localPath
+  );
+
+  const resourcePaths = filterPaths(
+    foundPaths.resourcePaths,
+    baseUrl,
+    localPath
+  );
+
+  return {
+    crawlablePaths,
+    resourcePaths,
+  };
+}

package/src/dev/crawler/pathsInCss.js

@@ -0,0 +1,51 @@
+import { parse, walk } from "css-tree";
+import { addHref } from "./utilities.js";
+
+const imageFunctions = ["cross-fade", "image", "image-set"];
+
+export default function pathsInCss(css, context = "stylesheet") {
+  const paths = {
+    crawlablePaths: [],
+    resourcePaths: [],
+  };
+
+  let ast;
+  try {
+    ast = parse(css, { context });
+  } catch (e) {
+    // If the CSS is invalid, we can't parse it, so we can't extract paths. For
+    // now we just return no paths.
+    return paths;
+  }
+
+  if (!ast) {
+    // Unclear why parser sometimes returns an undefined AST
+    return paths;
+  }
+
+  walk(
+    ast,
+    /** @this {any} */
+    function (node) {
+      const { type, value } = node;
+      if (
+        this.atrule?.name === "import" &&
+        (type === "String" || type === "Url")
+      ) {
+        // A plain string or url() in an @import
+        addHref(paths, value, true);
+      } else if (
+        type === "String" &&
+        imageFunctions.includes(this.function?.name)
+      ) {
+        // A plain string in an cross-fade(), image(), or image-set()
+        addHref(paths, value, false);
+      } else if (type === "Url") {
+        // A url() anywhere else
+        addHref(paths, value, false);
+      }
+    }
+  );
+
+  return paths;
+}