@weborigami/origami 0.0.71-beta.1 → 0.0.71

package/exports/exports.js

@@ -4,7 +4,7 @@ export { default as basename } from "../src/builtins/@basename.js";
 export { default as breakpoint } from "../src/builtins/@breakpoint.js";
 export { default as builtins } from "../src/builtins/@builtins.js";
 export { default as cache } from "../src/builtins/@cache.js";
-export { default as calendarTree } from "../src/builtins/@calendarTree.js";
+export * from "../src/builtins/@calendarTree.js";
 export { default as changes } from "../src/builtins/@changes.js";
 export { default as clean } from "../src/builtins/@clean.js";
 export { default as code } from "../src/builtins/@code.js";
@@ -23,8 +23,8 @@ export { default as deepValues } from "../src/builtins/@deepValues.js";
 export { default as defineds } from "../src/builtins/@defineds.js";
 export { default as document } from "../src/builtins/@document.js";
 export { default as equals } from "../src/builtins/@equals.js";
-export { default as exceptions } from "../src/builtins/@exceptions.js";
 export { default as explore } from "../src/builtins/@explore.js";
+export { default as exploreSite } from "../src/builtins/@exploreSite.js";
 export { default as fetch } from "../src/builtins/@fetch.js";
 export { default as files } from "../src/builtins/@files.js";
 export { default as filter } from "../src/builtins/@filter.js";
@@ -46,13 +46,11 @@ export { default as index } from "../src/builtins/@index.js";
 export { default as inherited } from "../src/builtins/@inherited.js";
 export { default as inline } from "../src/builtins/@inline.js";
 export { default as inners } from "../src/builtins/@inners.js";
-export { default as invoke } from "../src/builtins/@invoke.js";
 export { default as js } from "../src/builtins/@js.js";
 export { default as json } from "../src/builtins/@json.js";
 export { default as jsonKeys } from "../src/builtins/@jsonKeys.js";
 export { default as jsonParse } from "../src/builtins/@jsonParse.js";
 export { default as keys } from "../src/builtins/@keys.js";
-export { default as keysTree } from "../src/builtins/@keysTree.js";
 export { default as length } from "../src/builtins/@length.js";
 export { default as log } from "../src/builtins/@log.js";
 export { default as map } from "../src/builtins/@map.js";
@@ -60,9 +58,7 @@ export { default as mapFn } from "../src/builtins/@mapFn.js";
 export { default as match } from "../src/builtins/@match.js";
 export * from "../src/builtins/@math.js";
 export { default as mdHtml } from "../src/builtins/@mdHtml.js";
-export { default as mdTree } from "../src/builtins/@mdTree.js";
 export { default as merge } from "../src/builtins/@merge.js";
-export { default as mkdir } from "../src/builtins/@mkdir.js";
 export * from "../src/builtins/@naturalOrder.js";
 export { default as node } from "../src/builtins/@node.js";
 export { default as not } from "../src/builtins/@not.js";
@@ -74,7 +70,6 @@ export { default as package } from "../src/builtins/@package.js";
 export { default as paginate } from "../src/builtins/@paginate.js";
 export { default as paginateFn } from "../src/builtins/@paginateFn.js";
 export { default as parent } from "../src/builtins/@parent.js";
-export { default as perf } from "../src/builtins/@perf.js";
 export { default as plain } from "../src/builtins/@plain.js";
 export { default as post } from "../src/builtins/@post.js";
 export { default as project } from "../src/builtins/@project.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@weborigami/origami",
-  "version": "0.0.71-beta.1",
+  "version": "0.0.71",
   "description": "Web Origami language, CLI, framework, and server",
   "type": "module",
   "repository": {
@@ -17,9 +17,9 @@
     "typescript": "5.6.2"
   },
   "dependencies": {
-    "@weborigami/async-tree": "0.0.71-beta.1",
-    "@weborigami/language": "0.0.71-beta.1",
-    "@weborigami/types": "0.0.71-beta.1",
+    "@weborigami/async-tree": "0.0.71",
+    "@weborigami/language": "0.0.71",
+    "@weborigami/types": "0.0.71",
     "exif-parser": "0.1.12",
     "graphviz-wasm": "3.0.2",
     "highlight.js": "11.10.0",

package/src/builtins/@calendarTree.js

@@ -1,5 +1 @@
-import calendarTree from "@weborigami/async-tree/src/calendarTree.js";
-
-export default function calendarTreeBuiltin(start, end) {
-  return calendarTree(start, end);
-}
+export { calendarTree as default } from "@weborigami/async-tree";

package/src/builtins/@code.js

@@ -9,7 +9,7 @@ import getTreeArgument from "../misc/getTreeArgument.js";
  */
 export default async function code(value) {
   if (value === undefined) {
-    value = await getTreeArgument(this, arguments, value, "@clean");
+    value = await getTreeArgument(this, arguments, value, "@code");
   }
   if (value === undefined) {
     return undefined;

package/src/builtins/@crawl.js

@@ -8,7 +8,6 @@ import {
 } from "@weborigami/async-tree";
 import { InvokeFunctionsTransform } from "@weborigami/language";
 import crawlResources from "../crawler/crawlResources.js";
-import { normalizeKeys } from "../crawler/utilities.js";
 import getTreeArgument from "../misc/getTreeArgument.js";
 
 /**
@@ -74,7 +73,7 @@ export default async function crawlBuiltin(treelike, baseHref) {
     // Add indirect resource functions to the resource tree. When requested,
     // these functions will obtain the resource from the original site.
     for (const resourcePath of resourcePaths) {
-      const resourceKeys = normalizeKeys(keysFromPath(resourcePath));
+      const resourceKeys = keysFromPath(resourcePath);
       const fn = () => {
         return Tree.traverse(tree, ...resourceKeys);
       };

package/src/builtins/{@keysTree.js → @exploreSite.js}

@@ -10,7 +10,7 @@ import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
  * @param {string} host
  * @param  {...string} keys
  */
-export default function keysTree(host, ...keys) {
-  assertTreeIsDefined(this, "keysTree");
+export default function exploreSite(host, ...keys) {
+  assertTreeIsDefined(this, "exploreSite");
   return ops.explorableSite.call(this, host, ...keys);
 }

package/src/builtins/@sitemap.js

@@ -6,12 +6,10 @@ import fileTypeOrigami from "./ori_handler.js";
 
 const templateText = `(urls) => \`<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-\${ @map(urls, (url) => \`
-  <url>
+\${ @map(urls, (url) => \`  <url>
     <loc>\${ url }</loc>
   </url>
-\`) }
-</urlset>
+\`) }</urlset>
 \`
 `;
 

package/src/builtins/@svg.js

@@ -20,7 +20,7 @@ export default async function svg(treelike, options = {}) {
     await graphviz.loadWASM();
     graphvizLoaded = true;
   }
-  const tree = await getTreeArgument(this, arguments, treelike, "@svg");
+  const tree = await getTreeArgument(this, arguments, treelike, "@svg", true);
   const dotText = await dot.call(this, tree, options);
   if (dotText === undefined) {
     return undefined;
@@ -28,6 +28,7 @@ export default async function svg(treelike, options = {}) {
   const svgText = await graphviz.layout(dotText, "svg");
   /** @type {any} */
   const result = new String(svgText);
+  result.mediaType = "image/svg+xml";
   result.unpack = () => tree;
   return result;
 }

package/src/builtins/@unpack.js

@@ -10,5 +10,5 @@ import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
  */
 export default function unpack(obj) {
   assertTreeIsDefined(this, "unpack");
-  return obj?.unpack?.();
+  return obj?.unpack?.() ?? obj;
 }

package/src/crawler/crawlResources.js

@@ -5,7 +5,6 @@ import {
   Tree,
 } from "@weborigami/async-tree";
 import findPaths from "./findPaths.js";
-import { normalizeKeys } from "./utilities.js";
 
 /**
  * Crawl the paths for the given tree, starting at the given base URL, and yield
@@ -131,15 +130,20 @@ async function processPath(tree, path, baseUrl) {
 
   // Traverse tree to get value.
   let value = await Tree.traverse(tree, ...keys);
-  const normalizedKeys = normalizeKeys(keys);
+  const normalizedKeys = keys.slice();
   let normalizedPath = path;
   if (Tree.isTreelike(value)) {
-    // Path is actually a directory; see if it has an index.html
-    value = await Tree.traverse(value, "index.html");
+    // Path is actually a directory. See if we can get the empty string or
+    // "index.html".
+    value =
+      (await Tree.traverse(value, "")) ??
+      (await Tree.traverse(value, "index.html"));
     if (value !== undefined) {
       if (path.length > 0) {
         // Mark the path as ending in a slash
         normalizedPath = trailingSlash.add(path);
+        const key = normalizedKeys.pop();
+        normalizedKeys.push(trailingSlash.add(key));
       }
 
       // Add index.html to keys if it's not already there

package/src/crawler/findPaths.js

@@ -16,7 +16,11 @@ function filterPaths(paths, baseUrl, localPath) {
   // fumble the use of http and https, treating them interchangeably.
   const relativePaths = absoluteUrls.map((url) => {
     if (url.host === baseUrl.host && url.pathname.startsWith(basePathname)) {
-      return url.pathname.slice(basePathname.length);
+      const path = url.pathname.slice(basePathname.length);
+      // The process of creating the URLs will have escaped characters. We
+      // remove them. This has the side-effect of removing them if they existed
+      // in the original path; it would be better if we avoided that.
+      return decodeURIComponent(path);
     } else {
       return null;
     }
@@ -39,7 +43,6 @@ export default function findPaths(value, key, baseUrl, localPath) {
   // We guess the value is HTML is if its key has an .html extension or
   // doesn't have an extension, or the value starts with `<`.
   const ext = key ? extname(key).toLowerCase() : "";
-  const maybeHtml = ext === "" || text?.trim().startsWith("<");
   let foundPaths;
   if (ext === ".html" || ext === ".htm" || ext === ".xhtml") {
     foundPaths = findPathsInHtml(text);
@@ -53,7 +56,8 @@ export default function findPaths(value, key, baseUrl, localPath) {
     foundPaths = findPathsInRobotsTxt(text);
   } else if (key === "sitemap.xml") {
     foundPaths = findPathsInSitemapXml(text);
-  } else if (maybeHtml) {
+  } else if (ext === "" && text?.trim().startsWith("<")) {
+    // Probably HTML
     foundPaths = findPathsInHtml(text);
   } else {
     // Doesn't have an extension we want to process
@@ -90,7 +94,7 @@ function findPathsInCss(css) {
   while ((match = urlRegex.exec(css))) {
     const href = normalizeHref(match.groups?.href);
     if (href) {
-      resourcePaths.push();
+      resourcePaths.push(href);
     }
   }
 
@@ -217,6 +221,10 @@ function findPathsInHtml(html) {
     }
   }
 
+  // Also look for JS `import` statements that might be in <script type="module"> tags.
+  const jsResults = findPathsInJs(html);
+  crawlablePaths.push(...jsResults.crawlablePaths);
+
   return { crawlablePaths, resourcePaths };
 }
 

package/src/misc/getTreeArgument.js

@@ -1,5 +1,4 @@
 import { Tree, isUnpackable } from "@weborigami/async-tree";
-import { isTreelike } from "@weborigami/async-tree/src/Tree.js";
 import assertTreeIsDefined from "./assertTreeIsDefined.js";
 
 /**
@@ -34,7 +33,7 @@ export default async function getTreeArgument(
     if (isUnpackable(treelike)) {
       treelike = await treelike.unpack();
     }
-    if (isTreelike(treelike)) {
+    if (Tree.isTreelike(treelike)) {
       let tree = Tree.from(treelike, { deep });
       // If the tree was created from a treelike object and does not yet have a
       // parent, make the current tree its parent.

package/src/misc/treeDot.js

@@ -20,7 +20,7 @@ import { keySymbol } from "../common/utilities.js";
  * @param {PlainObject} [options]
  */
 export default async function dot(treelike, options = {}) {
-  const tree = Tree.from(treelike);
+  const tree = Tree.from(treelike, { deep: true });
   const rootLabel = tree[keySymbol] ?? "";
   const treeArcs = await statements(tree, "", rootLabel, options);
   return `digraph g {

package/src/server/constructResponse.js

@@ -10,8 +10,6 @@ import * as serialize from "../common/serialize.js";
 import { toString } from "../common/utilities.js";
 import { mediaTypeForExtension } from "./mediaTypes.js";
 
-const TypedArray = Object.getPrototypeOf(Uint8Array);
-
 /**
  * Given a resource that was returned from a route, construct an appropriate
  * HTTP Response indicating what should be sent to the client. Return null
@@ -78,22 +76,18 @@ export default async function constructResponse(request, resource) {
     mediaType = "text/yaml";
   }
 
-  let body;
-  if (mediaType) {
-    body = SiteTree.mediaTypeIsText(mediaType) ? toString(resource) : resource;
-  } else {
-    body = textOrObject(resource);
-    // Infer media type.
-    mediaType =
-      typeof body !== "string"
-        ? "application/octet-stream"
-        : body.trimStart().startsWith("<")
-        ? "text/html"
-        : "text/plain";
-  }
-
-  // Assume text is encoded in UTF-8.
-  if (SiteTree.mediaTypeIsText(mediaType)) {
+  // By default, the body will be the resource we got
+  let body = resource;
+  if (!mediaType) {
+    // Maybe it's HTML?
+    const text = toString(resource);
+    if (text && maybeHtml(text)) {
+      mediaType = "text/html";
+      body = text;
+    }
+  } else if (mediaType && SiteTree.mediaTypeIsText(mediaType)) {
+    // Assume text is encoded in UTF-8.
+    body = toString(resource);
     mediaType += "; charset=utf-8";
   }
 
@@ -108,26 +102,26 @@ export default async function constructResponse(request, resource) {
     return null;
   }
 
-  return new Response(body, {
-    headers: {
-      "Content-Type": mediaType,
-    },
-  });
+  const options = mediaType ? { headers: { "Content-Type": mediaType } } : {};
+  const response = new Response(body, options);
+  return response;
 }
 
-/**
- * Convert to a string if we can, but leave objects that convert to something
- * like "[object Object]" alone.
- *
- * @param {any} object
- */
-function textOrObject(object) {
-  if (object instanceof ArrayBuffer) {
-    // Convert to Uint8Array so we can write it to the Response.
-    return new Uint8Array(object);
-  } else if (object instanceof TypedArray) {
-    // Return typed arrays as is.
-    return object;
+// Return true if the resource appears to represent HTML
+function maybeHtml(text) {
+  if (!text) {
+    return false;
+  }
+  if (text.startsWith("<!DOCTYPE html>")) {
+    return true;
   }
-  return toString(object);
+  // Check if the text starts with an HTML tag.
+  // - start with possible whitespace
+  // - followed by '<'
+  // - followed by a letter
+  // - followed by letters, digits, hyphens, underscores, colons, or periods
+  // - followed by '>', or
+  // - followed by whitespace, anything that's not '>', then a '>'
+  const tagRegex = /^\s*<[a-zA-Z][a-zA-Z0-9-_:\.]+(>|[\s]+[^>]*>)/;
+  return tagRegex.test(text);
 }

package/src/builtins/@constructor.js

@@ -1,13 +0,0 @@
-import { ops } from "@weborigami/language";
-import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
-
-/**
- * @typedef {import("@weborigami/types").AsyncTree} AsyncTree
- *
- * @this {AsyncTree|null}
- * @param  {...any} keys
- */
-export default function constructor(...keys) {
-  assertTreeIsDefined(this, "constructor");
-  return ops.constructor.call(this, ...keys);
-}

package/src/builtins/@exceptions.js

@@ -1,45 +0,0 @@
-import { Tree } from "@weborigami/async-tree";
-import getTreeArgument from "../misc/getTreeArgument.js";
-import defineds from "./@defineds.js";
-
-/**
- * @typedef  {import("@weborigami/types").AsyncTree} AsyncTree
- * @typedef {import("@weborigami/async-tree").Treelike} Treelike
- * @this {AsyncTree|null}
- * @param {Treelike} treelike
- */
-export default async function exceptions(treelike) {
-  const tree = await getTreeArgument(this, arguments, treelike, "@exceptions");
-  const exceptionsTree = new ExceptionsTree(tree);
-  return defineds.call(this, exceptionsTree);
-}
-
-/**
- * @implements {AsyncTree}
- */
-class ExceptionsTree {
-  constructor(tree) {
-    this.tree = tree;
-  }
-
-  async get(key) {
-    try {
-      const value = await this.tree.get(key);
-      return Tree.isAsyncTree(value)
-        ? Reflect.construct(this.constructor, [value])
-        : undefined;
-    } catch (/** @type {any} */ error) {
-      return error.name && error.message
-        ? `${error.name}: ${error.message}`
-        : error.name ?? error.message ?? error;
-    }
-  }
-
-  async keys() {
-    return this.tree.keys();
-  }
-}
-
-exceptions.usage = `@exceptions tree\tReturn a tree of exceptions thrown in the tree`;
-exceptions.documentation =
-  "https://weborigami.org/cli/builtins.html#exceptions";

package/src/builtins/@invoke.js

@@ -1,36 +0,0 @@
-import { isUnpackable } from "@weborigami/async-tree";
-import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
-
-/**
- * Invoke the given text as an Origami function.
- *
- * This built-in exists to facilitate executing an Origami file as a script via
- * a [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) directive.
- *
- * You can execute a foo.ori file as a script by adding the following shebang
- * directive to the top of the file:
- *
- * ```sh
- * #!/usr/bin/env ori @invoke
- * ```
- *
- * Then mark the file as executable:
- *
- * ```sh
- * chmod +x foo.ori
- * ```
- *
- * @this {import("@weborigami/types").AsyncTree|null}
- */
-export default async function invoke(fn) {
-  assertTreeIsDefined(this, "invoke");
-  if (fn === undefined) {
-    throw new Error(
-      "An Origami function was called with an initial argument, but its value is undefined."
-    );
-  }
-  if (isUnpackable(fn)) {
-    fn = await fn.unpack();
-  }
-  return typeof fn === "function" ? fn.call(this) : fn;
-}

package/src/builtins/@mdTree.js

@@ -1,69 +0,0 @@
-import { toString } from "../common/utilities.js";
-
-/**
- * Returns the tree structure of a markdown document.
- *
- * @typedef {import("@weborigami/async-tree").StringLike} StringLike
- * @typedef {import("@weborigami/async-tree").Unpackable<StringLike>}
- * UnpackableStringlike
- *
- * @this {import("@weborigami/types").AsyncTree|null|void}
- * @param {StringLike|UnpackableStringlike} input
- */
-export default function mdStructure(input) {
-  const markdown = toString(input);
-  if (markdown === null) {
-    throw new Error("No markdown text provided.");
-  }
-
-  // The document acts as an entry for heading level zero. All level one
-  // headings will end up as its children.
-  const document = {};
-  const activeHeadings = [document];
-
-  // Split the text by lines that contain markdown headings.
-  const lines = markdown.split("\n");
-  lines.forEach((line) => {
-    const match = line.match(/^(?<levelMarkers>#{1,6})\s(?<heading>.*)$/);
-    if (!match?.groups) {
-      return;
-    }
-    const { levelMarkers, heading } = match.groups;
-    const level = levelMarkers.length;
-
-    // If we've gone up a level (or more), pop completed entries off the stack.
-    while (activeHeadings.length > level) {
-      activeHeadings.pop();
-    }
-
-    // If we've skipped a level (or more), add intermediate entries using
-    // symbols to avoid name collisions.
-    while (activeHeadings.length < level) {
-      const entry = {};
-      const parentEntry = activeHeadings[activeHeadings.length - 1];
-      parentEntry[Symbol()] = entry;
-      activeHeadings.push(entry);
-    }
-
-    // Add a new entry to the list of children under construction.
-    const entry = {};
-    const parentEntry = activeHeadings[activeHeadings.length - 1];
-    parentEntry[heading] = entry;
-    activeHeadings.push(entry);
-  });
-
-  return pruneEmptyObjects(document) ?? {};
-}
-
-// Replace empty objects in the tree with nulls.
-function pruneEmptyObjects(tree) {
-  const keys = [...Object.keys(tree), ...Object.getOwnPropertySymbols(tree)];
-  if (keys.length === 0) {
-    return null;
-  }
-  const result = {};
-  for (const key of keys) {
-    result[key] = pruneEmptyObjects(tree[key]);
-  }
-  return result;
-}

package/src/builtins/@mkdir.js

@@ -1,3 +0,0 @@
-// For now this is a synonym for @files.
-// If we like it, it might replace @files.
-export { default } from "./@files.js";

package/src/builtins/@perf.js

@@ -1,19 +0,0 @@
-import assertTreeIsDefined from "../misc/assertTreeIsDefined.js";
-
-/**
- * Return the number of milliseconds required to execute the given function the
- * specified number of times.
- *
- * @this {import("@weborigami/types").AsyncTree|null}
- * @param {Function} fn
- */
-export default async function perf(fn, count = 1) {
-  assertTreeIsDefined(this, "perf");
-  const start = performance.now();
-  for (let i = 0; i < count; i++) {
-    await fn.call(this);
-  }
-  const end = performance.now();
-  const milliseconds = Math.round(end - start);
-  return `${milliseconds} ms`;
-}