elm-pages 3.0.0-beta.2 → 3.0.0-beta.20

package/generator/src/request-cache.js

@@ -1,174 +1,189 @@
-const path = require("path");
-const fetch = require("node-fetch");
-const objectHash = require("object-hash");
-const kleur = require("kleur");
+import * as path from "path";
+import * as fsPromises from "fs/promises";
+import * as kleur from "kleur/colors";
+import { default as makeFetchHappenOriginal } from "make-fetch-happen";
 
-/**
- * To cache HTTP requests on disk with quick lookup and insertion, we store the hashed request.
- * This uses SHA1 hashes. They are uni-directional hashes, which works for this use case. Most importantly,
- * they're unique enough and can be expressed in a case-insensitive way so it works on Windows filesystems.
- * And they are 40 hex characters, so the length won't be too long no matter what the request payload.
- * @param {Object} request
- */
-function requestToString(request) {
-  return objectHash(request);
-}
-/**
- * @param {Object} request
- */
-function fullPath(portsHash, request, hasFsAccess) {
-  const requestWithPortHash =
-    request.url === "elm-pages-internal://port"
-      ? { portsHash, ...request }
-      : request;
-  if (hasFsAccess) {
-    return path.join(
-      process.cwd(),
-      ".elm-pages",
-      "http-response-cache",
-      requestToString(requestWithPortHash)
-    );
-  } else {
-    return path.join("/", requestToString(requestWithPortHash));
-  }
-}
+const defaultHttpCachePath = "./.elm-pages/http-cache";
+
+/** @typedef {{kind: 'cache-response-path', value: string} | {kind: 'response-json', value: JSON}} Response */
 
 /**
  * @param {string} mode
  * @param {{url: string;headers: {[x: string]: string;};method: string;body: Body;}} rawRequest
- * @returns {Promise<string>}
- * @param {string} portsFile
+ * @param {Record<string, unknown>} portsFile
  * @param {boolean} hasFsAccess
+ * @returns {Promise<Response>}
  */
-function lookupOrPerform(portsFile, mode, rawRequest, hasFsAccess, useCache) {
-  const { fs } = require("./request-cache-fs.js")(hasFsAccess);
+export function lookupOrPerform(
+  portsFile,
+  mode,
+  rawRequest,
+  hasFsAccess,
+  useCache
+) {
+  const makeFetchHappen = makeFetchHappenOriginal.defaults({
+    cache: mode === "build" ? "no-cache" : "default",
+  });
   return new Promise(async (resolve, reject) => {
     const request = toRequest(rawRequest);
-    const portsHash = (portsFile && portsFile.match(/-([^-]+)\.js$/)[1]) || "";
-    const responsePath = fullPath(portsHash, request, hasFsAccess);
 
-    // TODO check cache expiration time and delete and go to else if expired
-    if (useCache && (await checkFileExists(fs, responsePath))) {
-      // console.log("Skipping request, found file.");
-      resolve(responsePath);
-    } else {
-      let portDataSource = {};
-      let portDataSourceImportError = null;
-      try {
-        if (portsFile === undefined) {
-          throw "missing";
-        }
-        const portDataSourcePath = path.resolve(portsFile);
-        // On Windows, we need cannot use paths directly and instead must use a file:// URL.
-        // portDataSource = await require(url.pathToFileURL(portDataSourcePath).href);
-        portDataSource = require(portDataSourcePath);
-      } catch (e) {
-        portDataSourceImportError = e;
+    let portBackendTask = portsFile;
+    let portBackendTaskImportError = null;
+    try {
+      if (portsFile === undefined) {
+        throw "missing";
       }
+    } catch (e) {
+      portBackendTaskImportError = e;
+    }
 
-      if (request.url === "elm-pages-internal://port") {
-        try {
-          const { input, portName } = rawRequest.body.args[0];
+    if (request.url === "elm-pages-internal://port") {
+      try {
+        const { input, portName } = rawRequest.body.args[0];
 
-          if (!portDataSource[portName]) {
-            if (portDataSourceImportError === null) {
-              throw `DataSource.Port.send "${portName}" was called, but I couldn't find a function with that name in the port definitions file. Is it exported correctly?`;
-            } else if (portDataSourceImportError === "missing") {
-              throw `DataSource.Port.send "${portName}" was called, but I couldn't find the port definitions file. Be sure to create a 'port-data-source.ts' or 'port-data-source.js' file and maybe restart the dev server.`;
-            } else {
-              throw `DataSource.Port.send "${portName}" was called, but I couldn't import the port definitions file, because of this exception: \`${portDataSourceImportError}\` Are there syntax errors or expections thrown during import?`;
-            }
-          } else if (typeof portDataSource[portName] !== "function") {
-            throw `DataSource.Port.send "${portName}" was called, but it is not a function. Be sure to export a function with that name from port-data-source.js`;
-          }
-          await fs.promises.writeFile(
-            responsePath,
-            JSON.stringify(jsonResponse(await portDataSource[portName](input)))
-          );
-          resolve(responsePath);
-        } catch (error) {
-          console.trace(error);
-          reject({
-            title: "DataSource.Port Error",
-            message: error.toString(),
+        if (portBackendTask === null) {
+          resolve({
+            kind: "response-json",
+            value: jsonResponse({
+              "elm-pages-internal-error": "MissingCustomBackendTaskFile",
+            }),
           });
-        }
-      } else {
-        try {
-          console.time(`fetch ${request.url}`);
-          const response = await fetch(request.url, {
-            method: request.method,
-            body: request.body,
-            headers: {
-              "User-Agent": "request",
-              ...request.headers,
-            },
+        } else if (portBackendTask && portBackendTask.__internalElmPagesError) {
+          resolve({
+            kind: "response-json",
+            value: jsonResponse({
+              "elm-pages-internal-error": "ErrorInCustomBackendTaskFile",
+              error: portBackendTask.__internalElmPagesError,
+            }),
           });
-
-          console.timeEnd(`fetch ${request.url}`);
-          const expectString = request.headers["elm-pages-internal"];
-
-          if (response.ok || expectString === "ExpectResponse") {
-            let body;
-            let bodyKind;
-            if (expectString === "ExpectJson") {
-              body = await response.json();
-              bodyKind = "json";
-            } else if (
-              expectString === "ExpectBytes" ||
-              expectString === "ExpectBytesResponse"
-            ) {
-              bodyKind = "bytes";
-              const arrayBuffer = await response.arrayBuffer();
-              body = Buffer.from(arrayBuffer).toString("base64");
-            } else if (expectString === "ExpectWhatever") {
-              bodyKind = "whatever";
-              body = null;
-            } else if (
-              expectString === "ExpectResponse" ||
-              expectString === "ExpectString"
-            ) {
-              bodyKind = "string";
-              body = await response.text();
-            } else {
-              throw `Unexpected expectString ${expectString}`;
-            }
-
-            await fs.promises.writeFile(
-              responsePath,
-              JSON.stringify({
-                headers: Object.fromEntries(response.headers.entries()),
-                statusCode: response.status,
-                body: body,
-                bodyKind,
-                url: response.url,
-                statusText: response.statusText,
-              })
-            );
-
-            resolve(responsePath);
+        } else if (portBackendTask && !portBackendTask[portName]) {
+          if (portBackendTaskImportError === null) {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "CustomBackendTaskNotDefined",
+              }),
+            });
+          } else if (portBackendTaskImportError === "missing") {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "MissingCustomBackendTaskFile",
+              }),
+            });
           } else {
-            console.log("@@@ request-cache1 bad HTTP response");
-            reject({
-              title: "DataSource.Http Error",
-              message: `${kleur
-                .yellow()
-                .underline(request.url)} Bad HTTP response ${response.status} ${
-                response.statusText
-              }
-`,
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "ErrorInCustomBackendTaskFile",
+                error:
+                  (portBackendTaskImportError &&
+                    portBackendTaskImportError.stack) ||
+                  "",
+              }),
             });
           }
-        } catch (error) {
-          console.trace("@@@ request-cache2 HTTP error", error);
-          reject({
-            title: "DataSource.Http Error",
-            message: `${kleur
-              .yellow()
-              .underline(request.url)} ${error.toString()}
-`,
+        } else if (typeof portBackendTask[portName] !== "function") {
+          resolve({
+            kind: "response-json",
+            value: jsonResponse({
+              "elm-pages-internal-error": "ExportIsNotFunction",
+              error: typeof portBackendTask[portName],
+            }),
           });
+        } else {
+          try {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse(await portBackendTask[portName](input)),
+            });
+          } catch (portCallError) {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "CustomBackendTaskException",
+                error: portCallError,
+              }),
+            });
+          }
         }
+      } catch (error) {
+        console.trace(error);
+        reject({
+          title: "BackendTask.Custom Error",
+          message: error.toString(),
+        });
+      }
+    } else {
+      try {
+        console.time(`fetch ${request.url}`);
+        const response = await safeFetch(makeFetchHappen, request.url, {
+          method: request.method,
+          body: request.body,
+          headers: {
+            "User-Agent": "request",
+            ...request.headers,
+          },
+          ...rawRequest.cacheOptions,
+        });
+
+        console.timeEnd(`fetch ${request.url}`);
+        const expectString = request.headers["elm-pages-internal"];
+
+        let body;
+        let bodyKind;
+        if (expectString === "ExpectJson") {
+          try {
+            body = await response.buffer();
+            body = JSON.parse(body.toString("utf-8"));
+            bodyKind = "json";
+          } catch (error) {
+            body = body.toString("utf8");
+            bodyKind = "string";
+          }
+        } else if (
+          expectString === "ExpectBytes" ||
+          expectString === "ExpectBytesResponse"
+        ) {
+          body = await response.buffer();
+          try {
+            body = body.toString("base64");
+            bodyKind = "bytes";
+          } catch (e) {
+            body = body.toString("utf8");
+            bodyKind = "string";
+          }
+        } else if (expectString === "ExpectWhatever") {
+          bodyKind = "whatever";
+          body = null;
+        } else if (
+          expectString === "ExpectResponse" ||
+          expectString === "ExpectString"
+        ) {
+          bodyKind = "string";
+          body = await response.text();
+        } else {
+          throw `Unexpected expectString ${expectString}`;
+        }
+
+        resolve({
+          kind: "response-json",
+          value: {
+            headers: Object.fromEntries(response.headers.entries()),
+            statusCode: response.status,
+            body,
+            bodyKind,
+            url: response.url,
+            statusText: response.statusText,
+          },
+        });
+      } catch (error) {
+        console.trace("@@@ request-cache2 HTTP error", error);
+        reject({
+          title: "BackendTask.Http Error",
+          message: `${kleur.yellow().underline(request.url)} ${error.toString()}
+`,
+        });
       }
     }
   });
@@ -188,15 +203,6 @@ function toRequest(elmRequest) {
     body: toBody(elmRequest.body),
   };
 }
-/**
- * @param {string} file
- */
-function checkFileExists(fs, file) {
-  return fs.promises
-    .access(file, fs.constants.F_OK)
-    .then(() => true)
-    .catch(() => false);
-}
 /**
  * @param {Body} body
  */
@@ -208,6 +214,9 @@ function toBody(body) {
     case "StringBody": {
       return body.args[1];
     }
+    case "BytesBody": {
+      return Buffer.from(body.args[1], "base64");
+    }
     case "JsonBody": {
       return JSON.stringify(body.args[0]);
     }
@@ -226,13 +235,16 @@ function toContentType(body) {
     case "StringBody": {
       return { "Content-Type": body.args[0] };
     }
+    case "BytesBody": {
+      return { "Content-Type": body.args[0] };
+    }
     case "JsonBody": {
       return { "Content-Type": "application/json" };
     }
   }
 }
 
-/** @typedef { { tag: 'EmptyBody'} | { tag: 'StringBody'; args: [string, string] } | {tag: 'JsonBody'; args: [ Object ] } } Body  */
+/** @typedef { { tag: 'EmptyBody'} |{ tag: 'BytesBody'; args: [string, string] } |  { tag: 'StringBody'; args: [string, string] } | {tag: 'JsonBody'; args: [ Object ] } } Body  */
 function requireUncached(mode, filePath) {
   if (mode === "dev-server") {
     // for the build command, we can skip clearing the cache because it won't change while the build is running
@@ -249,4 +261,30 @@ function jsonResponse(json) {
   return { bodyKind: "json", body: json };
 }
 
-module.exports = { lookupOrPerform };
+async function safeFetch(makeFetchHappen, url, options) {
+  const { cachePath, ...optionsWithoutCachePath } = options;
+  const cachePathWithDefault = cachePath || defaultHttpCachePath;
+  if (await canAccess(cachePathWithDefault)) {
+    return await makeFetchHappen(url, {
+      cachePath: cachePathWithDefault,
+      ...options,
+    });
+  } else {
+    return await makeFetchHappen(url, {
+      cache: "no-store",
+      ...optionsWithoutCachePath,
+    });
+  }
+}
+
+async function canAccess(filePath) {
+  try {
+    await fsPromises.access(
+      filePath,
+      fsPromises.constants.R_OK | fsPromises.constants.W_OK
+    );
+    return true;
+  } catch {
+    return false;
+  }
+}

package/generator/src/rewrite-client-elm-json.js

@@ -1,6 +1,6 @@
-const fs = require("fs");
+import * as fs from "node:fs";
 
-module.exports = async function () {
+export async function rewriteClientElmJson() {
   var elmJson = JSON.parse(
     (await fs.promises.readFile("./elm.json")).toString()
   );
@@ -9,11 +9,11 @@ module.exports = async function () {
 
   await writeFileIfChanged(
     "./elm-stuff/elm-pages/client/elm.json",
-    JSON.stringify(rewriteElmJson(elmJson))
+    JSON.stringify(rewriteClientElmJsonHelp(elmJson))
   );
-};
+}
 
-function rewriteElmJson(elmJson) {
+function rewriteClientElmJsonHelp(elmJson) {
   // The internal generated file will be at:
   // ./elm-stuff/elm-pages/
   // So, we need to take the existing elmJson and

package/generator/src/rewrite-elm-json.js

@@ -1,19 +1,19 @@
-const fs = require("fs");
+import * as fs from "node:fs";
 
-module.exports = async function () {
+export async function rewriteElmJson(sourceElmJsonPath, targetElmJsonPath) {
   var elmJson = JSON.parse(
-    (await fs.promises.readFile("./elm.json")).toString()
+    (await fs.promises.readFile(sourceElmJsonPath)).toString()
   );
 
   // write new elm.json
 
   await writeFileIfChanged(
-    "./elm-stuff/elm-pages/elm.json",
-    JSON.stringify(rewriteElmJson(elmJson))
+    targetElmJsonPath,
+    JSON.stringify(rewriteElmJsonHelp(elmJson))
   );
-};
+}
 
-function rewriteElmJson(elmJson) {
+function rewriteElmJsonHelp(elmJson) {
   // The internal generated file will be at:
   // ./elm-stuff/elm-pages/
   // So, we need to take the existing elmJson and

package/generator/src/route-codegen-helpers.js

@@ -1,7 +1,7 @@
 /**
  * @param {string[]} name
  */
-function routeParams(name) {
+export function routeParams(name) {
   return name
     .map((section) => {
       const routeParamMatch = section.match(/([A-Z][A-Za-z0-9]*)__?$/);
@@ -17,7 +17,7 @@ function routeParams(name) {
  * @param {string[]} name
  * @returns {Segment[]}
  */
-function parseRouteParams(name) {
+export function parseRouteParams(name) {
   return name.flatMap((section) => {
     const routeParamMatch = section.match(/([A-Z][A-Za-z0-9]*)(_?_?)$/);
     const maybeParam = (routeParamMatch && routeParamMatch[1]) || "TODO";
@@ -68,7 +68,7 @@ function parseRouteParams(name) {
  * @param {string[]} name
  * @returns {( Segment | {kind: 'static'; name: string})[]}
  */
-function parseRouteParamsWithStatic(name) {
+export function parseRouteParamsWithStatic(name) {
   return name.flatMap((section) => {
     const routeParamMatch = section.match(/([A-Z][A-Za-z0-9]*)(_?_?)$/);
     const maybeParam = (routeParamMatch && routeParamMatch[1]) || "TODO";
@@ -123,7 +123,7 @@ function parseRouteParamsWithStatic(name) {
  * @param {string[]} name
  * @returns {string}
  */
-function routeVariantDefinition(name) {
+export function routeVariantDefinition(name) {
   const newLocal = parseRouteParams(name);
   if (newLocal.length == 0) {
     return routeVariant(name);
@@ -151,7 +151,7 @@ function routeVariantDefinition(name) {
  * @param {string[]} name
  * @returns {string}
  */
-function toPathPattern(name) {
+export function toPathPattern(name) {
   return (
     "/" +
     parseRouteParamsWithStatic(name)
@@ -181,7 +181,7 @@ function toPathPattern(name) {
  * @param {string[]} name
  * @returns {string[]}
  */
-function toPathPatterns(name) {
+export function toPathPatterns(name) {
   const segments = parseRouteParamsWithStatic(name);
 
   const lastSegment = segments[segments.length - 1];
@@ -199,7 +199,7 @@ function toPathPatterns(name) {
 /**
  * @param {string[]} segments
  */
-function joinPath(segments) {
+export function joinPath(segments) {
   const joined = segments.join("/");
   if (joined.startsWith("/")) {
     return joined;
@@ -211,7 +211,7 @@ function joinPath(segments) {
 /**
  * @return {string[]}
  */
-function newHelper(segments) {
+export function newHelper(segments) {
   return segments.map((param) => {
     switch (param.kind) {
       case "static": {
@@ -238,7 +238,7 @@ function newHelper(segments) {
  * @param {string[]} name
  * @returns {string}
  */
-function toElmPathPattern(name) {
+export function toElmPathPattern(name) {
   const parsedSegments = parseRouteParamsWithStatic(name);
 
   const foundEndings = parsedSegments.flatMap((segment) => {
@@ -297,14 +297,14 @@ function toElmPathPattern(name) {
  * @param {string} input
  * @returns {string}
  */
-function camelToKebab(input) {
+export function camelToKebab(input) {
   return input.replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase();
 }
 
 /**
  * @param {string[]} name
  */
-function paramsRecord(name) {
+export function paramsRecord(name) {
   return `{ ${parseRouteParams(name).map((param) => {
     switch (param.kind) {
       case "dynamic": {
@@ -326,7 +326,7 @@ function paramsRecord(name) {
 /**
  * @param {string[]} name
  */
-function routeVariant(name) {
+export function routeVariant(name) {
   return `${name.join("__")}`;
 }
 
@@ -334,44 +334,29 @@ function routeVariant(name) {
  * @param {string[]} name
  * @param {string} paramsName
  */
-function destructureRoute(name, paramsName) {
+export function destructureRoute(name, paramsName) {
   return emptyRouteParams(name)
     ? `Route.${routeVariant(name)}`
     : `(Route.${routeVariant(name)} ${paramsName})`;
 }
 
-function referenceRouteParams(name, paramsName) {
+export function referenceRouteParams(name, paramsName) {
   return emptyRouteParams(name) ? `{}` : paramsName;
 }
 /**
  * @param {string[]} name
  */
-function emptyRouteParams(name) {
+export function emptyRouteParams(name) {
   return parseRouteParams(name).length === 0;
 }
 
 /**
  * @param {string} name
  */
-function toFieldName(name) {
+export function toFieldName(name) {
   if (name === "SPLAT") {
     return "splat";
   } else {
     return name.charAt(0).toLowerCase() + name.slice(1);
   }
 }
-
-module.exports = {
-  routeParams,
-  routeVariantDefinition,
-  routeVariant,
-  toFieldName,
-  paramsRecord,
-  toPathPattern,
-  toPathPatterns,
-  parseRouteParams,
-  parseRouteParamsWithStatic,
-  toElmPathPattern,
-  destructureRoute,
-  referenceRouteParams,
-};

package/generator/src/seo-renderer.js

@@ -1,11 +1,9 @@
-module.exports = { gather };
-
 /** @typedef { { type: 'root'; keyValuePair: [string, string] } } RootTagModifier */
 
 /**
  * @param {( SeoTag | RootTagModifier )[]} tags
  */
-function gather(tags) {
+export function gather(tags) {
   const withoutRootModifiers = tags.flatMap((value) => {
     if (value.type === "root") {
       return [];
@@ -43,11 +41,16 @@ function headTag(rootModifiers) {
 
 function toString(/** @type { SeoTag[] }  */ tags) {
   return tags
-    .map((headTag) => {
+    .flatMap((headTag) => {
       if (headTag.type === "head") {
-        return appendTag(headTag);
+        return [appendTag(headTag)];
       } else if (headTag.type === "json-ld") {
-        return appendJsonLdTag(headTag);
+        return [appendJsonLdTag(headTag)];
+      } else if (headTag.type === "stripped") {
+        console.warn(
+          `WARNING: Head.nonLoadingTag value ignored because it used a loading tag: ${headTag.message}`
+        );
+        return [];
       } else {
         throw new Error(`Unknown tag type ${JSON.stringify(headTag)}`);
       }
@@ -55,7 +58,7 @@ function toString(/** @type { SeoTag[] }  */ tags) {
     .join("");
 }
 
-/** @typedef {HeadTag | JsonLdTag} SeoTag */
+/** @typedef {HeadTag | JsonLdTag | StrippedTag} SeoTag */
 
 /** @typedef {{ name: string; attributes: string[][]; type: 'head' }} HeadTag */
 function appendTag(/** @type {HeadTag} */ tagDetails) {
@@ -66,6 +69,8 @@ function appendTag(/** @type {HeadTag} */ tagDetails) {
 }
 
 /** @typedef {{ contents: Object; type: 'json-ld' }} JsonLdTag */
+/** @typedef {{ message: string; type: 'stripped' }} StrippedTag */
+
 function appendJsonLdTag(/** @type {JsonLdTag} */ tagDetails) {
   return `<script type="application/ld+json">
 ${JSON.stringify(tagDetails.contents)}