elm-pages 3.0.0-beta.13 → 3.0.0-beta.15

package/generator/src/request-cache.js

@@ -1,175 +1,171 @@
 const path = require("path");
-const fetch = require("node-fetch");
-const objectHash = require("object-hash");
 const kleur = require("kleur");
+const fsPromises = require("fs/promises");
 
-/**
- * To cache HTTP requests on disk with quick lookup and insertion, we store the hashed request.
- * This uses SHA1 hashes. They are uni-directional hashes, which works for this use case. Most importantly,
- * they're unique enough and can be expressed in a case-insensitive way so it works on Windows filesystems.
- * And they are 40 hex characters, so the length won't be too long no matter what the request payload.
- * @param {Object} request
- */
-function requestToString(request) {
-  return objectHash(request);
-}
-/**
- * @param {Object} request
- */
-function fullPath(portsHash, request, hasFsAccess) {
-  const requestWithPortHash =
-    request.url === "elm-pages-internal://port"
-      ? { portsHash, ...request }
-      : request;
-  if (hasFsAccess) {
-    return path.join(
-      process.cwd(),
-      // TODO use parameter or something other than global for this `global.isRunningGenerator` condition
-      ".elm-pages",
-      "http-response-cache",
-      requestToString(requestWithPortHash)
-    );
-  } else {
-    return path.join("/", requestToString(requestWithPortHash));
-  }
-}
+const defaultHttpCachePath = "./.elm-pages/http-cache";
+
+/** @typedef {{kind: 'cache-response-path', value: string} | {kind: 'response-json', value: JSON}} Response */
 
 /**
  * @param {string} mode
  * @param {{url: string;headers: {[x: string]: string;};method: string;body: Body;}} rawRequest
- * @returns {Promise<string>}
  * @param {string} portsFile
  * @param {boolean} hasFsAccess
+ * @returns {Promise<Response>}
  */
 function lookupOrPerform(portsFile, mode, rawRequest, hasFsAccess, useCache) {
-  const { fs } = require("./request-cache-fs.js")(hasFsAccess);
+  const makeFetchHappen = require("make-fetch-happen").defaults({
+    cache: mode === "build" ? "no-cache" : "default",
+  });
   return new Promise(async (resolve, reject) => {
     const request = toRequest(rawRequest);
-    const portsHash = (portsFile && portsFile.match(/-([^-]+)\.js$/)[1]) || "";
-    const responsePath = fullPath(portsHash, request, hasFsAccess);
 
-    // TODO check cache expiration time and delete and go to else if expired
-    if (useCache && (await checkFileExists(fs, responsePath))) {
-      // console.log("Skipping request, found file.");
-      resolve(responsePath);
-    } else {
-      let portDataSource = {};
-      let portDataSourceImportError = null;
-      try {
-        if (portsFile === undefined) {
-          throw "missing";
-        }
-        const portDataSourcePath = path.resolve(portsFile);
-        // On Windows, we need cannot use paths directly and instead must use a file:// URL.
-        // portDataSource = await require(url.pathToFileURL(portDataSourcePath).href);
-        portDataSource = require(portDataSourcePath);
-      } catch (e) {
-        portDataSourceImportError = e;
+    let portBackendTask = {};
+    let portBackendTaskImportError = null;
+    try {
+      if (portsFile === undefined) {
+        throw "missing";
       }
+      const portBackendTaskPath = path.resolve(portsFile);
+      // On Windows, we need cannot use paths directly and instead must use a file:// URL.
+      // portBackendTask = await require(url.pathToFileURL(portBackendTaskPath).href);
+      portBackendTask = require(portBackendTaskPath);
+    } catch (e) {
+      portBackendTaskImportError = e;
+    }
 
-      if (request.url === "elm-pages-internal://port") {
-        try {
-          const { input, portName } = rawRequest.body.args[0];
-
-          if (!portDataSource[portName]) {
-            if (portDataSourceImportError === null) {
-              throw `DataSource.Port.send "${portName}" was called, but I couldn't find a function with that name in the port definitions file. Is it exported correctly?`;
-            } else if (portDataSourceImportError === "missing") {
-              throw `DataSource.Port.send "${portName}" was called, but I couldn't find the port definitions file. Be sure to create a 'port-data-source.ts' or 'port-data-source.js' file and maybe restart the dev server.`;
-            } else {
-              throw `DataSource.Port.send "${portName}" was called, but I couldn't import the port definitions file, because of this exception: \`${portDataSourceImportError}\` Are there syntax errors or expections thrown during import?`;
-            }
-          } else if (typeof portDataSource[portName] !== "function") {
-            throw `DataSource.Port.send "${portName}" was called, but it is not a function. Be sure to export a function with that name from port-data-source.js`;
-          }
-          await fs.promises.writeFile(
-            responsePath,
-            JSON.stringify(jsonResponse(await portDataSource[portName](input)))
-          );
-          resolve(responsePath);
-        } catch (error) {
-          console.trace(error);
-          reject({
-            title: "DataSource.Port Error",
-            message: error.toString(),
-          });
-        }
-      } else {
-        try {
-          console.time(`fetch ${request.url}`);
-          const response = await fetch(request.url, {
-            method: request.method,
-            body: request.body,
-            headers: {
-              "User-Agent": "request",
-              ...request.headers,
-            },
-          });
+    if (request.url === "elm-pages-internal://port") {
+      try {
+        const { input, portName } = rawRequest.body.args[0];
 
-          console.timeEnd(`fetch ${request.url}`);
-          const expectString = request.headers["elm-pages-internal"];
-
-          if (response.ok || expectString === "ExpectResponse") {
-            let body;
-            let bodyKind;
-            if (expectString === "ExpectJson") {
-              body = await response.json();
-              bodyKind = "json";
-            } else if (
-              expectString === "ExpectBytes" ||
-              expectString === "ExpectBytesResponse"
-            ) {
-              bodyKind = "bytes";
-              const arrayBuffer = await response.arrayBuffer();
-              body = Buffer.from(arrayBuffer).toString("base64");
-            } else if (expectString === "ExpectWhatever") {
-              bodyKind = "whatever";
-              body = null;
-            } else if (
-              expectString === "ExpectResponse" ||
-              expectString === "ExpectString"
-            ) {
-              bodyKind = "string";
-              body = await response.text();
-            } else {
-              throw `Unexpected expectString ${expectString}`;
-            }
-
-            await fs.promises.writeFile(
-              responsePath,
-              JSON.stringify({
-                headers: Object.fromEntries(response.headers.entries()),
-                statusCode: response.status,
-                body: body,
-                bodyKind,
-                url: response.url,
-                statusText: response.statusText,
-              })
-            );
-
-            resolve(responsePath);
+        if (!portBackendTask[portName]) {
+          if (portBackendTaskImportError === null) {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "PortNotDefined",
+              }),
+            });
+          } else if (portBackendTaskImportError === "missing") {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "MissingPortsFile",
+              }),
+            });
           } else {
-            console.log("@@@ request-cache1 bad HTTP response");
-            reject({
-              title: "DataSource.Http Error",
-              message: `${kleur
-                .yellow()
-                .underline(request.url)} Bad HTTP response ${response.status} ${
-                response.statusText
-              }
-`,
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "ErrorInPortsFile",
+                error:
+                  (portBackendTaskImportError &&
+                    portBackendTaskImportError.stack) ||
+                  "",
+              }),
             });
           }
-        } catch (error) {
-          console.trace("@@@ request-cache2 HTTP error", error);
-          reject({
-            title: "DataSource.Http Error",
-            message: `${kleur
-              .yellow()
-              .underline(request.url)} ${error.toString()}
-`,
+        } else if (typeof portBackendTask[portName] !== "function") {
+          resolve({
+            kind: "response-json",
+            value: jsonResponse({
+              "elm-pages-internal-error": "ExportIsNotFunction",
+              error: typeof portBackendTask[portName],
+            }),
           });
+        } else {
+          try {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse(await portBackendTask[portName](input)),
+            });
+          } catch (portCallError) {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "PortCallException",
+                error: portCallError,
+              }),
+            });
+          }
         }
+      } catch (error) {
+        console.trace(error);
+        reject({
+          title: "BackendTask.Port Error",
+          message: error.toString(),
+        });
+      }
+    } else {
+      try {
+        console.time(`fetch ${request.url}`);
+        const response = await safeFetch(makeFetchHappen, request.url, {
+          method: request.method,
+          body: request.body,
+          headers: {
+            "User-Agent": "request",
+            ...request.headers,
+          },
+          ...rawRequest.cacheOptions,
+        });
+
+        console.timeEnd(`fetch ${request.url}`);
+        const expectString = request.headers["elm-pages-internal"];
+
+        let body;
+        let bodyKind;
+        if (expectString === "ExpectJson") {
+          try {
+            body = await response.buffer();
+            body = JSON.parse(body.toString("utf-8"));
+            bodyKind = "json";
+          } catch (error) {
+            body = body.toString("utf8");
+            bodyKind = "string";
+          }
+        } else if (
+          expectString === "ExpectBytes" ||
+          expectString === "ExpectBytesResponse"
+        ) {
+          body = await response.buffer();
+          try {
+            body = body.toString("base64");
+            bodyKind = "bytes";
+          } catch (e) {
+            body = body.toString("utf8");
+            bodyKind = "string";
+          }
+        } else if (expectString === "ExpectWhatever") {
+          bodyKind = "whatever";
+          body = null;
+        } else if (
+          expectString === "ExpectResponse" ||
+          expectString === "ExpectString"
+        ) {
+          bodyKind = "string";
+          body = await response.text();
+        } else {
+          throw `Unexpected expectString ${expectString}`;
+        }
+
+        resolve({
+          kind: "response-json",
+          value: {
+            headers: Object.fromEntries(response.headers.entries()),
+            statusCode: response.status,
+            body,
+            bodyKind,
+            url: response.url,
+            statusText: response.statusText,
+          },
+        });
+      } catch (error) {
+        console.trace("@@@ request-cache2 HTTP error", error);
+        reject({
+          title: "BackendTask.Http Error",
+          message: `${kleur.yellow().underline(request.url)} ${error.toString()}
+`,
+        });
       }
     }
   });
@@ -189,15 +185,6 @@ function toRequest(elmRequest) {
     body: toBody(elmRequest.body),
   };
 }
-/**
- * @param {string} file
- */
-function checkFileExists(fs, file) {
-  return fs.promises
-    .access(file, fs.constants.F_OK)
-    .then(() => true)
-    .catch(() => false);
-}
 /**
  * @param {Body} body
  */
@@ -209,6 +196,9 @@ function toBody(body) {
     case "StringBody": {
       return body.args[1];
     }
+    case "BytesBody": {
+      return Buffer.from(body.args[1], "base64");
+    }
     case "JsonBody": {
       return JSON.stringify(body.args[0]);
     }
@@ -227,13 +217,16 @@ function toContentType(body) {
     case "StringBody": {
       return { "Content-Type": body.args[0] };
     }
+    case "BytesBody": {
+      return { "Content-Type": body.args[0] };
+    }
     case "JsonBody": {
       return { "Content-Type": "application/json" };
     }
   }
 }
 
-/** @typedef { { tag: 'EmptyBody'} | { tag: 'StringBody'; args: [string, string] } | {tag: 'JsonBody'; args: [ Object ] } } Body  */
+/** @typedef { { tag: 'EmptyBody'} |{ tag: 'BytesBody'; args: [string, string] } |  { tag: 'StringBody'; args: [string, string] } | {tag: 'JsonBody'; args: [ Object ] } } Body  */
 function requireUncached(mode, filePath) {
   if (mode === "dev-server") {
     // for the build command, we can skip clearing the cache because it won't change while the build is running
@@ -250,4 +243,32 @@ function jsonResponse(json) {
   return { bodyKind: "json", body: json };
 }
 
+async function safeFetch(makeFetchHappen, url, options) {
+  const { cachePath, ...optionsWithoutCachePath } = options;
+  const cachePathWithDefault = cachePath || defaultHttpCachePath;
+  if (await canAccess(cachePathWithDefault)) {
+    return await makeFetchHappen(url, {
+      cachePath: cachePathWithDefault,
+      ...options,
+    });
+  } else {
+    return await makeFetchHappen(url, {
+      cache: "no-store",
+      ...optionsWithoutCachePath,
+    });
+  }
+}
+
+async function canAccess(filePath) {
+  try {
+    await fsPromises.access(
+      filePath,
+      fsPromises.constants.R_OK | fsPromises.constants.W_OK
+    );
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 module.exports = { lookupOrPerform };

package/generator/static-code/hmr.js

@@ -224,8 +224,11 @@ function htmlSanitize(str, type) {
   );
 }
 
-const parseHeader = (title, path) =>
-  `-- ${title.replace("-", " ")} --------------- ${path}`;
+function parseHeader(title, path) {
+  return `-- ${(title || "ERROR").replace("-", " ")} --------------- ${
+    path || ""
+  }`;
+}
 
 /*
   |-------------------------------------------------------------------------------
@@ -253,6 +256,14 @@ const parseConsoleErrors =
    * */
   (info) => {
     if (info.rule) {
+      if (info.details) {
+        return joinMessage(
+          info.details.reduce(consoleMsg, {
+            error: [consoleHeader(info.rule, path)],
+            style: [styleColor("blue")],
+          })
+        );
+      }
       return joinMessage(
         info.formatted.reduce(consoleMsg, {
           error: [consoleHeader(info.rule, path)],
@@ -313,6 +324,9 @@ const parseHtmlErrors = (path) => (info) => {
   if (info.rule) {
     return info.formatted.reduce(htmlMsg, htmlHeader(info.rule, path));
   } else {
+    if (info.details) {
+      return info.details.reduce(htmlMsg, htmlHeader(info.title, path));
+    }
     return info.message.reduce(htmlMsg, htmlHeader(info.title, path));
   }
 };

package/generator/template/app/Api.elm

@@ -1,14 +1,14 @@
 module Api exposing (routes)
 
 import ApiRoute exposing (ApiRoute)
-import DataSource exposing (DataSource)
+import BackendTask exposing (BackendTask)
 import Html exposing (Html)
 import Pages.Manifest as Manifest
 import Route exposing (Route)
 
 
 routes :
-    DataSource (List Route)
+    BackendTask (List Route)
     -> (Maybe { indent : Int, newLines : Bool } -> Html Never -> String)
     -> List (ApiRoute ApiRoute.Response)
 routes getStaticRoutes htmlToString =

package/generator/template/app/Route/Index.elm

@@ -1,6 +1,6 @@
 module Route.Index exposing (Data, Model, Msg, route)
 
-import DataSource exposing (DataSource)
+import BackendTask exposing (BackendTask)
 import Head
 import Head.Seo as Seo
 import Html
@@ -38,9 +38,9 @@ route =
         |> RouteBuilder.buildNoState { view = view }
 
 
-data : DataSource Data
+data : BackendTask Data
 data =
-    DataSource.succeed Data
+    BackendTask.succeed Data
 
 
 head :

package/generator/template/app/Shared.elm

@@ -1,7 +1,7 @@
 module Shared exposing (Data, Model, Msg(..), SharedMsg(..), template)
 
 import Browser.Navigation
-import DataSource
+import BackendTask
 import Html exposing (Html)
 import Html.Events
 import Pages.Flags
@@ -76,9 +76,9 @@ subscriptions _ _ =
     Sub.none
 
 
-data : DataSource.DataSource Data
+data : BackendTask.BackendTask Data
 data =
-    DataSource.succeed ()
+    BackendTask.succeed ()
 
 
 view :

package/generator/template/app/Site.elm

@@ -1,6 +1,6 @@
 module Site exposing (config)
 
-import DataSource exposing (DataSource)
+import BackendTask exposing (BackendTask)
 import Head
 import SiteConfig exposing (SiteConfig)
 
@@ -12,7 +12,7 @@ config =
     }
 
 
-head : DataSource (List Head.Tag)
+head : BackendTask (List Head.Tag)
 head =
     [ Head.metaName "viewport" (Head.raw "width=device-width,initial-scale=1")
     , Head.metaName "mobile-web-app-capable" (Head.raw "yes")
@@ -21,4 +21,4 @@ head =
     , Head.metaName "apple-mobile-web-app-status-bar-style" (Head.raw "black-translucent")
     , Head.sitemapLink "/sitemap.xml"
     ]
-        |> DataSource.succeed
+        |> BackendTask.succeed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "elm-pages",
-  "version": "3.0.0-beta.13",
+  "version": "3.0.0-beta.15",
   "homepage": "https://elm-pages.com",
   "moduleResolution": "node",
   "description": "Type-safe static sites, written in pure elm with your own custom elm-markup syntax.",
@@ -26,43 +26,43 @@
   "dependencies": {
     "busboy": "^1.0.0",
     "chokidar": "^3.5.3",
-    "commander": "9.4.1",
+    "commander": "^9.5.0",
     "connect": "^3.7.0",
     "cookie-signature": "^1.1.0",
     "cross-spawn": "7.0.3",
     "devcert": "^1.2.2",
     "elm-doc-preview": "^5.0.5",
     "elm-hot": "^1.1.6",
-    "esbuild": "^0.16.1",
+    "esbuild": "^0.16.15",
     "fs-extra": "^10.1.0",
     "globby": "11.0.4",
     "gray-matter": "^4.0.3",
     "jsesc": "^3.0.2",
     "kleur": "^4.1.5",
+    "make-fetch-happen": "^11.0.2",
     "memfs": "^3.4.7",
     "micromatch": "^4.0.5",
-    "node-fetch": "^2.6.7",
-    "object-hash": "^2.2.0",
     "serve-static": "^1.15.0",
     "terser": "^5.16.1",
-    "vite": "^3.2.5",
+    "vite": "^4.0.4",
     "which": "^2.0.2"
   },
   "devDependencies": {
     "@types/cross-spawn": "^6.0.2",
     "@types/fs-extra": "^9.0.13",
+    "@types/make-fetch-happen": "^10.0.1",
     "@types/micromatch": "^4.0.2",
     "@types/node": "^18.11.9",
     "@types/serve-static": "^1.15.0",
-    "cypress": "^11.1.0",
+    "cypress": "^12.3.0",
     "elm-codegen": "^0.2.0",
     "elm-optimize-level-2": "^0.3.5",
     "elm-review": "^2.8.2",
-    "elm-test": "^0.19.1-revision10",
-    "elm-tooling": "^1.10.0",
+    "elm-test": "^0.19.1-revision11",
+    "elm-tooling": "^1.11.0",
     "elm-verify-examples": "^5.2.0",
     "elmi-to-json": "^1.2.0",
-    "mocha": "^10.0.0",
+    "mocha": "^10.2.0",
     "typescript": "^4.9.3"
   },
   "files": [