elm-pages 3.0.0-beta.3 → 3.0.0-beta.31

package/generator/src/request-cache.js

@@ -1,179 +1,245 @@
-const path = require("path");
-const fetch = require("node-fetch");
-const objectHash = require("object-hash");
-const kleur = require("kleur");
+import * as path from "path";
+import * as fsPromises from "fs/promises";
+import * as kleur from "kleur/colors";
+import { default as makeFetchHappenOriginal } from "make-fetch-happen";
 
-/**
- * To cache HTTP requests on disk with quick lookup and insertion, we store the hashed request.
- * This uses SHA1 hashes. They are uni-directional hashes, which works for this use case. Most importantly,
- * they're unique enough and can be expressed in a case-insensitive way so it works on Windows filesystems.
- * And they are 40 hex characters, so the length won't be too long no matter what the request payload.
- * @param {Object} request
- */
-function requestToString(request) {
-  return objectHash(request);
-}
-/**
- * @param {Object} request
- */
-function fullPath(portsHash, request, hasFsAccess) {
-  const requestWithPortHash =
-    request.url === "elm-pages-internal://port"
-      ? { portsHash, ...request }
-      : request;
-  if (hasFsAccess) {
-    return path.join(
-      process.cwd(),
-      ".elm-pages",
-      "http-response-cache",
-      requestToString(requestWithPortHash)
-    );
-  } else {
-    return path.join("/", requestToString(requestWithPortHash));
-  }
-}
+const defaultHttpCachePath = "./.elm-pages/http-cache";
+
+/** @typedef {{kind: 'cache-response-path', value: string} | {kind: 'response-json', value: JSON}} Response */
 
 /**
  * @param {string} mode
  * @param {{url: string;headers: {[x: string]: string;};method: string;body: Body;}} rawRequest
- * @returns {Promise<string>}
- * @param {string} portsFile
+ * @param {Record<string, unknown>} portsFile
  * @param {boolean} hasFsAccess
+ * @returns {Promise<Response>}
  */
-function lookupOrPerform(portsFile, mode, rawRequest, hasFsAccess, useCache) {
-  const { fs } = require("./request-cache-fs.js")(hasFsAccess);
+export function lookupOrPerform(
+  portsFile,
+  mode,
+  rawRequest,
+  hasFsAccess,
+  useCache
+) {
+  const makeFetchHappen = makeFetchHappenOriginal.defaults({
+    cache: mode === "build" ? "no-cache" : "default",
+  });
   return new Promise(async (resolve, reject) => {
     const request = toRequest(rawRequest);
-    const portsHash = (portsFile && portsFile.match(/-([^-]+)\.js$/)[1]) || "";
-    const responsePath = fullPath(portsHash, request, hasFsAccess);
 
-    // TODO check cache expiration time and delete and go to else if expired
-    if (useCache && (await checkFileExists(fs, responsePath))) {
-      // console.log("Skipping request, found file.");
-      resolve(responsePath);
-    } else {
-      let portDataSource = {};
-      let portDataSourceImportError = null;
-      try {
-        if (portsFile === undefined) {
-          throw "missing";
-        }
-        const portDataSourcePath = path.resolve(portsFile);
-        // On Windows, we need cannot use paths directly and instead must use a file:// URL.
-        // portDataSource = await require(url.pathToFileURL(portDataSourcePath).href);
-        portDataSource = require(portDataSourcePath);
-      } catch (e) {
-        portDataSourceImportError = e;
+    let portBackendTask = portsFile;
+    let portBackendTaskImportError = null;
+    try {
+      if (portsFile === undefined) {
+        throw "missing";
       }
+    } catch (e) {
+      portBackendTaskImportError = e;
+    }
 
-      if (request.url === "elm-pages-internal://port") {
-        try {
-          const { input, portName } = rawRequest.body.args[0];
-
-          if (!portDataSource[portName]) {
-            if (portDataSourceImportError === null) {
-              throw `DataSource.Port.send "${portName}" was called, but I couldn't find a function with that name in the port definitions file. Is it exported correctly?`;
-            } else if (portDataSourceImportError === "missing") {
-              throw `DataSource.Port.send "${portName}" was called, but I couldn't find the port definitions file. Be sure to create a 'port-data-source.ts' or 'port-data-source.js' file and maybe restart the dev server.`;
-            } else {
-              throw `DataSource.Port.send "${portName}" was called, but I couldn't import the port definitions file, because of this exception: \`${portDataSourceImportError}\` Are there syntax errors or expections thrown during import?`;
-            }
-          } else if (typeof portDataSource[portName] !== "function") {
-            throw `DataSource.Port.send "${portName}" was called, but it is not a function. Be sure to export a function with that name from port-data-source.js`;
-          }
-          await fs.promises.writeFile(
-            responsePath,
-            JSON.stringify(jsonResponse(await portDataSource[portName](input)))
-          );
-          resolve(responsePath);
-        } catch (error) {
-          console.trace(error);
-          reject({
-            title: "DataSource.Port Error",
-            message: error.toString(),
+    if (request.url === "elm-pages-internal://port") {
+      try {
+        const { input, portName } = rawRequest.body.args[0];
+
+        if (portBackendTask === null) {
+          resolve({
+            kind: "response-json",
+            value: jsonResponse({
+              "elm-pages-internal-error": "MissingCustomBackendTaskFile",
+            }),
           });
-        }
-      } else {
-        try {
-          console.time(`fetch ${request.url}`);
-          const response = await fetch(request.url, {
-            method: request.method,
-            body: request.body,
-            headers: {
-              "User-Agent": "request",
-              ...request.headers,
-            },
+        } else if (portBackendTask && portBackendTask.__internalElmPagesError) {
+          resolve({
+            kind: "response-json",
+            value: jsonResponse({
+              "elm-pages-internal-error": "ErrorInCustomBackendTaskFile",
+              error: portBackendTask.__internalElmPagesError,
+            }),
           });
-
-          console.timeEnd(`fetch ${request.url}`);
-          const expectString = request.headers["elm-pages-internal"];
-
-          if (response.ok || expectString === "ExpectResponse") {
-            let body;
-            let bodyKind;
-            if (expectString === "ExpectJson") {
-              body = await response.json();
-              bodyKind = "json";
-            } else if (
-              expectString === "ExpectBytes" ||
-              expectString === "ExpectBytesResponse"
-            ) {
-              bodyKind = "bytes";
-              const arrayBuffer = await response.arrayBuffer();
-              body = Buffer.from(arrayBuffer).toString("base64");
-            } else if (expectString === "ExpectWhatever") {
-              bodyKind = "whatever";
-              body = null;
-            } else if (
-              expectString === "ExpectResponse" ||
-              expectString === "ExpectString"
-            ) {
-              bodyKind = "string";
-              body = await response.text();
-            } else {
-              throw `Unexpected expectString ${expectString}`;
-            }
-
-            await fs.promises.writeFile(
-              responsePath,
-              JSON.stringify({
-                headers: Object.fromEntries(response.headers.entries()),
-                statusCode: response.status,
-                body: body,
-                bodyKind,
-                url: response.url,
-                statusText: response.statusText,
-              })
-            );
-
-            resolve(responsePath);
+        } else if (portBackendTask && !portBackendTask[portName]) {
+          if (portBackendTaskImportError === null) {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "CustomBackendTaskNotDefined",
+              }),
+            });
+          } else if (portBackendTaskImportError === "missing") {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "MissingCustomBackendTaskFile",
+              }),
+            });
           } else {
-            console.log("@@@ request-cache1 bad HTTP response");
-            reject({
-              title: "DataSource.Http Error",
-              message: `${kleur
-                .yellow()
-                .underline(request.url)} Bad HTTP response ${response.status} ${
-                response.statusText
-              }
-`,
+            resolve({
+              kind: "response-json",
+              value: jsonResponse({
+                "elm-pages-internal-error": "ErrorInCustomBackendTaskFile",
+                error:
+                  (portBackendTaskImportError &&
+                    portBackendTaskImportError.stack) ||
+                  "",
+              }),
             });
           }
-        } catch (error) {
-          console.trace("@@@ request-cache2 HTTP error", error);
-          reject({
-            title: "DataSource.Http Error",
-            message: `${kleur
-              .yellow()
-              .underline(request.url)} ${error.toString()}
-`,
+        } else if (typeof portBackendTask[portName] !== "function") {
+          resolve({
+            kind: "response-json",
+            value: jsonResponse({
+              "elm-pages-internal-error": "ExportIsNotFunction",
+              error: typeof portBackendTask[portName],
+            }),
           });
+        } else {
+          console.time(`BackendTask.Custom.run "${portName}"`);
+          try {
+            resolve({
+              kind: "response-json",
+              value: jsonResponse(
+                toElmJson(await portBackendTask[portName](input))
+              ),
+            });
+          } catch (portCallError) {
+            if (portCallError instanceof Error) {
+              resolve({
+                kind: "response-json",
+                value: jsonResponse({
+                  "elm-pages-internal-error": "NonJsonException",
+                  error: portCallError.message,
+                  stack: portCallError.stack || null,
+                }),
+              });
+            }
+            try {
+              resolve({
+                kind: "response-json",
+                value: jsonResponse({
+                  "elm-pages-internal-error": "CustomBackendTaskException",
+                  error: JSON.parse(JSON.stringify(portCallError, null, 0)),
+                }),
+              });
+            } catch (jsonDecodeError) {
+              resolve({
+                kind: "response-json",
+                value: jsonResponse({
+                  "elm-pages-internal-error": "NonJsonException",
+                  error: portCallError.toString(),
+                }),
+              });
+            }
+          }
+          console.timeEnd(`BackendTask.Custom.run "${portName}"`);
         }
+      } catch (error) {
+        console.trace(error);
+        reject({
+          title: "BackendTask.Custom Error",
+          message: error.toString(),
+        });
+      }
+    } else {
+      try {
+        console.time(`fetch ${request.url}`);
+        const response = await safeFetch(makeFetchHappen, request.url, {
+          method: request.method,
+          body: request.body,
+          headers: {
+            "User-Agent": "request",
+            ...request.headers,
+          },
+          ...rawRequest.cacheOptions,
+        });
+
+        console.timeEnd(`fetch ${request.url}`);
+        const expectString = request.headers["elm-pages-internal"];
+
+        let body;
+        let bodyKind;
+        if (expectString === "ExpectJson") {
+          try {
+            body = await response.buffer();
+            body = JSON.parse(body.toString("utf-8"));
+            bodyKind = "json";
+          } catch (error) {
+            body = body.toString("utf8");
+            bodyKind = "string";
+          }
+        } else if (
+          expectString === "ExpectBytes" ||
+          expectString === "ExpectBytesResponse"
+        ) {
+          body = await response.buffer();
+          try {
+            body = body.toString("base64");
+            bodyKind = "bytes";
+          } catch (e) {
+            body = body.toString("utf8");
+            bodyKind = "string";
+          }
+        } else if (expectString === "ExpectWhatever") {
+          bodyKind = "whatever";
+          body = null;
+        } else if (
+          expectString === "ExpectResponse" ||
+          expectString === "ExpectString"
+        ) {
+          bodyKind = "string";
+          body = await response.text();
+        } else {
+          throw `Unexpected expectString ${expectString}`;
+        }
+
+        resolve({
+          kind: "response-json",
+          value: {
+            headers: Object.fromEntries(response.headers.entries()),
+            statusCode: response.status,
+            body,
+            bodyKind,
+            url: response.url,
+            statusText: response.statusText,
+          },
+        });
+      } catch (error) {
+        console.trace("@@@ request-cache2 HTTP error", error);
+        reject({
+          title: "BackendTask.Http Error",
+          message: `${kleur.yellow().underline(request.url)} ${error.toString()}
+`,
+        });
       }
     }
   });
 }
 
+/**
+ * @param {unknown} obj
+ * @returns {JSON}
+ */
+function toElmJson(obj) {
+  if (Array.isArray(obj)) {
+    return obj.map(toElmJson);
+  } else if (typeof obj === "object") {
+    for (let key in obj) {
+      const value = obj[key];
+      if (typeof value === "undefined") {
+        obj[key] = null;
+      } else if (value instanceof Date) {
+        obj[key] = {
+          "__elm-pages-normalized__": {
+            kind: "Date",
+            value: Math.floor(value.getTime()),
+          },
+        };
+        // } else if (value instanceof Object) {
+        //   toElmJson(obj);
+      }
+    }
+  }
+  return obj;
+}
+
 /**
  * @param {{url: string; headers: {[x: string]: string}; method: string; body: Body } } elmRequest
  */
@@ -188,15 +254,6 @@ function toRequest(elmRequest) {
     body: toBody(elmRequest.body),
   };
 }
-/**
- * @param {string} file
- */
-function checkFileExists(fs, file) {
-  return fs.promises
-    .access(file, fs.constants.F_OK)
-    .then(() => true)
-    .catch(() => false);
-}
 /**
  * @param {Body} body
  */
@@ -208,6 +265,9 @@ function toBody(body) {
     case "StringBody": {
       return body.args[1];
     }
+    case "BytesBody": {
+      return Buffer.from(body.args[1], "base64");
+    }
     case "JsonBody": {
       return JSON.stringify(body.args[0]);
     }
@@ -226,13 +286,16 @@ function toContentType(body) {
     case "StringBody": {
       return { "Content-Type": body.args[0] };
     }
+    case "BytesBody": {
+      return { "Content-Type": body.args[0] };
+    }
     case "JsonBody": {
       return { "Content-Type": "application/json" };
     }
   }
 }
 
-/** @typedef { { tag: 'EmptyBody'} | { tag: 'StringBody'; args: [string, string] } | {tag: 'JsonBody'; args: [ Object ] } } Body  */
+/** @typedef { { tag: 'EmptyBody'} |{ tag: 'BytesBody'; args: [string, string] } |  { tag: 'StringBody'; args: [string, string] } | {tag: 'JsonBody'; args: [ Object ] } } Body  */
 function requireUncached(mode, filePath) {
   if (mode === "dev-server") {
     // for the build command, we can skip clearing the cache because it won't change while the build is running
@@ -249,4 +312,30 @@ function jsonResponse(json) {
   return { bodyKind: "json", body: json };
 }
 
-module.exports = { lookupOrPerform };
+async function safeFetch(makeFetchHappen, url, options) {
+  const { cachePath, ...optionsWithoutCachePath } = options;
+  const cachePathWithDefault = cachePath || defaultHttpCachePath;
+  if (await canAccess(cachePathWithDefault)) {
+    return await makeFetchHappen(url, {
+      cachePath: cachePathWithDefault,
+      ...options,
+    });
+  } else {
+    return await makeFetchHappen(url, {
+      cache: "no-store",
+      ...optionsWithoutCachePath,
+    });
+  }
+}
+
+async function canAccess(filePath) {
+  try {
+    await fsPromises.access(
+      filePath,
+      fsPromises.constants.R_OK | fsPromises.constants.W_OK
+    );
+    return true;
+  } catch {
+    return false;
+  }
+}

package/generator/src/resolve-elm-module.js

@@ -0,0 +1,63 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+function findNearestElmJson(filePath) {
+  function searchForElmJson(directory) {
+    if (directory === "/") {
+      return null;
+    }
+
+    const elmJsonPath = path.join(directory, "elm.json");
+    return fs.existsSync(elmJsonPath)
+      ? elmJsonPath
+      : searchForElmJson(path.dirname(directory));
+  }
+
+  return searchForElmJson(path.dirname(filePath));
+}
+
+function getElmModuleName(inputPath) {
+  const filePath = path.normalize(
+    path.isAbsolute(inputPath) ? inputPath : path.resolve(inputPath)
+  );
+  const elmJsonPath = findNearestElmJson(filePath);
+
+  if (!elmJsonPath) {
+    throw new Error("No elm.json found");
+  }
+
+  const elmJson = JSON.parse(fs.readFileSync(elmJsonPath, "utf8"));
+  const sourceDirectories = elmJson["source-directories"];
+  const projectDirectory = path.dirname(elmJsonPath);
+
+  const matchingSourceDir = sourceDirectories
+    .map((sourceDir) => path.join(projectDirectory, sourceDir))
+    .find((absoluteSourceDir) => filePath.startsWith(absoluteSourceDir));
+
+  if (!matchingSourceDir) {
+    throw new Error(
+      "File is not in any source-directories specified in elm.json"
+    );
+  }
+
+  const relativePath = path.relative(matchingSourceDir, filePath);
+  const moduleName = relativePath
+    .replace(path.extname(relativePath), "")
+    .replace("/", ".");
+
+  return { projectDirectory, moduleName, sourceDirectory: matchingSourceDir };
+}
+
+export function resolveInputPathOrModuleName(inputPathOrModuleName) {
+  if (
+    /^[A-Z][a-zA-Z0-9_]*(\.[A-Z][a-zA-Z0-9_]*)*$/.test(inputPathOrModuleName)
+  ) {
+    return {
+      moduleName: inputPathOrModuleName,
+      projectDirectory: "./script",
+      sourceDirectory: "./script/src",
+    };
+  } else {
+    return getElmModuleName(inputPathOrModuleName);
+  }
+}

package/generator/src/rewrite-client-elm-json.js

@@ -1,6 +1,6 @@
-const fs = require("fs");
+import * as fs from "node:fs";
 
-module.exports = async function () {
+export async function rewriteClientElmJson() {
   var elmJson = JSON.parse(
     (await fs.promises.readFile("./elm.json")).toString()
   );
@@ -9,11 +9,11 @@ module.exports = async function () {
 
   await writeFileIfChanged(
     "./elm-stuff/elm-pages/client/elm.json",
-    JSON.stringify(rewriteElmJson(elmJson))
+    JSON.stringify(rewriteClientElmJsonHelp(elmJson))
   );
-};
+}
 
-function rewriteElmJson(elmJson) {
+function rewriteClientElmJsonHelp(elmJson) {
   // The internal generated file will be at:
   // ./elm-stuff/elm-pages/
   // So, we need to take the existing elmJson and
@@ -27,6 +27,7 @@ function rewriteElmJson(elmJson) {
   elmJson["source-directories"] = elmJson["source-directories"].map((item) => {
     return "../../../" + item;
   });
+  elmJson["dependencies"]["direct"]["lamdera/codecs"] = "1.0.0";
   // 3. add our own secret My.elm module 😈
   elmJson["source-directories"].push(".elm-pages");
   elmJson["source-directories"].push("app");

package/generator/src/rewrite-elm-json-help.js

@@ -0,0 +1,56 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+/**
+ * @param {string} sourceElmJsonPath
+ * @param {string} targetElmJsonPath
+ * @param {( (arg0: JSON) => JSON )?} modifyElmJson
+ */
+export async function rewriteElmJson(
+  sourceElmJsonPath,
+  targetElmJsonPath,
+  modifyElmJson
+) {
+  if (!modifyElmJson) {
+    modifyElmJson = function (json) {
+      return json;
+    };
+  }
+  var elmJson = JSON.parse(
+    (
+      await fs.promises.readFile(path.join(sourceElmJsonPath, "elm.json"))
+    ).toString()
+  );
+
+  let modifiedElmJson = modifyElmJson(elmJson);
+  // always add `lamdera/codecs` dependency
+  modifiedElmJson["dependencies"]["direct"]["lamdera/codecs"] = "1.0.0";
+
+  // write new elm.json
+  await writeFileIfChanged(
+    path.join(targetElmJsonPath, "elm.json"),
+    JSON.stringify(modifiedElmJson)
+  );
+}
+
+/**
+ * @param {fs.PathLike | fs.promises.FileHandle} filePath
+ * @param {string | NodeJS.ArrayBufferView | Iterable<string | NodeJS.ArrayBufferView> | AsyncIterable<string | NodeJS.ArrayBufferView> | import("stream").Stream} content
+ */
+async function writeFileIfChanged(filePath, content) {
+  if (
+    !(await fileExists(filePath)) ||
+    (await fs.promises.readFile(filePath, "utf8")) !== content
+  ) {
+    await fs.promises.writeFile(filePath, content);
+  }
+}
+/**
+ * @param {fs.PathLike} file
+ */
+function fileExists(file) {
+  return fs.promises
+    .access(file, fs.constants.F_OK)
+    .then(() => true)
+    .catch(() => false);
+}

package/generator/src/rewrite-elm-json.js

@@ -1,19 +1,23 @@
-const fs = require("fs");
+import * as fs from "node:fs";
 
-module.exports = async function () {
+export async function rewriteElmJson(
+  sourceElmJsonPath,
+  targetElmJsonPath,
+  options
+) {
   var elmJson = JSON.parse(
-    (await fs.promises.readFile("./elm.json")).toString()
+    (await fs.promises.readFile(sourceElmJsonPath)).toString()
   );
 
   // write new elm.json
 
   await writeFileIfChanged(
-    "./elm-stuff/elm-pages/elm.json",
-    JSON.stringify(rewriteElmJson(elmJson))
+    targetElmJsonPath,
+    JSON.stringify(rewriteElmJsonHelp(elmJson, options))
   );
-};
+}
 
-function rewriteElmJson(elmJson) {
+function rewriteElmJsonHelp(elmJson, options) {
   // The internal generated file will be at:
   // ./elm-stuff/elm-pages/
   // So, we need to take the existing elmJson and
@@ -27,6 +31,12 @@ function rewriteElmJson(elmJson) {
   elmJson["source-directories"] = elmJson["source-directories"].map((item) => {
     return "../../" + item;
   });
+  if (options && options.executableName === "elm") {
+    // elm, don't add lamdera/codecs
+  } else {
+    // lamdera, add codecs dependency
+    elmJson["dependencies"]["direct"]["lamdera/codecs"] = "1.0.0";
+  }
   // 3. add our own secret My.elm module 😈
   elmJson["source-directories"].push(".elm-pages");
   return elmJson;