elm-pages 3.0.0-beta.17 → 3.0.0-beta.19

package/generator/src/render.js

@@ -1,25 +1,21 @@
 // @ts-check
 
-const path = require("path");
-const mm = require("micromatch");
-const matter = require("gray-matter");
-const globby = require("globby");
-const fsPromises = require("fs").promises;
-const preRenderHtml = require("./pre-render-html.js");
-const { lookupOrPerform } = require("./request-cache.js");
-const kleur = require("kleur");
-const cookie = require("cookie-signature");
-const { compatibilityKey } = require("./compatibility-key.js");
-kleur.enabled = true;
+import * as path from "path";
+import { default as mm } from "micromatch";
+import { default as matter } from "gray-matter";
+import { globby } from "globby";
+import * as fsPromises from "fs/promises";
+import * as preRenderHtml from "./pre-render-html.js";
+import { lookupOrPerform } from "./request-cache.js";
+import * as kleur from "kleur/colors";
+import * as cookie from "cookie-signature";
+import { compatibilityKey } from "./compatibility-key.js";
+import * as fs from "fs";
 
 process.on("unhandledRejection", (error) => {
   console.error(error);
 });
 let foundErrors;
-let pendingBackendTaskResponses = new Map();
-let pendingBackendTaskCount;
-
-module.exports = { render, runGenerator };
 
 /**
  *
@@ -31,7 +27,7 @@ module.exports = { render, runGenerator };
  * @param {boolean} hasFsAccess
  * @returns
  */
-async function render(
+export async function render(
   portsFile,
   basePath,
   elmModule,
@@ -41,14 +37,12 @@ async function render(
   addBackendTaskWatcher,
   hasFsAccess
 ) {
-  const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(hasFsAccess);
-  resetInMemoryFs();
+  // const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(hasFsAccess);
+  // resetInMemoryFs();
   foundErrors = false;
-  pendingBackendTaskResponses = new Map();
-  pendingBackendTaskCount = 0;
   // since init/update are never called in pre-renders, and BackendTask.Http is called using pure NodeJS HTTP fetching
   // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
-  XMLHttpRequest = {};
+  global.XMLHttpRequest = {};
   const result = await runElmApp(
     portsFile,
     basePath,
@@ -57,7 +51,6 @@ async function render(
     path,
     request,
     addBackendTaskWatcher,
-    fs,
     hasFsAccess
   );
   return result;
@@ -68,25 +61,29 @@ async function render(
  * @returns
  * @param {string[]} cliOptions
  * @param {any} portsFile
+ * @param {string} scriptModuleName
  */
-async function runGenerator(cliOptions, portsFile, elmModule) {
+export async function runGenerator(
+  cliOptions,
+  portsFile,
+  elmModule,
+  scriptModuleName
+) {
   global.isRunningGenerator = true;
-  const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(true);
-  resetInMemoryFs();
+  // const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(true);
+  // resetInMemoryFs();
   foundErrors = false;
-  pendingBackendTaskResponses = new Map();
-  pendingBackendTaskCount = 0;
   // since init/update are never called in pre-renders, and BackendTask.Http is called using pure NodeJS HTTP fetching
   // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
-  XMLHttpRequest = {};
+  global.XMLHttpRequest = {};
   const result = await runGeneratorAppHelp(
     cliOptions,
     portsFile,
     "",
     elmModule,
+    scriptModuleName,
     "production",
     "",
-    fs,
     true
   );
   return result;
@@ -101,15 +98,16 @@ async function runGenerator(cliOptions, portsFile, elmModule) {
  * @param {any} portsFile
  * @param {typeof import("fs") | import("memfs").IFs} fs
  * @param {boolean} hasFsAccess
+ * @param {string} scriptModuleName
  */
 function runGeneratorAppHelp(
   cliOptions,
   portsFile,
   basePath,
   elmModule,
+  scriptModuleName,
   mode,
   pagePath,
-  fs,
   hasFsAccess
 ) {
   const isDevServer = mode !== "build";
@@ -122,7 +120,7 @@ function runGeneratorAppHelp(
     app = elmModule.Elm.Main.init({
       flags: {
         compatibilityKey,
-        argv: ["", "", ...cliOptions],
+        argv: ["", `elm-pages run ${scriptModuleName}`, ...cliOptions],
         versionMessage: "1.2.3",
       },
     });
@@ -166,33 +164,37 @@ function runGeneratorAppHelp(
           );
         }
       } else if (fromElm.tag === "DoHttp") {
-        const requestHash = fromElm.args[0];
-        const requestToPerform = fromElm.args[1];
-        if (
-          requestToPerform.url !== "elm-pages-internal://port" &&
-          requestToPerform.url.startsWith("elm-pages-internal://")
-        ) {
-          runInternalJob(
-            requestHash,
-            app,
-            mode,
-            requestToPerform,
-            fs,
-            hasFsAccess,
-            patternsToWatch
-          );
-        } else {
-          runHttpJob(
-            requestHash,
-            portsFile,
-            app,
-            mode,
-            requestToPerform,
-            fs,
-            hasFsAccess,
-            fromElm.args[1]
-          );
-        }
+        app.ports.gotBatchSub.send(
+          Object.fromEntries(
+            await Promise.all(
+              fromElm.args[0].map(([requestHash, requestToPerform]) => {
+                if (
+                  requestToPerform.url !== "elm-pages-internal://port" &&
+                  requestToPerform.url.startsWith("elm-pages-internal://")
+                ) {
+                  return runInternalJob(
+                    requestHash,
+                    app,
+                    mode,
+                    requestToPerform,
+                    hasFsAccess,
+                    patternsToWatch
+                  );
+                } else {
+                  return runHttpJob(
+                    requestHash,
+                    portsFile,
+                    app,
+                    mode,
+                    requestToPerform,
+                    hasFsAccess,
+                    requestToPerform
+                  );
+                }
+              })
+            )
+          )
+        );
       } else if (fromElm.tag === "Errors") {
         foundErrors = true;
         reject(fromElm.args[0].errorsJson);
@@ -227,7 +229,6 @@ function runElmApp(
   pagePath,
   request,
   addBackendTaskWatcher,
-  fs,
   hasFsAccess
 ) {
   const isDevServer = mode !== "build";
@@ -241,7 +242,6 @@ function runElmApp(
       .replace(/content\.dat\/?$/, "");
 
     const modifiedRequest = { ...request, path: route };
-    // console.log("StaticHttp cache keys", Object.keys(global.staticHttpCache));
     app = elmModule.Elm.Main.init({
       flags: {
         mode,
@@ -275,9 +275,6 @@ function runElmApp(
         console.log(fromElm.value);
       } else if (fromElm.tag === "ApiResponse") {
         const args = fromElm.args[0];
-        if (mode === "build") {
-          global.staticHttpCache = args.staticHttpCache;
-        }
 
         resolve({
           kind: "api-response",
@@ -287,10 +284,6 @@ function runElmApp(
         });
       } else if (fromElm.tag === "PageProgress") {
         const args = fromElm.args[0];
-        if (mode === "build") {
-          global.staticHttpCache = args.staticHttpCache;
-        }
-
         if (isBytes) {
           resolve({
             kind: "bytes",
@@ -309,33 +302,37 @@ function runElmApp(
           );
         }
       } else if (fromElm.tag === "DoHttp") {
-        const requestHash = fromElm.args[0];
-        const requestToPerform = fromElm.args[1];
-        if (
-          requestToPerform.url !== "elm-pages-internal://port" &&
-          requestToPerform.url.startsWith("elm-pages-internal://")
-        ) {
-          runInternalJob(
-            requestHash,
-            app,
-            mode,
-            requestToPerform,
-            fs,
-            hasFsAccess,
-            patternsToWatch
-          );
-        } else {
-          runHttpJob(
-            requestHash,
-            portsFile,
-            app,
-            mode,
-            requestToPerform,
-            fs,
-            hasFsAccess,
-            fromElm.args[1]
-          );
-        }
+        app.ports.gotBatchSub.send(
+          Object.fromEntries(
+            await Promise.all(
+              fromElm.args[0].map(([requestHash, requestToPerform]) => {
+                if (
+                  requestToPerform.url !== "elm-pages-internal://port" &&
+                  requestToPerform.url.startsWith("elm-pages-internal://")
+                ) {
+                  return runInternalJob(
+                    requestHash,
+                    app,
+                    mode,
+                    requestToPerform,
+                    hasFsAccess,
+                    patternsToWatch
+                  );
+                } else {
+                  return runHttpJob(
+                    requestHash,
+                    portsFile,
+                    app,
+                    mode,
+                    requestToPerform,
+                    hasFsAccess,
+                    requestToPerform
+                  );
+                }
+              })
+            )
+          )
+        );
       } else if (fromElm.tag === "Errors") {
         foundErrors = true;
         reject(fromElm.args[0].errorsJson);
@@ -400,11 +397,9 @@ async function runHttpJob(
   app,
   mode,
   requestToPerform,
-  fs,
   hasFsAccess,
   useCache
 ) {
-  pendingBackendTaskCount += 1;
   try {
     const lookupResponse = await lookupOrPerform(
       portsFile,
@@ -416,25 +411,29 @@ async function runHttpJob(
 
     if (lookupResponse.kind === "cache-response-path") {
       const responseFilePath = lookupResponse.value;
-      pendingBackendTaskResponses.set(requestHash, {
-        request: requestToPerform,
-        response: JSON.parse(
-          (await fs.promises.readFile(responseFilePath, "utf8")).toString()
-        ),
-      });
+      return [
+        requestHash,
+        {
+          request: requestToPerform,
+          response: JSON.parse(
+            (await fs.promises.readFile(responseFilePath, "utf8")).toString()
+          ),
+        },
+      ];
     } else if (lookupResponse.kind === "response-json") {
-      pendingBackendTaskResponses.set(requestHash, {
-        request: requestToPerform,
-        response: lookupResponse.value,
-      });
+      return [
+        requestHash,
+        {
+          request: requestToPerform,
+          response: lookupResponse.value,
+        },
+      ];
     } else {
       throw `Unexpected kind ${lookupResponse}`;
     }
   } catch (error) {
-    sendError(app, error);
-  } finally {
-    pendingBackendTaskCount -= 1;
-    flushIfDone(app);
+    console.log("@@@ERROR", error);
+    // sendError(app, error);
   }
 }
 
@@ -456,48 +455,33 @@ async function runInternalJob(
   app,
   mode,
   requestToPerform,
-  fs,
   hasFsAccess,
   patternsToWatch
 ) {
   try {
-    pendingBackendTaskCount += 1;
-
     if (requestToPerform.url === "elm-pages-internal://log") {
-      pendingBackendTaskResponses.set(
-        requestHash,
-        await runLogJob(requestToPerform)
-      );
+      return [requestHash, await runLogJob(requestToPerform)];
     } else if (requestToPerform.url === "elm-pages-internal://read-file") {
-      pendingBackendTaskResponses.set(
+      return [
         requestHash,
-        await readFileJobNew(requestToPerform, patternsToWatch)
-      );
+        await readFileJobNew(requestToPerform, patternsToWatch),
+      ];
     } else if (requestToPerform.url === "elm-pages-internal://glob") {
-      pendingBackendTaskResponses.set(
-        requestHash,
-        await runGlobNew(requestToPerform, patternsToWatch)
-      );
+      return [requestHash, await runGlobNew(requestToPerform, patternsToWatch)];
     } else if (requestToPerform.url === "elm-pages-internal://env") {
-      pendingBackendTaskResponses.set(
-        requestHash,
-        await runEnvJob(requestToPerform, patternsToWatch)
-      );
+      return [requestHash, await runEnvJob(requestToPerform, patternsToWatch)];
     } else if (requestToPerform.url === "elm-pages-internal://encrypt") {
-      pendingBackendTaskResponses.set(
+      return [
         requestHash,
-        await runEncryptJob(requestToPerform, patternsToWatch)
-      );
+        await runEncryptJob(requestToPerform, patternsToWatch),
+      ];
     } else if (requestToPerform.url === "elm-pages-internal://decrypt") {
-      pendingBackendTaskResponses.set(
+      return [
         requestHash,
-        await runDecryptJob(requestToPerform, patternsToWatch)
-      );
+        await runDecryptJob(requestToPerform, patternsToWatch),
+      ];
     } else if (requestToPerform.url === "elm-pages-internal://write-file") {
-      pendingBackendTaskResponses.set(
-        requestHash,
-        await runWriteFileJob(requestToPerform)
-      );
+      return [requestHash, await runWriteFileJob(requestToPerform)];
     } else {
       throw `Unexpected internal BackendTask request format: ${kleur.yellow(
         JSON.stringify(2, null, requestToPerform)
@@ -505,9 +489,6 @@ async function runInternalJob(
     }
   } catch (error) {
     sendError(app, error);
-  } finally {
-    pendingBackendTaskCount -= 1;
-    flushIfDone(app);
   }
 }
 
@@ -628,27 +609,6 @@ async function runDecryptJob(req, patternsToWatch) {
   }
 }
 
-function flushIfDone(app) {
-  if (foundErrors) {
-    pendingBackendTaskResponses = new Map();
-  } else if (pendingBackendTaskCount === 0) {
-    // console.log(
-    //   `Flushing ${pendingBackendTaskResponses.length} items in ${timeUntilThreshold}ms`
-    // );
-
-    flushQueue(app);
-  }
-}
-
-function flushQueue(app) {
-  // TODO - could the case where flush is called with size 0 be avoided on the Elm side?
-  // if (pendingBackendTaskResponses.size > 0) {
-  // console.log("@@@ FLUSHING", pendingBackendTaskResponses.size);
-  app.ports.gotBatchSub.send(Object.fromEntries(pendingBackendTaskResponses));
-  pendingBackendTaskResponses = new Map();
-  // }
-}
-
 /**
  * @param {{ ports: { fromJsPort: { send: (arg0: { tag: string; data: any; }) => void; }; }; }} app
  * @param {{ message: string; title: string; }} error

package/generator/src/request-cache.js

@@ -1,6 +1,7 @@
-const path = require("path");
-const kleur = require("kleur");
-const fsPromises = require("fs/promises");
+import * as path from "path";
+import * as fsPromises from "fs/promises";
+import * as kleur from "kleur/colors";
+import { default as makeFetchHappenOriginal } from "make-fetch-happen";
 
 const defaultHttpCachePath = "./.elm-pages/http-cache";
 
@@ -13,8 +14,14 @@ const defaultHttpCachePath = "./.elm-pages/http-cache";
  * @param {boolean} hasFsAccess
  * @returns {Promise<Response>}
  */
-function lookupOrPerform(portsFile, mode, rawRequest, hasFsAccess, useCache) {
-  const makeFetchHappen = require("make-fetch-happen").defaults({
+export function lookupOrPerform(
+  portsFile,
+  mode,
+  rawRequest,
+  hasFsAccess,
+  useCache
+) {
+  const makeFetchHappen = makeFetchHappenOriginal.defaults({
     cache: mode === "build" ? "no-cache" : "default",
   });
   return new Promise(async (resolve, reject) => {
@@ -29,7 +36,7 @@ function lookupOrPerform(portsFile, mode, rawRequest, hasFsAccess, useCache) {
       const portBackendTaskPath = path.resolve(portsFile);
       // On Windows, we need cannot use paths directly and instead must use a file:// URL.
       // portBackendTask = await require(url.pathToFileURL(portBackendTaskPath).href);
-      portBackendTask = require(portBackendTaskPath);
+      portBackendTask = await import(portBackendTaskPath);
     } catch (e) {
       portBackendTaskImportError = e;
     }
@@ -270,5 +277,3 @@ async function canAccess(filePath) {
     return false;
   }
 }
-
-module.exports = { lookupOrPerform };

package/generator/src/rewrite-client-elm-json.js

@@ -1,6 +1,6 @@
-const fs = require("fs");
+import * as fs from "fs";
 
-module.exports = async function () {
+export async function rewriteClientElmJson() {
   var elmJson = JSON.parse(
     (await fs.promises.readFile("./elm.json")).toString()
   );
@@ -9,11 +9,11 @@ module.exports = async function () {
 
   await writeFileIfChanged(
     "./elm-stuff/elm-pages/client/elm.json",
-    JSON.stringify(rewriteElmJson(elmJson))
+    JSON.stringify(rewriteClientElmJsonHelp(elmJson))
   );
-};
+}
 
-function rewriteElmJson(elmJson) {
+function rewriteClientElmJsonHelp(elmJson) {
   // The internal generated file will be at:
   // ./elm-stuff/elm-pages/
   // So, we need to take the existing elmJson and

package/generator/src/rewrite-elm-json.js

@@ -1,6 +1,6 @@
-const fs = require("fs");
+import * as fs from "fs";
 
-module.exports = async function (sourceElmJsonPath, targetElmJsonPath) {
+export async function rewriteElmJson(sourceElmJsonPath, targetElmJsonPath) {
   var elmJson = JSON.parse(
     (await fs.promises.readFile(sourceElmJsonPath)).toString()
   );
@@ -9,11 +9,11 @@ module.exports = async function (sourceElmJsonPath, targetElmJsonPath) {
 
   await writeFileIfChanged(
     targetElmJsonPath,
-    JSON.stringify(rewriteElmJson(elmJson))
+    JSON.stringify(rewriteElmJsonHelp(elmJson))
   );
-};
+}
 
-function rewriteElmJson(elmJson) {
+function rewriteElmJsonHelp(elmJson) {
   // The internal generated file will be at:
   // ./elm-stuff/elm-pages/
   // So, we need to take the existing elmJson and