elm-pages 3.0.0-beta.3 → 3.0.0-beta.30

package/generator/src/render.js

@@ -1,67 +1,222 @@
 // @ts-check
 
-const path = require("path");
-const mm = require("micromatch");
-const matter = require("gray-matter");
-const globby = require("globby");
-const fsPromises = require("fs").promises;
-const preRenderHtml = require("./pre-render-html.js");
-const { lookupOrPerform } = require("./request-cache.js");
-const kleur = require("kleur");
-const cookie = require("cookie-signature");
-kleur.enabled = true;
+import * as path from "node:path";
+import { default as mm } from "micromatch";
+import { default as matter } from "gray-matter";
+import * as globby from "globby";
+import * as fsPromises from "node:fs/promises";
+import * as preRenderHtml from "./pre-render-html.js";
+import { lookupOrPerform } from "./request-cache.js";
+import * as kleur from "kleur/colors";
+import * as cookie from "cookie-signature";
+import { compatibilityKey } from "./compatibility-key.js";
+import * as fs from "node:fs";
+import * as crypto from "node:crypto";
+import { restoreColorSafe } from "./error-formatter.js";
 
 process.on("unhandledRejection", (error) => {
   console.error(error);
 });
 let foundErrors;
-let pendingDataSourceResponses;
-let pendingDataSourceCount;
 
-module.exports =
-  /**
-   *
-   * @param {string} basePath
-   * @param {Object} elmModule
-   * @param {string} path
-   * @param {{ method: string; hostname: string; query: Record<string, string | undefined>; headers: Record<string, string>; host: string; pathname: string; port: number | null; protocol: string; rawUrl: string; }} request
-   * @param {(pattern: string) => void} addDataSourceWatcher
-   * @param {boolean} hasFsAccess
-   * @returns
-   */
-  async function run(
+/**
+ *
+ * @param {string} basePath
+ * @param {Object} elmModule
+ * @param {string} path
+ * @param {{ method: string; hostname: string; query: Record<string, string | undefined>; headers: Record<string, string>; host: string; pathname: string; port: number | null; protocol: string; rawUrl: string; }} request
+ * @param {(pattern: string) => void} addBackendTaskWatcher
+ * @param {boolean} hasFsAccess
+ * @returns
+ */
+export async function render(
+  portsFile,
+  basePath,
+  elmModule,
+  mode,
+  path,
+  request,
+  addBackendTaskWatcher,
+  hasFsAccess
+) {
+  // const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(hasFsAccess);
+  // resetInMemoryFs();
+  foundErrors = false;
+  // since init/update are never called in pre-renders, and BackendTask.Http is called using pure NodeJS HTTP fetching
+  // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
+  global.XMLHttpRequest = {};
+  const result = await runElmApp(
     portsFile,
     basePath,
     elmModule,
     mode,
     path,
     request,
-    addDataSourceWatcher,
+    addBackendTaskWatcher,
     hasFsAccess
-  ) {
-    const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(
-      hasFsAccess
-    );
-    resetInMemoryFs();
-    foundErrors = false;
-    pendingDataSourceResponses = [];
-    pendingDataSourceCount = 0;
-    // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
-    // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
-    XMLHttpRequest = {};
-    const result = await runElmApp(
+  );
+  return result;
+}
+
+/**
+ * @param {Object} elmModule
+ * @returns
+ * @param {string[]} cliOptions
+ * @param {any} portsFile
+ * @param {string} scriptModuleName
+ */
+export async function runGenerator(
+  cliOptions,
+  portsFile,
+  elmModule,
+  scriptModuleName
+) {
+  global.isRunningGenerator = true;
+  // const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(true);
+  // resetInMemoryFs();
+  foundErrors = false;
+  // since init/update are never called in pre-renders, and BackendTask.Http is called using pure NodeJS HTTP fetching
+  // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
+  global.XMLHttpRequest = {};
+  try {
+    const result = await runGeneratorAppHelp(
+      cliOptions,
       portsFile,
-      basePath,
+      "",
       elmModule,
-      mode,
-      path,
-      request,
-      addDataSourceWatcher,
-      fs,
-      hasFsAccess
+      scriptModuleName,
+      "production",
+      "",
+      true
     );
     return result;
-  };
+  } catch (error) {
+    console.log(restoreColorSafe(error));
+  }
+}
+/**
+ * @param {string} basePath
+ * @param {Object} elmModule
+ * @param {string} pagePath
+ * @param {string} mode
+ * @returns {Promise<({is404: boolean;} & ({kind: 'json';contentJson: string;} | {kind: 'html';htmlString: string;} | {kind: 'api-response';body: string;}))>}
+ * @param {string[]} cliOptions
+ * @param {any} portsFile
+ * @param {typeof import("fs") | import("memfs").IFs} fs
+ * @param {boolean} hasFsAccess
+ * @param {string} scriptModuleName
+ */
+function runGeneratorAppHelp(
+  cliOptions,
+  portsFile,
+  basePath,
+  elmModule,
+  scriptModuleName,
+  mode,
+  pagePath,
+  hasFsAccess
+) {
+  const isDevServer = mode !== "build";
+  let patternsToWatch = new Set();
+  let app = null;
+  let killApp;
+  return new Promise((resolve, reject) => {
+    const isBytes = pagePath.match(/content\.dat\/?$/);
+
+    app = elmModule.Elm.Main.init({
+      flags: {
+        compatibilityKey,
+        argv: ["", `elm-pages run ${scriptModuleName}`, ...cliOptions],
+        versionMessage: "1.2.3",
+      },
+    });
+
+    killApp = () => {
+      app.ports.toJsPort.unsubscribe(portHandler);
+      app.ports.sendPageData.unsubscribe(portHandler);
+      app.die();
+      app = null;
+      // delete require.cache[require.resolve(compiledElmPath)];
+    };
+
+    async function portHandler(/** @type { FromElm }  */ newThing) {
+      let fromElm;
+      let contentDatPayload;
+
+      fromElm = newThing;
+      if (fromElm.command === "log") {
+        console.log(fromElm.value);
+      } else if (fromElm.tag === "ApiResponse") {
+        // Finished successfully
+        process.exit(0);
+      } else if (fromElm.tag === "PageProgress") {
+        const args = fromElm.args[0];
+
+        if (isBytes) {
+          resolve({
+            kind: "bytes",
+            is404: false,
+            contentJson: JSON.stringify({
+              staticData: args.contentJson,
+              is404: false,
+            }),
+            statusCode: args.statusCode,
+            headers: args.headers,
+            contentDatPayload,
+          });
+        } else {
+          resolve(
+            outputString(basePath, fromElm, isDevServer, contentDatPayload)
+          );
+        }
+      } else if (fromElm.tag === "DoHttp") {
+        app.ports.gotBatchSub.send(
+          Object.fromEntries(
+            await Promise.all(
+              fromElm.args[0].map(([requestHash, requestToPerform]) => {
+                if (
+                  requestToPerform.url !== "elm-pages-internal://port" &&
+                  requestToPerform.url.startsWith("elm-pages-internal://")
+                ) {
+                  return runInternalJob(
+                    requestHash,
+                    app,
+                    mode,
+                    requestToPerform,
+                    hasFsAccess,
+                    patternsToWatch
+                  );
+                } else {
+                  return runHttpJob(
+                    requestHash,
+                    portsFile,
+                    app,
+                    mode,
+                    requestToPerform,
+                    hasFsAccess,
+                    requestToPerform
+                  );
+                }
+              })
+            )
+          )
+        );
+      } else if (fromElm.tag === "Errors") {
+        foundErrors = true;
+        reject(fromElm.args[0].errorsJson);
+      } else {
+        console.log(fromElm);
+      }
+    }
+    app.ports.toJsPort.subscribe(portHandler);
+    app.ports.sendPageData.subscribe(portHandler);
+  }).finally(() => {
+    try {
+      killApp();
+      killApp = null;
+    } catch (error) {}
+  });
+}
 
 /**
  * @param {string} basePath
@@ -69,7 +224,7 @@ module.exports =
  * @param {string} pagePath
  * @param {string} mode
  * @param {{ method: string; hostname: string; query: string; headers: Object; host: string; pathname: string; port: string; protocol: string; rawUrl: string; }} request
- * @param {(pattern: string) => void} addDataSourceWatcher
+ * @param {(pattern: string) => void} addBackendTaskWatcher
  * @returns {Promise<({is404: boolean} & ( { kind: 'json'; contentJson: string} | { kind: 'html'; htmlString: string } | { kind: 'api-response'; body: string; }) )>}
  */
 function runElmApp(
@@ -79,8 +234,7 @@ function runElmApp(
   mode,
   pagePath,
   request,
-  addDataSourceWatcher,
-  fs,
+  addBackendTaskWatcher,
   hasFsAccess
 ) {
   const isDevServer = mode !== "build";
@@ -94,10 +248,10 @@ function runElmApp(
       .replace(/content\.dat\/?$/, "");
 
     const modifiedRequest = { ...request, path: route };
-    // console.log("StaticHttp cache keys", Object.keys(global.staticHttpCache));
     app = elmModule.Elm.Main.init({
       flags: {
         mode,
+        compatibilityKey,
         request: {
           payload: modifiedRequest,
           kind: "single-page",
@@ -127,9 +281,6 @@ function runElmApp(
         console.log(fromElm.value);
       } else if (fromElm.tag === "ApiResponse") {
         const args = fromElm.args[0];
-        if (mode === "build") {
-          global.staticHttpCache = args.staticHttpCache;
-        }
 
         resolve({
           kind: "api-response",
@@ -139,10 +290,6 @@ function runElmApp(
         });
       } else if (fromElm.tag === "PageProgress") {
         const args = fromElm.args[0];
-        if (mode === "build") {
-          global.staticHttpCache = args.staticHttpCache;
-        }
-
         if (isBytes) {
           resolve({
             kind: "bytes",
@@ -161,30 +308,37 @@ function runElmApp(
           );
         }
       } else if (fromElm.tag === "DoHttp") {
-        const requestToPerform = fromElm.args[0];
-        if (
-          requestToPerform.url !== "elm-pages-internal://port" &&
-          requestToPerform.url.startsWith("elm-pages-internal://")
-        ) {
-          runInternalJob(
-            app,
-            mode,
-            requestToPerform,
-            fs,
-            hasFsAccess,
-            patternsToWatch
-          );
-        } else {
-          runHttpJob(
-            portsFile,
-            app,
-            mode,
-            requestToPerform,
-            fs,
-            hasFsAccess,
-            fromElm.args[1]
-          );
-        }
+        app.ports.gotBatchSub.send(
+          Object.fromEntries(
+            await Promise.all(
+              fromElm.args[0].map(([requestHash, requestToPerform]) => {
+                if (
+                  requestToPerform.url !== "elm-pages-internal://port" &&
+                  requestToPerform.url.startsWith("elm-pages-internal://")
+                ) {
+                  return runInternalJob(
+                    requestHash,
+                    app,
+                    mode,
+                    requestToPerform,
+                    hasFsAccess,
+                    patternsToWatch
+                  );
+                } else {
+                  return runHttpJob(
+                    requestHash,
+                    portsFile,
+                    app,
+                    mode,
+                    requestToPerform,
+                    hasFsAccess,
+                    requestToPerform
+                  );
+                }
+              })
+            )
+          )
+        );
       } else if (fromElm.tag === "Errors") {
         foundErrors = true;
         reject(fromElm.args[0].errorsJson);
@@ -195,7 +349,7 @@ function runElmApp(
     app.ports.toJsPort.subscribe(portHandler);
     app.ports.sendPageData.subscribe(portHandler);
   }).finally(() => {
-    addDataSourceWatcher(patternsToWatch);
+    addBackendTaskWatcher(patternsToWatch);
     try {
       killApp();
       killApp = null;
@@ -244,17 +398,16 @@ async function outputString(
 /** @typedef { { head: any[]; errors: any[]; contentJson: any[]; html: string; route: string; title: string; } } Arg */
 
 async function runHttpJob(
+  requestHash,
   portsFile,
   app,
   mode,
   requestToPerform,
-  fs,
   hasFsAccess,
   useCache
 ) {
-  pendingDataSourceCount += 1;
   try {
-    const responseFilePath = await lookupOrPerform(
+    const lookupResponse = await lookupOrPerform(
       portsFile,
       mode,
       requestToPerform,
@@ -262,17 +415,31 @@ async function runHttpJob(
       useCache
     );
 
-    pendingDataSourceResponses.push({
-      request: requestToPerform,
-      response: JSON.parse(
-        (await fs.promises.readFile(responseFilePath, "utf8")).toString()
-      ),
-    });
+    if (lookupResponse.kind === "cache-response-path") {
+      const responseFilePath = lookupResponse.value;
+      return [
+        requestHash,
+        {
+          request: requestToPerform,
+          response: JSON.parse(
+            (await fs.promises.readFile(responseFilePath, "utf8")).toString()
+          ),
+        },
+      ];
+    } else if (lookupResponse.kind === "response-json") {
+      return [
+        requestHash,
+        {
+          request: requestToPerform,
+          response: lookupResponse.value,
+        },
+      ];
+    } else {
+      throw `Unexpected kind ${lookupResponse}`;
+    }
   } catch (error) {
-    sendError(app, error);
-  } finally {
-    pendingDataSourceCount -= 1;
-    flushIfDone(app);
+    console.log("@@@ERROR", error);
+    // sendError(app, error);
   }
 }
 
@@ -290,46 +457,54 @@ function jsonResponse(request, json) {
 }
 
 async function runInternalJob(
+  requestHash,
   app,
   mode,
   requestToPerform,
-  fs,
   hasFsAccess,
   patternsToWatch
 ) {
   try {
-    pendingDataSourceCount += 1;
-
-    if (requestToPerform.url === "elm-pages-internal://read-file") {
-      pendingDataSourceResponses.push(
-        await readFileJobNew(requestToPerform, patternsToWatch)
-      );
+    if (requestToPerform.url === "elm-pages-internal://log") {
+      return [requestHash, await runLogJob(requestToPerform)];
+    } else if (requestToPerform.url === "elm-pages-internal://read-file") {
+      return [
+        requestHash,
+        await readFileJobNew(requestToPerform, patternsToWatch),
+      ];
     } else if (requestToPerform.url === "elm-pages-internal://glob") {
-      pendingDataSourceResponses.push(
-        await runGlobNew(requestToPerform, patternsToWatch)
-      );
+      return [requestHash, await runGlobNew(requestToPerform, patternsToWatch)];
+    } else if (requestToPerform.url === "elm-pages-internal://randomSeed") {
+      return [
+        requestHash,
+        jsonResponse(
+          requestToPerform,
+          crypto.getRandomValues(new Uint32Array(1))[0]
+        ),
+      ];
+    } else if (requestToPerform.url === "elm-pages-internal://now") {
+      return [requestHash, jsonResponse(requestToPerform, Date.now())];
     } else if (requestToPerform.url === "elm-pages-internal://env") {
-      pendingDataSourceResponses.push(
-        await runEnvJob(requestToPerform, patternsToWatch)
-      );
+      return [requestHash, await runEnvJob(requestToPerform, patternsToWatch)];
     } else if (requestToPerform.url === "elm-pages-internal://encrypt") {
-      pendingDataSourceResponses.push(
-        await runEncryptJob(requestToPerform, patternsToWatch)
-      );
+      return [
+        requestHash,
+        await runEncryptJob(requestToPerform, patternsToWatch),
+      ];
     } else if (requestToPerform.url === "elm-pages-internal://decrypt") {
-      pendingDataSourceResponses.push(
-        await runDecryptJob(requestToPerform, patternsToWatch)
-      );
+      return [
+        requestHash,
+        await runDecryptJob(requestToPerform, patternsToWatch),
+      ];
+    } else if (requestToPerform.url === "elm-pages-internal://write-file") {
+      return [requestHash, await runWriteFileJob(requestToPerform)];
     } else {
-      throw `Unexpected internal DataSource request format: ${kleur.yellow(
+      throw `Unexpected internal BackendTask request format: ${kleur.yellow(
         JSON.stringify(2, null, requestToPerform)
       )}`;
     }
   } catch (error) {
     sendError(app, error);
-  } finally {
-    pendingDataSourceCount -= 1;
-    flushIfDone(app);
   }
 }
 
@@ -344,6 +519,7 @@ async function readFileJobNew(req, patternsToWatch) {
           path.join(process.env.LAMBDA_TASK_ROOT || process.cwd(), filePath)
         )
       ).toString();
+    // TODO does this throw an error if there is invalid frontmatter?
     const parsedFile = matter(fileContents);
 
     return jsonResponse(req, {
@@ -352,10 +528,24 @@ async function readFileJobNew(req, patternsToWatch) {
       rawFile: fileContents,
     });
   } catch (error) {
+    return jsonResponse(req, {
+      errorCode: error.code,
+    });
+  }
+}
+async function runWriteFileJob(req) {
+  const data = req.body.args[0];
+  try {
+    const fullPathToWrite = path.join(process.cwd(), data.path);
+    await fsPromises.mkdir(path.dirname(fullPathToWrite), { recursive: true });
+    await fsPromises.writeFile(fullPathToWrite, data.body);
+    return jsonResponse(req, null);
+  } catch (error) {
+    console.trace(error);
     throw {
-      title: "DataSource.File Error",
-      message: `A DataSource.File read failed because I couldn't find this file: ${kleur.yellow(
-        filePath
+      title: "BackendTask Error",
+      message: `BackendTask.Generator.writeFile failed for file path: ${kleur.yellow(
+        data.path
       )}\n${kleur.red(error.toString())}`,
     };
   }
@@ -364,7 +554,7 @@ async function readFileJobNew(req, patternsToWatch) {
 async function runGlobNew(req, patternsToWatch) {
   try {
     const { pattern, options } = req.body.args[0];
-    const matchedPaths = await globby(pattern, options);
+    const matchedPaths = await globby.globby(pattern, options);
     patternsToWatch.add(pattern);
 
     return jsonResponse(
@@ -382,6 +572,15 @@ async function runGlobNew(req, patternsToWatch) {
   }
 }
 
+async function runLogJob(req) {
+  try {
+    console.log(req.body.args[0].message);
+    return jsonResponse(req, null);
+  } catch (e) {
+    console.log(`Error performing env '${JSON.stringify(req.body)}'`);
+    throw e;
+  }
+}
 async function runEnvJob(req, patternsToWatch) {
   try {
     const expectedEnv = req.body.args[0];
@@ -402,7 +601,7 @@ async function runEncryptJob(req, patternsToWatch) {
     );
   } catch (e) {
     throw {
-      title: "DataSource Encrypt Error",
+      title: "BackendTask Encrypt Error",
       message:
         e.toString() + e.stack + "\n\n" + JSON.stringify(rawRequest, null, 2),
     };
@@ -419,32 +618,13 @@ async function runDecryptJob(req, patternsToWatch) {
     return jsonResponse(req, JSON.parse(signed || "null"));
   } catch (e) {
     throw {
-      title: "DataSource Decrypt Error",
+      title: "BackendTask Decrypt Error",
       message:
         e.toString() + e.stack + "\n\n" + JSON.stringify(rawRequest, null, 2),
     };
   }
 }
 
-function flushIfDone(app) {
-  if (foundErrors) {
-    pendingDataSourceResponses = [];
-  } else if (pendingDataSourceCount === 0) {
-    // console.log(
-    //   `Flushing ${pendingDataSourceResponses.length} items in ${timeUntilThreshold}ms`
-    // );
-
-    flushQueue(app);
-  }
-}
-
-function flushQueue(app) {
-  const temp = pendingDataSourceResponses;
-  pendingDataSourceResponses = [];
-  // console.log("@@@ FLUSHING", temp.length);
-  app.ports.gotBatchSub.send(temp);
-}
-
 /**
  * @param {{ ports: { fromJsPort: { send: (arg0: { tag: string; data: any; }) => void; }; }; }} app
  * @param {{ message: string; title: string; }} error