elm-pages 3.0.0-beta.12 → 3.0.0-beta.14

package/generator/src/cli.js

@@ -111,6 +111,9 @@ async function main() {
       }
       try {
         // await codegen.generate("");
+        ensureDirSync(
+          path.join(process.cwd(), ".elm-pages", "http-response-cache")
+        );
         if (fs.existsSync("./codegen/")) {
           await runElmCodegenInstall();
         }
@@ -126,7 +129,7 @@ async function main() {
           "./script/elm-stuff/elm-pages/elm.json"
         );
 
-        const portDataSourceCompiled = esbuild
+        const portBackendTaskCompiled = esbuild
           .build({
             entryPoints: ["./port-data-source"],
             platform: "node",
@@ -147,16 +150,17 @@ async function main() {
             }
           })
           .catch((error) => {
-            const portDataSourceFileFound =
+            const portBackendTaskFileFound =
               globby.sync("./port-data-source.*").length > 0;
-            if (portDataSourceFileFound) {
+            if (portBackendTaskFileFound) {
               // don't present error if there are no files matching port-data-source
               // if there are files matching port-data-source, warn the user in case something went wrong loading it
               console.error("Failed to start port-data-source watcher", error);
             }
           });
-        const portsPath = await portDataSourceCompiled;
-        const resolvedPortsPath = path.join(process.cwd(), portsPath);
+        const portsPath = await portBackendTaskCompiled;
+        const resolvedPortsPath =
+          portsPath && path.join(process.cwd(), portsPath);
 
         process.chdir("./script");
         // TODO have option for compiling with --debug or not (maybe allow running with elm-optimize-level-2 as well?)
@@ -265,7 +269,8 @@ function generatorWrapperFile(moduleName) {
   return `port module Main exposing (main)
 
 import Bytes
-import DataSource exposing (DataSource)
+import BackendTask exposing (BackendTask)
+import Exception
 import Cli.Program as Program
 import Json.Decode as Decode
 import Json.Encode as Encode
@@ -273,7 +278,7 @@ import Pages.Internal.Platform.GeneratorApplication
 import ${moduleName}
 
 
-main : Program.StatefulProgram Pages.Internal.Platform.GeneratorApplication.Model Pages.Internal.Platform.GeneratorApplication.Msg (DataSource ()) Pages.Internal.Platform.GeneratorApplication.Flags
+main : Program.StatefulProgram Pages.Internal.Platform.GeneratorApplication.Model Pages.Internal.Platform.GeneratorApplication.Msg (BackendTask Exception.Throwable ()) Pages.Internal.Platform.GeneratorApplication.Flags
 main =
     Pages.Internal.Platform.GeneratorApplication.app
         { data = ${moduleName}.run

package/generator/src/compatibility-key.js

@@ -1 +1 @@
-module.exports = { compatibilityKey: 2 };
+module.exports = { compatibilityKey: 4 };

package/generator/src/dev-server.js

@@ -174,9 +174,9 @@ async function start(options) {
       console.log("Watching port-data-source...");
     })
     .catch((error) => {
-      const portDataSourceFileFound =
+      const portBackendTaskFileFound =
         globby.sync("./port-data-source.*").length > 0;
-      if (portDataSourceFileFound) {
+      if (portBackendTaskFileFound) {
         // don't present error if there are no files matching port-data-source
         // if there are files matching port-data-source, warn the user in case something went wrong loading it
         console.error("Failed to start port-data-source watcher", error);
@@ -282,18 +282,18 @@ async function start(options) {
       // TODO use similar logic in the workers? Or don't use cache at all?
       // const changedPathRelative = path.relative(process.cwd(), pathThatChanged);
       //
-      // Object.keys(global.staticHttpCache).forEach((dataSourceKey) => {
-      //   if (dataSourceKey.includes(`file://${changedPathRelative}`)) {
-      //     delete global.staticHttpCache[dataSourceKey];
+      // Object.keys(global.staticHttpCache).forEach((backendTaskKey) => {
+      //   if (backendTaskKey.includes(`file://${changedPathRelative}`)) {
+      //     delete global.staticHttpCache[backendTaskKey];
       //   } else if (
       //     (eventName === "add" ||
       //       eventName === "unlink" ||
       //       eventName === "change" ||
       //       eventName === "addDir" ||
       //       eventName === "unlinkDir") &&
-      //     dataSourceKey.startsWith("glob://")
+      //     backendTaskKey.startsWith("glob://")
       //   ) {
-      //     delete global.staticHttpCache[dataSourceKey];
+      //     delete global.staticHttpCache[backendTaskKey];
       //   }
       // });
       clients.forEach((client) => {

package/generator/src/render-test.js

@@ -0,0 +1,109 @@
+const renderer = require("../../generator/src/render");
+const path = require("path");
+const compiledElmPath = path.join(process.cwd(), "elm-stuff/elm-pages/elm.js");
+const codegen = require("./codegen.js");
+const {
+  compileElmForBrowser,
+  runElmReview,
+  compileCliApp,
+} = require("./compile-elm.js");
+const { restoreColorSafe } = require("./error-formatter");
+
+async function run({ pathname, serverRequest }) {
+  try {
+    const basePath = "/";
+    await codegen.generate(basePath);
+    console.log("Compiling...");
+    await compileCliApp(
+      { port: 1234, base: basePath, https: false, debug: true },
+      ".elm-pages/Main.elm",
+
+      path.join(process.cwd(), "elm-stuff/elm-pages/", "elm.js"),
+
+      // "elm.js",
+      "elm-stuff/elm-pages/",
+      path.join("elm-stuff/elm-pages/", "elm.js")
+    );
+    console.log("Compiling DONE");
+
+    const portsFilePath =
+      ".elm-pages/compiled-ports/port-data-source-FA2IJND6.js";
+    const mode = "dev-server";
+
+    const renderResult = await renderer.render(
+      portsFilePath,
+      basePath,
+      require(compiledElmPath),
+      mode,
+      pathname,
+      serverRequest,
+      function (patterns) {},
+      true
+    );
+    console.log("renderResult", renderResult);
+  } catch (error) {
+    console.log("ERROR");
+    console.log(restoreColorSafe(error));
+  }
+}
+
+// run({
+//   serverRequest: {
+//     method: "POST",
+//     headers: {
+//       host: "localhost:1234",
+//       cookie: "darkMode=%7B%7D.X1hjuYBa1OZulUomD5yrPy6VcYeY3sC7SKZVGViT0Q4",
+//       accept: "*/*",
+//       "accept-language": "en-US,en;q=0.9",
+//       connection: "keep-alive",
+//       "content-type": "application/x-www-form-urlencoded",
+//       origin: "http://localhost:1234",
+//       referer: "http://localhost:1234/dark-mode",
+//     },
+//     rawUrl: "http://localhost:1234/dark-mode/content.dat",
+//     body: "name=1",
+//     requestTime: 1671391652138,
+//     multiPartFormData: null,
+//   },
+//   pathname: "/dark-mode/content.dat",
+// });
+
+// run({
+//   pathname: "/fetcher/content.dat",
+//   serverRequest: {
+//     method: "POST",
+//     headers: {
+//       host: "localhost:1234",
+//       cookie: "darkMode=%7B%7D",
+//       accept: "*/*",
+//       "content-type": "application/x-www-form-urlencoded",
+//       origin: "http://localhost:1234",
+//       referer: "http://localhost:1234/fetcher",
+//     },
+//     rawUrl: "http://localhost:1234/fetcher/content.dat",
+//     body: "name=1",
+//     requestTime: 1671296755228,
+//     multiPartFormData: null,
+//   },
+// });
+
+run({
+  serverRequest: {
+    method: "POST",
+    headers: {
+      host: "localhost:1234",
+      "cache-control": "max-age=0",
+      accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+      "content-type": "application/x-www-form-urlencoded",
+      referer: "http://localhost:1234/login",
+      cookie:
+        "darkMode=%7B%22darkMode%22%3A%22%22%7D.vtrl8xZXjtx9E3iidJrgurAz4Vk7rnMQmf6zKiAbwNo; mysession=%7B%7D.YX1rW5PIpRFkjgIjLn4p9iyl5r9kWLYHqQUYxAYKJgQ",
+    },
+    rawUrl: "http://localhost:1234/login/content.dat",
+    body: "name=1",
+    requestTime: 1671391652138,
+    multiPartFormData: null,
+  },
+  pathname: "/login/content.dat",
+});

package/generator/src/render.js

@@ -16,8 +16,8 @@ process.on("unhandledRejection", (error) => {
   console.error(error);
 });
 let foundErrors;
-let pendingDataSourceResponses;
-let pendingDataSourceCount;
+let pendingBackendTaskResponses = new Map();
+let pendingBackendTaskCount;
 
 module.exports = { render, runGenerator };
 
@@ -27,7 +27,7 @@ module.exports = { render, runGenerator };
  * @param {Object} elmModule
  * @param {string} path
  * @param {{ method: string; hostname: string; query: Record<string, string | undefined>; headers: Record<string, string>; host: string; pathname: string; port: number | null; protocol: string; rawUrl: string; }} request
- * @param {(pattern: string) => void} addDataSourceWatcher
+ * @param {(pattern: string) => void} addBackendTaskWatcher
  * @param {boolean} hasFsAccess
  * @returns
  */
@@ -38,15 +38,15 @@ async function render(
   mode,
   path,
   request,
-  addDataSourceWatcher,
+  addBackendTaskWatcher,
   hasFsAccess
 ) {
   const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(hasFsAccess);
   resetInMemoryFs();
   foundErrors = false;
-  pendingDataSourceResponses = [];
-  pendingDataSourceCount = 0;
-  // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
+  pendingBackendTaskResponses = new Map();
+  pendingBackendTaskCount = 0;
+  // since init/update are never called in pre-renders, and BackendTask.Http is called using pure NodeJS HTTP fetching
   // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
   XMLHttpRequest = {};
   const result = await runElmApp(
@@ -56,7 +56,7 @@ async function render(
     mode,
     path,
     request,
-    addDataSourceWatcher,
+    addBackendTaskWatcher,
     fs,
     hasFsAccess
   );
@@ -74,9 +74,9 @@ async function runGenerator(cliOptions, portsFile, elmModule) {
   const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(true);
   resetInMemoryFs();
   foundErrors = false;
-  pendingDataSourceResponses = [];
-  pendingDataSourceCount = 0;
-  // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
+  pendingBackendTaskResponses = new Map();
+  pendingBackendTaskCount = 0;
+  // since init/update are never called in pre-renders, and BackendTask.Http is called using pure NodeJS HTTP fetching
   // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
   XMLHttpRequest = {};
   const result = await runGeneratorAppHelp(
@@ -178,12 +178,14 @@ function runGeneratorAppHelp(
           );
         }
       } else if (fromElm.tag === "DoHttp") {
-        const requestToPerform = fromElm.args[0];
+        const requestHash = fromElm.args[0];
+        const requestToPerform = fromElm.args[1];
         if (
           requestToPerform.url !== "elm-pages-internal://port" &&
           requestToPerform.url.startsWith("elm-pages-internal://")
         ) {
           runInternalJob(
+            requestHash,
             app,
             mode,
             requestToPerform,
@@ -193,6 +195,7 @@ function runGeneratorAppHelp(
           );
         } else {
           runHttpJob(
+            requestHash,
             portsFile,
             app,
             mode,
@@ -225,7 +228,7 @@ function runGeneratorAppHelp(
  * @param {string} pagePath
  * @param {string} mode
  * @param {{ method: string; hostname: string; query: string; headers: Object; host: string; pathname: string; port: string; protocol: string; rawUrl: string; }} request
- * @param {(pattern: string) => void} addDataSourceWatcher
+ * @param {(pattern: string) => void} addBackendTaskWatcher
  * @returns {Promise<({is404: boolean} & ( { kind: 'json'; contentJson: string} | { kind: 'html'; htmlString: string } | { kind: 'api-response'; body: string; }) )>}
  */
 function runElmApp(
@@ -235,7 +238,7 @@ function runElmApp(
   mode,
   pagePath,
   request,
-  addDataSourceWatcher,
+  addBackendTaskWatcher,
   fs,
   hasFsAccess
 ) {
@@ -318,12 +321,14 @@ function runElmApp(
           );
         }
       } else if (fromElm.tag === "DoHttp") {
-        const requestToPerform = fromElm.args[0];
+        const requestHash = fromElm.args[0];
+        const requestToPerform = fromElm.args[1];
         if (
           requestToPerform.url !== "elm-pages-internal://port" &&
           requestToPerform.url.startsWith("elm-pages-internal://")
         ) {
           runInternalJob(
+            requestHash,
             app,
             mode,
             requestToPerform,
@@ -333,6 +338,7 @@ function runElmApp(
           );
         } else {
           runHttpJob(
+            requestHash,
             portsFile,
             app,
             mode,
@@ -352,7 +358,7 @@ function runElmApp(
     app.ports.toJsPort.subscribe(portHandler);
     app.ports.sendPageData.subscribe(portHandler);
   }).finally(() => {
-    addDataSourceWatcher(patternsToWatch);
+    addBackendTaskWatcher(patternsToWatch);
     try {
       killApp();
       killApp = null;
@@ -401,6 +407,7 @@ async function outputString(
 /** @typedef { { head: any[]; errors: any[]; contentJson: any[]; html: string; route: string; title: string; } } Arg */
 
 async function runHttpJob(
+  requestHash,
   portsFile,
   app,
   mode,
@@ -409,9 +416,9 @@ async function runHttpJob(
   hasFsAccess,
   useCache
 ) {
-  pendingDataSourceCount += 1;
+  pendingBackendTaskCount += 1;
   try {
-    const responseFilePath = await lookupOrPerform(
+    const lookupResponse = await lookupOrPerform(
       portsFile,
       mode,
       requestToPerform,
@@ -419,16 +426,26 @@ async function runHttpJob(
       useCache
     );
 
-    pendingDataSourceResponses.push({
-      request: requestToPerform,
-      response: JSON.parse(
-        (await fs.promises.readFile(responseFilePath, "utf8")).toString()
-      ),
-    });
+    if (lookupResponse.kind === "cache-response-path") {
+      const responseFilePath = lookupResponse.value;
+      pendingBackendTaskResponses.set(requestHash, {
+        request: requestToPerform,
+        response: JSON.parse(
+          (await fs.promises.readFile(responseFilePath, "utf8")).toString()
+        ),
+      });
+    } else if (lookupResponse.kind === "response-json") {
+      pendingBackendTaskResponses.set(requestHash, {
+        request: requestToPerform,
+        response: lookupResponse.value,
+      });
+    } else {
+      throw `Unexpected kind ${lookupResponse}`;
+    }
   } catch (error) {
     sendError(app, error);
   } finally {
-    pendingDataSourceCount -= 1;
+    pendingBackendTaskCount -= 1;
     flushIfDone(app);
   }
 }
@@ -447,6 +464,7 @@ function jsonResponse(request, json) {
 }
 
 async function runInternalJob(
+  requestHash,
   app,
   mode,
   requestToPerform,
@@ -455,41 +473,52 @@ async function runInternalJob(
   patternsToWatch
 ) {
   try {
-    pendingDataSourceCount += 1;
+    pendingBackendTaskCount += 1;
 
     if (requestToPerform.url === "elm-pages-internal://log") {
-      pendingDataSourceResponses.push(await runLogJob(requestToPerform));
+      pendingBackendTaskResponses.set(
+        requestHash,
+        await runLogJob(requestToPerform)
+      );
     } else if (requestToPerform.url === "elm-pages-internal://read-file") {
-      pendingDataSourceResponses.push(
+      pendingBackendTaskResponses.set(
+        requestHash,
         await readFileJobNew(requestToPerform, patternsToWatch)
       );
     } else if (requestToPerform.url === "elm-pages-internal://glob") {
-      pendingDataSourceResponses.push(
+      pendingBackendTaskResponses.set(
+        requestHash,
         await runGlobNew(requestToPerform, patternsToWatch)
       );
     } else if (requestToPerform.url === "elm-pages-internal://env") {
-      pendingDataSourceResponses.push(
+      pendingBackendTaskResponses.set(
+        requestHash,
         await runEnvJob(requestToPerform, patternsToWatch)
       );
     } else if (requestToPerform.url === "elm-pages-internal://encrypt") {
-      pendingDataSourceResponses.push(
+      pendingBackendTaskResponses.set(
+        requestHash,
         await runEncryptJob(requestToPerform, patternsToWatch)
       );
     } else if (requestToPerform.url === "elm-pages-internal://decrypt") {
-      pendingDataSourceResponses.push(
+      pendingBackendTaskResponses.set(
+        requestHash,
         await runDecryptJob(requestToPerform, patternsToWatch)
       );
     } else if (requestToPerform.url === "elm-pages-internal://write-file") {
-      pendingDataSourceResponses.push(await runWriteFileJob(requestToPerform));
+      pendingBackendTaskResponses.set(
+        requestHash,
+        await runWriteFileJob(requestToPerform)
+      );
     } else {
-      throw `Unexpected internal DataSource request format: ${kleur.yellow(
+      throw `Unexpected internal BackendTask request format: ${kleur.yellow(
         JSON.stringify(2, null, requestToPerform)
       )}`;
     }
   } catch (error) {
     sendError(app, error);
   } finally {
-    pendingDataSourceCount -= 1;
+    pendingBackendTaskCount -= 1;
     flushIfDone(app);
   }
 }
@@ -505,6 +534,7 @@ async function readFileJobNew(req, patternsToWatch) {
           path.join(process.env.LAMBDA_TASK_ROOT || process.cwd(), filePath)
         )
       ).toString();
+    // TODO does this throw an error if there is invalid frontmatter?
     const parsedFile = matter(fileContents);
 
     return jsonResponse(req, {
@@ -513,12 +543,9 @@ async function readFileJobNew(req, patternsToWatch) {
       rawFile: fileContents,
     });
   } catch (error) {
-    throw {
-      title: "DataSource.File Error",
-      message: `A DataSource.File read failed because I couldn't find this file: ${kleur.yellow(
-        filePath
-      )}\n${kleur.red(error.toString())}`,
-    };
+    return jsonResponse(req, {
+      errorCode: error.code,
+    });
   }
 }
 async function runWriteFileJob(req) {
@@ -531,8 +558,8 @@ async function runWriteFileJob(req) {
   } catch (error) {
     console.trace(error);
     throw {
-      title: "DataSource Error",
-      message: `DataSource.Generator.writeFile failed for file path: ${kleur.yellow(
+      title: "BackendTask Error",
+      message: `BackendTask.Generator.writeFile failed for file path: ${kleur.yellow(
         data.path
       )}\n${kleur.red(error.toString())}`,
     };
@@ -589,7 +616,7 @@ async function runEncryptJob(req, patternsToWatch) {
     );
   } catch (e) {
     throw {
-      title: "DataSource Encrypt Error",
+      title: "BackendTask Encrypt Error",
       message:
         e.toString() + e.stack + "\n\n" + JSON.stringify(rawRequest, null, 2),
     };
@@ -606,7 +633,7 @@ async function runDecryptJob(req, patternsToWatch) {
     return jsonResponse(req, JSON.parse(signed || "null"));
   } catch (e) {
     throw {
-      title: "DataSource Decrypt Error",
+      title: "BackendTask Decrypt Error",
       message:
         e.toString() + e.stack + "\n\n" + JSON.stringify(rawRequest, null, 2),
     };
@@ -615,10 +642,10 @@ async function runDecryptJob(req, patternsToWatch) {
 
 function flushIfDone(app) {
   if (foundErrors) {
-    pendingDataSourceResponses = [];
-  } else if (pendingDataSourceCount === 0) {
+    pendingBackendTaskResponses = new Map();
+  } else if (pendingBackendTaskCount === 0) {
     // console.log(
-    //   `Flushing ${pendingDataSourceResponses.length} items in ${timeUntilThreshold}ms`
+    //   `Flushing ${pendingBackendTaskResponses.length} items in ${timeUntilThreshold}ms`
     // );
 
     flushQueue(app);
@@ -626,10 +653,9 @@ function flushIfDone(app) {
 }
 
 function flushQueue(app) {
-  const temp = pendingDataSourceResponses;
-  pendingDataSourceResponses = [];
+  app.ports.gotBatchSub.send(Object.fromEntries(pendingBackendTaskResponses));
+  pendingBackendTaskResponses = new Map();
   // console.log("@@@ FLUSHING", temp.length);
-  app.ports.gotBatchSub.send(temp);
 }
 
 /**