elm-pages 3.0.0-beta.10 → 3.0.0-beta.12

package/generator/review/elm-stuff/tests-0.19.1/js/node_runner.js

@@ -75,7 +75,7 @@ console.elmlog = (str) => logs.push(str + "\n");
 const { Elm } = require("./Runner.elm.js");
 
 // Start the Elm app
-const flags = { initialSeed: 2786629506, fuzzRuns: 100, filter: null };
+const flags = { initialSeed: 2434458748, fuzzRuns: 100, filter: null };
 const app = Elm.Runner.init({ flags: flags });
 
 // Record the timing at which we received the last "runTest" message

package/generator/review/elm-stuff/tests-0.19.1/js/node_supervisor.js

@@ -82,7 +82,7 @@ const verbosity = 0;
 // Create a long lived reporter worker
 const { Elm } = require("./Reporter.elm.js");
 const flags = {
-  initialSeed: 2786629506,
+  initialSeed: 2434458748,
   fuzzRuns: 100,
   mode: "consoleNoColor",
   verbosity: verbosity,

package/generator/review/elm.json

@@ -11,23 +11,23 @@
             "elm/html": "1.0.0",
             "elm/regex": "1.0.0",
             "elm-community/result-extra": "2.4.0",
-            "jfmengels/elm-review": "2.4.2",
+            "jfmengels/elm-review": "2.10.0",
             "mdgriffith/elm-codegen": "2.0.0",
-            "stil4m/elm-syntax": "7.2.5",
+            "stil4m/elm-syntax": "7.2.9",
             "the-sett/elm-syntax-dsl": "6.0.2"
         },
         "indirect": {
             "Chadtech/elm-bool-extra": "2.4.2",
+            "elm/bytes": "1.0.8",
             "elm/json": "1.1.3",
             "elm/parser": "1.1.0",
             "elm/project-metadata-utils": "1.0.2",
             "elm/random": "1.0.0",
             "elm/time": "1.0.0",
-            "elm/virtual-dom": "1.0.2",
+            "elm/virtual-dom": "1.0.3",
             "elm-community/basics-extra": "4.1.0",
-            "elm-community/list-extra": "8.3.0",
+            "elm-community/list-extra": "8.7.0",
             "elm-community/maybe-extra": "5.3.0",
-            "elm-explorations/test": "1.2.2",
             "miniBill/elm-unicode": "1.0.2",
             "rtfeldman/elm-hex": "1.0.0",
             "stil4m/structured-writer": "1.0.3",
@@ -36,7 +36,7 @@
     },
     "test-dependencies": {
         "direct": {
-            "elm-explorations/test": "1.2.2"
+            "elm-explorations/test": "2.0.1"
         },
         "indirect": {}
     }

package/generator/src/build.js

@@ -31,7 +31,6 @@ let pagesReadyCalled = false;
 let activeWorkers = 0;
 let buildError = false;
 
-const DIR_PATH = process.cwd();
 const OUTPUT_FILE_NAME = "elm.js";
 
 process.on("unhandledRejection", (error) => {
@@ -39,11 +38,9 @@ process.on("unhandledRejection", (error) => {
   process.exitCode = 1;
 });
 
-const ELM_FILE_PATH = path.join(
-  DIR_PATH,
-  "./elm-stuff/elm-pages",
-  OUTPUT_FILE_NAME
-);
+function ELM_FILE_PATH() {
+  return path.join(process.cwd(), "./elm-stuff/elm-pages", OUTPUT_FILE_NAME);
+}
 
 async function ensureRequiredDirs() {
   ensureDirSync(`dist`);
@@ -552,7 +549,7 @@ async function compileCliApp(options) {
     path.join(process.cwd(), "elm-stuff/elm-pages")
   );
 
-  const elmFileContent = await fsPromises.readFile(ELM_FILE_PATH, "utf-8");
+  const elmFileContent = await fsPromises.readFile(ELM_FILE_PATH(), "utf-8");
   // Source: https://github.com/elm-explorations/test/blob/d5eb84809de0f8bbf50303efd26889092c800609/src/Elm/Kernel/HtmlAsJson.js
   const forceThunksSource = ` _HtmlAsJson_toJson(x)
 }
@@ -598,7 +595,7 @@ function _HtmlAsJson_toJson(html) {
 `;
 
   await fsPromises.writeFile(
-    ELM_FILE_PATH,
+    ELM_FILE_PATH(),
     elmFileContent
       .replace(
         /return \$elm\$json\$Json\$Encode\$string\(.REPLACE_ME_WITH_JSON_STRINGIFY.\)/g,
@@ -675,4 +672,4 @@ function defaultPreloadForFile(file) {
  * @returns {string}
  */
 
-module.exports = { run };
+module.exports = { run, compileCliApp };

package/generator/src/cli.js

@@ -8,9 +8,13 @@ const codegen = require("./codegen.js");
 const fs = require("fs");
 const path = require("path");
 const { restoreColorSafe } = require("./error-formatter");
+const renderer = require("../../generator/src/render");
+const globby = require("globby");
+const esbuild = require("esbuild");
+const copyModifiedElmJson = require("./rewrite-elm-json.js");
+const { ensureDirSync, deleteIfExists } = require("./file-helpers.js");
 
 const commander = require("commander");
-const { compileCliApp } = require("./compile-elm.js");
 const { runElmCodegenInstall } = require("./elm-codegen.js");
 const Argument = commander.Argument;
 const Option = commander.Option;
@@ -84,66 +88,89 @@ async function main() {
     });
 
   program
-    .command("codegen <moduleName>")
-    .description("run a generator")
+    .command("run <moduleName>")
+    .description("run an elm-pages script")
     .allowUnknownOption()
     .allowExcessArguments()
     .action(async (moduleName, options, options2) => {
+      const unprocessedCliOptions = options2.args.splice(
+        options2.processedArgs.length,
+        options2.args.length
+      );
       if (!/^[A-Z][a-zA-Z0-9_]*(\.[A-Z][a-zA-Z0-9_]*)*$/.test(moduleName)) {
         throw `Invalid module name "${moduleName}", must be in the format of an Elm module`;
       }
       const splitModuleName = moduleName.split(".");
       const expectedFilePath = path.join(
         process.cwd(),
-        "codegen",
+        "script/src/",
         `${splitModuleName.join("/")}.elm`
       );
       if (!fs.existsSync(expectedFilePath)) {
         throw `I couldn't find a module named ${expectedFilePath}`;
       }
       try {
-        await codegen.generate("");
-        await runElmCodegenInstall();
-        await compileCliApp(
-          // { debug: true },
-          {},
-          `${splitModuleName.join("/")}.elm`,
-          path.join(process.cwd(), "codegen/elm-stuff/scaffold.js"),
-          "codegen",
-
-          path.join(process.cwd(), "codegen/elm-stuff/scaffold.js")
+        // await codegen.generate("");
+        if (fs.existsSync("./codegen/")) {
+          await runElmCodegenInstall();
+        }
+
+        ensureDirSync("./script/elm-stuff");
+        ensureDirSync("./script/elm-stuff/elm-pages/.elm-pages");
+        await fs.promises.writeFile(
+          path.join("./script/elm-stuff/elm-pages/.elm-pages/Main.elm"),
+          generatorWrapperFile(moduleName)
+        );
+        await copyModifiedElmJson(
+          "./script/elm.json",
+          "./script/elm-stuff/elm-pages/elm.json"
+        );
+
+        const portDataSourceCompiled = esbuild
+          .build({
+            entryPoints: ["./port-data-source"],
+            platform: "node",
+            outfile: ".elm-pages/compiled-ports/port-data-source.js",
+            assetNames: "[name]-[hash]",
+            chunkNames: "chunks/[name]-[hash]",
+            outExtension: { ".js": ".js" },
+            metafile: true,
+            bundle: true,
+            watch: false,
+            logLevel: "silent",
+          })
+          .then((result) => {
+            try {
+              return Object.keys(result.metafile.outputs)[0];
+            } catch (e) {
+              return null;
+            }
+          })
+          .catch((error) => {
+            const portDataSourceFileFound =
+              globby.sync("./port-data-source.*").length > 0;
+            if (portDataSourceFileFound) {
+              // don't present error if there are no files matching port-data-source
+              // if there are files matching port-data-source, warn the user in case something went wrong loading it
+              console.error("Failed to start port-data-source watcher", error);
+            }
+          });
+        const portsPath = await portDataSourceCompiled;
+        const resolvedPortsPath = path.join(process.cwd(), portsPath);
+
+        process.chdir("./script");
+        // TODO have option for compiling with --debug or not (maybe allow running with elm-optimize-level-2 as well?)
+        await build.compileCliApp({ debug: "debug" });
+        process.chdir("../");
+        await renderer.runGenerator(
+          unprocessedCliOptions,
+          resolvedPortsPath,
+          requireElm("./script/elm-stuff/elm-pages/elm.js")
         );
       } catch (error) {
         console.log(restoreColorSafe(error));
         process.exit(1);
       }
-
-      const elmScaffoldProgram = getAt(
-        splitModuleName,
-        require(path.join(process.cwd(), "./codegen/elm-stuff/scaffold.js")).Elm
-      );
-      const program = elmScaffoldProgram.init({
-        flags: { argv: ["", ...options2.args], versionMessage: "1.2.3" },
-      });
-
-      safeSubscribe(program, "print", (message) => {
-        console.log(message);
-      });
-      safeSubscribe(program, "printAndExitFailure", (message) => {
-        console.log(message);
-        process.exit(1);
-      });
-      safeSubscribe(program, "printAndExitSuccess", (message) => {
-        console.log(message);
-        process.exit(0);
-      });
-      safeSubscribe(program, "writeFile", async (info) => {
-        const filePath = path.join(process.cwd(), "app", info.path);
-        await dirHelpers.tryMkdir(path.dirname(filePath));
-        fs.writeFileSync(filePath, info.body);
-        console.log("Success! Created file", filePath);
-        process.exit(0);
-      });
     });
 
   program
@@ -180,7 +207,8 @@ function getAt(properties, object) {
 }
 
 function safeSubscribe(program, portName, subscribeFunction) {
-  program.ports[portName] &&
+  program.ports &&
+    program.ports[portName] &&
     program.ports[portName].subscribe(subscribeFunction);
 }
 
@@ -218,5 +246,55 @@ function normalizeUrl(rawPagePath) {
   // with detecting whether the path contains the base.
   return `/${segments.join("/")}`;
 }
+/**
+ * @param {string} compiledElmPath
+ */
+function requireElm(compiledElmPath) {
+  const warnOriginal = console.warn;
+  console.warn = function () {};
+
+  Elm = require(path.resolve(compiledElmPath));
+  console.warn = warnOriginal;
+  return Elm;
+}
+
+/**
+ * @param {string} moduleName
+ */
+function generatorWrapperFile(moduleName) {
+  return `port module Main exposing (main)
+
+import Bytes
+import DataSource exposing (DataSource)
+import Cli.Program as Program
+import Json.Decode as Decode
+import Json.Encode as Encode
+import Pages.Internal.Platform.GeneratorApplication
+import ${moduleName}
+
+
+main : Program.StatefulProgram Pages.Internal.Platform.GeneratorApplication.Model Pages.Internal.Platform.GeneratorApplication.Msg (DataSource ()) Pages.Internal.Platform.GeneratorApplication.Flags
+main =
+    Pages.Internal.Platform.GeneratorApplication.app
+        { data = ${moduleName}.run
+        , toJsPort = toJsPort
+        , fromJsPort = fromJsPort identity
+        , gotBatchSub = gotBatchSub identity
+        , sendPageData = sendPageData
+        }
+
+
+port toJsPort : Encode.Value -> Cmd msg
+
+
+port fromJsPort : (Decode.Value -> msg) -> Sub msg
+
+
+port gotBatchSub : (Decode.Value -> msg) -> Sub msg
+
+
+port sendPageData : { oldThing : Encode.Value, binaryPageData : Bytes.Bytes } -> Cmd msg
+`;
+}
 
 main();

package/generator/src/codegen.js

@@ -63,7 +63,7 @@ async function generate(basePath) {
       browserCode.fetcherModules
     ),
     // write modified elm.json to elm-stuff/elm-pages/
-    copyModifiedElmJson(),
+    copyModifiedElmJson("./elm.json", "./elm-stuff/elm-pages/elm.json"),
     // ...(await listFiles("./Pages/Internal")).map(copyToBoth),
   ]);
 }
@@ -174,15 +174,16 @@ async function copyToBoth(moduleToCopy) {
     copyFileEnsureDir(
       path.join(__dirname, moduleToCopy),
       path.join(`./.elm-pages/`, moduleToCopy)
-   ),
-   copyFileEnsureDir(
+    ),
+    copyFileEnsureDir(
       path.join(__dirname, moduleToCopy),
       path.join(`./elm-stuff/elm-pages/client/.elm-pages`, moduleToCopy)
-   ),
-  copyFileEnsureDir(
+    ),
+    copyFileEnsureDir(
       path.join(__dirname, moduleToCopy),
       path.join(`./elm-stuff/elm-pages/.elm-pages/`, moduleToCopy)
-   )]);
+    ),
+  ]);
 }
 
 /**
@@ -190,10 +191,10 @@ async function copyToBoth(moduleToCopy) {
  * @param {string} to
  */
 async function copyFileEnsureDir(from, to) {
-    await fs.promises.mkdir(path.dirname(to), {
-      recursive: true,
-    })
-    await fs.promises.copyFile(from, to)
+  await fs.promises.mkdir(path.dirname(to), {
+    recursive: true,
+  });
+  await fs.promises.copyFile(from, to);
 }
 
 /**

package/generator/src/compatibility-key.js

@@ -1 +1 @@
-module.exports = { compatibilityKey: 1 };
+module.exports = { compatibilityKey: 2 };

package/generator/src/elm-codegen.js

@@ -17,6 +17,9 @@ function runElmCodegenInstall() {
     subprocess.stderr.on("data", function (data) {
       commandOutput += data;
     });
+    subprocess.stdout.on("data", function (data) {
+      commandOutput += data;
+    });
     subprocess.on("error", function () {
       reject(commandOutput);
     });

package/generator/src/render-worker.js

@@ -10,7 +10,7 @@ global.staticHttpCache = {};
 async function run({ mode, pathname, serverRequest, portsFilePath }) {
   console.time(`${threadId} ${pathname}`);
   try {
-    const renderResult = await renderer(
+    const renderResult = await renderer.render(
       portsFilePath,
       workerData.basePath,
       requireElm(mode),

package/generator/src/render.js

@@ -19,18 +19,37 @@ let foundErrors;
 let pendingDataSourceResponses;
 let pendingDataSourceCount;
 
-module.exports =
-  /**
-   *
-   * @param {string} basePath
-   * @param {Object} elmModule
-   * @param {string} path
-   * @param {{ method: string; hostname: string; query: Record<string, string | undefined>; headers: Record<string, string>; host: string; pathname: string; port: number | null; protocol: string; rawUrl: string; }} request
-   * @param {(pattern: string) => void} addDataSourceWatcher
-   * @param {boolean} hasFsAccess
-   * @returns
-   */
-  async function run(
+module.exports = { render, runGenerator };
+
+/**
+ *
+ * @param {string} basePath
+ * @param {Object} elmModule
+ * @param {string} path
+ * @param {{ method: string; hostname: string; query: Record<string, string | undefined>; headers: Record<string, string>; host: string; pathname: string; port: number | null; protocol: string; rawUrl: string; }} request
+ * @param {(pattern: string) => void} addDataSourceWatcher
+ * @param {boolean} hasFsAccess
+ * @returns
+ */
+async function render(
+  portsFile,
+  basePath,
+  elmModule,
+  mode,
+  path,
+  request,
+  addDataSourceWatcher,
+  hasFsAccess
+) {
+  const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(hasFsAccess);
+  resetInMemoryFs();
+  foundErrors = false;
+  pendingDataSourceResponses = [];
+  pendingDataSourceCount = 0;
+  // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
+  // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
+  XMLHttpRequest = {};
+  const result = await runElmApp(
     portsFile,
     basePath,
     elmModule,
@@ -38,31 +57,167 @@ module.exports =
     path,
     request,
     addDataSourceWatcher,
+    fs,
     hasFsAccess
-  ) {
-    const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(
-      hasFsAccess
-    );
-    resetInMemoryFs();
-    foundErrors = false;
-    pendingDataSourceResponses = [];
-    pendingDataSourceCount = 0;
-    // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
-    // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
-    XMLHttpRequest = {};
-    const result = await runElmApp(
-      portsFile,
-      basePath,
-      elmModule,
-      mode,
-      path,
-      request,
-      addDataSourceWatcher,
-      fs,
-      hasFsAccess
-    );
-    return result;
-  };
+  );
+  return result;
+}
+
+/**
+ * @param {Object} elmModule
+ * @returns
+ * @param {string[]} cliOptions
+ * @param {any} portsFile
+ */
+async function runGenerator(cliOptions, portsFile, elmModule) {
+  global.isRunningGenerator = true;
+  const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(true);
+  resetInMemoryFs();
+  foundErrors = false;
+  pendingDataSourceResponses = [];
+  pendingDataSourceCount = 0;
+  // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
+  // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
+  XMLHttpRequest = {};
+  const result = await runGeneratorAppHelp(
+    cliOptions,
+    portsFile,
+    "",
+    elmModule,
+    "production",
+    "",
+    fs,
+    true
+  );
+  return result;
+}
+/**
+ * @param {string} basePath
+ * @param {Object} elmModule
+ * @param {string} pagePath
+ * @param {string} mode
+ * @returns {Promise<({is404: boolean;} & ({kind: 'json';contentJson: string;} | {kind: 'html';htmlString: string;} | {kind: 'api-response';body: string;}))>}
+ * @param {string[]} cliOptions
+ * @param {any} portsFile
+ * @param {typeof import("fs") | import("memfs").IFs} fs
+ * @param {boolean} hasFsAccess
+ */
+function runGeneratorAppHelp(
+  cliOptions,
+  portsFile,
+  basePath,
+  elmModule,
+  mode,
+  pagePath,
+  fs,
+  hasFsAccess
+) {
+  const isDevServer = mode !== "build";
+  let patternsToWatch = new Set();
+  let app = null;
+  let killApp;
+  return new Promise((resolve, reject) => {
+    const isBytes = pagePath.match(/content\.dat\/?$/);
+
+    app = elmModule.Elm.Main.init({
+      flags: {
+        compatibilityKey,
+        argv: ["", "", ...cliOptions],
+        versionMessage: "1.2.3",
+      },
+    });
+
+    killApp = () => {
+      app.ports.toJsPort.unsubscribe(portHandler);
+      app.ports.sendPageData.unsubscribe(portHandler);
+      app.die();
+      app = null;
+      // delete require.cache[require.resolve(compiledElmPath)];
+    };
+
+    async function portHandler(/** @type { FromElm }  */ newThing) {
+      let fromElm;
+      let contentDatPayload;
+
+      fromElm = newThing;
+      if (fromElm.command === "log") {
+        console.log(fromElm.value);
+      } else if (fromElm.tag === "ApiResponse") {
+        const args = fromElm.args[0];
+        if (mode === "build") {
+          global.staticHttpCache = args.staticHttpCache;
+        }
+
+        resolve({
+          kind: "api-response",
+          is404: args.is404,
+          statusCode: args.statusCode,
+          body: args.body,
+        });
+      } else if (fromElm.tag === "PageProgress") {
+        const args = fromElm.args[0];
+        if (mode === "build") {
+          global.staticHttpCache = args.staticHttpCache;
+        }
+
+        if (isBytes) {
+          resolve({
+            kind: "bytes",
+            is404: false,
+            contentJson: JSON.stringify({
+              staticData: args.contentJson,
+              is404: false,
+            }),
+            statusCode: args.statusCode,
+            headers: args.headers,
+            contentDatPayload,
+          });
+        } else {
+          resolve(
+            outputString(basePath, fromElm, isDevServer, contentDatPayload)
+          );
+        }
+      } else if (fromElm.tag === "DoHttp") {
+        const requestToPerform = fromElm.args[0];
+        if (
+          requestToPerform.url !== "elm-pages-internal://port" &&
+          requestToPerform.url.startsWith("elm-pages-internal://")
+        ) {
+          runInternalJob(
+            app,
+            mode,
+            requestToPerform,
+            fs,
+            hasFsAccess,
+            patternsToWatch
+          );
+        } else {
+          runHttpJob(
+            portsFile,
+            app,
+            mode,
+            requestToPerform,
+            fs,
+            hasFsAccess,
+            fromElm.args[1]
+          );
+        }
+      } else if (fromElm.tag === "Errors") {
+        foundErrors = true;
+        reject(fromElm.args[0].errorsJson);
+      } else {
+        console.log(fromElm);
+      }
+    }
+    app.ports.toJsPort.subscribe(portHandler);
+    app.ports.sendPageData.subscribe(portHandler);
+  }).finally(() => {
+    try {
+      killApp();
+      killApp = null;
+    } catch (error) {}
+  });
+}
 
 /**
  * @param {string} basePath
@@ -302,7 +457,9 @@ async function runInternalJob(
   try {
     pendingDataSourceCount += 1;
 
-    if (requestToPerform.url === "elm-pages-internal://read-file") {
+    if (requestToPerform.url === "elm-pages-internal://log") {
+      pendingDataSourceResponses.push(await runLogJob(requestToPerform));
+    } else if (requestToPerform.url === "elm-pages-internal://read-file") {
       pendingDataSourceResponses.push(
         await readFileJobNew(requestToPerform, patternsToWatch)
       );
@@ -322,6 +479,8 @@ async function runInternalJob(
       pendingDataSourceResponses.push(
         await runDecryptJob(requestToPerform, patternsToWatch)
       );
+    } else if (requestToPerform.url === "elm-pages-internal://write-file") {
+      pendingDataSourceResponses.push(await runWriteFileJob(requestToPerform));
     } else {
       throw `Unexpected internal DataSource request format: ${kleur.yellow(
         JSON.stringify(2, null, requestToPerform)
@@ -362,6 +521,23 @@ async function readFileJobNew(req, patternsToWatch) {
     };
   }
 }
+async function runWriteFileJob(req) {
+  const data = req.body.args[0];
+  try {
+    const fullPathToWrite = path.join(process.cwd(), data.path);
+    await fsPromises.mkdir(path.dirname(fullPathToWrite), { recursive: true });
+    await fsPromises.writeFile(fullPathToWrite, data.body);
+    return jsonResponse(req, null);
+  } catch (error) {
+    console.trace(error);
+    throw {
+      title: "DataSource Error",
+      message: `DataSource.Generator.writeFile failed for file path: ${kleur.yellow(
+        data.path
+      )}\n${kleur.red(error.toString())}`,
+    };
+  }
+}
 
 async function runGlobNew(req, patternsToWatch) {
   try {
@@ -384,6 +560,15 @@ async function runGlobNew(req, patternsToWatch) {
   }
 }
 
+async function runLogJob(req) {
+  try {
+    console.log(req.body.args[0].message);
+    return jsonResponse(req, null);
+  } catch (e) {
+    console.log(`Error performing env '${JSON.stringify(req.body)}'`);
+    throw e;
+  }
+}
 async function runEnvJob(req, patternsToWatch) {
   try {
     const expectedEnv = req.body.args[0];