elm-pages 3.0.0-beta.10 → 3.0.0-beta.13

package/generator/src/render.js

@@ -19,18 +19,37 @@ let foundErrors;
 let pendingDataSourceResponses;
 let pendingDataSourceCount;
 
-module.exports =
-  /**
-   *
-   * @param {string} basePath
-   * @param {Object} elmModule
-   * @param {string} path
-   * @param {{ method: string; hostname: string; query: Record<string, string | undefined>; headers: Record<string, string>; host: string; pathname: string; port: number | null; protocol: string; rawUrl: string; }} request
-   * @param {(pattern: string) => void} addDataSourceWatcher
-   * @param {boolean} hasFsAccess
-   * @returns
-   */
-  async function run(
+module.exports = { render, runGenerator };
+
+/**
+ *
+ * @param {string} basePath
+ * @param {Object} elmModule
+ * @param {string} path
+ * @param {{ method: string; hostname: string; query: Record<string, string | undefined>; headers: Record<string, string>; host: string; pathname: string; port: number | null; protocol: string; rawUrl: string; }} request
+ * @param {(pattern: string) => void} addDataSourceWatcher
+ * @param {boolean} hasFsAccess
+ * @returns
+ */
+async function render(
+  portsFile,
+  basePath,
+  elmModule,
+  mode,
+  path,
+  request,
+  addDataSourceWatcher,
+  hasFsAccess
+) {
+  const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(hasFsAccess);
+  resetInMemoryFs();
+  foundErrors = false;
+  pendingDataSourceResponses = [];
+  pendingDataSourceCount = 0;
+  // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
+  // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
+  XMLHttpRequest = {};
+  const result = await runElmApp(
     portsFile,
     basePath,
     elmModule,
@@ -38,31 +57,167 @@ module.exports =
     path,
     request,
     addDataSourceWatcher,
+    fs,
     hasFsAccess
-  ) {
-    const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(
-      hasFsAccess
-    );
-    resetInMemoryFs();
-    foundErrors = false;
-    pendingDataSourceResponses = [];
-    pendingDataSourceCount = 0;
-    // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
-    // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
-    XMLHttpRequest = {};
-    const result = await runElmApp(
-      portsFile,
-      basePath,
-      elmModule,
-      mode,
-      path,
-      request,
-      addDataSourceWatcher,
-      fs,
-      hasFsAccess
-    );
-    return result;
-  };
+  );
+  return result;
+}
+
+/**
+ * @param {Object} elmModule
+ * @returns
+ * @param {string[]} cliOptions
+ * @param {any} portsFile
+ */
+async function runGenerator(cliOptions, portsFile, elmModule) {
+  global.isRunningGenerator = true;
+  const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(true);
+  resetInMemoryFs();
+  foundErrors = false;
+  pendingDataSourceResponses = [];
+  pendingDataSourceCount = 0;
+  // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
+  // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
+  XMLHttpRequest = {};
+  const result = await runGeneratorAppHelp(
+    cliOptions,
+    portsFile,
+    "",
+    elmModule,
+    "production",
+    "",
+    fs,
+    true
+  );
+  return result;
+}
+/**
+ * @param {string} basePath
+ * @param {Object} elmModule
+ * @param {string} pagePath
+ * @param {string} mode
+ * @returns {Promise<({is404: boolean;} & ({kind: 'json';contentJson: string;} | {kind: 'html';htmlString: string;} | {kind: 'api-response';body: string;}))>}
+ * @param {string[]} cliOptions
+ * @param {any} portsFile
+ * @param {typeof import("fs") | import("memfs").IFs} fs
+ * @param {boolean} hasFsAccess
+ */
+function runGeneratorAppHelp(
+  cliOptions,
+  portsFile,
+  basePath,
+  elmModule,
+  mode,
+  pagePath,
+  fs,
+  hasFsAccess
+) {
+  const isDevServer = mode !== "build";
+  let patternsToWatch = new Set();
+  let app = null;
+  let killApp;
+  return new Promise((resolve, reject) => {
+    const isBytes = pagePath.match(/content\.dat\/?$/);
+
+    app = elmModule.Elm.Main.init({
+      flags: {
+        compatibilityKey,
+        argv: ["", "", ...cliOptions],
+        versionMessage: "1.2.3",
+      },
+    });
+
+    killApp = () => {
+      app.ports.toJsPort.unsubscribe(portHandler);
+      app.ports.sendPageData.unsubscribe(portHandler);
+      app.die();
+      app = null;
+      // delete require.cache[require.resolve(compiledElmPath)];
+    };
+
+    async function portHandler(/** @type { FromElm }  */ newThing) {
+      let fromElm;
+      let contentDatPayload;
+
+      fromElm = newThing;
+      if (fromElm.command === "log") {
+        console.log(fromElm.value);
+      } else if (fromElm.tag === "ApiResponse") {
+        const args = fromElm.args[0];
+        if (mode === "build") {
+          global.staticHttpCache = args.staticHttpCache;
+        }
+
+        resolve({
+          kind: "api-response",
+          is404: args.is404,
+          statusCode: args.statusCode,
+          body: args.body,
+        });
+      } else if (fromElm.tag === "PageProgress") {
+        const args = fromElm.args[0];
+        if (mode === "build") {
+          global.staticHttpCache = args.staticHttpCache;
+        }
+
+        if (isBytes) {
+          resolve({
+            kind: "bytes",
+            is404: false,
+            contentJson: JSON.stringify({
+              staticData: args.contentJson,
+              is404: false,
+            }),
+            statusCode: args.statusCode,
+            headers: args.headers,
+            contentDatPayload,
+          });
+        } else {
+          resolve(
+            outputString(basePath, fromElm, isDevServer, contentDatPayload)
+          );
+        }
+      } else if (fromElm.tag === "DoHttp") {
+        const requestToPerform = fromElm.args[0];
+        if (
+          requestToPerform.url !== "elm-pages-internal://port" &&
+          requestToPerform.url.startsWith("elm-pages-internal://")
+        ) {
+          runInternalJob(
+            app,
+            mode,
+            requestToPerform,
+            fs,
+            hasFsAccess,
+            patternsToWatch
+          );
+        } else {
+          runHttpJob(
+            portsFile,
+            app,
+            mode,
+            requestToPerform,
+            fs,
+            hasFsAccess,
+            fromElm.args[1]
+          );
+        }
+      } else if (fromElm.tag === "Errors") {
+        foundErrors = true;
+        reject(fromElm.args[0].errorsJson);
+      } else {
+        console.log(fromElm);
+      }
+    }
+    app.ports.toJsPort.subscribe(portHandler);
+    app.ports.sendPageData.subscribe(portHandler);
+  }).finally(() => {
+    try {
+      killApp();
+      killApp = null;
+    } catch (error) {}
+  });
+}
 
 /**
  * @param {string} basePath
@@ -302,7 +457,9 @@ async function runInternalJob(
   try {
     pendingDataSourceCount += 1;
 
-    if (requestToPerform.url === "elm-pages-internal://read-file") {
+    if (requestToPerform.url === "elm-pages-internal://log") {
+      pendingDataSourceResponses.push(await runLogJob(requestToPerform));
+    } else if (requestToPerform.url === "elm-pages-internal://read-file") {
       pendingDataSourceResponses.push(
         await readFileJobNew(requestToPerform, patternsToWatch)
       );
@@ -322,6 +479,8 @@ async function runInternalJob(
       pendingDataSourceResponses.push(
         await runDecryptJob(requestToPerform, patternsToWatch)
       );
+    } else if (requestToPerform.url === "elm-pages-internal://write-file") {
+      pendingDataSourceResponses.push(await runWriteFileJob(requestToPerform));
     } else {
       throw `Unexpected internal DataSource request format: ${kleur.yellow(
         JSON.stringify(2, null, requestToPerform)
@@ -362,6 +521,23 @@ async function readFileJobNew(req, patternsToWatch) {
     };
   }
 }
+async function runWriteFileJob(req) {
+  const data = req.body.args[0];
+  try {
+    const fullPathToWrite = path.join(process.cwd(), data.path);
+    await fsPromises.mkdir(path.dirname(fullPathToWrite), { recursive: true });
+    await fsPromises.writeFile(fullPathToWrite, data.body);
+    return jsonResponse(req, null);
+  } catch (error) {
+    console.trace(error);
+    throw {
+      title: "DataSource Error",
+      message: `DataSource.Generator.writeFile failed for file path: ${kleur.yellow(
+        data.path
+      )}\n${kleur.red(error.toString())}`,
+    };
+  }
+}
 
 async function runGlobNew(req, patternsToWatch) {
   try {
@@ -384,6 +560,15 @@ async function runGlobNew(req, patternsToWatch) {
   }
 }
 
+async function runLogJob(req) {
+  try {
+    console.log(req.body.args[0].message);
+    return jsonResponse(req, null);
+  } catch (e) {
+    console.log(`Error performing env '${JSON.stringify(req.body)}'`);
+    throw e;
+  }
+}
 async function runEnvJob(req, patternsToWatch) {
   try {
     const expectedEnv = req.body.args[0];

package/generator/src/request-cache.js

@@ -24,6 +24,7 @@ function fullPath(portsHash, request, hasFsAccess) {
   if (hasFsAccess) {
     return path.join(
       process.cwd(),
+      // TODO use parameter or something other than global for this `global.isRunningGenerator` condition
       ".elm-pages",
       "http-response-cache",
       requestToString(requestWithPortHash)

package/generator/src/rewrite-elm-json.js

@@ -1,14 +1,14 @@
 const fs = require("fs");
 
-module.exports = async function () {
+module.exports = async function (sourceElmJsonPath, targetElmJsonPath) {
   var elmJson = JSON.parse(
-    (await fs.promises.readFile("./elm.json")).toString()
+    (await fs.promises.readFile(sourceElmJsonPath)).toString()
   );
 
   // write new elm.json
 
   await writeFileIfChanged(
-    "./elm-stuff/elm-pages/elm.json",
+    targetElmJsonPath,
     JSON.stringify(rewriteElmJson(elmJson))
   );
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "elm-pages",
-  "version": "3.0.0-beta.10",
+  "version": "3.0.0-beta.13",
   "homepage": "https://elm-pages.com",
   "moduleResolution": "node",
   "description": "Type-safe static sites, written in pure elm with your own custom elm-markup syntax.",
@@ -26,14 +26,14 @@
   "dependencies": {
     "busboy": "^1.0.0",
     "chokidar": "^3.5.3",
-    "commander": "^9.4.0",
+    "commander": "9.4.1",
     "connect": "^3.7.0",
     "cookie-signature": "^1.1.0",
     "cross-spawn": "7.0.3",
     "devcert": "^1.2.2",
     "elm-doc-preview": "^5.0.5",
     "elm-hot": "^1.1.6",
-    "esbuild": "^0.15.5",
+    "esbuild": "^0.16.1",
     "fs-extra": "^10.1.0",
     "globby": "11.0.4",
     "gray-matter": "^4.0.3",
@@ -44,26 +44,26 @@
     "node-fetch": "^2.6.7",
     "object-hash": "^2.2.0",
     "serve-static": "^1.15.0",
-    "terser": "^5.14.2",
-    "vite": "^3.1.8",
+    "terser": "^5.16.1",
+    "vite": "^3.2.5",
     "which": "^2.0.2"
   },
   "devDependencies": {
     "@types/cross-spawn": "^6.0.2",
     "@types/fs-extra": "^9.0.13",
     "@types/micromatch": "^4.0.2",
-    "@types/node": "12.20.12",
+    "@types/node": "^18.11.9",
     "@types/serve-static": "^1.15.0",
-    "cypress": "^10.6.0",
+    "cypress": "^11.1.0",
     "elm-codegen": "^0.2.0",
-    "elm-optimize-level-2": "^0.1.5",
-    "elm-review": "^2.7.4",
-    "elm-test": "^0.19.1-revision9",
-    "elm-tooling": "^1.8.0",
+    "elm-optimize-level-2": "^0.3.5",
+    "elm-review": "^2.8.2",
+    "elm-test": "^0.19.1-revision10",
+    "elm-tooling": "^1.10.0",
     "elm-verify-examples": "^5.2.0",
     "elmi-to-json": "^1.2.0",
     "mocha": "^10.0.0",
-    "typescript": "^4.7.4"
+    "typescript": "^4.9.3"
   },
   "files": [
     "generator/src/",

package/src/ApiRoute.elm

@@ -1,8 +1,9 @@
 module ApiRoute exposing
-    ( ApiRoute, ApiRouteBuilder, Response
+    ( single, preRender
+    , serverRender
+    , preRenderWithFallback
+    , ApiRoute, ApiRouteBuilder, Response
     , capture, literal, slash, succeed
-    , single, preRender
-    , preRenderWithFallback, serverRender
     , withGlobalHeadTags
     , toJson, getBuildTimeRoutes, getGlobalHeadTagsDataSource
     )
@@ -11,19 +12,153 @@ module ApiRoute exposing
 to a DataSource so you can pull in HTTP data, etc. Because ApiRoutes don't hydrate into Elm apps (like pages in elm-pages do), you can pull in as much data as you want in
 the DataSource for your ApiRoutes, and it won't effect the payload size. Instead, the size of an ApiRoute is just the content you output for that route.
 
-@docs ApiRoute, ApiRouteBuilder, Response
-
-@docs capture, literal, slash, succeed
+Similar to your elm-pages Route Modules, ApiRoute's can be either server-rendered or pre-rendered. Let's compare the differences between pre-rendered and server-rendered ApiRoutes, and the different
+use cases they support.
 
 
 ## Pre-Rendering
 
+A pre-rendered ApiRoute is just a generated file. For example:
+
+  - [An RSS feed](https://github.com/dillonkearns/elm-pages/blob/131f7b750cdefb2ba7a34a06be06dfbfafc79a86/examples/docs/app/Api.elm#L77-L84) ([Output file](https://elm-pages.com/blog/feed.xml))
+  - [A calendar feed in the ical format](https://github.com/dillonkearns/incrementalelm.com/blob/d4934d899d06232dc66dcf9f4b5eccc74bbc60d3/src/Api.elm#L51-L60) ([Output file](https://incrementalelm.com/live.ics))
+  - A redirect file for a hosting provider like Netlify
+
+You could even generate a JavaScript file, an Elm file, or any file with a String body! It's really just a way to generate files, which are typically used to serve files to a user or Browser, but you execute them, copy them, etc. The only limit is your imagination!
+The beauty is that you have a way to 1) pull in type-safe data using DataSource's, and 2) write those files, and all in pure Elm!
+
 @docs single, preRender
 
 
 ## Server Rendering
 
-@docs preRenderWithFallback, serverRender
+You could use server-rendered ApiRoutes to do a lot of similar things, the main difference being that it will be served up through a URL and generated on-demand when that URL is requested.
+So for example, for an RSS feed or ical calendar feed like in the pre-rendered examples, you could build the same routes, but you would be pulling in the list of posts or calendar events on-demand rather
+than upfront at build-time. That means you can hit your database and serve up always-up-to-date data.
+
+Not only that, but your server-rendered ApiRoutes have access to the incoming HTTP request payload just like your server-rendered Route Modules do. Just as with server-rendered Route Modules,
+a server-rendered ApiRoute accesses the incoming HTTP request through a [Server.Request.Parser](Server-Request). Consider the use cases that this opens up:
+
+  - Serve up protected assets. For example, gated content, like a paid subscriber feed for a podcast that checks authentication information in a query parameter to authenticate that a user has an active paid subscription before serving up the Pro RSS feed.
+  - Serve up user-specific content, either through a cookie or other means of authentication
+  - Look at the [accepted content-type in the request headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept) and use that to choose a response format, like XML or JSON ([full example](https://github.com/dillonkearns/elm-pages/blob/131f7b750cdefb2ba7a34a06be06dfbfafc79a86/examples/end-to-end/app/Api.elm#L76-L107)).
+  - Look at the [accepted language in the request headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) and use that to choose a language for the response data.
+
+@docs serverRender
+
+You can also do a hybrid approach using `preRenderWithFallback`. This allows you to pre-render a set of routes at build-time, but build additional routes that weren't rendered at build-time on the fly on the server.
+Conceptually, this is just a delayed version of a pre-rendered route. Because of that, you _do not_ have access to the incoming HTTP request (no `Server.Request.Parser` like in server-rendered ApiRoute's).
+The strategy used to build these routes will differ depending on your hosting provider and the elm-pages adapter you have setup, but generally ApiRoute's that use `preRenderWithFallback` will be cached on the server
+so within a certain time interval (or in the case of [Netlify's DPR](https://www.netlify.com/blog/2021/04/14/distributed-persistent-rendering-a-new-jamstack-approach-for-faster-builds/), until a new build is done)
+that asset will be served up if that URL was already served up by the server.
+
+@docs preRenderWithFallback
+
+
+## Defining ApiRoute's
+
+You define your ApiRoute's in `app/Api.elm`. Here's a simple example:
+
+    module Api exposing (routes)
+
+    import ApiRoute
+    import DataSource exposing (DataSource)
+    import Server.Request
+
+    routes :
+        DataSource (List Route)
+        -> (Maybe { indent : Int, newLines : Bool } -> Html Never -> String)
+        -> List (ApiRoute.ApiRoute ApiRoute.Response)
+    routes getStaticRoutes htmlToString =
+        [ preRenderedExample
+        , requestPrinterExample
+        ]
+
+    {-| Generates the following files when you
+    run `elm-pages build`:
+
+      - `dist/users/1.json`
+      - `dist/users/2.json`
+      - `dist/users/3.json`
+
+    When you host it, these static assets will
+    be served at `/users/1.json`, etc.
+
+    -}
+    preRenderedExample : ApiRoute.ApiRoute ApiRoute.Response
+    preRenderedExample =
+        ApiRoute.succeed
+            (\userId ->
+                DataSource.succeed
+                    (Json.Encode.object
+                        [ ( "id", Json.Encode.string userId )
+                        , ( "name", "Data for user " ++ userId |> Json.Encode.string )
+                        ]
+                        |> Json.Encode.encode 2
+                    )
+            )
+            |> ApiRoute.literal "users"
+            |> ApiRoute.slash
+            |> ApiRoute.capture
+            |> ApiRoute.literal ".json"
+            |> ApiRoute.preRender
+                (\route ->
+                    DataSource.succeed
+                        [ route "1"
+                        , route "2"
+                        , route "3"
+                        ]
+                )
+
+    {-| This returns a JSON response that prints information about the incoming
+    HTTP request. In practice you'd want to do something useful with that data,
+    and use more of the high-level helpers from the Server.Request API.
+    -}
+    requestPrinterExample : ApiRoute ApiRoute.Response
+    requestPrinterExample =
+        ApiRoute.succeed
+            (Server.Request.map4
+                (\rawBody method cookies queryParams ->
+                    Encode.object
+                        [ ( "rawBody"
+                          , rawBody
+                                |> Maybe.map Encode.string
+                                |> Maybe.withDefault Encode.null
+                          )
+                        , ( "method"
+                          , method
+                                |> Server.Request.methodToString
+                                |> Encode.string
+                          )
+                        , ( "cookies"
+                          , cookies
+                                |> Encode.dict
+                                    identity
+                                    Encode.string
+                          )
+                        , ( "queryParams"
+                          , queryParams
+                                |> Encode.dict
+                                    identity
+                                    (Encode.list Encode.string)
+                          )
+                        ]
+                        |> Response.json
+                        |> DataSource.succeed
+                )
+                Server.Request.rawBody
+                Server.Request.method
+                Server.Request.allCookies
+                Server.Request.queryParams
+            )
+            |> ApiRoute.literal "api"
+            |> ApiRoute.slash
+            |> ApiRoute.literal "request-test"
+            |> ApiRoute.serverRender
+
+@docs ApiRoute, ApiRouteBuilder, Response
+
+@docs capture, literal, slash, succeed
 
 
 ## Including Head Tags
@@ -54,7 +189,9 @@ type alias ApiRoute response =
     Internal.ApiRoute.ApiRoute response
 
 
-{-| -}
+{-| Same as [`preRender`](#preRender), but for an ApiRoute that has no dynamic segments. This is just a bit simpler because
+since there are no dynamic segments, you don't need to provide a DataSource with the list of dynamic segments to pre-render because there is only a single possible route.
+-}
 single : ApiRouteBuilder (DataSource String) (List String) -> ApiRoute Response
 single handler =
     handler
@@ -235,7 +372,8 @@ toJson ((ApiRoute { kind }) as apiRoute) =
         ]
 
 
-{-| -}
+{-| A literal String segment of a route.
+-}
 literal : String -> ApiRouteBuilder a constructor -> ApiRouteBuilder a constructor
 literal segment (ApiRouteBuilder patterns pattern handler toString constructor) =
     ApiRouteBuilder

package/src/DataSource/Env.elm

@@ -1,6 +1,28 @@
 module DataSource.Env exposing (get, expect)
 
-{-|
+{-| Because DataSource's in `elm-pages` never run in the browser (see [the DataSource docs](DataSource)), you can access environment variables securely. As long as the environment variable isn't sent
+down into the final `Data` value, it won't end up in the client!
+
+    import DataSource exposing (DataSource)
+    import DataSource.Env
+
+    type alias EnvVariables =
+        { sendGridKey : String
+        , siteUrl : String
+        }
+
+    sendEmail : Email -> DataSource ()
+    sendEmail email =
+        DataSource.map2 EnvVariables
+            (DataSource.Env.expect "SEND_GRID_KEY")
+            (DataSource.Env.get "BASE_URL"
+                |> DataSource.map (Maybe.withDefault "http://localhost:1234")
+            )
+            |> DataSource.andThen (sendEmailDataSource email)
+
+    sendEmailDataSource : Email -> EnvVariables -> DataSource ()
+    sendEmailDataSource email envVariables =
+        Debug.todo "Not defined here"
 
 @docs get, expect
 
@@ -13,7 +35,8 @@ import Json.Decode as Decode
 import Json.Encode as Encode
 
 
-{-| -}
+{-| Get an environment variable, or Nothing if there is no environment variable matching that name.
+-}
 get : String -> DataSource (Maybe String)
 get envVariableName =
     DataSource.Internal.Request.request
@@ -25,7 +48,8 @@ get envVariableName =
         }
 
 
-{-| -}
+{-| Get an environment variable, or a DataSource failure if there is no environment variable matching that name.
+-}
 expect : String -> DataSource String
 expect envVariableName =
     envVariableName