elm-pages 2.1.11 → 3.0.0-beta.1

package/generator/src/render.js

@@ -1,12 +1,14 @@
 // @ts-check
 
 const path = require("path");
+const mm = require("micromatch");
 const matter = require("gray-matter");
 const globby = require("globby");
 const fsPromises = require("fs").promises;
 const preRenderHtml = require("./pre-render-html.js");
 const { lookupOrPerform } = require("./request-cache.js");
 const kleur = require("kleur");
+const cookie = require("cookie-signature");
 kleur.enabled = true;
 
 process.on("unhandledRejection", (error) => {
@@ -22,31 +24,41 @@ module.exports =
    * @param {string} basePath
    * @param {Object} elmModule
    * @param {string} path
-   * @param {import('aws-lambda').APIGatewayProxyEvent} request
+   * @param {{ method: string; hostname: string; query: Record<string, string | undefined>; headers: Record<string, string>; host: string; pathname: string; port: number | null; protocol: string; rawUrl: string; }} request
    * @param {(pattern: string) => void} addDataSourceWatcher
+   * @param {boolean} hasFsAccess
    * @returns
    */
   async function run(
+    portsFile,
     basePath,
     elmModule,
     mode,
     path,
     request,
-    addDataSourceWatcher
+    addDataSourceWatcher,
+    hasFsAccess
   ) {
+    const { fs, resetInMemoryFs } = require("./request-cache-fs.js")(
+      hasFsAccess
+    );
+    resetInMemoryFs();
     foundErrors = false;
     pendingDataSourceResponses = [];
     pendingDataSourceCount = 0;
-    // since init/update are never called in pre-renders, and DataSource.Http is called using undici
+    // since init/update are never called in pre-renders, and DataSource.Http is called using pure NodeJS HTTP fetching
     // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
     XMLHttpRequest = {};
     const result = await runElmApp(
+      portsFile,
       basePath,
       elmModule,
       mode,
       path,
       request,
-      addDataSourceWatcher
+      addDataSourceWatcher,
+      fs,
+      hasFsAccess
     );
     return result;
   };
@@ -56,49 +68,61 @@ module.exports =
  * @param {Object} elmModule
  * @param {string} pagePath
  * @param {string} mode
- * @param {import('aws-lambda').APIGatewayProxyEvent} request
+ * @param {{ method: string; hostname: string; query: string; headers: Object; host: string; pathname: string; port: string; protocol: string; rawUrl: string; }} request
  * @param {(pattern: string) => void} addDataSourceWatcher
  * @returns {Promise<({is404: boolean} & ( { kind: 'json'; contentJson: string} | { kind: 'html'; htmlString: string } | { kind: 'api-response'; body: string; }) )>}
  */
 function runElmApp(
+  portsFile,
   basePath,
   elmModule,
   mode,
   pagePath,
   request,
-  addDataSourceWatcher
+  addDataSourceWatcher,
+  fs,
+  hasFsAccess
 ) {
   const isDevServer = mode !== "build";
   let patternsToWatch = new Set();
   let app = null;
   let killApp;
   return new Promise((resolve, reject) => {
-    const isJson = pagePath.match(/content\.json\/?$/);
-    const route = pagePath.replace(/content\.json\/?$/, "");
+    const isBytes = pagePath.match(/content\.dat\/?$/);
+    const route = pagePath
+      .replace(/content\.json\/?$/, "")
+      .replace(/content\.dat\/?$/, "");
 
     const modifiedRequest = { ...request, path: route };
     // console.log("StaticHttp cache keys", Object.keys(global.staticHttpCache));
-    app = elmModule.Elm.TemplateModulesBeta.init({
+    app = elmModule.Elm.Main.init({
       flags: {
-        secrets: process.env,
-        staticHttpCache: global.staticHttpCache || {},
         mode,
         request: {
           payload: modifiedRequest,
           kind: "single-page",
-          jsonOnly: !!isJson,
+          jsonOnly: !!isBytes,
         },
       },
     });
 
     killApp = () => {
       app.ports.toJsPort.unsubscribe(portHandler);
+      app.ports.sendPageData.unsubscribe(portHandler);
       app.die();
       app = null;
       // delete require.cache[require.resolve(compiledElmPath)];
     };
 
-    async function portHandler(/** @type { FromElm }  */ fromElm) {
+    async function portHandler(/** @type { FromElm }  */ newThing) {
+      let fromElm;
+      let contentDatPayload;
+      if ("oldThing" in newThing) {
+        fromElm = newThing.oldThing;
+        contentDatPayload = newThing.binaryPageData;
+      } else {
+        fromElm = newThing;
+      }
       if (fromElm.command === "log") {
         console.log(fromElm.value);
       } else if (fromElm.tag === "ApiResponse") {
@@ -119,51 +143,63 @@ function runElmApp(
           global.staticHttpCache = args.staticHttpCache;
         }
 
-        if (isJson) {
+        if (isBytes) {
           resolve({
-            kind: "json",
-            is404: args.is404,
+            kind: "bytes",
+            is404: false,
             contentJson: JSON.stringify({
               staticData: args.contentJson,
-              is404: args.is404,
+              is404: false,
             }),
+            statusCode: args.statusCode,
+            headers: args.headers,
+            contentDatPayload,
           });
         } else {
-          resolve(outputString(basePath, fromElm, isDevServer));
-        }
-      } else if (fromElm.tag === "ReadFile") {
-        const filePath = fromElm.args[0];
-        try {
-          patternsToWatch.add(filePath);
-
-          runJob(app, filePath);
-        } catch (error) {
-          sendError(app, {
-            title: "DataSource.File Error",
-            message: `A DataSource.File read failed because I couldn't find this file: ${kleur.yellow(
-              filePath
-            )}`,
-          });
+          resolve(
+            outputString(basePath, fromElm, isDevServer, contentDatPayload)
+          );
         }
       } else if (fromElm.tag === "DoHttp") {
         const requestToPerform = fromElm.args[0];
-        runHttpJob(app, mode, requestToPerform);
-      } else if (fromElm.tag === "Glob") {
-        const globPattern = fromElm.args[0];
-        patternsToWatch.add(globPattern);
-        runGlobJob(app, globPattern);
+        if (
+          requestToPerform.url !== "elm-pages-internal://port" &&
+          requestToPerform.url.startsWith("elm-pages-internal://")
+        ) {
+          runInternalJob(
+            app,
+            mode,
+            requestToPerform,
+            fs,
+            hasFsAccess,
+            patternsToWatch
+          );
+        } else {
+          runHttpJob(
+            portsFile,
+            app,
+            mode,
+            requestToPerform,
+            fs,
+            hasFsAccess,
+            fromElm.args[1]
+          );
+        }
       } else if (fromElm.tag === "Errors") {
         foundErrors = true;
-        reject(fromElm.args[0]);
+        reject(fromElm.args[0].errorsJson);
       } else {
         console.log(fromElm);
       }
     }
     app.ports.toJsPort.subscribe(portHandler);
+    app.ports.sendPageData.subscribe(portHandler);
   }).finally(() => {
     addDataSourceWatcher(patternsToWatch);
-    killApp();
-    killApp = null;
+    try {
+      killApp();
+      killApp = null;
+    } catch (error) {}
   });
 }
 /**
@@ -174,21 +210,27 @@ function runElmApp(
 async function outputString(
   basePath,
   /** @type { PageProgress } */ fromElm,
-  isDevServer
+  isDevServer,
+  contentDatPayload
 ) {
   const args = fromElm.args[0];
   let contentJson = {};
   contentJson["staticData"] = args.contentJson;
   contentJson["is404"] = args.is404;
   contentJson["path"] = args.route;
+  contentJson["statusCode"] = args.statusCode;
+  contentJson["headers"] = args.headers;
   const normalizedRoute = args.route.replace(/index$/, "");
 
   return {
     is404: args.is404,
     route: normalizedRoute,
-    htmlString: preRenderHtml(basePath, args, contentJson, isDevServer),
+    htmlString: preRenderHtml.wrapHtml(basePath, args, contentDatPayload),
     contentJson: args.contentJson,
+    statusCode: args.statusCode,
+    headers: args.headers,
     kind: "html",
+    contentDatPayload,
   };
 }
 
@@ -201,73 +243,30 @@ async function outputString(
 
 /** @typedef { { head: any[]; errors: any[]; contentJson: any[]; html: string; route: string; title: string; } } Arg */
 
-/**
- * @param {string} string
- */
-function jsonOrNull(string) {
-  try {
-    return JSON.parse(string);
-  } catch (e) {
-    return { invalidJson: e.toString() };
-  }
-}
-
-async function runJob(app, filePath) {
-  pendingDataSourceCount += 1;
-  try {
-    const fileContents = (
-      await fsPromises.readFile(path.join(process.cwd(), filePath))
-    ).toString();
-    const parsedFile = matter(fileContents);
-
-    pendingDataSourceResponses.push({
-      request: {
-        masked: {
-          url: `file://${filePath}`,
-          method: "GET",
-          headers: [],
-          body: { tag: "EmptyBody", args: [] },
-        },
-        unmasked: {
-          url: `file://${filePath}`,
-          method: "GET",
-          headers: [],
-          body: { tag: "EmptyBody", args: [] },
-        },
-      },
-      response: JSON.stringify({
-        parsedFrontmatter: parsedFile.data,
-        withoutFrontmatter: parsedFile.content,
-        rawFile: fileContents,
-        jsonFile: jsonOrNull(fileContents),
-      }),
-    });
-  } catch (e) {
-    sendError(app, {
-      title: "Error reading file",
-      message: `A DataSource.File read failed because I couldn't find this file: ${kleur.yellow(
-        filePath
-      )}`,
-    });
-  } finally {
-    pendingDataSourceCount -= 1;
-    flushIfDone(app);
-  }
-}
-
-async function runHttpJob(app, mode, requestToPerform) {
+async function runHttpJob(
+  portsFile,
+  app,
+  mode,
+  requestToPerform,
+  fs,
+  hasFsAccess,
+  useCache
+) {
   pendingDataSourceCount += 1;
   try {
     const responseFilePath = await lookupOrPerform(
+      portsFile,
       mode,
-      requestToPerform.unmasked
+      requestToPerform,
+      hasFsAccess,
+      useCache
     );
 
     pendingDataSourceResponses.push({
       request: requestToPerform,
-      response: (
-        await fsPromises.readFile(responseFilePath, "utf8")
-      ).toString(),
+      response: JSON.parse(
+        (await fs.promises.readFile(responseFilePath, "utf8")).toString()
+      ),
     });
   } catch (error) {
     sendError(app, error);
@@ -277,129 +276,173 @@ async function runHttpJob(app, mode, requestToPerform) {
   }
 }
 
-async function runGlobJob(app, globPattern) {
+function stringResponse(request, string) {
+  return {
+    request,
+    response: { bodyKind: "string", body: string },
+  };
+}
+function jsonResponse(request, json) {
+  return {
+    request,
+    response: { bodyKind: "json", body: json },
+  };
+}
+
+async function runInternalJob(
+  app,
+  mode,
+  requestToPerform,
+  fs,
+  hasFsAccess,
+  patternsToWatch
+) {
   try {
-    // if (pendingDataSourceCount > 0) {
-    //   console.log(`Waiting for ${pendingDataSourceCount} pending data sources`);
-    // }
     pendingDataSourceCount += 1;
 
-    pendingDataSourceResponses.push(await globTask(globPattern));
+    if (requestToPerform.url === "elm-pages-internal://read-file") {
+      pendingDataSourceResponses.push(
+        await readFileJobNew(requestToPerform, patternsToWatch)
+      );
+    } else if (requestToPerform.url === "elm-pages-internal://glob") {
+      pendingDataSourceResponses.push(
+        await runGlobNew(requestToPerform, patternsToWatch)
+      );
+    } else if (requestToPerform.url === "elm-pages-internal://env") {
+      pendingDataSourceResponses.push(
+        await runEnvJob(requestToPerform, patternsToWatch)
+      );
+    } else if (requestToPerform.url === "elm-pages-internal://encrypt") {
+      pendingDataSourceResponses.push(
+        await runEncryptJob(requestToPerform, patternsToWatch)
+      );
+    } else if (requestToPerform.url === "elm-pages-internal://decrypt") {
+      pendingDataSourceResponses.push(
+        await runDecryptJob(requestToPerform, patternsToWatch)
+      );
+    } else {
+      throw `Unexpected internal DataSource request format: ${kleur.yellow(
+        JSON.stringify(2, null, requestToPerform)
+      )}`;
+    }
   } catch (error) {
-    console.log(`Error running glob pattern ${globPattern}`);
-    throw error;
+    sendError(app, error);
   } finally {
     pendingDataSourceCount -= 1;
     flushIfDone(app);
   }
 }
 
-function flushIfDone(app) {
-  if (foundErrors) {
-    pendingDataSourceResponses = [];
-  } else if (pendingDataSourceCount === 0) {
-    // console.log(
-    //   `Flushing ${pendingDataSourceResponses.length} items in ${timeUntilThreshold}ms`
-    // );
+async function readFileJobNew(req, patternsToWatch) {
+  const filePath = req.body.args[1];
+  try {
+    patternsToWatch.add(filePath);
 
-    flushQueue(app);
-  }
-}
+    const fileContents = // TODO can I remove this hack?
+      (
+        await fsPromises.readFile(
+          path.join(process.env.LAMBDA_TASK_ROOT || process.cwd(), filePath)
+        )
+      ).toString();
+    const parsedFile = matter(fileContents);
 
-function flushQueue(app) {
-  const temp = pendingDataSourceResponses;
-  pendingDataSourceResponses = [];
-  // console.log("@@@ FLUSHING", temp.length);
-  app.ports.fromJsPort.send({
-    tag: "GotBatch",
-    data: temp,
-  });
+    return jsonResponse(req, {
+      parsedFrontmatter: parsedFile.data,
+      withoutFrontmatter: parsedFile.content,
+      rawFile: fileContents,
+    });
+  } catch (error) {
+    throw {
+      title: "DataSource.File Error",
+      message: `A DataSource.File read failed because I couldn't find this file: ${kleur.yellow(
+        filePath
+      )}\n${kleur.red(error.toString())}`,
+    };
+  }
 }
 
-/**
- * @param {string} filePath
- * @returns {Promise<Object>}
- */
-async function readFileTask(app, filePath) {
-  // console.log(`Read file ${filePath}`);
+async function runGlobNew(req, patternsToWatch) {
   try {
-    const fileContents = (
-      await fsPromises.readFile(path.join(process.cwd(), filePath))
-    ).toString();
-    // console.log(`DONE reading file ${filePath}`);
-    const parsedFile = matter(fileContents);
+    const { pattern, options } = req.body.args[0];
+    const matchedPaths = await globby(pattern, options);
+    patternsToWatch.add(pattern);
 
-    return {
-      request: {
-        masked: {
-          url: `file://${filePath}`,
-          method: "GET",
-          headers: [],
-          body: { tag: "EmptyBody", args: [] },
-        },
-        unmasked: {
-          url: `file://${filePath}`,
-          method: "GET",
-          headers: [],
-          body: { tag: "EmptyBody", args: [] },
-        },
-      },
-      response: JSON.stringify({
-        parsedFrontmatter: parsedFile.data,
-        withoutFrontmatter: parsedFile.content,
-        rawFile: fileContents,
-        jsonFile: jsonOrNull(fileContents),
-      }),
-    };
+    return jsonResponse(
+      req,
+      matchedPaths.map((fullPath) => {
+        return {
+          fullPath,
+          captures: mm.capture(pattern, fullPath),
+        };
+      })
+    );
   } catch (e) {
-    sendError(app, {
-      title: "Error reading file",
-      message: `A DataSource.File read failed because I couldn't find this file: ${kleur.yellow(
-        filePath
-      )}`,
-    });
+    console.log(`Error performing glob '${JSON.stringify(req.body)}'`);
+    throw e;
   }
 }
 
-/**
- * @param {string} globPattern
- * @returns {Promise<Object>}
- */
-async function globTask(globPattern) {
+async function runEnvJob(req, patternsToWatch) {
   try {
-    const matchedPaths = await globby(globPattern);
-    // console.log("Got glob path", matchedPaths);
-
-    return {
-      request: {
-        masked: {
-          url: `glob://${globPattern}`,
-          method: "GET",
-          headers: [],
-          body: { tag: "EmptyBody", args: [] },
-        },
-        unmasked: {
-          url: `glob://${globPattern}`,
-          method: "GET",
-          headers: [],
-          body: { tag: "EmptyBody", args: [] },
-        },
-      },
-      response: JSON.stringify(matchedPaths),
-    };
+    const expectedEnv = req.body.args[0];
+    return jsonResponse(req, process.env[expectedEnv] || null);
   } catch (e) {
-    console.log(`Error performing glob '${globPattern}'`);
+    console.log(`Error performing env '${JSON.stringify(req.body)}'`);
     throw e;
   }
 }
+async function runEncryptJob(req, patternsToWatch) {
+  try {
+    return jsonResponse(
+      req,
+      cookie.sign(
+        JSON.stringify(req.body.args[0].values, null, 0),
+        req.body.args[0].secret
+      )
+    );
+  } catch (e) {
+    throw {
+      title: "DataSource Encrypt Error",
+      message:
+        e.toString() + e.stack + "\n\n" + JSON.stringify(rawRequest, null, 2),
+    };
+  }
+}
+async function runDecryptJob(req, patternsToWatch) {
+  try {
+    // TODO if unsign returns `false`, need to have an `Err` in Elm because decryption failed
+    const signed = tryDecodeCookie(
+      req.body.args[0].input,
+      req.body.args[0].secrets
+    );
+
+    return jsonResponse(req, JSON.parse(signed || "null"));
+  } catch (e) {
+    throw {
+      title: "DataSource Decrypt Error",
+      message:
+        e.toString() + e.stack + "\n\n" + JSON.stringify(rawRequest, null, 2),
+    };
+  }
+}
+
+function flushIfDone(app) {
+  if (foundErrors) {
+    pendingDataSourceResponses = [];
+  } else if (pendingDataSourceCount === 0) {
+    // console.log(
+    //   `Flushing ${pendingDataSourceResponses.length} items in ${timeUntilThreshold}ms`
+    // );
 
-function requireUncached(mode, filePath) {
-  if (mode === "dev-server") {
-    // for the build command, we can skip clearing the cache because it won't change while the build is running
-    // in the dev server, we want to clear the cache to get a the latest code each time it runs
-    delete require.cache[require.resolve(filePath)];
+    flushQueue(app);
   }
-  return require(filePath);
+}
+
+function flushQueue(app) {
+  const temp = pendingDataSourceResponses;
+  pendingDataSourceResponses = [];
+  // console.log("@@@ FLUSHING", temp.length);
+  app.ports.gotBatchSub.send(temp);
 }
 
 /**
@@ -414,3 +457,15 @@ function sendError(app, error) {
     data: error,
   });
 }
+function tryDecodeCookie(input, secrets) {
+  if (secrets.length > 0) {
+    const signed = cookie.unsign(input, secrets[0]);
+    if (signed) {
+      return signed;
+    } else {
+      return tryDecodeCookie(input, secrets.slice(1));
+    }
+  } else {
+    return null;
+  }
+}

package/generator/src/request-cache-fs.js

@@ -0,0 +1,18 @@
+module.exports = function (/** @type {boolean} */ hasFsAccess) {
+  if (hasFsAccess) {
+    return {
+      fs: require("fs"),
+      resetInMemoryFs: () => {},
+    };
+  } else {
+    const { vol, fs, Volume } = require("memfs");
+    vol.fromJSON({});
+    //     vol.reset();
+    return {
+      fs: fs,
+      resetInMemoryFs: () => {
+        vol.reset();
+      },
+    };
+  }
+};