@netlify/plugin-nextjs 4.0.0-beta.2 → 4.0.0-beta.6

package/README.md

@@ -14,6 +14,9 @@
   </a>
 </p>
 
+## What's new in this version
+
+Version 4 is a complete rewrite of the Essential Next.js plugin. For full details of everything that's new, see [the v4 release notes](https://github.com/netlify/netlify-plugin-nextjs/blob/main/docs/release-notes/v4.md)
 
 ## Installing the beta
 
@@ -31,7 +34,13 @@ publish = ".next"
 package = "@netlify/plugin-nextjs"
 ```
 
-If you previously set `target: "serverless"` in your `next.config.js` this is no longer needed and can be removed.
+If you previously set `target: "serverless"` or a custom `distDir` in your `next.config.js`, or set `node_bundler` or `external_node_modules` in your `netlify.toml` these are no longer needed and can be removed. 
+
+The `serverless` and `experimental-serverless-trace` targets are deprecated in Next 12, and all builds with this plugin will now use the default `server` target.
+
+If you are using a monorepo you will need to change `publish` to point to the full path to the built `.next` directory, which may be in a subdirectory. If you have changed your `distDir` then it will need to match that. 
+
+If you are using Nx, then you will need to point `publish` to the folder inside `dist`, e.g. `dist/apps/myapp/.next`. 
 
 ## Beta feedback
 

package/lib/helpers/config.js

@@ -1,6 +1,7 @@
 // @ts-check
-const { readJSON } = require('fs-extra');
+const { readJSON, existsSync } = require('fs-extra');
 const { join, dirname, relative } = require('pathe');
+const slash = require('slash');
 const defaultFailBuild = (message, { error }) => {
     throw new Error(`${message}\n${error && error.stack}`);
 };
@@ -58,12 +59,23 @@ exports.generateRedirects = async ({ netlifyConfig, basePath, i18n }) => {
     if (i18n) {
         netlifyConfig.redirects.push({ from: `${basePath}/:locale/_next/static/*`, to: `/static/:splat`, status: 200 });
     }
+    const staticManifest = join(netlifyConfig.build.publish, 'static-manifest.json');
+    if (process.env.EXPERIMENTAL_MOVE_STATIC_PAGES && existsSync(staticManifest)) {
+        // Static page files need to have a forced redirect for preview mode. Otherwise it's non-forced
+        const staticFiles = await readJSON(staticManifest);
+        netlifyConfig.redirects.push(...staticFiles.map((file) => ({
+            from: `${basePath}/${file}`,
+            to: HANDLER_FUNCTION_PATH,
+            status: 200,
+            force: true,
+            conditions: { Cookie: ['__prerender_bypass', '__next_preview_data'] },
+        })));
+    }
     // This is only used in prod, so dev uses `next dev` directly
     netlifyConfig.redirects.push({ from: `${basePath}/_next/static/*`, to: `/static/:splat`, status: 200 }, {
         from: `${basePath}/*`,
         to: HANDLER_FUNCTION_PATH,
         status: 200,
-        force: true,
         conditions: { Cookie: ['__prerender_bypass', '__next_preview_data'] },
     }, ...redirects.map((redirect) => ({
         from: `${basePath}${redirect}`,
@@ -91,24 +103,27 @@ const resolveModuleRoot = (moduleName) => {
         return null;
     }
 };
+const DEFAULT_EXCLUDED_MODULES = ['sharp', 'electron'];
 exports.configureHandlerFunctions = ({ netlifyConfig, publish, ignore = [] }) => {
     var _a;
     /* eslint-disable no-underscore-dangle */
     (_a = netlifyConfig.functions)._ipx || (_a._ipx = {});
-    netlifyConfig.functions._ipx.node_bundler = 'esbuild';
+    netlifyConfig.functions._ipx.node_bundler = 'nft';
     [HANDLER_FUNCTION_NAME, ODB_FUNCTION_NAME].forEach((functionName) => {
         var _a, _b;
         (_a = netlifyConfig.functions)[functionName] || (_a[functionName] = { included_files: [], external_node_modules: [] });
         netlifyConfig.functions[functionName].node_bundler = 'nft';
         (_b = netlifyConfig.functions[functionName]).included_files || (_b.included_files = []);
-        netlifyConfig.functions[functionName].included_files.push(`${publish}/server/**`, `${publish}/serverless/**`, `${publish}/*.json`, `${publish}/BUILD_ID`, ...ignore.map((path) => `!${path}`));
+        netlifyConfig.functions[functionName].included_files.push(`${publish}/server/**`, `${publish}/serverless/**`, `${publish}/*.json`, `${publish}/BUILD_ID`, `${publish}/static/chunks/webpack-middleware*.js`, `!${publish}/server/**/*.js.nft.json`, ...ignore.map((path) => `!${slash(path)}`));
         const nextRoot = resolveModuleRoot('next');
         if (nextRoot) {
             netlifyConfig.functions[functionName].included_files.push(`!${nextRoot}/dist/server/lib/squoosh/**/*.wasm`, `!${nextRoot}/dist/next-server/server/lib/squoosh/**/*.wasm`, `!${nextRoot}/dist/compiled/webpack/bundle4.js`, `!${nextRoot}/dist/compiled/webpack/bundle5.js`, `!${nextRoot}/dist/compiled/terser/bundle.min.js`);
         }
-        const sharpRoot = resolveModuleRoot('sharp');
-        if (sharpRoot) {
-            netlifyConfig.functions[functionName].included_files.push(`!${sharpRoot}/**/*`);
-        }
+        DEFAULT_EXCLUDED_MODULES.forEach((moduleName) => {
+            const moduleRoot = resolveModuleRoot(moduleName);
+            if (moduleRoot) {
+                netlifyConfig.functions[functionName].included_files.push(`!${moduleRoot}/**/*`);
+            }
+        });
     });
 };

package/lib/helpers/files.js

@@ -0,0 +1,51 @@
+//  @ts-check
+const { cpus } = require('os');
+const { existsSync, readJson, move, cpSync, copy, writeJson } = require('fs-extra');
+const globby = require('globby');
+const pLimit = require('p-limit');
+const { join } = require('pathe');
+const slash = require('slash');
+const TEST_ROUTE = /(|\/)\[[^/]+?](\/|\.html|$)/;
+const isDynamicRoute = (route) => TEST_ROUTE.test(route);
+exports.moveStaticPages = async ({ netlifyConfig, target, i18n, failBuild }) => {
+    console.log('Moving static page files to serve from CDN...');
+    const root = join(netlifyConfig.build.publish, target === 'server' ? 'server' : 'serverless', 'pages');
+    const files = [];
+    const moveFile = async (file) => {
+        const source = join(root, file);
+        files.push(file);
+        const dest = join(netlifyConfig.build.publish, file);
+        await move(source, dest);
+    };
+    // Move all static files, except error documents and nft manifests
+    const pages = await globby(['**/*.{html,json}', '!**/(500|404|*.js.nft).{html,json}'], {
+        cwd: root,
+        dot: true,
+    });
+    // Limit concurrent file moves to number of cpus or 2 if there is only 1
+    const limit = pLimit(Math.max(2, cpus().length));
+    const promises = pages.map(async (rawPath) => {
+        const filePath = slash(rawPath);
+        if (isDynamicRoute(filePath)) {
+            return;
+        }
+        return limit(moveFile, filePath);
+    });
+    await Promise.all(promises);
+    console.log(`Moved ${files.length} files`);
+    // Write the manifest for use in the serverless functions
+    await writeJson(join(netlifyConfig.build.publish, 'static-manifest.json'), files);
+    if (i18n === null || i18n === void 0 ? void 0 : i18n.defaultLocale) {
+        // Copy the default locale into the root
+        const defaultLocaleDir = join(netlifyConfig.build.publish, i18n.defaultLocale);
+        if (existsSync(defaultLocaleDir)) {
+            await copy(defaultLocaleDir, `${netlifyConfig.build.publish}/`);
+        }
+    }
+};
+exports.movePublicFiles = async ({ appDir, publish }) => {
+    const publicDir = join(appDir, 'public');
+    if (existsSync(publicDir)) {
+        await copy(publicDir, `${publish}/`);
+    }
+};

package/lib/helpers/functions.js

@@ -1,5 +1,5 @@
-const { join, relative } = require('path');
 const { copyFile, ensureDir, writeFile, writeJSON } = require('fs-extra');
+const { join, relative } = require('pathe');
 const { HANDLER_FUNCTION_NAME, ODB_FUNCTION_NAME, IMAGE_FUNCTION_NAME } = require('../constants');
 const getHandler = require('../templates/getHandler');
 const { getPageResolver } = require('../templates/getPageResolver');

package/lib/helpers/verification.js

@@ -1,13 +1,11 @@
+const { existsSync, promises } = require('fs');
 const path = require('path');
-const { yellowBright, greenBright, blueBright } = require('chalk');
-const { existsSync } = require('fs-extra');
+const { relative } = require('path');
+const { yellowBright, greenBright, blueBright, redBright } = require('chalk');
+const { async: StreamZip } = require('node-stream-zip');
 const outdent = require('outdent');
+const prettyBytes = require('pretty-bytes');
 const { satisfies } = require('semver');
-exports.verifyBuildTarget = (target) => {
-    if (target !== 'server') {
-        console.log(yellowBright `Setting target to ${target} is no longer required. You should check if target=server works for you.`);
-    }
-};
 // This is when nft support was added
 const REQUIRED_BUILD_VERSION = '>=18.16.0';
 exports.verifyNetlifyBuildVersion = ({ IS_LOCAL, NETLIFY_BUILD_VERSION, failBuild }) => {
@@ -41,7 +39,41 @@ exports.checkForRootPublish = ({ publish, failBuild }) => {
       In most cases it should be set to ".next", unless you have chosen a custom "distDir" in your Next config, or the Next site is in a subdirectory.`);
     }
 };
+// 50MB, which is the documented max, though the hard max seems to be higher
+const LAMBDA_MAX_SIZE = 1024 * 1024 * 50;
+exports.checkZipSize = async (file, maxSize = LAMBDA_MAX_SIZE) => {
+    if (!existsSync(file)) {
+        console.warn(`Could not check zip size because ${file} does not exist`);
+        return;
+    }
+    const size = await promises.stat(file).then(({ size }) => size);
+    if (size < maxSize) {
+        return;
+    }
+    // We don't fail the build, because the actual hard max size is larger so it might still succeed
+    console.log(redBright(outdent `
+
+    The function zip ${yellowBright(relative(process.cwd(), file))} size is ${prettyBytes(size)}, which is larger than the maximum supported size of ${prettyBytes(maxSize)}.
+    There are a few reasons this could happen. You may have accidentally bundled a large dependency, or you might have a
+    large number of pre-rendered pages included.
+  
+    `));
+    const zip = new StreamZip({ file });
+    console.log(`Contains ${await zip.entriesCount} files`);
+    const sortedFiles = Object.values(await zip.entries()).sort((a, b) => b.size - a.size);
+    const largest = {};
+    for (let i = 0; i < 10 && i < sortedFiles.length; i++) {
+        largest[`${i + 1}`] = {
+            File: sortedFiles[i].name,
+            'Compressed Size': prettyBytes(sortedFiles[i].compressedSize),
+            'Uncompressed Size': prettyBytes(sortedFiles[i].size),
+        };
+    }
+    console.log(yellowBright `\n\nThese are the largest files in the zip:`);
+    console.table(largest);
+    console.log(greenBright `\n\nFor more information on fixing this, see ${blueBright `https://ntl.fyi/large-next-functions`}`);
+};
 exports.logBetaMessage = () => console.log(greenBright(outdent `
         Thank you for trying the Essential Next.js beta plugin. 
-        Please share feedback at ${blueBright `https://ntl.fyi/next-beta-feedback`}
+        Please share feedback (both good and bad) at ${blueBright `https://ntl.fyi/next-beta-feedback`}
       `));

package/lib/index.js

@@ -1,29 +1,33 @@
 // @ts-check
 const { join, relative } = require('path');
-const { copy, existsSync } = require('fs-extra');
+const { ODB_FUNCTION_NAME } = require('./constants');
 const { restoreCache, saveCache } = require('./helpers/cache');
 const { getNextConfig, configureHandlerFunctions, generateRedirects } = require('./helpers/config');
+const { moveStaticPages, movePublicFiles } = require('./helpers/files');
 const { generateFunctions, setupImageFunction, generatePagesResolver } = require('./helpers/functions');
-const { verifyNetlifyBuildVersion, checkNextSiteHasBuilt, verifyBuildTarget, checkForRootPublish, logBetaMessage, } = require('./helpers/verification');
+const { verifyNetlifyBuildVersion, checkNextSiteHasBuilt, checkForRootPublish, logBetaMessage, checkZipSize, } = require('./helpers/verification');
 module.exports = {
     async onPreBuild({ constants, netlifyConfig, utils: { build: { failBuild }, cache, }, }) {
+        var _a;
         logBetaMessage();
         const { publish } = netlifyConfig.build;
         checkForRootPublish({ publish, failBuild });
         verifyNetlifyBuildVersion({ failBuild, ...constants });
         await restoreCache({ cache, publish });
+        (_a = netlifyConfig.build).environment || (_a.environment = {});
+        // eslint-disable-next-line unicorn/consistent-destructuring
+        netlifyConfig.build.environment.NEXT_PRIVATE_TARGET = 'server';
     },
     async onBuild({ constants, netlifyConfig, utils: { build: { failBuild }, }, }) {
         const { publish } = netlifyConfig.build;
         checkNextSiteHasBuilt({ publish, failBuild });
         const { appDir, basePath, i18n, images, target, ignore } = await getNextConfig({ publish, failBuild });
-        verifyBuildTarget(target);
         configureHandlerFunctions({ netlifyConfig, ignore, publish: relative(process.cwd(), publish) });
         await generateFunctions(constants, appDir);
         await generatePagesResolver({ netlifyConfig, target, constants });
-        const publicDir = join(appDir, 'public');
-        if (existsSync(publicDir)) {
-            await copy(publicDir, `${publish}/`);
+        await movePublicFiles({ appDir, publish });
+        if (process.env.EXPERIMENTAL_MOVE_STATIC_PAGES) {
+            await moveStaticPages({ target, failBuild, netlifyConfig, i18n });
         }
         await setupImageFunction({ constants, imageconfig: images, netlifyConfig, basePath });
         await generateRedirects({
@@ -32,8 +36,9 @@ module.exports = {
             i18n,
         });
     },
-    async onPostBuild({ netlifyConfig, utils: { cache } }) {
-        return saveCache({ cache, publish: netlifyConfig.build.publish });
+    async onPostBuild({ netlifyConfig, utils: { cache }, constants: { FUNCTIONS_DIST } }) {
+        await saveCache({ cache, publish: netlifyConfig.build.publish });
+        await checkZipSize(join(FUNCTIONS_DIST, `${ODB_FUNCTION_NAME}.zip`));
     },
     onEnd() {
         logBetaMessage();

package/lib/templates/getHandler.js

@@ -1,15 +1,64 @@
+const { promises, createWriteStream, existsSync } = require('fs');
 const { Server } = require('http');
+const { tmpdir } = require('os');
 const path = require('path');
+const { promisify } = require('util');
+const streamPipeline = promisify(require('stream').pipeline);
 const { Bridge } = require('@vercel/node/dist/bridge');
+const fetch = require('node-fetch');
 const makeHandler = () => 
 // We return a function and then call `toString()` on it to serialise it as the launcher function
-(conf, app) => {
-    // This is just so nft knows about the page entrypoints
+(conf, app, pageRoot, staticManifest = []) => {
+    // This is just so nft knows about the page entrypoints. It's not actually used
     try {
         // eslint-disable-next-line node/no-missing-require
         require.resolve('./pages.js');
     }
     catch { }
+    // Set during the request as it needs the host header. Hoisted so we can define the function once
+    let base;
+    // Only do this if we have some static files moved to the CDN
+    if (staticManifest.length !== 0) {
+        // These are static page files that have been removed from the function bundle
+        // In most cases these are served from the CDN, but for rewrites Next may try to read them
+        // from disk. We need to intercept these and load them from the CDN instead
+        // Sadly the only way to do this is to monkey-patch fs.promises. Yeah, I know.
+        const staticFiles = new Set(staticManifest);
+        // Yes, you can cache stuff locally in a Lambda
+        const cacheDir = path.join(tmpdir(), 'next-static-cache');
+        // Grab the real fs.promises.readFile...
+        const readfileOrig = promises.readFile;
+        // ...then money-patch it to see if it's requesting a CDN file
+        promises.readFile = async (file, options) => {
+            // We only care about page files
+            if (file.startsWith(pageRoot)) {
+                // We only want the part after `pages/`
+                const filePath = file.slice(pageRoot.length + 1);
+                // Is it in the CDN and not local?
+                if (staticFiles.has(filePath) && !existsSync(file)) {
+                    // This name is safe to use, because it's one that was already created by Next
+                    const cacheFile = path.join(cacheDir, filePath);
+                    // Have we already cached it? We ignore the cache if running locally to avoid staleness
+                    if ((!existsSync(cacheFile) || process.env.NETLIFY_DEV) && base) {
+                        await promises.mkdir(path.dirname(cacheFile), { recursive: true });
+                        // Append the path to our host and we can load it like a regular page
+                        const url = `${base}/${filePath}`;
+                        console.log(`Downloading ${url} to ${cacheFile}`);
+                        const response = await fetch(url);
+                        if (!response.ok) {
+                            // Next catches this and returns it as a not found file
+                            throw new Error(`Failed to fetch ${url}`);
+                        }
+                        // Stream it to disk
+                        await streamPipeline(response.body, createWriteStream(cacheFile));
+                    }
+                    // Return the cache file
+                    return readfileOrig(cacheFile, options);
+                }
+            }
+            return readfileOrig(file, options);
+        };
+    }
     let NextServer;
     try {
         // next >= 11.0.1. Yay breaking changes in patch releases!
@@ -53,10 +102,16 @@ const makeHandler = () =>
     const bridge = new Bridge(server);
     bridge.listen();
     return async (event, context) => {
-        var _a, _b;
+        var _a, _b, _c, _d;
         // Next expects to be able to parse the query from the URL
         const query = new URLSearchParams(event.queryStringParameters).toString();
         event.path = query ? `${event.path}?${query}` : event.path;
+        // Only needed if we're intercepting static files
+        if (staticManifest.length !== 0) {
+            const { host } = event.headers;
+            const protocol = event.headers['x-forwarded-proto'] || 'http';
+            base = `${protocol}://${host}`;
+        }
         const { headers, ...result } = await bridge.launcher(event, context);
         /** @type import("@netlify/functions").HandlerResponse */
         // Convert all headers to multiValueHeaders
@@ -73,6 +128,10 @@ const makeHandler = () =>
             delete multiValueHeaders.etag;
             multiValueHeaders['cache-control'] = ['no-cache'];
         }
+        // Sending SWR headers causes undefined behaviour with the Netlify CDN
+        if ((_d = (_c = multiValueHeaders['cache-control']) === null || _c === void 0 ? void 0 : _c[0]) === null || _d === void 0 ? void 0 : _d.includes('stale-while-revalidate')) {
+            multiValueHeaders['cache-control'] = ['public, max-age=0, must-revalidate'];
+        }
         return {
             ...result,
             multiValueHeaders,
@@ -82,13 +141,24 @@ const makeHandler = () =>
 };
 const getHandler = ({ isODB = false, publishDir = '../../../.next', appDir = '../../..' }) => `
 const { Server } = require("http");
+const { tmpdir } = require('os')
+const { promises, createWriteStream, existsSync } = require("fs");
+const { promisify } = require('util')
+const streamPipeline = promisify(require('stream').pipeline)
 // We copy the file here rather than requiring from the node module
 const { Bridge } = require("./bridge");
+const fetch = require('node-fetch')
+
 const { builder } = require("@netlify/functions");
 const { config }  = require("${publishDir}/required-server-files.json")
+let staticManifest 
+try {
+  staticManifest = require("${publishDir}/static-manifest.json")
+} catch {}
 const path = require("path");
+const pageRoot = path.resolve(path.join(__dirname, "${publishDir}", config.target === "server" ? "server" : "serverless", "pages"));
 exports.handler = ${isODB
-    ? `builder((${makeHandler().toString()})(config, "${appDir}"));`
-    : `(${makeHandler().toString()})(config, "${appDir}");`}
+    ? `builder((${makeHandler().toString()})(config, "${appDir}", pageRoot, staticManifest));`
+    : `(${makeHandler().toString()})(config, "${appDir}", pageRoot, staticManifest);`}
 `;
 module.exports = getHandler;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@netlify/plugin-nextjs",
-  "version": "4.0.0-beta.2",
+  "version": "4.0.0-beta.6",
   "description": "Run Next.js seamlessly on Netlify",
   "main": "lib/index.js",
   "files": [
@@ -10,7 +10,7 @@
   "scripts": {
     "build:demo": "next build demo",
     "cy:open": "cypress open --config-file cypress/config/all.json",
-    "cy:run": "cypress run --config-file cypress/config/all.json",
+    "cy:run": "cypress run --config-file ../cypress/config/ci.json",
     "dev:demo": "next dev demo",
     "format": "run-s format:check-fix:*",
     "format:ci": "run-s format:check:*",
@@ -52,14 +52,19 @@
   },
   "homepage": "https://github.com/netlify/netlify-plugin-nextjs#readme",
   "dependencies": {
-    "@netlify/functions": "^0.7.2",
+    "@netlify/functions": "^0.8.0",
     "@netlify/ipx": "^0.0.7",
     "@vercel/node": "^1.11.2-canary.4",
     "chalk": "^4.1.2",
     "fs-extra": "^10.0.0",
+    "globby": "^11.0.4",
     "moize": "^6.1.0",
+    "node-fetch": "^2.6.6",
+    "node-stream-zip": "^1.15.0",
     "outdent": "^0.8.0",
+    "p-limit": "^3.1.0",
     "pathe": "^0.2.0",
+    "pretty-bytes": "^5.6.0",
     "semver": "^7.3.5",
     "slash": "^3.0.0",
     "tiny-glob": "^0.2.9"
@@ -69,6 +74,7 @@
     "@babel/preset-env": "^7.15.8",
     "@netlify/eslint-config-node": "^3.3.0",
     "@testing-library/cypress": "^8.0.1",
+    "@types/fs-extra": "^9.0.13",
     "@types/jest": "^27.0.2",
     "@types/mocha": "^9.0.0",
     "babel-jest": "^27.2.5",
@@ -78,7 +84,7 @@
     "husky": "^4.3.0",
     "jest": "^27.0.0",
     "netlify-plugin-cypress": "^2.2.0",
-    "next": "^11.1.2",
+    "next": "^12.0.2",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.1.2",
     "react": "^17.0.1",