@netlify/build 29.0.0-rc → 29.0.0

package/lib/core/dry.js

@@ -0,0 +1,21 @@
+import pFilter from 'p-filter';
+import { logDryRunStart, logDryRunStep, logDryRunEnd } from '../log/messages/dry.js';
+import { runsOnlyOnBuildFailure } from '../plugins/events.js';
+// If the `dry` flag is specified, do a dry run
+export const doDryRun = async function ({ buildDir, steps, netlifyConfig, constants, buildbotServerSocket, logs }) {
+    const successSteps = await pFilter(steps, ({ event, condition }) => shouldIncludeStep({ buildDir, event, condition, netlifyConfig, constants, buildbotServerSocket }));
+    const eventWidth = Math.max(...successSteps.map(getEventLength));
+    const stepsCount = successSteps.length;
+    logDryRunStart({ logs, eventWidth, stepsCount });
+    successSteps.forEach((step, index) => {
+        logDryRunStep({ logs, step, index, netlifyConfig, eventWidth, stepsCount });
+    });
+    logDryRunEnd(logs);
+};
+const shouldIncludeStep = async function ({ buildDir, event, condition, netlifyConfig, constants, buildbotServerSocket, }) {
+    return (!runsOnlyOnBuildFailure(event) &&
+        (condition === undefined || (await condition({ buildDir, constants, netlifyConfig, buildbotServerSocket }))));
+};
+const getEventLength = function ({ event }) {
+    return event.length;
+};

package/lib/core/feature_flags.js

@@ -0,0 +1,21 @@
+// From CLI `--featureFlags=a,b,c` to programmatic `{ a: true, b: true, c: true }`
+export const normalizeCliFeatureFlags = function (cliFeatureFlags) {
+    return Object.assign({}, ...cliFeatureFlags.split(',').filter(isNotEmpty).map(getFeatureFlag));
+};
+const isNotEmpty = function (name) {
+    return name.trim() !== '';
+};
+const getFeatureFlag = function (name) {
+    return { [name]: true };
+};
+// Default values for feature flags
+export const DEFAULT_FEATURE_FLAGS = {
+    buildbot_es_modules_esbuild: false,
+    buildbot_zisi_trace_nft: false,
+    buildbot_zisi_esbuild_parser: false,
+    edge_functions_cache_cli: false,
+    edge_functions_system_logger: false,
+    // TODO: remove this flag once rolled out to everyone
+    // FF link: https://app.launchdarkly.com/default/production/features/plugins_break_builds_with_unsupported_plugin_versions/targeting
+    plugins_break_builds_with_unsupported_plugin_versions: false,
+};

package/lib/core/flags.js

@@ -0,0 +1,201 @@
+const jsonParse = function (value) {
+    return value === undefined ? undefined : JSON.parse(value);
+};
+// All CLI flags
+export const FLAGS = {
+    config: {
+        string: true,
+        describe: `Path to the configuration file.
+Defaults to any netlify.toml in the git repository root directory or the base directory`,
+    },
+    defaultConfig: {
+        string: true,
+        describe: `JSON configuration object containing default values.
+Each configuration default value is used unless overriden through the main configuration file.
+Default: none.`,
+        coerce: jsonParse,
+        hidden: true,
+    },
+    cachedConfig: {
+        string: true,
+        describe: `JSON configuration object returned by @netlify/config when --output=/ is used
+or when using @netlify/config programmatically.
+This is done as a performance optimization to cache the configuration loading logic.
+Default: none.`,
+        coerce: jsonParse,
+        hidden: true,
+    },
+    cachedConfigPath: {
+        string: true,
+        describe: `File path to the JSON configuration object returned by @netlify/config
+when --output=/path is used.
+This is done as a performance optimization to cache the configuration loading logic.
+Default: none.`,
+        hidden: true,
+    },
+    cwd: {
+        string: true,
+        describe: `Current directory. Used to retrieve the configuration file.
+Default: current directory`,
+    },
+    repositoryRoot: {
+        string: true,
+        describe: `Git repository root directory. Used to retrieve the configuration file.
+Default: automatically guessed`,
+    },
+    apiHost: {
+        string: true,
+        describe: `Netlify API endpoint.
+Default: automatically guessed`,
+    },
+    token: {
+        string: true,
+        describe: `Netlify API token for authentication.
+The NETLIFY_AUTH_TOKEN environment variable can be used as well.`,
+    },
+    siteId: {
+        string: true,
+        describe: `Netlify Site ID.`,
+    },
+    deployId: {
+        string: true,
+        describe: `Netlify Deploy ID.
+Default: automatically guessed`,
+    },
+    buildId: {
+        string: true,
+        describe: `Netlify Build ID.
+Default: automatically guessed`,
+    },
+    context: {
+        string: true,
+        describe: `Build context.
+Default: 'production'`,
+    },
+    branch: {
+        string: true,
+        describe: `Repository branch.
+Default: automatically guessed`,
+    },
+    framework: {
+        string: true,
+        describe: 'Front-end framework.',
+        hidden: true,
+    },
+    baseRelDir: {
+        boolean: true,
+        describe: `Feature flag meant for backward compatibility.
+When enabled, if the 'build.base' configuration property is defined, it is used
+to try to retrieve a second configuration file and discard the first one.
+Default: true`,
+        hidden: true,
+    },
+    dry: {
+        alias: 'dry-run',
+        boolean: true,
+        describe: `Run in dry mode, i.e. printing steps without executing them.
+Default: false`,
+    },
+    nodePath: {
+        string: true,
+        describe: `Path to the Node.js binary to use in the build command and plugins.
+Default: Current Node.js binary`,
+    },
+    functionsDistDir: {
+        string: true,
+        describe: `Path to the directory where packaged functions are kept.
+Default: automatically guessed`,
+        hidden: true,
+    },
+    edgeFunctionsDistDir: {
+        string: true,
+        describe: `Path to the directory where packaged Edge Functions are kept.
+Default: automatically guessed`,
+        hidden: true,
+    },
+    cacheDir: {
+        string: true,
+        describe: `Path to the cache directory.
+Default: .netlify/cache/`,
+        hidden: true,
+    },
+    buildbotServerSocket: {
+        string: true,
+        describe: `Path to the buildbot server socket. This is used to connect to the buildbot to trigger deploys.`,
+        hidden: true,
+    },
+    telemetry: {
+        boolean: true,
+        describe: `Enable telemetry.
+Default: false`,
+    },
+    mode: {
+        string: true,
+        describe: `Environment in which this is loaded. Can be:
+  - 'buildbot': within Netlify Buildbot
+  - 'cli': within Netlify CLI
+  - 'require': through import('@netlify/build')`,
+        hidden: true,
+    },
+    debug: {
+        boolean: true,
+        describe: 'Print user-facing debugging information',
+        hidden: true,
+    },
+    systemLogFile: {
+        type: 'number',
+        describe: 'File descriptor to where system logs should be piped',
+        hidden: true,
+    },
+    verbose: {
+        boolean: true,
+        describe: 'Print internal debugging information',
+        hidden: true,
+    },
+    sendStatus: {
+        boolean: true,
+        describe: 'Whether plugin statuses should be sent to the Netlify API',
+        hidden: true,
+    },
+    saveConfig: {
+        boolean: true,
+        describe: 'Whether configuration changes should be saved to netlify.toml',
+        hidden: true,
+    },
+    testOpts: {
+        describe: 'Options for testing only',
+        hidden: true,
+    },
+    featureFlags: {
+        describe: 'Comma-separated list of feature flags to enable unreleased features',
+        hidden: true,
+    },
+    statsd: {
+        describe: 'Statsd-related options, for performance measuring',
+        hidden: true,
+    },
+    'statsd.host': {
+        type: 'string',
+        describe: 'Statsd host',
+        hidden: true,
+    },
+    'statsd.port': {
+        type: 'number',
+        describe: 'Statsd port',
+        hidden: true,
+    },
+    offline: {
+        boolean: true,
+        describe: `Do not send requests to the Netlify API to retrieve site settings.
+Default: false`,
+    },
+    buffer: {
+        boolean: true,
+        describe: 'Buffer output instead of printing it',
+    },
+    timeline: {
+        string: true,
+        describe: 'The sequence of lifecycle events to run',
+        hidden: true,
+    },
+};

package/lib/core/lingering.js

@@ -0,0 +1,68 @@
+import psList from 'ps-list';
+import { logLingeringProcesses } from '../log/messages/core.js';
+// Print a warning when some build processes are still running.
+// We cannot rely on using the process tree:
+//  - This is because it is impossible to know whether a process was a child of
+//    of another once its parent process has exited. When that happens, the s
+//    child becomes inherited by `init`, changing its `ppid`. The information
+//    about the original parent is then lost.
+//  - The only way to implement this would be to repeatedly list processes as
+//    the build command is ongoing. However, this would fail to detect processes
+//    spawned just before the build commands ends.
+// We cannot list processes before and after the build command and use the
+// difference.
+//  - This is because other processes (unrelated to @netlify/build) might be
+//    running at the same time. This includes OS background processes.
+// Therefore, we run this in a controlled environment only (the buildbot) and
+// exclude specific processes manually. This is a lesser evil, although still
+// quite hacky.
+export const warnOnLingeringProcesses = async function ({ mode, logs, testOpts: { silentLingeringProcesses = false }, }) {
+    if (mode !== 'buildbot' || silentLingeringProcesses) {
+        return;
+    }
+    const processes = await psList();
+    const commands = processes.map(getCommand).filter(isNotIgnoredCommand);
+    if (commands.length === 0) {
+        return;
+    }
+    logLingeringProcesses(logs, commands);
+};
+// `cmd` is only available on Unix. Unlike `name`, it includes the arguments.
+const getCommand = function ({ name, cmd = name }) {
+    return cmd;
+};
+// We ignore any command known to be internal to the buildbot.
+// We also ignore commands known not to complete properly in builds if they are
+// widely used.
+const isNotIgnoredCommand = function (command) {
+    return !IGNORED_COMMANDS.some((ignoredCommand) => matchesIgnoredCommand(command, ignoredCommand));
+};
+const matchesIgnoredCommand = function (command, ignoredCommand) {
+    if (typeof ignoredCommand === 'string') {
+        return command.includes(ignoredCommand);
+    }
+    return ignoredCommand.test(command);
+};
+const IGNORED_COMMANDS = [
+    // TODO: Those can most likely be removed
+    'ps',
+    'grep',
+    'bash',
+    // Internal buildbot commands
+    '[build]',
+    /buildbot.*\[node]/,
+    // buildbot's main Bash script
+    '/opt/build-bin/build',
+    // `@netlify/build` binary itself
+    'netlify-build',
+    // Plugin child processes spawned by @netlify/build
+    '@netlify/build',
+    // Shown for parent processes with currently running child processes.
+    // Happens on `ps` itself.
+    'defunct',
+    // Processes often left running. We should report those but don't because of
+    // how common those are in production builds
+    'gatsby-telemetry',
+    'jest-worker',
+    'broccoli-babel-transpiler',
+];

package/lib/core/main.js

@@ -0,0 +1,110 @@
+import { handleBuildError } from '../error/handle.js';
+import { reportError } from '../error/report.js';
+import { getSystemLogger } from '../log/logger.js';
+import { logTimer, logBuildSuccess } from '../log/messages/core.js';
+import { trackBuildComplete } from '../telemetry/main.js';
+import { reportTimers } from '../time/report.js';
+import { execBuild, startBuild } from './build.js';
+import { getSeverity } from './severity.js';
+export { startDev } from './dev.js';
+export { runCoreSteps } from '../steps/run_core_steps.js';
+/**
+ * Main entry point of Netlify Build.
+ * Runs a builds and returns whether it succeeded or not.
+ *
+ * @param flags - build configuration CLI flags
+ */
+export default async function buildSite(flags = {}) {
+    const { errorMonitor, framework, mode, logs, debug, systemLogFile, testOpts, statsdOpts, dry, telemetry, buildId, deployId, ...flagsA } = startBuild(flags);
+    const errorParams = { errorMonitor, mode, logs, debug, testOpts };
+    const systemLog = getSystemLogger(logs, debug, systemLogFile);
+    try {
+        const { pluginsOptions, netlifyConfig: netlifyConfigA, siteInfo, userNodeVersion, stepsCount, timers, durationNs, configMutations, } = await execBuild({
+            ...flagsA,
+            buildId,
+            systemLogFile,
+            deployId,
+            dry,
+            errorMonitor,
+            mode,
+            logs,
+            debug,
+            testOpts,
+            errorParams,
+        });
+        await handleBuildSuccess({
+            framework,
+            dry,
+            logs,
+            timers,
+            durationNs,
+            statsdOpts,
+            systemLog,
+        });
+        const { success, severityCode, status } = getSeverity('success');
+        await telemetryReport({
+            buildId,
+            deployId,
+            status,
+            stepsCount,
+            pluginsOptions,
+            durationNs,
+            siteInfo,
+            telemetry,
+            userNodeVersion,
+            framework,
+            testOpts,
+            errorParams,
+        });
+        return { success, severityCode, netlifyConfig: netlifyConfigA, logs, configMutations };
+    }
+    catch (error) {
+        const { severity } = await handleBuildError(error, errorParams);
+        const { pluginsOptions, siteInfo, userNodeVersion } = errorParams;
+        const { success, severityCode, status } = getSeverity(severity);
+        await telemetryReport({
+            buildId,
+            deployId,
+            status,
+            pluginsOptions,
+            siteInfo,
+            telemetry,
+            userNodeVersion,
+            framework,
+            testOpts,
+            errorParams,
+        });
+        await reportError({ error, framework, statsdOpts });
+        return { success, severityCode, logs };
+    }
+}
+// Logs and reports that a build successfully ended
+const handleBuildSuccess = async function ({ framework, dry, logs, timers, durationNs, statsdOpts, systemLog }) {
+    if (dry) {
+        return;
+    }
+    logBuildSuccess(logs);
+    logTimer(logs, durationNs, 'Netlify Build', systemLog);
+    await reportTimers({ timers, statsdOpts, framework });
+};
+// Handles the calls and errors of telemetry reports
+const telemetryReport = async function ({ deployId, buildId, status, stepsCount, pluginsOptions, durationNs, siteInfo, telemetry, userNodeVersion, framework, testOpts, errorParams, }) {
+    try {
+        await trackBuildComplete({
+            deployId,
+            buildId,
+            status,
+            stepsCount,
+            pluginsOptions,
+            durationNs,
+            siteInfo,
+            telemetry,
+            userNodeVersion,
+            framework,
+            testOpts,
+        });
+    }
+    catch (error) {
+        await handleBuildError(error, errorParams);
+    }
+};

package/lib/core/missing_side_file.js

@@ -0,0 +1,17 @@
+import { relative } from 'path';
+import { pathExists } from 'path-exists';
+import { logMissingSideFile } from '../log/messages/core.js';
+// Some files like `_headers` and `_redirects` must be copied to the publish
+// directory to be used in production. When those are present in the repository
+// but not in the publish directory, this most likely indicates that the build
+// command accidentally forgot to copy those. We then print a warning message.
+export const warnOnMissingSideFiles = async function ({ buildDir, netlifyConfig: { build: { publish }, }, logs, }) {
+    await Promise.all(SIDE_FILES.map((sideFile) => warnOnMissingSideFile({ logs, sideFile, buildDir, publish })));
+};
+const SIDE_FILES = ['_headers', '_redirects'];
+const warnOnMissingSideFile = async function ({ logs, sideFile, buildDir, publish }) {
+    if (!(await pathExists(`${buildDir}/${sideFile}`)) || (await pathExists(`${publish}/${sideFile}`))) {
+        return;
+    }
+    logMissingSideFile(logs, sideFile, relative(buildDir, publish));
+};

package/lib/core/normalize_flags.js

@@ -0,0 +1,59 @@
+import { env, execPath } from 'process';
+import { logFlags } from '../log/messages/config.js';
+import { removeFalsy } from '../utils/remove_falsy.js';
+import { DEFAULT_FEATURE_FLAGS } from './feature_flags.js';
+const REQUIRE_MODE = 'require';
+const DEFAULT_EDGE_FUNCTIONS_DIST = '.netlify/edge-functions-dist/';
+const DEFAULT_FUNCTIONS_DIST = '.netlify/functions/';
+const DEFAULT_CACHE_DIR = '.netlify/cache/';
+const DEFAULT_STATSD_PORT = 8125;
+/** Normalize CLI flags  */
+export const normalizeFlags = function (flags, logs) {
+    const rawFlags = removeFalsy(flags);
+    // Combine the flags object env with the process.env
+    const combinedEnv = { ...env, ...rawFlags.env };
+    const defaultFlags = getDefaultFlags(rawFlags, combinedEnv);
+    // The telemetry flag requires specific logic to compute
+    const telemetryFlag = computeTelemetry(rawFlags, combinedEnv);
+    const mergedFlags = {
+        ...defaultFlags,
+        ...rawFlags,
+        ...telemetryFlag,
+        statsdOpts: { ...defaultFlags.statsd, ...rawFlags.statsd },
+        featureFlags: { ...defaultFlags.featureFlags, ...rawFlags.featureFlags },
+    };
+    const normalizedFlags = removeFalsy(mergedFlags);
+    logFlags(logs, rawFlags, normalizedFlags);
+    return normalizedFlags;
+};
+// Default values of CLI flags
+const getDefaultFlags = function ({ env: envOpt = {} }, combinedEnv) {
+    return {
+        env: envOpt,
+        nodePath: execPath,
+        token: combinedEnv.NETLIFY_AUTH_TOKEN,
+        mode: REQUIRE_MODE,
+        offline: false,
+        telemetry: false,
+        verbose: Boolean(combinedEnv.NETLIFY_BUILD_DEBUG),
+        functionsDistDir: DEFAULT_FUNCTIONS_DIST,
+        edgeFunctionsDistDir: DEFAULT_EDGE_FUNCTIONS_DIST,
+        cacheDir: DEFAULT_CACHE_DIR,
+        deployId: combinedEnv.DEPLOY_ID,
+        buildId: combinedEnv.BUILD_ID,
+        debug: Boolean(combinedEnv.NETLIFY_BUILD_DEBUG),
+        bugsnagKey: combinedEnv.BUGSNAG_KEY,
+        sendStatus: false,
+        saveConfig: false,
+        apiHost: 'api.netlify.com',
+        testOpts: {},
+        featureFlags: DEFAULT_FEATURE_FLAGS,
+        statsd: { port: DEFAULT_STATSD_PORT },
+        timeline: 'build',
+    };
+};
+// Compute the telemetry flag, it's disabled by default and we want to always disable it
+// if BUILD_TELEMETRY_DISABLED is passed.
+const computeTelemetry = function (flags, envOpts) {
+    return envOpts.BUILD_TELEMETRY_DISABLED ? { telemetry: false } : { telemetry: flags.telemetry };
+};

package/lib/core/severity.js

@@ -0,0 +1,21 @@
+const SEVERITY_MAP = {
+    success: { severityCode: 0, status: 'success' },
+    none: { severityCode: 1, status: 'cancelled' },
+    info: { severityCode: 2, status: 'user-error' },
+    warning: { severityCode: 3, status: 'plugin-error' },
+    error: { severityCode: 4, status: 'system-error' },
+};
+const SUCCESS_SEVERITY = 'success';
+/** Indicates a bug in our codebase */
+const FALLBACK_SEVERITY = 'error';
+/** Map error severities to exit codes and status (used for telemetry purposes) */
+export const FALLBACK_SEVERITY_ENTRY = SEVERITY_MAP[FALLBACK_SEVERITY];
+/**
+ * Used to extract exit codes and respective status strings
+ * 1|2|3 indicate whether this was a user|plugin|system error.
+ */
+export const getSeverity = (severity = FALLBACK_SEVERITY) => {
+    const severityEntry = severity in SEVERITY_MAP ? SEVERITY_MAP[severity] : FALLBACK_SEVERITY_ENTRY;
+    const success = severity === SUCCESS_SEVERITY;
+    return { success, ...severityEntry };
+};

package/lib/core/types.js

@@ -0,0 +1,8 @@
+export var SeverityCode;
+(function (SeverityCode) {
+    SeverityCode[SeverityCode["success"] = 1] = "success";
+    SeverityCode[SeverityCode["buildCancelled"] = 2] = "buildCancelled";
+    SeverityCode[SeverityCode["userError"] = 3] = "userError";
+    SeverityCode[SeverityCode["pluginError"] = 4] = "pluginError";
+    SeverityCode[SeverityCode["systemError"] = 5] = "systemError";
+})(SeverityCode = SeverityCode || (SeverityCode = {}));

package/lib/core/user_node_version.js

@@ -0,0 +1,32 @@
+import { version as currentVersion, execPath } from 'process';
+import { execa } from 'execa';
+import semver from 'semver';
+import { addErrorInfo } from '../error/info.js';
+// Retrieve Node.js version if the Node.js path is using nvm.
+// `node.exe` on Windows, `bin/node` on Unix.
+const NVM_NODE_VERSION_REGEXP = /[/\\]v(\d+\.\d+\.\d+)[/\\](bin[/\\]node|node.exe)$/;
+// Retrieve Node.js version from current process
+const getCurrentNodeVersion = function () {
+    return semver.clean(currentVersion);
+};
+// Retrieve Node.js version from `--node-path` or fallback to extracting the current Node.js process version
+export const getUserNodeVersion = async function (nodePath) {
+    // No `--node-path` CLI flag, use the current node process version
+    if (nodePath === execPath) {
+        return getCurrentNodeVersion();
+    }
+    // Extract version from path
+    const result = NVM_NODE_VERSION_REGEXP.exec(nodePath);
+    if (result !== null) {
+        return result[1];
+    }
+    // Fallback to actually running `node --version` with the given nodePath
+    const { stdout } = await execa(nodePath, ['--version'], { reject: false });
+    const version = semver.clean(stdout);
+    if (version === null) {
+        const error = new Error(`Invalid --node-path CLI flag: ${nodePath}`);
+        addErrorInfo(error, { type: 'resolveConfig' });
+        throw error;
+    }
+    return version;
+};

package/lib/env/changes.js

@@ -0,0 +1,43 @@
+import { env } from 'process';
+import filterObj from 'filter-obj';
+import mapObj from 'map-obj';
+// If plugins modify `process.env`, this is propagated in other plugins and in
+// `build.command`. Since those are different processes, we figure out when they
+// do this and communicate the new `process.env` to other processes.
+export const getNewEnvChanges = function (envBefore, netlifyConfig, netlifyConfigCopy) {
+    const processEnvChanges = diffEnv(envBefore, env);
+    const netlifyConfigEnvChanges = diffEnv(netlifyConfig.build.environment, netlifyConfigCopy.build.environment);
+    return { ...processEnvChanges, ...netlifyConfigEnvChanges };
+};
+const diffEnv = function (envBefore, envAfter) {
+    const envChanges = filterObj(envAfter, (name, value) => value !== envBefore[name]);
+    const deletedEnv = filterObj(envBefore, (name) => envAfter[name] === undefined);
+    const deletedEnvA = mapObj(deletedEnv, setToNull);
+    return { ...envChanges, ...deletedEnvA };
+};
+// `undefined` is not JSON-serializable (which is used in process IPC), so we
+// convert it to `null`
+// Note: `process.env[name] = undefined` actually does
+// `process.env[name] = 'undefined'` in Node.js.
+const setToNull = function (name) {
+    return [name, null];
+};
+// Set `process.env` changes from a previous different plugin.
+// Can also merge with a `currentEnv` plain object instead of `process.env`.
+export const setEnvChanges = function (envChanges, currentEnv = env) {
+    Object.entries(envChanges).forEach(([name, value]) => {
+        setEnvChange(name, value, currentEnv);
+    });
+    return { ...currentEnv };
+};
+const setEnvChange = function (name, value, currentEnv) {
+    if (currentEnv[name] === value) {
+        return;
+    }
+    if (value === null) {
+        // `currentEnv` is a mutable variable
+        delete currentEnv[name];
+        return;
+    }
+    currentEnv[name] = value;
+};

package/lib/env/main.js

@@ -0,0 +1,14 @@
+import { env } from 'process';
+import filterObj from 'filter-obj';
+import { getParentColorEnv } from '../log/colors.js';
+// Retrieve the environment variables passed to plugins and `build.command`
+// When run locally, this tries to emulate the production environment.
+export const getChildEnv = function ({ envOpt, env: allConfigEnv }) {
+    const parentColorEnv = getParentColorEnv();
+    const parentEnv = { ...env, ...allConfigEnv, ...envOpt, ...parentColorEnv };
+    return filterObj(parentEnv, shouldKeepEnv);
+};
+const shouldKeepEnv = function (key) {
+    return !REMOVED_PARENT_ENV.has(key.toLowerCase());
+};
+const REMOVED_PARENT_ENV = new Set(['bugsnag_key']);