@netlify/build 28.0.0-beta → 29.0.0-rc

package/src/core/main.js

@@ -1,29 +1,12 @@
-/* eslint-disable max-lines, import/max-dependencies */
 import { handleBuildError } from '../error/handle.js'
-import { getErrorInfo } from '../error/info.js'
-import { startErrorMonitor } from '../error/monitor/start.js'
-import { getBufferLogs } from '../log/logger.js'
-import { logBuildStart, logTimer, logBuildSuccess } from '../log/messages/core.js'
-import { loadPlugins } from '../plugins/load.js'
-import { getPluginsOptions } from '../plugins/options.js'
-import { pinPlugins } from '../plugins/pinned_version.js'
-import { startPlugins, stopPlugins } from '../plugins/spawn.js'
-import { addCorePlugins } from '../plugins_core/add.js'
-import { reportStatuses } from '../status/report.js'
-import { getSteps } from '../steps/get.js'
-import { runSteps } from '../steps/run_steps.js'
+import { logTimer, logBuildSuccess } from '../log/messages/core.js'
 import { trackBuildComplete } from '../telemetry/main.js'
-import { initTimers, measureDuration } from '../time/main.js'
 import { reportTimers } from '../time/report.js'
 
-import { getConfigOpts, loadConfig } from './config.js'
-import { getConstants } from './constants.js'
-import { doDryRun } from './dry.js'
-import { warnOnLingeringProcesses } from './lingering.js'
-import { warnOnMissingSideFiles } from './missing_side_file.js'
-import { normalizeFlags } from './normalize_flags.js'
+import { execBuild, startBuild } from './build.js'
 import { getSeverity } from './severity.js'
 
+export { startDev } from './dev.js'
 export { runCoreSteps } from '../steps/run_core_steps.js'
 
 /**
@@ -57,6 +40,7 @@ export default async function buildSite(flags = {}) {
     mode,
     logs,
     debug,
+    systemLogFile,
     testOpts,
     statsdOpts,
     dry,
@@ -80,6 +64,7 @@ export default async function buildSite(flags = {}) {
     } = await execBuild({
       ...flagsA,
       buildId,
+      systemLogFile,
       deployId,
       dry,
       errorMonitor,
@@ -133,517 +118,6 @@ export default async function buildSite(flags = {}) {
   }
 }
 
-// Performed on build start. Must be kept small and unlikely to fail since it
-// does not have proper error handling. Error handling relies on `errorMonitor`
-// being built, which relies itself on flags being normalized.
-const startBuild = function (flags) {
-  const timers = initTimers()
-
-  const logs = getBufferLogs(flags)
-  logBuildStart(logs)
-
-  const { bugsnagKey, ...flagsA } = normalizeFlags(flags, logs)
-  const errorMonitor = startErrorMonitor({ flags: flagsA, logs, bugsnagKey })
-
-  return { ...flagsA, errorMonitor, logs, timers }
-}
-
-const tExecBuild = async function ({
-  config,
-  defaultConfig,
-  cachedConfig,
-  cachedConfigPath,
-  cwd,
-  repositoryRoot,
-  apiHost,
-  token,
-  siteId,
-  context,
-  branch,
-  baseRelDir,
-  env: envOpt,
-  debug,
-  verbose,
-  nodePath,
-  functionsDistDir,
-  edgeFunctionsDistDir,
-  cacheDir,
-  dry,
-  mode,
-  offline,
-  deployId,
-  buildId,
-  testOpts,
-  errorMonitor,
-  errorParams,
-  logs,
-  timers,
-  buildbotServerSocket,
-  sendStatus,
-  saveConfig,
-  featureFlags,
-}) {
-  const configOpts = getConfigOpts({
-    config,
-    defaultConfig,
-    cwd,
-    repositoryRoot,
-    apiHost,
-    token,
-    siteId,
-    context,
-    branch,
-    baseRelDir,
-    envOpt,
-    mode,
-    offline,
-    deployId,
-    buildId,
-    testOpts,
-    featureFlags,
-  })
-  const {
-    netlifyConfig,
-    configPath,
-    headersPath,
-    redirectsPath,
-    buildDir,
-    repositoryRoot: repositoryRootA,
-    packageJson,
-    userNodeVersion,
-    childEnv,
-    context: contextA,
-    branch: branchA,
-    token: tokenA,
-    api,
-    siteInfo,
-    timers: timersA,
-  } = await loadConfig({
-    configOpts,
-    cachedConfig,
-    cachedConfigPath,
-    envOpt,
-    debug,
-    logs,
-    nodePath,
-    timers,
-  })
-  const constants = await getConstants({
-    configPath,
-    buildDir,
-    functionsDistDir,
-    edgeFunctionsDistDir,
-    cacheDir,
-    netlifyConfig,
-    siteInfo,
-    apiHost,
-    token: tokenA,
-    mode,
-    testOpts,
-  })
-  const pluginsOptions = addCorePlugins({ netlifyConfig, constants })
-  // `errorParams` is purposely stateful
-  // eslint-disable-next-line fp/no-mutating-assign
-  Object.assign(errorParams, { netlifyConfig, pluginsOptions, siteInfo, childEnv, userNodeVersion })
-
-  const {
-    pluginsOptions: pluginsOptionsA,
-    netlifyConfig: netlifyConfigA,
-    stepsCount,
-    timers: timersB,
-    configMutations,
-  } = await runAndReportBuild({
-    pluginsOptions,
-    netlifyConfig,
-    configOpts,
-    siteInfo,
-    configPath,
-    headersPath,
-    redirectsPath,
-    buildDir,
-    repositoryRoot: repositoryRootA,
-    nodePath,
-    packageJson,
-    userNodeVersion,
-    childEnv,
-    context: contextA,
-    branch: branchA,
-    dry,
-    mode,
-    api,
-    errorMonitor,
-    deployId,
-    errorParams,
-    logs,
-    debug,
-    verbose,
-    timers: timersA,
-    sendStatus,
-    saveConfig,
-    testOpts,
-    buildbotServerSocket,
-    constants,
-    featureFlags,
-  })
-  return {
-    pluginsOptions: pluginsOptionsA,
-    netlifyConfig: netlifyConfigA,
-    siteInfo,
-    userNodeVersion,
-    stepsCount,
-    timers: timersB,
-    configMutations,
-  }
-}
-
-const execBuild = measureDuration(tExecBuild, 'total', { parentTag: 'build_site' })
-
-// Runs a build then report any plugin statuses
-const runAndReportBuild = async function ({
-  pluginsOptions,
-  netlifyConfig,
-  configOpts,
-  siteInfo,
-  configPath,
-  headersPath,
-  redirectsPath,
-  buildDir,
-  repositoryRoot,
-  nodePath,
-  packageJson,
-  userNodeVersion,
-  childEnv,
-  context,
-  branch,
-  buildbotServerSocket,
-  constants,
-  dry,
-  mode,
-  api,
-  errorMonitor,
-  deployId,
-  errorParams,
-  logs,
-  debug,
-  verbose,
-  timers,
-  sendStatus,
-  saveConfig,
-  testOpts,
-  featureFlags,
-}) {
-  try {
-    const {
-      stepsCount,
-      netlifyConfig: netlifyConfigA,
-      statuses,
-      pluginsOptions: pluginsOptionsA,
-      failedPlugins,
-      timers: timersA,
-      configMutations,
-    } = await initAndRunBuild({
-      pluginsOptions,
-      netlifyConfig,
-      configOpts,
-      siteInfo,
-      configPath,
-      headersPath,
-      redirectsPath,
-      buildDir,
-      repositoryRoot,
-      nodePath,
-      packageJson,
-      userNodeVersion,
-      childEnv,
-      context,
-      branch,
-      dry,
-      mode,
-      api,
-      errorMonitor,
-      deployId,
-      errorParams,
-      logs,
-      debug,
-      verbose,
-      timers,
-      sendStatus,
-      saveConfig,
-      testOpts,
-      buildbotServerSocket,
-      constants,
-      featureFlags,
-    })
-    await Promise.all([
-      reportStatuses({
-        statuses,
-        childEnv,
-        api,
-        mode,
-        pluginsOptions: pluginsOptionsA,
-        netlifyConfig: netlifyConfigA,
-        errorMonitor,
-        deployId,
-        logs,
-        debug,
-        sendStatus,
-        testOpts,
-      }),
-      pinPlugins({
-        pluginsOptions: pluginsOptionsA,
-        failedPlugins,
-        api,
-        siteInfo,
-        childEnv,
-        mode,
-        netlifyConfig: netlifyConfigA,
-        errorMonitor,
-        logs,
-        debug,
-        testOpts,
-        sendStatus,
-      }),
-    ])
-
-    return {
-      pluginsOptions: pluginsOptionsA,
-      netlifyConfig: netlifyConfigA,
-      stepsCount,
-      timers: timersA,
-      configMutations,
-    }
-  } catch (error) {
-    const [{ statuses }] = getErrorInfo(error)
-    await reportStatuses({
-      statuses,
-      childEnv,
-      api,
-      mode,
-      pluginsOptions,
-      netlifyConfig,
-      errorMonitor,
-      deployId,
-      logs,
-      debug,
-      sendStatus,
-      testOpts,
-    })
-    throw error
-  }
-}
-
-// Initialize plugin processes then runs a build
-const initAndRunBuild = async function ({
-  pluginsOptions,
-  netlifyConfig,
-  configOpts,
-  siteInfo,
-  configPath,
-  headersPath,
-  redirectsPath,
-  buildDir,
-  repositoryRoot,
-  nodePath,
-  packageJson,
-  userNodeVersion,
-  childEnv,
-  context,
-  branch,
-  dry,
-  mode,
-  api,
-  errorMonitor,
-  deployId,
-  errorParams,
-  logs,
-  debug,
-  verbose,
-  sendStatus,
-  saveConfig,
-  timers,
-  testOpts,
-  buildbotServerSocket,
-  constants,
-  featureFlags,
-}) {
-  const { pluginsOptions: pluginsOptionsA, timers: timersA } = await getPluginsOptions({
-    pluginsOptions,
-    netlifyConfig,
-    siteInfo,
-    buildDir,
-    nodePath,
-    packageJson,
-    userNodeVersion,
-    mode,
-    api,
-    logs,
-    debug,
-    sendStatus,
-    timers,
-    testOpts,
-    featureFlags,
-  })
-  // eslint-disable-next-line fp/no-mutation, no-param-reassign
-  errorParams.pluginsOptions = pluginsOptionsA
-
-  const { childProcesses, timers: timersB } = await startPlugins({
-    pluginsOptions: pluginsOptionsA,
-    buildDir,
-    childEnv,
-    logs,
-    debug,
-    timers: timersA,
-  })
-
-  try {
-    const {
-      stepsCount,
-      netlifyConfig: netlifyConfigA,
-      statuses,
-      failedPlugins,
-      timers: timersC,
-      configMutations,
-    } = await runBuild({
-      childProcesses,
-      pluginsOptions: pluginsOptionsA,
-      netlifyConfig,
-      configOpts,
-      packageJson,
-      configPath,
-      headersPath,
-      redirectsPath,
-      buildDir,
-      repositoryRoot,
-      nodePath,
-      childEnv,
-      context,
-      branch,
-      dry,
-      buildbotServerSocket,
-      constants,
-      mode,
-      api,
-      errorMonitor,
-      deployId,
-      errorParams,
-      logs,
-      debug,
-      verbose,
-      saveConfig,
-      timers: timersB,
-      testOpts,
-      featureFlags,
-    })
-
-    await Promise.all([
-      warnOnMissingSideFiles({ buildDir, netlifyConfig: netlifyConfigA, logs }),
-      warnOnLingeringProcesses({ mode, logs, testOpts }),
-    ])
-
-    return {
-      stepsCount,
-      netlifyConfig: netlifyConfigA,
-      statuses,
-      pluginsOptions: pluginsOptionsA,
-      failedPlugins,
-      timers: timersC,
-      configMutations,
-    }
-  } finally {
-    stopPlugins(childProcesses)
-  }
-}
-
-// Load plugin main files, retrieve their event handlers then runs them,
-// together with the build command
-const runBuild = async function ({
-  childProcesses,
-  pluginsOptions,
-  netlifyConfig,
-  configOpts,
-  packageJson,
-  configPath,
-  headersPath,
-  redirectsPath,
-  buildDir,
-  repositoryRoot,
-  nodePath,
-  childEnv,
-  context,
-  branch,
-  dry,
-  buildbotServerSocket,
-  constants,
-  mode,
-  api,
-  errorMonitor,
-  deployId,
-  errorParams,
-  logs,
-  debug,
-  verbose,
-  saveConfig,
-  timers,
-  testOpts,
-  featureFlags,
-}) {
-  const { pluginsSteps, timers: timersA } = await loadPlugins({
-    pluginsOptions,
-    childProcesses,
-    packageJson,
-    timers,
-    logs,
-    debug,
-    verbose,
-  })
-
-  const { steps, events } = getSteps(pluginsSteps)
-
-  if (dry) {
-    await doDryRun({ buildDir, steps, netlifyConfig, constants, buildbotServerSocket, logs })
-    return { netlifyConfig }
-  }
-
-  const {
-    stepsCount,
-    netlifyConfig: netlifyConfigA,
-    statuses,
-    failedPlugins,
-    timers: timersB,
-    configMutations,
-  } = await runSteps({
-    steps,
-    buildbotServerSocket,
-    events,
-    configPath,
-    headersPath,
-    redirectsPath,
-    buildDir,
-    repositoryRoot,
-    nodePath,
-    childEnv,
-    context,
-    branch,
-    constants,
-    mode,
-    api,
-    errorMonitor,
-    deployId,
-    errorParams,
-    netlifyConfig,
-    configOpts,
-    logs,
-    debug,
-    verbose,
-    saveConfig,
-    timers: timersA,
-    testOpts,
-    featureFlags,
-  })
-
-  return { stepsCount, netlifyConfig: netlifyConfigA, statuses, failedPlugins, timers: timersB, configMutations }
-}
-
 // Logs and reports that a build successfully ended
 const handleBuildSuccess = async function ({ framework, dry, logs, timers, durationNs, statsdOpts }) {
   if (dry) {
@@ -689,4 +163,3 @@ const telemetryReport = async function ({
     await handleBuildError(error, errorParams)
   }
 }
-/* eslint-enable max-lines, import/max-dependencies */

package/src/core/normalize_flags.js

@@ -53,6 +53,7 @@ const getDefaultFlags = function ({ env: envOpt = {} }, combinedEnv) {
     testOpts: {},
     featureFlags: DEFAULT_FEATURE_FLAGS,
     statsd: { port: DEFAULT_STATSD_PORT },
+    timeline: 'build',
   }
 }
 

package/src/error/monitor/normalize.js

@@ -3,7 +3,15 @@
 // them consistent
 export const normalizeGroupingMessage = function (message, type) {
   const messageA = removeDependenciesLogs(message, type)
-  return NORMALIZE_REGEXPS.reduce(normalizeMessage, messageA)
+  const messageB = NORMALIZE_REGEXPS.reduce(normalizeMessage, messageA)
+
+  // If this is a functions bundling error, we'll use additional normalization
+  // rules to group errors more aggressively.
+  if (type === 'functionsBundling') {
+    return FUNCTIONS_BUNDLING_REGEXPS.reduce(normalizeMessage, messageB)
+  }
+
+  return messageB
 }
 
 // Discard debug/info installation information
@@ -32,9 +40,9 @@ const normalizeMessage = function (message, [regExp, replacement]) {
 
 const NORMALIZE_REGEXPS = [
   // Base64 URL
-  [/(data:[^;]+;base64),[\w+/-]+/g, 'dataURI'],
+  [/(data:[^;]+;base64),[\w+/-=]+/g, 'dataURI'],
   // File paths
-  [/(["'`, ]|^)([^"'`, \n]*[/\\][^"'`, \n]*)(["'`, ]|$)/gm, '$1/file/path$3'],
+  [/(["'`, ]|^)([^"'`, \n]*[/\\][^"'`, \n]*)(?=["'`, ]|$)/gm, '$1/file/path'],
   // Semantic versions
   [/\d+\.\d+\.\d+(-\d+)?/g, '1.0.0'],
   [/version "[^"]+"/g, 'version "1.0.0"'],
@@ -45,7 +53,7 @@ const NORMALIZE_REGEXPS = [
   // Numbers, e.g. number of issues/problems
   [/\d+/g, '0'],
   // Hexadecimal strings
-  [/[\da-fA-F]{6,}/g, 'hex'],
+  [/[\da-fA-F-]{6,}/g, 'hex'],
   // On unknown inputs, we print the inputs
   [/(does not accept any inputs but you specified: ).*/, '$1'],
   [/(Unknown inputs for plugin).*/, '$1'],
@@ -75,3 +83,14 @@ const NORMALIZE_REGEXPS = [
   // Multiple empty lines
   [/^\s*$/gm, ''],
 ]
+
+const FUNCTIONS_BUNDLING_REGEXPS = [
+  // String literals and identifiers
+  [/"([^"]+)"/g, '""'],
+  [/'([^']+)'/g, "''"],
+  [/`([^`]+)`/g, '``'],
+
+  // Rust crates
+  [/(?:Downloaded \S+ v[\d.]+\s*)+/gm, 'Downloaded crates'],
+  [/(?:Compiling \S+ v[\d.]+\s*)+/gm, 'Compiled crates'],
+]

package/src/error/monitor/report.js

@@ -21,7 +21,7 @@ export const reportBuildError = async function ({ error, errorMonitor, childEnv,
   const { errorInfo, type, severity, title, group = title } = parseErrorInfo(error)
   const severityA = getSeverity(severity, errorInfo)
   const groupA = getGroup(group, errorInfo)
-  const groupingHash = getGroupingHash(groupA, error, type)
+  const groupingHash = getGroupingHash(groupA, error, type, errorInfo)
   const metadata = getMetadata(errorInfo, childEnv, groupingHash)
   const app = getApp()
   const eventProps = getEventProps({ severity: severityA, group: groupA, groupingHash, metadata, app })
@@ -52,7 +52,12 @@ const getGroup = function (group, errorInfo) {
   return group(errorInfo)
 }
 
-const getGroupingHash = function (group, error, type) {
+const getGroupingHash = function (group, error, type, errorInfo = {}) {
+  // If the error has a `normalizedMessage`, we use it as the grouping hash.
+  if (errorInfo.normalizedMessage) {
+    return errorInfo.normalizedMessage
+  }
+
   const message = error instanceof Error && typeof error.message === 'string' ? error.message : String(error)
   const messageA = normalizeGroupingMessage(message, type)
   return `${group}\n${messageA}`

package/src/error/parse/location.js

@@ -23,8 +23,12 @@ const getBuildCommandLocation = function ({ buildCommand, buildCommandOrigin })
 ${buildCommand}`
 }
 
-const getFunctionsBundlingLocation = function ({ functionName }) {
-  return `While bundling Function "${functionName}"`
+const getFunctionsBundlingLocation = function ({ functionName, functionType }) {
+  if (functionType === 'edge') {
+    return 'While bundling edge function'
+  }
+
+  return `While bundling function "${functionName}"`
 }
 
 const getCoreStepLocation = function ({ coreStepName }) {

package/src/error/type.js

@@ -81,7 +81,14 @@ const TYPES = {
 
   // User error during Functions bundling
   functionsBundling: {
-    title: ({ location: { functionName } }) => `Bundling of Function "${functionName}" failed`,
+    title: ({ location: { functionName, functionType } }) => {
+      if (functionType === 'edge') {
+        return 'Bundling of edge function failed'
+      }
+
+      return `Bundling of function "${functionName}" failed`
+    },
+    group: ({ location: { functionType = 'serverless' } }) => `Bundling of ${functionType} function failed`,
     stackType: 'none',
     locationType: 'functionsBundling',
     severity: 'info',