@netlify/build 28.0.0-beta → 28.0.0-rc

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@netlify/build",
-  "version": "28.0.0-beta",
+  "version": "28.0.0-rc",
   "description": "Netlify build module",
   "type": "module",
   "exports": "./src/core/main.js",
@@ -56,16 +56,18 @@
   "license": "MIT",
   "dependencies": {
     "@bugsnag/js": "^7.0.0",
-    "@netlify/edge-bundler": "^1.4.2",
+    "@netlify/edge-bundler": "^1.8.0",
     "@netlify/cache-utils": "^4.0.0",
-    "@netlify/config": "^18.1.1",
-    "@netlify/functions-utils": "^4.2.0",
+    "@netlify/config": "^18.1.2",
+    "@netlify/functions-utils": "^4.2.2",
     "@netlify/git-utils": "^4.0.0",
-    "@netlify/plugins-list": "^6.29.0",
+    "@netlify/plugins-list": "^6.35.0",
     "@netlify/run-utils": "^4.0.0",
-    "@netlify/zip-it-and-ship-it": "6.0.0-beta",
+    "@netlify/zip-it-and-ship-it": "5.13.2",
     "@sindresorhus/slugify": "^2.0.0",
     "@types/node": "^16.0.0",
+    "ajv": "^8.11.0",
+    "ajv-errors": "^3.0.0",
     "ansi-escapes": "^5.0.0",
     "chalk": "^5.0.0",
     "clean-stack": "^4.0.0",

package/src/core/flags.js

@@ -145,6 +145,11 @@ Default: false`,
     describe: 'Print user-facing debugging information',
     hidden: true,
   },
+  systemLogFile: {
+    type: 'number',
+    describe: 'File descriptor to where system logs should be piped',
+    hidden: true,
+  },
   verbose: {
     boolean: true,
     describe: 'Print internal debugging information',

package/src/core/main.js

@@ -2,7 +2,7 @@
 import { handleBuildError } from '../error/handle.js'
 import { getErrorInfo } from '../error/info.js'
 import { startErrorMonitor } from '../error/monitor/start.js'
-import { getBufferLogs } from '../log/logger.js'
+import { getBufferLogs, getSystemLogger } from '../log/logger.js'
 import { logBuildStart, logTimer, logBuildSuccess } from '../log/messages/core.js'
 import { loadPlugins } from '../plugins/load.js'
 import { getPluginsOptions } from '../plugins/options.js'
@@ -57,6 +57,7 @@ export default async function buildSite(flags = {}) {
     mode,
     logs,
     debug,
+    systemLogFile,
     testOpts,
     statsdOpts,
     dry,
@@ -80,6 +81,7 @@ export default async function buildSite(flags = {}) {
     } = await execBuild({
       ...flagsA,
       buildId,
+      systemLogFile,
       deployId,
       dry,
       errorMonitor,
@@ -163,6 +165,7 @@ const tExecBuild = async function ({
   baseRelDir,
   env: envOpt,
   debug,
+  systemLogFile,
   verbose,
   nodePath,
   functionsDistDir,
@@ -241,6 +244,7 @@ const tExecBuild = async function ({
     mode,
     testOpts,
   })
+  const systemLog = getSystemLogger(logs, debug, systemLogFile)
   const pluginsOptions = addCorePlugins({ netlifyConfig, constants })
   // `errorParams` is purposely stateful
   // eslint-disable-next-line fp/no-mutating-assign
@@ -276,6 +280,7 @@ const tExecBuild = async function ({
     errorParams,
     logs,
     debug,
+    systemLog,
     verbose,
     timers: timersA,
     sendStatus,
@@ -325,6 +330,7 @@ const runAndReportBuild = async function ({
   errorParams,
   logs,
   debug,
+  systemLog,
   verbose,
   timers,
   sendStatus,
@@ -365,6 +371,7 @@ const runAndReportBuild = async function ({
       errorParams,
       logs,
       debug,
+      systemLog,
       verbose,
       timers,
       sendStatus,
@@ -457,6 +464,7 @@ const initAndRunBuild = async function ({
   errorParams,
   logs,
   debug,
+  systemLog,
   verbose,
   sendStatus,
   saveConfig,
@@ -528,6 +536,7 @@ const initAndRunBuild = async function ({
       errorParams,
       logs,
       debug,
+      systemLog,
       verbose,
       saveConfig,
       timers: timersB,
@@ -581,6 +590,7 @@ const runBuild = async function ({
   errorParams,
   logs,
   debug,
+  systemLog,
   verbose,
   saveConfig,
   timers,
@@ -634,6 +644,7 @@ const runBuild = async function ({
     configOpts,
     logs,
     debug,
+    systemLog,
     verbose,
     saveConfig,
     timers: timersA,

package/src/error/monitor/normalize.js

@@ -3,7 +3,15 @@
 // them consistent
 export const normalizeGroupingMessage = function (message, type) {
   const messageA = removeDependenciesLogs(message, type)
-  return NORMALIZE_REGEXPS.reduce(normalizeMessage, messageA)
+  const messageB = NORMALIZE_REGEXPS.reduce(normalizeMessage, messageA)
+
+  // If this is a functions bundling error, we'll use additional normalization
+  // rules to group errors more aggressively.
+  if (type === 'functionsBundling') {
+    return FUNCTIONS_BUNDLING_REGEXPS.reduce(normalizeMessage, messageB)
+  }
+
+  return messageB
 }
 
 // Discard debug/info installation information
@@ -32,9 +40,9 @@ const normalizeMessage = function (message, [regExp, replacement]) {
 
 const NORMALIZE_REGEXPS = [
   // Base64 URL
-  [/(data:[^;]+;base64),[\w+/-]+/g, 'dataURI'],
+  [/(data:[^;]+;base64),[\w+/-=]+/g, 'dataURI'],
   // File paths
-  [/(["'`, ]|^)([^"'`, \n]*[/\\][^"'`, \n]*)(["'`, ]|$)/gm, '$1/file/path$3'],
+  [/(["'`, ]|^)([^"'`, \n]*[/\\][^"'`, \n]*)(?=["'`, ]|$)/gm, '$1/file/path'],
   // Semantic versions
   [/\d+\.\d+\.\d+(-\d+)?/g, '1.0.0'],
   [/version "[^"]+"/g, 'version "1.0.0"'],
@@ -45,7 +53,7 @@ const NORMALIZE_REGEXPS = [
   // Numbers, e.g. number of issues/problems
   [/\d+/g, '0'],
   // Hexadecimal strings
-  [/[\da-fA-F]{6,}/g, 'hex'],
+  [/[\da-fA-F-]{6,}/g, 'hex'],
   // On unknown inputs, we print the inputs
   [/(does not accept any inputs but you specified: ).*/, '$1'],
   [/(Unknown inputs for plugin).*/, '$1'],
@@ -75,3 +83,14 @@ const NORMALIZE_REGEXPS = [
   // Multiple empty lines
   [/^\s*$/gm, ''],
 ]
+
+const FUNCTIONS_BUNDLING_REGEXPS = [
+  // String literals and identifiers
+  [/"([^"]+)"/g, '""'],
+  [/'([^']+)'/g, "''"],
+  [/`([^`]+)`/g, '``'],
+
+  // Rust crates
+  [/(?:Downloaded \S+ v[\d.]+\s*)+/gm, 'Downloaded crates'],
+  [/(?:Compiling \S+ v[\d.]+\s*)+/gm, 'Compiled crates'],
+]

package/src/error/monitor/report.js

@@ -21,7 +21,7 @@ export const reportBuildError = async function ({ error, errorMonitor, childEnv,
   const { errorInfo, type, severity, title, group = title } = parseErrorInfo(error)
   const severityA = getSeverity(severity, errorInfo)
   const groupA = getGroup(group, errorInfo)
-  const groupingHash = getGroupingHash(groupA, error, type)
+  const groupingHash = getGroupingHash(groupA, error, type, errorInfo)
   const metadata = getMetadata(errorInfo, childEnv, groupingHash)
   const app = getApp()
   const eventProps = getEventProps({ severity: severityA, group: groupA, groupingHash, metadata, app })
@@ -52,7 +52,12 @@ const getGroup = function (group, errorInfo) {
   return group(errorInfo)
 }
 
-const getGroupingHash = function (group, error, type) {
+const getGroupingHash = function (group, error, type, errorInfo = {}) {
+  // If the error has a `normalizedMessage`, we use it as the grouping hash.
+  if (errorInfo.normalizedMessage) {
+    return errorInfo.normalizedMessage
+  }
+
   const message = error instanceof Error && typeof error.message === 'string' ? error.message : String(error)
   const messageA = normalizeGroupingMessage(message, type)
   return `${group}\n${messageA}`

package/src/error/parse/location.js

@@ -23,8 +23,12 @@ const getBuildCommandLocation = function ({ buildCommand, buildCommandOrigin })
 ${buildCommand}`
 }
 
-const getFunctionsBundlingLocation = function ({ functionName }) {
-  return `While bundling Function "${functionName}"`
+const getFunctionsBundlingLocation = function ({ functionName, functionType }) {
+  if (functionType === 'edge') {
+    return 'While bundling edge function'
+  }
+
+  return `While bundling function "${functionName}"`
 }
 
 const getCoreStepLocation = function ({ coreStepName }) {

package/src/error/type.js

@@ -81,7 +81,14 @@ const TYPES = {
 
   // User error during Functions bundling
   functionsBundling: {
-    title: ({ location: { functionName } }) => `Bundling of Function "${functionName}" failed`,
+    title: ({ location: { functionName, functionType } }) => {
+      if (functionType === 'edge') {
+        return 'Bundling of edge function failed'
+      }
+
+      return `Bundling of function "${functionName}" failed`
+    },
+    group: ({ location: { functionType = 'serverless' } }) => `Bundling of ${functionType} function failed`,
     stackType: 'none',
     locationType: 'functionsBundling',
     severity: 'info',

package/src/log/logger.js

@@ -1,3 +1,5 @@
+import { createWriteStream } from 'fs'
+
 import figures from 'figures'
 import indentString from 'indent-string'
 
@@ -105,3 +107,36 @@ export const logErrorSubHeader = function (logs, string, opts) {
 export const logWarningSubHeader = function (logs, string, opts) {
   logSubHeader(logs, string, { color: THEME.warningSubHeader, ...opts })
 }
+
+// Combines an array of elements into a single string, separated by a space,
+// and with basic serialization of non-string types
+const reduceLogLines = function (lines) {
+  return lines
+    .map((input) => {
+      if (typeof input === 'object') {
+        return JSON.stringify(input)
+      }
+
+      return input.toString()
+    })
+    .join(' ')
+}
+
+// Builds a function for logging data to the system logger (i.e. hidden from
+// the user-facing build logs)
+export const getSystemLogger = function (logs, debug, systemLogFile) {
+  // If there's not a file descriptor (or it's set to 0, matching stdout), we
+  // send debug lines to the normal logger. The same happens if the `debug`
+  // flag is used, since that means we want to print logs to stdout.
+  if (!systemLogFile || debug) {
+    return (...args) => log(logs, reduceLogLines(args))
+  }
+
+  const fileDescriptor = createWriteStream(null, { fd: systemLogFile })
+
+  fileDescriptor.on('error', () => {
+    logError(logs, 'Could not write to system log file')
+  })
+
+  return (...args) => fileDescriptor.write(reduceLogLines(args))
+}

package/src/plugins/child/status.js

@@ -6,18 +6,19 @@ import { addErrorInfo } from '../../error/info.js'
 // Report status information to the UI
 export const show = function (runState, showArgs) {
   validateShowArgs(showArgs)
-  const { title, summary, text } = removeEmptyStrings(showArgs)
-  runState.status = { state: 'success', title, summary, text }
+  const { title, summary, text, extraData } = removeEmptyStrings(showArgs)
+  runState.status = { state: 'success', title, summary, text, extraData }
 }
 
 // Validate arguments of `utils.status.show()`
 const validateShowArgs = function (showArgs) {
   try {
     validateShowArgsObject(showArgs)
-    const { title, summary, text, ...otherArgs } = showArgs
+    const { title, summary, text, extraData, ...otherArgs } = showArgs
     validateShowArgsKeys(otherArgs)
     Object.entries({ title, summary, text }).forEach(validateStringArg)
     validateShowArgsSummary(summary)
+    validateShowArgsExtraData(extraData)
   } catch (error) {
     error.message = `utils.status.show() ${error.message}`
     addErrorInfo(error, { type: 'pluginValidation' })
@@ -54,6 +55,12 @@ const validateShowArgsSummary = function (summary) {
   }
 }
 
+const validateShowArgsExtraData = function (extraData) {
+  if (extraData !== undefined && Array.isArray(extraData) === false) {
+    throw new TypeError('provided extra data must be an array')
+  }
+}
+
 const removeEmptyStrings = function (showArgs) {
   return mapObj(showArgs, removeEmptyString)
 }

package/src/plugins_core/edge_functions/index.js

@@ -6,12 +6,15 @@ import { pathExists } from 'path-exists'
 
 import { logFunctionsToBundle } from '../../log/messages/core_steps.js'
 
+import { tagBundlingError } from './lib/error.js'
 import { parseManifest } from './lib/internal_manifest.js'
+import { validateEdgeFunctionsManifest } from './validate_manifest/validate_edge_functions_manifest.js'
 
 // TODO: Replace this with a custom cache directory.
 const DENO_CLI_CACHE_DIRECTORY = '.netlify/plugins/deno-cli'
 const IMPORT_MAP_FILENAME = 'edge-functions-import-map.json'
 
+// eslint-disable-next-line max-statements
 const coreStep = async function ({
   buildDir,
   constants: {
@@ -21,6 +24,7 @@ const coreStep = async function ({
     IS_LOCAL: isRunningLocally,
   },
   debug,
+  systemLog,
   featureFlags,
   logs,
   netlifyConfig,
@@ -45,13 +49,24 @@ const coreStep = async function ({
   // Edge Bundler expects the dist directory to exist.
   await fs.mkdir(distPath, { recursive: true })
 
-  await bundle(sourcePaths, distPath, declarations, {
-    cacheDirectory,
-    debug,
-    distImportMapPath,
-    featureFlags,
-    importMaps: [importMap].filter(Boolean),
-  })
+  try {
+    const { manifest } = await bundle(sourcePaths, distPath, declarations, {
+      basePath: buildDir,
+      cacheDirectory,
+      debug,
+      distImportMapPath,
+      featureFlags,
+      importMaps: [importMap].filter(Boolean),
+    })
+
+    systemLog('Edge Functions manifest:', manifest)
+  } catch (error) {
+    tagBundlingError(error)
+
+    throw error
+  }
+
+  await validateEdgeFunctionsManifest({ buildDir, constants: { EDGE_FUNCTIONS_DIST: distDirectory } })
 
   return {}
 }

package/src/plugins_core/edge_functions/lib/error.js

@@ -0,0 +1,21 @@
+import { CUSTOM_ERROR_KEY, getErrorInfo, isBuildError } from '../../../error/info.js'
+
+// If we have a custom error tagged with `functionsBundling` (which happens if
+// there is an issue with user code), we tag it as coming from an edge function
+// so that we can adjust the downstream error messages accordingly.
+export const tagBundlingError = (error) => {
+  if (!isBuildError(error)) {
+    return
+  }
+
+  const [errorInfo = {}] = getErrorInfo(error)
+
+  if (errorInfo.type !== 'functionsBundling') {
+    return
+  }
+
+  error[CUSTOM_ERROR_KEY].location = {
+    ...error[CUSTOM_ERROR_KEY].location,
+    functionType: 'edge',
+  }
+}

package/src/plugins_core/edge_functions/validate_manifest/validate_edge_functions_manifest.js

@@ -0,0 +1,89 @@
+import { promises as fs } from 'fs'
+import { join, resolve } from 'path'
+
+import Ajv from 'ajv'
+import ajvErrors from 'ajv-errors'
+
+import { addErrorInfo } from '../../../error/info.js'
+
+const ajv = new Ajv({ allErrors: true })
+ajvErrors(ajv)
+
+// regex pattern for manifest route pattern
+// checks if the pattern string starts with ^ and ends with $
+// we define this format in edge-bundler:
+// https://github.com/netlify/edge-bundler/blob/main/src/manifest.ts#L66
+const normalizedPatternRegex = /^\^.*\$$/
+ajv.addFormat('regexPattern', {
+  async: true,
+  validate: (data) => normalizedPatternRegex.test(data),
+})
+
+const bundlesSchema = {
+  $async: true,
+  type: 'object',
+  required: ['asset', 'format'],
+  properties: {
+    asset: { type: 'string' },
+    format: { type: 'string' },
+  },
+  additionalProperties: false,
+}
+
+const routesSchema = {
+  $async: true,
+  type: 'object',
+  required: ['function', 'pattern'],
+  properties: {
+    function: { type: 'string' },
+    pattern: { type: 'string', format: 'regexPattern', errorMessage: `must match format ${normalizedPatternRegex}` },
+  },
+  additionalProperties: false,
+}
+
+const edgeManifestSchema = {
+  $async: true,
+  type: 'object',
+  required: ['bundles', 'routes', 'bundler_version'],
+  properties: {
+    bundles: {
+      type: 'array',
+      items: bundlesSchema,
+    },
+    routes: {
+      type: 'array',
+      items: routesSchema,
+    },
+    bundler_version: { type: 'string' },
+  },
+  additionalProperties: false,
+  errorMessage: "Couldn't validate Edge Functions manifest.json",
+}
+
+const validateManifest = async (manifestData) => {
+  const validate = ajv.compile(edgeManifestSchema)
+
+  await validate(manifestData)
+}
+
+export const validateEdgeFunctionsManifest = async function ({
+  buildDir,
+  constants: { EDGE_FUNCTIONS_DIST: distDirectory },
+}) {
+  try {
+    const edgeFunctionsDistPath = resolve(buildDir, distDirectory)
+    const manifestPath = join(edgeFunctionsDistPath, 'manifest.json')
+    const data = await fs.readFile(manifestPath)
+    const manifestData = JSON.parse(data)
+
+    await validateManifest(manifestData)
+  } catch (error) {
+    const isValidationErr = error instanceof Ajv.ValidationError
+    const parsedErr = isValidationErr ? error.errors : error
+
+    addErrorInfo(parsedErr, { type: 'coreStep' })
+    throw new Error(isValidationErr ? JSON.stringify(parsedErr, null, 2) : parsedErr)
+  }
+
+  return {}
+}

package/src/status/report.js

@@ -104,7 +104,7 @@ const sendApiStatuses = async function ({
 
 const sendApiStatus = async function ({
   api,
-  status: { packageName, version, state, event, title, summary, text },
+  status: { packageName, version, state, event, title, summary, text, extraData },
   childEnv,
   mode,
   netlifyConfig,
@@ -117,7 +117,16 @@ const sendApiStatus = async function ({
   try {
     await api.createPluginRun({
       deploy_id: deployId,
-      body: { package: packageName, version, state, reporting_event: event, title, summary, text },
+      body: {
+        package: packageName,
+        version,
+        state,
+        reporting_event: event,
+        title,
+        summary,
+        text,
+        extra_data: extraData,
+      },
     })
     // Bitballoon API randomly fails with 502.
     // Builds should be successful when this API call fails, but we still want

package/src/steps/core_step.js

@@ -27,6 +27,7 @@ export const fireCoreStep = async function ({
   redirectsPath,
   featureFlags,
   debug,
+  systemLog,
   saveConfig,
 }) {
   try {
@@ -54,6 +55,7 @@ export const fireCoreStep = async function ({
       redirectsPath,
       featureFlags,
       debug,
+      systemLog,
       saveConfig,
     })
     const {

package/src/steps/run_step.js

@@ -48,6 +48,7 @@ export const runStep = async function ({
   redirectsPath,
   logs,
   debug,
+  systemLog,
   verbose,
   saveConfig,
   timers,
@@ -109,6 +110,7 @@ export const runStep = async function ({
     error,
     logs,
     debug,
+    systemLog,
     verbose,
     saveConfig,
     timers,
@@ -140,6 +142,7 @@ export const runStep = async function ({
     redirectsPath: redirectsPathA,
     logs,
     debug,
+    systemLog,
     timers: timersA,
     durationNs,
     testOpts,
@@ -237,6 +240,7 @@ const tFireStep = function ({
   error,
   logs,
   debug,
+  systemLog,
   verbose,
   saveConfig,
   errorParams,
@@ -271,6 +275,7 @@ const tFireStep = function ({
       redirectsPath,
       featureFlags,
       debug,
+      systemLog,
       saveConfig,
     })
   }
@@ -294,6 +299,7 @@ const tFireStep = function ({
     error,
     logs,
     debug,
+    systemLog,
     verbose,
   })
 }

package/src/steps/run_steps.js

@@ -34,6 +34,7 @@ export const runSteps = async function ({
   configOpts,
   logs,
   debug,
+  systemLog,
   verbose,
   saveConfig,
   timers,
@@ -127,6 +128,7 @@ export const runSteps = async function ({
         redirectsPath: redirectsPathA,
         logs,
         debug,
+        systemLog,
         verbose,
         saveConfig,
         timers: timersA,