@netlify/build 27.18.6-rc → 27.18.6

package/lib/plugins_core/build_command.js

@@ -0,0 +1,75 @@
+import { platform } from 'process'
+
+import { execa } from 'execa'
+
+import { addErrorInfo } from '../error/info.js'
+import { getBuildCommandDescription } from '../log/description.js'
+import { logBuildCommandStart } from '../log/messages/steps.js'
+import { getBuildCommandStdio, handleBuildCommandOutput } from '../log/stream.js'
+
+// Fire `build.command`
+const coreStep = async function ({
+  configPath,
+  buildDir,
+  nodePath,
+  childEnv,
+  logs,
+  netlifyConfig: {
+    build: { command: buildCommand, commandOrigin: buildCommandOrigin },
+  },
+}) {
+  logBuildCommandStart(logs, buildCommand)
+
+  const stdio = getBuildCommandStdio(logs)
+  const childProcess = execa(buildCommand, {
+    shell: SHELL,
+    cwd: buildDir,
+    preferLocal: true,
+    execPath: nodePath,
+    env: childEnv,
+    extendEnv: false,
+    stdio,
+  })
+
+  try {
+    const buildCommandOutput = await childProcess
+    handleBuildCommandOutput(buildCommandOutput, logs)
+    return {}
+  } catch (error) {
+    // In our test environment we use `stdio: 'pipe'` on the build command, meaning our `stdout/stderr` output are
+    // buffered and consequently added to `error.message`. To avoid this and end up with duplicated output in our
+    // logs/snapshots we need to rely on `error.shortMessage`.
+    error.message = error.shortMessage
+    handleBuildCommandOutput(error, logs)
+    addErrorInfo(error, { type: 'buildCommand', location: { buildCommand, buildCommandOrigin, configPath } })
+    throw error
+  }
+}
+
+// We use Bash on Unix and `cmd.exe` on Windows
+const SHELL = platform === 'win32' ? true : 'bash'
+
+const coreStepDescription = function ({
+  netlifyConfig: {
+    build: { commandOrigin: buildCommandOrigin },
+  },
+}) {
+  return getBuildCommandDescription(buildCommandOrigin)
+}
+
+const hasBuildCommand = function ({
+  netlifyConfig: {
+    build: { command: buildCommand },
+  },
+}) {
+  return buildCommand !== undefined && buildCommand !== ''
+}
+
+export const buildCommandCore = {
+  event: 'onBuild',
+  coreStep,
+  coreStepId: 'build_command',
+  coreStepName: 'build.command',
+  coreStepDescription,
+  condition: hasBuildCommand,
+}

package/lib/plugins_core/deploy/buildbot_client.js

@@ -0,0 +1,113 @@
+import net from 'net'
+import { normalize, resolve, relative } from 'path'
+import { promisify } from 'util'
+
+import { pEvent } from 'p-event'
+
+import { addErrorInfo } from '../../error/info.js'
+import { runsAfterDeploy } from '../../plugins/events.js'
+import { addAsyncErrorMessage } from '../../utils/errors.js'
+
+export const createBuildbotClient = function (buildbotServerSocket) {
+  const connectionOpts = getConnectionOpts(buildbotServerSocket)
+  const client = net.createConnection(connectionOpts)
+  return client
+}
+
+// Windows does not support Unix sockets well, so we also support `host:port`
+const getConnectionOpts = function (buildbotServerSocket) {
+  if (!buildbotServerSocket.includes(':')) {
+    return { path: buildbotServerSocket }
+  }
+
+  const [host, port] = buildbotServerSocket.split(':')
+  return { host, port }
+}
+
+const eConnectBuildbotClient = async function (client) {
+  await pEvent(client, 'connect')
+}
+
+export const connectBuildbotClient = addAsyncErrorMessage(eConnectBuildbotClient, 'Could not connect to buildbot')
+
+export const closeBuildbotClient = async function (client) {
+  if (client.destroyed) {
+    return
+  }
+
+  await promisify(client.end.bind(client))()
+}
+
+const cWritePayload = async function (buildbotClient, payload) {
+  await promisify(buildbotClient.write.bind(buildbotClient))(JSON.stringify(payload))
+}
+
+const writePayload = addAsyncErrorMessage(cWritePayload, 'Could not send payload to buildbot')
+
+const cGetNextParsedResponsePromise = async function (buildbotClient) {
+  const data = await pEvent(buildbotClient, 'data')
+  return JSON.parse(data)
+}
+
+const getNextParsedResponsePromise = addAsyncErrorMessage(
+  cGetNextParsedResponsePromise,
+  'Invalid response from buildbot',
+)
+
+export const deploySiteWithBuildbotClient = async function ({ client, events, buildDir, repositoryRoot, constants }) {
+  const action = shouldWaitForPostProcessing(events) ? 'deploySiteAndAwaitLive' : 'deploySite'
+  const deployDir = getDeployDir({ buildDir, repositoryRoot, constants })
+  const payload = { action, deployDir }
+
+  const [{ succeeded, values: { error, error_type: errorType } = {} }] = await Promise.all([
+    getNextParsedResponsePromise(client),
+    writePayload(client, payload),
+  ])
+
+  if (!succeeded) {
+    return handleDeployError(error, errorType)
+  }
+}
+
+// The file paths in the buildbot are relative to the repository root.
+// However, the file paths in Build plugins, including `constants.PUBLISH_DIR`
+// are relative to the build directory, which is different when there is a
+// base directory. This converts it.
+// We need to call `normalize()` in case the publish directory is the
+// repository root, so `deployDir` is "." not ""
+const getDeployDir = function ({ buildDir, repositoryRoot, constants: { PUBLISH_DIR } }) {
+  const absolutePublishDir = resolve(buildDir, PUBLISH_DIR)
+  const relativePublishDir = relative(repositoryRoot, absolutePublishDir)
+  const deployDir = normalize(relativePublishDir)
+  return deployDir
+}
+
+// We distinguish between user errors and system errors during deploys
+const handleDeployError = function (error, errorType) {
+  const errorIs422 = error !== undefined && error.code === '422'
+
+  const errMsg = errorIs422
+    ? `
+    File upload failed because of mismatched SHAs.
+    This can happen when files are changed after the deployment process has started but before they are uploaded.
+
+    Error: ${error}
+    `
+    : `Deploy did not succeed: ${error}`
+
+  const errorA = new Error(errMsg)
+  const errorInfo =
+    errorType === 'user' ? { type: 'resolveConfig' } : { type: 'coreStep', location: { coreStepName: 'Deploy site' } }
+  addErrorInfo(errorA, errorInfo)
+  throw errorA
+}
+
+// We only wait for post-processing (last stage before site deploy) if the build
+// has some plugins that do post-deploy logic
+const shouldWaitForPostProcessing = function (events) {
+  return events.some(hasPostDeployLogic)
+}
+
+const hasPostDeployLogic = function (event) {
+  return runsAfterDeploy(event)
+}

package/lib/plugins_core/deploy/index.js

@@ -0,0 +1,73 @@
+import { saveUpdatedConfig, restoreUpdatedConfig } from '../../core/config.js'
+import { logDeploySuccess } from '../../log/messages/plugins.js'
+
+import {
+  createBuildbotClient,
+  connectBuildbotClient,
+  closeBuildbotClient,
+  deploySiteWithBuildbotClient,
+} from './buildbot_client.js'
+
+const coreStep = async function ({
+  buildDir,
+  configPath,
+  repositoryRoot,
+  constants,
+  buildbotServerSocket,
+  events,
+  logs,
+  featureFlags,
+  context,
+  branch,
+  configMutations,
+  headersPath,
+  redirectsPath,
+  debug,
+  saveConfig,
+}) {
+  const client = createBuildbotClient(buildbotServerSocket)
+  try {
+    await connectBuildbotClient(client)
+    await saveUpdatedConfig({
+      configMutations,
+      buildDir,
+      repositoryRoot,
+      configPath,
+      headersPath,
+      redirectsPath,
+      logs,
+      featureFlags,
+      context,
+      branch,
+      debug,
+      saveConfig,
+    })
+    await deploySiteWithBuildbotClient({ client, events, buildDir, repositoryRoot, constants })
+    await restoreUpdatedConfig({
+      configMutations,
+      buildDir,
+      repositoryRoot,
+      configPath,
+      headersPath,
+      redirectsPath,
+      saveConfig,
+    })
+    logDeploySuccess(logs)
+    return {}
+  } finally {
+    await closeBuildbotClient(client)
+  }
+}
+
+const shouldDeploy = function ({ buildbotServerSocket }) {
+  return buildbotServerSocket !== undefined
+}
+
+export const deploySite = {
+  event: 'onPostBuild',
+  coreStep,
+  coreStepId: 'deploy_site',
+  coreStepName: 'Deploy site',
+  coreStepDescription: () => 'Deploy site',
+  condition: shouldDeploy,
+}

package/lib/plugins_core/deploy/manifest.yml

@@ -0,0 +1 @@
+name: '@netlify/plugin-deploy-core'

package/lib/plugins_core/edge_functions/index.js

@@ -0,0 +1,123 @@
+import { promises as fs } from 'fs'
+import { dirname, join, resolve } from 'path'
+
+import { bundle, find } from '@netlify/edge-bundler'
+import { pathExists } from 'path-exists'
+
+import { logFunctionsToBundle } from '../../log/messages/core_steps.js'
+
+import { tagBundlingError } from './lib/error.js'
+import { parseManifest } from './lib/internal_manifest.js'
+import { validateEdgeFunctionsManifest } from './validate_manifest/validate_edge_functions_manifest.js'
+
+// TODO: Replace this with a custom cache directory.
+const DENO_CLI_CACHE_DIRECTORY = '.netlify/plugins/deno-cli'
+const IMPORT_MAP_FILENAME = 'edge-functions-import-map.json'
+
+// eslint-disable-next-line complexity, max-statements
+const coreStep = async function ({
+  buildDir,
+  constants: {
+    EDGE_FUNCTIONS_DIST: distDirectory,
+    EDGE_FUNCTIONS_SRC: srcDirectory,
+    INTERNAL_EDGE_FUNCTIONS_SRC: internalSrcDirectory,
+    IS_LOCAL: isRunningLocally,
+  },
+  debug,
+  systemLog,
+  featureFlags,
+  logs,
+  netlifyConfig,
+}) {
+  const { edge_functions: configDeclarations = [] } = netlifyConfig
+  const distPath = resolve(buildDir, distDirectory)
+  const internalSrcPath = resolve(buildDir, internalSrcDirectory)
+  const distImportMapPath = join(dirname(internalSrcPath), IMPORT_MAP_FILENAME)
+  const srcPath = srcDirectory ? resolve(buildDir, srcDirectory) : undefined
+  const sourcePaths = [internalSrcPath, srcPath].filter(Boolean)
+
+  logFunctions({ internalSrcDirectory, internalSrcPath, logs, srcDirectory, srcPath })
+
+  const { declarations: internalDeclarations, importMap } = await parseManifest(internalSrcPath, systemLog)
+  const declarations = [...configDeclarations, ...internalDeclarations]
+
+  // If we're running in buildbot and the feature flag is enabled, we set the
+  // Deno cache dir to a directory that is persisted between builds.
+  const cacheDirectory =
+    !isRunningLocally && featureFlags.edge_functions_cache_cli ? resolve(buildDir, DENO_CLI_CACHE_DIRECTORY) : undefined
+
+  // Edge Bundler expects the dist directory to exist.
+  await fs.mkdir(distPath, { recursive: true })
+
+  try {
+    const { manifest } = await bundle(sourcePaths, distPath, declarations, {
+      basePath: buildDir,
+      cacheDirectory,
+      debug,
+      distImportMapPath,
+      featureFlags,
+      importMaps: [importMap].filter(Boolean),
+      systemLogger: featureFlags.edge_functions_system_logger ? systemLog : undefined,
+    })
+
+    systemLog('Edge Functions manifest:', manifest)
+  } catch (error) {
+    tagBundlingError(error)
+
+    throw error
+  }
+
+  await validateEdgeFunctionsManifest({ buildDir, constants: { EDGE_FUNCTIONS_DIST: distDirectory } })
+
+  return {}
+}
+
+// We run this core step if at least one of the functions directories (the
+// one configured by the user or the internal one) exists. We use a dynamic
+// `condition` because the directories might be created by the build command
+// or plugins.
+const hasEdgeFunctionsDirectories = async function ({
+  buildDir,
+  constants: { INTERNAL_EDGE_FUNCTIONS_SRC, EDGE_FUNCTIONS_SRC },
+}) {
+  const hasFunctionsSrc = EDGE_FUNCTIONS_SRC !== undefined && EDGE_FUNCTIONS_SRC !== ''
+
+  if (hasFunctionsSrc) {
+    return true
+  }
+
+  const internalFunctionsSrc = resolve(buildDir, INTERNAL_EDGE_FUNCTIONS_SRC)
+
+  return await pathExists(internalFunctionsSrc)
+}
+
+const logFunctions = async ({
+  internalSrcDirectory,
+  internalSrcPath,
+  logs,
+  srcDirectory: userFunctionsSrc,
+  srcPath,
+}) => {
+  const [userFunctions, internalFunctions] = await Promise.all([find([srcPath]), find([internalSrcPath])])
+  const userFunctionsSrcExists = await pathExists(srcPath)
+  const internalFunctionsSrc = internalSrcDirectory
+
+  logFunctionsToBundle({
+    logs,
+    userFunctions: userFunctions.map(({ name }) => name),
+    userFunctionsSrc,
+    userFunctionsSrcExists,
+    internalFunctions: internalFunctions.map(({ name }) => name),
+    internalFunctionsSrc,
+    type: 'Edge Functions',
+  })
+}
+
+export const bundleEdgeFunctions = {
+  event: 'onBuild',
+  coreStep,
+  coreStepId: 'edge_functions_bundling',
+  coreStepName: 'Edge Functions bundling',
+  coreStepDescription: () => 'Edge Functions bundling',
+  condition: hasEdgeFunctionsDirectories,
+}

package/lib/plugins_core/edge_functions/lib/error.js

@@ -0,0 +1,21 @@
+import { CUSTOM_ERROR_KEY, getErrorInfo, isBuildError } from '../../../error/info.js'
+
+// If we have a custom error tagged with `functionsBundling` (which happens if
+// there is an issue with user code), we tag it as coming from an edge function
+// so that we can adjust the downstream error messages accordingly.
+export const tagBundlingError = (error) => {
+  if (!isBuildError(error)) {
+    return
+  }
+
+  const [errorInfo = {}] = getErrorInfo(error)
+
+  if (errorInfo.type !== 'functionsBundling') {
+    return
+  }
+
+  error[CUSTOM_ERROR_KEY].location = {
+    ...error[CUSTOM_ERROR_KEY].location,
+    functionType: 'edge',
+  }
+}

package/lib/plugins_core/edge_functions/lib/internal_manifest.js

@@ -0,0 +1,60 @@
+import { promises as fs } from 'fs'
+import { dirname, join, resolve } from 'path'
+import { pathToFileURL } from 'url'
+
+const parseManifest = async (internalSourceDirectory, systemLog) => {
+  const manifestPath = join(internalSourceDirectory, 'manifest.json')
+
+  try {
+    const data = await fs.readFile(manifestPath)
+    const manifest = JSON.parse(data)
+
+    if (manifest.version !== 1) {
+      throw new Error('Unsupported manifest version')
+    }
+
+    const result = {
+      declarations: manifest.functions,
+    }
+
+    if (manifest.import_map) {
+      const importMapPath = resolve(dirname(manifestPath), manifest.import_map)
+      const importMap = await readImportMap(importMapPath)
+
+      return {
+        ...result,
+        importMap: {
+          baseURL: pathToFileURL(importMapPath),
+          ...importMap,
+        },
+      }
+    }
+
+    return result
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      systemLog('Error while parsing internal edge functions manifest:', error)
+    }
+  }
+
+  return {
+    declarations: [],
+  }
+}
+
+const readImportMap = async (path) => {
+  try {
+    const data = await fs.readFile(path)
+    const importMap = JSON.parse(data)
+
+    return importMap
+  } catch {
+    // no-op
+  }
+
+  return {
+    imports: {},
+  }
+}
+
+export { parseManifest }

package/lib/plugins_core/edge_functions/validate_manifest/validate_edge_functions_manifest.js

@@ -0,0 +1,89 @@
+import { promises as fs } from 'fs'
+import { join, resolve } from 'path'
+
+import Ajv from 'ajv'
+import ajvErrors from 'ajv-errors'
+
+import { addErrorInfo } from '../../../error/info.js'
+
+const ajv = new Ajv({ allErrors: true })
+ajvErrors(ajv)
+
+// regex pattern for manifest route pattern
+// checks if the pattern string starts with ^ and ends with $
+// we define this format in edge-bundler:
+// https://github.com/netlify/edge-bundler/blob/main/src/manifest.ts#L66
+const normalizedPatternRegex = /^\^.*\$$/
+ajv.addFormat('regexPattern', {
+  async: true,
+  validate: (data) => normalizedPatternRegex.test(data),
+})
+
+const bundlesSchema = {
+  $async: true,
+  type: 'object',
+  required: ['asset', 'format'],
+  properties: {
+    asset: { type: 'string' },
+    format: { type: 'string' },
+  },
+  additionalProperties: false,
+}
+
+const routesSchema = {
+  $async: true,
+  type: 'object',
+  required: ['function', 'pattern'],
+  properties: {
+    function: { type: 'string' },
+    pattern: { type: 'string', format: 'regexPattern', errorMessage: `must match format ${normalizedPatternRegex}` },
+  },
+  additionalProperties: false,
+}
+
+const edgeManifestSchema = {
+  $async: true,
+  type: 'object',
+  required: ['bundles', 'routes', 'bundler_version'],
+  properties: {
+    bundles: {
+      type: 'array',
+      items: bundlesSchema,
+    },
+    routes: {
+      type: 'array',
+      items: routesSchema,
+    },
+    bundler_version: { type: 'string' },
+  },
+  additionalProperties: false,
+  errorMessage: "Couldn't validate Edge Functions manifest.json",
+}
+
+const validateManifest = async (manifestData) => {
+  const validate = ajv.compile(edgeManifestSchema)
+
+  await validate(manifestData)
+}
+
+export const validateEdgeFunctionsManifest = async function ({
+  buildDir,
+  constants: { EDGE_FUNCTIONS_DIST: distDirectory },
+}) {
+  try {
+    const edgeFunctionsDistPath = resolve(buildDir, distDirectory)
+    const manifestPath = join(edgeFunctionsDistPath, 'manifest.json')
+    const data = await fs.readFile(manifestPath)
+    const manifestData = JSON.parse(data)
+
+    await validateManifest(manifestData)
+  } catch (error) {
+    const isValidationErr = error instanceof Ajv.ValidationError
+    const parsedErr = isValidationErr ? error.errors : error
+
+    addErrorInfo(parsedErr, { type: 'coreStep' })
+    throw new Error(isValidationErr ? JSON.stringify(parsedErr, null, 2) : parsedErr)
+  }
+
+  return {}
+}