@netlify/build 0.5.0 → 1.0.0-dl-test

package/src/plugins/manifest/validate.js

@@ -1,136 +0,0 @@
-const isPlainObj = require('is-plain-obj')
-
-const { serializeArray } = require('../../log/serialize')
-const { THEME } = require('../../log/theme')
-const { API_METHODS } = require('../child/api')
-
-// Validate `manifest.yml` syntax
-const validateManifest = function(manifest, rawManifest) {
-  try {
-    validateBasic(manifest)
-    validateUnknownProps(manifest)
-    validateName(manifest)
-    validateScopes(manifest)
-    validateInputs(manifest)
-  } catch (error) {
-    error.message = `Plugin's "manifest.yml" ${error.message}
-
-${THEME.errorSubHeader('manifest.yml')}
-${rawManifest.trim()}`
-    throw error
-  }
-}
-
-const validateBasic = function(manifest) {
-  if (!isPlainObj(manifest)) {
-    throw new Error('must be a plain object')
-  }
-}
-
-const validateUnknownProps = function(manifest) {
-  const unknownProp = Object.keys(manifest).find(key => !VALID_PROPS.includes(key))
-  if (unknownProp !== undefined) {
-    throw new Error(`unknown property "${unknownProp}"`)
-  }
-}
-
-const VALID_PROPS = ['name', 'inputs', 'scopes']
-
-const validateName = function({ name }) {
-  if (name === undefined) {
-    throw new Error('must contain a "name" property')
-  }
-
-  if (typeof name !== 'string') {
-    throw new Error('"name" property must be a string')
-  }
-}
-
-const validateScopes = function({ scopes }) {
-  if (scopes === undefined) {
-    return
-  }
-
-  if (!Array.isArray(scopes)) {
-    throw new Error('"scopes" property must be an array')
-  }
-
-  const wrongScope = scopes.find(scope => !isValidScope(scope))
-  if (wrongScope !== undefined) {
-    throw new Error(`scope "${wrongScope}" is invalid
-Please use a valid scope. One of:
-${serializeArray(ALLOWED_SCOPES)}`)
-  }
-}
-
-const isValidScope = function(scope) {
-  return ALLOWED_SCOPES.includes(scope)
-}
-
-const ALLOWED_SCOPES = ['*', ...API_METHODS]
-
-const validateInputs = function({ inputs }) {
-  if (inputs === undefined) {
-    return
-  }
-
-  if (!isArrayOfObjects(inputs)) {
-    throw new Error('"inputs" property must be an array of objects')
-  }
-
-  inputs.forEach(validateInput)
-}
-
-const isArrayOfObjects = function(objects) {
-  return Array.isArray(objects) && objects.every(isPlainObj)
-}
-
-const validateInput = function(input, index) {
-  try {
-    validateUnknownInputProps(input)
-    validateInputName(input)
-    validateInputDescription(input)
-    validateInputRequired(input)
-  } catch (error) {
-    error.message = `"inputs" property is invalid.
-Input at position ${index} ${error.message}.`
-    throw error
-  }
-}
-
-const validateUnknownInputProps = function(input) {
-  const unknownProp = Object.keys(input).find(key => !VALID_INPUT_PROPS.includes(key))
-  if (unknownProp !== undefined) {
-    throw new Error(`has an unknown property "${unknownProp}"`)
-  }
-}
-
-const VALID_INPUT_PROPS = ['name', 'description', 'required', 'default']
-
-const validateInputName = function({ name }) {
-  if (name === undefined) {
-    throw new Error('must contain a "name" property')
-  }
-
-  if (typeof name !== 'string') {
-    throw new Error('"name" property must be a string')
-  }
-}
-
-const validateInputDescription = function({ description }) {
-  if (description === undefined) {
-    return
-  }
-
-  if (typeof description !== 'string') {
-    throw new Error('"description" property must be a string')
-  }
-}
-
-const validateInputRequired = function({ required }) {
-  if (required !== undefined && typeof required !== 'boolean') {
-    throw new Error('"required" property must be a boolean')
-  }
-}
-
-module.exports = { validateManifest }

package/src/plugins/node_version.js

@@ -1,74 +0,0 @@
-const { version: currentVersion, execPath } = require('process')
-
-const execa = require('execa')
-const { satisfies, clean: cleanVersion } = require('semver')
-
-const {
-  engines: { node: coreNodeVersionRange },
-} = require('../../package.json')
-const { addErrorInfo } = require('../error/info')
-
-// Retrieve Node.js version from `--node-path`
-const getUserNodeVersion = async function(nodePath) {
-  // No `--node-path` CLI flag
-  if (nodePath === execPath) {
-    return getCurrentNodeVersion()
-  }
-
-  const result = NVM_NODE_VERSION_REGEXP.exec(nodePath)
-  if (result !== null) {
-    return result[1]
-  }
-
-  const { stdout } = await execa(nodePath, ['--version'], { reject: false })
-  const version = cleanVersion(stdout)
-  if (version === null) {
-    throwUserError(`Invalid --node-path CLI flag: ${nodePath}`)
-  }
-
-  return version
-}
-
-// Retrieve Node.js version if the Node.js path is using nvm.
-// `node.exe` on Windows, `bin/node` on Unix.
-const NVM_NODE_VERSION_REGEXP = /[/\\]v(\d+\.\d+\.\d+)[/\\](bin[/\\]node|node.exe)$/
-
-// Retrieve Node.js version from current process
-const getCurrentNodeVersion = function() {
-  return cleanVersion(currentVersion)
-}
-
-// Ensure Node.js version is recent enough to run this plugin
-const checkNodeVersion = async function({
-  childNodeVersion,
-  package,
-  packageJson: { engines: { node: pluginNodeVersionRange } = {} } = {},
-}) {
-  // Build plugins are instrumented with a wrapper code which has a minimal
-  // allowed Node.js version.
-  // A mismatch can only happen in the conditions described below since:
-  //  - production builds use a pinned version, except for local plugins and
-  //    plugins added to `package.json`
-  //  - Netlify CLI has the same minimal allowed Node.js version as
-  //    `@netlify/build`, so if users can run Netlify CLI, everything is good
-  if (!satisfies(childNodeVersion, coreNodeVersionRange)) {
-    throwUserError(`The Node.js version is ${childNodeVersion} but it should be ${coreNodeVersionRange} when using build plugins either:
-  - from the same repository (as opposed to npm modules)
-  - or added to "package.json"`)
-  }
-
-  // Plugins can also set a minimal version using `engines.node`
-  if (pluginNodeVersionRange && !satisfies(childNodeVersion, pluginNodeVersionRange)) {
-    throwUserError(
-      `The Node.js version is ${childNodeVersion} but the plugin "${package}" requires ${pluginNodeVersionRange}`,
-    )
-  }
-}
-
-const throwUserError = function(message) {
-  const error = new Error(message)
-  addErrorInfo(error, { type: 'resolveConfig' })
-  throw error
-}
-
-module.exports = { getUserNodeVersion, getCurrentNodeVersion, checkNodeVersion }

package/src/plugins/options.js

@@ -1,78 +0,0 @@
-const { dirname } = require('path')
-
-const corePackageJson = require('../../package.json')
-const { checkDeprecatedFunctionsInstall } = require('../install/functions')
-const { installLocalPluginsDependencies } = require('../install/local')
-const { getCorePlugins, CORE_PLUGINS, EARLY_CORE_PLUGINS } = require('../plugins_core/main')
-
-const { useManifest } = require('./manifest/main')
-const { getPackageJson } = require('./package')
-const { resolvePluginsPath } = require('./resolve')
-
-// Load plugin options (specified by user in `config.plugins`)
-// Do not allow user override of core plugins
-const getPluginsOptions = async function({
-  netlifyConfig: { plugins },
-  buildDir,
-  constants: { FUNCTIONS_SRC },
-  mode,
-  api,
-}) {
-  const corePlugins = getCorePlugins(FUNCTIONS_SRC).map(addCoreProperties)
-  const allCorePlugins = corePlugins.filter(corePlugin => !isOptionalCore(corePlugin, plugins))
-  const userPlugins = plugins.filter(isUserPlugin)
-  const pluginsOptions = [...allCorePlugins, ...userPlugins].map(normalizePluginOptions)
-  const pluginsOptionsA = await resolvePluginsPath({ pluginsOptions, buildDir, mode })
-  const pluginsOptionsB = await Promise.all(
-    pluginsOptionsA.map(pluginOptions => loadPluginFiles({ pluginOptions, mode, api })),
-  )
-  await Promise.all([
-    checkDeprecatedFunctionsInstall(plugins, FUNCTIONS_SRC, buildDir),
-    installLocalPluginsDependencies({ plugins, pluginsOptions: pluginsOptionsB, buildDir, mode }),
-  ])
-  return pluginsOptionsB
-}
-
-const addCoreProperties = function(corePlugin) {
-  return { ...corePlugin, loadedFrom: 'core', origin: 'core' }
-}
-
-// Optional core plugins requires user opt-in
-const isOptionalCore = function({ package, optional }, plugins) {
-  return optional && plugins.every(plugin => plugin.package !== package)
-}
-
-const isUserPlugin = function({ package }) {
-  return !CORE_PLUGINS.includes(package) && !EARLY_CORE_PLUGINS.includes(package)
-}
-
-const normalizePluginOptions = function({ package, pluginPath, loadedFrom, origin, inputs = {} }) {
-  return { package, pluginPath, loadedFrom, origin, inputs }
-}
-
-// Retrieve plugin's main file path.
-// Then load plugin's `package.json` and `manifest.yml`.
-const loadPluginFiles = async function({ pluginOptions: { pluginPath, ...pluginOptions }, mode, api }) {
-  const pluginDir = dirname(pluginPath)
-  const { packageDir, packageJson } = await getPackageJson(pluginDir)
-  const { manifest, inputs: inputsA } = await useManifest(pluginOptions, {
-    pluginDir,
-    packageDir,
-    packageJson,
-    mode,
-    api,
-  })
-  return { ...pluginOptions, pluginPath, packageDir, packageJson, manifest, inputs: inputsA }
-}
-
-// Retrieve information about @netlify/build when an error happens there and not
-// in a plugin
-const getSpawnInfo = function() {
-  const { name } = corePackageJson
-  return {
-    plugin: { package: name, packageJson: corePackageJson },
-    location: { event: 'load', package: name, loadedFrom: 'core', origin: 'core' },
-  }
-}
-
-module.exports = { getPluginsOptions, getSpawnInfo }

package/src/plugins/package.js

@@ -1,17 +0,0 @@
-const { dirname } = require('path')
-
-const readPkgUp = require('read-pkg-up')
-
-// Retrieve plugin's `package.json`
-const getPackageJson = async function(pluginDir) {
-  const packageObj = await readPkgUp({ cwd: pluginDir })
-  if (packageObj === undefined) {
-    return { packageJson: {} }
-  }
-
-  const { path, packageJson } = packageObj
-  const packageDir = dirname(path)
-  return { packageDir, packageJson }
-}
-
-module.exports = { getPackageJson }

package/src/plugins/resolve.js

@@ -1,159 +0,0 @@
-const { resolve } = require('path')
-const { env } = require('process')
-
-const pathExists = require('path-exists')
-
-const { installMissingPlugins, getAutoPluginsDirPath } = require('../install/missing')
-// const { logMissingPluginsWarning } = require('../log/main')
-const { resolvePath } = require('../utils/resolve')
-
-// Try to find plugins in four places, by priority order:
-//  - already loaded (core plugins)
-//  - local plugin
-//  - external plugin already installed in `node_modules`, most likely through `package.json`
-//  - cached in the build image
-//  - automatically installed by us (fallback)
-const resolvePluginsPath = async function({ pluginsOptions, buildDir, mode }) {
-  const autoPluginsDir = getAutoPluginsDirPath(buildDir)
-  const pluginsOptionsA = await Promise.all(
-    pluginsOptions.map(pluginOptions => resolvePluginPath({ pluginOptions, buildDir, autoPluginsDir, mode })),
-  )
-  await installMissingPlugins({ pluginsOptions: pluginsOptionsA, autoPluginsDir, mode })
-  const pluginsOptionsB = await Promise.all(
-    pluginsOptionsA.map(pluginOptions => resolveMissingPluginPath({ pluginOptions, autoPluginsDir })),
-  )
-  warnOnMissingPlugins(pluginsOptionsB, mode)
-  return pluginsOptionsB
-}
-
-const resolvePluginPath = async function({
-  pluginOptions,
-  pluginOptions: { package, loadedFrom },
-  buildDir,
-  autoPluginsDir,
-  mode,
-}) {
-  // Core plugins
-  if (loadedFrom !== undefined) {
-    return pluginOptions
-  }
-
-  // Local plugins
-  if (package.startsWith('.') || package.startsWith('/')) {
-    const localPath = resolve(buildDir, package)
-    return { ...pluginOptions, pluginPath: localPath, loadedFrom: 'local' }
-  }
-
-  // Plugin already installed in the project, most likely either local plugins,
-  // or external plugins added to `package.json`
-  const manualPath = await tryResolvePath(package, buildDir)
-  if (manualPath !== undefined) {
-    return { ...pluginOptions, pluginPath: manualPath, loadedFrom: 'package.json' }
-  }
-
-  // Cached in the build image
-  const buildImagePath = await tryBuildImagePath(package, mode, buildDir)
-  if (buildImagePath !== undefined) {
-    return { ...pluginOptions, pluginPath: buildImagePath, loadedFrom: 'image_cache' }
-  }
-
-  // Plugin previously automatically installed
-  const automaticPath = await tryAutomaticPath(package, autoPluginsDir)
-  if (automaticPath !== undefined) {
-    return { ...pluginOptions, pluginPath: automaticPath, loadedFrom: 'auto_install' }
-  }
-
-  // Otherwise, it must be automatically installed, as a fallback
-  return pluginOptions
-}
-
-// In production, we pre-install most Build plugins to that directory, for
-// performance reasons
-const tryBuildImagePath = async function(package, mode, buildDir) {
-  if (mode !== 'buildbot') {
-    return
-  }
-
-  const pluginsDir = getBuildImagePluginsDir()
-  const buildImagePath = `${pluginsDir}/${package}`
-  if (!(await pathExists(buildImagePath))) {
-    return
-  }
-
-  return resolvePath(buildImagePath, buildDir)
-}
-
-const getBuildImagePluginsDir = function() {
-  if (env.TEST_BUILD_IMAGE_PLUGINS_DIR) {
-    return env.TEST_BUILD_IMAGE_PLUGINS_DIR
-  }
-
-  return BUILD_IMAGE_PLUGINS_DIR
-}
-
-const BUILD_IMAGE_PLUGINS_DIR = '/opt/buildhome/.netlify-build-plugins/node_modules'
-
-// Try to find plugin previously automatically installed
-const tryAutomaticPath = async function(package, autoPluginsDir) {
-  if (!(await pathExists(autoPluginsDir))) {
-    return
-  }
-
-  return tryResolvePath(package, autoPluginsDir)
-}
-
-// Try to `resolve()` the plugin from the build directory
-const tryResolvePath = async function(package, baseDir) {
-  try {
-    return await resolvePath(package, baseDir)
-  } catch (error) {
-    return
-  }
-}
-
-// Resolve the plugins that just got automatically installed
-const resolveMissingPluginPath = async function({
-  pluginOptions,
-  pluginOptions: { package, pluginPath },
-  autoPluginsDir,
-}) {
-  if (pluginPath !== undefined) {
-    return pluginOptions
-  }
-
-  const automaticPath = await resolvePath(package, autoPluginsDir)
-  return { ...pluginOptions, pluginPath: automaticPath, loadedFrom: 'auto_install' }
-}
-
-// Warns when plugins have been automatically installed. This feature is a
-// fallback that should not be relied upon because:
-//  - it is much slower
-//  - npm can be unreliable
-// Warns both when installing the plugin, and when re-using it in a future build
-// Not done for local builds, since they cannot use the alternative
-// (build-image cached plugins).
-const warnOnMissingPlugins = function(pluginsOptions, mode) {
-  if (mode !== 'buildbot') {
-    return
-  }
-
-  const packages = pluginsOptions.filter(isAutomaticallyInstalled).map(getPackage)
-  if (packages.length === 0) {
-    return
-  }
-
-  // TODO: re-enable warning after the following are available:
-  //  - UI installs
-  //  - build-image pre-installed plugins
-  // logMissingPluginsWarning(packages)
-}
-
-const isAutomaticallyInstalled = function({ loadedFrom }) {
-  return loadedFrom === 'auto_install'
-}
-
-const getPackage = function({ package }) {
-  return package
-}
-
-module.exports = { resolvePluginsPath }

package/src/plugins/spawn.js

@@ -1,82 +0,0 @@
-const { execPath } = require('process')
-
-const execa = require('execa')
-
-const { logLoadingPlugins } = require('../log/main')
-
-const { getEventFromChild } = require('./ipc')
-const { getUserNodeVersion, getCurrentNodeVersion, checkNodeVersion } = require('./node_version')
-const { getSpawnInfo } = require('./options')
-
-const CHILD_MAIN_FILE = `${__dirname}/child/main.js`
-
-// Start child processes used by all plugins
-// We fire plugins through child processes so that:
-//  - each plugin is sandboxed, e.g. cannot access/modify its parent `process`
-//    (for both security and safety reasons)
-//  - logs can be buffered which allows manipulating them for log shipping,
-//    transforming and parallel plugins
-const startPlugins = async function({ pluginsOptions, buildDir, nodePath, childEnv, mode }) {
-  logLoadingPlugins(pluginsOptions)
-
-  const spawnInfo = getSpawnInfo()
-  const userNodeVersion = await getUserNodeVersion(nodePath)
-  return Promise.all(
-    pluginsOptions.map(({ package, packageJson, loadedFrom }) =>
-      startPlugin({ buildDir, nodePath, childEnv, package, packageJson, loadedFrom, mode, spawnInfo, userNodeVersion }),
-    ),
-  )
-}
-
-const startPlugin = async function({
-  buildDir,
-  nodePath,
-  childEnv,
-  package,
-  packageJson,
-  loadedFrom,
-  mode,
-  spawnInfo,
-  userNodeVersion,
-}) {
-  const { childNodePath, childNodeVersion } = getChildNodePath({ loadedFrom, nodePath, userNodeVersion, mode })
-
-  await checkNodeVersion({ childNodeVersion, package, packageJson })
-
-  const childProcess = execa.node(CHILD_MAIN_FILE, {
-    cwd: buildDir,
-    preferLocal: true,
-    nodePath: childNodePath,
-    execPath: childNodePath,
-    env: childEnv,
-    extendEnv: false,
-  })
-  await getEventFromChild(childProcess, 'ready', spawnInfo)
-  return { childProcess }
-}
-
-// Local plugins, `package.json`-installed plugins and local builds use user's
-// preferred Node.js version.
-// Other plugins use `@netlify/build` Node.js version.
-const getChildNodePath = function({ loadedFrom, nodePath, userNodeVersion, mode }) {
-  if (loadedFrom === 'local' || loadedFrom === 'package.json' || (loadedFrom !== 'core' && mode !== 'buildbot')) {
-    return { childNodePath: nodePath, childNodeVersion: userNodeVersion }
-  }
-
-  return { childNodePath: execPath, childNodeVersion: getCurrentNodeVersion() }
-}
-
-// Stop all plugins child processes
-const stopPlugins = async function(childProcesses) {
-  await Promise.all(childProcesses.map(stopPlugin))
-}
-
-const stopPlugin = async function({ childProcess }) {
-  if (childProcess.connected) {
-    childProcess.disconnect()
-  }
-
-  childProcess.kill()
-}
-
-module.exports = { startPlugins, stopPlugins }

package/src/plugins_core/cache/manifest.yml

@@ -1 +0,0 @@
-name: '@netlify/plugin-cache-core'

package/src/plugins_core/cache/plugin.js

@@ -1,75 +0,0 @@
-const { homedir } = require('os')
-const {
-  version,
-  env: { TEST_CACHE_PATH },
-} = require('process')
-
-const { logCacheDir } = require('../../log/main')
-
-// Save/restore cache core plugin
-const cachePlugin = {
-  async onPostBuild({ utils: { cache }, constants: { IS_LOCAL } }) {
-    await Promise.all(ARTIFACTS.map(({ path, digests }) => saveCache({ path, digests, cache, IS_LOCAL })))
-  },
-}
-
-// Cache a single directory
-const saveCache = async function({ path, digests, cache, IS_LOCAL }) {
-  // In tests we don't run caching since it is slow and make source directory
-  // much bigger
-  if (TEST_CACHE_PATH !== undefined && TEST_CACHE_PATH !== path) {
-    return
-  }
-
-  const success = await cache.save(path, { move: !IS_LOCAL, digests })
-
-  if (success) {
-    logCacheDir(path)
-  }
-}
-
-// List of directories to cache
-const HOME = homedir()
-const ARTIFACTS = [
-  // Node modules
-  { path: 'node_modules', digests: ['package-lock.json', 'yarn.lock'] },
-  // Bower components
-  { path: 'bower_components' },
-  // Ruby gems
-  { path: '.bundle' },
-  // Python virtualenv
-  { path: '.venv' },
-  // WAPM packages (WebAssembly)
-  { path: 'wapm_packages' },
-  // Yarn
-  { path: `${HOME}/.yarn_cache` },
-  // Pip
-  { path: `${HOME}/.cache/pip` },
-  // Emacs cask
-  { path: `${HOME}/.cask` },
-  // Emacs
-  { path: `${HOME}/.emacs.d` },
-  // Maven
-  { path: `${HOME}/.m2` },
-  // Boot
-  { path: `${HOME}/.boot` },
-  // Composer
-  { path: `${HOME}/.composer` },
-  // Wasmer
-  { path: `${HOME}/.wasmer/cache` },
-  // Go dependencies
-  { path: `${HOME}/.gimme_cache` },
-  // TODO: the buildbot currently uses different file paths. Once Netlify Build
-  // handles both saving cache and restoring cache, this should not be an issue
-  // anymore.
-  //   CACHE_DIR/node_version -> CACHE_DIR/.nvm/versions/node[/$NODE_VERSION]
-  //   CACHE_DIR/ruby_version -> CACHE_DIR/.rvm/rubies[/ruby-$RUBY_VERSION]
-  // nvm
-  { path: `${HOME}/.nvm/versions/node/${version}` },
-  // TODO: only cache rvm when the version is not already in the Docker image,
-  // i.e. it is a custom version
-  // rvm
-  { path: `${HOME}/.rvm/rubies` },
-]
-
-module.exports = cachePlugin

package/src/plugins_core/functions/manifest.yml

@@ -1 +0,0 @@
-name: '@netlify/plugin-functions-core'

package/src/plugins_core/functions/plugin.js

@@ -1,37 +0,0 @@
-const { zipFunctions } = require('@netlify/zip-it-and-ship-it')
-const pathExists = require('path-exists')
-const readdirp = require('readdirp')
-
-const { serializeArray } = require('../../log/serialize')
-
-// Plugin to package Netlify functions with @netlify/zip-it-and-ship-it
-const onPostBuild = async function({ constants: { FUNCTIONS_SRC, FUNCTIONS_DIST } }) {
-  if (!(await pathExists(FUNCTIONS_SRC))) {
-    return
-  }
-
-  console.log(`Packaging functions from ${FUNCTIONS_SRC}`)
-  await zipFunctions(FUNCTIONS_SRC, FUNCTIONS_DIST)
-
-  await logResults(FUNCTIONS_DIST)
-}
-
-// Print the list of paths to the packaged functions
-const logResults = async function(FUNCTIONS_DIST) {
-  const files = await readdirp.promise(FUNCTIONS_DIST)
-
-  if (files.length === 0) {
-    console.log('No functions were packaged')
-    return
-  }
-
-  const paths = files.map(getLoggedPath)
-  console.log(`Functions packaged in ${FUNCTIONS_DIST}
-${serializeArray(paths)}`)
-}
-
-const getLoggedPath = function({ path }) {
-  return path
-}
-
-module.exports = { onPostBuild }

package/src/plugins_core/functions_install/plugin.js

@@ -1,14 +0,0 @@
-const pathExists = require('path-exists')
-
-const { installFunctionDependencies } = require('../../install/functions')
-
-// Plugin to package Netlify functions with @netlify/zip-it-and-ship-it
-const onPreBuild = async function({ constants: { FUNCTIONS_SRC, IS_LOCAL } }) {
-  if (!(await pathExists(FUNCTIONS_SRC))) {
-    return
-  }
-
-  await installFunctionDependencies(FUNCTIONS_SRC, IS_LOCAL)
-}
-
-module.exports = { onPreBuild }