netlify-cli 10.0.0 → 10.1.0

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "netlify-cli",
-  "version": "10.0.0",
+  "version": "10.1.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "netlify-cli",
-      "version": "10.0.0",
+      "version": "10.1.0",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
@@ -138,6 +138,7 @@
         "husky": "^7.0.4",
         "ini": "^2.0.0",
         "mock-fs": "^5.1.2",
+        "nock": "^13.2.4",
         "p-timeout": "^4.0.0",
         "rewiremock": "^3.14.3",
         "seedrandom": "^3.0.5",
@@ -14405,6 +14406,12 @@
       "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
       "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w="
     },
+    "node_modules/lodash.set": {
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz",
+      "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=",
+      "dev": true
+    },
     "node_modules/lodash.some": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/lodash.some/-/lodash.some-4.6.0.tgz",
@@ -15831,6 +15838,21 @@
         "isarray": "0.0.1"
       }
     },
+    "node_modules/nock": {
+      "version": "13.2.4",
+      "resolved": "https://registry.npmjs.org/nock/-/nock-13.2.4.tgz",
+      "integrity": "sha512-8GPznwxcPNCH/h8B+XZcKjYPXnUV5clOKCjAqyjsiqA++MpNx9E9+t8YPp0MbThO+KauRo7aZJ1WuIZmOrT2Ug==",
+      "dev": true,
+      "dependencies": {
+        "debug": "^4.1.0",
+        "json-stringify-safe": "^5.0.1",
+        "lodash.set": "^4.3.2",
+        "propagate": "^2.0.0"
+      },
+      "engines": {
+        "node": ">= 10.13"
+      }
+    },
     "node_modules/node-domexception": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
@@ -18235,6 +18257,15 @@
       "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
       "dev": true
     },
+    "node_modules/propagate": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz",
+      "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==",
+      "dev": true,
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/proxy-addr": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -33416,6 +33447,12 @@
       "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
       "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w="
     },
+    "lodash.set": {
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz",
+      "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=",
+      "dev": true
+    },
     "lodash.some": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/lodash.some/-/lodash.some-4.6.0.tgz",
@@ -34486,6 +34523,18 @@
         }
       }
     },
+    "nock": {
+      "version": "13.2.4",
+      "resolved": "https://registry.npmjs.org/nock/-/nock-13.2.4.tgz",
+      "integrity": "sha512-8GPznwxcPNCH/h8B+XZcKjYPXnUV5clOKCjAqyjsiqA++MpNx9E9+t8YPp0MbThO+KauRo7aZJ1WuIZmOrT2Ug==",
+      "dev": true,
+      "requires": {
+        "debug": "^4.1.0",
+        "json-stringify-safe": "^5.0.1",
+        "lodash.set": "^4.3.2",
+        "propagate": "^2.0.0"
+      }
+    },
     "node-domexception": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
@@ -36300,6 +36349,12 @@
         }
       }
     },
+    "propagate": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz",
+      "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==",
+      "dev": true
+    },
     "proxy-addr": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "netlify-cli",
   "description": "Netlify command line tool",
-  "version": "10.0.0",
+  "version": "10.1.0",
   "author": "Netlify Inc.",
   "contributors": [
     "Abraham Schilling <AbrahamSchilling@gmail.com> (https://gitlab.com/n4bb12)",
@@ -103,6 +103,7 @@
     "Sam Holmes <samholmes1337@gmail.com> (https://samholmes.net)",
     "Sander de Groot (https://degroot.dev)",
     "Sarah Drasner <sarah.drasner@gmail.com> (https://twitter.com/sarah_edo)",
+    "Sarah Etter <sarah@sarahetter.com> (http://www.sarahetter.com)",
     "Scott Spence <spences10apps@gmail.com> (https://twitter.com/spences10)",
     "Sean Grove <sean@bushi.do> (https://twitter.com/sgrove)",
     "Sebastian Smolorz",
@@ -330,6 +331,7 @@
     "husky": "^7.0.4",
     "ini": "^2.0.0",
     "mock-fs": "^5.1.2",
+    "nock": "^13.2.4",
     "p-timeout": "^4.0.0",
     "rewiremock": "^3.14.3",
     "seedrandom": "^3.0.5",

package/src/commands/dev/dev.js

@@ -201,18 +201,33 @@ const FRAMEWORK_PORT_TIMEOUT = 6e5
  * @param {*} params.addonsUrls
  * @param {import('../base-command').NetlifyOptions["config"]} params.config
  * @param {() => Promise<object>} params.getUpdatedConfig
+ * @param {string} params.geolocationMode
  * @param {*} params.settings
+ * @param {boolean} params.offline
  * @param {*} params.site
+ * @param {import('../../utils/state-config').StateConfig} params.state
  * @returns
  */
-const startProxyServer = async ({ addonsUrls, config, getUpdatedConfig, settings, site }) => {
+const startProxyServer = async ({
+  addonsUrls,
+  config,
+  geolocationMode,
+  getUpdatedConfig,
+  offline,
+  settings,
+  site,
+  state,
+}) => {
   const url = await startProxy({
     addonsUrls,
     config,
     configPath: site.configPath,
+    geolocationMode,
     getUpdatedConfig,
+    offline,
     projectDir: site.root,
     settings,
+    state,
   })
 
   if (!url) {
@@ -365,7 +380,16 @@ const dev = async (options, command) => {
     return normalizedNewConfig
   }
 
-  let url = await startProxyServer({ settings, site, addonsUrls, config, getUpdatedConfig })
+  let url = await startProxyServer({
+    addonsUrls,
+    config,
+    geolocationMode: options.geo,
+    getUpdatedConfig,
+    offline: options.offline,
+    settings,
+    site,
+    state,
+  })
 
   const liveTunnelUrl = await handleLiveTunnel({ options, site, api, settings })
   url = liveTunnelUrl || url
@@ -484,17 +508,19 @@ const createDevCommand = (program) => {
     .option('-o ,--offline', 'disables any features that require network access')
     .option('-l, --live', 'start a public live session', false)
     .option('--functionsPort <port>', 'port of functions server', (value) => Number.parseInt(value))
+    .addOption(
+      new Option(
+        '--geo <mode>',
+        'force geolocation data to be updated, use cached data from the last 24h if found, or use a mock location',
+      )
+        .choices(['cache', 'mock', 'update'])
+        .default('cache'),
+    )
     .addOption(
       new Option('--staticServerPort <port>', 'port of the static app server used when no framework is detected')
         .argParser((value) => Number.parseInt(value))
         .hideHelp(),
     )
-    .addOption(
-      new Option(
-        '-g ,--locationDb <path>',
-        'specify the path to a local GeoIP location database in MMDB format',
-      ).hideHelp(),
-    )
     .addOption(new Option('--graph', 'enable Netlify Graph support').hideHelp())
     .addExamples([
       'netlify dev',

package/src/lib/edge-functions/headers.js

@@ -1,5 +1,6 @@
 module.exports = {
   Functions: 'x-deno-functions',
+  Geo: 'x-nf-geo',
   PassHost: 'X-NF-Pass-Host',
   Passthrough: 'x-deno-pass',
   RequestID: 'X-NF-Request-ID',

package/src/lib/edge-functions/proxy.js

@@ -1,10 +1,12 @@
 // @ts-check
-const { env } = require('process')
+const { relative } = require('path')
+const { cwd, env } = require('process')
 
 const getAvailablePort = require('get-port')
 const { v4: generateUUID } = require('uuid')
 
-const { NETLIFYDEVERR, chalk } = require('../../utils/command-helpers')
+const { NETLIFYDEVERR, NETLIFYDEVWARN, chalk, log } = require('../../utils/command-helpers')
+const { getGeoLocation } = require('../geo-location')
 const { getPathInProject } = require('../settings')
 const { startSpinner, stopSpinner } = require('../spinner')
 
@@ -40,7 +42,7 @@ const handleProxyRequest = (req, proxyReq) => {
   })
 }
 
-const initializeProxy = async ({ config, configPath, getUpdatedConfig, settings }) => {
+const initializeProxy = async ({ config, configPath, geolocationMode, getUpdatedConfig, offline, settings, state }) => {
   const { functions: internalFunctions, importMap, path: internalFunctionsPath } = await getInternalFunctions()
   const { port: mainPort } = settings
   const userFunctionsPath = config.build.edge_functions
@@ -65,24 +67,39 @@ const initializeProxy = async ({ config, configPath, getUpdatedConfig, settings
       return
     }
 
-    const { registry } = await server
+    const [geoLocation, { registry }] = await Promise.all([
+      getGeoLocation({ mode: geolocationMode, offline, state }),
+      server,
+    ])
+
+    // Setting header with geolocation.
+    req.headers[headers.Geo] = JSON.stringify(geoLocation)
 
     await registry.initialize()
 
-    const manifest = await registry.getManifest()
     const url = new URL(req.url, `http://${LOCAL_HOST}:${mainPort}`)
-    const routes = manifest.routes.map((route) => ({
-      ...route,
-      pattern: new RegExp(route.pattern),
-    }))
-    const matchingFunctions = routes.filter(({ pattern }) => pattern.test(url.pathname)).map((route) => route.function)
-
-    if (matchingFunctions.length === 0) {
+    const { functionNames, orphanedDeclarations } = await registry.matchURLPath(url.pathname)
+
+    // If the request matches a config declaration for an Edge Function without
+    // a matching function file, we warn the user.
+    orphanedDeclarations.forEach((functionName) => {
+      log(
+        `${NETLIFYDEVWARN} Request to ${chalk.yellow(
+          url.pathname,
+        )} matches declaration for edge function ${chalk.yellow(
+          functionName,
+        )}, but there's no matching function file in ${chalk.yellow(
+          relative(cwd(), userFunctionsPath),
+        )}. Please visit ${chalk.blue('https://ntl.fyi/edge-create')} for more information.`,
+      )
+    })
+
+    if (functionNames.length === 0) {
       return
     }
 
     req[headersSymbol] = {
-      [headers.Functions]: matchingFunctions.join(','),
+      [headers.Functions]: functionNames.join(','),
       [headers.PassHost]: `${LOCAL_HOST}:${mainPort}`,
       [headers.Passthrough]: 'passthrough',
       [headers.RequestID]: generateUUID(),

package/src/lib/edge-functions/registry.js

@@ -158,7 +158,10 @@ class EdgeFunctionsRegistry {
 
   getDeclarations(config) {
     const { edge_functions: userFunctions = [] } = config
-    const declarations = [...this.internalFunctions, ...userFunctions]
+
+    // The order is important, since we want to run user-defined functions
+    // before internal functions.
+    const declarations = [...userFunctions, ...this.internalFunctions]
 
     return declarations
   }
@@ -224,6 +227,45 @@ class EdgeFunctionsRegistry {
     log(`${NETLIFYDEVLOG} ${chalk.magenta('Removed')} edge function ${chalk.yellow(func.name)}`)
   }
 
+  /**
+   * @param {string} urlPath
+   */
+  async matchURLPath(urlPath) {
+    // `generateManifest` will only include functions for which there is both a
+    // function file and a config declaration, but we want to catch cases where
+    // a config declaration exists without a matching function file. To do that
+    // we compute a list of functions from the declarations (the `path` doesn't
+    // really matter) and later on match the resulting routes against the list
+    // of functions we have in the registry. Any functions found in the former
+    // but not the latter are treated as orphaned declarations.
+    const functions = this.declarations.map((declaration) => ({ name: declaration.function, path: '' }))
+    const manifest = await this.bundler.generateManifest({
+      declarations: this.declarations,
+      functions,
+    })
+    const routes = manifest.routes.map((route) => ({
+      ...route,
+      pattern: new RegExp(route.pattern),
+    }))
+    const orphanedDeclarations = new Set()
+    const functionNames = routes
+      .filter(({ pattern }) => pattern.test(urlPath))
+      .map((route) => {
+        const matchingFunction = this.functions.find(({ name }) => name === route.function)
+
+        if (matchingFunction === undefined) {
+          orphanedDeclarations.add(route.function)
+
+          return null
+        }
+
+        return matchingFunction.name
+      })
+      .filter(Boolean)
+
+    return { functionNames, orphanedDeclarations }
+  }
+
   processGraph(graph) {
     if (!graph) {
       warn('Could not process edge functions dependency graph. Live reload will not be available.')

package/src/lib/geo-location.js

@@ -0,0 +1,99 @@
+// @ts-check
+const fetch = require('node-fetch')
+
+const API_URL = 'https://netlifind.netlify.app'
+const STATE_GEO_PROPERTY = 'geolocation'
+
+// 24 hours
+const CACHE_TTL = 8.64e7
+
+// 10 seconds
+const REQUEST_TIMEOUT = 1e4
+
+/**
+ * @typedef GeoLocation
+ * @type {object}
+ * @property {string} city
+ * @property {object} country
+ * @property {string} country.code
+ * @property {string} country.name
+ * @property {object} country
+ * @property {string} country.code
+ * @property {string} country.name
+ */
+
+// The default location to be used if we're unable to talk to the API.
+const mockLocation = {
+  city: 'San Francisco',
+  country: { code: 'US', name: 'United States' },
+  subdivision: { code: 'CA', name: 'California' },
+}
+
+/**
+ * Returns geolocation data from a remote API, the local cache, or a mock
+ * location, depending on the mode selected.
+ *
+ * @param {object} params
+ * @param {string} params.geolocationMode
+ * @param {"cache"|"update"|"mock"} params.mode
+ * @param {boolean} params.offline
+ * @param {import('../utils/state-config').StateConfig} params.state
+ * @returns {Promise<GeoLocation>}
+ */
+const getGeoLocation = async ({ mode, offline, state }) => {
+  const cacheObject = state.get(STATE_GEO_PROPERTY)
+
+  // If we have cached geolocation data and the `--geo` option is set to
+  // `cache`, let's try to use it.
+  if (cacheObject !== undefined && mode === 'cache') {
+    const age = Date.now() - cacheObject.timestamp
+
+    // Let's use the cached data if it's not older than the TTL. Also, if the
+    // `--offline` option was used, it's best to use the cached location than
+    // the mock one.
+    if (age < CACHE_TTL || offline) {
+      return cacheObject.data
+    }
+  }
+
+  // If the `--geo` option is set to `mock`, we use the mock location. Also,
+  // if the `--offline` option was used, we can't talk to the API, so let's
+  // also use the mock location.
+  if (mode === 'mock' || offline) {
+    return mockLocation
+  }
+
+  // Trying to retrieve geolocation data from the API and caching it locally.
+  try {
+    const data = await getGeoLocationFromAPI()
+    const newCacheObject = {
+      data,
+      timestamp: Date.now(),
+    }
+
+    state.set(STATE_GEO_PROPERTY, newCacheObject)
+
+    return data
+  } catch {
+    // We couldn't get geolocation data from the API, so let's return the
+    // mock location.
+    return mockLocation
+  }
+}
+
+/**
+ * Returns geolocation data from a remote API
+ *
+ * @returns {Promise<GeoLocation>}
+ */
+const getGeoLocationFromAPI = async () => {
+  const res = await fetch(API_URL, {
+    method: 'GET',
+    timeout: REQUEST_TIMEOUT,
+  })
+  const { geo } = await res.json()
+
+  return geo
+}
+
+module.exports = { getGeoLocation, mockLocation }

package/src/utils/proxy.js

@@ -461,13 +461,26 @@ const onRequest = async ({ addonsUrls, edgeFunctionsProxy, functionsServer, prox
   proxy.web(req, res, options)
 }
 
-const startProxy = async function ({ addonsUrls, config, configPath, getUpdatedConfig, projectDir, settings }) {
+const startProxy = async function ({
+  addonsUrls,
+  config,
+  configPath,
+  geolocationMode,
+  getUpdatedConfig,
+  offline,
+  projectDir,
+  settings,
+  state,
+}) {
   const functionsServer = settings.functionsPort ? `http://localhost:${settings.functionsPort}` : null
   const edgeFunctionsProxy = await edgeFunctions.initializeProxy({
     config,
     configPath,
+    geolocationMode,
     getUpdatedConfig,
+    offline,
     settings,
+    state,
   })
   const proxy = await initializeProxy({
     port: settings.frameworkPort,