netlify-cli 8.12.0 → 8.14.0-scheduled-functions

package/src/lib/functions/netlify-function.js

@@ -32,6 +32,7 @@ class NetlifyFunction {
     // Determines whether this is a background function based on the function
     // name.
     this.isBackground = name.endsWith(BACKGROUND_SUFFIX)
+    this.schedule = null
 
     // List of the function's source files. This starts out as an empty set
     // and will get populated on every build.
@@ -44,6 +45,12 @@ class NetlifyFunction {
     return /^[A-Za-z0-9_-]+$/.test(this.name)
   }
 
+  async isScheduled() {
+    await this.buildQueue
+
+    return Boolean(this.schedule)
+  }
+
   // The `build` method transforms source files into invocable functions. Its
   // return value is an object with:
   //
@@ -61,12 +68,13 @@ class NetlifyFunction {
     this.buildQueue = buildFunction({ cache })
 
     try {
-      const { srcFiles, ...buildData } = await this.buildQueue
+      const { schedule, srcFiles, ...buildData } = await this.buildQueue
       const srcFilesSet = new Set(srcFiles)
       const srcFilesDiff = this.getSrcFilesDiff(srcFilesSet)
 
       this.buildData = buildData
       this.srcFiles = srcFilesSet
+      this.schedule = schedule
 
       return { srcFilesDiff }
     } catch (error) {

package/src/lib/functions/runtimes/js/builders/zisi.js

@@ -1,7 +1,7 @@
 const { mkdir, writeFile } = require('fs').promises
 const path = require('path')
 
-const { zipFunction } = require('@netlify/zip-it-and-ship-it')
+const { listFunction, zipFunction } = require('@netlify/zip-it-and-ship-it')
 const decache = require('decache')
 const readPkgUp = require('read-pkg-up')
 const sourceMapSupport = require('source-map-support')
@@ -35,7 +35,11 @@ const buildFunction = async ({ cache, config, directory, func, hasTypeModule, pr
   // root of the functions directory (e.g. `functions/my-func.js`). In
   // this case, we use `mainFile` as the function path of `zipFunction`.
   const entryPath = functionDirectory === directory ? func.mainFile : functionDirectory
-  const { inputs, path: functionPath } = await memoizedBuild({
+  const {
+    inputs,
+    path: functionPath,
+    schedule,
+  } = await memoizedBuild({
     cache,
     cacheKey: `zisi-${entryPath}`,
     command: () => zipFunction(entryPath, targetDirectory, zipOptions),
@@ -56,7 +60,19 @@ const buildFunction = async ({ cache, config, directory, func, hasTypeModule, pr
 
   clearFunctionsCache(targetDirectory)
 
-  return { buildPath, srcFiles }
+  return { buildPath, srcFiles, schedule }
+}
+
+/**
+ * @param {string} mainFile
+ */
+const parseForSchedule = async ({ config, mainFile, projectRoot }) => {
+  const listedFunction = await listFunction(mainFile, {
+    config: netlifyConfigToZisiConfig({ config, projectRoot }),
+    parseISC: true,
+  })
+
+  return listedFunction && listedFunction.schedule
 }
 
 // Clears the cache for any files inside the directory from which functions are
@@ -79,10 +95,11 @@ const getTargetDirectory = async ({ errorExit }) => {
   return targetDirectory
 }
 
+const netlifyConfigToZisiConfig = ({ config, projectRoot }) =>
+  addFunctionsConfigDefaults(normalizeFunctionsConfig({ functionsConfig: config.functions, projectRoot }))
+
 module.exports = async ({ config, directory, errorExit, func, projectRoot }) => {
-  const functionsConfig = addFunctionsConfigDefaults(
-    normalizeFunctionsConfig({ functionsConfig: config.functions, projectRoot }),
-  )
+  const functionsConfig = netlifyConfigToZisiConfig({ config, projectRoot })
 
   const packageJson = await readPkgUp(func.mainFile)
   const hasTypeModule = packageJson && packageJson.packageJson.type === 'module'
@@ -115,3 +132,5 @@ module.exports = async ({ config, directory, errorExit, func, projectRoot }) =>
     target: targetDirectory,
   }
 }
+
+module.exports.parseForSchedule = parseForSchedule

package/src/lib/functions/runtimes/js/index.js

@@ -46,8 +46,9 @@ const getBuildFunction = async ({ config, directory, errorExit, func, projectRoo
   // main file otherwise.
   const functionDirectory = dirname(func.mainFile)
   const srcFiles = functionDirectory === directory ? [func.mainFile] : [functionDirectory]
+  const schedule = await detectZisiBuilder.parseForSchedule({ mainFile: func.mainFile, config, projectRoot })
 
-  return () => ({ srcFiles })
+  return () => ({ schedule, srcFiles })
 }
 
 const invokeFunction = async ({ context, event, func, timeout }) => {

package/src/lib/functions/scheduled.js

@@ -0,0 +1,78 @@
+const ansi2html = require('ansi2html')
+
+const { CLOCKWORK_USERAGENT } = require('../../utils')
+
+const { formatLambdaError } = require('./utils')
+
+const handleScheduledFunction = ({ error, request, response, result }) => {
+  const acceptsHtml = request.headers.accept && request.headers.accept.includes('text/html')
+  const paragraph = (text) => {
+    text = text.trim()
+
+    if (acceptsHtml) {
+      return ansi2html(`<p>${text}</p>`)
+    }
+
+    text = text
+      .replaceAll('<pre><code>', '```\n')
+      .replaceAll('</code></pre>', '\n```')
+      .replaceAll(`<code>`, '`')
+      .replaceAll(`</code>`, '`')
+
+    return `${text}\n\n`
+  }
+
+  const isSimulatedRequest = request.headers['user-agent'] === CLOCKWORK_USERAGENT
+
+  let message = ''
+
+  if (!isSimulatedRequest) {
+    message += paragraph(`
+You performed an HTTP request to <code>${request.path}</code>, which is a scheduled function.
+You can do this to test your functions locally, but it won't work in production.
+    `)
+  }
+
+  if (error) {
+    message += paragraph(`
+There was an error during execution of your scheduled function:
+
+<pre><code>${formatLambdaError(error)}</code></pre>`)
+  }
+
+  if (result) {
+    const { statusCode } = result
+    if (statusCode >= 500) {
+      message += paragraph(`
+Your function returned a status code of <code>${statusCode}</code>.
+At the moment, Netlify does nothing about that. In the future, there might be a retry mechanism based on this.
+`)
+    }
+
+    const allowedKeys = new Set(['statusCode'])
+    const returnedKeys = Object.keys(result)
+    const ignoredKeys = returnedKeys.filter((key) => !allowedKeys.has(key))
+
+    if (ignoredKeys.length !== 0) {
+      message += paragraph(
+        `Your function returned ${ignoredKeys
+          .map((key) => `<code>${key}</code>`)
+          .join(', ')}. Is this an accident? It won't be interpreted by Netlify.`,
+      )
+    }
+  }
+
+  response.status(error ? 500 : 200)
+  if (acceptsHtml) {
+    response.set('Content-Type', 'text/html')
+    response.send(
+      `<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/water.css@2/out/water.css">\n
+      ${message}`,
+    )
+  } else {
+    response.set('Content-Type', 'text/plain')
+    response.send(message)
+  }
+}
+
+module.exports = { handleScheduledFunction }

package/src/lib/functions/server.js

@@ -6,6 +6,7 @@ const { NETLIFYDEVERR, NETLIFYDEVLOG, error: errorExit, getInternalFunctionsDir,
 const { handleBackgroundFunction, handleBackgroundFunctionResult } = require('./background')
 const { createFormSubmissionHandler } = require('./form-submissions-handler')
 const { FunctionsRegistry } = require('./registry')
+const { handleScheduledFunction } = require('./scheduled')
 const { handleSynchronousFunction } = require('./synchronous')
 const { shouldBase64Encode } = require('./utils')
 
@@ -105,6 +106,15 @@ const createHandler = function ({ functionsRegistry }) {
       const { error } = await func.invoke(event, clientContext)
 
       handleBackgroundFunctionResult(functionName, error)
+    } else if (await func.isScheduled()) {
+      const { error, result } = await func.invoke(event, clientContext)
+
+      handleScheduledFunction({
+        error,
+        result,
+        request,
+        response,
+      })
     } else {
       const { error, result } = await func.invoke(event, clientContext)
 

package/src/lib/one-graph/cli-client.js

@@ -0,0 +1,278 @@
+/* eslint-disable eslint-comments/disable-enable-pair */
+/* eslint-disable fp/no-loops */
+const os = require('os')
+
+const { GraphQL, InternalConsole, OneGraphClient } = require('netlify-onegraph-internal')
+const { NetlifyGraph } = require('netlify-onegraph-internal')
+
+const { chalk, error, log, warn } = require('../../utils')
+
+const { createCLISession, createPersistedQuery, ensureAppForSite, updateCLISessionMetadata } = OneGraphClient
+
+const {
+  generateFunctionsFile,
+  generateHandler,
+  readGraphQLOperationsSourceFile,
+  writeGraphQLOperationsSourceFile,
+  writeGraphQLSchemaFile,
+} = require('./cli-netlify-graph')
+
+const { parse } = GraphQL
+const { defaultExampleOperationsDoc, extractFunctionsFromOperationDoc } = NetlifyGraph
+
+const internalConsole = {
+  log,
+  warn,
+  error,
+  debug: console.debug,
+}
+
+InternalConsole.registerConsole(internalConsole)
+
+/**
+ * Start polling for CLI events for a given session to process locally
+ * @param {object} input
+ * @param {string} input.appId The app to query against, typically the siteId
+ * @param {string} input.netlifyToken The (typically netlify) access token that is used for authentication, if any
+ * @param {NetlifyGraphConfig} input.netlifyGraphConfig A standalone config object that contains all the information necessary for Netlify Graph to process events
+ * @param {function} input.onClose A function to call when the polling loop is closed
+ * @param {function} input.onError A function to call when an error occurs
+ * @param {function} input.onEvents A function to call when CLI events are received and need to be processed
+ * @param {string} input.sessionId The session id to monitor CLI events for
+ * @param {state} input.state A function to call to set/get the current state of the local Netlify project
+ * @returns
+ */
+const monitorCLISessionEvents = (input) => {
+  const { appId, netlifyGraphConfig, netlifyToken, onClose, onError, onEvents, sessionId, state } = input
+
+  const frequency = 5000
+  let shouldClose = false
+
+  const enabledServiceWatcher = async (innerNetlifyToken, siteId) => {
+    const enabledServices = state.get('oneGraphEnabledServices') || ['onegraph']
+    const enabledServicesInfo = await OneGraphClient.fetchEnabledServices(innerNetlifyToken, siteId)
+    if (!enabledServicesInfo) {
+      warn('Unable to fetch enabled services for site for code generation')
+      return
+    }
+    const newEnabledServices = enabledServicesInfo.map((service) => service.service)
+    const enabledServicesCompareKey = enabledServices.sort().join(',')
+    const newEnabledServicesCompareKey = newEnabledServices.sort().join(',')
+
+    if (enabledServicesCompareKey !== newEnabledServicesCompareKey) {
+      log(
+        `${chalk.magenta(
+          'Reloading',
+        )} Netlify Graph schema..., ${enabledServicesCompareKey} => ${newEnabledServicesCompareKey}`,
+      )
+      await refetchAndGenerateFromOneGraph({ netlifyGraphConfig, state, netlifyToken: innerNetlifyToken, siteId })
+      log(`${chalk.green('Reloaded')} Netlify Graph schema and regenerated functions`)
+    }
+  }
+
+  const close = () => {
+    shouldClose = true
+  }
+
+  let handle
+
+  const helper = async () => {
+    if (shouldClose) {
+      clearTimeout(handle)
+      onClose()
+    }
+
+    const next = await OneGraphClient.fetchCliSessionEvents({ appId, authToken: netlifyToken, sessionId })
+
+    if (next.errors) {
+      next.errors.forEach((fetchEventError) => {
+        onError(fetchEventError)
+      })
+    }
+
+    const { events } = next
+
+    if (events.length !== 0) {
+      const ackIds = await onEvents(events)
+      await OneGraphClient.ackCLISessionEvents({ appId, authToken: netlifyToken, sessionId, eventIds: ackIds })
+    }
+
+    await enabledServiceWatcher(netlifyToken, appId)
+
+    handle = setTimeout(helper, frequency)
+  }
+
+  // Fire immediately to start rather than waiting the initial `frequency`
+  helper()
+
+  return close
+}
+
+/**
+ * Fetch the schema for a site, and regenerate all of the downstream files
+ * @param {object} input
+ * @param {string} input.siteId The id of the site to query against
+ * @param {string} input.netlifyToken The (typically netlify) access token that is used for authentication, if any
+ * @param {NetlifyGraphConfig} input.netlifyGraphConfig A standalone config object that contains all the information necessary for Netlify Graph to process events
+ * @param {state} input.state A function to call to set/get the current state of the local Netlify project
+ * @returns {Promise<void>}
+ */
+const refetchAndGenerateFromOneGraph = async (input) => {
+  const { netlifyGraphConfig, netlifyToken, siteId, state } = input
+  await OneGraphClient.ensureAppForSite(netlifyToken, siteId)
+
+  const enabledServicesInfo = await OneGraphClient.fetchEnabledServices(netlifyToken, siteId)
+  if (!enabledServicesInfo) {
+    warn('Unable to fetch enabled services for site for code generation')
+    return
+  }
+
+  const enabledServices = enabledServicesInfo
+    .map((service) => service.service)
+    .sort((aString, bString) => aString.localeCompare(bString))
+  const schema = await OneGraphClient.fetchOneGraphSchema(siteId, enabledServices)
+
+  let currentOperationsDoc = readGraphQLOperationsSourceFile(netlifyGraphConfig)
+  if (currentOperationsDoc.trim().length === 0) {
+    currentOperationsDoc = defaultExampleOperationsDoc
+  }
+
+  const parsedDoc = parse(currentOperationsDoc)
+  const operations = extractFunctionsFromOperationDoc(parsedDoc)
+
+  generateFunctionsFile(netlifyGraphConfig, schema, currentOperationsDoc, operations)
+  writeGraphQLSchemaFile(netlifyGraphConfig, schema)
+  state.set('oneGraphEnabledServices', enabledServices)
+}
+
+/**
+ *
+ * @param {object} input
+ * @param {string} input.siteId The site id to query against
+ * @param {string} input.netlifyToken The (typically netlify) access token that is used for authentication, if any
+ * @param {string} input.docId The GraphQL operations document id to fetch
+ * @param {string} input.schema The GraphQL schema to use when generating code
+ * @param {NetlifyGraphConfig} input.netlifyGraphConfig A standalone config object that contains all the information necessary for Netlify Graph to process events
+ * @returns
+ */
+const updateGraphQLOperationsFile = async (input) => {
+  const { docId, netlifyGraphConfig, netlifyToken, schema, siteId } = input
+  const persistedDoc = await OneGraphClient.fetchPersistedQuery(netlifyToken, siteId, docId)
+  if (!persistedDoc) {
+    warn('No persisted doc found for:', docId)
+    return
+  }
+
+  const doc = persistedDoc.query
+
+  writeGraphQLOperationsSourceFile(netlifyGraphConfig, doc)
+  const appOperationsDoc = readGraphQLOperationsSourceFile(netlifyGraphConfig)
+  const parsedDoc = parse(appOperationsDoc, {
+    noLocation: true,
+  })
+  const operations = extractFunctionsFromOperationDoc(parsedDoc)
+  generateFunctionsFile(netlifyGraphConfig, schema, appOperationsDoc, operations)
+}
+
+const handleCliSessionEvent = async ({ event, netlifyGraphConfig, netlifyToken, schema, siteId }) => {
+  const { __typename, payload } = await event
+  switch (__typename) {
+    case 'OneGraphNetlifyCliSessionTestEvent':
+      await handleCliSessionEvent({ netlifyToken, event: payload, netlifyGraphConfig, schema, siteId })
+      break
+    case 'OneGraphNetlifyCliSessionGenerateHandlerEvent':
+      await generateHandler(netlifyGraphConfig, schema, payload.operationId, payload)
+      break
+    case 'OneGraphNetlifyCliSessionPersistedLibraryUpdatedEvent':
+      await updateGraphQLOperationsFile({ netlifyToken, docId: payload.docId, netlifyGraphConfig, schema, siteId })
+      break
+    default: {
+      warn(`Unrecognized event received, you may need to upgrade your CLI version`, __typename, payload)
+      break
+    }
+  }
+}
+
+/**
+ * Load the CLI session id from the local state
+ * @param {state} state
+ * @returns
+ */
+const loadCLISession = (state) => state.get('oneGraphSessionId')
+
+/**
+ * Idemponentially save the CLI session id to the local state and start monitoring for CLI events and upstream schema changes
+ * @param {object} input
+ * @param {string} input.netlifyToken The (typically netlify) access token that is used for authentication, if any
+ * @param {NetlifyGraphConfig} input.netlifyGraphConfig A standalone config object that contains all the information necessary for Netlify Graph to process events
+ * @param {state} input.state A function to call to set/get the current state of the local Netlify project
+ * @param {site} input.site The site object
+ */
+const startOneGraphCLISession = async (input) => {
+  const { netlifyGraphConfig, netlifyToken, site, state } = input
+  OneGraphClient.ensureAppForSite(netlifyToken, site.id)
+  let oneGraphSessionId = loadCLISession(state)
+  if (!oneGraphSessionId) {
+    const sessionName = generateSessionName()
+    const sessionMetadata = {}
+    const oneGraphSession = await OneGraphClient.createCLISession(netlifyToken, site.id, sessionName, sessionMetadata)
+    state.set('oneGraphSessionId', oneGraphSession.id)
+    oneGraphSessionId = state.get('oneGraphSessionId')
+  }
+
+  const enabledServices = []
+  const schema = await OneGraphClient.fetchOneGraphSchema(site.id, enabledServices)
+
+  monitorCLISessionEvents({
+    appId: site.id,
+    netlifyToken,
+    netlifyGraphConfig,
+    sessionId: oneGraphSessionId,
+    state,
+    onEvents: async (events) => {
+      for (const event of events) {
+        const eventName = OneGraphClient.friendlyEventName(event)
+        log(`${chalk.magenta('Handling')} Netlify Graph: ${eventName}...`)
+        await handleCliSessionEvent({ netlifyToken, event, netlifyGraphConfig, schema, siteId: site.id })
+        log(`${chalk.green('Finished handling')} Netlify Graph: ${eventName}...`)
+      }
+      return events.map((event) => event.id)
+    },
+    onError: (fetchEventError) => {
+      error(`Netlify Graph upstream error: ${fetchEventError}`)
+    },
+    onClose: () => {
+      log('Netlify Graph upstream closed')
+    },
+  })
+}
+
+/**
+ * Generate a session name that can be identified as belonging to the current checkout
+ * @returns {string} The name of the session to create
+ */
+const generateSessionName = () => {
+  const userInfo = os.userInfo({ encoding: 'utf-8' })
+  const sessionName = `${userInfo.username}-${Date.now()}`
+  log(`Generated Netlify Graph session name: ${sessionName}`)
+  return sessionName
+}
+
+const OneGraphCliClient = {
+  ackCLISessionEvents: OneGraphClient.ackCLISessionEvents,
+  createCLISession,
+  createPersistedQuery,
+  fetchCliSessionEvents: OneGraphClient.fetchCliSessionEvents,
+  ensureAppForSite,
+  updateCLISessionMetadata,
+}
+
+module.exports = {
+  OneGraphCliClient,
+  handleCliSessionEvent,
+  generateSessionName,
+  loadCLISession,
+  monitorCLISessionEvents,
+  refetchAndGenerateFromOneGraph,
+  startOneGraphCLISession,
+}