@pbvision/fastify-firestore-service 0.0.50

package/src/make-app.js

@@ -0,0 +1,223 @@
+import Ajv from 'ajv/dist/2020.js'
+import fastify from 'fastify'
+import { v4 as uuidv4 } from 'uuid'
+
+import ComponentRegistrar from './component-registrar.js'
+import { makePinoLoggerOptions } from './make-logger.js'
+import compressPlugin from './plugins/compress.js'
+import contentParserPlugin from './plugins/content-parser.js'
+import cookiePlugin from './plugins/cookie.js'
+import errorHandlerPlugin from './plugins/error-handler.js'
+import healthCheckPlugin from './plugins/health-check.js'
+import latencyTrackerPlugin from './plugins/latency-tracker.js'
+import swaggerPlugin from './plugins/swagger.js'
+
+/**
+ * @typedef {object} CookieConfig
+ * @property {boolean} [disabled=false] Adds fastify-cookie plugin
+ * @property {string} [secret] A secret to use to secure cookie
+ */
+const COOKIE_CONFIG = {
+  disabled: false,
+  secret: null
+}
+
+/**
+ * @typedef {object} LoggingConfig
+ * @property {boolean} [reportAllErrors=false] Whether include all API
+ *   validation errors in error logging. Recommend to keep it off for production,
+ *   on for testing.
+ * @property {boolean} [reportErrorDetail=false] Whether include all details
+ *   of an error. Recommend to keep it off for remote testing, on for local
+ *   testing.
+ * @property {String} [sentryDSN] The Sentry DSN to report errors to
+ * @property {Object} [sentryRateLimiter] Optional rate limiter instance (from
+ *   {@link createSentryRateLimiter}) to control Sentry reporting for errors
+ *   flagged via `RequestError.rateLimitSentry()`. Defaults to a fresh limiter
+ *   using the real clock; tests inject one with a controllable clock.
+ * @property {function} [customizePinoOpts] An optional function customize the
+ *   arguments to pass to Pino. Takes as input the options that will be sent
+ *   to be Pino by default (can be modified). Return the new options.
+ */
+const LOGGING_CONFIG = {
+  customizePinoOpts: null,
+  reportAllErrors: false,
+  reportErrorDetail: false,
+  sentryDSN: null,
+  sentryRateLimiter: null
+}
+
+/**
+ * @typedef {object} HealthCheckConfig
+ * @property {boolean} [disabled=false] Whether to add a health check endpoint
+ *   that simply returns 200.
+ * @property {string} [path='/'] The path to the health check endpoint.
+ */
+const HEALTH_CHECK_CONFIG = {
+  disabled: false,
+  path: '/'
+}
+
+/**
+ * @typedef {object} SwaggerConfig
+ * @property {boolean} [disabled=false] Whether to disable
+ * @property {Array<string>} [servers=[]] The host endpoint (scheme + domain) to
+ *   send requests to
+ * @property {Array<string>} [authHeaders=[]] Authentication headers
+ * @property {string} [routePrefix='/docs'] Authentication headers
+ */
+const SWAGGER_CONFIG = {
+  disabled: false,
+  servers: [],
+  authHeaders: [],
+  routePrefix: '/docs'
+}
+
+/**
+ * @typedef {object} LatencyTrackerConfig
+ * @property {boolean} [disabled=false] Whether to record how long from when we
+ *   start to process a request to when we start to send the response
+ * @property {string} [header='x-latency-ms'] response header to send the
+ *   latency info back in
+ */
+const LATENCY_TRACKER_CONFIG = {
+  disabled: false,
+  header: 'x-latency-ms'
+}
+
+const PARAMS_CONFIG = {
+  service: undefined,
+  components: undefined,
+  RegistrarCls: ComponentRegistrar,
+  cookie: {},
+  healthCheck: {},
+  latencyTracker: {},
+  logging: {},
+  swagger: {}
+}
+
+function loadConfigDefault (config, defaultConfig) {
+  for (const key of Object.keys(config)) {
+    if (!Object.prototype.hasOwnProperty.call(defaultConfig, key)) {
+      throw new Error(`Unknown config ${key}`)
+    }
+  }
+  if (!config.disabled) {
+    for (const [key, defaultValue] of Object.entries(defaultConfig)) {
+      if (defaultValue === undefined && !config[key]) {
+        throw new Error(`Missing required value for ${key}`)
+      }
+    }
+  }
+  Object.assign(config, { ...defaultConfig, ...config })
+}
+
+/**
+ * @param {Object} params
+ * @param {string} params.service Name of the service, for example, iam,
+ *   user-id, leaderboard. This affects API's prefixes.
+ * @param {Array<API|Model|component>} params.components A list of
+ *   components.
+ * @param {object} [params.RegistrarCls=ComponentRegistrar] A subclass of
+ *   ComponentRegistrar
+ * @param {CookieConfig} [params.cookie] Configures fastify-cookie.
+ * @param {HealthCheckConfig} [params.healthCheck] Configures health check endpoint.
+ * @param {LatencyTrackerConfig} [params.latencyTracker]
+ * @param {LoggingConfig} [params.logging] Configures logging.
+ * @param {SwaggerConfig} [params.swagger] Configures swagger.
+ * @returns {Promise<server>} fastify app with configured plugins
+ */
+export default async function makeService (params = {}) {
+  const configs = [
+    [() => params, PARAMS_CONFIG],
+    [() => params.cookie, COOKIE_CONFIG],
+    [() => params.healthCheck, HEALTH_CHECK_CONFIG],
+    [() => params.latencyTracker, LATENCY_TRACKER_CONFIG],
+    [() => params.logging, LOGGING_CONFIG],
+    [() => params.swagger, SWAGGER_CONFIG]
+  ]
+  for (const [getter, defaultConfig] of configs) {
+    loadConfigDefault(getter(), defaultConfig)
+  }
+  const {
+    service,
+    components,
+    RegistrarCls,
+    cookie,
+    healthCheck,
+    latencyTracker,
+    logging,
+    swagger
+  } = params
+  const fastifyServerId = uuidv4()
+  let requestCount = 0
+  const ajvParams = {
+    removeAdditional: false,
+    allErrors: logging.reportAllErrors,
+    useDefaults: true,
+    strictSchema: false,
+    strictRequired: true
+  }
+  const ajvObjs = {
+    coerceTypes: new Ajv({ ...ajvParams, coerceTypes: true }),
+    noCoerceTypes: new Ajv({ ...ajvParams, coerceTypes: false })
+  }
+  const logger = makePinoLoggerOptions(logging.customizePinoOpts)
+  const app = fastify({
+    disableRequestLogging: true,
+    logger,
+    genReqId: () => `${fastifyServerId}-${++requestCount}`
+  })
+    .setValidatorCompiler(({ httpPart, schema }) => {
+      // header, route, and query string always come as strings; coerce them
+      // to the correct types if possible; body should come exactly in the
+      // right type so don't allow type coercion there
+      const useCoerce = httpPart !== 'body'
+      const ajv = useCoerce ? ajvObjs.coerceTypes : ajvObjs.noCoerceTypes
+      const validate = ajv.compile(schema)
+      return (value) => !validate(value)
+        ? ({ value, error: validate.errors })
+        : ({ value })
+    })
+    .addHook('onResponse', (req, reply, done) => {
+      let objToLog = {}
+      try {
+        objToLog = logger.serializers.res(reply.raw)
+        Object.assign(objToLog, logger.serializers.req(req))
+        // istanbul ignore else
+        if (latencyTracker.header) {
+          // latency in milliseconds
+          objToLog.latency = reply.getHeader(latencyTracker.header)
+        }
+      } catch (err) /* istanbul ignore next */ {
+        // crashes in onResponse() aren't reported or caught unless we do this
+        if (typeof objToLog !== 'object') {
+          objToLog = {}
+        }
+        objToLog.failedToLog = JSON.stringify(err)
+      }
+      req.log.info(objToLog)
+      done()
+    })
+
+  const registrar = new RegistrarCls(app, service)
+
+  app.register(cookiePlugin, { cookie })
+    .register(compressPlugin)
+    .register(contentParserPlugin)
+    .register(latencyTrackerPlugin, { latencyTracker })
+    .register(errorHandlerPlugin, {
+      errorHandler: {
+        returnErrorDetail: logging.reportErrorDetail,
+        sentryDSN: logging.sentryDSN,
+        sentryRateLimiter: logging.sentryRateLimiter,
+        serverName: fastifyServerId,
+        service
+      }
+    })
+    .register(healthCheckPlugin, { healthCheck })
+    .register(swaggerPlugin, { swagger: { service, ...swagger } })
+
+  await registrar.registerComponents(components)
+  return app
+}

package/src/make-logger.js

@@ -0,0 +1,27 @@
+export function makePinoLoggerOptions (customizeOpts) {
+  function serializeReq (req) {
+    const q = req.query
+    const path = req.routeOptions.config.url
+    // istanbul ignore else
+    if (req.raw) {
+      req = req.raw
+    }
+    return {
+      uid: req.headers['x-uid'] || '',
+      method: req.method,
+      ua: req.headers['user-agent'] || '',
+      path,
+      q
+    }
+  }
+  customizeOpts = customizeOpts ?? (x => x)
+  const options = customizeOpts({
+    base: null, // omit pino default fields like pid and hostname
+    level: 'debug',
+    serializers: {
+      req: serializeReq,
+      res: res => ({ status: res.statusCode })
+    }
+  })
+  return options
+}

package/src/plugins/compress.js

@@ -0,0 +1,28 @@
+// eslint babel made changes to chained expressions which is breaking ast-types
+// https://github.com/babel/babel/issues/11908
+
+// ast-types has a fix, but it is not yet in a versioned release
+// https://github.com/benjamn/ast-types/pull/399
+
+// to circumvent problems caused by importing zlib, we do it in a separate file
+// so we can ignore it in linting
+import zlib from 'node:zlib'
+
+import { fastifyCompress } from '@fastify/compress'
+import fp from 'fastify-plugin'
+
+// TODO: fine tune configurations once we have a reliable usage pattern
+export default fp(function (fastify, options, next) {
+  fastify.register(fastifyCompress, {
+    threshold: 20,
+    brotliOptions: {
+      params: {
+        [zlib.constants.BROTLI_PARAM_QUALITY]: 4
+      }
+    }
+  })
+  next()
+}, {
+  fastify: '>=3.x',
+  name: 'compress'
+})

package/src/plugins/content-parser.js

@@ -0,0 +1,21 @@
+import fp from 'fastify-plugin'
+
+function addContentParser (fastify, options, next) {
+  fastify.addContentTypeParser('application/json',
+    { parseAs: 'string' },
+    function (req, body, done) {
+      try {
+        const json = JSON.parse(body || '{}')
+        done(null, json)
+      } catch (err) {
+        err.statusCode = 400
+        done(err, undefined)
+      }
+    })
+  next()
+}
+
+export default fp(addContentParser, {
+  fastify: '>=3.x',
+  name: 'content-parser'
+})

package/src/plugins/cookie.js

@@ -0,0 +1,17 @@
+import cookiePlugin from '@fastify/cookie'
+import fp from 'fastify-plugin'
+
+function addCookiePlugin (fastify, options, next) {
+  // istanbul ignore if
+  if (options.cookie.disabled || !options.cookie.secret) {
+    next()
+    return
+  }
+  fastify.register(cookiePlugin, { secret: options.cookie.secret })
+  next()
+}
+
+export default fp(addCookiePlugin, {
+  fastify: '>=3.x',
+  name: 'content-parser'
+})

package/src/plugins/error-handler.js

@@ -0,0 +1,180 @@
+import S from '@pbvision/schema'
+import * as Sentry from '@sentry/node'
+import fp from 'fastify-plugin'
+
+import { InvalidInputException, __RequestDone } from '../api/exception.js'
+import { createSentryRateLimiter } from './sentry-rate-limit.js'
+
+export default fp(function (fastify, options, next) {
+  const isLocalhost = process.env.NODE_ENV === 'localhost'
+  const sentryDSN = options.errorHandler.sentryDSN
+  // istanbul ignore next
+  const isSentryEnabled = sentryDSN && !isLocalhost
+  Sentry.init({
+    dsn: sentryDSN,
+    enabled: isSentryEnabled,
+    environment: process.env.NODE_ENV,
+    release: `${options.errorHandler.service}@${process.env.GIT_HASH ?? 'unknown'}`,
+    serverName: options.errorHandler.serverName
+  })
+
+  // Opt-in rate limiter: only errors flagged via RequestError.rateLimitSentry()
+  // consult this. See sentry-rate-limit.js for details.
+  const sentryRateLimiter = options.errorHandler.sentryRateLimiter ??
+    createSentryRateLimiter()
+
+  const returnErrorDetail = options.errorHandler.returnErrorDetail
+  // log any exception which occurs
+  fastify.setErrorHandler(async (error, req, reply) => {
+    // extract the relevant bit of the traceback: remove fastify lines
+    // istanbul ignore next
+    const traceback = error.stack?.split('\n') ?? []
+    // istanbul ignore next
+    const errorMessage = error.message?.split('\n') ?? ''
+    traceback.splice(0, errorMessage.length)
+    let removeFromIdx
+    if (error instanceof InvalidInputException) {
+      removeFromIdx = 1
+    } else {
+      for (let i = traceback.length - 1; i > 0; i--) {
+        const tbLine = traceback[i]
+        if (tbLine.indexOf('/fastify/lib') !== -1) {
+          removeFromIdx = i + 1
+          break
+        }
+      }
+    }
+    removeFromIdx = removeFromIdx ?? traceback.length
+
+    traceback.splice(removeFromIdx, traceback.length - removeFromIdx)
+
+    const response = reply.raw
+    /* istanbul ignore next */
+    const message = error.message || 'empty error message'
+    const statusCode = error.httpCode ?? error.statusCode ?? 500
+    reply.code(statusCode)
+    const errInfo = {
+      msg: message,
+      req,
+      status: statusCode,
+      stack: traceback
+    }
+
+    // improve the error emitted from bad requests (invalid input)
+    const isCrash = errInfo.status >= 500
+    let customFingerprint = false
+    if (!isCrash) {
+      const firstTB = traceback[0]
+      /* istanbul ignore else */
+      if (firstTB) {
+        /* istanbul ignore else */
+        if (firstTB.indexOf('fastify/lib/contentTypeParser.js') !== -1) {
+          customFingerprint = 'Content-Type Not Permitted'
+        } else if (error instanceof S.ValidationError) {
+          customFingerprint = message
+        }
+        /* istanbul ignore next */
+        if (customFingerprint) {
+          if (customFingerprint.indexOf(errInfo.msg) === -1) {
+            // prefix the error message with the custom fingerprint text if the
+            // fingerprint didn't already contain all of the error message text
+            errInfo.msg = customFingerprint + ': ' + errInfo.msg
+          }
+          // changing the error name results in a cleaner description on the
+          // Sentry dashboard
+          error.name = customFingerprint
+        }
+      }
+    }
+
+    Object.getOwnPropertyNames(error).forEach(key => {
+      if (key !== 'stack' && key !== 'message') {
+        if (!errInfo.error) {
+          errInfo.error = {}
+        }
+        errInfo.error[key] = error[key]
+      }
+    })
+    response.logged = true // don't double-log
+    if (statusCode >= 500) {
+      reply.log.error(errInfo)
+    } else {
+      reply.log.info(errInfo)
+    }
+
+    // Check if this error is opted in to Sentry rate limiting; if so and we're
+    // inside the window, skip Sentry.captureException entirely. The HTTP
+    // response and logs are unaffected regardless.
+    let shouldCaptureToSentry = true
+    let suppressedCount = 0
+    const rateLimitMs = error._sentryRateLimitMs
+    if (rateLimitMs) {
+      const rlKey = customFingerprint || message
+      const rlResult = sentryRateLimiter.shouldReport(rlKey, rateLimitMs)
+      shouldCaptureToSentry = rlResult.report
+      suppressedCount = rlResult.suppressedCount
+    }
+
+    if (shouldCaptureToSentry) {
+      Sentry.withScope(function (scope) {
+        if (customFingerprint) {
+          scope.setFingerprint(customFingerprint)
+        }
+        const user = {}
+        // istanbul ignore if
+        if (req.headers['x-uid']) {
+          user.id = req.headers['x-uid']
+        } else {
+          user.ip = req.ip
+        }
+        scope.setLevel(isCrash ? 'error' : 'warning')
+        scope.setUser({
+          ...user,
+          ...(req.__sentry?.userInfo ?? {})
+        })
+        scope.setTags({
+          ...(req.__sentry?.tags ?? {}),
+          method: req.method,
+          url: req.url,
+          status: errInfo.status,
+          // Tag -- not extra -- so operators can filter Sentry for "issues
+          // where rate-limiting kicked in" during an outage.
+          ...(suppressedCount > 0
+            ? { suppressedSimilarEvents: String(suppressedCount) }
+            : {})
+        })
+        const customContexts = req.__sentry?.context ?? {}
+        for (const [k, v] of Object.entries(customContexts)) {
+          scope.setContext(k, v)
+        }
+        const extras = {
+          msg: errInfo.message,
+          reqId: req.id,
+          userAgent: req.headers['user-agent'] ?? 'not set'
+        }
+        if (error instanceof __RequestDone) {
+          extras.data = error.data
+        }
+        scope.setExtras(extras)
+        Sentry.captureException(error)
+      })
+    }
+
+    const errorData = error.respData ?? {
+      code: error.constructor.name,
+      message: customFingerprint || error.message
+    }
+
+    // istanbul ignore else
+    if (returnErrorDetail && !error.respData) {
+      errorData.detail = errInfo.msg
+      errorData.stack = errInfo.stack
+    }
+
+    await reply.header('Content-Type', 'application/json; charset=utf-8')
+      .serializer(o => JSON.stringify(o, null, 2))
+      .send(errorData)
+  })
+
+  next()
+})

package/src/plugins/health-check.js

@@ -0,0 +1,19 @@
+import fp from 'fastify-plugin'
+
+export default fp(function (fastify, options, next) {
+  // istanbul ignore if
+  if (options.healthCheck.disabled) {
+    next()
+    return
+  }
+  // istanbul ignore next
+  const path = options.healthCheck.path ?? '/'
+  fastify.get(path, { schema: { hide: true } },
+    async (req, reply) => {
+      await reply.send()
+    })
+  next()
+}, {
+  fastify: '>=3.x',
+  name: 'healthCheck'
+})

package/src/plugins/latency-tracker.js

@@ -0,0 +1,34 @@
+import fp from 'fastify-plugin'
+const symbolRequestTime = Symbol('RequestTimer')
+
+export default fp(function (fastify, options, next) {
+  // istanbul ignore if
+  if (options.latencyTracker.disabled) {
+    next()
+    return
+  }
+
+  const header = options.latencyTracker.header
+  fastify.addHook('onRequest', function onRequestHandler (req, reply, done) {
+    // Start the recording of process time
+    req.raw[symbolRequestTime] = process.hrtime.bigint()
+
+    done()
+  })
+
+  fastify.addHook('onSend', async (req, reply, payload) => {
+    const fixedDigits = 3
+    // Calculate the duration, in nanoseconds
+    const hrDuration = process.hrtime.bigint() - req.raw[symbolRequestTime]
+    // convert it to milliseconds
+    const duration = (Number(hrDuration) / 1e6).toFixed(fixedDigits)
+    // add the header to the response
+    reply.header(header, duration)
+    return payload
+  })
+
+  next()
+}, {
+  fastify: '>=3.x',
+  name: 'latency-tracker'
+})

package/src/plugins/sentry-rate-limit.js

@@ -0,0 +1,37 @@
+/**
+ * Creates a rate limiter for Sentry event reporting, keyed by an arbitrary
+ * string (typically the Sentry fingerprint or error message). State lives in
+ * a Map per rate limiter instance: one instance per process is the expected
+ * usage.
+ *
+ * @param {Function} [nowFn=Date.now] clock function (injectable for tests)
+ * @returns {Object} rate limiter with one method, {@link shouldReport}.
+ */
+export function createSentryRateLimiter (nowFn = Date.now) {
+  const tracking = new Map()
+  return {
+    /**
+     * Record an event and decide whether it should be reported to Sentry.
+     *
+     * @param {String} key rate-limit key; typically the error fingerprint or
+     *   message so that Sentry grouping lines up with our suppression.
+     * @param {Number} windowMs rate-limit window in ms; within this window
+     *   after a report, subsequent events for the same key are suppressed.
+     * @returns {{report: Boolean, suppressedCount: Number}} when `report` is
+     *   true, `suppressedCount` is the number of events suppressed for this
+     *   key since the last report (0 if none were suppressed). When `report`
+     *   is false, `suppressedCount` is 0 (caller should not surface it).
+     */
+    shouldReport (key, windowMs) {
+      const now = nowFn()
+      const state = tracking.get(key)
+      if (state && now - state.lastReported < windowMs) {
+        state.suppressed++
+        return { report: false, suppressedCount: 0 }
+      }
+      const suppressedCount = state?.suppressed ?? 0
+      tracking.set(key, { lastReported: now, suppressed: 0 })
+      return { report: true, suppressedCount }
+    }
+  }
+}

package/src/plugins/swagger.js

@@ -0,0 +1,50 @@
+// istanbul ignore file
+import swagger from '@fastify/swagger'
+import swaggerUI from '@fastify/swagger-ui'
+import fp from 'fastify-plugin'
+
+export default fp(function (fastify, options, next) {
+  if (options.swagger.disabled) {
+    next()
+    return
+  }
+
+  const authHeaders = options.swagger.authHeaders
+  fastify.register(swagger, {
+    openapi: {
+      servers: options.swagger.servers.map(x => { return { url: x } }),
+      info: {
+        title: `${options.swagger.service.toUpperCase()} Service`
+      },
+      consumes: ['application/json'],
+      produces: ['application/json'],
+      components: {
+        securitySchemes: authHeaders.reduce((all, c) => {
+          all[c.replace('x-', '')] = {
+            type: 'apiKey',
+            name: c,
+            in: 'header'
+          }
+          return all
+        }, {})
+      }
+    }
+  })
+  fastify.register(swaggerUI, {
+    routePrefix: options.swagger.routePrefix,
+    exposeRoute: true,
+    uiConfig: {
+      docExpansion: 'list',
+      deepLinking: false,
+      filter: true
+    }
+  })
+  // fastify.addHook('onReady', function (done) {
+  //   fastify.swagger()
+  //   done()
+  // })
+  next()
+}, {
+  fastify: '>=3.x',
+  name: 'swagger'
+})