@pbvision/cloud-run-service 0.0.46

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "@pbvision/cloud-run-service",
+  "version": "0.0.46",
+  "description": "fastify-firestore-service Web Framework on Cloud Run",
+  "license": "Apache-2.0",
+  "type": "module",
+  "engines": {
+    "node": ">=20"
+  },
+  "exports": "./src/index.js",
+  "scripts": {
+    "coverage": "yarn -s start-local-db && yarn -s test --coverage",
+    "debug": "yarn start-local-db && INDEBUGGER=1 ./node_modules/nodemon/bin/nodemon.js --no-lazy --legacy-watch --watch ./src --watch ./test --exec 'node --inspect=9229 --env-file=.env --experimental-vm-modules ./node_modules/jest/bin/jest.js --config=./jest.config.json --runInBand'",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "setup": "yarn install --frozen-lockfile",
+    "start-local-db": "./node_modules/@pbvision/firestore-orm/scripts/start-local-db.sh",
+    "start-local": "yarn -s start-local-db && K_REVISION=localhost PORT=8090 node --env-file=.env test/main.js",
+    "test": "yarn -s start-local-db && yarn -s test-without-starting-db",
+    "test-without-starting-db": "node --env-file=.env --experimental-vm-modules ./node_modules/jest/bin/jest.js --config=./jest.config.json",
+    "watch": "yarn -s test --watch"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/pbv-public/cloud-run-service"
+  },
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org"
+  },
+  "dependencies": {
+    "@google-cloud/tasks": "^4.0.1",
+    "@pbvision/fastify-firestore-service": "^0.0.50",
+    "google-auth-library": "^9.4.2",
+    "ua-parser-js": "^1.0.37"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.17.12",
+    "@babel/eslint-parser": "^7.17.0",
+    "@babel/preset-env": "^7.17.12",
+    "@pbvision/jest-unit-test": "^0.2.3",
+    "babel-loader": "^9.1.3",
+    "eslint": "^8.22.0",
+    "eslint-config-standard": "17.1.0",
+    "eslint-import-resolver-webpack": "^0.13.8",
+    "eslint-plugin-import": "^2.22.0",
+    "eslint-plugin-n": "^16.6.2",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-promise": "^6.0.0",
+    "firebase-tools": "^13.5.2",
+    "jest": "^29.7.0",
+    "nodemon": "^3.1.0",
+    "standard": "^17.1.0",
+    "superagent": "^8",
+    "superagent-defaults": "^0.1.14",
+    "supertest": "^6.3.4",
+    "webpack": "^5.89.0"
+  },
+  "standard": {
+    "envs": [
+      "jest"
+    ],
+    "globals": [
+      "fail"
+    ],
+    "ignore": [
+      "**/node_modules/**"
+    ],
+    "parser": "@babel/eslint-parser"
+  }
+}

package/src/analytics.js

@@ -0,0 +1,164 @@
+import assert from 'node:assert'
+import crypto, { randomUUID } from 'node:crypto'
+
+import { DatabaseAPI, EXCEPTIONS } from '@pbvision/fastify-firestore-service'
+import UAParser from 'ua-parser-js'
+
+import { isProd } from './utils.js'
+
+// istanbul ignore next
+export const mixpanelToken = isProd ? '78c48e38f59ab21c1850740e2bb4ecff' : '52bd993b07bdba759c2f141345e7c32a'
+
+const mixpanelUpdateProfileURLs = {
+  $set: 'https://api.mixpanel.com/engage#profile-set',
+  $set_once: 'https://api.mixpanel.com/engage#profile-set-once'
+}
+
+// Analytics are only sent if the transaction commits. Aborting or throwing an
+// exception from the tx will result in analytics NOT being sent to mixpanel.
+export class DatabaseAPIWithAnalytics extends DatabaseAPI {
+  constructor (fastify, req, reply) {
+    super(fastify, req, reply)
+    this.__analyticsEvents = []
+    this.__analyticsUserProfileUpdates = {} // uid to $set/$set_once to changes
+  }
+
+  async postCommit (respData) {
+    // send analytics events after committing
+    await this.sendAnalyticsEvents()
+    return super.postCommit(respData)
+  }
+
+  logAnalyticsEvent (mixpanelUserId, eventName, inputProperties = {}, deviceId = null, insertId = null) {
+    if (insertId) {
+      // insert id must be <= 36 chars & only have alphanumeric & hyphen chars
+      insertId = crypto.createHash('md5').update(insertId).digest('hex')
+    } else {
+      insertId = randomUUID()
+    }
+
+    this.__analyticsEvents.push({
+      event: eventName,
+      properties: addSenderId(mixpanelUserId, {
+        ...inputProperties,
+        token: mixpanelToken,
+        time: new Date().getTime(),
+        $insert_id: insertId,
+        ip: this.req.ip
+      }, deviceId)
+    })
+  }
+
+  updateAnalyticsUserProfile (uid, key, value, method = '$set') {
+    assert(mixpanelUpdateProfileURLs[method])
+    // should be our user id not a device id... profile data is not recommended
+    // for anonymous users
+    assert(!uid.startsWith('$device:'))
+    if (!this.__analyticsUserProfileUpdates[method]) {
+      this.__analyticsUserProfileUpdates[method] = {}
+    }
+    const updatesByUser = this.__analyticsUserProfileUpdates[method]
+    if (!updatesByUser[uid]) {
+      updatesByUser[uid] = {}
+    }
+    updatesByUser[uid][key] = value
+  }
+
+  async sendAnalyticsEvents () {
+    const events = this.__analyticsEvents
+    this.__analyticsEvents = []
+    const userProfileUpdates = this.__analyticsUserProfileUpdates
+    this.__analyticsUserProfileUpdates = {}
+
+    const promises = []
+    if (events.length) {
+      const uaData = {}
+      const parser = new UAParser(this.req.headers['user-agent'])
+      const browser = parser.getBrowser()
+      if (browser.name) {
+        uaData.$browser = browser.name
+        // istanbul ignore else
+        if (browser.version) {
+          uaData.$browser += ` ${browser.version}`
+        }
+      }
+      const device = parser.getDevice()
+      const $device = [device.vendor, device.model, device.type].filter(x => !!x).join(' ')
+      if ($device) {
+        uaData.$device = $device
+      }
+      const os = parser.getOS()
+      if (os.name) {
+        uaData.$os = os.name
+        // istanbul ignore else
+        if (os.version) {
+          uaData.$os += ` ${os.version}`
+        }
+      }
+
+      for (const x of events) {
+        Object.assign(x.properties, uaData)
+      }
+      // send all the events in one Mixpanel API call
+      promises.push(this.callAPI({
+        method: 'POST',
+        url: 'https://api.mixpanel.com/track',
+        headers: { accept: 'text/plain' },
+        body: events
+      }))
+    }
+    const sent = []
+    for (const type of Object.keys(userProfileUpdates)) {
+      const updatesByUser = userProfileUpdates[type]
+      const body = []
+      for (const uid of Object.keys(updatesByUser)) {
+        const updates = updatesByUser[uid]
+        body.push({
+          $distinct_id: uid,
+          $token: mixpanelToken,
+          [type]: updates
+        })
+      }
+      promises.push(this.__sendUserProfileUpdates(type, body))
+      sent.push({ type, body })
+    }
+    const responses = await Promise.all(promises)
+    for (let i = 0; i < responses.length; i++) {
+      const resp = responses[i]
+      if (!resp.isOk || resp.data !== 1) {
+        console.log('error response from mixpanel', resp, sent)
+        throw new EXCEPTIONS.RequestError(
+          'failed to log analytics', { resp, sent }, 551)
+      }
+    }
+  }
+
+  async __sendUserProfileUpdates (type, body) {
+    const mixpanelApiURL = mixpanelUpdateProfileURLs[type]
+    assert(mixpanelApiURL) // make sure a valid type was passed
+    return this.callAPI({
+      method: 'POST',
+      url: mixpanelApiURL,
+      headers: { accept: 'text/plain' },
+      body
+    })
+  }
+}
+
+function addSenderId (mixpanelUserId, properties, deviceId) {
+  if (mixpanelUserId.startsWith('$device:')) {
+    properties.$device_id = mixpanelUserId.substring(8)
+  } else {
+    properties.$user_id = mixpanelUserId
+    if (deviceId) {
+      // istanbul ignore else
+      if (deviceId.startsWith('$device')) {
+        properties.$device_id = deviceId.substring(8)
+      } else {
+        properties.$device_id = deviceId
+      }
+    }
+  }
+  properties.distinct_id = mixpanelUserId
+  return properties
+}

package/src/app.js

@@ -0,0 +1,31 @@
+import { makeService } from '@pbvision/fastify-firestore-service'
+
+import { port } from './port.js'
+import { isProd } from './utils.js'
+
+export async function makePBVService (components, customizePinoOpts) {
+  return makeService({
+    service: process.env.SERVICE,
+    components,
+    cookie: {
+      secret: process.env.COOKIE_SECRET
+    },
+    healthCheck: {
+      path: '/_healthcheck'
+    },
+    latencyTracker: {
+      disabled: isProd
+    },
+    logging: {
+      customizePinoOpts,
+      reportErrorDetail: !isProd,
+      reportAllErrors: true,
+      sentryDSN: process.env.SENTRY_DSN
+    },
+    swagger: {
+      disabled: isProd,
+      servers: [`http://localhost:${port}`],
+      routePrefix: '/app/docs'
+    }
+  })
+}

package/src/call-service-api.js

@@ -0,0 +1,25 @@
+// add helper method to call APIs on this or other services; if the service is
+// internal then we'll make the request with our authorization token (only
+// works if this service has been granted access to the target service!)
+import { GoogleAuth } from 'google-auth-library'
+
+import { getServiceProtocolAndHost } from './utils.js'
+
+const auth = new GoogleAuth()
+
+// This function will be added to the API class.
+export async function callServiceAPI ({
+  path, service: serviceName,
+  body = undefined, qsParams = undefined,
+  method = 'POST', headers = {}, isServiceInternal = true
+}) {
+  const protocolAndHost = getServiceProtocolAndHost(serviceName)
+  const url = `${protocolAndHost}${path}`
+  if (isServiceInternal) {
+    const targetAudience = `${protocolAndHost}/`
+    const client = await auth.getIdTokenClient(targetAudience)
+    const token = await client.idTokenProvider.fetchIdToken(targetAudience)
+    headers.Authorization = `Bearer ${token}`
+  }
+  return this.callAPI({ method, headers, url, body, qsParams })
+}

package/src/index.js

@@ -0,0 +1,17 @@
+import { DatabaseAPIWithAnalytics, mixpanelToken } from './analytics.js'
+import { makeService, runService } from './main.js'
+import { enqueueCloudTask } from './tasks.js'
+import { isCloud, isDev, isProd, isLocalhost, isUnitTesting, getServiceProtocolAndHost } from './utils.js'
+
+export {
+  enqueueCloudTask,
+  makeService, // for use with unit testing
+  runService,
+
+  // analytics
+  DatabaseAPIWithAnalytics, mixpanelToken,
+
+  // utility
+  getServiceProtocolAndHost,
+  isCloud, isDev, isProd, isLocalhost, isUnitTesting
+}

package/src/main.js

@@ -0,0 +1,92 @@
+import assert from 'node:assert'
+
+import { API } from '@pbvision/fastify-firestore-service'
+
+import { makePBVService } from './app.js'
+import { callServiceAPI } from './call-service-api.js'
+import { port } from './port.js'
+import { isCloud, isLocalhost, isUnitTesting } from './utils.js'
+
+API.prototype.callServiceAPI = callServiceAPI
+
+let service
+const project = process.env.GCLOUD_PROJECT
+
+function verifyEnvironmentVariables () {
+  const requiredEnvKeys = [
+    'GCLOUD_PROJECT', 'K_REVISION', 'NODE_ENV', 'REGION', 'SERVICE']
+  // istanbul ignore else
+  if (isLocalhost) {
+    assert(['localhost', 'unittest'].indexOf(process.env.K_REVISION) !== -1,
+      'K_REVISION must be "localhost" or "unittest" in this environment')
+  } else {
+    // must have GIT_HASH to send to sentry in the cloud
+    requiredEnvKeys.push('GIT_HASH')
+  }
+  for (const k of requiredEnvKeys) {
+    assert(process.env[k], `${k} environment variable must be set`)
+  }
+}
+
+function makeCustomizeLoggingOptionsFunction () {
+  return options => {
+    options.formatters = {
+      level (label) {
+        return { severity: label }
+      },
+      // set messageKey to "message" for automatic parsing by GCP logs
+      messageKey: 'message'
+    }
+    const originalReqSerializer = options.serializers.req
+    options.serializers.req = req => {
+      const reqLog = originalReqSerializer(req)
+
+      // include the trace ID so logging can coordinate multiple logs from the
+      // same request per: https://github.com/GoogleCloudPlatform/cloud-run-microservice-template-nodejs/blob/main/utils/logging.js
+      const traceHeader = req.headers['X-Cloud-Trace-Context']
+      let trace
+      // istanbul ignore if
+      if (traceHeader) {
+        const [traceId] = traceHeader.split('/')
+        trace = `projects/${project}/traces/${traceId}`
+        reqLog['logging.googleapis.com/trace'] = trace
+      }
+      return reqLog
+    }
+    return options
+  }
+}
+
+export async function makeService (components) {
+  verifyEnvironmentVariables()
+  return makePBVService(components, makeCustomizeLoggingOptionsFunction())
+}
+
+// istanbul ignore next
+export async function runService (components) {
+  if (isCloud) {
+    // if the instance tells us it will shutdown, try to shut down gracefully (for
+    // example flushing logs)
+    process.on('SIGTERM', () => {
+      if (!service) {
+        return
+      }
+      // cloud run sends SIGTERM 10sec before killing the instance, so give the
+      // instance a little more time to finish any current requests then close
+      // the fastify instance (which kills any remaining requests, and should
+      // flush the logs and other clean up, if time permits)
+      setTimeout(() =>
+        service.close().then(() => {
+          console.log('successfully closed!')
+        }, (err) => {
+          console.log('an error happened', err)
+        }), 7000)
+    })
+  }
+  if (!isUnitTesting) {
+    // start the server
+    service = await makeService(components)
+    service.listen({ port, host: '0.0.0.0' })
+    return service
+  }
+}

package/src/placeholder.js

@@ -0,0 +1,71 @@
+// APIs in this file are not included in Docker builds so they never are
+// shipped to Cloud Run. They only exist for local testing.
+import assert from 'node:assert'
+
+import { API, DatabaseAPI } from '@pbvision/fastify-firestore-service'
+import db from '@pbvision/firestore-orm'
+import S from '@pbvision/schema'
+
+import { isLocalhost, isUnitTesting } from './utils.js'
+
+import { DatabaseAPIWithAnalytics } from './index.js'
+
+class Test extends db.Model {
+  static KEY = { id: S.str }
+  static FIELDS = { x: S.int }
+}
+
+export class TestAPI extends DatabaseAPI {
+  static METHOD = 'GET'
+  static PATH = '/time'
+  static DESC = 'Just for testing'
+  static RESPONSE = {
+    epoch: S.int
+  }
+
+  async computeResponse () {
+    const doesNotExist = await this.tx.get(Test, 'abc')
+    assert(doesNotExist === undefined)
+    return { epoch: Math.floor(new Date().getTime() / 1000) }
+  }
+}
+
+export class TestCallServiceAPI extends API {
+  static PATH = '/callService'
+  static DESC = 'This is used by unit tests only to test callServiceAPI.'
+  static BODY = S.obj()
+  static RESPONSE = { code: S.int, body: S.str }
+
+  async computeResponse () {
+    assert(isUnitTesting)
+    const resp = await this.callServiceAPI(this.req.body)
+    return {
+      code: resp.code,
+      body: typeof resp.data === 'object' ? JSON.stringify(resp.data) : (resp.data ?? '')
+    }
+  }
+}
+
+export class TestAnalyticsAPI extends DatabaseAPIWithAnalytics {
+  static PATH = '/analytics'
+  static DESC = 'This is used by unit tests only to test DatabaseAPIWithAnalytics.'
+  static BODY = S.obj()
+
+  async computeResponse () {
+    // istanbul ignore next
+    assert(isUnitTesting || isLocalhost)
+    const { eventCalls, profileUpdates, moreEventCalls } = this.req.body
+    for (const x of (eventCalls ?? [])) {
+      this.logAnalyticsEvent(...x)
+    }
+    for (const x of (profileUpdates ?? [])) {
+      this.updateAnalyticsUserProfile(...x)
+    }
+    if (moreEventCalls) {
+      await this.sendAnalyticsEvents()
+      for (const x of moreEventCalls) {
+        this.logAnalyticsEvent(...x)
+      }
+    }
+  }
+}

package/src/port.js

@@ -0,0 +1,2 @@
+// istanbul ignore next
+export const port = process.env.PORT ?? 8080

package/src/tasks.js

@@ -0,0 +1,117 @@
+import assert from 'node:assert'
+import crypto from 'node:crypto'
+
+import { CloudTasksClient } from '@google-cloud/tasks'
+import { credentials } from '@grpc/grpc-js'
+
+import { getServiceProtocolAndHost, usingEmulator } from './utils.js'
+
+const tasksClient = (() => {
+  // istanbul ignore else
+  if (usingEmulator) {
+    return new CloudTasksClient({
+      port: process.env.CLOUD_TASKS_EMULATOR_PORT,
+      servicePath: 'localhost',
+      sslCreds: credentials.createInsecure()
+    })
+  }
+  // istanbul ignore next
+  return new CloudTasksClient()
+})()
+
+/**
+ * Add a Task to a Cloud Tasks queue.
+ *
+ * The task will be routed to the internal API service on a path that matches
+ * the queue name (with hyphens replaced by underscores).
+ *
+ * @param {Object} task the task to enqueue
+ * @param {string} queue the name of the queue to add the task to
+ * @param {any} payload the data to convert to JSON add send as the task's body
+ * @param {string} [name] if provided, a name that is reused (on the same queue)
+ *   within about an hour will be rejected (TaskNameAlreadyExistsError)
+ * @param {Array<string>} [hashNameParts] if provided, the name param will be
+ *   generated from the combination of the queue name (namespacing this name)
+ *   and the name part(s) in this argument; the task name will be an md5 hash
+ *   of all this
+ * @param {string} [service="internal"] the service name that the task request
+ *   will be routed to
+ * @param {boolean} [ignoreNameAlreadyUsedError=false] if true, no error is
+ *   thrown due to a name already having been used
+ * @returns {boolean} true if a new task was added; false if the task name was
+ *   already recently used (no new task added, but a task was recently added
+ *   with this name)
+ */
+export async function enqueueCloudTask ({
+  queue, payload,
+  service = 'internal',
+  name = undefined,
+  hashNameParts = undefined,
+  ignoreNameAlreadyUsedError = false,
+  delaySecs = 0,
+  scheduledEpoch = undefined // can only be up to 30 days from now
+}) {
+  const project = process.env.GCLOUD_PROJECT
+  const region = process.env.REGION
+  const parent = tasksClient.queuePath(project, region, queue)
+  const protocolAndHost = getServiceProtocolAndHost(service)
+  const task = {
+    httpRequest: {
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      httpMethod: 'POST',
+      url: `${protocolAndHost}/${queue.replace(/-/g, '_')}`,
+      body: Buffer.from(JSON.stringify(payload)).toString('base64'),
+      oidcToken: {
+        serviceAccountEmail: `cr-${process.env.SERVICE}@${project}.iam.gserviceaccount.com`
+      }
+    }
+  }
+  const currentEpoch = new Date().getTime() / 1000
+  if (delaySecs) {
+    task.scheduleTime = {
+      seconds: Math.ceil(currentEpoch + delaySecs)
+    }
+    assert(!scheduledEpoch, 'cannot specify both delaysSecs and scheduledEpoch')
+  }
+  if (scheduledEpoch) {
+    scheduledEpoch = Math.floor(scheduledEpoch)
+    task.scheduleTime = { seconds: Math.floor(scheduledEpoch) }
+  }
+  if (task.scheduleTime) {
+    assert(task.scheduleTime.seconds <= currentEpoch + 30 * 86400, 'cannot delay for more than 30 days')
+  }
+  if (hashNameParts) {
+    assert(Array.isArray(hashNameParts))
+    assert(!name, 'cannot specify both name and hashNameParts')
+    const namespacedName = [queue, ...hashNameParts].join('|')
+    name = crypto.createHash('md5').update(namespacedName).digest('hex')
+  }
+  if (name) {
+    const fqName = tasksClient.taskPath(project, region, queue, name)
+    task.name = fqName
+  }
+  const request = { parent, task }
+  try {
+    await tasksClient.createTask(request)
+    return true
+  } catch (e) {
+    if (e.code === 6 && e.message.startsWith('6 ALREADY_EXISTS')) {
+      if (ignoreNameAlreadyUsedError) {
+        return false
+      } else {
+        throw new TaskNameAlreadyExistsError(name, e)
+      }
+    }
+    throw e
+  }
+}
+
+export class TaskNameAlreadyExistsError extends Error {
+  constructor (name, e) {
+    super(`task name already used recently: ${name}`)
+    this.taskName = name
+    this.originalError = e
+  }
+}

package/src/utils.js

@@ -0,0 +1,44 @@
+import assert from 'node:assert'
+
+import { port as portForThisService } from './port.js'
+
+// this refers to where the code is running; the database and other services
+// we are connected to depends on GCLOUD_PROJECT
+export const isLocalhost = process.env.NODE_ENV === 'localhost'
+export const isCloud = !isLocalhost
+export const isDev = process.env.NODE_ENV === 'dev'
+export const isProd = process.env.NODE_ENV === 'prod'
+// istanbul ignore next
+assert(isLocalhost || isDev || isProd,
+  'invalid NODE_ENV: must be on localhost, dev or prod')
+
+export const isUnitTesting = process.env.K_REVISION === 'unittest'
+assert(!isUnitTesting || isLocalhost, 'must be on localhost if unit testing')
+
+export const usingEmulator = !!process.env.FIRESTORE_EMULATOR_HOST
+assert(!usingEmulator || process.env.CLOUD_TASKS_EMULATOR_PORT,
+  'both firestore and tasks need to be either emulated or not')
+assert(usingEmulator === (process.env.GCLOUD_PROJECT === 'localhost-emulator'),
+  'when using the emulator, GCLOUD_PROJECT should be set to localhost-emulator')
+
+export function getServiceProtocolAndHost (serviceName) {
+  const host = getServiceHost(serviceName)
+  // istanbul ignore next
+  const protocol = host.startsWith('localhost') ? 'http' : 'https'
+  return `${protocol}://${host}`
+}
+
+export function getServiceHost (serviceName) {
+  // manually check NODE_ENV for testing purposes
+  if (process.env.NODE_ENV === 'localhost') {
+    if (process.env.SERVICE === serviceName) {
+      return `localhost:${portForThisService}`
+    }
+    const portMapping = JSON.parse(process.env.LOCAL_SERVICE_PORT_MAP)
+    const port = portMapping[serviceName]
+    assert(port, `unknown service or missing port for localhost ${serviceName}`)
+    return `localhost:${port}`
+  } else {
+    return serviceName + process.env.CLOUD_RUN_HOSTNAME_SUFFIX
+  }
+}

package/test/base-test.js

@@ -0,0 +1,17 @@
+import { BaseAppTest, BaseTest, runTests } from '../node_modules/@pbvision/fastify-firestore-service/test/base-test.js'
+
+const { TestAnalyticsAPI, TestAPI, TestCallServiceAPI } = await import('../src/placeholder.js')
+export {
+  BaseTest, runTests
+}
+
+export class AppTest extends BaseAppTest {
+  async getMakeServiceFunc () {
+    const { makeService } = await import('../src/main.js')
+    return () => makeService({
+      TestAnalyticsAPI,
+      TestAPI,
+      TestCallServiceAPI
+    })
+  }
+}

package/test/main.js

@@ -0,0 +1,4 @@
+import { runService } from '../src/index.js'
+const { TestAnalyticsAPI, TestAPI, TestCallServiceAPI } = await import('../src/placeholder.js')
+
+await runService({ TestAnalyticsAPI, TestAPI, TestCallServiceAPI })