@pbvision/cloud-run-service 0.0.49 → 0.0.53

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pbvision/cloud-run-service",
-  "version": "0.0.49",
+  "version": "0.0.53",
   "description": "fastify-firestore-service Web Framework on Cloud Run",
   "license": "Apache-2.0",
   "type": "module",
@@ -28,7 +28,8 @@
   },
   "dependencies": {
     "@google-cloud/tasks": "^4.0.1",
-    "@pbvision/fastify-firestore-service": "^0.0.53",
+    "@pbvision/fastify-firestore-service": "^0.0.55",
+    "@sentry/node": "^7.91.0",
     "google-auth-library": "^9.4.2",
     "ua-parser-js": "^1.0.37"
   },

package/src/analytics.js

@@ -1,7 +1,8 @@
 import assert from 'node:assert'
 import crypto, { randomUUID } from 'node:crypto'
 
-import { DatabaseAPI, EXCEPTIONS } from '@pbvision/fastify-firestore-service'
+import { DatabaseAPI } from '@pbvision/fastify-firestore-service'
+import * as Sentry from '@sentry/node'
 import UAParser from 'ua-parser-js'
 
 import { isProd } from './utils.js'
@@ -9,11 +10,53 @@ import { isProd } from './utils.js'
 // istanbul ignore next
 export const mixpanelToken = isProd ? '78c48e38f59ab21c1850740e2bb4ecff' : '52bd993b07bdba759c2f141345e7c32a'
 
+const mixpanelTrackURL = 'https://api.mixpanel.com/track'
 const mixpanelUpdateProfileURLs = {
   $set: 'https://api.mixpanel.com/engage#profile-set',
   $set_once: 'https://api.mixpanel.com/engage#profile-set-once'
 }
 
+// How long to wait before re-sending a Mixpanel request that failed in a way a
+// retry could fix. Short and few on purpose: this runs inside a live request,
+// so the caller waits out every retry.
+const MIXPANEL_RETRY_DELAYS_MS = [100, 300]
+
+// At most one Sentry report per failure kind per window, per process. A
+// Mixpanel outage affects every request we serve, so reporting each one would
+// exhaust the error budget in minutes.
+const ANALYTICS_FAILURE_REPORT_WINDOW_MS = 5 * 60 * 1000
+
+const analyticsFailureLastReportedAtMs = new Map()
+
+/**
+ * Whether a Mixpanel failure should be reported to Sentry now, or suppressed
+ * because an identical one was reported recently. Recording and deciding are
+ * one step: a suppressed failure does not extend the window.
+ *
+ * @param {String} key identifies the kind of failure; failures sharing a key
+ *   are suppressed as duplicates of each other
+ * @param {Number} [nowMs=Date.now()] current time (injectable for tests)
+ * @returns {Boolean} true if this failure should be sent to Sentry
+ */
+export function shouldReportAnalyticsFailure (key, nowMs = Date.now()) {
+  const lastReportedAtMs = analyticsFailureLastReportedAtMs.get(key)
+  if (lastReportedAtMs !== undefined &&
+      nowMs - lastReportedAtMs < ANALYTICS_FAILURE_REPORT_WINDOW_MS) {
+    return false
+  }
+  analyticsFailureLastReportedAtMs.set(key, nowMs)
+  return true
+}
+
+// Exported for tests: the suppression window outlives a single test case.
+export function resetAnalyticsFailureReporting () {
+  analyticsFailureLastReportedAtMs.clear()
+}
+
+function sleep (ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
 // Analytics are only sent if the transaction commits. Aborting or throwing an
 // exception from the tx will result in analytics NOT being sent to mixpanel.
 export class DatabaseAPIWithAnalytics extends DatabaseAPI {
@@ -64,13 +107,26 @@ export class DatabaseAPIWithAnalytics extends DatabaseAPI {
     updatesByUser[uid][key] = value
   }
 
+  /**
+   * Flushes everything queued by logAnalyticsEvent() and
+   * updateAnalyticsUserProfile() to Mixpanel.
+   *
+   * Never throws. This normally runs from postCommit(), i.e. after the
+   * transaction has already committed, so a Mixpanel failure here means the
+   * request did its work and only the telemetry about it was lost. It used to
+   * reject instead, which handed the caller a 500 (transport failure) or a 551
+   * (bad response) for a request that had succeeded: the write landed and the
+   * app still showed an error, and Mixpanel answering "temporary error ... try
+   * again in 30 seconds" was enough to do it. Failures are retried, then
+   * reported to Sentry as a warning and dropped.
+   */
   async sendAnalyticsEvents () {
     const events = this.__analyticsEvents
     this.__analyticsEvents = []
     const userProfileUpdates = this.__analyticsUserProfileUpdates
     this.__analyticsUserProfileUpdates = {}
 
-    const promises = []
+    const calls = []
     if (events.length) {
       const uaData = {}
       const parser = new UAParser(this.req.headers['user-agent'])
@@ -100,14 +156,8 @@ export class DatabaseAPIWithAnalytics extends DatabaseAPI {
         Object.assign(x.properties, uaData)
       }
       // send all the events in one Mixpanel API call
-      promises.push(this.callAPI({
-        method: 'POST',
-        url: 'https://api.mixpanel.com/track',
-        headers: { accept: 'text/plain' },
-        body: events
-      }))
+      calls.push(this.__makeMixpanelCall('track', mixpanelTrackURL, events))
     }
-    const sent = []
     for (const type of Object.keys(userProfileUpdates)) {
       const updatesByUser = userProfileUpdates[type]
       const body = []
@@ -119,28 +169,99 @@ export class DatabaseAPIWithAnalytics extends DatabaseAPI {
           [type]: updates
         })
       }
-      promises.push(this.__sendUserProfileUpdates(type, body))
-      sent.push({ type, body })
+      const url = mixpanelUpdateProfileURLs[type]
+      assert(url) // make sure a valid type was passed
+      calls.push(this.__makeMixpanelCall(type, url, body))
     }
-    const responses = await Promise.all(promises)
-    for (let i = 0; i < responses.length; i++) {
-      const resp = responses[i]
-      if (!resp.isOk || resp.data !== 1) {
-        console.log('error response from mixpanel', resp, sent)
-        throw new EXCEPTIONS.RequestError(
-          'failed to log analytics', { resp, sent }, 551)
+
+    const results = await Promise.all(
+      calls.map(call => this.__sendToMixpanel(call)))
+    const failures = results.filter(Boolean)
+    if (failures.length) {
+      this.__reportAnalyticsFailures(failures)
+    }
+  }
+
+  __makeMixpanelCall (type, url, body) {
+    return {
+      sent: { type, body },
+      request: {
+        method: 'POST',
+        url,
+        headers: { accept: 'text/plain' },
+        body
+      }
+    }
+  }
+
+  /**
+   * Sends one Mixpanel request, retrying the failures a retry can fix.
+   *
+   * @returns {Object|undefined} undefined if the call succeeded, otherwise a
+   *   description of the final failure
+   * @private
+   */
+  async __sendToMixpanel ({ request, sent }) {
+    const maxAttempts = MIXPANEL_RETRY_DELAYS_MS.length + 1
+    for (let attempt = 1; ; attempt++) {
+      const failure = await this.__attemptMixpanelCall(request, sent)
+      if (!failure) {
+        return undefined
       }
+      if (!failure.retryable || attempt === maxAttempts) {
+        return { ...failure, attempts: attempt }
+      }
+      await sleep(MIXPANEL_RETRY_DELAYS_MS[attempt - 1])
     }
   }
 
-  async __sendUserProfileUpdates (type, body) {
-    const mixpanelApiURL = mixpanelUpdateProfileURLs[type]
-    assert(mixpanelApiURL) // make sure a valid type was passed
-    return this.callAPI({
-      method: 'POST',
-      url: mixpanelApiURL,
-      headers: { accept: 'text/plain' },
-      body
+  /**
+   * Makes one attempt at a Mixpanel request.
+   *
+   * A rejected fetch (ECONNRESET, socket hang up) and a 5xx from Mixpanel are
+   * both transient, so they are worth another attempt. Anything else means
+   * Mixpanel understood us and said no -- a 4xx, or the `1` we expect in the
+   * body coming back as `0` -- which would fail identically on a retry.
+   *
+   * @returns {Object|undefined} undefined if the call succeeded, otherwise a
+   *   description of the failure
+   * @private
+   */
+  async __attemptMixpanelCall (request, sent) {
+    let resp
+    try {
+      resp = await this.callAPI(request)
+    } catch (err) {
+      return { url: request.url, sent, err: String(err), retryable: true }
+    }
+    if (resp.isOk && resp.data === 1) {
+      return undefined
+    }
+    return {
+      url: request.url,
+      sent,
+      resp,
+      retryable: !resp.isOk && resp.code >= 500
+    }
+  }
+
+  __reportAnalyticsFailures (failures) {
+    // One key per kind of failure, so an outage and a payload Mixpanel refuses
+    // are suppressed (and grouped in Sentry) separately.
+    const key = failures
+      .map(f => `${f.url} ${f.err ? 'transport error' : f.resp.code}`)
+      .sort().join(', ')
+    console.log('failed to send analytics to mixpanel', key, failures)
+    if (!shouldReportAnalyticsFailure(key)) {
+      return
+    }
+    Sentry.withScope(scope => {
+      // a warning, not an error: the request itself succeeded
+      scope.setLevel('warning')
+      scope.setFingerprint(['mixpanel-egress-failed', key])
+      scope.setTags({ method: this.req.method, url: this.req.url })
+      scope.setExtras({ failures, reqId: this.req.id })
+      Sentry.captureException(new Error(`failed to log analytics: ${key}`))
     })
   }
 }

package/test/unit-test-analytics.js

@@ -1,22 +1,70 @@
 import crypto from 'node:crypto'
 
-import { mixpanelToken } from '../src/analytics.js'
+import {
+  mixpanelToken, resetAnalyticsFailureReporting, shouldReportAnalyticsFailure
+} from '../src/analytics.js'
 
-import { AppTest, runTests } from './base-test.js'
+import { AppTest, BaseTest, runTests } from './base-test.js'
 
 class AnalyticsTest extends AppTest {
   async beforeEach () {
     await super.beforeEach()
     // mock using node-fetch to request the mixpanel APIs
     this.fetchMock.mockResp(1)
+    resetAnalyticsFailureReporting()
   }
 
-  async testMixpanelAPIFailure () {
+  // Mixpanel understood the request and refused it, so a retry would be
+  // refused too. The request still succeeds: only the telemetry is lost.
+  async testMixpanelRejectionIsNotRetried () {
     this.fetchMock.mockResp(0)
-    await this.sendBasicEvent(null, 551)
+    await this.sendBasicEvent()
+    expect(this.fetchMock.mock.calls.length).toBe(1)
 
+    this.fetchMock.mockClear()
+    this.fetchMock.mockResp('', 400)
+    await this.sendBasicEvent()
+    expect(this.fetchMock.mock.calls.length).toBe(1)
+  }
+
+  // A 5xx is transient (Mixpanel's own 502 body says to try again), so it is
+  // retried until the attempts run out.
+  async testMixpanelServerErrorIsRetried () {
     this.fetchMock.mockResp('', 500)
-    await this.sendBasicEvent(null, 551)
+    await this.sendBasicEvent()
+    expect(this.fetchMock.mock.calls.length).toBe(3)
+
+    // the second failure is the same kind as the first, so it is suppressed
+    // rather than reported to Sentry again
+    this.fetchMock.mockClear()
+    await this.sendBasicEvent()
+    expect(this.fetchMock.mock.calls.length).toBe(3)
+  }
+
+  // The failure that started this: node-fetch rejects before there is any
+  // response to inspect, e.g. "read ECONNRESET".
+  async testTransportFailureIsRetriedThenDropped () {
+    this.fetchMock.mockImplementation(
+      () => Promise.reject(new Error('read ECONNRESET')))
+    await this.sendBasicEvent()
+    expect(this.fetchMock.mock.calls.length).toBe(3)
+  }
+
+  async testTransportFailureRecoversOnRetry () {
+    let attempts = 0
+    this.fetchMock.mockImplementation(async () => {
+      attempts += 1
+      if (attempts === 1) {
+        throw new Error('socket hang up')
+      }
+      return {
+        status: 200,
+        headers: { get: () => 'application/json' },
+        text: async () => '1'
+      }
+    })
+    await this.sendBasicEvent()
+    expect(attempts).toBe(2)
   }
 
   async testNoAnalyticsLogged () {
@@ -277,4 +325,22 @@ class AnalyticsTest extends AppTest {
   }
 }
 
-runTests(AnalyticsTest)
+class AnalyticsFailureReportingTest extends BaseTest {
+  beforeEach () {
+    resetAnalyticsFailureReporting()
+  }
+
+  testSuppressesRepeatsInsideTheWindow () {
+    const key = 'some failure'
+    expect(shouldReportAnalyticsFailure(key, 1000)).toBe(true)
+    expect(shouldReportAnalyticsFailure(key, 1000)).toBe(false)
+    // a different kind of failure is tracked separately
+    expect(shouldReportAnalyticsFailure('another failure', 1000)).toBe(true)
+    // and a suppressed report does not push the window out
+    const fiveMinutesLater = 1000 + 5 * 60 * 1000
+    expect(shouldReportAnalyticsFailure(key, fiveMinutesLater - 1)).toBe(false)
+    expect(shouldReportAnalyticsFailure(key, fiveMinutesLater)).toBe(true)
+  }
+}
+
+runTests(AnalyticsTest, AnalyticsFailureReportingTest)