@pbvision/cloud-run-service 0.0.29 → 0.0.31

package/.env

@@ -1,5 +1,5 @@
 CLOUD_TASKS_EMULATOR_PORT=8123
-FIRESTORE_EMULATOR_HOST=[::1]:8404
+FIRESTORE_EMULATOR_HOST=127.0.0.1:9091
 GCLOUD_PROJECT=localhost-emulator
 K_REVISION=unittest
 LOCAL_SERVICE_PORT_MAP='{}'

package/.vscode/settings.json

@@ -33,6 +33,7 @@
     "gserviceaccount",
     "INDEBUGGER",
     "oidc",
-    "pino"
+    "pino",
+    "uids"
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pbvision/cloud-run-service",
-  "version": "0.0.29",
+  "version": "0.0.31",
   "description": "fastify-firestore-service Web Framework on Cloud Run",
   "license": "Apache-2.0",
   "type": "module",
@@ -10,11 +10,11 @@
   "exports": "./src/index.js",
   "scripts": {
     "coverage": "yarn -s start-local-db && yarn -s test --coverage",
-    "debug": "./node_modules/nodemon/bin/nodemon.js --no-lazy --legacy-watch --watch ./src --watch ./test --inspect=9229 node --experimental-vm-modules --exec yarn test --runInBand",
+    "debug": "yarn start-local-db && INDEBUGGER=1 ./node_modules/nodemon/bin/nodemon.js --no-lazy --legacy-watch --watch ./src --watch ./test --exec 'node --inspect=9229 --env-file=.env --experimental-vm-modules ./node_modules/jest/bin/jest.js --config=./jest.config.json --runInBand'",
     "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
     "setup": "yarn install --frozen-lockfile",
     "start-local-db": "./node_modules/@pbvision/firestore-orm/scripts/start-local-db.sh",
-    "start-local": "yarn -s start-local-db && K_REVISION=localhost node --env-file=.env test/main.js",
+    "start-local": "yarn -s start-local-db && K_REVISION=localhost PORT=8090 node --env-file=.env test/main.js",
     "test": "yarn -s start-local-db && yarn -s test-without-starting-db",
     "test-without-starting-db": "node --env-file=.env --experimental-vm-modules ./node_modules/jest/bin/jest.js --config=./jest.config.json",
     "watch": "yarn -s test --watch"
@@ -29,7 +29,8 @@
   "dependencies": {
     "@google-cloud/tasks": "^4.0.1",
     "@pbvision/fastify-firestore-service": "^0.0.38",
-    "google-auth-library": "^9.4.2"
+    "google-auth-library": "^9.4.2",
+    "ua-parser-js": "^1.0.37"
   },
   "devDependencies": {
     "@babel/core": "^7.17.12",
@@ -46,6 +47,7 @@
     "eslint-plugin-promise": "^6.0.0",
     "firebase-tools": "^13.5.2",
     "jest": "^29.7.0",
+    "nodemon": "^3.1.0",
     "standard": "^17.1.0",
     "superagent": "^8",
     "superagent-defaults": "^0.1.14",

package/src/analytics.js

@@ -0,0 +1,151 @@
+import assert from 'node:assert'
+import { randomUUID } from 'node:crypto'
+
+import { DatabaseAPI, EXCEPTIONS } from '@pbvision/fastify-firestore-service'
+import UAParser from 'ua-parser-js'
+
+import { isProd } from './utils.js'
+
+// istanbul ignore next
+export const mixpanelToken = isProd ? '78c48e38f59ab21c1850740e2bb4ecff' : '52bd993b07bdba759c2f141345e7c32a'
+
+const mixpanelUpdateProfileURLs = {
+  $set: 'https://api.mixpanel.com/engage#profile-set',
+  $set_once: 'https://api.mixpanel.com/engage#profile-set-once'
+}
+
+// Analytics are only sent if the transaction commits. Aborting or throwing an
+// exception from the tx will result in analytics NOT being sent to mixpanel.
+export class DatabaseAPIWithAnalytics extends DatabaseAPI {
+  constructor (fastify, req, reply) {
+    super(fastify, req, reply)
+    this.__analyticsEvents = []
+    this.__analyticsUserProfileUpdates = {} // uid to $set/$set_once to changes
+  }
+
+  async postCommit (respData) {
+    // send analytics events after committing
+    await this.sendAnalyticsEvents()
+    return super.postCommit(respData)
+  }
+
+  logAnalyticsEvent (mixpanelUserId, eventName, inputProperties, deviceId = null) {
+    this.__analyticsEvents.push({
+      event: eventName,
+      properties: addSenderId(mixpanelUserId, {
+        ...inputProperties,
+        token: mixpanelToken,
+        time: new Date().getTime(),
+        $insert_id: randomUUID(),
+        ip: this.req.ip
+      }, deviceId)
+    })
+  }
+
+  updateAnalyticsUserProfile (uid, key, value, method = '$set') {
+    assert(mixpanelUpdateProfileURLs[method])
+    // should be our user id not a device id... profile data is not recommended
+    // for anonymous users
+    assert(!uid.startsWith('$device:'))
+    if (!this.__analyticsUserProfileUpdates[method]) {
+      this.__analyticsUserProfileUpdates[method] = {}
+    }
+    const updatesByUser = this.__analyticsUserProfileUpdates[method]
+    if (!updatesByUser[uid]) {
+      updatesByUser[uid] = {}
+    }
+    updatesByUser[uid][key] = value
+  }
+
+  async sendAnalyticsEvents () {
+    const events = this.__analyticsEvents
+    this.__analyticsEvents = []
+    const userProfileUpdates = this.__analyticsUserProfileUpdates
+    this.__analyticsUserProfileUpdates = {}
+
+    const promises = []
+    if (events.length) {
+      const uaData = {}
+      const parser = new UAParser(this.req.headers['user-agent'])
+      const browser = parser.getBrowser()
+      if (browser.name) {
+        uaData.$browser = browser.name
+        // istanbul ignore else
+        if (browser.version) {
+          uaData.$browser += ` ${browser.version}`
+        }
+      }
+      const device = parser.getDevice()
+      const $device = [device.vendor, device.model, device.type].filter(x => !!x).join(' ')
+      if ($device) {
+        uaData.$device = $device
+      }
+      const os = parser.getOS()
+      if (os.name) {
+        uaData.$os = os.name
+        // istanbul ignore else
+        if (os.version) {
+          uaData.$os += ` ${os.version}`
+        }
+      }
+
+      for (const x of events) {
+        Object.assign(x.properties, uaData)
+      }
+      // send all the events in one Mixpanel API call
+      promises.push(this.callAPI({
+        method: 'POST',
+        url: 'https://api.mixpanel.com/track',
+        headers: { accept: 'text/plain' },
+        body: events
+      }))
+    }
+    const sent = []
+    for (const type of Object.keys(userProfileUpdates)) {
+      const updatesByUser = userProfileUpdates[type]
+      const body = []
+      for (const uid of Object.keys(updatesByUser)) {
+        const updates = updatesByUser[uid]
+        body.push({
+          $distinct_id: uid,
+          $token: mixpanelToken,
+          [type]: updates
+        })
+      }
+      promises.push(this.__sendUserProfileUpdates(type, body))
+      sent.push({ type, body })
+    }
+    const responses = await Promise.all(promises)
+    for (let i = 0; i < responses.length; i++) {
+      const resp = responses[i]
+      if (!resp.isOk || resp.data !== 1) {
+        console.log('error response from mixpanel', resp, sent)
+        throw new EXCEPTIONS.RequestError(
+          'failed to log analytics', { resp, sent }, 551)
+      }
+    }
+  }
+
+  async __sendUserProfileUpdates (type, body) {
+    const mixpanelApiURL = mixpanelUpdateProfileURLs[type]
+    assert(mixpanelApiURL) // make sure a valid type was passed
+    return this.callAPI({
+      method: 'POST',
+      url: mixpanelApiURL,
+      headers: { accept: 'text/plain' },
+      body
+    })
+  }
+}
+
+function addSenderId (mixpanelUserId, properties, deviceId) {
+  if (mixpanelUserId.startsWith('$device:')) {
+    properties.$device_id = mixpanelUserId
+  } else {
+    properties.$user_id = mixpanelUserId
+    if (deviceId) {
+      properties.$device_id = deviceId
+    }
+  }
+  return properties
+}

package/src/index.js

@@ -1,3 +1,4 @@
+import { DatabaseAPIWithAnalytics, mixpanelToken } from './analytics.js'
 import { makeService, runService } from './main.js'
 import { enqueueCloudTask } from './tasks.js'
 import { isCloud, isDev, isProd, isLocalhost, isUnitTesting, getServiceProtocolAndHost } from './utils.js'
@@ -7,6 +8,10 @@ export {
   makeService, // for use with unit testing
   runService,
 
+  // analytics
+  DatabaseAPIWithAnalytics, mixpanelToken,
+
+  // utility
   getServiceProtocolAndHost,
   isCloud, isDev, isProd, isLocalhost, isUnitTesting
 }

package/src/placeholder.js

@@ -6,7 +6,9 @@ import { API, DatabaseAPI } from '@pbvision/fastify-firestore-service'
 import db from '@pbvision/firestore-orm'
 import S from '@pbvision/schema'
 
-import { isUnitTesting } from './utils.js'
+import { isLocalhost, isUnitTesting } from './utils.js'
+
+import { DatabaseAPIWithAnalytics } from './index.js'
 
 class Test extends db.Model {
   static KEY = { id: S.str }
@@ -43,3 +45,27 @@ export class TestCallServiceAPI extends API {
     }
   }
 }
+
+export class TestAnalyticsAPI extends DatabaseAPIWithAnalytics {
+  static PATH = '/analytics'
+  static DESC = 'This is used by unit tests only to test DatabaseAPIWithAnalytics.'
+  static BODY = S.obj()
+
+  async computeResponse () {
+    // istanbul ignore next
+    assert(isUnitTesting || isLocalhost)
+    const { eventCalls, profileUpdates, moreEventCalls } = this.req.body
+    for (const x of (eventCalls ?? [])) {
+      this.logAnalyticsEvent(...x)
+    }
+    for (const x of (profileUpdates ?? [])) {
+      this.updateAnalyticsUserProfile(...x)
+    }
+    if (moreEventCalls) {
+      await this.sendAnalyticsEvents()
+      for (const x of moreEventCalls) {
+        this.logAnalyticsEvent(...x)
+      }
+    }
+  }
+}

package/test/base-test.js

@@ -1,6 +1,6 @@
 import { BaseAppTest, BaseTest, runTests } from '../node_modules/@pbvision/fastify-firestore-service/test/base-test.js'
-const { TestAPI, TestCallServiceAPI } = await import('../src/placeholder.js')
 
+const { TestAnalyticsAPI, TestAPI, TestCallServiceAPI } = await import('../src/placeholder.js')
 export {
   BaseTest, runTests
 }
@@ -8,6 +8,10 @@ export {
 export class AppTest extends BaseAppTest {
   async getMakeServiceFunc () {
     const { makeService } = await import('../src/main.js')
-    return () => makeService({ TestAPI, TestCallServiceAPI })
+    return () => makeService({
+      TestAnalyticsAPI,
+      TestAPI,
+      TestCallServiceAPI
+    })
   }
 }

package/test/main.js

@@ -1,4 +1,4 @@
 import { runService } from '../src/index.js'
-const { TestAPI, TestCallServiceAPI } = await import('../src/placeholder.js')
+const { TestAnalyticsAPI, TestAPI, TestCallServiceAPI } = await import('../src/placeholder.js')
 
-await runService({ TestAPI, TestCallServiceAPI })
+await runService({ TestAnalyticsAPI, TestAPI, TestCallServiceAPI })

package/test/unit-test-analytics.js

@@ -0,0 +1,235 @@
+import { mixpanelToken } from '../src/analytics.js'
+
+import { AppTest, runTests } from './base-test.js'
+
+class AnalyticsTest extends AppTest {
+  async beforeEach () {
+    await super.beforeEach()
+    // mock using node-fetch to request the mixpanel APIs
+    this.fetchMock.mockResp(1)
+  }
+
+  async testMixpanelAPIFailure () {
+    this.fetchMock.mockResp(0)
+    await this.sendBasicEvent(null, 551)
+
+    this.fetchMock.mockResp('', 500)
+    await this.sendBasicEvent(null, 551)
+  }
+
+  async testNoAnalyticsLogged () {
+    await this.app.post('/analytics')
+      .send({})
+      .expect(200)
+    expect(this.fetchMock).not.toHaveBeenCalled()
+  }
+
+  async testSendingEvents () {
+    await this.sendBasicEvent()
+    expect(this.fetchMock).toHaveBeenCalledWith(
+      'https://api.mixpanel.com/track',
+      expect.objectContaining({
+        headers: { accept: 'text/plain', 'content-type': 'application/json' },
+        method: 'POST',
+        compress: true
+      }))
+    const calls = this.fetchMock.mock.calls
+    expect(calls.length).toBe(1)
+    const actualBody = JSON.parse(calls[0][1].body)
+    expect(actualBody).toEqual([{
+      event: 'some event',
+      properties: expect.objectContaining({
+        cool: 1,
+        hi: 'world',
+        $user_id: 'some uid',
+        token: mixpanelToken
+      })
+    }])
+    const props = actualBody[0].properties
+    expect(typeof props.ip).toBe('string')
+    expect(new Date().getTime() - props.time).toBeLessThan(5000)
+    expect(props.$insert_id).toMatch(/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/)
+  }
+
+  async sendBasicEvent (userAgent, expRespCode = 200) {
+    const req = this.app.post('/analytics')
+    if (userAgent) {
+      req.set('User-Agent', userAgent)
+    }
+    await req.send({
+      eventCalls: [
+        ['some uid', 'some event', { cool: 1, hi: 'world' }]
+      ]
+    }).expect(expRespCode)
+  }
+
+  async testSendInBatches () {
+    await this.app.post('/analytics')
+      .send({
+        eventCalls: [
+          ['xyz', 'some event', { cool: 1, hi: 'world' }]
+        ],
+        moreEventCalls: [
+          ['xyz', 'another event', { x: 2 }]
+        ]
+      }).expect(200)
+    const calls = this.fetchMock.mock.calls
+    expect(calls.length).toBe(2)
+    const actualBody = JSON.parse(calls[0][1].body)
+    expect(actualBody).toEqual([{
+      event: 'some event',
+      properties: expect.objectContaining({
+        cool: 1,
+        hi: 'world',
+        $user_id: 'xyz',
+        token: mixpanelToken
+      })
+    }])
+    const actualBody2 = JSON.parse(calls[1][1].body)
+    expect(actualBody2).toEqual([{
+      event: 'another event',
+      properties: expect.objectContaining({
+        x: 2,
+        $user_id: 'xyz',
+        token: mixpanelToken
+      })
+    }])
+  }
+
+  async testDeviceId () {
+    await this.app.post('/analytics')
+      .send({
+        eventCalls: [
+          ['$device:xyz', 'some event', { cool: 1, hi: 'world' }]
+        ]
+      }).expect(200)
+    const calls = this.fetchMock.mock.calls
+    expect(calls.length).toBe(1)
+    const actualBody = JSON.parse(calls[0][1].body)
+    expect(actualBody).toEqual([{
+      event: 'some event',
+      properties: expect.objectContaining({
+        cool: 1,
+        hi: 'world',
+        $device_id: '$device:xyz',
+        token: mixpanelToken
+      })
+    }])
+  }
+
+  async testDeviceIdWithUserId () {
+    await this.app.post('/analytics')
+      .send({
+        eventCalls: [
+          ['some uid', 'some event', { cool: 1, hi: 'world' }, '$device:xyz']
+        ]
+      }).expect(200)
+    const calls = this.fetchMock.mock.calls
+    expect(calls.length).toBe(1)
+    const actualBody = JSON.parse(calls[0][1].body)
+    expect(actualBody).toEqual([{
+      event: 'some event',
+      properties: expect.objectContaining({
+        cool: 1,
+        hi: 'world',
+        $user_id: 'some uid',
+        $device_id: '$device:xyz',
+        token: mixpanelToken
+      })
+    }])
+  }
+
+  async testUserAgent () {
+    await this.sendBasicEvent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36')
+    const calls = this.fetchMock.mock.calls
+    expect(calls.length).toBe(1)
+    const actualBody = JSON.parse(calls[0][1].body)
+    const props = actualBody[0].properties
+    expect(props.$browser).toBe('Chrome 122.0.0.0')
+    expect(props.$device).toBe(undefined)
+    expect(props.$os).toBe('Windows 10')
+  }
+
+  async testUserAgentWithDevice () {
+    await this.sendBasicEvent('Mozilla/5.0 (PlayBook; U; RIM Tablet OS 1.0.0; en-US) AppleWebKit/534.11 (KHTML, like Gecko) Version/7.1.0.7 Safari/534.11')
+    const calls = this.fetchMock.mock.calls
+    expect(calls.length).toBe(1)
+    const actualBody = JSON.parse(calls[0][1].body)
+    const props = actualBody[0].properties
+    expect(props.$browser).toBe('Safari 7.1.0.7')
+    expect(props.$device).toBe('RIM PlayBook tablet')
+    expect(props.$os).toBe('RIM Tablet OS 1.0.0')
+  }
+
+  async sendUserProfileUpdate (profileUpdates) {
+    const req = this.app.post('/analytics')
+    await req.send({ profileUpdates }).expect(200)
+  }
+
+  async testUserProfileUpdates () {
+    const uids = ['uid0', 'uid1']
+    await this.sendUserProfileUpdate([
+      [uids[0], 'p1', true],
+      [uids[0], 'p2', 5],
+      [uids[1], 'p2', 6],
+      [uids[1], 'p1', 'nope', '$set'],
+      [uids[1], 'p1', 'cool'],
+      [uids[1], 'p1', 'once', '$set_once'],
+      [uids[0], 'p1', false],
+      [uids[0], 'p3', 'hi']
+    ])
+    const calls = this.fetchMock.mock.calls
+    expect(calls.length).toBe(2)
+    const setURL = 'https://api.mixpanel.com/engage#profile-set'
+    const setOnceURL = 'https://api.mixpanel.com/engage#profile-set-once'
+    expect(this.fetchMock).toHaveBeenCalledWith(
+      setURL,
+      expect.objectContaining({
+        headers: { accept: 'text/plain', 'content-type': 'application/json' },
+        method: 'POST',
+        compress: true
+      }))
+    expect(this.fetchMock).toHaveBeenCalledWith(
+      setOnceURL,
+      expect.objectContaining({
+        headers: { accept: 'text/plain', 'content-type': 'application/json' },
+        method: 'POST',
+        compress: true
+      }))
+    // sort $set first, then $set_once (just so we have a consistent order for
+    // the test to check)
+    calls.sort((a, b) => a[0].localeCompare(b[0]))
+
+    // check $set updates
+    const actualSetUpdates = JSON.parse(calls[0][1].body)
+    actualSetUpdates.sort((a, b) => a.$distinct_id.localeCompare(b.$distinct_id))
+    const expectedSets = [
+      { p1: false, p2: 5, p3: 'hi' },
+      { p1: 'cool', p2: 6 }
+    ]
+    for (let i = 0; i < actualSetUpdates.length; i++) {
+      const actualArgs = actualSetUpdates[i]
+      expect(actualArgs).toEqual({
+        $token: mixpanelToken,
+        $distinct_id: uids[i],
+        $set: expectedSets[i]
+      })
+    }
+
+    // check $set_once updates
+    const actualSetOnceUpdates = JSON.parse(calls[1][1].body)
+    const expectedSetOnces = [
+      { p1: 'once' }
+    ]
+    for (let i = 0; i < actualSetOnceUpdates.length; i++) {
+      const actualArgs = actualSetOnceUpdates[i]
+      expect(actualArgs).toEqual({
+        $token: mixpanelToken,
+        $distinct_id: uids[1],
+        $set_once: expectedSetOnces[i]
+      })
+    }
+  }
+}
+
+runTests(AnalyticsTest)