@stacksjs/ts-analytics 0.1.6

package/src/lib/ga4-api.ts

@@ -0,0 +1,244 @@
+/**
+ * GA4 Data API importer — Phase B of #155.
+ *
+ * Pulls a property's full daily history straight from Google's Analytics
+ * Data API and feeds it through the same write phase as the CSV importer
+ * (collision policy, zero-fill, batched rollup writes).
+ *
+ * Auth is a SERVICE ACCOUNT key, not OAuth: the user creates a service
+ * account in Google Cloud, grants its email Viewer access on the GA4
+ * property, and pastes the JSON key. No consent screens, no app
+ * verification, no refresh tokens. The key is used for this import only and
+ * is NEVER persisted.
+ *
+ * Test seams: GA4_TOKEN_URL / GA4_API_BASE env overrides let the test
+ * harness stand in for Google.
+ */
+import { type DayAccumulator, dayAcc, writeImportedDays, type GaImportResult } from './ga-import'
+
+export interface ServiceAccountKey {
+  client_email: string
+  private_key: string
+  token_uri?: string
+}
+
+const TOKEN_URL = (): string => process.env.GA4_TOKEN_URL || 'https://oauth2.googleapis.com/token'
+const API_BASE = (): string => process.env.GA4_API_BASE || 'https://analyticsdata.googleapis.com'
+
+function b64url(data: Uint8Array | string): string {
+  const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data
+  let bin = ''
+  for (const b of bytes) bin += String.fromCharCode(b)
+  return btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '')
+}
+
+/** PEM (PKCS8) → DER bytes (backed by a plain ArrayBuffer for WebCrypto). */
+function pemToDer(pem: string): Uint8Array<ArrayBuffer> {
+  const body = pem.replace(/-----[A-Z ]+-----/g, '').replace(/\s+/g, '')
+  const bin = atob(body)
+  const buf = new ArrayBuffer(bin.length)
+  const out = new Uint8Array(buf)
+  for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i)
+  return out
+}
+
+/** Exchange the service-account key for an access token (RS256 JWT grant). */
+export async function getAccessToken(key: ServiceAccountKey): Promise<string> {
+  const now = Math.floor(Date.now() / 1000)
+  const header = b64url(JSON.stringify({ alg: 'RS256', typ: 'JWT' }))
+  const claims = b64url(JSON.stringify({
+    iss: key.client_email,
+    scope: 'https://www.googleapis.com/auth/analytics.readonly',
+    aud: key.token_uri || TOKEN_URL(),
+    iat: now,
+    exp: now + 3600,
+  }))
+  const signingInput = `${header}.${claims}`
+  const cryptoKey = await crypto.subtle.importKey(
+    'pkcs8',
+    pemToDer(key.private_key),
+    { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' },
+    false,
+    ['sign'],
+  )
+  const signature = new Uint8Array(await crypto.subtle.sign('RSASSA-PKCS1-v1_5', cryptoKey, new TextEncoder().encode(signingInput)))
+  const jwt = `${signingInput}.${b64url(signature)}`
+
+  const res = await fetch(TOKEN_URL(), {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    body: `grant_type=${encodeURIComponent('urn:ietf:params:oauth:grant-type:jwt-bearer')}&assertion=${encodeURIComponent(jwt)}`,
+  })
+  if (!res.ok) {
+    const body = await res.text()
+    throw new Error(`Google token exchange failed (${res.status}): ${body.slice(0, 300)}`)
+  }
+  const data = await res.json() as { access_token?: string }
+  if (!data.access_token)
+    throw new Error('Google token exchange returned no access_token')
+  return data.access_token
+}
+
+interface GaReportRow {
+  dimensionValues?: Array<{ value?: string }>
+  metricValues?: Array<{ value?: string }>
+}
+
+interface RunReportResponse {
+  rows?: GaReportRow[]
+  rowCount?: number
+}
+
+async function runReport(
+  token: string,
+  propertyId: string,
+  body: Record<string, unknown>,
+): Promise<GaReportRow[]> {
+  const rows: GaReportRow[] = []
+  const limit = 100_000
+  let offset = 0
+  for (let page = 0; page < 30; page++) {
+    const res = await fetch(`${API_BASE()}/v1beta/properties/${propertyId}:runReport`, {
+      method: 'POST',
+      headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ ...body, limit, offset }),
+    })
+    if (!res.ok) {
+      const errBody = await res.text()
+      throw new Error(`GA4 runReport failed (${res.status}): ${errBody.slice(0, 300)}`)
+    }
+    const data = await res.json() as RunReportResponse
+    rows.push(...(data.rows || []))
+    offset += limit
+    if (!data.rows || data.rows.length < limit || rows.length >= (data.rowCount ?? rows.length))
+      break
+  }
+  return rows
+}
+
+const nnum = (v: string | undefined): number => {
+  const n = Number(v)
+  return Number.isFinite(n) ? n : 0
+}
+
+function isoDay(gaDate: string | undefined): string | null {
+  const m = String(gaDate || '').match(/^(\d{4})(\d{2})(\d{2})$/)
+  return m ? `${m[1]}-${m[2]}-${m[3]}` : null
+}
+
+/** The report set: one per rollup dimension family (+ daily scalars). */
+export const GA4_REPORTS: Array<{ kind: string, dimensions: string[], metrics: string[] }> = [
+  { kind: 'traffic', dimensions: ['date'], metrics: ['screenPageViews', 'activeUsers', 'sessions', 'bounceRate', 'averageSessionDuration', 'eventCount'] },
+  { kind: 'pages', dimensions: ['date', 'pagePath'], metrics: ['screenPageViews', 'activeUsers'] },
+  { kind: 'sources', dimensions: ['date', 'sessionSource'], metrics: ['activeUsers', 'sessions', 'screenPageViews'] },
+  { kind: 'devices', dimensions: ['date', 'deviceCategory'], metrics: ['activeUsers'] },
+  { kind: 'browsers', dimensions: ['date', 'browser'], metrics: ['activeUsers'] },
+  { kind: 'os', dimensions: ['date', 'operatingSystem'], metrics: ['activeUsers'] },
+  { kind: 'countries', dimensions: ['date', 'country'], metrics: ['activeUsers'] },
+  { kind: 'regions', dimensions: ['date', 'country', 'region'], metrics: ['activeUsers'] },
+  { kind: 'cities', dimensions: ['date', 'country', 'region', 'city'], metrics: ['activeUsers'] },
+  { kind: 'events', dimensions: ['date', 'eventName'], metrics: ['eventCount', 'activeUsers'] },
+]
+
+/**
+ * Fold one report's rows into the per-day accumulators. Pure — unit-testable
+ * with canned API responses.
+ */
+export function foldReport(days: Map<string, DayAccumulator>, kind: string, rows: GaReportRow[]): number {
+  let folded = 0
+  for (const row of rows) {
+    const dims = (row.dimensionValues || []).map(d => d.value || '')
+    const mets = (row.metricValues || []).map(m => m.value)
+    const day = isoDay(dims[0])
+    if (!day)
+      continue
+    const acc = dayAcc(days, day)
+    folded++
+    if (kind === 'traffic') {
+      acc.hasTraffic = true
+      acc.scalars.views += nnum(mets[0])
+      acc.scalars.visitors += nnum(mets[1])
+      acc.scalars.sessions += nnum(mets[2])
+      const sessions = nnum(mets[2]) || nnum(mets[1])
+      acc.scalars.bounces += Math.round(sessions * nnum(mets[3])) // bounceRate is 0..1
+      acc.scalars.totalDuration += Math.round(sessions * nnum(mets[4]) * 1000)
+      acc.scalars.events += nnum(mets[5])
+    }
+    else if (kind === 'pages') {
+      const path = dims[1] || '/'
+      const cell = (acc.pages[path] ||= { w: 0, v: 0, e: 0 })
+      cell.w += nnum(mets[0])
+      cell.v += nnum(mets[1])
+    }
+    else if (kind === 'sources') {
+      const raw = dims[1] || ''
+      const source = !raw || /^\(direct\)$|^\(none\)$/i.test(raw) ? 'Direct' : raw
+      const cell = (acc.sources[source] ||= { v: 0, w: 0 })
+      cell.v += nnum(mets[0])
+      cell.w += nnum(mets[2]) || nnum(mets[1])
+    }
+    else if (kind === 'events') {
+      const name = dims[1] || 'unknown'
+      const cell = (acc.events[name] ||= { c: 0, v: 0, val: 0 })
+      cell.c += nnum(mets[0])
+      cell.v += nnum(mets[1])
+    }
+    else if (kind === 'regions') {
+      const key = `${dims[1] || 'Unknown'}:${dims[2] || 'Unknown'}`
+      const cell = (acc.regions[key] ||= { v: 0 })
+      cell.v += nnum(mets[0])
+    }
+    else if (kind === 'cities') {
+      const key = `${dims[1] || 'Unknown'}:${dims[2] || 'Unknown'}:${dims[3] || 'Unknown'}`
+      const cell = (acc.cities[key] ||= { v: 0 })
+      cell.v += nnum(mets[0])
+    }
+    else if (kind === 'devices' || kind === 'browsers' || kind === 'os' || kind === 'countries') {
+      let value = (dims[1] || '').trim()
+      if (!value || value === '(not set)')
+        continue
+      if (kind === 'devices')
+        value = value.toLowerCase()
+      const cell = ((acc as any)[kind][value] ||= { v: 0 })
+      cell.v += nnum(mets[0])
+    }
+  }
+  return folded
+}
+
+export interface Ga4ApiImportOptions {
+  propertyId: string
+  serviceAccountKey: ServiceAccountKey
+  /** ISO date; defaults to GA4's earliest possible day. */
+  startDate?: string
+  /** ISO date; defaults to yesterday. */
+  endDate?: string
+}
+
+/** Pull the property's history from the Data API and import it. */
+export async function importFromGa4Api(siteId: string, options: Ga4ApiImportOptions): Promise<GaImportResult> {
+  const propertyId = options.propertyId.replace(/^properties\//, '').trim()
+  if (!/^\d+$/.test(propertyId))
+    throw new Error('propertyId must be the numeric GA4 property id')
+  const token = await getAccessToken(options.serviceAccountKey)
+
+  const dateRange = {
+    startDate: options.startDate || '2015-08-14',
+    endDate: options.endDate || 'yesterday',
+  }
+
+  const days = new Map<string, DayAccumulator>()
+  const files: GaImportResult['files'] = []
+  for (const report of GA4_REPORTS) {
+    const rows = await runReport(token, propertyId, {
+      dateRanges: [dateRange],
+      dimensions: report.dimensions.map(name => ({ name })),
+      metrics: report.metrics.map(name => ({ name })),
+    })
+    const folded = foldReport(days, report.kind, rows)
+    files.push({ name: `ga4-api:${report.kind}`, kind: report.kind, rows: folded, days: 0 })
+  }
+
+  const write = await writeImportedDays(siteId, days)
+  return { files, ...write }
+}

package/src/lib/goals.ts

@@ -0,0 +1,205 @@
+/**
+ * Goal matching and conversion logic
+ */
+
+import { generateId } from '../../src/index'
+import { Goal, Conversion } from '../../src/models/orm'
+import { getCachedGoals, setCachedGoals, hasConverted, markConverted } from '../utils/cache'
+import { dynamodb, TABLE_NAME, isConditionalCheckFailed } from './dynamodb'
+
+/**
+ * Get goals for a site (with caching)
+ */
+export async function getGoalsForSite(siteId: string): Promise<Goal[]> {
+  const cached = getCachedGoals(siteId)
+  if (cached) {
+    return cached
+  }
+
+  try {
+    const goals = await Goal.forSite(siteId).active().get()
+    setCachedGoals(siteId, goals)
+    return goals
+  }
+catch (err) {
+    console.error('[Goals] Failed to fetch goals:', err)
+    return []
+  }
+}
+
+/**
+ * Context for goal matching
+ */
+export interface GoalMatchContext {
+  path: string
+  eventName?: string
+  /**
+   * The triggering event's value — used as the conversion value for
+   * variable-price goals (e.g. a purchase), falling back to the goal's static
+   * value when the event carries none (#132).
+   */
+  eventValue?: number
+  sessionDurationMinutes?: number
+}
+
+/**
+ * Check if a goal matches the given context
+ */
+export function matchGoal(goal: Goal, context: GoalMatchContext): boolean {
+  if (!goal.isActive) return false
+
+  switch (goal.type) {
+    case 'pageview':
+      return matchPattern(goal.pattern, context.path, goal.matchType)
+
+    case 'event':
+      if (!context.eventName) return false
+      return matchPattern(goal.pattern, context.eventName, goal.matchType)
+
+    case 'duration':
+      if (context.sessionDurationMinutes === undefined) return false
+      const threshold = goal.durationMinutes || 0
+      return context.sessionDurationMinutes >= threshold
+
+    default:
+      return false
+  }
+}
+
+/**
+ * Match a pattern against a value
+ */
+function matchPattern(pattern: string, value: string, matchType: string): boolean {
+  if (!pattern || !value) return false
+
+  switch (matchType) {
+    case 'exact':
+      return value === pattern
+
+    case 'contains':
+      return value.includes(pattern)
+
+    case 'regex':
+      try {
+        const regex = new RegExp(pattern)
+        return regex.test(value)
+      }
+catch {
+        console.warn(`[Goals] Invalid regex pattern: ${pattern}`)
+        return false
+      }
+
+    default:
+      return value === pattern
+  }
+}
+
+/**
+ * Metadata for conversion attribution
+ */
+export interface ConversionMetadata {
+  referrerSource?: string
+  utmSource?: string
+  utmMedium?: string
+  utmCampaign?: string
+}
+
+
+/**
+ * Durable once-per-session conversion claim (#174). The in-process
+ * hasConverted Map dedupes only within one instance (capped at 1,000 sessions
+ * with arbitrary eviction) — on Lambda the same session converted once per
+ * concurrent instance, inflating a revenue-grade metric. A conditional put on
+ * a CONVLOCK item makes exactly one instance win; the Map stays as a cheap
+ * fast-path.
+ */
+async function claimConversionLock(siteId: string, sessionId: string, goalId: string): Promise<boolean> {
+  try {
+    await dynamodb.putItem({
+      TableName: TABLE_NAME,
+      Item: {
+        pk: { S: `SITE#${siteId}` },
+        sk: { S: `CONVLOCK#${sessionId}#${goalId}` },
+        // Sessions idle out after 30 minutes — 24h is generous headroom.
+        ttl: { N: String(Math.floor(Date.now() / 1000) + 24 * 60 * 60) },
+      },
+      ConditionExpression: 'attribute_not_exists(pk)',
+    })
+    return true
+  }
+  catch (e) {
+    if (isConditionalCheckFailed(e))
+      return false
+    throw e
+  }
+}
+
+/**
+ * Check and record conversions for all matching goals
+ */
+export async function checkAndRecordConversions(
+  siteId: string,
+  visitorId: string,
+  sessionId: string,
+  context: GoalMatchContext,
+  metadata: ConversionMetadata
+): Promise<void> {
+  try {
+    const goals = await getGoalsForSite(siteId)
+    if (goals.length === 0) return
+
+    const timestamp = new Date()
+
+    for (const goal of goals) {
+      // Skip if already converted in this session
+      if (hasConverted(siteId, sessionId, goal.id)) continue
+
+      if (matchGoal(goal, context)) {
+        // Durable claim first — exactly one instance records this conversion.
+        const claimed = await claimConversionLock(siteId, sessionId, goal.id)
+        if (!claimed) {
+          markConverted(siteId, sessionId, goal.id)
+          continue
+        }
+        // Record conversion
+        await Conversion.record({
+          id: generateId(),
+          siteId,
+          goalId: goal.id,
+          visitorId,
+          sessionId,
+          // Use the event's actual value for variable-price goals; fall back to
+          // the goal's configured static value otherwise (#132).
+          value: context.eventValue ?? goal.value,
+          path: context.path,
+          referrerSource: metadata.referrerSource,
+          utmSource: metadata.utmSource,
+          utmMedium: metadata.utmMedium,
+          utmCampaign: metadata.utmCampaign,
+          timestamp,
+        })
+
+        markConverted(siteId, sessionId, goal.id)
+        console.log(`[Goals] Conversion recorded: ${goal.name} for session ${sessionId}`)
+      }
+    }
+  }
+catch (err) {
+    console.error('[Goals] Error checking conversions:', err)
+  }
+}
+
+/**
+ * Calculate conversion rate
+ */
+export function calculateConversionRate(conversions: number, totalVisitors: number): number {
+  if (totalVisitors === 0) return 0
+  return (conversions / totalVisitors) * 100
+}
+
+/**
+ * Format conversion rate for display
+ */
+export function formatConversionRate(rate: number): string {
+  return `${rate.toFixed(2)}%`
+}

package/src/lib/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Lib barrel export
+ */
+
+export * from './dynamodb'
+export * from './goals'

package/src/lib/ingest-counters.ts

@@ -0,0 +1,123 @@
+/**
+ * Per-site hourly ingest counters (#175).
+ *
+ * Every beacon outcome — collected or dropped, and WHY — increments an
+ * in-process buffer that flushes to one item per site per hour
+ * (sk = INGEST#YYYY-MM-DDTHH) via atomic ADDs. This is what makes silent
+ * ingest failures visible: the CORS bug shipped twice with nothing to detect
+ * it because drops looked identical to no-traffic.
+ *
+ * Buffered (not per-event writes) so counting adds no per-beacon latency and
+ * at most one WCU per site-hour-flush. A dying instance loses at most the
+ * last FLUSH_MS of counts — acceptable for an ops signal.
+ */
+import { dynamodb, TABLE_NAME } from './dynamodb'
+import { log } from './log'
+
+export type IngestOutcome
+  = 'collected' | 'bot' | 'firewall' | 'dedup' | 'invalid'
+  | 'rate_limited' | 'excluded' | 'quota'
+
+const FLUSH_MS = 15_000
+const MAX_BUFFERED_KEYS = 500
+const COUNTER_TTL_S = 90 * 24 * 60 * 60
+
+// site|hour -> field -> pending increment
+const buffer = new Map<string, Map<string, number>>()
+let flushTimer: ReturnType<typeof setInterval> | null = null
+let flushing = false
+
+function hourKey(now: Date = new Date()): string {
+  return now.toISOString().slice(0, 13)
+}
+
+/** Sanitize a tracker version into a counter field name (v_1_2_3). */
+function versionField(version: string): string {
+  return `v_${version.replace(/[^\w.-]/g, '').replace(/[.-]/g, '_').slice(0, 24)}`
+}
+
+/** Count one beacon outcome for a site (buffered; flushes on a timer). */
+export function recordIngest(siteId: string, outcome: IngestOutcome, trackerVersion?: string): void {
+  if (!siteId)
+    return
+  const key = `${siteId}|${hourKey()}`
+  let fields = buffer.get(key)
+  if (!fields) {
+    fields = new Map()
+    buffer.set(key, fields)
+  }
+  fields.set(outcome, (fields.get(outcome) || 0) + 1)
+  if (outcome === 'collected' && trackerVersion)
+    fields.set(versionField(trackerVersion), (fields.get(versionField(trackerVersion)) || 0) + 1)
+
+  if (!flushTimer) {
+    flushTimer = setInterval(() => { void flushIngestCounters() }, FLUSH_MS)
+    // Never keep the process alive just to flush counters.
+    if (typeof (flushTimer as any).unref === 'function')
+      (flushTimer as any).unref()
+  }
+  if (buffer.size > MAX_BUFFERED_KEYS)
+    void flushIngestCounters()
+}
+
+/** Flush buffered counts to DynamoDB (atomic ADDs). Exported for tests/tick. */
+export async function flushIngestCounters(): Promise<number> {
+  if (flushing || buffer.size === 0)
+    return 0
+  flushing = true
+  const entries = [...buffer.entries()]
+  buffer.clear()
+  let flushed = 0
+  try {
+    for (const [key, fields] of entries) {
+      const [siteId, hour] = key.split('|')
+      const names: Record<string, string> = {}
+      const values: Record<string, unknown> = {}
+      const adds: string[] = []
+      let i = 0
+      for (const [field, n] of fields) {
+        names[`#f${i}`] = field
+        values[`:f${i}`] = { N: String(n) }
+        adds.push(`#f${i} :f${i}`)
+        i++
+      }
+      names['#ttl'] = 'ttl'
+      values[':ttl'] = { N: String(Math.floor(Date.now() / 1000) + COUNTER_TTL_S) }
+      await dynamodb.updateItem({
+        TableName: TABLE_NAME,
+        Key: { pk: { S: `SITE#${siteId}` }, sk: { S: `INGEST#${hour}` } },
+        UpdateExpression: `ADD ${adds.join(', ')} SET #ttl = if_not_exists(#ttl, :ttl)`,
+        ExpressionAttributeNames: names,
+        ExpressionAttributeValues: values,
+      })
+      flushed++
+    }
+  }
+  catch (e) {
+    log.warn('ingest_counters.flush_failed', { error: (e as Error).message })
+  }
+  finally {
+    flushing = false
+  }
+  return flushed
+}
+
+/** Hourly counter rows for a site over the trailing `hours` (default 48). */
+export async function readIngestCounters(siteId: string, hours = 48): Promise<Array<Record<string, unknown>>> {
+  const { queryAllItems, unmarshall } = await import('./dynamodb')
+  const start = new Date(Date.now() - hours * 60 * 60 * 1000)
+  const res = await queryAllItems({
+    TableName: TABLE_NAME,
+    KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
+    ExpressionAttributeValues: {
+      ':pk': { S: `SITE#${siteId}` },
+      ':start': { S: `INGEST#${hourKey(start)}` },
+      ':end': { S: `INGEST#${hourKey()}~` },
+    },
+  })
+  return (res.Items || []).map(unmarshall).map((it: any) => {
+    // eslint-disable-next-line pickier/no-unused-vars -- pk/ttl destructured to exclude them from the rest spread
+    const { pk, sk, ttl, ...counts } = it
+    return { hour: String(sk).slice('INGEST#'.length), ...counts }
+  })
+}

package/src/lib/log.ts

@@ -0,0 +1,36 @@
+/**
+ * Leveled logger (#175). JSON lines in production (parseable by CloudWatch /
+ * any log shipper), human-readable in dev. `debug` is emitted only when
+ * ANALYTICS_DEBUG=true. Use instead of bare console.* in operational paths so
+ * failures are queryable (level, event, fields) rather than prose.
+ */
+
+type Fields = Record<string, unknown>
+
+const JSON_LOGS: boolean = process.env.LOG_FORMAT === 'json'
+  || (process.env.NODE_ENV === 'production' && process.env.LOG_FORMAT !== 'pretty')
+
+function emit(level: 'debug' | 'info' | 'warn' | 'error', event: string, fields?: Fields): void {
+  if (level === 'debug' && process.env.ANALYTICS_DEBUG !== 'true')
+    return
+  if (JSON_LOGS) {
+    // eslint-disable-next-line no-console
+    console[level === 'debug' ? 'log' : level](JSON.stringify({ ts: new Date().toISOString(), level, event, ...fields }))
+    return
+  }
+  const suffix = fields && Object.keys(fields).length > 0 ? ` ${JSON.stringify(fields)}` : ''
+  // eslint-disable-next-line no-console
+  console[level === 'debug' ? 'log' : level](`[${event}]${suffix}`)
+}
+
+export const log: {
+  debug: (event: string, fields?: Fields) => void
+  info: (event: string, fields?: Fields) => void
+  warn: (event: string, fields?: Fields) => void
+  error: (event: string, fields?: Fields) => void
+} = {
+  debug: (event, fields) => emit('debug', event, fields),
+  info: (event, fields) => emit('info', event, fields),
+  warn: (event, fields) => emit('warn', event, fields),
+  error: (event, fields) => emit('error', event, fields),
+}