@growth-labs/monitoring 0.1.0 → 0.1.1

package/src/prober/runners/happy-path-runner.ts

@@ -0,0 +1,270 @@
+import { type CheckResult, errorMessage, isTimeoutMessage, type RuntimeFetch } from '../../types.js'
+import type { CodeSigninHappyPathConfig, HappyPathSurface } from '../surfaces.js'
+
+export interface GmailCodeClient {
+	findLatestCode(
+		config: CodeSigninHappyPathConfig['gmail'],
+		env: Record<string, unknown>,
+		runtime: RuntimeFetch,
+	): Promise<string | null>
+}
+
+export interface HappyPathRuntime extends RuntimeFetch {
+	env?: Record<string, unknown>
+	gmailClient?: GmailCodeClient
+	sleep?: (ms: number) => Promise<void>
+}
+
+export async function runHappyPath(
+	surface: HappyPathSurface,
+	runtime: HappyPathRuntime = {},
+): Promise<CheckResult> {
+	const start = Date.now()
+	const fetcher = runtime.fetcher ?? fetch
+	const env = runtime.env ?? {}
+	try {
+		if (surface.flow !== 'code-signin') {
+			return { status: 'fail', latencyMs: 0, errorMessage: `unsupported flow ${surface.flow}` }
+		}
+
+		const requestResponse = await fetcher(surface.config.codeRequestUrl, {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+			body: new URLSearchParams({ email: surface.config.testEmail }).toString(),
+			redirect: 'manual',
+			signal: AbortSignal.timeout(surface.timeoutMs),
+		})
+		if (requestResponse.status < 200 || requestResponse.status >= 300) {
+			return fail(
+				start,
+				`code request expected 2xx, got ${requestResponse.status}`,
+				requestResponse.status,
+			)
+		}
+
+		const code = await pollForCode(surface.config, env, runtime)
+		if (!code) return timeout(start, 'verification code email did not arrive before poll limit')
+
+		const verifyResponse = await fetcher(surface.config.codeVerifyUrl, {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+			body: new URLSearchParams({ email: surface.config.testEmail, code }).toString(),
+			redirect: 'manual',
+			signal: AbortSignal.timeout(surface.timeoutMs),
+		})
+		if (verifyResponse.status < 200 || verifyResponse.status >= 400) {
+			return fail(
+				start,
+				`code verify expected 2xx/3xx, got ${verifyResponse.status}`,
+				verifyResponse.status,
+			)
+		}
+
+		const cookies = collectSetCookies([requestResponse, verifyResponse])
+		const location = verifyResponse.headers.get('Location')
+		if (location) {
+			const callbackUrl = new URL(location, surface.config.callbackUrl).toString()
+			const callbackResponse = await fetcher(callbackUrl, {
+				method: 'GET',
+				headers: cookieHeader(cookies),
+				redirect: 'manual',
+				signal: AbortSignal.timeout(surface.timeoutMs),
+			})
+			cookies.push(...collectSetCookies([callbackResponse]))
+			if (callbackResponse.status >= 400) {
+				return fail(
+					start,
+					`callback expected <400, got ${callbackResponse.status}`,
+					callbackResponse.status,
+				)
+			}
+		}
+
+		const accountResponse = await fetcher(surface.config.accountUrl, {
+			method: 'GET',
+			headers: cookieHeader(cookies),
+			redirect: 'manual',
+			signal: AbortSignal.timeout(surface.timeoutMs),
+		})
+		if (accountResponse.status !== 200) {
+			return fail(
+				start,
+				`account expected 200, got ${accountResponse.status}`,
+				accountResponse.status,
+			)
+		}
+		const accountBody = await accountResponse.text()
+		if (!accountBody.includes(surface.config.testEmail)) {
+			return fail(
+				start,
+				`account page did not contain ${surface.config.testEmail}`,
+				accountResponse.status,
+			)
+		}
+
+		return { status: 'pass', statusCode: 200, latencyMs: Date.now() - start }
+	} catch (error) {
+		const message = errorMessage(error)
+		return {
+			status: isTimeoutMessage(message) ? 'timeout' : 'fail',
+			latencyMs: Date.now() - start,
+			errorMessage: message,
+		}
+	}
+}
+
+export async function extractCodeFromMessage(message: string): Promise<string | null> {
+	return message.match(/\b\d{6}\b/)?.[0] ?? null
+}
+
+async function pollForCode(
+	config: CodeSigninHappyPathConfig,
+	env: Record<string, unknown>,
+	runtime: HappyPathRuntime,
+): Promise<string | null> {
+	const client = runtime.gmailClient ?? defaultGmailClient
+	const sleep =
+		runtime.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)))
+	for (let attempt = 0; attempt < config.gmail.maxPollAttempts; attempt += 1) {
+		const code = await client.findLatestCode(config.gmail, env, runtime)
+		if (code) return code
+		if (attempt < config.gmail.maxPollAttempts - 1) await sleep(config.gmail.pollIntervalMs)
+	}
+	return null
+}
+
+const defaultGmailClient: GmailCodeClient = {
+	async findLatestCode(config, env, runtime) {
+		const fetcher = runtime.fetcher ?? fetch
+		const credentialsJson = env[config.credentialsSecret]
+		if (typeof credentialsJson !== 'string') {
+			throw new Error(`env.${config.credentialsSecret} must contain Gmail service account JSON`)
+		}
+		const token = await authorizeGmail(
+			JSON.parse(credentialsJson) as ServiceAccountCredentials,
+			config.subject,
+		)
+		const query = encodeURIComponent(config.query ?? 'newer_than:5m')
+		const messages = await fetcher(
+			`https://gmail.googleapis.com/gmail/v1/users/me/messages?q=${query}`,
+			{ headers: { Authorization: `Bearer ${token}` } },
+		)
+		const list = (await messages.json()) as { messages?: Array<{ id: string }> }
+		for (const message of list.messages ?? []) {
+			const response = await fetcher(
+				`https://gmail.googleapis.com/gmail/v1/users/me/messages/${message.id}?format=full`,
+				{ headers: { Authorization: `Bearer ${token}` } },
+			)
+			const body = JSON.stringify(await response.json())
+			const code = await extractCodeFromMessage(body)
+			if (code) return code
+		}
+		return null
+	},
+}
+
+interface ServiceAccountCredentials {
+	client_email: string
+	private_key: string
+	token_uri?: string
+}
+
+async function authorizeGmail(
+	credentials: ServiceAccountCredentials,
+	subject: string,
+): Promise<string> {
+	const now = Math.floor(Date.now() / 1000)
+	const assertion = await signJwt(
+		{
+			alg: 'RS256',
+			typ: 'JWT',
+		},
+		{
+			iss: credentials.client_email,
+			scope: 'https://www.googleapis.com/auth/gmail.readonly',
+			aud: credentials.token_uri ?? 'https://oauth2.googleapis.com/token',
+			exp: now + 3600,
+			iat: now,
+			sub: subject,
+		},
+		credentials.private_key,
+	)
+	const response = await fetch(credentials.token_uri ?? 'https://oauth2.googleapis.com/token', {
+		method: 'POST',
+		headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+		body: new URLSearchParams({
+			grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+			assertion,
+		}).toString(),
+	})
+	if (!response.ok) throw new Error(`Gmail JWT authorization failed with ${response.status}`)
+	const payload = (await response.json()) as { access_token?: string }
+	if (!payload.access_token) throw new Error('Gmail JWT authorization returned no access_token')
+	return payload.access_token
+}
+
+async function signJwt(
+	header: Record<string, unknown>,
+	claims: Record<string, unknown>,
+	privateKeyPem: string,
+): Promise<string> {
+	const encodedHeader = base64UrlEncode(new TextEncoder().encode(JSON.stringify(header)))
+	const encodedClaims = base64UrlEncode(new TextEncoder().encode(JSON.stringify(claims)))
+	const input = `${encodedHeader}.${encodedClaims}`
+	const key = await crypto.subtle.importKey(
+		'pkcs8',
+		pemToArrayBuffer(privateKeyPem),
+		{ name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' },
+		false,
+		['sign'],
+	)
+	const signature = await crypto.subtle.sign(
+		'RSASSA-PKCS1-v1_5',
+		key,
+		new TextEncoder().encode(input),
+	)
+	return `${input}.${base64UrlEncode(new Uint8Array(signature))}`
+}
+
+function pemToArrayBuffer(pem: string): ArrayBuffer {
+	const keyLabel = ['PRIVATE', 'KEY'].join(' ')
+	const beginLabel = ['-----BEGIN', `${keyLabel}-----`].join(' ')
+	const endLabel = ['-----END', `${keyLabel}-----`].join(' ')
+	const base64 = pem.replaceAll(beginLabel, '').replaceAll(endLabel, '').replace(/\s/g, '')
+	const binary = atob(base64)
+	const bytes = new Uint8Array(binary.length)
+	for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index)
+	return bytes.buffer
+}
+
+function base64UrlEncode(bytes: Uint8Array): string {
+	const binary = Array.from(bytes, (byte) => String.fromCharCode(byte)).join('')
+	return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '')
+}
+
+function fail(start: number, errorMessage: string, statusCode?: number): CheckResult {
+	return { status: 'fail', statusCode, latencyMs: Date.now() - start, errorMessage }
+}
+
+function timeout(start: number, errorMessage: string): CheckResult {
+	return { status: 'timeout', latencyMs: Date.now() - start, errorMessage }
+}
+
+function collectSetCookies(responses: Response[]): string[] {
+	const cookies: string[] = []
+	for (const response of responses) {
+		const getSetCookie = (response.headers as Headers & { getSetCookie?: () => string[] })
+			.getSetCookie
+		const values = getSetCookie ? getSetCookie.call(response.headers) : []
+		const single = response.headers.get('set-cookie')
+		for (const value of single ? [single, ...values] : values) {
+			const cookie = value.split(';')[0]?.trim()
+			if (cookie) cookies.push(cookie)
+		}
+	}
+	return cookies
+}
+
+function cookieHeader(cookies: string[]): HeadersInit {
+	return cookies.length > 0 ? { Cookie: cookies.join('; ') } : {}
+}

package/src/prober/runners/post-runner.ts

@@ -0,0 +1,50 @@
+import { type CheckResult, errorMessage, isTimeoutMessage, type RuntimeFetch } from '../../types.js'
+import type { PostSurface } from '../surfaces.js'
+
+export async function runPost(
+	surface: PostSurface,
+	runtime: RuntimeFetch = {},
+): Promise<CheckResult> {
+	const start = Date.now()
+	const fetcher = runtime.fetcher ?? fetch
+	try {
+		const body = new URLSearchParams(surface.bodyFormUrlEncoded).toString()
+		const response = await fetcher(surface.url, {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+			body,
+			redirect: 'manual',
+			signal: AbortSignal.timeout(surface.timeoutMs),
+		})
+		const latencyMs = Date.now() - start
+
+		if (response.status !== surface.assertions.statusCode) {
+			return {
+				status: 'fail',
+				statusCode: response.status,
+				latencyMs,
+				errorMessage: `expected ${surface.assertions.statusCode}, got ${response.status}`,
+			}
+		}
+
+		const location = response.headers.get('Location') ?? ''
+		if (surface.assertions.locationPattern && !surface.assertions.locationPattern.test(location)) {
+			return {
+				status: 'fail',
+				statusCode: response.status,
+				latencyMs,
+				errorMessage: `Location "${location}" does not match expected pattern`,
+			}
+		}
+
+		return { status: 'pass', statusCode: response.status, latencyMs }
+	} catch (error) {
+		const latencyMs = Date.now() - start
+		const message = errorMessage(error)
+		return {
+			status: isTimeoutMessage(message) ? 'timeout' : 'fail',
+			latencyMs,
+			errorMessage: message,
+		}
+	}
+}

package/src/prober/surfaces.ts

@@ -0,0 +1,52 @@
+import type { CheckType } from '../types.js'
+
+export interface BaseSurface {
+	name: string
+	kind: CheckType
+	schedule: string
+	timeoutMs: number
+}
+
+export interface GetSurface extends BaseSurface {
+	kind: 'get'
+	url: string
+	assertions: {
+		statusCode: number
+		contentMarker?: string
+	}
+}
+
+export interface PostSurface extends BaseSurface {
+	kind: 'post'
+	url: string
+	bodyFormUrlEncoded: Record<string, string>
+	assertions: {
+		statusCode: number
+		locationPattern?: RegExp
+	}
+}
+
+export interface GmailPollingConfig {
+	credentialsSecret: string
+	subject: string
+	pollIntervalMs: number
+	maxPollAttempts: number
+	query?: string
+}
+
+export interface CodeSigninHappyPathConfig {
+	codeRequestUrl: string
+	codeVerifyUrl: string
+	callbackUrl: string
+	accountUrl: string
+	testEmail: string
+	gmail: GmailPollingConfig
+}
+
+export interface HappyPathSurface extends BaseSurface {
+	kind: 'happy_path'
+	flow: 'code-signin'
+	config: CodeSigninHappyPathConfig
+}
+
+export type SurfaceConfig = GetSurface | PostSurface | HappyPathSurface

package/src/schemas/README.md

@@ -0,0 +1,14 @@
+# Monitoring Schemas
+
+The package ships raw SQL migrations and matching Drizzle schema definitions for
+consumers that prefer typed queries.
+
+Tables:
+
+- `gl_uptime_checks`
+- `gl_uptime_incidents`
+- `gl_errors`
+
+Consumers can point Wrangler or Drizzle tooling at
+`node_modules/@growth-labs/monitoring/src/schemas/migrations` or the copied
+`dist/schemas/migrations` directory after build.

package/src/schemas/drizzle/schema.ts

@@ -0,0 +1,59 @@
+import { index, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'
+
+export const uptimeChecks = sqliteTable(
+	'gl_uptime_checks',
+	{
+		id: text('id').primaryKey(),
+		surface: text('surface').notNull(),
+		checkType: text('check_type').notNull(),
+		status: text('status').notNull(),
+		statusCode: integer('status_code'),
+		latencyMs: integer('latency_ms'),
+		errorMessage: text('error_message'),
+		checkedAt: integer('checked_at').notNull(),
+	},
+	(table) => [
+		index('idx_uptime_surface_time').on(table.surface, table.checkedAt),
+		index('idx_uptime_status_time').on(table.status, table.checkedAt),
+	],
+)
+
+export const uptimeIncidents = sqliteTable(
+	'gl_uptime_incidents',
+	{
+		id: text('id').primaryKey(),
+		surface: text('surface').notNull(),
+		openedAt: integer('opened_at').notNull(),
+		closedAt: integer('closed_at'),
+		triggerCheckId: text('trigger_check_id'),
+		resolveCheckId: text('resolve_check_id'),
+		severity: text('severity').notNull(),
+		notes: text('notes'),
+	},
+	(table) => [
+		index('idx_incidents_surface_open').on(table.surface, table.openedAt),
+		index('idx_incidents_open').on(table.closedAt),
+	],
+)
+
+export const errors = sqliteTable(
+	'gl_errors',
+	{
+		id: text('id').primaryKey(),
+		realmKey: text('realm_key').notNull(),
+		surface: text('surface').notNull(),
+		severity: text('severity').notNull(),
+		message: text('message').notNull(),
+		stack: text('stack'),
+		requestId: text('request_id'),
+		statusCode: integer('status_code'),
+		durationMs: integer('duration_ms'),
+		occurredAt: integer('occurred_at').notNull(),
+		fingerprint: text('fingerprint').notNull(),
+	},
+	(table) => [
+		index('idx_errors_realm_time').on(table.realmKey, table.occurredAt),
+		index('idx_errors_fingerprint_time').on(table.fingerprint, table.occurredAt),
+		index('idx_errors_surface_time').on(table.surface, table.occurredAt),
+	],
+)

package/src/schemas/index.ts

@@ -0,0 +1 @@
+export { errors, uptimeChecks, uptimeIncidents } from './drizzle/schema.js'

package/src/tail/README.md

@@ -0,0 +1,18 @@
+# Monitoring Tail Worker
+
+`createTailWorker(config)` returns both `tail` and `tailHandler` aliases for
+Cloudflare Tail Worker exports.
+
+Pipeline:
+
+1. Categorize each trace event into zero or more records.
+2. Compute the fingerprint before redaction.
+3. Apply sampling, with an in-memory 100% override for surfaces that become
+   actively incidented within the same batch.
+4. Redact PII/secrets and truncate fields.
+5. Persist to `gl_errors` and write a WAE datapoint.
+6. Evaluate new-error and rate-spike alert actions.
+
+Surface names use `${scriptName}:${method} ${normalizedPath}`. The v0.1 path
+normalizer keeps alphabetic-only segments and replaces all other segments with
+`[slug]`.

package/src/tail/categorize.ts

@@ -0,0 +1,156 @@
+export type ErrorCategory =
+	| 'exception'
+	| 'fivexx'
+	| 'console-error'
+	| 'console-warn'
+	| 'slow-request'
+
+export interface CategorizedEvent {
+	category: ErrorCategory
+	surface: string
+	message: string
+	stack?: string
+	statusCode?: number
+	durationMs?: number
+	requestId?: string
+	occurredAt: number
+	fingerprint?: string
+}
+
+export interface TraceItem {
+	scriptName?: string
+	outcome?: string
+	exceptions?: Array<{ name?: string; message?: string; stack?: string }>
+	logs?: Array<{ level?: string; message?: unknown; args?: unknown[] }>
+	request?: { method?: string; url?: string; headers?: Record<string, string> }
+	response?: { status?: number }
+	eventTimestamp?: number | string
+	wallTime?: number
+	dispatchNamespace?: string
+}
+
+interface CategorizeOptions {
+	slowRequestThresholdMs?: number
+}
+
+export function categorize(
+	traceEvent: TraceItem,
+	options: CategorizeOptions = {},
+): CategorizedEvent[] {
+	const occurredAt = parseTimestamp(traceEvent.eventTimestamp)
+	const surface = surfaceForTrace(traceEvent)
+	const requestId =
+		traceEvent.request?.headers?.['cf-ray'] ?? traceEvent.request?.headers?.['CF-Ray']
+	const durationMs = typeof traceEvent.wallTime === 'number' ? traceEvent.wallTime : undefined
+	const events: CategorizedEvent[] = []
+
+	for (const exception of traceEvent.exceptions ?? []) {
+		const prefix = exception.name ? `${exception.name}: ` : ''
+		events.push({
+			category: 'exception',
+			surface,
+			message: `${prefix}${exception.message ?? 'Unhandled exception'}`,
+			stack: exception.stack,
+			requestId,
+			durationMs,
+			occurredAt,
+		})
+	}
+
+	const statusCode = traceEvent.response?.status
+	if (typeof statusCode === 'number' && statusCode >= 500) {
+		events.push({
+			category: 'fivexx',
+			surface,
+			message: `HTTP ${statusCode} response`,
+			statusCode,
+			requestId,
+			durationMs,
+			occurredAt,
+		})
+	}
+
+	for (const log of traceEvent.logs ?? []) {
+		if (log.level !== 'error' && log.level !== 'warn') continue
+		events.push({
+			category: log.level === 'error' ? 'console-error' : 'console-warn',
+			surface,
+			message: logMessage(log),
+			statusCode,
+			requestId,
+			durationMs,
+			occurredAt,
+		})
+	}
+
+	const slowThreshold = options.slowRequestThresholdMs
+	if (
+		events.length === 0 &&
+		typeof durationMs === 'number' &&
+		typeof slowThreshold === 'number' &&
+		durationMs > slowThreshold
+	) {
+		events.push({
+			category: 'slow-request',
+			surface,
+			message: `Slow request took ${durationMs}ms`,
+			statusCode,
+			requestId,
+			durationMs,
+			occurredAt,
+		})
+	}
+
+	return events
+}
+
+export function normalizePath(pathname: string): string {
+	return (
+		'/' +
+		pathname
+			.split('/')
+			.filter(Boolean)
+			.map((segment) => (/^[A-Za-z]+$/.test(segment) ? segment : '[slug]'))
+			.join('/')
+	)
+}
+
+function surfaceForTrace(traceEvent: TraceItem): string {
+	const scriptName = traceEvent.scriptName ?? traceEvent.dispatchNamespace ?? 'unknown'
+	const method = traceEvent.request?.method ?? 'GET'
+	const url = traceEvent.request?.url ?? 'https://unknown/'
+	let pathname = '/'
+	try {
+		pathname = new URL(url).pathname
+	} catch {
+		pathname = url.split('?')[0] || '/'
+	}
+	return `${scriptName}:${method} ${normalizePath(pathname)}`
+}
+
+function parseTimestamp(value: number | string | undefined): number {
+	if (typeof value === 'number') return value < 10_000_000_000 ? value * 1000 : value
+	if (typeof value === 'string') {
+		const numeric = Number(value)
+		if (Number.isFinite(numeric)) return parseTimestamp(numeric)
+		const parsed = Date.parse(value)
+		if (Number.isFinite(parsed)) return parsed
+	}
+	return Date.now()
+}
+
+function logMessage(log: { message?: unknown; args?: unknown[] }): string {
+	const value = log.message ?? log.args ?? ''
+	if (Array.isArray(value)) return value.map(formatLogPart).join(' ')
+	return formatLogPart(value)
+}
+
+function formatLogPart(value: unknown): string {
+	if (typeof value === 'string') return value
+	if (value instanceof Error) return value.message
+	try {
+		return JSON.stringify(value)
+	} catch {
+		return String(value)
+	}
+}

package/src/tail/fingerprint.ts

@@ -0,0 +1,21 @@
+import type { CategorizedEvent } from './categorize.js'
+
+const encoder = new TextEncoder()
+
+export async function computeFingerprint(event: CategorizedEvent): Promise<string> {
+	const firstStackFrame = event.stack?.split('\n')[1]?.trim() ?? ''
+	const normalizedMessage = normalizeMessage(event.message)
+	const input = `${event.category}|${normalizedMessage}|${firstStackFrame}`
+	const digest = await crypto.subtle.digest('SHA-256', encoder.encode(input))
+	return [...new Uint8Array(digest)]
+		.map((byte) => byte.toString(16).padStart(2, '0'))
+		.join('')
+		.slice(0, 16)
+}
+
+export function normalizeMessage(message: string): string {
+	return message
+		.replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, '[uuid]')
+		.replace(/\/[a-z0-9_-]+(?=\b)/gi, '/[seg]')
+		.replace(/\b\d{4,}\b/g, '[n]')
+}