@littlebearapps/platform-admin-sdk 1.0.0

package/templates/standard/workers/lib/error-collector/email-health-alerts.ts

@@ -0,0 +1,262 @@
+/**
+ * Email Health Alert Handler
+ *
+ * Processes email health check failures from platform-email-healthcheck
+ * and creates GitHub issues in the correct website repository.
+ *
+ * Uses existing GitHubClient and deduplication patterns from error-collector.
+ *
+ * @module workers/lib/error-collector/email-health-alerts
+ */
+
+import type { Env, EmailHealthAlertEvent } from './types';
+import { GitHubClient } from './github';
+
+// TODO: Set your GitHub organisation and dashboard URL
+const GITHUB_ORG = 'your-github-org';
+const DASHBOARD_URL = 'https://your-dashboard.example.com';
+const GATUS_URL = 'https://your-status.example.com';
+const EMAIL_HEALTHCHECK_URL = 'https://platform-email-healthcheck.your-subdomain.workers.dev';
+
+/**
+ * KV prefix for email health alert deduplication.
+ * Format: EMAIL_HEALTH:{brand}:{check_type}:{date}
+ * One issue per brand per check type per day maximum.
+ */
+const EMAIL_HEALTH_PREFIX = 'EMAIL_HEALTH';
+
+/** Labels applied to email health alert issues */
+const EMAIL_HEALTH_LABELS = ['cf:email-health', 'cf:priority:p2', 'cf:auto-generated'];
+
+/** Get today's date key in YYYY-MM-DD format (UTC) */
+function getDateKey(): string {
+  return new Date().toISOString().slice(0, 10);
+}
+
+/**
+ * Check if an email health alert has already been created for this brand+check today.
+ * @returns Issue number if exists, null otherwise
+ */
+async function checkDedup(
+  kv: KVNamespace,
+  brandId: string,
+  checkType: string
+): Promise<number | null> {
+  const key = `${EMAIL_HEALTH_PREFIX}:${brandId}:${checkType}:${getDateKey()}`;
+  const existing = await kv.get(key);
+  return existing ? parseInt(existing, 10) : null;
+}
+
+/** Record that an email health alert issue was created for today. */
+async function setDedup(
+  kv: KVNamespace,
+  brandId: string,
+  checkType: string,
+  issueNumber: number
+): Promise<void> {
+  const key = `${EMAIL_HEALTH_PREFIX}:${brandId}:${checkType}:${getDateKey()}`;
+  // TTL of 25 hours to cover the full day plus buffer
+  await kv.put(key, String(issueNumber), { expirationTtl: 90000 });
+}
+
+// TODO: Map your brand IDs to display names
+/** Brand display names for issue titles */
+const BRAND_NAMES: Record<string, string> = {
+  // Example:
+  // mybrand: 'My Brand',
+  // anotherbrand: 'Another Brand',
+};
+
+// TODO: Map your brand IDs to email domains
+/** Brand domain mappings for health check URLs */
+const BRAND_DOMAINS: Record<string, string> = {
+  // Example:
+  // mybrand: 'mybrand.com',
+  // anotherbrand: 'anotherbrand.io',
+};
+
+/** Format the GitHub issue body for email health check failures. */
+function formatIssueBody(event: EmailHealthAlertEvent): string {
+  const brandName = BRAND_NAMES[event.brand_id] ?? event.brand_id;
+  const brandDomain = BRAND_DOMAINS[event.brand_id] ?? `${event.brand_id}.example.com`;
+  const now = new Date().toISOString();
+
+  const failureRows = event.failures
+    .map((f) => `| \`${f.check_type}\` | ${f.error_msg} |`)
+    .join('\n');
+
+  return `## Email Health Check Failures
+
+| | |
+|---|---|
+| **Brand** | \`${event.brand_id}\` (${brandName}) |
+| **Failures** | ${event.failures.length} check(s) |
+| **Run ID** | \`${event.run_id}\` |
+| **Detected** | ${now} |
+
+### Failing Checks
+
+| Check | Error |
+|-------|-------|
+${failureRows}
+
+### Check Types Reference
+
+| Check | What It Validates |
+|-------|-------------------|
+| \`brand_config\` | Brand exists in D1, status=active, from_email set |
+| \`templates\` | confirmation + welcome email templates exist and active |
+| \`confirmation_page\` | /confirmed page returns 200 on website |
+| \`email_api_health\` | email.{domain}/email/_health returns ok |
+| \`resend_dns\` | Resend domain verified, DKIM + SPF records verified |
+| \`dmarc\` | DMARC record present with enforcing policy |
+| \`recent_sends\` | At least 1 email sent in last 7 days |
+
+### Investigation Steps
+
+1. **Run manual health check** for this brand:
+   \`\`\`bash
+   curl "${EMAIL_HEALTHCHECK_URL}/healthcheck?brand=${event.brand_id}"
+   \`\`\`
+
+2. **Check health check history**:
+   \`\`\`bash
+   curl "${EMAIL_HEALTHCHECK_URL}/history?brand=${event.brand_id}&limit=20"
+   \`\`\`
+
+3. **Check email API directly**:
+   \`\`\`bash
+   curl "https://email.${brandDomain}/email/_health"
+   \`\`\`
+
+4. **Check Resend domain status** via Resend dashboard or API
+
+5. **Check Gatus** for platform-email-healthcheck heartbeat:
+   - [Gatus Status Page](${GATUS_URL})
+
+### Quick Links
+
+- [Platform Dashboard](${DASHBOARD_URL})
+- [Resend Dashboard](https://resend.com/domains)
+- [Repository](https://github.com/${event.repository})
+- [Email Health Check Worker](https://github.com/${GITHUB_ORG}/platform/blob/main/workers/platform-email-healthcheck.ts)
+- [Email System Docs](https://github.com/${GITHUB_ORG}/platform/blob/main/docs/quickrefs/email-system.md)
+
+---
+Generated by [Platform Email Health Check](https://github.com/${GITHUB_ORG}/platform/blob/main/workers/platform-email-healthcheck.ts)
+`;
+}
+
+/**
+ * Process email health alert failures and create GitHub issues.
+ *
+ * Creates one issue per failing check type (deduped per brand+check+day).
+ * Returns results for each failure processed.
+ */
+export async function processEmailHealthAlerts(
+  event: EmailHealthAlertEvent,
+  env: Env
+): Promise<{
+  processed: number;
+  skipped: number;
+  issues: Array<{ check_type: string; issueNumber: number; issueUrl: string }>;
+  skippedChecks: Array<{ check_type: string; reason: string }>;
+}> {
+  const results = {
+    processed: 0,
+    skipped: 0,
+    issues: [] as Array<{ check_type: string; issueNumber: number; issueUrl: string }>,
+    skippedChecks: [] as Array<{ check_type: string; reason: string }>,
+  };
+
+  // Parse owner/repo
+  const [owner, repo] = event.repository.split('/');
+  if (!owner || !repo) {
+    results.skippedChecks.push(
+      ...event.failures.map((f) => ({
+        check_type: f.check_type,
+        reason: `Invalid repository format: ${event.repository}`,
+      }))
+    );
+    results.skipped = event.failures.length;
+    return results;
+  }
+
+  const github = new GitHubClient(env);
+  const brandName = BRAND_NAMES[event.brand_id] ?? event.brand_id;
+
+  for (const failure of event.failures) {
+    // Check dedup — one issue per brand per check type per day
+    const existingIssue = await checkDedup(env.PLATFORM_CACHE, event.brand_id, failure.check_type);
+    if (existingIssue) {
+      results.skipped++;
+      results.skippedChecks.push({
+        check_type: failure.check_type,
+        reason: `Issue #${existingIssue} already created today`,
+      });
+      continue;
+    }
+
+    try {
+      // Create a focused issue for this specific check failure
+      const issue = await github.createIssue({
+        owner,
+        repo,
+        title: `Email Health: ${brandName} ${failure.check_type} failing`,
+        body: formatIssueBody({
+          ...event,
+          failures: [failure], // Only include this specific failure
+        }),
+        labels: EMAIL_HEALTH_LABELS,
+      });
+
+      console.log(`Created email health issue #${issue.number} for ${event.brand_id}:${failure.check_type}`);
+
+      // Record dedup
+      await setDedup(env.PLATFORM_CACHE, event.brand_id, failure.check_type, issue.number);
+
+      results.processed++;
+      results.issues.push({
+        check_type: failure.check_type,
+        issueNumber: issue.number,
+        issueUrl: issue.html_url,
+      });
+    } catch (error) {
+      console.error(`Failed to create email health issue for ${event.brand_id}:${failure.check_type}:`, error);
+      results.skipped++;
+      results.skippedChecks.push({
+        check_type: failure.check_type,
+        reason: `GitHub API error: ${error instanceof Error ? error.message : String(error)}`,
+      });
+    }
+  }
+
+  // Create dashboard notification (summary for all failures)
+  if (results.processed > 0 && env.NOTIFICATIONS_API) {
+    try {
+      await env.NOTIFICATIONS_API.fetch(
+        'https://platform-notifications.internal/notifications',
+        {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            category: 'email_health',
+            source: 'platform-email-healthcheck',
+            source_id: event.run_id,
+            title: `Email Health: ${brandName} has ${event.failures.length} failing check(s)`,
+            description: event.failures.map((f) => `${f.check_type}: ${f.error_msg}`).join('; '),
+            priority: 'medium',
+            action_url: results.issues[0]?.issueUrl,
+            action_label: 'View Issue',
+            project: 'platform',
+          }),
+        }
+      );
+    } catch (e) {
+      // Non-blocking
+      console.error('Failed to create dashboard notification:', e);
+    }
+  }
+
+  return results;
+}

package/templates/standard/workers/lib/error-collector/fingerprint.ts

@@ -0,0 +1,258 @@
+/**
+ * Error Fingerprinting
+ * Creates stable hashes for error deduplication
+ *
+ * Supports both static patterns (from SDK) and dynamic patterns
+ * (loaded from KV/D1 at runtime via AI-assisted pattern discovery).
+ */
+
+import type { TailEvent, ErrorType } from './types';
+import { normalizeUrl, extractCoreMessage } from './capture';
+
+// Re-export SDK patterns and types for backward compatibility
+export {
+  TRANSIENT_ERROR_PATTERNS,
+  type TransientErrorPattern,
+} from '@littlebearapps/platform-consumer-sdk/patterns';
+
+export {
+  loadDynamicPatterns,
+  clearDynamicPatternsCache,
+  compileDynamicPatterns,
+  classifyWithDynamicPatterns,
+  DYNAMIC_PATTERNS_KV_KEY,
+  type DynamicPatternRule,
+  type CompiledPattern,
+} from '@littlebearapps/platform-consumer-sdk/dynamic-patterns';
+
+// Import for local use in classify/fingerprint functions
+import { TRANSIENT_ERROR_PATTERNS } from '@littlebearapps/platform-consumer-sdk/patterns';
+import type { CompiledPattern } from '@littlebearapps/platform-consumer-sdk/dynamic-patterns';
+
+/** Classification result including pattern source for analytics */
+export interface ClassificationResult {
+  category: string;
+  source: 'static' | 'dynamic';
+  patternId?: string;
+}
+
+/**
+ * Classify an error message into a semantic category for transient errors.
+ * Returns the category if matched, or null if the error should use
+ * standard message-based fingerprinting.
+ *
+ * Checks static patterns first (higher trust), then dynamic patterns.
+ *
+ * @example
+ * classifyError('[YOUTUBE_QUOTA_EXHAUSTED] Daily limit exceeded')
+ * // Returns: 'quota-exhausted'
+ *
+ * classifyError('TypeError: Cannot read property x')
+ * // Returns: null (not a transient error)
+ */
+export function classifyError(message: string): string | null {
+  const result = classifyErrorWithSource(message);
+  return result?.category ?? null;
+}
+
+/**
+ * Classify an error message with source information.
+ * Used internally and by analytics to track dynamic pattern effectiveness.
+ */
+export function classifyErrorWithSource(
+  message: string,
+  dynamicPatterns: CompiledPattern[] = []
+): ClassificationResult | null {
+  // Check static patterns first (trusted, from SDK)
+  for (const { pattern, category } of TRANSIENT_ERROR_PATTERNS) {
+    if (pattern.test(message)) {
+      return { category, source: 'static' };
+    }
+  }
+
+  // Check dynamic patterns (AI-suggested, human-approved)
+  for (const compiled of dynamicPatterns) {
+    if (compiled.test(message)) {
+      return {
+        category: compiled.category,
+        source: 'dynamic',
+        patternId: compiled.id,
+      };
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Check if an error is a transient (expected operational) error.
+ * Transient errors are expected to self-resolve and should not be
+ * treated as bugs or regressions.
+ */
+export function isTransientError(message: string): boolean {
+  return classifyError(message) !== null;
+}
+
+/**
+ * Normalize dynamic values in a message to create stable fingerprints.
+ * Replaces numbers, UUIDs, timestamps, and other variable content with placeholders.
+ *
+ * Example:
+ *   "Slow workflow step (durationMs: 116781, itemCount: 3)"
+ *   -> "Slow workflow step (durationMs: {N}, itemCount: {N})"
+ *
+ *   "Only 29 requests remaining!"
+ *   -> "Only {N} requests remaining!"
+ */
+export function normalizeDynamicValues(message: string): string {
+  return (
+    message
+      // Remove UUIDs (must be before numbers to avoid partial replacement)
+      .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '{UUID}')
+      // Remove hex hashes (16+ chars, e.g., correlation IDs, fingerprints)
+      .replace(/\b[0-9a-f]{16,}\b/gi, '{HASH}')
+      // Remove IPv6 addresses (e.g., 2001:0db8:85a3::8a2e:0370:7334)
+      .replace(/\b(?:[0-9a-f]{1,4}:){2,7}[0-9a-f]{1,4}\b/gi, '{IPV6}')
+      // Remove Base64-encoded strings (20+ chars to avoid false positives)
+      .replace(/\b[A-Za-z0-9+/]{20,}={0,2}\b/g, '{BASE64}')
+      // Remove ISO timestamps (2026-01-31T12:34:56.789Z)
+      .replace(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[.\d]*Z?/g, '{TS}')
+      // Remove date strings (2026-01-31)
+      .replace(/\d{4}-\d{2}-\d{2}/g, '{DATE}')
+      // Remove numbers (must be last to avoid breaking other patterns)
+      .replace(/\d+/g, '{N}')
+      // Normalize whitespace
+      .replace(/\s+/g, ' ')
+      .trim()
+  );
+}
+
+/**
+ * Result of fingerprint computation including classification metadata
+ */
+export interface FingerprintResult {
+  fingerprint: string;
+  category: string | null;
+  normalizedMessage: string | null;
+  patternSource?: 'static' | 'dynamic';
+  dynamicPatternId?: string;
+}
+
+/** Options for computing fingerprints with dynamic pattern support */
+export interface ComputeFingerprintOptions {
+  /** Pre-loaded dynamic patterns (optional, for performance) */
+  dynamicPatterns?: CompiledPattern[];
+}
+
+/**
+ * Compute a fingerprint for an error event
+ * Same fingerprint = same error = update existing issue
+ *
+ * For transient errors (quota exhaustion, rate limits, etc.), uses the
+ * error category instead of the message to ensure stable fingerprints
+ * even when external APIs return varying error messages.
+ *
+ * Supports both static patterns (from SDK) and dynamic patterns (from KV).
+ */
+export async function computeFingerprint(
+  event: TailEvent,
+  errorType: ErrorType,
+  options: ComputeFingerprintOptions = {}
+): Promise<FingerprintResult> {
+  const components: string[] = [event.scriptName, errorType];
+  let classification: ClassificationResult | null = null;
+  let normalizedMessage: string | null = null;
+  const dynamicPatterns = options.dynamicPatterns || [];
+
+  // For exceptions, include exception name and either category or normalized message
+  if (errorType === 'exception' && event.exceptions.length > 0) {
+    const exc = event.exceptions[0];
+    components.push(exc.name);
+
+    // Check for transient error classification first (static + dynamic)
+    classification = classifyErrorWithSource(exc.message, dynamicPatterns);
+    if (classification) {
+      // Use stable category instead of variable message
+      components.push(classification.category);
+      normalizedMessage = normalizeDynamicValues(exc.message).slice(0, 200);
+    } else {
+      // Standard message-based fingerprinting
+      normalizedMessage = normalizeDynamicValues(exc.message).slice(0, 100);
+      components.push(normalizedMessage);
+    }
+  }
+
+  // For CPU/memory limits, just use script name + type (already in components)
+  // These are script-level issues, not request-specific
+
+  // For soft errors, include the normalized error message or category
+  if (errorType === 'soft_error') {
+    const errorLog = event.logs.find((l) => l.level === 'error');
+    if (errorLog) {
+      const coreMsg = extractCoreMessage(errorLog.message[0]);
+
+      // Check for transient error classification (static + dynamic)
+      classification = classifyErrorWithSource(coreMsg, dynamicPatterns);
+      if (classification) {
+        components.push(classification.category);
+        normalizedMessage = normalizeDynamicValues(coreMsg).slice(0, 200);
+      } else {
+        normalizedMessage = normalizeDynamicValues(coreMsg).slice(0, 100);
+        components.push(normalizedMessage);
+      }
+    }
+  }
+
+  // For warnings, include the normalized warning message or category
+  if (errorType === 'warning') {
+    const warnLog = event.logs.find((l) => l.level === 'warn');
+    if (warnLog) {
+      const coreMsg = extractCoreMessage(warnLog.message[0]);
+
+      // Check for transient error classification (static + dynamic)
+      classification = classifyErrorWithSource(coreMsg, dynamicPatterns);
+      if (classification) {
+        components.push(classification.category);
+        normalizedMessage = normalizeDynamicValues(coreMsg).slice(0, 200);
+      } else {
+        normalizedMessage = normalizeDynamicValues(coreMsg).slice(0, 100);
+        components.push(normalizedMessage);
+      }
+    }
+  }
+
+  // Include normalized URL for HTTP errors (helps distinguish different endpoints)
+  // Note: Cron/scheduled events don't have request URLs
+  if (event.event?.request?.url && (errorType === 'exception' || errorType === 'soft_error')) {
+    components.push(normalizeUrl(event.event.request.url));
+  }
+
+  // Create hash
+  const data = components.join(':');
+  const hashBuffer = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(data));
+
+  // Return first 32 hex chars (16 bytes)
+  const fingerprint = Array.from(new Uint8Array(hashBuffer))
+    .slice(0, 16)
+    .map((b) => b.toString(16).padStart(2, '0'))
+    .join('');
+
+  return {
+    fingerprint,
+    category: classification?.category ?? null,
+    normalizedMessage,
+    patternSource: classification?.source,
+    dynamicPatternId: classification?.patternId,
+  };
+}
+
+/**
+ * Generate a unique ID for a new error occurrence
+ */
+export function generateId(): string {
+  const bytes = new Uint8Array(16);
+  crypto.getRandomValues(bytes);
+  return Array.from(bytes)
+    .map((b) => b.toString(16).padStart(2, '0'))
+    .join('');
+}