@littlebearapps/platform-admin-sdk 1.4.2 → 2.0.0

package/templates/full/workers/lib/auditor/comprehensive-report.ts

@@ -0,0 +1,407 @@
+/**
+ * Comprehensive Audit Report Module
+ *
+ * Aggregates findings from all audit dimensions:
+ *   - Gap detection (platform-sentinel gap_detection_log)
+ *   - Attribution checking (attribution_reports)
+ *   - Feature coverage audit
+ *   - AI Judge results
+ *
+ * Produces ranked action items sorted by severity.
+ *
+ * @module workers/lib/auditor/comprehensive-report
+ */
+
+import type { Logger } from '@littlebearapps/platform-consumer-sdk';
+import type { FeatureCoverageReport } from './feature-coverage';
+
+// =============================================================================
+// Types
+// =============================================================================
+
+/** Environment bindings (generic to work with both raw Env and TrackedEnv) */
+export interface ComprehensiveReportEnv {
+  PLATFORM_DB: {
+    prepare: (query: string) => {
+      bind: (...args: unknown[]) => {
+        run: () => Promise<unknown>;
+        all: <T>() => Promise<{ results?: T[] }>;
+        first: <T>() => Promise<T | null>;
+      };
+      all: <T>() => Promise<{ results?: T[] }>;
+      first: <T>() => Promise<T | null>;
+    };
+  };
+  PLATFORM_CACHE: {
+    get: (key: string) => Promise<string | null>;
+    put: (key: string, value: string, options?: { expirationTtl?: number }) => Promise<void>;
+  };
+}
+
+export type ActionSeverity = 'critical' | 'high' | 'medium' | 'low';
+
+export interface ActionItem {
+  category: 'gap' | 'attribution' | 'feature' | 'ai_judge';
+  severity: ActionSeverity;
+  title: string;
+  description: string;
+  recommendation: string;
+}
+
+export interface ComprehensiveReport {
+  id: string;
+  generatedAt: string;
+  gapSummary: {
+    eventsCount: number;
+    totalMissingHours: number;
+    worstGapDay: string | null;
+    averageSeverity: string;
+  };
+  attributionSummary: {
+    totalResources: number;
+    attributedCount: number;
+    unattributedCount: number;
+    unattributedResources: Array<{ type: string; name: string }>;
+  };
+  featureCoverageSummary: {
+    definedCount: number;
+    activeCount: number;
+    dormantCount: number;
+    undefinedCount: number;
+  };
+  aiJudgeSummary: {
+    averageScore: number | null;
+    recommendations: string[];
+  };
+  actionItems: ActionItem[];
+  criticalItemsCount: number;
+}
+
+// =============================================================================
+// Main Report Generator
+// =============================================================================
+
+/**
+ * Generate comprehensive audit report aggregating all dimensions.
+ */
+export async function generateComprehensiveReport(
+  env: ComprehensiveReportEnv,
+  featureCoverage: FeatureCoverageReport | null,
+  log: Logger
+): Promise<ComprehensiveReport> {
+  const id = crypto.randomUUID();
+  const generatedAt = new Date().toISOString();
+  const actionItems: ActionItem[] = [];
+
+  // 1. Gap detection summary (last 7 days)
+  const gapSummary = await getGapSummary(env, log);
+
+  if (gapSummary.totalMissingHours > 0) {
+    actionItems.push({
+      category: 'gap',
+      severity: gapSummary.totalMissingHours > 10 ? 'high' : 'medium',
+      title: `${gapSummary.totalMissingHours} missing hourly snapshots in last 7 days`,
+      description: `Gap detection found ${gapSummary.eventsCount} gap events. Worst day: ${gapSummary.worstGapDay || 'N/A'}`,
+      recommendation: 'Run backfill via npm run backfill or investigate collection issues',
+    });
+  }
+
+  // 2. Attribution summary (latest report)
+  const attributionSummary = await getAttributionSummary(env, log);
+
+  if (attributionSummary.unattributedCount > 0) {
+    actionItems.push({
+      category: 'attribution',
+      severity: attributionSummary.unattributedCount > 5 ? 'high' : 'medium',
+      title: `${attributionSummary.unattributedCount} unattributed Cloudflare resources`,
+      description: `Resources not mapped to any project: ${attributionSummary.unattributedResources
+        .map((r) => `${r.type}:${r.name}`)
+        .slice(0, 5)
+        .join(', ')}${attributionSummary.unattributedCount > 5 ? '...' : ''}`,
+      recommendation: 'Add resource-to-project mappings in services.yaml',
+    });
+  }
+
+  // 3. Feature coverage summary
+  const featureCoverageSummary = featureCoverage
+    ? {
+        definedCount: featureCoverage.summary.totalDefined,
+        activeCount: featureCoverage.summary.active,
+        dormantCount: featureCoverage.summary.dormant,
+        undefinedCount: featureCoverage.summary.undefined,
+      }
+    : { definedCount: 0, activeCount: 0, dormantCount: 0, undefinedCount: 0 };
+
+  if (featureCoverageSummary.dormantCount > 0) {
+    actionItems.push({
+      category: 'feature',
+      severity: 'low',
+      title: `${featureCoverageSummary.dormantCount} dormant features defined`,
+      description: 'Features defined in budgets.yaml but not sending heartbeats in 7 days',
+      recommendation: 'Review if these features are still used or can be removed from config',
+    });
+  }
+
+  if (featureCoverageSummary.undefinedCount > 0) {
+    actionItems.push({
+      category: 'feature',
+      severity: 'medium',
+      title: `${featureCoverageSummary.undefinedCount} undefined features reporting`,
+      description: 'Features sending telemetry but not defined in budgets.yaml',
+      recommendation: 'Add budget definitions for these features in budgets.yaml',
+    });
+  }
+
+  // 4. AI Judge summary (latest results)
+  const aiJudgeSummary = await getAIJudgeSummary(env, log);
+
+  if (aiJudgeSummary.recommendations.length > 0) {
+    for (const rec of aiJudgeSummary.recommendations.slice(0, 3)) {
+      actionItems.push({
+        category: 'ai_judge',
+        severity: 'medium',
+        title: 'AI Judge recommendation',
+        description: rec,
+        recommendation: 'Review and implement AI Judge suggestions',
+      });
+    }
+  }
+
+  // Sort by severity
+  const severityOrder: Record<ActionSeverity, number> = {
+    critical: 0,
+    high: 1,
+    medium: 2,
+    low: 3,
+  };
+  actionItems.sort((a, b) => severityOrder[a.severity] - severityOrder[b.severity]);
+
+  const report: ComprehensiveReport = {
+    id,
+    generatedAt,
+    gapSummary,
+    attributionSummary,
+    featureCoverageSummary,
+    aiJudgeSummary,
+    actionItems,
+    criticalItemsCount: actionItems.filter((i) => i.severity === 'critical').length,
+  };
+
+  log.info('Generated comprehensive audit report', {
+    id,
+    actionItems: actionItems.length,
+    criticalItems: report.criticalItemsCount,
+  });
+
+  return report;
+}
+
+// =============================================================================
+// Data Aggregation Helpers
+// =============================================================================
+
+/** Get gap detection summary from last 7 days */
+async function getGapSummary(
+  env: ComprehensiveReportEnv,
+  log: Logger
+): Promise<ComprehensiveReport['gapSummary']> {
+  try {
+    const result = await env.PLATFORM_DB.prepare(
+      `
+      SELECT
+        COUNT(*) as events_count,
+        SUM(missing_hours_count) as total_missing_hours,
+        MAX(CASE WHEN missing_hours_count = (
+          SELECT MAX(missing_hours_count) FROM gap_detection_log
+          WHERE detection_time > datetime('now', '-7 days')
+        ) THEN date(detection_time) END) as worst_gap_day,
+        AVG(CASE severity
+          WHEN 'critical' THEN 3
+          WHEN 'warning' THEN 2
+          WHEN 'ok' THEN 1
+          ELSE 0
+        END) as avg_severity_score
+      FROM gap_detection_log
+      WHERE detection_time > datetime('now', '-7 days')
+    `
+    ).first<{
+      events_count: number;
+      total_missing_hours: number;
+      worst_gap_day: string | null;
+      avg_severity_score: number;
+    }>();
+
+    if (!result) {
+      return { eventsCount: 0, totalMissingHours: 0, worstGapDay: null, averageSeverity: 'ok' };
+    }
+
+    const avgScore = result.avg_severity_score ?? 0;
+    let averageSeverity = 'ok';
+    if (avgScore >= 2.5) averageSeverity = 'critical';
+    else if (avgScore >= 1.5) averageSeverity = 'warning';
+
+    return {
+      eventsCount: result.events_count ?? 0,
+      totalMissingHours: result.total_missing_hours ?? 0,
+      worstGapDay: result.worst_gap_day,
+      averageSeverity,
+    };
+  } catch (error) {
+    log.error('Failed to get gap summary', error);
+    return { eventsCount: 0, totalMissingHours: 0, worstGapDay: null, averageSeverity: 'ok' };
+  }
+}
+
+/** Get attribution summary from latest report */
+async function getAttributionSummary(
+  env: ComprehensiveReportEnv,
+  log: Logger
+): Promise<ComprehensiveReport['attributionSummary']> {
+  try {
+    const result = await env.PLATFORM_DB.prepare(
+      `
+      SELECT report_json
+      FROM attribution_reports
+      ORDER BY discovery_time DESC
+      LIMIT 1
+    `
+    ).first<{ report_json: string }>();
+
+    if (!result?.report_json) {
+      return { totalResources: 0, attributedCount: 0, unattributedCount: 0, unattributedResources: [] };
+    }
+
+    const report = JSON.parse(result.report_json) as {
+      totalResources: number;
+      attributedCount: number;
+      unattributedCount: number;
+      unattributed: Array<{ resource: { resourceType: string; resourceName: string } }>;
+    };
+
+    return {
+      totalResources: report.totalResources,
+      attributedCount: report.attributedCount,
+      unattributedCount: report.unattributedCount,
+      unattributedResources: (report.unattributed || []).map((a) => ({
+        type: a.resource.resourceType,
+        name: a.resource.resourceName,
+      })),
+    };
+  } catch (error) {
+    log.error('Failed to get attribution summary', error);
+    return { totalResources: 0, attributedCount: 0, unattributedCount: 0, unattributedResources: [] };
+  }
+}
+
+/** Get AI Judge summary from latest audit results */
+async function getAIJudgeSummary(
+  env: ComprehensiveReportEnv,
+  log: Logger
+): Promise<ComprehensiveReport['aiJudgeSummary']> {
+  try {
+    const result = await env.PLATFORM_DB.prepare(
+      `
+      SELECT
+        AVG(ai_judge_score) as avg_score,
+        GROUP_CONCAT(ai_judge_summary) as summaries
+      FROM audit_results
+      WHERE created_at > datetime('now', '-7 days')
+        AND ai_judge_score IS NOT NULL
+    `
+    ).first<{ avg_score: number | null; summaries: string | null }>();
+
+    if (!result?.avg_score) {
+      return { averageScore: null, recommendations: [] };
+    }
+
+    const summaries = result.summaries?.split(',') ?? [];
+    const uniqueRecommendations = [...new Set(summaries)].filter((s) => s.trim()).slice(0, 5);
+
+    return {
+      averageScore: Math.round(result.avg_score),
+      recommendations: uniqueRecommendations,
+    };
+  } catch (error) {
+    log.error('Failed to get AI Judge summary', error);
+    return { averageScore: null, recommendations: [] };
+  }
+}
+
+// =============================================================================
+// Storage
+// =============================================================================
+
+/** Store comprehensive audit report in D1 */
+export async function storeComprehensiveReport(
+  env: ComprehensiveReportEnv,
+  report: ComprehensiveReport,
+  log: Logger
+): Promise<void> {
+  try {
+    await env.PLATFORM_DB.prepare(
+      `
+      INSERT INTO comprehensive_audit_reports (
+        id, generated_at,
+        gap_events_count, total_missing_hours, worst_gap_day, average_gap_severity,
+        total_resources, attributed_count, unattributed_count, unattributed_resources,
+        defined_features_count, active_features_count, dormant_features_count, undefined_features_count,
+        ai_judge_avg_score, ai_judge_recommendations,
+        action_items_count, critical_items_count, action_items,
+        report_json
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `
+    )
+      .bind(
+        report.id,
+        report.generatedAt,
+        report.gapSummary.eventsCount,
+        report.gapSummary.totalMissingHours,
+        report.gapSummary.worstGapDay,
+        report.gapSummary.averageSeverity,
+        report.attributionSummary.totalResources,
+        report.attributionSummary.attributedCount,
+        report.attributionSummary.unattributedCount,
+        JSON.stringify(report.attributionSummary.unattributedResources),
+        report.featureCoverageSummary.definedCount,
+        report.featureCoverageSummary.activeCount,
+        report.featureCoverageSummary.dormantCount,
+        report.featureCoverageSummary.undefinedCount,
+        report.aiJudgeSummary.averageScore,
+        JSON.stringify(report.aiJudgeSummary.recommendations),
+        report.actionItems.length,
+        report.criticalItemsCount,
+        JSON.stringify(report.actionItems),
+        JSON.stringify(report)
+      )
+      .run();
+
+    log.info('Stored comprehensive audit report', { id: report.id });
+  } catch (error) {
+    log.error('Failed to store comprehensive report', error);
+  }
+}
+
+/** Get latest comprehensive report from D1 */
+export async function getLatestComprehensiveReport(
+  env: ComprehensiveReportEnv,
+  log: Logger
+): Promise<ComprehensiveReport | null> {
+  try {
+    const result = await env.PLATFORM_DB.prepare(
+      `
+      SELECT report_json
+      FROM comprehensive_audit_reports
+      ORDER BY generated_at DESC
+      LIMIT 1
+    `
+    ).first<{ report_json: string }>();
+
+    if (result?.report_json) {
+      return JSON.parse(result.report_json) as ComprehensiveReport;
+    }
+    return null;
+  } catch (error) {
+    log.error('Failed to get latest comprehensive report', error);
+    return null;
+  }
+}