@littlebearapps/platform-admin-sdk 1.4.2 → 2.0.0

package/templates/full/workers/lib/auditor/feature-coverage.ts

@@ -0,0 +1,348 @@
+/**
+ * Feature Coverage Audit Module
+ *
+ * Audits which features from budgets.yaml are actively reporting vs dormant.
+ * Compares defined features in CONFIG:BUDGETS (KV) against actual heartbeats
+ * in system_health_checks (D1).
+ *
+ * Statuses:
+ *   active    — Heartbeats in last 7 days and events_count > 0
+ *   dormant   — Defined in config but no recent heartbeats
+ *   undefined — Reporting telemetry but not in budgets.yaml
+ *
+ * @module workers/lib/auditor/feature-coverage
+ */
+
+import type { Logger } from '@littlebearapps/platform-consumer-sdk';
+
+// =============================================================================
+// Types
+// =============================================================================
+
+/** Environment bindings (generic to work with both raw Env and TrackedEnv) */
+export interface FeatureCoverageEnv {
+  PLATFORM_DB: {
+    prepare: (query: string) => {
+      bind: (...args: unknown[]) => {
+        run: () => Promise<unknown>;
+        all: <T>() => Promise<{ results?: T[] }>;
+        first: <T>() => Promise<T | null>;
+      };
+      all: <T>() => Promise<{ results?: T[] }>;
+      first: <T>() => Promise<T | null>;
+    };
+  };
+  PLATFORM_CACHE: {
+    get: (key: string) => Promise<string | null>;
+    put: (key: string, value: string, options?: { expirationTtl?: number }) => Promise<void>;
+  };
+}
+
+export type FeatureStatus = 'active' | 'dormant' | 'undefined';
+
+export interface FeatureCoverageEntry {
+  project: string;
+  feature: string;
+  featureKey: string;
+  status: FeatureStatus;
+  lastHeartbeat: string | null;
+  eventsLast7d: number;
+  definedBudget: number | null;
+  budgetUnit: string | null;
+}
+
+export interface FeatureCoverageReport {
+  auditTime: string;
+  entries: FeatureCoverageEntry[];
+  summary: {
+    totalDefined: number;
+    active: number;
+    dormant: number;
+    undefined: number;
+  };
+  byProject: Record<string, { active: number; dormant: number; undefined: number }>;
+}
+
+interface BudgetConfig {
+  features: Record<
+    string,
+    Record<string, Record<string, { budget: number; unit: string; period: string }>>
+  >;
+}
+
+// =============================================================================
+// Main Audit Function
+// =============================================================================
+
+/**
+ * Run feature coverage audit.
+ * Compares defined features in CONFIG:BUDGETS against actual heartbeats.
+ */
+export async function runFeatureCoverageAudit(
+  env: FeatureCoverageEnv,
+  log: Logger
+): Promise<FeatureCoverageReport> {
+  const auditTime = new Date().toISOString();
+  const entries: FeatureCoverageEntry[] = [];
+
+  try {
+    const definedFeatures = await loadDefinedFeatures(env, log);
+    const activeFeatures = await getActiveFeatures(env, log);
+    const undefinedFeatures = await getUndefinedFeatures(env, definedFeatures, log);
+
+    // Build coverage entries for defined features
+    for (const [featureKey, feature] of Object.entries(definedFeatures)) {
+      const active = activeFeatures.get(featureKey);
+      const isActive = !!active && active.eventsLast7d > 0;
+
+      entries.push({
+        project: feature.project,
+        feature: feature.feature,
+        featureKey,
+        status: isActive ? 'active' : 'dormant',
+        lastHeartbeat: active?.lastHeartbeat ?? null,
+        eventsLast7d: active?.eventsLast7d ?? 0,
+        definedBudget: feature.budget,
+        budgetUnit: feature.unit,
+      });
+    }
+
+    // Add undefined features (reporting but not in config)
+    for (const feature of undefinedFeatures) {
+      entries.push({
+        project: feature.project,
+        feature: feature.feature,
+        featureKey: feature.featureKey,
+        status: 'undefined',
+        lastHeartbeat: feature.lastHeartbeat,
+        eventsLast7d: feature.eventsLast7d,
+        definedBudget: null,
+        budgetUnit: null,
+      });
+    }
+
+    const summary = {
+      totalDefined: Object.keys(definedFeatures).length,
+      active: entries.filter((e) => e.status === 'active').length,
+      dormant: entries.filter((e) => e.status === 'dormant').length,
+      undefined: entries.filter((e) => e.status === 'undefined').length,
+    };
+
+    const byProject: Record<string, { active: number; dormant: number; undefined: number }> = {};
+    for (const entry of entries) {
+      if (!byProject[entry.project]) {
+        byProject[entry.project] = { active: 0, dormant: 0, undefined: 0 };
+      }
+      byProject[entry.project][entry.status]++;
+    }
+
+    log.info('Feature coverage audit complete', {
+      defined: summary.totalDefined,
+      active: summary.active,
+      dormant: summary.dormant,
+      undefined: summary.undefined,
+    });
+
+    return { auditTime, entries, summary, byProject };
+  } catch (error) {
+    log.error('Feature coverage audit failed', error);
+    return {
+      auditTime,
+      entries: [],
+      summary: { totalDefined: 0, active: 0, dormant: 0, undefined: 0 },
+      byProject: {},
+    };
+  }
+}
+
+// =============================================================================
+// Internal Helpers
+// =============================================================================
+
+/** Load defined features from CONFIG:BUDGETS in KV */
+async function loadDefinedFeatures(
+  env: FeatureCoverageEnv,
+  log: Logger
+): Promise<
+  Record<
+    string,
+    { project: string; category: string; feature: string; budget: number; unit: string }
+  >
+> {
+  const features: Record<
+    string,
+    { project: string; category: string; feature: string; budget: number; unit: string }
+  > = {};
+
+  try {
+    const budgetsJson = await env.PLATFORM_CACHE.get('CONFIG:BUDGETS');
+    if (!budgetsJson) {
+      log.warn('CONFIG:BUDGETS not found in KV');
+      return features;
+    }
+
+    const config = JSON.parse(budgetsJson) as BudgetConfig;
+
+    for (const [project, categories] of Object.entries(config.features || {})) {
+      for (const [category, categoryFeatures] of Object.entries(categories || {})) {
+        for (const [featureName, featureConfig] of Object.entries(categoryFeatures || {})) {
+          const featureKey = `${project}:${category}:${featureName}`;
+          features[featureKey] = {
+            project,
+            category,
+            feature: featureName,
+            budget: featureConfig.budget,
+            unit: featureConfig.unit,
+          };
+        }
+      }
+    }
+
+    log.debug('Loaded defined features', { count: Object.keys(features).length });
+  } catch (error) {
+    log.error('Failed to load defined features', error);
+  }
+
+  return features;
+}
+
+/** Get active features from system_health_checks (last 7 days) */
+async function getActiveFeatures(
+  env: FeatureCoverageEnv,
+  log: Logger
+): Promise<Map<string, { lastHeartbeat: string; eventsLast7d: number }>> {
+  const active = new Map<string, { lastHeartbeat: string; eventsLast7d: number }>();
+
+  try {
+    const result = await env.PLATFORM_DB.prepare(
+      `
+      SELECT
+        feature_id AS feature_key,
+        MAX(datetime(last_heartbeat, 'unixepoch')) as last_heartbeat,
+        COUNT(*) as events_count
+      FROM system_health_checks
+      WHERE last_heartbeat > unixepoch('now', '-7 days')
+      GROUP BY feature_id
+    `
+    ).all<{ feature_key: string; last_heartbeat: string; events_count: number }>();
+
+    for (const row of result.results ?? []) {
+      active.set(row.feature_key, {
+        lastHeartbeat: row.last_heartbeat,
+        eventsLast7d: row.events_count,
+      });
+    }
+
+    log.debug('Found active features', { count: active.size });
+  } catch (error) {
+    log.error('Failed to get active features', error);
+  }
+
+  return active;
+}
+
+/** Get undefined features (reporting but not in config) */
+async function getUndefinedFeatures(
+  env: FeatureCoverageEnv,
+  definedFeatures: Record<string, unknown>,
+  log: Logger
+): Promise<
+  Array<{
+    project: string;
+    feature: string;
+    featureKey: string;
+    lastHeartbeat: string;
+    eventsLast7d: number;
+  }>
+> {
+  const undefinedFeatures: Array<{
+    project: string;
+    feature: string;
+    featureKey: string;
+    lastHeartbeat: string;
+    eventsLast7d: number;
+  }> = [];
+
+  try {
+    const result = await env.PLATFORM_DB.prepare(
+      `
+      SELECT
+        feature_id AS feature_key,
+        project_id,
+        MAX(datetime(last_heartbeat, 'unixepoch')) as last_heartbeat,
+        COUNT(*) as events_count
+      FROM system_health_checks
+      WHERE last_heartbeat > unixepoch('now', '-7 days')
+      GROUP BY feature_id
+    `
+    ).all<{
+      feature_key: string;
+      project_id: string;
+      last_heartbeat: string;
+      events_count: number;
+    }>();
+
+    for (const row of result.results ?? []) {
+      if (definedFeatures[row.feature_key]) continue;
+
+      const parts = row.feature_key.split(':');
+      const featureName = parts.length > 2 ? parts.slice(2).join(':') : row.feature_key;
+
+      undefinedFeatures.push({
+        project: row.project_id,
+        feature: featureName,
+        featureKey: row.feature_key,
+        lastHeartbeat: row.last_heartbeat,
+        eventsLast7d: row.events_count,
+      });
+    }
+
+    log.debug('Found undefined features', { count: undefinedFeatures.length });
+  } catch (error) {
+    log.error('Failed to get undefined features', error);
+  }
+
+  return undefinedFeatures;
+}
+
+// =============================================================================
+// Storage
+// =============================================================================
+
+/** Store feature coverage audit results in D1 */
+export async function storeFeatureCoverageAudit(
+  env: FeatureCoverageEnv,
+  report: FeatureCoverageReport,
+  log: Logger
+): Promise<void> {
+  try {
+    for (const entry of report.entries) {
+      const id = crypto.randomUUID();
+
+      await env.PLATFORM_DB.prepare(
+        `
+        INSERT INTO feature_coverage_audit (
+          id, audit_time, project, feature, status,
+          last_heartbeat, events_last_7d, defined_budget, budget_unit
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `
+      )
+        .bind(
+          id,
+          report.auditTime,
+          entry.project,
+          entry.featureKey,
+          entry.status,
+          entry.lastHeartbeat,
+          entry.eventsLast7d,
+          entry.definedBudget,
+          entry.budgetUnit
+        )
+        .run();
+    }
+
+    log.info('Stored feature coverage audit', { entries: report.entries.length });
+  } catch (error) {
+    log.error('Failed to store feature coverage audit', error);
+  }
+}

package/templates/full/workers/lib/auditor/index.ts

@@ -0,0 +1,9 @@
+/**
+ * Auditor Module Exports
+ *
+ * Barrel export for all audit modules.
+ */
+
+export * from './feature-coverage';
+export * from './comprehensive-report';
+export * from './types';

package/templates/full/workers/lib/auditor/types.ts

@@ -0,0 +1,167 @@
+/**
+ * Auditor Shared Types
+ *
+ * Type definitions for the platform-auditor worker and its library modules.
+ *
+ * @module workers/lib/auditor/types
+ */
+
+import type {
+  KVNamespace,
+  D1Database,
+  Fetcher,
+  ExecutionContext,
+} from '@cloudflare/workers-types';
+
+// =============================================================================
+// Environment
+// =============================================================================
+
+export interface AuditorEnv {
+  GITHUB_TOKEN: string;
+  PLATFORM_AI_GATEWAY_KEY: string;
+  GEMINI_API_KEY: string;
+  CLOUDFLARE_ACCOUNT_ID: string;
+  PLATFORM_DB: D1Database;
+  PLATFORM_CACHE: KVNamespace;
+  PLATFORM_TELEMETRY: Queue;
+  ALERT_ROUTER: Fetcher;
+  GATUS_HEARTBEAT_URL?: string;
+  GATUS_TOKEN?: string;
+}
+
+// =============================================================================
+// Audit Configuration
+// =============================================================================
+
+/** Scan type — full (all dimensions) or focused (weak dimensions only) */
+export type ScanType = 'full' | 'focused';
+
+/** Audit dimension for rubric scoring */
+export type AuditDimension = 'sdk' | 'observability' | 'cost' | 'security';
+
+/** Audit status determination */
+export type AuditStatus = 'HEALTHY' | 'ZOMBIE' | 'UNTRACKED' | 'BROKEN' | 'NOT_INTEGRATED';
+
+/** Deep scan files categorised by audit dimension */
+export interface DimensionFiles {
+  sdk: string[];
+  observability: string[];
+  cost: string[];
+  security: string[];
+}
+
+/** Audit target from config (audit-targets.yaml) */
+export interface AuditTarget {
+  repo: string;
+  wranglerPath: string;
+  entryPoint: string;
+  entryPointFallback: string | null;
+  expectedStatus: AuditStatus;
+  notes: string;
+  deepScanFiles: DimensionFiles;
+}
+
+/** Audit targets configuration (loaded from KV or YAML) */
+export interface AuditTargetsConfig {
+  targets: Record<string, AuditTarget>;
+  sdkPatterns: string[];
+  rubricWeights: {
+    sdk: number;
+    observability: number;
+    costProtection: number;
+    security: number;
+  };
+  heartbeatFreshnessHours: number;
+  focusedScanThreshold: number;
+  aiCacheTtlSeconds: number;
+  behavioralAnalysisDays: number;
+  hotspotMinCommits: number;
+}
+
+// =============================================================================
+// Check Results
+// =============================================================================
+
+/** Config check result (wrangler bindings) */
+export interface ConfigCheckResult {
+  hasPlatformCache: boolean;
+  hasPlatformTelemetry: boolean;
+  observabilityEnabled: boolean;
+  logsEnabled: boolean;
+  tracesEnabled: boolean;
+  traceSamplingRate: number | null;
+  logSamplingRate: number | null;
+  invocationLogsEnabled: boolean;
+  sourceMapsEnabled: boolean;
+  issues: string[];
+}
+
+/** Code smell check result (SDK pattern detection) */
+export interface CodeSmellResult {
+  hasSdkFolder: boolean;
+  hasWithFeatureBudget: boolean;
+  hasTrackedEnv: boolean;
+  hasCircuitBreakerError: boolean;
+  hasErrorLogging: boolean;
+  patternCounts: Record<string, number>;
+}
+
+/** Runtime check result (D1 heartbeat freshness) */
+export interface RuntimeCheckResult {
+  hasRecentHeartbeat: boolean;
+  hoursSinceHeartbeat: number | null;
+  lastHeartbeatTime: string | null;
+}
+
+/** AI Judge result (from Gemini analysis) */
+export interface AIJudgeResult {
+  score: number;
+  summary: string;
+  issues: string;
+  categorisedIssues: string;
+  reasoning: string;
+  rubricScores: {
+    sdk: number;
+    observability: number;
+    costProtection: number;
+    security: number;
+  };
+  rubricEvidence: string;
+  validationRetries: number;
+  cachedAt: string | null;
+}
+
+/** Behavioral analysis result (git history) */
+export interface BehavioralResult {
+  hotspots: Array<{
+    filePath: string;
+    changeCount: number;
+    hasSdkPatterns: boolean;
+    hotspotScore: number;
+  }>;
+  regressions: Array<{
+    commitSha: string;
+    commitMessage: string;
+    removedPattern: string;
+    filePath: string;
+  }>;
+  analysisWindow: number;
+}
+
+/** Combined audit result for a single project */
+export interface ProjectAuditResult {
+  project: string;
+  auditId: string;
+  status: AuditStatus;
+  statusMessage: string;
+  scanType: ScanType;
+  focusedDimensions: AuditDimension[] | null;
+  config: ConfigCheckResult;
+  codeSmells: CodeSmellResult;
+  runtime: RuntimeCheckResult;
+  aiJudge: AIJudgeResult | null;
+  behavioral: BehavioralResult | null;
+  compositeScore: number;
+  timestamp: string;
+}