@littlebearapps/platform-admin-sdk 1.4.2 → 2.0.0

package/templates/full/scripts/ops/universal-backfill.ts

@@ -0,0 +1,147 @@
+#!/usr/bin/env npx tsx
+/**
+ * Universal Backfill Script (Full Tier)
+ *
+ * Orchestrates the hourly → daily → monthly cascade with progress reporting.
+ * Runs each step in sequence, passing date ranges through the pipeline.
+ *
+ * Prerequisites:
+ *   CLOUDFLARE_API_TOKEN  — API token with Analytics:Read + D1:Write permissions
+ *   CLOUDFLARE_ACCOUNT_ID — Your Cloudflare account ID
+ *   D1_DATABASE_ID        — Your platform-metrics D1 database ID
+ *
+ * Usage:
+ *   npx tsx scripts/ops/universal-backfill.ts
+ *   npx tsx scripts/ops/universal-backfill.ts --dry-run
+ *   npx tsx scripts/ops/universal-backfill.ts --start 2026-02-01 --end 2026-02-28
+ *   npx tsx scripts/ops/universal-backfill.ts --skip-hourly
+ */
+
+import { execSync } from 'node:child_process';
+import { resolve } from 'node:path';
+
+const SCRIPTS_DIR = resolve(import.meta.dirname);
+
+interface Args {
+  start?: string;
+  end?: string;
+  dryRun: boolean;
+  skipHourly: boolean;
+}
+
+function parseArgs(): Args {
+  const args = process.argv.slice(2);
+  const result: Args = { dryRun: false, skipHourly: false };
+
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--start' && args[i + 1]) result.start = args[++i];
+    else if (args[i] === '--end' && args[i + 1]) result.end = args[++i];
+    else if (args[i] === '--dry-run') result.dryRun = true;
+    else if (args[i] === '--skip-hourly') result.skipHourly = true;
+  }
+
+  if (!result.start) {
+    const d = new Date();
+    d.setDate(d.getDate() - 7);
+    result.start = d.toISOString().slice(0, 10);
+  }
+  if (!result.end) {
+    result.end = new Date().toISOString().slice(0, 10);
+  }
+
+  return result;
+}
+
+function runStep(name: string, script: string, extraArgs: string[]) {
+  console.log(`\n${'='.repeat(60)}`);
+  console.log(`Step: ${name}`);
+  console.log('='.repeat(60));
+
+  const cmd = `npx tsx ${script} ${extraArgs.join(' ')}`;
+  console.log(`Running: ${cmd}\n`);
+
+  try {
+    execSync(cmd, { stdio: 'inherit', env: process.env });
+    console.log(`\n${name}: COMPLETE`);
+    return true;
+  } catch (err) {
+    console.error(`\n${name}: FAILED`);
+    return false;
+  }
+}
+
+async function main() {
+  const args = parseArgs();
+  const dateArgs = [];
+  if (args.start) dateArgs.push('--start', args.start);
+  if (args.end) dateArgs.push('--end', args.end);
+  if (args.dryRun) dateArgs.push('--dry-run');
+
+  console.log('Universal Backfill Pipeline');
+  console.log(`Range: ${args.start} → ${args.end}`);
+  console.log(`Dry run: ${args.dryRun}`);
+  console.log(`Skip hourly: ${args.skipHourly}`);
+
+  const startTime = Date.now();
+  const results: Array<{ step: string; success: boolean }> = [];
+
+  // Step 1: Hourly backfill (optional — requires GraphQL API, rate-limited)
+  if (!args.skipHourly) {
+    const success = runStep(
+      '1/3: Hourly Backfill',
+      resolve(SCRIPTS_DIR, 'backfill-cloudflare-hourly.ts'),
+      dateArgs
+    );
+    results.push({ step: 'Hourly', success });
+    if (!success && !args.dryRun) {
+      console.warn('\nHourly backfill failed — continuing with daily rollups using existing hourly data.');
+    }
+  } else {
+    console.log('\nSkipping hourly backfill (--skip-hourly)');
+    results.push({ step: 'Hourly', success: true });
+  }
+
+  // Step 2: Daily rollups
+  const dailySuccess = runStep(
+    '2/3: Daily Rollups',
+    resolve(SCRIPTS_DIR, 'backfill-cloudflare-daily.ts'),
+    dateArgs
+  );
+  results.push({ step: 'Daily', success: dailySuccess });
+
+  // Step 3: Monthly rollups
+  const monthStart = args.start?.slice(0, 7) ?? '';
+  const monthEnd = args.end?.slice(0, 7) ?? '';
+  const monthlyArgs = [];
+  if (monthStart) monthlyArgs.push('--start', monthStart);
+  if (monthEnd) monthlyArgs.push('--end', monthEnd);
+  if (args.dryRun) monthlyArgs.push('--dry-run');
+
+  const monthlySuccess = runStep(
+    '3/3: Monthly Rollups',
+    resolve(SCRIPTS_DIR, 'backfill-monthly-rollups.ts'),
+    monthlyArgs
+  );
+  results.push({ step: 'Monthly', success: monthlySuccess });
+
+  // Summary
+  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+  console.log(`\n${'='.repeat(60)}`);
+  console.log('Pipeline Summary');
+  console.log('='.repeat(60));
+  for (const r of results) {
+    console.log(`  ${r.success ? 'PASS' : 'FAIL'} ${r.step}`);
+  }
+  console.log(`\nTotal time: ${elapsed}s`);
+
+  const failed = results.filter((r) => !r.success);
+  if (failed.length > 0) {
+    console.error(`\n${failed.length} step(s) failed.`);
+    process.exit(1);
+  }
+}
+
+main().catch((err) => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

package/templates/full/workers/lib/ai-judge-schema.ts

@@ -0,0 +1,181 @@
+/**
+ * AI Judge Response Schema
+ *
+ * Zod schemas for validating AI Judge responses from Gemini via AI Gateway.
+ * Implements rubric-based scoring with chain-of-thought reasoning.
+ * Designed to be lenient to handle AI output variations.
+ *
+ * Rubric dimensions (1-5 scale):
+ *   sdk (30%)           — SDK integration patterns, feature budgets, tracking
+ *   observability (25%) — Logging, tracing, metrics, error reporting
+ *   costProtection (25%) — Circuit breakers, budgets, DLQ, rate limiting
+ *   security (20%)      — Auth, validation, secrets, CORS
+ *
+ * @module workers/lib/ai-judge-schema
+ */
+
+import { z } from 'zod';
+
+// -----------------------------------------------------------------------------
+// Rubric Score Schema (1-5 scale with evidence)
+// -----------------------------------------------------------------------------
+
+export const RubricScoreSchema = z.object({
+  score: z
+    .number()
+    .min(1)
+    .max(5)
+    .transform((v) => Math.round(v)),
+  evidence: z.array(z.string()).default(['No specific evidence cited']),
+});
+
+export type RubricScore = z.infer<typeof RubricScoreSchema>;
+
+// -----------------------------------------------------------------------------
+// Rubric Scores (all four dimensions) - lenient with aliases
+// -----------------------------------------------------------------------------
+
+export const RubricScoresSchema = z
+  .object({
+    sdk: RubricScoreSchema.optional(),
+    observability: RubricScoreSchema.optional(),
+    costProtection: RubricScoreSchema.optional(),
+    cost: RubricScoreSchema.optional(), // Alias for costProtection
+    security: RubricScoreSchema.optional(),
+  })
+  .transform((scores) => ({
+    sdk: scores.sdk ?? { score: 3, evidence: ['Not evaluated'] },
+    observability: scores.observability ?? { score: 3, evidence: ['Not evaluated'] },
+    costProtection: scores.costProtection ??
+      scores.cost ?? { score: 3, evidence: ['Not evaluated'] },
+    security: scores.security ?? { score: 3, evidence: ['Not evaluated'] },
+  }));
+
+export type RubricScores = z.infer<typeof RubricScoresSchema>;
+
+// -----------------------------------------------------------------------------
+// Static Check Correction (lenient)
+// -----------------------------------------------------------------------------
+
+export const StaticCheckCorrectionSchema = z.object({
+  field: z.string().optional().default('unknown'),
+  expected: z.boolean().optional().default(true),
+  actual: z.boolean().optional().default(false),
+  reason: z.string().optional().default(''),
+  confidence: z
+    .union([z.number(), z.string().transform((v) => parseFloat(v))])
+    .optional()
+    .default(0.8)
+    .transform((v) => (typeof v === 'number' && v > 1 ? v / 100 : v)),
+});
+
+export type StaticCheckCorrection = z.infer<typeof StaticCheckCorrectionSchema>;
+
+// -----------------------------------------------------------------------------
+// Categorised Issue (lenient)
+// -----------------------------------------------------------------------------
+
+export const CategorisedIssueSchema = z.object({
+  category: z.enum(['sdk', 'observability', 'cost', 'security']).catch('sdk'),
+  severity: z.enum(['critical', 'high', 'medium', 'low']).catch('medium'),
+  description: z.string().optional().default('No description'),
+  file: z.string().optional().nullable(),
+  line: z
+    .union([z.number(), z.string().transform((v) => parseInt(v, 10))])
+    .optional()
+    .nullable(),
+});
+
+export type CategorisedIssue = z.infer<typeof CategorisedIssueSchema>;
+
+// -----------------------------------------------------------------------------
+// Full AI Judge Response (lenient)
+// -----------------------------------------------------------------------------
+
+export const AIJudgeResponseSchema = z.object({
+  reasoning: z.string().optional().default('Analysis completed.'),
+  rubricScores: z.preprocess((val) => val ?? {}, RubricScoresSchema),
+  staticCheckCorrections: z.array(StaticCheckCorrectionSchema).optional().default([]),
+  issues: z.array(CategorisedIssueSchema).optional().default([]),
+  summary: z.string().optional().default('Analysis complete.'),
+});
+
+export type AIJudgeResponse = z.infer<typeof AIJudgeResponseSchema>;
+
+// -----------------------------------------------------------------------------
+// Rubric Weights (for composite score calculation)
+// -----------------------------------------------------------------------------
+
+/** Default rubric weights — configurable via audit-targets.yaml */
+export const DEFAULT_RUBRIC_WEIGHTS = {
+  sdk: 0.3,
+  observability: 0.25,
+  costProtection: 0.25,
+  security: 0.2,
+} as const;
+
+// -----------------------------------------------------------------------------
+// Composite Score Calculation
+// -----------------------------------------------------------------------------
+
+/**
+ * Calculate composite score from rubric scores.
+ * Converts 1-5 scale to 0-100 scale using weighted average.
+ */
+export function calculateCompositeScore(
+  rubricScores: RubricScores,
+  weights: typeof DEFAULT_RUBRIC_WEIGHTS = DEFAULT_RUBRIC_WEIGHTS
+): number {
+  const weightedSum =
+    rubricScores.sdk.score * weights.sdk +
+    rubricScores.observability.score * weights.observability +
+    rubricScores.costProtection.score * weights.costProtection +
+    rubricScores.security.score * weights.security;
+
+  return Math.round(((weightedSum - 1) / 4) * 100);
+}
+
+// -----------------------------------------------------------------------------
+// Validation Helper
+// -----------------------------------------------------------------------------
+
+export interface ValidationResult {
+  success: true;
+  data: AIJudgeResponse;
+}
+
+export interface ValidationError {
+  success: false;
+  errors: string[];
+}
+
+/**
+ * Validate and parse AI Judge response.
+ * Returns structured result with either parsed data or error messages.
+ */
+export function validateAIJudgeResponse(response: unknown): ValidationResult | ValidationError {
+  const result = AIJudgeResponseSchema.safeParse(response);
+
+  if (result.success) {
+    return { success: true, data: result.data };
+  }
+
+  return {
+    success: false,
+    errors: result.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`),
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Prompt Error Feedback
+// -----------------------------------------------------------------------------
+
+/**
+ * Format validation errors for inclusion in retry prompt.
+ */
+export function formatValidationErrors(errors: string[]): string {
+  return `Your previous response had validation errors:
+${errors.map((e) => `- ${e}`).join('\n')}
+
+Please fix these issues and respond again with valid JSON.`;
+}