@littlebearapps/platform-admin-sdk 1.4.2 → 1.5.0

package/templates/shared/scripts/ops/backfill-cloudflare-hourly.ts

@@ -0,0 +1,473 @@
+#!/usr/bin/env npx tsx
+/**
+ * Cloudflare Hourly Usage Backfill Script
+ *
+ * Backfills hourly Cloudflare usage data into hourly_usage_snapshots via the
+ * Cloudflare GraphQL Analytics API and D1 REST API.
+ *
+ * Queries Workers, D1, KV, R2, and Durable Objects metrics per hour, calculates
+ * estimated costs, and upserts into D1. Skips hours that already have data.
+ *
+ * Prerequisites:
+ *   CLOUDFLARE_API_TOKEN  — API token with Analytics:Read + D1:Write permissions
+ *   CLOUDFLARE_ACCOUNT_ID — Your Cloudflare account ID
+ *   D1_DATABASE_ID        — Your platform-metrics D1 database ID
+ *
+ * Usage:
+ *   npx tsx scripts/ops/backfill-cloudflare-hourly.ts
+ *   npx tsx scripts/ops/backfill-cloudflare-hourly.ts --dry-run
+ *   npx tsx scripts/ops/backfill-cloudflare-hourly.ts --start 2026-02-01 --end 2026-02-28
+ *   npx tsx scripts/ops/backfill-cloudflare-hourly.ts --days 7
+ *
+ * Note: Cloudflare GraphQL hourly data is retained for ~7 days. Older dates
+ * will return zeros. The script handles this gracefully.
+ */
+
+const GRAPHQL_ENDPOINT = 'https://api.cloudflare.com/client/v4/graphql';
+const REST_API_BASE = 'https://api.cloudflare.com/client/v4';
+
+// Rate limiting
+const RATE_LIMIT_MS = 500;
+const BUDGET_WAIT_MS = 310_000;
+
+// Pricing constants (Cloudflare Workers Paid Plan — current as of March 2026)
+const CF_PRICING = {
+  workers: {
+    baseCostMonthly: 5.0,
+    includedRequests: 10_000_000,
+    requestsPerMillion: 0.3,
+    cpuMsPerMillion: 0.02,
+  },
+  d1: {
+    rowsReadPerBillion: 0.001,
+    rowsWrittenPerMillion: 1.0,
+  },
+  kv: {
+    readsPerMillion: 0.5,
+    writesPerMillion: 5.0,
+    deletesPerMillion: 5.0,
+    listsPerMillion: 5.0,
+  },
+  r2: {
+    classAPerMillion: 4.5,
+    classBPerMillion: 0.36,
+  },
+  durableObjects: {
+    requestsPerMillion: 0.15,
+    gbSecondsPerMillion: 12.5,
+  },
+};
+
+interface HourlyMetrics {
+  hour: string;
+  workers: {
+    requests: number;
+    errors: number;
+    cpuTimeMs: number;
+    duration50thMs: number;
+    duration99thMs: number;
+  };
+  d1: { rowsRead: number; rowsWritten: number };
+  kv: { reads: number; writes: number; deletes: number; lists: number };
+  r2: { classAOps: number; classBOps: number; egressBytes: number };
+  durableObjects: { requests: number; gbSeconds: number };
+}
+
+// ---------------------------------------------------------------------------
+// CLI argument parsing
+// ---------------------------------------------------------------------------
+
+function parseArgs(): { startDate: string; endDate: string; dryRun: boolean } {
+  const args = process.argv.slice(2);
+  const now = new Date();
+  let startDate = '';
+  let endDate = '';
+  let dryRun = false;
+  let days = 7; // default lookback
+
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--start' && args[i + 1]) {
+      startDate = args[++i];
+    } else if (args[i] === '--end' && args[i + 1]) {
+      endDate = args[++i];
+    } else if (args[i] === '--days' && args[i + 1]) {
+      days = parseInt(args[++i], 10);
+    } else if (args[i] === '--dry-run') {
+      dryRun = true;
+    }
+  }
+
+  if (!startDate) {
+    const start = new Date(now);
+    start.setDate(start.getDate() - days);
+    startDate = start.toISOString().split('T')[0];
+  }
+  if (!endDate) {
+    const end = new Date(now);
+    end.setDate(end.getDate() - 1);
+    endDate = end.toISOString().split('T')[0];
+  }
+
+  return { startDate, endDate, dryRun };
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function generateId(): string {
+  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
+    const r = (Math.random() * 16) | 0;
+    const v = c === 'x' ? r : (r & 0x3) | 0x8;
+    return v.toString(16);
+  });
+}
+
+// ---------------------------------------------------------------------------
+// GraphQL helper with retry + budget-depleted handling
+// ---------------------------------------------------------------------------
+
+async function graphqlQuery(
+  apiToken: string,
+  query: string,
+  variables: Record<string, unknown>,
+  maxRetries = 3
+): Promise<unknown> {
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    const response = await fetch(GRAPHQL_ENDPOINT, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiToken}` },
+      body: JSON.stringify({ query, variables }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`GraphQL request failed: ${response.status} ${response.statusText}`);
+    }
+
+    const result = (await response.json()) as {
+      data?: unknown;
+      errors?: Array<{ message: string; extensions?: { code?: string } }>;
+    };
+
+    if (result.errors) {
+      const budgetError = result.errors.find(
+        (e) => e.message?.includes('budget depleted') || e.extensions?.code === 'budget'
+      );
+      if (budgetError && attempt < maxRetries) {
+        console.log(`  ⏳ Rate limit hit — waiting 5 min before retry (${attempt}/${maxRetries})…`);
+        await sleep(BUDGET_WAIT_MS);
+        continue;
+      }
+      throw new Error(`GraphQL errors: ${JSON.stringify(result.errors)}`);
+    }
+
+    return result.data;
+  }
+  throw new Error('Max retries exceeded');
+}
+
+// ---------------------------------------------------------------------------
+// Fetch hourly metrics from Cloudflare GraphQL
+// ---------------------------------------------------------------------------
+
+async function fetchHourlyMetrics(
+  apiToken: string,
+  accountId: string,
+  startHour: string,
+  endHour: string
+): Promise<Map<string, HourlyMetrics>> {
+  const metricsMap = new Map<string, HourlyMetrics>();
+
+  // Initialise all hours in range
+  const start = new Date(startHour);
+  const end = new Date(endHour);
+  for (let d = new Date(start); d <= end; d.setTime(d.getTime() + 3_600_000)) {
+    const hour = d.toISOString().replace(/:\d{2}\.\d{3}Z$/, ':00Z');
+    metricsMap.set(hour, {
+      hour,
+      workers: { requests: 0, errors: 0, cpuTimeMs: 0, duration50thMs: 0, duration99thMs: 0 },
+      d1: { rowsRead: 0, rowsWritten: 0 },
+      kv: { reads: 0, writes: 0, deletes: 0, lists: 0 },
+      r2: { classAOps: 0, classBOps: 0, egressBytes: 0 },
+      durableObjects: { requests: 0, gbSeconds: 0 },
+    });
+  }
+
+  const vars = { accountTag: accountId, startHour, endHour };
+
+  // Workers
+  try {
+    const workersQuery = `query($accountTag:String!,$startHour:Time!,$endHour:Time!){viewer{accounts(filter:{accountTag:$accountTag}){workersInvocationsAdaptive(filter:{datetimeHour_geq:$startHour,datetimeHour_leq:$endHour},limit:10000){sum{requests,errors}quantiles{cpuTimeP50,durationP50,durationP99}dimensions{datetimeHour}}}}}`;
+    const data = (await graphqlQuery(apiToken, workersQuery, vars)) as Record<string, unknown>;
+    const accounts = (data as { viewer?: { accounts?: unknown[] } })?.viewer?.accounts as
+      | Array<{ workersInvocationsAdaptive?: Array<{ sum: { requests: number; errors: number }; quantiles: { cpuTimeP50: number; durationP50: number; durationP99: number }; dimensions: { datetimeHour: string } }> }>
+      | undefined;
+    for (const w of accounts?.[0]?.workersInvocationsAdaptive ?? []) {
+      const m = metricsMap.get(w.dimensions.datetimeHour);
+      if (m) {
+        m.workers.requests += w.sum?.requests ?? 0;
+        m.workers.errors += w.sum?.errors ?? 0;
+        m.workers.cpuTimeMs += (w.quantiles?.cpuTimeP50 ?? 0) / 1000;
+        m.workers.duration50thMs = w.quantiles?.durationP50 ?? 0;
+        m.workers.duration99thMs = w.quantiles?.durationP99 ?? 0;
+      }
+    }
+  } catch (e) { console.warn(`  Workers query failed: ${e}`); }
+  await sleep(RATE_LIMIT_MS);
+
+  // D1
+  try {
+    const d1Query = `query($accountTag:String!,$startHour:Time!,$endHour:Time!){viewer{accounts(filter:{accountTag:$accountTag}){d1AnalyticsAdaptiveGroups(filter:{datetimeHour_geq:$startHour,datetimeHour_leq:$endHour},limit:10000){sum{rowsRead,rowsWritten}dimensions{datetimeHour}}}}}`;
+    const data = (await graphqlQuery(apiToken, d1Query, vars)) as Record<string, unknown>;
+    const accounts = (data as { viewer?: { accounts?: unknown[] } })?.viewer?.accounts as
+      | Array<{ d1AnalyticsAdaptiveGroups?: Array<{ sum: { rowsRead: number; rowsWritten: number }; dimensions: { datetimeHour: string } }> }>
+      | undefined;
+    for (const d of accounts?.[0]?.d1AnalyticsAdaptiveGroups ?? []) {
+      const m = metricsMap.get(d.dimensions.datetimeHour);
+      if (m) {
+        m.d1.rowsRead += d.sum?.rowsRead ?? 0;
+        m.d1.rowsWritten += d.sum?.rowsWritten ?? 0;
+      }
+    }
+  } catch (e) { console.warn(`  D1 query failed: ${e}`); }
+  await sleep(RATE_LIMIT_MS);
+
+  // KV
+  try {
+    const kvQuery = `query($accountTag:String!,$startHour:Time!,$endHour:Time!){viewer{accounts(filter:{accountTag:$accountTag}){kvOperationsAdaptiveGroups(filter:{datetimeHour_geq:$startHour,datetimeHour_leq:$endHour},limit:10000){sum{requests}dimensions{datetimeHour,actionType}}}}}`;
+    const data = (await graphqlQuery(apiToken, kvQuery, vars)) as Record<string, unknown>;
+    const accounts = (data as { viewer?: { accounts?: unknown[] } })?.viewer?.accounts as
+      | Array<{ kvOperationsAdaptiveGroups?: Array<{ sum: { requests: number }; dimensions: { datetimeHour: string; actionType: string } }> }>
+      | undefined;
+    for (const k of accounts?.[0]?.kvOperationsAdaptiveGroups ?? []) {
+      const m = metricsMap.get(k.dimensions.datetimeHour);
+      if (m) {
+        const action = k.dimensions.actionType ?? '';
+        const reqs = k.sum?.requests ?? 0;
+        if (action === 'read') m.kv.reads += reqs;
+        else if (action === 'write') m.kv.writes += reqs;
+        else if (action === 'delete') m.kv.deletes += reqs;
+        else if (action === 'list') m.kv.lists += reqs;
+      }
+    }
+  } catch (e) { console.warn(`  KV query failed: ${e}`); }
+  await sleep(RATE_LIMIT_MS);
+
+  // R2
+  try {
+    const r2Query = `query($accountTag:String!,$startHour:Time!,$endHour:Time!){viewer{accounts(filter:{accountTag:$accountTag}){r2OperationsAdaptiveGroups(filter:{datetimeHour_geq:$startHour,datetimeHour_leq:$endHour},limit:10000){sum{requests,responseObjectSize}dimensions{datetimeHour,actionType}}}}}`;
+    const data = (await graphqlQuery(apiToken, r2Query, vars)) as Record<string, unknown>;
+    const accounts = (data as { viewer?: { accounts?: unknown[] } })?.viewer?.accounts as
+      | Array<{ r2OperationsAdaptiveGroups?: Array<{ sum: { requests: number; responseObjectSize: number }; dimensions: { datetimeHour: string; actionType: string } }> }>
+      | undefined;
+    for (const r of accounts?.[0]?.r2OperationsAdaptiveGroups ?? []) {
+      const m = metricsMap.get(r.dimensions.datetimeHour);
+      if (m) {
+        const action = r.dimensions.actionType?.toUpperCase() ?? '';
+        if (['GET', 'HEAD'].includes(action)) {
+          m.r2.classBOps += r.sum?.requests ?? 0;
+          m.r2.egressBytes += r.sum?.responseObjectSize ?? 0;
+        } else {
+          m.r2.classAOps += r.sum?.requests ?? 0;
+        }
+      }
+    }
+  } catch (e) { console.warn(`  R2 query failed: ${e}`); }
+  await sleep(RATE_LIMIT_MS);
+
+  // Durable Objects
+  try {
+    const doQuery = `query($accountTag:String!,$startHour:Time!,$endHour:Time!){viewer{accounts(filter:{accountTag:$accountTag}){durableObjectsInvocationsAdaptiveGroups(filter:{datetimeHour_geq:$startHour,datetimeHour_leq:$endHour},limit:10000){sum{requests}dimensions{datetimeHour}}}}}`;
+    const data = (await graphqlQuery(apiToken, doQuery, vars)) as Record<string, unknown>;
+    const accounts = (data as { viewer?: { accounts?: unknown[] } })?.viewer?.accounts as
+      | Array<{ durableObjectsInvocationsAdaptiveGroups?: Array<{ sum: { requests: number }; dimensions: { datetimeHour: string } }> }>
+      | undefined;
+    for (const d of accounts?.[0]?.durableObjectsInvocationsAdaptiveGroups ?? []) {
+      const m = metricsMap.get(d.dimensions.datetimeHour);
+      if (m) m.durableObjects.requests += d.sum?.requests ?? 0;
+    }
+  } catch (e) { console.warn(`  DO query failed: ${e}`); }
+
+  return metricsMap;
+}
+
+// ---------------------------------------------------------------------------
+// Cost calculation
+// ---------------------------------------------------------------------------
+
+function calculateHourlyCosts(metrics: HourlyMetrics) {
+  const hourlyIncluded = CF_PRICING.workers.includedRequests / 30 / 24;
+  const overageReqs = Math.max(0, metrics.workers.requests - hourlyIncluded);
+  const workersCost =
+    (overageReqs / 1_000_000) * CF_PRICING.workers.requestsPerMillion +
+    (metrics.workers.cpuTimeMs / 1_000_000) * CF_PRICING.workers.cpuMsPerMillion;
+  const d1Cost =
+    (metrics.d1.rowsRead / 1_000_000_000) * CF_PRICING.d1.rowsReadPerBillion +
+    (metrics.d1.rowsWritten / 1_000_000) * CF_PRICING.d1.rowsWrittenPerMillion;
+  const kvCost =
+    (metrics.kv.reads / 1_000_000) * CF_PRICING.kv.readsPerMillion +
+    (metrics.kv.writes / 1_000_000) * CF_PRICING.kv.writesPerMillion +
+    (metrics.kv.deletes / 1_000_000) * CF_PRICING.kv.deletesPerMillion +
+    (metrics.kv.lists / 1_000_000) * CF_PRICING.kv.listsPerMillion;
+  const r2Cost =
+    (metrics.r2.classAOps / 1_000_000) * CF_PRICING.r2.classAPerMillion +
+    (metrics.r2.classBOps / 1_000_000) * CF_PRICING.r2.classBPerMillion;
+  const doCost =
+    (metrics.durableObjects.requests / 1_000_000) * CF_PRICING.durableObjects.requestsPerMillion +
+    (metrics.durableObjects.gbSeconds / 1_000_000) * CF_PRICING.durableObjects.gbSecondsPerMillion;
+  const totalCost = workersCost + d1Cost + kvCost + r2Cost + doCost;
+  return { workersCost, d1Cost, kvCost, r2Cost, doCost, totalCost };
+}
+
+// ---------------------------------------------------------------------------
+// D1 REST API helpers
+// ---------------------------------------------------------------------------
+
+async function upsertHourlySnapshot(
+  apiToken: string,
+  accountId: string,
+  databaseId: string,
+  metrics: HourlyMetrics,
+  costs: ReturnType<typeof calculateHourlyCosts>
+): Promise<void> {
+  const sql = `INSERT INTO hourly_usage_snapshots (
+    id, snapshot_hour, project,
+    workers_requests, workers_errors, workers_cpu_time_ms,
+    workers_duration_p50_ms, workers_duration_p99_ms, workers_cost_usd,
+    d1_rows_read, d1_rows_written, d1_cost_usd,
+    kv_reads, kv_writes, kv_deletes, kv_list_ops, kv_cost_usd,
+    r2_class_a_ops, r2_class_b_ops, r2_egress_bytes, r2_cost_usd,
+    do_requests, do_gb_seconds, do_cost_usd,
+    total_cost_usd, collection_timestamp, sampling_mode
+  ) VALUES (
+    '${generateId()}', '${metrics.hour}', 'all',
+    ${metrics.workers.requests}, ${metrics.workers.errors}, ${metrics.workers.cpuTimeMs},
+    ${metrics.workers.duration50thMs}, ${metrics.workers.duration99thMs}, ${costs.workersCost},
+    ${metrics.d1.rowsRead}, ${metrics.d1.rowsWritten}, ${costs.d1Cost},
+    ${metrics.kv.reads}, ${metrics.kv.writes}, ${metrics.kv.deletes}, ${metrics.kv.lists}, ${costs.kvCost},
+    ${metrics.r2.classAOps}, ${metrics.r2.classBOps}, ${metrics.r2.egressBytes}, ${costs.r2Cost},
+    ${metrics.durableObjects.requests}, ${metrics.durableObjects.gbSeconds}, ${costs.doCost},
+    ${costs.totalCost}, ${Math.floor(Date.now() / 1000)}, 'BACKFILL'
+  ) ON CONFLICT (snapshot_hour, project) DO UPDATE SET
+    workers_requests = excluded.workers_requests,
+    workers_errors = excluded.workers_errors,
+    d1_rows_read = excluded.d1_rows_read,
+    d1_rows_written = excluded.d1_rows_written,
+    total_cost_usd = excluded.total_cost_usd`;
+
+  const response = await fetch(
+    `${REST_API_BASE}/accounts/${accountId}/d1/database/${databaseId}/query`,
+    {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${apiToken}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ sql }),
+    }
+  );
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`D1 upsert failed: ${response.status} ${error}`);
+  }
+}
+
+async function hasExistingData(
+  apiToken: string,
+  accountId: string,
+  databaseId: string,
+  hour: string
+): Promise<boolean> {
+  const sql = `SELECT workers_requests FROM hourly_usage_snapshots WHERE snapshot_hour = '${hour}' AND project = 'all' AND workers_requests > 0`;
+  const response = await fetch(
+    `${REST_API_BASE}/accounts/${accountId}/d1/database/${databaseId}/query`,
+    {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${apiToken}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ sql }),
+    }
+  );
+  if (!response.ok) return false;
+  const result = (await response.json()) as { result?: Array<{ results?: unknown[] }> };
+  return (result?.result?.[0]?.results?.length ?? 0) > 0;
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main(): Promise<void> {
+  const { startDate, endDate, dryRun } = parseArgs();
+
+  const apiToken = process.env.CLOUDFLARE_API_TOKEN;
+  const accountId = process.env.CLOUDFLARE_ACCOUNT_ID;
+  const databaseId = process.env.D1_DATABASE_ID;
+
+  if (!apiToken || !accountId || !databaseId) {
+    console.error('Error: Required environment variables:');
+    if (!apiToken) console.error('  CLOUDFLARE_API_TOKEN');
+    if (!accountId) console.error('  CLOUDFLARE_ACCOUNT_ID');
+    if (!databaseId) console.error('  D1_DATABASE_ID');
+    process.exit(1);
+  }
+
+  console.log('='.repeat(60));
+  console.log('Cloudflare Hourly Usage Backfill');
+  console.log('='.repeat(60));
+  console.log(`Start Date: ${startDate}`);
+  console.log(`End Date:   ${endDate}`);
+  console.log(`Account:    ${accountId}`);
+  console.log(`Database:   ${databaseId}`);
+  console.log(`Dry Run:    ${dryRun}`);
+  console.log('='.repeat(60));
+  console.log('');
+
+  const startHour = `${startDate}T00:00:00Z`;
+  const endHour = `${endDate}T23:00:00Z`;
+
+  console.log('Fetching hourly metrics from Cloudflare GraphQL…');
+  const metricsMap = await fetchHourlyMetrics(apiToken, accountId, startHour, endHour);
+  console.log(`  Received data for ${metricsMap.size} hours`);
+  console.log('');
+
+  let processed = 0;
+  let skipped = 0;
+  let errors = 0;
+
+  for (const [hour, metrics] of metricsMap) {
+    process.stdout.write(`[${processed + skipped + 1}/${metricsMap.size}] ${hour}: `);
+
+    if (!dryRun) {
+      const hasData = await hasExistingData(apiToken, accountId, databaseId, hour);
+      if (hasData) {
+        console.log('SKIPPED (already has data)');
+        skipped++;
+        continue;
+      }
+    }
+
+    try {
+      const costs = calculateHourlyCosts(metrics);
+      console.log(
+        `requests=${metrics.workers.requests}, d1_reads=${metrics.d1.rowsRead}, cost=$${costs.totalCost.toFixed(6)}`
+      );
+      if (!dryRun) {
+        await upsertHourlySnapshot(apiToken, accountId, databaseId, metrics, costs);
+      }
+      processed++;
+    } catch (e) {
+      console.log(`ERROR: ${e}`);
+      errors++;
+    }
+
+    await sleep(100);
+  }
+
+  console.log('');
+  console.log('='.repeat(60));
+  console.log(`Processed: ${processed}  Skipped: ${skipped}  Errors: ${errors}  Total: ${metricsMap.size}`);
+  console.log('='.repeat(60));
+  if (dryRun) console.log('\nDRY RUN — no data was written to D1');
+}
+
+main().catch((e) => {
+  console.error('Fatal error:', e);
+  process.exit(1);
+});