@littlebearapps/platform-admin-sdk 1.4.2 → 2.0.0

package/templates/shared/scripts/ops/backfill-monthly-rollups.ts

@@ -0,0 +1,125 @@
+#!/usr/bin/env npx tsx
+/**
+ * Monthly Rollup Backfill Script
+ *
+ * Aggregates daily_usage_rollups into monthly_usage_rollups via the D1 REST API.
+ * Queries existing daily data and rolls it up to monthly granularity.
+ *
+ * Prerequisites:
+ *   CLOUDFLARE_API_TOKEN  — API token with D1:Write permissions
+ *   CLOUDFLARE_ACCOUNT_ID — Your Cloudflare account ID
+ *   D1_DATABASE_ID        — Your platform-metrics D1 database ID
+ *
+ * Usage:
+ *   npx tsx scripts/ops/backfill-monthly-rollups.ts
+ *   npx tsx scripts/ops/backfill-monthly-rollups.ts --dry-run
+ *   npx tsx scripts/ops/backfill-monthly-rollups.ts --start 2026-01 --end 2026-03
+ *   npx tsx scripts/ops/backfill-monthly-rollups.ts --limit 12
+ */
+
+const REST_API_BASE = 'https://api.cloudflare.com/client/v4';
+
+interface Args {
+  start?: string;
+  end?: string;
+  dryRun: boolean;
+  limit: number;
+}
+
+function parseArgs(): Args {
+  const args = process.argv.slice(2);
+  const result: Args = { dryRun: false, limit: 12 };
+
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--start' && args[i + 1]) result.start = args[++i];
+    else if (args[i] === '--end' && args[i + 1]) result.end = args[++i];
+    else if (args[i] === '--limit' && args[i + 1]) result.limit = Number(args[++i]);
+    else if (args[i] === '--dry-run') result.dryRun = true;
+  }
+
+  if (!result.start) {
+    const d = new Date();
+    d.setMonth(d.getMonth() - 6);
+    result.start = d.toISOString().slice(0, 7);
+  }
+  if (!result.end) {
+    result.end = new Date().toISOString().slice(0, 7);
+  }
+
+  return result;
+}
+
+function getEnvOrThrow(key: string): string {
+  const val = process.env[key];
+  if (!val) throw new Error(`Missing required env var: ${key}`);
+  return val;
+}
+
+async function d1Query(accountId: string, dbId: string, token: string, sql: string, params: unknown[] = []) {
+  const res = await fetch(`${REST_API_BASE}/accounts/${accountId}/d1/database/${dbId}/query`, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ sql, params }),
+  });
+  if (!res.ok) {
+    const text = await res.text();
+    throw new Error(`D1 query failed (${res.status}): ${text}`);
+  }
+  return res.json() as Promise<{ result: Array<{ results: unknown[] }> }>;
+}
+
+async function main() {
+  const args = parseArgs();
+  const token = getEnvOrThrow('CLOUDFLARE_API_TOKEN');
+  const accountId = getEnvOrThrow('CLOUDFLARE_ACCOUNT_ID');
+  const dbId = getEnvOrThrow('D1_DATABASE_ID');
+
+  console.log(`Backfilling monthly rollups: ${args.start} → ${args.end} (limit: ${args.limit}, dry-run: ${args.dryRun})`);
+
+  // Generate month list
+  const months: string[] = [];
+  const current = new Date(args.start + '-01');
+  const endDate = new Date(args.end + '-01');
+  while (current <= endDate && months.length < args.limit) {
+    months.push(current.toISOString().slice(0, 7));
+    current.setMonth(current.getMonth() + 1);
+  }
+
+  let inserted = 0;
+  for (const month of months) {
+    const monthStart = `${month}-01`;
+    const nextMonth = new Date(monthStart);
+    nextMonth.setMonth(nextMonth.getMonth() + 1);
+    const monthEnd = nextMonth.toISOString().slice(0, 10);
+
+    if (args.dryRun) {
+      console.log(`[DRY-RUN] Would roll up ${month}`);
+      continue;
+    }
+
+    await d1Query(accountId, dbId, token, `
+      INSERT OR REPLACE INTO monthly_usage_rollups (
+        project, snapshot_month,
+        d1_reads, d1_writes, kv_reads, kv_writes,
+        r2_reads, r2_writes, worker_requests, total_cost_usd
+      )
+      SELECT
+        project, ? as snapshot_month,
+        SUM(d1_reads), SUM(d1_writes), SUM(kv_reads), SUM(kv_writes),
+        SUM(r2_reads), SUM(r2_writes), SUM(worker_requests), SUM(total_cost_usd)
+      FROM daily_usage_rollups
+      WHERE project = 'all' AND snapshot_date >= ? AND snapshot_date < ?
+      GROUP BY project
+    `, [month, monthStart, monthEnd]);
+
+    inserted++;
+    console.log(`  Rolled up ${month} (${inserted}/${months.length})`);
+  }
+
+  console.log(`Done. Inserted ${inserted} monthly rollup rows.`);
+}
+
+main().catch((err) => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

package/templates/shared/scripts/ops/discover-graphql-datasets.ts

@@ -0,0 +1,482 @@
+#!/usr/bin/env npx tsx
+/**
+ * Cloudflare GraphQL Dataset Discovery Script
+ *
+ * Discovers all available GraphQL datasets for your Cloudflare account using
+ * the Settings node. Helps identify datasets you're not currently querying.
+ *
+ * The Cloudflare GraphQL API exposes 70+ datasets. This script:
+ * 1. Queries the account Settings node to list ALL available datasets
+ * 2. Shows which datasets are enabled for your account
+ * 3. Compares against your current implementation to find gaps
+ *
+ * Prerequisites:
+ * - CLOUDFLARE_API_TOKEN env var with Analytics read permissions
+ * - CLOUDFLARE_ACCOUNT_ID env var
+ *
+ * Usage:
+ *   npx tsx scripts/ops/discover-graphql-datasets.ts
+ *   npx tsx scripts/ops/discover-graphql-datasets.ts --json           # JSON output
+ *   npx tsx scripts/ops/discover-graphql-datasets.ts --compare        # Compare vs current
+ */
+
+const GRAPHQL_ENDPOINT = 'https://api.cloudflare.com/client/v4/graphql';
+
+// TODO: Add the datasets your platform-usage worker currently queries.
+// These are the datasets collected by the default data-collection.ts handler.
+// Uncomment/add entries as you enable collection for each service.
+const CURRENTLY_QUERIED_DATASETS = [
+  'workersInvocationsAdaptive',
+  // 'workersSubrequestsAdaptive',
+  'd1AnalyticsAdaptiveGroups',
+  'd1StorageAdaptiveGroups',
+  'kvOperationsAdaptiveGroups',
+  'kvStorageAdaptiveGroups',
+  'r2OperationsAdaptiveGroups',
+  'r2StorageAdaptiveGroups',
+  // 'durableObjectsInvocationsAdaptiveGroups',
+  // 'durableObjectsPeriodicGroups',
+  // 'durableObjectsStorageGroups',
+  // 'vectorizeV2QueriesAdaptiveGroups',
+  // 'vectorizeV2StorageAdaptiveGroups',
+  'queueMessageOperationsAdaptiveGroups',
+  'queueConsumerMetricsAdaptiveGroups',
+  // 'workflowsAdaptiveGroups',
+  // 'aiInferenceAdaptive',
+  // 'httpRequestsAdaptiveGroups',
+];
+
+// Known billable datasets — these have cost implications if used
+const BILLABLE_DATASETS = [
+  // Workers Platform
+  'workersInvocationsAdaptive',
+  'workersSubrequestsAdaptive',
+
+  // D1
+  'd1AnalyticsAdaptiveGroups',
+  'd1StorageAdaptiveGroups',
+
+  // KV
+  'kvOperationsAdaptiveGroups',
+  'kvStorageAdaptiveGroups',
+
+  // R2
+  'r2OperationsAdaptiveGroups',
+  'r2StorageAdaptiveGroups',
+
+  // Durable Objects
+  'durableObjectsInvocationsAdaptiveGroups',
+  'durableObjectsPeriodicGroups',
+  'durableObjectsStorageGroups',
+
+  // Vectorize
+  'vectorizeV2QueriesAdaptiveGroups',
+  'vectorizeV2StorageAdaptiveGroups',
+
+  // Queues
+  'queueMessageOperationsAdaptiveGroups',
+  'queueConsumerMetricsAdaptiveGroups',
+
+  // Workflows
+  'workflowsAdaptiveGroups',
+
+  // Workers AI
+  'aiInferenceAdaptive',
+
+  // Stream (video)
+  'streamMinutesViewedAdaptive',
+  'streamMinutesViewedAdaptiveGroups',
+
+  // Images
+  'imagesAdaptiveGroups',
+
+  // Pages
+  'pagesProjectsAdaptiveGroups',
+
+  // AI Gateway
+  'aiGatewayAdaptiveGroups',
+
+  // Hyperdrive
+  'hyperdriveAdaptiveGroups',
+
+  // Browser Rendering
+  'browserRenderingAdaptiveGroups',
+
+  // TURN/Calls
+  'callsTurnUsageAdaptiveGroups',
+
+  // Spectrum
+  'spectrumNetworkAnalyticsAdaptiveGroups',
+
+  // Email Routing
+  'emailRoutingAdaptiveGroups',
+
+  // Cache/CDN
+  'httpRequestsAdaptiveGroups',
+];
+
+interface DatasetInfo {
+  name: string;
+  enabled: boolean;
+  maxPageSize?: number;
+}
+
+interface DiscoveryResult {
+  timestamp: string;
+  accountId: string;
+  totalDatasetsFound: number;
+  enabledDatasets: DatasetInfo[];
+  disabledDatasets: string[];
+  currentlyQueried: string[];
+  missingBillable: string[];
+  notQueried: string[];
+}
+
+// Parse command line arguments
+function parseArgs(): { json: boolean; compare: boolean } {
+  const args = process.argv.slice(2);
+  return {
+    json: args.includes('--json'),
+    compare: args.includes('--compare'),
+  };
+}
+
+// Make GraphQL request
+async function graphqlRequest<T>(
+  query: string,
+  variables: Record<string, unknown> = {}
+): Promise<T> {
+  const token = process.env.CLOUDFLARE_API_TOKEN;
+  if (!token) {
+    throw new Error('CLOUDFLARE_API_TOKEN environment variable is required');
+  }
+
+  const response = await fetch(GRAPHQL_ENDPOINT, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${token}`,
+    },
+    body: JSON.stringify({ query, variables }),
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`GraphQL request failed: ${response.status} ${text}`);
+  }
+
+  const result = (await response.json()) as { data?: T; errors?: unknown[] };
+
+  if (result.errors) {
+    throw new Error(`GraphQL errors: ${JSON.stringify(result.errors, null, 2)}`);
+  }
+
+  return result.data as T;
+}
+
+// Query via GraphQL introspection to discover available datasets
+async function discoverAccountDatasets(accountId: string): Promise<DatasetInfo[]> {
+  const introspectionQuery = `
+    query IntrospectAccountFields {
+      __type(name: "AccountResolver") {
+        fields {
+          name
+          description
+          type {
+            name
+            kind
+          }
+        }
+      }
+    }
+  `;
+
+  try {
+    const introspectionResult = await graphqlRequest<{
+      __type: {
+        fields: Array<{
+          name: string;
+          description: string | null;
+          type: { name: string | null; kind: string };
+        }>;
+      } | null;
+    }>(introspectionQuery);
+
+    if (!introspectionResult.__type) {
+      console.log('Could not introspect AccountResolver, trying settings approach...');
+      return await discoverDatasetsViaSettings(accountId);
+    }
+
+    const datasets: DatasetInfo[] = [];
+    for (const field of introspectionResult.__type.fields) {
+      if (
+        field.name.includes('Adaptive') ||
+        field.name.includes('Analytics') ||
+        field.name.includes('Groups') ||
+        field.name.includes('Metrics')
+      ) {
+        datasets.push({ name: field.name, enabled: true });
+      }
+    }
+
+    return datasets;
+  } catch {
+    console.log('Introspection failed, trying Settings node approach...');
+    return await discoverDatasetsViaSettings(accountId);
+  }
+}
+
+// Alternative: Query settings node for dataset availability
+async function discoverDatasetsViaSettings(accountId: string): Promise<DatasetInfo[]> {
+  const settingsQuery = `
+    query AccountDatasetSettings($accountTag: String!) {
+      viewer {
+        accounts(filter: { accountTag: $accountTag }) {
+          settings {
+            workersInvocationsAdaptive { enabled maxPageSize }
+            d1AnalyticsAdaptiveGroups { enabled maxPageSize }
+            d1StorageAdaptiveGroups { enabled maxPageSize }
+            kvOperationsAdaptiveGroups { enabled maxPageSize }
+            kvStorageAdaptiveGroups { enabled maxPageSize }
+            r2OperationsAdaptiveGroups { enabled maxPageSize }
+            r2StorageAdaptiveGroups { enabled maxPageSize }
+            durableObjectsInvocationsAdaptiveGroups { enabled maxPageSize }
+            durableObjectsPeriodicGroups { enabled maxPageSize }
+            durableObjectsStorageGroups { enabled maxPageSize }
+            vectorizeV2QueriesAdaptiveGroups { enabled maxPageSize }
+            vectorizeV2StorageAdaptiveGroups { enabled maxPageSize }
+            queueMessageOperationsAdaptiveGroups { enabled maxPageSize }
+            queueConsumerMetricsAdaptiveGroups { enabled maxPageSize }
+            workflowsAdaptiveGroups { enabled maxPageSize }
+            aiInferenceAdaptive { enabled maxPageSize }
+            httpRequestsAdaptiveGroups { enabled maxPageSize }
+          }
+        }
+      }
+    }
+  `;
+
+  try {
+    const settingsResult = await graphqlRequest<{
+      viewer: {
+        accounts: Array<{
+          settings: Record<string, { enabled: boolean; maxPageSize: number } | null>;
+        }>;
+      };
+    }>(settingsQuery, { accountTag: accountId });
+
+    const settings = settingsResult.viewer?.accounts?.[0]?.settings;
+    if (!settings) {
+      console.log('No settings returned, falling back to probe approach...');
+      return await probeDatasets(accountId);
+    }
+
+    const datasets: DatasetInfo[] = [];
+    for (const [name, info] of Object.entries(settings)) {
+      if (info) {
+        datasets.push({ name, enabled: info.enabled, maxPageSize: info.maxPageSize });
+      }
+    }
+
+    return datasets;
+  } catch (error) {
+    console.log('Settings query failed, falling back to probe approach...');
+    console.log('Error:', error);
+    return await probeDatasets(accountId);
+  }
+}
+
+// Fallback: Probe individual datasets to check availability
+async function probeDatasets(accountId: string): Promise<DatasetInfo[]> {
+  const datasetsToProbe = BILLABLE_DATASETS;
+  const results: DatasetInfo[] = [];
+  const now = new Date();
+  const yesterday = new Date(now.getTime() - 24 * 60 * 60 * 1000);
+
+  console.log(`Probing ${datasetsToProbe.length} datasets...`);
+
+  for (const dataset of datasetsToProbe) {
+    const probeQuery = `
+      query ProbeDataset($accountTag: String!, $limit: Int!) {
+        viewer {
+          accounts(filter: { accountTag: $accountTag }) {
+            ${dataset}(limit: $limit, filter: { datetime_geq: "${yesterday.toISOString()}", datetime_leq: "${now.toISOString()}" }) {
+              dimensions {
+                datetime
+              }
+            }
+          }
+        }
+      }
+    `;
+
+    try {
+      await graphqlRequest(probeQuery, { accountTag: accountId, limit: 1 });
+      results.push({ name: dataset, enabled: true });
+      process.stdout.write('.');
+    } catch (error) {
+      const errorStr = String(error);
+      if (errorStr.includes('Cannot query field') || errorStr.includes('Unknown field')) {
+        results.push({ name: dataset, enabled: false });
+        process.stdout.write('x');
+      } else if (errorStr.includes('not enabled') || errorStr.includes('not available')) {
+        results.push({ name: dataset, enabled: false });
+        process.stdout.write('-');
+      } else {
+        results.push({ name: dataset, enabled: true });
+        process.stdout.write('?');
+      }
+    }
+
+    // Small delay between probes to avoid rate limiting
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+
+  console.log('');
+  return results;
+}
+
+// Main discovery function
+async function discoverDatasets(): Promise<DiscoveryResult> {
+  const accountId = process.env.CLOUDFLARE_ACCOUNT_ID;
+  if (!accountId) {
+    throw new Error('CLOUDFLARE_ACCOUNT_ID environment variable is required');
+  }
+
+  console.log('='.repeat(60));
+  console.log('Cloudflare GraphQL Dataset Discovery');
+  console.log('='.repeat(60));
+  console.log(`Account ID: ${accountId}`);
+  console.log(`Timestamp: ${new Date().toISOString()}`);
+  console.log('');
+
+  console.log('Discovering available datasets...');
+  const datasets = await discoverAccountDatasets(accountId);
+
+  const enabledDatasets = datasets.filter((d) => d.enabled);
+  const disabledDatasets = datasets.filter((d) => !d.enabled).map((d) => d.name);
+
+  const enabledNames = new Set(enabledDatasets.map((d) => d.name));
+  const queriedSet = new Set(CURRENTLY_QUERIED_DATASETS);
+
+  const missingBillable = BILLABLE_DATASETS.filter(
+    (d) => enabledNames.has(d) && !queriedSet.has(d)
+  );
+
+  const notQueried = enabledDatasets.filter((d) => !queriedSet.has(d.name)).map((d) => d.name);
+
+  return {
+    timestamp: new Date().toISOString(),
+    accountId,
+    totalDatasetsFound: datasets.length,
+    enabledDatasets,
+    disabledDatasets,
+    currentlyQueried: CURRENTLY_QUERIED_DATASETS,
+    missingBillable,
+    notQueried,
+  };
+}
+
+// Display results
+function displayResults(result: DiscoveryResult, jsonOutput: boolean, compareMode: boolean): void {
+  if (jsonOutput) {
+    console.log(JSON.stringify(result, null, 2));
+    return;
+  }
+
+  console.log('');
+  console.log('='.repeat(60));
+  console.log('DISCOVERY RESULTS');
+  console.log('='.repeat(60));
+
+  console.log(`\nTotal Datasets Found: ${result.totalDatasetsFound}`);
+  console.log(`Enabled: ${result.enabledDatasets.length}`);
+  console.log(`Disabled/Unavailable: ${result.disabledDatasets.length}`);
+
+  console.log('\n--- ENABLED DATASETS ---');
+  for (const dataset of result.enabledDatasets) {
+    const queried = CURRENTLY_QUERIED_DATASETS.includes(dataset.name)
+      ? '[QUERIED]'
+      : '[NOT QUERIED]';
+    const billable = BILLABLE_DATASETS.includes(dataset.name) ? '(billable)' : '';
+    console.log(`  ${queried} ${dataset.name} ${billable}`);
+  }
+
+  if (result.disabledDatasets.length > 0) {
+    console.log('\n--- DISABLED/UNAVAILABLE DATASETS ---');
+    for (const name of result.disabledDatasets) {
+      console.log(`  [-] ${name}`);
+    }
+  }
+
+  if (compareMode || result.missingBillable.length > 0) {
+    console.log('\n' + '='.repeat(60));
+    console.log('GAP ANALYSIS');
+    console.log('='.repeat(60));
+
+    if (result.missingBillable.length > 0) {
+      console.log('\n*** BILLABLE DATASETS NOT BEING QUERIED ***');
+      for (const name of result.missingBillable) {
+        console.log(`  [!] ${name}`);
+      }
+    } else {
+      console.log('\nNo billable dataset gaps found.');
+    }
+
+    console.log(`\n--- Currently Queried (${CURRENTLY_QUERIED_DATASETS.length}) ---`);
+    for (const name of CURRENTLY_QUERIED_DATASETS) {
+      const enabled = result.enabledDatasets.some((d) => d.name === name);
+      console.log(`  ${enabled ? '[OK]' : '[??]'} ${name}`);
+    }
+  }
+
+  console.log('\n' + '='.repeat(60));
+  console.log('RECOMMENDATIONS');
+  console.log('='.repeat(60));
+
+  if (result.missingBillable.length > 0) {
+    console.log('\nAction Required: Add queries for these billable datasets:');
+    for (const name of result.missingBillable) {
+      console.log(`  - ${name}`);
+    }
+  }
+
+  const unusedEnabled = result.notQueried.filter(
+    (name) =>
+      !name.includes('firewall') &&
+      !name.includes('Firewall') &&
+      !name.includes('healthCheck') &&
+      !name.includes('loadBalancing') &&
+      !name.includes('network')
+  );
+
+  if (unusedEnabled.length > 0) {
+    console.log('\nOther enabled datasets not being queried (may be relevant):');
+    for (const name of unusedEnabled.slice(0, 10)) {
+      console.log(`  - ${name}`);
+    }
+    if (unusedEnabled.length > 10) {
+      console.log(`  ... and ${unusedEnabled.length - 10} more`);
+    }
+  }
+}
+
+// Main
+async function main(): Promise<void> {
+  const { json, compare } = parseArgs();
+
+  try {
+    const result = await discoverDatasets();
+    displayResults(result, json, compare);
+
+    // Exit with non-zero if there are missing billable datasets
+    if (result.missingBillable.length > 0) {
+      process.exit(1);
+    }
+  } catch (error) {
+    console.error('Error:', error);
+    process.exit(2);
+  }
+}
+
+main();