@littlebearapps/platform-admin-sdk 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/README.md +112 -0
  2. package/dist/index.d.ts +16 -0
  3. package/dist/index.js +89 -0
  4. package/dist/prompts.d.ts +27 -0
  5. package/dist/prompts.js +80 -0
  6. package/dist/scaffold.d.ts +5 -0
  7. package/dist/scaffold.js +65 -0
  8. package/dist/templates.d.ts +16 -0
  9. package/dist/templates.js +131 -0
  10. package/package.json +46 -0
  11. package/templates/full/migrations/006_pattern_discovery.sql +199 -0
  12. package/templates/full/migrations/007_notifications_search.sql +127 -0
  13. package/templates/full/workers/lib/pattern-discovery/ai-prompt.ts +644 -0
  14. package/templates/full/workers/lib/pattern-discovery/clustering.ts +278 -0
  15. package/templates/full/workers/lib/pattern-discovery/shadow-evaluation.ts +603 -0
  16. package/templates/full/workers/lib/pattern-discovery/storage.ts +806 -0
  17. package/templates/full/workers/lib/pattern-discovery/types.ts +159 -0
  18. package/templates/full/workers/lib/pattern-discovery/validation.ts +278 -0
  19. package/templates/full/workers/pattern-discovery.ts +661 -0
  20. package/templates/full/workers/platform-alert-router.ts +1809 -0
  21. package/templates/full/workers/platform-notifications.ts +424 -0
  22. package/templates/full/workers/platform-search.ts +480 -0
  23. package/templates/full/workers/platform-settings.ts +436 -0
  24. package/templates/full/wrangler.alert-router.jsonc.hbs +34 -0
  25. package/templates/full/wrangler.notifications.jsonc.hbs +23 -0
  26. package/templates/full/wrangler.pattern-discovery.jsonc.hbs +33 -0
  27. package/templates/full/wrangler.search.jsonc.hbs +16 -0
  28. package/templates/full/wrangler.settings.jsonc.hbs +23 -0
  29. package/templates/shared/README.md.hbs +69 -0
  30. package/templates/shared/config/budgets.yaml.hbs +72 -0
  31. package/templates/shared/config/services.yaml.hbs +45 -0
  32. package/templates/shared/migrations/001_core_tables.sql +117 -0
  33. package/templates/shared/migrations/002_usage_warehouse.sql +830 -0
  34. package/templates/shared/migrations/003_feature_tracking.sql +250 -0
  35. package/templates/shared/migrations/004_settings_alerts.sql +452 -0
  36. package/templates/shared/migrations/seed.sql.hbs +4 -0
  37. package/templates/shared/package.json.hbs +21 -0
  38. package/templates/shared/scripts/sync-config.ts +242 -0
  39. package/templates/shared/tsconfig.json +12 -0
  40. package/templates/shared/workers/lib/analytics-engine.ts +357 -0
  41. package/templates/shared/workers/lib/billing.ts +293 -0
  42. package/templates/shared/workers/lib/circuit-breaker-middleware.ts +25 -0
  43. package/templates/shared/workers/lib/control.ts +292 -0
  44. package/templates/shared/workers/lib/economics.ts +368 -0
  45. package/templates/shared/workers/lib/metrics.ts +103 -0
  46. package/templates/shared/workers/lib/platform-settings.ts +407 -0
  47. package/templates/shared/workers/lib/shared/allowances.ts +333 -0
  48. package/templates/shared/workers/lib/shared/cloudflare.ts +1362 -0
  49. package/templates/shared/workers/lib/shared/types.ts +58 -0
  50. package/templates/shared/workers/lib/telemetry-sampling.ts +360 -0
  51. package/templates/shared/workers/lib/usage/collectors/example.ts +96 -0
  52. package/templates/shared/workers/lib/usage/collectors/index.ts +128 -0
  53. package/templates/shared/workers/lib/usage/handlers/audit.ts +306 -0
  54. package/templates/shared/workers/lib/usage/handlers/backfill.ts +845 -0
  55. package/templates/shared/workers/lib/usage/handlers/behavioral.ts +429 -0
  56. package/templates/shared/workers/lib/usage/handlers/data-queries.ts +507 -0
  57. package/templates/shared/workers/lib/usage/handlers/dlq-admin.ts +364 -0
  58. package/templates/shared/workers/lib/usage/handlers/health-trends.ts +222 -0
  59. package/templates/shared/workers/lib/usage/handlers/index.ts +35 -0
  60. package/templates/shared/workers/lib/usage/handlers/usage-admin.ts +421 -0
  61. package/templates/shared/workers/lib/usage/handlers/usage-features.ts +1262 -0
  62. package/templates/shared/workers/lib/usage/handlers/usage-metrics.ts +2420 -0
  63. package/templates/shared/workers/lib/usage/handlers/usage-settings.ts +610 -0
  64. package/templates/shared/workers/lib/usage/queue/budget-enforcement.ts +1032 -0
  65. package/templates/shared/workers/lib/usage/queue/cost-budget-enforcement.ts +128 -0
  66. package/templates/shared/workers/lib/usage/queue/cost-calculator.ts +77 -0
  67. package/templates/shared/workers/lib/usage/queue/dlq-handler.ts +161 -0
  68. package/templates/shared/workers/lib/usage/queue/index.ts +19 -0
  69. package/templates/shared/workers/lib/usage/queue/telemetry-processor.ts +790 -0
  70. package/templates/shared/workers/lib/usage/scheduled/anomaly-detection.ts +732 -0
  71. package/templates/shared/workers/lib/usage/scheduled/data-collection.ts +956 -0
  72. package/templates/shared/workers/lib/usage/scheduled/error-digest.ts +343 -0
  73. package/templates/shared/workers/lib/usage/scheduled/index.ts +18 -0
  74. package/templates/shared/workers/lib/usage/scheduled/rollups.ts +1561 -0
  75. package/templates/shared/workers/lib/usage/shared/constants.ts +362 -0
  76. package/templates/shared/workers/lib/usage/shared/index.ts +14 -0
  77. package/templates/shared/workers/lib/usage/shared/types.ts +1066 -0
  78. package/templates/shared/workers/lib/usage/shared/utils.ts +795 -0
  79. package/templates/shared/workers/platform-usage.ts +1915 -0
  80. package/templates/shared/wrangler.usage.jsonc.hbs +58 -0
  81. package/templates/standard/migrations/005_error_collection.sql +162 -0
  82. package/templates/standard/workers/error-collector.ts +2670 -0
  83. package/templates/standard/workers/lib/error-collector/capture.ts +213 -0
  84. package/templates/standard/workers/lib/error-collector/digest.ts +448 -0
  85. package/templates/standard/workers/lib/error-collector/email-health-alerts.ts +262 -0
  86. package/templates/standard/workers/lib/error-collector/fingerprint.ts +258 -0
  87. package/templates/standard/workers/lib/error-collector/gap-alerts.ts +293 -0
  88. package/templates/standard/workers/lib/error-collector/github.ts +329 -0
  89. package/templates/standard/workers/lib/error-collector/types.ts +262 -0
  90. package/templates/standard/workers/lib/sentinel/gap-detection.ts +734 -0
  91. package/templates/standard/workers/lib/shared/slack-alerts.ts +585 -0
  92. package/templates/standard/workers/platform-sentinel.ts +1744 -0
  93. package/templates/standard/wrangler.error-collector.jsonc.hbs +44 -0
  94. package/templates/standard/wrangler.sentinel.jsonc.hbs +45 -0
@@ -0,0 +1,1561 @@
1
+ /**
2
+ * Scheduled Rollup Functions
3
+ *
4
+ * Functions for aggregating usage data from hourly snapshots into daily and monthly rollups.
5
+ * These run during the scheduled cron job at midnight UTC.
6
+ *
7
+ * Reference: backlog/tasks/task-61 - Platform-Usage-Refactoring.md
8
+ */
9
+
10
+ import type { Env } from '../shared';
11
+ import { generateId, fetchBillingSettings } from '../shared';
12
+ import { createLoggerFromEnv } from '@littlebearapps/platform-consumer-sdk';
13
+ import { calculateBillingPeriod, type BillingPeriod } from '../../billing';
14
+ import { calculateDailyBillableCosts, type AccountDailyUsage } from '@littlebearapps/platform-consumer-sdk';
15
+ import { getDailyUsageFromAnalyticsEngine } from '../../analytics-engine';
16
+
17
+ // =============================================================================
18
+ // PRICING VERSION CACHING
19
+ // =============================================================================
20
+
21
+ // In-memory cache for current pricing version ID (per-request lifetime)
22
+ let cachedPricingVersionId: number | null = null;
23
+
24
+ /**
25
+ * Get the current pricing version ID from D1.
26
+ * Returns the ID of the pricing version with NULL effective_to (current pricing).
27
+ * Caches result in-memory for the duration of the request.
28
+ *
29
+ * @param env - Worker environment with D1 binding
30
+ * @returns Pricing version ID, or null if no versioned pricing exists
31
+ */
32
+ async function getCurrentPricingVersionId(env: Env): Promise<number | null> {
33
+ // Return cached value if already loaded this request
34
+ if (cachedPricingVersionId !== null) {
35
+ return cachedPricingVersionId;
36
+ }
37
+
38
+ try {
39
+ const result = await env.PLATFORM_DB.prepare(
40
+ `SELECT id FROM pricing_versions WHERE effective_to IS NULL ORDER BY effective_from DESC LIMIT 1`
41
+ ).first<{ id: number }>();
42
+
43
+ cachedPricingVersionId = result?.id ?? null;
44
+
45
+ // Pricing version ID loaded (may be null if no versioned pricing)
46
+ return cachedPricingVersionId;
47
+ } catch {
48
+ // Table may not exist yet (pre-migration)
49
+ return null;
50
+ }
51
+ }
52
+
53
+ /**
54
+ * Reset cached pricing version ID (call at start of each request if needed).
55
+ */
56
+ export function resetPricingVersionCache(): void {
57
+ cachedPricingVersionId = null;
58
+ }
59
+
60
+ // =============================================================================
61
+ // CACHE INVALIDATION
62
+ // =============================================================================
63
+
64
+ /**
65
+ * Invalidate daily usage cache keys in KV.
66
+ * Called at midnight to ensure fresh data for the new day.
67
+ *
68
+ * @returns Number of cache keys deleted
69
+ */
70
+ export async function invalidateDailyCache(env: Env): Promise<number> {
71
+ const log = createLoggerFromEnv(env, 'platform-usage', 'platform:usage:cache');
72
+ log.info('Invalidating daily usage cache keys', { tag: 'CACHE' });
73
+ let deletedCount = 0;
74
+
75
+ try {
76
+ // List all keys with 'daily:' prefix
77
+ const listResult = await env.PLATFORM_CACHE.list({ prefix: 'daily:' });
78
+
79
+ if (listResult.keys.length === 0) {
80
+ log.info('No daily cache keys to invalidate', { tag: 'CACHE' });
81
+ return 0;
82
+ }
83
+
84
+ // Delete each matching key
85
+ for (const key of listResult.keys) {
86
+ try {
87
+ await env.PLATFORM_CACHE.delete(key.name);
88
+ deletedCount++;
89
+ } catch (error) {
90
+ log.error(
91
+ `Failed to delete key ${key.name}`,
92
+ error instanceof Error ? error : new Error(String(error))
93
+ );
94
+ }
95
+ }
96
+
97
+ log.info(`Invalidated ${deletedCount} daily cache keys`, { tag: 'CACHE' });
98
+ } catch (error) {
99
+ log.error(
100
+ 'Failed to list/invalidate daily cache',
101
+ error instanceof Error ? error : new Error(String(error))
102
+ );
103
+ }
104
+
105
+ return deletedCount;
106
+ }
107
+
108
+ // =============================================================================
109
+ // DAILY ROLLUP
110
+ // =============================================================================
111
+
112
+ /**
113
+ * Run daily rollup: aggregate hourly snapshots into daily_usage_rollups.
114
+ * Called at midnight UTC.
115
+ *
116
+ * IMPORTANT: Cost columns store cumulative MTD (month-to-date) values.
117
+ * Daily cost = today's end-of-day MTD - previous day's end-of-day MTD.
118
+ *
119
+ * For the first day of the month, previous day is in a different month,
120
+ * so we use today's MTD value directly (it represents the full first day).
121
+ *
122
+ * @param env - Worker environment
123
+ * @param date - Date to run rollup for (YYYY-MM-DD)
124
+ * @returns Number of rows changed
125
+ */
126
+ export async function runDailyRollup(env: Env, date: string): Promise<number> {
127
+ const log = createLoggerFromEnv(env, 'platform-usage', 'platform:usage:scheduled');
128
+ log.info(`Running daily rollup for ${date}`, { tag: 'SCHEDULED' });
129
+
130
+ // Calculate the previous day's date
131
+ const targetDate = new Date(date + 'T00:00:00Z');
132
+ const prevDate = new Date(targetDate);
133
+ prevDate.setUTCDate(prevDate.getUTCDate() - 1);
134
+ const prevDateStr = prevDate.toISOString().split('T')[0];
135
+
136
+ // Check if this is the first day of the month
137
+ const isFirstDayOfMonth = targetDate.getUTCDate() === 1;
138
+
139
+ // Get current pricing version ID for audit trail
140
+ const pricingVersionId = await getCurrentPricingVersionId(env);
141
+
142
+ // Fetch billing settings for billing-aware calculations
143
+ const billingSettings = await fetchBillingSettings(env);
144
+ const billingPeriod = calculateBillingPeriod(billingSettings.billingCycleDay, targetDate);
145
+ log.info(
146
+ `Period: ${billingPeriod.startDate.toISOString().split('T')[0]} to ` +
147
+ `${billingPeriod.endDate.toISOString().split('T')[0]}, ` +
148
+ `${billingPeriod.daysElapsed}/${billingPeriod.daysInPeriod} days elapsed (${Math.round(billingPeriod.progress * 100)}%)`,
149
+ { tag: 'BILLING' }
150
+ );
151
+
152
+ // Get today's MAX values (end-of-day MTD totals) for each project
153
+ interface HourlyMaxRow {
154
+ project: string;
155
+ workers_requests: number;
156
+ workers_errors: number;
157
+ workers_cpu_time_ms: number;
158
+ workers_duration_p50_ms_avg: number;
159
+ workers_duration_p99_ms_max: number;
160
+ workers_cost_usd: number;
161
+ d1_rows_read: number;
162
+ d1_rows_written: number;
163
+ d1_storage_bytes_max: number;
164
+ d1_cost_usd: number;
165
+ kv_reads: number;
166
+ kv_writes: number;
167
+ kv_deletes: number;
168
+ kv_list_ops: number;
169
+ kv_storage_bytes_max: number;
170
+ kv_cost_usd: number;
171
+ r2_class_a_ops: number;
172
+ r2_class_b_ops: number;
173
+ r2_storage_bytes_max: number;
174
+ r2_egress_bytes: number;
175
+ r2_cost_usd: number;
176
+ do_requests: number;
177
+ do_gb_seconds: number;
178
+ do_websocket_connections: number;
179
+ do_storage_reads: number;
180
+ do_storage_writes: number;
181
+ do_storage_deletes: number;
182
+ do_cost_usd: number;
183
+ vectorize_queries: number;
184
+ vectorize_vectors_stored_max: number;
185
+ vectorize_cost_usd: number;
186
+ aigateway_requests: number;
187
+ aigateway_tokens_in: number;
188
+ aigateway_tokens_out: number;
189
+ aigateway_cached_requests: number;
190
+ aigateway_cost_usd: number;
191
+ pages_deployments: number;
192
+ pages_bandwidth_bytes: number;
193
+ pages_cost_usd: number;
194
+ queues_messages_produced: number;
195
+ queues_messages_consumed: number;
196
+ queues_cost_usd: number;
197
+ workersai_requests: number;
198
+ workersai_neurons: number;
199
+ workersai_cost_usd: number;
200
+ workflows_executions: number;
201
+ workflows_successes: number;
202
+ workflows_failures: number;
203
+ workflows_wall_time_ms: number;
204
+ workflows_cpu_time_ms: number;
205
+ workflows_cost_usd: number;
206
+ total_cost_usd: number;
207
+ samples_count: number;
208
+ }
209
+
210
+ // REGRESSION CHECK: D1 rows MUST use SUM() not MAX()
211
+ // Bug fix (2026-01-24): Using MAX() caused 825M read counts when actual was ~65K.
212
+ // The hourly_usage_snapshots table stores operational counts per hour.
213
+ // Daily rollups must SUM these hourly values, not take the MAX single-hour peak.
214
+ const todayResult = await env.PLATFORM_DB.prepare(
215
+ `
216
+ SELECT
217
+ project,
218
+ SUM(workers_requests) as workers_requests,
219
+ SUM(workers_errors) as workers_errors,
220
+ SUM(workers_cpu_time_ms) as workers_cpu_time_ms,
221
+ AVG(workers_duration_p50_ms) as workers_duration_p50_ms_avg,
222
+ MAX(workers_duration_p99_ms) as workers_duration_p99_ms_max,
223
+ MAX(workers_cost_usd) as workers_cost_usd,
224
+ SUM(d1_rows_read) as d1_rows_read, -- MUST be SUM, not MAX (regression check)
225
+ SUM(d1_rows_written) as d1_rows_written, -- MUST be SUM, not MAX (regression check)
226
+ MAX(d1_storage_bytes) as d1_storage_bytes_max,
227
+ MAX(d1_cost_usd) as d1_cost_usd,
228
+ SUM(kv_reads) as kv_reads,
229
+ SUM(kv_writes) as kv_writes,
230
+ SUM(kv_deletes) as kv_deletes,
231
+ SUM(kv_list_ops) as kv_list_ops,
232
+ MAX(kv_storage_bytes) as kv_storage_bytes_max,
233
+ MAX(kv_cost_usd) as kv_cost_usd,
234
+ SUM(r2_class_a_ops) as r2_class_a_ops,
235
+ SUM(r2_class_b_ops) as r2_class_b_ops,
236
+ MAX(r2_storage_bytes) as r2_storage_bytes_max,
237
+ SUM(r2_egress_bytes) as r2_egress_bytes,
238
+ MAX(r2_cost_usd) as r2_cost_usd,
239
+ SUM(do_requests) as do_requests,
240
+ MAX(COALESCE(do_gb_seconds, 0)) as do_gb_seconds,
241
+ SUM(do_websocket_connections) as do_websocket_connections,
242
+ SUM(do_storage_reads) as do_storage_reads,
243
+ SUM(do_storage_writes) as do_storage_writes,
244
+ SUM(do_storage_deletes) as do_storage_deletes,
245
+ MAX(do_cost_usd) as do_cost_usd,
246
+ SUM(vectorize_queries) as vectorize_queries,
247
+ MAX(vectorize_vectors_stored) as vectorize_vectors_stored_max,
248
+ MAX(vectorize_cost_usd) as vectorize_cost_usd,
249
+ SUM(aigateway_requests) as aigateway_requests,
250
+ SUM(aigateway_tokens_in) as aigateway_tokens_in,
251
+ SUM(aigateway_tokens_out) as aigateway_tokens_out,
252
+ SUM(aigateway_cached_requests) as aigateway_cached_requests,
253
+ MAX(aigateway_cost_usd) as aigateway_cost_usd,
254
+ SUM(pages_deployments) as pages_deployments,
255
+ SUM(pages_bandwidth_bytes) as pages_bandwidth_bytes,
256
+ MAX(pages_cost_usd) as pages_cost_usd,
257
+ SUM(queues_messages_produced) as queues_messages_produced,
258
+ SUM(queues_messages_consumed) as queues_messages_consumed,
259
+ MAX(queues_cost_usd) as queues_cost_usd,
260
+ SUM(workersai_requests) as workersai_requests,
261
+ SUM(workersai_neurons) as workersai_neurons,
262
+ MAX(workersai_cost_usd) as workersai_cost_usd,
263
+ SUM(COALESCE(workflows_executions, 0)) as workflows_executions,
264
+ SUM(COALESCE(workflows_successes, 0)) as workflows_successes,
265
+ SUM(COALESCE(workflows_failures, 0)) as workflows_failures,
266
+ SUM(COALESCE(workflows_wall_time_ms, 0)) as workflows_wall_time_ms,
267
+ SUM(COALESCE(workflows_cpu_time_ms, 0)) as workflows_cpu_time_ms,
268
+ MAX(COALESCE(workflows_cost_usd, 0)) as workflows_cost_usd,
269
+ MAX(total_cost_usd) as total_cost_usd,
270
+ COUNT(*) as samples_count
271
+ FROM hourly_usage_snapshots
272
+ WHERE DATE(snapshot_hour) = ?
273
+ GROUP BY project
274
+ `
275
+ )
276
+ .bind(date)
277
+ .all<HourlyMaxRow>();
278
+
279
+ if (!todayResult.results || todayResult.results.length === 0) {
280
+ log.info(`No hourly data found for ${date}`, { tag: 'SCHEDULED' });
281
+ return 0;
282
+ }
283
+
284
+ // Get previous day's MAX values (only needed if not first day of month)
285
+ interface PrevDayMaxRow {
286
+ project: string;
287
+ workers_cost_usd: number;
288
+ d1_rows_read: number;
289
+ d1_rows_written: number;
290
+ d1_cost_usd: number;
291
+ kv_cost_usd: number;
292
+ r2_cost_usd: number;
293
+ do_cost_usd: number;
294
+ vectorize_cost_usd: number;
295
+ aigateway_cost_usd: number;
296
+ pages_cost_usd: number;
297
+ queues_cost_usd: number;
298
+ workersai_cost_usd: number;
299
+ workflows_cost_usd: number;
300
+ total_cost_usd: number;
301
+ }
302
+
303
+ const prevDayMaxByProject: Map<string, PrevDayMaxRow> = new Map();
304
+
305
+ if (!isFirstDayOfMonth) {
306
+ // Previous day's aggregation for MTD calculation
307
+ // D1 rows use SUM (see regression check comment above)
308
+ const prevResult = await env.PLATFORM_DB.prepare(
309
+ `
310
+ SELECT
311
+ project,
312
+ MAX(workers_cost_usd) as workers_cost_usd,
313
+ SUM(d1_rows_read) as d1_rows_read, -- MUST be SUM, not MAX
314
+ SUM(d1_rows_written) as d1_rows_written, -- MUST be SUM, not MAX
315
+ MAX(d1_cost_usd) as d1_cost_usd,
316
+ MAX(kv_cost_usd) as kv_cost_usd,
317
+ MAX(r2_cost_usd) as r2_cost_usd,
318
+ MAX(do_cost_usd) as do_cost_usd,
319
+ MAX(vectorize_cost_usd) as vectorize_cost_usd,
320
+ MAX(aigateway_cost_usd) as aigateway_cost_usd,
321
+ MAX(pages_cost_usd) as pages_cost_usd,
322
+ MAX(queues_cost_usd) as queues_cost_usd,
323
+ MAX(workersai_cost_usd) as workersai_cost_usd,
324
+ MAX(COALESCE(workflows_cost_usd, 0)) as workflows_cost_usd,
325
+ MAX(total_cost_usd) as total_cost_usd
326
+ FROM hourly_usage_snapshots
327
+ WHERE DATE(snapshot_hour) = ?
328
+ GROUP BY project
329
+ `
330
+ )
331
+ .bind(prevDateStr)
332
+ .all<PrevDayMaxRow>();
333
+
334
+ if (prevResult.results) {
335
+ for (const row of prevResult.results) {
336
+ prevDayMaxByProject.set(row.project, row);
337
+ }
338
+ }
339
+ }
340
+
341
+ // ============================================================================
342
+ // ACCOUNT-LEVEL BILLABLE COST CALCULATION
343
+ // ============================================================================
344
+ // Calculate actual billable costs at account level BEFORE storing per-project.
345
+ // This ensures D1 daily_usage_rollups is the "Source of Truth" with accurate
346
+ // billable amounts that match Cloudflare invoices.
347
+ //
348
+ // Formula: billable_cost = max(0, account_usage - prorated_allowance) * rate
349
+ // ============================================================================
350
+
351
+ // Step 1: Aggregate all project usage to account level
352
+ const accountUsage: AccountDailyUsage = {
353
+ workersRequests: 0,
354
+ workersCpuMs: 0,
355
+ d1RowsRead: 0,
356
+ d1RowsWritten: 0,
357
+ d1StorageBytes: 0,
358
+ kvReads: 0,
359
+ kvWrites: 0,
360
+ kvDeletes: 0,
361
+ kvLists: 0,
362
+ kvStorageBytes: 0,
363
+ r2ClassA: 0,
364
+ r2ClassB: 0,
365
+ r2StorageBytes: 0,
366
+ doRequests: 0,
367
+ doGbSeconds: 0,
368
+ doStorageReads: 0,
369
+ doStorageWrites: 0,
370
+ doStorageDeletes: 0,
371
+ vectorizeQueries: 0,
372
+ vectorizeStoredDimensions: 0,
373
+ aiGatewayRequests: 0,
374
+ workersAINeurons: 0,
375
+ queuesMessagesProduced: 0,
376
+ queuesMessagesConsumed: 0,
377
+ pagesDeployments: 0,
378
+ pagesBandwidthBytes: 0,
379
+ workflowsExecutions: 0,
380
+ workflowsCpuMs: 0,
381
+ };
382
+
383
+ // Track raw costs for proportional distribution
384
+ const projectRawCosts = new Map<
385
+ string,
386
+ {
387
+ workers: number;
388
+ d1: number;
389
+ kv: number;
390
+ r2: number;
391
+ durableObjects: number;
392
+ vectorize: number;
393
+ aiGateway: number;
394
+ workersAI: number;
395
+ pages: number;
396
+ queues: number;
397
+ workflows: number;
398
+ total: number;
399
+ }
400
+ >();
401
+
402
+ let totalRawCost = 0;
403
+
404
+ for (const row of todayResult.results) {
405
+ // Calculate daily deltas for this project
406
+ const prev = prevDayMaxByProject.get(row.project);
407
+
408
+ // Raw cost deltas (before allowance subtraction)
409
+ const rawWorkersCost =
410
+ isFirstDayOfMonth || !prev
411
+ ? row.workers_cost_usd
412
+ : Math.max(0, row.workers_cost_usd - prev.workers_cost_usd);
413
+ const rawD1Cost =
414
+ isFirstDayOfMonth || !prev
415
+ ? row.d1_cost_usd
416
+ : Math.max(0, row.d1_cost_usd - prev.d1_cost_usd);
417
+ const rawKvCost =
418
+ isFirstDayOfMonth || !prev
419
+ ? row.kv_cost_usd
420
+ : Math.max(0, row.kv_cost_usd - prev.kv_cost_usd);
421
+ const rawR2Cost =
422
+ isFirstDayOfMonth || !prev
423
+ ? row.r2_cost_usd
424
+ : Math.max(0, row.r2_cost_usd - prev.r2_cost_usd);
425
+ const rawDoCost =
426
+ isFirstDayOfMonth || !prev
427
+ ? row.do_cost_usd
428
+ : Math.max(0, row.do_cost_usd - prev.do_cost_usd);
429
+ const rawVectorizeCost =
430
+ isFirstDayOfMonth || !prev
431
+ ? row.vectorize_cost_usd
432
+ : Math.max(0, row.vectorize_cost_usd - prev.vectorize_cost_usd);
433
+ const rawAiGatewayCost =
434
+ isFirstDayOfMonth || !prev
435
+ ? row.aigateway_cost_usd
436
+ : Math.max(0, row.aigateway_cost_usd - prev.aigateway_cost_usd);
437
+ const rawWorkersAICost =
438
+ isFirstDayOfMonth || !prev
439
+ ? row.workersai_cost_usd
440
+ : Math.max(0, row.workersai_cost_usd - prev.workersai_cost_usd);
441
+ const rawPagesCost =
442
+ isFirstDayOfMonth || !prev
443
+ ? row.pages_cost_usd
444
+ : Math.max(0, row.pages_cost_usd - prev.pages_cost_usd);
445
+ const rawQueuesCost =
446
+ isFirstDayOfMonth || !prev
447
+ ? row.queues_cost_usd
448
+ : Math.max(0, row.queues_cost_usd - prev.queues_cost_usd);
449
+ const rawWorkflowsCost =
450
+ isFirstDayOfMonth || !prev
451
+ ? row.workflows_cost_usd
452
+ : Math.max(0, row.workflows_cost_usd - prev.workflows_cost_usd);
453
+ const rawTotalCost =
454
+ rawWorkersCost +
455
+ rawD1Cost +
456
+ rawKvCost +
457
+ rawR2Cost +
458
+ rawDoCost +
459
+ rawVectorizeCost +
460
+ rawAiGatewayCost +
461
+ rawWorkersAICost +
462
+ rawPagesCost +
463
+ rawQueuesCost +
464
+ rawWorkflowsCost;
465
+
466
+ projectRawCosts.set(row.project, {
467
+ workers: rawWorkersCost,
468
+ d1: rawD1Cost,
469
+ kv: rawKvCost,
470
+ r2: rawR2Cost,
471
+ durableObjects: rawDoCost,
472
+ vectorize: rawVectorizeCost,
473
+ aiGateway: rawAiGatewayCost,
474
+ workersAI: rawWorkersAICost,
475
+ pages: rawPagesCost,
476
+ queues: rawQueuesCost,
477
+ workflows: rawWorkflowsCost,
478
+ total: rawTotalCost,
479
+ });
480
+
481
+ totalRawCost += rawTotalCost;
482
+
483
+ // Aggregate MTD usage to account level (for billable cost calculation)
484
+ accountUsage.workersRequests += row.workers_requests || 0;
485
+ accountUsage.workersCpuMs += row.workers_cpu_time_ms || 0;
486
+ accountUsage.d1RowsRead += row.d1_rows_read || 0;
487
+ accountUsage.d1RowsWritten += row.d1_rows_written || 0;
488
+ accountUsage.d1StorageBytes += row.d1_storage_bytes_max || 0;
489
+ accountUsage.kvReads += row.kv_reads || 0;
490
+ accountUsage.kvWrites += row.kv_writes || 0;
491
+ accountUsage.kvDeletes += row.kv_deletes || 0;
492
+ accountUsage.kvLists += row.kv_list_ops || 0;
493
+ accountUsage.kvStorageBytes += row.kv_storage_bytes_max || 0;
494
+ accountUsage.r2ClassA += row.r2_class_a_ops || 0;
495
+ accountUsage.r2ClassB += row.r2_class_b_ops || 0;
496
+ accountUsage.r2StorageBytes += row.r2_storage_bytes_max || 0;
497
+ accountUsage.doRequests += row.do_requests || 0;
498
+ accountUsage.doGbSeconds += row.do_gb_seconds || 0;
499
+ accountUsage.doStorageReads += row.do_storage_reads || 0;
500
+ accountUsage.doStorageWrites += row.do_storage_writes || 0;
501
+ accountUsage.doStorageDeletes += row.do_storage_deletes || 0;
502
+ accountUsage.vectorizeQueries += row.vectorize_queries || 0;
503
+ accountUsage.vectorizeStoredDimensions += row.vectorize_vectors_stored_max || 0;
504
+ accountUsage.aiGatewayRequests += row.aigateway_requests || 0;
505
+ accountUsage.workersAINeurons += row.workersai_neurons || 0;
506
+ accountUsage.queuesMessagesProduced += row.queues_messages_produced || 0;
507
+ accountUsage.queuesMessagesConsumed += row.queues_messages_consumed || 0;
508
+ accountUsage.pagesDeployments += row.pages_deployments || 0;
509
+ accountUsage.pagesBandwidthBytes += row.pages_bandwidth_bytes || 0;
510
+ accountUsage.workflowsExecutions += row.workflows_executions || 0;
511
+ accountUsage.workflowsCpuMs += row.workflows_cpu_time_ms || 0;
512
+ }
513
+
514
+ // Step 2: Calculate account-level billable costs with proper allowance subtraction
515
+ const accountBillableCosts = calculateDailyBillableCosts(
516
+ accountUsage,
517
+ billingPeriod.daysElapsed,
518
+ billingPeriod.daysInPeriod
519
+ );
520
+
521
+ // Log account-level billable costs
522
+ log.info(
523
+ `Account billable costs (day ${billingPeriod.daysElapsed}/${billingPeriod.daysInPeriod}): ` +
524
+ `rawTotal=$${totalRawCost.toFixed(4)}, billableTotal=$${accountBillableCosts.total.toFixed(4)}, ` +
525
+ `workers=$${accountBillableCosts.workers.toFixed(4)}, d1=$${accountBillableCosts.d1.toFixed(4)}, ` +
526
+ `kv=$${accountBillableCosts.kv.toFixed(4)}, do=$${accountBillableCosts.durableObjects.toFixed(4)}`,
527
+ { tag: 'BILLING' }
528
+ );
529
+
530
+ // Step 3: Calculate proportional distribution factor
531
+ // Each project gets: (projectRawCost / totalRawCost) * accountBillableCost
532
+ // This ensures the sum of project billable costs equals account billable cost
533
+ const costScaleFactor = totalRawCost > 0 ? accountBillableCosts.total / totalRawCost : 0;
534
+
535
+ log.info(
536
+ `Cost scale factor: ${costScaleFactor.toFixed(4)} ` +
537
+ `(billable $${accountBillableCosts.total.toFixed(4)} / raw $${totalRawCost.toFixed(4)})`,
538
+ { tag: 'BILLING' }
539
+ );
540
+
541
+ // Insert daily rollup with BILLABLE costs (proportionally distributed)
542
+ let totalChanges = 0;
543
+
544
+ for (const today of todayResult.results) {
545
+ const prev = prevDayMaxByProject.get(today.project);
546
+ const rawCosts = projectRawCosts.get(today.project)!;
547
+
548
+ // D1 rows are cumulative MTD, need delta calculation
549
+ const d1RowsReadDelta =
550
+ isFirstDayOfMonth || !prev
551
+ ? today.d1_rows_read
552
+ : Math.max(0, today.d1_rows_read - prev.d1_rows_read);
553
+ const d1RowsWrittenDelta =
554
+ isFirstDayOfMonth || !prev
555
+ ? today.d1_rows_written
556
+ : Math.max(0, today.d1_rows_written - prev.d1_rows_written);
557
+
558
+ // ========================================================================
559
+ // BILLABLE COSTS (Proportionally distributed from account-level)
560
+ // ========================================================================
561
+ // Each project's billable cost = rawCost * costScaleFactor
562
+ // This ensures: sum(projectBillableCost) == accountBillableCost
563
+ // The costScaleFactor adjusts for allowances at account level
564
+ // ========================================================================
565
+ const workersCostBillable = rawCosts.workers * costScaleFactor;
566
+ const d1CostBillable = rawCosts.d1 * costScaleFactor;
567
+ const kvCostBillable = rawCosts.kv * costScaleFactor;
568
+ const r2CostBillable = rawCosts.r2 * costScaleFactor;
569
+ const doCostBillable = rawCosts.durableObjects * costScaleFactor;
570
+ const vectorizeCostBillable = rawCosts.vectorize * costScaleFactor;
571
+ const aigatewayCostBillable = rawCosts.aiGateway * costScaleFactor;
572
+ const pagesCostBillable = rawCosts.pages * costScaleFactor;
573
+ const queuesCostBillable = rawCosts.queues * costScaleFactor;
574
+ const workersaiCostBillable = rawCosts.workersAI * costScaleFactor;
575
+ const workflowsCostBillable = rawCosts.workflows * costScaleFactor;
576
+ const totalCostBillable = rawCosts.total * costScaleFactor;
577
+
578
+ log.info(
579
+ `Rollup ${date} project=${today.project}: ` +
580
+ `rawCost=$${rawCosts.total.toFixed(4)}, ` +
581
+ `billableCost=$${totalCostBillable.toFixed(4)}, ` +
582
+ `scaleFactor=${costScaleFactor.toFixed(4)}, ` +
583
+ `d1WritesDaily=${d1RowsWrittenDelta.toLocaleString()}`,
584
+ { tag: 'SCHEDULED' }
585
+ );
586
+
587
+ const result = await env.PLATFORM_DB.prepare(
588
+ `
589
+ INSERT INTO daily_usage_rollups (
590
+ snapshot_date, project,
591
+ workers_requests, workers_errors, workers_cpu_time_ms,
592
+ workers_duration_p50_ms_avg, workers_duration_p99_ms_max, workers_cost_usd,
593
+ d1_rows_read, d1_rows_written, d1_storage_bytes_max, d1_cost_usd,
594
+ kv_reads, kv_writes, kv_deletes, kv_list_ops, kv_storage_bytes_max, kv_cost_usd,
595
+ r2_class_a_ops, r2_class_b_ops, r2_storage_bytes_max, r2_egress_bytes, r2_cost_usd,
596
+ do_requests, do_gb_seconds, do_websocket_connections, do_storage_reads, do_storage_writes, do_storage_deletes, do_cost_usd,
597
+ vectorize_queries, vectorize_vectors_stored_max, vectorize_cost_usd,
598
+ aigateway_requests, aigateway_tokens_in, aigateway_tokens_out, aigateway_cached_requests, aigateway_cost_usd,
599
+ pages_deployments, pages_bandwidth_bytes, pages_cost_usd,
600
+ queues_messages_produced, queues_messages_consumed, queues_cost_usd,
601
+ workersai_requests, workersai_neurons, workersai_cost_usd,
602
+ workflows_executions, workflows_successes, workflows_failures, workflows_wall_time_ms, workflows_cpu_time_ms, workflows_cost_usd,
603
+ total_cost_usd, samples_count, rollup_version, pricing_version_id
604
+ )
605
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 2, ?)
606
+ ON CONFLICT (snapshot_date, project) DO UPDATE SET
607
+ workers_requests = excluded.workers_requests,
608
+ workers_errors = excluded.workers_errors,
609
+ workers_cpu_time_ms = excluded.workers_cpu_time_ms,
610
+ workers_duration_p50_ms_avg = excluded.workers_duration_p50_ms_avg,
611
+ workers_duration_p99_ms_max = excluded.workers_duration_p99_ms_max,
612
+ workers_cost_usd = excluded.workers_cost_usd,
613
+ d1_rows_read = excluded.d1_rows_read,
614
+ d1_rows_written = excluded.d1_rows_written,
615
+ d1_storage_bytes_max = excluded.d1_storage_bytes_max,
616
+ d1_cost_usd = excluded.d1_cost_usd,
617
+ kv_reads = excluded.kv_reads,
618
+ kv_writes = excluded.kv_writes,
619
+ kv_deletes = excluded.kv_deletes,
620
+ kv_list_ops = excluded.kv_list_ops,
621
+ kv_storage_bytes_max = excluded.kv_storage_bytes_max,
622
+ kv_cost_usd = excluded.kv_cost_usd,
623
+ r2_class_a_ops = excluded.r2_class_a_ops,
624
+ r2_class_b_ops = excluded.r2_class_b_ops,
625
+ r2_storage_bytes_max = excluded.r2_storage_bytes_max,
626
+ r2_egress_bytes = excluded.r2_egress_bytes,
627
+ r2_cost_usd = excluded.r2_cost_usd,
628
+ do_requests = excluded.do_requests,
629
+ do_gb_seconds = excluded.do_gb_seconds,
630
+ do_websocket_connections = excluded.do_websocket_connections,
631
+ do_storage_reads = excluded.do_storage_reads,
632
+ do_storage_writes = excluded.do_storage_writes,
633
+ do_storage_deletes = excluded.do_storage_deletes,
634
+ do_cost_usd = excluded.do_cost_usd,
635
+ vectorize_queries = excluded.vectorize_queries,
636
+ vectorize_vectors_stored_max = excluded.vectorize_vectors_stored_max,
637
+ vectorize_cost_usd = excluded.vectorize_cost_usd,
638
+ aigateway_requests = excluded.aigateway_requests,
639
+ aigateway_tokens_in = excluded.aigateway_tokens_in,
640
+ aigateway_tokens_out = excluded.aigateway_tokens_out,
641
+ aigateway_cached_requests = excluded.aigateway_cached_requests,
642
+ aigateway_cost_usd = excluded.aigateway_cost_usd,
643
+ pages_deployments = excluded.pages_deployments,
644
+ pages_bandwidth_bytes = excluded.pages_bandwidth_bytes,
645
+ pages_cost_usd = excluded.pages_cost_usd,
646
+ queues_messages_produced = excluded.queues_messages_produced,
647
+ queues_messages_consumed = excluded.queues_messages_consumed,
648
+ queues_cost_usd = excluded.queues_cost_usd,
649
+ workersai_requests = excluded.workersai_requests,
650
+ workersai_neurons = excluded.workersai_neurons,
651
+ workersai_cost_usd = excluded.workersai_cost_usd,
652
+ workflows_executions = excluded.workflows_executions,
653
+ workflows_successes = excluded.workflows_successes,
654
+ workflows_failures = excluded.workflows_failures,
655
+ workflows_wall_time_ms = excluded.workflows_wall_time_ms,
656
+ workflows_cpu_time_ms = excluded.workflows_cpu_time_ms,
657
+ workflows_cost_usd = excluded.workflows_cost_usd,
658
+ total_cost_usd = excluded.total_cost_usd,
659
+ samples_count = excluded.samples_count,
660
+ rollup_version = excluded.rollup_version,
661
+ pricing_version_id = excluded.pricing_version_id
662
+ `
663
+ )
664
+ .bind(
665
+ date,
666
+ today.project,
667
+ today.workers_requests,
668
+ today.workers_errors,
669
+ today.workers_cpu_time_ms,
670
+ today.workers_duration_p50_ms_avg,
671
+ today.workers_duration_p99_ms_max,
672
+ workersCostBillable, // BILLABLE cost (account-level allowance applied)
673
+ d1RowsReadDelta,
674
+ d1RowsWrittenDelta,
675
+ today.d1_storage_bytes_max,
676
+ d1CostBillable, // BILLABLE cost
677
+ today.kv_reads,
678
+ today.kv_writes,
679
+ today.kv_deletes,
680
+ today.kv_list_ops,
681
+ today.kv_storage_bytes_max,
682
+ kvCostBillable, // BILLABLE cost
683
+ today.r2_class_a_ops,
684
+ today.r2_class_b_ops,
685
+ today.r2_storage_bytes_max,
686
+ today.r2_egress_bytes,
687
+ r2CostBillable, // BILLABLE cost
688
+ today.do_requests,
689
+ today.do_gb_seconds,
690
+ today.do_websocket_connections,
691
+ today.do_storage_reads,
692
+ today.do_storage_writes,
693
+ today.do_storage_deletes,
694
+ doCostBillable, // BILLABLE cost
695
+ today.vectorize_queries,
696
+ today.vectorize_vectors_stored_max,
697
+ vectorizeCostBillable, // BILLABLE cost
698
+ today.aigateway_requests,
699
+ today.aigateway_tokens_in,
700
+ today.aigateway_tokens_out,
701
+ today.aigateway_cached_requests,
702
+ aigatewayCostBillable, // BILLABLE cost (always 0 - free tier)
703
+ today.pages_deployments,
704
+ today.pages_bandwidth_bytes,
705
+ pagesCostBillable, // BILLABLE cost
706
+ today.queues_messages_produced,
707
+ today.queues_messages_consumed,
708
+ queuesCostBillable, // BILLABLE cost
709
+ today.workersai_requests,
710
+ today.workersai_neurons,
711
+ workersaiCostBillable, // BILLABLE cost
712
+ today.workflows_executions,
713
+ today.workflows_successes,
714
+ today.workflows_failures,
715
+ today.workflows_wall_time_ms,
716
+ today.workflows_cpu_time_ms,
717
+ workflowsCostBillable, // BILLABLE cost (always 0 - beta)
718
+ totalCostBillable, // BILLABLE total cost
719
+ today.samples_count,
720
+ pricingVersionId
721
+ )
722
+ .run();
723
+
724
+ totalChanges += result.meta.changes || 0;
725
+ }
726
+
727
+ // ============================================================================
728
+ // BILLING SUMMARY
729
+ // ============================================================================
730
+ // D1 daily_usage_rollups is now the "Source of Truth" for billable costs.
731
+ // Costs stored in *_cost_usd columns are ACTUAL BILLABLE amounts after:
732
+ // 1. Account-level allowance subtraction (PAID_ALLOWANCES in workers/lib/costs.ts)
733
+ // 2. Proportional distribution to projects (fair share based on raw cost proportion)
734
+ //
735
+ // Formula: projectBillableCost = projectRawCost * (accountBillableCost / accountRawCost)
736
+ //
737
+ // Pricing logic lives in:
738
+ // - workers/lib/costs.ts (calculateDailyBillableCosts, PRICING_TIERS, PAID_ALLOWANCES)
739
+ // - workers/lib/billing.ts (calculateBillingPeriod, proration utilities)
740
+ // ============================================================================
741
+
742
+ // Log overage warning if account is over allowance
743
+ if (accountBillableCosts.total > 0 && costScaleFactor > 0) {
744
+ log.warn(
745
+ `Account exceeds free tier: ` +
746
+ `billable=$${accountBillableCosts.total.toFixed(4)} ` +
747
+ `(workers=$${accountBillableCosts.workers.toFixed(4)}, ` +
748
+ `d1=$${accountBillableCosts.d1.toFixed(4)}, ` +
749
+ `kv=$${accountBillableCosts.kv.toFixed(4)}, ` +
750
+ `do=$${accountBillableCosts.durableObjects.toFixed(4)})`,
751
+ undefined,
752
+ { tag: 'BILLING' }
753
+ );
754
+ } else {
755
+ log.info(
756
+ `Account within free tier allowances ` +
757
+ `(day ${billingPeriod.daysElapsed}/${billingPeriod.daysInPeriod})`,
758
+ { tag: 'BILLING' }
759
+ );
760
+ }
761
+
762
+ log.info(
763
+ `Daily rollup complete: ${totalChanges} rows, billable=$${accountBillableCosts.total.toFixed(4)}`,
764
+ { tag: 'SCHEDULED' }
765
+ );
766
+ return totalChanges;
767
+ }
768
+
769
+ // =============================================================================
770
+ // FEATURE USAGE DAILY ROLLUP
771
+ // =============================================================================
772
+
773
+ /**
774
+ * Run feature-level daily rollup from Analytics Engine.
775
+ * Aggregates SDK telemetry from PLATFORM_ANALYTICS dataset into feature_usage_daily table.
776
+ * Called at midnight UTC for yesterday's data.
777
+ *
778
+ * This is the new "data tiering" approach where:
779
+ * - High-resolution telemetry is stored in Analytics Engine (SDK telemetry via queue)
780
+ * - Daily aggregates are stored in D1 (feature_usage_daily) for historical queries
781
+ *
782
+ * @param env - Worker environment
783
+ * @param date - Date to run rollup for (YYYY-MM-DD)
784
+ * @returns Number of rows inserted/updated
785
+ */
786
+ export async function runFeatureUsageDailyRollup(env: Env, date: string): Promise<number> {
787
+ const log = createLoggerFromEnv(env, 'platform-usage', 'platform:usage:scheduled');
788
+ log.info(`Running feature usage daily rollup for ${date}`, { tag: 'SCHEDULED' });
789
+
790
+ try {
791
+ // Query Analytics Engine for yesterday's SDK telemetry
792
+ const aggregations = await getDailyUsageFromAnalyticsEngine(
793
+ env.CLOUDFLARE_ACCOUNT_ID,
794
+ env.CLOUDFLARE_API_TOKEN,
795
+ 'platform-analytics'
796
+ );
797
+
798
+ if (aggregations.length === 0) {
799
+ log.info(`No SDK telemetry found in Analytics Engine for ${date}`, { tag: 'SCHEDULED' });
800
+ return 0;
801
+ }
802
+
803
+ log.info(`Found ${aggregations.length} feature aggregations from Analytics Engine`, {
804
+ tag: 'SCHEDULED',
805
+ });
806
+
807
+ let totalChanges = 0;
808
+
809
+ for (const agg of aggregations) {
810
+ // Insert or update feature_usage_daily
811
+ const result = await env.PLATFORM_DB.prepare(
812
+ `
813
+ INSERT INTO feature_usage_daily (
814
+ id, feature_key, usage_date,
815
+ d1_writes, d1_reads, kv_reads, kv_writes,
816
+ ai_neurons, requests
817
+ )
818
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
819
+ ON CONFLICT (feature_key, usage_date) DO UPDATE SET
820
+ d1_writes = excluded.d1_writes,
821
+ d1_reads = excluded.d1_reads,
822
+ kv_reads = excluded.kv_reads,
823
+ kv_writes = excluded.kv_writes,
824
+ ai_neurons = excluded.ai_neurons,
825
+ requests = excluded.requests
826
+ `
827
+ )
828
+ .bind(
829
+ generateId(),
830
+ agg.feature_id,
831
+ date,
832
+ agg.d1_writes + agg.d1_rows_written, // Combine write operations
833
+ agg.d1_reads + agg.d1_rows_read, // Combine read operations
834
+ agg.kv_reads,
835
+ agg.kv_writes + agg.kv_deletes + agg.kv_lists, // Combine write-like operations
836
+ agg.ai_neurons,
837
+ agg.interaction_count // Total telemetry events as requests
838
+ )
839
+ .run();
840
+
841
+ totalChanges += result.meta.changes || 0;
842
+ }
843
+
844
+ log.info('Feature usage daily rollup complete', {
845
+ tag: 'FEATURE_ROLLUP_COMPLETE',
846
+ totalChanges,
847
+ featureCount: aggregations.length,
848
+ });
849
+ return totalChanges;
850
+ } catch (error) {
851
+ const errorMessage = error instanceof Error ? error.message : String(error);
852
+ log.error('Feature usage daily rollup failed', error instanceof Error ? error : undefined, {
853
+ tag: 'FEATURE_ROLLUP_ERROR',
854
+ errorMessage,
855
+ });
856
+ // Don't throw - this is a non-critical operation
857
+ return 0;
858
+ }
859
+ }
860
+
861
+ // =============================================================================
862
+ // MONTHLY ROLLUP
863
+ // =============================================================================
864
+
865
+ /**
866
+ * Run monthly rollup: aggregate daily rollups into monthly_usage_rollups.
867
+ * Called on the 1st of each month for the previous month.
868
+ *
869
+ * @param env - Worker environment
870
+ * @param month - Month to run rollup for (YYYY-MM)
871
+ * @returns Number of rows changed
872
+ */
873
+ export async function runMonthlyRollup(env: Env, month: string): Promise<number> {
874
+ const log = createLoggerFromEnv(env, 'platform-usage', 'platform:usage:scheduled');
875
+ log.info('Running monthly rollup', { tag: 'MONTHLY_ROLLUP_START', month });
876
+
877
+ const result = await env.PLATFORM_DB.prepare(
878
+ `
879
+ INSERT INTO monthly_usage_rollups (
880
+ snapshot_month, project,
881
+ workers_requests, workers_errors, workers_cost_usd,
882
+ d1_rows_read, d1_rows_written, d1_cost_usd,
883
+ kv_reads, kv_writes, kv_cost_usd,
884
+ r2_class_a_ops, r2_class_b_ops, r2_egress_bytes, r2_cost_usd,
885
+ do_requests, do_gb_seconds, do_cost_usd,
886
+ aigateway_requests, aigateway_tokens_total, aigateway_cost_usd,
887
+ workersai_requests, workersai_neurons, workersai_cost_usd,
888
+ workflows_executions, workflows_failures, workflows_cpu_time_ms, workflows_cost_usd,
889
+ total_cost_usd, days_count, pricing_version_id
890
+ )
891
+ SELECT
892
+ SUBSTR(snapshot_date, 1, 7) as snapshot_month,
893
+ project,
894
+ SUM(workers_requests), SUM(workers_errors), SUM(workers_cost_usd),
895
+ SUM(d1_rows_read), SUM(d1_rows_written), SUM(d1_cost_usd),
896
+ SUM(kv_reads), SUM(kv_writes), SUM(kv_cost_usd),
897
+ SUM(r2_class_a_ops), SUM(r2_class_b_ops), SUM(r2_egress_bytes), SUM(r2_cost_usd),
898
+ SUM(do_requests), SUM(COALESCE(do_gb_seconds, 0)), SUM(do_cost_usd),
899
+ SUM(aigateway_requests), SUM(aigateway_tokens_in + aigateway_tokens_out), SUM(aigateway_cost_usd),
900
+ SUM(workersai_requests), SUM(workersai_neurons), SUM(workersai_cost_usd),
901
+ SUM(COALESCE(workflows_executions, 0)), SUM(COALESCE(workflows_failures, 0)),
902
+ SUM(COALESCE(workflows_cpu_time_ms, 0)), SUM(COALESCE(workflows_cost_usd, 0)),
903
+ SUM(total_cost_usd), COUNT(DISTINCT snapshot_date),
904
+ MAX(pricing_version_id) -- Use most recent pricing version from daily rollups
905
+ FROM daily_usage_rollups
906
+ WHERE SUBSTR(snapshot_date, 1, 7) = ?
907
+ GROUP BY SUBSTR(snapshot_date, 1, 7), project
908
+ ON CONFLICT (snapshot_month, project) DO UPDATE SET
909
+ workers_requests = excluded.workers_requests,
910
+ workers_errors = excluded.workers_errors,
911
+ workflows_executions = excluded.workflows_executions,
912
+ workflows_failures = excluded.workflows_failures,
913
+ workflows_cpu_time_ms = excluded.workflows_cpu_time_ms,
914
+ workflows_cost_usd = excluded.workflows_cost_usd,
915
+ total_cost_usd = excluded.total_cost_usd,
916
+ days_count = excluded.days_count,
917
+ pricing_version_id = excluded.pricing_version_id
918
+ `
919
+ )
920
+ .bind(month)
921
+ .run();
922
+
923
+ log.info('Monthly rollup complete', {
924
+ tag: 'MONTHLY_ROLLUP_COMPLETE',
925
+ rowsChanged: result.meta.changes,
926
+ month,
927
+ });
928
+ return result.meta.changes || 0;
929
+ }
930
+
931
+ // =============================================================================
932
+ // DATA CLEANUP
933
+ // =============================================================================
934
+
935
+ /**
936
+ * Clean up old data based on retention policies.
937
+ * - Hourly: 7 days
938
+ * - Daily: 90 days
939
+ * - Monthly: forever (no cleanup)
940
+ *
941
+ * @param env - Worker environment
942
+ * @returns Number of rows deleted by type
943
+ */
944
+ export async function cleanupOldData(
945
+ env: Env
946
+ ): Promise<{ hourlyDeleted: number; dailyDeleted: number }> {
947
+ // Delete hourly snapshots older than 7 days
948
+ const hourlyResult = await env.PLATFORM_DB.prepare(
949
+ `
950
+ DELETE FROM hourly_usage_snapshots
951
+ WHERE snapshot_hour < datetime('now', '-7 days')
952
+ `
953
+ ).run();
954
+
955
+ // Delete daily rollups older than 90 days
956
+ const dailyResult = await env.PLATFORM_DB.prepare(
957
+ `
958
+ DELETE FROM daily_usage_rollups
959
+ WHERE snapshot_date < date('now', '-90 days')
960
+ `
961
+ ).run();
962
+
963
+ const log = createLoggerFromEnv(env, 'platform-usage', 'platform:usage:scheduled');
964
+ log.info('Cleanup complete', {
965
+ tag: 'CLEANUP_COMPLETE',
966
+ hourlyDeleted: hourlyResult.meta.changes,
967
+ dailyDeleted: dailyResult.meta.changes,
968
+ });
969
+
970
+ return {
971
+ hourlyDeleted: hourlyResult.meta.changes || 0,
972
+ dailyDeleted: dailyResult.meta.changes || 0,
973
+ };
974
+ }
975
+
976
+ // =============================================================================
977
+ // USAGE VS ALLOWANCE PERCENTAGES
978
+ // =============================================================================
979
+
980
+ /**
981
+ * Calculate and persist usage vs allowance percentages.
982
+ * Computes month-to-date usage / monthly allowance * 100 for Cloudflare resources.
983
+ * Stores as new resource_type entries (e.g., workers_requests_usage_pct).
984
+ *
985
+ * @param env - Worker environment
986
+ * @param date - Date (YYYY-MM-DD) for the calculation
987
+ * @returns Number of D1 writes performed
988
+ */
989
+ export async function calculateUsageVsAllowancePercentages(
990
+ env: Env,
991
+ date: string
992
+ ): Promise<number> {
993
+ const log = createLoggerFromEnv(env, 'platform-usage', 'platform:usage:allowance');
994
+ log.info('Calculating usage vs allowance percentages', { date });
995
+
996
+ // Get current month (YYYY-MM)
997
+ const currentMonth = date.slice(0, 7);
998
+
999
+ // Mapping of usage metrics to their inclusion counterparts
1000
+ const usageToInclusionMap: Array<{
1001
+ usageColumn: string;
1002
+ inclusionType: string;
1003
+ pctType: string;
1004
+ }> = [
1005
+ {
1006
+ usageColumn: 'workers_requests',
1007
+ inclusionType: 'workers_requests_included',
1008
+ pctType: 'workers_requests_usage_pct',
1009
+ },
1010
+ {
1011
+ usageColumn: 'd1_rows_read',
1012
+ inclusionType: 'd1_rows_read_included',
1013
+ pctType: 'd1_rows_read_usage_pct',
1014
+ },
1015
+ {
1016
+ usageColumn: 'd1_rows_written',
1017
+ inclusionType: 'd1_rows_written_included',
1018
+ pctType: 'd1_rows_written_usage_pct',
1019
+ },
1020
+ {
1021
+ usageColumn: 'kv_reads',
1022
+ inclusionType: 'kv_reads_included',
1023
+ pctType: 'kv_reads_usage_pct',
1024
+ },
1025
+ {
1026
+ usageColumn: 'kv_writes',
1027
+ inclusionType: 'kv_writes_included',
1028
+ pctType: 'kv_writes_usage_pct',
1029
+ },
1030
+ {
1031
+ usageColumn: 'r2_class_a_ops',
1032
+ inclusionType: 'r2_class_a_included',
1033
+ pctType: 'r2_class_a_usage_pct',
1034
+ },
1035
+ {
1036
+ usageColumn: 'r2_class_b_ops',
1037
+ inclusionType: 'r2_class_b_included',
1038
+ pctType: 'r2_class_b_usage_pct',
1039
+ },
1040
+ {
1041
+ usageColumn: 'do_requests',
1042
+ inclusionType: 'do_requests_included',
1043
+ pctType: 'do_requests_usage_pct',
1044
+ },
1045
+ {
1046
+ usageColumn: 'workersai_neurons',
1047
+ inclusionType: 'workers_ai_neurons_included',
1048
+ pctType: 'workers_ai_neurons_usage_pct',
1049
+ },
1050
+ ];
1051
+
1052
+ let d1Writes = 0;
1053
+
1054
+ // Get month-to-date usage totals from daily_usage_rollups (sum across all projects)
1055
+ const mtdUsageResult = await env.PLATFORM_DB.prepare(
1056
+ `
1057
+ SELECT
1058
+ SUM(workers_requests) as workers_requests,
1059
+ SUM(d1_rows_read) as d1_rows_read,
1060
+ SUM(d1_rows_written) as d1_rows_written,
1061
+ SUM(kv_reads) as kv_reads,
1062
+ SUM(kv_writes) as kv_writes,
1063
+ SUM(r2_class_a_ops) as r2_class_a_ops,
1064
+ SUM(r2_class_b_ops) as r2_class_b_ops,
1065
+ SUM(do_requests) as do_requests,
1066
+ SUM(workersai_neurons) as workersai_neurons
1067
+ FROM daily_usage_rollups
1068
+ WHERE snapshot_date LIKE ? || '%'
1069
+ `
1070
+ )
1071
+ .bind(currentMonth)
1072
+ .first<Record<string, number | null>>();
1073
+
1074
+ if (!mtdUsageResult) {
1075
+ log.info('No usage data found for month', { month: currentMonth });
1076
+ return 0;
1077
+ }
1078
+
1079
+ // Get inclusions from third_party_usage (latest for Cloudflare)
1080
+ const inclusionsResult = await env.PLATFORM_DB.prepare(
1081
+ `
1082
+ SELECT resource_type, usage_value
1083
+ FROM third_party_usage
1084
+ WHERE provider = 'cloudflare'
1085
+ AND resource_type LIKE '%_included'
1086
+ AND snapshot_date = (
1087
+ SELECT MAX(snapshot_date) FROM third_party_usage
1088
+ WHERE provider = 'cloudflare' AND resource_type LIKE '%_included'
1089
+ )
1090
+ `
1091
+ ).all<{ resource_type: string; usage_value: number }>();
1092
+
1093
+ if (!inclusionsResult.results || inclusionsResult.results.length === 0) {
1094
+ log.info('No inclusions data found for Cloudflare');
1095
+ return 0;
1096
+ }
1097
+
1098
+ // Create a map of inclusion types to values
1099
+ const inclusionsMap = new Map<string, number>();
1100
+ for (const row of inclusionsResult.results) {
1101
+ inclusionsMap.set(row.resource_type, row.usage_value);
1102
+ }
1103
+
1104
+ // Calculate and persist percentages
1105
+ for (const mapping of usageToInclusionMap) {
1106
+ const usage = mtdUsageResult[mapping.usageColumn] || 0;
1107
+ const inclusion = inclusionsMap.get(mapping.inclusionType);
1108
+
1109
+ if (inclusion === undefined || inclusion === 0) {
1110
+ // Skip if no inclusion value (avoid division by zero)
1111
+ continue;
1112
+ }
1113
+
1114
+ const percentage = (usage / inclusion) * 100;
1115
+
1116
+ await persistThirdPartyUsage(
1117
+ env,
1118
+ date,
1119
+ 'cloudflare',
1120
+ mapping.pctType,
1121
+ Math.round(percentage * 100) / 100, // Round to 2 decimal places
1122
+ 'percent',
1123
+ 0
1124
+ );
1125
+ d1Writes++;
1126
+ }
1127
+
1128
+ // GitHub Actions: get MTD minutes and compare to included
1129
+ const githubMtdResult = await env.PLATFORM_DB.prepare(
1130
+ `
1131
+ SELECT SUM(usage_value) as mtd_minutes
1132
+ FROM third_party_usage
1133
+ WHERE provider = 'github'
1134
+ AND resource_type = 'actions_minutes'
1135
+ AND snapshot_date LIKE ? || '%'
1136
+ `
1137
+ )
1138
+ .bind(currentMonth)
1139
+ .first<{ mtd_minutes: number | null }>();
1140
+
1141
+ // Get GitHub inclusions separately
1142
+ const githubInclusionsResult = await env.PLATFORM_DB.prepare(
1143
+ `
1144
+ SELECT resource_type, usage_value
1145
+ FROM third_party_usage
1146
+ WHERE provider = 'github'
1147
+ AND resource_type LIKE '%_included'
1148
+ AND snapshot_date = (
1149
+ SELECT MAX(snapshot_date) FROM third_party_usage
1150
+ WHERE provider = 'github' AND resource_type LIKE '%_included'
1151
+ )
1152
+ `
1153
+ ).all<{ resource_type: string; usage_value: number }>();
1154
+
1155
+ if (
1156
+ githubMtdResult?.mtd_minutes &&
1157
+ githubInclusionsResult.results &&
1158
+ githubInclusionsResult.results.length > 0
1159
+ ) {
1160
+ const actionsIncluded = githubInclusionsResult.results.find(
1161
+ (r) => r.resource_type === 'actions_minutes_included'
1162
+ );
1163
+ if (actionsIncluded && actionsIncluded.usage_value > 0) {
1164
+ const pct = (githubMtdResult.mtd_minutes / actionsIncluded.usage_value) * 100;
1165
+ await persistThirdPartyUsage(
1166
+ env,
1167
+ date,
1168
+ 'github',
1169
+ 'actions_minutes_usage_pct',
1170
+ Math.round(pct * 100) / 100,
1171
+ 'percent',
1172
+ 0
1173
+ );
1174
+ d1Writes++;
1175
+ }
1176
+ }
1177
+
1178
+ log.info('Calculated usage vs allowance percentages', { d1Writes });
1179
+ return d1Writes;
1180
+ }
1181
+
1182
+ /**
1183
+ * Persist third-party usage data (GitHub billing, etc.).
1184
+ */
1185
+ async function persistThirdPartyUsage(
1186
+ env: Env,
1187
+ date: string,
1188
+ provider: string,
1189
+ resourceType: string,
1190
+ usageValue: number,
1191
+ usageUnit: string,
1192
+ costUsd: number = 0,
1193
+ resourceName?: string
1194
+ ): Promise<void> {
1195
+ await env.PLATFORM_DB.prepare(
1196
+ `
1197
+ INSERT INTO third_party_usage (
1198
+ id, snapshot_date, provider, resource_type, resource_name,
1199
+ usage_value, usage_unit, cost_usd, collection_timestamp
1200
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
1201
+ ON CONFLICT (snapshot_date, provider, resource_type, COALESCE(resource_name, ''))
1202
+ DO UPDATE SET
1203
+ usage_value = excluded.usage_value,
1204
+ cost_usd = excluded.cost_usd,
1205
+ collection_timestamp = excluded.collection_timestamp
1206
+ `
1207
+ )
1208
+ .bind(
1209
+ generateId(),
1210
+ date,
1211
+ provider,
1212
+ resourceType,
1213
+ resourceName || '',
1214
+ usageValue,
1215
+ usageUnit,
1216
+ costUsd,
1217
+ Math.floor(Date.now() / 1000)
1218
+ )
1219
+ .run();
1220
+ }
1221
+
1222
+ // =============================================================================
1223
+ // AI MODEL BREAKDOWN PERSISTENCE
1224
+ // =============================================================================
1225
+
1226
+ /**
1227
+ * Persist Workers AI model breakdown data to D1.
1228
+ * Stores per-project, per-model usage for historical analysis.
1229
+ *
1230
+ * @param env - Worker environment
1231
+ * @param snapshotHour - Hour of the snapshot (ISO format)
1232
+ * @param metrics - Array of model usage metrics
1233
+ * @returns Number of D1 writes performed
1234
+ */
1235
+ export async function persistWorkersAIModelBreakdown(
1236
+ env: Env,
1237
+ snapshotHour: string,
1238
+ metrics: Array<{
1239
+ project: string;
1240
+ model: string;
1241
+ requests: number;
1242
+ inputTokens: number;
1243
+ outputTokens: number;
1244
+ costUsd: number;
1245
+ isEstimated: boolean;
1246
+ }>
1247
+ ): Promise<number> {
1248
+ let writes = 0;
1249
+ for (const m of metrics) {
1250
+ await env.PLATFORM_DB.prepare(
1251
+ `
1252
+ INSERT INTO workersai_model_usage (
1253
+ id, snapshot_hour, project, model, requests,
1254
+ input_tokens, output_tokens, cost_usd, is_estimated
1255
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
1256
+ ON CONFLICT (snapshot_hour, project, model) DO UPDATE SET
1257
+ requests = excluded.requests,
1258
+ input_tokens = excluded.input_tokens,
1259
+ output_tokens = excluded.output_tokens,
1260
+ cost_usd = excluded.cost_usd,
1261
+ is_estimated = excluded.is_estimated
1262
+ `
1263
+ )
1264
+ .bind(
1265
+ generateId(),
1266
+ snapshotHour,
1267
+ m.project,
1268
+ m.model,
1269
+ m.requests,
1270
+ m.inputTokens,
1271
+ m.outputTokens,
1272
+ m.costUsd,
1273
+ m.isEstimated ? 1 : 0
1274
+ )
1275
+ .run();
1276
+ writes++;
1277
+ }
1278
+ return writes;
1279
+ }
1280
+
1281
+ /**
1282
+ * Persist AI Gateway model breakdown data to D1.
1283
+ * Stores per-gateway, per-provider, per-model usage for historical analysis.
1284
+ *
1285
+ * @param env - Worker environment
1286
+ * @param snapshotHour - Hour of the snapshot (ISO format)
1287
+ * @param gatewayId - AI Gateway ID
1288
+ * @param models - Array of model usage metrics
1289
+ * @returns Number of D1 writes performed
1290
+ */
1291
+ export async function persistAIGatewayModelBreakdown(
1292
+ env: Env,
1293
+ snapshotHour: string,
1294
+ gatewayId: string,
1295
+ models: Array<{
1296
+ provider: string;
1297
+ model: string;
1298
+ requests: number;
1299
+ cachedRequests: number;
1300
+ tokensIn: number;
1301
+ tokensOut: number;
1302
+ costUsd: number;
1303
+ }>
1304
+ ): Promise<number> {
1305
+ let writes = 0;
1306
+ for (const m of models) {
1307
+ await env.PLATFORM_DB.prepare(
1308
+ `
1309
+ INSERT INTO aigateway_model_usage (
1310
+ id, snapshot_hour, gateway_id, provider, model,
1311
+ requests, cached_requests, tokens_in, tokens_out, cost_usd
1312
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1313
+ ON CONFLICT (snapshot_hour, gateway_id, provider, model) DO UPDATE SET
1314
+ requests = excluded.requests,
1315
+ cached_requests = excluded.cached_requests,
1316
+ tokens_in = excluded.tokens_in,
1317
+ tokens_out = excluded.tokens_out,
1318
+ cost_usd = excluded.cost_usd
1319
+ `
1320
+ )
1321
+ .bind(
1322
+ generateId(),
1323
+ snapshotHour,
1324
+ gatewayId,
1325
+ m.provider,
1326
+ m.model,
1327
+ m.requests,
1328
+ m.cachedRequests,
1329
+ m.tokensIn,
1330
+ m.tokensOut,
1331
+ m.costUsd
1332
+ )
1333
+ .run();
1334
+ writes++;
1335
+ }
1336
+ return writes;
1337
+ }
1338
+
1339
+ /**
1340
+ * Persist feature-level AI model usage to D1.
1341
+ * Called from queue consumer when telemetry includes aiModelBreakdown.
1342
+ * Uses upsert to aggregate invocations for the same feature/model/date.
1343
+ *
1344
+ * @param env - Worker environment
1345
+ * @param featureKey - Feature key (project:category:feature)
1346
+ * @param modelBreakdown - Map of model name to invocation count
1347
+ * @param timestamp - Timestamp of the telemetry
1348
+ * @returns Number of D1 writes performed
1349
+ */
1350
+ export async function persistFeatureAIModelUsage(
1351
+ env: Env,
1352
+ featureKey: string,
1353
+ modelBreakdown: Record<string, number>,
1354
+ timestamp: Date
1355
+ ): Promise<number> {
1356
+ const usageDate = timestamp.toISOString().split('T')[0]; // YYYY-MM-DD
1357
+ let writes = 0;
1358
+
1359
+ for (const [model, invocations] of Object.entries(modelBreakdown)) {
1360
+ if (invocations <= 0) continue;
1361
+
1362
+ await env.PLATFORM_DB.prepare(
1363
+ `
1364
+ INSERT INTO feature_ai_model_usage (
1365
+ id, feature_key, model, usage_date, invocations, updated_at
1366
+ ) VALUES (?, ?, ?, ?, ?, unixepoch())
1367
+ ON CONFLICT (feature_key, model, usage_date) DO UPDATE SET
1368
+ invocations = invocations + excluded.invocations,
1369
+ updated_at = unixepoch()
1370
+ `
1371
+ )
1372
+ .bind(generateId(), featureKey, model, usageDate, invocations)
1373
+ .run();
1374
+ writes++;
1375
+ }
1376
+
1377
+ return writes;
1378
+ }
1379
+
1380
+ // =============================================================================
1381
+ // AI MODEL DAILY ROLLUPS
1382
+ // =============================================================================
1383
+
1384
+ /**
1385
+ * Run daily rollup for Workers AI model usage.
1386
+ * Aggregates hourly data into daily totals.
1387
+ *
1388
+ * @param env - Worker environment
1389
+ * @param date - Date to run rollup for (YYYY-MM-DD)
1390
+ * @returns Number of rows changed
1391
+ */
1392
+ export async function runWorkersAIModelDailyRollup(env: Env, date: string): Promise<number> {
1393
+ const startHour = `${date}T00:00:00Z`;
1394
+ const endHour = `${date}T23:59:59Z`;
1395
+
1396
+ const result = await env.PLATFORM_DB.prepare(
1397
+ `
1398
+ INSERT INTO workersai_model_daily (
1399
+ snapshot_date, project, model, requests, input_tokens, output_tokens, cost_usd, samples_count
1400
+ )
1401
+ SELECT
1402
+ ? as snapshot_date,
1403
+ project,
1404
+ model,
1405
+ COALESCE(SUM(requests), 0),
1406
+ COALESCE(SUM(input_tokens), 0),
1407
+ COALESCE(SUM(output_tokens), 0),
1408
+ COALESCE(SUM(cost_usd), 0),
1409
+ COUNT(*)
1410
+ FROM workersai_model_usage
1411
+ WHERE snapshot_hour >= ? AND snapshot_hour <= ?
1412
+ GROUP BY project, model
1413
+ ON CONFLICT (snapshot_date, project, model) DO UPDATE SET
1414
+ requests = excluded.requests,
1415
+ input_tokens = excluded.input_tokens,
1416
+ output_tokens = excluded.output_tokens,
1417
+ cost_usd = excluded.cost_usd,
1418
+ samples_count = excluded.samples_count
1419
+ `
1420
+ )
1421
+ .bind(date, startHour, endHour)
1422
+ .run();
1423
+
1424
+ return result.meta.changes || 0;
1425
+ }
1426
+
1427
+ /**
1428
+ * Run daily rollup for AI Gateway model usage.
1429
+ * Aggregates hourly data into daily totals.
1430
+ *
1431
+ * @param env - Worker environment
1432
+ * @param date - Date to run rollup for (YYYY-MM-DD)
1433
+ * @returns Number of rows changed
1434
+ */
1435
+ export async function runAIGatewayModelDailyRollup(env: Env, date: string): Promise<number> {
1436
+ const startHour = `${date}T00:00:00Z`;
1437
+ const endHour = `${date}T23:59:59Z`;
1438
+
1439
+ const result = await env.PLATFORM_DB.prepare(
1440
+ `
1441
+ INSERT INTO aigateway_model_daily (
1442
+ snapshot_date, gateway_id, provider, model, requests, cached_requests, tokens_in, tokens_out, cost_usd, samples_count
1443
+ )
1444
+ SELECT
1445
+ ? as snapshot_date,
1446
+ gateway_id,
1447
+ provider,
1448
+ model,
1449
+ COALESCE(SUM(requests), 0),
1450
+ COALESCE(SUM(cached_requests), 0),
1451
+ COALESCE(SUM(tokens_in), 0),
1452
+ COALESCE(SUM(tokens_out), 0),
1453
+ COALESCE(SUM(cost_usd), 0),
1454
+ COUNT(*)
1455
+ FROM aigateway_model_usage
1456
+ WHERE snapshot_hour >= ? AND snapshot_hour <= ?
1457
+ GROUP BY gateway_id, provider, model
1458
+ ON CONFLICT (snapshot_date, gateway_id, provider, model) DO UPDATE SET
1459
+ requests = excluded.requests,
1460
+ cached_requests = excluded.cached_requests,
1461
+ tokens_in = excluded.tokens_in,
1462
+ tokens_out = excluded.tokens_out,
1463
+ cost_usd = excluded.cost_usd,
1464
+ samples_count = excluded.samples_count
1465
+ `
1466
+ )
1467
+ .bind(date, startHour, endHour)
1468
+ .run();
1469
+
1470
+ return result.meta.changes || 0;
1471
+ }
1472
+
1473
+ // =============================================================================
1474
+ // GAP-FILLING: Self-healing daily rollups from hourly data
1475
+ // =============================================================================
1476
+
1477
+ /**
1478
+ * Finds gaps in daily_usage_rollups where storage metrics are 0 but hourly data exists.
1479
+ * Re-runs runDailyRollup for those days to fix the data.
1480
+ *
1481
+ * This is a self-healing mechanism that runs at midnight after the regular daily rollup.
1482
+ * It addresses the scenario where the backfill script overwrote good data with incomplete data.
1483
+ *
1484
+ * Limits to MAX_DAYS_PER_RUN days per cron run to stay within CPU budget.
1485
+ *
1486
+ * @param env - Worker environment
1487
+ * @returns Number of days fixed
1488
+ */
1489
+ export async function backfillMissingDays(env: Env): Promise<number> {
1490
+ const log = createLoggerFromEnv(env, 'platform-usage', 'platform:usage:gap-fill');
1491
+ const MAX_DAYS_PER_RUN = 3; // Stay within CPU budget
1492
+ const LOOKBACK_DAYS = 30; // Check last 30 days
1493
+
1494
+ log.info('Starting gap detection', { lookbackDays: LOOKBACK_DAYS });
1495
+
1496
+ // Find days where daily_usage_rollups has 0 storage but hourly_usage_snapshots has data
1497
+ // This indicates the backfill script overwrote good rollup data with incomplete data
1498
+ interface GapRow {
1499
+ snapshot_date: string;
1500
+ daily_storage: number | null;
1501
+ hourly_storage: number | null;
1502
+ }
1503
+
1504
+ const gapQuery = await env.PLATFORM_DB.prepare(
1505
+ `
1506
+ WITH daily_storage AS (
1507
+ SELECT snapshot_date, MAX(d1_storage_bytes_max) as storage
1508
+ FROM daily_usage_rollups
1509
+ WHERE project = 'all'
1510
+ AND snapshot_date >= date('now', '-${LOOKBACK_DAYS} days')
1511
+ GROUP BY snapshot_date
1512
+ ),
1513
+ hourly_storage AS (
1514
+ SELECT DATE(snapshot_hour) as snapshot_date, MAX(d1_storage_bytes) as storage
1515
+ FROM hourly_usage_snapshots
1516
+ WHERE project = 'all'
1517
+ AND snapshot_hour >= datetime('now', '-${LOOKBACK_DAYS} days')
1518
+ GROUP BY DATE(snapshot_hour)
1519
+ )
1520
+ SELECT
1521
+ h.snapshot_date,
1522
+ d.storage as daily_storage,
1523
+ h.storage as hourly_storage
1524
+ FROM hourly_storage h
1525
+ LEFT JOIN daily_storage d ON h.snapshot_date = d.snapshot_date
1526
+ WHERE (d.storage IS NULL OR d.storage = 0)
1527
+ AND h.storage > 0
1528
+ ORDER BY h.snapshot_date DESC
1529
+ LIMIT ?
1530
+ `
1531
+ )
1532
+ .bind(MAX_DAYS_PER_RUN)
1533
+ .all<GapRow>();
1534
+
1535
+ if (!gapQuery.results || gapQuery.results.length === 0) {
1536
+ log.info('No gaps found - all daily rollups have correct storage data');
1537
+ return 0;
1538
+ }
1539
+
1540
+ log.info('Found days needing fix', { count: gapQuery.results.length });
1541
+
1542
+ let fixedCount = 0;
1543
+ for (const gap of gapQuery.results) {
1544
+ log.info('Fixing gap', {
1545
+ date: gap.snapshot_date,
1546
+ dailyStorage: gap.daily_storage ?? 0,
1547
+ hourlyStorage: gap.hourly_storage,
1548
+ });
1549
+ try {
1550
+ await runDailyRollup(env, gap.snapshot_date);
1551
+ fixedCount++;
1552
+ log.info('Fixed gap', { date: gap.snapshot_date });
1553
+ } catch (error) {
1554
+ const errorMsg = error instanceof Error ? error.message : String(error);
1555
+ log.error(`Error fixing ${gap.snapshot_date}: ${errorMsg}`);
1556
+ }
1557
+ }
1558
+
1559
+ log.info('Gap-fill complete', { fixed: fixedCount, total: gapQuery.results.length });
1560
+ return fixedCount;
1561
+ }