@littlebearapps/platform-admin-sdk 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/README.md +112 -0
  2. package/dist/index.d.ts +16 -0
  3. package/dist/index.js +89 -0
  4. package/dist/prompts.d.ts +27 -0
  5. package/dist/prompts.js +80 -0
  6. package/dist/scaffold.d.ts +5 -0
  7. package/dist/scaffold.js +65 -0
  8. package/dist/templates.d.ts +16 -0
  9. package/dist/templates.js +131 -0
  10. package/package.json +46 -0
  11. package/templates/full/migrations/006_pattern_discovery.sql +199 -0
  12. package/templates/full/migrations/007_notifications_search.sql +127 -0
  13. package/templates/full/workers/lib/pattern-discovery/ai-prompt.ts +644 -0
  14. package/templates/full/workers/lib/pattern-discovery/clustering.ts +278 -0
  15. package/templates/full/workers/lib/pattern-discovery/shadow-evaluation.ts +603 -0
  16. package/templates/full/workers/lib/pattern-discovery/storage.ts +806 -0
  17. package/templates/full/workers/lib/pattern-discovery/types.ts +159 -0
  18. package/templates/full/workers/lib/pattern-discovery/validation.ts +278 -0
  19. package/templates/full/workers/pattern-discovery.ts +661 -0
  20. package/templates/full/workers/platform-alert-router.ts +1809 -0
  21. package/templates/full/workers/platform-notifications.ts +424 -0
  22. package/templates/full/workers/platform-search.ts +480 -0
  23. package/templates/full/workers/platform-settings.ts +436 -0
  24. package/templates/full/wrangler.alert-router.jsonc.hbs +34 -0
  25. package/templates/full/wrangler.notifications.jsonc.hbs +23 -0
  26. package/templates/full/wrangler.pattern-discovery.jsonc.hbs +33 -0
  27. package/templates/full/wrangler.search.jsonc.hbs +16 -0
  28. package/templates/full/wrangler.settings.jsonc.hbs +23 -0
  29. package/templates/shared/README.md.hbs +69 -0
  30. package/templates/shared/config/budgets.yaml.hbs +72 -0
  31. package/templates/shared/config/services.yaml.hbs +45 -0
  32. package/templates/shared/migrations/001_core_tables.sql +117 -0
  33. package/templates/shared/migrations/002_usage_warehouse.sql +830 -0
  34. package/templates/shared/migrations/003_feature_tracking.sql +250 -0
  35. package/templates/shared/migrations/004_settings_alerts.sql +452 -0
  36. package/templates/shared/migrations/seed.sql.hbs +4 -0
  37. package/templates/shared/package.json.hbs +21 -0
  38. package/templates/shared/scripts/sync-config.ts +242 -0
  39. package/templates/shared/tsconfig.json +12 -0
  40. package/templates/shared/workers/lib/analytics-engine.ts +357 -0
  41. package/templates/shared/workers/lib/billing.ts +293 -0
  42. package/templates/shared/workers/lib/circuit-breaker-middleware.ts +25 -0
  43. package/templates/shared/workers/lib/control.ts +292 -0
  44. package/templates/shared/workers/lib/economics.ts +368 -0
  45. package/templates/shared/workers/lib/metrics.ts +103 -0
  46. package/templates/shared/workers/lib/platform-settings.ts +407 -0
  47. package/templates/shared/workers/lib/shared/allowances.ts +333 -0
  48. package/templates/shared/workers/lib/shared/cloudflare.ts +1362 -0
  49. package/templates/shared/workers/lib/shared/types.ts +58 -0
  50. package/templates/shared/workers/lib/telemetry-sampling.ts +360 -0
  51. package/templates/shared/workers/lib/usage/collectors/example.ts +96 -0
  52. package/templates/shared/workers/lib/usage/collectors/index.ts +128 -0
  53. package/templates/shared/workers/lib/usage/handlers/audit.ts +306 -0
  54. package/templates/shared/workers/lib/usage/handlers/backfill.ts +845 -0
  55. package/templates/shared/workers/lib/usage/handlers/behavioral.ts +429 -0
  56. package/templates/shared/workers/lib/usage/handlers/data-queries.ts +507 -0
  57. package/templates/shared/workers/lib/usage/handlers/dlq-admin.ts +364 -0
  58. package/templates/shared/workers/lib/usage/handlers/health-trends.ts +222 -0
  59. package/templates/shared/workers/lib/usage/handlers/index.ts +35 -0
  60. package/templates/shared/workers/lib/usage/handlers/usage-admin.ts +421 -0
  61. package/templates/shared/workers/lib/usage/handlers/usage-features.ts +1262 -0
  62. package/templates/shared/workers/lib/usage/handlers/usage-metrics.ts +2420 -0
  63. package/templates/shared/workers/lib/usage/handlers/usage-settings.ts +610 -0
  64. package/templates/shared/workers/lib/usage/queue/budget-enforcement.ts +1032 -0
  65. package/templates/shared/workers/lib/usage/queue/cost-budget-enforcement.ts +128 -0
  66. package/templates/shared/workers/lib/usage/queue/cost-calculator.ts +77 -0
  67. package/templates/shared/workers/lib/usage/queue/dlq-handler.ts +161 -0
  68. package/templates/shared/workers/lib/usage/queue/index.ts +19 -0
  69. package/templates/shared/workers/lib/usage/queue/telemetry-processor.ts +790 -0
  70. package/templates/shared/workers/lib/usage/scheduled/anomaly-detection.ts +732 -0
  71. package/templates/shared/workers/lib/usage/scheduled/data-collection.ts +956 -0
  72. package/templates/shared/workers/lib/usage/scheduled/error-digest.ts +343 -0
  73. package/templates/shared/workers/lib/usage/scheduled/index.ts +18 -0
  74. package/templates/shared/workers/lib/usage/scheduled/rollups.ts +1561 -0
  75. package/templates/shared/workers/lib/usage/shared/constants.ts +362 -0
  76. package/templates/shared/workers/lib/usage/shared/index.ts +14 -0
  77. package/templates/shared/workers/lib/usage/shared/types.ts +1066 -0
  78. package/templates/shared/workers/lib/usage/shared/utils.ts +795 -0
  79. package/templates/shared/workers/platform-usage.ts +1915 -0
  80. package/templates/shared/wrangler.usage.jsonc.hbs +58 -0
  81. package/templates/standard/migrations/005_error_collection.sql +162 -0
  82. package/templates/standard/workers/error-collector.ts +2670 -0
  83. package/templates/standard/workers/lib/error-collector/capture.ts +213 -0
  84. package/templates/standard/workers/lib/error-collector/digest.ts +448 -0
  85. package/templates/standard/workers/lib/error-collector/email-health-alerts.ts +262 -0
  86. package/templates/standard/workers/lib/error-collector/fingerprint.ts +258 -0
  87. package/templates/standard/workers/lib/error-collector/gap-alerts.ts +293 -0
  88. package/templates/standard/workers/lib/error-collector/github.ts +329 -0
  89. package/templates/standard/workers/lib/error-collector/types.ts +262 -0
  90. package/templates/standard/workers/lib/sentinel/gap-detection.ts +734 -0
  91. package/templates/standard/workers/lib/shared/slack-alerts.ts +585 -0
  92. package/templates/standard/workers/platform-sentinel.ts +1744 -0
  93. package/templates/standard/wrangler.error-collector.jsonc.hbs +44 -0
  94. package/templates/standard/wrangler.sentinel.jsonc.hbs +45 -0
@@ -0,0 +1,2670 @@
1
+ /**
2
+ * Error Collector Tail Worker
3
+ *
4
+ * Captures errors from Cloudflare Workers across your projects and creates
5
+ * AI-agent-ready GitHub issues for investigation.
6
+ *
7
+ * @see docs/plans/2026-01-28-error-collector-tail-worker-design.md
8
+ */
9
+
10
+ import type {
11
+ TailEvent,
12
+ Env,
13
+ ScriptMapping,
14
+ ErrorType,
15
+ ErrorStatus,
16
+ GitHubIssueType,
17
+ } from './lib/error-collector/types';
18
+ import {
19
+ shouldCapture,
20
+ calculatePriority,
21
+ getLabels,
22
+ formatErrorTitle,
23
+ extractCoreMessage,
24
+ normalizeUrl,
25
+ } from './lib/error-collector/capture';
26
+ import {
27
+ computeFingerprint,
28
+ generateId,
29
+ normalizeDynamicValues,
30
+ isTransientError,
31
+ classifyError,
32
+ classifyErrorWithSource,
33
+ loadDynamicPatterns,
34
+ type FingerprintResult,
35
+ type CompiledPattern,
36
+ } from './lib/error-collector/fingerprint';
37
+ import { GitHubClient } from './lib/error-collector/github';
38
+ import { processWarningDigests, storeWarningForDigest } from './lib/error-collector/digest';
39
+ import { processGapAlert } from './lib/error-collector/gap-alerts';
40
+ import { processEmailHealthAlerts } from './lib/error-collector/email-health-alerts';
41
+ import { recordPatternMatchEvidence } from './lib/pattern-discovery/storage';
42
+ import type { GapAlertEvent, EmailHealthAlertEvent } from './lib/error-collector/types';
43
+ import { pingHeartbeat } from '@littlebearapps/platform-consumer-sdk';
44
+
45
+ // TODO: Set your GitHub organisation name
46
+ const GITHUB_ORG = 'your-github-org';
47
+
48
+ // Rate limit: max issues per script per hour
49
+ const MAX_ISSUES_PER_SCRIPT_PER_HOUR = 10;
50
+
51
+ /**
52
+ * Map error type to GitHub issue type
53
+ * - Exceptions, CPU/memory limits, soft errors → Bug
54
+ * - Warnings → Task
55
+ */
56
+ function getGitHubIssueType(errorType: ErrorType): GitHubIssueType {
57
+ if (errorType === 'warning') {
58
+ return 'Task';
59
+ }
60
+ return 'Bug';
61
+ }
62
+
63
+ /**
64
+ * Look up script mapping from KV
65
+ */
66
+ async function getScriptMapping(
67
+ kv: KVNamespace,
68
+ scriptName: string
69
+ ): Promise<ScriptMapping | null> {
70
+ const key = `SCRIPT_MAP:${scriptName}`;
71
+ const value = await kv.get(key);
72
+
73
+ if (!value) return null;
74
+
75
+ try {
76
+ return JSON.parse(value) as ScriptMapping;
77
+ } catch {
78
+ console.error(`Invalid script mapping for ${scriptName}`);
79
+ return null;
80
+ }
81
+ }
82
+
83
+ /**
84
+ * Check and update rate limit
85
+ * Returns true if within limits, false if rate limited
86
+ */
87
+ async function checkRateLimit(kv: KVNamespace, scriptName: string): Promise<boolean> {
88
+ const hour = Math.floor(Date.now() / (1000 * 60 * 60));
89
+ const key = `ERROR_RATE:${scriptName}:${hour}`;
90
+
91
+ const current = await kv.get(key);
92
+ const count = current ? parseInt(current, 10) : 0;
93
+
94
+ if (count >= MAX_ISSUES_PER_SCRIPT_PER_HOUR) {
95
+ return false;
96
+ }
97
+
98
+ await kv.put(key, String(count + 1), { expirationTtl: 7200 }); // 2 hour TTL
99
+ return true;
100
+ }
101
+
102
+ /**
103
+ * Get today's date key in YYYY-MM-DD format (UTC)
104
+ */
105
+ function getDateKey(): string {
106
+ return new Date().toISOString().slice(0, 10);
107
+ }
108
+
109
+ /**
110
+ * Check if a transient error already has an issue created today.
111
+ * For transient errors (quota exhaustion, rate limits, etc.), we only
112
+ * create one issue per 24-hour window to avoid noise.
113
+ *
114
+ * Returns the existing issue number if one exists, null otherwise.
115
+ */
116
+ async function checkTransientErrorWindow(
117
+ kv: KVNamespace,
118
+ scriptName: string,
119
+ category: string
120
+ ): Promise<number | null> {
121
+ const windowKey = `TRANSIENT:${scriptName}:${category}:${getDateKey()}`;
122
+ const existing = await kv.get(windowKey);
123
+ return existing ? parseInt(existing, 10) : null;
124
+ }
125
+
126
+ /**
127
+ * Record that a transient error issue was created for today's window.
128
+ */
129
+ async function setTransientErrorWindow(
130
+ kv: KVNamespace,
131
+ scriptName: string,
132
+ category: string,
133
+ issueNumber: number
134
+ ): Promise<void> {
135
+ const windowKey = `TRANSIENT:${scriptName}:${category}:${getDateKey()}`;
136
+ // TTL of 25 hours to cover the full day plus buffer
137
+ await kv.put(windowKey, String(issueNumber), { expirationTtl: 90000 });
138
+ }
139
+
140
+ /**
141
+ * Check if an issue is muted via the cf:muted label.
142
+ * Muted issues should not be reopened or receive comments.
143
+ */
144
+ async function isIssueMuted(
145
+ github: GitHubClient,
146
+ owner: string,
147
+ repo: string,
148
+ issueNumber: number
149
+ ): Promise<boolean> {
150
+ try {
151
+ const issue = await github.getIssue(owner, repo, issueNumber);
152
+ return (
153
+ issue.labels?.some(
154
+ (l: { name: string } | string) => (typeof l === 'string' ? l : l.name) === 'cf:muted'
155
+ ) ?? false
156
+ );
157
+ } catch {
158
+ // If we can't check, assume not muted
159
+ return false;
160
+ }
161
+ }
162
+
163
+ /** Labels that prevent issue reopening */
164
+ const SKIP_REOPEN_LABELS = ['cf:muted', 'cf:wont-fix'];
165
+
166
+ /**
167
+ * Acquire an optimistic lock for issue creation.
168
+ * Prevents race conditions where concurrent workers create duplicate issues.
169
+ *
170
+ * @returns true if lock acquired, false if another worker holds it
171
+ */
172
+ async function acquireIssueLock(kv: KVNamespace, fingerprint: string): Promise<boolean> {
173
+ const lockKey = `ISSUE_LOCK:${fingerprint}`;
174
+ const existing = await kv.get(lockKey);
175
+
176
+ if (existing) {
177
+ // Another worker is creating an issue for this fingerprint
178
+ return false;
179
+ }
180
+
181
+ // Set lock with 60s TTL (enough time for GitHub API calls)
182
+ await kv.put(lockKey, Date.now().toString(), { expirationTtl: 60 });
183
+ return true;
184
+ }
185
+
186
+ /**
187
+ * Release the issue creation lock.
188
+ */
189
+ async function releaseIssueLock(kv: KVNamespace, fingerprint: string): Promise<void> {
190
+ const lockKey = `ISSUE_LOCK:${fingerprint}`;
191
+ await kv.delete(lockKey);
192
+ }
193
+
194
+ /**
195
+ * Create a dashboard notification for P0-P2 errors.
196
+ * Non-blocking - failures are logged but don't affect issue creation.
197
+ */
198
+ async function createDashboardNotification(
199
+ api: Fetcher | undefined,
200
+ priority: string, // 'P0', 'P1', 'P2', etc.
201
+ errorType: ErrorType,
202
+ scriptName: string,
203
+ message: string,
204
+ issueNumber: number,
205
+ issueUrl: string,
206
+ project: string
207
+ ): Promise<void> {
208
+ // Extract numeric priority from string like 'P0', 'P1', etc.
209
+ const priorityNum = parseInt(priority.replace('P', ''), 10);
210
+
211
+ // Only create notifications for P0-P2 errors
212
+ if (priorityNum > 2 || isNaN(priorityNum) || !api) return;
213
+
214
+ const priorityMap: Record<number, 'critical' | 'high' | 'medium'> = {
215
+ 0: 'critical',
216
+ 1: 'high',
217
+ 2: 'medium',
218
+ };
219
+
220
+ try {
221
+ await api.fetch('https://platform-notifications.internal/notifications', {
222
+ method: 'POST',
223
+ headers: { 'Content-Type': 'application/json' },
224
+ body: JSON.stringify({
225
+ category: 'error',
226
+ source: 'error-collector',
227
+ source_id: String(issueNumber),
228
+ title: `${priority} ${errorType}: ${scriptName}`,
229
+ description: message.slice(0, 200),
230
+ priority: priorityMap[priorityNum],
231
+ action_url: issueUrl,
232
+ action_label: 'View Issue',
233
+ project: project || null,
234
+ }),
235
+ });
236
+ } catch (e) {
237
+ // Non-blocking - log and continue
238
+ console.error('Failed to create dashboard notification:', e);
239
+ }
240
+ }
241
+
242
+ /**
243
+ * Search GitHub for an existing issue with this fingerprint.
244
+ * Used as a fallback when D1/KV don't have the mapping.
245
+ *
246
+ * @returns Issue details if found, null otherwise
247
+ */
248
+ async function findExistingIssueByFingerprint(
249
+ github: GitHubClient,
250
+ owner: string,
251
+ repo: string,
252
+ fingerprint: string
253
+ ): Promise<{
254
+ number: number;
255
+ state: 'open' | 'closed';
256
+ shouldSkip: boolean; // true if muted/wontfix
257
+ } | null> {
258
+ try {
259
+ // Search for issues containing this fingerprint in the body
260
+ // The fingerprint appears as "Fingerprint: `{hash}`" in the issue body
261
+ const issues = await github.searchIssues(owner, repo, `"Fingerprint: \`${fingerprint}\`" in:body`);
262
+
263
+ if (issues.length === 0) return null;
264
+
265
+ // Prefer open issues over closed
266
+ const openIssue = issues.find((i) => i.state === 'open');
267
+ if (openIssue) {
268
+ const labelNames = openIssue.labels.map((l) => l.name);
269
+ const shouldSkip = SKIP_REOPEN_LABELS.some((l) => labelNames.includes(l));
270
+ return { number: openIssue.number, state: 'open', shouldSkip };
271
+ }
272
+
273
+ // Return most recent closed issue (search results are sorted by best match)
274
+ const closedIssue = issues[0];
275
+ const labelNames = closedIssue.labels.map((l) => l.name);
276
+ const shouldSkip = SKIP_REOPEN_LABELS.some((l) => labelNames.includes(l));
277
+
278
+ return {
279
+ number: closedIssue.number,
280
+ state: closedIssue.state as 'open' | 'closed',
281
+ shouldSkip,
282
+ };
283
+ } catch (error) {
284
+ // Fail open - if search fails, allow creating a new issue
285
+ console.error(`GitHub search failed for fingerprint ${fingerprint}:`, error);
286
+ return null;
287
+ }
288
+ }
289
+
290
+ /**
291
+ * Format a comment for when an error recurs and we're adding to an existing issue
292
+ */
293
+ function formatRecurrenceComment(
294
+ event: TailEvent,
295
+ errorType: ErrorType,
296
+ occurrenceCount: number,
297
+ isReopen: boolean
298
+ ): string {
299
+ const timestamp = new Date().toISOString();
300
+ const rayId = event.event?.rayId || event.event?.request?.headers?.['cf-ray'];
301
+
302
+ let comment = isReopen ? `## Error Recurrence (Reopened)\n\n` : `## New Occurrence\n\n`;
303
+
304
+ comment += `| | |\n|---|---|\n`;
305
+ comment += `| **Time** | ${timestamp} |\n`;
306
+ comment += `| **Total Occurrences** | ${occurrenceCount} |\n`;
307
+ comment += `| **Worker** | \`${event.scriptName}\` |\n`;
308
+
309
+ if (event.scriptVersion?.id) {
310
+ comment += `| **Version** | \`${event.scriptVersion.id.slice(0, 8)}\` |\n`;
311
+ }
312
+ if (rayId) {
313
+ comment += `| **Ray ID** | \`${rayId}\` |\n`;
314
+ }
315
+ if (event.event?.request?.cf?.colo) {
316
+ comment += `| **Colo** | ${event.event.request.cf.colo} |\n`;
317
+ }
318
+
319
+ // Include stack trace snippet for exceptions
320
+ if (errorType === 'exception' && event.exceptions.length > 0) {
321
+ const exc = event.exceptions[0];
322
+ const stackPreview = exc.message?.slice(0, 300) || 'N/A';
323
+ comment += `\n### Exception\n\`\`\`\n${exc.name}: ${stackPreview}${exc.message?.length > 300 ? '...' : ''}\n\`\`\`\n`;
324
+ }
325
+
326
+ if (isReopen) {
327
+ comment += `\n> This issue was reopened because the error recurred after being closed.\n`;
328
+ }
329
+
330
+ return comment;
331
+ }
332
+
333
+ /**
334
+ * Get or create error occurrence record
335
+ */
336
+ async function getOrCreateOccurrence(
337
+ db: D1Database,
338
+ kv: KVNamespace,
339
+ fingerprint: string,
340
+ scriptName: string,
341
+ project: string,
342
+ errorType: ErrorType,
343
+ priority: string,
344
+ repo: string
345
+ ): Promise<{
346
+ isNew: boolean;
347
+ occurrence: {
348
+ id: string;
349
+ occurrence_count: number;
350
+ github_issue_number?: number;
351
+ github_issue_url?: string;
352
+ status: ErrorStatus;
353
+ };
354
+ }> {
355
+ // Check KV cache first for existing fingerprint
356
+ const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
357
+ const cached = await kv.get(kvKey);
358
+
359
+ if (cached) {
360
+ const data = JSON.parse(cached) as {
361
+ issueNumber?: number;
362
+ issueUrl?: string;
363
+ status: ErrorStatus;
364
+ occurrenceCount: number;
365
+ };
366
+
367
+ // Update occurrence count in D1
368
+ await db
369
+ .prepare(
370
+ `
371
+ UPDATE error_occurrences
372
+ SET occurrence_count = occurrence_count + 1,
373
+ last_seen_at = unixepoch(),
374
+ updated_at = unixepoch()
375
+ WHERE fingerprint = ?
376
+ `
377
+ )
378
+ .bind(fingerprint)
379
+ .run();
380
+
381
+ // Update KV cache
382
+ await kv.put(
383
+ kvKey,
384
+ JSON.stringify({
385
+ ...data,
386
+ occurrenceCount: data.occurrenceCount + 1,
387
+ lastSeen: Date.now(),
388
+ }),
389
+ { expirationTtl: 90 * 24 * 60 * 60 }
390
+ ); // 90 days
391
+
392
+ return {
393
+ isNew: false,
394
+ occurrence: {
395
+ id: fingerprint,
396
+ occurrence_count: data.occurrenceCount + 1,
397
+ github_issue_number: data.issueNumber,
398
+ github_issue_url: data.issueUrl,
399
+ status: data.status,
400
+ },
401
+ };
402
+ }
403
+
404
+ // Check D1 for existing occurrence
405
+ const existing = await db
406
+ .prepare(
407
+ `
408
+ SELECT id, occurrence_count, github_issue_number, github_issue_url, status
409
+ FROM error_occurrences
410
+ WHERE fingerprint = ?
411
+ `
412
+ )
413
+ .bind(fingerprint)
414
+ .first<{
415
+ id: string;
416
+ occurrence_count: number;
417
+ github_issue_number?: number;
418
+ github_issue_url?: string;
419
+ status: ErrorStatus;
420
+ }>();
421
+
422
+ if (existing) {
423
+ // Update occurrence count
424
+ await db
425
+ .prepare(
426
+ `
427
+ UPDATE error_occurrences
428
+ SET occurrence_count = occurrence_count + 1,
429
+ last_seen_at = unixepoch(),
430
+ updated_at = unixepoch()
431
+ WHERE fingerprint = ?
432
+ `
433
+ )
434
+ .bind(fingerprint)
435
+ .run();
436
+
437
+ // Cache in KV
438
+ await kv.put(
439
+ kvKey,
440
+ JSON.stringify({
441
+ issueNumber: existing.github_issue_number,
442
+ issueUrl: existing.github_issue_url,
443
+ status: existing.status,
444
+ occurrenceCount: existing.occurrence_count + 1,
445
+ lastSeen: Date.now(),
446
+ }),
447
+ { expirationTtl: 90 * 24 * 60 * 60 }
448
+ );
449
+
450
+ return {
451
+ isNew: false,
452
+ occurrence: {
453
+ ...existing,
454
+ occurrence_count: existing.occurrence_count + 1,
455
+ },
456
+ };
457
+ }
458
+
459
+ // Create new occurrence with ON CONFLICT to handle race conditions
460
+ // If another concurrent request already created this fingerprint, just update it
461
+ const id = generateId();
462
+ const now = Math.floor(Date.now() / 1000);
463
+
464
+ const result = await db
465
+ .prepare(
466
+ `
467
+ INSERT INTO error_occurrences (
468
+ id, fingerprint, script_name, project, error_type, priority,
469
+ github_repo, status, first_seen_at, last_seen_at, occurrence_count,
470
+ created_at, updated_at
471
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, 'open', ?, ?, 1, ?, ?)
472
+ ON CONFLICT(fingerprint) DO UPDATE SET
473
+ occurrence_count = occurrence_count + 1,
474
+ last_seen_at = excluded.last_seen_at,
475
+ updated_at = excluded.updated_at
476
+ RETURNING id, occurrence_count, github_issue_number, github_issue_url, status
477
+ `
478
+ )
479
+ .bind(id, fingerprint, scriptName, project, errorType, priority, repo, now, now, now, now)
480
+ .first<{
481
+ id: string;
482
+ occurrence_count: number;
483
+ github_issue_number?: number;
484
+ github_issue_url?: string;
485
+ status: ErrorStatus;
486
+ }>();
487
+
488
+ // Check if this was an insert (count=1) or update (count>1)
489
+ const isNew = result?.occurrence_count === 1;
490
+
491
+ // Cache in KV
492
+ await kv.put(
493
+ kvKey,
494
+ JSON.stringify({
495
+ issueNumber: result?.github_issue_number,
496
+ issueUrl: result?.github_issue_url,
497
+ status: result?.status || ('open' as ErrorStatus),
498
+ occurrenceCount: result?.occurrence_count || 1,
499
+ firstSeen: Date.now(),
500
+ lastSeen: Date.now(),
501
+ }),
502
+ { expirationTtl: 90 * 24 * 60 * 60 }
503
+ );
504
+
505
+ return {
506
+ isNew,
507
+ occurrence: {
508
+ id: result?.id || id,
509
+ occurrence_count: result?.occurrence_count || 1,
510
+ github_issue_number: result?.github_issue_number,
511
+ github_issue_url: result?.github_issue_url,
512
+ status: result?.status || ('open' as ErrorStatus),
513
+ },
514
+ };
515
+ }
516
+
517
+ /**
518
+ * Update occurrence with GitHub issue details
519
+ */
520
+ async function updateOccurrenceWithIssue(
521
+ db: D1Database,
522
+ kv: KVNamespace,
523
+ fingerprint: string,
524
+ issueNumber: number,
525
+ issueUrl: string
526
+ ): Promise<void> {
527
+ await db
528
+ .prepare(
529
+ `
530
+ UPDATE error_occurrences
531
+ SET github_issue_number = ?,
532
+ github_issue_url = ?,
533
+ updated_at = unixepoch()
534
+ WHERE fingerprint = ?
535
+ `
536
+ )
537
+ .bind(issueNumber, issueUrl, fingerprint)
538
+ .run();
539
+
540
+ // Update KV cache
541
+ const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
542
+ const cached = await kv.get(kvKey);
543
+ if (cached) {
544
+ const data = JSON.parse(cached);
545
+ await kv.put(
546
+ kvKey,
547
+ JSON.stringify({
548
+ ...data,
549
+ issueNumber,
550
+ issueUrl,
551
+ }),
552
+ { expirationTtl: 90 * 24 * 60 * 60 }
553
+ );
554
+ }
555
+ }
556
+
557
+ /**
558
+ * Extract a normalized message for pattern discovery.
559
+ * Prioritizes: exception message > error logs > warning logs
560
+ * This ensures pattern-discovery can cluster errors even without exceptions.
561
+ */
562
+ function extractNormalizedMessage(event: TailEvent): string | null {
563
+ // Priority 1: Exception message
564
+ if (event.exceptions[0]?.message) {
565
+ return extractCoreMessage(event.exceptions[0].message);
566
+ }
567
+
568
+ // Priority 2: Error-level logs
569
+ const errorLog = event.logs.find((l) => l.level === 'error');
570
+ if (errorLog) {
571
+ return extractCoreMessage(errorLog.message[0]);
572
+ }
573
+
574
+ // Priority 3: Warning-level logs
575
+ const warnLog = event.logs.find((l) => l.level === 'warn');
576
+ if (warnLog) {
577
+ return extractCoreMessage(warnLog.message[0]);
578
+ }
579
+
580
+ return null;
581
+ }
582
+
583
+ /**
584
+ * Update occurrence with request context
585
+ */
586
+ async function updateOccurrenceContext(
587
+ db: D1Database,
588
+ fingerprint: string,
589
+ event: TailEvent
590
+ ): Promise<void> {
591
+ const url = event.event?.request?.url;
592
+ const method = event.event?.request?.method;
593
+ const colo = event.event?.request?.cf?.colo;
594
+ const country = event.event?.request?.cf?.country;
595
+ const cfRay = event.event?.request?.headers?.['cf-ray'];
596
+ const excName = event.exceptions[0]?.name;
597
+ const logsJson = JSON.stringify(event.logs.slice(-20));
598
+
599
+ // Extract normalized message for pattern discovery (works for all error types)
600
+ const normalizedMessage = extractNormalizedMessage(event);
601
+
602
+ // Use exception message if available, otherwise fall back to normalizedMessage.
603
+ // Soft errors (console.error) have no exceptions[] but DO have error logs —
604
+ // without this fallback, last_exception_message stays NULL and errors are
605
+ // invisible to pattern matching and GitHub issue diagnostics.
606
+ const excMessage = event.exceptions[0]?.message || normalizedMessage;
607
+
608
+ await db
609
+ .prepare(
610
+ `
611
+ UPDATE error_occurrences
612
+ SET last_request_url = ?,
613
+ last_request_method = ?,
614
+ last_colo = ?,
615
+ last_country = ?,
616
+ last_cf_ray = ?,
617
+ last_exception_name = ?,
618
+ last_exception_message = ?,
619
+ last_logs_json = ?,
620
+ normalized_message = COALESCE(?, normalized_message),
621
+ updated_at = unixepoch()
622
+ WHERE fingerprint = ?
623
+ `
624
+ )
625
+ .bind(
626
+ url || null,
627
+ method || null,
628
+ colo || null,
629
+ country || null,
630
+ cfRay || null,
631
+ excName || null,
632
+ excMessage || null,
633
+ logsJson,
634
+ normalizedMessage,
635
+ fingerprint
636
+ )
637
+ .run();
638
+ }
639
+
640
+ /**
641
+ * Format the GitHub issue body with AI-agent-ready context
642
+ * Includes comprehensive observability data for debugging
643
+ */
644
+ function formatIssueBody(
645
+ event: TailEvent,
646
+ errorType: ErrorType,
647
+ priority: string,
648
+ mapping: ScriptMapping,
649
+ fingerprint: string,
650
+ occurrenceCount: number
651
+ ): string {
652
+ const now = new Date().toISOString();
653
+ const eventTime = new Date(event.eventTimestamp).toISOString();
654
+ const exc = event.exceptions[0];
655
+ const req = event.event?.request;
656
+ const rayId = event.event?.rayId || req?.headers?.['cf-ray'];
657
+
658
+ let body = `## `;
659
+
660
+ // Header based on error type - extract clean message from JSON
661
+ if (errorType === 'exception') {
662
+ body += `🔴 Exception: ${exc?.name || 'Unknown'}: ${exc?.message || 'No message'}\n\n`;
663
+ } else if (errorType === 'cpu_limit') {
664
+ body += `🟠 Exceeded CPU Limit\n\n`;
665
+ } else if (errorType === 'memory_limit') {
666
+ body += `🟠 Exceeded Memory Limit\n\n`;
667
+ } else if (errorType === 'soft_error') {
668
+ const errorLog = event.logs.find((l) => l.level === 'error');
669
+ const cleanMsg = errorLog ? extractCoreMessage(errorLog.message[0]) : 'Unknown';
670
+ body += `🟡 Soft Error: ${cleanMsg}\n\n`;
671
+ } else {
672
+ const warnLog = event.logs.find((l) => l.level === 'warn');
673
+ const cleanMsg = warnLog ? extractCoreMessage(warnLog.message[0]) : 'Unknown';
674
+ body += `⚪ Warning: ${cleanMsg}\n\n`;
675
+ }
676
+
677
+ // Summary table for quick context
678
+ body += `| | |\n|---|---|\n`;
679
+ body += `| **Project** | ${mapping.displayName} |\n`;
680
+ body += `| **Worker** | \`${event.scriptName}\` |\n`;
681
+ body += `| **Priority** | ${priority} |\n`;
682
+ body += `| **Tier** | ${mapping.tier} |\n`;
683
+ body += `| **Event Type** | ${event.eventType || 'fetch'} |\n`;
684
+ if (event.scriptVersion?.id) {
685
+ body += `| **Version** | \`${event.scriptVersion.id.slice(0, 8)}\` |\n`;
686
+ }
687
+ body += `| **Outcome** | ${event.outcome} |\n\n`;
688
+
689
+ // Exception details - unescape newlines for readable stack traces
690
+ if (exc) {
691
+ const message = exc.message.replace(/\\n/g, '\n').replace(/\\t/g, '\t');
692
+ body += `### Exception\n\`\`\`\n${exc.name}: ${message}\n\`\`\`\n\n`;
693
+ }
694
+
695
+ // Event context based on type
696
+ if (req) {
697
+ // HTTP Request context
698
+ body += `### Request Context\n`;
699
+ body += `| Field | Value |\n`;
700
+ body += `|-------|-------|\n`;
701
+ body += `| **URL** | \`${req.method} ${req.url}\` |\n`;
702
+ if (event.event?.response?.status) {
703
+ body += `| **Response** | ${event.event.response.status} |\n`;
704
+ }
705
+ if (rayId) body += `| **Ray ID** | \`${rayId}\` |\n`;
706
+ if (req.cf?.colo) body += `| **Colo** | ${req.cf.colo} |\n`;
707
+ if (req.cf?.country) {
708
+ const geo = [req.cf.country, req.cf.city, req.cf.region].filter(Boolean).join(', ');
709
+ body += `| **Location** | ${geo} |\n`;
710
+ }
711
+ if (req.cf?.asOrganization)
712
+ body += `| **Network** | ${req.cf.asOrganization} (AS${req.cf.asn}) |\n`;
713
+ if (req.cf?.timezone) body += `| **Timezone** | ${req.cf.timezone} |\n`;
714
+ if (req.headers?.['user-agent']) {
715
+ const ua = req.headers['user-agent'].slice(0, 80);
716
+ body += `| **User Agent** | \`${ua}${req.headers['user-agent'].length > 80 ? '...' : ''}\` |\n`;
717
+ }
718
+ body += `| **Timestamp** | ${eventTime} |\n\n`;
719
+ } else if (event.event?.scheduledTime || event.event?.cron) {
720
+ // Scheduled/Cron context
721
+ body += `### Scheduled Event Context\n`;
722
+ body += `| Field | Value |\n`;
723
+ body += `|-------|-------|\n`;
724
+ if (event.event.cron) body += `| **Cron** | \`${event.event.cron}\` |\n`;
725
+ if (event.event.scheduledTime) {
726
+ body += `| **Scheduled** | ${new Date(event.event.scheduledTime).toISOString()} |\n`;
727
+ }
728
+ body += `| **Executed** | ${eventTime} |\n\n`;
729
+ } else if (event.event?.queue) {
730
+ // Queue context
731
+ body += `### Queue Event Context\n`;
732
+ body += `| Field | Value |\n`;
733
+ body += `|-------|-------|\n`;
734
+ body += `| **Queue** | ${event.event.queue} |\n`;
735
+ if (event.event.batchSize) body += `| **Batch Size** | ${event.event.batchSize} |\n`;
736
+ body += `| **Timestamp** | ${eventTime} |\n\n`;
737
+ }
738
+
739
+ // Performance metrics
740
+ body += `### Performance\n`;
741
+ body += `| Metric | Value |\n`;
742
+ body += `|--------|-------|\n`;
743
+ if (event.cpuTime !== undefined) body += `| **CPU Time** | ${event.cpuTime}ms |\n`;
744
+ if (event.wallTime !== undefined) body += `| **Wall Time** | ${event.wallTime}ms |\n`;
745
+ body += `| **Execution** | ${event.executionModel || 'stateless'} |\n\n`;
746
+
747
+ // Categorize logs by level
748
+ const errorLogs = event.logs.filter((l) => l.level === 'error' || l.level === 'warn');
749
+ const debugLogs = event.logs.filter((l) => l.level === 'debug');
750
+ const infoLogs = event.logs.filter((l) => l.level === 'info' || l.level === 'log');
751
+ const allLogs = event.logs;
752
+
753
+ // Show error/warning logs prominently
754
+ if (errorLogs.length > 0) {
755
+ body += `### Error/Warning Logs\n`;
756
+ for (const log of errorLogs.slice(-5)) {
757
+ const rawMsg = log.message
758
+ .map((m) => (typeof m === 'string' ? m : JSON.stringify(m, null, 2)))
759
+ .join(' ');
760
+ body += `\`\`\`\n[${log.level.toUpperCase()}] ${rawMsg}\n\`\`\`\n`;
761
+ }
762
+ }
763
+
764
+ // Debug context section - shows debug logs that provide investigation context
765
+ if (debugLogs.length > 0) {
766
+ body += `### 🔍 Debug Context\n`;
767
+ body += `> Debug logs from the same invocation - may contain useful investigation context\n\n`;
768
+ for (const log of debugLogs.slice(-10)) {
769
+ const ts = new Date(log.timestamp).toISOString().split('T')[1].slice(0, 12);
770
+ const rawMsg = log.message
771
+ .map((m) => (typeof m === 'string' ? m : JSON.stringify(m, null, 2)))
772
+ .join(' ')
773
+ .slice(0, 500);
774
+ body += `**${ts}**:\n\`\`\`\n${rawMsg}\n\`\`\`\n`;
775
+ }
776
+ if (debugLogs.length > 10) {
777
+ body += `_...and ${debugLogs.length - 10} more debug entries_\n`;
778
+ }
779
+ body += `\n`;
780
+ }
781
+
782
+ // Full log timeline in collapsible section
783
+ if (allLogs.length > 0) {
784
+ body += `<details>\n<summary>📋 Full Log Timeline (${allLogs.length} entries)</summary>\n\n`;
785
+ body += `| Time | Level | Message |\n`;
786
+ body += `|------|-------|--------|\n`;
787
+ for (const log of allLogs.slice(-30)) {
788
+ const ts = new Date(log.timestamp).toISOString().split('T')[1].slice(0, 12);
789
+ const msg = log.message
790
+ .map((m) => (typeof m === 'string' ? m : JSON.stringify(m)))
791
+ .join(' ')
792
+ .slice(0, 150)
793
+ .replace(/\|/g, '\\|')
794
+ .replace(/\n/g, ' ');
795
+ body += `| ${ts} | ${log.level.toUpperCase()} | ${msg}${log.message.join(' ').length > 150 ? '...' : ''} |\n`;
796
+ }
797
+ if (allLogs.length > 30) {
798
+ body += `| ... | ... | _(${allLogs.length - 30} more entries)_ |\n`;
799
+ }
800
+ body += `\n</details>\n\n`;
801
+ }
802
+
803
+ // Tracking metadata
804
+ body += `### Tracking\n`;
805
+ body += `| Field | Value |\n`;
806
+ body += `|-------|-------|\n`;
807
+ body += `| **Fingerprint** | \`${fingerprint}\` |\n`;
808
+ body += `| **First Seen** | ${now} |\n`;
809
+ body += `| **Occurrences** | ${occurrenceCount} |\n\n`;
810
+
811
+ // Quick links with observability deep link
812
+ body += `### Quick Links\n`;
813
+
814
+ // Observability link with Ray ID filter if available
815
+ if (rayId) {
816
+ body += `- 🔍 [View in CF Observability](https://dash.cloudflare.com/?to=/:account/workers/observability/logs?filters=%5B%7B%22key%22%3A%22%24metadata.requestId%22%2C%22operation%22%3A%22eq%22%2C%22value%22%3A%22${rayId}%22%7D%5D) ← **Start here**\n`;
817
+ } else {
818
+ body += `- 🔍 [CF Observability](https://dash.cloudflare.com/?to=/:account/workers/observability)\n`;
819
+ }
820
+
821
+ body += `- 📄 [Worker Source](https://github.com/${mapping.repository}/blob/main/workers/${event.scriptName}.ts)\n`;
822
+ body += `- 📊 [Worker Dashboard](https://dash.cloudflare.com/?to=/:account/workers/services/view/${event.scriptName})\n`;
823
+ body += `- 📁 [Repository](https://github.com/${mapping.repository})\n`;
824
+ body += `- 📖 [CLAUDE.md](https://github.com/${mapping.repository}/blob/main/CLAUDE.md)\n`;
825
+ body += `- 🔗 [Related Issues](https://github.com/${mapping.repository}/issues?q=is:issue+label:cf:error+${encodeURIComponent(event.scriptName)})\n\n`;
826
+
827
+ // Suggested investigation with more specific guidance
828
+ body += `### Investigation Steps\n`;
829
+ if (errorType === 'exception') {
830
+ body += `1. **Click the Observability link above** to see the full request trace\n`;
831
+ body += `2. Check the exception and logs for error context\n`;
832
+ body += `3. Open [Worker Source](https://github.com/${mapping.repository}/blob/main/workers/${event.scriptName}.ts) and find the failing code\n`;
833
+ body += `4. Review recent commits: \`git log --oneline -10 workers/${event.scriptName}.ts\`\n`;
834
+ } else if (errorType === 'cpu_limit' || errorType === 'memory_limit') {
835
+ body += `1. **Check Observability** for CPU/memory usage patterns\n`;
836
+ body += `2. Review [Worker Source](https://github.com/${mapping.repository}/blob/main/workers/${event.scriptName}.ts) for loops or heavy computation\n`;
837
+ body += `3. Look for recent changes that increased resource usage\n`;
838
+ body += `4. Consider optimizing, caching, or chunking work\n`;
839
+ } else {
840
+ body += `1. Review the logs above for context\n`;
841
+ body += `2. Search for the log message in [Worker Source](https://github.com/${mapping.repository}/blob/main/workers/${event.scriptName}.ts)\n`;
842
+ body += `3. Determine if this is expected or needs fixing\n`;
843
+ body += `4. Consider adding error handling or validation\n`;
844
+ }
845
+
846
+ // Reference documentation for Claude Code agents
847
+ body += `\n### Reference Documentation\n`;
848
+ body += `- 📚 [Error Collector Integration](https://github.com/${GITHUB_ORG}/platform/blob/main/docs/quickrefs/guides/error-collector-integration.md)\n`;
849
+ body += `- 📚 [Troubleshooting Guide](https://github.com/${GITHUB_ORG}/platform/blob/main/docs/quickrefs/troubleshooting.md)\n`;
850
+ body += `- 📚 [Workers Inventory](https://github.com/${GITHUB_ORG}/platform/blob/main/docs/quickrefs/workers-inventory.md)\n`;
851
+
852
+ body += `\n---\n`;
853
+ body += `_Auto-generated by [Platform Error Collector](https://github.com/${GITHUB_ORG}/platform/blob/main/workers/error-collector.ts)_ | Fingerprint: \`${fingerprint}\`\n`;
854
+
855
+ return body;
856
+ }
857
+
858
+ /**
859
+ * Compute fingerprint for a specific error log (for multi-error processing)
860
+ * Returns FingerprintResult with category for transient error detection
861
+ */
862
+ async function computeFingerprintForLog(
863
+ event: TailEvent,
864
+ errorType: ErrorType,
865
+ errorLog: { message: unknown[] },
866
+ dynamicPatterns: CompiledPattern[] = []
867
+ ): Promise<FingerprintResult> {
868
+ const components: string[] = [event.scriptName, errorType];
869
+
870
+ const coreMsg = extractCoreMessage(errorLog.message[0]);
871
+
872
+ // Check for transient error classification first (static + dynamic patterns)
873
+ const classification = classifyErrorWithSource(coreMsg, dynamicPatterns);
874
+ let normalizedMessage: string;
875
+
876
+ if (classification) {
877
+ // Use stable category instead of variable message
878
+ components.push(classification.category);
879
+ normalizedMessage = normalizeDynamicValues(coreMsg).slice(0, 200);
880
+ } else {
881
+ // Standard message-based fingerprinting
882
+ normalizedMessage = normalizeDynamicValues(coreMsg).slice(0, 100);
883
+ components.push(normalizedMessage);
884
+ }
885
+
886
+ // Include normalized URL for HTTP errors (helps distinguish different endpoints)
887
+ if (event.event?.request?.url) {
888
+ components.push(normalizeUrl(event.event.request.url));
889
+ }
890
+
891
+ // Create hash
892
+ const data = components.join(':');
893
+ const hashBuffer = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(data));
894
+
895
+ const fingerprint = Array.from(new Uint8Array(hashBuffer))
896
+ .slice(0, 16)
897
+ .map((b) => b.toString(16).padStart(2, '0'))
898
+ .join('');
899
+
900
+ return {
901
+ fingerprint,
902
+ category: classification?.category ?? null,
903
+ normalizedMessage,
904
+ patternSource: classification?.source,
905
+ dynamicPatternId: classification?.patternId,
906
+ };
907
+ }
908
+
909
+ /**
910
+ * Process a single soft error log from a tail event
911
+ * Called for each unique error in an invocation with multiple errors
912
+ */
913
+ async function processSoftErrorLog(
914
+ event: TailEvent,
915
+ env: Env,
916
+ github: GitHubClient,
917
+ mapping: ScriptMapping,
918
+ errorLog: { level: string; message: unknown[]; timestamp: number },
919
+ dynamicPatterns: CompiledPattern[] = []
920
+ ): Promise<void> {
921
+ const errorType: ErrorType = 'soft_error';
922
+
923
+ // Check rate limit
924
+ const withinLimits = await checkRateLimit(env.PLATFORM_CACHE, event.scriptName);
925
+ if (!withinLimits) {
926
+ const coreMsg = extractCoreMessage(errorLog.message[0]);
927
+ console.log(`Rate limited for script: ${event.scriptName} (error: ${coreMsg.slice(0, 50)})`);
928
+ return;
929
+ }
930
+
931
+ // Compute fingerprint for this specific error log (now returns FingerprintResult)
932
+ const fingerprintResult = await computeFingerprintForLog(event, errorType, errorLog, dynamicPatterns);
933
+ const { fingerprint, category, dynamicPatternId } = fingerprintResult;
934
+ const isTransient = category !== null;
935
+
936
+ // Log dynamic pattern matches for observability and record evidence
937
+ if (dynamicPatternId) {
938
+ console.log(`Dynamic pattern match (soft error): ${category} (pattern: ${dynamicPatternId})`);
939
+ // Record match evidence for human review context
940
+ await recordPatternMatchEvidence(env.PLATFORM_DB, {
941
+ patternId: dynamicPatternId,
942
+ scriptName: event.scriptName,
943
+ project: mapping.project,
944
+ errorFingerprint: fingerprint,
945
+ normalizedMessage: fingerprintResult.normalizedMessage ?? undefined,
946
+ errorType: 'soft_error',
947
+ priority: calculatePriority(errorType, mapping.tier, 1),
948
+ });
949
+ // Increment match_count so shadow evaluation has accurate stats
950
+ await env.PLATFORM_DB.prepare(
951
+ `UPDATE transient_pattern_suggestions SET match_count = match_count + 1, last_matched_at = unixepoch() WHERE id = ?`
952
+ ).bind(dynamicPatternId).run();
953
+ }
954
+
955
+ // For transient errors, check if we already have an issue for today's window
956
+ if (isTransient && category) {
957
+ const existingIssue = await checkTransientErrorWindow(
958
+ env.PLATFORM_CACHE,
959
+ event.scriptName,
960
+ category
961
+ );
962
+ if (existingIssue) {
963
+ // Just update occurrence count in D1, don't create new issue
964
+ await env.PLATFORM_DB.prepare(
965
+ `
966
+ UPDATE error_occurrences
967
+ SET occurrence_count = occurrence_count + 1,
968
+ last_seen_at = unixepoch(),
969
+ updated_at = unixepoch()
970
+ WHERE fingerprint = ?
971
+ `
972
+ )
973
+ .bind(fingerprint)
974
+ .run();
975
+ console.log(
976
+ `Transient soft error (${category}) for ${event.scriptName} - issue #${existingIssue} exists for today`
977
+ );
978
+ return;
979
+ }
980
+ }
981
+
982
+ // Get or create occurrence
983
+ const { isNew, occurrence } = await getOrCreateOccurrence(
984
+ env.PLATFORM_DB,
985
+ env.PLATFORM_CACHE,
986
+ fingerprint,
987
+ event.scriptName,
988
+ mapping.project,
989
+ errorType,
990
+ calculatePriority(errorType, mapping.tier, 1),
991
+ mapping.repository
992
+ );
993
+
994
+ // Update context
995
+ await updateOccurrenceContext(env.PLATFORM_DB, fingerprint, event);
996
+
997
+ // Calculate priority with actual occurrence count
998
+ const priority = calculatePriority(errorType, mapping.tier, occurrence.occurrence_count);
999
+
1000
+ // If this is a new error, create a GitHub issue (with dedup check)
1001
+ if (isNew) {
1002
+ try {
1003
+ const [owner, repo] = mapping.repository.split('/');
1004
+
1005
+ // RACE CONDITION PREVENTION: Acquire lock before searching/creating
1006
+ const lockAcquired = await acquireIssueLock(env.PLATFORM_CACHE, fingerprint);
1007
+ if (!lockAcquired) {
1008
+ console.log(`Lock held by another worker for ${fingerprint}, skipping`);
1009
+ return;
1010
+ }
1011
+
1012
+ try {
1013
+ // DEDUP CHECK: Search GitHub for existing issue with this fingerprint
1014
+ const existingIssue = await findExistingIssueByFingerprint(github, owner, repo, fingerprint);
1015
+
1016
+ if (existingIssue) {
1017
+ // Check if issue is muted/wontfix - don't reopen or create new
1018
+ if (existingIssue.shouldSkip) {
1019
+ console.log(`Issue #${existingIssue.number} is muted/wontfix, skipping`);
1020
+ // Still link D1 record to prevent future searches
1021
+ await updateOccurrenceWithIssue(
1022
+ env.PLATFORM_DB,
1023
+ env.PLATFORM_CACHE,
1024
+ fingerprint,
1025
+ existingIssue.number,
1026
+ `https://github.com/${owner}/${repo}/issues/${existingIssue.number}`
1027
+ );
1028
+ return;
1029
+ }
1030
+
1031
+ // Found existing issue - update it instead of creating new
1032
+ const comment = formatRecurrenceComment(
1033
+ event,
1034
+ errorType,
1035
+ occurrence.occurrence_count,
1036
+ existingIssue.state === 'closed'
1037
+ );
1038
+
1039
+ if (existingIssue.state === 'closed') {
1040
+ // Reopen the issue
1041
+ await github.updateIssue({
1042
+ owner,
1043
+ repo,
1044
+ issue_number: existingIssue.number,
1045
+ state: 'open',
1046
+ });
1047
+ await github.addLabels(owner, repo, existingIssue.number, ['cf:regression']);
1048
+ console.log(`Reopened existing issue #${existingIssue.number} (dedup: ${fingerprint})`);
1049
+ }
1050
+
1051
+ await github.addComment(owner, repo, existingIssue.number, comment);
1052
+
1053
+ // Update D1 with the found issue number
1054
+ await updateOccurrenceWithIssue(
1055
+ env.PLATFORM_DB,
1056
+ env.PLATFORM_CACHE,
1057
+ fingerprint,
1058
+ existingIssue.number,
1059
+ `https://github.com/${owner}/${repo}/issues/${existingIssue.number}`
1060
+ );
1061
+
1062
+ // For transient errors, record the issue in the window cache
1063
+ if (isTransient && category) {
1064
+ await setTransientErrorWindow(
1065
+ env.PLATFORM_CACHE,
1066
+ event.scriptName,
1067
+ category,
1068
+ existingIssue.number
1069
+ );
1070
+ }
1071
+
1072
+ return; // Don't create a new issue
1073
+ }
1074
+
1075
+ // No existing issue found - create new (original code)
1076
+ const coreMsg = extractCoreMessage(errorLog.message[0]);
1077
+ const title = `[${event.scriptName}] Error: ${coreMsg.slice(0, 60)}`.slice(0, 100);
1078
+ const body = formatIssueBody(
1079
+ event,
1080
+ errorType,
1081
+ priority,
1082
+ mapping,
1083
+ fingerprint,
1084
+ occurrence.occurrence_count
1085
+ );
1086
+ const labels = getLabels(errorType, priority);
1087
+
1088
+ // Add transient label for transient errors
1089
+ if (isTransient) {
1090
+ labels.push('cf:transient');
1091
+ }
1092
+
1093
+ const issue = await github.createIssue({
1094
+ owner,
1095
+ repo,
1096
+ title,
1097
+ body,
1098
+ labels,
1099
+ type: getGitHubIssueType(errorType),
1100
+ assignees: env.DEFAULT_ASSIGNEE ? [env.DEFAULT_ASSIGNEE] : [],
1101
+ });
1102
+
1103
+ console.log(
1104
+ `Created issue #${issue.number} for ${event.scriptName} - ${coreMsg.slice(0, 30)}${isTransient ? ` (transient: ${category})` : ''}`
1105
+ );
1106
+
1107
+ // Update occurrence with issue details
1108
+ await updateOccurrenceWithIssue(
1109
+ env.PLATFORM_DB,
1110
+ env.PLATFORM_CACHE,
1111
+ fingerprint,
1112
+ issue.number,
1113
+ issue.html_url
1114
+ );
1115
+
1116
+ // For transient errors, record the issue in the window cache
1117
+ if (isTransient && category) {
1118
+ await setTransientErrorWindow(env.PLATFORM_CACHE, event.scriptName, category, issue.number);
1119
+ }
1120
+
1121
+ // Add to project board
1122
+ try {
1123
+ const issueDetails = await github.getIssue(owner, repo, issue.number);
1124
+ await github.addToProject(issueDetails.node_id, env.GITHUB_PROJECT_ID);
1125
+ } catch (e) {
1126
+ console.error(`Failed to add to project board: ${e}`);
1127
+ }
1128
+
1129
+ // Create dashboard notification for P0-P2 errors
1130
+ await createDashboardNotification(
1131
+ env.NOTIFICATIONS_API,
1132
+ priority,
1133
+ errorType,
1134
+ event.scriptName,
1135
+ coreMsg,
1136
+ issue.number,
1137
+ issue.html_url,
1138
+ mapping.project
1139
+ );
1140
+ } finally {
1141
+ // Always release lock
1142
+ await releaseIssueLock(env.PLATFORM_CACHE, fingerprint);
1143
+ }
1144
+ } catch (e) {
1145
+ console.error(`Failed to create GitHub issue: ${e}`);
1146
+ }
1147
+ } else if (occurrence.github_issue_number && occurrence.status === 'resolved') {
1148
+ // Error recurred after being resolved
1149
+ // Skip regression logic for transient errors - they're expected to recur
1150
+ if (isTransient) {
1151
+ console.log(
1152
+ `Transient soft error (${category}) recurred for ${event.scriptName} - not marking as regression`
1153
+ );
1154
+ // Just update to open status without regression label
1155
+ await env.PLATFORM_DB.prepare(
1156
+ `
1157
+ UPDATE error_occurrences
1158
+ SET status = 'open',
1159
+ resolved_at = NULL,
1160
+ resolved_by = NULL,
1161
+ updated_at = unixepoch()
1162
+ WHERE fingerprint = ?
1163
+ `
1164
+ )
1165
+ .bind(fingerprint)
1166
+ .run();
1167
+ return;
1168
+ }
1169
+
1170
+ // Non-transient error: apply regression logic
1171
+ try {
1172
+ const [owner, repo] = mapping.repository.split('/');
1173
+
1174
+ // Check if issue is muted - if so, don't reopen or comment
1175
+ const muted = await isIssueMuted(github, owner, repo, occurrence.github_issue_number);
1176
+ if (muted) {
1177
+ console.log(`Issue #${occurrence.github_issue_number} is muted, skipping reopen`);
1178
+ return;
1179
+ }
1180
+
1181
+ await github.updateIssue({
1182
+ owner,
1183
+ repo,
1184
+ issue_number: occurrence.github_issue_number,
1185
+ state: 'open',
1186
+ });
1187
+
1188
+ await github.addLabels(owner, repo, occurrence.github_issue_number, ['cf:regression']);
1189
+
1190
+ await github.addComment(
1191
+ owner,
1192
+ repo,
1193
+ occurrence.github_issue_number,
1194
+ `⚠️ **Regression Detected**\n\nThis error has recurred after being marked as resolved.\n\n- **Occurrences**: ${occurrence.occurrence_count}\n- **Last Seen**: ${new Date().toISOString()}\n\nPlease investigate if the fix was incomplete.`
1195
+ );
1196
+
1197
+ console.log(`Reopened issue #${occurrence.github_issue_number} as regression`);
1198
+
1199
+ // Update status in D1
1200
+ await env.PLATFORM_DB.prepare(
1201
+ `
1202
+ UPDATE error_occurrences
1203
+ SET status = 'open',
1204
+ resolved_at = NULL,
1205
+ resolved_by = NULL,
1206
+ updated_at = unixepoch()
1207
+ WHERE fingerprint = ?
1208
+ `
1209
+ )
1210
+ .bind(fingerprint)
1211
+ .run();
1212
+ } catch (e) {
1213
+ console.error(`Failed to reopen issue: ${e}`);
1214
+ }
1215
+ } else if (occurrence.github_issue_number) {
1216
+ // Update existing issue with new occurrence count (every 10 occurrences)
1217
+ if (occurrence.occurrence_count % 10 === 0) {
1218
+ try {
1219
+ const [owner, repo] = mapping.repository.split('/');
1220
+
1221
+ // Check if issue is muted - if so, don't add comments
1222
+ const muted = await isIssueMuted(github, owner, repo, occurrence.github_issue_number);
1223
+ if (muted) {
1224
+ console.log(`Issue #${occurrence.github_issue_number} is muted, skipping comment`);
1225
+ return;
1226
+ }
1227
+
1228
+ await github.addComment(
1229
+ owner,
1230
+ repo,
1231
+ occurrence.github_issue_number,
1232
+ `📊 **Occurrence Update**\n\nThis error has now occurred **${occurrence.occurrence_count} times**.\n\n- **Last Seen**: ${new Date().toISOString()}\n- **Colo**: ${event.event?.request?.cf?.colo || 'unknown'}`
1233
+ );
1234
+ console.log(`Updated issue #${occurrence.github_issue_number} with occurrence count`);
1235
+ } catch (e) {
1236
+ console.error(`Failed to update issue: ${e}`);
1237
+ }
1238
+ }
1239
+ }
1240
+ }
1241
+
1242
+ /**
1243
+ * Process a single tail event
1244
+ */
1245
+ async function processEvent(
1246
+ event: TailEvent,
1247
+ env: Env,
1248
+ github: GitHubClient,
1249
+ dynamicPatterns: CompiledPattern[] = []
1250
+ ): Promise<void> {
1251
+ // Check if we should capture this event
1252
+ const decision = shouldCapture(event);
1253
+ if (!decision.capture || !decision.type) {
1254
+ return;
1255
+ }
1256
+
1257
+ const errorType = decision.type;
1258
+
1259
+ // Get script mapping
1260
+ const mapping = await getScriptMapping(env.PLATFORM_CACHE, event.scriptName);
1261
+ if (!mapping) {
1262
+ console.log(`No mapping found for script: ${event.scriptName}`);
1263
+ return;
1264
+ }
1265
+
1266
+ // P4 warnings go to daily digest instead of immediate issues
1267
+ // Process ALL warning logs, not just the first one
1268
+ if (errorType === 'warning') {
1269
+ const warnLogs = event.logs.filter((l) => l.level === 'warn');
1270
+ const seenNormalizedMessages = new Set<string>();
1271
+
1272
+ for (const warnLog of warnLogs) {
1273
+ const rawMessage = warnLog.message
1274
+ .map((m) => (typeof m === 'string' ? m : JSON.stringify(m)))
1275
+ .join(' ');
1276
+ const coreMessage = extractCoreMessage(warnLog.message[0]);
1277
+ const normalizedMessage = normalizeDynamicValues(coreMessage);
1278
+
1279
+ // Dedupe warnings with identical normalized messages within same invocation
1280
+ if (seenNormalizedMessages.has(normalizedMessage)) {
1281
+ continue;
1282
+ }
1283
+ seenNormalizedMessages.add(normalizedMessage);
1284
+
1285
+ const fingerprintResult = await computeFingerprintForLog(event, errorType, warnLog, dynamicPatterns);
1286
+
1287
+ await storeWarningForDigest(
1288
+ env.PLATFORM_DB,
1289
+ env.PLATFORM_CACHE,
1290
+ fingerprintResult.fingerprint,
1291
+ event.scriptName,
1292
+ mapping.project,
1293
+ mapping.repository,
1294
+ normalizedMessage,
1295
+ rawMessage
1296
+ );
1297
+
1298
+ console.log(
1299
+ `Stored warning for digest: ${event.scriptName} - ${normalizedMessage.slice(0, 50)}`
1300
+ );
1301
+ }
1302
+ return;
1303
+ }
1304
+
1305
+ // For soft_error: Process ALL error logs, not just the first one
1306
+ // This fixes the bug where multiple console.error() calls in one invocation
1307
+ // only resulted in one GitHub issue (task-296)
1308
+ if (errorType === 'soft_error') {
1309
+ const errorLogs = event.logs.filter((l) => l.level === 'error');
1310
+ const seenNormalizedMessages = new Set<string>();
1311
+
1312
+ for (const errorLog of errorLogs) {
1313
+ const coreMessage = extractCoreMessage(errorLog.message[0]);
1314
+ const normalizedMessage = normalizeDynamicValues(coreMessage);
1315
+
1316
+ // Dedupe errors with identical normalized messages within same invocation
1317
+ // (e.g., same error logged twice in a loop)
1318
+ if (seenNormalizedMessages.has(normalizedMessage)) {
1319
+ continue;
1320
+ }
1321
+ seenNormalizedMessages.add(normalizedMessage);
1322
+
1323
+ await processSoftErrorLog(event, env, github, mapping, errorLog, dynamicPatterns);
1324
+ }
1325
+ return;
1326
+ }
1327
+
1328
+ // For exceptions and resource limits: use original single-error processing
1329
+ // Check rate limit (only for non-warning errors that create immediate issues)
1330
+ const withinLimits = await checkRateLimit(env.PLATFORM_CACHE, event.scriptName);
1331
+ if (!withinLimits) {
1332
+ console.log(`Rate limited for script: ${event.scriptName}`);
1333
+ return;
1334
+ }
1335
+
1336
+ // Compute fingerprint (now returns FingerprintResult with category)
1337
+ // Pass dynamic patterns to enable AI-suggested pattern matching
1338
+ const fingerprintResult = await computeFingerprint(event, errorType, { dynamicPatterns });
1339
+ const { fingerprint, category, dynamicPatternId } = fingerprintResult;
1340
+ const isTransient = category !== null;
1341
+
1342
+ // Log dynamic pattern matches for observability and record evidence
1343
+ if (dynamicPatternId) {
1344
+ console.log(`Dynamic pattern match: ${category} (pattern: ${dynamicPatternId})`);
1345
+ // Record match evidence for human review context
1346
+ await recordPatternMatchEvidence(env.PLATFORM_DB, {
1347
+ patternId: dynamicPatternId,
1348
+ scriptName: event.scriptName,
1349
+ project: mapping.project,
1350
+ errorFingerprint: fingerprint,
1351
+ normalizedMessage: fingerprintResult.normalizedMessage ?? undefined,
1352
+ errorType: errorType,
1353
+ priority: calculatePriority(errorType, mapping.tier, 1),
1354
+ });
1355
+ // Increment match_count so shadow evaluation has accurate stats
1356
+ await env.PLATFORM_DB.prepare(
1357
+ `UPDATE transient_pattern_suggestions SET match_count = match_count + 1, last_matched_at = unixepoch() WHERE id = ?`
1358
+ ).bind(dynamicPatternId).run();
1359
+ }
1360
+
1361
+ // For transient errors, check if we already have an issue for today's window
1362
+ // This prevents noise from quota exhaustion errors that occur repeatedly
1363
+ if (isTransient && category) {
1364
+ const existingIssue = await checkTransientErrorWindow(
1365
+ env.PLATFORM_CACHE,
1366
+ event.scriptName,
1367
+ category
1368
+ );
1369
+ if (existingIssue) {
1370
+ // Just update occurrence count in D1, don't create new issue
1371
+ await env.PLATFORM_DB.prepare(
1372
+ `
1373
+ UPDATE error_occurrences
1374
+ SET occurrence_count = occurrence_count + 1,
1375
+ last_seen_at = unixepoch(),
1376
+ updated_at = unixepoch()
1377
+ WHERE fingerprint = ?
1378
+ `
1379
+ )
1380
+ .bind(fingerprint)
1381
+ .run();
1382
+ console.log(
1383
+ `Transient error (${category}) for ${event.scriptName} - issue #${existingIssue} exists for today`
1384
+ );
1385
+ return;
1386
+ }
1387
+ }
1388
+
1389
+ // Get or create occurrence
1390
+ const { isNew, occurrence } = await getOrCreateOccurrence(
1391
+ env.PLATFORM_DB,
1392
+ env.PLATFORM_CACHE,
1393
+ fingerprint,
1394
+ event.scriptName,
1395
+ mapping.project,
1396
+ errorType,
1397
+ calculatePriority(errorType, mapping.tier, 1),
1398
+ mapping.repository
1399
+ );
1400
+
1401
+ // Update context
1402
+ await updateOccurrenceContext(env.PLATFORM_DB, fingerprint, event);
1403
+
1404
+ // Calculate priority with actual occurrence count
1405
+ const priority = calculatePriority(errorType, mapping.tier, occurrence.occurrence_count);
1406
+
1407
+ // If this is a new error, create a GitHub issue (with dedup check)
1408
+ if (isNew) {
1409
+ try {
1410
+ const [owner, repo] = mapping.repository.split('/');
1411
+
1412
+ // RACE CONDITION PREVENTION: Acquire lock before searching/creating
1413
+ const lockAcquired = await acquireIssueLock(env.PLATFORM_CACHE, fingerprint);
1414
+ if (!lockAcquired) {
1415
+ console.log(`Lock held by another worker for ${fingerprint}, skipping`);
1416
+ return;
1417
+ }
1418
+
1419
+ try {
1420
+ // DEDUP CHECK: Search GitHub for existing issue with this fingerprint
1421
+ const existingIssue = await findExistingIssueByFingerprint(github, owner, repo, fingerprint);
1422
+
1423
+ if (existingIssue) {
1424
+ // Check if issue is muted/wontfix - don't reopen or create new
1425
+ if (existingIssue.shouldSkip) {
1426
+ console.log(`Issue #${existingIssue.number} is muted/wontfix, skipping`);
1427
+ // Still link D1 record to prevent future searches
1428
+ await updateOccurrenceWithIssue(
1429
+ env.PLATFORM_DB,
1430
+ env.PLATFORM_CACHE,
1431
+ fingerprint,
1432
+ existingIssue.number,
1433
+ `https://github.com/${owner}/${repo}/issues/${existingIssue.number}`
1434
+ );
1435
+ return;
1436
+ }
1437
+
1438
+ // Found existing issue - update it instead of creating new
1439
+ const comment = formatRecurrenceComment(
1440
+ event,
1441
+ errorType,
1442
+ occurrence.occurrence_count,
1443
+ existingIssue.state === 'closed'
1444
+ );
1445
+
1446
+ if (existingIssue.state === 'closed') {
1447
+ // Reopen the issue
1448
+ await github.updateIssue({
1449
+ owner,
1450
+ repo,
1451
+ issue_number: existingIssue.number,
1452
+ state: 'open',
1453
+ });
1454
+ await github.addLabels(owner, repo, existingIssue.number, ['cf:regression']);
1455
+ console.log(`Reopened existing issue #${existingIssue.number} (dedup: ${fingerprint})`);
1456
+ }
1457
+
1458
+ await github.addComment(owner, repo, existingIssue.number, comment);
1459
+
1460
+ // Update D1 with the found issue number
1461
+ await updateOccurrenceWithIssue(
1462
+ env.PLATFORM_DB,
1463
+ env.PLATFORM_CACHE,
1464
+ fingerprint,
1465
+ existingIssue.number,
1466
+ `https://github.com/${owner}/${repo}/issues/${existingIssue.number}`
1467
+ );
1468
+
1469
+ // For transient errors, record the issue in the window cache
1470
+ if (isTransient && category) {
1471
+ await setTransientErrorWindow(
1472
+ env.PLATFORM_CACHE,
1473
+ event.scriptName,
1474
+ category,
1475
+ existingIssue.number
1476
+ );
1477
+ }
1478
+
1479
+ return; // Don't create a new issue
1480
+ }
1481
+
1482
+ // No existing issue found - create new (original code)
1483
+ const title = formatErrorTitle(errorType, event, event.scriptName);
1484
+ const body = formatIssueBody(
1485
+ event,
1486
+ errorType,
1487
+ priority,
1488
+ mapping,
1489
+ fingerprint,
1490
+ occurrence.occurrence_count
1491
+ );
1492
+ const labels = getLabels(errorType, priority);
1493
+
1494
+ // Add transient label for transient errors
1495
+ if (isTransient) {
1496
+ labels.push('cf:transient');
1497
+ }
1498
+
1499
+ const issue = await github.createIssue({
1500
+ owner,
1501
+ repo,
1502
+ title,
1503
+ body,
1504
+ labels,
1505
+ type: getGitHubIssueType(errorType),
1506
+ assignees: env.DEFAULT_ASSIGNEE ? [env.DEFAULT_ASSIGNEE] : [],
1507
+ });
1508
+
1509
+ console.log(
1510
+ `Created issue #${issue.number} for ${event.scriptName}${isTransient ? ` (transient: ${category})` : ''}`
1511
+ );
1512
+
1513
+ // Update occurrence with issue details
1514
+ await updateOccurrenceWithIssue(
1515
+ env.PLATFORM_DB,
1516
+ env.PLATFORM_CACHE,
1517
+ fingerprint,
1518
+ issue.number,
1519
+ issue.html_url
1520
+ );
1521
+
1522
+ // For transient errors, record the issue in the window cache
1523
+ if (isTransient && category) {
1524
+ await setTransientErrorWindow(env.PLATFORM_CACHE, event.scriptName, category, issue.number);
1525
+ }
1526
+
1527
+ // Add to project board
1528
+ try {
1529
+ const issueDetails = await github.getIssue(owner, repo, issue.number);
1530
+ await github.addToProject(issueDetails.node_id, env.GITHUB_PROJECT_ID);
1531
+ console.log(`Added issue #${issue.number} to project board`);
1532
+ } catch (e) {
1533
+ console.error(`Failed to add to project board: ${e}`);
1534
+ }
1535
+
1536
+ // Create dashboard notification for P0-P2 errors
1537
+ await createDashboardNotification(
1538
+ env.NOTIFICATIONS_API,
1539
+ priority,
1540
+ errorType,
1541
+ event.scriptName,
1542
+ title,
1543
+ issue.number,
1544
+ issue.html_url,
1545
+ mapping.project
1546
+ );
1547
+ } finally {
1548
+ // Always release lock
1549
+ await releaseIssueLock(env.PLATFORM_CACHE, fingerprint);
1550
+ }
1551
+ } catch (e) {
1552
+ console.error(`Failed to create GitHub issue: ${e}`);
1553
+ }
1554
+ } else if (occurrence.github_issue_number && occurrence.status === 'resolved') {
1555
+ // Error recurred after being resolved
1556
+ // Skip regression logic for transient errors - they're expected to recur
1557
+ if (isTransient) {
1558
+ console.log(
1559
+ `Transient error (${category}) recurred for ${event.scriptName} - not marking as regression`
1560
+ );
1561
+ // Just update to open status without regression label
1562
+ await env.PLATFORM_DB.prepare(
1563
+ `
1564
+ UPDATE error_occurrences
1565
+ SET status = 'open',
1566
+ resolved_at = NULL,
1567
+ resolved_by = NULL,
1568
+ updated_at = unixepoch()
1569
+ WHERE fingerprint = ?
1570
+ `
1571
+ )
1572
+ .bind(fingerprint)
1573
+ .run();
1574
+ return;
1575
+ }
1576
+
1577
+ // Non-transient error: apply regression logic
1578
+ try {
1579
+ const [owner, repo] = mapping.repository.split('/');
1580
+
1581
+ // Check if issue is muted - if so, don't reopen or comment
1582
+ const muted = await isIssueMuted(github, owner, repo, occurrence.github_issue_number);
1583
+ if (muted) {
1584
+ console.log(`Issue #${occurrence.github_issue_number} is muted, skipping reopen`);
1585
+ return;
1586
+ }
1587
+
1588
+ await github.updateIssue({
1589
+ owner,
1590
+ repo,
1591
+ issue_number: occurrence.github_issue_number,
1592
+ state: 'open',
1593
+ });
1594
+
1595
+ await github.addLabels(owner, repo, occurrence.github_issue_number, ['cf:regression']);
1596
+
1597
+ await github.addComment(
1598
+ owner,
1599
+ repo,
1600
+ occurrence.github_issue_number,
1601
+ `⚠️ **Regression Detected**\n\nThis error has recurred after being marked as resolved.\n\n- **Occurrences**: ${occurrence.occurrence_count}\n- **Last Seen**: ${new Date().toISOString()}\n\nPlease investigate if the fix was incomplete.`
1602
+ );
1603
+
1604
+ console.log(`Reopened issue #${occurrence.github_issue_number} as regression`);
1605
+
1606
+ // Update status in D1
1607
+ await env.PLATFORM_DB.prepare(
1608
+ `
1609
+ UPDATE error_occurrences
1610
+ SET status = 'open',
1611
+ resolved_at = NULL,
1612
+ resolved_by = NULL,
1613
+ updated_at = unixepoch()
1614
+ WHERE fingerprint = ?
1615
+ `
1616
+ )
1617
+ .bind(fingerprint)
1618
+ .run();
1619
+ } catch (e) {
1620
+ console.error(`Failed to reopen issue: ${e}`);
1621
+ }
1622
+ } else if (occurrence.github_issue_number) {
1623
+ // Update existing issue with new occurrence count
1624
+ try {
1625
+ const [owner, repo] = mapping.repository.split('/');
1626
+
1627
+ // Check if issue is muted - if so, don't add comments
1628
+ const muted = await isIssueMuted(github, owner, repo, occurrence.github_issue_number);
1629
+ if (muted) {
1630
+ console.log(`Issue #${occurrence.github_issue_number} is muted, skipping comment`);
1631
+ return;
1632
+ }
1633
+
1634
+ // Add a comment every 10 occurrences to avoid spam
1635
+ if (occurrence.occurrence_count % 10 === 0) {
1636
+ await github.addComment(
1637
+ owner,
1638
+ repo,
1639
+ occurrence.github_issue_number,
1640
+ `📊 **Occurrence Update**\n\nThis error has now occurred **${occurrence.occurrence_count} times**.\n\n- **Last Seen**: ${new Date().toISOString()}\n- **Colo**: ${event.event?.request?.cf?.colo || 'unknown'}`
1641
+ );
1642
+ console.log(`Updated issue #${occurrence.github_issue_number} with occurrence count`);
1643
+ }
1644
+ } catch (e) {
1645
+ console.error(`Failed to update issue: ${e}`);
1646
+ }
1647
+ }
1648
+ }
1649
+
1650
+ /**
1651
+ * Tail handler - receives events from producer workers
1652
+ */
1653
+ async function tail(events: TailEvent[], env: Env): Promise<void> {
1654
+ const github = new GitHubClient(env);
1655
+
1656
+ // Load dynamic patterns once for all events in this batch
1657
+ // These are AI-suggested, human-approved patterns from pattern-discovery
1658
+ const dynamicPatterns = await loadDynamicPatterns(env.PLATFORM_CACHE);
1659
+ if (dynamicPatterns.length > 0) {
1660
+ console.log(`Loaded ${dynamicPatterns.length} dynamic patterns from KV`);
1661
+ }
1662
+
1663
+ for (const event of events) {
1664
+ try {
1665
+ await processEvent(event, env, github, dynamicPatterns);
1666
+ } catch (e) {
1667
+ console.error(`Error processing tail event: ${e}`);
1668
+ }
1669
+ }
1670
+ }
1671
+
1672
+ /**
1673
+ * Scheduled handler - auto-close stale errors and process warning digests
1674
+ */
1675
+ async function scheduled(
1676
+ event: ScheduledEvent,
1677
+ env: Env,
1678
+ ctx: ExecutionContext
1679
+ ): Promise<void> {
1680
+ const now = Math.floor(Date.now() / 1000);
1681
+ const autoCloseSeconds = parseInt(env.AUTO_CLOSE_HOURS, 10) * 60 * 60;
1682
+ const warningCloseSeconds = parseInt(env.WARNING_AUTO_CLOSE_DAYS, 10) * 24 * 60 * 60;
1683
+
1684
+ // At midnight UTC (0 0 * * *), process warning digests for the previous day
1685
+ const currentHour = new Date().getUTCHours();
1686
+ const currentMinute = new Date().getUTCMinutes();
1687
+ if (currentHour === 0 && currentMinute < 15) {
1688
+ console.log('Running daily warning digest processing...');
1689
+ try {
1690
+ const result = await processWarningDigests(env);
1691
+ console.log(
1692
+ `Digest complete: ${result.processed} warnings, ${result.issuesCreated} new issues, ${result.issuesUpdated} updated`
1693
+ );
1694
+ // Signal digest success to Gatus heartbeat
1695
+ pingHeartbeat(ctx, env.GATUS_HEARTBEAT_URL_DIGEST, env.GATUS_TOKEN, true);
1696
+ } catch (e) {
1697
+ console.error(`Failed to process warning digests: ${e}`);
1698
+ // Signal digest failure to Gatus heartbeat
1699
+ pingHeartbeat(ctx, env.GATUS_HEARTBEAT_URL_DIGEST, env.GATUS_TOKEN, false);
1700
+ }
1701
+ }
1702
+
1703
+ // Find errors that haven't recurred in AUTO_CLOSE_HOURS
1704
+ const staleErrors = await env.PLATFORM_DB.prepare(
1705
+ `
1706
+ SELECT fingerprint, github_issue_number, github_repo, error_type
1707
+ FROM error_occurrences
1708
+ WHERE status = 'open'
1709
+ AND github_issue_number IS NOT NULL
1710
+ AND last_seen_at < ?
1711
+ AND (error_type != 'warning' OR last_seen_at < ?)
1712
+ `
1713
+ )
1714
+ .bind(now - autoCloseSeconds, now - warningCloseSeconds)
1715
+ .all<{
1716
+ fingerprint: string;
1717
+ github_issue_number: number;
1718
+ github_repo: string;
1719
+ error_type: string;
1720
+ }>();
1721
+
1722
+ if (staleErrors.results?.length) {
1723
+ const github = new GitHubClient(env);
1724
+
1725
+ for (const error of staleErrors.results) {
1726
+ try {
1727
+ const [owner, repo] = error.github_repo.split('/');
1728
+
1729
+ await github.updateIssue({
1730
+ owner,
1731
+ repo,
1732
+ issue_number: error.github_issue_number,
1733
+ state: 'closed',
1734
+ });
1735
+
1736
+ await github.addComment(
1737
+ owner,
1738
+ repo,
1739
+ error.github_issue_number,
1740
+ `✅ **Auto-closed**\n\nThis error has not recurred in ${env.AUTO_CLOSE_HOURS} hours and has been automatically closed.\n\nIf you know which commit fixed this, please link it here. If the error recurs, this issue will be automatically reopened.`
1741
+ );
1742
+
1743
+ // Update status in D1
1744
+ await env.PLATFORM_DB.prepare(
1745
+ `
1746
+ UPDATE error_occurrences
1747
+ SET status = 'resolved',
1748
+ resolved_at = unixepoch(),
1749
+ resolved_by = 'auto-close',
1750
+ updated_at = unixepoch()
1751
+ WHERE fingerprint = ?
1752
+ `
1753
+ )
1754
+ .bind(error.fingerprint)
1755
+ .run();
1756
+
1757
+ // Update KV cache
1758
+ const kvKey = `ERROR_FINGERPRINT:${error.fingerprint}`;
1759
+ const cached = await env.PLATFORM_CACHE.get(kvKey);
1760
+ if (cached) {
1761
+ const data = JSON.parse(cached);
1762
+ await env.PLATFORM_CACHE.put(
1763
+ kvKey,
1764
+ JSON.stringify({
1765
+ ...data,
1766
+ status: 'resolved',
1767
+ }),
1768
+ { expirationTtl: 90 * 24 * 60 * 60 }
1769
+ );
1770
+ }
1771
+
1772
+ console.log(`Auto-closed issue #${error.github_issue_number}`);
1773
+ } catch (e) {
1774
+ console.error(`Failed to auto-close issue #${error.github_issue_number}: ${e}`);
1775
+ }
1776
+ }
1777
+ } else {
1778
+ console.log('No stale errors to auto-close');
1779
+ }
1780
+
1781
+ // Signal scheduled run success to Gatus heartbeat (must always execute)
1782
+ pingHeartbeat(ctx, env.GATUS_HEARTBEAT_URL_15M, env.GATUS_TOKEN, true);
1783
+ }
1784
+
1785
+ /**
1786
+ * Verify GitHub webhook signature
1787
+ */
1788
+ async function verifyWebhookSignature(
1789
+ payload: string,
1790
+ signature: string | null,
1791
+ secret: string
1792
+ ): Promise<boolean> {
1793
+ if (!signature) return false;
1794
+
1795
+ const encoder = new TextEncoder();
1796
+ const key = await crypto.subtle.importKey(
1797
+ 'raw',
1798
+ encoder.encode(secret),
1799
+ { name: 'HMAC', hash: 'SHA-256' },
1800
+ false,
1801
+ ['sign']
1802
+ );
1803
+
1804
+ const signatureBuffer = await crypto.subtle.sign('HMAC', key, encoder.encode(payload));
1805
+
1806
+ const expectedSignature =
1807
+ 'sha256=' +
1808
+ Array.from(new Uint8Array(signatureBuffer))
1809
+ .map((b) => b.toString(16).padStart(2, '0'))
1810
+ .join('');
1811
+
1812
+ return signature === expectedSignature;
1813
+ }
1814
+
1815
+ /**
1816
+ * Extract fingerprint from issue body
1817
+ */
1818
+ function extractFingerprint(body: string): string | null {
1819
+ const match = body.match(/Fingerprint: `([a-f0-9]+)`/);
1820
+ return match ? match[1] : null;
1821
+ }
1822
+
1823
+ /**
1824
+ * GitHub issue event payload
1825
+ */
1826
+ interface GitHubIssueEvent {
1827
+ action: 'closed' | 'reopened' | 'labeled' | 'unlabeled' | 'opened' | 'edited';
1828
+ issue: {
1829
+ number: number;
1830
+ state: 'open' | 'closed';
1831
+ labels: Array<{ name: string }>;
1832
+ body: string | null;
1833
+ html_url: string;
1834
+ };
1835
+ repository: {
1836
+ full_name: string;
1837
+ };
1838
+ label?: {
1839
+ name: string;
1840
+ };
1841
+ sender: {
1842
+ login: string;
1843
+ type: 'User' | 'Bot';
1844
+ };
1845
+ }
1846
+
1847
+ /**
1848
+ * Handle GitHub webhook for bidirectional sync
1849
+ */
1850
+ async function handleGitHubWebhook(
1851
+ event: GitHubIssueEvent,
1852
+ env: Env
1853
+ ): Promise<{ processed: boolean; message: string }> {
1854
+ // Ignore events from our own bot
1855
+ if (event.sender.type === 'Bot' && event.sender.login.includes('error-collector')) {
1856
+ return { processed: false, message: 'Ignoring bot event' };
1857
+ }
1858
+
1859
+ // Only process if issue has our auto-generated label
1860
+ const hasAutoLabel = event.issue.labels.some((l) => l.name === 'cf:error:auto-generated');
1861
+ if (!hasAutoLabel) {
1862
+ return { processed: false, message: 'Not an auto-generated error issue' };
1863
+ }
1864
+
1865
+ // Extract fingerprint from issue body
1866
+ const fingerprint = extractFingerprint(event.issue.body || '');
1867
+ if (!fingerprint) {
1868
+ return { processed: false, message: 'Could not extract fingerprint' };
1869
+ }
1870
+
1871
+ const repo = event.repository.full_name;
1872
+
1873
+ if (event.action === 'closed') {
1874
+ // Issue was closed - mark as resolved
1875
+ await env.PLATFORM_DB.prepare(
1876
+ `
1877
+ UPDATE error_occurrences
1878
+ SET status = 'resolved',
1879
+ resolved_at = unixepoch(),
1880
+ resolved_by = ?,
1881
+ updated_at = unixepoch()
1882
+ WHERE fingerprint = ?
1883
+ AND github_repo = ?
1884
+ `
1885
+ )
1886
+ .bind(`github:${event.sender.login}`, fingerprint, repo)
1887
+ .run();
1888
+
1889
+ // Update KV cache
1890
+ const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
1891
+ const cached = await env.PLATFORM_CACHE.get(kvKey);
1892
+ if (cached) {
1893
+ const data = JSON.parse(cached);
1894
+ await env.PLATFORM_CACHE.put(
1895
+ kvKey,
1896
+ JSON.stringify({
1897
+ ...data,
1898
+ status: 'resolved',
1899
+ }),
1900
+ { expirationTtl: 90 * 24 * 60 * 60 }
1901
+ );
1902
+ }
1903
+
1904
+ console.log(`Marked ${fingerprint} as resolved via GitHub (closed by ${event.sender.login})`);
1905
+ return { processed: true, message: `Marked as resolved by ${event.sender.login}` };
1906
+ }
1907
+
1908
+ if (event.action === 'reopened') {
1909
+ // Issue was reopened - clear resolved status
1910
+ await env.PLATFORM_DB.prepare(
1911
+ `
1912
+ UPDATE error_occurrences
1913
+ SET status = 'open',
1914
+ resolved_at = NULL,
1915
+ resolved_by = NULL,
1916
+ updated_at = unixepoch()
1917
+ WHERE fingerprint = ?
1918
+ AND github_repo = ?
1919
+ `
1920
+ )
1921
+ .bind(fingerprint, repo)
1922
+ .run();
1923
+
1924
+ // Update KV cache
1925
+ const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
1926
+ const cached = await env.PLATFORM_CACHE.get(kvKey);
1927
+ if (cached) {
1928
+ const data = JSON.parse(cached);
1929
+ await env.PLATFORM_CACHE.put(
1930
+ kvKey,
1931
+ JSON.stringify({
1932
+ ...data,
1933
+ status: 'open',
1934
+ }),
1935
+ { expirationTtl: 90 * 24 * 60 * 60 }
1936
+ );
1937
+ }
1938
+
1939
+ console.log(`Marked ${fingerprint} as open via GitHub (reopened by ${event.sender.login})`);
1940
+ return { processed: true, message: `Reopened by ${event.sender.login}` };
1941
+ }
1942
+
1943
+ if (event.action === 'labeled' && event.label) {
1944
+ // Check for specific status-changing labels
1945
+ if (event.label.name === 'cf:wont-fix') {
1946
+ await env.PLATFORM_DB.prepare(
1947
+ `
1948
+ UPDATE error_occurrences
1949
+ SET status = 'wont_fix',
1950
+ resolved_at = unixepoch(),
1951
+ resolved_by = ?,
1952
+ updated_at = unixepoch()
1953
+ WHERE fingerprint = ?
1954
+ AND github_repo = ?
1955
+ `
1956
+ )
1957
+ .bind(`github:${event.sender.login}`, fingerprint, repo)
1958
+ .run();
1959
+
1960
+ console.log(`Marked ${fingerprint} as won't fix`);
1961
+ return { processed: true, message: `Marked as won't fix` };
1962
+ }
1963
+ }
1964
+
1965
+ return { processed: false, message: 'No action taken' };
1966
+ }
1967
+
1968
+ // ============================================================================
1969
+ // Dashboard API Handlers
1970
+ // ============================================================================
1971
+
1972
+ /**
1973
+ * List errors with optional filtering
1974
+ */
1975
+ async function handleListErrors(
1976
+ request: Request,
1977
+ env: Env
1978
+ ): Promise<Response> {
1979
+ const url = new URL(request.url);
1980
+ const script = url.searchParams.get('script');
1981
+ const priority = url.searchParams.get('priority');
1982
+ const status = url.searchParams.get('status');
1983
+ const project = url.searchParams.get('project');
1984
+ const errorType = url.searchParams.get('error_type');
1985
+ const dateFrom = url.searchParams.get('date_from');
1986
+ const dateTo = url.searchParams.get('date_to');
1987
+ const limit = Math.min(parseInt(url.searchParams.get('limit') || '50', 10), 100);
1988
+ const offset = parseInt(url.searchParams.get('offset') || '0', 10);
1989
+
1990
+ // Sorting - validated against allowed columns to prevent SQL injection
1991
+ const allowedSortColumns = ['priority', 'script_name', 'status', 'occurrence_count', 'last_seen_at', 'first_seen_at', 'project'];
1992
+ const sortByParam = url.searchParams.get('sort_by');
1993
+ const sortBy = allowedSortColumns.includes(sortByParam || '') ? sortByParam : 'last_seen_at';
1994
+ const sortOrderParam = url.searchParams.get('sort_order')?.toLowerCase();
1995
+ const sortOrder = sortOrderParam === 'asc' ? 'ASC' : 'DESC';
1996
+
1997
+ // Build dynamic WHERE clause
1998
+ const conditions: string[] = [];
1999
+ const bindings: (string | number)[] = [];
2000
+
2001
+ if (script) {
2002
+ conditions.push('script_name = ?');
2003
+ bindings.push(script);
2004
+ }
2005
+ if (priority) {
2006
+ conditions.push('priority = ?');
2007
+ bindings.push(priority);
2008
+ }
2009
+ if (status) {
2010
+ conditions.push('status = ?');
2011
+ bindings.push(status);
2012
+ }
2013
+ if (project) {
2014
+ conditions.push('project = ?');
2015
+ bindings.push(project);
2016
+ }
2017
+ if (errorType) {
2018
+ conditions.push('error_type = ?');
2019
+ bindings.push(errorType);
2020
+ }
2021
+ if (dateFrom) {
2022
+ const fromTs = Math.floor(new Date(dateFrom).getTime() / 1000);
2023
+ conditions.push('last_seen_at >= ?');
2024
+ bindings.push(fromTs);
2025
+ }
2026
+ if (dateTo) {
2027
+ const toTs = Math.floor(new Date(dateTo).getTime() / 1000);
2028
+ conditions.push('last_seen_at <= ?');
2029
+ bindings.push(toTs);
2030
+ }
2031
+
2032
+ const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
2033
+
2034
+ // Count total
2035
+ const countQuery = `SELECT COUNT(*) as total FROM error_occurrences ${whereClause}`;
2036
+ const countResult = await env.PLATFORM_DB.prepare(countQuery)
2037
+ .bind(...bindings)
2038
+ .first<{ total: number }>();
2039
+
2040
+ // Fetch errors
2041
+ const query = `
2042
+ SELECT
2043
+ id, fingerprint, script_name, project, error_type, priority,
2044
+ github_issue_number, github_issue_url, github_repo,
2045
+ status, resolved_at, resolved_by,
2046
+ first_seen_at, last_seen_at, occurrence_count,
2047
+ last_request_url, last_request_method, last_colo, last_country, last_cf_ray,
2048
+ last_exception_name, last_exception_message,
2049
+ normalized_message, error_category
2050
+ FROM error_occurrences
2051
+ ${whereClause}
2052
+ ORDER BY ${sortBy} ${sortOrder}
2053
+ LIMIT ? OFFSET ?
2054
+ `;
2055
+
2056
+ const result = await env.PLATFORM_DB.prepare(query)
2057
+ .bind(...bindings, limit, offset)
2058
+ .all();
2059
+
2060
+ return new Response(JSON.stringify({
2061
+ errors: result.results || [],
2062
+ total: countResult?.total || 0,
2063
+ limit,
2064
+ offset,
2065
+ }), {
2066
+ headers: { 'Content-Type': 'application/json' },
2067
+ });
2068
+ }
2069
+
2070
+ /**
2071
+ * Get error statistics for dashboard overview
2072
+ */
2073
+ async function handleErrorStats(env: Env): Promise<Response> {
2074
+ // Counts by priority
2075
+ const priorityCounts = await env.PLATFORM_DB.prepare(`
2076
+ SELECT priority, COUNT(*) as count
2077
+ FROM error_occurrences
2078
+ WHERE status = 'open'
2079
+ GROUP BY priority
2080
+ `).all<{ priority: string; count: number }>();
2081
+
2082
+ // Counts by status
2083
+ const statusCounts = await env.PLATFORM_DB.prepare(`
2084
+ SELECT status, COUNT(*) as count
2085
+ FROM error_occurrences
2086
+ GROUP BY status
2087
+ `).all<{ status: string; count: number }>();
2088
+
2089
+ // Counts by error type
2090
+ const typeCounts = await env.PLATFORM_DB.prepare(`
2091
+ SELECT error_type, COUNT(*) as count
2092
+ FROM error_occurrences
2093
+ WHERE status = 'open'
2094
+ GROUP BY error_type
2095
+ `).all<{ error_type: string; count: number }>();
2096
+
2097
+ // Counts by project
2098
+ const projectCounts = await env.PLATFORM_DB.prepare(`
2099
+ SELECT project, COUNT(*) as count
2100
+ FROM error_occurrences
2101
+ WHERE status = 'open'
2102
+ GROUP BY project
2103
+ `).all<{ project: string; count: number }>();
2104
+
2105
+ // Recent errors (last 24h)
2106
+ const oneDayAgo = Math.floor(Date.now() / 1000) - 86400;
2107
+ const recentCount = await env.PLATFORM_DB.prepare(`
2108
+ SELECT COUNT(*) as count
2109
+ FROM error_occurrences
2110
+ WHERE last_seen_at > ?
2111
+ `).bind(oneDayAgo).first<{ count: number }>();
2112
+
2113
+ // Transient error categories
2114
+ const transientCounts = await env.PLATFORM_DB.prepare(`
2115
+ SELECT error_category, COUNT(DISTINCT fingerprint) as count
2116
+ FROM error_occurrences
2117
+ WHERE error_category IS NOT NULL AND status = 'open'
2118
+ GROUP BY error_category
2119
+ `).all<{ error_category: string; count: number }>();
2120
+
2121
+ // Total occurrences today
2122
+ const todayStart = Math.floor(new Date().setUTCHours(0, 0, 0, 0) / 1000);
2123
+ const todayOccurrences = await env.PLATFORM_DB.prepare(`
2124
+ SELECT SUM(occurrence_count) as total
2125
+ FROM error_occurrences
2126
+ WHERE last_seen_at >= ?
2127
+ `).bind(todayStart).first<{ total: number }>();
2128
+
2129
+ // Build stats object
2130
+ const byPriority: Record<string, number> = {};
2131
+ for (const row of priorityCounts.results || []) {
2132
+ byPriority[row.priority] = row.count;
2133
+ }
2134
+
2135
+ const byStatus: Record<string, number> = {};
2136
+ for (const row of statusCounts.results || []) {
2137
+ byStatus[row.status] = row.count;
2138
+ }
2139
+
2140
+ const byType: Record<string, number> = {};
2141
+ for (const row of typeCounts.results || []) {
2142
+ byType[row.error_type] = row.count;
2143
+ }
2144
+
2145
+ const byProject: Record<string, number> = {};
2146
+ for (const row of projectCounts.results || []) {
2147
+ byProject[row.project] = row.count;
2148
+ }
2149
+
2150
+ const byTransientCategory: Record<string, number> = {};
2151
+ for (const row of transientCounts.results || []) {
2152
+ byTransientCategory[row.error_category] = row.count;
2153
+ }
2154
+
2155
+ return new Response(JSON.stringify({
2156
+ byPriority,
2157
+ byStatus,
2158
+ byType,
2159
+ byProject,
2160
+ byTransientCategory,
2161
+ recentCount: recentCount?.count || 0,
2162
+ todayOccurrences: todayOccurrences?.total || 0,
2163
+ totalOpen: byStatus['open'] || 0,
2164
+ }), {
2165
+ headers: { 'Content-Type': 'application/json' },
2166
+ });
2167
+ }
2168
+
2169
+ // ============================================================================
2170
+ // Warning Digest API Handlers
2171
+ // ============================================================================
2172
+
2173
+ /**
2174
+ * List warning digests with filtering
2175
+ * GET /digests?script=&days=&limit=&offset=
2176
+ */
2177
+ async function handleListDigests(
2178
+ request: Request,
2179
+ env: Env
2180
+ ): Promise<Response> {
2181
+ const url = new URL(request.url);
2182
+ const script = url.searchParams.get('script');
2183
+ const days = parseInt(url.searchParams.get('days') || '30', 10);
2184
+ const limit = Math.min(parseInt(url.searchParams.get('limit') || '50', 10), 200);
2185
+ const offset = parseInt(url.searchParams.get('offset') || '0', 10);
2186
+
2187
+ const cutoffDate = new Date();
2188
+ cutoffDate.setDate(cutoffDate.getDate() - days);
2189
+ const cutoffDateStr = cutoffDate.toISOString().split('T')[0];
2190
+
2191
+ // Build query with optional filters
2192
+ let query = `
2193
+ SELECT
2194
+ id,
2195
+ digest_date,
2196
+ script_name,
2197
+ fingerprint,
2198
+ normalized_message,
2199
+ github_repo,
2200
+ github_issue_number,
2201
+ github_issue_url,
2202
+ occurrence_count,
2203
+ first_occurrence_at,
2204
+ last_occurrence_at,
2205
+ created_at,
2206
+ updated_at
2207
+ FROM warning_digests
2208
+ WHERE digest_date >= ?
2209
+ `;
2210
+ const params: (string | number)[] = [cutoffDateStr];
2211
+
2212
+ if (script) {
2213
+ query += ' AND script_name LIKE ?';
2214
+ params.push(`%${script}%`);
2215
+ }
2216
+
2217
+ // Get total count
2218
+ let countQuery = `SELECT COUNT(*) as total FROM warning_digests WHERE digest_date >= ?`;
2219
+ const countParams: (string | number)[] = [cutoffDateStr];
2220
+ if (script) {
2221
+ countQuery += ' AND script_name LIKE ?';
2222
+ countParams.push(`%${script}%`);
2223
+ }
2224
+
2225
+ const countResult = await env.PLATFORM_DB.prepare(countQuery)
2226
+ .bind(...countParams)
2227
+ .first<{ total: number }>();
2228
+
2229
+ // Add ordering and pagination
2230
+ query += ' ORDER BY digest_date DESC, occurrence_count DESC LIMIT ? OFFSET ?';
2231
+ params.push(limit, offset);
2232
+
2233
+ const result = await env.PLATFORM_DB.prepare(query)
2234
+ .bind(...params)
2235
+ .all();
2236
+
2237
+ return new Response(JSON.stringify({
2238
+ digests: result.results || [],
2239
+ total: countResult?.total || 0,
2240
+ limit,
2241
+ offset,
2242
+ }), {
2243
+ headers: { 'Content-Type': 'application/json' },
2244
+ });
2245
+ }
2246
+
2247
+ /**
2248
+ * Get warning digest statistics
2249
+ * GET /digests/stats
2250
+ */
2251
+ async function handleDigestStats(env: Env): Promise<Response> {
2252
+ // Digests by date (last 14 days)
2253
+ const twoWeeksAgo = new Date();
2254
+ twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14);
2255
+ const cutoffDateStr = twoWeeksAgo.toISOString().split('T')[0];
2256
+
2257
+ const byDate = await env.PLATFORM_DB.prepare(`
2258
+ SELECT digest_date, COUNT(*) as count, SUM(occurrence_count) as occurrences
2259
+ FROM warning_digests
2260
+ WHERE digest_date >= ?
2261
+ GROUP BY digest_date
2262
+ ORDER BY digest_date DESC
2263
+ `).bind(cutoffDateStr).all<{ digest_date: string; count: number; occurrences: number }>();
2264
+
2265
+ // Digests by script (all time, top 10)
2266
+ const byScript = await env.PLATFORM_DB.prepare(`
2267
+ SELECT script_name, COUNT(*) as count, SUM(occurrence_count) as occurrences
2268
+ FROM warning_digests
2269
+ GROUP BY script_name
2270
+ ORDER BY occurrences DESC
2271
+ LIMIT 10
2272
+ `).all<{ script_name: string; count: number; occurrences: number }>();
2273
+
2274
+ // Digests by repo (all time)
2275
+ const byRepo = await env.PLATFORM_DB.prepare(`
2276
+ SELECT github_repo, COUNT(*) as count, SUM(occurrence_count) as occurrences
2277
+ FROM warning_digests
2278
+ GROUP BY github_repo
2279
+ ORDER BY occurrences DESC
2280
+ `).all<{ github_repo: string; count: number; occurrences: number }>();
2281
+
2282
+ // Recent totals
2283
+ const oneDayAgo = new Date();
2284
+ oneDayAgo.setDate(oneDayAgo.getDate() - 1);
2285
+ const yesterday = oneDayAgo.toISOString().split('T')[0];
2286
+
2287
+ const todayDigests = await env.PLATFORM_DB.prepare(`
2288
+ SELECT COUNT(*) as count, COALESCE(SUM(occurrence_count), 0) as occurrences
2289
+ FROM warning_digests
2290
+ WHERE digest_date = date('now')
2291
+ `).first<{ count: number; occurrences: number }>();
2292
+
2293
+ const yesterdayDigests = await env.PLATFORM_DB.prepare(`
2294
+ SELECT COUNT(*) as count, COALESCE(SUM(occurrence_count), 0) as occurrences
2295
+ FROM warning_digests
2296
+ WHERE digest_date = ?
2297
+ `).bind(yesterday).first<{ count: number; occurrences: number }>();
2298
+
2299
+ // Total digests and occurrences
2300
+ const totals = await env.PLATFORM_DB.prepare(`
2301
+ SELECT COUNT(*) as totalDigests, COALESCE(SUM(occurrence_count), 0) as totalOccurrences
2302
+ FROM warning_digests
2303
+ `).first<{ totalDigests: number; totalOccurrences: number }>();
2304
+
2305
+ // Most common warning types (top 5)
2306
+ const topWarnings = await env.PLATFORM_DB.prepare(`
2307
+ SELECT normalized_message, SUM(occurrence_count) as occurrences, COUNT(*) as days_occurred
2308
+ FROM warning_digests
2309
+ GROUP BY normalized_message
2310
+ ORDER BY occurrences DESC
2311
+ LIMIT 5
2312
+ `).all<{ normalized_message: string; occurrences: number; days_occurred: number }>();
2313
+
2314
+ return new Response(JSON.stringify({
2315
+ byDate: byDate.results || [],
2316
+ byScript: byScript.results || [],
2317
+ byRepo: byRepo.results || [],
2318
+ todayDigests: todayDigests || { count: 0, occurrences: 0 },
2319
+ yesterdayDigests: yesterdayDigests || { count: 0, occurrences: 0 },
2320
+ totalDigests: totals?.totalDigests || 0,
2321
+ totalOccurrences: totals?.totalOccurrences || 0,
2322
+ topWarnings: topWarnings.results || [],
2323
+ }), {
2324
+ headers: { 'Content-Type': 'application/json' },
2325
+ });
2326
+ }
2327
+
2328
+ /**
2329
+ * Get single error by fingerprint
2330
+ */
2331
+ async function handleGetError(
2332
+ fingerprint: string,
2333
+ env: Env
2334
+ ): Promise<Response> {
2335
+ const error = await env.PLATFORM_DB.prepare(`
2336
+ SELECT *
2337
+ FROM error_occurrences
2338
+ WHERE fingerprint = ?
2339
+ `).bind(fingerprint).first();
2340
+
2341
+ if (!error) {
2342
+ return new Response(JSON.stringify({ error: 'Error not found' }), {
2343
+ status: 404,
2344
+ headers: { 'Content-Type': 'application/json' },
2345
+ });
2346
+ }
2347
+
2348
+ return new Response(JSON.stringify(error), {
2349
+ headers: { 'Content-Type': 'application/json' },
2350
+ });
2351
+ }
2352
+
2353
+ /**
2354
+ * Mute an error (add cf:muted label to GitHub issue)
2355
+ */
2356
+ async function handleMuteError(
2357
+ fingerprint: string,
2358
+ env: Env
2359
+ ): Promise<Response> {
2360
+ // Get error details
2361
+ const error = await env.PLATFORM_DB.prepare(`
2362
+ SELECT github_issue_number, github_repo
2363
+ FROM error_occurrences
2364
+ WHERE fingerprint = ?
2365
+ `).bind(fingerprint).first<{ github_issue_number: number; github_repo: string }>();
2366
+
2367
+ if (!error || !error.github_issue_number) {
2368
+ return new Response(JSON.stringify({ error: 'Error not found or has no GitHub issue' }), {
2369
+ status: 404,
2370
+ headers: { 'Content-Type': 'application/json' },
2371
+ });
2372
+ }
2373
+
2374
+ try {
2375
+ const github = new GitHubClient(env);
2376
+ const [owner, repo] = error.github_repo.split('/');
2377
+
2378
+ await github.addLabels(owner, repo, error.github_issue_number, ['cf:muted']);
2379
+
2380
+ return new Response(JSON.stringify({
2381
+ success: true,
2382
+ message: `Muted error - added cf:muted label to issue #${error.github_issue_number}`
2383
+ }), {
2384
+ headers: { 'Content-Type': 'application/json' },
2385
+ });
2386
+ } catch (e) {
2387
+ return new Response(JSON.stringify({ error: `Failed to mute: ${e}` }), {
2388
+ status: 500,
2389
+ headers: { 'Content-Type': 'application/json' },
2390
+ });
2391
+ }
2392
+ }
2393
+
2394
+ /**
2395
+ * Resolve an error manually
2396
+ */
2397
+ async function handleResolveError(
2398
+ fingerprint: string,
2399
+ env: Env
2400
+ ): Promise<Response> {
2401
+ // Get error details
2402
+ const error = await env.PLATFORM_DB.prepare(`
2403
+ SELECT github_issue_number, github_repo
2404
+ FROM error_occurrences
2405
+ WHERE fingerprint = ?
2406
+ `).bind(fingerprint).first<{ github_issue_number: number; github_repo: string }>();
2407
+
2408
+ if (!error) {
2409
+ return new Response(JSON.stringify({ error: 'Error not found' }), {
2410
+ status: 404,
2411
+ headers: { 'Content-Type': 'application/json' },
2412
+ });
2413
+ }
2414
+
2415
+ // Update D1
2416
+ await env.PLATFORM_DB.prepare(`
2417
+ UPDATE error_occurrences
2418
+ SET status = 'resolved',
2419
+ resolved_at = unixepoch(),
2420
+ resolved_by = 'dashboard',
2421
+ updated_at = unixepoch()
2422
+ WHERE fingerprint = ?
2423
+ `).bind(fingerprint).run();
2424
+
2425
+ // Update KV cache
2426
+ const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
2427
+ const cached = await env.PLATFORM_CACHE.get(kvKey);
2428
+ if (cached) {
2429
+ const data = JSON.parse(cached);
2430
+ await env.PLATFORM_CACHE.put(
2431
+ kvKey,
2432
+ JSON.stringify({ ...data, status: 'resolved' }),
2433
+ { expirationTtl: 90 * 24 * 60 * 60 }
2434
+ );
2435
+ }
2436
+
2437
+ // Close GitHub issue if exists
2438
+ if (error.github_issue_number) {
2439
+ try {
2440
+ const github = new GitHubClient(env);
2441
+ const [owner, repo] = error.github_repo.split('/');
2442
+
2443
+ await github.updateIssue({
2444
+ owner,
2445
+ repo,
2446
+ issue_number: error.github_issue_number,
2447
+ state: 'closed',
2448
+ });
2449
+
2450
+ await github.addComment(
2451
+ owner,
2452
+ repo,
2453
+ error.github_issue_number,
2454
+ `✅ **Resolved via Dashboard**\n\nThis error was marked as resolved from the Platform Dashboard.`
2455
+ );
2456
+ } catch (e) {
2457
+ console.error(`Failed to close GitHub issue: ${e}`);
2458
+ }
2459
+ }
2460
+
2461
+ return new Response(JSON.stringify({
2462
+ success: true,
2463
+ message: 'Error marked as resolved'
2464
+ }), {
2465
+ headers: { 'Content-Type': 'application/json' },
2466
+ });
2467
+ }
2468
+
2469
+ // ============================================================================
2470
+ // Main HTTP Handler
2471
+ // ============================================================================
2472
+
2473
+ /**
2474
+ * HTTP handler - webhook endpoint for GitHub events + dashboard API
2475
+ */
2476
+ async function fetch(request: Request, env: Env): Promise<Response> {
2477
+ const url = new URL(request.url);
2478
+
2479
+ // Health check
2480
+ if (url.pathname === '/health') {
2481
+ return new Response(JSON.stringify({ status: 'ok', worker: 'error-collector' }), {
2482
+ headers: { 'Content-Type': 'application/json' },
2483
+ });
2484
+ }
2485
+
2486
+ // ============================================================================
2487
+ // Dashboard API endpoints
2488
+ // ============================================================================
2489
+
2490
+ // GET /errors - List errors with filtering
2491
+ if (url.pathname === '/errors' && request.method === 'GET') {
2492
+ return handleListErrors(request, env);
2493
+ }
2494
+
2495
+ // GET /errors/stats - Get error statistics
2496
+ if (url.pathname === '/errors/stats' && request.method === 'GET') {
2497
+ return handleErrorStats(env);
2498
+ }
2499
+
2500
+ // GET /errors/:fingerprint - Get single error
2501
+ const errorMatch = url.pathname.match(/^\/errors\/([a-f0-9]+)$/);
2502
+ if (errorMatch && request.method === 'GET') {
2503
+ return handleGetError(errorMatch[1], env);
2504
+ }
2505
+
2506
+ // POST /errors/:fingerprint/mute - Mute an error
2507
+ const muteMatch = url.pathname.match(/^\/errors\/([a-f0-9]+)\/mute$/);
2508
+ if (muteMatch && request.method === 'POST') {
2509
+ return handleMuteError(muteMatch[1], env);
2510
+ }
2511
+
2512
+ // POST /errors/:fingerprint/resolve - Resolve an error
2513
+ const resolveMatch = url.pathname.match(/^\/errors\/([a-f0-9]+)\/resolve$/);
2514
+ if (resolveMatch && request.method === 'POST') {
2515
+ return handleResolveError(resolveMatch[1], env);
2516
+ }
2517
+
2518
+ // GET /digests - List warning digests with filtering
2519
+ if (url.pathname === '/digests' && request.method === 'GET') {
2520
+ return handleListDigests(request, env);
2521
+ }
2522
+
2523
+ // GET /digests/stats - Get digest statistics
2524
+ if (url.pathname === '/digests/stats' && request.method === 'GET') {
2525
+ return handleDigestStats(env);
2526
+ }
2527
+
2528
+ // POST /gap-alerts - Create GitHub issue for data coverage gap
2529
+ // Called by platform-sentinel when a project has low coverage
2530
+ if (url.pathname === '/gap-alerts' && request.method === 'POST') {
2531
+ try {
2532
+ const event = (await request.json()) as GapAlertEvent;
2533
+
2534
+ // Validate required fields
2535
+ if (!event.project || event.coveragePct === undefined) {
2536
+ return new Response(
2537
+ JSON.stringify({ error: 'Missing required fields: project, coveragePct' }),
2538
+ { status: 400, headers: { 'Content-Type': 'application/json' } }
2539
+ );
2540
+ }
2541
+
2542
+ const result = await processGapAlert(event, env);
2543
+
2544
+ return new Response(JSON.stringify(result), {
2545
+ status: result.processed ? 201 : 200,
2546
+ headers: { 'Content-Type': 'application/json' },
2547
+ });
2548
+ } catch (e) {
2549
+ console.error('Gap alert processing error:', e);
2550
+ return new Response(
2551
+ JSON.stringify({ error: 'Processing failed', details: String(e) }),
2552
+ { status: 500, headers: { 'Content-Type': 'application/json' } }
2553
+ );
2554
+ }
2555
+ }
2556
+
2557
+ // POST /email-health-alerts - Create GitHub issues for email health check failures
2558
+ // Called by platform-email-healthcheck when a brand has failing checks
2559
+ if (url.pathname === '/email-health-alerts' && request.method === 'POST') {
2560
+ try {
2561
+ const event = (await request.json()) as EmailHealthAlertEvent;
2562
+
2563
+ // Validate required fields
2564
+ if (!event.brand_id || !event.failures?.length || !event.repository) {
2565
+ return new Response(
2566
+ JSON.stringify({ error: 'Missing required fields: brand_id, failures, repository' }),
2567
+ { status: 400, headers: { 'Content-Type': 'application/json' } }
2568
+ );
2569
+ }
2570
+
2571
+ const result = await processEmailHealthAlerts(event, env);
2572
+
2573
+ return new Response(JSON.stringify(result), {
2574
+ status: result.processed > 0 ? 201 : 200,
2575
+ headers: { 'Content-Type': 'application/json' },
2576
+ });
2577
+ } catch (e) {
2578
+ console.error('Email health alert processing error:', e);
2579
+ return new Response(
2580
+ JSON.stringify({ error: 'Processing failed', details: String(e) }),
2581
+ { status: 500, headers: { 'Content-Type': 'application/json' } }
2582
+ );
2583
+ }
2584
+ }
2585
+
2586
+ // Test error endpoint - triggers an intentional error for testing the error collection pipeline
2587
+ // Usage: GET /test-error?type=exception|soft|warning
2588
+ if (url.pathname === '/test-error') {
2589
+ const errorType = url.searchParams.get('type') || 'exception';
2590
+
2591
+ if (errorType === 'soft') {
2592
+ console.error(
2593
+ 'TEST SOFT ERROR: This is a test soft error triggered via /test-error endpoint'
2594
+ );
2595
+ return new Response(JSON.stringify({ triggered: 'soft_error' }), {
2596
+ headers: { 'Content-Type': 'application/json' },
2597
+ });
2598
+ }
2599
+
2600
+ if (errorType === 'warning') {
2601
+ console.warn('TEST WARNING: This is a test warning triggered via /test-error endpoint');
2602
+ return new Response(JSON.stringify({ triggered: 'warning' }), {
2603
+ headers: { 'Content-Type': 'application/json' },
2604
+ });
2605
+ }
2606
+
2607
+ // Default: throw an exception
2608
+ throw new Error('TEST EXCEPTION: This is a test exception triggered via /test-error endpoint');
2609
+ }
2610
+
2611
+ // GitHub webhook endpoint
2612
+ if (url.pathname === '/webhooks/github' && request.method === 'POST') {
2613
+ const eventType = request.headers.get('X-GitHub-Event');
2614
+
2615
+ // Handle ping event (sent when webhook is created)
2616
+ if (eventType === 'ping') {
2617
+ console.log('Received GitHub webhook ping');
2618
+ return new Response(JSON.stringify({ message: 'pong' }), {
2619
+ headers: { 'Content-Type': 'application/json' },
2620
+ });
2621
+ }
2622
+
2623
+ // Only handle issue events
2624
+ if (eventType !== 'issues') {
2625
+ return new Response(
2626
+ JSON.stringify({ processed: false, reason: `Event type '${eventType}' not handled` }),
2627
+ {
2628
+ headers: { 'Content-Type': 'application/json' },
2629
+ }
2630
+ );
2631
+ }
2632
+
2633
+ try {
2634
+ const payload = await request.text();
2635
+ const signature = request.headers.get('X-Hub-Signature-256');
2636
+
2637
+ // Verify signature
2638
+ const isValid = await verifyWebhookSignature(payload, signature, env.GITHUB_WEBHOOK_SECRET);
2639
+ if (!isValid) {
2640
+ console.error('Invalid webhook signature');
2641
+ return new Response(JSON.stringify({ error: 'Invalid signature' }), {
2642
+ status: 401,
2643
+ headers: { 'Content-Type': 'application/json' },
2644
+ });
2645
+ }
2646
+
2647
+ // Parse and process issue event
2648
+ const event = JSON.parse(payload) as GitHubIssueEvent;
2649
+ const result = await handleGitHubWebhook(event, env);
2650
+
2651
+ return new Response(JSON.stringify(result), {
2652
+ headers: { 'Content-Type': 'application/json' },
2653
+ });
2654
+ } catch (e) {
2655
+ console.error(`Webhook processing error: ${e}`);
2656
+ return new Response(JSON.stringify({ error: 'Processing failed', details: String(e) }), {
2657
+ status: 500,
2658
+ headers: { 'Content-Type': 'application/json' },
2659
+ });
2660
+ }
2661
+ }
2662
+
2663
+ return new Response('Not Found', { status: 404 });
2664
+ }
2665
+
2666
+ export default {
2667
+ tail,
2668
+ scheduled,
2669
+ fetch,
2670
+ };