@littlebearapps/platform-admin-sdk 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +89 -0
- package/dist/prompts.d.ts +27 -0
- package/dist/prompts.js +80 -0
- package/dist/scaffold.d.ts +5 -0
- package/dist/scaffold.js +65 -0
- package/dist/templates.d.ts +16 -0
- package/dist/templates.js +131 -0
- package/package.json +46 -0
- package/templates/full/migrations/006_pattern_discovery.sql +199 -0
- package/templates/full/migrations/007_notifications_search.sql +127 -0
- package/templates/full/workers/lib/pattern-discovery/ai-prompt.ts +644 -0
- package/templates/full/workers/lib/pattern-discovery/clustering.ts +278 -0
- package/templates/full/workers/lib/pattern-discovery/shadow-evaluation.ts +603 -0
- package/templates/full/workers/lib/pattern-discovery/storage.ts +806 -0
- package/templates/full/workers/lib/pattern-discovery/types.ts +159 -0
- package/templates/full/workers/lib/pattern-discovery/validation.ts +278 -0
- package/templates/full/workers/pattern-discovery.ts +661 -0
- package/templates/full/workers/platform-alert-router.ts +1809 -0
- package/templates/full/workers/platform-notifications.ts +424 -0
- package/templates/full/workers/platform-search.ts +480 -0
- package/templates/full/workers/platform-settings.ts +436 -0
- package/templates/full/wrangler.alert-router.jsonc.hbs +34 -0
- package/templates/full/wrangler.notifications.jsonc.hbs +23 -0
- package/templates/full/wrangler.pattern-discovery.jsonc.hbs +33 -0
- package/templates/full/wrangler.search.jsonc.hbs +16 -0
- package/templates/full/wrangler.settings.jsonc.hbs +23 -0
- package/templates/shared/README.md.hbs +69 -0
- package/templates/shared/config/budgets.yaml.hbs +72 -0
- package/templates/shared/config/services.yaml.hbs +45 -0
- package/templates/shared/migrations/001_core_tables.sql +117 -0
- package/templates/shared/migrations/002_usage_warehouse.sql +830 -0
- package/templates/shared/migrations/003_feature_tracking.sql +250 -0
- package/templates/shared/migrations/004_settings_alerts.sql +452 -0
- package/templates/shared/migrations/seed.sql.hbs +4 -0
- package/templates/shared/package.json.hbs +21 -0
- package/templates/shared/scripts/sync-config.ts +242 -0
- package/templates/shared/tsconfig.json +12 -0
- package/templates/shared/workers/lib/analytics-engine.ts +357 -0
- package/templates/shared/workers/lib/billing.ts +293 -0
- package/templates/shared/workers/lib/circuit-breaker-middleware.ts +25 -0
- package/templates/shared/workers/lib/control.ts +292 -0
- package/templates/shared/workers/lib/economics.ts +368 -0
- package/templates/shared/workers/lib/metrics.ts +103 -0
- package/templates/shared/workers/lib/platform-settings.ts +407 -0
- package/templates/shared/workers/lib/shared/allowances.ts +333 -0
- package/templates/shared/workers/lib/shared/cloudflare.ts +1362 -0
- package/templates/shared/workers/lib/shared/types.ts +58 -0
- package/templates/shared/workers/lib/telemetry-sampling.ts +360 -0
- package/templates/shared/workers/lib/usage/collectors/example.ts +96 -0
- package/templates/shared/workers/lib/usage/collectors/index.ts +128 -0
- package/templates/shared/workers/lib/usage/handlers/audit.ts +306 -0
- package/templates/shared/workers/lib/usage/handlers/backfill.ts +845 -0
- package/templates/shared/workers/lib/usage/handlers/behavioral.ts +429 -0
- package/templates/shared/workers/lib/usage/handlers/data-queries.ts +507 -0
- package/templates/shared/workers/lib/usage/handlers/dlq-admin.ts +364 -0
- package/templates/shared/workers/lib/usage/handlers/health-trends.ts +222 -0
- package/templates/shared/workers/lib/usage/handlers/index.ts +35 -0
- package/templates/shared/workers/lib/usage/handlers/usage-admin.ts +421 -0
- package/templates/shared/workers/lib/usage/handlers/usage-features.ts +1262 -0
- package/templates/shared/workers/lib/usage/handlers/usage-metrics.ts +2420 -0
- package/templates/shared/workers/lib/usage/handlers/usage-settings.ts +610 -0
- package/templates/shared/workers/lib/usage/queue/budget-enforcement.ts +1032 -0
- package/templates/shared/workers/lib/usage/queue/cost-budget-enforcement.ts +128 -0
- package/templates/shared/workers/lib/usage/queue/cost-calculator.ts +77 -0
- package/templates/shared/workers/lib/usage/queue/dlq-handler.ts +161 -0
- package/templates/shared/workers/lib/usage/queue/index.ts +19 -0
- package/templates/shared/workers/lib/usage/queue/telemetry-processor.ts +790 -0
- package/templates/shared/workers/lib/usage/scheduled/anomaly-detection.ts +732 -0
- package/templates/shared/workers/lib/usage/scheduled/data-collection.ts +956 -0
- package/templates/shared/workers/lib/usage/scheduled/error-digest.ts +343 -0
- package/templates/shared/workers/lib/usage/scheduled/index.ts +18 -0
- package/templates/shared/workers/lib/usage/scheduled/rollups.ts +1561 -0
- package/templates/shared/workers/lib/usage/shared/constants.ts +362 -0
- package/templates/shared/workers/lib/usage/shared/index.ts +14 -0
- package/templates/shared/workers/lib/usage/shared/types.ts +1066 -0
- package/templates/shared/workers/lib/usage/shared/utils.ts +795 -0
- package/templates/shared/workers/platform-usage.ts +1915 -0
- package/templates/shared/wrangler.usage.jsonc.hbs +58 -0
- package/templates/standard/migrations/005_error_collection.sql +162 -0
- package/templates/standard/workers/error-collector.ts +2670 -0
- package/templates/standard/workers/lib/error-collector/capture.ts +213 -0
- package/templates/standard/workers/lib/error-collector/digest.ts +448 -0
- package/templates/standard/workers/lib/error-collector/email-health-alerts.ts +262 -0
- package/templates/standard/workers/lib/error-collector/fingerprint.ts +258 -0
- package/templates/standard/workers/lib/error-collector/gap-alerts.ts +293 -0
- package/templates/standard/workers/lib/error-collector/github.ts +329 -0
- package/templates/standard/workers/lib/error-collector/types.ts +262 -0
- package/templates/standard/workers/lib/sentinel/gap-detection.ts +734 -0
- package/templates/standard/workers/lib/shared/slack-alerts.ts +585 -0
- package/templates/standard/workers/platform-sentinel.ts +1744 -0
- package/templates/standard/wrangler.error-collector.jsonc.hbs +44 -0
- package/templates/standard/wrangler.sentinel.jsonc.hbs +45 -0
|
@@ -0,0 +1,2670 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Error Collector Tail Worker
|
|
3
|
+
*
|
|
4
|
+
* Captures errors from Cloudflare Workers across your projects and creates
|
|
5
|
+
* AI-agent-ready GitHub issues for investigation.
|
|
6
|
+
*
|
|
7
|
+
* @see docs/plans/2026-01-28-error-collector-tail-worker-design.md
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type {
|
|
11
|
+
TailEvent,
|
|
12
|
+
Env,
|
|
13
|
+
ScriptMapping,
|
|
14
|
+
ErrorType,
|
|
15
|
+
ErrorStatus,
|
|
16
|
+
GitHubIssueType,
|
|
17
|
+
} from './lib/error-collector/types';
|
|
18
|
+
import {
|
|
19
|
+
shouldCapture,
|
|
20
|
+
calculatePriority,
|
|
21
|
+
getLabels,
|
|
22
|
+
formatErrorTitle,
|
|
23
|
+
extractCoreMessage,
|
|
24
|
+
normalizeUrl,
|
|
25
|
+
} from './lib/error-collector/capture';
|
|
26
|
+
import {
|
|
27
|
+
computeFingerprint,
|
|
28
|
+
generateId,
|
|
29
|
+
normalizeDynamicValues,
|
|
30
|
+
isTransientError,
|
|
31
|
+
classifyError,
|
|
32
|
+
classifyErrorWithSource,
|
|
33
|
+
loadDynamicPatterns,
|
|
34
|
+
type FingerprintResult,
|
|
35
|
+
type CompiledPattern,
|
|
36
|
+
} from './lib/error-collector/fingerprint';
|
|
37
|
+
import { GitHubClient } from './lib/error-collector/github';
|
|
38
|
+
import { processWarningDigests, storeWarningForDigest } from './lib/error-collector/digest';
|
|
39
|
+
import { processGapAlert } from './lib/error-collector/gap-alerts';
|
|
40
|
+
import { processEmailHealthAlerts } from './lib/error-collector/email-health-alerts';
|
|
41
|
+
import { recordPatternMatchEvidence } from './lib/pattern-discovery/storage';
|
|
42
|
+
import type { GapAlertEvent, EmailHealthAlertEvent } from './lib/error-collector/types';
|
|
43
|
+
import { pingHeartbeat } from '@littlebearapps/platform-consumer-sdk';
|
|
44
|
+
|
|
45
|
+
// TODO: Set your GitHub organisation name
|
|
46
|
+
const GITHUB_ORG = 'your-github-org';
|
|
47
|
+
|
|
48
|
+
// Rate limit: max issues per script per hour
|
|
49
|
+
const MAX_ISSUES_PER_SCRIPT_PER_HOUR = 10;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Map error type to GitHub issue type
|
|
53
|
+
* - Exceptions, CPU/memory limits, soft errors → Bug
|
|
54
|
+
* - Warnings → Task
|
|
55
|
+
*/
|
|
56
|
+
function getGitHubIssueType(errorType: ErrorType): GitHubIssueType {
|
|
57
|
+
if (errorType === 'warning') {
|
|
58
|
+
return 'Task';
|
|
59
|
+
}
|
|
60
|
+
return 'Bug';
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Look up script mapping from KV
|
|
65
|
+
*/
|
|
66
|
+
async function getScriptMapping(
|
|
67
|
+
kv: KVNamespace,
|
|
68
|
+
scriptName: string
|
|
69
|
+
): Promise<ScriptMapping | null> {
|
|
70
|
+
const key = `SCRIPT_MAP:${scriptName}`;
|
|
71
|
+
const value = await kv.get(key);
|
|
72
|
+
|
|
73
|
+
if (!value) return null;
|
|
74
|
+
|
|
75
|
+
try {
|
|
76
|
+
return JSON.parse(value) as ScriptMapping;
|
|
77
|
+
} catch {
|
|
78
|
+
console.error(`Invalid script mapping for ${scriptName}`);
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Check and update rate limit
|
|
85
|
+
* Returns true if within limits, false if rate limited
|
|
86
|
+
*/
|
|
87
|
+
async function checkRateLimit(kv: KVNamespace, scriptName: string): Promise<boolean> {
|
|
88
|
+
const hour = Math.floor(Date.now() / (1000 * 60 * 60));
|
|
89
|
+
const key = `ERROR_RATE:${scriptName}:${hour}`;
|
|
90
|
+
|
|
91
|
+
const current = await kv.get(key);
|
|
92
|
+
const count = current ? parseInt(current, 10) : 0;
|
|
93
|
+
|
|
94
|
+
if (count >= MAX_ISSUES_PER_SCRIPT_PER_HOUR) {
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
await kv.put(key, String(count + 1), { expirationTtl: 7200 }); // 2 hour TTL
|
|
99
|
+
return true;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Get today's date key in YYYY-MM-DD format (UTC)
|
|
104
|
+
*/
|
|
105
|
+
function getDateKey(): string {
|
|
106
|
+
return new Date().toISOString().slice(0, 10);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Check if a transient error already has an issue created today.
|
|
111
|
+
* For transient errors (quota exhaustion, rate limits, etc.), we only
|
|
112
|
+
* create one issue per 24-hour window to avoid noise.
|
|
113
|
+
*
|
|
114
|
+
* Returns the existing issue number if one exists, null otherwise.
|
|
115
|
+
*/
|
|
116
|
+
async function checkTransientErrorWindow(
|
|
117
|
+
kv: KVNamespace,
|
|
118
|
+
scriptName: string,
|
|
119
|
+
category: string
|
|
120
|
+
): Promise<number | null> {
|
|
121
|
+
const windowKey = `TRANSIENT:${scriptName}:${category}:${getDateKey()}`;
|
|
122
|
+
const existing = await kv.get(windowKey);
|
|
123
|
+
return existing ? parseInt(existing, 10) : null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Record that a transient error issue was created for today's window.
|
|
128
|
+
*/
|
|
129
|
+
async function setTransientErrorWindow(
|
|
130
|
+
kv: KVNamespace,
|
|
131
|
+
scriptName: string,
|
|
132
|
+
category: string,
|
|
133
|
+
issueNumber: number
|
|
134
|
+
): Promise<void> {
|
|
135
|
+
const windowKey = `TRANSIENT:${scriptName}:${category}:${getDateKey()}`;
|
|
136
|
+
// TTL of 25 hours to cover the full day plus buffer
|
|
137
|
+
await kv.put(windowKey, String(issueNumber), { expirationTtl: 90000 });
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Check if an issue is muted via the cf:muted label.
|
|
142
|
+
* Muted issues should not be reopened or receive comments.
|
|
143
|
+
*/
|
|
144
|
+
async function isIssueMuted(
|
|
145
|
+
github: GitHubClient,
|
|
146
|
+
owner: string,
|
|
147
|
+
repo: string,
|
|
148
|
+
issueNumber: number
|
|
149
|
+
): Promise<boolean> {
|
|
150
|
+
try {
|
|
151
|
+
const issue = await github.getIssue(owner, repo, issueNumber);
|
|
152
|
+
return (
|
|
153
|
+
issue.labels?.some(
|
|
154
|
+
(l: { name: string } | string) => (typeof l === 'string' ? l : l.name) === 'cf:muted'
|
|
155
|
+
) ?? false
|
|
156
|
+
);
|
|
157
|
+
} catch {
|
|
158
|
+
// If we can't check, assume not muted
|
|
159
|
+
return false;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Labels that prevent issue reopening */
|
|
164
|
+
const SKIP_REOPEN_LABELS = ['cf:muted', 'cf:wont-fix'];
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Acquire an optimistic lock for issue creation.
|
|
168
|
+
* Prevents race conditions where concurrent workers create duplicate issues.
|
|
169
|
+
*
|
|
170
|
+
* @returns true if lock acquired, false if another worker holds it
|
|
171
|
+
*/
|
|
172
|
+
async function acquireIssueLock(kv: KVNamespace, fingerprint: string): Promise<boolean> {
|
|
173
|
+
const lockKey = `ISSUE_LOCK:${fingerprint}`;
|
|
174
|
+
const existing = await kv.get(lockKey);
|
|
175
|
+
|
|
176
|
+
if (existing) {
|
|
177
|
+
// Another worker is creating an issue for this fingerprint
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Set lock with 60s TTL (enough time for GitHub API calls)
|
|
182
|
+
await kv.put(lockKey, Date.now().toString(), { expirationTtl: 60 });
|
|
183
|
+
return true;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Release the issue creation lock.
|
|
188
|
+
*/
|
|
189
|
+
async function releaseIssueLock(kv: KVNamespace, fingerprint: string): Promise<void> {
|
|
190
|
+
const lockKey = `ISSUE_LOCK:${fingerprint}`;
|
|
191
|
+
await kv.delete(lockKey);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Create a dashboard notification for P0-P2 errors.
|
|
196
|
+
* Non-blocking - failures are logged but don't affect issue creation.
|
|
197
|
+
*/
|
|
198
|
+
async function createDashboardNotification(
|
|
199
|
+
api: Fetcher | undefined,
|
|
200
|
+
priority: string, // 'P0', 'P1', 'P2', etc.
|
|
201
|
+
errorType: ErrorType,
|
|
202
|
+
scriptName: string,
|
|
203
|
+
message: string,
|
|
204
|
+
issueNumber: number,
|
|
205
|
+
issueUrl: string,
|
|
206
|
+
project: string
|
|
207
|
+
): Promise<void> {
|
|
208
|
+
// Extract numeric priority from string like 'P0', 'P1', etc.
|
|
209
|
+
const priorityNum = parseInt(priority.replace('P', ''), 10);
|
|
210
|
+
|
|
211
|
+
// Only create notifications for P0-P2 errors
|
|
212
|
+
if (priorityNum > 2 || isNaN(priorityNum) || !api) return;
|
|
213
|
+
|
|
214
|
+
const priorityMap: Record<number, 'critical' | 'high' | 'medium'> = {
|
|
215
|
+
0: 'critical',
|
|
216
|
+
1: 'high',
|
|
217
|
+
2: 'medium',
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
try {
|
|
221
|
+
await api.fetch('https://platform-notifications.internal/notifications', {
|
|
222
|
+
method: 'POST',
|
|
223
|
+
headers: { 'Content-Type': 'application/json' },
|
|
224
|
+
body: JSON.stringify({
|
|
225
|
+
category: 'error',
|
|
226
|
+
source: 'error-collector',
|
|
227
|
+
source_id: String(issueNumber),
|
|
228
|
+
title: `${priority} ${errorType}: ${scriptName}`,
|
|
229
|
+
description: message.slice(0, 200),
|
|
230
|
+
priority: priorityMap[priorityNum],
|
|
231
|
+
action_url: issueUrl,
|
|
232
|
+
action_label: 'View Issue',
|
|
233
|
+
project: project || null,
|
|
234
|
+
}),
|
|
235
|
+
});
|
|
236
|
+
} catch (e) {
|
|
237
|
+
// Non-blocking - log and continue
|
|
238
|
+
console.error('Failed to create dashboard notification:', e);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Search GitHub for an existing issue with this fingerprint.
|
|
244
|
+
* Used as a fallback when D1/KV don't have the mapping.
|
|
245
|
+
*
|
|
246
|
+
* @returns Issue details if found, null otherwise
|
|
247
|
+
*/
|
|
248
|
+
async function findExistingIssueByFingerprint(
|
|
249
|
+
github: GitHubClient,
|
|
250
|
+
owner: string,
|
|
251
|
+
repo: string,
|
|
252
|
+
fingerprint: string
|
|
253
|
+
): Promise<{
|
|
254
|
+
number: number;
|
|
255
|
+
state: 'open' | 'closed';
|
|
256
|
+
shouldSkip: boolean; // true if muted/wontfix
|
|
257
|
+
} | null> {
|
|
258
|
+
try {
|
|
259
|
+
// Search for issues containing this fingerprint in the body
|
|
260
|
+
// The fingerprint appears as "Fingerprint: `{hash}`" in the issue body
|
|
261
|
+
const issues = await github.searchIssues(owner, repo, `"Fingerprint: \`${fingerprint}\`" in:body`);
|
|
262
|
+
|
|
263
|
+
if (issues.length === 0) return null;
|
|
264
|
+
|
|
265
|
+
// Prefer open issues over closed
|
|
266
|
+
const openIssue = issues.find((i) => i.state === 'open');
|
|
267
|
+
if (openIssue) {
|
|
268
|
+
const labelNames = openIssue.labels.map((l) => l.name);
|
|
269
|
+
const shouldSkip = SKIP_REOPEN_LABELS.some((l) => labelNames.includes(l));
|
|
270
|
+
return { number: openIssue.number, state: 'open', shouldSkip };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Return most recent closed issue (search results are sorted by best match)
|
|
274
|
+
const closedIssue = issues[0];
|
|
275
|
+
const labelNames = closedIssue.labels.map((l) => l.name);
|
|
276
|
+
const shouldSkip = SKIP_REOPEN_LABELS.some((l) => labelNames.includes(l));
|
|
277
|
+
|
|
278
|
+
return {
|
|
279
|
+
number: closedIssue.number,
|
|
280
|
+
state: closedIssue.state as 'open' | 'closed',
|
|
281
|
+
shouldSkip,
|
|
282
|
+
};
|
|
283
|
+
} catch (error) {
|
|
284
|
+
// Fail open - if search fails, allow creating a new issue
|
|
285
|
+
console.error(`GitHub search failed for fingerprint ${fingerprint}:`, error);
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Format a comment for when an error recurs and we're adding to an existing issue
|
|
292
|
+
*/
|
|
293
|
+
function formatRecurrenceComment(
|
|
294
|
+
event: TailEvent,
|
|
295
|
+
errorType: ErrorType,
|
|
296
|
+
occurrenceCount: number,
|
|
297
|
+
isReopen: boolean
|
|
298
|
+
): string {
|
|
299
|
+
const timestamp = new Date().toISOString();
|
|
300
|
+
const rayId = event.event?.rayId || event.event?.request?.headers?.['cf-ray'];
|
|
301
|
+
|
|
302
|
+
let comment = isReopen ? `## Error Recurrence (Reopened)\n\n` : `## New Occurrence\n\n`;
|
|
303
|
+
|
|
304
|
+
comment += `| | |\n|---|---|\n`;
|
|
305
|
+
comment += `| **Time** | ${timestamp} |\n`;
|
|
306
|
+
comment += `| **Total Occurrences** | ${occurrenceCount} |\n`;
|
|
307
|
+
comment += `| **Worker** | \`${event.scriptName}\` |\n`;
|
|
308
|
+
|
|
309
|
+
if (event.scriptVersion?.id) {
|
|
310
|
+
comment += `| **Version** | \`${event.scriptVersion.id.slice(0, 8)}\` |\n`;
|
|
311
|
+
}
|
|
312
|
+
if (rayId) {
|
|
313
|
+
comment += `| **Ray ID** | \`${rayId}\` |\n`;
|
|
314
|
+
}
|
|
315
|
+
if (event.event?.request?.cf?.colo) {
|
|
316
|
+
comment += `| **Colo** | ${event.event.request.cf.colo} |\n`;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// Include stack trace snippet for exceptions
|
|
320
|
+
if (errorType === 'exception' && event.exceptions.length > 0) {
|
|
321
|
+
const exc = event.exceptions[0];
|
|
322
|
+
const stackPreview = exc.message?.slice(0, 300) || 'N/A';
|
|
323
|
+
comment += `\n### Exception\n\`\`\`\n${exc.name}: ${stackPreview}${exc.message?.length > 300 ? '...' : ''}\n\`\`\`\n`;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
if (isReopen) {
|
|
327
|
+
comment += `\n> This issue was reopened because the error recurred after being closed.\n`;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
return comment;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Get or create error occurrence record
|
|
335
|
+
*/
|
|
336
|
+
async function getOrCreateOccurrence(
|
|
337
|
+
db: D1Database,
|
|
338
|
+
kv: KVNamespace,
|
|
339
|
+
fingerprint: string,
|
|
340
|
+
scriptName: string,
|
|
341
|
+
project: string,
|
|
342
|
+
errorType: ErrorType,
|
|
343
|
+
priority: string,
|
|
344
|
+
repo: string
|
|
345
|
+
): Promise<{
|
|
346
|
+
isNew: boolean;
|
|
347
|
+
occurrence: {
|
|
348
|
+
id: string;
|
|
349
|
+
occurrence_count: number;
|
|
350
|
+
github_issue_number?: number;
|
|
351
|
+
github_issue_url?: string;
|
|
352
|
+
status: ErrorStatus;
|
|
353
|
+
};
|
|
354
|
+
}> {
|
|
355
|
+
// Check KV cache first for existing fingerprint
|
|
356
|
+
const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
|
|
357
|
+
const cached = await kv.get(kvKey);
|
|
358
|
+
|
|
359
|
+
if (cached) {
|
|
360
|
+
const data = JSON.parse(cached) as {
|
|
361
|
+
issueNumber?: number;
|
|
362
|
+
issueUrl?: string;
|
|
363
|
+
status: ErrorStatus;
|
|
364
|
+
occurrenceCount: number;
|
|
365
|
+
};
|
|
366
|
+
|
|
367
|
+
// Update occurrence count in D1
|
|
368
|
+
await db
|
|
369
|
+
.prepare(
|
|
370
|
+
`
|
|
371
|
+
UPDATE error_occurrences
|
|
372
|
+
SET occurrence_count = occurrence_count + 1,
|
|
373
|
+
last_seen_at = unixepoch(),
|
|
374
|
+
updated_at = unixepoch()
|
|
375
|
+
WHERE fingerprint = ?
|
|
376
|
+
`
|
|
377
|
+
)
|
|
378
|
+
.bind(fingerprint)
|
|
379
|
+
.run();
|
|
380
|
+
|
|
381
|
+
// Update KV cache
|
|
382
|
+
await kv.put(
|
|
383
|
+
kvKey,
|
|
384
|
+
JSON.stringify({
|
|
385
|
+
...data,
|
|
386
|
+
occurrenceCount: data.occurrenceCount + 1,
|
|
387
|
+
lastSeen: Date.now(),
|
|
388
|
+
}),
|
|
389
|
+
{ expirationTtl: 90 * 24 * 60 * 60 }
|
|
390
|
+
); // 90 days
|
|
391
|
+
|
|
392
|
+
return {
|
|
393
|
+
isNew: false,
|
|
394
|
+
occurrence: {
|
|
395
|
+
id: fingerprint,
|
|
396
|
+
occurrence_count: data.occurrenceCount + 1,
|
|
397
|
+
github_issue_number: data.issueNumber,
|
|
398
|
+
github_issue_url: data.issueUrl,
|
|
399
|
+
status: data.status,
|
|
400
|
+
},
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
// Check D1 for existing occurrence
|
|
405
|
+
const existing = await db
|
|
406
|
+
.prepare(
|
|
407
|
+
`
|
|
408
|
+
SELECT id, occurrence_count, github_issue_number, github_issue_url, status
|
|
409
|
+
FROM error_occurrences
|
|
410
|
+
WHERE fingerprint = ?
|
|
411
|
+
`
|
|
412
|
+
)
|
|
413
|
+
.bind(fingerprint)
|
|
414
|
+
.first<{
|
|
415
|
+
id: string;
|
|
416
|
+
occurrence_count: number;
|
|
417
|
+
github_issue_number?: number;
|
|
418
|
+
github_issue_url?: string;
|
|
419
|
+
status: ErrorStatus;
|
|
420
|
+
}>();
|
|
421
|
+
|
|
422
|
+
if (existing) {
|
|
423
|
+
// Update occurrence count
|
|
424
|
+
await db
|
|
425
|
+
.prepare(
|
|
426
|
+
`
|
|
427
|
+
UPDATE error_occurrences
|
|
428
|
+
SET occurrence_count = occurrence_count + 1,
|
|
429
|
+
last_seen_at = unixepoch(),
|
|
430
|
+
updated_at = unixepoch()
|
|
431
|
+
WHERE fingerprint = ?
|
|
432
|
+
`
|
|
433
|
+
)
|
|
434
|
+
.bind(fingerprint)
|
|
435
|
+
.run();
|
|
436
|
+
|
|
437
|
+
// Cache in KV
|
|
438
|
+
await kv.put(
|
|
439
|
+
kvKey,
|
|
440
|
+
JSON.stringify({
|
|
441
|
+
issueNumber: existing.github_issue_number,
|
|
442
|
+
issueUrl: existing.github_issue_url,
|
|
443
|
+
status: existing.status,
|
|
444
|
+
occurrenceCount: existing.occurrence_count + 1,
|
|
445
|
+
lastSeen: Date.now(),
|
|
446
|
+
}),
|
|
447
|
+
{ expirationTtl: 90 * 24 * 60 * 60 }
|
|
448
|
+
);
|
|
449
|
+
|
|
450
|
+
return {
|
|
451
|
+
isNew: false,
|
|
452
|
+
occurrence: {
|
|
453
|
+
...existing,
|
|
454
|
+
occurrence_count: existing.occurrence_count + 1,
|
|
455
|
+
},
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
// Create new occurrence with ON CONFLICT to handle race conditions
|
|
460
|
+
// If another concurrent request already created this fingerprint, just update it
|
|
461
|
+
const id = generateId();
|
|
462
|
+
const now = Math.floor(Date.now() / 1000);
|
|
463
|
+
|
|
464
|
+
const result = await db
|
|
465
|
+
.prepare(
|
|
466
|
+
`
|
|
467
|
+
INSERT INTO error_occurrences (
|
|
468
|
+
id, fingerprint, script_name, project, error_type, priority,
|
|
469
|
+
github_repo, status, first_seen_at, last_seen_at, occurrence_count,
|
|
470
|
+
created_at, updated_at
|
|
471
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'open', ?, ?, 1, ?, ?)
|
|
472
|
+
ON CONFLICT(fingerprint) DO UPDATE SET
|
|
473
|
+
occurrence_count = occurrence_count + 1,
|
|
474
|
+
last_seen_at = excluded.last_seen_at,
|
|
475
|
+
updated_at = excluded.updated_at
|
|
476
|
+
RETURNING id, occurrence_count, github_issue_number, github_issue_url, status
|
|
477
|
+
`
|
|
478
|
+
)
|
|
479
|
+
.bind(id, fingerprint, scriptName, project, errorType, priority, repo, now, now, now, now)
|
|
480
|
+
.first<{
|
|
481
|
+
id: string;
|
|
482
|
+
occurrence_count: number;
|
|
483
|
+
github_issue_number?: number;
|
|
484
|
+
github_issue_url?: string;
|
|
485
|
+
status: ErrorStatus;
|
|
486
|
+
}>();
|
|
487
|
+
|
|
488
|
+
// Check if this was an insert (count=1) or update (count>1)
|
|
489
|
+
const isNew = result?.occurrence_count === 1;
|
|
490
|
+
|
|
491
|
+
// Cache in KV
|
|
492
|
+
await kv.put(
|
|
493
|
+
kvKey,
|
|
494
|
+
JSON.stringify({
|
|
495
|
+
issueNumber: result?.github_issue_number,
|
|
496
|
+
issueUrl: result?.github_issue_url,
|
|
497
|
+
status: result?.status || ('open' as ErrorStatus),
|
|
498
|
+
occurrenceCount: result?.occurrence_count || 1,
|
|
499
|
+
firstSeen: Date.now(),
|
|
500
|
+
lastSeen: Date.now(),
|
|
501
|
+
}),
|
|
502
|
+
{ expirationTtl: 90 * 24 * 60 * 60 }
|
|
503
|
+
);
|
|
504
|
+
|
|
505
|
+
return {
|
|
506
|
+
isNew,
|
|
507
|
+
occurrence: {
|
|
508
|
+
id: result?.id || id,
|
|
509
|
+
occurrence_count: result?.occurrence_count || 1,
|
|
510
|
+
github_issue_number: result?.github_issue_number,
|
|
511
|
+
github_issue_url: result?.github_issue_url,
|
|
512
|
+
status: result?.status || ('open' as ErrorStatus),
|
|
513
|
+
},
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Update occurrence with GitHub issue details
|
|
519
|
+
*/
|
|
520
|
+
async function updateOccurrenceWithIssue(
|
|
521
|
+
db: D1Database,
|
|
522
|
+
kv: KVNamespace,
|
|
523
|
+
fingerprint: string,
|
|
524
|
+
issueNumber: number,
|
|
525
|
+
issueUrl: string
|
|
526
|
+
): Promise<void> {
|
|
527
|
+
await db
|
|
528
|
+
.prepare(
|
|
529
|
+
`
|
|
530
|
+
UPDATE error_occurrences
|
|
531
|
+
SET github_issue_number = ?,
|
|
532
|
+
github_issue_url = ?,
|
|
533
|
+
updated_at = unixepoch()
|
|
534
|
+
WHERE fingerprint = ?
|
|
535
|
+
`
|
|
536
|
+
)
|
|
537
|
+
.bind(issueNumber, issueUrl, fingerprint)
|
|
538
|
+
.run();
|
|
539
|
+
|
|
540
|
+
// Update KV cache
|
|
541
|
+
const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
|
|
542
|
+
const cached = await kv.get(kvKey);
|
|
543
|
+
if (cached) {
|
|
544
|
+
const data = JSON.parse(cached);
|
|
545
|
+
await kv.put(
|
|
546
|
+
kvKey,
|
|
547
|
+
JSON.stringify({
|
|
548
|
+
...data,
|
|
549
|
+
issueNumber,
|
|
550
|
+
issueUrl,
|
|
551
|
+
}),
|
|
552
|
+
{ expirationTtl: 90 * 24 * 60 * 60 }
|
|
553
|
+
);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* Extract a normalized message for pattern discovery.
|
|
559
|
+
* Prioritizes: exception message > error logs > warning logs
|
|
560
|
+
* This ensures pattern-discovery can cluster errors even without exceptions.
|
|
561
|
+
*/
|
|
562
|
+
function extractNormalizedMessage(event: TailEvent): string | null {
|
|
563
|
+
// Priority 1: Exception message
|
|
564
|
+
if (event.exceptions[0]?.message) {
|
|
565
|
+
return extractCoreMessage(event.exceptions[0].message);
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// Priority 2: Error-level logs
|
|
569
|
+
const errorLog = event.logs.find((l) => l.level === 'error');
|
|
570
|
+
if (errorLog) {
|
|
571
|
+
return extractCoreMessage(errorLog.message[0]);
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
// Priority 3: Warning-level logs
|
|
575
|
+
const warnLog = event.logs.find((l) => l.level === 'warn');
|
|
576
|
+
if (warnLog) {
|
|
577
|
+
return extractCoreMessage(warnLog.message[0]);
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
return null;
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* Update occurrence with request context
|
|
585
|
+
*/
|
|
586
|
+
async function updateOccurrenceContext(
|
|
587
|
+
db: D1Database,
|
|
588
|
+
fingerprint: string,
|
|
589
|
+
event: TailEvent
|
|
590
|
+
): Promise<void> {
|
|
591
|
+
const url = event.event?.request?.url;
|
|
592
|
+
const method = event.event?.request?.method;
|
|
593
|
+
const colo = event.event?.request?.cf?.colo;
|
|
594
|
+
const country = event.event?.request?.cf?.country;
|
|
595
|
+
const cfRay = event.event?.request?.headers?.['cf-ray'];
|
|
596
|
+
const excName = event.exceptions[0]?.name;
|
|
597
|
+
const logsJson = JSON.stringify(event.logs.slice(-20));
|
|
598
|
+
|
|
599
|
+
// Extract normalized message for pattern discovery (works for all error types)
|
|
600
|
+
const normalizedMessage = extractNormalizedMessage(event);
|
|
601
|
+
|
|
602
|
+
// Use exception message if available, otherwise fall back to normalizedMessage.
|
|
603
|
+
// Soft errors (console.error) have no exceptions[] but DO have error logs —
|
|
604
|
+
// without this fallback, last_exception_message stays NULL and errors are
|
|
605
|
+
// invisible to pattern matching and GitHub issue diagnostics.
|
|
606
|
+
const excMessage = event.exceptions[0]?.message || normalizedMessage;
|
|
607
|
+
|
|
608
|
+
await db
|
|
609
|
+
.prepare(
|
|
610
|
+
`
|
|
611
|
+
UPDATE error_occurrences
|
|
612
|
+
SET last_request_url = ?,
|
|
613
|
+
last_request_method = ?,
|
|
614
|
+
last_colo = ?,
|
|
615
|
+
last_country = ?,
|
|
616
|
+
last_cf_ray = ?,
|
|
617
|
+
last_exception_name = ?,
|
|
618
|
+
last_exception_message = ?,
|
|
619
|
+
last_logs_json = ?,
|
|
620
|
+
normalized_message = COALESCE(?, normalized_message),
|
|
621
|
+
updated_at = unixepoch()
|
|
622
|
+
WHERE fingerprint = ?
|
|
623
|
+
`
|
|
624
|
+
)
|
|
625
|
+
.bind(
|
|
626
|
+
url || null,
|
|
627
|
+
method || null,
|
|
628
|
+
colo || null,
|
|
629
|
+
country || null,
|
|
630
|
+
cfRay || null,
|
|
631
|
+
excName || null,
|
|
632
|
+
excMessage || null,
|
|
633
|
+
logsJson,
|
|
634
|
+
normalizedMessage,
|
|
635
|
+
fingerprint
|
|
636
|
+
)
|
|
637
|
+
.run();
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* Format the GitHub issue body with AI-agent-ready context
|
|
642
|
+
* Includes comprehensive observability data for debugging
|
|
643
|
+
*/
|
|
644
|
+
function formatIssueBody(
|
|
645
|
+
event: TailEvent,
|
|
646
|
+
errorType: ErrorType,
|
|
647
|
+
priority: string,
|
|
648
|
+
mapping: ScriptMapping,
|
|
649
|
+
fingerprint: string,
|
|
650
|
+
occurrenceCount: number
|
|
651
|
+
): string {
|
|
652
|
+
const now = new Date().toISOString();
|
|
653
|
+
const eventTime = new Date(event.eventTimestamp).toISOString();
|
|
654
|
+
const exc = event.exceptions[0];
|
|
655
|
+
const req = event.event?.request;
|
|
656
|
+
const rayId = event.event?.rayId || req?.headers?.['cf-ray'];
|
|
657
|
+
|
|
658
|
+
let body = `## `;
|
|
659
|
+
|
|
660
|
+
// Header based on error type - extract clean message from JSON
|
|
661
|
+
if (errorType === 'exception') {
|
|
662
|
+
body += `🔴 Exception: ${exc?.name || 'Unknown'}: ${exc?.message || 'No message'}\n\n`;
|
|
663
|
+
} else if (errorType === 'cpu_limit') {
|
|
664
|
+
body += `🟠 Exceeded CPU Limit\n\n`;
|
|
665
|
+
} else if (errorType === 'memory_limit') {
|
|
666
|
+
body += `🟠 Exceeded Memory Limit\n\n`;
|
|
667
|
+
} else if (errorType === 'soft_error') {
|
|
668
|
+
const errorLog = event.logs.find((l) => l.level === 'error');
|
|
669
|
+
const cleanMsg = errorLog ? extractCoreMessage(errorLog.message[0]) : 'Unknown';
|
|
670
|
+
body += `🟡 Soft Error: ${cleanMsg}\n\n`;
|
|
671
|
+
} else {
|
|
672
|
+
const warnLog = event.logs.find((l) => l.level === 'warn');
|
|
673
|
+
const cleanMsg = warnLog ? extractCoreMessage(warnLog.message[0]) : 'Unknown';
|
|
674
|
+
body += `⚪ Warning: ${cleanMsg}\n\n`;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
// Summary table for quick context
|
|
678
|
+
body += `| | |\n|---|---|\n`;
|
|
679
|
+
body += `| **Project** | ${mapping.displayName} |\n`;
|
|
680
|
+
body += `| **Worker** | \`${event.scriptName}\` |\n`;
|
|
681
|
+
body += `| **Priority** | ${priority} |\n`;
|
|
682
|
+
body += `| **Tier** | ${mapping.tier} |\n`;
|
|
683
|
+
body += `| **Event Type** | ${event.eventType || 'fetch'} |\n`;
|
|
684
|
+
if (event.scriptVersion?.id) {
|
|
685
|
+
body += `| **Version** | \`${event.scriptVersion.id.slice(0, 8)}\` |\n`;
|
|
686
|
+
}
|
|
687
|
+
body += `| **Outcome** | ${event.outcome} |\n\n`;
|
|
688
|
+
|
|
689
|
+
// Exception details - unescape newlines for readable stack traces
|
|
690
|
+
if (exc) {
|
|
691
|
+
const message = exc.message.replace(/\\n/g, '\n').replace(/\\t/g, '\t');
|
|
692
|
+
body += `### Exception\n\`\`\`\n${exc.name}: ${message}\n\`\`\`\n\n`;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
// Event context based on type
|
|
696
|
+
if (req) {
|
|
697
|
+
// HTTP Request context
|
|
698
|
+
body += `### Request Context\n`;
|
|
699
|
+
body += `| Field | Value |\n`;
|
|
700
|
+
body += `|-------|-------|\n`;
|
|
701
|
+
body += `| **URL** | \`${req.method} ${req.url}\` |\n`;
|
|
702
|
+
if (event.event?.response?.status) {
|
|
703
|
+
body += `| **Response** | ${event.event.response.status} |\n`;
|
|
704
|
+
}
|
|
705
|
+
if (rayId) body += `| **Ray ID** | \`${rayId}\` |\n`;
|
|
706
|
+
if (req.cf?.colo) body += `| **Colo** | ${req.cf.colo} |\n`;
|
|
707
|
+
if (req.cf?.country) {
|
|
708
|
+
const geo = [req.cf.country, req.cf.city, req.cf.region].filter(Boolean).join(', ');
|
|
709
|
+
body += `| **Location** | ${geo} |\n`;
|
|
710
|
+
}
|
|
711
|
+
if (req.cf?.asOrganization)
|
|
712
|
+
body += `| **Network** | ${req.cf.asOrganization} (AS${req.cf.asn}) |\n`;
|
|
713
|
+
if (req.cf?.timezone) body += `| **Timezone** | ${req.cf.timezone} |\n`;
|
|
714
|
+
if (req.headers?.['user-agent']) {
|
|
715
|
+
const ua = req.headers['user-agent'].slice(0, 80);
|
|
716
|
+
body += `| **User Agent** | \`${ua}${req.headers['user-agent'].length > 80 ? '...' : ''}\` |\n`;
|
|
717
|
+
}
|
|
718
|
+
body += `| **Timestamp** | ${eventTime} |\n\n`;
|
|
719
|
+
} else if (event.event?.scheduledTime || event.event?.cron) {
|
|
720
|
+
// Scheduled/Cron context
|
|
721
|
+
body += `### Scheduled Event Context\n`;
|
|
722
|
+
body += `| Field | Value |\n`;
|
|
723
|
+
body += `|-------|-------|\n`;
|
|
724
|
+
if (event.event.cron) body += `| **Cron** | \`${event.event.cron}\` |\n`;
|
|
725
|
+
if (event.event.scheduledTime) {
|
|
726
|
+
body += `| **Scheduled** | ${new Date(event.event.scheduledTime).toISOString()} |\n`;
|
|
727
|
+
}
|
|
728
|
+
body += `| **Executed** | ${eventTime} |\n\n`;
|
|
729
|
+
} else if (event.event?.queue) {
|
|
730
|
+
// Queue context
|
|
731
|
+
body += `### Queue Event Context\n`;
|
|
732
|
+
body += `| Field | Value |\n`;
|
|
733
|
+
body += `|-------|-------|\n`;
|
|
734
|
+
body += `| **Queue** | ${event.event.queue} |\n`;
|
|
735
|
+
if (event.event.batchSize) body += `| **Batch Size** | ${event.event.batchSize} |\n`;
|
|
736
|
+
body += `| **Timestamp** | ${eventTime} |\n\n`;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
// Performance metrics
|
|
740
|
+
body += `### Performance\n`;
|
|
741
|
+
body += `| Metric | Value |\n`;
|
|
742
|
+
body += `|--------|-------|\n`;
|
|
743
|
+
if (event.cpuTime !== undefined) body += `| **CPU Time** | ${event.cpuTime}ms |\n`;
|
|
744
|
+
if (event.wallTime !== undefined) body += `| **Wall Time** | ${event.wallTime}ms |\n`;
|
|
745
|
+
body += `| **Execution** | ${event.executionModel || 'stateless'} |\n\n`;
|
|
746
|
+
|
|
747
|
+
// Categorize logs by level
|
|
748
|
+
const errorLogs = event.logs.filter((l) => l.level === 'error' || l.level === 'warn');
|
|
749
|
+
const debugLogs = event.logs.filter((l) => l.level === 'debug');
|
|
750
|
+
const infoLogs = event.logs.filter((l) => l.level === 'info' || l.level === 'log');
|
|
751
|
+
const allLogs = event.logs;
|
|
752
|
+
|
|
753
|
+
// Show error/warning logs prominently
|
|
754
|
+
if (errorLogs.length > 0) {
|
|
755
|
+
body += `### Error/Warning Logs\n`;
|
|
756
|
+
for (const log of errorLogs.slice(-5)) {
|
|
757
|
+
const rawMsg = log.message
|
|
758
|
+
.map((m) => (typeof m === 'string' ? m : JSON.stringify(m, null, 2)))
|
|
759
|
+
.join(' ');
|
|
760
|
+
body += `\`\`\`\n[${log.level.toUpperCase()}] ${rawMsg}\n\`\`\`\n`;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
// Debug context section - shows debug logs that provide investigation context
|
|
765
|
+
if (debugLogs.length > 0) {
|
|
766
|
+
body += `### 🔍 Debug Context\n`;
|
|
767
|
+
body += `> Debug logs from the same invocation - may contain useful investigation context\n\n`;
|
|
768
|
+
for (const log of debugLogs.slice(-10)) {
|
|
769
|
+
const ts = new Date(log.timestamp).toISOString().split('T')[1].slice(0, 12);
|
|
770
|
+
const rawMsg = log.message
|
|
771
|
+
.map((m) => (typeof m === 'string' ? m : JSON.stringify(m, null, 2)))
|
|
772
|
+
.join(' ')
|
|
773
|
+
.slice(0, 500);
|
|
774
|
+
body += `**${ts}**:\n\`\`\`\n${rawMsg}\n\`\`\`\n`;
|
|
775
|
+
}
|
|
776
|
+
if (debugLogs.length > 10) {
|
|
777
|
+
body += `_...and ${debugLogs.length - 10} more debug entries_\n`;
|
|
778
|
+
}
|
|
779
|
+
body += `\n`;
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
// Full log timeline in collapsible section
|
|
783
|
+
if (allLogs.length > 0) {
|
|
784
|
+
body += `<details>\n<summary>📋 Full Log Timeline (${allLogs.length} entries)</summary>\n\n`;
|
|
785
|
+
body += `| Time | Level | Message |\n`;
|
|
786
|
+
body += `|------|-------|--------|\n`;
|
|
787
|
+
for (const log of allLogs.slice(-30)) {
|
|
788
|
+
const ts = new Date(log.timestamp).toISOString().split('T')[1].slice(0, 12);
|
|
789
|
+
const msg = log.message
|
|
790
|
+
.map((m) => (typeof m === 'string' ? m : JSON.stringify(m)))
|
|
791
|
+
.join(' ')
|
|
792
|
+
.slice(0, 150)
|
|
793
|
+
.replace(/\|/g, '\\|')
|
|
794
|
+
.replace(/\n/g, ' ');
|
|
795
|
+
body += `| ${ts} | ${log.level.toUpperCase()} | ${msg}${log.message.join(' ').length > 150 ? '...' : ''} |\n`;
|
|
796
|
+
}
|
|
797
|
+
if (allLogs.length > 30) {
|
|
798
|
+
body += `| ... | ... | _(${allLogs.length - 30} more entries)_ |\n`;
|
|
799
|
+
}
|
|
800
|
+
body += `\n</details>\n\n`;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
// Tracking metadata
|
|
804
|
+
body += `### Tracking\n`;
|
|
805
|
+
body += `| Field | Value |\n`;
|
|
806
|
+
body += `|-------|-------|\n`;
|
|
807
|
+
body += `| **Fingerprint** | \`${fingerprint}\` |\n`;
|
|
808
|
+
body += `| **First Seen** | ${now} |\n`;
|
|
809
|
+
body += `| **Occurrences** | ${occurrenceCount} |\n\n`;
|
|
810
|
+
|
|
811
|
+
// Quick links with observability deep link
|
|
812
|
+
body += `### Quick Links\n`;
|
|
813
|
+
|
|
814
|
+
// Observability link with Ray ID filter if available
|
|
815
|
+
if (rayId) {
|
|
816
|
+
body += `- 🔍 [View in CF Observability](https://dash.cloudflare.com/?to=/:account/workers/observability/logs?filters=%5B%7B%22key%22%3A%22%24metadata.requestId%22%2C%22operation%22%3A%22eq%22%2C%22value%22%3A%22${rayId}%22%7D%5D) ← **Start here**\n`;
|
|
817
|
+
} else {
|
|
818
|
+
body += `- 🔍 [CF Observability](https://dash.cloudflare.com/?to=/:account/workers/observability)\n`;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
body += `- 📄 [Worker Source](https://github.com/${mapping.repository}/blob/main/workers/${event.scriptName}.ts)\n`;
|
|
822
|
+
body += `- 📊 [Worker Dashboard](https://dash.cloudflare.com/?to=/:account/workers/services/view/${event.scriptName})\n`;
|
|
823
|
+
body += `- 📁 [Repository](https://github.com/${mapping.repository})\n`;
|
|
824
|
+
body += `- 📖 [CLAUDE.md](https://github.com/${mapping.repository}/blob/main/CLAUDE.md)\n`;
|
|
825
|
+
body += `- 🔗 [Related Issues](https://github.com/${mapping.repository}/issues?q=is:issue+label:cf:error+${encodeURIComponent(event.scriptName)})\n\n`;
|
|
826
|
+
|
|
827
|
+
// Suggested investigation with more specific guidance
|
|
828
|
+
body += `### Investigation Steps\n`;
|
|
829
|
+
if (errorType === 'exception') {
|
|
830
|
+
body += `1. **Click the Observability link above** to see the full request trace\n`;
|
|
831
|
+
body += `2. Check the exception and logs for error context\n`;
|
|
832
|
+
body += `3. Open [Worker Source](https://github.com/${mapping.repository}/blob/main/workers/${event.scriptName}.ts) and find the failing code\n`;
|
|
833
|
+
body += `4. Review recent commits: \`git log --oneline -10 workers/${event.scriptName}.ts\`\n`;
|
|
834
|
+
} else if (errorType === 'cpu_limit' || errorType === 'memory_limit') {
|
|
835
|
+
body += `1. **Check Observability** for CPU/memory usage patterns\n`;
|
|
836
|
+
body += `2. Review [Worker Source](https://github.com/${mapping.repository}/blob/main/workers/${event.scriptName}.ts) for loops or heavy computation\n`;
|
|
837
|
+
body += `3. Look for recent changes that increased resource usage\n`;
|
|
838
|
+
body += `4. Consider optimizing, caching, or chunking work\n`;
|
|
839
|
+
} else {
|
|
840
|
+
body += `1. Review the logs above for context\n`;
|
|
841
|
+
body += `2. Search for the log message in [Worker Source](https://github.com/${mapping.repository}/blob/main/workers/${event.scriptName}.ts)\n`;
|
|
842
|
+
body += `3. Determine if this is expected or needs fixing\n`;
|
|
843
|
+
body += `4. Consider adding error handling or validation\n`;
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
// Reference documentation for Claude Code agents
|
|
847
|
+
body += `\n### Reference Documentation\n`;
|
|
848
|
+
body += `- 📚 [Error Collector Integration](https://github.com/${GITHUB_ORG}/platform/blob/main/docs/quickrefs/guides/error-collector-integration.md)\n`;
|
|
849
|
+
body += `- 📚 [Troubleshooting Guide](https://github.com/${GITHUB_ORG}/platform/blob/main/docs/quickrefs/troubleshooting.md)\n`;
|
|
850
|
+
body += `- 📚 [Workers Inventory](https://github.com/${GITHUB_ORG}/platform/blob/main/docs/quickrefs/workers-inventory.md)\n`;
|
|
851
|
+
|
|
852
|
+
body += `\n---\n`;
|
|
853
|
+
body += `_Auto-generated by [Platform Error Collector](https://github.com/${GITHUB_ORG}/platform/blob/main/workers/error-collector.ts)_ | Fingerprint: \`${fingerprint}\`\n`;
|
|
854
|
+
|
|
855
|
+
return body;
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
/**
|
|
859
|
+
* Compute fingerprint for a specific error log (for multi-error processing)
|
|
860
|
+
* Returns FingerprintResult with category for transient error detection
|
|
861
|
+
*/
|
|
862
|
+
async function computeFingerprintForLog(
|
|
863
|
+
event: TailEvent,
|
|
864
|
+
errorType: ErrorType,
|
|
865
|
+
errorLog: { message: unknown[] },
|
|
866
|
+
dynamicPatterns: CompiledPattern[] = []
|
|
867
|
+
): Promise<FingerprintResult> {
|
|
868
|
+
const components: string[] = [event.scriptName, errorType];
|
|
869
|
+
|
|
870
|
+
const coreMsg = extractCoreMessage(errorLog.message[0]);
|
|
871
|
+
|
|
872
|
+
// Check for transient error classification first (static + dynamic patterns)
|
|
873
|
+
const classification = classifyErrorWithSource(coreMsg, dynamicPatterns);
|
|
874
|
+
let normalizedMessage: string;
|
|
875
|
+
|
|
876
|
+
if (classification) {
|
|
877
|
+
// Use stable category instead of variable message
|
|
878
|
+
components.push(classification.category);
|
|
879
|
+
normalizedMessage = normalizeDynamicValues(coreMsg).slice(0, 200);
|
|
880
|
+
} else {
|
|
881
|
+
// Standard message-based fingerprinting
|
|
882
|
+
normalizedMessage = normalizeDynamicValues(coreMsg).slice(0, 100);
|
|
883
|
+
components.push(normalizedMessage);
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
// Include normalized URL for HTTP errors (helps distinguish different endpoints)
|
|
887
|
+
if (event.event?.request?.url) {
|
|
888
|
+
components.push(normalizeUrl(event.event.request.url));
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
// Create hash
|
|
892
|
+
const data = components.join(':');
|
|
893
|
+
const hashBuffer = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(data));
|
|
894
|
+
|
|
895
|
+
const fingerprint = Array.from(new Uint8Array(hashBuffer))
|
|
896
|
+
.slice(0, 16)
|
|
897
|
+
.map((b) => b.toString(16).padStart(2, '0'))
|
|
898
|
+
.join('');
|
|
899
|
+
|
|
900
|
+
return {
|
|
901
|
+
fingerprint,
|
|
902
|
+
category: classification?.category ?? null,
|
|
903
|
+
normalizedMessage,
|
|
904
|
+
patternSource: classification?.source,
|
|
905
|
+
dynamicPatternId: classification?.patternId,
|
|
906
|
+
};
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
/**
|
|
910
|
+
* Process a single soft error log from a tail event
|
|
911
|
+
* Called for each unique error in an invocation with multiple errors
|
|
912
|
+
*/
|
|
913
|
+
async function processSoftErrorLog(
|
|
914
|
+
event: TailEvent,
|
|
915
|
+
env: Env,
|
|
916
|
+
github: GitHubClient,
|
|
917
|
+
mapping: ScriptMapping,
|
|
918
|
+
errorLog: { level: string; message: unknown[]; timestamp: number },
|
|
919
|
+
dynamicPatterns: CompiledPattern[] = []
|
|
920
|
+
): Promise<void> {
|
|
921
|
+
const errorType: ErrorType = 'soft_error';
|
|
922
|
+
|
|
923
|
+
// Check rate limit
|
|
924
|
+
const withinLimits = await checkRateLimit(env.PLATFORM_CACHE, event.scriptName);
|
|
925
|
+
if (!withinLimits) {
|
|
926
|
+
const coreMsg = extractCoreMessage(errorLog.message[0]);
|
|
927
|
+
console.log(`Rate limited for script: ${event.scriptName} (error: ${coreMsg.slice(0, 50)})`);
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
// Compute fingerprint for this specific error log (now returns FingerprintResult)
|
|
932
|
+
const fingerprintResult = await computeFingerprintForLog(event, errorType, errorLog, dynamicPatterns);
|
|
933
|
+
const { fingerprint, category, dynamicPatternId } = fingerprintResult;
|
|
934
|
+
const isTransient = category !== null;
|
|
935
|
+
|
|
936
|
+
// Log dynamic pattern matches for observability and record evidence
|
|
937
|
+
if (dynamicPatternId) {
|
|
938
|
+
console.log(`Dynamic pattern match (soft error): ${category} (pattern: ${dynamicPatternId})`);
|
|
939
|
+
// Record match evidence for human review context
|
|
940
|
+
await recordPatternMatchEvidence(env.PLATFORM_DB, {
|
|
941
|
+
patternId: dynamicPatternId,
|
|
942
|
+
scriptName: event.scriptName,
|
|
943
|
+
project: mapping.project,
|
|
944
|
+
errorFingerprint: fingerprint,
|
|
945
|
+
normalizedMessage: fingerprintResult.normalizedMessage ?? undefined,
|
|
946
|
+
errorType: 'soft_error',
|
|
947
|
+
priority: calculatePriority(errorType, mapping.tier, 1),
|
|
948
|
+
});
|
|
949
|
+
// Increment match_count so shadow evaluation has accurate stats
|
|
950
|
+
await env.PLATFORM_DB.prepare(
|
|
951
|
+
`UPDATE transient_pattern_suggestions SET match_count = match_count + 1, last_matched_at = unixepoch() WHERE id = ?`
|
|
952
|
+
).bind(dynamicPatternId).run();
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
// For transient errors, check if we already have an issue for today's window
|
|
956
|
+
if (isTransient && category) {
|
|
957
|
+
const existingIssue = await checkTransientErrorWindow(
|
|
958
|
+
env.PLATFORM_CACHE,
|
|
959
|
+
event.scriptName,
|
|
960
|
+
category
|
|
961
|
+
);
|
|
962
|
+
if (existingIssue) {
|
|
963
|
+
// Just update occurrence count in D1, don't create new issue
|
|
964
|
+
await env.PLATFORM_DB.prepare(
|
|
965
|
+
`
|
|
966
|
+
UPDATE error_occurrences
|
|
967
|
+
SET occurrence_count = occurrence_count + 1,
|
|
968
|
+
last_seen_at = unixepoch(),
|
|
969
|
+
updated_at = unixepoch()
|
|
970
|
+
WHERE fingerprint = ?
|
|
971
|
+
`
|
|
972
|
+
)
|
|
973
|
+
.bind(fingerprint)
|
|
974
|
+
.run();
|
|
975
|
+
console.log(
|
|
976
|
+
`Transient soft error (${category}) for ${event.scriptName} - issue #${existingIssue} exists for today`
|
|
977
|
+
);
|
|
978
|
+
return;
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
// Get or create occurrence
|
|
983
|
+
const { isNew, occurrence } = await getOrCreateOccurrence(
|
|
984
|
+
env.PLATFORM_DB,
|
|
985
|
+
env.PLATFORM_CACHE,
|
|
986
|
+
fingerprint,
|
|
987
|
+
event.scriptName,
|
|
988
|
+
mapping.project,
|
|
989
|
+
errorType,
|
|
990
|
+
calculatePriority(errorType, mapping.tier, 1),
|
|
991
|
+
mapping.repository
|
|
992
|
+
);
|
|
993
|
+
|
|
994
|
+
// Update context
|
|
995
|
+
await updateOccurrenceContext(env.PLATFORM_DB, fingerprint, event);
|
|
996
|
+
|
|
997
|
+
// Calculate priority with actual occurrence count
|
|
998
|
+
const priority = calculatePriority(errorType, mapping.tier, occurrence.occurrence_count);
|
|
999
|
+
|
|
1000
|
+
// If this is a new error, create a GitHub issue (with dedup check)
|
|
1001
|
+
if (isNew) {
|
|
1002
|
+
try {
|
|
1003
|
+
const [owner, repo] = mapping.repository.split('/');
|
|
1004
|
+
|
|
1005
|
+
// RACE CONDITION PREVENTION: Acquire lock before searching/creating
|
|
1006
|
+
const lockAcquired = await acquireIssueLock(env.PLATFORM_CACHE, fingerprint);
|
|
1007
|
+
if (!lockAcquired) {
|
|
1008
|
+
console.log(`Lock held by another worker for ${fingerprint}, skipping`);
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
try {
|
|
1013
|
+
// DEDUP CHECK: Search GitHub for existing issue with this fingerprint
|
|
1014
|
+
const existingIssue = await findExistingIssueByFingerprint(github, owner, repo, fingerprint);
|
|
1015
|
+
|
|
1016
|
+
if (existingIssue) {
|
|
1017
|
+
// Check if issue is muted/wontfix - don't reopen or create new
|
|
1018
|
+
if (existingIssue.shouldSkip) {
|
|
1019
|
+
console.log(`Issue #${existingIssue.number} is muted/wontfix, skipping`);
|
|
1020
|
+
// Still link D1 record to prevent future searches
|
|
1021
|
+
await updateOccurrenceWithIssue(
|
|
1022
|
+
env.PLATFORM_DB,
|
|
1023
|
+
env.PLATFORM_CACHE,
|
|
1024
|
+
fingerprint,
|
|
1025
|
+
existingIssue.number,
|
|
1026
|
+
`https://github.com/${owner}/${repo}/issues/${existingIssue.number}`
|
|
1027
|
+
);
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
// Found existing issue - update it instead of creating new
|
|
1032
|
+
const comment = formatRecurrenceComment(
|
|
1033
|
+
event,
|
|
1034
|
+
errorType,
|
|
1035
|
+
occurrence.occurrence_count,
|
|
1036
|
+
existingIssue.state === 'closed'
|
|
1037
|
+
);
|
|
1038
|
+
|
|
1039
|
+
if (existingIssue.state === 'closed') {
|
|
1040
|
+
// Reopen the issue
|
|
1041
|
+
await github.updateIssue({
|
|
1042
|
+
owner,
|
|
1043
|
+
repo,
|
|
1044
|
+
issue_number: existingIssue.number,
|
|
1045
|
+
state: 'open',
|
|
1046
|
+
});
|
|
1047
|
+
await github.addLabels(owner, repo, existingIssue.number, ['cf:regression']);
|
|
1048
|
+
console.log(`Reopened existing issue #${existingIssue.number} (dedup: ${fingerprint})`);
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
await github.addComment(owner, repo, existingIssue.number, comment);
|
|
1052
|
+
|
|
1053
|
+
// Update D1 with the found issue number
|
|
1054
|
+
await updateOccurrenceWithIssue(
|
|
1055
|
+
env.PLATFORM_DB,
|
|
1056
|
+
env.PLATFORM_CACHE,
|
|
1057
|
+
fingerprint,
|
|
1058
|
+
existingIssue.number,
|
|
1059
|
+
`https://github.com/${owner}/${repo}/issues/${existingIssue.number}`
|
|
1060
|
+
);
|
|
1061
|
+
|
|
1062
|
+
// For transient errors, record the issue in the window cache
|
|
1063
|
+
if (isTransient && category) {
|
|
1064
|
+
await setTransientErrorWindow(
|
|
1065
|
+
env.PLATFORM_CACHE,
|
|
1066
|
+
event.scriptName,
|
|
1067
|
+
category,
|
|
1068
|
+
existingIssue.number
|
|
1069
|
+
);
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
return; // Don't create a new issue
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
// No existing issue found - create new (original code)
|
|
1076
|
+
const coreMsg = extractCoreMessage(errorLog.message[0]);
|
|
1077
|
+
const title = `[${event.scriptName}] Error: ${coreMsg.slice(0, 60)}`.slice(0, 100);
|
|
1078
|
+
const body = formatIssueBody(
|
|
1079
|
+
event,
|
|
1080
|
+
errorType,
|
|
1081
|
+
priority,
|
|
1082
|
+
mapping,
|
|
1083
|
+
fingerprint,
|
|
1084
|
+
occurrence.occurrence_count
|
|
1085
|
+
);
|
|
1086
|
+
const labels = getLabels(errorType, priority);
|
|
1087
|
+
|
|
1088
|
+
// Add transient label for transient errors
|
|
1089
|
+
if (isTransient) {
|
|
1090
|
+
labels.push('cf:transient');
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
const issue = await github.createIssue({
|
|
1094
|
+
owner,
|
|
1095
|
+
repo,
|
|
1096
|
+
title,
|
|
1097
|
+
body,
|
|
1098
|
+
labels,
|
|
1099
|
+
type: getGitHubIssueType(errorType),
|
|
1100
|
+
assignees: env.DEFAULT_ASSIGNEE ? [env.DEFAULT_ASSIGNEE] : [],
|
|
1101
|
+
});
|
|
1102
|
+
|
|
1103
|
+
console.log(
|
|
1104
|
+
`Created issue #${issue.number} for ${event.scriptName} - ${coreMsg.slice(0, 30)}${isTransient ? ` (transient: ${category})` : ''}`
|
|
1105
|
+
);
|
|
1106
|
+
|
|
1107
|
+
// Update occurrence with issue details
|
|
1108
|
+
await updateOccurrenceWithIssue(
|
|
1109
|
+
env.PLATFORM_DB,
|
|
1110
|
+
env.PLATFORM_CACHE,
|
|
1111
|
+
fingerprint,
|
|
1112
|
+
issue.number,
|
|
1113
|
+
issue.html_url
|
|
1114
|
+
);
|
|
1115
|
+
|
|
1116
|
+
// For transient errors, record the issue in the window cache
|
|
1117
|
+
if (isTransient && category) {
|
|
1118
|
+
await setTransientErrorWindow(env.PLATFORM_CACHE, event.scriptName, category, issue.number);
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
// Add to project board
|
|
1122
|
+
try {
|
|
1123
|
+
const issueDetails = await github.getIssue(owner, repo, issue.number);
|
|
1124
|
+
await github.addToProject(issueDetails.node_id, env.GITHUB_PROJECT_ID);
|
|
1125
|
+
} catch (e) {
|
|
1126
|
+
console.error(`Failed to add to project board: ${e}`);
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
// Create dashboard notification for P0-P2 errors
|
|
1130
|
+
await createDashboardNotification(
|
|
1131
|
+
env.NOTIFICATIONS_API,
|
|
1132
|
+
priority,
|
|
1133
|
+
errorType,
|
|
1134
|
+
event.scriptName,
|
|
1135
|
+
coreMsg,
|
|
1136
|
+
issue.number,
|
|
1137
|
+
issue.html_url,
|
|
1138
|
+
mapping.project
|
|
1139
|
+
);
|
|
1140
|
+
} finally {
|
|
1141
|
+
// Always release lock
|
|
1142
|
+
await releaseIssueLock(env.PLATFORM_CACHE, fingerprint);
|
|
1143
|
+
}
|
|
1144
|
+
} catch (e) {
|
|
1145
|
+
console.error(`Failed to create GitHub issue: ${e}`);
|
|
1146
|
+
}
|
|
1147
|
+
} else if (occurrence.github_issue_number && occurrence.status === 'resolved') {
|
|
1148
|
+
// Error recurred after being resolved
|
|
1149
|
+
// Skip regression logic for transient errors - they're expected to recur
|
|
1150
|
+
if (isTransient) {
|
|
1151
|
+
console.log(
|
|
1152
|
+
`Transient soft error (${category}) recurred for ${event.scriptName} - not marking as regression`
|
|
1153
|
+
);
|
|
1154
|
+
// Just update to open status without regression label
|
|
1155
|
+
await env.PLATFORM_DB.prepare(
|
|
1156
|
+
`
|
|
1157
|
+
UPDATE error_occurrences
|
|
1158
|
+
SET status = 'open',
|
|
1159
|
+
resolved_at = NULL,
|
|
1160
|
+
resolved_by = NULL,
|
|
1161
|
+
updated_at = unixepoch()
|
|
1162
|
+
WHERE fingerprint = ?
|
|
1163
|
+
`
|
|
1164
|
+
)
|
|
1165
|
+
.bind(fingerprint)
|
|
1166
|
+
.run();
|
|
1167
|
+
return;
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
// Non-transient error: apply regression logic
|
|
1171
|
+
try {
|
|
1172
|
+
const [owner, repo] = mapping.repository.split('/');
|
|
1173
|
+
|
|
1174
|
+
// Check if issue is muted - if so, don't reopen or comment
|
|
1175
|
+
const muted = await isIssueMuted(github, owner, repo, occurrence.github_issue_number);
|
|
1176
|
+
if (muted) {
|
|
1177
|
+
console.log(`Issue #${occurrence.github_issue_number} is muted, skipping reopen`);
|
|
1178
|
+
return;
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
await github.updateIssue({
|
|
1182
|
+
owner,
|
|
1183
|
+
repo,
|
|
1184
|
+
issue_number: occurrence.github_issue_number,
|
|
1185
|
+
state: 'open',
|
|
1186
|
+
});
|
|
1187
|
+
|
|
1188
|
+
await github.addLabels(owner, repo, occurrence.github_issue_number, ['cf:regression']);
|
|
1189
|
+
|
|
1190
|
+
await github.addComment(
|
|
1191
|
+
owner,
|
|
1192
|
+
repo,
|
|
1193
|
+
occurrence.github_issue_number,
|
|
1194
|
+
`⚠️ **Regression Detected**\n\nThis error has recurred after being marked as resolved.\n\n- **Occurrences**: ${occurrence.occurrence_count}\n- **Last Seen**: ${new Date().toISOString()}\n\nPlease investigate if the fix was incomplete.`
|
|
1195
|
+
);
|
|
1196
|
+
|
|
1197
|
+
console.log(`Reopened issue #${occurrence.github_issue_number} as regression`);
|
|
1198
|
+
|
|
1199
|
+
// Update status in D1
|
|
1200
|
+
await env.PLATFORM_DB.prepare(
|
|
1201
|
+
`
|
|
1202
|
+
UPDATE error_occurrences
|
|
1203
|
+
SET status = 'open',
|
|
1204
|
+
resolved_at = NULL,
|
|
1205
|
+
resolved_by = NULL,
|
|
1206
|
+
updated_at = unixepoch()
|
|
1207
|
+
WHERE fingerprint = ?
|
|
1208
|
+
`
|
|
1209
|
+
)
|
|
1210
|
+
.bind(fingerprint)
|
|
1211
|
+
.run();
|
|
1212
|
+
} catch (e) {
|
|
1213
|
+
console.error(`Failed to reopen issue: ${e}`);
|
|
1214
|
+
}
|
|
1215
|
+
} else if (occurrence.github_issue_number) {
|
|
1216
|
+
// Update existing issue with new occurrence count (every 10 occurrences)
|
|
1217
|
+
if (occurrence.occurrence_count % 10 === 0) {
|
|
1218
|
+
try {
|
|
1219
|
+
const [owner, repo] = mapping.repository.split('/');
|
|
1220
|
+
|
|
1221
|
+
// Check if issue is muted - if so, don't add comments
|
|
1222
|
+
const muted = await isIssueMuted(github, owner, repo, occurrence.github_issue_number);
|
|
1223
|
+
if (muted) {
|
|
1224
|
+
console.log(`Issue #${occurrence.github_issue_number} is muted, skipping comment`);
|
|
1225
|
+
return;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
await github.addComment(
|
|
1229
|
+
owner,
|
|
1230
|
+
repo,
|
|
1231
|
+
occurrence.github_issue_number,
|
|
1232
|
+
`📊 **Occurrence Update**\n\nThis error has now occurred **${occurrence.occurrence_count} times**.\n\n- **Last Seen**: ${new Date().toISOString()}\n- **Colo**: ${event.event?.request?.cf?.colo || 'unknown'}`
|
|
1233
|
+
);
|
|
1234
|
+
console.log(`Updated issue #${occurrence.github_issue_number} with occurrence count`);
|
|
1235
|
+
} catch (e) {
|
|
1236
|
+
console.error(`Failed to update issue: ${e}`);
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
/**
|
|
1243
|
+
* Process a single tail event
|
|
1244
|
+
*/
|
|
1245
|
+
async function processEvent(
|
|
1246
|
+
event: TailEvent,
|
|
1247
|
+
env: Env,
|
|
1248
|
+
github: GitHubClient,
|
|
1249
|
+
dynamicPatterns: CompiledPattern[] = []
|
|
1250
|
+
): Promise<void> {
|
|
1251
|
+
// Check if we should capture this event
|
|
1252
|
+
const decision = shouldCapture(event);
|
|
1253
|
+
if (!decision.capture || !decision.type) {
|
|
1254
|
+
return;
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
const errorType = decision.type;
|
|
1258
|
+
|
|
1259
|
+
// Get script mapping
|
|
1260
|
+
const mapping = await getScriptMapping(env.PLATFORM_CACHE, event.scriptName);
|
|
1261
|
+
if (!mapping) {
|
|
1262
|
+
console.log(`No mapping found for script: ${event.scriptName}`);
|
|
1263
|
+
return;
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
// P4 warnings go to daily digest instead of immediate issues
|
|
1267
|
+
// Process ALL warning logs, not just the first one
|
|
1268
|
+
if (errorType === 'warning') {
|
|
1269
|
+
const warnLogs = event.logs.filter((l) => l.level === 'warn');
|
|
1270
|
+
const seenNormalizedMessages = new Set<string>();
|
|
1271
|
+
|
|
1272
|
+
for (const warnLog of warnLogs) {
|
|
1273
|
+
const rawMessage = warnLog.message
|
|
1274
|
+
.map((m) => (typeof m === 'string' ? m : JSON.stringify(m)))
|
|
1275
|
+
.join(' ');
|
|
1276
|
+
const coreMessage = extractCoreMessage(warnLog.message[0]);
|
|
1277
|
+
const normalizedMessage = normalizeDynamicValues(coreMessage);
|
|
1278
|
+
|
|
1279
|
+
// Dedupe warnings with identical normalized messages within same invocation
|
|
1280
|
+
if (seenNormalizedMessages.has(normalizedMessage)) {
|
|
1281
|
+
continue;
|
|
1282
|
+
}
|
|
1283
|
+
seenNormalizedMessages.add(normalizedMessage);
|
|
1284
|
+
|
|
1285
|
+
const fingerprintResult = await computeFingerprintForLog(event, errorType, warnLog, dynamicPatterns);
|
|
1286
|
+
|
|
1287
|
+
await storeWarningForDigest(
|
|
1288
|
+
env.PLATFORM_DB,
|
|
1289
|
+
env.PLATFORM_CACHE,
|
|
1290
|
+
fingerprintResult.fingerprint,
|
|
1291
|
+
event.scriptName,
|
|
1292
|
+
mapping.project,
|
|
1293
|
+
mapping.repository,
|
|
1294
|
+
normalizedMessage,
|
|
1295
|
+
rawMessage
|
|
1296
|
+
);
|
|
1297
|
+
|
|
1298
|
+
console.log(
|
|
1299
|
+
`Stored warning for digest: ${event.scriptName} - ${normalizedMessage.slice(0, 50)}`
|
|
1300
|
+
);
|
|
1301
|
+
}
|
|
1302
|
+
return;
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
// For soft_error: Process ALL error logs, not just the first one
|
|
1306
|
+
// This fixes the bug where multiple console.error() calls in one invocation
|
|
1307
|
+
// only resulted in one GitHub issue (task-296)
|
|
1308
|
+
if (errorType === 'soft_error') {
|
|
1309
|
+
const errorLogs = event.logs.filter((l) => l.level === 'error');
|
|
1310
|
+
const seenNormalizedMessages = new Set<string>();
|
|
1311
|
+
|
|
1312
|
+
for (const errorLog of errorLogs) {
|
|
1313
|
+
const coreMessage = extractCoreMessage(errorLog.message[0]);
|
|
1314
|
+
const normalizedMessage = normalizeDynamicValues(coreMessage);
|
|
1315
|
+
|
|
1316
|
+
// Dedupe errors with identical normalized messages within same invocation
|
|
1317
|
+
// (e.g., same error logged twice in a loop)
|
|
1318
|
+
if (seenNormalizedMessages.has(normalizedMessage)) {
|
|
1319
|
+
continue;
|
|
1320
|
+
}
|
|
1321
|
+
seenNormalizedMessages.add(normalizedMessage);
|
|
1322
|
+
|
|
1323
|
+
await processSoftErrorLog(event, env, github, mapping, errorLog, dynamicPatterns);
|
|
1324
|
+
}
|
|
1325
|
+
return;
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
// For exceptions and resource limits: use original single-error processing
|
|
1329
|
+
// Check rate limit (only for non-warning errors that create immediate issues)
|
|
1330
|
+
const withinLimits = await checkRateLimit(env.PLATFORM_CACHE, event.scriptName);
|
|
1331
|
+
if (!withinLimits) {
|
|
1332
|
+
console.log(`Rate limited for script: ${event.scriptName}`);
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
// Compute fingerprint (now returns FingerprintResult with category)
|
|
1337
|
+
// Pass dynamic patterns to enable AI-suggested pattern matching
|
|
1338
|
+
const fingerprintResult = await computeFingerprint(event, errorType, { dynamicPatterns });
|
|
1339
|
+
const { fingerprint, category, dynamicPatternId } = fingerprintResult;
|
|
1340
|
+
const isTransient = category !== null;
|
|
1341
|
+
|
|
1342
|
+
// Log dynamic pattern matches for observability and record evidence
|
|
1343
|
+
if (dynamicPatternId) {
|
|
1344
|
+
console.log(`Dynamic pattern match: ${category} (pattern: ${dynamicPatternId})`);
|
|
1345
|
+
// Record match evidence for human review context
|
|
1346
|
+
await recordPatternMatchEvidence(env.PLATFORM_DB, {
|
|
1347
|
+
patternId: dynamicPatternId,
|
|
1348
|
+
scriptName: event.scriptName,
|
|
1349
|
+
project: mapping.project,
|
|
1350
|
+
errorFingerprint: fingerprint,
|
|
1351
|
+
normalizedMessage: fingerprintResult.normalizedMessage ?? undefined,
|
|
1352
|
+
errorType: errorType,
|
|
1353
|
+
priority: calculatePriority(errorType, mapping.tier, 1),
|
|
1354
|
+
});
|
|
1355
|
+
// Increment match_count so shadow evaluation has accurate stats
|
|
1356
|
+
await env.PLATFORM_DB.prepare(
|
|
1357
|
+
`UPDATE transient_pattern_suggestions SET match_count = match_count + 1, last_matched_at = unixepoch() WHERE id = ?`
|
|
1358
|
+
).bind(dynamicPatternId).run();
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
// For transient errors, check if we already have an issue for today's window
|
|
1362
|
+
// This prevents noise from quota exhaustion errors that occur repeatedly
|
|
1363
|
+
if (isTransient && category) {
|
|
1364
|
+
const existingIssue = await checkTransientErrorWindow(
|
|
1365
|
+
env.PLATFORM_CACHE,
|
|
1366
|
+
event.scriptName,
|
|
1367
|
+
category
|
|
1368
|
+
);
|
|
1369
|
+
if (existingIssue) {
|
|
1370
|
+
// Just update occurrence count in D1, don't create new issue
|
|
1371
|
+
await env.PLATFORM_DB.prepare(
|
|
1372
|
+
`
|
|
1373
|
+
UPDATE error_occurrences
|
|
1374
|
+
SET occurrence_count = occurrence_count + 1,
|
|
1375
|
+
last_seen_at = unixepoch(),
|
|
1376
|
+
updated_at = unixepoch()
|
|
1377
|
+
WHERE fingerprint = ?
|
|
1378
|
+
`
|
|
1379
|
+
)
|
|
1380
|
+
.bind(fingerprint)
|
|
1381
|
+
.run();
|
|
1382
|
+
console.log(
|
|
1383
|
+
`Transient error (${category}) for ${event.scriptName} - issue #${existingIssue} exists for today`
|
|
1384
|
+
);
|
|
1385
|
+
return;
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
// Get or create occurrence
|
|
1390
|
+
const { isNew, occurrence } = await getOrCreateOccurrence(
|
|
1391
|
+
env.PLATFORM_DB,
|
|
1392
|
+
env.PLATFORM_CACHE,
|
|
1393
|
+
fingerprint,
|
|
1394
|
+
event.scriptName,
|
|
1395
|
+
mapping.project,
|
|
1396
|
+
errorType,
|
|
1397
|
+
calculatePriority(errorType, mapping.tier, 1),
|
|
1398
|
+
mapping.repository
|
|
1399
|
+
);
|
|
1400
|
+
|
|
1401
|
+
// Update context
|
|
1402
|
+
await updateOccurrenceContext(env.PLATFORM_DB, fingerprint, event);
|
|
1403
|
+
|
|
1404
|
+
// Calculate priority with actual occurrence count
|
|
1405
|
+
const priority = calculatePriority(errorType, mapping.tier, occurrence.occurrence_count);
|
|
1406
|
+
|
|
1407
|
+
// If this is a new error, create a GitHub issue (with dedup check)
|
|
1408
|
+
if (isNew) {
|
|
1409
|
+
try {
|
|
1410
|
+
const [owner, repo] = mapping.repository.split('/');
|
|
1411
|
+
|
|
1412
|
+
// RACE CONDITION PREVENTION: Acquire lock before searching/creating
|
|
1413
|
+
const lockAcquired = await acquireIssueLock(env.PLATFORM_CACHE, fingerprint);
|
|
1414
|
+
if (!lockAcquired) {
|
|
1415
|
+
console.log(`Lock held by another worker for ${fingerprint}, skipping`);
|
|
1416
|
+
return;
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
try {
|
|
1420
|
+
// DEDUP CHECK: Search GitHub for existing issue with this fingerprint
|
|
1421
|
+
const existingIssue = await findExistingIssueByFingerprint(github, owner, repo, fingerprint);
|
|
1422
|
+
|
|
1423
|
+
if (existingIssue) {
|
|
1424
|
+
// Check if issue is muted/wontfix - don't reopen or create new
|
|
1425
|
+
if (existingIssue.shouldSkip) {
|
|
1426
|
+
console.log(`Issue #${existingIssue.number} is muted/wontfix, skipping`);
|
|
1427
|
+
// Still link D1 record to prevent future searches
|
|
1428
|
+
await updateOccurrenceWithIssue(
|
|
1429
|
+
env.PLATFORM_DB,
|
|
1430
|
+
env.PLATFORM_CACHE,
|
|
1431
|
+
fingerprint,
|
|
1432
|
+
existingIssue.number,
|
|
1433
|
+
`https://github.com/${owner}/${repo}/issues/${existingIssue.number}`
|
|
1434
|
+
);
|
|
1435
|
+
return;
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
// Found existing issue - update it instead of creating new
|
|
1439
|
+
const comment = formatRecurrenceComment(
|
|
1440
|
+
event,
|
|
1441
|
+
errorType,
|
|
1442
|
+
occurrence.occurrence_count,
|
|
1443
|
+
existingIssue.state === 'closed'
|
|
1444
|
+
);
|
|
1445
|
+
|
|
1446
|
+
if (existingIssue.state === 'closed') {
|
|
1447
|
+
// Reopen the issue
|
|
1448
|
+
await github.updateIssue({
|
|
1449
|
+
owner,
|
|
1450
|
+
repo,
|
|
1451
|
+
issue_number: existingIssue.number,
|
|
1452
|
+
state: 'open',
|
|
1453
|
+
});
|
|
1454
|
+
await github.addLabels(owner, repo, existingIssue.number, ['cf:regression']);
|
|
1455
|
+
console.log(`Reopened existing issue #${existingIssue.number} (dedup: ${fingerprint})`);
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
await github.addComment(owner, repo, existingIssue.number, comment);
|
|
1459
|
+
|
|
1460
|
+
// Update D1 with the found issue number
|
|
1461
|
+
await updateOccurrenceWithIssue(
|
|
1462
|
+
env.PLATFORM_DB,
|
|
1463
|
+
env.PLATFORM_CACHE,
|
|
1464
|
+
fingerprint,
|
|
1465
|
+
existingIssue.number,
|
|
1466
|
+
`https://github.com/${owner}/${repo}/issues/${existingIssue.number}`
|
|
1467
|
+
);
|
|
1468
|
+
|
|
1469
|
+
// For transient errors, record the issue in the window cache
|
|
1470
|
+
if (isTransient && category) {
|
|
1471
|
+
await setTransientErrorWindow(
|
|
1472
|
+
env.PLATFORM_CACHE,
|
|
1473
|
+
event.scriptName,
|
|
1474
|
+
category,
|
|
1475
|
+
existingIssue.number
|
|
1476
|
+
);
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
return; // Don't create a new issue
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
// No existing issue found - create new (original code)
|
|
1483
|
+
const title = formatErrorTitle(errorType, event, event.scriptName);
|
|
1484
|
+
const body = formatIssueBody(
|
|
1485
|
+
event,
|
|
1486
|
+
errorType,
|
|
1487
|
+
priority,
|
|
1488
|
+
mapping,
|
|
1489
|
+
fingerprint,
|
|
1490
|
+
occurrence.occurrence_count
|
|
1491
|
+
);
|
|
1492
|
+
const labels = getLabels(errorType, priority);
|
|
1493
|
+
|
|
1494
|
+
// Add transient label for transient errors
|
|
1495
|
+
if (isTransient) {
|
|
1496
|
+
labels.push('cf:transient');
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
const issue = await github.createIssue({
|
|
1500
|
+
owner,
|
|
1501
|
+
repo,
|
|
1502
|
+
title,
|
|
1503
|
+
body,
|
|
1504
|
+
labels,
|
|
1505
|
+
type: getGitHubIssueType(errorType),
|
|
1506
|
+
assignees: env.DEFAULT_ASSIGNEE ? [env.DEFAULT_ASSIGNEE] : [],
|
|
1507
|
+
});
|
|
1508
|
+
|
|
1509
|
+
console.log(
|
|
1510
|
+
`Created issue #${issue.number} for ${event.scriptName}${isTransient ? ` (transient: ${category})` : ''}`
|
|
1511
|
+
);
|
|
1512
|
+
|
|
1513
|
+
// Update occurrence with issue details
|
|
1514
|
+
await updateOccurrenceWithIssue(
|
|
1515
|
+
env.PLATFORM_DB,
|
|
1516
|
+
env.PLATFORM_CACHE,
|
|
1517
|
+
fingerprint,
|
|
1518
|
+
issue.number,
|
|
1519
|
+
issue.html_url
|
|
1520
|
+
);
|
|
1521
|
+
|
|
1522
|
+
// For transient errors, record the issue in the window cache
|
|
1523
|
+
if (isTransient && category) {
|
|
1524
|
+
await setTransientErrorWindow(env.PLATFORM_CACHE, event.scriptName, category, issue.number);
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
// Add to project board
|
|
1528
|
+
try {
|
|
1529
|
+
const issueDetails = await github.getIssue(owner, repo, issue.number);
|
|
1530
|
+
await github.addToProject(issueDetails.node_id, env.GITHUB_PROJECT_ID);
|
|
1531
|
+
console.log(`Added issue #${issue.number} to project board`);
|
|
1532
|
+
} catch (e) {
|
|
1533
|
+
console.error(`Failed to add to project board: ${e}`);
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
// Create dashboard notification for P0-P2 errors
|
|
1537
|
+
await createDashboardNotification(
|
|
1538
|
+
env.NOTIFICATIONS_API,
|
|
1539
|
+
priority,
|
|
1540
|
+
errorType,
|
|
1541
|
+
event.scriptName,
|
|
1542
|
+
title,
|
|
1543
|
+
issue.number,
|
|
1544
|
+
issue.html_url,
|
|
1545
|
+
mapping.project
|
|
1546
|
+
);
|
|
1547
|
+
} finally {
|
|
1548
|
+
// Always release lock
|
|
1549
|
+
await releaseIssueLock(env.PLATFORM_CACHE, fingerprint);
|
|
1550
|
+
}
|
|
1551
|
+
} catch (e) {
|
|
1552
|
+
console.error(`Failed to create GitHub issue: ${e}`);
|
|
1553
|
+
}
|
|
1554
|
+
} else if (occurrence.github_issue_number && occurrence.status === 'resolved') {
|
|
1555
|
+
// Error recurred after being resolved
|
|
1556
|
+
// Skip regression logic for transient errors - they're expected to recur
|
|
1557
|
+
if (isTransient) {
|
|
1558
|
+
console.log(
|
|
1559
|
+
`Transient error (${category}) recurred for ${event.scriptName} - not marking as regression`
|
|
1560
|
+
);
|
|
1561
|
+
// Just update to open status without regression label
|
|
1562
|
+
await env.PLATFORM_DB.prepare(
|
|
1563
|
+
`
|
|
1564
|
+
UPDATE error_occurrences
|
|
1565
|
+
SET status = 'open',
|
|
1566
|
+
resolved_at = NULL,
|
|
1567
|
+
resolved_by = NULL,
|
|
1568
|
+
updated_at = unixepoch()
|
|
1569
|
+
WHERE fingerprint = ?
|
|
1570
|
+
`
|
|
1571
|
+
)
|
|
1572
|
+
.bind(fingerprint)
|
|
1573
|
+
.run();
|
|
1574
|
+
return;
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
// Non-transient error: apply regression logic
|
|
1578
|
+
try {
|
|
1579
|
+
const [owner, repo] = mapping.repository.split('/');
|
|
1580
|
+
|
|
1581
|
+
// Check if issue is muted - if so, don't reopen or comment
|
|
1582
|
+
const muted = await isIssueMuted(github, owner, repo, occurrence.github_issue_number);
|
|
1583
|
+
if (muted) {
|
|
1584
|
+
console.log(`Issue #${occurrence.github_issue_number} is muted, skipping reopen`);
|
|
1585
|
+
return;
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
await github.updateIssue({
|
|
1589
|
+
owner,
|
|
1590
|
+
repo,
|
|
1591
|
+
issue_number: occurrence.github_issue_number,
|
|
1592
|
+
state: 'open',
|
|
1593
|
+
});
|
|
1594
|
+
|
|
1595
|
+
await github.addLabels(owner, repo, occurrence.github_issue_number, ['cf:regression']);
|
|
1596
|
+
|
|
1597
|
+
await github.addComment(
|
|
1598
|
+
owner,
|
|
1599
|
+
repo,
|
|
1600
|
+
occurrence.github_issue_number,
|
|
1601
|
+
`⚠️ **Regression Detected**\n\nThis error has recurred after being marked as resolved.\n\n- **Occurrences**: ${occurrence.occurrence_count}\n- **Last Seen**: ${new Date().toISOString()}\n\nPlease investigate if the fix was incomplete.`
|
|
1602
|
+
);
|
|
1603
|
+
|
|
1604
|
+
console.log(`Reopened issue #${occurrence.github_issue_number} as regression`);
|
|
1605
|
+
|
|
1606
|
+
// Update status in D1
|
|
1607
|
+
await env.PLATFORM_DB.prepare(
|
|
1608
|
+
`
|
|
1609
|
+
UPDATE error_occurrences
|
|
1610
|
+
SET status = 'open',
|
|
1611
|
+
resolved_at = NULL,
|
|
1612
|
+
resolved_by = NULL,
|
|
1613
|
+
updated_at = unixepoch()
|
|
1614
|
+
WHERE fingerprint = ?
|
|
1615
|
+
`
|
|
1616
|
+
)
|
|
1617
|
+
.bind(fingerprint)
|
|
1618
|
+
.run();
|
|
1619
|
+
} catch (e) {
|
|
1620
|
+
console.error(`Failed to reopen issue: ${e}`);
|
|
1621
|
+
}
|
|
1622
|
+
} else if (occurrence.github_issue_number) {
|
|
1623
|
+
// Update existing issue with new occurrence count
|
|
1624
|
+
try {
|
|
1625
|
+
const [owner, repo] = mapping.repository.split('/');
|
|
1626
|
+
|
|
1627
|
+
// Check if issue is muted - if so, don't add comments
|
|
1628
|
+
const muted = await isIssueMuted(github, owner, repo, occurrence.github_issue_number);
|
|
1629
|
+
if (muted) {
|
|
1630
|
+
console.log(`Issue #${occurrence.github_issue_number} is muted, skipping comment`);
|
|
1631
|
+
return;
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
// Add a comment every 10 occurrences to avoid spam
|
|
1635
|
+
if (occurrence.occurrence_count % 10 === 0) {
|
|
1636
|
+
await github.addComment(
|
|
1637
|
+
owner,
|
|
1638
|
+
repo,
|
|
1639
|
+
occurrence.github_issue_number,
|
|
1640
|
+
`📊 **Occurrence Update**\n\nThis error has now occurred **${occurrence.occurrence_count} times**.\n\n- **Last Seen**: ${new Date().toISOString()}\n- **Colo**: ${event.event?.request?.cf?.colo || 'unknown'}`
|
|
1641
|
+
);
|
|
1642
|
+
console.log(`Updated issue #${occurrence.github_issue_number} with occurrence count`);
|
|
1643
|
+
}
|
|
1644
|
+
} catch (e) {
|
|
1645
|
+
console.error(`Failed to update issue: ${e}`);
|
|
1646
|
+
}
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
/**
|
|
1651
|
+
* Tail handler - receives events from producer workers
|
|
1652
|
+
*/
|
|
1653
|
+
async function tail(events: TailEvent[], env: Env): Promise<void> {
|
|
1654
|
+
const github = new GitHubClient(env);
|
|
1655
|
+
|
|
1656
|
+
// Load dynamic patterns once for all events in this batch
|
|
1657
|
+
// These are AI-suggested, human-approved patterns from pattern-discovery
|
|
1658
|
+
const dynamicPatterns = await loadDynamicPatterns(env.PLATFORM_CACHE);
|
|
1659
|
+
if (dynamicPatterns.length > 0) {
|
|
1660
|
+
console.log(`Loaded ${dynamicPatterns.length} dynamic patterns from KV`);
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
for (const event of events) {
|
|
1664
|
+
try {
|
|
1665
|
+
await processEvent(event, env, github, dynamicPatterns);
|
|
1666
|
+
} catch (e) {
|
|
1667
|
+
console.error(`Error processing tail event: ${e}`);
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/**
|
|
1673
|
+
* Scheduled handler - auto-close stale errors and process warning digests
|
|
1674
|
+
*/
|
|
1675
|
+
async function scheduled(
|
|
1676
|
+
event: ScheduledEvent,
|
|
1677
|
+
env: Env,
|
|
1678
|
+
ctx: ExecutionContext
|
|
1679
|
+
): Promise<void> {
|
|
1680
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1681
|
+
const autoCloseSeconds = parseInt(env.AUTO_CLOSE_HOURS, 10) * 60 * 60;
|
|
1682
|
+
const warningCloseSeconds = parseInt(env.WARNING_AUTO_CLOSE_DAYS, 10) * 24 * 60 * 60;
|
|
1683
|
+
|
|
1684
|
+
// At midnight UTC (0 0 * * *), process warning digests for the previous day
|
|
1685
|
+
const currentHour = new Date().getUTCHours();
|
|
1686
|
+
const currentMinute = new Date().getUTCMinutes();
|
|
1687
|
+
if (currentHour === 0 && currentMinute < 15) {
|
|
1688
|
+
console.log('Running daily warning digest processing...');
|
|
1689
|
+
try {
|
|
1690
|
+
const result = await processWarningDigests(env);
|
|
1691
|
+
console.log(
|
|
1692
|
+
`Digest complete: ${result.processed} warnings, ${result.issuesCreated} new issues, ${result.issuesUpdated} updated`
|
|
1693
|
+
);
|
|
1694
|
+
// Signal digest success to Gatus heartbeat
|
|
1695
|
+
pingHeartbeat(ctx, env.GATUS_HEARTBEAT_URL_DIGEST, env.GATUS_TOKEN, true);
|
|
1696
|
+
} catch (e) {
|
|
1697
|
+
console.error(`Failed to process warning digests: ${e}`);
|
|
1698
|
+
// Signal digest failure to Gatus heartbeat
|
|
1699
|
+
pingHeartbeat(ctx, env.GATUS_HEARTBEAT_URL_DIGEST, env.GATUS_TOKEN, false);
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
// Find errors that haven't recurred in AUTO_CLOSE_HOURS
|
|
1704
|
+
const staleErrors = await env.PLATFORM_DB.prepare(
|
|
1705
|
+
`
|
|
1706
|
+
SELECT fingerprint, github_issue_number, github_repo, error_type
|
|
1707
|
+
FROM error_occurrences
|
|
1708
|
+
WHERE status = 'open'
|
|
1709
|
+
AND github_issue_number IS NOT NULL
|
|
1710
|
+
AND last_seen_at < ?
|
|
1711
|
+
AND (error_type != 'warning' OR last_seen_at < ?)
|
|
1712
|
+
`
|
|
1713
|
+
)
|
|
1714
|
+
.bind(now - autoCloseSeconds, now - warningCloseSeconds)
|
|
1715
|
+
.all<{
|
|
1716
|
+
fingerprint: string;
|
|
1717
|
+
github_issue_number: number;
|
|
1718
|
+
github_repo: string;
|
|
1719
|
+
error_type: string;
|
|
1720
|
+
}>();
|
|
1721
|
+
|
|
1722
|
+
if (staleErrors.results?.length) {
|
|
1723
|
+
const github = new GitHubClient(env);
|
|
1724
|
+
|
|
1725
|
+
for (const error of staleErrors.results) {
|
|
1726
|
+
try {
|
|
1727
|
+
const [owner, repo] = error.github_repo.split('/');
|
|
1728
|
+
|
|
1729
|
+
await github.updateIssue({
|
|
1730
|
+
owner,
|
|
1731
|
+
repo,
|
|
1732
|
+
issue_number: error.github_issue_number,
|
|
1733
|
+
state: 'closed',
|
|
1734
|
+
});
|
|
1735
|
+
|
|
1736
|
+
await github.addComment(
|
|
1737
|
+
owner,
|
|
1738
|
+
repo,
|
|
1739
|
+
error.github_issue_number,
|
|
1740
|
+
`✅ **Auto-closed**\n\nThis error has not recurred in ${env.AUTO_CLOSE_HOURS} hours and has been automatically closed.\n\nIf you know which commit fixed this, please link it here. If the error recurs, this issue will be automatically reopened.`
|
|
1741
|
+
);
|
|
1742
|
+
|
|
1743
|
+
// Update status in D1
|
|
1744
|
+
await env.PLATFORM_DB.prepare(
|
|
1745
|
+
`
|
|
1746
|
+
UPDATE error_occurrences
|
|
1747
|
+
SET status = 'resolved',
|
|
1748
|
+
resolved_at = unixepoch(),
|
|
1749
|
+
resolved_by = 'auto-close',
|
|
1750
|
+
updated_at = unixepoch()
|
|
1751
|
+
WHERE fingerprint = ?
|
|
1752
|
+
`
|
|
1753
|
+
)
|
|
1754
|
+
.bind(error.fingerprint)
|
|
1755
|
+
.run();
|
|
1756
|
+
|
|
1757
|
+
// Update KV cache
|
|
1758
|
+
const kvKey = `ERROR_FINGERPRINT:${error.fingerprint}`;
|
|
1759
|
+
const cached = await env.PLATFORM_CACHE.get(kvKey);
|
|
1760
|
+
if (cached) {
|
|
1761
|
+
const data = JSON.parse(cached);
|
|
1762
|
+
await env.PLATFORM_CACHE.put(
|
|
1763
|
+
kvKey,
|
|
1764
|
+
JSON.stringify({
|
|
1765
|
+
...data,
|
|
1766
|
+
status: 'resolved',
|
|
1767
|
+
}),
|
|
1768
|
+
{ expirationTtl: 90 * 24 * 60 * 60 }
|
|
1769
|
+
);
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
console.log(`Auto-closed issue #${error.github_issue_number}`);
|
|
1773
|
+
} catch (e) {
|
|
1774
|
+
console.error(`Failed to auto-close issue #${error.github_issue_number}: ${e}`);
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1777
|
+
} else {
|
|
1778
|
+
console.log('No stale errors to auto-close');
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
// Signal scheduled run success to Gatus heartbeat (must always execute)
|
|
1782
|
+
pingHeartbeat(ctx, env.GATUS_HEARTBEAT_URL_15M, env.GATUS_TOKEN, true);
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
/**
|
|
1786
|
+
* Verify GitHub webhook signature
|
|
1787
|
+
*/
|
|
1788
|
+
async function verifyWebhookSignature(
|
|
1789
|
+
payload: string,
|
|
1790
|
+
signature: string | null,
|
|
1791
|
+
secret: string
|
|
1792
|
+
): Promise<boolean> {
|
|
1793
|
+
if (!signature) return false;
|
|
1794
|
+
|
|
1795
|
+
const encoder = new TextEncoder();
|
|
1796
|
+
const key = await crypto.subtle.importKey(
|
|
1797
|
+
'raw',
|
|
1798
|
+
encoder.encode(secret),
|
|
1799
|
+
{ name: 'HMAC', hash: 'SHA-256' },
|
|
1800
|
+
false,
|
|
1801
|
+
['sign']
|
|
1802
|
+
);
|
|
1803
|
+
|
|
1804
|
+
const signatureBuffer = await crypto.subtle.sign('HMAC', key, encoder.encode(payload));
|
|
1805
|
+
|
|
1806
|
+
const expectedSignature =
|
|
1807
|
+
'sha256=' +
|
|
1808
|
+
Array.from(new Uint8Array(signatureBuffer))
|
|
1809
|
+
.map((b) => b.toString(16).padStart(2, '0'))
|
|
1810
|
+
.join('');
|
|
1811
|
+
|
|
1812
|
+
return signature === expectedSignature;
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1815
|
+
/**
|
|
1816
|
+
* Extract fingerprint from issue body
|
|
1817
|
+
*/
|
|
1818
|
+
function extractFingerprint(body: string): string | null {
|
|
1819
|
+
const match = body.match(/Fingerprint: `([a-f0-9]+)`/);
|
|
1820
|
+
return match ? match[1] : null;
|
|
1821
|
+
}
|
|
1822
|
+
|
|
1823
|
+
/**
|
|
1824
|
+
* GitHub issue event payload
|
|
1825
|
+
*/
|
|
1826
|
+
interface GitHubIssueEvent {
|
|
1827
|
+
action: 'closed' | 'reopened' | 'labeled' | 'unlabeled' | 'opened' | 'edited';
|
|
1828
|
+
issue: {
|
|
1829
|
+
number: number;
|
|
1830
|
+
state: 'open' | 'closed';
|
|
1831
|
+
labels: Array<{ name: string }>;
|
|
1832
|
+
body: string | null;
|
|
1833
|
+
html_url: string;
|
|
1834
|
+
};
|
|
1835
|
+
repository: {
|
|
1836
|
+
full_name: string;
|
|
1837
|
+
};
|
|
1838
|
+
label?: {
|
|
1839
|
+
name: string;
|
|
1840
|
+
};
|
|
1841
|
+
sender: {
|
|
1842
|
+
login: string;
|
|
1843
|
+
type: 'User' | 'Bot';
|
|
1844
|
+
};
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
/**
|
|
1848
|
+
* Handle GitHub webhook for bidirectional sync
|
|
1849
|
+
*/
|
|
1850
|
+
async function handleGitHubWebhook(
|
|
1851
|
+
event: GitHubIssueEvent,
|
|
1852
|
+
env: Env
|
|
1853
|
+
): Promise<{ processed: boolean; message: string }> {
|
|
1854
|
+
// Ignore events from our own bot
|
|
1855
|
+
if (event.sender.type === 'Bot' && event.sender.login.includes('error-collector')) {
|
|
1856
|
+
return { processed: false, message: 'Ignoring bot event' };
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
// Only process if issue has our auto-generated label
|
|
1860
|
+
const hasAutoLabel = event.issue.labels.some((l) => l.name === 'cf:error:auto-generated');
|
|
1861
|
+
if (!hasAutoLabel) {
|
|
1862
|
+
return { processed: false, message: 'Not an auto-generated error issue' };
|
|
1863
|
+
}
|
|
1864
|
+
|
|
1865
|
+
// Extract fingerprint from issue body
|
|
1866
|
+
const fingerprint = extractFingerprint(event.issue.body || '');
|
|
1867
|
+
if (!fingerprint) {
|
|
1868
|
+
return { processed: false, message: 'Could not extract fingerprint' };
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
const repo = event.repository.full_name;
|
|
1872
|
+
|
|
1873
|
+
if (event.action === 'closed') {
|
|
1874
|
+
// Issue was closed - mark as resolved
|
|
1875
|
+
await env.PLATFORM_DB.prepare(
|
|
1876
|
+
`
|
|
1877
|
+
UPDATE error_occurrences
|
|
1878
|
+
SET status = 'resolved',
|
|
1879
|
+
resolved_at = unixepoch(),
|
|
1880
|
+
resolved_by = ?,
|
|
1881
|
+
updated_at = unixepoch()
|
|
1882
|
+
WHERE fingerprint = ?
|
|
1883
|
+
AND github_repo = ?
|
|
1884
|
+
`
|
|
1885
|
+
)
|
|
1886
|
+
.bind(`github:${event.sender.login}`, fingerprint, repo)
|
|
1887
|
+
.run();
|
|
1888
|
+
|
|
1889
|
+
// Update KV cache
|
|
1890
|
+
const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
|
|
1891
|
+
const cached = await env.PLATFORM_CACHE.get(kvKey);
|
|
1892
|
+
if (cached) {
|
|
1893
|
+
const data = JSON.parse(cached);
|
|
1894
|
+
await env.PLATFORM_CACHE.put(
|
|
1895
|
+
kvKey,
|
|
1896
|
+
JSON.stringify({
|
|
1897
|
+
...data,
|
|
1898
|
+
status: 'resolved',
|
|
1899
|
+
}),
|
|
1900
|
+
{ expirationTtl: 90 * 24 * 60 * 60 }
|
|
1901
|
+
);
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
console.log(`Marked ${fingerprint} as resolved via GitHub (closed by ${event.sender.login})`);
|
|
1905
|
+
return { processed: true, message: `Marked as resolved by ${event.sender.login}` };
|
|
1906
|
+
}
|
|
1907
|
+
|
|
1908
|
+
if (event.action === 'reopened') {
|
|
1909
|
+
// Issue was reopened - clear resolved status
|
|
1910
|
+
await env.PLATFORM_DB.prepare(
|
|
1911
|
+
`
|
|
1912
|
+
UPDATE error_occurrences
|
|
1913
|
+
SET status = 'open',
|
|
1914
|
+
resolved_at = NULL,
|
|
1915
|
+
resolved_by = NULL,
|
|
1916
|
+
updated_at = unixepoch()
|
|
1917
|
+
WHERE fingerprint = ?
|
|
1918
|
+
AND github_repo = ?
|
|
1919
|
+
`
|
|
1920
|
+
)
|
|
1921
|
+
.bind(fingerprint, repo)
|
|
1922
|
+
.run();
|
|
1923
|
+
|
|
1924
|
+
// Update KV cache
|
|
1925
|
+
const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
|
|
1926
|
+
const cached = await env.PLATFORM_CACHE.get(kvKey);
|
|
1927
|
+
if (cached) {
|
|
1928
|
+
const data = JSON.parse(cached);
|
|
1929
|
+
await env.PLATFORM_CACHE.put(
|
|
1930
|
+
kvKey,
|
|
1931
|
+
JSON.stringify({
|
|
1932
|
+
...data,
|
|
1933
|
+
status: 'open',
|
|
1934
|
+
}),
|
|
1935
|
+
{ expirationTtl: 90 * 24 * 60 * 60 }
|
|
1936
|
+
);
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
console.log(`Marked ${fingerprint} as open via GitHub (reopened by ${event.sender.login})`);
|
|
1940
|
+
return { processed: true, message: `Reopened by ${event.sender.login}` };
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
if (event.action === 'labeled' && event.label) {
|
|
1944
|
+
// Check for specific status-changing labels
|
|
1945
|
+
if (event.label.name === 'cf:wont-fix') {
|
|
1946
|
+
await env.PLATFORM_DB.prepare(
|
|
1947
|
+
`
|
|
1948
|
+
UPDATE error_occurrences
|
|
1949
|
+
SET status = 'wont_fix',
|
|
1950
|
+
resolved_at = unixepoch(),
|
|
1951
|
+
resolved_by = ?,
|
|
1952
|
+
updated_at = unixepoch()
|
|
1953
|
+
WHERE fingerprint = ?
|
|
1954
|
+
AND github_repo = ?
|
|
1955
|
+
`
|
|
1956
|
+
)
|
|
1957
|
+
.bind(`github:${event.sender.login}`, fingerprint, repo)
|
|
1958
|
+
.run();
|
|
1959
|
+
|
|
1960
|
+
console.log(`Marked ${fingerprint} as won't fix`);
|
|
1961
|
+
return { processed: true, message: `Marked as won't fix` };
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1964
|
+
|
|
1965
|
+
return { processed: false, message: 'No action taken' };
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
// ============================================================================
|
|
1969
|
+
// Dashboard API Handlers
|
|
1970
|
+
// ============================================================================
|
|
1971
|
+
|
|
1972
|
+
/**
|
|
1973
|
+
* List errors with optional filtering
|
|
1974
|
+
*/
|
|
1975
|
+
async function handleListErrors(
|
|
1976
|
+
request: Request,
|
|
1977
|
+
env: Env
|
|
1978
|
+
): Promise<Response> {
|
|
1979
|
+
const url = new URL(request.url);
|
|
1980
|
+
const script = url.searchParams.get('script');
|
|
1981
|
+
const priority = url.searchParams.get('priority');
|
|
1982
|
+
const status = url.searchParams.get('status');
|
|
1983
|
+
const project = url.searchParams.get('project');
|
|
1984
|
+
const errorType = url.searchParams.get('error_type');
|
|
1985
|
+
const dateFrom = url.searchParams.get('date_from');
|
|
1986
|
+
const dateTo = url.searchParams.get('date_to');
|
|
1987
|
+
const limit = Math.min(parseInt(url.searchParams.get('limit') || '50', 10), 100);
|
|
1988
|
+
const offset = parseInt(url.searchParams.get('offset') || '0', 10);
|
|
1989
|
+
|
|
1990
|
+
// Sorting - validated against allowed columns to prevent SQL injection
|
|
1991
|
+
const allowedSortColumns = ['priority', 'script_name', 'status', 'occurrence_count', 'last_seen_at', 'first_seen_at', 'project'];
|
|
1992
|
+
const sortByParam = url.searchParams.get('sort_by');
|
|
1993
|
+
const sortBy = allowedSortColumns.includes(sortByParam || '') ? sortByParam : 'last_seen_at';
|
|
1994
|
+
const sortOrderParam = url.searchParams.get('sort_order')?.toLowerCase();
|
|
1995
|
+
const sortOrder = sortOrderParam === 'asc' ? 'ASC' : 'DESC';
|
|
1996
|
+
|
|
1997
|
+
// Build dynamic WHERE clause
|
|
1998
|
+
const conditions: string[] = [];
|
|
1999
|
+
const bindings: (string | number)[] = [];
|
|
2000
|
+
|
|
2001
|
+
if (script) {
|
|
2002
|
+
conditions.push('script_name = ?');
|
|
2003
|
+
bindings.push(script);
|
|
2004
|
+
}
|
|
2005
|
+
if (priority) {
|
|
2006
|
+
conditions.push('priority = ?');
|
|
2007
|
+
bindings.push(priority);
|
|
2008
|
+
}
|
|
2009
|
+
if (status) {
|
|
2010
|
+
conditions.push('status = ?');
|
|
2011
|
+
bindings.push(status);
|
|
2012
|
+
}
|
|
2013
|
+
if (project) {
|
|
2014
|
+
conditions.push('project = ?');
|
|
2015
|
+
bindings.push(project);
|
|
2016
|
+
}
|
|
2017
|
+
if (errorType) {
|
|
2018
|
+
conditions.push('error_type = ?');
|
|
2019
|
+
bindings.push(errorType);
|
|
2020
|
+
}
|
|
2021
|
+
if (dateFrom) {
|
|
2022
|
+
const fromTs = Math.floor(new Date(dateFrom).getTime() / 1000);
|
|
2023
|
+
conditions.push('last_seen_at >= ?');
|
|
2024
|
+
bindings.push(fromTs);
|
|
2025
|
+
}
|
|
2026
|
+
if (dateTo) {
|
|
2027
|
+
const toTs = Math.floor(new Date(dateTo).getTime() / 1000);
|
|
2028
|
+
conditions.push('last_seen_at <= ?');
|
|
2029
|
+
bindings.push(toTs);
|
|
2030
|
+
}
|
|
2031
|
+
|
|
2032
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
2033
|
+
|
|
2034
|
+
// Count total
|
|
2035
|
+
const countQuery = `SELECT COUNT(*) as total FROM error_occurrences ${whereClause}`;
|
|
2036
|
+
const countResult = await env.PLATFORM_DB.prepare(countQuery)
|
|
2037
|
+
.bind(...bindings)
|
|
2038
|
+
.first<{ total: number }>();
|
|
2039
|
+
|
|
2040
|
+
// Fetch errors
|
|
2041
|
+
const query = `
|
|
2042
|
+
SELECT
|
|
2043
|
+
id, fingerprint, script_name, project, error_type, priority,
|
|
2044
|
+
github_issue_number, github_issue_url, github_repo,
|
|
2045
|
+
status, resolved_at, resolved_by,
|
|
2046
|
+
first_seen_at, last_seen_at, occurrence_count,
|
|
2047
|
+
last_request_url, last_request_method, last_colo, last_country, last_cf_ray,
|
|
2048
|
+
last_exception_name, last_exception_message,
|
|
2049
|
+
normalized_message, error_category
|
|
2050
|
+
FROM error_occurrences
|
|
2051
|
+
${whereClause}
|
|
2052
|
+
ORDER BY ${sortBy} ${sortOrder}
|
|
2053
|
+
LIMIT ? OFFSET ?
|
|
2054
|
+
`;
|
|
2055
|
+
|
|
2056
|
+
const result = await env.PLATFORM_DB.prepare(query)
|
|
2057
|
+
.bind(...bindings, limit, offset)
|
|
2058
|
+
.all();
|
|
2059
|
+
|
|
2060
|
+
return new Response(JSON.stringify({
|
|
2061
|
+
errors: result.results || [],
|
|
2062
|
+
total: countResult?.total || 0,
|
|
2063
|
+
limit,
|
|
2064
|
+
offset,
|
|
2065
|
+
}), {
|
|
2066
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2067
|
+
});
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
/**
|
|
2071
|
+
* Get error statistics for dashboard overview
|
|
2072
|
+
*/
|
|
2073
|
+
async function handleErrorStats(env: Env): Promise<Response> {
|
|
2074
|
+
// Counts by priority
|
|
2075
|
+
const priorityCounts = await env.PLATFORM_DB.prepare(`
|
|
2076
|
+
SELECT priority, COUNT(*) as count
|
|
2077
|
+
FROM error_occurrences
|
|
2078
|
+
WHERE status = 'open'
|
|
2079
|
+
GROUP BY priority
|
|
2080
|
+
`).all<{ priority: string; count: number }>();
|
|
2081
|
+
|
|
2082
|
+
// Counts by status
|
|
2083
|
+
const statusCounts = await env.PLATFORM_DB.prepare(`
|
|
2084
|
+
SELECT status, COUNT(*) as count
|
|
2085
|
+
FROM error_occurrences
|
|
2086
|
+
GROUP BY status
|
|
2087
|
+
`).all<{ status: string; count: number }>();
|
|
2088
|
+
|
|
2089
|
+
// Counts by error type
|
|
2090
|
+
const typeCounts = await env.PLATFORM_DB.prepare(`
|
|
2091
|
+
SELECT error_type, COUNT(*) as count
|
|
2092
|
+
FROM error_occurrences
|
|
2093
|
+
WHERE status = 'open'
|
|
2094
|
+
GROUP BY error_type
|
|
2095
|
+
`).all<{ error_type: string; count: number }>();
|
|
2096
|
+
|
|
2097
|
+
// Counts by project
|
|
2098
|
+
const projectCounts = await env.PLATFORM_DB.prepare(`
|
|
2099
|
+
SELECT project, COUNT(*) as count
|
|
2100
|
+
FROM error_occurrences
|
|
2101
|
+
WHERE status = 'open'
|
|
2102
|
+
GROUP BY project
|
|
2103
|
+
`).all<{ project: string; count: number }>();
|
|
2104
|
+
|
|
2105
|
+
// Recent errors (last 24h)
|
|
2106
|
+
const oneDayAgo = Math.floor(Date.now() / 1000) - 86400;
|
|
2107
|
+
const recentCount = await env.PLATFORM_DB.prepare(`
|
|
2108
|
+
SELECT COUNT(*) as count
|
|
2109
|
+
FROM error_occurrences
|
|
2110
|
+
WHERE last_seen_at > ?
|
|
2111
|
+
`).bind(oneDayAgo).first<{ count: number }>();
|
|
2112
|
+
|
|
2113
|
+
// Transient error categories
|
|
2114
|
+
const transientCounts = await env.PLATFORM_DB.prepare(`
|
|
2115
|
+
SELECT error_category, COUNT(DISTINCT fingerprint) as count
|
|
2116
|
+
FROM error_occurrences
|
|
2117
|
+
WHERE error_category IS NOT NULL AND status = 'open'
|
|
2118
|
+
GROUP BY error_category
|
|
2119
|
+
`).all<{ error_category: string; count: number }>();
|
|
2120
|
+
|
|
2121
|
+
// Total occurrences today
|
|
2122
|
+
const todayStart = Math.floor(new Date().setUTCHours(0, 0, 0, 0) / 1000);
|
|
2123
|
+
const todayOccurrences = await env.PLATFORM_DB.prepare(`
|
|
2124
|
+
SELECT SUM(occurrence_count) as total
|
|
2125
|
+
FROM error_occurrences
|
|
2126
|
+
WHERE last_seen_at >= ?
|
|
2127
|
+
`).bind(todayStart).first<{ total: number }>();
|
|
2128
|
+
|
|
2129
|
+
// Build stats object
|
|
2130
|
+
const byPriority: Record<string, number> = {};
|
|
2131
|
+
for (const row of priorityCounts.results || []) {
|
|
2132
|
+
byPriority[row.priority] = row.count;
|
|
2133
|
+
}
|
|
2134
|
+
|
|
2135
|
+
const byStatus: Record<string, number> = {};
|
|
2136
|
+
for (const row of statusCounts.results || []) {
|
|
2137
|
+
byStatus[row.status] = row.count;
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
const byType: Record<string, number> = {};
|
|
2141
|
+
for (const row of typeCounts.results || []) {
|
|
2142
|
+
byType[row.error_type] = row.count;
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2145
|
+
const byProject: Record<string, number> = {};
|
|
2146
|
+
for (const row of projectCounts.results || []) {
|
|
2147
|
+
byProject[row.project] = row.count;
|
|
2148
|
+
}
|
|
2149
|
+
|
|
2150
|
+
const byTransientCategory: Record<string, number> = {};
|
|
2151
|
+
for (const row of transientCounts.results || []) {
|
|
2152
|
+
byTransientCategory[row.error_category] = row.count;
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
return new Response(JSON.stringify({
|
|
2156
|
+
byPriority,
|
|
2157
|
+
byStatus,
|
|
2158
|
+
byType,
|
|
2159
|
+
byProject,
|
|
2160
|
+
byTransientCategory,
|
|
2161
|
+
recentCount: recentCount?.count || 0,
|
|
2162
|
+
todayOccurrences: todayOccurrences?.total || 0,
|
|
2163
|
+
totalOpen: byStatus['open'] || 0,
|
|
2164
|
+
}), {
|
|
2165
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2166
|
+
});
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2169
|
+
// ============================================================================
|
|
2170
|
+
// Warning Digest API Handlers
|
|
2171
|
+
// ============================================================================
|
|
2172
|
+
|
|
2173
|
+
/**
|
|
2174
|
+
* List warning digests with filtering
|
|
2175
|
+
* GET /digests?script=&days=&limit=&offset=
|
|
2176
|
+
*/
|
|
2177
|
+
async function handleListDigests(
|
|
2178
|
+
request: Request,
|
|
2179
|
+
env: Env
|
|
2180
|
+
): Promise<Response> {
|
|
2181
|
+
const url = new URL(request.url);
|
|
2182
|
+
const script = url.searchParams.get('script');
|
|
2183
|
+
const days = parseInt(url.searchParams.get('days') || '30', 10);
|
|
2184
|
+
const limit = Math.min(parseInt(url.searchParams.get('limit') || '50', 10), 200);
|
|
2185
|
+
const offset = parseInt(url.searchParams.get('offset') || '0', 10);
|
|
2186
|
+
|
|
2187
|
+
const cutoffDate = new Date();
|
|
2188
|
+
cutoffDate.setDate(cutoffDate.getDate() - days);
|
|
2189
|
+
const cutoffDateStr = cutoffDate.toISOString().split('T')[0];
|
|
2190
|
+
|
|
2191
|
+
// Build query with optional filters
|
|
2192
|
+
let query = `
|
|
2193
|
+
SELECT
|
|
2194
|
+
id,
|
|
2195
|
+
digest_date,
|
|
2196
|
+
script_name,
|
|
2197
|
+
fingerprint,
|
|
2198
|
+
normalized_message,
|
|
2199
|
+
github_repo,
|
|
2200
|
+
github_issue_number,
|
|
2201
|
+
github_issue_url,
|
|
2202
|
+
occurrence_count,
|
|
2203
|
+
first_occurrence_at,
|
|
2204
|
+
last_occurrence_at,
|
|
2205
|
+
created_at,
|
|
2206
|
+
updated_at
|
|
2207
|
+
FROM warning_digests
|
|
2208
|
+
WHERE digest_date >= ?
|
|
2209
|
+
`;
|
|
2210
|
+
const params: (string | number)[] = [cutoffDateStr];
|
|
2211
|
+
|
|
2212
|
+
if (script) {
|
|
2213
|
+
query += ' AND script_name LIKE ?';
|
|
2214
|
+
params.push(`%${script}%`);
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
// Get total count
|
|
2218
|
+
let countQuery = `SELECT COUNT(*) as total FROM warning_digests WHERE digest_date >= ?`;
|
|
2219
|
+
const countParams: (string | number)[] = [cutoffDateStr];
|
|
2220
|
+
if (script) {
|
|
2221
|
+
countQuery += ' AND script_name LIKE ?';
|
|
2222
|
+
countParams.push(`%${script}%`);
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
const countResult = await env.PLATFORM_DB.prepare(countQuery)
|
|
2226
|
+
.bind(...countParams)
|
|
2227
|
+
.first<{ total: number }>();
|
|
2228
|
+
|
|
2229
|
+
// Add ordering and pagination
|
|
2230
|
+
query += ' ORDER BY digest_date DESC, occurrence_count DESC LIMIT ? OFFSET ?';
|
|
2231
|
+
params.push(limit, offset);
|
|
2232
|
+
|
|
2233
|
+
const result = await env.PLATFORM_DB.prepare(query)
|
|
2234
|
+
.bind(...params)
|
|
2235
|
+
.all();
|
|
2236
|
+
|
|
2237
|
+
return new Response(JSON.stringify({
|
|
2238
|
+
digests: result.results || [],
|
|
2239
|
+
total: countResult?.total || 0,
|
|
2240
|
+
limit,
|
|
2241
|
+
offset,
|
|
2242
|
+
}), {
|
|
2243
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2244
|
+
});
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
/**
|
|
2248
|
+
* Get warning digest statistics
|
|
2249
|
+
* GET /digests/stats
|
|
2250
|
+
*/
|
|
2251
|
+
async function handleDigestStats(env: Env): Promise<Response> {
|
|
2252
|
+
// Digests by date (last 14 days)
|
|
2253
|
+
const twoWeeksAgo = new Date();
|
|
2254
|
+
twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14);
|
|
2255
|
+
const cutoffDateStr = twoWeeksAgo.toISOString().split('T')[0];
|
|
2256
|
+
|
|
2257
|
+
const byDate = await env.PLATFORM_DB.prepare(`
|
|
2258
|
+
SELECT digest_date, COUNT(*) as count, SUM(occurrence_count) as occurrences
|
|
2259
|
+
FROM warning_digests
|
|
2260
|
+
WHERE digest_date >= ?
|
|
2261
|
+
GROUP BY digest_date
|
|
2262
|
+
ORDER BY digest_date DESC
|
|
2263
|
+
`).bind(cutoffDateStr).all<{ digest_date: string; count: number; occurrences: number }>();
|
|
2264
|
+
|
|
2265
|
+
// Digests by script (all time, top 10)
|
|
2266
|
+
const byScript = await env.PLATFORM_DB.prepare(`
|
|
2267
|
+
SELECT script_name, COUNT(*) as count, SUM(occurrence_count) as occurrences
|
|
2268
|
+
FROM warning_digests
|
|
2269
|
+
GROUP BY script_name
|
|
2270
|
+
ORDER BY occurrences DESC
|
|
2271
|
+
LIMIT 10
|
|
2272
|
+
`).all<{ script_name: string; count: number; occurrences: number }>();
|
|
2273
|
+
|
|
2274
|
+
// Digests by repo (all time)
|
|
2275
|
+
const byRepo = await env.PLATFORM_DB.prepare(`
|
|
2276
|
+
SELECT github_repo, COUNT(*) as count, SUM(occurrence_count) as occurrences
|
|
2277
|
+
FROM warning_digests
|
|
2278
|
+
GROUP BY github_repo
|
|
2279
|
+
ORDER BY occurrences DESC
|
|
2280
|
+
`).all<{ github_repo: string; count: number; occurrences: number }>();
|
|
2281
|
+
|
|
2282
|
+
// Recent totals
|
|
2283
|
+
const oneDayAgo = new Date();
|
|
2284
|
+
oneDayAgo.setDate(oneDayAgo.getDate() - 1);
|
|
2285
|
+
const yesterday = oneDayAgo.toISOString().split('T')[0];
|
|
2286
|
+
|
|
2287
|
+
const todayDigests = await env.PLATFORM_DB.prepare(`
|
|
2288
|
+
SELECT COUNT(*) as count, COALESCE(SUM(occurrence_count), 0) as occurrences
|
|
2289
|
+
FROM warning_digests
|
|
2290
|
+
WHERE digest_date = date('now')
|
|
2291
|
+
`).first<{ count: number; occurrences: number }>();
|
|
2292
|
+
|
|
2293
|
+
const yesterdayDigests = await env.PLATFORM_DB.prepare(`
|
|
2294
|
+
SELECT COUNT(*) as count, COALESCE(SUM(occurrence_count), 0) as occurrences
|
|
2295
|
+
FROM warning_digests
|
|
2296
|
+
WHERE digest_date = ?
|
|
2297
|
+
`).bind(yesterday).first<{ count: number; occurrences: number }>();
|
|
2298
|
+
|
|
2299
|
+
// Total digests and occurrences
|
|
2300
|
+
const totals = await env.PLATFORM_DB.prepare(`
|
|
2301
|
+
SELECT COUNT(*) as totalDigests, COALESCE(SUM(occurrence_count), 0) as totalOccurrences
|
|
2302
|
+
FROM warning_digests
|
|
2303
|
+
`).first<{ totalDigests: number; totalOccurrences: number }>();
|
|
2304
|
+
|
|
2305
|
+
// Most common warning types (top 5)
|
|
2306
|
+
const topWarnings = await env.PLATFORM_DB.prepare(`
|
|
2307
|
+
SELECT normalized_message, SUM(occurrence_count) as occurrences, COUNT(*) as days_occurred
|
|
2308
|
+
FROM warning_digests
|
|
2309
|
+
GROUP BY normalized_message
|
|
2310
|
+
ORDER BY occurrences DESC
|
|
2311
|
+
LIMIT 5
|
|
2312
|
+
`).all<{ normalized_message: string; occurrences: number; days_occurred: number }>();
|
|
2313
|
+
|
|
2314
|
+
return new Response(JSON.stringify({
|
|
2315
|
+
byDate: byDate.results || [],
|
|
2316
|
+
byScript: byScript.results || [],
|
|
2317
|
+
byRepo: byRepo.results || [],
|
|
2318
|
+
todayDigests: todayDigests || { count: 0, occurrences: 0 },
|
|
2319
|
+
yesterdayDigests: yesterdayDigests || { count: 0, occurrences: 0 },
|
|
2320
|
+
totalDigests: totals?.totalDigests || 0,
|
|
2321
|
+
totalOccurrences: totals?.totalOccurrences || 0,
|
|
2322
|
+
topWarnings: topWarnings.results || [],
|
|
2323
|
+
}), {
|
|
2324
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2325
|
+
});
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
/**
|
|
2329
|
+
* Get single error by fingerprint
|
|
2330
|
+
*/
|
|
2331
|
+
async function handleGetError(
|
|
2332
|
+
fingerprint: string,
|
|
2333
|
+
env: Env
|
|
2334
|
+
): Promise<Response> {
|
|
2335
|
+
const error = await env.PLATFORM_DB.prepare(`
|
|
2336
|
+
SELECT *
|
|
2337
|
+
FROM error_occurrences
|
|
2338
|
+
WHERE fingerprint = ?
|
|
2339
|
+
`).bind(fingerprint).first();
|
|
2340
|
+
|
|
2341
|
+
if (!error) {
|
|
2342
|
+
return new Response(JSON.stringify({ error: 'Error not found' }), {
|
|
2343
|
+
status: 404,
|
|
2344
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2345
|
+
});
|
|
2346
|
+
}
|
|
2347
|
+
|
|
2348
|
+
return new Response(JSON.stringify(error), {
|
|
2349
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2350
|
+
});
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
/**
|
|
2354
|
+
* Mute an error (add cf:muted label to GitHub issue)
|
|
2355
|
+
*/
|
|
2356
|
+
async function handleMuteError(
|
|
2357
|
+
fingerprint: string,
|
|
2358
|
+
env: Env
|
|
2359
|
+
): Promise<Response> {
|
|
2360
|
+
// Get error details
|
|
2361
|
+
const error = await env.PLATFORM_DB.prepare(`
|
|
2362
|
+
SELECT github_issue_number, github_repo
|
|
2363
|
+
FROM error_occurrences
|
|
2364
|
+
WHERE fingerprint = ?
|
|
2365
|
+
`).bind(fingerprint).first<{ github_issue_number: number; github_repo: string }>();
|
|
2366
|
+
|
|
2367
|
+
if (!error || !error.github_issue_number) {
|
|
2368
|
+
return new Response(JSON.stringify({ error: 'Error not found or has no GitHub issue' }), {
|
|
2369
|
+
status: 404,
|
|
2370
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2371
|
+
});
|
|
2372
|
+
}
|
|
2373
|
+
|
|
2374
|
+
try {
|
|
2375
|
+
const github = new GitHubClient(env);
|
|
2376
|
+
const [owner, repo] = error.github_repo.split('/');
|
|
2377
|
+
|
|
2378
|
+
await github.addLabels(owner, repo, error.github_issue_number, ['cf:muted']);
|
|
2379
|
+
|
|
2380
|
+
return new Response(JSON.stringify({
|
|
2381
|
+
success: true,
|
|
2382
|
+
message: `Muted error - added cf:muted label to issue #${error.github_issue_number}`
|
|
2383
|
+
}), {
|
|
2384
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2385
|
+
});
|
|
2386
|
+
} catch (e) {
|
|
2387
|
+
return new Response(JSON.stringify({ error: `Failed to mute: ${e}` }), {
|
|
2388
|
+
status: 500,
|
|
2389
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2390
|
+
});
|
|
2391
|
+
}
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2394
|
+
/**
|
|
2395
|
+
* Resolve an error manually
|
|
2396
|
+
*/
|
|
2397
|
+
async function handleResolveError(
|
|
2398
|
+
fingerprint: string,
|
|
2399
|
+
env: Env
|
|
2400
|
+
): Promise<Response> {
|
|
2401
|
+
// Get error details
|
|
2402
|
+
const error = await env.PLATFORM_DB.prepare(`
|
|
2403
|
+
SELECT github_issue_number, github_repo
|
|
2404
|
+
FROM error_occurrences
|
|
2405
|
+
WHERE fingerprint = ?
|
|
2406
|
+
`).bind(fingerprint).first<{ github_issue_number: number; github_repo: string }>();
|
|
2407
|
+
|
|
2408
|
+
if (!error) {
|
|
2409
|
+
return new Response(JSON.stringify({ error: 'Error not found' }), {
|
|
2410
|
+
status: 404,
|
|
2411
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2412
|
+
});
|
|
2413
|
+
}
|
|
2414
|
+
|
|
2415
|
+
// Update D1
|
|
2416
|
+
await env.PLATFORM_DB.prepare(`
|
|
2417
|
+
UPDATE error_occurrences
|
|
2418
|
+
SET status = 'resolved',
|
|
2419
|
+
resolved_at = unixepoch(),
|
|
2420
|
+
resolved_by = 'dashboard',
|
|
2421
|
+
updated_at = unixepoch()
|
|
2422
|
+
WHERE fingerprint = ?
|
|
2423
|
+
`).bind(fingerprint).run();
|
|
2424
|
+
|
|
2425
|
+
// Update KV cache
|
|
2426
|
+
const kvKey = `ERROR_FINGERPRINT:${fingerprint}`;
|
|
2427
|
+
const cached = await env.PLATFORM_CACHE.get(kvKey);
|
|
2428
|
+
if (cached) {
|
|
2429
|
+
const data = JSON.parse(cached);
|
|
2430
|
+
await env.PLATFORM_CACHE.put(
|
|
2431
|
+
kvKey,
|
|
2432
|
+
JSON.stringify({ ...data, status: 'resolved' }),
|
|
2433
|
+
{ expirationTtl: 90 * 24 * 60 * 60 }
|
|
2434
|
+
);
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
// Close GitHub issue if exists
|
|
2438
|
+
if (error.github_issue_number) {
|
|
2439
|
+
try {
|
|
2440
|
+
const github = new GitHubClient(env);
|
|
2441
|
+
const [owner, repo] = error.github_repo.split('/');
|
|
2442
|
+
|
|
2443
|
+
await github.updateIssue({
|
|
2444
|
+
owner,
|
|
2445
|
+
repo,
|
|
2446
|
+
issue_number: error.github_issue_number,
|
|
2447
|
+
state: 'closed',
|
|
2448
|
+
});
|
|
2449
|
+
|
|
2450
|
+
await github.addComment(
|
|
2451
|
+
owner,
|
|
2452
|
+
repo,
|
|
2453
|
+
error.github_issue_number,
|
|
2454
|
+
`✅ **Resolved via Dashboard**\n\nThis error was marked as resolved from the Platform Dashboard.`
|
|
2455
|
+
);
|
|
2456
|
+
} catch (e) {
|
|
2457
|
+
console.error(`Failed to close GitHub issue: ${e}`);
|
|
2458
|
+
}
|
|
2459
|
+
}
|
|
2460
|
+
|
|
2461
|
+
return new Response(JSON.stringify({
|
|
2462
|
+
success: true,
|
|
2463
|
+
message: 'Error marked as resolved'
|
|
2464
|
+
}), {
|
|
2465
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2466
|
+
});
|
|
2467
|
+
}
|
|
2468
|
+
|
|
2469
|
+
// ============================================================================
|
|
2470
|
+
// Main HTTP Handler
|
|
2471
|
+
// ============================================================================
|
|
2472
|
+
|
|
2473
|
+
/**
|
|
2474
|
+
* HTTP handler - webhook endpoint for GitHub events + dashboard API
|
|
2475
|
+
*/
|
|
2476
|
+
async function fetch(request: Request, env: Env): Promise<Response> {
|
|
2477
|
+
const url = new URL(request.url);
|
|
2478
|
+
|
|
2479
|
+
// Health check
|
|
2480
|
+
if (url.pathname === '/health') {
|
|
2481
|
+
return new Response(JSON.stringify({ status: 'ok', worker: 'error-collector' }), {
|
|
2482
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2483
|
+
});
|
|
2484
|
+
}
|
|
2485
|
+
|
|
2486
|
+
// ============================================================================
|
|
2487
|
+
// Dashboard API endpoints
|
|
2488
|
+
// ============================================================================
|
|
2489
|
+
|
|
2490
|
+
// GET /errors - List errors with filtering
|
|
2491
|
+
if (url.pathname === '/errors' && request.method === 'GET') {
|
|
2492
|
+
return handleListErrors(request, env);
|
|
2493
|
+
}
|
|
2494
|
+
|
|
2495
|
+
// GET /errors/stats - Get error statistics
|
|
2496
|
+
if (url.pathname === '/errors/stats' && request.method === 'GET') {
|
|
2497
|
+
return handleErrorStats(env);
|
|
2498
|
+
}
|
|
2499
|
+
|
|
2500
|
+
// GET /errors/:fingerprint - Get single error
|
|
2501
|
+
const errorMatch = url.pathname.match(/^\/errors\/([a-f0-9]+)$/);
|
|
2502
|
+
if (errorMatch && request.method === 'GET') {
|
|
2503
|
+
return handleGetError(errorMatch[1], env);
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2506
|
+
// POST /errors/:fingerprint/mute - Mute an error
|
|
2507
|
+
const muteMatch = url.pathname.match(/^\/errors\/([a-f0-9]+)\/mute$/);
|
|
2508
|
+
if (muteMatch && request.method === 'POST') {
|
|
2509
|
+
return handleMuteError(muteMatch[1], env);
|
|
2510
|
+
}
|
|
2511
|
+
|
|
2512
|
+
// POST /errors/:fingerprint/resolve - Resolve an error
|
|
2513
|
+
const resolveMatch = url.pathname.match(/^\/errors\/([a-f0-9]+)\/resolve$/);
|
|
2514
|
+
if (resolveMatch && request.method === 'POST') {
|
|
2515
|
+
return handleResolveError(resolveMatch[1], env);
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2518
|
+
// GET /digests - List warning digests with filtering
|
|
2519
|
+
if (url.pathname === '/digests' && request.method === 'GET') {
|
|
2520
|
+
return handleListDigests(request, env);
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
// GET /digests/stats - Get digest statistics
|
|
2524
|
+
if (url.pathname === '/digests/stats' && request.method === 'GET') {
|
|
2525
|
+
return handleDigestStats(env);
|
|
2526
|
+
}
|
|
2527
|
+
|
|
2528
|
+
// POST /gap-alerts - Create GitHub issue for data coverage gap
|
|
2529
|
+
// Called by platform-sentinel when a project has low coverage
|
|
2530
|
+
if (url.pathname === '/gap-alerts' && request.method === 'POST') {
|
|
2531
|
+
try {
|
|
2532
|
+
const event = (await request.json()) as GapAlertEvent;
|
|
2533
|
+
|
|
2534
|
+
// Validate required fields
|
|
2535
|
+
if (!event.project || event.coveragePct === undefined) {
|
|
2536
|
+
return new Response(
|
|
2537
|
+
JSON.stringify({ error: 'Missing required fields: project, coveragePct' }),
|
|
2538
|
+
{ status: 400, headers: { 'Content-Type': 'application/json' } }
|
|
2539
|
+
);
|
|
2540
|
+
}
|
|
2541
|
+
|
|
2542
|
+
const result = await processGapAlert(event, env);
|
|
2543
|
+
|
|
2544
|
+
return new Response(JSON.stringify(result), {
|
|
2545
|
+
status: result.processed ? 201 : 200,
|
|
2546
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2547
|
+
});
|
|
2548
|
+
} catch (e) {
|
|
2549
|
+
console.error('Gap alert processing error:', e);
|
|
2550
|
+
return new Response(
|
|
2551
|
+
JSON.stringify({ error: 'Processing failed', details: String(e) }),
|
|
2552
|
+
{ status: 500, headers: { 'Content-Type': 'application/json' } }
|
|
2553
|
+
);
|
|
2554
|
+
}
|
|
2555
|
+
}
|
|
2556
|
+
|
|
2557
|
+
// POST /email-health-alerts - Create GitHub issues for email health check failures
|
|
2558
|
+
// Called by platform-email-healthcheck when a brand has failing checks
|
|
2559
|
+
if (url.pathname === '/email-health-alerts' && request.method === 'POST') {
|
|
2560
|
+
try {
|
|
2561
|
+
const event = (await request.json()) as EmailHealthAlertEvent;
|
|
2562
|
+
|
|
2563
|
+
// Validate required fields
|
|
2564
|
+
if (!event.brand_id || !event.failures?.length || !event.repository) {
|
|
2565
|
+
return new Response(
|
|
2566
|
+
JSON.stringify({ error: 'Missing required fields: brand_id, failures, repository' }),
|
|
2567
|
+
{ status: 400, headers: { 'Content-Type': 'application/json' } }
|
|
2568
|
+
);
|
|
2569
|
+
}
|
|
2570
|
+
|
|
2571
|
+
const result = await processEmailHealthAlerts(event, env);
|
|
2572
|
+
|
|
2573
|
+
return new Response(JSON.stringify(result), {
|
|
2574
|
+
status: result.processed > 0 ? 201 : 200,
|
|
2575
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2576
|
+
});
|
|
2577
|
+
} catch (e) {
|
|
2578
|
+
console.error('Email health alert processing error:', e);
|
|
2579
|
+
return new Response(
|
|
2580
|
+
JSON.stringify({ error: 'Processing failed', details: String(e) }),
|
|
2581
|
+
{ status: 500, headers: { 'Content-Type': 'application/json' } }
|
|
2582
|
+
);
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
|
|
2586
|
+
// Test error endpoint - triggers an intentional error for testing the error collection pipeline
|
|
2587
|
+
// Usage: GET /test-error?type=exception|soft|warning
|
|
2588
|
+
if (url.pathname === '/test-error') {
|
|
2589
|
+
const errorType = url.searchParams.get('type') || 'exception';
|
|
2590
|
+
|
|
2591
|
+
if (errorType === 'soft') {
|
|
2592
|
+
console.error(
|
|
2593
|
+
'TEST SOFT ERROR: This is a test soft error triggered via /test-error endpoint'
|
|
2594
|
+
);
|
|
2595
|
+
return new Response(JSON.stringify({ triggered: 'soft_error' }), {
|
|
2596
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2597
|
+
});
|
|
2598
|
+
}
|
|
2599
|
+
|
|
2600
|
+
if (errorType === 'warning') {
|
|
2601
|
+
console.warn('TEST WARNING: This is a test warning triggered via /test-error endpoint');
|
|
2602
|
+
return new Response(JSON.stringify({ triggered: 'warning' }), {
|
|
2603
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2604
|
+
});
|
|
2605
|
+
}
|
|
2606
|
+
|
|
2607
|
+
// Default: throw an exception
|
|
2608
|
+
throw new Error('TEST EXCEPTION: This is a test exception triggered via /test-error endpoint');
|
|
2609
|
+
}
|
|
2610
|
+
|
|
2611
|
+
// GitHub webhook endpoint
|
|
2612
|
+
if (url.pathname === '/webhooks/github' && request.method === 'POST') {
|
|
2613
|
+
const eventType = request.headers.get('X-GitHub-Event');
|
|
2614
|
+
|
|
2615
|
+
// Handle ping event (sent when webhook is created)
|
|
2616
|
+
if (eventType === 'ping') {
|
|
2617
|
+
console.log('Received GitHub webhook ping');
|
|
2618
|
+
return new Response(JSON.stringify({ message: 'pong' }), {
|
|
2619
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2620
|
+
});
|
|
2621
|
+
}
|
|
2622
|
+
|
|
2623
|
+
// Only handle issue events
|
|
2624
|
+
if (eventType !== 'issues') {
|
|
2625
|
+
return new Response(
|
|
2626
|
+
JSON.stringify({ processed: false, reason: `Event type '${eventType}' not handled` }),
|
|
2627
|
+
{
|
|
2628
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2629
|
+
}
|
|
2630
|
+
);
|
|
2631
|
+
}
|
|
2632
|
+
|
|
2633
|
+
try {
|
|
2634
|
+
const payload = await request.text();
|
|
2635
|
+
const signature = request.headers.get('X-Hub-Signature-256');
|
|
2636
|
+
|
|
2637
|
+
// Verify signature
|
|
2638
|
+
const isValid = await verifyWebhookSignature(payload, signature, env.GITHUB_WEBHOOK_SECRET);
|
|
2639
|
+
if (!isValid) {
|
|
2640
|
+
console.error('Invalid webhook signature');
|
|
2641
|
+
return new Response(JSON.stringify({ error: 'Invalid signature' }), {
|
|
2642
|
+
status: 401,
|
|
2643
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2644
|
+
});
|
|
2645
|
+
}
|
|
2646
|
+
|
|
2647
|
+
// Parse and process issue event
|
|
2648
|
+
const event = JSON.parse(payload) as GitHubIssueEvent;
|
|
2649
|
+
const result = await handleGitHubWebhook(event, env);
|
|
2650
|
+
|
|
2651
|
+
return new Response(JSON.stringify(result), {
|
|
2652
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2653
|
+
});
|
|
2654
|
+
} catch (e) {
|
|
2655
|
+
console.error(`Webhook processing error: ${e}`);
|
|
2656
|
+
return new Response(JSON.stringify({ error: 'Processing failed', details: String(e) }), {
|
|
2657
|
+
status: 500,
|
|
2658
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2659
|
+
});
|
|
2660
|
+
}
|
|
2661
|
+
}
|
|
2662
|
+
|
|
2663
|
+
return new Response('Not Found', { status: 404 });
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
export default {
|
|
2667
|
+
tail,
|
|
2668
|
+
scheduled,
|
|
2669
|
+
fetch,
|
|
2670
|
+
};
|