@debugkit/dashboard 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/assets/_authed-3se5Egqg.js +1 -0
- package/dist/client/assets/_authed-BfFWbTOO.js +1 -0
- package/dist/client/assets/app-code-BKWbvUMR.js +1 -0
- package/dist/client/assets/closed-issues-Cl-8l4oR.js +1 -0
- package/dist/client/assets/{copy-button-ORgLIh59.js → copy-button-BwRd5q7l.js} +1 -1
- package/dist/client/assets/customers-Cf-R8BOg.js +1 -0
- package/dist/client/assets/empty-state-njgr1sWu.js +7 -0
- package/dist/client/assets/endpoints-BMqAUNyE.js +1 -0
- package/dist/client/assets/events._eventId-CVRZJ-bU.js +2 -0
- package/dist/client/assets/exchange-view-DVwSLcXF.js +2 -0
- package/dist/client/assets/index-DEnCDILF.js +10 -0
- package/dist/client/assets/{invite._token-CxXek2SP.js → invite._token-CKAfd5iB.js} +1 -1
- package/dist/client/assets/issues._fingerprint-Cm72LNM_.js +55 -0
- package/dist/client/assets/login-BTxOhEbN.js +1 -0
- package/dist/client/assets/request-panel-DmtFRF8Z.js +1 -0
- package/dist/client/assets/services._service-CU1a45f3.js +1 -0
- package/dist/client/assets/{sessions._sessionId-3wNImMqM.js → sessions._sessionId-LRo3QNQP.js} +1 -1
- package/dist/client/assets/settings-Ctzvl2LL.js +1 -0
- package/dist/client/assets/src-BiKxgjA_.js +3 -0
- package/dist/client/assets/styles-Tcq0pvWj.css +2 -0
- package/dist/client/assets/{team-Bud2-07A.js → team-xlrjhEGp.js} +1 -1
- package/dist/client/assets/{timeline-BSDhQPpY.js → timeline-WXE49N62.js} +1 -1
- package/dist/client/assets/ui-DRjjeSWy.js +1 -0
- package/dist/client/assets/version-banner-BYe4PVzv.js +1 -0
- package/dist/server/assets/_authed-Bn-IktVS.js +120 -0
- package/dist/server/assets/_authed-RXh7GpoE.js +252 -0
- package/dist/server/assets/_tanstack-start-manifest_v-C4dQ3MxD.js +122 -0
- package/dist/server/assets/app-code-BQL_FcSx.js +212 -0
- package/dist/server/assets/app-name-BcbY1BZN.js +11 -0
- package/dist/server/assets/closed-issues-o9U80FXv.js +45 -0
- package/dist/server/assets/config-Bp8m0B8q.js +57 -0
- package/dist/server/assets/{customers-CvSLt_xV.js → customers-CD84Bz4P.js} +11 -6
- package/dist/server/assets/{demo-BRTkdmQJ.js → demo-D-o8m75S.js} +225 -8
- package/dist/server/assets/{dynamodb-B0V3ZO20.js → dynamodb-CcH0cp7R.js} +55 -2
- package/dist/server/assets/empty-state-wz34oAi8.js +163 -0
- package/dist/server/assets/endpoints-8N389Poe.js +248 -0
- package/dist/server/assets/{endpoints-mPQXpH8C.js → endpoints-C0Cxc_-x.js} +38 -36
- package/dist/server/assets/{events._eventId-pkQBQ9sL.js → events._eventId-BOHkNO5W.js} +90 -351
- package/dist/server/assets/exchange-view-DzoSBt_R.js +112 -0
- package/dist/server/assets/{firestore-Jd-LeWZI.js → firestore-DyvYMiz3.js} +13 -2
- package/dist/server/assets/{fns-Di0vMpkQ.js → fns-BcVGbSID.js} +22 -7
- package/dist/server/assets/fns-DE03YExD.js +831 -0
- package/dist/server/assets/{invite._token-LwWF5YxP.js → invite._token-DhlACkIe.js} +3 -3
- package/dist/server/assets/{issues._fingerprint-DAA0QCuA.js → issues._fingerprint-BBd-_Nn5.js} +400 -28
- package/dist/server/assets/{login-DqjCU4DJ.js → login-CbfdHJvg.js} +3 -3
- package/dist/server/assets/{memory-aqsrtEUn.js → memory-CZBUh1Zc.js} +129 -5
- package/dist/server/assets/{errors-C2KEDL7l.js → queries-wha4cZbw.js} +1442 -129
- package/dist/server/assets/request-panel-DpCSwl2q.js +162 -0
- package/dist/server/assets/{router-C0Dauj2C.js → router-BXkzR89j.js} +197 -91
- package/dist/server/assets/services._service-DWjFFGON.js +309 -0
- package/dist/server/assets/{sessions._sessionId-D7JPV0M4.js → sessions._sessionId-DDspiPTe.js} +2 -2
- package/dist/server/assets/{settings-Drjd64d7.js → settings-DCLIaZ_p.js} +201 -24
- package/dist/server/assets/{src-C4fNgijW.js → src-B2GKPQyN.js} +1 -1
- package/dist/server/assets/{team-D7iXYnSE.js → team-jd_TmWP4.js} +3 -3
- package/dist/server/assets/ui-C3pl0edn.js +302 -0
- package/dist/server/assets/version-banner-BX34ijG5.js +106 -0
- package/dist/server/assets/versions-YI_zK1ql.js +58 -0
- package/dist/server/assets/warm-C3GbXP1_.js +16 -0
- package/dist/server/server.js +40 -28
- package/package.json +4 -4
- package/dist/client/assets/_authed-CNObwuZh.js +0 -7
- package/dist/client/assets/_authed-CuP-He8G.js +0 -1
- package/dist/client/assets/charts-CUvev8lV.js +0 -55
- package/dist/client/assets/customers-DTMEprFQ.js +0 -1
- package/dist/client/assets/endpoints-lMSO2CdM.js +0 -1
- package/dist/client/assets/events._eventId-bWag-i8J.js +0 -3
- package/dist/client/assets/index-BNpacHyi.js +0 -10
- package/dist/client/assets/issues._fingerprint-BQr85TVY.js +0 -1
- package/dist/client/assets/login-Dlk3GyyC.js +0 -1
- package/dist/client/assets/settings-CzbEYB7k.js +0 -1
- package/dist/client/assets/src-Dq5mIUf0.js +0 -3
- package/dist/client/assets/styles-DPsgRH7g.css +0 -2
- package/dist/client/assets/ui-CEv2y56s.js +0 -1
- package/dist/server/assets/_authed-DkkRwdZl.js +0 -87
- package/dist/server/assets/_authed-DsB8kjZB.js +0 -902
- package/dist/server/assets/_tanstack-start-manifest_v-D1_ErrJK.js +0 -97
- package/dist/server/assets/charts-DzTVqJ8z.js +0 -238
- package/dist/server/assets/fns-B3PHbFwV.js +0 -1295
- package/dist/server/assets/ui-B_XHlLUQ.js +0 -193
|
@@ -1,64 +1,110 @@
|
|
|
1
1
|
import { a as getCookie, i as deleteCookie, o as getRequestHeader, s as setCookie } from "../server.js";
|
|
2
|
-
import {
|
|
3
|
-
import { n as
|
|
2
|
+
import { A as hostNameFor, B as OWNER_INFO, C as maskPersonal, D as severityFromCount, E as releaseKey, F as validHostNames, H as markLibraryFrames, I as CATEGORY_INFO, L as CAUSE_INFO, M as nameUrl, N as nameUrlsIn, O as hostKindLabel, P as normaliseHostMatch, R as IMPACT_INFO, S as isTestIssue, T as percentileMs, V as cleanFnName, _ as TEST_TAG, f as OFFLINE_FINGERPRINT, g as SEVERITY_ORDER, j as hostOf$1, l as Envelope, p as OFFLINE_TITLE, r as compareVersions, u as AIRPLANE_TAG, v as compareVersions$1, w as parseStack, x as isTestEvent, y as isOfflineIssue } from "./memory-CZBUh1Zc.js";
|
|
3
|
+
import { n as validAppName, t as appLabel } from "./app-name-BcbY1BZN.js";
|
|
4
|
+
import { n as TEAM_ROLES, t as MAX_RETENTION_DAYS } from "./src-B2GKPQyN.js";
|
|
5
|
+
import { t as config } from "./config-Bp8m0B8q.js";
|
|
4
6
|
import { createHash, createHmac, randomBytes, scryptSync, timingSafeEqual } from "node:crypto";
|
|
5
|
-
|
|
6
|
-
var
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
7
|
+
/** Entries kept at most; the oldest go first. */
|
|
8
|
+
var MAX_ENTRIES = 2e3;
|
|
9
|
+
/** The reads that are cached. */
|
|
10
|
+
var CACHED_READS = /* @__PURE__ */ new Set([
|
|
11
|
+
"getSettings",
|
|
12
|
+
"getVersions",
|
|
13
|
+
"countUsers",
|
|
14
|
+
"listApps",
|
|
15
|
+
"listIssues",
|
|
16
|
+
"getIssue",
|
|
17
|
+
"getRollups",
|
|
18
|
+
"listIssueEvents",
|
|
19
|
+
"listSessionEvents"
|
|
20
|
+
]);
|
|
21
|
+
/** Calls that change data (demo and memory mode write through the store): they clear the cache. */
|
|
22
|
+
var WRITE = /^(write|upsert|increment|put|set|register|note|mark|delete|remove|restamp|create|accept|revoke|change)/;
|
|
23
|
+
/**
|
|
24
|
+
* A short-lived cache in front of the store, so moving between pages doesn't read the same data
|
|
25
|
+
* from Firestore or DynamoDB again and again.
|
|
26
|
+
*
|
|
27
|
+
* - The same read made twice at once (the health wall's summary and endpoints both read the issue
|
|
28
|
+
* list) is made once.
|
|
29
|
+
* - A read newer than FRESH_MS is served as it is; one up to STALE_MS old is served at once and
|
|
30
|
+
* read again in the background, so the next page has it fresh.
|
|
31
|
+
* - A write through the store (demo and memory mode), or clearStoreCache() after a change made
|
|
32
|
+
* through the ingest function's admin routes, clears it. A background read that started
|
|
33
|
+
* before the clear is dropped.
|
|
34
|
+
* - Only CACHED_READS are cached: team and sign-in reads always go to the store.
|
|
35
|
+
*
|
|
36
|
+
* Per server instance, like the sign-in limits in auth.ts: on hosting with several instances,
|
|
37
|
+
* another one can show data up to STALE_MS old after a change. The Refresh button clears this
|
|
38
|
+
* instance's cache.
|
|
39
|
+
*/
|
|
40
|
+
function cacheStore(store, now = Date.now) {
|
|
41
|
+
const entries = /* @__PURE__ */ new Map();
|
|
42
|
+
let generation = 0;
|
|
43
|
+
const read = (key, call) => {
|
|
44
|
+
const t = now();
|
|
45
|
+
const hit = entries.get(key);
|
|
46
|
+
if (hit) {
|
|
47
|
+
entries.delete(key);
|
|
48
|
+
entries.set(key, hit);
|
|
49
|
+
}
|
|
50
|
+
if (hit && t - hit.at < 3e4) return hit.value;
|
|
51
|
+
if (hit && t - hit.at < 12e4) {
|
|
52
|
+
if (!hit.refreshing) {
|
|
53
|
+
hit.refreshing = true;
|
|
54
|
+
const started = generation;
|
|
55
|
+
const next = call();
|
|
56
|
+
next.then(() => generation === started && entries.set(key, {
|
|
57
|
+
at: now(),
|
|
58
|
+
value: next
|
|
59
|
+
}), () => hit.refreshing = false);
|
|
60
|
+
}
|
|
61
|
+
return hit.value;
|
|
62
|
+
}
|
|
63
|
+
const value = call();
|
|
64
|
+
entries.set(key, {
|
|
65
|
+
at: t,
|
|
66
|
+
value
|
|
67
|
+
});
|
|
68
|
+
value.catch(() => entries.get(key)?.value === value && entries.delete(key));
|
|
69
|
+
while (entries.size > MAX_ENTRIES) entries.delete(entries.keys().next().value);
|
|
70
|
+
return value;
|
|
71
|
+
};
|
|
72
|
+
const clear = () => {
|
|
73
|
+
generation++;
|
|
74
|
+
entries.clear();
|
|
29
75
|
};
|
|
30
76
|
return {
|
|
31
|
-
store,
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
trackers: {
|
|
47
|
-
linear: env.DEBUG_LINEAR === "1" || !!env.DEBUG_LINEAR_TEAM,
|
|
48
|
-
linearTeam: env.DEBUG_LINEAR_TEAM || void 0,
|
|
49
|
-
githubRepo: env.DEBUG_GITHUB_REPO || void 0,
|
|
50
|
-
dashboardUrl: env.DEBUG_DASHBOARD_URL || void 0
|
|
51
|
-
},
|
|
52
|
-
secureCookies: env.NODE_ENV === "production" && env.DEBUG_INSECURE_COOKIES !== "1",
|
|
53
|
-
/** Firebase Hosting only forwards a cookie named __session to the function behind it. */
|
|
54
|
-
cookieName: /^[A-Za-z0-9_]{1,40}$/.test(env.DEBUG_COOKIE_NAME ?? "") ? env.DEBUG_COOKIE_NAME : "debug_session"
|
|
77
|
+
store: new Proxy(store, { get(target, prop, receiver) {
|
|
78
|
+
const v = Reflect.get(target, prop, receiver);
|
|
79
|
+
if (typeof v !== "function" || typeof prop !== "string") return v;
|
|
80
|
+
if (CACHED_READS.has(prop)) return (...args) => read(`${prop}:${JSON.stringify(args)}`, () => v.apply(target, args));
|
|
81
|
+
if (!WRITE.test(prop)) return v.bind(target);
|
|
82
|
+
return async (...args) => {
|
|
83
|
+
clear();
|
|
84
|
+
try {
|
|
85
|
+
return await v.apply(target, args);
|
|
86
|
+
} finally {
|
|
87
|
+
clear();
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
} }),
|
|
91
|
+
clear
|
|
55
92
|
};
|
|
56
93
|
}
|
|
57
94
|
// src/server/store.ts
|
|
58
95
|
var store;
|
|
96
|
+
var cache$1;
|
|
59
97
|
/** The dashboard only ever reads. Point it at read-only credentials. */
|
|
60
98
|
function getStore() {
|
|
61
|
-
return store ??= create()
|
|
99
|
+
return store ??= create().then((s) => {
|
|
100
|
+
cache$1 = cacheStore(s);
|
|
101
|
+
import("./warm-C3GbXP1_.js").then((m) => m.warmUp(cache$1.store)).catch(() => void 0);
|
|
102
|
+
return cache$1.store;
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
/** Forget every cached read: called after the dashboard changes anything (status, settings, team). */
|
|
106
|
+
function clearStoreCache() {
|
|
107
|
+
cache$1?.clear();
|
|
62
108
|
}
|
|
63
109
|
/**
|
|
64
110
|
* Demo mode only: the in-memory store also accepts events, so a local app can send to
|
|
@@ -75,23 +121,25 @@ async function create() {
|
|
|
75
121
|
case "firestore": {
|
|
76
122
|
const { initializeApp, getApps } = await import("firebase-admin/app");
|
|
77
123
|
const { getFirestore } = await import("firebase-admin/firestore");
|
|
78
|
-
const { FirestoreStore } = await import("./firestore-
|
|
124
|
+
const { FirestoreStore } = await import("./firestore-DyvYMiz3.js");
|
|
79
125
|
if (!getApps().length) initializeApp();
|
|
80
126
|
return new FirestoreStore(getFirestore(), { root: cfg.firestoreRoot });
|
|
81
127
|
}
|
|
82
128
|
case "dynamodb": {
|
|
83
129
|
const { DynamoDBClient } = await import("@aws-sdk/client-dynamodb");
|
|
84
130
|
const { DynamoDBDocumentClient } = await import("@aws-sdk/lib-dynamodb");
|
|
85
|
-
const { DynamoStore } = await import("./dynamodb-
|
|
131
|
+
const { DynamoStore } = await import("./dynamodb-CcH0cp7R.js");
|
|
86
132
|
return new DynamoStore(DynamoDBDocumentClient.from(new DynamoDBClient({})), { table: cfg.dynamoTable });
|
|
87
133
|
}
|
|
88
134
|
case "memory": {
|
|
89
|
-
const { MemoryStore } = await import("./memory-
|
|
135
|
+
const { MemoryStore } = await import("./memory-CZBUh1Zc.js").then((n) => n.n);
|
|
90
136
|
return new MemoryStore();
|
|
91
137
|
}
|
|
92
138
|
default: {
|
|
93
|
-
const { seedDemo } = await import("./demo-
|
|
94
|
-
|
|
139
|
+
const { keepUpstreamDown, seedDemo } = await import("./demo-D-o8m75S.js");
|
|
140
|
+
const demo = await seedDemo();
|
|
141
|
+
keepUpstreamDown(demo, clearStoreCache);
|
|
142
|
+
return demo;
|
|
95
143
|
}
|
|
96
144
|
}
|
|
97
145
|
}
|
|
@@ -115,6 +163,7 @@ async function admin(method, route, body, query) {
|
|
|
115
163
|
});
|
|
116
164
|
const json = await res.json().catch(() => ({}));
|
|
117
165
|
if (!res.ok) throw new Error(json.error ?? `The ingest function answered ${res.status}`);
|
|
166
|
+
if (method === "PUT") clearStoreCache();
|
|
118
167
|
return json;
|
|
119
168
|
}
|
|
120
169
|
async function readSettings(app) {
|
|
@@ -124,6 +173,7 @@ async function readSettings(app) {
|
|
|
124
173
|
const out = await admin("GET", "/settings", void 0, { app });
|
|
125
174
|
return {
|
|
126
175
|
app,
|
|
176
|
+
displayName: out.settings?.displayName ?? "",
|
|
127
177
|
retentionDays: out.retentionDays,
|
|
128
178
|
criticalFlows: out.settings?.criticalFlows ?? [],
|
|
129
179
|
alerts: out.settings?.alerts ?? {},
|
|
@@ -135,6 +185,7 @@ async function readSettings(app) {
|
|
|
135
185
|
} catch (err) {
|
|
136
186
|
return {
|
|
137
187
|
app,
|
|
188
|
+
displayName: "",
|
|
138
189
|
retentionDays: c.retentionDays ?? null,
|
|
139
190
|
criticalFlows: [],
|
|
140
191
|
alerts: {},
|
|
@@ -148,6 +199,7 @@ async function readSettings(app) {
|
|
|
148
199
|
const hook = s?.alerts?.slackWebhookUrl;
|
|
149
200
|
return {
|
|
150
201
|
app,
|
|
202
|
+
displayName: s?.displayName ?? "",
|
|
151
203
|
retentionDays: s?.retentionDays ?? c.retentionDays ?? (writer === "local" ? 30 : null),
|
|
152
204
|
criticalFlows: s?.criticalFlows ?? [],
|
|
153
205
|
alerts: {
|
|
@@ -161,6 +213,12 @@ async function readSettings(app) {
|
|
|
161
213
|
note: writer === "none" ? "Changing settings needs DEBUG_ADMIN_URL and DEBUG_ADMIN_TOKEN (npx debugkit init sets them)." : void 0
|
|
162
214
|
};
|
|
163
215
|
}
|
|
216
|
+
/** Every app's display name, by id: those with none set are left out (show the id). */
|
|
217
|
+
async function appNames(apps) {
|
|
218
|
+
const store = await getStore();
|
|
219
|
+
const names = await Promise.all(apps.map((a) => store.getSettings(a).then((s) => s?.displayName?.trim() ?? "", () => "")));
|
|
220
|
+
return Object.fromEntries(apps.map((a, i) => [a, names[i]]).filter(([, n]) => n));
|
|
221
|
+
}
|
|
164
222
|
/** Hosts the app called in the last 30 days, busiest first: suggestions for naming them. */
|
|
165
223
|
async function seenHosts(app, now = Date.now()) {
|
|
166
224
|
const day = (t) => new Date(t).toISOString().slice(0, 10);
|
|
@@ -189,12 +247,15 @@ async function writeRetention(app, days) {
|
|
|
189
247
|
else throw new Error("Settings can't be changed from this dashboard: DEBUG_ADMIN_URL and DEBUG_ADMIN_TOKEN aren't set.");
|
|
190
248
|
return readSettings(app);
|
|
191
249
|
}
|
|
192
|
-
/** Change critical flows, alert settings and/or host names. Validation happens in the ingest function (or here, locally). */
|
|
250
|
+
/** Change the display name, critical flows, alert settings and/or host names. Validation happens in the ingest function (or here, locally). */
|
|
193
251
|
async function writeSettings(app, patch) {
|
|
194
252
|
const writer = settingsWriter();
|
|
195
253
|
if (writer === "admin") await admin("PUT", "/settings", {
|
|
196
254
|
app,
|
|
197
255
|
...patch
|
|
256
|
+
}).catch((err) => {
|
|
257
|
+
if (patch.displayName !== void 0 && /nothing to change/i.test(err.message)) throw new Error("Your ingest function is older than app names: upgrade it with npx debugkit upgrade, then save again.");
|
|
258
|
+
throw err;
|
|
198
259
|
});
|
|
199
260
|
else if (writer === "local") {
|
|
200
261
|
const store = await getLocalWriteStore();
|
|
@@ -203,11 +264,14 @@ async function writeSettings(app, patch) {
|
|
|
203
264
|
if (patch.alerts && url && !url.includes("…") && !/^https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/_-]{10,200}$/.test(url)) throw new Error("Use a Slack incoming-webhook URL (https://hooks.slack.com/services/…).");
|
|
204
265
|
const hosts = patch.hosts ? validHostNames(patch.hosts) : void 0;
|
|
205
266
|
if (typeof hosts === "string") throw new Error(hosts);
|
|
267
|
+
const name = patch.displayName === void 0 ? void 0 : validAppName(patch.displayName);
|
|
268
|
+
if (typeof name === "string") throw new Error(name);
|
|
206
269
|
const alerts = patch.alerts ? {
|
|
207
270
|
...patch.alerts,
|
|
208
271
|
slackWebhookUrl: url?.includes("…") ? cur?.alerts?.slackWebhookUrl : url || void 0
|
|
209
272
|
} : void 0;
|
|
210
273
|
await store.putSettings(app, {
|
|
274
|
+
...name !== void 0 ? { displayName: name.name } : {},
|
|
211
275
|
...patch.criticalFlows ? { criticalFlows: patch.criticalFlows } : {},
|
|
212
276
|
...alerts ? { alerts } : {},
|
|
213
277
|
...hosts ? { hosts } : {}
|
|
@@ -255,6 +319,233 @@ async function restampStep(app, cursor) {
|
|
|
255
319
|
}
|
|
256
320
|
throw new Error("Settings can't be changed from this dashboard.");
|
|
257
321
|
}
|
|
322
|
+
// src/lib/health.ts
|
|
323
|
+
/** Worst first when sorted descending; healthy comes last. */
|
|
324
|
+
var HEALTH_ORDER = {
|
|
325
|
+
...SEVERITY_ORDER,
|
|
326
|
+
healthy: -1
|
|
327
|
+
};
|
|
328
|
+
/**
|
|
329
|
+
* What sets a service's level. Two ways to be bad: a large share of calls failing (with enough
|
|
330
|
+
* failures that it isn't one unlucky call), or a large number failing even when the share is
|
|
331
|
+
* small, because that's still many customers.
|
|
332
|
+
*/
|
|
333
|
+
var SERVICE_LEVELS = {
|
|
334
|
+
critical: {
|
|
335
|
+
rate: .25,
|
|
336
|
+
failed: 5,
|
|
337
|
+
volume: 500
|
|
338
|
+
},
|
|
339
|
+
high: {
|
|
340
|
+
rate: .1,
|
|
341
|
+
failed: 3,
|
|
342
|
+
volume: 100
|
|
343
|
+
},
|
|
344
|
+
medium: {
|
|
345
|
+
rate: .05,
|
|
346
|
+
failed: 1
|
|
347
|
+
},
|
|
348
|
+
/** With no request summaries from the SDK: stored failures alone. */
|
|
349
|
+
stored: {
|
|
350
|
+
critical: 50,
|
|
351
|
+
high: 20,
|
|
352
|
+
medium: 5
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
/**
|
|
356
|
+
* An outage: in the last 15 minutes, calls to a service got no answer or a 5xx for at least 3
|
|
357
|
+
* different people, and at least 5% of today's calls to it failed. The people make it more than
|
|
358
|
+
* one phone or one bad request; the share keeps a few unlucky calls on a busy service from
|
|
359
|
+
* reading as down. Request summaries are kept per day, so the share is today's (UTC), not the
|
|
360
|
+
* last 15 minutes'. Without summaries for today, the people alone decide.
|
|
361
|
+
*/
|
|
362
|
+
var OUTAGE = {
|
|
363
|
+
windowMs: 9e5,
|
|
364
|
+
users: 3,
|
|
365
|
+
share: .05
|
|
366
|
+
};
|
|
367
|
+
/** Whether recent no-answer and 5xx calls are an outage: enough people, and enough of today's calls. */
|
|
368
|
+
function isOutage(people, today) {
|
|
369
|
+
if (people < OUTAGE.users) return false;
|
|
370
|
+
return !today?.calls || today.failed / today.calls >= OUTAGE.share;
|
|
371
|
+
}
|
|
372
|
+
/** Failing within this long ago counts as still happening. */
|
|
373
|
+
var LIVE_MS = 36e5;
|
|
374
|
+
/** Still open: not resolved or ignored. */
|
|
375
|
+
var isOpen = (status) => status !== "resolved" && status !== "ignored";
|
|
376
|
+
/** How badly an endpoint fails: the larger of the SDK's failed calls and the stored failures. */
|
|
377
|
+
var failing = (r) => Math.max(r.failedCalls ?? 0, r.failures);
|
|
378
|
+
/** "27%", or "<1%" for a small share that isn't none. */
|
|
379
|
+
var pct$1 = (r) => r > 0 && r < .01 ? "<1%" : `${Math.round(r * 100)}%`;
|
|
380
|
+
/** "1 call", "126 calls"; "2 people" with a plural given. */
|
|
381
|
+
var plural$2 = (n, w, many = `${w}s`) => `${n.toLocaleString()} ${n === 1 ? w : many}`;
|
|
382
|
+
/** How bad a service is, by share and by number of failed calls. See SERVICE_LEVELS. */
|
|
383
|
+
function serviceLevel(s) {
|
|
384
|
+
const L = SERVICE_LEVELS;
|
|
385
|
+
if (s.calls > 0) {
|
|
386
|
+
if (!s.failedCalls && !s.failures) return "healthy";
|
|
387
|
+
const rate = s.failedCalls / s.calls;
|
|
388
|
+
if (s.failedCalls >= L.critical.volume || rate >= L.critical.rate && s.failedCalls >= L.critical.failed) return "critical";
|
|
389
|
+
if (s.failedCalls >= L.high.volume || rate >= L.high.rate && s.failedCalls >= L.high.failed) return "high";
|
|
390
|
+
if (rate >= L.medium.rate && s.failedCalls >= L.medium.failed) return "medium";
|
|
391
|
+
return "low";
|
|
392
|
+
}
|
|
393
|
+
const n = s.failures;
|
|
394
|
+
return !n ? "healthy" : n >= L.stored.critical ? "critical" : n >= L.stored.high ? "high" : n >= L.stored.medium ? "medium" : "low";
|
|
395
|
+
}
|
|
396
|
+
/** One endpoint's level: as a service's, and critical when it's down for many people right now (see OUTAGE). */
|
|
397
|
+
function endpointLevel(e) {
|
|
398
|
+
if (e.down && isOutage(e.down.who.length, e.today)) return "critical";
|
|
399
|
+
return serviceLevel({
|
|
400
|
+
calls: e.calls ?? 0,
|
|
401
|
+
failedCalls: e.failedCalls ?? 0,
|
|
402
|
+
failures: e.failures
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* How to group service names: case doesn't matter, and a plural joins its singular when both are
|
|
407
|
+
* there ("Balances" with "balance"). "status" stays "status" when there's no "statu".
|
|
408
|
+
*/
|
|
409
|
+
function serviceKeys(names) {
|
|
410
|
+
const lower = (n) => (n ?? "").trim().toLowerCase();
|
|
411
|
+
const known = new Set(names.map(lower));
|
|
412
|
+
return (name) => {
|
|
413
|
+
const l = lower(name);
|
|
414
|
+
return l.endsWith("s") && known.has(l.slice(0, -1)) ? l.slice(0, -1) : l;
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
/** Endpoints added up per service, most failures first. */
|
|
418
|
+
function servicesOf(endpoints) {
|
|
419
|
+
const keyOf = serviceKeys(endpoints.map((e) => e.service));
|
|
420
|
+
const by = /* @__PURE__ */ new Map();
|
|
421
|
+
for (const e of endpoints) by.set(keyOf(e.service), [...by.get(keyOf(e.service)) ?? [], e]);
|
|
422
|
+
return [...by.entries()].map(([key, list]) => {
|
|
423
|
+
const calls = list.reduce((n, e) => n + (e.calls ?? 0), 0);
|
|
424
|
+
const failedCalls = list.reduce((n, e) => n + (e.failedCalls ?? 0), 0);
|
|
425
|
+
const p95s = list.map((e) => e.p95).filter((v) => v !== null);
|
|
426
|
+
const busiest = [...list].sort((a, b) => (b.calls ?? 0) - (a.calls ?? 0));
|
|
427
|
+
const kinded = busiest.find((e) => e.kind);
|
|
428
|
+
return {
|
|
429
|
+
key: key || "-",
|
|
430
|
+
service: busiest[0].service,
|
|
431
|
+
...kinded ? {
|
|
432
|
+
kind: kinded.kind,
|
|
433
|
+
...kinded.provider ? { provider: kinded.provider } : {}
|
|
434
|
+
} : {},
|
|
435
|
+
down: downOfService(list),
|
|
436
|
+
endpoints: list.length,
|
|
437
|
+
calls,
|
|
438
|
+
failedCalls,
|
|
439
|
+
failureRate: calls ? failedCalls / calls : null,
|
|
440
|
+
worstP95: p95s.length ? Math.max(...p95s) : null,
|
|
441
|
+
failures: list.reduce((n, e) => n + e.failures, 0),
|
|
442
|
+
lastFailure: list.map((e) => e.lastFailure).filter((t) => !!t).sort().pop()
|
|
443
|
+
};
|
|
444
|
+
}).sort((a, b) => failing(b) - failing(a) || b.calls - a.calls);
|
|
445
|
+
}
|
|
446
|
+
/** A service's recent no-answer and 5xx calls, across its endpoints: people counted once. */
|
|
447
|
+
function downOfService(list) {
|
|
448
|
+
const downs = list.map((e) => e.down).filter((d) => !!d);
|
|
449
|
+
if (!downs.length) return void 0;
|
|
450
|
+
const statuses = /* @__PURE__ */ new Map();
|
|
451
|
+
for (const d of downs) for (const s of d.statuses) statuses.set(s.key, (statuses.get(s.key) ?? 0) + s.count);
|
|
452
|
+
const days = list.map((e) => e.today).filter((t) => !!t);
|
|
453
|
+
return {
|
|
454
|
+
...days.length ? { today: {
|
|
455
|
+
calls: days.reduce((n, t) => n + t.calls, 0),
|
|
456
|
+
failed: days.reduce((n, t) => n + t.failed, 0)
|
|
457
|
+
} } : {},
|
|
458
|
+
users: new Set(downs.flatMap((d) => d.who)).size,
|
|
459
|
+
failures: downs.reduce((n, d) => n + d.failures, 0),
|
|
460
|
+
statuses: [...statuses.entries()].map(([key, count]) => ({
|
|
461
|
+
key,
|
|
462
|
+
count
|
|
463
|
+
})).sort((a, b) => b.count - a.count || a.key.localeCompare(b.key)),
|
|
464
|
+
last: downs.map((d) => d.last).sort().pop()
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* "Cell C Balance is down: no answer or a 5xx for 6 people in the last 15 min": the outage line, when a
|
|
469
|
+
* service is failing broadly right now (see OUTAGE). Undefined when it isn't.
|
|
470
|
+
*/
|
|
471
|
+
function outageOf(s) {
|
|
472
|
+
if (!s.down || !isOutage(s.down.users, s.down.today)) return void 0;
|
|
473
|
+
const name = s.service ?? "Unnamed service";
|
|
474
|
+
return `${s.kind === "upstream" && s.provider && !name.toLowerCase().includes(s.provider.toLowerCase()) ? `${s.provider} ${name}` : name} is down: ${s.down.statuses.every((x) => /^\d+$/.test(x.key)) ? "a 5xx" : s.down.statuses.some((x) => /^\d+$/.test(x.key)) ? "no answer or a 5xx" : "no answer"} for ${plural$2(s.down.users, "person", "people")} in the last ${Math.round(OUTAGE.windowMs / 6e4)} min`;
|
|
475
|
+
}
|
|
476
|
+
/** A service's endpoints, by its key (as in the address). */
|
|
477
|
+
function endpointsOf(key, endpoints) {
|
|
478
|
+
const keyOf = serviceKeys(endpoints.map((e) => e.service));
|
|
479
|
+
const want = key === "-" ? "-" : keyOf(key) || "-";
|
|
480
|
+
return endpoints.filter((e) => (keyOf(e.service) || "-") === want);
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* What's wrong with a service, in one line, from its worst endpoint: the status, the backend's own
|
|
484
|
+
* code and message, and which call. "404 · 30007 Balance lookup unsuccessful · POST /ppd/api/balance/getBalances".
|
|
485
|
+
*/
|
|
486
|
+
function headlineOf(endpoints) {
|
|
487
|
+
const worst = [...endpoints].filter((e) => failing(e) > 0).sort((a, b) => failing(b) - failing(a))[0];
|
|
488
|
+
if (!worst) return void 0;
|
|
489
|
+
return `${[worst.statuses[0]?.key, [worst.backendCodes[0]?.key, worst.backendMessages[0]?.key].filter(Boolean).join(" ")].filter(Boolean).join(" · ") || "Failing"} · ${worst.method ? `${worst.method} ` : ""}${worst.path ?? worst.endpoint}`;
|
|
490
|
+
}
|
|
491
|
+
/** A service's box: its level, what's wrong, and how much. */
|
|
492
|
+
function serviceTile(s, endpoints, now) {
|
|
493
|
+
const outage = outageOf(s);
|
|
494
|
+
const level = outage ? "critical" : serviceLevel(s);
|
|
495
|
+
const measure = s.calls > 0 ? s.failedCalls ? `${pct$1(s.failedCalls / s.calls)} of ${plural$2(s.calls, "call")} failed` : `${plural$2(s.calls, "call")}, none failed` : s.failures ? plural$2(s.failures, "failure") : "No calls";
|
|
496
|
+
return {
|
|
497
|
+
key: s.key,
|
|
498
|
+
name: s.service ?? "Unnamed service",
|
|
499
|
+
kind: "service",
|
|
500
|
+
level,
|
|
501
|
+
headline: level === "healthy" ? void 0 : headlineOf(endpointsOf(s.key, endpoints)),
|
|
502
|
+
measure,
|
|
503
|
+
live: level !== "healthy" && !!s.lastFailure && now - Date.parse(s.lastFailure) < 36e5,
|
|
504
|
+
lastFailure: s.lastFailure,
|
|
505
|
+
weight: failing(s),
|
|
506
|
+
...s.kind ? {
|
|
507
|
+
side: s.kind,
|
|
508
|
+
sideLabel: hostKindLabel(s.kind, s.provider),
|
|
509
|
+
...s.provider ? { provider: s.provider } : {}
|
|
510
|
+
} : {},
|
|
511
|
+
...outage ? { outage } : {}
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
/** Worst level first, then down right now, then still happening, then most failures, then by name. */
|
|
515
|
+
var byHealth = (a, b) => HEALTH_ORDER[b.level] - HEALTH_ORDER[a.level] || Number(!!b.outage) - Number(!!a.outage) || Number(b.live) - Number(a.live) || b.weight - a.weight || a.name.localeCompare(b.name);
|
|
516
|
+
/**
|
|
517
|
+
* Who an outage is with, for a client: an upstream's ("It's Cell C's service, not our backend or
|
|
518
|
+
* the app.") or our own backend's. Nothing when nobody said whose service it is.
|
|
519
|
+
*/
|
|
520
|
+
function whoseLine(t) {
|
|
521
|
+
if (t.side === "upstream") return `It's ${t.provider ? `${t.provider}'s` : "a third party's"} service, not our backend or the app.`;
|
|
522
|
+
return t.side === "ours" ? "It's our own backend, not an upstream service or the app." : "";
|
|
523
|
+
}
|
|
524
|
+
/**
|
|
525
|
+
* The answer to "why is it breaking?", for a client: what's down right now and whose service it
|
|
526
|
+
* is, what's failing badly, the worst of it with what the server said, whether it's still
|
|
527
|
+
* happening, and how much is healthy.
|
|
528
|
+
*/
|
|
529
|
+
function answerOf(tiles, rangeLabel) {
|
|
530
|
+
const down = tiles.filter((t) => t.outage);
|
|
531
|
+
const downLines = down.map((t) => [`${t.outage}.`, whoseLine(t)].filter(Boolean).join(" "));
|
|
532
|
+
const healthyLine = `${tiles.filter((t) => t.level === "healthy").length} of ${tiles.length} ${tiles.length === 1 ? "part is" : "parts are"} healthy.`;
|
|
533
|
+
const failingBadly = tiles.filter((t) => t.level !== "healthy" && t.level !== "low");
|
|
534
|
+
if (!failingBadly.length) return `Nothing is failing badly in ${rangeLabel}. ${healthyLine}`;
|
|
535
|
+
const top = failingBadly.find((t) => !t.outage);
|
|
536
|
+
const names = (ts) => ts.length === 1 ? ts[0].name : `${ts.slice(0, -1).map((t) => t.name).join(", ")} and ${ts[ts.length - 1].name}`;
|
|
537
|
+
const critical = failingBadly.filter((t) => t.level === "critical" && !t.outage);
|
|
538
|
+
const others = failingBadly.filter((t) => !t.outage);
|
|
539
|
+
const lead = critical.length ? `${critical.length === 1 ? "1 part is" : `${critical.length} parts are`} ${down.length ? "also " : ""}failing badly in ${rangeLabel}: ${names(critical)}.` : others.length ? `${others.length === 1 ? "1 part has" : `${others.length} parts have`} ${down.length ? "other " : ""}problems in ${rangeLabel}: ${names(others)}.` : "";
|
|
540
|
+
const live = failingBadly.filter((t) => t.live).length;
|
|
541
|
+
return [
|
|
542
|
+
...downLines,
|
|
543
|
+
lead,
|
|
544
|
+
top ? `${down.length ? "Worst otherwise" : "Worst"}: ${top.name}, ${top.measure}${top.headline ? ` (${top.headline})` : ""}.` : "",
|
|
545
|
+
live ? `${live === 1 ? "1 is" : `${live} are`} still happening (failed in the last hour).` : "",
|
|
546
|
+
healthyLine
|
|
547
|
+
].filter(Boolean).join(" ");
|
|
548
|
+
}
|
|
258
549
|
var EMAIL = /^[^\s@/]{1,64}@[^\s@/]{1,190}\.[^\s@/]{2,63}$/;
|
|
259
550
|
var HASH = /^scrypt\.[0-9a-f]{32}\.[0-9a-f]{128}$/;
|
|
260
551
|
var bad = (error, status = 400) => ({
|
|
@@ -440,7 +731,7 @@ async function sendAlerts(app, outcome, issues, settings, post = defaultPost) {
|
|
|
440
731
|
u.flow && `flow \`${u.flow}\`${critical.has(u.flow) ? " (critical)" : ""}`,
|
|
441
732
|
u.version && `version ${u.version}`
|
|
442
733
|
].filter(Boolean).join(" · ");
|
|
443
|
-
const text = `${kind === "new" ? ":new: New issue" : ":rotating_light: Regressed"} in *${app}*: ${link ? `<${link}|${title}>` : title}${where ? `\n${where}` : ""}`;
|
|
734
|
+
const text = `${kind === "new" ? ":new: New issue" : ":rotating_light: Regressed"} in *${appLabel(app, settings?.displayName)}*: ${link ? `<${link}|${title}>` : title}${where ? `\n${where}` : ""}`;
|
|
444
735
|
try {
|
|
445
736
|
await post(a.slackWebhookUrl, { text });
|
|
446
737
|
sent++;
|
|
@@ -576,10 +867,7 @@ function countryOf(ctx) {
|
|
|
576
867
|
if (tz) return tz;
|
|
577
868
|
return /[-_]([A-Z]{2})(?:\b|$)/.exec(ctx.locale ?? "")?.[1] ?? "unknown";
|
|
578
869
|
}
|
|
579
|
-
// ../../packages/ingest/src/classify.ts
|
|
580
|
-
var FOREGROUND = "App came to the foreground";
|
|
581
870
|
var BACKGROUND = "App went to the background";
|
|
582
|
-
var NEVER_SENT_MS = 100;
|
|
583
871
|
/** How long after reopening a failure still counts as "on resume", if nothing has succeeded since. */
|
|
584
872
|
var RESUME_WINDOW_MS = 1e4;
|
|
585
873
|
var CRASH_SOURCES = /* @__PURE__ */ new Set(["global-handler", "uncaughtException"]);
|
|
@@ -637,7 +925,7 @@ function category(e) {
|
|
|
637
925
|
const source = e.tags?.source;
|
|
638
926
|
if (r && r.status === void 0) {
|
|
639
927
|
if (TIMED_OUT.test(e.error.message)) return "network.timeout";
|
|
640
|
-
if (r.durationMs !== void 0 && r.durationMs <
|
|
928
|
+
if (r.durationMs !== void 0 && r.durationMs < 100) return "network.never_sent";
|
|
641
929
|
return "network.failed";
|
|
642
930
|
}
|
|
643
931
|
if (r?.status !== void 0) {
|
|
@@ -652,11 +940,17 @@ function category(e) {
|
|
|
652
940
|
if (!r) return "app_reported";
|
|
653
941
|
return "other";
|
|
654
942
|
}
|
|
943
|
+
/**
|
|
944
|
+
* A request that never left the phone (ABA-132): no status, too fast to have reached a server, not
|
|
945
|
+
* a timeout. Every one counts as the phone being offline, including the first of a burst, which
|
|
946
|
+
* has no failed calls before it to show it was one.
|
|
947
|
+
*/
|
|
948
|
+
var isOfflineFailure = (labels) => labels.category === "network.never_sent";
|
|
655
949
|
/** The last foreground/background breadcrumb before the error. */
|
|
656
950
|
function lastLifecycle(crumbs) {
|
|
657
951
|
for (let i = crumbs.length - 1; i >= 0; i--) {
|
|
658
952
|
const b = crumbs[i];
|
|
659
|
-
if (b.category === "ui" && (b.message ===
|
|
953
|
+
if (b.category === "ui" && (b.message === "App came to the foreground" || b.message === "App went to the background")) return b;
|
|
660
954
|
}
|
|
661
955
|
}
|
|
662
956
|
function gather(e, crumbs, repeatedFailure) {
|
|
@@ -670,7 +964,7 @@ function gather(e, crumbs, repeatedFailure) {
|
|
|
670
964
|
r: crumbRequest(b)
|
|
671
965
|
})).filter((x) => !!x.r);
|
|
672
966
|
const life = lastLifecycle(crumbs);
|
|
673
|
-
const foreground = life?.message ===
|
|
967
|
+
const foreground = life?.message === "App came to the foreground" ? Date.parse(life.ts) : void 0;
|
|
674
968
|
const burst = requests.filter((x) => x.r.failed && within(x.b, 5e3));
|
|
675
969
|
const hosts = new Set(burst.map((x) => hostOf(x.r.url)));
|
|
676
970
|
if (e.request && (burst.length || repeatedFailure)) hosts.add(hostOf(e.request.url));
|
|
@@ -898,17 +1192,24 @@ function fingerprint(e) {
|
|
|
898
1192
|
const f = topInAppFrame(e.error.frames);
|
|
899
1193
|
const r = e.request;
|
|
900
1194
|
const where = r ? `${r.method.toUpperCase()} ${r.url} ${r.status ? `${Math.floor(r.status / 100)}xx` : "failed"}` : f ? `${cleanFnName(f.fn) ?? "?"}@${normaliseFile(f.file)}` : normaliseMessage(e.error.message);
|
|
901
|
-
|
|
1195
|
+
const parts = [
|
|
902
1196
|
e.error.name,
|
|
903
1197
|
where,
|
|
904
|
-
e.flow ?? ""
|
|
905
|
-
|
|
1198
|
+
e.flow ?? "",
|
|
1199
|
+
...isTestEvent(e) ? [TEST_TAG] : []
|
|
1200
|
+
];
|
|
1201
|
+
return createHash("sha256").update(parts.join("|")).digest("hex").slice(0, 24);
|
|
906
1202
|
}
|
|
907
1203
|
function titleOf(e) {
|
|
908
1204
|
return `${e.error.name}: ${e.error.message}`.slice(0, 200);
|
|
909
1205
|
}
|
|
1206
|
+
// ../../packages/ingest/src/version.ts
|
|
1207
|
+
/** This ingest's version: the package's, which every release bumps with the SDK's (one changeset group). */
|
|
1208
|
+
var INGEST_VERSION = "0.10.0";
|
|
1209
|
+
/** Sent on every ingest response, so an SDK (or the CLI) can tell how old the ingest it talks to is. */
|
|
1210
|
+
var VERSION_HEADER = "x-debugkit-version";
|
|
910
1211
|
// ../../packages/ingest/src/index.ts
|
|
911
|
-
/** "pk_abc:
|
|
1212
|
+
/** "pk_abc:megatron-native,pk_def:other-app" → { pk_abc: "megatron-native", ... } */
|
|
912
1213
|
function parseIngestKeys(raw) {
|
|
913
1214
|
const out = {};
|
|
914
1215
|
for (const pair of (raw ?? "").split(",").map((s) => s.trim()).filter(Boolean)) {
|
|
@@ -1006,6 +1307,8 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1006
1307
|
const settings = env.events.some((e) => e.type === "error") ? await cachedSettings(opts.store, app) : null;
|
|
1007
1308
|
const criticalFlows = settings?.criticalFlows ?? [];
|
|
1008
1309
|
const hosts = settings?.hosts ?? [];
|
|
1310
|
+
const errorsHere = env.events.filter((e) => e.type === "error");
|
|
1311
|
+
const onlyTestErrors = errorsHere.length > 0 && errorsHere.every((e) => isTestEvent(e));
|
|
1009
1312
|
/** This batch's errors, to group into incidents once they're all labelled. */
|
|
1010
1313
|
const batchErrors = [];
|
|
1011
1314
|
for (const raw of env.events) {
|
|
@@ -1019,6 +1322,7 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1019
1322
|
}
|
|
1020
1323
|
const day = dayOf(event.ts);
|
|
1021
1324
|
if (event.type === "session") {
|
|
1325
|
+
if (onlyTestErrors && (event.status === "errored" || event.status === "crashed")) continue;
|
|
1022
1326
|
rollups.push({
|
|
1023
1327
|
app,
|
|
1024
1328
|
day,
|
|
@@ -1067,6 +1371,10 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1067
1371
|
}));
|
|
1068
1372
|
continue;
|
|
1069
1373
|
}
|
|
1374
|
+
if (event.type !== "error" && isTestEvent(event)) {
|
|
1375
|
+
stored.push(storedEvent(day, event));
|
|
1376
|
+
continue;
|
|
1377
|
+
}
|
|
1070
1378
|
const kind = event.type === "error" ? "error" : "event";
|
|
1071
1379
|
const inc = (dimension, key) => rollups.push({
|
|
1072
1380
|
app,
|
|
@@ -1079,6 +1387,7 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1079
1387
|
let fp;
|
|
1080
1388
|
let labels;
|
|
1081
1389
|
let se;
|
|
1390
|
+
let offline = false;
|
|
1082
1391
|
if (event.type === "error") {
|
|
1083
1392
|
fp = fingerprint(event);
|
|
1084
1393
|
labels = classify({
|
|
@@ -1088,9 +1397,19 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1088
1397
|
criticalFlows,
|
|
1089
1398
|
hosts
|
|
1090
1399
|
});
|
|
1400
|
+
const test = isTestEvent(event);
|
|
1401
|
+
offline = !test && isOfflineFailure(labels);
|
|
1402
|
+
if (offline) {
|
|
1403
|
+
labels = {
|
|
1404
|
+
...labels,
|
|
1405
|
+
category: "network.offline",
|
|
1406
|
+
owner: "unknown"
|
|
1407
|
+
};
|
|
1408
|
+
fp = OFFLINE_FINGERPRINT;
|
|
1409
|
+
}
|
|
1091
1410
|
const cur = issues.get(fp);
|
|
1092
1411
|
se = storedEvent(day, event, fp, labels);
|
|
1093
|
-
batchErrors.push({
|
|
1412
|
+
if (!test && !offline) batchErrors.push({
|
|
1094
1413
|
ts: raw.ts,
|
|
1095
1414
|
labels,
|
|
1096
1415
|
se,
|
|
@@ -1099,9 +1418,9 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1099
1418
|
issues.set(fp, {
|
|
1100
1419
|
app,
|
|
1101
1420
|
fingerprint: fp,
|
|
1102
|
-
title: titleOf(event),
|
|
1103
|
-
culprit: culpritOf(event),
|
|
1104
|
-
flow: event.flow,
|
|
1421
|
+
title: offline ? OFFLINE_TITLE : titleOf(event),
|
|
1422
|
+
culprit: offline ? void 0 : culpritOf(event),
|
|
1423
|
+
flow: offline ? void 0 : event.flow,
|
|
1105
1424
|
by: (cur?.by ?? 0) + 1,
|
|
1106
1425
|
ts: cur && cur.ts > event.ts ? cur.ts : event.ts,
|
|
1107
1426
|
firstTs: cur && cur.firstTs < event.ts ? cur.firstTs : event.ts,
|
|
@@ -1110,7 +1429,6 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1110
1429
|
});
|
|
1111
1430
|
scopes.add(`issue:${fp}`);
|
|
1112
1431
|
inc("issue", fp);
|
|
1113
|
-
if (event.request) inc("endpoint", `${event.request.method.toUpperCase()} ${event.request.url}`);
|
|
1114
1432
|
rollups.push({
|
|
1115
1433
|
app,
|
|
1116
1434
|
day: event.ts.slice(0, 13),
|
|
@@ -1119,9 +1437,14 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1119
1437
|
kind,
|
|
1120
1438
|
by: 1
|
|
1121
1439
|
});
|
|
1440
|
+
if (test) {
|
|
1441
|
+
stored.push(se);
|
|
1442
|
+
continue;
|
|
1443
|
+
}
|
|
1444
|
+
if (event.request && !offline) inc("endpoint", `${event.request.method.toUpperCase()} ${event.request.url}`);
|
|
1122
1445
|
}
|
|
1123
1446
|
inc("total", "all");
|
|
1124
|
-
inc("flow", event.flow ?? "untagged");
|
|
1447
|
+
if (!offline) inc("flow", event.flow ?? "untagged");
|
|
1125
1448
|
inc("os", `${context.os.name} ${context.os.version ?? ""}`.trim());
|
|
1126
1449
|
inc("appVersion", context.app.version);
|
|
1127
1450
|
inc("deviceModel", context.device.model);
|
|
@@ -1134,7 +1457,7 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1134
1457
|
inc("country", countryOf(context));
|
|
1135
1458
|
if (labels) {
|
|
1136
1459
|
inc("category", labels.category);
|
|
1137
|
-
inc("owner", labels.owner);
|
|
1460
|
+
if (!offline) inc("owner", labels.owner);
|
|
1138
1461
|
inc("impact", labels.impact);
|
|
1139
1462
|
}
|
|
1140
1463
|
}
|
|
@@ -1180,9 +1503,16 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1180
1503
|
stored.length ? opts.store.writeEvents(stored, { retentionDays }) : void 0,
|
|
1181
1504
|
opts.store.upsertIssues([...issues.values()]),
|
|
1182
1505
|
opts.store.incrementRollups(rollups),
|
|
1183
|
-
markUsers(opts.store, app, env.user?.id ?? env.user?.anonymousId ?? env.session.id, scopes, dayOf(receivedAt))
|
|
1506
|
+
markUsers(opts.store, app, env.user?.id ?? env.user?.anonymousId ?? env.session.id, scopes, dayOf(receivedAt)),
|
|
1507
|
+
noteVersions(opts.store, app, env.sdk.version, receivedAt)
|
|
1184
1508
|
]);
|
|
1185
|
-
|
|
1509
|
+
const real = (fp) => !isOfflineIssue(issues.get(fp)) && !isTestIssue(issues.get(fp));
|
|
1510
|
+
const alerting = outcome && {
|
|
1511
|
+
...outcome,
|
|
1512
|
+
created: outcome.created.filter(real),
|
|
1513
|
+
regressed: outcome.regressed.filter(real)
|
|
1514
|
+
};
|
|
1515
|
+
if (alerting && (alerting.created.length || alerting.regressed.length)) await sendAlerts(app, alerting, issues, settings);
|
|
1186
1516
|
return {
|
|
1187
1517
|
status: 202,
|
|
1188
1518
|
body: {
|
|
@@ -1191,6 +1521,42 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1191
1521
|
}
|
|
1192
1522
|
};
|
|
1193
1523
|
}
|
|
1524
|
+
/**
|
|
1525
|
+
* What this instance last recorded (or tried to) per store and app, so a steady app costs one
|
|
1526
|
+
* versions write a day, not one per batch, and a store that's down isn't retried on every batch.
|
|
1527
|
+
*/
|
|
1528
|
+
var recordedVersions = /* @__PURE__ */ new WeakMap();
|
|
1529
|
+
var VERSIONS_REFRESH_MS = 864e5;
|
|
1530
|
+
var VERSIONS_RETRY_MS = 6e5;
|
|
1531
|
+
/**
|
|
1532
|
+
* Record the app's SDK version and this ingest's, when the SDK is newer than what this instance last
|
|
1533
|
+
* recorded, or a day has passed. A failure here never fails the envelope; it's tried again in ten minutes.
|
|
1534
|
+
*/
|
|
1535
|
+
async function noteVersions(store, app, sdk, at) {
|
|
1536
|
+
let byApp = recordedVersions.get(store);
|
|
1537
|
+
if (!byApp) recordedVersions.set(store, byApp = /* @__PURE__ */ new Map());
|
|
1538
|
+
const had = byApp.get(app);
|
|
1539
|
+
const now = Date.parse(at);
|
|
1540
|
+
if (had && compareVersions(sdk, had.sdk) <= 0 && now - had.at < VERSIONS_REFRESH_MS) return;
|
|
1541
|
+
const newest = compareVersions(sdk, had?.sdk) > 0 ? sdk : had?.sdk;
|
|
1542
|
+
try {
|
|
1543
|
+
await store.recordVersions(app, {
|
|
1544
|
+
sdk,
|
|
1545
|
+
ingest: INGEST_VERSION,
|
|
1546
|
+
at
|
|
1547
|
+
});
|
|
1548
|
+
byApp.set(app, {
|
|
1549
|
+
sdk: newest,
|
|
1550
|
+
at: now
|
|
1551
|
+
});
|
|
1552
|
+
} catch (err) {
|
|
1553
|
+
byApp.set(app, {
|
|
1554
|
+
sdk: newest,
|
|
1555
|
+
at: now - VERSIONS_REFRESH_MS + VERSIONS_RETRY_MS
|
|
1556
|
+
});
|
|
1557
|
+
console.warn("[deBug] couldn't record versions", err);
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1194
1560
|
/** Recently written (scope, member, day) marks, so a busy user costs one write per day, not per batch. */
|
|
1195
1561
|
var seenMarks = /* @__PURE__ */ new Set();
|
|
1196
1562
|
/**
|
|
@@ -1214,11 +1580,14 @@ async function markUsers(store, app, who, scopes, day) {
|
|
|
1214
1580
|
if (seenMarks.size > 5e4) seenMarks.clear();
|
|
1215
1581
|
if (marks.length) await store.markUsers(app, marks);
|
|
1216
1582
|
}
|
|
1583
|
+
/** On every ingest response: CORS for browser SDKs, and this ingest's version, readable from the browser too. */
|
|
1217
1584
|
var CORS_HEADERS = {
|
|
1218
1585
|
"access-control-allow-origin": "*",
|
|
1219
1586
|
"access-control-allow-methods": "POST, OPTIONS",
|
|
1220
1587
|
"access-control-allow-headers": "content-type, x-debug-key",
|
|
1221
|
-
"access-control-
|
|
1588
|
+
"access-control-expose-headers": VERSION_HEADER,
|
|
1589
|
+
"access-control-max-age": "86400",
|
|
1590
|
+
[VERSION_HEADER]: INGEST_VERSION
|
|
1222
1591
|
};
|
|
1223
1592
|
// src/server/auth.ts
|
|
1224
1593
|
var cookie = () => config().cookieName;
|
|
@@ -1447,9 +1816,11 @@ var PASS_TO = {
|
|
|
1447
1816
|
third_party: "whoever manages that third-party service",
|
|
1448
1817
|
unknown: "app team first, to find out whose it is"
|
|
1449
1818
|
};
|
|
1819
|
+
/** Nobody owns the phone's signal, but the app decides what happens without it (ABA-132). */
|
|
1820
|
+
var OFFLINE_PASS_TO = "Pass to: app team, to handle being offline. No endpoint or server is at fault.";
|
|
1450
1821
|
var TEMPLATES = {
|
|
1451
1822
|
"network.offline": {
|
|
1452
|
-
saw: (
|
|
1823
|
+
saw: () => "Requests never left the phone",
|
|
1453
1824
|
why: () => "The phone was offline.",
|
|
1454
1825
|
diagnosis: () => "No connection when the request was made.",
|
|
1455
1826
|
fix: () => "Check connectivity before sending (NetInfo), queue the request until the phone is back online, and show an offline state."
|
|
@@ -1464,7 +1835,7 @@ var TEMPLATES = {
|
|
|
1464
1835
|
saw: (n) => `${cap(n)} took too long and gave up`,
|
|
1465
1836
|
why: () => "The request was sent but no answer came back in time: a slow network or a slow server.",
|
|
1466
1837
|
diagnosis: (_ev, took) => `Timed out${took ? ` after ${took}` : ""} with no response.`,
|
|
1467
|
-
fix: () => "Check this endpoint's p95 under Response times. If
|
|
1838
|
+
fix: () => "Check this endpoint's p95 under Response times. If it's slow on every network, give that service's owner the timings and the times above; in the app, raise the timeout, retry idempotent requests, and show a retry state."
|
|
1468
1839
|
},
|
|
1469
1840
|
"network.failed": {
|
|
1470
1841
|
saw: (n) => `${cap(n)} couldn't connect`,
|
|
@@ -1554,6 +1925,7 @@ function explain(input) {
|
|
|
1554
1925
|
const e = ev0?.type === "error" ? ev0 : void 0;
|
|
1555
1926
|
const ev = e ? evidenceOf(e) : NO_EVIDENCE;
|
|
1556
1927
|
const t = TEMPLATES[labels.category] ?? TEMPLATES.other;
|
|
1928
|
+
const offline = labels.category === "network.offline";
|
|
1557
1929
|
const causes = labels.causes;
|
|
1558
1930
|
const model = input.lastEvent?.context.device.model;
|
|
1559
1931
|
const service = e?.request ? hostNameFor(e.request.url, input.hosts)?.host.name : void 0;
|
|
@@ -1565,16 +1937,19 @@ function explain(input) {
|
|
|
1565
1937
|
const action = userAction(e?.error.frames);
|
|
1566
1938
|
const doing = action ? ` (after ${action.action})` : "";
|
|
1567
1939
|
const causeSentences = causes.flatMap((c) => CAUSE_LINES[c]?.summary?.({ model }) ?? []);
|
|
1940
|
+
const said = !offline && (input.backend?.message || input.backend?.code) ? input.backend : void 0;
|
|
1941
|
+
const saidService = said?.service ?? service;
|
|
1942
|
+
const answer = said ? `${said.message ? `"${said.message}"` : ""}${said.code ? `${said.message ? " " : ""}(code ${said.code})` : ""}` : "";
|
|
1568
1943
|
const summary = [
|
|
1569
|
-
`${t.saw(s.name, s.plural ? "are" : "is")}${who}${where}${when}${doing}.`,
|
|
1570
|
-
t.why(ev, causes),
|
|
1944
|
+
said ? `${saidService ? `The ${saidService} service` : "The server"} answered${said.status ? ` ${said.status}` : ""} ${answer}${who}${where}${when}${doing}.` : `${t.saw(s.name, s.plural ? "are" : "is")}${who}${where}${when}${doing}.`,
|
|
1945
|
+
said ? "That's the service's own reason: start there." : t.why(ev, causes),
|
|
1571
1946
|
...causeSentences,
|
|
1572
1947
|
stillHappening(issue, rangeLabel, now),
|
|
1573
|
-
`Pass to: ${PASS_TO[labels.owner] ?? PASS_TO.unknown}${service && (labels.owner === "backend" || labels.owner === "third_party") ? ` (${service})` : ""}.`
|
|
1948
|
+
offline ? OFFLINE_PASS_TO : said ? `Pass to: whoever runs ${saidService ? `the ${saidService} service` : "that service"}, with ${said.code ? `code ${said.code}` : "its answer"}${said.traceId ? ` and trace ${said.traceId}` : ""}.` : `Pass to: ${PASS_TO[labels.owner] ?? PASS_TO.unknown}${service && (labels.owner === "backend" || labels.owner === "third_party") ? ` (${service})` : ""}.`
|
|
1574
1949
|
].join(" ");
|
|
1575
1950
|
const took = durationText(input.durations?.length ? input.durations : ev.durationMs !== void 0 ? [ev.durationMs] : []);
|
|
1576
|
-
const evidence = [t.diagnosis(ev, took)];
|
|
1577
|
-
if (e?.request) evidence.push(`Request: \`${e.request.method.toUpperCase()} ${e.request.url}\`${service ? ` (${service})` : ""}.`);
|
|
1951
|
+
const evidence = said ? [`HTTP ${said.status ?? ev.status ?? "error"}, and the backend's own answer: ${answer}${said.traceId ? `, trace \`${said.traceId}\`` : ""}.`, ...said.codes.length > 1 || (said.codes[0]?.count ?? 0) > 1 ? [`Backend codes in recent occurrences: ${said.codes.map((c) => `${c.key} × ${c.count}`).join(", ")}.`] : []] : [t.diagnosis(ev, took)];
|
|
1952
|
+
if (e?.request && !offline) evidence.push(`Request: \`${e.request.method.toUpperCase()} ${e.request.url}\`${service ? ` (${service})` : ""}.`);
|
|
1578
1953
|
if (causes.includes("resumed_from_background") && ev.msSinceForeground !== void 0) evidence.push(`The app came to the foreground ${seconds(ev.msSinceForeground)} before; no request has succeeded since.`);
|
|
1579
1954
|
if (ev.failedBefore > 0) evidence.push(`${plural$1(ev.failedBefore, "other request")} failed in the 5 s before${ev.failedHosts.length > 1 ? `, across ${ev.failedHosts.length} hosts` : ""}.`);
|
|
1580
1955
|
else if (ev.repeatedFailure) evidence.push("The same request had already failed just before.");
|
|
@@ -1583,7 +1958,12 @@ function explain(input) {
|
|
|
1583
1958
|
for (const c of causes) if (CAUSE_LINES[c]?.fix) evidence.push(`${CAUSE_INFO[c].name}: ${CAUSE_INFO[c].description}`);
|
|
1584
1959
|
if (issue.incidentTitle) evidence.push(`Part of the incident "${issue.incidentTitle}".`);
|
|
1585
1960
|
const whereParts = [issue.culprit ? `\`${issue.culprit}\`` : void 0, action ? `started by \`${action.fn}\` (${action.action})` : void 0].filter(Boolean);
|
|
1586
|
-
const
|
|
1961
|
+
const lookUp = said && [said.code && `code ${said.code}`, said.traceId && `trace ${said.traceId}`].filter(Boolean).join(" or ");
|
|
1962
|
+
const fixSteps = [
|
|
1963
|
+
said ? `Search the ${saidService ?? "backend"} logs${lookUp ? ` for ${lookUp}` : ""} at the times above: the service's answer (under The call that failed) says why it refused or failed.` : t.fix(ev, causes),
|
|
1964
|
+
...said && labels.category === "auth" ? [t.fix(ev, causes)] : [],
|
|
1965
|
+
...causes.flatMap((c) => CAUSE_LINES[c]?.fix ?? [])
|
|
1966
|
+
];
|
|
1587
1967
|
return {
|
|
1588
1968
|
summary,
|
|
1589
1969
|
technical: {
|
|
@@ -1594,8 +1974,12 @@ function explain(input) {
|
|
|
1594
1974
|
}
|
|
1595
1975
|
};
|
|
1596
1976
|
}
|
|
1597
|
-
/**
|
|
1598
|
-
|
|
1977
|
+
/**
|
|
1978
|
+
* The explanation as Markdown lines, without a title: for tickets and prefilled links. Specific
|
|
1979
|
+
* findings from the trail (ABA-146) come before the template's advice, which is labelled "General
|
|
1980
|
+
* advice" so it isn't mistaken for specific.
|
|
1981
|
+
*/
|
|
1982
|
+
function explanationLines(x, findings = []) {
|
|
1599
1983
|
return [
|
|
1600
1984
|
"### What happened",
|
|
1601
1985
|
x.summary,
|
|
@@ -1605,19 +1989,867 @@ function explanationLines(x) {
|
|
|
1605
1989
|
"",
|
|
1606
1990
|
...x.technical.evidence.map((l) => `- ${l}`),
|
|
1607
1991
|
...x.technical.where ? ["", `**Where:** ${x.technical.where}`] : [],
|
|
1992
|
+
...findings.length ? [
|
|
1993
|
+
"",
|
|
1994
|
+
"### What the trail shows",
|
|
1995
|
+
...findings.map(findingLine)
|
|
1996
|
+
] : [],
|
|
1608
1997
|
"",
|
|
1609
|
-
`**
|
|
1998
|
+
`**General advice:** ${x.technical.fix}`
|
|
1610
1999
|
];
|
|
1611
2000
|
}
|
|
2001
|
+
/** "- Why. Fix. (2 of 2 occurrences)" */
|
|
2002
|
+
var findingLine = (f) => `- ${f.why} ${f.fix}${f.of > 1 ? ` (${f.seen} of ${f.of} occurrences)` : ""}`;
|
|
1612
2003
|
/** The explanation as Markdown, for pasting into a Jira or Linear ticket by hand. */
|
|
1613
|
-
function explanationMarkdown(title, x, link) {
|
|
2004
|
+
function explanationMarkdown(title, x, link, findings = []) {
|
|
1614
2005
|
return [
|
|
1615
2006
|
`## ${title}`,
|
|
1616
2007
|
...link ? ["", `deBug: ${link}`] : [],
|
|
1617
2008
|
"",
|
|
1618
|
-
...explanationLines(x)
|
|
2009
|
+
...explanationLines(x, findings)
|
|
1619
2010
|
].join("\n");
|
|
1620
2011
|
}
|
|
2012
|
+
// src/server/exchange.ts
|
|
2013
|
+
function sideOf(s) {
|
|
2014
|
+
return {
|
|
2015
|
+
headers: s.headers && Object.keys(s.headers).length ? s.headers : void 0,
|
|
2016
|
+
body: s.body,
|
|
2017
|
+
bodySize: s.bodySize,
|
|
2018
|
+
truncated: s.truncated || void 0
|
|
2019
|
+
};
|
|
2020
|
+
}
|
|
2021
|
+
/** Whether the SDK captured anything beyond the call itself: only with network.bodies on. */
|
|
2022
|
+
function captured(r, crumbs) {
|
|
2023
|
+
return r.query !== void 0 || !!r.headers || r.body !== void 0 || r.bodySize !== void 0 || crumbs.some((b) => b.category === "network" && b.data?.hasBodies === true);
|
|
2024
|
+
}
|
|
2025
|
+
/** The request and response the SDK captured on an error, or undefined when it captured none. */
|
|
2026
|
+
function exchangeOf(ev) {
|
|
2027
|
+
const r = ev.request;
|
|
2028
|
+
if (!r || !(captured(r, ev.breadcrumbs ?? []) || ev.response)) return void 0;
|
|
2029
|
+
const fullUrl = r.url && !r.url.startsWith("/") ? `https://${r.url}` : void 0;
|
|
2030
|
+
return {
|
|
2031
|
+
request: {
|
|
2032
|
+
method: r.method.toUpperCase(),
|
|
2033
|
+
url: r.url,
|
|
2034
|
+
fullUrl,
|
|
2035
|
+
query: r.query?.replace(/^\?/, "") || void 0,
|
|
2036
|
+
...sideOf(r)
|
|
2037
|
+
},
|
|
2038
|
+
response: ev.response ? {
|
|
2039
|
+
status: ev.response.status,
|
|
2040
|
+
...sideOf(ev.response)
|
|
2041
|
+
} : void 0
|
|
2042
|
+
};
|
|
2043
|
+
}
|
|
2044
|
+
// src/server/errors.ts
|
|
2045
|
+
/** Firestore answers FAILED_PRECONDITION while an index deBug needs is missing or still building. */
|
|
2046
|
+
function isIndexError(message) {
|
|
2047
|
+
return /FAILED_PRECONDITION|requires an index|index is currently building/i.test(message);
|
|
2048
|
+
}
|
|
2049
|
+
/** A thrown Firestore error saying a composite index is missing (gRPC code 9), on the Firestore store only. */
|
|
2050
|
+
function isMissingIndex(e, store) {
|
|
2051
|
+
if (store !== "firestore") return false;
|
|
2052
|
+
const err = e;
|
|
2053
|
+
return (err?.code === 9 || err?.code === "failed-precondition") && /index/i.test(String(err?.message ?? ""));
|
|
2054
|
+
}
|
|
2055
|
+
/** A fatal error: labelled a crash, or uncaught and critical, which is how the SDK sends a fatal one. */
|
|
2056
|
+
function isCrash(e) {
|
|
2057
|
+
return e.event.type === "error" && (e.labels?.category === "crash" || !e.event.handled && e.event.severity === "critical");
|
|
2058
|
+
}
|
|
2059
|
+
/** crashed, errored (an uncaught error) or clean, as on the session page. */
|
|
2060
|
+
function sessionOutcome(events) {
|
|
2061
|
+
if (events.some(isCrash)) return "crashed";
|
|
2062
|
+
return events.some((e) => e.event.type === "error" && !e.event.handled) ? "errored" : "clean";
|
|
2063
|
+
}
|
|
2064
|
+
/** The prop that says why a flow failed, in the order apps tend to name it. */
|
|
2065
|
+
var REASON_KEYS = [
|
|
2066
|
+
"reason",
|
|
2067
|
+
"error",
|
|
2068
|
+
"errorKind",
|
|
2069
|
+
"code",
|
|
2070
|
+
"message"
|
|
2071
|
+
];
|
|
2072
|
+
var OUTCOME_WORD = {
|
|
2073
|
+
success: "succeeded",
|
|
2074
|
+
failure: "failed",
|
|
2075
|
+
abandoned: "abandoned"
|
|
2076
|
+
};
|
|
2077
|
+
/** "auth.login: failed, wrongOtp". */
|
|
2078
|
+
function describeEvent(name, outcome, props, step) {
|
|
2079
|
+
const reason = REASON_KEYS.map((k) => props?.[k]).find((v) => typeof v === "string" || typeof v === "number");
|
|
2080
|
+
const parts = [outcome ? OUTCOME_WORD[outcome] ?? outcome : void 0, reason !== void 0 ? String(reason) : void 0].filter(Boolean);
|
|
2081
|
+
return `${name}${step ? ` · ${step}` : ""}${parts.length ? `: ${parts.join(", ")}` : ""}`;
|
|
2082
|
+
}
|
|
2083
|
+
/**
|
|
2084
|
+
* Each breadcrumb once: errors in the same session carry overlapping trails. Given the events
|
|
2085
|
+
* oldest first, each breadcrumb's source is the earliest error that carried it, the one it led to.
|
|
2086
|
+
*
|
|
2087
|
+
* When the SDK merges a repeat it moves the breadcrumb's time to the latest repeat, so a later
|
|
2088
|
+
* trail can hold the same breadcrumb with a newer time and a higher count. Nothing else was
|
|
2089
|
+
* recorded between it and the breadcrumb before it, so an identical one in that window is the
|
|
2090
|
+
* same run: keep the copy with the highest count.
|
|
2091
|
+
*/
|
|
2092
|
+
function uniqueCrumbs(events) {
|
|
2093
|
+
const out = [];
|
|
2094
|
+
const exact = /* @__PURE__ */ new Map();
|
|
2095
|
+
const count = (b) => Number(b.data?.repeated ?? 1);
|
|
2096
|
+
for (const e of events) {
|
|
2097
|
+
if (e.event.type !== "error") continue;
|
|
2098
|
+
const source = e.fingerprint ? {
|
|
2099
|
+
eventId: e.event.id,
|
|
2100
|
+
fingerprint: e.fingerprint
|
|
2101
|
+
} : void 0;
|
|
2102
|
+
let after = -Infinity;
|
|
2103
|
+
for (const b of e.event.breadcrumbs ?? []) {
|
|
2104
|
+
const t = Date.parse(b.ts);
|
|
2105
|
+
const same = (c) => c.crumb.category === b.category && c.crumb.message === b.message && c.crumb.level === b.level;
|
|
2106
|
+
const had = exact.get(`${b.ts}|${b.category}|${b.message}`) ?? (count(b) > 1 ? out.find((c) => same(c) && Date.parse(c.crumb.ts) > after && Date.parse(c.crumb.ts) <= t) : void 0);
|
|
2107
|
+
if (!had) {
|
|
2108
|
+
const c = {
|
|
2109
|
+
crumb: b,
|
|
2110
|
+
source
|
|
2111
|
+
};
|
|
2112
|
+
out.push(c);
|
|
2113
|
+
exact.set(`${b.ts}|${b.category}|${b.message}`, c);
|
|
2114
|
+
} else if (count(b) > count(had.crumb)) {
|
|
2115
|
+
had.crumb = b;
|
|
2116
|
+
exact.set(`${b.ts}|${b.category}|${b.message}`, had);
|
|
2117
|
+
}
|
|
2118
|
+
after = t;
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
return out;
|
|
2122
|
+
}
|
|
2123
|
+
/** How far apart a custom event and its own breadcrumb, or an api.failed event and its call, can be. */
|
|
2124
|
+
var SAME_MS = 2e3;
|
|
2125
|
+
/** The SDK writes each custom event as an "event" breadcrumb too: the breadcrumb that is this event, if any. */
|
|
2126
|
+
function ownCrumb(name, ts, crumbs, taken) {
|
|
2127
|
+
const t = Date.parse(ts);
|
|
2128
|
+
return crumbs.find((c) => !taken.has(c) && c.crumb.category === "event" && c.crumb.message === name && Math.abs(Date.parse(c.crumb.ts) - t) <= SAME_MS);
|
|
2129
|
+
}
|
|
2130
|
+
/** The app's api.failed event says the same as the failed call's breadcrumb next to it. */
|
|
2131
|
+
function repeatsCrumb(e, crumbs) {
|
|
2132
|
+
if (e.event.type !== "event" || e.event.name !== "api.failed") return false;
|
|
2133
|
+
const path = e.event.props?.path;
|
|
2134
|
+
const t = Date.parse(e.event.ts);
|
|
2135
|
+
return crumbs.some((c) => c.api?.failed && c.api.path === path && Math.abs(Date.parse(c.ts) - t) <= SAME_MS);
|
|
2136
|
+
}
|
|
2137
|
+
async function buildSessionDetail(store, app, sessionId) {
|
|
2138
|
+
const [events, settings] = await Promise.all([store.listSessionEvents(app, sessionId, { limit: 501 }), store.getSettings(app).catch(() => null)]);
|
|
2139
|
+
if (!events.length) return null;
|
|
2140
|
+
const truncated = events.length > 500;
|
|
2141
|
+
const page = events.slice(-500);
|
|
2142
|
+
const hosts = settings?.hosts ?? [];
|
|
2143
|
+
const carried = withoutSdkTwinsCarried(uniqueCrumbs(page));
|
|
2144
|
+
const taken = /* @__PURE__ */ new Set();
|
|
2145
|
+
const own = /* @__PURE__ */ new Map();
|
|
2146
|
+
for (const e of page) {
|
|
2147
|
+
if (e.event.type !== "event") continue;
|
|
2148
|
+
const c = ownCrumb(e.event.name, e.event.ts, carried, taken);
|
|
2149
|
+
if (c) {
|
|
2150
|
+
taken.add(c);
|
|
2151
|
+
own.set(e, c);
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
const kept = carried.filter((c) => !taken.has(c)).map((c) => ({
|
|
2155
|
+
ts: c.crumb.ts,
|
|
2156
|
+
item: readCrumb(c.crumb, hosts),
|
|
2157
|
+
source: c.source
|
|
2158
|
+
}));
|
|
2159
|
+
const crumbs = kept.map((c) => c.item);
|
|
2160
|
+
const rows = kept.map((c) => ({
|
|
2161
|
+
kind: "crumb",
|
|
2162
|
+
...c
|
|
2163
|
+
}));
|
|
2164
|
+
let errors = 0;
|
|
2165
|
+
let handled = 0;
|
|
2166
|
+
for (const e of page) {
|
|
2167
|
+
const ev = e.event;
|
|
2168
|
+
if (ev.type === "error") {
|
|
2169
|
+
const crash = isCrash(e);
|
|
2170
|
+
errors++;
|
|
2171
|
+
if (ev.handled) handled++;
|
|
2172
|
+
rows.push({
|
|
2173
|
+
kind: "error",
|
|
2174
|
+
ts: ev.ts,
|
|
2175
|
+
eventId: ev.id,
|
|
2176
|
+
fingerprint: e.fingerprint,
|
|
2177
|
+
title: nameUrlsIn(`${ev.error.name}: ${ev.error.message}`, hosts),
|
|
2178
|
+
handled: ev.handled,
|
|
2179
|
+
crash
|
|
2180
|
+
});
|
|
2181
|
+
} else if (ev.type === "event" && !repeatsCrumb(e, crumbs)) rows.push({
|
|
2182
|
+
kind: "event",
|
|
2183
|
+
ts: ev.ts,
|
|
2184
|
+
eventId: ev.id,
|
|
2185
|
+
name: ev.name,
|
|
2186
|
+
outcome: ev.outcome,
|
|
2187
|
+
detail: describeEvent(ev.name, ev.outcome, ev.props, ev.step),
|
|
2188
|
+
failed: ev.outcome === "failure",
|
|
2189
|
+
source: own.get(e)?.source
|
|
2190
|
+
});
|
|
2191
|
+
}
|
|
2192
|
+
const order = {
|
|
2193
|
+
crumb: 0,
|
|
2194
|
+
event: 1,
|
|
2195
|
+
error: 2
|
|
2196
|
+
};
|
|
2197
|
+
const sorted = byTime(rows.sort((a, b) => order[a.kind] - order[b.kind]));
|
|
2198
|
+
const last = page[page.length - 1];
|
|
2199
|
+
return {
|
|
2200
|
+
app,
|
|
2201
|
+
sessionId,
|
|
2202
|
+
user: {
|
|
2203
|
+
id: last.user?.id,
|
|
2204
|
+
anonymousId: last.user?.anonymousId
|
|
2205
|
+
},
|
|
2206
|
+
context: last.context,
|
|
2207
|
+
start: sorted[0]?.ts ?? page[0].event.ts,
|
|
2208
|
+
end: last.event.ts,
|
|
2209
|
+
outcome: sessionOutcome(page),
|
|
2210
|
+
counts: {
|
|
2211
|
+
errors,
|
|
2212
|
+
handled,
|
|
2213
|
+
events: sorted.filter((r) => r.kind === "event").length
|
|
2214
|
+
},
|
|
2215
|
+
rows: sorted,
|
|
2216
|
+
truncated,
|
|
2217
|
+
read: page.length
|
|
2218
|
+
};
|
|
2219
|
+
}
|
|
2220
|
+
/** How far after an error its "what they did next" reaches. */
|
|
2221
|
+
var AFTER_MS = 3e5;
|
|
2222
|
+
/**
|
|
2223
|
+
* What the customer did after an error, from the trails of the session's later errors: the steps
|
|
2224
|
+
* within five minutes of it, at most `limit`. Empty when nothing later was stored or the session
|
|
2225
|
+
* can't be read (Firestore without its index).
|
|
2226
|
+
*/
|
|
2227
|
+
async function stepsAfter(store, app, sessionId, ts, hosts = [], limit = 30) {
|
|
2228
|
+
const events = await store.listSessionEvents(app, sessionId, { limit: 500 }).catch((e) => {
|
|
2229
|
+
if (isIndexError(String(e?.message))) return [];
|
|
2230
|
+
throw e;
|
|
2231
|
+
});
|
|
2232
|
+
const t = Date.parse(ts);
|
|
2233
|
+
return byTime(withoutSdkTwinsCarried(uniqueCrumbs(events)).filter((c) => {
|
|
2234
|
+
const d = Date.parse(c.crumb.ts) - t;
|
|
2235
|
+
return d > 0 && d <= AFTER_MS;
|
|
2236
|
+
}).map((c) => c.crumb)).slice(0, limit).map((b) => readCrumb(b, hosts));
|
|
2237
|
+
}
|
|
2238
|
+
/** withoutSdkTwins over carried breadcrumbs, keeping each one's source. */
|
|
2239
|
+
function withoutSdkTwinsCarried(carried) {
|
|
2240
|
+
const keep = new Set(withoutSdkTwins(carried.map((c) => c.crumb)));
|
|
2241
|
+
return carried.filter((c) => keep.has(c.crumb));
|
|
2242
|
+
}
|
|
2243
|
+
var str = (v) => typeof v === "string" && v ? v : typeof v === "number" ? String(v) : void 0;
|
|
2244
|
+
var num = (v) => typeof v === "number" && Number.isFinite(v) ? v : typeof v === "string" && /^\d+$/.test(v) ? Number(v) : void 0;
|
|
2245
|
+
/** The first key present: the Cell C app's keys and their alternatives ("backend.title" or "backendTitle"). */
|
|
2246
|
+
var pick = (d, ...keys) => keys.map((k) => d[k]).find((v) => v !== void 0 && v !== null && v !== "");
|
|
2247
|
+
/** API keys read into the call; the rest of the data is shown as it is. */
|
|
2248
|
+
var API_KEYS = /* @__PURE__ */ new Set([
|
|
2249
|
+
"service",
|
|
2250
|
+
"method",
|
|
2251
|
+
"path",
|
|
2252
|
+
"url",
|
|
2253
|
+
"status",
|
|
2254
|
+
"durationMs",
|
|
2255
|
+
"outcome",
|
|
2256
|
+
"retry",
|
|
2257
|
+
"backend.title",
|
|
2258
|
+
"backendTitle",
|
|
2259
|
+
"backend.code",
|
|
2260
|
+
"backendCode",
|
|
2261
|
+
"backendErrors",
|
|
2262
|
+
"traceId",
|
|
2263
|
+
"trace_id",
|
|
2264
|
+
"errorKind"
|
|
2265
|
+
]);
|
|
2266
|
+
/** Outcomes that mean the call did what it should. Anything else the app reports (unreachable, sessionExpired) is a failure. */
|
|
2267
|
+
var OK_OUTCOMES = /* @__PURE__ */ new Set([
|
|
2268
|
+
"answered",
|
|
2269
|
+
"cached",
|
|
2270
|
+
"cancelled",
|
|
2271
|
+
"canceled",
|
|
2272
|
+
"aborted",
|
|
2273
|
+
"success",
|
|
2274
|
+
"ok"
|
|
2275
|
+
]);
|
|
2276
|
+
/** The SDK's own network breadcrumb: "POST api.x.com/v1/pay → 503 (120ms)", or "→ failed". */
|
|
2277
|
+
var SDK_NETWORK = /^([A-Z]+) (\S+) → (\d{3})?/;
|
|
2278
|
+
/** A network breadcrumb as an API call, or undefined when it isn't one ("Device went offline"). */
|
|
2279
|
+
function apiCallOf(b, hosts = []) {
|
|
2280
|
+
if (b.category !== "network") return void 0;
|
|
2281
|
+
const d = b.data ?? {};
|
|
2282
|
+
const sdk = SDK_NETWORK.exec(b.message);
|
|
2283
|
+
const url = str(pick(d, "path", "url")) ?? sdk?.[2];
|
|
2284
|
+
const method = str(pick(d, "method"))?.toUpperCase() ?? sdk?.[1];
|
|
2285
|
+
const appService = str(d.service);
|
|
2286
|
+
if (!url && !method && !appService) return void 0;
|
|
2287
|
+
const status = num(d.status) ?? num(sdk?.[3]);
|
|
2288
|
+
const outcome = str(d.outcome);
|
|
2289
|
+
const named = url && !appService ? hostNameFor(url, hosts) : void 0;
|
|
2290
|
+
const call = {
|
|
2291
|
+
service: appService ?? named?.host.name,
|
|
2292
|
+
method,
|
|
2293
|
+
path: named ? named.rest || "/" : url,
|
|
2294
|
+
status,
|
|
2295
|
+
durationMs: num(d.durationMs),
|
|
2296
|
+
outcome,
|
|
2297
|
+
retry: typeof d.retry === "boolean" || typeof d.retry === "number" ? d.retry : str(d.retry),
|
|
2298
|
+
backendTitle: str(pick(d, "backend.title", "backendTitle")),
|
|
2299
|
+
backendCode: str(pick(d, "backend.code", "backendCode")),
|
|
2300
|
+
backendErrors: str(d.backendErrors),
|
|
2301
|
+
errorKind: str(d.errorKind),
|
|
2302
|
+
traceId: str(pick(d, "traceId", "trace_id")),
|
|
2303
|
+
failed: false
|
|
2304
|
+
};
|
|
2305
|
+
call.failed = b.level === "error" || status !== void 0 && status >= 400 || outcome !== void 0 && !OK_OUTCOMES.has(outcome);
|
|
2306
|
+
return call;
|
|
2307
|
+
}
|
|
2308
|
+
/** How close in time the SDK's breadcrumb and the app's own for the same call are. */
|
|
2309
|
+
var TWIN_MS = 2e3;
|
|
2310
|
+
/**
|
|
2311
|
+
* The SDK's own network breadcrumbs that repeat an app breadcrumb for the same call (same method,
|
|
2312
|
+
* the SDK's URL ending in the app's path, within TWIN_MS): the app's says more, so the SDK's is
|
|
2313
|
+
* left out of the timeline.
|
|
2314
|
+
*/
|
|
2315
|
+
function sdkTwins(crumbs) {
|
|
2316
|
+
const app = crumbs.filter((b) => b.category === "network" && str(b.data?.service) && str(b.data?.path));
|
|
2317
|
+
const out = /* @__PURE__ */ new Set();
|
|
2318
|
+
if (!app.length) return out;
|
|
2319
|
+
for (const b of crumbs) {
|
|
2320
|
+
const sdk = b.category === "network" && !b.data?.service ? SDK_NETWORK.exec(b.message) : null;
|
|
2321
|
+
if (!sdk) continue;
|
|
2322
|
+
const url = sdk[2].split(/[?#]/)[0];
|
|
2323
|
+
const t = Date.parse(b.ts);
|
|
2324
|
+
if (app.some((a) => str(a.data?.method)?.toUpperCase() === sdk[1] && url.endsWith(String(a.data.path)) && Math.abs(Date.parse(a.ts) - t) <= TWIN_MS)) out.add(b);
|
|
2325
|
+
}
|
|
2326
|
+
return out;
|
|
2327
|
+
}
|
|
2328
|
+
/** The breadcrumbs in time order (stable for equal times), each read for what it says. */
|
|
2329
|
+
function timelineOf(crumbs = [], hosts = []) {
|
|
2330
|
+
return byTime(withoutSdkTwins(crumbs)).map((b) => readCrumb(b, hosts));
|
|
2331
|
+
}
|
|
2332
|
+
/** Oldest first, stable for equal times. By time, not by string: "09:00:00Z" and "09:00:00.500Z" don't sort as text. */
|
|
2333
|
+
function byTime(items) {
|
|
2334
|
+
return items.map((x, i) => ({
|
|
2335
|
+
x,
|
|
2336
|
+
i
|
|
2337
|
+
})).sort((p, q) => Date.parse(p.x.ts) - Date.parse(q.x.ts) || p.i - q.i).map(({ x }) => x);
|
|
2338
|
+
}
|
|
2339
|
+
/** The breadcrumbs without the SDK's own copy of a call the app also described. */
|
|
2340
|
+
function withoutSdkTwins(crumbs) {
|
|
2341
|
+
const twins = sdkTwins(crumbs);
|
|
2342
|
+
return crumbs.filter((b) => !twins.has(b));
|
|
2343
|
+
}
|
|
2344
|
+
/** One breadcrumb read for what it says: the API call it describes, or its message. */
|
|
2345
|
+
function readCrumb(b, hosts = []) {
|
|
2346
|
+
const api = apiCallOf(b, hosts);
|
|
2347
|
+
const rest = Object.entries(b.data ?? {}).filter(([k]) => k !== "repeated" && !(api && API_KEYS.has(k)));
|
|
2348
|
+
const repeated = num(b.data?.repeated);
|
|
2349
|
+
return {
|
|
2350
|
+
ts: b.ts,
|
|
2351
|
+
category: b.category,
|
|
2352
|
+
message: nameUrlsIn(b.message, hosts),
|
|
2353
|
+
level: b.level,
|
|
2354
|
+
failed: api ? api.failed : b.level === "error",
|
|
2355
|
+
...api ? { api } : {},
|
|
2356
|
+
...rest.length ? { data: Object.fromEntries(rest) } : {},
|
|
2357
|
+
...repeated && repeated > 1 ? { repeated } : {}
|
|
2358
|
+
};
|
|
2359
|
+
}
|
|
2360
|
+
/** In-app frames first, keeping stack order within each group. */
|
|
2361
|
+
function framesInAppFirst(frames = []) {
|
|
2362
|
+
return [...frames.filter((f) => f.inApp), ...frames.filter((f) => !f.inApp)];
|
|
2363
|
+
}
|
|
2364
|
+
async function buildEventDetail(store, app, fingerprint, eventId) {
|
|
2365
|
+
const [issue, events, settings] = await Promise.all([
|
|
2366
|
+
store.getIssue(app, fingerprint),
|
|
2367
|
+
store.listIssueEvents(app, fingerprint, { limit: 200 }),
|
|
2368
|
+
store.getSettings(app).catch(() => null)
|
|
2369
|
+
]);
|
|
2370
|
+
if (!issue) return null;
|
|
2371
|
+
const errors = (events.length ? events : [issue.lastEvent]).filter((e) => e.event.type === "error");
|
|
2372
|
+
const index = errors.findIndex((e) => e.event.id === eventId);
|
|
2373
|
+
const stored = index >= 0 ? errors[index] : issue.lastEvent.event.id === eventId ? issue.lastEvent : void 0;
|
|
2374
|
+
if (!stored || stored.event.type !== "error") return null;
|
|
2375
|
+
const ev = stored.event;
|
|
2376
|
+
const hosts = settings?.hosts ?? [];
|
|
2377
|
+
const timeline = timelineOf(ev.breadcrumbs, hosts);
|
|
2378
|
+
return {
|
|
2379
|
+
app,
|
|
2380
|
+
issue: {
|
|
2381
|
+
fingerprint,
|
|
2382
|
+
title: nameUrlsIn(issue.title, hosts),
|
|
2383
|
+
status: issue.status ?? "open"
|
|
2384
|
+
},
|
|
2385
|
+
id: ev.id,
|
|
2386
|
+
ts: ev.ts,
|
|
2387
|
+
handled: ev.handled,
|
|
2388
|
+
error: {
|
|
2389
|
+
name: ev.error.name,
|
|
2390
|
+
message: nameUrlsIn(ev.error.message, hosts)
|
|
2391
|
+
},
|
|
2392
|
+
flow: ev.flow,
|
|
2393
|
+
step: ev.step,
|
|
2394
|
+
user: {
|
|
2395
|
+
id: stored.user?.id,
|
|
2396
|
+
anonymousId: stored.user?.anonymousId
|
|
2397
|
+
},
|
|
2398
|
+
sessionId: stored.sessionId,
|
|
2399
|
+
context: stored.context,
|
|
2400
|
+
tags: Object.entries(ev.tags ?? {}).map(([key, value]) => ({
|
|
2401
|
+
key,
|
|
2402
|
+
value: String(value)
|
|
2403
|
+
})).sort((a, b) => a.key.localeCompare(b.key)),
|
|
2404
|
+
timeline,
|
|
2405
|
+
request: failedRequest(ev.request, timeline, hosts),
|
|
2406
|
+
exchange: exchangeOf(ev),
|
|
2407
|
+
frames: framesInAppFirst(ev.error.frames),
|
|
2408
|
+
stack: ev.error.stack,
|
|
2409
|
+
after: await stepsAfter(store, app, stored.sessionId, ev.ts, hosts),
|
|
2410
|
+
newer: index > 0 ? errors[index - 1].event.id : void 0,
|
|
2411
|
+
older: index >= 0 && index < errors.length - 1 ? errors[index + 1].event.id : void 0
|
|
2412
|
+
};
|
|
2413
|
+
}
|
|
2414
|
+
/**
|
|
2415
|
+
* The call that failed: the error's own request, filled in from the app's breadcrumb for the same
|
|
2416
|
+
* call (service, backend code, traceId), or the last failing call when the error has no request.
|
|
2417
|
+
* A breadcrumb counts as the same call only when its method and path match.
|
|
2418
|
+
*/
|
|
2419
|
+
function failedRequest(request, timeline, hosts = []) {
|
|
2420
|
+
const lastFailing = [...timeline].reverse().find((t) => t.api?.failed)?.api;
|
|
2421
|
+
if (!request) return lastFailing;
|
|
2422
|
+
const named = hostNameFor(request.url, hosts);
|
|
2423
|
+
const own = {
|
|
2424
|
+
service: named?.host.name,
|
|
2425
|
+
method: request.method.toUpperCase(),
|
|
2426
|
+
path: named ? named.rest || "/" : request.url,
|
|
2427
|
+
status: request.status,
|
|
2428
|
+
durationMs: request.durationMs,
|
|
2429
|
+
failed: true
|
|
2430
|
+
};
|
|
2431
|
+
const bare = request.url.split(/[?#]/)[0];
|
|
2432
|
+
const sameCall = (c) => c.method === own.method && !!c.path && (bare.endsWith(c.path) || c.path === own.path);
|
|
2433
|
+
const match = [...timeline].reverse().find((t) => t.api?.failed && sameCall(t.api))?.api;
|
|
2434
|
+
return match ? {
|
|
2435
|
+
...match,
|
|
2436
|
+
...defined(own),
|
|
2437
|
+
service: match.service ?? own.service,
|
|
2438
|
+
path: named ? own.path : match.path
|
|
2439
|
+
} : own;
|
|
2440
|
+
}
|
|
2441
|
+
/** Drop undefined fields, so spreading doesn't erase a value from the breadcrumb. */
|
|
2442
|
+
function defined(o) {
|
|
2443
|
+
return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== void 0));
|
|
2444
|
+
}
|
|
2445
|
+
// src/server/hops.ts
|
|
2446
|
+
/** Response headers that carry a trace or request id, best first. */
|
|
2447
|
+
var TRACE_HEADERS = [
|
|
2448
|
+
"traceparent",
|
|
2449
|
+
"x-cloud-trace-context",
|
|
2450
|
+
"x-amzn-trace-id",
|
|
2451
|
+
"x-request-id",
|
|
2452
|
+
"x-amzn-requestid",
|
|
2453
|
+
"x-correlation-id",
|
|
2454
|
+
"request-id"
|
|
2455
|
+
];
|
|
2456
|
+
/** Body fields that carry one, as backends commonly name them. */
|
|
2457
|
+
var TRACE_FIELDS = [
|
|
2458
|
+
"traceId",
|
|
2459
|
+
"trace_id",
|
|
2460
|
+
"requestId",
|
|
2461
|
+
"request_id",
|
|
2462
|
+
"correlationId",
|
|
2463
|
+
"correlation_id"
|
|
2464
|
+
];
|
|
2465
|
+
/** W3C traceparent: version, 32-hex trace id (not all zeros), parent id, flags. */
|
|
2466
|
+
var TRACEPARENT = /^[\da-f]{2}-(?!0{32})([\da-f]{32})-[\da-f]{16}-[\da-f]{2}$/i;
|
|
2467
|
+
/** The trace id inside a header value, or undefined when the value isn't one. */
|
|
2468
|
+
function traceIdIn(header, value) {
|
|
2469
|
+
if (header === "traceparent") return TRACEPARENT.exec(value.trim())?.[1];
|
|
2470
|
+
if (header === "x-cloud-trace-context") return value.split("/")[0] || void 0;
|
|
2471
|
+
if (header === "x-amzn-trace-id") return /(?:^|;)\s*Root=([^;]+)/.exec(value)?.[1];
|
|
2472
|
+
return value.trim() || void 0;
|
|
2473
|
+
}
|
|
2474
|
+
/** A trace id or error code in a JSON response body, top level only. */
|
|
2475
|
+
function idsInBody(body) {
|
|
2476
|
+
if (!body || !body.trimStart().startsWith("{")) return {};
|
|
2477
|
+
try {
|
|
2478
|
+
const json = JSON.parse(body);
|
|
2479
|
+
const text = (v) => typeof v === "string" && v && v !== "[redacted]" ? v : typeof v === "number" ? String(v) : void 0;
|
|
2480
|
+
return {
|
|
2481
|
+
traceId: TRACE_FIELDS.map((k) => text(json[k])).find(Boolean),
|
|
2482
|
+
code: text(json.code) ?? text(json.errorCode)
|
|
2483
|
+
};
|
|
2484
|
+
} catch {
|
|
2485
|
+
return {};
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
/** The trace id the backend already sent: a response header, then the body, then the app's breadcrumb. */
|
|
2489
|
+
function traceOf(headers, body, call) {
|
|
2490
|
+
for (const h of TRACE_HEADERS) {
|
|
2491
|
+
const v = headers?.[h];
|
|
2492
|
+
const id = v && v !== "[redacted]" ? traceIdIn(h, v) : void 0;
|
|
2493
|
+
if (id) return {
|
|
2494
|
+
from: h,
|
|
2495
|
+
id
|
|
2496
|
+
};
|
|
2497
|
+
}
|
|
2498
|
+
const inBody = idsInBody(body).traceId;
|
|
2499
|
+
if (inBody) return {
|
|
2500
|
+
from: "traceId",
|
|
2501
|
+
id: inBody
|
|
2502
|
+
};
|
|
2503
|
+
if (call?.traceId) return {
|
|
2504
|
+
from: "traceId",
|
|
2505
|
+
id: call.traceId
|
|
2506
|
+
};
|
|
2507
|
+
}
|
|
2508
|
+
/** "08:31:44 UTC" for the logs, which are rarely in the phone's timezone. */
|
|
2509
|
+
function utcTime(ts) {
|
|
2510
|
+
const t = new Date(ts);
|
|
2511
|
+
return Number.isNaN(t.getTime()) ? ts : `${t.toISOString().slice(11, 19)} UTC`;
|
|
2512
|
+
}
|
|
2513
|
+
var callLine = (c, withService = true) => `${withService && c.service ? `${c.service} · ` : ""}${c.method ?? ""} ${c.path ?? ""}`.replace(/\s+/g, " ").trim();
|
|
2514
|
+
// src/server/detectors.ts
|
|
2515
|
+
/**
|
|
2516
|
+
* Specific fix advice from patterns in the customer's trail (ABA-146): rule-based, no AI. Each
|
|
2517
|
+
* finding says what happened, what to do, and the evidence it read, so it can be checked.
|
|
2518
|
+
*
|
|
2519
|
+
* No backend changes (ABA-137): a fix may tell the app team what to change, or say where to look
|
|
2520
|
+
* in the backend's logs (time, call, trace id). It never tells anyone to change their backend.
|
|
2521
|
+
*
|
|
2522
|
+
* Not detected: a call still running when the app crashed. Breadcrumbs are written when a call
|
|
2523
|
+
* ends, so one that never answered isn't in the trail (needs the SDK to record request starts).
|
|
2524
|
+
*/
|
|
2525
|
+
var FINDING_IDS = [
|
|
2526
|
+
"repeat_failed",
|
|
2527
|
+
"duplicate_call",
|
|
2528
|
+
"called_before_login",
|
|
2529
|
+
"no_refresh_before_401",
|
|
2530
|
+
"hung_in_background",
|
|
2531
|
+
"expected_answer"
|
|
2532
|
+
];
|
|
2533
|
+
/** Two sends of one call this close together are one action sending it twice. */
|
|
2534
|
+
var DUPLICATE_WINDOW_MS = 5e3;
|
|
2535
|
+
/** A success this recent, then a failure of the same call, is "the repeat failed". */
|
|
2536
|
+
var REPEAT_WINDOW_MS = 6e4;
|
|
2537
|
+
/** Longer than this in the background and a session token has usually expired. */
|
|
2538
|
+
var LONG_BACKGROUND_MS = 18e5;
|
|
2539
|
+
/** A timeout this short isn't a request left hanging. */
|
|
2540
|
+
var HUNG_MIN_MS = 5e3;
|
|
2541
|
+
/** At least this many calls, all with one status, before it looks like the endpoint's normal answer. */
|
|
2542
|
+
var EXPECTED_MIN_CALLS = 3;
|
|
2543
|
+
var ms = (ts) => Date.parse(ts);
|
|
2544
|
+
var isOk = (s) => s !== void 0 && s >= 200 && s < 300;
|
|
2545
|
+
/** "08:31:42.729 UTC", like the log search times (hops.ts utcTime). */
|
|
2546
|
+
var clock = (t) => Number.isNaN(t) ? "?" : `${new Date(t).toISOString().slice(11, 23)} UTC`;
|
|
2547
|
+
/** The path of a stored URL, without host, port, query or hash: "host:443/ppd/x?y" → "/ppd/x". */
|
|
2548
|
+
function pathOf(url) {
|
|
2549
|
+
const bare = url.split(/[?#]/)[0].replace(/^[a-z]+:\/\//i, "");
|
|
2550
|
+
const rest = hostOf$1(bare) ? bare.slice(bare.indexOf("/") < 0 ? bare.length : bare.indexOf("/")) : bare;
|
|
2551
|
+
return rest.startsWith("/") ? rest : `/${rest}`;
|
|
2552
|
+
}
|
|
2553
|
+
/**
|
|
2554
|
+
* Whether a breadcrumb's call is the error's endpoint. Same method; same host when the breadcrumb
|
|
2555
|
+
* names one; and the same path, or the app's path relative to its base URL ("/notifications/x"
|
|
2556
|
+
* under "/ppd/api") ending the request's path on whole segments, at least two of them.
|
|
2557
|
+
*/
|
|
2558
|
+
function sameEndpoint(api, req) {
|
|
2559
|
+
if (!api?.path || api.method !== req.method.toUpperCase()) return false;
|
|
2560
|
+
const host = hostOf$1(api.path);
|
|
2561
|
+
if (host && host !== hostOf$1(req.url)) return false;
|
|
2562
|
+
const want = pathOf(req.url);
|
|
2563
|
+
const have = pathOf(api.path);
|
|
2564
|
+
if (have === want) return true;
|
|
2565
|
+
return !host && have.split("/").length >= 3 && want.endsWith(have);
|
|
2566
|
+
}
|
|
2567
|
+
/** When the call was sent: its end minus its duration (from the data, or the SDK's "(120ms)"). */
|
|
2568
|
+
function sentAt(t) {
|
|
2569
|
+
const took = t.api?.durationMs ?? Number(/\((\d+) ?ms\)/.exec(t.message)?.[1] ?? 0);
|
|
2570
|
+
return ms(t.ts) - took;
|
|
2571
|
+
}
|
|
2572
|
+
/** A login that finished: the app's "Login Completed" or "Logged in", or a 2xx from a login call. */
|
|
2573
|
+
var isLogin = (t) => t.category === "event" && /^(log ?in (completed|succeeded|success)|logged in)$/i.test(t.message.trim()) || !!t.api && isOk(t.api.status) && /\/(login|sign-?in)$/i.test(pathOf(t.api.path ?? ""));
|
|
2574
|
+
/** A token refresh that worked. */
|
|
2575
|
+
var isRefresh = (t) => !!t.api && isOk(t.api.status) && /\/(refresh|refresh-token|token\/refresh|oauth\/token)$/i.test(pathOf(t.api.path ?? ""));
|
|
2576
|
+
var isRetry = (c) => c?.retry === true || typeof c?.retry === "number" && c.retry > 0 || typeof c?.retry === "string" && /^[1-9]/.test(c.retry);
|
|
2577
|
+
function read(ev, hosts) {
|
|
2578
|
+
const at = ms(ev.ts);
|
|
2579
|
+
const trail = timelineOf(ev.breadcrumbs, hosts).filter((t) => !(ms(t.ts) > at));
|
|
2580
|
+
const sent = at - (ev.request?.durationMs ?? 0);
|
|
2581
|
+
if (!ev.request) return {
|
|
2582
|
+
ev,
|
|
2583
|
+
trail,
|
|
2584
|
+
earlier: [],
|
|
2585
|
+
sent
|
|
2586
|
+
};
|
|
2587
|
+
const req = ev.request;
|
|
2588
|
+
const same = trail.filter((t) => sameEndpoint(t.api, req));
|
|
2589
|
+
let i = same.length - 1;
|
|
2590
|
+
while (i >= 0 && !same[i].api.failed) i--;
|
|
2591
|
+
return {
|
|
2592
|
+
ev,
|
|
2593
|
+
trail,
|
|
2594
|
+
call: failedRequest(req, trail, hosts),
|
|
2595
|
+
own: same[i],
|
|
2596
|
+
earlier: i < 0 ? same : same.slice(0, i),
|
|
2597
|
+
sent
|
|
2598
|
+
};
|
|
2599
|
+
}
|
|
2600
|
+
var statusOf = (t) => t.api.status ?? t.api.outcome ?? "no answer";
|
|
2601
|
+
/** Whether one action sent the error's call more than once: the send before it is close, and it isn't the app's own retry. */
|
|
2602
|
+
function sentTwice(r) {
|
|
2603
|
+
if (isRetry(r.own?.api)) return [];
|
|
2604
|
+
return r.earlier.filter((t) => r.sent - sentAt(t) <= DUPLICATE_WINDOW_MS && !isRetry(t.api));
|
|
2605
|
+
}
|
|
2606
|
+
var DETECTORS = {
|
|
2607
|
+
repeat_failed: (r) => {
|
|
2608
|
+
const failed = r.ev.request?.status;
|
|
2609
|
+
if (!r.call || failed === void 0 || !(failed >= 500 || failed === 409)) return void 0;
|
|
2610
|
+
const ok = [...r.earlier].reverse().find((t) => isOk(t.api.status) && r.sent - sentAt(t) <= REPEAT_WINDOW_MS);
|
|
2611
|
+
if (!ok) return void 0;
|
|
2612
|
+
const response = exchangeOf(r.ev)?.response;
|
|
2613
|
+
const trace = traceOf(response?.headers, response?.body, r.call);
|
|
2614
|
+
const service = r.call.service ?? (hostOf$1(r.ev.request.url) || "backend");
|
|
2615
|
+
const where = trace ? `trace id ${trace.id} (${trace.from})` : `${callLine(r.call, false)} sent at ${utcTime(new Date(r.sent).toISOString())}`;
|
|
2616
|
+
const overlapped = r.sent < ms(ok.ts);
|
|
2617
|
+
const twice = sentTwice(r).length > 0;
|
|
2618
|
+
return {
|
|
2619
|
+
why: overlapped ? `The app sent it again before the first call answered; the first worked (${ok.api.status}), the repeat failed (${failed}).` : `The first call worked (${ok.api.status}); the repeat ${seconds(r.sent - sentAt(ok))} later failed (${failed}).`,
|
|
2620
|
+
fix: `${twice ? "App: send it once per action. " : ""}To see why the repeat failed, search the ${service} logs for ${where}.`,
|
|
2621
|
+
evidence: [`${clock(sentAt(ok))} sent, ${clock(ms(ok.ts))} → ${ok.api.status}`, `${clock(r.sent)} sent, ${clock(ms(r.ev.ts))} → ${failed}`]
|
|
2622
|
+
};
|
|
2623
|
+
},
|
|
2624
|
+
duplicate_call: (r) => {
|
|
2625
|
+
const close = sentTwice(r);
|
|
2626
|
+
if (!r.call || !close.length) return void 0;
|
|
2627
|
+
return {
|
|
2628
|
+
why: `The app sends ${callLine(r.call)} ${close.length + 1} times in ${seconds(r.sent - sentAt(close[0]))}.`,
|
|
2629
|
+
fix: "App: send it once per action. Look for two screens or effects that both trigger it.",
|
|
2630
|
+
evidence: [...close.map((t) => `${clock(sentAt(t))} sent → ${statusOf(t)}`), `${clock(r.sent)} sent → ${r.ev.request.status ?? "no answer"}`]
|
|
2631
|
+
};
|
|
2632
|
+
},
|
|
2633
|
+
called_before_login: (r) => {
|
|
2634
|
+
if (!r.call) return void 0;
|
|
2635
|
+
const denied = r.earlier.filter((t) => t.api.status === 401 || t.api.status === 403);
|
|
2636
|
+
if (!denied.length) return void 0;
|
|
2637
|
+
const lastDenied = ms(denied[denied.length - 1].ts);
|
|
2638
|
+
const login = r.trail.find((t) => ms(t.ts) > lastDenied && ms(t.ts) <= r.sent && isLogin(t));
|
|
2639
|
+
if (!login) return void 0;
|
|
2640
|
+
const codes = [...new Set(denied.map((t) => t.api.status))].join(" or ");
|
|
2641
|
+
return {
|
|
2642
|
+
why: `${callLine(r.call)} ran ${denied.length} time${denied.length === 1 ? "" : "s"} before login and got ${codes}.`,
|
|
2643
|
+
fix: "App: wait until login has finished before making this call.",
|
|
2644
|
+
evidence: [...denied.map((t) => `${clock(ms(t.ts))} → ${t.api.status}`), `${clock(ms(login.ts))} ${login.message}`]
|
|
2645
|
+
};
|
|
2646
|
+
},
|
|
2647
|
+
no_refresh_before_401: (r) => {
|
|
2648
|
+
if (!r.call || r.ev.request?.status !== 401) return void 0;
|
|
2649
|
+
const lifecycle = r.trail.filter((t) => t.message === "App came to the foreground" || t.message === "App went to the background");
|
|
2650
|
+
const fore = lifecycle[lifecycle.length - 1];
|
|
2651
|
+
const back = lifecycle[lifecycle.length - 2];
|
|
2652
|
+
if (fore?.message !== "App came to the foreground" || back?.message !== "App went to the background" || ms(fore.ts) - ms(back.ts) < LONG_BACKGROUND_MS) return void 0;
|
|
2653
|
+
if (r.trail.some((t) => ms(t.ts) >= ms(fore.ts) && isRefresh(t))) return void 0;
|
|
2654
|
+
return {
|
|
2655
|
+
why: `The app came back after ${Math.round((ms(fore.ts) - ms(back.ts)) / 6e4)} min in the background and called ${callLine(r.call)} without refreshing its token.`,
|
|
2656
|
+
fix: "App: refresh the session token when the app comes back, before other calls.",
|
|
2657
|
+
evidence: [
|
|
2658
|
+
`${clock(ms(back.ts))} ${back.message}`,
|
|
2659
|
+
`${clock(ms(fore.ts))} ${fore.message}`,
|
|
2660
|
+
`${clock(ms(r.ev.ts))} → 401`
|
|
2661
|
+
]
|
|
2662
|
+
};
|
|
2663
|
+
},
|
|
2664
|
+
hung_in_background: (r) => {
|
|
2665
|
+
const req = r.ev.request;
|
|
2666
|
+
if (!r.call || !req || req.status !== void 0 || req.durationMs === void 0 || req.durationMs < HUNG_MIN_MS) return void 0;
|
|
2667
|
+
if (!/timed out/i.test(r.ev.error.message)) return void 0;
|
|
2668
|
+
const bg = r.trail.find((t) => t.message === "App went to the background" && ms(t.ts) >= r.sent && ms(t.ts) <= ms(r.ev.ts));
|
|
2669
|
+
if (!bg) return void 0;
|
|
2670
|
+
return {
|
|
2671
|
+
why: `${callLine(r.call)} was still running when the app went to the background, and gave up after ${seconds(req.durationMs)}.`,
|
|
2672
|
+
fix: "App: send it before the app goes to the background, or retry it when the app comes back.",
|
|
2673
|
+
evidence: [
|
|
2674
|
+
`${clock(r.sent)} sent (from its duration)`,
|
|
2675
|
+
`${clock(ms(bg.ts))} ${bg.message}`,
|
|
2676
|
+
`${clock(ms(r.ev.ts))} → no answer`
|
|
2677
|
+
]
|
|
2678
|
+
};
|
|
2679
|
+
}
|
|
2680
|
+
};
|
|
2681
|
+
/**
|
|
2682
|
+
* A 4xx an endpoint gives on every call can be its normal answer ("no update", "not found"). A
|
|
2683
|
+
* 5xx every time is an outage, and 401, 403 and 429 mean something to act on: never "expected".
|
|
2684
|
+
*/
|
|
2685
|
+
var MAYBE_EXPECTED = (s) => s >= 400 && s < 500 && s !== 401 && s !== 403 && s !== 429;
|
|
2686
|
+
/** Every call to the newest occurrence's endpoint, across these trails, answered the same 4xx. */
|
|
2687
|
+
function expectedAnswer(reads) {
|
|
2688
|
+
const head = reads.find((r) => r.call && r.ev.request?.status !== void 0);
|
|
2689
|
+
const status = head?.ev.request?.status;
|
|
2690
|
+
if (!head || status === void 0 || !MAYBE_EXPECTED(status)) return void 0;
|
|
2691
|
+
const calls = /* @__PURE__ */ new Map();
|
|
2692
|
+
let seen = 0;
|
|
2693
|
+
for (const r of reads) {
|
|
2694
|
+
const mine = r.trail.filter((t) => sameEndpoint(t.api, head.ev.request));
|
|
2695
|
+
if (mine.length) seen++;
|
|
2696
|
+
for (const t of mine) calls.set(`${t.ts}|${t.api.status}`, t);
|
|
2697
|
+
}
|
|
2698
|
+
const all = [...calls.values()];
|
|
2699
|
+
if (all.length < EXPECTED_MIN_CALLS || !all.every((t) => t.api.status === status)) return void 0;
|
|
2700
|
+
return {
|
|
2701
|
+
seen,
|
|
2702
|
+
draft: {
|
|
2703
|
+
why: `Every call to ${callLine(head.call)} in these trails answered ${status} (${all.length} of ${all.length}).`,
|
|
2704
|
+
fix: `If ${status} is this endpoint's normal answer, mark it as expected so it stops counting.`,
|
|
2705
|
+
evidence: all.slice(-5).map((t) => `${clock(ms(t.ts))} → ${t.api.status}`)
|
|
2706
|
+
}
|
|
2707
|
+
};
|
|
2708
|
+
}
|
|
2709
|
+
/**
|
|
2710
|
+
* What the trails of an issue's recent occurrences show, best first. Reads the newest
|
|
2711
|
+
* DETECTOR_LOOKBACK error events; each finding keeps the newest occurrence's wording and evidence
|
|
2712
|
+
* and counts how many of them show it. When the repeat-failed finding already says "send it
|
|
2713
|
+
* once", the duplicate-call finding isn't listed again.
|
|
2714
|
+
*/
|
|
2715
|
+
function findingsOf(events, hosts = []) {
|
|
2716
|
+
const reads = [...events].sort((a, b) => ms(b.ts) - ms(a.ts)).slice(0, 20).map((ev) => read(ev, hosts));
|
|
2717
|
+
const found = /* @__PURE__ */ new Map();
|
|
2718
|
+
for (const r of reads) for (const [id, detect] of Object.entries(DETECTORS)) {
|
|
2719
|
+
const draft = detect(r);
|
|
2720
|
+
if (!draft) continue;
|
|
2721
|
+
const had = found.get(id);
|
|
2722
|
+
if (had) had.seen++;
|
|
2723
|
+
else found.set(id, {
|
|
2724
|
+
id,
|
|
2725
|
+
...draft,
|
|
2726
|
+
seen: 1,
|
|
2727
|
+
of: reads.length
|
|
2728
|
+
});
|
|
2729
|
+
}
|
|
2730
|
+
if (found.get("repeat_failed")?.fix.startsWith("App: send it once")) found.delete("duplicate_call");
|
|
2731
|
+
const expected = expectedAnswer(reads);
|
|
2732
|
+
if (expected) found.set("expected_answer", {
|
|
2733
|
+
id: "expected_answer",
|
|
2734
|
+
...expected.draft,
|
|
2735
|
+
seen: expected.seen,
|
|
2736
|
+
of: reads.length
|
|
2737
|
+
});
|
|
2738
|
+
return FINDING_IDS.flatMap((id) => found.has(id) ? [found.get(id)] : []);
|
|
2739
|
+
}
|
|
2740
|
+
function offlineBursts(events, hosts = []) {
|
|
2741
|
+
const errors = events.filter((e) => e.event.type === "error");
|
|
2742
|
+
const byPhone = /* @__PURE__ */ new Map();
|
|
2743
|
+
for (const e of errors) {
|
|
2744
|
+
const phone = e.user?.anonymousId ?? e.sessionId;
|
|
2745
|
+
byPhone.set(phone, [...byPhone.get(phone) ?? [], e]);
|
|
2746
|
+
}
|
|
2747
|
+
const bursts = [];
|
|
2748
|
+
for (const [phone, list] of byPhone) {
|
|
2749
|
+
const sorted = [...list].sort((a, b) => Date.parse(a.event.ts) - Date.parse(b.event.ts));
|
|
2750
|
+
let cur = [];
|
|
2751
|
+
const close = () => {
|
|
2752
|
+
if (!cur.length) return;
|
|
2753
|
+
const first = cur[0];
|
|
2754
|
+
const last = cur[cur.length - 1];
|
|
2755
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2756
|
+
for (const e of cur) {
|
|
2757
|
+
const r = e.event.type === "error" ? e.event.request : void 0;
|
|
2758
|
+
const call = r ? `${r.method.toUpperCase()} ${nameUrl(r.url, hosts)}` : "unknown call";
|
|
2759
|
+
counts.set(call, (counts.get(call) ?? 0) + 1);
|
|
2760
|
+
}
|
|
2761
|
+
bursts.push({
|
|
2762
|
+
start: first.event.ts,
|
|
2763
|
+
end: last.event.ts,
|
|
2764
|
+
phone,
|
|
2765
|
+
os: `${last.context.os.name} ${last.context.os.version ?? ""}`.trim(),
|
|
2766
|
+
device: last.context.device.model,
|
|
2767
|
+
appVersion: last.context.app.version,
|
|
2768
|
+
calls: [...counts].map(([call, count]) => ({
|
|
2769
|
+
call,
|
|
2770
|
+
count
|
|
2771
|
+
})).sort((a, b) => b.count - a.count),
|
|
2772
|
+
failures: cur.length,
|
|
2773
|
+
airplaneMode: cur.some((e) => e.event.tags?.[AIRPLANE_TAG] === "true")
|
|
2774
|
+
});
|
|
2775
|
+
cur = [];
|
|
2776
|
+
};
|
|
2777
|
+
for (const e of sorted) {
|
|
2778
|
+
if (cur.length && Date.parse(e.event.ts) - Date.parse(cur[cur.length - 1].event.ts) > 6e4) close();
|
|
2779
|
+
cur.push(e);
|
|
2780
|
+
}
|
|
2781
|
+
close();
|
|
2782
|
+
}
|
|
2783
|
+
return bursts.sort((a, b) => b.start.localeCompare(a.start));
|
|
2784
|
+
}
|
|
2785
|
+
/** How many bursts on how many phones, for Health's Folded away line. */
|
|
2786
|
+
function burstSummary(bursts) {
|
|
2787
|
+
return {
|
|
2788
|
+
bursts: bursts.length,
|
|
2789
|
+
phones: new Set(bursts.map((b) => b.phone)).size,
|
|
2790
|
+
lastSeen: bursts[0]?.end
|
|
2791
|
+
};
|
|
2792
|
+
}
|
|
2793
|
+
/** The offline issue's occurrences in a range: [start, end]. */
|
|
2794
|
+
var inRange = (events, start, end) => events.filter((e) => e.event.ts >= start && e.event.ts <= end);
|
|
2795
|
+
// src/server/backend-said.ts
|
|
2796
|
+
/** A code or id from a JSON field: text or a number, kept to plain characters, else nothing. */
|
|
2797
|
+
var token = (v) => {
|
|
2798
|
+
const s = typeof v === "number" ? String(v) : typeof v === "string" ? v.trim() : "";
|
|
2799
|
+
return /^[\w.:-]{1,100}$/.test(s) ? s : void 0;
|
|
2800
|
+
};
|
|
2801
|
+
/**
|
|
2802
|
+
* What a gateway or framework says for any failure: not the service's own reason, so it doesn't
|
|
2803
|
+
* replace the explanation's diagnosis.
|
|
2804
|
+
*/
|
|
2805
|
+
var GENERIC = /^(internal server error|not found|bad request|forbidden|unauthori[sz]ed|missing authentication token|service unavailable|bad gateway|gateway time-?out|endpoint request timed out|too many requests|error|an error occurred)\.?$/i;
|
|
2806
|
+
/**
|
|
2807
|
+
* A message from the backend, safe to quote in a ticket or Slack: text only, phone numbers and
|
|
2808
|
+
* email addresses masked, no quotes or Markdown characters, at most 200 characters.
|
|
2809
|
+
*/
|
|
2810
|
+
var message = (v) => {
|
|
2811
|
+
if (typeof v !== "string") return void 0;
|
|
2812
|
+
const s = maskPersonal(v).replace(/[`*_<>[\]]/g, "").replace(/"/g, "'").replace(/\s+/g, " ").trim().slice(0, 200);
|
|
2813
|
+
return s && !GENERIC.test(s) ? s : void 0;
|
|
2814
|
+
};
|
|
2815
|
+
/** One occurrence's answer from the backend, or undefined when it gave none of its own. */
|
|
2816
|
+
function saidBy(se, hosts = []) {
|
|
2817
|
+
if (se.event.type !== "error") return void 0;
|
|
2818
|
+
const call = failedRequest(se.event.request, timelineOf(se.event.breadcrumbs, hosts), hosts);
|
|
2819
|
+
let body = {};
|
|
2820
|
+
try {
|
|
2821
|
+
const parsed = se.event.response?.body ? JSON.parse(se.event.response.body) : void 0;
|
|
2822
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) body = parsed;
|
|
2823
|
+
} catch {}
|
|
2824
|
+
const code = token(call?.backendCode) ?? token(body.code) ?? token(body.errorCode);
|
|
2825
|
+
const said = message(call?.backendTitle) ?? message(body.title) ?? message(body.message) ?? message(body.error);
|
|
2826
|
+
if (!code && !said) return void 0;
|
|
2827
|
+
return {
|
|
2828
|
+
status: se.event.response?.status ?? call?.status ?? se.event.request?.status,
|
|
2829
|
+
code,
|
|
2830
|
+
message: said,
|
|
2831
|
+
traceId: token(call?.traceId) ?? token(body.traceId) ?? token(body.trace_id),
|
|
2832
|
+
service: call?.service
|
|
2833
|
+
};
|
|
2834
|
+
}
|
|
2835
|
+
/**
|
|
2836
|
+
* What the backend said across an issue's occurrences (newest first): the newest answer that had
|
|
2837
|
+
* its own code or message, and every code with how often it came back. undefined when it never
|
|
2838
|
+
* gave one.
|
|
2839
|
+
*/
|
|
2840
|
+
function backendSaid(events, hosts = []) {
|
|
2841
|
+
const said = events.flatMap((e) => saidBy(e, hosts) ?? []);
|
|
2842
|
+
if (!said.length) return void 0;
|
|
2843
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2844
|
+
for (const s of said) if (s.code) counts.set(s.code, (counts.get(s.code) ?? 0) + 1);
|
|
2845
|
+
return {
|
|
2846
|
+
...said[0],
|
|
2847
|
+
codes: [...counts].map(([key, count]) => ({
|
|
2848
|
+
key,
|
|
2849
|
+
count
|
|
2850
|
+
})).sort((a, b) => b.count - a.count)
|
|
2851
|
+
};
|
|
2852
|
+
}
|
|
1621
2853
|
// src/server/range.ts
|
|
1622
2854
|
/** The date range a dashboard view covers, worked out from the URL (range=7d, or from/to). */
|
|
1623
2855
|
var RANGE_PRESETS = [
|
|
@@ -1675,6 +2907,17 @@ function resolveRange(input, now = Date.now()) {
|
|
|
1675
2907
|
granularity: preset === "24h" ? "hour" : days > 90 ? "week" : "day"
|
|
1676
2908
|
};
|
|
1677
2909
|
}
|
|
2910
|
+
/** The range as ISO instants: the last 24 hours for "24h", else whole UTC days from `from` to `to`. */
|
|
2911
|
+
function rangeBounds(range, now = Date.now()) {
|
|
2912
|
+
if (range.preset === "24h") return {
|
|
2913
|
+
start: (/* @__PURE__ */ new Date(now - 864e5)).toISOString(),
|
|
2914
|
+
end: new Date(now).toISOString()
|
|
2915
|
+
};
|
|
2916
|
+
return {
|
|
2917
|
+
start: `${range.from}T00:00:00.000Z`,
|
|
2918
|
+
end: `${range.to}T23:59:59.999Z`
|
|
2919
|
+
};
|
|
2920
|
+
}
|
|
1678
2921
|
/** Every UTC day from `from` to `to`, inclusive. */
|
|
1679
2922
|
function daysBetween(from, to) {
|
|
1680
2923
|
const out = [];
|
|
@@ -1713,11 +2956,15 @@ var SUMMARY_FIELDS = {
|
|
|
1713
2956
|
"totals.errors24h": "Error events in the trailing 24 hours (exact, from hourly rollups).",
|
|
1714
2957
|
"totals.errorsRange": "Error events in the selected range.",
|
|
1715
2958
|
"totals.changePct": "Percent change of errors vs the previous period of the same length. null when that period had fewer than 10 errors.",
|
|
1716
|
-
"totals.activeIssues": "Distinct grouped errors (issues) seen in the range.",
|
|
2959
|
+
"totals.activeIssues": "Distinct grouped errors (issues) seen in the range, without test errors.",
|
|
2960
|
+
"totals.testIssues": "Issues made on purpose to test deBug (tag test=true). Left out of every count; list them with ?tests=1.",
|
|
2961
|
+
connectivity: "Phones going offline (requests that never left the phone), as bursts per phone, shown on Health as Folded away. Not counted against any endpoint or owner. null when there were none.",
|
|
1717
2962
|
"totals.criticalIssues": "Issues whose 24h occurrence count meets the critical threshold.",
|
|
1718
2963
|
"flows[]": "Parts of the app as tagged by the developer (e.g. auth.login). Sorted by severity, then errors24h, then errorsRange.",
|
|
1719
2964
|
"flows[].failureRate": "failures/(failures+successes) in the range from events logged with an outcome. null when the app logs no outcomes for that flow.",
|
|
1720
|
-
"issues[]": "Grouped errors seen in the range. Sorted by severity, then count24h, then countRange. Narrow with ?status=open|resolved|ignored|all (default all), ?flow=, ?category=, ?owner= and ?impact= (ids from breakdowns); an unknown value is a 400. _filter echoes what was applied.",
|
|
2965
|
+
"issues[]": "Grouped errors seen in the range. Sorted by severity, then count24h, then countRange. Narrow with ?status=open|resolved|ignored|all (default all), ?flow=, ?category=, ?owner= and ?impact= (ids from breakdowns); an unknown value is a 400. Test errors are left out unless ?tests=1. _filter echoes what was applied.",
|
|
2966
|
+
"issues[].offline": "true for the one issue that collects offline bursts (calls that never left the phone). On Health's Folded away line, left out of this list unless ?category=network.offline, and never counted as active.",
|
|
2967
|
+
"issues[].test": "true for an issue made on purpose to test deBug (tag test=true): listed only with ?tests=1, never counted, always severity low.",
|
|
1721
2968
|
"issues[].severity": "Derived from count24h using thresholds critical/high/medium.",
|
|
1722
2969
|
"issues[].title": "Error name and message, with hosts named in Settings shown by their names.",
|
|
1723
2970
|
"issues[].culprit": "Top in-app stack frame: function (file:line).",
|
|
@@ -1814,14 +3061,15 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
|
|
|
1814
3061
|
let errors24h = 0;
|
|
1815
3062
|
let errorsPrev24h = 0;
|
|
1816
3063
|
const issue24h = /* @__PURE__ */ new Map();
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
3064
|
+
const testIssues = new Set(issues.filter(isTestIssue).map((i) => i.fingerprint));
|
|
3065
|
+
const realHours = hourRows.filter((r) => !testIssues.has(r.key));
|
|
3066
|
+
for (const r of hourRows) if (last24.has(r.day)) issue24h.set(r.key, (issue24h.get(r.key) ?? 0) + r.errors);
|
|
3067
|
+
for (const r of realHours) if (last24.has(r.day)) errors24h += r.errors;
|
|
3068
|
+
else errorsPrev24h += r.errors;
|
|
1821
3069
|
let trend;
|
|
1822
3070
|
if (hourly) trend = h24.map((h) => ({
|
|
1823
3071
|
day: h,
|
|
1824
|
-
errors:
|
|
3072
|
+
errors: realHours.filter((r) => r.day === h).reduce((a, r) => a + r.errors, 0),
|
|
1825
3073
|
events: 0
|
|
1826
3074
|
}));
|
|
1827
3075
|
else {
|
|
@@ -1851,22 +3099,37 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
|
|
|
1851
3099
|
const errorsPrev = hourly ? errorsPrev24h : sumRows(prevTotal);
|
|
1852
3100
|
const issueRange = /* @__PURE__ */ new Map();
|
|
1853
3101
|
for (const r of issueDays) issueRange.set(r.key, (issueRange.get(r.key) ?? 0) + r.errors);
|
|
1854
|
-
const
|
|
1855
|
-
const versionsInRange = [...new Set(sessionRows.map((r) => r.key.split("|")[0]).filter((v) => v && v !== "unknown"))].sort(compareVersions);
|
|
3102
|
+
const { start: rangeStart, end: rangeEnd } = rangeBounds(range, now);
|
|
3103
|
+
const versionsInRange = [...new Set(sessionRows.map((r) => r.key.split("|")[0]).filter((v) => v && v !== "unknown"))].sort(compareVersions$1);
|
|
1856
3104
|
const latestVersion = versionsInRange[versionsInRange.length - 1] ?? null;
|
|
1857
|
-
const issueRows = issues.filter((i) => i.lastSeen >= rangeStart && i.firstSeen <=
|
|
1858
|
-
const active = issueRows.filter((i) => i.status !== "ignored" && i.status !== "resolved");
|
|
3105
|
+
const issueRows = issues.filter((i) => i.lastSeen >= rangeStart && i.firstSeen <= rangeEnd).map((i) => toIssueRow(i, issue24h.get(i.fingerprint) ?? 0, hourly ? issue24h.get(i.fingerprint) ?? 0 : issueRange.get(i.fingerprint) ?? 0, thresholds, latestVersion, criticalFlows, hosts)).sort(bySeverityThenCount);
|
|
3106
|
+
const active = issueRows.filter((i) => !i.test && !i.offline && i.status !== "ignored" && i.status !== "resolved");
|
|
1859
3107
|
await Promise.all(active.slice(0, 10).map(async (i) => i.users = await countUsersSafe(store, app, `issue:${i.fingerprint}`, usersSince)));
|
|
1860
3108
|
const lastEvents = new Map(issues.map((i) => [i.fingerprint, i.lastEvent]));
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
3109
|
+
const offlineRow = issueRows.find((i) => i.offline);
|
|
3110
|
+
let connectivity = null;
|
|
3111
|
+
if (offlineRow) {
|
|
3112
|
+
const events = await store.listIssueEvents(app, offlineRow.fingerprint, { limit: 200 });
|
|
3113
|
+
connectivity = {
|
|
3114
|
+
fingerprint: offlineRow.fingerprint,
|
|
3115
|
+
...burstSummary(offlineBursts(inRange(events, rangeStart, rangeEnd), hosts)),
|
|
3116
|
+
failures: offlineRow.countRange,
|
|
3117
|
+
capped: events.length >= 200
|
|
3118
|
+
};
|
|
3119
|
+
}
|
|
3120
|
+
for (const row of issueRows) if (row.labels) {
|
|
3121
|
+
const last = lastEvents.get(row.fingerprint);
|
|
3122
|
+
row.explanation = explain({
|
|
3123
|
+
issue: row,
|
|
3124
|
+
labels: row.labels,
|
|
3125
|
+
lastEvent: last,
|
|
3126
|
+
users: row.users,
|
|
3127
|
+
hosts,
|
|
3128
|
+
rangeLabel: range.label,
|
|
3129
|
+
backend: last ? backendSaid([last], hosts) : void 0,
|
|
3130
|
+
now
|
|
3131
|
+
});
|
|
3132
|
+
}
|
|
1870
3133
|
const incidents = await buildIncidents(incidentRows, incidentIssueRows, issueRows, hourly, (key) => countUsersSafe(store, app, `incident:${key}`, usersSince));
|
|
1871
3134
|
const firstDay = /* @__PURE__ */ new Map();
|
|
1872
3135
|
for (const r of releaseRows) {
|
|
@@ -1940,12 +3203,14 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
|
|
|
1940
3203
|
eventsRange: sumRows(total, "events"),
|
|
1941
3204
|
activeIssues: active.length,
|
|
1942
3205
|
criticalIssues: active.filter((i) => i.severity === "critical").length,
|
|
3206
|
+
testIssues: issueRows.filter((i) => i.test).length,
|
|
1943
3207
|
failingFlows: flows.filter((f) => (hourly ? f.errors24h : f.errorsRange) > 0).length
|
|
1944
3208
|
},
|
|
1945
3209
|
narrative: [],
|
|
1946
3210
|
trend,
|
|
1947
3211
|
flows,
|
|
1948
3212
|
issues: issueRows,
|
|
3213
|
+
connectivity,
|
|
1949
3214
|
incidents,
|
|
1950
3215
|
breakdowns: {
|
|
1951
3216
|
os: top(os),
|
|
@@ -2069,7 +3334,7 @@ function buildHealth(sessionRows, endpointRows, today, hosts = []) {
|
|
|
2069
3334
|
...v,
|
|
2070
3335
|
crashFreeRate: rate(v.crashed, v.sessions),
|
|
2071
3336
|
errorFreeRate: rate(v.errored, v.sessions)
|
|
2072
|
-
})).filter((v) => v.sessions > 0).sort((a, b) => compareVersions(b.version, a.version)).slice(0, 8);
|
|
3337
|
+
})).filter((v) => v.sessions > 0).sort((a, b) => compareVersions$1(b.version, a.version)).slice(0, 8);
|
|
2073
3338
|
const sum = (k) => [...byVersion.values()].reduce((a, v) => a + v[k], 0);
|
|
2074
3339
|
const eps = /* @__PURE__ */ new Map();
|
|
2075
3340
|
for (const r of endpointRows) {
|
|
@@ -2179,7 +3444,9 @@ function issueLabels(i, latestVersion, criticalFlows = [], hosts = []) {
|
|
|
2179
3444
|
});
|
|
2180
3445
|
}
|
|
2181
3446
|
function toIssueRow(i, count24h, countRange, t, latestVersion = null, criticalFlows = [], hosts = []) {
|
|
2182
|
-
const
|
|
3447
|
+
const test = isTestIssue(i);
|
|
3448
|
+
const offline = isOfflineIssue(i);
|
|
3449
|
+
const base = test || offline ? "low" : severityFromCount(count24h, t);
|
|
2183
3450
|
const critical = new Set(criticalFlows);
|
|
2184
3451
|
return {
|
|
2185
3452
|
labels: issueLabels(i, latestVersion, criticalFlows, hosts),
|
|
@@ -2194,12 +3461,14 @@ function toIssueRow(i, count24h, countRange, t, latestVersion = null, criticalFl
|
|
|
2194
3461
|
title: nameUrlsIn(i.title, hosts),
|
|
2195
3462
|
culprit: i.culprit,
|
|
2196
3463
|
flow: i.flow,
|
|
2197
|
-
severity: i.flow && critical.has(i.flow) && count24h > 0 ? BUMP[base] : base,
|
|
3464
|
+
severity: !test && i.flow && critical.has(i.flow) && count24h > 0 ? BUMP[base] : base,
|
|
2198
3465
|
count24h,
|
|
2199
3466
|
countRange,
|
|
2200
3467
|
countTotal: i.count,
|
|
2201
3468
|
firstSeen: i.firstSeen,
|
|
2202
|
-
lastSeen: i.lastSeen
|
|
3469
|
+
lastSeen: i.lastSeen,
|
|
3470
|
+
test,
|
|
3471
|
+
offline
|
|
2203
3472
|
};
|
|
2204
3473
|
}
|
|
2205
3474
|
var bySeverityThenCount = (a, b) => SEVERITY_ORDER[b.severity] - SEVERITY_ORDER[a.severity] || b.count24h - a.count24h || b.countRange - a.countRange || b.lastSeen.localeCompare(a.lastSeen);
|
|
@@ -2222,7 +3491,7 @@ function narrate(s) {
|
|
|
2222
3491
|
const n = r.preset === "24h" ? f.errors24h : f.errorsRange;
|
|
2223
3492
|
out.push(`${f.flow}${f.critical ? " (a critical flow)" : ""} is the most affected flow (${f.severity}): ${plural(n, "error")} in ${r.label} from ${plural(f.issues, "issue")}.${failRate}`);
|
|
2224
3493
|
}
|
|
2225
|
-
const i = s.issues.find((x) => x.status !== "ignored" && x.status !== "resolved");
|
|
3494
|
+
const i = s.issues.find((x) => !x.test && !x.offline && x.status !== "ignored" && x.status !== "resolved");
|
|
2226
3495
|
if (i && i.countRange > 0) {
|
|
2227
3496
|
const isNew = Date.parse(s.generatedAt) - Date.parse(i.firstSeen) < 3 * DAY;
|
|
2228
3497
|
out.push(`Top issue: "${i.title}"${i.culprit ? ` in ${i.culprit}` : ""}, ${plural(i.countRange, "time")} in ${r.label}${isNew ? `, first seen ${relDays(s.generatedAt, i.firstSeen)}` : ""}.`);
|
|
@@ -2248,6 +3517,41 @@ function relDays(now, then) {
|
|
|
2248
3517
|
const d = Math.floor((Date.parse(now) - Date.parse(then)) / DAY);
|
|
2249
3518
|
return d === 0 ? "today" : d === 1 ? "yesterday" : `${d} days ago`;
|
|
2250
3519
|
}
|
|
3520
|
+
/**
|
|
3521
|
+
* The failed call to show on an issue's page, from occurrences newest first. `latest` is the
|
|
3522
|
+
* newest (what's happening now: during an outage, a timeout with no answer); when it captured
|
|
3523
|
+
* nothing, the newest that did. `lastAnswer` is the newest the server answered, when that's not
|
|
3524
|
+
* `latest`: what the server said last time it said anything.
|
|
3525
|
+
*/
|
|
3526
|
+
function callsOf(events, hosts) {
|
|
3527
|
+
const calls = events.flatMap((se) => {
|
|
3528
|
+
if (se.event.type !== "error") return [];
|
|
3529
|
+
const api = failedRequest(se.event.request, timelineOf(se.event.breadcrumbs, hosts), hosts);
|
|
3530
|
+
return api ? [{
|
|
3531
|
+
api,
|
|
3532
|
+
exchange: exchangeOf(se.event),
|
|
3533
|
+
eventId: se.event.id,
|
|
3534
|
+
ts: se.event.ts
|
|
3535
|
+
}] : [];
|
|
3536
|
+
});
|
|
3537
|
+
const latest = calls[0]?.exchange ? calls[0] : calls.find((c) => c.exchange) ?? calls[0];
|
|
3538
|
+
const answered = calls.find((c) => c.exchange?.response);
|
|
3539
|
+
return {
|
|
3540
|
+
latest,
|
|
3541
|
+
lastAnswer: answered && answered !== latest && !latest?.exchange?.response ? answered : void 0
|
|
3542
|
+
};
|
|
3543
|
+
}
|
|
3544
|
+
/** The service whose call an issue's newest occurrence failed, or the app's own code. */
|
|
3545
|
+
function areaOf(se, labels, hosts) {
|
|
3546
|
+
if (!labels || !isRequestCategory(labels.category)) return { kind: "app" };
|
|
3547
|
+
const e = se?.event.type === "error" ? se.event : void 0;
|
|
3548
|
+
if (!e) return null;
|
|
3549
|
+
const name = failedRequest(e.request, timelineOf(e.breadcrumbs, hosts), hosts)?.service ?? (e.request ? hostNameFor(e.request.url, hosts)?.host.name : void 0);
|
|
3550
|
+
return name ? {
|
|
3551
|
+
kind: "service",
|
|
3552
|
+
name
|
|
3553
|
+
} : null;
|
|
3554
|
+
}
|
|
2251
3555
|
async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.now(), opts = {}) {
|
|
2252
3556
|
const issue = await store.getIssue(app, fingerprint);
|
|
2253
3557
|
if (!issue) return null;
|
|
@@ -2297,6 +3601,7 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
|
|
|
2297
3601
|
if (row.labels) row.labels = withIssueCauses(row.labels, { deviceModels: events.map((e) => e.context.device.model) });
|
|
2298
3602
|
const hints = hintsFor(issue, breakdowns, events, !!row.labels && isRequestCategory(row.labels.category), hosts);
|
|
2299
3603
|
const errorEvents = events.filter((e) => e.event.type === "error");
|
|
3604
|
+
const backend = row.labels && isRequestCategory(row.labels.category) ? backendSaid(errorEvents.length ? errorEvents : issue.lastEvent ? [issue.lastEvent] : [], hosts) : void 0;
|
|
2300
3605
|
row.explanation = row.labels ? explain({
|
|
2301
3606
|
issue: row,
|
|
2302
3607
|
labels: row.labels,
|
|
@@ -2306,10 +3611,17 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
|
|
|
2306
3611
|
platforms: [...new Set(errorEvents.map((e) => e.context.os.name))],
|
|
2307
3612
|
durations: errorEvents.flatMap((e) => e.event.type === "error" && e.event.request?.durationMs !== void 0 ? [e.event.request.durationMs] : []),
|
|
2308
3613
|
rangeLabel: range.label,
|
|
3614
|
+
backend,
|
|
2309
3615
|
now
|
|
2310
3616
|
}) : void 0;
|
|
3617
|
+
const offline = row.offline;
|
|
3618
|
+
const findings = offline ? [] : findingsOf(errorEvents.flatMap((e) => e.event.type === "error" ? [e.event] : []), hosts);
|
|
2311
3619
|
const link = issueUrl(opts.trackers?.dashboardUrl, app, fingerprint);
|
|
3620
|
+
const bounds = rangeBounds(range, now);
|
|
3621
|
+
const inPeriod = errorEvents.filter((e) => e.event.ts >= bounds.start && e.event.ts <= bounds.end);
|
|
3622
|
+
const calls = row.labels && isRequestCategory(row.labels.category) ? callsOf(inPeriod.length ? inPeriod : errorEvents.length ? errorEvents : [issue.lastEvent], hosts) : void 0;
|
|
2312
3623
|
const detail = {
|
|
3624
|
+
area: row.test ? { kind: "tests" } : row.offline ? null : areaOf(issue.lastEvent, row.labels, hosts),
|
|
2313
3625
|
app,
|
|
2314
3626
|
range,
|
|
2315
3627
|
retention,
|
|
@@ -2320,6 +3632,8 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
|
|
|
2320
3632
|
affectedUsers,
|
|
2321
3633
|
links: {},
|
|
2322
3634
|
sample: events[0] ?? issue.lastEvent ?? null,
|
|
3635
|
+
calls,
|
|
3636
|
+
backend,
|
|
2323
3637
|
recent: events.slice(0, 25).map((e) => ({
|
|
2324
3638
|
id: e.event.id,
|
|
2325
3639
|
type: e.event.type,
|
|
@@ -2331,7 +3645,12 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
|
|
|
2331
3645
|
userId: e.user?.id
|
|
2332
3646
|
})),
|
|
2333
3647
|
hints,
|
|
2334
|
-
|
|
3648
|
+
findings,
|
|
3649
|
+
...offline ? {
|
|
3650
|
+
bursts: offlineBursts(inRange(events, bounds.start, bounds.end), hosts),
|
|
3651
|
+
burstsCapped: events.length >= 200
|
|
3652
|
+
} : {},
|
|
3653
|
+
ticketMarkdown: row.explanation ? explanationMarkdown(row.title, row.explanation, link, findings) : void 0
|
|
2335
3654
|
};
|
|
2336
3655
|
detail.links = trackerLinks(detail, opts.trackers ?? {});
|
|
2337
3656
|
return detail;
|
|
@@ -2359,6 +3678,11 @@ function issueStory(d, dashboardUrl) {
|
|
|
2359
3678
|
"Stack (in-app frames):",
|
|
2360
3679
|
...frames.map((f) => `- \`${f.fn ?? "?"}\` ${f.file ?? ""}:${f.line ?? ""}`)
|
|
2361
3680
|
] : [],
|
|
3681
|
+
...d.findings.length ? [
|
|
3682
|
+
"",
|
|
3683
|
+
"What the trail shows:",
|
|
3684
|
+
...d.findings.slice(0, 3).map(findingLine)
|
|
3685
|
+
] : [],
|
|
2362
3686
|
...d.hints.length ? [
|
|
2363
3687
|
"",
|
|
2364
3688
|
x ? "Also worth checking:" : "What to look at:",
|
|
@@ -2435,15 +3759,4 @@ function hintsFor(issue, b, events, explained = false, hosts = []) {
|
|
|
2435
3759
|
});
|
|
2436
3760
|
return hints;
|
|
2437
3761
|
}
|
|
2438
|
-
|
|
2439
|
-
/** Firestore answers FAILED_PRECONDITION while an index deBug needs is missing or still building. */
|
|
2440
|
-
function isIndexError(message) {
|
|
2441
|
-
return /FAILED_PRECONDITION|requires an index|index is currently building/i.test(message);
|
|
2442
|
-
}
|
|
2443
|
-
/** A thrown Firestore error saying a composite index is missing (gRPC code 9), on the Firestore store only. */
|
|
2444
|
-
function isMissingIndex(e, store) {
|
|
2445
|
-
if (store !== "firestore") return false;
|
|
2446
|
-
const err = e;
|
|
2447
|
-
return (err?.code === 9 || err?.code === "failed-precondition") && /index/i.test(String(err?.message ?? ""));
|
|
2448
|
-
}
|
|
2449
|
-
export { seenHosts as A, ingest as C, teamAction as D, hashInviteToken as E, writeRetention as F, writeSettings as I, getLocalWriteStore as L, testAlert as M, validRetention as N, readSettings as O, writeIssueStatus as P, getStore as R, CORS_HEADERS as S, isRequestCategory as T, hashPassword as _, buildSummary as a, startSession as b, resolveRange as c, canManage as d, checkCredentials as f, forgetMember as g, envOwnerEmails as h, buildIssueDetail as i, settingsWriter as j, restampStep as k, DEMO_USER as l, endSession as m, isMissingIndex as n, endpointTimings as o, demoLoginEnabled as p, SUMMARY_FIELDS as r, retentionFor as s, isIndexError as t, authorizeApi as u, readSession as v, parseIngestKeys as w, tooManyAttempts as x, recordFailure as y, config as z };
|
|
3762
|
+
export { testAlert as $, parseIngestKeys as A, endpointsOf as B, hashPassword as C, tooManyAttempts as D, startSession as E, LIVE_MS as F, serviceTile as G, isOpen as H, OUTAGE as I, appNames as J, servicesOf as K, answerOf as L, hashInviteToken as M, teamAction as N, CORS_HEADERS as O, HEALTH_ORDER as P, settingsWriter as Q, byHealth as R, forgetMember as S, recordFailure as T, pct$1 as U, failing as V, plural$2 as W, restampStep as X, readSettings as Y, seenHosts as Z, canManage as _, retentionFor as a, getLocalWriteStore as at, endSession as b, buildEventDetail as c, buildSessionDetail as d, validRetention as et, sessionOutcome as f, authorizeApi as g, DEMO_USER as h, endpointTimings as i, clearStoreCache as it, isRequestCategory as j, ingest as k, failedRequest as l, isMissingIndex as m, buildIssueDetail as n, writeRetention as nt, resolveRange as o, getStore as ot, isIndexError as p, whoseLine as q, buildSummary as r, writeSettings as rt, apiCallOf as s, SUMMARY_FIELDS as t, writeIssueStatus as tt, timelineOf as u, checkCredentials as v, readSession as w, envOwnerEmails as x, demoLoginEnabled as y, endpointLevel as z };
|