@debugkit/dashboard 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/assets/_authed-3se5Egqg.js +1 -0
- package/dist/client/assets/_authed-BfFWbTOO.js +1 -0
- package/dist/client/assets/app-code-BKWbvUMR.js +1 -0
- package/dist/client/assets/closed-issues-Cl-8l4oR.js +1 -0
- package/dist/client/assets/{copy-button-BJkvWYRY.js → copy-button-BwRd5q7l.js} +1 -1
- package/dist/client/assets/customers-Cf-R8BOg.js +1 -0
- package/dist/client/assets/empty-state-njgr1sWu.js +7 -0
- package/dist/client/assets/endpoints-BMqAUNyE.js +1 -0
- package/dist/client/assets/events._eventId-CVRZJ-bU.js +2 -0
- package/dist/client/assets/exchange-view-DVwSLcXF.js +2 -0
- package/dist/client/assets/index-DEnCDILF.js +10 -0
- package/dist/client/assets/{invite._token-DiDJMnQu.js → invite._token-CKAfd5iB.js} +1 -1
- package/dist/client/assets/issues._fingerprint-Cm72LNM_.js +55 -0
- package/dist/client/assets/login-BTxOhEbN.js +1 -0
- package/dist/client/assets/request-panel-DmtFRF8Z.js +1 -0
- package/dist/client/assets/services._service-CU1a45f3.js +1 -0
- package/dist/client/assets/{sessions._sessionId-B0NxM46y.js → sessions._sessionId-LRo3QNQP.js} +1 -1
- package/dist/client/assets/settings-Ctzvl2LL.js +1 -0
- package/dist/client/assets/src-BiKxgjA_.js +3 -0
- package/dist/client/assets/styles-Tcq0pvWj.css +2 -0
- package/dist/client/assets/{team-BRuVQ2bU.js → team-xlrjhEGp.js} +1 -1
- package/dist/client/assets/{timeline-BIGo9MCA.js → timeline-WXE49N62.js} +1 -1
- package/dist/client/assets/ui-DRjjeSWy.js +1 -0
- package/dist/client/assets/{version-banner-VFvyqbPh.js → version-banner-BYe4PVzv.js} +1 -1
- package/dist/server/assets/_authed-Bn-IktVS.js +120 -0
- package/dist/server/assets/_authed-RXh7GpoE.js +252 -0
- package/dist/server/assets/_tanstack-start-manifest_v-C4dQ3MxD.js +122 -0
- package/dist/server/assets/app-code-BQL_FcSx.js +212 -0
- package/dist/server/assets/app-name-BcbY1BZN.js +11 -0
- package/dist/server/assets/closed-issues-o9U80FXv.js +45 -0
- package/dist/server/assets/config-Bp8m0B8q.js +57 -0
- package/dist/server/assets/{customers-827fC5I3.js → customers-CD84Bz4P.js} +11 -6
- package/dist/server/assets/{demo-DDNiXCgj.js → demo-D-o8m75S.js} +222 -6
- package/dist/server/assets/{dynamodb-Cd4blXkh.js → dynamodb-CcH0cp7R.js} +2 -2
- package/dist/server/assets/empty-state-wz34oAi8.js +163 -0
- package/dist/server/assets/endpoints-8N389Poe.js +248 -0
- package/dist/server/assets/{endpoints-_T0NSOyA.js → endpoints-C0Cxc_-x.js} +38 -36
- package/dist/server/assets/{events._eventId-Dcsu4oVO.js → events._eventId-BOHkNO5W.js} +90 -351
- package/dist/server/assets/exchange-view-DzoSBt_R.js +112 -0
- package/dist/server/assets/{firestore-c8Sz2Q4O.js → firestore-DyvYMiz3.js} +2 -2
- package/dist/server/assets/{fns-BiUOnY1_.js → fns-BcVGbSID.js} +20 -7
- package/dist/server/assets/fns-DE03YExD.js +831 -0
- package/dist/server/assets/{invite._token-7DjEeeaD.js → invite._token-DhlACkIe.js} +3 -3
- package/dist/server/assets/{issues._fingerprint-vc9r3jgg.js → issues._fingerprint-BBd-_Nn5.js} +400 -28
- package/dist/server/assets/{login-CwKAx9Bj.js → login-CbfdHJvg.js} +3 -3
- package/dist/server/assets/{memory-Bv4C8uyK.js → memory-CZBUh1Zc.js} +75 -3
- package/dist/server/assets/{errors-Po4Tt5zd.js → queries-wha4cZbw.js} +1394 -128
- package/dist/server/assets/request-panel-DpCSwl2q.js +162 -0
- package/dist/server/assets/{router-Dk3XHn7B.js → router-BXkzR89j.js} +195 -91
- package/dist/server/assets/services._service-DWjFFGON.js +309 -0
- package/dist/server/assets/{sessions._sessionId-BvXxK9qF.js → sessions._sessionId-DDspiPTe.js} +2 -2
- package/dist/server/assets/{settings-B9NLQjl1.js → settings-DCLIaZ_p.js} +146 -25
- package/dist/server/assets/{src-oWPSAvXv.js → src-B2GKPQyN.js} +1 -1
- package/dist/server/assets/{team-Bsm2wFG0.js → team-jd_TmWP4.js} +3 -3
- package/dist/server/assets/ui-C3pl0edn.js +302 -0
- package/dist/server/assets/{version-banner-nf-yZurq.js → version-banner-BX34ijG5.js} +1 -1
- package/dist/server/assets/{versions-BOlL8jam.js → versions-YI_zK1ql.js} +3 -3
- package/dist/server/assets/warm-C3GbXP1_.js +16 -0
- package/dist/server/server.js +37 -29
- package/package.json +4 -4
- package/dist/client/assets/_authed-C29P68T-.js +0 -1
- package/dist/client/assets/_authed-D-dYbHoP.js +0 -7
- package/dist/client/assets/charts--S1o8iId.js +0 -55
- package/dist/client/assets/customers-Cwx5qEO1.js +0 -1
- package/dist/client/assets/endpoints-zz1TotR2.js +0 -1
- package/dist/client/assets/events._eventId-Bx6TfUjv.js +0 -3
- package/dist/client/assets/index-COHr_cc3.js +0 -10
- package/dist/client/assets/issues._fingerprint-Cv522h2j.js +0 -1
- package/dist/client/assets/login-CKkpc0t5.js +0 -1
- package/dist/client/assets/settings-CIlF10MI.js +0 -1
- package/dist/client/assets/src-BKSiJUs3.js +0 -3
- package/dist/client/assets/styles-GAsMUS4-.css +0 -2
- package/dist/client/assets/ui-CaEZB1j2.js +0 -1
- package/dist/server/assets/_authed-BJWDiF9i.js +0 -92
- package/dist/server/assets/_authed-X9u_5JUy.js +0 -902
- package/dist/server/assets/_tanstack-start-manifest_v-C6vaeBdY.js +0 -90
- package/dist/server/assets/charts-DzTVqJ8z.js +0 -238
- package/dist/server/assets/fns-CVbk8Gf9.js +0 -1310
- package/dist/server/assets/ui-SAkSlP7f.js +0 -193
|
@@ -1,66 +1,110 @@
|
|
|
1
1
|
import { a as getCookie, i as deleteCookie, o as getRequestHeader, s as setCookie } from "../server.js";
|
|
2
|
-
import { C as
|
|
3
|
-
import { n as
|
|
2
|
+
import { A as hostNameFor, B as OWNER_INFO, C as maskPersonal, D as severityFromCount, E as releaseKey, F as validHostNames, H as markLibraryFrames, I as CATEGORY_INFO, L as CAUSE_INFO, M as nameUrl, N as nameUrlsIn, O as hostKindLabel, P as normaliseHostMatch, R as IMPACT_INFO, S as isTestIssue, T as percentileMs, V as cleanFnName, _ as TEST_TAG, f as OFFLINE_FINGERPRINT, g as SEVERITY_ORDER, j as hostOf$1, l as Envelope, p as OFFLINE_TITLE, r as compareVersions, u as AIRPLANE_TAG, v as compareVersions$1, w as parseStack, x as isTestEvent, y as isOfflineIssue } from "./memory-CZBUh1Zc.js";
|
|
3
|
+
import { n as validAppName, t as appLabel } from "./app-name-BcbY1BZN.js";
|
|
4
|
+
import { n as TEAM_ROLES, t as MAX_RETENTION_DAYS } from "./src-B2GKPQyN.js";
|
|
5
|
+
import { t as config } from "./config-Bp8m0B8q.js";
|
|
4
6
|
import { createHash, createHmac, randomBytes, scryptSync, timingSafeEqual } from "node:crypto";
|
|
5
|
-
|
|
6
|
-
var
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
7
|
+
/** Entries kept at most; the oldest go first. */
|
|
8
|
+
var MAX_ENTRIES = 2e3;
|
|
9
|
+
/** The reads that are cached. */
|
|
10
|
+
var CACHED_READS = /* @__PURE__ */ new Set([
|
|
11
|
+
"getSettings",
|
|
12
|
+
"getVersions",
|
|
13
|
+
"countUsers",
|
|
14
|
+
"listApps",
|
|
15
|
+
"listIssues",
|
|
16
|
+
"getIssue",
|
|
17
|
+
"getRollups",
|
|
18
|
+
"listIssueEvents",
|
|
19
|
+
"listSessionEvents"
|
|
20
|
+
]);
|
|
21
|
+
/** Calls that change data (demo and memory mode write through the store): they clear the cache. */
|
|
22
|
+
var WRITE = /^(write|upsert|increment|put|set|register|note|mark|delete|remove|restamp|create|accept|revoke|change)/;
|
|
23
|
+
/**
|
|
24
|
+
* A short-lived cache in front of the store, so moving between pages doesn't read the same data
|
|
25
|
+
* from Firestore or DynamoDB again and again.
|
|
26
|
+
*
|
|
27
|
+
* - The same read made twice at once (the health wall's summary and endpoints both read the issue
|
|
28
|
+
* list) is made once.
|
|
29
|
+
* - A read newer than FRESH_MS is served as it is; one up to STALE_MS old is served at once and
|
|
30
|
+
* read again in the background, so the next page has it fresh.
|
|
31
|
+
* - A write through the store (demo and memory mode), or clearStoreCache() after a change made
|
|
32
|
+
* through the ingest function's admin routes, clears it. A background read that started
|
|
33
|
+
* before the clear is dropped.
|
|
34
|
+
* - Only CACHED_READS are cached: team and sign-in reads always go to the store.
|
|
35
|
+
*
|
|
36
|
+
* Per server instance, like the sign-in limits in auth.ts: on hosting with several instances,
|
|
37
|
+
* another one can show data up to STALE_MS old after a change. The Refresh button clears this
|
|
38
|
+
* instance's cache.
|
|
39
|
+
*/
|
|
40
|
+
function cacheStore(store, now = Date.now) {
|
|
41
|
+
const entries = /* @__PURE__ */ new Map();
|
|
42
|
+
let generation = 0;
|
|
43
|
+
const read = (key, call) => {
|
|
44
|
+
const t = now();
|
|
45
|
+
const hit = entries.get(key);
|
|
46
|
+
if (hit) {
|
|
47
|
+
entries.delete(key);
|
|
48
|
+
entries.set(key, hit);
|
|
49
|
+
}
|
|
50
|
+
if (hit && t - hit.at < 3e4) return hit.value;
|
|
51
|
+
if (hit && t - hit.at < 12e4) {
|
|
52
|
+
if (!hit.refreshing) {
|
|
53
|
+
hit.refreshing = true;
|
|
54
|
+
const started = generation;
|
|
55
|
+
const next = call();
|
|
56
|
+
next.then(() => generation === started && entries.set(key, {
|
|
57
|
+
at: now(),
|
|
58
|
+
value: next
|
|
59
|
+
}), () => hit.refreshing = false);
|
|
60
|
+
}
|
|
61
|
+
return hit.value;
|
|
62
|
+
}
|
|
63
|
+
const value = call();
|
|
64
|
+
entries.set(key, {
|
|
65
|
+
at: t,
|
|
66
|
+
value
|
|
67
|
+
});
|
|
68
|
+
value.catch(() => entries.get(key)?.value === value && entries.delete(key));
|
|
69
|
+
while (entries.size > MAX_ENTRIES) entries.delete(entries.keys().next().value);
|
|
70
|
+
return value;
|
|
71
|
+
};
|
|
72
|
+
const clear = () => {
|
|
73
|
+
generation++;
|
|
74
|
+
entries.clear();
|
|
29
75
|
};
|
|
30
76
|
return {
|
|
31
|
-
store,
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
retentionDays: num(env.DEBUG_RETENTION_DAYS, 0) || void 0,
|
|
47
|
-
/** "Create issue" buttons on the issue page. */
|
|
48
|
-
trackers: {
|
|
49
|
-
linear: env.DEBUG_LINEAR === "1" || !!env.DEBUG_LINEAR_TEAM,
|
|
50
|
-
linearTeam: env.DEBUG_LINEAR_TEAM || void 0,
|
|
51
|
-
githubRepo: env.DEBUG_GITHUB_REPO || void 0,
|
|
52
|
-
dashboardUrl: env.DEBUG_DASHBOARD_URL || void 0
|
|
53
|
-
},
|
|
54
|
-
secureCookies: env.NODE_ENV === "production" && env.DEBUG_INSECURE_COOKIES !== "1",
|
|
55
|
-
/** Firebase Hosting only forwards a cookie named __session to the function behind it. */
|
|
56
|
-
cookieName: /^[A-Za-z0-9_]{1,40}$/.test(env.DEBUG_COOKIE_NAME ?? "") ? env.DEBUG_COOKIE_NAME : "debug_session"
|
|
77
|
+
store: new Proxy(store, { get(target, prop, receiver) {
|
|
78
|
+
const v = Reflect.get(target, prop, receiver);
|
|
79
|
+
if (typeof v !== "function" || typeof prop !== "string") return v;
|
|
80
|
+
if (CACHED_READS.has(prop)) return (...args) => read(`${prop}:${JSON.stringify(args)}`, () => v.apply(target, args));
|
|
81
|
+
if (!WRITE.test(prop)) return v.bind(target);
|
|
82
|
+
return async (...args) => {
|
|
83
|
+
clear();
|
|
84
|
+
try {
|
|
85
|
+
return await v.apply(target, args);
|
|
86
|
+
} finally {
|
|
87
|
+
clear();
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
} }),
|
|
91
|
+
clear
|
|
57
92
|
};
|
|
58
93
|
}
|
|
59
94
|
// src/server/store.ts
|
|
60
95
|
var store;
|
|
96
|
+
var cache$1;
|
|
61
97
|
/** The dashboard only ever reads. Point it at read-only credentials. */
|
|
62
98
|
function getStore() {
|
|
63
|
-
return store ??= create()
|
|
99
|
+
return store ??= create().then((s) => {
|
|
100
|
+
cache$1 = cacheStore(s);
|
|
101
|
+
import("./warm-C3GbXP1_.js").then((m) => m.warmUp(cache$1.store)).catch(() => void 0);
|
|
102
|
+
return cache$1.store;
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
/** Forget every cached read: called after the dashboard changes anything (status, settings, team). */
|
|
106
|
+
function clearStoreCache() {
|
|
107
|
+
cache$1?.clear();
|
|
64
108
|
}
|
|
65
109
|
/**
|
|
66
110
|
* Demo mode only: the in-memory store also accepts events, so a local app can send to
|
|
@@ -77,23 +121,25 @@ async function create() {
|
|
|
77
121
|
case "firestore": {
|
|
78
122
|
const { initializeApp, getApps } = await import("firebase-admin/app");
|
|
79
123
|
const { getFirestore } = await import("firebase-admin/firestore");
|
|
80
|
-
const { FirestoreStore } = await import("./firestore-
|
|
124
|
+
const { FirestoreStore } = await import("./firestore-DyvYMiz3.js");
|
|
81
125
|
if (!getApps().length) initializeApp();
|
|
82
126
|
return new FirestoreStore(getFirestore(), { root: cfg.firestoreRoot });
|
|
83
127
|
}
|
|
84
128
|
case "dynamodb": {
|
|
85
129
|
const { DynamoDBClient } = await import("@aws-sdk/client-dynamodb");
|
|
86
130
|
const { DynamoDBDocumentClient } = await import("@aws-sdk/lib-dynamodb");
|
|
87
|
-
const { DynamoStore } = await import("./dynamodb-
|
|
131
|
+
const { DynamoStore } = await import("./dynamodb-CcH0cp7R.js");
|
|
88
132
|
return new DynamoStore(DynamoDBDocumentClient.from(new DynamoDBClient({})), { table: cfg.dynamoTable });
|
|
89
133
|
}
|
|
90
134
|
case "memory": {
|
|
91
|
-
const { MemoryStore } = await import("./memory-
|
|
135
|
+
const { MemoryStore } = await import("./memory-CZBUh1Zc.js").then((n) => n.n);
|
|
92
136
|
return new MemoryStore();
|
|
93
137
|
}
|
|
94
138
|
default: {
|
|
95
|
-
const { seedDemo } = await import("./demo-
|
|
96
|
-
|
|
139
|
+
const { keepUpstreamDown, seedDemo } = await import("./demo-D-o8m75S.js");
|
|
140
|
+
const demo = await seedDemo();
|
|
141
|
+
keepUpstreamDown(demo, clearStoreCache);
|
|
142
|
+
return demo;
|
|
97
143
|
}
|
|
98
144
|
}
|
|
99
145
|
}
|
|
@@ -117,6 +163,7 @@ async function admin(method, route, body, query) {
|
|
|
117
163
|
});
|
|
118
164
|
const json = await res.json().catch(() => ({}));
|
|
119
165
|
if (!res.ok) throw new Error(json.error ?? `The ingest function answered ${res.status}`);
|
|
166
|
+
if (method === "PUT") clearStoreCache();
|
|
120
167
|
return json;
|
|
121
168
|
}
|
|
122
169
|
async function readSettings(app) {
|
|
@@ -126,6 +173,7 @@ async function readSettings(app) {
|
|
|
126
173
|
const out = await admin("GET", "/settings", void 0, { app });
|
|
127
174
|
return {
|
|
128
175
|
app,
|
|
176
|
+
displayName: out.settings?.displayName ?? "",
|
|
129
177
|
retentionDays: out.retentionDays,
|
|
130
178
|
criticalFlows: out.settings?.criticalFlows ?? [],
|
|
131
179
|
alerts: out.settings?.alerts ?? {},
|
|
@@ -137,6 +185,7 @@ async function readSettings(app) {
|
|
|
137
185
|
} catch (err) {
|
|
138
186
|
return {
|
|
139
187
|
app,
|
|
188
|
+
displayName: "",
|
|
140
189
|
retentionDays: c.retentionDays ?? null,
|
|
141
190
|
criticalFlows: [],
|
|
142
191
|
alerts: {},
|
|
@@ -150,6 +199,7 @@ async function readSettings(app) {
|
|
|
150
199
|
const hook = s?.alerts?.slackWebhookUrl;
|
|
151
200
|
return {
|
|
152
201
|
app,
|
|
202
|
+
displayName: s?.displayName ?? "",
|
|
153
203
|
retentionDays: s?.retentionDays ?? c.retentionDays ?? (writer === "local" ? 30 : null),
|
|
154
204
|
criticalFlows: s?.criticalFlows ?? [],
|
|
155
205
|
alerts: {
|
|
@@ -163,6 +213,12 @@ async function readSettings(app) {
|
|
|
163
213
|
note: writer === "none" ? "Changing settings needs DEBUG_ADMIN_URL and DEBUG_ADMIN_TOKEN (npx debugkit init sets them)." : void 0
|
|
164
214
|
};
|
|
165
215
|
}
|
|
216
|
+
/** Every app's display name, by id: those with none set are left out (show the id). */
|
|
217
|
+
async function appNames(apps) {
|
|
218
|
+
const store = await getStore();
|
|
219
|
+
const names = await Promise.all(apps.map((a) => store.getSettings(a).then((s) => s?.displayName?.trim() ?? "", () => "")));
|
|
220
|
+
return Object.fromEntries(apps.map((a, i) => [a, names[i]]).filter(([, n]) => n));
|
|
221
|
+
}
|
|
166
222
|
/** Hosts the app called in the last 30 days, busiest first: suggestions for naming them. */
|
|
167
223
|
async function seenHosts(app, now = Date.now()) {
|
|
168
224
|
const day = (t) => new Date(t).toISOString().slice(0, 10);
|
|
@@ -191,12 +247,15 @@ async function writeRetention(app, days) {
|
|
|
191
247
|
else throw new Error("Settings can't be changed from this dashboard: DEBUG_ADMIN_URL and DEBUG_ADMIN_TOKEN aren't set.");
|
|
192
248
|
return readSettings(app);
|
|
193
249
|
}
|
|
194
|
-
/** Change critical flows, alert settings and/or host names. Validation happens in the ingest function (or here, locally). */
|
|
250
|
+
/** Change the display name, critical flows, alert settings and/or host names. Validation happens in the ingest function (or here, locally). */
|
|
195
251
|
async function writeSettings(app, patch) {
|
|
196
252
|
const writer = settingsWriter();
|
|
197
253
|
if (writer === "admin") await admin("PUT", "/settings", {
|
|
198
254
|
app,
|
|
199
255
|
...patch
|
|
256
|
+
}).catch((err) => {
|
|
257
|
+
if (patch.displayName !== void 0 && /nothing to change/i.test(err.message)) throw new Error("Your ingest function is older than app names: upgrade it with npx debugkit upgrade, then save again.");
|
|
258
|
+
throw err;
|
|
200
259
|
});
|
|
201
260
|
else if (writer === "local") {
|
|
202
261
|
const store = await getLocalWriteStore();
|
|
@@ -205,11 +264,14 @@ async function writeSettings(app, patch) {
|
|
|
205
264
|
if (patch.alerts && url && !url.includes("…") && !/^https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/_-]{10,200}$/.test(url)) throw new Error("Use a Slack incoming-webhook URL (https://hooks.slack.com/services/…).");
|
|
206
265
|
const hosts = patch.hosts ? validHostNames(patch.hosts) : void 0;
|
|
207
266
|
if (typeof hosts === "string") throw new Error(hosts);
|
|
267
|
+
const name = patch.displayName === void 0 ? void 0 : validAppName(patch.displayName);
|
|
268
|
+
if (typeof name === "string") throw new Error(name);
|
|
208
269
|
const alerts = patch.alerts ? {
|
|
209
270
|
...patch.alerts,
|
|
210
271
|
slackWebhookUrl: url?.includes("…") ? cur?.alerts?.slackWebhookUrl : url || void 0
|
|
211
272
|
} : void 0;
|
|
212
273
|
await store.putSettings(app, {
|
|
274
|
+
...name !== void 0 ? { displayName: name.name } : {},
|
|
213
275
|
...patch.criticalFlows ? { criticalFlows: patch.criticalFlows } : {},
|
|
214
276
|
...alerts ? { alerts } : {},
|
|
215
277
|
...hosts ? { hosts } : {}
|
|
@@ -257,6 +319,233 @@ async function restampStep(app, cursor) {
|
|
|
257
319
|
}
|
|
258
320
|
throw new Error("Settings can't be changed from this dashboard.");
|
|
259
321
|
}
|
|
322
|
+
// src/lib/health.ts
|
|
323
|
+
/** Worst first when sorted descending; healthy comes last. */
|
|
324
|
+
var HEALTH_ORDER = {
|
|
325
|
+
...SEVERITY_ORDER,
|
|
326
|
+
healthy: -1
|
|
327
|
+
};
|
|
328
|
+
/**
|
|
329
|
+
* What sets a service's level. Two ways to be bad: a large share of calls failing (with enough
|
|
330
|
+
* failures that it isn't one unlucky call), or a large number failing even when the share is
|
|
331
|
+
* small, because that's still many customers.
|
|
332
|
+
*/
|
|
333
|
+
var SERVICE_LEVELS = {
|
|
334
|
+
critical: {
|
|
335
|
+
rate: .25,
|
|
336
|
+
failed: 5,
|
|
337
|
+
volume: 500
|
|
338
|
+
},
|
|
339
|
+
high: {
|
|
340
|
+
rate: .1,
|
|
341
|
+
failed: 3,
|
|
342
|
+
volume: 100
|
|
343
|
+
},
|
|
344
|
+
medium: {
|
|
345
|
+
rate: .05,
|
|
346
|
+
failed: 1
|
|
347
|
+
},
|
|
348
|
+
/** With no request summaries from the SDK: stored failures alone. */
|
|
349
|
+
stored: {
|
|
350
|
+
critical: 50,
|
|
351
|
+
high: 20,
|
|
352
|
+
medium: 5
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
/**
|
|
356
|
+
* An outage: in the last 15 minutes, calls to a service got no answer or a 5xx for at least 3
|
|
357
|
+
* different people, and at least 5% of today's calls to it failed. The people make it more than
|
|
358
|
+
* one phone or one bad request; the share keeps a few unlucky calls on a busy service from
|
|
359
|
+
* reading as down. Request summaries are kept per day, so the share is today's (UTC), not the
|
|
360
|
+
* last 15 minutes'. Without summaries for today, the people alone decide.
|
|
361
|
+
*/
|
|
362
|
+
var OUTAGE = {
|
|
363
|
+
windowMs: 9e5,
|
|
364
|
+
users: 3,
|
|
365
|
+
share: .05
|
|
366
|
+
};
|
|
367
|
+
/** Whether recent no-answer and 5xx calls are an outage: enough people, and enough of today's calls. */
|
|
368
|
+
function isOutage(people, today) {
|
|
369
|
+
if (people < OUTAGE.users) return false;
|
|
370
|
+
return !today?.calls || today.failed / today.calls >= OUTAGE.share;
|
|
371
|
+
}
|
|
372
|
+
/** Failing within this long ago counts as still happening. */
|
|
373
|
+
var LIVE_MS = 36e5;
|
|
374
|
+
/** Still open: not resolved or ignored. */
|
|
375
|
+
var isOpen = (status) => status !== "resolved" && status !== "ignored";
|
|
376
|
+
/** How badly an endpoint fails: the larger of the SDK's failed calls and the stored failures. */
|
|
377
|
+
var failing = (r) => Math.max(r.failedCalls ?? 0, r.failures);
|
|
378
|
+
/** "27%", or "<1%" for a small share that isn't none. */
|
|
379
|
+
var pct$1 = (r) => r > 0 && r < .01 ? "<1%" : `${Math.round(r * 100)}%`;
|
|
380
|
+
/** "1 call", "126 calls"; "2 people" with a plural given. */
|
|
381
|
+
var plural$2 = (n, w, many = `${w}s`) => `${n.toLocaleString()} ${n === 1 ? w : many}`;
|
|
382
|
+
/** How bad a service is, by share and by number of failed calls. See SERVICE_LEVELS. */
|
|
383
|
+
function serviceLevel(s) {
|
|
384
|
+
const L = SERVICE_LEVELS;
|
|
385
|
+
if (s.calls > 0) {
|
|
386
|
+
if (!s.failedCalls && !s.failures) return "healthy";
|
|
387
|
+
const rate = s.failedCalls / s.calls;
|
|
388
|
+
if (s.failedCalls >= L.critical.volume || rate >= L.critical.rate && s.failedCalls >= L.critical.failed) return "critical";
|
|
389
|
+
if (s.failedCalls >= L.high.volume || rate >= L.high.rate && s.failedCalls >= L.high.failed) return "high";
|
|
390
|
+
if (rate >= L.medium.rate && s.failedCalls >= L.medium.failed) return "medium";
|
|
391
|
+
return "low";
|
|
392
|
+
}
|
|
393
|
+
const n = s.failures;
|
|
394
|
+
return !n ? "healthy" : n >= L.stored.critical ? "critical" : n >= L.stored.high ? "high" : n >= L.stored.medium ? "medium" : "low";
|
|
395
|
+
}
|
|
396
|
+
/** One endpoint's level: as a service's, and critical when it's down for many people right now (see OUTAGE). */
|
|
397
|
+
function endpointLevel(e) {
|
|
398
|
+
if (e.down && isOutage(e.down.who.length, e.today)) return "critical";
|
|
399
|
+
return serviceLevel({
|
|
400
|
+
calls: e.calls ?? 0,
|
|
401
|
+
failedCalls: e.failedCalls ?? 0,
|
|
402
|
+
failures: e.failures
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* How to group service names: case doesn't matter, and a plural joins its singular when both are
|
|
407
|
+
* there ("Balances" with "balance"). "status" stays "status" when there's no "statu".
|
|
408
|
+
*/
|
|
409
|
+
function serviceKeys(names) {
|
|
410
|
+
const lower = (n) => (n ?? "").trim().toLowerCase();
|
|
411
|
+
const known = new Set(names.map(lower));
|
|
412
|
+
return (name) => {
|
|
413
|
+
const l = lower(name);
|
|
414
|
+
return l.endsWith("s") && known.has(l.slice(0, -1)) ? l.slice(0, -1) : l;
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
/** Endpoints added up per service, most failures first. */
|
|
418
|
+
function servicesOf(endpoints) {
|
|
419
|
+
const keyOf = serviceKeys(endpoints.map((e) => e.service));
|
|
420
|
+
const by = /* @__PURE__ */ new Map();
|
|
421
|
+
for (const e of endpoints) by.set(keyOf(e.service), [...by.get(keyOf(e.service)) ?? [], e]);
|
|
422
|
+
return [...by.entries()].map(([key, list]) => {
|
|
423
|
+
const calls = list.reduce((n, e) => n + (e.calls ?? 0), 0);
|
|
424
|
+
const failedCalls = list.reduce((n, e) => n + (e.failedCalls ?? 0), 0);
|
|
425
|
+
const p95s = list.map((e) => e.p95).filter((v) => v !== null);
|
|
426
|
+
const busiest = [...list].sort((a, b) => (b.calls ?? 0) - (a.calls ?? 0));
|
|
427
|
+
const kinded = busiest.find((e) => e.kind);
|
|
428
|
+
return {
|
|
429
|
+
key: key || "-",
|
|
430
|
+
service: busiest[0].service,
|
|
431
|
+
...kinded ? {
|
|
432
|
+
kind: kinded.kind,
|
|
433
|
+
...kinded.provider ? { provider: kinded.provider } : {}
|
|
434
|
+
} : {},
|
|
435
|
+
down: downOfService(list),
|
|
436
|
+
endpoints: list.length,
|
|
437
|
+
calls,
|
|
438
|
+
failedCalls,
|
|
439
|
+
failureRate: calls ? failedCalls / calls : null,
|
|
440
|
+
worstP95: p95s.length ? Math.max(...p95s) : null,
|
|
441
|
+
failures: list.reduce((n, e) => n + e.failures, 0),
|
|
442
|
+
lastFailure: list.map((e) => e.lastFailure).filter((t) => !!t).sort().pop()
|
|
443
|
+
};
|
|
444
|
+
}).sort((a, b) => failing(b) - failing(a) || b.calls - a.calls);
|
|
445
|
+
}
|
|
446
|
+
/** A service's recent no-answer and 5xx calls, across its endpoints: people counted once. */
|
|
447
|
+
function downOfService(list) {
|
|
448
|
+
const downs = list.map((e) => e.down).filter((d) => !!d);
|
|
449
|
+
if (!downs.length) return void 0;
|
|
450
|
+
const statuses = /* @__PURE__ */ new Map();
|
|
451
|
+
for (const d of downs) for (const s of d.statuses) statuses.set(s.key, (statuses.get(s.key) ?? 0) + s.count);
|
|
452
|
+
const days = list.map((e) => e.today).filter((t) => !!t);
|
|
453
|
+
return {
|
|
454
|
+
...days.length ? { today: {
|
|
455
|
+
calls: days.reduce((n, t) => n + t.calls, 0),
|
|
456
|
+
failed: days.reduce((n, t) => n + t.failed, 0)
|
|
457
|
+
} } : {},
|
|
458
|
+
users: new Set(downs.flatMap((d) => d.who)).size,
|
|
459
|
+
failures: downs.reduce((n, d) => n + d.failures, 0),
|
|
460
|
+
statuses: [...statuses.entries()].map(([key, count]) => ({
|
|
461
|
+
key,
|
|
462
|
+
count
|
|
463
|
+
})).sort((a, b) => b.count - a.count || a.key.localeCompare(b.key)),
|
|
464
|
+
last: downs.map((d) => d.last).sort().pop()
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* "Cell C Balance is down: no answer or a 5xx for 6 people in the last 15 min": the outage line, when a
|
|
469
|
+
* service is failing broadly right now (see OUTAGE). Undefined when it isn't.
|
|
470
|
+
*/
|
|
471
|
+
function outageOf(s) {
|
|
472
|
+
if (!s.down || !isOutage(s.down.users, s.down.today)) return void 0;
|
|
473
|
+
const name = s.service ?? "Unnamed service";
|
|
474
|
+
return `${s.kind === "upstream" && s.provider && !name.toLowerCase().includes(s.provider.toLowerCase()) ? `${s.provider} ${name}` : name} is down: ${s.down.statuses.every((x) => /^\d+$/.test(x.key)) ? "a 5xx" : s.down.statuses.some((x) => /^\d+$/.test(x.key)) ? "no answer or a 5xx" : "no answer"} for ${plural$2(s.down.users, "person", "people")} in the last ${Math.round(OUTAGE.windowMs / 6e4)} min`;
|
|
475
|
+
}
|
|
476
|
+
/** A service's endpoints, by its key (as in the address). */
|
|
477
|
+
function endpointsOf(key, endpoints) {
|
|
478
|
+
const keyOf = serviceKeys(endpoints.map((e) => e.service));
|
|
479
|
+
const want = key === "-" ? "-" : keyOf(key) || "-";
|
|
480
|
+
return endpoints.filter((e) => (keyOf(e.service) || "-") === want);
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* What's wrong with a service, in one line, from its worst endpoint: the status, the backend's own
|
|
484
|
+
* code and message, and which call. "404 · 30007 Balance lookup unsuccessful · POST /ppd/api/balance/getBalances".
|
|
485
|
+
*/
|
|
486
|
+
function headlineOf(endpoints) {
|
|
487
|
+
const worst = [...endpoints].filter((e) => failing(e) > 0).sort((a, b) => failing(b) - failing(a))[0];
|
|
488
|
+
if (!worst) return void 0;
|
|
489
|
+
return `${[worst.statuses[0]?.key, [worst.backendCodes[0]?.key, worst.backendMessages[0]?.key].filter(Boolean).join(" ")].filter(Boolean).join(" · ") || "Failing"} · ${worst.method ? `${worst.method} ` : ""}${worst.path ?? worst.endpoint}`;
|
|
490
|
+
}
|
|
491
|
+
/** A service's box: its level, what's wrong, and how much. */
|
|
492
|
+
function serviceTile(s, endpoints, now) {
|
|
493
|
+
const outage = outageOf(s);
|
|
494
|
+
const level = outage ? "critical" : serviceLevel(s);
|
|
495
|
+
const measure = s.calls > 0 ? s.failedCalls ? `${pct$1(s.failedCalls / s.calls)} of ${plural$2(s.calls, "call")} failed` : `${plural$2(s.calls, "call")}, none failed` : s.failures ? plural$2(s.failures, "failure") : "No calls";
|
|
496
|
+
return {
|
|
497
|
+
key: s.key,
|
|
498
|
+
name: s.service ?? "Unnamed service",
|
|
499
|
+
kind: "service",
|
|
500
|
+
level,
|
|
501
|
+
headline: level === "healthy" ? void 0 : headlineOf(endpointsOf(s.key, endpoints)),
|
|
502
|
+
measure,
|
|
503
|
+
live: level !== "healthy" && !!s.lastFailure && now - Date.parse(s.lastFailure) < 36e5,
|
|
504
|
+
lastFailure: s.lastFailure,
|
|
505
|
+
weight: failing(s),
|
|
506
|
+
...s.kind ? {
|
|
507
|
+
side: s.kind,
|
|
508
|
+
sideLabel: hostKindLabel(s.kind, s.provider),
|
|
509
|
+
...s.provider ? { provider: s.provider } : {}
|
|
510
|
+
} : {},
|
|
511
|
+
...outage ? { outage } : {}
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
/** Worst level first, then down right now, then still happening, then most failures, then by name. */
|
|
515
|
+
var byHealth = (a, b) => HEALTH_ORDER[b.level] - HEALTH_ORDER[a.level] || Number(!!b.outage) - Number(!!a.outage) || Number(b.live) - Number(a.live) || b.weight - a.weight || a.name.localeCompare(b.name);
|
|
516
|
+
/**
|
|
517
|
+
* Who an outage is with, for a client: an upstream's ("It's Cell C's service, not our backend or
|
|
518
|
+
* the app.") or our own backend's. Nothing when nobody said whose service it is.
|
|
519
|
+
*/
|
|
520
|
+
function whoseLine(t) {
|
|
521
|
+
if (t.side === "upstream") return `It's ${t.provider ? `${t.provider}'s` : "a third party's"} service, not our backend or the app.`;
|
|
522
|
+
return t.side === "ours" ? "It's our own backend, not an upstream service or the app." : "";
|
|
523
|
+
}
|
|
524
|
+
/**
|
|
525
|
+
* The answer to "why is it breaking?", for a client: what's down right now and whose service it
|
|
526
|
+
* is, what's failing badly, the worst of it with what the server said, whether it's still
|
|
527
|
+
* happening, and how much is healthy.
|
|
528
|
+
*/
|
|
529
|
+
function answerOf(tiles, rangeLabel) {
|
|
530
|
+
const down = tiles.filter((t) => t.outage);
|
|
531
|
+
const downLines = down.map((t) => [`${t.outage}.`, whoseLine(t)].filter(Boolean).join(" "));
|
|
532
|
+
const healthyLine = `${tiles.filter((t) => t.level === "healthy").length} of ${tiles.length} ${tiles.length === 1 ? "part is" : "parts are"} healthy.`;
|
|
533
|
+
const failingBadly = tiles.filter((t) => t.level !== "healthy" && t.level !== "low");
|
|
534
|
+
if (!failingBadly.length) return `Nothing is failing badly in ${rangeLabel}. ${healthyLine}`;
|
|
535
|
+
const top = failingBadly.find((t) => !t.outage);
|
|
536
|
+
const names = (ts) => ts.length === 1 ? ts[0].name : `${ts.slice(0, -1).map((t) => t.name).join(", ")} and ${ts[ts.length - 1].name}`;
|
|
537
|
+
const critical = failingBadly.filter((t) => t.level === "critical" && !t.outage);
|
|
538
|
+
const others = failingBadly.filter((t) => !t.outage);
|
|
539
|
+
const lead = critical.length ? `${critical.length === 1 ? "1 part is" : `${critical.length} parts are`} ${down.length ? "also " : ""}failing badly in ${rangeLabel}: ${names(critical)}.` : others.length ? `${others.length === 1 ? "1 part has" : `${others.length} parts have`} ${down.length ? "other " : ""}problems in ${rangeLabel}: ${names(others)}.` : "";
|
|
540
|
+
const live = failingBadly.filter((t) => t.live).length;
|
|
541
|
+
return [
|
|
542
|
+
...downLines,
|
|
543
|
+
lead,
|
|
544
|
+
top ? `${down.length ? "Worst otherwise" : "Worst"}: ${top.name}, ${top.measure}${top.headline ? ` (${top.headline})` : ""}.` : "",
|
|
545
|
+
live ? `${live === 1 ? "1 is" : `${live} are`} still happening (failed in the last hour).` : "",
|
|
546
|
+
healthyLine
|
|
547
|
+
].filter(Boolean).join(" ");
|
|
548
|
+
}
|
|
260
549
|
var EMAIL = /^[^\s@/]{1,64}@[^\s@/]{1,190}\.[^\s@/]{2,63}$/;
|
|
261
550
|
var HASH = /^scrypt\.[0-9a-f]{32}\.[0-9a-f]{128}$/;
|
|
262
551
|
var bad = (error, status = 400) => ({
|
|
@@ -442,7 +731,7 @@ async function sendAlerts(app, outcome, issues, settings, post = defaultPost) {
|
|
|
442
731
|
u.flow && `flow \`${u.flow}\`${critical.has(u.flow) ? " (critical)" : ""}`,
|
|
443
732
|
u.version && `version ${u.version}`
|
|
444
733
|
].filter(Boolean).join(" · ");
|
|
445
|
-
const text = `${kind === "new" ? ":new: New issue" : ":rotating_light: Regressed"} in *${app}*: ${link ? `<${link}|${title}>` : title}${where ? `\n${where}` : ""}`;
|
|
734
|
+
const text = `${kind === "new" ? ":new: New issue" : ":rotating_light: Regressed"} in *${appLabel(app, settings?.displayName)}*: ${link ? `<${link}|${title}>` : title}${where ? `\n${where}` : ""}`;
|
|
446
735
|
try {
|
|
447
736
|
await post(a.slackWebhookUrl, { text });
|
|
448
737
|
sent++;
|
|
@@ -578,10 +867,7 @@ function countryOf(ctx) {
|
|
|
578
867
|
if (tz) return tz;
|
|
579
868
|
return /[-_]([A-Z]{2})(?:\b|$)/.exec(ctx.locale ?? "")?.[1] ?? "unknown";
|
|
580
869
|
}
|
|
581
|
-
// ../../packages/ingest/src/classify.ts
|
|
582
|
-
var FOREGROUND = "App came to the foreground";
|
|
583
870
|
var BACKGROUND = "App went to the background";
|
|
584
|
-
var NEVER_SENT_MS = 100;
|
|
585
871
|
/** How long after reopening a failure still counts as "on resume", if nothing has succeeded since. */
|
|
586
872
|
var RESUME_WINDOW_MS = 1e4;
|
|
587
873
|
var CRASH_SOURCES = /* @__PURE__ */ new Set(["global-handler", "uncaughtException"]);
|
|
@@ -639,7 +925,7 @@ function category(e) {
|
|
|
639
925
|
const source = e.tags?.source;
|
|
640
926
|
if (r && r.status === void 0) {
|
|
641
927
|
if (TIMED_OUT.test(e.error.message)) return "network.timeout";
|
|
642
|
-
if (r.durationMs !== void 0 && r.durationMs <
|
|
928
|
+
if (r.durationMs !== void 0 && r.durationMs < 100) return "network.never_sent";
|
|
643
929
|
return "network.failed";
|
|
644
930
|
}
|
|
645
931
|
if (r?.status !== void 0) {
|
|
@@ -654,11 +940,17 @@ function category(e) {
|
|
|
654
940
|
if (!r) return "app_reported";
|
|
655
941
|
return "other";
|
|
656
942
|
}
|
|
943
|
+
/**
|
|
944
|
+
* A request that never left the phone (ABA-132): no status, too fast to have reached a server, not
|
|
945
|
+
* a timeout. Every one counts as the phone being offline, including the first of a burst, which
|
|
946
|
+
* has no failed calls before it to show it was one.
|
|
947
|
+
*/
|
|
948
|
+
var isOfflineFailure = (labels) => labels.category === "network.never_sent";
|
|
657
949
|
/** The last foreground/background breadcrumb before the error. */
|
|
658
950
|
function lastLifecycle(crumbs) {
|
|
659
951
|
for (let i = crumbs.length - 1; i >= 0; i--) {
|
|
660
952
|
const b = crumbs[i];
|
|
661
|
-
if (b.category === "ui" && (b.message ===
|
|
953
|
+
if (b.category === "ui" && (b.message === "App came to the foreground" || b.message === "App went to the background")) return b;
|
|
662
954
|
}
|
|
663
955
|
}
|
|
664
956
|
function gather(e, crumbs, repeatedFailure) {
|
|
@@ -672,7 +964,7 @@ function gather(e, crumbs, repeatedFailure) {
|
|
|
672
964
|
r: crumbRequest(b)
|
|
673
965
|
})).filter((x) => !!x.r);
|
|
674
966
|
const life = lastLifecycle(crumbs);
|
|
675
|
-
const foreground = life?.message ===
|
|
967
|
+
const foreground = life?.message === "App came to the foreground" ? Date.parse(life.ts) : void 0;
|
|
676
968
|
const burst = requests.filter((x) => x.r.failed && within(x.b, 5e3));
|
|
677
969
|
const hosts = new Set(burst.map((x) => hostOf(x.r.url)));
|
|
678
970
|
if (e.request && (burst.length || repeatedFailure)) hosts.add(hostOf(e.request.url));
|
|
@@ -900,22 +1192,24 @@ function fingerprint(e) {
|
|
|
900
1192
|
const f = topInAppFrame(e.error.frames);
|
|
901
1193
|
const r = e.request;
|
|
902
1194
|
const where = r ? `${r.method.toUpperCase()} ${r.url} ${r.status ? `${Math.floor(r.status / 100)}xx` : "failed"}` : f ? `${cleanFnName(f.fn) ?? "?"}@${normaliseFile(f.file)}` : normaliseMessage(e.error.message);
|
|
903
|
-
|
|
1195
|
+
const parts = [
|
|
904
1196
|
e.error.name,
|
|
905
1197
|
where,
|
|
906
|
-
e.flow ?? ""
|
|
907
|
-
|
|
1198
|
+
e.flow ?? "",
|
|
1199
|
+
...isTestEvent(e) ? [TEST_TAG] : []
|
|
1200
|
+
];
|
|
1201
|
+
return createHash("sha256").update(parts.join("|")).digest("hex").slice(0, 24);
|
|
908
1202
|
}
|
|
909
1203
|
function titleOf(e) {
|
|
910
1204
|
return `${e.error.name}: ${e.error.message}`.slice(0, 200);
|
|
911
1205
|
}
|
|
912
1206
|
// ../../packages/ingest/src/version.ts
|
|
913
1207
|
/** This ingest's version: the package's, which every release bumps with the SDK's (one changeset group). */
|
|
914
|
-
var INGEST_VERSION = "0.
|
|
1208
|
+
var INGEST_VERSION = "0.10.0";
|
|
915
1209
|
/** Sent on every ingest response, so an SDK (or the CLI) can tell how old the ingest it talks to is. */
|
|
916
1210
|
var VERSION_HEADER = "x-debugkit-version";
|
|
917
1211
|
// ../../packages/ingest/src/index.ts
|
|
918
|
-
/** "pk_abc:
|
|
1212
|
+
/** "pk_abc:megatron-native,pk_def:other-app" → { pk_abc: "megatron-native", ... } */
|
|
919
1213
|
function parseIngestKeys(raw) {
|
|
920
1214
|
const out = {};
|
|
921
1215
|
for (const pair of (raw ?? "").split(",").map((s) => s.trim()).filter(Boolean)) {
|
|
@@ -1013,6 +1307,8 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1013
1307
|
const settings = env.events.some((e) => e.type === "error") ? await cachedSettings(opts.store, app) : null;
|
|
1014
1308
|
const criticalFlows = settings?.criticalFlows ?? [];
|
|
1015
1309
|
const hosts = settings?.hosts ?? [];
|
|
1310
|
+
const errorsHere = env.events.filter((e) => e.type === "error");
|
|
1311
|
+
const onlyTestErrors = errorsHere.length > 0 && errorsHere.every((e) => isTestEvent(e));
|
|
1016
1312
|
/** This batch's errors, to group into incidents once they're all labelled. */
|
|
1017
1313
|
const batchErrors = [];
|
|
1018
1314
|
for (const raw of env.events) {
|
|
@@ -1026,6 +1322,7 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1026
1322
|
}
|
|
1027
1323
|
const day = dayOf(event.ts);
|
|
1028
1324
|
if (event.type === "session") {
|
|
1325
|
+
if (onlyTestErrors && (event.status === "errored" || event.status === "crashed")) continue;
|
|
1029
1326
|
rollups.push({
|
|
1030
1327
|
app,
|
|
1031
1328
|
day,
|
|
@@ -1074,6 +1371,10 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1074
1371
|
}));
|
|
1075
1372
|
continue;
|
|
1076
1373
|
}
|
|
1374
|
+
if (event.type !== "error" && isTestEvent(event)) {
|
|
1375
|
+
stored.push(storedEvent(day, event));
|
|
1376
|
+
continue;
|
|
1377
|
+
}
|
|
1077
1378
|
const kind = event.type === "error" ? "error" : "event";
|
|
1078
1379
|
const inc = (dimension, key) => rollups.push({
|
|
1079
1380
|
app,
|
|
@@ -1086,6 +1387,7 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1086
1387
|
let fp;
|
|
1087
1388
|
let labels;
|
|
1088
1389
|
let se;
|
|
1390
|
+
let offline = false;
|
|
1089
1391
|
if (event.type === "error") {
|
|
1090
1392
|
fp = fingerprint(event);
|
|
1091
1393
|
labels = classify({
|
|
@@ -1095,9 +1397,19 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1095
1397
|
criticalFlows,
|
|
1096
1398
|
hosts
|
|
1097
1399
|
});
|
|
1400
|
+
const test = isTestEvent(event);
|
|
1401
|
+
offline = !test && isOfflineFailure(labels);
|
|
1402
|
+
if (offline) {
|
|
1403
|
+
labels = {
|
|
1404
|
+
...labels,
|
|
1405
|
+
category: "network.offline",
|
|
1406
|
+
owner: "unknown"
|
|
1407
|
+
};
|
|
1408
|
+
fp = OFFLINE_FINGERPRINT;
|
|
1409
|
+
}
|
|
1098
1410
|
const cur = issues.get(fp);
|
|
1099
1411
|
se = storedEvent(day, event, fp, labels);
|
|
1100
|
-
batchErrors.push({
|
|
1412
|
+
if (!test && !offline) batchErrors.push({
|
|
1101
1413
|
ts: raw.ts,
|
|
1102
1414
|
labels,
|
|
1103
1415
|
se,
|
|
@@ -1106,9 +1418,9 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1106
1418
|
issues.set(fp, {
|
|
1107
1419
|
app,
|
|
1108
1420
|
fingerprint: fp,
|
|
1109
|
-
title: titleOf(event),
|
|
1110
|
-
culprit: culpritOf(event),
|
|
1111
|
-
flow: event.flow,
|
|
1421
|
+
title: offline ? OFFLINE_TITLE : titleOf(event),
|
|
1422
|
+
culprit: offline ? void 0 : culpritOf(event),
|
|
1423
|
+
flow: offline ? void 0 : event.flow,
|
|
1112
1424
|
by: (cur?.by ?? 0) + 1,
|
|
1113
1425
|
ts: cur && cur.ts > event.ts ? cur.ts : event.ts,
|
|
1114
1426
|
firstTs: cur && cur.firstTs < event.ts ? cur.firstTs : event.ts,
|
|
@@ -1117,7 +1429,6 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1117
1429
|
});
|
|
1118
1430
|
scopes.add(`issue:${fp}`);
|
|
1119
1431
|
inc("issue", fp);
|
|
1120
|
-
if (event.request) inc("endpoint", `${event.request.method.toUpperCase()} ${event.request.url}`);
|
|
1121
1432
|
rollups.push({
|
|
1122
1433
|
app,
|
|
1123
1434
|
day: event.ts.slice(0, 13),
|
|
@@ -1126,9 +1437,14 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1126
1437
|
kind,
|
|
1127
1438
|
by: 1
|
|
1128
1439
|
});
|
|
1440
|
+
if (test) {
|
|
1441
|
+
stored.push(se);
|
|
1442
|
+
continue;
|
|
1443
|
+
}
|
|
1444
|
+
if (event.request && !offline) inc("endpoint", `${event.request.method.toUpperCase()} ${event.request.url}`);
|
|
1129
1445
|
}
|
|
1130
1446
|
inc("total", "all");
|
|
1131
|
-
inc("flow", event.flow ?? "untagged");
|
|
1447
|
+
if (!offline) inc("flow", event.flow ?? "untagged");
|
|
1132
1448
|
inc("os", `${context.os.name} ${context.os.version ?? ""}`.trim());
|
|
1133
1449
|
inc("appVersion", context.app.version);
|
|
1134
1450
|
inc("deviceModel", context.device.model);
|
|
@@ -1141,7 +1457,7 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1141
1457
|
inc("country", countryOf(context));
|
|
1142
1458
|
if (labels) {
|
|
1143
1459
|
inc("category", labels.category);
|
|
1144
|
-
inc("owner", labels.owner);
|
|
1460
|
+
if (!offline) inc("owner", labels.owner);
|
|
1145
1461
|
inc("impact", labels.impact);
|
|
1146
1462
|
}
|
|
1147
1463
|
}
|
|
@@ -1190,7 +1506,13 @@ async function ingest(rawBody, ingestKey, opts) {
|
|
|
1190
1506
|
markUsers(opts.store, app, env.user?.id ?? env.user?.anonymousId ?? env.session.id, scopes, dayOf(receivedAt)),
|
|
1191
1507
|
noteVersions(opts.store, app, env.sdk.version, receivedAt)
|
|
1192
1508
|
]);
|
|
1193
|
-
|
|
1509
|
+
const real = (fp) => !isOfflineIssue(issues.get(fp)) && !isTestIssue(issues.get(fp));
|
|
1510
|
+
const alerting = outcome && {
|
|
1511
|
+
...outcome,
|
|
1512
|
+
created: outcome.created.filter(real),
|
|
1513
|
+
regressed: outcome.regressed.filter(real)
|
|
1514
|
+
};
|
|
1515
|
+
if (alerting && (alerting.created.length || alerting.regressed.length)) await sendAlerts(app, alerting, issues, settings);
|
|
1194
1516
|
return {
|
|
1195
1517
|
status: 202,
|
|
1196
1518
|
body: {
|
|
@@ -1494,9 +1816,11 @@ var PASS_TO = {
|
|
|
1494
1816
|
third_party: "whoever manages that third-party service",
|
|
1495
1817
|
unknown: "app team first, to find out whose it is"
|
|
1496
1818
|
};
|
|
1819
|
+
/** Nobody owns the phone's signal, but the app decides what happens without it (ABA-132). */
|
|
1820
|
+
var OFFLINE_PASS_TO = "Pass to: app team, to handle being offline. No endpoint or server is at fault.";
|
|
1497
1821
|
var TEMPLATES = {
|
|
1498
1822
|
"network.offline": {
|
|
1499
|
-
saw: (
|
|
1823
|
+
saw: () => "Requests never left the phone",
|
|
1500
1824
|
why: () => "The phone was offline.",
|
|
1501
1825
|
diagnosis: () => "No connection when the request was made.",
|
|
1502
1826
|
fix: () => "Check connectivity before sending (NetInfo), queue the request until the phone is back online, and show an offline state."
|
|
@@ -1511,7 +1835,7 @@ var TEMPLATES = {
|
|
|
1511
1835
|
saw: (n) => `${cap(n)} took too long and gave up`,
|
|
1512
1836
|
why: () => "The request was sent but no answer came back in time: a slow network or a slow server.",
|
|
1513
1837
|
diagnosis: (_ev, took) => `Timed out${took ? ` after ${took}` : ""} with no response.`,
|
|
1514
|
-
fix: () => "Check this endpoint's p95 under Response times. If
|
|
1838
|
+
fix: () => "Check this endpoint's p95 under Response times. If it's slow on every network, give that service's owner the timings and the times above; in the app, raise the timeout, retry idempotent requests, and show a retry state."
|
|
1515
1839
|
},
|
|
1516
1840
|
"network.failed": {
|
|
1517
1841
|
saw: (n) => `${cap(n)} couldn't connect`,
|
|
@@ -1601,6 +1925,7 @@ function explain(input) {
|
|
|
1601
1925
|
const e = ev0?.type === "error" ? ev0 : void 0;
|
|
1602
1926
|
const ev = e ? evidenceOf(e) : NO_EVIDENCE;
|
|
1603
1927
|
const t = TEMPLATES[labels.category] ?? TEMPLATES.other;
|
|
1928
|
+
const offline = labels.category === "network.offline";
|
|
1604
1929
|
const causes = labels.causes;
|
|
1605
1930
|
const model = input.lastEvent?.context.device.model;
|
|
1606
1931
|
const service = e?.request ? hostNameFor(e.request.url, input.hosts)?.host.name : void 0;
|
|
@@ -1612,16 +1937,19 @@ function explain(input) {
|
|
|
1612
1937
|
const action = userAction(e?.error.frames);
|
|
1613
1938
|
const doing = action ? ` (after ${action.action})` : "";
|
|
1614
1939
|
const causeSentences = causes.flatMap((c) => CAUSE_LINES[c]?.summary?.({ model }) ?? []);
|
|
1940
|
+
const said = !offline && (input.backend?.message || input.backend?.code) ? input.backend : void 0;
|
|
1941
|
+
const saidService = said?.service ?? service;
|
|
1942
|
+
const answer = said ? `${said.message ? `"${said.message}"` : ""}${said.code ? `${said.message ? " " : ""}(code ${said.code})` : ""}` : "";
|
|
1615
1943
|
const summary = [
|
|
1616
|
-
`${t.saw(s.name, s.plural ? "are" : "is")}${who}${where}${when}${doing}.`,
|
|
1617
|
-
t.why(ev, causes),
|
|
1944
|
+
said ? `${saidService ? `The ${saidService} service` : "The server"} answered${said.status ? ` ${said.status}` : ""} ${answer}${who}${where}${when}${doing}.` : `${t.saw(s.name, s.plural ? "are" : "is")}${who}${where}${when}${doing}.`,
|
|
1945
|
+
said ? "That's the service's own reason: start there." : t.why(ev, causes),
|
|
1618
1946
|
...causeSentences,
|
|
1619
1947
|
stillHappening(issue, rangeLabel, now),
|
|
1620
|
-
`Pass to: ${PASS_TO[labels.owner] ?? PASS_TO.unknown}${service && (labels.owner === "backend" || labels.owner === "third_party") ? ` (${service})` : ""}.`
|
|
1948
|
+
offline ? OFFLINE_PASS_TO : said ? `Pass to: whoever runs ${saidService ? `the ${saidService} service` : "that service"}, with ${said.code ? `code ${said.code}` : "its answer"}${said.traceId ? ` and trace ${said.traceId}` : ""}.` : `Pass to: ${PASS_TO[labels.owner] ?? PASS_TO.unknown}${service && (labels.owner === "backend" || labels.owner === "third_party") ? ` (${service})` : ""}.`
|
|
1621
1949
|
].join(" ");
|
|
1622
1950
|
const took = durationText(input.durations?.length ? input.durations : ev.durationMs !== void 0 ? [ev.durationMs] : []);
|
|
1623
|
-
const evidence = [t.diagnosis(ev, took)];
|
|
1624
|
-
if (e?.request) evidence.push(`Request: \`${e.request.method.toUpperCase()} ${e.request.url}\`${service ? ` (${service})` : ""}.`);
|
|
1951
|
+
const evidence = said ? [`HTTP ${said.status ?? ev.status ?? "error"}, and the backend's own answer: ${answer}${said.traceId ? `, trace \`${said.traceId}\`` : ""}.`, ...said.codes.length > 1 || (said.codes[0]?.count ?? 0) > 1 ? [`Backend codes in recent occurrences: ${said.codes.map((c) => `${c.key} × ${c.count}`).join(", ")}.`] : []] : [t.diagnosis(ev, took)];
|
|
1952
|
+
if (e?.request && !offline) evidence.push(`Request: \`${e.request.method.toUpperCase()} ${e.request.url}\`${service ? ` (${service})` : ""}.`);
|
|
1625
1953
|
if (causes.includes("resumed_from_background") && ev.msSinceForeground !== void 0) evidence.push(`The app came to the foreground ${seconds(ev.msSinceForeground)} before; no request has succeeded since.`);
|
|
1626
1954
|
if (ev.failedBefore > 0) evidence.push(`${plural$1(ev.failedBefore, "other request")} failed in the 5 s before${ev.failedHosts.length > 1 ? `, across ${ev.failedHosts.length} hosts` : ""}.`);
|
|
1627
1955
|
else if (ev.repeatedFailure) evidence.push("The same request had already failed just before.");
|
|
@@ -1630,7 +1958,12 @@ function explain(input) {
|
|
|
1630
1958
|
for (const c of causes) if (CAUSE_LINES[c]?.fix) evidence.push(`${CAUSE_INFO[c].name}: ${CAUSE_INFO[c].description}`);
|
|
1631
1959
|
if (issue.incidentTitle) evidence.push(`Part of the incident "${issue.incidentTitle}".`);
|
|
1632
1960
|
const whereParts = [issue.culprit ? `\`${issue.culprit}\`` : void 0, action ? `started by \`${action.fn}\` (${action.action})` : void 0].filter(Boolean);
|
|
1633
|
-
const
|
|
1961
|
+
const lookUp = said && [said.code && `code ${said.code}`, said.traceId && `trace ${said.traceId}`].filter(Boolean).join(" or ");
|
|
1962
|
+
const fixSteps = [
|
|
1963
|
+
said ? `Search the ${saidService ?? "backend"} logs${lookUp ? ` for ${lookUp}` : ""} at the times above: the service's answer (under The call that failed) says why it refused or failed.` : t.fix(ev, causes),
|
|
1964
|
+
...said && labels.category === "auth" ? [t.fix(ev, causes)] : [],
|
|
1965
|
+
...causes.flatMap((c) => CAUSE_LINES[c]?.fix ?? [])
|
|
1966
|
+
];
|
|
1634
1967
|
return {
|
|
1635
1968
|
summary,
|
|
1636
1969
|
technical: {
|
|
@@ -1641,8 +1974,12 @@ function explain(input) {
|
|
|
1641
1974
|
}
|
|
1642
1975
|
};
|
|
1643
1976
|
}
|
|
1644
|
-
/**
|
|
1645
|
-
|
|
1977
|
+
/**
|
|
1978
|
+
* The explanation as Markdown lines, without a title: for tickets and prefilled links. Specific
|
|
1979
|
+
* findings from the trail (ABA-146) come before the template's advice, which is labelled "General
|
|
1980
|
+
* advice" so it isn't mistaken for specific.
|
|
1981
|
+
*/
|
|
1982
|
+
function explanationLines(x, findings = []) {
|
|
1646
1983
|
return [
|
|
1647
1984
|
"### What happened",
|
|
1648
1985
|
x.summary,
|
|
@@ -1652,19 +1989,867 @@ function explanationLines(x) {
|
|
|
1652
1989
|
"",
|
|
1653
1990
|
...x.technical.evidence.map((l) => `- ${l}`),
|
|
1654
1991
|
...x.technical.where ? ["", `**Where:** ${x.technical.where}`] : [],
|
|
1992
|
+
...findings.length ? [
|
|
1993
|
+
"",
|
|
1994
|
+
"### What the trail shows",
|
|
1995
|
+
...findings.map(findingLine)
|
|
1996
|
+
] : [],
|
|
1655
1997
|
"",
|
|
1656
|
-
`**
|
|
1998
|
+
`**General advice:** ${x.technical.fix}`
|
|
1657
1999
|
];
|
|
1658
2000
|
}
|
|
2001
|
+
/** "- Why. Fix. (2 of 2 occurrences)" */
|
|
2002
|
+
var findingLine = (f) => `- ${f.why} ${f.fix}${f.of > 1 ? ` (${f.seen} of ${f.of} occurrences)` : ""}`;
|
|
1659
2003
|
/** The explanation as Markdown, for pasting into a Jira or Linear ticket by hand. */
|
|
1660
|
-
function explanationMarkdown(title, x, link) {
|
|
2004
|
+
function explanationMarkdown(title, x, link, findings = []) {
|
|
1661
2005
|
return [
|
|
1662
2006
|
`## ${title}`,
|
|
1663
2007
|
...link ? ["", `deBug: ${link}`] : [],
|
|
1664
2008
|
"",
|
|
1665
|
-
...explanationLines(x)
|
|
2009
|
+
...explanationLines(x, findings)
|
|
1666
2010
|
].join("\n");
|
|
1667
2011
|
}
|
|
2012
|
+
// src/server/exchange.ts
|
|
2013
|
+
function sideOf(s) {
|
|
2014
|
+
return {
|
|
2015
|
+
headers: s.headers && Object.keys(s.headers).length ? s.headers : void 0,
|
|
2016
|
+
body: s.body,
|
|
2017
|
+
bodySize: s.bodySize,
|
|
2018
|
+
truncated: s.truncated || void 0
|
|
2019
|
+
};
|
|
2020
|
+
}
|
|
2021
|
+
/** Whether the SDK captured anything beyond the call itself: only with network.bodies on. */
|
|
2022
|
+
function captured(r, crumbs) {
|
|
2023
|
+
return r.query !== void 0 || !!r.headers || r.body !== void 0 || r.bodySize !== void 0 || crumbs.some((b) => b.category === "network" && b.data?.hasBodies === true);
|
|
2024
|
+
}
|
|
2025
|
+
/** The request and response the SDK captured on an error, or undefined when it captured none. */
|
|
2026
|
+
function exchangeOf(ev) {
|
|
2027
|
+
const r = ev.request;
|
|
2028
|
+
if (!r || !(captured(r, ev.breadcrumbs ?? []) || ev.response)) return void 0;
|
|
2029
|
+
const fullUrl = r.url && !r.url.startsWith("/") ? `https://${r.url}` : void 0;
|
|
2030
|
+
return {
|
|
2031
|
+
request: {
|
|
2032
|
+
method: r.method.toUpperCase(),
|
|
2033
|
+
url: r.url,
|
|
2034
|
+
fullUrl,
|
|
2035
|
+
query: r.query?.replace(/^\?/, "") || void 0,
|
|
2036
|
+
...sideOf(r)
|
|
2037
|
+
},
|
|
2038
|
+
response: ev.response ? {
|
|
2039
|
+
status: ev.response.status,
|
|
2040
|
+
...sideOf(ev.response)
|
|
2041
|
+
} : void 0
|
|
2042
|
+
};
|
|
2043
|
+
}
|
|
2044
|
+
// src/server/errors.ts
|
|
2045
|
+
/** Firestore answers FAILED_PRECONDITION while an index deBug needs is missing or still building. */
|
|
2046
|
+
function isIndexError(message) {
|
|
2047
|
+
return /FAILED_PRECONDITION|requires an index|index is currently building/i.test(message);
|
|
2048
|
+
}
|
|
2049
|
+
/** A thrown Firestore error saying a composite index is missing (gRPC code 9), on the Firestore store only. */
|
|
2050
|
+
function isMissingIndex(e, store) {
|
|
2051
|
+
if (store !== "firestore") return false;
|
|
2052
|
+
const err = e;
|
|
2053
|
+
return (err?.code === 9 || err?.code === "failed-precondition") && /index/i.test(String(err?.message ?? ""));
|
|
2054
|
+
}
|
|
2055
|
+
/** A fatal error: labelled a crash, or uncaught and critical, which is how the SDK sends a fatal one. */
|
|
2056
|
+
function isCrash(e) {
|
|
2057
|
+
return e.event.type === "error" && (e.labels?.category === "crash" || !e.event.handled && e.event.severity === "critical");
|
|
2058
|
+
}
|
|
2059
|
+
/** crashed, errored (an uncaught error) or clean, as on the session page. */
|
|
2060
|
+
function sessionOutcome(events) {
|
|
2061
|
+
if (events.some(isCrash)) return "crashed";
|
|
2062
|
+
return events.some((e) => e.event.type === "error" && !e.event.handled) ? "errored" : "clean";
|
|
2063
|
+
}
|
|
2064
|
+
/** The prop that says why a flow failed, in the order apps tend to name it. */
|
|
2065
|
+
var REASON_KEYS = [
|
|
2066
|
+
"reason",
|
|
2067
|
+
"error",
|
|
2068
|
+
"errorKind",
|
|
2069
|
+
"code",
|
|
2070
|
+
"message"
|
|
2071
|
+
];
|
|
2072
|
+
var OUTCOME_WORD = {
|
|
2073
|
+
success: "succeeded",
|
|
2074
|
+
failure: "failed",
|
|
2075
|
+
abandoned: "abandoned"
|
|
2076
|
+
};
|
|
2077
|
+
/** "auth.login: failed, wrongOtp". */
|
|
2078
|
+
function describeEvent(name, outcome, props, step) {
|
|
2079
|
+
const reason = REASON_KEYS.map((k) => props?.[k]).find((v) => typeof v === "string" || typeof v === "number");
|
|
2080
|
+
const parts = [outcome ? OUTCOME_WORD[outcome] ?? outcome : void 0, reason !== void 0 ? String(reason) : void 0].filter(Boolean);
|
|
2081
|
+
return `${name}${step ? ` · ${step}` : ""}${parts.length ? `: ${parts.join(", ")}` : ""}`;
|
|
2082
|
+
}
|
|
2083
|
+
/**
|
|
2084
|
+
* Each breadcrumb once: errors in the same session carry overlapping trails. Given the events
|
|
2085
|
+
* oldest first, each breadcrumb's source is the earliest error that carried it, the one it led to.
|
|
2086
|
+
*
|
|
2087
|
+
* When the SDK merges a repeat it moves the breadcrumb's time to the latest repeat, so a later
|
|
2088
|
+
* trail can hold the same breadcrumb with a newer time and a higher count. Nothing else was
|
|
2089
|
+
* recorded between it and the breadcrumb before it, so an identical one in that window is the
|
|
2090
|
+
* same run: keep the copy with the highest count.
|
|
2091
|
+
*/
|
|
2092
|
+
function uniqueCrumbs(events) {
|
|
2093
|
+
const out = [];
|
|
2094
|
+
const exact = /* @__PURE__ */ new Map();
|
|
2095
|
+
const count = (b) => Number(b.data?.repeated ?? 1);
|
|
2096
|
+
for (const e of events) {
|
|
2097
|
+
if (e.event.type !== "error") continue;
|
|
2098
|
+
const source = e.fingerprint ? {
|
|
2099
|
+
eventId: e.event.id,
|
|
2100
|
+
fingerprint: e.fingerprint
|
|
2101
|
+
} : void 0;
|
|
2102
|
+
let after = -Infinity;
|
|
2103
|
+
for (const b of e.event.breadcrumbs ?? []) {
|
|
2104
|
+
const t = Date.parse(b.ts);
|
|
2105
|
+
const same = (c) => c.crumb.category === b.category && c.crumb.message === b.message && c.crumb.level === b.level;
|
|
2106
|
+
const had = exact.get(`${b.ts}|${b.category}|${b.message}`) ?? (count(b) > 1 ? out.find((c) => same(c) && Date.parse(c.crumb.ts) > after && Date.parse(c.crumb.ts) <= t) : void 0);
|
|
2107
|
+
if (!had) {
|
|
2108
|
+
const c = {
|
|
2109
|
+
crumb: b,
|
|
2110
|
+
source
|
|
2111
|
+
};
|
|
2112
|
+
out.push(c);
|
|
2113
|
+
exact.set(`${b.ts}|${b.category}|${b.message}`, c);
|
|
2114
|
+
} else if (count(b) > count(had.crumb)) {
|
|
2115
|
+
had.crumb = b;
|
|
2116
|
+
exact.set(`${b.ts}|${b.category}|${b.message}`, had);
|
|
2117
|
+
}
|
|
2118
|
+
after = t;
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
return out;
|
|
2122
|
+
}
|
|
2123
|
+
/** How far apart a custom event and its own breadcrumb, or an api.failed event and its call, can be. */
|
|
2124
|
+
var SAME_MS = 2e3;
|
|
2125
|
+
/** The SDK writes each custom event as an "event" breadcrumb too: the breadcrumb that is this event, if any. */
|
|
2126
|
+
function ownCrumb(name, ts, crumbs, taken) {
|
|
2127
|
+
const t = Date.parse(ts);
|
|
2128
|
+
return crumbs.find((c) => !taken.has(c) && c.crumb.category === "event" && c.crumb.message === name && Math.abs(Date.parse(c.crumb.ts) - t) <= SAME_MS);
|
|
2129
|
+
}
|
|
2130
|
+
/** The app's api.failed event says the same as the failed call's breadcrumb next to it. */
|
|
2131
|
+
function repeatsCrumb(e, crumbs) {
|
|
2132
|
+
if (e.event.type !== "event" || e.event.name !== "api.failed") return false;
|
|
2133
|
+
const path = e.event.props?.path;
|
|
2134
|
+
const t = Date.parse(e.event.ts);
|
|
2135
|
+
return crumbs.some((c) => c.api?.failed && c.api.path === path && Math.abs(Date.parse(c.ts) - t) <= SAME_MS);
|
|
2136
|
+
}
|
|
2137
|
+
async function buildSessionDetail(store, app, sessionId) {
|
|
2138
|
+
const [events, settings] = await Promise.all([store.listSessionEvents(app, sessionId, { limit: 501 }), store.getSettings(app).catch(() => null)]);
|
|
2139
|
+
if (!events.length) return null;
|
|
2140
|
+
const truncated = events.length > 500;
|
|
2141
|
+
const page = events.slice(-500);
|
|
2142
|
+
const hosts = settings?.hosts ?? [];
|
|
2143
|
+
const carried = withoutSdkTwinsCarried(uniqueCrumbs(page));
|
|
2144
|
+
const taken = /* @__PURE__ */ new Set();
|
|
2145
|
+
const own = /* @__PURE__ */ new Map();
|
|
2146
|
+
for (const e of page) {
|
|
2147
|
+
if (e.event.type !== "event") continue;
|
|
2148
|
+
const c = ownCrumb(e.event.name, e.event.ts, carried, taken);
|
|
2149
|
+
if (c) {
|
|
2150
|
+
taken.add(c);
|
|
2151
|
+
own.set(e, c);
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
const kept = carried.filter((c) => !taken.has(c)).map((c) => ({
|
|
2155
|
+
ts: c.crumb.ts,
|
|
2156
|
+
item: readCrumb(c.crumb, hosts),
|
|
2157
|
+
source: c.source
|
|
2158
|
+
}));
|
|
2159
|
+
const crumbs = kept.map((c) => c.item);
|
|
2160
|
+
const rows = kept.map((c) => ({
|
|
2161
|
+
kind: "crumb",
|
|
2162
|
+
...c
|
|
2163
|
+
}));
|
|
2164
|
+
let errors = 0;
|
|
2165
|
+
let handled = 0;
|
|
2166
|
+
for (const e of page) {
|
|
2167
|
+
const ev = e.event;
|
|
2168
|
+
if (ev.type === "error") {
|
|
2169
|
+
const crash = isCrash(e);
|
|
2170
|
+
errors++;
|
|
2171
|
+
if (ev.handled) handled++;
|
|
2172
|
+
rows.push({
|
|
2173
|
+
kind: "error",
|
|
2174
|
+
ts: ev.ts,
|
|
2175
|
+
eventId: ev.id,
|
|
2176
|
+
fingerprint: e.fingerprint,
|
|
2177
|
+
title: nameUrlsIn(`${ev.error.name}: ${ev.error.message}`, hosts),
|
|
2178
|
+
handled: ev.handled,
|
|
2179
|
+
crash
|
|
2180
|
+
});
|
|
2181
|
+
} else if (ev.type === "event" && !repeatsCrumb(e, crumbs)) rows.push({
|
|
2182
|
+
kind: "event",
|
|
2183
|
+
ts: ev.ts,
|
|
2184
|
+
eventId: ev.id,
|
|
2185
|
+
name: ev.name,
|
|
2186
|
+
outcome: ev.outcome,
|
|
2187
|
+
detail: describeEvent(ev.name, ev.outcome, ev.props, ev.step),
|
|
2188
|
+
failed: ev.outcome === "failure",
|
|
2189
|
+
source: own.get(e)?.source
|
|
2190
|
+
});
|
|
2191
|
+
}
|
|
2192
|
+
const order = {
|
|
2193
|
+
crumb: 0,
|
|
2194
|
+
event: 1,
|
|
2195
|
+
error: 2
|
|
2196
|
+
};
|
|
2197
|
+
const sorted = byTime(rows.sort((a, b) => order[a.kind] - order[b.kind]));
|
|
2198
|
+
const last = page[page.length - 1];
|
|
2199
|
+
return {
|
|
2200
|
+
app,
|
|
2201
|
+
sessionId,
|
|
2202
|
+
user: {
|
|
2203
|
+
id: last.user?.id,
|
|
2204
|
+
anonymousId: last.user?.anonymousId
|
|
2205
|
+
},
|
|
2206
|
+
context: last.context,
|
|
2207
|
+
start: sorted[0]?.ts ?? page[0].event.ts,
|
|
2208
|
+
end: last.event.ts,
|
|
2209
|
+
outcome: sessionOutcome(page),
|
|
2210
|
+
counts: {
|
|
2211
|
+
errors,
|
|
2212
|
+
handled,
|
|
2213
|
+
events: sorted.filter((r) => r.kind === "event").length
|
|
2214
|
+
},
|
|
2215
|
+
rows: sorted,
|
|
2216
|
+
truncated,
|
|
2217
|
+
read: page.length
|
|
2218
|
+
};
|
|
2219
|
+
}
|
|
2220
|
+
/** How far after an error its "what they did next" reaches. */
|
|
2221
|
+
var AFTER_MS = 3e5;
|
|
2222
|
+
/**
|
|
2223
|
+
* What the customer did after an error, from the trails of the session's later errors: the steps
|
|
2224
|
+
* within five minutes of it, at most `limit`. Empty when nothing later was stored or the session
|
|
2225
|
+
* can't be read (Firestore without its index).
|
|
2226
|
+
*/
|
|
2227
|
+
async function stepsAfter(store, app, sessionId, ts, hosts = [], limit = 30) {
|
|
2228
|
+
const events = await store.listSessionEvents(app, sessionId, { limit: 500 }).catch((e) => {
|
|
2229
|
+
if (isIndexError(String(e?.message))) return [];
|
|
2230
|
+
throw e;
|
|
2231
|
+
});
|
|
2232
|
+
const t = Date.parse(ts);
|
|
2233
|
+
return byTime(withoutSdkTwinsCarried(uniqueCrumbs(events)).filter((c) => {
|
|
2234
|
+
const d = Date.parse(c.crumb.ts) - t;
|
|
2235
|
+
return d > 0 && d <= AFTER_MS;
|
|
2236
|
+
}).map((c) => c.crumb)).slice(0, limit).map((b) => readCrumb(b, hosts));
|
|
2237
|
+
}
|
|
2238
|
+
/** withoutSdkTwins over carried breadcrumbs, keeping each one's source. */
|
|
2239
|
+
function withoutSdkTwinsCarried(carried) {
|
|
2240
|
+
const keep = new Set(withoutSdkTwins(carried.map((c) => c.crumb)));
|
|
2241
|
+
return carried.filter((c) => keep.has(c.crumb));
|
|
2242
|
+
}
|
|
2243
|
+
var str = (v) => typeof v === "string" && v ? v : typeof v === "number" ? String(v) : void 0;
|
|
2244
|
+
var num = (v) => typeof v === "number" && Number.isFinite(v) ? v : typeof v === "string" && /^\d+$/.test(v) ? Number(v) : void 0;
|
|
2245
|
+
/** The first key present: the Cell C app's keys and their alternatives ("backend.title" or "backendTitle"). */
|
|
2246
|
+
var pick = (d, ...keys) => keys.map((k) => d[k]).find((v) => v !== void 0 && v !== null && v !== "");
|
|
2247
|
+
/** API keys read into the call; the rest of the data is shown as it is. */
|
|
2248
|
+
var API_KEYS = /* @__PURE__ */ new Set([
|
|
2249
|
+
"service",
|
|
2250
|
+
"method",
|
|
2251
|
+
"path",
|
|
2252
|
+
"url",
|
|
2253
|
+
"status",
|
|
2254
|
+
"durationMs",
|
|
2255
|
+
"outcome",
|
|
2256
|
+
"retry",
|
|
2257
|
+
"backend.title",
|
|
2258
|
+
"backendTitle",
|
|
2259
|
+
"backend.code",
|
|
2260
|
+
"backendCode",
|
|
2261
|
+
"backendErrors",
|
|
2262
|
+
"traceId",
|
|
2263
|
+
"trace_id",
|
|
2264
|
+
"errorKind"
|
|
2265
|
+
]);
|
|
2266
|
+
/** Outcomes that mean the call did what it should. Anything else the app reports (unreachable, sessionExpired) is a failure. */
|
|
2267
|
+
var OK_OUTCOMES = /* @__PURE__ */ new Set([
|
|
2268
|
+
"answered",
|
|
2269
|
+
"cached",
|
|
2270
|
+
"cancelled",
|
|
2271
|
+
"canceled",
|
|
2272
|
+
"aborted",
|
|
2273
|
+
"success",
|
|
2274
|
+
"ok"
|
|
2275
|
+
]);
|
|
2276
|
+
/** The SDK's own network breadcrumb: "POST api.x.com/v1/pay → 503 (120ms)", or "→ failed". */
|
|
2277
|
+
var SDK_NETWORK = /^([A-Z]+) (\S+) → (\d{3})?/;
|
|
2278
|
+
/** A network breadcrumb as an API call, or undefined when it isn't one ("Device went offline"). */
|
|
2279
|
+
function apiCallOf(b, hosts = []) {
|
|
2280
|
+
if (b.category !== "network") return void 0;
|
|
2281
|
+
const d = b.data ?? {};
|
|
2282
|
+
const sdk = SDK_NETWORK.exec(b.message);
|
|
2283
|
+
const url = str(pick(d, "path", "url")) ?? sdk?.[2];
|
|
2284
|
+
const method = str(pick(d, "method"))?.toUpperCase() ?? sdk?.[1];
|
|
2285
|
+
const appService = str(d.service);
|
|
2286
|
+
if (!url && !method && !appService) return void 0;
|
|
2287
|
+
const status = num(d.status) ?? num(sdk?.[3]);
|
|
2288
|
+
const outcome = str(d.outcome);
|
|
2289
|
+
const named = url && !appService ? hostNameFor(url, hosts) : void 0;
|
|
2290
|
+
const call = {
|
|
2291
|
+
service: appService ?? named?.host.name,
|
|
2292
|
+
method,
|
|
2293
|
+
path: named ? named.rest || "/" : url,
|
|
2294
|
+
status,
|
|
2295
|
+
durationMs: num(d.durationMs),
|
|
2296
|
+
outcome,
|
|
2297
|
+
retry: typeof d.retry === "boolean" || typeof d.retry === "number" ? d.retry : str(d.retry),
|
|
2298
|
+
backendTitle: str(pick(d, "backend.title", "backendTitle")),
|
|
2299
|
+
backendCode: str(pick(d, "backend.code", "backendCode")),
|
|
2300
|
+
backendErrors: str(d.backendErrors),
|
|
2301
|
+
errorKind: str(d.errorKind),
|
|
2302
|
+
traceId: str(pick(d, "traceId", "trace_id")),
|
|
2303
|
+
failed: false
|
|
2304
|
+
};
|
|
2305
|
+
call.failed = b.level === "error" || status !== void 0 && status >= 400 || outcome !== void 0 && !OK_OUTCOMES.has(outcome);
|
|
2306
|
+
return call;
|
|
2307
|
+
}
|
|
2308
|
+
/** How close in time the SDK's breadcrumb and the app's own for the same call are. */
|
|
2309
|
+
var TWIN_MS = 2e3;
|
|
2310
|
+
/**
|
|
2311
|
+
* The SDK's own network breadcrumbs that repeat an app breadcrumb for the same call (same method,
|
|
2312
|
+
* the SDK's URL ending in the app's path, within TWIN_MS): the app's says more, so the SDK's is
|
|
2313
|
+
* left out of the timeline.
|
|
2314
|
+
*/
|
|
2315
|
+
function sdkTwins(crumbs) {
|
|
2316
|
+
const app = crumbs.filter((b) => b.category === "network" && str(b.data?.service) && str(b.data?.path));
|
|
2317
|
+
const out = /* @__PURE__ */ new Set();
|
|
2318
|
+
if (!app.length) return out;
|
|
2319
|
+
for (const b of crumbs) {
|
|
2320
|
+
const sdk = b.category === "network" && !b.data?.service ? SDK_NETWORK.exec(b.message) : null;
|
|
2321
|
+
if (!sdk) continue;
|
|
2322
|
+
const url = sdk[2].split(/[?#]/)[0];
|
|
2323
|
+
const t = Date.parse(b.ts);
|
|
2324
|
+
if (app.some((a) => str(a.data?.method)?.toUpperCase() === sdk[1] && url.endsWith(String(a.data.path)) && Math.abs(Date.parse(a.ts) - t) <= TWIN_MS)) out.add(b);
|
|
2325
|
+
}
|
|
2326
|
+
return out;
|
|
2327
|
+
}
|
|
2328
|
+
/** The breadcrumbs in time order (stable for equal times), each read for what it says. */
|
|
2329
|
+
function timelineOf(crumbs = [], hosts = []) {
|
|
2330
|
+
return byTime(withoutSdkTwins(crumbs)).map((b) => readCrumb(b, hosts));
|
|
2331
|
+
}
|
|
2332
|
+
/** Oldest first, stable for equal times. By time, not by string: "09:00:00Z" and "09:00:00.500Z" don't sort as text. */
|
|
2333
|
+
function byTime(items) {
|
|
2334
|
+
return items.map((x, i) => ({
|
|
2335
|
+
x,
|
|
2336
|
+
i
|
|
2337
|
+
})).sort((p, q) => Date.parse(p.x.ts) - Date.parse(q.x.ts) || p.i - q.i).map(({ x }) => x);
|
|
2338
|
+
}
|
|
2339
|
+
/** The breadcrumbs without the SDK's own copy of a call the app also described. */
|
|
2340
|
+
function withoutSdkTwins(crumbs) {
|
|
2341
|
+
const twins = sdkTwins(crumbs);
|
|
2342
|
+
return crumbs.filter((b) => !twins.has(b));
|
|
2343
|
+
}
|
|
2344
|
+
/** One breadcrumb read for what it says: the API call it describes, or its message. */
|
|
2345
|
+
function readCrumb(b, hosts = []) {
|
|
2346
|
+
const api = apiCallOf(b, hosts);
|
|
2347
|
+
const rest = Object.entries(b.data ?? {}).filter(([k]) => k !== "repeated" && !(api && API_KEYS.has(k)));
|
|
2348
|
+
const repeated = num(b.data?.repeated);
|
|
2349
|
+
return {
|
|
2350
|
+
ts: b.ts,
|
|
2351
|
+
category: b.category,
|
|
2352
|
+
message: nameUrlsIn(b.message, hosts),
|
|
2353
|
+
level: b.level,
|
|
2354
|
+
failed: api ? api.failed : b.level === "error",
|
|
2355
|
+
...api ? { api } : {},
|
|
2356
|
+
...rest.length ? { data: Object.fromEntries(rest) } : {},
|
|
2357
|
+
...repeated && repeated > 1 ? { repeated } : {}
|
|
2358
|
+
};
|
|
2359
|
+
}
|
|
2360
|
+
/** In-app frames first, keeping stack order within each group. */
|
|
2361
|
+
function framesInAppFirst(frames = []) {
|
|
2362
|
+
return [...frames.filter((f) => f.inApp), ...frames.filter((f) => !f.inApp)];
|
|
2363
|
+
}
|
|
2364
|
+
async function buildEventDetail(store, app, fingerprint, eventId) {
|
|
2365
|
+
const [issue, events, settings] = await Promise.all([
|
|
2366
|
+
store.getIssue(app, fingerprint),
|
|
2367
|
+
store.listIssueEvents(app, fingerprint, { limit: 200 }),
|
|
2368
|
+
store.getSettings(app).catch(() => null)
|
|
2369
|
+
]);
|
|
2370
|
+
if (!issue) return null;
|
|
2371
|
+
const errors = (events.length ? events : [issue.lastEvent]).filter((e) => e.event.type === "error");
|
|
2372
|
+
const index = errors.findIndex((e) => e.event.id === eventId);
|
|
2373
|
+
const stored = index >= 0 ? errors[index] : issue.lastEvent.event.id === eventId ? issue.lastEvent : void 0;
|
|
2374
|
+
if (!stored || stored.event.type !== "error") return null;
|
|
2375
|
+
const ev = stored.event;
|
|
2376
|
+
const hosts = settings?.hosts ?? [];
|
|
2377
|
+
const timeline = timelineOf(ev.breadcrumbs, hosts);
|
|
2378
|
+
return {
|
|
2379
|
+
app,
|
|
2380
|
+
issue: {
|
|
2381
|
+
fingerprint,
|
|
2382
|
+
title: nameUrlsIn(issue.title, hosts),
|
|
2383
|
+
status: issue.status ?? "open"
|
|
2384
|
+
},
|
|
2385
|
+
id: ev.id,
|
|
2386
|
+
ts: ev.ts,
|
|
2387
|
+
handled: ev.handled,
|
|
2388
|
+
error: {
|
|
2389
|
+
name: ev.error.name,
|
|
2390
|
+
message: nameUrlsIn(ev.error.message, hosts)
|
|
2391
|
+
},
|
|
2392
|
+
flow: ev.flow,
|
|
2393
|
+
step: ev.step,
|
|
2394
|
+
user: {
|
|
2395
|
+
id: stored.user?.id,
|
|
2396
|
+
anonymousId: stored.user?.anonymousId
|
|
2397
|
+
},
|
|
2398
|
+
sessionId: stored.sessionId,
|
|
2399
|
+
context: stored.context,
|
|
2400
|
+
tags: Object.entries(ev.tags ?? {}).map(([key, value]) => ({
|
|
2401
|
+
key,
|
|
2402
|
+
value: String(value)
|
|
2403
|
+
})).sort((a, b) => a.key.localeCompare(b.key)),
|
|
2404
|
+
timeline,
|
|
2405
|
+
request: failedRequest(ev.request, timeline, hosts),
|
|
2406
|
+
exchange: exchangeOf(ev),
|
|
2407
|
+
frames: framesInAppFirst(ev.error.frames),
|
|
2408
|
+
stack: ev.error.stack,
|
|
2409
|
+
after: await stepsAfter(store, app, stored.sessionId, ev.ts, hosts),
|
|
2410
|
+
newer: index > 0 ? errors[index - 1].event.id : void 0,
|
|
2411
|
+
older: index >= 0 && index < errors.length - 1 ? errors[index + 1].event.id : void 0
|
|
2412
|
+
};
|
|
2413
|
+
}
|
|
2414
|
+
/**
|
|
2415
|
+
* The call that failed: the error's own request, filled in from the app's breadcrumb for the same
|
|
2416
|
+
* call (service, backend code, traceId), or the last failing call when the error has no request.
|
|
2417
|
+
* A breadcrumb counts as the same call only when its method and path match.
|
|
2418
|
+
*/
|
|
2419
|
+
function failedRequest(request, timeline, hosts = []) {
|
|
2420
|
+
const lastFailing = [...timeline].reverse().find((t) => t.api?.failed)?.api;
|
|
2421
|
+
if (!request) return lastFailing;
|
|
2422
|
+
const named = hostNameFor(request.url, hosts);
|
|
2423
|
+
const own = {
|
|
2424
|
+
service: named?.host.name,
|
|
2425
|
+
method: request.method.toUpperCase(),
|
|
2426
|
+
path: named ? named.rest || "/" : request.url,
|
|
2427
|
+
status: request.status,
|
|
2428
|
+
durationMs: request.durationMs,
|
|
2429
|
+
failed: true
|
|
2430
|
+
};
|
|
2431
|
+
const bare = request.url.split(/[?#]/)[0];
|
|
2432
|
+
const sameCall = (c) => c.method === own.method && !!c.path && (bare.endsWith(c.path) || c.path === own.path);
|
|
2433
|
+
const match = [...timeline].reverse().find((t) => t.api?.failed && sameCall(t.api))?.api;
|
|
2434
|
+
return match ? {
|
|
2435
|
+
...match,
|
|
2436
|
+
...defined(own),
|
|
2437
|
+
service: match.service ?? own.service,
|
|
2438
|
+
path: named ? own.path : match.path
|
|
2439
|
+
} : own;
|
|
2440
|
+
}
|
|
2441
|
+
/** Drop undefined fields, so spreading doesn't erase a value from the breadcrumb. */
|
|
2442
|
+
function defined(o) {
|
|
2443
|
+
return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== void 0));
|
|
2444
|
+
}
|
|
2445
|
+
// src/server/hops.ts
|
|
2446
|
+
/** Response headers that carry a trace or request id, best first. */
|
|
2447
|
+
var TRACE_HEADERS = [
|
|
2448
|
+
"traceparent",
|
|
2449
|
+
"x-cloud-trace-context",
|
|
2450
|
+
"x-amzn-trace-id",
|
|
2451
|
+
"x-request-id",
|
|
2452
|
+
"x-amzn-requestid",
|
|
2453
|
+
"x-correlation-id",
|
|
2454
|
+
"request-id"
|
|
2455
|
+
];
|
|
2456
|
+
/** Body fields that carry one, as backends commonly name them. */
|
|
2457
|
+
var TRACE_FIELDS = [
|
|
2458
|
+
"traceId",
|
|
2459
|
+
"trace_id",
|
|
2460
|
+
"requestId",
|
|
2461
|
+
"request_id",
|
|
2462
|
+
"correlationId",
|
|
2463
|
+
"correlation_id"
|
|
2464
|
+
];
|
|
2465
|
+
/** W3C traceparent: version, 32-hex trace id (not all zeros), parent id, flags. */
|
|
2466
|
+
var TRACEPARENT = /^[\da-f]{2}-(?!0{32})([\da-f]{32})-[\da-f]{16}-[\da-f]{2}$/i;
|
|
2467
|
+
/** The trace id inside a header value, or undefined when the value isn't one. */
|
|
2468
|
+
function traceIdIn(header, value) {
|
|
2469
|
+
if (header === "traceparent") return TRACEPARENT.exec(value.trim())?.[1];
|
|
2470
|
+
if (header === "x-cloud-trace-context") return value.split("/")[0] || void 0;
|
|
2471
|
+
if (header === "x-amzn-trace-id") return /(?:^|;)\s*Root=([^;]+)/.exec(value)?.[1];
|
|
2472
|
+
return value.trim() || void 0;
|
|
2473
|
+
}
|
|
2474
|
+
/** A trace id or error code in a JSON response body, top level only. */
|
|
2475
|
+
function idsInBody(body) {
|
|
2476
|
+
if (!body || !body.trimStart().startsWith("{")) return {};
|
|
2477
|
+
try {
|
|
2478
|
+
const json = JSON.parse(body);
|
|
2479
|
+
const text = (v) => typeof v === "string" && v && v !== "[redacted]" ? v : typeof v === "number" ? String(v) : void 0;
|
|
2480
|
+
return {
|
|
2481
|
+
traceId: TRACE_FIELDS.map((k) => text(json[k])).find(Boolean),
|
|
2482
|
+
code: text(json.code) ?? text(json.errorCode)
|
|
2483
|
+
};
|
|
2484
|
+
} catch {
|
|
2485
|
+
return {};
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
/** The trace id the backend already sent: a response header, then the body, then the app's breadcrumb. */
|
|
2489
|
+
function traceOf(headers, body, call) {
|
|
2490
|
+
for (const h of TRACE_HEADERS) {
|
|
2491
|
+
const v = headers?.[h];
|
|
2492
|
+
const id = v && v !== "[redacted]" ? traceIdIn(h, v) : void 0;
|
|
2493
|
+
if (id) return {
|
|
2494
|
+
from: h,
|
|
2495
|
+
id
|
|
2496
|
+
};
|
|
2497
|
+
}
|
|
2498
|
+
const inBody = idsInBody(body).traceId;
|
|
2499
|
+
if (inBody) return {
|
|
2500
|
+
from: "traceId",
|
|
2501
|
+
id: inBody
|
|
2502
|
+
};
|
|
2503
|
+
if (call?.traceId) return {
|
|
2504
|
+
from: "traceId",
|
|
2505
|
+
id: call.traceId
|
|
2506
|
+
};
|
|
2507
|
+
}
|
|
2508
|
+
/** "08:31:44 UTC" for the logs, which are rarely in the phone's timezone. */
|
|
2509
|
+
function utcTime(ts) {
|
|
2510
|
+
const t = new Date(ts);
|
|
2511
|
+
return Number.isNaN(t.getTime()) ? ts : `${t.toISOString().slice(11, 19)} UTC`;
|
|
2512
|
+
}
|
|
2513
|
+
var callLine = (c, withService = true) => `${withService && c.service ? `${c.service} · ` : ""}${c.method ?? ""} ${c.path ?? ""}`.replace(/\s+/g, " ").trim();
|
|
2514
|
+
// src/server/detectors.ts
|
|
2515
|
+
/**
|
|
2516
|
+
* Specific fix advice from patterns in the customer's trail (ABA-146): rule-based, no AI. Each
|
|
2517
|
+
* finding says what happened, what to do, and the evidence it read, so it can be checked.
|
|
2518
|
+
*
|
|
2519
|
+
* No backend changes (ABA-137): a fix may tell the app team what to change, or say where to look
|
|
2520
|
+
* in the backend's logs (time, call, trace id). It never tells anyone to change their backend.
|
|
2521
|
+
*
|
|
2522
|
+
* Not detected: a call still running when the app crashed. Breadcrumbs are written when a call
|
|
2523
|
+
* ends, so one that never answered isn't in the trail (needs the SDK to record request starts).
|
|
2524
|
+
*/
|
|
2525
|
+
var FINDING_IDS = [
|
|
2526
|
+
"repeat_failed",
|
|
2527
|
+
"duplicate_call",
|
|
2528
|
+
"called_before_login",
|
|
2529
|
+
"no_refresh_before_401",
|
|
2530
|
+
"hung_in_background",
|
|
2531
|
+
"expected_answer"
|
|
2532
|
+
];
|
|
2533
|
+
/** Two sends of one call this close together are one action sending it twice. */
|
|
2534
|
+
var DUPLICATE_WINDOW_MS = 5e3;
|
|
2535
|
+
/** A success this recent, then a failure of the same call, is "the repeat failed". */
|
|
2536
|
+
var REPEAT_WINDOW_MS = 6e4;
|
|
2537
|
+
/** Longer than this in the background and a session token has usually expired. */
|
|
2538
|
+
var LONG_BACKGROUND_MS = 18e5;
|
|
2539
|
+
/** A timeout this short isn't a request left hanging. */
|
|
2540
|
+
var HUNG_MIN_MS = 5e3;
|
|
2541
|
+
/** At least this many calls, all with one status, before it looks like the endpoint's normal answer. */
|
|
2542
|
+
var EXPECTED_MIN_CALLS = 3;
|
|
2543
|
+
var ms = (ts) => Date.parse(ts);
|
|
2544
|
+
var isOk = (s) => s !== void 0 && s >= 200 && s < 300;
|
|
2545
|
+
/** "08:31:42.729 UTC", like the log search times (hops.ts utcTime). */
|
|
2546
|
+
var clock = (t) => Number.isNaN(t) ? "?" : `${new Date(t).toISOString().slice(11, 23)} UTC`;
|
|
2547
|
+
/** The path of a stored URL, without host, port, query or hash: "host:443/ppd/x?y" → "/ppd/x". */
|
|
2548
|
+
function pathOf(url) {
|
|
2549
|
+
const bare = url.split(/[?#]/)[0].replace(/^[a-z]+:\/\//i, "");
|
|
2550
|
+
const rest = hostOf$1(bare) ? bare.slice(bare.indexOf("/") < 0 ? bare.length : bare.indexOf("/")) : bare;
|
|
2551
|
+
return rest.startsWith("/") ? rest : `/${rest}`;
|
|
2552
|
+
}
|
|
2553
|
+
/**
|
|
2554
|
+
* Whether a breadcrumb's call is the error's endpoint. Same method; same host when the breadcrumb
|
|
2555
|
+
* names one; and the same path, or the app's path relative to its base URL ("/notifications/x"
|
|
2556
|
+
* under "/ppd/api") ending the request's path on whole segments, at least two of them.
|
|
2557
|
+
*/
|
|
2558
|
+
function sameEndpoint(api, req) {
|
|
2559
|
+
if (!api?.path || api.method !== req.method.toUpperCase()) return false;
|
|
2560
|
+
const host = hostOf$1(api.path);
|
|
2561
|
+
if (host && host !== hostOf$1(req.url)) return false;
|
|
2562
|
+
const want = pathOf(req.url);
|
|
2563
|
+
const have = pathOf(api.path);
|
|
2564
|
+
if (have === want) return true;
|
|
2565
|
+
return !host && have.split("/").length >= 3 && want.endsWith(have);
|
|
2566
|
+
}
|
|
2567
|
+
/** When the call was sent: its end minus its duration (from the data, or the SDK's "(120ms)"). */
|
|
2568
|
+
function sentAt(t) {
|
|
2569
|
+
const took = t.api?.durationMs ?? Number(/\((\d+) ?ms\)/.exec(t.message)?.[1] ?? 0);
|
|
2570
|
+
return ms(t.ts) - took;
|
|
2571
|
+
}
|
|
2572
|
+
/** A login that finished: the app's "Login Completed" or "Logged in", or a 2xx from a login call. */
|
|
2573
|
+
var isLogin = (t) => t.category === "event" && /^(log ?in (completed|succeeded|success)|logged in)$/i.test(t.message.trim()) || !!t.api && isOk(t.api.status) && /\/(login|sign-?in)$/i.test(pathOf(t.api.path ?? ""));
|
|
2574
|
+
/** A token refresh that worked. */
|
|
2575
|
+
var isRefresh = (t) => !!t.api && isOk(t.api.status) && /\/(refresh|refresh-token|token\/refresh|oauth\/token)$/i.test(pathOf(t.api.path ?? ""));
|
|
2576
|
+
var isRetry = (c) => c?.retry === true || typeof c?.retry === "number" && c.retry > 0 || typeof c?.retry === "string" && /^[1-9]/.test(c.retry);
|
|
2577
|
+
function read(ev, hosts) {
|
|
2578
|
+
const at = ms(ev.ts);
|
|
2579
|
+
const trail = timelineOf(ev.breadcrumbs, hosts).filter((t) => !(ms(t.ts) > at));
|
|
2580
|
+
const sent = at - (ev.request?.durationMs ?? 0);
|
|
2581
|
+
if (!ev.request) return {
|
|
2582
|
+
ev,
|
|
2583
|
+
trail,
|
|
2584
|
+
earlier: [],
|
|
2585
|
+
sent
|
|
2586
|
+
};
|
|
2587
|
+
const req = ev.request;
|
|
2588
|
+
const same = trail.filter((t) => sameEndpoint(t.api, req));
|
|
2589
|
+
let i = same.length - 1;
|
|
2590
|
+
while (i >= 0 && !same[i].api.failed) i--;
|
|
2591
|
+
return {
|
|
2592
|
+
ev,
|
|
2593
|
+
trail,
|
|
2594
|
+
call: failedRequest(req, trail, hosts),
|
|
2595
|
+
own: same[i],
|
|
2596
|
+
earlier: i < 0 ? same : same.slice(0, i),
|
|
2597
|
+
sent
|
|
2598
|
+
};
|
|
2599
|
+
}
|
|
2600
|
+
var statusOf = (t) => t.api.status ?? t.api.outcome ?? "no answer";
|
|
2601
|
+
/** Whether one action sent the error's call more than once: the send before it is close, and it isn't the app's own retry. */
|
|
2602
|
+
function sentTwice(r) {
|
|
2603
|
+
if (isRetry(r.own?.api)) return [];
|
|
2604
|
+
return r.earlier.filter((t) => r.sent - sentAt(t) <= DUPLICATE_WINDOW_MS && !isRetry(t.api));
|
|
2605
|
+
}
|
|
2606
|
+
var DETECTORS = {
|
|
2607
|
+
repeat_failed: (r) => {
|
|
2608
|
+
const failed = r.ev.request?.status;
|
|
2609
|
+
if (!r.call || failed === void 0 || !(failed >= 500 || failed === 409)) return void 0;
|
|
2610
|
+
const ok = [...r.earlier].reverse().find((t) => isOk(t.api.status) && r.sent - sentAt(t) <= REPEAT_WINDOW_MS);
|
|
2611
|
+
if (!ok) return void 0;
|
|
2612
|
+
const response = exchangeOf(r.ev)?.response;
|
|
2613
|
+
const trace = traceOf(response?.headers, response?.body, r.call);
|
|
2614
|
+
const service = r.call.service ?? (hostOf$1(r.ev.request.url) || "backend");
|
|
2615
|
+
const where = trace ? `trace id ${trace.id} (${trace.from})` : `${callLine(r.call, false)} sent at ${utcTime(new Date(r.sent).toISOString())}`;
|
|
2616
|
+
const overlapped = r.sent < ms(ok.ts);
|
|
2617
|
+
const twice = sentTwice(r).length > 0;
|
|
2618
|
+
return {
|
|
2619
|
+
why: overlapped ? `The app sent it again before the first call answered; the first worked (${ok.api.status}), the repeat failed (${failed}).` : `The first call worked (${ok.api.status}); the repeat ${seconds(r.sent - sentAt(ok))} later failed (${failed}).`,
|
|
2620
|
+
fix: `${twice ? "App: send it once per action. " : ""}To see why the repeat failed, search the ${service} logs for ${where}.`,
|
|
2621
|
+
evidence: [`${clock(sentAt(ok))} sent, ${clock(ms(ok.ts))} → ${ok.api.status}`, `${clock(r.sent)} sent, ${clock(ms(r.ev.ts))} → ${failed}`]
|
|
2622
|
+
};
|
|
2623
|
+
},
|
|
2624
|
+
duplicate_call: (r) => {
|
|
2625
|
+
const close = sentTwice(r);
|
|
2626
|
+
if (!r.call || !close.length) return void 0;
|
|
2627
|
+
return {
|
|
2628
|
+
why: `The app sends ${callLine(r.call)} ${close.length + 1} times in ${seconds(r.sent - sentAt(close[0]))}.`,
|
|
2629
|
+
fix: "App: send it once per action. Look for two screens or effects that both trigger it.",
|
|
2630
|
+
evidence: [...close.map((t) => `${clock(sentAt(t))} sent → ${statusOf(t)}`), `${clock(r.sent)} sent → ${r.ev.request.status ?? "no answer"}`]
|
|
2631
|
+
};
|
|
2632
|
+
},
|
|
2633
|
+
called_before_login: (r) => {
|
|
2634
|
+
if (!r.call) return void 0;
|
|
2635
|
+
const denied = r.earlier.filter((t) => t.api.status === 401 || t.api.status === 403);
|
|
2636
|
+
if (!denied.length) return void 0;
|
|
2637
|
+
const lastDenied = ms(denied[denied.length - 1].ts);
|
|
2638
|
+
const login = r.trail.find((t) => ms(t.ts) > lastDenied && ms(t.ts) <= r.sent && isLogin(t));
|
|
2639
|
+
if (!login) return void 0;
|
|
2640
|
+
const codes = [...new Set(denied.map((t) => t.api.status))].join(" or ");
|
|
2641
|
+
return {
|
|
2642
|
+
why: `${callLine(r.call)} ran ${denied.length} time${denied.length === 1 ? "" : "s"} before login and got ${codes}.`,
|
|
2643
|
+
fix: "App: wait until login has finished before making this call.",
|
|
2644
|
+
evidence: [...denied.map((t) => `${clock(ms(t.ts))} → ${t.api.status}`), `${clock(ms(login.ts))} ${login.message}`]
|
|
2645
|
+
};
|
|
2646
|
+
},
|
|
2647
|
+
no_refresh_before_401: (r) => {
|
|
2648
|
+
if (!r.call || r.ev.request?.status !== 401) return void 0;
|
|
2649
|
+
const lifecycle = r.trail.filter((t) => t.message === "App came to the foreground" || t.message === "App went to the background");
|
|
2650
|
+
const fore = lifecycle[lifecycle.length - 1];
|
|
2651
|
+
const back = lifecycle[lifecycle.length - 2];
|
|
2652
|
+
if (fore?.message !== "App came to the foreground" || back?.message !== "App went to the background" || ms(fore.ts) - ms(back.ts) < LONG_BACKGROUND_MS) return void 0;
|
|
2653
|
+
if (r.trail.some((t) => ms(t.ts) >= ms(fore.ts) && isRefresh(t))) return void 0;
|
|
2654
|
+
return {
|
|
2655
|
+
why: `The app came back after ${Math.round((ms(fore.ts) - ms(back.ts)) / 6e4)} min in the background and called ${callLine(r.call)} without refreshing its token.`,
|
|
2656
|
+
fix: "App: refresh the session token when the app comes back, before other calls.",
|
|
2657
|
+
evidence: [
|
|
2658
|
+
`${clock(ms(back.ts))} ${back.message}`,
|
|
2659
|
+
`${clock(ms(fore.ts))} ${fore.message}`,
|
|
2660
|
+
`${clock(ms(r.ev.ts))} → 401`
|
|
2661
|
+
]
|
|
2662
|
+
};
|
|
2663
|
+
},
|
|
2664
|
+
hung_in_background: (r) => {
|
|
2665
|
+
const req = r.ev.request;
|
|
2666
|
+
if (!r.call || !req || req.status !== void 0 || req.durationMs === void 0 || req.durationMs < HUNG_MIN_MS) return void 0;
|
|
2667
|
+
if (!/timed out/i.test(r.ev.error.message)) return void 0;
|
|
2668
|
+
const bg = r.trail.find((t) => t.message === "App went to the background" && ms(t.ts) >= r.sent && ms(t.ts) <= ms(r.ev.ts));
|
|
2669
|
+
if (!bg) return void 0;
|
|
2670
|
+
return {
|
|
2671
|
+
why: `${callLine(r.call)} was still running when the app went to the background, and gave up after ${seconds(req.durationMs)}.`,
|
|
2672
|
+
fix: "App: send it before the app goes to the background, or retry it when the app comes back.",
|
|
2673
|
+
evidence: [
|
|
2674
|
+
`${clock(r.sent)} sent (from its duration)`,
|
|
2675
|
+
`${clock(ms(bg.ts))} ${bg.message}`,
|
|
2676
|
+
`${clock(ms(r.ev.ts))} → no answer`
|
|
2677
|
+
]
|
|
2678
|
+
};
|
|
2679
|
+
}
|
|
2680
|
+
};
|
|
2681
|
+
/**
|
|
2682
|
+
* A 4xx an endpoint gives on every call can be its normal answer ("no update", "not found"). A
|
|
2683
|
+
* 5xx every time is an outage, and 401, 403 and 429 mean something to act on: never "expected".
|
|
2684
|
+
*/
|
|
2685
|
+
var MAYBE_EXPECTED = (s) => s >= 400 && s < 500 && s !== 401 && s !== 403 && s !== 429;
|
|
2686
|
+
/** Every call to the newest occurrence's endpoint, across these trails, answered the same 4xx. */
|
|
2687
|
+
function expectedAnswer(reads) {
|
|
2688
|
+
const head = reads.find((r) => r.call && r.ev.request?.status !== void 0);
|
|
2689
|
+
const status = head?.ev.request?.status;
|
|
2690
|
+
if (!head || status === void 0 || !MAYBE_EXPECTED(status)) return void 0;
|
|
2691
|
+
const calls = /* @__PURE__ */ new Map();
|
|
2692
|
+
let seen = 0;
|
|
2693
|
+
for (const r of reads) {
|
|
2694
|
+
const mine = r.trail.filter((t) => sameEndpoint(t.api, head.ev.request));
|
|
2695
|
+
if (mine.length) seen++;
|
|
2696
|
+
for (const t of mine) calls.set(`${t.ts}|${t.api.status}`, t);
|
|
2697
|
+
}
|
|
2698
|
+
const all = [...calls.values()];
|
|
2699
|
+
if (all.length < EXPECTED_MIN_CALLS || !all.every((t) => t.api.status === status)) return void 0;
|
|
2700
|
+
return {
|
|
2701
|
+
seen,
|
|
2702
|
+
draft: {
|
|
2703
|
+
why: `Every call to ${callLine(head.call)} in these trails answered ${status} (${all.length} of ${all.length}).`,
|
|
2704
|
+
fix: `If ${status} is this endpoint's normal answer, mark it as expected so it stops counting.`,
|
|
2705
|
+
evidence: all.slice(-5).map((t) => `${clock(ms(t.ts))} → ${t.api.status}`)
|
|
2706
|
+
}
|
|
2707
|
+
};
|
|
2708
|
+
}
|
|
2709
|
+
/**
|
|
2710
|
+
* What the trails of an issue's recent occurrences show, best first. Reads the newest
|
|
2711
|
+
* DETECTOR_LOOKBACK error events; each finding keeps the newest occurrence's wording and evidence
|
|
2712
|
+
* and counts how many of them show it. When the repeat-failed finding already says "send it
|
|
2713
|
+
* once", the duplicate-call finding isn't listed again.
|
|
2714
|
+
*/
|
|
2715
|
+
function findingsOf(events, hosts = []) {
|
|
2716
|
+
const reads = [...events].sort((a, b) => ms(b.ts) - ms(a.ts)).slice(0, 20).map((ev) => read(ev, hosts));
|
|
2717
|
+
const found = /* @__PURE__ */ new Map();
|
|
2718
|
+
for (const r of reads) for (const [id, detect] of Object.entries(DETECTORS)) {
|
|
2719
|
+
const draft = detect(r);
|
|
2720
|
+
if (!draft) continue;
|
|
2721
|
+
const had = found.get(id);
|
|
2722
|
+
if (had) had.seen++;
|
|
2723
|
+
else found.set(id, {
|
|
2724
|
+
id,
|
|
2725
|
+
...draft,
|
|
2726
|
+
seen: 1,
|
|
2727
|
+
of: reads.length
|
|
2728
|
+
});
|
|
2729
|
+
}
|
|
2730
|
+
if (found.get("repeat_failed")?.fix.startsWith("App: send it once")) found.delete("duplicate_call");
|
|
2731
|
+
const expected = expectedAnswer(reads);
|
|
2732
|
+
if (expected) found.set("expected_answer", {
|
|
2733
|
+
id: "expected_answer",
|
|
2734
|
+
...expected.draft,
|
|
2735
|
+
seen: expected.seen,
|
|
2736
|
+
of: reads.length
|
|
2737
|
+
});
|
|
2738
|
+
return FINDING_IDS.flatMap((id) => found.has(id) ? [found.get(id)] : []);
|
|
2739
|
+
}
|
|
2740
|
+
function offlineBursts(events, hosts = []) {
|
|
2741
|
+
const errors = events.filter((e) => e.event.type === "error");
|
|
2742
|
+
const byPhone = /* @__PURE__ */ new Map();
|
|
2743
|
+
for (const e of errors) {
|
|
2744
|
+
const phone = e.user?.anonymousId ?? e.sessionId;
|
|
2745
|
+
byPhone.set(phone, [...byPhone.get(phone) ?? [], e]);
|
|
2746
|
+
}
|
|
2747
|
+
const bursts = [];
|
|
2748
|
+
for (const [phone, list] of byPhone) {
|
|
2749
|
+
const sorted = [...list].sort((a, b) => Date.parse(a.event.ts) - Date.parse(b.event.ts));
|
|
2750
|
+
let cur = [];
|
|
2751
|
+
const close = () => {
|
|
2752
|
+
if (!cur.length) return;
|
|
2753
|
+
const first = cur[0];
|
|
2754
|
+
const last = cur[cur.length - 1];
|
|
2755
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2756
|
+
for (const e of cur) {
|
|
2757
|
+
const r = e.event.type === "error" ? e.event.request : void 0;
|
|
2758
|
+
const call = r ? `${r.method.toUpperCase()} ${nameUrl(r.url, hosts)}` : "unknown call";
|
|
2759
|
+
counts.set(call, (counts.get(call) ?? 0) + 1);
|
|
2760
|
+
}
|
|
2761
|
+
bursts.push({
|
|
2762
|
+
start: first.event.ts,
|
|
2763
|
+
end: last.event.ts,
|
|
2764
|
+
phone,
|
|
2765
|
+
os: `${last.context.os.name} ${last.context.os.version ?? ""}`.trim(),
|
|
2766
|
+
device: last.context.device.model,
|
|
2767
|
+
appVersion: last.context.app.version,
|
|
2768
|
+
calls: [...counts].map(([call, count]) => ({
|
|
2769
|
+
call,
|
|
2770
|
+
count
|
|
2771
|
+
})).sort((a, b) => b.count - a.count),
|
|
2772
|
+
failures: cur.length,
|
|
2773
|
+
airplaneMode: cur.some((e) => e.event.tags?.[AIRPLANE_TAG] === "true")
|
|
2774
|
+
});
|
|
2775
|
+
cur = [];
|
|
2776
|
+
};
|
|
2777
|
+
for (const e of sorted) {
|
|
2778
|
+
if (cur.length && Date.parse(e.event.ts) - Date.parse(cur[cur.length - 1].event.ts) > 6e4) close();
|
|
2779
|
+
cur.push(e);
|
|
2780
|
+
}
|
|
2781
|
+
close();
|
|
2782
|
+
}
|
|
2783
|
+
return bursts.sort((a, b) => b.start.localeCompare(a.start));
|
|
2784
|
+
}
|
|
2785
|
+
/** How many bursts on how many phones, for Health's Folded away line. */
|
|
2786
|
+
function burstSummary(bursts) {
|
|
2787
|
+
return {
|
|
2788
|
+
bursts: bursts.length,
|
|
2789
|
+
phones: new Set(bursts.map((b) => b.phone)).size,
|
|
2790
|
+
lastSeen: bursts[0]?.end
|
|
2791
|
+
};
|
|
2792
|
+
}
|
|
2793
|
+
/** The offline issue's occurrences in a range: [start, end]. */
|
|
2794
|
+
var inRange = (events, start, end) => events.filter((e) => e.event.ts >= start && e.event.ts <= end);
|
|
2795
|
+
// src/server/backend-said.ts
|
|
2796
|
+
/** A code or id from a JSON field: text or a number, kept to plain characters, else nothing. */
|
|
2797
|
+
var token = (v) => {
|
|
2798
|
+
const s = typeof v === "number" ? String(v) : typeof v === "string" ? v.trim() : "";
|
|
2799
|
+
return /^[\w.:-]{1,100}$/.test(s) ? s : void 0;
|
|
2800
|
+
};
|
|
2801
|
+
/**
|
|
2802
|
+
* What a gateway or framework says for any failure: not the service's own reason, so it doesn't
|
|
2803
|
+
* replace the explanation's diagnosis.
|
|
2804
|
+
*/
|
|
2805
|
+
var GENERIC = /^(internal server error|not found|bad request|forbidden|unauthori[sz]ed|missing authentication token|service unavailable|bad gateway|gateway time-?out|endpoint request timed out|too many requests|error|an error occurred)\.?$/i;
|
|
2806
|
+
/**
|
|
2807
|
+
* A message from the backend, safe to quote in a ticket or Slack: text only, phone numbers and
|
|
2808
|
+
* email addresses masked, no quotes or Markdown characters, at most 200 characters.
|
|
2809
|
+
*/
|
|
2810
|
+
var message = (v) => {
|
|
2811
|
+
if (typeof v !== "string") return void 0;
|
|
2812
|
+
const s = maskPersonal(v).replace(/[`*_<>[\]]/g, "").replace(/"/g, "'").replace(/\s+/g, " ").trim().slice(0, 200);
|
|
2813
|
+
return s && !GENERIC.test(s) ? s : void 0;
|
|
2814
|
+
};
|
|
2815
|
+
/** One occurrence's answer from the backend, or undefined when it gave none of its own. */
|
|
2816
|
+
function saidBy(se, hosts = []) {
|
|
2817
|
+
if (se.event.type !== "error") return void 0;
|
|
2818
|
+
const call = failedRequest(se.event.request, timelineOf(se.event.breadcrumbs, hosts), hosts);
|
|
2819
|
+
let body = {};
|
|
2820
|
+
try {
|
|
2821
|
+
const parsed = se.event.response?.body ? JSON.parse(se.event.response.body) : void 0;
|
|
2822
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) body = parsed;
|
|
2823
|
+
} catch {}
|
|
2824
|
+
const code = token(call?.backendCode) ?? token(body.code) ?? token(body.errorCode);
|
|
2825
|
+
const said = message(call?.backendTitle) ?? message(body.title) ?? message(body.message) ?? message(body.error);
|
|
2826
|
+
if (!code && !said) return void 0;
|
|
2827
|
+
return {
|
|
2828
|
+
status: se.event.response?.status ?? call?.status ?? se.event.request?.status,
|
|
2829
|
+
code,
|
|
2830
|
+
message: said,
|
|
2831
|
+
traceId: token(call?.traceId) ?? token(body.traceId) ?? token(body.trace_id),
|
|
2832
|
+
service: call?.service
|
|
2833
|
+
};
|
|
2834
|
+
}
|
|
2835
|
+
/**
|
|
2836
|
+
* What the backend said across an issue's occurrences (newest first): the newest answer that had
|
|
2837
|
+
* its own code or message, and every code with how often it came back. undefined when it never
|
|
2838
|
+
* gave one.
|
|
2839
|
+
*/
|
|
2840
|
+
function backendSaid(events, hosts = []) {
|
|
2841
|
+
const said = events.flatMap((e) => saidBy(e, hosts) ?? []);
|
|
2842
|
+
if (!said.length) return void 0;
|
|
2843
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2844
|
+
for (const s of said) if (s.code) counts.set(s.code, (counts.get(s.code) ?? 0) + 1);
|
|
2845
|
+
return {
|
|
2846
|
+
...said[0],
|
|
2847
|
+
codes: [...counts].map(([key, count]) => ({
|
|
2848
|
+
key,
|
|
2849
|
+
count
|
|
2850
|
+
})).sort((a, b) => b.count - a.count)
|
|
2851
|
+
};
|
|
2852
|
+
}
|
|
1668
2853
|
// src/server/range.ts
|
|
1669
2854
|
/** The date range a dashboard view covers, worked out from the URL (range=7d, or from/to). */
|
|
1670
2855
|
var RANGE_PRESETS = [
|
|
@@ -1722,6 +2907,17 @@ function resolveRange(input, now = Date.now()) {
|
|
|
1722
2907
|
granularity: preset === "24h" ? "hour" : days > 90 ? "week" : "day"
|
|
1723
2908
|
};
|
|
1724
2909
|
}
|
|
2910
|
+
/** The range as ISO instants: the last 24 hours for "24h", else whole UTC days from `from` to `to`. */
|
|
2911
|
+
function rangeBounds(range, now = Date.now()) {
|
|
2912
|
+
if (range.preset === "24h") return {
|
|
2913
|
+
start: (/* @__PURE__ */ new Date(now - 864e5)).toISOString(),
|
|
2914
|
+
end: new Date(now).toISOString()
|
|
2915
|
+
};
|
|
2916
|
+
return {
|
|
2917
|
+
start: `${range.from}T00:00:00.000Z`,
|
|
2918
|
+
end: `${range.to}T23:59:59.999Z`
|
|
2919
|
+
};
|
|
2920
|
+
}
|
|
1725
2921
|
/** Every UTC day from `from` to `to`, inclusive. */
|
|
1726
2922
|
function daysBetween(from, to) {
|
|
1727
2923
|
const out = [];
|
|
@@ -1760,11 +2956,15 @@ var SUMMARY_FIELDS = {
|
|
|
1760
2956
|
"totals.errors24h": "Error events in the trailing 24 hours (exact, from hourly rollups).",
|
|
1761
2957
|
"totals.errorsRange": "Error events in the selected range.",
|
|
1762
2958
|
"totals.changePct": "Percent change of errors vs the previous period of the same length. null when that period had fewer than 10 errors.",
|
|
1763
|
-
"totals.activeIssues": "Distinct grouped errors (issues) seen in the range.",
|
|
2959
|
+
"totals.activeIssues": "Distinct grouped errors (issues) seen in the range, without test errors.",
|
|
2960
|
+
"totals.testIssues": "Issues made on purpose to test deBug (tag test=true). Left out of every count; list them with ?tests=1.",
|
|
2961
|
+
connectivity: "Phones going offline (requests that never left the phone), as bursts per phone, shown on Health as Folded away. Not counted against any endpoint or owner. null when there were none.",
|
|
1764
2962
|
"totals.criticalIssues": "Issues whose 24h occurrence count meets the critical threshold.",
|
|
1765
2963
|
"flows[]": "Parts of the app as tagged by the developer (e.g. auth.login). Sorted by severity, then errors24h, then errorsRange.",
|
|
1766
2964
|
"flows[].failureRate": "failures/(failures+successes) in the range from events logged with an outcome. null when the app logs no outcomes for that flow.",
|
|
1767
|
-
"issues[]": "Grouped errors seen in the range. Sorted by severity, then count24h, then countRange. Narrow with ?status=open|resolved|ignored|all (default all), ?flow=, ?category=, ?owner= and ?impact= (ids from breakdowns); an unknown value is a 400. _filter echoes what was applied.",
|
|
2965
|
+
"issues[]": "Grouped errors seen in the range. Sorted by severity, then count24h, then countRange. Narrow with ?status=open|resolved|ignored|all (default all), ?flow=, ?category=, ?owner= and ?impact= (ids from breakdowns); an unknown value is a 400. Test errors are left out unless ?tests=1. _filter echoes what was applied.",
|
|
2966
|
+
"issues[].offline": "true for the one issue that collects offline bursts (calls that never left the phone). On Health's Folded away line, left out of this list unless ?category=network.offline, and never counted as active.",
|
|
2967
|
+
"issues[].test": "true for an issue made on purpose to test deBug (tag test=true): listed only with ?tests=1, never counted, always severity low.",
|
|
1768
2968
|
"issues[].severity": "Derived from count24h using thresholds critical/high/medium.",
|
|
1769
2969
|
"issues[].title": "Error name and message, with hosts named in Settings shown by their names.",
|
|
1770
2970
|
"issues[].culprit": "Top in-app stack frame: function (file:line).",
|
|
@@ -1861,14 +3061,15 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
|
|
|
1861
3061
|
let errors24h = 0;
|
|
1862
3062
|
let errorsPrev24h = 0;
|
|
1863
3063
|
const issue24h = /* @__PURE__ */ new Map();
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
3064
|
+
const testIssues = new Set(issues.filter(isTestIssue).map((i) => i.fingerprint));
|
|
3065
|
+
const realHours = hourRows.filter((r) => !testIssues.has(r.key));
|
|
3066
|
+
for (const r of hourRows) if (last24.has(r.day)) issue24h.set(r.key, (issue24h.get(r.key) ?? 0) + r.errors);
|
|
3067
|
+
for (const r of realHours) if (last24.has(r.day)) errors24h += r.errors;
|
|
3068
|
+
else errorsPrev24h += r.errors;
|
|
1868
3069
|
let trend;
|
|
1869
3070
|
if (hourly) trend = h24.map((h) => ({
|
|
1870
3071
|
day: h,
|
|
1871
|
-
errors:
|
|
3072
|
+
errors: realHours.filter((r) => r.day === h).reduce((a, r) => a + r.errors, 0),
|
|
1872
3073
|
events: 0
|
|
1873
3074
|
}));
|
|
1874
3075
|
else {
|
|
@@ -1898,22 +3099,37 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
|
|
|
1898
3099
|
const errorsPrev = hourly ? errorsPrev24h : sumRows(prevTotal);
|
|
1899
3100
|
const issueRange = /* @__PURE__ */ new Map();
|
|
1900
3101
|
for (const r of issueDays) issueRange.set(r.key, (issueRange.get(r.key) ?? 0) + r.errors);
|
|
1901
|
-
const
|
|
3102
|
+
const { start: rangeStart, end: rangeEnd } = rangeBounds(range, now);
|
|
1902
3103
|
const versionsInRange = [...new Set(sessionRows.map((r) => r.key.split("|")[0]).filter((v) => v && v !== "unknown"))].sort(compareVersions$1);
|
|
1903
3104
|
const latestVersion = versionsInRange[versionsInRange.length - 1] ?? null;
|
|
1904
|
-
const issueRows = issues.filter((i) => i.lastSeen >= rangeStart && i.firstSeen <=
|
|
1905
|
-
const active = issueRows.filter((i) => i.status !== "ignored" && i.status !== "resolved");
|
|
3105
|
+
const issueRows = issues.filter((i) => i.lastSeen >= rangeStart && i.firstSeen <= rangeEnd).map((i) => toIssueRow(i, issue24h.get(i.fingerprint) ?? 0, hourly ? issue24h.get(i.fingerprint) ?? 0 : issueRange.get(i.fingerprint) ?? 0, thresholds, latestVersion, criticalFlows, hosts)).sort(bySeverityThenCount);
|
|
3106
|
+
const active = issueRows.filter((i) => !i.test && !i.offline && i.status !== "ignored" && i.status !== "resolved");
|
|
1906
3107
|
await Promise.all(active.slice(0, 10).map(async (i) => i.users = await countUsersSafe(store, app, `issue:${i.fingerprint}`, usersSince)));
|
|
1907
3108
|
const lastEvents = new Map(issues.map((i) => [i.fingerprint, i.lastEvent]));
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
3109
|
+
const offlineRow = issueRows.find((i) => i.offline);
|
|
3110
|
+
let connectivity = null;
|
|
3111
|
+
if (offlineRow) {
|
|
3112
|
+
const events = await store.listIssueEvents(app, offlineRow.fingerprint, { limit: 200 });
|
|
3113
|
+
connectivity = {
|
|
3114
|
+
fingerprint: offlineRow.fingerprint,
|
|
3115
|
+
...burstSummary(offlineBursts(inRange(events, rangeStart, rangeEnd), hosts)),
|
|
3116
|
+
failures: offlineRow.countRange,
|
|
3117
|
+
capped: events.length >= 200
|
|
3118
|
+
};
|
|
3119
|
+
}
|
|
3120
|
+
for (const row of issueRows) if (row.labels) {
|
|
3121
|
+
const last = lastEvents.get(row.fingerprint);
|
|
3122
|
+
row.explanation = explain({
|
|
3123
|
+
issue: row,
|
|
3124
|
+
labels: row.labels,
|
|
3125
|
+
lastEvent: last,
|
|
3126
|
+
users: row.users,
|
|
3127
|
+
hosts,
|
|
3128
|
+
rangeLabel: range.label,
|
|
3129
|
+
backend: last ? backendSaid([last], hosts) : void 0,
|
|
3130
|
+
now
|
|
3131
|
+
});
|
|
3132
|
+
}
|
|
1917
3133
|
const incidents = await buildIncidents(incidentRows, incidentIssueRows, issueRows, hourly, (key) => countUsersSafe(store, app, `incident:${key}`, usersSince));
|
|
1918
3134
|
const firstDay = /* @__PURE__ */ new Map();
|
|
1919
3135
|
for (const r of releaseRows) {
|
|
@@ -1987,12 +3203,14 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
|
|
|
1987
3203
|
eventsRange: sumRows(total, "events"),
|
|
1988
3204
|
activeIssues: active.length,
|
|
1989
3205
|
criticalIssues: active.filter((i) => i.severity === "critical").length,
|
|
3206
|
+
testIssues: issueRows.filter((i) => i.test).length,
|
|
1990
3207
|
failingFlows: flows.filter((f) => (hourly ? f.errors24h : f.errorsRange) > 0).length
|
|
1991
3208
|
},
|
|
1992
3209
|
narrative: [],
|
|
1993
3210
|
trend,
|
|
1994
3211
|
flows,
|
|
1995
3212
|
issues: issueRows,
|
|
3213
|
+
connectivity,
|
|
1996
3214
|
incidents,
|
|
1997
3215
|
breakdowns: {
|
|
1998
3216
|
os: top(os),
|
|
@@ -2226,7 +3444,9 @@ function issueLabels(i, latestVersion, criticalFlows = [], hosts = []) {
|
|
|
2226
3444
|
});
|
|
2227
3445
|
}
|
|
2228
3446
|
function toIssueRow(i, count24h, countRange, t, latestVersion = null, criticalFlows = [], hosts = []) {
|
|
2229
|
-
const
|
|
3447
|
+
const test = isTestIssue(i);
|
|
3448
|
+
const offline = isOfflineIssue(i);
|
|
3449
|
+
const base = test || offline ? "low" : severityFromCount(count24h, t);
|
|
2230
3450
|
const critical = new Set(criticalFlows);
|
|
2231
3451
|
return {
|
|
2232
3452
|
labels: issueLabels(i, latestVersion, criticalFlows, hosts),
|
|
@@ -2241,12 +3461,14 @@ function toIssueRow(i, count24h, countRange, t, latestVersion = null, criticalFl
|
|
|
2241
3461
|
title: nameUrlsIn(i.title, hosts),
|
|
2242
3462
|
culprit: i.culprit,
|
|
2243
3463
|
flow: i.flow,
|
|
2244
|
-
severity: i.flow && critical.has(i.flow) && count24h > 0 ? BUMP[base] : base,
|
|
3464
|
+
severity: !test && i.flow && critical.has(i.flow) && count24h > 0 ? BUMP[base] : base,
|
|
2245
3465
|
count24h,
|
|
2246
3466
|
countRange,
|
|
2247
3467
|
countTotal: i.count,
|
|
2248
3468
|
firstSeen: i.firstSeen,
|
|
2249
|
-
lastSeen: i.lastSeen
|
|
3469
|
+
lastSeen: i.lastSeen,
|
|
3470
|
+
test,
|
|
3471
|
+
offline
|
|
2250
3472
|
};
|
|
2251
3473
|
}
|
|
2252
3474
|
var bySeverityThenCount = (a, b) => SEVERITY_ORDER[b.severity] - SEVERITY_ORDER[a.severity] || b.count24h - a.count24h || b.countRange - a.countRange || b.lastSeen.localeCompare(a.lastSeen);
|
|
@@ -2269,7 +3491,7 @@ function narrate(s) {
|
|
|
2269
3491
|
const n = r.preset === "24h" ? f.errors24h : f.errorsRange;
|
|
2270
3492
|
out.push(`${f.flow}${f.critical ? " (a critical flow)" : ""} is the most affected flow (${f.severity}): ${plural(n, "error")} in ${r.label} from ${plural(f.issues, "issue")}.${failRate}`);
|
|
2271
3493
|
}
|
|
2272
|
-
const i = s.issues.find((x) => x.status !== "ignored" && x.status !== "resolved");
|
|
3494
|
+
const i = s.issues.find((x) => !x.test && !x.offline && x.status !== "ignored" && x.status !== "resolved");
|
|
2273
3495
|
if (i && i.countRange > 0) {
|
|
2274
3496
|
const isNew = Date.parse(s.generatedAt) - Date.parse(i.firstSeen) < 3 * DAY;
|
|
2275
3497
|
out.push(`Top issue: "${i.title}"${i.culprit ? ` in ${i.culprit}` : ""}, ${plural(i.countRange, "time")} in ${r.label}${isNew ? `, first seen ${relDays(s.generatedAt, i.firstSeen)}` : ""}.`);
|
|
@@ -2295,6 +3517,41 @@ function relDays(now, then) {
|
|
|
2295
3517
|
const d = Math.floor((Date.parse(now) - Date.parse(then)) / DAY);
|
|
2296
3518
|
return d === 0 ? "today" : d === 1 ? "yesterday" : `${d} days ago`;
|
|
2297
3519
|
}
|
|
3520
|
+
/**
|
|
3521
|
+
* The failed call to show on an issue's page, from occurrences newest first. `latest` is the
|
|
3522
|
+
* newest (what's happening now: during an outage, a timeout with no answer); when it captured
|
|
3523
|
+
* nothing, the newest that did. `lastAnswer` is the newest the server answered, when that's not
|
|
3524
|
+
* `latest`: what the server said last time it said anything.
|
|
3525
|
+
*/
|
|
3526
|
+
function callsOf(events, hosts) {
|
|
3527
|
+
const calls = events.flatMap((se) => {
|
|
3528
|
+
if (se.event.type !== "error") return [];
|
|
3529
|
+
const api = failedRequest(se.event.request, timelineOf(se.event.breadcrumbs, hosts), hosts);
|
|
3530
|
+
return api ? [{
|
|
3531
|
+
api,
|
|
3532
|
+
exchange: exchangeOf(se.event),
|
|
3533
|
+
eventId: se.event.id,
|
|
3534
|
+
ts: se.event.ts
|
|
3535
|
+
}] : [];
|
|
3536
|
+
});
|
|
3537
|
+
const latest = calls[0]?.exchange ? calls[0] : calls.find((c) => c.exchange) ?? calls[0];
|
|
3538
|
+
const answered = calls.find((c) => c.exchange?.response);
|
|
3539
|
+
return {
|
|
3540
|
+
latest,
|
|
3541
|
+
lastAnswer: answered && answered !== latest && !latest?.exchange?.response ? answered : void 0
|
|
3542
|
+
};
|
|
3543
|
+
}
|
|
3544
|
+
/** The service whose call an issue's newest occurrence failed, or the app's own code. */
|
|
3545
|
+
function areaOf(se, labels, hosts) {
|
|
3546
|
+
if (!labels || !isRequestCategory(labels.category)) return { kind: "app" };
|
|
3547
|
+
const e = se?.event.type === "error" ? se.event : void 0;
|
|
3548
|
+
if (!e) return null;
|
|
3549
|
+
const name = failedRequest(e.request, timelineOf(e.breadcrumbs, hosts), hosts)?.service ?? (e.request ? hostNameFor(e.request.url, hosts)?.host.name : void 0);
|
|
3550
|
+
return name ? {
|
|
3551
|
+
kind: "service",
|
|
3552
|
+
name
|
|
3553
|
+
} : null;
|
|
3554
|
+
}
|
|
2298
3555
|
async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.now(), opts = {}) {
|
|
2299
3556
|
const issue = await store.getIssue(app, fingerprint);
|
|
2300
3557
|
if (!issue) return null;
|
|
@@ -2344,6 +3601,7 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
|
|
|
2344
3601
|
if (row.labels) row.labels = withIssueCauses(row.labels, { deviceModels: events.map((e) => e.context.device.model) });
|
|
2345
3602
|
const hints = hintsFor(issue, breakdowns, events, !!row.labels && isRequestCategory(row.labels.category), hosts);
|
|
2346
3603
|
const errorEvents = events.filter((e) => e.event.type === "error");
|
|
3604
|
+
const backend = row.labels && isRequestCategory(row.labels.category) ? backendSaid(errorEvents.length ? errorEvents : issue.lastEvent ? [issue.lastEvent] : [], hosts) : void 0;
|
|
2347
3605
|
row.explanation = row.labels ? explain({
|
|
2348
3606
|
issue: row,
|
|
2349
3607
|
labels: row.labels,
|
|
@@ -2353,10 +3611,17 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
|
|
|
2353
3611
|
platforms: [...new Set(errorEvents.map((e) => e.context.os.name))],
|
|
2354
3612
|
durations: errorEvents.flatMap((e) => e.event.type === "error" && e.event.request?.durationMs !== void 0 ? [e.event.request.durationMs] : []),
|
|
2355
3613
|
rangeLabel: range.label,
|
|
3614
|
+
backend,
|
|
2356
3615
|
now
|
|
2357
3616
|
}) : void 0;
|
|
3617
|
+
const offline = row.offline;
|
|
3618
|
+
const findings = offline ? [] : findingsOf(errorEvents.flatMap((e) => e.event.type === "error" ? [e.event] : []), hosts);
|
|
2358
3619
|
const link = issueUrl(opts.trackers?.dashboardUrl, app, fingerprint);
|
|
3620
|
+
const bounds = rangeBounds(range, now);
|
|
3621
|
+
const inPeriod = errorEvents.filter((e) => e.event.ts >= bounds.start && e.event.ts <= bounds.end);
|
|
3622
|
+
const calls = row.labels && isRequestCategory(row.labels.category) ? callsOf(inPeriod.length ? inPeriod : errorEvents.length ? errorEvents : [issue.lastEvent], hosts) : void 0;
|
|
2359
3623
|
const detail = {
|
|
3624
|
+
area: row.test ? { kind: "tests" } : row.offline ? null : areaOf(issue.lastEvent, row.labels, hosts),
|
|
2360
3625
|
app,
|
|
2361
3626
|
range,
|
|
2362
3627
|
retention,
|
|
@@ -2367,6 +3632,8 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
|
|
|
2367
3632
|
affectedUsers,
|
|
2368
3633
|
links: {},
|
|
2369
3634
|
sample: events[0] ?? issue.lastEvent ?? null,
|
|
3635
|
+
calls,
|
|
3636
|
+
backend,
|
|
2370
3637
|
recent: events.slice(0, 25).map((e) => ({
|
|
2371
3638
|
id: e.event.id,
|
|
2372
3639
|
type: e.event.type,
|
|
@@ -2378,7 +3645,12 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
|
|
|
2378
3645
|
userId: e.user?.id
|
|
2379
3646
|
})),
|
|
2380
3647
|
hints,
|
|
2381
|
-
|
|
3648
|
+
findings,
|
|
3649
|
+
...offline ? {
|
|
3650
|
+
bursts: offlineBursts(inRange(events, bounds.start, bounds.end), hosts),
|
|
3651
|
+
burstsCapped: events.length >= 200
|
|
3652
|
+
} : {},
|
|
3653
|
+
ticketMarkdown: row.explanation ? explanationMarkdown(row.title, row.explanation, link, findings) : void 0
|
|
2382
3654
|
};
|
|
2383
3655
|
detail.links = trackerLinks(detail, opts.trackers ?? {});
|
|
2384
3656
|
return detail;
|
|
@@ -2406,6 +3678,11 @@ function issueStory(d, dashboardUrl) {
|
|
|
2406
3678
|
"Stack (in-app frames):",
|
|
2407
3679
|
...frames.map((f) => `- \`${f.fn ?? "?"}\` ${f.file ?? ""}:${f.line ?? ""}`)
|
|
2408
3680
|
] : [],
|
|
3681
|
+
...d.findings.length ? [
|
|
3682
|
+
"",
|
|
3683
|
+
"What the trail shows:",
|
|
3684
|
+
...d.findings.slice(0, 3).map(findingLine)
|
|
3685
|
+
] : [],
|
|
2409
3686
|
...d.hints.length ? [
|
|
2410
3687
|
"",
|
|
2411
3688
|
x ? "Also worth checking:" : "What to look at:",
|
|
@@ -2482,15 +3759,4 @@ function hintsFor(issue, b, events, explained = false, hosts = []) {
|
|
|
2482
3759
|
});
|
|
2483
3760
|
return hints;
|
|
2484
3761
|
}
|
|
2485
|
-
|
|
2486
|
-
/** Firestore answers FAILED_PRECONDITION while an index deBug needs is missing or still building. */
|
|
2487
|
-
function isIndexError(message) {
|
|
2488
|
-
return /FAILED_PRECONDITION|requires an index|index is currently building/i.test(message);
|
|
2489
|
-
}
|
|
2490
|
-
/** A thrown Firestore error saying a composite index is missing (gRPC code 9), on the Firestore store only. */
|
|
2491
|
-
function isMissingIndex(e, store) {
|
|
2492
|
-
if (store !== "firestore") return false;
|
|
2493
|
-
const err = e;
|
|
2494
|
-
return (err?.code === 9 || err?.code === "failed-precondition") && /index/i.test(String(err?.message ?? ""));
|
|
2495
|
-
}
|
|
2496
|
-
export { seenHosts as A, ingest as C, teamAction as D, hashInviteToken as E, writeRetention as F, writeSettings as I, getLocalWriteStore as L, testAlert as M, validRetention as N, readSettings as O, writeIssueStatus as P, getStore as R, CORS_HEADERS as S, isRequestCategory as T, hashPassword as _, buildSummary as a, startSession as b, resolveRange as c, canManage as d, checkCredentials as f, forgetMember as g, envOwnerEmails as h, buildIssueDetail as i, settingsWriter as j, restampStep as k, DEMO_USER as l, endSession as m, isMissingIndex as n, endpointTimings as o, demoLoginEnabled as p, SUMMARY_FIELDS as r, retentionFor as s, isIndexError as t, authorizeApi as u, readSession as v, parseIngestKeys as w, tooManyAttempts as x, recordFailure as y, config as z };
|
|
3762
|
+
export { testAlert as $, parseIngestKeys as A, endpointsOf as B, hashPassword as C, tooManyAttempts as D, startSession as E, LIVE_MS as F, serviceTile as G, isOpen as H, OUTAGE as I, appNames as J, servicesOf as K, answerOf as L, hashInviteToken as M, teamAction as N, CORS_HEADERS as O, HEALTH_ORDER as P, settingsWriter as Q, byHealth as R, forgetMember as S, recordFailure as T, pct$1 as U, failing as V, plural$2 as W, restampStep as X, readSettings as Y, seenHosts as Z, canManage as _, retentionFor as a, getLocalWriteStore as at, endSession as b, buildEventDetail as c, buildSessionDetail as d, validRetention as et, sessionOutcome as f, authorizeApi as g, DEMO_USER as h, endpointTimings as i, clearStoreCache as it, isRequestCategory as j, ingest as k, failedRequest as l, isMissingIndex as m, buildIssueDetail as n, writeRetention as nt, resolveRange as o, getStore as ot, isIndexError as p, whoseLine as q, buildSummary as r, writeSettings as rt, apiCallOf as s, SUMMARY_FIELDS as t, writeIssueStatus as tt, timelineOf as u, checkCredentials as v, readSession as w, envOwnerEmails as x, demoLoginEnabled as y, endpointLevel as z };
|