@debugkit/dashboard 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/client/assets/_authed-3se5Egqg.js +1 -0
  2. package/dist/client/assets/_authed-BfFWbTOO.js +1 -0
  3. package/dist/client/assets/app-code-BKWbvUMR.js +1 -0
  4. package/dist/client/assets/closed-issues-Cl-8l4oR.js +1 -0
  5. package/dist/client/assets/{copy-button-BJkvWYRY.js → copy-button-BwRd5q7l.js} +1 -1
  6. package/dist/client/assets/customers-Cf-R8BOg.js +1 -0
  7. package/dist/client/assets/empty-state-njgr1sWu.js +7 -0
  8. package/dist/client/assets/endpoints-BMqAUNyE.js +1 -0
  9. package/dist/client/assets/events._eventId-CVRZJ-bU.js +2 -0
  10. package/dist/client/assets/exchange-view-DVwSLcXF.js +2 -0
  11. package/dist/client/assets/index-DEnCDILF.js +10 -0
  12. package/dist/client/assets/{invite._token-DiDJMnQu.js → invite._token-CKAfd5iB.js} +1 -1
  13. package/dist/client/assets/issues._fingerprint-Cm72LNM_.js +55 -0
  14. package/dist/client/assets/login-BTxOhEbN.js +1 -0
  15. package/dist/client/assets/request-panel-DmtFRF8Z.js +1 -0
  16. package/dist/client/assets/services._service-CU1a45f3.js +1 -0
  17. package/dist/client/assets/{sessions._sessionId-B0NxM46y.js → sessions._sessionId-LRo3QNQP.js} +1 -1
  18. package/dist/client/assets/settings-Ctzvl2LL.js +1 -0
  19. package/dist/client/assets/src-BiKxgjA_.js +3 -0
  20. package/dist/client/assets/styles-Tcq0pvWj.css +2 -0
  21. package/dist/client/assets/{team-BRuVQ2bU.js → team-xlrjhEGp.js} +1 -1
  22. package/dist/client/assets/{timeline-BIGo9MCA.js → timeline-WXE49N62.js} +1 -1
  23. package/dist/client/assets/ui-DRjjeSWy.js +1 -0
  24. package/dist/client/assets/{version-banner-VFvyqbPh.js → version-banner-BYe4PVzv.js} +1 -1
  25. package/dist/server/assets/_authed-Bn-IktVS.js +120 -0
  26. package/dist/server/assets/_authed-RXh7GpoE.js +252 -0
  27. package/dist/server/assets/_tanstack-start-manifest_v-C4dQ3MxD.js +122 -0
  28. package/dist/server/assets/app-code-BQL_FcSx.js +212 -0
  29. package/dist/server/assets/app-name-BcbY1BZN.js +11 -0
  30. package/dist/server/assets/closed-issues-o9U80FXv.js +45 -0
  31. package/dist/server/assets/config-Bp8m0B8q.js +57 -0
  32. package/dist/server/assets/{customers-827fC5I3.js → customers-CD84Bz4P.js} +11 -6
  33. package/dist/server/assets/{demo-DDNiXCgj.js → demo-D-o8m75S.js} +222 -6
  34. package/dist/server/assets/{dynamodb-Cd4blXkh.js → dynamodb-CcH0cp7R.js} +2 -2
  35. package/dist/server/assets/empty-state-wz34oAi8.js +163 -0
  36. package/dist/server/assets/endpoints-8N389Poe.js +248 -0
  37. package/dist/server/assets/{endpoints-_T0NSOyA.js → endpoints-C0Cxc_-x.js} +38 -36
  38. package/dist/server/assets/{events._eventId-Dcsu4oVO.js → events._eventId-BOHkNO5W.js} +90 -351
  39. package/dist/server/assets/exchange-view-DzoSBt_R.js +112 -0
  40. package/dist/server/assets/{firestore-c8Sz2Q4O.js → firestore-DyvYMiz3.js} +2 -2
  41. package/dist/server/assets/{fns-BiUOnY1_.js → fns-BcVGbSID.js} +20 -7
  42. package/dist/server/assets/fns-DE03YExD.js +831 -0
  43. package/dist/server/assets/{invite._token-7DjEeeaD.js → invite._token-DhlACkIe.js} +3 -3
  44. package/dist/server/assets/{issues._fingerprint-vc9r3jgg.js → issues._fingerprint-BBd-_Nn5.js} +400 -28
  45. package/dist/server/assets/{login-CwKAx9Bj.js → login-CbfdHJvg.js} +3 -3
  46. package/dist/server/assets/{memory-Bv4C8uyK.js → memory-CZBUh1Zc.js} +75 -3
  47. package/dist/server/assets/{errors-Po4Tt5zd.js → queries-wha4cZbw.js} +1394 -128
  48. package/dist/server/assets/request-panel-DpCSwl2q.js +162 -0
  49. package/dist/server/assets/{router-Dk3XHn7B.js → router-BXkzR89j.js} +195 -91
  50. package/dist/server/assets/services._service-DWjFFGON.js +309 -0
  51. package/dist/server/assets/{sessions._sessionId-BvXxK9qF.js → sessions._sessionId-DDspiPTe.js} +2 -2
  52. package/dist/server/assets/{settings-B9NLQjl1.js → settings-DCLIaZ_p.js} +146 -25
  53. package/dist/server/assets/{src-oWPSAvXv.js → src-B2GKPQyN.js} +1 -1
  54. package/dist/server/assets/{team-Bsm2wFG0.js → team-jd_TmWP4.js} +3 -3
  55. package/dist/server/assets/ui-C3pl0edn.js +302 -0
  56. package/dist/server/assets/{version-banner-nf-yZurq.js → version-banner-BX34ijG5.js} +1 -1
  57. package/dist/server/assets/{versions-BOlL8jam.js → versions-YI_zK1ql.js} +3 -3
  58. package/dist/server/assets/warm-C3GbXP1_.js +16 -0
  59. package/dist/server/server.js +37 -29
  60. package/package.json +4 -4
  61. package/dist/client/assets/_authed-C29P68T-.js +0 -1
  62. package/dist/client/assets/_authed-D-dYbHoP.js +0 -7
  63. package/dist/client/assets/charts--S1o8iId.js +0 -55
  64. package/dist/client/assets/customers-Cwx5qEO1.js +0 -1
  65. package/dist/client/assets/endpoints-zz1TotR2.js +0 -1
  66. package/dist/client/assets/events._eventId-Bx6TfUjv.js +0 -3
  67. package/dist/client/assets/index-COHr_cc3.js +0 -10
  68. package/dist/client/assets/issues._fingerprint-Cv522h2j.js +0 -1
  69. package/dist/client/assets/login-CKkpc0t5.js +0 -1
  70. package/dist/client/assets/settings-CIlF10MI.js +0 -1
  71. package/dist/client/assets/src-BKSiJUs3.js +0 -3
  72. package/dist/client/assets/styles-GAsMUS4-.css +0 -2
  73. package/dist/client/assets/ui-CaEZB1j2.js +0 -1
  74. package/dist/server/assets/_authed-BJWDiF9i.js +0 -92
  75. package/dist/server/assets/_authed-X9u_5JUy.js +0 -902
  76. package/dist/server/assets/_tanstack-start-manifest_v-C6vaeBdY.js +0 -90
  77. package/dist/server/assets/charts-DzTVqJ8z.js +0 -238
  78. package/dist/server/assets/fns-CVbk8Gf9.js +0 -1310
  79. package/dist/server/assets/ui-SAkSlP7f.js +0 -193
@@ -1,66 +1,110 @@
1
1
  import { a as getCookie, i as deleteCookie, o as getRequestHeader, s as setCookie } from "../server.js";
2
- import { C as CAUSE_INFO, D as cleanFnName, E as OWNER_INFO, O as markLibraryFrames, S as CATEGORY_INFO, _ as severityFromCount, b as normaliseHostMatch, d as SEVERITY_ORDER, f as compareVersions$1, g as releaseKey, h as percentileMs, l as Envelope, m as parseStack, r as compareVersions, u as DEFAULT_SEVERITY_THRESHOLDS, v as hostNameFor, w as IMPACT_INFO, x as validHostNames, y as nameUrlsIn } from "./memory-Bv4C8uyK.js";
3
- import { n as TEAM_ROLES, t as MAX_RETENTION_DAYS } from "./src-oWPSAvXv.js";
2
+ import { A as hostNameFor, B as OWNER_INFO, C as maskPersonal, D as severityFromCount, E as releaseKey, F as validHostNames, H as markLibraryFrames, I as CATEGORY_INFO, L as CAUSE_INFO, M as nameUrl, N as nameUrlsIn, O as hostKindLabel, P as normaliseHostMatch, R as IMPACT_INFO, S as isTestIssue, T as percentileMs, V as cleanFnName, _ as TEST_TAG, f as OFFLINE_FINGERPRINT, g as SEVERITY_ORDER, j as hostOf$1, l as Envelope, p as OFFLINE_TITLE, r as compareVersions, u as AIRPLANE_TAG, v as compareVersions$1, w as parseStack, x as isTestEvent, y as isOfflineIssue } from "./memory-CZBUh1Zc.js";
3
+ import { n as validAppName, t as appLabel } from "./app-name-BcbY1BZN.js";
4
+ import { n as TEAM_ROLES, t as MAX_RETENTION_DAYS } from "./src-B2GKPQyN.js";
5
+ import { t as config } from "./config-Bp8m0B8q.js";
4
6
  import { createHash, createHmac, randomBytes, scryptSync, timingSafeEqual } from "node:crypto";
5
- // src/server/config.ts
6
- var loaded = false;
7
- function loadEnv() {
8
- if (loaded) return;
9
- loaded = true;
10
- try {
11
- process.loadEnvFile?.(process.env.DEBUG_ENV_FILE ?? ".env");
12
- } catch {}
13
- }
14
- var generatedSecret;
15
- function config() {
16
- loadEnv();
17
- const env = process.env;
18
- const store = env.DEBUG_STORE ?? "demo";
19
- let secret = env.DEBUG_SESSION_SECRET;
20
- if (!secret || secret.length < 32) {
21
- if (store !== "demo" && store !== "memory") throw new Error("DEBUG_SESSION_SECRET must be set (≥32 chars) when not in demo mode. Try: openssl rand -hex 32");
22
- secret = generatedSecret ??= randomBytes(32).toString("hex");
23
- }
24
- const num = (v, d) => v && !Number.isNaN(Number(v)) ? Number(v) : d;
25
- const severity = {
26
- critical: num(env.DEBUG_SEVERITY_CRITICAL, DEFAULT_SEVERITY_THRESHOLDS.critical),
27
- high: num(env.DEBUG_SEVERITY_HIGH, DEFAULT_SEVERITY_THRESHOLDS.high),
28
- medium: num(env.DEBUG_SEVERITY_MEDIUM, DEFAULT_SEVERITY_THRESHOLDS.medium)
7
+ /** Entries kept at most; the oldest go first. */
8
+ var MAX_ENTRIES = 2e3;
9
+ /** The reads that are cached. */
10
+ var CACHED_READS = /* @__PURE__ */ new Set([
11
+ "getSettings",
12
+ "getVersions",
13
+ "countUsers",
14
+ "listApps",
15
+ "listIssues",
16
+ "getIssue",
17
+ "getRollups",
18
+ "listIssueEvents",
19
+ "listSessionEvents"
20
+ ]);
21
+ /** Calls that change data (demo and memory mode write through the store): they clear the cache. */
22
+ var WRITE = /^(write|upsert|increment|put|set|register|note|mark|delete|remove|restamp|create|accept|revoke|change)/;
23
+ /**
24
+ * A short-lived cache in front of the store, so moving between pages doesn't read the same data
25
+ * from Firestore or DynamoDB again and again.
26
+ *
27
+ * - The same read made twice at once (the health wall's summary and endpoints both read the issue
28
+ * list) is made once.
29
+ * - A read newer than FRESH_MS is served as it is; one up to STALE_MS old is served at once and
30
+ * read again in the background, so the next page has it fresh.
31
+ * - A write through the store (demo and memory mode), or clearStoreCache() after a change made
32
+ * through the ingest function's admin routes, clears it. A background read that started
33
+ * before the clear is dropped.
34
+ * - Only CACHED_READS are cached: team and sign-in reads always go to the store.
35
+ *
36
+ * Per server instance, like the sign-in limits in auth.ts: on hosting with several instances,
37
+ * another one can show data up to STALE_MS old after a change. The Refresh button clears this
38
+ * instance's cache.
39
+ */
40
+ function cacheStore(store, now = Date.now) {
41
+ const entries = /* @__PURE__ */ new Map();
42
+ let generation = 0;
43
+ const read = (key, call) => {
44
+ const t = now();
45
+ const hit = entries.get(key);
46
+ if (hit) {
47
+ entries.delete(key);
48
+ entries.set(key, hit);
49
+ }
50
+ if (hit && t - hit.at < 3e4) return hit.value;
51
+ if (hit && t - hit.at < 12e4) {
52
+ if (!hit.refreshing) {
53
+ hit.refreshing = true;
54
+ const started = generation;
55
+ const next = call();
56
+ next.then(() => generation === started && entries.set(key, {
57
+ at: now(),
58
+ value: next
59
+ }), () => hit.refreshing = false);
60
+ }
61
+ return hit.value;
62
+ }
63
+ const value = call();
64
+ entries.set(key, {
65
+ at: t,
66
+ value
67
+ });
68
+ value.catch(() => entries.get(key)?.value === value && entries.delete(key));
69
+ while (entries.size > MAX_ENTRIES) entries.delete(entries.keys().next().value);
70
+ return value;
71
+ };
72
+ const clear = () => {
73
+ generation++;
74
+ entries.clear();
29
75
  };
30
76
  return {
31
- store,
32
- sessionSecret: secret,
33
- users: env.DEBUG_DASHBOARD_USERS ?? "",
34
- apiToken: env.DEBUG_API_TOKEN,
35
- /** Demo mode: keys accepted by POST /api/ingest. "*" = any app. */
36
- localIngestKeys: env.DEBUG_LOCAL_INGEST_KEYS ?? "pk_local:*",
37
- firestoreRoot: env.DEBUG_FIRESTORE_ROOT ?? "debug_apps",
38
- dynamoTable: env.DEBUG_DYNAMO_TABLE ?? "debug",
39
- severity,
40
- /** The ingest function's admin routes (…/admin) and their token, for changing settings. */
41
- adminUrl: env.DEBUG_ADMIN_URL?.replace(/\/+$/, ""),
42
- adminToken: env.DEBUG_ADMIN_TOKEN,
43
- /** The CLI environment this dashboard belongs to ("preview"), for commands it suggests (`--env preview`). */
44
- environment: /^[a-z][a-z0-9-]{0,30}$/.test(env.DEBUG_ENVIRONMENT ?? "") ? env.DEBUG_ENVIRONMENT : void 0,
45
- /** The deployment's default retention (what ingest uses before an app has a setting). */
46
- retentionDays: num(env.DEBUG_RETENTION_DAYS, 0) || void 0,
47
- /** "Create issue" buttons on the issue page. */
48
- trackers: {
49
- linear: env.DEBUG_LINEAR === "1" || !!env.DEBUG_LINEAR_TEAM,
50
- linearTeam: env.DEBUG_LINEAR_TEAM || void 0,
51
- githubRepo: env.DEBUG_GITHUB_REPO || void 0,
52
- dashboardUrl: env.DEBUG_DASHBOARD_URL || void 0
53
- },
54
- secureCookies: env.NODE_ENV === "production" && env.DEBUG_INSECURE_COOKIES !== "1",
55
- /** Firebase Hosting only forwards a cookie named __session to the function behind it. */
56
- cookieName: /^[A-Za-z0-9_]{1,40}$/.test(env.DEBUG_COOKIE_NAME ?? "") ? env.DEBUG_COOKIE_NAME : "debug_session"
77
+ store: new Proxy(store, { get(target, prop, receiver) {
78
+ const v = Reflect.get(target, prop, receiver);
79
+ if (typeof v !== "function" || typeof prop !== "string") return v;
80
+ if (CACHED_READS.has(prop)) return (...args) => read(`${prop}:${JSON.stringify(args)}`, () => v.apply(target, args));
81
+ if (!WRITE.test(prop)) return v.bind(target);
82
+ return async (...args) => {
83
+ clear();
84
+ try {
85
+ return await v.apply(target, args);
86
+ } finally {
87
+ clear();
88
+ }
89
+ };
90
+ } }),
91
+ clear
57
92
  };
58
93
  }
59
94
  // src/server/store.ts
60
95
  var store;
96
+ var cache$1;
61
97
  /** The dashboard only ever reads. Point it at read-only credentials. */
62
98
  function getStore() {
63
- return store ??= create();
99
+ return store ??= create().then((s) => {
100
+ cache$1 = cacheStore(s);
101
+ import("./warm-C3GbXP1_.js").then((m) => m.warmUp(cache$1.store)).catch(() => void 0);
102
+ return cache$1.store;
103
+ });
104
+ }
105
+ /** Forget every cached read: called after the dashboard changes anything (status, settings, team). */
106
+ function clearStoreCache() {
107
+ cache$1?.clear();
64
108
  }
65
109
  /**
66
110
  * Demo mode only: the in-memory store also accepts events, so a local app can send to
@@ -77,23 +121,25 @@ async function create() {
77
121
  case "firestore": {
78
122
  const { initializeApp, getApps } = await import("firebase-admin/app");
79
123
  const { getFirestore } = await import("firebase-admin/firestore");
80
- const { FirestoreStore } = await import("./firestore-c8Sz2Q4O.js");
124
+ const { FirestoreStore } = await import("./firestore-DyvYMiz3.js");
81
125
  if (!getApps().length) initializeApp();
82
126
  return new FirestoreStore(getFirestore(), { root: cfg.firestoreRoot });
83
127
  }
84
128
  case "dynamodb": {
85
129
  const { DynamoDBClient } = await import("@aws-sdk/client-dynamodb");
86
130
  const { DynamoDBDocumentClient } = await import("@aws-sdk/lib-dynamodb");
87
- const { DynamoStore } = await import("./dynamodb-Cd4blXkh.js");
131
+ const { DynamoStore } = await import("./dynamodb-CcH0cp7R.js");
88
132
  return new DynamoStore(DynamoDBDocumentClient.from(new DynamoDBClient({})), { table: cfg.dynamoTable });
89
133
  }
90
134
  case "memory": {
91
- const { MemoryStore } = await import("./memory-Bv4C8uyK.js").then((n) => n.n);
135
+ const { MemoryStore } = await import("./memory-CZBUh1Zc.js").then((n) => n.n);
92
136
  return new MemoryStore();
93
137
  }
94
138
  default: {
95
- const { seedDemo } = await import("./demo-DDNiXCgj.js");
96
- return seedDemo();
139
+ const { keepUpstreamDown, seedDemo } = await import("./demo-D-o8m75S.js");
140
+ const demo = await seedDemo();
141
+ keepUpstreamDown(demo, clearStoreCache);
142
+ return demo;
97
143
  }
98
144
  }
99
145
  }
@@ -117,6 +163,7 @@ async function admin(method, route, body, query) {
117
163
  });
118
164
  const json = await res.json().catch(() => ({}));
119
165
  if (!res.ok) throw new Error(json.error ?? `The ingest function answered ${res.status}`);
166
+ if (method === "PUT") clearStoreCache();
120
167
  return json;
121
168
  }
122
169
  async function readSettings(app) {
@@ -126,6 +173,7 @@ async function readSettings(app) {
126
173
  const out = await admin("GET", "/settings", void 0, { app });
127
174
  return {
128
175
  app,
176
+ displayName: out.settings?.displayName ?? "",
129
177
  retentionDays: out.retentionDays,
130
178
  criticalFlows: out.settings?.criticalFlows ?? [],
131
179
  alerts: out.settings?.alerts ?? {},
@@ -137,6 +185,7 @@ async function readSettings(app) {
137
185
  } catch (err) {
138
186
  return {
139
187
  app,
188
+ displayName: "",
140
189
  retentionDays: c.retentionDays ?? null,
141
190
  criticalFlows: [],
142
191
  alerts: {},
@@ -150,6 +199,7 @@ async function readSettings(app) {
150
199
  const hook = s?.alerts?.slackWebhookUrl;
151
200
  return {
152
201
  app,
202
+ displayName: s?.displayName ?? "",
153
203
  retentionDays: s?.retentionDays ?? c.retentionDays ?? (writer === "local" ? 30 : null),
154
204
  criticalFlows: s?.criticalFlows ?? [],
155
205
  alerts: {
@@ -163,6 +213,12 @@ async function readSettings(app) {
163
213
  note: writer === "none" ? "Changing settings needs DEBUG_ADMIN_URL and DEBUG_ADMIN_TOKEN (npx debugkit init sets them)." : void 0
164
214
  };
165
215
  }
216
+ /** Every app's display name, by id: those with none set are left out (show the id). */
217
+ async function appNames(apps) {
218
+ const store = await getStore();
219
+ const names = await Promise.all(apps.map((a) => store.getSettings(a).then((s) => s?.displayName?.trim() ?? "", () => "")));
220
+ return Object.fromEntries(apps.map((a, i) => [a, names[i]]).filter(([, n]) => n));
221
+ }
166
222
  /** Hosts the app called in the last 30 days, busiest first: suggestions for naming them. */
167
223
  async function seenHosts(app, now = Date.now()) {
168
224
  const day = (t) => new Date(t).toISOString().slice(0, 10);
@@ -191,12 +247,15 @@ async function writeRetention(app, days) {
191
247
  else throw new Error("Settings can't be changed from this dashboard: DEBUG_ADMIN_URL and DEBUG_ADMIN_TOKEN aren't set.");
192
248
  return readSettings(app);
193
249
  }
194
- /** Change critical flows, alert settings and/or host names. Validation happens in the ingest function (or here, locally). */
250
+ /** Change the display name, critical flows, alert settings and/or host names. Validation happens in the ingest function (or here, locally). */
195
251
  async function writeSettings(app, patch) {
196
252
  const writer = settingsWriter();
197
253
  if (writer === "admin") await admin("PUT", "/settings", {
198
254
  app,
199
255
  ...patch
256
+ }).catch((err) => {
257
+ if (patch.displayName !== void 0 && /nothing to change/i.test(err.message)) throw new Error("Your ingest function is older than app names: upgrade it with npx debugkit upgrade, then save again.");
258
+ throw err;
200
259
  });
201
260
  else if (writer === "local") {
202
261
  const store = await getLocalWriteStore();
@@ -205,11 +264,14 @@ async function writeSettings(app, patch) {
205
264
  if (patch.alerts && url && !url.includes("…") && !/^https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/_-]{10,200}$/.test(url)) throw new Error("Use a Slack incoming-webhook URL (https://hooks.slack.com/services/…).");
206
265
  const hosts = patch.hosts ? validHostNames(patch.hosts) : void 0;
207
266
  if (typeof hosts === "string") throw new Error(hosts);
267
+ const name = patch.displayName === void 0 ? void 0 : validAppName(patch.displayName);
268
+ if (typeof name === "string") throw new Error(name);
208
269
  const alerts = patch.alerts ? {
209
270
  ...patch.alerts,
210
271
  slackWebhookUrl: url?.includes("…") ? cur?.alerts?.slackWebhookUrl : url || void 0
211
272
  } : void 0;
212
273
  await store.putSettings(app, {
274
+ ...name !== void 0 ? { displayName: name.name } : {},
213
275
  ...patch.criticalFlows ? { criticalFlows: patch.criticalFlows } : {},
214
276
  ...alerts ? { alerts } : {},
215
277
  ...hosts ? { hosts } : {}
@@ -257,6 +319,233 @@ async function restampStep(app, cursor) {
257
319
  }
258
320
  throw new Error("Settings can't be changed from this dashboard.");
259
321
  }
322
+ // src/lib/health.ts
323
+ /** Worst first when sorted descending; healthy comes last. */
324
+ var HEALTH_ORDER = {
325
+ ...SEVERITY_ORDER,
326
+ healthy: -1
327
+ };
328
+ /**
329
+ * What sets a service's level. Two ways to be bad: a large share of calls failing (with enough
330
+ * failures that it isn't one unlucky call), or a large number failing even when the share is
331
+ * small, because that's still many customers.
332
+ */
333
+ var SERVICE_LEVELS = {
334
+ critical: {
335
+ rate: .25,
336
+ failed: 5,
337
+ volume: 500
338
+ },
339
+ high: {
340
+ rate: .1,
341
+ failed: 3,
342
+ volume: 100
343
+ },
344
+ medium: {
345
+ rate: .05,
346
+ failed: 1
347
+ },
348
+ /** With no request summaries from the SDK: stored failures alone. */
349
+ stored: {
350
+ critical: 50,
351
+ high: 20,
352
+ medium: 5
353
+ }
354
+ };
355
+ /**
356
+ * An outage: in the last 15 minutes, calls to a service got no answer or a 5xx for at least 3
357
+ * different people, and at least 5% of today's calls to it failed. The people make it more than
358
+ * one phone or one bad request; the share keeps a few unlucky calls on a busy service from
359
+ * reading as down. Request summaries are kept per day, so the share is today's (UTC), not the
360
+ * last 15 minutes'. Without summaries for today, the people alone decide.
361
+ */
362
+ var OUTAGE = {
363
+ windowMs: 9e5,
364
+ users: 3,
365
+ share: .05
366
+ };
367
+ /** Whether recent no-answer and 5xx calls are an outage: enough people, and enough of today's calls. */
368
+ function isOutage(people, today) {
369
+ if (people < OUTAGE.users) return false;
370
+ return !today?.calls || today.failed / today.calls >= OUTAGE.share;
371
+ }
372
+ /** Failing within this long ago counts as still happening. */
373
+ var LIVE_MS = 36e5;
374
+ /** Still open: not resolved or ignored. */
375
+ var isOpen = (status) => status !== "resolved" && status !== "ignored";
376
+ /** How badly an endpoint fails: the larger of the SDK's failed calls and the stored failures. */
377
+ var failing = (r) => Math.max(r.failedCalls ?? 0, r.failures);
378
+ /** "27%", or "<1%" for a small share that isn't none. */
379
+ var pct$1 = (r) => r > 0 && r < .01 ? "<1%" : `${Math.round(r * 100)}%`;
380
+ /** "1 call", "126 calls"; "2 people" with a plural given. */
381
+ var plural$2 = (n, w, many = `${w}s`) => `${n.toLocaleString()} ${n === 1 ? w : many}`;
382
+ /** How bad a service is, by share and by number of failed calls. See SERVICE_LEVELS. */
383
+ function serviceLevel(s) {
384
+ const L = SERVICE_LEVELS;
385
+ if (s.calls > 0) {
386
+ if (!s.failedCalls && !s.failures) return "healthy";
387
+ const rate = s.failedCalls / s.calls;
388
+ if (s.failedCalls >= L.critical.volume || rate >= L.critical.rate && s.failedCalls >= L.critical.failed) return "critical";
389
+ if (s.failedCalls >= L.high.volume || rate >= L.high.rate && s.failedCalls >= L.high.failed) return "high";
390
+ if (rate >= L.medium.rate && s.failedCalls >= L.medium.failed) return "medium";
391
+ return "low";
392
+ }
393
+ const n = s.failures;
394
+ return !n ? "healthy" : n >= L.stored.critical ? "critical" : n >= L.stored.high ? "high" : n >= L.stored.medium ? "medium" : "low";
395
+ }
396
+ /** One endpoint's level: as a service's, and critical when it's down for many people right now (see OUTAGE). */
397
+ function endpointLevel(e) {
398
+ if (e.down && isOutage(e.down.who.length, e.today)) return "critical";
399
+ return serviceLevel({
400
+ calls: e.calls ?? 0,
401
+ failedCalls: e.failedCalls ?? 0,
402
+ failures: e.failures
403
+ });
404
+ }
405
+ /**
406
+ * How to group service names: case doesn't matter, and a plural joins its singular when both are
407
+ * there ("Balances" with "balance"). "status" stays "status" when there's no "statu".
408
+ */
409
+ function serviceKeys(names) {
410
+ const lower = (n) => (n ?? "").trim().toLowerCase();
411
+ const known = new Set(names.map(lower));
412
+ return (name) => {
413
+ const l = lower(name);
414
+ return l.endsWith("s") && known.has(l.slice(0, -1)) ? l.slice(0, -1) : l;
415
+ };
416
+ }
417
+ /** Endpoints added up per service, most failures first. */
418
+ function servicesOf(endpoints) {
419
+ const keyOf = serviceKeys(endpoints.map((e) => e.service));
420
+ const by = /* @__PURE__ */ new Map();
421
+ for (const e of endpoints) by.set(keyOf(e.service), [...by.get(keyOf(e.service)) ?? [], e]);
422
+ return [...by.entries()].map(([key, list]) => {
423
+ const calls = list.reduce((n, e) => n + (e.calls ?? 0), 0);
424
+ const failedCalls = list.reduce((n, e) => n + (e.failedCalls ?? 0), 0);
425
+ const p95s = list.map((e) => e.p95).filter((v) => v !== null);
426
+ const busiest = [...list].sort((a, b) => (b.calls ?? 0) - (a.calls ?? 0));
427
+ const kinded = busiest.find((e) => e.kind);
428
+ return {
429
+ key: key || "-",
430
+ service: busiest[0].service,
431
+ ...kinded ? {
432
+ kind: kinded.kind,
433
+ ...kinded.provider ? { provider: kinded.provider } : {}
434
+ } : {},
435
+ down: downOfService(list),
436
+ endpoints: list.length,
437
+ calls,
438
+ failedCalls,
439
+ failureRate: calls ? failedCalls / calls : null,
440
+ worstP95: p95s.length ? Math.max(...p95s) : null,
441
+ failures: list.reduce((n, e) => n + e.failures, 0),
442
+ lastFailure: list.map((e) => e.lastFailure).filter((t) => !!t).sort().pop()
443
+ };
444
+ }).sort((a, b) => failing(b) - failing(a) || b.calls - a.calls);
445
+ }
446
+ /** A service's recent no-answer and 5xx calls, across its endpoints: people counted once. */
447
+ function downOfService(list) {
448
+ const downs = list.map((e) => e.down).filter((d) => !!d);
449
+ if (!downs.length) return void 0;
450
+ const statuses = /* @__PURE__ */ new Map();
451
+ for (const d of downs) for (const s of d.statuses) statuses.set(s.key, (statuses.get(s.key) ?? 0) + s.count);
452
+ const days = list.map((e) => e.today).filter((t) => !!t);
453
+ return {
454
+ ...days.length ? { today: {
455
+ calls: days.reduce((n, t) => n + t.calls, 0),
456
+ failed: days.reduce((n, t) => n + t.failed, 0)
457
+ } } : {},
458
+ users: new Set(downs.flatMap((d) => d.who)).size,
459
+ failures: downs.reduce((n, d) => n + d.failures, 0),
460
+ statuses: [...statuses.entries()].map(([key, count]) => ({
461
+ key,
462
+ count
463
+ })).sort((a, b) => b.count - a.count || a.key.localeCompare(b.key)),
464
+ last: downs.map((d) => d.last).sort().pop()
465
+ };
466
+ }
467
+ /**
468
+ * "Cell C Balance is down: no answer or a 5xx for 6 people in the last 15 min": the outage line, when a
469
+ * service is failing broadly right now (see OUTAGE). Undefined when it isn't.
470
+ */
471
+ function outageOf(s) {
472
+ if (!s.down || !isOutage(s.down.users, s.down.today)) return void 0;
473
+ const name = s.service ?? "Unnamed service";
474
+ return `${s.kind === "upstream" && s.provider && !name.toLowerCase().includes(s.provider.toLowerCase()) ? `${s.provider} ${name}` : name} is down: ${s.down.statuses.every((x) => /^\d+$/.test(x.key)) ? "a 5xx" : s.down.statuses.some((x) => /^\d+$/.test(x.key)) ? "no answer or a 5xx" : "no answer"} for ${plural$2(s.down.users, "person", "people")} in the last ${Math.round(OUTAGE.windowMs / 6e4)} min`;
475
+ }
476
+ /** A service's endpoints, by its key (as in the address). */
477
+ function endpointsOf(key, endpoints) {
478
+ const keyOf = serviceKeys(endpoints.map((e) => e.service));
479
+ const want = key === "-" ? "-" : keyOf(key) || "-";
480
+ return endpoints.filter((e) => (keyOf(e.service) || "-") === want);
481
+ }
482
+ /**
483
+ * What's wrong with a service, in one line, from its worst endpoint: the status, the backend's own
484
+ * code and message, and which call. "404 · 30007 Balance lookup unsuccessful · POST /ppd/api/balance/getBalances".
485
+ */
486
+ function headlineOf(endpoints) {
487
+ const worst = [...endpoints].filter((e) => failing(e) > 0).sort((a, b) => failing(b) - failing(a))[0];
488
+ if (!worst) return void 0;
489
+ return `${[worst.statuses[0]?.key, [worst.backendCodes[0]?.key, worst.backendMessages[0]?.key].filter(Boolean).join(" ")].filter(Boolean).join(" · ") || "Failing"} · ${worst.method ? `${worst.method} ` : ""}${worst.path ?? worst.endpoint}`;
490
+ }
491
+ /** A service's box: its level, what's wrong, and how much. */
492
+ function serviceTile(s, endpoints, now) {
493
+ const outage = outageOf(s);
494
+ const level = outage ? "critical" : serviceLevel(s);
495
+ const measure = s.calls > 0 ? s.failedCalls ? `${pct$1(s.failedCalls / s.calls)} of ${plural$2(s.calls, "call")} failed` : `${plural$2(s.calls, "call")}, none failed` : s.failures ? plural$2(s.failures, "failure") : "No calls";
496
+ return {
497
+ key: s.key,
498
+ name: s.service ?? "Unnamed service",
499
+ kind: "service",
500
+ level,
501
+ headline: level === "healthy" ? void 0 : headlineOf(endpointsOf(s.key, endpoints)),
502
+ measure,
503
+ live: level !== "healthy" && !!s.lastFailure && now - Date.parse(s.lastFailure) < 36e5,
504
+ lastFailure: s.lastFailure,
505
+ weight: failing(s),
506
+ ...s.kind ? {
507
+ side: s.kind,
508
+ sideLabel: hostKindLabel(s.kind, s.provider),
509
+ ...s.provider ? { provider: s.provider } : {}
510
+ } : {},
511
+ ...outage ? { outage } : {}
512
+ };
513
+ }
514
+ /** Worst level first, then down right now, then still happening, then most failures, then by name. */
515
+ var byHealth = (a, b) => HEALTH_ORDER[b.level] - HEALTH_ORDER[a.level] || Number(!!b.outage) - Number(!!a.outage) || Number(b.live) - Number(a.live) || b.weight - a.weight || a.name.localeCompare(b.name);
516
+ /**
517
+ * Who an outage is with, for a client: an upstream's ("It's Cell C's service, not our backend or
518
+ * the app.") or our own backend's. Nothing when nobody said whose service it is.
519
+ */
520
+ function whoseLine(t) {
521
+ if (t.side === "upstream") return `It's ${t.provider ? `${t.provider}'s` : "a third party's"} service, not our backend or the app.`;
522
+ return t.side === "ours" ? "It's our own backend, not an upstream service or the app." : "";
523
+ }
524
+ /**
525
+ * The answer to "why is it breaking?", for a client: what's down right now and whose service it
526
+ * is, what's failing badly, the worst of it with what the server said, whether it's still
527
+ * happening, and how much is healthy.
528
+ */
529
+ function answerOf(tiles, rangeLabel) {
530
+ const down = tiles.filter((t) => t.outage);
531
+ const downLines = down.map((t) => [`${t.outage}.`, whoseLine(t)].filter(Boolean).join(" "));
532
+ const healthyLine = `${tiles.filter((t) => t.level === "healthy").length} of ${tiles.length} ${tiles.length === 1 ? "part is" : "parts are"} healthy.`;
533
+ const failingBadly = tiles.filter((t) => t.level !== "healthy" && t.level !== "low");
534
+ if (!failingBadly.length) return `Nothing is failing badly in ${rangeLabel}. ${healthyLine}`;
535
+ const top = failingBadly.find((t) => !t.outage);
536
+ const names = (ts) => ts.length === 1 ? ts[0].name : `${ts.slice(0, -1).map((t) => t.name).join(", ")} and ${ts[ts.length - 1].name}`;
537
+ const critical = failingBadly.filter((t) => t.level === "critical" && !t.outage);
538
+ const others = failingBadly.filter((t) => !t.outage);
539
+ const lead = critical.length ? `${critical.length === 1 ? "1 part is" : `${critical.length} parts are`} ${down.length ? "also " : ""}failing badly in ${rangeLabel}: ${names(critical)}.` : others.length ? `${others.length === 1 ? "1 part has" : `${others.length} parts have`} ${down.length ? "other " : ""}problems in ${rangeLabel}: ${names(others)}.` : "";
540
+ const live = failingBadly.filter((t) => t.live).length;
541
+ return [
542
+ ...downLines,
543
+ lead,
544
+ top ? `${down.length ? "Worst otherwise" : "Worst"}: ${top.name}, ${top.measure}${top.headline ? ` (${top.headline})` : ""}.` : "",
545
+ live ? `${live === 1 ? "1 is" : `${live} are`} still happening (failed in the last hour).` : "",
546
+ healthyLine
547
+ ].filter(Boolean).join(" ");
548
+ }
260
549
  var EMAIL = /^[^\s@/]{1,64}@[^\s@/]{1,190}\.[^\s@/]{2,63}$/;
261
550
  var HASH = /^scrypt\.[0-9a-f]{32}\.[0-9a-f]{128}$/;
262
551
  var bad = (error, status = 400) => ({
@@ -442,7 +731,7 @@ async function sendAlerts(app, outcome, issues, settings, post = defaultPost) {
442
731
  u.flow && `flow \`${u.flow}\`${critical.has(u.flow) ? " (critical)" : ""}`,
443
732
  u.version && `version ${u.version}`
444
733
  ].filter(Boolean).join(" · ");
445
- const text = `${kind === "new" ? ":new: New issue" : ":rotating_light: Regressed"} in *${app}*: ${link ? `<${link}|${title}>` : title}${where ? `\n${where}` : ""}`;
734
+ const text = `${kind === "new" ? ":new: New issue" : ":rotating_light: Regressed"} in *${appLabel(app, settings?.displayName)}*: ${link ? `<${link}|${title}>` : title}${where ? `\n${where}` : ""}`;
446
735
  try {
447
736
  await post(a.slackWebhookUrl, { text });
448
737
  sent++;
@@ -578,10 +867,7 @@ function countryOf(ctx) {
578
867
  if (tz) return tz;
579
868
  return /[-_]([A-Z]{2})(?:\b|$)/.exec(ctx.locale ?? "")?.[1] ?? "unknown";
580
869
  }
581
- // ../../packages/ingest/src/classify.ts
582
- var FOREGROUND = "App came to the foreground";
583
870
  var BACKGROUND = "App went to the background";
584
- var NEVER_SENT_MS = 100;
585
871
  /** How long after reopening a failure still counts as "on resume", if nothing has succeeded since. */
586
872
  var RESUME_WINDOW_MS = 1e4;
587
873
  var CRASH_SOURCES = /* @__PURE__ */ new Set(["global-handler", "uncaughtException"]);
@@ -639,7 +925,7 @@ function category(e) {
639
925
  const source = e.tags?.source;
640
926
  if (r && r.status === void 0) {
641
927
  if (TIMED_OUT.test(e.error.message)) return "network.timeout";
642
- if (r.durationMs !== void 0 && r.durationMs < NEVER_SENT_MS) return "network.never_sent";
928
+ if (r.durationMs !== void 0 && r.durationMs < 100) return "network.never_sent";
643
929
  return "network.failed";
644
930
  }
645
931
  if (r?.status !== void 0) {
@@ -654,11 +940,17 @@ function category(e) {
654
940
  if (!r) return "app_reported";
655
941
  return "other";
656
942
  }
943
+ /**
944
+ * A request that never left the phone (ABA-132): no status, too fast to have reached a server, not
945
+ * a timeout. Every one counts as the phone being offline, including the first of a burst, which
946
+ * has no failed calls before it to show it was one.
947
+ */
948
+ var isOfflineFailure = (labels) => labels.category === "network.never_sent";
657
949
  /** The last foreground/background breadcrumb before the error. */
658
950
  function lastLifecycle(crumbs) {
659
951
  for (let i = crumbs.length - 1; i >= 0; i--) {
660
952
  const b = crumbs[i];
661
- if (b.category === "ui" && (b.message === FOREGROUND || b.message === BACKGROUND)) return b;
953
+ if (b.category === "ui" && (b.message === "App came to the foreground" || b.message === "App went to the background")) return b;
662
954
  }
663
955
  }
664
956
  function gather(e, crumbs, repeatedFailure) {
@@ -672,7 +964,7 @@ function gather(e, crumbs, repeatedFailure) {
672
964
  r: crumbRequest(b)
673
965
  })).filter((x) => !!x.r);
674
966
  const life = lastLifecycle(crumbs);
675
- const foreground = life?.message === FOREGROUND ? Date.parse(life.ts) : void 0;
967
+ const foreground = life?.message === "App came to the foreground" ? Date.parse(life.ts) : void 0;
676
968
  const burst = requests.filter((x) => x.r.failed && within(x.b, 5e3));
677
969
  const hosts = new Set(burst.map((x) => hostOf(x.r.url)));
678
970
  if (e.request && (burst.length || repeatedFailure)) hosts.add(hostOf(e.request.url));
@@ -900,22 +1192,24 @@ function fingerprint(e) {
900
1192
  const f = topInAppFrame(e.error.frames);
901
1193
  const r = e.request;
902
1194
  const where = r ? `${r.method.toUpperCase()} ${r.url} ${r.status ? `${Math.floor(r.status / 100)}xx` : "failed"}` : f ? `${cleanFnName(f.fn) ?? "?"}@${normaliseFile(f.file)}` : normaliseMessage(e.error.message);
903
- return createHash("sha256").update([
1195
+ const parts = [
904
1196
  e.error.name,
905
1197
  where,
906
- e.flow ?? ""
907
- ].join("|")).digest("hex").slice(0, 24);
1198
+ e.flow ?? "",
1199
+ ...isTestEvent(e) ? [TEST_TAG] : []
1200
+ ];
1201
+ return createHash("sha256").update(parts.join("|")).digest("hex").slice(0, 24);
908
1202
  }
909
1203
  function titleOf(e) {
910
1204
  return `${e.error.name}: ${e.error.message}`.slice(0, 200);
911
1205
  }
912
1206
  // ../../packages/ingest/src/version.ts
913
1207
  /** This ingest's version: the package's, which every release bumps with the SDK's (one changeset group). */
914
- var INGEST_VERSION = "0.9.0";
1208
+ var INGEST_VERSION = "0.10.0";
915
1209
  /** Sent on every ingest response, so an SDK (or the CLI) can tell how old the ingest it talks to is. */
916
1210
  var VERSION_HEADER = "x-debugkit-version";
917
1211
  // ../../packages/ingest/src/index.ts
918
- /** "pk_abc:megaton-native,pk_def:other-app" → { pk_abc: "megaton-native", ... } */
1212
+ /** "pk_abc:megatron-native,pk_def:other-app" → { pk_abc: "megatron-native", ... } */
919
1213
  function parseIngestKeys(raw) {
920
1214
  const out = {};
921
1215
  for (const pair of (raw ?? "").split(",").map((s) => s.trim()).filter(Boolean)) {
@@ -1013,6 +1307,8 @@ async function ingest(rawBody, ingestKey, opts) {
1013
1307
  const settings = env.events.some((e) => e.type === "error") ? await cachedSettings(opts.store, app) : null;
1014
1308
  const criticalFlows = settings?.criticalFlows ?? [];
1015
1309
  const hosts = settings?.hosts ?? [];
1310
+ const errorsHere = env.events.filter((e) => e.type === "error");
1311
+ const onlyTestErrors = errorsHere.length > 0 && errorsHere.every((e) => isTestEvent(e));
1016
1312
  /** This batch's errors, to group into incidents once they're all labelled. */
1017
1313
  const batchErrors = [];
1018
1314
  for (const raw of env.events) {
@@ -1026,6 +1322,7 @@ async function ingest(rawBody, ingestKey, opts) {
1026
1322
  }
1027
1323
  const day = dayOf(event.ts);
1028
1324
  if (event.type === "session") {
1325
+ if (onlyTestErrors && (event.status === "errored" || event.status === "crashed")) continue;
1029
1326
  rollups.push({
1030
1327
  app,
1031
1328
  day,
@@ -1074,6 +1371,10 @@ async function ingest(rawBody, ingestKey, opts) {
1074
1371
  }));
1075
1372
  continue;
1076
1373
  }
1374
+ if (event.type !== "error" && isTestEvent(event)) {
1375
+ stored.push(storedEvent(day, event));
1376
+ continue;
1377
+ }
1077
1378
  const kind = event.type === "error" ? "error" : "event";
1078
1379
  const inc = (dimension, key) => rollups.push({
1079
1380
  app,
@@ -1086,6 +1387,7 @@ async function ingest(rawBody, ingestKey, opts) {
1086
1387
  let fp;
1087
1388
  let labels;
1088
1389
  let se;
1390
+ let offline = false;
1089
1391
  if (event.type === "error") {
1090
1392
  fp = fingerprint(event);
1091
1393
  labels = classify({
@@ -1095,9 +1397,19 @@ async function ingest(rawBody, ingestKey, opts) {
1095
1397
  criticalFlows,
1096
1398
  hosts
1097
1399
  });
1400
+ const test = isTestEvent(event);
1401
+ offline = !test && isOfflineFailure(labels);
1402
+ if (offline) {
1403
+ labels = {
1404
+ ...labels,
1405
+ category: "network.offline",
1406
+ owner: "unknown"
1407
+ };
1408
+ fp = OFFLINE_FINGERPRINT;
1409
+ }
1098
1410
  const cur = issues.get(fp);
1099
1411
  se = storedEvent(day, event, fp, labels);
1100
- batchErrors.push({
1412
+ if (!test && !offline) batchErrors.push({
1101
1413
  ts: raw.ts,
1102
1414
  labels,
1103
1415
  se,
@@ -1106,9 +1418,9 @@ async function ingest(rawBody, ingestKey, opts) {
1106
1418
  issues.set(fp, {
1107
1419
  app,
1108
1420
  fingerprint: fp,
1109
- title: titleOf(event),
1110
- culprit: culpritOf(event),
1111
- flow: event.flow,
1421
+ title: offline ? OFFLINE_TITLE : titleOf(event),
1422
+ culprit: offline ? void 0 : culpritOf(event),
1423
+ flow: offline ? void 0 : event.flow,
1112
1424
  by: (cur?.by ?? 0) + 1,
1113
1425
  ts: cur && cur.ts > event.ts ? cur.ts : event.ts,
1114
1426
  firstTs: cur && cur.firstTs < event.ts ? cur.firstTs : event.ts,
@@ -1117,7 +1429,6 @@ async function ingest(rawBody, ingestKey, opts) {
1117
1429
  });
1118
1430
  scopes.add(`issue:${fp}`);
1119
1431
  inc("issue", fp);
1120
- if (event.request) inc("endpoint", `${event.request.method.toUpperCase()} ${event.request.url}`);
1121
1432
  rollups.push({
1122
1433
  app,
1123
1434
  day: event.ts.slice(0, 13),
@@ -1126,9 +1437,14 @@ async function ingest(rawBody, ingestKey, opts) {
1126
1437
  kind,
1127
1438
  by: 1
1128
1439
  });
1440
+ if (test) {
1441
+ stored.push(se);
1442
+ continue;
1443
+ }
1444
+ if (event.request && !offline) inc("endpoint", `${event.request.method.toUpperCase()} ${event.request.url}`);
1129
1445
  }
1130
1446
  inc("total", "all");
1131
- inc("flow", event.flow ?? "untagged");
1447
+ if (!offline) inc("flow", event.flow ?? "untagged");
1132
1448
  inc("os", `${context.os.name} ${context.os.version ?? ""}`.trim());
1133
1449
  inc("appVersion", context.app.version);
1134
1450
  inc("deviceModel", context.device.model);
@@ -1141,7 +1457,7 @@ async function ingest(rawBody, ingestKey, opts) {
1141
1457
  inc("country", countryOf(context));
1142
1458
  if (labels) {
1143
1459
  inc("category", labels.category);
1144
- inc("owner", labels.owner);
1460
+ if (!offline) inc("owner", labels.owner);
1145
1461
  inc("impact", labels.impact);
1146
1462
  }
1147
1463
  }
@@ -1190,7 +1506,13 @@ async function ingest(rawBody, ingestKey, opts) {
1190
1506
  markUsers(opts.store, app, env.user?.id ?? env.user?.anonymousId ?? env.session.id, scopes, dayOf(receivedAt)),
1191
1507
  noteVersions(opts.store, app, env.sdk.version, receivedAt)
1192
1508
  ]);
1193
- if (outcome && (outcome.created.length || outcome.regressed.length)) await sendAlerts(app, outcome, issues, settings);
1509
+ const real = (fp) => !isOfflineIssue(issues.get(fp)) && !isTestIssue(issues.get(fp));
1510
+ const alerting = outcome && {
1511
+ ...outcome,
1512
+ created: outcome.created.filter(real),
1513
+ regressed: outcome.regressed.filter(real)
1514
+ };
1515
+ if (alerting && (alerting.created.length || alerting.regressed.length)) await sendAlerts(app, alerting, issues, settings);
1194
1516
  return {
1195
1517
  status: 202,
1196
1518
  body: {
@@ -1494,9 +1816,11 @@ var PASS_TO = {
1494
1816
  third_party: "whoever manages that third-party service",
1495
1817
  unknown: "app team first, to find out whose it is"
1496
1818
  };
1819
+ /** Nobody owns the phone's signal, but the app decides what happens without it (ABA-132). */
1820
+ var OFFLINE_PASS_TO = "Pass to: app team, to handle being offline. No endpoint or server is at fault.";
1497
1821
  var TEMPLATES = {
1498
1822
  "network.offline": {
1499
- saw: (n, are) => `${cap(n)} ${are === "are" ? "weren't" : "wasn't"} available`,
1823
+ saw: () => "Requests never left the phone",
1500
1824
  why: () => "The phone was offline.",
1501
1825
  diagnosis: () => "No connection when the request was made.",
1502
1826
  fix: () => "Check connectivity before sending (NetInfo), queue the request until the phone is back online, and show an offline state."
@@ -1511,7 +1835,7 @@ var TEMPLATES = {
1511
1835
  saw: (n) => `${cap(n)} took too long and gave up`,
1512
1836
  why: () => "The request was sent but no answer came back in time: a slow network or a slow server.",
1513
1837
  diagnosis: (_ev, took) => `Timed out${took ? ` after ${took}` : ""} with no response.`,
1514
- fix: () => "Check this endpoint's p95 under Response times. If the server is slow, fix it there; otherwise raise the timeout, retry idempotent requests, and show a retry state."
1838
+ fix: () => "Check this endpoint's p95 under Response times. If it's slow on every network, give that service's owner the timings and the times above; in the app, raise the timeout, retry idempotent requests, and show a retry state."
1515
1839
  },
1516
1840
  "network.failed": {
1517
1841
  saw: (n) => `${cap(n)} couldn't connect`,
@@ -1601,6 +1925,7 @@ function explain(input) {
1601
1925
  const e = ev0?.type === "error" ? ev0 : void 0;
1602
1926
  const ev = e ? evidenceOf(e) : NO_EVIDENCE;
1603
1927
  const t = TEMPLATES[labels.category] ?? TEMPLATES.other;
1928
+ const offline = labels.category === "network.offline";
1604
1929
  const causes = labels.causes;
1605
1930
  const model = input.lastEvent?.context.device.model;
1606
1931
  const service = e?.request ? hostNameFor(e.request.url, input.hosts)?.host.name : void 0;
@@ -1612,16 +1937,19 @@ function explain(input) {
1612
1937
  const action = userAction(e?.error.frames);
1613
1938
  const doing = action ? ` (after ${action.action})` : "";
1614
1939
  const causeSentences = causes.flatMap((c) => CAUSE_LINES[c]?.summary?.({ model }) ?? []);
1940
+ const said = !offline && (input.backend?.message || input.backend?.code) ? input.backend : void 0;
1941
+ const saidService = said?.service ?? service;
1942
+ const answer = said ? `${said.message ? `"${said.message}"` : ""}${said.code ? `${said.message ? " " : ""}(code ${said.code})` : ""}` : "";
1615
1943
  const summary = [
1616
- `${t.saw(s.name, s.plural ? "are" : "is")}${who}${where}${when}${doing}.`,
1617
- t.why(ev, causes),
1944
+ said ? `${saidService ? `The ${saidService} service` : "The server"} answered${said.status ? ` ${said.status}` : ""} ${answer}${who}${where}${when}${doing}.` : `${t.saw(s.name, s.plural ? "are" : "is")}${who}${where}${when}${doing}.`,
1945
+ said ? "That's the service's own reason: start there." : t.why(ev, causes),
1618
1946
  ...causeSentences,
1619
1947
  stillHappening(issue, rangeLabel, now),
1620
- `Pass to: ${PASS_TO[labels.owner] ?? PASS_TO.unknown}${service && (labels.owner === "backend" || labels.owner === "third_party") ? ` (${service})` : ""}.`
1948
+ offline ? OFFLINE_PASS_TO : said ? `Pass to: whoever runs ${saidService ? `the ${saidService} service` : "that service"}, with ${said.code ? `code ${said.code}` : "its answer"}${said.traceId ? ` and trace ${said.traceId}` : ""}.` : `Pass to: ${PASS_TO[labels.owner] ?? PASS_TO.unknown}${service && (labels.owner === "backend" || labels.owner === "third_party") ? ` (${service})` : ""}.`
1621
1949
  ].join(" ");
1622
1950
  const took = durationText(input.durations?.length ? input.durations : ev.durationMs !== void 0 ? [ev.durationMs] : []);
1623
- const evidence = [t.diagnosis(ev, took)];
1624
- if (e?.request) evidence.push(`Request: \`${e.request.method.toUpperCase()} ${e.request.url}\`${service ? ` (${service})` : ""}.`);
1951
+ const evidence = said ? [`HTTP ${said.status ?? ev.status ?? "error"}, and the backend's own answer: ${answer}${said.traceId ? `, trace \`${said.traceId}\`` : ""}.`, ...said.codes.length > 1 || (said.codes[0]?.count ?? 0) > 1 ? [`Backend codes in recent occurrences: ${said.codes.map((c) => `${c.key} × ${c.count}`).join(", ")}.`] : []] : [t.diagnosis(ev, took)];
1952
+ if (e?.request && !offline) evidence.push(`Request: \`${e.request.method.toUpperCase()} ${e.request.url}\`${service ? ` (${service})` : ""}.`);
1625
1953
  if (causes.includes("resumed_from_background") && ev.msSinceForeground !== void 0) evidence.push(`The app came to the foreground ${seconds(ev.msSinceForeground)} before; no request has succeeded since.`);
1626
1954
  if (ev.failedBefore > 0) evidence.push(`${plural$1(ev.failedBefore, "other request")} failed in the 5 s before${ev.failedHosts.length > 1 ? `, across ${ev.failedHosts.length} hosts` : ""}.`);
1627
1955
  else if (ev.repeatedFailure) evidence.push("The same request had already failed just before.");
@@ -1630,7 +1958,12 @@ function explain(input) {
1630
1958
  for (const c of causes) if (CAUSE_LINES[c]?.fix) evidence.push(`${CAUSE_INFO[c].name}: ${CAUSE_INFO[c].description}`);
1631
1959
  if (issue.incidentTitle) evidence.push(`Part of the incident "${issue.incidentTitle}".`);
1632
1960
  const whereParts = [issue.culprit ? `\`${issue.culprit}\`` : void 0, action ? `started by \`${action.fn}\` (${action.action})` : void 0].filter(Boolean);
1633
- const fixSteps = [t.fix(ev, causes), ...causes.flatMap((c) => CAUSE_LINES[c]?.fix ?? [])];
1961
+ const lookUp = said && [said.code && `code ${said.code}`, said.traceId && `trace ${said.traceId}`].filter(Boolean).join(" or ");
1962
+ const fixSteps = [
1963
+ said ? `Search the ${saidService ?? "backend"} logs${lookUp ? ` for ${lookUp}` : ""} at the times above: the service's answer (under The call that failed) says why it refused or failed.` : t.fix(ev, causes),
1964
+ ...said && labels.category === "auth" ? [t.fix(ev, causes)] : [],
1965
+ ...causes.flatMap((c) => CAUSE_LINES[c]?.fix ?? [])
1966
+ ];
1634
1967
  return {
1635
1968
  summary,
1636
1969
  technical: {
@@ -1641,8 +1974,12 @@ function explain(input) {
1641
1974
  }
1642
1975
  };
1643
1976
  }
1644
- /** The explanation as Markdown lines, without a title: for tickets and prefilled links. */
1645
- function explanationLines(x) {
1977
+ /**
1978
+ * The explanation as Markdown lines, without a title: for tickets and prefilled links. Specific
1979
+ * findings from the trail (ABA-146) come before the template's advice, which is labelled "General
1980
+ * advice" so it isn't mistaken for specific.
1981
+ */
1982
+ function explanationLines(x, findings = []) {
1646
1983
  return [
1647
1984
  "### What happened",
1648
1985
  x.summary,
@@ -1652,19 +1989,867 @@ function explanationLines(x) {
1652
1989
  "",
1653
1990
  ...x.technical.evidence.map((l) => `- ${l}`),
1654
1991
  ...x.technical.where ? ["", `**Where:** ${x.technical.where}`] : [],
1992
+ ...findings.length ? [
1993
+ "",
1994
+ "### What the trail shows",
1995
+ ...findings.map(findingLine)
1996
+ ] : [],
1655
1997
  "",
1656
- `**Suggested fix:** ${x.technical.fix}`
1998
+ `**General advice:** ${x.technical.fix}`
1657
1999
  ];
1658
2000
  }
2001
+ /** "- Why. Fix. (2 of 2 occurrences)" */
2002
+ var findingLine = (f) => `- ${f.why} ${f.fix}${f.of > 1 ? ` (${f.seen} of ${f.of} occurrences)` : ""}`;
1659
2003
  /** The explanation as Markdown, for pasting into a Jira or Linear ticket by hand. */
1660
- function explanationMarkdown(title, x, link) {
2004
+ function explanationMarkdown(title, x, link, findings = []) {
1661
2005
  return [
1662
2006
  `## ${title}`,
1663
2007
  ...link ? ["", `deBug: ${link}`] : [],
1664
2008
  "",
1665
- ...explanationLines(x)
2009
+ ...explanationLines(x, findings)
1666
2010
  ].join("\n");
1667
2011
  }
2012
+ // src/server/exchange.ts
2013
+ function sideOf(s) {
2014
+ return {
2015
+ headers: s.headers && Object.keys(s.headers).length ? s.headers : void 0,
2016
+ body: s.body,
2017
+ bodySize: s.bodySize,
2018
+ truncated: s.truncated || void 0
2019
+ };
2020
+ }
2021
+ /** Whether the SDK captured anything beyond the call itself: only with network.bodies on. */
2022
+ function captured(r, crumbs) {
2023
+ return r.query !== void 0 || !!r.headers || r.body !== void 0 || r.bodySize !== void 0 || crumbs.some((b) => b.category === "network" && b.data?.hasBodies === true);
2024
+ }
2025
+ /** The request and response the SDK captured on an error, or undefined when it captured none. */
2026
+ function exchangeOf(ev) {
2027
+ const r = ev.request;
2028
+ if (!r || !(captured(r, ev.breadcrumbs ?? []) || ev.response)) return void 0;
2029
+ const fullUrl = r.url && !r.url.startsWith("/") ? `https://${r.url}` : void 0;
2030
+ return {
2031
+ request: {
2032
+ method: r.method.toUpperCase(),
2033
+ url: r.url,
2034
+ fullUrl,
2035
+ query: r.query?.replace(/^\?/, "") || void 0,
2036
+ ...sideOf(r)
2037
+ },
2038
+ response: ev.response ? {
2039
+ status: ev.response.status,
2040
+ ...sideOf(ev.response)
2041
+ } : void 0
2042
+ };
2043
+ }
2044
+ // src/server/errors.ts
2045
+ /** Firestore answers FAILED_PRECONDITION while an index deBug needs is missing or still building. */
2046
+ function isIndexError(message) {
2047
+ return /FAILED_PRECONDITION|requires an index|index is currently building/i.test(message);
2048
+ }
2049
+ /** A thrown Firestore error saying a composite index is missing (gRPC code 9), on the Firestore store only. */
2050
+ function isMissingIndex(e, store) {
2051
+ if (store !== "firestore") return false;
2052
+ const err = e;
2053
+ return (err?.code === 9 || err?.code === "failed-precondition") && /index/i.test(String(err?.message ?? ""));
2054
+ }
2055
+ /** A fatal error: labelled a crash, or uncaught and critical, which is how the SDK sends a fatal one. */
2056
+ function isCrash(e) {
2057
+ return e.event.type === "error" && (e.labels?.category === "crash" || !e.event.handled && e.event.severity === "critical");
2058
+ }
2059
+ /** crashed, errored (an uncaught error) or clean, as on the session page. */
2060
+ function sessionOutcome(events) {
2061
+ if (events.some(isCrash)) return "crashed";
2062
+ return events.some((e) => e.event.type === "error" && !e.event.handled) ? "errored" : "clean";
2063
+ }
2064
+ /** The prop that says why a flow failed, in the order apps tend to name it. */
2065
+ var REASON_KEYS = [
2066
+ "reason",
2067
+ "error",
2068
+ "errorKind",
2069
+ "code",
2070
+ "message"
2071
+ ];
2072
+ var OUTCOME_WORD = {
2073
+ success: "succeeded",
2074
+ failure: "failed",
2075
+ abandoned: "abandoned"
2076
+ };
2077
+ /** "auth.login: failed, wrongOtp". */
2078
+ function describeEvent(name, outcome, props, step) {
2079
+ const reason = REASON_KEYS.map((k) => props?.[k]).find((v) => typeof v === "string" || typeof v === "number");
2080
+ const parts = [outcome ? OUTCOME_WORD[outcome] ?? outcome : void 0, reason !== void 0 ? String(reason) : void 0].filter(Boolean);
2081
+ return `${name}${step ? ` · ${step}` : ""}${parts.length ? `: ${parts.join(", ")}` : ""}`;
2082
+ }
2083
+ /**
2084
+ * Each breadcrumb once: errors in the same session carry overlapping trails. Given the events
2085
+ * oldest first, each breadcrumb's source is the earliest error that carried it, the one it led to.
2086
+ *
2087
+ * When the SDK merges a repeat it moves the breadcrumb's time to the latest repeat, so a later
2088
+ * trail can hold the same breadcrumb with a newer time and a higher count. Nothing else was
2089
+ * recorded between it and the breadcrumb before it, so an identical one in that window is the
2090
+ * same run: keep the copy with the highest count.
2091
+ */
2092
+ function uniqueCrumbs(events) {
2093
+ const out = [];
2094
+ const exact = /* @__PURE__ */ new Map();
2095
+ const count = (b) => Number(b.data?.repeated ?? 1);
2096
+ for (const e of events) {
2097
+ if (e.event.type !== "error") continue;
2098
+ const source = e.fingerprint ? {
2099
+ eventId: e.event.id,
2100
+ fingerprint: e.fingerprint
2101
+ } : void 0;
2102
+ let after = -Infinity;
2103
+ for (const b of e.event.breadcrumbs ?? []) {
2104
+ const t = Date.parse(b.ts);
2105
+ const same = (c) => c.crumb.category === b.category && c.crumb.message === b.message && c.crumb.level === b.level;
2106
+ const had = exact.get(`${b.ts}|${b.category}|${b.message}`) ?? (count(b) > 1 ? out.find((c) => same(c) && Date.parse(c.crumb.ts) > after && Date.parse(c.crumb.ts) <= t) : void 0);
2107
+ if (!had) {
2108
+ const c = {
2109
+ crumb: b,
2110
+ source
2111
+ };
2112
+ out.push(c);
2113
+ exact.set(`${b.ts}|${b.category}|${b.message}`, c);
2114
+ } else if (count(b) > count(had.crumb)) {
2115
+ had.crumb = b;
2116
+ exact.set(`${b.ts}|${b.category}|${b.message}`, had);
2117
+ }
2118
+ after = t;
2119
+ }
2120
+ }
2121
+ return out;
2122
+ }
2123
+ /** How far apart a custom event and its own breadcrumb, or an api.failed event and its call, can be. */
2124
+ var SAME_MS = 2e3;
2125
+ /** The SDK writes each custom event as an "event" breadcrumb too: the breadcrumb that is this event, if any. */
2126
+ function ownCrumb(name, ts, crumbs, taken) {
2127
+ const t = Date.parse(ts);
2128
+ return crumbs.find((c) => !taken.has(c) && c.crumb.category === "event" && c.crumb.message === name && Math.abs(Date.parse(c.crumb.ts) - t) <= SAME_MS);
2129
+ }
2130
+ /** The app's api.failed event says the same as the failed call's breadcrumb next to it. */
2131
+ function repeatsCrumb(e, crumbs) {
2132
+ if (e.event.type !== "event" || e.event.name !== "api.failed") return false;
2133
+ const path = e.event.props?.path;
2134
+ const t = Date.parse(e.event.ts);
2135
+ return crumbs.some((c) => c.api?.failed && c.api.path === path && Math.abs(Date.parse(c.ts) - t) <= SAME_MS);
2136
+ }
2137
+ async function buildSessionDetail(store, app, sessionId) {
2138
+ const [events, settings] = await Promise.all([store.listSessionEvents(app, sessionId, { limit: 501 }), store.getSettings(app).catch(() => null)]);
2139
+ if (!events.length) return null;
2140
+ const truncated = events.length > 500;
2141
+ const page = events.slice(-500);
2142
+ const hosts = settings?.hosts ?? [];
2143
+ const carried = withoutSdkTwinsCarried(uniqueCrumbs(page));
2144
+ const taken = /* @__PURE__ */ new Set();
2145
+ const own = /* @__PURE__ */ new Map();
2146
+ for (const e of page) {
2147
+ if (e.event.type !== "event") continue;
2148
+ const c = ownCrumb(e.event.name, e.event.ts, carried, taken);
2149
+ if (c) {
2150
+ taken.add(c);
2151
+ own.set(e, c);
2152
+ }
2153
+ }
2154
+ const kept = carried.filter((c) => !taken.has(c)).map((c) => ({
2155
+ ts: c.crumb.ts,
2156
+ item: readCrumb(c.crumb, hosts),
2157
+ source: c.source
2158
+ }));
2159
+ const crumbs = kept.map((c) => c.item);
2160
+ const rows = kept.map((c) => ({
2161
+ kind: "crumb",
2162
+ ...c
2163
+ }));
2164
+ let errors = 0;
2165
+ let handled = 0;
2166
+ for (const e of page) {
2167
+ const ev = e.event;
2168
+ if (ev.type === "error") {
2169
+ const crash = isCrash(e);
2170
+ errors++;
2171
+ if (ev.handled) handled++;
2172
+ rows.push({
2173
+ kind: "error",
2174
+ ts: ev.ts,
2175
+ eventId: ev.id,
2176
+ fingerprint: e.fingerprint,
2177
+ title: nameUrlsIn(`${ev.error.name}: ${ev.error.message}`, hosts),
2178
+ handled: ev.handled,
2179
+ crash
2180
+ });
2181
+ } else if (ev.type === "event" && !repeatsCrumb(e, crumbs)) rows.push({
2182
+ kind: "event",
2183
+ ts: ev.ts,
2184
+ eventId: ev.id,
2185
+ name: ev.name,
2186
+ outcome: ev.outcome,
2187
+ detail: describeEvent(ev.name, ev.outcome, ev.props, ev.step),
2188
+ failed: ev.outcome === "failure",
2189
+ source: own.get(e)?.source
2190
+ });
2191
+ }
2192
+ const order = {
2193
+ crumb: 0,
2194
+ event: 1,
2195
+ error: 2
2196
+ };
2197
+ const sorted = byTime(rows.sort((a, b) => order[a.kind] - order[b.kind]));
2198
+ const last = page[page.length - 1];
2199
+ return {
2200
+ app,
2201
+ sessionId,
2202
+ user: {
2203
+ id: last.user?.id,
2204
+ anonymousId: last.user?.anonymousId
2205
+ },
2206
+ context: last.context,
2207
+ start: sorted[0]?.ts ?? page[0].event.ts,
2208
+ end: last.event.ts,
2209
+ outcome: sessionOutcome(page),
2210
+ counts: {
2211
+ errors,
2212
+ handled,
2213
+ events: sorted.filter((r) => r.kind === "event").length
2214
+ },
2215
+ rows: sorted,
2216
+ truncated,
2217
+ read: page.length
2218
+ };
2219
+ }
2220
+ /** How far after an error its "what they did next" reaches. */
2221
+ var AFTER_MS = 3e5;
2222
+ /**
2223
+ * What the customer did after an error, from the trails of the session's later errors: the steps
2224
+ * within five minutes of it, at most `limit`. Empty when nothing later was stored or the session
2225
+ * can't be read (Firestore without its index).
2226
+ */
2227
+ async function stepsAfter(store, app, sessionId, ts, hosts = [], limit = 30) {
2228
+ const events = await store.listSessionEvents(app, sessionId, { limit: 500 }).catch((e) => {
2229
+ if (isIndexError(String(e?.message))) return [];
2230
+ throw e;
2231
+ });
2232
+ const t = Date.parse(ts);
2233
+ return byTime(withoutSdkTwinsCarried(uniqueCrumbs(events)).filter((c) => {
2234
+ const d = Date.parse(c.crumb.ts) - t;
2235
+ return d > 0 && d <= AFTER_MS;
2236
+ }).map((c) => c.crumb)).slice(0, limit).map((b) => readCrumb(b, hosts));
2237
+ }
2238
+ /** withoutSdkTwins over carried breadcrumbs, keeping each one's source. */
2239
+ function withoutSdkTwinsCarried(carried) {
2240
+ const keep = new Set(withoutSdkTwins(carried.map((c) => c.crumb)));
2241
+ return carried.filter((c) => keep.has(c.crumb));
2242
+ }
2243
+ var str = (v) => typeof v === "string" && v ? v : typeof v === "number" ? String(v) : void 0;
2244
+ var num = (v) => typeof v === "number" && Number.isFinite(v) ? v : typeof v === "string" && /^\d+$/.test(v) ? Number(v) : void 0;
2245
+ /** The first key present: the Cell C app's keys and their alternatives ("backend.title" or "backendTitle"). */
2246
+ var pick = (d, ...keys) => keys.map((k) => d[k]).find((v) => v !== void 0 && v !== null && v !== "");
2247
+ /** API keys read into the call; the rest of the data is shown as it is. */
2248
+ var API_KEYS = /* @__PURE__ */ new Set([
2249
+ "service",
2250
+ "method",
2251
+ "path",
2252
+ "url",
2253
+ "status",
2254
+ "durationMs",
2255
+ "outcome",
2256
+ "retry",
2257
+ "backend.title",
2258
+ "backendTitle",
2259
+ "backend.code",
2260
+ "backendCode",
2261
+ "backendErrors",
2262
+ "traceId",
2263
+ "trace_id",
2264
+ "errorKind"
2265
+ ]);
2266
+ /** Outcomes that mean the call did what it should. Anything else the app reports (unreachable, sessionExpired) is a failure. */
2267
+ var OK_OUTCOMES = /* @__PURE__ */ new Set([
2268
+ "answered",
2269
+ "cached",
2270
+ "cancelled",
2271
+ "canceled",
2272
+ "aborted",
2273
+ "success",
2274
+ "ok"
2275
+ ]);
2276
+ /** The SDK's own network breadcrumb: "POST api.x.com/v1/pay → 503 (120ms)", or "→ failed". */
2277
+ var SDK_NETWORK = /^([A-Z]+) (\S+) → (\d{3})?/;
2278
+ /** A network breadcrumb as an API call, or undefined when it isn't one ("Device went offline"). */
2279
+ function apiCallOf(b, hosts = []) {
2280
+ if (b.category !== "network") return void 0;
2281
+ const d = b.data ?? {};
2282
+ const sdk = SDK_NETWORK.exec(b.message);
2283
+ const url = str(pick(d, "path", "url")) ?? sdk?.[2];
2284
+ const method = str(pick(d, "method"))?.toUpperCase() ?? sdk?.[1];
2285
+ const appService = str(d.service);
2286
+ if (!url && !method && !appService) return void 0;
2287
+ const status = num(d.status) ?? num(sdk?.[3]);
2288
+ const outcome = str(d.outcome);
2289
+ const named = url && !appService ? hostNameFor(url, hosts) : void 0;
2290
+ const call = {
2291
+ service: appService ?? named?.host.name,
2292
+ method,
2293
+ path: named ? named.rest || "/" : url,
2294
+ status,
2295
+ durationMs: num(d.durationMs),
2296
+ outcome,
2297
+ retry: typeof d.retry === "boolean" || typeof d.retry === "number" ? d.retry : str(d.retry),
2298
+ backendTitle: str(pick(d, "backend.title", "backendTitle")),
2299
+ backendCode: str(pick(d, "backend.code", "backendCode")),
2300
+ backendErrors: str(d.backendErrors),
2301
+ errorKind: str(d.errorKind),
2302
+ traceId: str(pick(d, "traceId", "trace_id")),
2303
+ failed: false
2304
+ };
2305
+ call.failed = b.level === "error" || status !== void 0 && status >= 400 || outcome !== void 0 && !OK_OUTCOMES.has(outcome);
2306
+ return call;
2307
+ }
2308
+ /** How close in time the SDK's breadcrumb and the app's own for the same call are. */
2309
+ var TWIN_MS = 2e3;
2310
+ /**
2311
+ * The SDK's own network breadcrumbs that repeat an app breadcrumb for the same call (same method,
2312
+ * the SDK's URL ending in the app's path, within TWIN_MS): the app's says more, so the SDK's is
2313
+ * left out of the timeline.
2314
+ */
2315
+ function sdkTwins(crumbs) {
2316
+ const app = crumbs.filter((b) => b.category === "network" && str(b.data?.service) && str(b.data?.path));
2317
+ const out = /* @__PURE__ */ new Set();
2318
+ if (!app.length) return out;
2319
+ for (const b of crumbs) {
2320
+ const sdk = b.category === "network" && !b.data?.service ? SDK_NETWORK.exec(b.message) : null;
2321
+ if (!sdk) continue;
2322
+ const url = sdk[2].split(/[?#]/)[0];
2323
+ const t = Date.parse(b.ts);
2324
+ if (app.some((a) => str(a.data?.method)?.toUpperCase() === sdk[1] && url.endsWith(String(a.data.path)) && Math.abs(Date.parse(a.ts) - t) <= TWIN_MS)) out.add(b);
2325
+ }
2326
+ return out;
2327
+ }
2328
+ /** The breadcrumbs in time order (stable for equal times), each read for what it says. */
2329
+ function timelineOf(crumbs = [], hosts = []) {
2330
+ return byTime(withoutSdkTwins(crumbs)).map((b) => readCrumb(b, hosts));
2331
+ }
2332
+ /** Oldest first, stable for equal times. By time, not by string: "09:00:00Z" and "09:00:00.500Z" don't sort as text. */
2333
+ function byTime(items) {
2334
+ return items.map((x, i) => ({
2335
+ x,
2336
+ i
2337
+ })).sort((p, q) => Date.parse(p.x.ts) - Date.parse(q.x.ts) || p.i - q.i).map(({ x }) => x);
2338
+ }
2339
+ /** The breadcrumbs without the SDK's own copy of a call the app also described. */
2340
+ function withoutSdkTwins(crumbs) {
2341
+ const twins = sdkTwins(crumbs);
2342
+ return crumbs.filter((b) => !twins.has(b));
2343
+ }
2344
+ /** One breadcrumb read for what it says: the API call it describes, or its message. */
2345
+ function readCrumb(b, hosts = []) {
2346
+ const api = apiCallOf(b, hosts);
2347
+ const rest = Object.entries(b.data ?? {}).filter(([k]) => k !== "repeated" && !(api && API_KEYS.has(k)));
2348
+ const repeated = num(b.data?.repeated);
2349
+ return {
2350
+ ts: b.ts,
2351
+ category: b.category,
2352
+ message: nameUrlsIn(b.message, hosts),
2353
+ level: b.level,
2354
+ failed: api ? api.failed : b.level === "error",
2355
+ ...api ? { api } : {},
2356
+ ...rest.length ? { data: Object.fromEntries(rest) } : {},
2357
+ ...repeated && repeated > 1 ? { repeated } : {}
2358
+ };
2359
+ }
2360
+ /** In-app frames first, keeping stack order within each group. */
2361
+ function framesInAppFirst(frames = []) {
2362
+ return [...frames.filter((f) => f.inApp), ...frames.filter((f) => !f.inApp)];
2363
+ }
2364
+ async function buildEventDetail(store, app, fingerprint, eventId) {
2365
+ const [issue, events, settings] = await Promise.all([
2366
+ store.getIssue(app, fingerprint),
2367
+ store.listIssueEvents(app, fingerprint, { limit: 200 }),
2368
+ store.getSettings(app).catch(() => null)
2369
+ ]);
2370
+ if (!issue) return null;
2371
+ const errors = (events.length ? events : [issue.lastEvent]).filter((e) => e.event.type === "error");
2372
+ const index = errors.findIndex((e) => e.event.id === eventId);
2373
+ const stored = index >= 0 ? errors[index] : issue.lastEvent.event.id === eventId ? issue.lastEvent : void 0;
2374
+ if (!stored || stored.event.type !== "error") return null;
2375
+ const ev = stored.event;
2376
+ const hosts = settings?.hosts ?? [];
2377
+ const timeline = timelineOf(ev.breadcrumbs, hosts);
2378
+ return {
2379
+ app,
2380
+ issue: {
2381
+ fingerprint,
2382
+ title: nameUrlsIn(issue.title, hosts),
2383
+ status: issue.status ?? "open"
2384
+ },
2385
+ id: ev.id,
2386
+ ts: ev.ts,
2387
+ handled: ev.handled,
2388
+ error: {
2389
+ name: ev.error.name,
2390
+ message: nameUrlsIn(ev.error.message, hosts)
2391
+ },
2392
+ flow: ev.flow,
2393
+ step: ev.step,
2394
+ user: {
2395
+ id: stored.user?.id,
2396
+ anonymousId: stored.user?.anonymousId
2397
+ },
2398
+ sessionId: stored.sessionId,
2399
+ context: stored.context,
2400
+ tags: Object.entries(ev.tags ?? {}).map(([key, value]) => ({
2401
+ key,
2402
+ value: String(value)
2403
+ })).sort((a, b) => a.key.localeCompare(b.key)),
2404
+ timeline,
2405
+ request: failedRequest(ev.request, timeline, hosts),
2406
+ exchange: exchangeOf(ev),
2407
+ frames: framesInAppFirst(ev.error.frames),
2408
+ stack: ev.error.stack,
2409
+ after: await stepsAfter(store, app, stored.sessionId, ev.ts, hosts),
2410
+ newer: index > 0 ? errors[index - 1].event.id : void 0,
2411
+ older: index >= 0 && index < errors.length - 1 ? errors[index + 1].event.id : void 0
2412
+ };
2413
+ }
2414
+ /**
2415
+ * The call that failed: the error's own request, filled in from the app's breadcrumb for the same
2416
+ * call (service, backend code, traceId), or the last failing call when the error has no request.
2417
+ * A breadcrumb counts as the same call only when its method and path match.
2418
+ */
2419
+ function failedRequest(request, timeline, hosts = []) {
2420
+ const lastFailing = [...timeline].reverse().find((t) => t.api?.failed)?.api;
2421
+ if (!request) return lastFailing;
2422
+ const named = hostNameFor(request.url, hosts);
2423
+ const own = {
2424
+ service: named?.host.name,
2425
+ method: request.method.toUpperCase(),
2426
+ path: named ? named.rest || "/" : request.url,
2427
+ status: request.status,
2428
+ durationMs: request.durationMs,
2429
+ failed: true
2430
+ };
2431
+ const bare = request.url.split(/[?#]/)[0];
2432
+ const sameCall = (c) => c.method === own.method && !!c.path && (bare.endsWith(c.path) || c.path === own.path);
2433
+ const match = [...timeline].reverse().find((t) => t.api?.failed && sameCall(t.api))?.api;
2434
+ return match ? {
2435
+ ...match,
2436
+ ...defined(own),
2437
+ service: match.service ?? own.service,
2438
+ path: named ? own.path : match.path
2439
+ } : own;
2440
+ }
2441
+ /** Drop undefined fields, so spreading doesn't erase a value from the breadcrumb. */
2442
+ function defined(o) {
2443
+ return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== void 0));
2444
+ }
2445
+ // src/server/hops.ts
2446
+ /** Response headers that carry a trace or request id, best first. */
2447
+ var TRACE_HEADERS = [
2448
+ "traceparent",
2449
+ "x-cloud-trace-context",
2450
+ "x-amzn-trace-id",
2451
+ "x-request-id",
2452
+ "x-amzn-requestid",
2453
+ "x-correlation-id",
2454
+ "request-id"
2455
+ ];
2456
+ /** Body fields that carry one, as backends commonly name them. */
2457
+ var TRACE_FIELDS = [
2458
+ "traceId",
2459
+ "trace_id",
2460
+ "requestId",
2461
+ "request_id",
2462
+ "correlationId",
2463
+ "correlation_id"
2464
+ ];
2465
+ /** W3C traceparent: version, 32-hex trace id (not all zeros), parent id, flags. */
2466
+ var TRACEPARENT = /^[\da-f]{2}-(?!0{32})([\da-f]{32})-[\da-f]{16}-[\da-f]{2}$/i;
2467
+ /** The trace id inside a header value, or undefined when the value isn't one. */
2468
+ function traceIdIn(header, value) {
2469
+ if (header === "traceparent") return TRACEPARENT.exec(value.trim())?.[1];
2470
+ if (header === "x-cloud-trace-context") return value.split("/")[0] || void 0;
2471
+ if (header === "x-amzn-trace-id") return /(?:^|;)\s*Root=([^;]+)/.exec(value)?.[1];
2472
+ return value.trim() || void 0;
2473
+ }
2474
+ /** A trace id or error code in a JSON response body, top level only. */
2475
+ function idsInBody(body) {
2476
+ if (!body || !body.trimStart().startsWith("{")) return {};
2477
+ try {
2478
+ const json = JSON.parse(body);
2479
+ const text = (v) => typeof v === "string" && v && v !== "[redacted]" ? v : typeof v === "number" ? String(v) : void 0;
2480
+ return {
2481
+ traceId: TRACE_FIELDS.map((k) => text(json[k])).find(Boolean),
2482
+ code: text(json.code) ?? text(json.errorCode)
2483
+ };
2484
+ } catch {
2485
+ return {};
2486
+ }
2487
+ }
2488
+ /** The trace id the backend already sent: a response header, then the body, then the app's breadcrumb. */
2489
+ function traceOf(headers, body, call) {
2490
+ for (const h of TRACE_HEADERS) {
2491
+ const v = headers?.[h];
2492
+ const id = v && v !== "[redacted]" ? traceIdIn(h, v) : void 0;
2493
+ if (id) return {
2494
+ from: h,
2495
+ id
2496
+ };
2497
+ }
2498
+ const inBody = idsInBody(body).traceId;
2499
+ if (inBody) return {
2500
+ from: "traceId",
2501
+ id: inBody
2502
+ };
2503
+ if (call?.traceId) return {
2504
+ from: "traceId",
2505
+ id: call.traceId
2506
+ };
2507
+ }
2508
+ /** "08:31:44 UTC" for the logs, which are rarely in the phone's timezone. */
2509
+ function utcTime(ts) {
2510
+ const t = new Date(ts);
2511
+ return Number.isNaN(t.getTime()) ? ts : `${t.toISOString().slice(11, 19)} UTC`;
2512
+ }
2513
+ var callLine = (c, withService = true) => `${withService && c.service ? `${c.service} · ` : ""}${c.method ?? ""} ${c.path ?? ""}`.replace(/\s+/g, " ").trim();
2514
+ // src/server/detectors.ts
2515
+ /**
2516
+ * Specific fix advice from patterns in the customer's trail (ABA-146): rule-based, no AI. Each
2517
+ * finding says what happened, what to do, and the evidence it read, so it can be checked.
2518
+ *
2519
+ * No backend changes (ABA-137): a fix may tell the app team what to change, or say where to look
2520
+ * in the backend's logs (time, call, trace id). It never tells anyone to change their backend.
2521
+ *
2522
+ * Not detected: a call still running when the app crashed. Breadcrumbs are written when a call
2523
+ * ends, so one that never answered isn't in the trail (needs the SDK to record request starts).
2524
+ */
2525
+ var FINDING_IDS = [
2526
+ "repeat_failed",
2527
+ "duplicate_call",
2528
+ "called_before_login",
2529
+ "no_refresh_before_401",
2530
+ "hung_in_background",
2531
+ "expected_answer"
2532
+ ];
2533
+ /** Two sends of one call this close together are one action sending it twice. */
2534
+ var DUPLICATE_WINDOW_MS = 5e3;
2535
+ /** A success this recent, then a failure of the same call, is "the repeat failed". */
2536
+ var REPEAT_WINDOW_MS = 6e4;
2537
+ /** Longer than this in the background and a session token has usually expired. */
2538
+ var LONG_BACKGROUND_MS = 18e5;
2539
+ /** A timeout this short isn't a request left hanging. */
2540
+ var HUNG_MIN_MS = 5e3;
2541
+ /** At least this many calls, all with one status, before it looks like the endpoint's normal answer. */
2542
+ var EXPECTED_MIN_CALLS = 3;
2543
+ var ms = (ts) => Date.parse(ts);
2544
+ var isOk = (s) => s !== void 0 && s >= 200 && s < 300;
2545
+ /** "08:31:42.729 UTC", like the log search times (hops.ts utcTime). */
2546
+ var clock = (t) => Number.isNaN(t) ? "?" : `${new Date(t).toISOString().slice(11, 23)} UTC`;
2547
+ /** The path of a stored URL, without host, port, query or hash: "host:443/ppd/x?y" → "/ppd/x". */
2548
+ function pathOf(url) {
2549
+ const bare = url.split(/[?#]/)[0].replace(/^[a-z]+:\/\//i, "");
2550
+ const rest = hostOf$1(bare) ? bare.slice(bare.indexOf("/") < 0 ? bare.length : bare.indexOf("/")) : bare;
2551
+ return rest.startsWith("/") ? rest : `/${rest}`;
2552
+ }
2553
+ /**
2554
+ * Whether a breadcrumb's call is the error's endpoint. Same method; same host when the breadcrumb
2555
+ * names one; and the same path, or the app's path relative to its base URL ("/notifications/x"
2556
+ * under "/ppd/api") ending the request's path on whole segments, at least two of them.
2557
+ */
2558
+ function sameEndpoint(api, req) {
2559
+ if (!api?.path || api.method !== req.method.toUpperCase()) return false;
2560
+ const host = hostOf$1(api.path);
2561
+ if (host && host !== hostOf$1(req.url)) return false;
2562
+ const want = pathOf(req.url);
2563
+ const have = pathOf(api.path);
2564
+ if (have === want) return true;
2565
+ return !host && have.split("/").length >= 3 && want.endsWith(have);
2566
+ }
2567
+ /** When the call was sent: its end minus its duration (from the data, or the SDK's "(120ms)"). */
2568
+ function sentAt(t) {
2569
+ const took = t.api?.durationMs ?? Number(/\((\d+) ?ms\)/.exec(t.message)?.[1] ?? 0);
2570
+ return ms(t.ts) - took;
2571
+ }
2572
+ /** A login that finished: the app's "Login Completed" or "Logged in", or a 2xx from a login call. */
2573
+ var isLogin = (t) => t.category === "event" && /^(log ?in (completed|succeeded|success)|logged in)$/i.test(t.message.trim()) || !!t.api && isOk(t.api.status) && /\/(login|sign-?in)$/i.test(pathOf(t.api.path ?? ""));
2574
+ /** A token refresh that worked. */
2575
+ var isRefresh = (t) => !!t.api && isOk(t.api.status) && /\/(refresh|refresh-token|token\/refresh|oauth\/token)$/i.test(pathOf(t.api.path ?? ""));
2576
+ var isRetry = (c) => c?.retry === true || typeof c?.retry === "number" && c.retry > 0 || typeof c?.retry === "string" && /^[1-9]/.test(c.retry);
2577
+ function read(ev, hosts) {
2578
+ const at = ms(ev.ts);
2579
+ const trail = timelineOf(ev.breadcrumbs, hosts).filter((t) => !(ms(t.ts) > at));
2580
+ const sent = at - (ev.request?.durationMs ?? 0);
2581
+ if (!ev.request) return {
2582
+ ev,
2583
+ trail,
2584
+ earlier: [],
2585
+ sent
2586
+ };
2587
+ const req = ev.request;
2588
+ const same = trail.filter((t) => sameEndpoint(t.api, req));
2589
+ let i = same.length - 1;
2590
+ while (i >= 0 && !same[i].api.failed) i--;
2591
+ return {
2592
+ ev,
2593
+ trail,
2594
+ call: failedRequest(req, trail, hosts),
2595
+ own: same[i],
2596
+ earlier: i < 0 ? same : same.slice(0, i),
2597
+ sent
2598
+ };
2599
+ }
2600
+ var statusOf = (t) => t.api.status ?? t.api.outcome ?? "no answer";
2601
+ /** Whether one action sent the error's call more than once: the send before it is close, and it isn't the app's own retry. */
2602
+ function sentTwice(r) {
2603
+ if (isRetry(r.own?.api)) return [];
2604
+ return r.earlier.filter((t) => r.sent - sentAt(t) <= DUPLICATE_WINDOW_MS && !isRetry(t.api));
2605
+ }
2606
+ var DETECTORS = {
2607
+ repeat_failed: (r) => {
2608
+ const failed = r.ev.request?.status;
2609
+ if (!r.call || failed === void 0 || !(failed >= 500 || failed === 409)) return void 0;
2610
+ const ok = [...r.earlier].reverse().find((t) => isOk(t.api.status) && r.sent - sentAt(t) <= REPEAT_WINDOW_MS);
2611
+ if (!ok) return void 0;
2612
+ const response = exchangeOf(r.ev)?.response;
2613
+ const trace = traceOf(response?.headers, response?.body, r.call);
2614
+ const service = r.call.service ?? (hostOf$1(r.ev.request.url) || "backend");
2615
+ const where = trace ? `trace id ${trace.id} (${trace.from})` : `${callLine(r.call, false)} sent at ${utcTime(new Date(r.sent).toISOString())}`;
2616
+ const overlapped = r.sent < ms(ok.ts);
2617
+ const twice = sentTwice(r).length > 0;
2618
+ return {
2619
+ why: overlapped ? `The app sent it again before the first call answered; the first worked (${ok.api.status}), the repeat failed (${failed}).` : `The first call worked (${ok.api.status}); the repeat ${seconds(r.sent - sentAt(ok))} later failed (${failed}).`,
2620
+ fix: `${twice ? "App: send it once per action. " : ""}To see why the repeat failed, search the ${service} logs for ${where}.`,
2621
+ evidence: [`${clock(sentAt(ok))} sent, ${clock(ms(ok.ts))} → ${ok.api.status}`, `${clock(r.sent)} sent, ${clock(ms(r.ev.ts))} → ${failed}`]
2622
+ };
2623
+ },
2624
+ duplicate_call: (r) => {
2625
+ const close = sentTwice(r);
2626
+ if (!r.call || !close.length) return void 0;
2627
+ return {
2628
+ why: `The app sends ${callLine(r.call)} ${close.length + 1} times in ${seconds(r.sent - sentAt(close[0]))}.`,
2629
+ fix: "App: send it once per action. Look for two screens or effects that both trigger it.",
2630
+ evidence: [...close.map((t) => `${clock(sentAt(t))} sent → ${statusOf(t)}`), `${clock(r.sent)} sent → ${r.ev.request.status ?? "no answer"}`]
2631
+ };
2632
+ },
2633
+ called_before_login: (r) => {
2634
+ if (!r.call) return void 0;
2635
+ const denied = r.earlier.filter((t) => t.api.status === 401 || t.api.status === 403);
2636
+ if (!denied.length) return void 0;
2637
+ const lastDenied = ms(denied[denied.length - 1].ts);
2638
+ const login = r.trail.find((t) => ms(t.ts) > lastDenied && ms(t.ts) <= r.sent && isLogin(t));
2639
+ if (!login) return void 0;
2640
+ const codes = [...new Set(denied.map((t) => t.api.status))].join(" or ");
2641
+ return {
2642
+ why: `${callLine(r.call)} ran ${denied.length} time${denied.length === 1 ? "" : "s"} before login and got ${codes}.`,
2643
+ fix: "App: wait until login has finished before making this call.",
2644
+ evidence: [...denied.map((t) => `${clock(ms(t.ts))} → ${t.api.status}`), `${clock(ms(login.ts))} ${login.message}`]
2645
+ };
2646
+ },
2647
+ no_refresh_before_401: (r) => {
2648
+ if (!r.call || r.ev.request?.status !== 401) return void 0;
2649
+ const lifecycle = r.trail.filter((t) => t.message === "App came to the foreground" || t.message === "App went to the background");
2650
+ const fore = lifecycle[lifecycle.length - 1];
2651
+ const back = lifecycle[lifecycle.length - 2];
2652
+ if (fore?.message !== "App came to the foreground" || back?.message !== "App went to the background" || ms(fore.ts) - ms(back.ts) < LONG_BACKGROUND_MS) return void 0;
2653
+ if (r.trail.some((t) => ms(t.ts) >= ms(fore.ts) && isRefresh(t))) return void 0;
2654
+ return {
2655
+ why: `The app came back after ${Math.round((ms(fore.ts) - ms(back.ts)) / 6e4)} min in the background and called ${callLine(r.call)} without refreshing its token.`,
2656
+ fix: "App: refresh the session token when the app comes back, before other calls.",
2657
+ evidence: [
2658
+ `${clock(ms(back.ts))} ${back.message}`,
2659
+ `${clock(ms(fore.ts))} ${fore.message}`,
2660
+ `${clock(ms(r.ev.ts))} → 401`
2661
+ ]
2662
+ };
2663
+ },
2664
+ hung_in_background: (r) => {
2665
+ const req = r.ev.request;
2666
+ if (!r.call || !req || req.status !== void 0 || req.durationMs === void 0 || req.durationMs < HUNG_MIN_MS) return void 0;
2667
+ if (!/timed out/i.test(r.ev.error.message)) return void 0;
2668
+ const bg = r.trail.find((t) => t.message === "App went to the background" && ms(t.ts) >= r.sent && ms(t.ts) <= ms(r.ev.ts));
2669
+ if (!bg) return void 0;
2670
+ return {
2671
+ why: `${callLine(r.call)} was still running when the app went to the background, and gave up after ${seconds(req.durationMs)}.`,
2672
+ fix: "App: send it before the app goes to the background, or retry it when the app comes back.",
2673
+ evidence: [
2674
+ `${clock(r.sent)} sent (from its duration)`,
2675
+ `${clock(ms(bg.ts))} ${bg.message}`,
2676
+ `${clock(ms(r.ev.ts))} → no answer`
2677
+ ]
2678
+ };
2679
+ }
2680
+ };
2681
+ /**
2682
+ * A 4xx an endpoint gives on every call can be its normal answer ("no update", "not found"). A
2683
+ * 5xx every time is an outage, and 401, 403 and 429 mean something to act on: never "expected".
2684
+ */
2685
+ var MAYBE_EXPECTED = (s) => s >= 400 && s < 500 && s !== 401 && s !== 403 && s !== 429;
2686
+ /** Every call to the newest occurrence's endpoint, across these trails, answered the same 4xx. */
2687
+ function expectedAnswer(reads) {
2688
+ const head = reads.find((r) => r.call && r.ev.request?.status !== void 0);
2689
+ const status = head?.ev.request?.status;
2690
+ if (!head || status === void 0 || !MAYBE_EXPECTED(status)) return void 0;
2691
+ const calls = /* @__PURE__ */ new Map();
2692
+ let seen = 0;
2693
+ for (const r of reads) {
2694
+ const mine = r.trail.filter((t) => sameEndpoint(t.api, head.ev.request));
2695
+ if (mine.length) seen++;
2696
+ for (const t of mine) calls.set(`${t.ts}|${t.api.status}`, t);
2697
+ }
2698
+ const all = [...calls.values()];
2699
+ if (all.length < EXPECTED_MIN_CALLS || !all.every((t) => t.api.status === status)) return void 0;
2700
+ return {
2701
+ seen,
2702
+ draft: {
2703
+ why: `Every call to ${callLine(head.call)} in these trails answered ${status} (${all.length} of ${all.length}).`,
2704
+ fix: `If ${status} is this endpoint's normal answer, mark it as expected so it stops counting.`,
2705
+ evidence: all.slice(-5).map((t) => `${clock(ms(t.ts))} → ${t.api.status}`)
2706
+ }
2707
+ };
2708
+ }
2709
+ /**
2710
+ * What the trails of an issue's recent occurrences show, best first. Reads the newest
2711
+ * DETECTOR_LOOKBACK error events; each finding keeps the newest occurrence's wording and evidence
2712
+ * and counts how many of them show it. When the repeat-failed finding already says "send it
2713
+ * once", the duplicate-call finding isn't listed again.
2714
+ */
2715
+ function findingsOf(events, hosts = []) {
2716
+ const reads = [...events].sort((a, b) => ms(b.ts) - ms(a.ts)).slice(0, 20).map((ev) => read(ev, hosts));
2717
+ const found = /* @__PURE__ */ new Map();
2718
+ for (const r of reads) for (const [id, detect] of Object.entries(DETECTORS)) {
2719
+ const draft = detect(r);
2720
+ if (!draft) continue;
2721
+ const had = found.get(id);
2722
+ if (had) had.seen++;
2723
+ else found.set(id, {
2724
+ id,
2725
+ ...draft,
2726
+ seen: 1,
2727
+ of: reads.length
2728
+ });
2729
+ }
2730
+ if (found.get("repeat_failed")?.fix.startsWith("App: send it once")) found.delete("duplicate_call");
2731
+ const expected = expectedAnswer(reads);
2732
+ if (expected) found.set("expected_answer", {
2733
+ id: "expected_answer",
2734
+ ...expected.draft,
2735
+ seen: expected.seen,
2736
+ of: reads.length
2737
+ });
2738
+ return FINDING_IDS.flatMap((id) => found.has(id) ? [found.get(id)] : []);
2739
+ }
2740
+ function offlineBursts(events, hosts = []) {
2741
+ const errors = events.filter((e) => e.event.type === "error");
2742
+ const byPhone = /* @__PURE__ */ new Map();
2743
+ for (const e of errors) {
2744
+ const phone = e.user?.anonymousId ?? e.sessionId;
2745
+ byPhone.set(phone, [...byPhone.get(phone) ?? [], e]);
2746
+ }
2747
+ const bursts = [];
2748
+ for (const [phone, list] of byPhone) {
2749
+ const sorted = [...list].sort((a, b) => Date.parse(a.event.ts) - Date.parse(b.event.ts));
2750
+ let cur = [];
2751
+ const close = () => {
2752
+ if (!cur.length) return;
2753
+ const first = cur[0];
2754
+ const last = cur[cur.length - 1];
2755
+ const counts = /* @__PURE__ */ new Map();
2756
+ for (const e of cur) {
2757
+ const r = e.event.type === "error" ? e.event.request : void 0;
2758
+ const call = r ? `${r.method.toUpperCase()} ${nameUrl(r.url, hosts)}` : "unknown call";
2759
+ counts.set(call, (counts.get(call) ?? 0) + 1);
2760
+ }
2761
+ bursts.push({
2762
+ start: first.event.ts,
2763
+ end: last.event.ts,
2764
+ phone,
2765
+ os: `${last.context.os.name} ${last.context.os.version ?? ""}`.trim(),
2766
+ device: last.context.device.model,
2767
+ appVersion: last.context.app.version,
2768
+ calls: [...counts].map(([call, count]) => ({
2769
+ call,
2770
+ count
2771
+ })).sort((a, b) => b.count - a.count),
2772
+ failures: cur.length,
2773
+ airplaneMode: cur.some((e) => e.event.tags?.[AIRPLANE_TAG] === "true")
2774
+ });
2775
+ cur = [];
2776
+ };
2777
+ for (const e of sorted) {
2778
+ if (cur.length && Date.parse(e.event.ts) - Date.parse(cur[cur.length - 1].event.ts) > 6e4) close();
2779
+ cur.push(e);
2780
+ }
2781
+ close();
2782
+ }
2783
+ return bursts.sort((a, b) => b.start.localeCompare(a.start));
2784
+ }
2785
+ /** How many bursts on how many phones, for Health's Folded away line. */
2786
+ function burstSummary(bursts) {
2787
+ return {
2788
+ bursts: bursts.length,
2789
+ phones: new Set(bursts.map((b) => b.phone)).size,
2790
+ lastSeen: bursts[0]?.end
2791
+ };
2792
+ }
2793
+ /** The offline issue's occurrences in a range: [start, end]. */
2794
+ var inRange = (events, start, end) => events.filter((e) => e.event.ts >= start && e.event.ts <= end);
2795
+ // src/server/backend-said.ts
2796
+ /** A code or id from a JSON field: text or a number, kept to plain characters, else nothing. */
2797
+ var token = (v) => {
2798
+ const s = typeof v === "number" ? String(v) : typeof v === "string" ? v.trim() : "";
2799
+ return /^[\w.:-]{1,100}$/.test(s) ? s : void 0;
2800
+ };
2801
+ /**
2802
+ * What a gateway or framework says for any failure: not the service's own reason, so it doesn't
2803
+ * replace the explanation's diagnosis.
2804
+ */
2805
+ var GENERIC = /^(internal server error|not found|bad request|forbidden|unauthori[sz]ed|missing authentication token|service unavailable|bad gateway|gateway time-?out|endpoint request timed out|too many requests|error|an error occurred)\.?$/i;
2806
+ /**
2807
+ * A message from the backend, safe to quote in a ticket or Slack: text only, phone numbers and
2808
+ * email addresses masked, no quotes or Markdown characters, at most 200 characters.
2809
+ */
2810
+ var message = (v) => {
2811
+ if (typeof v !== "string") return void 0;
2812
+ const s = maskPersonal(v).replace(/[`*_<>[\]]/g, "").replace(/"/g, "'").replace(/\s+/g, " ").trim().slice(0, 200);
2813
+ return s && !GENERIC.test(s) ? s : void 0;
2814
+ };
2815
+ /** One occurrence's answer from the backend, or undefined when it gave none of its own. */
2816
+ function saidBy(se, hosts = []) {
2817
+ if (se.event.type !== "error") return void 0;
2818
+ const call = failedRequest(se.event.request, timelineOf(se.event.breadcrumbs, hosts), hosts);
2819
+ let body = {};
2820
+ try {
2821
+ const parsed = se.event.response?.body ? JSON.parse(se.event.response.body) : void 0;
2822
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) body = parsed;
2823
+ } catch {}
2824
+ const code = token(call?.backendCode) ?? token(body.code) ?? token(body.errorCode);
2825
+ const said = message(call?.backendTitle) ?? message(body.title) ?? message(body.message) ?? message(body.error);
2826
+ if (!code && !said) return void 0;
2827
+ return {
2828
+ status: se.event.response?.status ?? call?.status ?? se.event.request?.status,
2829
+ code,
2830
+ message: said,
2831
+ traceId: token(call?.traceId) ?? token(body.traceId) ?? token(body.trace_id),
2832
+ service: call?.service
2833
+ };
2834
+ }
2835
+ /**
2836
+ * What the backend said across an issue's occurrences (newest first): the newest answer that had
2837
+ * its own code or message, and every code with how often it came back. undefined when it never
2838
+ * gave one.
2839
+ */
2840
+ function backendSaid(events, hosts = []) {
2841
+ const said = events.flatMap((e) => saidBy(e, hosts) ?? []);
2842
+ if (!said.length) return void 0;
2843
+ const counts = /* @__PURE__ */ new Map();
2844
+ for (const s of said) if (s.code) counts.set(s.code, (counts.get(s.code) ?? 0) + 1);
2845
+ return {
2846
+ ...said[0],
2847
+ codes: [...counts].map(([key, count]) => ({
2848
+ key,
2849
+ count
2850
+ })).sort((a, b) => b.count - a.count)
2851
+ };
2852
+ }
1668
2853
  // src/server/range.ts
1669
2854
  /** The date range a dashboard view covers, worked out from the URL (range=7d, or from/to). */
1670
2855
  var RANGE_PRESETS = [
@@ -1722,6 +2907,17 @@ function resolveRange(input, now = Date.now()) {
1722
2907
  granularity: preset === "24h" ? "hour" : days > 90 ? "week" : "day"
1723
2908
  };
1724
2909
  }
2910
+ /** The range as ISO instants: the last 24 hours for "24h", else whole UTC days from `from` to `to`. */
2911
+ function rangeBounds(range, now = Date.now()) {
2912
+ if (range.preset === "24h") return {
2913
+ start: (/* @__PURE__ */ new Date(now - 864e5)).toISOString(),
2914
+ end: new Date(now).toISOString()
2915
+ };
2916
+ return {
2917
+ start: `${range.from}T00:00:00.000Z`,
2918
+ end: `${range.to}T23:59:59.999Z`
2919
+ };
2920
+ }
1725
2921
  /** Every UTC day from `from` to `to`, inclusive. */
1726
2922
  function daysBetween(from, to) {
1727
2923
  const out = [];
@@ -1760,11 +2956,15 @@ var SUMMARY_FIELDS = {
1760
2956
  "totals.errors24h": "Error events in the trailing 24 hours (exact, from hourly rollups).",
1761
2957
  "totals.errorsRange": "Error events in the selected range.",
1762
2958
  "totals.changePct": "Percent change of errors vs the previous period of the same length. null when that period had fewer than 10 errors.",
1763
- "totals.activeIssues": "Distinct grouped errors (issues) seen in the range.",
2959
+ "totals.activeIssues": "Distinct grouped errors (issues) seen in the range, without test errors.",
2960
+ "totals.testIssues": "Issues made on purpose to test deBug (tag test=true). Left out of every count; list them with ?tests=1.",
2961
+ connectivity: "Phones going offline (requests that never left the phone), as bursts per phone, shown on Health as Folded away. Not counted against any endpoint or owner. null when there were none.",
1764
2962
  "totals.criticalIssues": "Issues whose 24h occurrence count meets the critical threshold.",
1765
2963
  "flows[]": "Parts of the app as tagged by the developer (e.g. auth.login). Sorted by severity, then errors24h, then errorsRange.",
1766
2964
  "flows[].failureRate": "failures/(failures+successes) in the range from events logged with an outcome. null when the app logs no outcomes for that flow.",
1767
- "issues[]": "Grouped errors seen in the range. Sorted by severity, then count24h, then countRange. Narrow with ?status=open|resolved|ignored|all (default all), ?flow=, ?category=, ?owner= and ?impact= (ids from breakdowns); an unknown value is a 400. _filter echoes what was applied.",
2965
+ "issues[]": "Grouped errors seen in the range. Sorted by severity, then count24h, then countRange. Narrow with ?status=open|resolved|ignored|all (default all), ?flow=, ?category=, ?owner= and ?impact= (ids from breakdowns); an unknown value is a 400. Test errors are left out unless ?tests=1. _filter echoes what was applied.",
2966
+ "issues[].offline": "true for the one issue that collects offline bursts (calls that never left the phone). On Health's Folded away line, left out of this list unless ?category=network.offline, and never counted as active.",
2967
+ "issues[].test": "true for an issue made on purpose to test deBug (tag test=true): listed only with ?tests=1, never counted, always severity low.",
1768
2968
  "issues[].severity": "Derived from count24h using thresholds critical/high/medium.",
1769
2969
  "issues[].title": "Error name and message, with hosts named in Settings shown by their names.",
1770
2970
  "issues[].culprit": "Top in-app stack frame: function (file:line).",
@@ -1861,14 +3061,15 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
1861
3061
  let errors24h = 0;
1862
3062
  let errorsPrev24h = 0;
1863
3063
  const issue24h = /* @__PURE__ */ new Map();
1864
- for (const r of hourRows) if (last24.has(r.day)) {
1865
- errors24h += r.errors;
1866
- issue24h.set(r.key, (issue24h.get(r.key) ?? 0) + r.errors);
1867
- } else errorsPrev24h += r.errors;
3064
+ const testIssues = new Set(issues.filter(isTestIssue).map((i) => i.fingerprint));
3065
+ const realHours = hourRows.filter((r) => !testIssues.has(r.key));
3066
+ for (const r of hourRows) if (last24.has(r.day)) issue24h.set(r.key, (issue24h.get(r.key) ?? 0) + r.errors);
3067
+ for (const r of realHours) if (last24.has(r.day)) errors24h += r.errors;
3068
+ else errorsPrev24h += r.errors;
1868
3069
  let trend;
1869
3070
  if (hourly) trend = h24.map((h) => ({
1870
3071
  day: h,
1871
- errors: hourRows.filter((r) => r.day === h).reduce((a, r) => a + r.errors, 0),
3072
+ errors: realHours.filter((r) => r.day === h).reduce((a, r) => a + r.errors, 0),
1872
3073
  events: 0
1873
3074
  }));
1874
3075
  else {
@@ -1898,22 +3099,37 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
1898
3099
  const errorsPrev = hourly ? errorsPrev24h : sumRows(prevTotal);
1899
3100
  const issueRange = /* @__PURE__ */ new Map();
1900
3101
  for (const r of issueDays) issueRange.set(r.key, (issueRange.get(r.key) ?? 0) + r.errors);
1901
- const rangeStart = hourly ? (/* @__PURE__ */ new Date(now - DAY)).toISOString() : `${from}T00:00:00.000Z`;
3102
+ const { start: rangeStart, end: rangeEnd } = rangeBounds(range, now);
1902
3103
  const versionsInRange = [...new Set(sessionRows.map((r) => r.key.split("|")[0]).filter((v) => v && v !== "unknown"))].sort(compareVersions$1);
1903
3104
  const latestVersion = versionsInRange[versionsInRange.length - 1] ?? null;
1904
- const issueRows = issues.filter((i) => i.lastSeen >= rangeStart && i.firstSeen <= `${to}T23:59:59.999Z`).map((i) => toIssueRow(i, issue24h.get(i.fingerprint) ?? 0, hourly ? issue24h.get(i.fingerprint) ?? 0 : issueRange.get(i.fingerprint) ?? 0, thresholds, latestVersion, criticalFlows, hosts)).sort(bySeverityThenCount);
1905
- const active = issueRows.filter((i) => i.status !== "ignored" && i.status !== "resolved");
3105
+ const issueRows = issues.filter((i) => i.lastSeen >= rangeStart && i.firstSeen <= rangeEnd).map((i) => toIssueRow(i, issue24h.get(i.fingerprint) ?? 0, hourly ? issue24h.get(i.fingerprint) ?? 0 : issueRange.get(i.fingerprint) ?? 0, thresholds, latestVersion, criticalFlows, hosts)).sort(bySeverityThenCount);
3106
+ const active = issueRows.filter((i) => !i.test && !i.offline && i.status !== "ignored" && i.status !== "resolved");
1906
3107
  await Promise.all(active.slice(0, 10).map(async (i) => i.users = await countUsersSafe(store, app, `issue:${i.fingerprint}`, usersSince)));
1907
3108
  const lastEvents = new Map(issues.map((i) => [i.fingerprint, i.lastEvent]));
1908
- for (const row of issueRows) if (row.labels) row.explanation = explain({
1909
- issue: row,
1910
- labels: row.labels,
1911
- lastEvent: lastEvents.get(row.fingerprint),
1912
- users: row.users,
1913
- hosts,
1914
- rangeLabel: range.label,
1915
- now
1916
- });
3109
+ const offlineRow = issueRows.find((i) => i.offline);
3110
+ let connectivity = null;
3111
+ if (offlineRow) {
3112
+ const events = await store.listIssueEvents(app, offlineRow.fingerprint, { limit: 200 });
3113
+ connectivity = {
3114
+ fingerprint: offlineRow.fingerprint,
3115
+ ...burstSummary(offlineBursts(inRange(events, rangeStart, rangeEnd), hosts)),
3116
+ failures: offlineRow.countRange,
3117
+ capped: events.length >= 200
3118
+ };
3119
+ }
3120
+ for (const row of issueRows) if (row.labels) {
3121
+ const last = lastEvents.get(row.fingerprint);
3122
+ row.explanation = explain({
3123
+ issue: row,
3124
+ labels: row.labels,
3125
+ lastEvent: last,
3126
+ users: row.users,
3127
+ hosts,
3128
+ rangeLabel: range.label,
3129
+ backend: last ? backendSaid([last], hosts) : void 0,
3130
+ now
3131
+ });
3132
+ }
1917
3133
  const incidents = await buildIncidents(incidentRows, incidentIssueRows, issueRows, hourly, (key) => countUsersSafe(store, app, `incident:${key}`, usersSince));
1918
3134
  const firstDay = /* @__PURE__ */ new Map();
1919
3135
  for (const r of releaseRows) {
@@ -1987,12 +3203,14 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
1987
3203
  eventsRange: sumRows(total, "events"),
1988
3204
  activeIssues: active.length,
1989
3205
  criticalIssues: active.filter((i) => i.severity === "critical").length,
3206
+ testIssues: issueRows.filter((i) => i.test).length,
1990
3207
  failingFlows: flows.filter((f) => (hourly ? f.errors24h : f.errorsRange) > 0).length
1991
3208
  },
1992
3209
  narrative: [],
1993
3210
  trend,
1994
3211
  flows,
1995
3212
  issues: issueRows,
3213
+ connectivity,
1996
3214
  incidents,
1997
3215
  breakdowns: {
1998
3216
  os: top(os),
@@ -2226,7 +3444,9 @@ function issueLabels(i, latestVersion, criticalFlows = [], hosts = []) {
2226
3444
  });
2227
3445
  }
2228
3446
  function toIssueRow(i, count24h, countRange, t, latestVersion = null, criticalFlows = [], hosts = []) {
2229
- const base = severityFromCount(count24h, t);
3447
+ const test = isTestIssue(i);
3448
+ const offline = isOfflineIssue(i);
3449
+ const base = test || offline ? "low" : severityFromCount(count24h, t);
2230
3450
  const critical = new Set(criticalFlows);
2231
3451
  return {
2232
3452
  labels: issueLabels(i, latestVersion, criticalFlows, hosts),
@@ -2241,12 +3461,14 @@ function toIssueRow(i, count24h, countRange, t, latestVersion = null, criticalFl
2241
3461
  title: nameUrlsIn(i.title, hosts),
2242
3462
  culprit: i.culprit,
2243
3463
  flow: i.flow,
2244
- severity: i.flow && critical.has(i.flow) && count24h > 0 ? BUMP[base] : base,
3464
+ severity: !test && i.flow && critical.has(i.flow) && count24h > 0 ? BUMP[base] : base,
2245
3465
  count24h,
2246
3466
  countRange,
2247
3467
  countTotal: i.count,
2248
3468
  firstSeen: i.firstSeen,
2249
- lastSeen: i.lastSeen
3469
+ lastSeen: i.lastSeen,
3470
+ test,
3471
+ offline
2250
3472
  };
2251
3473
  }
2252
3474
  var bySeverityThenCount = (a, b) => SEVERITY_ORDER[b.severity] - SEVERITY_ORDER[a.severity] || b.count24h - a.count24h || b.countRange - a.countRange || b.lastSeen.localeCompare(a.lastSeen);
@@ -2269,7 +3491,7 @@ function narrate(s) {
2269
3491
  const n = r.preset === "24h" ? f.errors24h : f.errorsRange;
2270
3492
  out.push(`${f.flow}${f.critical ? " (a critical flow)" : ""} is the most affected flow (${f.severity}): ${plural(n, "error")} in ${r.label} from ${plural(f.issues, "issue")}.${failRate}`);
2271
3493
  }
2272
- const i = s.issues.find((x) => x.status !== "ignored" && x.status !== "resolved");
3494
+ const i = s.issues.find((x) => !x.test && !x.offline && x.status !== "ignored" && x.status !== "resolved");
2273
3495
  if (i && i.countRange > 0) {
2274
3496
  const isNew = Date.parse(s.generatedAt) - Date.parse(i.firstSeen) < 3 * DAY;
2275
3497
  out.push(`Top issue: "${i.title}"${i.culprit ? ` in ${i.culprit}` : ""}, ${plural(i.countRange, "time")} in ${r.label}${isNew ? `, first seen ${relDays(s.generatedAt, i.firstSeen)}` : ""}.`);
@@ -2295,6 +3517,41 @@ function relDays(now, then) {
2295
3517
  const d = Math.floor((Date.parse(now) - Date.parse(then)) / DAY);
2296
3518
  return d === 0 ? "today" : d === 1 ? "yesterday" : `${d} days ago`;
2297
3519
  }
3520
+ /**
3521
+ * The failed call to show on an issue's page, from occurrences newest first. `latest` is the
3522
+ * newest (what's happening now: during an outage, a timeout with no answer); when it captured
3523
+ * nothing, the newest that did. `lastAnswer` is the newest the server answered, when that's not
3524
+ * `latest`: what the server said last time it said anything.
3525
+ */
3526
+ function callsOf(events, hosts) {
3527
+ const calls = events.flatMap((se) => {
3528
+ if (se.event.type !== "error") return [];
3529
+ const api = failedRequest(se.event.request, timelineOf(se.event.breadcrumbs, hosts), hosts);
3530
+ return api ? [{
3531
+ api,
3532
+ exchange: exchangeOf(se.event),
3533
+ eventId: se.event.id,
3534
+ ts: se.event.ts
3535
+ }] : [];
3536
+ });
3537
+ const latest = calls[0]?.exchange ? calls[0] : calls.find((c) => c.exchange) ?? calls[0];
3538
+ const answered = calls.find((c) => c.exchange?.response);
3539
+ return {
3540
+ latest,
3541
+ lastAnswer: answered && answered !== latest && !latest?.exchange?.response ? answered : void 0
3542
+ };
3543
+ }
3544
+ /** The service whose call an issue's newest occurrence failed, or the app's own code. */
3545
+ function areaOf(se, labels, hosts) {
3546
+ if (!labels || !isRequestCategory(labels.category)) return { kind: "app" };
3547
+ const e = se?.event.type === "error" ? se.event : void 0;
3548
+ if (!e) return null;
3549
+ const name = failedRequest(e.request, timelineOf(e.breadcrumbs, hosts), hosts)?.service ?? (e.request ? hostNameFor(e.request.url, hosts)?.host.name : void 0);
3550
+ return name ? {
3551
+ kind: "service",
3552
+ name
3553
+ } : null;
3554
+ }
2298
3555
  async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.now(), opts = {}) {
2299
3556
  const issue = await store.getIssue(app, fingerprint);
2300
3557
  if (!issue) return null;
@@ -2344,6 +3601,7 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
2344
3601
  if (row.labels) row.labels = withIssueCauses(row.labels, { deviceModels: events.map((e) => e.context.device.model) });
2345
3602
  const hints = hintsFor(issue, breakdowns, events, !!row.labels && isRequestCategory(row.labels.category), hosts);
2346
3603
  const errorEvents = events.filter((e) => e.event.type === "error");
3604
+ const backend = row.labels && isRequestCategory(row.labels.category) ? backendSaid(errorEvents.length ? errorEvents : issue.lastEvent ? [issue.lastEvent] : [], hosts) : void 0;
2347
3605
  row.explanation = row.labels ? explain({
2348
3606
  issue: row,
2349
3607
  labels: row.labels,
@@ -2353,10 +3611,17 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
2353
3611
  platforms: [...new Set(errorEvents.map((e) => e.context.os.name))],
2354
3612
  durations: errorEvents.flatMap((e) => e.event.type === "error" && e.event.request?.durationMs !== void 0 ? [e.event.request.durationMs] : []),
2355
3613
  rangeLabel: range.label,
3614
+ backend,
2356
3615
  now
2357
3616
  }) : void 0;
3617
+ const offline = row.offline;
3618
+ const findings = offline ? [] : findingsOf(errorEvents.flatMap((e) => e.event.type === "error" ? [e.event] : []), hosts);
2358
3619
  const link = issueUrl(opts.trackers?.dashboardUrl, app, fingerprint);
3620
+ const bounds = rangeBounds(range, now);
3621
+ const inPeriod = errorEvents.filter((e) => e.event.ts >= bounds.start && e.event.ts <= bounds.end);
3622
+ const calls = row.labels && isRequestCategory(row.labels.category) ? callsOf(inPeriod.length ? inPeriod : errorEvents.length ? errorEvents : [issue.lastEvent], hosts) : void 0;
2359
3623
  const detail = {
3624
+ area: row.test ? { kind: "tests" } : row.offline ? null : areaOf(issue.lastEvent, row.labels, hosts),
2360
3625
  app,
2361
3626
  range,
2362
3627
  retention,
@@ -2367,6 +3632,8 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
2367
3632
  affectedUsers,
2368
3633
  links: {},
2369
3634
  sample: events[0] ?? issue.lastEvent ?? null,
3635
+ calls,
3636
+ backend,
2370
3637
  recent: events.slice(0, 25).map((e) => ({
2371
3638
  id: e.event.id,
2372
3639
  type: e.event.type,
@@ -2378,7 +3645,12 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
2378
3645
  userId: e.user?.id
2379
3646
  })),
2380
3647
  hints,
2381
- ticketMarkdown: row.explanation ? explanationMarkdown(row.title, row.explanation, link) : void 0
3648
+ findings,
3649
+ ...offline ? {
3650
+ bursts: offlineBursts(inRange(events, bounds.start, bounds.end), hosts),
3651
+ burstsCapped: events.length >= 200
3652
+ } : {},
3653
+ ticketMarkdown: row.explanation ? explanationMarkdown(row.title, row.explanation, link, findings) : void 0
2382
3654
  };
2383
3655
  detail.links = trackerLinks(detail, opts.trackers ?? {});
2384
3656
  return detail;
@@ -2406,6 +3678,11 @@ function issueStory(d, dashboardUrl) {
2406
3678
  "Stack (in-app frames):",
2407
3679
  ...frames.map((f) => `- \`${f.fn ?? "?"}\` ${f.file ?? ""}:${f.line ?? ""}`)
2408
3680
  ] : [],
3681
+ ...d.findings.length ? [
3682
+ "",
3683
+ "What the trail shows:",
3684
+ ...d.findings.slice(0, 3).map(findingLine)
3685
+ ] : [],
2409
3686
  ...d.hints.length ? [
2410
3687
  "",
2411
3688
  x ? "Also worth checking:" : "What to look at:",
@@ -2482,15 +3759,4 @@ function hintsFor(issue, b, events, explained = false, hosts = []) {
2482
3759
  });
2483
3760
  return hints;
2484
3761
  }
2485
- // src/server/errors.ts
2486
- /** Firestore answers FAILED_PRECONDITION while an index deBug needs is missing or still building. */
2487
- function isIndexError(message) {
2488
- return /FAILED_PRECONDITION|requires an index|index is currently building/i.test(message);
2489
- }
2490
- /** A thrown Firestore error saying a composite index is missing (gRPC code 9), on the Firestore store only. */
2491
- function isMissingIndex(e, store) {
2492
- if (store !== "firestore") return false;
2493
- const err = e;
2494
- return (err?.code === 9 || err?.code === "failed-precondition") && /index/i.test(String(err?.message ?? ""));
2495
- }
2496
- export { seenHosts as A, ingest as C, teamAction as D, hashInviteToken as E, writeRetention as F, writeSettings as I, getLocalWriteStore as L, testAlert as M, validRetention as N, readSettings as O, writeIssueStatus as P, getStore as R, CORS_HEADERS as S, isRequestCategory as T, hashPassword as _, buildSummary as a, startSession as b, resolveRange as c, canManage as d, checkCredentials as f, forgetMember as g, envOwnerEmails as h, buildIssueDetail as i, settingsWriter as j, restampStep as k, DEMO_USER as l, endSession as m, isMissingIndex as n, endpointTimings as o, demoLoginEnabled as p, SUMMARY_FIELDS as r, retentionFor as s, isIndexError as t, authorizeApi as u, readSession as v, parseIngestKeys as w, tooManyAttempts as x, recordFailure as y, config as z };
3762
+ export { testAlert as $, parseIngestKeys as A, endpointsOf as B, hashPassword as C, tooManyAttempts as D, startSession as E, LIVE_MS as F, serviceTile as G, isOpen as H, OUTAGE as I, appNames as J, servicesOf as K, answerOf as L, hashInviteToken as M, teamAction as N, CORS_HEADERS as O, HEALTH_ORDER as P, settingsWriter as Q, byHealth as R, forgetMember as S, recordFailure as T, pct$1 as U, failing as V, plural$2 as W, restampStep as X, readSettings as Y, seenHosts as Z, canManage as _, retentionFor as a, getLocalWriteStore as at, endSession as b, buildEventDetail as c, buildSessionDetail as d, validRetention as et, sessionOutcome as f, authorizeApi as g, DEMO_USER as h, endpointTimings as i, clearStoreCache as it, isRequestCategory as j, ingest as k, failedRequest as l, isMissingIndex as m, buildIssueDetail as n, writeRetention as nt, resolveRange as o, getStore as ot, isIndexError as p, whoseLine as q, buildSummary as r, writeSettings as rt, apiCallOf as s, SUMMARY_FIELDS as t, writeIssueStatus as tt, timelineOf as u, checkCredentials as v, readSession as w, envOwnerEmails as x, demoLoginEnabled as y, endpointLevel as z };