@debugkit/dashboard 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/client/assets/_authed-3se5Egqg.js +1 -0
  2. package/dist/client/assets/_authed-BfFWbTOO.js +1 -0
  3. package/dist/client/assets/app-code-BKWbvUMR.js +1 -0
  4. package/dist/client/assets/closed-issues-Cl-8l4oR.js +1 -0
  5. package/dist/client/assets/{copy-button-ORgLIh59.js → copy-button-BwRd5q7l.js} +1 -1
  6. package/dist/client/assets/customers-Cf-R8BOg.js +1 -0
  7. package/dist/client/assets/empty-state-njgr1sWu.js +7 -0
  8. package/dist/client/assets/endpoints-BMqAUNyE.js +1 -0
  9. package/dist/client/assets/events._eventId-CVRZJ-bU.js +2 -0
  10. package/dist/client/assets/exchange-view-DVwSLcXF.js +2 -0
  11. package/dist/client/assets/index-DEnCDILF.js +10 -0
  12. package/dist/client/assets/{invite._token-CxXek2SP.js → invite._token-CKAfd5iB.js} +1 -1
  13. package/dist/client/assets/issues._fingerprint-Cm72LNM_.js +55 -0
  14. package/dist/client/assets/login-BTxOhEbN.js +1 -0
  15. package/dist/client/assets/request-panel-DmtFRF8Z.js +1 -0
  16. package/dist/client/assets/services._service-CU1a45f3.js +1 -0
  17. package/dist/client/assets/{sessions._sessionId-3wNImMqM.js → sessions._sessionId-LRo3QNQP.js} +1 -1
  18. package/dist/client/assets/settings-Ctzvl2LL.js +1 -0
  19. package/dist/client/assets/src-BiKxgjA_.js +3 -0
  20. package/dist/client/assets/styles-Tcq0pvWj.css +2 -0
  21. package/dist/client/assets/{team-Bud2-07A.js → team-xlrjhEGp.js} +1 -1
  22. package/dist/client/assets/{timeline-BSDhQPpY.js → timeline-WXE49N62.js} +1 -1
  23. package/dist/client/assets/ui-DRjjeSWy.js +1 -0
  24. package/dist/client/assets/version-banner-BYe4PVzv.js +1 -0
  25. package/dist/server/assets/_authed-Bn-IktVS.js +120 -0
  26. package/dist/server/assets/_authed-RXh7GpoE.js +252 -0
  27. package/dist/server/assets/_tanstack-start-manifest_v-C4dQ3MxD.js +122 -0
  28. package/dist/server/assets/app-code-BQL_FcSx.js +212 -0
  29. package/dist/server/assets/app-name-BcbY1BZN.js +11 -0
  30. package/dist/server/assets/closed-issues-o9U80FXv.js +45 -0
  31. package/dist/server/assets/config-Bp8m0B8q.js +57 -0
  32. package/dist/server/assets/{customers-CvSLt_xV.js → customers-CD84Bz4P.js} +11 -6
  33. package/dist/server/assets/{demo-BRTkdmQJ.js → demo-D-o8m75S.js} +225 -8
  34. package/dist/server/assets/{dynamodb-B0V3ZO20.js → dynamodb-CcH0cp7R.js} +55 -2
  35. package/dist/server/assets/empty-state-wz34oAi8.js +163 -0
  36. package/dist/server/assets/endpoints-8N389Poe.js +248 -0
  37. package/dist/server/assets/{endpoints-mPQXpH8C.js → endpoints-C0Cxc_-x.js} +38 -36
  38. package/dist/server/assets/{events._eventId-pkQBQ9sL.js → events._eventId-BOHkNO5W.js} +90 -351
  39. package/dist/server/assets/exchange-view-DzoSBt_R.js +112 -0
  40. package/dist/server/assets/{firestore-Jd-LeWZI.js → firestore-DyvYMiz3.js} +13 -2
  41. package/dist/server/assets/{fns-Di0vMpkQ.js → fns-BcVGbSID.js} +22 -7
  42. package/dist/server/assets/fns-DE03YExD.js +831 -0
  43. package/dist/server/assets/{invite._token-LwWF5YxP.js → invite._token-DhlACkIe.js} +3 -3
  44. package/dist/server/assets/{issues._fingerprint-DAA0QCuA.js → issues._fingerprint-BBd-_Nn5.js} +400 -28
  45. package/dist/server/assets/{login-DqjCU4DJ.js → login-CbfdHJvg.js} +3 -3
  46. package/dist/server/assets/{memory-aqsrtEUn.js → memory-CZBUh1Zc.js} +129 -5
  47. package/dist/server/assets/{errors-C2KEDL7l.js → queries-wha4cZbw.js} +1442 -129
  48. package/dist/server/assets/request-panel-DpCSwl2q.js +162 -0
  49. package/dist/server/assets/{router-C0Dauj2C.js → router-BXkzR89j.js} +197 -91
  50. package/dist/server/assets/services._service-DWjFFGON.js +309 -0
  51. package/dist/server/assets/{sessions._sessionId-D7JPV0M4.js → sessions._sessionId-DDspiPTe.js} +2 -2
  52. package/dist/server/assets/{settings-Drjd64d7.js → settings-DCLIaZ_p.js} +201 -24
  53. package/dist/server/assets/{src-C4fNgijW.js → src-B2GKPQyN.js} +1 -1
  54. package/dist/server/assets/{team-D7iXYnSE.js → team-jd_TmWP4.js} +3 -3
  55. package/dist/server/assets/ui-C3pl0edn.js +302 -0
  56. package/dist/server/assets/version-banner-BX34ijG5.js +106 -0
  57. package/dist/server/assets/versions-YI_zK1ql.js +58 -0
  58. package/dist/server/assets/warm-C3GbXP1_.js +16 -0
  59. package/dist/server/server.js +40 -28
  60. package/package.json +4 -4
  61. package/dist/client/assets/_authed-CNObwuZh.js +0 -7
  62. package/dist/client/assets/_authed-CuP-He8G.js +0 -1
  63. package/dist/client/assets/charts-CUvev8lV.js +0 -55
  64. package/dist/client/assets/customers-DTMEprFQ.js +0 -1
  65. package/dist/client/assets/endpoints-lMSO2CdM.js +0 -1
  66. package/dist/client/assets/events._eventId-bWag-i8J.js +0 -3
  67. package/dist/client/assets/index-BNpacHyi.js +0 -10
  68. package/dist/client/assets/issues._fingerprint-BQr85TVY.js +0 -1
  69. package/dist/client/assets/login-Dlk3GyyC.js +0 -1
  70. package/dist/client/assets/settings-CzbEYB7k.js +0 -1
  71. package/dist/client/assets/src-Dq5mIUf0.js +0 -3
  72. package/dist/client/assets/styles-DPsgRH7g.css +0 -2
  73. package/dist/client/assets/ui-CEv2y56s.js +0 -1
  74. package/dist/server/assets/_authed-DkkRwdZl.js +0 -87
  75. package/dist/server/assets/_authed-DsB8kjZB.js +0 -902
  76. package/dist/server/assets/_tanstack-start-manifest_v-D1_ErrJK.js +0 -97
  77. package/dist/server/assets/charts-DzTVqJ8z.js +0 -238
  78. package/dist/server/assets/fns-B3PHbFwV.js +0 -1295
  79. package/dist/server/assets/ui-B_XHlLUQ.js +0 -193
@@ -1,64 +1,110 @@
1
1
  import { a as getCookie, i as deleteCookie, o as getRequestHeader, s as setCookie } from "../server.js";
2
- import { C as cleanFnName, S as OWNER_INFO, _ as validHostNames, a as Envelope, b as IMPACT_INFO, c as compareVersions, d as percentileMs, f as releaseKey, g as normaliseHostMatch, h as nameUrlsIn, m as hostNameFor, o as DEFAULT_SEVERITY_THRESHOLDS, p as severityFromCount, s as SEVERITY_ORDER, u as parseStack, v as CATEGORY_INFO, w as markLibraryFrames, y as CAUSE_INFO } from "./memory-aqsrtEUn.js";
3
- import { n as TEAM_ROLES, t as MAX_RETENTION_DAYS } from "./src-C4fNgijW.js";
2
+ import { A as hostNameFor, B as OWNER_INFO, C as maskPersonal, D as severityFromCount, E as releaseKey, F as validHostNames, H as markLibraryFrames, I as CATEGORY_INFO, L as CAUSE_INFO, M as nameUrl, N as nameUrlsIn, O as hostKindLabel, P as normaliseHostMatch, R as IMPACT_INFO, S as isTestIssue, T as percentileMs, V as cleanFnName, _ as TEST_TAG, f as OFFLINE_FINGERPRINT, g as SEVERITY_ORDER, j as hostOf$1, l as Envelope, p as OFFLINE_TITLE, r as compareVersions, u as AIRPLANE_TAG, v as compareVersions$1, w as parseStack, x as isTestEvent, y as isOfflineIssue } from "./memory-CZBUh1Zc.js";
3
+ import { n as validAppName, t as appLabel } from "./app-name-BcbY1BZN.js";
4
+ import { n as TEAM_ROLES, t as MAX_RETENTION_DAYS } from "./src-B2GKPQyN.js";
5
+ import { t as config } from "./config-Bp8m0B8q.js";
4
6
  import { createHash, createHmac, randomBytes, scryptSync, timingSafeEqual } from "node:crypto";
5
- // src/server/config.ts
6
- var loaded = false;
7
- function loadEnv() {
8
- if (loaded) return;
9
- loaded = true;
10
- try {
11
- process.loadEnvFile?.(process.env.DEBUG_ENV_FILE ?? ".env");
12
- } catch {}
13
- }
14
- var generatedSecret;
15
- function config() {
16
- loadEnv();
17
- const env = process.env;
18
- const store = env.DEBUG_STORE ?? "demo";
19
- let secret = env.DEBUG_SESSION_SECRET;
20
- if (!secret || secret.length < 32) {
21
- if (store !== "demo" && store !== "memory") throw new Error("DEBUG_SESSION_SECRET must be set (≥32 chars) when not in demo mode. Try: openssl rand -hex 32");
22
- secret = generatedSecret ??= randomBytes(32).toString("hex");
23
- }
24
- const num = (v, d) => v && !Number.isNaN(Number(v)) ? Number(v) : d;
25
- const severity = {
26
- critical: num(env.DEBUG_SEVERITY_CRITICAL, DEFAULT_SEVERITY_THRESHOLDS.critical),
27
- high: num(env.DEBUG_SEVERITY_HIGH, DEFAULT_SEVERITY_THRESHOLDS.high),
28
- medium: num(env.DEBUG_SEVERITY_MEDIUM, DEFAULT_SEVERITY_THRESHOLDS.medium)
7
+ /** Entries kept at most; the oldest go first. */
8
+ var MAX_ENTRIES = 2e3;
9
+ /** The reads that are cached. */
10
+ var CACHED_READS = /* @__PURE__ */ new Set([
11
+ "getSettings",
12
+ "getVersions",
13
+ "countUsers",
14
+ "listApps",
15
+ "listIssues",
16
+ "getIssue",
17
+ "getRollups",
18
+ "listIssueEvents",
19
+ "listSessionEvents"
20
+ ]);
21
+ /** Calls that change data (demo and memory mode write through the store): they clear the cache. */
22
+ var WRITE = /^(write|upsert|increment|put|set|register|note|mark|delete|remove|restamp|create|accept|revoke|change)/;
23
+ /**
24
+ * A short-lived cache in front of the store, so moving between pages doesn't read the same data
25
+ * from Firestore or DynamoDB again and again.
26
+ *
27
+ * - The same read made twice at once (the health wall's summary and endpoints both read the issue
28
+ * list) is made once.
29
+ * - A read newer than FRESH_MS is served as it is; one up to STALE_MS old is served at once and
30
+ * read again in the background, so the next page has it fresh.
31
+ * - A write through the store (demo and memory mode), or clearStoreCache() after a change made
32
+ * through the ingest function's admin routes, clears it. A background read that started
33
+ * before the clear is dropped.
34
+ * - Only CACHED_READS are cached: team and sign-in reads always go to the store.
35
+ *
36
+ * Per server instance, like the sign-in limits in auth.ts: on hosting with several instances,
37
+ * another one can show data up to STALE_MS old after a change. The Refresh button clears this
38
+ * instance's cache.
39
+ */
40
+ function cacheStore(store, now = Date.now) {
41
+ const entries = /* @__PURE__ */ new Map();
42
+ let generation = 0;
43
+ const read = (key, call) => {
44
+ const t = now();
45
+ const hit = entries.get(key);
46
+ if (hit) {
47
+ entries.delete(key);
48
+ entries.set(key, hit);
49
+ }
50
+ if (hit && t - hit.at < 3e4) return hit.value;
51
+ if (hit && t - hit.at < 12e4) {
52
+ if (!hit.refreshing) {
53
+ hit.refreshing = true;
54
+ const started = generation;
55
+ const next = call();
56
+ next.then(() => generation === started && entries.set(key, {
57
+ at: now(),
58
+ value: next
59
+ }), () => hit.refreshing = false);
60
+ }
61
+ return hit.value;
62
+ }
63
+ const value = call();
64
+ entries.set(key, {
65
+ at: t,
66
+ value
67
+ });
68
+ value.catch(() => entries.get(key)?.value === value && entries.delete(key));
69
+ while (entries.size > MAX_ENTRIES) entries.delete(entries.keys().next().value);
70
+ return value;
71
+ };
72
+ const clear = () => {
73
+ generation++;
74
+ entries.clear();
29
75
  };
30
76
  return {
31
- store,
32
- sessionSecret: secret,
33
- users: env.DEBUG_DASHBOARD_USERS ?? "",
34
- apiToken: env.DEBUG_API_TOKEN,
35
- /** Demo mode: keys accepted by POST /api/ingest. "*" = any app. */
36
- localIngestKeys: env.DEBUG_LOCAL_INGEST_KEYS ?? "pk_local:*",
37
- firestoreRoot: env.DEBUG_FIRESTORE_ROOT ?? "debug_apps",
38
- dynamoTable: env.DEBUG_DYNAMO_TABLE ?? "debug",
39
- severity,
40
- /** The ingest function's admin routes (…/admin) and their token, for changing settings. */
41
- adminUrl: env.DEBUG_ADMIN_URL?.replace(/\/+$/, ""),
42
- adminToken: env.DEBUG_ADMIN_TOKEN,
43
- /** The deployment's default retention (what ingest uses before an app has a setting). */
44
- retentionDays: num(env.DEBUG_RETENTION_DAYS, 0) || void 0,
45
- /** "Create issue" buttons on the issue page. */
46
- trackers: {
47
- linear: env.DEBUG_LINEAR === "1" || !!env.DEBUG_LINEAR_TEAM,
48
- linearTeam: env.DEBUG_LINEAR_TEAM || void 0,
49
- githubRepo: env.DEBUG_GITHUB_REPO || void 0,
50
- dashboardUrl: env.DEBUG_DASHBOARD_URL || void 0
51
- },
52
- secureCookies: env.NODE_ENV === "production" && env.DEBUG_INSECURE_COOKIES !== "1",
53
- /** Firebase Hosting only forwards a cookie named __session to the function behind it. */
54
- cookieName: /^[A-Za-z0-9_]{1,40}$/.test(env.DEBUG_COOKIE_NAME ?? "") ? env.DEBUG_COOKIE_NAME : "debug_session"
77
+ store: new Proxy(store, { get(target, prop, receiver) {
78
+ const v = Reflect.get(target, prop, receiver);
79
+ if (typeof v !== "function" || typeof prop !== "string") return v;
80
+ if (CACHED_READS.has(prop)) return (...args) => read(`${prop}:${JSON.stringify(args)}`, () => v.apply(target, args));
81
+ if (!WRITE.test(prop)) return v.bind(target);
82
+ return async (...args) => {
83
+ clear();
84
+ try {
85
+ return await v.apply(target, args);
86
+ } finally {
87
+ clear();
88
+ }
89
+ };
90
+ } }),
91
+ clear
55
92
  };
56
93
  }
57
94
  // src/server/store.ts
58
95
  var store;
96
+ var cache$1;
59
97
  /** The dashboard only ever reads. Point it at read-only credentials. */
60
98
  function getStore() {
61
- return store ??= create();
99
+ return store ??= create().then((s) => {
100
+ cache$1 = cacheStore(s);
101
+ import("./warm-C3GbXP1_.js").then((m) => m.warmUp(cache$1.store)).catch(() => void 0);
102
+ return cache$1.store;
103
+ });
104
+ }
105
+ /** Forget every cached read: called after the dashboard changes anything (status, settings, team). */
106
+ function clearStoreCache() {
107
+ cache$1?.clear();
62
108
  }
63
109
  /**
64
110
  * Demo mode only: the in-memory store also accepts events, so a local app can send to
@@ -75,23 +121,25 @@ async function create() {
75
121
  case "firestore": {
76
122
  const { initializeApp, getApps } = await import("firebase-admin/app");
77
123
  const { getFirestore } = await import("firebase-admin/firestore");
78
- const { FirestoreStore } = await import("./firestore-Jd-LeWZI.js");
124
+ const { FirestoreStore } = await import("./firestore-DyvYMiz3.js");
79
125
  if (!getApps().length) initializeApp();
80
126
  return new FirestoreStore(getFirestore(), { root: cfg.firestoreRoot });
81
127
  }
82
128
  case "dynamodb": {
83
129
  const { DynamoDBClient } = await import("@aws-sdk/client-dynamodb");
84
130
  const { DynamoDBDocumentClient } = await import("@aws-sdk/lib-dynamodb");
85
- const { DynamoStore } = await import("./dynamodb-B0V3ZO20.js");
131
+ const { DynamoStore } = await import("./dynamodb-CcH0cp7R.js");
86
132
  return new DynamoStore(DynamoDBDocumentClient.from(new DynamoDBClient({})), { table: cfg.dynamoTable });
87
133
  }
88
134
  case "memory": {
89
- const { MemoryStore } = await import("./memory-aqsrtEUn.js").then((n) => n.n);
135
+ const { MemoryStore } = await import("./memory-CZBUh1Zc.js").then((n) => n.n);
90
136
  return new MemoryStore();
91
137
  }
92
138
  default: {
93
- const { seedDemo } = await import("./demo-BRTkdmQJ.js");
94
- return seedDemo();
139
+ const { keepUpstreamDown, seedDemo } = await import("./demo-D-o8m75S.js");
140
+ const demo = await seedDemo();
141
+ keepUpstreamDown(demo, clearStoreCache);
142
+ return demo;
95
143
  }
96
144
  }
97
145
  }
@@ -115,6 +163,7 @@ async function admin(method, route, body, query) {
115
163
  });
116
164
  const json = await res.json().catch(() => ({}));
117
165
  if (!res.ok) throw new Error(json.error ?? `The ingest function answered ${res.status}`);
166
+ if (method === "PUT") clearStoreCache();
118
167
  return json;
119
168
  }
120
169
  async function readSettings(app) {
@@ -124,6 +173,7 @@ async function readSettings(app) {
124
173
  const out = await admin("GET", "/settings", void 0, { app });
125
174
  return {
126
175
  app,
176
+ displayName: out.settings?.displayName ?? "",
127
177
  retentionDays: out.retentionDays,
128
178
  criticalFlows: out.settings?.criticalFlows ?? [],
129
179
  alerts: out.settings?.alerts ?? {},
@@ -135,6 +185,7 @@ async function readSettings(app) {
135
185
  } catch (err) {
136
186
  return {
137
187
  app,
188
+ displayName: "",
138
189
  retentionDays: c.retentionDays ?? null,
139
190
  criticalFlows: [],
140
191
  alerts: {},
@@ -148,6 +199,7 @@ async function readSettings(app) {
148
199
  const hook = s?.alerts?.slackWebhookUrl;
149
200
  return {
150
201
  app,
202
+ displayName: s?.displayName ?? "",
151
203
  retentionDays: s?.retentionDays ?? c.retentionDays ?? (writer === "local" ? 30 : null),
152
204
  criticalFlows: s?.criticalFlows ?? [],
153
205
  alerts: {
@@ -161,6 +213,12 @@ async function readSettings(app) {
161
213
  note: writer === "none" ? "Changing settings needs DEBUG_ADMIN_URL and DEBUG_ADMIN_TOKEN (npx debugkit init sets them)." : void 0
162
214
  };
163
215
  }
216
+ /** Every app's display name, by id: those with none set are left out (show the id). */
217
+ async function appNames(apps) {
218
+ const store = await getStore();
219
+ const names = await Promise.all(apps.map((a) => store.getSettings(a).then((s) => s?.displayName?.trim() ?? "", () => "")));
220
+ return Object.fromEntries(apps.map((a, i) => [a, names[i]]).filter(([, n]) => n));
221
+ }
164
222
  /** Hosts the app called in the last 30 days, busiest first: suggestions for naming them. */
165
223
  async function seenHosts(app, now = Date.now()) {
166
224
  const day = (t) => new Date(t).toISOString().slice(0, 10);
@@ -189,12 +247,15 @@ async function writeRetention(app, days) {
189
247
  else throw new Error("Settings can't be changed from this dashboard: DEBUG_ADMIN_URL and DEBUG_ADMIN_TOKEN aren't set.");
190
248
  return readSettings(app);
191
249
  }
192
- /** Change critical flows, alert settings and/or host names. Validation happens in the ingest function (or here, locally). */
250
+ /** Change the display name, critical flows, alert settings and/or host names. Validation happens in the ingest function (or here, locally). */
193
251
  async function writeSettings(app, patch) {
194
252
  const writer = settingsWriter();
195
253
  if (writer === "admin") await admin("PUT", "/settings", {
196
254
  app,
197
255
  ...patch
256
+ }).catch((err) => {
257
+ if (patch.displayName !== void 0 && /nothing to change/i.test(err.message)) throw new Error("Your ingest function is older than app names: upgrade it with npx debugkit upgrade, then save again.");
258
+ throw err;
198
259
  });
199
260
  else if (writer === "local") {
200
261
  const store = await getLocalWriteStore();
@@ -203,11 +264,14 @@ async function writeSettings(app, patch) {
203
264
  if (patch.alerts && url && !url.includes("…") && !/^https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/_-]{10,200}$/.test(url)) throw new Error("Use a Slack incoming-webhook URL (https://hooks.slack.com/services/…).");
204
265
  const hosts = patch.hosts ? validHostNames(patch.hosts) : void 0;
205
266
  if (typeof hosts === "string") throw new Error(hosts);
267
+ const name = patch.displayName === void 0 ? void 0 : validAppName(patch.displayName);
268
+ if (typeof name === "string") throw new Error(name);
206
269
  const alerts = patch.alerts ? {
207
270
  ...patch.alerts,
208
271
  slackWebhookUrl: url?.includes("…") ? cur?.alerts?.slackWebhookUrl : url || void 0
209
272
  } : void 0;
210
273
  await store.putSettings(app, {
274
+ ...name !== void 0 ? { displayName: name.name } : {},
211
275
  ...patch.criticalFlows ? { criticalFlows: patch.criticalFlows } : {},
212
276
  ...alerts ? { alerts } : {},
213
277
  ...hosts ? { hosts } : {}
@@ -255,6 +319,233 @@ async function restampStep(app, cursor) {
255
319
  }
256
320
  throw new Error("Settings can't be changed from this dashboard.");
257
321
  }
322
+ // src/lib/health.ts
323
+ /** Worst first when sorted descending; healthy comes last. */
324
+ var HEALTH_ORDER = {
325
+ ...SEVERITY_ORDER,
326
+ healthy: -1
327
+ };
328
+ /**
329
+ * What sets a service's level. Two ways to be bad: a large share of calls failing (with enough
330
+ * failures that it isn't one unlucky call), or a large number failing even when the share is
331
+ * small, because that's still many customers.
332
+ */
333
+ var SERVICE_LEVELS = {
334
+ critical: {
335
+ rate: .25,
336
+ failed: 5,
337
+ volume: 500
338
+ },
339
+ high: {
340
+ rate: .1,
341
+ failed: 3,
342
+ volume: 100
343
+ },
344
+ medium: {
345
+ rate: .05,
346
+ failed: 1
347
+ },
348
+ /** With no request summaries from the SDK: stored failures alone. */
349
+ stored: {
350
+ critical: 50,
351
+ high: 20,
352
+ medium: 5
353
+ }
354
+ };
355
+ /**
356
+ * An outage: in the last 15 minutes, calls to a service got no answer or a 5xx for at least 3
357
+ * different people, and at least 5% of today's calls to it failed. The people make it more than
358
+ * one phone or one bad request; the share keeps a few unlucky calls on a busy service from
359
+ * reading as down. Request summaries are kept per day, so the share is today's (UTC), not the
360
+ * last 15 minutes'. Without summaries for today, the people alone decide.
361
+ */
362
+ var OUTAGE = {
363
+ windowMs: 9e5,
364
+ users: 3,
365
+ share: .05
366
+ };
367
+ /** Whether recent no-answer and 5xx calls are an outage: enough people, and enough of today's calls. */
368
+ function isOutage(people, today) {
369
+ if (people < OUTAGE.users) return false;
370
+ return !today?.calls || today.failed / today.calls >= OUTAGE.share;
371
+ }
372
+ /** Failing within this long ago counts as still happening. */
373
+ var LIVE_MS = 36e5;
374
+ /** Still open: not resolved or ignored. */
375
+ var isOpen = (status) => status !== "resolved" && status !== "ignored";
376
+ /** How badly an endpoint fails: the larger of the SDK's failed calls and the stored failures. */
377
+ var failing = (r) => Math.max(r.failedCalls ?? 0, r.failures);
378
+ /** "27%", or "<1%" for a small share that isn't none. */
379
+ var pct$1 = (r) => r > 0 && r < .01 ? "<1%" : `${Math.round(r * 100)}%`;
380
+ /** "1 call", "126 calls"; "2 people" with a plural given. */
381
+ var plural$2 = (n, w, many = `${w}s`) => `${n.toLocaleString()} ${n === 1 ? w : many}`;
382
+ /** How bad a service is, by share and by number of failed calls. See SERVICE_LEVELS. */
383
+ function serviceLevel(s) {
384
+ const L = SERVICE_LEVELS;
385
+ if (s.calls > 0) {
386
+ if (!s.failedCalls && !s.failures) return "healthy";
387
+ const rate = s.failedCalls / s.calls;
388
+ if (s.failedCalls >= L.critical.volume || rate >= L.critical.rate && s.failedCalls >= L.critical.failed) return "critical";
389
+ if (s.failedCalls >= L.high.volume || rate >= L.high.rate && s.failedCalls >= L.high.failed) return "high";
390
+ if (rate >= L.medium.rate && s.failedCalls >= L.medium.failed) return "medium";
391
+ return "low";
392
+ }
393
+ const n = s.failures;
394
+ return !n ? "healthy" : n >= L.stored.critical ? "critical" : n >= L.stored.high ? "high" : n >= L.stored.medium ? "medium" : "low";
395
+ }
396
+ /** One endpoint's level: as a service's, and critical when it's down for many people right now (see OUTAGE). */
397
+ function endpointLevel(e) {
398
+ if (e.down && isOutage(e.down.who.length, e.today)) return "critical";
399
+ return serviceLevel({
400
+ calls: e.calls ?? 0,
401
+ failedCalls: e.failedCalls ?? 0,
402
+ failures: e.failures
403
+ });
404
+ }
405
+ /**
406
+ * How to group service names: case doesn't matter, and a plural joins its singular when both are
407
+ * there ("Balances" with "balance"). "status" stays "status" when there's no "statu".
408
+ */
409
+ function serviceKeys(names) {
410
+ const lower = (n) => (n ?? "").trim().toLowerCase();
411
+ const known = new Set(names.map(lower));
412
+ return (name) => {
413
+ const l = lower(name);
414
+ return l.endsWith("s") && known.has(l.slice(0, -1)) ? l.slice(0, -1) : l;
415
+ };
416
+ }
417
+ /** Endpoints added up per service, most failures first. */
418
+ function servicesOf(endpoints) {
419
+ const keyOf = serviceKeys(endpoints.map((e) => e.service));
420
+ const by = /* @__PURE__ */ new Map();
421
+ for (const e of endpoints) by.set(keyOf(e.service), [...by.get(keyOf(e.service)) ?? [], e]);
422
+ return [...by.entries()].map(([key, list]) => {
423
+ const calls = list.reduce((n, e) => n + (e.calls ?? 0), 0);
424
+ const failedCalls = list.reduce((n, e) => n + (e.failedCalls ?? 0), 0);
425
+ const p95s = list.map((e) => e.p95).filter((v) => v !== null);
426
+ const busiest = [...list].sort((a, b) => (b.calls ?? 0) - (a.calls ?? 0));
427
+ const kinded = busiest.find((e) => e.kind);
428
+ return {
429
+ key: key || "-",
430
+ service: busiest[0].service,
431
+ ...kinded ? {
432
+ kind: kinded.kind,
433
+ ...kinded.provider ? { provider: kinded.provider } : {}
434
+ } : {},
435
+ down: downOfService(list),
436
+ endpoints: list.length,
437
+ calls,
438
+ failedCalls,
439
+ failureRate: calls ? failedCalls / calls : null,
440
+ worstP95: p95s.length ? Math.max(...p95s) : null,
441
+ failures: list.reduce((n, e) => n + e.failures, 0),
442
+ lastFailure: list.map((e) => e.lastFailure).filter((t) => !!t).sort().pop()
443
+ };
444
+ }).sort((a, b) => failing(b) - failing(a) || b.calls - a.calls);
445
+ }
446
+ /** A service's recent no-answer and 5xx calls, across its endpoints: people counted once. */
447
+ function downOfService(list) {
448
+ const downs = list.map((e) => e.down).filter((d) => !!d);
449
+ if (!downs.length) return void 0;
450
+ const statuses = /* @__PURE__ */ new Map();
451
+ for (const d of downs) for (const s of d.statuses) statuses.set(s.key, (statuses.get(s.key) ?? 0) + s.count);
452
+ const days = list.map((e) => e.today).filter((t) => !!t);
453
+ return {
454
+ ...days.length ? { today: {
455
+ calls: days.reduce((n, t) => n + t.calls, 0),
456
+ failed: days.reduce((n, t) => n + t.failed, 0)
457
+ } } : {},
458
+ users: new Set(downs.flatMap((d) => d.who)).size,
459
+ failures: downs.reduce((n, d) => n + d.failures, 0),
460
+ statuses: [...statuses.entries()].map(([key, count]) => ({
461
+ key,
462
+ count
463
+ })).sort((a, b) => b.count - a.count || a.key.localeCompare(b.key)),
464
+ last: downs.map((d) => d.last).sort().pop()
465
+ };
466
+ }
467
+ /**
468
+ * "Cell C Balance is down: no answer or a 5xx for 6 people in the last 15 min": the outage line, when a
469
+ * service is failing broadly right now (see OUTAGE). Undefined when it isn't.
470
+ */
471
+ function outageOf(s) {
472
+ if (!s.down || !isOutage(s.down.users, s.down.today)) return void 0;
473
+ const name = s.service ?? "Unnamed service";
474
+ return `${s.kind === "upstream" && s.provider && !name.toLowerCase().includes(s.provider.toLowerCase()) ? `${s.provider} ${name}` : name} is down: ${s.down.statuses.every((x) => /^\d+$/.test(x.key)) ? "a 5xx" : s.down.statuses.some((x) => /^\d+$/.test(x.key)) ? "no answer or a 5xx" : "no answer"} for ${plural$2(s.down.users, "person", "people")} in the last ${Math.round(OUTAGE.windowMs / 6e4)} min`;
475
+ }
476
+ /** A service's endpoints, by its key (as in the address). */
477
+ function endpointsOf(key, endpoints) {
478
+ const keyOf = serviceKeys(endpoints.map((e) => e.service));
479
+ const want = key === "-" ? "-" : keyOf(key) || "-";
480
+ return endpoints.filter((e) => (keyOf(e.service) || "-") === want);
481
+ }
482
+ /**
483
+ * What's wrong with a service, in one line, from its worst endpoint: the status, the backend's own
484
+ * code and message, and which call. "404 · 30007 Balance lookup unsuccessful · POST /ppd/api/balance/getBalances".
485
+ */
486
+ function headlineOf(endpoints) {
487
+ const worst = [...endpoints].filter((e) => failing(e) > 0).sort((a, b) => failing(b) - failing(a))[0];
488
+ if (!worst) return void 0;
489
+ return `${[worst.statuses[0]?.key, [worst.backendCodes[0]?.key, worst.backendMessages[0]?.key].filter(Boolean).join(" ")].filter(Boolean).join(" · ") || "Failing"} · ${worst.method ? `${worst.method} ` : ""}${worst.path ?? worst.endpoint}`;
490
+ }
491
+ /** A service's box: its level, what's wrong, and how much. */
492
+ function serviceTile(s, endpoints, now) {
493
+ const outage = outageOf(s);
494
+ const level = outage ? "critical" : serviceLevel(s);
495
+ const measure = s.calls > 0 ? s.failedCalls ? `${pct$1(s.failedCalls / s.calls)} of ${plural$2(s.calls, "call")} failed` : `${plural$2(s.calls, "call")}, none failed` : s.failures ? plural$2(s.failures, "failure") : "No calls";
496
+ return {
497
+ key: s.key,
498
+ name: s.service ?? "Unnamed service",
499
+ kind: "service",
500
+ level,
501
+ headline: level === "healthy" ? void 0 : headlineOf(endpointsOf(s.key, endpoints)),
502
+ measure,
503
+ live: level !== "healthy" && !!s.lastFailure && now - Date.parse(s.lastFailure) < 36e5,
504
+ lastFailure: s.lastFailure,
505
+ weight: failing(s),
506
+ ...s.kind ? {
507
+ side: s.kind,
508
+ sideLabel: hostKindLabel(s.kind, s.provider),
509
+ ...s.provider ? { provider: s.provider } : {}
510
+ } : {},
511
+ ...outage ? { outage } : {}
512
+ };
513
+ }
514
+ /** Worst level first, then down right now, then still happening, then most failures, then by name. */
515
+ var byHealth = (a, b) => HEALTH_ORDER[b.level] - HEALTH_ORDER[a.level] || Number(!!b.outage) - Number(!!a.outage) || Number(b.live) - Number(a.live) || b.weight - a.weight || a.name.localeCompare(b.name);
516
+ /**
517
+ * Who an outage is with, for a client: an upstream's ("It's Cell C's service, not our backend or
518
+ * the app.") or our own backend's. Nothing when nobody said whose service it is.
519
+ */
520
+ function whoseLine(t) {
521
+ if (t.side === "upstream") return `It's ${t.provider ? `${t.provider}'s` : "a third party's"} service, not our backend or the app.`;
522
+ return t.side === "ours" ? "It's our own backend, not an upstream service or the app." : "";
523
+ }
524
+ /**
525
+ * The answer to "why is it breaking?", for a client: what's down right now and whose service it
526
+ * is, what's failing badly, the worst of it with what the server said, whether it's still
527
+ * happening, and how much is healthy.
528
+ */
529
+ function answerOf(tiles, rangeLabel) {
530
+ const down = tiles.filter((t) => t.outage);
531
+ const downLines = down.map((t) => [`${t.outage}.`, whoseLine(t)].filter(Boolean).join(" "));
532
+ const healthyLine = `${tiles.filter((t) => t.level === "healthy").length} of ${tiles.length} ${tiles.length === 1 ? "part is" : "parts are"} healthy.`;
533
+ const failingBadly = tiles.filter((t) => t.level !== "healthy" && t.level !== "low");
534
+ if (!failingBadly.length) return `Nothing is failing badly in ${rangeLabel}. ${healthyLine}`;
535
+ const top = failingBadly.find((t) => !t.outage);
536
+ const names = (ts) => ts.length === 1 ? ts[0].name : `${ts.slice(0, -1).map((t) => t.name).join(", ")} and ${ts[ts.length - 1].name}`;
537
+ const critical = failingBadly.filter((t) => t.level === "critical" && !t.outage);
538
+ const others = failingBadly.filter((t) => !t.outage);
539
+ const lead = critical.length ? `${critical.length === 1 ? "1 part is" : `${critical.length} parts are`} ${down.length ? "also " : ""}failing badly in ${rangeLabel}: ${names(critical)}.` : others.length ? `${others.length === 1 ? "1 part has" : `${others.length} parts have`} ${down.length ? "other " : ""}problems in ${rangeLabel}: ${names(others)}.` : "";
540
+ const live = failingBadly.filter((t) => t.live).length;
541
+ return [
542
+ ...downLines,
543
+ lead,
544
+ top ? `${down.length ? "Worst otherwise" : "Worst"}: ${top.name}, ${top.measure}${top.headline ? ` (${top.headline})` : ""}.` : "",
545
+ live ? `${live === 1 ? "1 is" : `${live} are`} still happening (failed in the last hour).` : "",
546
+ healthyLine
547
+ ].filter(Boolean).join(" ");
548
+ }
258
549
  var EMAIL = /^[^\s@/]{1,64}@[^\s@/]{1,190}\.[^\s@/]{2,63}$/;
259
550
  var HASH = /^scrypt\.[0-9a-f]{32}\.[0-9a-f]{128}$/;
260
551
  var bad = (error, status = 400) => ({
@@ -440,7 +731,7 @@ async function sendAlerts(app, outcome, issues, settings, post = defaultPost) {
440
731
  u.flow && `flow \`${u.flow}\`${critical.has(u.flow) ? " (critical)" : ""}`,
441
732
  u.version && `version ${u.version}`
442
733
  ].filter(Boolean).join(" · ");
443
- const text = `${kind === "new" ? ":new: New issue" : ":rotating_light: Regressed"} in *${app}*: ${link ? `<${link}|${title}>` : title}${where ? `\n${where}` : ""}`;
734
+ const text = `${kind === "new" ? ":new: New issue" : ":rotating_light: Regressed"} in *${appLabel(app, settings?.displayName)}*: ${link ? `<${link}|${title}>` : title}${where ? `\n${where}` : ""}`;
444
735
  try {
445
736
  await post(a.slackWebhookUrl, { text });
446
737
  sent++;
@@ -576,10 +867,7 @@ function countryOf(ctx) {
576
867
  if (tz) return tz;
577
868
  return /[-_]([A-Z]{2})(?:\b|$)/.exec(ctx.locale ?? "")?.[1] ?? "unknown";
578
869
  }
579
- // ../../packages/ingest/src/classify.ts
580
- var FOREGROUND = "App came to the foreground";
581
870
  var BACKGROUND = "App went to the background";
582
- var NEVER_SENT_MS = 100;
583
871
  /** How long after reopening a failure still counts as "on resume", if nothing has succeeded since. */
584
872
  var RESUME_WINDOW_MS = 1e4;
585
873
  var CRASH_SOURCES = /* @__PURE__ */ new Set(["global-handler", "uncaughtException"]);
@@ -637,7 +925,7 @@ function category(e) {
637
925
  const source = e.tags?.source;
638
926
  if (r && r.status === void 0) {
639
927
  if (TIMED_OUT.test(e.error.message)) return "network.timeout";
640
- if (r.durationMs !== void 0 && r.durationMs < NEVER_SENT_MS) return "network.never_sent";
928
+ if (r.durationMs !== void 0 && r.durationMs < 100) return "network.never_sent";
641
929
  return "network.failed";
642
930
  }
643
931
  if (r?.status !== void 0) {
@@ -652,11 +940,17 @@ function category(e) {
652
940
  if (!r) return "app_reported";
653
941
  return "other";
654
942
  }
943
+ /**
944
+ * A request that never left the phone (ABA-132): no status, too fast to have reached a server, not
945
+ * a timeout. Every one counts as the phone being offline, including the first of a burst, which
946
+ * has no failed calls before it to show it was one.
947
+ */
948
+ var isOfflineFailure = (labels) => labels.category === "network.never_sent";
655
949
  /** The last foreground/background breadcrumb before the error. */
656
950
  function lastLifecycle(crumbs) {
657
951
  for (let i = crumbs.length - 1; i >= 0; i--) {
658
952
  const b = crumbs[i];
659
- if (b.category === "ui" && (b.message === FOREGROUND || b.message === BACKGROUND)) return b;
953
+ if (b.category === "ui" && (b.message === "App came to the foreground" || b.message === "App went to the background")) return b;
660
954
  }
661
955
  }
662
956
  function gather(e, crumbs, repeatedFailure) {
@@ -670,7 +964,7 @@ function gather(e, crumbs, repeatedFailure) {
670
964
  r: crumbRequest(b)
671
965
  })).filter((x) => !!x.r);
672
966
  const life = lastLifecycle(crumbs);
673
- const foreground = life?.message === FOREGROUND ? Date.parse(life.ts) : void 0;
967
+ const foreground = life?.message === "App came to the foreground" ? Date.parse(life.ts) : void 0;
674
968
  const burst = requests.filter((x) => x.r.failed && within(x.b, 5e3));
675
969
  const hosts = new Set(burst.map((x) => hostOf(x.r.url)));
676
970
  if (e.request && (burst.length || repeatedFailure)) hosts.add(hostOf(e.request.url));
@@ -898,17 +1192,24 @@ function fingerprint(e) {
898
1192
  const f = topInAppFrame(e.error.frames);
899
1193
  const r = e.request;
900
1194
  const where = r ? `${r.method.toUpperCase()} ${r.url} ${r.status ? `${Math.floor(r.status / 100)}xx` : "failed"}` : f ? `${cleanFnName(f.fn) ?? "?"}@${normaliseFile(f.file)}` : normaliseMessage(e.error.message);
901
- return createHash("sha256").update([
1195
+ const parts = [
902
1196
  e.error.name,
903
1197
  where,
904
- e.flow ?? ""
905
- ].join("|")).digest("hex").slice(0, 24);
1198
+ e.flow ?? "",
1199
+ ...isTestEvent(e) ? [TEST_TAG] : []
1200
+ ];
1201
+ return createHash("sha256").update(parts.join("|")).digest("hex").slice(0, 24);
906
1202
  }
907
1203
  function titleOf(e) {
908
1204
  return `${e.error.name}: ${e.error.message}`.slice(0, 200);
909
1205
  }
1206
+ // ../../packages/ingest/src/version.ts
1207
+ /** This ingest's version: the package's, which every release bumps with the SDK's (one changeset group). */
1208
+ var INGEST_VERSION = "0.10.0";
1209
+ /** Sent on every ingest response, so an SDK (or the CLI) can tell how old the ingest it talks to is. */
1210
+ var VERSION_HEADER = "x-debugkit-version";
910
1211
  // ../../packages/ingest/src/index.ts
911
- /** "pk_abc:megaton-native,pk_def:other-app" → { pk_abc: "megaton-native", ... } */
1212
+ /** "pk_abc:megatron-native,pk_def:other-app" → { pk_abc: "megatron-native", ... } */
912
1213
  function parseIngestKeys(raw) {
913
1214
  const out = {};
914
1215
  for (const pair of (raw ?? "").split(",").map((s) => s.trim()).filter(Boolean)) {
@@ -1006,6 +1307,8 @@ async function ingest(rawBody, ingestKey, opts) {
1006
1307
  const settings = env.events.some((e) => e.type === "error") ? await cachedSettings(opts.store, app) : null;
1007
1308
  const criticalFlows = settings?.criticalFlows ?? [];
1008
1309
  const hosts = settings?.hosts ?? [];
1310
+ const errorsHere = env.events.filter((e) => e.type === "error");
1311
+ const onlyTestErrors = errorsHere.length > 0 && errorsHere.every((e) => isTestEvent(e));
1009
1312
  /** This batch's errors, to group into incidents once they're all labelled. */
1010
1313
  const batchErrors = [];
1011
1314
  for (const raw of env.events) {
@@ -1019,6 +1322,7 @@ async function ingest(rawBody, ingestKey, opts) {
1019
1322
  }
1020
1323
  const day = dayOf(event.ts);
1021
1324
  if (event.type === "session") {
1325
+ if (onlyTestErrors && (event.status === "errored" || event.status === "crashed")) continue;
1022
1326
  rollups.push({
1023
1327
  app,
1024
1328
  day,
@@ -1067,6 +1371,10 @@ async function ingest(rawBody, ingestKey, opts) {
1067
1371
  }));
1068
1372
  continue;
1069
1373
  }
1374
+ if (event.type !== "error" && isTestEvent(event)) {
1375
+ stored.push(storedEvent(day, event));
1376
+ continue;
1377
+ }
1070
1378
  const kind = event.type === "error" ? "error" : "event";
1071
1379
  const inc = (dimension, key) => rollups.push({
1072
1380
  app,
@@ -1079,6 +1387,7 @@ async function ingest(rawBody, ingestKey, opts) {
1079
1387
  let fp;
1080
1388
  let labels;
1081
1389
  let se;
1390
+ let offline = false;
1082
1391
  if (event.type === "error") {
1083
1392
  fp = fingerprint(event);
1084
1393
  labels = classify({
@@ -1088,9 +1397,19 @@ async function ingest(rawBody, ingestKey, opts) {
1088
1397
  criticalFlows,
1089
1398
  hosts
1090
1399
  });
1400
+ const test = isTestEvent(event);
1401
+ offline = !test && isOfflineFailure(labels);
1402
+ if (offline) {
1403
+ labels = {
1404
+ ...labels,
1405
+ category: "network.offline",
1406
+ owner: "unknown"
1407
+ };
1408
+ fp = OFFLINE_FINGERPRINT;
1409
+ }
1091
1410
  const cur = issues.get(fp);
1092
1411
  se = storedEvent(day, event, fp, labels);
1093
- batchErrors.push({
1412
+ if (!test && !offline) batchErrors.push({
1094
1413
  ts: raw.ts,
1095
1414
  labels,
1096
1415
  se,
@@ -1099,9 +1418,9 @@ async function ingest(rawBody, ingestKey, opts) {
1099
1418
  issues.set(fp, {
1100
1419
  app,
1101
1420
  fingerprint: fp,
1102
- title: titleOf(event),
1103
- culprit: culpritOf(event),
1104
- flow: event.flow,
1421
+ title: offline ? OFFLINE_TITLE : titleOf(event),
1422
+ culprit: offline ? void 0 : culpritOf(event),
1423
+ flow: offline ? void 0 : event.flow,
1105
1424
  by: (cur?.by ?? 0) + 1,
1106
1425
  ts: cur && cur.ts > event.ts ? cur.ts : event.ts,
1107
1426
  firstTs: cur && cur.firstTs < event.ts ? cur.firstTs : event.ts,
@@ -1110,7 +1429,6 @@ async function ingest(rawBody, ingestKey, opts) {
1110
1429
  });
1111
1430
  scopes.add(`issue:${fp}`);
1112
1431
  inc("issue", fp);
1113
- if (event.request) inc("endpoint", `${event.request.method.toUpperCase()} ${event.request.url}`);
1114
1432
  rollups.push({
1115
1433
  app,
1116
1434
  day: event.ts.slice(0, 13),
@@ -1119,9 +1437,14 @@ async function ingest(rawBody, ingestKey, opts) {
1119
1437
  kind,
1120
1438
  by: 1
1121
1439
  });
1440
+ if (test) {
1441
+ stored.push(se);
1442
+ continue;
1443
+ }
1444
+ if (event.request && !offline) inc("endpoint", `${event.request.method.toUpperCase()} ${event.request.url}`);
1122
1445
  }
1123
1446
  inc("total", "all");
1124
- inc("flow", event.flow ?? "untagged");
1447
+ if (!offline) inc("flow", event.flow ?? "untagged");
1125
1448
  inc("os", `${context.os.name} ${context.os.version ?? ""}`.trim());
1126
1449
  inc("appVersion", context.app.version);
1127
1450
  inc("deviceModel", context.device.model);
@@ -1134,7 +1457,7 @@ async function ingest(rawBody, ingestKey, opts) {
1134
1457
  inc("country", countryOf(context));
1135
1458
  if (labels) {
1136
1459
  inc("category", labels.category);
1137
- inc("owner", labels.owner);
1460
+ if (!offline) inc("owner", labels.owner);
1138
1461
  inc("impact", labels.impact);
1139
1462
  }
1140
1463
  }
@@ -1180,9 +1503,16 @@ async function ingest(rawBody, ingestKey, opts) {
1180
1503
  stored.length ? opts.store.writeEvents(stored, { retentionDays }) : void 0,
1181
1504
  opts.store.upsertIssues([...issues.values()]),
1182
1505
  opts.store.incrementRollups(rollups),
1183
- markUsers(opts.store, app, env.user?.id ?? env.user?.anonymousId ?? env.session.id, scopes, dayOf(receivedAt))
1506
+ markUsers(opts.store, app, env.user?.id ?? env.user?.anonymousId ?? env.session.id, scopes, dayOf(receivedAt)),
1507
+ noteVersions(opts.store, app, env.sdk.version, receivedAt)
1184
1508
  ]);
1185
- if (outcome && (outcome.created.length || outcome.regressed.length)) await sendAlerts(app, outcome, issues, settings);
1509
+ const real = (fp) => !isOfflineIssue(issues.get(fp)) && !isTestIssue(issues.get(fp));
1510
+ const alerting = outcome && {
1511
+ ...outcome,
1512
+ created: outcome.created.filter(real),
1513
+ regressed: outcome.regressed.filter(real)
1514
+ };
1515
+ if (alerting && (alerting.created.length || alerting.regressed.length)) await sendAlerts(app, alerting, issues, settings);
1186
1516
  return {
1187
1517
  status: 202,
1188
1518
  body: {
@@ -1191,6 +1521,42 @@ async function ingest(rawBody, ingestKey, opts) {
1191
1521
  }
1192
1522
  };
1193
1523
  }
1524
+ /**
1525
+ * What this instance last recorded (or tried to) per store and app, so a steady app costs one
1526
+ * versions write a day, not one per batch, and a store that's down isn't retried on every batch.
1527
+ */
1528
+ var recordedVersions = /* @__PURE__ */ new WeakMap();
1529
+ var VERSIONS_REFRESH_MS = 864e5;
1530
+ var VERSIONS_RETRY_MS = 6e5;
1531
+ /**
1532
+ * Record the app's SDK version and this ingest's, when the SDK is newer than what this instance last
1533
+ * recorded, or a day has passed. A failure here never fails the envelope; it's tried again in ten minutes.
1534
+ */
1535
+ async function noteVersions(store, app, sdk, at) {
1536
+ let byApp = recordedVersions.get(store);
1537
+ if (!byApp) recordedVersions.set(store, byApp = /* @__PURE__ */ new Map());
1538
+ const had = byApp.get(app);
1539
+ const now = Date.parse(at);
1540
+ if (had && compareVersions(sdk, had.sdk) <= 0 && now - had.at < VERSIONS_REFRESH_MS) return;
1541
+ const newest = compareVersions(sdk, had?.sdk) > 0 ? sdk : had?.sdk;
1542
+ try {
1543
+ await store.recordVersions(app, {
1544
+ sdk,
1545
+ ingest: INGEST_VERSION,
1546
+ at
1547
+ });
1548
+ byApp.set(app, {
1549
+ sdk: newest,
1550
+ at: now
1551
+ });
1552
+ } catch (err) {
1553
+ byApp.set(app, {
1554
+ sdk: newest,
1555
+ at: now - VERSIONS_REFRESH_MS + VERSIONS_RETRY_MS
1556
+ });
1557
+ console.warn("[deBug] couldn't record versions", err);
1558
+ }
1559
+ }
1194
1560
  /** Recently written (scope, member, day) marks, so a busy user costs one write per day, not per batch. */
1195
1561
  var seenMarks = /* @__PURE__ */ new Set();
1196
1562
  /**
@@ -1214,11 +1580,14 @@ async function markUsers(store, app, who, scopes, day) {
1214
1580
  if (seenMarks.size > 5e4) seenMarks.clear();
1215
1581
  if (marks.length) await store.markUsers(app, marks);
1216
1582
  }
1583
+ /** On every ingest response: CORS for browser SDKs, and this ingest's version, readable from the browser too. */
1217
1584
  var CORS_HEADERS = {
1218
1585
  "access-control-allow-origin": "*",
1219
1586
  "access-control-allow-methods": "POST, OPTIONS",
1220
1587
  "access-control-allow-headers": "content-type, x-debug-key",
1221
- "access-control-max-age": "86400"
1588
+ "access-control-expose-headers": VERSION_HEADER,
1589
+ "access-control-max-age": "86400",
1590
+ [VERSION_HEADER]: INGEST_VERSION
1222
1591
  };
1223
1592
  // src/server/auth.ts
1224
1593
  var cookie = () => config().cookieName;
@@ -1447,9 +1816,11 @@ var PASS_TO = {
1447
1816
  third_party: "whoever manages that third-party service",
1448
1817
  unknown: "app team first, to find out whose it is"
1449
1818
  };
1819
+ /** Nobody owns the phone's signal, but the app decides what happens without it (ABA-132). */
1820
+ var OFFLINE_PASS_TO = "Pass to: app team, to handle being offline. No endpoint or server is at fault.";
1450
1821
  var TEMPLATES = {
1451
1822
  "network.offline": {
1452
- saw: (n, are) => `${cap(n)} ${are === "are" ? "weren't" : "wasn't"} available`,
1823
+ saw: () => "Requests never left the phone",
1453
1824
  why: () => "The phone was offline.",
1454
1825
  diagnosis: () => "No connection when the request was made.",
1455
1826
  fix: () => "Check connectivity before sending (NetInfo), queue the request until the phone is back online, and show an offline state."
@@ -1464,7 +1835,7 @@ var TEMPLATES = {
1464
1835
  saw: (n) => `${cap(n)} took too long and gave up`,
1465
1836
  why: () => "The request was sent but no answer came back in time: a slow network or a slow server.",
1466
1837
  diagnosis: (_ev, took) => `Timed out${took ? ` after ${took}` : ""} with no response.`,
1467
- fix: () => "Check this endpoint's p95 under Response times. If the server is slow, fix it there; otherwise raise the timeout, retry idempotent requests, and show a retry state."
1838
+ fix: () => "Check this endpoint's p95 under Response times. If it's slow on every network, give that service's owner the timings and the times above; in the app, raise the timeout, retry idempotent requests, and show a retry state."
1468
1839
  },
1469
1840
  "network.failed": {
1470
1841
  saw: (n) => `${cap(n)} couldn't connect`,
@@ -1554,6 +1925,7 @@ function explain(input) {
1554
1925
  const e = ev0?.type === "error" ? ev0 : void 0;
1555
1926
  const ev = e ? evidenceOf(e) : NO_EVIDENCE;
1556
1927
  const t = TEMPLATES[labels.category] ?? TEMPLATES.other;
1928
+ const offline = labels.category === "network.offline";
1557
1929
  const causes = labels.causes;
1558
1930
  const model = input.lastEvent?.context.device.model;
1559
1931
  const service = e?.request ? hostNameFor(e.request.url, input.hosts)?.host.name : void 0;
@@ -1565,16 +1937,19 @@ function explain(input) {
1565
1937
  const action = userAction(e?.error.frames);
1566
1938
  const doing = action ? ` (after ${action.action})` : "";
1567
1939
  const causeSentences = causes.flatMap((c) => CAUSE_LINES[c]?.summary?.({ model }) ?? []);
1940
+ const said = !offline && (input.backend?.message || input.backend?.code) ? input.backend : void 0;
1941
+ const saidService = said?.service ?? service;
1942
+ const answer = said ? `${said.message ? `"${said.message}"` : ""}${said.code ? `${said.message ? " " : ""}(code ${said.code})` : ""}` : "";
1568
1943
  const summary = [
1569
- `${t.saw(s.name, s.plural ? "are" : "is")}${who}${where}${when}${doing}.`,
1570
- t.why(ev, causes),
1944
+ said ? `${saidService ? `The ${saidService} service` : "The server"} answered${said.status ? ` ${said.status}` : ""} ${answer}${who}${where}${when}${doing}.` : `${t.saw(s.name, s.plural ? "are" : "is")}${who}${where}${when}${doing}.`,
1945
+ said ? "That's the service's own reason: start there." : t.why(ev, causes),
1571
1946
  ...causeSentences,
1572
1947
  stillHappening(issue, rangeLabel, now),
1573
- `Pass to: ${PASS_TO[labels.owner] ?? PASS_TO.unknown}${service && (labels.owner === "backend" || labels.owner === "third_party") ? ` (${service})` : ""}.`
1948
+ offline ? OFFLINE_PASS_TO : said ? `Pass to: whoever runs ${saidService ? `the ${saidService} service` : "that service"}, with ${said.code ? `code ${said.code}` : "its answer"}${said.traceId ? ` and trace ${said.traceId}` : ""}.` : `Pass to: ${PASS_TO[labels.owner] ?? PASS_TO.unknown}${service && (labels.owner === "backend" || labels.owner === "third_party") ? ` (${service})` : ""}.`
1574
1949
  ].join(" ");
1575
1950
  const took = durationText(input.durations?.length ? input.durations : ev.durationMs !== void 0 ? [ev.durationMs] : []);
1576
- const evidence = [t.diagnosis(ev, took)];
1577
- if (e?.request) evidence.push(`Request: \`${e.request.method.toUpperCase()} ${e.request.url}\`${service ? ` (${service})` : ""}.`);
1951
+ const evidence = said ? [`HTTP ${said.status ?? ev.status ?? "error"}, and the backend's own answer: ${answer}${said.traceId ? `, trace \`${said.traceId}\`` : ""}.`, ...said.codes.length > 1 || (said.codes[0]?.count ?? 0) > 1 ? [`Backend codes in recent occurrences: ${said.codes.map((c) => `${c.key} × ${c.count}`).join(", ")}.`] : []] : [t.diagnosis(ev, took)];
1952
+ if (e?.request && !offline) evidence.push(`Request: \`${e.request.method.toUpperCase()} ${e.request.url}\`${service ? ` (${service})` : ""}.`);
1578
1953
  if (causes.includes("resumed_from_background") && ev.msSinceForeground !== void 0) evidence.push(`The app came to the foreground ${seconds(ev.msSinceForeground)} before; no request has succeeded since.`);
1579
1954
  if (ev.failedBefore > 0) evidence.push(`${plural$1(ev.failedBefore, "other request")} failed in the 5 s before${ev.failedHosts.length > 1 ? `, across ${ev.failedHosts.length} hosts` : ""}.`);
1580
1955
  else if (ev.repeatedFailure) evidence.push("The same request had already failed just before.");
@@ -1583,7 +1958,12 @@ function explain(input) {
1583
1958
  for (const c of causes) if (CAUSE_LINES[c]?.fix) evidence.push(`${CAUSE_INFO[c].name}: ${CAUSE_INFO[c].description}`);
1584
1959
  if (issue.incidentTitle) evidence.push(`Part of the incident "${issue.incidentTitle}".`);
1585
1960
  const whereParts = [issue.culprit ? `\`${issue.culprit}\`` : void 0, action ? `started by \`${action.fn}\` (${action.action})` : void 0].filter(Boolean);
1586
- const fixSteps = [t.fix(ev, causes), ...causes.flatMap((c) => CAUSE_LINES[c]?.fix ?? [])];
1961
+ const lookUp = said && [said.code && `code ${said.code}`, said.traceId && `trace ${said.traceId}`].filter(Boolean).join(" or ");
1962
+ const fixSteps = [
1963
+ said ? `Search the ${saidService ?? "backend"} logs${lookUp ? ` for ${lookUp}` : ""} at the times above: the service's answer (under The call that failed) says why it refused or failed.` : t.fix(ev, causes),
1964
+ ...said && labels.category === "auth" ? [t.fix(ev, causes)] : [],
1965
+ ...causes.flatMap((c) => CAUSE_LINES[c]?.fix ?? [])
1966
+ ];
1587
1967
  return {
1588
1968
  summary,
1589
1969
  technical: {
@@ -1594,8 +1974,12 @@ function explain(input) {
1594
1974
  }
1595
1975
  };
1596
1976
  }
1597
- /** The explanation as Markdown lines, without a title: for tickets and prefilled links. */
1598
- function explanationLines(x) {
1977
+ /**
1978
+ * The explanation as Markdown lines, without a title: for tickets and prefilled links. Specific
1979
+ * findings from the trail (ABA-146) come before the template's advice, which is labelled "General
1980
+ * advice" so it isn't mistaken for specific.
1981
+ */
1982
+ function explanationLines(x, findings = []) {
1599
1983
  return [
1600
1984
  "### What happened",
1601
1985
  x.summary,
@@ -1605,19 +1989,867 @@ function explanationLines(x) {
1605
1989
  "",
1606
1990
  ...x.technical.evidence.map((l) => `- ${l}`),
1607
1991
  ...x.technical.where ? ["", `**Where:** ${x.technical.where}`] : [],
1992
+ ...findings.length ? [
1993
+ "",
1994
+ "### What the trail shows",
1995
+ ...findings.map(findingLine)
1996
+ ] : [],
1608
1997
  "",
1609
- `**Suggested fix:** ${x.technical.fix}`
1998
+ `**General advice:** ${x.technical.fix}`
1610
1999
  ];
1611
2000
  }
2001
+ /** "- Why. Fix. (2 of 2 occurrences)" */
2002
+ var findingLine = (f) => `- ${f.why} ${f.fix}${f.of > 1 ? ` (${f.seen} of ${f.of} occurrences)` : ""}`;
1612
2003
  /** The explanation as Markdown, for pasting into a Jira or Linear ticket by hand. */
1613
- function explanationMarkdown(title, x, link) {
2004
+ function explanationMarkdown(title, x, link, findings = []) {
1614
2005
  return [
1615
2006
  `## ${title}`,
1616
2007
  ...link ? ["", `deBug: ${link}`] : [],
1617
2008
  "",
1618
- ...explanationLines(x)
2009
+ ...explanationLines(x, findings)
1619
2010
  ].join("\n");
1620
2011
  }
2012
+ // src/server/exchange.ts
2013
+ function sideOf(s) {
2014
+ return {
2015
+ headers: s.headers && Object.keys(s.headers).length ? s.headers : void 0,
2016
+ body: s.body,
2017
+ bodySize: s.bodySize,
2018
+ truncated: s.truncated || void 0
2019
+ };
2020
+ }
2021
+ /** Whether the SDK captured anything beyond the call itself: only with network.bodies on. */
2022
+ function captured(r, crumbs) {
2023
+ return r.query !== void 0 || !!r.headers || r.body !== void 0 || r.bodySize !== void 0 || crumbs.some((b) => b.category === "network" && b.data?.hasBodies === true);
2024
+ }
2025
+ /** The request and response the SDK captured on an error, or undefined when it captured none. */
2026
+ function exchangeOf(ev) {
2027
+ const r = ev.request;
2028
+ if (!r || !(captured(r, ev.breadcrumbs ?? []) || ev.response)) return void 0;
2029
+ const fullUrl = r.url && !r.url.startsWith("/") ? `https://${r.url}` : void 0;
2030
+ return {
2031
+ request: {
2032
+ method: r.method.toUpperCase(),
2033
+ url: r.url,
2034
+ fullUrl,
2035
+ query: r.query?.replace(/^\?/, "") || void 0,
2036
+ ...sideOf(r)
2037
+ },
2038
+ response: ev.response ? {
2039
+ status: ev.response.status,
2040
+ ...sideOf(ev.response)
2041
+ } : void 0
2042
+ };
2043
+ }
2044
+ // src/server/errors.ts
2045
+ /** Firestore answers FAILED_PRECONDITION while an index deBug needs is missing or still building. */
2046
+ function isIndexError(message) {
2047
+ return /FAILED_PRECONDITION|requires an index|index is currently building/i.test(message);
2048
+ }
2049
+ /** A thrown Firestore error saying a composite index is missing (gRPC code 9), on the Firestore store only. */
2050
+ function isMissingIndex(e, store) {
2051
+ if (store !== "firestore") return false;
2052
+ const err = e;
2053
+ return (err?.code === 9 || err?.code === "failed-precondition") && /index/i.test(String(err?.message ?? ""));
2054
+ }
2055
+ /** A fatal error: labelled a crash, or uncaught and critical, which is how the SDK sends a fatal one. */
2056
+ function isCrash(e) {
2057
+ return e.event.type === "error" && (e.labels?.category === "crash" || !e.event.handled && e.event.severity === "critical");
2058
+ }
2059
+ /** crashed, errored (an uncaught error) or clean, as on the session page. */
2060
+ function sessionOutcome(events) {
2061
+ if (events.some(isCrash)) return "crashed";
2062
+ return events.some((e) => e.event.type === "error" && !e.event.handled) ? "errored" : "clean";
2063
+ }
2064
+ /** The prop that says why a flow failed, in the order apps tend to name it. */
2065
+ var REASON_KEYS = [
2066
+ "reason",
2067
+ "error",
2068
+ "errorKind",
2069
+ "code",
2070
+ "message"
2071
+ ];
2072
+ var OUTCOME_WORD = {
2073
+ success: "succeeded",
2074
+ failure: "failed",
2075
+ abandoned: "abandoned"
2076
+ };
2077
+ /** "auth.login: failed, wrongOtp". */
2078
+ function describeEvent(name, outcome, props, step) {
2079
+ const reason = REASON_KEYS.map((k) => props?.[k]).find((v) => typeof v === "string" || typeof v === "number");
2080
+ const parts = [outcome ? OUTCOME_WORD[outcome] ?? outcome : void 0, reason !== void 0 ? String(reason) : void 0].filter(Boolean);
2081
+ return `${name}${step ? ` · ${step}` : ""}${parts.length ? `: ${parts.join(", ")}` : ""}`;
2082
+ }
2083
+ /**
2084
+ * Each breadcrumb once: errors in the same session carry overlapping trails. Given the events
2085
+ * oldest first, each breadcrumb's source is the earliest error that carried it, the one it led to.
2086
+ *
2087
+ * When the SDK merges a repeat it moves the breadcrumb's time to the latest repeat, so a later
2088
+ * trail can hold the same breadcrumb with a newer time and a higher count. Nothing else was
2089
+ * recorded between it and the breadcrumb before it, so an identical one in that window is the
2090
+ * same run: keep the copy with the highest count.
2091
+ */
2092
+ function uniqueCrumbs(events) {
2093
+ const out = [];
2094
+ const exact = /* @__PURE__ */ new Map();
2095
+ const count = (b) => Number(b.data?.repeated ?? 1);
2096
+ for (const e of events) {
2097
+ if (e.event.type !== "error") continue;
2098
+ const source = e.fingerprint ? {
2099
+ eventId: e.event.id,
2100
+ fingerprint: e.fingerprint
2101
+ } : void 0;
2102
+ let after = -Infinity;
2103
+ for (const b of e.event.breadcrumbs ?? []) {
2104
+ const t = Date.parse(b.ts);
2105
+ const same = (c) => c.crumb.category === b.category && c.crumb.message === b.message && c.crumb.level === b.level;
2106
+ const had = exact.get(`${b.ts}|${b.category}|${b.message}`) ?? (count(b) > 1 ? out.find((c) => same(c) && Date.parse(c.crumb.ts) > after && Date.parse(c.crumb.ts) <= t) : void 0);
2107
+ if (!had) {
2108
+ const c = {
2109
+ crumb: b,
2110
+ source
2111
+ };
2112
+ out.push(c);
2113
+ exact.set(`${b.ts}|${b.category}|${b.message}`, c);
2114
+ } else if (count(b) > count(had.crumb)) {
2115
+ had.crumb = b;
2116
+ exact.set(`${b.ts}|${b.category}|${b.message}`, had);
2117
+ }
2118
+ after = t;
2119
+ }
2120
+ }
2121
+ return out;
2122
+ }
2123
+ /** How far apart a custom event and its own breadcrumb, or an api.failed event and its call, can be. */
2124
+ var SAME_MS = 2e3;
2125
+ /** The SDK writes each custom event as an "event" breadcrumb too: the breadcrumb that is this event, if any. */
2126
+ function ownCrumb(name, ts, crumbs, taken) {
2127
+ const t = Date.parse(ts);
2128
+ return crumbs.find((c) => !taken.has(c) && c.crumb.category === "event" && c.crumb.message === name && Math.abs(Date.parse(c.crumb.ts) - t) <= SAME_MS);
2129
+ }
2130
+ /** The app's api.failed event says the same as the failed call's breadcrumb next to it. */
2131
+ function repeatsCrumb(e, crumbs) {
2132
+ if (e.event.type !== "event" || e.event.name !== "api.failed") return false;
2133
+ const path = e.event.props?.path;
2134
+ const t = Date.parse(e.event.ts);
2135
+ return crumbs.some((c) => c.api?.failed && c.api.path === path && Math.abs(Date.parse(c.ts) - t) <= SAME_MS);
2136
+ }
2137
+ async function buildSessionDetail(store, app, sessionId) {
2138
+ const [events, settings] = await Promise.all([store.listSessionEvents(app, sessionId, { limit: 501 }), store.getSettings(app).catch(() => null)]);
2139
+ if (!events.length) return null;
2140
+ const truncated = events.length > 500;
2141
+ const page = events.slice(-500);
2142
+ const hosts = settings?.hosts ?? [];
2143
+ const carried = withoutSdkTwinsCarried(uniqueCrumbs(page));
2144
+ const taken = /* @__PURE__ */ new Set();
2145
+ const own = /* @__PURE__ */ new Map();
2146
+ for (const e of page) {
2147
+ if (e.event.type !== "event") continue;
2148
+ const c = ownCrumb(e.event.name, e.event.ts, carried, taken);
2149
+ if (c) {
2150
+ taken.add(c);
2151
+ own.set(e, c);
2152
+ }
2153
+ }
2154
+ const kept = carried.filter((c) => !taken.has(c)).map((c) => ({
2155
+ ts: c.crumb.ts,
2156
+ item: readCrumb(c.crumb, hosts),
2157
+ source: c.source
2158
+ }));
2159
+ const crumbs = kept.map((c) => c.item);
2160
+ const rows = kept.map((c) => ({
2161
+ kind: "crumb",
2162
+ ...c
2163
+ }));
2164
+ let errors = 0;
2165
+ let handled = 0;
2166
+ for (const e of page) {
2167
+ const ev = e.event;
2168
+ if (ev.type === "error") {
2169
+ const crash = isCrash(e);
2170
+ errors++;
2171
+ if (ev.handled) handled++;
2172
+ rows.push({
2173
+ kind: "error",
2174
+ ts: ev.ts,
2175
+ eventId: ev.id,
2176
+ fingerprint: e.fingerprint,
2177
+ title: nameUrlsIn(`${ev.error.name}: ${ev.error.message}`, hosts),
2178
+ handled: ev.handled,
2179
+ crash
2180
+ });
2181
+ } else if (ev.type === "event" && !repeatsCrumb(e, crumbs)) rows.push({
2182
+ kind: "event",
2183
+ ts: ev.ts,
2184
+ eventId: ev.id,
2185
+ name: ev.name,
2186
+ outcome: ev.outcome,
2187
+ detail: describeEvent(ev.name, ev.outcome, ev.props, ev.step),
2188
+ failed: ev.outcome === "failure",
2189
+ source: own.get(e)?.source
2190
+ });
2191
+ }
2192
+ const order = {
2193
+ crumb: 0,
2194
+ event: 1,
2195
+ error: 2
2196
+ };
2197
+ const sorted = byTime(rows.sort((a, b) => order[a.kind] - order[b.kind]));
2198
+ const last = page[page.length - 1];
2199
+ return {
2200
+ app,
2201
+ sessionId,
2202
+ user: {
2203
+ id: last.user?.id,
2204
+ anonymousId: last.user?.anonymousId
2205
+ },
2206
+ context: last.context,
2207
+ start: sorted[0]?.ts ?? page[0].event.ts,
2208
+ end: last.event.ts,
2209
+ outcome: sessionOutcome(page),
2210
+ counts: {
2211
+ errors,
2212
+ handled,
2213
+ events: sorted.filter((r) => r.kind === "event").length
2214
+ },
2215
+ rows: sorted,
2216
+ truncated,
2217
+ read: page.length
2218
+ };
2219
+ }
2220
+ /** How far after an error its "what they did next" reaches. */
2221
+ var AFTER_MS = 3e5;
2222
+ /**
2223
+ * What the customer did after an error, from the trails of the session's later errors: the steps
2224
+ * within five minutes of it, at most `limit`. Empty when nothing later was stored or the session
2225
+ * can't be read (Firestore without its index).
2226
+ */
2227
+ async function stepsAfter(store, app, sessionId, ts, hosts = [], limit = 30) {
2228
+ const events = await store.listSessionEvents(app, sessionId, { limit: 500 }).catch((e) => {
2229
+ if (isIndexError(String(e?.message))) return [];
2230
+ throw e;
2231
+ });
2232
+ const t = Date.parse(ts);
2233
+ return byTime(withoutSdkTwinsCarried(uniqueCrumbs(events)).filter((c) => {
2234
+ const d = Date.parse(c.crumb.ts) - t;
2235
+ return d > 0 && d <= AFTER_MS;
2236
+ }).map((c) => c.crumb)).slice(0, limit).map((b) => readCrumb(b, hosts));
2237
+ }
2238
+ /** withoutSdkTwins over carried breadcrumbs, keeping each one's source. */
2239
+ function withoutSdkTwinsCarried(carried) {
2240
+ const keep = new Set(withoutSdkTwins(carried.map((c) => c.crumb)));
2241
+ return carried.filter((c) => keep.has(c.crumb));
2242
+ }
2243
+ var str = (v) => typeof v === "string" && v ? v : typeof v === "number" ? String(v) : void 0;
2244
+ var num = (v) => typeof v === "number" && Number.isFinite(v) ? v : typeof v === "string" && /^\d+$/.test(v) ? Number(v) : void 0;
2245
+ /** The first key present: the Cell C app's keys and their alternatives ("backend.title" or "backendTitle"). */
2246
+ var pick = (d, ...keys) => keys.map((k) => d[k]).find((v) => v !== void 0 && v !== null && v !== "");
2247
+ /** API keys read into the call; the rest of the data is shown as it is. */
2248
+ var API_KEYS = /* @__PURE__ */ new Set([
2249
+ "service",
2250
+ "method",
2251
+ "path",
2252
+ "url",
2253
+ "status",
2254
+ "durationMs",
2255
+ "outcome",
2256
+ "retry",
2257
+ "backend.title",
2258
+ "backendTitle",
2259
+ "backend.code",
2260
+ "backendCode",
2261
+ "backendErrors",
2262
+ "traceId",
2263
+ "trace_id",
2264
+ "errorKind"
2265
+ ]);
2266
+ /** Outcomes that mean the call did what it should. Anything else the app reports (unreachable, sessionExpired) is a failure. */
2267
+ var OK_OUTCOMES = /* @__PURE__ */ new Set([
2268
+ "answered",
2269
+ "cached",
2270
+ "cancelled",
2271
+ "canceled",
2272
+ "aborted",
2273
+ "success",
2274
+ "ok"
2275
+ ]);
2276
+ /** The SDK's own network breadcrumb: "POST api.x.com/v1/pay → 503 (120ms)", or "→ failed". */
2277
+ var SDK_NETWORK = /^([A-Z]+) (\S+) → (\d{3})?/;
2278
+ /** A network breadcrumb as an API call, or undefined when it isn't one ("Device went offline"). */
2279
+ function apiCallOf(b, hosts = []) {
2280
+ if (b.category !== "network") return void 0;
2281
+ const d = b.data ?? {};
2282
+ const sdk = SDK_NETWORK.exec(b.message);
2283
+ const url = str(pick(d, "path", "url")) ?? sdk?.[2];
2284
+ const method = str(pick(d, "method"))?.toUpperCase() ?? sdk?.[1];
2285
+ const appService = str(d.service);
2286
+ if (!url && !method && !appService) return void 0;
2287
+ const status = num(d.status) ?? num(sdk?.[3]);
2288
+ const outcome = str(d.outcome);
2289
+ const named = url && !appService ? hostNameFor(url, hosts) : void 0;
2290
+ const call = {
2291
+ service: appService ?? named?.host.name,
2292
+ method,
2293
+ path: named ? named.rest || "/" : url,
2294
+ status,
2295
+ durationMs: num(d.durationMs),
2296
+ outcome,
2297
+ retry: typeof d.retry === "boolean" || typeof d.retry === "number" ? d.retry : str(d.retry),
2298
+ backendTitle: str(pick(d, "backend.title", "backendTitle")),
2299
+ backendCode: str(pick(d, "backend.code", "backendCode")),
2300
+ backendErrors: str(d.backendErrors),
2301
+ errorKind: str(d.errorKind),
2302
+ traceId: str(pick(d, "traceId", "trace_id")),
2303
+ failed: false
2304
+ };
2305
+ call.failed = b.level === "error" || status !== void 0 && status >= 400 || outcome !== void 0 && !OK_OUTCOMES.has(outcome);
2306
+ return call;
2307
+ }
2308
+ /** How close in time the SDK's breadcrumb and the app's own for the same call are. */
2309
+ var TWIN_MS = 2e3;
2310
+ /**
2311
+ * The SDK's own network breadcrumbs that repeat an app breadcrumb for the same call (same method,
2312
+ * the SDK's URL ending in the app's path, within TWIN_MS): the app's says more, so the SDK's is
2313
+ * left out of the timeline.
2314
+ */
2315
+ function sdkTwins(crumbs) {
2316
+ const app = crumbs.filter((b) => b.category === "network" && str(b.data?.service) && str(b.data?.path));
2317
+ const out = /* @__PURE__ */ new Set();
2318
+ if (!app.length) return out;
2319
+ for (const b of crumbs) {
2320
+ const sdk = b.category === "network" && !b.data?.service ? SDK_NETWORK.exec(b.message) : null;
2321
+ if (!sdk) continue;
2322
+ const url = sdk[2].split(/[?#]/)[0];
2323
+ const t = Date.parse(b.ts);
2324
+ if (app.some((a) => str(a.data?.method)?.toUpperCase() === sdk[1] && url.endsWith(String(a.data.path)) && Math.abs(Date.parse(a.ts) - t) <= TWIN_MS)) out.add(b);
2325
+ }
2326
+ return out;
2327
+ }
2328
+ /** The breadcrumbs in time order (stable for equal times), each read for what it says. */
2329
+ function timelineOf(crumbs = [], hosts = []) {
2330
+ return byTime(withoutSdkTwins(crumbs)).map((b) => readCrumb(b, hosts));
2331
+ }
2332
+ /** Oldest first, stable for equal times. By time, not by string: "09:00:00Z" and "09:00:00.500Z" don't sort as text. */
2333
+ function byTime(items) {
2334
+ return items.map((x, i) => ({
2335
+ x,
2336
+ i
2337
+ })).sort((p, q) => Date.parse(p.x.ts) - Date.parse(q.x.ts) || p.i - q.i).map(({ x }) => x);
2338
+ }
2339
+ /** The breadcrumbs without the SDK's own copy of a call the app also described. */
2340
+ function withoutSdkTwins(crumbs) {
2341
+ const twins = sdkTwins(crumbs);
2342
+ return crumbs.filter((b) => !twins.has(b));
2343
+ }
2344
+ /** One breadcrumb read for what it says: the API call it describes, or its message. */
2345
+ function readCrumb(b, hosts = []) {
2346
+ const api = apiCallOf(b, hosts);
2347
+ const rest = Object.entries(b.data ?? {}).filter(([k]) => k !== "repeated" && !(api && API_KEYS.has(k)));
2348
+ const repeated = num(b.data?.repeated);
2349
+ return {
2350
+ ts: b.ts,
2351
+ category: b.category,
2352
+ message: nameUrlsIn(b.message, hosts),
2353
+ level: b.level,
2354
+ failed: api ? api.failed : b.level === "error",
2355
+ ...api ? { api } : {},
2356
+ ...rest.length ? { data: Object.fromEntries(rest) } : {},
2357
+ ...repeated && repeated > 1 ? { repeated } : {}
2358
+ };
2359
+ }
2360
+ /** In-app frames first, keeping stack order within each group. */
2361
+ function framesInAppFirst(frames = []) {
2362
+ return [...frames.filter((f) => f.inApp), ...frames.filter((f) => !f.inApp)];
2363
+ }
2364
+ async function buildEventDetail(store, app, fingerprint, eventId) {
2365
+ const [issue, events, settings] = await Promise.all([
2366
+ store.getIssue(app, fingerprint),
2367
+ store.listIssueEvents(app, fingerprint, { limit: 200 }),
2368
+ store.getSettings(app).catch(() => null)
2369
+ ]);
2370
+ if (!issue) return null;
2371
+ const errors = (events.length ? events : [issue.lastEvent]).filter((e) => e.event.type === "error");
2372
+ const index = errors.findIndex((e) => e.event.id === eventId);
2373
+ const stored = index >= 0 ? errors[index] : issue.lastEvent.event.id === eventId ? issue.lastEvent : void 0;
2374
+ if (!stored || stored.event.type !== "error") return null;
2375
+ const ev = stored.event;
2376
+ const hosts = settings?.hosts ?? [];
2377
+ const timeline = timelineOf(ev.breadcrumbs, hosts);
2378
+ return {
2379
+ app,
2380
+ issue: {
2381
+ fingerprint,
2382
+ title: nameUrlsIn(issue.title, hosts),
2383
+ status: issue.status ?? "open"
2384
+ },
2385
+ id: ev.id,
2386
+ ts: ev.ts,
2387
+ handled: ev.handled,
2388
+ error: {
2389
+ name: ev.error.name,
2390
+ message: nameUrlsIn(ev.error.message, hosts)
2391
+ },
2392
+ flow: ev.flow,
2393
+ step: ev.step,
2394
+ user: {
2395
+ id: stored.user?.id,
2396
+ anonymousId: stored.user?.anonymousId
2397
+ },
2398
+ sessionId: stored.sessionId,
2399
+ context: stored.context,
2400
+ tags: Object.entries(ev.tags ?? {}).map(([key, value]) => ({
2401
+ key,
2402
+ value: String(value)
2403
+ })).sort((a, b) => a.key.localeCompare(b.key)),
2404
+ timeline,
2405
+ request: failedRequest(ev.request, timeline, hosts),
2406
+ exchange: exchangeOf(ev),
2407
+ frames: framesInAppFirst(ev.error.frames),
2408
+ stack: ev.error.stack,
2409
+ after: await stepsAfter(store, app, stored.sessionId, ev.ts, hosts),
2410
+ newer: index > 0 ? errors[index - 1].event.id : void 0,
2411
+ older: index >= 0 && index < errors.length - 1 ? errors[index + 1].event.id : void 0
2412
+ };
2413
+ }
2414
+ /**
2415
+ * The call that failed: the error's own request, filled in from the app's breadcrumb for the same
2416
+ * call (service, backend code, traceId), or the last failing call when the error has no request.
2417
+ * A breadcrumb counts as the same call only when its method and path match.
2418
+ */
2419
+ function failedRequest(request, timeline, hosts = []) {
2420
+ const lastFailing = [...timeline].reverse().find((t) => t.api?.failed)?.api;
2421
+ if (!request) return lastFailing;
2422
+ const named = hostNameFor(request.url, hosts);
2423
+ const own = {
2424
+ service: named?.host.name,
2425
+ method: request.method.toUpperCase(),
2426
+ path: named ? named.rest || "/" : request.url,
2427
+ status: request.status,
2428
+ durationMs: request.durationMs,
2429
+ failed: true
2430
+ };
2431
+ const bare = request.url.split(/[?#]/)[0];
2432
+ const sameCall = (c) => c.method === own.method && !!c.path && (bare.endsWith(c.path) || c.path === own.path);
2433
+ const match = [...timeline].reverse().find((t) => t.api?.failed && sameCall(t.api))?.api;
2434
+ return match ? {
2435
+ ...match,
2436
+ ...defined(own),
2437
+ service: match.service ?? own.service,
2438
+ path: named ? own.path : match.path
2439
+ } : own;
2440
+ }
2441
+ /** Drop undefined fields, so spreading doesn't erase a value from the breadcrumb. */
2442
+ function defined(o) {
2443
+ return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== void 0));
2444
+ }
2445
+ // src/server/hops.ts
2446
+ /** Response headers that carry a trace or request id, best first. */
2447
+ var TRACE_HEADERS = [
2448
+ "traceparent",
2449
+ "x-cloud-trace-context",
2450
+ "x-amzn-trace-id",
2451
+ "x-request-id",
2452
+ "x-amzn-requestid",
2453
+ "x-correlation-id",
2454
+ "request-id"
2455
+ ];
2456
+ /** Body fields that carry one, as backends commonly name them. */
2457
+ var TRACE_FIELDS = [
2458
+ "traceId",
2459
+ "trace_id",
2460
+ "requestId",
2461
+ "request_id",
2462
+ "correlationId",
2463
+ "correlation_id"
2464
+ ];
2465
+ /** W3C traceparent: version, 32-hex trace id (not all zeros), parent id, flags. */
2466
+ var TRACEPARENT = /^[\da-f]{2}-(?!0{32})([\da-f]{32})-[\da-f]{16}-[\da-f]{2}$/i;
2467
+ /** The trace id inside a header value, or undefined when the value isn't one. */
2468
+ function traceIdIn(header, value) {
2469
+ if (header === "traceparent") return TRACEPARENT.exec(value.trim())?.[1];
2470
+ if (header === "x-cloud-trace-context") return value.split("/")[0] || void 0;
2471
+ if (header === "x-amzn-trace-id") return /(?:^|;)\s*Root=([^;]+)/.exec(value)?.[1];
2472
+ return value.trim() || void 0;
2473
+ }
2474
+ /** A trace id or error code in a JSON response body, top level only. */
2475
+ function idsInBody(body) {
2476
+ if (!body || !body.trimStart().startsWith("{")) return {};
2477
+ try {
2478
+ const json = JSON.parse(body);
2479
+ const text = (v) => typeof v === "string" && v && v !== "[redacted]" ? v : typeof v === "number" ? String(v) : void 0;
2480
+ return {
2481
+ traceId: TRACE_FIELDS.map((k) => text(json[k])).find(Boolean),
2482
+ code: text(json.code) ?? text(json.errorCode)
2483
+ };
2484
+ } catch {
2485
+ return {};
2486
+ }
2487
+ }
2488
+ /** The trace id the backend already sent: a response header, then the body, then the app's breadcrumb. */
2489
+ function traceOf(headers, body, call) {
2490
+ for (const h of TRACE_HEADERS) {
2491
+ const v = headers?.[h];
2492
+ const id = v && v !== "[redacted]" ? traceIdIn(h, v) : void 0;
2493
+ if (id) return {
2494
+ from: h,
2495
+ id
2496
+ };
2497
+ }
2498
+ const inBody = idsInBody(body).traceId;
2499
+ if (inBody) return {
2500
+ from: "traceId",
2501
+ id: inBody
2502
+ };
2503
+ if (call?.traceId) return {
2504
+ from: "traceId",
2505
+ id: call.traceId
2506
+ };
2507
+ }
2508
+ /** "08:31:44 UTC" for the logs, which are rarely in the phone's timezone. */
2509
+ function utcTime(ts) {
2510
+ const t = new Date(ts);
2511
+ return Number.isNaN(t.getTime()) ? ts : `${t.toISOString().slice(11, 19)} UTC`;
2512
+ }
2513
+ var callLine = (c, withService = true) => `${withService && c.service ? `${c.service} · ` : ""}${c.method ?? ""} ${c.path ?? ""}`.replace(/\s+/g, " ").trim();
2514
+ // src/server/detectors.ts
2515
+ /**
2516
+ * Specific fix advice from patterns in the customer's trail (ABA-146): rule-based, no AI. Each
2517
+ * finding says what happened, what to do, and the evidence it read, so it can be checked.
2518
+ *
2519
+ * No backend changes (ABA-137): a fix may tell the app team what to change, or say where to look
2520
+ * in the backend's logs (time, call, trace id). It never tells anyone to change their backend.
2521
+ *
2522
+ * Not detected: a call still running when the app crashed. Breadcrumbs are written when a call
2523
+ * ends, so one that never answered isn't in the trail (needs the SDK to record request starts).
2524
+ */
2525
+ var FINDING_IDS = [
2526
+ "repeat_failed",
2527
+ "duplicate_call",
2528
+ "called_before_login",
2529
+ "no_refresh_before_401",
2530
+ "hung_in_background",
2531
+ "expected_answer"
2532
+ ];
2533
+ /** Two sends of one call this close together are one action sending it twice. */
2534
+ var DUPLICATE_WINDOW_MS = 5e3;
2535
+ /** A success this recent, then a failure of the same call, is "the repeat failed". */
2536
+ var REPEAT_WINDOW_MS = 6e4;
2537
+ /** Longer than this in the background and a session token has usually expired. */
2538
+ var LONG_BACKGROUND_MS = 18e5;
2539
+ /** A timeout this short isn't a request left hanging. */
2540
+ var HUNG_MIN_MS = 5e3;
2541
+ /** At least this many calls, all with one status, before it looks like the endpoint's normal answer. */
2542
+ var EXPECTED_MIN_CALLS = 3;
2543
+ var ms = (ts) => Date.parse(ts);
2544
+ var isOk = (s) => s !== void 0 && s >= 200 && s < 300;
2545
+ /** "08:31:42.729 UTC", like the log search times (hops.ts utcTime). */
2546
+ var clock = (t) => Number.isNaN(t) ? "?" : `${new Date(t).toISOString().slice(11, 23)} UTC`;
2547
+ /** The path of a stored URL, without host, port, query or hash: "host:443/ppd/x?y" → "/ppd/x". */
2548
+ function pathOf(url) {
2549
+ const bare = url.split(/[?#]/)[0].replace(/^[a-z]+:\/\//i, "");
2550
+ const rest = hostOf$1(bare) ? bare.slice(bare.indexOf("/") < 0 ? bare.length : bare.indexOf("/")) : bare;
2551
+ return rest.startsWith("/") ? rest : `/${rest}`;
2552
+ }
2553
+ /**
2554
+ * Whether a breadcrumb's call is the error's endpoint. Same method; same host when the breadcrumb
2555
+ * names one; and the same path, or the app's path relative to its base URL ("/notifications/x"
2556
+ * under "/ppd/api") ending the request's path on whole segments, at least two of them.
2557
+ */
2558
+ function sameEndpoint(api, req) {
2559
+ if (!api?.path || api.method !== req.method.toUpperCase()) return false;
2560
+ const host = hostOf$1(api.path);
2561
+ if (host && host !== hostOf$1(req.url)) return false;
2562
+ const want = pathOf(req.url);
2563
+ const have = pathOf(api.path);
2564
+ if (have === want) return true;
2565
+ return !host && have.split("/").length >= 3 && want.endsWith(have);
2566
+ }
2567
+ /** When the call was sent: its end minus its duration (from the data, or the SDK's "(120ms)"). */
2568
+ function sentAt(t) {
2569
+ const took = t.api?.durationMs ?? Number(/\((\d+) ?ms\)/.exec(t.message)?.[1] ?? 0);
2570
+ return ms(t.ts) - took;
2571
+ }
2572
+ /** A login that finished: the app's "Login Completed" or "Logged in", or a 2xx from a login call. */
2573
+ var isLogin = (t) => t.category === "event" && /^(log ?in (completed|succeeded|success)|logged in)$/i.test(t.message.trim()) || !!t.api && isOk(t.api.status) && /\/(login|sign-?in)$/i.test(pathOf(t.api.path ?? ""));
2574
+ /** A token refresh that worked. */
2575
+ var isRefresh = (t) => !!t.api && isOk(t.api.status) && /\/(refresh|refresh-token|token\/refresh|oauth\/token)$/i.test(pathOf(t.api.path ?? ""));
2576
+ var isRetry = (c) => c?.retry === true || typeof c?.retry === "number" && c.retry > 0 || typeof c?.retry === "string" && /^[1-9]/.test(c.retry);
2577
+ function read(ev, hosts) {
2578
+ const at = ms(ev.ts);
2579
+ const trail = timelineOf(ev.breadcrumbs, hosts).filter((t) => !(ms(t.ts) > at));
2580
+ const sent = at - (ev.request?.durationMs ?? 0);
2581
+ if (!ev.request) return {
2582
+ ev,
2583
+ trail,
2584
+ earlier: [],
2585
+ sent
2586
+ };
2587
+ const req = ev.request;
2588
+ const same = trail.filter((t) => sameEndpoint(t.api, req));
2589
+ let i = same.length - 1;
2590
+ while (i >= 0 && !same[i].api.failed) i--;
2591
+ return {
2592
+ ev,
2593
+ trail,
2594
+ call: failedRequest(req, trail, hosts),
2595
+ own: same[i],
2596
+ earlier: i < 0 ? same : same.slice(0, i),
2597
+ sent
2598
+ };
2599
+ }
2600
+ var statusOf = (t) => t.api.status ?? t.api.outcome ?? "no answer";
2601
+ /** Whether one action sent the error's call more than once: the send before it is close, and it isn't the app's own retry. */
2602
+ function sentTwice(r) {
2603
+ if (isRetry(r.own?.api)) return [];
2604
+ return r.earlier.filter((t) => r.sent - sentAt(t) <= DUPLICATE_WINDOW_MS && !isRetry(t.api));
2605
+ }
2606
+ var DETECTORS = {
2607
+ repeat_failed: (r) => {
2608
+ const failed = r.ev.request?.status;
2609
+ if (!r.call || failed === void 0 || !(failed >= 500 || failed === 409)) return void 0;
2610
+ const ok = [...r.earlier].reverse().find((t) => isOk(t.api.status) && r.sent - sentAt(t) <= REPEAT_WINDOW_MS);
2611
+ if (!ok) return void 0;
2612
+ const response = exchangeOf(r.ev)?.response;
2613
+ const trace = traceOf(response?.headers, response?.body, r.call);
2614
+ const service = r.call.service ?? (hostOf$1(r.ev.request.url) || "backend");
2615
+ const where = trace ? `trace id ${trace.id} (${trace.from})` : `${callLine(r.call, false)} sent at ${utcTime(new Date(r.sent).toISOString())}`;
2616
+ const overlapped = r.sent < ms(ok.ts);
2617
+ const twice = sentTwice(r).length > 0;
2618
+ return {
2619
+ why: overlapped ? `The app sent it again before the first call answered; the first worked (${ok.api.status}), the repeat failed (${failed}).` : `The first call worked (${ok.api.status}); the repeat ${seconds(r.sent - sentAt(ok))} later failed (${failed}).`,
2620
+ fix: `${twice ? "App: send it once per action. " : ""}To see why the repeat failed, search the ${service} logs for ${where}.`,
2621
+ evidence: [`${clock(sentAt(ok))} sent, ${clock(ms(ok.ts))} → ${ok.api.status}`, `${clock(r.sent)} sent, ${clock(ms(r.ev.ts))} → ${failed}`]
2622
+ };
2623
+ },
2624
+ duplicate_call: (r) => {
2625
+ const close = sentTwice(r);
2626
+ if (!r.call || !close.length) return void 0;
2627
+ return {
2628
+ why: `The app sends ${callLine(r.call)} ${close.length + 1} times in ${seconds(r.sent - sentAt(close[0]))}.`,
2629
+ fix: "App: send it once per action. Look for two screens or effects that both trigger it.",
2630
+ evidence: [...close.map((t) => `${clock(sentAt(t))} sent → ${statusOf(t)}`), `${clock(r.sent)} sent → ${r.ev.request.status ?? "no answer"}`]
2631
+ };
2632
+ },
2633
+ called_before_login: (r) => {
2634
+ if (!r.call) return void 0;
2635
+ const denied = r.earlier.filter((t) => t.api.status === 401 || t.api.status === 403);
2636
+ if (!denied.length) return void 0;
2637
+ const lastDenied = ms(denied[denied.length - 1].ts);
2638
+ const login = r.trail.find((t) => ms(t.ts) > lastDenied && ms(t.ts) <= r.sent && isLogin(t));
2639
+ if (!login) return void 0;
2640
+ const codes = [...new Set(denied.map((t) => t.api.status))].join(" or ");
2641
+ return {
2642
+ why: `${callLine(r.call)} ran ${denied.length} time${denied.length === 1 ? "" : "s"} before login and got ${codes}.`,
2643
+ fix: "App: wait until login has finished before making this call.",
2644
+ evidence: [...denied.map((t) => `${clock(ms(t.ts))} → ${t.api.status}`), `${clock(ms(login.ts))} ${login.message}`]
2645
+ };
2646
+ },
2647
+ no_refresh_before_401: (r) => {
2648
+ if (!r.call || r.ev.request?.status !== 401) return void 0;
2649
+ const lifecycle = r.trail.filter((t) => t.message === "App came to the foreground" || t.message === "App went to the background");
2650
+ const fore = lifecycle[lifecycle.length - 1];
2651
+ const back = lifecycle[lifecycle.length - 2];
2652
+ if (fore?.message !== "App came to the foreground" || back?.message !== "App went to the background" || ms(fore.ts) - ms(back.ts) < LONG_BACKGROUND_MS) return void 0;
2653
+ if (r.trail.some((t) => ms(t.ts) >= ms(fore.ts) && isRefresh(t))) return void 0;
2654
+ return {
2655
+ why: `The app came back after ${Math.round((ms(fore.ts) - ms(back.ts)) / 6e4)} min in the background and called ${callLine(r.call)} without refreshing its token.`,
2656
+ fix: "App: refresh the session token when the app comes back, before other calls.",
2657
+ evidence: [
2658
+ `${clock(ms(back.ts))} ${back.message}`,
2659
+ `${clock(ms(fore.ts))} ${fore.message}`,
2660
+ `${clock(ms(r.ev.ts))} → 401`
2661
+ ]
2662
+ };
2663
+ },
2664
+ hung_in_background: (r) => {
2665
+ const req = r.ev.request;
2666
+ if (!r.call || !req || req.status !== void 0 || req.durationMs === void 0 || req.durationMs < HUNG_MIN_MS) return void 0;
2667
+ if (!/timed out/i.test(r.ev.error.message)) return void 0;
2668
+ const bg = r.trail.find((t) => t.message === "App went to the background" && ms(t.ts) >= r.sent && ms(t.ts) <= ms(r.ev.ts));
2669
+ if (!bg) return void 0;
2670
+ return {
2671
+ why: `${callLine(r.call)} was still running when the app went to the background, and gave up after ${seconds(req.durationMs)}.`,
2672
+ fix: "App: send it before the app goes to the background, or retry it when the app comes back.",
2673
+ evidence: [
2674
+ `${clock(r.sent)} sent (from its duration)`,
2675
+ `${clock(ms(bg.ts))} ${bg.message}`,
2676
+ `${clock(ms(r.ev.ts))} → no answer`
2677
+ ]
2678
+ };
2679
+ }
2680
+ };
2681
+ /**
2682
+ * A 4xx an endpoint gives on every call can be its normal answer ("no update", "not found"). A
2683
+ * 5xx every time is an outage, and 401, 403 and 429 mean something to act on: never "expected".
2684
+ */
2685
+ var MAYBE_EXPECTED = (s) => s >= 400 && s < 500 && s !== 401 && s !== 403 && s !== 429;
2686
+ /** Every call to the newest occurrence's endpoint, across these trails, answered the same 4xx. */
2687
+ function expectedAnswer(reads) {
2688
+ const head = reads.find((r) => r.call && r.ev.request?.status !== void 0);
2689
+ const status = head?.ev.request?.status;
2690
+ if (!head || status === void 0 || !MAYBE_EXPECTED(status)) return void 0;
2691
+ const calls = /* @__PURE__ */ new Map();
2692
+ let seen = 0;
2693
+ for (const r of reads) {
2694
+ const mine = r.trail.filter((t) => sameEndpoint(t.api, head.ev.request));
2695
+ if (mine.length) seen++;
2696
+ for (const t of mine) calls.set(`${t.ts}|${t.api.status}`, t);
2697
+ }
2698
+ const all = [...calls.values()];
2699
+ if (all.length < EXPECTED_MIN_CALLS || !all.every((t) => t.api.status === status)) return void 0;
2700
+ return {
2701
+ seen,
2702
+ draft: {
2703
+ why: `Every call to ${callLine(head.call)} in these trails answered ${status} (${all.length} of ${all.length}).`,
2704
+ fix: `If ${status} is this endpoint's normal answer, mark it as expected so it stops counting.`,
2705
+ evidence: all.slice(-5).map((t) => `${clock(ms(t.ts))} → ${t.api.status}`)
2706
+ }
2707
+ };
2708
+ }
2709
+ /**
2710
+ * What the trails of an issue's recent occurrences show, best first. Reads the newest
2711
+ * DETECTOR_LOOKBACK error events; each finding keeps the newest occurrence's wording and evidence
2712
+ * and counts how many of them show it. When the repeat-failed finding already says "send it
2713
+ * once", the duplicate-call finding isn't listed again.
2714
+ */
2715
+ function findingsOf(events, hosts = []) {
2716
+ const reads = [...events].sort((a, b) => ms(b.ts) - ms(a.ts)).slice(0, 20).map((ev) => read(ev, hosts));
2717
+ const found = /* @__PURE__ */ new Map();
2718
+ for (const r of reads) for (const [id, detect] of Object.entries(DETECTORS)) {
2719
+ const draft = detect(r);
2720
+ if (!draft) continue;
2721
+ const had = found.get(id);
2722
+ if (had) had.seen++;
2723
+ else found.set(id, {
2724
+ id,
2725
+ ...draft,
2726
+ seen: 1,
2727
+ of: reads.length
2728
+ });
2729
+ }
2730
+ if (found.get("repeat_failed")?.fix.startsWith("App: send it once")) found.delete("duplicate_call");
2731
+ const expected = expectedAnswer(reads);
2732
+ if (expected) found.set("expected_answer", {
2733
+ id: "expected_answer",
2734
+ ...expected.draft,
2735
+ seen: expected.seen,
2736
+ of: reads.length
2737
+ });
2738
+ return FINDING_IDS.flatMap((id) => found.has(id) ? [found.get(id)] : []);
2739
+ }
2740
+ function offlineBursts(events, hosts = []) {
2741
+ const errors = events.filter((e) => e.event.type === "error");
2742
+ const byPhone = /* @__PURE__ */ new Map();
2743
+ for (const e of errors) {
2744
+ const phone = e.user?.anonymousId ?? e.sessionId;
2745
+ byPhone.set(phone, [...byPhone.get(phone) ?? [], e]);
2746
+ }
2747
+ const bursts = [];
2748
+ for (const [phone, list] of byPhone) {
2749
+ const sorted = [...list].sort((a, b) => Date.parse(a.event.ts) - Date.parse(b.event.ts));
2750
+ let cur = [];
2751
+ const close = () => {
2752
+ if (!cur.length) return;
2753
+ const first = cur[0];
2754
+ const last = cur[cur.length - 1];
2755
+ const counts = /* @__PURE__ */ new Map();
2756
+ for (const e of cur) {
2757
+ const r = e.event.type === "error" ? e.event.request : void 0;
2758
+ const call = r ? `${r.method.toUpperCase()} ${nameUrl(r.url, hosts)}` : "unknown call";
2759
+ counts.set(call, (counts.get(call) ?? 0) + 1);
2760
+ }
2761
+ bursts.push({
2762
+ start: first.event.ts,
2763
+ end: last.event.ts,
2764
+ phone,
2765
+ os: `${last.context.os.name} ${last.context.os.version ?? ""}`.trim(),
2766
+ device: last.context.device.model,
2767
+ appVersion: last.context.app.version,
2768
+ calls: [...counts].map(([call, count]) => ({
2769
+ call,
2770
+ count
2771
+ })).sort((a, b) => b.count - a.count),
2772
+ failures: cur.length,
2773
+ airplaneMode: cur.some((e) => e.event.tags?.[AIRPLANE_TAG] === "true")
2774
+ });
2775
+ cur = [];
2776
+ };
2777
+ for (const e of sorted) {
2778
+ if (cur.length && Date.parse(e.event.ts) - Date.parse(cur[cur.length - 1].event.ts) > 6e4) close();
2779
+ cur.push(e);
2780
+ }
2781
+ close();
2782
+ }
2783
+ return bursts.sort((a, b) => b.start.localeCompare(a.start));
2784
+ }
2785
+ /** How many bursts on how many phones, for Health's Folded away line. */
2786
+ function burstSummary(bursts) {
2787
+ return {
2788
+ bursts: bursts.length,
2789
+ phones: new Set(bursts.map((b) => b.phone)).size,
2790
+ lastSeen: bursts[0]?.end
2791
+ };
2792
+ }
2793
+ /** The offline issue's occurrences in a range: [start, end]. */
2794
+ var inRange = (events, start, end) => events.filter((e) => e.event.ts >= start && e.event.ts <= end);
2795
+ // src/server/backend-said.ts
2796
+ /** A code or id from a JSON field: text or a number, kept to plain characters, else nothing. */
2797
+ var token = (v) => {
2798
+ const s = typeof v === "number" ? String(v) : typeof v === "string" ? v.trim() : "";
2799
+ return /^[\w.:-]{1,100}$/.test(s) ? s : void 0;
2800
+ };
2801
+ /**
2802
+ * What a gateway or framework says for any failure: not the service's own reason, so it doesn't
2803
+ * replace the explanation's diagnosis.
2804
+ */
2805
+ var GENERIC = /^(internal server error|not found|bad request|forbidden|unauthori[sz]ed|missing authentication token|service unavailable|bad gateway|gateway time-?out|endpoint request timed out|too many requests|error|an error occurred)\.?$/i;
2806
+ /**
2807
+ * A message from the backend, safe to quote in a ticket or Slack: text only, phone numbers and
2808
+ * email addresses masked, no quotes or Markdown characters, at most 200 characters.
2809
+ */
2810
+ var message = (v) => {
2811
+ if (typeof v !== "string") return void 0;
2812
+ const s = maskPersonal(v).replace(/[`*_<>[\]]/g, "").replace(/"/g, "'").replace(/\s+/g, " ").trim().slice(0, 200);
2813
+ return s && !GENERIC.test(s) ? s : void 0;
2814
+ };
2815
+ /** One occurrence's answer from the backend, or undefined when it gave none of its own. */
2816
+ function saidBy(se, hosts = []) {
2817
+ if (se.event.type !== "error") return void 0;
2818
+ const call = failedRequest(se.event.request, timelineOf(se.event.breadcrumbs, hosts), hosts);
2819
+ let body = {};
2820
+ try {
2821
+ const parsed = se.event.response?.body ? JSON.parse(se.event.response.body) : void 0;
2822
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) body = parsed;
2823
+ } catch {}
2824
+ const code = token(call?.backendCode) ?? token(body.code) ?? token(body.errorCode);
2825
+ const said = message(call?.backendTitle) ?? message(body.title) ?? message(body.message) ?? message(body.error);
2826
+ if (!code && !said) return void 0;
2827
+ return {
2828
+ status: se.event.response?.status ?? call?.status ?? se.event.request?.status,
2829
+ code,
2830
+ message: said,
2831
+ traceId: token(call?.traceId) ?? token(body.traceId) ?? token(body.trace_id),
2832
+ service: call?.service
2833
+ };
2834
+ }
2835
+ /**
2836
+ * What the backend said across an issue's occurrences (newest first): the newest answer that had
2837
+ * its own code or message, and every code with how often it came back. undefined when it never
2838
+ * gave one.
2839
+ */
2840
+ function backendSaid(events, hosts = []) {
2841
+ const said = events.flatMap((e) => saidBy(e, hosts) ?? []);
2842
+ if (!said.length) return void 0;
2843
+ const counts = /* @__PURE__ */ new Map();
2844
+ for (const s of said) if (s.code) counts.set(s.code, (counts.get(s.code) ?? 0) + 1);
2845
+ return {
2846
+ ...said[0],
2847
+ codes: [...counts].map(([key, count]) => ({
2848
+ key,
2849
+ count
2850
+ })).sort((a, b) => b.count - a.count)
2851
+ };
2852
+ }
1621
2853
  // src/server/range.ts
1622
2854
  /** The date range a dashboard view covers, worked out from the URL (range=7d, or from/to). */
1623
2855
  var RANGE_PRESETS = [
@@ -1675,6 +2907,17 @@ function resolveRange(input, now = Date.now()) {
1675
2907
  granularity: preset === "24h" ? "hour" : days > 90 ? "week" : "day"
1676
2908
  };
1677
2909
  }
2910
+ /** The range as ISO instants: the last 24 hours for "24h", else whole UTC days from `from` to `to`. */
2911
+ function rangeBounds(range, now = Date.now()) {
2912
+ if (range.preset === "24h") return {
2913
+ start: (/* @__PURE__ */ new Date(now - 864e5)).toISOString(),
2914
+ end: new Date(now).toISOString()
2915
+ };
2916
+ return {
2917
+ start: `${range.from}T00:00:00.000Z`,
2918
+ end: `${range.to}T23:59:59.999Z`
2919
+ };
2920
+ }
1678
2921
  /** Every UTC day from `from` to `to`, inclusive. */
1679
2922
  function daysBetween(from, to) {
1680
2923
  const out = [];
@@ -1713,11 +2956,15 @@ var SUMMARY_FIELDS = {
1713
2956
  "totals.errors24h": "Error events in the trailing 24 hours (exact, from hourly rollups).",
1714
2957
  "totals.errorsRange": "Error events in the selected range.",
1715
2958
  "totals.changePct": "Percent change of errors vs the previous period of the same length. null when that period had fewer than 10 errors.",
1716
- "totals.activeIssues": "Distinct grouped errors (issues) seen in the range.",
2959
+ "totals.activeIssues": "Distinct grouped errors (issues) seen in the range, without test errors.",
2960
+ "totals.testIssues": "Issues made on purpose to test deBug (tag test=true). Left out of every count; list them with ?tests=1.",
2961
+ connectivity: "Phones going offline (requests that never left the phone), as bursts per phone, shown on Health as Folded away. Not counted against any endpoint or owner. null when there were none.",
1717
2962
  "totals.criticalIssues": "Issues whose 24h occurrence count meets the critical threshold.",
1718
2963
  "flows[]": "Parts of the app as tagged by the developer (e.g. auth.login). Sorted by severity, then errors24h, then errorsRange.",
1719
2964
  "flows[].failureRate": "failures/(failures+successes) in the range from events logged with an outcome. null when the app logs no outcomes for that flow.",
1720
- "issues[]": "Grouped errors seen in the range. Sorted by severity, then count24h, then countRange. Narrow with ?status=open|resolved|ignored|all (default all), ?flow=, ?category=, ?owner= and ?impact= (ids from breakdowns); an unknown value is a 400. _filter echoes what was applied.",
2965
+ "issues[]": "Grouped errors seen in the range. Sorted by severity, then count24h, then countRange. Narrow with ?status=open|resolved|ignored|all (default all), ?flow=, ?category=, ?owner= and ?impact= (ids from breakdowns); an unknown value is a 400. Test errors are left out unless ?tests=1. _filter echoes what was applied.",
2966
+ "issues[].offline": "true for the one issue that collects offline bursts (calls that never left the phone). On Health's Folded away line, left out of this list unless ?category=network.offline, and never counted as active.",
2967
+ "issues[].test": "true for an issue made on purpose to test deBug (tag test=true): listed only with ?tests=1, never counted, always severity low.",
1721
2968
  "issues[].severity": "Derived from count24h using thresholds critical/high/medium.",
1722
2969
  "issues[].title": "Error name and message, with hosts named in Settings shown by their names.",
1723
2970
  "issues[].culprit": "Top in-app stack frame: function (file:line).",
@@ -1814,14 +3061,15 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
1814
3061
  let errors24h = 0;
1815
3062
  let errorsPrev24h = 0;
1816
3063
  const issue24h = /* @__PURE__ */ new Map();
1817
- for (const r of hourRows) if (last24.has(r.day)) {
1818
- errors24h += r.errors;
1819
- issue24h.set(r.key, (issue24h.get(r.key) ?? 0) + r.errors);
1820
- } else errorsPrev24h += r.errors;
3064
+ const testIssues = new Set(issues.filter(isTestIssue).map((i) => i.fingerprint));
3065
+ const realHours = hourRows.filter((r) => !testIssues.has(r.key));
3066
+ for (const r of hourRows) if (last24.has(r.day)) issue24h.set(r.key, (issue24h.get(r.key) ?? 0) + r.errors);
3067
+ for (const r of realHours) if (last24.has(r.day)) errors24h += r.errors;
3068
+ else errorsPrev24h += r.errors;
1821
3069
  let trend;
1822
3070
  if (hourly) trend = h24.map((h) => ({
1823
3071
  day: h,
1824
- errors: hourRows.filter((r) => r.day === h).reduce((a, r) => a + r.errors, 0),
3072
+ errors: realHours.filter((r) => r.day === h).reduce((a, r) => a + r.errors, 0),
1825
3073
  events: 0
1826
3074
  }));
1827
3075
  else {
@@ -1851,22 +3099,37 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
1851
3099
  const errorsPrev = hourly ? errorsPrev24h : sumRows(prevTotal);
1852
3100
  const issueRange = /* @__PURE__ */ new Map();
1853
3101
  for (const r of issueDays) issueRange.set(r.key, (issueRange.get(r.key) ?? 0) + r.errors);
1854
- const rangeStart = hourly ? (/* @__PURE__ */ new Date(now - DAY)).toISOString() : `${from}T00:00:00.000Z`;
1855
- const versionsInRange = [...new Set(sessionRows.map((r) => r.key.split("|")[0]).filter((v) => v && v !== "unknown"))].sort(compareVersions);
3102
+ const { start: rangeStart, end: rangeEnd } = rangeBounds(range, now);
3103
+ const versionsInRange = [...new Set(sessionRows.map((r) => r.key.split("|")[0]).filter((v) => v && v !== "unknown"))].sort(compareVersions$1);
1856
3104
  const latestVersion = versionsInRange[versionsInRange.length - 1] ?? null;
1857
- const issueRows = issues.filter((i) => i.lastSeen >= rangeStart && i.firstSeen <= `${to}T23:59:59.999Z`).map((i) => toIssueRow(i, issue24h.get(i.fingerprint) ?? 0, hourly ? issue24h.get(i.fingerprint) ?? 0 : issueRange.get(i.fingerprint) ?? 0, thresholds, latestVersion, criticalFlows, hosts)).sort(bySeverityThenCount);
1858
- const active = issueRows.filter((i) => i.status !== "ignored" && i.status !== "resolved");
3105
+ const issueRows = issues.filter((i) => i.lastSeen >= rangeStart && i.firstSeen <= rangeEnd).map((i) => toIssueRow(i, issue24h.get(i.fingerprint) ?? 0, hourly ? issue24h.get(i.fingerprint) ?? 0 : issueRange.get(i.fingerprint) ?? 0, thresholds, latestVersion, criticalFlows, hosts)).sort(bySeverityThenCount);
3106
+ const active = issueRows.filter((i) => !i.test && !i.offline && i.status !== "ignored" && i.status !== "resolved");
1859
3107
  await Promise.all(active.slice(0, 10).map(async (i) => i.users = await countUsersSafe(store, app, `issue:${i.fingerprint}`, usersSince)));
1860
3108
  const lastEvents = new Map(issues.map((i) => [i.fingerprint, i.lastEvent]));
1861
- for (const row of issueRows) if (row.labels) row.explanation = explain({
1862
- issue: row,
1863
- labels: row.labels,
1864
- lastEvent: lastEvents.get(row.fingerprint),
1865
- users: row.users,
1866
- hosts,
1867
- rangeLabel: range.label,
1868
- now
1869
- });
3109
+ const offlineRow = issueRows.find((i) => i.offline);
3110
+ let connectivity = null;
3111
+ if (offlineRow) {
3112
+ const events = await store.listIssueEvents(app, offlineRow.fingerprint, { limit: 200 });
3113
+ connectivity = {
3114
+ fingerprint: offlineRow.fingerprint,
3115
+ ...burstSummary(offlineBursts(inRange(events, rangeStart, rangeEnd), hosts)),
3116
+ failures: offlineRow.countRange,
3117
+ capped: events.length >= 200
3118
+ };
3119
+ }
3120
+ for (const row of issueRows) if (row.labels) {
3121
+ const last = lastEvents.get(row.fingerprint);
3122
+ row.explanation = explain({
3123
+ issue: row,
3124
+ labels: row.labels,
3125
+ lastEvent: last,
3126
+ users: row.users,
3127
+ hosts,
3128
+ rangeLabel: range.label,
3129
+ backend: last ? backendSaid([last], hosts) : void 0,
3130
+ now
3131
+ });
3132
+ }
1870
3133
  const incidents = await buildIncidents(incidentRows, incidentIssueRows, issueRows, hourly, (key) => countUsersSafe(store, app, `incident:${key}`, usersSince));
1871
3134
  const firstDay = /* @__PURE__ */ new Map();
1872
3135
  for (const r of releaseRows) {
@@ -1940,12 +3203,14 @@ async function buildSummary(store, appParam, thresholds, now = Date.now(), opts
1940
3203
  eventsRange: sumRows(total, "events"),
1941
3204
  activeIssues: active.length,
1942
3205
  criticalIssues: active.filter((i) => i.severity === "critical").length,
3206
+ testIssues: issueRows.filter((i) => i.test).length,
1943
3207
  failingFlows: flows.filter((f) => (hourly ? f.errors24h : f.errorsRange) > 0).length
1944
3208
  },
1945
3209
  narrative: [],
1946
3210
  trend,
1947
3211
  flows,
1948
3212
  issues: issueRows,
3213
+ connectivity,
1949
3214
  incidents,
1950
3215
  breakdowns: {
1951
3216
  os: top(os),
@@ -2069,7 +3334,7 @@ function buildHealth(sessionRows, endpointRows, today, hosts = []) {
2069
3334
  ...v,
2070
3335
  crashFreeRate: rate(v.crashed, v.sessions),
2071
3336
  errorFreeRate: rate(v.errored, v.sessions)
2072
- })).filter((v) => v.sessions > 0).sort((a, b) => compareVersions(b.version, a.version)).slice(0, 8);
3337
+ })).filter((v) => v.sessions > 0).sort((a, b) => compareVersions$1(b.version, a.version)).slice(0, 8);
2073
3338
  const sum = (k) => [...byVersion.values()].reduce((a, v) => a + v[k], 0);
2074
3339
  const eps = /* @__PURE__ */ new Map();
2075
3340
  for (const r of endpointRows) {
@@ -2179,7 +3444,9 @@ function issueLabels(i, latestVersion, criticalFlows = [], hosts = []) {
2179
3444
  });
2180
3445
  }
2181
3446
  function toIssueRow(i, count24h, countRange, t, latestVersion = null, criticalFlows = [], hosts = []) {
2182
- const base = severityFromCount(count24h, t);
3447
+ const test = isTestIssue(i);
3448
+ const offline = isOfflineIssue(i);
3449
+ const base = test || offline ? "low" : severityFromCount(count24h, t);
2183
3450
  const critical = new Set(criticalFlows);
2184
3451
  return {
2185
3452
  labels: issueLabels(i, latestVersion, criticalFlows, hosts),
@@ -2194,12 +3461,14 @@ function toIssueRow(i, count24h, countRange, t, latestVersion = null, criticalFl
2194
3461
  title: nameUrlsIn(i.title, hosts),
2195
3462
  culprit: i.culprit,
2196
3463
  flow: i.flow,
2197
- severity: i.flow && critical.has(i.flow) && count24h > 0 ? BUMP[base] : base,
3464
+ severity: !test && i.flow && critical.has(i.flow) && count24h > 0 ? BUMP[base] : base,
2198
3465
  count24h,
2199
3466
  countRange,
2200
3467
  countTotal: i.count,
2201
3468
  firstSeen: i.firstSeen,
2202
- lastSeen: i.lastSeen
3469
+ lastSeen: i.lastSeen,
3470
+ test,
3471
+ offline
2203
3472
  };
2204
3473
  }
2205
3474
  var bySeverityThenCount = (a, b) => SEVERITY_ORDER[b.severity] - SEVERITY_ORDER[a.severity] || b.count24h - a.count24h || b.countRange - a.countRange || b.lastSeen.localeCompare(a.lastSeen);
@@ -2222,7 +3491,7 @@ function narrate(s) {
2222
3491
  const n = r.preset === "24h" ? f.errors24h : f.errorsRange;
2223
3492
  out.push(`${f.flow}${f.critical ? " (a critical flow)" : ""} is the most affected flow (${f.severity}): ${plural(n, "error")} in ${r.label} from ${plural(f.issues, "issue")}.${failRate}`);
2224
3493
  }
2225
- const i = s.issues.find((x) => x.status !== "ignored" && x.status !== "resolved");
3494
+ const i = s.issues.find((x) => !x.test && !x.offline && x.status !== "ignored" && x.status !== "resolved");
2226
3495
  if (i && i.countRange > 0) {
2227
3496
  const isNew = Date.parse(s.generatedAt) - Date.parse(i.firstSeen) < 3 * DAY;
2228
3497
  out.push(`Top issue: "${i.title}"${i.culprit ? ` in ${i.culprit}` : ""}, ${plural(i.countRange, "time")} in ${r.label}${isNew ? `, first seen ${relDays(s.generatedAt, i.firstSeen)}` : ""}.`);
@@ -2248,6 +3517,41 @@ function relDays(now, then) {
2248
3517
  const d = Math.floor((Date.parse(now) - Date.parse(then)) / DAY);
2249
3518
  return d === 0 ? "today" : d === 1 ? "yesterday" : `${d} days ago`;
2250
3519
  }
3520
+ /**
3521
+ * The failed call to show on an issue's page, from occurrences newest first. `latest` is the
3522
+ * newest (what's happening now: during an outage, a timeout with no answer); when it captured
3523
+ * nothing, the newest that did. `lastAnswer` is the newest the server answered, when that's not
3524
+ * `latest`: what the server said last time it said anything.
3525
+ */
3526
+ function callsOf(events, hosts) {
3527
+ const calls = events.flatMap((se) => {
3528
+ if (se.event.type !== "error") return [];
3529
+ const api = failedRequest(se.event.request, timelineOf(se.event.breadcrumbs, hosts), hosts);
3530
+ return api ? [{
3531
+ api,
3532
+ exchange: exchangeOf(se.event),
3533
+ eventId: se.event.id,
3534
+ ts: se.event.ts
3535
+ }] : [];
3536
+ });
3537
+ const latest = calls[0]?.exchange ? calls[0] : calls.find((c) => c.exchange) ?? calls[0];
3538
+ const answered = calls.find((c) => c.exchange?.response);
3539
+ return {
3540
+ latest,
3541
+ lastAnswer: answered && answered !== latest && !latest?.exchange?.response ? answered : void 0
3542
+ };
3543
+ }
3544
+ /** The service whose call an issue's newest occurrence failed, or the app's own code. */
3545
+ function areaOf(se, labels, hosts) {
3546
+ if (!labels || !isRequestCategory(labels.category)) return { kind: "app" };
3547
+ const e = se?.event.type === "error" ? se.event : void 0;
3548
+ if (!e) return null;
3549
+ const name = failedRequest(e.request, timelineOf(e.breadcrumbs, hosts), hosts)?.service ?? (e.request ? hostNameFor(e.request.url, hosts)?.host.name : void 0);
3550
+ return name ? {
3551
+ kind: "service",
3552
+ name
3553
+ } : null;
3554
+ }
2251
3555
  async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.now(), opts = {}) {
2252
3556
  const issue = await store.getIssue(app, fingerprint);
2253
3557
  if (!issue) return null;
@@ -2297,6 +3601,7 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
2297
3601
  if (row.labels) row.labels = withIssueCauses(row.labels, { deviceModels: events.map((e) => e.context.device.model) });
2298
3602
  const hints = hintsFor(issue, breakdowns, events, !!row.labels && isRequestCategory(row.labels.category), hosts);
2299
3603
  const errorEvents = events.filter((e) => e.event.type === "error");
3604
+ const backend = row.labels && isRequestCategory(row.labels.category) ? backendSaid(errorEvents.length ? errorEvents : issue.lastEvent ? [issue.lastEvent] : [], hosts) : void 0;
2300
3605
  row.explanation = row.labels ? explain({
2301
3606
  issue: row,
2302
3607
  labels: row.labels,
@@ -2306,10 +3611,17 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
2306
3611
  platforms: [...new Set(errorEvents.map((e) => e.context.os.name))],
2307
3612
  durations: errorEvents.flatMap((e) => e.event.type === "error" && e.event.request?.durationMs !== void 0 ? [e.event.request.durationMs] : []),
2308
3613
  rangeLabel: range.label,
3614
+ backend,
2309
3615
  now
2310
3616
  }) : void 0;
3617
+ const offline = row.offline;
3618
+ const findings = offline ? [] : findingsOf(errorEvents.flatMap((e) => e.event.type === "error" ? [e.event] : []), hosts);
2311
3619
  const link = issueUrl(opts.trackers?.dashboardUrl, app, fingerprint);
3620
+ const bounds = rangeBounds(range, now);
3621
+ const inPeriod = errorEvents.filter((e) => e.event.ts >= bounds.start && e.event.ts <= bounds.end);
3622
+ const calls = row.labels && isRequestCategory(row.labels.category) ? callsOf(inPeriod.length ? inPeriod : errorEvents.length ? errorEvents : [issue.lastEvent], hosts) : void 0;
2312
3623
  const detail = {
3624
+ area: row.test ? { kind: "tests" } : row.offline ? null : areaOf(issue.lastEvent, row.labels, hosts),
2313
3625
  app,
2314
3626
  range,
2315
3627
  retention,
@@ -2320,6 +3632,8 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
2320
3632
  affectedUsers,
2321
3633
  links: {},
2322
3634
  sample: events[0] ?? issue.lastEvent ?? null,
3635
+ calls,
3636
+ backend,
2323
3637
  recent: events.slice(0, 25).map((e) => ({
2324
3638
  id: e.event.id,
2325
3639
  type: e.event.type,
@@ -2331,7 +3645,12 @@ async function buildIssueDetail(store, app, fingerprint, thresholds, now = Date.
2331
3645
  userId: e.user?.id
2332
3646
  })),
2333
3647
  hints,
2334
- ticketMarkdown: row.explanation ? explanationMarkdown(row.title, row.explanation, link) : void 0
3648
+ findings,
3649
+ ...offline ? {
3650
+ bursts: offlineBursts(inRange(events, bounds.start, bounds.end), hosts),
3651
+ burstsCapped: events.length >= 200
3652
+ } : {},
3653
+ ticketMarkdown: row.explanation ? explanationMarkdown(row.title, row.explanation, link, findings) : void 0
2335
3654
  };
2336
3655
  detail.links = trackerLinks(detail, opts.trackers ?? {});
2337
3656
  return detail;
@@ -2359,6 +3678,11 @@ function issueStory(d, dashboardUrl) {
2359
3678
  "Stack (in-app frames):",
2360
3679
  ...frames.map((f) => `- \`${f.fn ?? "?"}\` ${f.file ?? ""}:${f.line ?? ""}`)
2361
3680
  ] : [],
3681
+ ...d.findings.length ? [
3682
+ "",
3683
+ "What the trail shows:",
3684
+ ...d.findings.slice(0, 3).map(findingLine)
3685
+ ] : [],
2362
3686
  ...d.hints.length ? [
2363
3687
  "",
2364
3688
  x ? "Also worth checking:" : "What to look at:",
@@ -2435,15 +3759,4 @@ function hintsFor(issue, b, events, explained = false, hosts = []) {
2435
3759
  });
2436
3760
  return hints;
2437
3761
  }
2438
- // src/server/errors.ts
2439
- /** Firestore answers FAILED_PRECONDITION while an index deBug needs is missing or still building. */
2440
- function isIndexError(message) {
2441
- return /FAILED_PRECONDITION|requires an index|index is currently building/i.test(message);
2442
- }
2443
- /** A thrown Firestore error saying a composite index is missing (gRPC code 9), on the Firestore store only. */
2444
- function isMissingIndex(e, store) {
2445
- if (store !== "firestore") return false;
2446
- const err = e;
2447
- return (err?.code === 9 || err?.code === "failed-precondition") && /index/i.test(String(err?.message ?? ""));
2448
- }
2449
- export { seenHosts as A, ingest as C, teamAction as D, hashInviteToken as E, writeRetention as F, writeSettings as I, getLocalWriteStore as L, testAlert as M, validRetention as N, readSettings as O, writeIssueStatus as P, getStore as R, CORS_HEADERS as S, isRequestCategory as T, hashPassword as _, buildSummary as a, startSession as b, resolveRange as c, canManage as d, checkCredentials as f, forgetMember as g, envOwnerEmails as h, buildIssueDetail as i, settingsWriter as j, restampStep as k, DEMO_USER as l, endSession as m, isMissingIndex as n, endpointTimings as o, demoLoginEnabled as p, SUMMARY_FIELDS as r, retentionFor as s, isIndexError as t, authorizeApi as u, readSession as v, parseIngestKeys as w, tooManyAttempts as x, recordFailure as y, config as z };
3762
+ export { testAlert as $, parseIngestKeys as A, endpointsOf as B, hashPassword as C, tooManyAttempts as D, startSession as E, LIVE_MS as F, serviceTile as G, isOpen as H, OUTAGE as I, appNames as J, servicesOf as K, answerOf as L, hashInviteToken as M, teamAction as N, CORS_HEADERS as O, HEALTH_ORDER as P, settingsWriter as Q, byHealth as R, forgetMember as S, recordFailure as T, pct$1 as U, failing as V, plural$2 as W, restampStep as X, readSettings as Y, seenHosts as Z, canManage as _, retentionFor as a, getLocalWriteStore as at, endSession as b, buildEventDetail as c, buildSessionDetail as d, validRetention as et, sessionOutcome as f, authorizeApi as g, DEMO_USER as h, endpointTimings as i, clearStoreCache as it, isRequestCategory as j, ingest as k, failedRequest as l, isMissingIndex as m, buildIssueDetail as n, writeRetention as nt, resolveRange as o, getStore as ot, isIndexError as p, whoseLine as q, buildSummary as r, writeSettings as rt, apiCallOf as s, SUMMARY_FIELDS as t, writeIssueStatus as tt, timelineOf as u, checkCredentials as v, readSession as w, envOwnerEmails as x, demoLoginEnabled as y, endpointLevel as z };