@aglyn/tenant-data-admin 1.0.0-beta.221 → 1.0.0-beta.223

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/package.json +7 -7
  2. package/src/lib/server/api-keys.js +3 -3
  3. package/src/lib/server/api-keys.js.map +1 -1
  4. package/src/lib/server/api-v1-resources.d.ts +44 -3
  5. package/src/lib/server/api-v1-resources.js +82 -5
  6. package/src/lib/server/api-v1-resources.js.map +1 -1
  7. package/src/lib/server/collection-live-pages.d.ts +2 -2
  8. package/src/lib/server/collection-live-pages.js +1 -1
  9. package/src/lib/server/collection-live-pages.js.map +1 -1
  10. package/src/lib/server/email-delivery-log.d.ts +17 -73
  11. package/src/lib/server/email-delivery-log.js +44 -222
  12. package/src/lib/server/email-delivery-log.js.map +1 -1
  13. package/src/lib/server/email-flow-gate.js +27 -38
  14. package/src/lib/server/email-flow-gate.js.map +1 -1
  15. package/src/lib/server/erase-person.d.ts +34 -54
  16. package/src/lib/server/erase-person.js +44 -239
  17. package/src/lib/server/erase-person.js.map +1 -1
  18. package/src/lib/server/ga4-measurement-protocol.js +41 -28
  19. package/src/lib/server/ga4-measurement-protocol.js.map +1 -1
  20. package/src/lib/server/host-visitor-records.d.ts +25 -4
  21. package/src/lib/server/host-visitor-records.js +39 -11
  22. package/src/lib/server/host-visitor-records.js.map +1 -1
  23. package/src/lib/server/hosted-page-review.js +52 -36
  24. package/src/lib/server/hosted-page-review.js.map +1 -1
  25. package/src/lib/server/link-reputation-review.d.ts +6 -2
  26. package/src/lib/server/link-reputation-review.js +61 -20
  27. package/src/lib/server/link-reputation-review.js.map +1 -1
  28. package/src/lib/server/live-page-drops.d.ts +67 -0
  29. package/src/lib/server/live-page-drops.js +57 -0
  30. package/src/lib/server/live-page-drops.js.map +1 -0
  31. package/src/lib/server/media-bandwidth-cap.d.ts +78 -0
  32. package/src/lib/server/media-bandwidth-cap.js +180 -0
  33. package/src/lib/server/media-bandwidth-cap.js.map +1 -0
  34. package/src/lib/server/media-cdn-icon.d.ts +77 -0
  35. package/src/lib/server/media-cdn-icon.js +140 -0
  36. package/src/lib/server/media-cdn-icon.js.map +1 -0
  37. package/src/lib/server/media-delivery-regeneration.d.ts +132 -0
  38. package/src/lib/server/media-delivery-regeneration.js +216 -0
  39. package/src/lib/server/media-delivery-regeneration.js.map +1 -0
  40. package/src/lib/server/media-delivery.d.ts +2 -0
  41. package/src/lib/server/media-delivery.js +2 -1
  42. package/src/lib/server/media-delivery.js.map +1 -1
  43. package/src/lib/server/media-serve-count.d.ts +50 -0
  44. package/src/lib/server/media-serve-count.js +62 -0
  45. package/src/lib/server/media-serve-count.js.map +1 -0
  46. package/src/lib/server/media-signing.d.ts +13 -2
  47. package/src/lib/server/media-signing.js +12 -8
  48. package/src/lib/server/media-signing.js.map +1 -1
  49. package/src/lib/server/media-tombstone.js +7 -5
  50. package/src/lib/server/media-tombstone.js.map +1 -1
  51. package/src/lib/server/media-variants.d.ts +151 -20
  52. package/src/lib/server/media-variants.js +204 -27
  53. package/src/lib/server/media-variants.js.map +1 -1
  54. package/src/lib/server/organizations.d.ts +102 -2
  55. package/src/lib/server/organizations.js +388 -58
  56. package/src/lib/server/organizations.js.map +1 -1
  57. package/src/lib/server/outbound-send-review.js +11 -7
  58. package/src/lib/server/outbound-send-review.js.map +1 -1
  59. package/src/lib/server/paged-sweeps.d.ts +51 -0
  60. package/src/lib/server/paged-sweeps.js +79 -0
  61. package/src/lib/server/paged-sweeps.js.map +1 -0
  62. package/src/lib/server/payment-fraud-signal.d.ts +49 -9
  63. package/src/lib/server/payment-fraud-signal.js +192 -48
  64. package/src/lib/server/payment-fraud-signal.js.map +1 -1
  65. package/src/lib/server/platform-marketing-consent.d.ts +0 -49
  66. package/src/lib/server/platform-marketing-consent.js +27 -11
  67. package/src/lib/server/platform-marketing-consent.js.map +1 -1
  68. package/src/lib/server/render-monitor.d.ts +166 -0
  69. package/src/lib/server/render-monitor.js +412 -0
  70. package/src/lib/server/render-monitor.js.map +1 -0
  71. package/src/lib/server/risk-notice.d.ts +7 -0
  72. package/src/lib/server/risk-notice.js +6 -0
  73. package/src/lib/server/risk-notice.js.map +1 -1
  74. package/src/lib/server/serve-media-cdn.d.ts +69 -1
  75. package/src/lib/server/serve-media-cdn.js +424 -70
  76. package/src/lib/server/serve-media-cdn.js.map +1 -1
  77. package/src/lib/server/site-icon-render.d.ts +68 -0
  78. package/src/lib/server/site-icon-render.js +157 -0
  79. package/src/lib/server/site-icon-render.js.map +1 -0
  80. package/src/lib/server/staff-seat.d.ts +50 -0
  81. package/src/lib/server/staff-seat.js +118 -0
  82. package/src/lib/server/staff-seat.js.map +1 -0
  83. package/src/lib/server/transfer-export.d.ts +46 -0
  84. package/src/lib/server/transfer-export.js +221 -0
  85. package/src/lib/server/transfer-export.js.map +1 -0
  86. package/src/lib/server/transfer-jobs.d.ts +385 -0
  87. package/src/lib/server/transfer-jobs.js +2368 -0
  88. package/src/lib/server/transfer-jobs.js.map +1 -0
  89. package/src/lib/server/transfer-packages.d.ts +127 -0
  90. package/src/lib/server/transfer-packages.js +1036 -0
  91. package/src/lib/server/transfer-packages.js.map +1 -0
  92. package/src/lib/server/upgrade-proposal.d.ts +97 -0
  93. package/src/lib/server/upgrade-proposal.js +312 -0
  94. package/src/lib/server/upgrade-proposal.js.map +1 -0
  95. package/src/lib/server/upsert-contact.d.ts +17 -8
  96. package/src/lib/server/upsert-contact.js +82 -16
  97. package/src/lib/server/upsert-contact.js.map +1 -1
  98. package/src/lib/server/web-risk.d.ts +50 -9
  99. package/src/lib/server/web-risk.js +290 -155
  100. package/src/lib/server/web-risk.js.map +1 -1
  101. package/src/tenancy.d.ts +3 -3
  102. package/src/tenancy.js +5 -3
  103. package/src/tenancy.js.map +1 -1
  104. package/src/lib/server/campaign-attribution-store.d.ts +0 -55
  105. package/src/lib/server/campaign-attribution-store.js +0 -95
  106. package/src/lib/server/campaign-attribution-store.js.map +0 -1
  107. package/src/lib/server/campaign-conversion-attribution.d.ts +0 -283
  108. package/src/lib/server/campaign-conversion-attribution.js +0 -450
  109. package/src/lib/server/campaign-conversion-attribution.js.map +0 -1
  110. package/src/lib/server/crm-email-activity.d.ts +0 -118
  111. package/src/lib/server/crm-email-activity.js +0 -148
  112. package/src/lib/server/crm-email-activity.js.map +0 -1
  113. package/src/lib/server/crm-next-activity.d.ts +0 -74
  114. package/src/lib/server/crm-next-activity.js +0 -194
  115. package/src/lib/server/crm-next-activity.js.map +0 -1
  116. package/src/lib/server/dataset-live-pages.d.ts +0 -179
  117. package/src/lib/server/dataset-live-pages.js +0 -421
  118. package/src/lib/server/dataset-live-pages.js.map +0 -1
  119. package/src/lib/server/email-revenue-attribution.d.ts +0 -185
  120. package/src/lib/server/email-revenue-attribution.js +0 -321
  121. package/src/lib/server/email-revenue-attribution.js.map +0 -1
@@ -0,0 +1,412 @@
1
+ import { _ as _extends } from "@swc/helpers/_/_extends";
2
+ /**
3
+ * @license
4
+ * Copyright 2026 Aglyn LLC
5
+ *
6
+ * Licensed under the Apache License, Version 2.0 (the "License");
7
+ * you may not use this file except in compliance with the License.
8
+ * You may obtain a copy of the License at
9
+ *
10
+ * http://www.apache.org/licenses/LICENSE-2.0
11
+ *
12
+ * Unless required by applicable law or agreed to in writing, software
13
+ * distributed under the License is distributed on an "AS IS" BASIS,
14
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ * See the License for the specific language governing permissions and
16
+ * limitations under the License.
17
+ */ import { TENANT_APEX } from "@aglyn/aglyn/app-utils/tenant-apex";
18
+ /**
19
+ * THE RENDER MONITOR (AGL-3568): does a published site still render a page
20
+ * that was not already in a cache?
21
+ *
22
+ * On 2026-10-05 every uncached page render on the published-site runtime
23
+ * hung to the platform's 60 s limit for 28 minutes, and nothing alerted. The
24
+ * three monitors that existed each missed it for its own reason:
25
+ *
26
+ * - the GitHub uptime probe is scheduled every 15 minutes and GitHub ran it
27
+ * four times that day, none of them during the outage;
28
+ * - its page rows fetch `/`, which the ISR cache answers, so a visitor's
29
+ * page that was cached still loaded and the row stayed green;
30
+ * - `/api/health/render/site` runs the page LOADER in-process and never the
31
+ * site layout, which is where the request was stuck.
32
+ *
33
+ * So this fetches pages over HTTP, from the console (a different deployment
34
+ * from the one it watches), on Cloud Scheduler every five minutes, and asks
35
+ * for two pages no cache can answer:
36
+ *
37
+ * - `dynamic` — `/search`, a route every site has that renders on every
38
+ * request (`force-dynamic`) inside the full site layout. A 200 that is a
39
+ * complete document proves the layout and head ran just now.
40
+ * - `isr` — a path nobody has ever requested, under the catch-all page.
41
+ * A never-seen path is the one request the ISR cache cannot answer, so it
42
+ * renders in the same static context every published page renders in,
43
+ * which is the context the 2026-09-09 outage only appeared in. It is a
44
+ * 404 by construction, and the site's own not-found page is drawn inside
45
+ * the same layout, so a complete 404 document is a pass.
46
+ *
47
+ * A site that fails `threshold` runs in a row raises
48
+ * `system.siteRenderFailing` once; the first passing run after that raises
49
+ * `system.siteRenderRecovered` once. The state lives in
50
+ * `operatorHealthState/render-monitor--<host>`, beside every other health
51
+ * check, so Staff → Operator alerts lists it with its status and since when.
52
+ *
53
+ * Nothing here names an Aglyn host: the default target is the install's own
54
+ * demonstration site label under its own tenant apex, the convention the
55
+ * middleware and the render canaries already follow.
56
+ */ /** The state documents' collection: the health checks' own. */ const RENDER_MONITOR_COLLECTION = 'operatorHealthState';
57
+ /** Every render monitor document id starts with this. */ export const RENDER_MONITOR_CHECK_PREFIX = 'render-monitor--';
58
+ /** The beat id `/api/health/crons` reads for the scheduled run. */ export const RENDER_MONITOR_JOB_ID = 'render-monitor';
59
+ /**
60
+ * How long one page fetch may take. Well under the 60 s a platform function
61
+ * may run, so a hung render is graded here as a timeout rather than waiting
62
+ * for the platform's 504, and well under the route's own `maxDuration`.
63
+ */ export const RENDER_MONITOR_TIMEOUT_MS = 25000;
64
+ /** Failing runs in a row before the alert goes out. */ export const RENDER_MONITOR_DEFAULT_THRESHOLD = 2;
65
+ /**
66
+ * The prefix of the never-requested path the `isr` probe asks for. A
67
+ * lowercase slug on purpose: it must reach the catch-all page like any
68
+ * screen slug would, not a reserved or rewritten route.
69
+ */ export const RENDER_PROBE_ISR_PREFIX = '/aglyn-render-probe-';
70
+ export const RENDER_PROBES = [
71
+ {
72
+ kind: 'dynamic',
73
+ path: ()=>'/search',
74
+ expectStatus: 200
75
+ },
76
+ {
77
+ kind: 'isr',
78
+ path: (nonce)=>`${RENDER_PROBE_ISR_PREFIX}${nonce}`,
79
+ expectStatus: 404
80
+ }
81
+ ];
82
+ /** Vercel's challenge interstitial, matched on the body like the front doors. */ const CHECKPOINT_MARKER = /Vercel Security Checkpoint/i;
83
+ /**
84
+ * Cache states that mean the bytes came out of a store. A probe meant to
85
+ * prove a render cannot pass on one of these, whatever the status.
86
+ */ const CACHED_STATES = new Set([
87
+ 'HIT',
88
+ 'STALE',
89
+ 'PRERENDER'
90
+ ]);
91
+ /**
92
+ * Grade one probe response. Pure.
93
+ *
94
+ * The page markers are the front doors' (`tools/scripts/lib/front-door.mjs`)
95
+ * and for their reason: structural, never site copy. `</html>` proves the
96
+ * document finished streaming, `/_next/static/` that our build produced it.
97
+ */ export function gradeRenderProbe(response, expectStatus) {
98
+ var _response_body, _response_contentType, _response_cacheState;
99
+ const body = (_response_body = response.body) != null ? _response_body : '';
100
+ if (CHECKPOINT_MARKER.test(body)) {
101
+ return {
102
+ ok: false,
103
+ challenged: true,
104
+ code: 'challenged',
105
+ detail: `bot protection answered (HTTP ${response.status}) instead of the site; check AGLYN_PROBE_TOKEN on the console`
106
+ };
107
+ }
108
+ if (response.status >= 300 && response.status < 400) {
109
+ var _response_location;
110
+ return {
111
+ ok: false,
112
+ challenged: false,
113
+ code: 'redirected',
114
+ detail: `redirected to ${(_response_location = response.location) != null ? _response_location : '?'}, so no page was rendered`
115
+ };
116
+ }
117
+ if (response.status !== expectStatus) {
118
+ return {
119
+ ok: false,
120
+ challenged: false,
121
+ code: `http-${response.status}`,
122
+ detail: `HTTP ${response.status}`
123
+ };
124
+ }
125
+ if (!/text\/html/i.test((_response_contentType = response.contentType) != null ? _response_contentType : '')) {
126
+ return {
127
+ ok: false,
128
+ challenged: false,
129
+ code: 'not-html',
130
+ detail: `not HTML (${response.contentType || 'no content-type'})`
131
+ };
132
+ }
133
+ if (!body.includes('</html>')) {
134
+ return {
135
+ ok: false,
136
+ challenged: false,
137
+ code: 'incomplete',
138
+ detail: 'the document never finished (no </html>)'
139
+ };
140
+ }
141
+ if (!body.includes('/_next/static/')) {
142
+ return {
143
+ ok: false,
144
+ challenged: false,
145
+ code: 'not-our-render',
146
+ detail: 'no Next asset references, so the app did not render it'
147
+ };
148
+ }
149
+ const cache = String((_response_cacheState = response.cacheState) != null ? _response_cacheState : '').trim().toUpperCase();
150
+ if (CACHED_STATES.has(cache)) {
151
+ return {
152
+ ok: false,
153
+ challenged: false,
154
+ code: 'served-from-cache',
155
+ detail: `served from cache (${cache}), so it proves no render`
156
+ };
157
+ }
158
+ return {
159
+ ok: true,
160
+ challenged: false,
161
+ code: 'ok',
162
+ detail: 'rendered'
163
+ };
164
+ }
165
+ /** A configured value as an origin, or '' when it is not one. */ export function originOf(value) {
166
+ const trimmed = String(value != null ? value : '').trim().replace(/\/+$/, '');
167
+ return /^https?:\/\/[^/\s?#]+$/i.test(trimmed) ? trimmed.toLowerCase() : '';
168
+ }
169
+ /**
170
+ * Which sites the monitor renders, from the console's environment.
171
+ *
172
+ * 1. The install's demonstration site: `OPERATOR_HEALTH_TENANT_ORIGIN` when
173
+ * set (the one published site the operator alerts tick already watches),
174
+ * else `https://<AGLYN_TENANT_DEMO or demo>.<tenant apex>`.
175
+ * 2. Plus every origin in `RENDER_MONITOR_ORIGINS`, comma separated.
176
+ *
177
+ * `RENDER_MONITOR_ORIGINS=off` watches nothing, for an install with no
178
+ * published site to watch. Values that are not origins are dropped.
179
+ */ export function resolveRenderMonitorTargets(env) {
180
+ var _env_RENDER_MONITOR_ORIGINS, _env_NEXT_PUBLIC_TENANT_DOMAIN, _env_AGLYN_TENANT_DEMO;
181
+ const raw = String((_env_RENDER_MONITOR_ORIGINS = env['RENDER_MONITOR_ORIGINS']) != null ? _env_RENDER_MONITOR_ORIGINS : '').trim();
182
+ if (/^(off|none|false|0)$/i.test(raw)) return [];
183
+ const apex = String((_env_NEXT_PUBLIC_TENANT_DOMAIN = env['NEXT_PUBLIC_TENANT_DOMAIN']) != null ? _env_NEXT_PUBLIC_TENANT_DOMAIN : '').trim() || TENANT_APEX;
184
+ const label = String((_env_AGLYN_TENANT_DEMO = env['AGLYN_TENANT_DEMO']) != null ? _env_AGLYN_TENANT_DEMO : '').trim() || 'demo';
185
+ const demo = originOf(env['OPERATOR_HEALTH_TENANT_ORIGIN']) || originOf(`https://${label}.${apex}`);
186
+ const listed = raw.split(/[\s,]+/).map((value)=>originOf(value)).filter(Boolean);
187
+ return [
188
+ ...new Set([
189
+ demo,
190
+ ...listed
191
+ ].filter(Boolean))
192
+ ];
193
+ }
194
+ /** The state document id for one origin. */ export function renderMonitorCheckId(origin) {
195
+ return `${RENDER_MONITOR_CHECK_PREFIX}${new URL(origin).host}`;
196
+ }
197
+ /**
198
+ * The next state for one site, given what this run saw. Pure.
199
+ *
200
+ * The status turns `degraded` only once `threshold` runs in a row have
201
+ * failed, so one slow cold start or one dropped connection says nothing. It
202
+ * turns back on the first passing run, because a render that works is
203
+ * proof, and a recovery held back is a recovery nobody hears about.
204
+ */ export function nextRenderMonitorState(prior, observation, context) {
205
+ var _observation_detail, _ref, _ref1, _ref2, _ref3;
206
+ const { origin, label, threshold, now } = context;
207
+ const checkId = renderMonitorCheckId(origin);
208
+ const detail = String((_observation_detail = observation.detail) != null ? _observation_detail : '').slice(0, 1000);
209
+ const wasDegraded = (prior == null ? void 0 : prior.status) === 'degraded';
210
+ if (observation.ok) {
211
+ return {
212
+ next: {
213
+ checkId,
214
+ label,
215
+ origin,
216
+ status: 'ok',
217
+ sinceMs: !wasDegraded && (prior == null ? void 0 : prior.sinceMs) ? prior.sinceMs : now,
218
+ detail,
219
+ updatedAtMs: now,
220
+ consecutiveFailures: 0,
221
+ failingSinceMs: null,
222
+ lastOkAtMs: now
223
+ },
224
+ transition: wasDegraded ? 'recovered' : null
225
+ };
226
+ }
227
+ const failures = ((_ref = prior == null ? void 0 : prior.consecutiveFailures) != null ? _ref : 0) + 1;
228
+ const failingSinceMs = (_ref1 = prior == null ? void 0 : prior.failingSinceMs) != null ? _ref1 : now;
229
+ const degraded = wasDegraded || failures >= Math.max(1, threshold);
230
+ return {
231
+ next: {
232
+ checkId,
233
+ label,
234
+ origin,
235
+ status: degraded ? 'degraded' : 'ok',
236
+ sinceMs: degraded ? wasDegraded && (prior == null ? void 0 : prior.sinceMs) ? prior.sinceMs : failingSinceMs : (_ref2 = prior == null ? void 0 : prior.sinceMs) != null ? _ref2 : now,
237
+ detail,
238
+ updatedAtMs: now,
239
+ consecutiveFailures: failures,
240
+ failingSinceMs,
241
+ lastOkAtMs: (_ref3 = prior == null ? void 0 : prior.lastOkAtMs) != null ? _ref3 : null
242
+ },
243
+ transition: degraded && !wasDegraded ? 'failing' : null
244
+ };
245
+ }
246
+ /**
247
+ * The default store: one Firestore transaction on the site's document, so
248
+ * two overlapping runs cannot both decide they saw the edge.
249
+ */ const firestoreStore = async (checkId, work)=>{
250
+ const { default: firebaseAdmin } = await import("./firebase-admin.js");
251
+ const db = firebaseAdmin.app().firestore();
252
+ const ref = db.collection(RENDER_MONITOR_COLLECTION).doc(checkId);
253
+ return db.runTransaction(async (transaction)=>{
254
+ const snapshot = await transaction.get(ref);
255
+ const prior = snapshot.exists ? snapshot.data() : null;
256
+ const { next, result } = work(prior);
257
+ transaction.set(ref, next);
258
+ return result;
259
+ });
260
+ };
261
+ const defaultRaise = async (type, options)=>{
262
+ const { raiseOperatorAlert } = await import("./operator-alerts.js");
263
+ return raiseOperatorAlert(type, options);
264
+ };
265
+ function describeDuration(ms) {
266
+ const minutes = Math.max(1, Math.round(ms / 60000));
267
+ if (minutes < 60) return `${minutes} min`;
268
+ const hours = Math.floor(minutes / 60);
269
+ const rest = minutes % 60;
270
+ return rest ? `${hours} h ${rest} min` : `${hours} h`;
271
+ }
272
+ function probeLine(row) {
273
+ return `${row.kind} ${new URL(row.url).pathname}: ${row.detail} (${row.ms} ms)`;
274
+ }
275
+ /** Fetch and grade one probe. Never throws. */ async function runProbe(origin, spec, options) {
276
+ const url = `${origin}${spec.path(options.nonce)}`;
277
+ const startedAt = options.clock();
278
+ try {
279
+ var _response_headers_get;
280
+ const response = await options.fetcher(url, {
281
+ redirect: 'manual',
282
+ cache: 'no-store',
283
+ headers: _extends({
284
+ 'user-agent': 'aglyn-render-monitor',
285
+ accept: 'text/html,application/xhtml+xml'
286
+ }, options.headers),
287
+ signal: AbortSignal.timeout(options.timeoutMs)
288
+ });
289
+ const body = await response.text();
290
+ const verdict = gradeRenderProbe({
291
+ status: response.status,
292
+ contentType: response.headers.get('content-type'),
293
+ body,
294
+ location: response.headers.get('location'),
295
+ cacheState: (_response_headers_get = response.headers.get('x-vercel-cache')) != null ? _response_headers_get : response.headers.get('x-nextjs-cache')
296
+ }, spec.expectStatus);
297
+ return _extends({
298
+ kind: spec.kind,
299
+ url,
300
+ ms: options.clock() - startedAt,
301
+ httpStatus: response.status
302
+ }, verdict);
303
+ } catch (error) {
304
+ var _ref, _ref1;
305
+ var _error_cause;
306
+ const name = error == null ? void 0 : error.name;
307
+ const timedOut = name === 'TimeoutError' || name === 'AbortError';
308
+ return {
309
+ kind: spec.kind,
310
+ url,
311
+ ms: options.clock() - startedAt,
312
+ httpStatus: null,
313
+ ok: false,
314
+ challenged: false,
315
+ code: timedOut ? 'timeout' : 'unreachable',
316
+ detail: timedOut ? `no complete page in ${Math.round(options.timeoutMs / 1000)} s` : `unreachable (${(_ref = (_ref1 = error == null ? void 0 : (_error_cause = error.cause) == null ? void 0 : _error_cause.code) != null ? _ref1 : name) != null ? _ref : 'error'})`
317
+ };
318
+ }
319
+ }
320
+ /**
321
+ * One run: render every target, record each site's state, raise the edge.
322
+ *
323
+ * A dry run fetches and grades and writes nothing, so a person can ask
324
+ * "what would the monitor see right now" without moving its state.
325
+ */ export async function runRenderMonitor(options = {}) {
326
+ var _options_env, _options_now, _options_store, _options_raise, _env_RENDER_MONITOR_FAILURE_THRESHOLD, _options_nonce;
327
+ const env = (_options_env = options.env) != null ? _options_env : process.env;
328
+ const clock = (_options_now = options.now) != null ? _options_now : Date.now;
329
+ const store = (_options_store = options.store) != null ? _options_store : firestoreStore;
330
+ const raise = (_options_raise = options.raise) != null ? _options_raise : defaultRaise;
331
+ const dryRun = Boolean(options.dryRun);
332
+ const parsedThreshold = Number.parseInt(String((_env_RENDER_MONITOR_FAILURE_THRESHOLD = env['RENDER_MONITOR_FAILURE_THRESHOLD']) != null ? _env_RENDER_MONITOR_FAILURE_THRESHOLD : ''), 10);
333
+ const threshold = Number.isFinite(parsedThreshold) && parsedThreshold >= 1 ? parsedThreshold : RENDER_MONITOR_DEFAULT_THRESHOLD;
334
+ const nonce = (_options_nonce = options.nonce) != null ? _options_nonce : ()=>`${clock().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
335
+ const sites = await Promise.all(resolveRenderMonitorTargets(env).map(async (origin)=>{
336
+ const probes = await Promise.all(RENDER_PROBES.map((spec)=>{
337
+ var _options_fetcher, _options_headers, _options_timeoutMs;
338
+ return runProbe(origin, spec, {
339
+ fetcher: (_options_fetcher = options.fetcher) != null ? _options_fetcher : fetch,
340
+ headers: (_options_headers = options.headers) != null ? _options_headers : {},
341
+ nonce: nonce(),
342
+ timeoutMs: (_options_timeoutMs = options.timeoutMs) != null ? _options_timeoutMs : RENDER_MONITOR_TIMEOUT_MS,
343
+ clock
344
+ });
345
+ }));
346
+ const ok = probes.every((probe)=>probe.ok);
347
+ const site = {
348
+ origin,
349
+ ok,
350
+ probes
351
+ };
352
+ if (dryRun) return site;
353
+ const host = new URL(origin).host;
354
+ const failing = probes.filter((probe)=>!probe.ok);
355
+ const detail = ok ? 'Fresh renders pass.' : failing.map(probeLine).join('; ');
356
+ try {
357
+ const now = clock();
358
+ const { next, transition, failingSinceMs } = await store(renderMonitorCheckId(origin), (prior)=>{
359
+ var _ref, _ref1;
360
+ const decided = nextRenderMonitorState(prior, {
361
+ ok,
362
+ detail
363
+ }, {
364
+ origin,
365
+ label: `Page rendering on ${host}`,
366
+ threshold,
367
+ now
368
+ });
369
+ return {
370
+ next: decided.next,
371
+ result: _extends({}, decided, {
372
+ // When the failures a recovery ends began.
373
+ failingSinceMs: (_ref = (_ref1 = prior == null ? void 0 : prior.failingSinceMs) != null ? _ref1 : prior == null ? void 0 : prior.sinceMs) != null ? _ref : now
374
+ })
375
+ };
376
+ });
377
+ site.status = next.status;
378
+ site.consecutiveFailures = next.consecutiveFailures;
379
+ site.transition = transition;
380
+ if (transition === 'failing') {
381
+ await raise('system.siteRenderFailing', {
382
+ dedupeKey: host,
383
+ context: {
384
+ site: host,
385
+ failures: next.consecutiveFailures,
386
+ since: describeDuration(now - next.sinceMs),
387
+ detail
388
+ }
389
+ });
390
+ } else if (transition === 'recovered') {
391
+ await raise('system.siteRenderRecovered', {
392
+ dedupeKey: host,
393
+ context: {
394
+ site: host,
395
+ duration: describeDuration(now - failingSinceMs)
396
+ }
397
+ });
398
+ }
399
+ } catch (error) {
400
+ site.error = error instanceof Error ? error.message : String(error);
401
+ console.error(`[render-monitor] ${host} could not be recorded`, error);
402
+ }
403
+ return site;
404
+ }));
405
+ return {
406
+ dryRun,
407
+ threshold,
408
+ sites
409
+ };
410
+ }
411
+
412
+ //# sourceMappingURL=render-monitor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/render-monitor.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { TENANT_APEX } from '@aglyn/aglyn/app-utils/tenant-apex'\nimport type { OperatorHealthStateDoc } from './operator-health'\n\n/**\n * THE RENDER MONITOR (AGL-3568): does a published site still render a page\n * that was not already in a cache?\n *\n * On 2026-10-05 every uncached page render on the published-site runtime\n * hung to the platform's 60 s limit for 28 minutes, and nothing alerted. The\n * three monitors that existed each missed it for its own reason:\n *\n * - the GitHub uptime probe is scheduled every 15 minutes and GitHub ran it\n * four times that day, none of them during the outage;\n * - its page rows fetch `/`, which the ISR cache answers, so a visitor's\n * page that was cached still loaded and the row stayed green;\n * - `/api/health/render/site` runs the page LOADER in-process and never the\n * site layout, which is where the request was stuck.\n *\n * So this fetches pages over HTTP, from the console (a different deployment\n * from the one it watches), on Cloud Scheduler every five minutes, and asks\n * for two pages no cache can answer:\n *\n * - `dynamic` — `/search`, a route every site has that renders on every\n * request (`force-dynamic`) inside the full site layout. A 200 that is a\n * complete document proves the layout and head ran just now.\n * - `isr` — a path nobody has ever requested, under the catch-all page.\n * A never-seen path is the one request the ISR cache cannot answer, so it\n * renders in the same static context every published page renders in,\n * which is the context the 2026-09-09 outage only appeared in. It is a\n * 404 by construction, and the site's own not-found page is drawn inside\n * the same layout, so a complete 404 document is a pass.\n *\n * A site that fails `threshold` runs in a row raises\n * `system.siteRenderFailing` once; the first passing run after that raises\n * `system.siteRenderRecovered` once. The state lives in\n * `operatorHealthState/render-monitor--<host>`, beside every other health\n * check, so Staff → Operator alerts lists it with its status and since when.\n *\n * Nothing here names an Aglyn host: the default target is the install's own\n * demonstration site label under its own tenant apex, the convention the\n * middleware and the render canaries already follow.\n */\n\n/** The state documents' collection: the health checks' own. */\nconst RENDER_MONITOR_COLLECTION = 'operatorHealthState'\n\n/** Every render monitor document id starts with this. */\nexport const RENDER_MONITOR_CHECK_PREFIX = 'render-monitor--'\n\n/** The beat id `/api/health/crons` reads for the scheduled run. */\nexport const RENDER_MONITOR_JOB_ID = 'render-monitor'\n\n/**\n * How long one page fetch may take. Well under the 60 s a platform function\n * may run, so a hung render is graded here as a timeout rather than waiting\n * for the platform's 504, and well under the route's own `maxDuration`.\n */\nexport const RENDER_MONITOR_TIMEOUT_MS = 25_000\n\n/** Failing runs in a row before the alert goes out. */\nexport const RENDER_MONITOR_DEFAULT_THRESHOLD = 2\n\n/**\n * The prefix of the never-requested path the `isr` probe asks for. A\n * lowercase slug on purpose: it must reach the catch-all page like any\n * screen slug would, not a reserved or rewritten route.\n */\nexport const RENDER_PROBE_ISR_PREFIX = '/aglyn-render-probe-'\n\nexport type RenderProbeKind = 'dynamic' | 'isr'\n\nexport interface RenderProbeSpec {\n kind: RenderProbeKind\n /** The path to request, given a fresh nonce. */\n path: (nonce: string) => string\n /** The status a healthy render answers with. */\n expectStatus: number\n}\n\nexport const RENDER_PROBES: readonly RenderProbeSpec[] = [\n { kind: 'dynamic', path: () => '/search', expectStatus: 200 },\n {\n kind: 'isr',\n path: (nonce) => `${RENDER_PROBE_ISR_PREFIX}${nonce}`,\n expectStatus: 404,\n },\n]\n\n/** One fetched response, reduced to what grading reads. */\nexport interface RenderProbeResponse {\n status: number\n contentType?: string | null\n body?: string\n location?: string | null\n /** `x-vercel-cache` on Vercel, `x-nextjs-cache` on a plain `next start`. */\n cacheState?: string | null\n}\n\nexport interface RenderProbeVerdict {\n ok: boolean\n /** Bot protection answered instead of the site. */\n challenged: boolean\n /** A stable code; `ok` on a pass. */\n code: string\n detail: string\n}\n\n/** Vercel's challenge interstitial, matched on the body like the front doors. */\nconst CHECKPOINT_MARKER = /Vercel Security Checkpoint/i\n\n/**\n * Cache states that mean the bytes came out of a store. A probe meant to\n * prove a render cannot pass on one of these, whatever the status.\n */\nconst CACHED_STATES = new Set(['HIT', 'STALE', 'PRERENDER'])\n\n/**\n * Grade one probe response. Pure.\n *\n * The page markers are the front doors' (`tools/scripts/lib/front-door.mjs`)\n * and for their reason: structural, never site copy. `</html>` proves the\n * document finished streaming, `/_next/static/` that our build produced it.\n */\nexport function gradeRenderProbe(\n response: RenderProbeResponse,\n expectStatus: number,\n): RenderProbeVerdict {\n const body = response.body ?? ''\n if (CHECKPOINT_MARKER.test(body)) {\n return {\n ok: false,\n challenged: true,\n code: 'challenged',\n detail: `bot protection answered (HTTP ${response.status}) instead of the site; check AGLYN_PROBE_TOKEN on the console`,\n }\n }\n if (response.status >= 300 && response.status < 400) {\n return {\n ok: false,\n challenged: false,\n code: 'redirected',\n detail: `redirected to ${response.location ?? '?'}, so no page was rendered`,\n }\n }\n if (response.status !== expectStatus) {\n return {\n ok: false,\n challenged: false,\n code: `http-${response.status}`,\n detail: `HTTP ${response.status}`,\n }\n }\n if (!/text\\/html/i.test(response.contentType ?? '')) {\n return {\n ok: false,\n challenged: false,\n code: 'not-html',\n detail: `not HTML (${response.contentType || 'no content-type'})`,\n }\n }\n if (!body.includes('</html>')) {\n return {\n ok: false,\n challenged: false,\n code: 'incomplete',\n detail: 'the document never finished (no </html>)',\n }\n }\n if (!body.includes('/_next/static/')) {\n return {\n ok: false,\n challenged: false,\n code: 'not-our-render',\n detail: 'no Next asset references, so the app did not render it',\n }\n }\n const cache = String(response.cacheState ?? '').trim().toUpperCase()\n if (CACHED_STATES.has(cache)) {\n return {\n ok: false,\n challenged: false,\n code: 'served-from-cache',\n detail: `served from cache (${cache}), so it proves no render`,\n }\n }\n return { ok: true, challenged: false, code: 'ok', detail: 'rendered' }\n}\n\n/** A configured value as an origin, or '' when it is not one. */\nexport function originOf(value: string | undefined | null): string {\n const trimmed = String(value ?? '').trim().replace(/\\/+$/, '')\n return /^https?:\\/\\/[^/\\s?#]+$/i.test(trimmed) ? trimmed.toLowerCase() : ''\n}\n\n/**\n * Which sites the monitor renders, from the console's environment.\n *\n * 1. The install's demonstration site: `OPERATOR_HEALTH_TENANT_ORIGIN` when\n * set (the one published site the operator alerts tick already watches),\n * else `https://<AGLYN_TENANT_DEMO or demo>.<tenant apex>`.\n * 2. Plus every origin in `RENDER_MONITOR_ORIGINS`, comma separated.\n *\n * `RENDER_MONITOR_ORIGINS=off` watches nothing, for an install with no\n * published site to watch. Values that are not origins are dropped.\n */\nexport function resolveRenderMonitorTargets(\n env: Readonly<Record<string, string | undefined>>,\n): string[] {\n const raw = String(env['RENDER_MONITOR_ORIGINS'] ?? '').trim()\n if (/^(off|none|false|0)$/i.test(raw)) return []\n const apex =\n String(env['NEXT_PUBLIC_TENANT_DOMAIN'] ?? '').trim() || TENANT_APEX\n const label = String(env['AGLYN_TENANT_DEMO'] ?? '').trim() || 'demo'\n const demo =\n originOf(env['OPERATOR_HEALTH_TENANT_ORIGIN']) ||\n originOf(`https://${label}.${apex}`)\n const listed = raw\n .split(/[\\s,]+/)\n .map((value) => originOf(value))\n .filter(Boolean)\n return [...new Set([demo, ...listed].filter(Boolean))]\n}\n\n/** The state document id for one origin. */\nexport function renderMonitorCheckId(origin: string): string {\n return `${RENDER_MONITOR_CHECK_PREFIX}${new URL(origin).host}`\n}\n\n/** One render monitor state document. */\nexport interface RenderMonitorStateDoc extends OperatorHealthStateDoc {\n origin: string\n /** Failing runs in a row, reset by any passing run. */\n consecutiveFailures: number\n /** When the current run of failures began; null while passing. */\n failingSinceMs: number | null\n lastOkAtMs: number | null\n}\n\nexport type RenderMonitorTransition = 'failing' | 'recovered' | null\n\n/**\n * The next state for one site, given what this run saw. Pure.\n *\n * The status turns `degraded` only once `threshold` runs in a row have\n * failed, so one slow cold start or one dropped connection says nothing. It\n * turns back on the first passing run, because a render that works is\n * proof, and a recovery held back is a recovery nobody hears about.\n */\nexport function nextRenderMonitorState(\n prior: Partial<RenderMonitorStateDoc> | null,\n observation: { ok: boolean; detail: string },\n context: { origin: string; label: string; threshold: number; now: number },\n): { next: RenderMonitorStateDoc; transition: RenderMonitorTransition } {\n const { origin, label, threshold, now } = context\n const checkId = renderMonitorCheckId(origin)\n const detail = String(observation.detail ?? '').slice(0, 1_000)\n const wasDegraded = prior?.status === 'degraded'\n if (observation.ok) {\n return {\n next: {\n checkId,\n label,\n origin,\n status: 'ok',\n sinceMs: !wasDegraded && prior?.sinceMs ? prior.sinceMs : now,\n detail,\n updatedAtMs: now,\n consecutiveFailures: 0,\n failingSinceMs: null,\n lastOkAtMs: now,\n },\n transition: wasDegraded ? 'recovered' : null,\n }\n }\n const failures = (prior?.consecutiveFailures ?? 0) + 1\n const failingSinceMs = prior?.failingSinceMs ?? now\n const degraded = wasDegraded || failures >= Math.max(1, threshold)\n return {\n next: {\n checkId,\n label,\n origin,\n status: degraded ? 'degraded' : 'ok',\n sinceMs: degraded\n ? wasDegraded && prior?.sinceMs\n ? prior.sinceMs\n : failingSinceMs\n : (prior?.sinceMs ?? now),\n detail,\n updatedAtMs: now,\n consecutiveFailures: failures,\n failingSinceMs,\n lastOkAtMs: prior?.lastOkAtMs ?? null,\n },\n transition: degraded && !wasDegraded ? 'failing' : null,\n }\n}\n\n/** One probe as the report shows it. */\nexport interface RenderProbeRow extends RenderProbeVerdict {\n kind: RenderProbeKind\n url: string\n ms: number\n httpStatus: number | null\n}\n\n/** One site's result for a run. */\nexport interface RenderMonitorSiteResult {\n origin: string\n ok: boolean\n probes: RenderProbeRow[]\n /** The state written; absent on a dry run or a failed write. */\n status?: 'ok' | 'degraded'\n consecutiveFailures?: number\n transition?: RenderMonitorTransition\n error?: string\n}\n\nexport interface RenderMonitorReport {\n dryRun: boolean\n threshold: number\n sites: RenderMonitorSiteResult[]\n}\n\n/** The read-modify-write of one site's state, as one transaction. */\nexport type RenderMonitorStore = <T>(\n checkId: string,\n work: (prior: Partial<RenderMonitorStateDoc> | null) => {\n next: RenderMonitorStateDoc\n result: T\n },\n) => Promise<T>\n\n/**\n * The default store: one Firestore transaction on the site's document, so\n * two overlapping runs cannot both decide they saw the edge.\n */\nconst firestoreStore: RenderMonitorStore = async (checkId, work) => {\n const { default: firebaseAdmin } = await import('./firebase-admin')\n const db = firebaseAdmin.app().firestore()\n const ref = db.collection(RENDER_MONITOR_COLLECTION).doc(checkId)\n return db.runTransaction(async (transaction: any) => {\n const snapshot = await transaction.get(ref)\n const prior = snapshot.exists\n ? (snapshot.data() as Partial<RenderMonitorStateDoc>)\n : null\n const { next, result } = work(prior)\n transaction.set(ref, next)\n return result\n })\n}\n\ntype RaiseAlert = (\n type: string,\n options: { dedupeKey?: string; context?: Record<string, string | number> },\n) => Promise<unknown>\n\nconst defaultRaise: RaiseAlert = async (type, options) => {\n const { raiseOperatorAlert } = await import('./operator-alerts')\n return raiseOperatorAlert(type, options)\n}\n\nfunction describeDuration(ms: number): string {\n const minutes = Math.max(1, Math.round(ms / 60_000))\n if (minutes < 60) return `${minutes} min`\n const hours = Math.floor(minutes / 60)\n const rest = minutes % 60\n return rest ? `${hours} h ${rest} min` : `${hours} h`\n}\n\nfunction probeLine(row: RenderProbeRow): string {\n return `${row.kind} ${new URL(row.url).pathname}: ${row.detail} (${row.ms} ms)`\n}\n\n/** Fetch and grade one probe. Never throws. */\nasync function runProbe(\n origin: string,\n spec: RenderProbeSpec,\n options: {\n fetcher: typeof fetch\n headers: Record<string, string>\n nonce: string\n timeoutMs: number\n clock: () => number\n },\n): Promise<RenderProbeRow> {\n const url = `${origin}${spec.path(options.nonce)}`\n const startedAt = options.clock()\n try {\n const response = await options.fetcher(url, {\n redirect: 'manual',\n cache: 'no-store',\n headers: {\n 'user-agent': 'aglyn-render-monitor',\n accept: 'text/html,application/xhtml+xml',\n ...options.headers,\n },\n signal: AbortSignal.timeout(options.timeoutMs),\n })\n const body = await response.text()\n const verdict = gradeRenderProbe(\n {\n status: response.status,\n contentType: response.headers.get('content-type'),\n body,\n location: response.headers.get('location'),\n cacheState:\n response.headers.get('x-vercel-cache') ??\n response.headers.get('x-nextjs-cache'),\n },\n spec.expectStatus,\n )\n return {\n kind: spec.kind,\n url,\n ms: options.clock() - startedAt,\n httpStatus: response.status,\n ...verdict,\n }\n } catch (error) {\n const name = (error as { name?: string } | null)?.name\n const timedOut = name === 'TimeoutError' || name === 'AbortError'\n return {\n kind: spec.kind,\n url,\n ms: options.clock() - startedAt,\n httpStatus: null,\n ok: false,\n challenged: false,\n code: timedOut ? 'timeout' : 'unreachable',\n detail: timedOut\n ? `no complete page in ${Math.round(options.timeoutMs / 1_000)} s`\n : `unreachable (${(error as { cause?: { code?: string } } | null)?.cause?.code ?? name ?? 'error'})`,\n }\n }\n}\n\n/**\n * One run: render every target, record each site's state, raise the edge.\n *\n * A dry run fetches and grades and writes nothing, so a person can ask\n * \"what would the monitor see right now\" without moving its state.\n */\nexport async function runRenderMonitor(\n options: {\n dryRun?: boolean\n env?: Readonly<Record<string, string | undefined>>\n fetcher?: typeof fetch\n /** Our own edge's bypass headers, from the caller's environment. */\n headers?: Record<string, string>\n now?: () => number\n nonce?: () => string\n timeoutMs?: number\n store?: RenderMonitorStore\n raise?: RaiseAlert\n } = {},\n): Promise<RenderMonitorReport> {\n const env = options.env ?? process.env\n const clock = options.now ?? Date.now\n const store = options.store ?? firestoreStore\n const raise = options.raise ?? defaultRaise\n const dryRun = Boolean(options.dryRun)\n const parsedThreshold = Number.parseInt(\n String(env['RENDER_MONITOR_FAILURE_THRESHOLD'] ?? ''),\n 10,\n )\n const threshold =\n Number.isFinite(parsedThreshold) && parsedThreshold >= 1\n ? parsedThreshold\n : RENDER_MONITOR_DEFAULT_THRESHOLD\n const nonce =\n options.nonce ??\n (() => `${clock().toString(36)}${Math.random().toString(36).slice(2, 8)}`)\n\n const sites = await Promise.all(\n resolveRenderMonitorTargets(env).map(\n async (origin): Promise<RenderMonitorSiteResult> => {\n const probes = await Promise.all(\n RENDER_PROBES.map((spec) =>\n runProbe(origin, spec, {\n fetcher: options.fetcher ?? fetch,\n headers: options.headers ?? {},\n nonce: nonce(),\n timeoutMs: options.timeoutMs ?? RENDER_MONITOR_TIMEOUT_MS,\n clock,\n }),\n ),\n )\n const ok = probes.every((probe) => probe.ok)\n const site: RenderMonitorSiteResult = { origin, ok, probes }\n if (dryRun) return site\n const host = new URL(origin).host\n const failing = probes.filter((probe) => !probe.ok)\n const detail = ok\n ? 'Fresh renders pass.'\n : failing.map(probeLine).join('; ')\n try {\n const now = clock()\n const { next, transition, failingSinceMs } = await store(\n renderMonitorCheckId(origin),\n (prior) => {\n const decided = nextRenderMonitorState(\n prior,\n { ok, detail },\n {\n origin,\n label: `Page rendering on ${host}`,\n threshold,\n now,\n },\n )\n return {\n next: decided.next,\n result: {\n ...decided,\n // When the failures a recovery ends began.\n failingSinceMs: prior?.failingSinceMs ?? prior?.sinceMs ?? now,\n },\n }\n },\n )\n site.status = next.status\n site.consecutiveFailures = next.consecutiveFailures\n site.transition = transition\n if (transition === 'failing') {\n await raise('system.siteRenderFailing', {\n dedupeKey: host,\n context: {\n site: host,\n failures: next.consecutiveFailures,\n since: describeDuration(now - next.sinceMs),\n detail,\n },\n })\n } else if (transition === 'recovered') {\n await raise('system.siteRenderRecovered', {\n dedupeKey: host,\n context: {\n site: host,\n duration: describeDuration(now - failingSinceMs),\n },\n })\n }\n } catch (error) {\n site.error = error instanceof Error ? error.message : String(error)\n console.error(`[render-monitor] ${host} could not be recorded`, error)\n }\n return site\n },\n ),\n )\n return { dryRun, threshold, sites }\n}\n"],"names":["TENANT_APEX","RENDER_MONITOR_COLLECTION","RENDER_MONITOR_CHECK_PREFIX","RENDER_MONITOR_JOB_ID","RENDER_MONITOR_TIMEOUT_MS","RENDER_MONITOR_DEFAULT_THRESHOLD","RENDER_PROBE_ISR_PREFIX","RENDER_PROBES","kind","path","expectStatus","nonce","CHECKPOINT_MARKER","CACHED_STATES","Set","gradeRenderProbe","response","body","test","ok","challenged","code","detail","status","location","contentType","includes","cache","String","cacheState","trim","toUpperCase","has","originOf","value","trimmed","replace","toLowerCase","resolveRenderMonitorTargets","env","raw","apex","label","demo","listed","split","map","filter","Boolean","renderMonitorCheckId","origin","URL","host","nextRenderMonitorState","prior","observation","context","threshold","now","checkId","slice","wasDegraded","next","sinceMs","updatedAtMs","consecutiveFailures","failingSinceMs","lastOkAtMs","transition","failures","degraded","Math","max","firestoreStore","work","default","firebaseAdmin","db","app","firestore","ref","collection","doc","runTransaction","transaction","snapshot","get","exists","data","result","set","defaultRaise","type","options","raiseOperatorAlert","describeDuration","ms","minutes","round","hours","floor","rest","probeLine","row","url","pathname","runProbe","spec","startedAt","clock","fetcher","redirect","headers","accept","signal","AbortSignal","timeout","timeoutMs","text","verdict","httpStatus","error","name","timedOut","cause","runRenderMonitor","process","Date","store","raise","dryRun","parsedThreshold","Number","parseInt","isFinite","toString","random","sites","Promise","all","probes","fetch","every","probe","site","failing","join","decided","dedupeKey","since","duration","Error","message","console"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED,SAASA,WAAW,QAAQ,qCAAoC;AAGhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsCC,GAED,6DAA6D,GAC7D,MAAMC,4BAA4B;AAElC,uDAAuD,GACvD,OAAO,MAAMC,8BAA8B,mBAAkB;AAE7D,iEAAiE,GACjE,OAAO,MAAMC,wBAAwB,iBAAgB;AAErD;;;;CAIC,GACD,OAAO,MAAMC,4BAA4B,MAAM;AAE/C,qDAAqD,GACrD,OAAO,MAAMC,mCAAmC,EAAC;AAEjD;;;;CAIC,GACD,OAAO,MAAMC,0BAA0B,uBAAsB;AAY7D,OAAO,MAAMC,gBAA4C;IACvD;QAAEC,MAAM;QAAWC,MAAM,IAAM;QAAWC,cAAc;IAAI;IAC5D;QACEF,MAAM;QACNC,MAAM,CAACE,QAAU,GAAGL,0BAA0BK,OAAO;QACrDD,cAAc;IAChB;CACD,CAAA;AAqBD,+EAA+E,GAC/E,MAAME,oBAAoB;AAE1B;;;CAGC,GACD,MAAMC,gBAAgB,IAAIC,IAAI;IAAC;IAAO;IAAS;CAAY;AAE3D;;;;;;CAMC,GACD,OAAO,SAASC,iBACdC,QAA6B,EAC7BN,YAAoB;QAEPM,gBAyBWA,uBAwBHA;IAjDrB,MAAMC,QAAOD,iBAAAA,SAASC,IAAI,YAAbD,iBAAiB;IAC9B,IAAIJ,kBAAkBM,IAAI,CAACD,OAAO;QAChC,OAAO;YACLE,IAAI;YACJC,YAAY;YACZC,MAAM;YACNC,QAAQ,CAAC,8BAA8B,EAAEN,SAASO,MAAM,CAAC,6DAA6D,CAAC;QACzH;IACF;IACA,IAAIP,SAASO,MAAM,IAAI,OAAOP,SAASO,MAAM,GAAG,KAAK;YAKxBP;QAJ3B,OAAO;YACLG,IAAI;YACJC,YAAY;YACZC,MAAM;YACNC,QAAQ,CAAC,cAAc,GAAEN,qBAAAA,SAASQ,QAAQ,YAAjBR,qBAAqB,IAAI,yBAAyB,CAAC;QAC9E;IACF;IACA,IAAIA,SAASO,MAAM,KAAKb,cAAc;QACpC,OAAO;YACLS,IAAI;YACJC,YAAY;YACZC,MAAM,CAAC,KAAK,EAAEL,SAASO,MAAM,EAAE;YAC/BD,QAAQ,CAAC,KAAK,EAAEN,SAASO,MAAM,EAAE;QACnC;IACF;IACA,IAAI,CAAC,cAAcL,IAAI,EAACF,wBAAAA,SAASS,WAAW,YAApBT,wBAAwB,KAAK;QACnD,OAAO;YACLG,IAAI;YACJC,YAAY;YACZC,MAAM;YACNC,QAAQ,CAAC,UAAU,EAAEN,SAASS,WAAW,IAAI,kBAAkB,CAAC,CAAC;QACnE;IACF;IACA,IAAI,CAACR,KAAKS,QAAQ,CAAC,YAAY;QAC7B,OAAO;YACLP,IAAI;YACJC,YAAY;YACZC,MAAM;YACNC,QAAQ;QACV;IACF;IACA,IAAI,CAACL,KAAKS,QAAQ,CAAC,mBAAmB;QACpC,OAAO;YACLP,IAAI;YACJC,YAAY;YACZC,MAAM;YACNC,QAAQ;QACV;IACF;IACA,MAAMK,QAAQC,QAAOZ,uBAAAA,SAASa,UAAU,YAAnBb,uBAAuB,IAAIc,IAAI,GAAGC,WAAW;IAClE,IAAIlB,cAAcmB,GAAG,CAACL,QAAQ;QAC5B,OAAO;YACLR,IAAI;YACJC,YAAY;YACZC,MAAM;YACNC,QAAQ,CAAC,mBAAmB,EAAEK,MAAM,yBAAyB,CAAC;QAChE;IACF;IACA,OAAO;QAAER,IAAI;QAAMC,YAAY;QAAOC,MAAM;QAAMC,QAAQ;IAAW;AACvE;AAEA,+DAA+D,GAC/D,OAAO,SAASW,SAASC,KAAgC;IACvD,MAAMC,UAAUP,OAAOM,gBAAAA,QAAS,IAAIJ,IAAI,GAAGM,OAAO,CAAC,QAAQ;IAC3D,OAAO,0BAA0BlB,IAAI,CAACiB,WAAWA,QAAQE,WAAW,KAAK;AAC3E;AAEA;;;;;;;;;;CAUC,GACD,OAAO,SAASC,4BACdC,GAAiD;QAE9BA,6BAGVA,gCACYA;IAJrB,MAAMC,MAAMZ,QAAOW,8BAAAA,GAAG,CAAC,yBAAyB,YAA7BA,8BAAiC,IAAIT,IAAI;IAC5D,IAAI,wBAAwBZ,IAAI,CAACsB,MAAM,OAAO,EAAE;IAChD,MAAMC,OACJb,QAAOW,iCAAAA,GAAG,CAAC,4BAA4B,YAAhCA,iCAAoC,IAAIT,IAAI,MAAM9B;IAC3D,MAAM0C,QAAQd,QAAOW,yBAAAA,GAAG,CAAC,oBAAoB,YAAxBA,yBAA4B,IAAIT,IAAI,MAAM;IAC/D,MAAMa,OACJV,SAASM,GAAG,CAAC,gCAAgC,KAC7CN,SAAS,CAAC,QAAQ,EAAES,MAAM,CAAC,EAAED,MAAM;IACrC,MAAMG,SAASJ,IACZK,KAAK,CAAC,UACNC,GAAG,CAAC,CAACZ,QAAUD,SAASC,QACxBa,MAAM,CAACC;IACV,OAAO;WAAI,IAAIlC,IAAI;YAAC6B;eAASC;SAAO,CAACG,MAAM,CAACC;KAAU;AACxD;AAEA,0CAA0C,GAC1C,OAAO,SAASC,qBAAqBC,MAAc;IACjD,OAAO,GAAGhD,8BAA8B,IAAIiD,IAAID,QAAQE,IAAI,EAAE;AAChE;AAcA;;;;;;;CAOC,GACD,OAAO,SAASC,uBACdC,KAA4C,EAC5CC,WAA4C,EAC5CC,OAA0E;QAIpDD;IAFtB,MAAM,EAAEL,MAAM,EAAER,KAAK,EAAEe,SAAS,EAAEC,GAAG,EAAE,GAAGF;IAC1C,MAAMG,UAAUV,qBAAqBC;IACrC,MAAM5B,SAASM,QAAO2B,sBAAAA,YAAYjC,MAAM,YAAlBiC,sBAAsB,IAAIK,KAAK,CAAC,GAAG;IACzD,MAAMC,cAAcP,CAAAA,yBAAAA,MAAO/B,MAAM,MAAK;IACtC,IAAIgC,YAAYpC,EAAE,EAAE;QAClB,OAAO;YACL2C,MAAM;gBACJH;gBACAjB;gBACAQ;gBACA3B,QAAQ;gBACRwC,SAAS,CAACF,gBAAeP,yBAAAA,MAAOS,OAAO,IAAGT,MAAMS,OAAO,GAAGL;gBAC1DpC;gBACA0C,aAAaN;gBACbO,qBAAqB;gBACrBC,gBAAgB;gBAChBC,YAAYT;YACd;YACAU,YAAYP,cAAc,cAAc;QAC1C;IACF;IACA,MAAMQ,WAAW,SAACf,yBAAAA,MAAOW,mBAAmB,mBAAI,KAAK;IACrD,MAAMC,0BAAiBZ,yBAAAA,MAAOY,cAAc,oBAAIR;IAChD,MAAMY,WAAWT,eAAeQ,YAAYE,KAAKC,GAAG,CAAC,GAAGf;IACxD,OAAO;QACLK,MAAM;YACJH;YACAjB;YACAQ;YACA3B,QAAQ+C,WAAW,aAAa;YAChCP,SAASO,WACLT,gBAAeP,yBAAAA,MAAOS,OAAO,IAC3BT,MAAMS,OAAO,GACbG,0BACDZ,yBAAAA,MAAOS,OAAO,oBAAIL;YACvBpC;YACA0C,aAAaN;YACbO,qBAAqBI;YACrBH;YACAC,UAAU,WAAEb,yBAAAA,MAAOa,UAAU,oBAAI;QACnC;QACAC,YAAYE,YAAY,CAACT,cAAc,YAAY;IACrD;AACF;AAqCA;;;CAGC,GACD,MAAMY,iBAAqC,OAAOd,SAASe;IACzD,MAAM,EAAEC,SAASC,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,MAAMC,KAAKD,cAAcE,GAAG,GAAGC,SAAS;IACxC,MAAMC,MAAMH,GAAGI,UAAU,CAAChF,2BAA2BiF,GAAG,CAACvB;IACzD,OAAOkB,GAAGM,cAAc,CAAC,OAAOC;QAC9B,MAAMC,WAAW,MAAMD,YAAYE,GAAG,CAACN;QACvC,MAAM1B,QAAQ+B,SAASE,MAAM,GACxBF,SAASG,IAAI,KACd;QACJ,MAAM,EAAE1B,IAAI,EAAE2B,MAAM,EAAE,GAAGf,KAAKpB;QAC9B8B,YAAYM,GAAG,CAACV,KAAKlB;QACrB,OAAO2B;IACT;AACF;AAOA,MAAME,eAA2B,OAAOC,MAAMC;IAC5C,MAAM,EAAEC,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC;IAC5C,OAAOA,mBAAmBF,MAAMC;AAClC;AAEA,SAASE,iBAAiBC,EAAU;IAClC,MAAMC,UAAU1B,KAAKC,GAAG,CAAC,GAAGD,KAAK2B,KAAK,CAACF,KAAK;IAC5C,IAAIC,UAAU,IAAI,OAAO,GAAGA,QAAQ,IAAI,CAAC;IACzC,MAAME,QAAQ5B,KAAK6B,KAAK,CAACH,UAAU;IACnC,MAAMI,OAAOJ,UAAU;IACvB,OAAOI,OAAO,GAAGF,MAAM,GAAG,EAAEE,KAAK,IAAI,CAAC,GAAG,GAAGF,MAAM,EAAE,CAAC;AACvD;AAEA,SAASG,UAAUC,GAAmB;IACpC,OAAO,GAAGA,IAAI/F,IAAI,CAAC,CAAC,EAAE,IAAI2C,IAAIoD,IAAIC,GAAG,EAAEC,QAAQ,CAAC,EAAE,EAAEF,IAAIjF,MAAM,CAAC,EAAE,EAAEiF,IAAIP,EAAE,CAAC,IAAI,CAAC;AACjF;AAEA,6CAA6C,GAC7C,eAAeU,SACbxD,MAAc,EACdyD,IAAqB,EACrBd,OAMC;IAED,MAAMW,MAAM,GAAGtD,SAASyD,KAAKlG,IAAI,CAACoF,QAAQlF,KAAK,GAAG;IAClD,MAAMiG,YAAYf,QAAQgB,KAAK;IAC/B,IAAI;YAmBI7F;QAlBN,MAAMA,WAAW,MAAM6E,QAAQiB,OAAO,CAACN,KAAK;YAC1CO,UAAU;YACVpF,OAAO;YACPqF,SAAS;gBACP,cAAc;gBACdC,QAAQ;eACLpB,QAAQmB,OAAO;YAEpBE,QAAQC,YAAYC,OAAO,CAACvB,QAAQwB,SAAS;QAC/C;QACA,MAAMpG,OAAO,MAAMD,SAASsG,IAAI;QAChC,MAAMC,UAAUxG,iBACd;YACEQ,QAAQP,SAASO,MAAM;YACvBE,aAAaT,SAASgG,OAAO,CAAC1B,GAAG,CAAC;YAClCrE;YACAO,UAAUR,SAASgG,OAAO,CAAC1B,GAAG,CAAC;YAC/BzD,UAAU,GACRb,wBAAAA,SAASgG,OAAO,CAAC1B,GAAG,CAAC,6BAArBtE,wBACAA,SAASgG,OAAO,CAAC1B,GAAG,CAAC;QACzB,GACAqB,KAAKjG,YAAY;QAEnB,OAAO;YACLF,MAAMmG,KAAKnG,IAAI;YACfgG;YACAR,IAAIH,QAAQgB,KAAK,KAAKD;YACtBY,YAAYxG,SAASO,MAAM;WACxBgG;IAEP,EAAE,OAAOE,OAAO;YAaQ;YAAA;QAZtB,MAAMC,OAAQD,yBAAD,AAACA,MAAoCC,IAAI;QACtD,MAAMC,WAAWD,SAAS,kBAAkBA,SAAS;QACrD,OAAO;YACLlH,MAAMmG,KAAKnG,IAAI;YACfgG;YACAR,IAAIH,QAAQgB,KAAK,KAAKD;YACtBY,YAAY;YACZrG,IAAI;YACJC,YAAY;YACZC,MAAMsG,WAAW,YAAY;YAC7BrG,QAAQqG,WACJ,CAAC,oBAAoB,EAAEpD,KAAK2B,KAAK,CAACL,QAAQwB,SAAS,GAAG,MAAO,EAAE,CAAC,GAChE,CAAC,aAAa,GAAE,gBAACI,0BAAD,eAAA,AAACA,MAAgDG,KAAK,qBAAtD,aAAwDvG,IAAI,oBAAIqG,gBAAhE,OAAwE,QAAQ,CAAC,CAAC;QACxG;IACF;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAeG,iBACpBhC,UAWI,CAAC,CAAC;QAEMA,cACEA,cACAA,gBACAA,gBAGLtD,uCAQPsD;IAdF,MAAMtD,OAAMsD,eAAAA,QAAQtD,GAAG,YAAXsD,eAAeiC,QAAQvF,GAAG;IACtC,MAAMsE,SAAQhB,eAAAA,QAAQnC,GAAG,YAAXmC,eAAekC,KAAKrE,GAAG;IACrC,MAAMsE,SAAQnC,iBAAAA,QAAQmC,KAAK,YAAbnC,iBAAiBpB;IAC/B,MAAMwD,SAAQpC,iBAAAA,QAAQoC,KAAK,YAAbpC,iBAAiBF;IAC/B,MAAMuC,SAASlF,QAAQ6C,QAAQqC,MAAM;IACrC,MAAMC,kBAAkBC,OAAOC,QAAQ,CACrCzG,QAAOW,wCAAAA,GAAG,CAAC,mCAAmC,YAAvCA,wCAA2C,KAClD;IAEF,MAAMkB,YACJ2E,OAAOE,QAAQ,CAACH,oBAAoBA,mBAAmB,IACnDA,kBACA9H;IACN,MAAMM,SACJkF,iBAAAA,QAAQlF,KAAK,YAAbkF,iBACC,IAAM,GAAGgB,QAAQ0B,QAAQ,CAAC,MAAMhE,KAAKiE,MAAM,GAAGD,QAAQ,CAAC,IAAI3E,KAAK,CAAC,GAAG,IAAI;IAE3E,MAAM6E,QAAQ,MAAMC,QAAQC,GAAG,CAC7BrG,4BAA4BC,KAAKO,GAAG,CAClC,OAAOI;QACL,MAAM0F,SAAS,MAAMF,QAAQC,GAAG,CAC9BpI,cAAcuC,GAAG,CAAC,CAAC6D;gBAENd,kBACAA,kBAEEA;mBAJba,SAASxD,QAAQyD,MAAM;gBACrBG,OAAO,GAAEjB,mBAAAA,QAAQiB,OAAO,YAAfjB,mBAAmBgD;gBAC5B7B,OAAO,GAAEnB,mBAAAA,QAAQmB,OAAO,YAAfnB,mBAAmB,CAAC;gBAC7BlF,OAAOA;gBACP0G,SAAS,GAAExB,qBAAAA,QAAQwB,SAAS,YAAjBxB,qBAAqBzF;gBAChCyG;YACF;;QAGJ,MAAM1F,KAAKyH,OAAOE,KAAK,CAAC,CAACC,QAAUA,MAAM5H,EAAE;QAC3C,MAAM6H,OAAgC;YAAE9F;YAAQ/B;YAAIyH;QAAO;QAC3D,IAAIV,QAAQ,OAAOc;QACnB,MAAM5F,OAAO,IAAID,IAAID,QAAQE,IAAI;QACjC,MAAM6F,UAAUL,OAAO7F,MAAM,CAAC,CAACgG,QAAU,CAACA,MAAM5H,EAAE;QAClD,MAAMG,SAASH,KACX,wBACA8H,QAAQnG,GAAG,CAACwD,WAAW4C,IAAI,CAAC;QAChC,IAAI;YACF,MAAMxF,MAAMmD;YACZ,MAAM,EAAE/C,IAAI,EAAEM,UAAU,EAAEF,cAAc,EAAE,GAAG,MAAM8D,MACjD/E,qBAAqBC,SACrB,CAACI;oBAgBqBA;gBAfpB,MAAM6F,UAAU9F,uBACdC,OACA;oBAAEnC;oBAAIG;gBAAO,GACb;oBACE4B;oBACAR,OAAO,CAAC,kBAAkB,EAAEU,MAAM;oBAClCK;oBACAC;gBACF;gBAEF,OAAO;oBACLI,MAAMqF,QAAQrF,IAAI;oBAClB2B,QAAQ,aACH0D;wBACH,2CAA2C;wBAC3CjF,cAAc,GAAEZ,gBAAAA,yBAAAA,MAAOY,cAAc,oBAAIZ,yBAAAA,MAAOS,OAAO,YAAvCT,OAA2CI;;gBAE/D;YACF;YAEFsF,KAAKzH,MAAM,GAAGuC,KAAKvC,MAAM;YACzByH,KAAK/E,mBAAmB,GAAGH,KAAKG,mBAAmB;YACnD+E,KAAK5E,UAAU,GAAGA;YAClB,IAAIA,eAAe,WAAW;gBAC5B,MAAM6D,MAAM,4BAA4B;oBACtCmB,WAAWhG;oBACXI,SAAS;wBACPwF,MAAM5F;wBACNiB,UAAUP,KAAKG,mBAAmB;wBAClCoF,OAAOtD,iBAAiBrC,MAAMI,KAAKC,OAAO;wBAC1CzC;oBACF;gBACF;YACF,OAAO,IAAI8C,eAAe,aAAa;gBACrC,MAAM6D,MAAM,8BAA8B;oBACxCmB,WAAWhG;oBACXI,SAAS;wBACPwF,MAAM5F;wBACNkG,UAAUvD,iBAAiBrC,MAAMQ;oBACnC;gBACF;YACF;QACF,EAAE,OAAOuD,OAAO;YACduB,KAAKvB,KAAK,GAAGA,iBAAiB8B,QAAQ9B,MAAM+B,OAAO,GAAG5H,OAAO6F;YAC7DgC,QAAQhC,KAAK,CAAC,CAAC,iBAAiB,EAAErE,KAAK,sBAAsB,CAAC,EAAEqE;QAClE;QACA,OAAOuB;IACT;IAGJ,OAAO;QAAEd;QAAQzE;QAAWgF;IAAM;AACpC"}
@@ -217,6 +217,13 @@ export declare function riskNoticeProductName(brand: SystemEmailBrand): string;
217
217
  * ONE function, so every notice renders the same way.
218
218
  */
219
219
  export declare function renderRiskNoticeEmail(key: string, merge: Record<string, string>, brand?: SystemEmailBrand): Promise<RenderedRiskNoticeEmail>;
220
+ /**
221
+ * How long a caller on a page render's path waits on a notice before it
222
+ * carries on without the answer (AGL-3565). The notice is not canceled; it
223
+ * finishes behind the page. A held page or redirect is held either way —
224
+ * what is bounded is how long the visitor waits to be told so.
225
+ */
226
+ export declare const RISK_NOTICE_RENDER_DEADLINE_MS = 8000;
220
227
  /**
221
228
  * Tell the right people about one risk event. See the module header.
222
229
  * Never throws.
@@ -467,6 +467,12 @@ function emptyDelivery(skipped = null) {
467
467
  emails
468
468
  };
469
469
  }
470
+ /**
471
+ * How long a caller on a page render's path waits on a notice before it
472
+ * carries on without the answer (AGL-3565). The notice is not canceled; it
473
+ * finishes behind the page. A held page or redirect is held either way —
474
+ * what is bounded is how long the visitor waits to be told so.
475
+ */ export const RISK_NOTICE_RENDER_DEADLINE_MS = 8000;
470
476
  /**
471
477
  * Tell the right people about one risk event. See the module header.
472
478
  * Never throws.