@aglyn/tenant-data-admin 1.0.0-beta.223 → 1.0.0-beta.225
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/lib/server/after-response.d.ts +32 -0
- package/src/lib/server/after-response.js +94 -0
- package/src/lib/server/after-response.js.map +1 -0
- package/src/lib/server/capture-email-check.d.ts +2 -0
- package/src/lib/server/capture-email-check.js +20 -24
- package/src/lib/server/capture-email-check.js.map +1 -1
- package/src/lib/server/crm-records.d.ts +12 -3
- package/src/lib/server/crm-records.js +40 -5
- package/src/lib/server/crm-records.js.map +1 -1
- package/src/lib/server/render-monitor.d.ts +150 -2
- package/src/lib/server/render-monitor.js +390 -32
- package/src/lib/server/render-monitor.js.map +1 -1
- package/src/lib/server/serve-media-cdn.d.ts +7 -0
- package/src/lib/server/serve-media-cdn.js +37 -26
- package/src/lib/server/serve-media-cdn.js.map +1 -1
|
@@ -46,13 +46,46 @@ import { _ as _extends } from "@swc/helpers/_/_extends";
|
|
|
46
46
|
*
|
|
47
47
|
* A site that fails `threshold` runs in a row raises
|
|
48
48
|
* `system.siteRenderFailing` once; the first passing run after that raises
|
|
49
|
-
* `system.siteRenderRecovered` once.
|
|
49
|
+
* `system.siteRenderRecovered` once.
|
|
50
|
+
*
|
|
51
|
+
* ## A challenge is not a failed render (AGL-3580)
|
|
52
|
+
*
|
|
53
|
+
* When bot protection answers instead of the site, this run saw nothing about
|
|
54
|
+
* the site at all. On 2026-10-05 the console had no `AGLYN_PROBE_TOKEN`, every
|
|
55
|
+
* `/search` probe got the 429 checkpoint for three hours, and the monitor told
|
|
56
|
+
* the operator "Pages are not rendering on aglyn.com" and then "rendering
|
|
57
|
+
* again after 3 h" about a site that was serving pages the whole time. So a
|
|
58
|
+
* run whose only failures are challenges is BLIND: it leaves the site's
|
|
59
|
+
* verdict and failure count exactly as they were, and `threshold` blind runs
|
|
60
|
+
* in a row raise `system.renderMonitorBlind` once, which names the setup
|
|
61
|
+
* problem instead of an outage. A run with a real failure beside a challenge
|
|
62
|
+
* is still a failure, because the real one is evidence. The state lives in
|
|
50
63
|
* `operatorHealthState/render-monitor--<host>`, beside every other health
|
|
51
64
|
* check, so Staff → Operator alerts lists it with its status and since when.
|
|
52
65
|
*
|
|
53
|
-
* Nothing here names an Aglyn host: the default target is the
|
|
54
|
-
* demonstration site label under its own tenant apex, the
|
|
55
|
-
* middleware and the render canaries already follow.
|
|
66
|
+
* Nothing here names an Aglyn host for anyone else: the default target is the
|
|
67
|
+
* install's own demonstration site label under its own tenant apex, the
|
|
68
|
+
* convention the middleware and the render canaries already follow. The one
|
|
69
|
+
* exception is `PLATFORM_RENDER_PAGES` below, which applies on Aglyn's own
|
|
70
|
+
* console deployment and nowhere else.
|
|
71
|
+
*
|
|
72
|
+
* ## Real pages, rendered fresh (AGL-3571)
|
|
73
|
+
*
|
|
74
|
+
* The two probes above draw the layout. They do not draw a page BODY, and
|
|
75
|
+
* beta.223 hung only on page bodies with an image in them (an ancestor walk
|
|
76
|
+
* looped on the page root, AGL-3565): `demo` has no images and the not-found
|
|
77
|
+
* page has no body, so this monitor read green through it. So a third probe,
|
|
78
|
+
* `page`, renders a real published page through the full ISR path — through
|
|
79
|
+
* the tenant's `*.vercel.app` production domain (`RENDER_MONITOR_RENDER_ORIGIN`),
|
|
80
|
+
* which honors `?tenantHost=` and sits behind Deployment Protection, naming
|
|
81
|
+
* the site with a spelling no cache holds (`freshHostSpelling`). The cache key
|
|
82
|
+
* includes the `[host]` segment, and `normalizeHostAlias` resolves every
|
|
83
|
+
* spelling to the same site. Without the automation bypass the render origin
|
|
84
|
+
* answers its sign-in redirect, so no `page` probe is made at all.
|
|
85
|
+
*
|
|
86
|
+
* One page per site per run, rotating through that site's list: a platform
|
|
87
|
+
* break fails every page with an image, so every run, and alerts as fast as
|
|
88
|
+
* before; a break on one page alone is seen on its turn.
|
|
56
89
|
*/ /** The state documents' collection: the health checks' own. */ const RENDER_MONITOR_COLLECTION = 'operatorHealthState';
|
|
57
90
|
/** Every render monitor document id starts with this. */ export const RENDER_MONITOR_CHECK_PREFIX = 'render-monitor--';
|
|
58
91
|
/** The beat id `/api/health/crons` reads for the scheduled run. */ export const RENDER_MONITOR_JOB_ID = 'render-monitor';
|
|
@@ -162,6 +195,121 @@ export const RENDER_PROBES = [
|
|
|
162
195
|
detail: 'rendered'
|
|
163
196
|
};
|
|
164
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* The pages Aglyn's own console renders fresh when `RENDER_MONITOR_PAGES` is
|
|
200
|
+
* unset: the canary's real client pages (`tools/scripts/lib/prod-canary.mjs`
|
|
201
|
+
* `DEFAULT_TENANT_HOSTS`), images, reusable components, repeats and a form
|
|
202
|
+
* between them. Applied ONLY on that console deployment (`VERCEL_PROJECT_ID`),
|
|
203
|
+
* so no other install ever requests them.
|
|
204
|
+
*/ export const PLATFORM_RENDER_PAGES = [
|
|
205
|
+
'ready-to-roll.aglyn.app/',
|
|
206
|
+
'edr-construction.aglyn.app/',
|
|
207
|
+
'edr-construction.aglyn.app/services',
|
|
208
|
+
'edr-construction.aglyn.app/contact'
|
|
209
|
+
].join(' ');
|
|
210
|
+
/** Aglyn's console project on Vercel, the one deployment the list above is for. */ export const PLATFORM_CONSOLE_VERCEL_PROJECT_ID = 'prj_gEzxEXc0Lhs81rmaXIg2a1GbsDfl';
|
|
211
|
+
/**
|
|
212
|
+
* The tenant's production project domain on Vercel: follows every promote and
|
|
213
|
+
* rollback, honors `?tenantHost=`, and is reachable only with the bypass.
|
|
214
|
+
*/ export const PLATFORM_RENDER_ORIGIN = 'https://aglyn-tenant-aglyn.vercel.app';
|
|
215
|
+
/** The middleware's custom-domain sentinel (`apps/tenant/utils/get-host.ts`). */ const CNAME_HOST_PREFIX = 'cname--';
|
|
216
|
+
/** Letters past this are never recased, so the bit arithmetic stays in 32 bits. */ const MAX_SPELLING_BITS = 30;
|
|
217
|
+
function spellingLetters(host) {
|
|
218
|
+
return [
|
|
219
|
+
...host.toLowerCase()
|
|
220
|
+
].map((char, index)=>/[a-z]/.test(char) ? index : -1).filter((index)=>index >= 0).slice(0, MAX_SPELLING_BITS);
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Spelling number `n` of one site: a `?tenantHost=` value that resolves to
|
|
224
|
+
* that site and is a `[host]` route segment no other number produces, so the
|
|
225
|
+
* page it asks for has no cache entry to come from.
|
|
226
|
+
*
|
|
227
|
+
* - `{sub}.{apex}` → the full name, letters upper-cased by the bits of `n`,
|
|
228
|
+
* then one trailing dot per exhausted round of cases.
|
|
229
|
+
* - any other name → `cname--{name}`, recased the same way and cycling after
|
|
230
|
+
* `2 ** letters` (that form keeps trailing dots), with the prefix left
|
|
231
|
+
* lower-case because the canonical-domain redirect tests the raw segment.
|
|
232
|
+
*
|
|
233
|
+
* MUST stay identical to `freshHostSpelling` in
|
|
234
|
+
* `tools/scripts/lib/prod-canary.mjs`; `apps/tenant/specs/probe-host-spelling.spec.ts`
|
|
235
|
+
* pins both against `normalizeHostAlias`.
|
|
236
|
+
*/ export function freshHostSpelling(host, n, apex = TENANT_APEX) {
|
|
237
|
+
const subdomain = host.endsWith(`.${apex}`);
|
|
238
|
+
const prefix = subdomain ? '' : CNAME_HOST_PREFIX;
|
|
239
|
+
const chars = [
|
|
240
|
+
...host.toLowerCase()
|
|
241
|
+
];
|
|
242
|
+
const letters = spellingLetters(host);
|
|
243
|
+
const capacity = 2 ** letters.length;
|
|
244
|
+
const value = Math.max(0, Math.floor(Number(n) || 0));
|
|
245
|
+
const bits = value % capacity;
|
|
246
|
+
letters.forEach((index, bit)=>{
|
|
247
|
+
if (Math.floor(bits / 2 ** bit) % 2 === 1) chars[index] = chars[index].toUpperCase();
|
|
248
|
+
});
|
|
249
|
+
const dots = subdomain ? '.'.repeat(Math.floor(value / capacity)) : '';
|
|
250
|
+
return `${prefix}${chars.join('')}${dots}`;
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* The spelling number for a run at `nowMs`: minutes since 2026-10-01, so it
|
|
254
|
+
* never repeats on one deployment — a `{sub}.aglyn.app` name of 18 letters
|
|
255
|
+
* gains its first trailing dot after half a year.
|
|
256
|
+
*/ export function renderSpellingNumber(nowMs) {
|
|
257
|
+
return Math.max(1, Math.floor((nowMs - Date.UTC(2026, 9, 1)) / 60000));
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* `"a.example.app b.example.app/services, https://b.example.app/contact"` →
|
|
261
|
+
* one entry per host, paths merged, `/` when an entry names none. Entries that
|
|
262
|
+
* are not a hostname are dropped, as `RENDER_MONITOR_ORIGINS` drops them.
|
|
263
|
+
*/ export function parseRenderMonitorPages(text) {
|
|
264
|
+
const byHost = new Map();
|
|
265
|
+
for (const raw of String(text != null ? text : '').split(/[\s,]+/)){
|
|
266
|
+
var _byHost_get;
|
|
267
|
+
const entry = raw.trim().replace(/^https?:\/\//i, '');
|
|
268
|
+
if (!entry) continue;
|
|
269
|
+
const slash = entry.indexOf('/');
|
|
270
|
+
const host = (slash === -1 ? entry : entry.slice(0, slash)).toLowerCase();
|
|
271
|
+
if (!/^[a-z0-9-]+(\.[a-z0-9-]+)+$/.test(host)) continue;
|
|
272
|
+
const path = slash === -1 ? '/' : entry.slice(slash).replace(/\/+$/, '') || '/';
|
|
273
|
+
const paths = (_byHost_get = byHost.get(host)) != null ? _byHost_get : [];
|
|
274
|
+
if (!paths.includes(path)) paths.push(path);
|
|
275
|
+
byHost.set(host, paths);
|
|
276
|
+
}
|
|
277
|
+
return [
|
|
278
|
+
...byHost
|
|
279
|
+
].map(([host, paths])=>({
|
|
280
|
+
origin: `https://${host}`,
|
|
281
|
+
host,
|
|
282
|
+
paths
|
|
283
|
+
}));
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Which real pages the monitor renders, and through which origin.
|
|
287
|
+
*
|
|
288
|
+
* `RENDER_MONITOR_PAGES` (same grammar as the canary's `CANARY_TENANT_HOSTS`)
|
|
289
|
+
* and `RENDER_MONITOR_RENDER_ORIGIN`, else — on Aglyn's own console only —
|
|
290
|
+
* `PLATFORM_RENDER_PAGES` through `PLATFORM_RENDER_ORIGIN`. `off` (here or in
|
|
291
|
+
* `RENDER_MONITOR_ORIGINS`) renders none. No render origin, no pages.
|
|
292
|
+
*/ export function resolveRenderMonitorPages(env) {
|
|
293
|
+
var _env_RENDER_MONITOR_PAGES, _env_RENDER_MONITOR_ORIGINS;
|
|
294
|
+
const off = /^(off|none|false|0)$/i;
|
|
295
|
+
const raw = String((_env_RENDER_MONITOR_PAGES = env['RENDER_MONITOR_PAGES']) != null ? _env_RENDER_MONITOR_PAGES : '').trim();
|
|
296
|
+
if (off.test(raw) || off.test(String((_env_RENDER_MONITOR_ORIGINS = env['RENDER_MONITOR_ORIGINS']) != null ? _env_RENDER_MONITOR_ORIGINS : '').trim())) {
|
|
297
|
+
return {
|
|
298
|
+
renderOrigin: '',
|
|
299
|
+
pages: []
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
const platform = env['VERCEL_PROJECT_ID'] === PLATFORM_CONSOLE_VERCEL_PROJECT_ID;
|
|
303
|
+
const renderOrigin = originOf(env['RENDER_MONITOR_RENDER_ORIGIN']) || (platform ? PLATFORM_RENDER_ORIGIN : '');
|
|
304
|
+
const pages = parseRenderMonitorPages(raw || (platform ? PLATFORM_RENDER_PAGES : ''));
|
|
305
|
+
return renderOrigin ? {
|
|
306
|
+
renderOrigin,
|
|
307
|
+
pages
|
|
308
|
+
} : {
|
|
309
|
+
renderOrigin: '',
|
|
310
|
+
pages: []
|
|
311
|
+
};
|
|
312
|
+
}
|
|
165
313
|
/** A configured value as an origin, or '' when it is not one. */ export function originOf(value) {
|
|
166
314
|
const trimmed = String(value != null ? value : '').trim().replace(/\/+$/, '');
|
|
167
315
|
return /^https?:\/\/[^/\s?#]+$/i.test(trimmed) ? trimmed.toLowerCase() : '';
|
|
@@ -184,15 +332,21 @@ export const RENDER_PROBES = [
|
|
|
184
332
|
const label = String((_env_AGLYN_TENANT_DEMO = env['AGLYN_TENANT_DEMO']) != null ? _env_AGLYN_TENANT_DEMO : '').trim() || 'demo';
|
|
185
333
|
const demo = originOf(env['OPERATOR_HEALTH_TENANT_ORIGIN']) || originOf(`https://${label}.${apex}`);
|
|
186
334
|
const listed = raw.split(/[\s,]+/).map((value)=>originOf(value)).filter(Boolean);
|
|
335
|
+
// A site whose pages are rendered is watched whole: its layout probes too.
|
|
336
|
+
const paged = resolveRenderMonitorPages(env).pages.map((page)=>page.origin);
|
|
187
337
|
return [
|
|
188
338
|
...new Set([
|
|
189
339
|
demo,
|
|
190
|
-
...listed
|
|
340
|
+
...listed,
|
|
341
|
+
...paged
|
|
191
342
|
].filter(Boolean))
|
|
192
343
|
];
|
|
193
344
|
}
|
|
194
345
|
/** The state document id for one origin. */ export function renderMonitorCheckId(origin) {
|
|
195
|
-
return
|
|
346
|
+
return renderMonitorHostCheckId(new URL(origin).host);
|
|
347
|
+
}
|
|
348
|
+
/** The state document id for one host, as {@link renderMonitorCheckId} names it. */ export function renderMonitorHostCheckId(host) {
|
|
349
|
+
return `${RENDER_MONITOR_CHECK_PREFIX}${host}`;
|
|
196
350
|
}
|
|
197
351
|
/**
|
|
198
352
|
* The next state for one site, given what this run saw. Pure.
|
|
@@ -201,15 +355,47 @@ export const RENDER_PROBES = [
|
|
|
201
355
|
* failed, so one slow cold start or one dropped connection says nothing. It
|
|
202
356
|
* turns back on the first passing run, because a render that works is
|
|
203
357
|
* proof, and a recovery held back is a recovery nobody hears about.
|
|
358
|
+
*
|
|
359
|
+
* A BLIND observation (every failure was a bot-protection challenge) changes
|
|
360
|
+
* nothing about the site: its status, failure count and since-time carry over
|
|
361
|
+
* untouched, and only the blind count moves. It transitions to `blind` once,
|
|
362
|
+
* on the `threshold`-th blind run in a row.
|
|
204
363
|
*/ export function nextRenderMonitorState(prior, observation, context) {
|
|
205
364
|
var _observation_detail, _ref, _ref1, _ref2, _ref3;
|
|
206
365
|
const { origin, label, threshold, now } = context;
|
|
207
366
|
const checkId = renderMonitorCheckId(origin);
|
|
208
367
|
const detail = String((_observation_detail = observation.detail) != null ? _observation_detail : '').slice(0, 1000);
|
|
209
368
|
const wasDegraded = (prior == null ? void 0 : prior.status) === 'degraded';
|
|
210
|
-
if (observation.ok) {
|
|
369
|
+
if (!observation.ok && observation.blind) {
|
|
370
|
+
var _ref4, _ref5, _ref6, _ref7, _ref8, _ref9, _ref10;
|
|
371
|
+
const blindRuns = ((_ref4 = prior == null ? void 0 : prior.consecutiveBlindRuns) != null ? _ref4 : 0) + 1;
|
|
211
372
|
return {
|
|
212
373
|
next: {
|
|
374
|
+
checkId,
|
|
375
|
+
label,
|
|
376
|
+
origin,
|
|
377
|
+
status: wasDegraded ? 'degraded' : 'ok',
|
|
378
|
+
sinceMs: (_ref5 = prior == null ? void 0 : prior.sinceMs) != null ? _ref5 : now,
|
|
379
|
+
detail: `The monitor cannot see this site: ${detail}`.slice(0, 1000),
|
|
380
|
+
updatedAtMs: now,
|
|
381
|
+
consecutiveFailures: (_ref6 = prior == null ? void 0 : prior.consecutiveFailures) != null ? _ref6 : 0,
|
|
382
|
+
failingSinceMs: (_ref7 = prior == null ? void 0 : prior.failingSinceMs) != null ? _ref7 : null,
|
|
383
|
+
lastOkAtMs: (_ref8 = prior == null ? void 0 : prior.lastOkAtMs) != null ? _ref8 : null,
|
|
384
|
+
consecutiveBlindRuns: blindRuns,
|
|
385
|
+
blindSinceMs: (_ref9 = prior == null ? void 0 : prior.blindSinceMs) != null ? _ref9 : now,
|
|
386
|
+
lastObservedAtMs: (_ref10 = prior == null ? void 0 : prior.lastObservedAtMs) != null ? _ref10 : null
|
|
387
|
+
},
|
|
388
|
+
transition: blindRuns === Math.max(1, threshold) ? 'blind' : null
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
const sighted = {
|
|
392
|
+
consecutiveBlindRuns: 0,
|
|
393
|
+
blindSinceMs: null,
|
|
394
|
+
lastObservedAtMs: now
|
|
395
|
+
};
|
|
396
|
+
if (observation.ok) {
|
|
397
|
+
return {
|
|
398
|
+
next: _extends({
|
|
213
399
|
checkId,
|
|
214
400
|
label,
|
|
215
401
|
origin,
|
|
@@ -220,7 +406,7 @@ export const RENDER_PROBES = [
|
|
|
220
406
|
consecutiveFailures: 0,
|
|
221
407
|
failingSinceMs: null,
|
|
222
408
|
lastOkAtMs: now
|
|
223
|
-
},
|
|
409
|
+
}, sighted),
|
|
224
410
|
transition: wasDegraded ? 'recovered' : null
|
|
225
411
|
};
|
|
226
412
|
}
|
|
@@ -228,7 +414,7 @@ export const RENDER_PROBES = [
|
|
|
228
414
|
const failingSinceMs = (_ref1 = prior == null ? void 0 : prior.failingSinceMs) != null ? _ref1 : now;
|
|
229
415
|
const degraded = wasDegraded || failures >= Math.max(1, threshold);
|
|
230
416
|
return {
|
|
231
|
-
next: {
|
|
417
|
+
next: _extends({
|
|
232
418
|
checkId,
|
|
233
419
|
label,
|
|
234
420
|
origin,
|
|
@@ -239,7 +425,7 @@ export const RENDER_PROBES = [
|
|
|
239
425
|
consecutiveFailures: failures,
|
|
240
426
|
failingSinceMs,
|
|
241
427
|
lastOkAtMs: (_ref3 = prior == null ? void 0 : prior.lastOkAtMs) != null ? _ref3 : null
|
|
242
|
-
},
|
|
428
|
+
}, sighted),
|
|
243
429
|
transition: degraded && !wasDegraded ? 'failing' : null
|
|
244
430
|
};
|
|
245
431
|
}
|
|
@@ -269,11 +455,20 @@ function describeDuration(ms) {
|
|
|
269
455
|
const rest = minutes % 60;
|
|
270
456
|
return rest ? `${hours} h ${rest} min` : `${hours} h`;
|
|
271
457
|
}
|
|
458
|
+
/**
|
|
459
|
+
* The `page` probe's URL for one run: this run's page of the site's list
|
|
460
|
+
* (rotating every five-minute slot), under this run's spelling of the site.
|
|
461
|
+
*/ export function pageProbeUrl(renderOrigin, site, slot, spelling) {
|
|
462
|
+
var _site_paths_;
|
|
463
|
+
const path = (_site_paths_ = site.paths[Math.abs(slot) % site.paths.length]) != null ? _site_paths_ : '/';
|
|
464
|
+
const url = new URL(path, renderOrigin);
|
|
465
|
+
url.searchParams.set('tenantHost', freshHostSpelling(site.host, spelling));
|
|
466
|
+
return url.toString();
|
|
467
|
+
}
|
|
272
468
|
function probeLine(row) {
|
|
273
469
|
return `${row.kind} ${new URL(row.url).pathname}: ${row.detail} (${row.ms} ms)`;
|
|
274
470
|
}
|
|
275
|
-
/** Fetch and grade one probe. Never throws. */ async function runProbe(
|
|
276
|
-
const url = `${origin}${spec.path(options.nonce)}`;
|
|
471
|
+
/** Fetch and grade one probe. Never throws. */ async function runProbe(url, spec, options) {
|
|
277
472
|
const startedAt = options.clock();
|
|
278
473
|
try {
|
|
279
474
|
var _response_headers_get;
|
|
@@ -323,7 +518,7 @@ function probeLine(row) {
|
|
|
323
518
|
* A dry run fetches and grades and writes nothing, so a person can ask
|
|
324
519
|
* "what would the monitor see right now" without moving its state.
|
|
325
520
|
*/ export async function runRenderMonitor(options = {}) {
|
|
326
|
-
var _options_env, _options_now, _options_store, _options_raise, _env_RENDER_MONITOR_FAILURE_THRESHOLD, _options_nonce;
|
|
521
|
+
var _options_env, _options_now, _options_store, _options_raise, _env_RENDER_MONITOR_FAILURE_THRESHOLD, _options_nonce, _options_fetcher, _options_headers, _options_timeoutMs;
|
|
327
522
|
const env = (_options_env = options.env) != null ? _options_env : process.env;
|
|
328
523
|
const clock = (_options_now = options.now) != null ? _options_now : Date.now;
|
|
329
524
|
const store = (_options_store = options.store) != null ? _options_store : firestoreStore;
|
|
@@ -332,26 +527,42 @@ function probeLine(row) {
|
|
|
332
527
|
const parsedThreshold = Number.parseInt(String((_env_RENDER_MONITOR_FAILURE_THRESHOLD = env['RENDER_MONITOR_FAILURE_THRESHOLD']) != null ? _env_RENDER_MONITOR_FAILURE_THRESHOLD : ''), 10);
|
|
333
528
|
const threshold = Number.isFinite(parsedThreshold) && parsedThreshold >= 1 ? parsedThreshold : RENDER_MONITOR_DEFAULT_THRESHOLD;
|
|
334
529
|
const nonce = (_options_nonce = options.nonce) != null ? _options_nonce : ()=>`${clock().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
|
|
530
|
+
const probeOptions = {
|
|
531
|
+
fetcher: (_options_fetcher = options.fetcher) != null ? _options_fetcher : fetch,
|
|
532
|
+
headers: (_options_headers = options.headers) != null ? _options_headers : {},
|
|
533
|
+
timeoutMs: (_options_timeoutMs = options.timeoutMs) != null ? _options_timeoutMs : RENDER_MONITOR_TIMEOUT_MS,
|
|
534
|
+
clock
|
|
535
|
+
};
|
|
536
|
+
// Real pages need the render origin AND the bypass that gets past its
|
|
537
|
+
// Deployment Protection; without both a `page` probe could only fail.
|
|
538
|
+
const { renderOrigin, pages } = resolveRenderMonitorPages(env);
|
|
539
|
+
const canRenderPages = Boolean(renderOrigin && probeOptions.headers['x-vercel-protection-bypass']);
|
|
540
|
+
const runAt = clock();
|
|
541
|
+
const slot = Math.floor(runAt / 300000);
|
|
335
542
|
const sites = await Promise.all(resolveRenderMonitorTargets(env).map(async (origin)=>{
|
|
336
|
-
const
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
543
|
+
const layoutProbes = RENDER_PROBES.map((spec)=>runProbe(`${origin}${spec.path(nonce())}`, spec, probeOptions));
|
|
544
|
+
const site = canRenderPages ? pages.find((page)=>page.origin === origin) : undefined;
|
|
545
|
+
const pageProbe = site ? [
|
|
546
|
+
runProbe(pageProbeUrl(renderOrigin, site, slot, renderSpellingNumber(runAt)), {
|
|
547
|
+
kind: 'page',
|
|
548
|
+
expectStatus: 200
|
|
549
|
+
}, probeOptions)
|
|
550
|
+
] : [];
|
|
551
|
+
const probes = await Promise.all([
|
|
552
|
+
...layoutProbes,
|
|
553
|
+
...pageProbe
|
|
554
|
+
]);
|
|
346
555
|
const ok = probes.every((probe)=>probe.ok);
|
|
347
|
-
const
|
|
556
|
+
const failing = probes.filter((probe)=>!probe.ok);
|
|
557
|
+
const blind = !ok && failing.every((probe)=>probe.challenged);
|
|
558
|
+
const result = {
|
|
348
559
|
origin,
|
|
349
560
|
ok,
|
|
561
|
+
blind,
|
|
350
562
|
probes
|
|
351
563
|
};
|
|
352
|
-
if (dryRun) return
|
|
564
|
+
if (dryRun) return result;
|
|
353
565
|
const host = new URL(origin).host;
|
|
354
|
-
const failing = probes.filter((probe)=>!probe.ok);
|
|
355
566
|
const detail = ok ? 'Fresh renders pass.' : failing.map(probeLine).join('; ');
|
|
356
567
|
try {
|
|
357
568
|
const now = clock();
|
|
@@ -359,7 +570,8 @@ function probeLine(row) {
|
|
|
359
570
|
var _ref, _ref1;
|
|
360
571
|
const decided = nextRenderMonitorState(prior, {
|
|
361
572
|
ok,
|
|
362
|
-
detail
|
|
573
|
+
detail,
|
|
574
|
+
blind
|
|
363
575
|
}, {
|
|
364
576
|
origin,
|
|
365
577
|
label: `Page rendering on ${host}`,
|
|
@@ -374,9 +586,9 @@ function probeLine(row) {
|
|
|
374
586
|
})
|
|
375
587
|
};
|
|
376
588
|
});
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
589
|
+
result.status = next.status;
|
|
590
|
+
result.consecutiveFailures = next.consecutiveFailures;
|
|
591
|
+
result.transition = transition;
|
|
380
592
|
if (transition === 'failing') {
|
|
381
593
|
await raise('system.siteRenderFailing', {
|
|
382
594
|
dedupeKey: host,
|
|
@@ -387,6 +599,16 @@ function probeLine(row) {
|
|
|
387
599
|
detail
|
|
388
600
|
}
|
|
389
601
|
});
|
|
602
|
+
} else if (transition === 'blind') {
|
|
603
|
+
var _next_consecutiveBlindRuns;
|
|
604
|
+
await raise('system.renderMonitorBlind', {
|
|
605
|
+
dedupeKey: host,
|
|
606
|
+
context: {
|
|
607
|
+
site: host,
|
|
608
|
+
runs: (_next_consecutiveBlindRuns = next.consecutiveBlindRuns) != null ? _next_consecutiveBlindRuns : threshold,
|
|
609
|
+
detail
|
|
610
|
+
}
|
|
611
|
+
});
|
|
390
612
|
} else if (transition === 'recovered') {
|
|
391
613
|
await raise('system.siteRenderRecovered', {
|
|
392
614
|
dedupeKey: host,
|
|
@@ -397,10 +619,10 @@ function probeLine(row) {
|
|
|
397
619
|
});
|
|
398
620
|
}
|
|
399
621
|
} catch (error) {
|
|
400
|
-
|
|
622
|
+
result.error = error instanceof Error ? error.message : String(error);
|
|
401
623
|
console.error(`[render-monitor] ${host} could not be recorded`, error);
|
|
402
624
|
}
|
|
403
|
-
return
|
|
625
|
+
return result;
|
|
404
626
|
}));
|
|
405
627
|
return {
|
|
406
628
|
dryRun,
|
|
@@ -408,5 +630,141 @@ function probeLine(row) {
|
|
|
408
630
|
sites
|
|
409
631
|
};
|
|
410
632
|
}
|
|
633
|
+
/**
|
|
634
|
+
* A verdict older than this says the monitor stopped, not that the site did.
|
|
635
|
+
* Four missed five-minute runs.
|
|
636
|
+
*/ export const RENDER_PAGES_STALE_MS = 20 * 60000;
|
|
637
|
+
/** How many sites one `?site=` may ask about. */ export const RENDER_PAGES_MAX_SITES = 10;
|
|
638
|
+
/**
|
|
639
|
+
* Which watched hosts a `?site=` asks about. Pure.
|
|
640
|
+
*
|
|
641
|
+
* Absent or empty asks about every watched site. A host that is not watched
|
|
642
|
+
* is kept, so the caller can answer it red: a public monitor pointed at a
|
|
643
|
+
* site nobody renders is a monitor that can never go red, which is the
|
|
644
|
+
* mistake this door exists to undo.
|
|
645
|
+
*/ export function requestedRenderPagesHosts(query, watchedOrigins) {
|
|
646
|
+
const asked = String(query != null ? query : '').split(/[\s,]+/).map((entry)=>entry.trim().toLowerCase().replace(/^https?:\/\//, '').replace(/\/.*$/, '')).filter((entry)=>/^[a-z0-9-]+(\.[a-z0-9-]+)+$/.test(entry));
|
|
647
|
+
const hosts = asked.length ? asked : watchedOrigins.map((origin)=>new URL(origin).host);
|
|
648
|
+
return [
|
|
649
|
+
...new Set(hosts)
|
|
650
|
+
].slice(0, RENDER_PAGES_MAX_SITES);
|
|
651
|
+
}
|
|
652
|
+
/**
|
|
653
|
+
* The public verdict on real pages, per site, from the render monitor's
|
|
654
|
+
* stored state (AGL-3580). Pure.
|
|
655
|
+
*
|
|
656
|
+
* This is what the status page's `Published sites` and `Marketing site`
|
|
657
|
+
* monitors read. Before it they read `/api/health/render/*`, which builds a
|
|
658
|
+
* node tree inside a route handler and never draws a page, and both stayed at
|
|
659
|
+
* 100% through two outages on 2026-10-05 in which every fresh page hung.
|
|
660
|
+
*
|
|
661
|
+
* Red is reserved for the one thing a visitor would notice:
|
|
662
|
+
*
|
|
663
|
+
* - `not-rendering` — the monitor saw the site fail to render pages it could
|
|
664
|
+
* not have served from a cache, `threshold` runs in a row.
|
|
665
|
+
* - `not-watched` — the monitor renders nothing for this host, so this row
|
|
666
|
+
* could never go red; fix the monitor's config or the URL.
|
|
667
|
+
* - `state-unavailable` — the verdicts could not be read at all.
|
|
668
|
+
*
|
|
669
|
+
* Green with a code is a verdict this door cannot improve on, and is never
|
|
670
|
+
* reported as an outage, because it is not one:
|
|
671
|
+
*
|
|
672
|
+
* - `awaiting-first-run` — a site added since the monitor last ran.
|
|
673
|
+
* - `monitor-blind` — bot protection answered the monitor; the site's last
|
|
674
|
+
* real verdict stands, and the operator alert says what to fix.
|
|
675
|
+
* - `observation-stale` — no run in {@link RENDER_PAGES_STALE_MS}; the
|
|
676
|
+
* monitor's own `render-monitor` row on `/api/health/crons` is what goes
|
|
677
|
+
* red for that, so a stopped scheduler is never told as a site outage.
|
|
678
|
+
*/ export function renderPagesHealth(hosts, watchedOrigins, states, ms, now = Date.now()) {
|
|
679
|
+
const watched = new Set(watchedOrigins.map((origin)=>new URL(origin).host));
|
|
680
|
+
const iso = (value)=>typeof value === 'number' && Number.isFinite(value) ? new Date(value).toISOString() : null;
|
|
681
|
+
const checks = {};
|
|
682
|
+
if (!hosts.length) {
|
|
683
|
+
checks['sites'] = {
|
|
684
|
+
ok: false,
|
|
685
|
+
ms,
|
|
686
|
+
code: 'no-sites-watched',
|
|
687
|
+
lastObservedAt: null,
|
|
688
|
+
since: null
|
|
689
|
+
};
|
|
690
|
+
return checks;
|
|
691
|
+
}
|
|
692
|
+
for (const host of hosts){
|
|
693
|
+
var _ref, _state_lastObservedAtMs, _state_consecutiveBlindRuns, _state_updatedAtMs;
|
|
694
|
+
const blank = {
|
|
695
|
+
ms,
|
|
696
|
+
lastObservedAt: null,
|
|
697
|
+
since: null
|
|
698
|
+
};
|
|
699
|
+
if (!watched.has(host)) {
|
|
700
|
+
checks[host] = _extends({}, blank, {
|
|
701
|
+
ok: false,
|
|
702
|
+
code: 'not-watched'
|
|
703
|
+
});
|
|
704
|
+
continue;
|
|
705
|
+
}
|
|
706
|
+
if (states === null) {
|
|
707
|
+
checks[host] = _extends({}, blank, {
|
|
708
|
+
ok: false,
|
|
709
|
+
code: 'state-unavailable'
|
|
710
|
+
});
|
|
711
|
+
continue;
|
|
712
|
+
}
|
|
713
|
+
const state = states.get(renderMonitorHostCheckId(host));
|
|
714
|
+
if (!state) {
|
|
715
|
+
checks[host] = _extends({}, blank, {
|
|
716
|
+
ok: true,
|
|
717
|
+
code: 'awaiting-first-run'
|
|
718
|
+
});
|
|
719
|
+
continue;
|
|
720
|
+
}
|
|
721
|
+
const observed = (_ref = (_state_lastObservedAtMs = state.lastObservedAtMs) != null ? _state_lastObservedAtMs : state.lastOkAtMs) != null ? _ref : null;
|
|
722
|
+
const row = {
|
|
723
|
+
ms,
|
|
724
|
+
lastObservedAt: iso(observed),
|
|
725
|
+
since: iso(state.sinceMs)
|
|
726
|
+
};
|
|
727
|
+
if (state.status === 'degraded') {
|
|
728
|
+
checks[host] = _extends({}, row, {
|
|
729
|
+
ok: false,
|
|
730
|
+
code: 'not-rendering'
|
|
731
|
+
});
|
|
732
|
+
} else if (((_state_consecutiveBlindRuns = state.consecutiveBlindRuns) != null ? _state_consecutiveBlindRuns : 0) > 0) {
|
|
733
|
+
checks[host] = _extends({}, row, {
|
|
734
|
+
ok: true,
|
|
735
|
+
code: 'monitor-blind'
|
|
736
|
+
});
|
|
737
|
+
} else if (now - ((_state_updatedAtMs = state.updatedAtMs) != null ? _state_updatedAtMs : 0) > RENDER_PAGES_STALE_MS) {
|
|
738
|
+
checks[host] = _extends({}, row, {
|
|
739
|
+
ok: true,
|
|
740
|
+
code: 'observation-stale'
|
|
741
|
+
});
|
|
742
|
+
} else {
|
|
743
|
+
checks[host] = _extends({}, row, {
|
|
744
|
+
ok: true
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
return checks;
|
|
749
|
+
}
|
|
750
|
+
/**
|
|
751
|
+
* Read the stored verdicts for `hosts`, one batched read. Null when the read
|
|
752
|
+
* fails, which {@link renderPagesHealth} reports red rather than calm.
|
|
753
|
+
*/ export async function readRenderMonitorStates(hosts) {
|
|
754
|
+
try {
|
|
755
|
+
const { default: firebaseAdmin } = await import("./firebase-admin.js");
|
|
756
|
+
const db = firebaseAdmin.app().firestore();
|
|
757
|
+
const refs = hosts.map((host)=>db.collection(RENDER_MONITOR_COLLECTION).doc(renderMonitorHostCheckId(host)));
|
|
758
|
+
const snapshots = refs.length ? await db.getAll(...refs) : [];
|
|
759
|
+
const states = new Map();
|
|
760
|
+
for (const snapshot of snapshots){
|
|
761
|
+
if (snapshot.exists) states.set(snapshot.id, snapshot.data());
|
|
762
|
+
}
|
|
763
|
+
return states;
|
|
764
|
+
} catch (error) {
|
|
765
|
+
console.error('[render-monitor] stored verdicts could not be read', error);
|
|
766
|
+
return null;
|
|
767
|
+
}
|
|
768
|
+
}
|
|
411
769
|
|
|
412
770
|
//# sourceMappingURL=render-monitor.js.map
|