@ultimat3/cli 27.7.0 → 27.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ultimat3/cli",
3
- "version": "27.7.0",
3
+ "version": "27.8.0",
4
4
  "description": "The `x` binary: new, dev, build, verify, generate, db, mcp, doctor, deploy",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -39,33 +39,33 @@
39
39
  },
40
40
  "dependencies": {
41
41
  "@babel/core": "^8.0.1",
42
- "@ultimat3/action": "27.7.0",
43
- "@ultimat3/admin": "27.7.0",
44
- "@ultimat3/ai": "27.7.0",
45
- "@ultimat3/auth": "27.7.0",
46
- "@ultimat3/cache": "27.7.0",
47
- "@ultimat3/core": "27.7.0",
48
- "@ultimat3/db": "27.7.0",
49
- "@ultimat3/entity": "27.7.0",
50
- "@ultimat3/flags": "27.7.0",
51
- "@ultimat3/http": "27.7.0",
52
- "@ultimat3/i18n": "27.7.0",
53
- "@ultimat3/jobs": "27.7.0",
54
- "@ultimat3/mail": "27.7.0",
55
- "@ultimat3/manifest": "27.7.0",
56
- "@ultimat3/mcp": "27.7.0",
57
- "@ultimat3/money": "27.7.0",
58
- "@ultimat3/notify": "27.7.0",
59
- "@ultimat3/policy": "27.7.0",
60
- "@ultimat3/pwa": "27.7.0",
61
- "@ultimat3/query": "27.7.0",
62
- "@ultimat3/realtime": "27.7.0",
63
- "@ultimat3/render": "27.7.0",
64
- "@ultimat3/schema": "27.7.0",
65
- "@ultimat3/seo": "27.7.0",
66
- "@ultimat3/storage": "27.7.0",
67
- "@ultimat3/testing": "27.7.0",
68
- "@ultimat3/time": "27.7.0",
42
+ "@ultimat3/action": "27.8.0",
43
+ "@ultimat3/admin": "27.8.0",
44
+ "@ultimat3/ai": "27.8.0",
45
+ "@ultimat3/auth": "27.8.0",
46
+ "@ultimat3/cache": "27.8.0",
47
+ "@ultimat3/core": "27.8.0",
48
+ "@ultimat3/db": "27.8.0",
49
+ "@ultimat3/entity": "27.8.0",
50
+ "@ultimat3/flags": "27.8.0",
51
+ "@ultimat3/http": "27.8.0",
52
+ "@ultimat3/i18n": "27.8.0",
53
+ "@ultimat3/jobs": "27.8.0",
54
+ "@ultimat3/mail": "27.8.0",
55
+ "@ultimat3/manifest": "27.8.0",
56
+ "@ultimat3/mcp": "27.8.0",
57
+ "@ultimat3/money": "27.8.0",
58
+ "@ultimat3/notify": "27.8.0",
59
+ "@ultimat3/policy": "27.8.0",
60
+ "@ultimat3/pwa": "27.8.0",
61
+ "@ultimat3/query": "27.8.0",
62
+ "@ultimat3/realtime": "27.8.0",
63
+ "@ultimat3/render": "27.8.0",
64
+ "@ultimat3/schema": "27.8.0",
65
+ "@ultimat3/seo": "27.8.0",
66
+ "@ultimat3/storage": "27.8.0",
67
+ "@ultimat3/testing": "27.8.0",
68
+ "@ultimat3/time": "27.8.0",
69
69
  "babel-preset-solid": "^1.9.15"
70
70
  }
71
71
  }
@@ -113,10 +113,25 @@ const routeFacts = (): readonly RouteFact[] =>
113
113
  offline: route.offline,
114
114
  hydrate: route.hydrate,
115
115
  revalidateTags: route.revalidateTags,
116
+ ...revalidateOf(route),
116
117
  surface: route.surface,
117
118
  ...budgetOf(route),
118
119
  }));
119
120
 
121
+ /** An `isr` route's invalidation and key facts — `onInvalidate` always, the other two as declared. */
122
+ function revalidateOf(
123
+ route: RouteDescriptor,
124
+ ): Pick<RouteFact, 'onInvalidate' | 'maxStale' | 'query'> {
125
+ if (route.mode !== 'isr') return {};
126
+ const maxStale = route.revalidateMaxStale ?? null;
127
+ const query = route.revalidateQuery ?? null;
128
+ return {
129
+ onInvalidate: route.revalidateOnInvalidate ?? 'stale',
130
+ ...(maxStale === null ? {} : { maxStale }),
131
+ ...(query === null ? {} : { query }),
132
+ };
133
+ }
134
+
120
135
  function budgetOf(route: RouteDescriptor): { budget?: { js: string } } {
121
136
  return route.budgetJs === null ? {} : { budget: { js: route.budgetJs } };
122
137
  }
package/src/dev-boot.ts CHANGED
@@ -120,6 +120,8 @@ export async function startDev(options: StartDevOptions): Promise<DevServer> {
120
120
  appRuntime,
121
121
  'apply',
122
122
  options.roles ?? DEV_ROLES,
123
+ // Every role in one process: a bus that is away must not take the pages down with it.
124
+ true,
123
125
  );
124
126
  // The events binding the boot line reports is the bus the runtime chose, read off it.
125
127
  const services = withRealtimeEvents(resolved, env, runtime.realtime);
@@ -0,0 +1,63 @@
1
+ // What `x verify` says about the query an `isr` route keys on. Advice and not a finding: the scan
2
+ // reads source text, so it can miss a read made through a helper and can name one that is not a
3
+ // query read at all — and a route that declared nothing works as it always did. Both are exactly
4
+ // what a reader should hear about before a visitor finds it.
5
+
6
+ // why: Bun exposes no path-join primitive; Bun.file takes one already joined.
7
+ import { join } from 'node:path';
8
+ import { stripComments } from '@ultimat3/core';
9
+ import { describePages } from '@ultimat3/render';
10
+ import { msg } from './messages';
11
+
12
+ const MEMBER_READ = /\bquery\s*\??\.\s*([A-Za-z_$][\w$]*)\b(?!\s*\()/g;
13
+ const BRACKET_READ = /\bquery\s*(?:\?\.)?\[\s*['"]([^'"]+)['"]\s*\]/g;
14
+ const SEARCH_PARAM_READ = /\bsearchParams\s*\??\.\s*(?:get|getAll|has)\(\s*['"]([^'"]+)['"]/g;
15
+
16
+ /**
17
+ * The query parameter names a route module reads, as far as its text says: `query.name`,
18
+ * `query?.name`, `query['name']` and `searchParams.get('name')` (also `getAll`, `has`). A call
19
+ * (`query.where(…)`) is a method, not a parameter. Sorted, each once.
20
+ */
21
+ export function queryReads(source: string): readonly string[] {
22
+ const code = stripComments(source);
23
+ const names = new Set<string>();
24
+ for (const pattern of [MEMBER_READ, BRACKET_READ, SEARCH_PARAM_READ]) {
25
+ for (const match of code.matchAll(pattern)) {
26
+ if (match[1] !== undefined) names.add(match[1]);
27
+ }
28
+ }
29
+ return [...names].sort();
30
+ }
31
+
32
+ const listed = (names: readonly string[]): string => names.map((name) => `'${name}'`).join(', ');
33
+
34
+ /**
35
+ * One line per `isr` route with no `revalidate.query` — it renders and stores a page for every
36
+ * distinct query string, so `?x=1`, `?x=2`, … evict its real pages and cost a render each — and
37
+ * one per parameter a route's module reads and its declaration omits: the page is stored under a
38
+ * key without it, so every visitor gets the document rendered for whichever value arrived first.
39
+ */
40
+ export async function isrQueryAdvice(root: string): Promise<readonly string[]> {
41
+ const advice: string[] = [];
42
+ for (const route of describePages()) {
43
+ if (route.mode !== 'isr') continue;
44
+ const declared = route.revalidateQuery ?? null;
45
+ if (declared === null) {
46
+ advice.push(msg('cli.verify.isrQueryUndeclared', { file: route.file }));
47
+ continue;
48
+ }
49
+ const module = Bun.file(join(root, route.file));
50
+ if (!(await module.exists())) continue;
51
+ for (const name of queryReads(await module.text())) {
52
+ if (declared.includes(name)) continue;
53
+ advice.push(
54
+ msg('cli.verify.isrQueryOmitted', {
55
+ file: route.file,
56
+ name,
57
+ declared: listed([...declared, name].sort()),
58
+ }),
59
+ );
60
+ }
61
+ }
62
+ return advice;
63
+ }
@@ -0,0 +1,49 @@
1
+ // The `query` object a page component is handed. For an `isr` route that declared
2
+ // `revalidate.query` it holds the declared parameters only — and, outside production, it says so
3
+ // once when the page reads any other one: that read is a document varying on something its stored
4
+ // key does not carry, which is one visitor's page answered to every visitor after.
5
+
6
+ import type { ResolveEnvironmentOptions } from '@ultimat3/core';
7
+ import { logger, tryResolveEnvironment } from '@ultimat3/core';
8
+ import type { RouteEntry } from '@ultimat3/render';
9
+
10
+ type Query = Readonly<Record<string, string>>;
11
+
12
+ /** `<route file>\u0000<parameter>` already told — bounded by what the app's source can read. */
13
+ const told = new Set<string>();
14
+
15
+ /** Names every object answers and a renderer or a serializer may ask for: never a parameter. */
16
+ const isProbe = (name: string): boolean =>
17
+ name in Object.prototype || name === 'then' || name === 'toJSON';
18
+
19
+ export function routeQuery(
20
+ entry: RouteEntry,
21
+ url: URL,
22
+ env?: ResolveEnvironmentOptions['env'],
23
+ ): Query {
24
+ const query: Query = Object.fromEntries(url.searchParams);
25
+ const declared = entry.config.render === 'isr' ? entry.config.revalidate?.query : undefined;
26
+ if (declared === undefined) return query;
27
+ // Production pays nothing: the plain object, no trap. (`undefined`: an unreadable environment,
28
+ // which is not a development one either.)
29
+ const environment = tryResolveEnvironment(env === undefined ? undefined : { env });
30
+ if (environment === 'production' || environment === undefined) return query;
31
+ return new Proxy(query, {
32
+ get(target, name, receiver) {
33
+ if (typeof name === 'string' && !declared.includes(name) && !isProbe(name)) {
34
+ const key = `${entry.file}\u0000${name}`;
35
+ if (!told.has(key)) {
36
+ told.add(key);
37
+ const widened = [...declared, name].sort().map((one) => `'${one}'`);
38
+ logger.warn('isr.query.unkeyed-read', {
39
+ route: entry.file,
40
+ param: name,
41
+ cause: `the page read query.${name}, and revalidate.query does not list it — the stored page is keyed without it, so every visitor gets the document rendered for whichever value arrived first`,
42
+ fix: `query: [${widened.join(', ')}] // in revalidate, in ${entry.file}`,
43
+ });
44
+ }
45
+ }
46
+ return Reflect.get(target, name, receiver);
47
+ },
48
+ });
49
+ }
package/src/messages.ts CHANGED
@@ -328,6 +328,10 @@ const CATALOG = {
328
328
  // counts — so the line says how many and `x test <type>` prints the names.
329
329
  'cli.verify.allSkipped': 'found {skipped} test(s) and every one skipped itself',
330
330
  'cli.verify.warnings': '{count} warning(s) — --verbose or --json lists them',
331
+ 'cli.verify.isrQueryUndeclared':
332
+ "{file}: render 'isr' with no revalidate.query — its stored pages are keyed on the whole query string, so any visitor can mint one per request. Declare revalidate: { query: [] } (or the parameters the page varies on)",
333
+ 'cli.verify.isrQueryOmitted':
334
+ '{file}: reads the query parameter "{name}", which revalidate.query does not list — the page is stored under a key without it, so every visitor gets the document rendered for whichever value arrived first. Declare query: [{declared}], or stop reading it',
331
335
  // The other shape the same counts can take: a suite that reported no test at all, skipped ones
332
336
  // included. Its own line and not `allSkipped` with a zero, because the repair differs — there is
333
337
  // no skipped test to read a reason off, so the files the step selected hold no test.
@@ -0,0 +1,66 @@
1
+ // Single responsibility: what the `sync` role does when its node cannot start because the bus is
2
+ // away. A `ROLE=sync` container refuses its boot — it has nothing to serve without the bus.
3
+ // `x dev` runs every role in one process, so there the node stays mounted and NOT ready, and is
4
+ // started when the bus answers: the pages, the worker and the scheduler never went down with it.
5
+
6
+ import { isUltimateError, logger } from '@ultimat3/core';
7
+ import type { SyncNode, Transport } from '@ultimat3/realtime/server';
8
+
9
+ export interface BusWaitRuntime {
10
+ readonly transport: Pick<Transport, 'onReconnect'>;
11
+ /** `RunningServices.busTolerated`: the boot is `x dev`, and a bus that is away is waited for. */
12
+ readonly busTolerated?: boolean | undefined;
13
+ }
14
+
15
+ /**
16
+ * Start the node, or — tolerated, with the bus away — leave it unstarted and start it on the
17
+ * transport's next recovery. Until then an upgrade is shed with a 503 and a retry delay
18
+ * (`sync-upgrade.ts`), so an open page keeps asking and goes live with no restart. Any other
19
+ * refusal, and every refusal outside `x dev`, is the boot's. Returns what stops the wait.
20
+ */
21
+ export async function startOrWaitForBus(
22
+ node: Pick<SyncNode, 'start'>,
23
+ runtime: BusWaitRuntime,
24
+ ): Promise<() => void> {
25
+ const refused = await node.start().then(
26
+ () => undefined,
27
+ (error: unknown) => ({ error }),
28
+ );
29
+ if (refused === undefined) return () => undefined;
30
+ const { error } = refused;
31
+ if (
32
+ runtime.busTolerated !== true ||
33
+ !isUltimateError(error) ||
34
+ error.code !== 'X_TRANSPORT_UNAVAILABLE'
35
+ ) {
36
+ throw error;
37
+ }
38
+ // Once, with the refusal a container would have exited on.
39
+ logger.warn('sync node is waiting for the bus: sockets are refused until it answers', {
40
+ code: error.code,
41
+ cause: error.cause,
42
+ fix: error.fix,
43
+ });
44
+ let waiting = true;
45
+ let off = (): void => undefined;
46
+ const attempt = (): void => {
47
+ if (!waiting) return;
48
+ void node.start().then(
49
+ () => {
50
+ if (!waiting) return;
51
+ waiting = false;
52
+ off();
53
+ logger.info('sync node started: the bus answered');
54
+ },
55
+ // Still away, or away again: the next recovery asks once more.
56
+ () => undefined,
57
+ );
58
+ };
59
+ off = runtime.transport.onReconnect(attempt);
60
+ // The dial may have landed between the refusal and the line above; asking again costs nothing.
61
+ attempt();
62
+ return () => {
63
+ waiting = false;
64
+ off();
65
+ };
66
+ }
package/src/role-sync.ts CHANGED
@@ -26,6 +26,7 @@ import {
26
26
  import { neighbouringPort, PORT_RANGE, portPairAfter } from './flag-number';
27
27
  import { portFree } from './port-probe';
28
28
  import type { StartRolesOptions } from './role-start';
29
+ import { startOrWaitForBus } from './role-sync-bus-wait';
29
30
  import type { Env } from './runtime-bindings';
30
31
  import { syncAuthenticator } from './sync-authenticator';
31
32
  import { syncOriginsFrom } from './sync-url';
@@ -327,14 +328,17 @@ export async function prepareSync(options: StartRolesOptions): Promise<PreparedS
327
328
  ttlMs: options.runtime.presenceTtlMs,
328
329
  }),
329
330
  });
330
- await node.start();
331
+ const stopWaiting = await startOrWaitForBus(node, options.runtime);
331
332
  return {
332
333
  node,
333
334
  registry,
334
335
  // The node's OWN path, asked rather than restated: `SyncNodeOptions.path` is settable and a
335
336
  // second copy of `/_x/sync` here is the copy that stays behind when it moves.
336
337
  mount: { path: node.path, fetch: node.fetch, websocket: node.websocket },
337
- stop: () => node.stop(),
338
+ stop: async () => {
339
+ stopWaiting();
340
+ await node.stop();
341
+ },
338
342
  listen: async (appUrl) => {
339
343
  origins.push(...devOrigins(options, origins, appUrl));
340
344
  return await listen(options, node, { registry, hub }, appUrl);
@@ -8,9 +8,15 @@
8
8
  // the background and a dead one is `degraded`: the pod still boots, serves and runs jobs. Until
9
9
  // 2026-10 every role awaited the dial and the JetStream bucket, so a web pod that restarted while
10
10
  // NATS was down took the app's pages, webhooks and send jobs down with a feature none of them use.
11
+ //
12
+ // `x dev` is the one boot that TOLERATES a missing bus for a role that serves from it: it runs
13
+ // every role in one process, so the `sync` role's refusal took the pages, the worker and the
14
+ // scheduler down with it — the same outage, on the developer's own machine. There the refusal is
15
+ // logged once, coded, the dial goes on in the background, and the pages run without live until it
16
+ // lands. A `ROLE=sync` container keeps the hard refusal: it has nothing else to serve.
11
17
 
12
18
  import type { RealtimeConfig, Role } from '@ultimat3/core';
13
- import { logger, registerReadinessCheck } from '@ultimat3/core';
19
+ import { isUltimateError, logger, registerReadinessCheck } from '@ultimat3/core';
14
20
  import type { BusUse, SelectTransportOptions, Transport } from '@ultimat3/realtime/server';
15
21
  import { selectTransport, setChannelTransport } from '@ultimat3/realtime/server';
16
22
  import type { Env } from './runtime-bindings';
@@ -55,8 +61,20 @@ export interface RunningBus {
55
61
  stop(): Promise<void>;
56
62
  }
57
63
 
64
+ /**
65
+ * How long `x dev` waits for a bus its realtime roles need before it boots without one. Short on
66
+ * purpose: a NATS started beside it is reachable by then, and one that is not is dialled in the
67
+ * background anyway.
68
+ */
69
+ export const DEV_BUS_WAIT_MS = 3_000;
70
+
58
71
  export interface SelectedBus {
59
72
  readonly use: BusUse;
73
+ /**
74
+ * `x dev` with a role that serves from the bus: a bus that is away is waited for in the
75
+ * background rather than refused (`RunningServices.busTolerated`).
76
+ */
77
+ readonly tolerant: boolean;
60
78
  /** What the sync role must give `PresenceRegistry` (`RunningServices.presenceTtlMs`). */
61
79
  readonly presenceTtlMs: number;
62
80
  /**
@@ -76,6 +94,8 @@ export interface SelectBusInput {
76
94
  readonly override?: Transport | undefined;
77
95
  /** Injected by a test: a fake dial, a short wait. */
78
96
  readonly select?: SelectTransportOptions | undefined;
97
+ /** The boot is `x dev`: every role in one process, on the developer's machine. */
98
+ readonly dev?: boolean | undefined;
79
99
  }
80
100
 
81
101
  /**
@@ -84,9 +104,15 @@ export interface SelectBusInput {
84
104
  */
85
105
  export function selectBus(input: SelectBusInput): SelectedBus {
86
106
  const use = busUseFor(input.roles, input.realtime);
87
- const selection = selectTransport(input.env, input.realtime, { ...input.select, use });
107
+ const tolerant = input.dev === true && use !== 'publish';
108
+ const selection = selectTransport(input.env, input.realtime, {
109
+ ...(tolerant ? { connectWithinMs: DEV_BUS_WAIT_MS } : {}),
110
+ ...input.select,
111
+ use,
112
+ });
88
113
  return {
89
114
  use,
115
+ tolerant,
90
116
  presenceTtlMs: selection.presenceTtlMs,
91
117
  async start(): Promise<RunningBus> {
92
118
  const started: (() => void | Promise<void>)[] = [];
@@ -109,7 +135,20 @@ export function selectBus(input: SelectBusInput): SelectedBus {
109
135
  // Before the dial, not after: a dial refused for being late is still in flight inside
110
136
  // the client library, and closing the transport is what releases whatever it lands as.
111
137
  started.push(() => selection.transport.close());
112
- await selection.connect();
138
+ try {
139
+ await selection.connect();
140
+ } catch (error) {
141
+ if (!tolerant || !isUltimateError(error) || error.code !== 'X_TRANSPORT_UNAVAILABLE') {
142
+ throw error;
143
+ }
144
+ // Said ONCE, with the refusal a container would have exited on. The transport is not
145
+ // closed: its dial loop goes on, and `ultimate bus` is the line that says it landed.
146
+ logger.warn('ultimate bus is away: x dev serves without live until it answers', {
147
+ code: error.code,
148
+ cause: error.cause,
149
+ fix: error.fix,
150
+ });
151
+ }
113
152
  transport = selection.transport;
114
153
  }
115
154
  // The bus an app's own code publishes a channel event on (`publishChannelEvent`), in EVERY
@@ -129,7 +168,8 @@ export function selectBus(input: SelectBusInput): SelectedBus {
129
168
  () => connectable.connected,
130
169
  // A publisher serves without the bus, so losing it must not pull the pod from the
131
170
  // ingress — it is reported, and `/readyz?deep=1` is where a monitor reads it.
132
- { onFailure: use === 'publish' ? 'degraded' : 'failing' },
171
+ // `x dev` serves its pages from the same process: its bus is a degradation too.
172
+ { onFailure: use === 'publish' || tolerant ? 'degraded' : 'failing' },
133
173
  ),
134
174
  );
135
175
  // Each time the bus comes (back) up, on one line: the boot line may have said
@@ -12,6 +12,7 @@ import {
12
12
  isNoopPurgeDriver,
13
13
  lruTier,
14
14
  memoTier,
15
+ purgeEdgeAgain,
15
16
  receiveInvalidationBroadcast,
16
17
  redisTier,
17
18
  registerInvalidationBroadcast,
@@ -19,7 +20,13 @@ import {
19
20
  resetTiers,
20
21
  } from '@ultimat3/cache';
21
22
  import type { AppConfig, CacheTierName } from '@ultimat3/core';
22
- import { backoffDelay, defineConfig, logger, renderThrowable } from '@ultimat3/core';
23
+ import {
24
+ backoffDelay,
25
+ defineConfig,
26
+ isOutageMilestone,
27
+ logger,
28
+ renderThrowable,
29
+ } from '@ultimat3/core';
23
30
  import type { Transport, TransportSubscription } from '@ultimat3/realtime/server';
24
31
  import type { Env } from './runtime-bindings';
25
32
 
@@ -206,6 +213,11 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
206
213
  defer(batch);
207
214
  throw error;
208
215
  }
216
+ // The edge AGAIN, now that the peers are finally being told. It was purged when the bust ran,
217
+ // but every peer still held its copy then, and each request one of them answered since handed
218
+ // the edge the old page back — with no purge left to come. (Each peer purges after its own
219
+ // delete too; this is the sender's half, and does not depend on hearing its own publish.)
220
+ await purgeEdgeAgain(batch);
209
221
  };
210
222
  // ONE settle at a time. It is asked for by every reconnect and after every accepted publish,
211
223
  // and two that overlapped both published the same batch (and the same flush-all). A call that
@@ -274,12 +286,16 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
274
286
  },
275
287
  );
276
288
  subscribed = true;
277
- if (attempt > 1) flushDeaf('cache.bus-subscribed');
289
+ if (attempt > 1) {
290
+ // The outage's one closing line, beside the thinned warnings it ends.
291
+ logger.info('cache.broadcast.subscribe-failed recovered', { after: attempt - 1 });
292
+ flushDeaf('cache.bus-subscribed');
293
+ }
278
294
  return subscription;
279
295
  } catch (error) {
280
296
  // Attempts 1, 2, 4, 8, …: the dial's own thinning (`NatsTransport`), so an outage is not
281
297
  // one line per retry per pod for as long as it lasts.
282
- if ((attempt & (attempt - 1)) === 0) {
298
+ if (isOutageMilestone(attempt)) {
283
299
  logger.warn('cache.broadcast.subscribe-failed', {
284
300
  error: broadcastErrorText(error),
285
301
  attempt,
@@ -2,25 +2,66 @@
2
2
  // decided. A returned discriminated outcome, never a throw: the controller's regeneration treats
3
3
  // anything thrown as a failure and logs it, and a redirect is an ordinary answer, not a failure.
4
4
 
5
+ import { logger } from '@ultimat3/core';
5
6
  import type { RedirectIntent, RequestContext, RouteParams } from '@ultimat3/http';
6
7
  import { asCtx, takeRedirect } from '@ultimat3/http';
7
8
  import type { RenderResult, RouteData, RouteEntry } from '@ultimat3/render';
8
- import { routeDataFor, routeStatusOf } from '@ultimat3/render';
9
+ import { routeDataFor, routeNoStoreOf, routeStatusOf } from '@ultimat3/render';
9
10
  import type { IsrController, IsrEntry } from '@ultimat3/render/server';
10
- import { isrKey } from '@ultimat3/render/server';
11
+ import { isrKey, isrRequestUrl, undeclaredQuery } from '@ultimat3/render/server';
11
12
 
12
13
  export type IsrOutcome =
13
14
  | { readonly kind: 'page'; readonly result: RenderResult }
14
15
  | { readonly kind: 'redirect'; readonly to: RedirectIntent };
15
16
 
17
+ /** What `load` and `meta` are given; `url` is a string because that is what `ld.*` embeds. */
18
+ type IsrData = { readonly url: string; readonly params: RouteParams } & Record<string, unknown>;
19
+
16
20
  export interface IsrRequest {
17
21
  readonly entry: RouteEntry;
18
- /** What `load` and `meta` are given; `url` is a string because that is what `ld.*` embeds. */
19
- readonly data: { readonly url: string; readonly params: RouteParams } & Record<string, unknown>;
22
+ /** The request as it arrived. `load`, `meta` and the page get `keyedData`'s narrowing of it. */
23
+ readonly data: IsrData;
20
24
  readonly ctx: RequestContext;
21
25
  readonly isr: IsrController;
22
- /** The route's whole document for what `load` returned (`runtime-render.ts`'s `documentFrom`). */
23
- readonly document: (loaded: RouteData) => Promise<string>;
26
+ /**
27
+ * The route's whole document for what `load` returned (`runtime-render.ts`'s `documentFrom`),
28
+ * rendered AT the data this file hands it — the narrowed URL, never the request's own.
29
+ */
30
+ readonly document: (loaded: RouteData, at: IsrData) => Promise<string>;
31
+ }
32
+
33
+ /** Route files already told, once, about a request whose query they never declared. */
34
+ const toldUndeclared = new Set<string>();
35
+
36
+ /** Names are the visitor's: a few, cut short, is all a log line needs to name the edit. */
37
+ const NAMED_PARAMS = 5;
38
+ const NAMED_PARAM_CHARS = 40;
39
+
40
+ /**
41
+ * The request narrowed to what the route declared it varies on (`revalidate.query`). ONE narrowing
42
+ * feeds the store key, `load`, `meta` and the page, so a parameter outside the key cannot reach
43
+ * the document stored under it.
44
+ *
45
+ * A route that declared NOTHING still keys on the whole query, and is told so once, the first time
46
+ * a visitor uses that: every distinct query string is then a render and a stored page of its own.
47
+ */
48
+ function keyedData(entry: RouteEntry, data: IsrData): IsrData {
49
+ const url = new URL(data.url);
50
+ const declared = entry.config.revalidate?.query ?? null;
51
+ if (declared === null && !toldUndeclared.has(entry.file)) {
52
+ const minted = undeclaredQuery(url, declared);
53
+ if (minted.length > 0) {
54
+ toldUndeclared.add(entry.file);
55
+ logger.warn('isr.query.undeclared', {
56
+ route: entry.file,
57
+ params: minted.slice(0, NAMED_PARAMS).map((name) => name.slice(0, NAMED_PARAM_CHARS)),
58
+ cause:
59
+ 'the route declares no revalidate.query, so every distinct query string is rendered and stored as its own page',
60
+ fix: `revalidate: { query: [] } // in ${entry.file}; or query: ['<name>'] for each parameter the page varies on`,
61
+ });
62
+ }
63
+ }
64
+ return { ...data, url: isrRequestUrl(url, declared).href };
24
65
  }
25
66
 
26
67
  /**
@@ -54,11 +95,12 @@ function dropPage(isr: IsrController, key: string): void {
54
95
  * the request's own (its `load` may have read a cookie), so a request that joined a run whose load
55
96
  * redirected decides for itself, exactly as the static single-flight in `appRoutes` does.
56
97
  */
57
- export async function isrOutcome(request: IsrRequest): Promise<IsrOutcome> {
98
+ export async function isrOutcome(arrived: IsrRequest): Promise<IsrOutcome> {
99
+ const request: IsrRequest = { ...arrived, data: keyedData(arrived.entry, arrived.data) };
58
100
  const { entry, data, ctx, isr } = request;
59
- // `isrKey(url, locale)`, never `url.pathname`: the query is part of what was rendered — `meta`
60
- // reads `data.url` — so two URLs differing only in their query are two documents (#171). The
61
- // locale is `ctx.locale`, the one the `locale` stage negotiated for THIS request.
101
+ // `isrKey(url, locale)`, never `url.pathname`: the DECLARED query is part of what was rendered —
102
+ // `meta` reads `data.url` — so two URLs differing in it are two documents (#171). The locale is
103
+ // `ctx.locale`, the one the `locale` stage negotiated for THIS request.
62
104
  const key = isrKey(new URL(data.url), asCtx(ctx).locale);
63
105
  let decided: RedirectIntent | undefined;
64
106
  // `{ html, status }`, never the bare string: the entry stores the status beside the HTML and
@@ -66,8 +108,13 @@ export async function isrOutcome(request: IsrRequest): Promise<IsrOutcome> {
66
108
  const served = await isr.serve(key, async () => {
67
109
  const loaded = await routeDataFor(entry.config, data);
68
110
  const to = takeRedirect(ctx);
69
- if (to === undefined)
70
- return { html: await request.document(loaded), status: routeStatusOf(loaded) };
111
+ if (to === undefined) {
112
+ return {
113
+ html: await request.document(loaded, data),
114
+ status: routeStatusOf(loaded),
115
+ noStore: routeNoStoreOf(loaded),
116
+ };
117
+ }
71
118
  // Set by THIS request's producer only. On a stale page it runs behind the answer, and when it
72
119
  // has already decided by the time `serve` returns, the redirect is still this request's own.
73
120
  decided = to;
@@ -88,7 +135,7 @@ async function ownOutcome(request: IsrRequest): Promise<IsrOutcome> {
88
135
  const loaded = await routeDataFor(request.entry.config, request.data);
89
136
  const to = takeRedirect(request.ctx);
90
137
  if (to !== undefined) return { kind: 'redirect', to };
91
- const body = await request.document(loaded);
138
+ const body = await request.document(loaded, request.data);
92
139
  return {
93
140
  kind: 'page',
94
141
  result: {
@@ -2,6 +2,7 @@
2
2
  // and handed back with the detach the boot puts on its stop list. One function for `x dev` and the
3
3
  // container, so neither can build a controller `invalidateTags` never reaches (plan 101, s2-con #1).
4
4
 
5
+ import { describePages, RouteModeInvalidError } from '@ultimat3/render';
5
6
  import type { IsrController, IsrStore } from '@ultimat3/render/server';
6
7
  import { isrController } from '@ultimat3/render/server';
7
8
 
@@ -18,10 +19,16 @@ export interface AttachedIsr {
18
19
  export function attachedIsr(options: {
19
20
  readonly buildId: string;
20
21
  readonly store?: IsrStore;
22
+ /** `isrController`'s own; a test turns the cooldown off rather than waiting one out. */
23
+ readonly failureCooldownMs?: number;
21
24
  }): AttachedIsr {
25
+ if (options.store !== undefined) assertPurgeSafe(options.store);
22
26
  const isr = isrController({
23
27
  buildId: options.buildId,
24
28
  ...(options.store === undefined ? {} : { store: options.store }),
29
+ ...(options.failureCooldownMs === undefined
30
+ ? {}
31
+ : { failureCooldownMs: options.failureCooldownMs }),
25
32
  });
26
33
  const detach = isr.attach();
27
34
  let released = false;
@@ -34,3 +41,19 @@ export function attachedIsr(options: {
34
41
  },
35
42
  };
36
43
  }
44
+
45
+ /**
46
+ * A store the deployment supplied is one other processes may write. Under `onInvalidate: 'purge'`
47
+ * a replica that began a render before the purge, and has not heard of it yet, writes the purged
48
+ * page back unless the STORE refuses the write (`IsrStore.tagFence`) — its own fence only knows
49
+ * the busts it was told about. Refused here, by route, rather than left to happen in production.
50
+ */
51
+ function assertPurgeSafe(store: IsrStore): void {
52
+ if (store.tagFence !== undefined) return;
53
+ const purging = describePages().find((route) => route.revalidateOnInvalidate === 'purge');
54
+ if (purging === undefined) return;
55
+ throw new RouteModeInvalidError(
56
+ `${purging.file} declares revalidate.onInvalidate: 'purge', and the runtime's isrStore has no tagFence — a replica whose render began before a purge would write the purged page back into it`,
57
+ 'tagFence: { sample, bump, setIfCurrent } // implement IsrTagFence on the isrStore in apps/web/runtime.ts (setIfCurrent atomic in the store), or drop isrStore to keep one memory store per replica',
58
+ );
59
+ }
@@ -45,6 +45,7 @@ import {
45
45
  streamResult,
46
46
  } from '@ultimat3/render/server';
47
47
  import type { DocumentOptions } from './document-options';
48
+ import { routeQuery } from './isr-query-guard';
48
49
  import {
49
50
  type NavigationDocumentHead,
50
51
  navigationMetaOf,
@@ -193,7 +194,7 @@ export async function routeBody(
193
194
  data,
194
195
  params: ctx.params,
195
196
  url: ctx.url,
196
- query: Object.fromEntries(url.searchParams) as Readonly<Record<string, string>>,
197
+ query: routeQuery(entry, url),
197
198
  },
198
199
  entry.file,
199
200
  { islands },
@@ -486,8 +487,8 @@ async function renderIsr(
486
487
  options: DevRenderOptions,
487
488
  isr: IsrController,
488
489
  ): Promise<Rendered> {
489
- const document = (loaded: RouteData): Promise<string> =>
490
- documentFrom(entry, data, loaded, options);
490
+ const document = (loaded: RouteData, at: DevRouteData): Promise<string> =>
491
+ documentFrom(entry, at, loaded, options);
491
492
  const outcome = await isrOutcome({ entry, data, ctx, isr, document });
492
493
  if (outcome.kind === 'redirect') return { respond: () => loadRedirect(entry, outcome.to) };
493
494
  return { respond: () => responseOf(outcome.result) };
@@ -80,6 +80,8 @@ export interface RunningServices {
80
80
  * (`runtime-bus.ts`). Optional for the reason `workerConfig` is: a hand-built runtime has none.
81
81
  */
82
82
  readonly busUse?: BusUse;
83
+ /** `x dev` booted without a bus its `sync` role needs: that role waits for it (`role-sync.ts`). */
84
+ readonly busTolerated?: boolean;
83
85
  /**
84
86
  * What the sync role must give `PresenceRegistry`. It travels with the transport because the KV
85
87
  * bucket's age limit was derived from it — a registry given a longer TTL than the bucket honours
@@ -327,6 +329,8 @@ export async function startServices(
327
329
  * `sync` node or a replicator does (`runtime-bus.ts`). Unset is the strictest reading.
328
330
  */
329
331
  roles?: readonly Role[],
332
+ /** `x dev`: a bus the realtime roles need is waited for in the background, never refused. */
333
+ dev = false,
330
334
  ): Promise<RunningServices> {
331
335
  // The app's config is loaded ONCE, here, first, and every section below is read out of that one
332
336
  // validated object — mail's `retainMime` is the first reader, so it is loaded before mail.
@@ -350,7 +354,7 @@ export async function startServices(
350
354
  // signs one probe — a deploy with no keys, or two halves of different pairs, refuses here.
351
355
  const push = await selectWebPush(config, env);
352
356
  const workerConfig = workerConfigOf(config, env);
353
- const bus = selectBus({ env, realtime, roles, override: overrides?.transport });
357
+ const bus = selectBus({ env, realtime, roles, override: overrides?.transport, dev });
354
358
  // `env`, not the ambient one: this function is HANDED the boot's environment and every other
355
359
  // reader here already uses it, so a queue that asked `process.env` would decide the standby from
356
360
  // a different answer than the middleware that routes to it.
@@ -458,6 +462,7 @@ export async function startServices(
458
462
  rateLimitStore,
459
463
  transportDetail: running.detail,
460
464
  busUse: bus.use,
465
+ busTolerated: bus.tolerant,
461
466
  presenceTtlMs: bus.presenceTtlMs,
462
467
  purge,
463
468
  purgeDetail: cdn.detail,
package/src/seo-routes.ts CHANGED
@@ -2,7 +2,14 @@
2
2
  // role — the files the static export writes, for a deploy that serves its pages from a container.
3
3
  // Mounted by `x dev` and `runRole` alike, before the app's pages, for `style-routes.ts`' reason.
4
4
 
5
- import { DEFAULT_ENVIRONMENT, type Environment, tryResolveEnvironment } from '@ultimat3/core';
5
+ import {
6
+ DEFAULT_ENVIRONMENT,
7
+ type Environment,
8
+ finiteCount,
9
+ logger,
10
+ renderThrowable,
11
+ tryResolveEnvironment,
12
+ } from '@ultimat3/core';
6
13
  import type { Route, UltimateRequest } from '@ultimat3/http';
7
14
  import { applyCacheHeaders, jsonResponse, NO_STORE } from '@ultimat3/http';
8
15
  import { SITEMAP_PARTS_DIR } from '@ultimat3/seo';
@@ -17,6 +24,10 @@ export interface SeoRoutesOptions {
17
24
  readonly site?: SiteSettings;
18
25
  /** The app root route files are relative to — `seo.sitemap.lastmod: 'git' | 'mtime'` reads them. */
19
26
  readonly root?: string;
27
+ /** The memo's clock, in ms. Injected so an hour's expiry is provable without waiting one out. */
28
+ readonly now?: () => number;
29
+ /** How long one enumeration may run, in ms. Injected so a hung one is provable in a test. */
30
+ readonly enumerationTimeoutMs?: number;
20
31
  }
21
32
 
22
33
  /**
@@ -43,11 +54,28 @@ const SEO_CACHE = { mode: 'public', maxAgeSeconds: 3600 } as const;
43
54
  */
44
55
  const SEO_MEMO_ORIGINS = 8;
45
56
 
57
+ /** How soon a failed re-enumeration is asked again while the previous answer keeps being served. */
58
+ const SEO_RETRY_MS = 60_000;
59
+
60
+ /** How long one enumeration may run before its origin stops waiting on it. */
61
+ const SEO_ENUMERATION_TIMEOUT_MS = 30_000;
62
+
63
+ /** Consecutive failed refreshes after which `seo.enumeration.kept` is logged as an error. */
64
+ const SEO_FAILURES_BEFORE_ERROR = 3;
65
+
46
66
  interface Memo {
47
- readonly at: number;
48
- readonly answer: Promise<SiteSeo>;
67
+ at: number;
68
+ answer: Promise<SiteSeo>;
69
+ /** A refresh is running behind `answer`. */
70
+ refreshing: boolean;
71
+ /** The first enumeration has answered — only then is there something to refresh BEHIND. */
72
+ settled: boolean;
73
+ failures: number;
49
74
  }
50
75
 
76
+ /** A placeholder for the one statement between building a first memo and starting its enumeration. */
77
+ const NO_ANSWER: SiteSeo = { robots: '', sitemaps: [] };
78
+
51
79
  export function seoRoutes(options: SeoRoutesOptions): readonly Route[] {
52
80
  const environment: Environment =
53
81
  tryResolveEnvironment({ env: options.env }) ?? DEFAULT_ENVIRONMENT;
@@ -59,26 +87,98 @@ export function seoRoutes(options: SeoRoutesOptions): readonly Route[] {
59
87
  ...(options.site === undefined ? {} : { sitemap: options.site.sitemap }),
60
88
  ...(options.root === undefined ? {} : { root: options.root }),
61
89
  });
62
- // Single-flight per origin: concurrent requests share the one computation in progress. A failed
63
- // one is dropped, so the next request asks again rather than serving the failure for an hour.
90
+ // One answer per origin. A FIRST enumeration is single-flight and awaited; failed, it is dropped,
91
+ // so the next request asks again rather than serving the failure for an hour.
92
+ //
93
+ // A LATER one — the hourly refresh — runs BEHIND the answer it would replace, and every request
94
+ // meanwhile is served that answer: `prerender()` reads rows, and a database blip at the hour
95
+ // mark otherwise answered a crawler a 500, while a read that HUNG held every sitemap request on
96
+ // its pending promise. A failed or timed-out refresh keeps the last good sitemap and is asked
97
+ // again a minute later. A `prerender()` that THROWS on a failed read is what makes this hold:
98
+ // one that catches its own error and returns `[]` is a successful enumeration of nothing.
64
99
  const memo = new Map<string, Memo>();
100
+ const clock = options.now ?? Date.now;
101
+ const timeoutMs = finiteCount(
102
+ 'seoRoutes',
103
+ 'enumerationTimeoutMs',
104
+ options.enumerationTimeoutMs ?? SEO_ENUMERATION_TIMEOUT_MS,
105
+ 1,
106
+ );
107
+
108
+ /** `compute`, settled at most once: with the answer, or `undefined` for a failure or a timeout. */
109
+ const enumerate = (
110
+ origin: string,
111
+ settle: (answer: SiteSeo | undefined, why: string) => void,
112
+ ) => {
113
+ let settled = false;
114
+ const once = (answer: SiteSeo | undefined, why: string): void => {
115
+ if (settled) return;
116
+ settled = true;
117
+ clearTimeout(timer);
118
+ settle(answer, why);
119
+ };
120
+ const timer = setTimeout(() => {
121
+ once(undefined, `no answer within ${String(timeoutMs)}ms`);
122
+ }, timeoutMs);
123
+ const computed = compute(origin);
124
+ computed.then(
125
+ (answer) => once(answer, ''),
126
+ (error: unknown) => once(undefined, renderThrowable(error)),
127
+ );
128
+ return computed;
129
+ };
130
+
131
+ const refresh = (origin: string, kept: Memo): void => {
132
+ kept.refreshing = true;
133
+ void enumerate(origin, (answer, why) => {
134
+ if (memo.get(origin) !== kept) return;
135
+ if (answer !== undefined) {
136
+ memo.set(origin, {
137
+ at: clock(),
138
+ answer: Promise.resolve(answer),
139
+ refreshing: false,
140
+ settled: true,
141
+ failures: 0,
142
+ });
143
+ return;
144
+ }
145
+ kept.refreshing = false;
146
+ kept.failures += 1;
147
+ kept.at = clock() - SEO_CACHE.maxAgeSeconds * 1000 + SEO_RETRY_MS;
148
+ // A blip is a warning; a sitemap that has not refreshed for several tries is going stale.
149
+ const fields = { origin, failures: kept.failures, error: why };
150
+ if (kept.failures >= SEO_FAILURES_BEFORE_ERROR) logger.error('seo.enumeration.kept', fields);
151
+ else logger.warn('seo.enumeration.kept', fields);
152
+ }).catch(() => undefined);
153
+ };
154
+
65
155
  const answer = (request: UltimateRequest): Promise<SiteSeo> => {
66
156
  const origin = originOf(options, request);
67
- const now = Date.now();
68
157
  const kept = memo.get(origin);
69
- if (kept !== undefined && now - kept.at < SEO_CACHE.maxAgeSeconds * 1000) return kept.answer;
70
- memo.delete(origin);
158
+ if (kept !== undefined) {
159
+ const expired = clock() - kept.at >= SEO_CACHE.maxAgeSeconds * 1000;
160
+ if (expired && !kept.refreshing && kept.settled) refresh(origin, kept);
161
+ return kept.answer;
162
+ }
71
163
  if (memo.size >= SEO_MEMO_ORIGINS) {
72
164
  const oldest = memo.keys().next().value;
73
165
  if (oldest !== undefined) memo.delete(oldest);
74
166
  }
75
- const computed = compute(origin);
76
- memo.set(origin, { at: now, answer: computed });
77
- // Voided: the request awaits `computed` itself and reports its failure; this branch only forgets it.
78
- void computed.catch(() => {
79
- if (memo.get(origin)?.answer === computed) memo.delete(origin);
167
+ const first: Memo = {
168
+ at: clock(),
169
+ answer: Promise.resolve(NO_ANSWER),
170
+ refreshing: false,
171
+ settled: false,
172
+ failures: 0,
173
+ };
174
+ // The request awaits the enumeration itself and reports its failure; a failed or hung one is
175
+ // only forgotten here, so the next request starts its own rather than joining it.
176
+ first.answer = enumerate(origin, (found) => {
177
+ first.settled = true;
178
+ if (found === undefined && memo.get(origin) === first) memo.delete(origin);
80
179
  });
81
- return computed;
180
+ memo.set(origin, first);
181
+ return first.answer;
82
182
  };
83
183
 
84
184
  return [
@@ -23,6 +23,7 @@ import { checkDocumentStyles, documentSurfaces } from './document-styles';
23
23
  import { checkErrorCodeResolution, checkErrorFixReport } from './error-contract';
24
24
  import { guardFindings } from './guards';
25
25
  import { catalogFindings } from './i18n-registration';
26
+ import { isrQueryAdvice } from './isr-query-advice';
26
27
  import { unregisteredJobFindings } from './job-registration';
27
28
  import { liveRouteFindings } from './live-routes';
28
29
  import { withLoadFindings, withoutLoadFindings } from './load-findings';
@@ -292,9 +293,13 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
292
293
  // such key), so canonical checks are skipped rather than run against an origin this file
293
294
  // invented. `seo-meta.ts` spells out why that is the honest half.
294
295
  const report = validateMeta(scan.records);
296
+ // The scan filled the route table, so the advice costs no second load. Rides here because
297
+ // this is the step about public, shared-cache pages; it never fails it.
298
+ const warnings = await isrQueryAdvice(ctx.root);
295
299
  return {
296
300
  ok: scan.findings.length === 0 && report.ok,
297
301
  findings: [...scan.findings, ...report.issues.map(seoFinding)],
302
+ ...(warnings.length === 0 ? {} : { warnings }),
298
303
  };
299
304
  },
300
305
  },