@ultimat3/cli 27.6.1 → 27.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +1 -1
- package/package.json +28 -28
- package/src/admin-registration.ts +15 -14
- package/src/api-registration.ts +48 -47
- package/src/cmd-generate.ts +65 -49
- package/src/dev-boot.ts +8 -1
- package/src/generate-disk.ts +38 -0
- package/src/generate-files.ts +6 -1
- package/src/generate-follow-ups.ts +64 -0
- package/src/generate-grant-findings.ts +8 -6
- package/src/generate-grants.ts +79 -52
- package/src/generate-kinds.ts +7 -2
- package/src/generate-plural.ts +11 -0
- package/src/generate-sibling.ts +34 -0
- package/src/generated-imports.ts +9 -6
- package/src/handle-registration.ts +71 -29
- package/src/i18n-index.ts +30 -14
- package/src/mcp-host.ts +2 -1
- package/src/runtime-bus.ts +158 -0
- package/src/runtime-cache.ts +141 -9
- package/src/runtime-services.ts +25 -37
- package/src/serve-boot.ts +4 -1
- package/src/serve-types.ts +6 -0
- package/src/serve.ts +8 -1
- package/src/source-list-edit.ts +218 -0
- package/src/templates/entity-module.ts +29 -0
- package/src/templates/entity.ts +30 -14
- package/src/workspace-dep-edit.ts +13 -10
package/src/i18n-index.ts
CHANGED
|
@@ -9,7 +9,10 @@
|
|
|
9
9
|
// why: Bun has no synchronous existence check — `Bun.file(p).exists()` is async, and this decides
|
|
10
10
|
// whether to write at all, before any await the caller could interleave with.
|
|
11
11
|
import { existsSync } from 'node:fs';
|
|
12
|
+
import { stripComments } from '@ultimat3/core';
|
|
12
13
|
import { APP_CATALOGS_PATH } from '@ultimat3/i18n/app-catalogs';
|
|
14
|
+
import type { GenerateDisk } from './generate-disk';
|
|
15
|
+
import { appDisk } from './generate-disk';
|
|
13
16
|
import { containedPath } from './generate-write';
|
|
14
17
|
import type { Finding } from './output';
|
|
15
18
|
import { CATALOG_ROOT, i18nIndex, localeEntry, localeImport } from './templates';
|
|
@@ -30,6 +33,8 @@ export interface IndexSync {
|
|
|
30
33
|
/** False for an app with no i18n package, or an index this writer refused to edit. */
|
|
31
34
|
readonly registered: boolean;
|
|
32
35
|
readonly findings: readonly Finding[];
|
|
36
|
+
/** The index, when this changed its bytes — a run's file list names every file it touched. */
|
|
37
|
+
readonly edited?: readonly string[];
|
|
33
38
|
}
|
|
34
39
|
|
|
35
40
|
const CATALOG_IMPORT = /^import (\w+) from '\.\.\/catalogs\/([^']+)\.json';$/gm;
|
|
@@ -48,31 +53,42 @@ const importedLocales = (source: string): readonly string[] =>
|
|
|
48
53
|
* stay. A shape with no `locales: { … }` object to add to is refused with the edit named, never
|
|
49
54
|
* overwritten: it was, on every `x g`, with a template hard-coding `default: 'en'`.
|
|
50
55
|
*/
|
|
51
|
-
export async function syncI18nIndex(
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
+
export async function syncI18nIndex(
|
|
57
|
+
root: string,
|
|
58
|
+
disk: GenerateDisk = appDisk(root),
|
|
59
|
+
/** Locales whose catalog a dry run PLANS: on the real disk only once the run has written. */
|
|
60
|
+
planned: readonly string[] = [],
|
|
61
|
+
): Promise<IndexSync> {
|
|
62
|
+
const current = await disk.read(APP_CATALOGS_PATH);
|
|
63
|
+
if (current === undefined) return { registered: false, findings: [] };
|
|
64
|
+
const onDisk = [...new Set([...(await catalogLocales(root)), ...planned])].sort();
|
|
56
65
|
const imported = importedLocales(current);
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
66
|
+
const write = async (next: string): Promise<IndexSync> => {
|
|
67
|
+
if (next === current) return { registered: true, findings: [] };
|
|
68
|
+
await disk.write(APP_CATALOGS_PATH, next);
|
|
69
|
+
return { registered: true, findings: [], edited: [APP_CATALOGS_PATH] };
|
|
70
|
+
};
|
|
71
|
+
if (current === i18nIndex(imported)) return write(i18nIndex(onDisk));
|
|
61
72
|
const missing = onDisk.filter((locale) => !imported.includes(locale));
|
|
62
73
|
if (missing.length === 0) return { registered: true, findings: [] };
|
|
63
74
|
const edited = withLocales(current, missing);
|
|
64
75
|
if (edited === undefined) return { registered: false, findings: [refusal(missing)] };
|
|
65
|
-
|
|
66
|
-
return { registered: true, findings: [] };
|
|
76
|
+
return write(edited);
|
|
67
77
|
}
|
|
68
78
|
|
|
69
79
|
/** Each import after the last catalog import (or at the top), each entry at the object's end. */
|
|
70
80
|
function withLocales(source: string, locales: readonly string[]): string | undefined {
|
|
71
|
-
|
|
81
|
+
// Found on the comment-stripped text (same offsets): a `locales: { … }` in a comment above the
|
|
82
|
+
// call is prose, and it was the one this edited.
|
|
83
|
+
const object = LOCALES_OBJECT.exec(stripComments(source));
|
|
72
84
|
if (object === null) return undefined;
|
|
73
|
-
const
|
|
85
|
+
const [whole, head = '', inner = ''] = object;
|
|
86
|
+
// A comment INSIDE the object cannot be carried through a one-line rewrite: refused, with the
|
|
87
|
+
// edit named, rather than dropped.
|
|
88
|
+
if (source.slice(object.index, object.index + whole.length) !== whole) return undefined;
|
|
89
|
+
const body = inner.trim().replace(/,$/, '');
|
|
74
90
|
const entries = [body, ...locales.map(localeEntry)].filter((part) => part !== '').join(', ');
|
|
75
|
-
const withEntries = source.
|
|
91
|
+
const withEntries = `${source.slice(0, object.index)}${head} ${entries} }${source.slice(object.index + whole.length)}`;
|
|
76
92
|
const imports = locales.map(localeImport).join('\n');
|
|
77
93
|
const lastImport = [...withEntries.matchAll(CATALOG_IMPORT)].at(-1);
|
|
78
94
|
if (lastImport === undefined) return `${imports}\n${withEntries}`;
|
package/src/mcp-host.ts
CHANGED
|
@@ -105,7 +105,8 @@ export function lazyServices(input: DevHostInput): LazyServices {
|
|
|
105
105
|
// one connection wedged every later tool call in the session with that first error, and the only
|
|
106
106
|
// way out was restarting the host — `startServices` unwinds everything it started before it
|
|
107
107
|
// rejects, so there is nothing left over for a second attempt to collide with.
|
|
108
|
-
|
|
108
|
+
// `[]`: this host starts no role, so it only ever publishes to the bus and never waits for it.
|
|
109
|
+
const boot = retryMemo(() => startServices(services, input.env, undefined, 'apply', []));
|
|
109
110
|
let closed = false;
|
|
110
111
|
return {
|
|
111
112
|
services,
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// Single responsibility: the fanout bus as ONE boot holds it — what the roles it starts do with
|
|
2
|
+
// the bus, the dial that calls for, the readiness check, and the label the boot line prints.
|
|
3
|
+
//
|
|
4
|
+
// The rule is the role's, not the transport's. A `sync` node serves sockets from the bus and a
|
|
5
|
+
// replicator feeds them: neither can work without it, so each awaits the dial and refuses its
|
|
6
|
+
// boot, coded, inside a bounded wait. `web`, `worker` and `scheduler` only PUBLISH — channel
|
|
7
|
+
// events and cache invalidations, both of which mean "re-read" — so for them the bus is dialled in
|
|
8
|
+
// the background and a dead one is `degraded`: the pod still boots, serves and runs jobs. Until
|
|
9
|
+
// 2026-10 every role awaited the dial and the JetStream bucket, so a web pod that restarted while
|
|
10
|
+
// NATS was down took the app's pages, webhooks and send jobs down with a feature none of them use.
|
|
11
|
+
|
|
12
|
+
import type { RealtimeConfig, Role } from '@ultimat3/core';
|
|
13
|
+
import { logger, registerReadinessCheck } from '@ultimat3/core';
|
|
14
|
+
import type { BusUse, SelectTransportOptions, Transport } from '@ultimat3/realtime/server';
|
|
15
|
+
import { selectTransport, setChannelTransport } from '@ultimat3/realtime/server';
|
|
16
|
+
import type { Env } from './runtime-bindings';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* What a process running `roles` does with the bus. `undefined` — a boot that names no roles — is
|
|
20
|
+
* held to the strictest reading, the one every boot had. `realtime.enabled: false` starts neither
|
|
21
|
+
* realtime role (`role-realtime.ts`), so nothing in such a process waits for the bus.
|
|
22
|
+
*/
|
|
23
|
+
export function busUseFor(
|
|
24
|
+
roles: readonly Role[] | undefined,
|
|
25
|
+
realtime: Pick<RealtimeConfig, 'enabled'>,
|
|
26
|
+
): BusUse {
|
|
27
|
+
if (roles === undefined) return 'sockets';
|
|
28
|
+
if (!realtime.enabled) return 'publish';
|
|
29
|
+
if (roles.includes('sync')) return 'sockets';
|
|
30
|
+
return roles.includes('replicator') ? 'feed' : 'publish';
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** A transport that says whether it is connected. NATS does; the in-process bus has nothing to be. */
|
|
34
|
+
interface ConnectableTransport {
|
|
35
|
+
readonly connected: boolean;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const saysConnected = (transport: Transport): transport is Transport & ConnectableTransport =>
|
|
39
|
+
typeof (transport as Partial<ConnectableTransport>).connected === 'boolean';
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The boot line's `bus=` value: `nats(up)`, `nats(connecting)`, or the bare name of a bus with no
|
|
43
|
+
* connection to lose. A fixed vocabulary a script may parse, never a catalog lookup.
|
|
44
|
+
*/
|
|
45
|
+
export function busLabel(transport: Transport): string {
|
|
46
|
+
if (!saysConnected(transport)) return transport.name;
|
|
47
|
+
return `${transport.name}(${transport.connected ? 'up' : 'connecting'})`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface RunningBus {
|
|
51
|
+
readonly transport: Transport;
|
|
52
|
+
/** The env key that selected the bus — never the url behind it — or `runtime override`. */
|
|
53
|
+
readonly detail: string;
|
|
54
|
+
/** Everything `start()` acquired, newest first; the first failure is rethrown, every step runs. */
|
|
55
|
+
stop(): Promise<void>;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface SelectedBus {
|
|
59
|
+
readonly use: BusUse;
|
|
60
|
+
/** What the sync role must give `PresenceRegistry` (`RunningServices.presenceTtlMs`). */
|
|
61
|
+
readonly presenceTtlMs: number;
|
|
62
|
+
/**
|
|
63
|
+
* Dial as `use` calls for, install the transport as the process's channel bus, and register its
|
|
64
|
+
* readiness check. Rejects — with nothing left registered or open — only for a use that cannot
|
|
65
|
+
* work without the bus.
|
|
66
|
+
*/
|
|
67
|
+
start(): Promise<RunningBus>;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface SelectBusInput {
|
|
71
|
+
readonly env: Env;
|
|
72
|
+
readonly realtime: RealtimeConfig;
|
|
73
|
+
/** The roles this process will start; `undefined` when the caller does not know. */
|
|
74
|
+
readonly roles: readonly Role[] | undefined;
|
|
75
|
+
/** `RuntimeOverrides.transport`: already connected, and its owner's to close. */
|
|
76
|
+
readonly override?: Transport | undefined;
|
|
77
|
+
/** Injected by a test: a fake dial, a short wait. */
|
|
78
|
+
readonly select?: SelectTransportOptions | undefined;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Selection is pure — it parses env and constructs, it dials nothing — so a config and an
|
|
83
|
+
* environment that disagree refuse here, before any service starts (`selectTransport`).
|
|
84
|
+
*/
|
|
85
|
+
export function selectBus(input: SelectBusInput): SelectedBus {
|
|
86
|
+
const use = busUseFor(input.roles, input.realtime);
|
|
87
|
+
const selection = selectTransport(input.env, input.realtime, { ...input.select, use });
|
|
88
|
+
return {
|
|
89
|
+
use,
|
|
90
|
+
presenceTtlMs: selection.presenceTtlMs,
|
|
91
|
+
async start(): Promise<RunningBus> {
|
|
92
|
+
const started: (() => void | Promise<void>)[] = [];
|
|
93
|
+
const stop = async (): Promise<void> => {
|
|
94
|
+
let failure: { readonly error: unknown } | undefined;
|
|
95
|
+
for (const step of started.splice(0).reverse()) {
|
|
96
|
+
try {
|
|
97
|
+
await step();
|
|
98
|
+
} catch (error) {
|
|
99
|
+
failure ??= { error };
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (failure !== undefined) throw failure.error;
|
|
103
|
+
};
|
|
104
|
+
try {
|
|
105
|
+
// A supplied transport is already connected and is NOT closed here: whoever built it owns
|
|
106
|
+
// its socket, which is why the override skips both halves rather than only the dial.
|
|
107
|
+
let transport = input.override;
|
|
108
|
+
if (transport === undefined) {
|
|
109
|
+
// Before the dial, not after: a dial refused for being late is still in flight inside
|
|
110
|
+
// the client library, and closing the transport is what releases whatever it lands as.
|
|
111
|
+
started.push(() => selection.transport.close());
|
|
112
|
+
await selection.connect();
|
|
113
|
+
transport = selection.transport;
|
|
114
|
+
}
|
|
115
|
+
// The bus an app's own code publishes a channel event on (`publishChannelEvent`), in EVERY
|
|
116
|
+
// role: the hub is the `sync` role's, so a job or an action reached none, and an
|
|
117
|
+
// events-only channel — the one kind that needs no replicator — could be fed by nothing
|
|
118
|
+
// but a socket.
|
|
119
|
+
started.push(setChannelTransport(transport));
|
|
120
|
+
// Only a transport that can be disconnected gets a check. `NatsTransport.connected` is a
|
|
121
|
+
// synchronous getter over the client's own state, so no probe is needed; the in-process
|
|
122
|
+
// bus has nothing to lose a connection to, and a check that can only answer `true` is a
|
|
123
|
+
// number in `registered` that means nothing.
|
|
124
|
+
if (saysConnected(transport)) {
|
|
125
|
+
const connectable = transport;
|
|
126
|
+
started.push(
|
|
127
|
+
registerReadinessCheck(
|
|
128
|
+
'transport',
|
|
129
|
+
() => connectable.connected,
|
|
130
|
+
// A publisher serves without the bus, so losing it must not pull the pod from the
|
|
131
|
+
// ingress — it is reported, and `/readyz?deep=1` is where a monitor reads it.
|
|
132
|
+
{ onFailure: use === 'publish' ? 'degraded' : 'failing' },
|
|
133
|
+
),
|
|
134
|
+
);
|
|
135
|
+
// Each time the bus comes (back) up, on one line: the boot line may have said
|
|
136
|
+
// `connecting`, and this is the other half of it.
|
|
137
|
+
started.push(
|
|
138
|
+
connectable.onReconnect(() => {
|
|
139
|
+
logger.info('ultimate bus', { bus: busLabel(connectable), use });
|
|
140
|
+
}),
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
return {
|
|
144
|
+
transport,
|
|
145
|
+
// The env key that selected the bus — or the honest answer that no env key did, because
|
|
146
|
+
// a boot line reading `NATS_URL` over a transport the host handed in is a lie a script
|
|
147
|
+
// parses.
|
|
148
|
+
detail: input.override === undefined ? selection.detail : 'runtime override',
|
|
149
|
+
stop,
|
|
150
|
+
};
|
|
151
|
+
} catch (error) {
|
|
152
|
+
// The refusal is the failure worth reporting, not a release on the way out.
|
|
153
|
+
await stop().catch(() => undefined);
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
};
|
|
158
|
+
}
|
package/src/runtime-cache.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type { CacheTier, PurgeDriver } from '@ultimat3/cache';
|
|
|
8
8
|
import {
|
|
9
9
|
CacheDriverUnavailableError,
|
|
10
10
|
cdnTier,
|
|
11
|
+
flushProcessTiers,
|
|
11
12
|
isNoopPurgeDriver,
|
|
12
13
|
lruTier,
|
|
13
14
|
memoTier,
|
|
@@ -29,6 +30,19 @@ import type { Env } from './runtime-bindings';
|
|
|
29
30
|
*/
|
|
30
31
|
export const CACHE_INVALIDATE_SUBJECT = 'x.cache.invalidate';
|
|
31
32
|
|
|
33
|
+
/**
|
|
34
|
+
* "Drop everything you hold in process", on the same subject. An object, where a bust is an array
|
|
35
|
+
* of wire tags: a replica from before this marker existed parses it, finds no array and ignores
|
|
36
|
+
* it — its entries wait for their TTL, which is what a lost bust always cost it.
|
|
37
|
+
*/
|
|
38
|
+
export const CACHE_FLUSH_ALL = '{"flush":"all"}';
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* How many refused wire tags a process keeps for the bus's return. A bound, not a queue: past it
|
|
42
|
+
* the set is dropped and ONE flush-all is owed instead, so memory does not grow with the outage.
|
|
43
|
+
*/
|
|
44
|
+
export const MAX_DEFERRED_TAGS = 1_024;
|
|
45
|
+
|
|
32
46
|
export interface CacheTiersOptions {
|
|
33
47
|
readonly env: Env;
|
|
34
48
|
/** Already resolved by the boot — a `cdn` rung is built against a real edge or not at all. */
|
|
@@ -152,8 +166,87 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
|
|
|
152
166
|
for (const tier of built) registerTier(tier);
|
|
153
167
|
warnUnnamed(options);
|
|
154
168
|
|
|
169
|
+
const send = (payload: string): Promise<void> =>
|
|
170
|
+
options.transport.publish(CACHE_INVALIDATE_SUBJECT, payload);
|
|
171
|
+
// What the bus refused, kept for its return. A publish with no live connection is refused, not
|
|
172
|
+
// buffered (`NatsTransport`), so without this a bust made during a two-second NATS restart
|
|
173
|
+
// reached no peer at all. De-duplicated — a tag is a fact, not an event — and bounded.
|
|
174
|
+
const deferred = new Set<string>();
|
|
175
|
+
let flushOwed = false;
|
|
176
|
+
const defer = (wireTags: readonly string[]): void => {
|
|
177
|
+
if (flushOwed) return;
|
|
178
|
+
for (const wire of wireTags) deferred.add(wire);
|
|
179
|
+
if (deferred.size <= MAX_DEFERRED_TAGS) return;
|
|
180
|
+
deferred.clear();
|
|
181
|
+
flushOwed = true;
|
|
182
|
+
};
|
|
183
|
+
/**
|
|
184
|
+
* One pass over what is owed. It is TAKEN before the publish and put back if the bus refuses, so
|
|
185
|
+
* a bust refused while this publish is in flight is owed afresh rather than cleared by a success
|
|
186
|
+
* that never carried it — and a flush-all sent before that bust cannot stand in for it.
|
|
187
|
+
*/
|
|
188
|
+
const settleOnce = async (): Promise<void> => {
|
|
189
|
+
if (flushOwed) {
|
|
190
|
+
flushOwed = false;
|
|
191
|
+
try {
|
|
192
|
+
await send(CACHE_FLUSH_ALL);
|
|
193
|
+
} catch (error) {
|
|
194
|
+
deferred.clear();
|
|
195
|
+
flushOwed = true;
|
|
196
|
+
throw error;
|
|
197
|
+
}
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
if (deferred.size === 0) return;
|
|
201
|
+
const batch = [...deferred];
|
|
202
|
+
deferred.clear();
|
|
203
|
+
try {
|
|
204
|
+
await send(JSON.stringify(batch));
|
|
205
|
+
} catch (error) {
|
|
206
|
+
defer(batch);
|
|
207
|
+
throw error;
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
// ONE settle at a time. It is asked for by every reconnect and after every accepted publish,
|
|
211
|
+
// and two that overlapped both published the same batch (and the same flush-all). A call that
|
|
212
|
+
// arrives mid-run asks for one more pass instead, which finds whatever the run did not carry.
|
|
213
|
+
let settling: Promise<void> | undefined;
|
|
214
|
+
let again = false;
|
|
215
|
+
const settle = (): Promise<void> => {
|
|
216
|
+
if (settling !== undefined) {
|
|
217
|
+
again = true;
|
|
218
|
+
return settling;
|
|
219
|
+
}
|
|
220
|
+
settling = (async (): Promise<void> => {
|
|
221
|
+
try {
|
|
222
|
+
do {
|
|
223
|
+
again = false;
|
|
224
|
+
await settleOnce();
|
|
225
|
+
} while (again);
|
|
226
|
+
} finally {
|
|
227
|
+
again = false;
|
|
228
|
+
settling = undefined;
|
|
229
|
+
}
|
|
230
|
+
})();
|
|
231
|
+
return settling;
|
|
232
|
+
};
|
|
233
|
+
const settleQuietly = (): void => {
|
|
234
|
+
void settle().catch((error: unknown) => {
|
|
235
|
+
logger.warn('cache.broadcast.deferred-failed', { error: broadcastErrorText(error) });
|
|
236
|
+
});
|
|
237
|
+
};
|
|
238
|
+
|
|
155
239
|
registerInvalidationBroadcast(async (wireTags) => {
|
|
156
|
-
|
|
240
|
+
try {
|
|
241
|
+
await send(JSON.stringify(wireTags));
|
|
242
|
+
} catch (error) {
|
|
243
|
+
defer(wireTags);
|
|
244
|
+
// Still thrown: the report must say the peers have not been told YET.
|
|
245
|
+
throw error;
|
|
246
|
+
}
|
|
247
|
+
// The bus took this one, so it takes what an earlier publish could not — the case no
|
|
248
|
+
// reconnect announces, a refusal with the connection up.
|
|
249
|
+
if (flushOwed || deferred.size > 0) settleQuietly();
|
|
157
250
|
});
|
|
158
251
|
// Not awaited HERE: the boot must not block on a subscribe. The PROMISE is held rather than a
|
|
159
252
|
// handle assigned inside a `.then`, because the release ran first whenever `stop()` beat the round
|
|
@@ -162,28 +255,56 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
|
|
|
162
255
|
// logged once and never asked again, so the process missed every peer invalidation for its life.
|
|
163
256
|
const released = Promise.withResolvers<undefined>();
|
|
164
257
|
let stopped = false;
|
|
258
|
+
let subscribed = false;
|
|
259
|
+
/** Cuts the retry's wait short: the bus coming up is a better signal than a backoff. */
|
|
260
|
+
let wake: () => void = () => undefined;
|
|
165
261
|
const retryMs = options.subscribeRetryMs ?? defaultSubscribeRetryMs;
|
|
262
|
+
// This process heard no peer for a window nobody measured — since boot, or since the connection
|
|
263
|
+
// dropped — so nothing it holds in its own heap is known fresh. One cold cache, never a stale one.
|
|
264
|
+
const flushDeaf = (source: string): void => {
|
|
265
|
+
void flushProcessTiers(source);
|
|
266
|
+
};
|
|
166
267
|
const subscribing = (async (): Promise<TransportSubscription | undefined> => {
|
|
167
268
|
for (let attempt = 1; !stopped; attempt += 1) {
|
|
168
269
|
try {
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
270
|
+
const subscription = await options.transport.subscribe(
|
|
271
|
+
CACHE_INVALIDATE_SUBJECT,
|
|
272
|
+
(payload: string) => {
|
|
273
|
+
void applyBroadcast(payload);
|
|
274
|
+
},
|
|
275
|
+
);
|
|
276
|
+
subscribed = true;
|
|
277
|
+
if (attempt > 1) flushDeaf('cache.bus-subscribed');
|
|
278
|
+
return subscription;
|
|
172
279
|
} catch (error) {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
280
|
+
// Attempts 1, 2, 4, 8, …: the dial's own thinning (`NatsTransport`), so an outage is not
|
|
281
|
+
// one line per retry per pod for as long as it lasts.
|
|
282
|
+
if ((attempt & (attempt - 1)) === 0) {
|
|
283
|
+
logger.warn('cache.broadcast.subscribe-failed', {
|
|
284
|
+
error: broadcastErrorText(error),
|
|
285
|
+
attempt,
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
const woken = Promise.withResolvers<undefined>();
|
|
289
|
+
wake = (): void => woken.resolve(undefined);
|
|
290
|
+
await Promise.race([Bun.sleep(retryMs(attempt)), released.promise, woken.promise]);
|
|
178
291
|
}
|
|
179
292
|
}
|
|
180
293
|
return undefined;
|
|
181
294
|
})();
|
|
295
|
+
// The bus is (back) up. The subscription came back with the connection, so from here this
|
|
296
|
+
// process hears again: drop what it held while deaf, and say what it could not say.
|
|
297
|
+
const offReconnect = options.transport.onReconnect(() => {
|
|
298
|
+
wake();
|
|
299
|
+
if (subscribed) flushDeaf('cache.bus-reconnect');
|
|
300
|
+
settleQuietly();
|
|
301
|
+
});
|
|
182
302
|
|
|
183
303
|
// `resetTiers()` drops the registry AND the broadcast in one call: this boot is the only thing
|
|
184
304
|
// that registers either, and a tier left behind would purge for a process that has stopped.
|
|
185
305
|
return async () => {
|
|
186
306
|
stopped = true;
|
|
307
|
+
offReconnect();
|
|
187
308
|
released.resolve(undefined);
|
|
188
309
|
(await subscribing)?.unsubscribe();
|
|
189
310
|
resetTiers();
|
|
@@ -200,6 +321,13 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
|
|
|
200
321
|
*/
|
|
201
322
|
export const broadcastErrorText = (error: unknown): string => renderThrowable(error);
|
|
202
323
|
|
|
324
|
+
/** `CACHE_FLUSH_ALL`, parsed: a peer had more refused busts than it could keep. */
|
|
325
|
+
const isFlushAll = (parsed: unknown): boolean =>
|
|
326
|
+
typeof parsed === 'object' &&
|
|
327
|
+
parsed !== null &&
|
|
328
|
+
!Array.isArray(parsed) &&
|
|
329
|
+
Reflect.get(parsed, 'flush') === 'all';
|
|
330
|
+
|
|
203
331
|
/**
|
|
204
332
|
* A peer's wire tags, applied here. Never throws: a malformed frame or an undeclared tag must not
|
|
205
333
|
* kill the subscriber loop, because that would silently end cross-instance invalidation for the
|
|
@@ -208,6 +336,10 @@ export const broadcastErrorText = (error: unknown): string => renderThrowable(er
|
|
|
208
336
|
async function applyBroadcast(payload: string): Promise<void> {
|
|
209
337
|
try {
|
|
210
338
|
const parsed: unknown = JSON.parse(payload);
|
|
339
|
+
if (isFlushAll(parsed)) {
|
|
340
|
+
await flushProcessTiers('cache.broadcast');
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
211
343
|
if (!Array.isArray(parsed)) return;
|
|
212
344
|
const wire = parsed.filter((value): value is string => typeof value === 'string');
|
|
213
345
|
if (wire.length > 0) await receiveInvalidationBroadcast(wire);
|
package/src/runtime-services.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { isNoopPurgeDriver, selectPurgeDriver } from '@ultimat3/cache';
|
|
|
10
10
|
import {
|
|
11
11
|
isLocal,
|
|
12
12
|
type RealtimeConfig,
|
|
13
|
+
type Role,
|
|
13
14
|
registerReadinessCheck,
|
|
14
15
|
renderThrowable,
|
|
15
16
|
resolveEnvironment,
|
|
@@ -26,14 +27,14 @@ import {
|
|
|
26
27
|
resetMailDriver,
|
|
27
28
|
setMailDriver,
|
|
28
29
|
} from '@ultimat3/mail';
|
|
29
|
-
import type {
|
|
30
|
-
import { selectTransport, setChannelTransport } from '@ultimat3/realtime/server';
|
|
30
|
+
import type { BusUse, Transport } from '@ultimat3/realtime/server';
|
|
31
31
|
import type { Storage } from '@ultimat3/storage';
|
|
32
32
|
import { defineStorage, localDriver, s3Driver, usesDevStorageSecret } from '@ultimat3/storage';
|
|
33
33
|
import { loadAppConfig } from './app-config-load';
|
|
34
34
|
import { LocalDiskUnsafeError, StorageUnwritableError } from './errors';
|
|
35
35
|
import { msg } from './messages';
|
|
36
36
|
import type { DevServices, Env } from './runtime-bindings';
|
|
37
|
+
import { selectBus } from './runtime-bus';
|
|
37
38
|
import { cacheTiersOf, startCacheTiers } from './runtime-cache';
|
|
38
39
|
import { type WorkerConfig, workerConfigOf } from './runtime-jobs';
|
|
39
40
|
import { selectAppMailDriver } from './runtime-mail';
|
|
@@ -74,6 +75,11 @@ export interface RunningServices {
|
|
|
74
75
|
readonly mailDetail: string;
|
|
75
76
|
/** Same rule as `mailDetail`: the env key that selected the bus, never the url behind it. */
|
|
76
77
|
readonly transportDetail: string;
|
|
78
|
+
/**
|
|
79
|
+
* What this process does with the bus, which decided whether the boot waited for it
|
|
80
|
+
* (`runtime-bus.ts`). Optional for the reason `workerConfig` is: a hand-built runtime has none.
|
|
81
|
+
*/
|
|
82
|
+
readonly busUse?: BusUse;
|
|
77
83
|
/**
|
|
78
84
|
* What the sync role must give `PresenceRegistry`. It travels with the transport because the KV
|
|
79
85
|
* bucket's age limit was derived from it — a registry given a longer TTL than the bucket honours
|
|
@@ -278,14 +284,6 @@ function probedReadinessCheck(name: string, probe: () => Promise<unknown>): () =
|
|
|
278
284
|
});
|
|
279
285
|
}
|
|
280
286
|
|
|
281
|
-
/** A transport that says whether it is connected. NATS does; the in-process bus has nothing to be. */
|
|
282
|
-
interface ConnectableTransport {
|
|
283
|
-
readonly connected: boolean;
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
const saysConnected = (transport: Transport): transport is Transport & ConnectableTransport =>
|
|
287
|
-
typeof (transport as Partial<ConnectableTransport>).connected === 'boolean';
|
|
288
|
-
|
|
289
287
|
/**
|
|
290
288
|
* Release what has already started, newest first, and return every failure instead of throwing on
|
|
291
289
|
* the first: a step that rejects must not skip the ones after it, or one transport that will not
|
|
@@ -324,6 +322,11 @@ export async function startServices(
|
|
|
324
322
|
overrides?: RuntimeOverrides,
|
|
325
323
|
/** `'verify'` for a serving role on an external database — `runtime-queue.ts`'s `SchemaMode`. */
|
|
326
324
|
schema: SchemaMode = 'apply',
|
|
325
|
+
/**
|
|
326
|
+
* The roles this process will start, which decide whether the boot waits for the bus: only a
|
|
327
|
+
* `sync` node or a replicator does (`runtime-bus.ts`). Unset is the strictest reading.
|
|
328
|
+
*/
|
|
329
|
+
roles?: readonly Role[],
|
|
327
330
|
): Promise<RunningServices> {
|
|
328
331
|
// The app's config is loaded ONCE, here, first, and every section below is read out of that one
|
|
329
332
|
// validated object — mail's `retainMime` is the first reader, so it is loaded before mail.
|
|
@@ -341,13 +344,13 @@ export async function startServices(
|
|
|
341
344
|
// service starts, rather than quietly keeping the in-process bus. Which transport, which KV
|
|
342
345
|
// bucket and which presence TTL is `@ultimat3/realtime`'s decision, and it is the same call a
|
|
343
346
|
// `ROLE=sync` container makes, so this process cannot resolve the bus differently from the
|
|
344
|
-
// container it stands in for.
|
|
347
|
+
// container it stands in for. What the bus is USED for is the roles' (`runtime-bus.ts`).
|
|
345
348
|
const realtime = realtimeConfigOf(config);
|
|
346
349
|
// Fourth: `pwa.push` and its VAPID pair (`runtime-push.ts`). Resolving the pair reads env and
|
|
347
350
|
// signs one probe — a deploy with no keys, or two halves of different pairs, refuses here.
|
|
348
351
|
const push = await selectWebPush(config, env);
|
|
349
352
|
const workerConfig = workerConfigOf(config, env);
|
|
350
|
-
const bus
|
|
353
|
+
const bus = selectBus({ env, realtime, roles, override: overrides?.transport });
|
|
351
354
|
// `env`, not the ambient one: this function is HANDED the boot's environment and every other
|
|
352
355
|
// reader here already uses it, so a queue that asked `process.env` would decide the standby from
|
|
353
356
|
// a different answer than the middleware that routes to it.
|
|
@@ -405,28 +408,14 @@ export async function startServices(
|
|
|
405
408
|
// before `sync`, `worker` and `scheduler` start. The chart's `readinessProbe` and the container
|
|
406
409
|
// healthcheck both route on it, so a replica whose pool was gone kept taking traffic.
|
|
407
410
|
started.push(probedReadinessCheck('database', () => db.ping()));
|
|
408
|
-
// Dialled here rather than at selection:
|
|
409
|
-
//
|
|
410
|
-
//
|
|
411
|
-
//
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
transport = bus.transport;
|
|
417
|
-
}
|
|
418
|
-
// The bus an app's own code publishes a channel event on (`publishChannelEvent`), in EVERY
|
|
419
|
-
// role: the hub is the `sync` role's, so a job or an action reached none, and an events-only
|
|
420
|
-
// channel — the one kind that needs no replicator — could be fed by nothing but a socket.
|
|
421
|
-
started.push(setChannelTransport(transport));
|
|
422
|
-
// Only a transport that can be disconnected gets a check. `NatsTransport.connected` is a
|
|
423
|
-
// synchronous getter over the client's own state, so no probe is needed; the in-process bus
|
|
424
|
-
// has nothing to lose a connection to, and a check that can only answer `true` is a number in
|
|
425
|
-
// `registered` that means nothing.
|
|
426
|
-
if (saysConnected(transport)) {
|
|
427
|
-
const connectable = transport;
|
|
428
|
-
started.push(registerReadinessCheck('transport', () => connectable.connected));
|
|
429
|
-
}
|
|
411
|
+
// Dialled here rather than at selection: the socket is a resource the unwind below has to
|
|
412
|
+
// release. AWAITED only by a process that cannot work without the bus — a `sync` node, a
|
|
413
|
+
// replicator — where an unreachable one must fail the boot, not the first change nobody
|
|
414
|
+
// receives. Every other role dials in the background and boots regardless; the channel bus and
|
|
415
|
+
// the `transport` readiness check are installed either way (`runtime-bus.ts`).
|
|
416
|
+
const running = await bus.start();
|
|
417
|
+
started.push(() => running.stop());
|
|
418
|
+
const transport = running.transport;
|
|
430
419
|
// Before `loadApp` for the auth limiters' reason: `pushSubscribe()` is declared when the app's
|
|
431
420
|
// modules import, and reads this runtime per call.
|
|
432
421
|
if (push !== undefined) started.push(await installAppWebPush(push, executor));
|
|
@@ -467,9 +456,8 @@ export async function startServices(
|
|
|
467
456
|
// table: three readers of one executor, resolved once. `startWeb` is what an override
|
|
468
457
|
// replaces it at.
|
|
469
458
|
rateLimitStore,
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
transportDetail: overrides?.transport === undefined ? bus.detail : 'runtime override',
|
|
459
|
+
transportDetail: running.detail,
|
|
460
|
+
busUse: bus.use,
|
|
473
461
|
presenceTtlMs: bus.presenceTtlMs,
|
|
474
462
|
purge,
|
|
475
463
|
purgeDetail: cdn.detail,
|
package/src/serve-boot.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { startOtlpExport } from './otlp-export';
|
|
|
12
12
|
import { loadAppForRole, roleLoadFor } from './role-load';
|
|
13
13
|
import { startRoles } from './role-start';
|
|
14
14
|
import { resolveServices } from './runtime-bindings';
|
|
15
|
+
import { busLabel } from './runtime-bus';
|
|
15
16
|
import { warnIfIdempotencyProcessScoped } from './runtime-idempotency-scope';
|
|
16
17
|
import { warnUnsealedMfaSecrets } from './runtime-mfa-warning';
|
|
17
18
|
import { replicaOverrides } from './runtime-replica';
|
|
@@ -89,7 +90,8 @@ export async function bootServing(boot: {
|
|
|
89
90
|
// tables first, and this boot only verifies the stamp it left (`framework-schema-apply.ts`).
|
|
90
91
|
// The embedded database has no migrate step to have run, so it applies its own.
|
|
91
92
|
const schema = services.db.mode === 'external' ? 'verify' : 'apply';
|
|
92
|
-
|
|
93
|
+
// The role decides whether this boot waits for the bus: only `sync` and the replicator do.
|
|
94
|
+
const runtime = await startServices(services, options.env, options.runtime, schema, [boot.role]);
|
|
93
95
|
boot.acquired.push(() => runtime.stop());
|
|
94
96
|
return bootRoles({ ...boot, runtime });
|
|
95
97
|
}
|
|
@@ -190,6 +192,7 @@ async function bootRoles(boot: {
|
|
|
190
192
|
role,
|
|
191
193
|
url: running.url,
|
|
192
194
|
buildId,
|
|
195
|
+
bus: busLabel(runtime.transport),
|
|
193
196
|
running,
|
|
194
197
|
runtime,
|
|
195
198
|
async stop() {
|
package/src/serve-types.ts
CHANGED
|
@@ -39,6 +39,12 @@ export interface ServedApp {
|
|
|
39
39
|
/** `http://…` for the web role; null for the roles that open no HTTP socket. */
|
|
40
40
|
readonly url: string | null;
|
|
41
41
|
readonly buildId: string;
|
|
42
|
+
/**
|
|
43
|
+
* The bus as the boot left it: `nats(up)`, `nats(connecting)` or `in-process` (`busLabel`). Only
|
|
44
|
+
* a role that publishes and serves no socket can boot `connecting`. Optional: a hand-built
|
|
45
|
+
* `ServedApp` — a test double — has no bus to report, and the boot line then carries none.
|
|
46
|
+
*/
|
|
47
|
+
readonly bus?: string;
|
|
42
48
|
readonly running: RunningRoles;
|
|
43
49
|
readonly runtime: RunningServices;
|
|
44
50
|
stop(): Promise<void>;
|
package/src/serve.ts
CHANGED
|
@@ -203,7 +203,14 @@ export async function runRole(input: ServeOptions): Promise<StartedApp> {
|
|
|
203
203
|
}
|
|
204
204
|
const { app, hold } = booted;
|
|
205
205
|
retire?.adopt(app.running.worker);
|
|
206
|
-
logger.info('ultimate started', {
|
|
206
|
+
logger.info('ultimate started', {
|
|
207
|
+
role: app.role,
|
|
208
|
+
url: app.url,
|
|
209
|
+
buildId: app.buildId,
|
|
210
|
+
// `nats(connecting)` on a role that only publishes is a served pod with the bus down;
|
|
211
|
+
// `ultimate bus` is logged with `nats(up)` when the dial lands.
|
|
212
|
+
bus: app.bus,
|
|
213
|
+
});
|
|
207
214
|
await hold();
|
|
208
215
|
return app;
|
|
209
216
|
}
|