@ultimat3/cli 23.0.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +15 -7
- package/README.md +3 -3
- package/package.json +28 -28
- package/src/affected.ts +27 -35
- package/src/api-registration.ts +89 -0
- package/src/app-artifacts.ts +1 -1
- package/src/app-auth.ts +4 -27
- package/src/app-config-load.ts +51 -0
- package/src/app-env.ts +5 -11
- package/src/app-manifest.ts +2 -8
- package/src/app-mcp.ts +10 -23
- package/src/app-permissions-borrowed.ts +129 -0
- package/src/app-permissions-site.ts +24 -0
- package/src/app-permissions.ts +7 -17
- package/src/budgets.ts +4 -35
- package/src/cdp-shot-driver.ts +3 -1
- package/src/cmd-auth-spec.ts +16 -0
- package/src/cmd-auth.ts +65 -0
- package/src/cmd-build.ts +64 -15
- package/src/cmd-db-backfill.ts +25 -12
- package/src/cmd-db.ts +6 -5
- package/src/cmd-deploy-helm.ts +7 -6
- package/src/cmd-deploy.ts +4 -4
- package/src/cmd-dev.ts +9 -8
- package/src/cmd-doctor-spec.ts +2 -9
- package/src/cmd-doctor.ts +41 -14
- package/src/cmd-env.ts +26 -2
- package/src/cmd-generate.ts +79 -32
- package/src/cmd-help.ts +19 -6
- package/src/cmd-i18n.ts +10 -9
- package/src/cmd-jobs.ts +18 -6
- package/src/cmd-manifest-spec.ts +1 -1
- package/src/cmd-manifest.ts +6 -25
- package/src/cmd-mcp.ts +46 -15
- package/src/cmd-planned.ts +15 -8
- package/src/cmd-routes.ts +12 -2
- package/src/cmd-secrets.ts +47 -6
- package/src/cmd-shot-matrix.ts +1 -3
- package/src/cmd-shot.ts +12 -3
- package/src/cmd-tasks.ts +2 -2
- package/src/cmd-test-spec.ts +1 -7
- package/src/cmd-test.ts +13 -29
- package/src/cmd-verify.ts +4 -4
- package/src/db-subscribes.ts +12 -4
- package/src/dev-dashboard-guard.ts +76 -0
- package/src/dev-dashboard.ts +51 -29
- package/src/dev-lock.ts +4 -7
- package/src/dispatch.ts +3 -6
- package/src/doctor-app-url.ts +39 -0
- package/src/doctor-auth.ts +95 -0
- package/src/drift-replica-identity.ts +42 -0
- package/src/error-catalog.ts +6 -9
- package/src/error-codes.ts +23 -3
- package/src/error-unthrown.ts +29 -11
- package/src/errors.ts +54 -83
- package/src/framework-schema-apply.ts +78 -0
- package/src/framework-schema-stamp.ts +96 -0
- package/src/framework-schema.ts +13 -2
- package/src/generate-files.ts +4 -3
- package/src/generate-kinds.ts +33 -1
- package/src/generate-shadow.ts +46 -0
- package/src/generate-write.ts +47 -13
- package/src/gh-target.ts +12 -7
- package/src/i18n-audit.ts +4 -1
- package/src/index.ts +1 -3
- package/src/island-bundle.ts +3 -15
- package/src/jobs-drain.ts +82 -15
- package/src/jobs-report.ts +4 -4
- package/src/loopback-host.ts +13 -0
- package/src/manifest-staleness.ts +62 -0
- package/src/mcp-errors.ts +12 -3
- package/src/mcp-host.ts +16 -9
- package/src/mcp-test-run.ts +38 -0
- package/src/messages.ts +2 -0
- package/src/metrics-endpoint.ts +25 -1
- package/src/output.ts +6 -0
- package/src/page-navigation.ts +7 -23
- package/src/page-speculation.ts +5 -20
- package/src/prerender-out.ts +34 -7
- package/src/prerender.ts +8 -5
- package/src/pwa-artifacts.ts +26 -94
- package/src/registry.ts +2 -0
- package/src/revalidated-response.ts +4 -3
- package/src/role-replicator.ts +16 -0
- package/src/role-start.ts +9 -4
- package/src/role-sync.ts +55 -20
- package/src/runtime-assets.ts +11 -7
- package/src/runtime-bindings.ts +4 -4
- package/src/runtime-cache.ts +38 -47
- package/src/runtime-idempotency-scope.ts +23 -0
- package/src/runtime-isr.ts +36 -0
- package/src/runtime-jobs.ts +8 -33
- package/src/runtime-mfa-warning.ts +29 -0
- package/src/runtime-notify-retention.ts +10 -46
- package/src/runtime-overrides.ts +1 -1
- package/src/runtime-purge.ts +38 -16
- package/src/runtime-queue.ts +31 -10
- package/src/runtime-realtime.ts +9 -45
- package/src/runtime-render.ts +62 -35
- package/src/runtime-services.ts +20 -17
- package/src/runtime-storage.ts +10 -5
- package/src/schema-diff.ts +16 -0
- package/src/schema-drift.ts +45 -15
- package/src/seo-routes.ts +80 -9
- package/src/serve-boot.ts +34 -6
- package/src/serve-drain.ts +9 -33
- package/src/serve-env.ts +9 -6
- package/src/serve-web.ts +12 -7
- package/src/shot-locale.ts +5 -23
- package/src/site-assets.ts +3 -5
- package/src/site-config.ts +9 -48
- package/src/site-seo.ts +1 -1
- package/src/sitemap-lastmod.ts +5 -0
- package/src/solid-loader.ts +12 -3
- package/src/source-files.ts +6 -0
- package/src/stored-object-headers.ts +43 -0
- package/src/sync-url.ts +2 -2
- package/src/tasks-facts.ts +3 -29
- package/src/templates/action.ts +3 -0
- package/src/templates/naming.ts +28 -0
- package/src/templates/route.ts +3 -6
- package/src/templates/scaffold-app.ts +6 -2
- package/src/templates/scaffold-container-compose.ts +10 -1
- package/src/templates/scaffold-container.ts +4 -2
- package/src/templates/scaffold-db-client.ts +4 -2
- package/src/templates/scaffold-helm-fences.ts +76 -0
- package/src/templates/scaffold-helm-templates.ts +32 -7
- package/src/templates/scaffold-helm.ts +31 -1
- package/src/templates/scaffold-repo.ts +9 -1
- package/src/templates/scaffold-shell.ts +39 -36
- package/src/test-counts.ts +2 -6
- package/src/test-ignores.ts +25 -0
- package/src/test-passes.ts +0 -15
- package/src/test-select.ts +5 -14
- package/src/test-shards.ts +41 -93
- package/src/test-workers.ts +26 -4
- package/src/theme-boot.ts +6 -31
- package/src/trusted-hops-error.ts +17 -0
- package/src/unthrown-codes.ts +26 -0
- package/src/verify-checks.ts +17 -46
- package/src/verify-deadline.ts +130 -33
- package/src/verify-merge.ts +60 -27
- package/src/verify-part-step.ts +90 -0
- package/src/verify-run.ts +29 -8
- package/src/verify-shard.ts +13 -2
- package/src/verify-stalled.ts +152 -0
- package/src/verify-tests.ts +5 -2
- package/src/build-errors.ts +0 -54
- package/src/verify-errors.ts +0 -58
package/src/runtime-cache.ts
CHANGED
|
@@ -4,8 +4,6 @@
|
|
|
4
4
|
// and lru unconditionally, redis on `REDIS_URL` and cdn on any real purge driver, so the key was
|
|
5
5
|
// declared, validated at boot, documented, and read by nothing.
|
|
6
6
|
|
|
7
|
-
import { existsSync } from 'node:fs';
|
|
8
|
-
import { join } from 'node:path';
|
|
9
7
|
import type { CacheTier, PurgeDriver } from '@ultimat3/cache';
|
|
10
8
|
import {
|
|
11
9
|
CacheDriverUnavailableError,
|
|
@@ -19,11 +17,9 @@ import {
|
|
|
19
17
|
registerTier,
|
|
20
18
|
resetTiers,
|
|
21
19
|
} from '@ultimat3/cache';
|
|
22
|
-
import type { CacheTierName } from '@ultimat3/core';
|
|
23
|
-
import {
|
|
20
|
+
import type { AppConfig, CacheTierName } from '@ultimat3/core';
|
|
21
|
+
import { backoffDelay, defineConfig, logger, renderThrowable } from '@ultimat3/core';
|
|
24
22
|
import type { Transport, TransportSubscription } from '@ultimat3/realtime/server';
|
|
25
|
-
import { APP_CONFIG_EXPORT } from './app-auth';
|
|
26
|
-
import { APP_CONFIG_FILE } from './app-root';
|
|
27
23
|
import type { Env } from './runtime-bindings';
|
|
28
24
|
|
|
29
25
|
/**
|
|
@@ -41,11 +37,17 @@ export interface CacheTiersOptions {
|
|
|
41
37
|
/**
|
|
42
38
|
* `config.cache.tiers`, verbatim. REQUIRED, and that is the enforcement (axiom 3): a boot that
|
|
43
39
|
* has not read the app's declaration cannot call this function at all — it is a type error, not
|
|
44
|
-
* a convention to remember. `
|
|
40
|
+
* a convention to remember. `cacheTiersOf` is what a boot holding the loaded config calls for it.
|
|
45
41
|
*/
|
|
46
42
|
readonly tiers: readonly CacheTierName[];
|
|
43
|
+
/** The wait before re-subscribing after `attempt` failures (1-based). Injected by a test. */
|
|
44
|
+
readonly subscribeRetryMs?: ((attempt: number) => number) | undefined;
|
|
47
45
|
}
|
|
48
46
|
|
|
47
|
+
/** 1 s doubling to 30 s, full jitter: a bus that is down is retried without a fleet in lockstep. */
|
|
48
|
+
const defaultSubscribeRetryMs = (attempt: number): number =>
|
|
49
|
+
backoffDelay({ attempt, base: 1_000, max: 30_000, jitter: 'full' });
|
|
50
|
+
|
|
49
51
|
/**
|
|
50
52
|
* The rungs an app that declares none gets — ASKED of `defineConfig` rather than written out, for
|
|
51
53
|
* two reasons that point the same way: `defaults()` is private to `@ultimat3/core`, and a second
|
|
@@ -56,32 +58,9 @@ export const DEFAULT_CACHE_TIERS: readonly CacheTierName[] = defineConfig({
|
|
|
56
58
|
name: 'cache-defaults',
|
|
57
59
|
}).cache.tiers;
|
|
58
60
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
const isTierList = (value: unknown): value is readonly CacheTierName[] =>
|
|
63
|
-
Array.isArray(value) && value.every((entry) => CACHE_TIERS.some((name) => name === entry));
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* `cache.tiers` out of the app's own `app.config.ts` — the sibling of `app-auth.ts`'s
|
|
67
|
-
* `loadSignInPath`, and structural for the same reason: `defineConfig` returns a plain object, and
|
|
68
|
-
* a config that resolved through an older core simply has no `cache` section.
|
|
69
|
-
*
|
|
70
|
-
* A list this refuses cannot come from `defineConfig` — `validate()` rejects an unknown rung one
|
|
71
|
-
* `await import` above this line — so the fallback is for a hand-written config object, and the
|
|
72
|
-
* two rungs every app starts from are the honest answer for one.
|
|
73
|
-
*/
|
|
74
|
-
export async function loadCacheTiers(root: string): Promise<readonly CacheTierName[]> {
|
|
75
|
-
const configPath = join(root, APP_CONFIG_FILE);
|
|
76
|
-
if (!existsSync(configPath)) return DEFAULT_CACHE_TIERS;
|
|
77
|
-
const module = (await import(configPath)) as Record<string, unknown>;
|
|
78
|
-
const config = module[APP_CONFIG_EXPORT];
|
|
79
|
-
if (!isRecord(config)) return DEFAULT_CACHE_TIERS;
|
|
80
|
-
const cache = config['cache'];
|
|
81
|
-
if (!isRecord(cache)) return DEFAULT_CACHE_TIERS;
|
|
82
|
-
const { tiers } = cache;
|
|
83
|
-
return isTierList(tiers) ? tiers : DEFAULT_CACHE_TIERS;
|
|
84
|
-
}
|
|
61
|
+
/** `cache.tiers` out of the config `startServices` loads once; the defaults for a root with none. */
|
|
62
|
+
export const cacheTiersOf = (config: AppConfig | undefined): readonly CacheTierName[] =>
|
|
63
|
+
config?.cache.tiers ?? DEFAULT_CACHE_TIERS;
|
|
85
64
|
|
|
86
65
|
/**
|
|
87
66
|
* `REDIS_URL` is the same "an unset variable means the embedded default" law the db, events,
|
|
@@ -176,24 +155,36 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
|
|
|
176
155
|
registerInvalidationBroadcast(async (wireTags) => {
|
|
177
156
|
await options.transport.publish(CACHE_INVALIDATE_SUBJECT, JSON.stringify(wireTags));
|
|
178
157
|
});
|
|
179
|
-
// Not awaited HERE: the boot must not block on a subscribe
|
|
180
|
-
//
|
|
181
|
-
//
|
|
182
|
-
//
|
|
183
|
-
//
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
158
|
+
// Not awaited HERE: the boot must not block on a subscribe. The PROMISE is held rather than a
|
|
159
|
+
// handle assigned inside a `.then`, because the release ran first whenever `stop()` beat the round
|
|
160
|
+
// trip and the subscription that landed afterwards was live with nobody left holding it.
|
|
161
|
+
// RETRIED until it lands or the release runs (s1-con #8): a refused boot subscribe used to be
|
|
162
|
+
// logged once and never asked again, so the process missed every peer invalidation for its life.
|
|
163
|
+
const released = Promise.withResolvers<undefined>();
|
|
164
|
+
let stopped = false;
|
|
165
|
+
const retryMs = options.subscribeRetryMs ?? defaultSubscribeRetryMs;
|
|
166
|
+
const subscribing = (async (): Promise<TransportSubscription | undefined> => {
|
|
167
|
+
for (let attempt = 1; !stopped; attempt += 1) {
|
|
168
|
+
try {
|
|
169
|
+
return await options.transport.subscribe(CACHE_INVALIDATE_SUBJECT, (payload: string) => {
|
|
170
|
+
void applyBroadcast(payload);
|
|
171
|
+
});
|
|
172
|
+
} catch (error) {
|
|
173
|
+
logger.warn('cache.broadcast.subscribe-failed', {
|
|
174
|
+
error: broadcastErrorText(error),
|
|
175
|
+
attempt,
|
|
176
|
+
});
|
|
177
|
+
await Promise.race([Bun.sleep(retryMs(attempt)), released.promise]);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return undefined;
|
|
181
|
+
})();
|
|
193
182
|
|
|
194
183
|
// `resetTiers()` drops the registry AND the broadcast in one call: this boot is the only thing
|
|
195
184
|
// that registers either, and a tier left behind would purge for a process that has stopped.
|
|
196
185
|
return async () => {
|
|
186
|
+
stopped = true;
|
|
187
|
+
released.resolve(undefined);
|
|
197
188
|
(await subscribing)?.unsubscribe();
|
|
198
189
|
resetTiers();
|
|
199
190
|
};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// A container boot over an idempotency store that keeps its records per process. The boot installs
|
|
2
|
+
// the shared Postgres store (`runtime-queue.ts`); an app module that swapped in a per-process one
|
|
3
|
+
// after it left `idempotent: true` meaning "once per replica" — with nothing to say so, because
|
|
4
|
+
// `assertIdempotencyScope` only fires on a `'shared'` DECLARATION and nothing infers a replica count.
|
|
5
|
+
|
|
6
|
+
import { getIdempotencyStore, type IdempotencyStore } from '@ultimat3/action';
|
|
7
|
+
import { logger } from '@ultimat3/core';
|
|
8
|
+
|
|
9
|
+
/** `true` when it warned. Said, not refused: one replica is a real deployment and is correct. */
|
|
10
|
+
export function warnIfIdempotencyProcessScoped(
|
|
11
|
+
store: IdempotencyStore = getIdempotencyStore(),
|
|
12
|
+
): boolean {
|
|
13
|
+
if (store.scope === 'shared') return false;
|
|
14
|
+
logger.warn('X_CONFIG_INVALID', {
|
|
15
|
+
code: 'X_CONFIG_INVALID',
|
|
16
|
+
cause:
|
|
17
|
+
'the idempotency store this process ended its boot with keeps records per process, so on more than one replica a retried idempotent action that lands on another replica runs its handler again — docker/helm/values.yaml runs roles.web.replicas: 3',
|
|
18
|
+
// Declaring the scope is the enforcement: `assertIdempotencyScope` then refuses the store.
|
|
19
|
+
fix: "configureIdempotency({ scope: 'shared' })",
|
|
20
|
+
scope: store.scope ?? 'undeclared',
|
|
21
|
+
});
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// The ISR controller a boot serves pages through: built, ATTACHED as the framework's revalidator,
|
|
2
|
+
// and handed back with the detach the boot puts on its stop list. One function for `x dev` and the
|
|
3
|
+
// container, so neither can build a controller `invalidateTags` never reaches (plan 101, s2-con #1).
|
|
4
|
+
|
|
5
|
+
import type { IsrController, IsrStore } from '@ultimat3/render/server';
|
|
6
|
+
import { createIsrController } from '@ultimat3/render/server';
|
|
7
|
+
|
|
8
|
+
export interface AttachedIsr {
|
|
9
|
+
readonly isr: IsrController;
|
|
10
|
+
/** Detaches the revalidator and drops this controller's graph edges; idempotent. */
|
|
11
|
+
readonly release: () => void;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Unattached, a `revalidate: { tags }` page (no TTL) is fresh until invalidated — and nothing
|
|
16
|
+
* invalidates it: `invalidateTags` calls the registered revalidator, which was nobody.
|
|
17
|
+
*/
|
|
18
|
+
export function attachedIsr(options: {
|
|
19
|
+
readonly buildId: string;
|
|
20
|
+
readonly store?: IsrStore;
|
|
21
|
+
}): AttachedIsr {
|
|
22
|
+
const isr = createIsrController({
|
|
23
|
+
buildId: options.buildId,
|
|
24
|
+
...(options.store === undefined ? {} : { store: options.store }),
|
|
25
|
+
});
|
|
26
|
+
const detach = isr.attach();
|
|
27
|
+
let released = false;
|
|
28
|
+
return {
|
|
29
|
+
isr,
|
|
30
|
+
release: () => {
|
|
31
|
+
if (released) return;
|
|
32
|
+
released = true;
|
|
33
|
+
detach();
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
}
|
package/src/runtime-jobs.ts
CHANGED
|
@@ -1,14 +1,9 @@
|
|
|
1
1
|
// `jobs.queues`, `jobs.concurrency` and `jobs.visibilityTimeoutMs`, read out of the app's own
|
|
2
|
-
// `app.config.ts` for the `worker` role
|
|
3
|
-
//
|
|
4
|
-
// `default` at its own concurrency whatever the config said, so a `mail` queue ran nothing.
|
|
2
|
+
// `app.config.ts` for the `worker` role, out of the config `startServices` loads once. Until 22.0.0
|
|
3
|
+
// `createWorker` was called with none of them, so a `mail` queue ran nothing.
|
|
5
4
|
|
|
6
|
-
|
|
7
|
-
// `pathToFileURL` and `fileURLToPath`, and nothing that joins a path.
|
|
8
|
-
import { join } from 'node:path';
|
|
5
|
+
import type { AppConfig } from '@ultimat3/core';
|
|
9
6
|
import { registeredJobs, type WorkerOptions } from '@ultimat3/jobs';
|
|
10
|
-
import { APP_CONFIG_EXPORT } from './app-auth';
|
|
11
|
-
import { APP_CONFIG_FILE } from './app-root';
|
|
12
7
|
|
|
13
8
|
/** What the worker takes from the config. An absent number keeps the worker's own default. */
|
|
14
9
|
export interface WorkerConfig {
|
|
@@ -17,37 +12,17 @@ export interface WorkerConfig {
|
|
|
17
12
|
readonly visibilityTimeoutMs: number | undefined;
|
|
18
13
|
}
|
|
19
14
|
|
|
15
|
+
/** A root with no `app.config.ts`: no queue of its own, and the worker's own timings. */
|
|
20
16
|
const NO_WORKER_CONFIG: WorkerConfig = Object.freeze({
|
|
21
17
|
queues: [],
|
|
22
18
|
concurrency: undefined,
|
|
23
19
|
visibilityTimeoutMs: undefined,
|
|
24
20
|
});
|
|
25
21
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
* A number is handed on AS WRITTEN, never screened here: `createWorker`'s `resolveWorkerTimings`
|
|
31
|
-
* is the one refusal for a non-finite knob, and core's `defineConfig` screens both counts first.
|
|
32
|
-
*/
|
|
33
|
-
const numberOf = (value: unknown): number | undefined =>
|
|
34
|
-
typeof value === 'number' ? value : undefined;
|
|
35
|
-
|
|
36
|
-
export async function loadWorkerConfig(root: string): Promise<WorkerConfig> {
|
|
37
|
-
const path = join(root, APP_CONFIG_FILE);
|
|
38
|
-
if (!(await Bun.file(path).exists())) return NO_WORKER_CONFIG;
|
|
39
|
-
const module = (await import(path)) as Record<string, unknown>;
|
|
40
|
-
const config = module[APP_CONFIG_EXPORT];
|
|
41
|
-
const jobs = isRecord(config) ? config['jobs'] : undefined;
|
|
42
|
-
if (!isRecord(jobs)) return NO_WORKER_CONFIG;
|
|
43
|
-
const queues = Array.isArray(jobs['queues'])
|
|
44
|
-
? jobs['queues'].filter((queue): queue is string => typeof queue === 'string' && queue !== '')
|
|
45
|
-
: [];
|
|
46
|
-
return {
|
|
47
|
-
queues,
|
|
48
|
-
concurrency: numberOf(jobs['concurrency']),
|
|
49
|
-
visibilityTimeoutMs: numberOf(jobs['visibilityTimeoutMs']),
|
|
50
|
-
};
|
|
22
|
+
export function workerConfigOf(config: AppConfig | undefined): WorkerConfig {
|
|
23
|
+
if (config === undefined) return NO_WORKER_CONFIG;
|
|
24
|
+
const { queues, concurrency, visibilityTimeoutMs } = config.jobs;
|
|
25
|
+
return { queues, concurrency, visibilityTimeoutMs };
|
|
51
26
|
}
|
|
52
27
|
|
|
53
28
|
/**
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// The web role's boot asks once whether `x auth seal-mfa` still has work to do (wiki/Upgrading.md,
|
|
2
|
+
// the operator step for sealed MFA secrets). `x doctor` asks the same question only when someone
|
|
3
|
+
// runs it; an unsealed secret is a user refused at the second factor, so the boot says so too.
|
|
4
|
+
|
|
5
|
+
import { BuiltinAdapter, countUnsealedMfaSecrets } from '@ultimat3/auth';
|
|
6
|
+
import { logger } from '@ultimat3/core';
|
|
7
|
+
import type { DbClient } from '@ultimat3/db';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Answers the count it logged, `0` when there was nothing to say or nothing could be asked. Never
|
|
11
|
+
* throws: a database that cannot answer this is the readiness check's to report, not a reason for
|
|
12
|
+
* a pod to refuse traffic its other users can be served.
|
|
13
|
+
*/
|
|
14
|
+
export async function warnUnsealedMfaSecrets(client: DbClient): Promise<number> {
|
|
15
|
+
let unsealed = 0;
|
|
16
|
+
try {
|
|
17
|
+
unsealed = await countUnsealedMfaSecrets({ adapter: new BuiltinAdapter(client) });
|
|
18
|
+
} catch {
|
|
19
|
+
return 0;
|
|
20
|
+
}
|
|
21
|
+
if (unsealed === 0) return 0;
|
|
22
|
+
logger.warn('X_MFA_SECRET_UNSEALED', {
|
|
23
|
+
code: 'X_MFA_SECRET_UNSEALED',
|
|
24
|
+
cause: `${unsealed} user(s) hold a second-factor secret that is not sealed; the framework never reads a plaintext secret, so each of them is refused at the second factor until it is sealed`,
|
|
25
|
+
fix: 'x auth seal-mfa --json',
|
|
26
|
+
unsealed,
|
|
27
|
+
});
|
|
28
|
+
return unsealed;
|
|
29
|
+
}
|
|
@@ -1,18 +1,8 @@
|
|
|
1
|
-
// `notify.inboxReadRetentionMs` and `notify.inboxUnreadRetentionMs`,
|
|
2
|
-
// `app
|
|
3
|
-
//
|
|
4
|
-
// a config that resolved through an older core simply has no `notify` section.
|
|
5
|
-
//
|
|
6
|
-
// WHY A LOADER AND NOT A BOOT ARGUMENT: `startServices` has no `AppConfig` — the app's modules
|
|
7
|
-
// import after it, which is the same reason `configureAuthLimiters` takes a factory. A key read
|
|
8
|
-
// here is a key read from the file the operator edited, per boot.
|
|
1
|
+
// `notify.inboxReadRetentionMs` and `notify.inboxUnreadRetentionMs`, out of the config
|
|
2
|
+
// `startServices` loads once (`app-config-load.ts`) — before the app's modules import, so the window
|
|
3
|
+
// is read from the file the operator edited, per boot.
|
|
9
4
|
|
|
10
|
-
|
|
11
|
-
// `pathToFileURL` and `fileURLToPath`, and nothing that joins a path.
|
|
12
|
-
import { join } from 'node:path';
|
|
13
|
-
import { INBOX_RETENTION_KEYS } from '@ultimat3/core';
|
|
14
|
-
import { APP_CONFIG_EXPORT } from './app-auth';
|
|
15
|
-
import { APP_CONFIG_FILE } from './app-root';
|
|
5
|
+
import type { AppConfig } from '@ultimat3/core';
|
|
16
6
|
|
|
17
7
|
/**
|
|
18
8
|
* The two windows in milliseconds, each `undefined` where the app named none.
|
|
@@ -32,38 +22,12 @@ export const NO_INBOX_RETENTION: InboxRetention = Object.freeze({
|
|
|
32
22
|
unreadMs: undefined,
|
|
33
23
|
});
|
|
34
24
|
|
|
35
|
-
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
36
|
-
typeof value === 'object' && value !== null;
|
|
37
|
-
|
|
38
25
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
* assembled by hand, or one that resolved through a core too old to validate the section, reaches
|
|
42
|
-
* this line unchecked. A window that is not a positive finite number reads as absent: refusing
|
|
43
|
-
* would take the sweep over the other four framework tables down with it, and the whole point of
|
|
44
|
-
* the key is that not sweeping is a legal state.
|
|
26
|
+
* Core's validator (through the loader) refuses a window that is not a positive finite number, so
|
|
27
|
+
* a `NaN` cutoff — every row older than it, the inbox emptied on the first pass — cannot reach here.
|
|
45
28
|
*/
|
|
46
|
-
|
|
47
|
-
const
|
|
48
|
-
if (
|
|
49
|
-
return
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
export async function loadInboxRetention(root: string): Promise<InboxRetention> {
|
|
53
|
-
const configPath = join(root, APP_CONFIG_FILE);
|
|
54
|
-
// `Bun.file(p).exists()` rather than `existsSync`: this function is already async, so the
|
|
55
|
-
// `node:fs` import buys nothing here — and an import with nothing to say for itself is what
|
|
56
|
-
// `bun run node-imports` refuses.
|
|
57
|
-
if (!(await Bun.file(configPath).exists())) return NO_INBOX_RETENTION;
|
|
58
|
-
const module = (await import(configPath)) as Record<string, unknown>;
|
|
59
|
-
const config = module[APP_CONFIG_EXPORT];
|
|
60
|
-
if (!isRecord(config)) return NO_INBOX_RETENTION;
|
|
61
|
-
const notify = config['notify'];
|
|
62
|
-
if (!isRecord(notify)) return NO_INBOX_RETENTION;
|
|
63
|
-
// Keyed off core's own list rather than two string literals here, so the two names exist in one
|
|
64
|
-
// place. What actually stops a third window being declared-and-never-read is
|
|
65
|
-
// `bun run scripts/config-readers.ts` — every leaf key of `AppConfig` needs a reader or a pinned
|
|
66
|
-
// reason, and it is the guard whose own header calls that "the framework's most repeated defect".
|
|
67
|
-
const [readKey, unreadKey] = INBOX_RETENTION_KEYS;
|
|
68
|
-
return { readMs: windowOf(notify, readKey), unreadMs: windowOf(notify, unreadKey) };
|
|
29
|
+
export function inboxRetentionOf(config: AppConfig | undefined): InboxRetention {
|
|
30
|
+
const notify = config?.notify;
|
|
31
|
+
if (notify === undefined) return NO_INBOX_RETENTION;
|
|
32
|
+
return { readMs: notify.inboxReadRetentionMs, unreadMs: notify.inboxUnreadRetentionMs };
|
|
69
33
|
}
|
package/src/runtime-overrides.ts
CHANGED
|
@@ -69,7 +69,7 @@ export interface RuntimeOverrides {
|
|
|
69
69
|
* answers a document, and a protocol that fixes its own error shape is neither.
|
|
70
70
|
*/
|
|
71
71
|
readonly routes?: readonly Route[];
|
|
72
|
-
/** The `/media/*` transform. Omitted, `builtinImageDriver` — core's PNG/JPEG pipeline. */
|
|
72
|
+
/** The `/media/*` transform. Omitted, `builtinImageDriver` — core's PNG/JPEG/WebP pipeline. */
|
|
73
73
|
readonly images?: ImageTransformDriver;
|
|
74
74
|
/**
|
|
75
75
|
* Who is dialling the `sync` node. Omitted, the app's own `configureAuthenticator()` is adapted
|
package/src/runtime-purge.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
// The retention sweep this boot owns: the framework tables that grow with traffic, the `purge()`
|
|
2
2
|
// job that empties them and the `task` that fires it hourly.
|
|
3
3
|
//
|
|
4
|
-
//
|
|
5
|
-
// `x_auth`
|
|
6
|
-
// prose but here: this header said "three" for two releases
|
|
7
|
-
// boot's DDL, which is how `x_notify_inbox` became the one
|
|
4
|
+
// SEVEN TARGETS over seven tables, `As of 2026-10-02` — `x_idempotency`, `x_rate_limit`, the
|
|
5
|
+
// `x_auth` limiter's table, `x_notify_deliveries`, `x_notify_inbox`, `x_notify_digests` and
|
|
6
|
+
// `x_job_events`. Counted nowhere in prose but here: this header said "three" for two releases
|
|
7
|
+
// after the notify tables joined the boot's DDL, which is how `x_notify_inbox` became the one
|
|
8
|
+
// framework table nothing swept.
|
|
8
9
|
//
|
|
9
10
|
// WHY here and not in the packages that own the tables: `postgresIdempotencyStore` (tier 3),
|
|
10
11
|
// `postgresRateLimitStore` (tier 2) and `postgresAuthLimiter` (tier 2) cannot see each other and
|
|
@@ -20,9 +21,16 @@
|
|
|
20
21
|
import type { PostgresIdempotencyStore } from '@ultimat3/action';
|
|
21
22
|
import { purgeAuthLimits } from '@ultimat3/auth';
|
|
22
23
|
import type { PostgresRateLimitStore } from '@ultimat3/http';
|
|
23
|
-
import type { JobHandle, PurgeInput, PurgeTarget } from '@ultimat3/jobs';
|
|
24
|
-
import {
|
|
25
|
-
|
|
24
|
+
import type { EventBus, JobHandle, PurgeInput, PurgeTarget } from '@ultimat3/jobs';
|
|
25
|
+
import {
|
|
26
|
+
DEFAULT_PURGE_CRON,
|
|
27
|
+
eventsPurgeTarget,
|
|
28
|
+
getJob,
|
|
29
|
+
getTask,
|
|
30
|
+
purge,
|
|
31
|
+
task,
|
|
32
|
+
} from '@ultimat3/jobs';
|
|
33
|
+
import { purgeNotifyDeliveries, purgeNotifyDigests, purgeNotifyInbox } from '@ultimat3/notify';
|
|
26
34
|
import type { InboxRetention } from './runtime-notify-retention';
|
|
27
35
|
import { NO_INBOX_RETENTION } from './runtime-notify-retention';
|
|
28
36
|
|
|
@@ -32,7 +40,7 @@ export const PURGE_JOB_NAME = 'x.purge';
|
|
|
32
40
|
export const PURGE_TASK_NAME = 'x.purge.hourly';
|
|
33
41
|
|
|
34
42
|
/**
|
|
35
|
-
* The
|
|
43
|
+
* The stores this boot BUILT, handed over rather than rebuilt here. A second
|
|
36
44
|
* `postgresIdempotencyStore({ executor })` would sweep on the default window even where the boot
|
|
37
45
|
* had configured another — two answers to "how long is a record kept", and the shorter one
|
|
38
46
|
* deletes reservations a retry is still entitled to.
|
|
@@ -45,15 +53,22 @@ export interface RetentionStores {
|
|
|
45
53
|
readonly idempotency: PostgresIdempotencyStore;
|
|
46
54
|
readonly rateLimit: PostgresRateLimitStore;
|
|
47
55
|
/**
|
|
48
|
-
* The
|
|
49
|
-
* `
|
|
50
|
-
*
|
|
56
|
+
* The bus this boot installed for `step.waitForEvent`. REQUIRED: `x_job_events` takes a row per
|
|
57
|
+
* publish and `find` only filters an expired one, so a boot that left it out is the table
|
|
58
|
+
* growing for the life of the deployment — which it did until this member existed.
|
|
59
|
+
*/
|
|
60
|
+
readonly events: EventBus;
|
|
61
|
+
/**
|
|
62
|
+
* The app's own answer to "how long is an unread message kept", read off the config
|
|
63
|
+
* `startServices` loads (`inboxRetentionOf`). Absent windows mean the inbox is never swept,
|
|
64
|
+
* which is the default and a decision rather than an oversight — see `NotifyConfig` in
|
|
65
|
+
* `@ultimat3/core`.
|
|
51
66
|
*/
|
|
52
67
|
readonly inboxRetention?: InboxRetention | undefined;
|
|
53
68
|
}
|
|
54
69
|
|
|
55
70
|
/**
|
|
56
|
-
* The
|
|
71
|
+
* The three notify tables, and no store is in `RetentionStores` — deliberately, and for the
|
|
57
72
|
* reason `authTarget` is not either. `setNotifyStores` is an APP's boot line and runs when the
|
|
58
73
|
* app's modules import, which is AFTER this install; `framework-schema.ts` says so where it
|
|
59
74
|
* applies the DDL "whether or not this boot calls `setNotifyStores`". So the sweep can only ask,
|
|
@@ -79,6 +94,12 @@ function notifyTargets(retention: InboxRetention): readonly PurgeTarget[] {
|
|
|
79
94
|
retention.unreadMs === undefined ? undefined : new Date(nowMs - retention.unreadMs),
|
|
80
95
|
}),
|
|
81
96
|
},
|
|
97
|
+
{
|
|
98
|
+
name: 'x_notify_digests',
|
|
99
|
+
// The job's clock, and the store's own window (`createPgDigestStore({ retentionMs })`, a week
|
|
100
|
+
// by default) — a closed window only its dead-lettered flush would ever have drained.
|
|
101
|
+
purgeExpired: (nowMs: number): Promise<number> => purgeNotifyDigests(nowMs),
|
|
102
|
+
},
|
|
82
103
|
];
|
|
83
104
|
}
|
|
84
105
|
|
|
@@ -94,14 +115,13 @@ let installed: readonly PurgeTarget[] = [];
|
|
|
94
115
|
let sweep: JobHandle<PurgeInput> | undefined;
|
|
95
116
|
|
|
96
117
|
/**
|
|
97
|
-
* `
|
|
98
|
-
*
|
|
99
|
-
* tables would be a second delete that removes nothing.
|
|
118
|
+
* `x_auth_lockouts` — every credential bucket, one row per key. The target keeps the `x_auth`
|
|
119
|
+
* name it has always reported under: it is what a dashboard and `PurgeReport.swept` already key on.
|
|
100
120
|
*/
|
|
101
121
|
const authTarget: PurgeTarget = {
|
|
102
122
|
name: 'x_auth',
|
|
103
123
|
// No `nowMs`: a limiter built through the seam holds the clock its host handed it, which is the
|
|
104
|
-
// clock every
|
|
124
|
+
// clock every instant in that table was written from.
|
|
105
125
|
purgeExpired: () => purgeAuthLimits(),
|
|
106
126
|
};
|
|
107
127
|
|
|
@@ -121,6 +141,8 @@ function retentionTargets(stores: RetentionStores): readonly PurgeTarget[] {
|
|
|
121
141
|
},
|
|
122
142
|
authTarget,
|
|
123
143
|
...notifyTargets(stores.inboxRetention ?? NO_INBOX_RETENTION),
|
|
144
|
+
// Its own clock, and so no `nowMs`: every instant in that table is the database's.
|
|
145
|
+
eventsPurgeTarget(stores.events),
|
|
124
146
|
];
|
|
125
147
|
}
|
|
126
148
|
|
package/src/runtime-queue.ts
CHANGED
|
@@ -7,16 +7,17 @@
|
|
|
7
7
|
import {
|
|
8
8
|
type PostgresIdempotencyStore,
|
|
9
9
|
postgresIdempotencyStore,
|
|
10
|
+
requestDeadlineMs,
|
|
10
11
|
resetIdempotency,
|
|
11
12
|
setIdempotencyStore,
|
|
12
13
|
} from '@ultimat3/action';
|
|
13
14
|
import type { DbClient, PgliteClient, PostgresClient, SqlFragment } from '@ultimat3/db';
|
|
14
15
|
import {
|
|
16
|
+
baseClient,
|
|
15
17
|
createPgliteClient,
|
|
16
18
|
createPostgresClient,
|
|
17
19
|
currentTx,
|
|
18
20
|
pgliteDataDir,
|
|
19
|
-
raw,
|
|
20
21
|
setDbClient,
|
|
21
22
|
} from '@ultimat3/db';
|
|
22
23
|
import type { Tx } from '@ultimat3/entity';
|
|
@@ -26,13 +27,14 @@ import {
|
|
|
26
27
|
createPgDriver,
|
|
27
28
|
createPgEventBus,
|
|
28
29
|
createPgOutboxStore,
|
|
30
|
+
resetEventBus,
|
|
29
31
|
resetJobDriver,
|
|
30
32
|
resetJobsFacade,
|
|
31
33
|
setEventBus,
|
|
32
34
|
setJobDriver,
|
|
33
35
|
setJobsFacade,
|
|
34
36
|
} from '@ultimat3/jobs';
|
|
35
|
-
import {
|
|
37
|
+
import { applyLockedSchema, verifySchema } from './framework-schema-apply';
|
|
36
38
|
import { appExtensions } from './migration-extensions';
|
|
37
39
|
import type { DevServices } from './runtime-bindings';
|
|
38
40
|
import type { RuntimeOverrides } from './runtime-overrides';
|
|
@@ -134,10 +136,18 @@ export function pgExecutorFor(client: DbClient): PgExecutor {
|
|
|
134
136
|
* `@ultimat3/notify` because a package that holds no database dependency cannot apply its own
|
|
135
137
|
* schema — the same reason `SQL_JOBS_TABLE` is applied by the boot.
|
|
136
138
|
*/
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
139
|
+
/**
|
|
140
|
+
* `'apply'` — `ROLE=migrate`, `x dev` and every CLI command — runs the DDL under the migration
|
|
141
|
+
* lock with a bounded `lock_timeout` and stamps the build. `'verify'` — a serving role on an
|
|
142
|
+
* external database — runs none: an `alter table` queued behind one open transaction stalled every
|
|
143
|
+
* enqueue in the fleet on each pod start, and racing `create table`s were a first-deploy `23505`.
|
|
144
|
+
*/
|
|
145
|
+
export type SchemaMode = 'apply' | 'verify';
|
|
140
146
|
|
|
147
|
+
async function applySchema(client: DevDbClient, mode: SchemaMode): Promise<void> {
|
|
148
|
+
if (mode === 'verify') await verifySchema(client);
|
|
149
|
+
else await applyLockedSchema(client);
|
|
150
|
+
}
|
|
141
151
|
/**
|
|
142
152
|
* The dev queue is the real Postgres queue on the embedded Postgres — claiming, leases and the
|
|
143
153
|
* one-live-job-per-key index all behave here exactly as in production. A memory queue in dev
|
|
@@ -169,9 +179,10 @@ async function applySchema(client: DevDbClient): Promise<void> {
|
|
|
169
179
|
async function startJobs(
|
|
170
180
|
client: DevDbClient,
|
|
171
181
|
replica: PostgresClient | undefined,
|
|
172
|
-
overrides
|
|
182
|
+
overrides: RuntimeOverrides | undefined,
|
|
183
|
+
schema: SchemaMode,
|
|
173
184
|
): Promise<RunningQueue> {
|
|
174
|
-
await applySchema(client);
|
|
185
|
+
await applySchema(client, schema);
|
|
175
186
|
const executor = pgExecutorFor(client);
|
|
176
187
|
const driver = overrides?.jobs ?? createPgDriver({ executor });
|
|
177
188
|
setJobDriver(driver);
|
|
@@ -185,7 +196,14 @@ async function startJobs(
|
|
|
185
196
|
);
|
|
186
197
|
const events = createPgEventBus({ executor });
|
|
187
198
|
setEventBus(events);
|
|
188
|
-
const idempotency = postgresIdempotencyStore({
|
|
199
|
+
const idempotency = postgresIdempotencyStore({
|
|
200
|
+
executor,
|
|
201
|
+
// `baseClient`, read per call and never `client`: a transaction's origin is the AMBIENT client
|
|
202
|
+
// `startDb` installed, which is the replicated pair when a standby is configured.
|
|
203
|
+
origin: baseClient,
|
|
204
|
+
// The app's own `requestTimeoutMs`, read per reservation — it is declared after this boot.
|
|
205
|
+
reclaimAfterMs: requestDeadlineMs,
|
|
206
|
+
});
|
|
189
207
|
setIdempotencyStore(idempotency);
|
|
190
208
|
return {
|
|
191
209
|
db: client,
|
|
@@ -204,7 +222,8 @@ async function startJobs(
|
|
|
204
222
|
* startup entirely, so every query it makes fails on a connection this process already dropped.
|
|
205
223
|
*
|
|
206
224
|
* The facade goes with the driver for the same reason: an enqueue routed through a store bound to
|
|
207
|
-
* a closed client is a staged row nothing will ever publish
|
|
225
|
+
* a closed client is a staged row nothing will ever publish — and the event bus with both, or a
|
|
226
|
+
* `publishEvent` after `stop()` runs over the closed pool.
|
|
208
227
|
*/
|
|
209
228
|
async function releaseQueue(
|
|
210
229
|
db: DevDbClient,
|
|
@@ -215,6 +234,7 @@ async function releaseQueue(
|
|
|
215
234
|
// holds this client, and the next command in this process would reserve keys over a closed one.
|
|
216
235
|
resetIdempotency();
|
|
217
236
|
resetJobsFacade();
|
|
237
|
+
resetEventBus();
|
|
218
238
|
resetJobDriver();
|
|
219
239
|
setDbClient(undefined);
|
|
220
240
|
await jobs?.close?.();
|
|
@@ -239,13 +259,14 @@ export async function startQueue(
|
|
|
239
259
|
* this file reads it: see `startDb` for what two readers of one question cost.
|
|
240
260
|
*/
|
|
241
261
|
env: ReplicaEnv = process.env,
|
|
262
|
+
schema: SchemaMode = 'apply',
|
|
242
263
|
): Promise<RunningQueue> {
|
|
243
264
|
const { client: db, replica } = startDb(services, env);
|
|
244
265
|
try {
|
|
245
266
|
// Pay the Postgres boot here, so the first request is not the slow one and a broken database
|
|
246
267
|
// fails at boot rather than on some later query.
|
|
247
268
|
await db.ping();
|
|
248
|
-
return await startJobs(db, replica, overrides);
|
|
269
|
+
return await startJobs(db, replica, overrides, schema);
|
|
249
270
|
} catch (error) {
|
|
250
271
|
// `db.ping()` or `startJobs` is where a broken database is supposed to fail. Without this,
|
|
251
272
|
// the caller exits holding the PGlite lock and the ambient accessors, and nothing is left to
|