@ultimat3/cli 7.0.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CLAUDE.md +15 -1
  2. package/README.md +8 -3
  3. package/package.json +25 -25
  4. package/src/app-boundaries.ts +55 -5
  5. package/src/bin.ts +6 -3
  6. package/src/ci-log.ts +0 -0
  7. package/src/cmd-db-backfill.ts +240 -0
  8. package/src/cmd-db-branch.ts +3 -2
  9. package/src/cmd-db.ts +35 -156
  10. package/src/cmd-deploy.ts +37 -3
  11. package/src/cmd-dev.ts +7 -1
  12. package/src/cmd-errors.ts +2 -3
  13. package/src/cmd-fix.ts +3 -3
  14. package/src/cmd-i18n.ts +67 -5
  15. package/src/cmd-jobs.ts +27 -4
  16. package/src/cmd-mcp.ts +18 -9
  17. package/src/cmd-new.ts +91 -4
  18. package/src/cmd-policy.ts +3 -2
  19. package/src/cmd-pr.ts +55 -4
  20. package/src/cmd-registries.ts +3 -2
  21. package/src/cmd-shot.ts +68 -6
  22. package/src/cmd-tasks.ts +9 -4
  23. package/src/cmd-verify.ts +47 -6
  24. package/src/dev-cache.ts +1 -1
  25. package/src/dev-lock.ts +124 -12
  26. package/src/dev-queue.ts +12 -7
  27. package/src/dev-replicator.ts +3 -7
  28. package/src/dev-roles-fixture.ts +1 -1
  29. package/src/dev-roles.ts +40 -8
  30. package/src/dev-runtime.ts +96 -4
  31. package/src/dev-sync.ts +9 -4
  32. package/src/dispatch.ts +35 -5
  33. package/src/drift.ts +52 -7
  34. package/src/error-codes.ts +5 -0
  35. package/src/framework-scope.ts +57 -5
  36. package/src/generate-kinds.ts +19 -1
  37. package/src/i18n-registration.ts +67 -4
  38. package/src/index.ts +1 -1
  39. package/src/jobs-report.ts +10 -13
  40. package/src/mcp-errors.ts +3 -0
  41. package/src/messages.ts +12 -0
  42. package/src/output.ts +22 -2
  43. package/src/parse.ts +81 -37
  44. package/src/realtime-browser-probe-fixture.ts +9 -0
  45. package/src/runtime-overrides.ts +11 -3
  46. package/src/shot-settle.ts +57 -0
  47. package/src/shot-verdict.ts +27 -4
  48. package/src/sync-authenticator.ts +86 -14
  49. package/src/templates/guard-bare-error.ts +122 -0
  50. package/src/templates/guard-raw-colour.ts +138 -0
  51. package/src/templates/guard-untranslated-string.ts +138 -0
  52. package/src/templates/guard-unzoned-date.ts +142 -0
  53. package/src/templates/index.ts +3 -0
  54. package/src/templates/island.ts +2 -1
  55. package/src/templates/route.ts +1 -1
  56. package/src/templates/scaffold-app.ts +3 -82
  57. package/src/templates/scaffold-container.ts +30 -4
  58. package/src/templates/scaffold-db-package.ts +14 -6
  59. package/src/templates/scaffold-docs.ts +24 -13
  60. package/src/templates/scaffold-entries.ts +131 -0
  61. package/src/templates/scaffold-guards.ts +26 -0
  62. package/src/templates/scaffold-repo.ts +37 -6
  63. package/src/test-select.ts +4 -3
  64. package/src/verify-run.ts +25 -3
  65. package/src/verify-step.ts +11 -2
  66. package/src/verify-tests.ts +11 -3
  67. package/src/write-line.ts +23 -5
package/src/dev-lock.ts CHANGED
@@ -14,11 +14,12 @@
14
14
  // The lock file is what makes the second one nameable at all: nothing else in the process can tell
15
15
  // "another dev server owns this directory" from "the database is broken".
16
16
 
17
- import { unlinkSync } from 'node:fs';
17
+ import { closeSync, mkdirSync, openSync, unlinkSync, writeFileSync } from 'node:fs';
18
18
  import { join } from 'node:path';
19
19
  import { UltimateError } from '@ultimat3/core';
20
20
  import { docsFor } from './error-codes';
21
21
  import { exec, type Runner } from './exec';
22
+ import { quoteArg } from './shell-quote';
22
23
 
23
24
  /** Where the running dev server records itself, inside the state directory it already owns. */
24
25
  export const DEV_LOCK_FILE = 'dev.lock';
@@ -101,6 +102,28 @@ export class DevAlreadyRunningError extends UltimateError {
101
102
  }
102
103
  }
103
104
 
105
+ /**
106
+ * The lock file exists, this process could not read it and could not remove it — which is not
107
+ * "another x dev is running", and was reported as exactly that.
108
+ *
109
+ * `DevAlreadyRunningError` needs a `DevLock`, and the only one in hand on that path was `mine`:
110
+ * the refusal then read `pid <this process> is already running x dev` with `fix: … kill <this
111
+ * process>`, a remedy that kills the reader and a cause naming the wrong holder. Axiom 4 wants a
112
+ * runnable fix and an honest cause, so the honest answer is its own code — the file is the
113
+ * problem, and removing it is what a reader can actually do.
114
+ */
115
+ export class DevLockUnreadableError extends UltimateError {
116
+ constructor(input: { readonly path: string; readonly stateDir: string }) {
117
+ super({
118
+ code: 'X_DEV_LOCK_UNREADABLE',
119
+ cause: `${input.path} could not be parsed as a dev lock and could not be removed, so x dev cannot tell whether another process owns ${input.stateDir}`,
120
+ fix: `rm ${quoteArg(input.path)} # then re-run x dev`,
121
+ docs: docsFor('X_DEV_LOCK_UNREADABLE'),
122
+ meta: { path: input.path, stateDir: input.stateDir },
123
+ });
124
+ }
125
+ }
126
+
104
127
  /** Whatever is listening, as far as the OS will say. Both fields absent when it will not say. */
105
128
  export interface PortHolder {
106
129
  readonly pid?: number;
@@ -207,6 +230,41 @@ export const isPortBound = (port: number, hostname: string): boolean => {
207
230
  }
208
231
  };
209
232
 
233
+ /**
234
+ * Take the lock, or answer `false` because someone else holds it.
235
+ *
236
+ * `wx` is the whole mechanism: the create and the exclusivity are ONE syscall, so two boots racing
237
+ * this cannot both come back `true`. A check followed by a write is what this replaces, and the
238
+ * window between those two was seconds wide — `startDev` boots embedded Postgres, the queue, the
239
+ * transport and the app's modules before anything was written down.
240
+ *
241
+ * Only `EEXIST` is "someone else has it". Anything else — a read-only checkout, a `.x/` nobody may
242
+ * write — is rethrown as it arrives: it is the same failure `writeLock` would have raised seconds
243
+ * later, and inventing a code for it here would be a second answer to one condition.
244
+ */
245
+ function claimExclusive(path: string, lock: DevLock): boolean {
246
+ let fd: number;
247
+ try {
248
+ fd = openSync(path, 'wx');
249
+ } catch (error) {
250
+ if ((error as { code?: string }).code === 'EEXIST') return false;
251
+ throw error;
252
+ }
253
+ try {
254
+ writeFileSync(fd, `${JSON.stringify(lock, null, 2)}\n`);
255
+ } finally {
256
+ closeSync(fd);
257
+ }
258
+ return true;
259
+ }
260
+
261
+ /** Whatever is on disk right now, or `null` if it is absent or half-written. */
262
+ const readLock = async (path: string): Promise<DevLock | null> => {
263
+ const file = Bun.file(path);
264
+ if (!(await file.exists())) return null;
265
+ return parseLock(await file.text());
266
+ };
267
+
210
268
  export interface PreflightInput {
211
269
  readonly stateDir: string;
212
270
  readonly port: number;
@@ -220,22 +278,53 @@ export interface PreflightInput {
220
278
  readonly holder?: (port: number) => Promise<PortHolder>;
221
279
  }
222
280
 
281
+ export interface PreflightResult {
282
+ readonly clearedStale: boolean;
283
+ /**
284
+ * Give the directory back. The caller runs it when the boot it was preflighting FAILED —
285
+ * `serve.ts`'s `releaseBoot` shape — because a claim held by a process that gave up refuses
286
+ * every later boot in that shell for a pid that is gone.
287
+ */
288
+ release(): void;
289
+ }
290
+
223
291
  /**
224
- * Run before anything boots. Throws the coded refusal, or returns the stale lock it cleared so the
225
- * caller can say so — a lock left by a hard kill is normal and worth one line, not a failure.
292
+ * Run before anything boots, and it CLAIMS: it returns holding the directory, never having merely
293
+ * looked at it.
294
+ *
295
+ * That is the fix, not a detail. `preflight` read the lock, `startDev` booted for seconds, and
296
+ * `writeLock` ran last — so two `x dev` in one checkout both passed the check and both opened
297
+ * `.x/pgdata`, `X_DEV_ALREADY_RUNNING` was unreachable, and what the operator actually got was
298
+ * `X_DB_UNAVAILABLE` whose `fix:` reads "run `x dev`" — the incident this file's own header
299
+ * records. The claim closes the window it names.
300
+ *
301
+ * The stale path still exists and is still normal (a hard kill leaves a lock behind): the file is
302
+ * removed and the claim retried once. A retry that ALSO loses is another boot that claimed the
303
+ * cleared slot in that instant, which is a refusal and not a third boot.
226
304
  */
227
- export const preflight = async (input: PreflightInput): Promise<{ clearedStale: boolean }> => {
305
+ export const preflight = async (input: PreflightInput): Promise<PreflightResult> => {
228
306
  const alive = input.alive ?? isProcessAlive;
229
307
  const bound = input.portBound ?? isPortBound;
230
308
  const path = lockPath(input.stateDir);
231
- const file = Bun.file(path);
309
+ const release = (): void => clearLock(input.stateDir);
310
+ // The lock lives inside the state directory, which the database has not created yet — this runs
311
+ // before anything boots, which is the whole point of it.
312
+ mkdirSync(input.stateDir, { recursive: true });
313
+ const mine: DevLock = {
314
+ pid: process.pid,
315
+ port: input.port,
316
+ // Refined by `writeLock` once the server reports the address it really bound; until then this
317
+ // is what the refusal prints, and it is the address the boot is about to ask for.
318
+ url: `http://${input.hostname}:${input.port}`,
319
+ startedAt: new Date().toISOString(),
320
+ };
232
321
  let clearedStale = false;
233
322
 
234
- if (await file.exists()) {
235
- const lock = parseLock(await file.text());
236
- if (lock !== null && alive(lock.pid)) {
323
+ if (!claimExclusive(path, mine)) {
324
+ const held = await readLock(path);
325
+ if (held !== null && alive(held.pid)) {
237
326
  throw new DevAlreadyRunningError({
238
- lock,
327
+ lock: held,
239
328
  stateDir: input.stateDir,
240
329
  ...(input.embeddedDb === undefined ? {} : { embeddedDb: input.embeddedDb }),
241
330
  });
@@ -245,22 +334,45 @@ export const preflight = async (input: PreflightInput): Promise<{ clearedStale:
245
334
  try {
246
335
  unlinkSync(path);
247
336
  } catch {
248
- // Already gone, or not ours to remove. Either way the boot below is what decides.
337
+ // Already gone, or not ours to remove. The claim below is what decides.
249
338
  }
250
339
  clearedStale = true;
340
+ if (!claimExclusive(path, mine)) {
341
+ // Lost the race for the slot we just cleared. `held` is the best identity available — a
342
+ // half-written file parses as `null` for microseconds — and refusing on a stale pid beats
343
+ // the alternative, which is two processes writing one single-writer data directory.
344
+ //
345
+ // NEVER `mine`, which is what it fell back to: with an unparseable lock and a failed
346
+ // `unlinkSync`, both reads answer `null` and the refusal named THIS pid as the holder —
347
+ // `kill <self>` as the remedy for a file nobody could read.
348
+ const holder = (await readLock(path)) ?? held;
349
+ if (holder === null) throw new DevLockUnreadableError({ path, stateDir: input.stateDir });
350
+ throw new DevAlreadyRunningError({
351
+ lock: holder,
352
+ stateDir: input.stateDir,
353
+ ...(input.embeddedDb === undefined ? {} : { embeddedDb: input.embeddedDb }),
354
+ });
355
+ }
251
356
  }
252
357
 
253
358
  if (bound(input.port, input.hostname)) {
359
+ // Released before the throw: the remedy this refusal prints is `x dev --port <n>`, and a claim
360
+ // left behind would answer that command with X_DEV_ALREADY_RUNNING naming the process that
361
+ // just exited on it.
362
+ release();
254
363
  throw new DevPortInUseError({
255
364
  port: input.port,
256
365
  suggestion: suggestPort(input.port),
257
366
  holder: await (input.holder ?? portHolder)(input.port),
258
367
  });
259
368
  }
260
- return { clearedStale };
369
+ return { clearedStale, release };
261
370
  };
262
371
 
263
- /** Record this process. Written after the preflight passes and before the roles start. */
372
+ /**
373
+ * Record this process. The claim is already on disk — `preflight` took it — so this REFINES it
374
+ * with the address the server actually bound, which is the one field the preflight could not know.
375
+ */
264
376
  export const writeLock = async (stateDir: string, lock: DevLock): Promise<void> => {
265
377
  await Bun.write(lockPath(stateDir), `${JSON.stringify(lock, null, 2)}\n`);
266
378
  };
package/src/dev-queue.ts CHANGED
@@ -20,6 +20,7 @@ import {
20
20
  setDbClient,
21
21
  } from '@ultimat3/db';
22
22
  import type { Tx } from '@ultimat3/entity';
23
+ import { SQL_RATE_LIMIT_TABLE } from '@ultimat3/http';
23
24
  import type { EventBus, JobDriver, OutboxStore, PgExecutor } from '@ultimat3/jobs';
24
25
  import {
25
26
  createJobsFacade,
@@ -82,16 +83,20 @@ export function pgExecutorFor(client: DbClient): PgExecutor {
82
83
  * Every table this process's framework packages own, applied before anything reads one.
83
84
  *
84
85
  * PGlite speaks the extended protocol, which carries one statement per round trip, so the DDL is
85
- * applied statement by statement. Safe to split on `;`: both constants are fixed, with no
86
- * semicolon inside a literal, and each package's own SQL test is where that stays true.
86
+ * applied statement by statement. Safe to split on `;`: every constant is fixed, with no semicolon
87
+ * inside a literal, and each package's own SQL test is where that stays true.
87
88
  *
88
- * `SQL_IDEMPOTENCY_TABLE` is here and not in `@ultimat3/action` because a package that holds no
89
- * database dependency cannot apply its own schema — the same reason `SQL_JOBS_TABLE` is applied
90
- * here. Without it `postgresIdempotencyStore` is a store whose first reservation fails on a
91
- * missing relation, which is how a retried `POST /api/payments/charge` charges a card twice.
89
+ * `SQL_IDEMPOTENCY_TABLE` and `SQL_RATE_LIMIT_TABLE` are here and not in `@ultimat3/action` or
90
+ * `@ultimat3/http` because a package that holds no database dependency cannot apply its own schema
91
+ * — the same reason `SQL_JOBS_TABLE` is applied here. Each one absent is the same failure at a
92
+ * different door: a retried `POST /api/payments/charge` charges the card twice, and the FIRST
93
+ * request a `rateLimitStore` deployment serves dies on a missing `x_rate_limit` relation. The
94
+ * table is installed whether or not this boot passes `runtime.rateLimitStore` — `create table if
95
+ * not exists` on an unused table costs one round trip at boot, and a store installed later must
96
+ * not be the thing that discovers the schema was never applied.
92
97
  */
93
98
  async function applySchema(client: DevDbClient): Promise<void> {
94
- for (const ddl of [SQL_JOBS_TABLE, SQL_IDEMPOTENCY_TABLE]) {
99
+ for (const ddl of [SQL_JOBS_TABLE, SQL_IDEMPOTENCY_TABLE, SQL_RATE_LIMIT_TABLE]) {
95
100
  for (const statement of ddl.split(';')) {
96
101
  if (statement.trim().length > 0) await client.execute(raw(statement));
97
102
  }
@@ -4,13 +4,9 @@
4
4
  // roles reading the other end of that transport.
5
5
 
6
6
  import { describeEntities } from '@ultimat3/entity';
7
- import type { Replicator, Transport } from '@ultimat3/realtime';
8
- import {
9
- createReplicator,
10
- ReplicatorSlotHeldError,
11
- replicatorLockKey,
12
- selectChangeFeed,
13
- } from '@ultimat3/realtime';
7
+ import { ReplicatorSlotHeldError } from '@ultimat3/realtime';
8
+ import type { Replicator, Transport } from '@ultimat3/realtime/server';
9
+ import { createReplicator, replicatorLockKey, selectChangeFeed } from '@ultimat3/realtime/server';
14
10
  import type { DevServices, Env } from './dev-services';
15
11
  import { BadFlagError } from './errors';
16
12
 
@@ -16,7 +16,7 @@ import {
16
16
  resetTasks,
17
17
  } from '@ultimat3/jobs';
18
18
  import { createMemoryDriver as createMemoryMailDriver } from '@ultimat3/mail';
19
- import { DEFAULT_PRESENCE_TTL_MS, InProcessTransport } from '@ultimat3/realtime';
19
+ import { DEFAULT_PRESENCE_TTL_MS, InProcessTransport } from '@ultimat3/realtime/server';
20
20
  import { defineStorage, localDriver } from '@ultimat3/storage';
21
21
  import type { RunningServices } from './dev-runtime';
22
22
  import { resolveServices } from './dev-services';
package/src/dev-roles.ts CHANGED
@@ -8,7 +8,7 @@
8
8
 
9
9
  import type { Role } from '@ultimat3/core';
10
10
  import { createContext, isRole, logger, ROLES } from '@ultimat3/core';
11
- import type { Route, ServerHandle, ServerHooks } from '@ultimat3/http';
11
+ import type { RateLimitStore, Route, ServerHandle, ServerHooks } from '@ultimat3/http';
12
12
  import { configuredAuthenticator, createServer, defineHttpConfig } from '@ultimat3/http';
13
13
  import type { OutboxRelay, Scheduler, Worker } from '@ultimat3/jobs';
14
14
  import {
@@ -195,11 +195,43 @@ export function trustedHopsFromEnv(env: Env): number | null {
195
195
  return hops;
196
196
  }
197
197
 
198
+ /**
199
+ * A deployment that substituted a PER-PROCESS store is enforcing every declared number once per
200
+ * replica, and the shipped chart runs three. Not refused — `assertRateLimitScope` only fires on a
201
+ * `'shared'` declaration, and the scope below is DERIVED from the store, so the two can never
202
+ * contradict each other — but the multiplier is stated, in the `warnIfUnauthenticatable` shape:
203
+ * loud, coded, and naming the call that fixes it.
204
+ *
205
+ * Only for a store the host SUPPLIED. A boot with no store at all resolved one of its own
206
+ * (`startServices`), so the remaining silent case is a hand-built `RunningServices` in a test.
207
+ */
208
+ function warnIfProcessScoped(store: RateLimitStore): void {
209
+ if (store.scope === 'shared') return;
210
+ logger.warn(
211
+ `X_CONFIG_INVALID: the rate-limit store this deployment passed keeps its counters per process, so every limit the app declares is enforced once per replica — docker/helm/values.yaml runs roles.web.replicas: 3, which is 3x every number — fix: drop runtime.rateLimitStore and the boot installs postgresRateLimitStore({ executor }) on the pool it already opened`,
212
+ );
213
+ }
214
+
215
+ /**
216
+ * Where this web role's limiter keeps its counters: what the deployment SUBSTITUTED, else the
217
+ * shared Postgres store `startServices` resolved over the pool this boot already opened.
218
+ *
219
+ * One expression, one answer, in the order `RuntimeOverrides` documents — an override REPLACES the
220
+ * resolved default rather than sitting beside it. `undefined` is reachable only from a hand-built
221
+ * runtime, which is `createServer`'s per-process memory store and `scope: 'process'` below.
222
+ */
223
+ function rateLimitStoreFor(options: StartRolesOptions): RateLimitStore | undefined {
224
+ const supplied = options.overrides?.rateLimitStore;
225
+ if (supplied === undefined) return options.runtime.rateLimitStore;
226
+ warnIfProcessScoped(supplied);
227
+ return supplied;
228
+ }
229
+
198
230
  function startWeb(options: StartRolesOptions): ServerHandle {
199
231
  warnIfUnauthenticatable(options.routes);
200
232
  const binding = options.http ?? DEV_BINDING;
201
233
  const hops = trustedHopsFromEnv(options.env);
202
- const store = options.overrides?.rateLimitStore;
234
+ const store = rateLimitStoreFor(options);
203
235
  return createServer({
204
236
  routes: options.routes,
205
237
  role: 'web',
@@ -218,12 +250,12 @@ function startWeb(options: StartRolesOptions): ServerHandle {
218
250
  signInPath: options.signInPath ?? null,
219
251
  // One declaration, never half of one: `defineHttpConfig` refuses `trustProxy` without hops.
220
252
  ...(hops === null ? {} : { trustProxy: true, trustedProxyHops: hops }),
221
- // `scope` is mandatory since @ultimat3/http made an undeclared limiter a boot error: the
222
- // old `'process'` default meant the shipped chart's three `web` replicas enforced
223
- // `login: { limit: 5 }` as fifteen attempts, with `x verify` green. It is DERIVED from the
224
- // store rather than hardcoded — a deployment that hands `runtime.rateLimitStore` a shared
225
- // store is declaring the fleet-wide numbers, and `assertRateLimitScope` then holds the two
226
- // halves together instead of a literal here quietly contradicting the store beside it.
253
+ // `scope` is mandatory since @ultimat3/http made an undeclared limiter a boot error, and it
254
+ // is DERIVED from the store rather than hardcoded — a literal here would be a second
255
+ // declaration quietly contradicting the object beside it, and `assertRateLimitScope` holds
256
+ // the two halves together. It answered `'process'` on every real boot until `startServices`
257
+ // resolved a store, so the shipped chart's three `web` replicas enforced `login: { limit: 5 }`
258
+ // as fifteen attempts, with `x verify` green.
227
259
  rateLimit: { scope: store?.scope ?? 'process' },
228
260
  // Hashes, never `'unsafe-inline'`: a `render: 'static'` page is a file on disk, so
229
261
  // nothing can stamp a per-response nonce into it, but its body is fixed and a hash is a
@@ -6,7 +6,14 @@
6
6
  import { mkdirSync } from 'node:fs';
7
7
  import type { PurgeDriver } from '@ultimat3/cache';
8
8
  import { isNoopPurgeDriver, selectPurgeDriver } from '@ultimat3/cache';
9
- import { isLocal, renderThrowable, resolveEnvironment } from '@ultimat3/core';
9
+ import {
10
+ isLocal,
11
+ registerReadinessCheck,
12
+ renderThrowable,
13
+ resolveEnvironment,
14
+ } from '@ultimat3/core';
15
+ import type { RateLimitStore } from '@ultimat3/http';
16
+ import { postgresRateLimitStore } from '@ultimat3/http';
10
17
  import type { EventBus, JobDriver, OutboxStore } from '@ultimat3/jobs';
11
18
  import type { MailDriver } from '@ultimat3/mail';
12
19
  import {
@@ -16,13 +23,13 @@ import {
16
23
  selectMailDriver,
17
24
  setMailDriver,
18
25
  } from '@ultimat3/mail';
19
- import type { Transport, TransportSelection } from '@ultimat3/realtime';
20
- import { selectTransport } from '@ultimat3/realtime';
26
+ import type { Transport, TransportSelection } from '@ultimat3/realtime/server';
27
+ import { selectTransport } from '@ultimat3/realtime/server';
21
28
  import type { Storage } from '@ultimat3/storage';
22
29
  import { defineStorage, localDriver, s3Driver, usesDevStorageSecret } from '@ultimat3/storage';
23
30
  import { startCacheTiers } from './dev-cache';
24
31
  import type { DevDbClient } from './dev-queue';
25
- import { startQueue } from './dev-queue';
32
+ import { pgExecutorFor, startQueue } from './dev-queue';
26
33
  import type { DevServices, Env } from './dev-services';
27
34
  import { LocalDiskUnsafeError, StorageUnwritableError } from './errors';
28
35
  import { msg } from './messages';
@@ -51,6 +58,20 @@ export interface RunningServices {
51
58
  * would show members leaving that never left.
52
59
  */
53
60
  readonly presenceTtlMs: number;
61
+ /**
62
+ * Where the HTTP rate limiter keeps its counters, over the pool this boot already opened.
63
+ *
64
+ * Resolved HERE and not at `startWeb` for the reason every other driver is: `startServices` is
65
+ * what holds the connection, and a store built anywhere else would open a second pool against a
66
+ * url this boot resolved once. Until it did, no boot installed one at all — so `rateLimit.scope`
67
+ * derived to `'process'` on every deployment the framework produces, while `docker/helm` runs
68
+ * `roles.web.replicas: 3` and `x new` scaffolds two.
69
+ *
70
+ * OPTIONAL because a `RunningServices` can be hand-built: a test's fixture runtime has a stub
71
+ * client and no shared store, which is `createServer`'s memory store and `scope: 'process'` —
72
+ * the honest answer for it, and the one `startWeb` derives.
73
+ */
74
+ readonly rateLimitStore?: RateLimitStore;
54
75
  readonly purge: PurgeDriver;
55
76
  /** Same rule as `mailDetail`: the env key that selected the CDN, never the token behind it. */
56
77
  readonly purgeDetail: string;
@@ -176,6 +197,59 @@ export function startStorage(services: DevServices, env: Env, override?: Storage
176
197
  return defineStorage({ disks: { local: localDriver({ root }) }, default: 'local' });
177
198
  }
178
199
 
200
+ /**
201
+ * A readiness check over a dependency that can only be asked asynchronously.
202
+ *
203
+ * `ReadinessCheck` is synchronous and that is the mechanism, not a limitation: a probe that awaits
204
+ * a network call takes as long as the dependency does, so a slow database makes `/readyz` miss its
205
+ * `timeoutSeconds`, the kubelet reads that as unready, and capacity is pulled from an already
206
+ * struggling system — the outage the probe existed to prevent, caused by the probe.
207
+ *
208
+ * So the read answers the PREVIOUS probe and schedules the next, which is the standard cached
209
+ * health shape: staleness is one poll period, and the poll period is the operator's own
210
+ * `periodSeconds`. Deliberately not a `setInterval`: a timer would probe a process nobody is
211
+ * asking about, and in `x dev` every probe is a statement — one span, one trace id — so a
212
+ * one-per-second heartbeat would evict every real request from `/_x/timeline` inside a minute.
213
+ *
214
+ * It starts `true` because it is already proven: `startQueue` pinged this pool before this line
215
+ * ran, and a boot that could not would have thrown instead of reaching here.
216
+ */
217
+ function probedReadinessCheck(name: string, probe: () => Promise<unknown>): () => void {
218
+ let live = true;
219
+ let inFlight = false;
220
+ const refresh = (): void => {
221
+ if (inFlight) return;
222
+ inFlight = true;
223
+ void probe()
224
+ .then(
225
+ () => {
226
+ live = true;
227
+ },
228
+ // Every rejection, including the coded `X_DB_UNAVAILABLE` a closed pool answers with. A
229
+ // check that threw would be reported as failing anyway; catching it here keeps the reason
230
+ // out of the endpoint's own path, where nothing could report it.
231
+ () => {
232
+ live = false;
233
+ },
234
+ )
235
+ .finally(() => {
236
+ inFlight = false;
237
+ });
238
+ };
239
+ return registerReadinessCheck(name, () => {
240
+ refresh();
241
+ return live;
242
+ });
243
+ }
244
+
245
+ /** A transport that says whether it is connected. NATS does; the in-process bus has nothing to be. */
246
+ interface ConnectableTransport {
247
+ readonly connected: boolean;
248
+ }
249
+
250
+ const saysConnected = (transport: Transport): transport is Transport & ConnectableTransport =>
251
+ typeof (transport as Partial<ConnectableTransport>).connected === 'boolean';
252
+
179
253
  /**
180
254
  * Release what has already started, newest first, and return every failure instead of throwing on
181
255
  * the first: a step that rejects must not skip the ones after it, or one transport that will not
@@ -217,6 +291,12 @@ export async function startServices(
217
291
  // Boot is a sequence of external resources, and every step after the first can reject — the
218
292
  // queue is already up, so from here an unwind must release it exactly like everything after it.
219
293
  const started: (() => void | Promise<void>)[] = [() => queue.stop()];
294
+ // One readiness check per resource this boot OWNS, released with it. Nothing in the tree
295
+ // registered one, so `/readyz` was `markReady()` alone — "this process bound a socket" — and
296
+ // `packages/http/src/server.ts` calls that BEFORE `Bun.serve`, while `dev-roles.ts` calls it
297
+ // before `sync`, `worker` and `scheduler` start. The chart's `readinessProbe` and the container
298
+ // healthcheck both route on it, so a replica whose pool was gone kept taking traffic.
299
+ started.push(probedReadinessCheck('database', () => db.ping()));
220
300
  try {
221
301
  // Dialled here rather than at selection: an unreachable bus must fail at `x dev`, not on the
222
302
  // first change nobody receives, and the socket is a resource the unwind below has to release.
@@ -228,6 +308,14 @@ export async function startServices(
228
308
  started.push(() => bus.transport.close());
229
309
  transport = bus.transport;
230
310
  }
311
+ // Only a transport that can be disconnected gets a check. `NatsTransport.connected` is a
312
+ // synchronous getter over the client's own state, so no probe is needed; the in-process bus
313
+ // has nothing to lose a connection to, and a check that can only answer `true` is a number in
314
+ // `registered` that means nothing.
315
+ if (saysConnected(transport)) {
316
+ const connectable = transport;
317
+ started.push(registerReadinessCheck('transport', () => connectable.connected));
318
+ }
231
319
  const storage = startStorage(services, env, overrides?.storage);
232
320
  // With no credential this is the memory driver: caught, not sent, so the `/_x` mail panel can
233
321
  // show what a template renders in every locale without a mailbox or a message escaping to a
@@ -257,6 +345,10 @@ export async function startServices(
257
345
  mailDetail: selection.detail,
258
346
  // The env key that selected the bus — or the honest answer that no env key did, because a
259
347
  // boot line reading `NATS_URL` over a transport the host handed in is a lie a script parses.
348
+ // The same executor the jobs driver, the outbox, the event bus and the idempotency store
349
+ // run on — one pool, one `Bun.sql` that does NOT satisfy `PgExecutor` (`Bun.sql.query` is
350
+ // `undefined`), one wrapper. `startWeb` is what an override replaces it at.
351
+ rateLimitStore: postgresRateLimitStore({ executor: pgExecutorFor(db) }),
260
352
  transportDetail: overrides?.transport === undefined ? bus.detail : 'runtime override',
261
353
  presenceTtlMs: bus.presenceTtlMs,
262
354
  purge,
package/src/dev-sync.ts CHANGED
@@ -13,7 +13,7 @@ import {
13
13
  PresenceRegistry,
14
14
  RingChangeBuffer,
15
15
  SocketRegistry,
16
- } from '@ultimat3/realtime';
16
+ } from '@ultimat3/realtime/server';
17
17
  import type { StartRolesOptions } from './dev-roles';
18
18
  import { syncAuthenticator } from './sync-authenticator';
19
19
 
@@ -69,9 +69,14 @@ export async function startSync(options: StartRolesOptions): Promise<RunningSync
69
69
  const hub = new ChannelHub({ transport: options.runtime.transport, sockets });
70
70
  // The node evaluated no credential of its own and no host ever handed it one, so every socket
71
71
  // the framework opened was anonymous and every guard, gate, presence entry and tenant cap
72
- // decided against `null`. An explicit override first — only that one can carry an `expiresAt`
73
- // and a `refresh`, which is the whole of re-authorization — then the app's own HTTP resolver,
74
- // then nothing at all, which is what `x dev` with no authenticator should stay.
72
+ // decided against `null`. An explicit override first, then the app's own HTTP resolver, then
73
+ // nothing at all, which is what `x dev` with no authenticator should stay.
74
+ //
75
+ // BOTH of the first two re-authorize: `syncAuthenticator` carries an `expiresAt` and a `refresh`
76
+ // of its own (`SYNC_GRANT_TTL_MS`), re-asking the app's resolver with the upgrade's own
77
+ // `cookie`/`authorization`, so `logout` closes the socket and not only the HTTP session. The
78
+ // override is how a deployment states a window its credential already declares (a token's
79
+ // `exp`), or resolves identity from a header the adapter deliberately does not retain.
75
80
  const authenticate = options.overrides?.syncAuthenticate ?? syncAuthenticator(options.buildId);
76
81
  const node = createSyncNode({
77
82
  hub,
package/src/dispatch.ts CHANGED
@@ -3,6 +3,7 @@
3
3
  // path as results, so a failure is machine-readable exactly like a success.
4
4
 
5
5
  import { isAbsolute, resolve } from 'node:path';
6
+ import { setLogStream } from '@ultimat3/core';
6
7
  import { requireAppRoot, requireBunVersion } from './app-root';
7
8
  import { createHelpCommand } from './cmd-help';
8
9
  import { plannedCommandFor } from './cmd-planned';
@@ -23,8 +24,26 @@ export interface DispatchOptions {
23
24
  readonly bunVersion: string;
24
25
  readonly runner?: Runner;
25
26
  readonly write: (line: string) => void;
27
+ /**
28
+ * Where a result declaring `stream: 'stderr'` goes. Optional, and it falls back to `write`: an
29
+ * embedding caller that supplies one sink must still SEE the line, and `create-ultimate`'s entry
30
+ * point — which can only ever reach `x new` — would otherwise have to declare a sink for a case
31
+ * it cannot produce. `bin.ts` passes the real fd 2.
32
+ */
33
+ readonly writeError?: (line: string) => void;
26
34
  }
27
35
 
36
+ /**
37
+ * Which sink a result is written to. One function because `dispatch` writes in five places — the
38
+ * version refusal, the parse failure, the unknown command, the result and the catch — and a rule
39
+ * about fd 1 that four of them apply is not a rule.
40
+ */
41
+ export const sinkFor = (
42
+ result: CommandResult,
43
+ options: Pick<DispatchOptions, 'write' | 'writeError'>,
44
+ ): ((line: string) => void) =>
45
+ result.stream === 'stderr' ? (options.writeError ?? options.write) : options.write;
46
+
28
47
  const resolveCwd = (cwd: string, flag: string | undefined): string => {
29
48
  if (flag === undefined) return cwd;
30
49
  return isAbsolute(flag) ? flag : resolve(cwd, flag);
@@ -48,7 +67,8 @@ export async function dispatch(options: DispatchOptions): Promise<number> {
48
67
  try {
49
68
  requireBunVersion(options.bunVersion);
50
69
  } catch (error) {
51
- options.write(render(errorResult('x', error), wantsJson(options.argv)));
70
+ const failure = errorResult('x', error);
71
+ sinkFor(failure, options)(render(failure, wantsJson(options.argv)));
52
72
  return 1;
53
73
  }
54
74
 
@@ -70,7 +90,7 @@ export async function dispatch(options: DispatchOptions): Promise<number> {
70
90
  // `wantsJson`, not `includes('--json')`: a typo'd flag or a typo'd command is exactly the case
71
91
  // an agent hits while always passing `-j`, and the short form rendered prose it then parsed.
72
92
  const result = errorResult(planned?.name ?? 'x', failure);
73
- options.write(render(result, wantsJson(options.argv)));
93
+ sinkFor(result, options)(render(result, wantsJson(options.argv)));
74
94
  return 1;
75
95
  }
76
96
 
@@ -83,7 +103,7 @@ export async function dispatch(options: DispatchOptions): Promise<number> {
83
103
  known: SPECS.map((spec) => spec.name),
84
104
  }),
85
105
  );
86
- options.write(render(result, args.json));
106
+ sinkFor(result, options)(render(result, args.json));
87
107
  return 1;
88
108
  }
89
109
 
@@ -105,6 +125,16 @@ export async function dispatch(options: DispatchOptions): Promise<number> {
105
125
  };
106
126
 
107
127
  try {
128
+ // `--json` means fd 1 carries ONE document, and every command that boots an app logs through
129
+ // core's module-scope `logger`, whose default writer is stdout: `x db migrate --json` printed
130
+ // `ultimate migrate applied` and then the command's own object, so `json.load` on the output
131
+ // of a command whose whole contract is `--json` raised on the second. Decided HERE, once, for
132
+ // all thirty commands — a rule each booting command had to remember is a rule the next one
133
+ // forgets. A server's stdout stays its log stream; this is about the CLI process only.
134
+ // Set on EVERY dispatch, both ways: the stream is process-wide state, so `if (args.json)`
135
+ // alone left a JSON run's `stderr` in place for the next non-JSON one — one `x` process
136
+ // dispatching twice (the MCP host, a test, an embedding caller) lost its boot logs off stdout.
137
+ setLogStream(args.json ? 'stderr' : 'stdout');
108
138
  // The reader `CommandSpec.requiresApp` never had. Its doc said "the dispatcher enforces it" and
109
139
  // `dispatch` did not read the field at all: the guarantee held only because all 17 declaring
110
140
  // commands happen to call `requireAppRoot` themselves, so a new command that declares it and
@@ -115,7 +145,7 @@ export async function dispatch(options: DispatchOptions): Promise<number> {
115
145
  // command, which declares nothing: usage for a command must be readable from anywhere.
116
146
  if (target.spec.requiresApp === true) requireAppRoot(target.spec.name, ctx.cwd);
117
147
  const result = await target.run(ctx);
118
- options.write(render(result, args.json, args.flags.get('verbose') === true));
148
+ sinkFor(result, options)(render(result, args.json, args.flags.get('verbose') === true));
119
149
  // `x dev` and `x mcp serve --transport http` are still listening here: report first, so the
120
150
  // url is on stdout the moment it is reachable, then stay in the process until the drain that
121
151
  // stops them. Without this the exit code below is what takes the server down.
@@ -123,7 +153,7 @@ export async function dispatch(options: DispatchOptions): Promise<number> {
123
153
  return exitCodeFor(result);
124
154
  } catch (error) {
125
155
  const result = errorResult(args.command, error);
126
- options.write(render(result, args.json));
156
+ sinkFor(result, options)(render(result, args.json));
127
157
  return 1;
128
158
  }
129
159
  }