@tatara/cli 0.10.0 → 0.11.0

package/dist/bin.mjs

@@ -6,12 +6,12 @@ import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/pr
 import { dirname, join, relative, resolve, sep } from "node:path";
 import { glob } from "tinyglobby";
 import { existsSync, mkdirSync, readFileSync, readdirSync, watch } from "node:fs";
+import { homedir, tmpdir } from "node:os";
 import { x } from "tinyexec";
 import { parseEnv } from "node:util";
 import * as p from "@clack/prompts";
 import { createHash, randomBytes } from "node:crypto";
 import { stringify } from "yaml";
-import { homedir, tmpdir } from "node:os";
 //#region \0rolldown/runtime.js
 var __defProp = Object.defineProperty;
 var __exportAll = (all, no_symbols) => {
@@ -190,6 +190,61 @@ function ignored(v, text) {
 	return new RegExp(`tatara-check-ignore\\s+${v.rule}:\\s*\\S`).test(previous);
 }
 //#endregion
+//#region src/paths.ts
+/** Package root: one level above both src/ (dev) and dist/ (published, bundled). */
+const PACKAGE_ROOT = resolve(import.meta.dirname, "..");
+const TEMPLATES_DIR = resolve(PACKAGE_ROOT, "templates");
+function cliVersion() {
+	return JSON.parse(readFileSync(resolve(PACKAGE_ROOT, "package.json"), "utf8")).version;
+}
+let versions;
+/** Exact versions used in generated apps (templates/versions.json). */
+function templateVersions() {
+	versions ??= JSON.parse(readFileSync(resolve(TEMPLATES_DIR, "versions.json"), "utf8"));
+	return versions;
+}
+/** `TATARA_PACKAGE_SPECS='{"@tatara/core":"file:…"}'` installs local builds (e2e, bench). */
+function packageSpecOverride(pkg) {
+	const raw = process.env.TATARA_PACKAGE_SPECS;
+	if (!raw) return void 0;
+	return JSON.parse(raw)[pkg];
+}
+function versionOf(pkg) {
+	const override = packageSpecOverride(pkg);
+	if (override) return override;
+	if (pkg === "@tatara/cli") return process.env.TATARA_CLI_SPEC ?? `^${cliVersion()}`;
+	const version = templateVersions()[pkg];
+	if (!version) throw new Error(`templates/versions.json has no version for "${pkg}"`);
+	return pkg.startsWith("@tatara/") ? `^${version}` : version;
+}
+//#endregion
+//#region src/pm.ts
+function pmCommands(pm) {
+	switch (pm) {
+		case "pnpm": return {
+			install: ["pnpm", "install"],
+			run: "pnpm",
+			exec: "pnpm exec",
+			dlx: ["pnpm", "dlx"],
+			lockfile: "pnpm-lock.yaml"
+		};
+		case "npm": return {
+			install: ["npm", "install"],
+			run: "npm run",
+			exec: "npx",
+			dlx: ["npx", "-y"],
+			lockfile: "package-lock.json"
+		};
+		case "bun": return {
+			install: ["bun", "install"],
+			run: "bun run",
+			exec: "bunx",
+			dlx: ["bunx"],
+			lockfile: "bun.lock"
+		};
+	}
+}
+//#endregion
 //#region src/project/ejected.ts
 /** Packages the app ejected (`tatara eject`); their code lives in src/ejected/<name>. */
 function ejectedPackages(config) {
@@ -219,7 +274,10 @@ function identifier(path, prefix) {
 		"consumers",
 		"schedules",
 		"permissions",
-		"db"
+		"db",
+		"emails",
+		"notifications",
+		"channels"
 	].includes(p)).join("_")}`.replace(/[^A-Za-z0-9_]/g, "_").replace(/_+/g, "_");
 }
 async function find(root, patterns) {
@@ -239,6 +297,26 @@ ${imports.join("\n")}${imports.length ? "\n" : ""}
 export const ${options.exportName}: ${options.typeName}[] = [${names.join(", ")}];
 `;
 }
+/** The consumers keep their literal topics and schemas, so `TopicsOf<typeof consumers>` types enqueue. */
+function consumerList(files) {
+	const imports = files.map((f) => `import ${identifier(f, "consumer")} from "${specifier(f)}";`);
+	const names = files.map((f) => identifier(f, "consumer"));
+	return `${HEADER}import type { ConsumerDefinition } from "@tatara/jobs";
+${imports.join("\n")}${imports.length ? "\n" : ""}
+// biome-ignore lint/suspicious/noExplicitAny: every consumer, whatever its payload
+export const consumers = [${names.join(", ")}] as const satisfies readonly ConsumerDefinition<string, any>[];
+`;
+}
+/** A convention folder's default exports, with their literal names kept for typed lookups. */
+function definitionList(files, prefix, pkg, typeName, exportName) {
+	const imports = files.map((f) => `import ${identifier(f, prefix)} from "${specifier(f)}";`);
+	const names = files.map((f) => identifier(f, prefix));
+	return `${HEADER}import type { ${typeName} } from "${pkg}";
+${imports.join("\n")}${imports.length ? "\n" : ""}
+// biome-ignore lint/suspicious/noExplicitAny: every definition, whatever its data
+export const ${exportName} = [${names.join(", ")}] as const satisfies readonly ${typeName}<string, any>[];
+`;
+}
 /** What the app's installed (or ejected) @tatara/* packages contribute, by package name. */
 async function contributions(project, kind) {
 	const pkg = JSON.parse(await readFile(join(project.root, "package.json"), "utf8"));
@@ -318,12 +396,7 @@ async function sync(project = loadProject()) {
 	}), written);
 	if (project.has("jobs")) {
 		const consumers = await find(root, dirs.map((d) => `${d}/consumers/*.ts`));
-		await write(join(out, "consumers.ts"), listModule(consumers, {
-			prefix: "consumer",
-			typeImport: "import type { ConsumerDefinition } from \"@tatara/jobs\";",
-			typeName: "ConsumerDefinition",
-			exportName: "consumers"
-		}), written);
+		await write(join(out, "consumers.ts"), consumerList(consumers), written);
 		const schedules = await find(root, dirs.map((d) => `${d}/schedules/*.ts`));
 		await write(join(out, "schedules.ts"), listModule(schedules, {
 			prefix: "schedule",
@@ -332,6 +405,18 @@ async function sync(project = loadProject()) {
 			exportName: "schedules"
 		}), written);
 	}
+	if (project.has("email")) {
+		const emails = await find(root, ["src/modules/*/emails/*.ts"]);
+		await write(join(out, "emails.ts"), definitionList(emails, "email", "@tatara/email", "EmailDefinition", "emails"), written);
+	}
+	if (project.has("notifications")) {
+		const notifications = await find(root, ["src/modules/*/notifications/*.ts"]);
+		await write(join(out, "notifications.ts"), definitionList(notifications, "notification", "@tatara/notifications", "NotificationDefinition", "notifications"), written);
+	}
+	if (project.has("realtime")) {
+		const channels = await find(root, ["src/modules/*/channels/*.ts"]);
+		await write(join(out, "channels.ts"), definitionList(channels, "channel", "@tatara/realtime", "ChannelDefinition", "channels"), written);
+	}
 	if (project.has("authz")) {
 		const permissions = await find(root, ["src/modules/*/permissions/*.permissions.ts"]);
 		const imports = permissions.map((f) => `import ${identifier(f, "permissions")} from "${specifier(f)}";`);
@@ -371,6 +456,154 @@ export type PermissionKey =
 	return { written };
 }
 //#endregion
+//#region src/commands/routes.ts
+/** Computes the access label for a route based on its security requirements. */
+function parseAccessLabel(security, _securitySchemes, tags = []) {
+	if (tags.includes("Auth")) return "better-auth";
+	if (!security || security.length === 0) return "public";
+	const labels = [];
+	for (const requirement of security) {
+		const schemes = Object.keys(requirement);
+		if (schemes.length === 0) continue;
+		const schemeName = schemes[0] ?? "";
+		const roles = requirement[schemeName] ?? [];
+		if (schemeName === "bearerAuth" && roles.length > 0) labels.push(roles.join(", "));
+		else if (schemeName === "bearerAuth") labels.push("authenticated");
+		else labels.push(schemeName);
+	}
+	return labels.length > 1 ? labels.join(" OR ") : labels[0] ?? "authenticated";
+}
+/** Every app's OpenAPI document, as served at /openapi.json (with better-auth's merged in). */
+async function loadDocuments(project) {
+	await sync(project);
+	const apps = [["api", "#apps/api/app"], ...project.has("admin") ? [["admin", "#apps/admin/app"]] : []];
+	const code = `
+const out = {};
+for (const [name, spec] of ${JSON.stringify(apps)}) {
+	const { app } = await import(spec);
+	out[name] = await (await app.request("/openapi.json")).json();
+}
+// exit only once a large document has gone through the pipe
+process.stdout.write(JSON.stringify(out), () => process.exit(0));
+`;
+	return new Promise((resolve, reject) => {
+		const child = spawn(process.execPath, [
+			...NODE_FLAGS,
+			"--input-type=module",
+			"-e",
+			code
+		], {
+			cwd: project.root,
+			env: {
+				...process.env,
+				LOG_LEVEL: "silent"
+			}
+		});
+		let stdout = "";
+		let stderr = "";
+		child.stdout.on("data", (d) => {
+			stdout += d.toString();
+		});
+		child.stderr.on("data", (d) => {
+			stderr += d.toString();
+		});
+		child.on("exit", (code) => {
+			if (code !== 0) return reject(/* @__PURE__ */ new Error(`Could not load the apps:\n${stderr}`));
+			resolve(JSON.parse(stdout));
+		});
+	});
+}
+/** Lists every app's routes with what protects them. */
+async function listRoutes(project) {
+	const out = [];
+	for (const [app, doc] of Object.entries(await loadDocuments(project))) {
+		const schemes = doc.components?.securitySchemes ?? {};
+		for (const [path, item] of Object.entries(doc.paths ?? {})) for (const [method, op] of Object.entries(item)) out.push({
+			app,
+			method: method.toUpperCase(),
+			path,
+			summary: op.summary || op.description || "",
+			access: parseAccessLabel(op.security, schemes, op.tags)
+		});
+	}
+	return out;
+}
+function printRoutes(routes) {
+	const rows = routes.map((r) => [
+		r.app,
+		r.method,
+		r.path,
+		r.access,
+		r.summary
+	]);
+	const widths = [
+		0,
+		1,
+		2,
+		3
+	].map((i) => Math.max(...rows.map((r) => (r[i] ?? "").length), 4));
+	for (const r of rows) {
+		const [app = "", method = "", path = "", access = "", summary = ""] = r;
+		console.log(`${pc.dim(app.padEnd(widths[0] ?? 0))}  ${pc.bold(method.padEnd(widths[1] ?? 0))}  ${path.padEnd(widths[2] ?? 0)}  ${(access === "public" ? pc.yellow : pc.green)(access.padEnd(widths[3] ?? 0))}  ${pc.dim(summary)}`);
+	}
+}
+//#endregion
+//#region src/commands/contracts.ts
+async function openapiTypescript(project, input, output) {
+	const [cmd = "npx", ...args] = pmCommands(project.config.packageManager).dlx;
+	const spec = `openapi-typescript@${versionOf("openapi-typescript")}`;
+	const res = await x(cmd, [
+		...args,
+		spec,
+		input,
+		"-o",
+		output
+	], {
+		nodeOptions: { cwd: project.root },
+		throwOnError: false
+	});
+	if (res.exitCode !== 0) throw new Error(`openapi-typescript failed:\n${res.stderr || res.stdout}`);
+}
+/**
+* Writes each app's OpenAPI document and its TypeScript types, for typed clients:
+* `createClient<paths>({ baseUrl })` from openapi-fetch, with `paths` from `<app>.ts`.
+*/
+async function writeContracts(project, options = {}) {
+	const out = options.out ?? "contracts";
+	const generate = options.generateTypes ?? ((input, output) => openapiTypescript(project, input, output));
+	const docs = await loadDocuments(project);
+	const dir = options.check ? await mkdtemp(join(tmpdir(), "tatara-contracts-")) : join(project.root, out);
+	await mkdir(dir, { recursive: true });
+	const written = [];
+	try {
+		for (const [app, doc] of Object.entries(docs)) {
+			const json = join(dir, `${app}.openapi.json`);
+			await writeFile(json, `${JSON.stringify(doc, null, 2)}\n`);
+			await generate(json, join(dir, `${app}.ts`), project.root);
+			written.push(`${out}/${app}.openapi.json`, `${out}/${app}.ts`);
+		}
+		if (!options.check) return {
+			written,
+			stale: []
+		};
+		const stale = [];
+		for (const file of written) {
+			const name = file.slice(out.length + 1);
+			const fresh = await readFile(join(dir, name), "utf8");
+			if (await readFile(join(project.root, file), "utf8").catch(() => void 0) !== fresh) stale.push(file);
+		}
+		return {
+			written: [],
+			stale
+		};
+	} finally {
+		if (options.check) await rm(dir, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+//#endregion
 //#region src/commands/db.ts
 var DbError = class extends Error {};
 function requireDb(project) {
@@ -415,30 +648,6 @@ async function dbStudio(project) {
 	return runBin(project, "prisma", ["studio"]);
 }
 //#endregion
-//#region src/pm.ts
-function pmCommands(pm) {
-	switch (pm) {
-		case "pnpm": return {
-			install: ["pnpm", "install"],
-			run: "pnpm",
-			exec: "pnpm exec",
-			lockfile: "pnpm-lock.yaml"
-		};
-		case "npm": return {
-			install: ["npm", "install"],
-			run: "npm run",
-			exec: "npx",
-			lockfile: "package-lock.json"
-		};
-		case "bun": return {
-			install: ["bun", "install"],
-			run: "bun run",
-			exec: "bunx",
-			lockfile: "bun.lock"
-		};
-	}
-}
-//#endregion
 //#region src/commands/verify.ts
 const VERIFY_STEPS = [
 	"lint",
@@ -519,7 +728,7 @@ function prefixed(child, label) {
 		});
 	}
 }
-const CONVENTION = /\/(routes|commands|consumers|schedules|permissions|db)\/[^/]+\.ts$/;
+const CONVENTION = /\/(routes|commands|consumers|schedules|permissions|db|emails|notifications|channels)\/[^/]+\.ts$/;
 /** Re-runs `tatara sync` when convention files appear or disappear, so --watch picks them up. */
 function watchConventions(project) {
 	const watchers = [];
@@ -831,34 +1040,6 @@ async function eject(project, name, opts = {}) {
 	]) await opts.formatFile(file, project);
 	return report;
 }
-//#endregion
-//#region src/paths.ts
-/** Package root: one level above both src/ (dev) and dist/ (published, bundled). */
-const PACKAGE_ROOT = resolve(import.meta.dirname, "..");
-const TEMPLATES_DIR = resolve(PACKAGE_ROOT, "templates");
-function cliVersion() {
-	return JSON.parse(readFileSync(resolve(PACKAGE_ROOT, "package.json"), "utf8")).version;
-}
-let versions;
-/** Exact versions used in generated apps (templates/versions.json). */
-function templateVersions() {
-	versions ??= JSON.parse(readFileSync(resolve(TEMPLATES_DIR, "versions.json"), "utf8"));
-	return versions;
-}
-/** `TATARA_PACKAGE_SPECS='{"@tatara/core":"file:…"}'` installs local builds (e2e, bench). */
-function packageSpecOverride(pkg) {
-	const raw = process.env.TATARA_PACKAGE_SPECS;
-	if (!raw) return void 0;
-	return JSON.parse(raw)[pkg];
-}
-function versionOf(pkg) {
-	const override = packageSpecOverride(pkg);
-	if (override) return override;
-	if (pkg === "@tatara/cli") return process.env.TATARA_CLI_SPEC ?? `^${cliVersion()}`;
-	const version = templateVersions()[pkg];
-	if (!version) throw new Error(`templates/versions.json has no version for "${pkg}"`);
-	return pkg.startsWith("@tatara/") ? `^${version}` : version;
-}
 const authz = {
 	id: "authz",
 	kind: "addon",
@@ -920,22 +1101,44 @@ const authz = {
 - Effective permissions = role grants (assigned roles + built-in roles whose key matches an
   identity-provider role) + grant overlays − deny overlays. Deny always wins.
 - Custom roles and grant overlays can never grant \`role\`, \`member\` or \`audit_log\`.
-- Built-in roles live in \`src/core/authz/system-roles.ts\`. Run \`__PM_EXEC__ tatara run authz:sync-permissions\`
-  on every deploy (it is idempotent); \`db:seed\` runs it locally.
+- Built-in roles live in \`src/core/authz/system-roles.ts\`. \`authz:sync-permissions\` brings the database
+  in line with them; \`deploy\` runs it every deploy, \`db:seed\` locally.
 - In tests: \`asUser({ permissions: ["project:read"] })\` or \`asUser({ systemRoles: ["admin"] })\`.
 `
 		}
 	]
 };
+const REDIS_SERVICE = {
+	image: "redis:8-alpine",
+	ports: ["${REDIS_PORT:-6379}:6379"],
+	healthcheck: {
+		test: [
+			"CMD",
+			"redis-cli",
+			"ping"
+		],
+		interval: "2s",
+		timeout: "3s",
+		retries: 30
+	}
+};
+const jobsDriver = (drivers) => ({
+	key: "JOBS_DRIVER",
+	schema: `localDefault(z.enum(${JSON.stringify(drivers)}), "bullmq")`,
+	example: "bullmq",
+	test: "inline",
+	comment: `Where jobs wait: ${drivers.join(", ")} (inline runs them where they're sent, no worker)`
+});
 const jobs = {
 	id: "jobs",
 	kind: "addon",
-	label: "Jobs + cron (BullMQ)",
+	label: "Jobs + cron (BullMQ, Postgres or Kafka)",
 	dependencies: [
 		"@tatara/jobs",
 		"bullmq",
 		"ioredis",
-		"croner"
+		"croner",
+		"kafkajs"
 	],
 	allowBuilds: ["msgpackr-extract"],
 	scripts: {
@@ -965,22 +1168,57 @@ const jobs = {
 			key: "WORKER_CONCURRENCY",
 			schema: "z.coerce.number().int().positive().default(5)",
 			example: "5"
+		},
+		{
+			key: "KAFKA_BROKERS",
+			schema: "z.string().optional()",
+			example: "",
+			comment: "Comma-separated brokers, for JOBS_DRIVER=kafka"
+		},
+		{
+			key: "KAFKA_CLIENT_ID",
+			schema: "z.string().default(\"__APP_NAME__\")",
+			example: "__APP_NAME__"
+		},
+		{
+			key: "KAFKA_SSL",
+			schema: "z.stringbool().default(false)",
+			example: "false"
+		},
+		{
+			key: "KAFKA_SASL_USERNAME",
+			schema: "z.string().optional()",
+			example: ""
+		},
+		{
+			key: "KAFKA_SASL_PASSWORD",
+			schema: "z.string().optional()",
+			example: ""
+		},
+		{
+			key: "SCHEDULE_TRIGGER_SECRET",
+			schema: "z.string().min(16).optional()",
+			example: "",
+			comment: "Lets a platform scheduler POST /internal/schedules/<name> with Authorization: Bearer <secret>; unset, the route doesn't exist"
 		}
 	],
-	compose: { redis: {
-		image: "redis:8-alpine",
-		ports: ["${REDIS_PORT:-6379}:6379"],
-		healthcheck: {
-			test: [
-				"CMD",
-				"redis-cli",
-				"ping"
-			],
-			interval: "2s",
-			timeout: "3s",
-			retries: 30
-		}
-	} },
+	variants: [{
+		when: ["postgres"],
+		env: [jobsDriver([
+			"bullmq",
+			"postgres",
+			"kafka",
+			"inline"
+		])]
+	}, {
+		unless: ["postgres"],
+		env: [jobsDriver([
+			"bullmq",
+			"kafka",
+			"inline"
+		])]
+	}],
+	compose: { redis: REDIS_SERVICE },
 	slots: [
 		{
 			file: "tsdown.config.ts",
@@ -988,141 +1226,647 @@ const jobs = {
 			content: "\"apps/worker/index\": \"src/apps/worker/index.ts\",\n\"apps/cron/index\": \"src/apps/cron/index.ts\","
 		},
 		{
-			file: "tests/setup.ts",
-			slot: "setup",
+			file: "src/apps/http.ts",
+			slot: "imports",
+			content: "import { triggerSchedule } from \"@tatara/jobs\";\nimport { scheduleRunner } from \"#apps/cron/runner\";"
+		},
+		{
+			file: "src/apps/http.ts",
+			slot: "timeout-paths",
+			content: "\"/internal/schedules/*\": 15 * 60 * 1000,"
+		},
+		{
+			file: "src/apps/http.ts",
+			slot: "routes",
+			order: 3,
 			content: [
-				"",
-				"// One Redis database per worker.",
-				"if (process.env.REDIS_URL) {",
-				"	const redisUrl = new URL(process.env.REDIS_URL);",
-				"	redisUrl.pathname = `/${process.env.VITEST_POOL_ID ?? \"1\"}`;",
-				"	process.env.REDIS_URL = redisUrl.toString();",
+				"if (name === \"api\" && env.SCHEDULE_TRIGGER_SECRET) {",
+				"	const secret = env.SCHEDULE_TRIGGER_SECRET;",
+				"	app.post(\"/internal/schedules/:name\", async (c) => {",
+				"		const { status, body } = await triggerSchedule(c.req.param(\"name\"), c.req.header(\"authorization\"), {",
+				"			...scheduleRunner,",
+				"			logger: c.get(\"logger\"),",
+				"			secret,",
+				"		});",
+				"		return c.json(body, status);",
+				"	});",
 				"}"
 			].join("\n")
 		},
+		{
+			file: "src/apps/cron/runner.ts",
+			slot: "imports",
+			when: ["auth-orgs"],
+			content: "import { auth } from \"#modules/users/services/auth\";"
+		},
+		{
+			file: "src/apps/cron/runner.ts",
+			slot: "options",
+			when: ["auth-orgs"],
+			content: [
+				"async *orgs() {",
+				"	const { adapter } = await auth.$context;",
+				"	for (let offset = 0; ; offset += 100) {",
+				"		const page = await adapter.findMany<{ id: string }>({",
+				"			model: \"organization\",",
+				"			limit: 100,",
+				"			offset,",
+				"			sortBy: { field: \"id\", direction: \"asc\" },",
+				"		});",
+				"		for (const org of page) yield org.id;",
+				"		if (page.length < 100) return;",
+				"	}",
+				"},"
+			].join("\n")
+		},
+		{
+			file: "tests/setup.ts",
+			slot: "setup",
+			content: [
+				"",
+				"// One Redis database per worker.",
+				"if (process.env.REDIS_URL) {",
+				"	const redisUrl = new URL(process.env.REDIS_URL);",
+				"	redisUrl.pathname = `/${process.env.VITEST_POOL_ID ?? \"1\"}`;",
+				"	process.env.REDIS_URL = redisUrl.toString();",
+				"}"
+			].join("\n")
+		},
+		{
+			file: "AGENTS.md",
+			slot: "sections",
+			content: `## Jobs and schedules
+
+- Background work: \`enqueue(topic, payload, { orgId })\` from \`#core/jobs/index\`, typed by the app's
+  consumers. A module handles a topic in \`consumers/<name>.ts\`
+  (\`defineConsumer({ topic, schema, attempts?, backoffMs?, handle, onDeadLetter? })\` from \`@tatara/jobs\`).
+  The consumer validates the payload; a payload that never validates goes straight to dead.
+  Handlers must be idempotent: a failure retries with doubling backoff, and \`onDeadLetter\` runs once,
+  after the last attempt.
+- \`JOBS_DRIVER\` picks where jobs wait: \`bullmq\` (Redis, default), \`postgres\` (a \`tatara.jobs\` table;
+  pass \`{ executor: inTransaction(tx) }\` and the job commits with your writes, a transactional
+  outbox), \`kafka\`, or \`inline\` (tests: \`(jobs.driver as InlineDriver).drain()\`).
+- Recurring work: \`schedules/<name>.ts\` (\`defineSchedule({ name, cron, run })\`); \`run({ forEachOrg })\`
+  visits every org, one failure at a time. A lock (the cache) makes each tick run once however it
+  was started: \`apps/cron\`, \`tatara run schedule:run --name <name>\`, or
+  \`POST /internal/schedules/<name>\` with \`SCHEDULE_TRIGGER_SECRET\` (for a platform scheduler).
+- Processes: \`apps/worker\` runs consumers, \`apps/cron\` runs schedules. \`dev\` starts both.
+`
+		}
+	]
+};
+const cacheDriver = (drivers) => ({
+	key: "CACHE_DRIVER",
+	schema: `localDefault(z.enum(${JSON.stringify(drivers)}), "redis")`,
+	example: "redis",
+	test: "memory",
+	comment: `Where the cache lives: ${drivers.join(", ")} (memory is per process)`
+});
+const cache = {
+	id: "cache",
+	kind: "addon",
+	label: "Cache (Redis, Postgres or memory)",
+	dependencies: ["@tatara/cache", "ioredis"],
+	env: [{
+		key: "REDIS_PORT",
+		schema: "",
+		example: "6379",
+		composeOnly: true,
+		comment: "Host port for local Redis"
+	}, {
+		key: "REDIS_URL",
+		schema: "z.url()",
+		example: "redis://localhost:6379/0"
+	}],
+	variants: [{
+		when: ["postgres"],
+		env: [cacheDriver([
+			"redis",
+			"postgres",
+			"memory"
+		])]
+	}, {
+		unless: ["postgres"],
+		env: [cacheDriver(["redis", "memory"])]
+	}],
+	compose: { redis: REDIS_SERVICE },
+	slots: [{
+		file: "AGENTS.md",
+		slot: "sections",
+		content: `## Cache
+
+- \`cache\` from \`#core/cache/index\`: \`get/set/del\`, \`wrap(key, ttlMs, fn)\` (cache-aside; a failing
+  store falls back to \`fn\`), \`increment\` (fixed-window counters), \`add\`/\`take\` (single-use values)
+  and \`lock(key, ttlMs, fn)\` (one holder across processes).
+- Scope keys with \`orgKey(orgId, …)\` / \`userKey(userId, …)\` from \`@tatara/cache\`, and drop them all
+  with \`cache.delPrefix(orgKey(orgId))\`.
+- Rate-limit a route with \`middleware: [limitRequests({ name, windowMs, max })]\` from
+  \`#core/http/rate-limit\` (per caller IP, or per \`key\`); past the limit it answers 429 RATE_LIMITED.
+- \`CACHE_DRIVER\` picks the store (Redis by default; tests use memory). Values must be JSON.
+`
+	}]
+};
+const admin = {
+	id: "admin",
+	kind: "addon",
+	label: "Admin API app",
+	scripts: { "start:admin": "node dist/apps/admin/server.mjs" },
+	env: [
+		{
+			key: "ADMIN_PORT",
+			schema: "z.coerce.number().int().default(3001)",
+			example: "3001"
+		},
+		{
+			key: "ADMIN_URL",
+			schema: "localDefault(z.url(), \"http://localhost:3001\")",
+			example: "http://localhost:3001",
+			comment: "Where the admin app is served; staff sign in there"
+		},
+		{
+			key: "SEED_STAFF_EMAIL",
+			schema: "z.email().default(\"staff@example.com\")",
+			example: "staff@example.com",
+			comment: "Development staff account created by `db:seed` (never in production)"
+		},
+		{
+			key: "SEED_STAFF_PASSWORD",
+			schema: "z.string().min(10).optional()",
+			example: "",
+			secret: true,
+			comment: "Password for the development staff account (generated by tatara new)"
+		}
+	],
+	slots: [
+		{
+			file: "src/core/http/routes.ts",
+			slot: "app-names",
+			content: "\"admin\","
+		},
+		{
+			file: "tsdown.config.ts",
+			slot: "entries",
+			content: "\"apps/admin/server\": \"src/apps/admin/server.ts\","
+		},
+		{
+			file: "src/apps/http.ts",
+			slot: "imports",
+			content: [
+				"import { requireRole } from \"@tatara/auth\";",
+				"import { except } from \"hono/combine\";",
+				"import { attachStaffPrincipal, staffAuth } from \"#modules/staff/services/auth\";"
+			].join("\n")
+		},
+		{
+			file: "src/apps/http.ts",
+			slot: "middleware",
+			order: 50,
+			content: "app.use(name === \"admin\" ? attachStaffPrincipal : attachPrincipal);"
+		},
+		{
+			file: "src/apps/http.ts",
+			slot: "routes",
+			order: 2,
+			content: "if (name === \"admin\") app.on([\"GET\", \"POST\"], `${API_PREFIX}/auth/*`, (c) =>\n	staffAuth.handler(withClientIp(c.req.raw, clientIp(c, { trustedHops: env.TRUSTED_PROXY_HOPS }))),\n);"
+		},
+		{
+			file: "src/apps/http.ts",
+			slot: "routes",
+			order: 5,
+			content: "if (name === \"admin\") app.use(`${API_PREFIX}/*`, except(`${API_PREFIX}/auth/*`, requireRole(\"staff\")));"
+		},
+		{
+			file: "src/core/authz/system-roles.ts",
+			slot: "roles",
+			content: "{ key: \"staff\", name: \"Staff\", description: \"Internal staff: platform permissions on the admin app.\" },"
+		},
+		{
+			file: "src/modules/staff/services/auth.ts",
+			slot: "resolvers",
+			when: ["auth-keycloak"],
+			content: "() => (c) => keycloakPrincipal(auth, c.req.header(\"authorization\")),"
+		},
+		{
+			file: "src/modules/staff/services/auth.ts",
+			slot: "imports",
+			when: ["auth-keycloak"],
+			content: "import { keycloakPrincipal } from \"@tatara/keycloak\";\nimport { auth } from \"#modules/users/services/auth\";"
+		},
+		{
+			file: "tests/helpers/api.ts",
+			slot: "imports",
+			content: "import { app as adminApp } from \"#apps/admin/app\";"
+		},
+		{
+			file: "tests/helpers/api.ts",
+			slot: "exports",
+			content: "\n/** Calls the admin app in-process. */\nexport const adminApi = client(adminApp);"
+		},
+		{
+			file: "AGENTS.md",
+			slot: "sections",
+			content: `## Admin app and staff
+
+- \`apps/admin\` is a second HTTP app on \`ADMIN_PORT\`. Routes with \`defineRoutes({ app: "admin", … })\`
+  are served only there, and every \`/api\` route on it needs a staff principal (role \`staff\`).
+- Staff are separate accounts (\`src/modules/staff\`, tables \`staff_*\`), with their own better-auth
+  mounted at \`/api/auth\` on the admin app. They never share a table or a session with users.
+  There's no staff sign-up; the dev account comes from \`SEED_STAFF_EMAIL\`/\`SEED_STAFF_PASSWORD\`.
+- Staff act in the \`platform\` org, so grant platform permissions to the built-in \`staff\` role.
+- Test it with \`adminApi\` from \`#tests/helpers/api\` and \`asStaff()\` from \`#tests/helpers/staff\`.
+`
+		}
+	]
+};
+const MEMBERS = `/** The org's members, a page at a time. */
+async function* members(orgId: string) {
+	const { adapter } = await auth.$context;
+	for (let offset = 0; ; offset += 100) {
+		const page = await adapter.findMany<{ userId: string; role: string }>({
+			model: "member",
+			where: [{ field: "organizationId", value: orgId }],
+			limit: 100,
+			offset,
+			sortBy: { field: "id", direction: "asc" },
+		});
+		yield* page;
+		if (page.length < 100) return;
+	}
+}`;
+const hasRole = "m.role.split(\",\").map((r) => r.trim()).includes(role)";
+const notifications = {
+	id: "notifications",
+	kind: "addon",
+	label: "Notifications (in-app, email, SMS, push)",
+	dependencies: ["@tatara/notifications"],
+	env: [
+		{
+			key: "SMS_DRIVER",
+			schema: "z.enum([\"console\", \"twilio\"]).default(\"console\")",
+			example: "console",
+			test: "console",
+			comment: "How notifications send SMS; console logs them"
+		},
+		{
+			key: "TWILIO_ACCOUNT_SID",
+			schema: "z.string().optional()",
+			example: ""
+		},
+		{
+			key: "TWILIO_AUTH_TOKEN",
+			schema: "z.string().optional()",
+			example: ""
+		},
+		{
+			key: "TWILIO_FROM",
+			schema: "z.string().optional()",
+			example: "",
+			comment: "A Twilio number, or a messaging service sid (MG…)"
+		},
+		{
+			key: "PUSH_DRIVER",
+			schema: "z.enum([\"console\", \"fcm\"]).default(\"console\")",
+			example: "console",
+			test: "console",
+			comment: "How notifications push to devices; console logs them"
+		},
+		{
+			key: "FCM_PROJECT_ID",
+			schema: "z.string().optional()",
+			example: ""
+		},
+		{
+			key: "FCM_CLIENT_EMAIL",
+			schema: "z.string().optional()",
+			example: ""
+		},
+		{
+			key: "FCM_PRIVATE_KEY",
+			schema: "z.string().optional()",
+			example: "",
+			comment: "The Firebase service account's private key (\\n escapes are fine)"
+		}
+	],
+	slots: [
+		{
+			file: "src/modules/notifications/services/notifier.ts",
+			slot: "recipients",
+			unless: ["auth-orgs"],
+			content: [
+				"// without organizations there is no org to look people up in: notify users by id",
+				"recipients: {",
+				"	role() {",
+				"		throw new Error(\"Notifying by role needs organizations (the orgs auth method)\");",
+				"	},",
+				"	permission() {",
+				"		throw new Error(\"Notifying by permission needs organizations and the authz add-on\");",
+				"	},",
+				"},"
+			].join("\n")
+		},
+		{
+			file: "src/modules/notifications/services/notifier.ts",
+			slot: "helpers",
+			when: ["auth-orgs"],
+			content: MEMBERS
+		},
+		{
+			file: "src/modules/notifications/services/notifier.ts",
+			slot: "recipients",
+			when: ["auth-orgs"],
+			unless: ["authz"],
+			content: [
+				"recipients: {",
+				"	async *role(orgId, role) {",
+				`\t\tfor await (const m of members(orgId)) if (${hasRole}) yield m.userId;`,
+				"	},",
+				"	permission() {",
+				"		throw new Error(\"Notifying by permission needs the authz add-on\");",
+				"	},",
+				"},"
+			].join("\n")
+		},
+		{
+			file: "src/modules/notifications/services/notifier.ts",
+			slot: "recipients",
+			when: ["auth-orgs", "authz"],
+			content: [
+				"recipients: {",
+				"	async *role(orgId, role) {",
+				`\t\tfor await (const m of members(orgId)) if (${hasRole}) yield m.userId;`,
+				"	},",
+				"	// the same check a request makes: built-in roles, assigned roles and overlays",
+				"	async *permission(orgId, permission) {",
+				"		for await (const m of members(orgId)) {",
+				"			const who = { userId: m.userId, orgId, orgRole: m.role, roles: [], kind: \"user\" as const, claims: {} };",
+				"			if (await checkPermission(who, permission as PermissionKey)) yield m.userId;",
+				"		}",
+				"	},",
+				"},"
+			].join("\n")
+		},
+		{
+			file: "src/modules/notifications/services/notifier.ts",
+			slot: "imports",
+			when: ["auth-orgs", "authz"],
+			content: "import { checkPermission } from \"#core/authz/index\";\nimport type { PermissionKey } from \"#generated/permissions\";"
+		},
+		{
+			file: "src/modules/notifications/services/notifier.ts",
+			slot: "senders",
+			when: ["email"],
+			content: "email: { send: (message) => sendEmail(message) },"
+		},
+		{
+			file: "src/modules/notifications/services/notifier.ts",
+			slot: "imports",
+			when: ["email"],
+			content: "import { sendEmail } from \"#core/email/send\";"
+		},
+		{
+			file: "src/modules/notifications/routes/notifications.routes.ts",
+			slot: "guard",
+			when: ["auth-orgs"],
+			content: "requireOrg(),"
+		},
+		{
+			file: "src/modules/notifications/routes/notifications.routes.ts",
+			slot: "imports",
+			when: ["auth-orgs"],
+			content: "import { requireOrg } from \"#modules/users/services/auth\";"
+		},
+		{
+			file: "AGENTS.md",
+			slot: "sections",
+			content: `## Notifications
+
+- \`notify({ orgId, to, notification, data, channels?, idempotencyKey?, executor? })\` from
+  \`#modules/notifications/services/notifier\`. \`to\` is \`{ user }\`, \`{ users }\`, \`{ role }\` or
+  \`{ permission }\` (role and permission need organizations; permission needs authz).
+- A notification is \`notifications/<name>.ts\` in a module: \`defineNotification({ name, schema, channels,
+  inApp?, email?, sms?, push? })\`, each renderer taking \`(data, language)\`.
+- Delivery runs through jobs (\`notifications.fanout\`, then one \`notifications.deliver\` per user); each
+  channel goes out once per recipient, and the same \`idempotencyKey\` never notifies anyone twice.
+  Pass \`executor\` (with the Postgres jobs driver) to send only if your transaction commits.
+- The inbox: \`GET /api/notifications\` (cursor), \`/unread-count\`, \`POST /read\`, and
+  \`POST|DELETE /push-tokens\` for devices. \`SMS_DRIVER=twilio\` and \`PUSH_DRIVER=fcm\` send for real.
+- In tests, jobs run inline: \`await (jobs.driver as InlineDriver).drain()\` after \`notify\`.
+`
+		}
+	]
+};
+const realtime = {
+	id: "realtime",
+	kind: "addon",
+	label: "Realtime (websockets)",
+	dependencies: [
+		"@tatara/realtime",
+		"socket.io",
+		"@socket.io/redis-adapter",
+		"@socket.io/redis-emitter",
+		"ioredis"
+	],
+	devDependencies: ["socket.io-client"],
+	env: [
+		{
+			key: "REALTIME_DRIVER",
+			schema: "z.enum([\"socketio\", \"noop\"]).default(\"socketio\")",
+			example: "socketio",
+			comment: "noop turns websockets off (publishes do nothing)"
+		},
+		{
+			key: "REALTIME_ADAPTER",
+			schema: "localDefault(z.enum([\"redis\", \"memory\"]), \"redis\")",
+			example: "redis",
+			test: "memory",
+			comment: "redis shares rooms across api processes and lets workers publish"
+		},
+		{
+			key: "REALTIME_PATH",
+			schema: "z.string().default(\"/realtime\")",
+			example: "/realtime"
+		},
+		{
+			key: "REALTIME_TICKET_TTL_SECONDS",
+			schema: "z.coerce.number().int().positive().default(60)",
+			example: "60"
+		},
+		{
+			key: "REDIS_PORT",
+			schema: "",
+			example: "6379",
+			composeOnly: true,
+			comment: "Host port for local Redis"
+		},
+		{
+			key: "REDIS_URL",
+			schema: "z.url()",
+			example: "redis://localhost:6379/0"
+		}
+	],
+	compose: { redis: REDIS_SERVICE },
+	slots: [
+		{
+			file: "src/apps/api/server.ts",
+			slot: "imports",
+			content: "import { realtime } from \"#core/realtime/index\";\nimport { channels } from \"#generated/channels\";"
+		},
+		{
+			file: "src/apps/api/server.ts",
+			slot: "server-options",
+			content: [
+				"// websockets share the api's port; an api that can't serve them shouldn't run half-working",
+				"onServer: (server) => {",
+				"	realtime.attach(server as never, { channels }).catch((err: unknown) => {",
+				"		logger.fatal({ err }, \"realtime could not attach\");",
+				"		process.exit(1);",
+				"	});",
+				"},"
+			].join("\n")
+		},
+		{
+			file: "src/modules/realtime/routes/realtime.routes.ts",
+			slot: "guard",
+			when: ["auth-orgs"],
+			content: "requireOrg(),"
+		},
+		{
+			file: "src/modules/realtime/routes/realtime.routes.ts",
+			slot: "imports",
+			when: ["auth-orgs"],
+			content: "import { requireOrg } from \"#modules/users/services/auth\";"
+		},
+		{
+			file: "src/modules/notifications/services/notifier.ts",
+			slot: "senders",
+			when: ["notifications"],
+			content: [
+				"onInApp: (row) =>",
+				"	publishToUser(row.orgId, row.userId, \"notification.created\", {",
+				"		id: row.id,",
+				"		title: row.title,",
+				"		body: row.body ?? null,",
+				"		url: row.url ?? null,",
+				"	}),"
+			].join("\n")
+		},
+		{
+			file: "src/modules/notifications/services/notifier.ts",
+			slot: "imports",
+			when: ["notifications"],
+			content: "import { publishToUser } from \"#core/realtime/index\";"
+		},
 		{
 			file: "AGENTS.md",
 			slot: "sections",
-			content: `## Jobs and schedules
+			content: `## Realtime
 
-- Background work: \`enqueue(topic, payload)\` from \`#core/jobs/index\`; a module handles it in
-  \`consumers/<name>.ts\` (\`defineConsumer({ topic, schema, handle })\` from \`@tatara/jobs\`). Payloads are validated by the
-  consumer's zod schema. Handlers must be idempotent: jobs retry.
-- Recurring work: \`schedules/<name>.ts\` (\`defineSchedule({ name, cron, run })\`). A Redis lock makes
-  each tick run once across every cron process.
-- Processes: \`apps/worker\` runs consumers, \`apps/cron\` runs schedules. \`dev\` starts both.
+- A client asks \`POST /api/realtime/ticket\` (signed in) for a single-use ticket, then connects with
+  socket.io: \`io(API_URL, { path: "/realtime", transports: ["websocket"], auth: { ticket } })\`.
+  A socket belongs to one user in one org, and joins that user's and that org's rooms.
+- Publish from any process (api, worker, cron): \`publishToUser(orgId, userId, event, payload)\`,
+  \`publishToOrg(orgId, event, payload)\` from \`#core/realtime/index\`. \`disconnectUser\` /
+  \`disconnectOrg\` close sockets when access changes; clients reconnect with a new ticket.
+- Rooms of your own: \`channels/<name>.ts\` in a module, \`defineChannel({ name: "survey.{surveyId}",
+  events: { "answer.created": schema }, authorize(who, params) })\`. Clients \`emit("subscribe", name)\`;
+  publish with \`publishTo(channel, { orgId, params, event, payload })\` (the payload is checked).
+- With notifications, every inbox entry is also pushed as \`notification.created\`.
 `
 		}
 	]
 };
-const admin = {
-	id: "admin",
+const observability = {
+	id: "observability",
 	kind: "addon",
-	label: "Admin API app",
-	scripts: { "start:admin": "node dist/apps/admin/server.mjs" },
+	label: "Observability (OpenTelemetry)",
+	dependencies: [
+		"@tatara/observability",
+		"@opentelemetry/api",
+		"@opentelemetry/sdk-node",
+		"@opentelemetry/sdk-trace",
+		"@opentelemetry/sdk-metrics",
+		"@opentelemetry/resources",
+		"@opentelemetry/exporter-trace-otlp-http",
+		"@opentelemetry/exporter-metrics-otlp-http",
+		"@opentelemetry/auto-instrumentations-node",
+		"@opentelemetry/instrumentation",
+		"@hono/otel"
+	],
+	allowBuilds: ["protobufjs"],
+	scripts: { start: "node --import ./dist/apps/instrument.mjs dist/apps/api/server.mjs" },
+	variants: [{
+		when: ["jobs"],
+		scripts: {
+			"start:worker": "node --import ./dist/apps/instrument.mjs dist/apps/worker/index.mjs",
+			"start:cron": "node --import ./dist/apps/instrument.mjs dist/apps/cron/index.mjs"
+		}
+	}, {
+		when: ["admin"],
+		scripts: { "start:admin": "node --import ./dist/apps/instrument.mjs dist/apps/admin/server.mjs" }
+	}],
 	env: [
 		{
-			key: "ADMIN_PORT",
-			schema: "z.coerce.number().int().default(3001)",
-			example: "3001"
-		},
-		{
-			key: "ADMIN_URL",
-			schema: "localDefault(z.url(), \"http://localhost:3001\")",
-			example: "http://localhost:3001",
-			comment: "Where the admin app is served; staff sign in there"
+			key: "OTEL_EXPORTER_OTLP_ENDPOINT",
+			schema: "z.url().optional()",
+			example: "",
+			comment: "OTLP/HTTP collector, e.g. http://localhost:4318; empty sends nothing"
 		},
 		{
-			key: "SEED_STAFF_EMAIL",
-			schema: "z.email().default(\"staff@example.com\")",
-			example: "staff@example.com",
-			comment: "Development staff account created by `db:seed` (never in production)"
+			key: "OTEL_SERVICE_NAME",
+			schema: "z.string().default(\"__APP_NAME__\")",
+			example: "__APP_NAME__"
 		},
 		{
-			key: "SEED_STAFF_PASSWORD",
-			schema: "z.string().min(10).optional()",
-			example: "",
-			secret: true,
-			comment: "Password for the development staff account (generated by tatara new)"
+			key: "OTEL_TRACES_SAMPLE_RATIO",
+			schema: "z.coerce.number().min(0).max(1).default(1)",
+			example: "1",
+			comment: "Share of new traces kept (0–1)"
 		}
 	],
 	slots: [
-		{
-			file: "src/core/http/routes.ts",
-			slot: "app-names",
-			content: "\"admin\","
-		},
 		{
 			file: "tsdown.config.ts",
 			slot: "entries",
-			content: "\"apps/admin/server\": \"src/apps/admin/server.ts\","
-		},
-		{
-			file: "src/apps/http.ts",
-			slot: "imports",
-			content: [
-				"import { requireRole } from \"@tatara/auth\";",
-				"import { except } from \"hono/combine\";",
-				"import { attachStaffPrincipal, staffAuth } from \"#modules/staff/services/auth\";"
-			].join("\n")
+			content: "\"apps/instrument\": \"src/apps/instrument.ts\","
 		},
 		{
-			file: "src/apps/http.ts",
-			slot: "middleware",
-			order: 50,
-			content: "app.use(name === \"admin\" ? attachStaffPrincipal : attachPrincipal);"
+			file: "Dockerfile",
+			slot: "docker-runtime",
+			order: 60,
+			content: "ENV NODE_OPTIONS=\"--import /app/dist/apps/instrument.mjs\""
 		},
 		{
 			file: "src/apps/http.ts",
-			slot: "routes",
-			order: 2,
-			content: "if (name === \"admin\") app.on([\"GET\", \"POST\"], `${API_PREFIX}/auth/*`, (c) => staffAuth.handler(c.req.raw));"
+			slot: "imports",
+			content: "import { httpTracing } from \"@tatara/observability\";"
 		},
 		{
 			file: "src/apps/http.ts",
-			slot: "routes",
+			slot: "middleware",
 			order: 5,
-			content: "if (name === \"admin\") app.use(`${API_PREFIX}/*`, except(`${API_PREFIX}/auth/*`, requireRole(\"staff\")));"
+			content: "app.use(httpTracing(env.OTEL_SERVICE_NAME));"
 		},
 		{
-			file: "src/core/authz/system-roles.ts",
-			slot: "roles",
-			content: "{ key: \"staff\", name: \"Staff\", description: \"Internal staff: platform permissions on the admin app.\" },"
-		},
-		{
-			file: "src/modules/staff/services/auth.ts",
-			slot: "resolvers",
-			when: ["auth-keycloak"],
-			content: "() => (c) => keycloakPrincipal(auth, c.req.header(\"authorization\")),"
-		},
-		{
-			file: "src/modules/staff/services/auth.ts",
-			slot: "imports",
-			when: ["auth-keycloak"],
-			content: "import { keycloakPrincipal } from \"@tatara/keycloak\";\nimport { auth } from \"#modules/users/services/auth\";"
-		},
-		{
-			file: "tests/helpers/api.ts",
+			file: "src/core/logging/logger.ts",
 			slot: "imports",
-			content: "import { app as adminApp } from \"#apps/admin/app\";"
+			content: "import { traceIds } from \"@tatara/observability\";"
 		},
 		{
-			file: "tests/helpers/api.ts",
-			slot: "exports",
-			content: "\n/** Calls the admin app in-process. */\nexport const adminApi = client(adminApp);"
+			file: "src/core/logging/logger.ts",
+			slot: "logger-options",
+			content: "mixin: traceIds,"
 		},
 		{
 			file: "AGENTS.md",
 			slot: "sections",
-			content: `## Admin app and staff
+			content: `## Observability
 
-- \`apps/admin\` is a second HTTP app on \`ADMIN_PORT\`. Routes with \`defineRoutes({ app: "admin", … })\`
-  are served only there, and every \`/api\` route on it needs a staff principal (role \`staff\`).
-- Staff are separate accounts (\`src/modules/staff\`, tables \`staff_*\`), with their own better-auth
-  mounted at \`/api/auth\` on the admin app. They never share a table or a session with users.
-  There's no staff sign-up; the dev account comes from \`SEED_STAFF_EMAIL\`/\`SEED_STAFF_PASSWORD\`.
-- Staff act in the \`platform\` org, so grant platform permissions to the built-in \`staff\` role.
-- Test it with \`adminApi\` from \`#tests/helpers/api\` and \`asStaff()\` from \`#tests/helpers/staff\`.
+- Set \`OTEL_EXPORTER_OTLP_ENDPOINT\` (any OTLP/HTTP collector: Grafana, Honeycomb, Datadog, Jaeger…)
+  and every process sends traces and metrics. Requests, database, Redis and HTTP calls are traced
+  for you; log lines carry \`trace_id\` and \`span_id\`.
+- Processes start with \`node --import ./dist/apps/instrument.mjs\` (the Dockerfile sets it in
+  \`NODE_OPTIONS\`); without it nothing is instrumented.
+- Your own spans: \`trace.getTracer("billing").startActiveSpan(...)\` from \`@opentelemetry/api\`.
 `
 		}
 	]
@@ -1242,10 +1986,27 @@ const email = {
 	env: [
 		{
 			key: "EMAIL_DRIVER",
-			schema: "localDefault(z.enum([\"smtp\", \"console\"]), \"console\")",
+			schema: "localDefault(z.enum([\"smtp\", \"mailgun\", \"console\"]), \"console\")",
 			example: "smtp",
 			test: "console"
 		},
+		{
+			key: "MAILGUN_API_KEY",
+			schema: "z.string().optional()",
+			example: "",
+			comment: "For EMAIL_DRIVER=mailgun"
+		},
+		{
+			key: "MAILGUN_DOMAIN",
+			schema: "z.string().optional()",
+			example: ""
+		},
+		{
+			key: "MAILGUN_API_URL",
+			schema: "z.url().default(\"https://api.mailgun.net\")",
+			example: "https://api.mailgun.net",
+			comment: "https://api.eu.mailgun.net for EU domains"
+		},
 		{
 			key: "SMTP_URL",
 			schema: "localDefault(z.string(), \"smtp://localhost:1025\")",
@@ -1289,7 +2050,9 @@ const email = {
 		content: `## Email
 
 - \`sendEmail({ to, subject, text, html? })\` from \`#core/email/send\`. Locally it goes to Mailpit
-  (UI on \`MAILPIT_UI_PORT\`); tests use the console driver.
+  (UI on \`MAILPIT_UI_PORT\`); tests use the console driver; \`EMAIL_DRIVER=mailgun\` sends through Mailgun.
+- Templates: \`emails/<name>.ts\` in a module (\`defineEmail({ name, schema, render(data, language) })\` from
+  \`@tatara/email\`), sent with \`sendTemplate(name, to, data, language)\`.
 `
 	}]
 };
@@ -1347,7 +2110,7 @@ const auth = {
 		{
 			file: "src/apps/http.ts",
 			slot: "imports",
-			content: "import { attachPrincipal } from \"#modules/users/services/auth\";"
+			content: "import { clientIp, withClientIp } from \"@tatara/http\";\nimport { attachPrincipal } from \"#modules/users/services/auth\";"
 		},
 		{
 			file: "src/apps/http.ts",
@@ -1467,7 +2230,7 @@ const betterAuth = {
 			order: 1,
 			content: [
 				"if (name === \"api\") {",
-				"	app.on([\"GET\", \"POST\"], `${API_PREFIX}/auth/*`, (c) => auth.handler(c.req.raw));",
+				"	app.on([\"GET\", \"POST\"], `${API_PREFIX}/auth/*`, (c) =>\n		auth.handler(withClientIp(c.req.raw, clientIp(c, { trustedHops: env.TRUSTED_PROXY_HOPS }))),\n	);",
 				"	if (outbox.enabled) {",
 				"		app.get(`${API_PREFIX}/dev/outbox`, (c) => c.json({ messages: outbox.read(c.req.query(\"to\")) }));",
 				"	}",
@@ -1992,6 +2755,18 @@ const base = {
 			key: "BODY_LIMIT_BYTES",
 			schema: "z.coerce.number().int().positive().default(1_048_576)",
 			example: "1048576"
+		},
+		{
+			key: "REQUEST_TIMEOUT_MS",
+			schema: "z.coerce.number().int().positive().default(30_000)",
+			example: "30000",
+			comment: "Longest a request may take before a 504 (slow paths get their own in src/apps/http.ts)"
+		},
+		{
+			key: "TRUSTED_PROXY_HOPS",
+			schema: "z.coerce.number().int().min(0).default(0)",
+			example: "0",
+			comment: "Proxies in front of the app that append to X-Forwarded-For (a load balancer, Cloud Run: 1)"
 		}
 	],
 	entrypoints: [{
@@ -2163,6 +2938,16 @@ const db$1 = {
 				"process.env.DATABASE_URL = workerDatabaseUrl(process.env.DATABASE_URL ?? \"\", process.env.VITEST_POOL_ID ?? \"1\");"
 			].join("\n")
 		},
+		{
+			file: "src/apps/cli/index.ts",
+			slot: "imports",
+			content: "import { env } from \"#core/config/env\";\nimport { commandLedger } from \"#core/db/admin\";"
+		},
+		{
+			file: "src/apps/cli/index.ts",
+			slot: "options",
+			content: "ledger: commandLedger(env.DATABASE_URL),"
+		},
 		{
 			file: "AGENTS.md",
 			slot: "sections",
@@ -2312,20 +3097,31 @@ const FEATURES = Object.fromEntries([
 	auth,
 	authz,
 	jobs,
+	cache,
+	notifications,
+	realtime,
 	admin,
 	storage,
 	email,
-	i18n
+	i18n,
+	observability
 ].map((f) => [f.id, f]));
 //#endregion
 //#region src/engine/resolve.ts
 var OptionsError = class extends Error {};
+/** Add-ons others need: notifications deliver through jobs; schedules and realtime tickets use the cache. */
+function impliedAddons(addons) {
+	const out = new Set(addons);
+	if (out.has("notifications")) out.add("jobs");
+	if (out.has("jobs") || out.has("realtime")) out.add("cache");
+	return [...out];
+}
 /** Normalizes and validates options; throws OptionsError with a user-facing message. */
 function validateOptions(input) {
 	const options = {
 		...input,
 		authMethods: [...new Set(input.authMethods)],
-		addons: [...new Set(input.addons)]
+		addons: impliedAddons(input.addons)
 	};
 	if (!/^[a-z][a-z0-9-]*$/.test(options.name)) throw new OptionsError(`App name "${options.name}" must be lowercase letters, digits and dashes, starting with a letter.`);
 	if (options.orm === "none" !== (options.database === "none")) throw new OptionsError("Choose both an ORM and a database, or neither.");
@@ -2335,8 +3131,8 @@ function validateOptions(input) {
 		if (options.authMethods.filter((m) => m !== "orgs").length === 0) throw new OptionsError("better-auth needs at least one sign-in method: email-otp, phone-otp, password or keycloak.");
 	} else if (options.authMethods.length > 0) throw new OptionsError("Auth methods only apply to better-auth.");
 	for (const addon of options.addons) {
-		if ((addon === "authz" || addon === "admin") && options.auth === "none") throw new OptionsError(`The ${addon} add-on needs an auth provider.`);
-		if (addon === "authz" && !hasDb) throw new OptionsError("The authz add-on needs a database.");
+		if ((addon === "authz" || addon === "admin" || addon === "notifications" || addon === "realtime") && options.auth === "none") throw new OptionsError(`The ${addon} add-on needs an auth provider.`);
+		if ((addon === "authz" || addon === "notifications") && !hasDb) throw new OptionsError(`The ${addon} add-on needs a database.`);
 	}
 	return options;
 }
@@ -2999,6 +3795,10 @@ function normalizeAuthFlags(flags) {
 const ADDONS = [
 	"authz",
 	"jobs",
+	"cache",
+	"notifications",
+	"realtime",
+	"observability",
 	"admin",
 	"storage",
 	"email",
@@ -3105,6 +3905,10 @@ async function prompt(name, flags) {
 	const labels = {
 		authz: "Permission system",
 		jobs: "Jobs + cron (BullMQ)",
+		cache: "Cache (Redis, Postgres or memory)",
+		notifications: "Notifications (in-app, email, SMS, push)",
+		realtime: "Realtime (websockets)",
+		observability: "Observability (OpenTelemetry)",
 		admin: "Admin API app",
 		storage: "File storage",
 		email: "Email",
@@ -3268,111 +4072,6 @@ async function recordGenerated(root) {
 	await writeFile(path, `${JSON.stringify(config, null, "	")}\n`);
 }
 //#endregion
-//#region src/commands/routes.ts
-/** Reads every app's OpenAPI document in-process and lists its routes. */
-async function listRoutes(project) {
-	await sync(project);
-	const apps = [["api", "#apps/api/app"], ...project.has("admin") ? [["admin", "#apps/admin/app"]] : []];
-	const code = `
-const out = [];
-for (const [name, spec] of ${JSON.stringify(apps)}) {
-	const { app } = await import(spec);
-	// Fetch from /openapi.json endpoint to get the document with all extensions applied
-	const res = await app.request("/openapi.json");
-	const doc = await res.json();
-	const securitySchemes = doc.components?.securitySchemes ?? {};
-
-	for (const [path, item] of Object.entries(doc.paths ?? {})) {
-		for (const [method, op] of Object.entries(item)) {
-			const tags = op.tags ?? [];
-			const security = op.security ?? [];
-
-			// Parse access label based on security schemes and tags
-			let access = "public";
-
-			// Routes tagged "Auth" are from better-auth's merged document
-			if (tags.includes("Auth")) {
-				access = "better-auth";
-			} else if (security.length > 0) {
-				const labels = [];
-				for (const requirement of security) {
-					const schemes = Object.keys(requirement);
-					if (schemes.length === 0) continue;
-
-					const schemeName = schemes[0];
-					const roles = requirement[schemeName] ?? [];
-
-					// For bearer auth with specific roles, show the roles
-					if (schemeName === "bearerAuth" && roles.length > 0) {
-						labels.push(roles.join(", "));
-					}
-					// For bearer auth without roles, show "authenticated"
-					else if (schemeName === "bearerAuth") {
-						labels.push("authenticated");
-					}
-					// For other schemes (API keys, etc.), show the scheme name
-					else {
-						labels.push(schemeName);
-					}
-				}
-				access = labels.length > 1 ? labels.join(" OR ") : (labels[0] ?? "authenticated");
-			}
-
-			const summary = op.summary || op.description || "";
-			out.push({ app: name, method: method.toUpperCase(), path, summary, access });
-		}
-	}
-}
-process.stdout.write(JSON.stringify(out));
-process.exit(0);
-`;
-	return new Promise((resolve, reject) => {
-		const child = spawn(process.execPath, [
-			...NODE_FLAGS,
-			"--input-type=module",
-			"-e",
-			code
-		], {
-			cwd: project.root,
-			env: {
-				...process.env,
-				LOG_LEVEL: "silent"
-			}
-		});
-		let stdout = "";
-		let stderr = "";
-		child.stdout.on("data", (d) => {
-			stdout += d.toString();
-		});
-		child.stderr.on("data", (d) => {
-			stderr += d.toString();
-		});
-		child.on("exit", (code) => {
-			if (code !== 0) return reject(/* @__PURE__ */ new Error(`Could not load the apps:\n${stderr}`));
-			resolve(JSON.parse(stdout));
-		});
-	});
-}
-function printRoutes(routes) {
-	const rows = routes.map((r) => [
-		r.app,
-		r.method,
-		r.path,
-		r.access,
-		r.summary
-	]);
-	const widths = [
-		0,
-		1,
-		2,
-		3
-	].map((i) => Math.max(...rows.map((r) => (r[i] ?? "").length), 4));
-	for (const r of rows) {
-		const [app = "", method = "", path = "", access = "", summary = ""] = r;
-		console.log(`${pc.dim(app.padEnd(widths[0] ?? 0))}  ${pc.bold(method.padEnd(widths[1] ?? 0))}  ${path.padEnd(widths[2] ?? 0)}  ${(access === "public" ? pc.yellow : pc.green)(access.padEnd(widths[3] ?? 0))}  ${pc.dim(summary)}`);
-	}
-}
-//#endregion
 //#region src/commands/tinker.ts
 /** A REPL with env, logger, db and every module's services loaded. */
 async function tinker(project) {
@@ -3579,7 +4278,7 @@ function importsOf(text, specifier) {
 }
 //#endregion
 //#region src/upgrade/codemods/groups.ts
-async function read$5(ctx, path) {
+async function read$6(ctx, path) {
 	return ctx.read(path).catch(() => void 0);
 }
 function sourceFiles(ctx) {
@@ -3592,7 +4291,7 @@ function sourceFiles(ctx) {
 async function blockersOf$1(ctx, group, sources) {
 	const blockers = [];
 	for (const path of [...group.regenerate, ...group.remove]) {
-		const text = await read$5(ctx, path);
+		const text = await read$6(ctx, path);
 		if (text === void 0) continue;
 		if (!await ctx.baseline.isUnchanged(path, text)) blockers.push(`${path} is customised`);
 	}
@@ -3772,12 +4471,12 @@ const SPECIFIERS = {
 	"#core/auth/db/organization.schema": "#modules/users/db/organization.schema"
 };
 /** Files whose content this release changes in place (new imports, orgRole, single tenant). */
-const REGENERATED$2 = [
+const REGENERATED$4 = [
 	"src/core/http/context.ts",
 	"tests/helpers/auth.ts",
 	"src/core/authz/index.ts"
 ];
-async function read$4(ctx, path) {
+async function read$5(ctx, path) {
 	return ctx.read(path).catch(() => void 0);
 }
 /** Inserts the single-tenant option into an app's own createAuthz call, if it lacks one. */
@@ -3796,7 +4495,7 @@ const STAFF_FILES = [
 /** The admin app now signs in staff accounts of its own; the regenerated apps/http.ts imports them. */
 async function addStaff(ctx) {
 	for (const path of STAFF_FILES) {
-		if (await read$4(ctx, path) !== void 0) continue;
+		if (await read$5(ctx, path) !== void 0) continue;
 		const text = await ctx.renderCurrent(path);
 		if (text !== void 0) await ctx.write(path, text);
 	}
@@ -3806,7 +4505,7 @@ async function addStaff(ctx) {
 		".env.example",
 		".env.test"
 	]) {
-		const text = await read$4(ctx, path);
+		const text = await read$5(ctx, path);
 		if (text === void 0) continue;
 		if (await ctx.baseline.isUnchanged(path, text)) await ctx.regenerate(path);
 		else customisedEnv.push(path);
@@ -3822,23 +4521,23 @@ const appOwnedAuth = {
 		const { config } = ctx.project;
 		const orgs = config.authMethods.includes("orgs");
 		if (config.addons.includes("authz") && !orgs) {
-			const index = await read$4(ctx, "src/core/authz/index.ts");
+			const index = await read$5(ctx, "src/core/authz/index.ts");
 			const next = index && withSingleTenant(index);
 			if (next && next !== index) await ctx.write("src/core/authz/index.ts", next);
 		}
 		const blockers = [];
 		const moves = [];
 		for (const [from, to] of Object.entries(MOVED_IN_0_10)) {
-			const text = await read$4(ctx, from);
+			const text = await read$5(ctx, from);
 			if (text === void 0) continue;
-			const there = await read$4(ctx, to);
+			const there = await read$5(ctx, to);
 			if (there !== void 0 && there !== await ctx.renderCurrent(to)) blockers.push(`${to} already exists`);
 			else if (await ctx.baseline.isUnchanged(from, text)) moves.push([from, to]);
 			else blockers.push(`${from} is customised`);
 		}
 		const merged = [];
 		for (const from of Object.keys(MERGED_IN_0_10)) {
-			const text = await read$4(ctx, from);
+			const text = await read$5(ctx, from);
 			if (text === void 0) continue;
 			if (await ctx.baseline.isUnchanged(from, text)) merged.push(from);
 			else blockers.push(`${from} is customised`);
@@ -3854,8 +4553,8 @@ const appOwnedAuth = {
 			await ctx.removeIfUnchanged(from);
 		}
 		for (const from of merged) await ctx.removeIfUnchanged(from);
-		for (const path of REGENERATED$2) {
-			const text = await read$4(ctx, path);
+		for (const path of REGENERATED$4) {
+			const text = await read$5(ctx, path);
 			if (text !== void 0 && await ctx.baseline.isUnchanged(path, text)) await ctx.regenerate(path);
 		}
 		for (const [from, to] of Object.entries(SPECIFIERS)) await ctx.rewriteImports(from, to);
@@ -3942,7 +4641,7 @@ function groups(keycloak) {
 	});
 	return all;
 }
-async function read$3(ctx, path) {
+async function read$4(ctx, path) {
 	return ctx.read(path).catch(() => void 0);
 }
 /** True when `text` imports `entry` (or one of its gone names). */
@@ -3956,7 +4655,7 @@ function usesExport(text, entry) {
 async function blockersOf(ctx, group, sources) {
 	const blockers = [];
 	for (const path of [...group.regenerate, ...group.remove]) {
-		const text = await read$3(ctx, path);
+		const text = await read$4(ctx, path);
 		if (text === void 0) continue;
 		if (!await ctx.baseline.isUnchanged(path, text)) blockers.push(`${path} is customised`);
 	}
@@ -3990,7 +4689,7 @@ const authPackage = {
 		}
 		for (const group of ready) {
 			for (const path of group.regenerate) {
-				if (await read$3(ctx, path) === void 0) continue;
+				if (await read$4(ctx, path) === void 0) continue;
 				await ctx.regenerate(path);
 			}
 			for (const path of group.remove) await ctx.removeIfUnchanged(path);
@@ -4021,7 +4720,7 @@ const REMOVED = [
 	"services/sync.ts"
 ].map((f) => `${DIR}/${f}`);
 /** App files that stay (or are new), and become what this CLI generates. system-roles.ts is the app's config and stays as it is. */
-const REGENERATED$1 = [
+const REGENERATED$3 = [
 	`${DIR}/index.ts`,
 	`${DIR}/db/seed.ts`,
 	`${DIR}/commands/grant-role.ts`,
@@ -4087,7 +4786,7 @@ const MOVES = [
 ];
 /** Names the 0.5 constants file had that the package dropped. */
 const DROPPED_CONSTANTS = ["CACHE_TTL_MS"];
-async function read$2(ctx, path) {
+async function read$3(ctx, path) {
 	return ctx.read(path).catch(() => void 0);
 }
 const authzPackage = {
@@ -4099,8 +4798,8 @@ const authzPackage = {
 		await ctx.addDependency(AUTHZ, versionOf(AUTHZ));
 		const blockers = [];
 		const regenerate = [];
-		for (const path of REGENERATED$1) {
-			const text = await read$2(ctx, path);
+		for (const path of REGENERATED$3) {
+			const text = await read$3(ctx, path);
 			if (text === void 0) {
 				if (path === `${DIR}/index.ts`) regenerate.push(path);
 				continue;
@@ -4110,12 +4809,12 @@ const authzPackage = {
 		}
 		const removed = [];
 		for (const path of REMOVED) {
-			const text = await read$2(ctx, path);
+			const text = await read$3(ctx, path);
 			if (text === void 0) continue;
 			removed.push(path);
 			if (!await ctx.baseline.isUnchanged(path, text)) blockers.push(`${path} is customised`);
 		}
-		const own = /* @__PURE__ */ new Set([...REGENERATED$1, ...REMOVED]);
+		const own = /* @__PURE__ */ new Set([...REGENERATED$3, ...REMOVED]);
 		const sources = await glob(["src/**/*.ts", "tests/**/*.ts"], {
 			cwd: ctx.project.root,
 			ignore: ["src/generated/**"]
@@ -4298,13 +4997,13 @@ const MOVED$1 = [
 	}
 ];
 /** App files that stay, but become what this CLI generates (driver re-exports, thin helpers). */
-const REGENERATED = [
+const REGENERATED$2 = [
 	"src/core/db/admin.ts",
 	"src/core/db/migrate.ts",
 	"tests/helpers/api.ts",
 	"tests/global-setup.ts"
 ];
-async function read$1(ctx, path) {
+async function read$2(ctx, path) {
 	return ctx.read(path).catch(() => void 0);
 }
 const dbTesting = {
@@ -4316,8 +5015,8 @@ const dbTesting = {
 		if (hasDb) await ctx.addDependency("@tatara/db", versionOf("@tatara/db"));
 		await ctx.addDependency("@tatara/testing", versionOf("@tatara/testing"), { dev: true });
 		const regenerate = [];
-		for (const path of REGENERATED) {
-			const text = await read$1(ctx, path);
+		for (const path of REGENERATED$2) {
+			const text = await read$2(ctx, path);
 			if (text === void 0) continue;
 			if (await ctx.baseline.isUnchanged(path, text)) regenerate.push(path);
 			else ctx.report.notes.push(`${path} is customised and was kept; compare it with a newly generated app.`);
@@ -4337,6 +5036,67 @@ const dbTesting = {
 	}
 };
 //#endregion
+//#region src/upgrade/codemods/deploy-ledger.ts
+const REGENERATED$1 = [
+	"src/apps/cli/index.ts",
+	"src/core/db/admin.ts",
+	"src/apps/api/server.ts",
+	"src/core/logging/logger.ts"
+];
+const ADDED$1 = ["src/core/commands/db/command-runs.schema.ts", "prisma/models/command-runs.prisma"];
+/** `deploy`: migrations and deploy/once commands, with a ledger of what ran once. */
+const deployLedger = {
+	version: "0.11.0",
+	id: "deploy-ledger",
+	description: "A deploy command (migrations, then deploy and once commands) with a command ledger",
+	async run(ctx) {
+		const { config } = ctx.project;
+		const customised = [];
+		for (const path of REGENERATED$1) {
+			const text = await ctx.read(path).catch(() => void 0);
+			if (text === void 0) continue;
+			if (await ctx.baseline.isUnchanged(path, text)) await ctx.regenerate(path);
+			else customised.push(path);
+		}
+		let added = false;
+		for (const path of ADDED$1) {
+			if (await ctx.read(path).catch(() => void 0) !== void 0) continue;
+			const text = await ctx.renderCurrent(path);
+			if (text === void 0) continue;
+			await ctx.write(path, text);
+			added = true;
+		}
+		if (customised.length > 0) ctx.report.notes.push(`Compare ${customised.join(", ")} with a newly generated app: runCommand takes { ledger: commandLedger(env.DATABASE_URL) } (commandLedger from #core/db/admin), and the api server takes an options slot.`);
+		if (added) ctx.report.notes.push("MySQL keeps the command ledger in the app's own tatara_command_runs table: run `tatara db generate --name command_runs`.");
+		if (config.database !== "none") ctx.report.notes.push("Deploys now run `node dist/apps/cli/index.mjs deploy` (migrations, then commands with a phase, e.g. authz:sync-permissions) instead of separate db:migrate and sync steps.");
+	}
+};
+//#endregion
+//#region src/upgrade/codemods/email-v2.ts
+const FILES = [
+	"src/core/email/send.ts",
+	"src/core/config/env.ts",
+	".env.example",
+	".env.test"
+];
+/** Mailgun and email templates (`emails/*.ts`, `sendTemplate`) in apps with the email add-on. */
+const emailV2 = {
+	version: "0.11.0",
+	id: "email-v2",
+	description: "Email: a Mailgun driver and templates (emails/*.ts, sendTemplate)",
+	async run(ctx) {
+		if (!ctx.project.config.addons.includes("email")) return;
+		const customised = [];
+		for (const path of FILES) {
+			const text = await ctx.read(path).catch(() => void 0);
+			if (text === void 0) continue;
+			if (await ctx.baseline.isUnchanged(path, text)) await ctx.regenerate(path);
+			else customised.push(path);
+		}
+		if (customised.length > 0) ctx.report.notes.push(`Compare ${customised.join(", ")} with a newly generated app: EMAIL_DRIVER may be "mailgun" (MAILGUN_API_KEY, MAILGUN_DOMAIN, MAILGUN_API_URL), and src/core/email/send.ts exports sendTemplate for emails/*.ts templates.`);
+	}
+};
+//#endregion
 //#region src/upgrade/codemods/framework-tests.ts
 const frameworkTests = {
 	version: "0.9.0",
@@ -4351,6 +5111,29 @@ const frameworkTests = {
 	}
 };
 //#endregion
+//#region src/upgrade/codemods/http-limits.ts
+const ENV_FILES = [
+	"src/core/config/env.ts",
+	".env.example",
+	".env.test"
+];
+/** Request timeouts and the caller's real address for better-auth, in apps made before 0.11. */
+const httpLimits = {
+	version: "0.11.0",
+	id: "http-limits",
+	description: "Request timeouts (REQUEST_TIMEOUT_MS) and trusted proxies (TRUSTED_PROXY_HOPS)",
+	async run(ctx) {
+		const customised = [];
+		for (const path of ["src/apps/http.ts", ...ENV_FILES]) {
+			const text = await ctx.read(path).catch(() => void 0);
+			if (text === void 0) continue;
+			if (await ctx.baseline.isUnchanged(path, text)) await ctx.regenerate(path);
+			else customised.push(path);
+		}
+		if (customised.length > 0) ctx.report.notes.push(`Compare ${customised.join(", ")} with a newly generated app: env gains REQUEST_TIMEOUT_MS and TRUSTED_PROXY_HOPS; src/apps/http.ts passes timeouts: { defaultMs: env.REQUEST_TIMEOUT_MS } to createApp and hands better-auth the caller's address with withClientIp(c.req.raw, clientIp(c, { trustedHops: env.TRUSTED_PROXY_HOPS })), since better-auth otherwise rate-limits on an X-Forwarded-For the client can forge.`);
+	}
+};
+//#endregion
 //#region src/upgrade/codemods/http-package.ts
 const HTTP = "@tatara/http";
 /** Wiring files the app keeps; replaced by what this CLI generates when the app never changed them. */
@@ -4390,7 +5173,7 @@ const MOVED = [
 		usedBy: SERVERS
 	}
 ];
-async function read(ctx, path) {
+async function read$1(ctx, path) {
 	return ctx.read(path).catch(() => void 0);
 }
 /** Files outside the group that import one of the moved files (they would break). */
@@ -4426,14 +5209,14 @@ const httpPackage = {
 			ROUTES,
 			...SERVERS
 		]) {
-			const text = await read(ctx, path);
+			const text = await read$1(ctx, path);
 			if (text === void 0) continue;
 			if (await ctx.baseline.isUnchanged(path, text)) replace.push(path);
 			else blockers.push(`${path} is customised`);
 		}
 		const present = [];
 		for (const entry of MOVED) {
-			const text = await read(ctx, entry.path);
+			const text = await read$1(ctx, entry.path);
 			if (text === void 0) continue;
 			present.push(entry);
 			if (!await ctx.baseline.isUnchanged(entry.path, text)) blockers.push(`${entry.path} is customised`);
@@ -4454,6 +5237,60 @@ const httpPackage = {
 	}
 };
 //#endregion
+//#region src/upgrade/codemods/jobs-drivers.ts
+/** Generated jobs wiring that 0.11 replaces when the app hasn't changed it. */
+const REGENERATED = [
+	"src/core/jobs/index.ts",
+	"src/apps/worker/index.ts",
+	"src/apps/cron/index.ts",
+	"src/modules/example/tests/jobs.test.ts",
+	"src/core/config/env.ts",
+	".env.example",
+	".env.test"
+];
+/** New in 0.11; written only where the app has nothing. */
+const ADDED = [
+	"src/apps/cron/runner.ts",
+	"src/core/jobs/outbox.ts",
+	"src/core/cache/index.ts",
+	"src/core/http/rate-limit.ts"
+];
+async function read(ctx, path) {
+	return ctx.read(path).catch(() => void 0);
+}
+/** Jobs over drivers (inline, Postgres outbox, BullMQ, Kafka); schedules lock with the cache. */
+const jobsDrivers = {
+	version: "0.11.0",
+	id: "jobs-drivers",
+	description: "Jobs drivers (JOBS_DRIVER), typed enqueue, dead letters, schedule triggers; jobs bring the cache",
+	async run(ctx) {
+		const { config } = ctx.project;
+		if (!config.addons.includes("jobs")) return;
+		if (!config.addons.includes("cache")) {
+			config.addons.push("cache");
+			const raw = JSON.parse(await ctx.read("tatara.json"));
+			raw.addons = [.../* @__PURE__ */ new Set([...raw.addons, "cache"])];
+			await ctx.write("tatara.json", `${JSON.stringify(raw, null, "	")}\n`);
+			await ctx.addDependency("@tatara/cache", versionOf("@tatara/cache"));
+		}
+		await ctx.addDependency("kafkajs", versionOf("kafkajs"));
+		for (const path of ADDED) {
+			if (await read(ctx, path) !== void 0) continue;
+			const text = await ctx.renderCurrent(path);
+			if (text !== void 0) await ctx.write(path, text);
+		}
+		const customised = [];
+		for (const path of REGENERATED) {
+			const text = await read(ctx, path);
+			if (text === void 0) continue;
+			if (await ctx.baseline.isUnchanged(path, text)) await ctx.regenerate(path);
+			else customised.push(path);
+		}
+		if (customised.length > 0) ctx.report.notes.push(`Compare ${customised.join(", ")} with a newly generated app: env gains CACHE_DRIVER, JOBS_DRIVER, KAFKA_BROKERS, KAFKA_CLIENT_ID, KAFKA_SSL, KAFKA_SASL_USERNAME, KAFKA_SASL_PASSWORD and SCHEDULE_TRIGGER_SECRET; the queue is now \`jobs\` (createJobs over the JOBS_DRIVER driver), startWorker takes { driver: jobs.driver, consumers, … }, and startCron takes the schedule runner in src/apps/cron/runner.ts (it locks with the cache).`);
+		ctx.report.notes.push("Jobs now pick a driver with JOBS_DRIVER (bullmq stays the default; tests use inline). onDeadLetter runs once, after the last attempt. The app now has the cache add-on, which schedules lock with.");
+	}
+};
+//#endregion
 //#region src/upgrade/codemods/package-commands.ts
 /** Copies whose commands and seeds the packages contribute now (wired by `tatara sync`). */
 const COPIES = [
@@ -4501,7 +5338,11 @@ const CODEMODS = [
 			}
 		}
 	},
-	appOwnedAuth
+	appOwnedAuth,
+	httpLimits,
+	jobsDrivers,
+	emailV2,
+	deployLedger
 ];
 //#endregion
 //#region src/upgrade/context.ts
@@ -4797,6 +5638,14 @@ program.command("routes").description("List every route and what protects it").o
 	if (flags.json) console.log(JSON.stringify(routes, null, 2));
 	else printRoutes(routes);
 });
+program.command("contracts").description("Write each app's OpenAPI document and TypeScript types for typed clients").option("--out <dir>", "folder to write them to", "contracts").option("--check", "fail when the files are out of date instead of writing them").action(async (flags) => {
+	const { written, stale } = await writeContracts(loadProject(), flags);
+	for (const file of written) console.log(`${pc.green("✔")} ${file}`);
+	if (stale.length > 0) {
+		console.error(`Out of date: ${stale.join(", ")}. Run tatara contracts.`);
+		process.exitCode = 1;
+	}
+});
 program.command("upgrade").description("Run this CLI version's codemods on the app, then sync and verify").option("--dry-run", "show what would change without writing").option("--no-verify", "skip pnpm verify at the end").action(async (flags) => {
 	const project = loadProject();
 	const from = project.config.version;