@browserlessapi/inferlo 0.4.0 → 0.6.0

package/dist/app.js

@@ -4,16 +4,29 @@
  */
 import { bindTools, createAI } from "./ai/index.js";
 import { memory } from "./db/memory.js";
-import { createDb } from "./db/table.js";
+import { createAudit } from "./audit.js";
+import { checkRouteCache, createCache, databaseCacheStore, etagFor, etagMatches, memoryCacheStore, resolveCacheConfig, routeCacheKey } from "./cache.js";
+import { bindDb, createDb, guardAdapter } from "./db/table.js";
 import { describeApp } from "./describe.js";
 import { InferloError, didYouMean, suggest } from "./errors.js";
 import { callTool, handleMcp, toolDefinition } from "./mcp.js";
 import { buildOpenAPI } from "./openapi.js";
-import { isRoute } from "./route.js";
+import { collectRoutes } from "./route.js";
 import { checkRateLimit, databaseRateLimitStore, memoryRateLimitStore, rateLimitResult, windowMs } from "./rate-limit.js";
 import { clientIp, clientIps } from "./client-ip.js";
-import { collectCrons, collectJobs, createJobs, jobStoreFor } from "./jobs.js";
+import { collectCrons, collectJobs, createJobs, jobStoreFor, stopWorkers } from "./jobs.js";
 import { ValidationError } from "./schema.js";
+import { createRedactor, redactingLogger, redactValues, secretsOf } from "./redact.js";
+import { defaultLogger, loggingOptions, REQUEST_ID } from "./logging.js";
+import { clientTracing, jobTracing, requestTracing, traceAdapter, traceFields } from "./tracing.js";
+import { resolveDeprecation } from "./deprecation.js";
+import { createMetrics, hasMetricsToken, resolveMetrics } from "./metrics.js";
+import { INFERLO_VERSION } from "./describe.js";
+import { registerRequestMatcher } from "./request-matcher.js";
+import { versionDocPath, versionsOf } from "./api-version.js";
+import { resolveModules } from "./module.js";
+import { isPermissions } from "./permissions.js";
+import { applySecureHeaders, resolveSecureHeaders } from "./secure-headers.js";
 const testUsers = new WeakMap();
 function compile(def) {
     if (!def.path.startsWith("/")) {
@@ -39,29 +52,6 @@ function compile(def) {
         .join("");
     return { def, regex: new RegExp(`^${pattern || ""}/?$`), keys, score: score * 10 + parts.length };
 }
-function collectRoutes(sources) {
-    const out = [];
-    const seen = new Set();
-    const visit = (source, name) => {
-        if (!source || typeof source !== "object" || seen.has(source))
-            return;
-        seen.add(source);
-        if (isRoute(source)) {
-            out.push(name && !source.name ? { ...source, name } : source);
-            return;
-        }
-        if (Array.isArray(source)) {
-            for (const item of source)
-                visit(item);
-            return;
-        }
-        for (const [key, value] of Object.entries(source))
-            if (isRoute(value) || Array.isArray(value))
-                visit(value, key === "default" ? undefined : key);
-    };
-    visit(sources);
-    return out;
-}
 function corsHeaders(config, origin) {
     const allowed = config.origin === "*" ? "*" : origin && (Array.isArray(config.origin) ? config.origin : [config.origin]).includes(origin) ? origin : null;
     if (!allowed)
@@ -71,13 +61,15 @@ function corsHeaders(config, origin) {
         h["access-control-allow-credentials"] = "true";
     return h;
 }
-async function readBody(req, limit) {
+async function readBody(req, limit, signal) {
     const length = Number(req.headers.get("content-length") ?? "0");
     if (length > limit)
         throw new InferloError("IL2004", `Request body is larger than ${limit} bytes.`, { status: 413 });
     if (!req.body)
         return "";
     const reader = req.body.getReader();
+    // Stop reading a slow body once the request timed out.
+    signal?.addEventListener("abort", () => void reader.cancel().catch(() => { }), { once: true });
     const chunks = [];
     let size = 0;
     while (true) {
@@ -107,16 +99,35 @@ function queryObject(url) {
     }
     return out;
 }
+/** Sets a header unless the response already has it; copies responses with immutable headers. */
+function withHeader(response, name, value) {
+    if (response.headers.has(name))
+        return response;
+    try {
+        response.headers.set(name, value);
+        return response;
+    }
+    catch {
+        const copy = new Response(response.body, response);
+        copy.headers.set(name, value);
+        return copy;
+    }
+}
 function json(data, status, headers) {
     headers.set("content-type", "application/json; charset=utf-8");
     return new Response(JSON.stringify(data), { status, headers });
 }
 export function createApp(config = {}) {
-    const logger = config.logger ?? {
-        info: (m, d) => console.log(`[inferlo] ${m}`, d ?? ""),
-        error: (m, d) => console.error(`[inferlo] ${m}`, d ?? ""),
-    };
-    const models = (config.models ?? []);
+    // Everything the app logs goes through the redactor: credentials and secrets never reach the logs.
+    const redactor = createRedactor([...redactValues(config.redact), ...Object.values(config.ai?.providers ?? {}).flatMap((p) => secretsOf(p))]);
+    const logging = loggingOptions(config.logging, config.logger !== undefined);
+    const logger = redactingLogger(config.logger ?? defaultLogger(logging.format, config.name ?? "inferlo-app"), redactor);
+    const tracing = requestTracing(config.tracing, config.name ?? "inferlo-app", config.version ?? "0.0.0");
+    const tracingApi = tracing ? config.tracing.api : undefined;
+    // The server span of each request, for errors recorded while it runs.
+    const requestSpans = new WeakMap();
+    const { modules, modelOwner } = resolveModules(config.modules, config.models ?? []);
+    const models = [...(config.models ?? []), ...modules.flatMap((m) => m.models)];
     const modelNames = new Set();
     for (const m of models) {
         if (modelNames.has(m.name))
@@ -133,7 +144,15 @@ export function createApp(config = {}) {
             }
         }
     }
+    // Routes of the app itself, then each module's; routeModule remembers which module a route came from.
+    const routeModule = new Map();
     const routes = collectRoutes(config.routes);
+    for (const m of modules) {
+        for (const r of collectRoutes(m.routes)) {
+            routeModule.set(r, m.name);
+            routes.push(r);
+        }
+    }
     const compiled = routes.map(compile).sort((a, b) => b.score - a.score);
     const signatures = new Map();
     for (const r of routes) {
@@ -158,23 +177,151 @@ export function createApp(config = {}) {
             fix: 'Routes are private by default. Add auth: "public" to routes anyone may call, or pass createApp({ auth: { resolve(req) { ... } } }).',
         });
     }
+    if (config.permissions !== undefined && !isPermissions(config.permissions)) {
+        throw new InferloError("IL6026", "createApp({ permissions }) takes the result of permissions({ ... }).", { fix: 'Define them with permissions({ "tickets.update": ({ user, ticket }) => ticket.ownerId === user.id }) and pass that value.' });
+    }
+    for (const r of routes) {
+        const name = r.options.permission;
+        if (name === undefined)
+            continue;
+        const where = `${r.method} ${r.path}`;
+        if (typeof name !== "string" || !config.permissions?.has(name)) {
+            throw new InferloError("IL6026", config.permissions ? `The route ${where} uses the unknown permission "${String(name)}".` : `The route ${where} uses the permission "${String(name)}", but createApp has no permissions.`, {
+                fix: config.permissions ? `Use one of: ${config.permissions.list.map((p) => p.name).join(", ")}, or add it to permissions({ ... }).` : "Pass createApp({ permissions }) with the value from permissions({ ... }).",
+            });
+        }
+        if (r.options.auth === "public") {
+            throw new InferloError("IL6026", `The route ${where} has a permission but auth: "public".`, { fix: 'A permission needs a signed-in user: remove auth: "public" from the route.' });
+        }
+    }
     for (const r of routes)
         if (r.options.rateLimit !== undefined)
             checkRateLimit(r.options.rateLimit, `${r.method} ${r.path}`);
+    for (const r of routes)
+        if (r.options.cache !== undefined)
+            checkRouteCache(r.options.cache, r.method, `${r.method} ${r.path}`);
+    const requestTimeoutMs = config.requestTimeoutMs === undefined ? 60_000 : checkTimeout(config.requestTimeoutMs, "createApp({ requestTimeoutMs })");
+    for (const r of routes)
+        if (r.options.timeoutMs !== undefined)
+            checkTimeout(r.options.timeoutMs, `${r.method} ${r.path} { timeoutMs }`);
+    const deprecations = new Map();
+    for (const r of routes) {
+        const resolved = resolveDeprecation(r.options.deprecated, `${r.method} ${r.path}`);
+        if (resolved)
+            deprecations.set(r, resolved);
+    }
     const storeName = config.rateLimit?.store ?? "memory";
     if (storeName !== "memory" && storeName !== "database") {
         throw new InferloError("IL6007", `Invalid rateLimit store "${String(storeName)}".`, { fix: 'Use createApp({ rateLimit: { store: "memory" } }) or { store: "database" }.' });
     }
-    const adapter = config.db ?? memory();
+    const openapiPath = config.openapi === undefined ? "/openapi.json" : config.openapi;
+    const hasTools = routes.some((r) => r.options.tool);
+    const mcpPath = config.mcp === undefined ? (hasTools ? "/mcp" : false) : config.mcp;
+    const health = resolveHealth(config.health, routes, [openapiPath, mcpPath]);
+    // apiVersion("v1", ...) routes get their own OpenAPI document at /v1/openapi.json.
+    const versions = versionsOf(routes);
+    const versionDocs = new Map(openapiPath ? versions.map((v) => [versionDocPath(v, openapiPath), v]) : []);
+    for (const docPath of versionDocs.keys()) {
+        const clash = routes.find((r) => r.method === "GET" && r.path.replace(/\/+$/, "") === docPath);
+        if (clash)
+            throw new InferloError("IL6025", `The route GET ${clash.path} has the path of the OpenAPI document of API version ${versionDocs.get(docPath)}.`, { fix: "Rename the route; Inferlo serves each version's OpenAPI document there." });
+    }
+    const metricsConfig = resolveMetrics(config.metrics, [
+        ...[...versionDocs.keys()].map((path) => ({ path, what: "an OpenAPI document path" })),
+        { path: openapiPath, what: "the OpenAPI path" },
+        { path: mcpPath, what: "the MCP path" },
+        { path: health?.path ?? false, what: "the health path" },
+        { path: health?.readyPath ?? false, what: "the readiness path" },
+        ...routes.filter((r) => r.method === "GET").map((r) => ({ path: r.path, what: `the route GET ${r.path}` })),
+    ]);
+    // With tracing, database queries made while a request or job is traced get client spans.
+    const appTracing = tracing ? { ...config.tracing, tracerName: config.tracing.tracerName ?? config.name ?? "inferlo-app" } : undefined;
+    const adapter = appTracing ? traceAdapter(config.db ?? memory(), clientTracing(appTracing, "inferlo-app", config.version ?? "0.0.0")) : (config.db ?? memory());
     const db = createDb(models, adapter);
     const rateLimits = storeName === "database" ? databaseRateLimitStore(adapter) : memoryRateLimitStore();
-    const ai = createAI(config.ai);
-    const jobs = createJobs(collectJobs(config.jobs), { db: db, ai, store: jobStoreFor(config.jobQueue, adapter), queue: config.jobQueue?.store ?? "memory", logger }, collectCrons(config.jobs));
+    const cacheConfig = resolveCacheConfig(config.cache);
+    const cache = createCache(cacheConfig.store === "database" ? databaseCacheStore(adapter) : memoryCacheStore(cacheConfig.maxEntries), cacheConfig.store);
+    const metrics = metricsConfig
+        ? createMetrics({ app: config.name ?? "inferlo-app", version: config.version ?? "0.0.0", inferlo: INFERLO_VERSION }, metricsConfig.buckets)
+        : undefined;
+    // With metrics on, every AI call that reports usage is counted before the app's own onUsage runs.
+    // With tracing, AI calls get client spans too (unless the AI config has its own tracing).
+    const aiConfig = config.ai && (metrics || appTracing)
+        ? {
+            ...config.ai,
+            ...(metrics ? { onUsage: (usage) => (metrics.ai(usage), config.ai.onUsage?.(usage)) } : {}),
+            ...(appTracing ? { tracing: config.ai.tracing ?? appTracing } : {}),
+        }
+        : config.ai;
+    const ai = createAI(aiConfig);
+    const jobModule = new Map();
+    for (const m of modules)
+        for (const j of collectJobs(m.jobs))
+            jobModule.set(j.name, m.name);
+    const moduleInfo = { modules, routeModule, modelOwner, jobModule };
+    const jobSources = [...(config.jobs ?? []), ...modules.flatMap((m) => m.jobs)];
+    const jobs = createJobs(collectJobs(jobSources), {
+        db: db,
+        ai,
+        cache,
+        audit: (run) => createAudit(models, redactor, () => db, { userId: null, source: `job ${run.name}` }),
+        store: jobStoreFor(config.jobQueue, adapter),
+        queue: config.jobQueue?.store ?? "memory",
+        logger,
+        onRun: metrics ? (name, outcome, ms) => metrics.job(name, outcome, ms / 1000) : undefined,
+        tracing: appTracing ? jobTracing(appTracing, "inferlo-app", config.version ?? "0.0.0") : undefined,
+    }, collectCrons(jobSources));
     const maxBody = config.maxBodySize ?? 1024 * 1024;
+    const secureHeaders = resolveSecureHeaders(config.secureHeaders);
     const cors = config.cors === true ? { origin: "*" } : config.cors || undefined;
-    const openapiPath = config.openapi === undefined ? "/openapi.json" : config.openapi;
-    const hasTools = routes.some((r) => r.options.tool);
-    const mcpPath = config.mcp === undefined ? (hasTools ? "/mcp" : false) : config.mcp;
+    // Readiness: the database answers, every AI alias points to a configured provider, and the app's own checks pass.
+    const readinessChecks = {
+        database: async () => {
+            if (adapter.sql)
+                await adapter.sql("select 1");
+        },
+    };
+    const aiProviders = Object.keys(config.ai?.providers ?? {});
+    if (aiProviders.length || Object.keys(config.ai?.models ?? {}).length) {
+        readinessChecks.ai = () => {
+            for (const [alias, targets] of Object.entries(ai.aliases)) {
+                for (const target of targets) {
+                    const provider = target.split(":")[0];
+                    if (!aiProviders.includes(provider)) {
+                        throw new InferloError("IL5001", `AI alias "${alias}" uses provider "${provider}", which is not configured.`, { status: 503, expose: true });
+                    }
+                }
+            }
+        };
+    }
+    Object.assign(readinessChecks, health?.checks);
+    const runCheck = async (name, check) => {
+        const started = Date.now();
+        const timeoutMs = health?.timeoutMs ?? 5000;
+        let timer;
+        const timedOut = Symbol("timeout");
+        try {
+            const outcome = await Promise.race([
+                Promise.resolve().then(check),
+                new Promise((resolve) => {
+                    timer = setTimeout(() => resolve(timedOut), timeoutMs);
+                }),
+            ]);
+            if (outcome === timedOut) {
+                logger.error(`Readiness check "${name}" timed out after ${timeoutMs} ms`);
+                return { status: "error", durationMs: Date.now() - started, error: `Timed out after ${timeoutMs} ms.` };
+            }
+            return { status: "ok", durationMs: Date.now() - started };
+        }
+        catch (error) {
+            logger.error(`Readiness check "${name}" failed`, error);
+            const shown = (error instanceof InferloError && error.expose) || (config.exposeErrors && error instanceof Error);
+            return { status: "error", durationMs: Date.now() - started, error: shown ? redactor.text(error.message) : "Check failed." };
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    };
     // Routes passed to ai.generate({ tools }) are matched to the registered route by method and path.
     const registeredTool = (route) => {
         const found = routes.find((r) => r.method === route?.method && r.path === route?.path);
@@ -189,7 +336,7 @@ export function createApp(config = {}) {
         const { name, description, inputSchema } = registeredTool(route);
         return { name, description, inputSchema };
     };
-    const toolsFor = (req) => bindTools(ai, async (route, input) => {
+    const toolsFor = (req, signal) => bindTools(ai, async (route, input) => {
         const tool = registeredTool(route);
         const response = await callTool(app, tool, input, req, testUsers);
         const text = await response.text();
@@ -201,22 +348,26 @@ export function createApp(config = {}) {
             // Not JSON: the model sees the text as it is.
         }
         return { name: tool.name, status: response.status, output, text: text || String(response.status) };
-    }, describeTool);
+    }, describeTool, signal);
     const roles = (user) => config.auth?.roles?.(user) ?? (user.roles ?? []);
-    const errorResponse = (error, headers, req) => {
+    const errorResponse = (error, headers, req, requestId) => {
+        const span = requestSpans.get(req);
+        const fields = { requestId, ...traceFields(span) };
+        if (span && !(error instanceof InferloError && error.status < 500))
+            span.recordException(error);
         if (error instanceof InferloError) {
             if (error.status >= 500)
-                logger.error(`${req.method} ${new URL(req.url).pathname} failed: ${error.format()}`, error.cause ?? "");
+                logger.error(`${req.method} ${new URL(req.url).pathname} failed: ${error.format()}`, error.cause ?? "", fields);
             const body = error.toJSON();
             if (config.exposeErrors && !error.expose)
-                body.error.message = error.message;
+                body.error.message = redactor.text(error.message);
             return json(body, error.status, headers);
         }
-        logger.error(`${req.method} ${new URL(req.url).pathname} failed`, error);
-        const message = config.exposeErrors && error instanceof Error ? error.message : "Internal server error";
+        logger.error(`${req.method} ${new URL(req.url).pathname} failed`, error, fields);
+        const message = config.exposeErrors && error instanceof Error ? redactor.text(error.message) : "Internal server error";
         return json({ error: { code: "IL9000", message, docs: new InferloError("IL9000", "").docs } }, 500, headers);
     };
-    const dispatch = async (req, url, state, pending, ip) => {
+    const dispatch = async (req, url, state, pending, ip, requestId, seen) => {
         const headers = new Headers();
         if (cors)
             for (const [k, v] of Object.entries(corsHeaders(cors, req.headers.get("origin"))))
@@ -225,14 +376,57 @@ export function createApp(config = {}) {
             const method = req.method.toUpperCase();
             if (method === "OPTIONS" && cors) {
                 headers.set("access-control-allow-methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS");
-                headers.set("access-control-allow-headers", (cors.headers ?? ["content-type", "authorization"]).join(", "));
+                headers.set("access-control-allow-headers", (cors.headers ?? ["content-type", "authorization", "x-csrf-token"]).join(", "));
                 headers.set("access-control-max-age", String(cors.maxAge ?? 600));
                 return new Response(null, { status: 204, headers });
             }
-            if (openapiPath && url.pathname === openapiPath && method === "GET")
+            if (health && (method === "GET" || method === "HEAD") && (url.pathname === health.path || url.pathname === health.readyPath)) {
+                headers.set("cache-control", "no-store");
+                const report = url.pathname === health.path ? { status: "ok" } : await app.checkReady();
+                const response = json(report, report.status === "not_ready" ? 503 : 200, headers);
+                return method === "HEAD" ? new Response(null, { status: response.status, headers: response.headers }) : response;
+            }
+            if (metricsConfig && url.pathname === metricsConfig.path && (method === "GET" || method === "HEAD")) {
+                headers.set("cache-control", "no-store");
+                if (metricsConfig.token && !hasMetricsToken(req, metricsConfig.token)) {
+                    headers.set("www-authenticate", 'Bearer realm="metrics"');
+                    throw new InferloError("IL3001", "Sign in required.", { status: 401, fix: "Send the metrics token as Authorization: Bearer <token> (bearer_token in the Prometheus scrape config)." });
+                }
+                headers.set("content-type", "text/plain; version=0.0.4; charset=utf-8");
+                return new Response(method === "HEAD" ? null : metrics.render(), { status: 200, headers });
+            }
+            if (refusing) {
+                headers.set("connection", "close");
+                throw new InferloError("IL2006", "The server is shutting down.", { status: 503, fix: "Retry the request; another instance will answer it." });
+            }
+            if (method === "GET" && versionDocs.has(url.pathname)) {
+                seen.route = url.pathname;
+                return json(app.openapi({ version: versionDocs.get(url.pathname) }), 200, headers);
+            }
+            if (openapiPath && url.pathname === openapiPath && method === "GET") {
+                seen.route = openapiPath;
                 return json(app.openapi(), 200, headers);
-            if (mcpPath && url.pathname === mcpPath)
-                return await handleMcp(app, req, headers, testUsers);
+            }
+            if (mcpPath && url.pathname === mcpPath) {
+                seen.route = mcpPath;
+                // tools/list leaves out tools that need a role or permission the caller does not have.
+                let caller;
+                const visible = async (def) => {
+                    const level = def.options.auth ?? "user";
+                    if (!def.options.permission && (level === "public" || level === "user"))
+                        return true;
+                    caller ??= testUsers.has(req) ? Promise.resolve(testUsers.get(req) ?? null) : config.auth ? Promise.resolve(config.auth.resolve(req, { db: db, url })).then((u) => u ?? null) : Promise.resolve(null);
+                    const user = await caller;
+                    if (!user)
+                        return false;
+                    if (level !== "public" && level !== "user" && !roles(user).includes(level))
+                        return false;
+                    if (!def.options.permission)
+                        return true;
+                    return config.permissions.can(def.options.permission, { user }).catch(() => false);
+                };
+                return await handleMcp(app, req, headers, testUsers, visible);
+            }
             const lookup = method === "HEAD" ? "GET" : method;
             let match;
             const allowed = [];
@@ -255,131 +449,322 @@ export function createApp(config = {}) {
                 throw new InferloError("IL2001", `No route matches ${method} ${url.pathname}.${near ? ` Closest route: ${near}.` : ""}`, { status: 404 });
             }
             const { def, keys } = match.route;
+            seen.route = def.path;
+            // Deprecation headers go on every answer of the route, errors included.
+            for (const [name, value] of deprecations.get(def)?.headers ?? [])
+                headers.set(name, value);
             const opts = def.options;
-            // Auth
-            let user = null;
-            if (testUsers.has(req))
-                user = testUsers.get(req) ?? null;
-            else if (config.auth)
-                user = (await config.auth.resolve(req, { db: db, url })) ?? null;
-            const level = opts.auth ?? "user";
-            if (level !== "public") {
-                if (!user)
-                    throw new InferloError("IL3001", "Sign in required.", { status: 401, fix: "Send valid credentials, e.g. an Authorization: Bearer <token> header." });
-                if (level !== "user" && !roles(user).includes(level))
-                    throw new InferloError("IL3002", `This requires the "${level}" role.`, { status: 403 });
-            }
-            // Rate limit: counted per user (or IP without one), after auth so failed sign-ins on private routes do not use it up.
-            if (opts.rateLimit) {
-                const who = (opts.rateLimit.by ?? "user") === "user" && user ? `user:${user.id}` : `ip:${ip}`;
-                const hit = await rateLimits.hit(`${def.method} ${def.path} ${who}`, windowMs(opts.rateLimit));
-                const limited = rateLimitResult(opts.rateLimit, hit);
-                for (const [k, v] of Object.entries(limited.headers))
-                    headers.set(k, v);
-                if (limited.error)
-                    throw limited.error;
-            }
-            // Input
-            const rawParams = {};
-            keys.forEach((key, i) => {
-                const v = match.values[i];
-                if (v !== undefined)
-                    rawParams[key] = decodeURIComponent(v);
-            });
-            const params = opts.params ? parseOrThrow(opts.params, rawParams, "path parameters", true) : rawParams;
-            const rawQuery = queryObject(url);
-            const query = opts.query ? parseOrThrow(opts.query.strip(), rawQuery, "query string", true) : rawQuery;
-            let body;
-            if (opts.body) {
-                const text = method === "GET" || method === "HEAD" ? "" : await readBody(req, maxBody);
-                const type = req.headers.get("content-type") ?? "";
-                let raw = undefined;
-                if (text) {
-                    if (type.includes("application/x-www-form-urlencoded"))
-                        raw = Object.fromEntries(new URLSearchParams(text));
-                    else {
-                        try {
-                            raw = JSON.parse(text);
+            const limit = opts.timeoutMs === undefined ? requestTimeoutMs : opts.timeoutMs;
+            // timeout aborts when the route takes too long; signal (ctx.signal) also when the client disconnects.
+            const timeout = new AbortController();
+            const signal = AbortSignal.any([req.signal, timeout.signal]);
+            let requestDb;
+            const run = async () => {
+                // Auth
+                let user = null;
+                if (testUsers.has(req))
+                    user = testUsers.get(req) ?? null;
+                else if (config.auth)
+                    user = (await config.auth.resolve(req, { db: db, url })) ?? null;
+                const level = opts.auth ?? "user";
+                if (level !== "public") {
+                    if (!user)
+                        throw new InferloError("IL3001", "Sign in required.", { status: 401, fix: "Send valid credentials, e.g. an Authorization: Bearer <token> header." });
+                    if (level !== "user" && !roles(user).includes(level))
+                        throw new InferloError("IL3002", `This requires the "${level}" role.`, { status: 403 });
+                }
+                if (opts.permission)
+                    await config.permissions.authorize(opts.permission, { user });
+                // Rate limit: counted per user (or IP without one), after auth so failed sign-ins on private routes do not use it up.
+                if (opts.rateLimit) {
+                    const who = (opts.rateLimit.by ?? "user") === "user" && user ? `user:${user.id}` : `ip:${ip}`;
+                    const hit = await rateLimits.hit(`${def.method} ${def.path} ${who}`, windowMs(opts.rateLimit));
+                    const limited = rateLimitResult(opts.rateLimit, hit);
+                    for (const [k, v] of Object.entries(limited.headers))
+                        headers.set(k, v);
+                    if (limited.error)
+                        throw limited.error;
+                }
+                // Input
+                const rawParams = {};
+                keys.forEach((key, i) => {
+                    const v = match.values[i];
+                    if (v !== undefined)
+                        rawParams[key] = decodeURIComponent(v);
+                });
+                const params = opts.params ? parseOrThrow(opts.params, rawParams, "path parameters", true) : rawParams;
+                const rawQuery = queryObject(url);
+                const query = opts.query ? parseOrThrow(opts.query.strip(), rawQuery, "query string", true) : rawQuery;
+                let body;
+                if (opts.body) {
+                    const text = method === "GET" || method === "HEAD" ? "" : await readBody(req, maxBody, timeout.signal);
+                    const type = req.headers.get("content-type") ?? "";
+                    let raw = undefined;
+                    if (text) {
+                        if (type.includes("application/x-www-form-urlencoded")) {
+                            // _csrf is the sessions() CSRF field of HTML forms, checked before the handler and not part of the body.
+                            const form = new URLSearchParams(text);
+                            form.delete("_csrf");
+                            raw = Object.fromEntries(form);
                         }
-                        catch {
-                            throw new InferloError("IL2003", "Request body is not valid JSON.", { status: 400, fix: "Send JSON with Content-Type: application/json." });
+                        else {
+                            try {
+                                raw = JSON.parse(text);
+                            }
+                            catch {
+                                throw new InferloError("IL2003", "Request body is not valid JSON.", { status: 400, fix: "Send JSON with Content-Type: application/json." });
+                            }
                         }
                     }
+                    body = parseOrThrow(opts.body, raw, "request body", type.includes("application/x-www-form-urlencoded"));
                 }
-                body = parseOrThrow(opts.body, raw, "request body", type.includes("application/x-www-form-urlencoded"));
-            }
-            let status = 0;
-            const ctx = {
-                body,
-                query,
-                params,
-                user,
-                db: db,
-                ai: toolsFor(req),
-                jobs,
-                req,
-                url,
-                headers: req.headers,
-                state,
-                status: (code) => void (status = code),
-                header: (name, value) => headers.set(name, value),
-                waitUntil: (p) => void pending.push(p.catch((e) => logger.error("waitUntil task failed", e))),
+                // Response cache: after auth, permissions, rate limits and input checks, so a cached response is only
+                // served to callers who may call the route with valid input.
+                const routeCache = opts.cache;
+                const cacheKey = routeCache ? await routeCacheKey(def.method, def.path, routeCache, user?.id, url) : undefined;
+                const cacheHeaders = (outcome, etag) => {
+                    headers.set("x-inferlo-cache", outcome);
+                    headers.set("etag", etag);
+                    if (!headers.has("cache-control"))
+                        headers.set("cache-control", `${(routeCache.by ?? "user") === "public" && (opts.auth ?? "user") === "public" ? "public" : "private"}, no-cache`);
+                    if ((routeCache.by ?? "user") === "user")
+                        headers.append("vary", "Authorization, Cookie");
+                };
+                const cachedAnswer = (body, etag) => {
+                    if (etagMatches(req.headers.get("if-none-match"), etag))
+                        return new Response(null, { status: 304, headers });
+                    headers.set("content-type", "application/json; charset=utf-8");
+                    return new Response(method === "HEAD" ? null : body, { status: 200, headers });
+                };
+                if (cacheKey) {
+                    const hit = await cache.get(cacheKey);
+                    if (hit) {
+                        for (const [name, value] of hit.headers)
+                            headers.set(name, value);
+                        cacheHeaders("HIT", hit.etag);
+                        return cachedAnswer(JSON.stringify(hit.body), hit.etag);
+                    }
+                }
+                const handlerHeaders = [];
+                let status = 0;
+                const ctx = {
+                    body,
+                    query,
+                    params,
+                    user,
+                    get db() {
+                        // Built on first use: after a timeout, new queries fail with IL2007 instead of running.
+                        return (requestDb ??= bindDb(models, guardAdapter(adapter, timeout.signal)));
+                    },
+                    ai: toolsFor(req, signal),
+                    signal,
+                    requestId,
+                    jobs,
+                    cache,
+                    req,
+                    url,
+                    headers: req.headers,
+                    state,
+                    status: (code) => void (status = code),
+                    header: (name, value) => {
+                        // Set-Cookie is added, not replaced, so a handler can set several cookies.
+                        if (name.toLowerCase() === "set-cookie")
+                            headers.append(name, value);
+                        else
+                            headers.set(name, value);
+                        handlerHeaders.push([name.toLowerCase(), value]);
+                    },
+                    waitUntil: (p) => void pending.push(p.catch((e) => logger.error("waitUntil task failed", e, { requestId }))),
+                    audit: createAudit(models, redactor, () => ctx.db, { userId: user?.id ?? null, source: `${def.method} ${def.path}`, requestId }),
+                };
+                const result = await def.handler(ctx);
+                if (result instanceof Response) {
+                    headers.forEach((v, k) => {
+                        if (k !== "set-cookie" && !result.headers.has(k))
+                            result.headers.set(k, v);
+                    });
+                    for (const cookie of headers.getSetCookie())
+                        result.headers.append("set-cookie", cookie);
+                    return result;
+                }
+                if (result === undefined)
+                    return new Response(null, { status: status || 204, headers });
+                // Only plain 200 JSON answers are cached, never one that sets a cookie.
+                if (cacheKey && (status || 200) === 200 && !handlerHeaders.some(([name]) => name === "set-cookie")) {
+                    const body = JSON.stringify(result);
+                    const etag = await etagFor(body);
+                    await cache.set(cacheKey, { body: JSON.parse(body), headers: handlerHeaders, etag }, { ttlMs: routeCache.ttlMs, tags: routeCache.tags ?? [] });
+                    cacheHeaders("MISS", etag);
+                    return cachedAnswer(body, etag);
+                }
+                const response = json(result, status || 200, headers);
+                return method === "HEAD" ? new Response(null, { status: response.status, headers: response.headers }) : response;
             };
-            const result = await def.handler(ctx);
-            if (result instanceof Response) {
-                headers.forEach((v, k) => {
-                    if (!result.headers.has(k))
-                        result.headers.set(k, v);
-                });
-                return result;
+            if (limit === false)
+                return await run();
+            let timer;
+            const running = run();
+            // After a timeout the handler keeps running; its later result or error is dropped.
+            running.catch(() => { });
+            try {
+                return await Promise.race([
+                    running,
+                    new Promise((_, reject) => {
+                        timer = setTimeout(() => {
+                            const error = new InferloError("IL2007", `The request took longer than ${limit} ms.`, {
+                                status: 503,
+                                expose: true,
+                                fix: `Make the handler faster or move slow work to a background job; raise the route's timeoutMs or createApp({ requestTimeoutMs }) if it needs more time.`,
+                            });
+                            timeout.abort(error);
+                            reject(error);
+                        }, limit);
+                    }),
+                ]);
+            }
+            finally {
+                clearTimeout(timer);
             }
-            if (result === undefined)
-                return new Response(null, { status: status || 204, headers });
-            const response = json(result, status || 200, headers);
-            return method === "HEAD" ? new Response(null, { status: response.status, headers: response.headers }) : response;
         }
         catch (error) {
-            return errorResponse(error, headers, req);
+            return errorResponse(error, headers, req, requestId);
         }
     };
+    // Shutdown state: closing makes /ready answer 503, refusing (after delayMs) answers new requests with 503.
+    let closing = false;
+    let refusing = false;
+    let closed;
+    let inFlight = 0;
+    const tasks = new Set();
+    const servers = new Set();
+    let idle = [];
+    const settle = () => {
+        if (inFlight > 0 || tasks.size > 0)
+            return;
+        for (const resolve of idle)
+            resolve();
+        idle = [];
+    };
+    const whenIdle = () => new Promise((resolve) => {
+        idle.push(resolve);
+        settle();
+    });
+    /** Resolves true when promise settled before ms passed, false otherwise. */
+    const within = async (promise, ms) => {
+        let timer;
+        try {
+            return await Promise.race([promise.then(() => true, () => true), new Promise((resolve) => (timer = setTimeout(() => resolve(false), ms)))]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    };
+    async function handle(req, rest) {
+        const url = new URL(req.url);
+        // Health and metrics endpoints are called every few seconds by load balancers and scrapers: not logged, counted or traced.
+        const probe = url.pathname === health?.path || url.pathname === health?.readyPath || url.pathname === metricsConfig?.path;
+        if (!tracing || probe)
+            return handleTraced(req, rest, url, probe, undefined);
+        return tracing.run(req, url, async (span) => {
+            requestSpans.set(req, span);
+            let status = 500;
+            try {
+                const response = await handleTraced(req, rest, url, probe, span);
+                status = response.status;
+                return response;
+            }
+            finally {
+                span.setAttribute("http.response.status_code", status);
+                if (status >= 500)
+                    span.setStatus({ code: tracingApi.SpanStatusCode.ERROR });
+                span.end();
+            }
+        });
+    }
+    async function handleTraced(req, rest, url, probe, span) {
+        const started = performance.now();
+        // Behind a trusted proxy, keep the id it sent so log entries line up across services.
+        const incoming = config.trustProxy ? req.headers.get("x-request-id") : null;
+        const requestId = incoming && REQUEST_ID.test(incoming) ? incoming : crypto.randomUUID();
+        const ip = clientIp(req, Boolean(config.trustProxy), rest[0]);
+        if (ip !== "unknown")
+            clientIps.set(req, ip);
+        const state = {};
+        const pending = [];
+        const chain = [...(config.middleware ?? [])];
+        // The matched route pattern, for metrics ("unmatched" for 404s and responses from middleware).
+        const seen = { route: "unmatched" };
+        const run = (i) => {
+            const mw = chain[i];
+            if (!mw)
+                return dispatch(req, url, state, pending, ip, requestId, seen);
+            return mw({ req, url, state, requestId }, () => run(i + 1));
+        };
+        let response;
+        try {
+            response = await run(0);
+        }
+        catch (error) {
+            response = errorResponse(error, new Headers(), req, requestId);
+        }
+        response = withHeader(response, "x-request-id", requestId);
+        if (span && seen.route !== "unmatched") {
+            span.updateName(`${req.method.toUpperCase()} ${seen.route}`);
+            span.setAttribute("http.route", seen.route);
+        }
+        if (!probe) {
+            const elapsed = performance.now() - started;
+            const method = req.method.toUpperCase();
+            metrics?.request(method, seen.route, response.status, elapsed / 1000);
+            if (logging.requests) {
+                const durationMs = Math.round(elapsed * 10) / 10;
+                logger.info(`${method} ${url.pathname} ${response.status} ${durationMs}ms`, undefined, { requestId, ...traceFields(span), method, path: url.pathname, status: response.status, durationMs });
+            }
+        }
+        if (secureHeaders.length) {
+            const https = url.protocol === "https:" || (Boolean(config.trustProxy) && req.headers.get("x-forwarded-proto")?.split(",")[0]?.trim() === "https");
+            response = applySecureHeaders(response, secureHeaders, https);
+        }
+        if (pending.length) {
+            const all = Promise.all(pending);
+            const execution = rest[1];
+            if (execution?.waitUntil)
+                execution.waitUntil(all);
+            else {
+                // Kept until it settles, so app.close() waits for it.
+                tasks.add(all);
+                void all.finally(() => {
+                    tasks.delete(all);
+                    settle();
+                });
+            }
+        }
+        return response;
+    }
     const app = {
         name: config.name ?? "inferlo-app",
         version: config.version ?? "0.0.0",
         routes,
         models,
+        modules,
         db,
         ai,
         jobs,
+        cache,
+        get closing() {
+            return closing;
+        },
         async fetch(req, ...rest) {
-            const url = new URL(req.url);
-            const ip = clientIp(req, Boolean(config.trustProxy), rest[0]);
-            if (ip !== "unknown")
-                clientIps.set(req, ip);
-            const state = {};
-            const pending = [];
-            const chain = [...(config.middleware ?? [])];
-            const run = (i) => {
-                const mw = chain[i];
-                if (!mw)
-                    return dispatch(req, url, state, pending, ip);
-                return mw({ req, url, state }, () => run(i + 1));
-            };
-            let response;
+            inFlight++;
+            metrics?.inFlight(1);
             try {
-                response = await run(0);
-            }
-            catch (error) {
-                response = errorResponse(error, new Headers(), req);
+                return await handle(req, rest);
             }
-            if (pending.length) {
-                const all = Promise.all(pending);
-                const execution = rest[1];
-                if (execution?.waitUntil)
-                    execution.waitUntil(all);
-                else
-                    void all;
+            finally {
+                inFlight--;
+                metrics?.inFlight(-1);
+                settle();
             }
-            return response;
         },
         async request(path, options = {}) {
             const url = new URL(path, "http://inferlo.test");
@@ -399,17 +784,143 @@ export function createApp(config = {}) {
         },
         async listen(port = 3000, hostname = "0.0.0.0") {
             const { serve } = await import("./serve.js");
-            return serve(app, { port, hostname });
+            const server = await serve(app, { port, hostname });
+            servers.add(server);
+            return {
+                port: server.port,
+                close: async () => {
+                    servers.delete(server);
+                    await server.close();
+                },
+            };
+        },
+        async checkReady() {
+            if (closing)
+                return { status: "not_ready", checks: { shutdown: { status: "error", durationMs: 0, error: "The app is shutting down." } } };
+            const names = Object.keys(readinessChecks);
+            const results = await Promise.all(names.map((name) => runCheck(name, readinessChecks[name])));
+            const checks = {};
+            names.forEach((name, i) => (checks[name] = results[i]));
+            return { status: results.every((r) => r.status === "ok") ? "ready" : "not_ready", checks };
+        },
+        close(options = {}) {
+            if (closed)
+                return closed;
+            const delayMs = options.delayMs ?? 0;
+            const timeoutMs = options.timeoutMs ?? 10_000;
+            for (const [key, value] of [["delayMs", delayMs], ["timeoutMs", timeoutMs]]) {
+                if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+                    throw new InferloError("IL6015", `app.close({ ${key} }) must be a number of milliseconds of 0 or more, got ${String(value)}.`, {
+                        fix: "Use app.close({ delayMs: 5000, timeoutMs: 10000 }) or app.close().",
+                    });
+                }
+            }
+            closing = true;
+            closed = (async () => {
+                if (delayMs > 0)
+                    await new Promise((resolve) => setTimeout(resolve, delayMs));
+                refusing = true;
+                const deadline = Date.now() + timeoutMs;
+                let workersStopped = false;
+                const drained = Promise.all([whenIdle(), stopWorkers(jobs).then(() => void (workersStopped = true))]);
+                if (!(await within(drained, timeoutMs))) {
+                    const left = [`${inFlight} request(s)`, `${tasks.size} waitUntil task(s)`];
+                    if (!workersStopped)
+                        left.push("the running job");
+                    logger.error(`app.close(): stopped waiting after ${timeoutMs} ms with ${left.join(", ")} still running`);
+                }
+                const { forceClosers } = servers.size ? await import("./serve.js") : { forceClosers: undefined };
+                await Promise.all([...servers].map(async (server) => {
+                    servers.delete(server);
+                    if (!(await within(server.close(), Math.max(0, deadline - Date.now()))))
+                        forceClosers?.get(server)?.();
+                }));
+                await db.$close();
+            })();
+            return closed;
         },
         describe() {
-            return describeApp(app, { auth: Boolean(config.auth), openapi: openapiPath, mcp: mcpPath, description: config.description });
+            return describeApp(app, {
+                auth: Boolean(config.auth),
+                openapi: openapiPath,
+                mcp: mcpPath,
+                health: health?.path ?? false,
+                ready: health?.readyPath ?? false,
+                metrics: metricsConfig?.path ?? false,
+                secureHeaders: secureHeaders.map(([name]) => name),
+                requestTimeoutMs,
+                logging: { format: config.logger ? "custom" : logging.format, requests: logging.requests },
+                description: config.description,
+                modules: moduleInfo,
+                versions: versions.map((v) => ({ name: v, openapi: openapiPath ? versionDocPath(v, openapiPath) : false })),
+                permissions: config.permissions?.list.map((p) => (p.roles ? { name: p.name, roles: [...p.roles] } : { name: p.name })) ?? [],
+            });
         },
-        openapi() {
-            return buildOpenAPI(app, { description: config.description, auth: Boolean(config.auth) });
+        openapi(options = {}) {
+            if (options.version !== undefined && !versions.includes(options.version)) {
+                throw new InferloError("IL6025", `The app has no API version ${JSON.stringify(options.version)}.${versions.length ? ` Its versions: ${versions.join(", ")}.` : " It has no apiVersion() routes."}`, {
+                    fix: 'Pass a version used in apiVersion("v1", routes), or call app.openapi() for every route.',
+                });
+            }
+            return buildOpenAPI(app, { description: config.description, auth: Boolean(config.auth), modules: moduleInfo, version: options.version });
         },
     };
+    // Built-in endpoints with the methods dispatch answers them for (MCP takes every method).
+    const readEndpoints = [health?.path, health?.readyPath, metricsConfig?.path].filter((p) => typeof p === "string");
+    registerRequestMatcher(app, (rawMethod, pathname) => {
+        const method = rawMethod.toUpperCase();
+        if (mcpPath && pathname === mcpPath)
+            return true;
+        if (method === "GET" && ((openapiPath && pathname === openapiPath) || versionDocs.has(pathname)))
+            return true;
+        if ((method === "GET" || method === "HEAD") && readEndpoints.includes(pathname))
+            return true;
+        const lookup = method === "HEAD" ? "GET" : method;
+        // A CORS preflight belongs to the app when any of its routes has this path.
+        return compiled.some((c) => (method === "OPTIONS" || c.def.method === lookup) && c.regex.test(pathname));
+    });
     return app;
 }
+function resolveHealth(option, routes, taken) {
+    if (option === undefined || option === false)
+        return undefined;
+    const bad = (message, fix) => new InferloError("IL6014", message, { fix });
+    if (option !== true && (typeof option !== "object" || option === null))
+        throw bad("health must be true or an object.", "Use createApp({ health: true }) or createApp({ health: { checks: { ... } } }).");
+    const o = option === true ? {} : option;
+    const resolved = { path: o.path ?? "/health", readyPath: o.readyPath ?? "/ready", timeoutMs: o.timeoutMs ?? 5000, checks: o.checks ?? {} };
+    for (const [key, value] of [["path", resolved.path], ["readyPath", resolved.readyPath]]) {
+        if (typeof value !== "string" || !value.startsWith("/"))
+            throw bad(`health.${key} must be a path starting with "/", got ${JSON.stringify(value)}.`, `Use health: { ${key}: "/${String(value).replace(/^\/+/, "")}" }.`);
+    }
+    if (resolved.path === resolved.readyPath)
+        throw bad(`health.path and health.readyPath are both "${resolved.path}".`, 'Use different paths, for example "/health" and "/ready".');
+    if (!Number.isFinite(resolved.timeoutMs) || resolved.timeoutMs <= 0)
+        throw bad(`health.timeoutMs must be a positive number of milliseconds, got ${String(resolved.timeoutMs)}.`, "Use health: { timeoutMs: 5000 }.");
+    if (typeof resolved.checks !== "object" || resolved.checks === null)
+        throw bad("health.checks must be an object of functions.", "Use health: { checks: { redis: () => redis.ping() } }.");
+    for (const [name, check] of Object.entries(resolved.checks)) {
+        if (name === "database" || name === "ai")
+            throw bad(`health.checks.${name} is built in.`, `Give your check another name, for example "${name}Replica".`);
+        if (typeof check !== "function")
+            throw bad(`health.checks.${name} must be a function.`, `Use health: { checks: { ${name}: async () => { ... } } }; throw when it is not ready.`);
+    }
+    for (const path of [resolved.path, resolved.readyPath]) {
+        if (taken.includes(path))
+            throw bad(`The health endpoint ${path} is also the OpenAPI or MCP path.`, "Choose another health.path or health.readyPath.");
+        const route = routes.find((r) => r.method === "GET" && r.path.replace(/\/+$/, "") === path.replace(/\/+$/, ""));
+        if (route)
+            throw bad(`The health endpoint ${path} is also the route GET ${route.path}.`, `Remove the GET ${route.path} route (createApp({ health }) answers it now), or choose another health.path or health.readyPath.`);
+    }
+    return resolved;
+}
+function checkTimeout(value, where) {
+    if (value === false || (typeof value === "number" && Number.isFinite(value) && value > 0))
+        return value;
+    throw new InferloError("IL6017", `${where} must be a positive number of milliseconds or false, got ${typeof value === "string" ? JSON.stringify(value) : String(value)}.`, {
+        fix: "Use a number such as 30000 (30 seconds), or false for no timeout.",
+    });
+}
 function parseOrThrow(schema, value, where, coerce) {
     const result = schema.safeParse(value, { coerce });
     if (!result.ok)