@browserlessapi/inferlo 0.4.0 → 0.5.0

package/dist/app.js

@@ -4,7 +4,7 @@
  */
 import { bindTools, createAI } from "./ai/index.js";
 import { memory } from "./db/memory.js";
-import { createDb } from "./db/table.js";
+import { bindDb, createDb, guardAdapter } from "./db/table.js";
 import { describeApp } from "./describe.js";
 import { InferloError, didYouMean, suggest } from "./errors.js";
 import { callTool, handleMcp, toolDefinition } from "./mcp.js";
@@ -12,8 +12,14 @@ import { buildOpenAPI } from "./openapi.js";
 import { isRoute } from "./route.js";
 import { checkRateLimit, databaseRateLimitStore, memoryRateLimitStore, rateLimitResult, windowMs } from "./rate-limit.js";
 import { clientIp, clientIps } from "./client-ip.js";
-import { collectCrons, collectJobs, createJobs, jobStoreFor } from "./jobs.js";
+import { collectCrons, collectJobs, createJobs, jobStoreFor, stopWorkers } from "./jobs.js";
 import { ValidationError } from "./schema.js";
+import { createRedactor, redactingLogger, redactValues, secretsOf } from "./redact.js";
+import { defaultLogger, loggingOptions, REQUEST_ID } from "./logging.js";
+import { resolveDeprecation } from "./deprecation.js";
+import { createMetrics, hasMetricsToken, resolveMetrics } from "./metrics.js";
+import { INFERLO_VERSION } from "./describe.js";
+import { applySecureHeaders, resolveSecureHeaders } from "./secure-headers.js";
 const testUsers = new WeakMap();
 function compile(def) {
     if (!def.path.startsWith("/")) {
@@ -71,13 +77,15 @@ function corsHeaders(config, origin) {
         h["access-control-allow-credentials"] = "true";
     return h;
 }
-async function readBody(req, limit) {
+async function readBody(req, limit, signal) {
     const length = Number(req.headers.get("content-length") ?? "0");
     if (length > limit)
         throw new InferloError("IL2004", `Request body is larger than ${limit} bytes.`, { status: 413 });
     if (!req.body)
         return "";
     const reader = req.body.getReader();
+    // Stop reading a slow body once the request timed out.
+    signal?.addEventListener("abort", () => void reader.cancel().catch(() => { }), { once: true });
     const chunks = [];
     let size = 0;
     while (true) {
@@ -107,15 +115,29 @@ function queryObject(url) {
     }
     return out;
 }
+/** Sets a header unless the response already has it; copies responses with immutable headers. */
+function withHeader(response, name, value) {
+    if (response.headers.has(name))
+        return response;
+    try {
+        response.headers.set(name, value);
+        return response;
+    }
+    catch {
+        const copy = new Response(response.body, response);
+        copy.headers.set(name, value);
+        return copy;
+    }
+}
 function json(data, status, headers) {
     headers.set("content-type", "application/json; charset=utf-8");
     return new Response(JSON.stringify(data), { status, headers });
 }
 export function createApp(config = {}) {
-    const logger = config.logger ?? {
-        info: (m, d) => console.log(`[inferlo] ${m}`, d ?? ""),
-        error: (m, d) => console.error(`[inferlo] ${m}`, d ?? ""),
-    };
+    // Everything the app logs goes through the redactor: credentials and secrets never reach the logs.
+    const redactor = createRedactor([...redactValues(config.redact), ...Object.values(config.ai?.providers ?? {}).flatMap((p) => secretsOf(p))]);
+    const logging = loggingOptions(config.logging, config.logger !== undefined);
+    const logger = redactingLogger(config.logger ?? defaultLogger(logging.format, config.name ?? "inferlo-app"), redactor);
     const models = (config.models ?? []);
     const modelNames = new Set();
     for (const m of models) {
@@ -161,20 +183,101 @@ export function createApp(config = {}) {
     for (const r of routes)
         if (r.options.rateLimit !== undefined)
             checkRateLimit(r.options.rateLimit, `${r.method} ${r.path}`);
+    const requestTimeoutMs = config.requestTimeoutMs === undefined ? 60_000 : checkTimeout(config.requestTimeoutMs, "createApp({ requestTimeoutMs })");
+    for (const r of routes)
+        if (r.options.timeoutMs !== undefined)
+            checkTimeout(r.options.timeoutMs, `${r.method} ${r.path} { timeoutMs }`);
+    const deprecations = new Map();
+    for (const r of routes) {
+        const resolved = resolveDeprecation(r.options.deprecated, `${r.method} ${r.path}`);
+        if (resolved)
+            deprecations.set(r, resolved);
+    }
     const storeName = config.rateLimit?.store ?? "memory";
     if (storeName !== "memory" && storeName !== "database") {
         throw new InferloError("IL6007", `Invalid rateLimit store "${String(storeName)}".`, { fix: 'Use createApp({ rateLimit: { store: "memory" } }) or { store: "database" }.' });
     }
+    const openapiPath = config.openapi === undefined ? "/openapi.json" : config.openapi;
+    const hasTools = routes.some((r) => r.options.tool);
+    const mcpPath = config.mcp === undefined ? (hasTools ? "/mcp" : false) : config.mcp;
+    const health = resolveHealth(config.health, routes, [openapiPath, mcpPath]);
+    const metricsConfig = resolveMetrics(config.metrics, [
+        { path: openapiPath, what: "the OpenAPI path" },
+        { path: mcpPath, what: "the MCP path" },
+        { path: health?.path ?? false, what: "the health path" },
+        { path: health?.readyPath ?? false, what: "the readiness path" },
+        ...routes.filter((r) => r.method === "GET").map((r) => ({ path: r.path, what: `the route GET ${r.path}` })),
+    ]);
     const adapter = config.db ?? memory();
     const db = createDb(models, adapter);
     const rateLimits = storeName === "database" ? databaseRateLimitStore(adapter) : memoryRateLimitStore();
-    const ai = createAI(config.ai);
-    const jobs = createJobs(collectJobs(config.jobs), { db: db, ai, store: jobStoreFor(config.jobQueue, adapter), queue: config.jobQueue?.store ?? "memory", logger }, collectCrons(config.jobs));
+    const metrics = metricsConfig
+        ? createMetrics({ app: config.name ?? "inferlo-app", version: config.version ?? "0.0.0", inferlo: INFERLO_VERSION }, metricsConfig.buckets)
+        : undefined;
+    // With metrics on, every AI call that reports usage is counted before the app's own onUsage runs.
+    const aiConfig = metrics && config.ai
+        ? { ...config.ai, onUsage: (usage) => (metrics.ai(usage), config.ai.onUsage?.(usage)) }
+        : config.ai;
+    const ai = createAI(aiConfig);
+    const jobs = createJobs(collectJobs(config.jobs), {
+        db: db,
+        ai,
+        store: jobStoreFor(config.jobQueue, adapter),
+        queue: config.jobQueue?.store ?? "memory",
+        logger,
+        onRun: metrics ? (name, outcome, ms) => metrics.job(name, outcome, ms / 1000) : undefined,
+    }, collectCrons(config.jobs));
     const maxBody = config.maxBodySize ?? 1024 * 1024;
+    const secureHeaders = resolveSecureHeaders(config.secureHeaders);
     const cors = config.cors === true ? { origin: "*" } : config.cors || undefined;
-    const openapiPath = config.openapi === undefined ? "/openapi.json" : config.openapi;
-    const hasTools = routes.some((r) => r.options.tool);
-    const mcpPath = config.mcp === undefined ? (hasTools ? "/mcp" : false) : config.mcp;
+    // Readiness: the database answers, every AI alias points to a configured provider, and the app's own checks pass.
+    const readinessChecks = {
+        database: async () => {
+            if (adapter.sql)
+                await adapter.sql("select 1");
+        },
+    };
+    const aiProviders = Object.keys(config.ai?.providers ?? {});
+    if (aiProviders.length || Object.keys(config.ai?.models ?? {}).length) {
+        readinessChecks.ai = () => {
+            for (const [alias, targets] of Object.entries(ai.aliases)) {
+                for (const target of targets) {
+                    const provider = target.split(":")[0];
+                    if (!aiProviders.includes(provider)) {
+                        throw new InferloError("IL5001", `AI alias "${alias}" uses provider "${provider}", which is not configured.`, { status: 503, expose: true });
+                    }
+                }
+            }
+        };
+    }
+    Object.assign(readinessChecks, health?.checks);
+    const runCheck = async (name, check) => {
+        const started = Date.now();
+        const timeoutMs = health?.timeoutMs ?? 5000;
+        let timer;
+        const timedOut = Symbol("timeout");
+        try {
+            const outcome = await Promise.race([
+                Promise.resolve().then(check),
+                new Promise((resolve) => {
+                    timer = setTimeout(() => resolve(timedOut), timeoutMs);
+                }),
+            ]);
+            if (outcome === timedOut) {
+                logger.error(`Readiness check "${name}" timed out after ${timeoutMs} ms`);
+                return { status: "error", durationMs: Date.now() - started, error: `Timed out after ${timeoutMs} ms.` };
+            }
+            return { status: "ok", durationMs: Date.now() - started };
+        }
+        catch (error) {
+            logger.error(`Readiness check "${name}" failed`, error);
+            const shown = (error instanceof InferloError && error.expose) || (config.exposeErrors && error instanceof Error);
+            return { status: "error", durationMs: Date.now() - started, error: shown ? redactor.text(error.message) : "Check failed." };
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    };
     // Routes passed to ai.generate({ tools }) are matched to the registered route by method and path.
     const registeredTool = (route) => {
         const found = routes.find((r) => r.method === route?.method && r.path === route?.path);
@@ -189,7 +292,7 @@ export function createApp(config = {}) {
         const { name, description, inputSchema } = registeredTool(route);
         return { name, description, inputSchema };
     };
-    const toolsFor = (req) => bindTools(ai, async (route, input) => {
+    const toolsFor = (req, signal) => bindTools(ai, async (route, input) => {
         const tool = registeredTool(route);
         const response = await callTool(app, tool, input, req, testUsers);
         const text = await response.text();
@@ -201,22 +304,23 @@ export function createApp(config = {}) {
             // Not JSON: the model sees the text as it is.
         }
         return { name: tool.name, status: response.status, output, text: text || String(response.status) };
-    }, describeTool);
+    }, describeTool, signal);
     const roles = (user) => config.auth?.roles?.(user) ?? (user.roles ?? []);
-    const errorResponse = (error, headers, req) => {
+    const errorResponse = (error, headers, req, requestId) => {
+        const fields = { requestId };
         if (error instanceof InferloError) {
             if (error.status >= 500)
-                logger.error(`${req.method} ${new URL(req.url).pathname} failed: ${error.format()}`, error.cause ?? "");
+                logger.error(`${req.method} ${new URL(req.url).pathname} failed: ${error.format()}`, error.cause ?? "", fields);
             const body = error.toJSON();
             if (config.exposeErrors && !error.expose)
-                body.error.message = error.message;
+                body.error.message = redactor.text(error.message);
             return json(body, error.status, headers);
         }
-        logger.error(`${req.method} ${new URL(req.url).pathname} failed`, error);
-        const message = config.exposeErrors && error instanceof Error ? error.message : "Internal server error";
+        logger.error(`${req.method} ${new URL(req.url).pathname} failed`, error, fields);
+        const message = config.exposeErrors && error instanceof Error ? redactor.text(error.message) : "Internal server error";
         return json({ error: { code: "IL9000", message, docs: new InferloError("IL9000", "").docs } }, 500, headers);
     };
-    const dispatch = async (req, url, state, pending, ip) => {
+    const dispatch = async (req, url, state, pending, ip, requestId, seen) => {
         const headers = new Headers();
         if (cors)
             for (const [k, v] of Object.entries(corsHeaders(cors, req.headers.get("origin"))))
@@ -229,10 +333,33 @@ export function createApp(config = {}) {
                 headers.set("access-control-max-age", String(cors.maxAge ?? 600));
                 return new Response(null, { status: 204, headers });
             }
-            if (openapiPath && url.pathname === openapiPath && method === "GET")
+            if (health && (method === "GET" || method === "HEAD") && (url.pathname === health.path || url.pathname === health.readyPath)) {
+                headers.set("cache-control", "no-store");
+                const report = url.pathname === health.path ? { status: "ok" } : await app.checkReady();
+                const response = json(report, report.status === "not_ready" ? 503 : 200, headers);
+                return method === "HEAD" ? new Response(null, { status: response.status, headers: response.headers }) : response;
+            }
+            if (metricsConfig && url.pathname === metricsConfig.path && (method === "GET" || method === "HEAD")) {
+                headers.set("cache-control", "no-store");
+                if (metricsConfig.token && !hasMetricsToken(req, metricsConfig.token)) {
+                    headers.set("www-authenticate", 'Bearer realm="metrics"');
+                    throw new InferloError("IL3001", "Sign in required.", { status: 401, fix: "Send the metrics token as Authorization: Bearer <token> (bearer_token in the Prometheus scrape config)." });
+                }
+                headers.set("content-type", "text/plain; version=0.0.4; charset=utf-8");
+                return new Response(method === "HEAD" ? null : metrics.render(), { status: 200, headers });
+            }
+            if (refusing) {
+                headers.set("connection", "close");
+                throw new InferloError("IL2006", "The server is shutting down.", { status: 503, fix: "Retry the request; another instance will answer it." });
+            }
+            if (openapiPath && url.pathname === openapiPath && method === "GET") {
+                seen.route = openapiPath;
                 return json(app.openapi(), 200, headers);
-            if (mcpPath && url.pathname === mcpPath)
+            }
+            if (mcpPath && url.pathname === mcpPath) {
+                seen.route = mcpPath;
                 return await handleMcp(app, req, headers, testUsers);
+            }
             const lookup = method === "HEAD" ? "GET" : method;
             let match;
             const allowed = [];
@@ -255,93 +382,222 @@ export function createApp(config = {}) {
                 throw new InferloError("IL2001", `No route matches ${method} ${url.pathname}.${near ? ` Closest route: ${near}.` : ""}`, { status: 404 });
             }
             const { def, keys } = match.route;
+            seen.route = def.path;
+            // Deprecation headers go on every answer of the route, errors included.
+            for (const [name, value] of deprecations.get(def)?.headers ?? [])
+                headers.set(name, value);
             const opts = def.options;
-            // Auth
-            let user = null;
-            if (testUsers.has(req))
-                user = testUsers.get(req) ?? null;
-            else if (config.auth)
-                user = (await config.auth.resolve(req, { db: db, url })) ?? null;
-            const level = opts.auth ?? "user";
-            if (level !== "public") {
-                if (!user)
-                    throw new InferloError("IL3001", "Sign in required.", { status: 401, fix: "Send valid credentials, e.g. an Authorization: Bearer <token> header." });
-                if (level !== "user" && !roles(user).includes(level))
-                    throw new InferloError("IL3002", `This requires the "${level}" role.`, { status: 403 });
-            }
-            // Rate limit: counted per user (or IP without one), after auth so failed sign-ins on private routes do not use it up.
-            if (opts.rateLimit) {
-                const who = (opts.rateLimit.by ?? "user") === "user" && user ? `user:${user.id}` : `ip:${ip}`;
-                const hit = await rateLimits.hit(`${def.method} ${def.path} ${who}`, windowMs(opts.rateLimit));
-                const limited = rateLimitResult(opts.rateLimit, hit);
-                for (const [k, v] of Object.entries(limited.headers))
-                    headers.set(k, v);
-                if (limited.error)
-                    throw limited.error;
-            }
-            // Input
-            const rawParams = {};
-            keys.forEach((key, i) => {
-                const v = match.values[i];
-                if (v !== undefined)
-                    rawParams[key] = decodeURIComponent(v);
-            });
-            const params = opts.params ? parseOrThrow(opts.params, rawParams, "path parameters", true) : rawParams;
-            const rawQuery = queryObject(url);
-            const query = opts.query ? parseOrThrow(opts.query.strip(), rawQuery, "query string", true) : rawQuery;
-            let body;
-            if (opts.body) {
-                const text = method === "GET" || method === "HEAD" ? "" : await readBody(req, maxBody);
-                const type = req.headers.get("content-type") ?? "";
-                let raw = undefined;
-                if (text) {
-                    if (type.includes("application/x-www-form-urlencoded"))
-                        raw = Object.fromEntries(new URLSearchParams(text));
-                    else {
-                        try {
-                            raw = JSON.parse(text);
-                        }
-                        catch {
-                            throw new InferloError("IL2003", "Request body is not valid JSON.", { status: 400, fix: "Send JSON with Content-Type: application/json." });
+            const limit = opts.timeoutMs === undefined ? requestTimeoutMs : opts.timeoutMs;
+            // timeout aborts when the route takes too long; signal (ctx.signal) also when the client disconnects.
+            const timeout = new AbortController();
+            const signal = AbortSignal.any([req.signal, timeout.signal]);
+            let requestDb;
+            const run = async () => {
+                // Auth
+                let user = null;
+                if (testUsers.has(req))
+                    user = testUsers.get(req) ?? null;
+                else if (config.auth)
+                    user = (await config.auth.resolve(req, { db: db, url })) ?? null;
+                const level = opts.auth ?? "user";
+                if (level !== "public") {
+                    if (!user)
+                        throw new InferloError("IL3001", "Sign in required.", { status: 401, fix: "Send valid credentials, e.g. an Authorization: Bearer <token> header." });
+                    if (level !== "user" && !roles(user).includes(level))
+                        throw new InferloError("IL3002", `This requires the "${level}" role.`, { status: 403 });
+                }
+                // Rate limit: counted per user (or IP without one), after auth so failed sign-ins on private routes do not use it up.
+                if (opts.rateLimit) {
+                    const who = (opts.rateLimit.by ?? "user") === "user" && user ? `user:${user.id}` : `ip:${ip}`;
+                    const hit = await rateLimits.hit(`${def.method} ${def.path} ${who}`, windowMs(opts.rateLimit));
+                    const limited = rateLimitResult(opts.rateLimit, hit);
+                    for (const [k, v] of Object.entries(limited.headers))
+                        headers.set(k, v);
+                    if (limited.error)
+                        throw limited.error;
+                }
+                // Input
+                const rawParams = {};
+                keys.forEach((key, i) => {
+                    const v = match.values[i];
+                    if (v !== undefined)
+                        rawParams[key] = decodeURIComponent(v);
+                });
+                const params = opts.params ? parseOrThrow(opts.params, rawParams, "path parameters", true) : rawParams;
+                const rawQuery = queryObject(url);
+                const query = opts.query ? parseOrThrow(opts.query.strip(), rawQuery, "query string", true) : rawQuery;
+                let body;
+                if (opts.body) {
+                    const text = method === "GET" || method === "HEAD" ? "" : await readBody(req, maxBody, timeout.signal);
+                    const type = req.headers.get("content-type") ?? "";
+                    let raw = undefined;
+                    if (text) {
+                        if (type.includes("application/x-www-form-urlencoded"))
+                            raw = Object.fromEntries(new URLSearchParams(text));
+                        else {
+                            try {
+                                raw = JSON.parse(text);
+                            }
+                            catch {
+                                throw new InferloError("IL2003", "Request body is not valid JSON.", { status: 400, fix: "Send JSON with Content-Type: application/json." });
+                            }
                         }
                     }
+                    body = parseOrThrow(opts.body, raw, "request body", type.includes("application/x-www-form-urlencoded"));
                 }
-                body = parseOrThrow(opts.body, raw, "request body", type.includes("application/x-www-form-urlencoded"));
-            }
-            let status = 0;
-            const ctx = {
-                body,
-                query,
-                params,
-                user,
-                db: db,
-                ai: toolsFor(req),
-                jobs,
-                req,
-                url,
-                headers: req.headers,
-                state,
-                status: (code) => void (status = code),
-                header: (name, value) => headers.set(name, value),
-                waitUntil: (p) => void pending.push(p.catch((e) => logger.error("waitUntil task failed", e))),
+                let status = 0;
+                const ctx = {
+                    body,
+                    query,
+                    params,
+                    user,
+                    get db() {
+                        // Built on first use: after a timeout, new queries fail with IL2007 instead of running.
+                        return (requestDb ??= bindDb(models, guardAdapter(adapter, timeout.signal)));
+                    },
+                    ai: toolsFor(req, signal),
+                    signal,
+                    requestId,
+                    jobs,
+                    req,
+                    url,
+                    headers: req.headers,
+                    state,
+                    status: (code) => void (status = code),
+                    header: (name, value) => headers.set(name, value),
+                    waitUntil: (p) => void pending.push(p.catch((e) => logger.error("waitUntil task failed", e, { requestId }))),
+                };
+                const result = await def.handler(ctx);
+                if (result instanceof Response) {
+                    headers.forEach((v, k) => {
+                        if (!result.headers.has(k))
+                            result.headers.set(k, v);
+                    });
+                    return result;
+                }
+                if (result === undefined)
+                    return new Response(null, { status: status || 204, headers });
+                const response = json(result, status || 200, headers);
+                return method === "HEAD" ? new Response(null, { status: response.status, headers: response.headers }) : response;
             };
-            const result = await def.handler(ctx);
-            if (result instanceof Response) {
-                headers.forEach((v, k) => {
-                    if (!result.headers.has(k))
-                        result.headers.set(k, v);
-                });
-                return result;
+            if (limit === false)
+                return await run();
+            let timer;
+            const running = run();
+            // After a timeout the handler keeps running; its later result or error is dropped.
+            running.catch(() => { });
+            try {
+                return await Promise.race([
+                    running,
+                    new Promise((_, reject) => {
+                        timer = setTimeout(() => {
+                            const error = new InferloError("IL2007", `The request took longer than ${limit} ms.`, {
+                                status: 503,
+                                expose: true,
+                                fix: `Make the handler faster or move slow work to a background job; raise the route's timeoutMs or createApp({ requestTimeoutMs }) if it needs more time.`,
+                            });
+                            timeout.abort(error);
+                            reject(error);
+                        }, limit);
+                    }),
+                ]);
+            }
+            finally {
+                clearTimeout(timer);
             }
-            if (result === undefined)
-                return new Response(null, { status: status || 204, headers });
-            const response = json(result, status || 200, headers);
-            return method === "HEAD" ? new Response(null, { status: response.status, headers: response.headers }) : response;
         }
         catch (error) {
-            return errorResponse(error, headers, req);
+            return errorResponse(error, headers, req, requestId);
+        }
+    };
+    // Shutdown state: closing makes /ready answer 503, refusing (after delayMs) answers new requests with 503.
+    let closing = false;
+    let refusing = false;
+    let closed;
+    let inFlight = 0;
+    const tasks = new Set();
+    const servers = new Set();
+    let idle = [];
+    const settle = () => {
+        if (inFlight > 0 || tasks.size > 0)
+            return;
+        for (const resolve of idle)
+            resolve();
+        idle = [];
+    };
+    const whenIdle = () => new Promise((resolve) => {
+        idle.push(resolve);
+        settle();
+    });
+    /** Resolves true when promise settled before ms passed, false otherwise. */
+    const within = async (promise, ms) => {
+        let timer;
+        try {
+            return await Promise.race([promise.then(() => true, () => true), new Promise((resolve) => (timer = setTimeout(() => resolve(false), ms)))]);
+        }
+        finally {
+            clearTimeout(timer);
         }
     };
+    async function handle(req, rest) {
+        const started = performance.now();
+        const url = new URL(req.url);
+        // Behind a trusted proxy, keep the id it sent so log entries line up across services.
+        const incoming = config.trustProxy ? req.headers.get("x-request-id") : null;
+        const requestId = incoming && REQUEST_ID.test(incoming) ? incoming : crypto.randomUUID();
+        const ip = clientIp(req, Boolean(config.trustProxy), rest[0]);
+        if (ip !== "unknown")
+            clientIps.set(req, ip);
+        const state = {};
+        const pending = [];
+        const chain = [...(config.middleware ?? [])];
+        // The matched route pattern, for metrics ("unmatched" for 404s and responses from middleware).
+        const seen = { route: "unmatched" };
+        const run = (i) => {
+            const mw = chain[i];
+            if (!mw)
+                return dispatch(req, url, state, pending, ip, requestId, seen);
+            return mw({ req, url, state, requestId }, () => run(i + 1));
+        };
+        let response;
+        try {
+            response = await run(0);
+        }
+        catch (error) {
+            response = errorResponse(error, new Headers(), req, requestId);
+        }
+        response = withHeader(response, "x-request-id", requestId);
+        // Health and metrics endpoints are called every few seconds by load balancers and scrapers: not logged or counted.
+        const probe = url.pathname === health?.path || url.pathname === health?.readyPath || url.pathname === metricsConfig?.path;
+        if (!probe) {
+            const elapsed = performance.now() - started;
+            const method = req.method.toUpperCase();
+            metrics?.request(method, seen.route, response.status, elapsed / 1000);
+            if (logging.requests) {
+                const durationMs = Math.round(elapsed * 10) / 10;
+                logger.info(`${method} ${url.pathname} ${response.status} ${durationMs}ms`, undefined, { requestId, method, path: url.pathname, status: response.status, durationMs });
+            }
+        }
+        if (secureHeaders.length) {
+            const https = url.protocol === "https:" || (Boolean(config.trustProxy) && req.headers.get("x-forwarded-proto")?.split(",")[0]?.trim() === "https");
+            response = applySecureHeaders(response, secureHeaders, https);
+        }
+        if (pending.length) {
+            const all = Promise.all(pending);
+            const execution = rest[1];
+            if (execution?.waitUntil)
+                execution.waitUntil(all);
+            else {
+                // Kept until it settles, so app.close() waits for it.
+                tasks.add(all);
+                void all.finally(() => {
+                    tasks.delete(all);
+                    settle();
+                });
+            }
+        }
+        return response;
+    }
     const app = {
         name: config.name ?? "inferlo-app",
         version: config.version ?? "0.0.0",
@@ -350,36 +606,20 @@ export function createApp(config = {}) {
         db,
         ai,
         jobs,
+        get closing() {
+            return closing;
+        },
         async fetch(req, ...rest) {
-            const url = new URL(req.url);
-            const ip = clientIp(req, Boolean(config.trustProxy), rest[0]);
-            if (ip !== "unknown")
-                clientIps.set(req, ip);
-            const state = {};
-            const pending = [];
-            const chain = [...(config.middleware ?? [])];
-            const run = (i) => {
-                const mw = chain[i];
-                if (!mw)
-                    return dispatch(req, url, state, pending, ip);
-                return mw({ req, url, state }, () => run(i + 1));
-            };
-            let response;
+            inFlight++;
+            metrics?.inFlight(1);
             try {
-                response = await run(0);
-            }
-            catch (error) {
-                response = errorResponse(error, new Headers(), req);
+                return await handle(req, rest);
             }
-            if (pending.length) {
-                const all = Promise.all(pending);
-                const execution = rest[1];
-                if (execution?.waitUntil)
-                    execution.waitUntil(all);
-                else
-                    void all;
+            finally {
+                inFlight--;
+                metrics?.inFlight(-1);
+                settle();
             }
-            return response;
         },
         async request(path, options = {}) {
             const url = new URL(path, "http://inferlo.test");
@@ -399,10 +639,74 @@ export function createApp(config = {}) {
         },
         async listen(port = 3000, hostname = "0.0.0.0") {
             const { serve } = await import("./serve.js");
-            return serve(app, { port, hostname });
+            const server = await serve(app, { port, hostname });
+            servers.add(server);
+            return {
+                port: server.port,
+                close: async () => {
+                    servers.delete(server);
+                    await server.close();
+                },
+            };
+        },
+        async checkReady() {
+            if (closing)
+                return { status: "not_ready", checks: { shutdown: { status: "error", durationMs: 0, error: "The app is shutting down." } } };
+            const names = Object.keys(readinessChecks);
+            const results = await Promise.all(names.map((name) => runCheck(name, readinessChecks[name])));
+            const checks = {};
+            names.forEach((name, i) => (checks[name] = results[i]));
+            return { status: results.every((r) => r.status === "ok") ? "ready" : "not_ready", checks };
+        },
+        close(options = {}) {
+            if (closed)
+                return closed;
+            const delayMs = options.delayMs ?? 0;
+            const timeoutMs = options.timeoutMs ?? 10_000;
+            for (const [key, value] of [["delayMs", delayMs], ["timeoutMs", timeoutMs]]) {
+                if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+                    throw new InferloError("IL6015", `app.close({ ${key} }) must be a number of milliseconds of 0 or more, got ${String(value)}.`, {
+                        fix: "Use app.close({ delayMs: 5000, timeoutMs: 10000 }) or app.close().",
+                    });
+                }
+            }
+            closing = true;
+            closed = (async () => {
+                if (delayMs > 0)
+                    await new Promise((resolve) => setTimeout(resolve, delayMs));
+                refusing = true;
+                const deadline = Date.now() + timeoutMs;
+                let workersStopped = false;
+                const drained = Promise.all([whenIdle(), stopWorkers(jobs).then(() => void (workersStopped = true))]);
+                if (!(await within(drained, timeoutMs))) {
+                    const left = [`${inFlight} request(s)`, `${tasks.size} waitUntil task(s)`];
+                    if (!workersStopped)
+                        left.push("the running job");
+                    logger.error(`app.close(): stopped waiting after ${timeoutMs} ms with ${left.join(", ")} still running`);
+                }
+                const { forceClosers } = servers.size ? await import("./serve.js") : { forceClosers: undefined };
+                await Promise.all([...servers].map(async (server) => {
+                    servers.delete(server);
+                    if (!(await within(server.close(), Math.max(0, deadline - Date.now()))))
+                        forceClosers?.get(server)?.();
+                }));
+                await db.$close();
+            })();
+            return closed;
         },
         describe() {
-            return describeApp(app, { auth: Boolean(config.auth), openapi: openapiPath, mcp: mcpPath, description: config.description });
+            return describeApp(app, {
+                auth: Boolean(config.auth),
+                openapi: openapiPath,
+                mcp: mcpPath,
+                health: health?.path ?? false,
+                ready: health?.readyPath ?? false,
+                metrics: metricsConfig?.path ?? false,
+                secureHeaders: secureHeaders.map(([name]) => name),
+                requestTimeoutMs,
+                logging: { format: config.logger ? "custom" : logging.format, requests: logging.requests },
+                description: config.description,
+            });
         },
         openapi() {
             return buildOpenAPI(app, { description: config.description, auth: Boolean(config.auth) });
@@ -410,6 +714,46 @@ export function createApp(config = {}) {
     };
     return app;
 }
+function resolveHealth(option, routes, taken) {
+    if (option === undefined || option === false)
+        return undefined;
+    const bad = (message, fix) => new InferloError("IL6014", message, { fix });
+    if (option !== true && (typeof option !== "object" || option === null))
+        throw bad("health must be true or an object.", "Use createApp({ health: true }) or createApp({ health: { checks: { ... } } }).");
+    const o = option === true ? {} : option;
+    const resolved = { path: o.path ?? "/health", readyPath: o.readyPath ?? "/ready", timeoutMs: o.timeoutMs ?? 5000, checks: o.checks ?? {} };
+    for (const [key, value] of [["path", resolved.path], ["readyPath", resolved.readyPath]]) {
+        if (typeof value !== "string" || !value.startsWith("/"))
+            throw bad(`health.${key} must be a path starting with "/", got ${JSON.stringify(value)}.`, `Use health: { ${key}: "/${String(value).replace(/^\/+/, "")}" }.`);
+    }
+    if (resolved.path === resolved.readyPath)
+        throw bad(`health.path and health.readyPath are both "${resolved.path}".`, 'Use different paths, for example "/health" and "/ready".');
+    if (!Number.isFinite(resolved.timeoutMs) || resolved.timeoutMs <= 0)
+        throw bad(`health.timeoutMs must be a positive number of milliseconds, got ${String(resolved.timeoutMs)}.`, "Use health: { timeoutMs: 5000 }.");
+    if (typeof resolved.checks !== "object" || resolved.checks === null)
+        throw bad("health.checks must be an object of functions.", "Use health: { checks: { redis: () => redis.ping() } }.");
+    for (const [name, check] of Object.entries(resolved.checks)) {
+        if (name === "database" || name === "ai")
+            throw bad(`health.checks.${name} is built in.`, `Give your check another name, for example "${name}Replica".`);
+        if (typeof check !== "function")
+            throw bad(`health.checks.${name} must be a function.`, `Use health: { checks: { ${name}: async () => { ... } } }; throw when it is not ready.`);
+    }
+    for (const path of [resolved.path, resolved.readyPath]) {
+        if (taken.includes(path))
+            throw bad(`The health endpoint ${path} is also the OpenAPI or MCP path.`, "Choose another health.path or health.readyPath.");
+        const route = routes.find((r) => r.method === "GET" && r.path.replace(/\/+$/, "") === path.replace(/\/+$/, ""));
+        if (route)
+            throw bad(`The health endpoint ${path} is also the route GET ${route.path}.`, `Remove the GET ${route.path} route (createApp({ health }) answers it now), or choose another health.path or health.readyPath.`);
+    }
+    return resolved;
+}
+function checkTimeout(value, where) {
+    if (value === false || (typeof value === "number" && Number.isFinite(value) && value > 0))
+        return value;
+    throw new InferloError("IL6017", `${where} must be a positive number of milliseconds or false, got ${typeof value === "string" ? JSON.stringify(value) : String(value)}.`, {
+        fix: "Use a number such as 30000 (30 seconds), or false for no timeout.",
+    });
+}
 function parseOrThrow(schema, value, where, coerce) {
     const result = schema.safeParse(value, { coerce });
     if (!result.ok)