@browserlessapi/inferlo 0.6.0 → 0.8.0

package/dist/app.js

@@ -5,17 +5,28 @@
 import { bindTools, createAI } from "./ai/index.js";
 import { memory } from "./db/memory.js";
 import { createAudit } from "./audit.js";
+import { readEnv } from "./env.js";
+import { checkStorageStore, createStorage } from "./storage.js";
 import { checkRouteCache, createCache, databaseCacheStore, etagFor, etagMatches, memoryCacheStore, resolveCacheConfig, routeCacheKey } from "./cache.js";
-import { bindDb, createDb, guardAdapter } from "./db/table.js";
+import { bindDb, checkTenantId, createDb, guardAdapter } from "./db/table.js";
 import { describeApp } from "./describe.js";
 import { InferloError, didYouMean, suggest } from "./errors.js";
 import { callTool, handleMcp, toolDefinition } from "./mcp.js";
 import { buildOpenAPI } from "./openapi.js";
 import { collectRoutes } from "./route.js";
+import { createSocketHub } from "./sockets.js";
+import { checkSocketsConfig, redisSocketRelay } from "./redis-sockets.js";
+import { bunUpgrader, denoUpgrader, upgraders, upgrades, validKey } from "./websocket.js";
+import { checkRedisClient, pingRedis, redisCacheStore, redisMissing, redisPrefix, redisRateLimitStore } from "./redis.js";
 import { checkRateLimit, databaseRateLimitStore, memoryRateLimitStore, rateLimitResult, windowMs } from "./rate-limit.js";
-import { clientIp, clientIps } from "./client-ip.js";
-import { collectCrons, collectJobs, createJobs, jobStoreFor, stopWorkers } from "./jobs.js";
-import { ValidationError } from "./schema.js";
+import { clientIp, clientIps, clientSignals, keepEventStreamOpen, platformWaitUntil } from "./client-ip.js";
+import { redisJobStore } from "./redis-jobs.js";
+import { knownBodies } from "./response-body.js";
+import { checkFileContents, checkFileFields, fileBudget, isMultipart, readMultipart } from "./files.js";
+import { collectCrons, collectJobs, collectSubscriptions, createJobs, jobStoreFor, stopWorkers, withTenant } from "./jobs.js";
+import { checkEventsConfig, invalidTransport } from "./events.js";
+import { redisEventTransport } from "./redis-events.js";
+import { Schema, ValidationError } from "./schema.js";
 import { createRedactor, redactingLogger, redactValues, secretsOf } from "./redact.js";
 import { defaultLogger, loggingOptions, REQUEST_ID } from "./logging.js";
 import { clientTracing, jobTracing, requestTracing, traceAdapter, traceFields } from "./tracing.js";
@@ -62,11 +73,14 @@ function corsHeaders(config, origin) {
     return h;
 }
 async function readBody(req, limit, signal) {
+    return new TextDecoder().decode(await readBytes(req, limit, signal));
+}
+async function readBytes(req, limit, signal) {
     const length = Number(req.headers.get("content-length") ?? "0");
     if (length > limit)
         throw new InferloError("IL2004", `Request body is larger than ${limit} bytes.`, { status: 413 });
     if (!req.body)
-        return "";
+        return new Uint8Array(0);
     const reader = req.body.getReader();
     // Stop reading a slow body once the request timed out.
     signal?.addEventListener("abort", () => void reader.cancel().catch(() => { }), { once: true });
@@ -89,7 +103,7 @@ async function readBody(req, limit, signal) {
         all.set(c, offset);
         offset += c.byteLength;
     }
-    return new TextDecoder().decode(all);
+    return all;
 }
 function queryObject(url) {
     const out = {};
@@ -115,7 +129,10 @@ function withHeader(response, name, value) {
 }
 function json(data, status, headers) {
     headers.set("content-type", "application/json; charset=utf-8");
-    return new Response(JSON.stringify(data), { status, headers });
+    const text = JSON.stringify(data);
+    const response = new Response(text, { status, headers });
+    knownBodies.set(response, text);
+    return response;
 }
 export function createApp(config = {}) {
     // Everything the app logs goes through the redactor: credentials and secrets never reach the logs.
@@ -171,6 +188,14 @@ export function createApp(config = {}) {
         }
         names.set(r.name, r);
     }
+    // Routes with t.file() fields take multipart/form-data bodies up to maxBodySize plus their files' maxBytes.
+    const fileBudgets = new Map();
+    for (const r of routes) {
+        checkFileFields(r);
+        const budget = fileBudget(r.options.body);
+        if (budget > 0)
+            fileBudgets.set(r, budget);
+    }
     const needsAuth = routes.filter((r) => (r.options.auth ?? "user") !== "public");
     if (needsAuth.length && !config.auth) {
         throw new InferloError("IL3003", `${needsAuth.length} route(s) require a signed-in user, but createApp has no auth configured: ${needsAuth.map((r) => `${r.method} ${r.path}`).join(", ")}.`, {
@@ -197,10 +222,20 @@ export function createApp(config = {}) {
     for (const r of routes)
         if (r.options.rateLimit !== undefined)
             checkRateLimit(r.options.rateLimit, `${r.method} ${r.path}`);
+    for (const r of routes) {
+        if (!r.websocket)
+            continue;
+        const message = r.options.message;
+        if (message !== undefined && !(message instanceof Schema)) {
+            throw new InferloError("IL6035", `The WebSocket route ${r.path} has a message option that is not a schema.`, { fix: 'Pass a schema: route.ws("/chat", { message: t.object({ text: t.string() }) }).' });
+        }
+    }
     for (const r of routes)
         if (r.options.cache !== undefined)
             checkRouteCache(r.options.cache, r.method, `${r.method} ${r.path}`);
     const requestTimeoutMs = config.requestTimeoutMs === undefined ? 60_000 : checkTimeout(config.requestTimeoutMs, "createApp({ requestTimeoutMs })");
+    const validateResponses = resolveValidateResponses(config.validateResponses);
+    const warnedResponses = new Set();
     for (const r of routes)
         if (r.options.timeoutMs !== undefined)
             checkTimeout(r.options.timeoutMs, `${r.method} ${r.path} { timeoutMs }`);
@@ -211,13 +246,16 @@ export function createApp(config = {}) {
             deprecations.set(r, resolved);
     }
     const storeName = config.rateLimit?.store ?? "memory";
-    if (storeName !== "memory" && storeName !== "database") {
-        throw new InferloError("IL6007", `Invalid rateLimit store "${String(storeName)}".`, { fix: 'Use createApp({ rateLimit: { store: "memory" } }) or { store: "database" }.' });
+    if (storeName !== "memory" && storeName !== "database" && storeName !== "redis") {
+        throw new InferloError("IL6007", `Invalid rateLimit store "${String(storeName)}".`, { fix: 'Use createApp({ rateLimit: { store: "memory" } }), { store: "database" } or { store: "redis" }.' });
     }
+    const redis = checkRedisClient(config.redis);
+    if (storeName === "redis" && !redis)
+        throw redisMissing("rateLimit");
     const openapiPath = config.openapi === undefined ? "/openapi.json" : config.openapi;
     const hasTools = routes.some((r) => r.options.tool);
     const mcpPath = config.mcp === undefined ? (hasTools ? "/mcp" : false) : config.mcp;
-    const health = resolveHealth(config.health, routes, [openapiPath, mcpPath]);
+    const health = resolveHealth(config.health, routes, [openapiPath, mcpPath], redis ? ["redis"] : []);
     // apiVersion("v1", ...) routes get their own OpenAPI document at /v1/openapi.json.
     const versions = versionsOf(routes);
     const versionDocs = new Map(openapiPath ? versions.map((v) => [versionDocPath(v, openapiPath), v]) : []);
@@ -238,9 +276,13 @@ export function createApp(config = {}) {
     const appTracing = tracing ? { ...config.tracing, tracerName: config.tracing.tracerName ?? config.name ?? "inferlo-app" } : undefined;
     const adapter = appTracing ? traceAdapter(config.db ?? memory(), clientTracing(appTracing, "inferlo-app", config.version ?? "0.0.0")) : (config.db ?? memory());
     const db = createDb(models, adapter);
-    const rateLimits = storeName === "database" ? databaseRateLimitStore(adapter) : memoryRateLimitStore();
+    const rateLimits = storeName === "redis" ? redisRateLimitStore(redis, redisPrefix(config.name)) : storeName === "database" ? databaseRateLimitStore(adapter) : memoryRateLimitStore();
     const cacheConfig = resolveCacheConfig(config.cache);
-    const cache = createCache(cacheConfig.store === "database" ? databaseCacheStore(adapter) : memoryCacheStore(cacheConfig.maxEntries), cacheConfig.store);
+    if (cacheConfig.store === "redis" && !redis)
+        throw redisMissing("cache");
+    const cacheStore = cacheConfig.store === "redis" ? redisCacheStore(redis, redisPrefix(config.name)) : cacheConfig.store === "database" ? databaseCacheStore(adapter) : memoryCacheStore(cacheConfig.maxEntries);
+    const cache = createCache(cacheStore, cacheConfig.store);
+    const storage = createStorage(checkStorageStore(config.storage));
     const metrics = metricsConfig
         ? createMetrics({ app: config.name ?? "inferlo-app", version: config.version ?? "0.0.0", inferlo: INFERLO_VERSION }, metricsConfig.buckets)
         : undefined;
@@ -260,17 +302,39 @@ export function createApp(config = {}) {
             jobModule.set(j.name, m.name);
     const moduleInfo = { modules, routeModule, modelOwner, jobModule };
     const jobSources = [...(config.jobs ?? []), ...modules.flatMap((m) => m.jobs)];
+    const transportOption = checkEventsConfig(config.events);
+    let eventTransport;
+    if (transportOption === "redis") {
+        if (!redis)
+            throw new InferloError("IL6031", 'events transport "redis" needs a Redis client, but createApp has no redis option.', { fix: 'Create and connect a client with the redis package: const redis = await createClient({ url: process.env.REDIS_URL }).connect(); then createApp({ name: "billing", redis, events: { transport: "redis" } }).' });
+        // The app name is the consumer group: two services with one name would split each other's events.
+        if (!config.name)
+            throw invalidTransport('events transport "redis" needs createApp({ name }): each service reads events as its own consumer group, named after the app.');
+        eventTransport = redisEventTransport(redis, { group: config.name });
+    }
+    else
+        eventTransport = transportOption;
+    const socketSubscriber = checkSocketsConfig(config.sockets, redis, config.name);
+    const socketHub = createSocketHub(socketSubscriber ? redisSocketRelay(redis, socketSubscriber, `${redisPrefix(config.name)}sockets`, (message, error) => logger.error(`WebSocket broadcasts: ${message}`, error ?? "")) : undefined);
     const jobs = createJobs(collectJobs(jobSources), {
         db: db,
         ai,
         cache,
-        audit: (run) => createAudit(models, redactor, () => db, { userId: null, source: `job ${run.name}` }),
-        store: jobStoreFor(config.jobQueue, adapter),
+        storage,
+        sockets: socketHub,
+        audit: (run) => createAudit(models, redactor, () => db, { userId: null, tenantId: run.tenantId, source: `job ${run.name}` }),
+        store: jobStoreFor(config.jobQueue, adapter, () => {
+            if (!redis)
+                throw redisMissing("jobQueue");
+            return redisJobStore(redis, redisPrefix(config.name));
+        }),
         queue: config.jobQueue?.store ?? "memory",
         logger,
         onRun: metrics ? (name, outcome, ms) => metrics.job(name, outcome, ms / 1000) : undefined,
         tracing: appTracing ? jobTracing(appTracing, "inferlo-app", config.version ?? "0.0.0") : undefined,
-    }, collectCrons(jobSources));
+        eventTransport,
+    }, collectCrons(jobSources), collectSubscriptions(jobSources));
+    const events = jobs.events;
     const maxBody = config.maxBodySize ?? 1024 * 1024;
     const secureHeaders = resolveSecureHeaders(config.secureHeaders);
     const cors = config.cors === true ? { origin: "*" } : config.cors || undefined;
@@ -294,6 +358,8 @@ export function createApp(config = {}) {
             }
         };
     }
+    if (redis)
+        readinessChecks.redis = () => pingRedis(redis);
     Object.assign(readinessChecks, health?.checks);
     const runCheck = async (name, check) => {
         const started = Date.now();
@@ -330,6 +396,11 @@ export function createApp(config = {}) {
                 fix: "Pass routes that are registered in createApp({ routes }) to ai.generate({ tools }).",
             });
         }
+        if (fileBudgets.has(found)) {
+            throw new InferloError("IL1004", `Route ${found.method} ${found.path} takes files, so it cannot be an AI tool (tools send JSON).`, {
+                fix: "Pass routes without t.file() fields to ai.generate({ tools }).",
+            });
+        }
         return toolDefinition(found);
     };
     const describeTool = (route) => {
@@ -349,6 +420,13 @@ export function createApp(config = {}) {
         }
         return { name: tool.name, status: response.status, output, text: text || String(response.status) };
     }, describeTool, signal);
+    const tenantScope = (user) => {
+        if (!user)
+            return "none";
+        const id = config.auth?.tenant ? config.auth.tenant(user) : user.tenantId;
+        return id === null || id === undefined ? "none" : { tenant: checkTenantId(id) };
+    };
+    const tenantOf = (scope) => (typeof scope === "object" ? scope.tenant : null);
     const roles = (user) => config.auth?.roles?.(user) ?? (user.roles ?? []);
     const errorResponse = (error, headers, req, requestId) => {
         const span = requestSpans.get(req);
@@ -367,7 +445,218 @@ export function createApp(config = {}) {
         const message = config.exposeErrors && error instanceof Error ? redactor.text(error.message) : "Internal server error";
         return json({ error: { code: "IL9000", message, docs: new InferloError("IL9000", "").docs } }, 500, headers);
     };
-    const dispatch = async (req, url, state, pending, ip, requestId, seen) => {
+    // Rate limit counts are kept per route and per user, tenant or IP. by "tenant" counts users without a tenant one by
+    // one; by "user" and "tenant" count requests without a user by IP.
+    const rateLimitKey = (def, user, ip) => {
+        const by = def.options.rateLimit.by ?? "user";
+        const scope = by === "tenant" && user ? tenantScope(user) : "none";
+        const who = typeof scope === "object" ? `tenant:${scope.tenant}` : by !== "ip" && user ? `user:${user.id}` : `ip:${ip}`;
+        return `${def.method} ${def.path} ${who}`;
+    };
+    // Open WebSockets, closed with 1001 by app.close().
+    const sockets = new Set();
+    const encoder = new TextEncoder();
+    /** The JSON error message a socket gets when one of its messages fails (the same body as an HTTP error answer). */
+    const socketError = (error, def, requestId) => {
+        if (error instanceof InferloError) {
+            if (error.status >= 500)
+                logger.error(`WS ${def.path} message failed: ${error.format()}`, error.cause ?? "", { requestId });
+            const body = error.toJSON();
+            if (config.exposeErrors && !error.expose)
+                body.error.message = redactor.text(error.message);
+            return body;
+        }
+        logger.error(`WS ${def.path} message failed`, error, { requestId });
+        const message = config.exposeErrors && error instanceof Error ? redactor.text(error.message) : "Internal server error";
+        return { error: { code: "IL9000", message, docs: new InferloError("IL9000", "").docs } };
+    };
+    /** Checks the upgrade request of a WebSocket route (after auth, permissions, rate limits and input) and accepts it. */
+    const acceptSocket = async (def, req, url, headers, rest, conn) => {
+        if (req.method.toUpperCase() !== "GET" || req.headers.get("upgrade")?.toLowerCase() !== "websocket") {
+            headers.set("upgrade", "websocket");
+            throw new InferloError("IL2010", `${def.path} is a WebSocket route: open it with a WebSocket client.`, { status: 426, fix: `Connect with new WebSocket("wss://<host>${def.path}"); plain HTTP requests are not answered.` });
+        }
+        if (req.headers.get("sec-websocket-version") !== "13" || !validKey(req.headers.get("sec-websocket-key"))) {
+            headers.set("sec-websocket-version", "13");
+            throw new InferloError("IL2010", "Invalid WebSocket handshake.", { status: 400, fix: "Use a WebSocket client that speaks RFC 6455 (Sec-WebSocket-Version: 13 with a Sec-WebSocket-Key)." });
+        }
+        // Browsers send cookies with WebSocket upgrades from any page, so only the app's own origin (and the origins
+        // allowed by createApp({ cors })) may connect. Clients without an Origin header (servers, CLIs) may.
+        const origin = req.headers.get("origin");
+        if (origin !== null && !socketOriginAllowed(origin, req, url)) {
+            throw new InferloError("IL3006", `Connections from the origin ${origin} are not allowed.`, { status: 403, fix: "Open the socket from a page of the app's own origin, or list the page's origin in createApp({ cors: { origin: [...] } })." });
+        }
+        const upgrader = upgraders.get(req) ?? denoUpgrader(req, rest[0]) ?? bunUpgrader(req, rest[0]);
+        if (!upgrader) {
+            throw new InferloError("IL2011", "This server cannot accept WebSocket connections.", {
+                status: 501,
+                fix: 'Serve the app with app.listen() or serve(app), with Deno.serve(app.fetch) or deno serve on Deno, or with Bun.serve({ fetch: app.fetch, websocket: bunWebSocket }) on Bun. On a Node.js server another framework created, add server.on("upgrade", toNodeUpgradeHandler(app)). app.request() cannot upgrade: test against await app.listen(0).',
+            });
+        }
+        // With sockets: { transport: "redis" }, the first connection subscribes this instance to the app's broadcasts.
+        await socketHub.ready();
+        const response = await upgrader((transport) => openSocket(def, req, url, transport, conn));
+        upgrades.set(req, response ?? null);
+        // On Node.js and Bun the server already answered the handshake; this response is never sent.
+        return response ?? new Response(null, { status: 200, headers });
+    };
+    const socketOriginAllowed = (origin, req, url) => {
+        try {
+            if (new URL(origin).host === (req.headers.get("host") ?? url.host))
+                return true;
+        }
+        catch {
+            return false;
+        }
+        if (!cors)
+            return false;
+        return cors.origin === "*" || (Array.isArray(cors.origin) ? cors.origin : [cors.origin]).includes(origin);
+    };
+    /** One open WebSocket: runs the route's handler for each message, in order. */
+    const openSocket = (def, req, url, transport, conn) => {
+        const { user, requestId, ip } = conn;
+        const opts = def.options;
+        const closed = new AbortController();
+        let open = true;
+        const member = {
+            sendText: (text) => {
+                if (open)
+                    transport.send(text);
+            },
+        };
+        const finish = () => {
+            closed.abort();
+            sockets.delete(shutdown);
+            socketHub.drop(member);
+        };
+        const end = (code, reason) => {
+            if (!open)
+                return;
+            open = false;
+            transport.close(code, reason);
+            finish();
+        };
+        const shutdown = () => end(1001, "Server shutting down");
+        sockets.add(shutdown);
+        socketHub.open(member);
+        const scope = tenantScope(user);
+        const socket = {
+            id: crypto.randomUUID(),
+            send: (data) => {
+                if (open)
+                    transport.send(JSON.stringify(data));
+            },
+            close: (code = 1000, reason = "") => {
+                if (!(code === 1000 || (Number.isInteger(code) && code >= 3000 && code <= 4999))) {
+                    throw new InferloError("IL2012", `Invalid WebSocket close code ${String(code)}.`, { fix: "Close with 1000 (normal), or an application code from 3000 to 4999: socket.close(4000, \"reason\")." });
+                }
+                end(code, reason);
+            },
+            join: (room) => socketHub.join(member, room, tenantOf(scope)),
+            leave: (room) => socketHub.leave(member, room, tenantOf(scope)),
+            stream: async (result) => {
+                if (!result || typeof result[Symbol.asyncIterator] !== "function" || typeof result.usage !== "function") {
+                    throw new InferloError("IL2013", "socket.stream() takes the result of ai.stream(...).", { fix: 'Pass the stream itself: await socket.stream(ai.stream({ model: "fast", prompt })).' });
+                }
+                let text = "";
+                // After the socket closes, sends do nothing; with ctx.ai the stream itself is cancelled by ctx.signal.
+                for await (const chunk of result) {
+                    text += chunk;
+                    socket.send({ type: "text", text: chunk });
+                }
+                const usage = await result.usage();
+                socket.send({ type: "done", usage });
+                return { text, usage };
+            },
+        };
+        let socketDb;
+        let socketRooms;
+        let queues;
+        const tenantQueues = () => (queues ??= typeof scope === "object" ? withTenant({ jobs, events }, scope.tenant) : { jobs, events });
+        const handleMessage = async (data) => {
+            if (typeof data !== "string")
+                return end(1003, "Send JSON text messages");
+            if (data.length > maxBody / 3 && encoder.encode(data).byteLength > maxBody)
+                return end(1009, "Message too big");
+            try {
+                let raw;
+                try {
+                    raw = JSON.parse(data);
+                }
+                catch {
+                    throw new InferloError("IL2003", "Message is not valid JSON.", { status: 400, fix: "Send each message as JSON text: socket.send(JSON.stringify(message))." });
+                }
+                if (opts.rateLimit) {
+                    const limited = rateLimitResult(opts.rateLimit, await rateLimits.hit(rateLimitKey(def, user, ip), windowMs(opts.rateLimit)));
+                    if (limited.error)
+                        throw limited.error;
+                }
+                const message = opts.message ? parseOrThrow(opts.message, raw, "message", false) : raw;
+                const ctx = {
+                    message,
+                    query: conn.query,
+                    params: conn.params,
+                    user,
+                    socket,
+                    get db() {
+                        return (socketDb ??= bindDb(models, adapter, scope));
+                    },
+                    get ai() {
+                        const value = toolsFor(req, closed.signal);
+                        Object.defineProperty(ctx, "ai", { value, enumerable: true });
+                        return value;
+                    },
+                    get jobs() {
+                        return tenantQueues().jobs;
+                    },
+                    get events() {
+                        return tenantQueues().events;
+                    },
+                    cache,
+                    storage,
+                    get sockets() {
+                        return (socketRooms ??= socketHub.forTenant(tenantOf(scope)));
+                    },
+                    get audit() {
+                        const value = createAudit(models, redactor, () => ctx.db, { userId: user?.id ?? null, tenantId: tenantOf(scope), source: `WS ${def.path}`, requestId });
+                        Object.defineProperty(ctx, "audit", { value, enumerable: true });
+                        return value;
+                    },
+                    signal: closed.signal,
+                    requestId,
+                    url,
+                    headers: req.headers,
+                };
+                const result = await def.handler(ctx);
+                if (result !== undefined)
+                    socket.send(result);
+            }
+            catch (error) {
+                if (open)
+                    transport.send(JSON.stringify(socketError(error, def, requestId)));
+            }
+        };
+        let chain = Promise.resolve();
+        return {
+            message(data) {
+                if (!open)
+                    return;
+                // One message at a time, in order; app.close() waits for the one running.
+                const next = chain.then(() => (open ? handleMessage(data) : undefined));
+                chain = next;
+                tasks.add(next);
+                void next.finally(() => {
+                    tasks.delete(next);
+                    settle();
+                });
+            },
+            close() {
+                open = false;
+                finish();
+            },
+        };
+    };
+    const dispatch = async (req, url, state, pending, ip, requestId, seen, rest) => {
         const headers = new Headers();
         if (cors)
             for (const [k, v] of Object.entries(corsHeaders(cors, req.headers.get("origin"))))
@@ -457,8 +746,18 @@ export function createApp(config = {}) {
             const limit = opts.timeoutMs === undefined ? requestTimeoutMs : opts.timeoutMs;
             // timeout aborts when the route takes too long; signal (ctx.signal) also when the client disconnects.
             const timeout = new AbortController();
-            const signal = AbortSignal.any([req.signal, timeout.signal]);
+            // Combined on first use: AbortSignal.any is costly, and most handlers never read ctx.signal.
+            let combined;
+            const signal = () => (combined ??= AbortSignal.any([clientSignals.get(req) ?? req.signal, timeout.signal]));
             let requestDb;
+            let queues;
+            let requestUser = null;
+            const tenantQueues = () => {
+                if (queues)
+                    return queues;
+                const scope = tenantScope(requestUser);
+                return (queues = typeof scope === "object" ? withTenant({ jobs, events }, scope.tenant) : { jobs, events });
+            };
             const run = async () => {
                 // Auth
                 let user = null;
@@ -466,6 +765,7 @@ export function createApp(config = {}) {
                     user = testUsers.get(req) ?? null;
                 else if (config.auth)
                     user = (await config.auth.resolve(req, { db: db, url })) ?? null;
+                requestUser = user;
                 const level = opts.auth ?? "user";
                 if (level !== "public") {
                     if (!user)
@@ -475,10 +775,10 @@ export function createApp(config = {}) {
                 }
                 if (opts.permission)
                     await config.permissions.authorize(opts.permission, { user });
-                // Rate limit: counted per user (or IP without one), after auth so failed sign-ins on private routes do not use it up.
+                // Rate limit: counted per user, tenant or IP, after auth so failed sign-ins on private routes do not use it up.
+                // by "tenant" counts users without a tenant one by one; by "user" and "tenant" count requests without a user by IP.
                 if (opts.rateLimit) {
-                    const who = (opts.rateLimit.by ?? "user") === "user" && user ? `user:${user.id}` : `ip:${ip}`;
-                    const hit = await rateLimits.hit(`${def.method} ${def.path} ${who}`, windowMs(opts.rateLimit));
+                    const hit = await rateLimits.hit(rateLimitKey(def, user, ip), windowMs(opts.rateLimit));
                     const limited = rateLimitResult(opts.rateLimit, hit);
                     for (const [k, v] of Object.entries(limited.headers))
                         headers.set(k, v);
@@ -495,10 +795,24 @@ export function createApp(config = {}) {
                 const params = opts.params ? parseOrThrow(opts.params, rawParams, "path parameters", true) : rawParams;
                 const rawQuery = queryObject(url);
                 const query = opts.query ? parseOrThrow(opts.query.strip(), rawQuery, "query string", true) : rawQuery;
+                // WebSocket routes answer the upgrade here; their handler then runs once per message.
+                if (def.websocket)
+                    return await acceptSocket(def, req, url, headers, rest, { user, params, query, requestId, ip });
                 let body;
-                if (opts.body) {
+                const contentType = req.headers.get("content-type") ?? "";
+                const filesLimit = fileBudgets.get(def);
+                if (opts.body && filesLimit !== undefined && isMultipart(contentType) && method !== "GET" && method !== "HEAD") {
+                    // File uploads: read with the route's limit (maxBodySize plus its files' maxBytes), then check sizes,
+                    // types and file contents before the handler runs.
+                    const bytes = await readBytes(req, maxBody + filesLimit, timeout.signal);
+                    body = parseOrThrow(opts.body, await readMultipart(bytes, contentType, opts.body), "request body", true);
+                    const issues = await checkFileContents(opts.body, body);
+                    if (issues.length)
+                        throw new ValidationError(issues, "request body");
+                }
+                else if (opts.body) {
                     const text = method === "GET" || method === "HEAD" ? "" : await readBody(req, maxBody, timeout.signal);
-                    const type = req.headers.get("content-type") ?? "";
+                    const type = contentType;
                     let raw = undefined;
                     if (text) {
                         if (type.includes("application/x-www-form-urlencoded")) {
@@ -521,13 +835,13 @@ export function createApp(config = {}) {
                 // Response cache: after auth, permissions, rate limits and input checks, so a cached response is only
                 // served to callers who may call the route with valid input.
                 const routeCache = opts.cache;
-                const cacheKey = routeCache ? await routeCacheKey(def.method, def.path, routeCache, user?.id, url) : undefined;
+                const cacheKey = routeCache ? await routeCacheKey(def.method, def.path, routeCache, user?.id, url, routeCache.by === "tenant" && user ? tenantOf(tenantScope(user)) : undefined) : undefined;
                 const cacheHeaders = (outcome, etag) => {
                     headers.set("x-inferlo-cache", outcome);
                     headers.set("etag", etag);
                     if (!headers.has("cache-control"))
                         headers.set("cache-control", `${(routeCache.by ?? "user") === "public" && (opts.auth ?? "user") === "public" ? "public" : "private"}, no-cache`);
-                    if ((routeCache.by ?? "user") === "user")
+                    if ((routeCache.by ?? "user") !== "public")
                         headers.append("vary", "Authorization, Cookie");
                 };
                 const cachedAnswer = (body, etag) => {
@@ -554,13 +868,31 @@ export function createApp(config = {}) {
                     user,
                     get db() {
                         // Built on first use: after a timeout, new queries fail with IL2007 instead of running.
-                        return (requestDb ??= bindDb(models, guardAdapter(adapter, timeout.signal)));
+                        // Tenant-scoped tables are filtered by the signed-in user's tenant (none without one).
+                        return (requestDb ??= bindDb(models, guardAdapter(adapter, timeout.signal), tenantScope(user)));
+                    },
+                    // ctx.ai and ctx.audit are built on first use too, so routes that do not use them pay nothing.
+                    get ai() {
+                        const value = toolsFor(req, signal());
+                        Object.defineProperty(ctx, "ai", { value, enumerable: true });
+                        return value;
+                    },
+                    get signal() {
+                        return signal();
                     },
-                    ai: toolsFor(req, signal),
-                    signal,
                     requestId,
-                    jobs,
+                    // With a tenant, jobs and events queued by the request carry it.
+                    get jobs() {
+                        return tenantQueues().jobs;
+                    },
+                    get events() {
+                        return tenantQueues().events;
+                    },
                     cache,
+                    storage,
+                    get sockets() {
+                        return socketHub.forTenant(tenantOf(tenantScope(user)));
+                    },
                     req,
                     url,
                     headers: req.headers,
@@ -575,7 +907,11 @@ export function createApp(config = {}) {
                         handlerHeaders.push([name.toLowerCase(), value]);
                     },
                     waitUntil: (p) => void pending.push(p.catch((e) => logger.error("waitUntil task failed", e, { requestId }))),
-                    audit: createAudit(models, redactor, () => ctx.db, { userId: user?.id ?? null, source: `${def.method} ${def.path}`, requestId }),
+                    get audit() {
+                        const value = createAudit(models, redactor, () => ctx.db, { userId: user?.id ?? null, tenantId: tenantOf(tenantScope(user)), source: `${def.method} ${def.path}`, requestId });
+                        Object.defineProperty(ctx, "audit", { value, enumerable: true });
+                        return value;
+                    },
                 };
                 const result = await def.handler(ctx);
                 if (result instanceof Response) {
@@ -589,6 +925,24 @@ export function createApp(config = {}) {
                 }
                 if (result === undefined)
                     return new Response(null, { status: status || 204, headers });
+                if (validateResponses && opts.response && (status || 200) < 300) {
+                    const issues = responseIssues(opts.response, result);
+                    if (issues) {
+                        const where = `${def.method} ${def.path}`;
+                        const summary = issues.slice(0, 5).map((i) => `${i.path.length ? i.path.join(".") : "(root)"}: ${i.message}`).join("; ") + (issues.length > 5 ? ` (and ${issues.length - 5} more)` : "");
+                        if (validateResponses === "error") {
+                            throw new InferloError("IL2016", `${where}: the response does not match the route's response schema. ${summary}`, {
+                                expose: true,
+                                fix: "Return what the route's response schema describes, or change the schema. Fields not in an object schema count as mismatches, so the check also finds fields that should not be sent.",
+                                details: issues.map((i) => ({ path: i.path, code: i.code, message: i.message })),
+                            });
+                        }
+                        if (!warnedResponses.has(def)) {
+                            warnedResponses.add(def);
+                            logger.error(`IL2016 ${where}: the response does not match the route's response schema (sent anyway; logged once per route). ${summary}`, undefined, { requestId });
+                        }
+                    }
+                }
                 // Only plain 200 JSON answers are cached, never one that sets a cookie.
                 if (cacheKey && (status || 200) === 200 && !handlerHeaders.some(([name]) => name === "set-cookie")) {
                     const body = JSON.stringify(result);
@@ -697,7 +1051,7 @@ export function createApp(config = {}) {
         const run = (i) => {
             const mw = chain[i];
             if (!mw)
-                return dispatch(req, url, state, pending, ip, requestId, seen);
+                return dispatch(req, url, state, pending, ip, requestId, seen, rest);
             return mw({ req, url, state, requestId }, () => run(i + 1));
         };
         let response;
@@ -715,10 +1069,12 @@ export function createApp(config = {}) {
         if (!probe) {
             const elapsed = performance.now() - started;
             const method = req.method.toUpperCase();
-            metrics?.request(method, seen.route, response.status, elapsed / 1000);
+            // An accepted WebSocket upgrade is logged and counted as the 101 the client gets.
+            const status = upgrades.has(req) ? 101 : response.status;
+            metrics?.request(method, seen.route, status, elapsed / 1000);
             if (logging.requests) {
                 const durationMs = Math.round(elapsed * 10) / 10;
-                logger.info(`${method} ${url.pathname} ${response.status} ${durationMs}ms`, undefined, { requestId, ...traceFields(span), method, path: url.pathname, status: response.status, durationMs });
+                logger.info(`${method} ${url.pathname} ${status} ${durationMs}ms`, undefined, { requestId, ...traceFields(span), method, path: url.pathname, status, durationMs });
             }
         }
         if (secureHeaders.length) {
@@ -727,9 +1083,9 @@ export function createApp(config = {}) {
         }
         if (pending.length) {
             const all = Promise.all(pending);
-            const execution = rest[1];
-            if (execution?.waitUntil)
-                execution.waitUntil(all);
+            const execution = platformWaitUntil(rest);
+            if (execution)
+                execution(all);
             else {
                 // Kept until it settles, so app.close() waits for it.
                 tasks.add(all);
@@ -750,7 +1106,10 @@ export function createApp(config = {}) {
         db,
         ai,
         jobs,
+        events,
         cache,
+        storage,
+        sockets: { broadcast: socketHub.broadcast, count: socketHub.count },
         get closing() {
             return closing;
         },
@@ -758,7 +1117,16 @@ export function createApp(config = {}) {
             inFlight++;
             metrics?.inFlight(1);
             try {
-                return await handle(req, rest);
+                const response = await handle(req, rest);
+                // Deno needs its own upgrade response back, untouched; Bun.serve wants undefined for a request it upgraded.
+                const upgrade = upgrades.get(req);
+                if (upgrade)
+                    return upgrade;
+                if (upgrade === null)
+                    return undefined;
+                if (rest.length)
+                    keepEventStreamOpen(rest[0], req, response);
+                return response;
             }
             finally {
                 inFlight--;
@@ -777,6 +1145,13 @@ export function createApp(config = {}) {
                 body = JSON.stringify(options.json);
                 headers["content-type"] ??= "application/json";
             }
+            if (typeof FormData !== "undefined" && body instanceof FormData) {
+                // Sent as bytes: on Node.js 24, cancelling a request body stream made from FormData (as a too large upload
+                // does) raises an unhandled rejection inside fetch's FormData encoder.
+                const encoded = new Response(body);
+                headers["content-type"] ??= encoded.headers.get("content-type");
+                body = new Uint8Array(await encoded.arrayBuffer());
+            }
             const req = new Request(url, { method: (options.method ?? "GET").toUpperCase(), headers, body });
             if ("as" in options)
                 testUsers.set(req, options.as ?? null);
@@ -820,6 +1195,9 @@ export function createApp(config = {}) {
                 if (delayMs > 0)
                     await new Promise((resolve) => setTimeout(resolve, delayMs));
                 refusing = true;
+                // Open WebSockets are told the server is going away (1001); a message being handled still finishes.
+                for (const shutdown of [...sockets])
+                    shutdown();
                 const deadline = Date.now() + timeoutMs;
                 let workersStopped = false;
                 const drained = Promise.all([whenIdle(), stopWorkers(jobs).then(() => void (workersStopped = true))]);
@@ -835,6 +1213,7 @@ export function createApp(config = {}) {
                     if (!(await within(server.close(), Math.max(0, deadline - Date.now()))))
                         forceClosers?.get(server)?.();
                 }));
+                await socketHub.stop().catch((error) => logger.error("WebSocket broadcasts: unsubscribing failed", error));
                 await db.$close();
             })();
             return closed;
@@ -881,7 +1260,7 @@ export function createApp(config = {}) {
     });
     return app;
 }
-function resolveHealth(option, routes, taken) {
+function resolveHealth(option, routes, taken, builtIn) {
     if (option === undefined || option === false)
         return undefined;
     const bad = (message, fix) => new InferloError("IL6014", message, { fix });
@@ -898,9 +1277,9 @@ function resolveHealth(option, routes, taken) {
     if (!Number.isFinite(resolved.timeoutMs) || resolved.timeoutMs <= 0)
         throw bad(`health.timeoutMs must be a positive number of milliseconds, got ${String(resolved.timeoutMs)}.`, "Use health: { timeoutMs: 5000 }.");
     if (typeof resolved.checks !== "object" || resolved.checks === null)
-        throw bad("health.checks must be an object of functions.", "Use health: { checks: { redis: () => redis.ping() } }.");
+        throw bad("health.checks must be an object of functions.", "Use health: { checks: { search: () => search.ping() } }.");
     for (const [name, check] of Object.entries(resolved.checks)) {
-        if (name === "database" || name === "ai")
+        if (name === "database" || name === "ai" || builtIn.includes(name))
             throw bad(`health.checks.${name} is built in.`, `Give your check another name, for example "${name}Replica".`);
         if (typeof check !== "function")
             throw bad(`health.checks.${name} must be a function.`, `Use health: { checks: { ${name}: async () => { ... } } }; throw when it is not ready.`);
@@ -927,4 +1306,25 @@ function parseOrThrow(schema, value, where, coerce) {
         throw new ValidationError(result.issues, where);
     return result.value;
 }
+function resolveValidateResponses(value) {
+    if (value === undefined)
+        return (readEnv("INFERLO_ENV") || readEnv("NODE_ENV") || "development") === "production" ? false : "warn";
+    if (value === "warn" || value === "error" || value === false)
+        return value;
+    throw new InferloError("IL6040", `Invalid validateResponses option ${JSON.stringify(value)}.`, {
+        fix: 'Use createApp({ validateResponses: "warn" }) (the default outside production), "error" in tests, or false to turn the check off.',
+    });
+}
+/** The issues of the answer as clients receive it (JSON), or undefined when it matches. */
+function responseIssues(schema, result) {
+    let sent;
+    try {
+        sent = JSON.parse(JSON.stringify(result));
+    }
+    catch {
+        return undefined; // Not JSON: the answer itself fails with its own error.
+    }
+    const parsed = schema.safeParse(sent);
+    return parsed.ok ? undefined : parsed.issues;
+}
 //# sourceMappingURL=app.js.map