@thotischner/observability-mcp 1.8.1 → 3.0.1

package/dist/index.js

@@ -9,35 +9,56 @@ import { z } from "zod";
 import { loadConfig, saveConfig, DEFAULT_HEALTH_THRESHOLDS, DEFAULT_SETTINGS } from "./config/loader.js";
 import { ConnectorRegistry, getSupportedTypes } from "./connectors/registry.js";
 import { isTopologyProvider } from "./connectors/interface.js";
-import { defaultContext, principalContext } from "./context.js";
+import { defaultContext, principalContext, sessionContext, allowsTool } from "./context.js";
+import { parseKeyTenants } from "./tenancy/context.js";
 import { enforceEntitledAccess, enterpriseGateStatus, enterpriseGateInfo, enterprisePolicyView, enterpriseCatalogView, enterpriseAuditTail, authorizeAdmin, updateRbacPolicy, updateCatalog, } from "./enterprise-gate.js";
 import { loadCredentials, credentialsConfigured, extractToken, resolveToken, } from "./auth/credentials.js";
 import { issueSession, setCookieHeader, clearCookieHeader, generateSecret, } from "./auth/session.js";
-import { readUsersFile, authenticate, } from "./auth/local-users.js";
+import { readUsersFile, writeUsersFile, authenticate, } from "./auth/local-users.js";
 import { buildSessionAttacher, buildRequireSession, } from "./auth/middleware.js";
-import { buildRequirePermission, hasPermission, listGrantedPermissions, DEFAULT_POLICY, } from "./auth/rbac.js";
+import { buildRequirePermissionFromEngine, hasPermission, listGrantedPermissions, DEFAULT_POLICY, } from "./auth/rbac.js";
 import { resolveOidcConfig, buildOidcRuntime } from "./auth/oidc/runtime.js";
 import { registerOidcRoutes } from "./auth/oidc/endpoints.js";
+import { createScimStore } from "./scim/store.js";
+import { registerScimRoutes } from "./scim/routes.js";
 import { BuiltinPolicyEngine } from "./auth/policy/engine.js";
-import { loadPolicyFromFile, PolicyLoadError, VALID_RESOURCES, VALID_ACTIONS } from "./auth/policy/loader.js";
+import { loadPolicyFromFile, writePolicyFile, PolicyLoadError, VALID_RESOURCES, VALID_ACTIONS } from "./auth/policy/loader.js";
 import { OpaPolicyEngine } from "./auth/policy/opa.js";
+import { evaluateBatch, batchResultToCsv } from "./auth/policy/batch-dry-run.js";
 import { AuditLog } from "./audit/log.js";
 import { buildAuditMiddleware } from "./audit/middleware.js";
+import { WebhookSink } from "./audit/sinks/webhook.js";
+import { buildBypassBreadcrumb, buildBypassAuditParams } from "./audit/redaction-bypass.js";
 import { readCatalogFile, CatalogStore } from "./catalog/loader.js";
 import { readProductsFile, ProductsStore, validateProduct, writeProductsFile, ProductsLoadError } from "./products/loader.js";
+import { REGISTERED_TOOL_NAMES, REGISTERED_TOOLS, unknownToolNames } from "./tools/registry-names.js";
 import { redactValue } from "./policy/redact.js";
-import { IdentityRateLimiter, resolveToolRatePerMin } from "./quota/limiter.js";
+import { IdentityRateLimiter, resolveToolRatePerMin, parseKeyRateLimits } from "./quota/limiter.js";
 import { TokenBudget, estimateTokensFor, resolveDailyTokenLimit } from "./quota/token-budget.js";
+import { applyBudgetDecision } from "./quota/charge.js";
 import { getPluginLoader } from "./connectors/loader.js";
 import { resolveHubCatalogUrl, describeInstalled, mergeCatalog, fetchHubCatalog, } from "./connectors/hub.js";
 import { isValidConnectorName, installTarball } from "./connectors/install.js";
 import { PluginVerificationError } from "./connectors/verify.js";
-import { selfRegistry, withToolMetrics, apiRequests, mcpActiveSessions } from "./metrics/self.js";
+import { selfRegistry, withToolMetrics, apiRequests, mcpActiveSessions, auditDlqDepth } from "./metrics/self.js";
+import { initOtel } from "./observability/otel.js";
+import { WebSocketServerTransport } from "./transport/websocket.js";
+import { HookRegistry } from "./sdk/hooks.js";
+import { wrapToolHandler, wrapResourceHandler, wrapPromptHandler } from "./sdk/hook-wrappers.js";
+import { UpstreamClient } from "./federation/upstream.js";
+import { FederationRegistry, parseFederationEnv } from "./federation/registry.js";
+import { buildCsrfIssuer, buildCsrfEnforcer, csrfBypassFromEnv } from "./auth/csrf.js";
+import { checkOutboundUrl, ssrfGuardFromEnv } from "./middleware/ssrfGuard.js";
 import { buildOpenApiSpec } from "./openapi.js";
 import { listSourcesHandler } from "./tools/list-sources.js";
 import { listServicesHandler } from "./tools/list-services.js";
 import { queryMetricsHandler } from "./tools/query-metrics.js";
 import { queryLogsHandler } from "./tools/query-logs.js";
+import { queryTracesHandler } from "./tools/query-traces.js";
+import { getAnomalyHistoryHandler } from "./tools/get-anomaly-history.js";
+import { generatePostmortemHandler } from "./tools/generate-postmortem.js";
+import { PostmortemStore } from "./postmortem/store.js";
+import { AnomalyHistory, fromEnv as anomalyHistoryFromEnv } from "./analysis/history.js";
 import { getServiceHealthHandler, setHealthThresholds } from "./tools/get-service-health.js";
 import { detectAnomaliesHandler } from "./tools/detect-anomalies.js";
 import { getTopologyHandler, getBlastRadiusHandler } from "./tools/topology.js";
@@ -82,15 +103,7 @@ function qstr(v) {
  * so a downstream investigator can join the two channels there.
  */
 function emitBypassEvent(event, ctx, args) {
-    console.error(JSON.stringify({
-        event,
-        ts: new Date().toISOString(),
-        auth: ctx.auth,
-        tool: "query_logs",
-        service: args?.service ?? null,
-        correlationId: ctx.correlationId,
-        ...(event === "redaction_bypass_denied" ? { reason: "credential_not_in_OMCP_KEY_BYPASS_REDACTION" } : {}),
-    }));
+    console.error(JSON.stringify(buildBypassBreadcrumb(event, ctx, args)));
 }
 /** Bridge from the new PolicyEngine to the existing
  *  hasPermission/buildRequirePermission signatures (which still take
@@ -127,21 +140,25 @@ function getAvailableMetricNames(registry) {
 }
 /** Validate source URL: must be http/https, reject obviously dangerous targets */
 function validateSourceUrl(url) {
+    // Phase F11: delegate to the shared SSRF guard. Strict by default;
+    // operators add OMCP_ALLOW_PRIVATE_BACKENDS=true to allow in-cluster
+    // backends. Cloud-metadata IPs (AWS 169.254.169.254, GCE
+    // fd00:ec2::254) are rejected regardless.
+    const v = checkOutboundUrl(url, ssrfGuardFromEnv());
+    if (!v.allow)
+        return v.reason ?? `URL "${url}" is rejected by the SSRF guard`;
+    // Extra Google-metadata-hostname check (DNS-based, not in the
+    // numeric guard).
     try {
-        const parsed = new URL(url);
-        if (!["http:", "https:"].includes(parsed.protocol)) {
-            return `Invalid URL scheme "${parsed.protocol}". Only http and https are allowed.`;
-        }
-        // Block cloud metadata endpoints
-        const host = parsed.hostname.toLowerCase();
-        if (host === "169.254.169.254" || host === "metadata.google.internal") {
+        const host = new URL(url).hostname.toLowerCase();
+        if (host === "metadata.google.internal") {
             return "Access to cloud metadata endpoints is not allowed.";
         }
-        return null;
     }
     catch {
-        return `Invalid URL: "${url}"`;
+        /* already caught by checkOutboundUrl */
     }
+    return null;
 }
 // Hard cap for a downloaded/uploaded connector tarball (defence against
 // a hostile or accidental huge artifact OOM-ing the server).
@@ -169,6 +186,13 @@ async function main() {
     if (STDIO) {
         console.log = (...a) => console.error(...a);
     }
+    // OpenTelemetry self-tracing — opt-in via OMCP_OTEL_ENABLED. Init
+    // before express() so HTTP auto-instrumentation captures every
+    // /api/* and /mcp request. Skipped in stdio mode (no HTTP surface
+    // and the auto-instrumentation would emit noise per stdio call).
+    if (!STDIO) {
+        await initOtel({ serviceVersion: process.env.npm_package_version });
+    }
     let config = loadConfig();
     await getPluginLoader().load();
     const registry = new ConnectorRegistry();
@@ -238,37 +262,7 @@ async function main() {
         const text = result.content[0]?.text ?? "";
         const tokens = estimateTokensFor(text);
         const decision = tokenBudget.check(identityKey(ctx), tokens);
-        if (decision.allowed || decision.limit === 0)
-            return result;
-        // A single request larger than the entire daily cap can never
-        // succeed by waiting — surface a distinct error code so the
-        // agent doesn't loop. Otherwise the wait-then-retry path is the
-        // right answer (and freedAtRetry tells the agent how much they
-        // can request after the wait).
-        const requestExceedsCap = tokens > decision.limit;
-        const errBody = {
-            error: requestExceedsCap ? "OMCP_TOKEN_REQUEST_EXCEEDS_BUDGET" : "OMCP_TOKEN_BUDGET_EXCEEDED",
-            tool: toolName,
-            used: decision.used,
-            limit: decision.limit,
-            requested: tokens,
-            retryAfterSeconds: requestExceedsCap ? 0 : decision.retryAfterSeconds,
-            freedAtRetry: decision.freedAtRetry,
-            message: requestExceedsCap
-                ? `This single response (~${tokens} tokens) is larger than the entire daily budget (${decision.limit}). Retrying won't help — narrow the query (smaller window / lower limit / more selective filter) or raise OMCP_TOOL_DAILY_TOKENS.`
-                : `Daily token budget exceeded (${decision.used}/${decision.limit} tokens used in the trailing 24h; this call would have added ~${tokens}). Try again in ~${Math.ceil(decision.retryAfterSeconds / 3600)}h or raise OMCP_TOOL_DAILY_TOKENS.`,
-        };
-        // Preserve any additional content entries (e.g. a future
-        // tool returning [text, image]) — only the text payload of the
-        // first entry is replaced with the error JSON; everything after
-        // it passes through.
-        return {
-            ...result,
-            content: [
-                { ...result.content[0], text: JSON.stringify(errBody) },
-                ...result.content.slice(1),
-            ],
-        };
+        return applyBudgetDecision(result, decision, tokens, toolName);
     }
     const REDACTION_ENABLED = String(process.env.OMCP_REDACTION ?? "on").toLowerCase() !== "off";
     function redactToolText(result, opts = {}) {
@@ -303,13 +297,73 @@ async function main() {
             return result;
         }
     }
+    /**
+     * Returns the McpServer for the given context. The companion
+     * `toolHandlers` map carries every tool registered for this ctx
+     * (post-hook-wrapping) so the in-product Playground UI (Q13) can
+     * invoke a tool without going through the full Streamable HTTP
+     * transport stack. The map is keyed by tool name; values run the
+     * same wrapped handler the McpServer would dispatch over MCP.
+     */
     function createMcpServer(ctx) {
         const mcpServer = new McpServer({
             name: "observability-mcp",
             version: SERVER_VERSION,
         });
+        const toolHandlers = new Map();
         // --- Register tools with Zod schemas ---
-        mcpServer.tool("list_sources", [
+        // Product-aware registration: when the active credential is bound
+        // to a Product (OMCP_KEY_PRODUCTS), `ctx.allowedTools` carries that
+        // Product's `tools` allow-list and we skip the registration of any
+        // tool not in it. Anonymous + Product-less sessions leave
+        // allowedTools undefined and see every tool — the bypass is the
+        // back-compat path the open-source default relies on.
+        //
+        // The wrapper also wires Phase F7 hook fan-out: every tool dispatch
+        // fires tool_pre_invoke before the handler and tool_post_invoke after.
+        // Plugins can deny the call (allow:false → isError CallToolResult),
+        // mutate the args before dispatch, or mutate the result before it
+        // reaches the caller. When no hooks are registered (the default in
+        // the OSS demo) the wrapper is a thin pass-through.
+        const registerTool = ((name, ...rest) => {
+            if (!allowsTool(ctx.allowedTools, name))
+                return undefined;
+            if (rest.length > 0 && typeof rest[rest.length - 1] === "function") {
+                const originalHandler = rest[rest.length - 1];
+                const wrappedHandler = wrapToolHandler(hookRegistry, { principal: ctx.principalId, tenant: ctx.tenant || "default", target: name }, originalHandler);
+                rest[rest.length - 1] = wrappedHandler;
+                // Stash for the Playground endpoint — keyed by tool name. The
+                // wrapped handler honours pre/post hooks + the same RBAC the
+                // McpServer dispatch path runs. Per-ctx Map so a different
+                // user's allowedTools never leak.
+                toolHandlers.set(name, wrappedHandler);
+            }
+            return mcpServer.tool(name, ...rest);
+        });
+        // Q12: resource + prompt registrations get the same hook-fan-out
+        // treatment so a plugin's resource_pre_fetch / resource_post_fetch /
+        // prompt_pre_fetch / prompt_post_fetch handlers actually fire when
+        // a future resource/prompt registration lands. The wrappers stay
+        // thin pass-throughs when no hooks are registered (the OSS default).
+        // Wrappers are tested in mcp-server/src/sdk/hook-wrappers.test.ts.
+        const registerResource = ((name, ...rest) => {
+            if (rest.length > 0 && typeof rest[rest.length - 1] === "function") {
+                const originalHandler = rest[rest.length - 1];
+                rest[rest.length - 1] = wrapResourceHandler(hookRegistry, { principal: ctx.principalId, tenant: ctx.tenant || "default", target: name }, originalHandler);
+            }
+            return mcpServer.resource(name, ...rest);
+        });
+        const registerPrompt = ((name, ...rest) => {
+            if (rest.length > 0 && typeof rest[rest.length - 1] === "function") {
+                const originalHandler = rest[rest.length - 1];
+                rest[rest.length - 1] = wrapPromptHandler(hookRegistry, { principal: ctx.principalId, tenant: ctx.tenant || "default", target: name }, originalHandler);
+            }
+            return mcpServer.prompt(name, ...rest);
+        });
+        // Suppress unused-warn — kept for the moment registrations land.
+        void registerResource;
+        void registerPrompt;
+        registerTool("list_sources", [
             "List the configured observability backends (Prometheus, Loki, and any connector) and whether each is currently reachable.",
             "When to use: call this first to learn which source names exist and are healthy before passing `source` to other tools, or to debug why a query returns no data.",
             "Behavior: read-only, no side effects. Returns one entry per source with its name, type, configured URL, signal types (metrics/logs), and a live up/down status. Never throws for an unreachable backend — the backend is reported as down instead.",
@@ -318,7 +372,7 @@ async function main() {
             await enforceEntitledAccess(ctx, { tool: "list_sources" });
             return withToolMetrics("list_sources", () => listSourcesHandler(registry, ctx));
         });
-        mcpServer.tool("list_services", [
+        registerTool("list_services", [
             "Discover the service names that can be queried, aggregated across every connected backend.",
             "When to use: call this before `query_metrics`, `query_logs`, or `get_service_health` to obtain the exact, case-sensitive service name those tools require.",
             "Behavior: read-only, no side effects. Returns one entry per service with the service name, the source(s) it was discovered in, and which signals are available for it (metrics, logs, or both).",
@@ -336,7 +390,7 @@ async function main() {
         const metricsList = getAvailableMetricNames(registry);
         const metricNames = registry.getBySignal("metrics").flatMap(c => c.getMetrics().map(m => m.name));
         const uniqueNames = [...new Set(metricNames)];
-        mcpServer.tool("query_metrics", [
+        registerTool("query_metrics", [
             "Fetch the raw time-series for ONE metric of ONE service over a look-back window, returned together with pre-computed summary statistics.",
             "When to use: when you need the actual numeric values or the trend of a known metric. For a 'is this service OK?' verdict use `get_service_health`; to find which services are misbehaving use `detect_anomalies`.",
             "Prerequisites: get the exact service name from `list_services` and choose a metric from the list at the end of this description.",
@@ -366,7 +420,7 @@ async function main() {
             const result = await withToolMetrics("query_metrics", () => queryMetricsHandler(registry, args, ctx));
             return chargeTokenBudget(result, ctx, "query_metrics");
         });
-        mcpServer.tool("query_logs", [
+        registerTool("query_logs", [
             "Fetch recent log entries for ONE service over a look-back window, with a pre-computed summary (error/warning counts and the most frequent error patterns).",
             "When to use: to inspect what a service actually logged, or to investigate an error spike surfaced by `detect_anomalies` / `get_service_health`. For numeric metrics use `query_metrics` instead.",
             "Prerequisites: get the exact service name from `list_services` (the service must expose a logs signal).",
@@ -419,16 +473,7 @@ async function main() {
                 //      invocation is tamper-evident alongside the rest of
                 //      /api/*. Persists if OMCP_MGMT_AUDIT_FILE is set.
                 emitBypassEvent(bypass ? "redaction_bypass_engaged" : "redaction_bypass_denied", ctx, args);
-                void mgmtAudit.record({
-                    actor: { sub: ctx.principalId },
-                    tenant: ctx.tenant,
-                    resource: "redaction",
-                    action: "bypass",
-                    method: "MCP",
-                    path: "/mcp/query_logs",
-                    status: bypass ? 200 : 403,
-                    target: args?.service ?? undefined,
-                }).catch(() => {
+                void mgmtAudit.record(buildBypassAuditParams(bypass, ctx, args)).catch(() => {
                     // Audit record is best-effort — losing one entry must not
                     // crash the tool call. The chain itself remains intact.
                 });
@@ -436,7 +481,54 @@ async function main() {
             const redacted = redactToolText(result, { bypass });
             return chargeTokenBudget(redacted, ctx, "query_logs");
         });
-        mcpServer.tool("get_service_health", [
+        registerTool("get_anomaly_history", [
+            "Replay historical anomaly scores for a service from the TSDB the gateway writes to (omcp_anomaly_score series).",
+            "When to use: post-mortem reconstruction, trend analysis on detector noise, or pulling context for the LLM when an incident is reviewed after the fact.",
+            "Prerequisites: the operator must have OMCP_ANOMALY_HISTORY_REMOTE_WRITE configured AND a Prometheus source pointed at the same TSDB so the round-trip closes.",
+            "Behavior: read-only. Returns the time-series of scores. Empty result means either no anomalies in the window or history is disabled.",
+            "Related: `detect_anomalies` for the live scores; `query_metrics` if you want to write the PromQL by hand.",
+        ].join(" "), {
+            service: z.string().describe("Service name to filter on."),
+            duration: z.string().optional().describe("Rolling window, e.g. '1h', '24h'. Default '1h'."),
+            method: z.string().optional().describe("Filter by detector method ('mad' / 'seasonality' / 'correlator'). Optional."),
+        }, async (args) => {
+            await enforceEntitledAccess(ctx, { tool: "get_anomaly_history", service: args?.service });
+            const result = await withToolMetrics("get_anomaly_history", () => getAnomalyHistoryHandler(registry, args, ctx));
+            return chargeTokenBudget(result, ctx, "get_anomaly_history");
+        });
+        registerTool("generate_postmortem", [
+            "Stitch the gateway's primitives (anomaly history, blast-radius, traces, log highlights) into a single markdown post-mortem report for one service over a given window.",
+            "When to use: after an incident, when the operator or LLM wants 'one document the on-call can read in 60 seconds' instead of poking the individual tools.",
+            "Prerequisites: anomaly history requires OMCP_ANOMALY_HISTORY_REMOTE_WRITE + a Prometheus source. Traces require Tempo / Jaeger. Blast-radius requires a topology provider.",
+            "Behavior: read-only. Returns markdown by default; pass `format='json'` for the structured shape. Output capped (timeline 20 rows, blast-radius 30 nodes, 10 traces) — JSON shape carries the full data.",
+            "Related: `get_anomaly_history`, `query_traces`, `get_blast_radius` for the underlying primitives.",
+        ].join(" "), {
+            service: z.string().describe("Suspected root-cause service."),
+            duration: z.string().optional().describe("Window length, e.g. '1h', '6h'. Default '1h'."),
+            format: z.enum(["markdown", "json"]).optional().describe("'markdown' (default) or 'json'."),
+        }, async (args) => {
+            await enforceEntitledAccess(ctx, { tool: "generate_postmortem", service: args?.service });
+            const result = await withToolMetrics("generate_postmortem", () => generatePostmortemHandler(registry, args, ctx));
+            return chargeTokenBudget(result, ctx, "generate_postmortem");
+        });
+        registerTool("query_traces", [
+            "Query distributed traces for a service over a given timeframe.",
+            "Returns ranked trace summaries (duration, span count, error status) with a p50/p95 aggregate across the returned set.",
+            "When to use: investigate tail-latency outliers, walk call chains across services for a specific time window, or pull traces related to an anomaly that the metric/log tools surfaced first.",
+            "Prerequisites: get the exact service name from `list_services`. A Tempo / Jaeger / OTLP connector must be configured.",
+            "Behavior: read-only. `filter` accepts the backend's native query language (TraceQL on Tempo, tag query on Jaeger). When `errorsOnly=true`, only traces with at least one error span are returned. Default limit is 50.",
+        ].join(" "), {
+            service: z.string().describe("Service name (e.g. 'payment-service')."),
+            duration: z.string().optional().describe("Rolling time window, e.g. '5m', '1h'. Default '15m'."),
+            filter: z.string().optional().describe("Backend-native filter (TraceQL on Tempo, tag query on Jaeger). Optional."),
+            limit: z.number().int().positive().optional().describe("Soft cap on returned trace summaries. Default 50."),
+            errorsOnly: z.boolean().optional().describe("If true, only traces with at least one error span."),
+        }, async (args) => {
+            await enforceEntitledAccess(ctx, { tool: "query_traces", service: args?.service });
+            const result = await withToolMetrics("query_traces", () => queryTracesHandler(registry, args, ctx));
+            return chargeTokenBudget(result, ctx, "query_traces");
+        });
+        registerTool("get_service_health", [
             "Produce a single aggregated health verdict for ONE service by combining its metrics and logs.",
             "When to use: the fastest way to answer 'is this service healthy right now and why?'. Use `query_metrics`/`query_logs` to drill into the underlying numbers, or `detect_anomalies` to scan many services at once.",
             "Prerequisites: get the exact service name from `list_services`.",
@@ -451,7 +543,7 @@ async function main() {
             const enriched = enrichToolHealthText(result, String(args?.service ?? ""), ctx);
             return chargeTokenBudget(enriched, ctx, "get_service_health");
         });
-        mcpServer.tool("detect_anomalies", [
+        registerTool("detect_anomalies", [
             "Scan one or all monitored services for abnormal behavior and return the findings ranked by severity.",
             "When to use: the entry point for 'is anything wrong anywhere?' triage. Once a service is flagged, follow up with `get_service_health` for the verdict or `query_metrics`/`query_logs` for the raw evidence.",
             "Behavior: read-only, no side effects. Applies z-score analysis to metrics, detects log error-rate spikes, and correlates the two. Returns a list of anomalies, each with the affected service, metric/signal, severity, the deviation (e.g. σ and % change), and a short explanation. No anomalies yields an empty list, not an error.",
@@ -471,9 +563,11 @@ async function main() {
                 .describe("Optional. Detection threshold: 'low' flags only strong deviations (>3σ), 'medium' is balanced (>2σ), 'high' is most sensitive and noisier (>1.5σ). Default: 'medium'."),
         }, async (args) => {
             await enforceEntitledAccess(ctx, { tool: "detect_anomalies", source: args?.source, service: args?.service });
-            return withToolMetrics("detect_anomalies", () => detectAnomaliesHandler(registry, args, ctx));
+            // P1: pass the anomaly-history sink so detected scores flow
+            // into the TSDB and `get_anomaly_history` returns real data.
+            return withToolMetrics("detect_anomalies", () => detectAnomaliesHandler(registry, args, ctx, anomalyHistory));
         });
-        mcpServer.tool("get_topology", [
+        registerTool("get_topology", [
             "Return the infrastructure topology graph (Resources and Edges) from every topology-capable connector.",
             "When to use: when an agent needs to reason about which workload runs on which host, who owns whom, or which scope (namespace/project/folder) a resource belongs to. Pair with `get_blast_radius` for shared-host RCA.",
             "Behavior: read-only, no side effects. Returns `{ sources, resources, edges, total, truncated }`. Filters compose: `source` to one connector, `kind` to one resource type (e.g. 'pod', 'node', 'deployment'), `scope` to members of a namespace/folder/project. Output is capped by `limit` (default 500, max 5000) and edges referencing dropped resources are removed.",
@@ -502,7 +596,7 @@ async function main() {
             await enforceEntitledAccess(ctx, { tool: "get_topology", source: args?.source });
             return withToolMetrics("get_topology", () => getTopologyHandler(registry, args, ctx));
         });
-        mcpServer.tool("get_blast_radius", [
+        registerTool("get_blast_radius", [
             "Given a resource, return who else fails if its underlying host(s) fail.",
             "When to use: cross-cutting RCA — when several services degrade together and you suspect a shared host. Works for any RUNS_ON relationship: pod→node, vm→hypervisor, container→host.",
             "Behavior: read-only, no side effects. Resolves `resource` to a Resource (accepts canonical id, exact name, or unique substring), determines its host(s) via RUNS_ON, then lists every other resource that runs on those hosts, bucketed by ownership root (the terminal `OWNED_BY` target — e.g. the Deployment, not the ReplicaSet). If the target is itself a host, its tenants are reported. Returns a structured error if the resource is ambiguous or unknown.",
@@ -515,7 +609,41 @@ async function main() {
             await enforceEntitledAccess(ctx, { tool: "get_blast_radius" });
             return withToolMetrics("get_blast_radius", () => getBlastRadiusHandler(registry, args, ctx));
         });
-        return mcpServer;
+        // Phase F10: federated tools — every upstream MCP server's tools
+        // show up here under `<prefix>.<upstream-tool>`. The handler is a
+        // pure proxy: it forwards args verbatim and returns the upstream's
+        // CallToolResult unchanged. The wrapping registerTool() at the top
+        // of this function still fires F7 lifecycle hooks + the F1
+        // Product-allow-list gate, so federated tools obey the same policy
+        // surface as native ones.
+        for (const info of federationRegistry.getNamespacedTools()) {
+            // The MCP SDK's tool() signature wants a ZodRawShape (a map of
+            // field-name → Zod type), not a raw JSON Schema. Federated
+            // upstreams expose JSON Schema (the wire-format MCP uses on
+            // tools/list); we transcode to a permissive Zod shape so the
+            // SDK accepts the registration. Per-field types are `z.unknown()`
+            // because the upstream will validate the call args anyway; the
+            // local Zod check is only a "this is the field name set" gate.
+            // P7: this transcoding fixes the registration crash that broke
+            // every federation deploy before the E2E test caught it.
+            const upstreamProps = info.inputSchema?.properties ?? {};
+            // Every field is z.unknown().optional() — the SDK only uses this
+            // shape to know the field-name set; the upstream re-validates
+            // against its full JSON Schema (incl. its own `required` list)
+            // when the call arrives. Marking all fields optional here keeps
+            // calls with the upstream-defaults flowing through; without it
+            // the SDK rejects any call that omits a field upstream considers
+            // required even if the upstream would accept the omission.
+            const localShape = {};
+            for (const k of Object.keys(upstreamProps)) {
+                localShape[k] = z.unknown().optional();
+            }
+            registerTool(info.namespacedName, info.description || `Federated from upstream ${info.sourceName}.`, localShape, async (args) => {
+                await enforceEntitledAccess(ctx, { tool: info.namespacedName });
+                return withToolMetrics(info.namespacedName, () => federationRegistry.callNamespacedTool(info.namespacedName, args));
+            });
+        }
+        return { mcpServer, toolHandlers };
     }
     // --- Management-plane auth (basic mode) -----------------------------------
     // Off by default. Enable with `OMCP_AUTH=basic` + `OMCP_USERS_FILE` and
@@ -624,7 +752,12 @@ async function main() {
             app.set("trust proxy", trustProxy);
         }
     }
-    app.use(express.json({ limit: "1mb" }));
+    // Parse application/json AND any *+json media type. SCIM clients
+    // (Entra, Okta) send `application/scim+json` per RFC 7644 §3.1 —
+    // without the wildcard the body silently arrives empty and every
+    // SCIM POST/PATCH 400s. The wildcard also future-proofs other
+    // structured-suffix JSON content types.
+    app.use(express.json({ limit: "1mb", type: ["application/json", "application/*+json"] }));
     // Security headers
     app.use((req, res, next) => {
         res.setHeader("X-Content-Type-Options", "nosniff");
@@ -673,6 +806,17 @@ async function main() {
     // there is no string-match-based "is this public?" branch anywhere.
     app.use(buildSessionAttacher(authRuntime));
     const requireSession = buildRequireSession(authRuntime);
+    // Phase F11: CSRF — double-submit cookie pattern, enforced on every
+    // mutating /api/* request. The issuer runs top-of-pipe so any page
+    // render leaves a CSRF token cookie the SPA can read + echo back.
+    // Bearer-token clients (CI, agents, MCP clients) bypass by default
+    // since they can't be a browser confused-deputy.
+    const csrfCfg = {
+        bypassBearer: csrfBypassFromEnv(),
+        secureCookie: (r) => r.secure || r.headers["x-forwarded-proto"] === "https",
+    };
+    app.use(buildCsrfIssuer(csrfCfg));
+    app.use("/api", buildCsrfEnforcer(csrfCfg));
     // Active policy engine — built-in DEFAULT_POLICY by default. When
     // OMCP_RBAC_POLICY_FILE is set we load it and ALWAYS abort on
     // failure: OMCP_AUTH_ALLOW_FALLBACK is for *auth-mode* fallback
@@ -713,22 +857,42 @@ async function main() {
                 return;
             const resources = [...VALID_RESOURCES];
             const actions = [...VALID_ACTIONS];
+            // Tenant-aware pre-warm: the gate keys cache per
+            // (roles, resource, action, tenant) so a tenant-conditional
+            // Rego rule that fires for "acme" but not "bigco" produces a
+            // distinct cached verdict per tenant. The pre-warm iterates
+            // every known declared tenant + "default" so the first user
+            // request from a tenant'd identity gets a real decision
+            // instead of a warming-deny. OIDC tenants only known at
+            // runtime are still subject to first-request warming, but
+            // operator-set OMCP_KEY_TENANTS land here.
+            const knownTenants = new Set(["default"]);
+            // parseKeyTenants is the same parser the credentials layer
+            // uses, so the warm set is exactly what the gate will see.
+            for (const t of parseKeyTenants(process.env.OMCP_KEY_TENANTS).values()) {
+                if (t)
+                    knownTenants.add(t);
+            }
+            const tenants = Array.from(knownTenants);
             const tasks = [];
-            for (const role of roles) {
-                for (const resource of resources)
-                    for (const action of actions) {
-                        tasks.push(opaEngine.warmEvaluate([role], resource, action));
-                    }
-                tasks.push(opaEngine.warmList([role]));
+            for (const tenant of tenants) {
+                for (const role of roles) {
+                    for (const resource of resources)
+                        for (const action of actions) {
+                            tasks.push(opaEngine.warmEvaluate([role], resource, action, tenant));
+                        }
+                    tasks.push(opaEngine.warmList([role], tenant));
+                }
             }
             try {
                 const settled = await Promise.allSettled(tasks);
                 const failed = settled.filter((s) => s.status === "rejected").length;
+                const tlbl = tenants.length === 1 ? "1 tenant" : `${tenants.length} tenants`;
                 if (failed === 0) {
-                    console.log(`[auth] OPA cache pre-warmed: ${settled.length} decisions cached for ${roles.length} role(s)`);
+                    console.log(`[auth] OPA cache pre-warmed: ${settled.length} decisions cached for ${roles.length} role(s) × ${tlbl}`);
                 }
                 else {
-                    console.warn(`[auth] OPA cache pre-warmed: ${settled.length - failed}/${settled.length} ok, ${failed} failed (gates will retry on first user call)`);
+                    console.warn(`[auth] OPA cache pre-warmed: ${settled.length - failed}/${settled.length} ok, ${failed} failed across ${tlbl} (gates will retry on first user call)`);
                 }
             }
             catch { /* best-effort */ }
@@ -745,20 +909,108 @@ async function main() {
             process.exit(1);
         }
     }
-    const need = (resource, action) => buildRequirePermission(authRuntime, resource, action, policyEngineToMap(policyEngine));
+    // Use the engine-aware variant so tenant (session.tenant) flows into
+    // engine.evaluate() — required for tenant-conditional Rego rules
+    // (`input.tenant == "acme"` etc.) under OMCP_OPA_URL. Built-in /
+    // file-loaded engines ignore the tenant ctx, so the behaviour is
+    // unchanged for those deployments.
+    const need = (resource, action) => buildRequirePermissionFromEngine(authRuntime, resource, action, policyEngine);
     // Management-plane audit log. Records one entry per mutating /api/*
     // request. Writes JSONL to disk when OMCP_MGMT_AUDIT_FILE is set;
     // otherwise an in-memory ring of the last 500 entries keeps the
     // /api/audit endpoint useful in the demo / single-user case.
-    const mgmtAudit = new AuditLog({ file: process.env.OMCP_MGMT_AUDIT_FILE });
+    // External audit sinks — opt-in via env. Each chained entry is
+    // mirrored to every configured sink; the on-disk JSONL master
+    // remains the source of truth (the hash chain is never split).
+    const auditSinks = [];
+    if (process.env.OMCP_AUDIT_WEBHOOK_URL) {
+        auditSinks.push(new WebhookSink({
+            url: process.env.OMCP_AUDIT_WEBHOOK_URL,
+            token: process.env.OMCP_AUDIT_WEBHOOK_TOKEN,
+            deadLetterFile: process.env.OMCP_AUDIT_WEBHOOK_DLQ,
+        }));
+        console.log("AuditLog: webhook sink enabled -> %s%s", process.env.OMCP_AUDIT_WEBHOOK_URL, process.env.OMCP_AUDIT_WEBHOOK_DLQ
+            ? ` (DLQ: ${process.env.OMCP_AUDIT_WEBHOOK_DLQ})`
+            : "");
+    }
+    const mgmtAudit = new AuditLog({
+        file: process.env.OMCP_MGMT_AUDIT_FILE,
+        sinks: auditSinks,
+    });
     await mgmtAudit.bootstrap();
+    process.on("SIGTERM", () => {
+        mgmtAudit
+            .flushSinks()
+            .catch((err) => console.warn("AuditLog flushSinks failed:", err));
+    });
     const audit = (resource, action) => buildAuditMiddleware({ audit: mgmtAudit, resource, action });
+    // Plugin lifecycle hook registry — populated by the loader at boot
+    // (one entry per manifest `hooks[]` entry) and mutable at runtime
+    // when a connector is installed via /api/connectors/install. Each
+    // tool dispatch in createMcpServer fans through this registry's
+    // tool_pre_invoke / tool_post_invoke chains; resource and prompt
+    // hooks plug into their respective seams as they ship.
+    const hookRegistry = new HookRegistry();
+    // Phase F15: anomaly-history sink — opt-in via
+    // OMCP_ANOMALY_HISTORY_REMOTE_WRITE. When configured, anomaly
+    // scores written via anomalyHistory.record() flush to the
+    // configured TSDB on a 10-second timer. The MCP tool
+    // get_anomaly_history queries them back via any Prometheus source
+    // pointed at the same TSDB.
+    //
+    // The detector-side hook that actually records per-anomaly scores
+    // is plumbed in F15b (it requires passing this instance into the
+    // detectAnomaliesHandler — minor surgery deferred). The
+    // infrastructure ships now so externally-written omcp_anomaly_score
+    // metrics are already queryable end-to-end.
+    const anomalyHistory = new AnomalyHistory(anomalyHistoryFromEnv());
+    anomalyHistory.start();
+    if (anomalyHistory.isEnabled()) {
+        console.log("AnomalyHistory: TSDB sink enabled (OMCP_ANOMALY_HISTORY_REMOTE_WRITE set)");
+    }
+    process.on("SIGTERM", () => {
+        void anomalyHistory.stop().catch(() => undefined);
+    });
+    // Federation registry — populated from OMCP_FEDERATION_UPSTREAMS at
+    // boot. Each upstream connects, fetches tools/list, and exposes its
+    // tools under `<prefix>.<upstream-tool-name>` on the gateway's
+    // surface. Failures are logged + the upstream is left in `degraded`
+    // (no tools) so the gateway boots regardless of upstream health.
+    const federationRegistry = new FederationRegistry();
+    for (const cfg of parseFederationEnv()) {
+        const client = new UpstreamClient(cfg.kind === "stdio"
+            ? { transport: "stdio", name: cfg.name, command: cfg.command, args: cfg.args }
+            : cfg.kind === "ws"
+                ? { transport: "ws", name: cfg.name, url: cfg.url }
+                : { name: cfg.name, url: cfg.url, bearerToken: cfg.bearerToken });
+        federationRegistry.add(client);
+        client.connect().catch((err) => {
+            console.warn("federation upstream %s initial connect failed: %s", cfg.name, err instanceof Error ? err.message : String(err));
+        });
+    }
+    if (federationRegistry.list().length > 0) {
+        console.log("federation: %d upstream(s) configured: %s", federationRegistry.list().length, federationRegistry.list().map((u) => `${u.name}=${u.url}`).join(", "));
+    }
+    process.on("SIGTERM", () => {
+        federationRegistry
+            .closeAll()
+            .catch((err) => console.warn("federation closeAll failed:", err));
+    });
     // Service catalog: optional operator-curated ownership / criticality /
     // on-call metadata, keyed on the service name list_services returns.
     // No file ⇒ empty catalog, enrichment is a no-op (anonymous demos
     // see no behaviour change).
     const catalog = new CatalogStore(await readCatalogFile(process.env.OMCP_SERVICE_CATALOG_FILE));
-    const products = new ProductsStore(await readProductsFile(process.env.OMCP_PRODUCTS_FILE));
+    // Hot-reload aware: passing the path lets `products.maybeReload()`
+    // pick up out-of-band edits to OMCP_PRODUCTS_FILE without a restart.
+    // Each /api/products* handler awaits maybeReload() before reading,
+    // so a `kubectl apply` of an updated ConfigMap or a git-ops edit is
+    // visible on the very next request.
+    const productsPath = process.env.OMCP_PRODUCTS_FILE;
+    const products = new ProductsStore(await readProductsFile(productsPath), { path: productsPath });
+    // Seed the mtime cursor from the file we just loaded so the first
+    // maybeReload() call doesn't redundantly re-parse the boot state.
+    await products.pinMtimeAfterWrite();
     // Protected route prefixes. /api/me, /api/auth/*, /api/info,
     // /api/openapi.json deliberately don't appear here — they stay public.
     for (const prefix of [
@@ -784,6 +1036,39 @@ async function main() {
     // enough to skip the request-counter middleware.
     let ready = false;
     app.get("/healthz", (_req, res) => res.type("text").send("ok"));
+    // Procurement-time probe: the MCP spec revisions and transports the
+    // gateway supports. Static today — kept as a separate endpoint so a
+    // discovery tool / RFP probe / catalog scanner can resolve our
+    // compliance posture without sending a real MCP handshake.
+    // See docs/mcp-conformance.md for the test suite that proves it.
+    app.get("/api/conformance", (_req, res) => {
+        res.json({
+            revisions: ["2025-11-25"],
+            transports: ["streamable-http", "stdio", "websocket"],
+            methods: {
+                // Methods exercised by the conformance harness. "supported"
+                // is the union of methods that return a non -32601 envelope
+                // for any conforming caller. Per-method spec compliance is
+                // proven by src/conformance/mcp-2025-11-25.test.ts.
+                supported: [
+                    "initialize",
+                    "notifications/initialized",
+                    "ping",
+                    "tools/list",
+                    "tools/call",
+                ],
+                optional: [
+                    "resources/list",
+                    "resources/read",
+                    "prompts/list",
+                    "prompts/get",
+                    "logging/setLevel",
+                ],
+            },
+            harnessPath: "mcp-server/src/conformance/mcp-2025-11-25.test.ts",
+            docs: "docs/mcp-conformance.md",
+        });
+    });
     app.get("/readyz", (_req, res) => {
         if (ready)
             return res.type("text").send("ok");
@@ -799,6 +1084,24 @@ async function main() {
     // this endpoint when enabled.
     if (process.env.METRICS_ENABLED !== "false") {
         app.get("/metrics", async (_req, res) => {
+            // P9: refresh the audit-webhook DLQ depth before the scrape so
+            // Prometheus sees the current file state rather than whatever
+            // /api/audit/dlq last set. Best-effort; ENOENT or missing-env
+            // resets to 0 (the dlqPath being unset is the normal state).
+            try {
+                const dlqPath = process.env.OMCP_AUDIT_WEBHOOK_DLQ;
+                if (dlqPath) {
+                    const fs = await import("node:fs/promises");
+                    const raw = await fs.readFile(dlqPath, "utf8").catch(() => "");
+                    auditDlqDepth.set(raw.split("\n").filter((l) => l.trim()).length);
+                }
+                else {
+                    auditDlqDepth.set(0);
+                }
+            }
+            catch {
+                auditDlqDepth.set(0);
+            }
             res.set("Content-Type", selfRegistry.contentType);
             res.end(await selfRegistry.metrics());
         });
@@ -806,11 +1109,33 @@ async function main() {
     // Serve Web UI
     app.use(express.static(join(__dirname, "ui")));
     // --- API endpoints for Web UI ---
-    // List sources with health status
-    app.get("/api/sources", async (_req, res) => {
+    // List sources with health status — tenant-scoped.
+    // Non-admin callers see only their own tenant's sources + globals
+    // (untagged). Admins (users:delete) see everything, with optional
+    // ?tenant=acme drill-down. Anonymous mode (no session) sees
+    // everything — preserves single-tenant default. The `tenant` field
+    // is included on every entry so the UI can render scope badges.
+    app.get("/api/sources", async (req, res) => {
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        const requestedTenant = qstr(req.query.tenant);
         const health = await registry.healthCheckAll();
         const configs = registry.getSourceConfigs();
-        const sources = configs.map((c) => {
+        const filtered = configs.filter((c) => {
+            // Anonymous: every source.
+            if (!sess)
+                return true;
+            // Admin with ?tenant=X drill-down: untagged + that tenant.
+            if (isAdmin && requestedTenant)
+                return !c.tenant || c.tenant === requestedTenant;
+            // Admin no filter: every source (cross-tenant view).
+            if (isAdmin)
+                return true;
+            // Non-admin: own tenant + untagged.
+            return !c.tenant || c.tenant === callerTenant;
+        });
+        const sources = filtered.map((c) => {
             const connector = registry.getByName(c.name);
             return {
                 name: c.name,
@@ -819,6 +1144,7 @@ async function main() {
                 enabled: c.enabled,
                 auth: c.auth ? { type: c.auth.type } : undefined,
                 tls: c.tls || undefined,
+                tenant: c.tenant,
                 signalType: connector?.signalType || null,
                 status: health[c.name]?.status || (c.enabled ? "down" : "disabled"),
                 latencyMs: health[c.name]?.latencyMs || null,
@@ -831,6 +1157,46 @@ async function main() {
     app.get("/api/source-types", (_req, res) => {
         res.json(getSupportedTypes());
     });
+    // Get the registry of MCP tools the server can advertise (name +
+    // category + one-line summary). The Products modal uses this to
+    // populate the tools-allowlist picker so a typo can't happen at
+    // authoring time; the server-side typo guard (PR #343) stays as
+    // defence-in-depth. Open to every viewer — there's nothing
+    // sensitive in the catalogue, it's just static metadata.
+    app.get("/api/tools/registry", (_req, res) => {
+        res.json({ tools: REGISTERED_TOOLS });
+    });
+    // Q13: in-product Playground endpoint. Lets the operator invoke a
+    // registered tool against the live gateway without spinning up a
+    // separate MCP client. Re-uses the per-session ctx and the same
+    // wrapped handler the McpServer dispatch path would run (so RBAC,
+    // entitlements, rate-limit, audit, hook fan-out all apply
+    // identically).
+    app.post("/api/playground/invoke", async (req, res) => {
+        const ctx = await gateCtx(req, res);
+        if (!ctx)
+            return;
+        const body = (req.body ?? {});
+        const tool = typeof body.tool === "string" ? body.tool : "";
+        if (!tool) {
+            res.status(400).json({ error: "tool (string) is required" });
+            return;
+        }
+        const { toolHandlers } = createMcpServer(ctx);
+        const handler = toolHandlers.get(tool);
+        if (!handler) {
+            res.status(404).json({ error: `tool '${tool}' is not registered (or not allowed for this credential)` });
+            return;
+        }
+        try {
+            const result = await handler(body.args ?? {}, undefined);
+            res.json({ tool, result });
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            res.status(500).json({ error: message, tool });
+        }
+    });
     // Server info — version, loaded plugins, MCP protocol version, build metadata.
     // Used by the Web UI footer and by operators to confirm what's deployed.
     app.get("/api/info", async (_req, res) => {
@@ -865,6 +1231,16 @@ async function main() {
                 redaction: REDACTION_ENABLED,
                 trustProxy: !!(process.env.OMCP_TRUST_PROXY && process.env.OMCP_TRUST_PROXY !== "false"),
                 toolRatePerMin: resolveToolRatePerMin(process.env.OMCP_TOOL_RATE_PER_MIN),
+                // P1: posture flags so dashboards can alert when a shipped
+                // capability is configured but doing nothing useful.
+                anomalyHistoryActive: anomalyHistory.isEnabled(),
+                tracesCapabilityCount: registry
+                    .getAll()
+                    .filter((c) => typeof c.queryTraces === "function").length,
+                pluginsVerified: !/^(0|false|no|off)$/i.test(process.env.VERIFY_PLUGINS ?? "true"),
+                scimEnabled: !!process.env.OMCP_SCIM_TOKEN,
+                federationUpstreams: (process.env.OMCP_FEDERATION_UPSTREAMS ?? "")
+                    .split(",").map((s) => s.trim()).filter(Boolean).length,
             },
             plugins: loader.list().map((p) => ({
                 name: p.name,
@@ -923,9 +1299,18 @@ async function main() {
     // to non-admin sessions.
     app.get("/api/policy", need("users", "delete"), (req, res) => {
         const map = policyEngineToMap(policyEngine);
-        // Optional dry-run: ?roles=admin,operator&resource=sources&action=delete
+        // The OPA engine's kind() is prefixed `opa:` (see opa.ts:198).
+        // Surface a `tenantAware` boolean so operators can confirm at a
+        // glance whether the active engine honours session.tenant in
+        // .evaluate() — the BuiltinPolicyEngine ignores tenant ctx; OPA
+        // threads it into the Rego input. This is the property required
+        // for `allow { input.tenant == "acme" }` rules to actually fire.
+        const tenantAware = policyEngine.kind().startsWith("opa:");
+        // Optional dry-run: ?roles=admin,operator&resource=sources&action=delete[&tenant=acme]
         // returns { allowed, reason } so operators can probe the active
-        // engine without writing tests against a checkout.
+        // engine without writing tests against a checkout. Tenant defaults
+        // to the caller's session tenant; an admin can override via the
+        // ?tenant= query string to probe verdicts for any tenant.
         const q = req.query;
         if (q.resource && q.action) {
             const dryRoles = typeof q.roles === "string" ? q.roles.split(",").map((r) => r.trim()).filter(Boolean) : undefined;
@@ -940,12 +1325,30 @@ async function main() {
                 res.json({ dryRun: { roles: dryRoles ?? [], resource: q.resource, action: q.action, allowed: false, reason: `unknown action '${q.action}' (valid: ${[...VALID_ACTIONS].join(", ")})` } });
                 return;
             }
-            const result = policyEngine.evaluate(dryRoles, q.resource, q.action);
-            res.json({ dryRun: { roles: dryRoles ?? [], resource: q.resource, action: q.action, ...result } });
+            const callerSess = req.session;
+            // Tenant resolution: explicit ?tenant= override wins, else the
+            // caller's session tenant. The probe runs at users:delete (admin),
+            // so a cross-tenant override is intentional — exactly how an
+            // operator debugs "why doesn't my tenant-conditional Rego rule
+            // fire for tenant Acme?".
+            const probeTenant = typeof q.tenant === "string" && q.tenant
+                ? q.tenant.trim()
+                : callerSess?.tenant;
+            const result = policyEngine.evaluate(dryRoles, q.resource, q.action, probeTenant ? { tenant: probeTenant } : undefined);
+            res.json({
+                dryRun: {
+                    roles: dryRoles ?? [],
+                    resource: q.resource,
+                    action: q.action,
+                    tenant: probeTenant,
+                    ...result,
+                },
+            });
             return;
         }
         res.json({
             engine: policyEngine.kind(),
+            tenantAware,
             policy: map,
             roles: policyEngine.roles(),
             note: policyEngine.kind() === "builtin"
@@ -953,6 +1356,281 @@ async function main() {
                 : `policy loaded from ${policyEngine.kind()}; restart to reload.`,
         });
     });
+    // Phase F16: batch policy dry-run. Evaluates every
+    // (subject × resource × action) cell against the active engine and
+    // returns a matrix the UI heat-map renders. Gated identically to
+    // the single-call dry-run on GET /api/policy. Capped at 100×100×10
+    // cells per request — a single OPA query per cell is cheap on the
+    // BuiltinPolicyEngine but a careless caller could hose an external
+    // OPA, so the limit fences that. Operators get CSV via
+    // Accept: text/csv for ticket attachments.
+    app.post("/api/policy/dry-run-batch", need("users", "delete"), audit("policy", "read"), async (req, res) => {
+        const body = (req.body ?? {});
+        const subjects = Array.isArray(body.subjects) ? body.subjects : [];
+        const resources = Array.isArray(body.resources) ? body.resources : [];
+        const actions = Array.isArray(body.actions) ? body.actions : [];
+        const result = await evaluateBatch(policyEngine, { subjects, resources, actions }, VALID_RESOURCES, VALID_ACTIONS);
+        if (req.headers["accept"]?.toString().includes("text/csv")) {
+            res.type("text/csv").send(batchResultToCsv(result));
+            return;
+        }
+        res.json(result);
+    });
+    // --- /api/subjects — aggregated principals catalogue ------------------
+    // The third k8s-shaped RBAC view: who the deployment knows about.
+    // Three independent sources, returned in three independent arrays so
+    // the UI can table each section separately:
+    //   - users     : OMCP_USERS_FILE (basic-mode local users). Password
+    //                 hashes are never returned.
+    //   - apiKeys   : OMCP_API_KEYS names (the bearer-token catalogue).
+    //                 Tokens are never returned; only metadata (tenant,
+    //                 bound product, source allow-list, bypass flag).
+    //   - oidcGroups: keys of OMCP_OIDC_ROLE_MAP — every group the
+    //                 operator has explicitly mapped to an OMCP role.
+    //                 Runtime-only groups (claims that arrive without an
+    //                 OMCP-side mapping) are skipped on purpose; they
+    //                 produce no roles by definition.
+    // Gated identically to /api/policy.
+    app.get("/api/subjects", need("users", "delete"), async (_req, res) => {
+        // Local users.
+        const usersOut = [];
+        if (process.env.OMCP_USERS_FILE) {
+            try {
+                const f = await readUsersFile(process.env.OMCP_USERS_FILE);
+                if (f && Array.isArray(f.users)) {
+                    for (const u of f.users) {
+                        usersOut.push({
+                            username: u.username,
+                            name: u.name,
+                            roles: u.roles ? u.roles.slice() : [],
+                            tenant: u.tenant || "default",
+                        });
+                    }
+                }
+            }
+            catch (e) {
+                // Read failures don't 500 the whole endpoint — surface an
+                // empty users array; admins can check the boot log for the
+                // file-load diagnostic.
+                console.warn(`[/api/subjects] readUsersFile failed: ${e.message}`);
+            }
+        }
+        // API key credentials (tokens stripped).
+        const apiKeysOut = [];
+        for (const c of loadCredentials()) {
+            apiKeysOut.push({
+                name: c.name,
+                tenant: c.tenant || "default",
+                productId: c.productId,
+                bypassRedaction: !!c.bypassRedaction,
+                allowedSources: c.allowedSources,
+            });
+        }
+        // OIDC groups → role mappings.
+        const oidcGroupsOut = [];
+        const roleMapRaw = process.env.OMCP_OIDC_ROLE_MAP;
+        if (roleMapRaw) {
+            try {
+                const parsed = JSON.parse(roleMapRaw);
+                if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                    for (const [claim, role] of Object.entries(parsed)) {
+                        if (typeof role === "string" && claim) {
+                            oidcGroupsOut.push({ claim, role });
+                        }
+                    }
+                }
+            }
+            catch {
+                // The OIDC runtime already rejects an invalid role map at
+                // boot — if parsing fails here it's almost certainly a
+                // transient state during config reload. Surface empty.
+            }
+        }
+        res.json({
+            users: usersOut,
+            apiKeys: apiKeysOut,
+            oidcGroups: oidcGroupsOut,
+            // Surface which env vars actually drive each list so an
+            // admin diagnosing "where is my user?" sees the source path
+            // without having to read the deploy.
+            sources: {
+                users: process.env.OMCP_USERS_FILE || null,
+                apiKeys: process.env.OMCP_API_KEYS ? "OMCP_API_KEYS" : null,
+                oidcGroups: process.env.OMCP_OIDC_ROLE_MAP ? "OMCP_OIDC_ROLE_MAP" : null,
+            },
+        });
+    });
+    // Update a user's roles. Today this is the only binding-shape that
+    // OMCP can actually mutate at runtime: api-key roles aren't stored
+    // anywhere (creds carry sources / tenant / product but not roles),
+    // and OIDC group → role mappings come from OMCP_OIDC_ROLE_MAP which
+    // is read once at boot. The Bindings UI surface api-key + oidc rows
+    // explain the env-source path instead of offering an edit affordance.
+    app.put("/api/users/:username/roles", need("users", "delete"), audit("users", "write"), async (req, res) => {
+        const username = String(req.params.username);
+        const path = process.env.OMCP_USERS_FILE;
+        if (!path) {
+            res.status(409).json({ error: "OMCP_USERS_FILE is not configured — basic-mode user roles can't be edited via the API." });
+            return;
+        }
+        const body = req.body;
+        if (!body || !Array.isArray(body.roles) || !body.roles.every((r) => typeof r === "string")) {
+            res.status(400).json({ error: "body must include { roles: string[] }" });
+            return;
+        }
+        const requestedRoles = body.roles;
+        // Reject role names not in the active policy engine's catalogue —
+        // assigning a user a role that grants nothing is almost always a
+        // typo, not intent. Same defence-in-depth posture as the products
+        // typo guard (PR #343).
+        const knownRoles = new Set(policyEngine.roles());
+        const unknown = requestedRoles.filter((r) => !knownRoles.has(r));
+        if (unknown.length > 0) {
+            res.status(422).json({
+                error: `unknown role name(s) for user '${username}': ${unknown.join(", ")}`,
+                code: "OMCP_USER_UNKNOWN_ROLE",
+                unknown,
+                available: Array.from(knownRoles),
+            });
+            return;
+        }
+        const file = await readUsersFile(path);
+        if (!file) {
+            res.status(404).json({ error: `users file at ${path} is unreadable or empty` });
+            return;
+        }
+        const idx = file.users.findIndex((u) => u.username === username);
+        if (idx < 0) {
+            res.status(404).json({ error: `user '${username}' not found` });
+            return;
+        }
+        file.users[idx].roles = requestedRoles;
+        try {
+            await writeUsersFile(path, file);
+        }
+        catch (e) {
+            res.status(500).json({ error: `failed to persist users file: ${e.message}` });
+            return;
+        }
+        // Refresh the in-memory store so the next login picks up the new
+        // role set without a server restart. maybeReloadUsers stat()s the
+        // file's mtime, which we just bumped via the atomic rename.
+        await maybeReloadUsers();
+        res.json({ ok: true, username, roles: requestedRoles });
+    });
+    // Upsert a role in the file-backed RBAC policy. File engine only:
+    // built-in defaults are immutable in source; OPA is the Rego
+    // source of truth. The UI hides the affordance under non-file
+    // engines via the [data-engine-required="file"] CSS gate; the
+    // endpoint enforces the rule too for defence-in-depth.
+    app.put("/api/policy/roles/:name", need("users", "delete"), audit("users", "write"), async (req, res) => {
+        const name = String(req.params.name);
+        // Reject names with shell-unfriendly characters early so the
+        // YAML round-trip can't accidentally produce an exotic key.
+        if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(name)) {
+            res.status(400).json({ error: `role name '${name}' must match [A-Za-z0-9][A-Za-z0-9._-]{0,63}` });
+            return;
+        }
+        const policyFile = process.env.OMCP_RBAC_POLICY_FILE?.trim();
+        if (!policyEngine.kind().startsWith("file:")) {
+            // Built-in (immutable source) or OPA (Rego is the source of
+            // truth) — role authoring isn't available. Return distinct
+            // error codes so the UI can show the right hint without
+            // string-matching the message.
+            const code = policyEngine.kind() === "builtin"
+                ? "OMCP_POLICY_ENGINE_BUILTIN"
+                : policyEngine.kind().startsWith("opa:")
+                    ? "OMCP_POLICY_ENGINE_OPA"
+                    : "OMCP_POLICY_ENGINE_NOT_FILE";
+            res.status(409).json({
+                error: `role authoring requires the file engine — current is '${policyEngine.kind()}'`,
+                code,
+            });
+            return;
+        }
+        if (!policyFile) {
+            res.status(409).json({
+                error: "OMCP_RBAC_POLICY_FILE is not configured — role authoring writes through that file.",
+                code: "OMCP_POLICY_FILE_NOT_SET",
+            });
+            return;
+        }
+        const body = req.body;
+        if (!body || !Array.isArray(body.permissions)) {
+            res.status(400).json({ error: "body must include { permissions: [{resource, action}] }" });
+            return;
+        }
+        const cleanPerms = [];
+        for (let i = 0; i < body.permissions.length; i++) {
+            const p = body.permissions[i];
+            if (!p || typeof p !== "object" || typeof p.resource !== "string" || typeof p.action !== "string") {
+                res.status(400).json({ error: `body.permissions[${i}] must be { resource: string, action: string }` });
+                return;
+            }
+            if (!VALID_RESOURCES.has(p.resource)) {
+                res.status(422).json({
+                    error: `unknown resource '${p.resource}'`,
+                    code: "OMCP_POLICY_UNKNOWN_RESOURCE",
+                    unknown: p.resource,
+                    available: [...VALID_RESOURCES],
+                });
+                return;
+            }
+            if (!VALID_ACTIONS.has(p.action)) {
+                res.status(422).json({
+                    error: `unknown action '${p.action}'`,
+                    code: "OMCP_POLICY_UNKNOWN_ACTION",
+                    unknown: p.action,
+                    available: [...VALID_ACTIONS],
+                });
+                return;
+            }
+            cleanPerms.push({ resource: p.resource, action: p.action });
+        }
+        // De-duplicate exact (resource, action) pairs so the file
+        // doesn't accumulate redundant entries via re-saves.
+        const seen = new Set();
+        const dedup = [];
+        for (const p of cleanPerms) {
+            const k = p.resource + ":" + p.action;
+            if (seen.has(k))
+                continue;
+            seen.add(k);
+            dedup.push(p);
+        }
+        // Snapshot the existing map (via raw()) and overlay the upsert.
+        // BuiltinPolicyEngine is the only kind that reaches here per the
+        // checks above.
+        const current = {};
+        if (policyEngine instanceof BuiltinPolicyEngine) {
+            for (const [r, ps] of Object.entries(policyEngine.raw())) {
+                current[r] = ps.slice();
+            }
+        }
+        current[name] = dedup;
+        try {
+            await writePolicyFile(policyFile, current);
+        }
+        catch (e) {
+            if (e instanceof PolicyLoadError) {
+                res.status(422).json({ error: e.message });
+                return;
+            }
+            res.status(500).json({ error: `failed to persist policy: ${e.message}` });
+            return;
+        }
+        // Hot-swap the in-memory engine so the next gate evaluation
+        // picks up the new role without a restart. `replace()` mutates
+        // in-place, so existing middleware closures over `policyEngine`
+        // see the new map immediately.
+        if (policyEngine instanceof BuiltinPolicyEngine) {
+            const fresh = loadPolicyFromFile(policyFile);
+            if (fresh instanceof BuiltinPolicyEngine) {
+                policyEngine.replace(fresh.raw());
+            }
+        }
+        res.json({ ok: true, name, permissions: dedup });
+    });
     // --- /api/audit — management-plane audit feed -------------------------
     // Read-only, gated by the "audit:read" permission so only viewers /
     // operators / admins (basically anyone authenticated in the default
@@ -987,6 +1665,46 @@ async function main() {
             scopedTo: tenantFilter || (isAdmin ? null : callerTenant),
         });
     });
+    // --- /api/audit/dlq — webhook-sink dead-letter queue surface (P9) ---
+    // When the audit webhook is configured AND the receiver exhausted
+    // its retry budget, entries land in the DLQ file. This endpoint
+    // surfaces the count + the last N entries so operators can decide
+    // whether to replay manually. Also refreshes the
+    // `obsmcp_audit_webhook_dlq_depth` gauge so the /metrics scrape
+    // alongside it stays accurate.
+    app.get("/api/audit/dlq", need("audit", "read"), async (_req, res) => {
+        const dlqPath = process.env.OMCP_AUDIT_WEBHOOK_DLQ;
+        if (!dlqPath) {
+            auditDlqDepth.set(0);
+            res.json({ enabled: false, path: null, depth: 0, entries: [] });
+            return;
+        }
+        try {
+            const fs = await import("node:fs/promises");
+            const raw = await fs.readFile(dlqPath, "utf8");
+            const lines = raw.split("\n").filter((l) => l.trim());
+            auditDlqDepth.set(lines.length);
+            const tail = lines.slice(-50).map((l) => {
+                try {
+                    return JSON.parse(l);
+                }
+                catch {
+                    return { _raw: l, _parseError: true };
+                }
+            });
+            res.json({ enabled: true, path: dlqPath, depth: lines.length, entries: tail });
+        }
+        catch (err) {
+            const code = err.code;
+            if (code === "ENOENT") {
+                auditDlqDepth.set(0);
+                res.json({ enabled: true, path: dlqPath, depth: 0, entries: [] });
+                return;
+            }
+            console.warn("[/api/audit/dlq] read failed:", err);
+            res.status(500).json({ error: err?.message || "DLQ read failed" });
+        }
+    });
     // --- /api/usage — per-identity MCP rate-limit snapshot -----------------
     // Read-only view of the IdentityRateLimiter's bucket state. Gated by
     // need("audit","read") — the same role set that already sees the
@@ -1137,6 +1855,133 @@ async function main() {
         registerOidcRoutes(app, { sessionCfg, oidc: oidcRuntime });
         console.log("[auth] OIDC endpoints registered: /api/auth/oidc/{login,callback,logout}");
     }
+    // Phase F21 / Q6: SCIM 2.0 — opt-in. OMCP_SCIM_TOKEN gates access.
+    // The store backend is chosen by createScimStore from
+    // OMCP_SCIM_BACKEND (file | redis). file (default) → OMCP_SCIM_STORE
+    // on-disk JSON (mode 0600, atomic). redis → a shared snapshot so
+    // multi-replica deployments stay coherent (Q6); the redis client is
+    // built from OMCP_SCIM_REDIS_URL here, mirroring the session store.
+    const scimToken = process.env.OMCP_SCIM_TOKEN?.trim();
+    if (scimToken) {
+        try {
+            const scimBackend = (process.env.OMCP_SCIM_BACKEND?.trim() || "file");
+            let scimRedis;
+            if (scimBackend === "redis") {
+                const redisUrl = process.env.OMCP_SCIM_REDIS_URL?.trim();
+                if (!redisUrl)
+                    throw new Error("OMCP_SCIM_BACKEND=redis requires OMCP_SCIM_REDIS_URL");
+                const { createClient } = await import("redis");
+                const client = createClient({ url: redisUrl });
+                client.on("error", (err) => console.warn("[scim] redis client error: %s", err instanceof Error ? err.message : String(err)));
+                await client.connect();
+                scimRedis = client;
+            }
+            const scimStore = await createScimStore({
+                backend: scimBackend,
+                path: process.env.OMCP_SCIM_STORE?.trim() || "/tmp/scim.json",
+                redis: scimRedis,
+                redisKey: process.env.OMCP_SCIM_REDIS_KEY?.trim(),
+            });
+            registerScimRoutes(app, {
+                store: scimStore,
+                bearerToken: scimToken,
+                audit: (ev) => void mgmtAudit.record({
+                    actor: { sub: `scim:${ev.actor}` },
+                    tenant: "default",
+                    resource: "users",
+                    action: ev.action.includes("delete") ? "delete" : "write",
+                    method: "SCIM",
+                    path: `/scim/v2/${ev.action}`,
+                    status: ev.status,
+                    target: ev.target,
+                }).catch(() => undefined),
+            });
+            console.log("[scim] /scim/v2/* registered (backend: %s)", scimBackend);
+        }
+        catch (err) {
+            console.warn("[scim] enable failed (routes not mounted): %s", err instanceof Error ? err.message : String(err));
+        }
+    }
+    // Phase P6: Postmortems persistence. /api/postmortems lets the
+    // UI list / open / regenerate / delete previously-generated
+    // reports. Opt-in via OMCP_POSTMORTEMS_FILE (default
+    // /tmp/postmortems.jsonl). When the env is left at its default
+    // the demo still works — operators who want survival across
+    // restarts mount a PVC at the same path and set the env to it.
+    const postmortemStore = new PostmortemStore(process.env.OMCP_POSTMORTEMS_FILE?.trim() || "/tmp/postmortems.jsonl");
+    await postmortemStore.load();
+    // GET /api/postmortems — list (newest-first), tenant-scoped.
+    app.get("/api/postmortems", need("services", "read"), async (req, res) => {
+        const sess = req.session;
+        const tenant = sess?.tenant || "default";
+        const entries = postmortemStore.list(tenant);
+        res.json({
+            total: entries.length,
+            entries: entries.map((e) => ({
+                id: e.id,
+                ts: e.ts,
+                createdBy: e.createdBy,
+                service: e.report.service,
+                window: e.report.window,
+                synopsis: e.report.synopsis,
+            })),
+        });
+    });
+    // GET /api/postmortems/:id — full report (markdown + sections).
+    app.get("/api/postmortems/:id", need("services", "read"), async (req, res) => {
+        const sess = req.session;
+        const tenant = sess?.tenant || "default";
+        const id = String(req.params.id ?? "");
+        const entry = postmortemStore.get(id, tenant);
+        if (!entry) {
+            res.status(404).json({ error: `Postmortem ${id} not found` });
+            return;
+        }
+        res.json(entry);
+    });
+    // POST /api/postmortems — regenerate via the tool handler +
+    // persist. Body: { service, duration?, format? }. Returns the
+    // stored entry with its id.
+    app.post("/api/postmortems", need("services", "write"), async (req, res) => {
+        const body = (req.body ?? {});
+        if (!body.service || typeof body.service !== "string") {
+            res.status(400).json({ error: "service is required" });
+            return;
+        }
+        const sess = req.session;
+        const tenant = sess?.tenant || "default";
+        const createdBy = sess?.sub || sess?.name || "unknown";
+        try {
+            // Force JSON so we get the structured report shape back from
+            // the tool, not just the markdown body. We persist the full
+            // structured report; the markdown lives inside `report.markdown`.
+            const ctx = { ...defaultContext(), tenant, principalId: createdBy };
+            const result = await generatePostmortemHandler(registry, { service: body.service, duration: body.duration, format: "json" }, ctx);
+            const text = result?.content?.[0]?.text;
+            if (!text) {
+                res.status(500).json({ error: "generate_postmortem returned no content" });
+                return;
+            }
+            const report = JSON.parse(text);
+            const stored = await postmortemStore.append({ report, createdBy, tenant });
+            res.status(201).json(stored);
+        }
+        catch (e) {
+            console.warn(`[postmortems] regen failed:`, e);
+            res.status(500).json({ error: e?.message || "internal error" });
+        }
+    });
+    // DELETE /api/postmortems/:id — admin-gated.
+    app.delete("/api/postmortems/:id", need("services", "delete"), async (req, res) => {
+        const sess = req.session;
+        const tenant = sess?.tenant || "default";
+        const ok = await postmortemStore.delete(String(req.params.id ?? ""), tenant);
+        if (!ok) {
+            res.status(404).json({ error: `Postmortem ${req.params.id} not found` });
+            return;
+        }
+        res.status(204).end();
+    });
     // Connectors currently loaded into this server (builtin + filesystem
     // plugins), with manifest metadata — drives the UI "Connectors" page.
     app.get("/api/connectors", (_req, res) => {
@@ -1328,9 +2173,13 @@ async function main() {
                 rmSync(work, { recursive: true, force: true });
         }
     });
-    // Add a new source
+    // Add a new source — tenant-aware. Non-admins can only create
+    // sources in their own tenant; admins may set any tenant or leave
+    // unset (global). Untagged inputs default to undefined (global) for
+    // admins and to the caller's own tenant for non-admins, so a
+    // tenant-bound user can't accidentally pollute the global pool.
     app.post("/api/sources", installRateLimit, need("sources", "write"), audit("sources", "write"), async (req, res) => {
-        const { name, type, url, enabled, auth, tls } = req.body;
+        const { name, type, url, enabled, auth, tls, tenant: bodyTenant } = req.body;
         if (!name || !type || !url) {
             res.status(400).json({ error: "name, type, and url are required" });
             return;
@@ -1340,22 +2189,40 @@ async function main() {
             res.status(400).json({ error: urlErr });
             return;
         }
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        const resolvedTenant = isAdmin
+            ? (typeof bodyTenant === "string" && bodyTenant ? bodyTenant : undefined)
+            : (typeof bodyTenant === "string" && bodyTenant && bodyTenant !== callerTenant
+                ? "__deny__"
+                : callerTenant);
+        if (resolvedTenant === "__deny__") {
+            res.status(403).json({ error: "cannot create source in another tenant" });
+            return;
+        }
         const existing = registry.getSourceConfigs().find((s) => s.name === name);
         if (existing) {
             res.status(409).json({ error: `Source "${name}" already exists` });
             return;
         }
-        const source = { name, type, url, enabled: enabled !== false, auth, tls };
+        const source = { name, type, url, enabled: enabled !== false, auth, tls, tenant: resolvedTenant };
         await registry.addSource(source);
         saveConfig(config = { ...config, sources: registry.getSourceConfigs() });
         res.status(201).json({ ok: true, source });
     });
-    // Update an existing source
+    // Update an existing source — tenant-aware. Non-admins editing a
+    // cross-tenant source get the same 404 they'd get for "no such
+    // source" (no existence leak). Admins may move a source between
+    // tenants by setting body.tenant; non-admins cannot.
     app.put("/api/sources/:name", need("sources", "write"), audit("sources", "write"), async (req, res) => {
         const oldName = String(req.params.name);
-        const { name, type, url, enabled, auth, tls } = req.body;
+        const { name, type, url, enabled, auth, tls, tenant: bodyTenant } = req.body;
         const existing = registry.getSourceConfigs().find((s) => s.name === oldName);
-        if (!existing) {
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        if (!existing || (!isAdmin && existing.tenant && existing.tenant !== callerTenant)) {
             res.status(404).json({ error: `Source "${oldName}" not found` });
             return;
         }
@@ -1367,6 +2234,19 @@ async function main() {
                 return;
             }
         }
+        let nextTenant = existing.tenant;
+        if (bodyTenant !== undefined) {
+            if (!isAdmin) {
+                // Non-admin attempting tenant reassignment — disallow.
+                if (bodyTenant !== existing.tenant) {
+                    res.status(403).json({ error: "cannot change source tenant" });
+                    return;
+                }
+            }
+            else {
+                nextTenant = typeof bodyTenant === "string" && bodyTenant ? bodyTenant : undefined;
+            }
+        }
         const source = {
             name: name || oldName,
             type: type || existing.type,
@@ -1374,16 +2254,20 @@ async function main() {
             enabled: enabled !== undefined ? enabled : existing.enabled,
             auth: auth !== undefined ? auth : existing.auth,
             tls: tls !== undefined ? tls : existing.tls,
+            tenant: nextTenant,
         };
         await registry.updateSource(oldName, source);
         saveConfig(config = { ...config, sources: registry.getSourceConfigs() });
         res.json({ ok: true, source });
     });
-    // Delete a source
+    // Delete a source — same cross-tenant 404 posture.
     app.delete("/api/sources/:name", need("sources", "delete"), audit("sources", "delete"), async (req, res) => {
         const name = String(req.params.name);
         const existing = registry.getSourceConfigs().find((s) => s.name === name);
-        if (!existing) {
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        if (!existing || (!isAdmin && existing.tenant && existing.tenant !== callerTenant)) {
             res.status(404).json({ error: `Source "${name}" not found` });
             return;
         }
@@ -1417,7 +2301,10 @@ async function main() {
     app.patch("/api/sources/:name/toggle", need("sources", "write"), audit("sources", "write"), async (req, res) => {
         const name = String(req.params.name);
         const existing = registry.getSourceConfigs().find((s) => s.name === name);
-        if (!existing) {
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        if (!existing || (!isAdmin && existing.tenant && existing.tenant !== callerTenant)) {
             res.status(404).json({ error: `Source "${name}" not found` });
             return;
         }
@@ -1440,7 +2327,10 @@ async function main() {
         try {
             const sess = req.session;
             const callerTenant = sess?.tenant || "default";
-            const result = await listServicesHandler(registry, {}, defaultContext());
+            // sessionContext threads the caller's tenant into the handler so
+            // PR #331's per-tenant connector scoping fires for the dashboard
+            // surface too (was previously bypassed with defaultContext()).
+            const result = await listServicesHandler(registry, {}, sessionContext(sess));
             const parsed = parseToolResult(result);
             // Tenant-scope catalog enrichment so a viewer in tenant A
             // doesn't accidentally see acme's owner/SLO metadata on a
@@ -1483,7 +2373,9 @@ async function main() {
     // Same scoping / staging-visibility pattern as /api/catalog. Non-admins
     // see only their own tenant's PUBLISHED products; admins see all
     // tenants by default + staging.
-    app.get("/api/products", need("products", "read"), (req, res) => {
+    app.get("/api/products", need("products", "read"), async (req, res) => {
+        // Pick up out-of-band edits before serving — see ProductsStore docs.
+        await products.maybeReload();
         const sess = req.session;
         const isAdmin = hasPermission(sess?.roles, "users", "delete");
         const callerTenant = sess?.tenant || "default";
@@ -1497,6 +2389,67 @@ async function main() {
             includesStaging: includeStaging,
         });
     });
+    // Create a new product (REST convention: POST = create, 409 on
+    // conflict). Same tenancy + typo-guard posture as PUT. The PUT
+    // upsert path remains for the existing UI; new integrations that
+    // want strict create-vs-update semantics use POST.
+    app.post("/api/products", need("products", "write"), audit("products", "write"), async (req, res) => {
+        await products.maybeReload();
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        const body = req.body;
+        if (!body || typeof body !== "object" || Array.isArray(body)) {
+            res.status(400).json({ error: "body must be a product object" });
+            return;
+        }
+        if (typeof body.id !== "string" || !body.id) {
+            res.status(400).json({ error: "body.id is required" });
+            return;
+        }
+        let validated;
+        try {
+            validated = validateProduct(body, `POST /api/products`);
+        }
+        catch (e) {
+            if (e instanceof ProductsLoadError) {
+                res.status(400).json({ error: e.message });
+                return;
+            }
+            throw e;
+        }
+        if (validated.tools && validated.tools.length > 0) {
+            const unknown = unknownToolNames(validated.tools);
+            if (unknown.length > 0) {
+                res.status(422).json({
+                    error: `unknown tool name(s) in product '${validated.id}': ${unknown.join(", ")}`,
+                    code: "OMCP_PRODUCT_UNKNOWN_TOOL",
+                    unknown,
+                    available: REGISTERED_TOOL_NAMES,
+                });
+                return;
+            }
+        }
+        if (!isAdmin && (validated.tenant || "default") !== callerTenant) {
+            res.status(403).json({ error: "cannot create product in another tenant" });
+            return;
+        }
+        if (products.get(validated.id)) {
+            res.status(409).json({ error: `product '${validated.id}' already exists; use PUT to update` });
+            return;
+        }
+        const next = products.upsert(validated);
+        if (process.env.OMCP_PRODUCTS_FILE) {
+            try {
+                await writeProductsFile(process.env.OMCP_PRODUCTS_FILE, next);
+                await products.pinMtimeAfterWrite();
+            }
+            catch (e) {
+                console.warn(`[products] POST ${validated.id}: failed to persist to ${process.env.OMCP_PRODUCTS_FILE}: ${e.message} — in-memory state is still updated`);
+            }
+        }
+        res.status(201).json({ product: validated, persisted: !!process.env.OMCP_PRODUCTS_FILE });
+    });
     // Upsert a product. Body is the same shape as a single entry
     // in OMCP_PRODUCTS_FILE. The URL-path id must match the body id
     // (defence-in-depth: the gate keys on body, the path keys the
@@ -1504,6 +2457,9 @@ async function main() {
     // updated catalogue back to disk so the change survives a
     // restart; without the file, the upsert is in-memory only.
     app.put("/api/products/:id", need("products", "write"), audit("products", "write"), async (req, res) => {
+        // Hot-reload before mutating so a concurrent on-disk edit isn't
+        // silently clobbered by our in-memory snapshot.
+        await products.maybeReload();
         const id = String(req.params.id);
         const sess = req.session;
         const isAdmin = hasPermission(sess?.roles, "users", "delete");
@@ -1532,6 +2488,23 @@ async function main() {
             }
             throw e;
         }
+        // Typo guard: a Product whose `tools` allow-list names tools
+        // that don't actually register would bind a credential to an
+        // empty /mcp tool surface (silent dead session). Reject with
+        // 422 + a hint of valid tool names so the operator can see the
+        // intended typo immediately.
+        if (validated.tools && validated.tools.length > 0) {
+            const unknown = unknownToolNames(validated.tools);
+            if (unknown.length > 0) {
+                res.status(422).json({
+                    error: `unknown tool name(s) in product '${id}': ${unknown.join(", ")}`,
+                    code: "OMCP_PRODUCT_UNKNOWN_TOOL",
+                    unknown,
+                    available: REGISTERED_TOOL_NAMES,
+                });
+                return;
+            }
+        }
         // Tenant gate: non-admins can only write into their own tenant.
         if (!isAdmin && (validated.tenant || "default") !== callerTenant) {
             res.status(403).json({ error: "cannot write product into another tenant" });
@@ -1549,6 +2522,10 @@ async function main() {
         if (process.env.OMCP_PRODUCTS_FILE) {
             try {
                 await writeProductsFile(process.env.OMCP_PRODUCTS_FILE, next);
+                // Advance our mtime cursor past this write so the next
+                // maybeReload() doesn't treat our own change as an external
+                // edit and re-read what we just persisted.
+                await products.pinMtimeAfterWrite();
             }
             catch (e) {
                 console.warn(`[products] PUT ${id}: failed to persist to ${process.env.OMCP_PRODUCTS_FILE}: ${e.message} — in-memory state is still updated`);
@@ -1557,6 +2534,7 @@ async function main() {
         res.json({ product: validated, persisted: !!process.env.OMCP_PRODUCTS_FILE });
     });
     app.delete("/api/products/:id", need("products", "delete"), audit("products", "delete"), async (req, res) => {
+        await products.maybeReload();
         const id = String(req.params.id);
         const sess = req.session;
         const isAdmin = hasPermission(sess?.roles, "users", "delete");
@@ -1574,6 +2552,7 @@ async function main() {
         if (process.env.OMCP_PRODUCTS_FILE) {
             try {
                 await writeProductsFile(process.env.OMCP_PRODUCTS_FILE, next);
+                await products.pinMtimeAfterWrite();
             }
             catch (e) {
                 console.warn(`[products] DELETE ${id}: failed to persist to ${process.env.OMCP_PRODUCTS_FILE}: ${e.message} — in-memory state is still updated`);
@@ -1584,7 +2563,8 @@ async function main() {
     // Single product by id. Non-admins get a 404 (not 403) on a
     // cross-tenant probe so the existence of the product isn't leaked
     // — same posture as the rest of the tenancy layer.
-    app.get("/api/products/:id", need("products", "read"), (req, res) => {
+    app.get("/api/products/:id", need("products", "read"), async (req, res) => {
+        await products.maybeReload();
         const sess = req.session;
         const isAdmin = hasPermission(sess?.roles, "users", "delete");
         const callerTenant = sess?.tenant || "default";
@@ -1603,12 +2583,48 @@ async function main() {
         }
         res.json(p);
     });
+    // Agent preview — what would the /mcp tools/list response look
+    // like for a credential bound to this product? Same RBAC + tenant
+    // gate as the singular GET above. The body mirrors the actual
+    // tools/list shape (name + description + category), filtered the
+    // same way the /mcp transport filters it via allowsTool +
+    // registerTool — so the UI's Review pane shows the exact set the
+    // agent will see, not an approximation. Branding metadata travels
+    // alongside so the preview can render with the product's identity.
+    app.get("/api/products/:id/preview", need("products", "read"), async (req, res) => {
+        await products.maybeReload();
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        const tenantFilter = isAdmin ? undefined : callerTenant;
+        const id = String(req.params.id);
+        const p = products.get(id, tenantFilter);
+        if (!p) {
+            res.status(404).json({ error: "not found" });
+            return;
+        }
+        if (!isAdmin && p.status === "staging") {
+            res.status(404).json({ error: "not found" });
+            return;
+        }
+        const allowList = p.tools && p.tools.length > 0 ? p.tools : undefined;
+        const filteredTools = REGISTERED_TOOLS.filter((t) => allowsTool(allowList, t.name));
+        res.json({
+            product: { id: p.id, name: p.name, version: p.version, branding: p.branding, tenant: p.tenant, status: p.status },
+            // unrestricted = true when the product has no tools allow-list,
+            // i.e. the bound agent sees every registered tool. UI uses this
+            // to render a distinct "no filter" preview banner.
+            unrestricted: !allowList,
+            tools: filteredTools,
+        });
+    });
     // Health endpoint for UI dashboard
     app.get("/api/health/:service", async (req, res) => {
         try {
-            const callerTenant = req.session?.tenant || "default";
+            const sess = req.session;
+            const callerTenant = sess?.tenant || "default";
             const service = String(req.params.service);
-            const result = await getServiceHealthHandler(registry, { service }, defaultContext());
+            const result = await getServiceHealthHandler(registry, { service }, sessionContext(sess));
             const parsed = parseToolResult(result);
             const entry = catalog.get(service, callerTenant);
             if (entry && parsed && typeof parsed === "object")
@@ -1622,14 +2638,16 @@ async function main() {
     // Health for all services
     app.get("/api/health", async (req, res) => {
         try {
-            const callerTenant = req.session?.tenant || "default";
-            const servicesResult = await listServicesHandler(registry, {}, defaultContext());
+            const sess = req.session;
+            const callerTenant = sess?.tenant || "default";
+            const ctx = sessionContext(sess);
+            const servicesResult = await listServicesHandler(registry, {}, ctx);
             const parsed = parseToolResult(servicesResult);
             const services = parsed?.services || [];
             const health = {};
             for (const svc of services) {
                 try {
-                    const result = await getServiceHealthHandler(registry, { service: svc.name }, defaultContext());
+                    const result = await getServiceHealthHandler(registry, { service: svc.name }, ctx);
                     const h = parseToolResult(result);
                     // Same tenant scoping as /api/services to avoid the
                     // dashboard cross-tenant catalog leak the reviewer
@@ -1653,12 +2671,18 @@ async function main() {
     // Returns the union of topology snapshots across all topology-capable
     // connectors (today only "kubernetes"). One JSON document so the UI can
     // render summary + grouped views without N round-trips.
-    app.get("/api/topology", async (_req, res) => {
+    app.get("/api/topology", async (req, res) => {
         try {
+            const sess = req.session;
+            const callerTenant = sess?.tenant || "default";
             const sources = [];
             const allResources = [];
             const allEdges = [];
-            for (const c of registry.getAll()) {
+            // Tenant-scoped: non-anonymous callers only see topology from
+            // connectors their tenant can reach. Anonymous mode keeps the
+            // global view (single-tenant default).
+            const connectors = sess ? registry.getByTenant(callerTenant) : registry.getAll();
+            for (const c of connectors) {
                 if (!isTopologyProvider(c))
                     continue;
                 const snap = await c.getTopologySnapshot();
@@ -1710,9 +2734,19 @@ async function main() {
     // --- Per-Source Metrics API ---
     // Get metrics for a source (active metrics or defaults)
     app.get("/api/sources/:name/metrics", (req, res) => {
-        const connector = registry.getByName(String(req.params.name));
+        const name = String(req.params.name);
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        // Tenant-aware: getByNameForTenant returns undefined for both
+        // "doesn't exist" and "cross-tenant" — same no-leak posture as
+        // /api/sources GET/PUT/DELETE. Anonymous / admin keep the
+        // single-tenant behaviour by falling back to getByName.
+        const connector = (sess && !isAdmin)
+            ? registry.getByNameForTenant(name, callerTenant)
+            : registry.getByName(name);
         if (!connector) {
-            res.status(404).json({ error: `Source "${String(req.params.name)}" not found` });
+            res.status(404).json({ error: `Source "${name}" not found` });
             return;
         }
         res.json({
@@ -1720,11 +2754,15 @@ async function main() {
             defaults: connector.getDefaultMetrics(),
         });
     });
-    // Update metrics for a source
+    // Update metrics for a source — tenant-aware mutation.
     app.put("/api/sources/:name/metrics", need("sources", "write"), audit("sources", "write"), async (req, res) => {
         const name = String(req.params.name);
         const sourceIdx = config.sources.findIndex((s) => s.name === name);
-        if (sourceIdx === -1) {
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        const src = sourceIdx >= 0 ? config.sources[sourceIdx] : undefined;
+        if (!src || (!isAdmin && src.tenant && src.tenant !== callerTenant)) {
             res.status(404).json({ error: `Source "${name}" not found` });
             return;
         }
@@ -1734,11 +2772,15 @@ async function main() {
         saveConfig(config);
         res.json({ ok: true });
     });
-    // Reset a source's metrics to connector defaults
+    // Reset a source's metrics to connector defaults — tenant-aware.
     app.delete("/api/sources/:name/metrics", need("sources", "write"), audit("sources", "write"), async (req, res) => {
         const name = String(req.params.name);
         const sourceIdx = config.sources.findIndex((s) => s.name === name);
-        if (sourceIdx === -1) {
+        const sess = req.session;
+        const isAdmin = hasPermission(sess?.roles, "users", "delete");
+        const callerTenant = sess?.tenant || "default";
+        const src = sourceIdx >= 0 ? config.sources[sourceIdx] : undefined;
+        if (!src || (!isAdmin && src.tenant && src.tenant !== callerTenant)) {
             res.status(404).json({ error: `Source "${name}" not found` });
             return;
         }
@@ -1749,7 +2791,7 @@ async function main() {
     });
     // Stdio transport: one server over stdin/stdout, no HTTP listener.
     if (STDIO) {
-        const server = createMcpServer(defaultContext());
+        const { mcpServer: server } = createMcpServer(defaultContext());
         await server.connect(new StdioServerTransport());
         console.error(`observability-mcp running on stdio transport · connectors: ${registry
             .getAll()
@@ -1760,6 +2802,12 @@ async function main() {
     // MCP Streamable HTTP transport — stateful sessions
     const transports = new Map();
     const sessionLastActive = new Map();
+    // Phase F9: per-session tag identifying the virtual-server slug a
+    // session was issued under (or undefined for the root /mcp surface).
+    // Used to prevent a session minted on /mcp/v/foo from being probed
+    // via /mcp/v/bar — the GET/DELETE handlers refuse the cross-product
+    // lookup.
+    const sessionProduct = new Map();
     const SESSION_TTL_MS = 30 * 60 * 1000; // 30 min idle timeout
     // Clean up idle sessions every 5 minutes
     setInterval(() => {
@@ -1768,6 +2816,7 @@ async function main() {
             if (now - lastActive > SESSION_TTL_MS) {
                 transports.delete(sid);
                 sessionLastActive.delete(sid);
+                sessionProduct.delete(sid);
                 console.log(`Session ${sid} expired (idle)`);
             }
         }
@@ -1781,8 +2830,22 @@ async function main() {
     // 429 with a Retry-After. Anonymous /mcp traffic (no OMCP_API_KEYS
     // configured) bypasses this — the global express-rate-limit IP gate
     // still applies. Override via OMCP_TOOL_RATE_PER_MIN.
+    // Per-credential cap overrides: OMCP_KEY_RATE_PER_MIN="agent=600;ci=240"
+    // wins over the global OMCP_TOOL_RATE_PER_MIN for the named credentials.
+    // The bucket identity is "<tenant> <credName>"; the override map keys on
+    // credName, so the lookup pulls the cred-name back out of the composite.
+    const keyRateLimits = parseKeyRateLimits(process.env.OMCP_KEY_RATE_PER_MIN);
     const toolRateLimiter = new IdentityRateLimiter({
         limit: resolveToolRatePerMin(process.env.OMCP_TOOL_RATE_PER_MIN),
+        limitFor: keyRateLimits.size === 0 ? undefined : (identity) => {
+            // Composite identity is "<tenant> <credName>" — split on the
+            // single space that gateCtx put there (NUL would be safer but
+            // would break existing /api/usage actor labels; cred names are
+            // operator-set and don't contain spaces in practice).
+            const sp = identity.indexOf(" ");
+            const credName = sp >= 0 ? identity.slice(sp + 1) : identity;
+            return keyRateLimits.get(credName);
+        },
     });
     // Per-identity tracker key. Composes tenant + principalId so two
     // credentials of the same name in different tenants don't share
@@ -1822,7 +2885,7 @@ async function main() {
     }
     // Bearer/X-API-Key on every /mcp request; resolve the principal + its
     // coarse source allow-list into the RequestContext.
-    function gateCtx(req, res) {
+    async function gateCtx(req, res) {
         if (!credentialsConfigured())
             return defaultContext();
         const cred = resolveToken(extractToken(req.headers), loadCredentials());
@@ -1853,13 +2916,33 @@ async function main() {
             });
             return null;
         }
+        // Resolve the credential's bound Product (OMCP_KEY_PRODUCTS) into
+        // a concrete tools allow-list. Cross-tenant Products are invisible
+        // — products.get() returns undefined when the productId belongs to
+        // another tenant, mirroring the rest of the tenancy layer. A bound
+        // Product whose own `tools` field is absent / empty leaves the
+        // allow-list undefined (== unrestricted), matching the YAML
+        // loader's "no tools key = no restriction" semantics.
+        let allowedTools;
+        if (cred.productId) {
+            // Pick up out-of-band edits to OMCP_PRODUCTS_FILE before each
+            // /mcp request — cheap (one stat), keeps the binding live.
+            // Best-effort: if the catalogue reload fails we keep the prior
+            // good state (the store handles that internally) rather than
+            // failing the request.
+            await products.maybeReload().catch(() => undefined);
+            const p = products.get(cred.productId, credTenant);
+            if (p && p.tools && p.tools.length > 0)
+                allowedTools = p.tools.slice();
+        }
         return principalContext(cred.name, cred.allowedSources, {
             allowBypassRedaction: cred.bypassRedaction,
             tenant: cred.tenant,
+            allowedTools,
         });
     }
     app.post("/mcp", async (req, res) => {
-        const ctx = gateCtx(req, res);
+        const ctx = await gateCtx(req, res);
         if (!ctx)
             return;
         const sessionId = req.headers["mcp-session-id"];
@@ -1881,7 +2964,7 @@ async function main() {
                 }
                 mcpActiveSessions.set(transports.size);
             };
-            const sessionMcpServer = createMcpServer(ctx);
+            const { mcpServer: sessionMcpServer } = createMcpServer(ctx);
             await sessionMcpServer.connect(transport);
         }
         await transport.handleRequest(req, res, req.body);
@@ -1895,7 +2978,7 @@ async function main() {
         mcpActiveSessions.set(transports.size);
     });
     app.get("/mcp", async (req, res) => {
-        if (!gateCtx(req, res))
+        if (!(await gateCtx(req, res)))
             return;
         const sessionId = req.headers["mcp-session-id"];
         const transport = transports.get(sessionId);
@@ -1906,7 +2989,7 @@ async function main() {
         await transport.handleRequest(req, res);
     });
     app.delete("/mcp", async (req, res) => {
-        if (!gateCtx(req, res))
+        if (!(await gateCtx(req, res)))
             return;
         const sessionId = req.headers["mcp-session-id"];
         const transport = transports.get(sessionId);
@@ -1914,18 +2997,244 @@ async function main() {
             await transport.handleRequest(req, res);
             transports.delete(sessionId);
             sessionLastActive.delete(sessionId);
+            sessionProduct.delete(sessionId);
+        }
+        else {
+            res.status(400).json({ error: "No active session" });
+        }
+    });
+    // Phase F9: virtual servers — every Product gets its own MCP
+    // endpoint at /mcp/v/<slug> that exposes only the tools bound to
+    // that Product, with the caller's existing tenant + RBAC scoping
+    // preserved. The narrow ctx flows into createMcpServer's
+    // registerTool gate, so the surface a /mcp/v/<slug> client sees is
+    // strictly product.tools (intersected with any pre-existing
+    // allowedTools the credential already carries).
+    function intersectAllowed(a, b) {
+        if (!a)
+            return b;
+        if (!b)
+            return a;
+        const bSet = new Set(b);
+        return a.filter((t) => bSet.has(t));
+    }
+    async function resolveVirtualProduct(req, res, baseCtx) {
+        const slug = req.params.slug;
+        if (!slug || typeof slug !== "string") {
+            res.status(404).json({ error: "virtual server not found" });
+            return null;
+        }
+        // Hot-reload aware so newly-published products are visible
+        // without restart (same pattern /mcp uses for product changes).
+        await products.maybeReload().catch(() => undefined);
+        const tenant = baseCtx.tenant || "default";
+        const product = products.get(slug, tenant);
+        if (!product || product.status === "staging") {
+            // 404 (not 403) for cross-tenant or missing — matches the
+            // existence-hiding stance of the rest of the tenancy layer.
+            res.status(404).json({ error: "virtual server not found" });
+            return null;
+        }
+        const allowedTools = intersectAllowed(baseCtx.allowedTools, product.tools);
+        const ctx = { ...baseCtx, allowedTools };
+        return { product, ctx };
+    }
+    app.post("/mcp/v/:slug", async (req, res) => {
+        const baseCtx = await gateCtx(req, res);
+        if (!baseCtx)
+            return;
+        const resolved = await resolveVirtualProduct(req, res, baseCtx);
+        if (!resolved)
+            return;
+        const { ctx, product } = resolved;
+        const sessionId = req.headers["mcp-session-id"];
+        let transport;
+        if (sessionId && transports.has(sessionId)) {
+            // Cross-product session probe is rejected: the session is
+            // bound to whichever virtual server issued it.
+            if (sessionProduct.get(sessionId) !== product.id) {
+                res.status(404).json({ error: "virtual server not found" });
+                return;
+            }
+            transport = transports.get(sessionId);
+        }
+        else {
+            transport = new StreamableHTTPServerTransport({
+                sessionIdGenerator: () => randomUUID(),
+            });
+            transport.onclose = () => {
+                for (const [sid, t] of transports) {
+                    if (t === transport) {
+                        transports.delete(sid);
+                        sessionProduct.delete(sid);
+                        break;
+                    }
+                }
+                mcpActiveSessions.set(transports.size);
+            };
+            const { mcpServer: sessionMcpServer } = createMcpServer(ctx);
+            await sessionMcpServer.connect(transport);
+        }
+        await transport.handleRequest(req, res, req.body);
+        const sid = res.getHeader("mcp-session-id");
+        if (sid) {
+            if (!transports.has(sid)) {
+                transports.set(sid, transport);
+                sessionProduct.set(sid, product.id);
+            }
+            sessionLastActive.set(sid, Date.now());
+        }
+        mcpActiveSessions.set(transports.size);
+    });
+    app.get("/mcp/v/:slug", async (req, res) => {
+        const baseCtx = await gateCtx(req, res);
+        if (!baseCtx)
+            return;
+        const resolved = await resolveVirtualProduct(req, res, baseCtx);
+        if (!resolved)
+            return;
+        const sessionId = req.headers["mcp-session-id"];
+        const transport = transports.get(sessionId);
+        if (!transport || sessionProduct.get(sessionId) !== resolved.product.id) {
+            res.status(400).json({ error: "No active session" });
+            return;
+        }
+        await transport.handleRequest(req, res);
+    });
+    app.delete("/mcp/v/:slug", async (req, res) => {
+        const baseCtx = await gateCtx(req, res);
+        if (!baseCtx)
+            return;
+        const resolved = await resolveVirtualProduct(req, res, baseCtx);
+        if (!resolved)
+            return;
+        const sessionId = req.headers["mcp-session-id"];
+        const transport = transports.get(sessionId);
+        if (transport && sessionProduct.get(sessionId) === resolved.product.id) {
+            await transport.handleRequest(req, res);
+            transports.delete(sessionId);
+            sessionLastActive.delete(sessionId);
+            sessionProduct.delete(sessionId);
         }
         else {
             res.status(400).json({ error: "No active session" });
         }
     });
+    // Bearer-token resolver for WebSocket upgrade requests. Browsers
+    // can't set Authorization on a WS handshake, so we accept the token
+    // from any of: Authorization: Bearer X, ?token=X, or the
+    // Sec-WebSocket-Protocol subprotocol "bearer.X" (echoed back by the
+    // server when accepted so clients see which subprotocol won).
+    function extractWsToken(req) {
+        const auth = req.headers["authorization"];
+        if (typeof auth === "string") {
+            const m = auth.match(/^Bearer\s+(.+)$/i);
+            if (m)
+                return { token: m[1] };
+        }
+        try {
+            const url = new URL(req.url ?? "/", "http://localhost");
+            const q = url.searchParams.get("token");
+            if (q)
+                return { token: q };
+        }
+        catch {
+            /* malformed URL */
+        }
+        const sp = req.headers["sec-websocket-protocol"];
+        if (typeof sp === "string") {
+            const offered = sp.split(",").map((s) => s.trim());
+            const bearer = offered.find((p) => p.startsWith("bearer."));
+            if (bearer)
+                return { token: bearer.slice("bearer.".length), selectedSubprotocol: bearer };
+        }
+        return {};
+    }
+    async function gateWsCtx(req) {
+        const { token, selectedSubprotocol } = extractWsToken(req);
+        if (!credentialsConfigured()) {
+            return { ctx: defaultContext(), selectedSubprotocol };
+        }
+        if (!token) {
+            return { reject: 4401, reason: "unauthorized: token required" };
+        }
+        const cred = resolveToken(token, loadCredentials());
+        if (!cred) {
+            return { reject: 4401, reason: "unauthorized: invalid token" };
+        }
+        const credTenant = cred.tenant || "default";
+        const decision = toolRateLimiter.check(`${credTenant} ${cred.name}`);
+        if (!decision.allowed) {
+            return { reject: 4429, reason: "rate limit exceeded for identity" };
+        }
+        let allowedTools;
+        if (cred.productId) {
+            await products.maybeReload().catch(() => undefined);
+            const p = products.get(cred.productId, credTenant);
+            if (p && p.tools && p.tools.length > 0)
+                allowedTools = p.tools.slice();
+        }
+        return {
+            ctx: principalContext(cred.name, cred.allowedSources, {
+                allowBypassRedaction: cred.bypassRedaction,
+                tenant: cred.tenant,
+                allowedTools,
+            }),
+            selectedSubprotocol,
+        };
+    }
     const PORT = parseInt(process.env.PORT || "3000");
-    app.listen(PORT, () => {
+    const httpServer = app.listen(PORT, () => {
         ready = true;
         console.log(`observability-mcp server running on port ${PORT}`);
         console.log(`  MCP endpoint: http://localhost:${PORT}/mcp`);
+        console.log(`  MCP (WS):     ws://localhost:${PORT}/mcp/ws`);
         console.log(`  Web UI: http://localhost:${PORT}`);
         console.log(`  Connectors: ${registry.getAll().map((c) => c.name).join(", ")}`);
     });
+    // Mount the WebSocket MCP transport. One McpServer instance per
+    // accepted socket; per-connection state is carried in
+    // WebSocketServerTransport.sessionId so concurrent clients stay
+    // isolated. Dynamic import so the `ws` package only loads on
+    // platforms that actually use this transport.
+    const { WebSocketServer } = await import("ws");
+    const wss = new WebSocketServer({ noServer: true });
+    httpServer.on("upgrade", async (req, socket, head) => {
+        if (!req.url) {
+            socket.destroy();
+            return;
+        }
+        const path = req.url.split("?")[0];
+        if (path !== "/mcp/ws") {
+            socket.destroy();
+            return;
+        }
+        const auth = await gateWsCtx(req);
+        if ("reject" in auth) {
+            // Custom 4xxx codes during upgrade aren't expressible via HTTP
+            // status, so we accept the upgrade just long enough to close
+            // with the WS-level close code that carries our reason.
+            wss.handleUpgrade(req, socket, head, (ws) => {
+                ws.close(auth.reject === 4429 ? 1013 : 1008, auth.reason);
+            });
+            return;
+        }
+        wss.handleUpgrade(req, socket, head, async (ws) => {
+            try {
+                const transport = new WebSocketServerTransport(ws);
+                const { mcpServer: sessionMcpServer } = createMcpServer(auth.ctx);
+                await sessionMcpServer.connect(transport);
+            }
+            catch (err) {
+                console.warn("WS /mcp/ws session setup failed:", err);
+                try {
+                    ws.close(1011, "server error");
+                }
+                catch {
+                    /* socket already gone */
+                }
+            }
+        });
+    });
 }
 main().catch(console.error);