@arabold/docs-mcp-server 1.19.0 → 1.21.0

package/dist/index.js

@@ -1,9 +1,15 @@
 import "dotenv/config";
-import { Option, Command } from "commander";
+import { PostHog } from "posthog-node";
+import crypto, { randomUUID } from "node:crypto";
+import fs, { readFileSync, existsSync } from "node:fs";
 import path from "node:path";
+import envPaths from "env-paths";
+import { Option, Command } from "commander";
 import formBody from "@fastify/formbody";
 import fastifyStatic from "@fastify/static";
 import Fastify from "fastify";
+import { ProxyOAuthServerProvider } from "@modelcontextprotocol/sdk/server/auth/providers/proxyProvider.js";
+import { createRemoteJWKSet, jwtVerify } from "jose";
 import { SSEServerTransport } from "@modelcontextprotocol/sdk/server/sse.js";
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
 import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
@@ -17,7 +23,7 @@ import { chromium } from "playwright";
 import { gfm } from "@joplin/turndown-plugin-gfm";
 import TurndownService from "turndown";
 import iconv from "iconv-lite";
-import fs from "node:fs/promises";
+import fs$1 from "node:fs/promises";
 import * as mime from "mime-types";
 import axios from "axios";
 import { HeaderGenerator } from "header-generator";
@@ -25,7 +31,6 @@ import { initTRPC } from "@trpc/server";
 import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify";
 import { z as z$1 } from "zod";
 import { jsxs, jsx, Fragment } from "@kitajs/html/jsx-runtime";
-import fs$1, { readFileSync, existsSync } from "node:fs";
 import { unified } from "unified";
 import remarkParse from "remark-parse";
 import remarkGfm from "remark-gfm";
@@ -34,7 +39,6 @@ import DOMPurify from "dompurify";
 import { fileURLToPath, URL as URL$1 } from "node:url";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { createTRPCProxyClient, httpBatchLink } from "@trpc/client";
-import "env-paths";
 import "fuse.js";
 import "langchain/text_splitter";
 import "better-sqlite3";
@@ -101,10 +105,806 @@ const logger = {
     }
   }
 };
-const version = "1.18.0";
+class PostHogClient {
+  client;
+  enabled;
+  // PostHog configuration
+  static CONFIG = {
+    host: "https://app.posthog.com",
+    // Performance optimizations
+    flushAt: 20,
+    // Batch size - send after 20 events
+    flushInterval: 1e4,
+    // 10 seconds - send after time
+    // Privacy settings
+    disableGeoip: true,
+    // Don't collect IP geolocation
+    disableSessionRecording: true,
+    // Never record sessions
+    disableSurveys: true,
+    // No user surveys
+    // Data handling
+    persistence: "memory"
+    // No disk persistence for privacy
+  };
+  constructor(enabled) {
+    this.enabled = enabled;
+    if (this.enabled) {
+      try {
+        this.client = new PostHog("phc_g7pXZZdUiAQXdnwUANjloQWMvO0amEDTBaeDSWgXgrQ", {
+          host: PostHogClient.CONFIG.host,
+          flushAt: PostHogClient.CONFIG.flushAt,
+          flushInterval: PostHogClient.CONFIG.flushInterval,
+          disableGeoip: PostHogClient.CONFIG.disableGeoip
+        });
+        logger.debug("PostHog client initialized");
+      } catch (error) {
+        logger.debug(
+          `PostHog initialization failed: ${error instanceof Error ? error.message : "Unknown error"}`
+        );
+        this.enabled = false;
+      }
+    } else {
+      this.enabled = false;
+      logger.debug("PostHog client disabled");
+    }
+  }
+  /**
+   * Send event to PostHog
+   */
+  capture(distinctId, event, properties) {
+    if (!this.enabled || !this.client) return;
+    try {
+      this.client.capture({
+        distinctId,
+        event,
+        properties
+      });
+      logger.debug(`PostHog event captured: ${event}`);
+    } catch (error) {
+      logger.debug(
+        `PostHog capture error: ${error instanceof Error ? error.message : "Unknown error"}`
+      );
+    }
+  }
+  /**
+   * Graceful shutdown with event flushing
+   */
+  async shutdown() {
+    if (this.client) {
+      try {
+        await this.client.shutdown();
+        logger.debug("PostHog client shutdown complete");
+      } catch (error) {
+        logger.debug(
+          `PostHog shutdown error: ${error instanceof Error ? error.message : "Unknown error"}`
+        );
+      }
+    }
+  }
+  /**
+   * Check if client is enabled and ready
+   */
+  isEnabled() {
+    return this.enabled && !!this.client;
+  }
+}
+class SessionTracker {
+  sessionContext;
+  /**
+   * Start a new session with context
+   */
+  startSession(context) {
+    this.sessionContext = context;
+  }
+  /**
+   * End current session and return duration
+   */
+  endSession() {
+    if (!this.sessionContext) return null;
+    const duration = Date.now() - this.sessionContext.startTime.getTime();
+    const sessionInterface = this.sessionContext.interface;
+    this.sessionContext = void 0;
+    return { duration, interface: sessionInterface };
+  }
+  /**
+   * Get current session context
+   */
+  getSessionContext() {
+    return this.sessionContext;
+  }
+  /**
+   * Get enriched properties with session context
+   */
+  getEnrichedProperties(properties = {}) {
+    return {
+      ...this.sessionContext,
+      ...properties,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString()
+    };
+  }
+}
+class TelemetryConfig {
+  static instance;
+  enabled;
+  constructor() {
+    this.enabled = this.determineEnabledState();
+  }
+  /**
+   * Determines if telemetry should be enabled based on CLI flags and environment variables.
+   * Priority: CLI flags > environment variables > default (true)
+   */
+  determineEnabledState() {
+    if (process.env.DOCS_MCP_TELEMETRY === "false") {
+      return false;
+    }
+    const args = process.argv;
+    if (args.includes("--no-telemetry")) {
+      return false;
+    }
+    return true;
+  }
+  isEnabled() {
+    return this.enabled;
+  }
+  disable() {
+    this.enabled = false;
+  }
+  enable() {
+    this.enabled = true;
+  }
+  static getInstance() {
+    if (!TelemetryConfig.instance) {
+      TelemetryConfig.instance = new TelemetryConfig();
+    }
+    return TelemetryConfig.instance;
+  }
+}
+function generateInstallationId() {
+  try {
+    const envStorePath = process.env.DOCS_MCP_STORE_PATH;
+    const dataDir = envStorePath || envPaths("docs-mcp-server", { suffix: "" }).data;
+    const installationIdPath = path.join(dataDir, "installation.id");
+    if (fs.existsSync(installationIdPath)) {
+      const existingId = fs.readFileSync(installationIdPath, "utf8").trim();
+      if (existingId) {
+        return existingId;
+      }
+    }
+    const newId = randomUUID();
+    fs.mkdirSync(dataDir, { recursive: true });
+    fs.writeFileSync(installationIdPath, newId, "utf8");
+    return newId;
+  } catch {
+    return randomUUID();
+  }
+}
+function shouldEnableTelemetry() {
+  return TelemetryConfig.getInstance().isEnabled();
+}
+var TelemetryEvent = /* @__PURE__ */ ((TelemetryEvent2) => {
+  TelemetryEvent2["SESSION_STARTED"] = "session_started";
+  TelemetryEvent2["SESSION_ENDED"] = "session_ended";
+  TelemetryEvent2["APP_STARTED"] = "app_started";
+  TelemetryEvent2["APP_SHUTDOWN"] = "app_shutdown";
+  TelemetryEvent2["COMMAND_EXECUTED"] = "command_executed";
+  TelemetryEvent2["TOOL_USED"] = "tool_used";
+  TelemetryEvent2["HTTP_REQUEST_COMPLETED"] = "http_request_completed";
+  TelemetryEvent2["PIPELINE_JOB_PROGRESS"] = "pipeline_job_progress";
+  TelemetryEvent2["PIPELINE_JOB_COMPLETED"] = "pipeline_job_completed";
+  TelemetryEvent2["DOCUMENT_PROCESSED"] = "document_processed";
+  TelemetryEvent2["DOCUMENT_PROCESSING_FAILED"] = "document_processing_failed";
+  TelemetryEvent2["ERROR_OCCURRED"] = "error_occurred";
+  return TelemetryEvent2;
+})(TelemetryEvent || {});
+class Analytics {
+  postHogClient;
+  sessionTracker;
+  enabled = true;
+  distinctId;
+  constructor(enabled) {
+    this.enabled = enabled ?? TelemetryConfig.getInstance().isEnabled();
+    this.distinctId = generateInstallationId();
+    this.postHogClient = new PostHogClient(this.enabled);
+    this.sessionTracker = new SessionTracker();
+    if (this.enabled) {
+      logger.debug("Analytics enabled");
+    } else {
+      logger.debug("Analytics disabled");
+    }
+  }
+  /**
+   * Initialize session context - call once per session
+   */
+  startSession(context) {
+    if (!this.enabled) return;
+    this.sessionTracker.startSession(context);
+    this.track("session_started", {
+      interface: context.interface,
+      version: context.version,
+      platform: context.platform,
+      sessionDurationTarget: context.interface === "cli" ? "short" : "long",
+      authEnabled: context.authEnabled,
+      readOnly: context.readOnly,
+      servicesCount: context.servicesEnabled.length
+    });
+  }
+  /**
+   * Track an event with automatic session context inclusion
+   */
+  track(event, properties = {}) {
+    if (!this.enabled) return;
+    const eventProperties = this.sessionTracker.getEnrichedProperties(properties);
+    this.postHogClient.capture(this.distinctId, event, eventProperties);
+  }
+  /**
+   * Track session end with duration
+   */
+  endSession() {
+    if (!this.enabled) return;
+    const sessionInfo = this.sessionTracker.endSession();
+    if (sessionInfo) {
+      this.track("session_ended", {
+        durationMs: sessionInfo.duration,
+        interface: sessionInfo.interface
+      });
+    }
+  }
+  /**
+   * Graceful shutdown with event flushing
+   */
+  async shutdown() {
+    await this.postHogClient.shutdown();
+  }
+  /**
+   * Check if analytics is enabled
+   */
+  isEnabled() {
+    return this.enabled && this.postHogClient.isEnabled();
+  }
+  /**
+   * Get current session context
+   */
+  getSessionContext() {
+    return this.sessionTracker.getSessionContext();
+  }
+}
+const analytics = new Analytics();
+async function trackTool(toolName, operation, getProperties) {
+  const startTime = Date.now();
+  try {
+    const result = await operation();
+    analytics.track("tool_used", {
+      tool: toolName,
+      success: true,
+      durationMs: Date.now() - startTime,
+      ...getProperties ? getProperties(result) : {}
+    });
+    return result;
+  } catch (error) {
+    analytics.track("tool_used", {
+      tool: toolName,
+      success: false,
+      durationMs: Date.now() - startTime,
+      errorType: error instanceof Error ? error.constructor.name : "UnknownError"
+    });
+    throw error;
+  }
+}
+function extractHostname(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.hostname;
+  } catch {
+    return "invalid-hostname";
+  }
+}
+function extractProtocol(urlOrPath) {
+  try {
+    const parsed = new URL(urlOrPath);
+    return parsed.protocol.replace(":", "");
+  } catch {
+    if (urlOrPath.startsWith("/") || /^[A-Za-z]:/.test(urlOrPath)) {
+      return "file";
+    }
+    return "unknown";
+  }
+}
+function analyzeSearchQuery(query) {
+  return {
+    length: query.length,
+    wordCount: query.trim().split(/\s+/).length,
+    hasCodeTerms: /\b(function|class|import|export|const|let|var|def|async|await)\b/i.test(query),
+    hasSpecialChars: /[^\w\s]/.test(query)
+  };
+}
+function sanitizeErrorMessage(message) {
+  return message.replace(/https?:\/\/[^\s]+/gi, "[url]").replace(/file:\/\/[^\s]+/gi, "[file-url]").replace(/\/[^\s]*\.[a-z]{2,4}/gi, "[path]").replace(/[A-Za-z]:\\[^\s]+/g, "[path]").replace(/Bearer\s+[^\s]+/gi, "Bearer [token]").replace(/api[_-]?key[=:]\s*[^\s]+/gi, "api_key=[redacted]").replace(/token[=:]\s*[^\s]+/gi, "token=[redacted]").substring(0, 200);
+}
+function sanitizeError(error) {
+  return {
+    type: error.constructor.name,
+    message: sanitizeErrorMessage(error.message),
+    hasStack: Boolean(error.stack)
+  };
+}
+function extractCliFlags(argv) {
+  return argv.filter((arg) => arg.startsWith("--") || arg.startsWith("-"));
+}
+const version = "1.20.0";
 const packageJson = {
   version
 };
+function getPackageVersion() {
+  return packageJson.version;
+}
+function createCliSession(command, options) {
+  return {
+    sessionId: randomUUID(),
+    interface: "cli",
+    startTime: /* @__PURE__ */ new Date(),
+    version: getPackageVersion(),
+    platform: process.platform,
+    nodeVersion: process.version,
+    command: command || "unknown",
+    authEnabled: options?.authEnabled,
+    readOnly: options?.readOnly,
+    servicesEnabled: ["worker"]
+    // CLI typically runs embedded worker
+  };
+}
+function createMcpSession(options) {
+  return {
+    sessionId: randomUUID(),
+    interface: "mcp",
+    startTime: /* @__PURE__ */ new Date(),
+    version: getPackageVersion(),
+    platform: process.platform,
+    nodeVersion: process.version,
+    protocol: options.protocol || "stdio",
+    transport: options.transport,
+    authEnabled: options.authEnabled ?? false,
+    readOnly: options.readOnly ?? false,
+    servicesEnabled: options.servicesEnabled ?? ["mcp"]
+  };
+}
+function createTelemetryService() {
+  return {
+    startSession: (context) => {
+      analytics.startSession(context);
+    },
+    endSession: () => {
+      analytics.endSession();
+    },
+    shutdown: async () => {
+      await analytics.shutdown();
+    }
+  };
+}
+const telemetryService = createTelemetryService();
+function createAuthMiddleware(authManager) {
+  return async (request, reply) => {
+    try {
+      const authContext = await authManager.createAuthContext(
+        request.headers.authorization || "",
+        request
+      );
+      request.auth = authContext;
+      const isAuthEnabled = authManager.authConfig.enabled;
+      if (!isAuthEnabled) {
+        logger.debug("Authentication disabled, allowing request");
+        return;
+      }
+      if (!authContext.authenticated) {
+        const hasAuthHeader = !!request.headers.authorization;
+        if (hasAuthHeader) {
+          logger.debug("Token validation failed");
+          reply.status(401).header(
+            "WWW-Authenticate",
+            'Bearer realm="MCP Server", error="invalid_token"'
+          ).send({
+            error: "invalid_token",
+            error_description: "The access token is invalid"
+          });
+          return;
+        } else {
+          logger.debug("Missing authorization header");
+          reply.status(401).header("WWW-Authenticate", 'Bearer realm="MCP Server"').send({
+            error: "unauthorized",
+            error_description: "Authorization header required"
+          });
+          return;
+        }
+      }
+      logger.debug(
+        `Authentication successful for subject: ${authContext.subject || "anonymous"}`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Authentication failed";
+      logger.debug(`Authentication error: ${message}`);
+      reply.status(401).header("WWW-Authenticate", 'Bearer realm="MCP Server", error="invalid_token"').send({
+        error: "invalid_token",
+        error_description: "Token validation failed"
+      });
+    }
+  };
+}
+class ProxyAuthManager {
+  constructor(config) {
+    this.config = config;
+  }
+  proxyProvider = null;
+  discoveredEndpoints = null;
+  jwks = null;
+  /**
+   * Get the authentication configuration
+   */
+  get authConfig() {
+    return this.config;
+  }
+  /**
+   * Initialize the proxy auth manager with the configured OAuth provider.
+   */
+  async initialize() {
+    if (!this.config.enabled) {
+      logger.debug("Authentication disabled, skipping proxy auth manager initialization");
+      return;
+    }
+    if (!this.config.issuerUrl || !this.config.audience) {
+      throw new Error("Issuer URL and Audience are required when auth is enabled");
+    }
+    try {
+      logger.info("🔐 Initializing OAuth2 proxy authentication...");
+      this.discoveredEndpoints = await this.discoverEndpoints();
+      if (this.discoveredEndpoints.jwksUri) {
+        this.jwks = createRemoteJWKSet(new URL(this.discoveredEndpoints.jwksUri));
+        logger.debug(`JWKS configured from: ${this.discoveredEndpoints.jwksUri}`);
+      }
+      const capabilities = [];
+      if (this.discoveredEndpoints.jwksUri) capabilities.push("JWT validation via JWKS");
+      if (this.discoveredEndpoints.userinfoUrl)
+        capabilities.push("opaque token validation via userinfo");
+      logger.debug(`Token validation capabilities: ${capabilities.join(", ")}`);
+      if (capabilities.length === 0) {
+        logger.warn(
+          "⚠️ No token validation mechanisms available - authentication may fail"
+        );
+      }
+      this.proxyProvider = new ProxyOAuthServerProvider({
+        endpoints: {
+          authorizationUrl: this.discoveredEndpoints.authorizationUrl,
+          tokenUrl: this.discoveredEndpoints.tokenUrl,
+          revocationUrl: this.discoveredEndpoints.revocationUrl,
+          registrationUrl: this.discoveredEndpoints.registrationUrl
+        },
+        verifyAccessToken: this.verifyAccessToken.bind(this),
+        getClient: this.getClient.bind(this)
+      });
+      logger.info("✅ OAuth2 proxy authentication initialized successfully");
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      logger.error(`❌ Failed to initialize OAuth2 proxy authentication: ${message}`);
+      throw new Error(`Proxy authentication initialization failed: ${message}`);
+    }
+  }
+  /**
+   * Register OAuth2 endpoints on the Fastify server.
+   * This manually implements the necessary OAuth2 endpoints using the proxy provider.
+   */
+  registerRoutes(server, baseUrl) {
+    if (!this.proxyProvider) {
+      throw new Error("Proxy provider not initialized");
+    }
+    server.get("/.well-known/oauth-authorization-server", async (_request, reply) => {
+      const metadata = {
+        issuer: baseUrl.origin,
+        authorization_endpoint: `${baseUrl.origin}/oauth/authorize`,
+        token_endpoint: `${baseUrl.origin}/oauth/token`,
+        revocation_endpoint: `${baseUrl.origin}/oauth/revoke`,
+        registration_endpoint: `${baseUrl.origin}/oauth/register`,
+        scopes_supported: ["profile", "email"],
+        response_types_supported: ["code"],
+        grant_types_supported: ["authorization_code", "refresh_token"],
+        token_endpoint_auth_methods_supported: [
+          "client_secret_basic",
+          "client_secret_post",
+          "none"
+        ],
+        code_challenge_methods_supported: ["S256"]
+      };
+      reply.type("application/json").send(metadata);
+    });
+    server.get("/.well-known/oauth-protected-resource", async (request, reply) => {
+      const baseUrl2 = `${request.protocol}://${request.headers.host}`;
+      const metadata = {
+        resource: `${baseUrl2}/sse`,
+        authorization_servers: [this.config.issuerUrl],
+        scopes_supported: ["profile", "email"],
+        bearer_methods_supported: ["header"],
+        resource_name: "Documentation MCP Server",
+        resource_documentation: "https://github.com/arabold/docs-mcp-server#readme",
+        // Enhanced metadata for better discoverability
+        resource_server_metadata_url: `${baseUrl2}/.well-known/oauth-protected-resource`,
+        authorization_server_metadata_url: `${this.config.issuerUrl}/.well-known/openid-configuration`,
+        jwks_uri: `${this.config.issuerUrl}/.well-known/jwks.json`,
+        // Supported MCP transports
+        mcp_transports: [
+          {
+            transport: "sse",
+            endpoint: `${baseUrl2}/sse`,
+            description: "Server-Sent Events transport"
+          },
+          {
+            transport: "http",
+            endpoint: `${baseUrl2}/mcp`,
+            description: "Streaming HTTP transport"
+          }
+        ]
+      };
+      reply.type("application/json").send(metadata);
+    });
+    server.get("/oauth/authorize", async (request, reply) => {
+      const endpoints = await this.discoverEndpoints();
+      const params = new URLSearchParams(request.query);
+      if (!params.has("resource")) {
+        const resourceUrl = `${request.protocol}://${request.headers.host}/sse`;
+        params.set("resource", resourceUrl);
+      }
+      const redirectUrl = `${endpoints.authorizationUrl}?${params.toString()}`;
+      reply.redirect(redirectUrl);
+    });
+    server.post("/oauth/token", async (request, reply) => {
+      const endpoints = await this.discoverEndpoints();
+      const tokenBody = new URLSearchParams(request.body);
+      if (!tokenBody.has("resource")) {
+        const resourceUrl = `${request.protocol}://${request.headers.host}/sse`;
+        tokenBody.set("resource", resourceUrl);
+      }
+      const response = await fetch(endpoints.tokenUrl, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded"
+        },
+        body: tokenBody.toString()
+      });
+      const data = await response.json();
+      reply.status(response.status).type("application/json").send(data);
+    });
+    server.post("/oauth/revoke", async (request, reply) => {
+      const endpoints = await this.discoverEndpoints();
+      if (endpoints.revocationUrl) {
+        const response = await fetch(endpoints.revocationUrl, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/x-www-form-urlencoded"
+          },
+          body: new URLSearchParams(request.body).toString()
+        });
+        reply.status(response.status).send();
+      } else {
+        reply.status(404).send({ error: "Revocation not supported" });
+      }
+    });
+    server.post("/oauth/register", async (request, reply) => {
+      const endpoints = await this.discoverEndpoints();
+      if (endpoints.registrationUrl) {
+        const response = await fetch(endpoints.registrationUrl, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json"
+          },
+          body: JSON.stringify(request.body)
+        });
+        const data = await response.json();
+        reply.status(response.status).type("application/json").send(data);
+      } else {
+        reply.status(404).send({ error: "Dynamic client registration not supported" });
+      }
+    });
+    logger.debug("OAuth2 endpoints registered on Fastify server");
+  }
+  /**
+   * Discover OAuth endpoints from the OAuth2 authorization server.
+   * Uses OAuth2 discovery (RFC 8414) with OIDC discovery fallback.
+   * Supports both JWT and opaque token validation methods.
+   */
+  async discoverEndpoints() {
+    const oauthDiscoveryUrl = `${this.config.issuerUrl}/.well-known/oauth-authorization-server`;
+    try {
+      const oauthResponse = await fetch(oauthDiscoveryUrl);
+      if (oauthResponse.ok) {
+        const config2 = await oauthResponse.json();
+        logger.debug(
+          `Successfully discovered OAuth2 endpoints from: ${oauthDiscoveryUrl}`
+        );
+        const userinfoEndpoint = await this.discoverUserinfoEndpoint();
+        if (userinfoEndpoint) {
+          config2.userinfo_endpoint = userinfoEndpoint;
+        }
+        return this.buildEndpointsFromConfig(config2);
+      }
+    } catch (error) {
+      logger.debug(`OAuth2 discovery failed: ${error}, trying OIDC discovery`);
+    }
+    const oidcDiscoveryUrl = `${this.config.issuerUrl}/.well-known/openid-configuration`;
+    const oidcResponse = await fetch(oidcDiscoveryUrl);
+    if (!oidcResponse.ok) {
+      throw new Error(
+        `Failed to fetch configuration from both ${oauthDiscoveryUrl} and ${oidcDiscoveryUrl}`
+      );
+    }
+    const config = await oidcResponse.json();
+    logger.debug(`Successfully discovered OIDC endpoints from: ${oidcDiscoveryUrl}`);
+    return this.buildEndpointsFromConfig(config);
+  }
+  /**
+   * Try to discover userinfo endpoint for opaque token validation
+   */
+  async discoverUserinfoEndpoint() {
+    try {
+      const oidcDiscoveryUrl = `${this.config.issuerUrl}/.well-known/openid-configuration`;
+      const response = await fetch(oidcDiscoveryUrl);
+      if (response.ok) {
+        const config = await response.json();
+        return config.userinfo_endpoint || null;
+      }
+    } catch (error) {
+      logger.debug(`Failed to fetch userinfo endpoint: ${error}`);
+    }
+    return null;
+  }
+  /**
+   * Build endpoint configuration from discovery response.
+   */
+  buildEndpointsFromConfig(config) {
+    return {
+      authorizationUrl: config.authorization_endpoint,
+      tokenUrl: config.token_endpoint,
+      revocationUrl: config.revocation_endpoint,
+      registrationUrl: config.registration_endpoint,
+      jwksUri: config.jwks_uri,
+      userinfoUrl: config.userinfo_endpoint
+    };
+  }
+  /**
+   * Get supported resource URLs for this MCP server instance.
+   * This enables self-discovering resource validation per MCP Authorization spec.
+   */
+  getSupportedResources(request) {
+    const baseUrl = `${request.protocol}://${request.headers.host}`;
+    return [
+      `${baseUrl}/sse`,
+      // SSE transport
+      `${baseUrl}/mcp`,
+      // Streaming HTTP transport
+      `${baseUrl}`
+      // Server root
+    ];
+  }
+  /**
+   * Verify an access token using hybrid validation approach.
+   * First tries JWT validation with JWKS, falls back to userinfo endpoint for opaque tokens.
+   * This provides universal compatibility with all OAuth2 providers and token formats.
+   */
+  async verifyAccessToken(token, request) {
+    logger.debug(`Attempting to verify token: ${token.substring(0, 20)}...`);
+    if (this.jwks) {
+      try {
+        logger.debug("Attempting JWT validation with JWKS...");
+        const { payload } = await jwtVerify(token, this.jwks, {
+          issuer: this.config.issuerUrl,
+          audience: this.config.audience
+        });
+        logger.debug(
+          `JWT validation successful. Subject: ${payload.sub}, Audience: ${payload.aud}`
+        );
+        if (!payload.sub) {
+          throw new Error("JWT payload missing subject claim");
+        }
+        return {
+          token,
+          clientId: payload.sub,
+          scopes: ["*"]
+          // Full access for all authenticated users
+        };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        logger.debug(
+          `JWT validation failed: ${errorMessage}, trying userinfo fallback...`
+        );
+      }
+    }
+    if (this.discoveredEndpoints?.userinfoUrl) {
+      try {
+        logger.debug("Attempting userinfo endpoint validation...");
+        const response = await fetch(this.discoveredEndpoints.userinfoUrl, {
+          method: "GET",
+          headers: {
+            Authorization: `Bearer ${token}`,
+            Accept: "application/json"
+          }
+        });
+        if (!response.ok) {
+          throw new Error(
+            `Userinfo request failed: ${response.status} ${response.statusText}`
+          );
+        }
+        const userinfo = await response.json();
+        logger.debug(
+          `Token validation successful. User: ${userinfo.sub}, Email: ${userinfo.email}`
+        );
+        if (!userinfo.sub) {
+          throw new Error("Userinfo response missing subject");
+        }
+        if (request) {
+          const supportedResources = this.getSupportedResources(request);
+          logger.debug(`Supported resources: ${JSON.stringify(supportedResources)}`);
+        }
+        return {
+          token,
+          clientId: userinfo.sub,
+          scopes: ["*"]
+          // Full access for all authenticated users
+        };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        logger.debug(`Userinfo validation failed: ${errorMessage}`);
+      }
+    }
+    logger.debug("All token validation strategies exhausted");
+    throw new Error("Invalid access token");
+  }
+  /**
+   * Get client information for the given client ID.
+   * This is called by the proxy provider for client validation.
+   */
+  async getClient(clientId) {
+    return {
+      client_id: clientId,
+      redirect_uris: [`${this.config.audience}/callback`]
+      // Add other client metadata as needed
+    };
+  }
+  /**
+   * Create an authentication context from a token (for compatibility with existing middleware).
+   * Uses binary authentication - valid token grants full access.
+   */
+  async createAuthContext(authorization, request) {
+    if (!this.config.enabled) {
+      return {
+        authenticated: false,
+        scopes: /* @__PURE__ */ new Set()
+      };
+    }
+    try {
+      logger.debug(
+        `Processing authorization header: ${authorization.substring(0, 20)}...`
+      );
+      const match = authorization.match(/^Bearer\s+(.+)$/i);
+      if (!match) {
+        logger.debug("Authorization header does not match Bearer token pattern");
+        throw new Error("Invalid authorization header format");
+      }
+      const token = match[1];
+      logger.debug(`Extracted token: ${token.substring(0, 20)}...`);
+      const authInfo = await this.verifyAccessToken(token, request);
+      logger.debug(`Authentication successful for client: ${authInfo.clientId}`);
+      return {
+        authenticated: true,
+        scopes: /* @__PURE__ */ new Set(["*"]),
+        // Full access for authenticated users
+        subject: authInfo.clientId
+      };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : "Unknown error";
+      logger.debug(`Authentication failed: ${errorMessage}`);
+      return {
+        authenticated: false,
+        scopes: /* @__PURE__ */ new Set()
+      };
+    }
+  }
+}
 var PipelineJobStatus = /* @__PURE__ */ ((PipelineJobStatus2) => {
   PipelineJobStatus2["QUEUED"] = "queued";
   PipelineJobStatus2["RUNNING"] = "running";
@@ -185,7 +985,7 @@ class ClearCompletedJobsTool {
     try {
       const clearedCount = await this.pipeline.clearCompletedJobs();
       const message = clearedCount > 0 ? `Successfully cleared ${clearedCount} completed job${clearedCount === 1 ? "" : "s"} from the queue.` : "No completed jobs to clear.";
-      logger.debug(`[ClearCompletedJobsTool] ${message}`);
+      logger.debug(message);
       return {
         message,
         success: true,
@@ -193,7 +993,7 @@ class ClearCompletedJobsTool {
       };
     } catch (error) {
       const errorMessage = `Failed to clear completed jobs: ${error instanceof Error ? error.message : String(error)}`;
-      logger.error(`❌ [ClearCompletedJobsTool] ${errorMessage}`);
+      logger.error(`❌ ${errorMessage}`);
       return {
         message: errorMessage,
         success: false,
@@ -328,14 +1128,43 @@ class HtmlLinkExtractorMiddleware {
       return;
     }
     try {
+      let docBase = context.source;
+      try {
+        const baseEl = $("base[href]").first();
+        const rawBase = baseEl.attr("href");
+        if (rawBase && rawBase.trim() !== "") {
+          try {
+            const trimmed = rawBase.trim();
+            const candidate = new URL(trimmed, context.source);
+            const hasScheme = /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(trimmed);
+            const protocolRelative = trimmed.startsWith("//");
+            const firstSlash = trimmed.indexOf("/");
+            const firstColon = trimmed.indexOf(":");
+            const colonBeforeSlash = firstColon !== -1 && (firstSlash === -1 || firstColon < firstSlash);
+            const suspiciousColon = colonBeforeSlash && !hasScheme && !protocolRelative;
+            if (suspiciousColon || trimmed.startsWith(":")) {
+              logger.debug(
+                `Ignoring suspicious <base href> value (colon misuse): ${rawBase}`
+              );
+            } else {
+              docBase = candidate.href;
+            }
+          } catch {
+            logger.debug(`Ignoring invalid <base href> value: ${rawBase}`);
+          }
+        }
+      } catch {
+      }
       const linkElements = $("a[href]");
-      logger.debug(`Found ${linkElements.length} potential links in ${context.source}`);
+      logger.debug(
+        `Found ${linkElements.length} potential links in ${context.source} (base=${docBase})`
+      );
       const extractedLinks = [];
       linkElements.each((_index, element) => {
         const href = $(element).attr("href");
         if (href && href.trim() !== "") {
           try {
-            const urlObj = new URL(href, context.source);
+            const urlObj = new URL(href, docBase);
             if (!["http:", "https:", "file:"].includes(urlObj.protocol)) {
               logger.debug(`Ignoring link with invalid protocol: ${href}`);
               return;
@@ -1110,7 +1939,7 @@ class FetchUrlTool {
   async execute(options) {
     const { url, scrapeMode = ScrapeMode.Auto, headers } = options;
     const canFetchResults = this.fetchers.map((f) => f.canFetch(url));
-    const fetcherIndex = canFetchResults.findIndex((result) => result === true);
+    const fetcherIndex = canFetchResults.indexOf(true);
     if (fetcherIndex === -1) {
       throw new ToolError(
         `Invalid URL: ${url}. Must be an HTTP/HTTPS URL or a file:// URL.`,
@@ -1343,14 +2172,13 @@ class RemoveTool {
     this.pipeline = pipeline;
   }
   /**
-   * Executes the tool to remove the specified library version documents.
+   * Executes the tool to remove the specified library version completely.
    * Aborts any QUEUED/RUNNING job for the same library+version before deleting.
+   * Removes all documents, the version record, and the library if no other versions exist.
    */
   async execute(args) {
     const { library, version: version2 } = args;
-    logger.info(
-      `🗑️ Removing library: ${library}${version2 ? `, version: ${version2}` : " (unversioned)"}`
-    );
+    logger.info(`🗑️ Removing library: ${library}${version2 ? `@${version2}` : ""}`);
     try {
       const allJobs = await this.pipeline.getJobs();
       const jobs = allJobs.filter(
@@ -1363,12 +2191,12 @@ class RemoveTool {
         await this.pipeline.cancelJob(job.id);
         await this.pipeline.waitForJobCompletion(job.id);
       }
-      await this.documentManagementService.removeAllDocuments(library, version2);
-      const message = `Successfully removed documents for ${library}${version2 ? `@${version2}` : " (unversioned)"}.`;
+      await this.documentManagementService.removeVersion(library, version2);
+      const message = `Successfully removed ${library}${version2 ? `@${version2}` : ""}.`;
       logger.info(`✅ ${message}`);
       return { message };
     } catch (error) {
-      const errorMessage = `Failed to remove documents for ${library}${version2 ? `@${version2}` : " (unversioned)"}: ${error instanceof Error ? error.message : String(error)}`;
+      const errorMessage = `Failed to remove ${library}${version2 ? `@${version2}` : ""}: ${error instanceof Error ? error.message : String(error)}`;
       logger.error(`❌ Error removing library: ${errorMessage}`);
       throw new ToolError(errorMessage, this.constructor.name);
     }
@@ -1517,7 +2345,7 @@ function createError(text) {
     isError: true
   };
 }
-function createMcpServerInstance(tools) {
+function createMcpServerInstance(tools, readOnly = false) {
   const server = new McpServer(
     {
       name: "docs-mcp-server",
@@ -1531,54 +2359,56 @@ function createMcpServerInstance(tools) {
       }
     }
   );
-  server.tool(
-    "scrape_docs",
-    "Scrape and index documentation from a URL for a library. Use this tool to index a new library or a new version.",
-    {
-      url: z.string().url().describe("Documentation root URL to scrape."),
-      library: z.string().describe("Library name."),
-      version: z.string().optional().describe("Library version (optional)."),
-      maxPages: z.number().optional().default(DEFAULT_MAX_PAGES).describe(`Maximum number of pages to scrape (default: ${DEFAULT_MAX_PAGES}).`),
-      maxDepth: z.number().optional().default(DEFAULT_MAX_DEPTH$1).describe(`Maximum navigation depth (default: ${DEFAULT_MAX_DEPTH$1}).`),
-      scope: z.enum(["subpages", "hostname", "domain"]).optional().default("subpages").describe("Crawling boundary: 'subpages', 'hostname', or 'domain'."),
-      followRedirects: z.boolean().optional().default(true).describe("Follow HTTP redirects (3xx responses).")
-    },
-    {
-      title: "Scrape New Library Documentation",
-      destructiveHint: true,
-      // replaces existing docs
-      openWorldHint: true
-      // requires internet access
-    },
-    async ({ url, library, version: version2, maxPages, maxDepth, scope, followRedirects }) => {
-      try {
-        const result = await tools.scrape.execute({
-          url,
-          library,
-          version: version2,
-          waitForCompletion: false,
-          // Don't wait for completion
-          // onProgress: undefined, // Explicitly undefined or omitted
-          options: {
-            maxPages,
-            maxDepth,
-            scope,
-            followRedirects
+  if (!readOnly) {
+    server.tool(
+      "scrape_docs",
+      "Scrape and index documentation from a URL for a library. Use this tool to index a new library or a new version.",
+      {
+        url: z.string().url().describe("Documentation root URL to scrape."),
+        library: z.string().describe("Library name."),
+        version: z.string().optional().describe("Library version (optional)."),
+        maxPages: z.number().optional().default(DEFAULT_MAX_PAGES).describe(`Maximum number of pages to scrape (default: ${DEFAULT_MAX_PAGES}).`),
+        maxDepth: z.number().optional().default(DEFAULT_MAX_DEPTH$1).describe(`Maximum navigation depth (default: ${DEFAULT_MAX_DEPTH$1}).`),
+        scope: z.enum(["subpages", "hostname", "domain"]).optional().default("subpages").describe("Crawling boundary: 'subpages', 'hostname', or 'domain'."),
+        followRedirects: z.boolean().optional().default(true).describe("Follow HTTP redirects (3xx responses).")
+      },
+      {
+        title: "Scrape New Library Documentation",
+        destructiveHint: true,
+        // replaces existing docs
+        openWorldHint: true
+        // requires internet access
+      },
+      async ({ url, library, version: version2, maxPages, maxDepth, scope, followRedirects }) => {
+        try {
+          const result = await tools.scrape.execute({
+            url,
+            library,
+            version: version2,
+            waitForCompletion: false,
+            // Don't wait for completion
+            // onProgress: undefined, // Explicitly undefined or omitted
+            options: {
+              maxPages,
+              maxDepth,
+              scope,
+              followRedirects
+            }
+          });
+          if ("jobId" in result) {
+            return createResponse(`🚀 Scraping job started with ID: ${result.jobId}.`);
           }
-        });
-        if ("jobId" in result) {
-          return createResponse(`🚀 Scraping job started with ID: ${result.jobId}.`);
+          return createResponse(
+            `Scraping finished immediately (unexpectedly) with ${result.pagesScraped} pages.`
+          );
+        } catch (error) {
+          return createError(
+            `Failed to scrape documentation: ${error instanceof Error ? error.message : String(error)}`
+          );
         }
-        return createResponse(
-          `Scraping finished immediately (unexpectedly) with ${result.pagesScraped} pages.`
-        );
-      } catch (error) {
-        return createError(
-          `Failed to scrape documentation: ${error instanceof Error ? error.message : String(error)}`
-        );
       }
-    }
-  );
+    );
+  }
   server.tool(
     "search_docs",
     'Search up-to-date documentation for a library or package. Examples:\n\n- {library: "react", query: "hooks lifecycle"} -> matches latest version of React\n- {library: "react", version: "18.0.0", query: "hooks lifecycle"} -> matches React 18.0.0 or earlier\n- {library: "typescript", version: "5.x", query: "ReturnType example"} -> any TypeScript 5.x.x version\n- {library: "typescript", version: "5.2.x", query: "ReturnType example"} -> any TypeScript 5.2.x version',
@@ -1699,24 +2529,25 @@ ${result.libraries.map((lib) => `- ${lib.name}`).join("\n")}`
       }
     }
   );
-  server.tool(
-    "list_jobs",
-    "List all indexing jobs. Optionally filter by status.",
-    {
-      status: z.enum(["queued", "running", "completed", "failed", "cancelling", "cancelled"]).optional().describe("Filter jobs by status (optional).")
-    },
-    {
-      title: "List Indexing Jobs",
-      readOnlyHint: true,
-      destructiveHint: false
-    },
-    async ({ status }) => {
-      try {
-        const result = await tools.listJobs.execute({
-          status
-        });
-        const formattedJobs = result.jobs.map(
-          (job) => `- ID: ${job.id}
+  if (!readOnly) {
+    server.tool(
+      "list_jobs",
+      "List all indexing jobs. Optionally filter by status.",
+      {
+        status: z.enum(["queued", "running", "completed", "failed", "cancelling", "cancelled"]).optional().describe("Filter jobs by status (optional).")
+      },
+      {
+        title: "List Indexing Jobs",
+        readOnlyHint: true,
+        destructiveHint: false
+      },
+      async ({ status }) => {
+        try {
+          const result = await tools.listJobs.execute({
+            status
+          });
+          const formattedJobs = result.jobs.map(
+            (job) => `- ID: ${job.id}
   Status: ${job.status}
   Library: ${job.library}
   Version: ${job.version}
@@ -1724,100 +2555,101 @@ ${result.libraries.map((lib) => `- ${lib.name}`).join("\n")}`
   Started: ${job.startedAt}` : ""}${job.finishedAt ? `
   Finished: ${job.finishedAt}` : ""}${job.error ? `
   Error: ${job.error}` : ""}`
-        ).join("\n\n");
-        return createResponse(
-          result.jobs.length > 0 ? `Current Jobs:
+          ).join("\n\n");
+          return createResponse(
+            result.jobs.length > 0 ? `Current Jobs:
 
 ${formattedJobs}` : "No jobs found."
-        );
-      } catch (error) {
-        return createError(
-          `Failed to list jobs: ${error instanceof Error ? error.message : String(error)}`
-        );
-      }
-    }
-  );
-  server.tool(
-    "get_job_info",
-    "Get details for a specific indexing job. Use the 'list_jobs' tool to find the job ID.",
-    {
-      jobId: z.string().uuid().describe("Job ID to query.")
-    },
-    {
-      title: "Get Indexing Job Info",
-      readOnlyHint: true,
-      destructiveHint: false
-    },
-    async ({ jobId }) => {
-      try {
-        const result = await tools.getJobInfo.execute({ jobId });
-        if (!result.job) {
-          return createError(`Job with ID ${jobId} not found.`);
+          );
+        } catch (error) {
+          return createError(
+            `Failed to list jobs: ${error instanceof Error ? error.message : String(error)}`
+          );
         }
-        const job = result.job;
-        const formattedJob = `- ID: ${job.id}
+      }
+    );
+    server.tool(
+      "get_job_info",
+      "Get details for a specific indexing job. Use the 'list_jobs' tool to find the job ID.",
+      {
+        jobId: z.string().uuid().describe("Job ID to query.")
+      },
+      {
+        title: "Get Indexing Job Info",
+        readOnlyHint: true,
+        destructiveHint: false
+      },
+      async ({ jobId }) => {
+        try {
+          const result = await tools.getJobInfo.execute({ jobId });
+          if (!result.job) {
+            return createError(`Job with ID ${jobId} not found.`);
+          }
+          const job = result.job;
+          const formattedJob = `- ID: ${job.id}
   Status: ${job.status}
   Library: ${job.library}@${job.version}
   Created: ${job.createdAt}${job.startedAt ? `
   Started: ${job.startedAt}` : ""}${job.finishedAt ? `
   Finished: ${job.finishedAt}` : ""}${job.error ? `
   Error: ${job.error}` : ""}`;
-        return createResponse(`Job Info:
+          return createResponse(`Job Info:
 
 ${formattedJob}`);
-      } catch (error) {
-        return createError(
-          `Failed to get job info for ${jobId}: ${error instanceof Error ? error.message : String(error)}`
-        );
+        } catch (error) {
+          return createError(
+            `Failed to get job info for ${jobId}: ${error instanceof Error ? error.message : String(error)}`
+          );
+        }
       }
-    }
-  );
-  server.tool(
-    "cancel_job",
-    "Cancel a queued or running indexing job. Use the 'list_jobs' tool to find the job ID.",
-    {
-      jobId: z.string().uuid().describe("Job ID to cancel.")
-    },
-    {
-      title: "Cancel Indexing Job",
-      destructiveHint: true
-    },
-    async ({ jobId }) => {
-      try {
-        const result = await tools.cancelJob.execute({ jobId });
-        if (result.success) {
-          return createResponse(result.message);
+    );
+    server.tool(
+      "cancel_job",
+      "Cancel a queued or running indexing job. Use the 'list_jobs' tool to find the job ID.",
+      {
+        jobId: z.string().uuid().describe("Job ID to cancel.")
+      },
+      {
+        title: "Cancel Indexing Job",
+        destructiveHint: true
+      },
+      async ({ jobId }) => {
+        try {
+          const result = await tools.cancelJob.execute({ jobId });
+          if (result.success) {
+            return createResponse(result.message);
+          }
+          return createError(result.message);
+        } catch (error) {
+          return createError(
+            `Failed to cancel job ${jobId}: ${error instanceof Error ? error.message : String(error)}`
+          );
         }
-        return createError(result.message);
-      } catch (error) {
-        return createError(
-          `Failed to cancel job ${jobId}: ${error instanceof Error ? error.message : String(error)}`
-        );
       }
-    }
-  );
-  server.tool(
-    "remove_docs",
-    "Remove indexed documentation for a library version. Use only if explicitly instructed.",
-    {
-      library: z.string().describe("Library name."),
-      version: z.string().optional().describe("Library version (optional, removes unversioned if omitted).")
-    },
-    {
-      title: "Remove Library Documentation",
-      destructiveHint: true
-    },
-    async ({ library, version: version2 }) => {
-      try {
-        const result = await tools.remove.execute({ library, version: version2 });
-        return createResponse(result.message);
-      } catch (error) {
-        return createError(
-          `Failed to remove documents: ${error instanceof Error ? error.message : String(error)}`
-        );
+    );
+    server.tool(
+      "remove_docs",
+      "Remove indexed documentation for a library version. Use only if explicitly instructed.",
+      {
+        library: z.string().describe("Library name."),
+        version: z.string().optional().describe("Library version (optional, removes unversioned if omitted).")
+      },
+      {
+        title: "Remove Library Documentation",
+        destructiveHint: true
+      },
+      async ({ library, version: version2 }) => {
+        try {
+          const result = await tools.remove.execute({ library, version: version2 });
+          return createResponse(result.message);
+        } catch (error) {
+          return createError(
+            `Failed to remove documents: ${error instanceof Error ? error.message : String(error)}`
+          );
+        }
       }
-    }
-  );
+    );
+  }
   server.tool(
     "fetch_url",
     "Fetch a single URL and convert its content to Markdown. Use this tool to read the content of any web page.",
@@ -1881,74 +2713,76 @@ ${formattedJob}`);
       };
     }
   );
-  server.resource(
-    "jobs",
-    "docs://jobs",
-    {
-      description: "List indexing jobs, optionally filtering by status.",
-      mimeType: "application/json"
-    },
-    async (uri) => {
-      const statusParam = uri.searchParams.get("status");
-      let statusFilter;
-      if (statusParam) {
-        const validation = z.nativeEnum(PipelineJobStatus).safeParse(statusParam);
-        if (validation.success) {
-          statusFilter = validation.data;
-        } else {
-          logger.warn(`⚠️  Invalid status parameter received: ${statusParam}`);
+  if (!readOnly) {
+    server.resource(
+      "jobs",
+      "docs://jobs",
+      {
+        description: "List indexing jobs, optionally filtering by status.",
+        mimeType: "application/json"
+      },
+      async (uri) => {
+        const statusParam = uri.searchParams.get("status");
+        let statusFilter;
+        if (statusParam) {
+          const validation = z.nativeEnum(PipelineJobStatus).safeParse(statusParam);
+          if (validation.success) {
+            statusFilter = validation.data;
+          } else {
+            logger.warn(`⚠️  Invalid status parameter received: ${statusParam}`);
+          }
         }
-      }
-      const result = await tools.listJobs.execute({ status: statusFilter });
-      return {
-        contents: result.jobs.map((job) => ({
-          uri: new URL(job.id, uri).href,
-          mimeType: "application/json",
-          text: JSON.stringify({
-            id: job.id,
-            library: job.library,
-            version: job.version,
-            status: job.status,
-            error: job.error || void 0
-          })
-        }))
-      };
-    }
-  );
-  server.resource(
-    "job",
-    // A distinct name for this specific resource type
-    new ResourceTemplate("docs://jobs/{jobId}", { list: void 0 }),
-    {
-      description: "Get details for a specific indexing job by ID.",
-      mimeType: "application/json"
-    },
-    async (uri, { jobId }) => {
-      if (typeof jobId !== "string" || jobId.length === 0) {
-        logger.warn(`⚠️  Invalid jobId received in URI: ${jobId}`);
-        return { contents: [] };
-      }
-      const result = await tools.getJobInfo.execute({ jobId });
-      if (!result.job) {
-        return { contents: [] };
-      }
-      return {
-        contents: [
-          {
-            uri: uri.href,
+        const result = await tools.listJobs.execute({ status: statusFilter });
+        return {
+          contents: result.jobs.map((job) => ({
+            uri: new URL(job.id, uri).href,
             mimeType: "application/json",
             text: JSON.stringify({
-              id: result.job.id,
-              library: result.job.library,
-              version: result.job.version,
-              status: result.job.status,
-              error: result.job.error || void 0
+              id: job.id,
+              library: job.library,
+              version: job.version,
+              status: job.status,
+              error: job.error || void 0
             })
-          }
-        ]
-      };
-    }
-  );
+          }))
+        };
+      }
+    );
+    server.resource(
+      "job",
+      // A distinct name for this specific resource type
+      new ResourceTemplate("docs://jobs/{jobId}", { list: void 0 }),
+      {
+        description: "Get details for a specific indexing job by ID.",
+        mimeType: "application/json"
+      },
+      async (uri, { jobId }) => {
+        if (typeof jobId !== "string" || jobId.length === 0) {
+          logger.warn(`⚠️  Invalid jobId received in URI: ${jobId}`);
+          return { contents: [] };
+        }
+        const result = await tools.getJobInfo.execute({ jobId });
+        if (!result.job) {
+          return { contents: [] };
+        }
+        return {
+          contents: [
+            {
+              uri: uri.href,
+              mimeType: "application/json",
+              text: JSON.stringify({
+                id: result.job.id,
+                library: result.job.library,
+                version: result.job.version,
+                status: result.job.status,
+                error: result.job.error || void 0
+              })
+            }
+          ]
+        };
+      }
+    );
+  }
   return server;
 }
 class FileFetcher {
@@ -1963,7 +2797,7 @@ class FileFetcher {
     const rawPath = source.replace("file://", "");
     const filePath = decodeURIComponent(rawPath);
     try {
-      const content = await fs.readFile(filePath);
+      const content = await fs$1.readFile(filePath);
       const ext = path.extname(filePath).toLowerCase();
       const mimeType = mime.lookup(ext) || "application/octet-stream";
       return {
@@ -2054,9 +2888,49 @@ class HttpFetcher {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
   async fetch(source, options) {
+    const startTime = performance.now();
     const maxRetries = options?.maxRetries ?? FETCHER_MAX_RETRIES;
     const baseDelay = options?.retryDelay ?? FETCHER_BASE_DELAY;
     const followRedirects = options?.followRedirects ?? true;
+    try {
+      const result = await this.performFetch(
+        source,
+        options,
+        maxRetries,
+        baseDelay,
+        followRedirects
+      );
+      const duration = performance.now() - startTime;
+      analytics.track("http_request_completed", {
+        success: true,
+        hostname: extractHostname(source),
+        protocol: extractProtocol(source),
+        duration_ms: Math.round(duration),
+        content_size_bytes: result.content.length,
+        mime_type: result.mimeType,
+        has_encoding: !!result.encoding,
+        follow_redirects: followRedirects,
+        had_redirects: result.source !== source
+      });
+      return result;
+    } catch (error) {
+      const duration = performance.now() - startTime;
+      const axiosError = error;
+      const status = axiosError.response?.status;
+      analytics.track("http_request_completed", {
+        success: false,
+        hostname: extractHostname(source),
+        protocol: extractProtocol(source),
+        duration_ms: Math.round(duration),
+        status_code: status,
+        error_type: error instanceof CancellationError ? "cancellation" : error instanceof RedirectError ? "redirect" : error instanceof ScraperError ? "scraper" : "unknown",
+        error_code: axiosError.code,
+        follow_redirects: followRedirects
+      });
+      throw error;
+    }
+  }
+  async performFetch(source, options, maxRetries = FETCHER_MAX_RETRIES, baseDelay = FETCHER_BASE_DELAY, followRedirects = true) {
     for (let attempt = 0; attempt <= maxRetries; attempt++) {
       try {
         const fingerprint = this.fingerprintGenerator.generateHeaders();
@@ -2094,12 +2968,18 @@ class HttpFetcher {
         } else {
           content = Buffer.from(response.data);
         }
+        const finalUrl = (
+          // Node follow-redirects style
+          response.request?.res?.responseUrl || // Some adapters may expose directly
+          response.request?.responseUrl || // Fallback to axios recorded config URL
+          response.config?.url || source
+        );
         return {
           content,
           mimeType,
           charset,
           encoding: contentEncoding,
-          source
+          source: finalUrl
         };
       } catch (error) {
         const axiosError = error;
@@ -2150,20 +3030,35 @@ async function initializeTools(docService, pipeline) {
   };
   return tools;
 }
-async function registerMcpService(server, docService, pipeline) {
+async function registerMcpService(server, docService, pipeline, readOnly = false, authManager) {
   const mcpTools = await initializeTools(docService, pipeline);
-  const mcpServer = createMcpServerInstance(mcpTools);
+  const mcpServer = createMcpServerInstance(mcpTools, readOnly);
+  const authMiddleware = authManager ? createAuthMiddleware(authManager) : null;
   const sseTransports = {};
   server.route({
     method: "GET",
     url: "/sse",
+    preHandler: authMiddleware ? [authMiddleware] : void 0,
     handler: async (_request, reply) => {
       try {
         const transport = new SSEServerTransport("/messages", reply.raw);
         sseTransports[transport.sessionId] = transport;
+        if (analytics.isEnabled()) {
+          const session = createMcpSession({
+            protocol: "http",
+            transport: "sse",
+            authEnabled: !!authManager,
+            readOnly,
+            servicesEnabled: ["mcp"]
+          });
+          analytics.startSession(session);
+        }
         reply.raw.on("close", () => {
           delete sseTransports[transport.sessionId];
           transport.close();
+          if (analytics.isEnabled()) {
+            analytics.endSession();
+          }
         });
         await mcpServer.connect(transport);
       } catch (error) {
@@ -2198,16 +3093,30 @@ async function registerMcpService(server, docService, pipeline) {
   server.route({
     method: "POST",
     url: "/mcp",
+    preHandler: authMiddleware ? [authMiddleware] : void 0,
     handler: async (request, reply) => {
       try {
-        const requestServer = createMcpServerInstance(mcpTools);
+        const requestServer = createMcpServerInstance(mcpTools, readOnly);
         const requestTransport = new StreamableHTTPServerTransport({
           sessionIdGenerator: void 0
         });
+        if (analytics.isEnabled()) {
+          const session = createMcpSession({
+            protocol: "http",
+            transport: "streamable",
+            authEnabled: !!authManager,
+            readOnly,
+            servicesEnabled: ["mcp"]
+          });
+          analytics.startSession(session);
+        }
         reply.raw.on("close", () => {
           logger.debug("Streamable HTTP request closed");
           requestTransport.close();
           requestServer.close();
+          if (analytics.isEnabled()) {
+            analytics.endSession();
+          }
         });
         await requestServer.connect(requestTransport);
         await requestTransport.handleRequest(request.raw, reply.raw, request.body);
@@ -3031,410 +3940,475 @@ const Tooltip = ({ text, position = "top" }) => {
     }
   );
 };
-const ScrapeFormContent = () => /* @__PURE__ */ jsxs("div", { class: "mt-4 p-4 bg-white dark:bg-gray-800 rounded-lg shadow border border-gray-300 dark:border-gray-600", children: [
-  /* @__PURE__ */ jsx("h3", { class: "text-xl font-semibold text-gray-900 dark:text-white mb-2", children: "Queue New Scrape Job" }),
-  /* @__PURE__ */ jsxs(
-    "form",
-    {
-      "hx-post": "/web/jobs/scrape",
-      "hx-target": "#job-response",
-      "hx-swap": "innerHTML",
-      class: "space-y-2",
-      "x-data": "{\n        url: '',\n        hasPath: false,\n        headers: [],\n        checkUrlPath() {\n          try {\n            const url = new URL(this.url);\n            this.hasPath = url.pathname !== '/' && url.pathname !== '';\n          } catch (e) {\n            this.hasPath = false;\n          }\n        }\n      }",
-      children: [
-        /* @__PURE__ */ jsxs("div", { children: [
-          /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+const ScrapeFormContent = ({ defaultExcludePatterns }) => {
+  const defaultExcludePatternsText = defaultExcludePatterns?.join("\n") || "";
+  return /* @__PURE__ */ jsxs("div", { class: "mt-4 p-4 bg-white dark:bg-gray-800 rounded-lg shadow border border-gray-300 dark:border-gray-600", children: [
+    /* @__PURE__ */ jsx("h3", { class: "text-xl font-semibold text-gray-900 dark:text-white mb-2", children: "Queue New Scrape Job" }),
+    /* @__PURE__ */ jsxs(
+      "form",
+      {
+        "hx-post": "/web/jobs/scrape",
+        "hx-target": "#job-response",
+        "hx-swap": "innerHTML",
+        class: "space-y-2",
+        "x-data": "{\n        url: '',\n        hasPath: false,\n        headers: [],\n        checkUrlPath() {\n          try {\n            const url = new URL(this.url);\n            this.hasPath = url.pathname !== '/' && url.pathname !== '';\n          } catch (e) {\n            this.hasPath = false;\n          }\n        }\n      }",
+        children: [
+          /* @__PURE__ */ jsxs("div", { children: [
+            /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+              /* @__PURE__ */ jsx(
+                "label",
+                {
+                  for: "url",
+                  class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
+                  children: "URL"
+                }
+              ),
+              /* @__PURE__ */ jsx(
+                Tooltip,
+                {
+                  text: /* @__PURE__ */ jsxs("div", { children: [
+                    /* @__PURE__ */ jsx("p", { children: "Enter the URL of the documentation you want to scrape." }),
+                    /* @__PURE__ */ jsxs("p", { class: "mt-2", children: [
+                      "For local files/folders, you must use the ",
+                      /* @__PURE__ */ jsx("code", { children: "file://" }),
+                      " ",
+                      "prefix and ensure the path is accessible to the server."
+                    ] }),
+                    /* @__PURE__ */ jsxs("p", { class: "mt-2", children: [
+                      "If running in Docker, ",
+                      /* @__PURE__ */ jsx("b", { children: "mount the folder" }),
+                      " (see README for details)."
+                    ] })
+                  ] })
+                }
+              )
+            ] }),
             /* @__PURE__ */ jsx(
-              "label",
+              "input",
               {
-                for: "url",
-                class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
-                children: "URL"
+                type: "url",
+                name: "url",
+                id: "url",
+                required: true,
+                "x-model": "url",
+                "x-on:input": "checkUrlPath",
+                "x-on:paste": "$nextTick(() => checkUrlPath())",
+                class: "mt-0.5 block w-full px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
               }
             ),
             /* @__PURE__ */ jsx(
-              Tooltip,
+              "div",
               {
-                text: /* @__PURE__ */ jsxs("div", { children: [
-                  /* @__PURE__ */ jsx("p", { children: "Enter the URL of the documentation you want to scrape." }),
-                  /* @__PURE__ */ jsxs("p", { class: "mt-2", children: [
-                    "For local files/folders, you must use the ",
-                    /* @__PURE__ */ jsx("code", { children: "file://" }),
-                    " ",
-                    "prefix and ensure the path is accessible to the server."
-                  ] }),
-                  /* @__PURE__ */ jsxs("p", { class: "mt-2", children: [
-                    "If running in Docker, ",
-                    /* @__PURE__ */ jsx("b", { children: "mount the folder" }),
-                    " (see README for details)."
-                  ] })
-                ] })
+                "x-show": "hasPath && !(url.startsWith('file://'))",
+                "x-cloak": true,
+                "x-transition:enter": "transition ease-out duration-300",
+                "x-transition:enter-start": "opacity-0 transform -translate-y-2",
+                "x-transition:enter-end": "opacity-100 transform translate-y-0",
+                class: "mt-2",
+                children: /* @__PURE__ */ jsx(
+                  Alert,
+                  {
+                    type: "info",
+                    message: "By default, only subpages under the given URL will be scraped. To scrape the whole website, adjust the 'Scope' option in Advanced Options."
+                  }
+                )
               }
             )
           ] }),
-          /* @__PURE__ */ jsx(
-            "input",
-            {
-              type: "url",
-              name: "url",
-              id: "url",
-              required: true,
-              "x-model": "url",
-              "x-on:input": "checkUrlPath",
-              "x-on:paste": "$nextTick(() => checkUrlPath())",
-              class: "mt-0.5 block w-full px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
-            }
-          ),
-          /* @__PURE__ */ jsx(
-            "div",
-            {
-              "x-show": "hasPath && !(url.startsWith('file://'))",
-              "x-cloak": true,
-              "x-transition:enter": "transition ease-out duration-300",
-              "x-transition:enter-start": "opacity-0 transform -translate-y-2",
-              "x-transition:enter-end": "opacity-100 transform translate-y-0",
-              class: "mt-2",
-              children: /* @__PURE__ */ jsx(
-                Alert,
+          /* @__PURE__ */ jsxs("div", { children: [
+            /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+              /* @__PURE__ */ jsx(
+                "label",
                 {
-                  type: "info",
-                  message: "By default, only subpages under the given URL will be scraped. To scrape the whole website, adjust the 'Scope' option in Advanced Options."
+                  for: "library",
+                  class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
+                  children: "Library Name"
                 }
-              )
-            }
-          )
-        ] }),
-        /* @__PURE__ */ jsxs("div", { children: [
-          /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+              ),
+              /* @__PURE__ */ jsx(Tooltip, { text: "The name of the library you're documenting. This will be used when searching." })
+            ] }),
             /* @__PURE__ */ jsx(
-              "label",
+              "input",
               {
-                for: "library",
-                class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
-                children: "Library Name"
+                type: "text",
+                name: "library",
+                id: "library",
+                required: true,
+                class: "mt-0.5 block w-full px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
               }
-            ),
-            /* @__PURE__ */ jsx(Tooltip, { text: "The name of the library you're documenting. This will be used when searching." })
+            )
           ] }),
-          /* @__PURE__ */ jsx(
-            "input",
-            {
-              type: "text",
-              name: "library",
-              id: "library",
-              required: true,
-              class: "mt-0.5 block w-full px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
-            }
-          )
-        ] }),
-        /* @__PURE__ */ jsxs("div", { children: [
-          /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+          /* @__PURE__ */ jsxs("div", { children: [
+            /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+              /* @__PURE__ */ jsx(
+                "label",
+                {
+                  for: "version",
+                  class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
+                  children: "Version (optional)"
+                }
+              ),
+              /* @__PURE__ */ jsx(Tooltip, { text: "Specify the version of the library documentation you're indexing. This allows for version-specific searches." })
+            ] }),
             /* @__PURE__ */ jsx(
-              "label",
+              "input",
               {
-                for: "version",
-                class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
-                children: "Version (optional)"
+                type: "text",
+                name: "version",
+                id: "version",
+                class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
               }
-            ),
-            /* @__PURE__ */ jsx(Tooltip, { text: "Specify the version of the library documentation you're indexing. This allows for version-specific searches." })
+            )
           ] }),
-          /* @__PURE__ */ jsx(
-            "input",
-            {
-              type: "text",
-              name: "version",
-              id: "version",
-              class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
-            }
-          )
-        ] }),
-        /* @__PURE__ */ jsxs("details", { class: "bg-gray-50 dark:bg-gray-900 p-2 rounded-md", children: [
-          /* @__PURE__ */ jsx("summary", { class: "cursor-pointer text-sm font-medium text-gray-600 dark:text-gray-400", children: "Advanced Options" }),
-          /* @__PURE__ */ jsxs("div", { class: "mt-2 space-y-2", "x-data": "{ headers: [] }", children: [
-            /* @__PURE__ */ jsxs("div", { children: [
-              /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+          /* @__PURE__ */ jsxs("details", { class: "bg-gray-50 dark:bg-gray-900 p-2 rounded-md", children: [
+            /* @__PURE__ */ jsx("summary", { class: "cursor-pointer text-sm font-medium text-gray-600 dark:text-gray-400", children: "Advanced Options" }),
+            /* @__PURE__ */ jsxs("div", { class: "mt-2 space-y-2", "x-data": "{ headers: [] }", children: [
+              /* @__PURE__ */ jsxs("div", { children: [
+                /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+                  /* @__PURE__ */ jsx(
+                    "label",
+                    {
+                      for: "maxPages",
+                      class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
+                      children: "Max Pages"
+                    }
+                  ),
+                  /* @__PURE__ */ jsx(Tooltip, { text: "The maximum number of pages to scrape. Default is 1000. Setting this too high may result in longer processing times." })
+                ] }),
                 /* @__PURE__ */ jsx(
-                  "label",
+                  "input",
                   {
-                    for: "maxPages",
-                    class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
-                    children: "Max Pages"
+                    type: "number",
+                    name: "maxPages",
+                    id: "maxPages",
+                    min: "1",
+                    placeholder: "1000",
+                    class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
                   }
-                ),
-                /* @__PURE__ */ jsx(Tooltip, { text: "The maximum number of pages to scrape. Default is 1000. Setting this too high may result in longer processing times." })
+                )
               ] }),
-              /* @__PURE__ */ jsx(
-                "input",
-                {
-                  type: "number",
-                  name: "maxPages",
-                  id: "maxPages",
-                  min: "1",
-                  placeholder: "1000",
-                  class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
-                }
-              )
-            ] }),
-            /* @__PURE__ */ jsxs("div", { children: [
-              /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+              /* @__PURE__ */ jsxs("div", { children: [
+                /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+                  /* @__PURE__ */ jsx(
+                    "label",
+                    {
+                      for: "maxDepth",
+                      class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
+                      children: "Max Depth"
+                    }
+                  ),
+                  /* @__PURE__ */ jsx(Tooltip, { text: "How many links deep the scraper should follow. Default is 3. Higher values capture more content but increase processing time." })
+                ] }),
                 /* @__PURE__ */ jsx(
-                  "label",
+                  "input",
                   {
-                    for: "maxDepth",
-                    class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
-                    children: "Max Depth"
+                    type: "number",
+                    name: "maxDepth",
+                    id: "maxDepth",
+                    min: "0",
+                    placeholder: "3",
+                    class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
                   }
-                ),
-                /* @__PURE__ */ jsx(Tooltip, { text: "How many links deep the scraper should follow. Default is 3. Higher values capture more content but increase processing time." })
+                )
               ] }),
-              /* @__PURE__ */ jsx(
-                "input",
-                {
-                  type: "number",
-                  name: "maxDepth",
-                  id: "maxDepth",
-                  min: "0",
-                  placeholder: "3",
-                  class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
-                }
-              )
-            ] }),
-            /* @__PURE__ */ jsxs("div", { children: [
-              /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
-                /* @__PURE__ */ jsx(
-                  "label",
+              /* @__PURE__ */ jsxs("div", { children: [
+                /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+                  /* @__PURE__ */ jsx(
+                    "label",
+                    {
+                      for: "scope",
+                      class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
+                      children: "Scope"
+                    }
+                  ),
+                  /* @__PURE__ */ jsx(
+                    Tooltip,
+                    {
+                      text: /* @__PURE__ */ jsxs("div", { children: [
+                        "Controls which pages are scraped:",
+                        /* @__PURE__ */ jsxs("ul", { class: "list-disc pl-5", children: [
+                          /* @__PURE__ */ jsx("li", { children: "'Subpages' only scrapes under the given URL path," }),
+                          /* @__PURE__ */ jsx("li", { children: "'Hostname' scrapes all content on the same host (e.g., all of docs.example.com)," }),
+                          /* @__PURE__ */ jsx("li", { children: "'Domain' scrapes all content on the domain and its subdomains (e.g., all of example.com)." })
+                        ] })
+                      ] })
+                    }
+                  )
+                ] }),
+                /* @__PURE__ */ jsxs(
+                  "select",
                   {
-                    for: "scope",
-                    class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
-                    children: "Scope"
+                    name: "scope",
+                    id: "scope",
+                    class: "mt-0.5 block w-full max-w-sm pl-2 pr-10 py-1 text-base border border-gray-300 dark:border-gray-600 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white",
+                    children: [
+                      /* @__PURE__ */ jsx("option", { value: "subpages", selected: true, children: "Subpages (Default)" }),
+                      /* @__PURE__ */ jsx("option", { value: "hostname", children: "Hostname" }),
+                      /* @__PURE__ */ jsx("option", { value: "domain", children: "Domain" })
+                    ]
                   }
-                ),
+                )
+              ] }),
+              /* @__PURE__ */ jsxs("div", { children: [
+                /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+                  /* @__PURE__ */ jsx(
+                    "label",
+                    {
+                      for: "includePatterns",
+                      class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
+                      children: "Include Patterns"
+                    }
+                  ),
+                  /* @__PURE__ */ jsx(Tooltip, { text: "Glob or regex patterns for URLs to include. One per line or comma-separated. Regex patterns must be wrapped in slashes, e.g. /pattern/." })
+                ] }),
                 /* @__PURE__ */ jsx(
-                  Tooltip,
+                  "textarea",
                   {
-                    text: /* @__PURE__ */ jsxs("div", { children: [
-                      "Controls which pages are scraped:",
-                      /* @__PURE__ */ jsxs("ul", { class: "list-disc pl-5", children: [
-                        /* @__PURE__ */ jsx("li", { children: "'Subpages' only scrapes under the given URL path," }),
-                        /* @__PURE__ */ jsx("li", { children: "'Hostname' scrapes all content on the same host (e.g., all of docs.example.com)," }),
-                        /* @__PURE__ */ jsx("li", { children: "'Domain' scrapes all content on the domain and its subdomains (e.g., all of example.com)." })
-                      ] })
-                    ] })
+                    name: "includePatterns",
+                    id: "includePatterns",
+                    rows: "2",
+                    placeholder: "e.g. docs/* or /api\\/v1.*/",
+                    class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
                   }
                 )
               ] }),
-              /* @__PURE__ */ jsxs(
-                "select",
-                {
-                  name: "scope",
-                  id: "scope",
-                  class: "mt-0.5 block w-full max-w-sm pl-2 pr-10 py-1 text-base border border-gray-300 dark:border-gray-600 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white",
-                  children: [
-                    /* @__PURE__ */ jsx("option", { value: "subpages", selected: true, children: "Subpages (Default)" }),
-                    /* @__PURE__ */ jsx("option", { value: "hostname", children: "Hostname" }),
-                    /* @__PURE__ */ jsx("option", { value: "domain", children: "Domain" })
-                  ]
-                }
-              )
-            ] }),
-            /* @__PURE__ */ jsxs("div", { children: [
-              /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+              /* @__PURE__ */ jsxs("div", { children: [
+                /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+                  /* @__PURE__ */ jsx(
+                    "label",
+                    {
+                      for: "excludePatterns",
+                      class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
+                      children: "Exclude Patterns"
+                    }
+                  ),
+                  /* @__PURE__ */ jsx(Tooltip, { text: "Glob or regex patterns for URLs to exclude. One per line or comma-separated. Exclude takes precedence over include. Regex patterns must be wrapped in slashes, e.g. /pattern/. Edit or clear this field to customize exclusions." })
+                ] }),
                 /* @__PURE__ */ jsx(
-                  "label",
+                  "textarea",
                   {
-                    for: "includePatterns",
-                    class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
-                    children: "Include Patterns"
+                    name: "excludePatterns",
+                    id: "excludePatterns",
+                    rows: "5",
+                    class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white font-mono text-xs",
+                    children: defaultExcludePatternsText
                   }
                 ),
-                /* @__PURE__ */ jsx(Tooltip, { text: "Glob or regex patterns for URLs to include. One per line or comma-separated. Regex patterns must be wrapped in slashes, e.g. /pattern/." })
+                /* @__PURE__ */ jsx("p", { class: "mt-1 text-xs text-gray-500 dark:text-gray-400", children: "Default patterns are pre-filled. Edit to customize or clear to exclude nothing." })
               ] }),
-              /* @__PURE__ */ jsx(
-                "textarea",
-                {
-                  name: "includePatterns",
-                  id: "includePatterns",
-                  rows: "2",
-                  placeholder: "e.g. docs/* or /api\\/v1.*/",
-                  class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
-                }
-              )
-            ] }),
-            /* @__PURE__ */ jsxs("div", { children: [
-              /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
-                /* @__PURE__ */ jsx(
-                  "label",
+              /* @__PURE__ */ jsxs("div", { children: [
+                /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+                  /* @__PURE__ */ jsx(
+                    "label",
+                    {
+                      for: "scrapeMode",
+                      class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
+                      children: "Scrape Mode"
+                    }
+                  ),
+                  /* @__PURE__ */ jsx(
+                    Tooltip,
+                    {
+                      text: /* @__PURE__ */ jsx("div", { children: /* @__PURE__ */ jsxs("ul", { class: "list-disc pl-5", children: [
+                        /* @__PURE__ */ jsx("li", { children: "'Auto' automatically selects the best method," }),
+                        /* @__PURE__ */ jsx("li", { children: "'Fetch' uses simple HTTP requests (faster but may miss dynamic content)," }),
+                        /* @__PURE__ */ jsx("li", { children: "'Playwright' uses a headless browser (slower but better for JS-heavy sites)." })
+                      ] }) })
+                    }
+                  )
+                ] }),
+                /* @__PURE__ */ jsxs(
+                  "select",
                   {
-                    for: "excludePatterns",
-                    class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
-                    children: "Exclude Patterns"
+                    name: "scrapeMode",
+                    id: "scrapeMode",
+                    class: "mt-0.5 block w-full max-w-sm pl-2 pr-10 py-1 text-base border border-gray-300 dark:border-gray-600 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white",
+                    children: [
+                      /* @__PURE__ */ jsx("option", { value: ScrapeMode.Auto, selected: true, children: "Auto (Default)" }),
+                      /* @__PURE__ */ jsx("option", { value: ScrapeMode.Fetch, children: "Fetch" }),
+                      /* @__PURE__ */ jsx("option", { value: ScrapeMode.Playwright, children: "Playwright" })
+                    ]
                   }
-                ),
-                /* @__PURE__ */ jsx(Tooltip, { text: "Glob or regex patterns for URLs to exclude. One per line or comma-separated. Exclude takes precedence over include. Regex patterns must be wrapped in slashes, e.g. /pattern/." })
+                )
+              ] }),
+              /* @__PURE__ */ jsxs("div", { children: [
+                /* @__PURE__ */ jsxs("div", { class: "flex items-center mb-1", children: [
+                  /* @__PURE__ */ jsx("label", { class: "block text-sm font-medium text-gray-700 dark:text-gray-300", children: "Custom HTTP Headers" }),
+                  /* @__PURE__ */ jsx(Tooltip, { text: "Add custom HTTP headers (e.g., for authentication). These will be sent with every HTTP request." })
+                ] }),
+                /* @__PURE__ */ jsxs("div", { children: [
+                  /* @__PURE__ */ jsx("template", { "x-for": "(header, idx) in headers", children: /* @__PURE__ */ jsxs("div", { class: "flex space-x-2 mb-1", children: [
+                    /* @__PURE__ */ jsx(
+                      "input",
+                      {
+                        type: "text",
+                        class: "w-1/3 px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-xs",
+                        placeholder: "Header Name",
+                        "x-model": "header.name",
+                        required: true
+                      }
+                    ),
+                    /* @__PURE__ */ jsx("span", { class: "text-gray-500", children: ":" }),
+                    /* @__PURE__ */ jsx(
+                      "input",
+                      {
+                        type: "text",
+                        class: "w-1/2 px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-xs",
+                        placeholder: "Header Value",
+                        "x-model": "header.value",
+                        required: true
+                      }
+                    ),
+                    /* @__PURE__ */ jsx(
+                      "button",
+                      {
+                        type: "button",
+                        class: "text-red-500 hover:text-red-700 text-xs",
+                        "x-on:click": "headers.splice(idx, 1)",
+                        children: "Remove"
+                      }
+                    ),
+                    /* @__PURE__ */ jsx(
+                      "input",
+                      {
+                        type: "hidden",
+                        name: "header[]",
+                        "x-bind:value": "header.name && header.value ? header.name + ':' + header.value : ''"
+                      }
+                    )
+                  ] }) }),
+                  /* @__PURE__ */ jsx(
+                    "button",
+                    {
+                      type: "button",
+                      class: "mt-1 px-2 py-0.5 bg-indigo-100 dark:bg-indigo-900 text-indigo-700 dark:text-indigo-200 rounded text-xs",
+                      "x-on:click": "headers.push({ name: '', value: '' })",
+                      children: "+ Add Header"
+                    }
+                  )
+                ] })
               ] }),
-              /* @__PURE__ */ jsx(
-                "textarea",
-                {
-                  name: "excludePatterns",
-                  id: "excludePatterns",
-                  rows: "2",
-                  placeholder: "e.g. private/* or /internal/",
-                  class: "mt-0.5 block w-full max-w-sm px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm bg-white dark:bg-gray-700 text-gray-900 dark:text-white"
-                }
-              )
-            ] }),
-            /* @__PURE__ */ jsxs("div", { children: [
               /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
                 /* @__PURE__ */ jsx(
-                  "label",
+                  "input",
                   {
-                    for: "scrapeMode",
-                    class: "block text-sm font-medium text-gray-700 dark:text-gray-300",
-                    children: "Scrape Mode"
+                    id: "followRedirects",
+                    name: "followRedirects",
+                    type: "checkbox",
+                    checked: true,
+                    class: "h-4 w-4 text-indigo-600 focus:ring-indigo-500 border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-700"
                   }
                 ),
                 /* @__PURE__ */ jsx(
-                  Tooltip,
+                  "label",
                   {
-                    text: /* @__PURE__ */ jsx("div", { children: /* @__PURE__ */ jsxs("ul", { class: "list-disc pl-5", children: [
-                      /* @__PURE__ */ jsx("li", { children: "'Auto' automatically selects the best method," }),
-                      /* @__PURE__ */ jsx("li", { children: "'Fetch' uses simple HTTP requests (faster but may miss dynamic content)," }),
-                      /* @__PURE__ */ jsx("li", { children: "'Playwright' uses a headless browser (slower but better for JS-heavy sites)." })
-                    ] }) })
+                    for: "followRedirects",
+                    class: "ml-1 block text-sm text-gray-900 dark:text-gray-300",
+                    children: "Follow Redirects"
                   }
                 )
               ] }),
-              /* @__PURE__ */ jsxs(
-                "select",
-                {
-                  name: "scrapeMode",
-                  id: "scrapeMode",
-                  class: "mt-0.5 block w-full max-w-sm pl-2 pr-10 py-1 text-base border border-gray-300 dark:border-gray-600 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white",
-                  children: [
-                    /* @__PURE__ */ jsx("option", { value: ScrapeMode.Auto, selected: true, children: "Auto (Default)" }),
-                    /* @__PURE__ */ jsx("option", { value: ScrapeMode.Fetch, children: "Fetch" }),
-                    /* @__PURE__ */ jsx("option", { value: ScrapeMode.Playwright, children: "Playwright" })
-                  ]
-                }
-              )
-            ] }),
-            /* @__PURE__ */ jsxs("div", { children: [
-              /* @__PURE__ */ jsxs("div", { class: "flex items-center mb-1", children: [
-                /* @__PURE__ */ jsx("label", { class: "block text-sm font-medium text-gray-700 dark:text-gray-300", children: "Custom HTTP Headers" }),
-                /* @__PURE__ */ jsx(Tooltip, { text: "Add custom HTTP headers (e.g., for authentication). These will be sent with every HTTP request." })
-              ] }),
-              /* @__PURE__ */ jsxs("div", { children: [
-                /* @__PURE__ */ jsx("template", { "x-for": "(header, idx) in headers", children: /* @__PURE__ */ jsxs("div", { class: "flex space-x-2 mb-1", children: [
-                  /* @__PURE__ */ jsx(
-                    "input",
-                    {
-                      type: "text",
-                      class: "w-1/3 px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-xs",
-                      placeholder: "Header Name",
-                      "x-model": "header.name",
-                      required: true
-                    }
-                  ),
-                  /* @__PURE__ */ jsx("span", { class: "text-gray-500", children: ":" }),
-                  /* @__PURE__ */ jsx(
-                    "input",
-                    {
-                      type: "text",
-                      class: "w-1/2 px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-xs",
-                      placeholder: "Header Value",
-                      "x-model": "header.value",
-                      required: true
-                    }
-                  ),
-                  /* @__PURE__ */ jsx(
-                    "button",
-                    {
-                      type: "button",
-                      class: "text-red-500 hover:text-red-700 text-xs",
-                      "x-on:click": "headers.splice(idx, 1)",
-                      children: "Remove"
-                    }
-                  ),
-                  /* @__PURE__ */ jsx(
-                    "input",
-                    {
-                      type: "hidden",
-                      name: "header[]",
-                      "x-bind:value": "header.name && header.value ? header.name + ':' + header.value : ''"
-                    }
-                  )
-                ] }) }),
+              /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
+                /* @__PURE__ */ jsx(
+                  "input",
+                  {
+                    id: "ignoreErrors",
+                    name: "ignoreErrors",
+                    type: "checkbox",
+                    checked: true,
+                    class: "h-4 w-4 text-indigo-600 focus:ring-indigo-500 border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-700"
+                  }
+                ),
                 /* @__PURE__ */ jsx(
-                  "button",
+                  "label",
                   {
-                    type: "button",
-                    class: "mt-1 px-2 py-0.5 bg-indigo-100 dark:bg-indigo-900 text-indigo-700 dark:text-indigo-200 rounded text-xs",
-                    "x-on:click": "headers.push({ name: '', value: '' })",
-                    children: "+ Add Header"
+                    for: "ignoreErrors",
+                    class: "ml-1 block text-sm text-gray-900 dark:text-gray-300",
+                    children: "Ignore Errors During Scraping"
                   }
                 )
               ] })
-            ] }),
-            /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
-              /* @__PURE__ */ jsx(
-                "input",
-                {
-                  id: "followRedirects",
-                  name: "followRedirects",
-                  type: "checkbox",
-                  checked: true,
-                  class: "h-4 w-4 text-indigo-600 focus:ring-indigo-500 border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-700"
-                }
-              ),
-              /* @__PURE__ */ jsx(
-                "label",
-                {
-                  for: "followRedirects",
-                  class: "ml-1 block text-sm text-gray-900 dark:text-gray-300",
-                  children: "Follow Redirects"
-                }
-              )
-            ] }),
-            /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
-              /* @__PURE__ */ jsx(
-                "input",
-                {
-                  id: "ignoreErrors",
-                  name: "ignoreErrors",
-                  type: "checkbox",
-                  checked: true,
-                  class: "h-4 w-4 text-indigo-600 focus:ring-indigo-500 border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-700"
-                }
-              ),
-              /* @__PURE__ */ jsx(
-                "label",
-                {
-                  for: "ignoreErrors",
-                  class: "ml-1 block text-sm text-gray-900 dark:text-gray-300",
-                  children: "Ignore Errors During Scraping"
-                }
-              )
             ] })
-          ] })
-        ] }),
-        /* @__PURE__ */ jsx("div", { children: /* @__PURE__ */ jsx(
-          "button",
-          {
-            type: "submit",
-            class: "w-full flex justify-center py-1.5 px-3 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500",
-            children: "Queue Job"
-          }
-        ) })
-      ]
-    }
-  ),
-  /* @__PURE__ */ jsx("div", { id: "job-response", class: "mt-2 text-sm" })
-] });
-const ScrapeForm = () => /* @__PURE__ */ jsx("div", { id: "scrape-form-container", children: /* @__PURE__ */ jsx(ScrapeFormContent, {}) });
+          ] }),
+          /* @__PURE__ */ jsx("div", { children: /* @__PURE__ */ jsx(
+            "button",
+            {
+              type: "submit",
+              class: "w-full flex justify-center py-1.5 px-3 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500",
+              children: "Queue Job"
+            }
+          ) })
+        ]
+      }
+    ),
+    /* @__PURE__ */ jsx("div", { id: "job-response", class: "mt-2 text-sm" })
+  ] });
+};
+const ScrapeForm = ({ defaultExcludePatterns }) => /* @__PURE__ */ jsx("div", { id: "scrape-form-container", children: /* @__PURE__ */ jsx(ScrapeFormContent, { defaultExcludePatterns }) });
+const DEFAULT_FILE_EXCLUSIONS = [
+  // CHANGELOG files (case variations)
+  "**/CHANGELOG.md",
+  "**/changelog.md",
+  "**/CHANGELOG.mdx",
+  "**/changelog.mdx",
+  // LICENSE files (case variations)
+  "**/LICENSE",
+  "**/LICENSE.md",
+  "**/license.md",
+  // CODE_OF_CONDUCT files (case variations)
+  "**/CODE_OF_CONDUCT.md",
+  "**/code_of_conduct.md"
+];
+const DEFAULT_FOLDER_EXCLUSIONS = [
+  // Archive and deprecated content (matches anywhere in path)
+  "**/archive/**",
+  "**/archived/**",
+  "**/deprecated/**",
+  "**/legacy/**",
+  "**/old/**",
+  "**/outdated/**",
+  "**/previous/**",
+  "**/superseded/**",
+  // Specific paths that don't follow the general pattern
+  "docs/old/**",
+  // Internationalization folders - non-English locales
+  "**/i18n/ar*/**",
+  "**/i18n/de*/**",
+  "**/i18n/es*/**",
+  "**/i18n/fr*/**",
+  "**/i18n/hi*/**",
+  "**/i18n/it*/**",
+  "**/i18n/ja*/**",
+  "**/i18n/ko*/**",
+  "**/i18n/nl*/**",
+  "**/i18n/pl*/**",
+  "**/i18n/pt*/**",
+  "**/i18n/ru*/**",
+  "**/i18n/sv*/**",
+  "**/i18n/th*/**",
+  "**/i18n/tr*/**",
+  "**/i18n/vi*/**",
+  "**/i18n/zh*/**",
+  // Common locale folder patterns
+  "**/zh-cn/**",
+  "**/zh-hk/**",
+  "**/zh-mo/**",
+  "**/zh-sg/**",
+  "**/zh-tw/**"
+];
+const DEFAULT_EXCLUSION_PATTERNS = [
+  ...DEFAULT_FILE_EXCLUSIONS,
+  ...DEFAULT_FOLDER_EXCLUSIONS
+];
+function getEffectiveExclusionPatterns(userPatterns) {
+  if (userPatterns !== void 0) {
+    return userPatterns;
+  }
+  return DEFAULT_EXCLUSION_PATTERNS;
+}
 function registerNewJobRoutes(server, scrapeTool) {
   server.get("/web/jobs/new", async () => {
-    return /* @__PURE__ */ jsx(ScrapeForm, {});
+    return /* @__PURE__ */ jsx(ScrapeForm, { defaultExcludePatterns: DEFAULT_EXCLUSION_PATTERNS });
   });
   server.post(
     "/web/jobs/scrape",
@@ -3505,7 +4479,7 @@ function registerNewJobRoutes(server, scrapeTool) {
                 ] })
               }
             ),
-            /* @__PURE__ */ jsx("div", { id: "scrape-form-container", "hx-swap-oob": "innerHTML", children: /* @__PURE__ */ jsx(ScrapeFormContent, {}) })
+            /* @__PURE__ */ jsx("div", { id: "scrape-form-container", "hx-swap-oob": "innerHTML", children: /* @__PURE__ */ jsx(ScrapeFormContent, { defaultExcludePatterns: DEFAULT_EXCLUSION_PATTERNS }) })
           ] });
         }
         return /* @__PURE__ */ jsx(Alert, { type: "warning", message: "Job finished unexpectedly quickly." });
@@ -3924,16 +4898,59 @@ async function registerWorkerService(pipeline) {
   pipeline.setCallbacks({
     onJobProgress: async (job, progress) => {
       logger.debug(
-        `📊 Job ${job.id} progress: ${progress.pagesScraped}/${progress.totalPages} pages`
+        `Job ${job.id} progress: ${progress.pagesScraped}/${progress.totalPages} pages`
       );
+      analytics.track(TelemetryEvent.PIPELINE_JOB_PROGRESS, {
+        jobId: job.id,
+        // Job IDs are already anonymous
+        library: job.library,
+        pagesScraped: progress.pagesScraped,
+        totalPages: progress.totalPages,
+        totalDiscovered: progress.totalDiscovered,
+        progressPercent: Math.round(progress.pagesScraped / progress.totalPages * 100),
+        currentDepth: progress.depth,
+        maxDepth: progress.maxDepth,
+        discoveryRatio: Math.round(
+          progress.totalDiscovered / progress.totalPages * 100
+        ),
+        // How much we discovered vs limited total
+        queue_efficiency: progress.totalPages > 0 ? Math.round(progress.pagesScraped / progress.totalPages * 100) : 0
+      });
     },
     onJobStatusChange: async (job) => {
-      logger.debug(`🔄 Job ${job.id} status changed to: ${job.status}`);
+      logger.debug(`Job ${job.id} status changed to: ${job.status}`);
+      const duration = job.startedAt ? Date.now() - job.startedAt.getTime() : null;
+      const queueWaitTime = job.startedAt && job.createdAt ? job.startedAt.getTime() - job.createdAt.getTime() : null;
+      analytics.track(TelemetryEvent.PIPELINE_JOB_COMPLETED, {
+        jobId: job.id,
+        // Job IDs are already anonymous
+        library: job.library,
+        status: job.status,
+        duration_ms: duration,
+        queue_wait_time_ms: queueWaitTime,
+        pages_processed: job.progressPages || 0,
+        max_pages_configured: job.progressMaxPages || 0,
+        has_version: !!job.version,
+        has_error: !!job.error,
+        throughput_pages_per_second: duration && job.progressPages ? Math.round(job.progressPages / duration * 1e3) : 0
+      });
     },
     onJobError: async (job, error, document) => {
       logger.warn(
         `⚠️ Job ${job.id} error ${document ? `on document ${document.metadata.url}` : ""}: ${error.message}`
       );
+      const errorInfo = sanitizeError(error);
+      analytics.track(TelemetryEvent.ERROR_OCCURRED, {
+        jobId: job.id,
+        // Job IDs are already anonymous
+        library: job.library,
+        errorType: errorInfo.type,
+        errorMessage: errorInfo.message,
+        hasDocument: !!document,
+        stage: document ? "document_processing" : "job_setup",
+        hasStack: errorInfo.hasStack,
+        pages_processed_before_error: job.progressPages || 0
+      });
     }
   });
   await pipeline.start();
@@ -3952,7 +4969,7 @@ function getProjectRoot() {
   let currentDir = path.dirname(currentFilePath);
   while (true) {
     const packageJsonPath = path.join(currentDir, "package.json");
-    if (fs$1.existsSync(packageJsonPath)) {
+    if (fs.existsSync(packageJsonPath)) {
       projectRoot = currentDir;
       return projectRoot;
     }
@@ -3975,6 +4992,7 @@ class AppServer {
   }
   server;
   mcpServer = null;
+  authManager = null;
   config;
   /**
    * Validate the server configuration for invalid service combinations.
@@ -4005,15 +5023,48 @@ class AppServer {
    */
   async start() {
     this.validateConfig();
+    if (this.config.telemetry !== false && shouldEnableTelemetry()) {
+      try {
+        telemetryService.startSession({
+          sessionId: crypto.randomUUID(),
+          interface: "web",
+          startTime: /* @__PURE__ */ new Date(),
+          version: process.env.npm_package_version || "unknown",
+          platform: process.platform,
+          servicesEnabled: this.getActiveServicesList(),
+          authEnabled: Boolean(this.config.auth),
+          readOnly: Boolean(this.config.readOnly)
+        });
+      } catch (error) {
+        logger.debug(`Failed to initialize telemetry: ${error}`);
+      }
+    }
     await this.setupServer();
     try {
+      const startupStartTime = performance.now();
       const address = await this.server.listen({
         port: this.config.port,
         host: "0.0.0.0"
       });
+      const startupDuration = performance.now() - startupStartTime;
+      if (analytics.isEnabled()) {
+        analytics.track(TelemetryEvent.APP_STARTED, {
+          startup_success: true,
+          startup_duration_ms: Math.round(startupDuration),
+          listen_address: address,
+          active_services: this.getActiveServicesList()
+        });
+      }
       this.logStartupInfo(address);
       return this.server;
     } catch (error) {
+      if (analytics.isEnabled()) {
+        analytics.track(TelemetryEvent.APP_STARTED, {
+          startup_success: false,
+          error_type: error instanceof Error ? error.constructor.name : "UnknownError",
+          error_message: error instanceof Error ? error.message : String(error)
+        });
+      }
       logger.error(`❌ Failed to start AppServer: ${error}`);
       await this.server.close();
       throw error;
@@ -4024,24 +5075,121 @@ class AppServer {
    */
   async stop() {
     try {
+      if (analytics.isEnabled()) {
+        analytics.track(TelemetryEvent.APP_SHUTDOWN, {
+          graceful: true
+        });
+      }
       if (this.config.enableWorker) {
         await stopWorkerService(this.pipeline);
       }
       if (this.mcpServer) {
         await cleanupMcpService(this.mcpServer);
       }
+      telemetryService.endSession();
+      await telemetryService.shutdown();
       await this.server.close();
       logger.info("🛑 AppServer stopped");
     } catch (error) {
       logger.error(`❌ Failed to stop AppServer gracefully: ${error}`);
+      if (analytics.isEnabled()) {
+        analytics.track(TelemetryEvent.APP_SHUTDOWN, {
+          graceful: false,
+          error: error instanceof Error ? error.constructor.name : "UnknownError"
+        });
+        await telemetryService.shutdown();
+      }
       throw error;
     }
   }
+  /**
+   * Setup global error handling for telemetry
+   */
+  setupErrorHandling() {
+    if (!process.listenerCount("unhandledRejection")) {
+      process.on("unhandledRejection", (reason) => {
+        logger.error(`Unhandled Promise Rejection: ${reason}`);
+        if (analytics.isEnabled()) {
+          analytics.track(TelemetryEvent.ERROR_OCCURRED, {
+            error_type: "UnhandledPromiseRejection",
+            error_category: "system",
+            component: "AppServer",
+            severity: "critical",
+            context: "process_unhandled_rejection"
+          });
+        }
+      });
+    }
+    if (!process.listenerCount("uncaughtException")) {
+      process.on("uncaughtException", (error) => {
+        logger.error(`Uncaught Exception: ${error.message}`);
+        if (analytics.isEnabled()) {
+          analytics.track(TelemetryEvent.ERROR_OCCURRED, {
+            error_type: error.constructor.name,
+            error_category: "system",
+            component: "AppServer",
+            severity: "critical",
+            context: "process_uncaught_exception"
+          });
+        }
+      });
+    }
+    if (typeof this.server.setErrorHandler === "function") {
+      this.server.setErrorHandler(async (error, request, reply) => {
+        if (analytics.isEnabled()) {
+          analytics.track(TelemetryEvent.ERROR_OCCURRED, {
+            error_type: error.constructor.name,
+            error_category: "http",
+            component: "FastifyServer",
+            severity: "high",
+            status_code: error.statusCode || 500,
+            method: request.method,
+            route: request.routeOptions?.url || request.url,
+            context: "http_request_error"
+          });
+        }
+        logger.error(`HTTP Error on ${request.method} ${request.url}: ${error.message}`);
+        const statusCode = error.statusCode || 500;
+        reply.status(statusCode).send({
+          error: "Internal Server Error",
+          statusCode,
+          message: statusCode < 500 ? error.message : "An unexpected error occurred"
+        });
+      });
+    }
+  }
+  /**
+   * Get list of currently active services for telemetry
+   */
+  getActiveServicesList() {
+    const services = [];
+    if (this.config.enableMcpServer) services.push("mcp");
+    if (this.config.enableWebInterface) services.push("web");
+    if (this.config.enableApiServer) services.push("api");
+    if (this.config.enableWorker) services.push("worker");
+    return services;
+  }
   /**
    * Setup the server with plugins and conditionally enabled services.
    */
   async setupServer() {
+    this.setupErrorHandling();
+    if (this.config.auth?.enabled) {
+      await this.initializeAuth();
+    }
     await this.server.register(formBody);
+    if (this.config.auth?.enabled) {
+      this.server.addHook("onRequest", async (request) => {
+        if (request.url.includes("/oauth") || request.url.includes("/auth") || request.url.includes("/register")) {
+          logger.debug(
+            `${request.method} ${request.url} - Headers: ${JSON.stringify(request.headers)}`
+          );
+        }
+      });
+    }
+    if (this.config.auth?.enabled && this.authManager) {
+      await this.setupAuthMetadataEndpoint();
+    }
     if (this.config.enableWebInterface) {
       await this.enableWebInterface();
     }
@@ -4072,7 +5220,9 @@ class AppServer {
     this.mcpServer = await registerMcpService(
       this.server,
       this.docService,
-      this.pipeline
+      this.pipeline,
+      this.config.readOnly,
+      this.authManager || void 0
     );
     logger.debug("MCP server service enabled");
   }
@@ -4100,6 +5250,28 @@ class AppServer {
       index: false
     });
   }
+  /**
+   * Initialize OAuth2/OIDC authentication manager.
+   */
+  async initializeAuth() {
+    if (!this.config.auth) {
+      return;
+    }
+    this.authManager = new ProxyAuthManager(this.config.auth);
+    await this.authManager.initialize();
+    logger.debug("Proxy auth manager initialized");
+  }
+  /**
+   * Setup OAuth2 endpoints using ProxyAuthManager.
+   */
+  async setupAuthMetadataEndpoint() {
+    if (!this.authManager) {
+      return;
+    }
+    const baseUrl = new URL(`http://localhost:${this.config.port}`);
+    this.authManager.registerRoutes(this.server, baseUrl);
+    logger.debug("OAuth2 proxy endpoints registered");
+  }
   /**
    * Log startup information showing which services are enabled.
    */
@@ -4130,9 +5302,9 @@ async function startAppServer(docService, pipeline, config) {
   await appServer.start();
   return appServer;
 }
-async function startStdioServer(tools) {
+async function startStdioServer(tools, readOnly = false) {
   setLogLevel(LogLevel.ERROR);
-  const server = createMcpServerInstance(tools);
+  const server = createMcpServerInstance(tools, readOnly);
   const transport = new StdioServerTransport();
   await server.connect(transport);
   logger.info("🤖 MCP server listening on stdio");
@@ -4191,10 +5363,10 @@ async function applyMigrations(db) {
     logger.debug("Checking database migrations...");
     ensureMigrationsTable(db);
     const appliedMigrations = getAppliedMigrations(db);
-    if (!fs$1.existsSync(MIGRATIONS_DIR)) {
+    if (!fs.existsSync(MIGRATIONS_DIR)) {
       throw new StoreError("Migrations directory not found");
     }
-    const migrationFiles = fs$1.readdirSync(MIGRATIONS_DIR).filter((file) => file.endsWith(".sql")).sort();
+    const migrationFiles = fs.readdirSync(MIGRATIONS_DIR).filter((file) => file.endsWith(".sql")).sort();
     const pendingMigrations = migrationFiles.filter(
       (filename) => !appliedMigrations.has(filename)
     );
@@ -4205,12 +5377,12 @@ async function applyMigrations(db) {
     for (const filename of pendingMigrations) {
       logger.debug(`Applying migration: ${filename}`);
       const filePath = path.join(MIGRATIONS_DIR, filename);
-      const sql = fs$1.readFileSync(filePath, "utf8");
+      const sql = fs.readFileSync(filePath, "utf8");
       try {
         db.exec(sql);
         const insertStmt = db.prepare(`INSERT INTO ${MIGRATIONS_TABLE} (id) VALUES (?)`);
         insertStmt.run(filename);
-        logger.debug(`✅ Applied migration: ${filename}`);
+        logger.debug(`Applied migration: ${filename}`);
         appliedCount++;
       } catch (error) {
         logger.error(`❌ Failed to apply migration: ${filename} - ${error}`);
@@ -4284,12 +5456,12 @@ async function createDocumentManagement(options = {}) {
     await client.initialize();
     return client;
   }
-  const service = new (await import("./DocumentManagementService-BH02TJEe.js")).DocumentManagementService();
+  const service = new (await import("./DocumentManagementService-C1xAzouZ.js")).DocumentManagementService();
   await service.initialize();
   return service;
 }
 async function createLocalDocumentManagement() {
-  const service = new (await import("./DocumentManagementService-BH02TJEe.js")).DocumentManagementService();
+  const service = new (await import("./DocumentManagementService-C1xAzouZ.js")).DocumentManagementService();
   await service.initialize();
   return service;
 }
@@ -4459,17 +5631,42 @@ function validateUrl(url) {
     throw new InvalidUrlError(url, error instanceof Error ? error : void 0);
   }
 }
-function hasSameHostname(urlA, urlB) {
-  return urlA.hostname.toLowerCase() === urlB.hostname.toLowerCase();
+function extractPrimaryDomain(hostname) {
+  if (/^\d+\.\d+\.\d+\.\d+$/.test(hostname) || /^[0-9a-fA-F:]+$/.test(hostname)) {
+    return hostname;
+  }
+  if (!hostname.includes(".")) {
+    return hostname;
+  }
+  const domain = psl.get(hostname.toLowerCase());
+  return domain || hostname;
 }
-function hasSameDomain(urlA, urlB) {
-  const domainA = psl.get(urlA.hostname.toLowerCase());
-  const domainB = psl.get(urlB.hostname.toLowerCase());
-  return domainA !== null && domainA === domainB;
+function computeBaseDirectory(pathname) {
+  if (pathname === "") return "/";
+  if (pathname.endsWith("/")) return pathname;
+  const lastSegment = pathname.split("/").at(-1) || "";
+  const looksLikeFile = lastSegment.includes(".");
+  if (looksLikeFile) {
+    return pathname.replace(/\/[^/]*$/, "/");
+  }
+  return `${pathname}/`;
 }
-function isSubpath(baseUrl, targetUrl) {
-  const basePath = baseUrl.pathname.endsWith("/") ? baseUrl.pathname : `${baseUrl.pathname}/`;
-  return targetUrl.pathname.startsWith(basePath);
+function isInScope(baseUrl, targetUrl, scope) {
+  if (baseUrl.protocol !== targetUrl.protocol) return false;
+  switch (scope) {
+    case "subpages": {
+      if (baseUrl.hostname !== targetUrl.hostname) return false;
+      const baseDir = computeBaseDirectory(baseUrl.pathname);
+      return targetUrl.pathname.startsWith(baseDir);
+    }
+    case "hostname":
+      return baseUrl.hostname === targetUrl.hostname;
+    case "domain": {
+      return extractPrimaryDomain(baseUrl.hostname) === extractPrimaryDomain(targetUrl.hostname);
+    }
+    default:
+      return false;
+  }
 }
 function isRegexPattern(pattern) {
   return pattern.length > 2 && pattern.startsWith("/") && pattern.endsWith("/");
@@ -4512,29 +5709,12 @@ function shouldIncludeUrl(url, includePatterns, excludePatterns) {
     }
   }
   const stripSlash = (patterns) => patterns?.map((p) => p.startsWith("/") ? p.slice(1) : p);
-  if (matchesAnyPattern(normalizedPath, excludePatterns) || basename && matchesAnyPattern(basename, stripSlash(excludePatterns)))
+  const effectiveExcludePatterns = getEffectiveExclusionPatterns(excludePatterns);
+  if (matchesAnyPattern(normalizedPath, effectiveExcludePatterns) || basename && matchesAnyPattern(basename, stripSlash(effectiveExcludePatterns)))
     return false;
   if (!includePatterns || includePatterns.length === 0) return true;
   return matchesAnyPattern(normalizedPath, includePatterns) || (basename ? matchesAnyPattern(basename, stripSlash(includePatterns)) : false);
 }
-function isInScope(baseUrl, targetUrl, scope) {
-  if (baseUrl.protocol !== targetUrl.protocol) return false;
-  switch (scope) {
-    case "subpages": {
-      if (baseUrl.hostname !== targetUrl.hostname) return false;
-      const baseDir = baseUrl.pathname.endsWith("/") ? baseUrl.pathname : baseUrl.pathname.replace(/\/[^/]*$/, "/");
-      return targetUrl.pathname.startsWith(baseDir);
-    }
-    case "hostname":
-      return baseUrl.hostname === targetUrl.hostname;
-    case "domain": {
-      const getDomain = (host) => host.split(".").slice(-2).join(".");
-      return getDomain(baseUrl.hostname) === getDomain(targetUrl.hostname);
-    }
-    default:
-      return false;
-  }
-}
 const DEFAULT_MAX_DEPTH = 3;
 const DEFAULT_CONCURRENCY = 3;
 class BaseScraperStrategy {
@@ -4543,6 +5723,8 @@ class BaseScraperStrategy {
   totalDiscovered = 0;
   // Track total URLs discovered (unlimited)
   effectiveTotal = 0;
+  // Track effective total (limited by maxPages)
+  canonicalBaseUrl;
   options;
   constructor(options = {}) {
     this.options = options;
@@ -4554,7 +5736,7 @@ class BaseScraperStrategy {
   shouldProcessUrl(url, options) {
     if (options.scope) {
       try {
-        const base = new URL$1(options.url);
+        const base = this.canonicalBaseUrl ?? new URL$1(options.url);
         const target = new URL$1(url);
         if (!isInScope(base, target, options.scope)) return false;
       } catch {
@@ -4577,6 +5759,23 @@ class BaseScraperStrategy {
         }
         try {
           const result = await this.processItem(item, options, void 0, signal);
+          if (item.depth === 0 && !this.canonicalBaseUrl && result?.finalUrl) {
+            try {
+              const finalUrlStr = result.finalUrl;
+              const original = new URL$1(options.url);
+              const finalUrlObj = new URL$1(finalUrlStr);
+              if (finalUrlObj.href !== original.href && (finalUrlObj.protocol === "http:" || finalUrlObj.protocol === "https:")) {
+                this.canonicalBaseUrl = finalUrlObj;
+                logger.debug(
+                  `Updated scope base after redirect: ${original.href} -> ${finalUrlObj.href}`
+                );
+              } else {
+                this.canonicalBaseUrl = original;
+              }
+            } catch {
+              this.canonicalBaseUrl = new URL$1(options.url);
+            }
+          }
           if (result.document) {
             this.pageCount++;
             logger.info(
@@ -4637,7 +5836,8 @@ class BaseScraperStrategy {
     this.pageCount = 0;
     this.totalDiscovered = 1;
     this.effectiveTotal = 1;
-    const baseUrl = new URL$1(options.url);
+    this.canonicalBaseUrl = new URL$1(options.url);
+    let baseUrl = this.canonicalBaseUrl;
     const queue = [{ url: options.url, depth: 0 }];
     this.visited.add(normalizeUrl(options.url, this.options.urlNormalizerOptions));
     const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
@@ -4658,6 +5858,7 @@ class BaseScraperStrategy {
         queue.length
       );
       const batch = queue.splice(0, batchSize);
+      baseUrl = this.canonicalBaseUrl ?? baseUrl;
       const newUrls = await this.processBatch(
         batch,
         baseUrl,
@@ -4690,22 +5891,7 @@ class WebScraperStrategy extends BaseScraperStrategy {
       return false;
     }
   }
-  /**
-   * Determines if a target URL should be followed based on the scope setting.
-   */
-  isInScope(baseUrl, targetUrl, scope) {
-    try {
-      if (scope === "domain") {
-        return hasSameDomain(baseUrl, targetUrl);
-      }
-      if (scope === "hostname") {
-        return hasSameHostname(baseUrl, targetUrl);
-      }
-      return hasSameHostname(baseUrl, targetUrl) && isSubpath(baseUrl, targetUrl);
-    } catch {
-      return false;
-    }
-  }
+  // Removed custom isInScope logic; using shared scope utility for consistent behavior
   /**
    * Processes a single queue item by fetching its content and processing it through pipelines.
    * @param item - The queue item to process.
@@ -4746,12 +5932,12 @@ class WebScraperStrategy extends BaseScraperStrategy {
         );
         return { document: void 0, links: processed.links };
       }
-      const baseUrl = new URL(options.url);
+      const baseUrl = item.depth === 0 ? new URL(rawContent.source) : this.canonicalBaseUrl ?? new URL(options.url);
       const filteredLinks = processed.links.filter((link) => {
         try {
           const targetUrl = new URL(link);
           const scope = options.scope || "subpages";
-          return this.isInScope(baseUrl, targetUrl, scope) && (!this.shouldFollowLinkFn || this.shouldFollowLinkFn(baseUrl, targetUrl));
+          return isInScope(baseUrl, targetUrl, scope) && (!this.shouldFollowLinkFn || this.shouldFollowLinkFn(baseUrl, targetUrl));
         } catch {
           return false;
         }
@@ -4767,7 +5953,8 @@ class WebScraperStrategy extends BaseScraperStrategy {
             ...processed.metadata
           }
         },
-        links: filteredLinks
+        links: filteredLinks,
+        finalUrl: rawContent.source
       };
     } catch (error) {
       logger.error(`❌ Failed processing page ${url}: ${error}`);
@@ -4849,9 +6036,9 @@ class LocalFileStrategy extends BaseScraperStrategy {
   }
   async processItem(item, options, _progressCallback, _signal) {
     const filePath = decodeURIComponent(item.url.replace(/^file:\/\//, ""));
-    const stats = await fs.stat(filePath);
+    const stats = await fs$1.stat(filePath);
     if (stats.isDirectory()) {
-      const contents = await fs.readdir(filePath);
+      const contents = await fs$1.readdir(filePath);
       const links = contents.map((name) => `file://${path.join(filePath, name)}`).filter((url) => this.shouldProcessUrl(url, options));
       return { links };
     }
@@ -5682,11 +6869,11 @@ async function createPipelineWithCallbacks(docService, options = {}) {
   pipeline.setCallbacks({
     onJobProgress: async (job, progress) => {
       logger.debug(
-        `📊 Job ${job.id} progress: ${progress.pagesScraped}/${progress.totalPages} pages`
+        `Job ${job.id} progress: ${progress.pagesScraped}/${progress.totalPages} pages`
       );
     },
     onJobStatusChange: async (job) => {
-      logger.debug(`🔄 Job ${job.id} status changed to: ${job.status}`);
+      logger.debug(`Job ${job.id} status changed to: ${job.status}`);
     },
     onJobError: async (job, error, document) => {
       logger.warn(
@@ -5703,7 +6890,9 @@ function createAppServerConfig(options) {
     enableApiServer: options.enableApiServer ?? false,
     enableWorker: options.enableWorker ?? true,
     port: options.port,
-    externalWorkerUrl: options.externalWorkerUrl
+    externalWorkerUrl: options.externalWorkerUrl,
+    readOnly: options.readOnly ?? false,
+    auth: options.auth
   };
 }
 function parseHeaders(headerOptions) {
@@ -5724,8 +6913,84 @@ const CLI_DEFAULTS = {
   PROTOCOL: DEFAULT_PROTOCOL,
   HTTP_PORT: DEFAULT_HTTP_PORT,
   WEB_PORT: DEFAULT_WEB_PORT,
-  MAX_CONCURRENCY: DEFAULT_MAX_CONCURRENCY
+  MAX_CONCURRENCY: DEFAULT_MAX_CONCURRENCY,
+  TELEMETRY: true
 };
+function parseAuthConfig(options) {
+  const enabled = options.authEnabled ?? (process.env.DOCS_MCP_AUTH_ENABLED?.toLowerCase() === "true" || false);
+  if (!enabled) {
+    return void 0;
+  }
+  const issuerUrl = options.authIssuerUrl ?? process.env.DOCS_MCP_AUTH_ISSUER_URL;
+  const audience = options.authAudience ?? process.env.DOCS_MCP_AUTH_AUDIENCE;
+  return {
+    enabled,
+    issuerUrl,
+    audience,
+    scopes: ["openid", "profile"]
+    // Default scopes for OAuth2/OIDC
+  };
+}
+function validateAuthConfig(authConfig) {
+  if (!authConfig.enabled) {
+    return;
+  }
+  const errors = [];
+  if (!authConfig.issuerUrl) {
+    errors.push("--auth-issuer-url is required when auth is enabled");
+  } else {
+    try {
+      const url = new URL(authConfig.issuerUrl);
+      if (url.protocol !== "https:") {
+        errors.push("Issuer URL must use HTTPS protocol");
+      }
+    } catch {
+      errors.push("Issuer URL must be a valid URL");
+    }
+  }
+  if (!authConfig.audience) {
+    errors.push("--auth-audience is required when auth is enabled");
+  } else {
+    try {
+      const url = new URL(authConfig.audience);
+      if (url.protocol === "http:" && url.hostname !== "localhost") {
+        logger.warn(
+          "⚠️  Audience uses HTTP protocol - consider using HTTPS for production"
+        );
+      }
+      if (url.hash) {
+        errors.push("Audience must not contain URL fragments");
+      }
+    } catch {
+      if (authConfig.audience.startsWith("urn:")) {
+        const urnParts = authConfig.audience.split(":");
+        if (urnParts.length < 3 || !urnParts[1] || !urnParts[2]) {
+          errors.push("URN audience must follow format: urn:namespace:specific-string");
+        }
+      } else {
+        errors.push(
+          "Audience must be a valid absolute URL or URN (e.g., https://api.example.com or urn:company:service)"
+        );
+      }
+    }
+  }
+  if (errors.length > 0) {
+    throw new Error(`Auth configuration validation failed:
+${errors.join("\n")}`);
+  }
+}
+function warnHttpUsage(authConfig, port) {
+  if (!authConfig?.enabled) {
+    return;
+  }
+  const isLocalhost = process.env.NODE_ENV !== "production" || port === 6280 || // default dev port
+  process.env.HOSTNAME?.includes("localhost");
+  if (!isLocalhost) {
+    logger.warn(
+      "⚠️  Authentication is enabled but running over HTTP in production. Consider using HTTPS for security."
+    );
+  }
+}
 function createDefaultAction(program) {
   return program.addOption(
     new Option("--protocol <protocol>", "Protocol for MCP server").choices(["auto", "stdio", "http"]).default("auto")
@@ -5737,13 +7002,33 @@ function createDefaultAction(program) {
       }
       return String(n);
     }).default(CLI_DEFAULTS.HTTP_PORT.toString())
-  ).option("--resume", "Resume interrupted jobs on startup", false).option("--no-resume", "Do not resume jobs on startup").action(
+  ).option("--resume", "Resume interrupted jobs on startup", false).option("--no-resume", "Do not resume jobs on startup").option(
+    "--read-only",
+    "Run in read-only mode (only expose read tools, disable write/job tools)",
+    false
+  ).option(
+    "--auth-enabled",
+    "Enable OAuth2/OIDC authentication for MCP endpoints",
+    false
+  ).option("--auth-issuer-url <url>", "Issuer/discovery URL for OAuth2/OIDC provider").option(
+    "--auth-audience <id>",
+    "JWT audience claim (identifies this protected resource)"
+  ).action(
     async (options, command) => {
       const globalOptions = command.opts();
       const resolvedProtocol = resolveProtocol(options.protocol);
       setupLogging(globalOptions, resolvedProtocol);
       logger.debug("No subcommand specified, starting unified server by default...");
       const port = validatePort(options.port);
+      const authConfig = parseAuthConfig({
+        authEnabled: options.authEnabled,
+        authIssuerUrl: options.authIssuerUrl,
+        authAudience: options.authAudience
+      });
+      if (authConfig) {
+        validateAuthConfig(authConfig);
+        warnHttpUsage(authConfig, port);
+      }
       ensurePlaywrightBrowsersInstalled();
       const docService = await createLocalDocumentManagement();
       const pipelineOptions = {
@@ -5753,14 +7038,14 @@ function createDefaultAction(program) {
       };
       const pipeline = await createPipelineWithCallbacks(docService, pipelineOptions);
       if (resolvedProtocol === "stdio") {
-        logger.debug(`🔍 Auto-detected stdio protocol (no TTY)`);
+        logger.debug(`Auto-detected stdio protocol (no TTY)`);
         await pipeline.start();
         const mcpTools = await initializeTools(docService, pipeline);
-        await startStdioServer(mcpTools);
+        await startStdioServer(mcpTools, options.readOnly);
         await new Promise(() => {
         });
       } else {
-        logger.debug(`🔍 Auto-detected http protocol (TTY available)`);
+        logger.debug(`Auto-detected http protocol (TTY available)`);
         const config = createAppServerConfig({
           enableWebInterface: true,
           // Enable web interface in http mode
@@ -5770,7 +7055,9 @@ function createDefaultAction(program) {
           // Enable API (tRPC) in http mode
           enableWorker: true,
           // Always enable in-process worker for unified server
-          port
+          port,
+          readOnly: options.readOnly,
+          auth: authConfig
         });
         await startAppServer(docService, pipeline, config);
         await new Promise(() => {
@@ -5784,12 +7071,24 @@ async function fetchUrlAction(url, options, command) {
   setupLogging(globalOptions);
   const headers = parseHeaders(options.header);
   const fetchUrlTool = new FetchUrlTool(new HttpFetcher(), new FileFetcher());
-  const content = await fetchUrlTool.execute({
-    url,
-    followRedirects: options.followRedirects,
-    scrapeMode: options.scrapeMode,
-    headers: Object.keys(headers).length > 0 ? headers : void 0
-  });
+  const content = await trackTool(
+    "fetch_url",
+    () => fetchUrlTool.execute({
+      url,
+      followRedirects: options.followRedirects,
+      scrapeMode: options.scrapeMode,
+      headers: Object.keys(headers).length > 0 ? headers : void 0
+    }),
+    (content2) => ({
+      url_protocol: extractProtocol(url),
+      // Safe: only protocol, not full URL
+      follow_redirects: options.followRedirects,
+      scrape_mode: options.scrapeMode,
+      has_custom_headers: Object.keys(headers).length > 0,
+      content_length: content2.length,
+      cli_flags: extractCliFlags(process.argv)
+    })
+  );
   console.log(content);
 }
 function createFetchUrlCommand(program) {
@@ -5824,10 +7123,22 @@ async function findVersionAction(library, options, command) {
   const docService = await createDocumentManagement({ serverUrl });
   try {
     const findVersionTool = new FindVersionTool(docService);
-    const versionInfo = await findVersionTool.execute({
-      library,
-      targetVersion: options.version
-    });
+    const versionInfo = await trackTool(
+      "find_version",
+      () => findVersionTool.execute({
+        library,
+        targetVersion: options.version
+      }),
+      (versionInfo2) => ({
+        library,
+        // Safe: library names are public
+        has_target_version: !!options.version,
+        result_type: typeof versionInfo2,
+        // 'string'
+        using_remote_server: !!serverUrl,
+        cli_flags: extractCliFlags(process.argv)
+      })
+    );
     if (!versionInfo) throw new Error("Failed to get version information");
     console.log(versionInfo);
   } finally {
@@ -5847,7 +7158,15 @@ async function listAction(options, command) {
   const docService = await createDocumentManagement({ serverUrl });
   try {
     const listLibrariesTool = new ListLibrariesTool(docService);
-    const result = await listLibrariesTool.execute();
+    const result = await trackTool(
+      "list_libraries",
+      () => listLibrariesTool.execute(),
+      (result2) => ({
+        library_count: result2.libraries.length,
+        using_remote_server: !!serverUrl,
+        cli_flags: extractCliFlags(process.argv)
+      })
+    );
     console.log(formatOutput(result.libraries));
   } finally {
     await docService.shutdown();
@@ -5873,6 +7192,17 @@ function createMcpCommand(program) {
   ).option(
     "--server-url <url>",
     "URL of external pipeline worker RPC (e.g., http://localhost:6280/api)"
+  ).option(
+    "--read-only",
+    "Run in read-only mode (only expose read tools, disable write/job tools)",
+    false
+  ).option(
+    "--auth-enabled",
+    "Enable OAuth2/OIDC authentication for MCP endpoints",
+    false
+  ).option("--auth-issuer-url <url>", "Issuer/discovery URL for OAuth2/OIDC provider").option(
+    "--auth-audience <id>",
+    "JWT audience claim (identifies this protected resource)"
   ).action(
     async (cmdOptions, command) => {
       const globalOptions = command.parent?.opts() || {};
@@ -5880,6 +7210,14 @@ function createMcpCommand(program) {
       const serverUrl = cmdOptions.serverUrl;
       const resolvedProtocol = resolveProtocol(cmdOptions.protocol);
       setupLogging(globalOptions, resolvedProtocol);
+      const authConfig = parseAuthConfig({
+        authEnabled: cmdOptions.authEnabled,
+        authIssuerUrl: cmdOptions.authIssuerUrl,
+        authAudience: cmdOptions.authAudience
+      });
+      if (authConfig) {
+        validateAuthConfig(authConfig);
+      }
       try {
         const docService = await createDocumentManagement({
           serverUrl
@@ -5895,15 +7233,15 @@ function createMcpCommand(program) {
           pipelineOptions
         );
         if (resolvedProtocol === "stdio") {
-          logger.debug(`🔍 Auto-detected stdio protocol (no TTY)`);
+          logger.debug(`Auto-detected stdio protocol (no TTY)`);
           logger.info("🚀 Starting MCP server (stdio mode)");
           await pipeline.start();
           const mcpTools = await initializeTools(docService, pipeline);
-          await startStdioServer(mcpTools);
+          await startStdioServer(mcpTools, cmdOptions.readOnly);
           await new Promise(() => {
           });
         } else {
-          logger.debug(`🔍 Auto-detected http protocol (TTY available)`);
+          logger.debug(`Auto-detected http protocol (TTY available)`);
           logger.info("🚀 Starting MCP server (http mode)");
           const config = createAppServerConfig({
             enableWebInterface: false,
@@ -5913,7 +7251,9 @@ function createMcpCommand(program) {
             // Never enable API in mcp command
             enableWorker: !serverUrl,
             port,
-            externalWorkerUrl: serverUrl
+            externalWorkerUrl: serverUrl,
+            readOnly: cmdOptions.readOnly,
+            auth: authConfig
           });
           await startAppServer(docService, pipeline, config);
           await new Promise(() => {
@@ -5933,13 +7273,21 @@ async function removeAction(library, options, command) {
   const docService = await createDocumentManagement({ serverUrl });
   const { version: version2 } = options;
   try {
-    await docService.removeAllDocuments(library, version2);
-    console.log(
-      `✅ Successfully removed documents for ${library}${version2 ? `@${version2}` : " (unversioned)"}.`
+    await trackTool(
+      "remove_documents",
+      () => docService.removeAllDocuments(library, version2),
+      () => ({
+        library,
+        // Safe: library names are public
+        has_version: !!version2,
+        using_remote_server: !!serverUrl,
+        cli_flags: extractCliFlags(process.argv)
+      })
     );
+    console.log(`✅ Successfully removed ${library}${version2 ? `@${version2}` : ""}.`);
   } catch (error) {
     console.error(
-      `❌ Failed to remove documents for ${library}${version2 ? `@${version2}` : " (unversioned)"}:`,
+      `❌ Failed to remove ${library}${version2 ? `@${version2}` : ""}:`,
       error instanceof Error ? error.message : String(error)
     );
     throw error;
@@ -5975,23 +7323,48 @@ async function scrapeAction(library, url, options, command) {
     await pipeline.start();
     const scrapeTool = new ScrapeTool(pipeline);
     const headers = parseHeaders(options.header);
-    const result = await scrapeTool.execute({
-      url,
-      library,
-      version: options.version,
-      options: {
-        maxPages: Number.parseInt(options.maxPages),
-        maxDepth: Number.parseInt(options.maxDepth),
-        maxConcurrency: Number.parseInt(options.maxConcurrency),
-        ignoreErrors: options.ignoreErrors,
+    const result = await trackTool(
+      "scrape_docs",
+      () => scrapeTool.execute({
+        url,
+        library,
+        version: options.version,
+        options: {
+          maxPages: Number.parseInt(options.maxPages, 10),
+          maxDepth: Number.parseInt(options.maxDepth, 10),
+          maxConcurrency: Number.parseInt(options.maxConcurrency, 10),
+          ignoreErrors: options.ignoreErrors,
+          scope: options.scope,
+          followRedirects: options.followRedirects,
+          scrapeMode: options.scrapeMode,
+          includePatterns: Array.isArray(options.includePattern) && options.includePattern.length > 0 ? options.includePattern : void 0,
+          excludePatterns: Array.isArray(options.excludePattern) && options.excludePattern.length > 0 ? options.excludePattern : void 0,
+          headers: Object.keys(headers).length > 0 ? headers : void 0
+        }
+      }),
+      (result2) => ({
+        library,
+        // Safe: library names are public
+        url_protocol: extractProtocol(url),
+        // Safe: only protocol, not full URL
+        max_pages: Number.parseInt(options.maxPages, 10),
+        max_depth: Number.parseInt(options.maxDepth, 10),
+        max_concurrency: Number.parseInt(options.maxConcurrency, 10),
+        has_version: !!options.version,
         scope: options.scope,
-        followRedirects: options.followRedirects,
-        scrapeMode: options.scrapeMode,
-        includePatterns: Array.isArray(options.includePattern) && options.includePattern.length > 0 ? options.includePattern : void 0,
-        excludePatterns: Array.isArray(options.excludePattern) && options.excludePattern.length > 0 ? options.excludePattern : void 0,
-        headers: Object.keys(headers).length > 0 ? headers : void 0
-      }
-    });
+        scrape_mode: options.scrapeMode,
+        ignore_errors: options.ignoreErrors,
+        follow_redirects: options.followRedirects,
+        has_include_patterns: Array.isArray(options.includePattern) && options.includePattern.length > 0,
+        has_exclude_patterns: Array.isArray(options.excludePattern) && options.excludePattern.length > 0,
+        has_custom_headers: Object.keys(headers).length > 0,
+        using_remote_server: !!serverUrl,
+        cli_flags: extractCliFlags(process.argv),
+        is_async_job: !("pagesScraped" in result2),
+        // Pipeline mode vs direct mode
+        pages_scraped: "pagesScraped" in result2 ? result2.pagesScraped : void 0
+      })
+    );
     if ("pagesScraped" in result) {
       console.log(`✅ Successfully scraped ${result.pagesScraped} pages`);
     } else {
@@ -6073,13 +7446,28 @@ async function searchAction(library, query, options, command) {
   const docService = await createDocumentManagement({ serverUrl });
   try {
     const searchTool = new SearchTool(docService);
-    const result = await searchTool.execute({
-      library,
-      version: options.version,
-      query,
-      limit: Number.parseInt(options.limit),
-      exactMatch: options.exactMatch
-    });
+    const result = await trackTool(
+      "search_docs",
+      () => searchTool.execute({
+        library,
+        version: options.version,
+        query,
+        limit: Number.parseInt(options.limit, 10),
+        exactMatch: options.exactMatch
+      }),
+      (result2) => ({
+        library,
+        // Safe: library names are public
+        query_analysis: analyzeSearchQuery(query),
+        // Analyzed, not raw query
+        result_count: result2.results.length,
+        limit_used: Number.parseInt(options.limit, 10),
+        has_version_filter: !!options.version,
+        exact_match: options.exactMatch,
+        using_remote_server: !!serverUrl,
+        cli_flags: extractCliFlags(process.argv)
+      })
+    );
     console.log(formatOutput(result.results));
   } finally {
     await docService.shutdown();
@@ -6192,11 +7580,28 @@ function createCliProgram() {
   const program = new Command();
   program.name("docs-mcp-server").description("Unified CLI, MCP Server, and Web Interface for Docs MCP Server.").version(packageJson.version).addOption(
     new Option("--verbose", "Enable verbose (debug) logging").conflicts("silent")
-  ).addOption(new Option("--silent", "Disable all logging except errors")).enablePositionalOptions().allowExcessArguments(false).showHelpAfterError(true);
-  program.hook("preAction", (thisCommand, _actionCommand) => {
+  ).addOption(new Option("--silent", "Disable all logging except errors")).addOption(new Option("--no-telemetry", "Disable telemetry collection")).enablePositionalOptions().allowExcessArguments(false).showHelpAfterError(true);
+  program.hook("preAction", (thisCommand, actionCommand) => {
     const globalOptions = thisCommand.opts();
     if (globalOptions.silent) setLogLevel(LogLevel.ERROR);
     else if (globalOptions.verbose) setLogLevel(LogLevel.DEBUG);
+    if (shouldEnableTelemetry()) {
+      const commandName = actionCommand.name();
+      const session = createCliSession(commandName, {
+        authEnabled: false,
+        // CLI doesn't use auth
+        readOnly: false
+      });
+      analytics.startSession(session);
+    } else {
+      TelemetryConfig.getInstance().disable();
+    }
+  });
+  program.hook("postAction", async () => {
+    if (analytics.isEnabled()) {
+      analytics.endSession();
+      await analytics.shutdown();
+    }
   });
   createMcpCommand(program);
   createWebCommand(program);
@@ -6244,6 +7649,10 @@ const sigintHandler = async () => {
       activeDocService = null;
       logger.debug("SIGINT: DocumentManagementService shut down.");
     }
+    if (analytics.isEnabled()) {
+      await analytics.shutdown();
+      logger.debug("SIGINT: Analytics shut down.");
+    }
     logger.info("✅ Graceful shutdown completed");
     process.exit(0);
   } catch (error) {
@@ -6319,6 +7728,7 @@ export {
   EMBEDDING_BATCH_CHARS as E,
   LibraryNotFoundError as L,
   StoreError as S,
+  TelemetryEvent as T,
   VECTOR_DIMENSION as V,
   applyMigrations as a,
   EMBEDDING_BATCH_SIZE as b,
@@ -6328,7 +7738,9 @@ export {
   SPLITTER_MAX_CHUNK_SIZE as f,
   getProjectRoot as g,
   VersionNotFoundError as h,
-  SPLITTER_MIN_CHUNK_SIZE as i,
+  analytics as i,
+  extractHostname as j,
+  SPLITTER_MIN_CHUNK_SIZE as k,
   logger as l,
   mapDbDocumentToDocument as m,
   normalizeVersionName as n