@arabold/docs-mcp-server 1.16.1 → 1.17.0

package/dist/index.js

@@ -42,7 +42,7 @@ import Fastify from "fastify";
 import { jsxs, jsx, Fragment } from "@kitajs/html/jsx-runtime";
 import DOMPurify from "dompurify";
 const name = "@arabold/docs-mcp-server";
-const version = "1.16.0";
+const version = "1.16.1";
 const description = "MCP server for fetching and searching documentation";
 const type = "module";
 const bin = { "docs-mcp-server": "dist/index.js" };
@@ -179,6 +179,41 @@ class CancelJobTool {
     }
   }
 }
+class ClearCompletedJobsTool {
+  manager;
+  /**
+   * Creates an instance of ClearCompletedJobsTool.
+   * @param manager The PipelineManager instance.
+   */
+  constructor(manager) {
+    this.manager = manager;
+  }
+  /**
+   * Executes the tool to clear all completed jobs from the pipeline.
+   * @param input - The input parameters (currently unused).
+   * @returns A promise that resolves with the outcome of the clear operation.
+   */
+  async execute(input) {
+    try {
+      const clearedCount = await this.manager.clearCompletedJobs();
+      const message = clearedCount > 0 ? `Successfully cleared ${clearedCount} completed job${clearedCount === 1 ? "" : "s"} from the queue.` : "No completed jobs to clear.";
+      logger.debug(`[ClearCompletedJobsTool] ${message}`);
+      return {
+        message,
+        success: true,
+        clearedCount
+      };
+    } catch (error) {
+      const errorMessage = `Failed to clear completed jobs: ${error instanceof Error ? error.message : String(error)}`;
+      logger.error(`❌ [ClearCompletedJobsTool] ${errorMessage}`);
+      return {
+        message: errorMessage,
+        success: false,
+        clearedCount: 0
+      };
+    }
+  }
+}
 class ToolError extends Error {
   constructor(message, toolName) {
     super(message);
@@ -473,6 +508,7 @@ class HtmlPlaywrightMiddleware {
    * - Parses credentials from the URL (if present).
    * - Uses browser.newContext({ httpCredentials }) for HTTP Basic Auth on the main page and subresources.
    * - Injects Authorization header for all same-origin requests if credentials are present and not already set.
+   * - Forwards all custom headers from context.options?.headers to Playwright requests.
    * - Waits for common loading indicators to disappear before extracting HTML.
    *
    * @param context The middleware context containing the HTML and source URL.
@@ -494,20 +530,8 @@ class HtmlPlaywrightMiddleware {
     let page = null;
     let browserContext = null;
     let renderedHtml = null;
-    let credentials = null;
-    let origin = null;
-    try {
-      const url = new URL(context.source);
-      origin = url.origin;
-      if (url.username && url.password) {
-        credentials = { username: url.username, password: url.password };
-        logger.debug(
-          `Playwright: Detected credentials for ${origin} (username: ${url.username})`
-        );
-      }
-    } catch (e) {
-      logger.warn(`⚠️  Could not parse URL for credential extraction: ${context.source}`);
-    }
+    const { credentials, origin } = extractCredentialsAndOrigin(context.source);
+    const customHeaders = context.options?.headers ?? {};
     try {
       const browser = await this.ensureBrowser();
       if (credentials) {
@@ -537,17 +561,14 @@ class HtmlPlaywrightMiddleware {
         if (["image", "stylesheet", "font", "media"].includes(resourceType)) {
           return route.abort();
         }
-        if (credentials && origin && reqOrigin === origin && !route.request().headers().authorization) {
-          const basic = Buffer.from(
-            `${credentials.username}:${credentials.password}`
-          ).toString("base64");
-          const headers = {
-            ...route.request().headers(),
-            Authorization: `Basic ${basic}`
-          };
-          return route.continue({ headers });
-        }
-        return route.continue();
+        const headers = mergePlaywrightHeaders(
+          route.request().headers(),
+          customHeaders,
+          credentials ?? void 0,
+          origin ?? void 0,
+          reqOrigin ?? void 0
+        );
+        return route.continue({ headers });
       });
       await page.goto(context.source, { waitUntil: "load" });
       await page.waitForSelector("body");
@@ -581,6 +602,38 @@ class HtmlPlaywrightMiddleware {
     await next();
   }
 }
+function extractCredentialsAndOrigin(urlString) {
+  try {
+    const url = new URL(urlString);
+    const origin = url.origin;
+    if (url.username && url.password) {
+      return {
+        credentials: { username: url.username, password: url.password },
+        origin
+      };
+    }
+    return { credentials: null, origin };
+  } catch {
+    return { credentials: null, origin: null };
+  }
+}
+function mergePlaywrightHeaders(requestHeaders, customHeaders, credentials, origin, reqOrigin) {
+  let headers = { ...requestHeaders };
+  for (const [key, value] of Object.entries(customHeaders)) {
+    if (key.toLowerCase() === "authorization" && headers.authorization) continue;
+    headers[key] = value;
+  }
+  if (credentials && origin && reqOrigin === origin && !headers.authorization) {
+    const basic = Buffer.from(`${credentials.username}:${credentials.password}`).toString(
+      "base64"
+    );
+    headers = {
+      ...headers,
+      Authorization: `Basic ${basic}`
+    };
+  }
+  return headers;
+}
 class HtmlSanitizerMiddleware {
   // Default selectors to remove
   defaultSelectorsToRemove = [
@@ -1001,7 +1054,7 @@ class FetchUrlTool {
    * @throws {ToolError} If fetching or processing fails
    */
   async execute(options) {
-    const { url, scrapeMode = ScrapeMode.Auto } = options;
+    const { url, scrapeMode = ScrapeMode.Auto, headers } = options;
     const canFetchResults = this.fetchers.map((f) => f.canFetch(url));
     const fetcherIndex = canFetchResults.findIndex((result) => result === true);
     if (fetcherIndex === -1) {
@@ -1018,7 +1071,9 @@ class FetchUrlTool {
       logger.info(`📡 Fetching ${url}...`);
       const rawContent = await fetcher.fetch(url, {
         followRedirects: options.followRedirects ?? true,
-        maxRetries: 3
+        maxRetries: 3,
+        headers
+        // propagate custom headers
       });
       logger.info("🔄 Processing content...");
       let processed;
@@ -1037,7 +1092,9 @@ class FetchUrlTool {
               followRedirects: options.followRedirects ?? true,
               excludeSelectors: void 0,
               ignoreErrors: false,
-              scrapeMode
+              scrapeMode,
+              headers
+              // propagate custom headers
             },
             fetcher
           );
@@ -1298,7 +1355,9 @@ class ScrapeTool {
       scrapeMode: scraperOptions?.scrapeMode ?? ScrapeMode.Auto,
       // Pass scrapeMode enum
       includePatterns: scraperOptions?.includePatterns,
-      excludePatterns: scraperOptions?.excludePatterns
+      excludePatterns: scraperOptions?.excludePatterns,
+      headers: scraperOptions?.headers
+      // <-- propagate headers
     });
     if (waitForCompletion) {
       try {
@@ -1906,6 +1965,24 @@ async function startStdioServer(tools) {
   logger.info("🤖 MCP server listening on stdio");
   return server;
 }
+class PipelineError extends Error {
+  constructor(message, cause) {
+    super(message);
+    this.cause = cause;
+    this.name = this.constructor.name;
+    if (cause?.stack) {
+      this.stack = `${this.stack}
+Caused by: ${cause.stack}`;
+    }
+  }
+}
+class PipelineStateError extends PipelineError {
+}
+class CancellationError extends PipelineError {
+  constructor(message = "Operation cancelled") {
+    super(message);
+  }
+}
 class FingerprintGenerator {
   headerGenerator;
   /**
@@ -1997,6 +2074,9 @@ class HttpFetcher {
         const axiosError = error;
         const status = axiosError.response?.status;
         const code = axiosError.code;
+        if (options?.signal?.aborted || code === "ERR_CANCELED") {
+          throw new CancellationError("HTTP fetch cancelled");
+        }
         if (!followRedirects && status && status >= 300 && status < 400) {
           const location = axiosError.response?.headers?.location;
           if (location) {
@@ -2064,6 +2144,7 @@ async function initializeTools(docService, pipelineManager) {
     listJobs: new ListJobsTool(pipelineManager),
     getJobInfo: new GetJobInfoTool(pipelineManager),
     cancelJob: new CancelJobTool(pipelineManager),
+    // clearCompletedJobs: new ClearCompletedJobsTool(pipelineManager),
     remove: new RemoveTool(docService, pipelineManager),
     fetchUrl: new FetchUrlTool(new HttpFetcher(), new FileFetcher())
   };
@@ -2172,24 +2253,6 @@ function isSubpath(baseUrl, targetUrl) {
   const basePath = baseUrl.pathname.endsWith("/") ? baseUrl.pathname : `${baseUrl.pathname}/`;
   return targetUrl.pathname.startsWith(basePath);
 }
-class PipelineError extends Error {
-  constructor(message, cause) {
-    super(message);
-    this.cause = cause;
-    this.name = this.constructor.name;
-    if (cause?.stack) {
-      this.stack = `${this.stack}
-Caused by: ${cause.stack}`;
-    }
-  }
-}
-class PipelineStateError extends PipelineError {
-}
-class CancellationError extends PipelineError {
-  constructor(message = "Operation cancelled") {
-    super(message);
-  }
-}
 function isRegexPattern(pattern) {
   return pattern.length > 2 && pattern.startsWith("/") && pattern.endsWith("/");
 }
@@ -2416,12 +2479,22 @@ class WebScraperStrategy extends BaseScraperStrategy {
       return false;
     }
   }
+  /**
+   * Processes a single queue item by fetching its content and processing it through pipelines.
+   * @param item - The queue item to process.
+   * @param options - Scraper options including headers for HTTP requests.
+   * @param _progressCallback - Optional progress callback (not used here).
+   * @param signal - Optional abort signal for request cancellation.
+   * @returns An object containing the processed document and extracted links.
+   */
   async processItem(item, options, _progressCallback, signal) {
     const { url } = item;
     try {
       const fetchOptions = {
         signal,
-        followRedirects: options.followRedirects
+        followRedirects: options.followRedirects,
+        headers: options.headers
+        // Forward custom headers
       };
       const rawContent = await this.httpFetcher.fetch(url, fetchOptions);
       let processed;
@@ -2725,7 +2798,7 @@ class PipelineWorker {
         // Pass signal to scraper service
       );
       if (signal.aborted) {
-        throw new CancellationError("Job cancelled shortly after scraping finished");
+        throw new CancellationError("Job cancelled");
       }
       logger.debug(`[${jobId}] Worker finished job successfully.`);
     } catch (error) {
@@ -2862,13 +2935,21 @@ class PipelineManager {
   }
   /**
    * Returns a promise that resolves when the specified job completes, fails, or is cancelled.
+   * For cancelled jobs, this resolves successfully rather than rejecting.
    */
   async waitForJobCompletion(jobId) {
     const job = this.jobMap.get(jobId);
     if (!job) {
       throw new PipelineStateError(`Job not found: ${jobId}`);
     }
-    await job.completionPromise;
+    try {
+      await job.completionPromise;
+    } catch (error) {
+      if (error instanceof CancellationError || job.status === PipelineJobStatus.CANCELLED) {
+        return;
+      }
+      throw error;
+    }
   }
   /**
    * Attempts to cancel a queued or running job.
@@ -2907,6 +2988,35 @@ class PipelineManager {
         break;
     }
   }
+  /**
+   * Removes all jobs that are in a final state (completed, cancelled, or failed).
+   * Only removes jobs that are not currently in the queue or actively running.
+   * @returns The number of jobs that were cleared.
+   */
+  async clearCompletedJobs() {
+    const completedStatuses = [
+      PipelineJobStatus.COMPLETED,
+      PipelineJobStatus.CANCELLED,
+      PipelineJobStatus.FAILED
+    ];
+    let clearedCount = 0;
+    const jobsToRemove = [];
+    for (const [jobId, job] of this.jobMap.entries()) {
+      if (completedStatuses.includes(job.status)) {
+        jobsToRemove.push(jobId);
+        clearedCount++;
+      }
+    }
+    for (const jobId of jobsToRemove) {
+      this.jobMap.delete(jobId);
+    }
+    if (clearedCount > 0) {
+      logger.info(`🧹 Cleared ${clearedCount} completed job(s) from the queue`);
+    } else {
+      logger.debug("No completed jobs to clear");
+    }
+    return clearedCount;
+  }
   // --- Private Methods ---
   /**
    * Processes the job queue, starting new workers if capacity allows.
@@ -2961,10 +3071,10 @@ class PipelineManager {
       if (error instanceof CancellationError || signal.aborted) {
         job.status = PipelineJobStatus.CANCELLED;
         job.finishedAt = /* @__PURE__ */ new Date();
-        job.error = error instanceof CancellationError ? error : new CancellationError("Job cancelled by signal");
-        logger.info(`🚫 Job execution cancelled: ${jobId}: ${job.error.message}`);
+        const cancellationError = error instanceof CancellationError ? error : new CancellationError("Job cancelled by signal");
+        logger.info(`🚫 Job execution cancelled: ${jobId}: ${cancellationError.message}`);
         await this.callbacks.onJobStatusChange?.(job);
-        job.rejectCompletion(job.error);
+        job.rejectCompletion(cancellationError);
       } else {
         job.status = PipelineJobStatus.FAILED;
         job.error = error instanceof Error ? error : new Error(String(error));
@@ -4676,8 +4786,23 @@ function registerIndexRoute(server) {
     reply.type("text/html");
     return "<!DOCTYPE html>" + /* @__PURE__ */ jsxs(Layout, { title: "MCP Docs", children: [
       /* @__PURE__ */ jsxs("section", { class: "mb-4 p-4 bg-white rounded-lg shadow dark:bg-gray-800 border border-gray-300 dark:border-gray-600", children: [
-        /* @__PURE__ */ jsx("h2", { class: "text-xl font-semibold mb-2 text-gray-900 dark:text-white", children: "Job Queue" }),
-        /* @__PURE__ */ jsx("div", { id: "jobQueue", "hx-get": "/api/jobs", "hx-trigger": "load, every 1s", children: /* @__PURE__ */ jsxs("div", { class: "animate-pulse", children: [
+        /* @__PURE__ */ jsxs("div", { class: "flex items-center justify-between mb-2", children: [
+          /* @__PURE__ */ jsx("h2", { class: "text-xl font-semibold text-gray-900 dark:text-white", children: "Job Queue" }),
+          /* @__PURE__ */ jsx(
+            "button",
+            {
+              type: "button",
+              class: "text-xs px-3 py-1.5 text-gray-700 bg-gray-100 border border-gray-300 rounded-lg hover:bg-gray-200 focus:ring-4 focus:outline-none focus:ring-gray-100 dark:bg-gray-600 dark:text-gray-300 dark:border-gray-500 dark:hover:bg-gray-700 dark:focus:ring-gray-700 transition-colors duration-150",
+              title: "Clear all completed, cancelled, and failed jobs",
+              "hx-post": "/api/jobs/clear-completed",
+              "hx-trigger": "click",
+              "hx-on": "htmx:afterRequest: document.dispatchEvent(new Event('job-list-refresh'))",
+              "hx-swap": "none",
+              children: "Clear Completed Jobs"
+            }
+          )
+        ] }),
+        /* @__PURE__ */ jsx("div", { id: "job-queue", "hx-get": "/api/jobs", "hx-trigger": "load, every 1s", children: /* @__PURE__ */ jsxs("div", { class: "animate-pulse", children: [
           /* @__PURE__ */ jsx("div", { class: "h-[0.8em] bg-gray-200 rounded-full dark:bg-gray-700 w-48 mb-4" }),
           /* @__PURE__ */ jsx("div", { class: "h-[0.8em] bg-gray-200 rounded-full dark:bg-gray-700 w-full mb-2.5" }),
           /* @__PURE__ */ jsx("div", { class: "h-[0.8em] bg-gray-200 rounded-full dark:bg-gray-700 w-full mb-2.5" })
@@ -4693,7 +4818,7 @@ function registerIndexRoute(server) {
         /* @__PURE__ */ jsx(
           "div",
           {
-            id: "indexedDocs",
+            id: "indexed-docs",
             "hx-get": "/api/libraries",
             "hx-trigger": "load, every 10s",
             children: /* @__PURE__ */ jsxs("div", { class: "animate-pulse", children: [
@@ -4707,12 +4832,75 @@ function registerIndexRoute(server) {
     ] });
   });
 }
+function registerCancelJobRoute(server, cancelJobTool) {
+  server.post(
+    "/api/jobs/:jobId/cancel",
+    async (request, reply) => {
+      const { jobId } = request.params;
+      const result = await cancelJobTool.execute({ jobId });
+      if (result.success) {
+        return { success: true, message: result.message };
+      } else {
+        reply.status(400);
+        return { success: false, message: result.message };
+      }
+    }
+  );
+}
+function registerClearCompletedJobsRoute(server, clearCompletedJobsTool) {
+  server.post("/api/jobs/clear-completed", async (_, reply) => {
+    try {
+      const result = await clearCompletedJobsTool.execute({});
+      reply.type("application/json");
+      return {
+        success: result.success,
+        message: result.message
+      };
+    } catch (error) {
+      reply.code(500);
+      return {
+        success: false,
+        message: `Internal server error: ${error instanceof Error ? error.message : String(error)}`
+      };
+    }
+  });
+}
 const VersionBadge = ({ version: version2 }) => {
   if (!version2) {
     return null;
   }
   return /* @__PURE__ */ jsx("span", { class: "bg-purple-100 text-purple-800 text-xs font-medium me-2 px-1.5 py-0.5 rounded dark:bg-purple-900 dark:text-purple-300", children: /* @__PURE__ */ jsx("span", { safe: true, children: version2 }) });
 };
+const LoadingSpinner = () => /* @__PURE__ */ jsxs(
+  "svg",
+  {
+    class: "animate-spin h-4 w-4 text-white",
+    xmlns: "http://www.w3.org/2000/svg",
+    fill: "none",
+    viewBox: "0 0 24 24",
+    children: [
+      /* @__PURE__ */ jsx(
+        "circle",
+        {
+          class: "opacity-25",
+          cx: "12",
+          cy: "12",
+          r: "10",
+          stroke: "currentColor",
+          "stroke-width": "4"
+        }
+      ),
+      /* @__PURE__ */ jsx(
+        "path",
+        {
+          class: "opacity-75",
+          fill: "currentColor",
+          d: "M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
+        }
+      )
+    ]
+  }
+);
 const JobItem = ({ job }) => (
   // Use Flowbite Card structure with reduced padding and added border
   /* @__PURE__ */ jsx("div", { class: "block p-2 bg-gray-50 dark:bg-gray-700 rounded-lg border border-gray-200 dark:border-gray-600", children: /* @__PURE__ */ jsxs("div", { class: "flex items-center justify-between", children: [
@@ -4728,19 +4916,99 @@ const JobItem = ({ job }) => (
       ] })
     ] }),
     /* @__PURE__ */ jsxs("div", { class: "flex flex-col items-end gap-1", children: [
-      /* @__PURE__ */ jsx(
-        "span",
-        {
-          class: `px-1.5 py-0.5 text-xs font-medium me-2 rounded ${job.status === PipelineJobStatus.COMPLETED ? "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300" : job.error ? "bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-300" : "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-300"}`,
-          children: job.status
-        }
-      ),
+      /* @__PURE__ */ jsxs("div", { class: "flex items-center gap-2", children: [
+        /* @__PURE__ */ jsx(
+          "span",
+          {
+            class: `px-1.5 py-0.5 text-xs font-medium rounded ${job.status === PipelineJobStatus.COMPLETED ? "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300" : job.error ? "bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-300" : "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-300"}`,
+            children: job.status
+          }
+        ),
+        (job.status === PipelineJobStatus.QUEUED || job.status === PipelineJobStatus.RUNNING) && /* @__PURE__ */ jsxs(
+          "button",
+          {
+            type: "button",
+            class: "font-medium rounded-lg text-xs p-1 text-center inline-flex items-center transition-colors duration-150 ease-in-out border border-gray-300 bg-white text-red-600 hover:bg-red-50 focus:ring-4 focus:outline-none focus:ring-red-100 dark:border-gray-600 dark:bg-gray-800 dark:text-red-400 dark:hover:bg-gray-700 dark:focus:ring-red-900",
+            title: "Stop this job",
+            "x-data": "{}",
+            "x-on:click": `
+                if ($store.confirmingAction.type === 'job-cancel' && $store.confirmingAction.id === '${job.id}') {
+                  $store.confirmingAction.isStopping = true;
+                  fetch('/api/jobs/' + '${job.id}' + '/cancel', {
+                    method: 'POST',
+                    headers: { 'Accept': 'application/json' },
+                  })
+                    .then(r => r.json())
+                    .then(() => {
+                      $store.confirmingAction.type = null;
+                      $store.confirmingAction.id = null;
+                      $store.confirmingAction.isStopping = false;
+                      if ($store.confirmingAction.timeoutId) { clearTimeout($store.confirmingAction.timeoutId); $store.confirmingAction.timeoutId = null; }
+                      document.dispatchEvent(new CustomEvent('job-list-refresh'));
+                    })
+                    .catch(() => { $store.confirmingAction.isStopping = false; });
+                } else {
+                  if ($store.confirmingAction.timeoutId) { clearTimeout($store.confirmingAction.timeoutId); $store.confirmingAction.timeoutId = null; }
+                  $store.confirmingAction.type = 'job-cancel';
+                  $store.confirmingAction.id = '${job.id}';
+                  $store.confirmingAction.isStopping = false;
+                  $store.confirmingAction.timeoutId = setTimeout(() => {
+                    $store.confirmingAction.type = null;
+                    $store.confirmingAction.id = null;
+                    $store.confirmingAction.isStopping = false;
+                    $store.confirmingAction.timeoutId = null;
+                  }, 3000);
+                }
+              `,
+            "x-bind:disabled": `$store.confirmingAction.type === 'job-cancel' && $store.confirmingAction.id === '${job.id}' && $store.confirmingAction.isStopping`,
+            children: [
+              /* @__PURE__ */ jsxs(
+                "span",
+                {
+                  "x-show": `$store.confirmingAction.type !== 'job-cancel' || $store.confirmingAction.id !== '${job.id}' || $store.confirmingAction.isStopping`,
+                  children: [
+                    /* @__PURE__ */ jsx(
+                      "svg",
+                      {
+                        class: "w-4 h-4",
+                        "aria-hidden": "true",
+                        fill: "currentColor",
+                        viewBox: "0 0 20 20",
+                        children: /* @__PURE__ */ jsx("rect", { x: "5", y: "5", width: "10", height: "10", rx: "2" })
+                      }
+                    ),
+                    /* @__PURE__ */ jsx("span", { class: "sr-only", children: "Stop job" })
+                  ]
+                }
+              ),
+              /* @__PURE__ */ jsx(
+                "span",
+                {
+                  "x-show": `$store.confirmingAction.type === 'job-cancel' && $store.confirmingAction.id === '${job.id}' && !$store.confirmingAction.isStopping`,
+                  class: "px-2",
+                  children: "Cancel?"
+                }
+              ),
+              /* @__PURE__ */ jsxs(
+                "span",
+                {
+                  "x-show": `$store.confirmingAction.type === 'job-cancel' && $store.confirmingAction.id === '${job.id}' && $store.confirmingAction.isStopping`,
+                  children: [
+                    /* @__PURE__ */ jsx(LoadingSpinner, {}),
+                    /* @__PURE__ */ jsx("span", { class: "sr-only", children: "Stopping..." })
+                  ]
+                }
+              )
+            ]
+          }
+        )
+      ] }),
       job.error && // Keep the error badge for clarity if an error occurred
-      /* @__PURE__ */ jsx("span", { class: "bg-red-100 text-red-800 text-xs font-medium me-2 px-1.5 py-0.5 rounded dark:bg-red-900 dark:text-red-300", children: "Error" })
+      /* @__PURE__ */ jsx("span", { class: "bg-red-100 text-red-800 text-xs font-medium px-1.5 py-0.5 rounded dark:bg-red-900 dark:text-red-300", children: "Error" })
     ] })
   ] }) })
 );
-const JobList = ({ jobs }) => /* @__PURE__ */ jsx("div", { class: "space-y-2", children: jobs.length === 0 ? /* @__PURE__ */ jsx("p", { class: "text-center text-gray-500 dark:text-gray-400", children: "No pending jobs." }) : jobs.map((job) => /* @__PURE__ */ jsx(JobItem, { job })) });
+const JobList = ({ jobs }) => /* @__PURE__ */ jsx("div", { id: "job-list", class: "space-y-2", children: jobs.length === 0 ? /* @__PURE__ */ jsx("p", { class: "text-center text-gray-500 dark:text-gray-400", children: "No pending jobs." }) : jobs.map((job) => /* @__PURE__ */ jsx(JobItem, { job })) });
 function registerJobListRoutes(server, listJobsTool) {
   server.get("/api/jobs", async () => {
     const result = await listJobsTool.execute({});
@@ -4899,7 +5167,7 @@ const ScrapeFormContent = () => /* @__PURE__ */ jsxs("div", { class: "mt-4 p-4 b
       "hx-target": "#job-response",
       "hx-swap": "innerHTML",
       class: "space-y-2",
-      "x-data": "{\n        url: '',\n        hasPath: false,\n        checkUrlPath() {\n          try {\n            const url = new URL(this.url);\n            this.hasPath = url.pathname !== '/' && url.pathname !== '';\n          } catch (e) {\n            this.hasPath = false;\n          }\n        }\n      }",
+      "x-data": "{\n        url: '',\n        hasPath: false,\n        headers: [],\n        checkUrlPath() {\n          try {\n            const url = new URL(this.url);\n            this.hasPath = url.pathname !== '/' && url.pathname !== '';\n          } catch (e) {\n            this.hasPath = false;\n          }\n        }\n      }",
       children: [
         /* @__PURE__ */ jsxs("div", { children: [
           /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
@@ -5010,7 +5278,7 @@ const ScrapeFormContent = () => /* @__PURE__ */ jsxs("div", { class: "mt-4 p-4 b
         ] }),
         /* @__PURE__ */ jsxs("details", { class: "bg-gray-50 dark:bg-gray-900 p-2 rounded-md", children: [
           /* @__PURE__ */ jsx("summary", { class: "cursor-pointer text-sm font-medium text-gray-600 dark:text-gray-400", children: "Advanced Options" }),
-          /* @__PURE__ */ jsxs("div", { class: "mt-2 space-y-2", children: [
+          /* @__PURE__ */ jsxs("div", { class: "mt-2 space-y-2", "x-data": "{ headers: [] }", children: [
             /* @__PURE__ */ jsxs("div", { children: [
               /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
                 /* @__PURE__ */ jsx(
@@ -5178,6 +5446,63 @@ const ScrapeFormContent = () => /* @__PURE__ */ jsxs("div", { class: "mt-4 p-4 b
                 }
               )
             ] }),
+            /* @__PURE__ */ jsxs("div", { children: [
+              /* @__PURE__ */ jsxs("div", { class: "flex items-center mb-1", children: [
+                /* @__PURE__ */ jsx("label", { class: "block text-sm font-medium text-gray-700 dark:text-gray-300", children: "Custom HTTP Headers" }),
+                /* @__PURE__ */ jsx(Tooltip, { text: "Add custom HTTP headers (e.g., for authentication). These will be sent with every HTTP request." })
+              ] }),
+              /* @__PURE__ */ jsxs("div", { children: [
+                /* @__PURE__ */ jsx("template", { "x-for": "(header, idx) in headers", children: /* @__PURE__ */ jsxs("div", { class: "flex space-x-2 mb-1", children: [
+                  /* @__PURE__ */ jsx(
+                    "input",
+                    {
+                      type: "text",
+                      class: "w-1/3 px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-xs",
+                      placeholder: "Header Name",
+                      "x-model": "header.name",
+                      required: true
+                    }
+                  ),
+                  /* @__PURE__ */ jsx("span", { class: "text-gray-500", children: ":" }),
+                  /* @__PURE__ */ jsx(
+                    "input",
+                    {
+                      type: "text",
+                      class: "w-1/2 px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-xs",
+                      placeholder: "Header Value",
+                      "x-model": "header.value",
+                      required: true
+                    }
+                  ),
+                  /* @__PURE__ */ jsx(
+                    "button",
+                    {
+                      type: "button",
+                      class: "text-red-500 hover:text-red-700 text-xs",
+                      "x-on:click": "headers.splice(idx, 1)",
+                      children: "Remove"
+                    }
+                  ),
+                  /* @__PURE__ */ jsx(
+                    "input",
+                    {
+                      type: "hidden",
+                      name: "header[]",
+                      "x-bind:value": "header.name && header.value ? header.name + ':' + header.value : ''"
+                    }
+                  )
+                ] }) }),
+                /* @__PURE__ */ jsx(
+                  "button",
+                  {
+                    type: "button",
+                    class: "mt-1 px-2 py-0.5 bg-indigo-100 dark:bg-indigo-900 text-indigo-700 dark:text-indigo-200 rounded text-xs",
+                    "x-on:click": "headers.push({ name: '', value: '' })",
+                    children: "+ Add Header"
+                  }
+                )
+              ] })
+            ] }),
             /* @__PURE__ */ jsxs("div", { class: "flex items-center", children: [
               /* @__PURE__ */ jsx(
                 "input",
@@ -5247,6 +5572,19 @@ function registerNewJobRoutes(server, scrapeTool) {
         let parsePatterns = function(input) {
           if (!input) return void 0;
           return input.split(/\n|,/).map((s) => s.trim()).filter((s) => s.length > 0);
+        }, parseHeaders = function(input) {
+          if (!input) return void 0;
+          const arr = Array.isArray(input) ? input : [input];
+          const headers = {};
+          for (const entry of arr) {
+            const idx = entry.indexOf(":");
+            if (idx > 0) {
+              const name2 = entry.slice(0, idx).trim();
+              const value = entry.slice(idx + 1).trim();
+              if (name2) headers[name2] = value;
+            }
+          }
+          return Object.keys(headers).length > 0 ? headers : void 0;
         };
         if (!body.url || !body.library) {
           reply.status(400);
@@ -5275,7 +5613,9 @@ function registerNewJobRoutes(server, scrapeTool) {
             followRedirects: body.followRedirects === "on",
             ignoreErrors: body.ignoreErrors === "on",
             includePatterns: parsePatterns(body.includePatterns),
-            excludePatterns: parsePatterns(body.excludePatterns)
+            excludePatterns: parsePatterns(body.excludePatterns),
+            headers: parseHeaders(body["header[]"])
+            // <-- propagate custom headers from web UI
           }
         };
         const result = await scrapeTool.execute(scrapeOptions);
@@ -5314,36 +5654,6 @@ function registerNewJobRoutes(server, scrapeTool) {
     }
   );
 }
-const LoadingSpinner = () => /* @__PURE__ */ jsxs(
-  "svg",
-  {
-    class: "animate-spin h-4 w-4 text-white",
-    xmlns: "http://www.w3.org/2000/svg",
-    fill: "none",
-    viewBox: "0 0 24 24",
-    children: [
-      /* @__PURE__ */ jsx(
-        "circle",
-        {
-          class: "opacity-25",
-          cx: "12",
-          cy: "12",
-          r: "10",
-          stroke: "currentColor",
-          "stroke-width": "4"
-        }
-      ),
-      /* @__PURE__ */ jsx(
-        "path",
-        {
-          class: "opacity-75",
-          fill: "currentColor",
-          d: "M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
-        }
-      )
-    ]
-  }
-);
 const VersionDetailsRow = ({
   version: version2,
   libraryName,
@@ -5352,7 +5662,7 @@ const VersionDetailsRow = ({
 }) => {
   const indexedDate = version2.indexedAt ? new Date(version2.indexedAt).toLocaleDateString() : "N/A";
   const versionLabel = version2.version || "Unversioned";
-  const versionParam = version2.version || "unversioned";
+  const versionParam = version2.version || "";
   const sanitizedLibraryName = libraryName.replace(/[^a-zA-Z0-9-_]/g, "-");
   const sanitizedVersionParam = versionParam.replace(/[^a-zA-Z0-9-_]/g, "-");
   const rowId = `row-${sanitizedLibraryName}-${sanitizedVersionParam}`;
@@ -5397,43 +5707,80 @@ const VersionDetailsRow = ({
               type: "button",
               class: "ml-2 font-medium rounded-lg text-sm p-1 text-center inline-flex items-center transition-colors duration-150 ease-in-out",
               title: "Remove this version",
-              "x-data": "{ confirming: false, isDeleting: false, timeoutId: null }",
-              "x-bind:class": `confirming ? "${confirmingStateClasses}" : "${defaultStateClasses}"`,
-              "x-bind:disabled": "isDeleting",
-              "x-on:click": "\n            if (confirming) {\n              clearTimeout(timeoutId);\n              timeoutId = null;\n              isDeleting = true; // Set deleting state directly\n              // Dispatch a standard browser event instead of calling htmx directly\n              $el.dispatchEvent(new CustomEvent('confirmed-delete', { bubbles: true }));\n            } else {\n              confirming = true;\n              timeoutId = setTimeout(() => { confirming = false; timeoutId = null; }, 3000);\n            }\n          ",
+              "x-data": "{}",
+              "x-bind:class": `$store.confirmingAction.type === 'version-delete' && $store.confirmingAction.id === '${libraryName}:${versionParam}' ? '${confirmingStateClasses}' : '${defaultStateClasses}'`,
+              "x-bind:disabled": `$store.confirmingAction.type === 'version-delete' && $store.confirmingAction.id === '${libraryName}:${versionParam}' && $store.confirmingAction.isDeleting`,
+              "x-on:click": `
+            if ($store.confirmingAction.type === 'version-delete' && $store.confirmingAction.id === '${libraryName}:${versionParam}') {
+              $store.confirmingAction.isDeleting = true;
+              $el.dispatchEvent(new CustomEvent('confirmed-delete', { bubbles: true }));
+            } else {
+              if ($store.confirmingAction.timeoutId) { clearTimeout($store.confirmingAction.timeoutId); $store.confirmingAction.timeoutId = null; }
+              $store.confirmingAction.type = 'version-delete';
+              $store.confirmingAction.id = '${libraryName}:${versionParam}';
+              $store.confirmingAction.isDeleting = false;
+              $store.confirmingAction.timeoutId = setTimeout(() => {
+                $store.confirmingAction.type = null;
+                $store.confirmingAction.id = null;
+                $store.confirmingAction.isDeleting = false;
+                $store.confirmingAction.timeoutId = null;
+              }, 3000);
+            }
+          `,
               "hx-delete": `/api/libraries/${encodeURIComponent(libraryName)}/versions/${encodeURIComponent(versionParam)}`,
               "hx-target": `#${rowId}`,
               "hx-swap": "outerHTML",
               "hx-trigger": "confirmed-delete",
               children: [
-                /* @__PURE__ */ jsxs("span", { "x-show": "!confirming && !isDeleting", children: [
-                  /* @__PURE__ */ jsx(
-                    "svg",
-                    {
-                      class: "w-4 h-4",
-                      "aria-hidden": "true",
-                      xmlns: "http://www.w3.org/2000/svg",
-                      fill: "none",
-                      viewBox: "0 0 18 20",
-                      children: /* @__PURE__ */ jsx(
-                        "path",
+                /* @__PURE__ */ jsxs(
+                  "span",
+                  {
+                    "x-show": `!($store.confirmingAction.type === 'version-delete' && $store.confirmingAction.id === '${libraryName}:${versionParam}' && $store.confirmingAction.isDeleting)`,
+                    children: [
+                      /* @__PURE__ */ jsx(
+                        "svg",
                         {
-                          stroke: "currentColor",
-                          "stroke-linecap": "round",
-                          "stroke-linejoin": "round",
-                          "stroke-width": "2",
-                          d: "M1 5h16M7 8v8m4-8v8M7 1h4a1 1 0 0 1 1 1v3H6V2a1 1 0 0 1 1-1ZM3 5h12v13a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V5Z"
+                          class: "w-4 h-4",
+                          "aria-hidden": "true",
+                          xmlns: "http://www.w3.org/2000/svg",
+                          fill: "none",
+                          viewBox: "0 0 18 20",
+                          children: /* @__PURE__ */ jsx(
+                            "path",
+                            {
+                              stroke: "currentColor",
+                              "stroke-linecap": "round",
+                              "stroke-linejoin": "round",
+                              "stroke-width": "2",
+                              d: "M1 5h16M7 8v8m4-8v8M7 1h4a1 1 0 0 1 1 1v3H6V2a1 1 0 0 1-1-1ZM3 5h12v13a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V5Z"
+                            }
+                          )
                         }
-                      )
-                    }
-                  ),
-                  /* @__PURE__ */ jsx("span", { class: "sr-only", children: "Remove version" })
-                ] }),
-                /* @__PURE__ */ jsx("span", { "x-show": "confirming && !isDeleting", children: "Confirm?" }),
-                /* @__PURE__ */ jsxs("span", { "x-show": "isDeleting", children: [
-                  /* @__PURE__ */ jsx(LoadingSpinner, {}),
-                  /* @__PURE__ */ jsx("span", { class: "sr-only", children: "Loading..." })
-                ] })
+                      ),
+                      /* @__PURE__ */ jsx("span", { class: "sr-only", children: "Remove version" })
+                    ]
+                  }
+                ),
+                /* @__PURE__ */ jsxs(
+                  "span",
+                  {
+                    "x-show": `$store.confirmingAction.type === 'version-delete' && $store.confirmingAction.id === '${libraryName}:${versionParam}' && !$store.confirmingAction.isDeleting`,
+                    children: [
+                      "Confirm?",
+                      /* @__PURE__ */ jsx("span", { class: "sr-only", children: "Confirm delete" })
+                    ]
+                  }
+                ),
+                /* @__PURE__ */ jsxs(
+                  "span",
+                  {
+                    "x-show": `$store.confirmingAction.type === 'version-delete' && $store.confirmingAction.id === '${libraryName}:${versionParam}' && $store.confirmingAction.isDeleting`,
+                    children: [
+                      /* @__PURE__ */ jsx(LoadingSpinner, {}),
+                      /* @__PURE__ */ jsx("span", { class: "sr-only", children: "Loading..." })
+                    ]
+                  }
+                )
               ]
             }
           )
@@ -5446,14 +5793,7 @@ const LibraryDetailCard = ({ library }) => (
   // Use Flowbite Card structure with updated padding and border, and white background
   /* @__PURE__ */ jsxs("div", { class: "block p-4 bg-white dark:bg-gray-800 rounded-lg shadow-sm border border-gray-300 dark:border-gray-600 mb-4", children: [
     /* @__PURE__ */ jsx("h3", { class: "text-lg font-medium text-gray-900 dark:text-white mb-1", children: /* @__PURE__ */ jsx("span", { safe: true, children: library.name }) }),
-    /* @__PURE__ */ jsx("div", { class: "mt-1", children: library.versions.length > 0 ? library.versions.sort((a, b) => {
-      if (!a.version) return -1;
-      if (!b.version) return 1;
-      return semver__default.compare(
-        semver__default.coerce(b.version)?.version ?? "0.0.0",
-        semver__default.coerce(a.version)?.version ?? "0.0.0"
-      );
-    }).map((version2) => /* @__PURE__ */ jsx(
+    /* @__PURE__ */ jsx("div", { class: "mt-1", children: library.versions.length > 0 ? library.versions.map((version2) => /* @__PURE__ */ jsx(
       VersionDetailsRow,
       {
         libraryName: library.name,
@@ -5467,14 +5807,6 @@ const LibraryDetailCard = ({ library }) => (
   ] })
 );
 const LibrarySearchCard = ({ library }) => {
-  const sortedVersions = library.versions.sort((a, b) => {
-    if (!a.version) return -1;
-    if (!b.version) return 1;
-    return semver__default.compare(
-      semver__default.coerce(b.version)?.version ?? "0.0.0",
-      semver__default.coerce(a.version)?.version ?? "0.0.0"
-    );
-  });
   return /* @__PURE__ */ jsxs("div", { class: "block p-4 bg-white dark:bg-gray-800 rounded-lg shadow-sm border border-gray-300 dark:border-gray-600 mb-4", children: [
     /* @__PURE__ */ jsxs("h2", { class: "text-xl font-semibold mb-2 text-gray-900 dark:text-white", safe: true, children: [
       "Search ",
@@ -5498,7 +5830,7 @@ const LibrarySearchCard = ({ library }) => {
               children: [
                 /* @__PURE__ */ jsx("option", { value: "", children: "Latest" }),
                 " ",
-                sortedVersions.map((version2) => /* @__PURE__ */ jsx("option", { value: version2.version || "unversioned", safe: true, children: version2.version || "Unversioned" }))
+                library.versions.map((version2) => /* @__PURE__ */ jsx("option", { value: version2.version || "unversioned", safe: true, children: version2.version || "Unversioned" }))
               ]
             }
           ),
@@ -5636,14 +5968,7 @@ const LibraryItem = ({ library }) => (
         children: /* @__PURE__ */ jsx("span", { safe: true, children: library.name })
       }
     ) }),
-    /* @__PURE__ */ jsx("div", { class: "mt-1", children: library.versions.length > 0 ? library.versions.sort((a, b) => {
-      if (!a.version) return -1;
-      if (!b.version) return 1;
-      return semver__default.compare(
-        semver__default.coerce(b.version)?.version ?? "0.0.0",
-        semver__default.coerce(a.version)?.version ?? "0.0.0"
-      );
-    }).map((version2) => /* @__PURE__ */ jsx(VersionDetailsRow, { libraryName: library.name, version: version2 })) : (
+    /* @__PURE__ */ jsx("div", { class: "mt-1", children: library.versions.length > 0 ? library.versions.map((version2) => /* @__PURE__ */ jsx(VersionDetailsRow, { libraryName: library.name, version: version2 })) : (
       // Display message if no versions are indexed
       /* @__PURE__ */ jsx("p", { class: "text-sm text-gray-500 dark:text-gray-400 italic", children: "No versions indexed." })
     ) })
@@ -5690,8 +6015,10 @@ async function startWebServer(port, docService, pipelineManager) {
   const listLibrariesTool = new ListLibrariesTool(docService);
   const listJobsTool = new ListJobsTool(pipelineManager);
   const scrapeTool = new ScrapeTool(docService, pipelineManager);
-  const removeTool = new RemoveTool(docService);
+  const removeTool = new RemoveTool(docService, pipelineManager);
   const searchTool = new SearchTool(docService);
+  const cancelJobTool = new CancelJobTool(pipelineManager);
+  const clearCompletedJobsTool = new ClearCompletedJobsTool(pipelineManager);
   await server.register(fastifyStatic, {
     // Use project root to construct absolute path to public directory
     root: path.join(getProjectRoot(), "public"),
@@ -5702,6 +6029,8 @@ async function startWebServer(port, docService, pipelineManager) {
   registerIndexRoute(server);
   registerJobListRoutes(server, listJobsTool);
   registerNewJobRoutes(server, scrapeTool);
+  registerCancelJobRoute(server, cancelJobTool);
+  registerClearCompletedJobsRoute(server, clearCompletedJobsTool);
   registerLibrariesRoutes(server, listLibrariesTool, removeTool);
   registerLibraryDetailRoutes(server, listLibrariesTool, searchTool);
   try {
@@ -5825,7 +6154,7 @@ async function main() {
   }
   try {
     const program = new Command();
-    program.name("docs-mcp-server").description("Unified CLI, MCP Server, and Web Interface for Docs MCP Server.").version(packageJson.version).option("--verbose", "Enable verbose (debug) logging", false).option("--silent", "Disable all logging except errors", false).enablePositionalOptions().option(
+    program.name("docs-mcp-server").description("Unified CLI, MCP Server, and Web Interface for Docs MCP Server.").version(packageJson.version).option("--verbose", "Enable verbose (debug) logging", false).option("--silent", "Disable all logging except errors", false).enablePositionalOptions().showHelpAfterError(true).option(
       "--protocol <type>",
       "Protocol for MCP server (stdio or http)",
       DEFAULT_PROTOCOL
@@ -5910,6 +6239,11 @@ async function main() {
       "Glob or regex pattern for URLs to exclude (can be specified multiple times, takes precedence over include). Regex patterns must be wrapped in slashes, e.g. /pattern/.",
       (val, prev = []) => prev.concat([val]),
       []
+    ).option(
+      "--header <name:value>",
+      "Custom HTTP header to send with each request (can be specified multiple times)",
+      (val, prev = []) => prev.concat([val]),
+      []
     ).action(async (library, url, options) => {
       commandExecuted = true;
       const docService = new DocumentManagementService();
@@ -5919,6 +6253,17 @@ async function main() {
         pipelineManager = new PipelineManager(docService);
         await pipelineManager.start();
         const scrapeTool = new ScrapeTool(docService, pipelineManager);
+        const headers = {};
+        if (Array.isArray(options.header)) {
+          for (const entry of options.header) {
+            const idx = entry.indexOf(":");
+            if (idx > 0) {
+              const name2 = entry.slice(0, idx).trim();
+              const value = entry.slice(idx + 1).trim();
+              if (name2) headers[name2] = value;
+            }
+          }
+        }
         const result = await scrapeTool.execute({
           url,
           library,
@@ -5932,7 +6277,8 @@ async function main() {
             followRedirects: options.followRedirects,
             scrapeMode: options.scrapeMode,
             includePatterns: Array.isArray(options.includePattern) && options.includePattern.length > 0 ? options.includePattern : void 0,
-            excludePatterns: Array.isArray(options.excludePattern) && options.excludePattern.length > 0 ? options.excludePattern : void 0
+            excludePatterns: Array.isArray(options.excludePattern) && options.excludePattern.length > 0 ? options.excludePattern : void 0,
+            headers: Object.keys(headers).length > 0 ? headers : void 0
           }
         });
         if ("pagesScraped" in result)
@@ -6034,13 +6380,30 @@ async function main() {
         return value;
       },
       ScrapeMode.Auto
+    ).option(
+      "--header <name:value>",
+      "Custom HTTP header to send with the request (can be specified multiple times)",
+      (val, prev = []) => prev.concat([val]),
+      []
     ).action(async (url, options) => {
       commandExecuted = true;
+      const headers = {};
+      if (Array.isArray(options.header)) {
+        for (const entry of options.header) {
+          const idx = entry.indexOf(":");
+          if (idx > 0) {
+            const name2 = entry.slice(0, idx).trim();
+            const value = entry.slice(idx + 1).trim();
+            if (name2) headers[name2] = value;
+          }
+        }
+      }
       const fetchUrlTool = new FetchUrlTool(new HttpFetcher(), new FileFetcher());
       const content = await fetchUrlTool.execute({
         url,
         followRedirects: options.followRedirects,
-        scrapeMode: options.scrapeMode
+        scrapeMode: options.scrapeMode,
+        headers: Object.keys(headers).length > 0 ? headers : void 0
       });
       console.log(content);
     });