@duckmind/dm-darwin-x64 0.66.5 → 0.66.6

package/extensions/dm-web-access/duckduckgo.js

@@ -48,7 +48,7 @@ export async function searchWithDuckDuckGo(query, options = {}) {
       method: "GET",
       headers: {
         Accept: "text/html",
-        "User-Agent": "Mozilla/5.0 (compatible; dm-web-access/1.0; +https://github.com/nicobailon/dm-web-access)"
+        "User-Agent": "Mozilla/5.0 (compatible; dm-web-access/1.0; +https://github.com/DuckMind/extensions/tree/main/dm-web-access)"
       },
       signal: options.signal ? AbortSignal.any([AbortSignal.timeout(SEARCH_TIMEOUT_MS), options.signal]) : AbortSignal.timeout(SEARCH_TIMEOUT_MS)
     });
@@ -58,6 +58,9 @@ export async function searchWithDuckDuckGo(query, options = {}) {
     }
     const { parseHTML } = await import("linkedom");
     const { document } = parseHTML(await response.text());
+    if (response.status === 202 || document.querySelector(".anomaly-modal__box, #challenge-form")) {
+      throw new Error(`DuckDuckGo blocked the request with a bot check (HTTP ${response.status}, rate limited); use another search provider`);
+    }
     const filters = normalizeDomainFilters(options.domainFilter);
     const results = [];
     let parseableResults = 0;
@@ -78,8 +81,8 @@ export async function searchWithDuckDuckGo(query, options = {}) {
       if (results.length >= normalizeSearchResultCount(options.numResults))
         break;
     }
-    if (parseableResults === 0) {
-      throw new Error("DuckDuckGo returned no parseable results (invalid response)");
+    if (parseableResults === 0 && !document.querySelector(".no-results__message")) {
+      throw new Error("DuckDuckGo returned no parseable results (invalid response; the page format may have changed)");
     }
     activityMonitor.logComplete(activityId, response.status);
     const answer = results.map((result) => result.snippet ? `${result.snippet}

package/extensions/dm-web-access/endpoint-pdf-extract.js

@@ -0,0 +1,210 @@
+import { existsSync, readFileSync } from "node:fs";
+import { fetchRemoteUrl, loadSsrfConfig } from "./ssrf-protection.js";
+import { getWebSearchConfigPath, isLoopbackHostname } from "./utils.js";
+const DEFAULT_TIMEOUT_MS = 120000;
+const MAX_POLL_INTERVAL_MS = 3000;
+const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
+const MAX_ERROR_BODY_BYTES = 300;
+const CONFIG_PATH = getWebSearchConfigPath();
+function loadConfig() {
+  if (!existsSync(CONFIG_PATH))
+    return {};
+  try {
+    const parsed = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+    return parsed && typeof parsed === "object" ? parsed : {};
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+}
+function pdfSection() {
+  const pdf = loadConfig().pdf;
+  return pdf && typeof pdf === "object" ? pdf : {};
+}
+function normalizeString(value) {
+  if (typeof value !== "string")
+    return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+export function getEndpointUrl() {
+  const configured = normalizeString(process.env.PDF_ENDPOINT_URL) ?? normalizeString(pdfSection().endpointUrl);
+  if (!configured)
+    return null;
+  let url;
+  try {
+    url = new URL(configured);
+  } catch {
+    throw new Error(`Failed to parse pdf.endpointUrl: expected an absolute http(s) URL, got "${configured}"`);
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error(`Failed to parse pdf.endpointUrl: expected an http(s) URL, got "${configured}"`);
+  }
+  return configured.replace(/\/+$/, "");
+}
+export function getEndpointEngine() {
+  return normalizeString(pdfSection().endpointEngine);
+}
+export function getEndpointTimeoutMs() {
+  const configured = pdfSection().endpointTimeoutMs;
+  if (typeof configured === "number" && Number.isFinite(configured) && configured > 0) {
+    return Math.min(configured, 300000);
+  }
+  return DEFAULT_TIMEOUT_MS;
+}
+export function isEndpointApiAvailable() {
+  try {
+    return getEndpointUrl() !== null;
+  } catch {
+    return true;
+  }
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+async function readCapped(response, maxBytes, overflowThrows) {
+  const body = response.body;
+  if (!body)
+    return "";
+  const reader = body.getReader();
+  const decoder = new TextDecoder;
+  let text = "";
+  let received = 0;
+  for (;; ) {
+    const { done, value } = await reader.read();
+    if (done)
+      break;
+    const room = maxBytes - received;
+    received += value.byteLength;
+    if (received > maxBytes) {
+      await reader.cancel();
+      if (overflowThrows) {
+        throw new Error("PDF parser endpoint returned an oversized response");
+      }
+      return text + decoder.decode(value.subarray(0, room), { stream: true });
+    }
+    text += decoder.decode(value, { stream: true });
+    if (!overflowThrows && received === maxBytes) {
+      await reader.cancel();
+      return text + decoder.decode();
+    }
+  }
+  return text + decoder.decode();
+}
+async function readJson(response) {
+  const text = await readCapped(response, MAX_RESPONSE_BYTES, true);
+  try {
+    const parsed = JSON.parse(text);
+    if (parsed && typeof parsed === "object")
+      return parsed;
+  } catch {}
+  throw new Error("PDF parser endpoint returned a non-JSON response");
+}
+async function errorBody(response) {
+  try {
+    const text = (await readCapped(response, MAX_ERROR_BODY_BYTES, false)).slice(0, MAX_ERROR_BODY_BYTES);
+    return text ? `: ${text}` : "";
+  } catch {
+    return "";
+  }
+}
+export function stripNullCharacters(markdown) {
+  return markdown.includes("\x00") ? markdown.replace(/\x00/g, "") : markdown;
+}
+function markdownResult(payload) {
+  const markdown = typeof payload.markdown === "string" ? stripNullCharacters(payload.markdown) : "";
+  if (markdown.trim().length === 0) {
+    const error = normalizeString(payload.error);
+    throw new Error(error ? `PDF parser endpoint failed: ${error}` : "PDF parser endpoint returned no markdown");
+  }
+  const pages = typeof payload.pages === "number" ? payload.pages : undefined;
+  return { markdown, ...pages !== undefined ? { pages } : {} };
+}
+export async function extractPDFViaEndpoint(buffer, options) {
+  options.signal?.throwIfAborted();
+  const endpointUrl = getEndpointUrl();
+  if (!endpointUrl) {
+    throw new Error("PDF endpoint extraction requires a configured pdf.endpointUrl");
+  }
+  const ssrf = loadSsrfConfig();
+  const guard = {
+    allowRanges: ssrf.allowRanges,
+    trustEnvProxy: ssrf.trustEnvProxy,
+    allowLoopback: isLoopbackHostname(new URL(endpointUrl).hostname),
+    ...options.lookup ? { lookup: options.lookup } : {}
+  };
+  const engine = getEndpointEngine();
+  const form = new FormData;
+  form.append("file", new Blob([buffer], { type: "application/pdf" }), "document.pdf");
+  if (engine)
+    form.append("engine", engine);
+  form.append("max_pages", String(options.maxPages));
+  const deadline = Date.now() + (options.timeoutMs ?? getEndpointTimeoutMs());
+  const signal = options.signal;
+  const timeout = AbortSignal.timeout(Math.max(1, deadline - Date.now()));
+  const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
+  const response = await fetchRemoteUrl(endpointUrl, { method: "POST", body: form, signal: combined }, guard);
+  if (!response.ok) {
+    throw new Error(`PDF parser endpoint returned HTTP ${response.status}${await errorBody(response)}`);
+  }
+  let payload = await readJson(response);
+  if (typeof payload.markdown === "string" || !normalizeString(payload.job_id)) {
+    return markdownResult(payload);
+  }
+  const jobId = normalizeString(payload.job_id);
+  const jobsUrl = new URL(`jobs/${encodeURIComponent(jobId)}`, endpointUrl).toString();
+  let interval = 1000;
+  while (Date.now() < deadline) {
+    const remaining = deadline - Date.now();
+    if (remaining <= 0)
+      break;
+    const wait = Math.min(interval, Math.max(1, remaining), MAX_POLL_INTERVAL_MS);
+    interval = Math.min(interval * 2, MAX_POLL_INTERVAL_MS);
+    await new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        signal?.removeEventListener("abort", onAbort);
+        resolve();
+      }, wait);
+      const onAbort = () => {
+        clearTimeout(timer);
+        reject(new Error("PDF endpoint poll aborted"));
+      };
+      if (signal?.aborted) {
+        clearTimeout(timer);
+        reject(new Error("PDF endpoint poll aborted"));
+        return;
+      }
+      signal?.addEventListener("abort", onAbort, { once: true });
+    });
+    const poll = await fetchRemoteUrl(jobsUrl, { signal: combined }, guard);
+    if (!poll.ok) {
+      throw new Error(`PDF parser job status returned HTTP ${poll.status}${await errorBody(poll)}`);
+    }
+    payload = await readJson(poll);
+    const status = normalizeString(payload.status)?.toLowerCase();
+    if (status === "error" || status === "failed") {
+      const error = normalizeString(payload.error) ?? "unknown parser error";
+      throw new Error(`PDF parser endpoint job failed: ${error}`);
+    }
+    if (status === "done") {
+      const result = payload.result;
+      if (typeof result === "string") {
+        let parsed;
+        try {
+          parsed = JSON.parse(result);
+        } catch {
+          return { markdown: stripNullCharacters(result) };
+        }
+        if (parsed && typeof parsed === "object") {
+          return markdownResult(parsed);
+        }
+        throw new Error("PDF parser endpoint job completed without a result");
+      }
+      if (result && typeof result === "object") {
+        return markdownResult(result);
+      }
+      throw new Error("PDF parser endpoint job completed without a result");
+    }
+  }
+  throw new Error("PDF parser endpoint job timed out");
+}

package/extensions/dm-web-access/exa.js

@@ -80,6 +80,7 @@ function exaSearchArgs(query, options) {
     type: "auto",
     numResults: options.numResults ?? 5,
     ...mapDomainFilter(options.domainFilter),
+    ...options.category ? { category: options.category } : {},
     ...startDate ? { startPublishedDate: startDate } : {}
   };
 }
@@ -88,6 +89,14 @@ function normalizeHighlights(value) {
     return [];
   return value.filter((item) => typeof item === "string" && item.trim().length > 0);
 }
+function fallbackSourceLabel(url, index) {
+  try {
+    const hostname = url ? new URL(url).hostname : "";
+    if (hostname)
+      return hostname;
+  } catch {}
+  return `Source ${index + 1}`;
+}
 function buildAnswerFromSearchResults(results) {
   if (!results?.length)
     return "";
@@ -100,7 +109,7 @@ function buildAnswerFromSearchResults(results) {
     const content = highlights.length > 0 ? highlights.join(" ") : typeof item.text === "string" ? item.text.trim().slice(0, 1000) : "";
     if (!content)
       continue;
-    const sourceTitle = item.title || `Source ${i + 1}`;
+    const sourceTitle = item.title || fallbackSourceLabel(item.url, i);
     parts.push(`${content}
 Source: ${sourceTitle} (${item.url})`);
   }
@@ -117,7 +126,7 @@ function mapResults(results) {
     if (!item?.url)
       continue;
     mapped.push({
-      title: item.title || `Source ${i + 1}`,
+      title: item.title || fallbackSourceLabel(item.url, i),
       url: item.url,
       snippet: ""
     });
@@ -127,12 +136,7 @@ function mapResults(results) {
 function mapInlineContent(results) {
   if (!results?.length)
     return [];
-  return results.filter((r) => !!r?.url && typeof r.text === "string" && r.text.length > 0).map((r) => ({
-    url: r.url,
-    title: r.title || "",
-    content: r.text,
-    error: null
-  }));
+  return results.flatMap((r, i) => r?.url && typeof r.text === "string" && r.text.length > 0 ? [{ url: r.url, title: r.title || fallbackSourceLabel(r.url, i), content: r.text, error: null }] : []);
 }
 function toSearchResponse(answer, results, inlineContent) {
   const response = { answer, results };
@@ -238,7 +242,7 @@ function buildAnswerFromMcpResults(results) {
     const snippet = result.content.replace(/\s+/g, " ").trim().slice(0, 500);
     if (!snippet)
       continue;
-    const sourceTitle = result.title || `Source ${i + 1}`;
+    const sourceTitle = result.title || fallbackSourceLabel(result.url, i);
     parts.push(`${snippet}
 Source: ${sourceTitle} (${result.url})`);
   }
@@ -247,12 +251,7 @@ Source: ${sourceTitle} (${result.url})`);
 `);
 }
 function mapMcpInlineContent(results) {
-  return results.filter((result) => result.content.length > 0).map((result) => ({
-    url: result.url,
-    title: result.title,
-    content: result.content,
-    error: null
-  }));
+  return results.flatMap((result, i) => result.content.length > 0 ? [{ url: result.url, title: result.title || fallbackSourceLabel(result.url, i), content: result.content, error: null }] : []);
 }
 function buildMcpQuery(query, options) {
   const parts = [query];
@@ -278,6 +277,8 @@ function buildMcpQuery(query, options) {
         break;
     }
   }
+  if (options.category)
+    parts.push(options.category);
   return parts.join(" ");
 }
 function isAbortMessage(message) {
@@ -318,7 +319,7 @@ async function searchWithFilteredExaMcp(query, options, basicArgs) {
 async function searchWithExaMcp(query, options = {}) {
   const activityId = activityMonitor.logStart({ type: "api", query });
   const basicArgs = { query: buildMcpQuery(query, options), numResults: options.numResults ?? 5 };
-  const filtered = !!options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length;
+  const filtered = !!options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length || !!options.category;
   try {
     const response = filtered ? await searchWithFilteredExaMcp(query, options, basicArgs) : await searchWithExaMcpTool(EXA_MCP_BASIC_TOOL, basicArgs, options);
     activityMonitor.logComplete(activityId, 200);
@@ -349,27 +350,8 @@ export async function searchWithExa(query, options = {}) {
     return searchWithExaMcp(query, options);
   }
   const apiBaseUrl = getApiBaseUrl();
-  const useSearch = options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length || !!(options.numResults && options.numResults !== 5);
   const activityId = activityMonitor.logStart({ type: "api", query });
   try {
-    if (!useSearch) {
-      const response = await fetchWithCredentialRedirects(`${apiBaseUrl}/answer`, {
-        method: "POST",
-        headers: exaApiHeaders(apiKey),
-        body: JSON.stringify({ query }),
-        signal: requestSignal(options.signal)
-      }, ["x-api-key"]);
-      if (!response.ok) {
-        const errorText = redactCredential(await response.text(), apiKey);
-        throw new Error(`Exa API error ${response.status}: ${errorText.slice(0, 300)}`);
-      }
-      const data = await response.json();
-      activityMonitor.logComplete(activityId, response.status);
-      return {
-        answer: data.answer || "",
-        results: mapResults(data.citations)
-      };
-    }
     const response = await fetchWithCredentialRedirects(`${apiBaseUrl}/search`, {
       method: "POST",
       headers: exaApiHeaders(apiKey),

package/extensions/dm-web-access/extract.js

@@ -121,6 +121,9 @@ function isAbortException(err) {
 function isRedirectPolicyError(message) {
   return message.startsWith("Authenticated fetch refused cross-origin redirect") || message.startsWith("Blocked internal ") || message.startsWith("Blocked hostname by fetch_content domain policy") || message.startsWith("Hostname not allowed by fetch_content domain policy") || message.startsWith("Too many redirects fetching ") || message === "Only HTTP and HTTPS URLs can be fetched remotely" || message === "URL must include a hostname" || message.startsWith("Failed to resolve ");
 }
+function isNonRecoverableHttpError(error) {
+  return !!error && (NON_RECOVERABLE_ERRORS.some((prefix) => error.startsWith(prefix)) || isRedirectPolicyError(error) || isConfigParseError(error));
+}
 function imageGateError() {
   try {
     return isImageEnabled() ? null : "Image fetching is disabled by image.enabled";
@@ -145,6 +148,7 @@ async function fetchAuthenticatedRemoteUrl(url, init, validationOptions, profile
     allowRanges: validationOptions.ssrf.allowRanges,
     trustEnvProxy: validationOptions.ssrf.trustEnvProxy,
     domainPolicy: validationOptions.domainPolicy,
+    proxy: validationOptions.proxy,
     ...validationOptions.lookup ? { lookup: validationOptions.lookup } : {}
   });
   let requestInit = init;
@@ -164,6 +168,7 @@ async function fetchAuthenticatedRemoteUrl(url, init, validationOptions, profile
       allowRanges: validationOptions.ssrf.allowRanges,
       trustEnvProxy: validationOptions.ssrf.trustEnvProxy,
       domainPolicy: validationOptions.domainPolicy,
+      proxy: validationOptions.proxy,
       ...validationOptions.lookup ? { lookup: validationOptions.lookup } : {}
     });
     authFetchRedirectGuard(profile, from, current);
@@ -237,6 +242,16 @@ function notFoundGuidance(result, toolNames) {
   return lines.join(`
 `);
 }
+function jinaReaderGuidance(routing, providerOrder, needsRemoteOptIn) {
+  if (providerOrder.includes("jina"))
+    return null;
+  const privacy = "target URLs are fetched through Jina's infrastructure";
+  const optInCaveat = "this also allows the other hosted providers in your fetch provider order";
+  if (routing.providers.includes("jina")) {
+    return `  • Enable the keyless Jina Reader fallback: set fetchRouting.allowRemoteHostedProviders to true in ${WEB_SEARCH_CONFIG_PATH} (Jina is already in your fetch provider order; ${optInCaveat}; ${privacy})`;
+  }
+  return needsRemoteOptIn ? `  • Enable the keyless Jina Reader fallback: add "jina" to your existing fetchRouting.providers and set fetchRouting.allowRemoteHostedProviders to true in ${WEB_SEARCH_CONFIG_PATH} (${optInCaveat}; ${privacy})` : `  • Enable the keyless Jina Reader fallback: add "jina" to your existing fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} (${privacy})`;
+}
 function abortedResult(url) {
   return { url, title: "", content: "", error: "Aborted" };
 }
@@ -396,6 +411,7 @@ export async function extractContent(url, signal, options) {
         allowRanges: ssrf.allowRanges,
         trustEnvProxy: ssrf.trustEnvProxy,
         domainPolicy,
+        proxy: options?.proxy,
         ...options?.lookup ? { lookup: options.lookup } : {}
       });
     } catch (err) {
@@ -649,7 +665,7 @@ export async function extractContent(url, signal, options) {
       return abortedResult(url);
     if (!httpResult.error)
       return httpResult;
-    if (NON_RECOVERABLE_ERRORS.some((prefix) => httpResult.error.startsWith(prefix)) || isRedirectPolicyError(httpResult.error) || isConfigParseError(httpResult.error)) {
+    if (isNonRecoverableHttpError(httpResult.error) || httpResult.error === options?.rejectDirectImages) {
       return httpResult;
     }
     return null;
@@ -891,6 +907,7 @@ export async function extractContent(url, signal, options) {
     return { ...finalHttpResult, error: notFoundGuidance(finalHttpResult, options?.toolNames) };
   }
   const searchToolName = options?.toolNames?.webSearch;
+  const jinaHint = jinaReaderGuidance(fetchRouting, providerOrder, Boolean(remoteUrl) && !fetchRouting.allowRemoteHostedProviders);
   const guidance = [
     finalHttpResult?.error ?? "No fetch_content provider returned content",
     ...firecrawlError ? [`Firecrawl fallback failed: ${firecrawlError}`] : [],
@@ -905,6 +922,7 @@ export async function extractContent(url, signal, options) {
     ...brightdataError ? [`Bright Data fallback failed: ${brightdataError}`] : [],
     "",
     "Fallback options:",
+    ...jinaHint ? [jinaHint] : [],
     `  • Set firecrawlBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Firecrawl instance`,
     `  • Set crawl4aiBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Crawl4AI instance`,
     `  • Set tinyfishApiKey in ${WEB_SEARCH_CONFIG_PATH} or TINYFISH_API_KEY`,
@@ -921,6 +939,13 @@ export async function extractContent(url, signal, options) {
 `);
   return { ...finalHttpResult ?? { url, title: "", content: "", error: null }, error: guidance };
 }
+function isCloudflareChallenge(response, text, isHTML) {
+  if (response.status !== 200)
+    return false;
+  if (response.headers.get("cf-mitigated") === "challenge")
+    return true;
+  return isHTML && /<title>\s*Just a moment\.\.\.\s*<\/title>/i.test(text) && text.includes("window._cf_chl_opt") && text.includes("/cdn-cgi/challenge-platform/");
+}
 function isLikelyJSRendered(html) {
   const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
   if (!bodyMatch)
@@ -988,7 +1013,9 @@ function pdfSizeLimitError(maxSizeMB) {
 function responseSizeLimitError(maxBytes) {
   return new Error(`Response too large (${Math.round(maxBytes / 1024 / 1024)}MB)`);
 }
-async function extractViaHttp(url, timeoutMs, signal, options) {
+const BROWSER_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8";
+const MARKDOWN_FIRST_ACCEPT = "text/markdown,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,image/avif;q=0.9,image/webp;q=0.9,image/apng;q=0.9,*/*;q=0.7";
+async function extractViaHttp(url, timeoutMs, signal, options, preferMarkdown = options?.mode !== "raw") {
   const activityId = activityMonitor.logStart({ type: "fetch", url });
   const controller = new AbortController;
   const startedAt = Date.now();
@@ -999,13 +1026,12 @@ async function extractViaHttp(url, timeoutMs, signal, options) {
     const ssrf = loadSsrfConfig();
     const domainPolicy = loadFetchContentDomainPolicy();
     const authProfile = options?.authFetchProfile;
-    const trustEnvProxy = options?.proxy === undefined && ssrf.trustEnvProxy;
     const requestInit = {
       signal: controller.signal,
       __proxy: options?.proxy,
       headers: {
         "User-Agent": "OpenAI File Downloader, XaiImageApiFetch/1.0",
-        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+        Accept: preferMarkdown ? MARKDOWN_FIRST_ACCEPT : BROWSER_ACCEPT,
         "Accept-Language": "en-US,en;q=0.9",
         "Cache-Control": "no-cache",
         "Sec-Fetch-Dest": "document",
@@ -1015,10 +1041,11 @@ async function extractViaHttp(url, timeoutMs, signal, options) {
         "Upgrade-Insecure-Requests": "1"
       }
     };
-    const response = authProfile ? await fetchAuthenticatedRemoteUrl(url, requestInit, { ssrf: { ...ssrf, trustEnvProxy }, domainPolicy, ...options?.lookup ? { lookup: options.lookup } : {} }, authProfile) : await fetchRemoteUrl(url, requestInit, {
+    const response = authProfile ? await fetchAuthenticatedRemoteUrl(url, requestInit, { ssrf, domainPolicy, proxy: options?.proxy, ...options?.lookup ? { lookup: options.lookup } : {} }, authProfile) : await fetchRemoteUrl(url, requestInit, {
       allowRanges: ssrf.allowRanges,
-      trustEnvProxy,
+      trustEnvProxy: ssrf.trustEnvProxy,
       domainPolicy,
+      proxy: options?.proxy,
       ...options?.lookup ? { lookup: options.lookup } : {}
     });
     if (!response.ok && options?.mode !== "raw") {
@@ -1056,7 +1083,8 @@ async function extractViaHttp(url, timeoutMs, signal, options) {
     if (options?.mode === "raw") {
       if (!isTextContentType(contentType)) {
         activityMonitor.logComplete(activityId, response.status);
-        return { url, title: "", content: "", error: `Unsupported content type in raw mode: ${mimeType || "missing"}`, mimeType, status: response.status };
+        const hint = isPDFContent ? ' — raw mode does not decode PDFs; retry without mode (Markdown extraction) or with mode "answer" and a prompt' : "";
+        return { url, title: "", content: "", error: `Unsupported content type in raw mode: ${mimeType || "missing"}${hint}`, mimeType, status: response.status };
       }
       const text = await readTextResponseWithLimit(response, maxResponseSize);
       activityMonitor.logComplete(activityId, response.status);
@@ -1068,6 +1096,10 @@ async function extractViaHttp(url, timeoutMs, signal, options) {
         activityMonitor.logComplete(activityId, response.status);
         return { url, title: "", content: "", error: disabled, mimeType, status: response.status };
       }
+      if (options?.rejectDirectImages) {
+        activityMonitor.logComplete(activityId, response.status);
+        return { url, title: "", content: "", error: options.rejectDirectImages, mimeType, status: response.status };
+      }
       try {
         const buffer = await readResponseBufferWithLimit(response, maxResponseSize, () => responseSizeLimitError(maxResponseSize));
         const { resizeImage } = await import("@duckmind/dm-coding-agent");
@@ -1103,8 +1135,8 @@ async function extractViaHttp(url, timeoutMs, signal, options) {
           title: result.title,
           content: options?.mode === "answer" ? result.content : `PDF extracted and saved to: ${result.outputPath}
 
-Pages: ${result.pages}
-Characters: ${result.chars}`,
+${result.pages > 0 ? `Pages: ${result.pages}
+` : ""}Characters: ${result.chars}`,
           error: null
         };
       } catch (err) {
@@ -1130,10 +1162,31 @@ Characters: ${result.chars}`,
     }
     const text = await readTextResponseWithLimit(response, maxResponseSize);
     const isHTML = contentType.includes("text/html") || contentType.includes("application/xhtml+xml");
+    if (isCloudflareChallenge(response, text, isHTML)) {
+      activityMonitor.logComplete(activityId, response.status);
+      return { url, title: "", content: "", error: `HTTP ${response.status}: Blocked by Cloudflare challenge page`, status: response.status };
+    }
     if (!isHTML) {
       activityMonitor.logComplete(activityId, response.status);
       const title = extractTextTitle(text, url);
-      return { url, title, content: text, error: null };
+      if (mimeType !== "text/markdown" && mimeType !== "text/x-markdown")
+        return { url, title, content: text, error: null };
+      const declaredLinks = discoverDeclaredWebLinks(null, response.headers.get("link"), response.url || url);
+      const markdown = { url, title, content: appendDeclaredWebLinks(text, declaredLinks), error: null, declaredLinks };
+      if (!preferMarkdown || text.trim().length >= MIN_USEFUL_CONTENT)
+        return markdown;
+      const normal = await extractViaHttp(url, Math.max(1, timeoutMs - (Date.now() - startedAt)), signal, options, false);
+      if (signal?.aborted || isNonRecoverableHttpError(normal.error) || normal.status === 404 || normal.status === 410 || isAbortError(normal.error))
+        return normal;
+      const normalHasBody = normal.content.trim() !== appendDeclaredWebLinks("", normal.declaredLinks ?? []).trim();
+      if (normalHasBody || !markdown.content.trim())
+        return normal;
+      return {
+        ...normal,
+        ...markdown,
+        error: "Extracted content appears incomplete" + (normal.error ? `
+Browser retry failed: ${normal.error}` : "")
+      };
     }
     const { parseHTML } = await import("linkedom");
     const { document } = parseHTML(text);

package/extensions/dm-web-access/firecrawl.js

@@ -100,6 +100,29 @@ function allowFreshScrape() {
     throw new Error(`firecrawlFreshScrape in ${CONFIG_PATH} must be a boolean`);
   return configured;
 }
+function pdfParserOptions() {
+  const configured = loadConfig().firecrawlPdfMaxPages;
+  if (configured === undefined || configured === null)
+    return {};
+  if (typeof configured !== "number" || !Number.isInteger(configured) || configured < 1) {
+    throw new Error(`firecrawlPdfMaxPages in ${CONFIG_PATH} must be a positive integer`);
+  }
+  if (getApiVersion() === "v1") {
+    throw new Error(`firecrawlPdfMaxPages in ${CONFIG_PATH} requires Firecrawl API v2; v1 has no PDF page cap. ` + 'Use firecrawlApiVersion "v2" or remove firecrawlPdfMaxPages');
+  }
+  return { parsers: [{ type: "pdf", maxPages: configured }] };
+}
+function withPdfTruncationNotice(content, metadata) {
+  const parsed = metadata?.numPages;
+  const total = metadata?.totalPages;
+  if (typeof parsed !== "number" || typeof total !== "number" || parsed >= total)
+    return content;
+  return `${content}
+
+---
+
+*[Truncated: Only first ${parsed} of ${total} PDF pages parsed by Firecrawl]*`;
+}
 async function getApiKey(signal) {
   const configKey = loadConfig().firecrawlApiKey;
   return resolveCredential({
@@ -165,7 +188,8 @@ function scrapeBody(url) {
     url,
     formats: ["markdown"],
     onlyMainContent: true,
-    ...allowFreshScrape() ? {} : { lockdown: true }
+    ...allowFreshScrape() ? {} : { lockdown: true },
+    ...pdfParserOptions()
   };
 }
 function parseDomainFilter(domainFilter) {
@@ -209,6 +233,7 @@ function mapRecencyFilter(value) {
   }
 }
 function searchBody(query, options, numResults, filters) {
+  const parserOptions = pdfParserOptions();
   return {
     query,
     limit: numResults,
@@ -220,7 +245,8 @@ function searchBody(query, options, numResults, filters) {
       scrapeOptions: {
         formats: ["markdown"],
         onlyMainContent: true,
-        ...allowFreshScrape() ? {} : { lockdown: true }
+        ...allowFreshScrape() ? {} : { lockdown: true },
+        ...parserOptions
       }
     } : {}
   };
@@ -252,7 +278,7 @@ function mapSearchResults(data, numResults, filters) {
     results.push({ title, url, snippet });
     const markdown = firstString(item.markdown);
     if (markdown)
-      inlineContent.push({ url, title, content: markdown, error: null });
+      inlineContent.push({ url, title, content: withPdfTruncationNotice(markdown, item.metadata), error: null });
     if (results.length >= numResults)
       break;
   }
@@ -336,5 +362,5 @@ export async function extractWithFirecrawl(url, signal, options) {
     return null;
   const metadataTitle = scrape.metadata?.title;
   const title = typeof metadataTitle === "string" && metadataTitle.trim() ? metadataTitle.trim() : typeof scrape.title === "string" ? scrape.title.trim() : "";
-  return { url, title, content, error: null };
+  return { url, title, content: withPdfTruncationNotice(content, scrape.metadata), error: null };
 }