@bacnh85/pi-web 0.5.2 → 0.5.4

package/extensions/lib/brave.ts

@@ -4,38 +4,38 @@ import { sanitizeSnippet, type SearchResultItem } from "./format";
 import { withRetry } from "./retry";
 
 export interface BraveResult extends SearchResultItem {
-	age: string;
+  age: string;
 }
 
 export async function fetchBraveResults(
-	query: string,
-	count: number,
-	country: string,
-	freshness: string,
-	apiKey: string,
-	signal?: AbortSignal,
+  query: string,
+  count: number,
+  country: string,
+  freshness: string,
+  apiKey: string,
+  signal?: AbortSignal,
 ): Promise<BraveResult[]> {
-	return withRetry(async () => {
-		if (signal?.aborted) throw signal.reason ?? new Error("Operation aborted.");
-		const params = new URLSearchParams({ q: query, count: String(count), country });
-		if (freshness) params.append("freshness", freshness);
-		const response = await fetch(`https://api.search.brave.com/res/v1/web/search?${params}`, {
-			headers: {
-				Accept: "application/json",
-				"Accept-Encoding": "gzip",
-				"X-Subscription-Token": apiKey,
-			},
-			signal,
-		});
-		if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}\n${await response.text()}`);
-		const data: any = await response.json();
-		return (data.web?.results || [])
-			.slice(0, count)
-			.map((r: any) => ({
-				title: sanitizeSnippet(r.title || ""),
-				url: r.url || "",
-				snippet: sanitizeSnippet(r.description || ""),
-				age: sanitizeSnippet(r.age || r.page_age || ""),
-			}));
-	}, undefined, signal);
+  return withRetry(async () => {
+    if (signal?.aborted) throw signal.reason ?? new Error("Operation aborted.");
+    const params = new URLSearchParams({ q: query, count: String(count), country });
+    if (freshness) params.append("freshness", freshness);
+    const response = await fetch(`https://api.search.brave.com/res/v1/web/search?${params}`, {
+      headers: {
+        Accept: "application/json",
+        "Accept-Encoding": "gzip",
+        "X-Subscription-Token": apiKey,
+      },
+      signal,
+    });
+    if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}\n${await response.text()}`);
+    const data: any = await response.json();
+    return (data.web?.results || [])
+      .slice(0, count)
+      .map((r: any) => ({
+        title: sanitizeSnippet(r.title || ""),
+        url: r.url || "",
+        snippet: sanitizeSnippet(r.description || ""),
+        age: sanitizeSnippet(r.age || r.page_age || ""),
+      }));
+  }, undefined, signal);
 }

package/extensions/lib/config.ts

@@ -9,70 +9,70 @@ import path from "node:path";
 // ---------------------------------------------------------------------------
 
 export const HOSTED_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev/v2";
-export const DEFAULT_SEARXNG_BASE_URL = "http://172.30.55.22:8888";
-export const DEFAULT_CRAWL4AI_API_URL = "http://172.30.55.22:11235";
+export const DEFAULT_SEARXNG_BASE_URL = "http://127.0.0.1:8888";
+export const DEFAULT_CRAWL4AI_API_URL = "http://127.0.0.1:11235";
 
 // ---------------------------------------------------------------------------
 // Environment loading
 // ---------------------------------------------------------------------------
 
 export function piConfigDirs(): string[] {
-	return process.env.PI_CODING_AGENT_DIR
-		? [process.env.PI_CODING_AGENT_DIR]
-		: [path.join(os.homedir(), ".pi", "agent")];
+  return process.env.PI_CODING_AGENT_DIR
+    ? [process.env.PI_CODING_AGENT_DIR]
+    : [path.join(os.homedir(), ".pi", "agent")];
 }
 
 export function envFileCandidates(cwd = process.cwd(), includeCwd = true): string[] {
-	return [
-		...(includeCwd ? [path.resolve(cwd, ".env.local"), path.resolve(cwd, ".env")] : []),
-		...piConfigDirs().flatMap((dir) => [path.join(dir, ".env.local"), path.join(dir, ".env")]),
-	];
+  return [
+    ...(includeCwd ? [path.resolve(cwd, ".env.local"), path.resolve(cwd, ".env")] : []),
+    ...piConfigDirs().flatMap((dir) => [path.join(dir, ".env.local"), path.join(dir, ".env")]),
+  ];
 }
 
 export function stripInlineComment(value: string): string {
-	let quote = "";
-	let escaped = false;
-	for (let i = 0; i < value.length; i++) {
-		const char = value[i];
-		if (escaped) { escaped = false; continue; }
-		if (char === "\\") { escaped = true; continue; }
-		if (quote) { if (char === quote) quote = ""; continue; }
-		if (char === '"' || char === "'") { quote = char; continue; }
-		if (char === "#" && (i === 0 || /\s/.test(value[i - 1]))) return value.slice(0, i);
-	}
-	return value;
+  let quote = "";
+  let escaped = false;
+  for (let i = 0; i < value.length; i++) {
+    const char = value[i];
+    if (escaped) { escaped = false; continue; }
+    if (char === "\\") { escaped = true; continue; }
+    if (quote) { if (char === quote) quote = ""; continue; }
+    if (char === '"' || char === "'") { quote = char; continue; }
+    if (char === "#" && (i === 0 || /\s/.test(value[i - 1]))) return value.slice(0, i);
+  }
+  return value;
 }
 
 export function parseDotenvValue(rawValue: string): string {
-	let value = stripInlineComment(rawValue).trim();
-	if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
-		value = value.slice(1, -1);
-		return value.replace(/\\n/g, "\n").replace(/\\r/g, "\r").replace(/\\t/g, "\t");
-	}
-	return value.trim();
+  let value = stripInlineComment(rawValue).trim();
+  if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+    value = value.slice(1, -1);
+    return value.replace(/\\n/g, "\n").replace(/\\r/g, "\r").replace(/\\t/g, "\t");
+  }
+  return value.trim();
 }
 
 export function parseDotenvFile(file: string): Record<string, string> | null {
-	let text: string;
-	try { text = fs.readFileSync(file, "utf8"); } catch (e: any) { if (e?.code === "ENOENT") return null; throw e; }
-	const values: Record<string, string> = {};
-	for (const rawLine of text.split(/\r?\n/)) {
-		const line = rawLine.trim();
-		if (!line || line.startsWith("#")) continue;
-		const match = line.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
-		if (match) values[match[1]] = parseDotenvValue(match[2]);
-	}
-	return values;
+  let text: string;
+  try { text = fs.readFileSync(file, "utf8"); } catch { return null; }
+  const values: Record<string, string> = {};
+  for (const rawLine of text.split(/\r?\n/)) {
+    const line = rawLine.trim();
+    if (!line || line.startsWith("#")) continue;
+    const match = line.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
+    if (match) values[match[1]] = parseDotenvValue(match[2]);
+  }
+  return values;
 }
 
 export function findEnvValue(name: string, cwd = process.cwd(), includeCwd = true): { value?: string; source: string; checkedFiles: string[] } {
-	if (process.env[name]) return { value: process.env[name], source: "process.env", checkedFiles: [] };
-	const checkedFiles = envFileCandidates(cwd, includeCwd);
-	for (const file of checkedFiles) {
-		const parsed = parseDotenvFile(file);
-		if (parsed?.[name]) return { value: parsed[name], source: file, checkedFiles };
-	}
-	return { value: undefined, source: "", checkedFiles };
+  if (process.env[name]) return { value: process.env[name], source: "process.env", checkedFiles: [] };
+  const checkedFiles = envFileCandidates(cwd, includeCwd);
+  for (const file of checkedFiles) {
+    const parsed = parseDotenvFile(file);
+    if (parsed?.[name]) return { value: parsed[name], source: file, checkedFiles };
+  }
+  return { value: undefined, source: "", checkedFiles };
 }
 
 // ---------------------------------------------------------------------------
@@ -80,11 +80,11 @@ export function findEnvValue(name: string, cwd = process.cwd(), includeCwd = tru
 // ---------------------------------------------------------------------------
 
 export function cwdFromContext(ctx: Record<string, unknown>): string {
-	return typeof ctx?.cwd === "string" && (ctx.cwd as string) ? (ctx.cwd as string) : process.cwd();
+  return typeof ctx?.cwd === "string" && (ctx.cwd as string) ? (ctx.cwd as string) : process.cwd();
 }
 
 export function includeProjectEnv(ctx: Record<string, unknown>): boolean {
-	return typeof ctx?.isProjectTrusted === "function" ? (ctx.isProjectTrusted as () => boolean)() : false;
+  return typeof ctx?.isProjectTrusted === "function" ? (ctx.isProjectTrusted as () => boolean)() : false;
 }
 
 // ---------------------------------------------------------------------------
@@ -92,13 +92,13 @@ export function includeProjectEnv(ctx: Record<string, unknown>): boolean {
 // ---------------------------------------------------------------------------
 
 function normalizeHttpUrl(raw: string | undefined, fallback: string): string {
-	let value = (raw || fallback).trim();
-	if (!/^https?:\/\//i.test(value)) {
-		value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value)
-			? `http://${value}`
-			: `https://${value}`;
-	}
-	return value.replace(/\/+$/, "");
+  let value = (raw || fallback).trim();
+  if (!/^https?:\/\//i.test(value)) {
+    value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value)
+      ? `http://${value}`
+      : `https://${value}`;
+  }
+  return value.replace(/\/+$/, "");
 }
 
 // ---------------------------------------------------------------------------
@@ -106,20 +106,20 @@ function normalizeHttpUrl(raw: string | undefined, fallback: string): string {
 // ---------------------------------------------------------------------------
 
 export interface SearxngConfig {
-	baseUrl: string;
-	source: string;
+  baseUrl: string;
+  source: string;
 }
 
 export function normalizeSearxngBaseUrl(raw?: string): string {
-	return normalizeHttpUrl(raw, DEFAULT_SEARXNG_BASE_URL);
+  return normalizeHttpUrl(raw, DEFAULT_SEARXNG_BASE_URL);
 }
 
 export function loadSearxngConfig(params: Record<string, unknown> = {}, cwd = process.cwd(), includeCwdEnv = false): SearxngConfig {
-	const baseUrlEnv = findEnvValue("SEARXNG_BASE_URL", cwd, includeCwdEnv);
-	const explicitBaseUrl = params.searxng_base_url as string | undefined;
-	const baseUrl = normalizeSearxngBaseUrl(explicitBaseUrl || (params.base_url as string) || baseUrlEnv.value);
-	const source = explicitBaseUrl || params.base_url ? "tool parameter" : baseUrlEnv.source || "default local";
-	return { baseUrl, source };
+  const baseUrlEnv = findEnvValue("SEARXNG_BASE_URL", cwd, includeCwdEnv);
+  const explicitBaseUrl = params.searxng_base_url as string | undefined;
+  const baseUrl = normalizeSearxngBaseUrl(explicitBaseUrl || (params.base_url as string) || baseUrlEnv.value);
+  const source = explicitBaseUrl || params.base_url ? "tool parameter" : baseUrlEnv.source || "default local";
+  return { baseUrl, source };
 }
 
 // ---------------------------------------------------------------------------
@@ -127,26 +127,29 @@ export function loadSearxngConfig(params: Record<string, unknown> = {}, cwd = pr
 // ---------------------------------------------------------------------------
 
 export interface Crawl4aiConfig {
-	baseUrl: string;
-	apiToken: string;
-	timeoutMs: number;
+  baseUrl: string;
+  apiToken: string;
+  timeoutMs: number;
 }
 
 export function normalizeCrawl4aiApiUrl(raw?: string): string {
-	return normalizeHttpUrl(raw, DEFAULT_CRAWL4AI_API_URL);
+  return normalizeHttpUrl(raw, DEFAULT_CRAWL4AI_API_URL);
 }
 
 export function loadCrawl4aiConfig(params: Record<string, unknown> = {}, cwd = process.cwd(), includeCwdEnv = false): Crawl4aiConfig {
-	const apiUrlLookup = findEnvValue("CRAWL4AI_API_URL", cwd, includeCwdEnv);
-	const apiTokenLookup = findEnvValue("CRAWL4AI_API_TOKEN", cwd, includeCwdEnv);
-	const explicitApiUrl = params.crawl4ai_api_url as string | undefined;
-	const explicitApiToken = params.crawl4ai_api_token as string | undefined;
-	const baseUrl = normalizeCrawl4aiApiUrl(explicitApiUrl || apiUrlLookup.value);
-	const apiToken = explicitApiToken || apiTokenLookup.value || "";
-	const timeoutValue = (params.timeout_ms as number) || findEnvValue("CRAWL4AI_API_TIMEOUT_MS", cwd, includeCwdEnv).value;
-	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
-	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("CRAWL4AI_API_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
-	return { baseUrl, apiToken, timeoutMs };
+  const apiUrlLookup = findEnvValue("CRAWL4AI_API_URL", cwd, includeCwdEnv);
+  const apiTokenLookup = findEnvValue("CRAWL4AI_API_TOKEN", cwd, includeCwdEnv);
+  const explicitApiUrl = params.crawl4ai_api_url as string | undefined;
+  const explicitApiToken = params.crawl4ai_api_token as string | undefined;
+  const baseUrl = normalizeCrawl4aiApiUrl(explicitApiUrl || apiUrlLookup.value);
+  // Security: only use env-var API token when URL is default or also from env var.
+  // If the URL is overridden via params but no explicit token was provided, don't send
+  // the env-var token to a potentially attacker-controlled URL.
+  const apiToken = explicitApiToken || (explicitApiUrl ? "" : apiTokenLookup.value) || "";
+  const timeoutValue = (params.timeout_ms as number) || findEnvValue("CRAWL4AI_API_TIMEOUT_MS", cwd, includeCwdEnv).value;
+  const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
+  if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("CRAWL4AI_API_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
+  return { baseUrl, apiToken, timeoutMs };
 }
 
 // ---------------------------------------------------------------------------
@@ -154,29 +157,32 @@ export function loadCrawl4aiConfig(params: Record<string, unknown> = {}, cwd = p
 // ---------------------------------------------------------------------------
 
 export interface FirecrawlConfig {
-	baseUrl: string;
-	apiKey: string;
-	isHosted: boolean;
-	timeoutMs: number;
+  baseUrl: string;
+  apiKey: string;
+  isHosted: boolean;
+  timeoutMs: number;
 }
 
 export function normalizeFirecrawlBaseUrl(raw?: string): string {
-	let value = normalizeHttpUrl(raw, HOSTED_FIRECRAWL_BASE_URL);
-	if (!/\/v\d+$/i.test(value)) value += "/v2";
-	return value;
+  let value = normalizeHttpUrl(raw, HOSTED_FIRECRAWL_BASE_URL);
+  if (!/\/v\d+$/i.test(value)) value += "/v2";
+  return value;
 }
 
 export function loadFirecrawlConfig(params: Record<string, unknown> = {}, cwd = process.cwd(), includeCwdEnv = false): FirecrawlConfig {
-	const apiUrlLookup = findEnvValue("FIRECRAWL_API_URL", cwd, includeCwdEnv);
-	const apiKeyLookup = findEnvValue("FIRECRAWL_API_KEY", cwd, includeCwdEnv);
-	const explicitApiKey = (params.firecrawl_api_key as string) || (params.api_key as string);
-	const apiUrl = (params.firecrawl_api_url as string) || (params.api_url as string) || apiUrlLookup.value;
-	const apiKey = explicitApiKey || apiKeyLookup.value || "";
-	const timeoutValue = (params.timeout_ms as number) || findEnvValue("FIRECRAWL_TIMEOUT_MS", cwd, includeCwdEnv).value;
-	const baseUrl = normalizeFirecrawlBaseUrl(apiUrl);
-	const isHosted = !apiUrl || baseUrl.startsWith(HOSTED_FIRECRAWL_BASE_URL);
-	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
-	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("FIRECRAWL_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
-	if (isHosted && !apiKey) throw new Error("FIRECRAWL_API_KEY is required for hosted Firecrawl.");
-	return { baseUrl, apiKey, isHosted, timeoutMs };
+  const apiUrlLookup = findEnvValue("FIRECRAWL_API_URL", cwd, includeCwdEnv);
+  const apiKeyLookup = findEnvValue("FIRECRAWL_API_KEY", cwd, includeCwdEnv);
+  const explicitApiKey = (params.firecrawl_api_key as string) || (params.api_key as string);
+  const apiUrl = (params.firecrawl_api_url as string) || (params.api_url as string) || apiUrlLookup.value;
+  // Security: only use env-var API key when URL is default or also from env var.
+  // If the URL is overridden via params but no explicit key was provided, don't send
+  // the env-var key to a potentially attacker-controlled URL.
+  const apiKey = explicitApiKey || (apiUrl !== apiUrlLookup.value && apiUrl ? "" : apiKeyLookup.value) || "";
+  const timeoutValue = (params.timeout_ms as number) || findEnvValue("FIRECRAWL_TIMEOUT_MS", cwd, includeCwdEnv).value;
+  const baseUrl = normalizeFirecrawlBaseUrl(apiUrl);
+  const isHosted = !apiUrl || baseUrl.startsWith(HOSTED_FIRECRAWL_BASE_URL);
+  const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
+  if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("FIRECRAWL_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
+  if (isHosted && !apiKey) throw new Error("FIRECRAWL_API_KEY is required for hosted Firecrawl.");
+  return { baseUrl, apiKey, isHosted, timeoutMs };
 }

package/extensions/lib/content.ts

@@ -6,73 +6,81 @@ import { signalWithTimeout } from "./retry";
 const require = createRequire(import.meta.url);
 
 export interface ReadableContentDeps {
-	Readability: any;
-	JSDOM: any;
-	TurndownService: any;
-	gfm: unknown;
+  Readability: any;
+  JSDOM: any;
+  TurndownService: any;
+  gfm: unknown;
 }
 
 export function loadReadableContentDependencies(): ReadableContentDeps {
-	try {
-		return {
-			Readability: require("@mozilla/readability").Readability,
-			JSDOM: require("jsdom").JSDOM,
-			TurndownService: require("turndown"),
-			gfm: require("turndown-plugin-gfm").gfm,
-		};
-	} catch (error: any) {
-		throw new Error(
-			`Readable content extraction dependencies are missing. Install pi-web dependencies with \`npm install\` in the pi-web extension directory, or install the package with Pi so runtime dependencies are installed. Original error: ${error?.message || error}`,
-		);
-	}
+  try {
+    return {
+      Readability: require("@mozilla/readability").Readability,
+      JSDOM: require("jsdom").JSDOM,
+      TurndownService: require("turndown"),
+      gfm: require("turndown-plugin-gfm").gfm,
+    };
+  } catch (error: any) {
+    throw new Error(
+      `Readable content extraction dependencies are missing. Install pi-web dependencies with \`npm install\` in the pi-web extension directory, or install the package with Pi so runtime dependencies are installed. Original error: ${error?.message || error}`,
+    );
+  }
 }
 
 export function htmlToMarkdown(html: string, deps = loadReadableContentDependencies()): string {
-	const turndown = new deps.TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
-	turndown.use(deps.gfm as any);
-	turndown.addRule("removeEmptyLinks", {
-		filter: (node: any) => node.nodeName === "A" && !node.textContent?.trim(),
-		replacement: () => "",
-	});
-	return turndown
-		.turndown(html)
-		.replace(/\[\\?\[\s*\\?\]\]\([^)]*\)/g, "")
-		.replace(/ +/g, " ")
-		.replace(/\s+,/g, ",")
-		.replace(/\s+\./g, ".")
-		.replace(/\n{3,}/g, "\n\n")
-		.trim();
+  const turndown = new deps.TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+  turndown.use(deps.gfm as any);
+  turndown.addRule("removeEmptyLinks", {
+    filter: (node: any) => node.nodeName === "A" && !node.textContent?.trim(),
+    replacement: () => "",
+  });
+  return turndown
+    .turndown(html)
+    .replace(/\[\\?\[\s*\\?\]\]\([^)]*\)/g, "")
+    .replace(/ +/g, " ")
+    .replace(/\s+,/g, ",")
+    .replace(/\s+\./g, ".")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
 }
 
 export async function fetchReadableContent(
-	url: string,
-	timeoutMs = 15000,
-	signal?: AbortSignal,
+  url: string,
+  timeoutMs = 15000,
+  signal?: AbortSignal,
 ): Promise<{ title: string; markdown: string }> {
-	const response = await fetch(url, {
-		headers: {
-			"User-Agent":
-				"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120 Safari/537.36",
-			Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-			"Accept-Language": "en-US,en;q=0.9",
-		},
-		signal: signalWithTimeout(timeoutMs, signal),
-	});
-	if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-	const html = await response.text();
-	const deps = loadReadableContentDependencies();
-	const dom = new deps.JSDOM(html, { url });
-	const article = new deps.Readability(dom.window.document).parse();
-	if (article?.content) {
-		return { title: article.title || "", markdown: htmlToMarkdown(article.content, deps) };
-	}
-	// Fallback: strip non-content elements and use main/article/body
-	const fallbackDoc = new deps.JSDOM(html, { url });
-	const doc = fallbackDoc.window.document;
-	doc.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach((el: any) => el.remove());
-	const title = doc.querySelector("title")?.textContent?.trim() || "";
-	const main = doc.querySelector("main, article, [role='main'], .content, #content") || doc.body;
-	const fallbackHtml = main?.innerHTML || "";
-	if (fallbackHtml.trim().length <= 100) throw new Error("Could not extract readable content from this page.");
-	return { title, markdown: htmlToMarkdown(fallbackHtml, deps) };
+  const response = await fetch(url, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120 Safari/537.36",
+      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "Accept-Language": "en-US,en;q=0.9",
+    },
+    signal: signalWithTimeout(timeoutMs, signal),
+  });
+  if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  const html = await response.text();
+  const deps = loadReadableContentDependencies();
+  const dom = new deps.JSDOM(html, { url });
+  try {
+    const article = new deps.Readability(dom.window.document).parse();
+    if (article?.content) {
+      return { title: article.title || "", markdown: htmlToMarkdown(article.content, deps) };
+    }
+  } finally {
+    dom.window.close();
+  }
+  // Fallback: strip non-content elements and use main/article/body
+  const fallbackDoc = new deps.JSDOM(html, { url });
+  try {
+    const doc = fallbackDoc.window.document;
+    doc.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach((el: any) => el.remove());
+    const title = doc.querySelector("title")?.textContent?.trim() || "";
+    const main = doc.querySelector("main, article, [role='main'], .content, #content") || doc.body;
+    const fallbackHtml = main?.innerHTML || "";
+    if (fallbackHtml.trim().length <= 100) throw new Error("Could not extract readable content from this page.");
+    return { title, markdown: htmlToMarkdown(fallbackHtml, deps) };
+  } finally {
+    fallbackDoc.window.close();
+  }
 }