@prometheus-ai/agent 0.5.4 → 0.5.8

package/src/web/scrapers/github.ts

@@ -1,14 +1,29 @@
 import { $env, ptree } from "@prometheus-ai/utils";
 import type { RenderResult, SpecialHandler } from "./types";
-import { buildResult, loadPage } from "./types";
+import { buildResult, formatMediaDuration, loadPage } from "./types";
 
 interface GitHubUrl {
-	type: "blob" | "tree" | "repo" | "issue" | "issues" | "pull" | "pulls" | "discussion" | "discussions" | "other";
+	type:
+		| "blob"
+		| "tree"
+		| "repo"
+		| "commit"
+		| "issue"
+		| "issues"
+		| "pull"
+		| "pulls"
+		| "discussion"
+		| "discussions"
+		| "actions-run"
+		| "actions-job"
+		| "other";
 	owner: string;
 	repo: string;
 	ref?: string;
 	path?: string;
 	number?: number;
+	runId?: number;
+	jobId?: number;
 }
 
 interface GitHubIssueComment {
@@ -20,7 +35,7 @@ interface GitHubIssueComment {
 /**
  * Parse GitHub URL into components
  */
-function parseGitHubUrl(url: string): GitHubUrl | null {
+export function parseGitHubUrl(url: string): GitHubUrl | null {
 	try {
 		const parsed = new URL(url);
 		if (parsed.hostname !== "github.com") return null;
@@ -42,6 +57,11 @@ function parseGitHubUrl(url: string): GitHubUrl | null {
 				const [ref, ...pathParts] = subParts;
 				return { type: section, owner, repo, ref, path: pathParts.join("/") };
 			}
+			case "commit":
+				if (subParts.length > 0 && subParts[0]) {
+					return { type: "commit", owner, repo, ref: subParts[0] };
+				}
+				return { type: "other", owner, repo };
 			case "issues":
 				if (subParts.length > 0 && /^\d+$/.test(subParts[0])) {
 					return { type: "issue", owner, repo, number: parseInt(subParts[0], 10) };
@@ -54,6 +74,20 @@ function parseGitHubUrl(url: string): GitHubUrl | null {
 				return { type: "pulls", owner, repo };
 			case "pulls":
 				return { type: "pulls", owner, repo };
+			case "actions": {
+				// /actions/runs/{runId}                      → run summary + jobs
+				// /actions/runs/{runId}/job/{jobId}          → single job (web URL uses singular "job")
+				// /actions/runs/{runId}/jobs/{jobId}         → single job (API-style plural)
+				if (subParts[0] === "runs" && /^\d+$/.test(subParts[1] ?? "")) {
+					const runId = parseInt(subParts[1], 10);
+					const seg = subParts[2];
+					if ((seg === "job" || seg === "jobs") && /^\d+$/.test(subParts[3] ?? "")) {
+						return { type: "actions-job", owner, repo, runId, jobId: parseInt(subParts[3], 10) };
+					}
+					return { type: "actions-run", owner, repo, runId };
+				}
+				return { type: "other", owner, repo };
+			}
 			case "discussions":
 				if (subParts.length > 0 && /^\d+$/.test(subParts[0])) {
 					return { type: "discussion", owner, repo, number: parseInt(subParts[0], 10) };
@@ -205,6 +239,87 @@ async function renderGitHubIssue(
 	return { content: md, ok: true };
 }
 
+interface GitHubCommitFile {
+	filename: string;
+	status: string;
+	additions: number;
+	deletions: number;
+	changes: number;
+	patch?: string;
+	previous_filename?: string;
+}
+
+/**
+ * Render a GitHub commit (metadata, message, and per-file diff) to markdown.
+ *
+ * The commits API (`/repos/{owner}/{repo}/commits/{ref}`) returns the full
+ * unified diff inline via `files[].patch`, so a single request yields both the
+ * summary and the diff. Binary files have no `patch` and are flagged instead.
+ */
+async function renderGitHubCommit(
+	gh: GitHubUrl,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean }> {
+	const result = await fetchGitHubApi(`/repos/${gh.owner}/${gh.repo}/commits/${gh.ref}`, timeout, signal);
+	if (!result.ok || !result.data) return { content: "", ok: false };
+
+	const commit = result.data as {
+		sha: string;
+		html_url: string;
+		commit: {
+			author?: { name?: string; date?: string } | null;
+			committer?: { name?: string; date?: string } | null;
+			message: string;
+		};
+		author?: { login: string } | null;
+		committer?: { login: string } | null;
+		parents?: Array<{ sha: string }>;
+		stats?: { total?: number; additions?: number; deletions?: number };
+		files?: GitHubCommitFile[];
+	};
+
+	const message = commit.commit.message ?? "";
+	const [subject, ...bodyLines] = message.split("\n");
+	const authorName = commit.author?.login ? `@${commit.author.login}` : (commit.commit.author?.name ?? "unknown");
+	const authoredAt = commit.commit.author?.date ?? "";
+
+	let md = `# ${subject || commit.sha.slice(0, 7)}\n\n`;
+	md += `**${commit.sha.slice(0, 12)}** · authored by ${authorName}`;
+	if (authoredAt) md += ` · ${authoredAt}`;
+	md += `\n`;
+	if (commit.stats) {
+		const { additions = 0, deletions = 0 } = commit.stats;
+		const fileCount = commit.files?.length ?? 0;
+		md += `${fileCount} file${fileCount === 1 ? "" : "s"} changed · +${additions} −${deletions}\n`;
+	}
+	if (commit.parents && commit.parents.length > 0) {
+		md += `Parents: ${commit.parents.map(p => p.sha.slice(0, 12)).join(", ")}\n`;
+	}
+
+	const body = bodyLines.join("\n").trim();
+	if (body) {
+		md += `\n${body}\n`;
+	}
+
+	const files = commit.files ?? [];
+	if (files.length > 0) {
+		md += `\n---\n\n## Files (${files.length})\n\n`;
+		for (const file of files) {
+			const name = file.previous_filename ? `${file.previous_filename} → ${file.filename}` : file.filename;
+			md += `### ${name}\n\n`;
+			md += `${file.status} · +${file.additions} −${file.deletions}\n\n`;
+			if (file.patch) {
+				md += `\`\`\`diff\n${file.patch}\n\`\`\`\n\n`;
+			} else {
+				md += `*No textual diff (binary or too large).*\n\n`;
+			}
+		}
+	}
+
+	return { content: md, ok: true };
+}
+
 /**
  * Render GitHub issues list to markdown
  */
@@ -371,6 +486,212 @@ async function renderGitHubRepo(
 	return { content: md, ok: true };
 }
 
+interface GitHubActionsStep {
+	name: string;
+	status: string;
+	conclusion: string | null;
+	number: number;
+	started_at: string | null;
+	completed_at: string | null;
+}
+
+interface GitHubActionsJob {
+	id: number;
+	run_id: number;
+	name: string;
+	status: string;
+	conclusion: string | null;
+	started_at: string | null;
+	completed_at: string | null;
+	html_url: string | null;
+	steps?: GitHubActionsStep[];
+	runner_name?: string | null;
+	labels?: string[];
+	workflow_name?: string | null;
+	head_branch?: string | null;
+	head_sha?: string;
+}
+
+interface GitHubActionsRun {
+	id: number;
+	name?: string | null;
+	display_title?: string;
+	run_number: number;
+	run_attempt?: number;
+	event: string;
+	status: string;
+	conclusion: string | null;
+	head_branch?: string | null;
+	head_sha?: string;
+	html_url: string;
+	created_at: string;
+	updated_at: string;
+	run_started_at?: string;
+	actor?: { login: string };
+	triggering_actor?: { login: string };
+}
+
+/** Combine status + conclusion into a single label, e.g. `completed (failure)`. */
+function statusLabel(status: string, conclusion: string | null | undefined): string {
+	return conclusion ? `${status} (${conclusion})` : status;
+}
+
+/** Wall-clock duration between two ISO timestamps, formatted HH:MM:SS / MM:SS. Empty when unknown. */
+function actionDuration(start?: string | null, end?: string | null): string {
+	if (!start || !end) return "";
+	const ms = Date.parse(end) - Date.parse(start);
+	if (!Number.isFinite(ms) || ms < 0) return "";
+	return formatMediaDuration(Math.round(ms / 1000));
+}
+
+/** Escape `|` so step/job names can't break a markdown table row. */
+function escapeCell(text: string): string {
+	return text.replaceAll("|", "\\|");
+}
+
+/**
+ * Strip the per-line ISO-8601 timestamp prefix GitHub prepends to every job log line.
+ * Cuts ~28 bytes/line of noise while preserving the message text. Also drops the leading
+ * UTF-8 BOM GitHub puts at the start of the log file (otherwise the first line's timestamp
+ * survives because `^` no longer sits before a digit).
+ */
+export function stripActionsLogTimestamps(logs: string): string {
+	return logs.replace(/^\uFEFF/, "").replace(/^\d{4}-\d{2}-\d{2}T[\d:.]+Z /gm, "");
+}
+
+/** Render a job's steps as a markdown table. Empty string when there are no steps. */
+function renderActionsSteps(steps?: GitHubActionsStep[]): string {
+	if (!steps || steps.length === 0) return "";
+	let md = "| # | Step | Status | Conclusion | Duration |\n";
+	md += "|---|------|--------|------------|----------|\n";
+	for (const step of steps) {
+		const dur = actionDuration(step.started_at, step.completed_at) || "-";
+		md += `| ${step.number} | ${escapeCell(step.name)} | ${step.status} | ${step.conclusion ?? "-"} | ${dur} |\n`;
+	}
+	return `${md}\n`;
+}
+
+/** Run-level metadata lines shared by the run and job renderers. */
+function renderActionsRunMeta(run: GitHubActionsRun): string {
+	let md = `**Workflow:** ${run.name ?? "(unknown)"}\n`;
+	md += `**Run:** #${run.run_number}`;
+	if (run.run_attempt && run.run_attempt > 1) md += ` (attempt ${run.run_attempt})`;
+	md += ` · ${statusLabel(run.status, run.conclusion)}\n`;
+	if (run.head_branch) {
+		md += `**Branch:** ${run.head_branch}${run.head_sha ? ` @ ${run.head_sha.slice(0, 7)}` : ""}\n`;
+	}
+	const actor = run.triggering_actor?.login ?? run.actor?.login;
+	md += `**Event:** ${run.event}${actor ? ` · by @${actor}` : ""}\n`;
+	const started = run.run_started_at ?? run.created_at;
+	const dur = actionDuration(started, run.updated_at);
+	md += `Started: ${started}${dur ? ` · Duration: ${dur}` : ""}\n`;
+	md += `URL: ${run.html_url}\n`;
+	return md;
+}
+
+/** Fetch a job's plain-text logs. Returns null when unavailable (no token / expired / private). */
+async function fetchGitHubJobLogs(
+	owner: string,
+	repo: string,
+	jobId: number,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<string | null> {
+	const headers: Record<string, string> = {
+		Accept: "application/vnd.github+json",
+		"X-GitHub-Api-Version": "2022-11-28",
+	};
+	const token = $env.GITHUB_TOKEN || $env.GH_TOKEN;
+	if (token) headers.Authorization = `Bearer ${token}`;
+
+	// 302 → signed log URL on a different origin; fetch strips Authorization on the cross-origin hop.
+	const result = await loadPage(`https://api.github.com/repos/${owner}/${repo}/actions/jobs/${jobId}/logs`, {
+		timeout,
+		headers,
+		signal,
+	});
+	return result.ok && result.content ? result.content : null;
+}
+
+/**
+ * Render a workflow run: run metadata plus a per-job breakdown. Steps are listed for any job that
+ * did not succeed (the debugging-relevant ones); successful jobs collapse to a single line.
+ */
+async function renderGitHubActionsRun(
+	gh: GitHubUrl,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean }> {
+	const runResult = await fetchGitHubApi(`/repos/${gh.owner}/${gh.repo}/actions/runs/${gh.runId}`, timeout, signal);
+	if (!runResult.ok || !runResult.data) return { content: "", ok: false };
+
+	const run = runResult.data as GitHubActionsRun;
+	let md = `# ${run.display_title || run.name || `Run #${run.run_number}`}\n\n`;
+	md += renderActionsRunMeta(run);
+	md += `\n---\n\n`;
+
+	const jobsResult = await fetchGitHubApi(
+		`/repos/${gh.owner}/${gh.repo}/actions/runs/${gh.runId}/jobs?per_page=100`,
+		timeout,
+		signal,
+	);
+	if (jobsResult.ok && jobsResult.data) {
+		const jobs = (jobsResult.data as { jobs?: GitHubActionsJob[] }).jobs ?? [];
+		md += `## Jobs (${jobs.length})\n\n`;
+		for (const job of jobs) {
+			const dur = actionDuration(job.started_at, job.completed_at);
+			md += `### ${escapeCell(job.name)} — ${statusLabel(job.status, job.conclusion)}${dur ? ` (${dur})` : ""}\n\n`;
+			if (job.conclusion !== "success") {
+				md += renderActionsSteps(job.steps);
+			}
+		}
+	}
+
+	return { content: md, ok: true };
+}
+
+/**
+ * Render a single workflow job: run context, step table, and the full job logs.
+ */
+async function renderGitHubActionsJob(
+	gh: GitHubUrl,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean }> {
+	const jobResult = await fetchGitHubApi(`/repos/${gh.owner}/${gh.repo}/actions/jobs/${gh.jobId}`, timeout, signal);
+	if (!jobResult.ok || !jobResult.data) return { content: "", ok: false };
+
+	const job = jobResult.data as GitHubActionsJob;
+
+	// Best-effort run context for nicer headers; the job render stands on its own without it.
+	const runResult = await fetchGitHubApi(`/repos/${gh.owner}/${gh.repo}/actions/runs/${job.run_id}`, timeout, signal);
+	const run = runResult.ok && runResult.data ? (runResult.data as GitHubActionsRun) : null;
+
+	let md = `# ${escapeCell(job.name)}\n\n`;
+	if (run) {
+		md += renderActionsRunMeta(run);
+	} else if (job.workflow_name) {
+		md += `**Workflow:** ${job.workflow_name}\n`;
+		if (job.head_branch) md += `**Branch:** ${job.head_branch}\n`;
+	}
+	const dur = actionDuration(job.started_at, job.completed_at);
+	md += `**Job:** ${escapeCell(job.name)} · ${statusLabel(job.status, job.conclusion)}${dur ? ` · ${dur}` : ""}\n`;
+	if (job.runner_name) md += `**Runner:** ${job.runner_name}\n`;
+	if (job.html_url) md += `URL: ${job.html_url}\n`;
+	md += `\n---\n\n`;
+
+	const steps = renderActionsSteps(job.steps);
+	if (steps) md += `## Steps\n\n${steps}`;
+
+	const logs = await fetchGitHubJobLogs(gh.owner, gh.repo, job.id, timeout, signal);
+	md += `## Logs\n\n`;
+	md += logs
+		? stripActionsLogTimestamps(logs)
+		: "*Logs unavailable — requires a GITHUB_TOKEN/GH_TOKEN with read access, or the run's logs have expired.*\n";
+
+	return { content: md, ok: true };
+}
+
 /**
  * Handle GitHub URLs specially
  */
@@ -413,6 +734,15 @@ export const handleGitHub: SpecialHandler = async (
 			break;
 		}
 
+		case "commit": {
+			notes.push(`Fetched via GitHub API`);
+			const result = await renderGitHubCommit(gh, timeout, signal);
+			if (result.ok) {
+				return buildResult(result.content, { url, method: "github-commit", fetchedAt, notes });
+			}
+			break;
+		}
+
 		case "issue":
 		case "pull": {
 			notes.push(`Fetched via GitHub API`);
@@ -445,6 +775,24 @@ export const handleGitHub: SpecialHandler = async (
 			}
 			break;
 		}
+
+		case "actions-run": {
+			notes.push(`Fetched via GitHub API`);
+			const result = await renderGitHubActionsRun(gh, timeout, signal);
+			if (result.ok) {
+				return buildResult(result.content, { url, method: "github-actions-run", fetchedAt, notes });
+			}
+			break;
+		}
+
+		case "actions-job": {
+			notes.push(`Fetched via GitHub API`);
+			const result = await renderGitHubActionsJob(gh, timeout, signal);
+			if (result.ok) {
+				return buildResult(result.content, { url, method: "github-actions-job", fetchedAt, notes });
+			}
+			break;
+		}
 	}
 
 	// Fall back to null (let normal rendering handle it)

package/src/web/scrapers/go-pkg.ts

@@ -1,5 +1,4 @@
 import { tryParseJson } from "@prometheus-ai/utils";
-import { parseHTML } from "linkedom";
 import type { RenderResult, SpecialHandler } from "./types";
 import { buildResult, htmlToBasicMarkdown, loadPage } from "./types";
 
@@ -97,6 +96,7 @@ export const handleGoPkg: SpecialHandler = async (
 			});
 		}
 
+		const { parseHTML } = await import("linkedom");
 		const doc = parseHTML(pageResult.content).document;
 
 		// Extract actual module path from breadcrumb or header

package/src/web/scrapers/iacr.ts

@@ -1,4 +1,3 @@
-import { parseHTML } from "linkedom";
 import type { RenderResult, SpecialHandler } from "./types";
 import { buildResult, loadPage } from "./types";
 import { convertWithMarkit, fetchBinary } from "./utils";
@@ -30,6 +29,7 @@ export const handleIacr: SpecialHandler = async (
 
 		if (!result.ok) return null;
 
+		const { parseHTML } = await import("linkedom");
 		const doc = parseHTML(result.content).document;
 
 		// Extract metadata from the page

package/src/web/scrapers/readthedocs.ts

@@ -1,7 +1,6 @@
 /**
  * Read the Docs handler for web-fetch
  */
-import { parseHTML } from "linkedom";
 import { buildResult, htmlToBasicMarkdown, loadPage, type RenderResult, type SpecialHandler } from "./types";
 
 export const handleReadTheDocs: SpecialHandler = async (
@@ -39,6 +38,7 @@ export const handleReadTheDocs: SpecialHandler = async (
 	}
 
 	// Parse HTML
+	const { parseHTML } = await import("linkedom");
 	const root = parseHTML(result.content).document;
 
 	// Extract main content from common Read the Docs selectors

package/src/web/scrapers/twitter.ts

@@ -1,4 +1,4 @@
-import { type HTMLElement, parseHTML } from "linkedom";
+import type { HTMLElement } from "linkedom";
 import { ToolAbortError } from "../../tools/tool-errors";
 import type { RenderResult, SpecialHandler } from "./types";
 import { buildResult, loadPage } from "./types";
@@ -33,6 +33,7 @@ export const handleTwitter: SpecialHandler = async (
 
 			if (result.ok && result.content.length > 500) {
 				// Parse the Nitter HTML
+				const { parseHTML } = await import("linkedom");
 				const doc = parseHTML(result.content).document;
 
 				// Extract tweet content

package/src/web/scrapers/types.ts

@@ -1,6 +1,7 @@
 /**
  * Shared types and utilities for web-fetch handlers
  */
+import { scheduler } from "node:timers/promises";
 import { ptree } from "@prometheus-ai/utils";
 import type TurndownService from "turndown";
 
@@ -70,6 +71,12 @@ export interface LoadPageOptions {
 	body?: string;
 	maxBytes?: number;
 	signal?: AbortSignal;
+	/**
+	 * Return true to skip reading the response body for this content type
+	 * (lowercased mime, no params). The caller is expected to re-fetch the
+	 * payload as binary; this avoids streaming + decoding huge binaries twice.
+	 */
+	skipBodyForContentType?: (contentType: string) => boolean;
 }
 
 export interface LoadPageResult {
@@ -78,6 +85,51 @@ export interface LoadPageResult {
 	finalUrl: string;
 	ok: boolean;
 	status?: number;
+	/** True when the body was cut mid-stream at maxBytes. */
+	truncated?: boolean;
+	/** Last transport-level error message when ok is false. */
+	error?: string;
+	/** True when the body read was skipped via skipBodyForContentType. */
+	bodySkipped?: boolean;
+}
+
+const RETRY_AFTER_MAX_MS = 10_000;
+
+/** Parse a Retry-After header (seconds or HTTP-date) into a bounded delay. */
+function parseRetryAfterMs(value: string | null): number {
+	if (!value) return 1_000;
+	const seconds = Number(value);
+	if (Number.isFinite(seconds)) return Math.min(Math.max(seconds, 0) * 1000, RETRY_AFTER_MAX_MS);
+	const date = Date.parse(value);
+	if (!Number.isNaN(date)) return Math.min(Math.max(date - Date.now(), 0), RETRY_AFTER_MAX_MS);
+	return 1_000;
+}
+
+function charsetFromContentType(header: string): string | undefined {
+	return /charset\s*=\s*"?([\w-]+)"?/i.exec(header)?.[1];
+}
+
+/**
+ * Decode a response body honoring the declared charset (Content-Type header,
+ * then a cheap <meta charset> sniff), falling back to UTF-8.
+ */
+function decodeBody(bytes: Buffer, contentTypeHeader: string): string {
+	let label = charsetFromContentType(contentTypeHeader);
+	if (!label) {
+		// All charsets we can decode are ASCII-compatible in the prefix, so a
+		// latin1 view of the first 2KB is enough to find a <meta charset>.
+		label = /<meta[^>]+charset\s*=\s*["']?([\w-]+)/i.exec(bytes.subarray(0, 2048).toString("latin1"))?.[1];
+	}
+	if (label && !/^utf-?8$/i.test(label)) {
+		try {
+			// Bun.Encoding's union is narrower than the runtime, which accepts
+			// WHATWG labels (shift_jis, euc-kr, gbk, big5, …); unknowns throw here.
+			return new TextDecoder(label as Bun.Encoding).decode(bytes);
+		} catch {
+			// Unknown/unsupported label — fall back to UTF-8.
+		}
+	}
+	return bytes.toString("utf-8");
 }
 
 /**
@@ -86,6 +138,8 @@ export interface LoadPageResult {
 export async function loadPage(url: string, options: LoadPageOptions = {}): Promise<LoadPageResult> {
 	const { timeout = 20, headers = {}, maxBytes = MAX_BYTES, signal, method = "GET", body } = options;
 
+	let lastError: string | undefined;
+	let retried429 = false;
 	for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) {
 		if (signal?.aborted) {
 			throw new ToolAbortError();
@@ -114,9 +168,31 @@ export async function loadPage(url: string, options: LoadPageOptions = {}): Prom
 
 			const response = await fetch(url, requestInit);
 
-			const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? "";
+			const rawContentType = response.headers.get("content-type") ?? "";
+			const contentType = rawContentType.split(";")[0]?.trim().toLowerCase() ?? "";
 			const finalUrl = response.url;
 
+			if (response.status === 429 && !retried429) {
+				// Rate limited: retry once, honoring a bounded Retry-After. The
+				// wait observes the caller's signal so an Esc during the backoff
+				// does not stall for up to the full delay.
+				retried429 = true;
+				const delayMs = parseRetryAfterMs(response.headers.get("retry-after"));
+				void response.body?.cancel().catch(() => {});
+				try {
+					await scheduler.wait(delayMs, { signal });
+				} catch {
+					throw new ToolAbortError();
+				}
+				attempt--; // Reuse the same user agent for the retry.
+				continue;
+			}
+
+			if (response.ok && options.skipBodyForContentType?.(contentType)) {
+				void response.body?.cancel().catch(() => {});
+				return { content: "", contentType, finalUrl, ok: true, status: response.status, bodySkipped: true };
+			}
+
 			const reader = response.body?.getReader();
 			if (!reader) {
 				return { content: "", contentType, finalUrl, ok: false, status: response.status };
@@ -124,6 +200,7 @@ export async function loadPage(url: string, options: LoadPageOptions = {}): Prom
 
 			const chunks: Uint8Array[] = [];
 			let totalSize = 0;
+			let truncated = false;
 
 			while (true) {
 				const { done, value } = await reader.read();
@@ -133,32 +210,34 @@ export async function loadPage(url: string, options: LoadPageOptions = {}): Prom
 				totalSize += value.length;
 
 				if (totalSize > maxBytes) {
-					reader.cancel();
+					truncated = true;
+					void reader.cancel().catch(() => {});
 					break;
 				}
 			}
 
-			const content = Buffer.concat(chunks).toString("utf-8");
+			const content = decodeBody(Buffer.concat(chunks), rawContentType);
 			if (isBotBlocked(response.status, content) && attempt < USER_AGENTS.length - 1) {
 				continue;
 			}
 
 			if (!response.ok) {
-				return { content, contentType, finalUrl, ok: false, status: response.status };
+				return { content, contentType, finalUrl, ok: false, status: response.status, truncated };
 			}
 
-			return { content, contentType, finalUrl, ok: true, status: response.status };
-		} catch {
+			return { content, contentType, finalUrl, ok: true, status: response.status, truncated };
+		} catch (error) {
 			if (signal?.aborted) {
 				throw new ToolAbortError();
 			}
+			lastError = error instanceof Error ? error.message : String(error);
 			if (attempt === USER_AGENTS.length - 1) {
-				return { content: "", contentType: "", finalUrl: url, ok: false };
+				return { content: "", contentType: "", finalUrl: url, ok: false, error: lastError };
 			}
 		}
 	}
 
-	return { content: "", contentType: "", finalUrl: url, ok: false };
+	return { content: "", contentType: "", finalUrl: url, ok: false, error: lastError };
 }
 
 /** Module-level Turndown instance — built lazily on first use. */

package/src/web/scrapers/wikipedia.ts

@@ -1,4 +1,3 @@
-import { parseHTML } from "linkedom";
 import type { RenderResult, SpecialHandler } from "./types";
 import { buildResult, loadPage } from "./types";
 
@@ -45,6 +44,7 @@ export const handleWikipedia: SpecialHandler = async (
 		const contentResult = await loadPage(contentUrl, { timeout, signal });
 
 		if (contentResult.ok) {
+			const { parseHTML } = await import("linkedom");
 			const doc = parseHTML(contentResult.content).document;
 
 			// Extract main content sections

package/src/web/scrapers/youtube.ts

@@ -113,8 +113,9 @@ export const handleYouTube: SpecialHandler = async (
 	const notes: string[] = [];
 	const videoUrl = `https://www.youtube.com/watch?v=${yt.videoId}`;
 
-	// Prefer Parallel extract when credentials are available
-	if (settings.get("providers.parallelFetch") && findParallelApiKey(storage)) {
+	// Prefer Parallel extract when it sits in the reader chain and creds exist
+	const fetchPreference = settings.get("providers.fetch");
+	if ((fetchPreference === "auto" || fetchPreference === "parallel") && findParallelApiKey(storage)) {
 		try {
 			const parallelResult = await extractWithParallel(
 				[videoUrl],
@@ -287,12 +288,17 @@ export const handleYouTube: SpecialHandler = async (
 			}
 		}
 	} finally {
-		throwIfAborted(signal);
 		// Cleanup temp files (fire-and-forget with error suppression)
 		Array.fromAsync(new Bun.Glob(`${tmpBase}*`).scan({ absolute: true }))
 			.then(tmpFiles => Promise.all(tmpFiles.map(f => fs.unlink(f).catch(() => {}))))
 			.catch(() => {});
 	}
+	// Only a user-initiated abort is fatal; the per-fetch time budget expiring
+	// just means partial metadata/transcript, which we surface as a note.
+	throwIfAborted(userSignal);
+	if (signal?.aborted) {
+		notes.push("Fetch time budget exhausted; metadata/transcript may be incomplete");
+	}
 
 	// Build markdown output
 	let md = `# ${title}\n\n`;