@vertana/context-web 0.1.0-dev.1

package/LICENSE

@@ -0,0 +1,20 @@
+MIT License
+
+Copyright 2025 Hong Minhee
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,76 @@
+# @vertana/context-web
+
+[![JSR][JSR badge]][JSR]
+[![npm][npm badge]][npm]
+
+Web context gathering for [Vertana] — fetch and extract content from
+linked pages to provide additional context for translation.
+
+[JSR]: https://jsr.io/@vertana/context-web
+[JSR badge]: https://jsr.io/badges/@vertana/context-web
+[npm]: https://www.npmjs.com/package/@vertana/context-web
+[npm badge]: https://img.shields.io/npm/v/@vertana/context-web
+[Vertana]: https://vertana.org/
+
+
+Features
+--------
+
+ -  **fetchWebPage**: A passive context source that fetches a single URL
+    and extracts the main content using Mozilla's Readability algorithm.
+ -  **fetchLinkedPages**: A required context source factory that extracts
+    all links from the source text and fetches their content.
+ -  **extractLinks**: A utility function to extract URLs from text
+    in various formats (plain text, Markdown, HTML).
+
+
+Installation
+------------
+
+### Deno
+
+~~~~ bash
+deno add jsr:@vertana/context-web
+~~~~
+
+### npm
+
+~~~~ bash
+npm add @vertana/context-web
+~~~~
+
+### pnpm
+
+~~~~ bash
+pnpm add @vertana/context-web
+~~~~
+
+
+Usage
+-----
+
+~~~~ typescript
+import { translate } from "@vertana/facade";
+import { fetchLinkedPages, fetchWebPage } from "@vertana/context-web";
+import { openai } from "@ai-sdk/openai";
+
+const text = `
+Check out this article: https://example.com/article
+It explains the concept in detail.
+`;
+
+const result = await translate(openai("gpt-4o"), "ko", text, {
+  contextSources: [
+    // Automatically fetch all links in the text
+    fetchLinkedPages({ text, mediaType: "text/plain" }),
+    // Allow LLM to fetch additional URLs on demand
+    fetchWebPage,
+  ],
+});
+~~~~
+
+
+License
+-------
+
+[MIT License](../../LICENSE)

package/dist/extract-links.cjs

@@ -0,0 +1,117 @@
+let htmlparser2 = require("htmlparser2");
+
+//#region src/extract-links.ts
+/**
+* Extracts URLs from text based on the media type.
+*
+* @param text The text to extract URLs from.
+* @param mediaType The media type of the text.
+* @returns An array of unique URLs found in the text.
+* @since 0.1.0
+*/
+function extractLinks(text, mediaType) {
+	switch (mediaType) {
+		case "text/plain": return extractFromPlainText(text);
+		case "text/markdown": return extractFromMarkdown(text);
+		case "text/html": return extractFromHtml(text);
+	}
+}
+/**
+* URL pattern for plain text extraction.
+* Matches http:// and https:// URLs.
+*/
+const URL_PATTERN = /https?:\/\/[^\s<>"')\]]+/g;
+/**
+* Characters that should be trimmed from the end of URLs.
+*/
+const TRAILING_PUNCTUATION = /[.,;:!?)]+$/;
+/**
+* Extracts URLs from plain text.
+*/
+function extractFromPlainText(text) {
+	const matches = text.match(URL_PATTERN);
+	if (matches == null) return [];
+	const urls = /* @__PURE__ */ new Set();
+	for (const match of matches) {
+		const cleanUrl = match.replace(TRAILING_PUNCTUATION, "");
+		if (isValidUrl(cleanUrl)) urls.add(cleanUrl);
+	}
+	return [...urls];
+}
+/**
+* Markdown link patterns.
+*/
+const MARKDOWN_INLINE_LINK = /\[([^\]]*)\]\(([^)]+)\)/g;
+const MARKDOWN_REFERENCE_LINK = /^\[([^\]]+)\]:\s*(\S+)/gm;
+const MARKDOWN_AUTOLINK = /<(https?:\/\/[^>]+)>/g;
+const MARKDOWN_CODE_BLOCK = /```[\s\S]*?```|`[^`]+`/g;
+/**
+* Extracts URLs from Markdown text.
+*/
+function extractFromMarkdown(text) {
+	const textWithoutCode = text.replace(MARKDOWN_CODE_BLOCK, "");
+	const urls = /* @__PURE__ */ new Set();
+	let match;
+	MARKDOWN_INLINE_LINK.lastIndex = 0;
+	while ((match = MARKDOWN_INLINE_LINK.exec(textWithoutCode)) != null) {
+		const url = match[2];
+		if (isValidUrl(url)) urls.add(url);
+	}
+	MARKDOWN_REFERENCE_LINK.lastIndex = 0;
+	while ((match = MARKDOWN_REFERENCE_LINK.exec(textWithoutCode)) != null) {
+		const url = match[2];
+		if (isValidUrl(url)) urls.add(url);
+	}
+	MARKDOWN_AUTOLINK.lastIndex = 0;
+	while ((match = MARKDOWN_AUTOLINK.exec(textWithoutCode)) != null) {
+		const url = match[1];
+		if (isValidUrl(url)) urls.add(url);
+	}
+	const bareUrls = extractFromPlainText(textWithoutCode);
+	for (const url of bareUrls) urls.add(url);
+	return [...urls];
+}
+/**
+* Determines if a node is an element.
+*/
+function isElement(node) {
+	return node.type === "tag";
+}
+/**
+* Extracts URLs from HTML.
+*/
+function extractFromHtml(html) {
+	const doc = (0, htmlparser2.parseDocument)(html, {
+		lowerCaseTags: true,
+		lowerCaseAttributeNames: true
+	});
+	const urls = /* @__PURE__ */ new Set();
+	function traverse(node) {
+		if (isElement(node)) {
+			if (node.name === "a") {
+				const href = node.attribs.href;
+				if (href != null && isValidUrl(href)) urls.add(href);
+			}
+			for (const child of node.children) traverse(child);
+		}
+	}
+	for (const child of doc.children) traverse(child);
+	return [...urls];
+}
+/**
+* Checks if a URL is valid for extraction.
+* Only allows http:// and https:// URLs.
+*/
+function isValidUrl(url) {
+	if (url.length === 0 || url === "#") return false;
+	if (!/^https?:\/\//i.test(url)) return false;
+	try {
+		new URL(url);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+//#endregion
+exports.extractLinks = extractLinks;

package/dist/extract-links.d.cts

@@ -0,0 +1,18 @@
+//#region src/extract-links.d.ts
+/**
+ * Supported media types for link extraction.
+ *
+ * @since 0.1.0
+ */
+type MediaType = "text/plain" | "text/markdown" | "text/html";
+/**
+ * Extracts URLs from text based on the media type.
+ *
+ * @param text The text to extract URLs from.
+ * @param mediaType The media type of the text.
+ * @returns An array of unique URLs found in the text.
+ * @since 0.1.0
+ */
+declare function extractLinks(text: string, mediaType: MediaType): readonly string[];
+//#endregion
+export { MediaType, extractLinks };

package/dist/extract-links.d.ts

@@ -0,0 +1,18 @@
+//#region src/extract-links.d.ts
+/**
+ * Supported media types for link extraction.
+ *
+ * @since 0.1.0
+ */
+type MediaType = "text/plain" | "text/markdown" | "text/html";
+/**
+ * Extracts URLs from text based on the media type.
+ *
+ * @param text The text to extract URLs from.
+ * @param mediaType The media type of the text.
+ * @returns An array of unique URLs found in the text.
+ * @since 0.1.0
+ */
+declare function extractLinks(text: string, mediaType: MediaType): readonly string[];
+//#endregion
+export { MediaType, extractLinks };

package/dist/extract-links.js

@@ -0,0 +1,117 @@
+import { parseDocument } from "htmlparser2";
+
+//#region src/extract-links.ts
+/**
+* Extracts URLs from text based on the media type.
+*
+* @param text The text to extract URLs from.
+* @param mediaType The media type of the text.
+* @returns An array of unique URLs found in the text.
+* @since 0.1.0
+*/
+function extractLinks(text, mediaType) {
+	switch (mediaType) {
+		case "text/plain": return extractFromPlainText(text);
+		case "text/markdown": return extractFromMarkdown(text);
+		case "text/html": return extractFromHtml(text);
+	}
+}
+/**
+* URL pattern for plain text extraction.
+* Matches http:// and https:// URLs.
+*/
+const URL_PATTERN = /https?:\/\/[^\s<>"')\]]+/g;
+/**
+* Characters that should be trimmed from the end of URLs.
+*/
+const TRAILING_PUNCTUATION = /[.,;:!?)]+$/;
+/**
+* Extracts URLs from plain text.
+*/
+function extractFromPlainText(text) {
+	const matches = text.match(URL_PATTERN);
+	if (matches == null) return [];
+	const urls = /* @__PURE__ */ new Set();
+	for (const match of matches) {
+		const cleanUrl = match.replace(TRAILING_PUNCTUATION, "");
+		if (isValidUrl(cleanUrl)) urls.add(cleanUrl);
+	}
+	return [...urls];
+}
+/**
+* Markdown link patterns.
+*/
+const MARKDOWN_INLINE_LINK = /\[([^\]]*)\]\(([^)]+)\)/g;
+const MARKDOWN_REFERENCE_LINK = /^\[([^\]]+)\]:\s*(\S+)/gm;
+const MARKDOWN_AUTOLINK = /<(https?:\/\/[^>]+)>/g;
+const MARKDOWN_CODE_BLOCK = /```[\s\S]*?```|`[^`]+`/g;
+/**
+* Extracts URLs from Markdown text.
+*/
+function extractFromMarkdown(text) {
+	const textWithoutCode = text.replace(MARKDOWN_CODE_BLOCK, "");
+	const urls = /* @__PURE__ */ new Set();
+	let match;
+	MARKDOWN_INLINE_LINK.lastIndex = 0;
+	while ((match = MARKDOWN_INLINE_LINK.exec(textWithoutCode)) != null) {
+		const url = match[2];
+		if (isValidUrl(url)) urls.add(url);
+	}
+	MARKDOWN_REFERENCE_LINK.lastIndex = 0;
+	while ((match = MARKDOWN_REFERENCE_LINK.exec(textWithoutCode)) != null) {
+		const url = match[2];
+		if (isValidUrl(url)) urls.add(url);
+	}
+	MARKDOWN_AUTOLINK.lastIndex = 0;
+	while ((match = MARKDOWN_AUTOLINK.exec(textWithoutCode)) != null) {
+		const url = match[1];
+		if (isValidUrl(url)) urls.add(url);
+	}
+	const bareUrls = extractFromPlainText(textWithoutCode);
+	for (const url of bareUrls) urls.add(url);
+	return [...urls];
+}
+/**
+* Determines if a node is an element.
+*/
+function isElement(node) {
+	return node.type === "tag";
+}
+/**
+* Extracts URLs from HTML.
+*/
+function extractFromHtml(html) {
+	const doc = parseDocument(html, {
+		lowerCaseTags: true,
+		lowerCaseAttributeNames: true
+	});
+	const urls = /* @__PURE__ */ new Set();
+	function traverse(node) {
+		if (isElement(node)) {
+			if (node.name === "a") {
+				const href = node.attribs.href;
+				if (href != null && isValidUrl(href)) urls.add(href);
+			}
+			for (const child of node.children) traverse(child);
+		}
+	}
+	for (const child of doc.children) traverse(child);
+	return [...urls];
+}
+/**
+* Checks if a URL is valid for extraction.
+* Only allows http:// and https:// URLs.
+*/
+function isValidUrl(url) {
+	if (url.length === 0 || url === "#") return false;
+	if (!/^https?:\/\//i.test(url)) return false;
+	try {
+		new URL(url);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+//#endregion
+export { extractLinks };

package/dist/fetch.cjs

@@ -0,0 +1,235 @@
+const require_extract_links = require('./extract-links.cjs');
+let _logtape_logtape = require("@logtape/logtape");
+let _mozilla_readability = require("@mozilla/readability");
+let linkedom = require("linkedom");
+let zod = require("zod");
+
+//#region src/fetch.ts
+const logger = (0, _logtape_logtape.getLogger)([
+	"vertana",
+	"context-web",
+	"fetch"
+]);
+/**
+* Extracts the main content from an HTML page using Mozilla's Readability.
+*
+* @param html The HTML content to extract from.
+* @param url The URL of the page (used for resolving relative links).
+* @returns The extracted content, or null if extraction failed.
+* @since 0.1.0
+*/
+function extractContent(html, url) {
+	const document = (0, linkedom.parseHTML)(html, "text/html").document;
+	const baseElement = document.createElement("base");
+	baseElement.href = url;
+	document.head.appendChild(baseElement);
+	const article = new _mozilla_readability.Readability(document).parse();
+	if (article == null) return null;
+	const title = article.title ?? "";
+	const content = article.textContent ?? "";
+	if (title.length === 0 && content.length === 0) return null;
+	return {
+		title,
+		content,
+		byline: article.byline ?? void 0,
+		excerpt: article.excerpt ?? void 0
+	};
+}
+/**
+* Fetches a URL and extracts its main content.
+*
+* @param url The URL to fetch.
+* @param options Fetch options.
+* @returns The extracted content, or null if fetch or extraction failed.
+*/
+async function fetchAndExtract(url, options) {
+	const timeout = options?.timeout ?? 1e4;
+	logger.debug("Fetching URL: {url}...", { url });
+	try {
+		const controller = new AbortController();
+		const timeoutId = setTimeout(() => controller.abort(), timeout);
+		if (options?.signal != null) options.signal.addEventListener("abort", () => controller.abort());
+		const response = await fetch(url, {
+			signal: controller.signal,
+			headers: {
+				"User-Agent": "Mozilla/5.0 (compatible; Vertana/0.1; +https://vertana.org)",
+				Accept: "text/html,application/xhtml+xml"
+			}
+		});
+		clearTimeout(timeoutId);
+		if (!response.ok) {
+			logger.warn("Failed to fetch URL: {url}, status: {status}", {
+				url,
+				status: response.status
+			});
+			return null;
+		}
+		const contentType = response.headers.get("content-type");
+		if (contentType != null && !contentType.includes("text/html")) {
+			logger.debug("Skipping non-HTML content: {url}, type: {contentType}", {
+				url,
+				contentType
+			});
+			return null;
+		}
+		const content = extractContent(await response.text(), url);
+		if (content == null) {
+			logger.debug("Failed to extract content from: {url}", { url });
+			return null;
+		}
+		logger.debug("Extracted content from: {url}, title: {title}", {
+			url,
+			title: content.title
+		});
+		return content;
+	} catch (error) {
+		if (error instanceof Error && error.name === "AbortError") logger.debug("Fetch aborted for: {url}", { url });
+		else logger.warn("Error fetching URL: {url}, error: {error}", {
+			url,
+			error: String(error)
+		});
+		return null;
+	}
+}
+/**
+* A passive context source that fetches a single web page and extracts
+* its main content.
+*
+* This source is exposed as a tool that the LLM can call when it needs
+* to fetch additional context from a specific URL.
+*
+* @example
+* ```typescript
+* import { translate } from "@vertana/facade";
+* import { fetchWebPage } from "@vertana/context-web";
+*
+* const result = await translate(model, "ko", text, {
+*   contextSources: [fetchWebPage],
+* });
+* ```
+*
+* @since 0.1.0
+*/
+const fetchWebPage = {
+	name: "fetch-web-page",
+	description: "Fetches a web page and extracts its main content. Use this when you need additional context from a linked article or page.",
+	mode: "passive",
+	parameters: zod.z.object({ url: zod.z.string().url().describe("The URL of the web page to fetch") }),
+	async gather(params, options) {
+		const content = await fetchAndExtract(params.url, { signal: options?.signal });
+		if (content == null) return {
+			content: `Failed to fetch or extract content from: ${params.url}`,
+			metadata: {
+				url: params.url,
+				success: false
+			}
+		};
+		return {
+			content: formatContent(content, params.url),
+			metadata: {
+				url: params.url,
+				title: content.title,
+				success: true
+			}
+		};
+	}
+};
+/**
+* Creates a required context source that extracts all links from the given
+* text and fetches their content.
+*
+* This source is invoked automatically before translation begins, providing
+* context from all linked pages.
+*
+* @param options Options for the context source.
+* @returns A required context source.
+*
+* @example
+* ```typescript
+* import { translate } from "@vertana/facade";
+* import { fetchLinkedPages } from "@vertana/context-web";
+*
+* const text = "Check out https://example.com for details.";
+* const result = await translate(model, "ko", text, {
+*   contextSources: [
+*     fetchLinkedPages({ text, mediaType: "text/plain" }),
+*   ],
+* });
+* ```
+*
+* @since 0.1.0
+*/
+function fetchLinkedPages(options) {
+	const maxLinks = options.maxLinks ?? 10;
+	const timeout = options.timeout ?? 1e4;
+	const linksToFetch = require_extract_links.extractLinks(options.text, options.mediaType).slice(0, maxLinks);
+	return {
+		name: "fetch-linked-pages",
+		description: `Fetches content from ${linksToFetch.length} linked page(s) to provide additional context for translation.`,
+		mode: "required",
+		async gather(gatherOptions) {
+			if (linksToFetch.length === 0) {
+				logger.debug("No links to fetch.");
+				return {
+					content: "",
+					metadata: {
+						linkCount: 0,
+						fetchedCount: 0
+					}
+				};
+			}
+			logger.info("Fetching {count} linked page(s)...", { count: linksToFetch.length });
+			const results = [];
+			for (const url of linksToFetch) {
+				gatherOptions?.signal?.throwIfAborted();
+				const content = await fetchAndExtract(url, {
+					signal: gatherOptions?.signal,
+					timeout
+				});
+				if (content != null) results.push({
+					url,
+					content
+				});
+			}
+			if (results.length === 0) {
+				logger.debug("No content could be extracted from any linked pages.");
+				return {
+					content: "",
+					metadata: {
+						linkCount: linksToFetch.length,
+						fetchedCount: 0
+					}
+				};
+			}
+			logger.info("Successfully extracted content from {count} of {total} page(s).", {
+				count: results.length,
+				total: linksToFetch.length
+			});
+			return {
+				content: results.map(({ url, content }) => formatContent(content, url)).join("\n\n---\n\n"),
+				metadata: {
+					linkCount: linksToFetch.length,
+					fetchedCount: results.length,
+					urls: results.map((r) => r.url)
+				}
+			};
+		}
+	};
+}
+/**
+* Formats extracted content for inclusion in the translation context.
+*/
+function formatContent(content, url) {
+	const parts = [];
+	parts.push(`# ${content.title}`);
+	parts.push(`Source: ${url}`);
+	if (content.byline != null) parts.push(`Author: ${content.byline}`);
+	parts.push("");
+	parts.push(content.content);
+	return parts.join("\n");
+}
+
+//#endregion
+exports.extractContent = extractContent;
+exports.fetchLinkedPages = fetchLinkedPages;
+exports.fetchWebPage = fetchWebPage;

package/dist/fetch.d.cts

@@ -0,0 +1,121 @@
+import { MediaType } from "./extract-links.cjs";
+import { PassiveContextSource, RequiredContextSource } from "@vertana/core/context";
+
+//#region src/fetch.d.ts
+
+/**
+ * Result of extracting content from a web page.
+ *
+ * @since 0.1.0
+ */
+interface ExtractedContent {
+  /**
+   * The title of the article.
+   */
+  readonly title: string;
+  /**
+   * The extracted main content as plain text.
+   */
+  readonly content: string;
+  /**
+   * The byline (author) if available.
+   */
+  readonly byline?: string;
+  /**
+   * The excerpt if available.
+   */
+  readonly excerpt?: string;
+}
+/**
+ * Extracts the main content from an HTML page using Mozilla's Readability.
+ *
+ * @param html The HTML content to extract from.
+ * @param url The URL of the page (used for resolving relative links).
+ * @returns The extracted content, or null if extraction failed.
+ * @since 0.1.0
+ */
+declare function extractContent(html: string, url: string): ExtractedContent | null;
+/**
+ * Parameters for the fetchWebPage context source.
+ */
+interface FetchWebPageParams {
+  /**
+   * The URL to fetch.
+   */
+  readonly url: string;
+}
+/**
+ * A passive context source that fetches a single web page and extracts
+ * its main content.
+ *
+ * This source is exposed as a tool that the LLM can call when it needs
+ * to fetch additional context from a specific URL.
+ *
+ * @example
+ * ```typescript
+ * import { translate } from "@vertana/facade";
+ * import { fetchWebPage } from "@vertana/context-web";
+ *
+ * const result = await translate(model, "ko", text, {
+ *   contextSources: [fetchWebPage],
+ * });
+ * ```
+ *
+ * @since 0.1.0
+ */
+declare const fetchWebPage: PassiveContextSource<FetchWebPageParams>;
+/**
+ * Options for creating a fetchLinkedPages context source.
+ *
+ * @since 0.1.0
+ */
+interface FetchLinkedPagesOptions {
+  /**
+   * The text to extract links from.
+   */
+  readonly text: string;
+  /**
+   * The media type of the text.
+   */
+  readonly mediaType: MediaType;
+  /**
+   * Maximum number of links to fetch.
+   *
+   * @default 10
+   */
+  readonly maxLinks?: number;
+  /**
+   * Timeout for each fetch request in milliseconds.
+   *
+   * @default 10000
+   */
+  readonly timeout?: number;
+}
+/**
+ * Creates a required context source that extracts all links from the given
+ * text and fetches their content.
+ *
+ * This source is invoked automatically before translation begins, providing
+ * context from all linked pages.
+ *
+ * @param options Options for the context source.
+ * @returns A required context source.
+ *
+ * @example
+ * ```typescript
+ * import { translate } from "@vertana/facade";
+ * import { fetchLinkedPages } from "@vertana/context-web";
+ *
+ * const text = "Check out https://example.com for details.";
+ * const result = await translate(model, "ko", text, {
+ *   contextSources: [
+ *     fetchLinkedPages({ text, mediaType: "text/plain" }),
+ *   ],
+ * });
+ * ```
+ *
+ * @since 0.1.0
+ */
+declare function fetchLinkedPages(options: FetchLinkedPagesOptions): RequiredContextSource;
+//#endregion
+export { ExtractedContent, FetchLinkedPagesOptions, extractContent, fetchLinkedPages, fetchWebPage };

package/dist/fetch.d.ts

@@ -0,0 +1,121 @@
+import { MediaType } from "./extract-links.js";
+import { PassiveContextSource, RequiredContextSource } from "@vertana/core/context";
+
+//#region src/fetch.d.ts
+
+/**
+ * Result of extracting content from a web page.
+ *
+ * @since 0.1.0
+ */
+interface ExtractedContent {
+  /**
+   * The title of the article.
+   */
+  readonly title: string;
+  /**
+   * The extracted main content as plain text.
+   */
+  readonly content: string;
+  /**
+   * The byline (author) if available.
+   */
+  readonly byline?: string;
+  /**
+   * The excerpt if available.
+   */
+  readonly excerpt?: string;
+}
+/**
+ * Extracts the main content from an HTML page using Mozilla's Readability.
+ *
+ * @param html The HTML content to extract from.
+ * @param url The URL of the page (used for resolving relative links).
+ * @returns The extracted content, or null if extraction failed.
+ * @since 0.1.0
+ */
+declare function extractContent(html: string, url: string): ExtractedContent | null;
+/**
+ * Parameters for the fetchWebPage context source.
+ */
+interface FetchWebPageParams {
+  /**
+   * The URL to fetch.
+   */
+  readonly url: string;
+}
+/**
+ * A passive context source that fetches a single web page and extracts
+ * its main content.
+ *
+ * This source is exposed as a tool that the LLM can call when it needs
+ * to fetch additional context from a specific URL.
+ *
+ * @example
+ * ```typescript
+ * import { translate } from "@vertana/facade";
+ * import { fetchWebPage } from "@vertana/context-web";
+ *
+ * const result = await translate(model, "ko", text, {
+ *   contextSources: [fetchWebPage],
+ * });
+ * ```
+ *
+ * @since 0.1.0
+ */
+declare const fetchWebPage: PassiveContextSource<FetchWebPageParams>;
+/**
+ * Options for creating a fetchLinkedPages context source.
+ *
+ * @since 0.1.0
+ */
+interface FetchLinkedPagesOptions {
+  /**
+   * The text to extract links from.
+   */
+  readonly text: string;
+  /**
+   * The media type of the text.
+   */
+  readonly mediaType: MediaType;
+  /**
+   * Maximum number of links to fetch.
+   *
+   * @default 10
+   */
+  readonly maxLinks?: number;
+  /**
+   * Timeout for each fetch request in milliseconds.
+   *
+   * @default 10000
+   */
+  readonly timeout?: number;
+}
+/**
+ * Creates a required context source that extracts all links from the given
+ * text and fetches their content.
+ *
+ * This source is invoked automatically before translation begins, providing
+ * context from all linked pages.
+ *
+ * @param options Options for the context source.
+ * @returns A required context source.
+ *
+ * @example
+ * ```typescript
+ * import { translate } from "@vertana/facade";
+ * import { fetchLinkedPages } from "@vertana/context-web";
+ *
+ * const text = "Check out https://example.com for details.";
+ * const result = await translate(model, "ko", text, {
+ *   contextSources: [
+ *     fetchLinkedPages({ text, mediaType: "text/plain" }),
+ *   ],
+ * });
+ * ```
+ *
+ * @since 0.1.0
+ */
+declare function fetchLinkedPages(options: FetchLinkedPagesOptions): RequiredContextSource;
+//#endregion
+export { ExtractedContent, FetchLinkedPagesOptions, extractContent, fetchLinkedPages, fetchWebPage };

package/dist/fetch.js

@@ -0,0 +1,233 @@
+import { extractLinks } from "./extract-links.js";
+import { getLogger } from "@logtape/logtape";
+import { Readability } from "@mozilla/readability";
+import { parseHTML } from "linkedom";
+import { z } from "zod";
+
+//#region src/fetch.ts
+const logger = getLogger([
+	"vertana",
+	"context-web",
+	"fetch"
+]);
+/**
+* Extracts the main content from an HTML page using Mozilla's Readability.
+*
+* @param html The HTML content to extract from.
+* @param url The URL of the page (used for resolving relative links).
+* @returns The extracted content, or null if extraction failed.
+* @since 0.1.0
+*/
+function extractContent(html, url) {
+	const document = parseHTML(html, "text/html").document;
+	const baseElement = document.createElement("base");
+	baseElement.href = url;
+	document.head.appendChild(baseElement);
+	const article = new Readability(document).parse();
+	if (article == null) return null;
+	const title = article.title ?? "";
+	const content = article.textContent ?? "";
+	if (title.length === 0 && content.length === 0) return null;
+	return {
+		title,
+		content,
+		byline: article.byline ?? void 0,
+		excerpt: article.excerpt ?? void 0
+	};
+}
+/**
+* Fetches a URL and extracts its main content.
+*
+* @param url The URL to fetch.
+* @param options Fetch options.
+* @returns The extracted content, or null if fetch or extraction failed.
+*/
+async function fetchAndExtract(url, options) {
+	const timeout = options?.timeout ?? 1e4;
+	logger.debug("Fetching URL: {url}...", { url });
+	try {
+		const controller = new AbortController();
+		const timeoutId = setTimeout(() => controller.abort(), timeout);
+		if (options?.signal != null) options.signal.addEventListener("abort", () => controller.abort());
+		const response = await fetch(url, {
+			signal: controller.signal,
+			headers: {
+				"User-Agent": "Mozilla/5.0 (compatible; Vertana/0.1; +https://vertana.org)",
+				Accept: "text/html,application/xhtml+xml"
+			}
+		});
+		clearTimeout(timeoutId);
+		if (!response.ok) {
+			logger.warn("Failed to fetch URL: {url}, status: {status}", {
+				url,
+				status: response.status
+			});
+			return null;
+		}
+		const contentType = response.headers.get("content-type");
+		if (contentType != null && !contentType.includes("text/html")) {
+			logger.debug("Skipping non-HTML content: {url}, type: {contentType}", {
+				url,
+				contentType
+			});
+			return null;
+		}
+		const content = extractContent(await response.text(), url);
+		if (content == null) {
+			logger.debug("Failed to extract content from: {url}", { url });
+			return null;
+		}
+		logger.debug("Extracted content from: {url}, title: {title}", {
+			url,
+			title: content.title
+		});
+		return content;
+	} catch (error) {
+		if (error instanceof Error && error.name === "AbortError") logger.debug("Fetch aborted for: {url}", { url });
+		else logger.warn("Error fetching URL: {url}, error: {error}", {
+			url,
+			error: String(error)
+		});
+		return null;
+	}
+}
+/**
+* A passive context source that fetches a single web page and extracts
+* its main content.
+*
+* This source is exposed as a tool that the LLM can call when it needs
+* to fetch additional context from a specific URL.
+*
+* @example
+* ```typescript
+* import { translate } from "@vertana/facade";
+* import { fetchWebPage } from "@vertana/context-web";
+*
+* const result = await translate(model, "ko", text, {
+*   contextSources: [fetchWebPage],
+* });
+* ```
+*
+* @since 0.1.0
+*/
+const fetchWebPage = {
+	name: "fetch-web-page",
+	description: "Fetches a web page and extracts its main content. Use this when you need additional context from a linked article or page.",
+	mode: "passive",
+	parameters: z.object({ url: z.string().url().describe("The URL of the web page to fetch") }),
+	async gather(params, options) {
+		const content = await fetchAndExtract(params.url, { signal: options?.signal });
+		if (content == null) return {
+			content: `Failed to fetch or extract content from: ${params.url}`,
+			metadata: {
+				url: params.url,
+				success: false
+			}
+		};
+		return {
+			content: formatContent(content, params.url),
+			metadata: {
+				url: params.url,
+				title: content.title,
+				success: true
+			}
+		};
+	}
+};
+/**
+* Creates a required context source that extracts all links from the given
+* text and fetches their content.
+*
+* This source is invoked automatically before translation begins, providing
+* context from all linked pages.
+*
+* @param options Options for the context source.
+* @returns A required context source.
+*
+* @example
+* ```typescript
+* import { translate } from "@vertana/facade";
+* import { fetchLinkedPages } from "@vertana/context-web";
+*
+* const text = "Check out https://example.com for details.";
+* const result = await translate(model, "ko", text, {
+*   contextSources: [
+*     fetchLinkedPages({ text, mediaType: "text/plain" }),
+*   ],
+* });
+* ```
+*
+* @since 0.1.0
+*/
+function fetchLinkedPages(options) {
+	const maxLinks = options.maxLinks ?? 10;
+	const timeout = options.timeout ?? 1e4;
+	const linksToFetch = extractLinks(options.text, options.mediaType).slice(0, maxLinks);
+	return {
+		name: "fetch-linked-pages",
+		description: `Fetches content from ${linksToFetch.length} linked page(s) to provide additional context for translation.`,
+		mode: "required",
+		async gather(gatherOptions) {
+			if (linksToFetch.length === 0) {
+				logger.debug("No links to fetch.");
+				return {
+					content: "",
+					metadata: {
+						linkCount: 0,
+						fetchedCount: 0
+					}
+				};
+			}
+			logger.info("Fetching {count} linked page(s)...", { count: linksToFetch.length });
+			const results = [];
+			for (const url of linksToFetch) {
+				gatherOptions?.signal?.throwIfAborted();
+				const content = await fetchAndExtract(url, {
+					signal: gatherOptions?.signal,
+					timeout
+				});
+				if (content != null) results.push({
+					url,
+					content
+				});
+			}
+			if (results.length === 0) {
+				logger.debug("No content could be extracted from any linked pages.");
+				return {
+					content: "",
+					metadata: {
+						linkCount: linksToFetch.length,
+						fetchedCount: 0
+					}
+				};
+			}
+			logger.info("Successfully extracted content from {count} of {total} page(s).", {
+				count: results.length,
+				total: linksToFetch.length
+			});
+			return {
+				content: results.map(({ url, content }) => formatContent(content, url)).join("\n\n---\n\n"),
+				metadata: {
+					linkCount: linksToFetch.length,
+					fetchedCount: results.length,
+					urls: results.map((r) => r.url)
+				}
+			};
+		}
+	};
+}
+/**
+* Formats extracted content for inclusion in the translation context.
+*/
+function formatContent(content, url) {
+	const parts = [];
+	parts.push(`# ${content.title}`);
+	parts.push(`Source: ${url}`);
+	if (content.byline != null) parts.push(`Author: ${content.byline}`);
+	parts.push("");
+	parts.push(content.content);
+	return parts.join("\n");
+}
+
+//#endregion
+export { extractContent, fetchLinkedPages, fetchWebPage };

package/dist/index.cjs

@@ -0,0 +1,7 @@
+const require_extract_links = require('./extract-links.cjs');
+const require_fetch = require('./fetch.cjs');
+
+exports.extractContent = require_fetch.extractContent;
+exports.extractLinks = require_extract_links.extractLinks;
+exports.fetchLinkedPages = require_fetch.fetchLinkedPages;
+exports.fetchWebPage = require_fetch.fetchWebPage;

package/dist/index.d.cts

@@ -0,0 +1,3 @@
+import { MediaType, extractLinks } from "./extract-links.cjs";
+import { ExtractedContent, FetchLinkedPagesOptions, extractContent, fetchLinkedPages, fetchWebPage } from "./fetch.cjs";
+export { type ExtractedContent, type FetchLinkedPagesOptions, type MediaType, extractContent, extractLinks, fetchLinkedPages, fetchWebPage };

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+import { MediaType, extractLinks } from "./extract-links.js";
+import { ExtractedContent, FetchLinkedPagesOptions, extractContent, fetchLinkedPages, fetchWebPage } from "./fetch.js";
+export { type ExtractedContent, type FetchLinkedPagesOptions, type MediaType, extractContent, extractLinks, fetchLinkedPages, fetchWebPage };

package/dist/index.js

@@ -0,0 +1,4 @@
+import { extractLinks } from "./extract-links.js";
+import { extractContent, fetchLinkedPages, fetchWebPage } from "./fetch.js";
+
+export { extractContent, extractLinks, fetchLinkedPages, fetchWebPage };

package/package.json

@@ -0,0 +1,92 @@
+{
+  "name": "@vertana/context-web",
+  "version": "0.1.0-dev.1",
+  "description": "Web context gathering for Vertana - fetch and extract content from linked pages",
+  "keywords": [
+    "LLM",
+    "translation",
+    "context",
+    "web",
+    "readability"
+  ],
+  "license": "MIT",
+  "author": {
+    "name": "Hong Minhee",
+    "email": "hong@minhee.org",
+    "url": "https://hongminhee.org/"
+  },
+  "homepage": "https://vertana.org/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/dahlia/vertana.git",
+    "directory": "packages/context-web"
+  },
+  "bugs": {
+    "url": "https://github.com/dahlia/vertana/issues"
+  },
+  "funding": [
+    "https://github.com/sponsors/dahlia"
+  ],
+  "engines": {
+    "node": ">=20.0.0",
+    "bun": ">=1.2.0",
+    "deno": ">=2.3.0"
+  },
+  "files": [
+    "dist/",
+    "package.json",
+    "README.md"
+  ],
+  "type": "module",
+  "module": "./dist/index.js",
+  "main": "./dist/index.cjs",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": {
+        "require": "./dist/index.d.cts",
+        "import": "./dist/index.d.ts"
+      },
+      "require": "./dist/index.cjs",
+      "import": "./dist/index.js"
+    },
+    "./fetch": {
+      "types": {
+        "require": "./dist/fetch.d.cts",
+        "import": "./dist/fetch.d.ts"
+      },
+      "require": "./dist/fetch.cjs",
+      "import": "./dist/fetch.js"
+    },
+    "./extract-links": {
+      "types": {
+        "require": "./dist/extract-links.d.cts",
+        "import": "./dist/extract-links.d.ts"
+      },
+      "require": "./dist/extract-links.cjs",
+      "import": "./dist/extract-links.js"
+    }
+  },
+  "sideEffects": false,
+  "dependencies": {
+    "@logtape/logtape": "^1.3.5",
+    "@mozilla/readability": "^0.6.0",
+    "@vertana/core": "",
+    "htmlparser2": "^10.0.0",
+    "linkedom": "^0.18.12",
+    "zod": "4.2.1"
+  },
+  "devDependencies": {
+    "@types/node": "^20.19.9",
+    "tsdown": "^0.18.3",
+    "typescript": "^5.9.3"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "prepublish": "tsdown",
+    "test": "tsdown && node --experimental-transform-types --test --test-concurrency=4",
+    "test:bun": "tsdown && bun test",
+    "test:deno": "deno test --allow-env --allow-net",
+    "test-all": "tsdown && node --experimental-transform-types --test && bun test && deno test"
+  }
+}