@bastani/atomic 0.9.2-alpha.1 → 0.9.3-alpha.1

package/dist/core/tools/fetch-url.d.ts

@@ -0,0 +1,74 @@
+import type { TruncationResult } from "./truncate.ts";
+export declare const FETCH_DEFAULT_MAX_LINES = 300;
+export declare const READ_URL_CACHE_MAX_ENTRIES = 100;
+export interface LineRange {
+    start: number;
+    end?: number;
+}
+export interface ParsedReadUrlTarget {
+    url: string;
+    raw: boolean;
+    offset?: number;
+    limit?: number;
+    ranges?: readonly LineRange[];
+}
+export interface ReadUrlToolDetails {
+    url: string;
+    finalUrl: string;
+    contentType: string;
+    method: string;
+    truncated: boolean;
+    notes: string[];
+    meta?: {
+        artifactId?: string;
+        truncation?: TruncationResult;
+    };
+}
+export interface LoadedPage {
+    ok: boolean;
+    status: number;
+    content: string;
+    contentType: string;
+    finalUrl: string;
+}
+export declare function repairCollapsedScheme(url: string): string;
+export declare function isReadableUrlPath(readPath: string): boolean;
+export declare function parseReadUrlTarget(readPath: string): ParsedReadUrlTarget | null;
+export declare function getReadUrlCacheKey(scope: string, requestedUrl: string, raw: boolean): string;
+export declare function loadPage(url: string, timeoutMs: number, signal?: AbortSignal, accept?: string): Promise<LoadedPage>;
+export interface RenderedUrl {
+    content: string;
+    finalUrl: string;
+    contentType: string;
+    method: string;
+    notes: string[];
+    truncated: boolean;
+}
+export declare function renderUrl(url: string, raw: boolean, signal?: AbortSignal): Promise<RenderedUrl>;
+export declare function buildUrlReadOutput(result: RenderedUrl): string;
+export interface ExecuteReadUrlResult {
+    content: string;
+    details: ReadUrlToolDetails;
+    artifactId?: string;
+}
+export interface LoadedReadUrl {
+    output: string;
+    details: ReadUrlToolDetails;
+    artifactId?: string;
+    raw: boolean;
+}
+/**
+ * Resolve a URL to its full rendered (or raw) output, using the session-scoped
+ * cache and persisting the rendered body as a `read` artifact. Returns the
+ * complete untruncated output so callers can apply their own line selection.
+ */
+export declare function loadReadUrlOutput(scope: string, params: {
+    path: string;
+    raw?: boolean;
+}, artifactsDir: string | undefined, signal?: AbortSignal): Promise<LoadedReadUrl>;
+export declare function executeReadUrl(scope: string, params: {
+    path: string;
+    raw?: boolean;
+}, artifactsDir: string | undefined, signal?: AbortSignal): Promise<ExecuteReadUrlResult>;
+export declare function resetReadUrlCache(): void;
+//# sourceMappingURL=fetch-url.d.ts.map

package/dist/core/tools/fetch-url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-url.d.ts","sourceRoot":"","sources":["../../../src/core/tools/fetch-url.ts"],"names":[],"mappings":"AAuBA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEtD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAM9C,MAAM,WAAW,SAAS;IAAG,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE;AAE1D,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,OAAO,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,gBAAgB,CAAA;KAAE,CAAC;CAC9D;AAQD,MAAM,WAAW,UAAU;IAC1B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CACjB;AASD,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAG3D;AAkDD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAoB/E;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,MAAM,CAE5F;AAwFD,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAqCzH;AA6ED,MAAM,WAAW,WAAW;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;CACnB;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAoCrG;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CAG9D;AAOD,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,kBAAkB,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,kBAAkB,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;CACb;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAA;CAAE,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAgB9K;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAA;CAAE,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA+BlL;AAED,wBAAgB,iBAAiB,IAAI,IAAI,CAExC","sourcesContent":["/// <reference path=\"../../utils/turndown.d.ts\" />\n/**\n * URL fetch / cache / artifact / markdown-discovery pipeline, mirrored from\n * oh-my-pi's `packages/coding-agent/src/tools/fetch.ts` at 15b5c1397fc.\n *\n * Responsibilities:\n *  - parseReadUrlTarget: URL + line selectors (:raw, :N, :A-B, :A+B, :A-B,C-D),\n *    collapsed-scheme repair, host-port protection.\n *  - session-scoped LRU cache keyed by `<scope>::<raw|rendered>::<normalizedUrl>`,\n *    cached under both requested and final (redirected) URLs.\n *  - renderUrl: markdown discovery (alternate <link>, .md suffix, content\n *    negotiation, llms.txt/llms.md endpoints) + native HTML-to-markdown, with\n *    quality gating.\n *  - artifact persistence of the rendered output, with artifactId surfaced in\n *    truncation metadata when the visible output is head-truncated.\n */\nimport { lookup } from \"node:dns/promises\";\nimport { ipFamily, isPrivateIpAddress, normalizeIpLiteralHost } from \"./url-ip-guards.ts\";\nimport { LRUCache } from \"lru-cache\";\nimport TurndownService from \"turndown\";\nimport { Agent, request as undiciRequest, type Dispatcher } from \"undici\";\nimport { getArtifactManager } from \"./artifacts.ts\";\nimport { extractDocumentMarkdown, isDocumentPath } from \"./read-document-extract.ts\";\nimport type { TruncationResult } from \"./truncate.ts\";\n\nexport const FETCH_DEFAULT_MAX_LINES = 300;\nexport const READ_URL_CACHE_MAX_ENTRIES = 100;\nconst REMOTE_FETCH_TIMEOUT_MS = 10_000;\nconst MAX_URL_REDIRECTS = 5;\nconst MAX_URL_RESPONSE_BYTES = 10 * 1024 * 1024;\nconst MIN_CONTENT_LENGTH = 100;\n\nexport interface LineRange { start: number; end?: number }\n\nexport interface ParsedReadUrlTarget {\n\turl: string;\n\traw: boolean;\n\toffset?: number;\n\tlimit?: number;\n\tranges?: readonly LineRange[];\n}\n\nexport interface ReadUrlToolDetails {\n\turl: string;\n\tfinalUrl: string;\n\tcontentType: string;\n\tmethod: string;\n\ttruncated: boolean;\n\tnotes: string[];\n\tmeta?: { artifactId?: string; truncation?: TruncationResult };\n}\n\ninterface ReadUrlCacheEntry {\n\tartifactId?: string;\n\tdetails: ReadUrlToolDetails;\n\toutput: string;\n}\n\nexport interface LoadedPage {\n\tok: boolean;\n\tstatus: number;\n\tcontent: string;\n\tcontentType: string;\n\tfinalUrl: string;\n}\n\nconst readUrlCache = new LRUCache<string, ReadUrlCacheEntry>({ max: READ_URL_CACHE_MAX_ENTRIES });\nlet turndown: TurndownService | undefined;\nfunction getTurndown(): TurndownService {\n\tif (!turndown) turndown = new TurndownService({ headingStyle: \"atx\", codeBlockStyle: \"fenced\", bulletListMarker: \"-\" });\n\treturn turndown;\n}\n\nexport function repairCollapsedScheme(url: string): string {\n\treturn url.replace(/^(https?):\\/([^/])/i, \"$1://$2\");\n}\n\nexport function isReadableUrlPath(readPath: string): boolean {\n\tconst repaired = repairCollapsedScheme(readPath);\n\treturn /^https?:\\/\\//i.test(repaired) || /^www\\./i.test(readPath);\n}\n\nfunction normalizeUrlForCache(url: string): string {\n\tconst repaired = repairCollapsedScheme(url);\n\tif (/^[a-z][a-z0-9+.-]*:\\/\\//i.test(repaired)) return repaired;\n\treturn /^https?:\\/\\//i.test(repaired) ? repaired : `https://${repaired}`;\n}\nconst RANGE_TOKEN_RE = /^(raw|\\d+(?:\\+(?:\\d+)|(?:-|\\.\\.)\\d+)?)$/i;\nconst MULTI_RANGE_RE = /^\\d+(?:(?:-|\\.\\.|,|\\+)\\d+)*$/;\n\nfunction isUrlSelectorToken(token: string): boolean {\n\treturn /^raw$/i.test(token) || RANGE_TOKEN_RE.test(token) || MULTI_RANGE_RE.test(token);\n}\n\nfunction parseSingleRange(token: string): LineRange | undefined {\n\tconst plus = token.match(/^(\\d+)\\+(\\d+)$/);\n\tif (plus) {\n\t\tconst start = Number.parseInt(plus[1]!, 10), count = Number.parseInt(plus[2]!, 10);\n\t\treturn start < 1 || count < 1 ? undefined : { start, end: start + count - 1 };\n\t}\n\tconst range = token.match(/^(\\d+)(?:-|\\.\\.)(\\d+)$/);\n\tif (range) { const start = Number.parseInt(range[1]!, 10); const end = Number.parseInt(range[2]!, 10); return start < 1 || end < start ? undefined : { start, end }; }\n\tconst single = token.match(/^(\\d+)$/);\n\tif (single) { const start = Number.parseInt(single[1]!, 10); return start < 1 ? undefined : { start }; }\n\treturn undefined;\n}\n\nfunction parseRangeGroup(token: string): LineRange[] | undefined {\n\tconst groups = token.split(\",\");\n\tconst ranges: LineRange[] = [];\n\tfor (const group of groups) { const r = parseSingleRange(group); if (!r) return undefined; ranges.push(r); }\n\treturn ranges;\n}\n\nfunction tryExtractEmbeddedUrlSelector(readPath: string): { path: string; sels: string[] } | null {\n\tlet basePath = readPath;\n\tconst sels: string[] = [];\n\tfor (;;) {\n\t\tconst lastColonIndex = basePath.lastIndexOf(\":\");\n\t\tif (lastColonIndex <= 0) break;\n\t\tconst candidate = basePath.slice(lastColonIndex + 1);\n\t\tconst remainder = basePath.slice(0, lastColonIndex);\n\t\tif (!isReadableUrlPath(remainder) || !isUrlSelectorToken(candidate)) break;\n\t\ttry { new URL(remainder.startsWith(\"http://\") || remainder.startsWith(\"https://\") ? remainder : `https://${remainder}`); } catch { break; }\n\t\tsels.unshift(candidate);\n\t\tbasePath = remainder;\n\t}\n\treturn sels.length === 0 ? null : { path: basePath, sels };\n}\n\nexport function parseReadUrlTarget(readPath: string): ParsedReadUrlTarget | null {\n\tconst repaired = repairCollapsedScheme(readPath);\n\tconst embedded = tryExtractEmbeddedUrlSelector(repaired);\n\tconst urlPath = embedded?.path ?? repaired;\n\tif (!isReadableUrlPath(urlPath)) return null;\n\tlet raw = false;\n\tlet ranges: LineRange[] | undefined;\n\tfor (const sel of embedded?.sels ?? []) {\n\t\tif (/^raw$/i.test(sel)) { raw = true; continue; }\n\t\tif (ranges !== undefined) throw new Error(\"URL selector has multiple range groups; combine them with commas (e.g. `:5-10,20-30`).\");\n\t\tconst parsed = parseRangeGroup(sel);\n\t\tif (!parsed) throw new Error(`Invalid URL line selector: ${sel}`);\n\t\tranges = parsed;\n\t}\n\tif (!ranges || ranges.length === 0) return { url: urlPath, raw };\n\tif (ranges.length === 1) {\n\t\tconst r = ranges[0]!;\n\t\treturn { url: urlPath, raw, offset: r.start, limit: r.end !== undefined ? r.end - r.start + 1 : undefined };\n\t}\n\treturn { url: urlPath, raw, ranges };\n}\n\nexport function getReadUrlCacheKey(scope: string, requestedUrl: string, raw: boolean): string {\n\treturn `${scope}::${raw ? \"raw\" : \"rendered\"}::${normalizeUrlForCache(requestedUrl)}`;\n}\n\nfunction looksLikeHtml(text: string): boolean {\n\treturn /<html[\\s>]/i.test(text) || (/<head[\\s>]/i.test(text) && /<body[\\s>]/i.test(text));\n}\n\nfunction documentExtensionFromContentType(contentType: string): string | undefined {\n\tif (/pdf/i.test(contentType)) return \".pdf\";\n\tif (/msword/i.test(contentType)) return \".doc\";\n\tif (/wordprocessingml|officedocument\\.word/i.test(contentType)) return \".docx\";\n\tif (/presentationml|officedocument\\.presentation/i.test(contentType)) return \".pptx\";\n\tif (/ms-powerpoint|vnd\\.ms-powerpoint/i.test(contentType)) return \".ppt\";\n\tif (/spreadsheetml|officedocument\\.spreadsheet/i.test(contentType)) return \".xlsx\";\n\tif (/ms-excel|vnd\\.ms-excel/i.test(contentType)) return \".xls\";\n\tif (/epub/i.test(contentType)) return \".epub\";\n\tif (/\\bjson\\b|ipynb|jupyter/i.test(contentType)) return \".ipynb\";\n\tif (/rtf/i.test(contentType)) return \".rtf\";\n\treturn undefined;\n}\n\ninterface SafeFetchAddress { address: string; family: 4 | 6; /** True only when the address came from DNS resolution and must be pinned to prevent rebinding at connect time. */ pinned: boolean }\ntype LookupCallback = (error: NodeJS.ErrnoException | null, address: string | SafeFetchAddress[], family?: number) => void;\n\nfunction createPinnedDispatcher(pinned: SafeFetchAddress): Agent {\n\treturn new Agent({ connect: { lookup: (_hostname: string, options: { all?: boolean }, callback: LookupCallback) => { if (options.all) callback(null, [pinned]); else callback(null, pinned.address, pinned.family); } } });\n}\n\nasync function closePinnedDispatcher(dispatcher: Agent | undefined): Promise<void> {\n\tconst close = (dispatcher as { close?: () => Promise<void> | void } | undefined)?.close;\n\tif (typeof close === \"function\") await close.call(dispatcher);\n}\n/**\n * Fetch a URL through undici with the pinned dispatcher so the validated\n * address is the one actually connected to. Unlike `globalThis.fetch`, which\n * silently ignores the undici `dispatcher` option under Bun's compiled binary\n * (reopening a DNS-rebinding TOCTOU window for hostname targets), undici's own\n * client honors the dispatcher's custom `lookup` regardless of host runtime.\n * Adapts the undici response to the WHATWG-Response shape loadPage consumes.\n */\nasync function pinnedFetch(url: string, dispatcher: Dispatcher, options: { signal: AbortSignal; headers?: Record<string, string> }): Promise<Response> {\n\tconst result = await undiciRequest(url, { method: \"GET\", dispatcher, signal: options.signal, headers: options.headers });\n\tconst body = result.body;\n\tconst status = result.statusCode;\n\tconst headers = new Headers();\n\tfor (const [key, value] of Object.entries(result.headers as Record<string, string | string[] | undefined>)) {\n\t\tif (value === undefined) continue;\n\t\tconst values = Array.isArray(value) ? value : [value];\n\t\tfor (const entry of values) headers.append(key, entry);\n\t}\n\tconst stream = new ReadableStream({\n\t\tstart(controller) { body.on(\"data\", (chunk: Buffer) => controller.enqueue(new Uint8Array(chunk))); body.on(\"end\", () => controller.close()); body.on(\"error\", (error: unknown) => controller.error(error)); },\n\t\tcancel() { body.destroy(); },\n\t});\n\treturn new Response(stream, { status, headers });\n}\n\n\nasync function assertSafeFetchUrl(url: string): Promise<SafeFetchAddress | undefined> {\n\tconst parsed = new URL(url);\n\tif (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") throw new Error(`Unsupported URL protocol: ${parsed.protocol}`);\n\tif (process.env.ATOMIC_ALLOW_PRIVATE_URL_READS === \"1\") return undefined;\n\tconst hostname = parsed.hostname.replace(/^\\[|\\]$/g, \"\").replace(/\\.$/, \"\").toLowerCase();\n\tif (hostname === \"localhost\" || hostname.endsWith(\".localhost\") || hostname === \"metadata.google.internal\") throw new Error(`Refusing to fetch private or metadata URL: ${url}`);\n\tconst literalAddress = normalizeIpLiteralHost(hostname);\n\tif (literalAddress) { if (isPrivateIpAddress(literalAddress)) throw new Error(`Refusing to fetch private or metadata URL: ${url}`); return { address: literalAddress, family: ipFamily(literalAddress), pinned: false }; }\n\tconst addresses = await lookup(hostname, { all: true }).catch((error: unknown) => { throw new Error(`Could not resolve URL host ${hostname}: ${error instanceof Error ? error.message : String(error)}`); });\n\tif (addresses.length === 0) throw new Error(`Could not resolve URL host ${hostname}: no addresses returned`);\n\tfor (const address of addresses) if (isPrivateIpAddress(address.address)) throw new Error(`Refusing to fetch private or metadata URL: ${url}`);\n\tconst first = addresses[0]!;\n\treturn { address: first.address, family: first.family === 6 ? 6 : 4, pinned: true };\n}\n\nasync function readResponseTextWithLimit(response: Response): Promise<string> {\n\tif (!response.body) { const buffer = Buffer.from(await response.arrayBuffer()); if (buffer.length > MAX_URL_RESPONSE_BYTES) throw new Error(`URL response exceeds ${MAX_URL_RESPONSE_BYTES} byte limit`); return buffer.toString(\"utf8\"); }\n\tconst reader = response.body.getReader();\n\tconst decoder = new TextDecoder();\n\tconst chunks: string[] = [];\n\tlet totalBytes = 0;\n\tfor (;;) {\n\t\tconst { done, value } = await reader.read();\n\t\tif (done) break;\n\t\ttotalBytes += value.byteLength;\n\t\tif (totalBytes > MAX_URL_RESPONSE_BYTES) { await reader.cancel(); throw new Error(`URL response exceeds ${MAX_URL_RESPONSE_BYTES} byte limit`); }\n\t\tchunks.push(decoder.decode(value, { stream: true }));\n\t}\n\tchunks.push(decoder.decode());\n\treturn chunks.join(\"\");\n}\nexport async function loadPage(url: string, timeoutMs: number, signal?: AbortSignal, accept?: string): Promise<LoadedPage> {\n\turl = normalizeUrlForCache(url);\n\tconst controller = new AbortController();\n\tconst timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, REMOTE_FETCH_TIMEOUT_MS));\n\tconst onParentAbort = () => controller.abort();\n\tsignal?.addEventListener(\"abort\", onParentAbort, { once: true });\n\ttry {\n\t\tlet currentUrl = url;\n\t\tfor (let redirects = 0; redirects <= MAX_URL_REDIRECTS; redirects++) {\n\t\t\tconst pinned = await assertSafeFetchUrl(currentUrl);\n\t\t\t// Only a DNS-resolved hostname needs pinning: a literal-IP host is\n\t\t\t// validated directly and cannot be re-resolved at connect time, so\n\t\t\t// global fetch is sufficient and stays redirect/mock-friendly.\n\t\t\tconst dispatcher = pinned?.pinned ? createPinnedDispatcher(pinned) : undefined;\n\t\t\ttry {\n\t\t\t\tconst headers = accept ? { Accept: accept } : undefined;\n\t\t\t\t// Route pinned (hostname) fetches through undici directly so the\n\t\t\t\t// dispatcher's custom lookup is honored even under Bun's compiled\n\t\t\t\t// fetch, which silently ignores the `dispatcher` option.\n\t\t\t\tconst response = dispatcher\n\t\t\t\t\t? await pinnedFetch(currentUrl, dispatcher, { signal: controller.signal, headers })\n\t\t\t\t\t: await fetch(currentUrl, { signal: controller.signal, headers, redirect: \"manual\" });\n\t\t\t\tif (response.status >= 300 && response.status < 400 && response.headers.get(\"location\")) { await response.body?.cancel().catch(() => {}); currentUrl = new URL(response.headers.get(\"location\")!, currentUrl).href; continue; }\n\t\t\t\tconst contentLength = Number.parseInt(response.headers.get(\"content-length\") ?? \"0\", 10);\n\t\t\t\tif (contentLength > MAX_URL_RESPONSE_BYTES) throw new Error(`URL response exceeds ${MAX_URL_RESPONSE_BYTES} byte limit: ${currentUrl}`);\n\t\t\t\tconst contentType = response.headers.get(\"content-type\") ?? \"\";\n\t\t\t\tconst content = await readResponseTextWithLimit(response);\n\t\t\t\treturn { ok: response.ok, status: response.status, content, contentType, finalUrl: response.url || currentUrl };\n\t\t\t} finally {\n\t\t\t\tawait closePinnedDispatcher(dispatcher).catch(() => undefined);\n\t\t\t}\n\t\t}\n\t\tthrow new Error(`Too many redirects while fetching URL: ${url}`);\n\t} finally {\n\t\tclearTimeout(timer);\n\t\tsignal?.removeEventListener(\"abort\", onParentAbort);\n\t}\n}\n\nfunction isLowQuality(content: string): boolean {\n\tconst lower = content.toLowerCase();\n\tif (/(enable javascript|javascript required|turn on javascript|please enable javascript|browser not supported)/.test(lower) && content.trim().length < 2000) return true;\n\tconst nonblank = content.split(\"\\n\").filter((line) => line.trim().length > 0);\n\tif (nonblank.length > 10 && nonblank.filter((line) => line.trim().length < 40).length / nonblank.length > 0.7) return true;\n\treturn false;\n}\n\nfunction buildLlmEndpointCandidates(url: string): string[] {\n\tconst parsed = (() => { try { return new URL(url); } catch { return undefined; } })();\n\tif (!parsed) return [];\n\tconst origin = parsed.origin;\n\tif (parsed.pathname === \"/\" || parsed.pathname === \"\") return [`${origin}/.well-known/llms.txt`, `${origin}/llms.txt`, `${origin}/llms.md`];\n\tconst trimmed = parsed.pathname.replace(/\\/+$/, \"\");\n\tconst segments = trimmed.split(\"/\").filter(Boolean);\n\tconst candidates: string[] = [];\n\tconst depth = parsed.pathname.endsWith(\"/\") ? segments.length : Math.max(segments.length - 1, 1);\n\tfor (let scope = Math.min(depth, segments.length); scope >= 1; scope--) {\n\t\tconst base = `${origin}/${segments.slice(0, scope).join(\"/\")}`;\n\t\tcandidates.push(`${base}/llms.txt`, `${base}/llms.md`);\n\t}\n\tcandidates.push(`${origin}/llms.txt`, `${origin}/llms.md`);\n\treturn [...new Set(candidates)];\n}\n\nasync function tryLlmEndpoints(url: string, signal?: AbortSignal): Promise<{ content: string; endpoint: string } | null> {\n\tfor (const endpoint of buildLlmEndpointCandidates(url)) {\n\t\ttry {\n\t\t\tconst page = await loadPage(endpoint, 5000, signal);\n\t\t\tif (page.ok && page.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(page.content)) return { content: page.content, endpoint };\n\t\t} catch {}\n\t}\n\treturn null;\n}\n\nfunction parseAlternateMarkdownLink(html: string, pageUrl: string): string | undefined {\n\tconst headEnd = html.toLowerCase().indexOf(\"</head>\");\n\tconst head = html.slice(0, headEnd >= 0 ? headEnd + 7 : html.length);\n\tconst links = [...head.matchAll(/<link\\b[^>]*>/gi)].map((m) => m[0] ?? \"\");\n\tfor (const tag of links) {\n\t\tconst rel = /rel=[\"']?([^\"'>]+)[\"']?/i.exec(tag)?.[1] ?? \"\";\n\t\tconst type = /type=[\"']?([^\"'>]+)[\"']?/i.exec(tag)?.[1] ?? \"\";\n\t\tconst href = /href=[\"']?([^\"'>]+)[\"']?/i.exec(tag)?.[1] ?? \"\";\n\t\tif (!/alternate/i.test(rel) || !href) continue;\n\t\tif (/RecentChanges|Special:|\\/feed\\/|action=feed/i.test(href)) continue;\n\t\tif (type.includes(\"markdown\") || href.endsWith(\".md\")) {\n\t\t\ttry { return new URL(href, pageUrl).href; } catch {}\n\t\t}\n\t}\n\treturn undefined;\n}\n\nasync function tryMdSuffix(url: string, signal?: AbortSignal): Promise<string | null> {\n\tconst candidates: string[] = [];\n\tif (/\\/$/.test(url)) candidates.push(`${url}index.html.md`);\n\telse if (/\\.\\w+\\/?$/.test(url)) candidates.push(`${url}.md`);\n\telse candidates.push(`${url}.md`, `${url}/index.html.md`);\n\tfor (const candidate of candidates) {\n\t\ttry {\n\t\t\tconst page = await loadPage(candidate, 5000, signal);\n\t\t\tif (page.ok && page.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(page.content)) return page.content;\n\t\t} catch {}\n\t}\n\treturn null;\n}\n\nasync function tryContentNegotiation(url: string, signal?: AbortSignal): Promise<string | null> {\n\ttry {\n\t\tconst page = await loadPage(url, 5000, signal, \"text/markdown, text/plain;q=0.9, text/html;q=0.8\");\n\t\tconst mime = page.contentType.toLowerCase();\n\t\tif (page.ok && page.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(page.content) && (mime.includes(\"markdown\") || mime.includes(\"text/plain\"))) return page.content;\n\t} catch {}\n\treturn null;\n}\n\nexport interface RenderedUrl {\n\tcontent: string;\n\tfinalUrl: string;\n\tcontentType: string;\n\tmethod: string;\n\tnotes: string[];\n\ttruncated: boolean;\n}\n\nexport async function renderUrl(url: string, raw: boolean, signal?: AbortSignal): Promise<RenderedUrl> {\n\tconst notes: string[] = [];\n\tlet method = \"text\";\n\tlet contentType = \"\";\n\tlet finalUrl: string;\n\tlet body = \"\";\n\tconst page = await loadPage(url, REMOTE_FETCH_TIMEOUT_MS, signal);\n\tfinalUrl = page.finalUrl;\n\tcontentType = page.contentType;\n\tif (!page.ok) return { content: page.content || `(request failed with status ${page.status})`, finalUrl, contentType, method: \"failed\", notes, truncated: false };\n\tif (raw) return { content: page.content, finalUrl, contentType, method: \"raw\", notes, truncated: false };\n\tconst buffer = Buffer.from(page.content, \"utf8\");\n\tconst docExt = documentExtensionFromContentType(contentType) ?? (isDocumentPath(finalUrl) ? (finalUrl.match(/\\.\\w+(?:$|[?#])/)?.[0] ?? \"\") : \"\");\n\tif (docExt) {\n\t\ttry { body = await extractDocumentMarkdown(buffer, `${finalUrl}${docExt}`); method = \"document\"; }\n\t\tcatch (error) { body = `[Cannot read ${docExt} file: ${error instanceof Error ? error.message : \"conversion failed\"}]`; method = \"document-error\"; }\n\t\treturn { content: body, finalUrl, contentType, method, notes, truncated: false };\n\t}\n\tconst isHtml = /html/i.test(contentType) || looksLikeHtml(page.content);\n\tif (isHtml) {\n\t\tconst altMd = parseAlternateMarkdownLink(page.content, finalUrl);\n\t\tif (altMd) { try { const alt = await loadPage(altMd, 5000, signal); if (alt.ok && alt.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(alt.content)) { return { content: alt.content, finalUrl: alt.finalUrl, contentType: alt.contentType, method: \"alternate-markdown\", notes, truncated: false }; } } catch {} notes.push(\"alternate markdown link unusable\"); }\n\t\tconst md = await tryMdSuffix(finalUrl, signal); if (md) return { content: md, finalUrl, contentType: \"text/markdown\", method: \"md-suffix\", notes, truncated: false };\n\t\tconst neg = await tryContentNegotiation(finalUrl, signal); if (neg) return { content: neg, finalUrl, contentType: \"text/markdown\", method: \"content-negotiation\", notes, truncated: false };\n\t\tlet rendered = getTurndown().turndown(page.content).trim();\n\t\tmethod = \"native\";\n\t\tif (rendered.length <= MIN_CONTENT_LENGTH || isLowQuality(rendered)) {\n\t\t\tconst llms = await tryLlmEndpoints(finalUrl, signal);\n\t\t\tif (llms) { return { content: llms.content, finalUrl, contentType: \"text/plain\", method: `llms-txt (${llms.endpoint})`, notes, truncated: false }; }\n\t\t\tnotes.push(\"rendered content low quality; no markdown source discovered\");\n\t\t}\n\t\tbody = rendered;\n\t} else {\n\t\tbody = page.content;\n\t}\n\treturn { content: body, finalUrl, contentType, method, notes, truncated: false };\n}\n\nexport function buildUrlReadOutput(result: RenderedUrl): string {\n\tconst header = [`URL: ${result.finalUrl}`, `Content-Type: ${result.contentType}`, `Method: ${result.method}`, ...(result.notes.length ? [`Notes: ${result.notes.join(\"; \")}`] : [])].join(\"\\n\");\n\treturn `${header}\\n\\n---\\n\\n${result.content}`;\n}\n\nfunction persistReadUrlArtifact(artifactsDir: string | undefined, output: string): string | undefined {\n\tif (!artifactsDir) return undefined;\n\treturn getArtifactManager(artifactsDir).save(output, \"read\");\n}\n\nexport interface ExecuteReadUrlResult {\n\tcontent: string;\n\tdetails: ReadUrlToolDetails;\n\tartifactId?: string;\n}\n\nexport interface LoadedReadUrl {\n\toutput: string;\n\tdetails: ReadUrlToolDetails;\n\tartifactId?: string;\n\traw: boolean;\n}\n\n/**\n * Resolve a URL to its full rendered (or raw) output, using the session-scoped\n * cache and persisting the rendered body as a `read` artifact. Returns the\n * complete untruncated output so callers can apply their own line selection.\n */\nexport async function loadReadUrlOutput(scope: string, params: { path: string; raw?: boolean }, artifactsDir: string | undefined, signal?: AbortSignal): Promise<LoadedReadUrl> {\n\tconst parsed = parseReadUrlTarget(params.path);\n\tif (!parsed) throw new Error(`Not a readable URL: ${params.path}`);\n\tconst raw = params.raw ?? parsed.raw;\n\tconst key = getReadUrlCacheKey(scope, parsed.url, raw);\n\tlet entry = readUrlCache.get(key);\n\tif (!entry) {\n\t\tconst rendered = await renderUrl(parsed.url, raw, signal);\n\t\tconst output = raw ? rendered.content : buildUrlReadOutput(rendered);\n\t\tconst artifactId = persistReadUrlArtifact(artifactsDir, output);\n\t\tconst details: ReadUrlToolDetails = { url: parsed.url, finalUrl: rendered.finalUrl, contentType: rendered.contentType, method: rendered.method, truncated: rendered.truncated, notes: rendered.notes, meta: artifactId ? { artifactId } : undefined };\n\t\tentry = { artifactId, details, output };\n\t\treadUrlCache.set(key, entry);\n\t\treadUrlCache.set(getReadUrlCacheKey(scope, rendered.finalUrl, raw), entry);\n\t}\n\treturn { output: entry.output, details: entry.details, artifactId: entry.artifactId, raw };\n}\n\nexport async function executeReadUrl(scope: string, params: { path: string; raw?: boolean }, artifactsDir: string | undefined, signal?: AbortSignal): Promise<ExecuteReadUrlResult> {\n\tconst loaded = await loadReadUrlOutput(scope, params, artifactsDir, signal);\n\tconst entry = { output: loaded.output, details: loaded.details, artifactId: loaded.artifactId };\n\tconst fetchDefaultBytes = 50 * 1024;\n\tconst lines = entry.output.split(\"\\n\");\n\tconst visibleLines = lines.slice(0, FETCH_DEFAULT_MAX_LINES);\n\tlet content = visibleLines.join(\"\\n\");\n\tconst bytesBeforeByteClamp = Buffer.byteLength(content, \"utf8\");\n\tconst byteTruncated = bytesBeforeByteClamp > fetchDefaultBytes;\n\tconst lineTruncated = lines.length > FETCH_DEFAULT_MAX_LINES;\n\tif (byteTruncated) content = content.slice(0, fetchDefaultBytes);\n\tconst truncated = lineTruncated || byteTruncated;\n\tlet artifactId = entry.artifactId;\n\tif (truncated && !artifactId) artifactId = persistReadUrlArtifact(artifactsDir, entry.output);\n\tconst outputLines = content.length === 0 ? 0 : content.split(\"\\n\").length;\n\tconst truncation: TruncationResult | undefined = truncated ? {\n\t\tcontent,\n\t\ttruncated: true,\n\t\ttruncatedBy: byteTruncated ? \"bytes\" : \"lines\",\n\t\ttotalLines: lines.length,\n\t\ttotalBytes: Buffer.byteLength(entry.output, \"utf8\"),\n\t\toutputLines,\n\t\toutputBytes: Buffer.byteLength(content, \"utf8\"),\n\t\tlastLinePartial: byteTruncated,\n\t\tfirstLineExceedsLimit: false,\n\t\tmaxLines: FETCH_DEFAULT_MAX_LINES,\n\t\tmaxBytes: fetchDefaultBytes,\n\t} : undefined;\n\tconst details: ReadUrlToolDetails = { ...entry.details, truncated, meta: { ...(entry.details.meta ?? {}), ...(artifactId ? { artifactId } : {}), ...(truncation ? { truncation } : {}) } };\n\tif (truncated) content += `\\n\\n[Showing first ${visibleLines.length} of ${lines.length} lines.${artifactId ? ` Full output: artifact://${artifactId}` : \"\"}]`;\n\treturn { content, details, artifactId };\n}\n\nexport function resetReadUrlCache(): void {\n\treadUrlCache.clear();\n}\n"]}

package/dist/core/tools/fetch-url.js

@@ -0,0 +1,518 @@
+/// <reference path="../../utils/turndown.d.ts" />
+/**
+ * URL fetch / cache / artifact / markdown-discovery pipeline, mirrored from
+ * oh-my-pi's `packages/coding-agent/src/tools/fetch.ts` at 15b5c1397fc.
+ *
+ * Responsibilities:
+ *  - parseReadUrlTarget: URL + line selectors (:raw, :N, :A-B, :A+B, :A-B,C-D),
+ *    collapsed-scheme repair, host-port protection.
+ *  - session-scoped LRU cache keyed by `<scope>::<raw|rendered>::<normalizedUrl>`,
+ *    cached under both requested and final (redirected) URLs.
+ *  - renderUrl: markdown discovery (alternate <link>, .md suffix, content
+ *    negotiation, llms.txt/llms.md endpoints) + native HTML-to-markdown, with
+ *    quality gating.
+ *  - artifact persistence of the rendered output, with artifactId surfaced in
+ *    truncation metadata when the visible output is head-truncated.
+ */
+import { lookup } from "node:dns/promises";
+import { ipFamily, isPrivateIpAddress, normalizeIpLiteralHost } from "./url-ip-guards.js";
+import { LRUCache } from "lru-cache";
+import TurndownService from "turndown";
+import { Agent, request as undiciRequest } from "undici";
+import { getArtifactManager } from "./artifacts.js";
+import { extractDocumentMarkdown, isDocumentPath } from "./read-document-extract.js";
+export const FETCH_DEFAULT_MAX_LINES = 300;
+export const READ_URL_CACHE_MAX_ENTRIES = 100;
+const REMOTE_FETCH_TIMEOUT_MS = 10_000;
+const MAX_URL_REDIRECTS = 5;
+const MAX_URL_RESPONSE_BYTES = 10 * 1024 * 1024;
+const MIN_CONTENT_LENGTH = 100;
+const readUrlCache = new LRUCache({ max: READ_URL_CACHE_MAX_ENTRIES });
+let turndown;
+function getTurndown() {
+    if (!turndown)
+        turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
+    return turndown;
+}
+export function repairCollapsedScheme(url) {
+    return url.replace(/^(https?):\/([^/])/i, "$1://$2");
+}
+export function isReadableUrlPath(readPath) {
+    const repaired = repairCollapsedScheme(readPath);
+    return /^https?:\/\//i.test(repaired) || /^www\./i.test(readPath);
+}
+function normalizeUrlForCache(url) {
+    const repaired = repairCollapsedScheme(url);
+    if (/^[a-z][a-z0-9+.-]*:\/\//i.test(repaired))
+        return repaired;
+    return /^https?:\/\//i.test(repaired) ? repaired : `https://${repaired}`;
+}
+const RANGE_TOKEN_RE = /^(raw|\d+(?:\+(?:\d+)|(?:-|\.\.)\d+)?)$/i;
+const MULTI_RANGE_RE = /^\d+(?:(?:-|\.\.|,|\+)\d+)*$/;
+function isUrlSelectorToken(token) {
+    return /^raw$/i.test(token) || RANGE_TOKEN_RE.test(token) || MULTI_RANGE_RE.test(token);
+}
+function parseSingleRange(token) {
+    const plus = token.match(/^(\d+)\+(\d+)$/);
+    if (plus) {
+        const start = Number.parseInt(plus[1], 10), count = Number.parseInt(plus[2], 10);
+        return start < 1 || count < 1 ? undefined : { start, end: start + count - 1 };
+    }
+    const range = token.match(/^(\d+)(?:-|\.\.)(\d+)$/);
+    if (range) {
+        const start = Number.parseInt(range[1], 10);
+        const end = Number.parseInt(range[2], 10);
+        return start < 1 || end < start ? undefined : { start, end };
+    }
+    const single = token.match(/^(\d+)$/);
+    if (single) {
+        const start = Number.parseInt(single[1], 10);
+        return start < 1 ? undefined : { start };
+    }
+    return undefined;
+}
+function parseRangeGroup(token) {
+    const groups = token.split(",");
+    const ranges = [];
+    for (const group of groups) {
+        const r = parseSingleRange(group);
+        if (!r)
+            return undefined;
+        ranges.push(r);
+    }
+    return ranges;
+}
+function tryExtractEmbeddedUrlSelector(readPath) {
+    let basePath = readPath;
+    const sels = [];
+    for (;;) {
+        const lastColonIndex = basePath.lastIndexOf(":");
+        if (lastColonIndex <= 0)
+            break;
+        const candidate = basePath.slice(lastColonIndex + 1);
+        const remainder = basePath.slice(0, lastColonIndex);
+        if (!isReadableUrlPath(remainder) || !isUrlSelectorToken(candidate))
+            break;
+        try {
+            new URL(remainder.startsWith("http://") || remainder.startsWith("https://") ? remainder : `https://${remainder}`);
+        }
+        catch {
+            break;
+        }
+        sels.unshift(candidate);
+        basePath = remainder;
+    }
+    return sels.length === 0 ? null : { path: basePath, sels };
+}
+export function parseReadUrlTarget(readPath) {
+    const repaired = repairCollapsedScheme(readPath);
+    const embedded = tryExtractEmbeddedUrlSelector(repaired);
+    const urlPath = embedded?.path ?? repaired;
+    if (!isReadableUrlPath(urlPath))
+        return null;
+    let raw = false;
+    let ranges;
+    for (const sel of embedded?.sels ?? []) {
+        if (/^raw$/i.test(sel)) {
+            raw = true;
+            continue;
+        }
+        if (ranges !== undefined)
+            throw new Error("URL selector has multiple range groups; combine them with commas (e.g. `:5-10,20-30`).");
+        const parsed = parseRangeGroup(sel);
+        if (!parsed)
+            throw new Error(`Invalid URL line selector: ${sel}`);
+        ranges = parsed;
+    }
+    if (!ranges || ranges.length === 0)
+        return { url: urlPath, raw };
+    if (ranges.length === 1) {
+        const r = ranges[0];
+        return { url: urlPath, raw, offset: r.start, limit: r.end !== undefined ? r.end - r.start + 1 : undefined };
+    }
+    return { url: urlPath, raw, ranges };
+}
+export function getReadUrlCacheKey(scope, requestedUrl, raw) {
+    return `${scope}::${raw ? "raw" : "rendered"}::${normalizeUrlForCache(requestedUrl)}`;
+}
+function looksLikeHtml(text) {
+    return /<html[\s>]/i.test(text) || (/<head[\s>]/i.test(text) && /<body[\s>]/i.test(text));
+}
+function documentExtensionFromContentType(contentType) {
+    if (/pdf/i.test(contentType))
+        return ".pdf";
+    if (/msword/i.test(contentType))
+        return ".doc";
+    if (/wordprocessingml|officedocument\.word/i.test(contentType))
+        return ".docx";
+    if (/presentationml|officedocument\.presentation/i.test(contentType))
+        return ".pptx";
+    if (/ms-powerpoint|vnd\.ms-powerpoint/i.test(contentType))
+        return ".ppt";
+    if (/spreadsheetml|officedocument\.spreadsheet/i.test(contentType))
+        return ".xlsx";
+    if (/ms-excel|vnd\.ms-excel/i.test(contentType))
+        return ".xls";
+    if (/epub/i.test(contentType))
+        return ".epub";
+    if (/\bjson\b|ipynb|jupyter/i.test(contentType))
+        return ".ipynb";
+    if (/rtf/i.test(contentType))
+        return ".rtf";
+    return undefined;
+}
+function createPinnedDispatcher(pinned) {
+    return new Agent({ connect: { lookup: (_hostname, options, callback) => { if (options.all)
+                callback(null, [pinned]);
+            else
+                callback(null, pinned.address, pinned.family); } } });
+}
+async function closePinnedDispatcher(dispatcher) {
+    const close = dispatcher?.close;
+    if (typeof close === "function")
+        await close.call(dispatcher);
+}
+/**
+ * Fetch a URL through undici with the pinned dispatcher so the validated
+ * address is the one actually connected to. Unlike `globalThis.fetch`, which
+ * silently ignores the undici `dispatcher` option under Bun's compiled binary
+ * (reopening a DNS-rebinding TOCTOU window for hostname targets), undici's own
+ * client honors the dispatcher's custom `lookup` regardless of host runtime.
+ * Adapts the undici response to the WHATWG-Response shape loadPage consumes.
+ */
+async function pinnedFetch(url, dispatcher, options) {
+    const result = await undiciRequest(url, { method: "GET", dispatcher, signal: options.signal, headers: options.headers });
+    const body = result.body;
+    const status = result.statusCode;
+    const headers = new Headers();
+    for (const [key, value] of Object.entries(result.headers)) {
+        if (value === undefined)
+            continue;
+        const values = Array.isArray(value) ? value : [value];
+        for (const entry of values)
+            headers.append(key, entry);
+    }
+    const stream = new ReadableStream({
+        start(controller) { body.on("data", (chunk) => controller.enqueue(new Uint8Array(chunk))); body.on("end", () => controller.close()); body.on("error", (error) => controller.error(error)); },
+        cancel() { body.destroy(); },
+    });
+    return new Response(stream, { status, headers });
+}
+async function assertSafeFetchUrl(url) {
+    const parsed = new URL(url);
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
+        throw new Error(`Unsupported URL protocol: ${parsed.protocol}`);
+    if (process.env.ATOMIC_ALLOW_PRIVATE_URL_READS === "1")
+        return undefined;
+    const hostname = parsed.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
+    if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal")
+        throw new Error(`Refusing to fetch private or metadata URL: ${url}`);
+    const literalAddress = normalizeIpLiteralHost(hostname);
+    if (literalAddress) {
+        if (isPrivateIpAddress(literalAddress))
+            throw new Error(`Refusing to fetch private or metadata URL: ${url}`);
+        return { address: literalAddress, family: ipFamily(literalAddress), pinned: false };
+    }
+    const addresses = await lookup(hostname, { all: true }).catch((error) => { throw new Error(`Could not resolve URL host ${hostname}: ${error instanceof Error ? error.message : String(error)}`); });
+    if (addresses.length === 0)
+        throw new Error(`Could not resolve URL host ${hostname}: no addresses returned`);
+    for (const address of addresses)
+        if (isPrivateIpAddress(address.address))
+            throw new Error(`Refusing to fetch private or metadata URL: ${url}`);
+    const first = addresses[0];
+    return { address: first.address, family: first.family === 6 ? 6 : 4, pinned: true };
+}
+async function readResponseTextWithLimit(response) {
+    if (!response.body) {
+        const buffer = Buffer.from(await response.arrayBuffer());
+        if (buffer.length > MAX_URL_RESPONSE_BYTES)
+            throw new Error(`URL response exceeds ${MAX_URL_RESPONSE_BYTES} byte limit`);
+        return buffer.toString("utf8");
+    }
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    const chunks = [];
+    let totalBytes = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        totalBytes += value.byteLength;
+        if (totalBytes > MAX_URL_RESPONSE_BYTES) {
+            await reader.cancel();
+            throw new Error(`URL response exceeds ${MAX_URL_RESPONSE_BYTES} byte limit`);
+        }
+        chunks.push(decoder.decode(value, { stream: true }));
+    }
+    chunks.push(decoder.decode());
+    return chunks.join("");
+}
+export async function loadPage(url, timeoutMs, signal, accept) {
+    url = normalizeUrlForCache(url);
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, REMOTE_FETCH_TIMEOUT_MS));
+    const onParentAbort = () => controller.abort();
+    signal?.addEventListener("abort", onParentAbort, { once: true });
+    try {
+        let currentUrl = url;
+        for (let redirects = 0; redirects <= MAX_URL_REDIRECTS; redirects++) {
+            const pinned = await assertSafeFetchUrl(currentUrl);
+            // Only a DNS-resolved hostname needs pinning: a literal-IP host is
+            // validated directly and cannot be re-resolved at connect time, so
+            // global fetch is sufficient and stays redirect/mock-friendly.
+            const dispatcher = pinned?.pinned ? createPinnedDispatcher(pinned) : undefined;
+            try {
+                const headers = accept ? { Accept: accept } : undefined;
+                // Route pinned (hostname) fetches through undici directly so the
+                // dispatcher's custom lookup is honored even under Bun's compiled
+                // fetch, which silently ignores the `dispatcher` option.
+                const response = dispatcher
+                    ? await pinnedFetch(currentUrl, dispatcher, { signal: controller.signal, headers })
+                    : await fetch(currentUrl, { signal: controller.signal, headers, redirect: "manual" });
+                if (response.status >= 300 && response.status < 400 && response.headers.get("location")) {
+                    await response.body?.cancel().catch(() => { });
+                    currentUrl = new URL(response.headers.get("location"), currentUrl).href;
+                    continue;
+                }
+                const contentLength = Number.parseInt(response.headers.get("content-length") ?? "0", 10);
+                if (contentLength > MAX_URL_RESPONSE_BYTES)
+                    throw new Error(`URL response exceeds ${MAX_URL_RESPONSE_BYTES} byte limit: ${currentUrl}`);
+                const contentType = response.headers.get("content-type") ?? "";
+                const content = await readResponseTextWithLimit(response);
+                return { ok: response.ok, status: response.status, content, contentType, finalUrl: response.url || currentUrl };
+            }
+            finally {
+                await closePinnedDispatcher(dispatcher).catch(() => undefined);
+            }
+        }
+        throw new Error(`Too many redirects while fetching URL: ${url}`);
+    }
+    finally {
+        clearTimeout(timer);
+        signal?.removeEventListener("abort", onParentAbort);
+    }
+}
+function isLowQuality(content) {
+    const lower = content.toLowerCase();
+    if (/(enable javascript|javascript required|turn on javascript|please enable javascript|browser not supported)/.test(lower) && content.trim().length < 2000)
+        return true;
+    const nonblank = content.split("\n").filter((line) => line.trim().length > 0);
+    if (nonblank.length > 10 && nonblank.filter((line) => line.trim().length < 40).length / nonblank.length > 0.7)
+        return true;
+    return false;
+}
+function buildLlmEndpointCandidates(url) {
+    const parsed = (() => { try {
+        return new URL(url);
+    }
+    catch {
+        return undefined;
+    } })();
+    if (!parsed)
+        return [];
+    const origin = parsed.origin;
+    if (parsed.pathname === "/" || parsed.pathname === "")
+        return [`${origin}/.well-known/llms.txt`, `${origin}/llms.txt`, `${origin}/llms.md`];
+    const trimmed = parsed.pathname.replace(/\/+$/, "");
+    const segments = trimmed.split("/").filter(Boolean);
+    const candidates = [];
+    const depth = parsed.pathname.endsWith("/") ? segments.length : Math.max(segments.length - 1, 1);
+    for (let scope = Math.min(depth, segments.length); scope >= 1; scope--) {
+        const base = `${origin}/${segments.slice(0, scope).join("/")}`;
+        candidates.push(`${base}/llms.txt`, `${base}/llms.md`);
+    }
+    candidates.push(`${origin}/llms.txt`, `${origin}/llms.md`);
+    return [...new Set(candidates)];
+}
+async function tryLlmEndpoints(url, signal) {
+    for (const endpoint of buildLlmEndpointCandidates(url)) {
+        try {
+            const page = await loadPage(endpoint, 5000, signal);
+            if (page.ok && page.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(page.content))
+                return { content: page.content, endpoint };
+        }
+        catch { }
+    }
+    return null;
+}
+function parseAlternateMarkdownLink(html, pageUrl) {
+    const headEnd = html.toLowerCase().indexOf("</head>");
+    const head = html.slice(0, headEnd >= 0 ? headEnd + 7 : html.length);
+    const links = [...head.matchAll(/<link\b[^>]*>/gi)].map((m) => m[0] ?? "");
+    for (const tag of links) {
+        const rel = /rel=["']?([^"'>]+)["']?/i.exec(tag)?.[1] ?? "";
+        const type = /type=["']?([^"'>]+)["']?/i.exec(tag)?.[1] ?? "";
+        const href = /href=["']?([^"'>]+)["']?/i.exec(tag)?.[1] ?? "";
+        if (!/alternate/i.test(rel) || !href)
+            continue;
+        if (/RecentChanges|Special:|\/feed\/|action=feed/i.test(href))
+            continue;
+        if (type.includes("markdown") || href.endsWith(".md")) {
+            try {
+                return new URL(href, pageUrl).href;
+            }
+            catch { }
+        }
+    }
+    return undefined;
+}
+async function tryMdSuffix(url, signal) {
+    const candidates = [];
+    if (/\/$/.test(url))
+        candidates.push(`${url}index.html.md`);
+    else if (/\.\w+\/?$/.test(url))
+        candidates.push(`${url}.md`);
+    else
+        candidates.push(`${url}.md`, `${url}/index.html.md`);
+    for (const candidate of candidates) {
+        try {
+            const page = await loadPage(candidate, 5000, signal);
+            if (page.ok && page.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(page.content))
+                return page.content;
+        }
+        catch { }
+    }
+    return null;
+}
+async function tryContentNegotiation(url, signal) {
+    try {
+        const page = await loadPage(url, 5000, signal, "text/markdown, text/plain;q=0.9, text/html;q=0.8");
+        const mime = page.contentType.toLowerCase();
+        if (page.ok && page.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(page.content) && (mime.includes("markdown") || mime.includes("text/plain")))
+            return page.content;
+    }
+    catch { }
+    return null;
+}
+export async function renderUrl(url, raw, signal) {
+    const notes = [];
+    let method = "text";
+    let contentType = "";
+    let finalUrl;
+    let body = "";
+    const page = await loadPage(url, REMOTE_FETCH_TIMEOUT_MS, signal);
+    finalUrl = page.finalUrl;
+    contentType = page.contentType;
+    if (!page.ok)
+        return { content: page.content || `(request failed with status ${page.status})`, finalUrl, contentType, method: "failed", notes, truncated: false };
+    if (raw)
+        return { content: page.content, finalUrl, contentType, method: "raw", notes, truncated: false };
+    const buffer = Buffer.from(page.content, "utf8");
+    const docExt = documentExtensionFromContentType(contentType) ?? (isDocumentPath(finalUrl) ? (finalUrl.match(/\.\w+(?:$|[?#])/)?.[0] ?? "") : "");
+    if (docExt) {
+        try {
+            body = await extractDocumentMarkdown(buffer, `${finalUrl}${docExt}`);
+            method = "document";
+        }
+        catch (error) {
+            body = `[Cannot read ${docExt} file: ${error instanceof Error ? error.message : "conversion failed"}]`;
+            method = "document-error";
+        }
+        return { content: body, finalUrl, contentType, method, notes, truncated: false };
+    }
+    const isHtml = /html/i.test(contentType) || looksLikeHtml(page.content);
+    if (isHtml) {
+        const altMd = parseAlternateMarkdownLink(page.content, finalUrl);
+        if (altMd) {
+            try {
+                const alt = await loadPage(altMd, 5000, signal);
+                if (alt.ok && alt.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(alt.content)) {
+                    return { content: alt.content, finalUrl: alt.finalUrl, contentType: alt.contentType, method: "alternate-markdown", notes, truncated: false };
+                }
+            }
+            catch { }
+            notes.push("alternate markdown link unusable");
+        }
+        const md = await tryMdSuffix(finalUrl, signal);
+        if (md)
+            return { content: md, finalUrl, contentType: "text/markdown", method: "md-suffix", notes, truncated: false };
+        const neg = await tryContentNegotiation(finalUrl, signal);
+        if (neg)
+            return { content: neg, finalUrl, contentType: "text/markdown", method: "content-negotiation", notes, truncated: false };
+        let rendered = getTurndown().turndown(page.content).trim();
+        method = "native";
+        if (rendered.length <= MIN_CONTENT_LENGTH || isLowQuality(rendered)) {
+            const llms = await tryLlmEndpoints(finalUrl, signal);
+            if (llms) {
+                return { content: llms.content, finalUrl, contentType: "text/plain", method: `llms-txt (${llms.endpoint})`, notes, truncated: false };
+            }
+            notes.push("rendered content low quality; no markdown source discovered");
+        }
+        body = rendered;
+    }
+    else {
+        body = page.content;
+    }
+    return { content: body, finalUrl, contentType, method, notes, truncated: false };
+}
+export function buildUrlReadOutput(result) {
+    const header = [`URL: ${result.finalUrl}`, `Content-Type: ${result.contentType}`, `Method: ${result.method}`, ...(result.notes.length ? [`Notes: ${result.notes.join("; ")}`] : [])].join("\n");
+    return `${header}\n\n---\n\n${result.content}`;
+}
+function persistReadUrlArtifact(artifactsDir, output) {
+    if (!artifactsDir)
+        return undefined;
+    return getArtifactManager(artifactsDir).save(output, "read");
+}
+/**
+ * Resolve a URL to its full rendered (or raw) output, using the session-scoped
+ * cache and persisting the rendered body as a `read` artifact. Returns the
+ * complete untruncated output so callers can apply their own line selection.
+ */
+export async function loadReadUrlOutput(scope, params, artifactsDir, signal) {
+    const parsed = parseReadUrlTarget(params.path);
+    if (!parsed)
+        throw new Error(`Not a readable URL: ${params.path}`);
+    const raw = params.raw ?? parsed.raw;
+    const key = getReadUrlCacheKey(scope, parsed.url, raw);
+    let entry = readUrlCache.get(key);
+    if (!entry) {
+        const rendered = await renderUrl(parsed.url, raw, signal);
+        const output = raw ? rendered.content : buildUrlReadOutput(rendered);
+        const artifactId = persistReadUrlArtifact(artifactsDir, output);
+        const details = { url: parsed.url, finalUrl: rendered.finalUrl, contentType: rendered.contentType, method: rendered.method, truncated: rendered.truncated, notes: rendered.notes, meta: artifactId ? { artifactId } : undefined };
+        entry = { artifactId, details, output };
+        readUrlCache.set(key, entry);
+        readUrlCache.set(getReadUrlCacheKey(scope, rendered.finalUrl, raw), entry);
+    }
+    return { output: entry.output, details: entry.details, artifactId: entry.artifactId, raw };
+}
+export async function executeReadUrl(scope, params, artifactsDir, signal) {
+    const loaded = await loadReadUrlOutput(scope, params, artifactsDir, signal);
+    const entry = { output: loaded.output, details: loaded.details, artifactId: loaded.artifactId };
+    const fetchDefaultBytes = 50 * 1024;
+    const lines = entry.output.split("\n");
+    const visibleLines = lines.slice(0, FETCH_DEFAULT_MAX_LINES);
+    let content = visibleLines.join("\n");
+    const bytesBeforeByteClamp = Buffer.byteLength(content, "utf8");
+    const byteTruncated = bytesBeforeByteClamp > fetchDefaultBytes;
+    const lineTruncated = lines.length > FETCH_DEFAULT_MAX_LINES;
+    if (byteTruncated)
+        content = content.slice(0, fetchDefaultBytes);
+    const truncated = lineTruncated || byteTruncated;
+    let artifactId = entry.artifactId;
+    if (truncated && !artifactId)
+        artifactId = persistReadUrlArtifact(artifactsDir, entry.output);
+    const outputLines = content.length === 0 ? 0 : content.split("\n").length;
+    const truncation = truncated ? {
+        content,
+        truncated: true,
+        truncatedBy: byteTruncated ? "bytes" : "lines",
+        totalLines: lines.length,
+        totalBytes: Buffer.byteLength(entry.output, "utf8"),
+        outputLines,
+        outputBytes: Buffer.byteLength(content, "utf8"),
+        lastLinePartial: byteTruncated,
+        firstLineExceedsLimit: false,
+        maxLines: FETCH_DEFAULT_MAX_LINES,
+        maxBytes: fetchDefaultBytes,
+    } : undefined;
+    const details = { ...entry.details, truncated, meta: { ...(entry.details.meta ?? {}), ...(artifactId ? { artifactId } : {}), ...(truncation ? { truncation } : {}) } };
+    if (truncated)
+        content += `\n\n[Showing first ${visibleLines.length} of ${lines.length} lines.${artifactId ? ` Full output: artifact://${artifactId}` : ""}]`;
+    return { content, details, artifactId };
+}
+export function resetReadUrlCache() {
+    readUrlCache.clear();
+}
+//# sourceMappingURL=fetch-url.js.map