@pentoshi/clai 3.8.44 → 3.8.46

package/dist/tools/http.js

@@ -1,7 +1,9 @@
 import net from "node:net";
+import { createHash } from "node:crypto";
 import { lookup } from "node:dns/promises";
 import { Agent } from "undici";
 import { isBlockedAddress } from "./web/ssrf-guard.js";
+import { decodeTextBody } from "./web/decode.js";
 import { toReadableText } from "./web/readable.js";
 import { selectOutput, } from "./output-selection.js";
 /** Shared undici agent that skips cert verification (authorized pentest only). */
@@ -16,6 +18,53 @@ function getInsecureTlsAgent() {
     }
     return insecureTlsAgent;
 }
+/**
+ * Pin Fetch to addresses that already passed policy. If policy pre-resolution
+ * was unavailable (for example, an injected test transport), the dispatcher's
+ * own lookup performs the SSRF classification immediately before connect so
+ * there is still no unchecked second DNS resolution.
+ */
+function createPinnedAgent(addresses, insecureTls, allowBlockedAddresses) {
+    const preResolved = toLookupAddresses(addresses);
+    return new Agent({
+        connect: {
+            rejectUnauthorized: !insecureTls,
+            lookup: (hostname, options, callback) => {
+                const send = (resolved) => {
+                    if (resolved.length === 0) {
+                        callback(new Error(`DNS resolution returned no addresses for ${hostname}`), "");
+                        return;
+                    }
+                    if (!allowBlockedAddresses &&
+                        (isBlockedAddress(hostname) ||
+                            resolved.some((entry) => isBlockedAddress(entry.address)))) {
+                        callback(new Error(`Refusing private/loopback/metadata DNS result for ${hostname}`), "");
+                        return;
+                    }
+                    if (options && typeof options === "object" && options.all === true) {
+                        callback(null, resolved);
+                        return;
+                    }
+                    const selected = resolved[0];
+                    callback(null, selected.address, selected.family);
+                };
+                if (preResolved.length > 0) {
+                    send(preResolved);
+                    return;
+                }
+                void lookup(hostname, { all: true }).then((results) => send(toLookupAddresses(results.map((entry) => entry.address))), (error) => callback(error instanceof Error ? error : new Error(String(error)), ""));
+            },
+        },
+    });
+}
+function toLookupAddresses(addresses) {
+    return addresses
+        .map((address) => ({ address, family: net.isIP(address) }))
+        .filter((entry) => entry.family === 4 || entry.family === 6);
+}
+async function closeAgents(agents) {
+    await Promise.allSettled(agents.map(async (agent) => agent.close()));
+}
 /** Flatten error + nested cause(s) for TLS code matching. */
 function errorText(error, depth = 0) {
     if (depth > 4 || error == null)
@@ -77,8 +126,10 @@ function formatTlsNetworkError(error, url) {
         `  3. shell: curl -k -i ${url}\n` +
         `Evidence taken with insecureTls is still valid for recon; record that TLS verification was disabled.`);
 }
-/** Capture budget for body bytes (artifact + evidence build). Model context is capped later. */
+/** Capture budget for decoded response-body bytes (artifact + evidence build). */
 const DEFAULT_MAX_BYTES = 128 * 1024;
+/** Hard memory ceiling for one response capture; raise maxBytes up to this cap. */
+const MAX_CAPTURE_BYTES = 16 * 1024 * 1024;
 /**
  * Default retries for remote evidence: 0 so intentional 5xx/probe responses are
  * not silently rewritten. Callers may pass retries; owned loopback still soft-
@@ -217,6 +268,7 @@ function clampTimeoutMs(value) {
     return Math.max(MIN_TIMEOUT_MS, Math.min(MAX_TIMEOUT_MS, Math.floor(value)));
 }
 export async function httpFetch(url, options = {}) {
+    const startedAt = Date.now();
     let target;
     try {
         target = new URL(url);
@@ -241,12 +293,14 @@ export async function httpFetch(url, options = {}) {
     }
     const startHost = target.hostname.replace(/^\[|\]$/g, "");
     const ownedLoopback = Boolean(options.iOwnThis && isLoopbackHost(startHost));
+    const authorizedAddresses = new Map();
     // SSRF and engagement-policy checks run for the initial destination and
     // every redirect hop. Owned loopback (local app verify) skips engagement
     // hop checks — a leftover pentest scope must not block http://localhost:5173.
     const authorizeDestination = async (destination) => {
         const hostname = destination.hostname.replace(/^\[|\]$/g, "");
         const resolvedAddresses = await resolveHosts(hostname);
+        authorizedAddresses.set(hostname.toLowerCase(), resolvedAddresses);
         const blocked = isBlockedAddress(hostname) || resolvedAddresses.some(isBlockedAddress);
         const destLoopback = isLoopbackHost(hostname);
         if (blocked && !options.iOwnThis) {
@@ -285,18 +339,20 @@ export async function httpFetch(url, options = {}) {
     if (options.body !== undefined && method !== "GET" && method !== "HEAD") {
         init.body = options.body;
     }
-    // Node undici: skip cert checks for authorized IP/lab HTTPS probes.
-    if (insecureTls && target.protocol === "https:") {
-        init.dispatcher = getInsecureTlsAgent();
-    }
+    // The insecure dispatcher is selected per hop below because redirects may
+    // change scheme. A single shared dispatcher keeps authorized lab probes
+    // cheap without weakening unrelated requests.
     // Local dev: try localhost / 127.0.0.1 / ::1 until one accepts (Vite dual-stack).
     const urlCandidates = ownedLoopback && (method === "GET" || method === "HEAD")
         ? loopbackUrlCandidates(url)
         : [url];
     let response;
+    const pinnedAgents = [];
     let attempts = 0;
     let lastNetworkError;
+    // The active URL may advance through redirects within this attempt.
     let usedUrl = url;
+    let finalMethod = method;
     let redirectHops = [];
     let responseCleanup;
     const timeoutMs = clampTimeoutMs(options.timeoutMs);
@@ -334,31 +390,81 @@ export async function httpFetch(url, options = {}) {
                 }
                 try {
                     let requestUrl = candidateUrl;
+                    let requestMethod = method;
+                    let requestBody = init.body;
+                    let requestHeaders = new Headers(headers);
                     let redirects = 0;
                     redirectHops = [];
                     for (;;) {
-                        response = await fetch(requestUrl, {
+                        const hopInit = {
                             ...init,
+                            method: requestMethod,
+                            headers: requestHeaders,
                             signal: controller.signal,
-                        });
+                        };
+                        if (requestBody === undefined || requestBody === null) {
+                            delete hopInit.body;
+                        }
+                        else {
+                            hopInit.body = requestBody;
+                        }
+                        const hopUrl = new URL(requestUrl);
+                        const hopHost = hopUrl.hostname.replace(/^\[|\]$/g, "").toLowerCase();
+                        const pinned = authorizedAddresses.get(hopHost) ?? [];
+                        if (!isLoopbackHost(hopHost)) {
+                            const dispatcher = createPinnedAgent(pinned, insecureTls && hopUrl.protocol === "https:", options.iOwnThis === true);
+                            pinnedAgents.push(dispatcher);
+                            hopInit.dispatcher = dispatcher;
+                        }
+                        else if (insecureTls && hopUrl.protocol === "https:") {
+                            // Owned loopback keeps the shared dispatcher so localhost can
+                            // retain its IPv4/IPv6 fallback behavior.
+                            hopInit.dispatcher = getInsecureTlsAgent();
+                        }
+                        else {
+                            delete hopInit.dispatcher;
+                        }
+                        usedUrl = requestUrl;
+                        response = await fetch(requestUrl, hopInit);
+                        finalMethod = requestMethod;
                         const location = response.headers.get("location");
                         if (!(response.status >= 300 && response.status < 400 && location))
                             break;
                         if (redirects >= 10)
                             throw new Error("Too many redirects (maximum 10)");
-                        const setCookies = typeof response.headers.getSetCookie === "function"
-                            ? response.headers.getSetCookie()
-                            : multiHeader(response.headers, "set-cookie");
                         redirectHops.push({
                             status: response.status,
+                            statusText: response.statusText,
+                            method: requestMethod,
                             url: requestUrl,
                             location,
-                            setCookies,
+                            headers: snapshotHeaders(response.headers),
                         });
                         const next = new URL(location, requestUrl);
                         const denial = await authorizeDestination(next);
                         if (denial)
                             throw new Error(denial);
+                        if (!options.forwardSensitiveHeaders &&
+                            new URL(requestUrl).origin !== next.origin) {
+                            requestHeaders = new Headers(requestHeaders);
+                            requestHeaders.delete("authorization");
+                            requestHeaders.delete("proxy-authorization");
+                            requestHeaders.delete("cookie");
+                        }
+                        // Match widely deployed user-agent redirect semantics while
+                        // preserving methods for 307/308. This avoids replaying a POST
+                        // body onto a 303 destination and records the actual method used.
+                        if (requestMethod !== "HEAD" &&
+                            (response.status === 303 ||
+                                ((response.status === 301 || response.status === 302) &&
+                                    requestMethod === "POST"))) {
+                            requestMethod = "GET";
+                            requestBody = undefined;
+                            requestHeaders = new Headers(requestHeaders);
+                            requestHeaders.delete("content-length");
+                            requestHeaders.delete("content-type");
+                            requestHeaders.delete("transfer-encoding");
+                        }
                         await drainResponse(response);
                         requestUrl = next.toString();
                         redirects += 1;
@@ -410,6 +516,7 @@ export async function httpFetch(url, options = {}) {
         }
     }
     catch (error) {
+        await closeAgents(pinnedAgents);
         const tried = urlCandidates.length > 1
             ? ` (tried ${urlCandidates.join(" → ")})`
             : "";
@@ -421,15 +528,15 @@ export async function httpFetch(url, options = {}) {
         };
     }
     if (!response) {
+        await closeAgents(pinnedAgents);
         return {
             ok: false,
             output: "Network error: no response was received",
             exitCode: 1,
         };
     }
-    const limit = options.maxBytes ?? DEFAULT_MAX_BYTES;
-    const decoder = new TextDecoder("utf-8", { fatal: false });
-    let collected = "";
+    const limit = clampCaptureBytes(options.maxBytes);
+    const chunks = [];
     let bytesRead = 0;
     let truncated = false;
     let bodyReadError;
@@ -471,9 +578,12 @@ export async function httpFetch(url, options = {}) {
                         }
                         break;
                     }
+                    const captured = value.byteLength > remaining
+                        ? value.subarray(0, remaining)
+                        : value;
+                    chunks.push(Buffer.from(captured));
+                    bytesRead += captured.byteLength;
                     if (value.byteLength > remaining) {
-                        collected += decoder.decode(value.subarray(0, remaining), { stream: true });
-                        bytesRead += remaining;
                         truncated = true;
                         try {
                             await reader.cancel();
@@ -483,10 +593,7 @@ export async function httpFetch(url, options = {}) {
                         }
                         break;
                     }
-                    collected += decoder.decode(value, { stream: true });
-                    bytesRead += value.byteLength;
                 }
-                collected += decoder.decode();
             }
             finally {
                 try {
@@ -497,9 +604,6 @@ export async function httpFetch(url, options = {}) {
                 }
             }
         }
-        else {
-            collected = "";
-        }
     }
     catch (error) {
         bodyReadError = error;
@@ -512,6 +616,7 @@ export async function httpFetch(url, options = {}) {
         const timedOut = bodyReadError instanceof Error &&
             bodyReadError.name === "AbortError" &&
             !options.signal?.aborted;
+        await closeAgents(pinnedAgents);
         return {
             ok: false,
             output: timedOut
@@ -520,11 +625,15 @@ export async function httpFetch(url, options = {}) {
             exitCode: timedOut ? 124 : options.signal?.aborted ? 130 : 1,
         };
     }
+    const capturedBody = Buffer.concat(chunks, bytesRead);
     const contentType = response.headers.get("content-type") ?? "";
-    const isBinary = isBinaryContentType(contentType) || isBinaryContent(collected);
+    const decoded = decodeTextBody(capturedBody, contentType);
+    const isBinary = isBinaryContentType(contentType) ||
+        (!isTextualContentType(contentType) && isBinaryContent(capturedBody));
     const finalUrl = response.url || usedUrl;
     const evidence = formatHttpEvidence({
-        method,
+        requestedMethod: method,
+        finalMethod,
         requestedUrl: url,
         usedUrl,
         finalUrl,
@@ -532,24 +641,36 @@ export async function httpFetch(url, options = {}) {
         statusText: response.statusText,
         attempts,
         bytesRead,
+        bodySha256: createHash("sha256").update(capturedBody).digest("hex"),
+        bodyCharset: decoded.charset,
+        charsetSource: decoded.charsetSource,
+        unsupportedCharset: decoded.unsupportedCharset,
         truncated,
         limit,
         contentType,
         headers: response.headers,
-        body: collected,
+        body: decoded.text,
         isBinary,
         redirectHops,
         lastNetworkError,
         insecureTls,
-        responseMode: options.responseMode ?? "readable",
+        forwardSensitiveHeaders: options.forwardSensitiveHeaders === true,
+        responseMode: options.responseMode ?? "raw",
         responsePart: options.responsePart ?? "full",
     });
     const output = selectOutput(evidence, options);
+    await closeAgents(pinnedAgents);
     return {
         ok: true,
         output,
         exitCode: 0,
         truncated: truncated || output !== evidence,
+        stats: {
+            bytesRead,
+            bytesDropped: 0,
+            linesRead: decoded.text.split("\n").length,
+            elapsedMs: Date.now() - startedAt,
+        },
     };
 }
 function multiHeader(headers, name) {
@@ -560,57 +681,100 @@ function multiHeader(headers, name) {
     });
     return out;
 }
+/**
+ * Snapshot every response header exposed by the Fetch runtime. Header names
+ * and most repeat values are normalized by Fetch; Set-Cookie is recovered as
+ * separate lines when Node/undici exposes getSetCookie().
+ */
+function snapshotHeaders(headers) {
+    const out = [];
+    headers.forEach((value, key) => {
+        if (key.toLowerCase() !== "set-cookie")
+            out.push([key, value]);
+    });
+    const setCookies = typeof headers.getSetCookie === "function"
+        ? headers.getSetCookie()
+        : multiHeader(headers, "set-cookie");
+    for (const value of setCookies)
+        out.push(["set-cookie", value]);
+    return out;
+}
 /** Headers always kept in evidence (security / fingerprint / cookies). */
 const PRIORITY_HEADER_RE = /^(set-cookie|server|x-powered-by|x-aspnet|x-generator|x-frame-options|x-content-type-options|x-xss-protection|content-security-policy|content-type|location|www-authenticate|access-control-|strict-transport|referrer-policy|permissions-policy|cross-origin-|cache-control|via|cf-ray|x-request-id|x-amz-|x-served|link|allow|retry-after)$/i;
 function formatHttpEvidence(input) {
     const lines = [];
-    lines.push(`${input.status} ${input.statusText} ${input.method} ${input.finalUrl}`);
+    lines.push(`${input.status} ${input.statusText} ${input.finalMethod} ${input.finalUrl}`);
     if (input.insecureTls) {
         lines.push("TLS: verification DISABLED (insecureTls=true) — hostname/cert not validated; authorized-test only.");
     }
     if (input.redirectHops.length > 0) {
         const chain = [
-            ...input.redirectHops.map((h) => `${h.status} ${h.url} → ${h.location}`),
-            `${input.status} ${input.finalUrl}`,
+            ...input.redirectHops.map((h) => `${h.status} ${h.method} ${h.url} → ${h.location}`),
+            `${input.status} ${input.finalMethod} ${input.finalUrl}`,
         ];
         lines.push(`redirects: ${chain.join(" → ")}`);
+        const crossedOrigin = input.redirectHops.some((hop) => {
+            try {
+                return new URL(hop.url).origin !== new URL(hop.location, hop.url).origin;
+            }
+            catch {
+                return false;
+            }
+        });
+        if (crossedOrigin) {
+            lines.push(input.forwardSensitiveHeaders
+                ? "redirect credentials: explicitly forwarded across origin change"
+                : "redirect credentials: Authorization, Proxy-Authorization, and Cookie stripped across origin change");
+        }
+        lines.push("");
+        lines.push("Redirect responses (headers runtime-normalized):");
         for (const hop of input.redirectHops) {
-            if (hop.setCookies.length) {
-                lines.push(`  hop ${hop.status} set-cookie: ${hop.setCookies.join("; ")}`);
+            lines.push(`  ${hop.status} ${hop.statusText} ${hop.method} ${hop.url}`, `  location: ${hop.location}`);
+            for (const [key, value] of hop.headers) {
+                lines.push(`  ${key}: ${value}`);
             }
+            lines.push("");
+        }
+    }
+    const metaBits = [`attempts=${input.attempts}`];
+    if (input.responsePart === "headers") {
+        metaBits.push("bodyCapture=skipped(responsePart=headers)");
+    }
+    else {
+        metaBits.push(`bodyBytes=${input.bytesRead}`, `bodySha256=${input.bodySha256}`, `charset=${input.bodyCharset}(${input.charsetSource})`);
+        if (input.unsupportedCharset) {
+            metaBits.push(`unsupportedCharset=${input.unsupportedCharset}`);
         }
+        if (input.truncated)
+            metaBits.push(`truncated@${input.limit}`);
     }
-    const metaBits = [
-        `attempts=${input.attempts}`,
-        `bytes=${input.bytesRead}`,
-    ];
-    if (input.truncated)
-        metaBits.push(`truncated@${input.limit}`);
     if (input.usedUrl !== input.requestedUrl)
         metaBits.push(`via=${input.usedUrl}`);
-    if (input.requestedUrl !== input.finalUrl && input.redirectHops.length === 0) {
-        metaBits.push(`requested=${input.requestedUrl}`);
+    if (input.requestedMethod !== input.finalMethod ||
+        (input.requestedUrl !== input.finalUrl && input.redirectHops.length === 0)) {
+        metaBits.push(`requested=${input.requestedMethod} ${input.requestedUrl}`);
     }
     lines.push(metaBits.join(" "));
+    if (input.responsePart !== "headers") {
+        lines.push("capture: response-body bytes after Fetch transfer/content decoding; SHA-256 covers exactly the captured bytes");
+    }
     if (input.finalUrl.startsWith("https:") && !input.insecureTls) {
         lines.push("TLS: leaf cert not captured by http.fetch — use web.fetch with includeTls=true for fingerprint/SAN.");
     }
-    // ALL response headers, always — cheap vs body and critical for pentest.
-    // Security/fingerprint headers first for scannability; remainder sorted after.
-    const allHeaders = [];
-    input.headers.forEach((v, k) => allHeaders.push([k, v]));
-    allHeaders.sort(([a], [b]) => a.localeCompare(b));
+    // Fetch normalizes names and most repeated values. Preserve every exposed
+    // field, with individual Set-Cookie lines, and be explicit about the limit.
+    const allHeaders = snapshotHeaders(input.headers);
     const priority = [];
     const rest = [];
-    for (const [k, v] of allHeaders) {
-        const line = `${k}: ${v}`;
-        if (PRIORITY_HEADER_RE.test(k))
+    for (const [key, value] of allHeaders) {
+        const line = `${key}: ${value}`;
+        if (PRIORITY_HEADER_RE.test(key))
             priority.push(line);
         else
             rest.push(line);
     }
     lines.push("");
-    lines.push("Headers:");
+    lines.push("Final response headers (runtime-normalized; Set-Cookie preserved separately):");
     for (const line of priority)
         lines.push(line);
     for (const line of rest)
@@ -620,21 +784,21 @@ function formatHttpEvidence(input) {
         lines.push("");
         lines.push(`Tech hints: ${tech}`);
     }
-    if (input.method !== "HEAD") {
+    if (input.finalMethod !== "HEAD") {
         lines.push("");
         lines.push("Body:");
         if (input.isBinary) {
-            lines.push(`[Binary content (${input.contentType || "unknown content type"}) — raw body suppressed]`);
+            lines.push(`[Binary content (${input.contentType || "unknown content type"}) — textual rendering suppressed; use bodySha256 and headers as evidence]`);
         }
         else {
-            const ct = input.contentType.toLowerCase();
-            const isHtml = ct.includes("html");
-            // Single representation: readable HTML OR raw captured source.
+            const isHtml = input.contentType.toLowerCase().includes("html");
+            // Raw is the forensic default and preserves source comments, tags,
+            // attributes, values, whitespace, and malformed markup after decoding.
             const bodyText = isHtml && input.responseMode === "readable"
-                ? toReadableText(input.body) || input.body
+                ? toReadableText(input.body, input.finalUrl) || input.body
                 : input.body;
             if (input.truncated) {
-                lines.push(`${bodyText || "(empty)"}\n... (wire capture stopped at ${input.limit.toLocaleString()} bytes — raise maxBytes for more body)`);
+                lines.push(`${bodyText || "(empty)"}\n... (capture stopped at ${input.limit.toLocaleString()} decoded response-body bytes — raise maxBytes for more body)`);
             }
             else {
                 lines.push(bodyText || "(empty)");
@@ -700,6 +864,11 @@ function deriveTechHints(headers, body, contentType) {
     }
     return bits.join("; ");
 }
+function clampCaptureBytes(value) {
+    if (value === undefined || !Number.isFinite(value))
+        return DEFAULT_MAX_BYTES;
+    return Math.max(0, Math.min(MAX_CAPTURE_BYTES, Math.floor(value)));
+}
 function clampRetries(value) {
     if (!Number.isFinite(value))
         return DEFAULT_RETRIES;
@@ -713,10 +882,10 @@ function sleep(ms) {
 }
 async function drainResponse(response) {
     try {
-        await response.arrayBuffer();
+        await response.body?.cancel();
     }
     catch {
-        // Best effort only; retrying is more important than draining.
+        // Best effort only; redirect/retry can proceed after a socket-close race.
     }
 }
 function isBinaryContentType(contentType) {
@@ -734,19 +903,29 @@ function isBinaryContentType(contentType) {
     }
     return false;
 }
-function isBinaryContent(text) {
-    const sample = text.slice(0, 2048);
-    if (sample.includes("\u0000"))
-        return true;
+function isTextualContentType(contentType) {
+    const ct = contentType.toLowerCase().split(";", 1)[0]?.trim() ?? "";
+    return (ct.startsWith("text/") ||
+        ct === "image/svg+xml" ||
+        ct === "application/json" ||
+        ct.endsWith("+json") ||
+        ct === "application/xml" ||
+        ct.endsWith("+xml") ||
+        ct === "application/javascript" ||
+        ct === "application/x-javascript" ||
+        ct === "application/x-www-form-urlencoded");
+}
+function isBinaryContent(body) {
+    const sample = body.subarray(0, 2_048);
     let nonPrintableCount = 0;
-    for (let i = 0; i < sample.length; i++) {
-        const code = sample.charCodeAt(i);
-        // Control chars other than tab (9), lf (10), cr (13)
-        if (code < 32 && code !== 9 && code !== 10 && code !== 13) {
-            nonPrintableCount++;
-            if (nonPrintableCount > 5) {
+    for (const byte of sample) {
+        if (byte === 0)
+            return true;
+        // Control bytes other than tab (9), lf (10), cr (13).
+        if (byte < 32 && byte !== 9 && byte !== 10 && byte !== 13) {
+            nonPrintableCount += 1;
+            if (nonPrintableCount > 5)
                 return true;
-            }
         }
     }
     return false;