@massa-ai/core 1.2.1

package/dist/services/web/index.js

@@ -0,0 +1,15 @@
+/**
+ * Web ingestion service — SSRF-guarded fetch + HTML→md + index.
+ *
+ * Public surface:
+ *   - classifyIp / assertUrlSafe / fetchWithSsrfGuard / SsrfBlockedError (ssrf)
+ *   - htmlToMarkdown / jsonToKeyPathChunks (html-to-md)
+ *   - fetchAndConvertOne / composeFetchCacheKey / MAX_FETCH_BYTES /
+ *     DEFAULT_FETCH_TTL_MS + types (fetcher)
+ *   - WebController (orchestration)
+ */
+export { classifyIp, assertUrlSafe, fetchWithSsrfGuard, SsrfBlockedError, setDnsResolver, MAX_REDIRECTS, } from "./ssrf.js";
+export { htmlToMarkdown, jsonToKeyPathChunks, } from "./html-to-md.js";
+export { fetchAndConvertOne, composeFetchCacheKey, MAX_FETCH_BYTES, DEFAULT_FETCH_TTL_MS, } from "./fetcher.js";
+export { WebController, } from "./web-controller.js";
+//# sourceMappingURL=index.js.map

package/dist/services/web/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/services/web/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,UAAU,EACV,aAAa,EACb,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,aAAa,GAKd,MAAM,WAAW,CAAC;AAEnB,OAAO,EACL,cAAc,EACd,mBAAmB,GAEpB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EACf,oBAAoB,GAMrB,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,aAAa,GAId,MAAM,qBAAqB,CAAC"}

package/dist/services/web/ssrf.d.ts

@@ -0,0 +1,142 @@
+/**
+ * SSRF guard — Server-Side Request Forgery defense for `fetch_and_index`.
+ *
+ * Policy (massa-ai is STRICT BY DEFAULT, unlike context-mode's lenient mode):
+ *
+ *   HARD BLOCK — no legitimate fetch use case reaches these:
+ *     - non-http(s) schemes (file://, gopher://, javascript:, data:, …)
+ *     - 169.254.0.0/16 link-local — INCLUDES 169.254.169.254 (AWS/GCP/Azure
+ *       IMDS cloud-credential endpoint, the highest-value SSRF target)
+ *     - IPv6 link-local fe80::/10
+ *     - multicast (224.0.0.0/4 IPv4, ff00::/8 IPv6)
+ *     - unspecified / current-network (0.0.0.0/8, ::)
+ *     - loopback (127.0.0.0/8, ::1) — local dev servers are out of scope here
+ *     - RFC1918 private (10/8, 172.16/12, 192.168/16) and IPv6 ULA (fc00::/7)
+ *     - malformed / non-IP strings
+ *
+ * The massa-ai trust model treats fetch targets as OPEN WORLD (arbitrary
+ * URLs the agent fetched at the user's request), so a private/loopback hit is
+ * treated as a rebinding attack probe, not a local-dev convenience. This is the
+ * conservative default; there is no lenient escape hatch (unlike context-mode's
+ * `CTX_FETCH_STRICT` toggle) — if a future task needs to fetch localhost it can
+ * add an explicit allowlist.
+ *
+ * DNS-rebinding defense (TOCTOU-safe):
+ *   1. Resolve the hostname ONCE and classify EACH resolved address.
+ *   2. PIN the connection to the already-classified resolved IP by rewriting
+ *      the fetch URL to the literal IP and setting the `Host` header to the
+ *      original hostname. This closes the TOCTOU: the fetch's own connect-time
+ *      DNS re-resolution is bypassed entirely because the URL host is now a
+ *      literal IP (no lookup occurs). An attacker cannot serve a public IP at
+ *      check-time and a private IP at connect-time for the SAME request — the
+ *      connect-time resolution never happens.
+ *      (The alternative — a custom undici `connect.lookup` hook — does NOT work
+ *      under bun: bun's native fetch ignores the undici Dispatcher entirely,
+ *      and even the bundled undici shim does not invoke `connect.lookup`. The
+ *      literal-IP + Host-header approach is the only one that works under bun
+ *      and is also portable to node/undici.)
+ *   3. On fetch, walk redirects MANUALLY (do not let fetch auto-follow) and
+ *      re-resolve + re-classify + re-PIN the hostname of every hop. An attacker
+ *      can return a public IP for the first request, then 302 to an internal
+ *      host whose DNS now resolves to 169.254.169.254. Manual redirect walking
+ *      with per-hop resolution+pinning defeats this.
+ *
+ * Exported seams:
+ *   - classifyIp(ip) — pure IP classifier (tested directly with literals).
+ *   - assertUrlSafe(url) — resolve + classify, returns pinned IP(s); throws on
+ *     any blocked IP.
+ *   - fetchWithSsrfGuard(url, opts) — fetch with DNS-pinned connect + manual
+ *     redirect walk that re-runs assertUrlSafe (and re-pins) at every hop.
+ */
+export type IpClass = "block" | "public";
+/** Hard cap on redirect hops we will walk before giving up (loop defense). */
+export declare const MAX_REDIRECTS = 8;
+/**
+ * Injectable DNS resolver seam. Defaults to node:dns/promises `lookup`. Tests
+ * swap this (via `setDnsResolver`) to simulate DNS rebinding / IMDS aliases
+ * WITHOUT patching the read-only module namespace. Production code never calls
+ * the setter.
+ */
+export type DnsResolver = (hostname: string) => Promise<{
+    address: string;
+}[]>;
+/** Test seam: override the DNS resolver. Returns a restore function. */
+export declare function setDnsResolver(resolver: DnsResolver): () => void;
+/**
+ * Classify a single IP literal.
+ *
+ * Returns "block" for link-local / loopback / private / multicast / unspecified
+ * / malformed. Returns "public" only for genuinely routable addresses.
+ *
+ * IPv4-mapped IPv6 (`::ffff:127.0.0.1`) is routed through the IPv4 classifier so
+ * loopback mapped into the v6 space is still caught.
+ *
+ * RFC 6874 zone identifiers (`fe80::1%eth0`, URL-encoded `%25eth0`) are stripped
+ * BEFORE classification — without the strip, `::1%eth0` no longer matches
+ * `=== "::1"` and would fall through to "public", silently bypassing the guard.
+ */
+export declare function classifyIp(rawIp: string): IpClass;
+/** Error thrown when a URL or one of its redirect hops is blocked by SSRF. */
+export declare class SsrfBlockedError extends Error {
+    readonly url: string;
+    readonly resolvedIp?: string | undefined;
+    constructor(message: string, url: string, resolvedIp?: string | undefined);
+}
+/**
+ * Result of `assertUrlSafe`: the connection-pin parameters the caller must use
+ * when issuing the fetch so the TOCTOU gap between check-time and connect-time
+ * DNS resolution is closed.
+ *
+ *   - `pinnedIp`: the already-classified IP literal to connect to (replaces the
+ *     hostname in the fetch URL). For a URL that already names a literal IP,
+ *     this is that same IP. For a hostname, this is the first public resolved
+ *     address.
+ *   - `originalHost`: the hostname to send in the `Host` header so virtual
+ *     hosting + TLS SNI still work. Equals the URL's hostname (brackets stripped
+ *     for IPv6). When the URL already names a literal IP, this equals pinnedIp
+ *     and no Host header rewrite is needed.
+ *   - `isLiteralIpUrl`: true when the original URL host was already an IP
+ *     literal (no DNS resolution occurred). Lets the caller skip the rewrite.
+ */
+export interface UrlSafetyResult {
+    pinnedIp: string;
+    originalHost: string;
+    isLiteralIpUrl: boolean;
+}
+/**
+ * Resolve `hostname` and assert every resolved IP is public. Throws
+ * `SsrfBlockedError` on the first blocked address. A hostname that resolves to
+ * a MIX of public + private is still rejected (TOCTOU-safe: we re-resolve on
+ * fetch, so even if the resolver rotates answers, every hop is checked).
+ *
+ * Returns the `UrlSafetyResult` (pinned IP + original hostname) so the caller
+ * can pin the fetch connection to the already-classified IP and close the
+ * TOCTOU between check-time and connect-time DNS resolution.
+ */
+export declare function assertUrlSafe(rawUrl: string): Promise<UrlSafetyResult>;
+export interface FetchGuardOptions {
+    /** Per-request timeout (ms). Default 30_000. */
+    timeoutMs?: number;
+    /** AbortSignal from the caller (e.g. for test cancellation). */
+    signal?: AbortSignal;
+    /** Composed AbortSignal: combines the timeout timer and the caller's signal. */
+    _compose?: (signals: AbortSignal[]) => AbortSignal;
+}
+/**
+ * Fetch a URL with the SSRF DNS-pin + redirect-walk defense.
+ *
+ * TWO-LAYER DEFENSE (TOCTOU-safe):
+ *   1. DNS PIN: `assertUrlSafe` resolves the hostname once and classifies the
+ *      IP. We then rewrite the fetch URL to connect to that LITERAL IP and set
+ *      the `Host` header to the original hostname. Because the fetch URL's host
+ *      is now a literal IP, NO DNS lookup occurs at connect time — the attacker
+ *      cannot serve a different (private) IP at connect. bun's fetch honors the
+ *      `Host` header for both HTTP virtual hosting and HTTPS TLS SNI.
+ *   2. REDIRECT WALK: does NOT auto-follow. Each 3xx Location header is
+ *      re-resolved + re-classified + re-pinned before the next hop. Any blocked
+ *      hop throws `SsrfBlockedError`. Capped at MAX_REDIRECTS to defeat loops.
+ *
+ * Returns the final (non-redirect) Response. The caller reads the body.
+ */
+export declare function fetchWithSsrfGuard(rawUrl: string, opts?: FetchGuardOptions): Promise<Response>;
+//# sourceMappingURL=ssrf.d.ts.map

package/dist/services/web/ssrf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ssrf.d.ts","sourceRoot":"","sources":["../../../src/services/web/ssrf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAKH,MAAM,MAAM,OAAO,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEzC,8EAA8E;AAC9E,eAAO,MAAM,aAAa,IAAI,CAAC;AAE/B;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,CACxB,QAAQ,EAAE,MAAM,KACb,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAAC;AAOpC,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,QAAQ,EAAE,WAAW,GAAG,MAAM,IAAI,CAMhE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CA0DjD;AAED,8EAA8E;AAC9E,qBAAa,gBAAiB,SAAQ,KAAK;IAGvC,QAAQ,CAAC,GAAG,EAAE,MAAM;IACpB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM;gBAF5B,OAAO,EAAE,MAAM,EACN,GAAG,EAAE,MAAM,EACX,UAAU,CAAC,EAAE,MAAM,YAAA;CAK/B;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;;GASG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAyE5E;AAkBD,MAAM,WAAW,iBAAiB;IAChC,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,WAAW,CAAC;CACpD;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,iBAAsB,GAC3B,OAAO,CAAC,QAAQ,CAAC,CAgEnB"}

package/dist/services/web/ssrf.js

@@ -0,0 +1,334 @@
+/**
+ * SSRF guard — Server-Side Request Forgery defense for `fetch_and_index`.
+ *
+ * Policy (massa-ai is STRICT BY DEFAULT, unlike context-mode's lenient mode):
+ *
+ *   HARD BLOCK — no legitimate fetch use case reaches these:
+ *     - non-http(s) schemes (file://, gopher://, javascript:, data:, …)
+ *     - 169.254.0.0/16 link-local — INCLUDES 169.254.169.254 (AWS/GCP/Azure
+ *       IMDS cloud-credential endpoint, the highest-value SSRF target)
+ *     - IPv6 link-local fe80::/10
+ *     - multicast (224.0.0.0/4 IPv4, ff00::/8 IPv6)
+ *     - unspecified / current-network (0.0.0.0/8, ::)
+ *     - loopback (127.0.0.0/8, ::1) — local dev servers are out of scope here
+ *     - RFC1918 private (10/8, 172.16/12, 192.168/16) and IPv6 ULA (fc00::/7)
+ *     - malformed / non-IP strings
+ *
+ * The massa-ai trust model treats fetch targets as OPEN WORLD (arbitrary
+ * URLs the agent fetched at the user's request), so a private/loopback hit is
+ * treated as a rebinding attack probe, not a local-dev convenience. This is the
+ * conservative default; there is no lenient escape hatch (unlike context-mode's
+ * `CTX_FETCH_STRICT` toggle) — if a future task needs to fetch localhost it can
+ * add an explicit allowlist.
+ *
+ * DNS-rebinding defense (TOCTOU-safe):
+ *   1. Resolve the hostname ONCE and classify EACH resolved address.
+ *   2. PIN the connection to the already-classified resolved IP by rewriting
+ *      the fetch URL to the literal IP and setting the `Host` header to the
+ *      original hostname. This closes the TOCTOU: the fetch's own connect-time
+ *      DNS re-resolution is bypassed entirely because the URL host is now a
+ *      literal IP (no lookup occurs). An attacker cannot serve a public IP at
+ *      check-time and a private IP at connect-time for the SAME request — the
+ *      connect-time resolution never happens.
+ *      (The alternative — a custom undici `connect.lookup` hook — does NOT work
+ *      under bun: bun's native fetch ignores the undici Dispatcher entirely,
+ *      and even the bundled undici shim does not invoke `connect.lookup`. The
+ *      literal-IP + Host-header approach is the only one that works under bun
+ *      and is also portable to node/undici.)
+ *   3. On fetch, walk redirects MANUALLY (do not let fetch auto-follow) and
+ *      re-resolve + re-classify + re-PIN the hostname of every hop. An attacker
+ *      can return a public IP for the first request, then 302 to an internal
+ *      host whose DNS now resolves to 169.254.169.254. Manual redirect walking
+ *      with per-hop resolution+pinning defeats this.
+ *
+ * Exported seams:
+ *   - classifyIp(ip) — pure IP classifier (tested directly with literals).
+ *   - assertUrlSafe(url) — resolve + classify, returns pinned IP(s); throws on
+ *     any blocked IP.
+ *   - fetchWithSsrfGuard(url, opts) — fetch with DNS-pinned connect + manual
+ *     redirect walk that re-runs assertUrlSafe (and re-pins) at every hop.
+ */
+import { lookup as dnsLookup } from "node:dns/promises";
+import { logger } from "@massa-ai/shared";
+/** Hard cap on redirect hops we will walk before giving up (loop defense). */
+export const MAX_REDIRECTS = 8;
+let dnsResolver = async (hostname) => {
+    // node:dns/promises `lookup` with `all: true` returns {address,family}[].
+    const records = await dnsLookup(hostname, { all: true, verbatim: true });
+    return records.map((r) => ({ address: r.address }));
+};
+/** Test seam: override the DNS resolver. Returns a restore function. */
+export function setDnsResolver(resolver) {
+    const prev = dnsResolver;
+    dnsResolver = resolver;
+    return () => {
+        dnsResolver = prev;
+    };
+}
+/**
+ * Classify a single IP literal.
+ *
+ * Returns "block" for link-local / loopback / private / multicast / unspecified
+ * / malformed. Returns "public" only for genuinely routable addresses.
+ *
+ * IPv4-mapped IPv6 (`::ffff:127.0.0.1`) is routed through the IPv4 classifier so
+ * loopback mapped into the v6 space is still caught.
+ *
+ * RFC 6874 zone identifiers (`fe80::1%eth0`, URL-encoded `%25eth0`) are stripped
+ * BEFORE classification — without the strip, `::1%eth0` no longer matches
+ * `=== "::1"` and would fall through to "public", silently bypassing the guard.
+ */
+export function classifyIp(rawIp) {
+    // Strip RFC 6874 zone id first (e.g. `fe80::1%eth0` or `%25eth0`).
+    const pctIdx = rawIp.indexOf("%");
+    const ip = pctIdx === -1 ? rawIp : rawIp.slice(0, pctIdx);
+    const lower = ip.toLowerCase();
+    // IPv6 (detected by presence of `:` so IPv4-mapped addresses route correctly).
+    if (lower.includes(":")) {
+        // IPv4-mapped IPv6 (`::ffff:127.0.0.1`) — recurse through IPv4 classifier.
+        // Node's URL normalizes these to hex (`::ffff:7f00:1`), so handle BOTH the
+        // dotted-decimal and the hex-compressed forms. Without the hex form, an
+        // attacker could reach IMDS via `http://[::ffff:169.254.169.254]/` because
+        // URL.hostname yields `[::ffff:a9fe:a9fe]` which the decimal-only regex missed.
+        const v4MappedDecimal = lower.match(/^::ffff:([\d.]+)$/);
+        if (v4MappedDecimal)
+            return classifyIp(v4MappedDecimal[1]);
+        const v4MappedHex = lower.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
+        if (v4MappedHex) {
+            const hi = parseInt(v4MappedHex[1], 16);
+            const lo = parseInt(v4MappedHex[2], 16);
+            const a = (hi >> 8) & 0xff;
+            const b = hi & 0xff;
+            const c = (lo >> 8) & 0xff;
+            const d = lo & 0xff;
+            return classifyIp(`${a}.${b}.${c}.${d}`);
+        }
+        if (lower === "::")
+            return "block"; // unspecified
+        // fe80::/10 link-local — match the high-nibble range fe8..feb.
+        if (lower.startsWith("fe8") ||
+            lower.startsWith("fe9") ||
+            lower.startsWith("fea") ||
+            lower.startsWith("feb")) {
+            return "block";
+        }
+        if (lower.startsWith("ff"))
+            return "block"; // ff00::/8 multicast
+        if (lower === "::1")
+            return "block"; // loopback
+        // fc00::/7 unique-local (fc.. or fd..)
+        if (lower.startsWith("fc") || lower.startsWith("fd"))
+            return "block";
+        return "public";
+    }
+    // IPv4 — or a non-IP string (malformed = block).
+    if (!ip.includes("."))
+        return "block";
+    const parts = ip.split(".").map((p) => parseInt(p, 10));
+    if (parts.length !== 4 || parts.some((p) => Number.isNaN(p) || p < 0 || p > 255)) {
+        return "block";
+    }
+    const [a, b] = parts;
+    if (a === 169 && b === 254)
+        return "block"; // link-local incl. 169.254.169.254 (IMDS)
+    if (a === 0)
+        return "block"; // 0.0.0.0/8 current network
+    if (a >= 224)
+        return "block"; // 224.0.0.0/4 multicast + 240+ reserved
+    if (a === 127)
+        return "block"; // 127.0.0.0/8 loopback
+    if (a === 10)
+        return "block"; // 10.0.0.0/8 RFC1918
+    if (a === 172 && b >= 16 && b <= 31)
+        return "block"; // 172.16.0.0/12 RFC1918
+    if (a === 192 && b === 168)
+        return "block"; // 192.168.0.0/16 RFC1918
+    if (a === 100 && b >= 64 && b <= 127)
+        return "block"; // 100.64.0.0/10 CGNAT
+    return "public";
+}
+/** Error thrown when a URL or one of its redirect hops is blocked by SSRF. */
+export class SsrfBlockedError extends Error {
+    url;
+    resolvedIp;
+    constructor(message, url, resolvedIp) {
+        super(message);
+        this.url = url;
+        this.resolvedIp = resolvedIp;
+        this.name = "SsrfBlockedError";
+    }
+}
+/**
+ * Resolve `hostname` and assert every resolved IP is public. Throws
+ * `SsrfBlockedError` on the first blocked address. A hostname that resolves to
+ * a MIX of public + private is still rejected (TOCTOU-safe: we re-resolve on
+ * fetch, so even if the resolver rotates answers, every hop is checked).
+ *
+ * Returns the `UrlSafetyResult` (pinned IP + original hostname) so the caller
+ * can pin the fetch connection to the already-classified IP and close the
+ * TOCTOU between check-time and connect-time DNS resolution.
+ */
+export async function assertUrlSafe(rawUrl) {
+    let parsed;
+    try {
+        parsed = new URL(rawUrl);
+    }
+    catch {
+        throw new SsrfBlockedError(`invalid URL: ${rawUrl}`, rawUrl);
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+        throw new SsrfBlockedError(`URL scheme "${parsed.protocol}" not allowed (only http: and https:)`, rawUrl);
+    }
+    // Bracketed IPv6 host literal (`[::1]`) — Node's `URL.hostname` does NOT
+    // strip the brackets (it returns `"[::1]"`, `"[fe80::1]"`, etc.); the old
+    // comment claiming otherwise was wrong and caused a CRITICAL SSRF bypass
+    // where `http://[::1]/` and `http://[fe80::1]/` fell through to "public".
+    // Strip brackets so classifyIp sees the bare IPv6 literal.
+    const hostname = parsed.hostname.replace(/^\[|\]$/g, "");
+    // Literal IP in the URL (no DNS needed): URL parses `http://127.0.0.1/`
+    // with hostname === "127.0.0.1". classifyIp handles it directly. A hostname
+    // that is NOT an IP literal goes through DNS resolution below.
+    if (looksLikeIpLiteral(hostname)) {
+        if (classifyIp(hostname) === "block") {
+            throw new SsrfBlockedError(`URL host ${hostname} is a blocked IP range`, rawUrl, hostname);
+        }
+        return { pinnedIp: hostname, originalHost: hostname, isLiteralIpUrl: true };
+    }
+    let records;
+    try {
+        records = await dnsResolver(hostname);
+    }
+    catch (err) {
+        const code = err?.code ?? "";
+        const msg = err instanceof Error ? err.message : String(err);
+        // Surface transient DNS codes so the caller can distinguish "blocked" from
+        // "resolver hiccup" and retry sensibly.
+        throw new SsrfBlockedError(`DNS lookup failed for "${hostname}" (${code}): ${msg}`, rawUrl);
+    }
+    if (records.length === 0) {
+        throw new SsrfBlockedError(`DNS returned no records for "${hostname}"`, rawUrl);
+    }
+    for (const rec of records) {
+        if (classifyIp(rec.address) === "block") {
+            throw new SsrfBlockedError(`"${hostname}" resolves to blocked IP ${rec.address} (loopback / private / link-local / multicast)`, rawUrl, rec.address);
+        }
+    }
+    // Pin to the first classified-public IP. Every record was checked above, so
+    // the first is safe to connect to. (We do not round-robin across resolved
+    // IPs because only the one we pin is guaranteed classified; a later answer
+    // could be attacker-rotated between assert and connect.)
+    return {
+        pinnedIp: records[0].address,
+        originalHost: hostname,
+        isLiteralIpUrl: false,
+    };
+}
+/**
+ * Heuristic: does `hostname` parse as an IP literal (v4 or v6)? Used to skip
+ * DNS when the URL already names an IP directly. IPv6 zone ids are tolerated.
+ */
+function looksLikeIpLiteral(hostname) {
+    const h = hostname.split("%")[0]; // tolerate zone id
+    if (h.includes(":"))
+        return true; // IPv6
+    // IPv4 = four dot-separated 0-255 octets.
+    const parts = h.split(".");
+    if (parts.length !== 4)
+        return false;
+    return parts.every((p) => {
+        const n = Number(p);
+        return Number.isInteger(n) && n >= 0 && n <= 255 && String(n) === p;
+    });
+}
+/**
+ * Fetch a URL with the SSRF DNS-pin + redirect-walk defense.
+ *
+ * TWO-LAYER DEFENSE (TOCTOU-safe):
+ *   1. DNS PIN: `assertUrlSafe` resolves the hostname once and classifies the
+ *      IP. We then rewrite the fetch URL to connect to that LITERAL IP and set
+ *      the `Host` header to the original hostname. Because the fetch URL's host
+ *      is now a literal IP, NO DNS lookup occurs at connect time — the attacker
+ *      cannot serve a different (private) IP at connect. bun's fetch honors the
+ *      `Host` header for both HTTP virtual hosting and HTTPS TLS SNI.
+ *   2. REDIRECT WALK: does NOT auto-follow. Each 3xx Location header is
+ *      re-resolved + re-classified + re-pinned before the next hop. Any blocked
+ *      hop throws `SsrfBlockedError`. Capped at MAX_REDIRECTS to defeat loops.
+ *
+ * Returns the final (non-redirect) Response. The caller reads the body.
+ */
+export async function fetchWithSsrfGuard(rawUrl, opts = {}) {
+    const { timeoutMs = 30_000, signal, _compose } = opts;
+    const safety = await assertUrlSafe(rawUrl);
+    let url = rawUrl;
+    let pin = safety;
+    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+        const composed = _compose && signal
+            ? _compose([signal, AbortSignal.timeout(timeoutMs)])
+            : (signal
+                ? _compose
+                    ? _compose([signal])
+                    : signal
+                : AbortSignal.timeout(timeoutMs));
+        // Build the DNS-pinned fetch URL + Host header. When the URL already names
+        // a literal IP (isLiteralIpUrl), no rewrite is needed — the connect-time
+        // DNS lookup is already absent. Otherwise, replace the hostname in the URL
+        // with the pinned classified IP and send the original hostname as Host.
+        const { fetchUrl, hostHeader } = pinConnection(url, pin);
+        // `redirect: "manual"` keeps the Response opaque-redirect; we read the
+        // Location header ourselves and re-validate the target.
+        const resp = await fetch(fetchUrl, {
+            redirect: "manual",
+            signal: composed,
+            // Conservative default headers — identify as a fetcher, not a browser.
+            // Host header pins virtual hosting + TLS SNI to the original hostname.
+            headers: {
+                "user-agent": "massa-ai-fetch/1.0",
+                ...(hostHeader ? { host: hostHeader } : {}),
+            },
+        });
+        // 3xx with a Location → walk it.
+        if (resp.status >= 300 && resp.status < 400) {
+            const location = resp.headers.get("location");
+            // Drain so the underlying socket can be reused.
+            await resp.body?.cancel().catch(() => { });
+            if (!location) {
+                throw new SsrfBlockedError(`redirect ${resp.status} without Location header`, url);
+            }
+            const next = new URL(location, url).toString(); // resolve relative redirects
+            if (hop === MAX_REDIRECTS) {
+                throw new SsrfBlockedError(`exceeded ${MAX_REDIRECTS} redirects (loop?)`, next);
+            }
+            logger.debug?.("ssrf redirect hop", { from: url, to: next, status: resp.status });
+            // Re-resolve + re-classify + re-pin the NEXT hop (DNS may now differ).
+            pin = await assertUrlSafe(next);
+            url = next;
+            continue;
+        }
+        return resp;
+    }
+    // Unreachable: the loop either returns or throws on the last iteration.
+    throw new SsrfBlockedError("redirect walk exhausted", url);
+}
+/**
+ * Build a DNS-pinned fetch URL + Host header from the original URL and the
+ * `UrlSafetyResult` (pinned IP + original hostname).
+ *
+ * When the URL already names a literal IP (`isLiteralIpUrl`), returns the URL
+ * unchanged with no Host header (no pinning needed — connect-time DNS is
+ * already absent). Otherwise, rewrites the URL's host to the pinned IP literal
+ * and returns the original hostname as the Host header value.
+ *
+ * IPv6 pinned IPs are bracketed in the URL (`http://[2606:4700::1]/`) per
+ * RFC 3986. The Host header uses the bare original hostname (no brackets — it
+ * was a hostname, not an IP, in the non-literal branch).
+ */
+function pinConnection(url, pin) {
+    if (pin.isLiteralIpUrl) {
+        return { fetchUrl: url, hostHeader: null };
+    }
+    const parsed = new URL(url);
+    const ipIsV6 = pin.pinnedIp.includes(":");
+    parsed.hostname = ipIsV6 ? `[${pin.pinnedIp}]` : pin.pinnedIp;
+    return { fetchUrl: parsed.toString(), hostHeader: pin.originalHost };
+}
+//# sourceMappingURL=ssrf.js.map

package/dist/services/web/ssrf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ssrf.js","sourceRoot":"","sources":["../../../src/services/web/ssrf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAEH,OAAO,EAAE,MAAM,IAAI,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAI1C,8EAA8E;AAC9E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC;AAW/B,IAAI,WAAW,GAAgB,KAAK,EAAE,QAAQ,EAAE,EAAE;IAChD,0EAA0E;IAC1E,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,QAAqB;IAClD,MAAM,IAAI,GAAG,WAAW,CAAC;IACzB,WAAW,GAAG,QAAQ,CAAC;IACvB,OAAO,GAAG,EAAE;QACV,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,mEAAmE;IACnE,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAE/B,+EAA+E;IAC/E,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,2EAA2E;QAC3E,2EAA2E;QAC3E,wEAAwE;QACxE,2EAA2E;QAC3E,gFAAgF;QAChF,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACzD,IAAI,eAAe;YAAE,OAAO,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC5E,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,EAAE,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACxC,MAAM,EAAE,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;YAC3B,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;YAC3B,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;YACpB,OAAO,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,OAAO,CAAC,CAAC,cAAc;QAClD,+DAA+D;QAC/D,IACE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;YACvB,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;YACvB,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;YACvB,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,EACvB,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,OAAO,CAAC,CAAC,qBAAqB;QACjE,IAAI,KAAK,KAAK,KAAK;YAAE,OAAO,OAAO,CAAC,CAAC,WAAW;QAChD,uCAAuC;QACvC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,OAAO,CAAC;QACrE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,iDAAiD;IACjD,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC;IACtC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACxD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC;QACjF,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;IACrB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,OAAO,CAAC,CAAC,0CAA0C;IACtF,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,4BAA4B;IACzD,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,OAAO,CAAC,CAAC,wCAAwC;IACtE,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,OAAO,CAAC,CAAC,uBAAuB;IACtD,IAAI,CAAC,KAAK,EAAE;QAAE,OAAO,OAAO,CAAC,CAAC,qBAAqB;IACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,OAAO,CAAC,CAAC,wBAAwB;IAC7E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,OAAO,CAAC,CAAC,yBAAyB;IACrE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,OAAO,CAAC,CAAC,sBAAsB;IAC5E,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAG9B;IACA;IAHX,YACE,OAAe,EACN,GAAW,EACX,UAAmB;QAE5B,KAAK,CAAC,OAAO,CAAC,CAAC;QAHN,QAAG,GAAH,GAAG,CAAQ;QACX,eAAU,GAAV,UAAU,CAAS;QAG5B,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAwBD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc;IAChD,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,gBAAgB,CAAC,gBAAgB,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAChE,MAAM,IAAI,gBAAgB,CACxB,eAAe,MAAM,CAAC,QAAQ,uCAAuC,EACrE,MAAM,CACP,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,0EAA0E;IAC1E,yEAAyE;IACzE,0EAA0E;IAC1E,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEzD,wEAAwE;IACxE,4EAA4E;IAC5E,+DAA+D;IAC/D,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,OAAO,EAAE,CAAC;YACrC,MAAM,IAAI,gBAAgB,CACxB,YAAY,QAAQ,wBAAwB,EAC5C,MAAM,EACN,QAAQ,CACT,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IAED,IAAI,OAA8B,CAAC;IACnC,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GAAI,GAAyC,EAAE,IAAI,IAAI,EAAE,CAAC;QACpE,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,2EAA2E;QAC3E,wCAAwC;QACxC,MAAM,IAAI,gBAAgB,CACxB,0BAA0B,QAAQ,MAAM,IAAI,MAAM,GAAG,EAAE,EACvD,MAAM,CACP,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,gBAAgB,CAAC,gCAAgC,QAAQ,GAAG,EAAE,MAAM,CAAC,CAAC;IAClF,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,OAAO,EAAE,CAAC;YACxC,MAAM,IAAI,gBAAgB,CACxB,IAAI,QAAQ,4BAA4B,GAAG,CAAC,OAAO,gDAAgD,EACnG,MAAM,EACN,GAAG,CAAC,OAAO,CACZ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,2EAA2E;IAC3E,yDAAyD;IACzD,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;QAC5B,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,QAAgB;IAC1C,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;IACrD,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,OAAO;IACzC,0CAA0C;IAC1C,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;AACL,CAAC;AAWD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAc,EACd,OAA0B,EAAE;IAE5B,MAAM,EAAE,SAAS,GAAG,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IACtD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAE3C,IAAI,GAAG,GAAG,MAAM,CAAC;IACjB,IAAI,GAAG,GAAG,MAAM,CAAC;IACjB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,aAAa,EAAE,GAAG,EAAE,EAAE,CAAC;QAC9C,MAAM,QAAQ,GACZ,QAAQ,IAAI,MAAM;YAChB,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC,MAAM;gBACL,CAAC,CAAC,QAAQ;oBACR,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC;oBACpB,CAAC,CAAC,MAAM;gBACV,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAE1C,2EAA2E;QAC3E,yEAAyE;QACzE,2EAA2E;QAC3E,wEAAwE;QACxE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAEzD,uEAAuE;QACvE,wDAAwD;QACxD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YACjC,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,QAAQ;YAChB,uEAAuE;YACvE,uEAAuE;YACvE,OAAO,EAAE;gBACP,YAAY,EAAE,oBAAoB;gBAClC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5C;SACF,CAAC,CAAC;QAEH,iCAAiC;QACjC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC9C,gDAAgD;YAChD,MAAM,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,gBAAgB,CACxB,YAAY,IAAI,CAAC,MAAM,0BAA0B,EACjD,GAAG,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,6BAA6B;YAC7E,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;gBAC1B,MAAM,IAAI,gBAAgB,CACxB,YAAY,aAAa,oBAAoB,EAC7C,IAAI,CACL,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,KAAK,EAAE,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,uEAAuE;YACvE,GAAG,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;YAChC,GAAG,GAAG,IAAI,CAAC;YACX,SAAS;QACX,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IACD,wEAAwE;IACxE,MAAM,IAAI,gBAAgB,CAAC,yBAAyB,EAAE,GAAG,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,aAAa,CACpB,GAAW,EACX,GAAoB;IAEpB,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;QACvB,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC;AACvE,CAAC"}

package/dist/services/web/web-controller.d.ts

@@ -0,0 +1,70 @@
+/**
+ * WebController — orchestration for `fetch_and_index`.
+ *
+ * Owns a single set of injected index deps (vector store + keyword search) and
+ * drives the parallel-fetch / serial-index pipeline:
+ *
+ *   1. Build a `PoolJob` per request URL.
+ *   2. runPool fetches them in parallel (concurrency from the caller, cpu-capped
+ *      for batch). Each job runs fetchAndConvertOne, whose internal index drain
+ *      is already SERIAL — so even when N fetches race, the per-URL indexing
+ *      never overlaps with itself. Cross-URL overlap is acceptable for
+ *      pgvector (row locks) but would contend on PostgreSQL WAL; the per-URL
+ *      serial drain keeps the worst case to N concurrent single-writers, which
+ *      PostgreSQL tolerates and pgvector handles trivially.
+ *   3. Drain the allSettled-shaped results in input order and return per-URL
+ *      status + chunk counts.
+ *
+ * The controller is the composition root that wires the real vector/keyword
+ * stores into the fetcher's `WebIndexDeps`. Tests construct it with a captured
+ * in-memory deps map instead.
+ */
+import type { IVectorStore, IKeywordSearch } from "@massa-ai/shared";
+import { type FetchOneResult } from "./fetcher.js";
+export interface FetchRequest {
+    url: string;
+    source?: string;
+}
+export interface FetchAndIndexParams {
+    /** Single URL (legacy single shape). */
+    url?: string;
+    /** Label for the single-`url` case. */
+    source?: string;
+    /** Batch shape. Wins over `url` when both are provided. */
+    requests?: FetchRequest[];
+    /** Max URLs fetched in parallel (1-8, default 1). */
+    concurrency?: number;
+    /** Bypass the TTL cache. */
+    force?: boolean;
+    /** Per-call cache window override (ms). `0` bypasses like force. */
+    ttl?: number;
+    /** projectId scoping the indexed chunks. Defaults to "web". */
+    projectId?: string;
+}
+export interface WebControllerDeps {
+    vectorStore: IVectorStore;
+    keywordSearch: IKeywordSearch;
+}
+export declare class WebController {
+    private static instance;
+    private readonly deps;
+    private readonly cache;
+    constructor(deps: WebControllerDeps);
+    static getInstance(): WebController;
+    /**
+     * Boot the singleton with real store deps. The Tools API route calls this lazily
+     * on first request (after resolving the async vector store). Idempotent.
+     */
+    static instantiate(deps: WebControllerDeps): WebController;
+    /** For tests: reset the singleton + clear the cache. */
+    static resetInstance(): void;
+    /** Build the WebIndexDeps seam from the real stores + the in-memory cache. */
+    private indexDeps;
+    fetchAndIndex(params: FetchAndIndexParams): Promise<{
+        success: boolean;
+        results: FetchOneResult[];
+        concurrency: number;
+        capped: boolean;
+    }>;
+}
+//# sourceMappingURL=web-controller.d.ts.map

package/dist/services/web/web-controller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"web-controller.d.ts","sourceRoot":"","sources":["../../../src/services/web/web-controller.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAGrF,OAAO,EAEL,KAAK,cAAc,EAGpB,MAAM,cAAc,CAAC;AAEtB,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IAClC,wCAAwC;IACxC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,YAAY,EAAE,CAAC;IAC1B,qDAAqD;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,oEAAoE;IACpE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,YAAY,CAAC;IAC1B,aAAa,EAAE,cAAc,CAAC;CAC/B;AAcD,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAA8B;IACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAoB;IACzC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuB;gBAEjC,IAAI,EAAE,iBAAiB;IAInC,MAAM,CAAC,WAAW,IAAI,aAAa;IAUnC;;;OAGG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,iBAAiB,GAAG,aAAa;IAO1D,wDAAwD;IACxD,MAAM,CAAC,aAAa,IAAI,IAAI;IAI5B,8EAA8E;IAC9E,OAAO,CAAC,SAAS;IAuCX,aAAa,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC;QACxD,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,cAAc,EAAE,CAAC;QAC1B,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,OAAO,CAAC;KACjB,CAAC;CAkDH"}