@pentoshi/clai 0.10.5 → 0.11.0

package/dist/tools/web/providers/tavily.js

@@ -0,0 +1,285 @@
+/**
+ * Tavily search-provider adapter for `web.search`.
+ *
+ * Implements the {@link SearchProvider} contract from `./provider.ts` and
+ * registers itself in the {@link searchProviders} registry on import. The
+ * adapter performs exactly one outbound HTTPS request per invocation
+ * (Requirement 6.7), forwards the caller-provided {@link AbortSignal} to
+ * the underlying transport so the 15-second `web.search` timeout is
+ * honored (Requirement 1.8), and returns a {@link RawProviderResponse}
+ * describing the HTTP outcome plus the raw hit list.
+ *
+ * Status-to-error-kind classification (`401/403 → auth`, `429 → rate-limit`,
+ * `5xx → server`, non-JSON → `parse`, other non-2xx → `http`) is the
+ * responsibility of the `web.search` handler; this adapter only exposes
+ * the raw HTTP `status` and the parsed (or `parseError`-flagged) hit list
+ * so that mapping can be applied uniformly across providers (Requirements
+ * 6.1, 6.2, 6.5, 6.6).
+ *
+ * Endpoint and request shape match the design's "Per-provider notes →
+ * Tavily" section (`.kiro/specs/web-search-and-fetch/design.md`):
+ *
+ *   - POST `https://api.tavily.com/search`
+ *   - Body: `{ api_key, query, max_results, search_depth: "basic" }`
+ *     where `max_results` is clamped to `[1..20]` defensively.
+ *   - Response: `{ results: [{ title, url, content }] }` mapped into
+ *     `SearchResult { title, url, snippet }`.
+ */
+import { Buffer } from "node:buffer";
+import https from "node:https";
+import { searchProviders, } from "./provider.js";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Tavily search endpoint (host + path). */
+const TAVILY_HOST = "api.tavily.com";
+const TAVILY_PATH = "/search";
+/**
+ * `max_results` accepted by Tavily. The provider's documented range
+ * already matches the `maxResults` clamp the `web.search` handler
+ * enforces; we re-clamp here so the adapter is self-consistent if
+ * invoked directly (e.g. from a unit test) with an out-of-range value.
+ */
+const TAVILY_MIN_RESULTS = 1;
+const TAVILY_MAX_RESULTS = 20;
+/**
+ * Search depth passed to Tavily. The design specifies `"basic"` for
+ * cost predictability; users who want deeper retrieval can layer that
+ * on later via per-provider configuration.
+ */
+const TAVILY_SEARCH_DEPTH = "basic";
+/** User-Agent sent on outbound Tavily requests. */
+const DEFAULT_USER_AGENT = "clai-web-search/1.0";
+/**
+ * Hard cap on the number of body bytes we read from the provider before
+ * giving up and surfacing a `parse` error. Tavily responses for 20 hits
+ * are well under 100 KiB; this cap is defensive against a misbehaving
+ * upstream that streams an unbounded body.
+ */
+const MAX_RESPONSE_BYTES = 1_048_576; // 1 MiB
+let httpsRequestFn = https.request;
+/**
+ * Test-only seam: swap the HTTPS transport used by the adapter.
+ * Production callers never invoke this; tests use it to inject a
+ * stubbed `request` implementation that emits scripted responses.
+ */
+export function __setTavilyHttpsRequestForTesting(fn) {
+    httpsRequestFn = fn ?? https.request;
+}
+/**
+ * Clamp `maxResults` to the Tavily-supported range. The handler is
+ * expected to pass an already-clamped value, but defending here keeps
+ * the adapter self-consistent if invoked directly.
+ */
+function clampMaxResults(count) {
+    if (!Number.isFinite(count))
+        return TAVILY_MIN_RESULTS;
+    const rounded = Math.trunc(count);
+    if (rounded < TAVILY_MIN_RESULTS)
+        return TAVILY_MIN_RESULTS;
+    if (rounded > TAVILY_MAX_RESULTS)
+        return TAVILY_MAX_RESULTS;
+    return rounded;
+}
+/**
+ * Drain `res` into a UTF-8 string, capped at {@link MAX_RESPONSE_BYTES}.
+ * The body cap defends against a misbehaving upstream; the
+ * {@link AbortSignal} short-circuits the read on `web.search`'s
+ * 15-second timeout (Requirement 1.8).
+ */
+function readBody(res, signal) {
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        let total = 0;
+        let aborted = false;
+        const onAbort = () => {
+            if (aborted)
+                return;
+            aborted = true;
+            res.destroy(new Error("aborted"));
+            reject(signal.reason instanceof Error
+                ? signal.reason
+                : new Error("aborted"));
+        };
+        if (signal.aborted) {
+            onAbort();
+            return;
+        }
+        signal.addEventListener("abort", onAbort, { once: true });
+        res.on("data", (chunk) => {
+            if (aborted)
+                return;
+            total += chunk.length;
+            if (total > MAX_RESPONSE_BYTES) {
+                aborted = true;
+                signal.removeEventListener("abort", onAbort);
+                res.destroy();
+                reject(new Error("response body exceeded 1 MiB cap"));
+                return;
+            }
+            chunks.push(chunk);
+        });
+        res.on("end", () => {
+            if (aborted)
+                return;
+            signal.removeEventListener("abort", onAbort);
+            resolve(Buffer.concat(chunks).toString("utf8"));
+        });
+        res.on("error", (err) => {
+            if (aborted)
+                return;
+            aborted = true;
+            signal.removeEventListener("abort", onAbort);
+            reject(err);
+        });
+    });
+}
+/**
+ * Issue the Tavily HTTPS POST and return `{status, body}` once the
+ * response is fully read. Network failures (DNS, connect, TLS, socket
+ * reset) propagate as a thrown error so the adapter can map them to
+ * a `RawProviderResponse` with a `status: 0` placeholder for the
+ * search handler's `network` classification (Requirement 6.3).
+ */
+function dispatchRequest(query, maxResults, apiKey, signal) {
+    // Tavily authenticates via a body field rather than a header, per
+    // the design's "Per-provider notes" section. We embed `api_key`,
+    // the user query, the clamped `max_results`, and the documented
+    // `search_depth: "basic"` cost-control hint.
+    const payload = JSON.stringify({
+        api_key: apiKey,
+        query,
+        max_results: maxResults,
+        search_depth: TAVILY_SEARCH_DEPTH,
+    });
+    const bodyBytes = Buffer.from(payload, "utf8");
+    return new Promise((resolve, reject) => {
+        let req;
+        try {
+            req = httpsRequestFn({
+                method: "POST",
+                host: TAVILY_HOST,
+                path: TAVILY_PATH,
+                signal,
+                headers: {
+                    accept: "application/json",
+                    "content-type": "application/json",
+                    "content-length": String(bodyBytes.length),
+                    "user-agent": DEFAULT_USER_AGENT,
+                },
+            }, (res) => {
+                const status = res.statusCode ?? 0;
+                readBody(res, signal).then((body) => resolve({ status, body }), (err) => reject(err));
+            });
+        }
+        catch (err) {
+            reject(err);
+            return;
+        }
+        req.on("error", (err) => {
+            reject(err);
+        });
+        req.write(bodyBytes);
+        req.end();
+    });
+}
+/**
+ * Extract the Tavily `results[]` array from a parsed JSON body and map
+ * it to the {@link RawProviderResponse.hits} shape. Each hit
+ * contributes `title`/`url`/`content` (Tavily's snippet field)
+ * verbatim into `title`/`url`/`snippet`; the `web.search` handler is
+ * responsible for further validation (URL shape, whitespace, control
+ * chars — Requirement 7.3).
+ *
+ * Returns `null` when the JSON body did not have the expected
+ * `{ results: [...] }` shape, signalling the adapter to surface a
+ * `parseError` so the handler can emit `error.kind="parse"`
+ * (Requirement 6.5).
+ */
+function extractHits(parsed) {
+    if (!parsed || typeof parsed !== "object")
+        return null;
+    const results = parsed.results;
+    if (!Array.isArray(results))
+        return null;
+    const hits = [];
+    for (const entry of results) {
+        if (!entry || typeof entry !== "object")
+            continue;
+        const e = entry;
+        const hit = {};
+        if (typeof e.title === "string")
+            hit.title = e.title;
+        if (typeof e.url === "string")
+            hit.url = e.url;
+        if (typeof e.content === "string")
+            hit.snippet = e.content;
+        hits.push(hit);
+    }
+    return hits;
+}
+// ---------------------------------------------------------------------------
+// Provider definition
+// ---------------------------------------------------------------------------
+/**
+ * Tavily adapter. Registered in {@link searchProviders} as a
+ * side-effect of importing this module — `web.search` resolves the
+ * active provider via the registry.
+ */
+export const tavilyProvider = {
+    id: "tavily",
+    displayName: "Tavily",
+    needsApiKey: true,
+    envVar: "TAVILY_API_KEY",
+    async search(query, maxResults, auth, signal) {
+        // Defensive: the handler resolves the key before calling us. If
+        // somehow we're invoked without one, surface a 0-status response
+        // so the handler can map it to `missing-key` / `network` rather
+        // than dispatching an unauthenticated request to Tavily.
+        if (!auth.apiKey) {
+            return {
+                status: 0,
+                hits: [],
+                parseError: "missing api key",
+            };
+        }
+        const clamped = clampMaxResults(maxResults);
+        const { status, body } = await dispatchRequest(query, clamped, auth.apiKey, signal);
+        // Non-2xx: forward the status with an empty hit list. The search
+        // handler maps the status to the appropriate error kind
+        // (`auth` for 401/403, `rate-limit` for 429, `server` for 5xx,
+        // `http` for everything else).
+        if (status < 200 || status >= 300) {
+            return { status, hits: [] };
+        }
+        // 2xx: parse JSON. Anything that does not parse, or whose shape
+        // does not match `{ results: [...] }`, surfaces as `parseError`
+        // so the handler emits `error.kind="parse"` (Requirement 6.5).
+        let parsed;
+        try {
+            parsed = JSON.parse(body);
+        }
+        catch (err) {
+            return {
+                status,
+                hits: [],
+                parseError: err instanceof Error
+                    ? `non-JSON response: ${err.message}`
+                    : "non-JSON response",
+            };
+        }
+        const hits = extractHits(parsed);
+        if (hits === null) {
+            return {
+                status,
+                hits: [],
+                parseError: "missing results array in Tavily response",
+            };
+        }
+        return { status, hits };
+    },
+};
+// Register on import so `searchProviders.tavily` is populated by the
+// time the `web.search` handler dispatches.
+searchProviders.tavily = tavilyProvider;
+//# sourceMappingURL=tavily.js.map

package/dist/tools/web/providers/tavily.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tavily.js","sourceRoot":"","sources":["../../../../src/tools/web/providers/tavily.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,KAAK,MAAM,YAAY,CAAC;AAG/B,OAAO,EACL,eAAe,GAGhB,MAAM,eAAe,CAAC;AAEvB,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,4CAA4C;AAC5C,MAAM,WAAW,GAAG,gBAAgB,CAAC;AACrC,MAAM,WAAW,GAAG,SAAS,CAAC;AAE9B;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B;;;;GAIG;AACH,MAAM,mBAAmB,GAAG,OAAO,CAAC;AAEpC,mDAAmD;AACnD,MAAM,kBAAkB,GAAG,qBAAqB,CAAC;AAEjD;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,QAAQ;AAiB9C,IAAI,cAAc,GAAmB,KAAK,CAAC,OAAO,CAAC;AAEnD;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAC/C,EAA8B;IAE9B,cAAc,GAAG,EAAE,IAAI,KAAK,CAAC,OAAO,CAAC;AACvC,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,KAAa;IACpC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACvD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,OAAO,GAAG,kBAAkB;QAAE,OAAO,kBAAkB,CAAC;IAC5D,IAAI,OAAO,GAAG,kBAAkB;QAAE,OAAO,kBAAkB,CAAC;IAC5D,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,SAAS,QAAQ,CAAC,GAAoB,EAAE,MAAmB;IACzD,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,MAAM,OAAO,GAAG,GAAS,EAAE;YACzB,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;YAClC,MAAM,CACJ,MAAM,CAAC,MAAM,YAAY,KAAK;gBAC5B,CAAC,CAAC,MAAM,CAAC,MAAM;gBACf,CAAC,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CACzB,CAAC;QACJ,CAAC,CAAC;QAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QACD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1D,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YAC/B,IAAI,OAAO;gBAAE,OAAO;YACpB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,IAAI,KAAK,GAAG,kBAAkB,EAAE,CAAC;gBAC/B,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC7C,GAAG,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC,CAAC;gBACtD,OAAO;YACT,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACjB,IAAI,OAAO;gBAAE,OAAO;YACpB,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACtB,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CACtB,KAAa,EACb,UAAkB,EAClB,MAAc,EACd,MAAmB;IAEnB,kEAAkE;IAClE,iEAAiE;IACjE,gEAAgE;IAChE,6CAA6C;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM;QACf,KAAK;QACL,WAAW,EAAE,UAAU;QACvB,YAAY,EAAE,mBAAmB;KAClC,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAE/C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,IAAI,GAAkB,CAAC;QACvB,IAAI,CAAC;YACH,GAAG,GAAG,cAAc,CAClB;gBACE,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,WAAW;gBACjB,MAAM;gBACN,OAAO,EAAE;oBACP,MAAM,EAAE,kBAAkB;oBAC1B,cAAc,EAAE,kBAAkB;oBAClC,gBAAgB,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;oBAC1C,YAAY,EAAE,kBAAkB;iBACjC;aACF,EACD,CAAC,GAAG,EAAE,EAAE;gBACN,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;gBACnC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CACxB,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EACnC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CACrB,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,CAAC,CAAC;YACZ,OAAO;QACT,CAAC;QAED,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACtB,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACrB,GAAG,CAAC,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,WAAW,CAClB,MAAe;IAEf,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACvD,MAAM,OAAO,GAAI,MAAgC,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,MAAM,IAAI,GAAgC,EAAE,CAAC;IAC7C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,SAAS;QAClD,MAAM,CAAC,GAAG,KAIT,CAAC;QACF,MAAM,GAAG,GAAwC,EAAE,CAAC;QACpD,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;YAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QACrD,IAAI,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ;YAAE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC;QAC/C,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ;YAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;QAC3D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAmB;IAC5C,EAAE,EAAE,QAAQ;IACZ,WAAW,EAAE,QAAQ;IACrB,WAAW,EAAE,IAAI;IACjB,MAAM,EAAE,gBAAgB;IAExB,KAAK,CAAC,MAAM,CACV,KAAa,EACb,UAAkB,EAClB,IAAyB,EACzB,MAAmB;QAEnB,gEAAgE;QAChE,iEAAiE;QACjE,gEAAgE;QAChE,yDAAyD;QACzD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO;gBACL,MAAM,EAAE,CAAC;gBACT,IAAI,EAAE,EAAE;gBACR,UAAU,EAAE,iBAAiB;aAC9B,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAe,CAC5C,KAAK,EACL,OAAO,EACP,IAAI,CAAC,MAAM,EACX,MAAM,CACP,CAAC;QAEF,iEAAiE;QACjE,wDAAwD;QACxD,+DAA+D;QAC/D,+BAA+B;QAC/B,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YAClC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;QAC9B,CAAC;QAED,gEAAgE;QAChE,gEAAgE;QAChE,+DAA+D;QAC/D,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,MAAM;gBACN,IAAI,EAAE,EAAE;gBACR,UAAU,EACR,GAAG,YAAY,KAAK;oBAClB,CAAC,CAAC,sBAAsB,GAAG,CAAC,OAAO,EAAE;oBACrC,CAAC,CAAC,mBAAmB;aAC1B,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,OAAO;gBACL,MAAM;gBACN,IAAI,EAAE,EAAE;gBACR,UAAU,EAAE,0CAA0C;aACvD,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;CACF,CAAC;AAEF,qEAAqE;AACrE,4CAA4C;AAC5C,eAAe,CAAC,MAAM,GAAG,cAAc,CAAC"}

package/dist/tools/web/readable.d.ts

@@ -0,0 +1,67 @@
+/**
+ * HTML-to-readable-text conversion and a permissive `Set-Cookie` parser.
+ *
+ * Two helpers live here so both `web.fetch` (via `capture.ts`) and the
+ * DuckDuckGo provider's lite-HTML adapter share one implementation:
+ *
+ * - {@link toReadableText} strips obvious chrome and non-rendering content
+ *   from an HTML document and returns the visible prose with whitespace
+ *   collapsed. It satisfies Requirements 2.4, 2.5, and 2.28 and follows
+ *   the design's "HTML-to-readable-text strategy" (cheerio-based, no
+ *   browser/jsdom dependency).
+ * - {@link parseSetCookie} parses one `Set-Cookie` header value into a
+ *   {@link CookieInfo}, supporting only the public attributes the
+ *   `web.fetch` tool surfaces. It is regex-driven and intentionally
+ *   permissive: missing or malformed attributes are simply absent in the
+ *   returned object instead of producing a hard error.
+ */
+import type { CookieInfo } from "./types.js";
+/**
+ * Convert an HTML document into a single readable text string.
+ *
+ * The conversion:
+ * 1. Parses `html` with cheerio (no DOM/browser dependency).
+ * 2. Removes `<script>`, `<style>`, `<noscript>`, `<nav>`, `<header>`,
+ *    `<footer>`, and `<aside>` subtrees outright.
+ * 3. Removes every HTML comment node anywhere in the tree.
+ * 4. Extracts the remaining text via `$.root().text()`.
+ * 5. Collapses every run of ASCII/Unicode whitespace (including newlines,
+ *    tabs, NBSPs) into a single space and trims the result so the agent
+ *    receives compact prose.
+ *
+ * Empty input, whitespace-only input, and input that contains only
+ * stripped elements all yield the empty string.
+ */
+export declare function toReadableText(html: string): string;
+/**
+ * Parse a single `Set-Cookie` header value into a {@link CookieInfo}.
+ *
+ * The parser is intentionally permissive: it never throws for malformed
+ * input. The first `;`-separated attribute is treated as the
+ * `name=value` pair (with everything after the first `=` taken verbatim
+ * as the value, matching common server practice). Subsequent attributes
+ * are matched case-insensitively against the public RFC 6265 set the
+ * `web.fetch` tool surfaces:
+ *
+ * - `Domain`        → {@link CookieInfo.domain}
+ * - `Path`          → {@link CookieInfo.path}
+ * - `Expires`       → {@link CookieInfo.expires} as an ISO 8601 string
+ *                     (omitted if the date string fails to parse)
+ * - `Max-Age`       → {@link CookieInfo.maxAge} as a finite integer
+ *                     (omitted if not a finite integer)
+ * - `HttpOnly`      → {@link CookieInfo.httpOnly} = `true`
+ * - `Secure`        → {@link CookieInfo.secure}   = `true`
+ * - `SameSite=…`    → {@link CookieInfo.sameSite} normalized to
+ *                     `"Strict"`/`"Lax"`/`"None"` (omitted if value is
+ *                     unknown)
+ *
+ * Unknown attributes (e.g. `Priority`, `Partitioned`) are ignored. When
+ * an attribute is missing, malformed, or unrecognised, the corresponding
+ * field is simply absent from the returned object.
+ *
+ * The header value is expected to be a single cookie. Callers that
+ * receive multiple cookies in a single header (which servers must not
+ * do, but a few do) should split on the appropriate boundary before
+ * calling this function.
+ */
+export declare function parseSetCookie(value: string): CookieInfo;

package/dist/tools/web/readable.js

@@ -0,0 +1,248 @@
+/**
+ * HTML-to-readable-text conversion and a permissive `Set-Cookie` parser.
+ *
+ * Two helpers live here so both `web.fetch` (via `capture.ts`) and the
+ * DuckDuckGo provider's lite-HTML adapter share one implementation:
+ *
+ * - {@link toReadableText} strips obvious chrome and non-rendering content
+ *   from an HTML document and returns the visible prose with whitespace
+ *   collapsed. It satisfies Requirements 2.4, 2.5, and 2.28 and follows
+ *   the design's "HTML-to-readable-text strategy" (cheerio-based, no
+ *   browser/jsdom dependency).
+ * - {@link parseSetCookie} parses one `Set-Cookie` header value into a
+ *   {@link CookieInfo}, supporting only the public attributes the
+ *   `web.fetch` tool surfaces. It is regex-driven and intentionally
+ *   permissive: missing or malformed attributes are simply absent in the
+ *   returned object instead of producing a hard error.
+ */
+import * as cheerio from "cheerio";
+// ---------------------------------------------------------------------------
+// HTML → readable text
+// ---------------------------------------------------------------------------
+/**
+ * Selectors for elements whose text content should never appear in the
+ * readable view. `script`/`style`/`noscript` carry executable or styling
+ * payloads (Requirement 2.4); `nav`/`header`/`footer`/`aside` are the
+ * obvious chrome regions called out in the design's
+ * "HTML-to-readable-text strategy".
+ */
+const STRIPPED_SELECTORS = [
+    "script",
+    "style",
+    "noscript",
+    "nav",
+    "header",
+    "footer",
+    "aside",
+].join(", ");
+/**
+ * Convert an HTML document into a single readable text string.
+ *
+ * The conversion:
+ * 1. Parses `html` with cheerio (no DOM/browser dependency).
+ * 2. Removes `<script>`, `<style>`, `<noscript>`, `<nav>`, `<header>`,
+ *    `<footer>`, and `<aside>` subtrees outright.
+ * 3. Removes every HTML comment node anywhere in the tree.
+ * 4. Extracts the remaining text via `$.root().text()`.
+ * 5. Collapses every run of ASCII/Unicode whitespace (including newlines,
+ *    tabs, NBSPs) into a single space and trims the result so the agent
+ *    receives compact prose.
+ *
+ * Empty input, whitespace-only input, and input that contains only
+ * stripped elements all yield the empty string.
+ */
+export function toReadableText(html) {
+    if (typeof html !== "string" || html.length === 0)
+        return "";
+    const $ = cheerio.load(html);
+    // 1. Remove non-content elements outright.
+    $(STRIPPED_SELECTORS).remove();
+    // 2. Remove every comment node still attached to the tree. cheerio
+    //    represents comments with `type === "comment"`; we walk the full
+    //    contents of `*` so nested comments inside any remaining element
+    //    are caught.
+    $("*")
+        .contents()
+        .filter(function () {
+        return this.type === "comment";
+    })
+        .remove();
+    const raw = $.root().text();
+    return collapseWhitespace(raw);
+}
+/**
+ * Collapse every contiguous run of whitespace characters into a single
+ * ASCII space and trim leading/trailing whitespace.
+ *
+ * Treated as whitespace: the standard `\s` set (space, tab, CR, LF, FF,
+ * VT) plus the most common non-breaking and zero-width characters that
+ * show up in real-world web HTML — `\u00a0` (NBSP), `\u200b` (zero-width
+ * space), `\u200c`/`\u200d` (zero-width joiner/non-joiner), `\ufeff`
+ * (BOM/zero-width no-break space), and the assorted `\u2000-\u200a` set
+ * of Unicode spaces.
+ */
+function collapseWhitespace(text) {
+    return text
+        .replace(/[\s\u00a0\u2000-\u200a\u200b\u200c\u200d\u2028\u2029\ufeff]+/g, " ")
+        .trim();
+}
+// ---------------------------------------------------------------------------
+// Set-Cookie parser
+// ---------------------------------------------------------------------------
+/**
+ * Allowed values for `SameSite`, lower-cased for case-insensitive lookup.
+ * The map preserves the canonical capitalisation used in the
+ * {@link CookieInfo.sameSite} field.
+ */
+const SAME_SITE_VALUES = new Map([
+    ["strict", "Strict"],
+    ["lax", "Lax"],
+    ["none", "None"],
+]);
+/**
+ * Parse a single `Set-Cookie` header value into a {@link CookieInfo}.
+ *
+ * The parser is intentionally permissive: it never throws for malformed
+ * input. The first `;`-separated attribute is treated as the
+ * `name=value` pair (with everything after the first `=` taken verbatim
+ * as the value, matching common server practice). Subsequent attributes
+ * are matched case-insensitively against the public RFC 6265 set the
+ * `web.fetch` tool surfaces:
+ *
+ * - `Domain`        → {@link CookieInfo.domain}
+ * - `Path`          → {@link CookieInfo.path}
+ * - `Expires`       → {@link CookieInfo.expires} as an ISO 8601 string
+ *                     (omitted if the date string fails to parse)
+ * - `Max-Age`       → {@link CookieInfo.maxAge} as a finite integer
+ *                     (omitted if not a finite integer)
+ * - `HttpOnly`      → {@link CookieInfo.httpOnly} = `true`
+ * - `Secure`        → {@link CookieInfo.secure}   = `true`
+ * - `SameSite=…`    → {@link CookieInfo.sameSite} normalized to
+ *                     `"Strict"`/`"Lax"`/`"None"` (omitted if value is
+ *                     unknown)
+ *
+ * Unknown attributes (e.g. `Priority`, `Partitioned`) are ignored. When
+ * an attribute is missing, malformed, or unrecognised, the corresponding
+ * field is simply absent from the returned object.
+ *
+ * The header value is expected to be a single cookie. Callers that
+ * receive multiple cookies in a single header (which servers must not
+ * do, but a few do) should split on the appropriate boundary before
+ * calling this function.
+ */
+export function parseSetCookie(value) {
+    if (typeof value !== "string") {
+        return { name: "", value: "" };
+    }
+    // Split on `;` to peel attributes off the name=value pair. We do not
+    // split on `,` because RFC 6265 §4.1 forbids commas in cookie values
+    // unrelated to date attributes, and Node/undici always hand us one
+    // header value per Set-Cookie line.
+    const parts = value.split(";");
+    const head = (parts[0] ?? "").trim();
+    const eqIdx = head.indexOf("=");
+    let name;
+    let cookieValue;
+    if (eqIdx === -1) {
+        // No `=` at all: treat the whole token as the name, value empty.
+        name = head;
+        cookieValue = "";
+    }
+    else {
+        name = head.slice(0, eqIdx).trim();
+        // Per RFC 6265 the value runs to end-of-attribute; trim outer
+        // whitespace but keep internal characters verbatim.
+        cookieValue = head.slice(eqIdx + 1).trim();
+    }
+    // Build the result one field at a time so `exactOptionalPropertyTypes`
+    // sees an absent key for any attribute we did not observe.
+    const result = { name, value: cookieValue };
+    for (let i = 1; i < parts.length; i++) {
+        const attr = parts[i];
+        if (typeof attr !== "string")
+            continue;
+        const trimmed = attr.trim();
+        if (trimmed.length === 0)
+            continue;
+        const attrEq = trimmed.indexOf("=");
+        const attrName = attrEq === -1 ? trimmed : trimmed.slice(0, attrEq).trim();
+        const attrValue = attrEq === -1 ? "" : trimmed.slice(attrEq + 1).trim();
+        const lowerName = attrName.toLowerCase();
+        switch (lowerName) {
+            case "domain": {
+                if (attrValue.length > 0)
+                    result.domain = attrValue;
+                break;
+            }
+            case "path": {
+                if (attrValue.length > 0)
+                    result.path = attrValue;
+                break;
+            }
+            case "expires": {
+                const iso = parseHttpDate(attrValue);
+                if (iso !== undefined)
+                    result.expires = iso;
+                break;
+            }
+            case "max-age": {
+                const n = parseMaxAge(attrValue);
+                if (n !== undefined)
+                    result.maxAge = n;
+                break;
+            }
+            case "httponly": {
+                result.httpOnly = true;
+                break;
+            }
+            case "secure": {
+                result.secure = true;
+                break;
+            }
+            case "samesite": {
+                const canonical = SAME_SITE_VALUES.get(attrValue.toLowerCase());
+                if (canonical !== undefined)
+                    result.sameSite = canonical;
+                break;
+            }
+            default:
+                // Ignore unknown attributes (Priority, Partitioned, etc.).
+                break;
+        }
+    }
+    return result;
+}
+/**
+ * Parse an HTTP-date string (RFC 7231 §7.1.1.1, including the legacy
+ * RFC 850 and asctime forms) into an ISO 8601 timestamp. Returns
+ * `undefined` when the value does not parse to a finite, valid date.
+ */
+function parseHttpDate(value) {
+    if (value.length === 0)
+        return undefined;
+    const ms = Date.parse(value);
+    if (!Number.isFinite(ms))
+        return undefined;
+    return new Date(ms).toISOString();
+}
+/**
+ * Parse a `Max-Age` attribute value into a finite integer. Returns
+ * `undefined` for empty input, non-numeric input, fractional values, or
+ * values outside the safe-integer range.
+ */
+function parseMaxAge(value) {
+    if (value.length === 0)
+        return undefined;
+    // RFC 6265 §5.2.2 specifies the value as DIGIT *DIGIT (optionally
+    // preceded by `-` for delete-now semantics). Reject anything that
+    // does not match that shape.
+    if (!/^-?\d+$/.test(value))
+        return undefined;
+    const n = Number(value);
+    if (!Number.isFinite(n) || !Number.isInteger(n))
+        return undefined;
+    if (!Number.isSafeInteger(n))
+        return undefined;
+    return n;
+}
+//# sourceMappingURL=readable.js.map

package/dist/tools/web/readable.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readable.js","sourceRoot":"","sources":["../../../src/tools/web/readable.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG;IACzB,QAAQ;IACR,OAAO;IACP,UAAU;IACV,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,OAAO;CACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE7D,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,2CAA2C;IAC3C,CAAC,CAAC,kBAAkB,CAAC,CAAC,MAAM,EAAE,CAAC;IAE/B,mEAAmE;IACnE,qEAAqE;IACrE,qEAAqE;IACrE,iBAAiB;IACjB,CAAC,CAAC,GAAG,CAAC;SACH,QAAQ,EAAE;SACV,MAAM,CAAC;QACN,OAAO,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC;IACjC,CAAC,CAAC;SACD,MAAM,EAAE,CAAC;IAEZ,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC5B,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,IAAI;SACR,OAAO,CAAC,+DAA+D,EAAE,GAAG,CAAC;SAC7E,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,gBAAgB,GAAwC,IAAI,GAAG,CAAC;IACpE,CAAC,QAAQ,EAAE,QAAQ,CAAC;IACpB,CAAC,KAAK,EAAE,KAAK,CAAC;IACd,CAAC,MAAM,EAAE,MAAM,CAAC;CACjB,CAAC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,qEAAqE;IACrE,qEAAqE;IACrE,mEAAmE;IACnE,oCAAoC;IACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAErC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,IAAY,CAAC;IACjB,IAAI,WAAmB,CAAC;IACxB,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,iEAAiE;QACjE,IAAI,GAAG,IAAI,CAAC;QACZ,WAAW,GAAG,EAAE,CAAC;IACnB,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACnC,8DAA8D;QAC9D,oDAAoD;QACpD,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,uEAAuE;IACvE,2DAA2D;IAC3D,MAAM,MAAM,GAAe,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;IAExD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,SAAS;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEnC,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,QAAQ,GACZ,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACxE,MAAM,SAAS,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QAEzC,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;oBAAE,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;gBACpD,MAAM;YACR,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;oBAAE,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC;gBAClD,MAAM;YACR,CAAC;YACD,KAAK,SAAS,CAAC,CAAC,CAAC;gBACf,MAAM,GAAG,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;gBACrC,IAAI,GAAG,KAAK,SAAS;oBAAE,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC;gBAC5C,MAAM;YACR,CAAC;YACD,KAAK,SAAS,CAAC,CAAC,CAAC;gBACf,MAAM,CAAC,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBACjC,IAAI,CAAC,KAAK,SAAS;oBAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;gBACvC,MAAM;YACR,CAAC;YACD,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACvB,MAAM;YACR,CAAC;YACD,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC;gBACrB,MAAM;YACR,CAAC;YACD,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;gBAChE,IAAI,SAAS,KAAK,SAAS;oBAAE,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC;gBACzD,MAAM;YACR,CAAC;YACD;gBACE,2DAA2D;gBAC3D,MAAM;QACV,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,KAAa;IAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACpC,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzC,kEAAkE;IAClE,kEAAkE;IAClE,6BAA6B;IAC7B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACxB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAClE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAC/C,OAAO,CAAC,CAAC;AACX,CAAC"}