@stackfactor/agent-utils 1.10.0 → 1.12.0

package/dist/esm/tavily.js

@@ -53,6 +53,10 @@ export const assertTavilyUsable = (config, options) => {
     if (!config?.tavilyAPIKey) {
         throw errorHandlingHelper.create(constants.HTTP_CODES.INTERNAL_SERVER_ERROR, "Tavily tools are enabled but config.tavilyAPIKey is missing.");
     }
+    if (options.allowedUrls !== undefined &&
+        !(Array.isArray(options.allowedUrls) && options.allowedUrls.every((url) => typeof url === "string"))) {
+        throw errorHandlingHelper.create(constants.HTTP_CODES.BAD_REQUEST, "config.tavily.allowedUrls must be a list of URLs.");
+    }
     const unknown = (options.tools || []).filter((name) => !["search", "extract", "crawl", "map"].includes(name));
     if (unknown.length > 0) {
         throw errorHandlingHelper.create(constants.HTTP_CODES.BAD_REQUEST, `Unknown Tavily tool(s): ${unknown.join(", ")}. Valid values are search, extract, crawl, map.`);
@@ -160,6 +164,52 @@ const recordTavilyUsage = (tracker, _config, credits, sources) => {
         }
     }
 };
+/**
+ * Adds the URLs `web_extract` is about to ask Tavily for to
+ * `tracker.attemptedUrls`, each once across the run. Recorded before the call,
+ * so a URL the call then fails on is still there.
+ * @param tracker - The caller's usage accumulator, when it passed one
+ * @param urls - The URLs sent
+ */
+const recordAttemptedUrls = (tracker, urls) => {
+    if (!tracker || urls.length === 0)
+        return;
+    const attempted = tracker.attemptedUrls || (tracker.attemptedUrls = []);
+    for (const url of urls) {
+        if (!attempted.includes(url))
+            attempted.push(url);
+    }
+};
+/**
+ * How a URL is matched, against Tavily's answer or `allowedUrls`: trimmed,
+ * lowercased, no trailing slash.
+ */
+const urlMatchKey = (url) => url.trim().replace(/\/+$/, "").toLowerCase();
+/** What `web_extract` answers for each URL `allowedUrls` refused. */
+const REFUSED_HEADING = "Refused, not read (only the referenced pages may be read):";
+/**
+ * Splits the URLs the model asked `web_extract` for into those it may read —
+ * as `allowedUrls` lists them — and those it may not, each once. With no
+ * `allowedUrls`, every URL may be read, as asked.
+ * @param urls - The URLs the model asked for
+ * @param allowed - `allowedUrls` by match key, or `null` when unset
+ */
+const screenUrls = (urls, allowed) => {
+    if (!allowed)
+        return { read: urls, refused: [] };
+    const read = [];
+    const refused = [];
+    for (const url of urls) {
+        const listed = allowed.get(urlMatchKey(url));
+        if (listed === undefined) {
+            if (!refused.includes(url))
+                refused.push(url);
+        }
+        else if (!read.includes(listed))
+            read.push(listed);
+    }
+    return { read, refused };
+};
 /**
  * Trims one document to whatever the per-URL and per-call budgets allow,
  * appending a note when anything was cut.
@@ -230,6 +280,9 @@ export const buildTavilyTools = (config, options, usageTracker = null) => {
     const maxCharsPerUrl = options.maxCharsPerUrl ?? DEFAULT_MAX_CHARS_PER_URL;
     const maxCharsTotal = options.maxCharsTotal ?? DEFAULT_MAX_CHARS_TOTAL;
     const crawlLimit = options.crawlLimit ?? DEFAULT_CRAWL_LIMIT;
+    const allowed = Array.isArray(options.allowedUrls)
+        ? new Map(options.allowedUrls.map((url) => [urlMatchKey(url), url.trim()]))
+        : null;
     const newBudget = () => ({
         remaining: maxCharsTotal,
         perUrl: maxCharsPerUrl,
@@ -277,8 +330,22 @@ export const buildTavilyTools = (config, options, usageTracker = null) => {
     }
     if (enabled.includes("extract")) {
         tools.push(tool(async ({ urls, query }) => {
+            // Outside `allowedUrls` nothing is sent: the model is told per URL,
+            // and the run goes on with what it may read.
+            const { read, refused } = screenUrls(urls, allowed);
+            const refusal = refused.length
+                ? `${REFUSED_HEADING}\n${refused
+                    .map((url) => `- ${url}: not one of the referenced pages`)
+                    .join("\n")}`
+                : "";
+            if (refused.length > 0) {
+                logger.log(null, logger.levels.warn, `Tavily extract refused ${refused.length} URL(s) not among the allowed pages: ${refused.join(", ")}`);
+            }
+            if (read.length === 0)
+                return refusal;
+            recordAttemptedUrls(usageTracker, read);
             const data = await callTavily("extract", {
-                urls,
+                urls: read,
                 extract_depth: options.extractDepth,
                 format,
                 // `query` switches the API from whole-document extraction to
@@ -309,10 +376,14 @@ export const buildTavilyTools = (config, options, usageTracker = null) => {
                     .join("\n");
                 rendered += `\n\n---\n\nFailed to extract:\n${lines}`;
             }
+            if (refusal)
+                rendered += `\n\n---\n\n${refusal}`;
             return rendered;
         }, {
             name: "web_extract",
-            description: "Read the content of specific URLs. Pass a query to get back only the passages relevant to it — much cheaper and usually sufficient. Omit the query to retrieve the whole document, which is appropriate when you need its full structure. Long documents are truncated, and the result says so when that happens.",
+            description: `Read the content of specific URLs. Pass a query to get back only the passages relevant to it — much cheaper and usually sufficient. Omit the query to retrieve the whole document, which is appropriate when you need its full structure. Long documents are truncated, and the result says so when that happens.${allowed
+                ? " Only the referenced pages can be read; any other URL is refused."
+                : ""}`,
             schema: z.object({
                 urls: z
                     .array(z.string())
@@ -390,8 +461,6 @@ export const buildTavilyTools = (config, options, usageTracker = null) => {
 const EXTRACT_MAX_URLS = 20;
 /** Seconds. A reachability check needs the page fetched, not rendered at length. */
 const REACHABILITY_TIMEOUT_SECONDS = 20;
-/** How a URL is matched against Tavily's answer: trimmed, lowercased, no trailing slash. */
-const urlMatchKey = (url) => url.trim().replace(/\/+$/, "").toLowerCase();
 /** True for an `http:` or `https:` URL; false for anything else, unparsable included. */
 const isWebUrl = (url) => {
     try {

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.10.0",
+  "version": "1.12.0",
   "description": "",
   "main": "dist/cjs/index.js",
   "module": "dist/esm/index.js",