@stackfactor/agent-utils 1.8.0 → 1.9.0

package/README.md

@@ -419,6 +419,21 @@ A model that cannot tell it received half a regulation will reason over the half
 - Per-URL failures are surfaced in the tool result instead of being dropped, so a fetch failure is not mistaken for an absence of content.
 - Tool calls honour the caller's abort signal, so a cancelled run stops paying for in-flight crawls.
 
+### Checking that cited links resolve
+
+`checkUrlsReachable(urls, config, usageTracker?)` returns the URLs, exactly as passed, that Tavily could fetch. Use it in place of requesting a link yourself. Cited links come from web pages anyone can publish, so a link, its redirects or its DNS can point inside StackFactor's network; a request this process sends would go there through StackFactor's own egress addresses. Tavily fetches from its own infrastructure.
+
+```js
+const { checkUrlsReachable } = require("@stackfactor/agent-utils");
+
+const live = await checkUrlsReachable(references.map((r) => r.url), config, usageTracker);
+const kept = references.filter((r) => !r.url || live.has(r.url));
+```
+
+- Only `http:` and `https:` URLs are sent, 20 per call (Tavily's limit), at basic depth: 1 credit per 5 URLs fetched, failed ones free. Credits are billed like any Tavily call; the URLs are not added to `webSearchSources`.
+- A URL Tavily lists under `failed_results` is unreachable. A batch whose call fails, or a config without `tavilyAPIKey`, confirms none of its URLs. It never throws.
+- No `config.tavily` or `agentic` is needed: it calls Tavily directly, not through the model.
+
 ---
 
 ## Supported Models

package/dist/cjs/index.d.ts

@@ -7,11 +7,13 @@ import { callAgent, checkAgent } from "./client.js";
 import * as runtimeContext from "./runtimeContext.js";
 import { recordUsage, USAGE_UNITS } from "./usage.js";
 import stackfactorKnowledge from "./stackfactorKnowledge.js";
+import { checkUrlsReachable } from "./tavily.js";
 export { constants };
 export { errorHandling, AppError };
 export type { ParsedError } from "./errorHandling.js";
 export { langChain };
 export type { UsageTracker } from "./langChain.js";
+export { checkUrlsReachable };
 export type { TavilyConfig, TavilyToolName } from "./tavily.js";
 export { logger };
 export { serve, runOnce, isRunOnceExecution, EXECUTION_ENV };

package/dist/cjs/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAEhC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAEhC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC"}

package/dist/cjs/index.js

@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.stackfactorKnowledge = exports.USAGE_UNITS = exports.recordUsage = exports.runtimeContext = exports.checkAgent = exports.callAgent = exports.EXECUTION_ENV = exports.isRunOnceExecution = exports.runOnce = exports.serve = exports.logger = exports.langChain = exports.AppError = exports.errorHandling = exports.constants = void 0;
+exports.stackfactorKnowledge = exports.USAGE_UNITS = exports.recordUsage = exports.runtimeContext = exports.checkAgent = exports.callAgent = exports.EXECUTION_ENV = exports.isRunOnceExecution = exports.runOnce = exports.serve = exports.logger = exports.checkUrlsReachable = exports.langChain = exports.AppError = exports.errorHandling = exports.constants = void 0;
 const const_js_1 = __importDefault(require("./const.js"));
 exports.constants = const_js_1.default;
 const errorHandling_js_1 = __importStar(require("./errorHandling.js"));
@@ -61,3 +61,5 @@ Object.defineProperty(exports, "recordUsage", { enumerable: true, get: function
 Object.defineProperty(exports, "USAGE_UNITS", { enumerable: true, get: function () { return usage_js_1.USAGE_UNITS; } });
 const stackfactorKnowledge_js_1 = __importDefault(require("./stackfactorKnowledge.js"));
 exports.stackfactorKnowledge = stackfactorKnowledge_js_1.default;
+const tavily_js_1 = require("./tavily.js");
+Object.defineProperty(exports, "checkUrlsReachable", { enumerable: true, get: function () { return tavily_js_1.checkUrlsReachable; } });

package/dist/cjs/tavily.d.ts

@@ -82,9 +82,40 @@ export declare const assertTavilyUsable: (config: any, options: TavilyConfig) =>
  * @returns LangChain tool instances to append to the agent's tool list
  */
 export declare const buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
+/**
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
+ * this process connecting to them.
+ *
+ * An agent that confirms a cited link by requesting it itself sends that
+ * request from inside StackFactor's network, through its fixed egress
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
+ * links come from web pages anyone can publish, so that is a request to an
+ * address an outsider chose (server-side request forgery). Tavily fetches from
+ * its own infrastructure, so nothing here ever connects to such a host.
+ *
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
+ * like every other Tavily call, but the URLs are not added to the tracker's
+ * sources: a link worth checking was found there in the first place.
+ *
+ * Tavily accounts for every URL it is given, listing the ones it could not
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
+ * other URL of a batch Tavily answered is reachable (the results may name a
+ * URL by its address after a redirect, which is why they are not matched). A
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
+ * confirms none of its URLs: a link that could not be checked is not trusted.
+ *
+ * @param urls - The URLs to check; a URL given twice is checked once
+ * @param config - The integration config; supplies `tavilyAPIKey`
+ * @param usageTracker - Optional accumulator the credits are counted into
+ * @returns The given URLs, exactly as passed, that resolve
+ */
+export declare const checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
 declare const _default: {
     assertTavilyUsable: (config: any, options: TavilyConfig) => void;
     buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
+    checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
     getTavilyOptions: (config: any) => TavilyConfig | null;
     recordTavilyUsage: (tracker: UsageTracker | null | undefined, _config: any, credits: number, sources: {
         url: string;

package/dist/cjs/tavily.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;;iCA5dyC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BAhPiC,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAmWP,wBAKE"}
+{"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;AAqBF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,eAAO,MAAM,kBAAkB,GAC7B,MAAM,SAAS,MAAM,EAAE,EACvB,QAAQ,GAAG,EACX,eAAc,YAAY,GAAG,IAAW,KACvC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CA6DrB,CAAC;;iCA/kByC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BA8SA,SAAS,MAAM,EAAE,UACf,GAAG,iBACG,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;+BAjiBkB,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAsdP,wBAME"}

package/dist/cjs/tavily.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildTavilyTools = exports.assertTavilyUsable = exports.getTavilyOptions = void 0;
+exports.checkUrlsReachable = exports.buildTavilyTools = exports.assertTavilyUsable = exports.getTavilyOptions = void 0;
 const langchain_1 = require("langchain");
 const zod_1 = require("zod");
 const const_js_1 = __importDefault(require("./const.js"));
@@ -395,9 +395,96 @@ const buildTavilyTools = (config, options, usageTracker = null) => {
     return tools;
 };
 exports.buildTavilyTools = buildTavilyTools;
+/** Tavily's own limit on URLs per `/extract` request. */
+const EXTRACT_MAX_URLS = 20;
+/** Seconds. A reachability check needs the page fetched, not rendered at length. */
+const REACHABILITY_TIMEOUT_SECONDS = 20;
+/** How a URL is matched against Tavily's answer: trimmed, lowercased, no trailing slash. */
+const urlMatchKey = (url) => url.trim().replace(/\/+$/, "").toLowerCase();
+/** True for an `http:` or `https:` URL; false for anything else, unparsable included. */
+const isWebUrl = (url) => {
+    try {
+        const { protocol } = new URL(url);
+        return protocol === "http:" || protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+};
+/**
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
+ * this process connecting to them.
+ *
+ * An agent that confirms a cited link by requesting it itself sends that
+ * request from inside StackFactor's network, through its fixed egress
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
+ * links come from web pages anyone can publish, so that is a request to an
+ * address an outsider chose (server-side request forgery). Tavily fetches from
+ * its own infrastructure, so nothing here ever connects to such a host.
+ *
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
+ * like every other Tavily call, but the URLs are not added to the tracker's
+ * sources: a link worth checking was found there in the first place.
+ *
+ * Tavily accounts for every URL it is given, listing the ones it could not
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
+ * other URL of a batch Tavily answered is reachable (the results may name a
+ * URL by its address after a redirect, which is why they are not matched). A
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
+ * confirms none of its URLs: a link that could not be checked is not trusted.
+ *
+ * @param urls - The URLs to check; a URL given twice is checked once
+ * @param config - The integration config; supplies `tavilyAPIKey`
+ * @param usageTracker - Optional accumulator the credits are counted into
+ * @returns The given URLs, exactly as passed, that resolve
+ */
+const checkUrlsReachable = async (urls, config, usageTracker = null) => {
+    const reachable = new Set();
+    const candidates = [
+        ...new Set(urls.filter((url) => typeof url === "string" && isWebUrl(url.trim()))),
+    ];
+    if (candidates.length === 0)
+        return reachable;
+    const apiKey = config?.tavilyAPIKey;
+    if (!apiKey) {
+        logger_js_1.default.log(null, logger_js_1.default.levels.warn, `Tavily reachability: no tavilyAPIKey, so none of ${candidates.length} URL(s) is confirmed`);
+        return reachable;
+    }
+    const batches = [];
+    for (let start = 0; start < candidates.length; start += EXTRACT_MAX_URLS) {
+        batches.push(candidates.slice(start, start + EXTRACT_MAX_URLS));
+    }
+    await Promise.all(batches.map(async (batch) => {
+        let data;
+        try {
+            data = await callTavily("extract", {
+                urls: batch.map((url) => url.trim()),
+                extract_depth: "basic",
+                format: "text",
+            }, apiKey, REACHABILITY_TIMEOUT_SECONDS);
+        }
+        catch (err) {
+            logger_js_1.default.log(null, logger_js_1.default.levels.warn, `Tavily reachability: the check of ${batch.length} URL(s) failed (${err?.message || String(err)}), so none of them is confirmed`);
+            return;
+        }
+        recordTavilyUsage(usageTracker, config, data?.usage?.credits || 0, []);
+        const failed = new Set((data?.failed_results || [])
+            .map((entry) => typeof entry?.url === "string" ? urlMatchKey(entry.url) : "")
+            .filter(Boolean));
+        for (const url of batch) {
+            if (!failed.has(urlMatchKey(url)))
+                reachable.add(url);
+        }
+    }));
+    return reachable;
+};
+exports.checkUrlsReachable = checkUrlsReachable;
 exports.default = {
     assertTavilyUsable: exports.assertTavilyUsable,
     buildTavilyTools: exports.buildTavilyTools,
+    checkUrlsReachable: exports.checkUrlsReachable,
     getTavilyOptions: exports.getTavilyOptions,
     recordTavilyUsage,
 };

package/dist/esm/index.d.ts

@@ -7,11 +7,13 @@ import { callAgent, checkAgent } from "./client.js";
 import * as runtimeContext from "./runtimeContext.js";
 import { recordUsage, USAGE_UNITS } from "./usage.js";
 import stackfactorKnowledge from "./stackfactorKnowledge.js";
+import { checkUrlsReachable } from "./tavily.js";
 export { constants };
 export { errorHandling, AppError };
 export type { ParsedError } from "./errorHandling.js";
 export { langChain };
 export type { UsageTracker } from "./langChain.js";
+export { checkUrlsReachable };
 export type { TavilyConfig, TavilyToolName } from "./tavily.js";
 export { logger };
 export { serve, runOnce, isRunOnceExecution, EXECUTION_ENV };

package/dist/esm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAEhC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAEhC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC"}

package/dist/esm/index.js

@@ -7,9 +7,11 @@ import { callAgent, checkAgent } from "./client.js";
 import * as runtimeContext from "./runtimeContext.js";
 import { recordUsage, USAGE_UNITS } from "./usage.js";
 import stackfactorKnowledge from "./stackfactorKnowledge.js";
+import { checkUrlsReachable } from "./tavily.js";
 export { constants };
 export { errorHandling, AppError };
 export { langChain };
+export { checkUrlsReachable };
 export { logger };
 export { serve, runOnce, isRunOnceExecution, EXECUTION_ENV };
 export { callAgent };

package/dist/esm/tavily.d.ts

@@ -82,9 +82,40 @@ export declare const assertTavilyUsable: (config: any, options: TavilyConfig) =>
  * @returns LangChain tool instances to append to the agent's tool list
  */
 export declare const buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
+/**
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
+ * this process connecting to them.
+ *
+ * An agent that confirms a cited link by requesting it itself sends that
+ * request from inside StackFactor's network, through its fixed egress
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
+ * links come from web pages anyone can publish, so that is a request to an
+ * address an outsider chose (server-side request forgery). Tavily fetches from
+ * its own infrastructure, so nothing here ever connects to such a host.
+ *
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
+ * like every other Tavily call, but the URLs are not added to the tracker's
+ * sources: a link worth checking was found there in the first place.
+ *
+ * Tavily accounts for every URL it is given, listing the ones it could not
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
+ * other URL of a batch Tavily answered is reachable (the results may name a
+ * URL by its address after a redirect, which is why they are not matched). A
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
+ * confirms none of its URLs: a link that could not be checked is not trusted.
+ *
+ * @param urls - The URLs to check; a URL given twice is checked once
+ * @param config - The integration config; supplies `tavilyAPIKey`
+ * @param usageTracker - Optional accumulator the credits are counted into
+ * @returns The given URLs, exactly as passed, that resolve
+ */
+export declare const checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
 declare const _default: {
     assertTavilyUsable: (config: any, options: TavilyConfig) => void;
     buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
+    checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
     getTavilyOptions: (config: any) => TavilyConfig | null;
     recordTavilyUsage: (tracker: UsageTracker | null | undefined, _config: any, credits: number, sources: {
         url: string;

package/dist/esm/tavily.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;;iCA5dyC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BAhPiC,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAmWP,wBAKE"}
+{"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;AAqBF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,eAAO,MAAM,kBAAkB,GAC7B,MAAM,SAAS,MAAM,EAAE,EACvB,QAAQ,GAAG,EACX,eAAc,YAAY,GAAG,IAAW,KACvC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CA6DrB,CAAC;;iCA/kByC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BA8SA,SAAS,MAAM,EAAE,UACf,GAAG,iBACG,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;+BAjiBkB,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAsdP,wBAME"}

package/dist/esm/tavily.js

@@ -386,9 +386,95 @@ export const buildTavilyTools = (config, options, usageTracker = null) => {
     logger.log(null, logger.levels.info, `Tavily tools enabled: ${tools.map((entry) => entry.name).join(", ")}`);
     return tools;
 };
+/** Tavily's own limit on URLs per `/extract` request. */
+const EXTRACT_MAX_URLS = 20;
+/** Seconds. A reachability check needs the page fetched, not rendered at length. */
+const REACHABILITY_TIMEOUT_SECONDS = 20;
+/** How a URL is matched against Tavily's answer: trimmed, lowercased, no trailing slash. */
+const urlMatchKey = (url) => url.trim().replace(/\/+$/, "").toLowerCase();
+/** True for an `http:` or `https:` URL; false for anything else, unparsable included. */
+const isWebUrl = (url) => {
+    try {
+        const { protocol } = new URL(url);
+        return protocol === "http:" || protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+};
+/**
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
+ * this process connecting to them.
+ *
+ * An agent that confirms a cited link by requesting it itself sends that
+ * request from inside StackFactor's network, through its fixed egress
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
+ * links come from web pages anyone can publish, so that is a request to an
+ * address an outsider chose (server-side request forgery). Tavily fetches from
+ * its own infrastructure, so nothing here ever connects to such a host.
+ *
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
+ * like every other Tavily call, but the URLs are not added to the tracker's
+ * sources: a link worth checking was found there in the first place.
+ *
+ * Tavily accounts for every URL it is given, listing the ones it could not
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
+ * other URL of a batch Tavily answered is reachable (the results may name a
+ * URL by its address after a redirect, which is why they are not matched). A
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
+ * confirms none of its URLs: a link that could not be checked is not trusted.
+ *
+ * @param urls - The URLs to check; a URL given twice is checked once
+ * @param config - The integration config; supplies `tavilyAPIKey`
+ * @param usageTracker - Optional accumulator the credits are counted into
+ * @returns The given URLs, exactly as passed, that resolve
+ */
+export const checkUrlsReachable = async (urls, config, usageTracker = null) => {
+    const reachable = new Set();
+    const candidates = [
+        ...new Set(urls.filter((url) => typeof url === "string" && isWebUrl(url.trim()))),
+    ];
+    if (candidates.length === 0)
+        return reachable;
+    const apiKey = config?.tavilyAPIKey;
+    if (!apiKey) {
+        logger.log(null, logger.levels.warn, `Tavily reachability: no tavilyAPIKey, so none of ${candidates.length} URL(s) is confirmed`);
+        return reachable;
+    }
+    const batches = [];
+    for (let start = 0; start < candidates.length; start += EXTRACT_MAX_URLS) {
+        batches.push(candidates.slice(start, start + EXTRACT_MAX_URLS));
+    }
+    await Promise.all(batches.map(async (batch) => {
+        let data;
+        try {
+            data = await callTavily("extract", {
+                urls: batch.map((url) => url.trim()),
+                extract_depth: "basic",
+                format: "text",
+            }, apiKey, REACHABILITY_TIMEOUT_SECONDS);
+        }
+        catch (err) {
+            logger.log(null, logger.levels.warn, `Tavily reachability: the check of ${batch.length} URL(s) failed (${err?.message || String(err)}), so none of them is confirmed`);
+            return;
+        }
+        recordTavilyUsage(usageTracker, config, data?.usage?.credits || 0, []);
+        const failed = new Set((data?.failed_results || [])
+            .map((entry) => typeof entry?.url === "string" ? urlMatchKey(entry.url) : "")
+            .filter(Boolean));
+        for (const url of batch) {
+            if (!failed.has(urlMatchKey(url)))
+                reachable.add(url);
+        }
+    }));
+    return reachable;
+};
 export default {
     assertTavilyUsable,
     buildTavilyTools,
+    checkUrlsReachable,
     getTavilyOptions,
     recordTavilyUsage,
 };

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.8.0",
+  "version": "1.9.0",
   "description": "",
   "main": "dist/cjs/index.js",
   "module": "dist/esm/index.js",
@@ -20,7 +20,7 @@
     "work:release": "bash scripts/release.sh",
     "release": "bash scripts/release.sh",
     "release:fanout": "bash scripts/release-fanout.sh",
-    "ar-auth": "npm config set @stackfactor:registry https://us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/; TOKEN=$(gcloud auth print-access-token 2>/dev/null || curl -sfH 'Metadata-Flavor: Google' http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token | sed -n 's/.*\"access_token\":\"\\([^\"]*\\)\".*/\\1/p'); test -n \"$TOKEN\" || { echo 'ar-auth: no credential \u2014 run: gcloud auth login' >&2; exit 1; }; npm config set //us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/:_authToken \"$TOKEN\"; npm config set //us-central1-npm.pkg.dev/virtual-development-team/:_authToken \"$TOKEN\"; echo 'ar-auth: Artifact Registry scope + token configured'",
+    "ar-auth": "npm config set @stackfactor:registry https://us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/; TOKEN=$(gcloud auth print-access-token 2>/dev/null || curl -sfH 'Metadata-Flavor: Google' http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token | sed -n 's/.*\"access_token\":\"\\([^\"]*\\)\".*/\\1/p'); test -n \"$TOKEN\" || { echo 'ar-auth: no credential — run: gcloud auth login' >&2; exit 1; }; npm config set //us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/:_authToken \"$TOKEN\"; npm config set //us-central1-npm.pkg.dev/virtual-development-team/:_authToken \"$TOKEN\"; echo 'ar-auth: Artifact Registry scope + token configured'",
     "reinstall": "npm run ar-auth && npm ci",
     "test:coverage": "npm run build && node --experimental-test-coverage --test --test-reporter=tap test/*.test.js"
   },