@stackfactor/agent-utils 1.8.0 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. package/README.md +15 -0
  2. package/dist/cjs/index.d.ts +2 -0
  3. package/dist/cjs/index.d.ts.map +1 -1
  4. package/dist/cjs/index.js +3 -1
  5. package/dist/cjs/stackfactorKnowledge.d.ts +8 -1
  6. package/dist/cjs/stackfactorKnowledge.d.ts.map +1 -1
  7. package/dist/cjs/stackfactorKnowledge.js +30 -20
  8. package/dist/cjs/tavily.d.ts +31 -0
  9. package/dist/cjs/tavily.d.ts.map +1 -1
  10. package/dist/cjs/tavily.js +88 -1
  11. package/dist/esm/index.d.ts +2 -0
  12. package/dist/esm/index.d.ts.map +1 -1
  13. package/dist/esm/index.js +2 -0
  14. package/dist/esm/stackfactorKnowledge.d.ts +8 -1
  15. package/dist/esm/stackfactorKnowledge.d.ts.map +1 -1
  16. package/dist/esm/stackfactorKnowledge.js +30 -20
  17. package/dist/esm/tavily.d.ts +31 -0
  18. package/dist/esm/tavily.d.ts.map +1 -1
  19. package/dist/esm/tavily.js +86 -0
  20. package/package.json +2 -2
package/README.md CHANGED
@@ -419,6 +419,21 @@ A model that cannot tell it received half a regulation will reason over the half
419
419
  - Per-URL failures are surfaced in the tool result instead of being dropped, so a fetch failure is not mistaken for an absence of content.
420
420
  - Tool calls honour the caller's abort signal, so a cancelled run stops paying for in-flight crawls.
421
421
 
422
+ ### Checking that cited links resolve
423
+
424
+ `checkUrlsReachable(urls, config, usageTracker?)` returns the URLs, exactly as passed, that Tavily could fetch. Use it in place of requesting a link yourself. Cited links come from web pages anyone can publish, so a link, its redirects or its DNS can point inside StackFactor's network; a request this process sends would go there through StackFactor's own egress addresses. Tavily fetches from its own infrastructure.
425
+
426
+ ```js
427
+ const { checkUrlsReachable } = require("@stackfactor/agent-utils");
428
+
429
+ const live = await checkUrlsReachable(references.map((r) => r.url), config, usageTracker);
430
+ const kept = references.filter((r) => !r.url || live.has(r.url));
431
+ ```
432
+
433
+ - Only `http:` and `https:` URLs are sent, 20 per call (Tavily's limit), at basic depth: 1 credit per 5 URLs fetched, failed ones free. Credits are billed like any Tavily call; the URLs are not added to `webSearchSources`.
434
+ - A URL Tavily lists under `failed_results` is unreachable. A batch whose call fails, or a config without `tavilyAPIKey`, confirms none of its URLs. It never throws.
435
+ - No `config.tavily` or `agentic` is needed: it calls Tavily directly, not through the model.
436
+
422
437
  ---
423
438
 
424
439
  ## Supported Models
package/dist/cjs/index.d.ts CHANGED
@@ -7,11 +7,13 @@ import { callAgent, checkAgent } from "./client.js";
7
7
  import * as runtimeContext from "./runtimeContext.js";
8
8
  import { recordUsage, USAGE_UNITS } from "./usage.js";
9
9
  import stackfactorKnowledge from "./stackfactorKnowledge.js";
10
+ import { checkUrlsReachable } from "./tavily.js";
10
11
  export { constants };
11
12
  export { errorHandling, AppError };
12
13
  export type { ParsedError } from "./errorHandling.js";
13
14
  export { langChain };
14
15
  export type { UsageTracker } from "./langChain.js";
16
+ export { checkUrlsReachable };
15
17
  export type { TavilyConfig, TavilyToolName } from "./tavily.js";
16
18
  export { logger };
17
19
  export { serve, runOnce, isRunOnceExecution, EXECUTION_ENV };
package/dist/cjs/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAEhC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAEhC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC"}
package/dist/cjs/index.js CHANGED
@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
36
36
  return (mod && mod.__esModule) ? mod : { "default": mod };
37
37
  };
38
38
  Object.defineProperty(exports, "__esModule", { value: true });
39
- exports.stackfactorKnowledge = exports.USAGE_UNITS = exports.recordUsage = exports.runtimeContext = exports.checkAgent = exports.callAgent = exports.EXECUTION_ENV = exports.isRunOnceExecution = exports.runOnce = exports.serve = exports.logger = exports.langChain = exports.AppError = exports.errorHandling = exports.constants = void 0;
39
+ exports.stackfactorKnowledge = exports.USAGE_UNITS = exports.recordUsage = exports.runtimeContext = exports.checkAgent = exports.callAgent = exports.EXECUTION_ENV = exports.isRunOnceExecution = exports.runOnce = exports.serve = exports.logger = exports.checkUrlsReachable = exports.langChain = exports.AppError = exports.errorHandling = exports.constants = void 0;
40
40
  const const_js_1 = __importDefault(require("./const.js"));
41
41
  exports.constants = const_js_1.default;
42
42
  const errorHandling_js_1 = __importStar(require("./errorHandling.js"));
@@ -61,3 +61,5 @@ Object.defineProperty(exports, "recordUsage", { enumerable: true, get: function
61
61
  Object.defineProperty(exports, "USAGE_UNITS", { enumerable: true, get: function () { return usage_js_1.USAGE_UNITS; } });
62
62
  const stackfactorKnowledge_js_1 = __importDefault(require("./stackfactorKnowledge.js"));
63
63
  exports.stackfactorKnowledge = stackfactorKnowledge_js_1.default;
64
+ const tavily_js_1 = require("./tavily.js");
65
+ Object.defineProperty(exports, "checkUrlsReachable", { enumerable: true, get: function () { return tavily_js_1.checkUrlsReachable; } });
package/dist/cjs/stackfactorKnowledge.d.ts CHANGED
@@ -17,6 +17,8 @@ export interface KnowledgeChunk {
17
17
  knowledgeBaseId?: string;
18
18
  documentId: string;
19
19
  documentName?: string | null;
20
+ /** The document's page at its source (sent by the platform's search since backend-core #630). */
21
+ documentUrl?: string | null;
20
22
  sectionTitle?: string | null;
21
23
  pageNumber?: number | null;
22
24
  text?: string;
@@ -48,6 +50,8 @@ export interface GroundingPassage {
48
50
  knowledgeBaseId: string;
49
51
  documentId: string;
50
52
  documentName: string | null;
53
+ /** The document's page at its source, an http(s) address; null when the platform has none. */
54
+ documentUrl?: string | null;
51
55
  sectionTitle: string | null;
52
56
  pageNumber: number | null;
53
57
  text: string;
@@ -72,6 +76,8 @@ export interface StackFactorReference {
72
76
  kind: "STACKFACTOR";
73
77
  title: string;
74
78
  description: string | null;
79
+ /** The document's page at its source; absent when the platform has none. */
80
+ url?: string;
75
81
  target: {
76
82
  elementType: "KNOWLEDGE_BASE_DOCUMENT";
77
83
  elementId: string;
@@ -142,7 +148,8 @@ export declare const passagesForPrompt: (passages: GroundingPassage[]) => {
142
148
  /**
143
149
  * STACKFACTOR references for passages: one per passage, in their order, with no
144
150
  * cap. With `tags`, only the passages the model cited by tag (any tag it made
145
- * up is passed over); without, every passage.
151
+ * up is passed over); without, every passage. Each links to its document's
152
+ * page at the source (`url`) when the passage has one.
146
153
  * @param passages From `retrievePassages`
147
154
  * @param tags The tags the model cited ("k3", "K3" and " k3 " alike)
148
155
  * @returns The references, ready for a skill's, a microskill's or a question's `references`
package/dist/cjs/stackfactorKnowledge.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"stackfactorKnowledge.d.ts","sourceRoot":"","sources":["../../src/stackfactorKnowledge.ts"],"names":[],"mappings":"AA8BA,kGAAkG;AAClG,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AAEH,0FAA0F;AAC1F,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,0EAA0E;AAC1E,eAAO,MAAM,gBAAgB,OAAO,CAAC;AAErC,0EAA0E;AAC1E,eAAO,MAAM,aAAa,IAAI,CAAC;AAQ/B,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3D,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,oFAAoF;AACpF,MAAM,MAAM,eAAe,GAAG,CAC5B,eAAe,EAAE,MAAM,EACvB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,EAClD,KAAK,EAAE,MAAM,KACV,OAAO,CAAC;IAAE,MAAM,EAAE,cAAc,EAAE,CAAA;CAAE,CAAC,CAAC;AAE3C,8EAA8E;AAC9E,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IACzD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,4DAA4D;AAC5D,MAAM,WAAW,aAAa;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,mFAAmF;IACnF,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kFAAkF;AAClF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE;QAAE,WAAW,EAAE,yBAAyB,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,aAAa,EAAE;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;IACF,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;;;;GAQG;AACH,eAAO,MAAM,qBAAqB,GAAI,YAAY,OAAO,KAAG,eAAe,EAmB1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,OAAO,KAAG,eAAe,EAWtE,CAAC;AA8BF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gBAAgB,GAAU,SAAS;IAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,KAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAgFrE,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAC5B,UAAU,gBAAgB,EAAE,KAC3B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAM5E,CAAC;AAEN;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa,GACxB,UAAU,gBAAgB,EAAE,EAC5B,OAAO,OAAO,EAAE,KACf,oBAAoB,EAyBtB,CAAC;;;;;;;;;;;kCA7CU,gBAAgB,EAAE,KAC3B;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;8BAiBvE,gBAAgB,EAAE,SACrB,OAAO,EAAE,KACf,oBAAoB,EAAE;gCAlHuB;QAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;QAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,MAAM,EAAE,eAAe,CAAC;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,KAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,aAAa,EAAE,CAAA;KAAE,CAAC;oCA7DzB,OAAO,KAAG,eAAe,EAAE;wCA5BvB,OAAO,KAAG,eAAe,EAAE;;AAgO7E,wBAUE"}
1
+ {"version":3,"file":"stackfactorKnowledge.d.ts","sourceRoot":"","sources":["../../src/stackfactorKnowledge.ts"],"names":[],"mappings":"AAgCA,kGAAkG;AAClG,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AAEH,0FAA0F;AAC1F,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,0EAA0E;AAC1E,eAAO,MAAM,gBAAgB,OAAO,CAAC;AAErC,0EAA0E;AAC1E,eAAO,MAAM,aAAa,IAAI,CAAC;AAQ/B,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iGAAiG;IACjG,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3D,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,oFAAoF;AACpF,MAAM,MAAM,eAAe,GAAG,CAC5B,eAAe,EAAE,MAAM,EACvB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,EAClD,KAAK,EAAE,MAAM,KACV,OAAO,CAAC;IAAE,MAAM,EAAE,cAAc,EAAE,CAAA;CAAE,CAAC,CAAC;AAE3C,8EAA8E;AAC9E,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,8FAA8F;IAC9F,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IACzD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,4DAA4D;AAC5D,MAAM,WAAW,aAAa;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,mFAAmF;IACnF,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kFAAkF;AAClF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,4EAA4E;IAC5E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,WAAW,EAAE,yBAAyB,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,aAAa,EAAE;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;IACF,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB;AAQD;;;;;;;;GAQG;AACH,eAAO,MAAM,qBAAqB,GAAI,YAAY,OAAO,KAAG,eAAe,EAmB1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,OAAO,KAAG,eAAe,EAWtE,CAAC;AA8BF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gBAAgB,GAAU,SAAS;IAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,KAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAiFrE,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAC5B,UAAU,gBAAgB,EAAE,KAC3B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAM5E,CAAC;AAEN;;;;;;;;GAQG;AACH,eAAO,MAAM,aAAa,GACxB,UAAU,gBAAgB,EAAE,EAC5B,OAAO,OAAO,EAAE,KACf,oBAAoB,EA6BtB,CAAC;;;;;;;;;;;kCAlDU,gBAAgB,EAAE,KAC3B;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;8BAkBvE,gBAAgB,EAAE,SACrB,OAAO,EAAE,KACf,oBAAoB,EAAE;gCApHuB;QAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;QAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,MAAM,EAAE,eAAe,CAAC;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,KAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,aAAa,EAAE,CAAA;KAAE,CAAC;oCA7DzB,OAAO,KAAG,eAAe,EAAE;wCA5BvB,OAAO,KAAG,eAAe,EAAE;;AAsO7E,wBAUE"}
package/dist/cjs/stackfactorKnowledge.js CHANGED
@@ -19,7 +19,9 @@
19
19
  // generates without it.
20
20
  //
21
21
  // A reference points at the content and never copies it: the passage is
22
- // searched again, under the reader's own access, whenever it is used. It is
22
+ // searched again, under the reader's own access, whenever it is used. Its
23
+ // `url` is the document's page at its source (GitHub, Confluence…), when the
24
+ // platform knows it, so a person can open what was cited. It is
23
25
  // EDIT-only (`visibility: "EDIT"`): the platform strips it from what is
24
26
  // published, so a knowledge base's titles never reach a learner; agents read
25
27
  // it from the draft.
@@ -48,6 +50,8 @@ const CONCURRENCY = 4;
48
50
  /** Statuses that mean the person may not read the knowledge base, or it is gone. */
49
51
  const NO_ACCESS = new Set([401, 403, 404]);
50
52
  const text = (value) => (typeof value === "string" ? value.trim() : "");
53
+ /** An http(s) address, or null: a reference's url lands in an `href`. */
54
+ const webUrl = (value) => (/^https?:\/\//i.test(text(value)) ? text(value) : null);
51
55
  const id = (value) => typeof value === "string" ? value.trim() : value != null && typeof value === "object" ? String(value) : "";
52
56
  /**
53
57
  * The knowledge bases, and the documents in them, that a list of references
@@ -200,6 +204,7 @@ const retrievePassages = async (options) => {
200
204
  chunkId,
201
205
  documentId: id(chunk.documentId),
202
206
  documentName: text(chunk.documentName) || null,
207
+ documentUrl: webUrl(chunk.documentUrl),
203
208
  knowledgeBaseId: id(chunk.knowledgeBaseId) || knowledgeBaseId,
204
209
  pageNumber: typeof chunk.pageNumber === "number" ? chunk.pageNumber : null,
205
210
  score: typeof chunk.score === "number" ? chunk.score : null,
@@ -229,7 +234,8 @@ exports.passagesForPrompt = passagesForPrompt;
229
234
  /**
230
235
  * STACKFACTOR references for passages: one per passage, in their order, with no
231
236
  * cap. With `tags`, only the passages the model cited by tag (any tag it made
232
- * up is passed over); without, every passage.
237
+ * up is passed over); without, every passage. Each links to its document's
238
+ * page at the source (`url`) when the passage has one.
233
239
  * @param passages From `retrievePassages`
234
240
  * @param tags The tags the model cited ("k3", "K3" and " k3 " alike)
235
241
  * @returns The references, ready for a skill's, a microskill's or a question's `references`
@@ -241,24 +247,28 @@ const referencesFor = (passages, tags) => {
241
247
  : null;
242
248
  return list
243
249
  .filter((passage) => !cited || cited.has(passage.tag.toLowerCase()))
244
- .map((passage) => ({
245
- description: passage.sectionTitle,
246
- kind: exports.STACKFACTOR_REFERENCE.KIND,
247
- knowledgeBase: {
248
- chunkId: passage.chunkId,
249
- documentId: passage.documentId,
250
- knowledgeBaseId: passage.knowledgeBaseId,
251
- sectionTitle: passage.sectionTitle,
252
- },
253
- origin: exports.STACKFACTOR_REFERENCE.ORIGIN,
254
- target: {
255
- elementId: passage.documentId,
256
- elementType: exports.STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE,
257
- parentId: passage.knowledgeBaseId,
258
- },
259
- title: passage.documentName || passage.sectionTitle || "Knowledge base document",
260
- visibility: exports.STACKFACTOR_REFERENCE.VISIBILITY,
261
- }));
250
+ .map((passage) => {
251
+ const url = webUrl(passage.documentUrl);
252
+ return {
253
+ description: passage.sectionTitle,
254
+ kind: exports.STACKFACTOR_REFERENCE.KIND,
255
+ knowledgeBase: {
256
+ chunkId: passage.chunkId,
257
+ documentId: passage.documentId,
258
+ knowledgeBaseId: passage.knowledgeBaseId,
259
+ sectionTitle: passage.sectionTitle,
260
+ },
261
+ origin: exports.STACKFACTOR_REFERENCE.ORIGIN,
262
+ target: {
263
+ elementId: passage.documentId,
264
+ elementType: exports.STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE,
265
+ parentId: passage.knowledgeBaseId,
266
+ },
267
+ title: passage.documentName || passage.sectionTitle || "Knowledge base document",
268
+ ...(url ? { url } : {}),
269
+ visibility: exports.STACKFACTOR_REFERENCE.VISIBILITY,
270
+ };
271
+ });
262
272
  };
263
273
  exports.referencesFor = referencesFor;
264
274
  exports.default = {
package/dist/cjs/tavily.d.ts CHANGED
@@ -82,9 +82,40 @@ export declare const assertTavilyUsable: (config: any, options: TavilyConfig) =>
82
82
  * @returns LangChain tool instances to append to the agent's tool list
83
83
  */
84
84
  export declare const buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
85
+ /**
86
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
87
+ * this process connecting to them.
88
+ *
89
+ * An agent that confirms a cited link by requesting it itself sends that
90
+ * request from inside StackFactor's network, through its fixed egress
91
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
92
+ * links come from web pages anyone can publish, so that is a request to an
93
+ * address an outsider chose (server-side request forgery). Tavily fetches from
94
+ * its own infrastructure, so nothing here ever connects to such a host.
95
+ *
96
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
97
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
98
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
99
+ * like every other Tavily call, but the URLs are not added to the tracker's
100
+ * sources: a link worth checking was found there in the first place.
101
+ *
102
+ * Tavily accounts for every URL it is given, listing the ones it could not
103
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
104
+ * other URL of a batch Tavily answered is reachable (the results may name a
105
+ * URL by its address after a redirect, which is why they are not matched). A
106
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
107
+ * confirms none of its URLs: a link that could not be checked is not trusted.
108
+ *
109
+ * @param urls - The URLs to check; a URL given twice is checked once
110
+ * @param config - The integration config; supplies `tavilyAPIKey`
111
+ * @param usageTracker - Optional accumulator the credits are counted into
112
+ * @returns The given URLs, exactly as passed, that resolve
113
+ */
114
+ export declare const checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
85
115
  declare const _default: {
86
116
  assertTavilyUsable: (config: any, options: TavilyConfig) => void;
87
117
  buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
118
+ checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
88
119
  getTavilyOptions: (config: any) => TavilyConfig | null;
89
120
  recordTavilyUsage: (tracker: UsageTracker | null | undefined, _config: any, credits: number, sources: {
90
121
  url: string;
package/dist/cjs/tavily.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;;iCA5dyC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BAhPiC,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAmWP,wBAKE"}
1
+ {"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;AAqBF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,eAAO,MAAM,kBAAkB,GAC7B,MAAM,SAAS,MAAM,EAAE,EACvB,QAAQ,GAAG,EACX,eAAc,YAAY,GAAG,IAAW,KACvC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CA6DrB,CAAC;;iCA/kByC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BA8SA,SAAS,MAAM,EAAE,UACf,GAAG,iBACG,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;+BAjiBkB,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAsdP,wBAME"}
package/dist/cjs/tavily.js CHANGED
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.buildTavilyTools = exports.assertTavilyUsable = exports.getTavilyOptions = void 0;
6
+ exports.checkUrlsReachable = exports.buildTavilyTools = exports.assertTavilyUsable = exports.getTavilyOptions = void 0;
7
7
  const langchain_1 = require("langchain");
8
8
  const zod_1 = require("zod");
9
9
  const const_js_1 = __importDefault(require("./const.js"));
@@ -395,9 +395,96 @@ const buildTavilyTools = (config, options, usageTracker = null) => {
395
395
  return tools;
396
396
  };
397
397
  exports.buildTavilyTools = buildTavilyTools;
398
+ /** Tavily's own limit on URLs per `/extract` request. */
399
+ const EXTRACT_MAX_URLS = 20;
400
+ /** Seconds. A reachability check needs the page fetched, not rendered at length. */
401
+ const REACHABILITY_TIMEOUT_SECONDS = 20;
402
+ /** How a URL is matched against Tavily's answer: trimmed, lowercased, no trailing slash. */
403
+ const urlMatchKey = (url) => url.trim().replace(/\/+$/, "").toLowerCase();
404
+ /** True for an `http:` or `https:` URL; false for anything else, unparsable included. */
405
+ const isWebUrl = (url) => {
406
+ try {
407
+ const { protocol } = new URL(url);
408
+ return protocol === "http:" || protocol === "https:";
409
+ }
410
+ catch {
411
+ return false;
412
+ }
413
+ };
414
+ /**
415
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
416
+ * this process connecting to them.
417
+ *
418
+ * An agent that confirms a cited link by requesting it itself sends that
419
+ * request from inside StackFactor's network, through its fixed egress
420
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
421
+ * links come from web pages anyone can publish, so that is a request to an
422
+ * address an outsider chose (server-side request forgery). Tavily fetches from
423
+ * its own infrastructure, so nothing here ever connects to such a host.
424
+ *
425
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
426
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
427
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
428
+ * like every other Tavily call, but the URLs are not added to the tracker's
429
+ * sources: a link worth checking was found there in the first place.
430
+ *
431
+ * Tavily accounts for every URL it is given, listing the ones it could not
432
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
433
+ * other URL of a batch Tavily answered is reachable (the results may name a
434
+ * URL by its address after a redirect, which is why they are not matched). A
435
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
436
+ * confirms none of its URLs: a link that could not be checked is not trusted.
437
+ *
438
+ * @param urls - The URLs to check; a URL given twice is checked once
439
+ * @param config - The integration config; supplies `tavilyAPIKey`
440
+ * @param usageTracker - Optional accumulator the credits are counted into
441
+ * @returns The given URLs, exactly as passed, that resolve
442
+ */
443
+ const checkUrlsReachable = async (urls, config, usageTracker = null) => {
444
+ const reachable = new Set();
445
+ const candidates = [
446
+ ...new Set(urls.filter((url) => typeof url === "string" && isWebUrl(url.trim()))),
447
+ ];
448
+ if (candidates.length === 0)
449
+ return reachable;
450
+ const apiKey = config?.tavilyAPIKey;
451
+ if (!apiKey) {
452
+ logger_js_1.default.log(null, logger_js_1.default.levels.warn, `Tavily reachability: no tavilyAPIKey, so none of ${candidates.length} URL(s) is confirmed`);
453
+ return reachable;
454
+ }
455
+ const batches = [];
456
+ for (let start = 0; start < candidates.length; start += EXTRACT_MAX_URLS) {
457
+ batches.push(candidates.slice(start, start + EXTRACT_MAX_URLS));
458
+ }
459
+ await Promise.all(batches.map(async (batch) => {
460
+ let data;
461
+ try {
462
+ data = await callTavily("extract", {
463
+ urls: batch.map((url) => url.trim()),
464
+ extract_depth: "basic",
465
+ format: "text",
466
+ }, apiKey, REACHABILITY_TIMEOUT_SECONDS);
467
+ }
468
+ catch (err) {
469
+ logger_js_1.default.log(null, logger_js_1.default.levels.warn, `Tavily reachability: the check of ${batch.length} URL(s) failed (${err?.message || String(err)}), so none of them is confirmed`);
470
+ return;
471
+ }
472
+ recordTavilyUsage(usageTracker, config, data?.usage?.credits || 0, []);
473
+ const failed = new Set((data?.failed_results || [])
474
+ .map((entry) => typeof entry?.url === "string" ? urlMatchKey(entry.url) : "")
475
+ .filter(Boolean));
476
+ for (const url of batch) {
477
+ if (!failed.has(urlMatchKey(url)))
478
+ reachable.add(url);
479
+ }
480
+ }));
481
+ return reachable;
482
+ };
483
+ exports.checkUrlsReachable = checkUrlsReachable;
398
484
  exports.default = {
399
485
  assertTavilyUsable: exports.assertTavilyUsable,
400
486
  buildTavilyTools: exports.buildTavilyTools,
487
+ checkUrlsReachable: exports.checkUrlsReachable,
401
488
  getTavilyOptions: exports.getTavilyOptions,
402
489
  recordTavilyUsage,
403
490
  };
package/dist/esm/index.d.ts CHANGED
@@ -7,11 +7,13 @@ import { callAgent, checkAgent } from "./client.js";
7
7
  import * as runtimeContext from "./runtimeContext.js";
8
8
  import { recordUsage, USAGE_UNITS } from "./usage.js";
9
9
  import stackfactorKnowledge from "./stackfactorKnowledge.js";
10
+ import { checkUrlsReachable } from "./tavily.js";
10
11
  export { constants };
11
12
  export { errorHandling, AppError };
12
13
  export type { ParsedError } from "./errorHandling.js";
13
14
  export { langChain };
14
15
  export type { UsageTracker } from "./langChain.js";
16
+ export { checkUrlsReachable };
15
17
  export type { TavilyConfig, TavilyToolName } from "./tavily.js";
16
18
  export { logger };
17
19
  export { serve, runOnce, isRunOnceExecution, EXECUTION_ENV };
package/dist/esm/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAEhC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAEhC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC"}
package/dist/esm/index.js CHANGED
@@ -7,9 +7,11 @@ import { callAgent, checkAgent } from "./client.js";
7
7
  import * as runtimeContext from "./runtimeContext.js";
8
8
  import { recordUsage, USAGE_UNITS } from "./usage.js";
9
9
  import stackfactorKnowledge from "./stackfactorKnowledge.js";
10
+ import { checkUrlsReachable } from "./tavily.js";
10
11
  export { constants };
11
12
  export { errorHandling, AppError };
12
13
  export { langChain };
14
+ export { checkUrlsReachable };
13
15
  export { logger };
14
16
  export { serve, runOnce, isRunOnceExecution, EXECUTION_ENV };
15
17
  export { callAgent };
package/dist/esm/stackfactorKnowledge.d.ts CHANGED
@@ -17,6 +17,8 @@ export interface KnowledgeChunk {
17
17
  knowledgeBaseId?: string;
18
18
  documentId: string;
19
19
  documentName?: string | null;
20
+ /** The document's page at its source (sent by the platform's search since backend-core #630). */
21
+ documentUrl?: string | null;
20
22
  sectionTitle?: string | null;
21
23
  pageNumber?: number | null;
22
24
  text?: string;
@@ -48,6 +50,8 @@ export interface GroundingPassage {
48
50
  knowledgeBaseId: string;
49
51
  documentId: string;
50
52
  documentName: string | null;
53
+ /** The document's page at its source, an http(s) address; null when the platform has none. */
54
+ documentUrl?: string | null;
51
55
  sectionTitle: string | null;
52
56
  pageNumber: number | null;
53
57
  text: string;
@@ -72,6 +76,8 @@ export interface StackFactorReference {
72
76
  kind: "STACKFACTOR";
73
77
  title: string;
74
78
  description: string | null;
79
+ /** The document's page at its source; absent when the platform has none. */
80
+ url?: string;
75
81
  target: {
76
82
  elementType: "KNOWLEDGE_BASE_DOCUMENT";
77
83
  elementId: string;
@@ -142,7 +148,8 @@ export declare const passagesForPrompt: (passages: GroundingPassage[]) => {
142
148
  /**
143
149
  * STACKFACTOR references for passages: one per passage, in their order, with no
144
150
  * cap. With `tags`, only the passages the model cited by tag (any tag it made
145
- * up is passed over); without, every passage.
151
+ * up is passed over); without, every passage. Each links to its document's
152
+ * page at the source (`url`) when the passage has one.
146
153
  * @param passages From `retrievePassages`
147
154
  * @param tags The tags the model cited ("k3", "K3" and " k3 " alike)
148
155
  * @returns The references, ready for a skill's, a microskill's or a question's `references`
package/dist/esm/stackfactorKnowledge.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"stackfactorKnowledge.d.ts","sourceRoot":"","sources":["../../src/stackfactorKnowledge.ts"],"names":[],"mappings":"AA8BA,kGAAkG;AAClG,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AAEH,0FAA0F;AAC1F,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,0EAA0E;AAC1E,eAAO,MAAM,gBAAgB,OAAO,CAAC;AAErC,0EAA0E;AAC1E,eAAO,MAAM,aAAa,IAAI,CAAC;AAQ/B,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3D,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,oFAAoF;AACpF,MAAM,MAAM,eAAe,GAAG,CAC5B,eAAe,EAAE,MAAM,EACvB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,EAClD,KAAK,EAAE,MAAM,KACV,OAAO,CAAC;IAAE,MAAM,EAAE,cAAc,EAAE,CAAA;CAAE,CAAC,CAAC;AAE3C,8EAA8E;AAC9E,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IACzD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,4DAA4D;AAC5D,MAAM,WAAW,aAAa;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,mFAAmF;IACnF,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kFAAkF;AAClF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE;QAAE,WAAW,EAAE,yBAAyB,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,aAAa,EAAE;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;IACF,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;;;;GAQG;AACH,eAAO,MAAM,qBAAqB,GAAI,YAAY,OAAO,KAAG,eAAe,EAmB1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,OAAO,KAAG,eAAe,EAWtE,CAAC;AA8BF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gBAAgB,GAAU,SAAS;IAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,KAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAgFrE,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAC5B,UAAU,gBAAgB,EAAE,KAC3B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAM5E,CAAC;AAEN;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa,GACxB,UAAU,gBAAgB,EAAE,EAC5B,OAAO,OAAO,EAAE,KACf,oBAAoB,EAyBtB,CAAC;;;;;;;;;;;kCA7CU,gBAAgB,EAAE,KAC3B;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;8BAiBvE,gBAAgB,EAAE,SACrB,OAAO,EAAE,KACf,oBAAoB,EAAE;gCAlHuB;QAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;QAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,MAAM,EAAE,eAAe,CAAC;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,KAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,aAAa,EAAE,CAAA;KAAE,CAAC;oCA7DzB,OAAO,KAAG,eAAe,EAAE;wCA5BvB,OAAO,KAAG,eAAe,EAAE;;AAgO7E,wBAUE"}
1
+ {"version":3,"file":"stackfactorKnowledge.d.ts","sourceRoot":"","sources":["../../src/stackfactorKnowledge.ts"],"names":[],"mappings":"AAgCA,kGAAkG;AAClG,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AAEH,0FAA0F;AAC1F,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,0EAA0E;AAC1E,eAAO,MAAM,gBAAgB,OAAO,CAAC;AAErC,0EAA0E;AAC1E,eAAO,MAAM,aAAa,IAAI,CAAC;AAQ/B,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iGAAiG;IACjG,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3D,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,oFAAoF;AACpF,MAAM,MAAM,eAAe,GAAG,CAC5B,eAAe,EAAE,MAAM,EACvB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,EAClD,KAAK,EAAE,MAAM,KACV,OAAO,CAAC;IAAE,MAAM,EAAE,cAAc,EAAE,CAAA;CAAE,CAAC,CAAC;AAE3C,8EAA8E;AAC9E,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,8FAA8F;IAC9F,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IACzD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,4DAA4D;AAC5D,MAAM,WAAW,aAAa;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,mFAAmF;IACnF,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kFAAkF;AAClF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,4EAA4E;IAC5E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,WAAW,EAAE,yBAAyB,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,aAAa,EAAE;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;IACF,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB;AAQD;;;;;;;;GAQG;AACH,eAAO,MAAM,qBAAqB,GAAI,YAAY,OAAO,KAAG,eAAe,EAmB1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,OAAO,KAAG,eAAe,EAWtE,CAAC;AA8BF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gBAAgB,GAAU,SAAS;IAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,KAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAiFrE,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAC5B,UAAU,gBAAgB,EAAE,KAC3B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAM5E,CAAC;AAEN;;;;;;;;GAQG;AACH,eAAO,MAAM,aAAa,GACxB,UAAU,gBAAgB,EAAE,EAC5B,OAAO,OAAO,EAAE,KACf,oBAAoB,EA6BtB,CAAC;;;;;;;;;;;kCAlDU,gBAAgB,EAAE,KAC3B;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;8BAkBvE,gBAAgB,EAAE,SACrB,OAAO,EAAE,KACf,oBAAoB,EAAE;gCApHuB;QAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;QAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,MAAM,EAAE,eAAe,CAAC;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,KAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,aAAa,EAAE,CAAA;KAAE,CAAC;oCA7DzB,OAAO,KAAG,eAAe,EAAE;wCA5BvB,OAAO,KAAG,eAAe,EAAE;;AAsO7E,wBAUE"}
package/dist/esm/stackfactorKnowledge.js CHANGED
@@ -18,7 +18,9 @@
18
18
  // generates without it.
19
19
  //
20
20
  // A reference points at the content and never copies it: the passage is
21
- // searched again, under the reader's own access, whenever it is used. It is
21
+ // searched again, under the reader's own access, whenever it is used. Its
22
+ // `url` is the document's page at its source (GitHub, Confluence…), when the
23
+ // platform knows it, so a person can open what was cited. It is
22
24
  // EDIT-only (`visibility: "EDIT"`): the platform strips it from what is
23
25
  // published, so a knowledge base's titles never reach a learner; agents read
24
26
  // it from the draft.
@@ -45,6 +47,8 @@ const CONCURRENCY = 4;
45
47
  /** Statuses that mean the person may not read the knowledge base, or it is gone. */
46
48
  const NO_ACCESS = new Set([401, 403, 404]);
47
49
  const text = (value) => (typeof value === "string" ? value.trim() : "");
50
+ /** An http(s) address, or null: a reference's url lands in an `href`. */
51
+ const webUrl = (value) => (/^https?:\/\//i.test(text(value)) ? text(value) : null);
48
52
  const id = (value) => typeof value === "string" ? value.trim() : value != null && typeof value === "object" ? String(value) : "";
49
53
  /**
50
54
  * The knowledge bases, and the documents in them, that a list of references
@@ -195,6 +199,7 @@ export const retrievePassages = async (options) => {
195
199
  chunkId,
196
200
  documentId: id(chunk.documentId),
197
201
  documentName: text(chunk.documentName) || null,
202
+ documentUrl: webUrl(chunk.documentUrl),
198
203
  knowledgeBaseId: id(chunk.knowledgeBaseId) || knowledgeBaseId,
199
204
  pageNumber: typeof chunk.pageNumber === "number" ? chunk.pageNumber : null,
200
205
  score: typeof chunk.score === "number" ? chunk.score : null,
@@ -222,7 +227,8 @@ export const passagesForPrompt = (passages) => (Array.isArray(passages) ? passag
222
227
  /**
223
228
  * STACKFACTOR references for passages: one per passage, in their order, with no
224
229
  * cap. With `tags`, only the passages the model cited by tag (any tag it made
225
- * up is passed over); without, every passage.
230
+ * up is passed over); without, every passage. Each links to its document's
231
+ * page at the source (`url`) when the passage has one.
226
232
  * @param passages From `retrievePassages`
227
233
  * @param tags The tags the model cited ("k3", "K3" and " k3 " alike)
228
234
  * @returns The references, ready for a skill's, a microskill's or a question's `references`
@@ -234,24 +240,28 @@ export const referencesFor = (passages, tags) => {
234
240
  : null;
235
241
  return list
236
242
  .filter((passage) => !cited || cited.has(passage.tag.toLowerCase()))
237
- .map((passage) => ({
238
- description: passage.sectionTitle,
239
- kind: STACKFACTOR_REFERENCE.KIND,
240
- knowledgeBase: {
241
- chunkId: passage.chunkId,
242
- documentId: passage.documentId,
243
- knowledgeBaseId: passage.knowledgeBaseId,
244
- sectionTitle: passage.sectionTitle,
245
- },
246
- origin: STACKFACTOR_REFERENCE.ORIGIN,
247
- target: {
248
- elementId: passage.documentId,
249
- elementType: STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE,
250
- parentId: passage.knowledgeBaseId,
251
- },
252
- title: passage.documentName || passage.sectionTitle || "Knowledge base document",
253
- visibility: STACKFACTOR_REFERENCE.VISIBILITY,
254
- }));
243
+ .map((passage) => {
244
+ const url = webUrl(passage.documentUrl);
245
+ return {
246
+ description: passage.sectionTitle,
247
+ kind: STACKFACTOR_REFERENCE.KIND,
248
+ knowledgeBase: {
249
+ chunkId: passage.chunkId,
250
+ documentId: passage.documentId,
251
+ knowledgeBaseId: passage.knowledgeBaseId,
252
+ sectionTitle: passage.sectionTitle,
253
+ },
254
+ origin: STACKFACTOR_REFERENCE.ORIGIN,
255
+ target: {
256
+ elementId: passage.documentId,
257
+ elementType: STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE,
258
+ parentId: passage.knowledgeBaseId,
259
+ },
260
+ title: passage.documentName || passage.sectionTitle || "Knowledge base document",
261
+ ...(url ? { url } : {}),
262
+ visibility: STACKFACTOR_REFERENCE.VISIBILITY,
263
+ };
264
+ });
255
265
  };
256
266
  export default {
257
267
  DEFAULT_TOP_K,
package/dist/esm/tavily.d.ts CHANGED
@@ -82,9 +82,40 @@ export declare const assertTavilyUsable: (config: any, options: TavilyConfig) =>
82
82
  * @returns LangChain tool instances to append to the agent's tool list
83
83
  */
84
84
  export declare const buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
85
+ /**
86
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
87
+ * this process connecting to them.
88
+ *
89
+ * An agent that confirms a cited link by requesting it itself sends that
90
+ * request from inside StackFactor's network, through its fixed egress
91
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
92
+ * links come from web pages anyone can publish, so that is a request to an
93
+ * address an outsider chose (server-side request forgery). Tavily fetches from
94
+ * its own infrastructure, so nothing here ever connects to such a host.
95
+ *
96
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
97
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
98
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
99
+ * like every other Tavily call, but the URLs are not added to the tracker's
100
+ * sources: a link worth checking was found there in the first place.
101
+ *
102
+ * Tavily accounts for every URL it is given, listing the ones it could not
103
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
104
+ * other URL of a batch Tavily answered is reachable (the results may name a
105
+ * URL by its address after a redirect, which is why they are not matched). A
106
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
107
+ * confirms none of its URLs: a link that could not be checked is not trusted.
108
+ *
109
+ * @param urls - The URLs to check; a URL given twice is checked once
110
+ * @param config - The integration config; supplies `tavilyAPIKey`
111
+ * @param usageTracker - Optional accumulator the credits are counted into
112
+ * @returns The given URLs, exactly as passed, that resolve
113
+ */
114
+ export declare const checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
85
115
  declare const _default: {
86
116
  assertTavilyUsable: (config: any, options: TavilyConfig) => void;
87
117
  buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
118
+ checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
88
119
  getTavilyOptions: (config: any) => TavilyConfig | null;
89
120
  recordTavilyUsage: (tracker: UsageTracker | null | undefined, _config: any, credits: number, sources: {
90
121
  url: string;
package/dist/esm/tavily.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;;iCA5dyC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BAhPiC,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAmWP,wBAKE"}
1
+ {"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;AAqBF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,eAAO,MAAM,kBAAkB,GAC7B,MAAM,SAAS,MAAM,EAAE,EACvB,QAAQ,GAAG,EACX,eAAc,YAAY,GAAG,IAAW,KACvC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CA6DrB,CAAC;;iCA/kByC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BA8SA,SAAS,MAAM,EAAE,UACf,GAAG,iBACG,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;+BAjiBkB,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAsdP,wBAME"}
package/dist/esm/tavily.js CHANGED
@@ -386,9 +386,95 @@ export const buildTavilyTools = (config, options, usageTracker = null) => {
386
386
  logger.log(null, logger.levels.info, `Tavily tools enabled: ${tools.map((entry) => entry.name).join(", ")}`);
387
387
  return tools;
388
388
  };
389
+ /** Tavily's own limit on URLs per `/extract` request. */
390
+ const EXTRACT_MAX_URLS = 20;
391
+ /** Seconds. A reachability check needs the page fetched, not rendered at length. */
392
+ const REACHABILITY_TIMEOUT_SECONDS = 20;
393
+ /** How a URL is matched against Tavily's answer: trimmed, lowercased, no trailing slash. */
394
+ const urlMatchKey = (url) => url.trim().replace(/\/+$/, "").toLowerCase();
395
+ /** True for an `http:` or `https:` URL; false for anything else, unparsable included. */
396
+ const isWebUrl = (url) => {
397
+ try {
398
+ const { protocol } = new URL(url);
399
+ return protocol === "http:" || protocol === "https:";
400
+ }
401
+ catch {
402
+ return false;
403
+ }
404
+ };
405
+ /**
406
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
407
+ * this process connecting to them.
408
+ *
409
+ * An agent that confirms a cited link by requesting it itself sends that
410
+ * request from inside StackFactor's network, through its fixed egress
411
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
412
+ * links come from web pages anyone can publish, so that is a request to an
413
+ * address an outsider chose (server-side request forgery). Tavily fetches from
414
+ * its own infrastructure, so nothing here ever connects to such a host.
415
+ *
416
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
417
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
418
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
419
+ * like every other Tavily call, but the URLs are not added to the tracker's
420
+ * sources: a link worth checking was found there in the first place.
421
+ *
422
+ * Tavily accounts for every URL it is given, listing the ones it could not
423
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
424
+ * other URL of a batch Tavily answered is reachable (the results may name a
425
+ * URL by its address after a redirect, which is why they are not matched). A
426
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
427
+ * confirms none of its URLs: a link that could not be checked is not trusted.
428
+ *
429
+ * @param urls - The URLs to check; a URL given twice is checked once
430
+ * @param config - The integration config; supplies `tavilyAPIKey`
431
+ * @param usageTracker - Optional accumulator the credits are counted into
432
+ * @returns The given URLs, exactly as passed, that resolve
433
+ */
434
+ export const checkUrlsReachable = async (urls, config, usageTracker = null) => {
435
+ const reachable = new Set();
436
+ const candidates = [
437
+ ...new Set(urls.filter((url) => typeof url === "string" && isWebUrl(url.trim()))),
438
+ ];
439
+ if (candidates.length === 0)
440
+ return reachable;
441
+ const apiKey = config?.tavilyAPIKey;
442
+ if (!apiKey) {
443
+ logger.log(null, logger.levels.warn, `Tavily reachability: no tavilyAPIKey, so none of ${candidates.length} URL(s) is confirmed`);
444
+ return reachable;
445
+ }
446
+ const batches = [];
447
+ for (let start = 0; start < candidates.length; start += EXTRACT_MAX_URLS) {
448
+ batches.push(candidates.slice(start, start + EXTRACT_MAX_URLS));
449
+ }
450
+ await Promise.all(batches.map(async (batch) => {
451
+ let data;
452
+ try {
453
+ data = await callTavily("extract", {
454
+ urls: batch.map((url) => url.trim()),
455
+ extract_depth: "basic",
456
+ format: "text",
457
+ }, apiKey, REACHABILITY_TIMEOUT_SECONDS);
458
+ }
459
+ catch (err) {
460
+ logger.log(null, logger.levels.warn, `Tavily reachability: the check of ${batch.length} URL(s) failed (${err?.message || String(err)}), so none of them is confirmed`);
461
+ return;
462
+ }
463
+ recordTavilyUsage(usageTracker, config, data?.usage?.credits || 0, []);
464
+ const failed = new Set((data?.failed_results || [])
465
+ .map((entry) => typeof entry?.url === "string" ? urlMatchKey(entry.url) : "")
466
+ .filter(Boolean));
467
+ for (const url of batch) {
468
+ if (!failed.has(urlMatchKey(url)))
469
+ reachable.add(url);
470
+ }
471
+ }));
472
+ return reachable;
473
+ };
389
474
  export default {
390
475
  assertTavilyUsable,
391
476
  buildTavilyTools,
477
+ checkUrlsReachable,
392
478
  getTavilyOptions,
393
479
  recordTavilyUsage,
394
480
  };
package/package.json CHANGED
@@ -3,7 +3,7 @@
3
3
  "publishConfig": {
4
4
  "access": "public"
5
5
  },
6
- "version": "1.8.0",
6
+ "version": "1.10.0",
7
7
  "description": "",
8
8
  "main": "dist/cjs/index.js",
9
9
  "module": "dist/esm/index.js",
@@ -20,7 +20,7 @@
20
20
  "work:release": "bash scripts/release.sh",
21
21
  "release": "bash scripts/release.sh",
22
22
  "release:fanout": "bash scripts/release-fanout.sh",
23
- "ar-auth": "npm config set @stackfactor:registry https://us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/; TOKEN=$(gcloud auth print-access-token 2>/dev/null || curl -sfH 'Metadata-Flavor: Google' http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token | sed -n 's/.*\"access_token\":\"\\([^\"]*\\)\".*/\\1/p'); test -n \"$TOKEN\" || { echo 'ar-auth: no credential \u2014 run: gcloud auth login' >&2; exit 1; }; npm config set //us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/:_authToken \"$TOKEN\"; npm config set //us-central1-npm.pkg.dev/virtual-development-team/:_authToken \"$TOKEN\"; echo 'ar-auth: Artifact Registry scope + token configured'",
23
+ "ar-auth": "npm config set @stackfactor:registry https://us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/; TOKEN=$(gcloud auth print-access-token 2>/dev/null || curl -sfH 'Metadata-Flavor: Google' http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token | sed -n 's/.*\"access_token\":\"\\([^\"]*\\)\".*/\\1/p'); test -n \"$TOKEN\" || { echo 'ar-auth: no credential — run: gcloud auth login' >&2; exit 1; }; npm config set //us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/:_authToken \"$TOKEN\"; npm config set //us-central1-npm.pkg.dev/virtual-development-team/:_authToken \"$TOKEN\"; echo 'ar-auth: Artifact Registry scope + token configured'",
24
24
  "reinstall": "npm run ar-auth && npm ci",
25
25
  "test:coverage": "npm run build && node --experimental-test-coverage --test --test-reporter=tap test/*.test.js"
26
26
  },