@stackfactor/agent-utils 1.11.0 → 1.12.0

package/README.md

@@ -392,6 +392,7 @@ const result = await langChain.runPromptWithModel(
 | `excludeDomains`  | —                        | Exclude these domains from `web_search`                                                       |
 | `crawlLimit`      | `10`                     | Maximum pages one `web_crawl` or `web_map` call may return                                    |
 | `format`          | `"markdown"`             | `"markdown"` preserves table and heading structure at fewer tokens than the prose equivalent  |
+| `allowedUrls`     | —                        | The only URLs `web_extract` may read; any other gets an error line in the tool result and is never sent to Tavily |
 
 ### Context management
 
@@ -416,6 +417,7 @@ A model that cannot tell it received half a regulation will reason over the half
 - Works with every supported provider.
 - Credits come from each response's own `usage` block rather than an estimate. They are reported to the platform as `credits` of the model `tavily` (see [Usage](#usage)) and counted on `usageTracker.tokens.tavilyCredits`.
 - URLs are collected on `usageTracker.webSearchSources`, deduplicated across the whole run — Tavily's terms require citing the original sources when its content is shown to end users.
+- Every URL `web_extract` asked Tavily for is on `usageTracker.attemptedUrls`, each once, whether Tavily could read it or not; a URL `allowedUrls` refused was never asked for and is not there. `webSearchSources` holds what Tavily answered, under the address it answered with; `attemptedUrls` is what the run tried to read, so a caller can check a model's claim that it read a page.
 - Per-URL failures are surfaced in the tool result instead of being dropped, so a fetch failure is not mistaken for an absence of content.
 - Tool calls honour the caller's abort signal, so a cancelled run stops paying for in-flight crawls.
 
@@ -436,6 +438,28 @@ const kept = references.filter((r) => !r.url || live.has(r.url));
 
 ---
 
+## Reading references (`referenceContext`)
+
+An agent reading a skill's (or a microskill's, a question's) references gathers what they point at once, with `referenceContext.gatherReferenceContext({ references, queries, search, token, topK?, maxPassages?, maxPages? })`:
+
+- **pages**: WEB and PUBLICATION references with an http(s) url, each `{ id: "r<N>", title, url }`, opened by the model through Tavily `extract`. All of them by default; `maxPages` keeps the first N in reference order.
+- **passages**: the passages of the knowledge base documents STACKFACTOR references cite, searched under the run's token (client-api `knowledgeBaseAgent.search`). A passage names the references citing its chunk (`knowledgeBase.chunkId`); when none does, every reference citing its document. The best-ranked 40 are kept (`maxPassages`).
+
+**A cited knowledge base that cannot be read stops the run.** With no token, or when the person may not open a cited knowledge base (401, 403, 404) or its search fails, it throws `KnowledgeAccessError` (exported from the package and from `referenceContext`). A search that fails for any reason other than access is tried once more, 200 ms later, before it counts as failed; a refusal is not asked again.
+
+| Field       | Value                                                                                        |
+| ----------- | -------------------------------------------------------------------------------------------- |
+| `code`      | `403` when every skipped knowledge base refused the person (or there was no token), else `502` |
+| `documents` | The titles of the STACKFACTOR references citing those knowledge bases, each once              |
+| `skipped`   | The knowledge bases that could not be searched, and why                                       |
+| `message`   | `The knowledge base documents this content cites could not be read with your access: "A", "B".` (502: `…could not be read: the knowledge base search failed for "A".`) |
+
+The agent answers with `code` and `message`, so the person sees which documents could not be read. An agent that does not catch it answers the same: `serve` and `runOnce` turn an uncaught `KnowledgeAccessError` (known by its `name`, so a second copy of this package counts too) into an error with its own `code` and `message`, not a 500. A search that works and finds nothing is not an error: the run gets no passages and the `none` guidance. Grounding from skill coverage (`stackfactorKnowledge.retrievePassages`) still skips a knowledge base it cannot read.
+
+`referenceContext.toolsFor(context)` gives the run its tools: `{ tavily: { tools: ["extract"], allowedUrls } }` when there are pages (their urls), else `{}`. Spread it after `tavily: undefined`, so `web_extract` reads only the referenced pages.
+
+---
+
 ## Supported Models
 
 ### Text / Chat

package/dist/cjs/index.d.ts

@@ -7,7 +7,7 @@ import { callAgent, checkAgent } from "./client.js";
 import * as runtimeContext from "./runtimeContext.js";
 import { recordUsage, USAGE_UNITS } from "./usage.js";
 import stackfactorKnowledge from "./stackfactorKnowledge.js";
-import referenceContext from "./referenceContext.js";
+import referenceContext, { KnowledgeAccessError } from "./referenceContext.js";
 import { checkUrlsReachable } from "./tavily.js";
 export { constants };
 export { errorHandling, AppError };
@@ -24,7 +24,7 @@ export type { AgentExecuteRequest, CallAgentOptions, CheckAgentOptions, AgentHea
 export { runtimeContext };
 export { recordUsage, USAGE_UNITS };
 export type { UsageEntry, UsageUnit } from "./usage.js";
-export { stackfactorKnowledge, referenceContext };
+export { stackfactorKnowledge, referenceContext, KnowledgeAccessError };
 export type { GroundingPassage, KnowledgeChunk, KnowledgeSearch, KnowledgeSource, SkippedSource, StackFactorReference, } from "./stackfactorKnowledge.js";
 export type { ReferenceContext, ReferencePage, ReferencePassage, SourceGuidance, StoredReference, } from "./referenceContext.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/cjs/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAC7D,OAAO,gBAAgB,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,CAAC;AAElD,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EACV,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,eAAe,GAChB,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAC7D,OAAO,gBAAgB,EAAE,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,CAAC;AAExE,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EACV,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,eAAe,GAChB,MAAM,uBAAuB,CAAC"}

package/dist/cjs/index.js

@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.referenceContext = exports.stackfactorKnowledge = exports.USAGE_UNITS = exports.recordUsage = exports.runtimeContext = exports.checkAgent = exports.callAgent = exports.EXECUTION_ENV = exports.isRunOnceExecution = exports.runOnce = exports.serve = exports.logger = exports.checkUrlsReachable = exports.langChain = exports.AppError = exports.errorHandling = exports.constants = void 0;
+exports.KnowledgeAccessError = exports.referenceContext = exports.stackfactorKnowledge = exports.USAGE_UNITS = exports.recordUsage = exports.runtimeContext = exports.checkAgent = exports.callAgent = exports.EXECUTION_ENV = exports.isRunOnceExecution = exports.runOnce = exports.serve = exports.logger = exports.checkUrlsReachable = exports.langChain = exports.AppError = exports.errorHandling = exports.constants = void 0;
 const const_js_1 = __importDefault(require("./const.js"));
 exports.constants = const_js_1.default;
 const errorHandling_js_1 = __importStar(require("./errorHandling.js"));
@@ -61,7 +61,8 @@ Object.defineProperty(exports, "recordUsage", { enumerable: true, get: function
 Object.defineProperty(exports, "USAGE_UNITS", { enumerable: true, get: function () { return usage_js_1.USAGE_UNITS; } });
 const stackfactorKnowledge_js_1 = __importDefault(require("./stackfactorKnowledge.js"));
 exports.stackfactorKnowledge = stackfactorKnowledge_js_1.default;
-const referenceContext_js_1 = __importDefault(require("./referenceContext.js"));
+const referenceContext_js_1 = __importStar(require("./referenceContext.js"));
 exports.referenceContext = referenceContext_js_1.default;
+Object.defineProperty(exports, "KnowledgeAccessError", { enumerable: true, get: function () { return referenceContext_js_1.KnowledgeAccessError; } });
 const tavily_js_1 = require("./tavily.js");
 Object.defineProperty(exports, "checkUrlsReachable", { enumerable: true, get: function () { return tavily_js_1.checkUrlsReachable; } });

package/dist/cjs/langChain.d.ts

@@ -31,6 +31,14 @@ export type UsageTracker = {
         url: string;
         title?: string;
     }[];
+    /**
+     * Every URL `web_extract` asked Tavily for, each once, in the order first
+     * asked, whether Tavily could read it or not (a failed URL, a failed call).
+     * A URL refused by `allowedUrls` was never asked for and is not here. What
+     * Tavily answered is `webSearchSources`, under the address it answered with
+     * (after a redirect); this is what the run actually tried to read.
+     */
+    attemptedUrls?: string[];
 };
 /**
  * Instantiates and returns the appropriate LangChain chat model based on the model

package/dist/cjs/langChain.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"langChain.d.ts","sourceRoot":"","sources":["../../src/langChain.ts"],"names":[],"mappings":"AAoGA,KAAK,UAAU,GAAG;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IAKtB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACvC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACtD,CAAC;AA0iBF;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,eAAO,MAAM,WAAW,GACtB,WAAW,MAAM,EACjB,QAAQ,GAAG,EACX,SAAQ,GAAU,KACjB,GAwHF,CAAC;;8CAkb+C,OAAO,UAAU,GAAG,KAAG,OAAO;oCAWxC,GAAG,KAAG,OAAO;0CAgBtC,MAAM,cACN,GAAG,SACR,GAAG,KACT,MAAM;2CA0CqC,GAAG,UAAU,GAAG,KAAG,MAAM;sCApxC9B,MAAM,KAAG,OAAO;0CA+FZ,GAAG,KAAG,IAAI;wBA8sB/C,MAAM,aACD,MAAM,gBACH,MAAM,SACb,GAAG,EAAE,kBACI,GAAG,UACX,GAAG,iBACG,YAAY,GAAG,IAAI,KAChC,GAAG;sBAsCG,GAAG,UACF,MAAM,UACN,GAAG,eACC,QAAQ,GAAG,IAAI,iBACb,YAAY,GAAG,IAAI,aACvB,MAAM,KACf,OAAO,CAAC,GAAG,CAAC;oCAghBF,MAAM,UACT,GAAG,UACH,GAAG,oBACO,GAAG,eACT,MAAM,eACN,MAAM,wBACG,OAAO,WACpB,GAAG,cACA,MAAM,UACV,GAAG,EAAE,iBACE,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC;sDA2sBF,MAAM,UACT,GAAG,UACH,MAAM,YACL,GAAG,iBACE,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC;sDAmTF,MAAM,UACT,GAAG,UACH,MAAM,YACL,GAAG,iBACE,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC;0CAx6C8B,GAAG,KAAG,MAAM;uCAxyB9C,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,YACP,GAAG,WACJ,GAAG,YACF,QAAQ,GAAG,QAAQ,KAC5B,IAAI;kCA5CI,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,SACV,UAAU,GAAG,IAAI,YACd,GAAG,KACZ,IAAI;+CA0FI,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,kBACD,MAAM,YACZ,GAAG,KACZ,IAAI;uCAoBI,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,WACR,MAAM,YACL,GAAG,KACZ,IAAI;mCAyuBU,MAAM,mBACJ,MAAM,EAAE,KACxB,MAAM;;AA28CT,wBAkBE"}
+{"version":3,"file":"langChain.d.ts","sourceRoot":"","sources":["../../src/langChain.ts"],"names":[],"mappings":"AAoGA,KAAK,UAAU,GAAG;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IAKtB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACvC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACrD;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B,CAAC;AA0iBF;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,eAAO,MAAM,WAAW,GACtB,WAAW,MAAM,EACjB,QAAQ,GAAG,EACX,SAAQ,GAAU,KACjB,GAwHF,CAAC;;8CAkb+C,OAAO,UAAU,GAAG,KAAG,OAAO;oCAWxC,GAAG,KAAG,OAAO;0CAgBtC,MAAM,cACN,GAAG,SACR,GAAG,KACT,MAAM;2CA0CqC,GAAG,UAAU,GAAG,KAAG,MAAM;sCA5xC9B,MAAM,KAAG,OAAO;0CAuGZ,GAAG,KAAG,IAAI;wBA8sB/C,MAAM,aACD,MAAM,gBACH,MAAM,SACb,GAAG,EAAE,kBACI,GAAG,UACX,GAAG,iBACG,YAAY,GAAG,IAAI,KAChC,GAAG;sBAsCG,GAAG,UACF,MAAM,UACN,GAAG,eACC,QAAQ,GAAG,IAAI,iBACb,YAAY,GAAG,IAAI,aACvB,MAAM,KACf,OAAO,CAAC,GAAG,CAAC;oCAihBF,MAAM,UACT,GAAG,UACH,GAAG,oBACO,GAAG,eACT,MAAM,eACN,MAAM,wBACG,OAAO,WACpB,GAAG,cACA,MAAM,UACV,GAAG,EAAE,iBACE,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC;sDA2sBF,MAAM,UACT,GAAG,UACH,MAAM,YACL,GAAG,iBACE,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC;sDAmTF,MAAM,UACT,GAAG,UACH,MAAM,YACL,GAAG,iBACE,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC;0CAz6C8B,GAAG,KAAG,MAAM;uCAxyB9C,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,YACP,GAAG,WACJ,GAAG,YACF,QAAQ,GAAG,QAAQ,KAC5B,IAAI;kCA5CI,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,SACV,UAAU,GAAG,IAAI,YACd,GAAG,KACZ,IAAI;+CA0FI,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,kBACD,MAAM,YACZ,GAAG,KACZ,IAAI;uCAoBI,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,WACR,MAAM,YACL,GAAG,KACZ,IAAI;mCAyuBU,MAAM,mBACJ,MAAM,EAAE,KACxB,MAAM;;AA48CT,wBAkBE"}

package/dist/cjs/langChain.js

@@ -1210,8 +1210,9 @@ const parseAgenticJsonResponse = (rawContent, schema) => {
  *
  * Setting `config.tavily` adds the Tavily web tools (`web_search`, `web_extract`, and
  * optionally `web_map` / `web_crawl`). They execute in this process, so they require
- * `config.agentic` to be `true`; the credits spent and the URLs returned are recorded
- * on `usageTracker`.
+ * `config.agentic` to be `true`; the credits spent, the URLs returned and the URLs
+ * `web_extract` asked for are recorded on `usageTracker`. `tavily.allowedUrls` keeps
+ * `web_extract` to those URLs.
  * @param modelName - The model identifier, e.g. `"gpt-4o"`, `"claude-3-5-sonnet"`,
  *   `"gemini-1.5-pro"`
  * @param config - Configuration object with API keys, `temperature`, optional `agentic`

package/dist/cjs/referenceContext.d.ts

@@ -10,7 +10,10 @@ export interface ReferencePage {
 }
 /** A passage of a document that references cite. */
 export interface ReferencePassage {
-    /** The short ids of every STACKFACTOR reference that cites this passage's document. */
+    /**
+     * The short ids of the STACKFACTOR references that cite this passage's chunk
+     * (`knowledgeBase.chunkId`); when none does, of every one citing its document.
+     */
     references: string[];
     document: string | null;
     section: string | null;
@@ -23,21 +26,56 @@ export interface ReferencePassage {
 export interface ReferenceContext {
     pages: ReferencePage[];
     passages: ReferencePassage[];
+    /**
+     * Always empty from `gatherReferenceContext`: a cited knowledge base that
+     * cannot be read throws KnowledgeAccessError, which carries the skips.
+     */
     skipped: SkippedSource[];
-    /** True when the references cite at least one knowledge base document, whether or not it could be read. */
+    /** True when the references cite at least one knowledge base document (all of them readable, or it would have thrown). */
     citesKnowledge: boolean;
 }
+/**
+ * The knowledge base documents a run's references cite could not be read: the
+ * run has no token, the person may not open a cited knowledge base (401, 403,
+ * 404), or its search failed. The agent answers with it — `code` and `message`
+ * are what the person sees — and generates nothing.
+ */
+export declare class KnowledgeAccessError extends Error {
+    /** 403 when every skipped knowledge base refused the person (or there was no token), else 502. */
+    readonly code: 403 | 502;
+    /** The knowledge bases that could not be searched, and why; empty when there was no token. */
+    readonly skipped: SkippedSource[];
+    /** The titles of the STACKFACTOR references citing those knowledge bases, each once. */
+    readonly documents: string[];
+    /**
+     * @param details `skipped` (from the search) and `documents` (titles); the
+     *   code and the message follow from them
+     */
+    constructor(details?: {
+        skipped?: SkippedSource[];
+        documents?: unknown[];
+    });
+}
 /**
  * What a list of references points at: its pages, and the passages of the
  * documents its STACKFACTOR references cite. One search per knowledge base per
- * query (per batch of 500 documents), however many references there are. With
- * no token no search is made and the pages are still listed. A knowledge base
- * the person cannot open is in `skipped`; the other passages stand.
+ * query (per batch of 500 documents), however many references there are.
+ *
+ * References that cite a knowledge base the run cannot read stop it: with no
+ * token, or when any cited knowledge base is skipped (refused or failed), it
+ * throws a KnowledgeAccessError naming the documents. A search that works and
+ * finds nothing is not an error. With no knowledge cited, no token is needed.
+ *
+ * A passage is attributed to the references citing its chunk, else to every
+ * reference citing its document. The best-ranked `maxPassages` are kept, and
+ * the first `maxPages` pages in reference order.
  * @param options `references` (a skill's, a microskill's…), `queries` to
  *   search the cited documents with, `search` (client-api
  *   `knowledgeBaseAgent.search`), `token` (the run's AGENT token), `topK` per
- *   search (default 5)
- * @returns `{ pages, passages, skipped, citesKnowledge }`
+ *   search (default 5), `maxPassages` (default 40), `maxPages` (default: all)
+ * @returns `{ pages, passages, skipped, citesKnowledge }`; `skipped` is always
+ *   empty, since a skip throws
+ * @throws KnowledgeAccessError When cited knowledge bases cannot be read
  */
 export declare const gatherReferenceContext: (options: {
     references: unknown;
@@ -45,6 +83,8 @@ export declare const gatherReferenceContext: (options: {
     search: KnowledgeSearch;
     token: string | null | undefined;
     topK?: number;
+    maxPassages?: number;
+    maxPages?: number;
 }) => Promise<ReferenceContext>;
 /**
  * The passages as a prompt shows them: who cites each, where it came from, and
@@ -92,23 +132,26 @@ export interface StoredReference {
 }
 /**
  * The tools a run is given. Never `search`, `map` or `crawl`: a run reads what
- * its references point at and discovers nothing. `extract` only, and only when
- * there are pages to open.
+ * its references point at and discovers nothing. `extract` only, only when
+ * there are pages to open, and kept to those pages: any other url the model
+ * asks for is refused (`allowedUrls`).
  * @param context From `gatherReferenceContext`
- * @returns `{ tavily: { tools: ["extract"] } }` with pages, else `{}`. With no
- *   pages it returns `{}`, so a caller spreading it over a config that already
- *   has a `tavily` key keeps that key (and `tavily: true` means search plus
- *   extract): callers must not carry a base `tavily` key.
+ * @returns `{ tavily: { tools: ["extract"], allowedUrls } }` with pages, else
+ *   `{}`. With no pages it returns `{}`, so a caller spreading it over a config
+ *   that already has a `tavily` key keeps that key (and `tavily: true` means
+ *   search plus extract): callers must not carry a base `tavily` key.
  */
 export declare const toolsFor: (context: ReferenceContext) => {
     tavily?: {
         tools: string[];
+        allowedUrls: string[];
     };
 };
 /**
  * The stage prompt followed by the guidance for each kind of source the run
- * has: pages, then passages; when it has neither, the `none` guidance. An empty
- * guidance adds nothing.
+ * has: pages, then passages; when it has neither (knowledge cited and readable
+ * but no passage found included), the `none` guidance. An empty guidance adds
+ * nothing.
  * @param stagePrompt The stage's own prompt
  * @param context From `gatherReferenceContext`
  * @param guidance From `sourceGuidanceFrom`
@@ -159,6 +202,7 @@ export declare const citedReferences: (ids: unknown, supplied: unknown, origin?:
  */
 export declare const sourceGuidanceFrom: (config: Record<string, unknown>) => SourceGuidance;
 declare const _default: {
+    KnowledgeAccessError: typeof KnowledgeAccessError;
     REFERENCE_PASSAGES_KEY: string;
     citedReferences: (ids: unknown, supplied: unknown, origin?: string) => StoredReference[];
     composeSourceInstructions: (stagePrompt: string, context: ReferenceContext, guidance: SourceGuidance) => string;
@@ -168,6 +212,8 @@ declare const _default: {
         search: KnowledgeSearch;
         token: string | null | undefined;
         topK?: number;
+        maxPassages?: number;
+        maxPages?: number;
     }) => Promise<ReferenceContext>;
     referenceForModel: <T extends {
         kind?: string;
@@ -184,6 +230,7 @@ declare const _default: {
     toolsFor: (context: ReferenceContext) => {
         tavily?: {
             tools: string[];
+            allowedUrls: string[];
         };
     };
 };

package/dist/cjs/referenceContext.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"referenceContext.d.ts","sourceRoot":"","sources":["../../src/referenceContext.ts"],"names":[],"mappings":"AAiBA,OAAO,EACL,eAAe,EACf,aAAa,EAId,MAAM,2BAA2B,CAAC;AAKnC,iEAAiE;AACjE,eAAO,MAAM,sBAAsB,4BAA4B,CAAC;AAEhE,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,sCAAsC;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED,oDAAoD;AACpD,MAAM,WAAW,gBAAgB;IAC/B,uFAAuF;IACvF,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,2CAA2C;AAC3C,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,2GAA2G;IAC3G,cAAc,EAAE,OAAO,CAAC;CACzB;AAoDD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,sBAAsB,GAAU,SAAS;IACpD,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,KAAG,OAAO,CAAC,gBAAgB,CA+B3B,CAAC;AAYF;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,GACrC,SAAS,gBAAgB,KACxB;IAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAMrF,CAAC;AAEN,qFAAqF;AACrF,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,iFAAiF;AACjF,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7F,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACvE,aAAa,CAAC,EAAE;QAAE,eAAe,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;CACjH;AAED;;;;;;;;;GASG;AACH,eAAO,MAAM,QAAQ,GAAI,SAAS,gBAAgB,KAAG;IAAE,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,CAAA;CACf,CAAC;AAErE;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,GACpC,aAAa,MAAM,EACnB,SAAS,gBAAgB,EACzB,UAAU,cAAc,KACvB,MAMF,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,iBAAiB,GAAI,CAAC,SAAS;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,WAAW,CAAC,KAAG,CAY7E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,kBAAkB,GAAI,YAAY,OAAO,KAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAQrF,CAAC;AAmBF;;;;;;;GAOG;AACH,eAAO,MAAM,iBAAiB,GAAI,WAAW,GAAG,EAAE,SAAQ,MAAuB,KAAG,eAoBnF,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,GAAI,KAAK,OAAO,EAAE,UAAU,OAAO,EAAE,SAAS,MAAM,KAAG,eAAe,EAYjG,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAG,cAInE,CAAC;;;2BAvBkC,OAAO,YAAY,OAAO,WAAW,MAAM,KAAG,eAAe,EAAE;6CAzGrF,MAAM,WACV,gBAAgB,YACf,cAAc,KACvB,MAAM;sCA/G6C;QACpD,UAAU,EAAE,OAAO,CAAC;QACpB,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,MAAM,EAAE,eAAe,CAAC;QACxB,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,KAAG,OAAO,CAAC,gBAAgB,CAAC;wBAyHK,CAAC,SAAS;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,aAAa,CAAC,KAAG,CAAC;0CAvEpE,gBAAgB,KACxB;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;qCA2F7C,OAAO,KAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;iCAoF5C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAG,cAAc;mCAjDtC,GAAG,WAAU,MAAM,KAAoB,eAAe;wBAxFjE,gBAAgB,KAAG;QAAE,MAAM,CAAC,EAAE;YAAE,KAAK,EAAE,MAAM,EAAE,CAAA;SAAE,CAAA;KAAE;;AA+IrF,wBAWE"}
+{"version":3,"file":"referenceContext.d.ts","sourceRoot":"","sources":["../../src/referenceContext.ts"],"names":[],"mappings":"AAuBA,OAAO,EACL,eAAe,EACf,aAAa,EAId,MAAM,2BAA2B,CAAC;AAQnC,iEAAiE;AACjE,eAAO,MAAM,sBAAsB,4BAA4B,CAAC;AAEhE,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,sCAAsC;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED,oDAAoD;AACpD,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,2CAA2C;AAC3C,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B;;;OAGG;IACH,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,0HAA0H;IAC1H,cAAc,EAAE,OAAO,CAAC;CACzB;AAID;;;;;GAKG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,kGAAkG;IAClG,QAAQ,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,CAAC;IACzB,8FAA8F;IAC9F,QAAQ,CAAC,OAAO,EAAE,aAAa,EAAE,CAAC;IAClC,wFAAwF;IACxF,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;IAE7B;;;OAGG;gBACS,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAA;KAAO;CAgB/E;AAsFD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,sBAAsB,GAAU,SAAS;IACpD,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,KAAG,OAAO,CAAC,gBAAgB,CA2C3B,CAAC;AAwBF;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,GACrC,SAAS,gBAAgB,KACxB;IAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAMrF,CAAC;AAEN,qFAAqF;AACrF,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,iFAAiF;AACjF,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7F,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACvE,aAAa,CAAC,EAAE;QAAE,eAAe,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;CACjH;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,QAAQ,GAAI,SAAS,gBAAgB,KAAG;IAAE,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,CAAA;CAGlG,CAAC;AAET;;;;;;;;;GASG;AACH,eAAO,MAAM,yBAAyB,GACpC,aAAa,MAAM,EACnB,SAAS,gBAAgB,EACzB,UAAU,cAAc,KACvB,MAMF,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,iBAAiB,GAAI,CAAC,SAAS;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,WAAW,CAAC,KAAG,CAY7E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,kBAAkB,GAAI,YAAY,OAAO,KAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAQrF,CAAC;AAmBF;;;;;;;GAOG;AACH,eAAO,MAAM,iBAAiB,GAAI,WAAW,GAAG,EAAE,SAAQ,MAAuB,KAAG,eAoBnF,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,GAAI,KAAK,OAAO,EAAE,UAAU,OAAO,EAAE,SAAS,MAAM,KAAG,eAAe,EAYjG,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAG,cAInE,CAAC;;;;2BAvBkC,OAAO,YAAY,OAAO,WAAW,MAAM,KAAG,eAAe,EAAE;6CAzGrF,MAAM,WACV,gBAAgB,YACf,cAAc,KACvB,MAAM;sCA7I6C;QACpD,UAAU,EAAE,OAAO,CAAC;QACpB,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,MAAM,EAAE,eAAe,CAAC;QACxB,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,KAAG,OAAO,CAAC,gBAAgB,CAAC;wBAqJK,CAAC,SAAS;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,aAAa,CAAC,KAAG,CAAC;0CA3EpE,gBAAgB,KACxB;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;qCA+F7C,OAAO,KAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;iCAoF5C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAG,cAAc;mCAjDtC,GAAG,WAAU,MAAM,KAAoB,eAAe;wBA3FjE,gBAAgB,KAAG;QAAE,MAAM,CAAC,EAAE;YAAE,KAAK,EAAE,MAAM,EAAE,CAAC;YAAC,WAAW,EAAE,MAAM,EAAE,CAAA;SAAE,CAAA;KAAE;;AAkJ5G,wBAYE"}

package/dist/cjs/referenceContext.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sourceGuidanceFrom = exports.citedReferences = exports.toStoredReference = exports.referencesForModel = exports.referenceForModel = exports.composeSourceInstructions = exports.toolsFor = exports.referencePassagesForPrompt = exports.gatherReferenceContext = exports.REFERENCE_PASSAGES_KEY = void 0;
+exports.sourceGuidanceFrom = exports.citedReferences = exports.toStoredReference = exports.referencesForModel = exports.referenceForModel = exports.composeSourceInstructions = exports.toolsFor = exports.referencePassagesForPrompt = exports.gatherReferenceContext = exports.KnowledgeAccessError = exports.REFERENCE_PASSAGES_KEY = void 0;
 //
 // Reference context — gathering what a list of references points at
 //
@@ -17,6 +17,12 @@ exports.sourceGuidanceFrom = exports.citedReferences = exports.toStoredReference
 //     references cite, searched under the run's own token, so a run reads only
 //     what the person it acts for may open.
 //
+// A cited knowledge base that cannot be read stops the run (Candin,
+// 2026-10-13): with no token, a refusal or a failed search, the gathering
+// throws a KnowledgeAccessError naming the documents, and the agent answers
+// with it rather than writing as if the content cited nothing. A search that
+// works and finds nothing is not an error.
+//
 // A reference is cited by its short id `r<N>`, N being its index in the
 // supplied list, whatever its kind.
 //
@@ -24,9 +30,44 @@ const logger_js_1 = __importDefault(require("./logger.js"));
 const stackfactorKnowledge_js_1 = require("./stackfactorKnowledge.js");
 /** How many passages one search asks for when the caller does not say. */
 const DEFAULT_REFERENCE_TOP_K = 5;
+/** How many passages a gathering keeps when the caller does not say: the best-ranked. */
+const DEFAULT_MAX_PASSAGES = 40;
 /** The prompt key the passages are handed to the model under. */
 exports.REFERENCE_PASSAGES_KEY = "Knowledge Base Passages";
 const text = (value) => (typeof value === "string" ? value.trim() : "");
+/**
+ * The knowledge base documents a run's references cite could not be read: the
+ * run has no token, the person may not open a cited knowledge base (401, 403,
+ * 404), or its search failed. The agent answers with it — `code` and `message`
+ * are what the person sees — and generates nothing.
+ */
+class KnowledgeAccessError extends Error {
+    /** 403 when every skipped knowledge base refused the person (or there was no token), else 502. */
+    code;
+    /** The knowledge bases that could not be searched, and why; empty when there was no token. */
+    skipped;
+    /** The titles of the STACKFACTOR references citing those knowledge bases, each once. */
+    documents;
+    /**
+     * @param details `skipped` (from the search) and `documents` (titles); the
+     *   code and the message follow from them
+     */
+    constructor(details = {}) {
+        const skipped = Array.isArray(details.skipped) ? details.skipped : [];
+        const documents = [...new Set((Array.isArray(details.documents) ? details.documents : []).map(text).filter(Boolean))];
+        const code = skipped.every((source) => source?.noAccess) ? 403 : 502;
+        const titles = documents.map((title) => `"${title}"`).join(", ");
+        super(code === 403
+            ? `The knowledge base documents this content cites could not be read with your access${titles ? `: ${titles}` : ""}.`
+            : `The knowledge base documents this content cites could not be read: the knowledge base search failed${titles ? ` for ${titles}` : ""}.`);
+        this.name = "KnowledgeAccessError";
+        this.code = code;
+        this.skipped = skipped;
+        this.documents = documents;
+        Object.setPrototypeOf(this, KnowledgeAccessError.prototype);
+    }
+}
+exports.KnowledgeAccessError = KnowledgeAccessError;
 const id = (value) => typeof value === "string" ? value.trim() : value != null && typeof value === "object" ? String(value) : "";
 const shortId = (index) => `r${index}`;
 /** The document a STACKFACTOR reference cites, or "" when it cites none. */
@@ -35,6 +76,14 @@ const documentOf = (reference) => {
     return (id(reference.knowledgeBase?.documentId) ||
         (target.elementType === stackfactorKnowledge_js_1.STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE ? id(target.elementId) : ""));
 };
+/** The knowledge base a STACKFACTOR reference's document is in, or "" when it cites no document. */
+const knowledgeBaseOf = (reference) => {
+    const target = reference.target ?? {};
+    return (id(reference.knowledgeBase?.knowledgeBaseId) ||
+        (target.elementType === stackfactorKnowledge_js_1.STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE ? id(target.parentId) : ""));
+};
+/** A cap the caller passed (a whole number, 0 or more), else `fallback`. */
+const capOf = (value, fallback) => typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : fallback;
 /** The kinds the platform retired; it reads them as PUBLICATION. */
 const RETIRED_KINDS = new Set(["DOCUMENT", "KNOWLEDGE_BASE", "NOTE"]);
 /** A reference's kind as the platform reads it: missing or retired is PUBLICATION. */
@@ -60,39 +109,71 @@ const pagesOf = (references) => {
     });
     return pages;
 };
-/** The short ids of the STACKFACTOR references citing each document. */
-const citersByDocument = (references) => {
-    const citers = new Map();
+/**
+ * The short ids of the STACKFACTOR references citing each document, and of
+ * those citing each chunk (`knowledgeBase.chunkId`) of a document.
+ */
+const citersOf = (references) => {
+    const byDocument = new Map();
+    const byChunk = new Map();
     references.forEach((reference, index) => {
         if (reference?.kind !== stackfactorKnowledge_js_1.STACKFACTOR_REFERENCE.KIND)
             return;
         const documentId = documentOf(reference);
-        if (documentId)
-            citers.set(documentId, [...(citers.get(documentId) ?? []), shortId(index)]);
+        if (!documentId)
+            return;
+        byDocument.set(documentId, [...(byDocument.get(documentId) ?? []), shortId(index)]);
+        const chunkId = id(reference.knowledgeBase?.chunkId);
+        if (chunkId)
+            byChunk.set(chunkId, [...(byChunk.get(chunkId) ?? []), shortId(index)]);
     });
-    return citers;
+    return { byChunk, byDocument };
 };
+/**
+ * The error for cited knowledge bases that could not be read: it names the
+ * documents of the STACKFACTOR references citing them.
+ */
+const accessError = (references, unread, skipped) => new KnowledgeAccessError({
+    documents: references
+        .filter((reference) => reference?.kind === stackfactorKnowledge_js_1.STACKFACTOR_REFERENCE.KIND && documentOf(reference) && unread.has(knowledgeBaseOf(reference)))
+        .map((reference) => reference.title),
+    skipped,
+});
 /**
  * What a list of references points at: its pages, and the passages of the
  * documents its STACKFACTOR references cite. One search per knowledge base per
- * query (per batch of 500 documents), however many references there are. With
- * no token no search is made and the pages are still listed. A knowledge base
- * the person cannot open is in `skipped`; the other passages stand.
+ * query (per batch of 500 documents), however many references there are.
+ *
+ * References that cite a knowledge base the run cannot read stop it: with no
+ * token, or when any cited knowledge base is skipped (refused or failed), it
+ * throws a KnowledgeAccessError naming the documents. A search that works and
+ * finds nothing is not an error. With no knowledge cited, no token is needed.
+ *
+ * A passage is attributed to the references citing its chunk, else to every
+ * reference citing its document. The best-ranked `maxPassages` are kept, and
+ * the first `maxPages` pages in reference order.
  * @param options `references` (a skill's, a microskill's…), `queries` to
  *   search the cited documents with, `search` (client-api
  *   `knowledgeBaseAgent.search`), `token` (the run's AGENT token), `topK` per
- *   search (default 5)
- * @returns `{ pages, passages, skipped, citesKnowledge }`
+ *   search (default 5), `maxPassages` (default 40), `maxPages` (default: all)
+ * @returns `{ pages, passages, skipped, citesKnowledge }`; `skipped` is always
+ *   empty, since a skip throws
+ * @throws KnowledgeAccessError When cited knowledge bases cannot be read
  */
 const gatherReferenceContext = async (options) => {
     const references = Array.isArray(options.references) ? options.references : [];
     const sources = (0, stackfactorKnowledge_js_1.sourcesFromReferences)(references);
     const citesKnowledge = sources.length > 0;
-    const pages = pagesOf(references);
-    if (!options.token || !citesKnowledge) {
-        logContext(pages.length, 0, [], citesKnowledge && !options.token);
+    const pages = pagesOf(references).slice(0, capOf(options.maxPages, Infinity));
+    if (!citesKnowledge) {
+        logContext(pages.length, 0, []);
         return { citesKnowledge, pages, passages: [], skipped: [] };
     }
+    if (!options.token) {
+        const error = accessError(references, new Set(sources.map((source) => source.knowledgeBaseId)), []);
+        logContext(pages.length, 0, [], { error, noToken: true });
+        throw error;
+    }
     const found = await (0, stackfactorKnowledge_js_1.retrievePassages)({
         queries: options.queries,
         search: options.search,
@@ -100,31 +181,45 @@ const gatherReferenceContext = async (options) => {
         token: options.token,
         topK: options.topK ?? DEFAULT_REFERENCE_TOP_K,
     });
-    const citers = citersByDocument(references);
+    if (found.skipped.length > 0) {
+        const unread = new Set(found.skipped.map((source) => source.knowledgeBaseId));
+        const error = accessError(references, unread, found.skipped);
+        logContext(pages.length, 0, found.skipped, { error });
+        throw error;
+    }
+    const citers = citersOf(references);
     const passages = found.passages
-        .filter((passage) => citers.has(passage.documentId))
+        .filter((passage) => citers.byDocument.has(passage.documentId))
+        .slice(0, capOf(options.maxPassages, DEFAULT_MAX_PASSAGES))
         .map((passage) => ({
         chunkId: passage.chunkId,
         document: passage.documentName,
         documentId: passage.documentId,
         knowledgeBaseId: passage.knowledgeBaseId,
-        references: citers.get(passage.documentId),
+        references: citers.byChunk.get(passage.chunkId) ?? citers.byDocument.get(passage.documentId),
         section: passage.sectionTitle,
         text: passage.text,
     }));
-    logContext(pages.length, passages.length, found.skipped, false);
-    return { citesKnowledge, pages, passages, skipped: found.skipped };
+    logContext(pages.length, passages.length, []);
+    return { citesKnowledge, pages, passages, skipped: [] };
 };
 exports.gatherReferenceContext = gatherReferenceContext;
-/** The one summary line of a gathering, for the logs. */
-const logContext = (pages, passages, skipped, noToken) => {
+/**
+ * The one summary line of a gathering, for the logs: a warning when it was
+ * refused. Document titles stay out of the logs.
+ */
+const logContext = (pages, passages, skipped, refusal) => {
     const parts = [`${pages} page${pages === 1 ? "" : "s"}`, `${passages} passage${passages === 1 ? "" : "s"}`];
     if (skipped.length > 0) {
         parts.push(`skipped knowledge bases: ${skipped.map((s) => `${s.knowledgeBaseId} (noAccess: ${s.noAccess})`).join(", ")}`);
     }
-    if (noToken)
-        parts.push("no token, so no passages");
-    logger_js_1.default.log(null, logger_js_1.default.levels.info, `Reference context: ${parts.join("; ")}`);
+    if (refusal?.noToken)
+        parts.push("no token");
+    if (refusal) {
+        const count = refusal.error.documents.length;
+        parts.push(`refused (${refusal.error.code}): ${count} document${count === 1 ? "" : "s"} unreadable`);
+    }
+    logger_js_1.default.log(null, refusal ? logger_js_1.default.levels.warn : logger_js_1.default.levels.info, `Reference context: ${parts.join("; ")}`);
 };
 /**
  * The passages as a prompt shows them: who cites each, where it came from, and
@@ -141,20 +236,24 @@ const referencePassagesForPrompt = (context) => context.passages.map((passage) =
 exports.referencePassagesForPrompt = referencePassagesForPrompt;
 /**
  * The tools a run is given. Never `search`, `map` or `crawl`: a run reads what
- * its references point at and discovers nothing. `extract` only, and only when
- * there are pages to open.
+ * its references point at and discovers nothing. `extract` only, only when
+ * there are pages to open, and kept to those pages: any other url the model
+ * asks for is refused (`allowedUrls`).
  * @param context From `gatherReferenceContext`
- * @returns `{ tavily: { tools: ["extract"] } }` with pages, else `{}`. With no
- *   pages it returns `{}`, so a caller spreading it over a config that already
- *   has a `tavily` key keeps that key (and `tavily: true` means search plus
- *   extract): callers must not carry a base `tavily` key.
+ * @returns `{ tavily: { tools: ["extract"], allowedUrls } }` with pages, else
+ *   `{}`. With no pages it returns `{}`, so a caller spreading it over a config
+ *   that already has a `tavily` key keeps that key (and `tavily: true` means
+ *   search plus extract): callers must not carry a base `tavily` key.
  */
-const toolsFor = (context) => context.pages.length > 0 ? { tavily: { tools: ["extract"] } } : {};
+const toolsFor = (context) => context.pages.length > 0
+    ? { tavily: { tools: ["extract"], allowedUrls: context.pages.map((page) => page.url) } }
+    : {};
 exports.toolsFor = toolsFor;
 /**
  * The stage prompt followed by the guidance for each kind of source the run
- * has: pages, then passages; when it has neither, the `none` guidance. An empty
- * guidance adds nothing.
+ * has: pages, then passages; when it has neither (knowledge cited and readable
+ * but no passage found included), the `none` guidance. An empty guidance adds
+ * nothing.
  * @param stagePrompt The stage's own prompt
  * @param context From `gatherReferenceContext`
  * @param guidance From `sourceGuidanceFrom`
@@ -286,6 +385,7 @@ const sourceGuidanceFrom = (config) => ({
 });
 exports.sourceGuidanceFrom = sourceGuidanceFrom;
 exports.default = {
+    KnowledgeAccessError,
     REFERENCE_PASSAGES_KEY: exports.REFERENCE_PASSAGES_KEY,
     citedReferences: exports.citedReferences,
     composeSourceInstructions: exports.composeSourceInstructions,

package/dist/cjs/serve.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"serve.d.ts","sourceRoot":"","sources":["../../src/serve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,eAAe,CAAC;AAWtC,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAwPpB;;;;;GAKG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,CAC1B,OAAO,EAAE,GAAG,KACT,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,GAAG,CAAA;CAAE,CAAC,CAAC;AAEzC,MAAM,WAAW,YAAY;IAC3B,4EAA4E;IAC5E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,wFAAwF;IACxF,YAAY,CAAC,EAAE,aAAa,CAAC;IAC7B,uEAAuE;IACvE,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,sEAAsE;IACtE,SAAS,CAAC,EAAE,YAAY,CAAC;IACzB,iEAAiE;IACjE,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC9B,mFAAmF;IACnF,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,mFAAmF;IACnF,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACxD,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,EAAE,CACN,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,KACV,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACpC;AAKD;;;;;;GAMG;AACH,eAAO,MAAM,aAAa;;;;;CAKhB,CAAC;AAEX,mFAAmF;AACnF,eAAO,MAAM,kBAAkB,GAAI,MAAK,MAAM,CAAC,UAAwB,KAAG,OACJ,CAAC;AAKvE;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,qBAAqB,GAAI,OAAO,GAAG,EAAE,SAAS,GAAG,KAAG,OAEhB,CAAC;AAKlD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,cAAc,GAAI,OAAO,GAAG,EAAE,SAAS,GAAG,KAAG,OAGN,CAAC;AA4JrD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,OAAO,GAClB,MAAM,SAAS,EACf,UAAS,YAAiB,KACzB,OAAO,CAAC,IAAI,CA8Id,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,KAAK,GAChB,MAAM,SAAS,EACf,UAAS,YAAiB,KACzB,IAAI,CAAC,MAyOP,CAAC"}
+{"version":3,"file":"serve.d.ts","sourceRoot":"","sources":["../../src/serve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,eAAe,CAAC;AAWtC,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAwPpB;;;;;GAKG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,CAC1B,OAAO,EAAE,GAAG,KACT,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,GAAG,CAAA;CAAE,CAAC,CAAC;AAEzC,MAAM,WAAW,YAAY;IAC3B,4EAA4E;IAC5E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,wFAAwF;IACxF,YAAY,CAAC,EAAE,aAAa,CAAC;IAC7B,uEAAuE;IACvE,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,sEAAsE;IACtE,SAAS,CAAC,EAAE,YAAY,CAAC;IACzB,iEAAiE;IACjE,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC9B,mFAAmF;IACnF,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,mFAAmF;IACnF,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACxD,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,EAAE,CACN,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,KACV,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACpC;AAKD;;;;;;GAMG;AACH,eAAO,MAAM,aAAa;;;;;CAKhB,CAAC;AAEX,mFAAmF;AACnF,eAAO,MAAM,kBAAkB,GAAI,MAAK,MAAM,CAAC,UAAwB,KAAG,OACJ,CAAC;AAKvE;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,qBAAqB,GAAI,OAAO,GAAG,EAAE,SAAS,GAAG,KAAG,OAEhB,CAAC;AAKlD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,cAAc,GAAI,OAAO,GAAG,EAAE,SAAS,GAAG,KAAG,OAGN,CAAC;AAmKrD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,OAAO,GAClB,MAAM,SAAS,EACf,UAAS,YAAiB,KACzB,OAAO,CAAC,IAAI,CA8Id,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,KAAK,GAChB,MAAM,SAAS,EACf,UAAS,YAAiB,KACzB,IAAI,CAAC,MAyOP,CAAC"}

package/dist/cjs/serve.js

@@ -381,8 +381,15 @@ const safeParse = (value, fallback) => {
 /**
  * Agents that fail throw `new Error(JSON.stringify({ code, message }))`. Unwrap
  * that back into a structured gRPC error frame; fall back to a 500 otherwise.
+ * A KnowledgeAccessError an agent did not catch (referenceContext) answers
+ * with its own code (403, 502) and message, which names the documents that
+ * could not be read. It is known by its name, so one thrown by another copy of
+ * this package in the agent's dependencies is known too.
  */
 const toErrorFrame = (error) => {
+    if (error?.name === "KnowledgeAccessError" && typeof error.code === "number") {
+        return { code: error.code, message: String(error.message ?? "") };
+    }
     try {
         const parsed = JSON.parse(error?.message);
         if (parsed && typeof parsed.code === "number") {

package/dist/cjs/stackfactorKnowledge.d.ts

@@ -67,7 +67,10 @@ export interface SkippedSource {
     knowledgeBaseId: string;
     /** The HTTP status, when the platform answered one. */
     status: number | null;
-    /** True when the person may not read it (or it is gone): expected, not a fault. */
+    /**
+     * True when the platform refused the person (401, 403) or does not have the
+     * knowledge base (404); false when the search failed otherwise.
+     */
     noAccess: boolean;
     message: string;
 }
@@ -116,9 +119,11 @@ export declare const sourcesFromCoverage: (coverage: unknown) => KnowledgeSource
  * model to cite. A source with documents is searched within them (in batches of
  * SEARCH_DOCUMENT_IDS_MAX); one without, whole. A source the person may not
  * read (401, 403, 404) is skipped: none of its passages are used and no more of
- * its searches are made. A search that fails otherwise is reported and the
- * passages its other searches found stand. Either way it is in `skipped`, once
- * per knowledge base, and the passages of the others stand.
+ * its searches are made. A search that fails otherwise is tried once more; when
+ * that fails too it is reported and the passages its other searches found
+ * stand. A refusal is never tried again. Either way it is in `skipped`, once
+ * per knowledge base, and the passages of the others stand; whether a skip is
+ * an error is the caller's to say.
  * @param options `sources`, `queries`, `search` (client-api
  *   `knowledgeBaseAgent.search`), `token` (the run's AGENT token), `topK`
  *   per search (default DEFAULT_TOP_K)